1 /* Linux-specific atomic operations for ARM EABI.
2 Copyright (C) 2008 Free Software Foundation, Inc.
3 Contributed by CodeSourcery.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 In addition to the permissions in the GNU General Public License, the
13 Free Software Foundation gives you unlimited permission to link the
14 compiled version of this file into combinations with other programs,
15 and to distribute those combinations without any restriction coming
16 from the use of this file. (The General Public License restrictions
17 do apply in other respects; for example, they cover modification of
18 the file, and distribution when not linked into a combine
21 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
22 WARRANTY; without even the implied warranty of MERCHANTABILITY or
23 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
26 You should have received a copy of the GNU General Public License
27 along with GCC; see the file COPYING. If not, write to the Free
28 Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA
31 /* Kernel helper for compare-and-exchange. */
32 typedef int (__kernel_cmpxchg_t) (int oldval, int newval, int *ptr);
33 #define __kernel_cmpxchg (*(__kernel_cmpxchg_t *) 0xffff0fc0)
35 /* Kernel helper for memory barrier. */
36 typedef void (__kernel_dmb_t) (void);
37 #define __kernel_dmb (*(__kernel_dmb_t *) 0xffff0fa0)
39 /* Note: we implement byte, short and int versions of atomic operations using
40 the above kernel helpers, but there is no support for "long long" (64-bit)
43 #define HIDDEN __attribute__ ((visibility ("hidden")))
46 #define INVERT_MASK_1 0
47 #define INVERT_MASK_2 0
49 #define INVERT_MASK_1 24
50 #define INVERT_MASK_2 16
54 #define MASK_2 0xffffu
56 #define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP) \
58 __sync_fetch_and_##OP##_4 (int *ptr, int val) \
64 failure = __kernel_cmpxchg (tmp, PFX_OP tmp INF_OP val, ptr); \
65 } while (failure != 0); \
70 FETCH_AND_OP_WORD (add, , +)
71 FETCH_AND_OP_WORD (sub, , -)
72 FETCH_AND_OP_WORD (or, , |)
73 FETCH_AND_OP_WORD (and, , &)
74 FETCH_AND_OP_WORD (xor, , ^)
75 FETCH_AND_OP_WORD (nand, ~, &)
77 #define NAME_oldval(OP, WIDTH) __sync_fetch_and_##OP##_##WIDTH
78 #define NAME_newval(OP, WIDTH) __sync_##OP##_and_fetch_##WIDTH
80 /* Implement both __sync_<op>_and_fetch and __sync_fetch_and_<op> for
81 subword-sized quantities. */
83 #define SUBWORD_SYNC_OP(OP, PFX_OP, INF_OP, TYPE, WIDTH, RETURN) \
85 NAME##_##RETURN (OP, WIDTH) (TYPE *ptr, TYPE val) \
87 int *wordptr = (int *) ((unsigned int) ptr & ~3); \
88 unsigned int mask, shift, oldval, newval; \
91 shift = (((unsigned int) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH; \
92 mask = MASK_##WIDTH << shift; \
96 newval = ((PFX_OP ((oldval & mask) >> shift) \
97 INF_OP (unsigned int) val) << shift) & mask; \
98 newval |= oldval & ~mask; \
99 failure = __kernel_cmpxchg (oldval, newval, wordptr); \
100 } while (failure != 0); \
102 return (RETURN & mask) >> shift; \
105 SUBWORD_SYNC_OP (add, , +, short, 2, oldval)
106 SUBWORD_SYNC_OP (sub, , -, short, 2, oldval)
107 SUBWORD_SYNC_OP (or, , |, short, 2, oldval)
108 SUBWORD_SYNC_OP (and, , &, short, 2, oldval)
109 SUBWORD_SYNC_OP (xor, , ^, short, 2, oldval)
110 SUBWORD_SYNC_OP (nand, ~, &, short, 2, oldval)
112 SUBWORD_SYNC_OP (add, , +, char, 1, oldval)
113 SUBWORD_SYNC_OP (sub, , -, char, 1, oldval)
114 SUBWORD_SYNC_OP (or, , |, char, 1, oldval)
115 SUBWORD_SYNC_OP (and, , &, char, 1, oldval)
116 SUBWORD_SYNC_OP (xor, , ^, char, 1, oldval)
117 SUBWORD_SYNC_OP (nand, ~, &, char, 1, oldval)
119 #define OP_AND_FETCH_WORD(OP, PFX_OP, INF_OP) \
121 __sync_##OP##_and_fetch_4 (int *ptr, int val) \
127 failure = __kernel_cmpxchg (tmp, PFX_OP tmp INF_OP val, ptr); \
128 } while (failure != 0); \
130 return PFX_OP tmp INF_OP val; \
133 OP_AND_FETCH_WORD (add, , +)
134 OP_AND_FETCH_WORD (sub, , -)
135 OP_AND_FETCH_WORD (or, , |)
136 OP_AND_FETCH_WORD (and, , &)
137 OP_AND_FETCH_WORD (xor, , ^)
138 OP_AND_FETCH_WORD (nand, ~, &)
140 SUBWORD_SYNC_OP (add, , +, short, 2, newval)
141 SUBWORD_SYNC_OP (sub, , -, short, 2, newval)
142 SUBWORD_SYNC_OP (or, , |, short, 2, newval)
143 SUBWORD_SYNC_OP (and, , &, short, 2, newval)
144 SUBWORD_SYNC_OP (xor, , ^, short, 2, newval)
145 SUBWORD_SYNC_OP (nand, ~, &, short, 2, newval)
147 SUBWORD_SYNC_OP (add, , +, char, 1, newval)
148 SUBWORD_SYNC_OP (sub, , -, char, 1, newval)
149 SUBWORD_SYNC_OP (or, , |, char, 1, newval)
150 SUBWORD_SYNC_OP (and, , &, char, 1, newval)
151 SUBWORD_SYNC_OP (xor, , ^, char, 1, newval)
152 SUBWORD_SYNC_OP (nand, ~, &, char, 1, newval)
155 __sync_val_compare_and_swap_4 (int *ptr, int oldval, int newval)
157 int actual_oldval, fail;
161 actual_oldval = *ptr;
163 if (oldval != actual_oldval)
164 return actual_oldval;
166 fail = __kernel_cmpxchg (actual_oldval, newval, ptr);
173 #define SUBWORD_VAL_CAS(TYPE, WIDTH) \
175 __sync_val_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval, \
178 int *wordptr = (int *)((unsigned int) ptr & ~3), fail; \
179 unsigned int mask, shift, actual_oldval, actual_newval; \
181 shift = (((unsigned int) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH; \
182 mask = MASK_##WIDTH << shift; \
186 actual_oldval = *wordptr; \
188 if (((actual_oldval & mask) >> shift) != (unsigned int) oldval) \
189 return (actual_oldval & mask) >> shift; \
191 actual_newval = (actual_oldval & ~mask) \
192 | (((unsigned int) newval << shift) & mask); \
194 fail = __kernel_cmpxchg (actual_oldval, actual_newval, \
202 SUBWORD_VAL_CAS (short, 2)
203 SUBWORD_VAL_CAS (char, 1)
205 typedef unsigned char bool;
208 __sync_bool_compare_and_swap_4 (int *ptr, int oldval, int newval)
210 int failure = __kernel_cmpxchg (oldval, newval, ptr);
211 return (failure == 0);
214 #define SUBWORD_BOOL_CAS(TYPE, WIDTH) \
216 __sync_bool_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval, \
220 = __sync_val_compare_and_swap_##WIDTH (ptr, oldval, newval); \
221 return (oldval == actual_oldval); \
224 SUBWORD_BOOL_CAS (short, 2)
225 SUBWORD_BOOL_CAS (char, 1)
228 __sync_synchronize (void)
234 __sync_lock_test_and_set_4 (int *ptr, int val)
240 failure = __kernel_cmpxchg (oldval, val, ptr);
241 } while (failure != 0);
246 #define SUBWORD_TEST_AND_SET(TYPE, WIDTH) \
248 __sync_lock_test_and_set_##WIDTH (TYPE *ptr, TYPE val) \
251 unsigned int oldval, newval, shift, mask; \
252 int *wordptr = (int *) ((unsigned int) ptr & ~3); \
254 shift = (((unsigned int) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH; \
255 mask = MASK_##WIDTH << shift; \
259 newval = (oldval & ~mask) \
260 | (((unsigned int) val << shift) & mask); \
261 failure = __kernel_cmpxchg (oldval, newval, wordptr); \
262 } while (failure != 0); \
264 return (oldval & mask) >> shift; \
267 SUBWORD_TEST_AND_SET (short, 2)
268 SUBWORD_TEST_AND_SET (char, 1)
270 #define SYNC_LOCK_RELEASE(TYPE, WIDTH) \
272 __sync_lock_release_##WIDTH (TYPE *ptr) \
278 SYNC_LOCK_RELEASE (int, 4)
279 SYNC_LOCK_RELEASE (short, 2)
280 SYNC_LOCK_RELEASE (char, 1)