2 Copyright (C) 2011, 2012
3 Free Software Foundation, Inc.
4 Contributed by Walter Lee (walt@tilera.com)
6 This file is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
11 This file is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 General Public License for more details.
16 Under Section 7 of GPL version 3, you are granted additional
17 permissions described in the GCC Runtime Library Exception, version
18 3.1, as published by the Free Software Foundation.
20 You should have received a copy of the GNU General Public License and
21 a copy of the GCC Runtime Library Exception along with this program;
22 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
29 /* This code should be inlined by the compiler, but for now support
30 it as out-of-line methods in libgcc. */
33 pre_atomic_barrier (int model)
35 switch ((enum memmodel) model)
37 case MEMMODEL_RELEASE:
38 case MEMMODEL_ACQ_REL:
39 case MEMMODEL_SEQ_CST:
40 __atomic_thread_fence (model);
49 post_atomic_barrier (int model)
51 switch ((enum memmodel) model)
53 case MEMMODEL_ACQUIRE:
54 case MEMMODEL_ACQ_REL:
55 case MEMMODEL_SEQ_CST:
56 __atomic_thread_fence (model);
64 #define __unused __attribute__((unused))
66 #define __atomic_fetch_and_do(type, size, opname) \
68 __atomic_fetch_##opname##_##size(type* p, type i, int model) \
70 pre_atomic_barrier(model); \
71 type rv = arch_atomic_##opname(p, i); \
72 post_atomic_barrier(model); \
76 __atomic_fetch_and_do (int, 4, add)
77 __atomic_fetch_and_do (int, 4, sub)
78 __atomic_fetch_and_do (int, 4, or)
79 __atomic_fetch_and_do (int, 4, and)
80 __atomic_fetch_and_do (int, 4, xor)
81 __atomic_fetch_and_do (int, 4, nand)
82 __atomic_fetch_and_do (long long, 8, add)
83 __atomic_fetch_and_do (long long, 8, sub)
84 __atomic_fetch_and_do (long long, 8, or)
85 __atomic_fetch_and_do (long long, 8, and)
86 __atomic_fetch_and_do (long long, 8, xor)
87 __atomic_fetch_and_do (long long, 8, nand)
89 #define __atomic_do_and_fetch(type, size, opname, op, op2) \
91 __atomic_##opname##_fetch_##size(type* p, type i, int model) \
93 pre_atomic_barrier(model); \
94 type rv = op2 (arch_atomic_##opname(p, i) op i); \
95 post_atomic_barrier(model); \
98 __atomic_do_and_fetch (int, 4, add, +, )
99 __atomic_do_and_fetch (int, 4, sub, -, )
100 __atomic_do_and_fetch (int, 4, or, |, )
101 __atomic_do_and_fetch (int, 4, and, &, )
102 __atomic_do_and_fetch (int, 4, xor, |, )
103 __atomic_do_and_fetch (int, 4, nand, &, ~)
104 __atomic_do_and_fetch (long long, 8, add, +, )
105 __atomic_do_and_fetch (long long, 8, sub, -, )
106 __atomic_do_and_fetch (long long, 8, or, |, )
107 __atomic_do_and_fetch (long long, 8, and, &, )
108 __atomic_do_and_fetch (long long, 8, xor, |, )
109 __atomic_do_and_fetch (long long, 8, nand, &, ~)
111 #define __atomic_exchange_methods(type, size) \
113 __atomic_compare_exchange_##size(volatile type* ptr, type* oldvalp, \
114 type newval, bool weak __unused, \
115 int models, int modelf __unused) \
117 type oldval = *oldvalp; \
118 pre_atomic_barrier(models); \
119 type retval = arch_atomic_val_compare_and_exchange(ptr, oldval, newval); \
120 post_atomic_barrier(models); \
121 bool success = (retval == oldval); \
127 __atomic_exchange_##size(volatile type* ptr, type val, int model) \
129 pre_atomic_barrier(model); \
130 type retval = arch_atomic_exchange(ptr, val); \
131 post_atomic_barrier(model); \
135 __atomic_exchange_methods (int, 4)
136 __atomic_exchange_methods (long long, 8)
138 /* Subword methods require the same approach for both TILEPro and
139 TILE-Gx. We load the background data for the word, insert the
140 desired subword piece, then compare-and-exchange it into place. */
141 #define u8 unsigned char
142 #define u16 unsigned short
144 #define __atomic_subword_cmpxchg(type, size) \
147 __atomic_compare_exchange_##size(volatile type* ptr, type* guess, \
148 type val, bool weak __unused, int models, \
149 int modelf __unused) \
151 pre_atomic_barrier(models); \
152 unsigned int *p = (unsigned int *)((unsigned long)ptr & ~3UL); \
153 const int shift = ((unsigned long)ptr & 3UL) * 8; \
154 const unsigned int valmask = (1 << (sizeof(type) * 8)) - 1; \
155 const unsigned int bgmask = ~(valmask << shift); \
156 unsigned int oldword = *p; \
157 type oldval = (oldword >> shift) & valmask; \
158 if (__builtin_expect((oldval == *guess), 1)) { \
159 unsigned int word = (oldword & bgmask) | ((val & valmask) << shift); \
160 oldword = arch_atomic_val_compare_and_exchange(p, oldword, word); \
161 oldval = (oldword >> shift) & valmask; \
163 post_atomic_barrier(models); \
164 bool success = (oldval == *guess); \
169 __atomic_subword_cmpxchg (u8, 1)
170 __atomic_subword_cmpxchg (u16, 2)
172 /* For the atomic-update subword methods, we use the same approach as
173 above, but we retry until we succeed if the compare-and-exchange
175 #define __atomic_subword(type, proto, top, expr, bottom) \
179 unsigned int *p = (unsigned int *)((unsigned long)ptr & ~3UL); \
180 const int shift = ((unsigned long)ptr & 3UL) * 8; \
181 const unsigned int valmask = (1 << (sizeof(type) * 8)) - 1; \
182 const unsigned int bgmask = ~(valmask << shift); \
183 unsigned int oldword, xword = *p; \
187 oldval = (oldword >> shift) & valmask; \
189 unsigned int word = (oldword & bgmask) | ((val & valmask) << shift); \
190 xword = arch_atomic_val_compare_and_exchange(p, oldword, word); \
191 } while (__builtin_expect(xword != oldword, 0)); \
195 #define __atomic_subword_fetch(type, funcname, expr, retval) \
196 __atomic_subword(type, \
197 type __atomic_ ## funcname(volatile type *ptr, type i, int model), \
198 pre_atomic_barrier(model);, \
200 post_atomic_barrier(model); return retval;)
202 __atomic_subword_fetch (u8, fetch_add_1, oldval + i, oldval)
203 __atomic_subword_fetch (u8, fetch_sub_1, oldval - i, oldval)
204 __atomic_subword_fetch (u8, fetch_or_1, oldval | i, oldval)
205 __atomic_subword_fetch (u8, fetch_and_1, oldval & i, oldval)
206 __atomic_subword_fetch (u8, fetch_xor_1, oldval ^ i, oldval)
207 __atomic_subword_fetch (u8, fetch_nand_1, ~(oldval & i), oldval)
209 __atomic_subword_fetch (u16, fetch_add_2, oldval + i, oldval)
210 __atomic_subword_fetch (u16, fetch_sub_2, oldval - i, oldval)
211 __atomic_subword_fetch (u16, fetch_or_2, oldval | i, oldval)
212 __atomic_subword_fetch (u16, fetch_and_2, oldval & i, oldval)
213 __atomic_subword_fetch (u16, fetch_xor_2, oldval ^ i, oldval)
214 __atomic_subword_fetch (u16, fetch_nand_2, ~(oldval & i), oldval)
216 __atomic_subword_fetch (u8, add_fetch_1, oldval + i, val)
217 __atomic_subword_fetch (u8, sub_fetch_1, oldval - i, val)
218 __atomic_subword_fetch (u8, or_fetch_1, oldval | i, val)
219 __atomic_subword_fetch (u8, and_fetch_1, oldval & i, val)
220 __atomic_subword_fetch (u8, xor_fetch_1, oldval ^ i, val)
221 __atomic_subword_fetch (u8, nand_fetch_1, ~(oldval & i), val)
223 __atomic_subword_fetch (u16, add_fetch_2, oldval + i, val)
224 __atomic_subword_fetch (u16, sub_fetch_2, oldval - i, val)
225 __atomic_subword_fetch (u16, or_fetch_2, oldval | i, val)
226 __atomic_subword_fetch (u16, and_fetch_2, oldval & i, val)
227 __atomic_subword_fetch (u16, xor_fetch_2, oldval ^ i, val)
228 __atomic_subword_fetch (u16, nand_fetch_2, ~(oldval & i), val)
230 #define __atomic_subword_lock(type, size) \
232 __atomic_subword(type, \
233 type __atomic_exchange_##size(volatile type* ptr, type nval, int model), \
234 pre_atomic_barrier(model);, \
236 post_atomic_barrier(model); return oldval;)
238 __atomic_subword_lock (u8, 1)
239 __atomic_subword_lock (u16, 2)