gcc/config/arm/linux-atomic.c

   1 /* Linux-specific atomic operations for ARM EABI.
   2    Copyright (C) 2008 Free Software Foundation, Inc.
   3    Contributed by CodeSourcery.
   4
   5 This file is part of GCC.
   6
   7 GCC is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 In addition to the permissions in the GNU General Public License, the
  13 Free Software Foundation gives you unlimited permission to link the
  14 compiled version of this file into combinations with other programs,
  15 and to distribute those combinations without any restriction coming
  16 from the use of this file.  (The General Public License restrictions
  17 do apply in other respects; for example, they cover modification of
  18 the file, and distribution when not linked into a combine
  19 executable.)
  20
  21 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  22 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  23 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  24 for more details.
  25
  26 You should have received a copy of the GNU General Public License
  27 along with GCC; see the file COPYING.  If not, write to the Free
  28 Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA
  29 02110-1301, USA.  */
  30
  31 /* Kernel helper for compare-and-exchange.  */
  32 typedef int (__kernel_cmpxchg_t) (int oldval, int newval, int *ptr);
  33 #define __kernel_cmpxchg (*(__kernel_cmpxchg_t *) 0xffff0fc0)
  34
  35 /* Kernel helper for memory barrier.  */
  36 typedef void (__kernel_dmb_t) (void);
  37 #define __kernel_dmb (*(__kernel_dmb_t *) 0xffff0fa0)
  38
  39 /* Note: we implement byte, short and int versions of atomic operations using
  40    the above kernel helpers, but there is no support for "long long" (64-bit)
  41    operations as yet.  */
  42
  43 #define HIDDEN __attribute__ ((visibility ("hidden")))
  44
  45 #ifdef __ARMEL__
  46 #define INVERT_MASK_1 0
  47 #define INVERT_MASK_2 0
  48 #else
  49 #define INVERT_MASK_1 24
  50 #define INVERT_MASK_2 16
  51 #endif
  52
  53 #define MASK_1 0xffu
  54 #define MASK_2 0xffffu
  55
  56 #define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP)                           \
  57   int HIDDEN                                                            \
  58   __sync_fetch_and_##OP##_4 (int *ptr, int val)                         \
  59   {                                                                     \
  60     int failure, tmp;                                                   \
  61                                                                         \
  62     do {                                                                \
  63       tmp = *ptr;                                                       \
  64       failure = __kernel_cmpxchg (tmp, PFX_OP tmp INF_OP val, ptr);     \
  65     } while (failure != 0);                                             \
  66                                                                         \
  67     return tmp;                                                         \
  68   }
  69
  70 FETCH_AND_OP_WORD (add,   , +)
  71 FETCH_AND_OP_WORD (sub,   , -)
  72 FETCH_AND_OP_WORD (or,    , |)
  73 FETCH_AND_OP_WORD (and,   , &)
  74 FETCH_AND_OP_WORD (xor,   , ^)
  75 FETCH_AND_OP_WORD (nand, ~, &)
  76
  77 #define NAME_oldval(OP, WIDTH) __sync_fetch_and_##OP##_##WIDTH
  78 #define NAME_newval(OP, WIDTH) __sync_##OP##_and_fetch_##WIDTH
  79
  80 /* Implement both __sync_<op>_and_fetch and __sync_fetch_and_<op> for
  81    subword-sized quantities.  */
  82
  83 #define SUBWORD_SYNC_OP(OP, PFX_OP, INF_OP, TYPE, WIDTH, RETURN)        \
  84   TYPE HIDDEN                                                           \
  85   NAME##_##RETURN (OP, WIDTH) (TYPE *ptr, TYPE val)                     \
  86   {                                                                     \
  87     int *wordptr = (int *) ((unsigned int) ptr & ~3);                   \
  88     unsigned int mask, shift, oldval, newval;                           \
  89     int failure;                                                        \
  90                                                                         \
  91     shift = (((unsigned int) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH;      \
  92     mask = MASK_##WIDTH << shift;                                       \
  93                                                                         \
  94     do {                                                                \
  95       oldval = *wordptr;                                                \
  96       newval = ((PFX_OP ((oldval & mask) >> shift)                      \
  97                  INF_OP (unsigned int) val) << shift) & mask;           \
  98       newval |= oldval & ~mask;                                         \
  99       failure = __kernel_cmpxchg (oldval, newval, wordptr);             \
 100     } while (failure != 0);                                             \
 101                                                                         \
 102     return (RETURN & mask) >> shift;                                    \
 103   }
 104
 105 SUBWORD_SYNC_OP (add,   , +, short, 2, oldval)
 106 SUBWORD_SYNC_OP (sub,   , -, short, 2, oldval)
 107 SUBWORD_SYNC_OP (or,    , |, short, 2, oldval)
 108 SUBWORD_SYNC_OP (and,   , &, short, 2, oldval)
 109 SUBWORD_SYNC_OP (xor,   , ^, short, 2, oldval)
 110 SUBWORD_SYNC_OP (nand, ~, &, short, 2, oldval)
 111
 112 SUBWORD_SYNC_OP (add,   , +, char, 1, oldval)
 113 SUBWORD_SYNC_OP (sub,   , -, char, 1, oldval)
 114 SUBWORD_SYNC_OP (or,    , |, char, 1, oldval)
 115 SUBWORD_SYNC_OP (and,   , &, char, 1, oldval)
 116 SUBWORD_SYNC_OP (xor,   , ^, char, 1, oldval)
 117 SUBWORD_SYNC_OP (nand, ~, &, char, 1, oldval)
 118
 119 #define OP_AND_FETCH_WORD(OP, PFX_OP, INF_OP)                           \
 120   int HIDDEN                                                            \
 121   __sync_##OP##_and_fetch_4 (int *ptr, int val)                         \
 122   {                                                                     \
 123     int tmp, failure;                                                   \
 124                                                                         \
 125     do {                                                                \
 126       tmp = *ptr;                                                       \
 127       failure = __kernel_cmpxchg (tmp, PFX_OP tmp INF_OP val, ptr);     \
 128     } while (failure != 0);                                             \
 129                                                                         \
 130     return PFX_OP tmp INF_OP val;                                       \
 131   }
 132
 133 OP_AND_FETCH_WORD (add,   , +)
 134 OP_AND_FETCH_WORD (sub,   , -)
 135 OP_AND_FETCH_WORD (or,    , |)
 136 OP_AND_FETCH_WORD (and,   , &)
 137 OP_AND_FETCH_WORD (xor,   , ^)
 138 OP_AND_FETCH_WORD (nand, ~, &)
 139
 140 SUBWORD_SYNC_OP (add,   , +, short, 2, newval)
 141 SUBWORD_SYNC_OP (sub,   , -, short, 2, newval)
 142 SUBWORD_SYNC_OP (or,    , |, short, 2, newval)
 143 SUBWORD_SYNC_OP (and,   , &, short, 2, newval)
 144 SUBWORD_SYNC_OP (xor,   , ^, short, 2, newval)
 145 SUBWORD_SYNC_OP (nand, ~, &, short, 2, newval)
 146
 147 SUBWORD_SYNC_OP (add,   , +, char, 1, newval)
 148 SUBWORD_SYNC_OP (sub,   , -, char, 1, newval)
 149 SUBWORD_SYNC_OP (or,    , |, char, 1, newval)
 150 SUBWORD_SYNC_OP (and,   , &, char, 1, newval)
 151 SUBWORD_SYNC_OP (xor,   , ^, char, 1, newval)
 152 SUBWORD_SYNC_OP (nand, ~, &, char, 1, newval)
 153
 154 int HIDDEN
 155 __sync_val_compare_and_swap_4 (int *ptr, int oldval, int newval)
 156 {
 157   int actual_oldval, fail;
 158
 159   while (1)
 160     {
 161       actual_oldval = *ptr;
 162
 163       if (oldval != actual_oldval)
 164         return actual_oldval;
 165
 166       fail = __kernel_cmpxchg (actual_oldval, newval, ptr);
 167
 168       if (!fail)
 169         return oldval;
 170     }
 171 }
 172
 173 #define SUBWORD_VAL_CAS(TYPE, WIDTH)                                    \
 174   TYPE HIDDEN                                                           \
 175   __sync_val_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval,          \
 176                                        TYPE newval)                     \
 177   {                                                                     \
 178     int *wordptr = (int *)((unsigned int) ptr & ~3), fail;              \
 179     unsigned int mask, shift, actual_oldval, actual_newval;             \
 180                                                                         \
 181     shift = (((unsigned int) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH;      \
 182     mask = MASK_##WIDTH << shift;                                       \
 183                                                                         \
 184     while (1)                                                           \
 185       {                                                                 \
 186         actual_oldval = *wordptr;                                       \
 187                                                                         \
 188         if (((actual_oldval & mask) >> shift) != (unsigned int) oldval) \
 189           return (actual_oldval & mask) >> shift;                       \
 190                                                                         \
 191         actual_newval = (actual_oldval & ~mask)                         \
 192                         | (((unsigned int) newval << shift) & mask);    \
 193                                                                         \
 194         fail = __kernel_cmpxchg (actual_oldval, actual_newval,          \
 195                                  wordptr);                              \
 196                                                                         \
 197         if (!fail)                                                      \
 198           return oldval;                                                \
 199       }                                                                 \
 200   }
 201
 202 SUBWORD_VAL_CAS (short, 2)
 203 SUBWORD_VAL_CAS (char,  1)
 204
 205 typedef unsigned char bool;
 206
 207 bool HIDDEN
 208 __sync_bool_compare_and_swap_4 (int *ptr, int oldval, int newval)
 209 {
 210   int failure = __kernel_cmpxchg (oldval, newval, ptr);
 211   return (failure == 0);
 212 }
 213
 214 #define SUBWORD_BOOL_CAS(TYPE, WIDTH)                                   \
 215   bool HIDDEN                                                           \
 216   __sync_bool_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval,         \
 217                                         TYPE newval)                    \
 218   {                                                                     \
 219     TYPE actual_oldval                                                  \
 220       = __sync_val_compare_and_swap_##WIDTH (ptr, oldval, newval);      \
 221     return (oldval == actual_oldval);                                   \
 222   }
 223
 224 SUBWORD_BOOL_CAS (short, 2)
 225 SUBWORD_BOOL_CAS (char,  1)
 226
 227 void HIDDEN
 228 __sync_synchronize (void)
 229 {
 230   __kernel_dmb ();
 231 }
 232
 233 int HIDDEN
 234 __sync_lock_test_and_set_4 (int *ptr, int val)
 235 {
 236   int failure, oldval;
 237
 238   do {
 239     oldval = *ptr;
 240     failure = __kernel_cmpxchg (oldval, val, ptr);
 241   } while (failure != 0);
 242
 243   return oldval;
 244 }
 245
 246 #define SUBWORD_TEST_AND_SET(TYPE, WIDTH)                               \
 247   TYPE HIDDEN                                                           \
 248   __sync_lock_test_and_set_##WIDTH (TYPE *ptr, TYPE val)                \
 249   {                                                                     \
 250     int failure;                                                        \
 251     unsigned int oldval, newval, shift, mask;                           \
 252     int *wordptr = (int *) ((unsigned int) ptr & ~3);                   \
 253                                                                         \
 254     shift = (((unsigned int) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH;      \
 255     mask = MASK_##WIDTH << shift;                                       \
 256                                                                         \
 257     do {                                                                \
 258       oldval = *wordptr;                                                \
 259       newval = (oldval & ~mask)                                         \
 260                | (((unsigned int) val << shift) & mask);                \
 261       failure = __kernel_cmpxchg (oldval, newval, wordptr);             \
 262     } while (failure != 0);                                             \
 263                                                                         \
 264     return (oldval & mask) >> shift;                                    \
 265   }
 266
 267 SUBWORD_TEST_AND_SET (short, 2)
 268 SUBWORD_TEST_AND_SET (char,  1)
 269
 270 #define SYNC_LOCK_RELEASE(TYPE, WIDTH)                                  \
 271   void HIDDEN                                                           \
 272   __sync_lock_release_##WIDTH (TYPE *ptr)                               \
 273   {                                                                     \
 274     *ptr = 0;                                                           \
 275     __kernel_dmb ();                                                    \
 276   }
 277
 278 SYNC_LOCK_RELEASE (int,   4)
 279 SYNC_LOCK_RELEASE (short, 2)
 280 SYNC_LOCK_RELEASE (char,  1)