"Fossies" - the Fresh Open Source Software Archive

Member "haproxy-2.0.8/include/import/atomic-ops.h" (23 Oct 2019, 31803 Bytes) of package /linux/misc/haproxy-2.0.8.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "atomic-ops.h" see the Fossies "Dox" file reference documentation.

    1 #ifndef PL_ATOMIC_OPS_H
    2 #define PL_ATOMIC_OPS_H
    3 
    4 
    5 /* compiler-only memory barrier, for use around locks */
    6 #define pl_barrier() do {           \
    7         asm volatile("" ::: "memory");  \
    8     } while (0)
    9 
   10 #if defined(__i386__) || defined (__i486__) || defined (__i586__) || defined (__i686__) || defined (__x86_64__)
   11 
   12 /* full memory barrier using mfence when SSE2 is supported, falling back to
   13  * "lock add %esp" (gcc uses "lock add" or "lock or").
   14  */
   15 #if defined(__SSE2__)
   16 
   17 #define pl_mb() do {                                 \
   18         asm volatile("mfence" ::: "memory"); \
   19     } while (0)
   20 
   21 #elif defined(__x86_64__)
   22 
   23 #define pl_mb() do {                                                       \
   24         asm volatile("lock addl $0,0 (%%rsp)" ::: "memory", "cc"); \
   25     } while (0)
   26 
   27 #else /* ix86 */
   28 
   29 #define pl_mb() do {                                                       \
   30         asm volatile("lock addl $0,0 (%%esp)" ::: "memory", "cc"); \
   31     } while (0)
   32 
   33 #endif /* end of pl_mb() case for sse2/x86_64/x86 */
   34 
   35 /*
   36  * Generic functions common to the x86 family
   37  */
   38 
   39 #define pl_cpu_relax() do {                   \
   40         asm volatile("rep;nop\n");    \
   41     } while (0)
   42 
   43 /* increment integer value pointed to by pointer <ptr>, and return non-zero if
   44  * result is non-null.
   45  */
   46 #define pl_inc(ptr) (                                                         \
   47     (sizeof(long) == 8 && sizeof(*(ptr)) == 8) ? ({                       \
   48         unsigned char ret;                                            \
   49         asm volatile("lock incq %0\n"                                 \
   50                  "setne %1\n"                                     \
   51                  : "+m" (*(ptr)), "=qm" (ret)                     \
   52                  :                                                \
   53                  : "cc");                                         \
   54         ret; /* return value */                                       \
   55     }) : (sizeof(*(ptr)) == 4) ? ({                                       \
   56         unsigned char ret;                                            \
   57         asm volatile("lock incl %0\n"                                 \
   58                  "setne %1\n"                                     \
   59                  : "+m" (*(ptr)), "=qm" (ret)                     \
   60                  :                                                \
   61                  : "cc");                                         \
   62         ret; /* return value */                                       \
   63     }) : (sizeof(*(ptr)) == 2) ? ({                                       \
   64         unsigned char ret;                                            \
   65         asm volatile("lock incw %0\n"                                 \
   66                  "setne %1\n"                                     \
   67                  : "+m" (*(ptr)), "=qm" (ret)                     \
   68                  :                                                \
   69                  : "cc");                                         \
   70         ret; /* return value */                                       \
   71     }) : (sizeof(*(ptr)) == 1) ? ({                                       \
   72         unsigned char ret;                                            \
   73         asm volatile("lock incb %0\n"                                 \
   74                  "setne %1\n"                                     \
   75                  : "+m" (*(ptr)), "=qm" (ret)                     \
   76                  :                                                \
   77                  : "cc");                                         \
   78         ret; /* return value */                                       \
   79     }) : ({                                                               \
   80         void __unsupported_argument_size_for_pl_inc__(char *,int);    \
   81         if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 &&                      \
   82             sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \
   83             __unsupported_argument_size_for_pl_inc__(__FILE__,__LINE__);   \
   84         0;                                                            \
   85     })                                                                    \
   86 )
   87 
   88 /* decrement integer value pointed to by pointer <ptr>, and return non-zero if
   89  * result is non-null.
   90  */
   91 #define pl_dec(ptr) (                                                         \
   92     (sizeof(long) == 8 && sizeof(*(ptr)) == 8) ? ({                       \
   93         unsigned char ret;                                            \
   94         asm volatile("lock decq %0\n"                                 \
   95                  "setne %1\n"                                     \
   96                  : "+m" (*(ptr)), "=qm" (ret)                     \
   97                  :                                                \
   98                  : "cc");                                         \
   99         ret; /* return value */                                       \
  100     }) : (sizeof(*(ptr)) == 4) ? ({                                       \
  101         unsigned char ret;                                            \
  102         asm volatile("lock decl %0\n"                                 \
  103                  "setne %1\n"                                     \
  104                  : "+m" (*(ptr)), "=qm" (ret)                     \
  105                  :                                                \
  106                  : "cc");                                         \
  107         ret; /* return value */                                       \
  108     }) : (sizeof(*(ptr)) == 2) ? ({                                       \
  109         unsigned char ret;                                            \
  110         asm volatile("lock decw %0\n"                                 \
  111                  "setne %1\n"                                     \
  112                  : "+m" (*(ptr)), "=qm" (ret)                     \
  113                  :                                                \
  114                  : "cc");                                         \
  115         ret; /* return value */                                       \
  116     }) : (sizeof(*(ptr)) == 1) ? ({                                       \
  117         unsigned char ret;                                            \
  118         asm volatile("lock decb %0\n"                                 \
  119                  "setne %1\n"                                     \
  120                  : "+m" (*(ptr)), "=qm" (ret)                     \
  121                  :                                                \
  122                  : "cc");                                         \
  123         ret; /* return value */                                       \
  124     }) : ({                                                               \
  125         void __unsupported_argument_size_for_pl_dec__(char *,int);    \
  126         if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 &&                      \
  127             sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \
  128             __unsupported_argument_size_for_pl_dec__(__FILE__,__LINE__);   \
  129         0;                                                            \
  130     })                                                                    \
  131 )
  132 
  133 /* increment integer value pointed to by pointer <ptr>, no return */
  134 #define pl_inc_noret(ptr) ({                                                  \
  135     if (sizeof(long) == 8 && sizeof(*(ptr)) == 8) {                       \
  136         asm volatile("lock incq %0\n"                                 \
  137                  : "+m" (*(ptr))                                  \
  138                  :                                                \
  139                  : "cc");                                         \
  140     } else if (sizeof(*(ptr)) == 4) {                                     \
  141         asm volatile("lock incl %0\n"                                 \
  142                  : "+m" (*(ptr))                                  \
  143                  :                                                \
  144                  : "cc");                                         \
  145     } else if (sizeof(*(ptr)) == 2) {                                     \
  146         asm volatile("lock incw %0\n"                                 \
  147                  : "+m" (*(ptr))                                  \
  148                  :                                                \
  149                  : "cc");                                         \
  150     } else if (sizeof(*(ptr)) == 1) {                                     \
  151         asm volatile("lock incb %0\n"                                 \
  152                  : "+m" (*(ptr))                                  \
  153                  :                                                \
  154                  : "cc");                                         \
  155     } else {                                                              \
  156         void __unsupported_argument_size_for_pl_inc_noret__(char *,int);   \
  157         if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 &&                          \
  158             sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8))     \
  159             __unsupported_argument_size_for_pl_inc_noret__(__FILE__,__LINE__); \
  160     }                                                                     \
  161 })
  162 
  163 /* decrement integer value pointed to by pointer <ptr>, no return */
  164 #define pl_dec_noret(ptr) ({                                                  \
  165     if (sizeof(long) == 8 && sizeof(*(ptr)) == 8) {                       \
  166         asm volatile("lock decq %0\n"                                 \
  167                  : "+m" (*(ptr))                                  \
  168                  :                                                \
  169                  : "cc");                                         \
  170     } else if (sizeof(*(ptr)) == 4) {                                     \
  171         asm volatile("lock decl %0\n"                                 \
  172                  : "+m" (*(ptr))                                  \
  173                  :                                                \
  174                  : "cc");                                         \
  175     } else if (sizeof(*(ptr)) == 2) {                                     \
  176         asm volatile("lock decw %0\n"                                 \
  177                  : "+m" (*(ptr))                                  \
  178                  :                                                \
  179                  : "cc");                                         \
  180     } else if (sizeof(*(ptr)) == 1) {                                     \
  181         asm volatile("lock decb %0\n"                                 \
  182                  : "+m" (*(ptr))                                  \
  183                  :                                                \
  184                  : "cc");                                         \
  185     } else {                                                              \
  186         void __unsupported_argument_size_for_pl_dec_noret__(char *,int);   \
  187         if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 &&                          \
  188             sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8))     \
  189             __unsupported_argument_size_for_pl_dec_noret__(__FILE__,__LINE__); \
  190     }                                                                     \
  191 })
  192 
  193 /* add integer constant <x> to integer value pointed to by pointer <ptr>,
  194  * no return. Size of <x> is not checked.
  195  */
  196 #define pl_add(ptr, x) ({                                                     \
  197     if (sizeof(long) == 8 && sizeof(*(ptr)) == 8) {                       \
  198         asm volatile("lock addq %1, %0\n"                             \
  199                  : "+m" (*(ptr))                                  \
  200                  : "er" ((unsigned long)(x))                      \
  201                  : "cc");                                         \
  202     } else if (sizeof(*(ptr)) == 4) {                                     \
  203         asm volatile("lock addl %1, %0\n"                             \
  204                  : "+m" (*(ptr))                                  \
  205                  : "er" ((unsigned int)(x))                       \
  206                  : "cc");                                         \
  207     } else if (sizeof(*(ptr)) == 2) {                                     \
  208         asm volatile("lock addw %1, %0\n"                             \
  209                  : "+m" (*(ptr))                                  \
  210                  : "er" ((unsigned short)(x))                     \
  211                  : "cc");                                         \
  212     } else if (sizeof(*(ptr)) == 1) {                                     \
  213         asm volatile("lock addb %1, %0\n"                             \
  214                  : "+m" (*(ptr))                                  \
  215                  : "er" ((unsigned char)(x))                      \
  216                  : "cc");                                         \
  217     } else {                                                              \
  218         void __unsupported_argument_size_for_pl_add__(char *,int);    \
  219         if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 &&                          \
  220             sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8))     \
  221             __unsupported_argument_size_for_pl_add__(__FILE__,__LINE__);       \
  222     }                                                                     \
  223 })
  224 
  225 /* subtract integer constant <x> from integer value pointed to by pointer
  226  * <ptr>, no return. Size of <x> is not checked.
  227  */
  228 #define pl_sub(ptr, x) ({                                                     \
  229     if (sizeof(long) == 8 && sizeof(*(ptr)) == 8) {                       \
  230         asm volatile("lock subq %1, %0\n"                             \
  231                  : "+m" (*(ptr))                                  \
  232                  : "er" ((unsigned long)(x))                      \
  233                  : "cc");                                         \
  234     } else if (sizeof(*(ptr)) == 4) {                                     \
  235         asm volatile("lock subl %1, %0\n"                             \
  236                  : "+m" (*(ptr))                                  \
  237                  : "er" ((unsigned int)(x))                       \
  238                  : "cc");                                         \
  239     } else if (sizeof(*(ptr)) == 2) {                                     \
  240         asm volatile("lock subw %1, %0\n"                             \
  241                  : "+m" (*(ptr))                                  \
  242                  : "er" ((unsigned short)(x))                     \
  243                  : "cc");                                         \
  244     } else if (sizeof(*(ptr)) == 1) {                                     \
  245         asm volatile("lock subb %1, %0\n"                             \
  246                  : "+m" (*(ptr))                                  \
  247                  : "er" ((unsigned char)(x))                      \
  248                  : "cc");                                         \
  249     } else {                                                              \
  250         void __unsupported_argument_size_for_pl_sub__(char *,int);    \
  251         if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 &&                      \
  252             sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \
  253             __unsupported_argument_size_for_pl_sub__(__FILE__,__LINE__);   \
  254     }                                                                     \
  255 })
  256 
  257 /* binary and integer value pointed to by pointer <ptr> with constant <x>, no
  258  * return. Size of <x> is not checked.
  259  */
  260 #define pl_and(ptr, x) ({                                                     \
  261     if (sizeof(long) == 8 && sizeof(*(ptr)) == 8) {                       \
  262         asm volatile("lock andq %1, %0\n"                             \
  263                  : "+m" (*(ptr))                                  \
  264                  : "er" ((unsigned long)(x))                      \
  265                  : "cc");                                         \
  266     } else if (sizeof(*(ptr)) == 4) {                                     \
  267         asm volatile("lock andl %1, %0\n"                             \
  268                  : "+m" (*(ptr))                                  \
  269                  : "er" ((unsigned int)(x))                       \
  270                  : "cc");                                         \
  271     } else if (sizeof(*(ptr)) == 2) {                                     \
  272         asm volatile("lock andw %1, %0\n"                             \
  273                  : "+m" (*(ptr))                                  \
  274                  : "er" ((unsigned short)(x))                     \
  275                  : "cc");                                         \
  276     } else if (sizeof(*(ptr)) == 1) {                                     \
  277         asm volatile("lock andb %1, %0\n"                             \
  278                  : "+m" (*(ptr))                                  \
  279                  : "er" ((unsigned char)(x))                      \
  280                  : "cc");                                         \
  281     } else {                                                              \
  282         void __unsupported_argument_size_for_pl_and__(char *,int);    \
  283         if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 &&                       \
  284             sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8))  \
  285             __unsupported_argument_size_for_pl_and__(__FILE__,__LINE__);    \
  286     }                                                                     \
  287 })
  288 
  289 /* binary or integer value pointed to by pointer <ptr> with constant <x>, no
  290  * return. Size of <x> is not checked.
  291  */
  292 #define pl_or(ptr, x) ({                                                      \
  293     if (sizeof(long) == 8 && sizeof(*(ptr)) == 8) {                       \
  294         asm volatile("lock orq %1, %0\n"                              \
  295                  : "+m" (*(ptr))                                  \
  296                  : "er" ((unsigned long)(x))                      \
  297                  : "cc");                                         \
  298     } else if (sizeof(*(ptr)) == 4) {                                     \
  299         asm volatile("lock orl %1, %0\n"                              \
  300                  : "+m" (*(ptr))                                  \
  301                  : "er" ((unsigned int)(x))                       \
  302                  : "cc");                                         \
  303     } else if (sizeof(*(ptr)) == 2) {                                     \
  304         asm volatile("lock orw %1, %0\n"                              \
  305                  : "+m" (*(ptr))                                  \
  306                  : "er" ((unsigned short)(x))                     \
  307                  : "cc");                                         \
  308     } else if (sizeof(*(ptr)) == 1) {                                     \
  309         asm volatile("lock orb %1, %0\n"                              \
  310                  : "+m" (*(ptr))                                  \
  311                  : "er" ((unsigned char)(x))                      \
  312                  : "cc");                                         \
  313     } else {                                                              \
  314         void __unsupported_argument_size_for_pl_or__(char *,int);     \
  315         if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 &&                       \
  316             sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8))  \
  317             __unsupported_argument_size_for_pl_or__(__FILE__,__LINE__);     \
  318     }                                                                     \
  319 })
  320 
  321 /* binary xor integer value pointed to by pointer <ptr> with constant <x>, no
  322  * return. Size of <x> is not checked.
  323  */
  324 #define pl_xor(ptr, x) ({                                                     \
  325     if (sizeof(long) == 8 && sizeof(*(ptr)) == 8) {                       \
  326         asm volatile("lock xorq %1, %0\n"                             \
  327                  : "+m" (*(ptr))                                  \
  328                  : "er" ((unsigned long)(x))                      \
  329                  : "cc");                                         \
  330     } else if (sizeof(*(ptr)) == 4) {                                     \
  331         asm volatile("lock xorl %1, %0\n"                             \
  332                  : "+m" (*(ptr))                                  \
  333                  : "er" ((unsigned int)(x))                       \
  334                  : "cc");                                         \
  335     } else if (sizeof(*(ptr)) == 2) {                                     \
  336         asm volatile("lock xorw %1, %0\n"                             \
  337                  : "+m" (*(ptr))                                  \
  338                  : "er" ((unsigned short)(x))                     \
  339                  : "cc");                                         \
  340     } else if (sizeof(*(ptr)) == 1) {                                     \
  341         asm volatile("lock xorb %1, %0\n"                             \
  342                  : "+m" (*(ptr))                                  \
  343                  : "er" ((unsigned char)(x))                      \
  344                  : "cc");                                         \
  345     } else {                                                              \
  346         void __unsupported_argument_size_for_pl_xor__(char *,int);    \
  347         if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 &&                       \
  348             sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8))  \
  349         __unsupported_argument_size_for_pl_xor__(__FILE__,__LINE__);            \
  350     }                                                                     \
  351 })
  352 
  353 /* test and set bit <bit> in integer value pointed to by pointer <ptr>. Returns
  354  * 0 if the bit was not set, or ~0 of the same type as *ptr if it was set. Note
  355  * that there is no 8-bit equivalent operation.
  356  */
  357 #define pl_bts(ptr, bit) (                                                    \
  358     (sizeof(long) == 8 && sizeof(*(ptr)) == 8) ? ({                       \
  359         unsigned long ret;                                            \
  360         asm volatile("lock btsq %2, %0\n\t"                           \
  361                  "sbb %1, %1\n\t"                                 \
  362                  : "+m" (*(ptr)), "=r" (ret)                      \
  363                  : "Ir" ((unsigned long)(bit))                    \
  364                  : "cc");                                         \
  365         ret; /* return value */                                       \
  366     }) : (sizeof(*(ptr)) == 4) ? ({                                       \
  367         unsigned int ret;                                             \
  368         asm volatile("lock btsl %2, %0\n\t"                           \
  369                  "sbb %1, %1\n\t"                                 \
  370                  : "+m" (*(ptr)), "=r" (ret)                      \
  371                  : "Ir" ((unsigned int)(bit))                     \
  372                  : "cc");                                         \
  373         ret; /* return value */                                       \
  374     }) : (sizeof(*(ptr)) == 2) ? ({                                       \
  375         unsigned short ret;                                           \
  376         asm volatile("lock btsw %2, %0\n\t"                           \
  377                  "sbb %1, %1\n\t"                                 \
  378                  : "+m" (*(ptr)), "=r" (ret)                      \
  379                  : "Ir" ((unsigned short)(bit))                   \
  380                  : "cc");                                         \
  381         ret; /* return value */                                       \
  382     }) : ({                                                               \
  383         void __unsupported_argument_size_for_pl_bts__(char *,int);    \
  384         if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 &&                      \
  385             sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \
  386             __unsupported_argument_size_for_pl_bts__(__FILE__,__LINE__);   \
  387         0;                                                            \
  388     })                                                                    \
  389 )
  390 
  391 /* Note: for an unclear reason, gcc's __sync_fetch_and_add() implementation
  392  * produces less optimal than hand-crafted asm code so let's implement here the
  393  * operations we need for the most common archs.
  394  */
  395 
  396 /* fetch-and-add: fetch integer value pointed to by pointer <ptr>, add <x> to
  397  * to <*ptr> and return the previous value.
  398  */
  399 #define pl_xadd(ptr, x) (                                                     \
  400     (sizeof(long) == 8 && sizeof(*(ptr)) == 8) ? ({                       \
  401         unsigned long ret = (unsigned long)(x);                       \
  402         asm volatile("lock xaddq %0, %1\n"                            \
  403                  :  "=r" (ret), "+m" (*(ptr))                     \
  404                  : "0" (ret)                                      \
  405                  : "cc");                                         \
  406         ret; /* return value */                                       \
  407     }) : (sizeof(*(ptr)) == 4) ? ({                                       \
  408         unsigned int ret = (unsigned int)(x);                         \
  409         asm volatile("lock xaddl %0, %1\n"                            \
  410                  :  "=r" (ret), "+m" (*(ptr))                     \
  411                  : "0" (ret)                                      \
  412                  : "cc");                                         \
  413         ret; /* return value */                                       \
  414     }) : (sizeof(*(ptr)) == 2) ? ({                                       \
  415         unsigned short ret = (unsigned short)(x);                     \
  416         asm volatile("lock xaddw %0, %1\n"                            \
  417                  :  "=r" (ret), "+m" (*(ptr))                     \
  418                  : "0" (ret)                                      \
  419                  : "cc");                                         \
  420         ret; /* return value */                                       \
  421     }) : (sizeof(*(ptr)) == 1) ? ({                                       \
  422         unsigned char ret = (unsigned char)(x);                       \
  423         asm volatile("lock xaddb %0, %1\n"                            \
  424                  :  "=r" (ret), "+m" (*(ptr))                     \
  425                  : "0" (ret)                                      \
  426                  : "cc");                                         \
  427         ret; /* return value */                                       \
  428     }) : ({                                                               \
  429         void __unsupported_argument_size_for_pl_xadd__(char *,int);   \
  430         if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 &&                       \
  431             sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8))  \
  432             __unsupported_argument_size_for_pl_xadd__(__FILE__,__LINE__);   \
  433         0;                                                            \
  434     })                                                                    \
  435 )
  436 
  437 /* exchage value <x> with integer value pointed to by pointer <ptr>, and return
  438  * previous <*ptr> value. <x> must be of the same size as <*ptr>.
  439  */
  440 #define pl_xchg(ptr, x) (                                                     \
  441     (sizeof(long) == 8 && sizeof(*(ptr)) == 8) ? ({                       \
  442         unsigned long ret = (unsigned long)(x);                       \
  443         asm volatile("xchgq %0, %1\n"                                 \
  444                  :  "=r" (ret), "+m" (*(ptr))                     \
  445                  : "0" (ret)                                      \
  446                  : "cc");                                         \
  447         ret; /* return value */                                       \
  448     }) : (sizeof(*(ptr)) == 4) ? ({                                       \
  449         unsigned int ret = (unsigned int)(x);                         \
  450         asm volatile("xchgl %0, %1\n"                                 \
  451                  :  "=r" (ret), "+m" (*(ptr))                     \
  452                  : "0" (ret)                                      \
  453                  : "cc");                                         \
  454         ret; /* return value */                                       \
  455     }) : (sizeof(*(ptr)) == 2) ? ({                                       \
  456         unsigned short ret = (unsigned short)(x);                     \
  457         asm volatile("xchgw %0, %1\n"                                 \
  458                  :  "=r" (ret), "+m" (*(ptr))                     \
  459                  : "0" (ret)                                      \
  460                  : "cc");                                         \
  461         ret; /* return value */                                       \
  462     }) : (sizeof(*(ptr)) == 1) ? ({                                       \
  463         unsigned char ret = (unsigned char)(x);                       \
  464         asm volatile("xchgb %0, %1\n"                                 \
  465                  :  "=r" (ret), "+m" (*(ptr))                     \
  466                  : "0" (ret)                                      \
  467                  : "cc");                                         \
  468         ret; /* return value */                                       \
  469     }) : ({                                                               \
  470         void __unsupported_argument_size_for_pl_xchg__(char *,int);   \
  471         if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 &&                       \
  472             sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8))  \
  473         __unsupported_argument_size_for_pl_xchg__(__FILE__,__LINE__);           \
  474         0;                                                            \
  475     })                                                                    \
  476 )
  477 
  478 /* compare integer value <*ptr> with <old> and exchange it with <new> if
  479  * it matches, and return <old>. <old> and <new> must be of the same size as
  480  * <*ptr>.
  481  */
  482 #define pl_cmpxchg(ptr, old, new) (                                           \
  483     (sizeof(long) == 8 && sizeof(*(ptr)) == 8) ? ({                       \
  484         unsigned long ret;                                            \
  485         asm volatile("lock cmpxchgq %2,%1"                            \
  486                  : "=a" (ret), "+m" (*(ptr))                      \
  487                  : "r" ((unsigned long)(new)),                    \
  488                    "0" ((unsigned long)(old))                     \
  489                  : "cc");                                         \
  490         ret; /* return value */                                       \
  491     }) : (sizeof(*(ptr)) == 4) ? ({                                       \
  492         unsigned int ret;                                             \
  493         asm volatile("lock cmpxchgl %2,%1"                            \
  494                  : "=a" (ret), "+m" (*(ptr))                      \
  495                  : "r" ((unsigned int)(new)),                     \
  496                    "0" ((unsigned int)(old))                      \
  497                  : "cc");                                         \
  498         ret; /* return value */                                       \
  499     }) : (sizeof(*(ptr)) == 2) ? ({                                       \
  500         unsigned short ret;                                           \
  501         asm volatile("lock cmpxchgw %2,%1"                            \
  502                  : "=a" (ret), "+m" (*(ptr))                      \
  503                  : "r" ((unsigned short)(new)),                   \
  504                    "0" ((unsigned short)(old))                    \
  505                  : "cc");                                         \
  506         ret; /* return value */                                       \
  507     }) : (sizeof(*(ptr)) == 1) ? ({                                       \
  508         unsigned char ret;                                            \
  509         asm volatile("lock cmpxchgb %2,%1"                            \
  510                  : "=a" (ret), "+m" (*(ptr))                      \
  511                  : "r" ((unsigned char)(new)),                    \
  512                    "0" ((unsigned char)(old))                     \
  513                  : "cc");                                         \
  514         ret; /* return value */                                       \
  515     }) : ({                                                               \
  516         void __unsupported_argument_size_for_pl_cmpxchg__(char *,int);   \
  517         if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 &&                      \
  518             sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \
  519         __unsupported_argument_size_for_pl_cmpxchg__(__FILE__,__LINE__);       \
  520         0;                                                            \
  521     })                                                                    \
  522 )
  523 
  524 #else
  525 /* generic implementations */
  526 
  527 #define pl_cpu_relax() do {             \
  528         asm volatile("");       \
  529     } while (0)
  530 
  531 /* full memory barrier */
  532 #define pl_mb() do {                    \
  533         __sync_synchronize();   \
  534     } while (0)
  535 
  536 #define pl_inc_noret(ptr)     ({ __sync_add_and_fetch((ptr), 1);   })
  537 #define pl_dec_noret(ptr)     ({ __sync_sub_and_fetch((ptr), 1);   })
  538 #define pl_inc(ptr)           ({ __sync_add_and_fetch((ptr), 1);   })
  539 #define pl_dec(ptr)           ({ __sync_sub_and_fetch((ptr), 1);   })
  540 #define pl_add(ptr, x)        ({ __sync_add_and_fetch((ptr), (x)); })
  541 #define pl_and(ptr, x)        ({ __sync_and_and_fetch((ptr), (x)); })
  542 #define pl_or(ptr, x)         ({ __sync_or_and_fetch((ptr), (x));  })
  543 #define pl_xor(ptr, x)        ({ __sync_xor_and_fetch((ptr), (x)); })
  544 #define pl_sub(ptr, x)        ({ __sync_sub_and_fetch((ptr), (x)); })
  545 #define pl_bts(ptr, bit)      ({ typeof(*(ptr)) __pl_t = (1u << (bit));         \
  546                                  __sync_fetch_and_or((ptr), __pl_t) & __pl_t;   \
  547                               })
  548 #define pl_xadd(ptr, x)       ({ __sync_fetch_and_add((ptr), (x)); })
  549 #define pl_cmpxchg(ptr, o, n) ({ __sync_val_compare_and_swap((ptr), (o), (n)); })
  550 #define pl_xchg(ptr, x)       ({ typeof(*(ptr)) __pl_t;                                       \
  551                                  do { __pl_t = *(ptr);                                        \
  552                                  } while (!__sync_bool_compare_and_swap((ptr), __pl_t, (x))); \
  553                                  __pl_t;                                                      \
  554                               })
  555 
  556 #endif
  557 
  558 #endif /* PL_ATOMIC_OPS_H */