Home | History | Annotate | Download | only in sys
      1 /*
      2  * CDDL HEADER START
      3  *
      4  * The contents of this file are subject to the terms of the
      5  * Common Development and Distribution License (the "License").
      6  * You may not use this file except in compliance with the License.
      7  *
      8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
      9  * or http://www.opensolaris.org/os/licensing.
     10  * See the License for the specific language governing permissions
     11  * and limitations under the License.
     12  *
     13  * When distributing Covered Code, include this CDDL HEADER in each
     14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
     15  * If applicable, add the following below this CDDL HEADER, with the
     16  * fields enclosed by brackets "[]" replaced with your own identifying
     17  * information: Portions Copyright [yyyy] [name of copyright owner]
     18  *
     19  * CDDL HEADER END
     20  */
     21 
     22 /*
     23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
     24  * Use is subject to license terms.
     25  */
     26 
     27 #ifndef	_SYS_ATOMIC_H
     28 #define	_SYS_ATOMIC_H
     29 
     30 #pragma ident	"@(#)atomic.h	1.15	05/09/28 SMI"
     31 
     32 #include <sys/types.h>
     33 #include <sys/inttypes.h>
     34 
     35 #ifdef	__cplusplus
     36 extern "C" {
     37 #endif
     38 
     39 #if defined(_KERNEL) && defined(__GNUC__) && defined(_ASM_INLINES) && \
     40 	(defined(__i386) || defined(__amd64))
     41 #include <asm/atomic.h>
     42 #endif
     43 
     44 #if defined(_KERNEL) || defined(__STDC__)
     45 /*
     46  * Increment target.
     47  */
     48 extern void atomic_inc_8(volatile uint8_t *);
     49 extern void atomic_inc_uchar(volatile uchar_t *);
     50 extern void atomic_inc_16(volatile uint16_t *);
     51 extern void atomic_inc_ushort(volatile ushort_t *);
     52 extern void atomic_inc_32(volatile uint32_t *);
     53 extern void atomic_inc_uint(volatile uint_t *);
     54 extern void atomic_inc_ulong(volatile ulong_t *);
     55 #if defined(_KERNEL) || defined(_INT64_TYPE)
     56 extern void atomic_inc_64(volatile uint64_t *);
     57 #endif
     58 
     59 /*
     60  * Decrement target
     61  */
     62 extern void atomic_dec_8(volatile uint8_t *);
     63 extern void atomic_dec_uchar(volatile uchar_t *);
     64 extern void atomic_dec_16(volatile uint16_t *);
     65 extern void atomic_dec_ushort(volatile ushort_t *);
     66 extern void atomic_dec_32(volatile uint32_t *);
     67 extern void atomic_dec_uint(volatile uint_t *);
     68 extern void atomic_dec_ulong(volatile ulong_t *);
     69 #if defined(_KERNEL) || defined(_INT64_TYPE)
     70 extern void atomic_dec_64(volatile uint64_t *);
     71 #endif
     72 
     73 /*
     74  * Add delta to target
     75  */
     76 extern void atomic_add_8(volatile uint8_t *, int8_t);
     77 extern void atomic_add_char(volatile uchar_t *, signed char);
     78 extern void atomic_add_16(volatile uint16_t *, int16_t);
     79 extern void atomic_add_short(volatile ushort_t *, short);
     80 extern void atomic_add_32(volatile uint32_t *, int32_t);
     81 extern void atomic_add_int(volatile uint_t *, int);
     82 extern void atomic_add_ptr(volatile void *, ssize_t);
     83 extern void atomic_add_long(volatile ulong_t *, long);
     84 #if defined(_KERNEL) || defined(_INT64_TYPE)
     85 extern void atomic_add_64(volatile uint64_t *, int64_t);
     86 #endif
     87 
     88 /*
     89  * logical OR bits with target
     90  */
     91 extern void atomic_or_8(volatile uint8_t *, uint8_t);
     92 extern void atomic_or_uchar(volatile uchar_t *, uchar_t);
     93 extern void atomic_or_16(volatile uint16_t *, uint16_t);
     94 extern void atomic_or_ushort(volatile ushort_t *, ushort_t);
     95 extern void atomic_or_32(volatile uint32_t *, uint32_t);
     96 extern void atomic_or_uint(volatile uint_t *, uint_t);
     97 extern void atomic_or_ulong(volatile ulong_t *, ulong_t);
     98 #if defined(_KERNEL) || defined(_INT64_TYPE)
     99 extern void atomic_or_64(volatile uint64_t *, uint64_t);
    100 #endif
    101 
    102 /*
    103  * logical AND bits with target
    104  */
    105 extern void atomic_and_8(volatile uint8_t *, uint8_t);
    106 extern void atomic_and_uchar(volatile uchar_t *, uchar_t);
    107 extern void atomic_and_16(volatile uint16_t *, uint16_t);
    108 extern void atomic_and_ushort(volatile ushort_t *, ushort_t);
    109 extern void atomic_and_32(volatile uint32_t *, uint32_t);
    110 extern void atomic_and_uint(volatile uint_t *, uint_t);
    111 extern void atomic_and_ulong(volatile ulong_t *, ulong_t);
    112 #if defined(_KERNEL) || defined(_INT64_TYPE)
    113 extern void atomic_and_64(volatile uint64_t *, uint64_t);
    114 #endif
    115 
    116 /*
    117  * As above, but return the new value.  Note that these _nv() variants are
    118  * substantially more expensive on some platforms than the no-return-value
    119  * versions above, so don't use them unless you really need to know the
    120  * new value *atomically* (e.g. when decrementing a reference count and
    121  * checking whether it went to zero).
    122  */
    123 
    124 /*
    125  * Increment target and return new value.
    126  */
    127 extern uint8_t atomic_inc_8_nv(volatile uint8_t *);
    128 extern uchar_t atomic_inc_uchar_nv(volatile uchar_t *);
    129 extern uint16_t atomic_inc_16_nv(volatile uint16_t *);
    130 extern ushort_t atomic_inc_ushort_nv(volatile ushort_t *);
    131 extern uint32_t atomic_inc_32_nv(volatile uint32_t *);
    132 extern uint_t atomic_inc_uint_nv(volatile uint_t *);
    133 extern ulong_t atomic_inc_ulong_nv(volatile ulong_t *);
    134 #if defined(_KERNEL) || defined(_INT64_TYPE)
    135 extern uint64_t atomic_inc_64_nv(volatile uint64_t *);
    136 #endif
    137 
    138 /*
    139  * Decrement target and return new value.
    140  */
    141 extern uint8_t atomic_dec_8_nv(volatile uint8_t *);
    142 extern uchar_t atomic_dec_uchar_nv(volatile uchar_t *);
    143 extern uint16_t atomic_dec_16_nv(volatile uint16_t *);
    144 extern ushort_t atomic_dec_ushort_nv(volatile ushort_t *);
    145 extern uint32_t atomic_dec_32_nv(volatile uint32_t *);
    146 extern uint_t atomic_dec_uint_nv(volatile uint_t *);
    147 extern ulong_t atomic_dec_ulong_nv(volatile ulong_t *);
    148 #if defined(_KERNEL) || defined(_INT64_TYPE)
    149 extern uint64_t atomic_dec_64_nv(volatile uint64_t *);
    150 #endif
    151 
    152 /*
    153  * Add delta to target
    154  */
    155 extern uint8_t atomic_add_8_nv(volatile uint8_t *, int8_t);
    156 extern uchar_t atomic_add_char_nv(volatile uchar_t *, signed char);
    157 extern uint16_t atomic_add_16_nv(volatile uint16_t *, int16_t);
    158 extern ushort_t atomic_add_short_nv(volatile ushort_t *, short);
    159 extern uint32_t atomic_add_32_nv(volatile uint32_t *, int32_t);
    160 extern uint_t atomic_add_int_nv(volatile uint_t *, int);
    161 extern void *atomic_add_ptr_nv(volatile void *, ssize_t);
    162 extern ulong_t atomic_add_long_nv(volatile ulong_t *, long);
    163 #if defined(_KERNEL) || defined(_INT64_TYPE)
    164 extern uint64_t atomic_add_64_nv(volatile uint64_t *, int64_t);
    165 #endif
    166 
    167 /*
    168  * logical OR bits with target and return new value.
    169  */
    170 extern uint8_t atomic_or_8_nv(volatile uint8_t *, uint8_t);
    171 extern uchar_t atomic_or_uchar_nv(volatile uchar_t *, uchar_t);
    172 extern uint16_t atomic_or_16_nv(volatile uint16_t *, uint16_t);
    173 extern ushort_t atomic_or_ushort_nv(volatile ushort_t *, ushort_t);
    174 extern uint32_t atomic_or_32_nv(volatile uint32_t *, uint32_t);
    175 extern uint_t atomic_or_uint_nv(volatile uint_t *, uint_t);
    176 extern ulong_t atomic_or_ulong_nv(volatile ulong_t *, ulong_t);
    177 #if defined(_KERNEL) || defined(_INT64_TYPE)
    178 extern uint64_t atomic_or_64_nv(volatile uint64_t *, uint64_t);
    179 #endif
    180 
    181 /*
    182  * logical AND bits with target and return new value.
    183  */
    184 extern uint8_t atomic_and_8_nv(volatile uint8_t *, uint8_t);
    185 extern uchar_t atomic_and_uchar_nv(volatile uchar_t *, uchar_t);
    186 extern uint16_t atomic_and_16_nv(volatile uint16_t *, uint16_t);
    187 extern ushort_t atomic_and_ushort_nv(volatile ushort_t *, ushort_t);
    188 extern uint32_t atomic_and_32_nv(volatile uint32_t *, uint32_t);
    189 extern uint_t atomic_and_uint_nv(volatile uint_t *, uint_t);
    190 extern ulong_t atomic_and_ulong_nv(volatile ulong_t *, ulong_t);
    191 #if defined(_KERNEL) || defined(_INT64_TYPE)
    192 extern uint64_t atomic_and_64_nv(volatile uint64_t *, uint64_t);
    193 #endif
    194 
    195 /*
    196  * If *arg1 == arg2, set *arg1 = arg3; return old value
    197  */
    198 extern uint8_t atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t);
    199 extern uchar_t atomic_cas_uchar(volatile uchar_t *, uchar_t, uchar_t);
    200 extern uint16_t atomic_cas_16(volatile uint16_t *, uint16_t, uint16_t);
    201 extern ushort_t atomic_cas_ushort(volatile ushort_t *, ushort_t, ushort_t);
    202 extern uint32_t atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t);
    203 extern uint_t atomic_cas_uint(volatile uint_t *, uint_t, uint_t);
    204 extern void *atomic_cas_ptr(volatile void *, void *, void *);
    205 extern ulong_t atomic_cas_ulong(volatile ulong_t *, ulong_t, ulong_t);
    206 #if defined(_KERNEL) || defined(_INT64_TYPE)
    207 extern uint64_t atomic_cas_64(volatile uint64_t *, uint64_t, uint64_t);
    208 #endif
    209 
    210 /*
    211  * Swap target and return old value
    212  */
    213 extern uint8_t atomic_swap_8(volatile uint8_t *, uint8_t);
    214 extern uchar_t atomic_swap_uchar(volatile uchar_t *, uchar_t);
    215 extern uint16_t atomic_swap_16(volatile uint16_t *, uint16_t);
    216 extern ushort_t atomic_swap_ushort(volatile ushort_t *, ushort_t);
    217 extern uint32_t atomic_swap_32(volatile uint32_t *, uint32_t);
    218 extern uint_t atomic_swap_uint(volatile uint_t *, uint_t);
    219 extern void *atomic_swap_ptr(volatile void *, void *);
    220 extern ulong_t atomic_swap_ulong(volatile ulong_t *, ulong_t);
    221 #if defined(_KERNEL) || defined(_INT64_TYPE)
    222 extern uint64_t atomic_swap_64(volatile uint64_t *, uint64_t);
    223 #endif
    224 
    225 /*
    226  * Perform an exclusive atomic bit set/clear on a target.
    227  * Returns 0 if bit was sucessfully set/cleared, or -1
    228  * if the bit was already set/cleared.
    229  */
    230 extern int atomic_set_long_excl(volatile ulong_t *, uint_t);
    231 extern int atomic_clear_long_excl(volatile ulong_t *, uint_t);
    232 
    233 /*
    234  * Generic memory barrier used during lock entry, placed after the
    235  * memory operation that acquires the lock to guarantee that the lock
    236  * protects its data.  No stores from after the memory barrier will
    237  * reach visibility, and no loads from after the barrier will be
    238  * resolved, before the lock acquisition reaches global visibility.
    239  */
    240 extern void membar_enter(void);
    241 
    242 /*
    243  * Generic memory barrier used during lock exit, placed before the
    244  * memory operation that releases the lock to guarantee that the lock
    245  * protects its data.  All loads and stores issued before the barrier
    246  * will be resolved before the subsequent lock update reaches visibility.
    247  */
    248 extern void membar_exit(void);
    249 
    250 /*
    251  * Arrange that all stores issued before this point in the code reach
    252  * global visibility before any stores that follow; useful in producer
    253  * modules that update a data item, then set a flag that it is available.
    254  * The memory barrier guarantees that the available flag is not visible
    255  * earlier than the updated data, i.e. it imposes store ordering.
    256  */
    257 extern void membar_producer(void);
    258 
    259 /*
    260  * Arrange that all loads issued before this point in the code are
    261  * completed before any subsequent loads; useful in consumer modules
    262  * that check to see if data is available and read the data.
    263  * The memory barrier guarantees that the data is not sampled until
    264  * after the available flag has been seen, i.e. it imposes load ordering.
    265  */
    266 extern void membar_consumer(void);
    267 #endif
    268 
    269 #if !defined(_KERNEL) && !defined(__STDC__)
    270 extern void atomic_inc_8();
    271 extern void atomic_inc_uchar();
    272 extern void atomic_inc_16();
    273 extern void atomic_inc_ushort();
    274 extern void atomic_inc_32();
    275 extern void atomic_inc_uint();
    276 extern void atomic_inc_ulong();
    277 #if defined(_INT64_TYPE)
    278 extern void atomic_inc_64();
    279 #endif /* defined(_INT64_TYPE) */
    280 extern void atomic_dec_8();
    281 extern void atomic_dec_uchar();
    282 extern void atomic_dec_16();
    283 extern void atomic_dec_ushort();
    284 extern void atomic_dec_32();
    285 extern void atomic_dec_uint();
    286 extern void atomic_dec_ulong();
    287 #if defined(_INT64_TYPE)
    288 extern void atomic_dec_64();
    289 #endif /* defined(_INT64_TYPE) */
    290 extern void atomic_add_8();
    291 extern void atomic_add_char();
    292 extern void atomic_add_16();
    293 extern void atomic_add_short();
    294 extern void atomic_add_32();
    295 extern void atomic_add_int();
    296 extern void atomic_add_ptr();
    297 extern void atomic_add_long();
    298 #if defined(_INT64_TYPE)
    299 extern void atomic_add_64();
    300 #endif /* defined(_INT64_TYPE) */
    301 extern void atomic_or_8();
    302 extern void atomic_or_uchar();
    303 extern void atomic_or_16();
    304 extern void atomic_or_ushort();
    305 extern void atomic_or_32();
    306 extern void atomic_or_uint();
    307 extern void atomic_or_ulong();
    308 #if defined(_INT64_TYPE)
    309 extern void atomic_or_64();
    310 #endif /* defined(_INT64_TYPE) */
    311 extern void atomic_and_8();
    312 extern void atomic_and_uchar();
    313 extern void atomic_and_16();
    314 extern void atomic_and_ushort();
    315 extern void atomic_and_32();
    316 extern void atomic_and_uint();
    317 extern void atomic_and_ulong();
    318 #if defined(_INT64_TYPE)
    319 extern void atomic_and_64();
    320 #endif /* defined(_INT64_TYPE) */
    321 extern uint8_t atomic_inc_8_nv();
    322 extern uchar_t atomic_inc_uchar_nv();
    323 extern uint16_t atomic_inc_16_nv();
    324 extern ushort_t atomic_inc_ushort_nv();
    325 extern uint32_t atomic_inc_32_nv();
    326 extern uint_t atomic_inc_uint_nv();
    327 extern ulong_t atomic_inc_ulong_nv();
    328 #if defined(_INT64_TYPE)
    329 extern uint64_t atomic_inc_64_nv();
    330 #endif /* defined(_INT64_TYPE) */
    331 extern uint8_t atomic_dec_8_nv();
    332 extern uchar_t atomic_dec_uchar_nv();
    333 extern uint16_t atomic_dec_16_nv();
    334 extern ushort_t atomic_dec_ushort_nv();
    335 extern uint32_t atomic_dec_32_nv();
    336 extern uint_t atomic_dec_uint_nv();
    337 extern ulong_t atomic_dec_ulong_nv();
    338 #if defined(_INT64_TYPE)
    339 extern uint64_t atomic_dec_64_nv();
    340 #endif /* defined(_INT64_TYPE) */
    341 extern uint8_t atomic_add_8_nv();
    342 extern uchar_t atomic_add_char_nv();
    343 extern uint16_t atomic_add_16_nv();
    344 extern ushort_t atomic_add_short_nv();
    345 extern uint32_t atomic_add_32_nv();
    346 extern uint_t atomic_add_int_nv();
    347 extern void *atomic_add_ptr_nv();
    348 extern ulong_t atomic_add_long_nv();
    349 #if defined(_INT64_TYPE)
    350 extern uint64_t atomic_add_64_nv();
    351 #endif /* defined(_INT64_TYPE) */
    352 extern uint8_t atomic_or_8_nv();
    353 extern uchar_t atomic_or_uchar_nv();
    354 extern uint16_t atomic_or_16_nv();
    355 extern ushort_t atomic_or_ushort_nv();
    356 extern uint32_t atomic_or_32_nv();
    357 extern uint_t atomic_or_uint_nv();
    358 extern ulong_t atomic_or_ulong_nv();
    359 #if defined(_INT64_TYPE)
    360 extern uint64_t atomic_or_64_nv();
    361 #endif /* defined(_INT64_TYPE) */
    362 extern uint8_t atomic_and_8_nv();
    363 extern uchar_t atomic_and_uchar_nv();
    364 extern uint16_t atomic_and_16_nv();
    365 extern ushort_t atomic_and_ushort_nv();
    366 extern uint32_t atomic_and_32_nv();
    367 extern uint_t atomic_and_uint_nv();
    368 extern ulong_t atomic_and_ulong_nv();
    369 #if defined(_INT64_TYPE)
    370 extern uint64_t atomic_and_64_nv();
    371 #endif /* defined(_INT64_TYPE) */
    372 extern uint8_t atomic_cas_8();
    373 extern uchar_t atomic_cas_uchar();
    374 extern uint16_t atomic_cas_16();
    375 extern ushort_t atomic_cas_ushort();
    376 extern uint32_t atomic_cas_32();
    377 extern uint_t atomic_cas_uint();
    378 extern void *atomic_cas_ptr();
    379 extern ulong_t atomic_cas_ulong();
    380 #if defined(_INT64_TYPE)
    381 extern uint64_t atomic_cas_64();
    382 #endif /* defined(_INT64_TYPE) */
    383 extern uint8_t atomic_swap_8();
    384 extern uchar_t atomic_swap_uchar();
    385 extern uint16_t atomic_swap_16();
    386 extern ushort_t atomic_swap_ushort();
    387 extern uint32_t atomic_swap_32();
    388 extern uint_t atomic_swap_uint();
    389 extern void *atomic_swap_ptr();
    390 extern ulong_t atomic_swap_ulong();
    391 #if defined(_INT64_TYPE)
    392 extern uint64_t atomic_swap_64();
    393 #endif /* defined(_INT64_TYPE) */
    394 
    395 
    396 extern int atomic_set_long_excl();
    397 extern int atomic_clear_long_excl();
    398 
    399 extern void membar_enter();
    400 extern void membar_exit();
    401 extern void membar_producer();
    402 extern void membar_consumer();
    403 
    404 #endif
    405 
    406 #if defined(_KERNEL)
    407 
    408 #if defined(_LP64) || defined(_ILP32)
    409 #define	atomic_add_ip		atomic_add_long
    410 #define	atomic_add_ip_nv	atomic_add_long_nv
    411 #define	casip			atomic_cas_ulong
    412 #endif
    413 
    414 #if defined(__sparc)
    415 extern uint8_t ldstub(uint8_t *);
    416 #endif
    417 
    418 /*
    419  * Legacy kernel interfaces; they will go away (eventually).
    420  */
    421 extern uint8_t cas8(uint8_t *, uint8_t, uint8_t);
    422 extern uint32_t cas32(uint32_t *, uint32_t, uint32_t);
    423 extern uint64_t cas64(uint64_t *, uint64_t, uint64_t);
    424 extern ulong_t caslong(ulong_t *, ulong_t, ulong_t);
    425 extern void *casptr(void *, void *, void *);
    426 extern void atomic_and_long(ulong_t *, ulong_t);
    427 extern void atomic_or_long(ulong_t *, ulong_t);
    428 #if defined(__sparc)
    429 extern uint32_t swapl(uint32_t *, uint32_t);
    430 #endif
    431 
    432 #endif	/* _KERNEL */
    433 
    434 #ifdef	__cplusplus
    435 }
    436 #endif
    437 
    438 #endif	/* _SYS_ATOMIC_H */
    439