1 : /*
2 : * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
3 : *
4 : * @APPLE_LICENSE_HEADER_START@
5 : *
6 : * This file contains Original Code and/or Modifications of Original Code
7 : * as defined in and that are subject to the Apple Public Source License
8 : * Version 2.0 (the 'License'). You may not use this file except in
9 : * compliance with the License. Please obtain a copy of the License at
10 : * http://www.opensource.apple.com/apsl/ and read it before using this
11 : * file.
12 : *
13 : * The Original Code and all software distributed under the License are
14 : * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 : * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 : * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 : * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 : * Please see the License for the specific language governing rights and
19 : * limitations under the License.
20 : *
21 : * @APPLE_LICENSE_HEADER_END@
22 : */
23 :
24 : #ifndef _OSATOMIC_H_
25 : #define _OSATOMIC_H_
26 :
27 : #include <stddef.h>
28 : #include <sys/cdefs.h>
29 : #include <stdint.h>
30 : #include <stdbool.h>
31 :
32 : /* These are the preferred versions of the atomic and synchronization operations.
33 : * Their implementation is customized at boot time for the platform, including
34 : * late-breaking errata fixes as necessary. They are thread safe.
35 : *
36 : * WARNING: all addresses passed to these functions must be "naturally aligned", ie
37 : * int32_t's must be 32-bit aligned (low 2 bits of address zero), and int64_t's
38 : * must be 64-bit aligned (low 3 bits of address zero.)
39 : *
40 : * Note that some versions of the atomic functions incorporate memory barriers,
41 : * and some do not. Barriers strictly order memory access on a weakly-ordered
42 : * architecture such as PPC. All loads and stores executed in sequential program
43 : * order before the barrier will complete before any load or store executed after
44 : * the barrier. On a uniprocessor, the barrier operation is typically a nop.
45 : * On a multiprocessor, the barrier can be quite expensive on some platforms,
46 : * eg PPC.
47 : *
48 : * Most code will want to use the barrier functions to insure that memory shared
49 : * between threads is properly synchronized. For example, if you want to initialize
50 : * a shared data structure and then atomically increment a variable to indicate
51 : * that the initialization is complete, then you must use OSAtomicIncrement32Barrier()
52 : * to ensure that the stores to your data structure complete before the atomic add.
53 : * Likewise, the consumer of that data structure must use OSAtomicDecrement32Barrier(),
54 : * in order to ensure that their loads of the structure are not executed before
55 : * the atomic decrement. On the other hand, if you are simply incrementing a global
56 : * counter, then it is safe and potentially faster to use OSAtomicIncrement32().
57 : *
58 : * If you are unsure which version to use, prefer the barrier variants as they are
59 : * safer.
60 : *
61 : * The spinlock and queue operations always incorporate a barrier.
62 : */
63 : __BEGIN_DECLS
64 :
65 :
66 : /* Arithmetic functions. They return the new value.
67 : */
68 : int32_t OSAtomicAdd32( int32_t __theAmount, volatile int32_t *__theValue );
69 : int32_t OSAtomicAdd32Barrier( int32_t __theAmount, volatile int32_t *__theValue );
70 :
71 : __inline static
72 : int32_t OSAtomicIncrement32( volatile int32_t *__theValue )
73 0 : { return OSAtomicAdd32( 1, __theValue); }
74 : __inline static
75 : int32_t OSAtomicIncrement32Barrier( volatile int32_t *__theValue )
76 0 : { return OSAtomicAdd32Barrier( 1, __theValue); }
77 :
78 : __inline static
79 : int32_t OSAtomicDecrement32( volatile int32_t *__theValue )
80 0 : { return OSAtomicAdd32( -1, __theValue); }
81 : __inline static
82 : int32_t OSAtomicDecrement32Barrier( volatile int32_t *__theValue )
83 0 : { return OSAtomicAdd32Barrier( -1, __theValue); }
84 :
85 : #if defined(__ppc64__) || defined(__i386__) || defined(__x86_64__) || defined(__arm__)
86 :
87 : int64_t OSAtomicAdd64( int64_t __theAmount, volatile int64_t *__theValue );
88 : int64_t OSAtomicAdd64Barrier( int64_t __theAmount, volatile int64_t *__theValue );
89 :
90 : __inline static
91 : int64_t OSAtomicIncrement64( volatile int64_t *__theValue )
92 0 : { return OSAtomicAdd64( 1, __theValue); }
93 : __inline static
94 : int64_t OSAtomicIncrement64Barrier( volatile int64_t *__theValue )
95 0 : { return OSAtomicAdd64Barrier( 1, __theValue); }
96 :
97 : __inline static
98 : int64_t OSAtomicDecrement64( volatile int64_t *__theValue )
99 0 : { return OSAtomicAdd64( -1, __theValue); }
100 : __inline static
101 : int64_t OSAtomicDecrement64Barrier( volatile int64_t *__theValue )
102 0 : { return OSAtomicAdd64Barrier( -1, __theValue); }
103 :
104 : #endif /* defined(__ppc64__) || defined(__i386__) || defined(__x86_64__) || defined(__arm__) */
105 :
106 :
107 : /* Boolean functions (and, or, xor.) These come in four versions for each operation:
108 : * with and without barriers, and returning the old or new value of the operation.
109 : * The "Orig" versions return the original value, ie before the operation, the non-Orig
110 : * versions return the value after the operation. All are layered on top of
111 : * compare-and-swap.
112 : */
113 : int32_t OSAtomicOr32( uint32_t __theMask, volatile uint32_t *__theValue );
114 : int32_t OSAtomicOr32Barrier( uint32_t __theMask, volatile uint32_t *__theValue );
115 : int32_t OSAtomicOr32Orig( uint32_t __theMask, volatile uint32_t *__theValue );
116 : int32_t OSAtomicOr32OrigBarrier( uint32_t __theMask, volatile uint32_t *__theValue );
117 :
118 : int32_t OSAtomicAnd32( uint32_t __theMask, volatile uint32_t *__theValue );
119 : int32_t OSAtomicAnd32Barrier( uint32_t __theMask, volatile uint32_t *__theValue );
120 : int32_t OSAtomicAnd32Orig( uint32_t __theMask, volatile uint32_t *__theValue );
121 : int32_t OSAtomicAnd32OrigBarrier( uint32_t __theMask, volatile uint32_t *__theValue );
122 :
123 : int32_t OSAtomicXor32( uint32_t __theMask, volatile uint32_t *__theValue );
124 : int32_t OSAtomicXor32Barrier( uint32_t __theMask, volatile uint32_t *__theValue );
125 : int32_t OSAtomicXor32Orig( uint32_t __theMask, volatile uint32_t *__theValue );
126 : int32_t OSAtomicXor32OrigBarrier( uint32_t __theMask, volatile uint32_t *__theValue );
127 :
128 :
129 : /* Compare and swap. They return true if the swap occured. There are several versions,
130 : * depending on data type and whether or not a barrier is used.
131 : */
132 : bool OSAtomicCompareAndSwap32( int32_t __oldValue, int32_t __newValue, volatile int32_t *__theValue );
133 : bool OSAtomicCompareAndSwap32Barrier( int32_t __oldValue, int32_t __newValue, volatile int32_t *__theValue );
134 : bool OSAtomicCompareAndSwapPtr( void *__oldValue, void *__newValue, void * volatile *__theValue );
135 : bool OSAtomicCompareAndSwapPtrBarrier( void *__oldValue, void *__newValue, void * volatile *__theValue );
136 : bool OSAtomicCompareAndSwapInt( int __oldValue, int __newValue, volatile int *__theValue );
137 : bool OSAtomicCompareAndSwapIntBarrier( int __oldValue, int __newValue, volatile int *__theValue );
138 : bool OSAtomicCompareAndSwapLong( long __oldValue, long __newValue, volatile long *__theValue );
139 : bool OSAtomicCompareAndSwapLongBarrier( long __oldValue, long __newValue, volatile long *__theValue );
140 :
141 : #if defined(__ppc64__) || defined(__i386__) || defined(__x86_64__) || defined(__arm__)
142 :
143 : bool OSAtomicCompareAndSwap64( int64_t __oldValue, int64_t __newValue, volatile int64_t *__theValue );
144 : bool OSAtomicCompareAndSwap64Barrier( int64_t __oldValue, int64_t __newValue, volatile int64_t *__theValue );
145 :
146 : #endif /* defined(__ppc64__) || defined(__i386__) || defined(__x86_64__) || defined(__arm__) */
147 :
148 :
149 : /* Test and set. They return the original value of the bit, and operate on bit (0x80>>(n&7))
150 : * in byte ((char*)theAddress + (n>>3)).
151 : */
152 : bool OSAtomicTestAndSet( uint32_t __n, volatile void *__theAddress );
153 : bool OSAtomicTestAndSetBarrier( uint32_t __n, volatile void *__theAddress );
154 : bool OSAtomicTestAndClear( uint32_t __n, volatile void *__theAddress );
155 : bool OSAtomicTestAndClearBarrier( uint32_t __n, volatile void *__theAddress );
156 :
157 :
158 : /* Spinlocks. These use memory barriers as required to synchronize access to shared
159 : * memory protected by the lock. The lock operation spins, but employs various strategies
160 : * to back off if the lock is held, making it immune to most priority-inversion livelocks.
161 : * The try operation immediately returns false if the lock was held, true if it took the
162 : * lock. The convention is that unlocked is zero, locked is nonzero.
163 : */
164 : #define OS_SPINLOCK_INIT 0
165 :
166 : typedef int32_t OSSpinLock;
167 :
168 : bool OSSpinLockTry( volatile OSSpinLock *__lock );
169 : void OSSpinLockLock( volatile OSSpinLock *__lock );
170 : void OSSpinLockUnlock( volatile OSSpinLock *__lock );
171 :
172 :
173 : /* Lockless atomic enqueue and dequeue. These routines manipulate singly
174 : * linked LIFO lists. Ie, a dequeue will return the most recently enqueued
175 : * element, or NULL if the list is empty. The "offset" parameter is the offset
176 : * in bytes of the link field within the data structure being queued. The
177 : * link field should be a pointer type. Memory barriers are incorporated as
178 : * needed to permit thread-safe access to the queue element.
179 : */
180 : #if defined(__x86_64__)
181 :
182 : typedef volatile struct {
183 : void *opaque1;
184 : long opaque2;
185 : } OSQueueHead __attribute__ ((aligned (16)));
186 :
187 : #else
188 :
189 : typedef volatile struct {
190 : void *opaque1;
191 : long opaque2;
192 : } OSQueueHead;
193 :
194 : #endif
195 :
196 : #define OS_ATOMIC_QUEUE_INIT { NULL, 0 }
197 :
198 : void OSAtomicEnqueue( OSQueueHead *__list, void *__new, size_t __offset);
199 : void* OSAtomicDequeue( OSQueueHead *__list, size_t __offset);
200 :
201 :
202 : /* Memory barrier. It is both a read and write barrier.
203 : */
204 : void OSMemoryBarrier( void );
205 :
206 :
207 : __END_DECLS
208 :
209 : #endif /* _OSATOMIC_H_ */
|
