4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License"). You may not use this file except in compliance
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 2009 by Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
32 * All operations are implemented by serializing them through a global
33 * pthread mutex. This provides a correct generic implementation.
34 * However all supported architectures are encouraged to provide a
35 * native implementation is assembly for performance reasons.
37 pthread_mutex_t atomic_lock = PTHREAD_MUTEX_INITIALIZER;
40 * Theses are the void returning variants
43 #define ATOMIC_INC(name, type) \
44 void atomic_inc_##name(volatile type *target) \
46 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
48 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
51 ATOMIC_INC(long, unsigned long)
52 ATOMIC_INC(8, uint8_t)
53 ATOMIC_INC(uchar, uchar_t)
54 ATOMIC_INC(16, uint16_t)
55 ATOMIC_INC(ushort, ushort_t)
56 ATOMIC_INC(32, uint32_t)
57 ATOMIC_INC(uint, uint_t)
58 ATOMIC_INC(ulong, ulong_t)
59 ATOMIC_INC(64, uint64_t)
62 #define ATOMIC_DEC(name, type) \
63 void atomic_dec_##name(volatile type *target) \
65 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
67 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
70 ATOMIC_DEC(long, unsigned long)
71 ATOMIC_DEC(8, uint8_t)
72 ATOMIC_DEC(uchar, uchar_t)
73 ATOMIC_DEC(16, uint16_t)
74 ATOMIC_DEC(ushort, ushort_t)
75 ATOMIC_DEC(32, uint32_t)
76 ATOMIC_DEC(uint, uint_t)
77 ATOMIC_DEC(ulong, ulong_t)
78 ATOMIC_DEC(64, uint64_t)
81 #define ATOMIC_ADD(name, type1, type2) \
82 void atomic_add_##name(volatile type1 *target, type2 bits) \
84 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
86 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
89 ATOMIC_ADD(8, uint8_t, int8_t)
90 ATOMIC_ADD(char, uchar_t, signed char)
91 ATOMIC_ADD(16, uint16_t, int16_t)
92 ATOMIC_ADD(short, ushort_t, short)
93 ATOMIC_ADD(32, uint32_t, int32_t)
94 ATOMIC_ADD(int, uint_t, int)
95 ATOMIC_ADD(long, ulong_t, long)
96 ATOMIC_ADD(64, uint64_t, int64_t)
98 void atomic_add_ptr(volatile void *target, ssize_t bits)
100 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
101 *(caddr_t *)target += bits;
102 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
106 #define ATOMIC_SUB(name, type1, type2) \
107 void atomic_sub_##name(volatile type1 *target, type2 bits) \
109 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
111 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
114 ATOMIC_SUB(8, uint8_t, int8_t)
115 ATOMIC_SUB(char, uchar_t, signed char)
116 ATOMIC_SUB(16, uint16_t, int16_t)
117 ATOMIC_SUB(short, ushort_t, short)
118 ATOMIC_SUB(32, uint32_t, int32_t)
119 ATOMIC_SUB(int, uint_t, int)
120 ATOMIC_SUB(long, ulong_t, long)
121 ATOMIC_SUB(64, uint64_t, int64_t)
123 void atomic_sub_ptr(volatile void *target, ssize_t bits)
125 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
126 *(caddr_t *)target -= bits;
127 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
131 #define ATOMIC_OR(name, type) \
132 void atomic_or_##name(volatile type *target, type bits) \
134 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
136 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
139 ATOMIC_OR(8, uint8_t)
140 ATOMIC_OR(uchar, uchar_t)
141 ATOMIC_OR(16, uint16_t)
142 ATOMIC_OR(ushort, ushort_t)
143 ATOMIC_OR(32, uint32_t)
144 ATOMIC_OR(uint, uint_t)
145 ATOMIC_OR(ulong, ulong_t)
146 ATOMIC_OR(64, uint64_t)
149 #define ATOMIC_AND(name, type) \
150 void atomic_and_##name(volatile type *target, type bits) \
152 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
154 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
157 ATOMIC_AND(8, uint8_t)
158 ATOMIC_AND(uchar, uchar_t)
159 ATOMIC_AND(16, uint16_t)
160 ATOMIC_AND(ushort, ushort_t)
161 ATOMIC_AND(32, uint32_t)
162 ATOMIC_AND(uint, uint_t)
163 ATOMIC_AND(ulong, ulong_t)
164 ATOMIC_AND(64, uint64_t)
168 * New value returning variants
171 #define ATOMIC_INC_NV(name, type) \
172 type atomic_inc_##name##_nv(volatile type *target) \
175 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
176 rc = (++(*target)); \
177 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
181 ATOMIC_INC_NV(long, unsigned long)
182 ATOMIC_INC_NV(8, uint8_t)
183 ATOMIC_INC_NV(uchar, uchar_t)
184 ATOMIC_INC_NV(16, uint16_t)
185 ATOMIC_INC_NV(ushort, ushort_t)
186 ATOMIC_INC_NV(32, uint32_t)
187 ATOMIC_INC_NV(uint, uint_t)
188 ATOMIC_INC_NV(ulong, ulong_t)
189 ATOMIC_INC_NV(64, uint64_t)
192 #define ATOMIC_DEC_NV(name, type) \
193 type atomic_dec_##name##_nv(volatile type *target) \
196 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
197 rc = (--(*target)); \
198 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
202 ATOMIC_DEC_NV(long, unsigned long)
203 ATOMIC_DEC_NV(8, uint8_t)
204 ATOMIC_DEC_NV(uchar, uchar_t)
205 ATOMIC_DEC_NV(16, uint16_t)
206 ATOMIC_DEC_NV(ushort, ushort_t)
207 ATOMIC_DEC_NV(32, uint32_t)
208 ATOMIC_DEC_NV(uint, uint_t)
209 ATOMIC_DEC_NV(ulong, ulong_t)
210 ATOMIC_DEC_NV(64, uint64_t)
213 #define ATOMIC_ADD_NV(name, type1, type2) \
214 type1 atomic_add_##name##_nv(volatile type1 *target, type2 bits)\
217 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
218 rc = (*target += bits); \
219 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
223 ATOMIC_ADD_NV(8, uint8_t, int8_t)
224 ATOMIC_ADD_NV(char, uchar_t, signed char)
225 ATOMIC_ADD_NV(16, uint16_t, int16_t)
226 ATOMIC_ADD_NV(short, ushort_t, short)
227 ATOMIC_ADD_NV(32, uint32_t, int32_t)
228 ATOMIC_ADD_NV(int, uint_t, int)
229 ATOMIC_ADD_NV(long, ulong_t, long)
230 ATOMIC_ADD_NV(64, uint64_t, int64_t)
232 void *atomic_add_ptr_nv(volatile void *target, ssize_t bits)
236 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
237 ptr = (*(caddr_t *)target += bits);
238 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
244 #define ATOMIC_SUB_NV(name, type1, type2) \
245 type1 atomic_sub_##name##_nv(volatile type1 *target, type2 bits)\
248 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
249 rc = (*target -= bits); \
250 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
254 ATOMIC_SUB_NV(8, uint8_t, int8_t)
255 ATOMIC_SUB_NV(char, uchar_t, signed char)
256 ATOMIC_SUB_NV(16, uint16_t, int16_t)
257 ATOMIC_SUB_NV(short, ushort_t, short)
258 ATOMIC_SUB_NV(32, uint32_t, int32_t)
259 ATOMIC_SUB_NV(int, uint_t, int)
260 ATOMIC_SUB_NV(long, ulong_t, long)
261 ATOMIC_SUB_NV(64, uint64_t, int64_t)
263 void *atomic_sub_ptr_nv(volatile void *target, ssize_t bits)
267 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
268 ptr = (*(caddr_t *)target -= bits);
269 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
275 #define ATOMIC_OR_NV(name, type) \
276 type atomic_or_##name##_nv(volatile type *target, type bits) \
279 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
280 rc = (*target |= bits); \
281 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
285 ATOMIC_OR_NV(long, unsigned long)
286 ATOMIC_OR_NV(8, uint8_t)
287 ATOMIC_OR_NV(uchar, uchar_t)
288 ATOMIC_OR_NV(16, uint16_t)
289 ATOMIC_OR_NV(ushort, ushort_t)
290 ATOMIC_OR_NV(32, uint32_t)
291 ATOMIC_OR_NV(uint, uint_t)
292 ATOMIC_OR_NV(ulong, ulong_t)
293 ATOMIC_OR_NV(64, uint64_t)
296 #define ATOMIC_AND_NV(name, type) \
297 type atomic_and_##name##_nv(volatile type *target, type bits) \
300 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
301 rc = (*target &= bits); \
302 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
306 ATOMIC_AND_NV(long, unsigned long)
307 ATOMIC_AND_NV(8, uint8_t)
308 ATOMIC_AND_NV(uchar, uchar_t)
309 ATOMIC_AND_NV(16, uint16_t)
310 ATOMIC_AND_NV(ushort, ushort_t)
311 ATOMIC_AND_NV(32, uint32_t)
312 ATOMIC_AND_NV(uint, uint_t)
313 ATOMIC_AND_NV(ulong, ulong_t)
314 ATOMIC_AND_NV(64, uint64_t)
318 * If *arg1 == arg2, set *arg1 = arg3; return old value
321 #define ATOMIC_CAS(name, type) \
322 type atomic_cas_##name(volatile type *target, type arg1, type arg2) \
325 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
329 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
333 ATOMIC_CAS(8, uint8_t)
334 ATOMIC_CAS(uchar, uchar_t)
335 ATOMIC_CAS(16, uint16_t)
336 ATOMIC_CAS(ushort, ushort_t)
337 ATOMIC_CAS(32, uint32_t)
338 ATOMIC_CAS(uint, uint_t)
339 ATOMIC_CAS(ulong, ulong_t)
340 ATOMIC_CAS(64, uint64_t)
342 void *atomic_cas_ptr(volatile void *target, void *arg1, void *arg2)
346 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
347 old = *(void **)target;
349 *(void **)target = arg2;
350 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
357 * Swap target and return old value
360 #define ATOMIC_SWAP(name, type) \
361 type atomic_swap_##name(volatile type *target, type bits) \
364 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
367 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
371 ATOMIC_SWAP(8, uint8_t)
372 ATOMIC_SWAP(uchar, uchar_t)
373 ATOMIC_SWAP(16, uint16_t)
374 ATOMIC_SWAP(ushort, ushort_t)
375 ATOMIC_SWAP(32, uint32_t)
376 ATOMIC_SWAP(uint, uint_t)
377 ATOMIC_SWAP(ulong, ulong_t)
378 ATOMIC_SWAP(64, uint64_t)
380 void *atomic_swap_ptr(volatile void *target, void *bits)
384 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
385 old = *(void **)target;
386 *(void **)target = bits;
387 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
393 int atomic_set_long_excl(volatile ulong_t *target, uint_t value)
397 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
398 bit = (1UL << value);
399 if ((*target & bit) != 0) {
400 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
404 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
409 int atomic_clear_long_excl(volatile ulong_t *target, uint_t value)
413 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
414 bit = (1UL << value);
415 if ((*target & bit) != 0) {
416 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
420 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
425 void membar_enter(void)
427 /* XXX - Implement me */
430 void membar_exit(void)
432 /* XXX - Implement me */
435 void membar_producer(void)
437 /* XXX - Implement me */
440 void membar_consumer(void)
442 /* XXX - Implement me */
445 /* Legacy kernel interfaces; they will go away (eventually). */
447 uint8_t cas8(uint8_t *target, uint8_t arg1, uint8_t arg2)
449 return atomic_cas_8(target, arg1, arg2);
452 uint32_t cas32(uint32_t *target, uint32_t arg1, uint32_t arg2)
454 return atomic_cas_32(target, arg1, arg2);
457 uint64_t cas64(uint64_t *target, uint64_t arg1, uint64_t arg2)
459 return atomic_cas_64(target, arg1, arg2);
462 ulong_t caslong(ulong_t *target, ulong_t arg1, ulong_t arg2)
464 return atomic_cas_ulong(target, arg1, arg2);
467 void *casptr(void *target, void *arg1, void *arg2)
469 return atomic_cas_ptr(target, arg1, arg2);
472 void atomic_and_long(ulong_t *target, ulong_t bits)
474 return atomic_and_ulong(target, bits);
477 void atomic_or_long(ulong_t *target, ulong_t bits)
479 return atomic_or_ulong(target, bits);