/* * Copyright (C) 2011 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef BIONIC_ATOMIC_ARM_H #define BIONIC_ATOMIC_ARM_H #include /* Some of the harware instructions used below are not available in Thumb-1 * mode (they are if you build in ARM or Thumb-2 mode though). To solve this * problem, we're going to use the same technique than libatomics_ops, * which is to temporarily switch to ARM, do the operation, then switch * back to Thumb-1. * * This results in two 'bx' jumps, just like a normal function call, but * everything is kept inlined, avoids loading or computing the function's * address, and prevents a little I-cache trashing too. * * However, it is highly recommended to avoid compiling any C library source * file that use these functions in Thumb-1 mode. * * Define three helper macros to implement this: */ #if defined(__thumb__) && !defined(__thumb2__) # define __ATOMIC_SWITCH_TO_ARM \ "adr r3, 5f\n" \ "bx r3\n" \ ".align\n" \ ".arm\n" \ "5:\n" /* note: the leading \n below is intentional */ # define __ATOMIC_SWITCH_TO_THUMB \ "\n" \ "adr r3, 6f\n" \ "bx r3\n" \ ".thumb" \ "6:\n" # define __ATOMIC_CLOBBERS "r3" /* list of clobbered registers */ /* Warn the user that ARM mode should really be preferred! */ # warning Rebuilding this source file in ARM mode is highly recommended for performance!! #else # define __ATOMIC_SWITCH_TO_ARM /* nothing */ # define __ATOMIC_SWITCH_TO_THUMB /* nothing */ # define __ATOMIC_CLOBBERS /* nothing */ #endif /* Define a full memory barrier, this is only needed if we build the * platform for a multi-core device. For the record, using a 'dmb' * instruction on a Nexus One device can take up to 180 ns even if * it is completely un-necessary on this device. * * NOTE: This is where the platform and NDK headers atomic headers are * going to diverge. With the NDK, we don't know if the generated * code is going to run on a single or multi-core device, so we * need to be cautious. * * Fortunately, we can use the kernel helper function that is * mapped at address 0xffff0fa0 in all user process, and that * provides a device-specific barrier operation. * * I.e. on single-core devices, the helper immediately returns, * on multi-core devices, it uses "dmb" or any other means to * perform a full-memory barrier. * * There are three cases to consider for the platform: * * - multi-core ARMv7-A => use the 'dmb' hardware instruction * - multi-core ARMv6 => use the coprocessor * - single core ARMv5TE/6/7 => do not use any hardware barrier */ #if defined(ANDROID_SMP) && ANDROID_SMP == 1 /* Sanity check, multi-core is only supported starting from ARMv6 */ # if __ARM_ARCH__ < 6 # error ANDROID_SMP should not be set to 1 for an ARM architecture less than 6 # endif # ifdef __ARM_HAVE_DMB /* For ARMv7-A, we can use the 'dmb' instruction directly */ __ATOMIC_INLINE__ void __bionic_memory_barrier(void) { /* Note: we always build in ARM or Thumb-2 on ARMv7-A, so don't * bother with __ATOMIC_SWITCH_TO_ARM */ __asm__ __volatile__ ( "dmb" : : : "memory" ); } # else /* !__ARM_HAVE_DMB */ /* Otherwise, i.e. for multi-core ARMv6, we need to use the coprocessor, * which requires the use of a general-purpose register, which is slightly * less efficient. */ __ATOMIC_INLINE__ void __bionic_memory_barrier(void) { __asm__ __volatile__ ( __SWITCH_TO_ARM "mcr p15, 0, %0, c7, c10, 5" __SWITCH_TO_THUMB : : "r" (0) : __ATOMIC_CLOBBERS "memory"); } # endif /* !__ARM_HAVE_DMB */ #else /* !ANDROID_SMP */ __ATOMIC_INLINE__ void __bionic_memory_barrier(void) { /* A simple compiler barrier */ __asm__ __volatile__ ( "" : : : "memory" ); } #endif /* !ANDROID_SMP */ /* Compare-and-swap, without any explicit barriers. Note that this functions * returns 0 on success, and 1 on failure. The opposite convention is typically * used on other platforms. * * There are two cases to consider: * * - ARMv6+ => use LDREX/STREX instructions * - < ARMv6 => use kernel helper function mapped at 0xffff0fc0 * * LDREX/STREX are only available starting from ARMv6 */ #ifdef __ARM_HAVE_LDREX_STREX __ATOMIC_INLINE__ int __bionic_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr) { int32_t prev, status; do { __asm__ __volatile__ ( __ATOMIC_SWITCH_TO_ARM "ldrex %0, [%3]\n" "mov %1, #0\n" "teq %0, %4\n" #ifdef __thumb2__ "it eq\n" #endif "strexeq %1, %5, [%3]" __ATOMIC_SWITCH_TO_THUMB : "=&r" (prev), "=&r" (status), "+m"(*ptr) : "r" (ptr), "Ir" (old_value), "r" (new_value) : __ATOMIC_CLOBBERS "cc"); } while (__builtin_expect(status != 0, 0)); return prev != old_value; } # else /* !__ARM_HAVE_LDREX_STREX */ /* Use the handy kernel helper function mapped at 0xffff0fc0 */ typedef int (kernel_cmpxchg)(int32_t, int32_t, volatile int32_t *); __ATOMIC_INLINE__ int __kernel_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr) { /* Note: the kernel function returns 0 on success too */ return (*(kernel_cmpxchg *)0xffff0fc0)(old_value, new_value, ptr); } __ATOMIC_INLINE__ int __bionic_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr) { return __kernel_cmpxchg(old_value, new_value, ptr); } #endif /* !__ARM_HAVE_LDREX_STREX */ /* Swap operation, without any explicit barriers. * There are again two similar cases to consider: * * ARMv6+ => use LDREX/STREX * < ARMv6 => use SWP instead. */ #ifdef __ARM_HAVE_LDREX_STREX __ATOMIC_INLINE__ int32_t __bionic_swap(int32_t new_value, volatile int32_t* ptr) { int32_t prev, status; do { __asm__ __volatile__ ( __ATOMIC_SWITCH_TO_ARM "ldrex %0, [%3]\n" "strex %1, %4, [%3]" __ATOMIC_SWITCH_TO_THUMB : "=&r" (prev), "=&r" (status), "+m" (*ptr) : "r" (ptr), "r" (new_value) : __ATOMIC_CLOBBERS "cc"); } while (__builtin_expect(status != 0, 0)); return prev; } #else /* !__ARM_HAVE_LDREX_STREX */ __ATOMIC_INLINE__ int32_t __bionic_swap(int32_t new_value, volatile int32_t* ptr) { int32_t prev; /* NOTE: SWP is available in Thumb-1 too */ __asm__ __volatile__ ("swp %0, %2, [%3]" : "=&r" (prev), "+m" (*ptr) : "r" (new_value), "r" (ptr) : "cc"); return prev; } #endif /* !__ARM_HAVE_LDREX_STREX */ /* Atomic increment - without any barriers * This returns the old value */ #ifdef __ARM_HAVE_LDREX_STREX __ATOMIC_INLINE__ int32_t __bionic_atomic_inc(volatile int32_t* ptr) { int32_t prev, tmp, status; do { __asm__ __volatile__ ( __ATOMIC_SWITCH_TO_ARM "ldrex %0, [%4]\n" "add %1, %0, #1\n" "strex %2, %1, [%4]" __ATOMIC_SWITCH_TO_THUMB : "=&r" (prev), "=&r" (tmp), "=&r" (status), "+m"(*ptr) : "r" (ptr) : __ATOMIC_CLOBBERS "cc"); } while (__builtin_expect(status != 0, 0)); return prev; } #else __ATOMIC_INLINE__ int32_t __bionic_atomic_inc(volatile int32_t* ptr) { int32_t prev, status; do { prev = *ptr; status = __kernel_cmpxchg(prev, prev+1, ptr); } while (__builtin_expect(status != 0, 0)); return prev; } #endif /* Atomic decrement - without any barriers * This returns the old value. */ #ifdef __ARM_HAVE_LDREX_STREX __ATOMIC_INLINE__ int32_t __bionic_atomic_dec(volatile int32_t* ptr) { int32_t prev, tmp, status; do { __asm__ __volatile__ ( __ATOMIC_SWITCH_TO_ARM "ldrex %0, [%4]\n" "sub %1, %0, #1\n" "strex %2, %1, [%4]" __ATOMIC_SWITCH_TO_THUMB : "=&r" (prev), "=&r" (tmp), "=&r" (status), "+m"(*ptr) : "r" (ptr) : __ATOMIC_CLOBBERS "cc"); } while (__builtin_expect(status != 0, 0)); return prev; } #else __ATOMIC_INLINE__ int32_t __bionic_atomic_dec(volatile int32_t* ptr) { int32_t prev, status; do { prev = *ptr; status = __kernel_cmpxchg(prev, prev-1, ptr); } while (__builtin_expect(status != 0, 0)); return prev; } #endif #endif /* SYS_ATOMICS_ARM_H */