diff options
Diffstat (limited to 'src/sync/atomic/asm_linux_arm.s')
| -rw-r--r-- | src/sync/atomic/asm_linux_arm.s | 216 |
1 files changed, 216 insertions, 0 deletions
diff --git a/src/sync/atomic/asm_linux_arm.s b/src/sync/atomic/asm_linux_arm.s new file mode 100644 index 000000000..944758441 --- /dev/null +++ b/src/sync/atomic/asm_linux_arm.s @@ -0,0 +1,216 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !race + +#include "textflag.h" + +// Linux/ARM atomic operations. + +// Because there is so much variation in ARM devices, +// the Linux kernel provides an appropriate compare-and-swap +// implementation at address 0xffff0fc0. Caller sets: +// R0 = old value +// R1 = new value +// R2 = addr +// LR = return address +// The function returns with CS true if the swap happened. +// http://lxr.linux.no/linux+v2.6.37.2/arch/arm/kernel/entry-armv.S#L850 +// On older kernels (before 2.6.24) the function can incorrectly +// report a conflict, so we have to double-check the compare ourselves +// and retry if necessary. +// +// http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git;a=commit;h=b49c0f24cf6744a3f4fd09289fe7cade349dead5 +// +TEXT cas<>(SB),NOSPLIT,$0 + MOVW $0xffff0fc0, PC + +TEXT ·CompareAndSwapInt32(SB),NOSPLIT,$0 + B ·CompareAndSwapUint32(SB) + +// Implement using kernel cas for portability. +TEXT ·CompareAndSwapUint32(SB),NOSPLIT,$0-13 + MOVW addr+0(FP), R2 + // trigger potential paging fault here, + // because we don't know how to traceback through __kuser_cmpxchg + MOVW (R2), R0 + MOVW old+4(FP), R0 +casagain: + MOVW new+8(FP), R1 + BL cas<>(SB) + BCC cascheck + MOVW $1, R0 +casret: + MOVB R0, swapped+12(FP) + RET +cascheck: + // Kernel lies; double-check. + MOVW addr+0(FP), R2 + MOVW old+4(FP), R0 + MOVW 0(R2), R3 + CMP R0, R3 + BEQ casagain + MOVW $0, R0 + B casret + +TEXT ·CompareAndSwapUintptr(SB),NOSPLIT,$0 + B ·CompareAndSwapUint32(SB) + +TEXT ·CompareAndSwapPointer(SB),NOSPLIT,$0 + B ·CompareAndSwapUint32(SB) + +TEXT ·AddInt32(SB),NOSPLIT,$0 + B ·AddUint32(SB) + +// Implement using kernel cas for portability. +TEXT ·AddUint32(SB),NOSPLIT,$0-12 + MOVW addr+0(FP), R2 + MOVW delta+4(FP), R4 +addloop1: + MOVW 0(R2), R0 + MOVW R0, R1 + ADD R4, R1 + BL cas<>(SB) + BCC addloop1 + MOVW R1, new+8(FP) + RET + +TEXT ·AddUintptr(SB),NOSPLIT,$0 + B ·AddUint32(SB) + +TEXT ·SwapInt32(SB),NOSPLIT,$0 + B ·SwapUint32(SB) + +// Implement using kernel cas for portability. +TEXT ·SwapUint32(SB),NOSPLIT,$0-12 + MOVW addr+0(FP), R2 + MOVW new+4(FP), R1 +swaploop1: + MOVW 0(R2), R0 + MOVW R0, R4 // cas smashes R0 + BL cas<>(SB) + BCC swaploop1 + MOVW R4, old+8(FP) + RET + +TEXT ·SwapUintptr(SB),NOSPLIT,$0 + B ·SwapUint32(SB) + +TEXT ·SwapPointer(SB),NOSPLIT,$0 + B ·SwapUint32(SB) + +TEXT cas64<>(SB),NOSPLIT,$0 + MOVW $0xffff0f60, PC // __kuser_cmpxchg64: Linux-3.1 and above + +TEXT kernelCAS64<>(SB),NOSPLIT,$0-21 + // int (*__kuser_cmpxchg64_t)(const int64_t *oldval, const int64_t *newval, volatile int64_t *ptr); + MOVW addr+0(FP), R2 // ptr + // trigger potential paging fault here, + // because we don't know how to traceback through __kuser_cmpxchg64 + MOVW (R2), R0 + // make unaligned atomic access panic + AND.S $7, R2, R1 + BEQ 2(PC) + MOVW R1, (R1) + MOVW $4(FP), R0 // oldval + MOVW $12(FP), R1 // newval + BL cas64<>(SB) + MOVW.CS $1, R0 // C is set if the kernel has changed *ptr + MOVW.CC $0, R0 + MOVW R0, 20(FP) + RET + +TEXT ·generalCAS64(SB),NOSPLIT,$0-21 + B runtime·cas64(SB) + +GLOBL armCAS64(SB), NOPTR, $4 + +TEXT setupAndCallCAS64<>(SB),NOSPLIT,$-4-21 + MOVW $0xffff0ffc, R0 // __kuser_helper_version + MOVW (R0), R0 + // __kuser_cmpxchg64 only present if helper version >= 5 + CMP $5, R0 + MOVW.CS $kernelCAS64<>(SB), R1 + MOVW.CS R1, armCAS64(SB) + MOVW.CS R1, PC + MOVB runtime·armArch(SB), R0 + // LDREXD, STREXD only present on ARMv6K or higher + CMP $6, R0 // TODO(minux): how to differentiate ARMv6 with ARMv6K? + MOVW.CS $·armCompareAndSwapUint64(SB), R1 + MOVW.CS R1, armCAS64(SB) + MOVW.CS R1, PC + // we are out of luck, can only use runtime's emulated 64-bit cas + MOVW $·generalCAS64(SB), R1 + MOVW R1, armCAS64(SB) + MOVW R1, PC + +TEXT ·CompareAndSwapInt64(SB),NOSPLIT,$0 + B ·CompareAndSwapUint64(SB) + +TEXT ·CompareAndSwapUint64(SB),NOSPLIT,$-4-21 + MOVW armCAS64(SB), R0 + CMP $0, R0 + MOVW.NE R0, PC + B setupAndCallCAS64<>(SB) + +TEXT ·AddInt64(SB),NOSPLIT,$0 + B ·addUint64(SB) + +TEXT ·AddUint64(SB),NOSPLIT,$0 + B ·addUint64(SB) + +TEXT ·SwapInt64(SB),NOSPLIT,$0 + B ·swapUint64(SB) + +TEXT ·SwapUint64(SB),NOSPLIT,$0 + B ·swapUint64(SB) + +TEXT ·LoadInt32(SB),NOSPLIT,$0 + B ·LoadUint32(SB) + +TEXT ·LoadUint32(SB),NOSPLIT,$0-8 + MOVW addr+0(FP), R2 +loadloop1: + MOVW 0(R2), R0 + MOVW R0, R1 + BL cas<>(SB) + BCC loadloop1 + MOVW R1, val+4(FP) + RET + +TEXT ·LoadInt64(SB),NOSPLIT,$0 + B ·loadUint64(SB) + +TEXT ·LoadUint64(SB),NOSPLIT,$0 + B ·loadUint64(SB) + +TEXT ·LoadUintptr(SB),NOSPLIT,$0 + B ·LoadUint32(SB) + +TEXT ·LoadPointer(SB),NOSPLIT,$0 + B ·LoadUint32(SB) + +TEXT ·StoreInt32(SB),NOSPLIT,$0 + B ·StoreUint32(SB) + +TEXT ·StoreUint32(SB),NOSPLIT,$0-8 + MOVW addr+0(FP), R2 + MOVW val+4(FP), R1 +storeloop1: + MOVW 0(R2), R0 + BL cas<>(SB) + BCC storeloop1 + RET + +TEXT ·StoreInt64(SB),NOSPLIT,$0 + B ·storeUint64(SB) + +TEXT ·StoreUint64(SB),NOSPLIT,$0 + B ·storeUint64(SB) + +TEXT ·StoreUintptr(SB),NOSPLIT,$0 + B ·StoreUint32(SB) + +TEXT ·StorePointer(SB),NOSPLIT,$0 + B ·StoreUint32(SB) |