/* * Copyright IBM Corp. 1999, 2011 * * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>, */ #ifndef __ASM_CMPXCHG_H #define __ASM_CMPXCHG_H #include <linux/mmdebug.h> #include <linux/types.h> #include <linux/bug.h> extern void __xchg_called_with_bad_pointer(void); static inline unsigned long __xchg(unsigned long x, void *ptr, int size) { unsigned long addr, old; int shift; switch (size) { case 1: addr = (unsigned long) ptr; shift = (3 ^ (addr & 3)) << 3; addr ^= addr & 3; asm volatile( " l %0,%4\n" "0: lr 0,%0\n" " nr 0,%3\n" " or 0,%2\n" " cs %0,0,%4\n" " jl 0b\n" : "=&d" (old), "=Q" (*(int *) addr) : "d" ((x & 0xff) << shift), "d" (~(0xff << shift)), "Q" (*(int *) addr) : "memory", "cc", "0"); return old >> shift; case 2: addr = (unsigned long) ptr; shift = (2 ^ (addr & 2)) << 3; addr ^= addr & 2; asm volatile( " l %0,%4\n" "0: lr 0,%0\n" " nr 0,%3\n" " or 0,%2\n" " cs %0,0,%4\n" " jl 0b\n" : "=&d" (old), "=Q" (*(int *) addr) : "d" ((x & 0xffff) << shift), "d" (~(0xffff << shift)), "Q" (*(int *) addr) : "memory", "cc", "0"); return old >> shift; case 4: asm volatile( " l %0,%3\n" "0: cs %0,%2,%3\n" " jl 0b\n" : "=&d" (old), "=Q" (*(int *) ptr) : "d" (x), "Q" (*(int *) ptr) : "memory", "cc"); return old; #ifdef CONFIG_64BIT case 8: asm volatile( " lg %0,%3\n" "0: csg %0,%2,%3\n" " jl 0b\n" : "=&d" (old), "=m" (*(long *) ptr) : "d" (x), "Q" (*(long *) ptr) : "memory", "cc"); return old; #endif /* CONFIG_64BIT */ } __xchg_called_with_bad_pointer(); return x; } #define xchg(ptr, x) \ ({ \ __typeof__(*(ptr)) __ret; \ __ret = (__typeof__(*(ptr))) \ __xchg((unsigned long)(x), (void *)(ptr), sizeof(*(ptr)));\ __ret; \ }) /* * Atomic compare and exchange. Compare OLD with MEM, if identical, * store NEW in MEM. Return the initial value in MEM. Success is * indicated by comparing RETURN with OLD. */ #define __HAVE_ARCH_CMPXCHG extern void __cmpxchg_called_with_bad_pointer(void); static inline unsigned long __cmpxchg(void *ptr, unsigned long old, unsigned long new, int size) { unsigned long addr, prev, tmp; int shift; switch (size) { case 1: addr = (unsigned long) ptr; shift = (3 ^ (addr & 3)) << 3; addr ^= addr & 3; asm volatile( " l %0,%2\n" "0: nr %0,%5\n" " lr %1,%0\n" " or %0,%3\n" " or %1,%4\n" " cs %0,%1,%2\n" " jnl 1f\n" " xr %1,%0\n" " nr %1,%5\n" " jnz 0b\n" "1:" : "=&d" (prev), "=&d" (tmp), "+Q" (*(int *) addr) : "d" ((old & 0xff) << shift), "d" ((new & 0xff) << shift), "d" (~(0xff << shift)) : "memory", "cc"); return prev >> shift; case 2: addr = (unsigned long) ptr; shift = (2 ^ (addr & 2)) << 3; addr ^= addr & 2; asm volatile( " l %0,%2\n" "0: nr %0,%5\n" " lr %1,%0\n" " or %0,%3\n" " or %1,%4\n" " cs %0,%1,%2\n" " jnl 1f\n" " xr %1,%0\n" " nr %1,%5\n" " jnz 0b\n" "1:" : "=&d" (prev), "=&d" (tmp), "+Q" (*(int *) addr) : "d" ((old & 0xffff) << shift), "d" ((new & 0xffff) << shift), "d" (~(0xffff << shift)) : "memory", "cc"); return prev >> shift; case 4: asm volatile( " cs %0,%3,%1\n" : "=&d" (prev), "=Q" (*(int *) ptr) : "0" (old), "d" (new), "Q" (*(int *) ptr) : "memory", "cc"); return prev; #ifdef CONFIG_64BIT case 8: asm volatile( " csg %0,%3,%1\n" : "=&d" (prev), "=Q" (*(long *) ptr) : "0" (old), "d" (new), "Q" (*(long *) ptr) : "memory", "cc"); return prev; #endif /* CONFIG_64BIT */ } __cmpxchg_called_with_bad_pointer(); return old; } #define cmpxchg(ptr, o, n) \ ({ \ __typeof__(*(ptr)) __ret; \ __ret = (__typeof__(*(ptr))) \ __cmpxchg((ptr), (unsigned long)(o), (unsigned long)(n), \ sizeof(*(ptr))); \ __ret; \ }) #ifdef CONFIG_64BIT #define cmpxchg64(ptr, o, n) \ ({ \ cmpxchg((ptr), (o), (n)); \ }) #else /* CONFIG_64BIT */ static inline unsigned long long __cmpxchg64(void *ptr, unsigned long long old, unsigned long long new) { register_pair rp_old = {.pair = old}; register_pair rp_new = {.pair = new}; unsigned long long *ullptr = ptr; asm volatile( " cds %0,%2,%1" : "+d" (rp_old), "+Q" (*ullptr) : "d" (rp_new) : "memory", "cc"); return rp_old.pair; } #define cmpxchg64(ptr, o, n) \ ({ \ __typeof__(*(ptr)) __ret; \ __ret = (__typeof__(*(ptr))) \ __cmpxchg64((ptr), \ (unsigned long long)(o), \ (unsigned long long)(n)); \ __ret; \ }) #endif /* CONFIG_64BIT */ #define __cmpxchg_double_op(p1, p2, o1, o2, n1, n2, insn) \ ({ \ register __typeof__(*(p1)) __old1 asm("2") = (o1); \ register __typeof__(*(p2)) __old2 asm("3") = (o2); \ register __typeof__(*(p1)) __new1 asm("4") = (n1); \ register __typeof__(*(p2)) __new2 asm("5") = (n2); \ int cc; \ asm volatile( \ insn " %[old],%[new],%[ptr]\n" \ " ipm %[cc]\n" \ " srl %[cc],28" \ : [cc] "=d" (cc), [old] "+d" (__old1), "+d" (__old2) \ : [new] "d" (__new1), "d" (__new2), \ [ptr] "Q" (*(p1)), "Q" (*(p2)) \ : "memory", "cc"); \ !cc; \ }) #define __cmpxchg_double_4(p1, p2, o1, o2, n1, n2) \ __cmpxchg_double_op(p1, p2, o1, o2, n1, n2, "cds") #define __cmpxchg_double_8(p1, p2, o1, o2, n1, n2) \ __cmpxchg_double_op(p1, p2, o1, o2, n1, n2, "cdsg") extern void __cmpxchg_double_called_with_bad_pointer(void); #define __cmpxchg_double(p1, p2, o1, o2, n1, n2) \ ({ \ int __ret; \ switch (sizeof(*(p1))) { \ case 4: \ __ret = __cmpxchg_double_4(p1, p2, o1, o2, n1, n2); \ break; \ case 8: \ __ret = __cmpxchg_double_8(p1, p2, o1, o2, n1, n2); \ break; \ default: \ __cmpxchg_double_called_with_bad_pointer(); \ } \ __ret; \ }) #define cmpxchg_double(p1, p2, o1, o2, n1, n2) \ ({ \ __typeof__(p1) __p1 = (p1); \ __typeof__(p2) __p2 = (p2); \ int __ret; \ BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \ BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \ VM_BUG_ON((unsigned long)((__p1) + 1) != (unsigned long)(__p2));\ if (sizeof(long) == 4) \ __ret = __cmpxchg_double_4(__p1, __p2, o1, o2, n1, n2); \ else \ __ret = __cmpxchg_double_8(__p1, __p2, o1, o2, n1, n2); \ __ret; \ }) #define system_has_cmpxchg_double() 1 #include <asm-generic/cmpxchg-local.h> static inline unsigned long __cmpxchg_local(void *ptr, unsigned long old, unsigned long new, int size) { switch (size) { case 1: case 2: case 4: #ifdef CONFIG_64BIT case 8: #endif return __cmpxchg(ptr, old, new, size); default: return __cmpxchg_local_generic(ptr, old, new, size); } return old; } /* * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make * them available. */ #define cmpxchg_local(ptr, o, n) \ ({ \ __typeof__(*(ptr)) __ret; \ __ret = (__typeof__(*(ptr))) \ __cmpxchg_local((ptr), (unsigned long)(o), \ (unsigned long)(n), sizeof(*(ptr))); \ __ret; \ }) #define cmpxchg64_local(ptr, o, n) cmpxchg64((ptr), (o), (n)) #endif /* __ASM_CMPXCHG_H */