diff --git a/base_c/include/embb/base/c/internal/atomic/and_assign.h b/base_c/include/embb/base/c/internal/atomic/and_assign.h index bf3ed84..b156b1f 100644 --- a/base_c/include/embb/base/c/internal/atomic/and_assign.h +++ b/base_c/include/embb/base/c/internal/atomic/and_assign.h @@ -131,14 +131,21 @@ EMBB_DEFINE_AND_ASSIGN(4, "") #define EMBB_DEFINE_AND_ASSIGN(EMBB_PARAMETER_SIZE_BYTE, EMBB_ATOMIC_TC_SIZE_SUFFIX)\ EMBB_PLATFORM_INLINE void EMBB_CAT2(embb_internal__atomic_and_assign_, \ EMBB_PARAMETER_SIZE_BYTE)(EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) volatile* pointer_to_value, EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) value) { \ - EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) result=0; \ - __asm__ __volatile__("dsync\n\t" \ - "ld" EMBB_ATOMIC_TC_SIZE_SUFFIX " %[result], %[pointer_to_value]\n\t" \ - "and %[result], %[value]\n\t" \ - "st" EMBB_ATOMIC_TC_SIZE_SUFFIX " %[pointer_to_value], %[result]\n\t" \ - : [pointer_to_value]"+m" (*pointer_to_value), [result]"+d" (result) \ - : [value]"d" (value)\ - : "memory"); \ + register uint64_t e_reg = sizeof(value) == sizeof(uint32_t) ? (uint64_t)value : 0xFFFFFFFFull; \ + register uint32_t add = ((uint32_t)pointer_to_value) & 0xFFFFFFFC; \ + uint32_t pos = ((uint32_t)pointer_to_value & 0x3) * 8; \ + uint32_t width = sizeof(value) == sizeof(uint32_t) ? 31 : sizeof(value) * 8; \ + register uint64_t e_ins = ((uint64_t)width << 32) | pos; \ + __asm__ __volatile__ ("dsync\n\t" \ + "mov.a %[pointer_to_value], %[add]\n\t" \ + "loop_:\tld.w %H0, [%2]0\n\t" \ + "insert %L0, %H0, %[value], %A1\n\t" \ + "and %L0, %H0\n\t" \ + "cmpswap.w [%2]0, %A0\n\t" \ + "jne %H0, %L0, loop_" \ + : "+d"(e_reg), [e_ins]"+d"(e_ins) \ + : [pointer_to_value]"a"(pointer_to_value), [value] "r" (value), [add] "d" (add) \ + : "memory"); \ } #else #error "No atomic fetch and store implementation found" @@ -146,7 +153,7 @@ EMBB_DEFINE_AND_ASSIGN(4, "") EMBB_DEFINE_AND_ASSIGN(1, ".b") EMBB_DEFINE_AND_ASSIGN(2, ".h") -EMBB_DEFINE_AND_ASSIGN(4, ".w") +EMBB_DEFINE_AND_ASSIGN(4, "") #else #error "Unknown architecture" diff --git a/base_c/include/embb/base/c/internal/atomic/compare_and_swap.h b/base_c/include/embb/base/c/internal/atomic/compare_and_swap.h index 1da58ba..447e50d 100644 --- a/base_c/include/embb/base/c/internal/atomic/compare_and_swap.h +++ b/base_c/include/embb/base/c/internal/atomic/compare_and_swap.h @@ -132,25 +132,25 @@ EMBB_DEFINE_COMPARE_AND_SWAP(4, "") EMBB_PLATFORM_INLINE int EMBB_CAT2(embb_internal__atomic_compare_and_swap_, \ EMBB_PARAMETER_SIZE_BYTE)(EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) volatile* pointer_to_value, EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) volatile* expected, \ EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) desired) { \ - register uint64_t e_reg = ((uint64_t)*expected << 32U) | desired; \ - __asm__ __volatile__ ("cmpswap" EMBB_ATOMIC_TC_SIZE_SUFFIX " [%1]0, %A0\n\t" \ - : "+d"(e_reg) \ - : "a"(pointer_to_value) \ + EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) result; \ + register uint64_t e_reg = desired; \ + __asm__ __volatile__ ("dsync\n\t" \ + "ld.w %H0, [%3]0\n\t" \ + "cmpswap.w [%2]0, %A0\n\t" \ + "eq %[result], %H0, %L0" \ + : "+d"(e_reg), [result]"=&d" (result) \ + : "a"(pointer_to_value), "a"(expected) \ : "memory"); \ - __asm__ __volatile__ ("dsync" : : : "memory"); \ - if(*pointer_to_value == desired) { \ - return 1; \ - } else { \ - return 0; \ - } \ + return result; \ } #else #error "No atomic fetch and store implementation found" #endif +//TODO: CAS-byte and CAS-short not needed yet, implement if needed EMBB_DEFINE_COMPARE_AND_SWAP(1, ".b") EMBB_DEFINE_COMPARE_AND_SWAP(2, ".h") -EMBB_DEFINE_COMPARE_AND_SWAP(4, ".w") +EMBB_DEFINE_COMPARE_AND_SWAP(4, "") #else #error "Unknown architecture" diff --git a/base_c/include/embb/base/c/internal/atomic/fetch_and_add.h b/base_c/include/embb/base/c/internal/atomic/fetch_and_add.h index 5bab66e..56b75e7 100644 --- a/base_c/include/embb/base/c/internal/atomic/fetch_and_add.h +++ b/base_c/include/embb/base/c/internal/atomic/fetch_and_add.h @@ -123,16 +123,24 @@ EMBB_DEFINE_FETCH_AND_ADD(4, "") #define EMBB_DEFINE_FETCH_AND_ADD(EMBB_PARAMETER_SIZE_BYTE, EMBB_ATOMIC_TC_SIZE_SUFFIX) \ EMBB_PLATFORM_INLINE EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) EMBB_CAT2(embb_internal__atomic_fetch_and_add_, EMBB_PARAMETER_SIZE_BYTE) \ (EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) volatile* pointer_to_value, EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) new_value) { \ - EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) \ - result=0,tmp1=0; \ - __asm__ __volatile__ ("dsync\n\t" \ - "ld" EMBB_ATOMIC_TC_SIZE_SUFFIX " %[result], %[pointer_to_value]\n\t" \ - "add %[tmp1],%[new_value], %[result]\n\t" \ - "st" EMBB_ATOMIC_TC_SIZE_SUFFIX " %[pointer_to_value], %[tmp1]\n\t" \ - : [result] "+d" (result), [tmp1]"+d" (tmp1), [pointer_to_value]"+m" (*pointer_to_value) \ - : [new_value]"d" (new_value) \ - : "memory", "cc" ); \ - return result; \ + EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) result; \ + register uint64_t e_reg = sizeof(value) == sizeof(uint32_t) ? (uint64_t)value : 0ull; \ + register uint32_t add = (uint32_t)pointer_to_value & 0xFFFFFFFC; \ + uint32_t pos = ((uint32_t)pointer_to_value & 0x3) * 8; \ + uint32_t width = sizeof(value) == sizeof(uint32_t) ? 31 : sizeof(value) * 8; \ + register uint64_t e_mask = ((uint64_t)width << 32) | pos; \ + __asm__ __volatile__ ("dsync\n\t" \ + "mov.a %[pointer_to_value], %[add]\n\t" \ + "loop_:\tld.w %H0, [%3]0\n\t" \ + "insert %L0, %L0, %[value], %A1\n\t" \ + "add" EMBB_ATOMIC_TC_SIZE_SUFFIX " %L0, %L0, %H0\n\t" \ + "cmpswap.w [%3]0, %A0\n\t" \ + "jne %H0, %L0, loop_\n\t" \ + "extr.u %[result], %H0, %A1" \ + : "+d"(e_reg), [e_mask]"+d"(e_mask), [result] "=&d" (result) \ + : [pointer_to_value]"a"(pointer_to_value), [value] "r" (value), [add] "d" (add) \ + : "memory"); \ + return result; \ } #else #error "No atomic fetch and store implementation found" @@ -140,7 +148,7 @@ EMBB_DEFINE_FETCH_AND_ADD(4, "") EMBB_DEFINE_FETCH_AND_ADD(1, ".b") EMBB_DEFINE_FETCH_AND_ADD(2, ".h") -EMBB_DEFINE_FETCH_AND_ADD(4, ".w") +EMBB_DEFINE_FETCH_AND_ADD(4, "") #else #error "Unknown architecture" diff --git a/base_c/include/embb/base/c/internal/atomic/or_assign.h b/base_c/include/embb/base/c/internal/atomic/or_assign.h index 695452a..8b1d1e1 100644 --- a/base_c/include/embb/base/c/internal/atomic/or_assign.h +++ b/base_c/include/embb/base/c/internal/atomic/or_assign.h @@ -116,15 +116,21 @@ EMBB_DEFINE_OR_ASSIGN(4, "") #define EMBB_DEFINE_OR_ASSIGN(EMBB_PARAMETER_SIZE_BYTE, EMBB_ATOMIC_TC_SIZE_SUFFIX) \ EMBB_PLATFORM_INLINE void EMBB_CAT2(embb_internal__atomic_or_assign_, EMBB_PARAMETER_SIZE_BYTE)(\ EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) volatile* pointer_to_value, EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) value) { \ - EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) result=0; \ - __asm__ __volatile__( \ - "dsync\n\t" \ - "ld" EMBB_ATOMIC_TC_SIZE_SUFFIX " %[result], %[pointer_to_value]\n\t"\ - "or %[result], %[value]\n\t" \ - "st" EMBB_ATOMIC_TC_SIZE_SUFFIX " %[pointer_to_value], %[result]\n\t" \ - : [result]"+d" (result), [pointer_to_value]"+m" (*pointer_to_value) \ - : [value]"d" (value) \ - : "memory"); \ + register uint64_t e_reg = sizeof(value) == sizeof(uint32_t) ? (uint64_t)value : 0ull; \ + register uint32_t add = (uint32_t)pointer_to_value & 0xFFFFFFFC; \ + uint32_t pos = ((uint32_t)pointer_to_value & 0x3) * 8; \ + uint32_t width = sizeof(value) == sizeof(uint32_t) ? 31 : sizeof(value) * 8; \ + register uint64_t e_ins = ((uint64_t)width << 32) | pos; \ + __asm__ __volatile__ ("dsync\n\t" \ + "mov.a %[pointer_to_value], %[add]\n\t" \ + "loop_:\tld.w %H0, [%2]0\n\t" \ + "insert %L0, %L0, %[value], %A1\n\t" \ + "or %L0, %H0\n\t" \ + "cmpswap.w [%2]0, %A0\n\t" \ + "jne %H0, %L0, loop_" \ + : "+d"(e_reg), [e_ins]"+d"(e_ins) \ + : [pointer_to_value]"a"(pointer_to_value), [value] "r" (value), [add] "d" (add) \ + : "memory"); \ } #else @@ -133,7 +139,7 @@ EMBB_DEFINE_OR_ASSIGN(4, "") EMBB_DEFINE_OR_ASSIGN(1, ".b") EMBB_DEFINE_OR_ASSIGN(2, ".h") -EMBB_DEFINE_OR_ASSIGN(4, ".w") +EMBB_DEFINE_OR_ASSIGN(4, "") #else #error "Unknown architecture" diff --git a/base_c/include/embb/base/c/internal/atomic/swap.h b/base_c/include/embb/base/c/internal/atomic/swap.h index 026b182..2f0ae68 100644 --- a/base_c/include/embb/base/c/internal/atomic/swap.h +++ b/base_c/include/embb/base/c/internal/atomic/swap.h @@ -118,17 +118,12 @@ EMBB_DEFINE_SWAP(4, "") #if defined(EMBB_PLATFORM_COMPILER_GNUC) #define EMBB_DEFINE_SWAP(EMBB_PARAMETER_SIZE_BYTE, EMBB_ATOMIC_TC_SIZE_SUFFIX) \ EMBB_PLATFORM_INLINE EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) EMBB_CAT2(embb_internal__atomic_swap_, EMBB_PARAMETER_SIZE_BYTE)(\ - EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) volatile* pointer_to_value, EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) new_value)\ - { \ - EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) \ - result=0; \ - __asm__ __volatile__("dsync\n\t" \ - "ld" EMBB_ATOMIC_TC_SIZE_SUFFIX "%[result], %[pointer_to_value]" \ - "st" EMBB_ATOMIC_TC_SIZE_SUFFIX "%[pointer_to_value], %[new_value]" \ - : [result]"+d"(result), [pointer_to_value]"+m" (*pointer_to_value) \ - : [new_value]"d" (new_value) \ + EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) volatile* pointer_to_value, EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) new_value) { \ + __asm__ __volatile__("swap" EMBB_ATOMIC_TC_SIZE_SUFFIX " [%0]0, %[new_value]" \ + : "+a" (pointer_to_value), [new_value]"+d" (new_value) \ + : \ : "memory"); \ - return result; \ + return new_value; \ } #else diff --git a/base_c/include/embb/base/c/internal/atomic/xor_assign.h b/base_c/include/embb/base/c/internal/atomic/xor_assign.h index 04c14d8..0859854 100644 --- a/base_c/include/embb/base/c/internal/atomic/xor_assign.h +++ b/base_c/include/embb/base/c/internal/atomic/xor_assign.h @@ -119,15 +119,21 @@ EMBB_DEFINE_XOR_ASSIGN(4, "") EMBB_PLATFORM_INLINE void EMBB_CAT2(embb_internal__atomic_xor_assign_, EMBB_PARAMETER_SIZE_BYTE)(\ EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) volatile* pointer_to_value, \ EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) value) { \ - EMBB_CAT2(EMBB_BASE_BASIC_TYPE_SIZE_, EMBB_PARAMETER_SIZE_BYTE) result=0; \ - __asm__ __volatile__( \ - "dsync\n\t" \ - "ld" EMBB_ATOMIC_TC_SIZE_SUFFIX " %[result], %[pointer_to_value]\n\t" \ - "xor %[result], %[value]\n\t" \ - "st" EMBB_ATOMIC_TC_SIZE_SUFFIX " %[pointer_to_value], %[result]\n\t" \ - : [value]"+d" (value), [result]"+d" (result) \ - : [pointer_to_value]"m" (*pointer_to_value) \ - : "memory"); \ + register uint64_t e_reg = sizeof(value) == sizeof(uint32_t) ? (uint64_t)value : 0ull; \ + register uint32_t add = (uint32_t)pointer_to_value & 0xFFFFFFFC; \ + uint32_t pos = ((uint32_t)pointer_to_value & 0x3) * 8; \ + uint32_t width = sizeof(value) == sizeof(uint32_t) ? 31 : sizeof(value) * 8; \ + register uint64_t e_ins = ((uint64_t)width << 32) | pos; \ + __asm__ __volatile__ ("dsync\n\t" \ + "mov.a %[pointer_to_value], %[add]\n\t" \ + "loop_:\tld.w %H0, [%2]0\n\t" \ + "insert %L0, %L0, %[value], %A1\n\t" \ + "xor %L0, %H0\n\t" \ + "cmpswap.w [%2]0, %A0\n\t" \ + "jne %H0, %L0, loop_" \ + : "+d"(e_reg), [e_ins]"+d"(e_ins) \ + : [pointer_to_value]"a"(pointer_to_value), [value] "r" (value), [add] "d" (add) \ + : "memory"); \ } #else @@ -136,7 +142,7 @@ EMBB_DEFINE_XOR_ASSIGN(4, "") EMBB_DEFINE_XOR_ASSIGN(1, ".b") EMBB_DEFINE_XOR_ASSIGN(2, ".h") -EMBB_DEFINE_XOR_ASSIGN(4, ".w") +EMBB_DEFINE_XOR_ASSIGN(4, "") #else #error "Unknown architecture"