From 42b6c93c36d818a75344d16e353144a0e609e01e Mon Sep 17 00:00:00 2001
From: Craig Tiller <ctiller@google.com>
Date: Thu, 30 Jul 2015 09:59:25 -0700
Subject: [PATCH] Make TSAN happy with our lock free stack
---
include/grpc/support/atm_gcc_atomic.h | 2 ++
include/grpc/support/atm_gcc_sync.h | 5 +++++
include/grpc/support/atm_win32.h | 5 +++++
src/core/support/stack_lockfree.c | 12 ++++++++++--
4 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/include/grpc/support/atm_gcc_atomic.h b/include/grpc/support/atm_gcc_atomic.h
index 65d3d0c60f..a2c8386028 100644
--- a/include/grpc/support/atm_gcc_atomic.h
+++ b/include/grpc/support/atm_gcc_atomic.h
@@ -46,6 +46,8 @@ typedef gpr_intptr gpr_atm;
#define gpr_atm_no_barrier_load(p) (__atomic_load_n((p), __ATOMIC_RELAXED))
#define gpr_atm_rel_store(p, value) \
(__atomic_store_n((p), (gpr_intptr)(value), __ATOMIC_RELEASE))
+#define gpr_atm_no_barrier_store(p, value) \
+ (__atomic_store_n((p), (gpr_intptr)(value), __ATOMIC_RELAXED))
#define gpr_atm_no_barrier_fetch_add(p, delta) \
(__atomic_fetch_add((p), (gpr_intptr)(delta), __ATOMIC_RELAXED))
diff --git a/include/grpc/support/atm_gcc_sync.h b/include/grpc/support/atm_gcc_sync.h
index 4955e4436f..38b5a9eec2 100644
--- a/include/grpc/support/atm_gcc_sync.h
+++ b/include/grpc/support/atm_gcc_sync.h
@@ -68,6 +68,11 @@ static __inline void gpr_atm_rel_store(gpr_atm *p, gpr_atm value) {
*p = value;
}
+static __inline void gpr_atm_no_barrier_store(gpr_atm *p, gpr_atm value) {
+ GPR_ATM_COMPILE_BARRIER_();
+ *p = value;
+}
+
#undef GPR_ATM_LS_BARRIER_
#undef GPR_ATM_COMPILE_BARRIER_
diff --git a/include/grpc/support/atm_win32.h b/include/grpc/support/atm_win32.h
index da99021c24..694528a9ba 100644
--- a/include/grpc/support/atm_win32.h
+++ b/include/grpc/support/atm_win32.h
@@ -57,6 +57,11 @@ static __inline void gpr_atm_rel_store(gpr_atm *p, gpr_atm value) {
*p = value;
}
+static __inline void gpr_atm_no_barrier_store(gpr_atm *p, gpr_atm value) {
+ /* TODO(ctiller): Can we implement something better here? */
+ gpr_atm_rel_store(p, value);
+}
+
static __inline int gpr_atm_no_barrier_cas(gpr_atm *p, gpr_atm o, gpr_atm n) {
/* InterlockedCompareExchangePointerNoFence() not available on vista or
windows7 */
diff --git a/src/core/support/stack_lockfree.c b/src/core/support/stack_lockfree.c
index f24e272207..bc741f8c70 100644
--- a/src/core/support/stack_lockfree.c
+++ b/src/core/support/stack_lockfree.c
@@ -95,6 +95,8 @@ gpr_stack_lockfree *gpr_stack_lockfree_create(int entries) {
memset(&stack->pushed, 0, sizeof(stack->pushed));
#endif
+ GPR_ASSERT(sizeof(stack->entries->atm) == sizeof(stack->entries->contents));
+
/* Point the head at reserved dummy entry */
stack->head.contents.index = INVALID_ENTRY_INDEX;
return stack;
@@ -108,11 +110,15 @@ void gpr_stack_lockfree_destroy(gpr_stack_lockfree *stack) {
int gpr_stack_lockfree_push(gpr_stack_lockfree *stack, int entry) {
lockfree_node head;
lockfree_node newhead;
+ lockfree_node curent;
+ lockfree_node newent;
/* First fill in the entry's index and aba ctr for new head */
newhead.contents.index = (gpr_uint16)entry;
/* Also post-increment the aba_ctr */
- newhead.contents.aba_ctr = stack->entries[entry].contents.aba_ctr++;
+ curent.atm = gpr_atm_no_barrier_load(&stack->entries[entry].atm);
+ newhead.contents.aba_ctr = ++curent.contents.aba_ctr;
+ gpr_atm_no_barrier_store(&stack->entries[entry].atm, curent.atm);
#ifndef NDEBUG
/* Check for double push */
@@ -131,7 +137,9 @@ int gpr_stack_lockfree_push(gpr_stack_lockfree *stack, int entry) {
/* Atomically get the existing head value for use */
head.atm = gpr_atm_no_barrier_load(&(stack->head.atm));
/* Point to it */
- stack->entries[entry].contents.index = head.contents.index;
+ newent.atm = gpr_atm_no_barrier_load(&stack->entries[entry].atm);
+ newent.contents.index = head.contents.index;
+ gpr_atm_no_barrier_store(&stack->entries[entry].atm, newent.atm);
} while (!gpr_atm_rel_cas(&(stack->head.atm), head.atm, newhead.atm));
/* Use rel_cas above to make sure that entry index is set properly */
return head.contents.index == INVALID_ENTRY_INDEX;
--
GitLab