ck_epoch: Allow for forward progress in concurrent epoch sections.

This work is derived directly from the work of John Esmet and Paul
Khuong ({jesmet,pkhuong}@appnexus.com) and PR34.

include/ck_epoch.h

@@ -43,6 +43,12 @@
 #define CK_EPOCH_LENGTH 4
 #endif
 
+/*
+ * This is used for sense detection with-respect to concurrent
+ * epoch sections.
+ */
+#define CK_EPOCH_SENSE	2
+
 struct ck_epoch_entry;
 typedef struct ck_epoch_entry ck_epoch_entry_t;
 typedef void ck_epoch_cb_t(ck_epoch_entry_t *);
@@ -56,16 +62,34 @@ struct ck_epoch_entry {
 	ck_stack_entry_t stack_entry;
 };
 
+/*
+ * A section object may be passed to every begin-end pair to allow for
+ * forward progress guarantees with-in prolonged active sections.
+ */
+struct ck_epoch_section {
+	unsigned int bucket;
+};
+typedef struct ck_epoch_section ck_epoch_section_t;
+
 /*
  * Return pointer to ck_epoch_entry container object.
  */
-#define CK_EPOCH_CONTAINER(T, M, N) CK_CC_CONTAINER(struct ck_epoch_entry, T, M, N)
+#define CK_EPOCH_CONTAINER(T, M, N) \
+	CK_CC_CONTAINER(struct ck_epoch_entry, T, M, N)
+
+struct ck_epoch_ref {
+	unsigned int epoch;
+	unsigned int count;
+};
 
 struct ck_epoch_record {
 	struct ck_epoch *global;
 	unsigned int state;
 	unsigned int epoch;
 	unsigned int active;
+	struct {
+		struct ck_epoch_ref bucket[CK_EPOCH_SENSE];
+	} local CK_CC_CACHELINE;
 	unsigned int n_pending;
 	unsigned int n_peak;
 	unsigned long n_dispatch;
@@ -82,11 +106,17 @@ struct ck_epoch {
 };
 typedef struct ck_epoch ck_epoch_t;
 
+/*
+ * Internal functions.
+ */
+void _ck_epoch_addref(ck_epoch_record_t *, ck_epoch_section_t *);
+void _ck_epoch_delref(ck_epoch_record_t *, ck_epoch_section_t *);
+
 /*
  * Marks the beginning of an epoch-protected section.
  */
-CK_CC_INLINE static void
-ck_epoch_begin(ck_epoch_record_t *record)
+CK_CC_FORCE_INLINE static void
+ck_epoch_begin(ck_epoch_record_t *record, ck_epoch_section_t *section)
 {
 	struct ck_epoch *epoch = record->global;
 
@@ -111,23 +141,29 @@ ck_epoch_begin(ck_epoch_record_t *record)
 		ck_pr_store_uint(&record->active, 1);
 		ck_pr_fence_store_load();
 #endif
-
-		return;
+	} else {
+		ck_pr_store_uint(&record->active, record->active + 1);
 	}
 
-	ck_pr_store_uint(&record->active, record->active + 1);
+	if (section != NULL)
+		_ck_epoch_addref(record, section);
+
 	return;
 }
 
 /*
  * Marks the end of an epoch-protected section.
  */
-CK_CC_INLINE static void
-ck_epoch_end(ck_epoch_record_t *record)
+CK_CC_FORCE_INLINE static void
+ck_epoch_end(ck_epoch_record_t *record, ck_epoch_section_t *section)
 {
 
 	ck_pr_fence_release();
 	ck_pr_store_uint(&record->active, record->active - 1);
+
+	if (section != NULL)
+		_ck_epoch_delref(record, section);
+
 	return;
 }
 
@@ -136,7 +172,7 @@ ck_epoch_end(ck_epoch_record_t *record)
  * argument until an epoch counter loop. This allows for a
  * non-blocking deferral.
  */
-CK_CC_INLINE static void
+CK_CC_FORCE_INLINE static void
 ck_epoch_call(ck_epoch_record_t *record,
 	      ck_epoch_entry_t *entry,
 	      ck_epoch_cb_t *function)

regressions/ck_epoch/validate/Makefile

@@ -1,6 +1,7 @@
 .PHONY: check clean distribution
 
-OBJECTS=ck_stack ck_epoch_synchronize ck_epoch_poll ck_epoch_call
+OBJECTS=ck_stack ck_epoch_synchronize ck_epoch_poll ck_epoch_call \
+	ck_epoch_section
 HALF=`expr $(CORES) / 2`
 
 all: $(OBJECTS)
@@ -9,13 +10,17 @@ check: all
 	./ck_stack $(CORES) 1
 	./ck_epoch_synchronize $(HALF) $(HALF) 1
 	./ck_epoch_poll $(CORES) 1 1
+	./ck_epoch_section
 
 ck_epoch_synchronize: ck_epoch_synchronize.c ../../../include/ck_stack.h ../../../include/ck_epoch.h ../../../src/ck_epoch.c
 	$(CC) $(CFLAGS) -o ck_epoch_synchronize ck_epoch_synchronize.c ../../../src/ck_epoch.c
 
 ck_epoch_poll: ck_epoch_poll.c ../../../include/ck_stack.h ../../../include/ck_epoch.h ../../../src/ck_epoch.c
 	$(CC) $(CFLAGS) -o ck_epoch_poll ck_epoch_poll.c ../../../src/ck_epoch.c
 
+ck_epoch_section: ck_epoch_section.c ../../../include/ck_stack.h ../../../include/ck_epoch.h ../../../src/ck_epoch.c
+	$(CC) $(CFLAGS) -o ck_epoch_section ck_epoch_section.c ../../../src/ck_epoch.c
+
 ck_epoch_call: ck_epoch_call.c ../../../include/ck_stack.h ../../../include/ck_epoch.h ../../../src/ck_epoch.c
 	$(CC) $(CFLAGS) -o ck_epoch_call ck_epoch_call.c ../../../src/ck_epoch.c
 

regressions/ck_epoch/validate/ck_epoch_poll.c

@@ -108,15 +108,15 @@ read_thread(void *unused CK_CC_UNUSED)
 
 	j = 0;
 	for (;;) {
-		ck_epoch_begin(&record);
+		ck_epoch_begin(&record, NULL);
 		CK_STACK_FOREACH(&stack, cursor) {
 			if (cursor == NULL)
 				continue;
 
 			n = CK_STACK_NEXT(cursor);
 			j += ck_pr_load_ptr(&n) != NULL;
 		}
-		ck_epoch_end(&record);
+		ck_epoch_end(&record, NULL);
 
 		if (j != 0 && ck_pr_load_uint(&readers) == 0)
 			ck_pr_store_uint(&readers, 1);
@@ -178,10 +178,10 @@ write_thread(void *unused CK_CC_UNUSED)
 		}
 
 		for (i = 0; i < PAIRS_S; i++) {
-			ck_epoch_begin(&record);
+			ck_epoch_begin(&record, NULL);
 			s = ck_stack_pop_upmc(&stack);
 			e = stack_container(s);
-			ck_epoch_end(&record);
+			ck_epoch_end(&record, NULL);
 
 			ck_epoch_call(&record, &e->epoch_entry, destructor);
 			ck_epoch_poll(&record);