ghOSt: Add bpf helper to perform a sync commit

PiperOrigin-RevId: 518714623

abi/latest/kernel/ghost.h

@@ -26,7 +26,7 @@
  * process are the same version as each other. Each successive version changes
  * values in this header file, assumptions about operations in the kernel, etc.
  */
-#define GHOST_VERSION 86
+#define GHOST_VERSION 87
 
 /*
  * Define SCHED_GHOST via the ghost uapi unless it has already been defined
@@ -457,6 +457,22 @@ struct bpf_ghost_halt_poll {
 	__u32 type; /* one of the enum GHOST_***_HALT_POLL options */
 };
 
+#define BPF_SYNC_COMMIT_MAX_CPUS        2
+
+/* Used with bpf_sync_commit */
+struct bpf_sync_commit_args {
+	/*
+	 * Fewer than MAX cpus can be used in the sync commit by specifying -1
+	 * for the other cpus.
+	 */
+	int cpus[BPF_SYNC_COMMIT_MAX_CPUS];
+	uint64_t gtids[BPF_SYNC_COMMIT_MAX_CPUS];
+	uint32_t task_barriers[BPF_SYNC_COMMIT_MAX_CPUS];
+
+	int run_flags[BPF_SYNC_COMMIT_MAX_CPUS];
+	int commit_flags[BPF_SYNC_COMMIT_MAX_CPUS];
+};
+
 #ifdef __cplusplus
 #include <atomic>
 typedef std::atomic<uint32_t> _ghost_ring_index_t;

lib/enclave.cc

@@ -756,10 +756,24 @@ std::string RunRequest::StateToString(ghost_txn_state state) {
 
 void LocalRunRequest::Open(const RunRequestOptions& options) {
   // Wait for current owner to relinquish ownership of the sync_group txn.
+  // NOTE: We must acquire ownership of the sync group first, before we modify
+  // any transaction fields. This serves as a pseudo-lock to avoid transaction
+  // fields getting clobbered by BPF (e.g. via bpf_sync_commit).
   if (options.sync_group_owner != kSyncGroupNotOwned) {
-    while (sync_group_owned()) {
-      Pause();
+    while (true) {
+      int32_t current_owner = sync_group_owner_get();
+
+      if (current_owner == kSyncGroupNotOwned) {
+        if (sync_group_take_ownership(options.sync_group_owner)) {
+          break;
+        }
+      } else if (current_owner != options.sync_group_owner) {
+        Pause();
+      } else {
+        break;
+      }
     }
+    CHECK(sync_group_owned());
   } else {
     CHECK(!sync_group_owned());
   }
@@ -776,9 +790,6 @@ void LocalRunRequest::Open(const RunRequestOptions& options) {
   txn_->task_barrier = options.target_barrier;
   txn_->run_flags = options.run_flags;
   txn_->commit_flags = options.commit_flags;
-  if (options.sync_group_owner != kSyncGroupNotOwned) {
-    sync_group_owner_set(options.sync_group_owner);
-  }
   allow_txn_target_on_cpu_ = options.allow_txn_target_on_cpu;
   if (allow_txn_target_on_cpu_) CHECK(sync_group_owned());
   txn_->state.store(GHOST_TXN_READY, std::memory_order_release);

lib/enclave.h

@@ -374,6 +374,9 @@ class RunRequest {
   // Updates owner of the sync_group in the associated txn.
   virtual void sync_group_owner_set(int32_t owner) = 0;
 
+  // Attempts to take ownership of the sync_group, if not already taken.
+  virtual bool sync_group_take_ownership(int32_t owner) = 0;
+
   // Returns true if the txn has a valid sync_group owner and false otherwise.
   virtual bool sync_group_owned() const = 0;
 
@@ -442,6 +445,15 @@ class LocalRunRequest : public RunRequest {
     txn_->u.sync_group_owner.store(owner, std::memory_order_release);
   }
 
+  // Attempts to take ownership of the sync_group, if not already taken.
+  // Uses the weak form of CAS, so this may fail spuriously (it is expected to
+  // be used in a loop).
+  bool sync_group_take_ownership(int32_t owner) override {
+    int32_t no_owner = kSyncGroupNotOwned;
+    return txn_->u.sync_group_owner.compare_exchange_weak(
+        no_owner, owner, std::memory_order_acq_rel);
+  }
+
   // Returns true if the txn has a valid sync_group owner and false otherwise.
   bool sync_group_owned() const override {
     return sync_group_owner_get() != kSyncGroupNotOwned;

third_party/bpf/common.bpf.h

@@ -148,6 +148,12 @@ static long (*bpf_ghost_resched_cpu)(__u32 cpu, __u64 cpu_seqnum) = (void *) 300
 #if GHOST_VERSION >= 79
 static long (*bpf_ghost_resched_cpu2)(__u32 cpu, __u32 flags) = (void *) 3003;
 #endif
+
+#if GHOST_VERSION >= 87
+static long (*bpf_ghost_sync_commit)(struct bpf_dynptr *args) =
+    (void *)3004;
+#endif
+
 #endif  // !GHOST_BPF */
 
 #if GHOST_VERSION >= 84