partial implement

.gitignore

@@ -1,10 +1,11 @@
 # Generated by Cargo
 # will have compiled files and executables
-/target/
+target/
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
 Cargo.lock
 
 # These are backup files generated by rustfmt
 **/*.rs.bk
+.idea

Cargo.toml

@@ -6,3 +6,15 @@ edition = "2018"
 
 [dependencies]
 failure = "0.1"
+bytes = "0.5"
+crc = "1.8"
+rand = "0.7"
+proto = { path = "proto" }
+skiplist = { path = "skiplist" }
+memmap = "0.7"
+
+[workspace]
+members = [
+    "proto",
+    "skiplist",
+]

proto/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "proto"
+version = "0.1.0"
+authors = ["Jay Lee <[email protected]>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+protobuf = "2.10.0"

proto/proto/meta.proto

@@ -0,0 +1,75 @@
+syntax = "proto3";
+
+import "rustproto.proto";
+
+option (rustproto.lite_runtime_all) = true;
+
+package meta;
+
+message KV {
+  bytes key = 1;
+  bytes value = 2;
+  bytes user_meta = 3;
+  uint64 version = 4;
+  uint64 expires_at = 5;
+  bytes meta = 6;
+
+  // Stream id is used to identify which stream the KV came from.
+  uint32 stream_id = 10;
+  // Stream done is used to indicate end of stream.
+  bool stream_done = 11;
+}
+
+message KVList {
+  repeated KV kv = 1;
+}
+
+message ManifestChangeSet {
+  // A set of changes that are applied atomically.
+  repeated ManifestChange changes = 1;
+}
+
+enum EncryptionAlgo {
+  aes = 0;
+}
+
+message ManifestChange {
+  uint64 Id = 1;            // Table ID.
+  enum Operation {
+          CREATE = 0;
+          DELETE = 1;
+  }
+  Operation Op   = 2;
+  uint32 Level   = 3;       // Only used for CREATE.
+  uint64 key_id  = 4;
+  EncryptionAlgo encryption_algo = 5;
+  uint32 compression = 6;   // Only used for CREATE Op.
+}
+
+message BlockOffset {
+  bytes key = 1;
+  uint32 offset = 2;
+  uint32 len = 3;
+}
+
+message TableIndex {
+  repeated BlockOffset offsets = 1;
+  bytes bloom_filter = 2;
+  uint64 estimated_size = 3;
+}
+
+message Checksum {
+  enum Algorithm {
+    CRC32C = 0;
+    XXHash64 = 1;
+  }
+  Algorithm algo = 1; // For storing type of Checksum algorithm used
+  uint64 sum = 2;
+}
+
+message DataKey {
+  uint64 key_id      = 1;
+  bytes  data       = 2;
+  bytes  iv         = 3;
+  int64  created_at = 4;
+}

proto/proto/rustproto.proto

@@ -0,0 +1,47 @@
+syntax = "proto2";
+
+import "google/protobuf/descriptor.proto";
+
+// see https://github.com/gogo/protobuf/blob/master/gogoproto/gogo.proto
+// for the original idea
+
+package rustproto;
+
+extend google.protobuf.FileOptions {
+    // When true, oneof field is generated public
+    optional bool expose_oneof_all = 17001;
+    // When true all fields are public, and not accessors generated
+    optional bool expose_fields_all = 17003;
+    // When false, `get_`, `set_`, `mut_` etc. accessors are not generated
+    optional bool generate_accessors_all = 17004;
+    // Use `bytes::Bytes` for `bytes` fields
+    optional bool carllerche_bytes_for_bytes_all = 17011;
+    // Use `bytes::Bytes` for `string` fields
+    optional bool carllerche_bytes_for_string_all = 17012;
+    // When true, will only generate codes that works with lite runtime.
+    optional bool lite_runtime_all = 17035;
+}
+
+extend google.protobuf.MessageOptions {
+    // When true, oneof field is generated public
+    optional bool expose_oneof = 17001;
+    // When true all fields are public, and not accessors generated
+    optional bool expose_fields = 17003;
+    // When false, `get_`, `set_`, `mut_` etc. accessors are not generated
+    optional bool generate_accessors = 17004;
+    // Use `bytes::Bytes` for `bytes` fields
+    optional bool carllerche_bytes_for_bytes = 17011;
+    // Use `bytes::Bytes` for `string` fields
+    optional bool carllerche_bytes_for_string = 17012;
+}
+
+extend google.protobuf.FieldOptions {
+    // When true all fields are public, and not accessors generated
+    optional bool expose_fields_field = 17003;
+    // When false, `get_`, `set_`, `mut_` etc. accessors are not generated
+    optional bool generate_accessors_field = 17004;
+    // Use `bytes::Bytes` for `bytes` fields
+    optional bool carllerche_bytes_for_bytes_field = 17011;
+    // Use `bytes::Bytes` for `string` fields
+    optional bool carllerche_bytes_for_string_field = 17012;
+}

proto/src/lib.rs

@@ -0,0 +1 @@
+pub mod meta;