Add raw split/raw checksum/raw write sst (tikv#10524)

* Add raw split/raw checksum/raw write sst Signed-off-by: Andy Lok <[email protected]> * Fix build Signed-off-by: Andy Lok <[email protected]> * fix warning Signed-off-by: Andy Lok <[email protected]> * fix clippy Signed-off-by: Andy Lok <[email protected]> * format Signed-off-by: Andy Lok <[email protected]> * add metric Signed-off-by: Andy Lok <[email protected]> * fix clippy Signed-off-by: Andy Lok <[email protected]> * fix metrics Signed-off-by: Andy Lok <[email protected]> * fix test Signed-off-by: Andy Lok <[email protected]> * remove todo Signed-off-by: Andy Lok <[email protected]> * update kvproto Signed-off-by: Andy Lok <[email protected]>
chejinge · Jul 21, 2021 · 5a9e2de · 5a9e2de
1 parent 8defb3b
commit 5a9e2de
Show file tree

Hide file tree

Showing 32 changed files with 1,202 additions and 746 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -237,7 +237,6 @@ test_util = { path = "components/test_util", default-features = false }
 tokio = { version = "1.5", features = ["macros", "rt-multi-thread", "time"] }
 zipf = "6.1.0"
 
-
 [patch.crates-io]
 # TODO: remove this when new raft-rs is published.
 raft = { git = "https://github.com/tikv/raft-rs", branch = "master", default-features = false }
@@ -256,6 +255,12 @@ rusoto_sts = { git = "https://github.com/tikv/rusoto", branch = "gh1482-s3-addr-
 [target.'cfg(target_os = "linux")'.dependencies]
 procinfo = { git = "https://github.com/tikv/procinfo-rs", rev = "5125fc1a69496b73b26b3c08b6e8afc3c665a56e" }
 
+# When you modify TiKV cooperatively with kvproto, this will be useful to submit the PR to TiKV and the PR to
+# kvproto at the same time.
+# After the PR to kvproto is merged, remember to comment this out and run `cargo update -p kvproto`.
+# [patch.'https://github.com/pingcap/kvproto']
+# kvproto = {git = "https://github.com/your_github_id/kvproto", branch="your_branch"}
+
 [workspace]
 # See https://github.com/rust-lang/rfcs/blob/master/text/2957-cargo-features2.md
 # Without resolver = 2, using `cargo build --features x` to build `cmd`

diff --git a/components/engine_traits/Cargo.toml b/components/engine_traits/Cargo.toml
@@ -23,6 +23,7 @@ prost-codec = [
   "txn_types/prost-codec",
   "file_system/prost-codec",
 ]
+failpoints = ["fail/failpoints"]
 
 [dependencies]
 error_code = { path = "../error_code", default-features = false }
@@ -38,6 +39,7 @@ slog = { version = "2.3", features = ["max_level_trace", "release_max_level_debu
 slog-global = { version = "0.1", git = "https://github.com/breeswish/slog-global.git", rev = "d592f88e4dbba5eb439998463054f1a44fbf17b9" }
 kvproto = { git = "https://github.com/pingcap/kvproto.git", default-features = false }
 raft = { version = "0.6.0-alpha", default-features = false }
+fail = "0.4"
 
 [dev-dependencies]
 toml = "0.5"

diff --git a/components/engine_traits/src/lib.rs b/components/engine_traits/src/lib.rs
@@ -259,6 +259,8 @@ extern crate tikv_alloc;
 #[macro_use]
 extern crate serde_derive;
 extern crate slog_global;
+#[macro_use(fail_point)]
+extern crate fail;
 
 // These modules contain traits that need to be implemented by engines, either
 // they are required by KvEngine or are an associated type of KvEngine. It is

diff --git a/components/engine_traits/src/util.rs b/components/engine_traits/src/util.rs
@@ -3,6 +3,7 @@
 use super::{Error, Result};
 use tikv_util::codec;
 use tikv_util::codec::number::{self, NumberEncoder};
+use tikv_util::time::UnixSecs;
 
 /// Check if key in range [`start_key`, `end_key`).
 #[allow(dead_code)]
@@ -24,6 +25,18 @@ pub fn check_key_in_range(
     }
 }
 
+pub fn ttl_current_ts() -> u64 {
+    fail_point!("ttl_current_ts", |r| r.map_or(2, |e| e.parse().unwrap()));
+    UnixSecs::now().into_inner()
+}
+
+pub fn ttl_to_expire_ts(ttl: u64) -> u64 {
+    if ttl == 0 {
+        return 0;
+    }
+    ttl.saturating_add(ttl_current_ts())
+}
+
 pub fn append_expire_ts(value: &mut Vec<u8>, expire_ts: u64) {
     value.encode_u64(expire_ts).unwrap();
 }

diff --git a/components/error_code/src/sst_importer.rs b/components/error_code/src/sst_importer.rs
@@ -17,5 +17,6 @@ define_error_codes!(
     CANNOT_READ_EXTERNAL_STORAGE => ("CannotReadExternalStorage", "", ""),
     WRONG_KEY_PREFIX => ("WrongKeyPrefix", "", ""),
     BAD_FORMAT => ("BadFormat", "", ""),
-    FILE_CONFLICT => ("FileConflict", "", "")
+    FILE_CONFLICT => ("FileConflict", "", ""),
+    TTL_NOT_ENABLED => ("TTLNotEnabled", "", "")
 );
diff --git a/components/raftstore/src/store/fsm/apply.rs b/components/raftstore/src/store/fsm/apply.rs
@@ -4086,7 +4086,7 @@ mod tests {
     pub fn create_tmp_importer(path: &str) -> (TempDir, Arc<SSTImporter>) {
         let dir = Builder::new().prefix(path).tempdir().unwrap();
         let importer =
-            Arc::new(SSTImporter::new(&ImportConfig::default(), dir.path(), None).unwrap());
+            Arc::new(SSTImporter::new(&ImportConfig::default(), dir.path(), None, false).unwrap());
         (dir, importer)
     }
 

diff --git a/components/server/src/server.rs b/components/server/src/server.rs
@@ -730,6 +730,7 @@ impl<ER: RaftEngine> TiKVServer<ER> {
                 &self.config.import,
                 import_path,
                 self.encryption_key_manager.clone(),
+                self.config.storage.enable_ttl,
             )
             .unwrap(),
         );

diff --git a/components/sst_importer/src/errors.rs b/components/sst_importer/src/errors.rs
@@ -107,6 +107,9 @@ pub enum Error {
 
     #[error("ingest file conflict")]
     FileConflict,
+
+    #[error("ttl is not enabled")]
+    TTLNotEnabled,
 }
 
 impl From<String> for Error {
@@ -148,6 +151,7 @@ impl ErrorCodeExt for Error {
             Error::Encryption(e) => e.error_code(),
             Error::CodecError(e) => e.error_code(),
             Error::FileConflict => error_code::sst_importer::FILE_CONFLICT,
+            Error::TTLNotEnabled => error_code::sst_importer::TTL_NOT_ENABLED,
         }
     }
 }