Problem: sst files are ingested into too bottom level by default (#871)

* Problem: sst files are ingested into too bottom level by default

Solution:
- set temporary num_level when ingesting sst files

* comments

* revert rocksdb version change

* changelog

Author: yihuang

CHANGELOG.md

@@ -1,6 +1,6 @@
 # Changelog
 
-*Feb 09, 2022*
+*Feb 15, 2022*
 
 ## v1.0.4
 
@@ -13,7 +13,8 @@
 
 - [#813](https://github.com/crypto-org-chain/cronos/pull/813) Tune up rocksdb options.
 - [#791](https://github.com/crypto-org-chain/cronos/pull/791) Implement versiondb and migration commands.
-- [#779](https://github.com/crypto-org-chain/cronos/pull/779) Add config iavl-lazy-loading to enable lazy loading of iavl store.
+- [#779](https://github.com/crypto-org-chain/cronos/pull/779) Add config `iavl-lazy-loading` to enable lazy loading of iavl store.
+- [#871](https://github.com/crypto-org-chain/cronos/pull/871) Only ingest sst files to level 3 in versiondb migration.
 
 *Feb 08, 2023*
 

integration_tests/test_versiondb.py

@@ -63,11 +63,15 @@ def test_versiondb_migration(cronos: Cronos):
     print("restore versiondb for node0")
     sst_dir = tempfile.mkdtemp(dir=cronos.base_dir)
     print(cli0.changeset_build_versiondb_sst(changeset_dir, sst_dir))
+    tmpdir = tempfile.mkdtemp(dir=cronos.base_dir)
     print(
         cli0.changeset_ingest_versiondb_sst(
-            cli0.data_dir / "data/versiondb", sst_dir, maximum_version=latest_version
+            tmpdir, sst_dir, maximum_version=latest_version
         )
     )
+    versiondb_dir = cli0.data_dir / "data/versiondb"
+    shutil.rmtree(versiondb_dir)
+    shutil.move(tmpdir, versiondb_dir)
 
     print("start all nodes")
     print(cronos.supervisorctl("start", "cronos_777-1-node0", "cronos_777-1-node1"))

nix/rocksdb.nix

@@ -22,10 +22,10 @@ stdenv.mkDerivation rec {
   version = "7.9.2";
 
   src = fetchFromGitHub {
-    owner = "yihuang";
+    owner = "facebook";
     repo = pname;
-    rev = "82d63cdd7db25cce3b97f30e11000a382c116aff"; # release/v7.9.x
-    sha256 = "sha256-ProQ5g5CqruWwnmbzD0HN4z9tgal9YCOgNiD9AHjjqo=";
+    rev = "v${version}";
+    sha256 = "sha256-5P7IqJ14EZzDkbjaBvbix04ceGGdlWBuVFH/5dpD5VM=";
   };
 
   nativeBuildInputs = [ cmake ninja ];

versiondb/client/flags.go

@@ -20,4 +20,5 @@ const (
 	flagLoadSnapshot     = "load-snapshot"
 	flagSorterChunkSize  = "sorter-chunk-size"
 	flagInitialVersion   = "initial-version"
+	flagNumLevels        = "num-levels"
 )

versiondb/client/ingest_sst.go

@@ -20,8 +20,19 @@ func IngestVersionDBSSTCmd() *cobra.Command {
 			if err != nil {
 				return err
 			}
-
-			db, cfHandle, err := tsrocksdb.OpenVersionDB(dbPath)
+			numLevels, err := cmd.Flags().GetInt(flagNumLevels)
+			if err != nil {
+				return err
+			}
+			opts := tsrocksdb.NewVersionDBOpts(false)
+			// it's a workaround for rocksdb issue(https://github.com/facebook/rocksdb/issues/11212),
+			// because rocksdb always ingest files into level `num_levels-1`,
+			// the new data will take a very long time to reach that level,
+			// level3 is the bottommost level in practice.
+			if numLevels > 0 {
+				opts.SetNumLevels(numLevels)
+			}
+			db, cfHandle, err := tsrocksdb.OpenVersionDBWithOpts(dbPath, opts)
 			if err != nil {
 				return err
 			}
@@ -56,5 +67,6 @@ func IngestVersionDBSSTCmd() *cobra.Command {
 	}
 	cmd.Flags().Bool(flagMoveFiles, false, "move sst files instead of copy them")
 	cmd.Flags().Int64(flagMaximumVersion, 0, "Specify the maximum version covered by the ingested files, if it's bigger than existing recorded latest version, will update it.")
+	cmd.Flags().Int(flagNumLevels, 4, "Override the num levels in default options, when ingesting into a new db, the sst files will be ingested into level `num-levels-1`, set to 0 to not override.")
 	return cmd
 }

versiondb/client/restore_app_db.go

@@ -63,6 +63,10 @@ func RestoreAppDBCmd(opts Options) *cobra.Command {
 			if err != nil {
 				return err
 			}
+			numLevels, err := cmd.Flags().GetInt(flagNumLevels)
+			if err != nil {
+				return err
+			}
 			stores, err := GetStoresOrDefault(cmd, opts.DefaultStores)
 			if err != nil {
 				return err
@@ -143,7 +147,15 @@ func RestoreAppDBCmd(opts Options) *cobra.Command {
 			defer ingestOpts.Destroy()
 			ingestOpts.SetMoveFiles(true)
 
-			db, err := grocksdb.OpenDb(opts.AppRocksDBOptions(false), iavlDir)
+			opts := opts.AppRocksDBOptions(false)
+			// it's a workaround for rocksdb issue(https://github.com/facebook/rocksdb/issues/11212),
+			// because rocksdb always ingest files into level `num_levels-1`,
+			// the new data will take a very long time to reach that level,
+			// level3 is the bottommost level in practice.
+			if numLevels > 0 {
+				opts.SetNumLevels(numLevels)
+			}
+			db, err := grocksdb.OpenDb(opts, iavlDir)
 			if err != nil {
 				return errors.Wrap(err, "open iavl db fail")
 			}
@@ -167,6 +179,7 @@ func RestoreAppDBCmd(opts Options) *cobra.Command {
 	cmd.Flags().String(flagStores, "", "list of store names, default to the current store list in application")
 	cmd.Flags().Uint64(flagSorterChunkSize, DefaultSorterChunkSizeIAVL, "uncompressed chunk size for external sorter, it decides the peak ram usage, on disk it'll be snappy compressed")
 	cmd.Flags().Int(flagConcurrency, runtime.NumCPU(), "Number concurrent goroutines to parallelize the work")
+	cmd.Flags().Int(flagNumLevels, 4, "Override the num levels in default options, when ingesting into a new db, the sst files will be ingested into level `num-levels-1`, set to 0 to not override.")
 
 	return cmd
 }

versiondb/tsrocksdb/opts.go

@@ -46,23 +46,27 @@ func NewVersionDBOpts(sstFileWriter bool) *grocksdb.Options {
 	return opts
 }
 
-// OpenVersionDB opens versiondb, the default column family is used for metadata,
-// actually key-value pairs are stored on another column family named with "versiondb",
-// which has user-defined timestamp enabled.
-func OpenVersionDB(dir string) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, error) {
+func OpenVersionDBWithOpts(dir string, versionDBOpts *grocksdb.Options) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, error) {
 	opts := grocksdb.NewDefaultOptions()
 	opts.SetCreateIfMissing(true)
 	opts.SetCreateIfMissingColumnFamilies(true)
 	db, cfHandles, err := grocksdb.OpenDbColumnFamilies(
 		opts, dir, []string{"default", VersionDBCFName},
-		[]*grocksdb.Options{opts, NewVersionDBOpts(false)},
+		[]*grocksdb.Options{opts, versionDBOpts},
 	)
 	if err != nil {
 		return nil, nil, err
 	}
 	return db, cfHandles[1], nil
 }
 
+// OpenVersionDB opens versiondb, the default column family is used for metadata,
+// actually key-value pairs are stored on another column family named with "versiondb",
+// which has user-defined timestamp enabled.
+func OpenVersionDB(dir string) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, error) {
+	return OpenVersionDBWithOpts(dir, NewVersionDBOpts(false))
+}
+
 // OpenVersionDBAndTrimHistory opens versiondb similar to `OpenVersionDB`,
 // but it also trim the versions newer than target one, can be used for rollback.
 func OpenVersionDBAndTrimHistory(dir string, version int64) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, error) {