From 1b0c172d39ffcce7db8d33935ad149ff40210b36 Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 19 Aug 2025 15:09:10 +0800 Subject: [PATCH 1/4] add a new option 'bottom_level_check_range_overlap' for manual compact Signed-off-by: glorv --- db/db_impl/db_impl_compaction_flush.cc | 5 ++++- db/external_sst_file_ingestion_job.cc | 10 ++++++++++ include/rocksdb/options.h | 5 +++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/db/db_impl/db_impl_compaction_flush.cc b/db/db_impl/db_impl_compaction_flush.cc index c2bd7af04765..d1635a8382a1 100644 --- a/db/db_impl/db_impl_compaction_flush.cc +++ b/db/db_impl/db_impl_compaction_flush.cc @@ -1234,7 +1234,10 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options, check_overlap_within_file = false; } } - if (!check_overlap_within_file) { + if (!check_overlap_within_file || + (!overlap && options.bottom_level_check_range_overlap && + level == + current_version->storage_info()->num_non_empty_levels() - 1)) { overlap = current_version->storage_info()->OverlapInLevel(level, begin, end); } diff --git a/db/external_sst_file_ingestion_job.cc b/db/external_sst_file_ingestion_job.cc index 3747aed252f6..3487de8b3f95 100644 --- a/db/external_sst_file_ingestion_job.cc +++ b/db/external_sst_file_ingestion_job.cc @@ -925,6 +925,9 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile( // the keys that we overlap with in this level, We also need to assign // this file a seqno to overwrite the existing keys in level `lvl` overlap_with_db = true; + ROCKS_LOG_INFO(db_options_.info_log, + "Ingest file overlap with level %d, file: %s", lvl, + file_to_ingest->internal_file_path.c_str()); break; } @@ -1110,6 +1113,10 @@ bool ExternalSstFileIngestionJob::IngestedFileFitInLevel( &file_largest_user_key)) { // File overlap with another files in this level, we cannot // add it to this level + ROCKS_LOG_INFO( + db_options_.info_log, + "Ingest file overlap with level file range, level %d, ingset_file: %s", + level, file_to_ingest->internal_file_path.c_str()); return false; } @@ -1117,6 +1124,9 @@ bool ExternalSstFileIngestionJob::IngestedFileFitInLevel( file_largest_user_key, level)) { // File overlap with a running compaction output that will be stored // in this level, we cannot add this file to this level + ROCKS_LOG_INFO(db_options_.info_log, + "Ingest file overlap with compaction in level %d, file: %s", + level, file_to_ingest->internal_file_path.c_str()); return false; } diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index d0975a6716df..e971cd4bb37f 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -2020,6 +2020,11 @@ struct CompactRangeOptions { // user-provided setting. This enables customers to selectively override the // age cutoff. double blob_garbage_collection_age_cutoff = -1; + + // If set to true, it will check file range overlap instead of keys overlap + // for the bottom level. This is used in manual compact for SST ingestion + // scenario. + bool bottom_level_check_range_overlap = false; }; // IngestExternalFileOptions is used by IngestExternalFile() From 424ac4fdacea53b0a229feb571db775d331a5d99 Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 17 Nov 2025 16:23:20 +0800 Subject: [PATCH 2/4] add a comment Signed-off-by: glorv --- db/db_impl/db_impl_compaction_flush.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/db/db_impl/db_impl_compaction_flush.cc b/db/db_impl/db_impl_compaction_flush.cc index d1635a8382a1..9bb30bb54588 100644 --- a/db/db_impl/db_impl_compaction_flush.cc +++ b/db/db_impl/db_impl_compaction_flush.cc @@ -1234,6 +1234,10 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options, check_overlap_within_file = false; } } + // `bottom_level_check_range_overlap` set to true means we check SST + // range overlap instead of real kv overlap to ensure that Manual + // Compact can be trigger on the overlapped SST files, so the SST files + // with big range can be split by the CompactionPartitioner. if (!check_overlap_within_file || (!overlap && options.bottom_level_check_range_overlap && level == From 4bc323cb882abb5434d49c93b3f06fdb56977c4b Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 18 Nov 2025 13:21:46 +0800 Subject: [PATCH 3/4] rename the new config name Signed-off-by: glorv --- db/db_impl/db_impl_compaction_flush.cc | 4 ++-- include/rocksdb/options.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/db/db_impl/db_impl_compaction_flush.cc b/db/db_impl/db_impl_compaction_flush.cc index 9bb30bb54588..7e951feab5f5 100644 --- a/db/db_impl/db_impl_compaction_flush.cc +++ b/db/db_impl/db_impl_compaction_flush.cc @@ -1234,12 +1234,12 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options, check_overlap_within_file = false; } } - // `bottom_level_check_range_overlap` set to true means we check SST + // `check_range_overlap_at_bottom_level` set to true means we check SST // range overlap instead of real kv overlap to ensure that Manual // Compact can be trigger on the overlapped SST files, so the SST files // with big range can be split by the CompactionPartitioner. if (!check_overlap_within_file || - (!overlap && options.bottom_level_check_range_overlap && + (!overlap && options.check_range_overlap_at_bottom_level && level == current_version->storage_info()->num_non_empty_levels() - 1)) { overlap = current_version->storage_info()->OverlapInLevel(level, diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index e971cd4bb37f..623bc36cc2df 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -2024,7 +2024,7 @@ struct CompactRangeOptions { // If set to true, it will check file range overlap instead of keys overlap // for the bottom level. This is used in manual compact for SST ingestion // scenario. - bool bottom_level_check_range_overlap = false; + bool check_range_overlap_at_bottom_level = false; }; // IngestExternalFileOptions is used by IngestExternalFile() From c24fc993839069c5a774bca82e1c3f728e94bfbc Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 18 Nov 2025 13:27:57 +0800 Subject: [PATCH 4/4] rename Signed-off-by: glorv --- db/db_impl/db_impl_compaction_flush.cc | 4 ++-- include/rocksdb/options.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/db/db_impl/db_impl_compaction_flush.cc b/db/db_impl/db_impl_compaction_flush.cc index 7e951feab5f5..9835c99b2b64 100644 --- a/db/db_impl/db_impl_compaction_flush.cc +++ b/db/db_impl/db_impl_compaction_flush.cc @@ -1234,12 +1234,12 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options, check_overlap_within_file = false; } } - // `check_range_overlap_at_bottom_level` set to true means we check SST + // `check_range_overlap_on_bottom_level` set to true means we check SST // range overlap instead of real kv overlap to ensure that Manual // Compact can be trigger on the overlapped SST files, so the SST files // with big range can be split by the CompactionPartitioner. if (!check_overlap_within_file || - (!overlap && options.check_range_overlap_at_bottom_level && + (!overlap && options.check_range_overlap_on_bottom_level && level == current_version->storage_info()->num_non_empty_levels() - 1)) { overlap = current_version->storage_info()->OverlapInLevel(level, diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index 623bc36cc2df..312c65fc2566 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -2024,7 +2024,7 @@ struct CompactRangeOptions { // If set to true, it will check file range overlap instead of keys overlap // for the bottom level. This is used in manual compact for SST ingestion // scenario. - bool check_range_overlap_at_bottom_level = false; + bool check_range_overlap_on_bottom_level = false; }; // IngestExternalFileOptions is used by IngestExternalFile()