From f38d0043aab39d3dfc4fe0421095781dcc698955 Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Tue, 8 Jul 2025 11:20:07 -0700 Subject: [PATCH 01/11] Add IVF index support in SVS. IVF index building can be done either with standard 1 level clustering or faster two level hierarchical clustering. Clustering algorithm uses AMX on supported Intel (R) Xeon (R) systems. Also, added support for BF16 datatype. --- .gitignore | 2 +- CMakeLists.txt | 7 + THIRD-PARTY-PROGRAMS | 2 +- benchmark/CMakeLists.txt | 11 + benchmark/include/svs-benchmark/ivf/build.h | 280 +++++++ benchmark/include/svs-benchmark/ivf/common.h | 41 + benchmark/include/svs-benchmark/ivf/search.h | 218 ++++++ .../include/svs-benchmark/ivf/static_traits.h | 139 ++++ benchmark/include/svs-benchmark/ivf/test.h | 152 ++++ .../include/svs-benchmark/ivf/uncompressed.h | 39 + benchmark/include/svs-benchmark/test.h | 3 + benchmark/src/ivf/build.cpp | 152 ++++ benchmark/src/ivf/search.cpp | 120 +++ benchmark/src/ivf/test.cpp | 118 +++ benchmark/src/ivf/uncompressed.cpp | 289 ++++++++ benchmark/src/main.cpp | 8 + bindings/python/CMakeLists.txt | 1 + bindings/python/include/svs/python/core.h | 2 +- bindings/python/include/svs/python/ivf.h | 94 +++ bindings/python/src/ivf.cpp | 640 ++++++++++++++++ bindings/python/src/python_bindings.cpp | 35 +- bindings/python/src/vamana.cpp | 2 +- cmake/mkl_functions | 2 + cmake/options.cmake | 6 + .../ivf_clustering/clusters_0.bin | Bin 0 -> 41032 bytes data/test_dataset/ivf_clustering/data_1.svs | Bin 0 -> 33792 bytes .../ivf_clustering/svs_config.toml | 34 + .../test_dataset/reference/ivf_reference.toml | 570 ++++++++++++++ include/svs/core/data.h | 2 +- include/svs/core/data/simple.h | 5 + include/svs/core/data/view.h | 1 + include/svs/index/ivf/clustering.h | 358 +++++++++ include/svs/index/ivf/common.h | 700 ++++++++++++++++++ include/svs/index/ivf/extensions.h | 220 ++++++ include/svs/index/ivf/hierarchical_kmeans.h | 369 +++++++++ include/svs/index/ivf/index.h | 345 +++++++++ include/svs/index/ivf/kmeans.h | 154 ++++ include/svs/index/ivf/sorted_buffer.h | 202 +++++ include/svs/lib/bfloat16.h | 143 ++++ include/svs/lib/datatype.h | 14 +- include/svs/lib/file_iterator.h | 3 +- include/svs/lib/narrow.h | 2 +- include/svs/lib/neighbor.h | 23 + include/svs/orchestrators/ivf.h | 180 +++++ tests/CMakeLists.txt | 19 + tests/integration/ivf/index_build.cpp | 119 +++ tests/integration/ivf/index_search.cpp | 147 ++++ tests/svs/core/data/data.h | 1 + tests/svs/index/ivf/common.cpp | 71 ++ tests/svs/index/ivf/hierarchical_kmeans.cpp | 75 ++ tests/svs/index/ivf/kmeans.cpp | 69 ++ tests/svs/lib/bfloat16.cpp | 79 ++ tests/svs/lib/datatype.cpp | 4 + tests/utils/ivf_reference.cpp | 46 ++ tests/utils/ivf_reference.h | 81 ++ tests/utils/test_dataset.cpp | 4 + tests/utils/test_dataset.h | 3 + .../benchmark_inputs/ivf/test-generator.toml | 34 + utils/CMakeLists.txt | 8 + utils/build_ivf.cpp | 130 ++++ utils/convert_data_to_bfloat16.cpp | 67 ++ utils/search_ivf.cpp | 240 ++++++ 62 files changed, 6876 insertions(+), 9 deletions(-) create mode 100644 benchmark/include/svs-benchmark/ivf/build.h create mode 100644 benchmark/include/svs-benchmark/ivf/common.h create mode 100644 benchmark/include/svs-benchmark/ivf/search.h create mode 100644 benchmark/include/svs-benchmark/ivf/static_traits.h create mode 100644 benchmark/include/svs-benchmark/ivf/test.h create mode 100644 benchmark/include/svs-benchmark/ivf/uncompressed.h create mode 100644 benchmark/src/ivf/build.cpp create mode 100644 benchmark/src/ivf/search.cpp create mode 100644 benchmark/src/ivf/test.cpp create mode 100644 benchmark/src/ivf/uncompressed.cpp create mode 100644 bindings/python/include/svs/python/ivf.h create mode 100644 bindings/python/src/ivf.cpp create mode 100644 data/test_dataset/ivf_clustering/clusters_0.bin create mode 100644 data/test_dataset/ivf_clustering/data_1.svs create mode 100644 data/test_dataset/ivf_clustering/svs_config.toml create mode 100644 data/test_dataset/reference/ivf_reference.toml create mode 100644 include/svs/index/ivf/clustering.h create mode 100644 include/svs/index/ivf/common.h create mode 100644 include/svs/index/ivf/extensions.h create mode 100644 include/svs/index/ivf/hierarchical_kmeans.h create mode 100644 include/svs/index/ivf/index.h create mode 100644 include/svs/index/ivf/kmeans.h create mode 100644 include/svs/index/ivf/sorted_buffer.h create mode 100644 include/svs/lib/bfloat16.h create mode 100644 include/svs/orchestrators/ivf.h create mode 100644 tests/integration/ivf/index_build.cpp create mode 100644 tests/integration/ivf/index_search.cpp create mode 100644 tests/svs/index/ivf/common.cpp create mode 100644 tests/svs/index/ivf/hierarchical_kmeans.cpp create mode 100644 tests/svs/index/ivf/kmeans.cpp create mode 100644 tests/svs/lib/bfloat16.cpp create mode 100644 tests/utils/ivf_reference.cpp create mode 100644 tests/utils/ivf_reference.h create mode 100644 tools/benchmark_inputs/ivf/test-generator.toml create mode 100644 utils/build_ivf.cpp create mode 100644 utils/convert_data_to_bfloat16.cpp create mode 100644 utils/search_ivf.cpp diff --git a/.gitignore b/.gitignore index 6aa7c701..efaf738f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ usr/ wheelhouse/ # Bundled test data -/data/ +/data/temp # Misc tool related files *.swp diff --git a/CMakeLists.txt b/CMakeLists.txt index 903a436b..95cdc9ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,6 +85,13 @@ include("cmake/fmt.cmake") include("cmake/spdlog.cmake") include("cmake/toml.cmake") +# IVF requires Intel(R) MKL support +if(SVS_EXPERIMENTAL_BUILD_IVF) + include("cmake/mkl.cmake") + target_compile_options(${SVS_LIB} INTERFACE "-DSVS_HAVE_MKL=1") +endif() + + ##### ##### Build Objects ##### diff --git a/THIRD-PARTY-PROGRAMS b/THIRD-PARTY-PROGRAMS index 9b976e48..7b1f9557 100644 --- a/THIRD-PARTY-PROGRAMS +++ b/THIRD-PARTY-PROGRAMS @@ -159,7 +159,7 @@ Please also refer to the file .github/CONTRIBUTING.md, which clarifies licensing external contributions to this project including patches, pull requests, etc. -------------------------------------------------------------------------------- -7. MKL (cmake/mkl.cmake, https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl.html) +7. Intel(R) MKL (cmake/mkl.cmake, https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl.html) Copyright (c) Intel Corporation, All rights reserved. diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 0e4bc916..e1ceeb16 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -48,6 +48,17 @@ set(SHARED_LIBRARY_FILES src/inverted/memory/executables/memory_test.cpp ) +# ivf +if (SVS_EXPERIMENTAL_BUILD_IVF) + list(APPEND SHARED_LIBRARY_FILES + src/ivf/uncompressed.cpp + src/ivf/search.cpp + src/ivf/build.cpp + src/ivf/test.cpp + ) +endif() + + add_library(svs_benchmark_library SHARED ${SHARED_LIBRARY_FILES}) target_include_directories(svs_benchmark_library PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include) diff --git a/benchmark/include/svs-benchmark/ivf/build.h b/benchmark/include/svs-benchmark/ivf/build.h new file mode 100644 index 00000000..46481bbb --- /dev/null +++ b/benchmark/include/svs-benchmark/ivf/build.h @@ -0,0 +1,280 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs-benchmark +#include "svs-benchmark/benchmark.h" +#include "svs-benchmark/build.h" +#include "svs-benchmark/datasets.h" +#include "svs-benchmark/index_traits.h" +#include "svs-benchmark/ivf/search.h" +#include "svs-benchmark/search.h" + +// svs +#include "svs/orchestrators/ivf.h" + +// stl +#include +#include +#include +#include +#include +#include + +namespace svsbenchmark::ivf { + +struct StaticBenchmark {}; + +// Forward declarations +struct BuildJob; + +template struct AssociatedJob; + +template <> struct AssociatedJob { + using type = BuildJob; +}; + +template using associated_job_t = typename AssociatedJob::type; + +// Job names +inline constexpr std::string_view benchmark_name(StaticBenchmark) { + return "ivf_static_build"; +} + +// Entry-point for registering the static index building executable. +std::unique_ptr static_workflow(); + +// Shared struct between the static and dynamic paths. +struct BuildJobBase { + public: + // A descriptive name for this workload. + std::string description_; + + // The dataset to load + Dataset dataset_; + + // Paths + std::filesystem::path data_; + std::filesystem::path queries_; + + // The number of queries (taken form queries) to use in the training set. + size_t queries_in_training_set_; + + // Dataset Parameters + svs::DataType data_type_; + svs::DataType query_type_; + svs::DistanceType distance_; + Extent ndims_; + + // Build Parameters + svs::index::ivf::IVFBuildParameters build_parameters_; + size_t num_threads_; + + public: + ///// Contructor + BuildJobBase( + std::string_view description, + svsbenchmark::Dataset dataset, + std::filesystem::path data, + std::filesystem::path queries, + size_t queries_in_training_set, + svs::DataType data_type, + svs::DataType query_type, + svs::DistanceType distance, + size_t ndims, + const svs::index::ivf::IVFBuildParameters& build_parameters, + size_t num_threads + ) + : description_{description} + , dataset_{dataset} + , data_{std::move(data)} + , queries_{std::move(queries)} + , queries_in_training_set_{queries_in_training_set} + , data_type_{data_type} + , query_type_{query_type} + , distance_{distance} + , ndims_{ndims} + , build_parameters_{build_parameters} + , num_threads_{num_threads} {} + + // Compatibility with `ExpectedResults`. + const svs::index::ivf::IVFBuildParameters& get_build_parameters() const { + return build_parameters_; + } + svs::DistanceType get_distance() const { return distance_; } + + // Return an example BuildJob that can be used to generate sample config files. + static BuildJobBase example() { + return BuildJobBase( + "example index build", + Dataset::example(), + "data.fvecs", + "queries.fvecs", + 5000, + svs::DataType::float32, + svs::DataType::float32, + svs::DistanceType::L2, + svs::Dynamic, + svs::index::ivf::IVFBuildParameters(128, 10000, 10, false, 0.1), + 8 + ); + } + + svs::lib::SaveTable + to_toml(std::string_view schema, const svs::lib::Version& version) const { + return svs::lib::SaveTable( + schema, + version, + {SVS_LIST_SAVE_(description), + SVS_LIST_SAVE_(dataset), + SVS_LIST_SAVE_(data), + SVS_LIST_SAVE_(queries), + SVS_LIST_SAVE_(queries_in_training_set), + SVS_LIST_SAVE_(data_type), + SVS_LIST_SAVE_(query_type), + SVS_LIST_SAVE_(distance), + SVS_LIST_SAVE_(ndims), + SVS_LIST_SAVE_(build_parameters), + SVS_LIST_SAVE_(num_threads)} + ); + } + + static BuildJobBase from_toml( + const svs::lib::ContextFreeLoadTable& table, + const std::optional& root + ) { + namespace lib = svs::lib; + return BuildJobBase( + SVS_LOAD_MEMBER_AT_(table, description), + SVS_LOAD_MEMBER_AT_(table, dataset, root), + svsbenchmark::extract_filename(table, "data", root), + svsbenchmark::extract_filename(table, "queries", root), + SVS_LOAD_MEMBER_AT_(table, queries_in_training_set), + SVS_LOAD_MEMBER_AT_(table, data_type), + SVS_LOAD_MEMBER_AT_(table, query_type), + SVS_LOAD_MEMBER_AT_(table, distance), + SVS_LOAD_MEMBER_AT_(table, ndims), + SVS_LOAD_MEMBER_AT_(table, build_parameters), + SVS_LOAD_MEMBER_AT_(table, num_threads) + ); + } +}; + +// Parsed setup for a static index build job. +struct BuildJob : public BuildJobBase { + public: + // Paths + std::filesystem::path groundtruth_; + // Preset search parameters + std::vector preset_parameters_; + // Post-build validation parameters. + svsbenchmark::search::SearchParameters search_parameters_; + // Directory to save the built index. + // An empty optional implies no saving. + std::optional save_directory_; + + public: + template + BuildJob( + std::filesystem::path groundtruth, + std::vector preset_parameters, + svsbenchmark::search::SearchParameters search_parameters, + std::optional save_directory, + Args&&... args + ) + : BuildJobBase(std::forward(args)...) + , groundtruth_{std::move(groundtruth)} + , preset_parameters_{std::move(preset_parameters)} + , search_parameters_{std::move(search_parameters)} + , save_directory_{std::move(save_directory)} {} + + // Return an example BuildJob that can be used to generate sample config files. + static BuildJob example() { + return BuildJob( + "groundtruth.ivecs", // groundtruth + {{10, 1.0}, {10, 4.0}, {50, 1.0}}, // preset_parameters + svsbenchmark::search::SearchParameters::example(), // search_parameters + std::nullopt, // save_directory + BuildJobBase::example() // base-class + ); + } + + // Compatibility with abstract search-space. + std::vector get_search_configs() const { + return preset_parameters_; + } + const svsbenchmark::search::SearchParameters& get_search_parameters() const { + return search_parameters_; + } + + template + auto invoke(F&& f, const Checkpoint& SVS_UNUSED(checkpoint)) const { + return f(dataset_, query_type_, data_type_, distance_, ndims_, *this); + } + + // Save the index if the `save_directory` field is non-empty. + template void maybe_save_index(Index& index) const { + if (!save_directory_) { + return; + } + const auto& root = save_directory_.value(); + svs::lib::save_to_disk(index, root / "clustering"); + } + + static constexpr svs::lib::Version save_version = svs::lib::Version(0, 0, 0); + static constexpr std::string_view serialization_schema = "benchmark_ivf_build_job"; + + // Save the BuildJob to a TOML table. + svs::lib::SaveTable save() const { + // Get a base table. + auto table = BuildJobBase::to_toml(serialization_schema, save_version); + + // Append the extra information needed by the static BuildJob. + SVS_INSERT_SAVE_(table, groundtruth); + SVS_INSERT_SAVE_(table, preset_parameters); + SVS_INSERT_SAVE_(table, search_parameters); + table.insert("save_directory", svs::lib::save(save_directory_.value_or(""))); + return table; + } + + // Load a BuildJob from a TOML table. + static BuildJob load( + const svs::lib::ContextFreeLoadTable& table, + const std::optional& root, + svsbenchmark::SaveDirectoryChecker& checker + ) { + return BuildJob( + svsbenchmark::extract_filename(table, "groundtruth", root), + SVS_LOAD_MEMBER_AT_(table, preset_parameters), + SVS_LOAD_MEMBER_AT_(table, search_parameters), + checker.extract(table.unwrap(), "save_directory"), + BuildJobBase::from_toml(table, root) + ); + } +}; + +// Dispatchers +using StaticBuildDispatcher = svs::lib::Dispatcher< + toml::table, + svsbenchmark::Dataset, + svs::DataType, + svs::DataType, + svs::DistanceType, + Extent, + const BuildJob&>; + +} // namespace svsbenchmark::ivf diff --git a/benchmark/include/svs-benchmark/ivf/common.h b/benchmark/include/svs-benchmark/ivf/common.h new file mode 100644 index 00000000..0ffdc10a --- /dev/null +++ b/benchmark/include/svs-benchmark/ivf/common.h @@ -0,0 +1,41 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// svs-benchmark +#include "svs-benchmark/benchmark.h" + +// svs +#include "svs/core/distance.h" +#include "svs/index/ivf/common.h" + +// stl +#include + +namespace svsbenchmark::ivf { + +// Test Routines +SVS_BENCHMARK_FOR_TESTS_ONLY inline search::SearchParameters test_search_parameters() { + return search::SearchParameters{10, {0.5, 0.8, 0.9}}; +} + +SVS_BENCHMARK_FOR_TESTS_ONLY inline std::vector +test_search_configs() { + return std::vector( + {{{10, 1.0}, {50, 1.0}}} + ); +} + +} // namespace svsbenchmark::ivf diff --git a/benchmark/include/svs-benchmark/ivf/search.h b/benchmark/include/svs-benchmark/ivf/search.h new file mode 100644 index 00000000..434f0a3f --- /dev/null +++ b/benchmark/include/svs-benchmark/ivf/search.h @@ -0,0 +1,218 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs-benchmark +#include "svs-benchmark/benchmark.h" +#include "svs-benchmark/datasets.h" +#include "svs-benchmark/index_traits.h" +#include "svs-benchmark/search.h" + +// svs +#include "svs/core/distance.h" +#include "svs/index/ivf/common.h" +#include "svs/lib/dispatcher.h" + +// stl +#include + +namespace svsbenchmark::ivf { + +inline constexpr std::string_view search_benchmark_name() { return "ivf_static_search"; } + +// Entry point for searching. +std::unique_ptr search_static_workflow(); + +// The current state of the index. +struct IVFState { + public: + svs::index::ivf::IVFSearchParameters search_parameters_; + size_t num_threads_; + + public: + IVFState( + svs::index::ivf::IVFSearchParameters search_parameters, size_t num_threads + ) + : search_parameters_{search_parameters} + , num_threads_{num_threads} {} + + template + explicit IVFState(const Index& index) + : IVFState(index.get_search_parameters(), index.get_num_threads()) {} + + // Saving + static constexpr svs::lib::Version save_version{0, 0, 0}; + static constexpr std::string_view serialization_schema = "benchmark_ivf_state"; + svs::lib::SaveTable save() const { + return svs::lib::SaveTable( + serialization_schema, + save_version, + {SVS_LIST_SAVE_(search_parameters), SVS_LIST_SAVE_(num_threads)} + ); + } +}; + +struct SearchJob { + public: + std::string description_; + svsbenchmark::Dataset dataset_; + std::filesystem::path config_; + std::filesystem::path graph_; + std::filesystem::path data_; + std::filesystem::path queries_; + std::filesystem::path groundtruth_; + size_t queries_in_training_set_; + svs::DataType data_type_; + svs::DataType query_type_; + svs::DistanceType distance_; + Extent ndims_; + size_t num_threads_; + svsbenchmark::search::SearchParameters search_parameters_; + std::vector preset_parameters_; + + public: + SearchJob( + std::string description, + svsbenchmark::Dataset dataset, + std::filesystem::path config, + std::filesystem::path graph, + std::filesystem::path data, + std::filesystem::path queries, + std::filesystem::path groundtruth, + size_t queries_in_training_set, + svs::DataType data_type, + svs::DataType query_type, + svs::DistanceType distance, + Extent ndims, + size_t num_threads, + const svsbenchmark::search::SearchParameters& search_parameters, + std::vector preset_parameters + ) + : description_{std::move(description)} + , dataset_{std::move(dataset)} + , config_{std::move(config)} + , graph_{std::move(graph)} + , data_{std::move(data)} + , queries_{std::move(queries)} + , groundtruth_{std::move(groundtruth)} + , queries_in_training_set_{queries_in_training_set} + , data_type_{data_type} + , query_type_{query_type} + , distance_{distance} + , ndims_{ndims} + , num_threads_{num_threads} + , search_parameters_{search_parameters} + , preset_parameters_{std::move(preset_parameters)} {} + + // Return the benchmark search parameters + const svsbenchmark::search::SearchParameters& get_search_parameters() const { + return search_parameters_; + } + + // Compatbility with `ExpectedResults` + static std::nullopt_t get_build_parameters() { return std::nullopt; } + svs::DistanceType get_distance() const { return distance_; } + + // Return the preset search configurations. + const std::vector& + get_search_configs() const { + return preset_parameters_; + } + + static SearchJob example() { + return SearchJob{ + "index search", // description + Dataset::example(), // dataset + "path/to/clustering dir", // clustering dir + "path/not/used", // Not applicable + "path/to/data", // data + "path/to/queries", // queries + "path/to/groundtruth", // groundtruth + 5000, // queries_in_training_set + svs::DataType::float32, // data_type + svs::DataType::float32, // query_type + svs::DistanceType::L2, // distance + Extent{svs::Dynamic}, // ndims + 4, // num_threads + search::SearchParameters::example(), // search_parameters + {{10, 1.0}, {10, 4.0}} // preset_parameters + }; + } + + template + auto invoke(F&& f, const Checkpoint& SVS_UNUSED(checkpoiner)) const { + return f(dataset_, query_type_, data_type_, distance_, ndims_, *this); + } + + ///// Save/Load + static constexpr svs::lib::Version save_version{0, 0, 0}; + static constexpr std::string_view serialization_schema = "benchmark_ivf_search_job"; + svs::lib::SaveTable save() const { + return svs::lib::SaveTable( + serialization_schema, + save_version, + {SVS_LIST_SAVE_(description), + SVS_LIST_SAVE_(dataset), + SVS_LIST_SAVE_(config), + SVS_LIST_SAVE_(graph), + SVS_LIST_SAVE_(data), + SVS_LIST_SAVE_(queries), + SVS_LIST_SAVE_(groundtruth), + SVS_LIST_SAVE_(queries_in_training_set), + SVS_LIST_SAVE_(data_type), + SVS_LIST_SAVE_(query_type), + SVS_LIST_SAVE_(distance), + SVS_LIST_SAVE_(ndims), + SVS_LIST_SAVE_(num_threads), + SVS_LIST_SAVE_(search_parameters), + SVS_LIST_SAVE_(preset_parameters)} + ); + } + + static SearchJob load( + const svs::lib::ContextFreeLoadTable& table, + const std::optional& root = {} + ) { + return SearchJob{ + SVS_LOAD_MEMBER_AT_(table, description), + SVS_LOAD_MEMBER_AT_(table, dataset, root), + svsbenchmark::extract_filename(table, "config", root), + svsbenchmark::extract_filename(table, "graph", root), + svsbenchmark::extract_filename(table, "data", root), + svsbenchmark::extract_filename(table, "queries", root), + svsbenchmark::extract_filename(table, "groundtruth", root), + SVS_LOAD_MEMBER_AT_(table, queries_in_training_set), + SVS_LOAD_MEMBER_AT_(table, data_type), + SVS_LOAD_MEMBER_AT_(table, query_type), + SVS_LOAD_MEMBER_AT_(table, distance), + SVS_LOAD_MEMBER_AT_(table, ndims), + SVS_LOAD_MEMBER_AT_(table, num_threads), + SVS_LOAD_MEMBER_AT_(table, search_parameters), + SVS_LOAD_MEMBER_AT_(table, preset_parameters)}; + } +}; + +using StaticSearchDispatcher = svs::lib::Dispatcher< + toml::table, + Dataset, + svs::DataType, + svs::DataType, + svs::DistanceType, + Extent, + const SearchJob&>; + +} // namespace svsbenchmark::ivf diff --git a/benchmark/include/svs-benchmark/ivf/static_traits.h b/benchmark/include/svs-benchmark/ivf/static_traits.h new file mode 100644 index 00000000..baab997e --- /dev/null +++ b/benchmark/include/svs-benchmark/ivf/static_traits.h @@ -0,0 +1,139 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs-benchmark +#include "svs-benchmark/index_traits.h" + +// svs +#include "svs/orchestrators/ivf.h" + +// stl +#include +#include + +namespace svsbenchmark { + +template <> struct IndexTraits { + using index_type = svs::IVF; + using config_type = svs::index::ivf::IVFSearchParameters; + using state_type = svsbenchmark::ivf::IVFState; + static std::string name() { return "static ivf index"; } + + // Configuration Space. + static void apply_config(svs::IVF& index, const config_type& config) { + index.set_search_parameters(config); + } + + template + static auto search( + svs::IVF& index, + const Queries& queries, + size_t num_neighbors, + const config_type& config + ) { + apply_config(index, config); + return index.search(queries, num_neighbors); + } + + static state_type report_state(const svs::IVF& index) { return state_type(index); } + + template + static config_type calibrate( + index_type& index, + const Queries& queries, + const Groundtruth& groundtruth, + size_t num_neighbors, + double target_recall, + svsbenchmark::CalibrateContext SVS_UNUSED(ctx), + svsbenchmark::Placeholder SVS_UNUSED(placeholder) + ) { + config_type baseline = index.get_search_parameters(); + + auto valid_configurations = std::vector(); + + // Search over epsilon search space first. + auto k_reorders = std::vector({1, 4, 10}); + auto n_probes_range = svs::threads::UnitRange(1, 200); + for (auto k_reorder : k_reorders) { + auto copy = baseline; + copy.k_reorder_ = k_reorder; + + copy.n_probes_ = *std::lower_bound( + n_probes_range.begin(), + n_probes_range.end(), + target_recall, + [&](size_t n_probes, double recall) { + copy.n_probes_ = n_probes; + index.set_search_parameters(copy); + + auto result = index.search(queries, num_neighbors); + auto this_recall = svs::k_recall_at_n(groundtruth, result); + return this_recall < recall; + } + ); + + valid_configurations.push_back(copy); + } + + // Loop through each valid configuration - find the fastest. + size_t best_config = std::numeric_limits::max(); + double lowest_latency = std::numeric_limits::max(); + + size_t config_index = 0; + for (auto& config : valid_configurations) { + apply_config(index, config); + auto latencies = std::vector(); + for (size_t i = 0; i < 5; ++i) { + auto tic = svs::lib::now(); + index.search(queries, num_neighbors); + latencies.push_back(svs::lib::time_difference(tic)); + } + + auto min_latency = *std::min_element(latencies.begin(), latencies.end()); + + std::cout << svs::lib::save_to_table(config) << '\n'; + SVS_SHOW(min_latency); + + if (min_latency < lowest_latency) { + best_config = config_index; + lowest_latency = min_latency; + } + ++config_index; + } + + return valid_configurations[best_config]; + } + + template + static config_type calibrate_with_hint( + index_type& index, + const Queries& queries, + const Groundtruth& groundtruth, + size_t num_neighbors, + double target_recall, + svsbenchmark::CalibrateContext ctx, + const config_type& SVS_UNUSED(preset), + svsbenchmark::Placeholder placeholder + ) { + return calibrate( + index, queries, groundtruth, num_neighbors, target_recall, ctx, placeholder + ); + } +}; + +} // namespace svsbenchmark diff --git a/benchmark/include/svs-benchmark/ivf/test.h b/benchmark/include/svs-benchmark/ivf/test.h new file mode 100644 index 00000000..9f9041b0 --- /dev/null +++ b/benchmark/include/svs-benchmark/ivf/test.h @@ -0,0 +1,152 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs-benchmark +#include "svs-benchmark/benchmark.h" +#include "svs-benchmark/test.h" +#include "svs-benchmark/ivf/build.h" +#include "svs-benchmark/ivf/search.h" +#include "svs-benchmark/ivf/static_traits.h" + +// svs +#include "svs/orchestrators/ivf.h" + +// stl +#include +#include +#include +#include + +namespace svsbenchmark::ivf { + +inline constexpr std::string_view test_benchmark_name() { return "ivf_test_generator"; } + +// A benchmark that generates reference inputs for unit tests. +std::unique_ptr test_generator(); + +///// Test Runner +struct IVFTest { + public: + std::vector groundtruths_; + std::filesystem::path data_f32_; + std::filesystem::path index_config_; + std::filesystem::path graph_; + std::filesystem::path queries_f32_; + size_t queries_in_training_set_; + // Backend-specific members + std::filesystem::path leanvec_data_matrix_; + std::filesystem::path leanvec_query_matrix_; + // Runtime values + size_t num_threads_; + + public: + IVFTest( + std::vector groundtruths, + std::filesystem::path data_f32, + std::filesystem::path index_config, + std::filesystem::path graph, + std::filesystem::path queries_f32, + size_t queries_in_training_set, + // backend-specific members + std::filesystem::path leanvec_data_matrix, + std::filesystem::path leanvec_query_matrix, + // Runtime values + size_t num_threads + ) + : groundtruths_{std::move(groundtruths)} + , data_f32_{std::move(data_f32)} + , index_config_{std::move(index_config)} + , graph_{std::move(graph)} + , queries_f32_{std::move(queries_f32)} + , queries_in_training_set_{queries_in_training_set} + , leanvec_data_matrix_{std::move(leanvec_data_matrix)} + , leanvec_query_matrix_{std::move(leanvec_query_matrix)} + , num_threads_{num_threads} {} + + static IVFTest example() { + return IVFTest{ + {DistanceAndGroundtruth::example()}, // groundtruths + "path/to/data_f32", // data_f32 + "path/to/config", // index_config + "path/to/graph", // graph + "path/to/queries_f32", // queries_f32 + 10000, // queries_in_training_set + "path/to/leanvec_data_matrix", // LeanVec data matrix + "path/to/leanvec_query_matrix", // LeanVec query matrix + 0, // Num Threads (not-saved) + }; + } + + const std::filesystem::path& groundtruth_for(svs::DistanceType distance) const { + for (const auto& pair : groundtruths_) { + if (pair.distance_ == distance) { + return pair.path_; + } + } + throw ANNEXCEPTION("Could not find a groundtruth for {} distance!", distance); + } + + ///// Save/Load + static constexpr svs::lib::Version save_version{0, 0, 0}; + static constexpr std::string_view serialization_schema = "benchmark_ivf_test"; + svs::lib::SaveTable save() const { + return svs::lib::SaveTable( + serialization_schema, + save_version, + {SVS_LIST_SAVE_(groundtruths), + SVS_LIST_SAVE_(data_f32), + SVS_LIST_SAVE_(index_config), + SVS_LIST_SAVE_(graph), + SVS_LIST_SAVE_(queries_f32), + SVS_LIST_SAVE_(queries_in_training_set), + SVS_LIST_SAVE_(leanvec_data_matrix), + SVS_LIST_SAVE_(leanvec_query_matrix)} + ); + } + + static IVFTest load( + const svs::lib::ContextFreeLoadTable& table, + size_t num_threads, + const std::optional& root = {} + ) { + return IVFTest{ + SVS_LOAD_MEMBER_AT_(table, groundtruths, root), + svsbenchmark::extract_filename(table, "data_f32", root), + svsbenchmark::extract_filename(table, "index_config", root), + svsbenchmark::extract_filename(table, "graph", root), + svsbenchmark::extract_filename(table, "queries_f32", root), + SVS_LOAD_MEMBER_AT_(table, queries_in_training_set), + svsbenchmark::extract_filename(table, "leanvec_data_matrix", root), + svsbenchmark::extract_filename(table, "leanvec_query_matrix", root), + num_threads}; + } +}; + +// Specialize ConfigAndResult for `svs::IVF`. +using ConfigAndResult = + svsbenchmark::ConfigAndResultPrototype; + +// Specialize ExpectedResult for `svs::IVF`. +using ExpectedResult = svsbenchmark::ExpectedResultPrototype< + svs::index::ivf::IVFBuildParameters, + svs::index::ivf::IVFSearchParameters>; + +// Test functions take the test input and returns a `TestFunctionReturn` with the results. +using TestFunction = std::function; + +} // namespace svsbenchmark::ivf diff --git a/benchmark/include/svs-benchmark/ivf/uncompressed.h b/benchmark/include/svs-benchmark/ivf/uncompressed.h new file mode 100644 index 00000000..c8af8afd --- /dev/null +++ b/benchmark/include/svs-benchmark/ivf/uncompressed.h @@ -0,0 +1,39 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs-benchmark +#include "svs-benchmark/ivf/build.h" +#include "svs-benchmark/ivf/search.h" +#include "svs-benchmark/ivf/test.h" + +// stl +#include + +namespace svsbenchmark::ivf { + +///// target-registration +// search +void register_uncompressed_static_search(ivf::StaticSearchDispatcher&); + +// build +void register_uncompressed_static_build(ivf::StaticBuildDispatcher&); + +// test +std::vector register_uncompressed_test_routines(); + +} // namespace svsbenchmark::ivf diff --git a/benchmark/include/svs-benchmark/test.h b/benchmark/include/svs-benchmark/test.h index d11a091f..a8190b99 100644 --- a/benchmark/include/svs-benchmark/test.h +++ b/benchmark/include/svs-benchmark/test.h @@ -247,6 +247,9 @@ inline int8_t convert_to(svs::lib::Type, float x) { inline svs::Float16 convert_to(svs::lib::Type, float x) { return svs::Float16{x}; } +inline svs::BFloat16 convert_to(svs::lib::Type, float x) { + return svs::BFloat16{x}; +} inline float convert_to(svs::lib::Type, float x) { return x; } } // namespace detail diff --git a/benchmark/src/ivf/build.cpp b/benchmark/src/ivf/build.cpp new file mode 100644 index 00000000..20d3d679 --- /dev/null +++ b/benchmark/src/ivf/build.cpp @@ -0,0 +1,152 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// svs-benchmark +#include "svs-benchmark/ivf/build.h" +#include "svs-benchmark/benchmark.h" +#include "svs-benchmark/executable.h" +#include "svs-benchmark/ivf/uncompressed.h" + +// svs +#include "svs/lib/saveload.h" +#include "svs/third-party/toml.h" + +// third-party +#include "fmt/core.h" +#include "fmt/ranges.h" + +// stl +#include +#include +#include +#include +#include + +namespace svsbenchmark::ivf { +namespace { + +///// +///// Dispatchers +///// + +template struct BuildDispatcher; + +// Dispatcher for Uncompressed Data. +template <> struct BuildDispatcher { + using type = ivf::StaticBuildDispatcher; + static type dispatcher() { + auto dispatcher = type{}; + ivf::register_uncompressed_static_build(dispatcher); + return dispatcher; + } +}; + +///// +///// Executable +///// + +constexpr std::string_view HELP_TEMPLATE = R"( +Run a {} benchmark for the IVF index. + +Usage: + (1) src-file.toml (output-file.toml/--validate) [basename] + (2) --help + (3) --example + +1. Run all the benchmarks in the global `{}` array in `src-file.toml`. + All elements in the array must be parseable as a `{}`. + + Results will be saved to `output-file.toml`. + + If `--validate` is given as the second argument, then all pre-run checks will be + performed on the input file and arguments but not benchmark will actually be run. + + Optional third argument `basename` will be used as the root for all file paths parsed. + +2. Print this help message. + +3. Display an example input TOML file to `stdout`. + +Backend specializations are dispatched on the following fields of the input TOML file: +* build_type: The dataset type to use. +* query_type: The element type of the query dataset. +* data_type: The input type of the source dataset. +* distance: The distance function to use. +* ndims: The compile-time dimensionality. + +Compiled specializations are listed below: +{{ build_type, query_type, data_type, distance, ndims }} +)"; + +template struct Exe { + public: + // type alises + using job_type = associated_job_t; + using dispatcher_type = typename BuildDispatcher::type; + + static dispatcher_type dispatcher() { + return BuildDispatcher::dispatcher(); + } + + static std::string_view name() { return benchmark_name(BenchmarkType()); } + + static void print_help() { + if constexpr (std::is_same_v) { + fmt::print( + HELP_TEMPLATE, + "static build and search", + name(), + "svsbenchmark::IVF::BuildJob" + ); + } else { + throw ANNEXCEPTION("Unreachable"); + } + auto f = dispatcher(); + for (size_t i = 0; i < f.size(); ++i) { + auto dispatch_strings = std::array{ + f.description(i, 0), + f.description(i, 1), + f.description(i, 2), + f.description(i, 3), + f.description(i, 4), + }; + fmt::print("{{ {} }}\n", fmt::join(dispatch_strings, ", ")); + } + } + + static job_type example() { return job_type::example(); } + + template + static std::optional> + parse_args_and_invoke(F&& f, std::span args) { + auto root = std::optional(); + if (args.size() == 1) { + root = std::filesystem::path(args[0]); + } + if constexpr (std::is_same_v) { + return f(root, svsbenchmark::SaveDirectoryChecker()); + } else { + return f(root); + } + } +}; +} // namespace + +// Return an executor for this benchmark. +std::unique_ptr static_workflow() { + return std::make_unique>>(); +} +} // namespace svsbenchmark::ivf diff --git a/benchmark/src/ivf/search.cpp b/benchmark/src/ivf/search.cpp new file mode 100644 index 00000000..dfd29fc9 --- /dev/null +++ b/benchmark/src/ivf/search.cpp @@ -0,0 +1,120 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// svs-benchmark +#include "svs-benchmark/ivf/search.h" +#include "svs-benchmark/benchmark.h" +#include "svs-benchmark/executable.h" +#include "svs-benchmark/ivf/uncompressed.h" + +// svs +#include "svs/lib/saveload.h" +#include "svs/third-party/toml.h" + +// third-party +#include "fmt/core.h" +#include "fmt/ranges.h" + +// stl +#include +#include +#include +#include +#include + +namespace svsbenchmark::ivf { +namespace { + +const char* HELP = R"( +Run a search-only benchmark for the IVF index. + +Usage: + (1) src-file.toml (output-file.toml/--validate) [basename] + (2) --help + (3) --example + +1. Run all the benchmarks in the global `search_ivf_static` array in `src-file.toml`. + All elements in the array must be parseable as a ``svsbenchmark::ivf::SearchJob``. + + Results will be saved to `output-file.toml`. + + If `--validate` is given as the second argument, then all pre-run checks will be + performed on the input file and arguments but not benchmark will actually be run. + + Optional third argument `basename` will be used as the root for all file paths parsed. + +2. Print this help message. + +3. Display an example input TOML file to `stdout`. + +Backend specializations are dispatched on the following fields of the input TOML file: +* build_type: The dataset type to use. +* query_type: The element type of the query dataset. +* data_type: The input type of the source dataset. +* distance: The distance function to use. +* ndims: The compile-time dimensionality. + +Compiled specializations are listed below: +{build_type, query_type, data_type, distance, ndims} +)"; + +struct Exe { + public: + using job_type = ivf::SearchJob; + using dispatcher_type = ivf::StaticSearchDispatcher; + + static dispatcher_type dispatcher() { + auto dispatcher = dispatcher_type{}; + ivf::register_uncompressed_static_search(dispatcher); + return dispatcher; + } + + static std::string_view name() { return search_benchmark_name(); } + + static void print_help() { + fmt::print("{}", HELP); + auto f = dispatcher(); + for (size_t i = 0; i < f.size(); ++i) { + auto dispatch_strings = std::array{ + f.description(i, 0), + f.description(i, 1), + f.description(i, 2), + f.description(i, 3), + f.description(i, 4), + }; + fmt::print("{{ {} }}\n", fmt::join(dispatch_strings, ", ")); + } + } + + static job_type example() { return job_type::example(); } + + template + static std::optional> + parse_args_and_invoke(F&& f, std::span args) { + auto root = std::optional(); + if (args.size() == 1) { + root = std::filesystem::path(args[0]); + } + return f(root); + } +}; +} // namespace + +// Return an executor for this benchmark. +std::unique_ptr search_static_workflow() { + return std::make_unique>(); +} +} // namespace svsbenchmark::ivf diff --git a/benchmark/src/ivf/test.cpp b/benchmark/src/ivf/test.cpp new file mode 100644 index 00000000..e39268b9 --- /dev/null +++ b/benchmark/src/ivf/test.cpp @@ -0,0 +1,118 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// svs-benchmark +#include "svs-benchmark/ivf/test.h" +#include "svs-benchmark/benchmark.h" +#include "svs-benchmark/executable.h" +#include "svs-benchmark/ivf/uncompressed.h" + +// svs +#include "svs/lib/saveload.h" +#include "svs/third-party/toml.h" + +// third-party +#include "fmt/core.h" +#include "fmt/ranges.h" + +// stl +#include +#include +#include +#include +#include +#include + +namespace svsbenchmark::ivf { +namespace { + +const char* HELP = R"( +Generate reference results for the IVF index. + +Usage: + (1) src-file.toml output-file.toml num_threads [basename] + (2) --help + (3) --example + +1. Run the test generators using `src-file.toml` as the test driver input. (see (3)) + Store the post-processed results into `output-file.toml`. + Third argument `num_threads` sets the number of worker threads to use for each job. + Optional fourth argument `basename` will be used as the root for all file-paths parsed + from `src-file.toml`. + + The output results will be saved to `output-file.toml` as a dictionary with the following + structure: + + "ivf_test_search" : Array of serialized `svsbenchmark::ivf::ExpectedResult` for + each search-only job registered. None of these entries should have the + `build_parameters` field present. + + "ivf_test_build" : Array of serialized `svsbenchmark::ivf::ExpectedResult` for + each build-job registered. All of these entries should have the `build_parameters` + field present. + +2. Print this message. + +3. Display an example input TOML file to `stdout`. +)"; + +struct TestGenerator { + public: + // Type Aliases + using job_type = ivf::IVFTest; + using test_type = std::vector; + + constexpr TestGenerator() = default; + + static std::string_view name() { return ivf::test_benchmark_name(); } + + static test_type tests() { + auto generator = test_type{}; + svsbenchmark::append_to(generator, register_uncompressed_test_routines()); + return generator; + } + + static job_type example() { return job_type::example(); } + static void print_help() { fmt::print("{}", HELP); } + + template + static std::optional + parse_args_and_invoke(F&& f, std::span args) { + // We should have 1 or 2 additional arguments to parse, corresponding to the + // number of threads and an optional data root. + auto nargs = args.size(); + bool nargs_okay = nargs == 1 || nargs == 2; + if (!nargs_okay) { + fmt::print("Received too few arguments for Inverted test generation!"); + print_help(); + return std::nullopt; + } + + auto num_threads = std::stoull(std::string{args[0]}); + auto data_root = std::optional(); + if (nargs == 2) { + data_root = args[1]; + } + return f(num_threads, data_root); + } +}; +} // namespace + +// Return an executor for this benchmark. +std::unique_ptr test_generator() { + return std::make_unique>(); +} +} // namespace svsbenchmark::ivf diff --git a/benchmark/src/ivf/uncompressed.cpp b/benchmark/src/ivf/uncompressed.cpp new file mode 100644 index 00000000..bf819183 --- /dev/null +++ b/benchmark/src/ivf/uncompressed.cpp @@ -0,0 +1,289 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// svs-benchmark +#include "svs-benchmark/ivf/uncompressed.h" +#include "svs-benchmark/benchmark.h" +#include "svs-benchmark/datasets/uncompressed.h" +#include "svs-benchmark/ivf/build.h" +#include "svs-benchmark/ivf/common.h" +#include "svs-benchmark/ivf/static_traits.h" + +// svs +#include "svs/core/distance.h" +#include "svs/lib/dispatcher.h" +#include "svs/third-party/toml.h" + +// stl +#include + +namespace svsbenchmark::ivf { + +namespace { + +// Specializations +#define X(Q, T, D, N) f.template operator()() +template void for_standard_specializations(F&& f) { + if constexpr (!is_minimal) { + X(float, svs::Float16, svs::distance::DistanceL2, 96); // deep + X(float, svs::Float16, svs::distance::DistanceL2, 100); // msturing + X(float, svs::Float16, svs::distance::DistanceIP, 200); // text2image + X(float, svs::Float16, svs::distance::DistanceIP, 512); // open-images, laion + X(float, svs::Float16, svs::distance::DistanceIP, 768); // dpr, rqa + // // Generic fallbacks + // X(float, svs::Float16, svs::distance::DistanceL2, svs::Dynamic); + // X(float, svs::Float16, svs::distance::DistanceIP, svs::Dynamic); + } +} +#undef X + +// Load and Search +template +toml::table run_static_search( + // dispatch arguments. + svsbenchmark::TypedUncompressed SVS_UNUSED(tag), + DispatchType SVS_UNUSED(query_type), + DispatchType SVS_UNUSED(data_type), + D distance, + svs::lib::ExtentTag SVS_UNUSED(extent), + // feed-forward arguments + const SearchJob& job +) { + auto tic = svs::lib::now(); + auto index = svs::IVF::assemble_from_file( + job.config_, + svs::data::SimpleData>::load(job.data_), + distance, + job.num_threads_ + ); + + double load_time = svs::lib::time_difference(tic); + auto queries = svs::data::SimpleData::load(job.queries_); + auto groundtruth = svs::data::SimpleData::load(job.groundtruth_); + + auto results = svsbenchmark::search::run_search( + index, + job, + search::QuerySet(queries, groundtruth, job.queries_in_training_set_), + svsbenchmark::LoadTime{load_time}, + svsbenchmark::Placeholder{} + ); + return svs::lib::save_to_table(results); +} + +// Static build and search +template +toml::table run_static_uncompressed( + // dispatch arguments. + TypedUncompressed SVS_UNUSED(tag), + DispatchType SVS_UNUSED(query_type), + DispatchType SVS_UNUSED(data_type), + D distance, + svs::lib::ExtentTag SVS_UNUSED(extent), + // feed-forward arguments + const BuildJob& job +) { + auto data = svs::data::SimpleData::load(job.data_); + auto tic = svs::lib::now(); + auto clustering = svs::IVF::build_clustering( + job.build_parameters_, + data, + distance, + job.num_threads_ + ); + double build_time = svs::lib::time_difference(tic); + auto index = svs::IVF::assemble_from_clustering( + clustering, + data, + distance, + job.num_threads_ + ); + + // Save the index if requested by the caller. + job.maybe_save_index(clustering); + + // Load and run queries. + auto queries = svs::data::SimpleData::load(job.queries_); + auto groundtruth = svs::data::SimpleData::load(job.groundtruth_); + auto results = svsbenchmark::search::run_search( + index, + job, + search::QuerySet(queries, groundtruth, job.queries_in_training_set_), + svsbenchmark::BuildTime{build_time}, + svsbenchmark::Placeholder{} + ); + return svs::lib::save_to_table(results); +} + +template +svsbenchmark::TestFunctionReturn test_search(const IVFTest& job) { + // Get the groundtruth for the distance. + constexpr svs::DistanceType distance = svs::distance_type_v; + const auto& groundtruth_path = job.groundtruth_for(distance); + auto kind = svsbenchmark::Uncompressed(svs::datatype_v); + + // Construct a `SearchJob` for this operation. + auto search_job = SearchJob{ + "IVF uncompressed reference search", + kind, + job.index_config_, + job.graph_, + job.data_f32_, + job.queries_f32_, + groundtruth_path, + job.queries_in_training_set_, + svs::datatype_v, + svs::DataType::float32, + svs::distance_type_v, + Extent(svs::Dynamic), + job.num_threads_, + test_search_parameters(), + test_search_configs()}; + + // Load the components for the test. + auto tic = svs::lib::now(); + auto data_loader = svs::lib::Lazy([&]() { + return svsbenchmark::convert_data( + svs::lib::Type(), svs::data::SimpleData::load(job.data_f32_) + ); + }); + auto index = svs::IVF::assemble_from_file( + job.index_config_, + data_loader, + Distance(), + job.num_threads_ + ); + double load_time = svs::lib::time_difference(tic); + auto queries = svs::data::SimpleData::load(job.queries_f32_); + auto groundtruth = svs::data::SimpleData::load(groundtruth_path); + + auto results = svsbenchmark::search::run_search( + index, + search_job, + svsbenchmark::search::QuerySet{ + std::move(queries), std::move(groundtruth), job.queries_in_training_set_}, + svsbenchmark::LoadTime{load_time}, + svsbenchmark::Placeholder{} + ); + + return TestFunctionReturn{ + .key_ = "ivf_test_search", + .results_ = + svs::lib::save_to_table(ivf::ExpectedResult(std::move(kind), results))}; +} + +template +svsbenchmark::TestFunctionReturn test_build(const IVFTest& job) { + // Get the groundtruth for the distance. + constexpr svs::DistanceType distance = svs::distance_type_v; + const auto& groundtruth_path = job.groundtruth_for(distance); + + auto build_parameters = svs::index::ivf::IVFBuildParameters{ + 128, 10000, 10, Hierarchical, 0.1}; + + auto kind = svsbenchmark::Uncompressed(svs::datatype_v); + + // Construct a `SearchJob` for this operation. + auto build_job = BuildJob{ + groundtruth_path, + {{10, 1.0}, {50, 1.0}}, + test_search_parameters(), + std::nullopt, + "IVF uncompressed reference build", + kind, + job.data_f32_, + job.queries_f32_, + job.queries_in_training_set_, + svs::datatype_v, + svs::DataType::float32, + svs::distance_type_v, + Extent(svs::Dynamic), + build_parameters, + job.num_threads_}; + + // Load the components for the test. + auto data = svsbenchmark::convert_data( + svs::lib::Type(), svs::data::SimpleData::load(job.data_f32_) + ); + auto tic = svs::lib::now(); + auto clustering = svs::IVF::build_clustering( + build_parameters, data, Distance(), job.num_threads_ + ); + double build_time = svs::lib::time_difference(tic); + auto index = svs::IVF::assemble_from_clustering( + clustering, + data, + distance, + job.num_threads_ + ); + + auto queries = svs::data::SimpleData::load(job.queries_f32_); + auto groundtruth = svs::data::SimpleData::load(groundtruth_path); + + auto results = svsbenchmark::search::run_search( + index, + build_job, + svsbenchmark::search::QuerySet{ + std::move(queries), std::move(groundtruth), job.queries_in_training_set_}, + svsbenchmark::BuildTime{build_time}, + svsbenchmark::Placeholder{} + ); + + return TestFunctionReturn{ + .key_ = "ivf_test_build", + .results_ = + svs::lib::save_to_table(ivf::ExpectedResult(std::move(kind), results))}; +} + +} // namespace + +// target-registration. +void register_uncompressed_static_search(ivf::StaticSearchDispatcher& dispatcher) { + for_standard_specializations( + [&dispatcher]() { + auto method = &run_static_search; + dispatcher.register_target(svs::lib::dispatcher_build_docs, method); + } + ); +} + +void register_uncompressed_static_build(ivf::StaticBuildDispatcher& dispatcher) { + for_standard_specializations( + [&dispatcher]() { + auto method = &run_static_uncompressed; + dispatcher.register_target(svs::lib::dispatcher_build_docs, method); + } + ); +} + +std::vector register_uncompressed_test_routines() { + if constexpr (build_test_generators) { + return std::vector({ + // Searching + &test_search, + &test_search, + // Building + &test_build, + &test_build, + &test_build, + &test_build, + }); + } else { + return std::vector(); + } +} + +} // namespace svsbenchmark::ivf diff --git a/benchmark/src/main.cpp b/benchmark/src/main.cpp index 7c06fb28..05079949 100644 --- a/benchmark/src/main.cpp +++ b/benchmark/src/main.cpp @@ -24,6 +24,10 @@ #include "svs-benchmark/vamana/test.h" // inverted #include "svs-benchmark/inverted/inverted.h" +// ivf +#include "svs-benchmark/ivf/build.h" +#include "svs-benchmark/ivf/search.h" +#include "svs-benchmark/ivf/test.h" // stl #include @@ -43,6 +47,10 @@ svsbenchmark::ExecutableDispatcher build_dispatcher() { dispatcher.register_executable(svsbenchmark::vamana::iterator_benchmark()); // inverted svsbenchmark::inverted::register_executables(dispatcher); + // ivf + dispatcher.register_executable(svsbenchmark::ivf::search_static_workflow()); + dispatcher.register_executable(svsbenchmark::ivf::static_workflow()); + dispatcher.register_executable(svsbenchmark::ivf::test_generator()); // documentation svsbenchmark::register_dataset_documentation(dispatcher); return dispatcher; diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 279b51df..ed630a81 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -37,6 +37,7 @@ set(CPP_FILES src/vamana.cpp src/vamana_common.cpp src/svs_mkl.cpp + src/ivf.cpp ) set(LIB_NAME "_svs") diff --git a/bindings/python/include/svs/python/core.h b/bindings/python/include/svs/python/core.h index ed378375..50864281 100644 --- a/bindings/python/include/svs/python/core.h +++ b/bindings/python/include/svs/python/core.h @@ -104,7 +104,7 @@ class UnspecializedGraphLoader { const std::filesystem::path& path() const { return path_; } const Allocator& allocator() const { return allocator_; } - svs::graphs::SimpleGraph load() const { + auto load() const { using other = std::allocator_traits::rebind_alloc; return svs::graphs::SimpleGraph::load(path_, other(allocator_)); } diff --git a/bindings/python/include/svs/python/ivf.h b/bindings/python/include/svs/python/ivf.h new file mode 100644 index 00000000..5f0dc0a3 --- /dev/null +++ b/bindings/python/include/svs/python/ivf.h @@ -0,0 +1,94 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs python bindings +#include "svs/python/common.h" +#include "svs/python/core.h" + +#include "svs/core/distance.h" +#include "svs/lib/bfloat16.h" +#include "svs/lib/datatype.h" +#include "svs/lib/float16.h" +#include "svs/lib/meta.h" +#include "svs/lib/misc.h" + +#include +#include + +namespace svs::python { +namespace ivf_specializations { +/// +/// Flag to selectively enable index building. +/// +enum class EnableBuild { None, FromFile, FromFileAndArray }; + +template +inline constexpr bool enable_build_from_file = + (B == EnableBuild::FromFile || B == EnableBuild::FromFileAndArray); + +template +inline constexpr bool enable_build_from_array = (B == EnableBuild::FromFileAndArray); + +// Define all desired specializations for searching and building. +template void for_standard_specializations(F&& f) { +#define X(Q, T, N, B) f.template operator()(); +#define XN(Q, T, N) X(Q, T, N, EnableBuild::None) + // Pattern: + // QueryType, DataType, Dimensionality, Enable Building + // clang-format off + X(float, svs::BFloat16, 512, EnableBuild::FromFileAndArray); + + XN(float, float, 512); + XN(float, svs::Float16, 512); + + X(float, svs::BFloat16, Dynamic, EnableBuild::FromFileAndArray); + XN(float, float, Dynamic); + XN(float, svs::Float16, Dynamic); + // clang-format on +#undef XN +#undef X +} +} // namespace ivf_specializations + +namespace ivf { +template void add_interface(pybind11::class_& manager) { + manager.def_property_readonly( + "experimental_backend_string", + &Manager::experimental_backend_string, + R"( + Read Only (str): Get a string identifying the full-type of the backend implementation. + + This property is experimental and subject to change without a deprecation warning.)" + ); + + manager.def_property( + "search_parameters", + &Manager::get_search_parameters, + &Manager::set_search_parameters, + R"( + "Read/Write (svs.IVFSearchParameters): Get/set the current search parameters for the + index. These parameters modify both the algorithmic properties of search (affecting recall) + and non-algorthmic properties of search (affecting queries-per-second). + + See also: `svs.IVFSearchParameters`.)" + ); +} + +void wrap(pybind11::module& m); +} // namespace ivf +} // namespace svs::python diff --git a/bindings/python/src/ivf.cpp b/bindings/python/src/ivf.cpp new file mode 100644 index 00000000..a96fde8b --- /dev/null +++ b/bindings/python/src/ivf.cpp @@ -0,0 +1,640 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// svs python bindings +#include "svs/python/ivf.h" +#include "svs/python/common.h" +#include "svs/python/core.h" +#include "svs/python/dispatch.h" +#include "svs/python/manager.h" + +// svs +#include "svs/core/data/simple.h" +#include "svs/core/distance.h" +#include "svs/lib/array.h" +#include "svs/lib/datatype.h" +#include "svs/lib/dispatcher.h" +#include "svs/lib/float16.h" +#include "svs/lib/meta.h" +#include "svs/orchestrators/ivf.h" + +// pybind +#include +#include +#include + +// stl +#include +#include +#include +#include + +///// +///// IVF +///// + +namespace py = pybind11; +using namespace svs::python::ivf_specializations; + +namespace svs::python::ivf { +// The build process in IVF uses Kmeans to get centroids and assignments of data. +// This sparse clustering can be saved with centroids stored as float datatype. +// While assembling, the sparse clustering is used to create DenseClusters and +// centroids datatype can be changed as per the search specializations. +// By default, BFloat16 centroids are used to take advantage of AMX +// template +using Clustering = + svs::index::ivf::Clustering, uint32_t>; + +namespace detail { + +///// +///// Assembly +///// + +template +svs::IVF assemble_uncompressed( + Clustering clustering, + svs::VectorDataLoader> data, + svs::DistanceType distance_type, + size_t num_threads, + size_t intra_query_threads = 1 +) { + return svs::IVF::assemble_from_clustering( + std::move(clustering), + std::move(data), + distance_type, + num_threads, + intra_query_threads + ); +} + +template +void register_uncompressed_ivf_assemble(Dispatcher& dispatcher) { + for_standard_specializations( + [&dispatcher]() { + auto method = &assemble_uncompressed; + dispatcher.register_target(svs::lib::dispatcher_build_docs, method); + } + ); +} + +template void register_ivf_assembly(Dispatcher& dispatcher) { + register_uncompressed_ivf_assemble(dispatcher); +} + +///// +///// Assembly from File +///// +// N.B (IB): quite a bit of repetition in Assemble and AssembleFromFile functions. +// Loading the cluster from file and then using the Assemble from clustering +// shows performance loss, mainly due to the threadpool used for loading. +// This needs to be revisited. +template +svs::IVF assemble_from_file_uncompressed( + const std::filesystem::path& cluster_path, + svs::VectorDataLoader> data, + svs::DistanceType distance_type, + size_t num_threads, + size_t intra_query_threads = 1 +) { + return svs::IVF::assemble_from_file( + cluster_path, std::move(data), distance_type, num_threads, intra_query_threads + ); +} + +template +void register_uncompressed_ivf_assemble_from_file(Dispatcher& dispatcher) { + for_standard_specializations( + [&dispatcher]() { + auto method = &assemble_from_file_uncompressed; + dispatcher.register_target(svs::lib::dispatcher_build_docs, method); + } + ); +} + +template +void register_ivf_assembly_from_file(Dispatcher& dispatcher) { + register_uncompressed_ivf_assemble_from_file(dispatcher); +} + +using IVFAssembleTypes = + std::variant; + +///// +///// Build From File +///// + +template +Clustering build_uncompressed( + const svs::index::ivf::IVFBuildParameters& parameters, + svs::VectorDataLoader> data, + svs::DistanceType distance_type, + size_t num_threads +) { + return svs::IVF::build_clustering( + parameters, std::move(data), distance_type, num_threads + ); +} + +template +void register_uncompressed_ivf_build_from_file(Dispatcher& dispatcher) { + for_standard_specializations( + [&dispatcher]() { + if constexpr (enable_build_from_file) { + auto method = &build_uncompressed; + dispatcher.register_target(svs::lib::dispatcher_build_docs, method); + } + } + ); +} + +template void register_ivf_build_from_file(Dispatcher& dispatcher) { + register_uncompressed_ivf_build_from_file(dispatcher); +} + +using IVFBuildTypes = std::variant; + +///// +///// Build from Array +///// + +template +Clustering uncompressed_build_from_array( + const svs::index::ivf::IVFBuildParameters& parameters, + svs::data::ConstSimpleDataView view, + svs::DistanceType distance_type, + size_t num_threads +) { + auto data = + svs::data::SimpleData>(view.size(), view.dimensions()); + svs::data::copy(view, data); + return svs::IVF::build_clustering( + parameters, std::move(data), distance_type, num_threads + ); +} + +template void register_ivf_build_from_array(Dispatcher& dispatcher) { + for_standard_specializations( + [&dispatcher]() { + if constexpr (enable_build_from_array) { + auto method = &uncompressed_build_from_array; + dispatcher.register_target(svs::lib::dispatcher_build_docs, method); + } + } + ); +} + +///// +///// Dispatch Invocation +///// + +using AssemblyDispatcher = svs::lib:: + Dispatcher; + +AssemblyDispatcher assembly_dispatcher() { + auto dispatcher = AssemblyDispatcher{}; + + // Register available backend methods. + register_ivf_assembly(dispatcher); + return dispatcher; +} + +// Assemble +svs::IVF assemble_from_clustering( + Clustering clustering, + IVFAssembleTypes data_kind, + svs::DistanceType distance_type, + svs::DataType SVS_UNUSED(query_type), + bool SVS_UNUSED(enforce_dims), + size_t num_threads, + size_t intra_query_threads = 1 +) { + return assembly_dispatcher().invoke( + std::move(clustering), + std::move(data_kind), + distance_type, + num_threads, + intra_query_threads + ); +} + +using AssemblyFromFileDispatcher = svs::lib::Dispatcher< + svs::IVF, + const std::filesystem::path&, + IVFAssembleTypes, + svs::DistanceType, + size_t, + size_t>; + +AssemblyFromFileDispatcher assembly_from_file_dispatcher() { + auto dispatcher = AssemblyFromFileDispatcher{}; + + // Register available backend methods. + register_ivf_assembly_from_file(dispatcher); + return dispatcher; +} + +// Assemble from file +svs::IVF assemble_from_file( + const std::string& cluster_path, + IVFAssembleTypes data_kind, + svs::DistanceType distance_type, + svs::DataType SVS_UNUSED(query_type), + bool SVS_UNUSED(enforce_dims), + size_t num_threads, + size_t intra_query_threads = 1 +) { + return assembly_from_file_dispatcher().invoke( + cluster_path, std::move(data_kind), distance_type, num_threads, intra_query_threads + ); +} + +constexpr std::string_view ASSEMBLE_DOCSTRING_PROTO = R"( +Assemble a searchable IVF index from provided clustering and data + +Args: + clustering_path/clustering: Path to the directory where the clustering was generated. + OR directly provide the loaded Clustering. + data_loader: The loader for the dataset. See comment below for accepted types. + distance: The distance function to use. + query_type: The data type of the queries. + enforce_dims: Require that the compiled dimensionality of the returned index matches + the dimensionality provided in the ``data_loader`` argument. If a match is not + found, an exception is thrown. + + This is meant to ensure that specialized dimensionality is provided without falling + back to generic implementations. Leaving the ``dims`` out when constructing the + ``data_loader`` will with `enable_dims = True` will always attempt to use a generic + implementation. + num_threads: The number of threads to use for queries (can't be changed after loading). + intra_query_threads: (default: 1) these many threads work on a single query. + Total number of threads required = ``query_batch_size`` * ``intra_query_threads``. + Where ``query_batch_size`` is the number of queries processed in parallel. + Use this parameter only when the ``query_batch_size`` is smaller and ensure your + system has sufficient threads available. Set ``num_threads`` = ``query_batch_size`` + +The top level type is an abstract type backed by various specialized backends that will +be instantiated based on their applicability to the particular problem instance. + +The arguments upon which specialization is conducted are: + +* `data_loader`: Both kind (type of loader) and inner aspects of the loader like data type, + quantization type, and number of dimensions. +* `distance`: The distance measure being used. + +Specializations compiled into the binary are listed below. + +{} +)"; + +void wrap_assemble(py::class_& ivf) { + auto dispatcher = assembly_dispatcher(); + // Procedurally generate the dispatch string. + auto dynamic = std::string{}; + for (size_t i = 0; i < dispatcher.size(); ++i) { + fmt::format_to( + std::back_inserter(dynamic), + R"( +Method {}: + - data_loader: {} + - distance: {} +)", + i, + dispatcher.description(i, 2), + dispatcher.description(i, 3) + ); + } + + ivf.def_static( + "assemble_from_clustering", + &detail::assemble_from_clustering, + py::arg("clustering"), + py::arg("data_loader"), + py::arg("distance") = svs::L2, + py::arg("query_type") = svs::DataType::float32, + py::arg("enforce_dims") = false, + py::arg("num_threads") = 1, + py::arg("intra_query_threads") = 1, + fmt::format(ASSEMBLE_DOCSTRING_PROTO, dynamic).c_str() + ); + ivf.def_static( + "assemble_from_file", + &detail::assemble_from_file, + py::arg("clustering_path"), + py::arg("data_loader"), + py::arg("distance") = svs::L2, + py::arg("query_type") = svs::DataType::float32, + py::arg("enforce_dims") = false, + py::arg("num_threads") = 1, + py::arg("intra_query_threads") = 1, + fmt::format(ASSEMBLE_DOCSTRING_PROTO, dynamic).c_str() + ); +} + +// Build from file +using BuildFromFileDispatcher = svs::lib::Dispatcher< + Clustering, + const svs::index::ivf::IVFBuildParameters&, + IVFBuildTypes, + svs::DistanceType, + size_t>; + +BuildFromFileDispatcher build_from_file_dispatcher() { + auto dispatcher = BuildFromFileDispatcher{}; + register_ivf_build_from_file(dispatcher); + return dispatcher; +} + +Clustering build_from_file( + const svs::index::ivf::IVFBuildParameters& parameters, + IVFBuildTypes data_source, + svs::DistanceType distance_type, + size_t num_threads +) { + return build_from_file_dispatcher().invoke( + parameters, std::move(data_source), distance_type, num_threads + ); +} + +// Build from array. +using BuildFromArrayDispatcher = svs::lib::Dispatcher< + Clustering, + const svs::index::ivf::IVFBuildParameters&, + AnonymousVectorData, + svs::DistanceType, + size_t>; + +BuildFromArrayDispatcher build_from_array_dispatcher() { + auto dispatcher = BuildFromArrayDispatcher{}; + register_ivf_build_from_array(dispatcher); + return dispatcher; +} + +Clustering build_from_array( + const svs::index::ivf::IVFBuildParameters& parameters, + AnonymousVectorData py_data, + svs::DistanceType distance_type, + size_t num_threads +) { + return build_from_array_dispatcher().invoke( + parameters, py_data, distance_type, num_threads + ); +} + +template +void add_build_specialization(py::class_& clustering) { + clustering.def_static( + "build", + [](const svs::index::ivf::IVFBuildParameters& parameters, + py_contiguous_array_t py_data, + svs::DistanceType distance, + size_t num_threads) { + return build_from_array( + parameters, AnonymousVectorData(py_data), distance, num_threads + ); + }, + py::arg("build_parameters"), + py::arg("py_data"), + py::arg("distance"), + py::arg("num_threads") = 1, + R"( + Build IVF clustering over the given data and return a sparse clustering. + Use this clustering to assemble a searcheable IVF index. + + Args: + parameters: Parameters controlling kmeans clustering. + py_data: The dataset to index. + distance: The distance type to use for this dataset. + num_threads: The number of threads to use for index construction. Default: 1. +)" + ); +} + +void wrap_build_from_file(py::class_& clustering) { + constexpr std::string_view docstring_proto = R"( + Build IVF clustering over the given data and return a sparse clustering. + Use the returned clustering to assemble a searcheable IVF index. + + Args: + build_parameters (:py:class:`svs.IVFBuildParameters`): Hyper-parameters + controlling clustering build. + data_loader: The source of the data on-disk. Can either be + :py:class:`svs.DataFile` to represent a standard uncompressed dataset + distance: The similarity-function to use for this index. + num_threads: The number of threads to use for index construction. Default: 1. + + The top level type is an abstract type backed by various specialized backends that will + be instantiated based on their applicability to the particular problem instance. + + The arguments upon which specialization is conducted are: + +* `data_loader`: Only uncompressed data types are supported for IVF cluster building +* `distance`: The distance measure being used. + + Specializations compiled into the binary are listed below. + +{} +)"; + + auto dispatcher = build_from_file_dispatcher(); + // Procedurally generate the dispatch string. + auto dynamic = std::string{}; + for (size_t i = 0; i < dispatcher.size(); ++i) { + fmt::format_to( + std::back_inserter(dynamic), + R"( + Method {}: + - data_loader: {} + - distance: {} +)", + i, + dispatcher.description(i, 1), + dispatcher.description(i, 2) + ); + } + + clustering.def_static( + "build", + &detail::build_from_file, + py::arg("build_parameters"), + py::arg("data_loader"), + py::arg("distance"), + py::arg("num_threads") = 1, + fmt::format(docstring_proto, dynamic).c_str() + ); +} + +// Save the sparse clustering to a directory +void save_clustering(Clustering& clustering, const std::string& clustering_path) { + svs::lib::save_to_disk(clustering, clustering_path); +} + +// Save the sparse clustering to a directory +auto load_clustering(const std::string& clustering_path, size_t num_threads = 1) { + auto threadpool = threads::as_threadpool(num_threads); + return svs::lib::load_from_disk(clustering_path, threadpool); +} + +} // namespace detail + +void wrap(py::module& m) { + // wrap_common(m); + + /// Build Parameters + using IVFBuildParameters = svs::index::ivf::IVFBuildParameters; + py::class_ parameters( + m, "IVFBuildParameters", "Build parameters for kmeans clustering." + ); + + parameters + .def( + py::init([](size_t num_centroids, + size_t minibatch_size, + size_t num_iterations, + bool is_hierarchical, + float training_fraction, + size_t hierarchical_level1_clusters, + size_t seed) { + return svs::index::ivf::IVFBuildParameters{ + num_centroids, + minibatch_size, + num_iterations, + is_hierarchical, + training_fraction, + hierarchical_level1_clusters, + seed}; + }), + py::arg("num_centroids") = 1000, + py::arg("minibatch_size") = 10'000, + py::arg("num_iterations") = 10, + py::arg("is_hierarchical") = false, + py::arg("training_fraction") = 0.1, + py::arg("hierarchical_level1_clusters") = 0, + py::arg("seed") = 0xc0ffee, + R"( + Construct a new instance from keyword arguments. + + Args: + num_centroids: The target number of clusters in the final result. + minibatch_size: The size of each minibatch used to process data at a time. + num_iterations: The number of iterations used in kmeans training. + is_hierarchical: Use hierarchical Kmeans or not. + training_fraction: Fraction of dataset used for training + hierarchical_level1_clusters: Level1 clusters for hierarchical kmeans. + Use heuristic if 0. + seed: The initial seed for the random number generator. + )" + ) + .def_readwrite("num_centroids", &IVFBuildParameters::num_centroids_) + .def_readwrite("minibatch_size", &IVFBuildParameters::minibatch_size_) + .def_readwrite("num_iterations", &IVFBuildParameters::num_iterations_) + .def_readwrite("is_hierarchical", &IVFBuildParameters::is_hierarchical_) + .def_readwrite("training_fraction", &IVFBuildParameters::training_fraction_) + .def_readwrite("hierarchical_level1_clusters", &IVFBuildParameters::hierarchical_level1_clusters_); + + /// Search Parameters + using IVFSearchParameters = svs::index::ivf::IVFSearchParameters; + auto params = py::class_{ + m, + "IVFSearchParameters", + R"( + Parameters controlling recall and performance of the IVF Index. + Args: + n_probes: The number of nearest clusters to be explored + k_reorder: Level of reordering or reranking done when using compressed datasets + )"}; + + params + .def(py::init(), py::arg("n_probes") = 1, py::arg("k_reorder") = 1.0) + .def_readwrite("n_probes", &IVFSearchParameters::n_probes_) + .def_readwrite("k_reorder", &IVFSearchParameters::k_reorder_); + + /// + /// IVF Static Module + /// + std::string name = "IVF"; + py::class_ ivf(m, name.c_str(), "Top level class for the IVF index."); + + detail::wrap_assemble(ivf); + + // Make the IVF type searchable. + add_search_specialization(ivf); + add_search_specialization(ivf); + + // Add threading layer. + add_threading_interface(ivf); + + // Data layer + add_data_interface(ivf); + + // IVF Specific Extensions. + add_interface(ivf); + + // Reconstruction. + // add_reconstruct_interface(ivf); + + name = "Clustering"; + py::class_ clustering( + m, name.c_str(), "Top level class for sparse IVF clustering" + ); + + /// Index building + // Build from Numpy array. + detail::add_build_specialization(clustering); + detail::add_build_specialization(clustering); + detail::add_build_specialization(clustering); + + // Build from datasets on file. + detail::wrap_build_from_file(clustering); + + /// Index Saving and Loading. + clustering.def( + "save_clustering", + &detail::save_clustering, + py::arg("clustering_directory"), + R"( + Save a constructed IVF clustering to disk (useful following build). + + Args: + clustering_directory: Directory where clustering will be saved. + + Note: All directories should be separate to avoid accidental name collision + with any auxiliary files that are needed when saving the various components of + the index. + + If the directory does not exist, it will be created if its parent exists. + + It is the caller's responsibilty to ensure that no existing data will be + overwritten when saving the index to this directory. + )" + ); + clustering.def_static( + "load_clustering", + &detail::load_clustering, + py::arg("clustering_directory"), + py::arg("num_threads") = 1, + R"( + Load IVF clustering from disk (maybe used before assembling). + + Args: + clustering_directory: Directory from where to load the clustering. + num_threads: Number of threads to use when loading (default: 1). + )" + ); +} + +} // namespace svs::python::ivf diff --git a/bindings/python/src/python_bindings.cpp b/bindings/python/src/python_bindings.cpp index 30e03acc..cab9debc 100644 --- a/bindings/python/src/python_bindings.cpp +++ b/bindings/python/src/python_bindings.cpp @@ -20,6 +20,7 @@ #include "svs/python/core.h" #include "svs/python/dynamic_vamana.h" #include "svs/python/flat.h" +#include "svs/python/ivf.h" #include "svs/python/svs_mkl.h" #include "svs/python/vamana.h" @@ -27,6 +28,7 @@ #include "svs/core/distance.h" #include "svs/core/io.h" #include "svs/lib/array.h" +#include "svs/lib/bfloat16.h" #include "svs/lib/datatype.h" #include "svs/lib/float16.h" #include "svs/third-party/toml.h" @@ -59,6 +61,17 @@ void convert_fvecs_to_float16( } } +// Convert fvecs to bfloat16 +void convert_fvecs_to_bfloat16( + const std::string& filename_f32, const std::string& filename_bf16 +) { + auto reader = svs::io::vecs::VecsReader{filename_f32}; + auto writer = svs::io::vecs::VecsWriter{filename_bf16, reader.ndims()}; + for (auto i : reader) { + writer << i; + } +} + // Convert fvecs to svs - typed template void convert_vecs_to_svs_impl(const std::string& vecs_file, const std::string& svs_file) { @@ -91,13 +104,14 @@ void wrap_conversion(py::module& m) { Convert the vecs file (containing the specified element types) to the svs native format. Args: - vecs_file: The source [f/h/i/b]vecs file. + vecs_file: The source [f/h/bfi/b]vecs file. svs_file: The destination native file. dtype: The svs.DataType of the vecs file. Supported types: ({}). File extension type map: * fvecs = svs.DataType.float32 * hvecs = svs.DataType.float16 +* bfvecs = svs.DataType.bfloat16 * ivecs = svs.DataType.uint32 * bvecs = svs.DataType.uint8 )"; @@ -165,6 +179,7 @@ PYBIND11_MODULE(_svs, m) { .value("int32", svs::DataType::int32, "32-bit signed integer.") .value("int64", svs::DataType::int64, "64-bit signed integer.") .value("float16", svs::DataType::float16, "16-bit IEEE floating point.") + .value("bfloat16", svs::DataType::bfloat16, "16-bit brain floating point.") .value("float32", svs::DataType::float32, "32-bit IEEE floating point.") .value("float64", svs::DataType::float64, "64-bit IEEE floating point.") .export_values(); @@ -179,6 +194,21 @@ PYBIND11_MODULE(_svs, m) { Convert the `fvecs` file on disk with 32-bit floating point entries to a `fvecs` file with 16-bit floating point entries. +Args: + source_file: The source file path to convert. + destination_file: The destination file to generate. + )" + ); + + m.def( + "convert_fvecs_to_bfloat16", + &convert_fvecs_to_bfloat16, + py::arg("source_file"), + py::arg("destination_file"), + R"( +Convert the `fvecs` file on disk with 32-bit floating point entries to a `fvecs` file with +16-bit brain floating (bfloat16) point entries. + Args: source_file: The source file path to convert. destination_file: The destination file to generate. @@ -213,4 +243,7 @@ Convert the `fvecs` file on disk with 32-bit floating point entries to a `fvecs` // Vamana svs::python::vamana::wrap(m); svs::python::dynamic_vamana::wrap(m); + + // IVF + svs::python::ivf::wrap(m); } diff --git a/bindings/python/src/vamana.cpp b/bindings/python/src/vamana.cpp index 603f5007..8fb7b434 100644 --- a/bindings/python/src/vamana.cpp +++ b/bindings/python/src/vamana.cpp @@ -546,4 +546,4 @@ overwritten when saving the index to this directory. )" ); } -} // namespace svs::python::vamana \ No newline at end of file +} // namespace svs::python::vamana diff --git a/cmake/mkl_functions b/cmake/mkl_functions index 9835ea96..9fba8595 100644 --- a/cmake/mkl_functions +++ b/cmake/mkl_functions @@ -3,3 +3,5 @@ mkl_dimatcopy LAPACKE_dgesvd mkl_get_max_threads MKL_Free_Buffers +cblas_gemm_bf16bf16f32 +cblas_gemm_f16f16f32 diff --git a/cmake/options.cmake b/cmake/options.cmake index e374a548..c6b833bf 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -92,6 +92,11 @@ option(SVS_EXPERIMENTAL_ENABLE_NUMA OFF # disabled by default ) +option(SVS_EXPERIMENTAL_BUILD_IVF + "Build IVF implementation. Requires Intel(R) MKL support" + OFF # disabled by default +) + ##### ##### svsbenchmark ##### @@ -161,6 +166,7 @@ target_compile_options( -Wextra -Wpedantic -Wno-gnu-zero-variadic-macro-arguments + -Wno-address-of-packed-member # When calling Intel(R) MKL GEMMs with BFloat16/Float16 -Wno-parentheses # GCC in CI has issues without it ) diff --git a/data/test_dataset/ivf_clustering/clusters_0.bin b/data/test_dataset/ivf_clustering/clusters_0.bin new file mode 100644 index 0000000000000000000000000000000000000000..9e5f956609e68c5c33df8d8dac9962baa0785ddb GIT binary patch literal 41032 zcmYh>1(a1~+c4k*GYlmS206gcNH>Udmq<5ADBYdX(hVXlB}g|CBHi5~-O?%XUxU2s z`}bNG2WHOM`?=$}_a2~c;J>5C$Jdx%v+@^5=o=vrSjJ7JMGORP@R*`Ofj~)WFp$}l z2o3}~a)i57h!hA6;&)nw1Ok~O2LdhV!c_Kfo)}RAfqL|!LDWE?8*_OAZ46Dpd6FgL*uxC zK%97iKmr=EjL+i-0&_T&zfM~ z=ea$F;YDR)`q;GVcjis2jAuj9H0d?sR=Kw$L8 z&wS}M6PYQ=D(>=(63*9%PE6q!F3>1qAdoCb{pm(Ob}-nr=MgcIoRENWe8Whp>7DcB zw7%xdWIuO#!#jObOs@=J3y}o&Ewx$839{;qpE$-9?u#?3{zyn-rZa~%Z0A1zP(zQ* z=BPe-Km@tz!%S`wL$B1JGZ*QnS90i+UX15=?h{$xv||LDIKdSX$#Z(LGLj`+;3l{D zm#F$`EbBSPZC)`!efE=Ce^sX;sq|QLHqlg%{i3H<(@ih^%nc%_=_iDenqK@twq$yQ z8KLsXRT3vx2X2x{pS7en8`wkalz~72zNI0HNT}bg@RSLu0)gcmWT2i~z&l##lkg)4 zCD_M%Li9-ts?(6x#E%#Vd_@nAurf&R>z9f&qrV>6NW73hARTR3!h5|DNpFNumxla6 zBRw*lL1xNORv5q|IKcC%Pq@wySX9HJwPh>rCjI#P5qq=V- zmOAHTCR>QCrlZ(IHFZp)hE?gnR?hLBH0n8>{%Sc;EvGP#3#3=m?`h3#nyG7F#&DFY z3{lhj)Kk-1q56-GjG%CGdyJ*jO%VvBRkxkwRkQq5qz~EEZ5q?575MKcr6#NRhmqx6Qhjp`id39IeT%k+@Z{+93S@*Gi~%TSf~q?g|wETN52c0t z-XfvA=A|Yr*&?TH#E>h^{~>NbK`)Ka^KTqRWf%Fu~X zoL9e+>eq>Yx}~EIUFoBSzjKWNq1MSpHj_my^HPjYQusbmQ#wECxXwg1Y@A9B=+88@ z3rZ6RNbiSZyt?g_=T>qYUtXul+bywHa{g`JFT@W=Ebq%sb#-m0hMAn89iRDJ4Yf(; zvl)E0qMZFps1NoM>n}tSV|lTx6~jR1&1b!*oqM*mgqJe>Xy#lCoa+=7oUf5J9HqQ` zmiPHIKKn(eYlv|Qo5kjBAC7L3^sXNI$~nf_v#p$TrCK($h9>H8MP7e%%^Px2z;!!{ zBmD68fY1ZV6DZ9{#v_s^0&16%9OR`DO=wF$#xs>)n9FvaQcBIL zQk|CcU?j8H#%>OCkLLs{e0mDdijH)mKO;HKb?)7VGv}Z7rSw(>WZxxzEv5!H;!PCklIf~r)f5d#^*3>Fa0 zJc>tV@=}bF)S@1<+0IT*@HbJ-_&gM#Ncg;`G*xLqXSy(i9h~MR5zVgXB%?6B7{NHE z6W?6@lrN}AXGXA#y&U2+*SX1UdYPreI7rM;_i3{79mlywAh~@;9Fo$QIou~=3Tq-Z zaj8SRl=^}(%w{QTctDmg=jKmNaE`m=N+oABp#vis$5fWFmi25RMQXjlH&mw?T^Pk& z{vv{9haac(_niNJ8z}5`63ci)GH5-?v5}M14$^xp<&$7_V+~K} z6iNQ65TdWy!)qd&rBUh6O-h@c6&S%3Vw;m`$V?sDagqk+U~hh7GyjmutZPD3X0U=a z#52?4Q^KsOLQP_uLs^)?V&a)M-*AA!W=S2Evxj2lLvM@p66R+pF7lba4C5=x)0PW-mRc?-OLzM5fCz#Aj`!Y^>ErLqdadFc z?@1X^{v&xtAeY|xRKGkWr(PJ#KsAq`wmI2GRkggYRxenqP94-Gs+uIDr&<)1>p9_h zm*bprSy1kJ@;h7PtEn9Q#3T-dika#1a*i_MufcvYi@O+ING<5x=dI(M%JSj(RZY5l{QX?7 z8`;5qQg~BsM)Ma%)Nm=AeP%nAgY_BJ*}@K1tKkbWSW6CabC?clSY7R!GmV$LCZqM% zB!-&RepzoK&t)cm7~IA*szGj`0sq z2n+mo{O+Z)8s6n`h~89}1L|;_mFjR;{(rY$uF83G`Td?Ia`{x=X39|)dFf3OxkyKL za*#+qDiAF8k+c_U4|0g@d+RJ@0e_Lq`m$P2DQ*(s@QCNUCXMrCWEgW;$VQ&>hBB_zgw}MR3q2Xf1m^HNtGPh1UjKxcWFS8y z_=VN1|*!j&qJXqzG{@AQwfcNqxpKo{21A zF*`ZIB_8pN2x^;zP%`or1*uFO8nBQxY~m_Uh@clj$;TkJvW?5!;4Vq@N;w8HgjD(_ zBZXPaQXUYhm#R>W$^6P9PIHl~JmeMc_}sp)NM|N7jT!7Bt65Nify`k!*GZ;V^HYF= zG@~O!n93IRaDh+Ej`)hM@yJeDs?m;tOkxXLc}s5d=qIK! zkKb6vA&zmLhkRjXrJ*EMs7777(v1mB<{mNazfe+;l2XiJC8xPU!2T@CLKgFucLbYt zA>`&88qkc^bfGtkS;KygaGghdVI$_I1+8d9PsTBwUpdHao>IzOs7yne(Ul$yU;;<^ z%q&e#KE9$b)oDyyI@6n>tYf@9wg*qCyAkl=*kVQ%GF!sTi-UY(U)S-TuB@=P^4c>M)S$NW^rhCv z&kgZfJc7N+3+lQ+FR}*7H_3v{TG|solIPkG^N$p+mxFQqOG@*uDp$y09UBP$N6ErW zBA7{Unw}q_@6`&i{jR$M^Z- ztdhp?qaw3d!CI~oDS~?@<=Mq6GN@evzNS2q6F+hEX zvxw#F<{t03Z0&j7>q^jqA7l9_Yp*4se3Y#MBRMm`+)VsYYKyqONXH5p7}`K+Wr>a&Ey9HqNn8P1;^B#IdwjZgTJ zOxz%jUP?wPD$t2(?Bynr_1Ne9zyPMQkw3XcD!q4$2KsI~%hIOpg;rDhkqsZcL|TDm_BB)MhT<5iO0`8u;({%}c}x z{z`)aEGMg;-9fe>_h*`NgAu`IiT*685C0&#UQ9|c_Of38T_TmnWpZi4l| zC(Pt3*Y!dywa>*8?(%|n>RnrHW2xt7OyPza4p6U^BuQ?jv4x51P&B0;C$pSpk<0S* zWGqtm;kYFh?f>BjleZf3)=S<-F@ZhIkf*&w^WJfkG=I0SG(2t}t}muZ;c;1GY3oSu z`cnMi2yd}_-nUIYGRwtU@g4BKC#n8#&P8_3E949DOyIofpobvvx&KnR@ zOYvCdhhw0N?Dqb_&YAt=*OV@hTt9s7O-1oDSor7b zO+S)bLjzt=M2*W?dn;>crH(noT3P+tsNFy&QpOr;s9Q5;lGK{NC6yX=p_KJcW)X`+ z{ahwT8^!ZpJki4R8DG+YY;u!GjeF*V+<{0rL=%v^*J6A4hi?0P|ct>5aPNK4!eaB99NFnz0Jm+72lZPOANFoQ{ zFoPj-aD{&uC>L?WUYi_xwuATwlT++@dCVNKe=XKQ#1n5h9`i!n|Eq6q>)h!aDScgd zAC79)ywsm3fBdsk1=3QK=3L+<`80TGmU4)*ya+#ulV_(qRcXmAws4nMd>zU8_=BZ# z*-W4P$Sy8%jg)fx4-fcRUKeqQeDd3iKS*o+xhSQ#l3RB{dUK1sa$SSp_(cBWlAZz-XA!%(#C=k$NondagAVF5ghv!tr(fC3d4lwAp78#q4juWCVJu}O ztN2Mx7pY~4TBam5d1yp0exi&%uS-)paGU#tsaY$MsC_buFpE`eU>g@nZI4>SNMSp6dL<5~}LK zy+qMht?0oU;_1clf&Y%|Udqve>qLp5$H`Acwh}d>p9zBO_h4&gH75y<!k+4W#i8Zw^)q|t|&DN1=N5kXJJ=S%9-hQ7?=HxlW`(rluDUR=x*V(GYsiL50f`85H*#5 zUtk6|`9vQjr3#Zdg*N$ctd!@h*4E1Bw9kj*qaVX7;wr! zJzCP5IsC~kPH>LPJRvB8&l8VCWFZ?lC`=`q(~5TVV-YJ@#~w~{o~zvD5s!&(!C#Pw zwB)A{9U0D8rZAQBT;w$&;)+H*5)evC(o>kSRHPA&X+kIZFqQ@U$tliqo;Sn@R%^18 zpW>9KCUvMwQ+hL$(Trg>kBJmXtw=&LQjv|~)S?dcX+$&H)0N)zVFo)n#u=XQil7iP zfJh`HIcdqs_q3ob?dV8f1~Pp6F&>4B}9nHngP+UFk+21~ZhO7|B@1F`1?8<_b3ma!-mx zVzQB)+SI23jc800exMV5SjrmyU>7I2#1sA{qI+37GLoCGDME3Y(VUj_U@$`&#&{+& zlZ|X*I|n$+9qtm*{V_71lZ;T(@+CRQMn$nK;bf+g{n9m}1v6~b8&1=FuyJa9N`S^-Lw4)ybn8fd_WEJb! zz#-0XmWMpyC0}^{OF<5DQ;14bp&2b`O;^S+mia7XJ)61DLmu;lSe_r_keUo+BRi$2 zNMnXFoteyMHCx!nA&zmAn4a^#AQ_>gBLiQOgA$aZ8g*$*dw!%pquIi7u5y=0Jm+8H zdbW*ES~8H67PO`#eHg?LhBAp6%wsW2S;=~KaE~~ihZB&PT-2p0-_wmjOk@$O*}!JD zvXk9h<1P<)$P-=@;<-998Ocu}%2JI+d`Dy2(UtCuU>p;e!ZhYGk0tEoFef;}dHyDD zs5r<#Hom7VJsH9%=CF#@tm7|svxlP`;~clSN0j8A7fDMtvQvoaG^Hhd8N^7&F`lW+ zU_Ogk#cI~Gi=&+8AJELvTv-i0H&6 zJz2?11DZ0JkxXO;D_P57j&g}>+~YA%2nsVh2_-wZ$VX}F(}K3NV<02>nW;=;7IRt6 zI`(js*ZfPwRIWiJl9Gpf)T1daXv0uuvxq<0#cuX;lq)>sDX)l~+WnH0WFRxy$w?0e zGl?0@Vl!L#7muMIj-P|XK`hTDy*)z=cYo?@R!=60ey?f$@YflT$S3h!$OFU$mbtJQnfaigd9Hg21 z^BQ(r+f#mXZ#qv2Ydp_IvY7RUscBA6qPE$3jHe_t6Z6uNZNxO!=9)eK@GAV(%y>Xe z@y56BKIKa`bCs7Q5_}d4Q;MnVBd%U=$SaoW(dPQ87k{abP(K_k>j@&5x!mKj z`b6}B_+(@_9j)U}+PlyU>#3w3(^$ZCi;e8rzBu30lgr`MDTSI;cg;uSRgbu?-JiGI z6whPN9sFKxb4~Kg)7SKu&rj0m>5sVyf2qVXxrrN5PcWH-944`RRiQ3z zSx#0tyT?;f$XfkV*dX(t@76qKb8+#AoV~nAUXRJGJRa4E4!OA--b)Z-}O5X=%jw z{J<)9a)w;?*KRITUR{^*shYN?JM+|ZAJ2%Qu1i^@wprD*998Mb7@qJ@eczE;jSq64 z_gq!qPt>{*Gl6rv>G@f(+k z7s>pi01a5d0j_e5M0zVJ-%y7&oZ|{v1yF?=Oky6}C~K~MPgi;~n00I>mW*ZvtGG`}J(`b7)S(%j*u{Pl=-DKENedRRfj@XpJiS|n4h&}=f3cI3oZ%fM z_4T*>!e-9$kXZVA9C!IePyfyeR`Hxl`g$18DX52wvY0LG=OHom?iXxhHwU@MHNMo# z4cSMKeojCcD$#`#oZ&5z^ms1Hagy_N)9X{XM+N=ffXYn>4R2YH+@9bRiBkCc9IDfVLCj(a_sE~pGdUwzz#%U4FB!wkaO%;B@%+X%qNOss z_<|~QVj+v^oZ5Yf16<}MQPce2{WFz4IV+->scr!|tU-ObOC$b$Vk%+XC0#$iZ-*bH zsLV0C`0Q`&BbRe5V>gdz7Ua*teowJT*2^?|>Ld?HWDgBzA6x8?(e}VK!u0AnJ(@;O z#?gz1S#Q?7aZ0X>Gi@rnp*cr_7PcKS8ybpSk4nsJeMgMDo%0L zOW~PLY*iS*1G&5`XF233AV(9tgdf5S)TTaV{9y>A*hCw#mJ(|gaTaDSXGtpF5&XhJ zw(y3iA?iqRdNGqI^3jo1Jfo?c{K|8J&6m%}#U>8&m`HLKjo5TxB2nb8JoRZ#3l>t$ z{20$t-Z9FYn9d`qW%ORf8P;J^Xf{Dy0zgi7qHQV@$ zNNQJsnzUpd+c`u;v!gq6`J2nUB)@)~#%%6WK<#_5jc$5i4=*XI2e#>fQ*_q@v-y)M zdY~DX3DX0?>fe~ToYMpGLd`5H(v%b2Bw}*;=1aOTmap_lRnBpXJG>>bp4rQ50%7i5 zbYU2YQn?oAsgT+;038^?uV|SM$IW2p3Xwapr1|)@Z@unjX9Qo1C@*yMfw+33u-X@8 zhkDOb%O`5KM~%{|$4#EnJiHe2o=<*{@I=mTIsa5S>L)jS?D6H?lAES-b4N~I$Vp)_ zHD(OKVyM6_28&^UHJ`_FKO9H&)kfDI>e^j>ed))q3nfsD$#l2=;ml?|`?xnL&j`U(DU&(DHS~7qoTqOJ--aaEGMHt3D&T^aha`GwFsYg4$kfZeEq$6Dz zz)3z)uN0)D3MV+nd-AB`_hePe0W4%EkBO$PsTj&Qj&qXN#IS#|^SOP~l_9(&r9Q|; zAsR57<;2z-eL2n}V%SqTXh0`6@TJ~a&Sqljp*(El4^rx-FKNpT?vPDyeM37=5mTR~ zqY@GITMYdchxC*P^>_Q+;t?;2kX-C^U@zyn%olp|TUzrw>j_C|4Wtc|5B}jj;r}q2 zmXkapNowC`A&XefY0hyQZSmo_>DM|Dx~U+(ZQOJ@w4^*?ph%q-y1EWo(`5j zwHzW(W5k-s_e=PE_z}L0+;pWMd)dz^B6!aiB%vNn8Npso@tRMaKQ)zTOK)cK2bW0_ zF_GO|;5x7Q(%LdpmMZ+pAMED^c_XOR`#H;9{^4`=DM~R) zP?7$OV;)O5&A+@Smc3nuw)E!=ZwdcTwo%DM5h^o+RjlSMG3@V{3}q}obDqfdcxnpK zn<*USB@y*SD)LZ{fqDGNUS5&Z z9LY`riZOz9Y~oMu@R)bRvTssRjq%Lo3<0wy9*Iayeu~kRDXd@<|L}=96q{sZSEPkis)6 z|By4Keak?`GliLKXAcKC%N-(xd9I}+eObsN)^mzWyrEz!&o<2B46k`d`qZ93C`lc9 z^AqdXjnVbt=r70h#b|qdI7Wxag*hf3*au4fa-^E8Ib=^f`O{_%58 z6WATWUZ;eUe@iPmGl>hqe&&xPR;Ief1vBX>U0iDe$;^_h6f`61QOr!}N)-$Hk+Do* z509;_v$fT<@4B*th}L(Mj`m4cYT6$QI7LDGVK5i;bY#8Sg!YW5yj~p1aYhSb4rj&p zhEif|u17jDnzwu-S8W(hl2CJpaTHPO&g!w0wPchVsrYa#b}p&=a2)V!68Q*~gQpyo zgDv7eZS8fvDTDpk#=7=1-{(HHhS;t5;)ggF=rNU zm~%}R^Ib7b|M-2W3Jl@}MVug=wfxFF?$J8PJ(OeQQ?pL2XEQO?E6WolRX?GsXDiz3#+)yOVWxXE4ArNeD%-HdqUJbJGH387S0h#FEnB^ z$BC;ia`BXedN->1-Iabs6?{=z(wUJwAgA1a!_TZDyE;^(9!ojKHInPAEOPrVZz&?j z6{tmBqR4kn4seD~*U~#6gn-fG-yM^566%+Ktb`tB0 z(tOK!4so93`lKH7c|;UF6FJn+nf$<~dM6c``GM~2W*^B@*e~4X3;mUptt1XpXMSQd zNBAU_S;tsj5+SvFDCyY9c6RbNvC{nCylLWmNqkd=56Imu0;1^c<39!z2< zCpb?~BzuvJWacXh(V8wyWEsoZ%yr%qKg9Rx&k9!Y7iYN3171?bUg^!x+~6P5*e~g* zMQ6G(hrig)F3xbB81_&)GLVB(l;?Z8)0Z*K;WW4T)ILi?Rq8W?)tuo8Z+J@rd+;+7 zl8&z!$ZR%piM#y6GhUO-{!B-0exMISnampg;tV%QWbgK68r!(R6JGI}B=&O|s#25o zEMWs%Il*P_@}3ZlG@niE<{*bSPa?A*DPfeN0u^aO2PQC?8Qda@8Ig%P3}-AWSphF@69CcDok-|(47HHVFer6!wZ7Uu@&s+8ZphVc#P$Dma?3^JRy=f zmXk)bq!;6uz!FxohQHX!5zg_57Ywrbk8q7!JR^;{H<%#|<2RP@l$X4ru$fqqPV{3O zYxs*j1e=XX$U$w|(3Yu8V=w1;OgwWl2R#_ZQr2*kTRi1Gxy;rAd{1*avXJHM;WiI= z$UEK>5~{A$q!XPP$}~1`npVkuo&JQR@N+56=|V61a*f0(&2q|6iBZgAAJ0e{rf0}X zHY(GLas0v@ma&GjT;>%?Qn|NLhdzvACs(;m!_;OfP3goL9#SUo-|@YdRt#e&|3*~T zAb-yi?B{?;zGkj1Fs~NT#0GFJ_q^K{>aHd=GgFI~O)w@6IzoYEcb9YFtuQF1bOZ>w~ z{SjB~TXBJ6YMLojz18Xw(bOd+eWGN-toD0=c0~v59KJ=$SsCK z^kpW8X)UH>tP@kc5cg-j(1|l-7h6eAQd)eK^g?%bbDY6q-9c}0<`L&rGU$tId@A1G z`By)@A(hx)kWKv6Il%=o$U{H%pUf;4vWy&Z(Tc4kl#d-eB)XgwWB^y?=3nCIf%2^5 z5z+KP4F+(6hI(RlsJTqZ6oEhk+7ctBn$w2)oaPgG>`h)h@h{T&;W+1eQuyIWFBey> z{eOSnCx%JB?@}L*?|fq=XDJ?}?(*A{?ed#WUW@RXoHm!kPvvj3{3UjdX6zArCUG8P zuHGLfzR4`*h#trzzJ)FnT}*X}7;10oiB~D?!H?H(@3jjDxh{ZO)=|(pjuXW?TCtE7 z1d@P}keNskV>hV22n8P6ZW(0FdpieT=N8Tp0iOnRF z!)%o2f}Z(79xusbBzY`Hedh9l*DR3ReN>j)4O9-bN91=j*ZEwYFOnmrpHZkywJ^O# z;Z)W{Ahq`~kCp75u08$SfAKfz!kZ z(eLD?09_fwA+GU|*!m|w)oD#n2JsWa*v&o;bCD}NC$?Uz&sf${NUxQk0^_+x5|Fa9L99!o`9 z8qtAX%w-uJMDXw#eyF|CAN>M)dHOkp8G zW=XPGmEuAeHDyZ)UKV zlRV}*uld|;N=g>KqAPRwjVEL@qq?$*XXH1l3Q~n1XiXPpvy4@2B&nO=JJ|o$>oeN+A@kUOk^#8aft6zm|cuuG~-#$3Jwz|r8ua-545He z!&t^fws4G7#0~TJALOI}ZCJo!c5|F-+#zNv>tO|}*uxPn5oYstH z9E(}P26l0YyF^Iye{;NB@c;ciZQp?VZ3Ox9wJ!F;`tUx~XE*dwCOvRP?Y7FzVQcJY zEw=TCV~t-oM6?$<$1^Slc^+_{Z$tEuIhWLaeyr!(>yPfvvrU~=r?8J)FZ_svQ55i{ zKsxf$kx`6c9t+sSRxa{4DSbX2c_~CS!vEf948OC6V;tu#39Riil9GvRG-En5n9UmY zvY$g-=N+-dl!Y9Wp&A{S!4j6Si@ogQC}()br*e^(Qq-d@y%@%JE^&+Jydr{xMB#Ii zQt2PsDKli9yn=eWZ&TH513vV_%aCKOfW-aU4#arGJ$t;ONG?I{-Y}BA8^=Ly+dNY|{SiG(TLk7!3%#<7r9{J{;P zhv`oek%_|8=6imiH9sEU zMqdX0JFd$JocKG*NJ5wCZZw9%)T0o;y#afLr(Lq9z)no5;Lt2>xpEReMUZ(vx}(a z)Mq3m2cwwBGxD2J2?_lF^*%$YyexJ^Rd;(?l~@exNfO>12Lv;V&Ya z8PSPB4VDwlTsTBo`@SNRxkNnsw+DTh$UfrQza5BXkEWp*wV202?(>$)_G3(YEf#6X z&bLIfhm!F%ttc02Z*!2r`g|HIiJ!tgrV2yYPL`DNA7*|0!BrlSQ*ZU;1|h_yiK)-7(?oBLVlvfXJ@%1W-!vtGp80{^e64qC6IJibW`JZ3VjIDFCYBy4!AkaVp9=cr z4|ejDmvq!K2YA3Nz0z5q{K+F;P*h*^W+A%?(IdI|n#NopM30o@I|eeC6`bKYo%PHn zPLep({H8zclH2qA&JA9$DTV8^A*H?#(|4p~3HykeS}lm62f~kta{0gC7Z!umd^o~< zu^8LbKUJ_jqRzo;ctM>ms!vUI>B1y+IK@66Y$tE0thbr#+$PWecb(#L^}p-1^B$-D zaFp=b3TmSiAC4K`^pER3;4{7RhCZ%)ivq5lPd&q2Cj*6?XB0(~d&VNKxInwy^yWwY(BCDkwKe<5?Ya%+>b2|EwWvX?AX<12^A#i5#A*KJEhY7FEsoG$ z+|$V=R~6-Y8avn~aS!-RJd1cn5^;SiKm8a(5xH+gYlf=vMpBA9Cryd1Pt#GCpUJIH zOYtq~<@YOnSDhwIB$_-Wm!}p)Na1>93Q(LXG^Q=x=)*XE;a66&o(o)f`!V-Sy`6sH{F z>#0ghy3&_^3}GFYxX(-8@Rq1S&c$bBrZm+V%uuGWkY_ySC85Fko^R<(KL#+6aZF?m z8`#YmZt#*A`tDQGkd80OPEPVukPu2%E?wx#NpA3(sP zPkv=PC%M2~KC!>!keC!Sqa)oJ$9N_&n*}UkEeAQrQ(o{dZ-{9gBp?&nC{1?;@G~>{ zjTNlr5Qn+RZSL`ySEMvg@=}0ubf!B$GM+iC;xuQ8X6_`W7~fEv2J~Vazp|KRoa8c3 zh;CNJ=Tp8Q8`;T8A<9yh7PO)R{TRg*wsV@>+~qM(c|$C-FEODMqXaEz%>q`ipCcUS z5?8pzGoqN8Uyz8@q#+$yC_-tf@B?Ewz*)}mmiNRrYcrFF5>%xcHE2mUhBA_|EaML@ zaES=!bVL$Sk@k$^XAX0fnE(~yfi)S)TE7|mGbu#~l&>nzM%>W zSj8@mkiz{THzlY{7y2@cam;2JLGB;HgbgMpvdXm)}{- zeopg{c>NIsD3UR&szK_v4s+K^AJzjBfN`4r^G?QI2tn>pbH%Q9{isJ|`i`NI@6{C{A4( z(ShmAWIp>j#U1VvDY@qf@>86WRG~g?=tx(7ViI$i$2vB%lf#_kBF}kGloa}l)TARD z*~vk9y0enYJmEd5Q`#>Sp%#tl$RLKYi{t#wE8Y_-%(!fhJ5^e4X@)_$Y!q4(;FAFmdp{=g%UxYDTy9zzcQ00Y-JBe$r{P?CU=Pz zB4;$>S5}kWv%qxT63sn51N}J83v#+w7iSQ8+>gsKg7)sWTRBA=_tB~ROIP>6^zL1` zDMCxeQ^@^kE~`1tb+WkkWM!`V#A_m%_t8np7EY4e{BFZ0Ld@b!^kX%L_|)93Ky`Dp zJquaKZW@@Cv-pcI%)`vIB%#?=*t{9XZ=7U+`7x0PyeE>`5QF68V;LLx#Qy)BH8yD!%f~W zSR8w~M>08lDsM;RCRi>Sag_U%l9xGLB(dD2U<5PSz+rO9QC05BLHJ=A;s3QLke@=7 zr#dxh!~iBRof$0V8V^bB92v+>QOZ$)%GBitdN7b_Z00Zz35lrod`=DuQ;PbuV*rb| z!*e32RdnJKpL`VHE9%jZA818qy3(Bk9O5d8)HgX9=)(|xVg{=?$RW;ghuo1oJ20QW zxzAf7=#QAh;tRf}3Z3Z2P-Ziat?cIpPl%_F5|E5+l%_1zsL2RU5zlW$h(}5mLX9mmon-F~+lZ1Rp0luaJ-|_?P>B~SSFp1we z$Ppsg6G41VDzcJ;{1l=H75`sR_5$Byxd#AzHlwl}m&~oS3v6Bh2Nvl|qjDkZXus!qEB7>3q)n`Fss~-}iZ*-}C$Zp8udN zo#;YOMlp>=tm81h5vZoh(1Moq=5^j>9NYPwTNF}v6{$omn$wxVyv-7pah<&SZAm)w zCKH*%A#(rIHG(SCC5E=NV>YWe!e#D|Uws#$7$ta|`g}k7GcG^Q!(ZLxoj7faWQL?#8up)nwdNiaj?=qfv5?IU% zeq=8vILQUB60DzwP>8~W6G1sD5KRnC7|aKZBc5rbv5HJKahd?VuQ;`6LQ~q&n~}^R zg;cV+Nt9+)g=ktcn0P)RiEHF=mq+x{FrFfsjtnA++00=cCpbf(UR#n_CNPbKEMhSy z$?rUTn95Y62K|X+98;J{Dl5p~HpQHqC8$FiI?#>ftY9tMIZg+Ceh?Fw$YPfA4eR-X zhn%s6C`MK4(uHBfF@q&6WgibZHzKG(9b#xp4+is3j&p%9=Sw)P7{Hs1WI35^;yjng zAs|fs6HEvdd4d>Tpgn!)$3P}9nVF=pjC8J2I^1Ujn$d-kjAaf>Sjq9aYZ)TSAu`GPBimi2DR6GL}iC5;R=v6-FxPN{O% zO>5fHjraMKnapD|d)UXX6pnI^F@y=s2gu=*JU**3BTx<#nb+D` zL>l+F8SGvnohk2;#&TLa8$$H{{Cf8Y*7LN!o2H+Z(+kH^Opm+4B|WI99y5e5NTH0L z(1F{OQOjG@+fQofwi=l(-}A}i`lQR%rgG|2X33T2Vd~vpC9;cb@xRP9alXOJVzJhG zE$jYuM(zgv_4_RI8=!|j>ihruyN%wv%X`~-&J^xnrWf#|7rbNrJ*;swBdljCS=MvK z4Y#SGH*RC4H6&R>su%2Gv~{$whTR-B|GDN_)%#D;@~?@tF-9l)F@TTBWG@%EMh*8i zp&$KO%0>#?t4FCzSNb!71pZ-RZ5crVvslSCc5;GWxJ-cc6(fdWY~?OZgZ-ZU1myEG zFm370WRm!nH3Z0o3c2~f%S>i5ODP~n3Q~?n4CVu-lg<_H5Foz6gi@CF^kXFBh-V5( zd`T)B*~~Wfa)4u;q`q8i%rR~fV&5xKjRq{BxI8UQEhaLB1$@aRg5*h4TF{k23}p@r zI7*Ow$j4mL$Ydu+DJV}GGmxQ-CWQ^`A)8vx^k-_8J*vpbs?=mGGnhpt1>|ZKTF{3$CXmHtuJ8wS<#R(?^BSX>P9jTK&bMsk zGS>)ozJ^nQCuq)le9BLp;3P%VNJF~NlRgaKBhpyLx9sEJ{7M@&^&S(M%yCXpTfc~= zH~kq-JX1+w6X&@>4nb-%jPg898`9avVII|U`tUx9EaofLk;Qd7tLd)v;x*!#!c3O2 zfg@b!0sSYOm*_+v=8!@bdpN*3E_00s^rY$}Q9^HuVjQ#ijxC(v4xwSrW8P&d)A@`g zEM+ARhszxzs7PnlkjYM>BgBn=v5SL*MEcA}E7~%Og{1QhMa%l^NhkU;jNv4*i@bVW z6qSkPUE=wU9sIyuibNTQ{(M9d%gA6OM>tEq^8fG8KjqQq0>z5ZAh9Doug_n4PPF>Z zR^Lym;m_4*FLicF-cOc8h2+ayjFtyA#r8pK-(s#`o5$nEh{J97FQpjJo1Gtfp%85Gm>>Og+rctqVEuyghx}c$JM8XbjW5f1UH|-}9N1 z!LkiJ#=U=1%D%q(*FB~iP>E{hvBi2b&7q97M)48TIY+}_y?~eKPcq+dgHU7CqX`}8 zMH)#aGCy}R*7Gv|BFDPMSl4+jGhHkfvznKza}IO4Kz?g}f@p@*T;J=%Xl97}XPo3V z<*acuQ@KZReX2A)$YXy(c$?Mi=MQc%ODr$ZNGzXYEP3>h_OueS`7GcoGU+T}T}WjG zZSD1DTB+@LimTsHF@Avr`;trtd$Wp7G!*+}riG~ks@k*PS+6Gk7U}QV%-pipR8AjZ zG5fhhOX1ADCUKXJ_H7*TtRSyFjJJmqNaGN-g8gi2KPEAc)f^?&-t@93h3!cNc5#@m z?8|i?vL|gQE#7;0QJl|EQhcL0A{W!ecZfLtP9AYgCQ=Mvq@ozU$y-zuyXxZAmHC8- z*;9PL24cnSC0^wak>aQX>s`8TJu_GmT}K{CfHo3V=M1p zx5V2N?qAQD$2ww-oA<99a=(|m4vz#(B$=h8^Bw1T$i2mh=6%L+k^I*8Bu~+Xjtpc9 z3rJ@xJJ`iO&e6@FDP*vkKPh1nHK|W0mavy(s(HF=U4+Q_F~^ko1OC}HoTS;sw^ z*!PK?;|WPxjT*!;mwC)5gMHkioBDi}I1-pgCWkmpUUl1&IOem3-5lo`x%~$FxWyf! z0F7IhhJF+Pp;asIHiLB%%#Uss!x-@1OS)3tHS^b9&bmR>tkxZL%dKrsY z%0(JPxsT=C<$?15?-g&`lbhoBxc3GI{P#-m=TLLV{rBzMD>qd2X+Rfx(1#Js;ByX> z+iUU@LMa*$OFOzThiuMK&|uAZg*e7BoyDwY6WcgLV32*H6yZb@LobFindv05khNrU zm!Q1rjmL;4ktBApmy?|0Hu;0)3stE@Bbw2dcs^q>DJ*3pyE({dE)bH>csxcW8qtCg z%w!g;Sj#Oc>pfLz$aEI4j6+-|LeF`cQOsch%hhl>G$59Cbf+g1NM;qA z*};BJa)&m0Ts(_dK?a#TpwE?|8tv&x9RDPP4P>!{A34lfO6!LWXh?qsvzRq(<|rq) zL?Qh$lu}fpHXZ0fPX_P~Blw6(B=9+_SjSd&bAZGALO%VqJ^dKLD8?|Jsmx{>>&fCL z4setl^69^!)FzhBOko<|bBaKHxF}_)KrQMMOGo-MnFOYh$Y*45nDdm=w=41#F|?#R z{TRdu#*xGZwo_35c!XzZNjrM+CJB7bEDmsxBV47Fz8^s}F?3@nOUPgk`}sGQ32`Qr zpgxUhLMQt23IiF!nxImGp;DbsX$d? z>C7~~;A^(BmxG+)3Xx&znC|r8H3l(^i6oH7X0~#Kok@)7&OJ(z;p8KF;zhm$^gXvhLwoo}&%p7|-V{-F_AQO@G~d5$ZhUWpuC@T0{(lY z1o{ll_#kzjSIn8rc8Ul49K!&{lafz-oj+xrEm0J7X0)OU_egg>tmR=nf49EkLE@#eI#{FxoUtbZE zwOka3RO?T+)*Eiv;!L@1zI}{YE6mSKV*J17hI=kNj~*Um46$hJzCS5o?#s-%v^noL z-!aA->bV;CN3$MZ%XqF*pcpfNVT|^k^At9ZDQskZuyd9s_GSoW%q_&+!tFzAV|HT$ zHOzA}|IpWGit}!9eMvkkh-WkNxnw@O&8NEgWK+$08k1v9L1I@vLa$(9?pJXdN-F7P Poy+96hUaKcvbFpN8Fp?` literal 0 HcmV?d00001 diff --git a/data/test_dataset/ivf_clustering/data_1.svs b/data/test_dataset/ivf_clustering/data_1.svs new file mode 100644 index 0000000000000000000000000000000000000000..5c760e8da5b373f0a009f6f148d950edcb6dfb43 GIT binary patch literal 33792 zcmeFZiCip59E$^iq(`;oX<2w)>>C+s%GNrsd#B7 z9pWK6Q&f9vUtTFBO|1%9%od-b+T4h{S(Dfzw{~+X3jO6vx=DpRPQ0Jrr#SxlynaK)$HVoe^OQ>H0$urhRP5hBIH^CaF@iav%+>ZRBe{xmb z%0?r3g0A9NJ|vNLFK%wGU7Hs( zjr_Spa9-gw3Zl!_WvdJIre6Fxjh1TsUpht)X+5vTSPOWp_QLv{pofy`T;py!Q6A#e zYC2BeQQ|H%Q6ijm9K+Z7vU7rJ&|N8T%5Y~da83Jw5KnXmV-=4{hTbT=?7XJwJeF^` zgEURI^8%U4yJ>~?(|38Zey(}e_cYk*t$9vetZk0gb*5=!UBnZeEc{$W^W=FQY<1xq z6eKsC20Ykmrb#-J-_(iD_uN){Xr5a&M30fkL}|+(Nqb#DZ6t^m@lM216V8!h>lu1M zKSp#ETNAaSFp*;l&#K39S)a*dTEO+pKb%50kR6kqF1nbXrMrl!7j&h5EE6P;pOtT9 zB}LRpP_*q zA`57NG$0S|Bo(pPlB?PM)Ws=Ql+NOFJegzMAJensI1b`xx=yAO{1KJWLvAARk}rjN z8teNhKX5$MLgTC|Tw=x1zcmRFID!V~SZToPv=P6{pK+-y6~|dXU3IA}${b%7;bzC;yHptR-=+>6R(vtH-^+FZrDEfv;j%H%hA58bCqe48r7-+a$~csyo) z3m3^B{3Gpg(wvI^1(<;-PM4+X#ppdef}%7Cv+mFYTE^@3Qze`&l)A9XRYGwx1alQd zTj?3(dNEC*6?{wf^F{eUyK!}Ml}`#$9#2Ki4biE3g2u_aSc_%qILBofrPHgt+KDLi z*E(!~On03E>8}T@3DQER^E4~MDWiS-3o&w}54C4Uhw(JLasY9&LE9p>b_rRvG*?$! zDR}?iQkU!KB3-T9u(}!aE5&hZzQZ4Lu)e|H(jXZp;o60p>S$}1MoAx>cL~<>S^Yvn z`2}Dk>k?{9Gi0|;a||7@KI6l{q*XkXPs?@Z6K>8OG+NF$>3Z3nq~CEbCd-d=bf=a} z4~?K49f?fMavbY2?*B1u)caJQE3sJ1bDv>jyo7_7i~xh*ABK{2|EYnX+6-OA-Ax?c;#LjiJDA9573 zrHZMovot~0Anx1ScR0&2yj^Nwg>rc?n5jZ6E)&ao0!%2ENIuC6b+eX==yhwZ?&n)H z9Jt*;=V=|2Z_D*B9>(#qoh)$Af4GH~ zIMdkWo*+e1SH1F%F4PjsW$o4F;He87O{Eg8ZOsF@&wVi$+w{J(4H4N95gX6jX|mQJ zKio@#JIh&$%-+J;GL83G_3RnkiksY-IKge&VtptS{DIiW76 zsTScrn8&CESn&hwxGPU;;nc(P^z|Elvd+}Ze3 zO}!~D^=OQ{m{v+tN&r8%<7C|_>v@lk*FdoFa$NwNbeTm|#QUsN{S1He#~5OqST5z6 z@)FJ_;+N6g$#;R3?*x?BpH#A~eWOsg9Wc!>0v+>H!+9 zF@=-Wg}58RRk#~a>qne*rY2ZEz4Ab#xx#9vITBO2nVZs08Ne6$NBrK_e5@l8!>hnh zj}TQcgN(aPT;Fn{_?#wDHlHUKeTA&amvsG}16iCsT*#TYf3FClyt19lFKYz_ zYn%)+C&9Z}PB#v)!+8nra;xq`na`0k(&|GJ)pu9(IYxN~1VU z)H6Cq4I;iAC_I83XAXM~q{$InwK11cJb8if9$;xh&ICgHVtYTLEf+E01i9<4w3&rc<50g@=ImbFjuKYRiXUl^N7ss$q1k zb(8eu!}K{6+z0%cq&u1V9_`d{T`Pt*$#J?M>vf%cL49>T&*fQGP2NEk=I`I`e2gqy z6Qu@N`Blf|T1gZ6Sv^m0XqXJKx?@bCJk@y%QM6nukj=A@1s)1B!6wlz z)dCH($%NYJ8feFARTF9Z+XL-rjn!zol%cA`3k@aYLJX9eRkmoj@j$oatJlPH9B?EK zDm75!aLjED)rO`>lek33n>Z#!c%YU6uR=|46OBB^s;IDV49*>@B3>%z2jI|EILhC| zA%?w(LO(;c%OG0RMXAP@Gr2&YHMXDqfch3Jk|!2 zk9P%f6BEsZYrbXZ&-kL6OAP2!=cr2t3~4b?cr_BdXNRxiGUM;U)RU;RvqanUF{1`(H}aa^FG_#N@AF?Kn7 zB?-}50B*|Dl^nz`120^zY@Go;a!;NEMpYaqC*O(W`aq)7T923dG@_o+?KD#&G~Yudj%YxsO=0WeUyE z?s67USVY6+ur7zj`i1Wc#65;yk;cIGdYS>=nbL}ku(QD3a_aXY)dK2gR zLh8wD{8$G`7O-&$j=SdU&TnEj!F~QryR?%YIi5>%xVF=t$LWtc5&!#B0IvlW zI?(9*XeT&UwJBn01{l}nilGcB+<(}Sc>Na2b)FWx+wf5>hIR@813krzyO4zw!LT22 zjI-972qq_OM>nmZ8iTW3p{9t%Y~XeRPnKY)1()f@U~sd6*Mg-xKuwI)xf%@g|6JGe zNZrki%taDP;Xh<9#?}o;Fq12rxqM7-@bB6Vf8Qz7fm2KMYyJ;-K#(V)c2nym8fX^r zS!AWq2ra}2yGbhNLSYj!rHZ*JMZ5qU{gM}SWC zawaOpxjc>_r~T|`JKB^Xb0-;~j|AE-Pb{Y(Bg23o>FT9^cxS9BfPdgJWyn_-I00vL zxxf@jS_YJk2PdG`A=k+IMSh^X{-^0E*nA_k=Fu|SSI1~g zj^IRoLp*$v`_QLQo5AoVYT?++$m!K`RLiXGJcM85YX9R5HH0pVf$H3d*cYo26x}eH zkBY2{$>Ak5h?7uT%tjTrf%B}<9%>vEJa>x=SuoH3^h-Ak`Ab z<&@GV{FU|u$0%p%RqHDKMTMwkyK7IYvUUPz)MwF`fvH{js&hepr&o9l>idJb!hKM` z)od$_U#4ZK!ICkW0K1c9Q)O+Te^E!RqbDgFUu!j+?$BC(5i`&q)n6yAj(RW*)$CC* zI-EoKj+{YW-l0VK5~`{p@V35FVO@3C*ZPIos6XO3QEO^_{+|EEi-2jdTwQ;q>+}ku zKU8aTBHaO_{o-_@d8j@vBY_ar%j?+2lv?&vxCKpa{@ze*juA^5r- zm?d5>85d$c92{1JYe83=Kq#14jxi+~hWPi=14MMHJ>9rmqz|+l95%`{#JdocrYf@M zwl)M8<#Gu!%@14`W;Zc$CKp*)guKbI!EzL7mnx#tDMQkfB6G?S(Hb)oaB!ZiUu zrK!RLX~F-dRBFs$7d8Wu{)~7|(OneGmGme2jlQ5<@z+Ycls=JOv=REep=8SovQ8?? zM$5~a=|y;Zb@<5f*>s!M;an}~bs5APt#`Dq{w%BLnp2G*NMoJFeXMJ8L7LH5^e25M z9ykv@IFug91Z}BH-QQ7T=-1t{QL6CgypTr8^E6EZG)yOQW2-s0!;GAy{Zv~H>W@@i zkD;P?6F%HtY6eUqFyL-$l6KIqrBC4n+70(_B`453(#8as>(b5+;hk^^Gx5KRkLgqH zYChuLKzTpify}rFtZ`Y$DY*zQYrF~Nt=i0XFz2Pv3p;VO`#A1Rec3*b{H+T_3^0|A z1;4DeNkk-BN_MEMg7INQS@+l3mJOP-rL*pb%ywuJFnwlmbW#ghWs%@s>8Y@wwu9RXzT<~db2ko^;GIXDH6PT5*>vf;Tpdy`! zN_DNZU;6@gL)m2q+Wnwu8lXBVra)ZtA#mw?-u z&-j+Q`ENK~8_5H$lYlCqo+Kfb`=FkBNNb>vp2*U|_q4Heh6djR%~G2i%Xhle$>0mT zU72feV?@jmXSn3*b?K@t-Mtz-$wzp;+3ohk4n-iGfyh@OKk+XJ+`Eq=U(a_v}6Y@b!usCClup3j-P))94~bSqHc)S zhX2DGQ>c&p$zLO)@5)`o*AaeO$7+PTwLa9w<|E+sKghJ#5p8v}CRlZm-nBLX!QY0v z*F$^2iK&1iSc@X`BR#`;j_+?-vY8*t6u2l8s5d+Uk3=KJzoXB17hJ-ld{{pOch1)X za$WN%h~7mEZv^J9CO`d7w&BZTev)MV2~`+ajK=W4wM71lJp+$(hAMLq^?*K%aQ6Xv zc2$@C9Gt@Ez$@eUvMdB{9*}S@1AZLg%T`zYT$h1OEb#UcSqN6}QeI&Y*tdm7K#A0% zAv{UzQw#WkpTbFb8>p~Kk8pRsYSlzcHc|Y|>I9sq%ilYnO_v&(r$gz#Qq1 zBX;3T2Me3iDOyS!;7MfTtc_5$wt`oE5HWDjT8mt$1wZwIGY}PIHN@pFVA#W`eU5QM zjYlp$OOt$9Bo7ohNA+|RJcOIrYq|lg+!T9Z;bv_i>qQxl>ZSvn%SXcC!KRu5{j?RB zCqU|%*cYpTwT-b{JRR#B1IKM7ub~(q$`?@C2{Mz`$aL8UWjvR^#2(e}^pf^NMt0*v zx>Ej-7vZ(d)KK`>DNqz2adWsek(7&{U8sAfLa)9Gru0fT>p0^6igRBI^`?lE4##kb zcILzINT+j!^8k@>ie}0Mb#$QC)kW}4c2Z|OK?kMy_!{!#G`(Uiz|RG|#guWVPBLDo zJQp-1INj3^{Jz2Xv_ZHH^$R|AnIyXavnVpjK#_VN&VjMfrj#QML3M^g%~rKbbgmKZ z4pvCV?Bv4(A@Z?C5#HMnuljI3*j{DIfnky0>G1|NGqgtm`y;>d!O&Ywl9mFi3-Eui zUBc03I=&ZZ8I^Dua=?Y7P*tl>Str`TCYJ>YBgGcXkjvA=jxpi(|EgvIK6#~~P0|mn z?ugcqq z_g#{YSHC2WR8hAa^&%hB-Fl5*!951qpXfktXRb;m6Na(R(h4VA z{sgnW0)@F372rv%=1@AJD+>RE^K`^m>hqV@>pWCF^e>F$F}z?~J8>s&ijk}+41)u> z0#!?*zNLLp->;CGJPJs6U;H@&T$s&cHCE2?DL9OWWGl~t*MAIX5=&xL6uO+-a$nAH zD)K$hx0=%1qp_DDF1hv`>)bLsOTZ~x`hx!(0SzoIQ`IU~?cIh6e zZWihfIzxWfXsql5%0(5^mxp6FG+J^gL<-2G_w+a z5iu303>w&*pehoemXbuMitV-j#QxJEozEFE1v{@9@*D0c9X@C}b=T&==^yz4J>fWb z;IWd!HQ{)Erq4i8{sFDl)QQryG*`V^gc|W8YR#KG0&9Lx63qZTg}GZxUt+HRz)z=W zr#yq5s2n+k+P6Yh@**0_P%5ats^h$E;6hK-M)p)_f(qp9W_V{q;b-r|I{Wy%4OBx8 znEEWT)1xV}fLCG#iojA|a|+nHCe&I}sbO!x+V?Y`X{x3GiEcW}@mB&Owb5gIktR!1 z9jX=9v%Cj-qaD1A$x?xu{0e;3N5J!|z=xZtm2ocJuP;%tbrbGf8~nCas{?ob<`#Af z`wcw+?J`Y+$TrnXg!VwKRLcb0zhJlaGDRc5SIaOaXtW3(q$89NpH1+}GT_WKo(3P} z98^S*25B{T=%aNPS2Dx&nr;E=PQy+|CDa0g*ef3VwwhYm>-0tQr_Khm*0+0rCoa=v zCl9Vlklw}4W`eY}N6QvO$P%!~V|||splq^`KhYQ=&H(1j!tUx|`!0Cw0Nv-u6v7$E zvTD8>kMpo6zfJdGWY@G0#_$Gqj!Ey*V|5+eqS(39z2L)f7-xjZqz}bqRoA0XH21Bs zx){2pfG0Siln<_&4o_;fy4-Q7Ob+s0?uvDKh<%3hR@5LIl zhK>{2$E8vQGn}ZoQi7WKhE&K2B2G{#oUTu3f*jIj{Fjtij*Q|}+=hEo2@pDw0!>3b z?~FiBZk0eiL}Q7ZP^%>uNDXMCme`w(f~NAYh4`)Lze+srPT|5_4J$8sRSH^$X5JoNk0;wp!+M zIk5JCT;?_taWoW2jyc(mZV+=U37B9hFw6@P6e-~pl z>)Gd^Yer*#*(;6g$8w!}njoxQG!S=^*@Z}KWJhE4C5VqcP&~i#5;=_+sAEQBcF!R; z(zKDu*HAN>cJXcA!qMCU*q=u}|7|fZL;t`5sf)gW?HsNTWt#fj$O-%vb)f~+kGFxH zYDf*<4lijM-PiB6hJ1)@YYQHEo|3Rz`_SsmZXU+(U{9_VB5eU&lmM*H1Sqx$>7r-Q zRgi!=ILH&J5_V`tY6D=+U5dpFtkOtMMZGr~U%9+n>YL}02Oea@R`P>#=%TOkUN|J> z@KUBhU;A+326{0jQY!Fp8^(Xy`G{7NKTx=eJ}(hGo*to&ctex*M*sDC+W9ZAY=lN4 zDn0TBN9b-zLzhJ;pQg*S9<09*PR`&V*&SQ!t)=SCE~w z4Qn|Y>w5t4?Qwpq#Ee4q#&#^)HE6Jzr1u?OEq3Aw5sjpzqARW@h>Rm5VM0{v&+@7 zpeVS5`BA56ppLY^MZ{mH0`@UdW7NzKsD`@Wn{7}F^Iz3Q*Al;exVn9a9dt%bGS}%k z_A@HLLm`wdAE745;QFSN12Bq~rUuV437mx8kj6UFj6sc8gs~LsNnl#IrrEyh68FOY zxy*I~?PtgNnY-H+;xYxCMRK?dm^#p&z>PVM#94r`_yN5q+b2*7{)IUhY9}cC8sul* z@m5?HJQ+``_%ZggpGcN9j`jl8f`DM3!d=Tdo_G8UdW^cpaU#T^?zPa169KnvoLrVv zo&*J*sM-9O+VLlaldCoK$A=RL-wxfS&FP)<>ZK*G4ZCo#d9V*9JYr{KrAkAcPwOd?ub_7( zlIm-!cGS0Vm&NkBCZb!aHFoV!qoZ!Rb6D5wKQHA_tS7W4RpQ#* zp8M*vPzOzDqq~Fl)%ElP^uQx(k8CPK<&&$&;l(}=6nsSY+(-0NJ&GKf!}ki;@>kXg z-RKn4yL7KGmHSdnhomE?VN|VfSDk@IUvs3^-~#N|zr-<|ifZ^G9d+(GFJWFf+dZH! z#(|4#+S4=+IG&)F|0jbF=-)u1)mnzSG6df5YRQ6!dKY{!3@9*~HiAX#8Zb21@{Lb_ zgI2f$mQeEZT-z^nx^mTIL!SppnbF*fhut}bir(Jejj@a=R@sR z0|t$QF2C!;a^!Gl`#b&Cm(4)yUAh{vy9}IO%@)jwgIGBP%~1=8u@2Q0xXO70@A&}^ z!7w1qcO1aA>^${yYrC@jDv+rbFVxsWeYlq~K;Vs9hFz+2$fx1v0d!*)T^e}K|j9{Dr8X~2@i*M1nGlrk?V0dm8ujk&H ziHzSvPql%aB~P@uZDIE$z?4J3qDx2j!BN9rqdAK^_?MjvN2C~ta0suhRFC|Oz10x% zSRKr7VCx}R*92OuMf?$vtRGOd81B#mZiYMm&2iH`832yxg&di$wR9W!ZV*MnzfRM8 z#4?S3m-$vXl=NtLP%-W;w2*JeNL0POWxYIcI`TBivd%l*w7EWzg;q~)L|@ZU@<%nb zLkp}R?tr?vfm7hT%>y_XIxCuLA)lYqFD;hmWw9iqMw-fD*iSo0m%#RuXrK;64t++s z^eLU9MnIT3GKW57n=Cp@N2nE|`A>Im%A(57epKmsg`dEQbxTionbUw~SsSIQdWu!TJp3zka-R= zr#Gq_=x?5mJh<+>g~$sB4mXF(5Cv4Xh|r_+6uKz^wKnz?^%#0Vve5OJs2Q{c(KwR( zVRaL6L=Infn%JdOQCRMCh8$I|dfa8Ila0ABS<@rXpzv4tyRJTmdw@DOc>o|4R+p?S&@k3h59l@4wV(0b+Zl#|STm~R|xSQ{*4a14xLT5bD zgW8yj@Rtyp3688`Zczp><_?b93^XsJ05F2bor8F}?zq&2n&AiJV17*(FN1pZ*2I|ytLX2xNs=Qwm%5)@tn{h*;(oe8*Ro=gKTczgCR z^P$G_+>fQYU4rV&MFISby$l|2Jwql4nm^C_TU#3!^#Su%$YrY;^s+?>T-jcN**qi5 zcm(u9Ge*}i?jS<2mkRyX0}7vH9L{ryJM8^o-=|+a~>6F0(yWKLh+-v zlb>*{U-8cCw1f^osWbyyodfIa0WZzg6&%EIn#POa7?rC)6;?B$CJ_4e44r|-84q+^ zg$&-qRdpaoX`nO)1}A`#UGgUX1E1u&^#dxAP&)%C9d27X-CmE%HA-69({bz?I2`wY z%0K8N^%TYc!7t%&ma|WzX*{aGAbUUNZ!mOb49*q;M!-I|?!<~epsL8|>L!3n{Oi@frbtBTVn3yStS+1wGwe>zO1t0ZVulXCkUI zfy;UP8Ae_pGh{elK>Q|H0a_c_GY1`Ri}{I+!2WqIwc$zdThG(Ws4Bi8m#etYK$#efemit=j`JFRUO-JXR6O!P z{J1BF+2>HHl<6Mej~`r$Mbrf|yjGfuM+4cXOPcEf;J^lW6U+Epje)XVi#^!yp_7vM z5nR_(ggza937Rv;nLwwc0#6*oLSLhC8S}hW<2XZ0Q8`wZ#+cnaK1@x4h&|DrJp}Rn zJXSP^t~hWju?O=Ay`ER_B*04f44D+jGqjujE=|#OepR1RQ|lBhk_A!*GkWtTW{s?ABbdMoI!-k*S!2>TrCZ++O$NZr5XvcQW>& zJ8CYnb+vYo_Sj_@0K{8Hj*iw2^allLU*u3L-UYYk3*-wX2i?&jSmT~J-vxB>Hn#1JP$bFvJMt*=VAJyyeMgi0`#s~9Ryyh zEYeEw{~N()FTvU`lA+d0%<6N97Z3d+hv_FSbFzWeBdD$&i5jW_2TCZiHBhH;Z%)u% zK-xc{0N|Q%B#_^O9Uu>w=eE>l^nCIZc$=BX(hAhaUvre~;vvY|ZMdWF*^jZ;2@L42 z1MJU%$w^@A54EFi){VRw`S2F{s^-D*oN01tfNp@gn}K>g8mu({+>}AJIT*}-U;FV} zU@gLk+jBfHGf>LG6j9vHoJ93fBq7kw1N1&+b2xAy1R0-)vHzQM#f!+>rz7YT##e|O zYioSYPL3AvO|4`yaHnhaEd1PuP7S(j9Roj2=OEdsi>a=4QQPWQh>CPn=LOesrqqR}SHN>~dp)&&Uo(3HDH<^f>Uoiu#>6k2J- zK&j4UJjanLZ>W(z$n$Wjq5J7Sybf628aUPo2;k8zP9|our7nbrcDr!7v_LK7tJS*d z5%(be0ek9oaP^JI_P=S8zRXkPA-c61LMt6PzC&)411(dkwZ*8RY8tLzSfjypKhSc{ z=M3w8sFNr?Vx7|pi+LLmq=`P1`LaWbq%&8eYt|H}vD9HW9XuKw?$P$zS#KgMTR>5~ zg9`8mX~oC2KT!UxB*-d6&Sq$p#@GeTkZrV)6HyE8(N(m?nvD#quntiTZq7A$qb%1{ zs)U`^EHJ-;a;##9@HLKxca)*kO@(98V|JP28Uk)v$E3Z?0px7Bfp4NwsA;O11lgtQ z5RZSN>gr@Xavpg&O)J^&qsFr+0%wY_cJXDYXYf1`cwnzq0DpW=;5hJeTNWVS-~1g1 zo6C}cith|^en4SB~t6#f#~@AT!ZW< zG>U6LpM>gMzKvBbrzqoUoWzcwIgOQR zYJ>3#j{~0{VBgOJp8S^j>OjW6j%+|Z7jqDu#2lwf)N5VUTN10-DFFin@Y823^6ve-*Kr z3io9aqqhmwSiYuef}5;zjKeFLGJ!^FvbhcYzY|{NB&6^Ti{q>mjC$(~_BKx)hPnXGjD2i>egx+mdz0Ta~H&HQcQ$FfRt=x5}KjsB-JD6Y-45{R=vr%)1hU_0_&9p?A} zUq@?0%jG)b9F_yr6^uSe#~@$N;E99|h^{d02MxZ24)Y56weVTa!DtQu5!+(Ny0TWY zGgZt~+XLPg>1`MD9OL3Hw#z~9fSEv!Y~VNbH=l}N2mLRhG{>F-RGESKtYzZ#H1AhS zLUn*S%9r4QOyLUmN%Z=rBYuC=C{qDkscz0g_oPAHT2zWEWxaN@tDC=oq8`+xW5C!W zpfCK?W&F$=}4$10Zj8dkAkwEKU%Y@i^aN?<_%YT2<6vKW2|c z+9Me~BKA(y86SWJi`~UCoj(V@o&~cUMJLdAGi8=#^Z50@sGKhWKL5>?)2)HnX<=suzAfk~ODI9=e)M{vXv zk!^pmKTpK@_G>Zaa1NsWCDbhq&>@bey>P@zDt3q35hjE$B4U~%XBYEKA__F^py@y7 zD(Io}r&xVNFUe8th3{Zgci6L7gt!exf8Sx50Cjy2y9e#44HwgRTw@zQkZ!t_iu>2F zMfT`)j68&!Lg&n+r?gGOu-ooNwuB+8cj4LIm3V@DJJ$IU>en)Q0kvHMbZbp+Kym2A zOQ2BXqc}Tdu7+wC9PvGLU^|LrboL>(0BBT<3%@8C_m42 zc@gd4mr!%v141TC0~v{0I)#UWzdx22cmux*Wb)Ow8@R4cg^PItu@S8cpo(j1uzmxz zdqw^E1U!Y#cn)}N;ZVH7xWDf)5|Rd}3A>;dVJnf=MJMkH^OY$AvX%gA;Yh;6jOBi2 zJUr!0j{`U zSg0KcS0mi?wo~mS9*g{PAwR>>b2Q$bWD7;w@o+M|nq;F!I=@2sbNNR5q@rB zy4#2oEyGtnufiyba2_%ff%d7|PbZ^7^0QqMVpq{($v~`6W^`t9JU>E}7UN9Dli7Pv zoiEk>i1eAjktrq^j2kJw{DAywOnWMX(`vs&|&GQ`9g8!InXPX zEJNPm9pJZ3V4miB9*BGlydTrE3^i9dX3igX5@O!dL(1G0{!B1?sWDI^UFu_JXPcft zOa-D^+Mzq7KR+nELBsVWXpwSC12RwJN1P4S8l+)dpSDU3i?l7=q776DADvl!J@I4|b5;=#^An*Lj|2(7gu~^+JMgjxP8I{m&x@ z7l;K{YCrZN+G7^h6#j+$8%O)CRx%g9OuV&>u7a~YR<<6~XYCD|hJk9*)Cr89bw_HF%$9@>OkVJNR@6I2X2`Jr-3)unE9?bu&S>pSCx*c>-n0 z7|y4D<{s|D1%2Aktm2}>U2S}Jfrju)<_4ZeO5z1j^i@n-(}WYS+k0Kk;^+++PYo?# zFVuGhUDTs+lkiCvCfztzI6C%x-p?!cO8rPvP)DU8uU(WykN5$48RBS)I*8zKdnmF% zbdMx)0MO#<5Q`fZ}O_+#%u#B2EqD?miZ42lR(y^|Af! zEZxrQp@^$t>=k%&EZTT+>kp$qVQN|_3h{Tl|NB(wW-N)Ie8NBondP;oS0KOCSKlU`C zYF+GaTvnXAZOf1y2SjgWqgxcHnveKTu(H5$+3f4OAo-B*Q@o}@PaHz8*I0P-_c)D8 z)n$%q8+)*E;9~xPtSpd@x(RnwTgUS`nv3}R9^J>Ka98H&Ri~vDhrN*%PJJht--J6k zf}ep}d1RrBT@q-lu5fME_0b zKrfXNo=d4)BUvJex~ zpjHyGOWM<#MjPa!ti&rWS2^6%0qz0lK|YU>STdcnr4=gFN6rc-85|Uf_+QQIxTW(f zt~kIw8kmc)K?lSnY>F6{+JrXH8*5;-6>1pQKlxnRmz)E#xH z7B@ysI@o#_PTMr!+wZ5Svt_E%j?^VAn#GH5wU{N)c9X_&<60 z{E8ceWpV@EQ!9XJi=~QrD81>1^FF8Wa`JFzpm!c!;<`=@+`C~~E@wHF20M9pVk-(Y zPz?0^T{?!*)Z%)`8cVv+V7^6FkZDl=+8MDn9{O$#V)K1%3;pw$p2}+cypKFcl=c_6MCp*C(NY<;U z9q&+>Zs0`1vjmv6`cP|fzHp)c8;?ckG- z_?}kPp1_i4brL-8%6vzL>)SZzUSLL$`2-I9@ANk+`In#x3lPbZWE0M~29a_VyES>J zhp@k9odJfofvfttJhW~cKZcp?1J`Lb_VzMSlOCYev;ym~8=V}d9pC2&9Ks3@;*nfn zJ*D|vSD(Tq>VciO0z|@C{*d3LiFAv4@nFg22pNJk*Z|zGrqeZ<7f2g;wL##bG~Qsr z&()_=04F$^H=+}-3$pQ5ZmHAcZ|wQ41}FOSE$OYBunV~mj^14;`N6!MbMOg2Lr@cq z}DN;Y<;c^~myy`J=h(iz1v;@~GV-J)~47$;Mz4{*Tv4?zG zDAMFZ8*BnExa7hA{vGHTtN+C{ye5-9R;+rt1Y<4}!ZW6L&JOR4=3u;5gnoLLC)EUE zyhV6dZyZ;`>0ql^ybr2P@=@c4BDdnzgZtiKHUUW&*wH4!+(&h~O4ox8dxMY8^8+wJ z0zQeOo!!SgfpZX|anK?a(D*07N`9PZuf!)_JmFQEf*$Vz-C&nMhlEHVX5~#(v#4zm z&DSNAuIoT!gZak5cUcAPyi^@va02c>URR>}O~lIGhnkoGz8GhZ!$`vMcMqM{Q0i&w z0HcR+lKOh;f=#p?rA_Q=c0R6C&j_8Dt9%r5a#F#aCh>3p>XM4Wl~5G>@b_@s{RFVj zc^q5IbTjv%Il{36QzDh@3HB@YD)yk-dy_-BuDOW6qB}-=qpL64q-mHTy)Rp}vxy^o zx(x?FjjY#8{D~RCKPuLdig9$PR9MUG-k7&h_J@eSv$_~#uW*ku_01vu2#TP&$*Vy9mjYnqQYe!p;oQkJIs8j*Qthju?7|HqnN3| zn5pxOz9)jii2A9!eID7FqDffMt-1m4CVt&yYOFc00VWiCNrCJK6fWY{_6T^pkDwD~ zo8B5@&V%<|5&#csJ$BKr(yROq@H(N;F1AO`8K} z@d;vjA9GxvPm>>ZfkKg?uj*22i5Xu6rFEJHp^}M}uQeI}R!C)3fk7y4Zw>V0Z*D*6cXgJ*H&HA(M2a}}GeYkg@@_u>~yBLm@CC}(v8pvtRN4yhA zH(0)h5{{ExZLN(^ds);KZstey9oBIKRBLOl&9~rIHwCW^1df(dG)L+I#@-gR%B#9v zTXALnmJ%c#`4$YeY2h5DO}t#5r|E?!-D|mpHDC4t#S3IQI2uplVTb5#bnQ(=r^`_N zmv-m+K-!vAPRD`q%Q1IVB^mfy>Lf#Z{9xhp0*jjen$cuA}#y7$~QEg;TXZvelt_*kv7x zr{%U=Z}TQjMm z$h{mz&?0lRXCl-5Hd2G zo&^SV)Hy>ysQVxZPt6zm0ti7UU3deqz2)+EPVPBzOSUZ zaBozWI(Mvj*cl9yi>NEA>kxc8(jn^~_(DK1bfj&fM6qck{Qod%q4Q;@wNQFtA7nlK z1wLNrPSxL$9jD;b7demUj@40%+$+gX!mxMLN$0^;?}RKDfaHJ|Idl>6&MIH6NdV0!PS0S_0nbXUF3=5B}@mLd{7+}qAYl$2p*bFq&341Wl}x$JoM0tq8=57X^T zQ^s-N=~C^7bKviiYw#2JXs&S?k#zVP@o*QjB*uxtr$JQ%i#NAlG~J-W@!YLkq>JFx zNNCz5tbZf)u{1?L;T`RO+5bjE^clE9pXqp@<~<@X&OV@Z8|+6kfWK+MvliN^i?o1! z&wy7*IoB~MK&GisZZl~n{RZUkrjMwTxd}J)S=1C+aCy4XCd6J6RPkX7#ubawEwY>K@+Z1x~I+U6qYbx!6hjP-XVR z=WtDC;T)OE-Dsyo@**6+2plF-&0dQ0-?XL!i}3`MJi@cbex@CHvD*6t_a#HsrAL9H zfALKR`@XywE58Pxv^5#`Hy!?CZF5gzu#Qc2t*%A)@l1IUce9f^@Qd10gSi1KwD3vn z7VNXGNEKYoA0t?gv$Yd{S&UJW=79g&pu?uQsg6CI4SWgsordw_9(a)~M>ejdD6sG% zXppspr!aAkm$)C#)qUiFR|hY*umcZ31YV()aBaHT#Z*%Zxf{2We2nTxy#jxxsq;Cs z?LMe%tfbuy&r_z#GO2{mK=gG1HZIKJlhg(|lX1LE_Bczxdzt8ah`_GJdD@L#kGJ%R z1nXJqkDl>=A^L+DJ#FqfV8)iHFy=^0{oRe{HRuvDv=cD>H~JJ$G)19TEKM&vuTpDm zg&mC`UzW;ZT+ui~xi2cJbOI|ufuuSPyp)yPP#YJ%g1zKUsHrO9(<_%~1h;Z(qb{qC zr-bY3Ks_N-u^U&Hu1T2w2<2UwTj^lLO?%zK%fY0_t#QDd_B;b;t}7D@jr_($1nj;-`xc=z;XuU_L=y$pzlSIIA-^e%EEm*-x`kUm?%h12g z^k?i=zrk;!3n+_D;(aS9jwVnm-6db5_8N;l_80LKVjo`NUN65`4N+eYl-oQ^XM(Zb zM2z>OA06Eo^OeS)S4xmOe?LGEwVP*cITJEG_IA&C1uhoO{z)MB<4KNkxR0R zqHLEnTPs$!Rj(BNx1Im{?5o!_<99jdd%owne4fv9Ip?vW*K048=K528o#}>)>>Vrc zX1hEKd7J^zv1L{tCFhHH*i`BU^!Pp*BzHuxiQ zp8m-$_mAaac!t+u8D7$!_7z!x>(QXI@Xc%WVYyZgM$dy8o{z2tCH;|lnjA37!*bk4 z`AE+46CEwbut`VcZF!$O+EYY1ljC-vgZp(EnTYHCb-5GFb{p$%;j@L4LGo4+v*B9$vFT+3J?=3IM%3j|PGH|$jlR6r)wxI?8W#j}}TB9i<$iW}{ z?>%iB+9ypzR;GBPv>< zd=#!tGMR(q8aCs%GVEGb27^y3(;c&&#DOQ$^K!<#+vrBBSsHqoweX(F23g|&pmJ;_JMo$= zg$G&g1BeZ;gFE@y4@GfNrsz%h?z8s7_4YPNopi)=B!QOi~#3^6src$Zs%v4#}uP%GM%RW!f=C z;z`~Aj@L0{iOnBQ4tk1km89+#?xxPS)ow?=~>3pa~w@yCo8hd z|Hnf_4}ho-5&`r~%*xq?ouDT{ss!JY16Bt$#~^4bJS#bZN!HWrDh%Qeglg>S2qZQO zY8R>G`lj&thXdy_WR-P~2s?dfGAm3(6HQNW+`(w}SD`0s$Yd)e!AV2glu{#gpeSe-5defx4pCU& zN;_R$f>-~Q@+~<#X{zRA^O3Tpeok`81}FFFVBO<=*{c*i@3Gym5leKBcBX5vNzz0I zhp9wBb0c-_Cn z_DqLgo1_fqQin@KaWLdEnJU#=BY zd5i^h^fmMO|Lf9yL77NW^c#3Qct?5%)5^V%?3YgWalB3P<290B`<3<}zt$3)NOzd^ z<&4Y7KMWyWUZc&xOd0x+?)CqI&%0>i6>(qN1s)&fvxuB`(P8@pJn%Z*BPXT8Ho;Mi z^9o|ES-MHq&{sTGQ>JqlwJm3ZF?V^_qz!#zTd-~`^d8Q-Kwptg$a}Tq7Zt~)@z*}k z7kSV+E3xTVwJOW1csd#81MFc_H0M6AqJHogwYY1fWpWJGJ5NeD-WpKg3fZVHXe5oj zjD209RaD<*p?{}hGw`((>#8Hcx z!*;g+RoB{jTkY4d*Au|mHxV-zgAe;kBb`YEv{Mg7lc;aI(LR+@zR`s>-L18-y0iTj z*p$!gGfNp7auBl8sq@rp!D4+pwqkO74YV!2hySt{eOFu+4FnCogkDZ$N>L&mb&GY8 z4mw*q$ZVZQo}`l(%Qx0bZ_)hPR@m8ltJ>*N{ zPg&(kdC*?4Dw|7xWLw=7eQtN@dtj!IrHRgn*IGYvobR*#opP#Zu&$C$#L_qFS+;3Q z54kDc4{u#nw2*$cOZC*|mM{tPq>;aE6ZIkKC#Nj(&pBTsrXS9$-5{H=P%Xf&XMp?9 z@hk9`Gldw3`C(^k6Io?P$?Pmew&qjU)QedAd^DsgnlWG=oZx)8@7n0e+6p;hGqya0 ze!=&}CBzq_{bzcs{v%p~H-1_FPi~6-9JRq3wv?a1N9Eg1aWAS#mxxiTJO}2aJZT@+ zVzsAW5%*%fR^WkZ6raKG6MhgTt7mHWBXEi5B{_O)assIXdBRJIc-yCTWs zEO!vGW06291<6i)n_&@_@Cq{xk#;_ZC9cChW~~w%4etQj6U&53yaOx}xGkN+`dd7T zghI}{R!^mzP1Goq;Ctr>*d3Z3A`3E1(N_T;vks4)!QYL*zl=^E z3nxHO{exH`E1beADz(JZd5dfHC2IHv65WmRRdENjaxM%`el&?1%stW}8Rcz!KAEqL zSd$^x$)VV+-nPPjty_FNQd>+$axwXo3Rz9H=OUuWA7vq#lWv|HU&u_10ts+>$GC3~ zas2Q7US#>#zR=oXbvlY`@I+yEMDlZYO<;NH41p7VlqoUI%h!p#1D{SXs zJ8YZa3eO{Mycs)H2D0y9jqP!72X0zpli~{LsjF>;{2-0#+U};UiNuQhU2RKjc7whl zSMte*prd8}DD~*S!qa|Z2e67&wH>fU%m)CYSJlq;AIX-~6OZIj)nDTI>=?Dm_#V3N zWj*#K-$$_O5^^@3lC1Pg>gn-}Er&|$nc7dwum^eAhmEocR{BT%Q5!omm3j3RaP~1D zLexB)>{C5laekDC&1;a%LQ`|yu=^EOMi(*OhuYo)z6JY|v14*+c*Czs#%P2$Ek{F6 zqUQ^!tf*uS)!Hjy1(Ap`NY62>aE_YHRQi}%`)8bOrdP15KO*5{bTw;Ez2#gzsZ~~# zeCA)1!R@BK{0TpzD|M)kmt{6SIzpak9Tso{4D9F8K&C+KvnRcR%+W={WLaHC_A*y^ zLWI2P4Sc1YgQhPdhgs|6t$}x@*6V9IX>`ew;p!CrV@s*07~&m$sxh6N>fct_sFm@f z?ApqxOvlR^a1dWxeKZX#exrWQwD~={i!5n9e8c$oR&upVJmZO8pl=}EGw~2DLnhNJ zOhyw6_Xyr`58s8+~SD{bIK$X!jem7b0E z__q1}7W4BmAk@2Ma?z(Uj*Q$Bd?B$X8Ai3eUqvP@8aGFctI4<{_yd^ov;b4U(@})fG%cZ{# zN6R{GInyVC?Jd5^Tve=S+L7_ad={jL8&ir_^4W#=eWjMP;>ANREO_7`eU zx4>^T$J;d_N27S_!?w-t^RHx<^wS@0DxPc=-2Kb;A+@Ko{L^TZjl$}0qYnNM`Zlj< zKe@^2{<6IsZ;J;bhiB`&xR}=ncZB+@oS?8^&<1Gm}Wl zy>YsqQjvNSB7kMw{av8KlU!w0Q~{rvPWfe4ulMCs`I^h>CRsP|bT#`jTUR4z*|-#> zeUPK)gr)io5mYsG3EjB5jWRj34EZ47fB8%hQ<*QobG1%3${Ic7lY9rBnYmP~d6MkY zdvz0jVT)4v1=HGzRX9X8E(RD69n{JydHF$&Y z_!agT`%(^jpRdJCEN$jP@Eyann;xLsGS}yDzGwAu6+bg(VxiSPR zoF6~Jy?xaWvy*S9|A*%oOY}C0JakUD87}b$`Ll2EeLh{Nj7f@#dVa&b{lR{K&1>cl z5}~|6G&dbflK~-=lV_hUmwP)uW*M1nPw4CPl8hqWsgi0cqb>+9&{z6$yf6Mc(-Kcw zxi`?E*r!#)Ghd-hC-Ke9Sbo!vSe0B< z+gI-7GmEsHcJ!zH%jg0a;&l4$Ry_6zd4{=&qr4-Y@`w0<{avo|BUrhuc0^V)H*tqd z)t}fxUriq4N4ZT$${Hyod%PrSt~>+B`pDnN1OGifWjBI{7E)Dqfi4qfR>+RHv-I`5 zn3z>1z2alCoD9ew*&to@T}z_QzMpH(*YWx#^B}*MDr?PE4Uj!#=X!ha^~f<%uMczT z0`wi?YVqesI1FSo( z7W>FvkzDP^nfsEhD2?K1q23GIwnE;}{pj9z`fu4}fADQq5GFYD!O+$2NMa6r8TYz!M?n=*_IAOL7{UvrT@9m#mM6pl1!ylFj^onEn;yInJ`+s;b&wvijNl zU5u92W1o+E7b%YOWd~IW;%m^wW_B!o-Jju1!~CS|CE_de%k?E+?OV9wifAk-K3`^V zFEXJ}57{m`1fTFLd_;*g!&0uY@4%y7^a&NRtr@Shp2$^!t=B@mMz1AH`X}A(_t48< zrVoPTcA)(uc+Su@RA@8j!(*N$^cbvYHjbEYI2t7K0GP9n4s!AaMGcabeyb0rzjpyB zB&}CHNcBjL93=B_A1KF=h#Q#(l6H4|z}cIEXG`hu&lWZJy|#snZDrINbXScfI44E5 z{)M-tA0q~pwGDSr+3=l~QX5mNwO(VD@}c#{M$FI*pL&}b*mPDzYdj|z6TBf$&VrS= z3!k`;)%=2;s*$~tCy!`z?}ZL`_ip&#qqS%PS~6RidF_MrYuzbdNlX7wUPMdR$S~NR zPDJi)Y`C|vNkqr{Y^1Lt?wz2QVM$AM5l3&t-I=J&=ZLzaH#^bfjGYmV+9dBMFH0$w zMm=NgxVAQ+rw^qqdh?=OD2>BMayR=plPp^sdskkysbHji(mdKkM(r(-abw*_U)0Og z3V&u3u}oXA)qV8}JoN$H7Qe_74|ZDT5Xlz#t$k@*KxSuqJzo8ny5CxQKk_4OSpApM zN#{kIie@?!nd}UFNVz;u6qIYKb=c4vnSd_-Ro7`}yh%-*jYeU^T1C%t51#cY+QBEl zPvE8eA}jVnJ7A+hc;oR>59nn+$fltI)!6+**wT(RM{|;us`?+chK7iI zzGaY1(jodHRds9qcCXaFNcY|PHFGgqk#l6eoKeq{W)P_u9c~MJkZiQe_-r%0V1I3) zB_Q5aprkDOx*NYyEtR10GCWl#YDrw(Ik`Bwk2w9`QiHvZ^sMkctlLVE)dnviZ@HK3 zK-cgtQNjoq#X=c&&BhGwC!pVZUjb(fOEe^cm+$@@fvJmX7VIkNh$$vzzNnF3MF8-zut^HQFk{9MFV|?Zm z6%QT4d3LoQ_w)%R%tiMWI!B(?vj5|f2?=PV*K1Kx;uJ{)n<8m=^SYocJR9R ze0h#2K1&QyC)=e#D8mYOrsXt>!v+=jyFx1=Y`leT+jc54#dcQwsUodH~ z)Q85IOsnKp(CczL6OLkCE8bK881jo>%5%U4THf7mT;fVTae>SbE1`!$xKXU zkTK0eno>3g%>wzi4QVbalWmILW`czr;>I+VOS7#b*ozz@vy5aV&CeRYvtB86N_9)p zHOUAP5!UKaEev;rl(%aedW5<(zQZy!N^(LbvlfQ2BuSrsQb62Tp70!#VEBzp(l#ma zGTqBgJrUZ&Akq(%RC6ryO4yykq?&K1*`O0dN0Z_C8ihtl0V`vj8NnZ=v0<9sy$2+A z*lW=60ZB2@-hsG_e}G;-pS) z3mGD*1N86rbLJpm4Pw0VQk@=Rs|F(-vR6O|3qbA%;^&=j*lHr63iLcb$&ZJy!>h1J z2mFv#;Gtg9e(3lRKRb}$fmymf^!7-aBv>wMjJL@~xkQDLKIgq*FB$U@z#1IZ=akI(uO%n38%iF$9;wsAY; zzCZoJeetKwKmw!vbiBe|u_`$nZ8JfihWeRUfmh;(u{q7ygR$7CcjXEnPRzAf-_Uz` zI?Nexlym*RHru`_+Dx_Ca=Dl1(H^$8e!0w{T5lL^?K5&F_aWcUNDMjnhzKnU*LV{7 zxkqP)Dkee=lof%GA4$`uVO5YAT3mi%eN{qTm zyCn(sy~4+U;mz`WP(n;D_#1fe#`v9FEtfiC+}8MojqpoPd53Ue$Ylc8ACq+IE=)i7 zDHq9m$qT^IlMK!jYtSLIP1f_BZJ>3g8*|s2fj1H~D_3&;;xLJduB@-3B6gLn&`()S z2Ym`}GX{-e`f*XQmP&o=e}%+xY6i#7g^#lskcQ7b%%hAEi&S6LbA4ZI-O_dZ|Z#BHd$~eI-$P z9!HxV?h7d%y^6d3q*C#0S?gTWOjQPsD@xRK4Ah6eNw| zujYd!_TnuT%Obmu_<4u55T@10Uu2*$zs+y5L?*Ipx9IIey@PD7uGH6v8*4y;7ZM|{ zvDWcQyNmetac@jTZeM>;ZnELJ4qHA?M@bD&#hoT0rutvMh&p*6y!}PUd^uD5wo6TP zmhZG?f-Ukb%-J{_y=xA7$jGzIXIVIX2Wy9eB2kKxO1!_S_! z61zhaeMi2tO6%{}Ye)R-PwS)$kAK@eppPmDW3tX=qSblqS;W=d>Ti(a*v`scuq*XP=blae$-*T-~H|l1t;~tn3 zCZZruJV(OM^vCVz#Nvy+VRC+W)9xf=dY;eGyYqvJU2>ii*fb(Xe{^0qOjN7esxQi9CV}*SgHBuyGZpk# zM3zh8K=R~L{{zUlMY0_1Umvv#59)S(S_k+FtFWuEq{n2K_r-!df$xl?*L;?C2#v8r znanI~Q?c&HQk7cp3fQYw;jrYB`z(hASRsGFHZz&eo1kAY5$bHh{$ui@Pr~lxN*x@= zB64W?RB1h}(}@_if++K;^_hYXY{*YjEXp%@*<%qI`=~{7BYOmLz?V0Z=@OA|=xaQ! zoSyhXnNCO1cs%$*Ux~y{wF`W%rR;MtQNe@$s>~oG)j1hOBu0#Z-F_G*ZI*tg9q|G$ z`$C!T!>JR^Mf2#9u+3QKS^D`FreEt^UC6Wi4uCAbCbu(*wa(RNe5!PVVY<)%NFKv% z5K&)0zs=YBF2T-bT6vYt(fjCr{lP{-Gfw|4p$Yr{`>T^N0&Og6w0Jma_NK5|#(VB7!5=g~=4dyf7;YMum8 literal 0 HcmV?d00001 diff --git a/data/test_dataset/ivf_clustering/svs_config.toml b/data/test_dataset/ivf_clustering/svs_config.toml new file mode 100644 index 00000000..1cf94f94 --- /dev/null +++ b/data/test_dataset/ivf_clustering/svs_config.toml @@ -0,0 +1,34 @@ +# Copyright 2025 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = 'v0.0.2' + +[object] +__schema__ = 'IVF clustering' +__version__ = 'v0.0.0' +data_type = 'bfloat16' +filepath = 'clusters_0.bin' +filesize = 41032 +integer_type = 'uint32' +num_clusters = 128 + + [object.centroids] + __schema__ = 'uncompressed_data' + __version__ = 'v0.0.0' + binary_file = 'data_1.svs' + dims = 128 + eltype = 'bfloat16' + name = 'uncompressed' + num_vectors = 128 + uuid = '0cd42b81-8e7a-4fdd-b4d6-81d6d5880fb0' diff --git a/data/test_dataset/reference/ivf_reference.toml b/data/test_dataset/reference/ivf_reference.toml new file mode 100644 index 00000000..4edf53c9 --- /dev/null +++ b/data/test_dataset/reference/ivf_reference.toml @@ -0,0 +1,570 @@ +# Copyright 2025 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +start_time = 2024-09-18T19:44:08 +stop_time = 2024-09-18T19:44:52 + +[[ivf_test_build]] +__schema__ = 'benchmark_expected_result' +__version__ = 'v0.0.0' +distance = 'L2' + + [ivf_test_build.build_parameters] + __schema__ = 'ivf_build_parameters' + __version__ = 'v0.0.0' + hierarchical_level1_clusters = 0 + is_hierarchical = true + minibatch_size = 10000 + num_centroids = 128 + num_iterations = 10 + seed = 12648430 + training_fraction = 0.10000000149011612 + + [ivf_test_build.dataset] + __schema__ = 'benchmark_dataset_abstract' + __version__ = 'v0.0.0' + kind = 'uncompressed' + + [ivf_test_build.dataset.dataset] + __schema__ = 'benchmark_dataset_uncompressed' + __version__ = 'v0.0.0' + data_type = 'float32' + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.55700000000000005 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 10 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.98044444444444445 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 50 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.52144444444444449 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 9 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.80588888888888888 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 22 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.90688888888888886 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 32 + +[[ivf_test_build]] +__schema__ = 'benchmark_expected_result' +__version__ = 'v0.0.0' +distance = 'MIP' + + [ivf_test_build.build_parameters] + __schema__ = 'ivf_build_parameters' + __version__ = 'v0.0.0' + hierarchical_level1_clusters = 0 + is_hierarchical = true + minibatch_size = 10000 + num_centroids = 128 + num_iterations = 10 + seed = 12648430 + training_fraction = 0.10000000149011612 + + [ivf_test_build.dataset] + __schema__ = 'benchmark_dataset_abstract' + __version__ = 'v0.0.0' + kind = 'uncompressed' + + [ivf_test_build.dataset.dataset] + __schema__ = 'benchmark_dataset_uncompressed' + __version__ = 'v0.0.0' + data_type = 'float16' + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.24455555555555555 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 10 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.67688888888888887 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 50 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.50755555555555554 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 31 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.80544444444444441 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 70 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.90300000000000002 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 91 + +[[ivf_test_build]] +__schema__ = 'benchmark_expected_result' +__version__ = 'v0.0.0' +distance = 'L2' + + [ivf_test_build.build_parameters] + __schema__ = 'ivf_build_parameters' + __version__ = 'v0.0.0' + hierarchical_level1_clusters = 0 + is_hierarchical = true + minibatch_size = 10000 + num_centroids = 128 + num_iterations = 10 + seed = 12648430 + training_fraction = 0.10000000149011612 + + [ivf_test_build.dataset] + __schema__ = 'benchmark_dataset_abstract' + __version__ = 'v0.0.0' + kind = 'uncompressed' + + [ivf_test_build.dataset.dataset] + __schema__ = 'benchmark_dataset_uncompressed' + __version__ = 'v0.0.0' + data_type = 'bfloat16' + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.54955555555555557 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 10 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.97633333333333339 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 50 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.51855555555555555 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 9 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.81177777777777782 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 22 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.90322222222222226 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 31 + +[[ivf_test_build]] +__schema__ = 'benchmark_expected_result' +__version__ = 'v0.0.0' +distance = 'MIP' + + [ivf_test_build.build_parameters] + __schema__ = 'ivf_build_parameters' + __version__ = 'v0.0.0' + hierarchical_level1_clusters = 0 + is_hierarchical = false + minibatch_size = 10000 + num_centroids = 128 + num_iterations = 10 + seed = 12648430 + training_fraction = 0.10000000149011612 + + [ivf_test_build.dataset] + __schema__ = 'benchmark_dataset_abstract' + __version__ = 'v0.0.0' + kind = 'uncompressed' + + [ivf_test_build.dataset.dataset] + __schema__ = 'benchmark_dataset_uncompressed' + __version__ = 'v0.0.0' + data_type = 'bfloat16' + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.26800000000000002 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 10 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.70766666666666667 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 50 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.504 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 27 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.80444444444444441 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 65 + + [[ivf_test_build.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.90288888888888885 + recall_k = 10 + + [ivf_test_build.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 86 + +[[ivf_test_search]] +__schema__ = 'benchmark_expected_result' +__version__ = 'v0.0.0' +distance = 'L2' + + [ivf_test_search.dataset] + __schema__ = 'benchmark_dataset_abstract' + __version__ = 'v0.0.0' + kind = 'uncompressed' + + [ivf_test_search.dataset.dataset] + __schema__ = 'benchmark_dataset_uncompressed' + __version__ = 'v0.0.0' + data_type = 'float32' + + [[ivf_test_search.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.48399999999999999 + recall_k = 10 + + [ivf_test_search.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 10 + + [[ivf_test_search.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.95644444444444443 + recall_k = 10 + + [ivf_test_search.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 50 + + [[ivf_test_search.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.51244444444444448 + recall_k = 10 + + [ivf_test_search.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 11 + + [[ivf_test_search.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.80844444444444441 + recall_k = 10 + + [ivf_test_search.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 27 + + [[ivf_test_search.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.90000000000000002 + recall_k = 10 + + [ivf_test_search.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 38 + +[[ivf_test_search]] +__schema__ = 'benchmark_expected_result' +__version__ = 'v0.0.0' +distance = 'MIP' + + [ivf_test_search.dataset] + __schema__ = 'benchmark_dataset_abstract' + __version__ = 'v0.0.0' + kind = 'uncompressed' + + [ivf_test_search.dataset.dataset] + __schema__ = 'benchmark_dataset_uncompressed' + __version__ = 'v0.0.0' + data_type = 'float16' + + [[ivf_test_search.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.066000000000000003 + recall_k = 10 + + [ivf_test_search.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 10 + + [[ivf_test_search.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.57833333333333337 + recall_k = 10 + + [ivf_test_search.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 50 + + [[ivf_test_search.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.50155555555555553 + recall_k = 10 + + [ivf_test_search.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 44 + + [[ivf_test_search.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.80577777777777781 + recall_k = 10 + + [ivf_test_search.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 72 + + [[ivf_test_search.config_and_recall]] + __schema__ = 'benchmark_config_and_result' + __version__ = 'v0.0.0' + num_neighbors = 10 + num_queries = 900 + recall = 0.90044444444444449 + recall_k = 10 + + [ivf_test_search.config_and_recall.search_parameters] + __schema__ = 'ivf_search_parameters' + __version__ = 'v0.0.0' + k_reorder = 1.0 + n_probes = 85 diff --git a/include/svs/core/data.h b/include/svs/core/data.h index b0d2eb8b..39d9b7fe 100644 --- a/include/svs/core/data.h +++ b/include/svs/core/data.h @@ -126,7 +126,7 @@ class VectorDataLoader { template VectorDataLoader(const UnspecializedVectorDataLoader& other) - : VectorDataLoader{other.path_, other.allocator_} { + : VectorDataLoader{other.path_} { // Validate the refinement. if (datatype_v != other.type_) { throw ANNEXCEPTION("Type mismatch!"); diff --git a/include/svs/core/data/simple.h b/include/svs/core/data/simple.h index 6d0fd7bd..86ccd6d0 100644 --- a/include/svs/core/data/simple.h +++ b/include/svs/core/data/simple.h @@ -245,6 +245,9 @@ class SimpleData { /// The type used to return a constant handle to stored vectors. using const_value_type = std::span; + /// Data wrapped in the library allocator. + using lib_alloc_data_type = SimpleData>; + /// Return the underlying allocator. const allocator_type& get_allocator() const { return data_.get_allocator(); } @@ -600,6 +603,8 @@ class SimpleData> { using value_type = std::span; using const_value_type = std::span; + using lib_alloc_data_type = SimpleData>>; + ///// Constructors SimpleData(size_t n_elements, size_t n_dimensions, const Blocked& alloc) : blocksize_{lib::prevpow2( diff --git a/include/svs/core/data/view.h b/include/svs/core/data/view.h index 21e34ce9..e9aa0c9e 100644 --- a/include/svs/core/data/view.h +++ b/include/svs/core/data/view.h @@ -66,6 +66,7 @@ template class DataViewImpl { using raw_reference = Data&; using raw_const_reference = const Data&; + using element_type = typename raw_data_type::element_type; using value_type = typename raw_data_type::value_type; using const_value_type = typename raw_data_type::const_value_type; diff --git a/include/svs/index/ivf/clustering.h b/include/svs/index/ivf/clustering.h new file mode 100644 index 00000000..707cf0c1 --- /dev/null +++ b/include/svs/index/ivf/clustering.h @@ -0,0 +1,358 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/concepts/data.h" +#include "svs/core/data.h" +#include "svs/core/logging.h" +#include "svs/index/ivf/extensions.h" +#include "svs/lib/meta.h" +#include "svs/lib/misc.h" +#include "svs/lib/saveload.h" +#include "svs/lib/threads.h" +#include "svs/lib/timing.h" + +namespace svs::index::ivf { + +struct ClusteringStats { + public: + size_t min_size_ = std::numeric_limits::max(); + size_t max_size_ = std::numeric_limits::min(); + size_t empty_clusters_ = 0; + size_t num_clusters_ = 0; + size_t num_leaves_ = 0; + double mean_size_ = 0; + double std_size_ = 0; + + public: + template ClusteringStats(Iter begin, Iter end) { + for (auto it = begin; it != end; ++it) { + const auto& list = *it; + num_clusters_++; + auto these_leaves = list.size(); + num_leaves_ += these_leaves; + min_size_ = std::min(min_size_, these_leaves); + max_size_ = std::max(max_size_, these_leaves); + if (these_leaves == 0) { + empty_clusters_ += 1; + } + } + mean_size_ = static_cast(num_leaves_) / num_clusters_; + + // Compute the standard deviation + double accum = 0; + for (auto it = begin; it != end; ++it) { + const auto& list = *it; + auto x = static_cast(list.size()) - mean_size_; + accum += x * x; + } + std_size_ = std::sqrt(accum / static_cast(num_clusters_)); + } + + std::vector prepare_report() const { + return std::vector({ + SVS_SHOW_STRING_(min_size), + SVS_SHOW_STRING_(max_size), + SVS_SHOW_STRING_(empty_clusters), + SVS_SHOW_STRING_(num_clusters), + SVS_SHOW_STRING_(num_leaves), + SVS_SHOW_STRING_(mean_size), + SVS_SHOW_STRING_(std_size), + }); + } + + [[nodiscard]] std::string report() const { return report(", "); } + [[nodiscard]] std::string report(std::string_view separator) const { + return fmt::format("{}", fmt::join(prepare_report(), separator)); + } +}; + +template class Clustering { + public: + Data centroids_; + std::vector> clusters_; + + public: + using vector_type = std::vector; + using T = typename Data::element_type; + + // Type Aliases + using iterator = typename std::vector::iterator; + using const_iterator = typename std::vector::const_iterator; + + public: + Clustering() = default; + + Clustering(size_t n_clusters, size_t n_dims) + : centroids_{n_clusters, n_dims} + , clusters_(n_clusters) {} + + Clustering(Data centroids, std::vector clusters) + : centroids_{std::move(centroids)} + , clusters_{std::move(clusters)} {} + + size_t size() const { return clusters_.size(); } + + void check_valid(size_t cluster_id) const { + if (cluster_id > size()) { + throw ANNEXCEPTION( + "Cluster id {} can't be higher than the number of clusters!", + cluster_id, + size() + ); + } + } + + size_t size(size_t id) const { + check_valid(id); + return clusters_[id].size(); + } + + const vector_type& cluster(size_t id) const { + check_valid(id); + return clusters_[id]; + } + + vector_type& cluster(size_t id) { + check_valid(id); + return clusters_[id]; + } + + Data centroids() { return centroids_; } + + // Iterators + iterator begin() { return clusters_.begin(); } + iterator end() { return clusters_.end(); } + + const_iterator cbegin() const { return clusters_.cbegin(); } + const_iterator cend() const { return clusters_.cend(); } + + template void for_each_cluster(F&& f) const { + for (size_t i = 0; i < size(); i++) { + f(cluster(i)); + } + } + + template + void for_each_cluster_parallel(F&& f, Pool& threadpool) const { + threads::parallel_for( + threadpool, + threads::StaticPartition{size()}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + f(cluster(i), i); + } + } + ); + } + + ClusteringStats statistics() const { return ClusteringStats(cbegin(), cend()); } + + // Serializing and Deserializing. + size_t serialize_clusters(std::ostream& stream) const { + size_t bytes = lib::write_binary(stream, size()); + for (size_t i = 0; i < size(); i++) { + bytes += lib::write_binary(stream, size(i)); + bytes += lib::write_binary(stream, clusters_[i]); + } + return bytes; + } + + static std::vector deserialize_clusters(std::istream& stream) { + size_t n_clusters = lib::read_binary(stream); + std::vector clusters(n_clusters); + for (size_t i = 0; i < n_clusters; i++) { + size_t cluster_size = lib::read_binary(stream); + clusters[i].resize(cluster_size); + lib::read_binary(stream, clusters[i]); + } + return clusters; + } + + // Saving and Loading. + static constexpr lib::Version save_version{0, 0, 0}; + static constexpr std::string_view serialization_schema = "IVF clustering"; + lib::SaveTable save(const lib::SaveContext& ctx) const { + // Serialize all clusters into an auxiliary file. + auto fullpath = ctx.generate_name("clusters", "bin"); + size_t filesize = 0; + { + auto io = lib::open_write(fullpath); + filesize += serialize_clusters(io); + } + + return lib::SaveTable( + serialization_schema, + save_version, + {SVS_LIST_SAVE_(centroids, ctx), + {"filepath", lib::save(fullpath.filename())}, + SVS_LIST_SAVE(filesize), + {"data_type", lib::save(datatype_v)}, + {"integer_type", lib::save(datatype_v)}, + {"num_clusters", lib::save(size())}} + ); + } + + template + static Clustering load(const lib::LoadTable& table, Pool& threadpool) { + auto saved_data_type = lib::load_at(table, "data_type"); + + // Ensure we have the correct integer type when decoding. + auto saved_integer_type = lib::load_at(table, "integer_type"); + if (saved_integer_type != datatype_v) { + auto type = datatype_v; + throw ANNEXCEPTION( + "Clustering was saved using {} but we're trying to reload it using {}!", + saved_integer_type, + type + ); + } + + auto expected_filesize = lib::load_at(table, "filesize"); + + auto file = table.resolve_at("filepath"); + size_t actual_filesize = std::filesystem::file_size(file); + if (actual_filesize != expected_filesize) { + throw ANNEXCEPTION( + "Expected cluster file size to be {}. Instead, it is {}!", + actual_filesize, + expected_filesize + ); + } + + auto io = lib::open_read(file); + if (saved_data_type != datatype_v) { + auto centroids_orig = + lib::load_at>(table, "centroids"); + if constexpr (std::is_same_v || std::is_same_v) { + auto centroids = convert_data(centroids_orig, threadpool); + return Clustering{centroids, deserialize_clusters(io)}; + } else { + throw ANNEXCEPTION("Centroids datatype {} not supported!", datatype_v); + } + } + + return Clustering{ + SVS_LOAD_MEMBER_AT_(table, centroids), deserialize_clusters(io)}; + } +}; + +template struct DenseCluster { + public: + DenseCluster(Data data, std::vector ids) + : data_{std::move(data)} + , ids_{std::move(ids)} { + if (data_.size() != ids_.size()) { + throw ANNEXCEPTION("Size mismatch!"); + } + } + + size_t size() const { return data_.size(); } + + template + void on_leaves(Callback&& f, size_t prefetch_offset) const { + size_t p = 0; + size_t clustersize = size(); + auto accessor = extensions::accessor(data_); + + for (size_t pmax = std::min(prefetch_offset, clustersize); p < pmax; ++p) { + accessor.prefetch(data_, p); + } + + for (size_t i = 0; i < clustersize; ++i) { + if (p < clustersize) { + accessor.prefetch(data_, p); + ++p; + } + f(accessor(data_, i), ids_[i], i); + } + } + + auto get_datum(size_t id) const { return data_.get_datum(id); } + auto get_secondary(size_t id) const { + return data_.get_secondary(id); + } + auto get_global_id(size_t local_id) const { return ids_[local_id]; } + const Data& view_cluster() const { return data_; } + + public: + Data data_; + std::vector ids_; +}; + +template < + data::ImmutableMemoryDataset Centroids, + std::integral I, + data::ImmutableMemoryDataset Data> +class DenseClusteredDataset { + public: + // Type aliases + using index_type = I; + using data_type = Data; + + // Constructor + template + DenseClusteredDataset( + const Clustering& clustering, + const Original& original, + Pool& threadpool, + const Alloc& allocator + ) + : clusters_{} { + clustering.for_each_cluster([&](const auto& cluster) { + size_t cluster_size = cluster.size(); + clusters_.emplace_back( + extensions::create_dense_cluster(original, cluster_size, allocator), + std::vector(cluster_size) + ); + }); + + clustering.for_each_cluster_parallel( + [&](const auto& cluster, size_t cluster_id) { + auto& leaf = clusters_[cluster_id]; + extensions::set_dense_cluster(original, leaf.data_, cluster, leaf.ids_); + }, + threadpool + ); + } + + template void on_leaves(Callback&& f, size_t cluster) const { + clusters_.at(cluster).on_leaves(SVS_FWD(f), prefetch_offset_); + } + + size_t get_prefetch_offset() const { return prefetch_offset_; } + void set_prefetch_offset(size_t offset) { prefetch_offset_ = offset; } + auto get_datum(size_t cluster, size_t id) const { + return clusters_.at(cluster).get_datum(id); + } + auto get_secondary(size_t cluster, size_t id) const { + return clusters_.at(cluster).get_secondary(id); + } + auto get_global_id(size_t cluster, size_t id) const { + return clusters_.at(cluster).get_global_id(id); + } + const Data& view_cluster(size_t cluster) const { + return clusters_.at(cluster).view_cluster(); + } + + private: + std::vector> clusters_; + size_t prefetch_offset_ = 8; +}; + +} // namespace svs::index::ivf diff --git a/include/svs/index/ivf/common.h b/include/svs/index/ivf/common.h new file mode 100644 index 00000000..02aa1439 --- /dev/null +++ b/include/svs/index/ivf/common.h @@ -0,0 +1,700 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/concepts/data.h" +#include "svs/concepts/distance.h" +#include "svs/core/data/simple.h" +#include "svs/core/data/view.h" +#include "svs/core/distance.h" +#include "svs/core/logging.h" +#include "svs/index/ivf/sorted_buffer.h" +#include "svs/lib/exception.h" +#include "svs/lib/neighbor.h" +#include "svs/lib/threads/threadpool.h" +#include "svs/lib/timing.h" +#include "svs/lib/type_traits.h" + +// external +#include "tsl/robin_set.h" + +//Intel(R) MKL +#include + +// stl +#include + +// Common definitions. +namespace svs::index::ivf { + +#define EPSILON (1 / 1024.) + +/// @brief Parameters controlling the IVF build/k-means algortihm. +struct IVFBuildParameters { + public: + /// The target number of clusters in the final result. + size_t num_centroids_ = 1000; + /// The size of each minibatch. + size_t minibatch_size_ = 10'000; + /// The number of iterations used in kmeans training. + size_t num_iterations_ = 10; + /// Use hierarchical Kmeans + bool is_hierarchical_ = true; + /// Fraction of dataset used for training + float training_fraction_ = 0.1; + // Level1 clusters for hierarchical kmeans (use heuristic when 0) + size_t hierarchical_level1_clusters_ = 0; + /// The initial seed for the random number generator. + size_t seed_ = 0xc0ffee; + + public: + IVFBuildParameters() = default; + IVFBuildParameters( + size_t num_centroids, + size_t minibatch_size = 10'000, + size_t num_iterations = 10, + bool is_hierarchical = true, + float training_fraction = 0.1, + size_t hierarchical_level1_clusters = 0, + size_t seed = 0xc0ffee + ) + : num_centroids_{num_centroids} + , minibatch_size_{minibatch_size} + , num_iterations_{num_iterations} + , is_hierarchical_{is_hierarchical} + , training_fraction_{training_fraction} + , hierarchical_level1_clusters_{hierarchical_level1_clusters} + , seed_{seed} {} + + // Chain setters to help with construction. + SVS_CHAIN_SETTER_(IVFBuildParameters, num_centroids); + SVS_CHAIN_SETTER_(IVFBuildParameters, minibatch_size); + SVS_CHAIN_SETTER_(IVFBuildParameters, num_iterations); + SVS_CHAIN_SETTER_(IVFBuildParameters, is_hierarchical); + SVS_CHAIN_SETTER_(IVFBuildParameters, training_fraction); + SVS_CHAIN_SETTER_(IVFBuildParameters, hierarchical_level1_clusters); + SVS_CHAIN_SETTER_(IVFBuildParameters, seed); + + // Comparison + friend constexpr bool + operator==(const IVFBuildParameters&, const IVFBuildParameters&) = default; + + // Saving and Loading. + static constexpr svs::lib::Version save_version{0, 0, 0}; + static constexpr std::string_view serialization_schema = "ivf_build_parameters"; + lib::SaveTable save() const { + return lib::SaveTable( + serialization_schema, + save_version, + { + SVS_LIST_SAVE_(num_centroids), + SVS_LIST_SAVE_(minibatch_size), + SVS_LIST_SAVE_(num_iterations), + SVS_LIST_SAVE_(is_hierarchical), + SVS_LIST_SAVE_(training_fraction), + SVS_LIST_SAVE_(hierarchical_level1_clusters), + {"seed", lib::save(lib::FullUnsigned(seed_))}, + } + ); + } + + static IVFBuildParameters load(const lib::ContextFreeLoadTable& table) { + return IVFBuildParameters( + SVS_LOAD_MEMBER_AT_(table, num_centroids), + SVS_LOAD_MEMBER_AT_(table, minibatch_size), + SVS_LOAD_MEMBER_AT_(table, num_iterations), + SVS_LOAD_MEMBER_AT_(table, is_hierarchical), + SVS_LOAD_MEMBER_AT_(table, training_fraction), + SVS_LOAD_MEMBER_AT_(table, hierarchical_level1_clusters), + lib::load_at(table, "seed") + ); + } +}; + +/// @brief Parameters controlling the IVF search algorithm. +struct IVFSearchParameters { + public: + /// The number of nearest clusters to be explored + size_t n_probes_ = 1; + /// Level of reordering or reranking done when using compressed datasets + float k_reorder_ = 1.0; + + public: + IVFSearchParameters() = default; + + IVFSearchParameters(size_t n_probes, float k_reorder) + : n_probes_{n_probes} + , k_reorder_{k_reorder} {} + + SVS_CHAIN_SETTER_(IVFSearchParameters, n_probes); + SVS_CHAIN_SETTER_(IVFSearchParameters, k_reorder); + + // Saving and Loading. + static constexpr lib::Version save_version{0, 0, 0}; + static constexpr std::string_view serialization_schema = "ivf_search_parameters"; + lib::SaveTable save() const { + return lib::SaveTable( + serialization_schema, + save_version, + {SVS_LIST_SAVE_(n_probes), SVS_LIST_SAVE_(k_reorder)} + ); + } + + static IVFSearchParameters load(const lib::ContextFreeLoadTable& table) { + return IVFSearchParameters{ + SVS_LOAD_MEMBER_AT_(table, n_probes), SVS_LOAD_MEMBER_AT_(table, k_reorder)}; + } + + constexpr friend bool + operator==(const IVFSearchParameters&, const IVFSearchParameters&) = default; +}; + +// Helper functions to convert data from one type to another using threadpool +template +void convert_data(D1& src, D2& dst, Pool& threadpool) { + // Note: Destination size can be bigger than the source as we preallocate a bigger + // buffer and reuse it to reduce the cost of frequent allocations + if (src.size() > dst.size() || src.dimensions() != dst.dimensions()) { + throw ANNEXCEPTION( + "Unexpected data shapes sizes: {}, {}; dims: {}, {}!", + src.size(), + dst.size(), + src.dimensions(), + dst.dimensions() + ); + } + + threads::parallel_for( + threadpool, + threads::StaticPartition{src.size()}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + dst.set_datum(i, src.get_datum(i)); + } + } + ); +} + +template +auto convert_data(Data& src, Pool& threadpool) { + auto dst = svs::data::SimpleData(src.size(), src.dimensions()); + convert_data(src, dst, threadpool); + return dst; +} + +// Partial specialization to preserve the dimensionality and Allocation type +template +auto convert_data( + svs::data::SimpleData& src, Pool& threadpool +) { + using allocator_type = svs::lib::rebind_allocator_t; + allocator_type rebound_alloctor = {}; + + auto dst = svs::data::SimpleData( + src.size(), src.dimensions(), rebound_alloctor + ); + convert_data(src, dst, threadpool); + return dst; +} + +template +auto convert_data(Data& src) { + auto dst = svs::data::SimpleData(src.size(), src.dimensions()); + auto threadpool = threads::as_threadpool(1); + convert_data(src, dst, threadpool); + return dst; +} + +template +void compute_matmul( + const T* data, const T* centroids, float* results, size_t m, size_t n, size_t k +) { + if constexpr (std::is_same_v) { + cblas_sgemm( + CblasRowMajor, // CBLAS_LAYOUT layout + CblasNoTrans, // CBLAS_TRANSPOSE TransA + CblasTrans, // CBLAS_TRANSPOSE TransB + m, // const int M + n, // const int N + k, // const int K + 1.0, // float alpha + data, // const float* A + k, // const int lda + centroids, // const float* B + k, // const int ldb + 0.0, // const float beta + results, // float* c + n // const int ldc + ); + } else if constexpr (std::is_same_v) { + cblas_gemm_bf16bf16f32( + CblasRowMajor, // CBLAS_LAYOUT layout + CblasNoTrans, // CBLAS_TRANSPOSE TransA + CblasTrans, // CBLAS_TRANSPOSE TransB + m, // const int M + n, // const int N + k, // const int K + 1.0, // float alpha + (const uint16_t*)data, // const *uint16_t A + k, // const int lda + (const uint16_t*)centroids, // const uint16_t* B + k, // const int ldb + 0.0, // const float beta + results, // float* c + n // const int ldc + ); + } else if constexpr (std::is_same_v) { + cblas_gemm_f16f16f32( + CblasRowMajor, // CBLAS_LAYOUT layout + CblasNoTrans, // CBLAS_TRANSPOSE TransA + CblasTrans, // CBLAS_TRANSPOSE TransB + m, // const int M + n, // const int N + k, // const int K + 1.0, // float alpha + (const uint16_t*)data, // const *uint16_t A + k, // const int lda + (const uint16_t*)centroids, // const uint16_t* B + k, // const int ldb + 0.0, // const float beta + results, // float* c + n // const int ldc + ); + } else { + throw ANNEXCEPTION("GEMM type not supported!"); + } +} + +inline static void generate_unique_ids(std::vector& ids, size_t id_range, std::mt19937& rng) { + size_t n = ids.size(); + tsl::robin_set seen; + seen.reserve(n); + + while (seen.size() < n) { + auto j = rng() % id_range; + seen.insert(j); + } + + size_t i = 0; + for (auto it = seen.begin(); it != seen.end(); ++it, ++i) { + ids[i] = *it; + } +} + +template +void normalize_centroids( + data::SimpleData& centroids, Pool& threadpool, lib::Timer& timer +) { + auto normalize_centroids_t = timer.push_back("normalize centroids"); + threads::parallel_for( + threadpool, + threads::StaticPartition{centroids.size()}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + auto datum = centroids.get_datum(i); + float norm = distance::norm(datum); + if (norm != 0.0) { + float norm_inv = 1.0 / norm; + for (size_t j = 0; j < datum.size(); j++) { + datum[j] = datum[j] * norm_inv; + } + } + } + } + ); + normalize_centroids_t.finish(); +} + +template < + data::ImmutableMemoryDataset Data, + typename T, + typename Distance, + threads::ThreadPool Pool> +void centroid_assignment( + Data& data, + std::vector& data_norm, + threads::UnitRange batch_range, + Distance& SVS_UNUSED(distance), + data::SimpleData& centroids, + std::vector& centroids_norm, + std::vector& assignments, + data::SimpleData& matmul_results, + Pool& threadpool, + lib::Timer& timer +) { + auto generate_assignments = timer.push_back("generate assignments"); + threads::parallel_for( + threadpool, + threads::StaticPartition{batch_range.size()}, + [&](auto indices, auto /*tid*/) { + auto range = threads::UnitRange(indices); + compute_matmul( + data.get_datum(range.start()).data(), + centroids.data(), + matmul_results.get_datum(range.start()).data(), + range.size(), + centroids.size(), + data.dimensions() + ); + if constexpr (std::is_same_v) { + for (auto i : indices) { + auto nearest = + type_traits::sentinel_v, std::greater<>>; + auto dists = matmul_results.get_datum(i); + for (size_t j = 0; j < centroids.size(); j++) { + nearest = std::max(nearest, Neighbor(j, dists[j])); + } + assignments[batch_range.start() + i] = nearest.id(); + } + } else if constexpr (std::is_same_v) { + for (auto i : indices) { + auto nearest = type_traits::sentinel_v, std::less<>>; + auto dists = matmul_results.get_datum(i); + for (size_t j = 0; j < centroids.size(); j++) { + auto dist = data_norm[batch_range.start() + i] + centroids_norm[j] - + 2 * dists[j]; + nearest = std::min(nearest, Neighbor(j, dist)); + } + assignments[batch_range.start() + i] = nearest.id(); + } + } else { + throw ANNEXCEPTION("Only L2 and MIP distances supported in IVF build!"); + } + } + ); + generate_assignments.finish(); +} + +template +void centroid_adjustment( + Data& data, + data::SimpleData& centroids, + std::vector& assignments, + std::vector& counts, + Pool& threadpool, + lib::Timer& timer +) { + auto adjust_centroids = timer.push_back("adjust centroids"); + size_t n_threads = threadpool.size(); + size_t n_centroids = centroids.size(); + + threads::parallel_for( + threadpool, + threads::StaticPartition{n_threads}, + [&](auto /*indices*/, auto tid) { + size_t centroid_start = (n_centroids * tid) / n_threads; + size_t centroid_end = (n_centroids * (tid + 1)) / n_threads; + for (auto i : data.eachindex()) { + auto assignment = assignments[i]; + if (assignment >= centroid_start && assignment < centroid_end) { + counts.at(assignment)++; + auto datum = data.get_datum(i); + auto this_centroid = centroids.get_datum(assignment); + for (size_t p = 0, pmax = this_centroid.size(); p < pmax; ++p) { + this_centroid[p] += lib::narrow_cast(datum[p]); + } + } + } + } + ); + + threads::parallel_for( + threadpool, + threads::StaticPartition{n_centroids}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + if (counts.at(i) != 0) { + auto this_centroid = centroids.get_datum(i); + float norm = 1.0 / (counts.at(i) + 1); + // float norm = 1.0 / counts.at(i); + for (size_t p = 0, pmax = this_centroid.size(); p < pmax; ++p) { + this_centroid[p] *= norm; + } + } + } + } + ); + adjust_centroids.finish(); +} + +template +void centroid_split( + Data& data, + data::SimpleData& centroids, + std::vector& counts, + std::mt19937& rng, + Pool& SVS_UNUSED(threadpool), + lib::Timer& timer +) { + auto split_centroids = timer.push_back("split centroids"); + + auto num_centroids = centroids.size(); + auto dims = centroids.dimensions(); + auto num_data = data.size(); + + auto distribution = std::uniform_real_distribution(0.0, 1.0); + for (size_t i = 0; i < num_centroids; i++) { + if (counts.at(i) == 0) { + size_t j; + for (j = 0; true; j = (j + 1) % num_centroids) { + if (counts.at(j) == 0) { + continue; + } + float p = counts.at(j) / float(num_data); + float r = distribution(rng); + if (r < p) { + break; + } + } + centroids.set_datum(i, centroids.get_datum(j)); + for (size_t k = 0; k < dims; k++) { + if (k % 2 == 0) { + centroids.get_datum(i)[k] *= 1 + EPSILON; + centroids.get_datum(j)[k] *= 1 - EPSILON; + } else { + centroids.get_datum(i)[k] *= 1 - EPSILON; + centroids.get_datum(j)[k] *= 1 + EPSILON; + } + } + counts.at(i) = counts.at(j) / 2; + counts.at(j) -= counts.at(i); + } + } + split_centroids.finish(); +} + +template +void generate_norms( + Data& data, std::vector& norms, Pool& threadpool +) { + norms.resize(data.size()); + threads::parallel_for( + threadpool, + threads::StaticPartition{data.size()}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + norms[i] = distance::norm_square(data.get_datum(i)); + } + } + ); +} + +template < + data::ImmutableMemoryDataset Data, + typename T, + typename Distance, + threads::ThreadPool Pool> +auto kmeans_training( + const IVFBuildParameters& parameters, + Data& data, + Distance& distance, + data::SimpleData& centroids, + data::SimpleData& matmul_results, + std::mt19937& rng, + Pool& threadpool, + lib::Timer& timer +) { + auto training_timer = timer.push_back("Kmeans training"); + data::SimpleData centroids_fp32 = convert_data(centroids, threadpool); + + if constexpr (std::is_same_v) { + normalize_centroids(centroids_fp32, threadpool, timer); + } + + auto assignments = std::vector(data.size()); + std::vector data_norm; + if constexpr (std::is_same_v) { + generate_norms(data, data_norm, threadpool); + } + std::vector centroids_norm; + + for (size_t iter = 0; iter < parameters.num_iterations_; ++iter) { + auto iter_timer = timer.push_back("iteration"); + auto batchsize = parameters.minibatch_size_; + auto num_batches = lib::div_round_up(data.size(), batchsize); + if constexpr (std::is_same_v) { + generate_norms(centroids_fp32, centroids_norm, threadpool); + } + + // Convert from fp32 to fp16/bf16 + convert_data(centroids_fp32, centroids, threadpool); + + for (size_t batch = 0; batch < num_batches; ++batch) { + auto this_batch = threads::UnitRange{ + batch * batchsize, std::min((batch + 1) * batchsize, data.size())}; + auto data_batch = data::make_view(data, this_batch); + centroid_assignment( + data_batch, + data_norm, + this_batch, + distance, + centroids, + centroids_norm, + assignments, + matmul_results, + threadpool, + timer + ); + } + + // Convert back to fp32 + convert_data(centroids, centroids_fp32, threadpool); + + auto counts = std::vector(centroids.size()); + centroid_adjustment(data, centroids_fp32, assignments, counts, threadpool, timer); + + centroid_split(data, centroids_fp32, counts, rng, threadpool, timer); + + if constexpr (std::is_same_v) { + normalize_centroids(centroids_fp32, threadpool, timer); + } + } + + // Finally call the conversion to get the updated centroids + // after adjustment and split + convert_data(centroids_fp32, centroids, threadpool); + training_timer.finish(); + return centroids_fp32; +} + +template < + data::ImmutableMemoryDataset Queries, + typename Centroids, + typename MatMulResults, + threads::ThreadPool Pool> +void compute_centroid_distances( + const Queries& queries, + const Centroids& centroids, + MatMulResults& matmul_results, + Pool& threadpool +) { + using TQ = typename Queries::element_type; + using TC = typename Centroids::element_type; + size_t num_centroids = centroids.size(); + size_t num_queries = queries.size(); + data::SimpleData queries_conv; + + // Convert if Queries and Centroids datatypes are not same + if constexpr (!std::is_same_v) { + queries_conv = convert_data(queries, threadpool); + } + + threads::parallel_for( + threadpool, + threads::StaticPartition(num_centroids), + [&](auto is, auto tid) { + auto batch = threads::UnitRange{is}; + if constexpr (!std::is_same_v) { + compute_matmul( + queries_conv.data(), + centroids.get_datum(batch.start()).data(), + matmul_results[tid].data(), + num_queries, + batch.size(), + queries.dimensions() + ); + } else { + compute_matmul( + queries.get_datum(0).data(), + centroids.get_datum(batch.start()).data(), + matmul_results[tid].data(), + num_queries, + batch.size(), + queries.dimensions() + ); + } + } + ); +} + +template +void search_centroids( + const Query& query, + Dist& SVS_UNUSED(dist), + const MatMulResults& matmul_results, + Buffer& buffer, + size_t query_id, + const std::vector& centroids_norm, + size_t num_threads +) { + unsigned int count = 0; + buffer.clear(); + if constexpr (std::is_same_v) { + for (size_t j = 0; j < num_threads; j++) { + auto distance = matmul_results[j].get_datum(query_id); + for (size_t k = 0; k < distance.size(); k++) { + buffer.insert({count, distance[k]}); + count++; + } + } + } else if constexpr (std::is_same_v) { + float query_norm = distance::norm_square(query); + for (size_t j = 0; j < num_threads; j++) { + auto distance = matmul_results[j].get_datum(query_id); + for (size_t k = 0; k < distance.size(); k++) { + float dist = query_norm + centroids_norm[count] - 2 * distance[k]; + buffer.insert({count, dist}); + count++; + } + } + } else { + throw ANNEXCEPTION("Only L2 and MIP distances supported in IVF search!"); + } +} + +template < + typename Query, + typename Dist, + typename Cluster, + typename BufferCentroids, + typename BufferLeaves, + threads::ThreadPool Pool> +void search_leaves( + const Query& query, + Dist& dist, + const Cluster& cluster, + const BufferCentroids& buffer_centroids, + BufferLeaves& buffer_leaves, + Pool& threadpool_inner +) { + for (size_t j = 0; j < buffer_leaves.size(); j++) { + buffer_leaves[j].clear(); + } + distance::maybe_fix_argument(dist, query); + threads::parallel_for( + threadpool_inner, + threads::DynamicPartition(buffer_centroids.size(), 1), + [&](auto js, auto tid_inner) { + for (auto j : js) { + auto candidate = buffer_centroids[j]; + auto cluster_id = candidate.id(); + + // Compute the distance between the query and each leaf element. + cluster.on_leaves( + [&](const auto& datum, unsigned int /*gid*/, unsigned int lid) { + auto distance = distance::compute(dist, query, datum); + buffer_leaves[tid_inner].insert({cluster_id, distance, lid}); + }, + cluster_id + ); + } + } + ); +} + +} // namespace svs::index::ivf diff --git a/include/svs/index/ivf/extensions.h b/include/svs/index/ivf/extensions.h new file mode 100644 index 00000000..0ef83587 --- /dev/null +++ b/include/svs/index/ivf/extensions.h @@ -0,0 +1,220 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/core/distance.h" +#include "svs/core/query_result.h" +#include "svs/index/ivf/common.h" +#include "svs/lib/invoke.h" + +namespace svs::index::ivf::extensions { + +struct IVFAccessor { + template + svs::svs_invoke_result_t operator()(const Data& data) const { + return svs::svs_invoke(*this, data); + } +}; + +inline constexpr IVFAccessor accessor{}; + +template +data::GetDatumAccessor svs_invoke(svs::tag_t, const Data& SVS_UNUSED(data)) { + return data::GetDatumAccessor{}; +} + +struct IVFPerThreadBatchSearchSetupType { + template + using result_t = svs:: + svs_invoke_result_t; + + template + result_t + operator()(const Data& dataset, const Distance& distance) const { + return svs::svs_invoke(*this, dataset, distance); + } +}; + +inline constexpr IVFPerThreadBatchSearchSetupType per_thread_batch_search_setup{}; + +// Default Implementation. +template +Distance svs_invoke( + svs::tag_t, + const Data& SVS_UNUSED(dataset), + const Distance& distance +) { + return threads::shallow_copy(distance); +} + +/// +/// @brief Customization point for working with a batch of threads. +/// +struct IVFPerThreadBatchSearchType { + template < + data::ImmutableMemoryDataset Data, + typename Cluster, + typename BufferCentroids, + typename BufferLeaves, + typename Scratch, + data::ImmutableMemoryDataset Queries, + std::integral I, + typename SearchCentroids, + typename SearchLeaves> + SVS_FORCE_INLINE void operator()( + const Data& data, + const Cluster& cluster, + BufferCentroids& buffer_centroids, + BufferLeaves& buffer_leaves, + Scratch& scratch, + const Queries& queries, + QueryResultView& result, + threads::UnitRange thread_indices, + size_t tid, + const SearchCentroids& search_centroids, + const SearchLeaves& search_leaves + ) const { + svs::svs_invoke( + *this, + data, + cluster, + buffer_centroids, + buffer_leaves, + scratch, + queries, + result, + thread_indices, + tid, + search_centroids, + search_leaves + ); + } +}; + +/// @brief Customization point object for batch search. +inline constexpr IVFPerThreadBatchSearchType per_thread_batch_search{}; + +// Default Implementation +template < + typename Data, + typename Cluster, + typename BufferCentroids, + typename BufferLeaves, + typename Distance, + typename Queries, + std::integral I, + typename SearchCentroids, + typename SearchLeaves> +void svs_invoke( + svs::tag_t, + const Data& SVS_UNUSED(dataset), + const Cluster& cluster, + BufferCentroids& buffer_centroids, + BufferLeaves& buffer_leaves, + Distance& distance, + const Queries& queries, + QueryResultView& result, + threads::UnitRange thread_indices, + const size_t tid, + const SearchCentroids& search_centroids, + const SearchLeaves& search_leaves +) { + size_t n_inner_threads = buffer_leaves.size(); + size_t buffer_leaves_size = buffer_leaves[0].capacity(); + size_t num_neighbors = result.n_neighbors(); + + // Fallback implementation + for (auto i : thread_indices) { + const auto& query = queries.get_datum(i); + search_centroids(query, buffer_centroids, i); + search_leaves(query, distance, buffer_centroids, buffer_leaves, tid); + + // Accumulate results from intra-query threads + for (size_t j = 1; j < n_inner_threads; ++j) { + for (size_t k = 0; k < buffer_leaves_size; ++k) { + buffer_leaves[0].insert(buffer_leaves[j][k]); + } + } + + for (size_t j = 0; j < buffer_leaves_size; ++j) { + auto& neighbor = buffer_leaves[0][j]; + auto cluster_id = neighbor.id(); + auto local_id = neighbor.get_local_id(); + auto global_id = cluster.get_global_id(cluster_id, local_id); + neighbor.set_id(global_id); + } + + // Store results + for (size_t j = 0; j < num_neighbors; ++j) { + result.set(buffer_leaves[0][j], i, j); + } + } +} + +struct CreateDenseCluster { + using This = CreateDenseCluster; + + template + using return_type = svs::svs_invoke_result_t; + + template + return_type + operator()(const Data& original, size_t new_size, const Alloc& allocator) const { + return svs::svs_invoke(*this, original, new_size, allocator); + } +}; + +inline constexpr CreateDenseCluster create_dense_cluster{}; + +template +svs::data::SimpleData svs_invoke( + svs::tag_t, + const svs::data::SimpleData& original, + size_t new_size, + const NewAlloc& SVS_UNUSED(allocator) +) { + return svs::data::SimpleData(new_size, original.dimensions()); +} + +struct SetDenseCluster { + template + void operator()( + const Src& src, Dst& dst, const std::vector& src_ids, std::vector& dst_ids + ) const { + return svs::svs_invoke(*this, src, dst, src_ids, dst_ids); + } +}; + +inline constexpr SetDenseCluster set_dense_cluster{}; + +template +void svs_invoke( + svs::tag_t, + const Src& src, + Dst& dst, + const std::vector& src_ids, + std::vector& dst_ids +) { + size_t i = 0; + for (auto id : src_ids) { + dst.set_datum(i, src.get_datum(id)); + dst_ids[i] = id; + ++i; + } +} + +} // namespace svs::index::ivf::extensions diff --git a/include/svs/index/ivf/hierarchical_kmeans.h b/include/svs/index/ivf/hierarchical_kmeans.h new file mode 100644 index 00000000..395eb4c2 --- /dev/null +++ b/include/svs/index/ivf/hierarchical_kmeans.h @@ -0,0 +1,369 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/index/ivf/common.h" + +// stdlib +#include + +namespace svs::index::ivf { + +template +auto calc_level2_clusters( + size_t num_clusters, + size_t num_level1_clusters, + std::vector>& clusters_level1, + size_t num_training_data, + std::mt19937& rng +) { + auto num_level2_clusters = std::vector(num_level1_clusters); + size_t total_centroids_l2 = 0; + for (size_t cluster = 0; cluster < num_level1_clusters; cluster++) { + num_level2_clusters[cluster] = + (1.0 * clusters_level1[cluster].size()) / num_training_data * num_clusters; + total_centroids_l2 += num_level2_clusters[cluster]; + } + while (total_centroids_l2 < num_clusters) { + size_t j = rng() % num_level1_clusters; + if (clusters_level1[j].size() != 0) { + num_level2_clusters[j]++; + total_centroids_l2++; + } + } + return num_level2_clusters; +} + +template < + typename BuildType, + data::ImmutableMemoryDataset Data, + typename Distance, + threads::ThreadPool Pool, + std::integral I = uint32_t> +auto hierarchical_kmeans_clustering_impl( + const IVFBuildParameters& parameters, + Data& data, + Distance& distance, + Pool& threadpool, + lib::Type SVS_UNUSED(integer_type) = {} +) { + auto timer = lib::Timer(); + auto kmeans_timer = timer.push_back("Hierarchical kmeans clustering"); + auto init_timer = timer.push_back("init"); + auto init_trainset_centroid = timer.push_back("create trainset and centroids"); + + constexpr size_t Dims = Data::extent; + using Alloc = svs::HugepageAllocator; + size_t ndims = data.dimensions(); + auto num_clusters = parameters.num_centroids_; + + size_t num_level1_clusters = parameters.hierarchical_level1_clusters_; + if (num_level1_clusters == 0) { + num_level1_clusters = std::sqrt(num_clusters); + } + + fmt::print("Level1 clusters: {}\n", num_level1_clusters); + + size_t num_training_data = + lib::narrow(std::ceil(data.size() * parameters.training_fraction_)); + if (num_training_data < num_clusters || num_training_data > data.size()) { + throw ANNEXCEPTION( + "Invalid number of training data: {}, num_clusters: {}, total data size: " + "{}\n", + num_training_data, + num_clusters, + data.size() + ); + } + + auto centroids_level1 = data::SimpleData{num_level1_clusters, ndims}; + auto data_train = data::SimpleData{num_training_data, ndims}; + auto matmul_results_level1 = + data::SimpleData{parameters.minibatch_size_, num_level1_clusters}; + auto rng = std::mt19937(parameters.seed_); + + std::vector v(num_training_data); + generate_unique_ids(v, data.size(), rng); + threads::parallel_for( + threadpool, + threads::StaticPartition{num_training_data}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + data_train.set_datum(i, data.get_datum(v[i])); + } + } + ); + + v.resize(num_level1_clusters); + generate_unique_ids(v, data_train.size(), rng); + threads::parallel_for( + threadpool, + threads::StaticPartition{num_level1_clusters}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + centroids_level1.set_datum(i, data_train.get_datum(v[i])); + } + } + ); + init_trainset_centroid.finish(); + init_timer.finish(); + + auto level1_training_time = timer.push_back("Level1 training"); + auto centroids_level1_fp32 = kmeans_training( + parameters, + data_train, + distance, + centroids_level1, + matmul_results_level1, + rng, + threadpool, + timer + ); + auto assignments_level1 = std::vector(data_train.size()); + auto batchsize = parameters.minibatch_size_; + auto num_batches = lib::div_round_up(data_train.size(), batchsize); + + std::vector data_norm; + if constexpr (std::is_same_v) { + generate_norms(data_train, data_norm, threadpool); + } + std::vector centroids_level1_norm; + if constexpr (std::is_same_v) { + generate_norms(centroids_level1_fp32, centroids_level1_norm, threadpool); + } + + for (size_t batch = 0; batch < num_batches; ++batch) { + auto this_batch = threads::UnitRange{ + batch * batchsize, std::min((batch + 1) * batchsize, data_train.size())}; + auto data_batch = data::make_view(data_train, this_batch); + centroid_assignment( + data_batch, + data_norm, + this_batch, + distance, + centroids_level1, + centroids_level1_norm, + assignments_level1, + matmul_results_level1, + threadpool, + timer + ); + } + auto clusters_level1 = std::vector>(num_level1_clusters); + for (auto i : data_train.eachindex()) { + clusters_level1[assignments_level1[i]].push_back(i); + } + + auto all_assignments_time = timer.push_back("level1 all assignments"); + auto all_assignments_alloc = timer.push_back("level1 all assignments alloc"); + auto assignments_level1_all = std::vector(data.size()); + all_assignments_alloc.finish(); + + batchsize = parameters.minibatch_size_; + num_batches = lib::div_round_up(data.size(), batchsize); + + if constexpr (std::is_same_v) { + generate_norms(data, data_norm, threadpool); + } + + auto data_batch = data::SimpleData{batchsize, ndims}; + for (size_t batch = 0; batch < num_batches; ++batch) { + auto this_batch = threads::UnitRange{ + batch * batchsize, std::min((batch + 1) * batchsize, data.size())}; + auto data_batch_view = data::make_view(data, this_batch); + auto all_assignments_convert = timer.push_back("level1 all assignments convert"); + convert_data(data_batch_view, data_batch, threadpool); + all_assignments_convert.finish(); + centroid_assignment( + data_batch, + data_norm, + this_batch, + distance, + centroids_level1, + centroids_level1_norm, + assignments_level1_all, + matmul_results_level1, + threadpool, + timer + ); + } + auto all_assignments_cluster = timer.push_back("level1 all assignments clusters"); + auto clusters_level1_all = std::vector>(num_level1_clusters); + for (auto i : data.eachindex()) { + clusters_level1_all[assignments_level1_all[i]].push_back(i); + } + all_assignments_cluster.finish(); + + all_assignments_time.finish(); + level1_training_time.finish(); + + auto level2_training_time = timer.push_back("Level2 training"); + auto num_level2_clusters = calc_level2_clusters( + num_clusters, num_level1_clusters, clusters_level1, num_training_data, rng + ); + + auto centroids_final = data::SimpleData{num_clusters, ndims}; + auto clusters_final = std::vector>(num_clusters); + + size_t max_data_per_cluster = 0; + for (size_t cluster = 0; cluster < num_level1_clusters; cluster++) { + max_data_per_cluster = clusters_level1_all[cluster].size() > max_data_per_cluster + ? clusters_level1_all[cluster].size() + : max_data_per_cluster; + } + auto data_level2 = data::SimpleData{max_data_per_cluster, ndims}; + auto assignments_level2_all = std::vector(max_data_per_cluster); + + size_t cluster_start = 0; + for (size_t cluster = 0; cluster < num_level1_clusters; cluster++) { + size_t num_clusters_l2 = num_level2_clusters[cluster]; + size_t num_assignments_l2 = clusters_level1[cluster].size(); + size_t num_assignments_l2_all = clusters_level1_all[cluster].size(); + + auto centroids_level2 = data::SimpleData{num_clusters_l2, ndims}; + auto matmul_results_level2 = + data::SimpleData{parameters.minibatch_size_, num_clusters_l2}; + auto data_train_level2 = data::SimpleData{num_assignments_l2, ndims}; + + threads::parallel_for( + threadpool, + threads::StaticPartition{num_assignments_l2}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + data_train_level2.set_datum( + i, data_train.get_datum(clusters_level1[cluster][i]) + ); + } + } + ); + + v.resize(num_clusters_l2); + generate_unique_ids(v, data_train_level2.size(), rng); + threads::parallel_for( + threadpool, + threads::StaticPartition{num_clusters_l2}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + centroids_level2.set_datum(i, data_train_level2.get_datum(v[i])); + } + } + ); + + auto centroids_level2_fp32 = kmeans_training( + parameters, + data_train_level2, + distance, + centroids_level2, + matmul_results_level2, + rng, + threadpool, + timer + ); + + auto all_assignments_level2 = timer.push_back("level2 all assignments"); + threads::parallel_for( + threadpool, + threads::StaticPartition{num_assignments_l2_all}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + data_level2.set_datum( + i, data.get_datum(clusters_level1_all[cluster][i]) + ); + } + } + ); + + batchsize = parameters.minibatch_size_; + num_batches = lib::div_round_up(num_assignments_l2_all, batchsize); + + if constexpr (std::is_same_v) { + generate_norms(data_level2, data_norm, threadpool); + } + std::vector centroids_level2_norm; + if constexpr (std::is_same_v) { + generate_norms(centroids_level2_fp32, centroids_level2_norm, threadpool); + } + for (size_t batch = 0; batch < num_batches; ++batch) { + auto this_batch = threads::UnitRange{ + batch * batchsize, + std::min((batch + 1) * batchsize, num_assignments_l2_all)}; + auto data_batch = data::make_view(data_level2, this_batch); + centroid_assignment( + data_batch, + data_norm, + this_batch, + distance, + centroids_level2, + centroids_level2_norm, + assignments_level2_all, + matmul_results_level2, + threadpool, + timer + ); + } + + for (size_t i = 0; i < num_assignments_l2_all; i++) { + clusters_final[cluster_start + assignments_level2_all[i]].push_back( + clusters_level1_all[cluster][i] + ); + } + + threads::parallel_for( + threadpool, + threads::StaticPartition{num_clusters_l2}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + centroids_final.set_datum( + cluster_start + i, centroids_level2_fp32.get_datum(i) + ); + } + } + ); + + cluster_start += num_clusters_l2; + all_assignments_level2.finish(); + } + level2_training_time.finish(); + + kmeans_timer.finish(); + svs::logging::debug("{}", timer); + fmt::print( + "hierarchical kmeans clustering time: {}\n", lib::as_seconds(timer.elapsed()) + ); + + return std::make_tuple(std::move(centroids_final), std::move(clusters_final)); +} + +template < + typename BuildType, + data::ImmutableMemoryDataset Data, + typename Distance, + threads::ThreadPool Pool, + std::integral I = uint32_t> +auto hierarchical_kmeans_clustering( + const IVFBuildParameters& parameters, + Data& data, + Distance& distance, + Pool& threadpool, + lib::Type integer_type = {} +) { + return hierarchical_kmeans_clustering_impl( + parameters, data, distance, threadpool, integer_type + ); +} + +} // namespace svs::index::ivf diff --git a/include/svs/index/ivf/index.h b/include/svs/index/ivf/index.h new file mode 100644 index 00000000..58d81e49 --- /dev/null +++ b/include/svs/index/ivf/index.h @@ -0,0 +1,345 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svs +#include "svs/concepts/data.h" +#include "svs/core/loading.h" +#include "svs/core/query_result.h" +#include "svs/index/ivf/clustering.h" +#include "svs/index/ivf/extensions.h" +#include "svs/index/ivf/hierarchical_kmeans.h" +#include "svs/index/ivf/kmeans.h" +#include "svs/lib/timing.h" + +// format +#include "fmt/core.h" + +// stl +#include +#include + +namespace svs::index::ivf { + +const size_t MAX_QUERY_BATCH_SIZE = 10000; + +template +class IVFIndex { + public: + using Idx = typename Cluster::index_type; + using Data = typename Cluster::data_type; + using search_parameters_type = IVFSearchParameters; + + IVFIndex( + Centroids centroids, + Cluster cluster, + Dist distance_function, + ThreadPoolProto threadpool_proto, + size_t n_inner_threads = 1 + ) + : centroids_{std::move(centroids)} + , cluster_{std::move(cluster)} + , cluster0_{cluster_.view_cluster(0)} + , distance_{std::move(distance_function)} + , threadpool_{threads::as_threadpool(std::move(threadpool_proto))} + , n_inner_threads_{n_inner_threads} { + // Initialize threadpools for intra-query parallelism + for (size_t i = 0; i < threadpool_.size(); i++) { + threadpool_inner_.push_back(threads::as_threadpool(n_inner_threads_)); + } + + // The first level of search to find the n_probes nearest centroids is done + // using matmul (single thread) and thread batching on the number of centroids. + // Matmul results of each batch with appropriate sizes are initialized here, + // assuming that the static partition will keep the batching same. + auto batches = std::vector>(threadpool_.size()); + threads::parallel_for( + threadpool_, + threads::StaticPartition(centroids_.size()), + [&](auto is, auto tid) { batches[tid] = threads::UnitRange{is}; } + ); + for (size_t i = 0; i < threadpool_.size(); i++) { + matmul_results_.emplace_back(MAX_QUERY_BATCH_SIZE, batches[i].size()); + } + + // Precalculate centroid norms (required in L2 distances) + if constexpr (std::is_same_v) { + for (size_t i = 0; i < centroids_.size(); i++) { + centroids_norm_.push_back(distance::norm_square(centroids_.get_datum(i))); + } + } + } + + ///// Threading Interface + + static constexpr bool can_change_threads() { return false; } + /// + /// @brief Return the current number of threads used for search. + /// + size_t get_num_threads() const { return threadpool_.size(); } + + void set_threadpool(threads::ThreadPoolHandle threadpool) { + if (threadpool.size() != threadpool_.size()) { + throw ANNEXCEPTION("Threadpool change not supported for IVFIndex!"); + } + + threadpool_ = std::move(threadpool); + } + + /// + /// @brief Destroy the original thread pool and set to the provided one. + /// + /// @param threadpool An acceptable thread pool. + /// + /// @copydoc threadpool_requirements + /// + template + void set_threadpool(Pool threadpool) + requires(!std::is_same_v) + { + set_threadpool(threads::ThreadPoolHandle(std::move(threadpool))); + } + + /// + /// @brief Return the current thread pool handle. + /// + threads::ThreadPoolHandle& get_threadpool_handle() { return threadpool_; } + + + size_t size() const { return centroids_.size(); } + size_t dimensions() const { return centroids_.dimensions(); } + + ///// Search Parameter Setting + search_parameters_type get_search_parameters() const { return search_parameters_; } + + void set_search_parameters(const search_parameters_type& search_parameters) { + search_parameters_ = search_parameters; + } + + auto search_centroids_closure() { + return [&](const auto& query, auto& buffer, size_t id) { + search_centroids( + query, + distance_, + matmul_results_, + buffer, + id, + centroids_norm_, + get_num_threads() + ); + }; + } + + auto search_leaves_closure() { + return [&](const auto& query, + auto& distance, + const auto& buffer_centroids, + auto& buffer_leaves, + size_t tid) { + search_leaves( + query, + distance, + cluster_, + buffer_centroids, + buffer_leaves, + threadpool_inner_[tid] + ); + }; + } + + // Search + template + void search( + QueryResultView results, + const Queries& queries, + const search_parameters_type& search_parameters, + const lib::DefaultPredicate& SVS_UNUSED(cancel) = lib::Returns(lib::Const()) + ) { + if (queries.size() > MAX_QUERY_BATCH_SIZE) { + throw ANNEXCEPTION( + "Number of queries {} higher than expected {}, increase value of " + "MAX_QUERY_BATCH_SIZE", + queries.size(), + MAX_QUERY_BATCH_SIZE + ); + } + + size_t num_neighbors = results.n_neighbors(); + size_t buffer_leaves_size = search_parameters.k_reorder_ * num_neighbors; + compute_centroid_distances(queries, centroids_, matmul_results_, threadpool_); + + threads::parallel_for( + threadpool_, + threads::StaticPartition(queries.size()), + [&](auto is, auto tid) { + auto buffer_centroids = SortedBuffer>( + search_parameters.n_probes_, distance::comparator(distance_) + ); + std::vector>> buffer_leaves; + for (size_t j = 0; j < n_inner_threads_; j++) { + buffer_leaves.push_back(SortedBuffer>( + buffer_leaves_size, distance::comparator(distance_) + )); + } + + // Pre-allocate scratch space needed by the dataset implementation. + auto scratch = + extensions::per_thread_batch_search_setup(cluster0_, distance_); + + // Perform a search over the batch of queries. + extensions::per_thread_batch_search( + cluster0_, + cluster_, + buffer_centroids, + buffer_leaves, + scratch, + queries, + results, + threads::UnitRange{is}, + tid, + search_centroids_closure(), + search_leaves_closure() + ); + } + ); + } + + std::string name() const { return "IVFIndex"; } + + private: + Centroids centroids_; + Cluster cluster_; + Data cluster0_; + Dist distance_; + threads::ThreadPoolHandle threadpool_; + const size_t n_inner_threads_ = 1; + std::vector threadpool_inner_; + std::vector> matmul_results_; + std::vector centroids_norm_; + + // Tunable Parameters + search_parameters_type search_parameters_{}; +}; + +template < + typename BuildType, + typename DataProto, + typename Distance, + typename ThreadpoolProto> +auto build_clustering( + const IVFBuildParameters& parameters, + const DataProto& data_proto, + Distance distance, + ThreadpoolProto threadpool_proto +) { + auto threadpool = threads::as_threadpool(std::move(threadpool_proto)); + auto data = svs::detail::dispatch_load(data_proto, threadpool); + + auto tic = svs::lib::now(); + data::SimpleData centroids; + std::vector> clusters; + if (parameters.is_hierarchical_) { + std::tie(centroids, clusters) = hierarchical_kmeans_clustering( + parameters, data, distance, threadpool + ); + } else { + std::tie(centroids, clusters) = + kmeans_clustering(parameters, data, distance, threadpool); + } + + Clustering clustering(std::move(centroids), std::move(clusters)); + auto build_time = svs::lib::time_difference(tic); + fmt::print("IVF build time: {} seconds\n", build_time); + + auto logger = svs::logging::get(); + svs::logging::debug( + logger, "IVF Clustering Stats: {}", clustering.statistics().report("\n") + ); + + return clustering; +} + +template < + typename Clustering, + typename DataProto, + typename Distance, + typename ThreadpoolProto> +auto assemble_from_clustering( + Clustering clustering, + const DataProto& data_proto, + Distance distance, + ThreadpoolProto threadpool_proto, + const size_t n_inner_threads = 1 +) { + auto timer = lib::Timer(); + auto assemble_timer = timer.push_back("Total Assembling time"); + auto data_load_timer = timer.push_back("Data loading"); + auto threadpool = threads::as_threadpool(std::move(threadpool_proto)); + auto data = svs::detail::dispatch_load(data_proto, threadpool); + data_load_timer.finish(); + + auto dense_cluster_timer = timer.push_back("Dense clustering"); + using centroids_type = data::SimpleData; + using data_type = typename decltype(data)::lib_alloc_data_type; + + auto dense_clusters = DenseClusteredDataset( + clustering, data, threadpool, lib::Allocator() + ); + dense_cluster_timer.finish(); + + auto index_build_timer = timer.push_back("IVF index construction"); + auto ivf_index = IVFIndex( + std::move(clustering.centroids()), + std::move(dense_clusters), + std::move(distance), + std::move(threadpool), + n_inner_threads + ); + index_build_timer.finish(); + assemble_timer.finish(); + svs::logging::debug("{}", timer); + return ivf_index; +} + +template < + typename Centroids, + typename DataProto, + typename Distance, + typename ThreadpoolProto> +auto assemble_from_file( + const std::filesystem::path& clustering_path, + const DataProto& data_proto, + Distance distance, + ThreadpoolProto threadpool_proto, + const size_t n_inner_threads = 1 +) { + using centroids_type = data::SimpleData; + auto threadpool = threads::as_threadpool(std::move(threadpool_proto)); + auto clustering = svs::lib::load_from_disk>( + clustering_path, threadpool + ); + + return assemble_from_clustering( + std::move(clustering), + data_proto, + std::move(distance), + std::move(threadpool), + n_inner_threads + ); +} + +} // namespace svs::index::ivf diff --git a/include/svs/index/ivf/kmeans.h b/include/svs/index/ivf/kmeans.h new file mode 100644 index 00000000..17e8747b --- /dev/null +++ b/include/svs/index/ivf/kmeans.h @@ -0,0 +1,154 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/index/ivf/common.h" + +namespace svs::index::ivf { + +template < + typename BuildType, + data::ImmutableMemoryDataset Data, + typename Distance, + threads::ThreadPool Pool, + std::integral I = uint32_t> +auto kmeans_clustering_impl( + const IVFBuildParameters& parameters, + Data& data, + Distance& distance, + Pool& threadpool, + lib::Type SVS_UNUSED(integer_type) = {} +) { + auto timer = lib::Timer(); + auto kmeans_timer = timer.push_back("Non-hierarchical kmeans clustering"); + auto init_timer = timer.push_back("init"); + + constexpr size_t Dims = Data::extent; + using Alloc = svs::HugepageAllocator; + size_t ndims = data.dimensions(); + auto num_centroids = parameters.num_centroids_; + size_t num_training_data = + lib::narrow(std::ceil(data.size() * parameters.training_fraction_)); + if (num_training_data < num_centroids || num_training_data > data.size()) { + throw ANNEXCEPTION( + "Invalid number of training data: {}, num_centroids: {}, total data size: " + "{}\n", + num_training_data, + num_centroids, + data.size() + ); + } + + // The cluster centroids + auto centroids = data::SimpleData{num_centroids, ndims}; + auto data_train = data::SimpleData{num_training_data, ndims}; + auto matmul_results = + data::SimpleData{parameters.minibatch_size_, num_centroids}; + auto rng = std::mt19937(parameters.seed_); + + std::vector v(num_training_data); + generate_unique_ids(v, data.size(), rng); + threads::parallel_for( + threadpool, + threads::StaticPartition{num_training_data}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + data_train.set_datum(i, data.get_datum(v[i])); + } + } + ); + + v.resize(num_centroids); + generate_unique_ids(v, data_train.size(), rng); + threads::parallel_for( + threadpool, + threads::StaticPartition{num_centroids}, + [&](auto indices, auto /*tid*/) { + for (auto i : indices) { + centroids.set_datum(i, data_train.get_datum(v[i])); + } + } + ); + init_timer.finish(); + + auto centroids_fp32 = kmeans_training( + parameters, data_train, distance, centroids, matmul_results, rng, threadpool, timer + ); + + auto final_assignments_time = timer.push_back("final assignments"); + auto assignments = std::vector(data.size()); + auto batchsize = parameters.minibatch_size_; + auto num_batches = lib::div_round_up(data.size(), batchsize); + + std::vector data_norm; + if constexpr (std::is_same_v) { + generate_norms(data, data_norm, threadpool); + } + std::vector centroids_norm; + if constexpr (std::is_same_v) { + generate_norms(centroids_fp32, centroids_norm, threadpool); + } + + auto data_batch = data::SimpleData{batchsize, ndims}; + for (size_t batch = 0; batch < num_batches; ++batch) { + auto this_batch = threads::UnitRange{ + batch * batchsize, std::min((batch + 1) * batchsize, data.size())}; + auto data_batch_view = data::make_view(data, this_batch); + convert_data(data_batch_view, data_batch, threadpool); + centroid_assignment( + data_batch, + data_norm, + this_batch, + distance, + centroids, + centroids_norm, + assignments, + matmul_results, + threadpool, + timer + ); + } + + auto clusters = std::vector>(num_centroids); + for (auto i : data.eachindex()) { + clusters[assignments[i]].push_back(i); + } + final_assignments_time.finish(); + kmeans_timer.finish(); + svs::logging::debug("{}", timer); + fmt::print("kmeans clustering time: {}\n", lib::as_seconds(timer.elapsed())); + return std::make_tuple(centroids, std::move(clusters)); +} + +template < + typename BuildType, + data::ImmutableMemoryDataset Data, + typename Distance, + threads::ThreadPool Pool, + std::integral I = uint32_t> +auto kmeans_clustering( + const IVFBuildParameters& parameters, + Data& data, + Distance& distance, + Pool& threadpool, + lib::Type integer_type = {} +) { + return kmeans_clustering_impl( + parameters, data, distance, threadpool, integer_type + ); +} +} // namespace svs::index::ivf diff --git a/include/svs/index/ivf/sorted_buffer.h b/include/svs/index/ivf/sorted_buffer.h new file mode 100644 index 00000000..85939ef6 --- /dev/null +++ b/include/svs/index/ivf/sorted_buffer.h @@ -0,0 +1,202 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/index/vamana/filter.h" +#include "svs/lib/datatype.h" +#include "svs/lib/neighbor.h" +#include "svs/lib/prefetch.h" +#include "svs/lib/threads/threadlocal.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace svs::index::ivf { +/// +/// @brief Class used to store search results for static greedy search. +/// +/// @tparam Idx Type used to uniquely identify DB vectors +/// @tparam Cmp Type of the comparison function used to sort neighbors by distance. +/// +template > class SortedBuffer { + public: + // External type aliases + using value_type = IVFNeighbor; + using reference = value_type&; + using const_reference = const value_type&; + using compare_type = Cmp; + + using vector_type = std::vector>; + using iterator = typename vector_type::iterator; + using const_iterator = typename vector_type::const_iterator; + + /// A visited filter with 65,535 entries with a memory footpring of 128 kiB. + using set_type = index::vamana::VisitedFilter; + + /// + /// @brief Initialize a buffer with zero capacity. + /// + /// In order to use a buffer that has been default constructed, use the + /// @ref change_maxsize(size_t) method. + /// + SortedBuffer() = default; + + /// + /// @brief Construct a search buffer with the target capacity and comparison function. + /// + /// @param size The number of valid elements to return from a search operation. + /// @param compare The functor used to compare two ``Neighbor``s together. + /// + explicit SortedBuffer(size_t size, Cmp compare = {}) + : compare_{std::move(compare)} + , capacity_{size} + , candidates_{capacity_ + 1} {} + + /// + /// @brief Perform an efficient copy. + /// + /// Copy the portions of the SortedBuffer that matter for the purposes of scratch + /// space. + /// + /// Perserves the sizes of various containers but not necessarily the values. + /// + SortedBuffer shallow_copy() const { return SortedBuffer{capacity_, compare_}; } + + /// + /// @brief Change the target number of elements to return after search. + /// + /// @param new_size The new number of elements to return. + /// + /// Post conditions + /// - The capacity of the search buffer will be set to the new size. + /// - The actual size (number of valid elements) will be the minimum of the current + /// size and the new size. + /// + void change_maxsize(size_t new_size) { + capacity_ = new_size; + candidates_.resize(new_size + 1); + size_ = std::min(size_, new_size); + } + + /// + /// @brief Prepare the buffer for a new search operation. + /// + void clear() { size_ = 0; } + + /// @brief Return the current number of valid elements in the buffer. + size_t size() const { return size_; } + + /// @brief Return the maximum number of neighbors that can be held by the buffer. + size_t capacity() const { return capacity_; } + + /// @brief Return whether or not the buffer is full of valid elements. + bool full() const { return size() == capacity(); } + + /// @brief Access the neighbor at position `i`. + reference operator[](size_t i) { return candidates_[i]; } + + /// @brief Access the neighbor at position `i`. + const_reference operator[](size_t i) const { return candidates_[i]; } + + /// @brief Return the furtherst valid neighbor. + reference back() { return candidates_[size_ - 1]; } + + /// @brief Return the furtherst valid neighbor. + const_reference back() const { return candidates_[size_ - 1]; } + + // Define iterators. + constexpr const_iterator begin() const noexcept { return candidates_.begin(); } + constexpr const_iterator end() const noexcept { return begin() + size(); } + constexpr iterator begin() noexcept { return candidates_.begin(); } + constexpr iterator end() noexcept { return begin() + size(); } + + void unsafe_insert(value_type neighbor, iterator index) { + std::copy_backward(index, end(), end() + 1); + (*index) = neighbor; + } + + /// + /// @brief Return ``true`` if a neighbor with the given distance can be skipped. + /// + bool can_skip(float distance) const { + return compare_(back().distance(), distance) && full(); + } + + /// + /// @brief Insert the neighbor into the buffer. + /// + /// @param neighbor The neighbor to insert. + /// + /// @returns The position where the neighbor was inserted. + /// + size_t insert(value_type neighbor) { + if (can_skip(neighbor.distance())) { + return size(); + } + return insert_inner(neighbor); + } + + size_t insert_inner(value_type neighbor) { + const auto start = begin(); + // Binary search to the first location where `distance` is less than the stored + // neighbor. + auto pos = std::lower_bound( + start, + end(), + neighbor.distance(), + [&](const value_type& other, const float& d) { + return !compare_(d, other.distance()); + } + ); + + size_t i = pos - start; + unsafe_insert(neighbor, pos); + size_ = std::min(size_ + 1, capacity()); + return i; + } + + /// + /// @brief Sort the elements in the buffer according to the internal comparison functor. + /// + void sort() { std::sort(begin(), end(), compare_); } + + /// + /// @brief Return ``true`` if the visited set is enabled. Otherwise, return ``false``. + /// + bool visited_set_enabled() const { return visited_.has_value(); } + + private: + // The comparison functor. + [[no_unique_address]] Cmp compare_ = Cmp{}; + // The current number of valid neighbors. + size_t size_ = 0; + // The maximum capacity of the buffer. + size_t capacity_ = 0; + // Storage for the neighbors. + vector_type candidates_ = {}; + // The visited set. Implemented as a `std::optional` to allow enablind and disabling + // without always requiring allocation of the data structure. + std::optional visited_{std::nullopt}; +}; + +} // namespace svs::index::ivf diff --git a/include/svs/lib/bfloat16.h b/include/svs/lib/bfloat16.h new file mode 100644 index 00000000..e162cd60 --- /dev/null +++ b/include/svs/lib/bfloat16.h @@ -0,0 +1,143 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/lib/narrow.h" +#include "svs/lib/type_traits.h" +#include "svs/third-party/fmt.h" + +#include +#include +#include +#include +#include +#if defined(__AVX512_BF16__) +#include +#endif + +namespace svs { +namespace bfloat16 { +namespace detail { + +// TODO: Update to `bitcast` if its available in the standard library. +inline uint32_t bitcast_float_to_uint32(const float x) { + static_assert(sizeof(float) == sizeof(uint32_t)); + uint32_t u; + memcpy(&u, &x, sizeof(x)); + return u; +} + +inline float bitcast_uint32_to_float(const uint32_t x) { + static_assert(sizeof(float) == sizeof(uint32_t)); + float f; + memcpy(&f, &x, sizeof(x)); + return f; +} + +inline float bfloat16_to_float_untyped(const uint16_t x) { + const uint32_t u = x << 16; + return bitcast_uint32_to_float(u); +} + +inline uint16_t float_to_bfloat16_untyped(const float x) { + const uint32_t u = bitcast_float_to_uint32(x); + return lib::narrow(u >> 16); +} + +} // namespace detail + +// On GCC - we need to add this attribute so that BFloat16 members can appear inside +// packed structs. +class __attribute__((packed)) BFloat16 { + public: + BFloat16() = default; + + // converting constructors + explicit BFloat16(float x) + : value_{detail::float_to_bfloat16_untyped(x)} {} + explicit BFloat16(double x) + : BFloat16(lib::narrow_cast(x)) {} + explicit BFloat16(size_t x) + : BFloat16(lib::narrow(x)) {} + explicit BFloat16(int x) + : BFloat16(lib::narrow(x)) {} + + // conversion functions + operator float() const { return detail::bfloat16_to_float_untyped(value_); } + BFloat16& operator=(float x) { + value_ = detail::float_to_bfloat16_untyped(x); + return *this; + } + + // Allow users to set and expect the contents of the class as a uint16_t using an + // explicit API. + static BFloat16 from_raw(uint16_t value) { return BFloat16{value, FromRawTag{}}; } + uint16_t raw() const { return value_; } + + private: + // Use a tag to construct from a raw value in order to still allow a constructor + // for a lone `uint16_t`. + struct FromRawTag {}; + explicit BFloat16(uint16_t value, FromRawTag /*unused*/) + : value_{value} {} + uint16_t value_; +}; +static_assert(std::is_trivial_v); +static_assert(std::is_standard_layout_v); + +///// +///// Operators +///// + +// For equality, still use `float` rather than the underlying bit pattern to handle cases +// like signed zeros. +inline bool operator==(BFloat16 x, BFloat16 y) { return float{x} == float{y}; } + +///// +///// Pretty Printing +///// + +} // namespace bfloat16 + +using BFloat16 = bfloat16::BFloat16; + +// SVS local arithmetric trait. +template <> inline constexpr bool is_arithmetic_v = true; +template <> inline constexpr bool is_signed_v = true; +template <> inline constexpr bool allow_lossy_conversion = true; + +} // namespace svs + +// Apply hashing to `BFloat16` +namespace std { +template <> struct hash { + inline std::size_t operator()(const svs::BFloat16& x) const noexcept { + return std::hash()(x); + } +}; +} // namespace std + +// Formatting and Printing +template <> struct fmt::formatter : svs::format_empty { + auto format(svs::BFloat16 x, auto& ctx) const { + return fmt::format_to(ctx.out(), "{}f16", float{x}); + } +}; + +inline std::ostream& operator<<(std::ostream& stream, svs::BFloat16 x) { + return stream << fmt::format("{}", x); +} diff --git a/include/svs/lib/datatype.h b/include/svs/lib/datatype.h index 68ddf958..9b5b0aa2 100644 --- a/include/svs/lib/datatype.h +++ b/include/svs/lib/datatype.h @@ -22,6 +22,7 @@ /// // local deps +#include "svs/lib/bfloat16.h" #include "svs/lib/exception.h" #include "svs/lib/float16.h" #include "svs/third-party/fmt.h" @@ -50,6 +51,7 @@ enum class DataType { int32, int64, float16, + bfloat16, float32, float64, byte, @@ -74,6 +76,9 @@ template <> inline constexpr std::string_view name() { return " template <> inline constexpr std::string_view name() { return "float16"; } +template <> inline constexpr std::string_view name() { + return "bfloat16"; +} template <> inline constexpr std::string_view name() { return "float32"; } @@ -101,6 +106,7 @@ inline constexpr std::string_view name(DataType type) { case DataType::int64: { return name(); } case DataType::float16: { return name(); } + case DataType::bfloat16: { return name(); } case DataType::float32: { return name(); } case DataType::float64: { return name(); } @@ -126,6 +132,7 @@ inline constexpr size_t element_size(DataType type) { case DataType::int64: { return sizeof(int64_t); } case DataType::float16: { return sizeof(svs::Float16); } + case DataType::bfloat16: { return sizeof(svs::BFloat16); } case DataType::float32: { return sizeof(float); } case DataType::float64: { return sizeof(double); } @@ -140,6 +147,9 @@ inline constexpr DataType parse_datatype_floating(std::string_view name) { if (name == "float16") { return DataType::float16; } + if (name == "bfloat16") { + return DataType::bfloat16; + } if (name == "float32") { return DataType::float32; } @@ -191,7 +201,7 @@ inline constexpr DataType parse_datatype(std::string_view name) { } // Floating point. - if (name.starts_with("float")) { + if (name.starts_with("float") || name.starts_with("bfloat")) { return parse_datatype_floating(name); } if (name.starts_with("uint")) { @@ -263,6 +273,7 @@ template <> struct CppType { using type = int32_t; }; template <> struct CppType { using type = int64_t; }; template <> struct CppType { using type = Float16; }; +template <> struct CppType { using type = BFloat16; }; template <> struct CppType { using type = float; }; template <> struct CppType { using type = double; }; @@ -289,6 +300,7 @@ template<> inline constexpr DataType datatype_v = DataType::int32; template<> inline constexpr DataType datatype_v = DataType::int64; template<> inline constexpr DataType datatype_v = DataType::float16; +template<> inline constexpr DataType datatype_v = DataType::bfloat16; template<> inline constexpr DataType datatype_v = DataType::float32; template<> inline constexpr DataType datatype_v = DataType::float64; diff --git a/include/svs/lib/file_iterator.h b/include/svs/lib/file_iterator.h index 0ef8b79e..63f7bcd3 100644 --- a/include/svs/lib/file_iterator.h +++ b/include/svs/lib/file_iterator.h @@ -54,9 +54,10 @@ template io_convert_type_t io_convert(U u) { return narrow>(u); } -// We expect the conversion to `Float16` to be lossy. +// We expect the conversion to `Float16` and `BFloat16` to be lossy. // Hence, don't require precise narrowing. template <> inline Float16 io_convert(float u) { return Float16(u); } +template <> inline BFloat16 io_convert(float u) { return BFloat16(u); } ///// ///// Iterator Helpers diff --git a/include/svs/lib/narrow.h b/include/svs/lib/narrow.h index d7d11f1a..6c07f667 100644 --- a/include/svs/lib/narrow.h +++ b/include/svs/lib/narrow.h @@ -3,7 +3,7 @@ * * The code in this file is a modified version of code from Microsoft Corporation, * published under an MIT License. This modified version is licensed under the - * GNU Affero General Public License version 3 + * Apache License, Version 2.0 * * ORIGINAL LICENSE * ------------------------------------------------------------------------------ diff --git a/include/svs/lib/neighbor.h b/include/svs/lib/neighbor.h index 4f790dd7..bd3b7290 100644 --- a/include/svs/lib/neighbor.h +++ b/include/svs/lib/neighbor.h @@ -62,6 +62,7 @@ template struct Neighbor : public Meta { constexpr float distance() const { return distance_; } constexpr void set_distance(float new_distance) { distance_ = new_distance; } constexpr Idx id() const { return id_; } + constexpr void set_id(Idx new_id) { id_ = new_id; } // Members Idx id_; @@ -82,7 +83,9 @@ template struct Neighbor { : Neighbor(lib::narrow(other.id()), other.distance()) {} constexpr float distance() const { return distance_; } + constexpr void set_distance(float new_distance) { distance_ = new_distance; } constexpr Idx id() const { return id_; } + constexpr void set_id(Idx new_id) { id_ = new_id; } // Members Idx id_; @@ -246,4 +249,24 @@ class ValidVisit { /// Type alias for skippable neighbor. template using PredicatedSearchNeighbor = Neighbor; +///// +///// IVF Neighbor +///// + +template struct LocalId { + constexpr LocalId(Idx local_id = 0) + : local_id_{local_id} {} + constexpr Idx get_local_id() const { return local_id_; } + constexpr void set_local_id(Idx local_id) { local_id_ = local_id; } + friend constexpr bool operator==(LocalId, LocalId) = default; + + // members + Idx local_id_; +}; + +/// +/// Type alias for the IVF neighbor +/// +template using IVFNeighbor = Neighbor>; + } // namespace svs diff --git a/include/svs/orchestrators/ivf.h b/include/svs/orchestrators/ivf.h new file mode 100644 index 00000000..b77b8aa5 --- /dev/null +++ b/include/svs/orchestrators/ivf.h @@ -0,0 +1,180 @@ +/* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/index/ivf/index.h" +#include "svs/orchestrators/manager.h" + +namespace svs { + +class IVFInterface { + public: + using search_parameters_type = svs::index::ivf::IVFSearchParameters; + + ///// Beckend Information Inteface + virtual std::string experimental_backend_string() const = 0; +}; + +template +class IVFImpl : public manager::ManagerImpl { + private: + // Null-terminated array of characters. + static constexpr auto typename_impl = lib::generate_typename(); + + public: + using base_type = manager::ManagerImpl; + using base_type::impl; + using search_parameters_type = typename IFace::search_parameters_type; + + explicit IVFImpl(Impl impl) + : base_type{std::move(impl)} {} + + ///// Parameter Interface + [[nodiscard]] search_parameters_type get_search_parameters() const override { + return impl().get_search_parameters(); + } + + void set_search_parameters(const search_parameters_type& search_parameters) override { + impl().set_search_parameters(search_parameters); + } + + ///// Backend Information Interface + [[nodiscard]] std::string experimental_backend_string() const override { + return std::string{typename_impl.begin(), typename_impl.end() - 1}; + } +}; + +///// +///// IVFManager +///// + +class IVF : public manager::IndexManager { + // Type Alises + public: + using base_type = manager::IndexManager; + using search_parameters_type = typename IVFInterface::search_parameters_type; + + // Constructors + IVF(std::unique_ptr> impl) + : base_type{std::move(impl)} {} + + template + IVF(std::in_place_t, QueryTypes SVS_UNUSED(type), Impl&& impl) + : base_type{std::make_unique>(SVS_FWD(impl))} {} + + ///// Backend String + std::string experimental_backend_string() const { + return impl_->experimental_backend_string(); + } + + ///// Assembling + template < + manager::QueryTypeDefinition QueryTypes, + typename Clustering, + typename DataProto, + typename Distance, + typename ThreadpoolProto> + static IVF assemble_from_clustering( + Clustering clustering, + const DataProto& data_proto, + const Distance& distance, + ThreadpoolProto threadpool_proto, + size_t intra_query_threads = 1 + ) { + auto threadpool = threads::as_threadpool(std::move(threadpool_proto)); + if constexpr (std::is_same_v, DistanceType>) { + auto dispatcher = DistanceDispatcher(distance); + return dispatcher([&](auto distance_function) { + return IVF( + std::in_place, + manager::as_typelist{}, + index::ivf::assemble_from_clustering( + std::move(clustering), + data_proto, + std::move(distance_function), + std::move(threadpool), + intra_query_threads + ) + ); + }); + } else { + return IVF( + std::in_place, + manager::as_typelist{}, + index::ivf::assemble_from_clustering( + std::move(clustering), + data_proto, + distance, + std::move(threadpool), + intra_query_threads + ) + ); + } + } + + template < + manager::QueryTypeDefinition QueryTypes, + typename Centroids, + typename DataProto, + typename Distance, + typename ThreadpoolProto> + static IVF assemble_from_file( + const std::filesystem::path& clustering_path, + const DataProto& data_proto, + const Distance& distance, + ThreadpoolProto threadpool_proto, + size_t intra_query_threads = 1 + ) { + using centroids_type = data::SimpleData; + auto threadpool = threads::as_threadpool(std::move(threadpool_proto)); + auto clustering = + svs::lib::load_from_disk>( + clustering_path, threadpool + ); + return assemble_from_clustering( + std::move(clustering), + data_proto, + distance, + std::move(threadpool), + intra_query_threads + ); + } + + ///// Building + template + static auto build_clustering( + const index::ivf::IVFBuildParameters& build_parameters, + const DataProto& data_proto, + const Distance& distance, + size_t num_threads + ) { + if constexpr (std::is_same_v, DistanceType>) { + auto dispatcher = DistanceDispatcher(distance); + return dispatcher([&](auto distance_function) { + return index::ivf::build_clustering( + build_parameters, data_proto, std::move(distance_function), num_threads + ); + }); + } else { + return index::ivf::build_clustering( + build_parameters, data_proto, distance, num_threads + ); + } + } +}; + +} // namespace svs diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8bda49ac..9058a7e0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -55,6 +55,7 @@ set(TEST_SOURCES ${TEST_DIR}/utils/schemas.cpp ${TEST_DIR}/utils/test_dataset.cpp ${TEST_DIR}/utils/vamana_reference.cpp + ${TEST_DIR}/utils/ivf_reference.cpp # Lib ${TEST_DIR}/svs/lib/algorithms.cpp ${TEST_DIR}/svs/lib/array.cpp @@ -65,6 +66,7 @@ set(TEST_SOURCES ${TEST_DIR}/svs/lib/invoke.cpp ${TEST_DIR}/svs/lib/file.cpp ${TEST_DIR}/svs/lib/file_iterator.cpp + ${TEST_DIR}/svs/lib/bfloat16.cpp ${TEST_DIR}/svs/lib/float16.cpp ${TEST_DIR}/svs/lib/memory.cpp ${TEST_DIR}/svs/lib/meta.cpp @@ -169,6 +171,14 @@ SET(INTEGRATION_TESTS # ${TEST_DIR}/integration/numa_search.cpp -- requires SVS_EXPERIMENTAL_ENABLE_NUMA ) +if (SVS_EXPERIMENTAL_BUILD_IVF) + message("Enabling IVF Integration tests!") + list(APPEND INTEGRATION_TESTS + ${TEST_DIR}/integration/ivf/index_build.cpp + ${TEST_DIR}/integration/ivf/index_search.cpp + ) +endif() + if ((NOT CMAKE_BUILD_TYPE STREQUAL "Debug") OR SVS_FORCE_INTEGRATION_TESTS) message("Enabling Integration Tests!") list(APPEND TEST_SOURCES ${INTEGRATION_TESTS}) @@ -188,6 +198,15 @@ if (SVS_EXPERIMENTAL_ENABLE_NUMA) list(APPEND TEST_SOURCES ${NUMA_TESTS}) endif() +if (SVS_EXPERIMENTAL_BUILD_IVF) + message("Enabling IVF tests!") + list(APPEND TEST_SOURCES + ${TEST_DIR}/svs/index/ivf/kmeans.cpp + ${TEST_DIR}/svs/index/ivf/hierarchical_kmeans.cpp + ${TEST_DIR}/svs/index/ivf/common.cpp + ) +endif() + add_executable(tests ${TEST_SOURCES}) # Path to the test dataset. diff --git a/tests/integration/ivf/index_build.cpp b/tests/integration/ivf/index_build.cpp new file mode 100644 index 00000000..a59acf96 --- /dev/null +++ b/tests/integration/ivf/index_build.cpp @@ -0,0 +1,119 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// svs +#include "svs/core/data/simple.h" +#include "svs/core/recall.h" +#include "svs/lib/timing.h" +#include "svs/orchestrators/ivf.h" + +// svsbenchmark +#include "svs-benchmark/benchmark.h" + +// fmt +#include "fmt/core.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +// tests +#include "tests/utils/test_dataset.h" +#include "tests/utils/utils.h" +#include "tests/utils/ivf_reference.h" + +// stl +#include +#include +#include +#include + +namespace { + +template +auto build_index( + const svs::index::ivf::IVFBuildParameters parameters, + const std::filesystem::path& data_path, + size_t num_threads, + size_t num_inner_threads, + const Distance& dist_type +) { + auto data = svs::data::SimpleData::load(data_path); + auto clustering = svs::IVF::build_clustering( + parameters, data, dist_type, num_threads + ); + + return svs::IVF::assemble_from_clustering( + std::move(clustering), + std::move(data), + dist_type, + num_threads, + num_inner_threads + ); +} + +template +void test_build(const Distance& distance, size_t num_inner_threads = 1) { + const double epsilon = 0.005; + const auto queries = svs::data::SimpleData::load(test_dataset::query_file()); + CATCH_REQUIRE(svs_test::prepare_temp_directory()); + size_t num_threads = 2; + + auto expected_result = test_dataset::ivf::expected_build_results( + svs::distance_type_v, svsbenchmark::Uncompressed(svs::datatype_v) + ); + auto index = build_index( + expected_result.build_parameters_.value(), + test_dataset::data_svs_file(), + num_threads, + num_inner_threads, + distance + ); + + auto groundtruth = test_dataset::load_groundtruth(svs::distance_type_v); + for (const auto& expected : expected_result.config_and_recall_) { + auto these_queries = test_dataset::get_test_set(queries, expected.num_queries_); + auto these_groundtruth = + test_dataset::get_test_set(groundtruth, expected.num_queries_); + index.set_search_parameters(expected.search_parameters_); + auto results = index.search(these_queries, expected.num_neighbors_); + double recall = svs::k_recall_at_n( + these_groundtruth, results, expected.num_neighbors_, expected.recall_k_ + ); + + fmt::print( + "n_probes: {}, Expected Recall: {}, Actual Recall: {}\n", + index.get_search_parameters().n_probes_, + expected.recall_, + recall + ); + CATCH_REQUIRE(recall > expected.recall_ - epsilon); + CATCH_REQUIRE(recall < expected.recall_ + epsilon); + } +} + +} // namespace + +CATCH_TEST_CASE("IVF Build/Clustering", "[integration][build][ivf]") { + test_build(svs::DistanceL2()); + test_build(svs::DistanceIP()); + + test_build(svs::DistanceL2()); + test_build(svs::DistanceIP()); + + // With 4 inner threads + test_build(svs::DistanceL2(), 4); + test_build(svs::DistanceIP(), 4); +} diff --git a/tests/integration/ivf/index_search.cpp b/tests/integration/ivf/index_search.cpp new file mode 100644 index 00000000..48b890b6 --- /dev/null +++ b/tests/integration/ivf/index_search.cpp @@ -0,0 +1,147 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// stl +#include +#include +#include +#include +#include +#include + +// svs +#include "svs/core/recall.h" +#include "svs/lib/saveload.h" +#include "svs/orchestrators/ivf.h" + +// svsbenchmark +#include "svs-benchmark/benchmark.h" +#include "svs-benchmark/test.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" +#include "catch2/matchers/catch_matchers_string.hpp" + +// fmt +#include "fmt/core.h" + +// tests +#include "tests/utils/test_dataset.h" +#include "tests/utils/utils.h" +#include "tests/utils/ivf_reference.h" + +namespace { + +void run_search( + svs::IVF& index, + const svs::data::SimpleData& queries_all, + const svs::data::SimpleData& groundtruth_all, + const std::vector& expected_results +) { + // If we make a change that somehow improves accuracy, we'll want to know. + // Use `epsilon` to add to the expected results to set an upper bound on the + // achieved accuracy. + double epsilon = 0.005; + + // Ensure we have at least one entry in the expected results. + CATCH_REQUIRE(!expected_results.empty()); + + const auto queries_in_test_set = expected_results.at(0).num_queries_; + + auto queries = test_dataset::get_test_set(queries_all, queries_in_test_set); + auto groundtruth = test_dataset::get_test_set(groundtruth_all, queries_in_test_set); + + for (const auto& expected : expected_results) { + // Update the query set if needed. + auto num_queries = expected.num_queries_; + if (num_queries != queries.size()) { + queries = test_dataset::get_test_set(queries_all, num_queries); + groundtruth = test_dataset::get_test_set(groundtruth_all, num_queries); + } + + // Configure the index with the current parameters. + // Ensure that the result sticks. + index.set_search_parameters(expected.search_parameters_); + CATCH_REQUIRE(index.get_search_parameters() == expected.search_parameters_); + + // Float32 + auto results = index.search(queries, expected.num_neighbors_); + auto recall = svs::k_recall_at_n( + groundtruth, results, expected.num_neighbors_, expected.recall_k_ + ); + fmt::print( + "n_probes: {}, Expected Recall: {}, Actual Recall: {}\n", + index.get_search_parameters().n_probes_, + expected.recall_, + recall + ); + + CATCH_REQUIRE(recall > expected.recall_ - epsilon); + CATCH_REQUIRE(recall < expected.recall_ + epsilon); + } +} + +template +void test_search( + svs::data::SimpleData data, + const Distance& distance, + const svs::data::SimpleData& queries, + const svs::data::SimpleData& groundtruth, + const size_t num_inner_threads = 1 +) { + size_t num_threads = 2; + + // Find the expected results for this dataset. + auto expected_result = test_dataset::ivf::expected_search_results( + svs::distance_type_v, svsbenchmark::Uncompressed(svs::datatype_v) + ); + + auto index = svs::IVF::assemble_from_file( + test_dataset::clustering_directory(), + data, + distance, + num_threads, + num_inner_threads + ); + CATCH_REQUIRE(index.get_num_threads() == num_threads); + + run_search( + index, queries, groundtruth, expected_result.config_and_recall_ + ); + CATCH_REQUIRE(index.dimensions() == test_dataset::NUM_DIMENSIONS); +} + +} // namespace + +CATCH_TEST_CASE("IVF Search", "[integration][search][ivf]") { + namespace ivf = svs::index::ivf; + + auto datafile = test_dataset::data_svs_file(); + auto queries = test_dataset::queries(); + auto gt_l2 = test_dataset::groundtruth_euclidean(); + auto gt_ip = test_dataset::groundtruth_mip(); + + auto dist_l2 = svs::distance::DistanceL2(); + auto dist_ip = svs::distance::DistanceIP(); + + auto data = svs::data::SimpleData::load(datafile); + auto data_f16 = svs::index::ivf::convert_data(data); + test_search(data, dist_l2, queries, gt_l2); + test_search(data, dist_l2, queries, gt_l2, 2); + + test_search(data_f16, dist_ip, queries, gt_ip); + test_search(data_f16, dist_ip, queries, gt_ip, 2); +} diff --git a/tests/svs/core/data/data.h b/tests/svs/core/data/data.h index 4ebe2a07..72cd9821 100644 --- a/tests/svs/core/data/data.h +++ b/tests/svs/core/data/data.h @@ -25,6 +25,7 @@ namespace svs_test::data { // A mock dataset with integer entries. class MockDataset { public: + using element_type = int64_t; using value_type = int64_t; using const_value_type = int64_t; diff --git a/tests/svs/index/ivf/common.cpp b/tests/svs/index/ivf/common.cpp new file mode 100644 index 00000000..39df8503 --- /dev/null +++ b/tests/svs/index/ivf/common.cpp @@ -0,0 +1,71 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// header under test +#include "svs/index/ivf/common.h" + +// tests +#include "tests/utils/test_dataset.h" +#include "tests/utils/utils.h" + +// catch +#include "catch2/catch_test_macros.hpp" + +CATCH_TEST_CASE("Kmeans Clustering", "[ivf][parameters]") { + namespace ivf = svs::index::ivf; + CATCH_SECTION("IVF Build Parameters") { + auto p = ivf::IVFBuildParameters(); + + // Test the setter methods. +#define XX(name, v) \ + CATCH_REQUIRE(p.name##_ != v); \ + CATCH_REQUIRE(p.name(v).name##_ == v); \ + CATCH_REQUIRE(p.name##_ == v); + + XX(num_centroids, 10); + XX(minibatch_size, 100); + XX(num_iterations, 1000); + XX(is_hierarchical, false); + XX(training_fraction, 0.05F); + XX(hierarchical_level1_clusters, 10); + XX(seed, 0x1234); +#undef XX + + // Saving and loading + svs_test::prepare_temp_directory(); + auto dir = svs_test::temp_directory(); + CATCH_REQUIRE(svs::lib::test_self_save_load(p, dir)); + } + + CATCH_SECTION("IVF Search Parameters") { + auto p = ivf::IVFSearchParameters(); + + // Test the setter methods. +#define XX(name, v) \ + CATCH_REQUIRE(p.name##_ != v); \ + CATCH_REQUIRE(p.name(v).name##_ == v); \ + CATCH_REQUIRE(p.name##_ == v); + + XX(n_probes, 10); + XX(k_reorder, 100); +#undef XX + + // Saving and loading + svs_test::prepare_temp_directory(); + auto dir = svs_test::temp_directory(); + CATCH_REQUIRE(svs::lib::test_self_save_load(p, dir)); + } +} diff --git a/tests/svs/index/ivf/hierarchical_kmeans.cpp b/tests/svs/index/ivf/hierarchical_kmeans.cpp new file mode 100644 index 00000000..23cbc53c --- /dev/null +++ b/tests/svs/index/ivf/hierarchical_kmeans.cpp @@ -0,0 +1,75 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// header under test +#include "svs/index/ivf/hierarchical_kmeans.h" + +// tests +#include "tests/utils/test_dataset.h" +#include "tests/utils/utils.h" + +// catch +#include "catch2/catch_test_macros.hpp" + +// stl +#include + +namespace { + +template +void test_hierarchical_kmeans_clustering(const Data& data, Distance distance) { + namespace ivf = svs::index::ivf; + + for (size_t n_centroids : {155}) { + for (size_t minibatch : {25}) { + for (size_t iters : {3}) { + for (float training_fraction : {0.55}) { + for (size_t l1_clusters : {0, 9}) { + auto params = ivf::IVFBuildParameters() + .num_centroids(n_centroids) + .minibatch_size(minibatch) + .num_iterations(iters) + .is_hierarchical(true) + .training_fraction(training_fraction) + .hierarchical_level1_clusters(l1_clusters); + + auto threadpool = svs::threads::as_threadpool(10); + auto [centroids, clusters] = + hierarchical_kmeans_clustering( + params, data, distance, threadpool + ); + + CATCH_REQUIRE(centroids.size() == n_centroids); + CATCH_REQUIRE(centroids.dimensions() == data.dimensions()); + CATCH_REQUIRE(clusters.size() == n_centroids); + } + } + } + } + } +} + +} // namespace + +CATCH_TEST_CASE("Hierarchical Kmeans Param Check", "[ivf][hierarchial_parameter_check]") { + CATCH_SECTION("Uncompressed Data") { + auto data = svs::data::SimpleData::load(test_dataset::data_svs_file()); + test_hierarchical_kmeans_clustering(data, svs::DistanceIP()); + test_hierarchical_kmeans_clustering(data, svs::DistanceIP()); + test_hierarchical_kmeans_clustering(data, svs::DistanceIP()); + test_hierarchical_kmeans_clustering(data, svs::DistanceL2()); + } +} diff --git a/tests/svs/index/ivf/kmeans.cpp b/tests/svs/index/ivf/kmeans.cpp new file mode 100644 index 00000000..69f71df1 --- /dev/null +++ b/tests/svs/index/ivf/kmeans.cpp @@ -0,0 +1,69 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// header under test +#include "svs/index/ivf/kmeans.h" + +// tests +#include "tests/utils/test_dataset.h" +#include "tests/utils/utils.h" + +// catch +#include "catch2/catch_test_macros.hpp" + +// stl +#include + +namespace { + +template +void test_kmeans_clustering(const Data& data, Distance distance) { + namespace ivf = svs::index::ivf; + + for (size_t n_centroids : {1, 99}) { + for (size_t minibatch : {25}) { + for (size_t iters : {3}) { + for (float training_fraction : {0.55}) { + auto params = ivf::IVFBuildParameters() + .num_centroids(n_centroids) + .minibatch_size(minibatch) + .num_iterations(iters) + .is_hierarchical(false) + .training_fraction(training_fraction); + auto threadpool = svs::threads::as_threadpool(10); + auto [centroids, clusters] = + ivf::kmeans_clustering(params, data, distance, threadpool); + + CATCH_REQUIRE(centroids.size() == n_centroids); + CATCH_REQUIRE(centroids.dimensions() == data.dimensions()); + CATCH_REQUIRE(clusters.size() == n_centroids); + } + } + } + } +} + +} // namespace + +CATCH_TEST_CASE("Build Kmeans Param Check", "[ivf][parameter_check]") { + CATCH_SECTION("Uncompressed Data") { + auto data = svs::data::SimpleData::load(test_dataset::data_svs_file()); + test_kmeans_clustering(data, svs::DistanceIP()); + test_kmeans_clustering(data, svs::DistanceIP()); + test_kmeans_clustering(data, svs::DistanceIP()); + test_kmeans_clustering(data, svs::DistanceL2()); + } +} diff --git a/tests/svs/lib/bfloat16.cpp b/tests/svs/lib/bfloat16.cpp new file mode 100644 index 00000000..ae011b77 --- /dev/null +++ b/tests/svs/lib/bfloat16.cpp @@ -0,0 +1,79 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "svs/lib/bfloat16.h" +#include "svs/lib/narrow.h" + +#include "catch2/catch_test_macros.hpp" + +CATCH_TEST_CASE("Testing BFloat16", "[core][bfloat16]") { + CATCH_SECTION("Implicit Conversion") { + svs::BFloat16 x{1.0f}; + float y = x; + CATCH_REQUIRE(y == 1.0f); + + x = svs::BFloat16{-1}; + CATCH_REQUIRE(float{x} == -1.0f); + + // Construct from `size_t` + x = svs::BFloat16{size_t{100}}; + CATCH_REQUIRE(float{x} == 100.0f); + + // Default Construction. + CATCH_REQUIRE(svs::BFloat16{} == svs::BFloat16(float{0})); + } + + CATCH_SECTION("Arithmetic") { + CATCH_REQUIRE(svs::is_arithmetic_v); + CATCH_REQUIRE(svs::is_signed_v); + + auto x = svs::BFloat16{1}; + auto y = svs::BFloat16{2}; + CATCH_REQUIRE(x + y == 3); + CATCH_REQUIRE(x != y); + CATCH_REQUIRE(x < y); + CATCH_REQUIRE(!(y < x)); + CATCH_REQUIRE(y - x == svs::BFloat16(1)); + } + + CATCH_SECTION("Narrow") { + float x_good{1.0f}; + auto y_good = svs::lib::narrow(x_good); + CATCH_REQUIRE(float{y_good} == x_good); + + // Use low precision outside the bounds of `BFloat16`. + float x_bad{0.000012f}; + CATCH_REQUIRE_THROWS_AS( + svs::lib::narrow(x_bad), svs::lib::narrowing_error + ); + + // Fail when constructing from typemax integers + CATCH_REQUIRE_THROWS_AS( + svs::BFloat16(std::numeric_limits::max() - 1), svs::lib::narrowing_error + ); + + CATCH_REQUIRE_THROWS_AS( + svs::BFloat16(std::numeric_limits::max() - int{1}), + svs::lib::narrowing_error + ); + CATCH_REQUIRE_THROWS_AS( + svs::BFloat16(std::numeric_limits::min() + int{1}), + svs::lib::narrowing_error + ); + } +} diff --git a/tests/svs/lib/datatype.cpp b/tests/svs/lib/datatype.cpp index a0e7ca22..a3573531 100644 --- a/tests/svs/lib/datatype.cpp +++ b/tests/svs/lib/datatype.cpp @@ -65,6 +65,7 @@ CATCH_TEST_CASE("Data Type", "[core][datatype]") { CATCH_REQUIRE(svs::parse_datatype("int128") == DataType::undef); test("float16"); + test("bfloat16"); test("float32"); test("float64"); CATCH_REQUIRE(svs::parse_datatype("float128") == DataType::undef); @@ -86,12 +87,15 @@ CATCH_TEST_CASE("Data Type", "[core][datatype]") { // Use in a hash table. std::unordered_map table{}; table[DataType::float16] = 5; + table[DataType::bfloat16] = 6; table[DataType::float32] = 10; CATCH_REQUIRE(!table.contains(DataType::int8)); CATCH_REQUIRE(table.contains(DataType::float16)); + CATCH_REQUIRE(table.contains(DataType::bfloat16)); CATCH_REQUIRE(table.contains(DataType::float32)); CATCH_REQUIRE(table[DataType::float16] == 5); + CATCH_REQUIRE(table[DataType::bfloat16] == 6); CATCH_REQUIRE(table[DataType::float32] == 10); } diff --git a/tests/utils/ivf_reference.cpp b/tests/utils/ivf_reference.cpp new file mode 100644 index 00000000..5bb932f8 --- /dev/null +++ b/tests/utils/ivf_reference.cpp @@ -0,0 +1,46 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// svstest +#include "tests/utils/test_dataset.h" + +// svsbenchmark +#include "svs-benchmark/ivf/test.h" + +// svs +#include "svs/core/distance.h" +#include "svs/lib/timing.h" +#include "svs/third-party/toml.h" + +// stl +#include +#include +#include + +namespace test_dataset::ivf { +namespace { +std::filesystem::path reference_path() { + return test_dataset::reference_directory() / "ivf_reference.toml"; +} +} // namespace + +const toml::table& parse_expected() { + // Make the expected results a magic static variable. + // This shaves off a bit of run time as we only need to parse the toml file once. + static toml::table expected = toml::parse_file(reference_path().native()); + return expected; +} +} // namespace test_dataset::ivf diff --git a/tests/utils/ivf_reference.h b/tests/utils/ivf_reference.h new file mode 100644 index 00000000..64674740 --- /dev/null +++ b/tests/utils/ivf_reference.h @@ -0,0 +1,81 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// svsbenchmark +#include "svs-benchmark/datasets.h" +#include "svs-benchmark/ivf/test.h" + +// svs +#include "svs/core/distance.h" + +// stl +#include +#include +#include + +namespace test_dataset::ivf { + +// Implemented in CPP file. +const toml::table& parse_expected(); + +template +std::vector +expected_results(std::string_view key, svs::DistanceType distance, const T& dataset) { + const auto& table = parse_expected(); + auto v = svs::lib::load>( + svs::lib::node_view_at(table, key), std::nullopt + ); + auto output = std::vector(); + for (const auto& i : v) { + if ((i.distance_ == distance) && i.dataset_.match(dataset)) { + output.push_back(i); + } + } + return output; +} + +/// Return the only reference build for the requested parameters. +/// Throws ANNException if the number of matches is not equal to one. +template +svsbenchmark::ivf::ExpectedResult +expected_build_results(svs::DistanceType distance, const T& dataset) { + auto results = ivf::expected_results("ivf_test_build", distance, dataset); + if (results.size() != 1) { + throw ANNEXCEPTION("Got {} results when only one was expected!", results.size()); + } + // Make sure the only result has build parameters. + auto result = results[0]; + if (!result.build_parameters_.has_value()) { + throw ANNEXCEPTION("Expected build result does not have build parameters!"); + } + return result; +} + +/// Return the only reference search for the requested parameters. +/// Throws ANNException if the number of dataset is not equal to one. +template +svsbenchmark::ivf::ExpectedResult +expected_search_results(svs::DistanceType distance, const T& dataset) { + auto results = ivf::expected_results("ivf_test_search", distance, dataset); + if (results.size() != 1) { + throw ANNEXCEPTION("Got {} results when only one was expected!", results.size()); + } + return results[0]; +} + +} // namespace test_dataset::ivf diff --git a/tests/utils/test_dataset.cpp b/tests/utils/test_dataset.cpp index a89aa042..37ea49ed 100644 --- a/tests/utils/test_dataset.cpp +++ b/tests/utils/test_dataset.cpp @@ -78,6 +78,10 @@ std::filesystem::path groundtruth_cosine_file() { return dataset_directory() / "groundtruth_cosine.ivecs"; } +std::filesystem::path clustering_directory() { + return dataset_directory() / "ivf_clustering"; +} + svs::data::SimpleData queries() { return svs::load_data(query_file()); } svs::data::SimpleData groundtruth_euclidean() { return svs::load_data(groundtruth_euclidean_file()); diff --git a/tests/utils/test_dataset.h b/tests/utils/test_dataset.h index 7d075ce4..9df58c55 100644 --- a/tests/utils/test_dataset.h +++ b/tests/utils/test_dataset.h @@ -58,6 +58,9 @@ std::filesystem::path groundtruth_mip_file(); // Groundtruth of for the queries with respect to the dataset using cosine similarity. std::filesystem::path groundtruth_cosine_file(); +// The directory containing the IVF clustering. +std::filesystem::path clustering_directory(); + ///// Helper Functions svs::data::SimpleData queries(); svs::data::SimpleData groundtruth_euclidean(); diff --git a/tools/benchmark_inputs/ivf/test-generator.toml b/tools/benchmark_inputs/ivf/test-generator.toml new file mode 100644 index 00000000..963ee7b0 --- /dev/null +++ b/tools/benchmark_inputs/ivf/test-generator.toml @@ -0,0 +1,34 @@ +# Copyright 2025 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[ivf_test_generator] +__schema__ = 'benchmark_ivf_test' +__version__ = 'v0.0.0' +data_f32 = 'data_f32.fvecs' +graph = '' +index_config = 'ivf_clustering/' +queries_f32 = 'queries_f32.fvecs' +queries_in_training_set = 100 + + [[ivf_test_generator.groundtruths]] + __schema__ = 'benchmark_distance_and_groundtruth' + __version__ = 'v0.0.0' + distance = 'L2' + path = 'groundtruth_euclidean.ivecs' + + [[ivf_test_generator.groundtruths]] + __schema__ = 'benchmark_distance_and_groundtruth' + __version__ = 'v0.0.0' + distance = 'MIP' + path = 'groundtruth_mip.ivecs' diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index d27bb33c..84f44057 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -46,9 +46,17 @@ create_utility(upgrade_vamana_index_parameters upgrade_vamana_index_parameters.c create_utility(build_index build_index.cpp) create_utility(compute_recall compute_recall.cpp) create_utility(convert_data_to_float16 convert_data_to_float16.cpp) +create_utility(convert_data_to_bfloat16 convert_data_to_bfloat16.cpp) create_utility(search_index search_index.cpp) create_utility(consolidate characterization/consolidate.cpp) create_utility(logging logging.cpp) + +# IVF build and search +if (SVS_EXPERIMENTAL_BUILD_IVF) + create_utility(build_ivf build_ivf.cpp) + create_utility(search_ivf search_ivf.cpp) +endif() + # create_utility(mutable characterization/mutable.cpp) # # Benchmark diff --git a/utils/build_ivf.cpp b/utils/build_ivf.cpp new file mode 100644 index 00000000..890e746f --- /dev/null +++ b/utils/build_ivf.cpp @@ -0,0 +1,130 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "svs/orchestrators/ivf.h" +#include "svsmain.h" + +// stl +#include +#include +#include +#include +#include +#include +#include + +using namespace svs::index::ivf; + +template +void build_ivf_clustering( + Params params, + Data data, + Dist dist, + size_t n_threads, + const std::string& clustering_directory +) { + auto clustering = svs::IVF::build_clustering( + std::move(params), std::move(data), std::move(dist), n_threads + ); + svs::lib::save_to_disk(clustering, clustering_directory); +} + +const std::string HELP = + R"( +The required arguments are as follows: + +(1) Data Element Type (string). Options: (int8, uint8, float, float16, bfloat16) +(2) Path to vector dataset (.vecs format) (string). +(3) Number of clusters to be built +(4) Number of threads to use for index construction (integer). +(5) Should use hierarchical Kmeans? (0/1) +(6) Clustering directory for saving. +(7) Distance type (string - distance type) +)"; + +int svs_main(std::vector args) { + if (args.size() != 8) { + std::cout << "Expected 7 arguments. Instead, got " << args.size() - 1 << ". " + << "The required positional arguments are given below." << std::endl + << std::endl + << HELP << std::endl; + return 1; + } + + size_t i = 1; + const auto& data_type(args[i++]); + const auto& vecs_filename(args[i++]); + const size_t n_clusters = std::stoull(args[i++]); + const size_t n_threads = std::stoull(args[i++]); + const size_t is_hierarchical = std::stoull(args[i++]); + const std::string& clustering_directory(args[i++]); + const auto& distance_type = args[i++]; + + const size_t D = svs::Dynamic; + using Alloc = svs::HugepageAllocator; + + auto data = svs::VectorDataLoader(vecs_filename); + + auto dist_disp = [&](dist_type dist) { + auto params = svs::index::ivf::IVFBuildParameters(n_clusters, 10000, 10, false, .1); + if (is_hierarchical) { + params.is_hierarchical_ = true; + } + if (data_type == "float") { + build_ivf_clustering( + std::move(params), + std::move(data), + std::move(dist), + n_threads, + clustering_directory + ); + } else if (data_type == "float16") { + build_ivf_clustering( + std::move(params), + std::move(data), + std::move(dist), + n_threads, + clustering_directory + ); + } else if (data_type == "bfloat16") { + build_ivf_clustering( + std::move(params), + std::move(data), + std::move(dist), + n_threads, + clustering_directory + ); + } else { + throw ANNEXCEPTION("Unsupported data type: ", data_type, '.'); + } + }; + + if (distance_type == std::string("L2")) { + dist_disp(svs::distance::DistanceL2{}); + } else if (distance_type == std::string("MIP")) { + dist_disp(svs::distance::DistanceIP{}); + } else { + throw ANNEXCEPTION( + "Unsupported distance type. Valid values: L2/MIP. Recieved: ", + distance_type, + '!' + ); + } + + return 0; +} + +SVS_DEFINE_MAIN(); diff --git a/utils/convert_data_to_bfloat16.cpp b/utils/convert_data_to_bfloat16.cpp new file mode 100644 index 00000000..34e1d5ac --- /dev/null +++ b/utils/convert_data_to_bfloat16.cpp @@ -0,0 +1,67 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include "svs/core/io.h" +#include "svs/lib/bfloat16.h" +#include "svsmain.h" + +int svs_main(std::vector args) { + if (args.size() != 4) { + std::cout << "Specify the right parameters: input index, output index, " + "vector_type: 0 for SVS data, 1 for fvecs, 2 for fbin" + << std::endl; + return 1; + } + const std::string& filename_f32 = args[1]; + const std::string& filename_bf16 = args[2]; + const size_t file_type = std::stoull(args[3]); + + if (file_type == 0) { + std::cout << "Converting SVS data!" << std::endl; + auto reader = svs::io::v1::NativeFile{filename_f32}.reader(svs::lib::Type()); + auto writer = svs::io::NativeFile{filename_bf16}.writer( + svs::lib::Type(), reader.ndims() + ); + + for (auto i : reader) { + writer << i; + } + } else if (file_type == 1) { + std::cout << "Converting Vecs data!" << std::endl; + auto reader = svs::io::vecs::VecsReader{filename_f32}; + auto writer = + svs::io::vecs::VecsWriter{filename_bf16, reader.ndims()}; + for (auto i : reader) { + writer << i; + } + } else if (file_type == 2) { + std::cout << "Converting Bin data!" << std::endl; + auto reader = svs::io::binary::BinaryReader{filename_f32}; + auto writer = svs::io::binary::BinaryWriter{ + filename_bf16, reader.nvectors(), reader.ndims()}; + for (auto i : reader) { + writer << i; + } + } + + return 0; +} + +SVS_DEFINE_MAIN(); diff --git a/utils/search_ivf.cpp b/utils/search_ivf.cpp new file mode 100644 index 00000000..1c7d9d6b --- /dev/null +++ b/utils/search_ivf.cpp @@ -0,0 +1,240 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "svs/core/recall.h" +#include "svs/orchestrators/ivf.h" +#include "svsmain.h" + +// stl +#include +#include +#include +#include +#include +#include +#include + +using namespace svs::index::ivf; + +template +using SearchIndexFunction = std::function; + +auto std_dev(std::vector v) { + double sum = std::accumulate(std::begin(v), std::end(v), 0.0); + double m = sum / v.size(); + + double accum = 0.0; + std::for_each(std::begin(v), std::end(v), [&](const double d) { + accum += (d - m) * (d - m); + }); + + double stdev = sqrt(accum / (v.size() - 1)); + return stdev; +} + +template +auto batch_queries( + svs::data::SimpleData query_data, size_t num_batches, size_t batchsize +) { + std::vector> query_batch; + for (size_t batch = 0; batch < num_batches; ++batch) { + auto this_batch = svs::threads::UnitRange{ + batch * batchsize, std::min((batch + 1) * batchsize, query_data.size())}; + query_batch.push_back( + svs::data::SimpleData(this_batch.size(), query_data.dimensions()) + ); + for (size_t i = 0; i < this_batch.size(); i++) { + query_batch[batch].set_datum(i, query_data.get_datum(this_batch[i])); + } + } + return query_batch; +} + +template +void search_index( + const std::string& query_filename, + const std::string& gt_filename, + const size_t n_probes, + const size_t n_neighbors, + const size_t batch_size, + const size_t n_threads, + const size_t n_inner_threads, + const std::filesystem::path& clustering_path, + const std::filesystem::path& data_path, + const Distance dist_type, + const int n_reps +) { + const size_t dim = svs::Dynamic; + + auto data = svs::VectorDataLoader>(data_path); + + auto ivf_index = svs::IVF::assemble_from_file( + clustering_path, std::move(data), dist_type, n_threads, n_inner_threads + ); + + const auto query_data = svs::load_data(query_filename); + const auto groundtruth = svs::load_data(gt_filename); + + ivf_index.set_search_parameters(IVFSearchParameters(n_probes, 1.0)); + + size_t batchsize = query_data.size(); + if (batch_size != 0) { + batchsize = batch_size; + } + + auto query_results = + svs::Matrix{svs::make_dims(query_data.size(), n_neighbors)}; + auto num_batches = svs::lib::div_round_up(query_data.size(), batchsize); + auto query_batch = batch_queries(query_data, num_batches, batchsize); + + auto tic = svs::lib::now(); + for (size_t batch = 0; batch < num_batches; ++batch) { + auto query_result = ivf_index.search(query_batch[batch], n_neighbors); + for (size_t i = 0; i < query_result.n_queries(); i++) { + for (size_t j = 0; j < n_neighbors; j++) { + query_results.at(batch * batchsize + i, j) = query_result.index(i, j); + } + } + } + auto search_time = svs::lib::time_difference(tic); + + std::vector qps; + for (int i = 0; i < n_reps; i++) { + tic = svs::lib::now(); + for (size_t batch = 0; batch < num_batches; ++batch) { + ivf_index.search(query_batch[batch], n_neighbors); + } + search_time = svs::lib::time_difference(tic); + qps.push_back(query_data.size() / search_time); + } + + fmt::print("Raw QPS: {:7.3f} \n", fmt::join(qps, ", ")); + fmt::print( + "Batch Size: {}, Recall: {:.4f}, QPS (Avg: {:7.3f}, Max: {:7.3f}, StdDev: {:7.3f} ) " + "\n", + batchsize, + svs::k_recall_at_n(groundtruth, query_results, 10, 10), + std::reduce(qps.begin(), qps.end()) / qps.size(), + *std::max_element(qps.begin(), qps.end()), + std_dev(qps) + ); +} + +constexpr std::string_view HELP = + R"( +The required arguments are as follows: +(1) Query Element Type (string). Options: (int8, uint8, float) +(2) Data Element Type (string). Options: (int8, uint8, float, float16, bfloat16) +(3) Query File Path (string). Supported extentions: (.vecs, .bin) +(4) Groundtruth File Path (string). Supported extentions: (.vecs, .bin) +(5) n_probes (number of clusters to search) (integer) +(6) Number of neighbors to recall (integer) +(7) Batch size (integer) +(8) Number of threads (integer) +(9) Number of intra-query threads (integer) +(10) Clustering directory (string) +(11) Data directory (string) +(12) Number of repetitions to be run for benchmarking purposes (integer) +(13) Distance type (string - distance type) +)"; + +int svs_main(std::vector&& args) { + if (args.size() != 14) { + std::cout << "Expected 13 arguments. Instead, got " << args.size() - 1 << ". " + << "The required positional arguments are given below." << std::endl + << std::endl + << HELP << std::endl; + return 1; + } + + size_t i = 1; + const auto& query_data_type = args[i++]; + const auto& db_data_type = args[i++]; + const auto& query_filename = args[i++]; + const auto& gt_filename = args[i++]; + const size_t n_probes = std::stoull(args[i++]); + const size_t n_neighbors = std::stoull(args[i++]); + const size_t batch_size = std::stoull(args[i++]); + const size_t n_threads = std::stoull(args[i++]); + const size_t n_inner_threads = std::stoull(args[i++]); + const auto& clustering_path = args[i++]; + const auto& data_path = args[i++]; + const int nreps = std::stoull(args[i++]); + const auto& distance_type = args[i++]; + + auto dist_disp = [&](dist_type dist) { + using KeyType = std::pair; + using ValueType = SearchIndexFunction; + const auto dispatcher = std::map{ + {{"float", "float16"}, search_index}, + {{"float", "bfloat16"}, search_index}, + {{"float", "float"}, search_index}}; + + auto it = dispatcher.find({query_data_type, db_data_type}); + if (it == dispatcher.end()) { + throw ANNEXCEPTION( + "Unsupported Query and Data type pair: (", + query_data_type, + ", ", + db_data_type, + ")!" + ); + } + + // Unpack and call + const auto& f = it->second; + f(query_filename, + gt_filename, + n_probes, + n_neighbors, + batch_size, + n_threads, + n_inner_threads, + clustering_path, + data_path, + dist, + nreps); + }; + + if (distance_type == std::string("L2")) { + dist_disp(svs::distance::DistanceL2{}); + } else if (distance_type == std::string("MIP")) { + dist_disp(svs::distance::DistanceIP{}); + } else { + throw ANNEXCEPTION( + "Unsupported distance type. Valid values: L2/MIP. Recieved: ", + distance_type, + '!' + ); + } + + return 0; +} + +// Include the helper main function. +SVS_DEFINE_MAIN(); From 1a5949b7ada0213970d6f30f691b0ce3ee33660a Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Tue, 8 Jul 2025 11:39:01 -0700 Subject: [PATCH 02/11] Clang formatting --- benchmark/include/svs-benchmark/ivf/common.h | 4 +-- benchmark/include/svs-benchmark/ivf/search.h | 7 ++--- benchmark/include/svs-benchmark/ivf/test.h | 2 +- benchmark/src/ivf/uncompressed.cpp | 33 ++++++-------------- bindings/python/src/ivf.cpp | 5 ++- include/svs/index/ivf/clustering.h | 4 +-- include/svs/index/ivf/common.h | 16 ++++------ include/svs/index/ivf/hierarchical_kmeans.h | 3 +- include/svs/index/ivf/index.h | 1 - include/svs/lib/datatype.h | 2 +- tests/integration/ivf/index_build.cpp | 13 +++----- tests/integration/ivf/index_search.cpp | 12 ++----- tests/svs/index/ivf/hierarchical_kmeans.cpp | 12 +++---- tests/svs/index/ivf/kmeans.cpp | 5 +-- tests/utils/test_dataset.cpp | 2 +- utils/search_ivf.cpp | 3 +- 16 files changed, 46 insertions(+), 78 deletions(-) diff --git a/benchmark/include/svs-benchmark/ivf/common.h b/benchmark/include/svs-benchmark/ivf/common.h index 0ffdc10a..2ff38b9a 100644 --- a/benchmark/include/svs-benchmark/ivf/common.h +++ b/benchmark/include/svs-benchmark/ivf/common.h @@ -33,9 +33,7 @@ SVS_BENCHMARK_FOR_TESTS_ONLY inline search::SearchParameters test_search_paramet SVS_BENCHMARK_FOR_TESTS_ONLY inline std::vector test_search_configs() { - return std::vector( - {{{10, 1.0}, {50, 1.0}}} - ); + return std::vector({{{10, 1.0}, {50, 1.0}}}); } } // namespace svsbenchmark::ivf diff --git a/benchmark/include/svs-benchmark/ivf/search.h b/benchmark/include/svs-benchmark/ivf/search.h index 434f0a3f..7a0c563a 100644 --- a/benchmark/include/svs-benchmark/ivf/search.h +++ b/benchmark/include/svs-benchmark/ivf/search.h @@ -44,9 +44,7 @@ struct IVFState { size_t num_threads_; public: - IVFState( - svs::index::ivf::IVFSearchParameters search_parameters, size_t num_threads - ) + IVFState(svs::index::ivf::IVFSearchParameters search_parameters, size_t num_threads) : search_parameters_{search_parameters} , num_threads_{num_threads} {} @@ -128,8 +126,7 @@ struct SearchJob { svs::DistanceType get_distance() const { return distance_; } // Return the preset search configurations. - const std::vector& - get_search_configs() const { + const std::vector& get_search_configs() const { return preset_parameters_; } diff --git a/benchmark/include/svs-benchmark/ivf/test.h b/benchmark/include/svs-benchmark/ivf/test.h index 9f9041b0..95c9b307 100644 --- a/benchmark/include/svs-benchmark/ivf/test.h +++ b/benchmark/include/svs-benchmark/ivf/test.h @@ -18,10 +18,10 @@ // svs-benchmark #include "svs-benchmark/benchmark.h" -#include "svs-benchmark/test.h" #include "svs-benchmark/ivf/build.h" #include "svs-benchmark/ivf/search.h" #include "svs-benchmark/ivf/static_traits.h" +#include "svs-benchmark/test.h" // svs #include "svs/orchestrators/ivf.h" diff --git a/benchmark/src/ivf/uncompressed.cpp b/benchmark/src/ivf/uncompressed.cpp index bf819183..2ddc7cfd 100644 --- a/benchmark/src/ivf/uncompressed.cpp +++ b/benchmark/src/ivf/uncompressed.cpp @@ -99,18 +99,11 @@ toml::table run_static_uncompressed( auto data = svs::data::SimpleData::load(job.data_); auto tic = svs::lib::now(); auto clustering = svs::IVF::build_clustering( - job.build_parameters_, - data, - distance, - job.num_threads_ + job.build_parameters_, data, distance, job.num_threads_ ); double build_time = svs::lib::time_difference(tic); - auto index = svs::IVF::assemble_from_clustering( - clustering, - data, - distance, - job.num_threads_ - ); + auto index = + svs::IVF::assemble_from_clustering(clustering, data, distance, job.num_threads_); // Save the index if requested by the caller. job.maybe_save_index(clustering); @@ -161,10 +154,7 @@ svsbenchmark::TestFunctionReturn test_search(const IVFTest& job) { ); }); auto index = svs::IVF::assemble_from_file( - job.index_config_, - data_loader, - Distance(), - job.num_threads_ + job.index_config_, data_loader, Distance(), job.num_threads_ ); double load_time = svs::lib::time_difference(tic); auto queries = svs::data::SimpleData::load(job.queries_f32_); @@ -181,8 +171,7 @@ svsbenchmark::TestFunctionReturn test_search(const IVFTest& job) { return TestFunctionReturn{ .key_ = "ivf_test_search", - .results_ = - svs::lib::save_to_table(ivf::ExpectedResult(std::move(kind), results))}; + .results_ = svs::lib::save_to_table(ivf::ExpectedResult(std::move(kind), results))}; } template @@ -191,8 +180,8 @@ svsbenchmark::TestFunctionReturn test_build(const IVFTest& job) { constexpr svs::DistanceType distance = svs::distance_type_v; const auto& groundtruth_path = job.groundtruth_for(distance); - auto build_parameters = svs::index::ivf::IVFBuildParameters{ - 128, 10000, 10, Hierarchical, 0.1}; + auto build_parameters = + svs::index::ivf::IVFBuildParameters{128, 10000, 10, Hierarchical, 0.1}; auto kind = svsbenchmark::Uncompressed(svs::datatype_v); @@ -224,10 +213,7 @@ svsbenchmark::TestFunctionReturn test_build(const IVFTest& job) { ); double build_time = svs::lib::time_difference(tic); auto index = svs::IVF::assemble_from_clustering( - clustering, - data, - distance, - job.num_threads_ + clustering, data, distance, job.num_threads_ ); auto queries = svs::data::SimpleData::load(job.queries_f32_); @@ -244,8 +230,7 @@ svsbenchmark::TestFunctionReturn test_build(const IVFTest& job) { return TestFunctionReturn{ .key_ = "ivf_test_build", - .results_ = - svs::lib::save_to_table(ivf::ExpectedResult(std::move(kind), results))}; + .results_ = svs::lib::save_to_table(ivf::ExpectedResult(std::move(kind), results))}; } } // namespace diff --git a/bindings/python/src/ivf.cpp b/bindings/python/src/ivf.cpp index a96fde8b..2c0646c9 100644 --- a/bindings/python/src/ivf.cpp +++ b/bindings/python/src/ivf.cpp @@ -544,7 +544,10 @@ void wrap(py::module& m) { .def_readwrite("num_iterations", &IVFBuildParameters::num_iterations_) .def_readwrite("is_hierarchical", &IVFBuildParameters::is_hierarchical_) .def_readwrite("training_fraction", &IVFBuildParameters::training_fraction_) - .def_readwrite("hierarchical_level1_clusters", &IVFBuildParameters::hierarchical_level1_clusters_); + .def_readwrite( + "hierarchical_level1_clusters", + &IVFBuildParameters::hierarchical_level1_clusters_ + ); /// Search Parameters using IVFSearchParameters = svs::index::ivf::IVFSearchParameters; diff --git a/include/svs/index/ivf/clustering.h b/include/svs/index/ivf/clustering.h index 707cf0c1..9b2343ef 100644 --- a/include/svs/index/ivf/clustering.h +++ b/include/svs/index/ivf/clustering.h @@ -284,9 +284,7 @@ template struct DenseCluster { } auto get_datum(size_t id) const { return data_.get_datum(id); } - auto get_secondary(size_t id) const { - return data_.get_secondary(id); - } + auto get_secondary(size_t id) const { return data_.get_secondary(id); } auto get_global_id(size_t local_id) const { return ids_[local_id]; } const Data& view_cluster() const { return data_; } diff --git a/include/svs/index/ivf/common.h b/include/svs/index/ivf/common.h index 02aa1439..29fb159d 100644 --- a/include/svs/index/ivf/common.h +++ b/include/svs/index/ivf/common.h @@ -33,7 +33,7 @@ // external #include "tsl/robin_set.h" -//Intel(R) MKL +// Intel(R) MKL #include // stl @@ -199,9 +199,7 @@ auto convert_data(Data& src, Pool& threadpool) { // Partial specialization to preserve the dimensionality and Allocation type template -auto convert_data( - svs::data::SimpleData& src, Pool& threadpool -) { +auto convert_data(svs::data::SimpleData& src, Pool& threadpool) { using allocator_type = svs::lib::rebind_allocator_t; allocator_type rebound_alloctor = {}; @@ -212,8 +210,7 @@ auto convert_data( return dst; } -template -auto convert_data(Data& src) { +template auto convert_data(Data& src) { auto dst = svs::data::SimpleData(src.size(), src.dimensions()); auto threadpool = threads::as_threadpool(1); convert_data(src, dst, threadpool); @@ -280,7 +277,8 @@ void compute_matmul( } } -inline static void generate_unique_ids(std::vector& ids, size_t id_range, std::mt19937& rng) { +inline static void +generate_unique_ids(std::vector& ids, size_t id_range, std::mt19937& rng) { size_t n = ids.size(); tsl::robin_set seen; seen.reserve(n); @@ -479,9 +477,7 @@ void centroid_split( } template -void generate_norms( - Data& data, std::vector& norms, Pool& threadpool -) { +void generate_norms(Data& data, std::vector& norms, Pool& threadpool) { norms.resize(data.size()); threads::parallel_for( threadpool, diff --git a/include/svs/index/ivf/hierarchical_kmeans.h b/include/svs/index/ivf/hierarchical_kmeans.h index 395eb4c2..fca610c9 100644 --- a/include/svs/index/ivf/hierarchical_kmeans.h +++ b/include/svs/index/ivf/hierarchical_kmeans.h @@ -225,7 +225,8 @@ auto hierarchical_kmeans_clustering_impl( ? clusters_level1_all[cluster].size() : max_data_per_cluster; } - auto data_level2 = data::SimpleData{max_data_per_cluster, ndims}; + auto data_level2 = + data::SimpleData{max_data_per_cluster, ndims}; auto assignments_level2_all = std::vector(max_data_per_cluster); size_t cluster_start = 0; diff --git a/include/svs/index/ivf/index.h b/include/svs/index/ivf/index.h index 58d81e49..1662ad51 100644 --- a/include/svs/index/ivf/index.h +++ b/include/svs/index/ivf/index.h @@ -119,7 +119,6 @@ class IVFIndex { /// threads::ThreadPoolHandle& get_threadpool_handle() { return threadpool_; } - size_t size() const { return centroids_.size(); } size_t dimensions() const { return centroids_.dimensions(); } diff --git a/include/svs/lib/datatype.h b/include/svs/lib/datatype.h index 65460420..49955707 100644 --- a/include/svs/lib/datatype.h +++ b/include/svs/lib/datatype.h @@ -77,7 +77,7 @@ template <> inline constexpr std::string_view name() { return "float16"; } template <> inline constexpr std::string_view name() { - return "bfloat16"; + return "bfloat16"; } template <> inline constexpr std::string_view name() { return "float32"; diff --git a/tests/integration/ivf/index_build.cpp b/tests/integration/ivf/index_build.cpp index a59acf96..3e362203 100644 --- a/tests/integration/ivf/index_build.cpp +++ b/tests/integration/ivf/index_build.cpp @@ -30,9 +30,9 @@ #include "catch2/catch_test_macros.hpp" // tests +#include "tests/utils/ivf_reference.h" #include "tests/utils/test_dataset.h" #include "tests/utils/utils.h" -#include "tests/utils/ivf_reference.h" // stl #include @@ -51,16 +51,11 @@ auto build_index( const Distance& dist_type ) { auto data = svs::data::SimpleData::load(data_path); - auto clustering = svs::IVF::build_clustering( - parameters, data, dist_type, num_threads - ); + auto clustering = + svs::IVF::build_clustering(parameters, data, dist_type, num_threads); return svs::IVF::assemble_from_clustering( - std::move(clustering), - std::move(data), - dist_type, - num_threads, - num_inner_threads + std::move(clustering), std::move(data), dist_type, num_threads, num_inner_threads ); } diff --git a/tests/integration/ivf/index_search.cpp b/tests/integration/ivf/index_search.cpp index 48b890b6..cec5fcda 100644 --- a/tests/integration/ivf/index_search.cpp +++ b/tests/integration/ivf/index_search.cpp @@ -39,9 +39,9 @@ #include "fmt/core.h" // tests +#include "tests/utils/ivf_reference.h" #include "tests/utils/test_dataset.h" #include "tests/utils/utils.h" -#include "tests/utils/ivf_reference.h" namespace { @@ -110,17 +110,11 @@ void test_search( ); auto index = svs::IVF::assemble_from_file( - test_dataset::clustering_directory(), - data, - distance, - num_threads, - num_inner_threads + test_dataset::clustering_directory(), data, distance, num_threads, num_inner_threads ); CATCH_REQUIRE(index.get_num_threads() == num_threads); - run_search( - index, queries, groundtruth, expected_result.config_and_recall_ - ); + run_search(index, queries, groundtruth, expected_result.config_and_recall_); CATCH_REQUIRE(index.dimensions() == test_dataset::NUM_DIMENSIONS); } diff --git a/tests/svs/index/ivf/hierarchical_kmeans.cpp b/tests/svs/index/ivf/hierarchical_kmeans.cpp index 23cbc53c..db15940c 100644 --- a/tests/svs/index/ivf/hierarchical_kmeans.cpp +++ b/tests/svs/index/ivf/hierarchical_kmeans.cpp @@ -39,12 +39,12 @@ void test_hierarchical_kmeans_clustering(const Data& data, Distance distance) { for (float training_fraction : {0.55}) { for (size_t l1_clusters : {0, 9}) { auto params = ivf::IVFBuildParameters() - .num_centroids(n_centroids) - .minibatch_size(minibatch) - .num_iterations(iters) - .is_hierarchical(true) - .training_fraction(training_fraction) - .hierarchical_level1_clusters(l1_clusters); + .num_centroids(n_centroids) + .minibatch_size(minibatch) + .num_iterations(iters) + .is_hierarchical(true) + .training_fraction(training_fraction) + .hierarchical_level1_clusters(l1_clusters); auto threadpool = svs::threads::as_threadpool(10); auto [centroids, clusters] = diff --git a/tests/svs/index/ivf/kmeans.cpp b/tests/svs/index/ivf/kmeans.cpp index 69f71df1..49e20a10 100644 --- a/tests/svs/index/ivf/kmeans.cpp +++ b/tests/svs/index/ivf/kmeans.cpp @@ -44,8 +44,9 @@ void test_kmeans_clustering(const Data& data, Distance distance) { .is_hierarchical(false) .training_fraction(training_fraction); auto threadpool = svs::threads::as_threadpool(10); - auto [centroids, clusters] = - ivf::kmeans_clustering(params, data, distance, threadpool); + auto [centroids, clusters] = ivf::kmeans_clustering( + params, data, distance, threadpool + ); CATCH_REQUIRE(centroids.size() == n_centroids); CATCH_REQUIRE(centroids.dimensions() == data.dimensions()); diff --git a/tests/utils/test_dataset.cpp b/tests/utils/test_dataset.cpp index 37ea49ed..d1ff3aba 100644 --- a/tests/utils/test_dataset.cpp +++ b/tests/utils/test_dataset.cpp @@ -79,7 +79,7 @@ std::filesystem::path groundtruth_cosine_file() { } std::filesystem::path clustering_directory() { - return dataset_directory() / "ivf_clustering"; + return dataset_directory() / "ivf_clustering"; } svs::data::SimpleData queries() { return svs::load_data(query_file()); } diff --git a/utils/search_ivf.cpp b/utils/search_ivf.cpp index 1c7d9d6b..4e19b20a 100644 --- a/utils/search_ivf.cpp +++ b/utils/search_ivf.cpp @@ -135,7 +135,8 @@ void search_index( fmt::print("Raw QPS: {:7.3f} \n", fmt::join(qps, ", ")); fmt::print( - "Batch Size: {}, Recall: {:.4f}, QPS (Avg: {:7.3f}, Max: {:7.3f}, StdDev: {:7.3f} ) " + "Batch Size: {}, Recall: {:.4f}, QPS (Avg: {:7.3f}, Max: {:7.3f}, StdDev: {:7.3f} " + ") " "\n", batchsize, svs::k_recall_at_n(groundtruth, query_results, 10, 10), From 8fd2a179939daec0aea22af666183857b37357ee Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Wed, 9 Jul 2025 15:08:47 -0700 Subject: [PATCH 03/11] Enable IVF by default (experimental) --- bindings/python/src/ivf.cpp | 2 +- cmake/options.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/python/src/ivf.cpp b/bindings/python/src/ivf.cpp index 2c0646c9..443d358c 100644 --- a/bindings/python/src/ivf.cpp +++ b/bindings/python/src/ivf.cpp @@ -132,7 +132,7 @@ void register_ivf_assembly_from_file(Dispatcher& dispatcher) { } using IVFAssembleTypes = - std::variant; + std::variant; ///// ///// Build From File diff --git a/cmake/options.cmake b/cmake/options.cmake index c6b833bf..d1dbc074 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -94,7 +94,7 @@ option(SVS_EXPERIMENTAL_ENABLE_NUMA option(SVS_EXPERIMENTAL_BUILD_IVF "Build IVF implementation. Requires Intel(R) MKL support" - OFF # disabled by default + ON # disabled by default ) ##### From 36d69cb282385e33fa8324fd8b38742568c3051d Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Thu, 31 Jul 2025 07:58:48 -0700 Subject: [PATCH 04/11] Selectively enable/disable IVF --- CMakeLists.txt | 2 +- benchmark/CMakeLists.txt | 2 +- benchmark/src/main.cpp | 7 +++++++ cmake/options.cmake | 12 +++++++++--- tests/CMakeLists.txt | 4 ++-- utils/CMakeLists.txt | 2 +- 6 files changed, 21 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c3bb0833..54095bf1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,7 +78,7 @@ include("cmake/spdlog.cmake") include("cmake/toml.cmake") # IVF requires Intel(R) MKL support -if(SVS_EXPERIMENTAL_BUILD_IVF) +if(SVS_EXPERIMENTAL_ENABLE_IVF) include("cmake/mkl.cmake") target_compile_options(${SVS_LIB} INTERFACE "-DSVS_HAVE_MKL=1") endif() diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index e1ceeb16..62995be7 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -49,7 +49,7 @@ set(SHARED_LIBRARY_FILES ) # ivf -if (SVS_EXPERIMENTAL_BUILD_IVF) +if (SVS_EXPERIMENTAL_ENABLE_IVF) list(APPEND SHARED_LIBRARY_FILES src/ivf/uncompressed.cpp src/ivf/search.cpp diff --git a/benchmark/src/main.cpp b/benchmark/src/main.cpp index 05079949..6a8e5c24 100644 --- a/benchmark/src/main.cpp +++ b/benchmark/src/main.cpp @@ -24,10 +24,14 @@ #include "svs-benchmark/vamana/test.h" // inverted #include "svs-benchmark/inverted/inverted.h" + // ivf +SVS_VALIDATE_BOOL_ENV(SVS_ENABLE_IVF) +#if SVS_ENABLE_IVF #include "svs-benchmark/ivf/build.h" #include "svs-benchmark/ivf/search.h" #include "svs-benchmark/ivf/test.h" +#endif // SVS_ENABLE_IVF // stl #include @@ -48,9 +52,12 @@ svsbenchmark::ExecutableDispatcher build_dispatcher() { // inverted svsbenchmark::inverted::register_executables(dispatcher); // ivf +SVS_VALIDATE_BOOL_ENV(SVS_ENABLE_IVF) +#if SVS_ENABLE_IVF dispatcher.register_executable(svsbenchmark::ivf::search_static_workflow()); dispatcher.register_executable(svsbenchmark::ivf::static_workflow()); dispatcher.register_executable(svsbenchmark::ivf::test_generator()); +#endif // SVS_ENABLE_IVF // documentation svsbenchmark::register_dataset_documentation(dispatcher); return dispatcher; diff --git a/cmake/options.cmake b/cmake/options.cmake index d1dbc074..2f9dd416 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -92,9 +92,9 @@ option(SVS_EXPERIMENTAL_ENABLE_NUMA OFF # disabled by default ) -option(SVS_EXPERIMENTAL_BUILD_IVF - "Build IVF implementation. Requires Intel(R) MKL support" - ON # disabled by default +option(SVS_EXPERIMENTAL_ENABLE_IVF + "Enable IVF implementation. Requires Intel(R) MKL support" + OFF # disabled by default ) ##### @@ -145,6 +145,12 @@ else() target_compile_options(${SVS_LIB} INTERFACE -DSVS_INITIALIZE_LOGGER=0) endif() +if (SVS_EXPERIMENTAL_ENABLE_IVF) + target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_IVF=1) +else() + target_compile_options(${SVS_LIB} INTERFACE -DSVS_ENABLE_IVF=0) +endif() + ##### ##### Helper target to apply relevant compiler optimizations. ##### diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9058a7e0..e41b52d6 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -171,7 +171,7 @@ SET(INTEGRATION_TESTS # ${TEST_DIR}/integration/numa_search.cpp -- requires SVS_EXPERIMENTAL_ENABLE_NUMA ) -if (SVS_EXPERIMENTAL_BUILD_IVF) +if (SVS_EXPERIMENTAL_ENABLE_IVF) message("Enabling IVF Integration tests!") list(APPEND INTEGRATION_TESTS ${TEST_DIR}/integration/ivf/index_build.cpp @@ -198,7 +198,7 @@ if (SVS_EXPERIMENTAL_ENABLE_NUMA) list(APPEND TEST_SOURCES ${NUMA_TESTS}) endif() -if (SVS_EXPERIMENTAL_BUILD_IVF) +if (SVS_EXPERIMENTAL_ENABLE_IVF) message("Enabling IVF tests!") list(APPEND TEST_SOURCES ${TEST_DIR}/svs/index/ivf/kmeans.cpp diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 84f44057..888b32f4 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -52,7 +52,7 @@ create_utility(consolidate characterization/consolidate.cpp) create_utility(logging logging.cpp) # IVF build and search -if (SVS_EXPERIMENTAL_BUILD_IVF) +if (SVS_EXPERIMENTAL_ENABLE_IVF) create_utility(build_ivf build_ivf.cpp) create_utility(search_ivf search_ivf.cpp) endif() From 2856d4d0a4ba494e1a253f0679225d6f1f235a42 Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Thu, 31 Jul 2025 08:00:46 -0700 Subject: [PATCH 05/11] clang format --- benchmark/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/src/main.cpp b/benchmark/src/main.cpp index 6a8e5c24..c8db6003 100644 --- a/benchmark/src/main.cpp +++ b/benchmark/src/main.cpp @@ -52,7 +52,7 @@ svsbenchmark::ExecutableDispatcher build_dispatcher() { // inverted svsbenchmark::inverted::register_executables(dispatcher); // ivf -SVS_VALIDATE_BOOL_ENV(SVS_ENABLE_IVF) + SVS_VALIDATE_BOOL_ENV(SVS_ENABLE_IVF) #if SVS_ENABLE_IVF dispatcher.register_executable(svsbenchmark::ivf::search_static_workflow()); dispatcher.register_executable(svsbenchmark::ivf::static_workflow()); From bbadac63b132dbf1f03b5f5303acaf2da9791e14 Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Thu, 31 Jul 2025 09:14:21 -0700 Subject: [PATCH 06/11] Python bindings enable/disable ivf --- bindings/python/CMakeLists.txt | 9 ++++++++- bindings/python/src/python_bindings.cpp | 8 ++++++++ cmake/mkl.cmake | 7 ++++++- cmake/mkl_functions | 2 -- cmake/mkl_functions_ivf | 7 +++++++ tests/CMakeLists.txt | 2 +- 6 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 cmake/mkl_functions_ivf diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index ed630a81..81ae4f2a 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -37,9 +37,16 @@ set(CPP_FILES src/vamana.cpp src/vamana_common.cpp src/svs_mkl.cpp - src/ivf.cpp ) +# ivf +if (SVS_EXPERIMENTAL_ENABLE_IVF) + list(APPEND CPP_FILES + src/ivf.cpp + ) +endif() + + set(LIB_NAME "_svs") pybind11_add_module(${LIB_NAME} MODULE ${CPP_FILES}) target_link_libraries(${LIB_NAME} PRIVATE pybind11::module) diff --git a/bindings/python/src/python_bindings.cpp b/bindings/python/src/python_bindings.cpp index f40613a6..ae79638a 100644 --- a/bindings/python/src/python_bindings.cpp +++ b/bindings/python/src/python_bindings.cpp @@ -20,7 +20,12 @@ #include "svs/python/core.h" #include "svs/python/dynamic_vamana.h" #include "svs/python/flat.h" + +SVS_VALIDATE_BOOL_ENV(SVS_ENABLE_IVF) +#if SVS_ENABLE_IVF #include "svs/python/ivf.h" +#endif // SVS_ENABLE_IVF + #include "svs/python/svs_mkl.h" #include "svs/python/vamana.h" @@ -245,5 +250,8 @@ Convert the `fvecs` file on disk with 32-bit floating point entries to a `fvecs` svs::python::dynamic_vamana::wrap(m); // IVF + SVS_VALIDATE_BOOL_ENV(SVS_ENABLE_IVF) +#if SVS_ENABLE_IVF svs::python::ivf::wrap(m); +#endif // SVS_ENABLE_IVF } diff --git a/cmake/mkl.cmake b/cmake/mkl.cmake index 13af27fc..088cfbea 100644 --- a/cmake/mkl.cmake +++ b/cmake/mkl.cmake @@ -45,6 +45,11 @@ if (SVS_EXPERIMENTAL_BUILD_CUSTOM_MKL) NO_DEFAULT_PATH REQUIRED ) + if (SVS_EXPERIMENTAL_ENABLE_IVF) + set(mkl_functions_file ${CMAKE_CURRENT_LIST_DIR}/mkl_functions_ivf) + else() + set(mkl_functions_file ${CMAKE_CURRENT_LIST_DIR}/mkl_functions) + endif() # This command creates the custom shared-object that will be linked to. add_custom_command( @@ -56,7 +61,7 @@ if (SVS_EXPERIMENTAL_BUILD_CUSTOM_MKL) "libintel64" "interface=ilp64" "threading=sequential" - "export=${CMAKE_CURRENT_LIST_DIR}/mkl_functions" + "export=${mkl_functions_file}" "MKLROOT=${MKL_ROOT}" "name=${CMAKE_CURRENT_BINARY_DIR}/${SVS_MKL_CUSTOM_LIBRARY_NAME}" ) diff --git a/cmake/mkl_functions b/cmake/mkl_functions index 951760fa..8f470043 100644 --- a/cmake/mkl_functions +++ b/cmake/mkl_functions @@ -3,5 +3,3 @@ mkl_simatcopy LAPACKE_sgesvd mkl_get_max_threads MKL_Free_Buffers -cblas_gemm_bf16bf16f32 -cblas_gemm_f16f16f32 diff --git a/cmake/mkl_functions_ivf b/cmake/mkl_functions_ivf new file mode 100644 index 00000000..951760fa --- /dev/null +++ b/cmake/mkl_functions_ivf @@ -0,0 +1,7 @@ +cblas_sgemm +mkl_simatcopy +LAPACKE_sgesvd +mkl_get_max_threads +MKL_Free_Buffers +cblas_gemm_bf16bf16f32 +cblas_gemm_f16f16f32 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e41b52d6..bab566f2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -55,7 +55,6 @@ set(TEST_SOURCES ${TEST_DIR}/utils/schemas.cpp ${TEST_DIR}/utils/test_dataset.cpp ${TEST_DIR}/utils/vamana_reference.cpp - ${TEST_DIR}/utils/ivf_reference.cpp # Lib ${TEST_DIR}/svs/lib/algorithms.cpp ${TEST_DIR}/svs/lib/array.cpp @@ -201,6 +200,7 @@ endif() if (SVS_EXPERIMENTAL_ENABLE_IVF) message("Enabling IVF tests!") list(APPEND TEST_SOURCES + ${TEST_DIR}/utils/ivf_reference.cpp ${TEST_DIR}/svs/index/ivf/kmeans.cpp ${TEST_DIR}/svs/index/ivf/hierarchical_kmeans.cpp ${TEST_DIR}/svs/index/ivf/common.cpp From a25e5f94d8603d42da56973432baf934c7124338 Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Thu, 31 Jul 2025 09:24:13 -0700 Subject: [PATCH 07/11] Remove mkl_functions_ivf from license check --- .github/.licenserc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/.licenserc.yaml b/.github/.licenserc.yaml index ad6f7f31..1b5f8976 100644 --- a/.github/.licenserc.yaml +++ b/.github/.licenserc.yaml @@ -46,6 +46,7 @@ header: - '.github/renovate.json' - '.github/CODEOWNERS' - 'cmake/mkl_functions' + - 'cmake/mkl_functions_ivf' - 'cmake/patches/tomlplusplus_v330.patch' - 'docker/x86_64/manylinux2014/oneAPI.repo' - 'docs/cpp/index/loader-compatibility.csv' From 38d9169a874485312dc592395cbb324464225f4d Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Wed, 6 Aug 2025 14:02:24 -0700 Subject: [PATCH 08/11] Fix some typos --- include/svs/core/data.h | 2 +- include/svs/index/ivf/clustering.h | 2 +- include/svs/lib/narrow.h | 2 +- include/svs/orchestrators/ivf.h | 2 +- utils/build_ivf.cpp | 2 +- utils/search_ivf.cpp | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/svs/core/data.h b/include/svs/core/data.h index 352e39e1..ba8d85c1 100644 --- a/include/svs/core/data.h +++ b/include/svs/core/data.h @@ -126,7 +126,7 @@ class VectorDataLoader { template VectorDataLoader(const UnspecializedVectorDataLoader& other) - : VectorDataLoader{other.path_} { + : VectorDataLoader{other.path_, other.allocator_} { // Validate the refinement. if (datatype_v != other.type_) { throw ANNEXCEPTION("Type mismatch!"); diff --git a/include/svs/index/ivf/clustering.h b/include/svs/index/ivf/clustering.h index 9b2343ef..8f055517 100644 --- a/include/svs/index/ivf/clustering.h +++ b/include/svs/index/ivf/clustering.h @@ -109,7 +109,7 @@ template class Clustering { size_t size() const { return clusters_.size(); } void check_valid(size_t cluster_id) const { - if (cluster_id > size()) { + if (cluster_id >= size()) { throw ANNEXCEPTION( "Cluster id {} can't be higher than the number of clusters!", cluster_id, diff --git a/include/svs/lib/narrow.h b/include/svs/lib/narrow.h index 6c07f667..d7d11f1a 100644 --- a/include/svs/lib/narrow.h +++ b/include/svs/lib/narrow.h @@ -3,7 +3,7 @@ * * The code in this file is a modified version of code from Microsoft Corporation, * published under an MIT License. This modified version is licensed under the - * Apache License, Version 2.0 + * GNU Affero General Public License version 3 * * ORIGINAL LICENSE * ------------------------------------------------------------------------------ diff --git a/include/svs/orchestrators/ivf.h b/include/svs/orchestrators/ivf.h index b77b8aa5..7c035f11 100644 --- a/include/svs/orchestrators/ivf.h +++ b/include/svs/orchestrators/ivf.h @@ -25,7 +25,7 @@ class IVFInterface { public: using search_parameters_type = svs::index::ivf::IVFSearchParameters; - ///// Beckend Information Inteface + ///// Backend information interface virtual std::string experimental_backend_string() const = 0; }; diff --git a/utils/build_ivf.cpp b/utils/build_ivf.cpp index 890e746f..58dbe93f 100644 --- a/utils/build_ivf.cpp +++ b/utils/build_ivf.cpp @@ -118,7 +118,7 @@ int svs_main(std::vector args) { dist_disp(svs::distance::DistanceIP{}); } else { throw ANNEXCEPTION( - "Unsupported distance type. Valid values: L2/MIP. Recieved: ", + "Unsupported distance type. Valid values: L2/MIP. Received: ", distance_type, '!' ); diff --git a/utils/search_ivf.cpp b/utils/search_ivf.cpp index 4e19b20a..6009c84c 100644 --- a/utils/search_ivf.cpp +++ b/utils/search_ivf.cpp @@ -228,7 +228,7 @@ int svs_main(std::vector&& args) { dist_disp(svs::distance::DistanceIP{}); } else { throw ANNEXCEPTION( - "Unsupported distance type. Valid values: L2/MIP. Recieved: ", + "Unsupported distance type. Valid values: L2/MIP. Received: ", distance_type, '!' ); From a55d4a994f555c28d9b1d26b723ed345381eb9cf Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Wed, 6 Aug 2025 14:21:54 -0700 Subject: [PATCH 09/11] Some more code improvements --- include/svs/index/ivf/common.h | 5 ++++- include/svs/index/ivf/index.h | 4 ++++ include/svs/lib/bfloat16.h | 2 +- utils/search_ivf.cpp | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/include/svs/index/ivf/common.h b/include/svs/index/ivf/common.h index 29fb159d..f993ad6b 100644 --- a/include/svs/index/ivf/common.h +++ b/include/svs/index/ivf/common.h @@ -42,7 +42,10 @@ // Common definitions. namespace svs::index::ivf { -#define EPSILON (1 / 1024.) +// Small epsilon value used for floating-point comparisons to avoid precision +// issues. The value 1/1024 (approximately 0.0009765625) is chosen as a reasonable +// threshold for numerical stability in algorithms such as k-means clustering, where exact +// equality of floating-point values is rare due to rounding errors. /// @brief Parameters controlling the IVF build/k-means algortihm. struct IVFBuildParameters { diff --git a/include/svs/index/ivf/index.h b/include/svs/index/ivf/index.h index 1662ad51..eaafe10e 100644 --- a/include/svs/index/ivf/index.h +++ b/include/svs/index/ivf/index.h @@ -35,6 +35,10 @@ namespace svs::index::ivf { +// The maximum batch size for queries is set to 10,000 to balance memory usage and +// performance. This value was chosen based on empirical testing to avoid excessive memory +// allocation while supporting large batch operations typical in high-throughput +// environments. const size_t MAX_QUERY_BATCH_SIZE = 10000; template diff --git a/include/svs/lib/bfloat16.h b/include/svs/lib/bfloat16.h index e162cd60..a1c257a6 100644 --- a/include/svs/lib/bfloat16.h +++ b/include/svs/lib/bfloat16.h @@ -60,7 +60,7 @@ inline uint16_t float_to_bfloat16_untyped(const float x) { } // namespace detail -// On GCC - we need to add this attribute so that BFloat16 members can appear inside +// On GCC, we need to add this attribute so that BFloat16 members can appear inside // packed structs. class __attribute__((packed)) BFloat16 { public: diff --git a/utils/search_ivf.cpp b/utils/search_ivf.cpp index 6009c84c..f1c42d4c 100644 --- a/utils/search_ivf.cpp +++ b/utils/search_ivf.cpp @@ -185,7 +185,7 @@ int svs_main(std::vector&& args) { const size_t n_inner_threads = std::stoull(args[i++]); const auto& clustering_path = args[i++]; const auto& data_path = args[i++]; - const int nreps = std::stoull(args[i++]); + const size_t nreps = std::stoull(args[i++]); const auto& distance_type = args[i++]; auto dist_disp = [&](dist_type dist) { From 584f9a6efbf92f213554f77f8c52c25535cce5bb Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Wed, 6 Aug 2025 14:35:16 -0700 Subject: [PATCH 10/11] typo --- include/svs/index/ivf/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/svs/index/ivf/common.h b/include/svs/index/ivf/common.h index f993ad6b..e6e756c0 100644 --- a/include/svs/index/ivf/common.h +++ b/include/svs/index/ivf/common.h @@ -45,7 +45,7 @@ namespace svs::index::ivf { // Small epsilon value used for floating-point comparisons to avoid precision // issues. The value 1/1024 (approximately 0.0009765625) is chosen as a reasonable // threshold for numerical stability in algorithms such as k-means clustering, where exact -// equality of floating-point values is rare due to rounding errors. +constexpr double EPSILON = 1.0 / 1024.0; /// @brief Parameters controlling the IVF build/k-means algortihm. struct IVFBuildParameters { From 2268429df547f8d16bdbd60b5ec23e9822e32dc4 Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Wed, 27 Aug 2025 10:44:50 -0700 Subject: [PATCH 11/11] 1. Clarify inter/intra query threadpools and stream lined code in IVFIndex class 2. Add docstring in the high level class and methods 3. Use logging instead of fmt::print --- include/svs/index/ivf/hierarchical_kmeans.h | 18 +- include/svs/index/ivf/index.h | 405 +++++++++++++++----- include/svs/index/ivf/kmeans.h | 14 +- 3 files changed, 326 insertions(+), 111 deletions(-) diff --git a/include/svs/index/ivf/hierarchical_kmeans.h b/include/svs/index/ivf/hierarchical_kmeans.h index fca610c9..35d68c1b 100644 --- a/include/svs/index/ivf/hierarchical_kmeans.h +++ b/include/svs/index/ivf/hierarchical_kmeans.h @@ -59,7 +59,8 @@ auto hierarchical_kmeans_clustering_impl( Data& data, Distance& distance, Pool& threadpool, - lib::Type SVS_UNUSED(integer_type) = {} + lib::Type SVS_UNUSED(integer_type) = {}, + svs::logging::logger_ptr logger = svs::logging::get() ) { auto timer = lib::Timer(); auto kmeans_timer = timer.push_back("Hierarchical kmeans clustering"); @@ -76,7 +77,7 @@ auto hierarchical_kmeans_clustering_impl( num_level1_clusters = std::sqrt(num_clusters); } - fmt::print("Level1 clusters: {}\n", num_level1_clusters); + svs::logging::debug(logger, "Level1 clusters: {}\n", num_level1_clusters); size_t num_training_data = lib::narrow(std::ceil(data.size() * parameters.training_fraction_)); @@ -341,9 +342,11 @@ auto hierarchical_kmeans_clustering_impl( level2_training_time.finish(); kmeans_timer.finish(); - svs::logging::debug("{}", timer); - fmt::print( - "hierarchical kmeans clustering time: {}\n", lib::as_seconds(timer.elapsed()) + svs::logging::debug(logger, "{}", timer); + svs::logging::debug( + logger, + "Hierarchical kmeans clustering time: {}\n", + lib::as_seconds(timer.elapsed()) ); return std::make_tuple(std::move(centroids_final), std::move(clusters_final)); @@ -360,10 +363,11 @@ auto hierarchical_kmeans_clustering( Data& data, Distance& distance, Pool& threadpool, - lib::Type integer_type = {} + lib::Type integer_type = {}, + svs::logging::logger_ptr logger = svs::logging::get() ) { return hierarchical_kmeans_clustering_impl( - parameters, data, distance, threadpool, integer_type + parameters, data, distance, threadpool, integer_type, std::move(logger) ); } diff --git a/include/svs/index/ivf/index.h b/include/svs/index/ivf/index.h index eaafe10e..fb3f3c80 100644 --- a/include/svs/index/ivf/index.h +++ b/include/svs/index/ivf/index.h @@ -41,6 +41,39 @@ namespace svs::index::ivf { // environments. const size_t MAX_QUERY_BATCH_SIZE = 10000; +/// @brief IVF (Inverted File) Index implementation for efficient similarity search +/// +/// This class implements an IVF index structure that partitions the search space using +/// centroids and supports a two-level hierarchical threading model for parallel search +/// operations: +/// +/// Threading Model Details: +/// ---------------------- +/// 1. Inter-Query Parallelism (Outer Threading): +/// - Distributes query batches across multiple threads +/// - Each thread processes its assigned queries independently +/// - Used for initial centroid search to find n_probe nearest centroids +/// - Managed by inter_query_threadpool_ +/// +/// 2. Intra-Query Parallelism (Inner Threading): +/// - Activated after finding n_probe nearest centroids for a query +/// - Parallelizes cluster exploration for each query +/// - Multiple threads explore different clusters of the same query concurrently +/// - Each outer thread has its own pool of inner threads (intra_query_threadpools_) +/// - Enables parallel processing of leaf nodes within identified clusters +/// +/// Search Process Flow: +/// ------------------ +/// 1. Queries are distributed across outer threads for initial processing +/// 2. Each outer thread finds n_probe nearest centroids for its assigned queries +/// 3. For each query, its n_probe clusters are distributed across inner threads +/// 4. Inner threads concurrently explore assigned clusters to find nearest neighbors +/// +/// @tparam Centroids Type representing centroid storage +/// @tparam Cluster Type representing cluster storage +/// @tparam Dist Distance metric type +/// @tparam ThreadPoolProto Thread pool prototype type +/// template class IVFIndex { public: @@ -48,91 +81,94 @@ class IVFIndex { using Data = typename Cluster::data_type; using search_parameters_type = IVFSearchParameters; + // Thread-related type aliases for clarity + using InterQueryThreadPool = threads::ThreadPoolHandle; // For inter-query parallelism + using IntraQueryThreadPool = threads::DefaultThreadPool; // For intra-query parallelism + + /// @brief Construct a new IVF Index + /// + /// @param centroids Collection of centroids for space partitioning + /// @param cluster Cluster storage implementation + /// @param distance_function Distance metric for similarity computation + /// @param threadpool_proto Primary thread pool prototype for inter-query parallelism + /// @param intra_query_thread_count Number of threads for intra-query parallelism + /// (default: 1) + /// @param logger logger for per-index logging customization. + /// @throws std::invalid_argument if thread configuration is invalid IVFIndex( Centroids centroids, Cluster cluster, Dist distance_function, ThreadPoolProto threadpool_proto, - size_t n_inner_threads = 1 + const size_t intra_query_thread_count = 1, + svs::logging::logger_ptr logger = svs::logging::get() ) : centroids_{std::move(centroids)} , cluster_{std::move(cluster)} , cluster0_{cluster_.view_cluster(0)} , distance_{std::move(distance_function)} - , threadpool_{threads::as_threadpool(std::move(threadpool_proto))} - , n_inner_threads_{n_inner_threads} { - // Initialize threadpools for intra-query parallelism - for (size_t i = 0; i < threadpool_.size(); i++) { - threadpool_inner_.push_back(threads::as_threadpool(n_inner_threads_)); - } + , inter_query_threadpool_{threads::as_threadpool(std::move(threadpool_proto))} + , intra_query_thread_count_{intra_query_thread_count} + , logger_{std::move(logger)} { + validate_thread_configuration(); + initialize_thread_pools(); + initialize_search_buffers(); + initialize_distance_metadata(); + } - // The first level of search to find the n_probes nearest centroids is done - // using matmul (single thread) and thread batching on the number of centroids. - // Matmul results of each batch with appropriate sizes are initialized here, - // assuming that the static partition will keep the batching same. - auto batches = std::vector>(threadpool_.size()); - threads::parallel_for( - threadpool_, - threads::StaticPartition(centroids_.size()), - [&](auto is, auto tid) { batches[tid] = threads::UnitRange{is}; } - ); - for (size_t i = 0; i < threadpool_.size(); i++) { - matmul_results_.emplace_back(MAX_QUERY_BATCH_SIZE, batches[i].size()); - } + ///// Index Information ///// - // Precalculate centroid norms (required in L2 distances) - if constexpr (std::is_same_v) { - for (size_t i = 0; i < centroids_.size(); i++) { - centroids_norm_.push_back(distance::norm_square(centroids_.get_datum(i))); - } - } - } + /// @brief Get the number of centroids in the index + size_t size() const { return centroids_.size(); } + + /// @brief Get the dimensionality of the indexed vectors + size_t dimensions() const { return centroids_.dimensions(); } + + /// @brief Get the index type name + std::string name() const { return "IVFIndex"; } - ///// Threading Interface + /// @brief Getter method for logger + svs::logging::logger_ptr get_logger() const { return logger_; } + ///// Threading Configuration ///// + + /// @brief Indicates if the number of threads can be changed at runtime static constexpr bool can_change_threads() { return false; } - /// - /// @brief Return the current number of threads used for search. - /// - size_t get_num_threads() const { return threadpool_.size(); } - void set_threadpool(threads::ThreadPoolHandle threadpool) { - if (threadpool.size() != threadpool_.size()) { - throw ANNEXCEPTION("Threadpool change not supported for IVFIndex!"); - } + /// @brief Get the number of threads used for inter-query parallelism + size_t get_num_threads() const { return inter_query_threadpool_.size(); } - threadpool_ = std::move(threadpool); - } + /// @brief Get the number of threads used for intra-query cluster exploration + size_t get_num_intra_query_threads() const { return intra_query_thread_count_; } - /// - /// @brief Destroy the original thread pool and set to the provided one. - /// - /// @param threadpool An acceptable thread pool. - /// - /// @copydoc threadpool_requirements - /// - template - void set_threadpool(Pool threadpool) - requires(!std::is_same_v) - { - set_threadpool(threads::ThreadPoolHandle(std::move(threadpool))); + /// @brief Set a new thread pool for inter-query parallelism + /// @throws std::runtime_error if thread count differs from original + void set_threadpool(InterQueryThreadPool threadpool) { + if (threadpool.size() != inter_query_threadpool_.size()) { + throw std::runtime_error("Threadpool change not supported for IVFIndex - " + "thread count must remain constant"); + } + inter_query_threadpool_ = std::move(threadpool); } - /// - /// @brief Return the current thread pool handle. - /// - threads::ThreadPoolHandle& get_threadpool_handle() { return threadpool_; } + /// @brief Get the thread pool handle for inter-query parallelism + InterQueryThreadPool& get_threadpool_handle() { return inter_query_threadpool_; } - size_t size() const { return centroids_.size(); } - size_t dimensions() const { return centroids_.dimensions(); } + ///// Search Parameters ///// - ///// Search Parameter Setting + /// @brief Get current search parameters search_parameters_type get_search_parameters() const { return search_parameters_; } + /// @brief Update search parameters void set_search_parameters(const search_parameters_type& search_parameters) { search_parameters_ = search_parameters; } + ///// Search Implementation ///// + + /// @brief Search closure for centroid distance computation + /// @return Function object handling initial centroid search phase (inter-query + /// parallel) auto search_centroids_closure() { return [&](const auto& query, auto& buffer, size_t id) { search_centroids( @@ -147,6 +183,8 @@ class IVFIndex { }; } + /// @brief Search closure for cluster traversal + /// @return Function object handling cluster exploration (intra-query parallel) auto search_leaves_closure() { return [&](const auto& query, auto& distance, @@ -159,12 +197,26 @@ class IVFIndex { cluster_, buffer_centroids, buffer_leaves, - threadpool_inner_[tid] + intra_query_threadpools_[tid] ); }; } - // Search + /// @brief Perform similarity search for given queries + /// + /// Search Process: + /// 1. Inter-query parallel: Distribute queries across primary threads + /// 2. For each query: Find n_probe nearest centroids + /// 3. Intra-query parallel: Explore identified clusters using inner threads + /// 4. Combine results from all explored clusters + /// + /// @tparam Idx Index type for results + /// @tparam Queries Query dataset type + /// @param results View for storing search results + /// @param queries Query vectors to search for + /// @param search_parameters Search configuration parameters + /// @param cancel Optional cancellation predicate + /// @throws std::runtime_error if query batch size exceeds limits template void search( QueryResultView results, @@ -172,38 +224,30 @@ class IVFIndex { const search_parameters_type& search_parameters, const lib::DefaultPredicate& SVS_UNUSED(cancel) = lib::Returns(lib::Const()) ) { - if (queries.size() > MAX_QUERY_BATCH_SIZE) { - throw ANNEXCEPTION( - "Number of queries {} higher than expected {}, increase value of " - "MAX_QUERY_BATCH_SIZE", - queries.size(), - MAX_QUERY_BATCH_SIZE - ); - } + validate_query_batch_size(queries.size()); size_t num_neighbors = results.n_neighbors(); size_t buffer_leaves_size = search_parameters.k_reorder_ * num_neighbors; - compute_centroid_distances(queries, centroids_, matmul_results_, threadpool_); + // Phase 1: Inter-query parallel - Compute distances to centroids + compute_centroid_distances( + queries, centroids_, matmul_results_, inter_query_threadpool_ + ); + + // Phase 2: Process queries in parallel threads::parallel_for( - threadpool_, + inter_query_threadpool_, threads::StaticPartition(queries.size()), [&](auto is, auto tid) { - auto buffer_centroids = SortedBuffer>( - search_parameters.n_probes_, distance::comparator(distance_) - ); - std::vector>> buffer_leaves; - for (size_t j = 0; j < n_inner_threads_; j++) { - buffer_leaves.push_back(SortedBuffer>( - buffer_leaves_size, distance::comparator(distance_) - )); - } - - // Pre-allocate scratch space needed by the dataset implementation. + // Initialize search buffers + auto buffer_centroids = create_centroid_buffer(search_parameters.n_probes_); + auto buffer_leaves = create_leaf_buffers(buffer_leaves_size); + + // Prepare cluster search scratch space auto scratch = extensions::per_thread_batch_search_setup(cluster0_, distance_); - // Perform a search over the batch of queries. + // Execute search with intra-query parallelism extensions::per_thread_batch_search( cluster0_, cluster_, @@ -221,23 +265,119 @@ class IVFIndex { ); } - std::string name() const { return "IVFIndex"; } - private: + ///// Core Components ///// Centroids centroids_; Cluster cluster_; Data cluster0_; Dist distance_; - threads::ThreadPoolHandle threadpool_; - const size_t n_inner_threads_ = 1; - std::vector threadpool_inner_; + + ///// Threading Infrastructure ///// + InterQueryThreadPool inter_query_threadpool_; // Handles parallelism across queries + const size_t intra_query_thread_count_; // Number of threads per query processing + std::vector + intra_query_threadpools_; // Per-query parallel cluster exploration + + ///// Search Data ///// std::vector> matmul_results_; std::vector centroids_norm_; - - // Tunable Parameters search_parameters_type search_parameters_{}; + + // SVS logger for per index logging + svs::logging::logger_ptr logger_; + + ///// Initialization Methods ///// + + void validate_thread_configuration() { + if (intra_query_thread_count_ < 1) { + throw std::invalid_argument("Intra-query thread count must be at least 1"); + } + } + + void initialize_thread_pools() { + // Create thread pools for intra-query (cluster-level) parallelism + for (size_t i = 0; i < inter_query_threadpool_.size(); i++) { + intra_query_threadpools_.push_back( + threads::as_threadpool(intra_query_thread_count_) + ); + } + } + + void initialize_search_buffers() { + // Initialize matmul result buffers for centroid distance computation + auto batches = + std::vector>(inter_query_threadpool_.size()); + threads::parallel_for( + inter_query_threadpool_, + threads::StaticPartition(centroids_.size()), + [&](auto is, auto tid) { batches[tid] = threads::UnitRange{is}; } + ); + + for (size_t i = 0; i < inter_query_threadpool_.size(); i++) { + matmul_results_.emplace_back(MAX_QUERY_BATCH_SIZE, batches[i].size()); + } + } + + void initialize_distance_metadata() { + // Precalculate centroid norms for L2 distance + if constexpr (std::is_same_v) { + centroids_norm_.reserve(centroids_.size()); + for (size_t i = 0; i < centroids_.size(); i++) { + centroids_norm_.push_back(distance::norm_square(centroids_.get_datum(i))); + } + } + } + + ///// Helper Methods ///// + + void validate_query_batch_size(size_t query_size) const { + if (query_size > MAX_QUERY_BATCH_SIZE) { + throw std::runtime_error(fmt::format( + "Query batch size {} exceeds maximum allowed {}", + query_size, + MAX_QUERY_BATCH_SIZE + )); + } + } + + auto create_centroid_buffer(size_t n_probes) const { + return SortedBuffer>( + n_probes, distance::comparator(distance_) + ); + } + + auto create_leaf_buffers(size_t buffer_size) const { + std::vector>> buffers; + buffers.reserve(intra_query_thread_count_); + for (size_t j = 0; j < intra_query_thread_count_; j++) { + buffers.push_back(SortedBuffer>( + buffer_size, distance::comparator(distance_) + )); + } + return buffers; + } }; +/// @brief Build an IVF clustering using either hierarchical or flat k-means +/// +/// This function builds an IVF (Inverted File) clustering structure by: +/// 1. Loading the input data using the provided data prototype +/// 2. Performing either hierarchical or flat k-means clustering based on parameters +/// 3. Measuring and logging the build performance +/// +/// @tparam BuildType The data type used for building (e.g., float, float16, bfloat16) +/// @tparam DataProto Type of the data prototype for loading +/// @tparam Distance Distance metric type +/// @tparam ThreadpoolProto Thread pool prototype type +/// +/// @param parameters IVF build configuration parameters +/// @param data_proto Data prototype for loading the dataset +/// @param distance Distance metric for clustering +/// @param threadpool_proto Thread pool for parallel processing +/// @param logger logger for logging customization. +/// +/// @return Clustering object containing centroids and cluster assignments +/// template < typename BuildType, typename DataProto, @@ -247,28 +387,36 @@ auto build_clustering( const IVFBuildParameters& parameters, const DataProto& data_proto, Distance distance, - ThreadpoolProto threadpool_proto + ThreadpoolProto threadpool_proto, + svs::logging::logger_ptr logger = svs::logging::get() ) { auto threadpool = threads::as_threadpool(std::move(threadpool_proto)); auto data = svs::detail::dispatch_load(data_proto, threadpool); + // Start timing the clustering process auto tic = svs::lib::now(); data::SimpleData centroids; std::vector> clusters; + + using Idx = lib::Type; + // Choose clustering method based on parameters if (parameters.is_hierarchical_) { std::tie(centroids, clusters) = hierarchical_kmeans_clustering( - parameters, data, distance, threadpool + parameters, data, distance, threadpool, Idx{}, logger ); } else { - std::tie(centroids, clusters) = - kmeans_clustering(parameters, data, distance, threadpool); + std::tie(centroids, clusters) = kmeans_clustering( + parameters, data, distance, threadpool, Idx{}, logger + ); } + // Create and validate clustering Clustering clustering(std::move(centroids), std::move(clusters)); + + // Log performance metrics auto build_time = svs::lib::time_difference(tic); - fmt::print("IVF build time: {} seconds\n", build_time); + svs::logging::debug(logger, "IVF build time: {} seconds\n", build_time); - auto logger = svs::logging::get(); svs::logging::debug( logger, "IVF Clustering Stats: {}", clustering.statistics().report("\n") ); @@ -276,6 +424,28 @@ auto build_clustering( return clustering; } +/// @brief Assemble an IVF index from an existing clustering +/// +/// This function creates a complete IVF index by: +/// 1. Loading the dataset +/// 2. Creating dense cluster representations +/// 3. Constructing the final IVF index with parallel search support +/// +/// @tparam Clustering Type of the clustering structure +/// @tparam DataProto Type of the data prototype for loading +/// @tparam Distance Distance metric type +/// @tparam ThreadpoolProto Thread pool prototype type +/// +/// @param clustering Existing clustering structure +/// @param data_proto Data prototype for loading the dataset +/// @param distance Distance metric for searching +/// @param threadpool_proto Thread pool for parallel processing +/// @param intra_query_thread_count Number of threads for intra-query parallelism (default: +/// 1) +/// @param logger logger for logging customization. +/// +/// @return Fully constructed IVF index ready for searching +/// template < typename Clustering, typename DataProto, @@ -286,15 +456,20 @@ auto assemble_from_clustering( const DataProto& data_proto, Distance distance, ThreadpoolProto threadpool_proto, - const size_t n_inner_threads = 1 + const size_t intra_query_thread_count = 1, + svs::logging::logger_ptr logger = svs::logging::get() ) { + // Initialize timing infrastructure auto timer = lib::Timer(); auto assemble_timer = timer.push_back("Total Assembling time"); + + // Phase 1: Load dataset auto data_load_timer = timer.push_back("Data loading"); auto threadpool = threads::as_threadpool(std::move(threadpool_proto)); auto data = svs::detail::dispatch_load(data_proto, threadpool); data_load_timer.finish(); + // Phase 2: Create dense cluster representation auto dense_cluster_timer = timer.push_back("Dense clustering"); using centroids_type = data::SimpleData; using data_type = typename decltype(data)::lib_alloc_data_type; @@ -304,20 +479,46 @@ auto assemble_from_clustering( ); dense_cluster_timer.finish(); + // Phase 3: Construct IVF index auto index_build_timer = timer.push_back("IVF index construction"); auto ivf_index = IVFIndex( std::move(clustering.centroids()), std::move(dense_clusters), std::move(distance), std::move(threadpool), - n_inner_threads + intra_query_thread_count, + logger ); index_build_timer.finish(); + + // Log timing results assemble_timer.finish(); - svs::logging::debug("{}", timer); + svs::logging::debug(logger, "{}", timer); return ivf_index; } +/// @brief Assemble an IVF index from a saved clustering file +/// +/// This function loads a previously saved clustering from disk and creates +/// a complete IVF index. It's a convenience wrapper around assemble_from_clustering +/// that handles the clustering loading step. +/// +/// @tparam Centroids Type of the centroid data +/// @tparam DataProto Type of the data prototype for loading +/// @tparam Distance Distance metric type +/// @tparam ThreadpoolProto Thread pool prototype type +/// +/// @param clustering_path Path to the saved clustering file +/// @param data_proto Data prototype for loading the dataset +/// @param distance Distance metric for searching +/// @param threadpool_proto Thread pool for parallel processing +/// @param n_inner_threads Number of threads for intra-query parallelism (default: 1) +/// @param intra_query_thread_count Number of threads for intra-query parallelism (default: +/// 1) +/// @param logger logger for logging customization. +/// +/// @return Fully constructed IVF index ready for searching +/// template < typename Centroids, typename DataProto, @@ -328,20 +529,26 @@ auto assemble_from_file( const DataProto& data_proto, Distance distance, ThreadpoolProto threadpool_proto, - const size_t n_inner_threads = 1 + const size_t intra_query_thread_count = 1, + svs::logging::logger_ptr logger = svs::logging::get() ) { + // Define the clustering type using centroids_type = data::SimpleData; + + // Initialize thread pool and load clustering from disk auto threadpool = threads::as_threadpool(std::move(threadpool_proto)); auto clustering = svs::lib::load_from_disk>( clustering_path, threadpool ); + // Delegate to the main assembly function return assemble_from_clustering( std::move(clustering), data_proto, std::move(distance), std::move(threadpool), - n_inner_threads + intra_query_thread_count, + std::move(logger) ); } diff --git a/include/svs/index/ivf/kmeans.h b/include/svs/index/ivf/kmeans.h index 17e8747b..ca584315 100644 --- a/include/svs/index/ivf/kmeans.h +++ b/include/svs/index/ivf/kmeans.h @@ -31,7 +31,8 @@ auto kmeans_clustering_impl( Data& data, Distance& distance, Pool& threadpool, - lib::Type SVS_UNUSED(integer_type) = {} + lib::Type SVS_UNUSED(integer_type) = {}, + svs::logging::logger_ptr logger = svs::logging::get() ) { auto timer = lib::Timer(); auto kmeans_timer = timer.push_back("Non-hierarchical kmeans clustering"); @@ -129,8 +130,10 @@ auto kmeans_clustering_impl( } final_assignments_time.finish(); kmeans_timer.finish(); - svs::logging::debug("{}", timer); - fmt::print("kmeans clustering time: {}\n", lib::as_seconds(timer.elapsed())); + svs::logging::debug(logger, "{}", timer); + svs::logging::debug( + logger, "kmeans clustering time: {}\n", lib::as_seconds(timer.elapsed()) + ); return std::make_tuple(centroids, std::move(clusters)); } @@ -145,10 +148,11 @@ auto kmeans_clustering( Data& data, Distance& distance, Pool& threadpool, - lib::Type integer_type = {} + lib::Type integer_type = {}, + svs::logging::logger_ptr logger = svs::logging::get() ) { return kmeans_clustering_impl( - parameters, data, distance, threadpool, integer_type + parameters, data, distance, threadpool, integer_type, std::move(logger) ); } } // namespace svs::index::ivf