Add more pre-commit hooks

.github/CODEOWNERS

@@ -1,10 +1,10 @@
 # C/C++ library code
 /include/               @ibhati @ahuber21
-/cmake/                 @mihaic @ibhati @ahuber21 
-/tests/                 @mihaic @ibhati @ahuber21 @yuejiaointel 
-/benchmark/             @mihaic @ibhati @ahuber21 
-/tools/                 @mihaic @ibhati @ahuber21 
-/utils/                 @mihaic @ibhati @ahuber21 
+/cmake/                 @mihaic @ibhati @ahuber21
+/tests/                 @mihaic @ibhati @ahuber21 @yuejiaointel
+/benchmark/             @mihaic @ibhati @ahuber21
+/tools/                 @mihaic @ibhati @ahuber21
+/utils/                 @mihaic @ibhati @ahuber21
 
 # Python and other language bindings
 /bindings/              @ethanglaser @ibhati @ahuber21
@@ -14,7 +14,7 @@
 /data/                  @ibhati @ahuber21
 
 # Docker and build environment
-/docker/                @mihaic @ahuber21 
+/docker/                @mihaic @ahuber21
 
 # CI and infra
 /.github                @mihaic @homksei @yuejiaointel @ethanglaser

.pre-commit-config.yaml

@@ -27,3 +27,14 @@ repos:
     hooks:
       - id: check-yaml
         files: \.ya?ml$
+      - id: check-added-large-files
+      - id: check-illegal-windows-names
+      - id: check-json
+        files: \.json$
+      - id: check-merge-conflict
+      - id: check-toml
+        files: \.toml$
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
+      - id: mixed-line-ending

CMakeLists.txt

@@ -177,4 +177,3 @@ install(FILES
     ${SVS_CMAKE_FIND_FILES}
     DESTINATION "${LIB_CONFIG_INSTALL_DIR}"
 )
-

HISTORY.md

@@ -707,4 +707,3 @@ affect performance or users relying on internal APIs.
 ## Third Party
 
 * Bump [fmtlib](https://github.com/fmtlib/fmt) from 9.1.0 to 10.1.1.
-

LICENSE

@@ -202,10 +202,10 @@
    limitations under the License.
   ============================================================================
    Intel® oneAPI Math Kernel Library (Intel® oneMKL): Copyright 2022 Intel Corporation
-    
+
    Intel® Scalable Vector Search: Copyright (C) Intel Corporation provides shared library in binary form with optional algorithms implementations that can be used with Scalable Vector Search library
-   
-   ------------------------------   
+
+   ------------------------------
    Intel Simplified Software License (Version October 2022)
 
    Use and Redistribution.  You may use and redistribute the software, which is
@@ -277,4 +277,3 @@
    objections. THE UNITED NATIONS CONVENTION ON CONTRACTS FOR THE INTERNATIONAL
    SALE OF GOODS (1980) IS SPECIFICALLY EXCLUDED AND WILL NOT APPLY TO THE
    SOFTWARE.
-

SECURITY.md

@@ -19,4 +19,3 @@ Intel is committed to rapidly addressing security vulnerabilities affecting our
 
 ## Reporting a Vulnerability
 Please report any security vulnerabilities in this project [utilizing the guidelines here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html).
-

bindings/cpp/src/api_defs.cpp

@@ -43,4 +43,4 @@ void Status::destroy_message() noexcept {
     message_storage_ = nullptr;
 }
 } // namespace runtime
-} // namespace svs
+} // namespace svs

bindings/python/.gitignore

@@ -2,4 +2,3 @@
 _skbuild/
 build/
 *.egg-info
-

bindings/python/CMakeLists.txt

@@ -90,4 +90,3 @@ if(DEFINED SKBUILD)
     # wheel.
     install(FILES "${CMAKE_CURRENT_LIST_DIR}/../../data/serialization.toml" DESTINATION .)
 endif()
-

bindings/python/src/svs/__init__.py

@@ -30,4 +30,3 @@
 
 # Make the upgrader available without explicit import.
 from . import upgrader
-

bindings/python/src/svs/upgrader.py

@@ -297,4 +297,3 @@ def upgrade(
         )
 
     print(f"File {path} is up to date!")
-

bindings/python/tests/__init__.py

@@ -11,4 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-

bindings/python/tests/dynamic.py

@@ -81,4 +81,3 @@ def ground_truth(self, num_neighbors: int):
         index = svs.Flat(sub_dataset, svs.DistanceType.L2, self.num_threads)
         I, D = index.search(self.queries, num_neighbors)
         return ids_np[I]
-

bindings/python/tests/test_dynamic_flat.py

@@ -72,7 +72,7 @@ def recall_check(
             print(f"Reloaded index has {len(reloaded.all_ids())} IDs")
             print(f"Original IDs sample: {sorted(list(index.all_ids()))[:10]}")
             print(f"Reloaded IDs sample: {sorted(list(reloaded.all_ids()))[:10]}")
-            
+
             I, D = reloaded.search(reference.queries, num_neighbors)
             reloaded_recall = svs.k_recall_at(gt, I, num_neighbors, num_neighbors)
 

bindings/python/tests/test_dynamic_vamana.py

@@ -201,4 +201,3 @@ def test_build_from_loader(self):
         recall = svs.k_recall_at(groundtruth, I, k, k)
         # Recall in plausible range
         self.assertTrue(0.5 < recall <= 1.0)
-

bindings/python/tests/test_reconstruction.py

@@ -90,4 +90,3 @@ def test_reconstruction(self):
                 self.assertTrue(np.array_equal(shuffled_data, r))
             else:
                 raise Exception(f"Unhandled loader kind: {loader}")
-

bindings/python/tests/test_vamana_common.py

@@ -64,4 +64,3 @@ def test_vamana_search_parameters(self):
         x = svs.VamanaSearchParameters(svs.SearchBufferConfig(10, 20), True)
         self.assertEqual(x.buffer_config, svs.SearchBufferConfig(10, 20))
         self.assertEqual(x.search_buffer_visited_set, True)
-

cmake/FindNuma.cmake

@@ -49,4 +49,3 @@ endif()
 mark_as_advanced(NUMA_INCLUDE_DIRS NUMA_LIBRARY_DIR NUMA_LIBRARY)
 
 find_package_handle_standard_args(Numa REQUIRED_VARS NUMA_INCLUDE_DIRS NUMA_LIBRARY)
-

cmake/openmp.cmake

@@ -18,4 +18,3 @@ if (OPENMP_CXX_FOUND)
 else()
     message(FATAL_ERROR "no OpenMP support")
 endif()
-

cmake/options.cmake

@@ -212,4 +212,3 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
         target_compile_options(svs_compile_options INTERFACE -Wno-uninitialized)
     endif()
 endif()
-

cmake/patches/tomlplusplus_v330.patch

@@ -4,7 +4,7 @@ index 1179d37..59cf878 100644
 +++ b/CMakeLists.txt
 @@ -31,7 +31,8 @@ target_include_directories(
  target_compile_features(tomlplusplus_tomlplusplus INTERFACE cxx_std_17)
- 
+
  # ---- Install rules and examples ----
 -if(PROJECT_IS_TOP_LEVEL)
 +option(TOMLPLUSPLUS_INSTALL "Enable cmake installation" OFF)
@@ -19,7 +19,7 @@ index 4b754b7..5ebc665 100644
 @@ -134,6 +134,18 @@ TOML_IMPL_NAMESPACE_START
  		return node_ptr{ make_node_impl(static_cast<T&&>(val), flags) };
  	}
- 
+
 +	TOML_NODISCARD
 +	inline node_ptr make_node(node_ptr&& val, value_flags flags = preserve_source_value_flags)
 +	{

data/serialization.toml

@@ -321,4 +321,3 @@ schema_version = "v0.0.0"
     graph = "str"
     queries_f32 = "str"
     queries_in_training_set = "int"
-

docker/x86_64/manylinux2014/Dockerfile

@@ -17,7 +17,7 @@
 COPY ./oneAPI.repo /etc/yum.repos.d/oneAPI.repo
 
 # Install gcc-11
 RUN yum --disablerepo=epel install -y \
     devtoolset-11-gcc \
     devtoolset-11-gcc-c++ \
     intel-oneapi-mkl \
@@ -26,4 +26,3 @@
 # Enable CMake to find the config files for Intel(R) MKL.
 COPY ./entry.sh .
 ENTRYPOINT ["./entry.sh"]
-

examples/python/vamana_with_compression_lvq.py

@@ -60,7 +60,7 @@ def run():
         svs.DataType.float32
     )
 
-    # Next - we construct a LVQLoader which is configured to use LVQ compression with 4 
+    # Next - we construct a LVQLoader which is configured to use LVQ compression with 4
     # bits for the primary and 8 bits for the residual quantization.
     B1 = 4    # Number of bits for the first level LVQ quantization
     B2 = 8    # Number of bits for the residuals quantization
@@ -69,9 +69,9 @@ def run():
         residual=B2,
     )
     # [create-loader]
-    
+
     # An index can be constructed using a LeanVec dataset.
-    # [build-parameters]    
+    # [build-parameters]
     parameters = svs.VamanaBuildParameters(
         graph_max_degree = 64,
         window_size = 128,
@@ -89,19 +89,19 @@ def run():
 
     # Set the search window size of the index and perform queries and load the queries.
     # [perform-queries]
-    n_neighbors = 10    
+    n_neighbors = 10
     index.search_window_size = 20
-    index.num_threads = 4    
+    index.num_threads = 4
 
-    queries = svs.read_vecs(os.path.join(test_data_dir, "queries.fvecs"))    
+    queries = svs.read_vecs(os.path.join(test_data_dir, "queries.fvecs"))
     I, D = index.search(queries, n_neighbors)
     # [perform-queries]
 
     # Compare with the groundtruth.
     # [recall]
-    groundtruth = svs.read_vecs(os.path.join(test_data_dir, "groundtruth.ivecs"))    
+    groundtruth = svs.read_vecs(os.path.join(test_data_dir, "groundtruth.ivecs"))
     recall = svs.k_recall_at(groundtruth, I, n_neighbors, n_neighbors)
-    print(f"Recall = {recall}")    
+    print(f"Recall = {recall}")
     # [recall]
     assert_equal(recall, 0.953)
 
@@ -112,10 +112,10 @@ def run():
         os.path.join(test_data_dir, "example_graph"),
         os.path.join(test_data_dir, "example_data"),
     )
-    
+
     index = svs.Vamana(
         os.path.join(test_data_dir, "example_config"),
-        os.path.join(test_data_dir, "example_graph"),        
+        os.path.join(test_data_dir, "example_graph"),
         os.path.join(test_data_dir, "example_data"),
         svs.DistanceType.L2,
         num_threads = 4,

include/svs/index/flat/dynamic_flat.h

@@ -767,4 +767,4 @@ auto auto_dynamic_assemble(
     );
 }
 
-} // namespace svs::index::flat
+} // namespace svs::index::flat

include/svs/index/vamana/build_params.h

@@ -130,4 +130,4 @@ struct VamanaBuildParameters {
         );
     }
 };
-} // namespace svs::index::vamana
+} // namespace svs::index::vamana

tests/svs/index/inverted/memory_based.cpp

@@ -72,4 +72,4 @@ CATCH_TEST_CASE("InvertedIndex Logging Test", "[long][logging]") {
     CATCH_REQUIRE(global_captured_logs.empty());
     CATCH_REQUIRE(captured_logs[0].find("Vamana Build Parameters:") != std::string::npos);
     CATCH_REQUIRE(captured_logs[1].find("Number of syncs") != std::string::npos);
-}
+}

tools/clang-format.sh

@@ -22,4 +22,3 @@ for i in "${DIRECTORIES[@]}"
 do
     find "./$i" \( -iname "*.h" -o -iname "*.cpp" \) ! -iname "*toml_impl.h" | xargs "$CLANGFORMAT" -i
 done
-