diff --git a/benchs/bench_6bit_codec.cpp b/benchs/bench_6bit_codec.cpp index b4ac0b04b6..6757855747 100644 --- a/benchs/bench_6bit_codec.cpp +++ b/benchs/bench_6bit_codec.cpp @@ -19,6 +19,7 @@ using namespace faiss; static void bench(benchmark::State& state) { int d = 128; int n = 2000; + state.SetLabel(faiss::get_compile_options()); std::vector x(d * n); diff --git a/faiss/perf_tests/bench_scalar_quantizer_accuracy.cpp b/faiss/perf_tests/bench_scalar_quantizer_accuracy.cpp new file mode 100644 index 0000000000..7d8e70f958 --- /dev/null +++ b/faiss/perf_tests/bench_scalar_quantizer_accuracy.cpp @@ -0,0 +1,91 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace faiss; +DEFINE_uint32(d, 128, "dimension"); +DEFINE_uint32(n, 2000, "dimension"); +DEFINE_uint32(iterations, 20, "iterations"); + +static void bench_reconstruction_error( + benchmark::State& state, + ScalarQuantizer::QuantizerType type, + int d, + int n) { + state.SetLabel(faiss::get_compile_options()); + std::vector x(d * n); + + float_rand(x.data(), d * n, 12345); + + // make sure it's idempotent + ScalarQuantizer sq(d, type); + + sq.train(n, x.data()); + + size_t code_size = sq.code_size; + state.counters["code_size"] = sq.code_size; + + // encode + std::vector codes(code_size * n); + sq.compute_codes(x.data(), codes.data(), n); + + // decode + std::vector x2(d * n); + sq.decode(codes.data(), x2.data(), n); + + state.counters["sql2_recons_error"] = + fvec_L2sqr(x.data(), x2.data(), n * d) / n; + + // encode again + std::vector codes2(code_size * n); + sq.compute_codes(x2.data(), codes2.data(), n); + + size_t ndiff = 0; + for (size_t i = 0; i < codes.size(); i++) { + if (codes[i] != codes2[i]) + ndiff++; + } + + state.counters["ndiff_for_idempotence"] = ndiff; + + state.counters["code_size_two"] = codes.size(); + state.SetLabel(faiss::get_compile_options()); +} + +int main(int argc, char** argv) { + benchmark::Initialize(&argc, argv); + gflags::AllowCommandLineReparsing(); + gflags::ParseCommandLineFlags(&argc, &argv, true); + int iterations = FLAGS_iterations; + int d = FLAGS_d; + int n = FLAGS_n; + auto benchs = ::perf_tests::sq_types(); + + for (auto& [bench_name, quantizer_type] : benchs) { + benchmark::RegisterBenchmark( + fmt::format("{}_{}d_{}n", bench_name, d, n).c_str(), + bench_reconstruction_error, + quantizer_type, + d, + n) + ->Iterations(iterations); + } + + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); +} diff --git a/faiss/perf_tests/bench_scalar_quantizer_decode.cpp b/faiss/perf_tests/bench_scalar_quantizer_decode.cpp new file mode 100644 index 0000000000..6cf3713a9b --- /dev/null +++ b/faiss/perf_tests/bench_scalar_quantizer_decode.cpp @@ -0,0 +1,77 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace faiss; +DEFINE_uint32(d, 128, "dimension"); +DEFINE_uint32(n, 2000, "dimension"); +DEFINE_uint32(iterations, 20, "iterations"); + +static void bench_decode( + benchmark::State& state, + ScalarQuantizer::QuantizerType type, + int d, + int n) { + state.SetLabel(faiss::get_compile_options()); + std::vector x(d * n); + + float_rand(x.data(), d * n, 12345); + + // make sure it's idempotent + ScalarQuantizer sq(d, type); + + omp_set_num_threads(1); + + sq.train(n, x.data()); + + size_t code_size = sq.code_size; + state.counters["code_size"] = sq.code_size; + + // encode + std::vector codes(code_size * n); + sq.compute_codes(x.data(), codes.data(), n); + std::vector x2(d * n); + + for (auto _ : state) { + // decode + sq.decode(codes.data(), x2.data(), n); + } +} + +int main(int argc, char** argv) { + benchmark::Initialize(&argc, argv); + gflags::AllowCommandLineReparsing(); + gflags::ParseCommandLineFlags(&argc, &argv, true); + int iterations = FLAGS_iterations; + int d = FLAGS_d; + int n = FLAGS_n; + auto benchs = ::perf_tests::sq_types(); + + for (auto& [bench_name, quantizer_type] : benchs) { + benchmark::RegisterBenchmark( + fmt::format("{}_{}d_{}n", bench_name, d, n).c_str(), + bench_decode, + quantizer_type, + d, + n) + ->Iterations(iterations); + } + + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); +} diff --git a/faiss/perf_tests/bench_scalar_quantizer_distance.cpp b/faiss/perf_tests/bench_scalar_quantizer_distance.cpp new file mode 100644 index 0000000000..b621ed39b3 --- /dev/null +++ b/faiss/perf_tests/bench_scalar_quantizer_distance.cpp @@ -0,0 +1,87 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace faiss; +DEFINE_uint32(d, 128, "dimension"); +DEFINE_uint32(n, 2000, "dimension"); +DEFINE_uint32(iterations, 20, "iterations"); + +static void bench_distance( + benchmark::State& state, + ScalarQuantizer::QuantizerType type, + int n, + int d) { + state.SetLabel(faiss::get_compile_options()); + std::vector x(d * n); + + float_rand(x.data(), d * n, 12345); + + // make sure it's idempotent + ScalarQuantizer sq(d, type); + + omp_set_num_threads(1); + + sq.train(n, x.data()); + + size_t code_size = sq.code_size; + state.counters["code_size"] = sq.code_size; + + // encode + std::vector codes(code_size * n); + sq.compute_codes(x.data(), codes.data(), n); + + state.SetLabel(faiss::get_compile_options()); + + std::unique_ptr dc( + sq.get_distance_computer()); + dc->codes = codes.data(); + dc->code_size = sq.code_size; + + for (auto _ : state) { + float sum_dis = 0; + for (int i = 0; i < n; i++) { + dc->set_query(&x[i * d]); + for (int j = 0; j < n; j++) { + benchmark::DoNotOptimize(sum_dis += (*dc)(j)); + } + } + } +} + +int main(int argc, char** argv) { + benchmark::Initialize(&argc, argv); + gflags::AllowCommandLineReparsing(); + gflags::ParseCommandLineFlags(&argc, &argv, true); + int iterations = FLAGS_iterations; + int d = FLAGS_d; + int n = FLAGS_n; + auto benchs = ::perf_tests::sq_types(); + + for (auto& [bench_name, quantizer_type] : benchs) { + benchmark::RegisterBenchmark( + fmt::format("{}_{}d_{}n", bench_name, d, n).c_str(), + bench_distance, + quantizer_type, + d, + n) + ->Iterations(iterations); + } + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); +} diff --git a/faiss/perf_tests/bench_scalar_quantizer_encode.cpp b/faiss/perf_tests/bench_scalar_quantizer_encode.cpp new file mode 100644 index 0000000000..3f745ada9a --- /dev/null +++ b/faiss/perf_tests/bench_scalar_quantizer_encode.cpp @@ -0,0 +1,71 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +using namespace faiss; +DEFINE_uint32(d, 128, "dimension"); +DEFINE_uint32(n, 2000, "dimension"); +DEFINE_uint32(iterations, 20, "iterations"); + +static void bench_encode( + benchmark::State& state, + ScalarQuantizer::QuantizerType type, + int d, + int n) { + state.SetLabel(faiss::get_compile_options()); + std::vector x(d * n); + + float_rand(x.data(), d * n, 12345); + ScalarQuantizer sq(d, type); + + omp_set_num_threads(1); + size_t code_size = sq.code_size; + + sq.train(n, x.data()); + state.counters["code_size"] = sq.code_size; + std::vector codes(code_size * n); + + for (auto _ : state) { + // encode + sq.compute_codes(x.data(), codes.data(), n); + } +} + +int main(int argc, char** argv) { + benchmark::Initialize(&argc, argv); + gflags::AllowCommandLineReparsing(); + gflags::ParseCommandLineFlags(&argc, &argv, true); + int iterations = FLAGS_iterations; + int d = FLAGS_d; + int n = FLAGS_n; + auto benchs = ::perf_tests::sq_types(); + + for (auto& [bench_name, quantizer_type] : benchs) { + benchmark::RegisterBenchmark( + fmt::format("{}_{}d_{}n", bench_name, d, n).c_str(), + bench_encode, + quantizer_type, + d, + n) + ->Iterations(iterations); + } + + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); +} diff --git a/faiss/perf_tests/utils.cpp b/faiss/perf_tests/utils.cpp new file mode 100644 index 0000000000..3e6c33220d --- /dev/null +++ b/faiss/perf_tests/utils.cpp @@ -0,0 +1,20 @@ +#include +namespace faiss::perf_tests { +std::map sq_types() { + static std::map + sq_types = { + {"QT_8bit", faiss::ScalarQuantizer::QT_8bit}, + {"QT_4bit", faiss::ScalarQuantizer::QT_4bit}, + {"QT_8bit_uniform", + faiss::ScalarQuantizer::QT_8bit_uniform}, + {"QT_4bit_uniform", + faiss::ScalarQuantizer::QT_4bit_uniform}, + {"QT_fp16", faiss::ScalarQuantizer::QT_fp16}, + {"QT_8bit_direct", faiss::ScalarQuantizer::QT_8bit_direct}, + {"QT_6bit", faiss::ScalarQuantizer::QT_6bit}, + {"QT_bf16", faiss::ScalarQuantizer::QT_bf16}, + {"QT_8bit_direct_signed", + faiss::ScalarQuantizer::QT_8bit_direct_signed}}; + return sq_types; +} +} // namespace faiss::perf_tests diff --git a/faiss/perf_tests/utils.h b/faiss/perf_tests/utils.h new file mode 100644 index 0000000000..e3065b9d4c --- /dev/null +++ b/faiss/perf_tests/utils.h @@ -0,0 +1,11 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#pragma once +#include +#include + +namespace faiss::perf_tests { + +std::map sq_types(); + +} // namespace faiss::perf_tests