Merge branch 'main' into histogram-neon

Author: wx257osn2

.circleci/config.yml

@@ -45,29 +45,31 @@ jobs:
       - checkout
       - run:
           name: Build faiss library
+          no_output_timeout: 30m
           command: |
             cmake -B build -DBUILD_TESTING=ON -DFAISS_ENABLE_GPU=OFF \
                   -DFAISS_OPT_LEVEL=<< parameters.opt_level >> \
                   -DFAISS_ENABLE_C_API=ON \
                   -DCMAKE_BUILD_TYPE=Release -DBLA_VENDOR=Intel10_64_dyn .
-            make -k -C build -j3 faiss
+            make -k -C build -j$(nproc) faiss
       - when:
           condition:
             equal: [ "avx2", << parameters.opt_level >> ]
           steps:
             - run:
                 name: Build faiss_avx2 library
-                command: make -k -C build -j3 faiss_avx2 swigfaiss_avx2
+                no_output_timeout: 30m
+                command: make -k -C build -j$(nproc) faiss_avx2 swigfaiss_avx2
       - run:
           name: Test faiss library
           command: |
-            make -C build -j3 faiss_test
+            make -C build -j$(nproc) faiss_test
             export GTEST_OUTPUT="xml:$(realpath .)/test-results/googletest/"
             make -C build test
       - run:
           name: Build python extension
           command: |
-            make -C build -j3 swigfaiss
+            make -C build -j$(nproc) swigfaiss
             cd build/faiss/python
             python3 setup.py build
       - run:
@@ -87,10 +89,12 @@ jobs:
   build_linux_conda:
     docker:
       - image: continuumio/miniconda3
+    resource_class: medium+
     steps:
       - checkout
       - run:
           name: Conda build
+          no_output_timeout: 30m
           command: |
             conda install -y -q conda-build
             cd conda
@@ -166,17 +170,17 @@ jobs:
                   -DFAISS_ENABLE_C_API=ON \
                   -DCMAKE_BUILD_TYPE=Release \
                   -DPython_EXECUTABLE=$PYTHON .
-            make -k -C build -j3 faiss
+            make -k -C build -j$(nproc) faiss
       - run:
           name: Test faiss library
           command: |
-            make -C build -j3 faiss_test
+            make -C build -j$(nproc) faiss_test
             export GTEST_OUTPUT="xml:$(realpath .)/test-results/googletest/"
             make -C build test
       - run:
           name: Build python extension
           command: |
-            make -C build -j3 swigfaiss
+            make -C build -j$(nproc) swigfaiss
             cd build/faiss/python
             $PYTHON setup.py build
       - run:
@@ -217,6 +221,7 @@ jobs:
         default: main
     docker:
       - image: continuumio/miniconda3
+    resource_class: medium+
     steps:
       - checkout
       - run:
@@ -226,6 +231,7 @@ jobs:
             conda config --set anaconda_upload yes
       - run:
           name: Build packages
+          no_output_timeout: 30m
           environment:
             PACKAGE_TYPE: <<parameters.label>>
           command: |

Dockerfile

@@ -13,7 +13,7 @@ ENV LD_PRELOAD /usr/lib64/libgomp.so.1:/opt/intel/mkl/lib/intel64/libmkl_def.so:
 # Install necessary build tools
 RUN yum install -y gcc-c++ make swig3
 
-# Install necesary headers/libs
+# Install necessary headers/libs
 RUN yum install -y python-devel numpy
 
 COPY . /opt/faiss

Doxyfile

@@ -786,7 +786,7 @@ EXCLUDE_SYMLINKS       = NO
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories for example use the pattern */test/*
 
-EXCLUDE_PATTERNS       = */impl/*
+EXCLUDE_PATTERNS       = 
 
 # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
 # (namespaces, classes, functions, etc.) that should be excluded from the

README.md

@@ -74,6 +74,8 @@ For public discussion of Faiss or for questions, there is a Facebook group at ht
 We monitor the [issues page](http://github.com/facebookresearch/faiss/issues) of the repository.
 You can report bugs, ask questions, etc.
 
-## License
+## Legal
 
-Faiss is MIT-licensed.
+Faiss is MIT-licensed, refer to the [LICENSE file](https://github.com/facebookresearch/faiss/blob/main/LICENSE) in the top level directory.
+
+Copyright © Meta Platforms, Inc. See the [Terms of Use](https://opensource.fb.com/legal/terms/) and [Privacy Policy](https://opensource.fb.com/legal/privacy/) for this project.

benchs/bench_all_ivf/bench_all_ivf.py

@@ -3,15 +3,20 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
+import argparse
 import os
 import sys
 import time
-import pdb
-import numpy as np
+
 import faiss
-import argparse
-import datasets
-from datasets import sanitize
+import numpy as np
+
+try:
+    import datasets_fb as datasets
+except ModuleNotFoundError:
+    import datasets_oss as datasets
+
+sanitize = datasets.sanitize
 
 
 ######################################################
@@ -262,8 +267,7 @@ def apply_AQ_options(index, args):
         print("Getting centroids from", args.get_centroids_from)
         src_index = faiss.read_index(args.get_centroids_from)
         src_quant = faiss.downcast_index(src_index.quantizer)
-        centroids = faiss.vector_to_array(src_quant.xb)
-        centroids = centroids.reshape(-1, d)
+        centroids = src_quant.reconstruct_n()
         print("  centroid table shape", centroids.shape)
 
         if isinstance(vec_transform, faiss.VectorTransform):
@@ -333,7 +337,7 @@ def apply_AQ_options(index, args):
 
 xq = sanitize(ds.get_queries())
 gt = ds.get_groundtruth(k=args.k)
-assert gt.shape[1] == args.k, pdb.set_trace()
+assert gt.shape[1] == args.k
 
 if args.searchthreads != -1:
     print("Setting nb of threads to", args.searchthreads)

benchs/bench_all_ivf/cmp_with_scann.py

@@ -75,28 +75,37 @@ def aa(*args, **kwargs):
     k = args.k
     nrun = args.nrun
 
-    if args.lib == "faiss":
+    if not os.path.exists(cache_dir + "xb.npy"):
         # prepare cache
-        import faiss
         from datasets import load_dataset
-
         ds = load_dataset(args.db, download=args.download)
         print(ds)
-        if not os.path.exists(cache_dir + "xb.npy"):
-            # store for SCANN
-            os.system(f"rm -rf {cache_dir}; mkdir -p {cache_dir}")
-            tosave = dict(
-                # xt = ds.get_train(10),
-                xb = ds.get_database(),
-                xq = ds.get_queries(),
-                gt = ds.get_groundtruth()
-            )
-            for name, v in tosave.items():
-                fname = cache_dir + "/" + name + ".npy"
-                print("save", fname)
-                np.save(fname, v)
-
-            open(cache_dir + "metric", "w").write(ds.metric)
+        # store for SCANN
+        os.system(f"rm -rf {cache_dir}; mkdir -p {cache_dir}")
+        tosave = dict(
+            xb = ds.get_database(),
+            xq = ds.get_queries(),
+            gt = ds.get_groundtruth()
+        )
+        for name, v in tosave.items():
+            fname = cache_dir + "/" + name + ".npy"
+            print("save", fname)
+            np.save(fname, v)
+
+        open(cache_dir + "metric", "w").write(ds.metric)
+        
+    dataset = {}
+    for kn in "xb xq gt".split():
+        fname = cache_dir + "/" + kn + ".npy"
+        print("load", fname)
+        dataset[kn] = np.load(fname)
+    xb = dataset["xb"]
+    xq = dataset["xq"]
+    gt = dataset["gt"] 
+    distance_measure = open(cache_dir + "metric").read()
+    
+    if args.lib == "faiss":
+        import faiss
 
         name1_to_metric = {
             "IP": faiss.METRIC_INNER_PRODUCT,
@@ -106,14 +115,10 @@ def aa(*args, **kwargs):
         index_fname = cache_dir + "index.faiss"
         if not os.path.exists(index_fname):
             index = faiss_make_index(
-                ds.get_database(), name1_to_metric[ds.metric], index_fname)
+                xb, name1_to_metric[distance_measure], index_fname)
         else:
             index = faiss.read_index(index_fname)
 
-        xb = ds.get_database()
-        xq = ds.get_queries()
-        gt = ds.get_groundtruth()
-
         faiss_eval_search(
                 index, xq, xb, nprobe_tab, pre_reorder_k_tab, k, gt,
                 nrun, args.measure
@@ -122,32 +127,22 @@ def aa(*args, **kwargs):
     if args.lib == "scann":
         from scann.scann_ops.py import scann_ops_pybind
 
-        dataset = {}
-        for kn in "xb xq gt".split():
-            fname = cache_dir + "/" + kn + ".npy"
-            print("load", fname)
-            dataset[kn] = np.load(fname)
         name1_to_name2 = {
             "IP": "dot_product",
             "L2": "squared_l2"
         }
-        distance_measure = name1_to_name2[open(cache_dir + "metric").read()]
-
-        xb = dataset["xb"]
-        xq = dataset["xq"]
-        gt = dataset["gt"]
 
         scann_dir = cache_dir + "/scann1.1.1_serialized"
         if os.path.exists(scann_dir + "/scann_config.pb"):
             searcher = scann_ops_pybind.load_searcher(scann_dir)
         else:
-            searcher = scann_make_index(xb, distance_measure, scann_dir, 0)
+            searcher = scann_make_index(xb, name1_to_name2[distance_measure], scann_dir, 0)
 
         scann_dir = cache_dir + "/scann1.1.1_serialized_reorder"
         if os.path.exists(scann_dir + "/scann_config.pb"):
             searcher_reo = scann_ops_pybind.load_searcher(scann_dir)
         else:
-            searcher_reo = scann_make_index(xb, distance_measure, scann_dir, 100)
+            searcher_reo = scann_make_index(xb, name1_to_name2[distance_measure], scann_dir, 100)
 
         scann_eval_search(
             searcher, searcher_reo,
@@ -256,7 +251,6 @@ def faiss_make_index(xb, metric_type, fname):
     #    index.by_residual = False
 
     print("train")
-    # index.train(ds.get_train())
     index.train(xb[:250000])
     print("add")
     index.add(xb)

benchs/bench_all_ivf/datasets.py renamed to benchs/bench_all_ivf/datasets_oss.py

@@ -81,7 +81,6 @@ def load_dataset(dataset='deep1M', compute_gt=False, download=False):
             f"{centdir}/clustering.dbdeep1M.IVF{ncent}.faissindex"
         )
 
-
     elif dataset.startswith("deep"):
 
         szsuf = dataset[4:]

benchs/bench_big_batch_ivf.py

@@ -0,0 +1,109 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+import time
+
+import faiss
+
+import numpy as np
+
+from faiss.contrib.datasets import SyntheticDataset
+from faiss.contrib.ivf_tools import big_batch_search
+
+parser = argparse.ArgumentParser()
+
+
+def aa(*args, **kwargs):
+    group.add_argument(*args, **kwargs)
+
+
+group = parser.add_argument_group('dataset options')
+aa('--dim', type=int, default=64)
+aa('--size', default="S")
+
+group = parser.add_argument_group('index options')
+aa('--nlist', type=int, default=100)
+aa('--factory_string', default="", help="overrides nlist")
+aa('--k', type=int, default=10)
+aa('--nprobe', type=int, default=5)
+aa('--nt', type=int, default=-1, help="nb search threads")
+aa('--method', default="pairwise_distances", help="")
+
+args = parser.parse_args()
+print("args:", args)
+
+if args.size == "S":
+    ds = SyntheticDataset(32, 2000, 4000, 1000)
+elif args.size == "M":
+    ds = SyntheticDataset(32, 20000, 40000, 10000)
+elif args.size == "L":
+    ds = SyntheticDataset(32, 200000, 400000, 100000)
+else:
+    raise RuntimeError(f"dataset size {args.size} not supported")
+
+nlist = args.nlist
+nprobe = args.nprobe
+k = args.k
+
+
+def tic(name):
+    global tictoc
+    tictoc = (name, time.time())
+    print(name, end="\r", flush=True)
+
+
+def toc():
+    global tictoc
+    name, t0 = tictoc
+    dt = time.time() - t0
+    print(f"{name}: {dt:.3f} s")
+    return dt
+
+
+print(f"dataset {ds}, {nlist=:} {nprobe=:} {k=:}")
+
+if args.factory_string == "":
+    factory_string = f"IVF{nlist},Flat"
+else:
+    factory_string = args.factory_string
+
+print(f"instantiate {factory_string}")
+index = faiss.index_factory(ds.d, factory_string)
+
+if args.factory_string != "":
+    nlist = index.nlist
+
+print("nlist", nlist)
+
+tic("train")
+index.train(ds.get_train())
+toc()
+
+tic("add")
+index.add(ds.get_database())
+toc()
+
+if args.nt != -1:
+    print("setting nb of threads to", args.nt)
+    faiss.omp_set_num_threads(args.nt)
+
+tic("reference search")
+index.nprobe
+index.nprobe = nprobe
+Dref, Iref = index.search(ds.get_queries(), k)
+t_ref = toc()
+
+tic("block search")
+Dnew, Inew = big_batch_search(
+    index, ds.get_queries(),
+    k, method=args.method, verbose=10
+)
+t_tot = toc()
+
+assert (Inew != Iref).sum() / Iref.size < 1e-4
+np.testing.assert_almost_equal(Dnew, Dref, decimal=4)
+
+print(f"total block search time {t_tot:.3f} s, speedup {t_ref / t_tot:.3f}x")