Add a Faiss codec for KNN searches

.github/workflows/run-special-checks-sandbox.yml

@@ -0,0 +1,53 @@
+name: "Run special checks: module lucene/sandbox"
+
+on:
+  workflow_dispatch:
+
+  pull_request:
+    branches:
+      - '*'
+
+  push:
+    branches:
+      - 'main'
+      - 'branch_10x'
+
+jobs:
+  faiss-tests:
+    name: tests for the Faiss codec (v${{ matrix.faiss-version }} with JDK ${{ matrix.java }} on ${{ matrix.os }})
+    timeout-minutes: 15
+
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        java: [ '24' ]
+        faiss-version: [ '1.11.0' ]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Install Faiss
+        uses: mamba-org/setup-micromamba@b09ef9b599704322748535812ca03efb2625677b #v2.0.5
+        id: setup
+        with:
+          environment-name: faiss-env
+          # TODO: Use only conda-forge if possible, see https://github.com/conda-forge/faiss-split-feedstock/pull/88
+          create-args: >-
+            -c pytorch
+            -c conda-forge
+            faiss-cpu=${{ matrix.faiss-version }}
+
+      - name: Checkout Lucene
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Prepare Lucene workspace
+        uses: ./.github/actions/prepare-for-build
+
+      - name: Run tests for Faiss codec
+        env:
+          LD_LIBRARY_PATH: ${{ steps.setup.outputs.environment-path }}/lib
+        run: ./gradlew -p lucene/sandbox -Dtests.faiss.run=true test --tests "org.apache.lucene.sandbox.codecs.faiss.*"
+
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}

gradle/testing/defaults-tests.gradle

@@ -145,6 +145,7 @@ allprojects {
               ':lucene:core',
               ':lucene:codecs',
               ":lucene:distribution.tests",
+              ':lucene:sandbox',
               ":lucene:test-framework"
       ] ? 'ALL-UNNAMED' : 'org.apache.lucene.core')
 

gradle/testing/randomization.gradle

@@ -116,6 +116,7 @@ allprojects {
             description: "Forces use of integer vectors even when slow."],
           [propName: 'tests.defaultvectorization', value: false,
             description: "Uses defaults for running tests with correct JVM settings to test Panama vectorization (tests.jvmargs, tests.vectorsize, tests.forceintegervectors)."],
+          [propName: 'tests.faiss.run', value: false, description: "Explicitly run tests for the Faiss codec."],
       ]
     }
   }

gradle/testing/randomization/policies/tests.policy

@@ -80,6 +80,9 @@ grant {
   permission java.io.FilePermission "${hunspell.corpora}${/}-", "read";
   permission java.io.FilePermission "${hunspell.dictionaries}", "read";
   permission java.io.FilePermission "${hunspell.dictionaries}${/}-", "read";
+
+  // allow loading specific library for the Faiss codec
+  permission java.lang.RuntimePermission "loadLibrary.faiss_c";
 };
 
 // Permissions for jacoco code coverage

lucene/CHANGES.txt

@@ -30,6 +30,8 @@ New Features
 ---------------------
 * GITHUB#14097: Binary partitioning merge policy over float-valued vector field. (Mike Sokolov)
 
+* GITHUB#14178: Add a Faiss-based vector format in the sandbox module. (Kaival Parikh)
+
 Improvements
 ---------------------
 

lucene/sandbox/src/java/module-info.java

@@ -22,6 +22,7 @@
   requires org.apache.lucene.facet;
 
   exports org.apache.lucene.payloads;
+  exports org.apache.lucene.sandbox.codecs.faiss;
   exports org.apache.lucene.sandbox.codecs.idversion;
   exports org.apache.lucene.sandbox.codecs.quantization;
   exports org.apache.lucene.sandbox.document;
@@ -39,4 +40,6 @@
 
   provides org.apache.lucene.codecs.PostingsFormat with
       org.apache.lucene.sandbox.codecs.idversion.IDVersionPostingsFormat;
+  provides org.apache.lucene.codecs.KnnVectorsFormat with
+      org.apache.lucene.sandbox.codecs.faiss.FaissKnnVectorsFormat;
 }

lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsFormat.java

@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.sandbox.codecs.faiss;
+
+import java.io.IOException;
+import java.util.Locale;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.KnnVectorsReader;
+import org.apache.lucene.codecs.KnnVectorsWriter;
+import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;
+import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+
+/**
+ * A format which uses <a href="https://github.com/facebookresearch/faiss">Faiss</a> to create and
+ * search vector indexes, using {@link LibFaissC} to interact with the native library.
+ *
+ * <p>TODO: There is no guarantee of backwards compatibility!
+ *
+ * <p>A separate Faiss index is created per-segment, and uses the following files:
+ *
+ * <ul>
+ *   <li><code>.faissm</code> (metadata file): stores field number, offset and length of actual
+ *       Faiss index in data file.
+ *   <li><code>.faissd</code> (data file): stores concatenated Faiss indexes for all fields.
+ *   <li>All files required by {@link Lucene99FlatVectorsFormat} for storing raw vectors.
+ * </ul>
+ *
+ * <p>Note: Set the {@code $OMP_NUM_THREADS} environment variable to control <a
+ * href="https://github.com/facebookresearch/faiss/wiki/Threads-and-asynchronous-calls">internal
+ * threading</a>.
+ *
+ * @lucene.experimental
+ */
+public final class FaissKnnVectorsFormat extends KnnVectorsFormat {
+  public static final String NAME = FaissKnnVectorsFormat.class.getSimpleName();
+  static final int VERSION_START = 0;
+  static final int VERSION_CURRENT = VERSION_START;
+  static final String META_CODEC_NAME = NAME + "Meta";
+  static final String DATA_CODEC_NAME = NAME + "Data";
+  static final String META_EXTENSION = "faissm";
+  static final String DATA_EXTENSION = "faissd";
+
+  private final String description;
+  private final String indexParams;
+  private final FlatVectorsFormat rawVectorsFormat;
+
+  public FaissKnnVectorsFormat() {
+    this("IDMap,HNSW32", "efConstruction=200");
+  }
+
+  public FaissKnnVectorsFormat(String description, String indexParams) {
+    super(NAME);
+    this.description = description;
+    this.indexParams = indexParams;
+    this.rawVectorsFormat =
+        new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer());
+  }
+
+  @Override
+  public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+    return new FaissKnnVectorsWriter(
+        description, indexParams, state, rawVectorsFormat.fieldsWriter(state));
+  }
+
+  @Override
+  public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {
+    return new FaissKnnVectorsReader(state, rawVectorsFormat.fieldsReader(state));
+  }
+
+  @Override
+  public int getMaxDimensions(String fieldName) {
+    return DEFAULT_MAX_DIMENSIONS;
+  }
+
+  @Override
+  public String toString() {
+    return String.format(
+        Locale.ROOT, "%s(description=%s indexParams=%s)", NAME, description, indexParams);
+  }
+}