Add AVX512 support to k-nn FAISS

.github/workflows/CI.yml

@@ -73,13 +73,17 @@ jobs:
         # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip.
         run: |
           chown -R 1000:1000 `pwd`
-          if lscpu  | grep -i avx2
+          if lscpu  | grep -i avx512f | grep -i avx512cd | grep -i avx512vl | grep -i avx512dq | grep -i avx512bw          
           then
-            echo "avx2 available on system"
+            echo "avx512 available on system"
             su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dnproc.count=`nproc`"
+          elif lscpu  | grep -i avx2
+          then
+            echo "avx2 available on system"
+            su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dnproc.count=`nproc` -Davx512.enabled=false"
           else
-            echo "avx2 not available on system"
-            su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dsimd.enabled=false -Dnproc.count=`nproc`"
+            echo "avx512 and avx2 not available on system"
+            su `id -un 1000` -c "whoami && java -version && ./gradlew build -Davx2.enabled=false -Davx512.enabled=false -Dnproc.count=`nproc`"
           fi  
 
 
@@ -126,7 +130,7 @@ jobs:
               ./gradlew build -Dnproc.count=3
           else
               echo "avx2 not available on system"
-              ./gradlew build -Dsimd.enabled=false -Dnproc.count=3
+              ./gradlew build -Davx2.enabled=false -Davx512.enabled=false -Dnproc.count=3
           fi
 
   Build-k-NN-Windows:
@@ -187,4 +191,4 @@ jobs:
       # TODO: Detect processor count and set the value of nproc.count
       - name: Run build
         run: |
-          ./gradlew.bat build -D'simd.enabled=false' -D'nproc.count=4'
+          ./gradlew.bat build -D'avx2.enabled=false' -D'avx512.enabled=false' -D'nproc.count=4'

.github/workflows/test_security.yml

@@ -70,4 +70,4 @@ jobs:
         # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip.
         run: |
           chown -R 1000:1000 `pwd`
-          su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true -Dsimd.enabled=true -Dnproc.count=`nproc`"
+          su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true -Davx2.enabled=true -Davx512.enabled=true -Dnproc.count=`nproc`"

CHANGELOG.md

@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.17...2.x)
 ### Features
+* Add AVX512 support to k-NN for FAISS library [#2069](https://github.com/opensearch-project/k-NN/pull/2069)
 ### Enhancements
 ### Bug Fixes
 ### Infrastructure

DEVELOPER_GUIDE.md

@@ -283,19 +283,22 @@ make -j 4
 
 ### Enable SIMD Optimization
 SIMD(Single Instruction/Multiple Data) Optimization is enabled by default on Linux and Mac which boosts the performance
-by enabling `AVX2` on `x86 architecture` and `NEON` on `ARM64 architecture` while building the Faiss library. But to enable SIMD, the underlying processor
-should support this (AVX2 or NEON). It can be disabled by setting the parameter `simd.enabled` to `false`. As of now, it is not supported on Windows OS.
+by enabling `AVX2` and `AVX512` on `x86 architecture` and `NEON` on `ARM64 architecture` where applicable while building the Faiss library. But to enable SIMD,
+the underlying processor should support these capabilities (AVX512, AVX2 or NEON). It can be disabled by setting the parameter `avx2.enabled` to `false` and
+`avx512.enabled` to `false`. If your processor supports `AVX512` or `AVX2`, they can be set by enabling the setting . By default, these values are enabled on 
+OpenSearch. Some exceptions: As of now, SIMD support is not supported on Windows OS, and AVX512 is not present on MAC systems due to hardware not supporting the
+feature. 
 
 ```
 # While building OpenSearch k-NN
-./gradlew build -Dsimd.enabled=true
+./gradlew build -Davx2.enabled=true -Davx512.enabled=true
 
 # While running OpenSearch k-NN
-./gradlew run -Dsimd.enabled=true
+./gradlew run -Davx2.enabled=true -Davx512.enabled=true
 
 # While building the JNI libraries
 cd jni
-cmake . -DSIMD_ENABLED=true
+cmake . -DAVX2_ENABLED=true -DAVX512_ENABLED=true
 ```
 
 ## Run OpenSearch k-NN

bin/main/META-INF/services/org.apache.lucene.codecs.Codec

@@ -0,0 +1,10 @@
+org.opensearch.knn.index.codec.KNN80Codec.KNN80Codec
+org.opensearch.knn.index.codec.KNN84Codec.KNN84Codec
+org.opensearch.knn.index.codec.KNN86Codec.KNN86Codec
+org.opensearch.knn.index.codec.KNN87Codec.KNN87Codec
+org.opensearch.knn.index.codec.KNN910Codec.KNN910Codec
+org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec
+org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec
+org.opensearch.knn.index.codec.KNN950Codec.KNN950Codec
+org.opensearch.knn.index.codec.KNN990Codec.KNN990Codec
+org.opensearch.knn.index.codec.KNN990Codec.UnitTestCodec

bin/main/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat

@@ -0,0 +1,12 @@
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+
+org.opensearch.knn.index.codec.KNN990Codec.NativeEngines990KnnVectorsFormat

bin/main/META-INF/services/org.opensearch.painless.spi.PainlessExtension

@@ -0,0 +1,4 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+org.opensearch.knn.plugin.script.KNNAllowlistExtension

bin/main/mappings/model-index.json

@@ -0,0 +1,46 @@
+{
+  "properties": {
+    "model_id": {
+      "type": "keyword"
+    },
+    "engine": {
+      "type": "keyword"
+    },
+    "space_type": {
+      "type": "keyword"
+    },
+    "dimension": {
+      "type": "integer"
+    },
+    "state": {
+      "type": "keyword"
+    },
+    "timestamp": {
+      "type": "date"
+    },
+    "description": {
+      "type": "keyword"
+    },
+    "error": {
+      "type": "keyword"
+    },
+    "model_blob": {
+      "type": "binary"
+    },
+    "node_assignment": {
+      "type": "keyword"
+    },
+    "method_component_context": {
+      "type": "keyword"
+    },
+    "mode": {
+      "type": "keyword"
+    },
+    "compression_level": {
+      "type": "keyword"
+    },
+    "model_version": {
+      "type": "keyword"
+    }
+  }
+}