Skip to content

Fix the endianness issue in AIX while running the benchmark. #3345

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
KamathForAIX wants to merge 16 commits into from
Closed

Fix the endianness issue in AIX while running the benchmark. #3345

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
2fa5993
Fix the endianness issue in AIX while running the benchmark.
KamathForAIX Apr 4, 2024
ca42db4
Merge branch 'main' into gh-3330
KamathForAIX Apr 4, 2024
a86c2fd
Attempt to Fix Windows Build break
KamathForAIX Apr 4, 2024
e0412c5
Change or to and
KamathForAIX Apr 4, 2024
0561f7d
Fix format issue
KamathForAIX Apr 4, 2024
aad8301
Move macros to faiss/impl/platform_macros.h to simplify logic
KamathForAIX Apr 8, 2024
be1a750
Fix Format issues
KamathForAIX Apr 8, 2024
db1a5eb
Merge test case fix due to endianness in AIX
KamathForAIX Apr 8, 2024
82eac21
fix Format issues
KamathForAIX Apr 8, 2024
391f931
fix Format issues
KamathForAIX Apr 8, 2024
553eba6
Add Swap2Bytes definition for little endian in Level2-inl.h
KamathForAIX Apr 8, 2024
be9f6ef
Make FAISS_BIG_ENDIAN definition only in platform-macros.h
KamathForAIX Apr 11, 2024
62caf79
Make FAISS_BIG_ENDIAN to not be defined as 0 or 1
KamathForAIX Apr 23, 2024
dbfd236
Fix Format issue
KamathForAIX Apr 23, 2024
1809905
Fix Format issue
KamathForAIX Apr 23, 2024
d3bc22e
Move isBigEndian to faiss::cppcontrib namespace
KamathForAIX Apr 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions contrib/vecs_io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import sys
import numpy as np

"""
Expand All @@ -13,6 +14,8 @@

def ivecs_read(fname):
a = np.fromfile(fname, dtype='int32')
if sys.byteorder == 'big':
a.byteswap(inplace=True)
d = a[0]
return a.reshape(-1, d + 1)[:, 1:].copy()

Expand Down
12 changes: 12 additions & 0 deletions faiss/impl/pq4_fast_scan.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,14 @@ void pq4_pack_codes(
return;
}
memset(blocks, 0, nb * nsq / 2);
#if !defined(_MSC_VER) && \
(defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
const uint8_t perm0[16] = {
8, 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7};
#else
const uint8_t perm0[16] = {
0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15};
#endif

uint8_t* codes2 = blocks;
for (size_t i0 = 0; i0 < nb; i0 += bbs) {
Expand Down Expand Up @@ -93,8 +99,14 @@ void pq4_pack_codes_range(
size_t bbs,
size_t nsq,
uint8_t* blocks) {
#if !defined(_MSC_VER) && \
(defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
const uint8_t perm0[16] = {
8, 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7};
#else
const uint8_t perm0[16] = {
0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15};
#endif

// range of affected blocks
size_t block0 = i0 / bbs;
Expand Down
13 changes: 13 additions & 0 deletions faiss/python/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,20 @@ else()
find_package(faiss REQUIRED)
endif()

if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
swig_add_library(swigfaiss
TYPE MODULE
LANGUAGE python
SOURCES swigfaiss.swig
)
else ()
swig_add_library(swigfaiss
TYPE SHARED
LANGUAGE python
SOURCES swigfaiss.swig
)
endif()

set_property(TARGET swigfaiss PROPERTY SWIG_COMPILE_OPTIONS -doxygen)

set_property(SOURCE swigfaiss_avx2.swig
Expand Down Expand Up @@ -160,6 +169,10 @@ set_property(TARGET faiss_python_callbacks
PROPERTY POSITION_INDEPENDENT_CODE ON
)

if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
target_link_libraries(faiss_python_callbacks PRIVATE faiss)
endif()

# Hack so that python_callbacks.h can be included as
# `#include <faiss/python/python_callbacks.h>`.
target_include_directories(faiss_python_callbacks PRIVATE ${PROJECT_SOURCE_DIR}/../..)
Expand Down