Increase supported Y-level from 127 to 255

CMakeLists.txt

@@ -13,6 +13,7 @@ set(CMAKE_CROSSCOMPILING True)
 set(WITH_SANITIZER False)
 set(ZLIBNG_ENABLE_TESTS False)
 set(ZLIB_ENABLE_TESTS False)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 add_subdirectory(zlib-ng)
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20 -Wall")
@@ -21,18 +22,19 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0")
 message("${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
 
 file(GLOB_RECURSE SRC src/*)
-list(REMOVE_ITEM SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/benchmark_main.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/main.cpp")
+list(REMOVE_ITEM SRC "${CMAKE_CURRENT_SOURCE_DIR}/src/benchmark_main.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/main.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/baritone_pather.cpp")
 
 if (TESTING)
     add_executable(nether_pathfinder ${SRC} src/main.cpp)
     if(GBENCHMARK_FOUND)
-        add_executable(nether_pathfinder_benchmark ${SRC} src/benchmark_main.cpp)
-        target_link_libraries(nether_pathfinder_benchmark PkgConfig::GBENCHMARK)
+        #add_executable(nether_pathfinder_benchmark ${SRC} src/benchmark_main.cpp)
+        #target_link_libraries(nether_pathfinder_benchmark PkgConfig::GBENCHMARK)
+        #target_include_directories(nether_pathfinder_benchmark PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/zlib-ng)
     endif()
 else()
-    add_executable(baritone_pather ${SRC} src/baritone_pather.cpp)
-    target_include_directories(baritone_pather PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/zlib-ng)
-    target_link_libraries(baritone_pather PRIVATE zlibstatic)
+    #add_executable(baritone_pather ${SRC} src/baritone_pather.cpp)
+    #target_include_directories(baritone_pather PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/zlib-ng)
+    #target_link_libraries(baritone_pather PRIVATE zlibstatic)
 
     add_library(nether_pathfinder SHARED ${SRC})
 endif()

flake.nix

@@ -21,7 +21,8 @@
             # zig 0.10.1 = segfault
             # zig 0.9.1 only works on linux
             # zig 0.11.0 only works on macos
-            zig_0_9
+            # zig 0.13.0 fails for windows
+            zig_0_12
           ];
           buildInputs = with pkgs; [
             gbenchmark

java/multiplat_build.sh

@@ -19,7 +19,7 @@ function do_build {
 
     ninja -j `nproc`
 
-    cp libnether_pathfinder.* ../$output_path
+    cp libnether_pathfinder.so ../$output_path
     popd
     rm -rf build
 }
@@ -28,7 +28,7 @@ do_build $1 x86_64-linux-gnu libnether_pathfinder-x86_64.so
 do_build $1 aarch64-linux-gnu libnether_pathfinder-aarch64.so
 # zig 0.9.1 requires macos-gnu
 # zig 0.11.0 requires macos-none
-do_build $1 x86_64-macos-gnu libnether_pathfinder-x86_64.dylib
-do_build $1 aarch64-macos-gnu libnether_pathfinder-aarch64.dylib
+do_build $1 x86_64-macos-none libnether_pathfinder-x86_64.dylib
+do_build $1 aarch64-macos-none libnether_pathfinder-aarch64.dylib
 do_build $1 x86_64-windows-gnu nether_pathfinder-x86_64.dll
 do_build $1 aarch64-windows-gnu nether_pathfinder-aarch64.dll

java/src/main/java/dev/babbaj/pathfinder/NetherPathfinder.java

@@ -13,12 +13,20 @@
 
 public class NetherPathfinder {
 
-    // How the raytracer will treat chunks that aren't actually observed
+    // How the raytracer will treat chunks that aren't actually observed.
+    // When chunk generation is used the caller must synchronize calls that read from the cache with calls that may mutate the cache.
+    // No synchronization is done within the jni code
     public static int CACHE_MISS_GENERATE = 0;
     public static int CACHE_MISS_AIR = 1;
     public static int CACHE_MISS_SOLID = 2;
 
-    public static native long newContext(long seed, String baritoneCacheDirCanBeNull);
+    public static int DIMENSION_OVERWORLD = 0;
+    public static int DIMENSION_NETHER = 1;
+    public static int DIMENSION_END = 2;
+
+    // pass true to use the custom chunk allocator that will reduce memory usage and maybe be faster. false to just use new/delete
+    // this is only supported on systems with 4k pages
+    public static native long newContext(long seed, String baritoneCacheDirCanBeNull, int dimension, int maxHeight, boolean allocator);
     public static native void freeContext(long pointer);
 
     /*
@@ -29,19 +37,24 @@ private static int getIndex(int x, int y, int z)
     }
 
     chunkX and chunkZ are chunk coords, not block coords.
-    This is currently not thread safe
     */
     public static native void insertChunkData(long context, int chunkX, int chunkZ, boolean[] data);
 
-    public static native long getOrCreateChunk(long context, int x, int z);
+    public static native long allocateAndInsertChunk(long context, int x, int z);
+
+    // do not write to the chunk this returns
+    public static native long getChunkOrDefault(long context, int x, int z, boolean solid);
+
+    public static native long getChunk(long context, int x, int z);
 
-    public static native long getChunkPointer(long context, int x, int z);
+    // returns true if the chunk existed and the change was made
+    public static native boolean setChunkState(long context, int x, int z, boolean fromJava);
 
     public static native boolean hasChunkFromJava(long context, int x, int z);
 
     public static native void cullFarChunks(long context, int chunkX, int chunkZ, int maxDistanceBlocks);
 
-    public static native PathSegment pathFind(long context, int x1, int y1, int z1, int x2, int y2, int z2, boolean atLeastX4, boolean refine, int failTimeoutInMillis, boolean useAirIfChunkNotLoaded, double fakeChunkCost);
+    public static native PathSegment pathFind(long context, int x1, int y1, int z1, int x2, int y2, int z2, boolean atLeastX4, boolean refine, int failTimeoutInMillis, boolean defaultAirElseGenerate, double fakeChunkCost);
 
     private static native void raytrace0(long context, int fakeChunkMode, int inputs, double[] start, double[] end, boolean[] hitsOut, double[] hitPosOutCanBeNull);
 

java/src/main/java/dev/babbaj/pathfinder/Octree.java

@@ -24,7 +24,7 @@ public class Octree {
     public static final int SIZEOF_X4 = SIZEOF_X2 * 8;
     public static final int SIZEOF_X8 = SIZEOF_X4 * 8;
     public static final int SIZEOF_X16 = SIZEOF_X8 * 8;
-    public static final int SIZEOF_CHUNK = SIZEOF_X16 * 8;
+    public static final int SIZEOF_CHUNK = SIZEOF_X16 * 24;
 
     public static int x16Index(int y) {
         return y >> 4;
@@ -49,11 +49,11 @@ public static int bitIndex(int x, int y, int z) {
     private static long getX2Ptr(long chunk, int x, int y, int z) {
         //final int offset = (x16Index(y) * SIZEOF_X16) + (x8Index(x, y, z) * SIZEOF_X8) + (x4Index(x, y, z) * SIZEOF_X4) + (x2Index(x, y, z) * SIZEOF_X2);
 
-        // std::array<std::array<std::array<uint16_t, 64>, 8>, 8>
+        // std::array<std::array<std::array<uint16_t, 192>, 8>, 8>
         // auto x2Idx = X2_INDEX[x/2][z/2][y/2];
-        // std::array<uint16_t, 64> = 128 bytes
-        // std::array<std::array<uint16_t, 64>, 8> = 1024 bytes
-        final int indexOffset = (1024 * (x/2)) + (128 * (z/2)) + (2 * (y/2));
+        // std::array<uint16_t, 192> = 384 bytes
+        // std::array<std::array<uint16_t, 192>, 8> = 3072 bytes
+        final int indexOffset = (3072 * (x/2)) + (384 * (z/2)) + (2 * (y/2));
         // uint16_t to signed int
         final int offset = UNSAFE.getShort(X2_INDEX_PTR + indexOffset);
         return chunk + offset;
@@ -91,12 +91,4 @@ public static boolean getBlock(long pointer, int x, int y, int z) {
         final byte x2 = UNSAFE.getByte(x2Ptr);
         return ((x2 >> bit) & 1) != 0;
     }
-
-    public static void setIsFromJava(long pointer) {
-        UNSAFE.putByte(pointer + SIZEOF_CHUNK, (byte) 1);
-    }
-
-    public static boolean getIsFromJava(long pointer) {
-        return UNSAFE.getByte(pointer + SIZEOF_CHUNK) != 0;
-    }
 }

src/Allocator.cpp

@@ -0,0 +1,119 @@
+#include "Allocator.h"
+
+#ifdef _WIN32
+#include <windows.h>
+#include <atomic>
+#else
+#include <sys/mman.h>
+#include <unistd.h>
+#endif
+
+void* alignToPoolSize(void* base) {
+    return reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(base) + POOL_SIZE - 1) & ~(POOL_SIZE - 1));
+}
+
+size_t getPageSize() {
+#ifdef _WIN32
+    SYSTEM_INFO si;
+    GetSystemInfo(&si);
+    return si.dwPageSize;
+#else
+    return sysconf(_SC_PAGESIZE);
+#endif
+}
+
+#ifdef _WIN32
+// in case multiple threads want to change the global pool list at the same time but this does not need to be held to read it
+std::mutex pool_mutate_mutex;
+std::shared_ptr<std::vector<void*>> all_pools = std::make_shared<std::vector<void*>>();
+
+bool is_pool_pointer(uintptr_t address) {
+    auto pools = all_pools;
+    return std::find(pools->rbegin(), pools->rend(), (void*) (address & POOL_PTR_MASK)) != pools->rend();
+}
+
+LONG page_handler(PEXCEPTION_POINTERS ptr) {
+    PEXCEPTION_RECORD record = ptr->ExceptionRecord;
+    if (record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
+        uintptr_t fault_address = (uintptr_t) (record->ExceptionInformation[1]);
+        if (is_pool_pointer(fault_address)) {
+            void* page_aligned = (void*) (fault_address & ~(4096 - 1));
+            if (VirtualAlloc(page_aligned, 4096, MEM_COMMIT, PAGE_READWRITE)) {
+                return EXCEPTION_CONTINUE_EXECUTION;
+            }
+        }
+    }
+    return EXCEPTION_CONTINUE_SEARCH;
+}
+
+std::atomic_flag handler_added;
+void init_page_handler() {
+    if (!handler_added.test_and_set()) {
+        AddVectoredExceptionHandler(0, PVECTORED_EXCEPTION_HANDLER(page_handler));
+    }
+}
+
+void add_pool_global(void* pool) {
+    std::lock_guard lock{pool_mutate_mutex};
+    std::vector<void*> new_pools;
+    new_pools.reserve(all_pools->size() + 1);
+    std::copy(all_pools->begin(), all_pools->end(), std::back_inserter(new_pools));
+    new_pools.push_back(pool);
+    all_pools = std::make_shared<std::vector<void*>>(std::move(new_pools));
+}
+
+void remove_pools_global(std::span<void*> pools) {
+    std::lock_guard lock{pool_mutate_mutex};
+    auto& old_pools = *all_pools;
+    std::vector<void*> new_pools;
+    new_pools.reserve(old_pools.size());
+    for (auto p : old_pools) {
+        if (std::find(pools.begin(), pools.end(), p) == pools.end()) {
+            new_pools.push_back(p);
+        }
+    }
+    all_pools = std::make_shared<std::vector<void*>>(new_pools);
+}
+
+std::pair<void*, void*> alloc_pool() {
+    void* original = VirtualAlloc(nullptr, POOL_SIZE * 2, MEM_RESERVE, PAGE_NOACCESS);
+    if (!original) {
+        return {nullptr, nullptr};
+    }
+
+    void* aligned = alignToPoolSize(original);
+    return {aligned, original};
+}
+
+void free_pool(void* ptr) {
+    VirtualFree(ptr, 0, MEM_RELEASE);
+}
+
+void decommit(void* ptr, size_t len) {
+    // can probably just MEM_RESET
+    VirtualFree(ptr, len, MEM_DECOMMIT);
+}
+#else
+std::pair<void*, void*> alloc_pool() {
+    void* original = mmap(nullptr, POOL_SIZE * 2, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (original == MAP_FAILED) {
+        return {nullptr, nullptr};
+    }
+    void* aligned = alignToPoolSize(original);
+    return {aligned, original};
+}
+
+void free_pool(void* ptr) {
+    munmap(ptr, POOL_SIZE * 2);
+}
+
+void decommit(void* ptr, size_t len) {
+    madvise(ptr, len, MADV_DONTNEED);
+    //mmap(ptr, len, PROT_NONE, MAP_FIXED, -1, 0);
+}
+
+// not needed on linux
+void add_pool_global(void*) {}
+void remove_pools_global(std::span<void*>) {}
+void init_page_handler() {}
+#endif