diff --git a/Make.inc b/Make.inc index 4382393ab21dd..2a9b4301de6ce 100644 --- a/Make.inc +++ b/Make.inc @@ -378,8 +378,10 @@ ifeq ($(XC_HOST),) CROSS_COMPILE:= # delayed expansion of $(CC), since it won't be computed until later HOSTCC = $(CC) +HOSTCXX = $(CXX) else HOSTCC ?= gcc +HOSTCXX ?= g++ OPENBLAS_DYNAMIC_ARCH := 1 override CROSS_COMPILE:=$(XC_HOST)- ifneq (,$(findstring mingw,$(XC_HOST))) diff --git a/deps/NATIVE.cmake b/deps/NATIVE.cmake deleted file mode 100644 index 026ee4de66934..0000000000000 --- a/deps/NATIVE.cmake +++ /dev/null @@ -1,4 +0,0 @@ -# native toolchain file to fix llvm cross-compilation finickiness -# ref http://lists.llvm.org/pipermail/llvm-dev/2016-February/095366.html -set(CMAKE_C_COMPILER cc) -set(CMAKE_CXX_COMPILER c++) diff --git a/deps/llvm.mk b/deps/llvm.mk index a0f4a4f3fed54..0f6bfe4e55438 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -129,19 +129,14 @@ LLVM_CMAKE += -DLLVM_BINDINGS_LIST="" -DLLVM_INCLUDE_DOCS=Off -DLLVM_ENABLE_TERM ifeq ($(LLVM_ASSERTIONS), 1) LLVM_CMAKE += -DLLVM_ENABLE_ASSERTIONS:BOOL=ON endif # LLVM_ASSERTIONS -ifeq ($(LLVM_DEBUG), 1) -ifeq ($(OS), WINNT) -LLVM_CXXFLAGS += -Wa,-mbig-obj -endif # OS == WINNT -endif # LLVM_DEBUG ifeq ($(OS), WINNT) LLVM_CPPFLAGS += -D__USING_SJLJ_EXCEPTIONS__ -D__CRT__NO_INLINE -ifneq ($(BUILD_OS),WINNT) -LLVM_CMAKE += -DCROSS_TOOLCHAIN_FLAGS_NATIVE=-DCMAKE_TOOLCHAIN_FILE=$(SRCDIR)/NATIVE.cmake -endif # BUILD_OS != WINNT endif # OS == WINNT +ifneq ($(HOSTCC),$(CC)) +LLVM_CMAKE += -DCROSS_TOOLCHAIN_FLAGS_NATIVE="-DCMAKE_C_COMPILER=$$(which $(HOSTCC));-DCMAKE_CXX_COMPILER=$$(which $(HOSTCXX))" +endif ifeq ($(OS), emscripten) -LLVM_CMAKE += -DCMAKE_TOOLCHAIN_FILE=$(EMSCRIPTEN)/cmake/Modules/Platform/Emscripten.cmake -DCROSS_TOOLCHAIN_FLAGS_NATIVE=-DCMAKE_TOOLCHAIN_FILE=$(SRCDIR)/NATIVE.cmake -DLLVM_INCLUDE_TOOLS=OFF -DLLVM_BUILD_TOOLS=OFF -DLLVM_INCLUDE_TESTS=OFF -DLLVM_ENABLE_THREADS=OFF -DLLVM_BUILD_UTILS=OFF +LLVM_CMAKE += -DCMAKE_TOOLCHAIN_FILE=$(EMSCRIPTEN)/cmake/Modules/Platform/Emscripten.cmake -DLLVM_INCLUDE_TOOLS=OFF -DLLVM_BUILD_TOOLS=OFF -DLLVM_INCLUDE_TESTS=OFF -DLLVM_ENABLE_THREADS=OFF -DLLVM_BUILD_UTILS=OFF endif # OS == emscripten ifeq ($(USE_LLVM_SHLIB),1) # NOTE: we could also --disable-static here (on the condition we link tools diff --git a/deps/tools/common.mk b/deps/tools/common.mk index 28f87cfc91e1a..35effcb75bd96 100644 --- a/deps/tools/common.mk +++ b/deps/tools/common.mk @@ -35,14 +35,12 @@ CMAKE_COMMON += -DCMAKE_CXX_COMPILER="$(CXX_BASE)" ifneq ($(strip $(CMAKE_CXX_ARG)),) CMAKE_COMMON += -DCMAKE_CXX_COMPILER_ARG1="$(CMAKE_CXX_ARG)" endif -CMAKE_COMMON += -DCMAKE_LINKER="$(shell which $(LD))" -DCMAKE_AR="$(shell which $(AR))" -DCMAKE_RANLIB="$(shell which $(RANLIB))" +CMAKE_COMMON += -DCMAKE_LINKER="$$(which $(LD))" -DCMAKE_AR="$$(which $(AR))" -DCMAKE_RANLIB="$$(which $(RANLIB))" ifeq ($(OS),WINNT) CMAKE_COMMON += -DCMAKE_SYSTEM_NAME=Windows -ifneq ($(BUILD_OS),WINNT) CMAKE_COMMON += -DCMAKE_RC_COMPILER="$$(which $(CROSS_COMPILE)windres)" endif -endif # For now this is LLVM specific, but I expect it won't be in the future ifeq ($(CMAKE_GENERATOR),Ninja) diff --git a/src/anticodegen.c b/src/anticodegen.c index ff65f9bcdf96f..df2738b0f67d5 100644 --- a/src/anticodegen.c +++ b/src/anticodegen.c @@ -13,7 +13,7 @@ void jl_write_malloc_log(void) UNAVAILABLE void jl_write_coverage_data(void) UNAVAILABLE JL_DLLEXPORT void jl_clear_malloc_data(void) UNAVAILABLE -JL_DLLEXPORT void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) UNAVAILABLE +JL_DLLEXPORT int jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) UNAVAILABLE JL_DLLEXPORT void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt) UNAVAILABLE JL_DLLEXPORT jl_value_t *jl_dump_method_asm(jl_method_instance_t *linfo, size_t world, int raw_mc, char getwrapper, const char* asm_variant, const char *debuginfo) UNAVAILABLE JL_DLLEXPORT const jl_value_t *jl_dump_function_ir(void *f, uint8_t strip_ir_metadata, uint8_t dump_module, const char *debuginfo) UNAVAILABLE diff --git a/src/codegen.cpp b/src/codegen.cpp index 3fc19055368f4..e13d63d702286 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5396,7 +5396,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con // do codegen to create a C-callable alias/wrapper, or if sysimg_handle is set, // restore one from a loaded system image. -void jl_generate_ccallable(void *llvmmod, void *sysimg_handle, jl_value_t *declrt, jl_value_t *sigt, jl_codegen_params_t ¶ms) +const char *jl_generate_ccallable(void *llvmmod, void *sysimg_handle, jl_value_t *declrt, jl_value_t *sigt, jl_codegen_params_t ¶ms) { jl_datatype_t *ft = (jl_datatype_t*)jl_tparam0(sigt); jl_value_t *ff = ft->instance; @@ -5438,7 +5438,7 @@ void jl_generate_ccallable(void *llvmmod, void *sysimg_handle, jl_value_t *declr gen_cfun_wrapper((Module*)llvmmod, params, sig, ff, name, declrt, lam, NULL, NULL, NULL); } JL_GC_POP(); - return; + return name; } err = jl_get_exceptionf(jl_errorexception_type, "%s", sig.err_msg.c_str()); } @@ -7587,7 +7587,6 @@ static void init_jit_functions(void) add_named_global(except_enter_func, (void*)NULL); #ifdef _OS_WINDOWS_ -#ifndef FORCE_ELF #if defined(_CPU_X86_64_) #if defined(_COMPILER_GCC_) add_named_global("___chkstk_ms", &___chkstk_ms); @@ -7602,7 +7601,6 @@ static void init_jit_functions(void) #endif #endif #endif -#endif #define BOX_F(ct) add_named_global("jl_box_"#ct, &jl_box_##ct); BOX_F(int8); BOX_F(uint8); @@ -7751,7 +7749,7 @@ extern "C" void jl_init_llvm(void) #endif init_julia_llvm_meta(); - jl_ExecutionEngine = new JuliaOJIT(*jl_TargetMachine); + jl_ExecutionEngine = new JuliaOJIT(*jl_TargetMachine, &jl_LLVMContext); // Mark our address spaces as non-integral jl_data_layout = jl_ExecutionEngine->getDataLayout(); diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 318cec338aa07..ee867f291a75c 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -227,16 +227,13 @@ class JuliaJITEventListener: public JITEventListener if (!SavedObject.first) { auto NewBuffer = MemoryBuffer::getMemBufferCopy( Object.getData(), Object.getFileName()); - auto NewObj = object::ObjectFile::createObjectFile(NewBuffer->getMemBufferRef()); - assert(NewObj); - SavedObject = std::make_pair(std::move(*NewObj), std::move(NewBuffer)); + auto NewObj = cantFail(object::ObjectFile::createObjectFile(NewBuffer->getMemBufferRef())); + SavedObject = std::make_pair(std::move(NewObj), std::move(NewBuffer)); } const object::ObjectFile &debugObj = *SavedObject.first.release(); SavedObject.second.release(); - object::section_iterator Section = debugObj.section_begin(); object::section_iterator EndSection = debugObj.section_end(); - std::map loadedSections; for (const object::SectionRef &lSection: Object.sections()) { #if JL_LLVM_VERSION >= 100000 @@ -316,16 +313,14 @@ class JuliaJITEventListener: public JITEventListener #endif #if defined(_OS_WINDOWS_) - uint64_t SectionAddrCheck = 0; // assert that all of the Sections are at the same location + uint64_t SectionAddrCheck = 0; + uint64_t SectionLoadCheck = 0; + uint64_t SectionWriteCheck = 0; uint8_t *UnwindData = NULL; #if defined(_CPU_X86_64_) - uint64_t SectionLoadOffset = 1; // The real offset shouldn't be 1. uint8_t *catchjmp = NULL; for (const object::SymbolRef &sym_iter : debugObj.symbols()) { - StringRef sName; - auto sNameOrError = sym_iter.getName(); - assert(sNameOrError); - sName = sNameOrError.get(); + StringRef sName = cantFail(sym_iter.getName()); uint8_t **pAddr = NULL; if (sName.equals("__UnwindData")) { pAddr = &UnwindData; @@ -334,58 +329,48 @@ class JuliaJITEventListener: public JITEventListener pAddr = &catchjmp; } if (pAddr) { - uint64_t Addr, SectionAddr, SectionLoadAddr; - auto AddrOrError = sym_iter.getAddress(); - assert(AddrOrError); - Addr = AddrOrError.get(); - auto SectionOrError = sym_iter.getSection(); - assert(SectionOrError); - Section = SectionOrError.get(); + uint64_t Addr = cantFail(sym_iter.getAddress()); + auto Section = cantFail(sym_iter.getSection()); assert(Section != EndSection && Section->isText()); - SectionAddr = Section->getAddress(); + uint64_t SectionAddr = Section->getAddress(); #if JL_LLVM_VERSION >= 100000 - auto secName = Section->getName(); - assert(secName); - SectionLoadAddr = getLoadAddress(*secName); + sName = cantFail(Section->getName()); #else Section->getName(sName); - SectionLoadAddr = getLoadAddress(sName); #endif - Addr -= SectionAddr - SectionLoadAddr; - *pAddr = (uint8_t*)Addr; - if (SectionAddrCheck) - assert(SectionAddrCheck == SectionLoadAddr); - else - SectionAddrCheck = SectionLoadAddr; + uint64_t SectionLoadAddr = getLoadAddress(sName); + assert(SectionLoadAddr); + if (SectionAddrCheck) // assert that all of the Sections are at the same location + assert(SectionAddrCheck == SectionAddr && + SectionLoadCheck == SectionLoadAddr); + SectionAddrCheck = SectionAddr; + SectionLoadCheck = SectionLoadAddr; + SectionWriteCheck = SectionLoadAddr; if (memmgr) - SectionAddr = - (uintptr_t)lookupWriteAddressFor(memmgr, - (void*)SectionLoadAddr); - if (SectionLoadOffset != 1) - assert(SectionLoadOffset == SectionAddr - SectionLoadAddr); - else - SectionLoadOffset = SectionAddr - SectionLoadAddr; + SectionWriteCheck = (uintptr_t)lookupWriteAddressFor(memmgr, + (void*)SectionLoadAddr); + Addr += SectionWriteCheck - SectionLoadAddr; + *pAddr = (uint8_t*)Addr; } } assert(catchjmp); assert(UnwindData); assert(SectionAddrCheck); - assert(SectionLoadOffset != 1); - catchjmp[SectionLoadOffset] = 0x48; - catchjmp[SectionLoadOffset + 1] = 0xb8; // mov RAX, QWORD PTR [&__julia_personality] - *(uint64_t*)(&catchjmp[SectionLoadOffset + 2]) = - (uint64_t)&__julia_personality; - catchjmp[SectionLoadOffset + 10] = 0xff; - catchjmp[SectionLoadOffset + 11] = 0xe0; // jmp RAX - UnwindData[SectionLoadOffset] = 0x09; // version info, UNW_FLAG_EHANDLER - UnwindData[SectionLoadOffset + 1] = 4; // size of prolog (bytes) - UnwindData[SectionLoadOffset + 2] = 2; // count of unwind codes (slots) - UnwindData[SectionLoadOffset + 3] = 0x05; // frame register (rbp) = rsp - UnwindData[SectionLoadOffset + 4] = 4; // second instruction - UnwindData[SectionLoadOffset + 5] = 0x03; // mov RBP, RSP - UnwindData[SectionLoadOffset + 6] = 1; // first instruction - UnwindData[SectionLoadOffset + 7] = 0x50; // push RBP - *(DWORD*)&UnwindData[SectionLoadOffset + 8] = (DWORD)(catchjmp - (uint8_t*)SectionAddrCheck); // relative location of catchjmp + assert(SectionLoadCheck); + catchjmp[0] = 0x48; + catchjmp[1] = 0xb8; // mov RAX, QWORD PTR [&__julia_personality] + *(uint64_t*)(&catchjmp[2]) = (uint64_t)&__julia_personality; + catchjmp[10] = 0xff; + catchjmp[11] = 0xe0; // jmp RAX + UnwindData[0] = 0x09; // version info, UNW_FLAG_EHANDLER + UnwindData[1] = 4; // size of prolog (bytes) + UnwindData[2] = 2; // count of unwind codes (slots) + UnwindData[3] = 0x05; // frame register (rbp) = rsp + UnwindData[4] = 4; // second instruction + UnwindData[5] = 0x03; // mov RBP, RSP + UnwindData[6] = 1; // first instruction + UnwindData[7] = 0x50; // push RBP + *(DWORD*)&UnwindData[8] = (DWORD)(catchjmp - (uint8_t*)SectionWriteCheck); // relative location of catchjmp #endif // defined(_OS_X86_64_) #endif // defined(_OS_WINDOWS_) @@ -393,39 +378,30 @@ class JuliaJITEventListener: public JITEventListener bool first = true; for (const auto &sym_size : symbols) { const object::SymbolRef &sym_iter = sym_size.first; - auto SymbolTypeOrError = sym_iter.getType(); - assert(SymbolTypeOrError); - object::SymbolRef::Type SymbolType = SymbolTypeOrError.get(); + object::SymbolRef::Type SymbolType = cantFail(sym_iter.getType()); if (SymbolType != object::SymbolRef::ST_Function) continue; - auto AddrOrError = sym_iter.getAddress(); - assert(AddrOrError); - uint64_t Addr = AddrOrError.get(); - auto SectionOrError = sym_iter.getSection(); - assert(SectionOrError); - Section = SectionOrError.get(); + uint64_t Addr = cantFail(sym_iter.getAddress()); + auto Section = cantFail(sym_iter.getSection()); if (Section == EndSection) continue; if (!Section->isText()) continue; uint64_t SectionAddr = Section->getAddress(); #if JL_LLVM_VERSION >= 100000 - Expected secName = Section->getName(); - assert(secName); - uint64_t SectionLoadAddr = getLoadAddress(*secName); + StringRef secName = cantFail(Section->getName()); #else StringRef secName; Section->getName(secName); - uint64_t SectionLoadAddr = getLoadAddress(secName); #endif + uint64_t SectionLoadAddr = getLoadAddress(secName); Addr -= SectionAddr - SectionLoadAddr; - auto sNameOrError = sym_iter.getName(); - assert(sNameOrError); - StringRef sName = sNameOrError.get(); + StringRef sName = cantFail(sym_iter.getName()); uint64_t SectionSize = Section->getSize(); size_t Size = sym_size.second; #if defined(_OS_WINDOWS_) if (SectionAddrCheck) - assert(SectionAddrCheck == SectionLoadAddr); - else - SectionAddrCheck = SectionLoadAddr; + assert(SectionAddrCheck == SectionAddr && + SectionLoadCheck == SectionLoadAddr); + SectionAddrCheck = SectionAddr; + SectionLoadCheck = SectionLoadAddr; create_PRUNTIME_FUNCTION( (uint8_t*)(uintptr_t)Addr, (size_t)Size, sName, (uint8_t*)(uintptr_t)SectionLoadAddr, (size_t)SectionSize, UnwindData); @@ -821,9 +797,8 @@ static void get_function_name_and_base(llvm::object::SectionRef Section, size_t } if (distance != (size_t)-1) { if (needs_saddr) { - auto addr = sym_found.getAddress(); - assert(addr); - *saddr = (void*)(uintptr_t)(addr.get() - slide); + uintptr_t addr = cantFail(sym_found.getAddress()); + *saddr = (void*)(addr - slide); needs_saddr = false; } if (needs_name) { diff --git a/src/dump.c b/src/dump.c index 654d47f35a18a..0daad95c03b5c 100644 --- a/src/dump.c +++ b/src/dump.c @@ -2576,7 +2576,9 @@ static jl_value_t *_jl_restore_incremental(ios_t *f, jl_array_t *mod_array) for (int i = 0; i < ccallable_list.len; i++) { jl_svec_t *item = (jl_svec_t*)ccallable_list.items[i]; JL_GC_PROMISE_ROOTED(item); - jl_compile_extern_c(NULL, NULL, NULL, jl_svecref(item, 0), jl_svecref(item, 1)); + int success = jl_compile_extern_c(NULL, NULL, NULL, jl_svecref(item, 0), jl_svecref(item, 1)); + if (!success) + jl_safe_printf("@ccallable was already defined for this method name\n"); } arraylist_free(&ccallable_list); jl_value_t *ret = (jl_value_t*)jl_svec(2, restored, init_order); diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index e6627dd023290..fa225b5e03082 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -5,19 +5,21 @@ #include "llvm-version.h" #include "platform.h" -#include -#include -#include -#include -#include -#include +#include "llvm/IR/Mangler.h" #include - +#include +#include +#include +#include +#include +#include +#include #include +#include #include -#include -#include +#include +#include using namespace llvm; @@ -214,11 +216,11 @@ static jl_callptr_t _jl_compile_codeinst( return fptr; } -void jl_generate_ccallable(void *llvmmod, void *sysimg_handle, jl_value_t *declrt, jl_value_t *sigt, jl_codegen_params_t ¶ms); +const char *jl_generate_ccallable(void *llvmmod, void *sysimg_handle, jl_value_t *declrt, jl_value_t *sigt, jl_codegen_params_t ¶ms); // compile a C-callable alias -extern "C" JL_DLLEXPORT -void jl_compile_extern_c(void *llvmmod, void *p, void *sysimg, jl_value_t *declrt, jl_value_t *sigt) +extern "C" +int jl_compile_extern_c(void *llvmmod, void *p, void *sysimg, jl_value_t *declrt, jl_value_t *sigt) { JL_LOCK(&codegen_lock); uint64_t compiler_start_time = jl_hrtime(); @@ -229,20 +231,25 @@ void jl_compile_extern_c(void *llvmmod, void *p, void *sysimg, jl_value_t *declr Module *into = (Module*)llvmmod; if (into == NULL) into = jl_create_llvm_module("cextern"); - jl_generate_ccallable(into, sysimg, declrt, sigt, *pparams); + const char *name = jl_generate_ccallable(into, sysimg, declrt, sigt, *pparams); + bool success = true; if (!sysimg) { - if (p == NULL) { + if (jl_ExecutionEngine->getGlobalValueAddress(name)) { + success = false; + } + if (success && p == NULL) { jl_jit_globals(params.globals); assert(params.workqueue.empty()); if (params._shared_module) jl_add_to_ee(std::unique_ptr(params._shared_module)); } - if (llvmmod == NULL) + if (success && llvmmod == NULL) jl_add_to_ee(std::unique_ptr(into)); } if (codegen_lock.count == 1) jl_cumulative_compile_time += (jl_hrtime() - compiler_start_time); JL_UNLOCK(&codegen_lock); + return success; } bool jl_type_mappable_to_c(jl_value_t *ty); @@ -287,7 +294,9 @@ void jl_extern_c(jl_value_t *declrt, jl_tupletype_t *sigt) JL_GC_POP(); // create the alias in the current runtime environment - jl_compile_extern_c(NULL, NULL, NULL, declrt, (jl_value_t*)sigt); + int success = jl_compile_extern_c(NULL, NULL, NULL, declrt, (jl_value_t*)sigt); + if (!success) + jl_error("@ccallable was already defined for this method name"); } // this compiles li and emits fptr @@ -433,80 +442,76 @@ jl_value_t *jl_dump_method_asm(jl_method_instance_t *mi, size_t world, return jl_dump_llvm_asm(jl_get_llvmf_defn(mi, world, getwrapper, true, jl_default_cgparams), asm_variant, debuginfo); } -#if defined(_OS_LINUX_) || defined(_OS_WINDOWS_) || defined(_OS_FREEBSD_) -// Resolve non-lock free atomic functions in the libatomic1 library. -// This is the library that provides support for c11/c++11 atomic operations. -static uint64_t resolve_atomic(const char *name) -{ -#if defined(_OS_LINUX_) || defined(_OS_FREEBSD_) - static const char *const libatomic = "libatomic.so.1"; -#elif defined(_OS_WINDOWS_) - static const char *const libatomic = "libatomic-1.dll"; -#endif - static void *atomic_hdl = jl_load_dynamic_library(libatomic, - JL_RTLD_LOCAL, 0); - static const char *const atomic_prefix = "__atomic_"; - if (!atomic_hdl) - return 0; - if (strncmp(name, atomic_prefix, strlen(atomic_prefix)) != 0) - return 0; - uintptr_t value; - jl_dlsym(atomic_hdl, name, (void **)&value, 0); - return value; -} -#endif +// A simple forwarding class, since OrcJIT v2 needs a unique_ptr, while we have a shared_ptr +class ForwardingMemoryManager : public RuntimeDyld::MemoryManager { +private: + std::shared_ptr MemMgr; + +public: + ForwardingMemoryManager(std::shared_ptr MemMgr) : MemMgr(MemMgr) {} + virtual ~ForwardingMemoryManager() = default; + virtual uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, + StringRef SectionName) override { + return MemMgr->allocateCodeSection(Size, Alignment, SectionID, SectionName); + } + virtual uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, + StringRef SectionName, + bool IsReadOnly) override { + return MemMgr->allocateDataSection(Size, Alignment, SectionID, SectionName, IsReadOnly); + } + virtual void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign, + uintptr_t RODataSize, + uint32_t RODataAlign, + uintptr_t RWDataSize, + uint32_t RWDataAlign) override { + return MemMgr->reserveAllocationSpace(CodeSize, CodeAlign, RODataSize, RODataAlign, RWDataSize, RWDataAlign); + } + virtual bool needsToReserveAllocationSpace() override { + return MemMgr->needsToReserveAllocationSpace(); + } + virtual void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, + size_t Size) override { + return MemMgr->registerEHFrames(Addr, LoadAddr, Size); + } + virtual void deregisterEHFrames() override { + return MemMgr->deregisterEHFrames(); + } + virtual bool finalizeMemory(std::string *ErrMsg = nullptr) override { + return MemMgr->finalizeMemory(ErrMsg); + } + virtual void notifyObjectLoaded(RuntimeDyld &RTDyld, + const object::ObjectFile &Obj) override { + return MemMgr->notifyObjectLoaded(RTDyld, Obj); + } +}; + // Custom object emission notification handler for the JuliaOJIT extern JITEventListener *CreateJuliaJITEventListener(); -JuliaOJIT::DebugObjectRegistrar::DebugObjectRegistrar(JuliaOJIT &JIT) - : JuliaListener(CreateJuliaJITEventListener()), - JIT(JIT) {} JL_DLLEXPORT void ORCNotifyObjectEmitted(JITEventListener *Listener, const object::ObjectFile &obj, const RuntimeDyld::LoadedObjectInfo &L, RTDyldMemoryManager *memmgr); +#if JL_LLVM_VERSION >= 120000 template -void JuliaOJIT::DebugObjectRegistrar::registerObject(RTDyldObjHandleT H, const ObjT &Obj, - const LoadResult &LO) +void JuliaOJIT::registerObject(const ObjT &Obj, const LoadResult &LO) { const ObjT* Object = &Obj; - - JIT.NotifyFinalizer(H, *Object, *LO); - ORCNotifyObjectEmitted(JuliaListener.get(), *Object, *LO, JIT.MemMgr.get()); - - // record all of the exported symbols defined in this object - // in the primary hash table for the enclosing JIT - for (auto &Symbol : Object->symbols()) { -#if JL_LLVM_VERSION >= 110000 - uint32_t Flags = Symbol.getFlags().get(); -#else - uint32_t Flags = Symbol.getFlags(); -#endif - if (Flags & object::BasicSymbolRef::SF_Undefined) - continue; - if (!(Flags & object::BasicSymbolRef::SF_Exported)) - continue; - auto NameOrError = Symbol.getName(); - assert(NameOrError); - auto Name = NameOrError.get(); - auto Sym = JIT.CompileLayer.findSymbolIn(H, Name.str(), true); - assert(Sym); - // note: calling getAddress here eagerly finalizes H - // as an alternative, we could store the JITSymbol instead - // (which would present a lazy-initializer functor interface instead) - JIT.LocalSymbolTable[Name] = (void*)(uintptr_t)cantFail(Sym.getAddress()); - } + ORCNotifyObjectEmitted(JuliaListener.get(), *Object, *LO, MemMgr.get()); } - -template -void JuliaOJIT::DebugObjectRegistrar::operator()(RTDyldObjHandleT H, - const ObjSetT &Object, const LoadResult &LOS) +#else +template +void JuliaOJIT::registerObject(RTDyldObjHandleT H, const ObjT &Obj, const LoadResult &LO) { - registerObject(H, Object, - static_cast(&LOS)); + const ObjT* Object = &Obj; + NotifyFinalizer(H, *Object, *LO); + ORCNotifyObjectEmitted(JuliaListener.get(), *Object, *LO, MemMgr.get()); } +#endif CodeGenOpt::Level CodeGenOptLevelFor(int optlevel) { @@ -630,38 +635,40 @@ CompilerResultT JuliaOJIT::CompilerT::operator()(Module &M) return CompilerResultT(std::move(ObjBuffer)); } -JuliaOJIT::JuliaOJIT(TargetMachine &TM) +JuliaOJIT::JuliaOJIT(TargetMachine &TM, LLVMContext *LLVMCtx) : TM(TM), DL(TM.createDataLayout()), ObjStream(ObjBufferSV), MemMgr(createRTDyldMemoryManager()), - registrar(*this), + JuliaListener(CreateJuliaJITEventListener()), + TSCtx(std::unique_ptr(LLVMCtx)), ES(), - SymbolResolver(llvm::orc::createLegacyLookupResolver( - ES, - [this](StringRef name) -> llvm::JITSymbol { - return this->resolveSymbol(name); - }, - [](llvm::Error Err) { - cantFail(std::move(Err), "resolveSymbol failed"); - })), + GlobalJD(ES.createBareJITDylib("JuliaGlobals")), + JD(ES.createBareJITDylib("JuliaOJIT")), ObjectLayer( - AcknowledgeORCv1Deprecation, - ES, - [this](RTDyldObjHandleT) { - ObjLayerT::Resources result; - result.MemMgr = MemMgr; - result.Resolver = SymbolResolver; - return result; - }, - std::ref(registrar) + ES, + [this]() { + std::unique_ptr result(new ForwardingMemoryManager(MemMgr)); + return result; + } ), - CompileLayer( - AcknowledgeORCv1Deprecation, - ObjectLayer, - CompilerT(this) - ) + CompileLayer(ES, ObjectLayer, std::make_unique(this)) { +#if JL_LLVM_VERSION >= 120000 + ObjectLayer.setNotifyLoaded( + [this](orc::MaterializationResponsibility &MR, + const object::ObjectFile &Object, + const RuntimeDyld::LoadedObjectInfo &LOS) { + registerObject(Object, &LOS); + }); +#else + ObjectLayer.setNotifyLoaded( + [this](RTDyldObjHandleT H, + const object::ObjectFile &Object, + const RuntimeDyld::LoadedObjectInfo &LOS) { + registerObject(H, Object, &LOS); + }); +#endif for (int i = 0; i < 4; i++) { TMs[i] = TM.getTarget().createTargetMachine(TM.getTargetTriple().getTriple(), TM.getTargetCPU(), TM.getTargetFeatureString(), TM.Options, Reloc::Static, TM.getCodeModel(), @@ -678,36 +685,51 @@ JuliaOJIT::JuliaOJIT(TargetMachine &TM) std::string ErrorStr; if (sys::DynamicLibrary::LoadLibraryPermanently(nullptr, &ErrorStr)) report_fatal_error("FATAL: unable to dlopen self\n" + ErrorStr); -} -void JuliaOJIT::addGlobalMapping(StringRef Name, uint64_t Addr) -{ - std::string MangleName = getMangledName(Name); - bool successful = GlobalSymbolTable.insert(std::make_pair(MangleName, (void*)Addr)).second; - (void)successful; - assert(successful); -} + GlobalJD.addGenerator( + std::move(*orc::DynamicLibrarySearchGenerator::GetForCurrentProcess( + DL.getGlobalPrefix()))); -void *JuliaOJIT::getPointerToGlobalIfAvailable(StringRef S) -{ - SymbolTableT::const_iterator pos = GlobalSymbolTable.find(S); - if (pos != GlobalSymbolTable.end()) - return pos->second; - return nullptr; + // Resolve non-lock free atomic functions in the libatomic1 library. + // This is the library that provides support for c11/c++11 atomic operations. + const char *const libatomic = +#if defined(_OS_LINUX_) || defined(_OS_FREEBSD_) + "libatomic.so.1"; +#elif defined(_OS_WINDOWS_) + "libatomic-1.dll"; +#else + NULL; +#endif + if (libatomic) { + static void *atomic_hdl = jl_load_dynamic_library(libatomic, JL_RTLD_LOCAL, 0); + if (atomic_hdl != NULL) { + GlobalJD.addGenerator( + std::move(*orc::DynamicLibrarySearchGenerator::Load( + libatomic, + DL.getGlobalPrefix(), + [&](const orc::SymbolStringPtr &S) { + const char *const atomic_prefix = "__atomic_"; + return (*S).startswith(atomic_prefix); + }))); + } + } + + JD.addToLinkOrder(GlobalJD, orc::JITDylibLookupFlags::MatchExportedSymbolsOnly); } -void *JuliaOJIT::getPointerToGlobalIfAvailable(const GlobalValue *GV) +void JuliaOJIT::addGlobalMapping(StringRef Name, uint64_t Addr) { - return getPointerToGlobalIfAvailable(getMangledName(GV)); + std::string MangleName = getMangledName(Name); + cantFail(JD.define(orc::absoluteSymbols({{ES.intern(MangleName), JITEvaluatedSymbol::fromPointer((void*)Addr)}}))); } - void JuliaOJIT::addModule(std::unique_ptr M) { - std::vector NewExports; + JL_TIMING(LLVM_MODULE_FINISH); + std::vector NewExports; for (auto &F : M->functions()) { if (!F.isDeclaration() && F.getLinkage() == GlobalValue::ExternalLinkage) { - NewExports.push_back(strdup(F.getName().str().c_str())); + NewExports.push_back(getMangledName(F.getName())); } } #ifndef JL_NDEBUG @@ -730,42 +752,39 @@ void JuliaOJIT::addModule(std::unique_ptr M) } } #endif - JL_TIMING(LLVM_MODULE_FINISH); - +#if JL_LLVM_VERSION >= 120000 + // TODO: what is the performance characteristics of this? + cantFail(CompileLayer.add(JD, orc::ThreadSafeModule(std::move(M), TSCtx))); +#else auto key = ES.allocateVModule(); - cantFail(CompileLayer.addModule(key, std::move(M))); - // Force LLVM to emit the module so that we can register the symbols - // in our lookup table. - Error Err = CompileLayer.emitAndFinalize(key); - // Check for errors to prevent LLVM from crashing the program. - if (Err) - report_fatal_error(std::move(Err)); - // record a stable name for this fptr address - for (auto Name : NewExports) { - void *addr = LocalSymbolTable[getMangledName(Name)]; - ReverseLocalSymbolTable[addr] = Name; - } + // TODO: what is the performance characteristics of this? + cantFail(CompileLayer.add(JD, orc::ThreadSafeModule(std::move(M), TSCtx), key)); +#endif + // force eager compilation (for now), due to memory management specifics + // (can't handle compilation recursion) + for (auto Name : NewExports) + cantFail(ES.lookup({&JD}, Name)); + } +#if JL_LLVM_VERSION < 120000 void JuliaOJIT::removeModule(ModuleHandleT H) { - (void)CompileLayer.removeModule(H); + //(void)CompileLayer.remove(H); } +#endif JL_JITSymbol JuliaOJIT::findSymbol(StringRef Name, bool ExportedSymbolsOnly) { - void *Addr = nullptr; if (ExportedSymbolsOnly) { - // Step 1: Check against list of known external globals - Addr = getPointerToGlobalIfAvailable(Name); - } - // Step 2: Search all previously emitted symbols - if (Addr == nullptr) { - auto it = LocalSymbolTable.find(Name); - if (it != LocalSymbolTable.end()) - Addr = it->second; + auto Sym = ES.lookup({&GlobalJD}, Name); + if (Sym) + return *Sym; } - return JL_JITSymbol((uintptr_t)Addr, JITSymbolFlags::Exported); + auto Sym = ES.lookup({&JD}, Name); + if (Sym) + return *Sym; + return Sym.takeError(); } JL_JITSymbol JuliaOJIT::findUnmangledSymbol(StringRef Name) @@ -773,36 +792,16 @@ JL_JITSymbol JuliaOJIT::findUnmangledSymbol(StringRef Name) return findSymbol(getMangledName(Name), true); } -JL_JITSymbol JuliaOJIT::resolveSymbol(StringRef Name) -{ - // Step 0: ObjectLinkingLayer has checked whether it is in the current module - // Step 1: See if it's something known to the ExecutionEngine - if (auto Sym = findSymbol(Name, true)) { - // `findSymbol` already eagerly resolved the address - // return it directly. - return Sym; - } - // Step 2: Search the program symbols - if (uint64_t addr = SectionMemoryManager::getSymbolAddressInProcess(Name.str())) - return JL_SymbolInfo(addr, JITSymbolFlags::Exported); -#if defined(_OS_LINUX_) || defined(_OS_WINDOWS_) || defined(_OS_FREEBSD_) - if (uint64_t addr = resolve_atomic(Name.str().c_str())) - return JL_SymbolInfo(addr, JITSymbolFlags::Exported); -#endif - // Return failure code - return JL_SymbolInfo(nullptr); -} - uint64_t JuliaOJIT::getGlobalValueAddress(StringRef Name) { - auto addr = findSymbol(getMangledName(Name), false).getAddress(); - return addr ? addr.get() : 0; + auto addr = findSymbol(getMangledName(Name), false); + return addr ? addr.getAddress().get() : 0; } uint64_t JuliaOJIT::getFunctionAddress(StringRef Name) { - auto addr = findSymbol(getMangledName(Name), false).getAddress(); - return addr ? addr.get() : 0; + auto addr = findSymbol(getMangledName(Name), false); + return addr ? addr.getAddress().get() : 0; } static int globalUniqueGeneratedNames; @@ -828,7 +827,7 @@ StringRef JuliaOJIT::getFunctionAtAddress(uint64_t Addr, jl_code_instance_t *cod const char* unadorned_name = jl_symbol_name(codeinst->def->def.method->name); stream_fname << unadorned_name << "_" << globalUniqueGeneratedNames++; fname = strdup(stream_fname.str().c_str()); - LocalSymbolTable[getMangledName(string_fname)] = (void*)(uintptr_t)Addr; + addGlobalMapping(fname, Addr); } return fname; } @@ -838,9 +837,14 @@ void JuliaOJIT::RegisterJITEventListener(JITEventListener *L) { if (!L) return; +#if JL_LLVM_VERSION >= 120000 + this->ObjectLayer.registerJITEventListener(*L); +#else EventListeners.push_back(L); +#endif } +#if JL_LLVM_VERSION < 120000 void JuliaOJIT::NotifyFinalizer(RTDyldObjHandleT Key, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &LoadedObjectInfo) @@ -848,6 +852,7 @@ void JuliaOJIT::NotifyFinalizer(RTDyldObjHandleT Key, for (auto &Listener : EventListeners) Listener->notifyObjectLoaded(Key, Obj, LoadedObjectInfo); } +#endif const DataLayout& JuliaOJIT::getDataLayout() const { @@ -1106,7 +1111,9 @@ static void jl_add_to_ee(std::unique_ptr &M, StringMapgetFunctionAddress(fname); + auto addr = jl_ExecutionEngine->getFunctionAddress(fname); + assert(addr); + return addr; } // helper function for adding a DLLImport (dlsym) address to the execution engine diff --git a/src/jitlayers.h b/src/jitlayers.h index 979fe15c73839..01fcf3c325df9 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -7,12 +7,9 @@ #include "llvm/IR/LegacyPassManager.h" #include -#include "llvm/ExecutionEngine/Orc/CompileUtils.h" -#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" -#include "llvm/ExecutionEngine/Orc/LambdaResolver.h" -#include "llvm/ExecutionEngine/Orc/LazyEmittingLayer.h" -#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" -#include "llvm/ExecutionEngine/JITEventListener.h" +#include +#include +#include #include #include "julia_assert.h" @@ -142,58 +139,59 @@ typedef JITSymbol JL_JITSymbol; // is expected. typedef JITSymbol JL_SymbolInfo; +#if JL_LLVM_VERSION < 120000 using RTDyldObjHandleT = orc::VModuleKey; +#endif + #if JL_LLVM_VERSION >= 100000 -using CompilerResultT = Expected; +using CompilerResultT = Expected>; #else using CompilerResultT = std::unique_ptr; #endif class JuliaOJIT { - // Custom object emission notification handler for the JuliaOJIT - class DebugObjectRegistrar { - public: - DebugObjectRegistrar(JuliaOJIT &JIT); - template - void operator()(RTDyldObjHandleT H, const ObjSetT &Object, const LoadResult &LOS); - private: - template - void registerObject(RTDyldObjHandleT H, const ObjT &Obj, const LoadResult &LO); - std::unique_ptr JuliaListener; - JuliaOJIT &JIT; - }; - - struct CompilerT { + struct CompilerT : public orc::IRCompileLayer::IRCompiler { CompilerT(JuliaOJIT *pjit) - : jit(*pjit) - {} - CompilerResultT operator()(Module &M); + : IRCompiler(orc::IRSymbolMapper::ManglingOptions{}), + jit(*pjit) {} + virtual CompilerResultT operator()(Module &M) override; private: JuliaOJIT &jit; }; +#if JL_LLVM_VERSION >= 120000 + // Custom object emission notification handler for the JuliaOJIT + template + void registerObject(const ObjT &Obj, const LoadResult &LO); +#else + // Custom object emission notification handler for the JuliaOJIT + template + void registerObject(RTDyldObjHandleT H, const ObjT &Obj, const LoadResult &LO); +#endif public: - typedef orc::LegacyRTDyldObjectLinkingLayer ObjLayerT; - typedef orc::LegacyIRCompileLayer CompileLayerT; - typedef orc::VModuleKey ModuleHandleT; - typedef StringMap SymbolTableT; + typedef orc::RTDyldObjectLinkingLayer ObjLayerT; + typedef orc::IRCompileLayer CompileLayerT; +#if JL_LLVM_VERSION < 120000 + typedef RTDyldObjHandleT ModuleHandleT; +#endif typedef object::OwningBinary OwningObj; - JuliaOJIT(TargetMachine &TM); + JuliaOJIT(TargetMachine &TM, LLVMContext *Ctx); void RegisterJITEventListener(JITEventListener *L); +#if JL_LLVM_VERSION < 120000 std::vector EventListeners; void NotifyFinalizer(RTDyldObjHandleT Key, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &LoadedObjectInfo); +#endif void addGlobalMapping(StringRef Name, uint64_t Addr); - void *getPointerToGlobalIfAvailable(StringRef S); - void *getPointerToGlobalIfAvailable(const GlobalValue *GV); void addModule(std::unique_ptr M); +#if JL_LLVM_VERSION < 120000 void removeModule(ModuleHandleT H); +#endif JL_JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly); JL_JITSymbol findUnmangledSymbol(StringRef Name); - JL_JITSymbol resolveSymbol(StringRef Name); uint64_t getGlobalValueAddress(StringRef Name); uint64_t getFunctionAddress(StringRef Name); StringRef getFunctionAtAddress(uint64_t Addr, jl_code_instance_t *codeinst); @@ -217,16 +215,17 @@ class JuliaOJIT { TargetMachine *TMs[4]; MCContext *Ctx; std::shared_ptr MemMgr; - DebugObjectRegistrar registrar; + std::unique_ptr JuliaListener; + - llvm::orc::ExecutionSession ES; - std::shared_ptr SymbolResolver; + orc::ThreadSafeContext TSCtx; + orc::ExecutionSession ES; + orc::JITDylib &GlobalJD; + orc::JITDylib &JD; ObjLayerT ObjectLayer; CompileLayerT CompileLayer; - SymbolTableT GlobalSymbolTable; - SymbolTableT LocalSymbolTable; DenseMap ReverseLocalSymbolTable; }; extern JuliaOJIT *jl_ExecutionEngine; diff --git a/src/julia_internal.h b/src/julia_internal.h index 21bdbc85ec901..cf794c082cda0 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -527,7 +527,7 @@ jl_binding_t *jl_get_module_binding(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t void jl_binding_deprecation_warning(jl_module_t *m, jl_binding_t *b); extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED; extern htable_t jl_current_modules JL_GLOBALLY_ROOTED; -JL_DLLEXPORT void jl_compile_extern_c(void *llvmmod, void *params, void *sysimg, jl_value_t *declrt, jl_value_t *sigt); +int jl_compile_extern_c(void *llvmmod, void *params, void *sysimg, jl_value_t *declrt, jl_value_t *sigt); // Each tuple can exist in one of 4 Vararg states: // NONE: no vararg Tuple{Int,Float32} diff --git a/src/staticdata.c b/src/staticdata.c index 5b63c6d4fcbe6..cdd675a5a6f44 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1401,7 +1401,8 @@ static void jl_reinit_item(jl_value_t *v, int how) JL_GC_DISABLED } case 3: { // install ccallable entry point in JIT jl_svec_t *sv = ((jl_method_t*)v)->ccallable; - jl_compile_extern_c(NULL, NULL, jl_sysimg_handle, jl_svecref(sv, 0), jl_svecref(sv, 1)); + int success = jl_compile_extern_c(NULL, NULL, jl_sysimg_handle, jl_svecref(sv, 0), jl_svecref(sv, 1)); + assert(success); (void)success; break; } default: diff --git a/test/llvmcall.jl b/test/llvmcall.jl index 0fab2e2f84802..25951757b9e9d 100644 --- a/test/llvmcall.jl +++ b/test/llvmcall.jl @@ -152,8 +152,12 @@ module ObjLoadTest didcall = true nothing end + @test_throws(ErrorException("@ccallable was already defined for this method name"), + @eval @ccallable Cvoid jl_the_callback(not_the_method::Int) = "other") # Make sure everything up until here gets compiled - jl_the_callback(); didcall = false + @test jl_the_callback() === nothing + @test jl_the_callback(1) == "other" + didcall = false function do_the_call() llvmcall( ("""declare void @jl_the_callback() diff --git a/test/precompile.jl b/test/precompile.jl index 7e2e82b88ff0b..3358fda58f0e3 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -262,9 +262,11 @@ precompile_test_harness(false) do dir cachefile = joinpath(cachedir, "$Foo_module.ji") # use _require_from_serialized to ensure that the test fails if # the module doesn't reload from the image: - @test_logs (:warn, "Replacing module `$Foo_module`") begin - ms = Base._require_from_serialized(cachefile) - @test isa(ms, Array{Any,1}) + @test_warn "@ccallable was already defined for this method name" begin + @test_logs (:warn, "Replacing module `$Foo_module`") begin + ms = Base._require_from_serialized(cachefile) + @test isa(ms, Array{Any,1}) + end end @test_throws MethodError Foo.foo(17) # world shouldn't be visible yet