From 23748d03aeddf2685d93315d8f0e1ff6618f120e Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sun, 9 Feb 2020 14:42:53 +0100 Subject: [PATCH 01/80] clear everything out --- .appveyor.yml | 47 ---- .codecov.yml | 1 - .github/FUNDING.yml | 1 - .github/workflows/TagBot.yml | 11 - .gitignore | 20 -- .mailmap | 8 - .travis.yml | 46 ---- LICENSE.md | 22 -- Project.toml | 30 --- README.md | 306 --------------------- deps/build.jl | 73 ----- examples/REQUIRE | 1 - examples/hello.jl | 12 - examples/program.c | 54 ---- juliac.jl | 189 ------------- src/PackageCompiler.jl | 166 ------------ src/api.jl | 125 --------- src/compiler_flags.jl | 265 ------------------ src/incremental.jl | 178 ------------ src/pkg.jl | 178 ------------ src/snooping.jl | 149 ---------- src/static_julia.jl | 373 -------------------------- src/system_image.jl | 66 ----- test/REQUIRE | 6 - test/TestPackage/Project.toml | 4 - test/TestPackage/src/TestPackage.jl | 5 - test/TestPackage/test/runtests.jl | 3 - test/TestPackage2/Project.toml | 4 - test/TestPackage2/snoop/snoopfile.jl | 3 - test/TestPackage2/src/TestPackage2.jl | 5 - test/runtests.jl | 161 ----------- test/shipping.jl | 49 ---- 32 files changed, 2561 deletions(-) delete mode 100644 .appveyor.yml delete mode 100644 .codecov.yml delete mode 100644 .github/FUNDING.yml delete mode 100644 .github/workflows/TagBot.yml delete mode 100644 .gitignore delete mode 100644 .mailmap delete mode 100644 .travis.yml delete mode 100644 LICENSE.md delete mode 100644 Project.toml delete mode 100644 README.md delete mode 100644 deps/build.jl delete mode 100644 examples/REQUIRE delete mode 100644 examples/hello.jl delete mode 100644 examples/program.c delete mode 100644 juliac.jl delete mode 100644 src/PackageCompiler.jl delete mode 100644 src/api.jl delete mode 100644 src/compiler_flags.jl delete mode 100644 src/incremental.jl delete mode 100644 src/pkg.jl delete mode 100644 src/snooping.jl delete mode 100644 src/static_julia.jl delete mode 100644 src/system_image.jl delete mode 100644 test/REQUIRE delete mode 100644 test/TestPackage/Project.toml delete mode 100644 test/TestPackage/src/TestPackage.jl delete mode 100644 test/TestPackage/test/runtests.jl delete mode 100644 test/TestPackage2/Project.toml delete mode 100644 test/TestPackage2/snoop/snoopfile.jl delete mode 100644 test/TestPackage2/src/TestPackage2.jl delete mode 100644 test/runtests.jl delete mode 100644 test/shipping.jl diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 5d3cab74..00000000 --- a/.appveyor.yml +++ /dev/null @@ -1,47 +0,0 @@ -environment: - matrix: - - julia_version: 1.0 - - julia_version: 1.3 - - julia_version: nightly - -platform: - - x86 # 32-bit - - x64 # 64-bit - -matrix: - allow_failures: - - julia_version: nightly - -# # Uncomment the following lines to allow failures on nightly julia -# # (tests will run but not make your overall status red) -# matrix: -# allow_failures: -# - julia_version: latest - -branches: - only: - - master - - /release-.*/ - -notifications: - - provider: Email - on_build_success: false - on_build_failure: false - on_build_status_changed: false - -install: - - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1")) - -build_script: - - echo "%JL_BUILD_SCRIPT%" - - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%" - -test_script: - - echo "%JL_TEST_SCRIPT%" - - C:\julia\bin\julia -e "%JL_TEST_SCRIPT%" - -# # Uncomment to support code coverage upload. Should only be enabled for packages -# # which would have coverage gaps without running on Windows -# on_success: -# - echo "%JL_CODECOV_SCRIPT%" -# - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%" diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index 69cb7601..00000000 --- a/.codecov.yml +++ /dev/null @@ -1 +0,0 @@ -comment: false diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index cc27b731..00000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1 +0,0 @@ -custom: https://numfocus.salsalabs.org/donate-to-julia/index.html diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml deleted file mode 100644 index d77d3a0c..00000000 --- a/.github/workflows/TagBot.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: TagBot -on: - schedule: - - cron: 0 * * * * -jobs: - TagBot: - runs-on: ubuntu-latest - steps: - - uses: JuliaRegistries/TagBot@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore deleted file mode 100644 index ee7e4b0e..00000000 --- a/.gitignore +++ /dev/null @@ -1,20 +0,0 @@ -*.jl.cov -*.jl.*.cov -*.jl.mem -deps.jl -*.csv -snoopy.jl -precompile.jl -sysimg -*.ji -*.o -*.so -*.so.* -*.dylib -*.dll -hello -hello.exe -packages -deps/usr/ -deps/build.log -Manifest.toml diff --git a/.mailmap b/.mailmap deleted file mode 100644 index af815b8d..00000000 --- a/.mailmap +++ /dev/null @@ -1,8 +0,0 @@ -Luca Trevisani -Luca Trevisani - -Simon Danisch - -Viral B. Shah -Viral B. Shah -Viral B. Shah diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 70e59b3c..00000000 --- a/.travis.yml +++ /dev/null @@ -1,46 +0,0 @@ -## Documentation: http://docs.travis-ci.com/user/languages/julia/ - -language: julia - -os: - - linux - - osx - -julia: - - 1.0 - - 1.3 - - nightly - -matrix: - allow_failures: - - julia: nightly - -notifications: - email: false - -git: - depth: 99999999 - -## uncomment the following lines to allow failures on nightly julia -## (tests will run but not make your overall status red) -# matrix: -# allow_failures: -# - julia: nightly - -## uncomment and modify the following lines to manually install system packages -#addons: -# apt: # apt-get for linux -# packages: -# - gfortran -#before_script: # homebrew for mac -# - if [ $TRAVIS_OS_NAME = osx ]; then brew install gcc; fi - -## uncomment the following lines to override the default test script -#script: -# - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("PackageCompiler"); Pkg.test("PackageCompiler"; coverage=true)' - -after_success: - # push coverage results to Coveralls - - julia -e 'cd(normpath(Base.find_package("PackageCompiler"), "..", "..")); using Pkg; Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())' - # push coverage results to Codecov - - julia -e 'cd(normpath(Base.find_package("PackageCompiler"), "..", "..")); using Pkg; Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())' diff --git a/LICENSE.md b/LICENSE.md deleted file mode 100644 index adcc4f70..00000000 --- a/LICENSE.md +++ /dev/null @@ -1,22 +0,0 @@ -The PackageCompiler.jl package is licensed under the MIT "Expat" License: - -> Copyright (c) 2017: SimonDanisch. -> -> Permission is hereby granted, free of charge, to any person obtaining a copy -> of this software and associated documentation files (the "Software"), to deal -> in the Software without restriction, including without limitation the rights -> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -> copies of the Software, and to permit persons to whom the Software is -> furnished to do so, subject to the following conditions: -> -> The above copyright notice and this permission notice shall be included in all -> copies or substantial portions of the Software. -> -> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -> SOFTWARE. -> diff --git a/Project.toml b/Project.toml deleted file mode 100644 index a585ba7a..00000000 --- a/Project.toml +++ /dev/null @@ -1,30 +0,0 @@ -name = "PackageCompiler" -uuid = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" -version = "0.6.5" - -[deps] -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" -Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" -WinRPM = "c17dfb99-b4f7-5aad-8812-456da1ad7187" - -[compat] -julia = "1" -WinRPM = "0.4.3, 1" - -[extras] -ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" -DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -FixedPointNumbers = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" -JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" -Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -UnicodePlots = "b8865327-cd53-5732-bb35-84acbb429228" -ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" - - -[targets] -test = ["OffsetArrays", "JSON", "Pkg", "DataStructures", "ColorTypes", "Test", "UnicodePlots", "FixedPointNumbers", "ArgParse"] diff --git a/README.md b/README.md deleted file mode 100644 index 7ccb81b8..00000000 --- a/README.md +++ /dev/null @@ -1,306 +0,0 @@ -# PackageCompiler -[![Build Status](https://travis-ci.org/JuliaLang/PackageCompiler.jl.svg?branch=master)](https://travis-ci.org/JuliaLang/PackageCompiler.jl) - -[![Coverage Status](https://coveralls.io/repos/JuliaLang/PackageCompiler.jl/badge.svg?branch=master&service=github)](https://coveralls.io/github/JuliaLang/PackageCompiler.jl?branch=master) - -[![codecov.io](http://codecov.io/github/JuliaLang/PackageCompiler.jl/coverage.svg?branch=master)](http://codecov.io/github/JuliaLang/PackageCompiler.jl?branch=master) - -Remove just-in-time compilation overhead from your package and compile it into a system image. - -## Usage example - -One can try ahead of time compiled images online with nextjournal! -Here are some images for a few popular packages: - -[dataframes + query](https://nextjournal.com/sdanisch/data-remix) - -[plots & gr backend](https://nextjournal.com/sdanisch/plots-remix) - -[makie & opengl backend](https://nextjournal.com/sdanisch/glmakie-remix) - -[makie & cairo backend](https://nextjournal.com/sdanisch/cairomakie-remix) - -If you find a package to be missing, anyone can create these images and share them here! -One can also download the docker images for local usage: -[instructions](https://nextjournal.com/sdanisch/static-cairomakie) - -(signup code for nextjournal: `julia1.0`) - - -# compile_package -```Julia -using PackageCompiler - -# This command will use the `runtest.jl` of `ColorTypes` + `FixedPointNumbers` to find out what functions to precompile! -# `force = false` to not force overwriting Julia's current system image -compile_package("ColorTypes", "FixedPointNumbers", force = false) - -# force = false is the default and recommended, since overwriting your standard system image can make Julia unusable. - -# If you used force and want your old system image back (force will overwrite the default system image Julia uses) you can run: -revert() - -``` - -# compile_incremental - -This function works like the above, but incrementally adds the newly cached binary to your old system image. -That means that all precompiled code in the system image (e.g. REPL code) is preserved and therefore one gets a lag free start of the Julia REPL. -Also, the compilation times are much faster: - -help?> compile_incremental -``` -compile_incremental( - toml_path::String, snoopfile::String; - force = false, precompile_file = nothing, verbose = true, - debug = false, cc_flags = nothing -) - -Extract all calls from `snoopfile` and ahead of time compiles them -incrementally into the current system image. -`force = true` will replace the old system image with the new one. -The argument `toml_path` should contain a project file of the packages that `snoopfile` explicitly uses. -Implicitly used packages & modules don't need to be contained! - -To compile just a single package, see the simpler version `compile_incremental(package::Symbol)`: -``` - -``` -compile_incremental( - packages::Symbol...; - force = false, reuse = false, verbose = true, - debug = false, cc_flags = nothing -) - -Incrementally compile `package` into the current system image. -`force = true` will replace the old system image with the new one. -`compile_incremental` will run the `Package/test/runtests.jl` file to -record the functions getting compiled. The coverage of the Package's tests will -thus determine what is getting ahead of time compiled. -For a more explicit version of compile_incremental, see: -`compile_incremental(toml_path::String, snoopfile::String)` -``` - - -# more functionality - -```julia -# Or if you simply want to get a native system image e.g. when you have downloaded the generic Julia install: -force_native_image!() - -# Build an executable -build_executable( - "hello.jl", # Julia script containing a `julia_main` function, e.g. like `examples/hello.jl` - snoopfile = "call_functions.jl", # Julia script which calls functions that you want to make sure to have precompiled [optional] - builddir = "path/to/builddir" # that's where the compiled artifacts will end up [optional] -) - -# Build a shared library -build_shared_lib("hello.jl") -``` - - -# Static Julia Compiler - -Build shared libraries and executables from Julia code. - -Run `juliac.jl -h` for help: - -``` -usage: juliac.jl [-v] [-q] [-d ] [-n ] [-p ] [-c] - [-a] [-o] [-s] [-i] [-e] [-t] [-j] [-f ] [-r] - [-R] [-J ] [-H ] [--startup-file {yes|no}] - [--handle-signals {yes|no}] - [--sysimage-native-code {yes|no}] - [--compiled-modules {yes|no}] - [--depwarn {yes|no|error}] - [--warn-overwrite {yes|no}] - [--compile {yes|no|all|min}] [-C ] - [-O {0,1,2,3}] [-g ] [--inline {yes|no}] - [--check-bounds {yes|no}] [--math-mode {ieee,fast}] - [--cc ] [--cc-flag ] [--version] [-h] - juliaprog [cprog] - -Static Julia Compiler - -positional arguments: - juliaprog Julia program to compile - cprog C program to compile (required only when - building an executable, if not provided a - minimal driver program is used) - -optional arguments: - -v, --verbose increase verbosity - -q, --quiet suppress non-error messages - -d, --builddir build directory - -n, --outname output files basename - -p, --snoopfile - specify script calling functions to precompile - -c, --clean remove build directory - -a, --autodeps automatically build required dependencies - -o, --object build object file - -s, --shared build shared library - -i, --init-shared add `init_jl_runtime` and `exit_jl_runtime` to - shared library for runtime initialization - -e, --executable build executable file - -t, --rmtemp remove temporary build files - -j, --copy-julialibs copy Julia libraries to build directory - -f, --copy-file - copy file to build directory, can be repeated - for multiple files - -r, --release build in release mode, implies `-O3 -g0` - unless otherwise specified - -R, --Release perform a fully automated release build, - equivalent to `-atjr` - -J, --sysimage - start up with the given system image file - -H, --home set location of `julia` executable - --startup-file {yes|no} - load `~/.julia/config/startup.jl` - --handle-signals {yes|no} - enable or disable Julia's default signal - handlers - --sysimage-native-code {yes|no} - use native code from system image if available - --compiled-modules {yes|no} - enable or disable incremental precompilation - of modules - --depwarn {yes|no|error} - enable or disable syntax and method - deprecation warnings - --warn-overwrite {yes|no} - enable or disable method overwrite warnings - --compile {yes|no|all|min} - enable or disable JIT compiler, or request - exhaustive compilation - -C, --cpu-target - limit usage of CPU features up to - (implies default `--sysimage-native-code=no`) - -O, --optimize {0,1,2,3} - set the optimization level (type: Int64) - -g, --debug enable / set the level of debug info - generation (type: Int64) - --inline {yes|no} control whether inlining is permitted - --check-bounds {yes|no} - emit bounds checks always or never - --math-mode {ieee,fast} - disallow or enable unsafe floating point - optimizations - --cc system C compiler - --cc-flag pass custom flag to the system C compiler when - building a shared library or executable, can - be repeated for multiple flags - --version show version information and exit - -h, --help show this help message and exit - -examples: - juliac.jl -vae hello.jl # verbose, build executable and deps - juliac.jl -vae hello.jl prog.c # embed into user defined C program - juliac.jl -qo hello.jl # quiet, build object file only - juliac.jl -vosej hello.jl # build all and copy Julia libs - juliac.jl -vRe hello.jl # fully automated release build -``` - -## Building a shared library -`PackageCompiler` can compile a julia library into a linkable shared library, -built for a specific architecture, with a `C`-compatible ABI which can be -linked against from another program. This can be done either from the julia -api, `build_shared_lib("src/HelloLib.jl", "hello")`, or on the command line, -`$ juliac.jl -vas src/HelloLib.jl`. This will generate a shared library called -`builddir/libhello.{so,dylib,dll}` depending on your system. - -The provided julia file, `src/HelloLib.jl`, is `PackageCompiler`'s entry point -into the library, so it should be the "top level" library file. Any julia code -that it `include`s or `import`s will be compiled into the shared library. - -Note that for a julia function to be callable from `C`, it must be defined with -`Base.@ccallable`, e.g. `Base.@ccallable foo()::Cint = 3`. - -## Building an executable -To compile a Julia program into an executable, you can use either the julia -api, `build_executable("hello.jl", "hello")`, or the command line, `$ -juliac.jl -vae hello.jl`. - -The provided julia file, `hello.jl`, is `PackageCompiler`'s entry point into the -program, and should be the program's "main" file. Any julia code that it -`include`s or `import`s will be compiled into the shared library, which will be -linked against the provided `C` program to create an executable at -`builddir/hello`. - -If you choose to use the default `C` program, your julia code _must_ define -`julia_main` as its entry point. The resultant executable will start by calling -that function, so all of your program's logic should proceed from that -function. For example: - -``` -Base.@ccallable function julia_main(ARGS::Vector{String})::Cint - hello_main(ARGS) # call your program's logic. - return 0 -end -``` - -Please see -[examples/hello.jl](https://github.com/JuliaLang/PackageCompiler.jl/blob/master/examples/hello.jl) -for an example Julia program. - -### Notes - -1. The `juliac.jl` script is located in the `PackageCompiler` root - folder (`normpath(Base.find_package("PackageCompiler"), "..", "..")`). - -2. A shared library containing the system image `hello.so`, and a - driver binary `hello` are created in the `builddir` directory. - Running `hello` produces the following output: - -``` - hello, world - sin(0.0) = 0.0 - ┌─────────────────────────────────────────────────┐ - 1 │⠀⠀⠀⠀⠀⠀⠀⡠⠊⠉⠉⠉⠢⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ - │⠀⠀⠀⠀⠀⢠⠎⠀⠀⠀⠀⠀⠀⠘⢆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ - │⠀⠀⠀⠀⢠⠃⠀⠀⠀⠀⠀⠀⠀⠀⠀⠳⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ - │⠀⠀⠀⢠⠃⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠱⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ - │⠀⠀⢠⠃⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠳⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ - │⠀⢀⠇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢣⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ - │⠀⡎⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀│ - │⠼⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠬⢦⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⠤⢤│ - │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⡆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠇│ - │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠘⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡎⠀│ - │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠱⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡞⠀⠀│ - │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠱⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡜⠀⠀⠀│ - │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠱⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡞⠀⠀⠀⠀│ - │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠘⢆⠀⠀⠀⠀⠀⠀⢠⠎⠀⠀⠀⠀⠀│ - -1 │⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠑⢄⣀⣀⣀⠔⠁⠀⠀⠀⠀⠀⠀│ - └─────────────────────────────────────────────────┘ - 0 100 -``` - -3. Currently, before another program can call any of the functions defined in - the created shared library, that program must first initialize the julia - runtime. (See - [#53](https://github.com/JuliaLang/PackageCompiler.jl/issues/53) for - details.) - - -## Under the hood - -The `juliac.jl` script uses the `--output-o` switch to compile the user -script into object code, and then builds it into the system image -specified by the `-J` switch. This prepares an object file, which is -then linked into a shared library containing the system image and user -code. A driver script such as the one in `program.c` can then be used -to build a binary that runs the Julia code. - -Instead of a driver script, the generated system image can be embedded -into a larger program, see the -[Embedding Julia](https://docs.julialang.org/en/stable/manual/embedding/) -section of the Julia manual. Note that the name of the generated system -image (`"libhello"` for `hello.jl`) is accessible from C in the -preprocessor macro `JULIAC_PROGRAM_LIBNAME`. - -For more information on static Julia compilation see:\ -https://juliacomputing.com/blog/2016/02/09/static-julia.html - -## Side effects - -1. Using `PackageCompiler` makes it impossible to load changed package code automatically - it must be `eval`'ed in from the current session. This becomes a problem when developing packages. diff --git a/deps/build.jl b/deps/build.jl deleted file mode 100644 index bd8254b6..00000000 --- a/deps/build.jl +++ /dev/null @@ -1,73 +0,0 @@ -function verify_gcc(gcc) - try - return success(`$gcc --version`) - catch - return false - end -end - -if Sys.iswindows() - using WinRPM -end - -function build() - gccpath = "" - if isfile("deps.jl") - include("deps.jl") - if verify_gcc(gcc) - @info "GCC already installed and package already built" - return - else - rm("deps.jl") - end - end - - if haskey(ENV, "CC") - if !verify_gcc(`$(ENV["CC"]) -v`) - error("Using compiler override from environment variable CC = $(ENV["CC"]), but unable to run `$(ENV["CC"]) -v`.") - end - gccpath = ENV["CC"] - @info "Using `$gccpath` as C compiler from environment variable CC" - end - - @info "Looking for GCC" - - gccargs = `` - if verify_gcc("cc") - gccpath = "cc" - @info "Using `cc` as C compiler" - elseif Sys.iswindows() - sysroot = joinpath(WinRPM.installdir, "usr", "$(Sys.ARCH)-w64-mingw32", "sys-root") - gccpath = joinpath(sysroot, "mingw", "bin", "gcc.exe") - if !isfile(gccpath) - @info "Could not find GCC, installing using WinRPM" - WinRPM.install("gcc", yes = true) - end - if !isfile(gccpath) - error("Couldn't install gcc via WinRPM") - end - gccargs = `$gccargs --sysroot $sysroot` - @info "Using `gcc` from WinRPM as C compiler" - elseif Sys.isunix() && verify_gcc("gcc") - gccpath = "gcc" - @info "Using `gcc` as C compiler" - end - - if isempty(gccpath) - error(""" - Please make sure to provide a working gcc in your path! - You may need to install GCC. - """ - ) - Sys.isapple() && @info "You can install GCC using Homebrew by `brew install gcc`" - Sys.islinux() && @info """ - You can install GCC using `sudo apt-get install gcc` on Debian, - or use your distro's package manager. - """ - end - open("deps.jl", "w") do io - println(io, "const gcc = ", repr(`$gccpath $gccargs`)) - end -end - -build() diff --git a/examples/REQUIRE b/examples/REQUIRE deleted file mode 100644 index 9d79edf3..00000000 --- a/examples/REQUIRE +++ /dev/null @@ -1 +0,0 @@ -UnicodePlots diff --git a/examples/hello.jl b/examples/hello.jl deleted file mode 100644 index 6b6a7a2c..00000000 --- a/examples/hello.jl +++ /dev/null @@ -1,12 +0,0 @@ -module Hello - -using UnicodePlots - -Base.@ccallable function julia_main(ARGS::Vector{String})::Cint - println("hello, world") - @show sin(0.0) - println(lineplot(1:100, sin.(range(0, stop=2π, length=100)))) - return 0 -end - -end diff --git a/examples/program.c b/examples/program.c deleted file mode 100644 index 67a6d0a1..00000000 --- a/examples/program.c +++ /dev/null @@ -1,54 +0,0 @@ -// This file is a part of Julia. License is MIT: http://julialang.org/license - -// Standard headers -#include -#include - -// Julia headers (for initialization and gc commands) -#include "uv.h" -#include "julia.h" - -#ifdef JULIA_DEFINE_FAST_TLS // only available in Julia v0.7 and above -JULIA_DEFINE_FAST_TLS() -#endif - -// Declare C prototype of a function defined in Julia -extern int julia_main(jl_array_t*); - -// main function (windows UTF16 -> UTF8 argument conversion code copied from julia's ui/repl.c) -int main(int argc, char *argv[]) -{ - int retcode; - int i; - uv_setup_args(argc, argv); // no-op on Windows - - // initialization - libsupport_init(); - - // jl_options.compile_enabled = JL_OPTIONS_COMPILE_OFF; - // JULIAC_PROGRAM_LIBNAME defined on command-line for compilation - jl_options.image_file = JULIAC_PROGRAM_LIBNAME; - julia_init(JL_IMAGE_JULIA_HOME); - - // Initialize Core.ARGS with the full argv. - jl_set_ARGS(argc, argv); - - // Set PROGRAM_FILE to argv[0]. - jl_set_global(jl_base_module, - jl_symbol("PROGRAM_FILE"), (jl_value_t*)jl_cstr_to_string(argv[0])); - - // Set Base.ARGS to `String[ unsafe_string(argv[i]) for i = 1:argc ]` - jl_array_t *ARGS = (jl_array_t*)jl_get_global(jl_base_module, jl_symbol("ARGS")); - jl_array_grow_end(ARGS, argc - 1); - for (i = 1; i < argc; i++) { - jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]); - jl_arrayset(ARGS, s, i - 1); - } - - // call the work function, and get back a value - retcode = julia_main(ARGS); - - // Cleanup and gracefully exit - jl_atexit_hook(retcode); - return retcode; -} diff --git a/juliac.jl b/juliac.jl deleted file mode 100644 index 282b256a..00000000 --- a/juliac.jl +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env julia - -# ArgParse can't be a dependency here, -# so users need to install it. -using ArgParse, PackageCompiler - -Base.@ccallable function julia_main(args::Vector{String})::Cint - - s = ArgParseSettings("Static Julia Compiler", - version = "$(basename(@__FILE__)) version 0.7", - add_version = true) - - @add_arg_table s begin - "juliaprog" - arg_type = String - required = true - help = "Julia program to compile" - "cprog" - arg_type = String - help = "C program to compile (required only when building an executable, if not provided a minimal driver program is used)" - "--verbose", "-v" - action = :store_true - help = "increase verbosity" - "--quiet", "-q" - action = :store_true - help = "suppress non-error messages" - "--builddir", "-d" - arg_type = String - metavar = "" - help = "build directory" - "--outname", "-n" - arg_type = String - metavar = "" - help = "output files basename" - "--snoopfile", "-p" - arg_type = String - metavar = "" - help = "specify script calling functions to precompile" - "--clean", "-c" - action = :store_true - help = "remove build directory" - "--autodeps", "-a" - action = :store_true - help = "automatically build required dependencies" - "--object", "-o" - action = :store_true - help = "build object file" - "--shared", "-s" - action = :store_true - help = "build shared library" - "--init-shared", "-i" - action = :store_true - help = "add `init_jl_runtime` and `exit_jl_runtime` to shared library for runtime initialization" - "--executable", "-e" - action = :store_true - help = "build executable file" - "--rmtemp", "-t" - action = :store_true - help = "remove temporary build files" - "--copy-julialibs", "-j" - action = :store_true - help = "copy Julia libraries to build directory" - "--copy-file", "-f" - arg_type = String - action = :append_arg - dest_name = "copy-files" - metavar = "" - help = "copy file to build directory, can be repeated for multiple files" - "--release", "-r" - action = :store_true - help = "build in release mode, implies `-O3 -g0` unless otherwise specified" - "--Release", "-R" - action = :store_true - help = "perform a fully automated release build, equivalent to `-atjr`" - "--sysimage", "-J" - arg_type = String - metavar = "" - help = "start up with the given system image file" - "--home", "-H" - arg_type = String - metavar = "" - help = "set location of `julia` executable" - "--startup-file" - arg_type = String - metavar = "{yes|no}" - range_tester = (x -> x ∈ ("yes", "no")) - help = "load `~/.julia/config/startup.jl`" - "--handle-signals" - arg_type = String - metavar = "{yes|no}" - range_tester = (x -> x ∈ ("yes", "no")) - help = "enable or disable Julia's default signal handlers" - "--sysimage-native-code" - arg_type = String - metavar = "{yes|no}" - range_tester = (x -> x ∈ ("yes", "no")) - help = "use native code from system image if available" - "--compiled-modules" - arg_type = String - metavar = "{yes|no}" - range_tester = (x -> x ∈ ("yes", "no")) - help = "enable or disable incremental precompilation of modules" - "--depwarn" - arg_type = String - metavar = "{yes|no|error}" - range_tester = (x -> x ∈ ("yes", "no", "error")) - help = "enable or disable syntax and method deprecation warnings" - "--warn-overwrite" - arg_type = String - metavar = "{yes|no}" - range_tester = (x -> x ∈ ("yes", "no")) - help = "enable or disable method overwrite warnings" - "--compile" - arg_type = String - metavar = "{yes|no|all|min}" - range_tester = (x -> x ∈ ("yes", "no", "all", "min")) - help = "enable or disable JIT compiler, or request exhaustive compilation" - "--cpu-target", "-C" - arg_type = String - metavar = "" - help = "limit usage of CPU features up to (implies default `--sysimage-native-code=no`)" - "--optimize", "-O" - arg_type = Int - metavar = "{0,1,2,3}" - range_tester = (x -> x ∈ (0, 1, 2, 3)) - help = "set the optimization level" - "--debug", "-g" - arg_type = Int - metavar = "" - range_tester = (x -> x ∈ (0, 1, 2)) - help = "enable / set the level of debug info generation" - "--inline" - arg_type = String - metavar = "{yes|no}" - range_tester = (x -> x ∈ ("yes", "no")) - help = "control whether inlining is permitted" - "--check-bounds" - arg_type = String - metavar = "{yes|no}" - range_tester = (x -> x ∈ ("yes", "no")) - help = "emit bounds checks always or never" - "--math-mode" - arg_type = String - metavar = "{ieee,fast}" - range_tester = (x -> x ∈ ("ieee", "fast")) - help = "disallow or enable unsafe floating point optimizations" - "--cc" - arg_type = String - metavar = "" - help = "system C compiler" - "--cc-flag" - arg_type = String - action = :append_arg - dest_name = "cc-flags" - metavar = "" - help = "pass custom flag to the system C compiler when building a shared library or executable, can be repeated for multiple flags" - end - - s.epilog = """ - examples:\n - \ua0\ua0juliac.jl -vae hello.jl # verbose, build executable and deps\n - \ua0\ua0juliac.jl -vae hello.jl prog.c # embed into user defined C program\n - \ua0\ua0juliac.jl -qo hello.jl # quiet, build object file only\n - \ua0\ua0juliac.jl -vosej hello.jl # build all and copy Julia libs\n - \ua0\ua0juliac.jl -vRe hello.jl # fully automated release build - """ - - parsed_args = parse_args(args, s) - - parsed_args["copy-files"] == String[] && (parsed_args["copy-files"] = nothing) - - # TODO: in future it may be possible to broadcast dictionary indexing, see: https://discourse.julialang.org/t/accessing-multiple-values-of-a-dictionary/8648 - if getindex.(Ref(parsed_args), ["clean", "object", "shared", "executable", "rmtemp", "copy-julialibs", "copy-files"]) == [false, false, false, false, false, false, nothing] - parsed_args["quiet"] || println("nothing to do, exiting\ntry \"$(basename(@__FILE__)) -h\" for more information") - exit(0) - end - - juliaprog = pop!(parsed_args, "juliaprog") - filter!(kv -> kv.second !== nothing && kv.second !== false, parsed_args) - kw_args = [Symbol(replace(kv.first, "-" => "_")) => kv.second for kv in parsed_args] - - static_julia(juliaprog; kw_args...) - - return 0 -end - -if get(ENV, "COMPILE_STATIC", "false") == "false" - julia_main(ARGS) -end diff --git a/src/PackageCompiler.jl b/src/PackageCompiler.jl deleted file mode 100644 index 332fc17d..00000000 --- a/src/PackageCompiler.jl +++ /dev/null @@ -1,166 +0,0 @@ -module PackageCompiler - -using Pkg, Serialization, Libdl, UUIDs -using Pkg: TOML, Operations, Types - - -include("compiler_flags.jl") -include("static_julia.jl") -include("api.jl") -include("snooping.jl") -include("system_image.jl") -include("pkg.jl") -include("incremental.jl") - - -const sysimage_binaries = ("sys.$(Libdl.dlext)",) - -function copy_system_image(src, dest, ignore_missing = false) - for file in sysimage_binaries - # backup - srcfile = joinpath(src, file) - destfile = joinpath(dest, file) - if !isfile(srcfile) - ignore_missing && continue - error("No file: $srcfile") - end - if isfile(destfile) - if isfile(destfile * ".backup") - rm(destfile * ".backup", force = true) - end - mv(destfile, destfile * ".backup", force = true) - end - @info "Copying system image: $srcfile to $destfile" - cp(srcfile, destfile, force = true) - end -end - -julia_cpu_target(x) = error("CPU target needs to be a string or `nothing`") -julia_cpu_target(x::String) = x # TODO: match against available targets -function julia_cpu_target(::Nothing) - replace(Base.julia_cmd().exec[2], "-C" => "") -end - -""" -Reverts a forced compilation of the system image. -This will restore any previously backed up system image files, or -build a new, clean system image. -""" -function revert(debug = false) - syspath = default_sysimg_path(debug) - sysimg_backup = dirname(get_backup!(debug)) - copy_system_image(sysimg_backup, syspath) -end - -function get_root_dir(path) - path, name = splitdir(path) - if isempty(name) - return splitdir(path)[2] - else - name - end -end - -function sysimg_folder(files...) - base_path = normpath(abspath(joinpath(@__DIR__, "..", "sysimg"))) - isdir(base_path) || mkpath(base_path) - normpath(abspath(joinpath(base_path, files...))) -end - -function sysimgbackup_folder(files...) - backup = sysimg_folder("backup") - isdir(backup) || mkpath(backup) - sysimg_folder("backup", files...) -end - -function package_folder(package...) - packages = normpath(abspath(joinpath(@__DIR__, "..", "packages"))) - isdir(packages) || mkpath(packages) - normpath(abspath(joinpath(packages, package...))) -end - -""" - compile_package(packages...; kw_args...) - -with packages being either a string naming a package, or a tuple `(package_name, precompile_file)`. -If no precompile file is given, it will use the packages `runtests.jl`, which is a good canditate -for figuring out what functions to compile! -""" -function compile_package(packages...; kw_args...) - args = map(packages) do package - # If no explicit path to a seperate precompile file, use runtests - isa(package, String) && return (package, "test/runtests.jl") - isa(package, Tuple{String, String}) && return package - error("Unrecognized package. Use `packagename::String`, or `(packagename::String, rel_path_to_testfile::String)`. Found: `$package`") - end - compile_package(args...; kw_args...) -end - -""" - compile_package( - packages::Tuple{String, String}...; - force = false, reuse = false, debug = false, cpu_target = nothing, - additional_packages = Symbol[] - ) - -Compile a list of packages. Each package comes as a tuple of `(package_name, precompile_file)` -where the precompile file should contain all function calls, that should get compiled into the system image. -Usually the `runtests.jl` file is a good candidate, since it should run all important functions of a package. -You can pass `additional_packages` a vector of symbols with package names, to help AOT compiling -uninstalled, recursive dependencies of `packages`. Look at `compile_incremental` to -use a toml instead. -""" -function compile_package( - packages::Tuple{String, String}...; - force = false, reuse = false, debug = false, - cpu_target = nothing, verbose = false - ) - userimg = sysimg_folder("precompile.jl") - if !reuse - # TODO that's a pretty weak way to check that it's not a path... - ispackage = all(x-> !occursin(Base.Filesystem.path_separator, x), first.(packages)) - isruntests = all(x-> x == "test/runtests.jl", last.(packages)) - if ispackage && isruntests - snoop_packages(Symbol.(first.(packages))...; file = userimg) - else - ispackage || @warn "Giving path to package deprecated. Use Package name!" - isruntests || @warn "Giving a snoopfile is deprecated. Use runtests from package!" - end - end - !isfile(userimg) && reuse && error("Nothing to reuse. Please run `compile_package(reuse = true)`") - image_path = sysimg_folder() - build_sysimg(image_path, userimg, cpu_target=cpu_target, verbose = verbose) - imgfile = joinpath(image_path, "sys.$(Libdl.dlext)") - syspath = joinpath(default_sysimg_path(debug), "sys.$(Libdl.dlext)") - if force - try - backup = syspath * ".packagecompiler_backup" - isfile(backup) || mv(syspath, backup) - cp(imgfile, syspath, force=true) - @info """ - Replaced system image successfully. Next start of julia will load the newly compiled system image. - If you encounter any errors with the new julia image, try `PackageCompiler.revert([debug = false])`. - """ - catch e - @warn "An error occured while replacing sysimg files:" error = e - @info "Recovering old system image from backup" - # if any file is missing in default system image, revert! - if !isfile(syspath) - @info "$syspath missing. Reverting!" - revert(debug) - end - end - else - @info """ - Not replacing system image. - You can start julia with $(`julia -J $imgfile`) at a posix shell to load the compiled files. - """ - end - imgfile -end - - - -export compile_package, revert, force_native_image!, executable_ext, build_executable, build_shared_lib, static_julia, compile_incremental - -end # module diff --git a/src/api.jl b/src/api.jl deleted file mode 100644 index 6aae6289..00000000 --- a/src/api.jl +++ /dev/null @@ -1,125 +0,0 @@ -""" - build_sysimg(sysimg_path=default_sysimg_path(), userimg_path=nothing; cpu_target="native", force=false) - -Rebuild the system image. Store it in `sysimg_path`, which defaults to a file named `sys.ji` -that sits in the same folder as `libjulia.{so,dylib}`, except on Windows where it defaults -to `Sys.BINDIR/../lib/julia/sys.ji`. Use the cpu instruction set given by `cpu_target`. -Valid CPU targets are the same as for the `-C` option to `julia`, or the `-march` option to -`gcc`. Defaults to `native`, which means to use all CPU instructions available on the -current processor. Include the user image file given by `userimg_path`, which should contain -directives such as `using MyPackage` to include that package in the new system image. New -system image will not replace an older image unless `force` is set to true. -""" -function build_sysimg( - sysimg_path = default_sysimg_path(), userimg_path = nothing; - verbose = false, quiet = false, release = false, - home = nothing, startup_file = nothing, handle_signals = nothing, - sysimage_native_code = nothing, compiled_modules = nothing, - depwarn = nothing, warn_overwrite = nothing, - compile = nothing, cpu_target = nothing, optimize = nothing, debug = nothing, - inline = nothing, check_bounds = nothing, math_mode = nothing, - cc = nothing, cc_flags = nothing - ) - # build vanilla backup system image - clean_sysimg = get_backup!(occursin("debug", basename(Base.julia_cmd().exec[1])), cpu_target) - static_julia( - userimg_path, verbose = verbose, quiet = quiet, builddir = sysimg_path, outname = "sys", - autodeps = true, shared = true, release = release, - sysimage = clean_sysimg, home = home, startup_file = startup_file, handle_signals = handle_signals, - sysimage_native_code = sysimage_native_code, compiled_modules = compiled_modules, - depwarn = depwarn, warn_overwrite = warn_overwrite, - compile = compile, cpu_target = cpu_target, optimize = optimize, debug = debug, - inline = inline, check_bounds = check_bounds, math_mode = math_mode, - cc = cc, cc_flags = cc_flags - ) -end - -""" - build_shared_lib( - julia_program, output_name = nothing; - snoopfile = nothing, builddir = nothing, verbose = false, quiet = false, - init_shared = false, copy_julialibs = true, copy_files = nothing, release = false, Release = false, - sysimage = nothing, home = nothing, startup_file = nothing, handle_signals = nothing, - sysimage_native_code = nothing, compiled_modules = nothing, - depwarn = nothing, warn_overwrite = nothing, - compile = nothing, cpu_target = nothing, optimize = nothing, debug = nothing, - inline = nothing, check_bounds = nothing, math_mode = nothing, - cc = nothing, cc_flags = nothing - ) - `julia_program` needs to be a Julia script containing a `julia_main` function, e.g. like `examples/hello.jl` - `snoopfile` is optional and can be a Julia script which calls functions that you want to make sure to have precompiled - `builddir` is where the compiled artifacts will end up -""" -function build_shared_lib( - julia_program, output_name = nothing; - snoopfile = nothing, builddir = nothing, verbose = false, quiet = false, - init_shared = false, copy_julialibs = true, copy_files = nothing, release = false, Release = false, - sysimage = nothing, home = nothing, startup_file = nothing, handle_signals = nothing, - sysimage_native_code = nothing, compiled_modules = nothing, - depwarn = nothing, warn_overwrite = nothing, - compile = nothing, cpu_target = nothing, optimize = nothing, debug = nothing, - inline = nothing, check_bounds = nothing, math_mode = nothing, - cc = nothing, cc_flags = nothing - ) - static_julia( - julia_program, verbose = verbose, quiet = quiet, - builddir = builddir, outname = output_name, snoopfile = snoopfile, autodeps = true, shared = true, - init_shared = init_shared, copy_julialibs = copy_julialibs, copy_files = copy_files, release = release, Release = release, - sysimage = sysimage, home = home, startup_file = startup_file, handle_signals = handle_signals, - sysimage_native_code = sysimage_native_code, compiled_modules = compiled_modules, - depwarn = depwarn, warn_overwrite = warn_overwrite, - compile = compile, cpu_target = cpu_target, optimize = optimize, debug = debug, - inline = inline, check_bounds = check_bounds, math_mode = math_mode, - cc = cc, cc_flags = cc_flags - ) -end - -""" - build_executable( - julia_program, output_name = nothing, c_program = nothing; - snoopfile = nothing, builddir = nothing, verbose = false, quiet = false, - copy_julialibs = true, copy_files = nothing, release = false, Release = false, - sysimage = nothing, home = nothing, startup_file = nothing, handle_signals = nothing, - sysimage_native_code = nothing, compiled_modules = nothing, - depwarn = nothing, warn_overwrite = nothing, - compile = nothing, cpu_target = nothing, optimize = nothing, debug = nothing, - inline = nothing, check_bounds = nothing, math_mode = nothing, - cc = nothing, cc_flags = nothing - ) - `julia_program` needs to be a Julia script containing a `julia_main` function, e.g. like `examples/hello.jl` - `snoopfile` is optional and can be a Julia script which calls functions that you want to make sure to have precompiled - `builddir` is where the compiled artifacts will end up -""" -function build_executable( - julia_program, output_name = nothing, c_program = nothing; - snoopfile = nothing, builddir = nothing, verbose = false, quiet = false, - copy_julialibs = true, copy_files = nothing, release = false, Release = false, - sysimage = nothing, home = nothing, startup_file = nothing, handle_signals = nothing, - sysimage_native_code = nothing, compiled_modules = nothing, - depwarn = nothing, warn_overwrite = nothing, - compile = nothing, cpu_target = nothing, optimize = nothing, debug = nothing, - inline = nothing, check_bounds = nothing, math_mode = nothing, - cc = nothing, cc_flags = nothing - ) - static_julia( - julia_program, cprog = c_program, verbose = verbose, quiet = quiet, - builddir = builddir, outname = output_name, snoopfile = snoopfile, autodeps = true, executable = true, - copy_julialibs = copy_julialibs, copy_files = copy_files, release = release, Release = release, - sysimage = sysimage, home = home, startup_file = startup_file, handle_signals = handle_signals, - sysimage_native_code = sysimage_native_code, compiled_modules = compiled_modules, - depwarn = depwarn, warn_overwrite = warn_overwrite, - compile = compile, cpu_target = cpu_target, optimize = optimize, debug = debug, - inline = inline, check_bounds = check_bounds, math_mode = math_mode, - cc = cc, cc_flags = cc_flags - ) -end - -""" - force_native_image!() -Builds a clean system image, similar to a fresh Julia install. -Can also be used to build a native system image for a downloaded cross compiled julia binary. -""" -function force_native_image!(debug = false) # debug is ignored right now - sysimg = get_backup!(debug, "native") - copy_system_image(dirname(sysimg), default_sysimg_path(debug)) -end diff --git a/src/compiler_flags.jl b/src/compiler_flags.jl deleted file mode 100644 index 65d2b0bb..00000000 --- a/src/compiler_flags.jl +++ /dev/null @@ -1,265 +0,0 @@ -# This code is derived from `julia-config.jl` (part of Julia) and should be kept aligned with it. - -threadingOn() = ccall(:jl_threading_enabled, Cint, ()) != 0 - -function shell_escape(str) - str = replace(str, "'" => "'\''") - return "'$str'" -end - -function libDir() - return if ccall(:jl_is_debugbuild, Cint, ()) != 0 - dirname(abspath(Libdl.dlpath("libjulia-debug"))) - else - dirname(abspath(Libdl.dlpath("libjulia"))) - end -end - -private_libDir() = abspath(Sys.BINDIR, Base.PRIVATE_LIBDIR) - -function includeDir() - return abspath(Sys.BINDIR, Base.INCLUDEDIR, "julia") -end - -function ldflags() - fl = "-L$(shell_escape(libDir()))" - if Sys.iswindows() - fl = fl * " -Wl,--stack,8388608" - elseif Sys.islinux() - fl = fl * " -Wl,--export-dynamic" - end - return fl -end - -function ldlibs() - libname = if ccall(:jl_is_debugbuild, Cint, ()) != 0 - "julia-debug" - else - "julia" - end - if Sys.isunix() - return "-Wl,-rpath,$(shell_escape(libDir())) -Wl,-rpath,$(shell_escape(private_libDir())) -l$libname" - else - return "-l$libname -lopenlibm" - end -end - -function cflags() - flags = IOBuffer() - print(flags, "-std=gnu99") - include = shell_escape(includeDir()) - print(flags, " -I", include) - if threadingOn() - print(flags, " -DJULIA_ENABLE_THREADING=1") - end - if Sys.isunix() - print(flags, " -fPIC") - end - return String(take!(flags)) -end - -function allflags() - return "$(cflags()) $(ldflags()) $(ldlibs())" -end - - - - -const short_flag_to_jloptions = Dict( - "C" => :cpu_target, - "J" => :image_file, - "O" => :opt_level, - "g" => :debug_level -) - -const flag_to_jloptions = Dict( - "check-bounds" => :check_bounds, - "code-coverage" => :code_coverage, - "compile" => :compile_enabled, - "compiled-modules" => :use_compiled_modules, - "cpu-target" => :cpu_target, - "depwarn" => :depwarn, - "handle-signals" => :handle_signals, - "history-file" => :historyfile, - "machine-file" => :machine_file, - "math-mode" => :fast_math, - "optimize" => :opt_level, - "output-bc" => :outputbc, - "output-ji" => :outputji, - "output-jitbc" => :outputjitbc, - "output-o" => :outputo, - "output-unoptbc" => :outputunoptbc, - "project" => :project, - "startup-file" => :startupfile, - "sysimage" => :image_file, - "sysimage-native-code" => :use_sysimage_native_code, - "trace-compile" => :trace_compile, - "track-allocation" => :malloc_log, - "warn-overwrite" => :warn_overwrite, - "inline" => :can_inline -) - -const jl_options_to_flag = Dict( - :can_inline => "inline", - :handle_signals => "handle-signals", - :opt_level => "optimize", - :depwarn => "depwarn", - :malloc_log => "track-allocation", - :outputo => "output-o", - :startupfile => "startup-file", - :compile_enabled => "compile", - :trace_compile => "trace-compile", - :check_bounds => "check-bounds", - :outputji => "output-ji", - :use_sysimage_native_code => "sysimage-native-code", - :outputunoptbc => "output-unoptbc", - :historyfile => "history-file", - :outputbc => "output-bc", - :warn_overwrite => "warn-overwrite", - :machine_file => "machine-file", - :code_coverage => "code-coverage", - :image_file => "sysimage", - :cpu_target => "cpu-target", - :outputjitbc => "output-jitbc", - :project => "project", - :fast_math => "math-mode", - :use_compiled_modules => "compiled-modules", - :debug_level => "g" -) - -const flags_with_cmdval = Set([ - :handle_signals, - :use_sysimage_native_code, - :depwarn, - :can_inline, - :historyfile, - :startupfile, - :use_compiled_modules, - :warn_overwrite, - :check_bounds, - :fast_math, - :compile_enabled, - :malloc_log, - :code_coverage -]) - -function to_cmd_val(key::Symbol, val) - # undocumented auto!? well we can only skip it I guess - if key in (:depwarn, :warn_overwrite) - val == 0 && return "no" - val == 1 && return "yes" - val == 2 && return "error" - end - if key in (:code_coverage, :malloc_log) - val == 0 && return "none" - val == 1 && return "user" - val == 2 && return "all" - end - if key == :compile_enabled - val == 0 && return "no" - val == 1 && return "yes" - val == 2 && return "all" - end - if key == :fast_math - val == 0 && return "ieee" - val == 1 && return "fast" - end - val in (0, -1) && return "" - val == 1 && return "yes" - val == 2 && return "no" -end - - -function jl_option_value(opts, key) - value = getfield(opts, key) - if value isa Ptr{UInt8} - return value == C_NULL ? "" : unsafe_string(value) - end - if key in flags_with_cmdval - return to_cmd_val(key, value) - end - return value -end - -is_short_flag(flag) = haskey(short_flag_to_jloptions, flag) - -function jl_option_key(flag::Symbol) - # if symbol is used, we can also check the fields directly. - # TODO should we also do this for strings? - flag in fieldnames(Base.JLOptions) && return flag - jl_option_key(string(flag)) -end - -function jl_option_key(flag::String) - haskey(short_flag_to_jloptions, flag) && return short_flag_to_jloptions[flag] - haskey(flag_to_jloptions, flag) && return flag_to_jloptions[flag] - m_flag = replace(flag, "_" => "-") - haskey(flag_to_jloptions, m_flag) && return flag_to_jloptions[m_flag] - flags = replace.([keys(flag_to_jloptions)..., keys(short_flag_to_jloptions)...], ("-" => "_",)) - error("Flag $flag not a valid Julia Options. Valid flags are:\n$(join(flags, " "))") -end - -""" - extract_flag(flag, jl_cmd = Base.julia_cmd()) - -Extracts the value for `flag` from `jl_cmd`. -""" -function jl_flag_value(flag, jl_options = Base.JLOptions()) - jl_option_value(jl_options, jl_option_key(flag)) -end - -current_systemimage() = jl_flag_value("J") - -""" - run_julia( - code::String; - g = nothing, O = nothing, output_o = nothing, J = nothing, - startup_file = "no" - ) - -Run `code` in a julia command. -You can overwrite any julia command line flag by setting it to a value. -If the flag has the value `nothing`, the value of the flag of the current julia process is used. -""" -function run_julia(code::String; kw...) - run(julia_code_cmd(code; kw...)) -end - -function jl_command(flag, value) - (value === nothing || isempty(value)) && return "" - if is_short_flag(flag) - string("-", flag, value) - else - string("--", flag, "=", value) - end -end - - -function push_command!(cmd, flag, value) - command = jl_command(flag, value) - isempty(command) || push!(cmd.exec, command) -end - -function julia_code_cmd( - code::String, jl_options = Base.JLOptions(); - kw... - ) - jl_cmd = Base.julia_cmd() - cmd = `$(jl_cmd.exec[1])` # extract julia exe - # Add the commands from the keyword arguments - for (k, v) in kw - jl_key = jl_option_key(k) - flag = jl_options_to_flag[jl_key] - push_command!(cmd, flag, v) - end - # add remaining commands from JLOptions - for key in setdiff(keys(jl_options_to_flag), keys(kw)) - flag = jl_options_to_flag[key] - push_command!(cmd, flag, jl_option_value(jl_options, key)) - end - # for better debug, let's not make a tmp file which would get lost! - file = sysimg_folder("run_julia_code.jl") - open(io-> println(io, code), file, "w") - push!(cmd.exec, file) - cmd -end diff --git a/src/incremental.jl b/src/incremental.jl deleted file mode 100644 index d87e29b8..00000000 --- a/src/incremental.jl +++ /dev/null @@ -1,178 +0,0 @@ -""" -Init basic C libraries -""" -function InitBase() - """ - Base.__init__() - Sys.__init__() #fix https://github.com/JuliaLang/julia/issues/30479 - """ -end - -""" -# Initialize REPL module for Docs -""" -function InitREPL() - """ - using REPL - Base.REPL_MODULE_REF[] = REPL - """ -end -function Include(path) - """ - Mod = @eval module \$(gensym("anon_module")) end - # Include into anonymous module to not polute namespace - Mod.include($(repr(path))) - """ -end - -""" -Exit hooks can get serialized and therefore end up in weird behaviour -When incrementally compiling -""" -function ExitHooksStart() - """ - atexit_hook_copy = copy(Base.atexit_hooks) # make backup - # clean state so that any package we use can carelessly call atexit - empty!(Base.atexit_hooks) - """ -end - -function ExitHooksEnd() - """ - Base._atexit() # run all exit hooks we registered during precompile - empty!(Base.atexit_hooks) # don't serialize the exit hooks we run + added - # atexit_hook_copy should be empty, but who knows what base will do in the future - append!(Base.atexit_hooks, atexit_hook_copy) - """ -end - -function PackageCallbacksStart() - """ - package_callbacks_copy = copy(Base.package_callbacks) - empty!(Base.package_callbacks) - """ -end - -function PackageCallbacksEnd() - """ - empty!(Base.package_callbacks) - append!(Base.package_callbacks, package_callbacks_copy) - """ -end - -function REPLHooksStart() - """ - repl_hooks_copy = copy(Base.repl_hooks) - empty!(Base.repl_hooks) - """ -end - -function REPLHooksEnd() - """ - empty!(Base.repl_hooks) - append!(Base.repl_hooks, repl_hooks_copy) - """ -end - -function DisableLibraryThreadingHooksStart() - """ - if isdefined(Base, :disable_library_threading_hooks) - disable_library_threading_hooks_copy = copy(Base.disable_library_threading_hooks) - empty!(Base.disable_library_threading_hooks) - end - """ -end - -function DisableLibraryThreadingHooksEnd() - """ - if isdefined(Base, :disable_library_threading_hooks) - empty!(Base.disable_library_threading_hooks) - append!(Base.disable_library_threading_hooks, disable_library_threading_hooks_copy) - end - """ -end - -""" -The command to pass to julia --output-o, that runs the julia code in `path` during compilation. -""" -function PrecompileCommand(path) - ExitHooksStart() * - PackageCallbacksStart() * - REPLHooksStart() * - DisableLibraryThreadingHooksStart() * - InitBase() * - InitREPL() * - Include(path) * - DisableLibraryThreadingHooksEnd() * - REPLHooksEnd() * - PackageCallbacksEnd() * - ExitHooksEnd() -end - - - -""" - compile_incremental( - toml_path::String, snoopfile::String; - force = false, precompile_file = nothing, verbose = true, - debug = false, cc_flags = nothing - ) - - Extract all calls from `snoopfile` and ahead of time compiles them - incrementally into the current system image. - `force = true` will replace the old system image with the new one. - The argument `toml_path` should contain a project file of the packages that `snoopfile` explicitly uses. - Implicitly used packages & modules don't need to be contained! - - To compile just a single package, see the simpler version `compile_incremental(package::Symbol)`: -""" -function compile_incremental( - toml_path::Union{String, Nothing}, precompiles::String; - force = false, verbose = true, - debug = false, cc_flags = nothing - ) - systemp = sysimg_folder("sys.a") - sysout = sysimg_folder("sys.$(Libdl.dlext)") - code = PrecompileCommand(precompiles) - run_julia( - code, O = 3, output_o = systemp, g = 1, - track_allocation = "none", startup_file = "no", code_coverage = "none" - ) - build_shared(sysout, systemp, false, sysimg_folder(), verbose, "3", debug, system_compiler, cc_flags) - curr_syso = current_systemimage() - force && cp(sysout, curr_syso, force = true) - return sysout, curr_syso -end - -""" - compile_incremental( - packages::Symbol...; - force = false, reuse = false, verbose = true, - debug = false, cc_flags = nothing, - blacklist::Vector = [] - ) - - Incrementally compile `package` into the current system image. - `force = true` will replace the old system image with the new one. - This process requires a script that julia will run in order to determine - which functions to compile. A package may define a script called `snoopfile.jl` - for this purpose. If this file cannot be found the package's test script - `Package/test/runtests.jl` will be used. `compile_incremental` will search - for `snoopfile.jl` in the package's root directory and in the folders - `Package/src` and `Package/snoop`. For a more explicit version of compile_incremental, - see: `compile_incremental(toml_path::String, snoopfile::String)` - - Not all packages can currently be compiled into the system image. By default, - `compile_incremental(:Package) will also compile all of Package's dependencies. - It can still be desirable to compile packages with dependencies that cannot be - compiled. For this reason `compile_incremental` offers - the ability for the user to pass a list of blacklisted packages - that will be ignored during the compilation process. These are passed as a - vector of package names (defined as either strings or symbols) using the - `blacklist keyword argument` -""" -function compile_incremental(pkg::Symbol, packages::Symbol...; - blacklist::Vector=[], kw...) - toml, precompile = snoop_packages(pkg, packages...; blacklist=blacklist) - compile_incremental(toml, precompile; kw...) -end diff --git a/src/pkg.jl b/src/pkg.jl deleted file mode 100644 index 8f5f59fb..00000000 --- a/src/pkg.jl +++ /dev/null @@ -1,178 +0,0 @@ -#= -genfile & create_project_from_require have been taken from the PR -https://github.com/JuliaLang/PkgDev.jl/pull/144 -which was created by https://github.com/KristofferC - -THIS IS JUST A TEMPORARY SOLUTION FOR PACKAGES WITHOUT A TOML AND WILL GET MOVED OUT! -=# - -function packages_from_require(reqfile::String) - ctx = Pkg.Types.Context() - pkgs = Types.PackageSpec[] - compatibility = Pair{String, String}[] - for r in Pkg.Pkg2.Reqs.read(reqfile) - r isa Pkg.Pkg2.Reqs.Requirement || continue - r.package == "julia" && continue - push!(pkgs, Types.PackageSpec(r.package)) - intervals = r.versions.intervals - if length(intervals) != 1 - @warn "Project.toml creator cannot handle multiple requirements for $(r.package), ignoring" - else - l = intervals[1].lower - h = intervals[1].upper - if l != v"0.0.0-" - # no upper bound - if h == typemax(VersionNumber) - push!(compatibility, r.package => string(">=", VersionNumber(l.major, l.minor, l.patch))) - else # assume semver - push!(compatibility, r.package => string(">=", VersionNumber(l.major, l.minor, l.patch), ", ", - "<", VersionNumber(h.major, h.minor, h.patch))) - end - end - end - end - Operations.registry_resolve!(ctx.env, pkgs) - Operations.ensure_resolved(ctx.env, pkgs) - pkgs -end -function create_project_from_require(pkgname::String, path::String, toml_path::String) - ctx = Pkg.Types.Context() - # Package data - path = abspath(path) - mainpkg = Types.PackageSpec(pkgname) - Pkg.Operations.registry_resolve!(ctx.env, [mainpkg]) - if !Operations.has_uuid(mainpkg) - uuid = UUIDs.uuid1() - @info "Unregistered package $pkgname, giving it a new UUID: $uuid" - mainpkg.version = v"0.1.0" - else - uuid = mainpkg.uuid - @info "Registered package $pkgname, using already given UUID: $(mainpkg.uuid)" - Pkg.Operations.set_maximum_version_registry!(ctx.env, mainpkg) - v = mainpkg.version - # Remove the build - mainpkg.version = VersionNumber(v.major, v.minor, v.patch) - end - # Dependency data - dep_pkgs = Types.PackageSpec[] - test_pkgs = Types.PackageSpec[] - compatibility = Pair{String, String}[] - - reqfiles = [joinpath(path, "REQUIRE"), joinpath(path, "test", "REQUIRE")] - for (reqfile, pkgs) in zip(reqfiles, [dep_pkgs, test_pkgs]) - if isfile(reqfile) - append!(pkgs, packages_from_require(reqfile)) - end - end - - stdlib_deps = Pkg.Operations.find_stdlib_deps(ctx, path) - for (stdlib_uuid, stdlib) in stdlib_deps - pkg = Types.PackageSpec(stdlib, stdlib_uuid) - if stdlib == "Test" - push!(test_pkgs, pkg) - else - push!(dep_pkgs, pkg) - end - end - - # Write project - - project = Dict( - "name" => pkgname, - "uuid" => string(uuid), - "version" => string(mainpkg.version), - "deps" => Dict(pkg.name => string(pkg.uuid) for pkg in dep_pkgs) - ) - - if !isempty(compatibility) - project["compat"] = - Dict(name => ver for (name, ver) in compatibility) - end - - if !isempty(test_pkgs) - project["extras"] = Dict(pkg.name => string(pkg.uuid) for pkg in test_pkgs) - project["targets"] = Dict("test" => [pkg.name for pkg in test_pkgs]) - end - - open(toml_path, "w") do io - Pkg.TOML.print(io, project, sorted=true, by=key -> (Types.project_key_order(key), key)) - end -end - -function package_toml(package::Symbol) - pstr = string(package) - # could use eval using here?! Not sure what is actually better - pkg_module = Base.require(Module(), package) - pkg_root = normpath(joinpath(dirname(pathof(pkg_module)), "..")) - toml = joinpath(pkg_root, "Project.toml") - # Check for snoopfile and fall back to runtests.jl - # if it can't be found - snoopfile = get_snoopfile(pkg_root) - # We will create a new toml, based that will include all test dependencies etc - # We're also using the precompile toml as a temp toml for packages not having a toml - precompile_toml = package_folder(pstr, "Project.toml") - isdir(dirname(precompile_toml)) || mkpath(dirname(precompile_toml)) - test_deps = Dict() - if !isfile(toml) - create_project_from_require(pstr, pkg_root, precompile_toml) - else - testreq = joinpath(pkg_root, "test", "REQUIRE") - if isfile(testreq) - pkgs = packages_from_require(testreq) - test_deps = Dict(pkg.name => string(pkg.uuid) for pkg in pkgs) - end - cp(toml, precompile_toml, force = true) - chmod(precompile_toml, 0o644) - end - # remove any old manifest - if isfile(package_folder(pstr, "Manifest.toml")) - rm(package_folder(pstr, "Manifest.toml")) - end - # add ourselves as dependencies and ensure we have a manifest - run_julia(""" - using Pkg - Pkg.instantiate() - pkg"add PackageCompiler Pkg" - """, project = precompile_toml, startup_file = "no") - - toml = TOML.parsefile(precompile_toml) - - deps = merge(get(toml, "deps", Dict()), test_deps) - # Add the package itself - deps[toml["name"]] = toml["uuid"] - # Add the packages we need - test_deps = get(toml, "extras", Dict()) - compile_toml = Dict() - compile_toml["name"] = string(package, "Precompile") - compile_toml["deps"] = merge(test_deps, deps) - if haskey(toml, "compat") - compile_toml["compat"] = toml["compat"] - end - write_toml(precompile_toml, compile_toml) - precompile_toml, snoopfile -end - -function write_toml(path, dict) - open(path, "w") do io - TOML.print( - io, dict, - sorted = true, by = key-> (Types.project_key_order(key), key) - ) - end -end - -function get_snoopfile(pkg_root) - snoopfileroot = joinpath(pkg_root,"snoopfile.jl") - snoopfilesnoopdir = joinpath(pkg_root, "snoop", "snoopfile.jl") - snoopfilesrc = joinpath(pkg_root, "src", "snoopfile.jl") - if isfile(snoopfileroot) - snoopfile = snoopfileroot - elseif isfile(snoopfilesnoopdir) - snoopfile = snoopfilesnoopdir - elseif isfile(snoopfilesrc) - snoopfile = snoopfilesrc - else - snoopfile = joinpath(pkg_root, "test", "runtests.jl") - end - return snoopfile -end diff --git a/src/snooping.jl b/src/snooping.jl deleted file mode 100644 index f53573d1..00000000 --- a/src/snooping.jl +++ /dev/null @@ -1,149 +0,0 @@ - - -function snoop(package, tomlpath, snoopfile, outputfile, reuse = false, blacklist = []) - - command = """ - using Pkg, PackageCompiler - """ - - if tomlpath !== nothing - command *= """ - empty!(Base.LOAD_PATH) - # Take LOAD_PATH from parent process - append!(Base.LOAD_PATH, $(repr(Base.LOAD_PATH))) - Pkg.activate($(repr(tomlpath))) - Pkg.resolve() - Pkg.instantiate() - """ - end - - command *= """ - # let's wrap the snoop file in a try catch... - # This way we still do some snooping even if there is an error in the tests! - try - include($(repr(snoopfile))) - catch e - @warn("Snoop file errored. Precompile statements were recorded untill error!", exception = e) - end - """ - - # let's use a file in the PackageCompiler dir, - # so it doesn't get lost if later steps fail - tmp_file = package_folder("precompile_tmp.jl") - if !reuse - run_julia(command, compile = "all", O = 0, g = 1, trace_compile = tmp_file, startup_file = "no") - end - used_packages = Set{String}() # e.g. from test/REQUIRE - if package !== nothing - push!(used_packages, string(package)) - end - usings = "" - if tomlpath !== nothing - # add toml packages, in case extract_used_packages misses a package - deps = get(TOML.parsefile(tomlpath), "deps", Dict{String, Any}()) - union!(used_packages, string.(keys(deps))) - end - if !isempty(used_packages) - packages = join(setdiff(used_packages,string.(blacklist)), ", ") - usings *= """ - using $packages - for Mod in [$packages] - isdefined(Mod, :__init__) && Mod.__init__() - end - """ - end - - line_idx = 0; missed = 0 - open(outputfile, "w") do io - println(io, """ - # We need to use all used packages in the precompile file for maximum - # usage of the precompile statements. - # Since this can be any recursive dependency of the package we AOT compile, - # we decided to just use them without installing them. An added - # benefit is, that we can call __init__ this way more easily, since - # incremental sysimage compilation won't call __init__ on `using` - # https://github.com/JuliaLang/julia/issues/22910 - $usings - # bring recursive dependencies of used packages and standard libraries into namespace - for Mod in Base.loaded_modules_array() - if !Core.isdefined(@__MODULE__, nameof(Mod)) - Core.eval(@__MODULE__, Expr(:const, Expr(:(=), nameof(Mod), Mod))) - end - end - """) - for line in eachline(tmp_file) - line_idx += 1 - # replace function instances, which turn up as typeof(func)(). - # TODO why would they be represented in a way that doesn't actually work? - line = replace(line, r"typeof\(([\u00A0-\uFFFF\w_!´\.]*@?[\u00A0-\uFFFF\w_!´]+)\)\(\)" => s"\1") - # Is this ridicilous? Yes, it is! But we need a unique symbol to replace `_`, - # which otherwise ends up as an uncatchable syntax error - line = replace(line, r"\b_\b" => "🐃") - try - expr = Meta.parse(line, raise = true) - if expr.head != :incomplete - # after all this, we still need to wrap into try catch, - # since some anonymous symbols won't be found... - println(io, "try;", line, "; catch e; @debug \"couldn't precompile statement $line_idx\" exception = e; end") - else - missed += 1 - @debug "Incomplete line in precompile file: $line" - end - catch e - missed += 1 - @debug "Parse error in precompile file: $line" exception=e - end - end - end - @info "used $(line_idx - missed) out of $line_idx precompile statements" - outputfile -end - - -function snoop_packages(packages::Symbol...; blacklist = [], file = package_folder("incremental_precompile.jl")) - finaltoml = Dict{Any, Any}( - "deps" => Dict(), - "compat" => Dict(), - ) - toml_path = package_folder("Project.toml") - manifest_dict = Dict{String, Vector{Dict{String, Any}}}() - open(file, "w") do compile_io - println(compile_io, "# Precompile file for $(join(packages, " "))") - # make sure we have all packages from toml installed - println(compile_io, """ - using Pkg - empty!(Base.LOAD_PATH) - # Take LOAD_PATH from parent process - append!(Base.LOAD_PATH, $(repr(Base.LOAD_PATH))) - Pkg.activate($(repr(toml_path))) - Pkg.instantiate() - """) - for package in packages - precompiles = package_folder(string(package), "incremental_precompile.jl") - toml, snoopfile = package_toml(package) - snoop(package, toml, snoopfile, precompiles, false, blacklist) - pkg_toml = TOML.parsefile(toml) - manifest = joinpath(dirname(toml), "Manifest.toml") - if isfile(manifest) # not all get a manifest (only if pkg operations are executed I suppose) - pkg_manifest = TOML.parsefile(manifest) - for (name, pkgs) in pkg_manifest - pkg_vec = get!(()-> Dict{String, Any}[], manifest_dict, name) - append!(pkg_vec, pkgs); unique!(pkg_vec) - end - end - merge!(finaltoml["deps"], get(pkg_toml, "deps", Dict())) - merge!(finaltoml["compat"], get(pkg_toml, "compat", Dict())) - println(compile_io) - write(compile_io, read(precompiles)) - end - end - finaltoml["name"] = "PackagesPrecompile" - write_toml(toml_path, finaltoml) - manifest_path = package_folder("Manifest.toml") - # make sure we don't reuse old manifests - isfile(manifest_path) && rm(manifest_path) - if !isempty(manifest_dict) - write_toml(manifest_path, manifest_dict) - end - return toml_path, file -end diff --git a/src/static_julia.jl b/src/static_julia.jl deleted file mode 100644 index 3a7bb14d..00000000 --- a/src/static_julia.jl +++ /dev/null @@ -1,373 +0,0 @@ -const depsfile = normpath(@__DIR__, "..", "deps", "deps.jl") - -if isfile(depsfile) - include(depsfile) - gccworks = try - success(`$gcc -v`) - catch - false - end - if !gccworks - error("GCC wasn't found. Please make sure that gcc is on the path and run Pkg.build(\"PackageCompiler\")") - end -else - error("Package wasn't built correctly. Please run Pkg.build(\"PackageCompiler\")") -end - -system_compiler = gcc -executable_ext = Sys.iswindows() ? ".exe" : "" - -if Sys.iswindows() - function run_PATH(cmd) - bindir = dirname(cmd.exec[1]) - run(setenv(cmd, ["PATH=" * bindir * ";" * ENV["PATH"]])) - end -else - const run_PATH = run -end - - -""" - static_julia(juliaprog::String; kw_args...) - -compiles the Julia file at path `juliaprog` with keyword arguments: - - cprog C program to compile (required only when building an executable, if not provided a minimal driver program is used) - verbose increase verbosity - quiet suppress non-error messages - builddir build directory - outname output files basename - snoopfile specify script calling functions to precompile - clean remove build directory - autodeps automatically build required dependencies - object build object file - shared build shared library - init_shared add `init_jl_runtime` and `exit_jl_runtime` to shared library for runtime initialization - executable build executable file - rmtemp remove temporary build files - copy_julialibs copy Julia libraries to build directory - copy_files copy user-specified files to build directory (either `nothing` or a string array) - release build in release mode, implies `-O3 -g0` unless otherwise specified - Release perform a fully automated release build, equivalent to `-atjr` - sysimage start up with the given system image file - home set location of `julia` executable - startup_file {yes|no} load `~/.julia/config/startup.jl` - handle_signals {yes|no} enable or disable Julia's default signal handlers - sysimage_native_code {yes|no} use native code from system image if available - compiled_modules {yes|no} enable or disable incremental precompilation of modules - depwarn {yes|no|error} enable or disable syntax and method deprecation warnings - warn_overwrite {yes|no} enable or disable method overwrite warnings - compile {yes|no|all|min} enable or disable JIT compiler, or request exhaustive compilation - cpu_target limit usage of CPU features up to (implies default `--sysimage_native_code=no`) - optimize {0,1,2,3} set the optimization level - debug enable / set the level of debug info generation - inline {yes|no} control whether inlining is permitted - check_bounds {yes|no} emit bounds checks always or never - math_mode {ieee,fast} disallow or enable unsafe floating point optimizations - cc system C compiler - cc_flags pass custom flags to the system C compiler when building a shared library or executable (either `nothing` or a string array) -""" -function static_julia( - juliaprog; - cprog = nothing, verbose = false, quiet = false, builddir = nothing, outname = nothing, snoopfile = nothing, - clean = false, autodeps = false, object = false, shared = false, init_shared = false, executable = false, rmtemp = false, - copy_julialibs = false, copy_files = nothing, release = false, Release = false, - sysimage = nothing, home = nothing, startup_file = nothing, handle_signals = nothing, - sysimage_native_code = nothing, compiled_modules = nothing, - depwarn = nothing, warn_overwrite = nothing, - compile = nothing, cpu_target = nothing, optimize = nothing, debug = nothing, - inline = nothing, check_bounds = nothing, math_mode = nothing, - cc = nothing, cc_flags = nothing - ) - - cprog == nothing && (cprog = normpath(@__DIR__, "..", "examples", "program.c")) - builddir == nothing && (builddir = "builddir") - outname == nothing && (outname = splitext(basename(juliaprog))[1]) - cc == nothing && (cc = system_compiler) - - verbose && quiet && (quiet = false) - - if Release - autodeps = rmtemp = copy_julialibs = release = true - end - - if autodeps - executable && (shared = true) - shared && (object = true) - end - - if release - optimize == nothing && (optimize = "3") - debug == nothing && (debug = "0") - end - - if juliaprog != nothing - juliaprog = abspath(juliaprog) - isfile(juliaprog) || error("Cannot find file: \"$juliaprog\"") - quiet || println("Julia program file:\n \"$juliaprog\"") - end - - if executable - cprog = abspath(cprog) - isfile(cprog) || error("Cannot find file: \"$cprog\"") - quiet || println("C program file:\n \"$cprog\"") - end - - builddir = abspath(builddir) - quiet || println("Build directory:\n \"$builddir\"") - - if [clean, object, shared, executable, rmtemp, copy_julialibs, copy_files] == [false, false, false, false, false, false, nothing] - quiet || println("Nothing to do") - return - end - - if clean - if isdir(builddir) - verbose && println("Remove build directory") - rm(builddir, recursive = true) - else - verbose && println("Build directory does not exist") - end - end - - if [object, shared, executable, rmtemp, copy_julialibs, copy_files] == [false, false, false, false, false, nothing] - quiet || println("Clean completed") - return - end - - if !isdir(builddir) - verbose && println("Make build directory") - mkpath(builddir) - end - - o_file = outname * ".a" - s_file = outname * ".$(Libdl.dlext)" - e_file = outname * executable_ext - - if object - if snoopfile != nothing - snoopfile = abspath(snoopfile) - verbose && println("Executing snoopfile: \"$snoopfile\"") - precompfile = joinpath(builddir, "precompiled.jl") - snoop(nothing, nothing, snoopfile, precompfile) - jlmain = joinpath(builddir, "julia_main.jl") - open(jlmain, "w") do io - # this file will get included via @eval Module() include(...) - # but the juliaprog should be included in the main module - juliaprog != nothing && println(io, "Base.include(Main, $(repr(relpath(juliaprog, builddir))))") - println(io, "Base.include(@__MODULE__, $(repr(relpath(precompfile, builddir))))") - end - juliaprog = jlmain - end - build_object( - juliaprog, o_file, builddir, verbose, - sysimage, home, startup_file, handle_signals, sysimage_native_code, compiled_modules, - depwarn, warn_overwrite, compile, cpu_target, optimize, debug, inline, check_bounds, math_mode - ) - end - - shared && build_shared(s_file, o_file, init_shared, builddir, verbose, optimize, debug, cc, cc_flags) - - executable && build_exec(e_file, cprog, s_file, builddir, verbose, optimize, debug, cc, cc_flags) - - rmtemp && remove_temp_files(builddir, verbose) - - copy_julialibs && copy_julia_libs(builddir, verbose) - - copy_files != nothing && copy_files_array(copy_files, builddir, verbose, "Copy user-specified files to build directory:") - - quiet || println("All done") -end - -function julia_flags(optimize, debug, cc_flags) - allflags = Base.shell_split(PackageCompiler.allflags()) - bitness_flag = Sys.ARCH == :arm ? "-mbe32" : Sys.ARCH == :aarch64 ? `` : Int == Int32 ? "-m32" : "-m64" - allflags = `$allflags $bitness_flag` - optimize == nothing || (allflags = `$allflags -O$optimize`) - debug == 2 && (allflags = `$allflags -g`) - cc_flags == nothing || isempty(cc_flags) || (allflags = `$allflags $cc_flags`) - allflags -end - -function build_julia_cmd( - sysimage, home, startup_file, handle_signals, sysimage_native_code, compiled_modules, - depwarn, warn_overwrite, compile, cpu_target, optimize, debug, inline, check_bounds, math_mode - ) - # TODO: `sysimage_native_code` and `compiled_modules` may be removed in future, see: https://github.com/JuliaLang/PackageCompiler.jl/issues/47 - sysimage_native_code == nothing && cpu_target != nothing && (sysimage_native_code = "no") - compiled_modules == nothing && (compiled_modules = "no") - # TODO: `startup_file` may be removed in future with `julia-compile`, see: https://github.com/JuliaLang/julia/issues/15864 - startup_file == nothing && (startup_file = "no") - julia_cmd = `$(Base.julia_cmd())` - function get_flag_idx(julia_cmd, flag_str) - findfirst(f->(length(f) > length(flag_str) && - f[1:length(flag_str)]==flag_str), julia_cmd.exec) - end - function set_flag(julia_cmd, flag_str, val) - flag_idx = get_flag_idx(julia_cmd, flag_str) - if flag_idx != nothing - julia_cmd.exec[flag_idx] = "$flag_str$val" - else - push!(julia_cmd.exec, "$flag_str$val") - end - end - sysimage == nothing || set_flag(julia_cmd, "-J", sysimage) - home == nothing || set_flag(julia_cmd, "-H=", home) - startup_file == nothing || set_flag(julia_cmd, "--startup-file=", startup_file) - handle_signals == nothing || set_flag(julia_cmd, "--handle-signals=", handle_signals) - sysimage_native_code == nothing || set_flag(julia_cmd, "--sysimage-native-code=", sysimage_native_code) - compiled_modules == nothing || set_flag(julia_cmd, "--compiled-modules=", compiled_modules) - depwarn == nothing || set_flag(julia_cmd, "--depwarn=", depwarn) - warn_overwrite == nothing || set_flag(julia_cmd, "--warn-overwrite=", warn_overwrite) - compile == nothing || set_flag(julia_cmd, "--compile=", compile) - cpu_target == nothing || set_flag(julia_cmd, "-C", cpu_target) - optimize == nothing || set_flag(julia_cmd, "-O", optimize) - debug == nothing || set_flag(julia_cmd, "-g", debug) - inline == nothing || set_flag(julia_cmd, "--inline=", inline) - check_bounds == nothing || set_flag(julia_cmd, "--check-bounds=", check_bounds) - math_mode == nothing || set_flag(julia_cmd, "--math-mode=", math_mode) - # Disable incompatible flags - set_flag(julia_cmd, "--code-coverage=", "none") - - julia_cmd -end - -function build_object( - juliaprog, o_file, builddir, verbose, - sysimage, home, startup_file, handle_signals, sysimage_native_code, compiled_modules, - depwarn, warn_overwrite, compile, cpu_target, optimize, debug, inline, check_bounds, math_mode - ) - # TODO really refactor this :D - build_object( - juliaprog, o_file, builddir, verbose; - sysimage = sysimage, startup_file = startup_file, - handle_signals = handle_signals, sysimage_native_code = sysimage_native_code, - compiled_modules = compiled_modules, - depwarn = depwarn, warn_overwrite = warn_overwrite, - compile = compile, cpu_target = cpu_target, optimize = optimize, - debug_level = debug, inline = inline, check_bounds = check_bounds, math_mode = math_mode - ) -end - -function build_object( - juliaprog, o_file, builddir, verbose; julia_flags... - ) - Sys.iswindows() && (juliaprog != nothing) && (juliaprog = replace(juliaprog, "\\" => "\\\\")) - command = ExitHooksStart() * InitBase() * InitREPL() - if juliaprog != nothing - command *= Include(juliaprog) - end - command *= ExitHooksEnd() - verbose && println("Build static library $(repr(o_file)):\n $command") - cd(builddir) do - run_julia( - command; julia_flags..., - output_o = o_file, track_allocation = "none", code_coverage = "none", startup_file = "no", - ) - end -end - -function build_shared(s_file, o_file, init_shared, builddir, verbose, optimize, debug, cc, cc_flags) - if init_shared - i_file = joinpath(builddir, "lib_init.c") - open(i_file, "w") do io - print(io, """ - // Julia headers (for initialization and gc commands) - #include "uv.h" - #include "julia.h" - void init_jl_runtime() // alternate name for jl_init_with_image, with hardcoded library name - { - // JULIAC_PROGRAM_LIBNAME defined on command-line for compilation - const char rel_libname[] = JULIAC_PROGRAM_LIBNAME; - jl_init_with_image(NULL, rel_libname); - } - void exit_jl_runtime(int retcode) // alternate name for jl_atexit_hook - { - jl_atexit_hook(retcode); - } - """ - ) - end - i_file = `$i_file` - else - i_file = `` - end - # Prevent compiler from stripping all symbols from the shared lib. - if Sys.isapple() - o_file = `-Wl,-all_load $o_file` - else - o_file = `-Wl,--whole-archive $o_file -Wl,--no-whole-archive` - end - command = `$cc -shared -DJULIAC_PROGRAM_LIBNAME=\"$s_file\" -o $s_file $o_file $i_file $(julia_flags(optimize, debug, cc_flags))` - if Sys.isapple() - command = `$command -Wl,-install_name,@rpath/$s_file` - elseif Sys.iswindows() - command = `$command -Wl,--export-all-symbols` - end - verbose && println("Build shared library $(repr(s_file)):\n $command") - cd(builddir) do - run_PATH(command) - end -end - -function build_exec(e_file, cprog, s_file, builddir, verbose, optimize, debug, cc, cc_flags) - command = `$cc -DJULIAC_PROGRAM_LIBNAME=\"$s_file\" -o $e_file $cprog $s_file $(julia_flags(optimize, debug, cc_flags))` - if Sys.iswindows() - # functionality doesn't readily exist on this platform - elseif Sys.isapple() - command = `$command -Wl,-rpath,@executable_path` - else - command = `$command -Wl,-rpath,\$ORIGIN` - end - if Int == Int32 - # TODO: this was added because of an error with julia on win32 that suggested this line, it seems to work but I'm not sure if it's correct - command = `$command -march=pentium4` - end - verbose && println("Build executable $(repr(e_file)):\n $command") - cd(builddir) do - run_PATH(command) - end -end - -function remove_temp_files(builddir, verbose) - verbose && println("Remove temporary files:") - remove = false - for tmp in filter(x -> endswith(x, ".a") || startswith(x, "cache_ji_v"), readdir(builddir)) - verbose && println(" $tmp") - rm(joinpath(builddir, tmp), recursive = true) - remove = true - end - verbose && !remove && println(" none") -end - -function copy_files_array(files_array, builddir, verbose, message) - verbose && println(message) - copy = false - for src in files_array - isfile(src) || error("Cannot find file: \"$src\"") - dst = joinpath(builddir, basename(src)) - if filesize(src) != filesize(dst) || ctime(src) > ctime(dst) || mtime(src) > mtime(dst) - verbose && println(" $(basename(src))") - cp(src, dst, force = true, follow_symlinks = false) - copy = true - end - end - verbose && !copy && println(" none") -end - -function copy_julia_libs(builddir, verbose) - # TODO: these flags should probably be emitted also by `julia-config.jl` and `compiler_flags.jl` - shlibdir = Sys.iswindows() ? Sys.BINDIR : joinpath(Sys.BINDIR, Base.LIBDIR) - private_shlibdir = joinpath(Sys.BINDIR, Base.PRIVATE_LIBDIR) - libfiles = String[] - dlext = "." * Libdl.dlext - for dir in (shlibdir, private_shlibdir) - if Sys.iswindows() || Sys.isapple() - append!(libfiles, joinpath.(dir, filter(x -> endswith(x, dlext) && !startswith(x, "sys"), readdir(dir)))) - else - append!(libfiles, joinpath.(dir, filter(x -> occursin(r"^lib.+\.so(?:\.\d+)*$", x), readdir(dir)))) - end - end - filter!(v -> !occursin(r"debug", v), libfiles) - copy_files_array(libfiles, builddir, verbose, "Copy Julia libraries to build directory:") -end diff --git a/src/system_image.jl b/src/system_image.jl deleted file mode 100644 index 530507cf..00000000 --- a/src/system_image.jl +++ /dev/null @@ -1,66 +0,0 @@ -# This code is derived from `build_sysimg.jl` (part of Julia) and should be kept aligned with it. - -function default_sysimg_path(debug = false) - ext = debug ? "sys-debug" : "sys" - if Sys.isunix() - dirname(Libdl.dlpath(ext)) - else - normpath(Sys.BINDIR, "..", "lib", "julia") - end -end - -function compile_system_image(sysimg_path, cpu_target = nothing; debug = false) - # Enter base and setup some useful paths - base_dir = dirname(Base.find_source_file("sysimg.jl")) - cd(base_dir) do - # This can probably get merged with build_object. - # At some point, I will need to understand build_object a bit better before doing that move, though! - julia_cmd = Base.julia_cmd() - julia = julia_cmd.exec[1] - cpu_target = if cpu_target === nothing - replace(julia_cmd.exec[2], "-C" => "") - else - cpu_target - end - cc = system_compiler - # Ensure we have write-permissions to wherever we're trying to write to - try - touch("$sysimg_path.ji") - catch - error("Unable to modify $sysimg_path.ji, ensure that parent directory exists and is writable") - end - compiler_path = joinpath(dirname(sysimg_path), "basecompiler") - compiler = "compiler/compiler.jl" - - # Start by building inference.{ji,o} - inference_path = joinpath(dirname(sysimg_path), "inference") - command = `$julia -C $cpu_target --output-ji $compiler_path.ji --output-o $compiler_path.o $compiler` - @info "Building `inference.o`:\n$command" - run(command) - - # Bootstrap off of that to create sys.{ji,o} - command = `$julia -C $cpu_target --output-ji $sysimg_path.ji --output-o $sysimg_path.o -J $compiler_path.ji --startup-file=no sysimg.jl` - @info "Building `sys.o`:\n$command" - run(command) - - build_shared( - "$sysimg_path.$(Libdl.dlext)", "$sysimg_path.o", false, - ".", true, nothing, debug ? 2 : nothing, cc, nothing - ) - end -end - -""" -Returns the system image file stored in the backup folder. -If there is no backup, this function will automatically generate a system image -in the backup folder. -""" -function get_backup!(debug, cpu_target = nothing) - target = julia_cpu_target(cpu_target) - sysimg_backup = sysimgbackup_folder(target) - isdir(sysimg_backup) || mkpath(sysimg_backup) - if !all(x-> isfile(joinpath(sysimg_backup, x)), sysimage_binaries) # we have a backup - compile_system_image(joinpath(sysimg_backup, "sys"), target; debug = debug) - end - return joinpath(sysimg_backup, "sys.$(Libdl.dlext)") -end diff --git a/test/REQUIRE b/test/REQUIRE deleted file mode 100644 index d85c9081..00000000 --- a/test/REQUIRE +++ /dev/null @@ -1,6 +0,0 @@ -ColorTypes -FixedPointNumbers -UnicodePlots -JSON -DataStructures -OffsetArrays diff --git a/test/TestPackage/Project.toml b/test/TestPackage/Project.toml deleted file mode 100644 index c5484ce4..00000000 --- a/test/TestPackage/Project.toml +++ /dev/null @@ -1,4 +0,0 @@ -name = "TestPackage" -uuid = "d4c01e28-d041-5f16-991d-b5745e8a2c25" -authors = ["SimonDanisch "] -version = "0.1.0" diff --git a/test/TestPackage/src/TestPackage.jl b/test/TestPackage/src/TestPackage.jl deleted file mode 100644 index 537eab5a..00000000 --- a/test/TestPackage/src/TestPackage.jl +++ /dev/null @@ -1,5 +0,0 @@ -module TestPackage - -greet() = print("Hello World!") - -end # module diff --git a/test/TestPackage/test/runtests.jl b/test/TestPackage/test/runtests.jl deleted file mode 100644 index 447bcf8c..00000000 --- a/test/TestPackage/test/runtests.jl +++ /dev/null @@ -1,3 +0,0 @@ -using TestPackage - -TestPackage.greet() diff --git a/test/TestPackage2/Project.toml b/test/TestPackage2/Project.toml deleted file mode 100644 index a71f41d0..00000000 --- a/test/TestPackage2/Project.toml +++ /dev/null @@ -1,4 +0,0 @@ -name = "TestPackage2" -uuid = "886979a0-3551-11e9-32b6-2991215e940c" -authors = ["Patrick Belliveau "] -version = "0.1.0" diff --git a/test/TestPackage2/snoop/snoopfile.jl b/test/TestPackage2/snoop/snoopfile.jl deleted file mode 100644 index 1fd89165..00000000 --- a/test/TestPackage2/snoop/snoopfile.jl +++ /dev/null @@ -1,3 +0,0 @@ -using TestPackage2 - -TestPackage2.greet() diff --git a/test/TestPackage2/src/TestPackage2.jl b/test/TestPackage2/src/TestPackage2.jl deleted file mode 100644 index 00841e76..00000000 --- a/test/TestPackage2/src/TestPackage2.jl +++ /dev/null @@ -1,5 +0,0 @@ -module TestPackage2 - -greet() = print("Hello World!") - -end # module diff --git a/test/runtests.jl b/test/runtests.jl deleted file mode 100644 index a0cc69ff..00000000 --- a/test/runtests.jl +++ /dev/null @@ -1,161 +0,0 @@ -using PackageCompiler, Test - -@testset "compilage_package" begin - @testset "FixedPointNumbers" begin - sysimage = PackageCompiler.compile_package("FixedPointNumbers", verbose = true) - test_code = """ - using FixedPointNumbers; N0f8(0.5); println("no segfaults, yay") - """ - cmd = PackageCompiler.julia_code_cmd(test_code, J = sysimage) - @test read(cmd, String) == "no segfaults, yay\n" - end - @testset "FixedPointNumbers ColorTypes" begin - sysimage = PackageCompiler.compile_package("FixedPointNumbers", "ColorTypes", verbose = true) - test_code = """ - using FixedPointNumbers, ColorTypes; N0f8(0.5); RGB(0.0, 0.0, 0.0); println("no segfaults, yay") - """ - cmd = PackageCompiler.julia_code_cmd(test_code, J = sysimage) - @test read(cmd, String) == "no segfaults, yay\n" - end -end - - -@testset "compile_incremental" begin - @testset "unregistered package with runtests.jl" begin - path = joinpath(@__DIR__, "TestPackage") - push!(Base.LOAD_PATH, path) - syso, syso_old = PackageCompiler.compile_incremental(:TestPackage) - test_code = """ - push!(Base.LOAD_PATH, $(repr(path))) - using TestPackage; TestPackage.greet() - """ - cmd = PackageCompiler.julia_code_cmd(test_code, J = syso) - @test read(cmd, String) == "Hello World!" - pop!(Base.LOAD_PATH) - end - @testset "unregistered package with snoopfile.jl" begin - path = joinpath(@__DIR__, "TestPackage2") - push!(Base.LOAD_PATH, path) - syso, syso_old = PackageCompiler.compile_incremental(:TestPackage2) - test_code = """ - push!(Base.LOAD_PATH, $(repr(path))) - using TestPackage2; TestPackage2.greet() - """ - cmd = PackageCompiler.julia_code_cmd(test_code, J = syso) - @test read(cmd, String) == "Hello World!" - pop!(Base.LOAD_PATH) - end - @testset "FixedPointNumbers" begin - # This is the new compile_package - syso, syso_old = PackageCompiler.compile_incremental(:FixedPointNumbers) - test_code = """ - using FixedPointNumbers; N0f8(0.5); println("no segfaults, yay") - """ - cmd = PackageCompiler.julia_code_cmd(test_code, J = syso) - @test read(cmd, String) == "no segfaults, yay\n" - end - @testset "FixedPointNumbers ColorTypes" begin - syso, syso_old = PackageCompiler.compile_incremental(:FixedPointNumbers, :ColorTypes) - test_code = """ - using FixedPointNumbers, ColorTypes; N0f8(0.5); RGB(0.0, 0.0, 0.0); println("no segfaults, yay") - """ - cmd = PackageCompiler.julia_code_cmd(test_code, J = syso) - @test read(cmd, String) == "no segfaults, yay\n" - end - @testset "JSON with Distributed blacklisted" begin - # This is the new compile_package - syso, syso_old = PackageCompiler.compile_incremental(:JSON, blacklist=[:Distributed]) - test_code = """ - using JSON - s = \"{\\"a_number\\" : 5.0, \\"an_array\\" : [\\"string\\", 9]}\" - j = JSON.parse(s) - println(\"no segfaults, yay\") - """ - cmd = PackageCompiler.julia_code_cmd(test_code, J = syso) - @test read(cmd, String) == "no segfaults, yay\n" - end -end - -julia = Base.julia_cmd().exec[1] - -@testset "build_executable" begin - jlfile = joinpath(@__DIR__, "..", "examples", "hello.jl") - basedir = mktempdir() - relativebuilddir = "build" - cd(basedir) do - mkdir(relativebuilddir) - snoopfile = "snoop.jl" - open(snoopfile, "w") do io - write(io, open(read, jlfile)) - println(io) - println(io, "using .Hello; Hello.julia_main(String[])") - end - build_executable( - jlfile, snoopfile = snoopfile, builddir = relativebuilddir, verbose = true - ) - end - builddir = joinpath(basedir, relativebuilddir) - @test isfile(joinpath(builddir, "hello.$(PackageCompiler.Libdl.dlext)")) - @test isfile(joinpath(builddir, "hello$executable_ext")) - @test startswith(read(`$(joinpath(builddir, "hello$executable_ext"))`, String), "hello, world") - for i = 1:100 - # Windows seems to have problems with removing files - it can error - # making this test fail. - try rm(basedir, recursive = true) catch end - sleep(1/100) - end -end - -@testset "program.c" begin - @testset "args" begin - basedir = mktempdir(); - argsjlfile = mktemp(basedir)[1]; - write(argsjlfile, raw""" - Base.@ccallable function julia_main(argv::Vector{String})::Cint - println("@__FILE__: $(@__FILE__)") - println("PROGRAM_FILE: $(PROGRAM_FILE)") - println("argv: $(argv)") - # Sometimes code accesses ARGS directly, as a global - println("ARGS: $ARGS") - println("Base.ARGS: $(Base.ARGS)") - println("Core.ARGS: $(Core.ARGS)") - return 0 - end - """) - builddir = joinpath(basedir, "builddir") - outname = "args" - build_executable( - argsjlfile, outname; builddir = builddir - ) - # Check that the output from the program is as expected: - exe = joinpath(builddir, outname*executable_ext) - output = read(`$exe a b c`, String) - println(output) - @test output == - "@__FILE__: $argsjlfile\n" * - "PROGRAM_FILE: $exe\n" * - "argv: [\"a\", \"b\", \"c\"]\n" * - "ARGS: [\"a\", \"b\", \"c\"]\n" * - "Base.ARGS: [\"a\", \"b\", \"c\"]\n" * - "Core.ARGS: Any[\"$(Sys.iswindows() ? replace(exe, "\\" => "\\\\") : exe)\", \"a\", \"b\", \"c\"]\n" - end -end - -@testset "juliac" begin - mktempdir() do builddir - juliac = joinpath(@__DIR__, "..", "juliac.jl") - jlfile = joinpath(@__DIR__, "..", "examples", "hello.jl") - cfile = joinpath(@__DIR__, "..", "examples", "program.c") - @test success(`$julia $juliac -vaej $jlfile $cfile --builddir $builddir`) - @test isfile(joinpath(builddir, "hello.$(PackageCompiler.Libdl.dlext)")) - @test isfile(joinpath(builddir, "hello$executable_ext")) - @test success(`$(joinpath(builddir, "hello$executable_ext"))`) - @testset "--cc-flag" begin - # Try passing `--help` to $cc. This should work for any system compiler. - # Then grep the output for "-g", which should be present on any system. - @test occursin("-g", read(`$julia $juliac -se --cc-flag="--help" $jlfile $cfile --builddir $builddir`, String)) - # Just as a control, make sure that without passing '--help', we don't see "-g" - @test !occursin("-g", read(`$julia $juliac -se $jlfile $cfile --builddir $builddir`, String)) - end - end -end diff --git a/test/shipping.jl b/test/shipping.jl deleted file mode 100644 index ea925363..00000000 --- a/test/shipping.jl +++ /dev/null @@ -1,49 +0,0 @@ -using PackageCompiler, Pkg - -dir(f...) = joinpath(@__DIR__, f...) -cd(@__DIR__) -build_executable( - dir("makietest.jl"), - "makie", - dir("..", "examples", "program.c"); - snoopfile = dir("makiesnoop.jl"), - builddir = dir("build"), - verbose = true, quiet = false, - cpu_target = "x86-64", optimize = "3" -) -PackageCompiler.build_object( - dir("build", "julia_main.jl"), dir("build"), dir("build", "makie.o"), true, - nothing, nothing, "x86-64", "3", nothing, nothing, nothing, - nothing, nothing -) - - -packages = [ - "Quaternions", - "GLVisualize", - "StaticArrays", - "GeometryTypes", - "Reactive", - "GLAbstraction", - "GLWindow", - "AbstractNumbers", - "Contour", - "FileIO", - "Images", - "UnicodeFun", - "ColorBrewer", - "Interact", # for displaying signals of Image - a bit unfortunate - "Hiccup", - "Media", - "Juno", - "ModernGL", - "GLFW", - "Fontconfig", - "FreeType", - "FreeTypeAbstraction", - "ImageMagick", -] - -for elem in packages - cp(normpath(Base.find_package(elem), "..", ".."), dir("build", elem)) -end From 7d84910b7bebd82e9bd849e09283e553f4a4bfba Mon Sep 17 00:00:00 2001 From: KristofferC Date: Wed, 16 Oct 2019 15:16:05 +0200 Subject: [PATCH 02/80] Files generated by PkgTemplates --- .gitignore | 6 ++++++ .travis.yml | 16 ++++++++++++++++ LICENSE | 19 +++++++++++++++++++ Project.toml | 13 +++++++++++++ README.md | 4 ++++ src/PackageCompilerX.jl | 5 +++++ test/runtests.jl | 6 ++++++ 7 files changed, 69 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 LICENSE create mode 100644 Project.toml create mode 100644 README.md create mode 100644 src/PackageCompilerX.jl create mode 100644 test/runtests.jl diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..1388e96a --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.jl.*.cov +*.jl.cov +*.jl.mem +.DS_Store +/Manifest.toml +/dev/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..e374e1c9 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,16 @@ +# Documentation: http://docs.travis-ci.com/user/languages/julia/ +language: julia +os: + - linux + - osx +julia: + - 1.0 + - nightly +matrix: + allow_failures: + - julia: nightly + fast_finish: true +notifications: + email: false +after_success: + - julia -e 'using Pkg; Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder())' diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..9adf483e --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2019 KristofferC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Project.toml b/Project.toml new file mode 100644 index 00000000..2d355123 --- /dev/null +++ b/Project.toml @@ -0,0 +1,13 @@ +name = "PackageCompilerX" +uuid = "dffaa6cc-da53-48e5-b007-4292dfcc27f1" +authors = ["KristofferC "] +version = "0.1.0" + +[compat] +julia = "1" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/README.md b/README.md new file mode 100644 index 00000000..aa9f4405 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +# PackageCompilerX + +[![Build Status](https://travis-ci.com/KristofferC/PackageCompilerX.jl.svg?branch=master)](https://travis-ci.com/KristofferC/PackageCompilerX.jl) +[![Codecov](https://codecov.io/gh/KristofferC/PackageCompilerX.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/KristofferC/PackageCompilerX.jl) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl new file mode 100644 index 00000000..0eda4ec9 --- /dev/null +++ b/src/PackageCompilerX.jl @@ -0,0 +1,5 @@ +module PackageCompilerX + +greet() = print("Hello World!") + +end # module diff --git a/test/runtests.jl b/test/runtests.jl new file mode 100644 index 00000000..18c46204 --- /dev/null +++ b/test/runtests.jl @@ -0,0 +1,6 @@ +using PackageCompilerX +using Test + +@testset "PackageCompilerX.jl" begin + # Write your own tests here. +end From a5a55a3eedcffd00ce79e24e3d1360c8e264d2bf Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 22 Oct 2019 20:02:54 +0200 Subject: [PATCH 03/80] test build script from PackageCompiler (#1) make a hardcoded app --- .travis.yml | 10 ++----- Project.toml | 4 +++ README.md | 4 +++ examples/hello.jl | 6 ++++ src/PackageCompilerX.jl | 42 +++++++++++++++++++++++++- src/embedding_wrapper.c | 52 +++++++++++++++++++++++++++++++++ src/juliaconfig.jl | 65 +++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 8 +++++ 8 files changed, 183 insertions(+), 8 deletions(-) create mode 100644 examples/hello.jl create mode 100644 src/embedding_wrapper.c create mode 100644 src/juliaconfig.jl diff --git a/.travis.yml b/.travis.yml index e374e1c9..36ff6983 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,11 @@ -# Documentation: http://docs.travis-ci.com/user/languages/julia/ language: julia os: - linux - osx + - windows + # - arm64 julia: - - 1.0 - - nightly -matrix: - allow_failures: - - julia: nightly - fast_finish: true + - 1.3 notifications: email: false after_success: diff --git a/Project.toml b/Project.toml index 2d355123..d634dbf8 100644 --- a/Project.toml +++ b/Project.toml @@ -3,6 +3,10 @@ uuid = "dffaa6cc-da53-48e5-b007-4292dfcc27f1" authors = ["KristofferC "] version = "0.1.0" +[deps] +Example = "7876af07-990d-54b4-ab0e-23690620f79a" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + [compat] julia = "1" diff --git a/README.md b/README.md index aa9f4405..e2803c15 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,7 @@ [![Build Status](https://travis-ci.com/KristofferC/PackageCompilerX.jl.svg?branch=master)](https://travis-ci.com/KristofferC/PackageCompilerX.jl) [![Codecov](https://codecov.io/gh/KristofferC/PackageCompilerX.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/KristofferC/PackageCompilerX.jl) + + +This package requires a locally installed C compiler. By default, the compilers `gcc` and `clang` will be l +On Windows, simple ways to install compilers are either by installing LLVM (which includes the compiler Clang) from https://releases.llvm.org/download.html or installing `gcc` via [MinGW](http://www.mingw.org/). On macOS and Linux, getting a compiler is usually trivial. diff --git a/examples/hello.jl b/examples/hello.jl new file mode 100644 index 00000000..f77d132d --- /dev/null +++ b/examples/hello.jl @@ -0,0 +1,6 @@ +Base.@ccallable function julia_main(ARGS::Vector{String})::Cint + println("hello, world") + @show sin(0.0) + return 0 +end + diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 0eda4ec9..5915fb21 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -1,5 +1,45 @@ module PackageCompilerX -greet() = print("Hello World!") +using Libdl + +include("juliaconfig.jl") + +function create_object(package::Symbol, project=Base.active_project(); precompilefile="precompile.jl") + example = joinpath(@__DIR__, "..", "examples", "hello.jl") + julia_code = """Base.__init__(); using $package; include("$(example)")""" + julia_path = joinpath(Sys.BINDIR, Base.julia_exename()) + image_file = unsafe_string(Base.JLOptions().image_file) + + cmd = `$julia_path -J$image_file --color=yes --project=$project --output-o=sys.o --startup-file=no -e $julia_code` + run(cmd) +end + +function create_shared_library(input_object::String, output_library::String) + julia_libdir = dirname(Libdl.dlpath("libjulia")) + + # Prevent compiler from stripping all symbols from the shared lib. + if Sys.isapple() + o_file = `-Wl,-all_load $input_object` + else + o_file = `-Wl,--whole-archive $input_object -Wl,--no-whole-archive` + end + + run(`clang -v -shared -L$(julia_libdir) -o $output_library $o_file -ljulia`) +end + +function create_executable() + flags = join((cflags(), ldflags(), ldlibs()), " ") + flags = Base.shell_split(flags) + wrapper = joinpath(@__DIR__, "embedding_wrapper.c") + if Sys.iswindows() + # functionality doesn't readily exist on this platform + elseif Sys.isapple() + rpath = `-Wl,-rpath,@executable_path` + else + rpath = `-Wl,-rpath,\$ORIGIN` + end + sysimg = "sys." * Libdl.dlext + run(`clang -DJULIAC_PROGRAM_LIBNAME=\"$sysimg\" -o myapp $(wrapper) $sysimg -O2 $rpath $flags`) +end end # module diff --git a/src/embedding_wrapper.c b/src/embedding_wrapper.c new file mode 100644 index 00000000..93ba3d9a --- /dev/null +++ b/src/embedding_wrapper.c @@ -0,0 +1,52 @@ +// This file is a part of Julia. License is MIT: http://julialang.org/license + +// Standard headers +#include +#include + +// Julia headers (for initialization and gc commands) +#include "uv.h" +#include "julia.h" + +JULIA_DEFINE_FAST_TLS() + +// Declare C prototype of a function defined in Julia +extern int julia_main(jl_array_t*); + +// main function (windows UTF16 -> UTF8 argument conversion code copied from julia's ui/repl.c) +int main(int argc, char *argv[]) +{ + int retcode; + int i; + uv_setup_args(argc, argv); // no-op on Windows + + // initialization + libsupport_init(); + + // jl_options.compile_enabled = JL_OPTIONS_COMPILE_OFF; + // JULIAC_PROGRAM_LIBNAME defined on command-line for compilation + jl_options.image_file = JULIAC_PROGRAM_LIBNAME; + julia_init(JL_IMAGE_JULIA_HOME); + + // Initialize Core.ARGS with the full argv. + jl_set_ARGS(argc, argv); + + // Set PROGRAM_FILE to argv[0]. + jl_set_global(jl_base_module, + jl_symbol("PROGRAM_FILE"), (jl_value_t*)jl_cstr_to_string(argv[0])); + + // Set Base.ARGS to `String[ unsafe_string(argv[i]) for i = 1:argc ]` + jl_array_t *ARGS = (jl_array_t*)jl_get_global(jl_base_module, jl_symbol("ARGS")); + jl_array_grow_end(ARGS, argc - 1); + for (i = 1; i < argc; i++) { + jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]); + jl_arrayset(ARGS, s, i - 1); + } + + // call the work function, and get back a value + retcode = julia_main(ARGS); + + // Cleanup and gracefully exit + jl_atexit_hook(retcode); + return retcode; +} diff --git a/src/juliaconfig.jl b/src/juliaconfig.jl new file mode 100644 index 00000000..dd64ee70 --- /dev/null +++ b/src/juliaconfig.jl @@ -0,0 +1,65 @@ +# adopted from https://github.com/JuliaLang/julia/blob/release-0.6/contrib/julia-config.jl + +threading_on() = ccall(:jl_threading_enabled, Cint, ()) != 0 + +function shell_escape(str) + str = replace(str, "'" => "'\''") + return "'$str'" +end + +function julia_libdir() + return if ccall(:jl_is_debugbuild, Cint, ()) != 0 + dirname(abspath(Libdl.dlpath("libjulia-debug"))) + else + dirname(abspath(Libdl.dlpath("libjulia"))) + end +end + +function julia_private_libdir() + @static if Sys.iswindows() + return julia_libdir() + else + return abspath(Sys.BINDIR, Base.PRIVATE_LIBDIR) + end +end + +julia_includedir() = abspath(Sys.BINDIR, Base.INCLUDEDIR, "julia") + +function ldflags() + fl = "-L$(shell_escape(julia_libdir()))" + if Sys.iswindows() + fl = fl * " -Wl,--stack,8388608" + elseif Sys.islinux() + fl = fl * " -Wl,--export-dynamic" + end + return fl +end + +# TODO +function ldlibs(relative_path=nothing) + libname = if ccall(:jl_is_debugbuild, Cint, ()) != 0 + "julia-debug" + else + "julia" + end + if Sys.isunix() + return "-Wl,-rpath,$(shell_escape(julia_libdir())) -Wl,-rpath,$(shell_escape(julia_private_libdir())) -l$libname" + else + return "-l$libname -lopenlibm" + end +end + +function cflags() + flags = IOBuffer() + print(flags, "-std=gnu99") + include = shell_escape(julia_includedir()) + print(flags, " -I", include) + if threading_on() + print(flags, " -DJULIA_ENABLE_THREADING=1") + end + if Sys.isunix() + print(flags, " -fPIC") + end + return String(take!(flags)) +end + diff --git a/test/runtests.jl b/test/runtests.jl index 18c46204..1edf1e42 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,14 @@ using PackageCompilerX using Test +using Libdl @testset "PackageCompilerX.jl" begin # Write your own tests here. + PackageCompilerX.create_object(:Example) + sysimg = "sys." * Libdl.dlext + PackageCompilerX.create_shared_library("sys.o", sysimg) + run(`$(Base.julia_cmd()) -J $(sysimg) -e 'println(1337)'`) + PackageCompilerX.create_executable() + output = read(`./myapp`, String) + @test occursin("hello, world", output) end From 8e949945e8d3ed268efc1557442619be4bbcdd3d Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 22 Oct 2019 22:46:26 +0200 Subject: [PATCH 04/80] implement support for precompile file (#5) * implement support for precompile file --- .travis.yml | 2 +- src/PackageCompilerX.jl | 59 ++++++++++++++++++++++++++++++++++++----- test/precompile.jl | 3 +++ test/runtests.jl | 2 +- 4 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 test/precompile.jl diff --git a/.travis.yml b/.travis.yml index 36ff6983..02bc13d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ os: - linux - osx - windows - # - arm64 +# - arm64 julia: - 1.3 notifications: diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 5915fb21..97ef416f 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -1,17 +1,63 @@ module PackageCompilerX +using Base: active_project using Libdl include("juliaconfig.jl") -function create_object(package::Symbol, project=Base.active_project(); precompilefile="precompile.jl") - example = joinpath(@__DIR__, "..", "examples", "hello.jl") - julia_code = """Base.__init__(); using $package; include("$(example)")""" +function get_julia_cmd() julia_path = joinpath(Sys.BINDIR, Base.julia_exename()) image_file = unsafe_string(Base.JLOptions().image_file) + cmd = `$julia_path -J$image_file --color=yes --startup-file=no` +end + +# Returns a vector of precompile statemenets +function run_precompilation_script(project::String, precompile_file::String) + tracefile = tempname() + julia_code = """Base.__init__(); include($(repr(precompile_file)))""" + run(`$(get_julia_cmd()) --project=$project --trace-compile=$tracefile -e $julia_code`) + return tracefile +end - cmd = `$julia_path -J$image_file --color=yes --project=$project --output-o=sys.o --startup-file=no -e $julia_code` - run(cmd) +function create_object_file(package::Symbol, project::String=active_project(); + precompile_file::Union{String, Nothing}=nothing) + julia_code = """ + if !isdefined(Base, :uv_eventloop) + Base.reinit_stdio() + end + Base.__init__(); + using $package + """ + example = joinpath(@__DIR__, "..", "examples", "hello.jl") + julia_code *= """ + include($(repr(example))) + """ + if precompile_file !== nothing + tracefile = run_precompilation_script(project, precompile_file) + precompile_code = """ + # This @eval prevents symbols from being put into Main + @eval Module() begin + PrecompileStagingArea = Module() + for (_pkgid, _mod) in Base.loaded_modules + if !(_pkgid.name in ("Main", "Core", "Base")) + eval(PrecompileStagingArea, :(const \$(Symbol(_mod)) = \$_mod)) + end + end + precompile_statements = readlines($(repr(tracefile))) + for statement in sort(precompile_statements) + # println(statement) + try + Base.include_string(PrecompileStagingArea, statement) + catch + # See #28808 + @error "failed to execute \$statement" + end + end + end # module + """ + julia_code *= precompile_code + end + run(`$(get_julia_cmd()) --project=$project --output-o=sys.o -e $julia_code`) end function create_shared_library(input_object::String, output_library::String) @@ -23,7 +69,6 @@ function create_shared_library(input_object::String, output_library::String) else o_file = `-Wl,--whole-archive $input_object -Wl,--no-whole-archive` end - run(`clang -v -shared -L$(julia_libdir) -o $output_library $o_file -ljulia`) end @@ -39,7 +84,7 @@ function create_executable() rpath = `-Wl,-rpath,\$ORIGIN` end sysimg = "sys." * Libdl.dlext - run(`clang -DJULIAC_PROGRAM_LIBNAME=\"$sysimg\" -o myapp $(wrapper) $sysimg -O2 $rpath $flags`) + run(`clang -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimg)) -o myapp $(wrapper) $sysimg -O2 $rpath $flags`) end end # module diff --git a/test/precompile.jl b/test/precompile.jl new file mode 100644 index 00000000..786fde11 --- /dev/null +++ b/test/precompile.jl @@ -0,0 +1,3 @@ +using Example + +Example.domath(5) diff --git a/test/runtests.jl b/test/runtests.jl index 1edf1e42..74d8d637 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,7 +4,7 @@ using Libdl @testset "PackageCompilerX.jl" begin # Write your own tests here. - PackageCompilerX.create_object(:Example) + PackageCompilerX.create_object_file(:Example; precompile_file="precompile.jl") sysimg = "sys." * Libdl.dlext PackageCompilerX.create_shared_library("sys.o", sysimg) run(`$(Base.julia_cmd()) -J $(sysimg) -e 'println(1337)'`) From f1e4ee49618cc85ec7c7d6d5e5dccf55bfbd6009 Mon Sep 17 00:00:00 2001 From: KristofferC Date: Wed, 23 Oct 2019 09:14:01 +0200 Subject: [PATCH 05/80] use gcc for now --- src/PackageCompilerX.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 97ef416f..3115e84d 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -5,6 +5,8 @@ using Libdl include("juliaconfig.jl") +const CC = `gcc` + function get_julia_cmd() julia_path = joinpath(Sys.BINDIR, Base.julia_exename()) image_file = unsafe_string(Base.JLOptions().image_file) @@ -64,12 +66,13 @@ function create_shared_library(input_object::String, output_library::String) julia_libdir = dirname(Libdl.dlpath("libjulia")) # Prevent compiler from stripping all symbols from the shared lib. + # TODO: On clang on windows this is called something else if Sys.isapple() o_file = `-Wl,-all_load $input_object` else o_file = `-Wl,--whole-archive $input_object -Wl,--no-whole-archive` end - run(`clang -v -shared -L$(julia_libdir) -o $output_library $o_file -ljulia`) + run(`$CC -v -shared -L$(julia_libdir) -o $output_library $o_file -ljulia`) end function create_executable() @@ -84,7 +87,7 @@ function create_executable() rpath = `-Wl,-rpath,\$ORIGIN` end sysimg = "sys." * Libdl.dlext - run(`clang -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimg)) -o myapp $(wrapper) $sysimg -O2 $rpath $flags`) + run(`$CC -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimg)) -o myapp $(wrapper) $sysimg -O2 $rpath $flags`) end end # module From c4e433868e0b2974e085b42f7978049546145102 Mon Sep 17 00:00:00 2001 From: KristofferC Date: Wed, 23 Oct 2019 09:19:57 +0200 Subject: [PATCH 06/80] allow for multiple packages --- src/PackageCompilerX.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 3115e84d..77dffbe5 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -21,15 +21,18 @@ function run_precompilation_script(project::String, precompile_file::String) return tracefile end -function create_object_file(package::Symbol, project::String=active_project(); +function create_object_file(packages::Union{Symbol, Vector{Symbol}}, project::String=active_project(); precompile_file::Union{String, Nothing}=nothing) + packages = vcat(packages) julia_code = """ if !isdefined(Base, :uv_eventloop) Base.reinit_stdio() end Base.__init__(); - using $package """ + for package in packages + julia_code *= "using $package\n" + end example = joinpath(@__DIR__, "..", "examples", "hello.jl") julia_code *= """ include($(repr(example))) From 3666883f797349687bdef4155f7800df2e563c60 Mon Sep 17 00:00:00 2001 From: KristofferC Date: Wed, 23 Oct 2019 11:10:33 +0200 Subject: [PATCH 07/80] some generalizations --- .gitignore | 4 + README.md | 5 +- src/PackageCompilerX.jl | 94 ++++++++++++++++--- ...{precompile.jl => precompile_execution.jl} | 0 test/precompile_statements.jl | 1 + test/runtests.jl | 16 ++-- 6 files changed, 97 insertions(+), 23 deletions(-) rename test/{precompile.jl => precompile_execution.jl} (100%) create mode 100644 test/precompile_statements.jl diff --git a/.gitignore b/.gitignore index 1388e96a..0500fc6b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ .DS_Store /Manifest.toml /dev/ +*.so +*.dll +*.dylib +*.o diff --git a/README.md b/README.md index e2803c15..92659754 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,5 @@ [![Build Status](https://travis-ci.com/KristofferC/PackageCompilerX.jl.svg?branch=master)](https://travis-ci.com/KristofferC/PackageCompilerX.jl) [![Codecov](https://codecov.io/gh/KristofferC/PackageCompilerX.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/KristofferC/PackageCompilerX.jl) - -This package requires a locally installed C compiler. By default, the compilers `gcc` and `clang` will be l -On Windows, simple ways to install compilers are either by installing LLVM (which includes the compiler Clang) from https://releases.llvm.org/download.html or installing `gcc` via [MinGW](http://www.mingw.org/). On macOS and Linux, getting a compiler is usually trivial. +This package requires `gcc` to be installed and on the `PATH`. +On Windows, we recommend installing `gcc` via [MinGW](http://www.mingw.org/). diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 77dffbe5..e96f19ba 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -21,8 +21,10 @@ function run_precompilation_script(project::String, precompile_file::String) return tracefile end -function create_object_file(packages::Union{Symbol, Vector{Symbol}}, project::String=active_project(); - precompile_file::Union{String, Nothing}=nothing) +function create_object_file(object_file::String, packages::Union{Symbol, Vector{Symbol}}, project::String=active_project(); + precompile_execution_file::Union{String, Nothing}=nothing, + precompile_statements_file::Union{String, Nothing}=nothing) + # include all packages into the sysimage packages = vcat(packages) julia_code = """ if !isdefined(Base, :uv_eventloop) @@ -33,12 +35,24 @@ function create_object_file(packages::Union{Symbol, Vector{Symbol}}, project::St for package in packages julia_code *= "using $package\n" end + + # include the "App file" containing julia_main example = joinpath(@__DIR__, "..", "examples", "hello.jl") julia_code *= """ - include($(repr(example))) + include($(repr(example))) """ - if precompile_file !== nothing - tracefile = run_precompilation_script(project, precompile_file) + + # handle precompilation + if precompile_execution_file !== nothing || precompile_statements_file !== nothing + precompile_statements = "" + if precompile_execution_file !== nothing + tracefile = run_precompilation_script(project, precompile_execution_file) + precompile_statements *= "append!(precompile_statements, readlines($(repr(tracefile))))\n" + end + if precompile_statements_file != nothing + precompile_statements *= "append!(precompile_statements, readlines($(repr(precompile_statements_file))))\n" + end + precompile_code = """ # This @eval prevents symbols from being put into Main @eval Module() begin @@ -48,13 +62,14 @@ function create_object_file(packages::Union{Symbol, Vector{Symbol}}, project::St eval(PrecompileStagingArea, :(const \$(Symbol(_mod)) = \$_mod)) end end - precompile_statements = readlines($(repr(tracefile))) + precompile_statements = String[] + $precompile_statements for statement in sort(precompile_statements) # println(statement) try Base.include_string(PrecompileStagingArea, statement) catch - # See #28808 + # See julia issue #28808 @error "failed to execute \$statement" end end @@ -62,10 +77,46 @@ function create_object_file(packages::Union{Symbol, Vector{Symbol}}, project::St """ julia_code *= precompile_code end - run(`$(get_julia_cmd()) --project=$project --output-o=sys.o -e $julia_code`) + + # finally, make julia output the resulting object file + @debug "creating object file at $object_file" + @info "PackageCompilerX: creating object file, this might take a while..." + run(`$(get_julia_cmd()) --project=$project --output-o=$(object_file) -e $julia_code`) +end + +default_sysimage_path() = joinpath(julia_private_libdir(), "sys." * Libdl.dlext) +backup_sysimage_path() = default_sysimage_path() * ".backup" + +function create_sysimage(packages::Union{Symbol, Vector{Symbol}}=Symbol[]; + sysimage_path::Union{String,Nothing}=nothing, + project::String=active_project(), + precompile_execution_file::Union{String, Nothing}=nothing, + precompile_statements_file::Union{String, Nothing}=nothing, + replace_default_sysimage::Bool=false) + if sysimage_path === nothing && replace_default_sysimage == false + error("`sysimage_path` cannot be `nothing` if `replace_default_sysimage` is `false`") + end + if sysimage_path === nothing + sysimage_path = string(tempname(), ".", Libdl.dlext) + end + + object_file = tempname() * ".o" + create_object_file(object_file, packages, project; precompile_execution_file=precompile_execution_file, + precompile_statements_file=precompile_statements_file) + @show sysimage_path + create_sysimage_from_object_file(object_file, sysimage_path) + if replace_default_sysimage + if !isfile(backup_sysimage_path()) + cp(default_sysimage_path(), backup_sysimage_path()) + @debug "making a backup of sysimage" + end + @info "PackageCompilerX: default sysimage replaced, restart Julia for the new sysimage to be in effect" + cp(sysimage_path, default_sysimage_path(); force=true) + end + # TODO: Remove object file end -function create_shared_library(input_object::String, output_library::String) +function create_sysimage_from_object_file(input_object::String, sysimage_path::String) julia_libdir = dirname(Libdl.dlpath("libjulia")) # Prevent compiler from stripping all symbols from the shared lib. @@ -75,22 +126,37 @@ function create_shared_library(input_object::String, output_library::String) else o_file = `-Wl,--whole-archive $input_object -Wl,--no-whole-archive` end - run(`$CC -v -shared -L$(julia_libdir) -o $output_library $o_file -ljulia`) + run(`$CC -v -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia`) + return nothing +end + +function restore_default_sysimage() + if !isfile(backup_sysimage_path()) + error("did not find a backup sysimage") + end + cp(backup_sysimage_path(), default_sysimage_path(); force=true) + rm(backup_sysimage_path()) + @info "PackageCompilerX: default sysimage restored, restart Julia for the new sysimage to be in effect" + return nothing end -function create_executable() +# This requires that the sysimage have been built so that there is a ccallable `julia_main` +# in Main. +function create_executable_from_sysimage(;sysimage_path::String, + executable_path::String) flags = join((cflags(), ldflags(), ldlibs()), " ") flags = Base.shell_split(flags) wrapper = joinpath(@__DIR__, "embedding_wrapper.c") if Sys.iswindows() - # functionality doesn't readily exist on this platform + # Cannot create an executable without copying dlls on Windows... + rpath = `` # functionality doesn't readily exist on this platform elseif Sys.isapple() rpath = `-Wl,-rpath,@executable_path` else rpath = `-Wl,-rpath,\$ORIGIN` end - sysimg = "sys." * Libdl.dlext - run(`$CC -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimg)) -o myapp $(wrapper) $sysimg -O2 $rpath $flags`) + run(`$CC -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) -o $(executable_path) $(wrapper) $sysimage_path -O2 $rpath $flags`) + return nothing end end # module diff --git a/test/precompile.jl b/test/precompile_execution.jl similarity index 100% rename from test/precompile.jl rename to test/precompile_execution.jl diff --git a/test/precompile_statements.jl b/test/precompile_statements.jl new file mode 100644 index 00000000..1e598ca8 --- /dev/null +++ b/test/precompile_statements.jl @@ -0,0 +1 @@ +precompile(Tuple{typeof(Base.peek), Base.IOStream}) diff --git a/test/runtests.jl b/test/runtests.jl index 74d8d637..2b4487bc 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,11 +4,15 @@ using Libdl @testset "PackageCompilerX.jl" begin # Write your own tests here. - PackageCompilerX.create_object_file(:Example; precompile_file="precompile.jl") - sysimg = "sys." * Libdl.dlext - PackageCompilerX.create_shared_library("sys.o", sysimg) - run(`$(Base.julia_cmd()) -J $(sysimg) -e 'println(1337)'`) - PackageCompilerX.create_executable() - output = read(`./myapp`, String) + sysimage_path = "sys." * Libdl.dlext + PackageCompilerX.create_sysimage(:Example; sysimage_path=sysimage_path, + precompile_execution_file="precompile_execution.jl", + precompile_statements_file="precompile_statements.jl") + run(`$(Base.julia_cmd()) -J $(sysimage_path) -e 'println(1337)'`) + PackageCompilerX.create_executable_from_sysimage(sysimage_path=sysimage_path, + executable_path="myapp") + + appname = abspath("myapp" * (Sys.iswindows() ? ".exe" : "")) + output = read(`$(appname)`, String) @test occursin("hello, world", output) end From 80ccbe1de30460ffd12ea3f37382fd3d72c34e38 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 23 Oct 2019 14:14:46 +0200 Subject: [PATCH 08/80] add an example (#7) --- .github/workflows/documenter-workflow.yml | 23 +++++ docs/.gitignore | 1 + docs/Manifest.toml | 105 ++++++++++++++++++++++ docs/Project.toml | 3 + docs/make.jl | 15 ++++ docs/src/examples/ohmyrepl.md | 49 ++++++++++ docs/src/examples/omr_install.png | Bin 0 -> 52966 bytes docs/src/index.md | 13 +++ src/PackageCompilerX.jl | 2 +- 9 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/documenter-workflow.yml create mode 100644 docs/.gitignore create mode 100644 docs/Manifest.toml create mode 100644 docs/Project.toml create mode 100644 docs/make.jl create mode 100644 docs/src/examples/ohmyrepl.md create mode 100644 docs/src/examples/omr_install.png create mode 100644 docs/src/index.md diff --git a/.github/workflows/documenter-workflow.yml b/.github/workflows/documenter-workflow.yml new file mode 100644 index 00000000..6d433383 --- /dev/null +++ b/.github/workflows/documenter-workflow.yml @@ -0,0 +1,23 @@ +name: Documentation + +on: [push] + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + julia-version: [1.2.0] + julia-arch: [x86] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v1.0.0 + - uses: julia-actions/setup-julia@latest + with: + version: ${{ matrix.julia-version }} + - name: Install dependencies + run: julia --project=docs/ -e 'using Pkg; Pkg.instantiate()' + - name: Build and deploy + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: julia --project=docs/ docs/make.jl diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..567609b1 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/docs/Manifest.toml b/docs/Manifest.toml new file mode 100644 index 00000000..4a24f822 --- /dev/null +++ b/docs/Manifest.toml @@ -0,0 +1,105 @@ +# This file is machine-generated - editing it directly is not advised + +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[DocStringExtensions]] +deps = ["LibGit2", "Markdown", "Pkg", "Test"] +git-tree-sha1 = "88bb0edb352b16608036faadcc071adda068582a" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.8.1" + +[[Documenter]] +deps = ["Base64", "Dates", "DocStringExtensions", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] +git-tree-sha1 = "2b45ba82d7de1083af18f09af4ebdeb4dd99b9f3" +repo-rev = "master" +repo-url = "https://github.com/JuliaDocs/Documenter.jl.git" +uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +version = "0.24.0-DEV" + +[[Example]] +git-tree-sha1 = "46e44e869b4d90b96bd8ed1fdcf32244fddfb6cc" +uuid = "7876af07-990d-54b4-ab0e-23690620f79a" +version = "0.5.3" + +[[InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "b34d7cef7b337321e97d22242c3c2b91f476748e" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.0" + +[[LibGit2]] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[PackageCompilerX]] +deps = ["Example", "Libdl"] +path = ".." +uuid = "dffaa6cc-da53-48e5-b007-4292dfcc27f1" +version = "0.1.0" + +[[Parsers]] +deps = ["Dates", "Test"] +git-tree-sha1 = "ef0af6c8601db18c282d092ccbd2f01f3f0cd70b" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "0.3.7" + +[[Pkg]] +deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[Test]] +deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" diff --git a/docs/Project.toml b/docs/Project.toml new file mode 100644 index 00000000..57aa2754 --- /dev/null +++ b/docs/Project.toml @@ -0,0 +1,3 @@ +[deps] +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +PackageCompilerX = "dffaa6cc-da53-48e5-b007-4292dfcc27f1" diff --git a/docs/make.jl b/docs/make.jl new file mode 100644 index 00000000..67b4a8b4 --- /dev/null +++ b/docs/make.jl @@ -0,0 +1,15 @@ +using Documenter, PackageCompilerX + +makedocs( + sitename = "PackageCompilerX", + pages = Any[ + "Home" => "index.md", + "Examples" => Any[ + "examples/ohmyrepl.md" + ] + ] +) + +deploydocs( + repo = "github.com/KristofferC/PackageCompilerX.jl.git", +) diff --git a/docs/src/examples/ohmyrepl.md b/docs/src/examples/ohmyrepl.md new file mode 100644 index 00000000..2fe85503 --- /dev/null +++ b/docs/src/examples/ohmyrepl.md @@ -0,0 +1,49 @@ +# Creating a sysimage with OhMyREPL + +[OhMyREPL.jl](https://github.com/KristofferC/OhMyREPL.jl) is a package that enhances the REPL with, for example, syntax highlighting. +It does, however, come with a bit of a startup time increase +so compiling a new system image with OhMyREPL included is useful. +Importing the OhMyREPL package is not the only factor that contributes to the extra load time from using OhMyREPL +In addition, the time of compiling functions that OhMyREPL uses is also a factor. +Therefore, we also want to +do Profile Guided Precompilation (PGP) where we record what functions gets compiled when using +OhMyREPL so they can be cached into the system image. OhMyREPL is a bit different from +most other packages in that is used interactive. Normally to do PGP with PackageCompilerX we pass a script to +to execute as the `precompile_exectution_file` which is used to collect compilation data, +but in this case, we will use Julia to manually collect this data. + +First install `OhMyREPL` in the global environement using `import Pkg; Pkg.add("OhMyREPL")`. +Run `using OhMyREPL` and write something (like `1+1`). It should be syntax highlighted but you might +have noticed that there was a bit of a delay before the characters appeared. This is the extra latency +from using the package that we want to get rid off. + +![OhMyREPL installation](omr_install.png) + +The first goal is to have Julia emit the functions it compiles when running OhMyREPL. +To this end, start Julia with the `--trace-compile=ohmyrepl_precompile` flag. This will +start a normal-looking Julia session but all functions that get compiled are output +to the file `ohmyrepl_precompile`. In the Julia session, load OhMyREPL, use the REPL a bit +so that the functionality of OhMyREPL is exercised. Quit Julia and look into the file `ohmyrepl_precompile`. +It should be filled with lines like: + +``` +precompile(Tuple{typeof(OhMyREPL.Prompt.insert_keybindings), Any}) +precompile(Tuple{typeof(OhMyREPL.__init__)}) +``` + +This are functions that Julia compile. We now just tell `create_sysimage` to use these precompile statements +when creating the system image: + +```jl +PackageCompilerX.create_sysimage(:OhMyREPL; precompile_statements_file="ohmyrepl_precompile", replace_default_sysimage=true) +``` + +Restart julia and start typing something. If everything went well you should see the type text become highlighted with a significantly smaller delay than before creating the new system image + + +!!! note + If you want to go back to the default sysimage you can run + + ```jl + PackageCompilerX.restore_default_sysimage() + ``` diff --git a/docs/src/examples/omr_install.png b/docs/src/examples/omr_install.png new file mode 100644 index 0000000000000000000000000000000000000000..8cdc605d7399379fed1310e759466b2873ec72a4 GIT binary patch literal 52966 zcmd3uWn33g+pWPy1f->F$oRd3@?UzxRAU^F`5t z;lF3k-uGJTy2tOUq!7$|^!E@D5HKRb{4x*_FFYY2AiutQ1OBCh!X^y-@yhy(h}=7H zxV+Q#1z)4t2q@XeTIk!@Ygy?*7?@j_=}}tiSn27RTN_%~9KCGj0w+;DpTuXSr)6Vg zVg6ap$V?AHQBU_X9sOr19h=W|G_E$>d!VWFkCnUKuHxT|h&d z>+PO`Q-o{MEJ9`8BS-^I%8I2#=RaUKE`4R@t|{ z36cF}dyklC+Sf^g37mRR0_ZQPW23BxAS)A@aHZd2K**{Uu*&Ald-D=t% zJ%6J7>7s~5Sy|aX12^H<;Gpde@7u?%mx@)f98Y zrK@Odu0MiPqH@CV^tP{tVUk{R>uew6zTc7upDHEot8TQWxg<7p9X7Rm2K7$_NB9U| zFEeCxhGLDh`vdFYTGvCz+_A9<1$=t^w{kT^Y_znYM-7{KRBdp-dQd|HADu9Fjg4m* zN>=01WOJ3vi8MpdWtZ}hdwp`s$)3M1oUcezo&E{yj|V=UCscUX$==w;Ld|MOeLV&@ zrSd=tE`AO2$DHpRXlQ5?`37E=@;cfYT_X(w^kd^=ss5*IX0r{EuFhBK`T3slss>RM z4^YFQ={MceQbdSoXq_2Q;Tvt4Sy{d6%@*@B^#(Z0L$QP27Z|-gJtW}_&vU5n=Zm2f zX&9`odn)dNdeas(8AN)$X`SiHG*Mfu$2iBt66ceO(~`7EA7thbA4{7HbuUDVHHC1= zBnxkCPm9hn+aYc%x~SkThCS)%JVx?#b!E?q4sq-|L2Ii&e01YwMk5Vul($M&dvdR; zC53(?#TR|Qwb0Si>pgU3*PDLed}DcZ$T?e?*ocMksOB^57hYjF`_oLdTE}4ew-mIYiVj%_whM6))a(A;iiU2VJ1X`DluTJN?KT1n~9hZD} z)2vVW(=FMvGf{32*A;}(9=KB;!)8WTxi>*XMU~WxnDr4GyVa}TK(B0h@M5jLRs-Um z#vbzS3hOJ`)9mal2ejK8p06W_c?yjBZ5b!qdwbzAF-km33vCG(?gbn7H_-PL4p5Gc zjybuxv}U(*3yX_AL%mviQH3@SV~LCPSABh3j80BY+dDh(7#OO%AzOD-wJewz>~wwB zucnwTMkgl;DHsBSFqkdZN2Fr|y}jRzXGx`&&!etyZkD<31;02U!8?niI!ts#IR8~L zLgRHYAl2;V%DQ9i-&}3_a^_&K8?ct=xf_t;dePwfV zDq>g369EIos{0FO^@T+itMyO#g0L@JMn*m+k9Ws#jt9sROZB(eZqFovaUCyRL~U>) zJn#z%35lNES!BQ3o+~6bqOVAepsHLUqG^kG2@E_);1&h6l_GMimpF*T?~dASd6wCLM@&@lUBYR{W9?x66iBDH2l6+ z1BdZA)3Z&Rbe+J`E5%Kv6|p7UvBqpv$6(ua)CJSDJ>hktL%hc7L{O+w9l`lj7SE2) zDO6(BN!GMG=h^NPT(;f)+G|L5q9|iJ+8mrB&sQJmKK^uM zu)3Bbq6d?ft=3Mvt3vq$Hp_>3DN|ZT`p5tw_}FO%^RbwgmU)5Q2=EKl)UZILrKgKX zh(vOZPfn&vF0#E^3WLM>;evGl?M zn{2I>UV?f8YvS;Z^TjT*!EnN#eslGWw&$s8Z7|%A*?MniYmf=Q&&n%31CR6S~l zI)*F-7IgCc;T~37>|n@9UPf?m6H%)DpPhpZ;YC3Nh&G%C?SmOHx#E1O_9?~^Gi6U~ zdi2n^nBGL|Vb*3!-@pme$eo1f=e`rkwtD5rc21_U>eL+7kra&*V3V7>MLo4H~ZGQItv;UoapFs7qzwH&(ykRCj{1du|EEAv$}d*xJGTJq^=psB$y}h zTkAVEOG`_mA#bxp1%*aV4oS6m4Q9@`@R#_<3z{6FX3^rXrmO|q^>Kc@WFjl$alSa4 zT!H%5DHi*}!9x0BKc~Dp+|+^s3M*@CJx=7(cynel|Q1l3cg@uL9 zDBp`67nSB&cjBbktvFd&FMq|jfjja@#V|UeHHlFdoXpvmFG0*Lxr_QM}B{_vR*c& zM#jMjY2t15ViSVJ(c8bK4a8wDZJ}S@dpZV-dYz5!pJABz?Xk|FoF0-Q&2Y_*QrCG{ z*nM_i0SRbtSe&+oBe&0FWGa$bC9IyMrIuFH{&U`By#oUsDZj1YpU1Q(rvnYiFW@?- z;ZGVYU%04EqOOlH0p79rtKk=aMn+hF+x&Th4I+t$}O|BSP>WE8D$DZ^!%{ZP;_`0T_!$mG%e zOCi&?@!^U1s8sBG62p~vbT@t>0pu6EJJ;?$Tre;&8;2O5(0jZE^mev-_Ff`Y7<_wr z^vuZ0vb^Z4e8D#lwmr-R_utg@wi{ruzCV>gzlC)`EjgH(Vfpzp=_Atlt5T%_i~OkZ zbXAM7@#TTH3c5onX;rDBry;QaDC9{Zj7?$(+_?pjn&N4^$oxo5EJeAkbZDyk}~LjCY3u8{Yk&Tf<`_|-CWm$i zlg8dJG&+YkT@?5a*jr{~WSml5qg^J?^)6OdSO2kFNtEeC!$9n>DxB{n?Qnd>;;`}& z1!ePm?}#^%t@4CxMcvfJKr)$-lQOZD?=5Lz!1GA%PL{77Vs)a9LV*F5dnLN@L6A%0 zMT*3elg%f6+4W1>Qh3sgY~9q{!Y&UliSHK`nU2MEWm}1YOf8e+DD_qP9uqg#7Zsh8 z<59a&xBNZ?;o*@1Sykwgi6%13ndoM3`;U|rCe!HH-o&XbJw(VSq0VshN4Lt>=p z(mH8=LPw_qWsmW6QK#Z!{b-t=wcWu2zH^gxH&xU2zQ6M{)kJ}EI6S`3i3p#nT>~St zsC3e6pOG?o=)l0hk%_Sg)d3ll_4Rcc=PXnvrYRRrr-=Q>U^DjP?d9mHLRIN}1quDZ zBwjna)kYQb!$elMQxZ)ZJ4#t~`CM!101dGIKtV%yXXes_n_z2eOC%NRo79w)uuz0m zzq6Q)+9e+ZAJFN-sW11VPKJMz5tUkPZCvou(2%5>WQ}pY`XGa^`|=!TB;-vMObnL} zdZqES_SU6(rhhSkMw3f#CvEon2sT~zdrfJ%DRHAxv%oF5(aT~oQI92 z$a*;ctAr@mv0GlS#bo_?mE?v~`}DIHQwL*( z->^Ad_>(mrBK-IIHuAT3cS~HS6=Y;&lyBzAhLY<)JG*^*;tkA4r?1C(w_OhH+(ayiw05-}yiPu!FmZC7WTD|>fvZchu}#*^b|i;M@_ zI?sBChV=h*NdSR^TmHLH=bTLO%}Qqgs$O0;nJrbY^0)#0ya^Fkc5rA?XzOUZ5Y#c; z@9^22^@}bzP?+c{SBMvx%A=>`gEY~RL?sQ2XiXOTu=SYhuz@} zQp9)AFgjC#7ANjv7V>58%aAJ7MxCuMeUQSqU9R8#Ar98-$kDn#J)?ujQ)JfaE;6&S z8XX{Wohz*`{X*Qm zj%;rRJ0>S5Ex+54*i7!+2}&oDlL67fv&^GucPk_yps4s3s9$dwoVF&{w_NB#A2XQF zn>F1L!B2$z0T%{zp7pefZwFo(z4bfyWB>I0U%ewQL_Z^pmK(rj4aW%D*)iE4E$HlT zpS+g*1nz8BtXMxDp5rh{jv4 zY!rHXc(ge4I4r;BLg8XDI1Z0uJU+G4Tfy53^Ahu`4SM!B#wt_~=)}^~w6$emfQ4Yn zT?+0Dm+*^w3tHzPadcJ}m8HC%lli_47jApW+%{}BD+qAf{mX|KR%fTuGMxMexe6C~ z23Ia7q{20om%XC{l#8E>+W~%P&9!|l$btX2yUZjv#kiR@bmtR*$maDk>GPzOkPGGV# zJ7{0I-kdSINMq$blfFRZ3@pg!!%a%j;J< z#D9&4sA_Z^hN~m)lz3aF9%Et>voDuWlTO zB3`Kks!#jJI|^OlEAOc!eDc|=!}rz?t8(~XWvM_S%fs7MW zoeN&$Kz`{@Cmmcb)%2Z&3mJ{H2on9-T#Dtqv$tG4dcWjzMgRR6qV`Y!#Kc4ltqEIn zQNq?jCw(9$V*;NLERA6gPi$ahWTvT7M$KwA)q;^GiqGW(9zeN|BxN9SA;{4r|LL^^C?dkR@%Os?)S{u=CeAlBK^~{ zT3A?4kVy?3+W4VFp?}?zYWm>poI5n#cxW#WmE7j-*PP7mBX6SEvp8R+1Fh2?B29)R zu%De#=McYk`l#tb9Wk1n(GZLW#>_=Rp3HD9I+pkR zb7cjwm^CNcSj0sN6`Nz?Z?ssf@#DC1KaeIz!7l}aT128JqsWsMHj$#IZ=loQ!Bs{> zZ^fD9m6^Kj{(M|$cw&E$jur$zr$j_2;$I(tK)+6j4IMS#IvsZN`tFy`@>JlP!F)s~ z<|KDmABYjh0b(Uqc|u0q@QMtIIoLc+Q#Ha=b&>dDA!j;^LzkpioSk!7O$EG<;5o~Z zj34-U6Y%aBE(OTodm4ADShU#v`zzG@5iQS~v~u!teK#>fVG^(-qe0p&KyE?Eb zWGF8qB%~QWQh#!OX_aA)4+Y^zyK`O89MYkNdRL{d`?N>UBr((I(016VGu>6eZ*bhV z^sVRsj))!KXX;^3dQk@F_vL$)`k3E9t8dOWb^Idy1I%YwEa;@g0umJ!mHRSXQZN}` z{$>n@NJO;qt**r<1Pi-yh z&d!cny;I`Egq*g{ABX#G%h+|3<)w!Ljry=W`TWz<4Kac)ggC~hKuaAWlE|+AOOoF= zGT*=Pe&cdBC>K|G4XRD?U~I0wB+ffw36WIAqof2*Yso`3RiIFOL)&MXT#3QWJCdI+ zu{A?esxT0fk@@F_H?_8m zcob>YSzmASc4*To7OUypT_#$c9?Vr|ItA_CH|p&mZlB!Ln*Eh-oS z55b+-U)?eTF>Y|i@eRN1$g=RcSwH%w&0WJj23zOp4v*CEM!Bj-v0QB4xzM;{YU z$vMGKfYN)o@x)E6`dFaDc;UX~;(Pj#e54kUu;1YF)bGQ{kjp<(2Xc=*Q8Z zb%n4xd!H{BI(EWyqNfx(_X_fd|Cd;b&mw^$FL|WW1TN~vR%Q3Hh#)cu^+F><@rgB; z1?mWv$4be7Z^t@YOVE{dMkb0bH5nJF2hAY9*GrB4J7xzD!LLVrxYhZ_42)op6Z|p} zvFsd+?V9d~MfzR-qd zLHNXg`~VPvU~|Kgm5k(9+#>nptw~o&IH1O(e!Tuo60Sf>O656HZ#i+_;k(+;g-?hZ z-Y1`dlh5vDI=i+|i2}C690gS=GV90JwXBXnLc?D$z=zk}It-YJPn55}XfGVa?OoU1 zNXsqsmfea2qdji-eHEGsL1(;IbeNY<6(-~e9T6v15V&nGB)wmb80o&`*M=9=9}7tb z7KozMj&Y-h>?%JO>GdepZ$WW3Df6yxJB9-(om34UMAFQm&fu24Y)ziNO;yu707np1 zZvtXs;UMhLA;@j(TUK~kqB9Ts1_rICvHr^Oavu$Bi1_U6syh`RG4tWaTlHRs2@4P3 z*`AZ0!#HNi@z^h4`KVH5+`1!cAS@|D#7?fdvlJXnt!Z`AMo1);*i)H5?Hd>wRXrxH?JP)!KE;azGeZ znf-UwWYj6t%L3(16lg3SuP_&mBn0)}NJ^mh!z7ZVh2)*eBlJ~49ciDvc!7hUcZq1b zJY2m^v*29^+qI2u7={!-^w1xgFnbR^Xer~B;Zi=8cBwB|g%jqk^^3ZH=>4dXs%RgW z`$&leQ?&0Y8n6^qwk1-!v;V;AG5mHa`+>#;A#J?Y`jD0KP6b(*n3*^v-Q;krtQgrA z9Ze^fXstDpB$j^rS-R)&{9C&F;SGIm0T$Kw3c(Vc&_4&>u-0#b*XFH?xk5GDQDlZt^EQwf2VPQkaMmP0K?uN~QwHd#5 zLt~@U_J!A(Dw*~Fkw!FWV!a( zuUx&PDNtpea5=x--y2*Bld^s^WS!;8_0g5|ZRS{+BoymG)o$jD4ojYn>#7S;Drl$wutHZ^uZUOFm ziXmGN<*l|0fmtQ4XBqWcFVtS(=J!&-6>s6orR;yP0FF~$K$G5OekmY&ocV53*)lr$ zGbh{6w-axc<^)WsjkFM8^h*v&J{K*(8-D|bE$nhn?ou4xtfPWdSH}icCpy#XkCzAR z$H|tjU_PAJTHTP{T{by~WE2&_qNB%66v;y0Ngcd5eWf<~Rc{G-mCuEG3H7L+>N%T5 zZAGJasf{nwBRZUFj;}Mp(9z&8iQht;5Kr9JCxjd7o5bl%14{g|^72~eEo>d3km3<* zO%#easeq_KR@dDRCW=oQOS=us2nYzu_4AD?)UQl3Xs< z-s>7-IpC_(xfmLl1y-4AK7RjJ?S0W!obgJ}G5_%rV1E*8yjO?ca2f(Oel0ZY%^-wT{_@2 zmZx>Nr1&NBC!6NV=ZKb;^xg(WKsw_CxiM>gZm#8Z3^Sh(oFj4U+QaIDXZQ7xCR=+RWAkWBRjc!OKE>CrBzZ5gIW~E>_$6d5MoZj($bYYd2$@h3 zaq)Hb;@XWuL9dO+hqpG?M;c4U5#;9TZ0XsGvFhx0jp5>!@9c8e{PO~UsWe_)ZR5Ek zorIqYY)tWsuL6nY|1XMwQM5@wpuGKbB%9{WXn(yYZduWC^IVxoD&Y&*USTkpI;N^1 zuZ|Yoi*H@GbMhp~`>J!*5?iO^C6krhaSX0hw$Y3bF+X}92@a zP<9hl(I0VQ`wGY5D{zb@O!?oN2$H+$f=3e_)-zA3exI2R~AoX zik-7HMnpv&UWbycao2RR3XFF)@P@_s>E*>L9V)%Q)EHf;TGbo8s8Zv=h2ZxK04@x6 zR{VT?o|N&`r_))3Wj6eH@a;k$GKGqssyp~{x6wZvi9k8q0#PiW<37= z*>Xj@jz~Pc979&}FnbT+P>g0T2xYPDl;qSsY)EQVE{p>ooEr9*s<2dRTyDtn6&gZi6%`E*;g&~BW#58FVy4uL zcyHQ_q%6??i#CPUTE+>;J_GyeBB7w5v<7#VS)DgAzm^rw&CZvu4%>o!UL+uja;D0b z-Odi?rXrNnyPoJ#f0UF+gj_a6a1a*S2I~ux_*-C9 zBcPynbp^$*K8rp>XMPGB?Z0?=aO9{-7OMI8@)1Q&vp9KSIE>p|I8}e@e05LKp-`!M*1}f|Y7NW&7 z>fN3AE(SUlC;&zY57?Aata7XhVhx;JK|P-Z6Q@XgZqD(92@9EW79&++l_zTag~&)m zxdKb>TDg~y)@J4#NL2a?VwVxn5&EwE{^V$>@JSMpY(oKFAtpxd&voxlYN5aQf=5OF zt$V*>LpiOn;@IZiXwpDDdonSvRB2T%waIs|m<8pI6P15^#Cvi7GDo%=mU|#sOka?Q z{p{h3tYzos?q5mr-+qtThNq0*@7DD>d*>iZgNMMZk_zgR7kWg|66obff`CPD;) z2vrt~sTBivRqboLU(nRz*vfV8B%Q9#Q>4?SZrqkIzkK;}!fGkA%2H8TIeWb2Boh{X zqrWSTr^&B9-V%q-xU{YbuR1w0veIFRgU#jA5s@2yJ!C@F9g_TQYCrbA(b43`nICd) zaamRd+v^;a$ryk;l%7XxiinD)fy!n;lP1LX_#JSLk$+E5Ok}9vRe=fkJM=^!-%(%= zMt%l2;@*vPULR!|n+#MZj*3|l9&HER9U15V;Zd&nW*NGQH2F5guN+Y*1XHJXSj^IN zzvULDxxX>nzv~}1jn2^*frrf|xc~p0n!!k_=zkk9O8+uoutM@i$Hvy83B~l9NzU9e zIE6yAAL?MrG(v$?dR!_CGGAh-8Yr|Wo^NsD$KV_Qk zfYT9UVb#guWM<$yXB>;`q>JwmP2uMjqjAF0MeDd-F6pJ-V_oFvJZuk)!MKXVi z5|nq@GDF{pr-Nf+W7n2$#PIMl(-DS~`=xNDI~2}NPitIu-UEdS@l0}1Qc3{6g@%Ua zGZ&WzN;2MsyflmDg3zH8s8-MSXFWG51>`3gzJR%uX%w^t^GDt&_OyggezJ3%xE0*So+2D> zC-LDW&IteZuCBP_<9D(6N}v2^jDd4dTU%SURC3=<4%RYn=BAV{;C|82j2QHk;MUYI z@M=@=N=RV6eCc{`aogW(wp``;@(xy8di-Zs+CWz!9m#$OTObR4I^P? zO9@KDs99fzQQ3{0Kxz?{g%jXLeV2w7RHwB|g){YHllNUD+vC^!vr5GJoPqVSSbvdL z8jcS%mxIH(7O<1f4@VVu;G+JAx0D%0`bm(KF(NIat|vVO#<$bB)BEO1F!abQ)R*6X zwv~M*&)4N3Bg_^TA)v>3PaR}qa;^U>HBx!?xpF5Zg&3d{yc8&B^LZQR+f(u9=Qf#h zLDi%|Sb3)pKhx&H1H$ku))O8Y9B^%yG^8wh`up+0c4cXK(@vfR$78OcwG8psw93f| zibU3%osuW$(>00hh+%*@Cs2_qCdcBcG&g?1qaqV!n%ZfyTQ3|fr^==vP(Xx7%pTn_ zW1}G;@+ViNoDN-OtZSemDOF9$FO~`~)OX+R(!=p9N+FxZ-5%R z9AG6Jx3GY?{4EF@^u8lAJKHBWmlW9JURjfM66D5{MV(R2k5n?rYr1{yuwr<$wD9U! z>azg%>B?|wHACB-D5O*zy4Y2QM?jF_DXO7!`ThjCxvBqp6J|pwvPwlw-OvzeA*_F9 zCZ>b`CRJCU|Ltc+Mr2A#v&;(@tm1MpL?-i@U(4LgzW)A1w7Aktn94~1K}jI}K^;}a zC$L;>UpX1}HEU3ge`{@@n2yfBJH*p)IWt+&a^kw|!z|<>TYa zl1LUkn5(=7ag$1g=}=Lkv+l6{;Y=rLqPpeLoL3y937z&j4VRH&Kd@>7`(6t4l@P>+ zoh~);vEQB_1Vu&?PUgoyFD^U9*WJ9Y$9B0}Jn_35Esazh(%G#hzF+Pykzeg;D#!Zk z``DW=XZ|RJiHfoX$jXOL6F9C9KaaA3SvuYn`=wBYapyLZJ-OPf{m^COJ`NCkpwt1h zhSKdGLPBEvH*<4AP!x|>YvRNPl9^T3)Rh>0`-fsIa#h`(Ebe~(jqQwn78lV>P~c_l3a zD`~O+m_AvawaC%w7-7HR)II&?;1jlx&|5GT0kYv;R($IIRjYYDiXtQ?I{q`(mPgJ4 zCVNZ7dYfg3cQ+^?UHm%T`bqtzM40w?tvhQNeLN2GZ_cVs>_O*A0Sc6p9_Bb%jO76= zmJ*7OSAKRe${1kkv_>{c8I8`%Tyl^yB{p)uJ-%tV1q-Xh7aKPFAutNoS6bRVpi4>80mB8Tl#>EDgS=r0!)?J8cReb7}K7CYOkKZgw0?Tt7^4a zHHqW-F}$nOMR>r`wBA#@$q+?wB1>RalzuFQ^)kThL5%eQlL2D5SSTziYUqzp2(-(m zVznlJ;Q7qkn-Ky7EwZ_A?Y;laI&(-KEAxAwYxV2;QKpRx2_7(aVSTVBCY4>5i{ROK|GG;B2kwt4l9Y(73Ij& z=f?W%4e-lZOP)T%;c!rVwm+D&t*EG+X)+->7cyOHM+#H`b9TaXbp$ExO10 z!eHX3($mx1+g=CEmW$Q-RpQyQ{co-!;pWBhDt=%xOB8&%_euY7Kaw%CoSsi#gXWQO zzW;W*@DlQJ=CK?Cf0hAOXc|hsNQRI9gOc7%5U}M!8^MC5*|I=_0<32NY|zG|caS@3Z4e%u zy6Jkh>~3|miU=U8GnowydVE_sk%d-6`Ghh2q^te>`pGF#1ik`dDzf?6;T}`!F&16?i%}R4b@}+ zJ3A$UpAR{{PbEQ-do-nhbNa`@b0cFJT?IIuzefaX8;vEdTy7DcnKM3u4!wibA3eu> z{D0CrU-ey%IX&&n!~y;i`4FT&y_@)lp_;bCOpPzjF8|^1z1wG(r79R;!)*s{Ha1P+i$pB$?&7%+2_eOFUcMnp^s1ib?1ffa@$ z4Y$q`syrz)|1#p9Lg^|0;X-nmE*ZtJtV?3<#%YW1p^a3ingM{0wLWttYfP^XCV`6&jHLUyp2qZt+~fl6k?WT6_9w%zxEO(F9`DI2 zV4%WOnWOMlMQeHJgF2sthCrsJnLEuRmw)n|@|G;r!xl3x5k+jJP9}zsu_0&^QDvW) z=%m20idw(^5LYvq5xy0l+StbAhkYLo4e-D3dPlMa>f$5JQ$*WcVxW+TN~|p~$?>~; z#cn(=<{KiuF}3q<4z7}j3D1i$otmuR3eb0ncl--Hec&WO}DZH(VTb!LlV$B(#q9zxVX3l$`8(=KECm)G3AdTl?mML zKax?XajB`LQY)U>7$#S;!$Wi3t@RSyeuX->ouR`;=h+6M0P&&0MtVLUnAWzo1b|B0 z9WC`wQO+ZA9WMfP-0c!fV^u@Ud~e=&2={3w1eD|>puYqZ{3HnqpxXb$Z2o~hYgl`? zq8E7I@ijFLL{c$r4|j07mRCFT7~#7_1cZc^AX*>K9*pSJ`bVRXh28Yg5v{MY{Za1@ zC4%24E^QJ{r;C;ur7sRjEX~F0!=U*?mu`7Dkq>4Zox}As;u32~u42kcIg=D*$4xN*LQbZJfy z+_#dy0M1aYKgva%oPeyV#O8-8EV<+?%}HEJ(g1@WmJ`NGud6L#;Pd@zEX7M*@4g&% zghoYp0iW|cs1l^H*-?q`#^~Ch`2mkrqt556?so)W6M?K%`{qb+RA4m6ll75nb*7pz zo%d|_*HX|{ayu@B4fYw7f}}h^Z3w_ssyyFwlPkjmhlC#RFbz!)!EazpXG)AXTU%Sz zSg37*X~(z;l75PZ4Mot-VmMmHdWJfHkH_)zxkJIw#H4(6_#%UV_-t?@mcPljBkKnv zqwxd@!yxc0%MU{sFB~NozcPO| zIA5Qk!mr@)B-YtpYdxyb(bLV=>O_CpKtM(w9K?96aa$$Z`e1jPhRD?|)rZNeH#YX4 zb{namN!n(z$q^B9%ADd!YpAH z_E3GS-d>;c6X2%$%)lT%WNG^U!eid={av|I-W{!fxQv;7XDdOge*x1wG_Y}+lnwDo zZnEpqnjSpr7&aMoXr0c10-+K|e61auH}3=AM$4QV2vp}lT7+wy|* zoX!V*c^l0o*4^e)h3eE-;-M9{mwWw}X6(8`R}>oOh@hmzM;)?0Cz&dq|GWni5E!_2 zL^;1b^&sYz$m@|V6ilnN#^R=OPY=4`V&daROIC^R8yrMHr{iCLsMc9%H=@z50b*cq zP((^E^Bdm_e{0GO2S4*_ik)V+@EXVNycsWURfJ^9qRDX(o!N0jxU?< zP`djjn^V)%Ww%=|2{YwE;}w9b0g6p;!SLMB;sQ+yh$O&zbpf6^hC%PA+C6Z@33sED z!@05np2{Y{ZM{0y*Ib?Q0jwjNS5a10R^xuen`)HE?i`HE_tx@yh8J`N4IRwe zAG03vnVKfZu&GWkY_+)E;HUpJYB+11sgs(2*Gq~>@-DaVY~23c>Knr8uf>|j8BFD0 zP8XZ)u#`dax>4($Z9#PPbI^cjU&SF@ zXga`hjNfa4d=)b?`>|9vbL;D!EYzDxGF;keLNXS=YxhJpym^uC~^WM`oqM!@fNy3sVPzezXDGK9-s>Q4+)#;+4pr8!Ib4PP= z-XS1gWMLzT=6~c*`}rgl%TU;J`mXHAdIb<__`(nz+#iHHfjEsCzL zuF*zSYIJtiwe4*WA0ZsUKup0SY}ZKpmZd!CQYE<-K|sswhBm{>=Y>3%K`ria_VWfb zaQDI92u{q{ocNSLy}tO&MgRo?LrLo<^o8|o8h>YbQ|Q%A=kHJ9usk4w1}pelN8Mck zhI=f%yNGi{0lMPr&qNenprpx*+5KjUizQQKm-~iC#TpSIdx>>X|A+2tk3BAH~KcmR|>&ntF@&RHum<YwLoEf2P%?rAG;f2(&@whT83U2S>xqOip88OyeS%pbZF{UtV6N+u2?K$M6~# zm6Rl6VuHoMFq!296BP*B*?*%tEYU?gOKaPqI-p6;TMB(n8|<#7>D~Plg&@0ro>sf^ zCMVnM5a658Ss7%=f=foe*PcTD!{m@Vc`en1{ zQ;uz$n~~j`pTlsN;Qpg73q6Tkb;KmAmVF;QENib4AWonw)^G?U~Mg{>1|H z@t>bxtgivv*avu(QIB=647P}#SI5n>sq=&B>=6cQD=W}i3k`<837b8!GZuk$MhjCI_h6pK$N0 z{h6ln^_I+uY|!NxjCs|SX*4QXnx_kZ0??MiAy*c6J$t|c>J;hxRCr1!RV%CZm#_~< zOqRp4hBI>!PD7JDeLbu4x@1>ptlx%T=+M$lrfQkUGLYwnVkY-qyd2iew}KopjiO>I zwJ;D&6zu8{%Cb6Wp}&{vf#crN-6%WSQUwY{eFHd4f3t^wP&v8dL;C@ z+{KBe_@XEjy|Q9Zpxjv1W7Gbn{o3dRp(&a9t>&^jc(}a37@6H#(}Ug}WsuB*EzfkJ z1}gpyldisQkM8B+tPl{%FWa_hglb0|Ij*_{Ky0@OLUC>p!l2LwO()m3nvfv$%Ep63>%rs5teRlTeJwvP2|W#Rh#cWyBRaKxcLB1}HBY z$F&K%v5I|ZMa8YL$uk#vTW?dZe=0tnPF_Mz&ljgGbt)^{#=H~b{2kixwoi*$n~xhk z2w&W;!317i-|ql?S8rdR_VLP(j_&5RtGYGGlD~qn<&N+BoBrnqv~0{eXTIkXBMfbE z+W5O;sI5(?&P##&6nMYw+48sJ2I2rPi?6poq_sPDpgV;@eC}`ohaBcFE$zRb2lB@g z4VkjUf4W1o@K`*(zGl|J!eR}UnyTGCK*|eL7uSu|+2)@r@I6=TE0n)3l~;6DJ+0qx z8!5j848R*IviioM0kaR57}LMA7LB8o(ftwrgN}d~QUn7`Wb3ZhS`_(mB=jYC6T~Xc zcK5*Sct9lMe_nk-R_Nc?9Y_5GF2vq3np*+MES=2wj^c9)h2Gi+l^KQB{`trg_ll^+ z@d<@?&*$q`L)LuSi*Tq%%pD)OeRd&fN97oS$RJtGk(g1hV%{@{) z0W19k`7l#zg7MrE@(hn$TVnzCjlA%z+FHLK^EU=0=zn(eoz2F?juWz%wqi6ZGdFIyWw2^R@u*I>Jur=V~IOLer ztv0>5aFI>gTsK5D`!+bVn)WGUc~Ge%4VpmknYjZ9}>%E z#5^R^p@2ZdV<(@_phHz7UCAX*?iy?R=}QIA@w8_3;>8PxQ>bx+ zE!rs@Tk0p}*tET62aDr5aJLEtHxb~ zfD2brt@i8Wx=R~w<3g>0F%32(>0N;gqUx6_JE{@K;U0m*79ax|)_^w_uMGzlehN^~LiQ}{^T5PCnxzy}0^HlPc93TYF zT?e{t>xdWX=*CY*-A}C&$&&$#6|q5c zsNj0M`FKG>SVzI2OvqW8720}5_g%xB@=D}VAJ*b^$Bc`_s)CA-kKd5s z_5S<70HUjFeU;-ZT1raFpTe=B?}lNGFKCwgM^2sK4)528zi6~)v|Ry5$v0_fl=P=< z36uj~(=jKTu!@^%$Ai2HH@^Dh1uPmr0h0iGXB{3rFYdSd+r2wIBs1Wu!$il6tP>mv zN*TS1QqkGUxrI=<*=g0^HaMIXu?!c~l1P~SAojbL#Qi6c7Ob1Yu4aQZ9^@pr z@%>zn-&`N;-iEj^_uQ;z{$5-tt_P8IskNo%lU7kJba|Wb+g+NRM&Cg5x8WTgS2wpx z51a9})L*~2;7~to`1w2#**B!LU7@U%)GZV9(xo%O2ZTecoexl@69-PWAN%HQARUQ6 znH&+0I+JZmaxzo0^ECfxQ|uXxn5gzs#Bu5kDyn4KKVSbdx3kbXj%3?7 z0FfULHQ6}q%9m=}&T1yUFl5B_aPF~v^yD0-)#~1K&cb3JCPHnuRox)0adZq~Ip9=RQK5I?qavrEHF(yqleHrL;=NWo z=uYv1{z#m@6Dtrm~sp zndKK9y5LQH{erejI}L~H^FV+{XUpn8{y64$L9x!n`3n}gUxgCx~N5c@JP#pf`Veh^#Hy_tL{gy z0^x{6d~$Nnlum8ts3|+9PZHplva&uo?LYHpJEJ5O)f11Qn^35Tp&^^|(adDwdR1-h zmd?}tcZXg|3JipyG_j2_GAplS9s^D=g^WRf+SuHzy4`0<;fik$NeoX=G!7N+sK~D9 zObvOp@^3Y}v!~0z#^zU5Rh8OsIK%zJgB%4$3cGjI*cc<@7?bM5MH2ZqtBcg27mYgL zb_WWW>^^YWCG>0$8}zOjr2yIT3Cks6jCd}XamG^zxP~T13hG%;5EF2<(F{uO5-P4% zx3~LGTX}|e74jQAZy4`>0mtE>`+3FBKd<=v z$t?>1Dw@IF8W*)-4#T(>wBwfEI@?e0dQ6Ofh^W?b;08LmB%c2jtcBjcDNyV?nL)Q) zxQ1xoMrI4I9Ig-6?N22`z8pSZSPl@*TYYpE78c9P%R9jxKgZ!=&1{?xN=tzN)~)hj zHusB%bHP&M-kv#RScO(+^iESj%WvF?m(auD@??yfj$T*SCybDzd&bV-U;iKpC1rzB zPrf1*XhUORuhj$Nj z0incjXhCGQWglDm1Fx98JR?Js0R;^Wljyvr7wc7xshG7jVK_IlfD2Jet^J#`?XP^= z)@h?gu*cAy;R2>PVj-&7C|A}))T7h8Q!L3gb-bL~%%@-e=j2O);=>C?F9E*ABUEO0XxIU8V zSpI^aKHB;4V59e!J47*N*Ri=B(>WEGAm6700Xu|YCC z529tId}lqhH(z(F`qS3`KKczF$&@g7dXh$J&c|!gPQwx!^w^&NV*p{%L4roEhNWhZ zw=!yn2n;#H9}Ak;1`ae*oi7F^K$of{hjAKvOA*AI9 zEwFdGesHn9LZW^!Oz7PsvDIlpeWBo8S}>T%OKo#K`Kzy*Vg5u@I}y;7?k(udlkw)n z!^tDU7Yef|+C_TjD*h+VjZak@*}QbLWdYZnD0%#vh*c^?VFRA$`z|MiN3tDo;hSpfeQVEWZUS^U0vN1n5Zxsst6<#J2>{jcz*j9hGpvkku~c%tYQN*hm%C+Ka+=_mrl(%b-q9};&mLj&TK+t=%kVh9sSy{D z3rlG)rYb)Cl2}p6W`O0);<}(7qYx@y(B0SV%d`G!XJI+|eK^aaUUs445BW?lq_68W zOCee_v4q)Wn{38>*xj@T;uCLqzu}vPfqN?uoA#FAEya`qeG&5|B1Fl!Z^Iu4Tp?_b z9v5cw`P%B0z@FLr4y?XDB%(Sk8xR@~W+1VTuhg&B(^DM7XthY*!ov%LUE^+4jh%zV z>nI8>8i(1H7%a^ChNeV}?}d}*_6}z|yctCD1sD<%5|6hBOzfN-csW+PGp)C47KCZ6 zv7lwmb8ZNd(t#8fzx?k7#fM_y1# zJ}WoDRlGS0Pp8p;X&$q4)TY)Ia zS2+746Nv&Hs==Ov#Pr&phuiA=)QZ~5cF<=5`~k1uyusUWs=DI`Y48e7Pap_-d!1Ze8O$7KO4_VFbZ(X2 zDRp~3F~^4kZL!8p7}`%Rv1AC-_4=I&?t|;=wy%p7F17sHg~S)CFYc!E@N*U~+z|W- zQNUP65UO;xuuL!c-`sQvlJlVZ>(nDoIWy|l88W!4|F(uSE4Ia54b$J#rQ)*CrSyGz z(+fxE06mxWd-$V-=j`nr=8^wvtl$^dM=Z24eMF8bl{*!KbodwU;W()aRy#}|VMd&3 zDeHA-)}~?((@B}^%t(L8mHtse`7j^TwIaeVTQs+2XD0!>%=LSuTsDqIYia9l?rHx; z?PF!?Q>m7eloSo`(>dby_z+v7^FtjzJUp`A1zS;hRnOHvUTvF%iaQjDZ^z}+Q5neIdR(;6HnM~`-(dy6|S+K2x0~sA%I%Zk3 zu)Y2GS5+H-j$zclPhQsy)Y@K&S{}6qMpn6O%%V7SC4%>p6Blk-!-0>!9Q zugLBA)_<9PzoclZI1-AiaU2#g5#EFkYyD;Fk2^l%U2a<%N;^0R_IymILprrMYD~^* z@Eq{cVZI_Dd+S3U^`~_75Ixh@Dy3~0n;?_9EDsW-=gf-2h`}Ph80_Z-MKv5i*Gl?O zcN|`RIsXKByuK!AvrR2`>+LP#P`ZrGsmP`C7*-E$bs5)5($bJoi^7o+-@CaXtI_1^ z;k@aYuyefmNDv}&7iX9JnFP?=Zd?LZi;`yXR489RF3=;^<00Zm|m0Hsv<(HfTD0+Qj)NO z*LYOWWS*dUoDQ&d*AeVj=GYxb3!`}pE)v##FN+(IKEr~sa|mKC7pYa1IsQ&Us5 z>1Q$MZ0JCjJqk+NQAEa9FJFNzq-|;nWdwZ12HTIpr;HEqG~k(UkrBIQXK_PeH%_UB zNK>Zt22W^aa|l{5;d-ux z7I#>tY)7Ce8;nyBG?NJ7>?v8LEaEdo2DsM23~8WGyG{G$i_60m(kioLa9*(#M3{Y# z%S?mw*#%$R-eRVXJoKO{mwc1K(m7!%Ati$#{%pMgPO2||IfuA0{5aPyoB%J>x%5~7 zpRrfWe7-IE4d1(;yuwYWGztn@*=%l}3TMeLXJ+T+baa+C6fL^TBh*JmGD!FNSwCjB z4d#K37<98#wBdKwx|uMi-Q17|eP~RN&(5+{%F>=w@gzK}a8o!qI3SJ_kdtEwG>JB~ zwCrpf{gKy*##Tq?P~{Z!o?#^TC&aB}Z*svyQ$(11D$&bg2o$ooqa6Pr!?4jzbt49 zoX(v6FUz8;O^Hu@QS^z4Sh zf1HJ}K?%_0z)^lQH^79)l0!&CZ2t%Kvv-T=`CHy76z!=R{aFsRlL5Al>l;5oh;k3D zUvpwJcT)KFhn(8N7vGZ#VHe8<gsBg|rQeJ`G>Q)Mg^m9Q7$+B6# z?)-@vBLLcVS@KBk9_P<3Y1y)iZEj{X?9HdiAj58En1aYxWE6c$ul>Lf`@B3{`q2r( zwqD0CV62pNzuXJ-^YdG}jd7qn2%QxX7AAM9aRRq`mJ+;R!(Fwsw9FJ)?C-DP$Htl7 zqQH0e&-ZVf{tWP!*HDm<(~}D}qxtgX?b{I^?c(OruCd9QSDQwgj?njmBYFZLdEsrD zep6jCtKv~rSp9VSrFSH6Js<0_I;?3}Lbr$}8RBOzT`V(cn9AS6#lA>bS-(NIwg4aImQ zyvA)RoA-xQVZzkdKPXpnHptgW*A=VxmM!OGkD5~SmKgty(Yf~i8o9(Pv z{jxDJx%7-hFkMcsu85b@kdxV<*8z!g`>l#kc3#$t@UXl$_VMG?vyWnonEp}QqJMg08#_PFEEqor-Sc6KKWluAu-p@)LGA95tzF#`8fZgs7{jy~mvm&B@sy=W^35tE~+~?~(Zw-3zv> zOpEE#&Ym6x6^&nlS;*p4)YQF0EpNYT()J84BJ=w2A|M7!&P|u>@r;EB4h!zmcx>h~ zH2Nfu>2;{)-rP=S)mFaHfJ@|S%<+rIW#tNiE^!-`({feB1Z8n%317c&;t2k;+M*as z@sB5aIdHAcii7lEuw$|(&4X~S^U~+Df>PFq#P|f49}C-SH-hHFYp_9m%R+v8HbylA z24pMFz4|+-Bi=2Hrw&}}HzJ>8HfO}$J6dIU);yYizr`bq6+*znD6!a;3?H|#YjG@1 z$+ZX=^NQgWVN0B%h8&2~=zMiR;u4|G61imndn4H0MBILuLFo~|La$4A_Axnj7tf6X z^A(ujrIuCo>qQOPE$mRjP*l}4Mypi9Ez}+#JGkv9-i8qb3(4gpGftS-ly-=?xpBOL zLm{T1_&pJ$IAP8b8cMZz=cfW%r>z0B`8}p@TuNEFz14qG@v5)NkRjH_@dNlGL>6g3vj1 z&_9jH_@a;+51>DJAPxp%5W1DHf%#E5{jn>-1-3{W!_t$4utwakxq2xhcT5u`d{*yV z%fgSIK>ofTt0b4*xzv?(O%VU@TIvoi8WM~-i5U>yBPFLX`OPTO7lp@}+@?srkaB2r z>IUQhY1dc8u+oQ}L&HcS@XtBc(U&?UdWz`Y@|qf}NiN6B%S%yGG_CTLm7Q-^B`0SD zD$0sz9tZVHmDY=m0HG0+lFHT|9WY=CynJXY+E7xOncaO_nVXx*y8arjF&7^Q@Z!Z| z3KdM_eyO8xW3tntX4=T!?`g|vMCulJ7ySFTL!n}xkiAp)wbK55VUw#(21jFx2Xff8 zrO~viyHo2VEaF4gj~&MT3?L6eu^C`4_)^$y(I%}S!Q<#3C_!?9?9fdz@?#LHJ*tuJ z(-QT)l)y0^00STli1IjYb{Cgd#OoQ@h&c%f1VhE=IQwibt3R?#9it*Eoc>tW`Bh$P z%8&|DKHEEsX)pzWMq2 z+BVvOf?4}1`X6IIMXLd&$Zz$Mth_v|*uHWWnOx;ea` zA9w=N&HDQKBGg53>C1VkVuJuw-#CPqVb##&y<)7vrNRC6Ye8x0s_Vf?UL|wJsHtoL z9&iZ*c@AuiI({=*n$(2HQ$J45BFH1@+Y;E61xZQM8T zh7)-NWw&xA+v9w!#`xGMP*Fbu_3w6Y7|ugIK}H3il)B}|MUk5d7pJ7QTSP#^fd~f5 zZsX%>UGX#X>&FrW@^A)<$iIKw-p_FM$tto%#j6iZg-89NV0?TW1G;+(MWjUBfaqkF zLSbYww@FJkXh<7Z;D5shyVSYAK-O~C? z91wxSw?3Foe(wb@@+D@5qMG5JgDPWsdOL2Lyaiv2m(5PhPOK?zGu+U~A}~0g=879b zl3%j)?Q4*czs?M;cI(d|uI{dGz4%le4rf;J=KfSJ>MiX5JosBz_Bj~5UZtHn8XtxK z?<3wGioEud0{fiE@ZCN9?&0W?klv|ZG!CbuDBxwm(@o!;5YR5bAJyxYgFK^nhBL0U zdAw}!a+PSc{bxQ2E zhPLju8;hMJwZFM3#;?z<)#PL~*&e96 z;A-aZmVox{v_-UTxxqBb^M{k^pia_MRaWSBO;v#3vG6@Q+{~ZeT0dSsKf2wDKS_IM z6B>-|pjx}3rrdJFsjZ!6+&d1MA0IZ=GG3ipMGt^PsFXoZ%PC*7O=~NeW6zU4Y(7qTzM;(8Q5&SAS$& z0?A`#WsQlCJ=?@Cay37RE?M#s5O{?@WkX6wN5qUrn8KMj%=7efcicEKHP*^8aesdw z=qE?e63JLB&3zI5 zeF}r&LPe0TcaBa5959htKCp|irI&h$z#ZwJ1WFpbK1 zUqp;zzs8|)u1S@_&J2TZ)H()x!v1pFQ+NC>#UaJ-t7sFiN^ys)6Y-+gg>*(B2r7fj>@wN}No(+Ib*&WQ-@!W0j3PV3LGyf#= zzSq8Tn+g#T!l(6<@vt@k?-G#r*?=Hh?22&I6Jm~R(W+)r zdj_V|zf}V`#~TQsSjtK?U;Is`Uf#+l@|of?371R{h_Ay2RIkEGzT0X;d`L@Pw|4MB zvE`BvR0pPRLjJxZZ_qs*9X%R{SMP(@-K7~LPP5uq#iGgQItw(PSG#JpY4BNl?R+rP zb^qqf(JZ9kj7AHQ{^-%&YqLl)?ezPR0|*PcuFG|bY#BEe=HiQ{|1sv!1bdyg?de@s zCqpYEiJ3X+U$P48!^*wWLcF>8>#R>tP47uGWhYqm&B%;)2I&0`UuA3BT~?C;?88oj zP99L^!wdwttQyI+d0A=oAb&j?hWj@8ha7B>cJucCt`Fy8)Zfh7uhnkat~&Pn0msat z%v>NpMop+A?uJzO=wqYPI<^xDh^EL+p%cBaA+FU{geTSA&$QpxUU&D!a*g@jqt{WU zi6ari@e>ntuyw25ac)K(ZjB`V~Vvl z^SxZq7OUeU6$c~O!;~e!;g$pLl)J|CBZJ4~=~C0vMxZ}pgm|to7caQnX`@~{xQoB~ zg2b0JZ)%UuE_g42d~Lm~KmNpr$a|pvj{JB7J(wfi`HUkXx$${U)asg;^Z*(b#`Z{a zyo#2^1r?~2qv6rJ*H>1)wLU&vJ3-6b_d`p-)ps%2Tg9hd&H^L%_;8ZHvAMNdI(_jY zgTW~~*TUR#(KN8)2FC%4ZEZpnJ^T*ARbN`#*G8aq`@+2u@I(`Af#i zw`Nsaw6XfTI_3Qi*K4b!-2$SBmm8c%)aD`%~Cr)ZmYr-IOIKco^ur&)?>$aF&)40i!&pvZi`>DhBaN zh^`7|vUz%tx+I$GFHIo$qSlB5mUBvK>NCyu@nf6&Zmv8fuvZwYcS9rlH?8FJc*oKf z89eLk&~bNhF*GT4&+=$;(uZ|NR`_7elDRJZWxlo*mMK?xOcNp?AaD?R_}WZcjaXp5 z038tw_UYyUQxEniDbQCgMWX`6a)aX;04sqq?rt#bz*>$wY6BpPs)j~|=}9yI_I#7nAFwP9_M16M{TV_E@bfgsd3_nK z!WgdniSH8^u%Jj=>1D4oKuKf-K}f856J|YC?>hTZHbG0ssN&`K^ZHNxcRQ#zfU&>_ z#lq@=SyA1>RBz#6pQc>KLzMFW11Ih?j=Sg!%yn|As{c-nhSv|)26EuYE4{YqM04fC zdeU^kwYeAz|G@InhR6me6P6B)%ut7hqz4k&emy?it!*6z`1_NAJ4jQc=|w%N3|yQ5 zF4rGzG)!v*9E=xJN-8S*UM==FK}*|7dw40OcX~7URpIg4Za+Y?9fxO@HVj_?7(~Rx zdP_KYa^=O$#nq$tIbpBQ&SY*&$ju72;Zdh;IBW_A#BY3|CCT`>1aZN0B0xuLE z1pN@&GbOHCD4;WhzNp=Yv~m77_(BpP5EsdYJfGJFN2zf~h<#59l?d%bxl+D-PT9`# z=z)q^8qN=`N};z3!D_0HOW!CPj+UnfgR{&hG_PzPthAaf6RW=<0Qhn3(UX=uPH&sk zV50eL&h+8<0yA4;osfCvV!kk))x6|srNy>2s38s>q8B?XgBEm8-X(Cq);sc=H|{}(D; z=VDx@RaQfz2WTj+Z*TTWm}-ZT*VU^#XBzbmSRMp$?Vt;e3Ui<-^Se2TWNz|Os)AR4 zM+fnsx`*rI$1k*Dz;4kPhNa?kRCZ`hG}iyB!2h*Uc2c4I8%+N>Pwtl=BGfEiAuQ8l zL#!6>4}eh*)DM;q6oH)&H;u}j_yu^?IuU?Bw{qf*e_GPt*)zzum_^#1Wc5>F!=(7P zY47*#o$gNg$JelLDvx}YS(Ed@c(=5)fYn!5D5(UB_W9X;9*JD}=H_HV>CD<%EDMWd z*XlieNpGfKi%CYG5`8 zok-`u+$>RC$rA$p*1f;IqoWrR;}T7y-m|a`m%%ri;o&)tkt^Q|0>;g-i4MKNt)-@> zCa4E~Y2z8}RdJO07hL%ZQgux!KZ*HQTQkk2LFS@mY zw@w+ZAP&^a2EMjUrqQ|p9UIPdFq+fWAVM*meF|VQQ^WWc4PC!T#`u?V1xJ4&bQmvi znn?{X;~@JD3}YodE0f-kSO{)ac~RT?R91?)qBsjcVb5`izHL9I0R(y%1s_NfVh|1u zc5ursj1SsN-3fTRzwETn_-h5?t-_hU8U&yfqqCmMBG(U^0xCI>8o;9!{1kZ_O~);} z%DcJ_6SkU!P>POI2H|Fzg-}^o~dT2D);x&kpAlex7aw2}t z&l}c~*GKb%MJ4_8=>wY!Tl7m=ur1ll^^2@%ap4gVnE&k?U->IiZKe}7Kj(FT zC6ofYy6f3t?9YIJcU2~3y++nylunP~`jSB4ki>uHv`%HvlAYAwvv4(3;u#L#ewa15 zgnNJTPz5qL-<7nZ87_^FbV?EK#or$+kUe0JZn^#WP~k{o#YMxZ^_P<@pkfdAdY{nK z1Ka5q-dJ$q0;wzxH;rJOwsxUz|D6fWWBgD^61FtJ!8ZKj8(iG~w0TcZA#?Ha_5k!_ zccLTp9U{D0g@4Ly(L-?g@J|>>4-Kdp&J*9km~zEqXwqBSS@d1XL?zZUNByNxnV^CT zvjQr2l8=9stj_A|$$-dI&{UP8KaSSs%=J_wen-%hjWcu}%kkM;fhY?u4=SQ19tdlge#?F5bC z&N=f>naTELYKj@w9*8)2k|nt)he-_Q^hA3q$VqD{iluuE86*Z!<|c;dNFz$|MSDE&HL zQCT~Nf0^Cbm;|cUO4nWmK%!Pw(VZ?-(M3c%h}A(VE*f$zKvSP#m*mpW_}%5vbZ|P0 z!zw=(>R0x`KL(?w%o9^nrxRKJuI_GGic|n){s**;8zXHUW3s0Du1=63kWsb zGr@TB9-4cS;)lytVBR=A;@V2)SSc!NUAezGC-*V5`D|r(z<<|>UbdU6Y78Jv!a50sIFkwr(21bxJ^ zcG1(zeC>qrk{EQ`W$SM{+O);$$aUM>s7%RrSej!2#ioC-pYju>QJ zWbS;%-*jPGT6kvW=3d}!*mGdh8p>(s@v7wli9n`aCddr4N55-ocP0L;VrFHe%KY&7+S9NhU`Ok3 z!3|5T!UXWTe1g$}YnacSTZpyhzuP%s@zX+%7P92(_36hIoYp`1kZwGb+tg^y?nae0 zq%qS8yh3pdQ0#B5)ZtF_);P#4pUKj9nEX1q>h%cWnbH_Fh~f3id$Yb_Eg)u%d^eR7 z=~X+C!yPb)GL9dOUcGMb&3%<2c5|MLA$aw^#qqd8AbWHO)7w;loI~u8IW_@cZo0~c zJk^p?PN7R1CsKn2@VX{8B3@oz(F!HF9sK5e0)A1&h_gR`!m=fUWIL{WL>AIJ<+Q0# z)DRObp`}HT(Kb9Bt_^Xw2N|2WdG*&Uf)5`6D1<5`)J2*ml~qyk9!>-(3v9To@ItmKe$?)rY2GebGeAcut{tw27Fob)(a%uUgEQMONy*5y;)m!}&CPVSwl?(o0rtpo zXW?CBEU@e0Lrf`x$bQo=1XF6L{2p*k-srE8ecL`V@z*dlijt-*rS|`Dnl?@w{qym{ z^hKnA7yHSXP?pGbq~mdADQ5prZ;(j~Zuc+|>0S+|Gw#OV#^$Ie_cn2j5;Dw$BFjzy zS!0Tv*p-~OBjHLeHc}zpBuy=Ap^~{4PZ!4|yJNX=r$K7XtKTZrQambBcJ_68yL)=Y zcN!l!dZ+uQpMDS@xltqlj{IB6*ocijF^8DZhZ|`wt?4kCXdq9JE@L2=y;{yQacICb z^6Kwdic)AgHwpSDQABKBPsb-$`srCHbB%6#g+dB=8zg+57{F_fRjJ5)`0(L;DnCU+ z&t`xJEW(1K5(9ASvipF^wR3-OPdvLYa=k|;A}x*2R+JtLKY9FT<}WdGN$Noe!RnG0 zdU|}{zF(f`Z#vP$m6j3!FHJr$QGjNo{J=?ca>0}$xfJ77!$z$kvB1R{;Uh2-@6z&e z*%ljO^g@eE*T|r4E`FW8`J*4$tJjG#)5%Z%8f69C=v$eH0l$h_#WdmJlroF$qu_hM zXYX2#d*u-Aehzm0v2yXnQ2x`eTU{dyF#~~pb=U{$)!OuJ+hl+6BM;`b7 zrp0}MzDVjum_@pULka(&HN-7Lpc zZ24t}OODG0x*5`^(I@d7m%mn9^aJu4lX6?Wps1*0t~*IO)prNj4}j!JK}W|%UK`Md z=V!R4%y@g|`^U$&`Pr?!{^X9Tb`B05^Yg*$6`zx&wpVnAzyEjzY$-tPiH0^Lt5kpt z$PakHmV@lup5g8jFu?4NWWEC84WY>y(N*Y=^oqMiEs)u~kfolsAu9s1*nP+T!r1#; zDNc}PfwcuI3Exe(AXIj73@Dp8Wh|s*OyQ|SG(2lp%jELU@rgF)ZKIQTZfak_ng`Ql zsGCiCHy8ggo<3k0WgNn#sx2#VkMa;#jI!hmkZ)xhwS%}XmrujCX_kO{H~B1bD}|5+ zbSZoG<9m7%QK(vao!P0g4@L;)u>s&E*Urwy!zu;eFCWD4J#cFcTClZT{s|7Km@2Dr}`_qhy@TKs}(MHXPax;|N14lXv1OPo@qv)SOK=EWr&^l z|7b7J4)Kt%u(5USWZ{8I0twfh4(vXLI#<7^aabl%?nxOFk#u;88l4ZtL2Fka(-0+{%gCv$i~_1T7*O*7vHJ1ZRd#v# zzZ|c{!XuK0glox|rAifVyzIL=N&?mprKB<+8gpv=a;FH~90^X>M7^qx>!U#h*!@a$ z2_R7;G5<+akzDl~???g!$pfa_R0?mP)@fhj1_GOg?M!LQCl^K~G6V>-6og6DK_Ftf zZ9c5^fg6@|?mInQi7yxs1M312M4YB_L?G1c4WY}Me@E1ZXx=L`4cn)(9}fGcOio@c zCxdUaF>dIb{Q+xRV!KbR(U<}-OAe0FW{|P|(uvbark-+#>QC-DMCfT@hKtm8G1v*0 zDB1r}mS$Hjuw_8g{XAcXhNjL#$pV6z$(ymZS9ZR~)?j}1xTufYgz^rMh=i`pRH$M& z%@GTfDDQK6#>B@n*{;C&`uc*iOoU35$7lYGL3mx2w6th|q@NtRubkV1Sz)gB1;*O@ z@|oEyN)O#MH?cIxe^Fd8!IMVCCyTFj;Y9IR9CRI5ty=&J63=nq} zu_$*|3`|aIsdAInK3Js=i_xwF4Tb*7g!IAvfNc!SGmSN5ukpqYOc)J~d1Z)ziH%J- z(A~d51myq36dv!goske3SmpaPpy@cZc}>#91oIx|Ah&X~=`vBXxVDmiYU(3uX-Oze z(DGS*u2Cy@0aC;@a+Jy}iQh@fxY)!6UufCkOckF-Y(5m-bx=!nyE&}`SQDd{U zqq7bnuyBomh<-$H$k9Xx%`Xv_2_ z+%E}c?vF4b7r+Ywzca^orDp!Ijc&moJi~eb0W3w<%Q!_9=IZOxoRVxL6B&GY1LUU> zdNQKx2dB3d!C2$ZtX*BJGcqjbrbR(Tn+I5EVEN;8Lf|h^j!yS!$dy6{nSFZ)7w|yQ zf`6{=?t2$ov7)4nLzGzZl#GdFVPRo^CaKKL&E*P^G|m*wG0kJu>on)@*n)r!=b0pr zP+wW!2ZE_riMZ2*O`DsW?=cBLqb8W#Tr$Uqzj|`p2sy zW*y{gip$39QR`#=*i!&=TItTX77RqlzQsS~{d!tqnQiSYP_noE(OcJ}O0xs~b1}+v zKSOe;avPPtf*lO5x}z?E(NLn>M#z-e_GwHM?Bw!bEf22OZqkDYzrgi1CZzt;tu#{S z6kqr5Vm#969vfu@*y3*hFAMOJqP*r2zw>B+MT{xEZOg8VU%tS_!H#Z#qqzKqC{VE- zh&UV`f3gw*lu@q3>*THD$o@xKz_Bhg%+(pme#&&%rUz!R(9lptMMYcMn+2%Bbz)v| zF;anl3;^`9L(kk4NwA;f3`O+E7Wn!V86sEsTta(6zL* zlr5@h(^xZLeuwqHv;eb$kSiiY1cb<5!`HB|Xj2?9fNk6!AET_-zh4=!%A_v}s{4Fs zeQ|lhMW<(E()+!aFC+I>SKpGH%1%MO@d2&RmcebnOy6%gEBZF zp==!HFoG75r%{AX%K}y>PLvGx2F9~x=l5v%{0lt%iJUGhAgCeoP|3$FZnZ5;c-D3A ztm?wbAm%Zb1ppa<(J77}8aybWe}E|`|HbM&)HcOj8fPAgl3;lBLjxvr_KsOZHNeZ7v1Gm{>*7B%n9$?35FmG=8{ z1U4P|p^TjH1-$4j|Jh?i9G{L3k*AlJwVU-?)98f*dOSvaHIV9v0;yZQ>zw_tKpSc2 zM$ota`06>){2wxV?)U$&+n^xSKja~J7JY!95%2#~ zM~vQ<@!y^8T9W^5QQ&bGhWAL%X)E{&UH^aa#H!|9Joxhuhp6Aye)Ru|*<+;^gtOuD z&C)8MD%ox?SYw?J|DP|G0nKa6P^8jpACH7d;gu6CschSk%nrCY{aZEKI1H3-o|m1g z$$3}dw$D#7Os9EJc;olXDlRoet1%IhZEA{pVCSoqm3d;y*vMevP-4gz^StZ6lJE+tmBy!k=X_he7pRxs?B_<5WKHrIfq0k zjL+o$<5^dJqY>!Na==?14}J%r5q|14Fw&8dc1K6x3`frR6>B{qgAHp$;;c?ZXEe9r zv9)a4bJ8$L!DMX)ylYvpE4&G|iB?iW`0(mPB71b0(sOZpM9yY~t60R`{MF2*h!GN{ z)&)eXoX&^e;`x?n@?^mF4s>f8HFVVYhK~6(7ktsWWPZA8 z6G`EW{o5Ic?g~wR(p^fqN%c|6X{AR>Nh6g<8U?gz(P zuJnG!O4O$K4;4VfeeGAuO`M(YsbTg>Z2; z&$=)Jpz5Ff{4`3F{1BhW&FBSAw$KJSlyq16mjAZ)SPANX9`M#_FY7&qlYMHPE+B6x zMpO%Fv>{byq-M2QYR`XasU<>0L`?J0<3Rr`ogtSCx_w_ry6b-8y)XBFHe27;8xN%n z0U@ntll3c@0?GoUW)m1DV`;Xxu<4$A@PvsB$na=>r++5RL1FT2nJKup2!YndnwlSBJ)oBy8>NvkD9Qb{t5bC$$T;Ur%RHk}x25 zkLPPsSeM=SYEu9QtjVrrKh6xCEx=_N5QdaY0Q=3%8yTb>AZ)2N zx>EoZl@u6cRn=N4kzLSX^JIFn|GE{G*LcAT-SsrE3-*NsFm<^LU+IBEQ3tmsL8sNR zu^}eo3D-E@JMJ^Wbcrj%>7|+!E+_*yhrPd_?jf|tc=#jtrw@QLjPA3ru@T8%kp|$v zm_}Mcf}o;f&Qc}lNvCUJ(cjF0c)fN;_spf#@HJpj-;gD0 zT-xuL!1j;@Z+6<&4_pjlURK$&eFsozIDh5}IMxY3hVts_!ou#tMW=oL!)3z!Up&7h zPU9xhChGxEcKd4zm?zimuL0pW#$!C3jw3uB5QcX}#ecwQwn{!~MwmoC=kOqh1BwH} z zV9<{E`?3|yB5UK{Q>Ord%zk$R% zig}lZcEF2(Qw7M4U668fjmWp1Hcv4j!=NN$=dj@71roxQtu67}8;g>#T(aDlU6*_c=fa=3P#o4i;TpF(#+~oshBARpDcH zk=np+IKz;VZ}b@2*p#s>4=siLe0}Iz#XkcpD@z+ELAN zaCCNcgVw#;hHl_X@|#kR{^gupj@KG2Idu?&!w!DgZ)CI)UscIQJ&MEg*@YSd;Z5Pd z2j`M{{=ciDQR~$<@9c^UdUl4@nNId=o0-e8%zD)h)b0F-0|-{9>B^!`zLmImP*PH= zs;Qgh#fp>BDpBpnC=^Yma6Ei2QZ6-K`Th~C@bP*v1HZa5is!=-dOoilYN;>{_s)ua zE})iHSLe+9n91xuBx2#6EzF|{U$34uvj**{0*7tdBZv~d0QUOO{CqgTN#(d-m>!l*M}Bk&BeAf@S=J)(Hm$}Fx>+o-@&4y z{X|1T68>m&FgE2$Vl$)UottwkcS%Jx)8@vOE-gU_F+<}9eA^`TG!DBd42#7SF)3*b zXl!R-VF9LEhea8%WOd#+Yy+8cE;!B>xV-HEg@RV6f%A0nh(z#6K3%p5+Cauo+DMUe z%^#nH1ESIJa>1n@%{CSh`;>@Ap+;Saha$Vbz|PL@-tPXj&9G}QQ~vNf0s;lY=fm8m z9x8wAGA$dg2S_EIgm#mYa}Luf7FEHcAh5FM`KI~dpnV7uzt~_a0fX}H$=IFD&&_Qv zUMBo*xovb2Z92*Mk5Yps2bRz}3L)qI<{yXI%M&Bckm6tu0uSzej(6B`g<`PDE*E2s zotFES(r2aG-yR3aM)@c$AFk2pTGf7`knBf&Nba4?ma8U^m8gdJ>;=91nzpEu^o@gNe5|pW;`?A&}y*`a-oUhKLPj}Y}FolYC zZ9e1fCHA1-q2r$Hxz=2`0#z*M1^? z49R7wFXsl9mIHy~J+3C*mV*S{!-c@I&1iP|n$y`;%Y7NFu-_&L(!(njBoH|~%473{ zr}DNOlKf+fmKmNUtuvI&HOkmDN0o z*lYji_eqgvoT(I1!VrHmTlnGA%V=92cFwlS7*|(ikxR<(WHa-#e0(*(?f7Yr(mC_dcaLi((3DHs{%1hw)qoIwe`y<91Ho&BQd!}vZQ z4<0vs07Bf^?3Y}SNhJ@F9&yu_Nn}3C>-@52`=o+}`RV0~r+#|}zJ~RVLE7@?60bpc z>8)#R`L?akP66N7rw&!7QFgRH+mkpxg$FU$hK(NXdpiR*JHqdPxQfyftFB5Dd6ThfHU|fXM+>OB9iVD^zFTqg1KB;So_{X;faz>+ z!bdJkY~@Rb@6V+d{F+Tdu!YSICN?&M(WN6|9a;-JlPepC_fptmbClgsibuLUx8eTgjTyD_-+qNW zE9pK^6&Kf^=in`SRBG4b+1FEqE`aTb*z!0Mfd(7?x$MCs#9!7NN^&$DYU%3>KGI9G zG}P7t?DEgxdUuuE&R^VbE_48!vs<;>W7rU^Im2LvQWBrbtx{0CTUAwcyjT+r7^C^{ zdf6mg^{9jhR1Cka(-Ya9dyOz9rC7VYSzp;$M!orXRJi^5a-5c(-Se@WrlKk>m*400 zwP{thY@Af_&bBbKoQZ93Xs&#fiy{!|t+l=t4V-1-;GkrB8)2nXqVwn1kw~n)qoZE$ zxcEH75nS?LzU`Z=^%%SWO}ce9j3^>&Zu$tmHlGiINaP1|CO&63;E zhk&1|N%e#c#e>KAGv9UU$+g?!+3nv6H|T~{Qe7wOUGKITNe60)LBjjeVA`D)yzUtL zd)jaHFEG5TFU?}EAQvBucSJSRsyhGFVU@w|R*r+hd#-MdsD~J@NM+CoO(spQ5dKK* z+UQID4e$_c9ev>xWUCd#wpsiP&(D|G(!=KiKS-w`*veIwO)3h-Qo2j%nX)l-BLg6d z-B7nOnDT~3(Z+DQk{Z;P0i5l@8YNy7rL6vWVO@Xx75hZKGTGa=5j7AQSZC-zNWUEp z{s0B;qtnwj6x@CTNj2sBu(P!dez)Q*07kKrP7(r31wc~<7U~_ZTqZaGLp7`C-0Nq6 z)j=@L!~L6s!#lukeLw4WO;DsmleM;4(x&wmD-`Flz2JKoHffq4?^NweXwg+_ng<;J zJ7yfipPrt;8|w>`$p&h0jKbk#+=zp&G}f|vv59el;J0k?a48m}31F-s3J)!p6Q;(04N=~k~) z!TX-`M|j-2CrT(DTf^`17_``Hi&pRm3CF4zg5Ao^>vC{c#@1<_&Z?9T_a;8>XwdDuJ`B1; zFflRdY!q(UZC!H%H79cw$nC8~KxSBd(_A(HZL{jXfg!kuY6_;14!xkdvy~>&fa3IW zSN^u-6W_(WEpUV(8_NjLv;iB?`Di>{>ZQb-JH$l4uaUy)5u|k0Lde18as0GM+uY(a z_(R?GC6klw=2u{qDvPU#?Zk5WNiOyV)w|gqef>xAgRP3q6f4H_VRv=1n&a^nCSax?7P3&xyHzc-)eD zWB_8$igSOq@j_J}N5lRM<)NlR#I?GHrY4(#%e8}1f9;6d*}8dSm_fd^jwYz-V(>{P7n$*0lk~q#&M+z zIM}<1tfn%sspmU?Jd>Nw?&)w_T9WLxhkfzKpgL>0FL|dfmoa?+X6VQzZm>V)grUvv zIi%Q86Ks*-T)DV=$s02T%BADiMOR?u9GZ-OHq(lt2rVlu{knF@1b#Om*tvPHF3&ncvP!5D?!)g$*p?xAg8>y(L{JrDj(Npzyoj``Ou#nR8 zyQ2=w)HzQpi@QsPXk+u}qg&NBn(0J-UX>ddQt;~sann3Sgq)-ILFk4u2>TAn+}#tkhph=Rm4;*jqkPRtNIebqslND?;q(UL#4$&yQZ`w47X}$L$rue}h>7h~qpG@EDC0BlNNLue!3i*Rk|wQ~Mi{|K2FK99=-zTaoMZGNG2Y7IX-rBlbEnNEquU@E+5y@f`@F9g*&UN*S1 z)Pb`YpZ9rvA|{lN=(iCik_>uU+xTP{@CO6x4-c8k0%O=FTu+;*ARzzS%xj%Regj zW@gL>mzcPXQJ|!&#!;62+Uh~NJS?oXzJ8=pQZ_S74=KRKyl@Sa-KXn~nm6M5W)GSm zx3r8*--Ze4oG#qo+%1e!$|k>KXUF+25SG>g2<~ZVC3kmsKS@bFdILnNi06(kEt z01WB}!s>MOvMll}oj(uoGCS<`d)6FX+2TIM#O&&VWEcuy;IV#3-??RLUW{W!DRlA zYA(N(h=DmWn%FMMRmj5v(IqDr&&clqG^tch#e(@YYU~7FgG>t5@PGjQRliU*172P} zuzul(w|7S!&lU(ZgLstt+9qg~RWA|m-jQYHYSdw6iH0xT+}(unH{AxZeABQ2N}g#5 ztH)eiT*5!ZhWE3pPj{foJLR9PwTy3hQ+OPIX{RBJ#1^PlJV~m)25@L`83{w68J9Rz zSysk`fq~J$W&TtEafl7|sPHo!NIXLPc?=>+6PNF276nxKbgz-)O74Ro8H9Dv<)xF$ z>8_u$B4arTzI@)$g&#ju0Kiq-JR1cdGAU1%+C(I$9lwURTH`v-pDh&MTYNn|y^lgd z4>W%_4T8W0kHhdC$k~kNsc7W^CmX=2lc!!&twaP)w8YeG zF_cvxo$}Fx;nTEU+nVy|N28qnjs3@`;Oa2C4uw_z( zKU#^(QJpdnI9wD@lEd5Eg{67JXHRyt{?948kU^viu!?7Z8ZQWBLh#sz|Ex71RT9?Q7(qZYIuMPw3zrvM#rel-X?5N|{KXDMpXWoH)W6$QYAot5BWZ>rzGVB z1>*Rb*G{VpWJ9~MA=zf+nBo5H?z2)wOuUa@ zD{NK}hsVg#yk6VlqEc#Yvkwiel10%YBfFhtB&&OzjN8$nXIvYIdesxv*w{Gf#^cI^ zg;j}Nuq)Oy9Q?ngEK~YSot>R`=bNHZoA@qu_QLHBW}L6dRvzV z+!k@?y#K0+@7mZS1mN`c=TRRV->F#4mU}Ie`ITS#kb@V2vIpBL^WDprF9rKz>7QGg z0fIdEQJ1)MU3?b;$ z*VgC{r`@dGsnYBKrxvf%Ard8}7&rh%HAX=F<~_ewJ!L~}JT!sNN83XUvej7czQZn^ z$ga~rEeEo7KLooJptkn&Kgv$`=8~AQx^Uw^Y*wTNb1RdsFK-F%urnSGhRpo-dLIT2{7uy#-EG> z5w&1@y&xo%>{AjE7$iFseFKtGB4Ivgn8CnE6n75$Zv8hFV6p)u)UikmFH6D`t4zP@ z>;Fn9mahpDbX?-2;o~D_VL8DnDhLOuj+IsH%IbxGlevQe+Al&=$M%se#=HG#k(DK1 zsl#ub00ndU1mR(B3Xj9;)>dnJ>%%;)MuK{hdY&U|kSPRYc2S^%gMg?nDz(6%XK)Zl znL}Au58d<6KW79Jm4g#N+rSzGDUDOjZq(nk_->9xn*|f^fCg-@?+LDR7WBiGL@?o$)zbQUwqobNfOJ$h-Y8(Qe+JUaiFeO->;aQLC75tUje_Crt62CItSxo@ z-~9|ywB$gwQkD^OfEPrlL@7k}z1xH*0zusSA{KG{#Ts=IQ907AW1^y>tfp&iRkX8u zRY5^BAS?l>mK4%)U_|4SV4e_s8z~UPKxC4HN*0?!NVt?Jbw(&R{_uJfBUVF^Zv&f2 zhd>hOnM%Q>OArqXEprIgn9h6+Cx(m3%>nxXN)oX}YKOP8q+B3hqpPbcE-uqL0H`z? zn_$zc&vUAB+If@!6I{{oRs(OFtTo^P`E|M~NCBCADz0kPIH zs3bk`BOY5BoJgq1S)J>PyVz(Buk)?jsECjM`3oPWN#s7H*xmd#3u5I6=P)0=YE@H% z@ooYSiaKlPN&}og*tVe46KFkDx-^l4<546};rRTtrnIbTBp*`y?;%9ZjIWU1(%zdm zZ(g+_$9P?CW7}1I-G&@vh!Gag#TjW=p+AI9E3qUCjVk!EdSoh=Ovv$;k2`URSpDgZ zh1C(Q0J(aNIT;H}6o)^C&io}4pr^CBnY4hD)``CXugZ~i9!^$LyPfu6719*}9?$h1 z#R4`12XknkEx8eJLjr)m0$iW|vqIfyXbpjI+@=yOFH!x^ga0p$CVq5Df#SB_l66=e}FIG z*Z3U(2z{FF5Mm8G&RNWd8Vi-yEP%N4IX-v1$ruES1ZQx5=D_A?h1_SGsU4_RIpw+Y zfO=Mu@|CvdLL11%GP&%2LR=g{kQDM(yzle`oFtB0l(N6%$z%RglIo5wqZDLjbRA*> z+?j~T6R zpKq5a6M_x$3j$#QLRd8Q3Zg|y*jQM-JS>K0?hIg`Bk}O?ED~|Y*DR(ADU9>+BQ`yE z=jx$Ldo(hD82WDfe#t42W;kDac;q5*OFqAXD-La{AsEK%; z0=hr7g~#y!S4J_F*E$F!sPRDi7RYJ8XMG=Clha^v_&V(Bps8z(Ix4>AE5^CQIY>*E zZp>9B9vS*k?Q878T@YgrVm;3TCtV0{IgUmm!_Q1VLTz8Co7FI=sH&D(?3`mwWesta z3|UV>Lik~})^pib{kBE}diT|ZOLx))Rn^^Fd&W4D8=lYeT!!6cPO(&O+Z;6Ceo4m> z-^jslX+La->uJE(+HBCJqD$_xX``c&HPFPP%8%4cAS{qc=I^{8oc#|TN@qLJIPoaT z;j*@!?Ae?1O84_C8xr6MNbyQ(JbjBmXSdWEI$EVvxmwR(L5qtJa(!}_EFj4;%31Io zmWFy%>FiWH)?i>en2f-#QX44Mv5Jh3Z=VrJTTr$I>ExA-jm~blwwY>+E>D`5a50dy z-(1Whv^5+Ub5`&H1t2$21-aM5Mk4a;CC2kTe;yRH4xI+5xUE*PzSz%MhW0>&=U#F? z&8?w9SZ|}Z&nS%%)=_Q2d3vL>m~?QFCiMgqlHm29n5pioKXP8TJEZhv_m^)aWFiIc zi$4L+r2ap4(O)}E{>=C|5d{fZEVEk<7OVOKLy<0TvO6OBlFoauD@K*)Z~aP0J)@wZ*o%)Z!>YEE_JZ|9Vav1*f4U z>{`-@-kAl*GHcc>tt1+Q5#fZ=8k_a19INI1aU?N9Muz#B!LTt8*&>p^qg1YB%zQ78 zw?c9wsj$_^Lp@&~VADN9m#jLDcSLOxnfgaIuV%N>-=5u<28DQ~xqH4Es<9h6LPd}(H?^vy2*WX)Tfl&UzL88r zdDw$Kg){5+;_}CigIy7+g_)w*Mu~RE!bQF7Es2CcMJXa{F#GoBYy)d9PAl$#?bgL7 z%UgU&8HbsL(|7ml??^j6>Uj<%m$P)^i3FKQ0;LalSk&g!09n@>p{Ul1`hA;AhOG@zU&N$>MRV9f`D=P;pr~>oen5U6ataw0!ka zH6QPvVFj8mW%-6$dV-K=#N(UAE{(eJKusyzL}D0*%v)|wU#Mq;c(A3Sv%LyXTh8rV zZQfLP7V-k4^UZ$sA8)BZkpbY_Yjfn%%#V&PV)YZ!H@ZJ zuF^kKJ>?`4H~!~AY&C@)-d5;w89VVpk@4_kjNhJy^>L|COvFI8kKC{J7%dxzaf$?R zOXS=fX4qGNFcsQG3N)^@sr@mZ|>9rdNBRwIj2|Miu{Ry>NHn z1G5qVuisG5!ZTkXFB%qqP1<6l&H0pFrLHgrY=Y0(IrM9S(`S!lEKxT)^Nhz6xSmug zC+lYsMTS;J^(7hNb94OsJGF52Xe#^=43F=U=<2hW#nNn~5Ft`hY|;c#!$bmGT5LbJ zY47+Y6y8tWYXIed#2uRFFO=6Zvhb{17Qubz?s$|9B@1Xwk_h9*mA55JbUb=T&L5@h zH!)(G6$xRBb;wDcC+g4kdHG%x5~m-9^qD4F!66s@dszfEPL6|ub)lq#1I~rKP+#MJ zZLAz+=J+_s!ioL-9Y)xGZ0%&_N3e=|%cuw(4~JD5#ZvmJ!x_hP^fl*qrdL&mylP7o zq?xB13gRZo4b_co6CKYU99uC2Vo2!hbj3Mc4RG7xsm85MOBIuZX6}7gOSFF=+x6lb zrvznZ8W!28KM;?s2NhJV zxM&n5&Zx&R1)!f6aDlp&4>S7q!4}r zIplO@D{a&9LW~xW8!WuuHe=9!Ms>0D_d~)nqg#N){EAH`jlV6~af1oO@a4=EDxlt$ zuXN@@aD@8TW-!^S4L#;EKdb6B)j$ zWj6bF8jNXK@fB!Z+ojCE_?~2>E3RB{T09PEB;ygJvk}gQ z2%afWT%Z}v2L0C0ThsLdm=adsi|71aH!t&&7u<0P#aOnh^1k3mN9Q?U?jB0p=o+7? z|47o$__Dz_;S>fv)$x^lHMhHpMjw^YZxHAHAh2_09Zz+6n^iJnix$=G!jc{fnp%Qer;uE2ei-6l z?wN+AV*Oi7ABe8voIgNKF$+CFRb%!|4q;6x$0*|Y;@(O5KJCsYojj^4Ruip^V!H{y zX-{|N@}~LVga(`=J+Bg+RPM&-Kpe3{^|sF_A_{?Yl3#^JXQnnoT+=GwQ|nqk zeC;8y(~bvyT1{FIB0=Bi1J-=R=6j#8;#etoPReA1tUnP2-}u%j^kc=lUb#vgzcRNDfCcx?qL$cJqmJn=I5AD& z6cCX+4wS1XZjty~M3(h1t*j0B$IaemHXH^Y_ukd0$LGYAD#dcmd2gHazP3wHK@Mrs z_Voyq;WSEmVVT%r3Nt>RAvc{Uk?rp|O2HExQ^sZ;BQ-}yqB>-@uBQQoIyB`@nOc?z<3 zPZiC3Jfe}(KYz}?Fiz$Rs~g|%Xik(HxW^?;~bfr zjQqB%ES)@bYbPYXX!hsEN!p3@dua)-sS9esDY4}2faDK1Zb^A|w!=uvPokmG$+bU<1&-C7m zi?Gpyi1}-l*ktoI7YF$KpF>+Ty>WM37fP;?s>w4fEA>8Bv-4^4L+hBQiP`EE?3L_Y zMNxJ737)7!yvjY88CBwJy*hqkPcNht#``MU6;smi_&)Sz$lV72GZGhg5N|M=ix3%z zZ}q;U;9_DZeulR2r`9+^6!yxMKd4YN-XvCOMFr%_r^|YMfQd1lqm4Br#!_L*JJf2v zo{+C2w#;J@0OU?H6&uUVCd5sRE~R=m3^_ybexuz1*`y3()Y^4(FEnn#DUG*YMGn2@ zWmimNmhIhIPpv&C>{}?ojl^+6^~dFrcc1-IzV91B5`9P3119ZPg;?OFH5Zo&W~s&b zyNMh+(bw35rc2~Dqa@!lhHUSDyM>z?R=p7)o1wd(eIrHgz8H4RP}*f1Pi1c%dZLih zxMt)aTuYxhcTnkt%aHdIwlC!PuRsJPN{KH8T+pur=+UA<%(73~N+_{B_+F-+TY_WJ z$Z=u)P`2hn#fQ$($>>tEz|%BnG(qTz>mfk&sqk$*QZ$_&JtSA&uCdP^+U@)W$*px^ z4bs~=_!*;xSl#?1t+-n`bnuqPC_$pu1cx7ADX~4@Ht6BLsr+w+Mwv=+j}o)!Km^AY zp6*v6axiVBa|{Bh2g!O)>t1|5STsk)H*J!A9UU688bpeZ=F1qq&47|lL-W_SlKg5R zjg5SKE)GWB>7e`{+W@QKL~mY%u(?}}neJP_)eg>R9r1<#kh-!px{atew-tBvDeN=< z9A-Rm<;0dnpH+&8V3r>@}SB2*tgw1 zpUAdO1(D3tT#>=d*skt1xl9nI=?WS6mvf;2|A)7}ckM~!s6t~IHbCrl7MGt-v!JQ( zNd}L!QVXL%4sXIaOh$@mN-zo}`$#5g-8h5dm!glK;}1*keu?3lUdJ=ZX=I@f_#kJ3 z)mpxm&8&B$*BHdnKbDu2*C-ntTvwTpxRyD{;N~_9(y!B1wj5TCd@-1C9BiUj8ec(Q zPcs8MNZ0hQ>OHBzMr+oAyfG|JofoLoG}99O^af{Q+c+n*Xz$w=+XKQhrFc$GpV6m! zggjrYG=`hs8X)*3>)m}fEzIsLVHDM&>Z@=y(nuN7_UTEle0f~#-?}7RTX`;u1%pao zEsl1>qFw=W%`QQXuRw*?B!4PywO#61mfpom3V_$h{$!}=pISyIe_#OkDF0+mR{d-A z+25n3DYP;qO1aGCb#@}mvLGwP3mx)7S(TC~{VHQB-$woXCdVaXu` zdU~o(B}IXOsGx8U95q&p$sc=j4zJXzS$U7*1&0@Oo;_aUn=hHjVQib8jqScumoc>C zTwD8WyBK%1ei{u?iinC3HPjE7gcrs?HFr~c@Kvsef#Q5{+~uOcKLZcuIiGZS*HJ~y z_qk(XObt5w&Of@+2yu}C5(;0GGOaxR3t_J8_>KB9Y@{#*ZliQ=h=)&$PSOTjv-B0p z_$MZzt@yPvlHIVf?xvg(f7`w;x%+?@dO3WJAsC zx5bj9V`9+pQ?TuFx$EF?Zy}0!&S5*6wutc4X+%_l*Jyu3i-GFJpR&SYKGD(g-xqDa zq%+!cbR>oN-ThIf2G@s}*;^~i2~BDq*lqX87s%nghQE}?Z1PF~0SNC{?|$xz9@3|J z=fE9#_o`Usl**s;=4{JQI?wmW=?1fs4101rlU zcaw6{CrB_cl^O`eJJvzt4lj^R@?c}_;|a|ZgIS*)Id>4>2VLN0@V~q-W3kiqd26}0 zOl%yP&bW-*mHGX&ZQI6Gay?@Nl05Pi$lx&tA0jPXyDrcYakZK(@YNU&HP0Bd4VxN)jFN5eod%jdzhT##CKRUs}|k``J9H zs%?7kWp*I`v-P&{9bkoYrq)Dg9En5f)PcUm(+@_m7A;shf{z}cC)c|8|Zxxl5 z1OCM>hdJ)$ge438WT@=Be4T`X_iG1YeZ2efai4JJPu6PY&_DFaoARj{+N(&bLHMO8 z*nf;aUdnNPa%=kNLFod2s)Q+aPCHn ztfl&$=k==+>l2g90gq@3L$&Z&ao#J!PWE$!eFVk6l?)jzL(pMW-=|AGtHA1=Y>Ge) z;C%B|FKNmC-7WW#-?RSAT}`_9h7Iy=pKN!`=&kxdk&cNBkq0H!0Y>}z-lXw#y{XyG z7}^8R^=F1AYMrOHP5i-xW+rd_-^A$A2_IoZW!3$it)jP5%}JNd{T*^0zi*2ppFVpl z{5X4z<#InvhAR45xgwV^ion-{UbFT^_ruDSf8p{5xx;2-Fk6Fjrw+yCxq%eh!p08kC|uGTLg;M2xH0_OL;1nTC(8v)Dtr)#&5(a;w{ z|KwR28$06E`Pq&zhZ3k7(go#9 z#JY|<#3<7Fpknc)NY}Z?V)6NndpcrkfUE04Cs3Gvb(b2PAnp{skz;T~UIoKo++vzO<#R1?3H{?%vaesYCJ-s8h@Kgn-= z9-hm1?Rn>!X!HTZFp`a$**TAISFXfTg&grrTiIR@w6{M3SYCinNMtwr14=;>pM&pj z5cJLw`!2YH&Q=P5dpHD94)xXc}8ws*+@h@P#H8GhpMsc4i1B2uzF)wGRV;99`)Ykuo z39@f6IsYe2SbvC>ta^agMZNP%sGnJNjleH6xklt~QjT<#<&vYzTu)=1LMq(a$R(tH zvRuI+Y|<_|P{OEAGwdZ2JC`Wu8lG(*@5c(>PGRYCIR|_ZhQB7yCh|FAFD~PiNQgmZRyN zYyR3szR2hNy7t+|*Oc5W*G*^>75ztpm0!OJk>_;&26(#WRrkeDb~#_FQ6*kUh1iNv zXZ=l=9zBoT&lfYdu9XExX2wwrJ}FGXk2y|#1K<4Hhux76P%g36Xp8>ybeH#p2DZcO zf!$r5_UNa318e66%qBFvA#*=D^6P4(=<9UJXb#Xlps1-7;vZrdV z+$~S^diB9#g-=EsbUbmYL4@U;bUxVES0tg58Z2?~jGnLc4i;Nmb3Ul6lw-vMeQuSf zt5_V&?wzcF8N2ni_J+K8RW%CewC!s)M@GjhY*@r>lu@CJg_C9~TS4FTMZ*x(^w;f$ zL)8M*08A6haPw2M+0rv7^6q%r!9C?>hZkUC0P+rQ;t$F(cP=1KRV+{@XJ-!s_?g3{ z)`tL`Au;|vE)JKm%BpLUeom5Tt%8c2+!ypIy?p&z5q#fd8I|mMou@=!1y)5}9akV~ zpj(9tdPKxs>aFr(wdIWH9mlnfp`ntS4j5nGQSm3x3-uPzE7rhwOApA#>+rtq2f9%H zwm%RWPI>dE?T>vdfXXqs?$gO6b9asiV6YhO=00FVwclIxrJv(Cg4N0IUfc6nVGw$M zx%zN=af?%WJ*U{8$QSj{|J)i(2Zd}(?wFP}XyV(0`e$`vjsS>{IlzTgz+@}#mbA7f zPM)MI6(uIRe;Fqip){i8SJ~~#$U?b1DUrW_Tb#Cu)o>F)8o#o0wrso`l1gNjD+uAg zf#k+pn47ci&CSdmr9#~XPO}xC>>9#T4a6<}OdE zf77iCZtd>E;}ZZ|gSm7R9#r4y!GDwyaUom-b)P&%#aLc7xYHXdaRul%Rws9)G*SL! zKF`DsN&clxQh$7qWY;c+C$Cb5}%KsPOyN+X9 z{PCnd0KQlCf)1^|kOZp=rx7d$%?4orySn-sK?-2Dpa^f$TLn<`jjrcMJzae!vxTT; zlOCW)NESeUz6+iq#jD4&c>LkSxu$=&Df7X;apz}jEJh1}__3Od6XC}7?h4e_wCpi+ za2V}jnthU#UQfZd4QRp*b!_}y`};SJO4EmIXS$#{fsPJ1Sa$&bjx3+jywA7w&w%MN$Dx*=#qD)ig$=@tr@g`hsctVR+}&8RP=X2 z-lAX-jnf*s*AS%hG~J=N;U`ZgNM+HplC*D~txow-@>DGf+vctp`t(4f1O7oq}=b zlWY%L4h%{`0wYtG>ZCr9k9QvQw$Oi@`7Zcyx^yuO;Jugj7e2%?sQL+I4AZdu3n-7h zz40L9=Jp$d3?B9k?Huhb$QNihIX$X(*rx!1bpWb*K|&H1F8k4E2l)dF%c^G@9Sk(f z=o=c^0I-#gj*g3!E$FGV2K`~3a|hp+;g114i_>lM=?)-cuMZnlnnzF%#iKkYYFXsF znW=DP1o#(#)o?A=tRHLr7qxbys6GF+6Y9ps258WbFL01%(5zL+pL_f> zIr$Haw(0$Q&PR9XNnoxlz^GA!p9VnCpkK8D>An{kHkjJh+lvLjBQ2#vBGy53V8C1f zBj2I+!BCz&1Qi{19l);bny^V;V`KZczU|2b=KdCx&SKeFSZozkX;Q8LGbO0$*z9c3 z4>Vlf9KGjfc7l$Kyaa1TDv3V^W(h?K%+1weoQzz}zObMzz?^;=YkstvqMd*%~ zi)97_K$8gz3uA$X=oM}Wb-5dqUJgS5;&U*c+Sh#6U>)__-pp)(mSJN>qroX~#unN- zJDaGi<(v>VTa)D+Z1@Y39hH>SH(@p<(e%>yX}xP4XcZl>)78xsWQwEs%X`eA>gkD4 z$oEhCXp3e`^xDszAxM6{8UV?GRrvPp+tOswT;e(tQ`vh;O5E8TiR{t$)!@9A$mx%2 zZU6CTclYeB+@lhEDa}(hjXjFYKOCn{%TlX&u))*i)$=@*;rD3dIiFw6B*`TncaJ4N z6bQEZX-)6od~%jdcVmlL(GrhA%l6zC^Zt%n+ZzlM6O&6;(@#6>LnigRfK|lMoo#v5PLGli_<1QQ#9)Dvqbh?U zB(xXKd{kvNm8V%BI|%<5$9Z^OT#x`;+x-C$zJ}cV{9e=3Dr(iHSHeuuN_^MP8eRtY z`_~U%c9s8(jP#DljxkirQ&{F9o)$1tg!}YDAw|%l;1v)8=0kCR$O2 z!=9Q(z1lTj!+n(_+L`&X>j?Y4Ar5rDareQeZdkq@B5Aqg5i1tBFgY%omseD|A~$K? z65n25=&WtNR4C92{k>=p1fA1c9YN}u@A|&2=-s<_s)!lNO7ocqKp=3o8IJXkgb-Vq z0ANZ*8?P5Zr_w_J==+)BQ&v?)KNkP-`5&x~FT$Cn0b2c$Si!v?o8BnsZskA4%86izSFQFEp(!BVHAmfublrSm9%#Nh zn&0ClP~)YOddJe%VwIWzOW4Sv?Cq9{(_wpsg_YR=$L6043=NLaVH=0Sp5|g8wFpV_ zw~w4OD3fP*%tlqZj)XM(U5nx?eN-;djO|*L)711?;po`uBHh`n;%wxX#4shiM2v#uR5VlWyt5KM=uO8)FG8J%Vz>7BT>AnI9X&>G zDO*Bv@URlI1p+8K?H%lMHxHE55&nMYM3d4Wuu;IWX2MvRj{tP~9 zB)}Vo!y9aD%31kt6-!Y3W3%Xx3Ze&ainjCT9tk2n^)|rAKkdNLlkYgQv6#lA4DZUq zBqLRHUd^F!+IxtER7$7B`uqVB(pLleD|1~WBqVtAQy=gTg(%)5B%}{af8HP=eGh#3 z1Q`iQD&)gc@J8Jqg5dAiWcQzn$J2R%gw)dhzkZ}FP3J6fNHJ~-ycr2eTtr5=Sm>+w F{{>9+2Y~

Date: Wed, 23 Oct 2019 14:28:38 +0200 Subject: [PATCH 09/80] use Travis for docs (#8) --- .travis.yml | 9 +++++++++ README.md | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/.travis.yml b/.travis.yml index 02bc13d8..4a1b1c0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,5 +8,14 @@ julia: - 1.3 notifications: email: false +jobs: + include: + - stage: "Documentation" + julia: 1.3 + os: linux + script: + - julia --project=docs/ -e 'using Pkg; Pkg.instantiate()' + - julia --project=docs/ docs/make.jl + after_success: skip after_success: - julia -e 'using Pkg; Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder())' diff --git a/README.md b/README.md index 92659754..ceeafc03 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ [![Build Status](https://travis-ci.com/KristofferC/PackageCompilerX.jl.svg?branch=master)](https://travis-ci.com/KristofferC/PackageCompilerX.jl) [![Codecov](https://codecov.io/gh/KristofferC/PackageCompilerX.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/KristofferC/PackageCompilerX.jl) +[![][docs-stable-img]][docs-stable-url] This package requires `gcc` to be installed and on the `PATH`. On Windows, we recommend installing `gcc` via [MinGW](http://www.mingw.org/). + +[docs-stable-img]: https://img.shields.io/badge/docs-stable-blue.svg +[docs-stable-url]: https://kristofferc.github.io/PackageCompilerX.jl/stable From 7b15882cad1a7fc283523d16ebe8728345f99b0d Mon Sep 17 00:00:00 2001 From: KristofferC Date: Wed, 23 Oct 2019 15:31:13 +0200 Subject: [PATCH 10/80] fixups --- .travis.yml | 9 +++++++-- src/juliaconfig.jl | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4a1b1c0a..2c8eb3d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,13 @@ language: julia os: - linux - osx - - windows -# - arm64 +# - windows +# - arm64 +branches: + only: + - master + - /^release-.*/ + - /^v[0-9]+\.[0-9]+\.[0-9]+$/ # version tags julia: - 1.3 notifications: diff --git a/src/juliaconfig.jl b/src/juliaconfig.jl index dd64ee70..05765ff9 100644 --- a/src/juliaconfig.jl +++ b/src/juliaconfig.jl @@ -29,6 +29,7 @@ function ldflags() fl = "-L$(shell_escape(julia_libdir()))" if Sys.iswindows() fl = fl * " -Wl,--stack,8388608" + fl = fl * " -Wl,--export-all-symbols" elseif Sys.islinux() fl = fl * " -Wl,--export-dynamic" end From 48fb1176088ebb747ea52f010dac0c7c90097c14 Mon Sep 17 00:00:00 2001 From: KristofferC Date: Wed, 23 Oct 2019 16:44:52 +0200 Subject: [PATCH 11/80] fix windows --- README.md | 7 ++++--- src/PackageCompilerX.jl | 8 +++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ceeafc03..b1906672 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,9 @@ [![Codecov](https://codecov.io/gh/KristofferC/PackageCompilerX.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/KristofferC/PackageCompilerX.jl) [![][docs-stable-img]][docs-stable-url] -This package requires `gcc` to be installed and on the `PATH`. -On Windows, we recommend installing `gcc` via [MinGW](http://www.mingw.org/). +On Linux or Mac this package requires `gcc` to be installed and be on the `PATH`. +On Windows, a cygwin setup needs to be installed as decribed [here](https://github.com/JuliaLang/julia/blob/master/doc/build/windows.md#cygwin-to-mingw-cross-compiling) and Julia need to be run in the cygwin environement. + [docs-stable-img]: https://img.shields.io/badge/docs-stable-blue.svg -[docs-stable-url]: https://kristofferc.github.io/PackageCompilerX.jl/stable +[docs-stable-url]: https://kristofferc.github.io/PackageCompilerX.jl/dev diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 39d8d05e..36882f46 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -5,7 +5,7 @@ using Libdl include("juliaconfig.jl") -const CC = `gcc` +const CC = (Sys.iswindows() ? `x86_64-w64-mingw32-gcc` : `gcc`) function get_julia_cmd() julia_path = joinpath(Sys.BINDIR, Base.julia_exename()) @@ -126,7 +126,8 @@ function create_sysimage_from_object_file(input_object::String, sysimage_path::S else o_file = `-Wl,--whole-archive $input_object -Wl,--no-whole-archive` end - run(`$CC -v -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia`) + extra = Sys.iswindows() ? `-Wl,--export-all-symbols` : `` + run(`$CC -v -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia $extra`) return nothing end @@ -155,7 +156,8 @@ function create_executable_from_sysimage(;sysimage_path::String, else rpath = `-Wl,-rpath,\$ORIGIN` end - run(`$CC -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) -o $(executable_path) $(wrapper) $sysimage_path -O2 $rpath $flags`) + extra = Sys.iswindows() ? `-Wl,--export-all-symbols` : `` + run(`$CC -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) -o $(executable_path) $(wrapper) $sysimage_path -O2 $rpath $flags $extra`) return nothing end From f421ab968faaf0eafce31704466d0e7634e6ab2b Mon Sep 17 00:00:00 2001 From: KristofferC Date: Wed, 23 Oct 2019 17:59:57 +0200 Subject: [PATCH 12/80] start with an create_app function --- Project.toml | 1 + examples/MyApp/Manifest.toml | 6 +++++ examples/MyApp/Project.toml | 7 ++++++ examples/{hello.jl => MyApp/src/MyApp.jl} | 8 +++++++ src/PackageCompilerX.jl | 28 +++++++++++++++------- test/myapp | Bin 0 -> 17280 bytes test/runtests.jl | 11 +++++---- 7 files changed, 47 insertions(+), 14 deletions(-) create mode 100644 examples/MyApp/Manifest.toml create mode 100644 examples/MyApp/Project.toml rename examples/{hello.jl => MyApp/src/MyApp.jl} (55%) create mode 100755 test/myapp diff --git a/Project.toml b/Project.toml index d634dbf8..2f4fdfec 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "0.1.0" [deps] Example = "7876af07-990d-54b4-ab0e-23690620f79a" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [compat] julia = "1" diff --git a/examples/MyApp/Manifest.toml b/examples/MyApp/Manifest.toml new file mode 100644 index 00000000..550c2420 --- /dev/null +++ b/examples/MyApp/Manifest.toml @@ -0,0 +1,6 @@ +# This file is machine-generated - editing it directly is not advised + +[[Example]] +git-tree-sha1 = "46e44e869b4d90b96bd8ed1fdcf32244fddfb6cc" +uuid = "7876af07-990d-54b4-ab0e-23690620f79a" +version = "0.5.3" diff --git a/examples/MyApp/Project.toml b/examples/MyApp/Project.toml new file mode 100644 index 00000000..b43f22af --- /dev/null +++ b/examples/MyApp/Project.toml @@ -0,0 +1,7 @@ +name = "MyApp" +uuid = "f943f3d7-887a-4ed5-b0c0-a1d6899aa8f5" +authors = ["KristofferC "] +version = "0.1.0" + +[deps] +Example = "7876af07-990d-54b4-ab0e-23690620f79a" diff --git a/examples/hello.jl b/examples/MyApp/src/MyApp.jl similarity index 55% rename from examples/hello.jl rename to examples/MyApp/src/MyApp.jl index f77d132d..73153db8 100644 --- a/examples/hello.jl +++ b/examples/MyApp/src/MyApp.jl @@ -1,6 +1,14 @@ +module MyApp + +using Example + +greet() = print("Hello World!") + Base.@ccallable function julia_main(ARGS::Vector{String})::Cint println("hello, world") + @show Example.domath(5) @show sin(0.0) return 0 end +end # module diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 36882f46..47d0f744 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -1,7 +1,8 @@ module PackageCompilerX using Base: active_project -using Libdl +using Libdl: Libdl +using Pkg: Pkg include("juliaconfig.jl") @@ -21,7 +22,8 @@ function run_precompilation_script(project::String, precompile_file::String) return tracefile end -function create_object_file(object_file::String, packages::Union{Symbol, Vector{Symbol}}, project::String=active_project(); +function create_object_file(object_file::String, packages::Union{Symbol, Vector{Symbol}}; + project::String=active_project(), precompile_execution_file::Union{String, Nothing}=nothing, precompile_statements_file::Union{String, Nothing}=nothing) # include all packages into the sysimage @@ -35,12 +37,6 @@ function create_object_file(object_file::String, packages::Union{Symbol, Vector{ for package in packages julia_code *= "using $package\n" end - - # include the "App file" containing julia_main - example = joinpath(@__DIR__, "..", "examples", "hello.jl") - julia_code *= """ - include($(repr(example))) - """ # handle precompilation if precompile_execution_file !== nothing || precompile_statements_file !== nothing @@ -101,7 +97,7 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}=Symbol[]; end object_file = tempname() * ".o" - create_object_file(object_file, packages, project; precompile_execution_file=precompile_execution_file, + create_object_file(object_file, packages; project=project, precompile_execution_file=precompile_execution_file, precompile_statements_file=precompile_statements_file) @show sysimage_path create_sysimage_from_object_file(object_file, sysimage_path) @@ -161,4 +157,18 @@ function create_executable_from_sysimage(;sysimage_path::String, return nothing end +function create_app(app_dir::String, + precompile_execution_file::Union{String, Nothing}=nothing, + precompile_statements_file::Union{String, Nothing}=nothing) + + project_path = Pkg.Types.projectfile_path(app_dir; strict=true) + project_toml = Pkg.TOML.parsefile(project_path) + app_name = get(() -> error("expected package to have a name entry"), project_toml, "name") + sysimage_file = app_name * "." * Libdl.dlext + cd(app_dir) do + create_sysimage(Symbol(app_name); sysimage_path=sysimage_file, project = app_dir) + create_executable_from_sysimage(;sysimage_path=sysimage_file, executable_path=app_name) + end +end + end # module diff --git a/test/myapp b/test/myapp new file mode 100755 index 0000000000000000000000000000000000000000..b9d9bd2ae59bac2539b8d01b0418cae93fa94f80 GIT binary patch literal 17280 zcmeHOeQ;FO6~CKJVENjt2$+JF6~P8=HUuOFFm*SdFE)G#!H-sa*WxX#uozyHPrxcIvQT%I#eXc6fg0TKICUrS^*T zbBxZLDP0y(8xHl!&tiGrGoT}Q3EEe!7ugAqj8&h$ z^BeOftCtQ{-+%d}mEpxQHXuiTsDlOdsX19kAIM`Yce(Zpaq$erF|}fvqg;4V_8B1X-+uLvM<1BF{N5=KfAQ%4SH5z? zg^$g8_WEfvjywQnmZcp!L_@Sdeh+X~_z#QVj~2oITm*lk2wq;KKj#<0mw+EP9OhC} zM1K$ztibiG9Li983g{PaGYm>uLR7>|1(AO6Gi%YvWTA*MfAG>cflP}4W-o;S}P09prJ)3;Od6dR#pw{Zt9zWkjbt2zQzAbs$FnZVJY6GJ6a35$E~;a>DOr zocgnY@HYW>i}CFq1pGXl)ZDrvkN=&3)035(}iDW1!|wB>dYz7wfUO7K%laSnW+}2~~?hV+r&Jy-B9x@fh?Un-Zzb=sy() z^M0uhUD^s=7>LC-$>vO*h0a4zP=cpn=oWCkwe9OrB736tCt_RFU^IYD_TtIcL-=~P z`qjQr)E^FQ$7rI6qA6pg1_OT04}zYgj7vgerQT#p?=bl1h!;eWI;M8428R$jHI)no zjMmU>2zYEjZ&I{(b~H7si#>}yOU3-vYdhLIRxPg`h(&_6n|f<~@pw4Y>z8q?mDi@G z&a=d`s3y@1lYgk^+%>Q4Zhu)*Aq|IL58DWW&ZdtT>LvqO{cY4I-I$oyWw z&($%SgsM!>TNjNBw+Y`%`VOz|pZzfZ^wm0!&s=CYUk({?9&0Fk*?`Z=Yr}h*0iR~T zGX@-Ht9%(V;5>Js@O=YroQJy!$7g7Mj6;3Efa?)QFG9o(IKM(6zSV&9oP+qF0hjNA z2tv8TfE(XSb{TN&6PMivoMVl$JqCO-D+B$p0jF0}Dcx(pD-HVl3^-j{Dcx_tX(}{W zm8zu(lp;`yz$cG@GIYYB?5ud9RtV+m`?a#%A!X=E$4K5$Zs}zp$}RXj{I8trg&Oh$ zD9?`Ope&e&JWko!Lz173JWkcw{gR)KJWkQsy^^1dJWkEoJ(4d&9;f8&F3Er30UoE~ z?4ab|MINW%Y+Ulkk;kbw+b#Jc$m63eyGHV_B9BvW)+_nvk;kbwyGZiCM;;$}S$AIE zw+nb#l;OrKoJ@8H$nbo9>wR78mGpb-(x)&~nTG+-w8K9L+FtPVIG(vOJZ+y7NLq1c zHlZy2_w-*f-;!Flii`s*!;Rkqm5%i9GOwJ;ElZJV8_m@=4XBmC@Ad! zrOdCOjv&Y~Q^0sQnNxaCD(R;>(obf#gK+4S?Q3-{!<)|39e&GEIkyGd9#MAQ^135) zKl(|(+L?Y6t#6fHn{6G-UQfLSSmreV(odyFGT%lCT&<71zKS!U-Og=h=7y%yKgygRc|+(%T(wdJFr(` zS3ucWz5r`V#hxc}xe?#>9qE^R*L9>%!j(^d=z|C>$Sf3My)s-g26SEL!ejDA7` zJMf`hn^SkVaxPrS4?5E)Thj0Nax-32hDK}(MEp1gSP!r7W}gau5IC~rm@61%*nv_h zMW7UcQUpp7C`F(Yfl>rY5hz8V6oFC%{=X5xuh;Zb%9^#S+t>P5s%;&etwR1f08QKG z*jM0hxbj=)ST464>Z3sS104f;0;u>$E{7j&$AIFOfzfw!xtCE6v;pun|IFnEah(hW ze-FiC+rC!Vwz_O{rcQM1vQ2a$j=%K}f)D=93H7Y?g_uwJ-S6abqp0X~wK=PT zy35|`bPt!eI;-!jP@IcypP)D!h9 z*Aw!E9%R!AYItu-m2NI(5@kYWLZN*qycY-8cUX9x#(QNh&|z^0HN^NJZ9?LHJIJ13 z+({_QXNj`jshoa(pe`Z!oll-`HyOgOdXgu10+xG{uOj(>h%YCw76^| zdL7X|qMM1{PV^q44-x$((St-^B>D!?lSIw!ep24y)h*hao0q$**Y~8NTFTuZx6sw6 zBv-dk=Re-F9Xh5htcjeB6{6! zu|F6k_WUe()&|FWIedURi|93+zb5eJX{dAl!lh8R(ys%&4C3!KZn^_(3E^=Q-bOe- zAApqwzbV-a{pa~;lf=iz?QX!aU4HIJC-4t9{@X0#U#$O+O8x2L7ZKK`^Q% zwu!!kKN3^}sYqlSm>4*jAyA;Is%`KCM624mss-~k+B;V_`8w6rZEfpXyVNdUQ)ern zEjO<6t?X#V=94jXz#k3BR1e7fV7o%cm{uV3H1tFVnGFIEBy*{)3N@&-thGpmfaC?s zY!1qVFeW(|(>XpW;Q?HJOo|6)WSmn{hs=E8whHoIz=a{*!<1t|!gEe(`RD{2nG%A5 zpnMHu#)#KzOxY;N^`r$3tV^rS+NJ{SrR@Q8qCv|dK3!j#vkB$m2d zM|ueyxHe>aULP=>k4lDOUcc3Y%xLeXbwQpQY_V##ZvpHitPQz+FWEEgMk6S^p62za zjta2~D)4ra@BbdMXIe!(uD>x(Y|rc5>i|O)ZlBjLOuJbCI#|$$vD^#{u6o9xZ~Cs=HU3dWS}dELa6*Z17M`TTc~eG@t4?*&Zx`vThI zHwCl(U4UUsuz#}fiv>~s4#8sc^-qJqXwU04rlV{~)Z|`#$7G*>8K9Q}?k~Gxp6T~Z z_PqXM>gT|k+Gl@1Fxm4uktu&yVSV%Y|Ag%G&A3UED1XNyblmuP1OymAws(^q(@r)d zYHt5=z>W5CvS+HG&`>yT3_SxC#O&ml*AJX0&g(wbH`_l80@!jd#~|vM?fH763=7*c zJq%bS95l@H>l!!gKF7j(I(L?5{7n$xz03Bze&+mtHwBjOZ?@xp?FL&65!>_kIi(gr zDEoAh|IhYJ$3SAV7hY0bq*HE(&~fL3`OfHz+voRV7uj<=Y;U@p;uHuFBXw32Y=21} z(JO3kq#>F Date: Wed, 27 Nov 2019 12:51:53 +0100 Subject: [PATCH 13/80] Implement a create app function that bundles everything needed to run it. (#13) --- .gitignore | 1 + examples/MyApp/Artifacts.toml | 12 +++ examples/MyApp/Manifest.toml | 56 ++++++++++++ examples/MyApp/Project.toml | 1 + examples/MyApp/src/MyApp.jl | 33 ++++++- src/PackageCompilerX.jl | 166 ++++++++++++++++++++++++++++------ src/embedding_wrapper.c | 26 ++++-- src/juliaconfig.jl | 5 - test/runtests.jl | 14 +-- 9 files changed, 265 insertions(+), 49 deletions(-) create mode 100644 examples/MyApp/Artifacts.toml diff --git a/.gitignore b/.gitignore index 0500fc6b..c3d31939 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.jl.mem .DS_Store /Manifest.toml +/examples/MyApp/MyApp /dev/ *.so *.dll diff --git a/examples/MyApp/Artifacts.toml b/examples/MyApp/Artifacts.toml new file mode 100644 index 00000000..d90994d1 --- /dev/null +++ b/examples/MyApp/Artifacts.toml @@ -0,0 +1,12 @@ +# Example Artifacts.toml file +[socrates] +git-tree-sha1 = "43563e7631a7eafae1f9f8d9d332e3de44ad7239" +lazy = false + + [[socrates.download]] + url = "https://github.com/staticfloat/small_bin/raw/master/socrates.tar.gz" + sha256 = "e65d2f13f2085f2c279830e863292312a72930fee5ba3c792b14c33ce5c5cc58" + + [[socrates.download]] + url = "https://github.com/staticfloat/small_bin/raw/master/socrates.tar.bz2" + sha256 = "13fc17b97be41763b02cbb80e9d048302cec3bd3d446c2ed6e8210bddcd3ac76" diff --git a/examples/MyApp/Manifest.toml b/examples/MyApp/Manifest.toml index 550c2420..7f04b86e 100644 --- a/examples/MyApp/Manifest.toml +++ b/examples/MyApp/Manifest.toml @@ -1,6 +1,62 @@ # This file is machine-generated - editing it directly is not advised +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + [[Example]] git-tree-sha1 = "46e44e869b4d90b96bd8ed1fdcf32244fddfb6cc" uuid = "7876af07-990d-54b4-ab0e-23690620f79a" version = "0.5.3" + +[[InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[LibGit2]] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[Pkg]] +deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" diff --git a/examples/MyApp/Project.toml b/examples/MyApp/Project.toml index b43f22af..8c061a07 100644 --- a/examples/MyApp/Project.toml +++ b/examples/MyApp/Project.toml @@ -5,3 +5,4 @@ version = "0.1.0" [deps] Example = "7876af07-990d-54b4-ab0e-23690620f79a" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" diff --git a/examples/MyApp/src/MyApp.jl b/examples/MyApp/src/MyApp.jl index 73153db8..8e955358 100644 --- a/examples/MyApp/src/MyApp.jl +++ b/examples/MyApp/src/MyApp.jl @@ -1,14 +1,39 @@ module MyApp using Example +using Pkg.Artifacts -greet() = print("Hello World!") +socrates = joinpath(ensure_artifact_installed("socrates", joinpath(@__DIR__, "..", "Artifacts.toml")), + "bin", "socrates") -Base.@ccallable function julia_main(ARGS::Vector{String})::Cint - println("hello, world") +Base.@ccallable function julia_main()::Cint + try + real_main() + catch + Base.invokelatest(Base.display_error, Base.catch_stack()) + return 1 + end + return 0 +end + +function real_main() + @show ARGS + @show Base.PROGRAM_FILE + @show DEPOT_PATH + @show LOAD_PATH + @show pwd() + @show Base.active_project() + @show Sys.BINDIR + @info "Running an artifact:" + run(`$socrates`) + @show unsafe_string(Base.JLOptions().image_file) @show Example.domath(5) @show sin(0.0) - return 0 + return +end + +if abspath(PROGRAM_FILE) == @__FILE__ + real_main() end end # module diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 47d0f744..e8e7c183 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -1,11 +1,20 @@ module PackageCompilerX +# TODO: Add good debugging statements + using Base: active_project using Libdl: Libdl using Pkg: Pkg +if isdefined(Pkg, :Artifacts) + const SUPPORTS_ARTIFACTS = true +else + const SUPPORTS_ARTIFACTS = false +end + include("juliaconfig.jl") +# TODO: Check more carefully how to just use mingw on windows without using cygwin. const CC = (Sys.iswindows() ? `x86_64-w64-mingw32-gcc` : `gcc`) function get_julia_cmd() @@ -14,11 +23,14 @@ function get_julia_cmd() cmd = `$julia_path -J$image_file --color=yes --startup-file=no -Cnative` end +# TODO: Add output file? # Returns a vector of precompile statemenets function run_precompilation_script(project::String, precompile_file::String) tracefile = tempname() julia_code = """Base.__init__(); include($(repr(precompile_file)))""" - run(`$(get_julia_cmd()) --project=$project --trace-compile=$tracefile -e $julia_code`) + cmd = `$(get_julia_cmd()) --project=$project --trace-compile=$tracefile -e $julia_code` + @debug "run_precompilation_script: running $cmd" + run(cmd) return tracefile end @@ -42,6 +54,7 @@ function create_object_file(object_file::String, packages::Union{Symbol, Vector{ if precompile_execution_file !== nothing || precompile_statements_file !== nothing precompile_statements = "" if precompile_execution_file !== nothing + @debug "running precompilation execution script..." tracefile = run_precompilation_script(project, precompile_execution_file) precompile_statements *= "append!(precompile_statements, readlines($(repr(tracefile))))\n" end @@ -77,11 +90,15 @@ function create_object_file(object_file::String, packages::Union{Symbol, Vector{ # finally, make julia output the resulting object file @debug "creating object file at $object_file" @info "PackageCompilerX: creating object file, this might take a while..." - run(`$(get_julia_cmd()) --project=$project --output-o=$(object_file) -e $julia_code`) + cmd = `$(get_julia_cmd()) --project=$project --output-o=$(object_file) -e $julia_code` + @debug "running $cmd" + run(cmd) end default_sysimage_path() = joinpath(julia_private_libdir(), "sys." * Libdl.dlext) -backup_sysimage_path() = default_sysimage_path() * ".backup" +default_sysimage_name() = basename(default_sysimage_path()) +backup_default_sysimage_path() = default_sysimage_path() * ".backup" +backup_default_sysimage_name() = basename(backup_default_sysimage_path()) function create_sysimage(packages::Union{Symbol, Vector{Symbol}}=Symbol[]; sysimage_path::Union{String,Nothing}=nothing, @@ -97,14 +114,15 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}=Symbol[]; end object_file = tempname() * ".o" - create_object_file(object_file, packages; project=project, precompile_execution_file=precompile_execution_file, + create_object_file(object_file, packages; + project=project, + precompile_execution_file=precompile_execution_file, precompile_statements_file=precompile_statements_file) - @show sysimage_path create_sysimage_from_object_file(object_file, sysimage_path) if replace_default_sysimage - if !isfile(backup_sysimage_path()) - cp(default_sysimage_path(), backup_sysimage_path()) - @debug "making a backup of sysimage" + if !isfile(backup_default_sysimage_path()) + @debug "making a backup of default sysimage" + cp(default_sysimage_path(), backup_default_sysimage_path()) end @info "PackageCompilerX: default sysimage replaced, restart Julia for the new sysimage to be in effect" cp(sysimage_path, default_sysimage_path(); force=true) @@ -123,16 +141,16 @@ function create_sysimage_from_object_file(input_object::String, sysimage_path::S o_file = `-Wl,--whole-archive $input_object -Wl,--no-whole-archive` end extra = Sys.iswindows() ? `-Wl,--export-all-symbols` : `` - run(`$CC -v -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia $extra`) + run(`$CC -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia $extra`) return nothing end function restore_default_sysimage() - if !isfile(backup_sysimage_path()) + if !isfile(backup_default_sysimage_path()) error("did not find a backup sysimage") end - cp(backup_sysimage_path(), default_sysimage_path(); force=true) - rm(backup_sysimage_path()) + cp(backup_default_sysimage_path(), default_sysimage_path(); force=true) + rm(backup_default_sysimage_path()) @info "PackageCompilerX: default sysimage restored, restart Julia for the new sysimage to be in effect" return nothing end @@ -140,35 +158,127 @@ end # This requires that the sysimage have been built so that there is a ccallable `julia_main` # in Main. function create_executable_from_sysimage(;sysimage_path::String, - executable_path::String) + executable_path::String) flags = join((cflags(), ldflags(), ldlibs()), " ") flags = Base.shell_split(flags) wrapper = joinpath(@__DIR__, "embedding_wrapper.c") if Sys.iswindows() - # Cannot create an executable without copying dlls on Windows... - rpath = `` # functionality doesn't readily exist on this platform + rpath = `` elseif Sys.isapple() - rpath = `-Wl,-rpath,@executable_path` + # TODO: Only add `../julia` when bundling + rpath = `-Wl,-rpath,@executable_path:@executable_path/../lib` else - rpath = `-Wl,-rpath,\$ORIGIN` + rpath = `-Wl,-rpath,\$ORIGIN:\$ORIGIN/../lib` end - extra = Sys.iswindows() ? `-Wl,--export-all-symbols` : `` - run(`$CC -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) -o $(executable_path) $(wrapper) $sysimage_path -O2 $rpath $flags $extra`) + cmd = `$CC -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) -o $(executable_path) $(wrapper) $(sysimage_path) -O2 $rpath $flags` + @debug "running $cmd" + run(cmd) return nothing end -function create_app(app_dir::String, - precompile_execution_file::Union{String, Nothing}=nothing, - precompile_statements_file::Union{String, Nothing}=nothing) - - project_path = Pkg.Types.projectfile_path(app_dir; strict=true) - project_toml = Pkg.TOML.parsefile(project_path) - app_name = get(() -> error("expected package to have a name entry"), project_toml, "name") +function create_app(package_dir::String, + app_dir::String, + precompile_execution_file::Union{String,Nothing}=nothing, + precompile_statements_file::Union{String,Nothing}=nothing, + # sysimage_path::Union{String,Nothing}=nothing, # optional sysimage + bundle=true, + force=false) + project_toml_path = abspath(Pkg.Types.projectfile_path(package_dir; strict=true)) + manifest_toml_path = abspath(Pkg.Types.manifestfile_path(package_dir)) + if manifest_toml_path === nothing + @warn "it is not recommended to create an app without a preexisting manifest" + end + project_toml = Pkg.TOML.parsefile(project_toml_path) + project_path = abspath(package_dir) + app_name = get(project_toml, "name") do + error("expected package to have a `name`-entry") + end sysimage_file = app_name * "." * Libdl.dlext + + ctx = Pkg.Types.Context(env=Pkg.Types.EnvCache(project_toml_path)) + @debug "instantiating project at \"$project_toml_path\"" + Pkg.instantiate(ctx) + + if isdir(app_dir) + if !force + error("directory $(repr(app_dir)) already exists, use `force=true` to overwrite (will completely", + " remove the directory") + end + rm(app_dir; force=true, recursive=true) + end + + mkpath(app_dir) + + if bundle + bundle_julia_libraries(app_dir) + if SUPPORTS_ARTIFACTS + bundle_artifacts(ctx, app_dir) + end + end + + # TODO: Maybe avoid this cd? cd(app_dir) do - create_sysimage(Symbol(app_name); sysimage_path=sysimage_file, project = app_dir) - create_executable_from_sysimage(;sysimage_path=sysimage_file, executable_path=app_name) + create_sysimage(Symbol(app_name); sysimage_path=sysimage_file, project=project_path) + mkpath("bin") + create_executable_from_sysimage(; sysimage_path=sysimage_file, executable_path=joinpath("bin", app_name)) + mv(sysimage_file, joinpath("bin", sysimage_file)) + end + return +end + +function bundle_julia_libraries(app_dir) + app_libdir = joinpath(app_dir, Sys.isunix() ? "lib" : "bin") + cp(julia_libdir(), app_libdir; force=true) + # We do not want to bundle the sysimage (nor the backup): + rm(joinpath(app_libdir, "julia", default_sysimage_name()); force=true) + rm(joinpath(app_libdir, "julia", backup_default_sysimage_name()); force=true) + return +end + +function bundle_artifacts(ctx, app_dir) + @debug "bundling artifacts..." + + pkgs = Pkg.Types.PackageSpec[] + Pkg.Operations.load_all_deps!(ctx, pkgs) + + # Also want artifacts for the project itself + if ctx.env.pkg !== nothing + # This is kinda ugly... + ctx.env.pkg.path = dirname(ctx.env.project_file) + push!(pkgs, ctx.env.pkg) + end + + # Collect all artifacts needed for the project + artifact_paths = String[] + for pkg in pkgs + pkg_source_path = Pkg.Operations.source_path(pkg) + pkg_source_path === nothing && continue + # Check to see if this package has an (Julia)Artifacts.toml + for f in Pkg.Artifacts.artifact_names + artifacts_toml_path = joinpath(pkg_source_path, f) + if isfile(artifacts_toml_path) + @debug "bundling artifacts for $(pkg.name)" + artifact_dict = Pkg.Artifacts.load_artifacts_toml(artifacts_toml_path) + for name in keys(artifact_dict) + @debug " \"$name\"" + push!(artifact_paths, Pkg.Artifacts.ensure_artifact_installed(name, artifacts_toml_path)) + end + break + end + end + end + + # Copy the artifacts needed to the app directory + artifact_app_path = joinpath(app_dir, "artifacts") + if !isempty(artifact_paths) + mkpath(artifact_app_path) + end + for artifact_path in artifact_paths + artifact_name = basename(artifact_path) + # force=true? + cp(artifact_path, joinpath(artifact_app_path, artifact_name)) end + return end end # module diff --git a/src/embedding_wrapper.c b/src/embedding_wrapper.c index 93ba3d9a..627b876b 100644 --- a/src/embedding_wrapper.c +++ b/src/embedding_wrapper.c @@ -3,6 +3,7 @@ // Standard headers #include #include +#include // Julia headers (for initialization and gc commands) #include "uv.h" @@ -10,20 +11,33 @@ JULIA_DEFINE_FAST_TLS() +// TODO: Windows wmain handling as in repl.c + // Declare C prototype of a function defined in Julia -extern int julia_main(jl_array_t*); +int julia_main(jl_array_t*); // main function (windows UTF16 -> UTF8 argument conversion code copied from julia's ui/repl.c) int main(int argc, char *argv[]) { - int retcode; - int i; uv_setup_args(argc, argv); // no-op on Windows // initialization libsupport_init(); - // jl_options.compile_enabled = JL_OPTIONS_COMPILE_OFF; + // Get the current exe path so we can compute a relative depot path + char *free_path = (char*)malloc(PATH_MAX); + size_t path_size = PATH_MAX; + if (!free_path) + jl_errorf("fatal error: failed to allocate memory: %s", strerror(errno)); + if (uv_exepath(free_path, &path_size)) { + jl_error("fatal error: unexpected error while retrieving exepath"); + } + + char buf[PATH_MAX]; + snprintf(buf, sizeof(buf), "JULIA_DEPOT_PATH=%s/../", free_path); + putenv(buf); + putenv("JULIA_LOAD_PATH=@"); + // JULIAC_PROGRAM_LIBNAME defined on command-line for compilation jl_options.image_file = JULIAC_PROGRAM_LIBNAME; julia_init(JL_IMAGE_JULIA_HOME); @@ -38,13 +52,13 @@ int main(int argc, char *argv[]) // Set Base.ARGS to `String[ unsafe_string(argv[i]) for i = 1:argc ]` jl_array_t *ARGS = (jl_array_t*)jl_get_global(jl_base_module, jl_symbol("ARGS")); jl_array_grow_end(ARGS, argc - 1); - for (i = 1; i < argc; i++) { + for (int i = 1; i < argc; i++) { jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]); jl_arrayset(ARGS, s, i - 1); } // call the work function, and get back a value - retcode = julia_main(ARGS); + int retcode = julia_main(ARGS); // Cleanup and gracefully exit jl_atexit_hook(retcode); diff --git a/src/juliaconfig.jl b/src/juliaconfig.jl index 05765ff9..f8fc6a78 100644 --- a/src/juliaconfig.jl +++ b/src/juliaconfig.jl @@ -1,7 +1,5 @@ # adopted from https://github.com/JuliaLang/julia/blob/release-0.6/contrib/julia-config.jl -threading_on() = ccall(:jl_threading_enabled, Cint, ()) != 0 - function shell_escape(str) str = replace(str, "'" => "'\''") return "'$str'" @@ -55,9 +53,6 @@ function cflags() print(flags, "-std=gnu99") include = shell_escape(julia_includedir()) print(flags, " -I", include) - if threading_on() - print(flags, " -DJULIA_ENABLE_THREADING=1") - end if Sys.isunix() print(flags, " -fPIC") end diff --git a/test/runtests.jl b/test/runtests.jl index bb76b58c..d6d794ec 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,17 +3,19 @@ using Test using Libdl @testset "PackageCompilerX.jl" begin - # Write your own tests here. - sysimage_path = "sys." * Libdl.dlext + tmp = mktempdir() + sysimage_path = joinpath(tmp, "sys." * Libdl.dlext) PackageCompilerX.create_sysimage(:Example; sysimage_path=sysimage_path, precompile_execution_file="precompile_execution.jl", precompile_statements_file="precompile_statements.jl") run(`$(Base.julia_cmd()) -J $(sysimage_path) -e 'println(1337)'`) - # Test creating an app without bundling - app_dir = joinpath(@__DIR__, "..", "examples/MyApp/") - PackageCompilerX.create_app(app_dir) - app_path = abspath(app_dir, "MyApp" * (Sys.iswindows() ? ".exe" : "")) + # Test creating an app + app_source_dir = joinpath(@__DIR__, "..", "examples/MyApp/") + app_exe_dir = joinpath(tmp, "MyApp") + PackageCompilerX.create_app(app_source_dir, app_exe_dir) + app_path = abspath(app_exe_dir, "bin", "MyApp" * (Sys.iswindows() ? ".exe" : "")) app_output = read(`$app_path`, String) @test occursin("Example.domath", app_output) + @test occursin("ἔοικα γοῦν τούτου", app_output) end From 613f7c25e4d11b66f2b2d03165c137dedc54d1eb Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 27 Nov 2019 14:05:01 +0100 Subject: [PATCH 14/80] add an audit function (#17) --- Project.toml | 5 ++- src/PackageCompilerX.jl | 84 +++++++++++++++++++++++++++++------------ test/runtests.jl | 1 + 3 files changed, 64 insertions(+), 26 deletions(-) diff --git a/Project.toml b/Project.toml index 2f4fdfec..a4c3e021 100644 --- a/Project.toml +++ b/Project.toml @@ -4,15 +4,16 @@ authors = ["KristofferC "] version = "0.1.0" [deps] -Example = "7876af07-990d-54b4-ab0e-23690620f79a" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] julia = "1" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Example = "7876af07-990d-54b4-ab0e-23690620f79a" [targets] -test = ["Test"] +test = ["Test", "Example"] diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index e8e7c183..0aab29b0 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -5,6 +5,7 @@ module PackageCompilerX using Base: active_project using Libdl: Libdl using Pkg: Pkg +using UUIDs: UUID if isdefined(Pkg, :Artifacts) const SUPPORTS_ARTIFACTS = true @@ -155,25 +156,39 @@ function restore_default_sysimage() return nothing end -# This requires that the sysimage have been built so that there is a ccallable `julia_main` -# in Main. -function create_executable_from_sysimage(;sysimage_path::String, - executable_path::String) - flags = join((cflags(), ldflags(), ldlibs()), " ") - flags = Base.shell_split(flags) - wrapper = joinpath(@__DIR__, "embedding_wrapper.c") - if Sys.iswindows() - rpath = `` - elseif Sys.isapple() - # TODO: Only add `../julia` when bundling - rpath = `-Wl,-rpath,@executable_path:@executable_path/../lib` - else - rpath = `-Wl,-rpath,\$ORIGIN:\$ORIGIN/../lib` +const REQUIRES = "Requires" => UUID("ae029012-a4dd-5104-9daa-d747884805df") + +# Check for things that might indicate that the app or dependencies +function audit_app(package_dir::String) + project_toml_path = abspath(Pkg.Types.projectfile_path(package_dir; strict=true)) + ctx = Pkg.Types.Context(env=Pkg.Types.EnvCache(project_toml_path)) + return audit_app(ctx) +end +function audit_app(ctx::Pkg.Types.Context) + # Check for Requires.jl usage + if REQUIRES in ctx.env.project.deps + @warn "Project has a dependency on Requires.jl, code in `@require` will not be run" end - cmd = `$CC -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) -o $(executable_path) $(wrapper) $(sysimage_path) -O2 $rpath $flags` - @debug "running $cmd" - run(cmd) - return nothing + for (uuid, pkg) in ctx.env.manifest + if REQUIRES in pkg.deps + @warn "$(pkg.name) has a dependency on Requires.jl, code in `@require` will not be run" + end + end + + # Check for build script usage + if isfile(joinpath(dirname(ctx.env.project_file), "deps", "build.jl")) + @warn "Project has a build script, this might indicate that it is not relocatable" + end + pkgs = Pkg.Types.PackageSpec[] + Pkg.Operations.load_all_deps!(ctx, pkgs) + for pkg in pkgs + pkg_source = Pkg.Operations.source_path(pkg) + pkg_source === nothing && continue + if isfile(joinpath(pkg_source, "deps", "build.jl")) + @warn "Package $(pkg.name) has a build script, this might indicate that it is not relocatable" + end + end + return end function create_app(package_dir::String, @@ -181,7 +196,7 @@ function create_app(package_dir::String, precompile_execution_file::Union{String,Nothing}=nothing, precompile_statements_file::Union{String,Nothing}=nothing, # sysimage_path::Union{String,Nothing}=nothing, # optional sysimage - bundle=true, + audit=true, force=false) project_toml_path = abspath(Pkg.Types.projectfile_path(package_dir; strict=true)) manifest_toml_path = abspath(Pkg.Types.manifestfile_path(package_dir)) @@ -206,16 +221,17 @@ function create_app(package_dir::String, end rm(app_dir; force=true, recursive=true) end + + audit && audit_app(ctx) mkpath(app_dir) - if bundle - bundle_julia_libraries(app_dir) - if SUPPORTS_ARTIFACTS - bundle_artifacts(ctx, app_dir) - end + bundle_julia_libraries(app_dir) + if SUPPORTS_ARTIFACTS + bundle_artifacts(ctx, app_dir) end + # TODO: Create in a temp dir and then move it into place? # TODO: Maybe avoid this cd? cd(app_dir) do create_sysimage(Symbol(app_name); sysimage_path=sysimage_file, project=project_path) @@ -226,6 +242,26 @@ function create_app(package_dir::String, return end +# This requires that the sysimage have been built so that there is a ccallable `julia_main` +# in Main. +function create_executable_from_sysimage(;sysimage_path::String, + executable_path::String) + flags = join((cflags(), ldflags(), ldlibs()), " ") + flags = Base.shell_split(flags) + wrapper = joinpath(@__DIR__, "embedding_wrapper.c") + if Sys.iswindows() + rpath = `` + elseif Sys.isapple() + rpath = `-Wl,-rpath,@executable_path:@executable_path/../lib` + else + rpath = `-Wl,-rpath,\$ORIGIN:\$ORIGIN/../lib` + end + cmd = `$CC -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) -o $(executable_path) $(wrapper) $(sysimage_path) -O2 $rpath $flags` + @debug "running $cmd" + run(cmd) + return nothing +end + function bundle_julia_libraries(app_dir) app_libdir = joinpath(app_dir, Sys.isunix() ? "lib" : "bin") cp(julia_libdir(), app_libdir; force=true) diff --git a/test/runtests.jl b/test/runtests.jl index d6d794ec..9448a0a9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -12,6 +12,7 @@ using Libdl # Test creating an app app_source_dir = joinpath(@__DIR__, "..", "examples/MyApp/") + @test_logs PackageCompilerX.audit_app(app_source_dir) app_exe_dir = joinpath(tmp, "MyApp") PackageCompilerX.create_app(app_source_dir, app_exe_dir) app_path = abspath(app_exe_dir, "bin", "MyApp" * (Sys.iswindows() ? ".exe" : "")) From b59a7cb75668d219ba2833a69ea974fdea9adfac Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 27 Nov 2019 16:40:02 +0100 Subject: [PATCH 15/80] add option to compile with incremental=false and set to default (#19) --- src/PackageCompilerX.jl | 141 ++++++++++++++++++++++++++-------------- test/runtests.jl | 22 ++++--- 2 files changed, 105 insertions(+), 58 deletions(-) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 0aab29b0..5fe8b24a 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -1,12 +1,15 @@ module PackageCompilerX # TODO: Add good debugging statements +# TODO: sysimage or sysimg... using Base: active_project using Libdl: Libdl using Pkg: Pkg using UUIDs: UUID +export create_sysimage, create_app + if isdefined(Pkg, :Artifacts) const SUPPORTS_ARTIFACTS = true else @@ -18,28 +21,59 @@ include("juliaconfig.jl") # TODO: Check more carefully how to just use mingw on windows without using cygwin. const CC = (Sys.iswindows() ? `x86_64-w64-mingw32-gcc` : `gcc`) +# TODO: Be able to set target for -C? +# TODO: Change to commented default ? +# const DEFAULT_TARGET = "generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)" +const DEFAULT_TARGET = "generic" +current_process_sysimage_path() = unsafe_string(Base.JLOptions().image_file) + function get_julia_cmd() julia_path = joinpath(Sys.BINDIR, Base.julia_exename()) - image_file = unsafe_string(Base.JLOptions().image_file) - cmd = `$julia_path -J$image_file --color=yes --startup-file=no -Cnative` + cmd = `$julia_path --color=yes --startup-file=no --cpu-target=$DEFAULT_TARGET` end # TODO: Add output file? -# Returns a vector of precompile statemenets function run_precompilation_script(project::String, precompile_file::String) tracefile = tempname() julia_code = """Base.__init__(); include($(repr(precompile_file)))""" - cmd = `$(get_julia_cmd()) --project=$project --trace-compile=$tracefile -e $julia_code` + cmd = `$(get_julia_cmd()) --sysimage=$(current_process_sysimage_path()) --project=$project + --compile=all --trace-compile=$tracefile -e $julia_code` @debug "run_precompilation_script: running $cmd" run(cmd) return tracefile end -function create_object_file(object_file::String, packages::Union{Symbol, Vector{Symbol}}; - project::String=active_project(), - precompile_execution_file::Union{String, Nothing}=nothing, - precompile_statements_file::Union{String, Nothing}=nothing) - # include all packages into the sysimage + +function bootstrap_sys() + tmp = mktempdir() + sysimg_source_path = Base.find_source_file("sysimg.jl") + base_dir = dirname(sysimg_source_path) + tmp_corecompiler_ji = joinpath(tmp, "corecompiler.ji") + tmp_sys_ji = joinpath(tmp, "sys.ji") + compiler_source_path = joinpath(base_dir, "compiler", "compiler.jl") + + @info "PackageCompilerX: creating base system image (incremental=false), this might take a while..." + cd(base_dir) do + # Create corecompiler.ji + cmd = `$(get_julia_cmd()) --output-ji $tmp_corecompiler_ji -g0 -O0 $compiler_source_path` + @debug "running $cmd" + read(cmd) + + # Use that to create sys.ji + cmd = `$(get_julia_cmd()) --sysimage=$tmp_corecompiler_ji --output-ji=$tmp_sys_ji $sysimg_source_path` + @debug "running $cmd" + read(cmd) + end + + return tmp_sys_ji +end + +function create_sysimg_object_file(object_file::String, packages::Union{Symbol, Vector{Symbol}}; + project::String, + base_sysimg::String, + precompile_execution_file::Union{String, Nothing}, + precompile_statements_file::Union{String, Nothing}) + # include all packages into the sysimg packages = vcat(packages) julia_code = """ if !isdefined(Base, :uv_eventloop) @@ -90,48 +124,57 @@ function create_object_file(object_file::String, packages::Union{Symbol, Vector{ # finally, make julia output the resulting object file @debug "creating object file at $object_file" - @info "PackageCompilerX: creating object file, this might take a while..." - cmd = `$(get_julia_cmd()) --project=$project --output-o=$(object_file) -e $julia_code` + @info "PackageCompilerX: creating system image object file, this might take a while..." + cmd = `$(get_julia_cmd()) --sysimage=$base_sysimg --project=$project --output-o=$(object_file) -e $julia_code` @debug "running $cmd" run(cmd) end -default_sysimage_path() = joinpath(julia_private_libdir(), "sys." * Libdl.dlext) -default_sysimage_name() = basename(default_sysimage_path()) -backup_default_sysimage_path() = default_sysimage_path() * ".backup" -backup_default_sysimage_name() = basename(backup_default_sysimage_path()) +default_sysimg_path() = joinpath(julia_private_libdir(), "sys." * Libdl.dlext) +default_sysimg_name() = basename(default_sysimg_path()) +backup_default_sysimg_path() = default_sysimg_path() * ".backup" +backup_default_sysimg_name() = basename(backup_default_sysimg_path()) function create_sysimage(packages::Union{Symbol, Vector{Symbol}}=Symbol[]; sysimage_path::Union{String,Nothing}=nothing, project::String=active_project(), precompile_execution_file::Union{String, Nothing}=nothing, precompile_statements_file::Union{String, Nothing}=nothing, - replace_default_sysimage::Bool=false) - if sysimage_path === nothing && replace_default_sysimage == false - error("`sysimage_path` cannot be `nothing` if `replace_default_sysimage` is `false`") + incremental=true, + replace_default_sysimg::Bool=false) + if sysimage_path === nothing && replace_default_sysimg == false + error("`sysimage_path` cannot be `nothing` if `replace_default_sysimg` is `false`") end if sysimage_path === nothing - sysimage_path = string(tempname(), ".", Libdl.dlext) + tmp = mktempdir() + sysimage_path = joinpath(tmp, string("sys.", Libdl.dlext)) + end + + if !incremental + base_sysimg = bootstrap_sys() + else + base_sysimg = current_process_sysimage_path() end object_file = tempname() * ".o" - create_object_file(object_file, packages; - project=project, - precompile_execution_file=precompile_execution_file, - precompile_statements_file=precompile_statements_file) - create_sysimage_from_object_file(object_file, sysimage_path) - if replace_default_sysimage - if !isfile(backup_default_sysimage_path()) - @debug "making a backup of default sysimage" - cp(default_sysimage_path(), backup_default_sysimage_path()) + create_sysimg_object_file(object_file, packages; + project=project, + base_sysimg=base_sysimg, + precompile_execution_file=precompile_execution_file, + precompile_statements_file=precompile_statements_file) + create_sysimg_from_object_file(object_file, sysimage_path) + if replace_default_sysimg + if !isfile(backup_default_sysimg_path()) + @debug "making a backup of default sysimg" + cp(default_sysimg_path(), backup_default_sysimg_path()) end - @info "PackageCompilerX: default sysimage replaced, restart Julia for the new sysimage to be in effect" - cp(sysimage_path, default_sysimage_path(); force=true) + @info "PackageCompilerX: default sysimg replaced, restart Julia for the new sysimg to be in effect" + mv(sysimage_path, default_sysimg_path(); force=true) end # TODO: Remove object file end -function create_sysimage_from_object_file(input_object::String, sysimage_path::String) +function create_sysimg_from_object_file(input_object::String, sysimage_path::String) julia_libdir = dirname(Libdl.dlpath("libjulia")) # Prevent compiler from stripping all symbols from the shared lib. @@ -146,13 +189,13 @@ function create_sysimage_from_object_file(input_object::String, sysimage_path::S return nothing end -function restore_default_sysimage() - if !isfile(backup_default_sysimage_path()) - error("did not find a backup sysimage") +function restore_default_sysimg() + if !isfile(backup_default_sysimg_path()) + error("did not find a backup sysimg") end - cp(backup_default_sysimage_path(), default_sysimage_path(); force=true) - rm(backup_default_sysimage_path()) - @info "PackageCompilerX: default sysimage restored, restart Julia for the new sysimage to be in effect" + cp(backup_default_sysimg_path(), default_sysimg_path(); force=true) + rm(backup_default_sysimg_path()) + @info "PackageCompilerX: default sysimg restored, restart Julia for the new sysimg to be in effect" return nothing end @@ -192,10 +235,10 @@ function audit_app(ctx::Pkg.Types.Context) end function create_app(package_dir::String, - app_dir::String, + app_dir::String; precompile_execution_file::Union{String,Nothing}=nothing, precompile_statements_file::Union{String,Nothing}=nothing, - # sysimage_path::Union{String,Nothing}=nothing, # optional sysimage + incremental=false, audit=true, force=false) project_toml_path = abspath(Pkg.Types.projectfile_path(package_dir; strict=true)) @@ -208,7 +251,7 @@ function create_app(package_dir::String, app_name = get(project_toml, "name") do error("expected package to have a `name`-entry") end - sysimage_file = app_name * "." * Libdl.dlext + sysimg_file = app_name * "." * Libdl.dlext ctx = Pkg.Types.Context(env=Pkg.Types.EnvCache(project_toml_path)) @debug "instantiating project at \"$project_toml_path\"" @@ -234,18 +277,18 @@ function create_app(package_dir::String, # TODO: Create in a temp dir and then move it into place? # TODO: Maybe avoid this cd? cd(app_dir) do - create_sysimage(Symbol(app_name); sysimage_path=sysimage_file, project=project_path) + create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=project_path, incremental=incremental) mkpath("bin") - create_executable_from_sysimage(; sysimage_path=sysimage_file, executable_path=joinpath("bin", app_name)) - mv(sysimage_file, joinpath("bin", sysimage_file)) + create_executable_from_sysimg(; sysimage_path=sysimg_file, executable_path=joinpath("bin", app_name)) + mv(sysimg_file, joinpath("bin", sysimg_file)) end return end -# This requires that the sysimage have been built so that there is a ccallable `julia_main` +# This requires that the sysimg have been built so that there is a ccallable `julia_main` # in Main. -function create_executable_from_sysimage(;sysimage_path::String, - executable_path::String) +function create_executable_from_sysimg(;sysimage_path::String, + executable_path::String) flags = join((cflags(), ldflags(), ldlibs()), " ") flags = Base.shell_split(flags) wrapper = joinpath(@__DIR__, "embedding_wrapper.c") @@ -265,9 +308,9 @@ end function bundle_julia_libraries(app_dir) app_libdir = joinpath(app_dir, Sys.isunix() ? "lib" : "bin") cp(julia_libdir(), app_libdir; force=true) - # We do not want to bundle the sysimage (nor the backup): - rm(joinpath(app_libdir, "julia", default_sysimage_name()); force=true) - rm(joinpath(app_libdir, "julia", backup_default_sysimage_name()); force=true) + # We do not want to bundle the sysimg (nor the backup): + rm(joinpath(app_libdir, "julia", default_sysimg_name()); force=true) + rm(joinpath(app_libdir, "julia", backup_default_sysimg_name()); force=true) return end diff --git a/test/runtests.jl b/test/runtests.jl index 9448a0a9..379a575a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,22 +1,26 @@ -using PackageCompilerX +using PackageCompilerX: PackageCompilerX, create_sysimage, create_app using Test using Libdl @testset "PackageCompilerX.jl" begin tmp = mktempdir() sysimage_path = joinpath(tmp, "sys." * Libdl.dlext) - PackageCompilerX.create_sysimage(:Example; sysimage_path=sysimage_path, - precompile_execution_file="precompile_execution.jl", - precompile_statements_file="precompile_statements.jl") + create_sysimage(:Example; sysimage_path=sysimage_path, + precompile_execution_file="precompile_execution.jl", + precompile_statements_file="precompile_statements.jl") run(`$(Base.julia_cmd()) -J $(sysimage_path) -e 'println(1337)'`) # Test creating an app app_source_dir = joinpath(@__DIR__, "..", "examples/MyApp/") + + # TODO: Also test something that actually gives audit warnings @test_logs PackageCompilerX.audit_app(app_source_dir) app_exe_dir = joinpath(tmp, "MyApp") - PackageCompilerX.create_app(app_source_dir, app_exe_dir) - app_path = abspath(app_exe_dir, "bin", "MyApp" * (Sys.iswindows() ? ".exe" : "")) - app_output = read(`$app_path`, String) - @test occursin("Example.domath", app_output) - @test occursin("ἔοικα γοῦν τούτου", app_output) + for incremental in (true, false) + create_app(app_source_dir, app_exe_dir; incremental=incremental, force=true) + app_path = abspath(app_exe_dir, "bin", "MyApp" * (Sys.iswindows() ? ".exe" : "")) + app_output = read(`$app_path`, String) + @test occursin("Example.domath", app_output) + @test occursin("ἔοικα γοῦν τούτου", app_output) + end end From c6e2bac55ab3314f6200e829c3c931637ec06b33 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 28 Nov 2019 13:54:19 +0100 Subject: [PATCH 16/80] fix on macos (#21) --- Project.toml | 2 +- src/PackageCompilerX.jl | 9 ++++++++- src/juliaconfig.jl | 1 + 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index a4c3e021..fc276005 100644 --- a/Project.toml +++ b/Project.toml @@ -12,8 +12,8 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" julia = "1" [extras] -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Example = "7876af07-990d-54b4-ab0e-23690620f79a" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] test = ["Test", "Example"] diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 5fe8b24a..7b5b85ab 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -281,6 +281,13 @@ function create_app(package_dir::String, mkpath("bin") create_executable_from_sysimg(; sysimage_path=sysimg_file, executable_path=joinpath("bin", app_name)) mv(sysimg_file, joinpath("bin", sysimg_file)) + if Sys.isapple() + cd("bin") do + cmd = `install_name_tool -change $sysimg_file @rpath/$sysimg_file $app_name` + @debug "running $cmd" + run(cmd) + end + end end return end @@ -295,7 +302,7 @@ function create_executable_from_sysimg(;sysimage_path::String, if Sys.iswindows() rpath = `` elseif Sys.isapple() - rpath = `-Wl,-rpath,@executable_path:@executable_path/../lib` + rpath = `-Wl,-rpath,'@executable_path' -Wl,-rpath,'@executable_path/../lib'` else rpath = `-Wl,-rpath,\$ORIGIN:\$ORIGIN/../lib` end diff --git a/src/juliaconfig.jl b/src/juliaconfig.jl index f8fc6a78..0c86a168 100644 --- a/src/juliaconfig.jl +++ b/src/juliaconfig.jl @@ -42,6 +42,7 @@ function ldlibs(relative_path=nothing) "julia" end if Sys.isunix() + # TODO, these should not be needed since it adds an rpath to the current julia process libraries. return "-Wl,-rpath,$(shell_escape(julia_libdir())) -Wl,-rpath,$(shell_escape(julia_private_libdir())) -l$libname" else return "-l$libname -lopenlibm" From d6c0109706ebcf69bbece09d25b8f36c47fb0cff Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 28 Nov 2019 18:17:26 +0100 Subject: [PATCH 17/80] only include needed stdlibs in apps (#25) * wip only using needed stdlibs * only include needed stdlibs in apps * make default off --- examples/MyApp/src/MyApp.jl | 5 +- src/PackageCompilerX.jl | 146 ++++++++++++++++++++++++------------ test/runtests.jl | 22 ++++-- 3 files changed, 117 insertions(+), 56 deletions(-) diff --git a/examples/MyApp/src/MyApp.jl b/examples/MyApp/src/MyApp.jl index 8e955358..45502c43 100644 --- a/examples/MyApp/src/MyApp.jl +++ b/examples/MyApp/src/MyApp.jl @@ -23,9 +23,12 @@ function real_main() @show LOAD_PATH @show pwd() @show Base.active_project() + @show Threads.nthreads() @show Sys.BINDIR - @info "Running an artifact:" + display(Base.loaded_modules) + println("Running an artifact:") run(`$socrates`) + println() @show unsafe_string(Base.JLOptions().image_file) @show Example.domath(5) @show sin(0.0) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 7b5b85ab..d86d4a06 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -32,20 +32,19 @@ function get_julia_cmd() cmd = `$julia_path --color=yes --startup-file=no --cpu-target=$DEFAULT_TARGET` end -# TODO: Add output file? -function run_precompilation_script(project::String, precompile_file::String) - tracefile = tempname() - julia_code = """Base.__init__(); include($(repr(precompile_file)))""" - cmd = `$(get_julia_cmd()) --sysimage=$(current_process_sysimage_path()) --project=$project - --compile=all --trace-compile=$tracefile -e $julia_code` - @debug "run_precompilation_script: running $cmd" - run(cmd) - return tracefile -end +all_stdlibs() = readdir(Sys.STDLIB) + +function rewrite_sysimg_jl_only_needed_stdlibs(stdlibs::Vector{String}) + sysimg_source_path = Base.find_source_file("sysimg.jl") + sysimg_content = read(sysimg_source_path, String) + # replaces the hardcoded list of stdlibs in sysimg.jl with + # the stdlibs that is given as argument + return replace(sysimg_content, r"stdlibs = \[(.*?)\]"s => string("stdlibs = [", join(":" .* string.(stdlibs), ",\n"), "]")) +end -function bootstrap_sys() - tmp = mktempdir() +function create_fresh_base_sysimage(stdlibs::Vector{String}) + tmp = mktempdir(cleanup=false) sysimg_source_path = Base.find_source_file("sysimg.jl") base_dir = dirname(sysimg_source_path) tmp_corecompiler_ji = joinpath(tmp, "corecompiler.ji") @@ -60,14 +59,38 @@ function bootstrap_sys() read(cmd) # Use that to create sys.ji - cmd = `$(get_julia_cmd()) --sysimage=$tmp_corecompiler_ji --output-ji=$tmp_sys_ji $sysimg_source_path` - @debug "running $cmd" - read(cmd) + new_sysimage_content = rewrite_sysimg_jl_only_needed_stdlibs(stdlibs) + new_sysimage_source_path = joinpath(base_dir, "sysimage_packagecompiler_x.jl") + write(new_sysimage_source_path, new_sysimage_content) + try + cmd = `$(get_julia_cmd()) --sysimage=$tmp_corecompiler_ji -g1 -O0 --output-ji=$tmp_sys_ji $new_sysimage_source_path` + @debug "running $cmd" + read(cmd) + finally + rm(new_sysimage_source_path; force=true) + end end return tmp_sys_ji end +# TODO: Add output file? +function run_precompilation_script(project::String, precompile_file::Union{String, Nothing}) + # TODO: Audit tempname usage + tracefile = tempname() + if precompile_file == nothing + arg = `-e ''` + else + arg = `$precompile_file` + end + touch(tracefile) + cmd = `$(get_julia_cmd()) --sysimage=$(current_process_sysimage_path()) --project=$project + --compile=all --trace-compile=$tracefile $arg` + @debug "run_precompilation_script: running $cmd" + run(cmd) + return tracefile +end + function create_sysimg_object_file(object_file::String, packages::Union{Symbol, Vector{Symbol}}; project::String, base_sysimg::String, @@ -86,41 +109,37 @@ function create_sysimg_object_file(object_file::String, packages::Union{Symbol, end # handle precompilation - if precompile_execution_file !== nothing || precompile_statements_file !== nothing - precompile_statements = "" - if precompile_execution_file !== nothing - @debug "running precompilation execution script..." - tracefile = run_precompilation_script(project, precompile_execution_file) - precompile_statements *= "append!(precompile_statements, readlines($(repr(tracefile))))\n" - end - if precompile_statements_file != nothing - precompile_statements *= "append!(precompile_statements, readlines($(repr(precompile_statements_file))))\n" - end + precompile_statements = "" + @debug "running precompilation execution script..." + tracefile = run_precompilation_script(project, precompile_execution_file) + precompile_statements *= "append!(precompile_statements, readlines($(repr(tracefile))))\n" + if precompile_statements_file != nothing + precompile_statements *= "append!(precompile_statements, readlines($(repr(precompile_statements_file))))\n" + end - precompile_code = """ - # This @eval prevents symbols from being put into Main - @eval Module() begin - PrecompileStagingArea = Module() - for (_pkgid, _mod) in Base.loaded_modules - if !(_pkgid.name in ("Main", "Core", "Base")) - eval(PrecompileStagingArea, :(const \$(Symbol(_mod)) = \$_mod)) - end + precompile_code = """ + # This @eval prevents symbols from being put into Main + @eval Module() begin + PrecompileStagingArea = Module() + for (_pkgid, _mod) in Base.loaded_modules + if !(_pkgid.name in ("Main", "Core", "Base")) + eval(PrecompileStagingArea, :(const \$(Symbol(_mod)) = \$_mod)) end - precompile_statements = String[] - $precompile_statements - for statement in sort(precompile_statements) - # println(statement) - try - Base.include_string(PrecompileStagingArea, statement) - catch - # See julia issue #28808 - @error "failed to execute \$statement" - end + end + precompile_statements = String[] + $precompile_statements + for statement in sort(precompile_statements) + # println(statement) + try + Base.include_string(PrecompileStagingArea, statement) + catch + # See julia issue #28808 + @error "failed to execute \$statement" end - end # module - """ - julia_code *= precompile_code - end + end + end # module + """ + julia_code *= precompile_code # finally, make julia output the resulting object file @debug "creating object file at $object_file" @@ -135,12 +154,28 @@ default_sysimg_name() = basename(default_sysimg_path()) backup_default_sysimg_path() = default_sysimg_path() * ".backup" backup_default_sysimg_name() = basename(backup_default_sysimg_path()) +# TODO: Also check UUIDs for stdlibs, not only names +function gather_stdlibs_project(project::String) + project_toml_path = abspath(Pkg.Types.projectfile_path(project; strict=true)) + ctx = Pkg.Types.Context(env=Pkg.Types.EnvCache(project_toml_path)) + @assert ctx.env.manifest !== nothing + stdlibs = all_stdlibs() + stdlibs_project = String[] + for (uuid, pkg) in ctx.env.manifest + if pkg.name in stdlibs + push!(stdlibs_project, pkg.name) + end + end + return stdlibs_project +end + function create_sysimage(packages::Union{Symbol, Vector{Symbol}}=Symbol[]; sysimage_path::Union{String,Nothing}=nothing, project::String=active_project(), precompile_execution_file::Union{String, Nothing}=nothing, precompile_statements_file::Union{String, Nothing}=nothing, - incremental=true, + incremental::Bool=true, + filter_stdlibs=false, replace_default_sysimg::Bool=false) if sysimage_path === nothing && replace_default_sysimg == false error("`sysimage_path` cannot be `nothing` if `replace_default_sysimg` is `false`") @@ -150,8 +185,17 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}=Symbol[]; sysimage_path = joinpath(tmp, string("sys.", Libdl.dlext)) end + if filter_stdlibs && incremental + error("must use `incremental=false` to use `filter_stdlibs=true`") + end + if !incremental - base_sysimg = bootstrap_sys() + if filter_stdlibs + stdlibs = gather_stdlibs_project(project) + else + stdlibs= all_stdlibs() + end + base_sysimg = create_fresh_base_sysimage(stdlibs) else base_sysimg = current_process_sysimage_path() end @@ -239,6 +283,7 @@ function create_app(package_dir::String, precompile_execution_file::Union{String,Nothing}=nothing, precompile_statements_file::Union{String,Nothing}=nothing, incremental=false, + filter_stdlibs=false, audit=true, force=false) project_toml_path = abspath(Pkg.Types.projectfile_path(package_dir; strict=true)) @@ -277,7 +322,8 @@ function create_app(package_dir::String, # TODO: Create in a temp dir and then move it into place? # TODO: Maybe avoid this cd? cd(app_dir) do - create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=project_path, incremental=incremental) + create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=project_path, + incremental=incremental, filter_stdlibs=filter_stdlibs) mkpath("bin") create_executable_from_sysimg(; sysimage_path=sysimg_file, executable_path=joinpath("bin", app_name)) mv(sysimg_file, joinpath("bin", sysimg_file)) diff --git a/test/runtests.jl b/test/runtests.jl index 379a575a..ae12cfca 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -17,10 +17,22 @@ using Libdl @test_logs PackageCompilerX.audit_app(app_source_dir) app_exe_dir = joinpath(tmp, "MyApp") for incremental in (true, false) - create_app(app_source_dir, app_exe_dir; incremental=incremental, force=true) - app_path = abspath(app_exe_dir, "bin", "MyApp" * (Sys.iswindows() ? ".exe" : "")) - app_output = read(`$app_path`, String) - @test occursin("Example.domath", app_output) - @test occursin("ἔοικα γοῦν τούτου", app_output) + if incremental == false + filter_stdlibs = (true, false) + else + filter_stdlibs = (false,) + end + for filter in filter_stdlibs + create_app(app_source_dir, app_exe_dir; incremental=incremental, force=true, filter_stdlibs=filter) + app_path = abspath(app_exe_dir, "bin", "MyApp" * (Sys.iswindows() ? ".exe" : "")) + app_output = read(`$app_path`, String) + if filter == true + @test !(occursin("LinearAlgebra", app_output)) + else + @test occursin("LinearAlgebra", app_output) + end + @test occursin("Example.domath", app_output) + @test occursin("ἔοικα γοῦν τούτου", app_output) + end end end From 5f0a90ee3f3bfb76c937dd50025cbfeee6b82016 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sat, 30 Nov 2019 17:51:03 +0100 Subject: [PATCH 18/80] try not use rpath to local julia install on linux (#23) Don't add an rpath to local absolute path --- src/PackageCompilerX.jl | 19 +++++++++---------- src/juliaconfig.jl | 9 +++++---- test/runtests.jl | 2 ++ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index d86d4a06..f470a387 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -144,6 +144,7 @@ function create_sysimg_object_file(object_file::String, packages::Union{Symbol, # finally, make julia output the resulting object file @debug "creating object file at $object_file" @info "PackageCompilerX: creating system image object file, this might take a while..." + cmd = `$(get_julia_cmd()) --sysimage=$base_sysimg --project=$project --output-o=$(object_file) -e $julia_code` @debug "running $cmd" run(cmd) @@ -321,18 +322,16 @@ function create_app(package_dir::String, # TODO: Create in a temp dir and then move it into place? # TODO: Maybe avoid this cd? - cd(app_dir) do - create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=project_path, + binpath = joinpath(app_dir, "bin") + mkpath(binpath) + cd(binpath) do + create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=project_path, incremental=incremental, filter_stdlibs=filter_stdlibs) - mkpath("bin") - create_executable_from_sysimg(; sysimage_path=sysimg_file, executable_path=joinpath("bin", app_name)) - mv(sysimg_file, joinpath("bin", sysimg_file)) + create_executable_from_sysimg(; sysimage_path=sysimg_file, executable_path=app_name) if Sys.isapple() - cd("bin") do - cmd = `install_name_tool -change $sysimg_file @rpath/$sysimg_file $app_name` - @debug "running $cmd" - run(cmd) - end + cmd = `install_name_tool -change $sysimg_file @rpath/$sysimg_file $app_name` + @debug "running $cmd" + run(cmd) end end return diff --git a/src/juliaconfig.jl b/src/juliaconfig.jl index 0c86a168..dda5bc95 100644 --- a/src/juliaconfig.jl +++ b/src/juliaconfig.jl @@ -41,11 +41,12 @@ function ldlibs(relative_path=nothing) else "julia" end - if Sys.isunix() - # TODO, these should not be needed since it adds an rpath to the current julia process libraries. - return "-Wl,-rpath,$(shell_escape(julia_libdir())) -Wl,-rpath,$(shell_escape(julia_private_libdir())) -l$libname" - else + if Sys.islinux() + return "-Wl,-rpath-link,$(shell_escape(julia_libdir())) -Wl,-rpath-link,$(shell_escape(julia_private_libdir())) -l$libname" + elseif Sys.iswindows() return "-l$libname -lopenlibm" + else + return "-l$libname" end end diff --git a/test/runtests.jl b/test/runtests.jl index ae12cfca..8b3a808c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,6 +2,8 @@ using PackageCompilerX: PackageCompilerX, create_sysimage, create_app using Test using Libdl +ENV["JULIA_DEBUG"] = "PackageCompilerX" + @testset "PackageCompilerX.jl" begin tmp = mktempdir() sysimage_path = joinpath(tmp, "sys." * Libdl.dlext) From 76ed66c465ab99238adb31dad1e82dd5b859818f Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 2 Dec 2019 17:11:23 +0100 Subject: [PATCH 19/80] Initial work on docs (#34) --- Project.toml | 3 +- docs/Manifest.toml | 11 +- docs/make.jl | 15 ++- docs/src/apps.md | 34 ++++++ docs/src/examples/ohmyrepl.md | 4 +- docs/src/index.md | 18 +++- docs/src/prereq.md | 7 ++ docs/src/refs.md | 8 ++ docs/src/sysimages.md | 191 +++++++++++++++++++++++++++++++++ src/PackageCompilerX.jl | 107 ++++++++++++------ test/precompile_statements2.jl | 1 + test/runtests.jl | 3 +- 12 files changed, 351 insertions(+), 51 deletions(-) create mode 100644 docs/src/apps.md create mode 100644 docs/src/prereq.md create mode 100644 docs/src/refs.md create mode 100644 docs/src/sysimages.md create mode 100644 test/precompile_statements2.jl diff --git a/Project.toml b/Project.toml index fc276005..5e5c1513 100644 --- a/Project.toml +++ b/Project.toml @@ -4,12 +4,13 @@ authors = ["KristofferC "] version = "0.1.0" [deps] +DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] -julia = "1" +julia = "1.3" [extras] Example = "7876af07-990d-54b4-ab0e-23690620f79a" diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 4a24f822..6564d5df 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -25,11 +25,6 @@ repo-url = "https://github.com/JuliaDocs/Documenter.jl.git" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" version = "0.24.0-DEV" -[[Example]] -git-tree-sha1 = "46e44e869b4d90b96bd8ed1fdcf32244fddfb6cc" -uuid = "7876af07-990d-54b4-ab0e-23690620f79a" -version = "0.5.3" - [[InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" @@ -57,16 +52,16 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[PackageCompilerX]] -deps = ["Example", "Libdl"] +deps = ["DocStringExtensions", "Libdl", "Pkg", "UUIDs"] path = ".." uuid = "dffaa6cc-da53-48e5-b007-4292dfcc27f1" version = "0.1.0" [[Parsers]] deps = ["Dates", "Test"] -git-tree-sha1 = "ef0af6c8601db18c282d092ccbd2f01f3f0cd70b" +git-tree-sha1 = "0139ba59ce9bc680e2925aec5b7db79065d60556" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "0.3.7" +version = "0.3.10" [[Pkg]] deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] diff --git a/docs/make.jl b/docs/make.jl index 67b4a8b4..74a08a53 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,12 +1,23 @@ using Documenter, PackageCompilerX makedocs( + format = Documenter.HTML( + # prettyurls on travis + prettyurls = haskey(ENV, "HAS_JOSH_K_SEAL_OF_APPROVAL"), + ), sitename = "PackageCompilerX", pages = Any[ "Home" => "index.md", + "Manual" => [ + "prereq.md" + "sysimages.md" + "apps.md" + ], + "Examples" => Any[ - "examples/ohmyrepl.md" - ] + "examples/ohmyrepl.md", + ], + "References" => "refs.md", ] ) diff --git a/docs/src/apps.md b/docs/src/apps.md new file mode 100644 index 00000000..85140f3c --- /dev/null +++ b/docs/src/apps.md @@ -0,0 +1,34 @@ +# Apps + +## Relocatability + + +### Artifacts + +PackageCompilerX provdes a function `audit_app(project::String)[@ref]` that tries to find common problems + +## Incremental vs non-incremental sysimage + +Creating a sysimage can in PackageCompilerX either be done "from scratch" (`incremental=false`) or it can +be done as a + + +### Incremental vs non-incremental sysimages + +By default, when creating a sysimage with PackageCompilerX, the sysimage is created in "incremental"-mode. +This means that the +This has the benefit that + + +## Standard library filtering + +As an example, + + +## What things are being leaked + +### Absolute paths of build machine + +### Lowered code + +### Name and fieldname of types diff --git a/docs/src/examples/ohmyrepl.md b/docs/src/examples/ohmyrepl.md index 2fe85503..06ea6d58 100644 --- a/docs/src/examples/ohmyrepl.md +++ b/docs/src/examples/ohmyrepl.md @@ -1,4 +1,4 @@ -# Creating a sysimage with OhMyREPL +# Creating a sysimage with OhMyREPL [OhMyREPL.jl](https://github.com/KristofferC/OhMyREPL.jl) is a package that enhances the REPL with, for example, syntax highlighting. It does, however, come with a bit of a startup time increase @@ -35,7 +35,7 @@ This are functions that Julia compile. We now just tell `create_sysimage` to use when creating the system image: ```jl -PackageCompilerX.create_sysimage(:OhMyREPL; precompile_statements_file="ohmyrepl_precompile", replace_default_sysimage=true) +PackageCompilerX.create_sysimage(:OhMyREPL; precompile_statements_file="ohmyrepl_precompile", replace_default=true) ``` Restart julia and start typing something. If everything went well you should see the type text become highlighted with a significantly smaller delay than before creating the new system image diff --git a/docs/src/index.md b/docs/src/index.md index 198fd127..05ebd52c 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,12 +1,26 @@ # PackageCompilerX -Compile packages for reduced latency and +PackageCompilerX is a Julia package with two main purposes: + +1. Creating custom sysimages for reduced latency when working locally with + packages that has a high startup time. + +2. Creating "apps" which are a bundle of files including an executable that can + be sent and run on other machines without Julia being installed on that machine. + +The manual contains some uses of Linux commands like `ls` (`dir` in Windows) +and `cat` but hopefully these commands are common enough that the points still +come across). + ## Manual Outline ```@contents Pages = [ - "examples/ohmyrepl.md" + "prereq.md", + "sysimages.md", + "apps.md", + "examples/ohmyrepl.md", ] Depth = 1 ``` diff --git a/docs/src/prereq.md b/docs/src/prereq.md new file mode 100644 index 00000000..9f9a17ec --- /dev/null +++ b/docs/src/prereq.md @@ -0,0 +1,7 @@ +# Prerequisites + +In order to use this package you need either `gcc` or `clang` installed and on the path. + +The package currently only works on Linux and Windows 64 bit. + +In the future, we hope to avoid this by automatically providing a working compiler with the package. diff --git a/docs/src/refs.md b/docs/src/refs.md new file mode 100644 index 00000000..edfc6056 --- /dev/null +++ b/docs/src/refs.md @@ -0,0 +1,8 @@ +# References + +```@docs +PackageCompilerX.create_sysimage +PackageCompilerX.restore_default_sysimg +PackageCompilerX.create_app +PackageCompilerX.audit_app +``` diff --git a/docs/src/sysimages.md b/docs/src/sysimages.md new file mode 100644 index 00000000..7b0d7f16 --- /dev/null +++ b/docs/src/sysimages.md @@ -0,0 +1,191 @@ +# Sysimages + +## What is a sysimage + +A sysimage is a file which, in a loose sense, contains a Julia session +serialized to a file. A "Julia session" include things like loaded packages, +global variables, inferred and compiled code, etc. By starting Julia with a +sysimage, the stored Julia session is deserialized and loaded. The idea behind +the sysimage is that this deserialization is faster than having to reload +packages and recompile code from scratch. + +Julia ships with a sysimage that is used by default when Julia is started. That +sysimage contains the Julia compiler itself, the standard libraries and also +compiled code (precompile statements) that has been put there to reduce the +time required to do common operations, like working in the REPL. + +Sometimes, it is desirable to create a custom sysimage with custom precompile +statements. This is the case if one has some dependencies that take a +significant time to load or where the compilation time for the first call is +uncomfortably long. This section of the documentation is intended to document +how to use PackageCompilerX to create such sysimages. + +### Drawbacks to custom sysimages + +It should be clearly stated that there are some drawbacks to using a custom +sysimage, thereby sidestepping the standard Julia package precompilation +system. The biggest drawback is that packages that are compiled into a +sysimage (including their dependencies!) are "locked" to the version they where +at when the sysimage was ( created. This means that no matter what package +version you have installed in your current project, the one in the sysimage +will take precedence. This can lead to bugs where you start with a project that +needs a specific version of a package, but you have another one compiled into +the sysimage. + +Putting packages in the sysimage is therefore only recommended if the load time +of the packages getting put in there is a significant problem and that these +are not frequently updated. In addition, compiling "workflow packages" like +Revise.jl and OhMyREPL.jl might make sense. + +## Creating a sysimage using PackageCompilerX + +PackageCompilerX provides the function [`create_sysimage`](@ref) to create a +sysimage. It takes as the first argument a package or a list of packages that +should be embedded in the resulting sysimage. By default, the given packages are +loaded from the active project but a specific project can be specified by +giving a path with the `project` keyword. The location of the resulting +sysimage is given by the `sysimage_path` keyword. After the sysimage is +created, giving the command flag `-Jpath/to/sysimage` will start Julia with the +given sysimage. + +As an example, below, a new sysimage in a separate project is created with the +package Example.jl in it. Using `Base.loaded_modules` it can be seen that the +package is loaded without having to explicitly `import` it. +``` +~ +❯ mkdir NewSysImageEnv + +~ +❯ cd NewSysImageEnv + +~/NewSysImageEnv 29s +❯ julia -q + +julia> using PackageCompilerX +[ Info: Precompiling PackageCompilerX [dffaa6cc-da53-48e5-b007-4292dfcc27f1] + +(v1.3) pkg> activate . +Activating new environment at `~/NewSysImageEnv/Project.toml` + +(NewSysImageEnv) pkg> add Example + Updating registry at `~/.julia/registries/General` + Updating git-repo `https://github.com/JuliaRegistries/General.git` + Resolving package versions... + Updating `~/NewSysImageEnv/Project.toml` + [7876af07] + Example v0.5.3 + Updating `~/NewSysImageEnv/Manifest.toml` + [7876af07] + Example v0.5.3 + +julia> create_sysimage(:Example; sysimage_path="ExampleSysimage.so") +[ Info: PackageCompilerX: creating system image object file, this might take a while... + +julia> exit() + +~/NewSysImageEnv +❯ ls +ExampleSysimage.so Manifest.toml Project.toml + +~/NewSysImageEnv +❯ julia -q -JExampleSysimage.so + +julia> Base.loaded_modules +Dict{Base.PkgId,Module} with 34 entries: +... + Example [7876af07-990d-54b4-ab0e-23690620f79a] => Example +... +``` + +Alternatively, instead of giving a path to where the new sysimage should appear, one +can choose to replace the default sysimage. +This is done by omitting the `sysimage_path` keyword and instead adding `replace_default=true`, for example: + +``` +create_sysimage([:Debugger, :OhMyREPL]; replace_default=true) +``` + +If this is the first time `create_sysimage` is called with `replace_default`, a +backup of the default sysimage is created. The default sysimage can then be +restored with [`restore_default_sysimg()`](@ref). + +Note that sysimages are created "incrementally" in the sense that they add to +the sysimage of the process running PackageCompilerX. If the default sysimage +has been replaced, the next `create_sysimage` call will create a new sysimage +based on the replaced sysimage. It is possible to create a sysimage +non-incrementally by passing the `incremental=false` keyword. This will create +a new system image from scratch, however, it will lose the special +precompilation that the Julia bundled sysimage provides which is what make the +REPL and package manager snappy. It is therefore unlikely that +`incremental=false` is of much use unless in special cases for sysimage +creation (for apps it is a different story though). + +### Precompilation + +The step where we included Example.jl in the sysimage meant that loading +Example is now pretty much instant (the package is already loaded when Julia +starts). However, functions inside Example.jl still need to be compiled when +executed for the first time. One way we can see this is by using the +`--trace-compile=stderr` flag which outputs a "precompile statement" every +time Julia compiles a function. Running the `hello` function inside Example.jl +we can see that it needs to be compiled (it shows the function +`Example.hello` was compiled for the input type `String`. + +``` +~/NewSysImageEnv +❯ julia -JExampleSysimage.so --trace-compile=stderr -e 'import Example; Example.hello("friend")' +precompile(Tuple{typeof(Example.hello), String}) +``` + +To remedy this, we can give a "precompile script" to `create_sysimage` which +causes functions executed in that script to be baked into the sysimage. As an +example, the script below simply calls the `hello` function in `Example`: + +``` +~/NewSysImageEnv +❯ cat precompile_example.jl +using Example +Example.hello("friend") +``` + +We now create a new system image called `ExampleSysimagePrecompile.so` where +the `precompile_statements_file` keyword argument has been giving, pointing to +the file just shown above: + +``` +~/NewSysImageEnv +❯ julia-q + +julia> using PackageCompilerX + +(v1.3) pkg> activate . +Activating environment at `~/NewSysImageEnv/Project.toml` + +julia> PackageCompilerX.create_sysimage(:Example; sysimage_path="ExampleSysimagePrecompile.so", + precompile_statements_file="precompile_example.jl") +[ Info: PackageCompilerX: creating system image object file, this might take a while... + +julia> exit() +``` + +Using the just created system image, we can see that the `hello` function no longer needs to get compiled_: + +``` +~/NewSysImageEnv +❯ julia -JExampleSysimagePrecompile.so --trace-compile=stderr -e 'import Example; Example.hello("friend")' + +~/NewSysImageEnv +❯ +``` + +### Using a manually generated list of precompile statements + +Starting Julia with `--trace-compile=file.jl` will emit precompilation +statements to `file.jl` for the duration of the started Julia process. This +can be useful in cases where it is difficult to give a script that executes the +code (like with interactive use). A file with a list of such precompile +statements can be used when creating a sysimage by passing the keyword argument +`precompile_statements_file`. See the OhMyREPL.jl example in the docs for more +details on how to use `--trace-compile` with PackageCompilerX. + +It is also possible to use [SnoopCompile.jl][snoop-url] to create files with precompilation statements. + +[snoop-url]: https://timholy.github.io/SnoopCompile.jl/stable/snoopi/#auto-1 diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index f470a387..5b13a4f2 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -1,5 +1,6 @@ module PackageCompilerX + # TODO: Add good debugging statements # TODO: sysimage or sysimg... @@ -7,19 +8,25 @@ using Base: active_project using Libdl: Libdl using Pkg: Pkg using UUIDs: UUID +using DocStringExtensions: SIGNATURES, TYPEDEF -export create_sysimage, create_app - -if isdefined(Pkg, :Artifacts) - const SUPPORTS_ARTIFACTS = true -else - const SUPPORTS_ARTIFACTS = false -end +export create_sysimage, create_app, audit_app, restore_default_sysimg include("juliaconfig.jl") # TODO: Check more carefully how to just use mingw on windows without using cygwin. -const CC = (Sys.iswindows() ? `x86_64-w64-mingw32-gcc` : `gcc`) +function get_compiler() + if Sys.iswindows() + return `x86_64-w64-mingw32-gcc` + else + if Sys.which("gcc") !== nothing + return `gcc` + elseif Sys.which("clang") !== nothing + return `clang` + end + error("could not find a compiler, looked for `gcc` and `clang`") + end +end # TODO: Be able to set target for -C? # TODO: Change to commented default ? @@ -32,7 +39,6 @@ function get_julia_cmd() cmd = `$julia_path --color=yes --startup-file=no --cpu-target=$DEFAULT_TARGET` end - all_stdlibs() = readdir(Sys.STDLIB) function rewrite_sysimg_jl_only_needed_stdlibs(stdlibs::Vector{String}) @@ -91,13 +97,12 @@ function run_precompilation_script(project::String, precompile_file::Union{Strin return tracefile end -function create_sysimg_object_file(object_file::String, packages::Union{Symbol, Vector{Symbol}}; +function create_sysimg_object_file(object_file::String, packages::Vector{Symbol}; project::String, base_sysimg::String, - precompile_execution_file::Union{String, Nothing}, - precompile_statements_file::Union{String, Nothing}) + precompile_execution_file::Union{Vector{String}, Nothing}, + precompile_statements_file::Union{Vector{String}, Nothing}) # include all packages into the sysimg - packages = vcat(packages) julia_code = """ if !isdefined(Base, :uv_eventloop) Base.reinit_stdio() @@ -111,10 +116,16 @@ function create_sysimg_object_file(object_file::String, packages::Union{Symbol, # handle precompilation precompile_statements = "" @debug "running precompilation execution script..." - tracefile = run_precompilation_script(project, precompile_execution_file) - precompile_statements *= "append!(precompile_statements, readlines($(repr(tracefile))))\n" + tracefiles = String[] + for file in (precompile_execution_file === nothing ? (nothing,) : precompile_execution_file) + tracefile = run_precompilation_script(project, file) + precompile_statements *= "append!(precompile_statements, readlines($(repr(tracefile))))\n" + end if precompile_statements_file != nothing - precompile_statements *= "append!(precompile_statements, readlines($(repr(precompile_statements_file))))\n" + for file in precompile_statements_file + precompile_statements *= + "append!(precompile_statements, readlines($(repr(file))))\n" + end end precompile_code = """ @@ -170,18 +181,22 @@ function gather_stdlibs_project(project::String) return stdlibs_project end -function create_sysimage(packages::Union{Symbol, Vector{Symbol}}=Symbol[]; +""" + $SIGNATURES +""" +function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; sysimage_path::Union{String,Nothing}=nothing, project::String=active_project(), - precompile_execution_file::Union{String, Nothing}=nothing, - precompile_statements_file::Union{String, Nothing}=nothing, + precompile_execution_file::Union{String, Vector{String}, Nothing}=nothing, + precompile_statements_file::Union{String, Vector{String}, Nothing}=nothing, incremental::Bool=true, filter_stdlibs=false, - replace_default_sysimg::Bool=false) - if sysimage_path === nothing && replace_default_sysimg == false - error("`sysimage_path` cannot be `nothing` if `replace_default_sysimg` is `false`") - end + replace_defaut::Bool=false) if sysimage_path === nothing + if replace_defaut == false + error("`sysimage_path` cannot be `nothing` if `replace_defaut` is `false`") + end + # We will replace the default sysimage so just put it somewhere for now tmp = mktempdir() sysimage_path = joinpath(tmp, string("sys.", Libdl.dlext)) end @@ -190,6 +205,11 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}=Symbol[]; error("must use `incremental=false` to use `filter_stdlibs=true`") end + # Functions lower down handles precompilation file as arrays so convert here + packages = vcat(packages) + precompile_execution_file !== nothing && (precompile_execution_file = vcat(precompile_execution_file)) + precompile_statements_file !== nothing && (precompile_statements = vcat(precompile_statements_file)) + if !incremental if filter_stdlibs stdlibs = gather_stdlibs_project(project) @@ -208,7 +228,7 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}=Symbol[]; precompile_execution_file=precompile_execution_file, precompile_statements_file=precompile_statements_file) create_sysimg_from_object_file(object_file, sysimage_path) - if replace_default_sysimg + if replace_defaut if !isfile(backup_default_sysimg_path()) @debug "making a backup of default sysimg" cp(default_sysimg_path(), backup_default_sysimg_path()) @@ -230,10 +250,15 @@ function create_sysimg_from_object_file(input_object::String, sysimage_path::Str o_file = `-Wl,--whole-archive $input_object -Wl,--no-whole-archive` end extra = Sys.iswindows() ? `-Wl,--export-all-symbols` : `` - run(`$CC -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia $extra`) + run(`$(get_compiler()) -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia $extra`) return nothing end +""" + $SIGNATURES + +lalala +""" function restore_default_sysimg() if !isfile(backup_default_sysimg_path()) error("did not find a backup sysimg") @@ -247,8 +272,18 @@ end const REQUIRES = "Requires" => UUID("ae029012-a4dd-5104-9daa-d747884805df") # Check for things that might indicate that the app or dependencies -function audit_app(package_dir::String) - project_toml_path = abspath(Pkg.Types.projectfile_path(package_dir; strict=true)) +""" + $SIGNATURES + +Check for possible problems with regfards to relocatability at +the project at `project_dir`. + +!!! warning + This cannot guarantee that the project is free of relocatability problems, + it can only detect some known bad cases and warn about those. +""" +function audit_app(project_dir::String) + project_toml_path = abspath(Pkg.Types.projectfile_path(project_dir; strict=true)) ctx = Pkg.Types.Context(env=Pkg.Types.EnvCache(project_toml_path)) return audit_app(ctx) end @@ -279,10 +314,13 @@ function audit_app(ctx::Pkg.Types.Context) return end +""" + $SIGNATURES +""" function create_app(package_dir::String, app_dir::String; - precompile_execution_file::Union{String,Nothing}=nothing, - precompile_statements_file::Union{String,Nothing}=nothing, + precompile_execution_file::Union{String, Vector{String}, Nothing}=nothing, + precompile_statements_file::Union{String, Vector{String}, Nothing}=nothing, incremental=false, filter_stdlibs=false, audit=true, @@ -316,17 +354,16 @@ function create_app(package_dir::String, mkpath(app_dir) bundle_julia_libraries(app_dir) - if SUPPORTS_ARTIFACTS - bundle_artifacts(ctx, app_dir) - end + bundle_artifacts(ctx, app_dir) # TODO: Create in a temp dir and then move it into place? - # TODO: Maybe avoid this cd? binpath = joinpath(app_dir, "bin") mkpath(binpath) cd(binpath) do create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=project_path, - incremental=incremental, filter_stdlibs=filter_stdlibs) + incremental=incremental, filter_stdlibs=filter_stdlibs, + precompile_execution_file = precompile_execution_file, + precompile_statements_file = precompile_statements_file) create_executable_from_sysimg(; sysimage_path=sysimg_file, executable_path=app_name) if Sys.isapple() cmd = `install_name_tool -change $sysimg_file @rpath/$sysimg_file $app_name` @@ -351,7 +388,7 @@ function create_executable_from_sysimg(;sysimage_path::String, else rpath = `-Wl,-rpath,\$ORIGIN:\$ORIGIN/../lib` end - cmd = `$CC -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) -o $(executable_path) $(wrapper) $(sysimage_path) -O2 $rpath $flags` + cmd = `$(get_compiler()) -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) -o $(executable_path) $(wrapper) $(sysimage_path) -O2 $rpath $flags` @debug "running $cmd" run(cmd) return nothing diff --git a/test/precompile_statements2.jl b/test/precompile_statements2.jl new file mode 100644 index 00000000..1e598ca8 --- /dev/null +++ b/test/precompile_statements2.jl @@ -0,0 +1 @@ +precompile(Tuple{typeof(Base.peek), Base.IOStream}) diff --git a/test/runtests.jl b/test/runtests.jl index 8b3a808c..ab207c22 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -9,7 +9,8 @@ ENV["JULIA_DEBUG"] = "PackageCompilerX" sysimage_path = joinpath(tmp, "sys." * Libdl.dlext) create_sysimage(:Example; sysimage_path=sysimage_path, precompile_execution_file="precompile_execution.jl", - precompile_statements_file="precompile_statements.jl") + precompile_statements_file=["precompile_statements.jl", + "precompile_statements2.jl"]) run(`$(Base.julia_cmd()) -J $(sysimage_path) -e 'println(1337)'`) # Test creating an app From a3372bcf0e2b36acf0ab7aaacc6919d2895a41d4 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 2 Dec 2019 19:53:22 +0100 Subject: [PATCH 20/80] add docs for apps (#37) --- docs/src/apps.md | 261 ++++++++++++++++++++++++++++++++++++---- docs/src/index.md | 16 +-- src/PackageCompilerX.jl | 6 +- 3 files changed, 241 insertions(+), 42 deletions(-) diff --git a/docs/src/apps.md b/docs/src/apps.md index 85140f3c..a3dc668b 100644 --- a/docs/src/apps.md +++ b/docs/src/apps.md @@ -1,34 +1,247 @@ # Apps -## Relocatability - - -### Artifacts - -PackageCompilerX provdes a function `audit_app(project::String)[@ref]` that tries to find common problems - -## Incremental vs non-incremental sysimage +With an "app" we here mean a bundle of files where one of these files is an +executable and where the bundle can be sent to another machine while still allowing +the executable to run. -Creating a sysimage can in PackageCompilerX either be done "from scratch" (`incremental=false`) or it can -be done as a +Use cases for Julia-apps is for example when one wants to provide some kind of +functionality where the fact that it was written in Julia is just an +implementation detail and forcing the user to download and use Julia to run the +code would be a distraction. There is also no need to provide the original +Julia source code for apps since everything gets baked into the sysimage. -### Incremental vs non-incremental sysimages - -By default, when creating a sysimage with PackageCompilerX, the sysimage is created in "incremental"-mode. -This means that the -This has the benefit that - - -## Standard library filtering - -As an example, +## Relocatability +Since we want to send the app to other machines the app we create must be +"relocatable". With an app being "relocatable" we mean it does not rely on +specifics of the machine where the app was created. Relocatability is not an +absolute measure, most apps assume some properties of the machine they will run +on, like what operating system is installed and the presence of graphics +drivers if one want to show graphics. On the other hand, embedding things into +the app that is most likely unique to the machine, like absolute paths, means +that the application almost surely will not run properly on another machine. + +For something to be relocatable, everything that it depends on must also be +relocatable. In the case of an app, the app itself and all the Julia packages +it depends on must also relocatable. This is a bit of an issue because the +Julia package ecosystem has not thought much about relocatability since app +making has not been common in the Julia community. + +The main problem with relocatability of Julia packages is that many packages +are encoding fundamentally non-relocatable *into the source code*. As an +example, many packages tend to use a `build.jl` file (which runs when the +package is first installed) that looks something like: + +``` +lib_path = find_library("libfoo") +write("deps.jl", "const LIBFOO_PATH = $(repr(lib_path))") +``` + +The main package file then contains + +``` +if !isfile("../build/deps.jl") + error("run Pkg.build(\"Package\") to re-build Package") +end +include("../build/deps.jl") +function __init__() + libfoo = Libdl.dlopen(LIBFOO_PATH) +end + +``` + +The absolute path to `lib_path` that `find_library` found is thus effectively +included into the source code of the package. Arguably, the whole build system +in Julia is inherently non-relocatable because it runs when the package is +being installed which is a concept that doesn't make sense when distributing an +app. + +Some packages do need to call into external libraries and use external binaries +so how are these packages supposed to do this in a relocatable way? The answer +is to use the "artifact system" which was described in the following [blog +post][artifact-blog-url]. The artifact system is a declarative way of +downloading and using "external files" like binaries and libraries. How this +is used in practice is described a bit later in this document. + + +## Creating an app + +The source of an app is a package with a project and manifest file. +It should define a function with the signature + +```jl +Base.@ccallable function julia_main()::Cint + ... +end +``` + +which will be the entry point of the app (the function that runs when the +executable in the app is run). A skeleton of an app to start working from can +be found at +https://github.com/KristofferC/PackageCompilerX.jl/tree/master/examples/MyApp. + +Regarding relocatability, PackageCompilerX provides a function +[`audit_app(app_dir::String)`](@ref) that tries to find common problems with +relocatability in the app. + +The app is then compiled using the [`create_app`](@ref) function that takes a +path to the source code of the app and the destination where the app should be +compiled to. This will bundle all required libraries for the app to run on +another machine where the same Julia that created the app could run. As an +example, below the example app linked above is compiled and run: + +``` +~/PackageCompilerX.jl/examples +❯ julia -q --project + +julia> using PackageCompilerX + +julia> create_app("MyApp/", "MyAppCompiled") +[ Info: PackageCompilerX: creating base system image (incremental=false), this might take a while... +[ Info: PackageCompilerX: creating system image object file, this might take a while... + +julia> exit() + +~/PackageCompilerX.jl/examples +❯ MyAppCompiled/bin/MyApp +ARGS = ["foo", "bar"] +Base.PROGRAM_FILE = "MyAppCompiled/bin/MyApp" +... +ἔοικα γοῦν τούτου γε σμικρῷ τινι αὐτῷ τούτῳ σοφώτερος εἶναι, ὅτι ἃ μὴ οἶδα οὐδὲ οἴομαι εἰδέναι. +unsafe_string((Base.JLOptions()).image_file) = "/home/kc/PackageCompilerX.jl/examples/MyAppCompiled/bin/MyApp.so" +Example.domath(5) = 10 +sin(0.0) = 0.0 +``` + +The resulting executable is found in the `bin` folder in the compiled app +directory. The compiled app directory `MyAppCompiled` could now be put into an +archive and sent to another machine or an installer could be wrapped around the +directory, perhaps providing a better user experience than just an archive of +files. + +### Precompilation + +In the same way as files for precompilation could be given when creating +sysimages, the same keyword arguments are used to add precompilation to apps. + +### Incremental vs non-incremental sysimage + +In the section about creating sysimages, there was a short discussion about +incremental vs non-incremental sysimages. In short, an incremental sysimage is +built on top of another sysimage while a non-incremental is created from +scratch. For sysimages, it made sense to use an incremental sysimage built on +top of Julia's default sysimage since we wanted the benefit of having a snappy +REPL that it provides. For apps, this is no longer the case, the sysimage is +not meant to be used when working interactively, it only needs to be +specialized for the specific app. Therefore, by default, `incremental=true` is +used for `create_app`. If, for some reason, one wants an incremental sysimage, +`incremental=true` could be passed to `create_app`. With the example app, a +non-incremental sysimage is about 70MB smaller than the default sysimage. + +### Filtering stdlibs + +By default, all standard libraries are included in the sysimage. It is +possible to only include those standard libraries that the project needs by +passing the keyword argument `filter_stdlibs=true` to `create_app`. This +causes the sysimage to be smaller, and possibly load faster. The reason this +is not the default is that it is possible to "accidentally" depend on a +standard library without it being reflected in the Project file. For example, +it is possible to call `rand()` from a package without depending on Random, +even though that is where it is defined. If Random was excluded from the +sysimage that call would then error. Same applies to matrix multiplication, +`rand(3,3) * rand(3,3)` requires both `LinearAlgebra` and `Random` This is +because these standard libraries do "type piracy" so just loading them can +cause code to change behavior. + +Nevertheless, the option is there to use, just make sure to properly test the +app with the resulting sysimage. -## What things are being leaked -### Absolute paths of build machine +### Artifacts -### Lowered code +The way to depend on external libraries or binaries when creating apps is by +using the [artifact system][artifact-pkg-url]. PackageCompilerX will bundle all +artifacts needed by the project, and set up things so that they can be found +during runtime on other machines. + +The example app uses the artifact system to depend on a very simple toy binary +that prints some greek text. It is instructive to see how the [artifact +file](https://github.com/KristofferC/PackageCompilerX.jl/blob/d12d8d9b2286bb7d57ca28ad0f2a8cd130c70c81/examples/MyApp/Artifacts.toml) +is [used in the source +code](https://github.com/KristofferC/PackageCompilerX.jl/blob/d12d8d9b2286bb7d57ca28ad0f2a8cd130c70c81/examples/MyApp/src/MyApp.jl#L4-L7) + +### What things are being leaked about the build machine and the source code + +While the created app is relocatable and no source code is bundled with it, +there are still some things about the build machine that can be observed. + +#### Absolute paths of build machine + +Julia records the paths and line-numbers for methods when they are getting +compiled. These get cached into the sysimage and can be found e.g. by dumping +all strings in the sysimage: + +``` +~/PackageCompilerX.jl/examples/MyAppCompiled/bin +❯ strings MyApp.so | grep MyApp +MyApp +/home/kc/PackageCompilerX.jl/examples/MyApp/ +MyApp +/home/kc/PackageCompilerX.jl/examples/MyApp/src/MyApp.jl +/home/kc/PackageCompilerX.jl/examples/MyApp/src +MyApp.jl +/home/kc/PackageCompilerX.jl/examples/MyApp/src/MyApp.jl +``` + +This is a problem that Julia itself has: + +``` +julia> @which rand() +rand() in Random at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.3/Random/src/Random.jl:256 +``` + +#### Using reflection and finding lowered code + +There is nothing preventing someone from starting Julia with the sysimage that +comes with the app. And while the source code is not available one can read +the "lowered code" and use reflection to find things like the name of fields in +structs and global variables etc: + +``` +~/PackageCompilerX.jl/examples/MyAppCompiled/bin kc/docs_apps* +❯ julia -q -JMyApp.so +julia> MyApp = Base.loaded_modules[Base.PkgId(Base.UUID("f943f3d7-887a-4ed5-b0c0-a1d6899aa8f5"), "MyApp")] +MyApp + +julia> names(MyApp; all=true) +10-element Array{Symbol,1}: + Symbol("#eval") + Symbol("#include") + Symbol("#julia_main") + Symbol("#real_main") + :MyApp + :eval + :include + :julia_main + :real_main + :socrates + +julia> @code_lowered MyApp.real_main() +CodeInfo( +1 ─ %1 = MyApp.ARGS +│ value@_2 = %1 +│ %3 = Base.repr(%1) +│ Base.println("ARGS = ", %3) +│ value@_2 +│ %6 = Base.PROGRAM_FILE +│ value@_3 = %6 +│ %8 = Base.repr(%6) +│ Base.println("Base.PROGRAM_FILE = ", %8) +│ value@_3 +│ %11 = MyApp.DEPOT_PATH +``` + +[artifact-blog-url]: https://julialang.org/blog/2019/11/artifacts +[artifact-pkg-url]: https://julialang.github.io/Pkg.jl/v1/artifacts/ -### Name and fieldname of types diff --git a/docs/src/index.md b/docs/src/index.md index 05ebd52c..dd5f783e 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -10,18 +10,4 @@ PackageCompilerX is a Julia package with two main purposes: The manual contains some uses of Linux commands like `ls` (`dir` in Windows) and `cat` but hopefully these commands are common enough that the points still -come across). - - -## Manual Outline - -```@contents -Pages = [ - "prereq.md", - "sysimages.md", - "apps.md", - "examples/ohmyrepl.md", -] -Depth = 1 -``` - +come across. diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 5b13a4f2..7c9bf888 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -276,14 +276,14 @@ const REQUIRES = "Requires" => UUID("ae029012-a4dd-5104-9daa-d747884805df") $SIGNATURES Check for possible problems with regfards to relocatability at -the project at `project_dir`. +the project at `app_dir`. !!! warning This cannot guarantee that the project is free of relocatability problems, it can only detect some known bad cases and warn about those. """ -function audit_app(project_dir::String) - project_toml_path = abspath(Pkg.Types.projectfile_path(project_dir; strict=true)) +function audit_app(app_dir::String) + project_toml_path = abspath(Pkg.Types.projectfile_path(app_dir; strict=true)) ctx = Pkg.Types.Context(env=Pkg.Types.EnvCache(project_toml_path)) return audit_app(ctx) end From 0ec0309ad2b59f7a9b2e8192553854aa6f8a477c Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 2 Dec 2019 20:47:39 +0100 Subject: [PATCH 21/80] fix support --- docs/src/prereq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/prereq.md b/docs/src/prereq.md index 9f9a17ec..82c6165e 100644 --- a/docs/src/prereq.md +++ b/docs/src/prereq.md @@ -2,6 +2,6 @@ In order to use this package you need either `gcc` or `clang` installed and on the path. -The package currently only works on Linux and Windows 64 bit. +The package currently only works on Linux and macOS with Windows support being a work in progress. In the future, we hope to avoid this by automatically providing a working compiler with the package. From 8692e968f0de0319dd78f51415c99ba9544459cc Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 3 Dec 2019 17:15:46 +0100 Subject: [PATCH 22/80] fix typo in docs --- docs/src/sysimages.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/sysimages.md b/docs/src/sysimages.md index 7b0d7f16..86537f88 100644 --- a/docs/src/sysimages.md +++ b/docs/src/sysimages.md @@ -147,7 +147,7 @@ Example.hello("friend") ``` We now create a new system image called `ExampleSysimagePrecompile.so` where -the `precompile_statements_file` keyword argument has been giving, pointing to +the `precompile_execution_file` keyword argument has been giving, pointing to the file just shown above: ``` @@ -160,7 +160,7 @@ julia> using PackageCompilerX Activating environment at `~/NewSysImageEnv/Project.toml` julia> PackageCompilerX.create_sysimage(:Example; sysimage_path="ExampleSysimagePrecompile.so", - precompile_statements_file="precompile_example.jl") + precompile_execution_file="precompile_example.jl") [ Info: PackageCompilerX: creating system image object file, this might take a while... julia> exit() From 8a8c6123857781d43becba1baec7623fcc5e1153 Mon Sep 17 00:00:00 2001 From: Colin Caine Date: Tue, 3 Dec 2019 18:00:28 +0000 Subject: [PATCH 23/80] docs: fix typo (#40) --- docs/src/examples/ohmyrepl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/examples/ohmyrepl.md b/docs/src/examples/ohmyrepl.md index 06ea6d58..8131f70e 100644 --- a/docs/src/examples/ohmyrepl.md +++ b/docs/src/examples/ohmyrepl.md @@ -31,7 +31,7 @@ precompile(Tuple{typeof(OhMyREPL.Prompt.insert_keybindings), Any}) precompile(Tuple{typeof(OhMyREPL.__init__)}) ``` -This are functions that Julia compile. We now just tell `create_sysimage` to use these precompile statements +These are functions that Julia compiled. We now just tell `create_sysimage` to use these precompile statements when creating the system image: ```jl From 2dac50721c8ac76ce0fb5e0a2274350fe8c88061 Mon Sep 17 00:00:00 2001 From: Rogerluo Date: Sun, 8 Dec 2019 03:37:40 -0500 Subject: [PATCH 24/80] fix a typo (#41) --- src/PackageCompilerX.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 7c9bf888..c421736d 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -191,10 +191,10 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; precompile_statements_file::Union{String, Vector{String}, Nothing}=nothing, incremental::Bool=true, filter_stdlibs=false, - replace_defaut::Bool=false) + replace_default::Bool=false) if sysimage_path === nothing - if replace_defaut == false - error("`sysimage_path` cannot be `nothing` if `replace_defaut` is `false`") + if replace_default == false + error("`sysimage_path` cannot be `nothing` if `replace_default` is `false`") end # We will replace the default sysimage so just put it somewhere for now tmp = mktempdir() @@ -228,7 +228,7 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; precompile_execution_file=precompile_execution_file, precompile_statements_file=precompile_statements_file) create_sysimg_from_object_file(object_file, sysimage_path) - if replace_defaut + if replace_default if !isfile(backup_default_sysimg_path()) @debug "making a backup of default sysimg" cp(default_sysimg_path(), backup_default_sysimg_path()) From c43f07c10ac94020d56a01304cdd9fbf6be528e0 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 9 Dec 2019 08:12:11 +0100 Subject: [PATCH 25/80] add a mention that this is still WIP --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b1906672..eee5056f 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ [![Codecov](https://codecov.io/gh/KristofferC/PackageCompilerX.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/KristofferC/PackageCompilerX.jl) [![][docs-stable-img]][docs-stable-url] +**This package is a work in progress. Its current state of functionality / docs should not be indicative of the quality of the finished package** + On Linux or Mac this package requires `gcc` to be installed and be on the `PATH`. On Windows, a cygwin setup needs to be installed as decribed [here](https://github.com/JuliaLang/julia/blob/master/doc/build/windows.md#cygwin-to-mingw-cross-compiling) and Julia need to be run in the cygwin environement. From 4cb1fd216ec4734beb3d1957a2fbf777db5e3e7c Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Wed, 11 Dec 2019 04:57:06 -0500 Subject: [PATCH 26/80] Create CompatHelper.yml (#48) --- .github/workflows/CompatHelper.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/CompatHelper.yml diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml new file mode 100644 index 00000000..68dbe39c --- /dev/null +++ b/.github/workflows/CompatHelper.yml @@ -0,0 +1,24 @@ +name: CompatHelper + +on: + schedule: + - cron: '00 * * * *' + +jobs: + CompatHelper: + runs-on: ${{ matrix.os }} + strategy: + matrix: + julia-version: [1.2.0] + julia-arch: [x86] + os: [ubuntu-latest] + steps: + - uses: julia-actions/setup-julia@latest + with: + version: ${{ matrix.julia-version }} + - name: Pkg.add("CompatHelper") + run: julia -e 'using Pkg; Pkg.add("CompatHelper")' + - name: CompatHelper.main() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: julia -e 'using CompatHelper; CompatHelper.main()' From 15eddce3144a34083c9368763217bf0b373c230f Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Wed, 11 Dec 2019 06:50:02 -0500 Subject: [PATCH 27/80] CompatHelper: only update root manifest (#50) --- .github/workflows/CompatHelper.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml index 68dbe39c..ad15d36f 100644 --- a/.github/workflows/CompatHelper.yml +++ b/.github/workflows/CompatHelper.yml @@ -21,4 +21,4 @@ jobs: - name: CompatHelper.main() env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: julia -e 'using CompatHelper; CompatHelper.main()' + run: julia -e 'using CompatHelper; CompatHelper.main( (; registries) -> CompatHelper._update_manifests(pwd(); registries = registries) )' From 8f7d453d2e10fb6c5680d9599bac581f93252c45 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 11 Dec 2019 13:13:05 +0100 Subject: [PATCH 28/80] improvements to cpu targets (#43) * improvements to cpu targets * more * more * remove precompile from testing due to https://github.com/JuliaLang/julia/issues/34076 --- src/PackageCompilerX.jl | 83 ++++++++++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 26 deletions(-) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index c421736d..b2bb2606 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -14,6 +14,13 @@ export create_sysimage, create_app, audit_app, restore_default_sysimg include("juliaconfig.jl") +const NATIVE_CPU_TARGET = "native" +const APP_CPU_TARGET = "generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)" + +current_process_sysimage_path() = unsafe_string(Base.JLOptions().image_file) + +all_stdlibs() = readdir(Sys.STDLIB) + # TODO: Check more carefully how to just use mingw on windows without using cygwin. function get_compiler() if Sys.iswindows() @@ -28,19 +35,11 @@ function get_compiler() end end -# TODO: Be able to set target for -C? -# TODO: Change to commented default ? -# const DEFAULT_TARGET = "generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)" -const DEFAULT_TARGET = "generic" -current_process_sysimage_path() = unsafe_string(Base.JLOptions().image_file) - function get_julia_cmd() julia_path = joinpath(Sys.BINDIR, Base.julia_exename()) - cmd = `$julia_path --color=yes --startup-file=no --cpu-target=$DEFAULT_TARGET` + cmd = `$julia_path --color=yes --startup-file=no` end -all_stdlibs() = readdir(Sys.STDLIB) - function rewrite_sysimg_jl_only_needed_stdlibs(stdlibs::Vector{String}) sysimg_source_path = Base.find_source_file("sysimg.jl") sysimg_content = read(sysimg_source_path, String) @@ -49,7 +48,7 @@ function rewrite_sysimg_jl_only_needed_stdlibs(stdlibs::Vector{String}) return replace(sysimg_content, r"stdlibs = \[(.*?)\]"s => string("stdlibs = [", join(":" .* string.(stdlibs), ",\n"), "]")) end -function create_fresh_base_sysimage(stdlibs::Vector{String}) +function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String) tmp = mktempdir(cleanup=false) sysimg_source_path = Base.find_source_file("sysimg.jl") base_dir = dirname(sysimg_source_path) @@ -57,10 +56,11 @@ function create_fresh_base_sysimage(stdlibs::Vector{String}) tmp_sys_ji = joinpath(tmp, "sys.ji") compiler_source_path = joinpath(base_dir, "compiler", "compiler.jl") - @info "PackageCompilerX: creating base system image (incremental=false), this might take a while..." + @info "PackageCompilerX: creating base system image (incremental=false)..." cd(base_dir) do # Create corecompiler.ji - cmd = `$(get_julia_cmd()) --output-ji $tmp_corecompiler_ji -g0 -O0 $compiler_source_path` + cmd = `$(get_julia_cmd()) --cpu-target $cpu_target --output-ji $tmp_corecompiler_ji + -g0 -O0 $compiler_source_path` @debug "running $cmd" read(cmd) @@ -69,7 +69,9 @@ function create_fresh_base_sysimage(stdlibs::Vector{String}) new_sysimage_source_path = joinpath(base_dir, "sysimage_packagecompiler_x.jl") write(new_sysimage_source_path, new_sysimage_content) try - cmd = `$(get_julia_cmd()) --sysimage=$tmp_corecompiler_ji -g1 -O0 --output-ji=$tmp_sys_ji $new_sysimage_source_path` + cmd = `$(get_julia_cmd()) --cpu-target $cpu_target + --sysimage=$tmp_corecompiler_ji + -g1 -O0 --output-ji=$tmp_sys_ji $new_sysimage_source_path` @debug "running $cmd" read(cmd) finally @@ -93,15 +95,16 @@ function run_precompilation_script(project::String, precompile_file::Union{Strin cmd = `$(get_julia_cmd()) --sysimage=$(current_process_sysimage_path()) --project=$project --compile=all --trace-compile=$tracefile $arg` @debug "run_precompilation_script: running $cmd" - run(cmd) + read(cmd) return tracefile end function create_sysimg_object_file(object_file::String, packages::Vector{Symbol}; project::String, - base_sysimg::String, + base_sysimage::String, precompile_execution_file::Union{Vector{String}, Nothing}, - precompile_statements_file::Union{Vector{String}, Nothing}) + precompile_statements_file::Union{Vector{String}, Nothing}, + cpu_target::String) # include all packages into the sysimg julia_code = """ if !isdefined(Base, :uv_eventloop) @@ -156,7 +159,7 @@ function create_sysimg_object_file(object_file::String, packages::Vector{Symbol} @debug "creating object file at $object_file" @info "PackageCompilerX: creating system image object file, this might take a while..." - cmd = `$(get_julia_cmd()) --sysimage=$base_sysimg --project=$project --output-o=$(object_file) -e $julia_code` + cmd = `$(get_julia_cmd()) --cpu-target=$cpu_target --sysimage=$base_sysimage --project=$project --output-o=$(object_file) -e $julia_code` @debug "running $cmd" run(cmd) end @@ -191,7 +194,9 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; precompile_statements_file::Union{String, Vector{String}, Nothing}=nothing, incremental::Bool=true, filter_stdlibs=false, - replace_default::Bool=false) + replace_default::Bool=false, + cpu_target::String=NATIVE_CPU_TARGET, + base_sysimage::Union{Nothing, String}=nothing) if sysimage_path === nothing if replace_default == false error("`sysimage_path` cannot be `nothing` if `replace_default` is `false`") @@ -211,22 +216,28 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; precompile_statements_file !== nothing && (precompile_statements = vcat(precompile_statements_file)) if !incremental + if base_sysimage !== nothing + error("cannot specify `base_sysimage` when `incremental=false`") + end if filter_stdlibs stdlibs = gather_stdlibs_project(project) else stdlibs= all_stdlibs() end - base_sysimg = create_fresh_base_sysimage(stdlibs) + base_sysimage = create_fresh_base_sysimage(stdlibs; cpu_target=cpu_target) else - base_sysimg = current_process_sysimage_path() + if base_sysimage == nothing + base_sysimage = current_process_sysimage_path() + end end object_file = tempname() * ".o" create_sysimg_object_file(object_file, packages; project=project, - base_sysimg=base_sysimg, + base_sysimage=base_sysimage, precompile_execution_file=precompile_execution_file, - precompile_statements_file=precompile_statements_file) + precompile_statements_file=precompile_statements_file, + cpu_target=cpu_target) create_sysimg_from_object_file(object_file, sysimage_path) if replace_default if !isfile(backup_default_sysimg_path()) @@ -360,10 +371,30 @@ function create_app(package_dir::String, binpath = joinpath(app_dir, "bin") mkpath(binpath) cd(binpath) do - create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=project_path, - incremental=incremental, filter_stdlibs=filter_stdlibs, - precompile_execution_file = precompile_execution_file, - precompile_statements_file = precompile_statements_file) + if !incremental + tmp = mktempdir() + # Use workaround at https://github.com/JuliaLang/julia/issues/34064#issuecomment-563950633 + # by first creating a normal "empty" sysimage and then use that to finally create the one + # with the @ccallable function + tmp_base_sysimage = joinpath(tmp, "tmp_sys.so") + create_sysimage(Symbol[]; sysimage_path=tmp_base_sysimage, project=project_path, + incremental=false, filter_stdlibs=filter_stdlibs, + cpu_target=APP_CPU_TARGET) + + create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=project_path, + incremental=true, + precompile_execution_file=precompile_execution_file, + precompile_statements_file=precompile_statements_file, + cpu_target=APP_CPU_TARGET, + base_sysimage=tmp_base_sysimage) + else + create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=project_path, + incremental=incremental, filter_stdlibs=filter_stdlibs, + precompile_execution_file=precompile_execution_file, + precompile_statements_file=precompile_statements_file, + cpu_target=APP_CPU_TARGET) + end + create_executable_from_sysimg(; sysimage_path=sysimg_file, executable_path=app_name) if Sys.isapple() cmd = `install_name_tool -change $sysimg_file @rpath/$sysimg_file $app_name` From a9d336a63277c62cc6298048e72bd088f2f92c3e Mon Sep 17 00:00:00 2001 From: KristofferC Date: Wed, 11 Dec 2019 16:45:06 +0100 Subject: [PATCH 29/80] fix some links --- docs/src/apps.md | 16 +++++++--------- docs/src/sysimages.md | 5 +++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/src/apps.md b/docs/src/apps.md index a3dc668b..1295b563 100644 --- a/docs/src/apps.md +++ b/docs/src/apps.md @@ -60,9 +60,10 @@ app. Some packages do need to call into external libraries and use external binaries so how are these packages supposed to do this in a relocatable way? The answer is to use the "artifact system" which was described in the following [blog -post][artifact-blog-url]. The artifact system is a declarative way of -downloading and using "external files" like binaries and libraries. How this -is used in practice is described a bit later in this document. +post](https://julialang.org/blog/2019/11/artifacts). The artifact system is a +declarative way of downloading and using "external files" like binaries and +libraries. How this is used in practice is described a bit later in this +document. ## Creating an app @@ -161,9 +162,9 @@ app with the resulting sysimage. ### Artifacts The way to depend on external libraries or binaries when creating apps is by -using the [artifact system][artifact-pkg-url]. PackageCompilerX will bundle all -artifacts needed by the project, and set up things so that they can be found -during runtime on other machines. +using the [artifact system](https://julialang.github.io/Pkg.jl/v1/artifacts/). +PackageCompilerX will bundle all artifacts needed by the project, and set up +things so that they can be found during runtime on other machines. The example app uses the artifact system to depend on a very simple toy binary that prints some greek text. It is instructive to see how the [artifact @@ -242,6 +243,3 @@ CodeInfo( │ %11 = MyApp.DEPOT_PATH ``` -[artifact-blog-url]: https://julialang.org/blog/2019/11/artifacts -[artifact-pkg-url]: https://julialang.github.io/Pkg.jl/v1/artifacts/ - diff --git a/docs/src/sysimages.md b/docs/src/sysimages.md index 86537f88..8723c452 100644 --- a/docs/src/sysimages.md +++ b/docs/src/sysimages.md @@ -186,6 +186,7 @@ statements can be used when creating a sysimage by passing the keyword argument `precompile_statements_file`. See the OhMyREPL.jl example in the docs for more details on how to use `--trace-compile` with PackageCompilerX. -It is also possible to use [SnoopCompile.jl][snoop-url] to create files with precompilation statements. +It is also possible to use +[SnoopCompile.jl](https://timholy.github.io/SnoopCompile.jl/stable/snoopi/#auto-1) +to create files with precompilation statements. -[snoop-url]: https://timholy.github.io/SnoopCompile.jl/stable/snoopi/#auto-1 From 403176680bc2b0eff49961fd66d8973815ade7e2 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 11 Dec 2019 22:25:49 +0100 Subject: [PATCH 30/80] WIP on making app work on windows. (#55) * wip make app work on windows * wip win * test artifact load path * realpath --- examples/MyApp/Artifacts.toml | 39 ++++++++++++++++++++++++----------- examples/MyApp/Manifest.toml | 8 +++++++ examples/MyApp/Project.toml | 1 + examples/MyApp/src/MyApp.jl | 19 +++++++++++++---- src/PackageCompilerX.jl | 7 ++++++- test/runtests.jl | 16 ++++++++++---- 6 files changed, 69 insertions(+), 21 deletions(-) diff --git a/examples/MyApp/Artifacts.toml b/examples/MyApp/Artifacts.toml index d90994d1..2b831cc3 100644 --- a/examples/MyApp/Artifacts.toml +++ b/examples/MyApp/Artifacts.toml @@ -1,12 +1,27 @@ -# Example Artifacts.toml file -[socrates] -git-tree-sha1 = "43563e7631a7eafae1f9f8d9d332e3de44ad7239" -lazy = false - - [[socrates.download]] - url = "https://github.com/staticfloat/small_bin/raw/master/socrates.tar.gz" - sha256 = "e65d2f13f2085f2c279830e863292312a72930fee5ba3c792b14c33ce5c5cc58" - - [[socrates.download]] - url = "https://github.com/staticfloat/small_bin/raw/master/socrates.tar.bz2" - sha256 = "13fc17b97be41763b02cbb80e9d048302cec3bd3d446c2ed6e8210bddcd3ac76" +[[fooifier]] +arch = "x86_64" +git-tree-sha1 = "98d93024ca384050c59d554415b75d61e467fd8c" +libc = "glibc" +os = "linux" + + [[fooifier.download]] + sha256 = "5208c63a9d07e592c78f541fc13caa8cd191b11e7e77b31d407237c2b13ec391" + url = "https://github.com/staticfloat/small_bin/raw/master/libfoo/libfoo.x86_64-linux-gnu.tar.gz" + +[[fooifier]] +arch = "x86_64" +git-tree-sha1 = "f413ff2438a4e9e9dd69b23c35ca30de6af069cc" +os = "macos" + + [[fooifier.download]] + sha256 = "fcc268772d6f21d65b45fcf3854a3142679b78e53c7673dac26c95d6ccc89a24" + url = "https://github.com/staticfloat/small_bin/raw/master/libfoo/libfoo.x86_64-apple-darwin14.tar.gz" + +[[fooifier]] +arch = "x86_64" +git-tree-sha1 = "d61f806c76b57e54f343634c5219d00d4c81b077" # FIX! +os = "windows" + + [[fooifier.download]] + sha256 = "7f8939e9529835b83810d3ae7e2556f6e002d571f619894e54ece42ea5262b7f" + url = "https://github.com/staticfloat/small_bin/raw/master/libfoo/libfoo.x86_64-w64-mingw32.tar.gz" diff --git a/examples/MyApp/Manifest.toml b/examples/MyApp/Manifest.toml index 7f04b86e..d590ae1d 100644 --- a/examples/MyApp/Manifest.toml +++ b/examples/MyApp/Manifest.toml @@ -12,6 +12,14 @@ git-tree-sha1 = "46e44e869b4d90b96bd8ed1fdcf32244fddfb6cc" uuid = "7876af07-990d-54b4-ab0e-23690620f79a" version = "0.5.3" +[[HelloWorldC_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "8912dbb81d8c8bd7117426a9072068d58c38d1f4" +repo-rev = "master" +repo-url = "https://github.com/JuliaBinaryWrappers/HelloWorldC_jll.jl" +uuid = "dca1746e-5efc-54fc-8249-22745bc95a49" +version = "1.0.6+1" + [[InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" diff --git a/examples/MyApp/Project.toml b/examples/MyApp/Project.toml index 8c061a07..db2daf45 100644 --- a/examples/MyApp/Project.toml +++ b/examples/MyApp/Project.toml @@ -5,4 +5,5 @@ version = "0.1.0" [deps] Example = "7876af07-990d-54b4-ab0e-23690620f79a" +HelloWorldC_jll = "dca1746e-5efc-54fc-8249-22745bc95a49" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" diff --git a/examples/MyApp/src/MyApp.jl b/examples/MyApp/src/MyApp.jl index 45502c43..b0d55ed9 100644 --- a/examples/MyApp/src/MyApp.jl +++ b/examples/MyApp/src/MyApp.jl @@ -1,10 +1,11 @@ module MyApp using Example +using HelloWorldC_jll using Pkg.Artifacts -socrates = joinpath(ensure_artifact_installed("socrates", joinpath(@__DIR__, "..", "Artifacts.toml")), - "bin", "socrates") +const fooifier = joinpath(ensure_artifact_installed("fooifier", joinpath(@__DIR__, "..", "Artifacts.toml")), + "bin", "fooifier" * (Sys.iswindows() ? ".exe" : "")) Base.@ccallable function julia_main()::Cint try @@ -26,8 +27,18 @@ function real_main() @show Threads.nthreads() @show Sys.BINDIR display(Base.loaded_modules) - println("Running an artifact:") - run(`$socrates`) + + println("Running a jll package:") + HelloWorldC_jll.hello_world() do x + println("HelloWorld artifact at $(realpath(x))") + run(`$x`) + end + println() + + println("Running the artifact") + res = read(`$fooifier 5 10`, String) + println("The result of 2*5^2 - 10 == $res") + println() @show unsafe_string(Base.JLOptions().image_file) @show Example.domath(5) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index b2bb2606..5b571151 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -24,7 +24,11 @@ all_stdlibs() = readdir(Sys.STDLIB) # TODO: Check more carefully how to just use mingw on windows without using cygwin. function get_compiler() if Sys.iswindows() - return `x86_64-w64-mingw32-gcc` + if Sys.which("gcc") !== nothing + return `gcc` + elseif Sys.which("x86_64-w64-mingw32-gcc") !== nothing + return `x86_64-w64-mingw32-gcc` + end else if Sys.which("gcc") !== nothing return `gcc` @@ -336,6 +340,7 @@ function create_app(package_dir::String, filter_stdlibs=false, audit=true, force=false) + # Need to check if nothing was returned below project_toml_path = abspath(Pkg.Types.projectfile_path(package_dir; strict=true)) manifest_toml_path = abspath(Pkg.Types.manifestfile_path(package_dir)) if manifest_toml_path === nothing diff --git a/test/runtests.jl b/test/runtests.jl index ab207c22..e0d1f7cb 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -18,7 +18,7 @@ ENV["JULIA_DEBUG"] = "PackageCompilerX" # TODO: Also test something that actually gives audit warnings @test_logs PackageCompilerX.audit_app(app_source_dir) - app_exe_dir = joinpath(tmp, "MyApp") + app_compiled_dir = joinpath(tmp, "MyApp") for incremental in (true, false) if incremental == false filter_stdlibs = (true, false) @@ -26,16 +26,24 @@ ENV["JULIA_DEBUG"] = "PackageCompilerX" filter_stdlibs = (false,) end for filter in filter_stdlibs - create_app(app_source_dir, app_exe_dir; incremental=incremental, force=true, filter_stdlibs=filter) - app_path = abspath(app_exe_dir, "bin", "MyApp" * (Sys.iswindows() ? ".exe" : "")) + create_app(app_source_dir, app_compiled_dir; incremental=incremental, force=true, filter_stdlibs=filter) + app_path = abspath(app_compiled_dir, "bin", "MyApp" * (Sys.iswindows() ? ".exe" : "")) app_output = read(`$app_path`, String) + + # Check stdlib filtering if filter == true @test !(occursin("LinearAlgebra", app_output)) else @test occursin("LinearAlgebra", app_output) end + # Check dependency run @test occursin("Example.domath", app_output) - @test occursin("ἔοικα γοῦν τούτου", app_output) + # Check jll package runs + @test occursin("Hello, World!", app_output) + # Check artifact runs + @test occursin("The result of 2*5^2 - 10 == 40.000000", app_output) + # Check artifact gets run from the correct place + @test occursin("HelloWorld artifact at $(realpath(app_compiled_dir))", app_output) end end end From ead31731c49a6042be44c9238d0ef30d6940b1dc Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 12 Dec 2019 09:48:36 +0100 Subject: [PATCH 31/80] workaround julia#34076 (#54) * workaround julia#34076 * fix typo --- examples/MyApp/precompile_app.jl | 4 ++++ src/PackageCompilerX.jl | 24 ++++++++++++++++-------- test/runtests.jl | 3 ++- 3 files changed, 22 insertions(+), 9 deletions(-) create mode 100644 examples/MyApp/precompile_app.jl diff --git a/examples/MyApp/precompile_app.jl b/examples/MyApp/precompile_app.jl new file mode 100644 index 00000000..d70278d6 --- /dev/null +++ b/examples/MyApp/precompile_app.jl @@ -0,0 +1,4 @@ +using MyApp + +push!(ARGS, "arg") +MyApp.julia_main() diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 5b571151..fc203e84 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -21,6 +21,8 @@ current_process_sysimage_path() = unsafe_string(Base.JLOptions().image_file) all_stdlibs() = readdir(Sys.STDLIB) +yesno(b::Bool) = b ? "yes" : "no" + # TODO: Check more carefully how to just use mingw on windows without using cygwin. function get_compiler() if Sys.iswindows() @@ -108,7 +110,8 @@ function create_sysimg_object_file(object_file::String, packages::Vector{Symbol} base_sysimage::String, precompile_execution_file::Union{Vector{String}, Nothing}, precompile_statements_file::Union{Vector{String}, Nothing}, - cpu_target::String) + cpu_target::String, + compiled_modules::Bool) # include all packages into the sysimg julia_code = """ if !isdefined(Base, :uv_eventloop) @@ -126,12 +129,12 @@ function create_sysimg_object_file(object_file::String, packages::Vector{Symbol} tracefiles = String[] for file in (precompile_execution_file === nothing ? (nothing,) : precompile_execution_file) tracefile = run_precompilation_script(project, file) - precompile_statements *= "append!(precompile_statements, readlines($(repr(tracefile))))\n" + precompile_statements *= " append!(precompile_statements, readlines($(repr(tracefile))))\n" end if precompile_statements_file != nothing for file in precompile_statements_file precompile_statements *= - "append!(precompile_statements, readlines($(repr(file))))\n" + " append!(precompile_statements, readlines($(repr(file))))\n" end end @@ -163,7 +166,8 @@ function create_sysimg_object_file(object_file::String, packages::Vector{Symbol} @debug "creating object file at $object_file" @info "PackageCompilerX: creating system image object file, this might take a while..." - cmd = `$(get_julia_cmd()) --cpu-target=$cpu_target --sysimage=$base_sysimage --project=$project --output-o=$(object_file) -e $julia_code` + cmd = `$(get_julia_cmd()) --compiled-modules=$(yesno(compiled_modules)) --cpu-target=$cpu_target + --sysimage=$base_sysimage --project=$project --output-o=$(object_file) -e $julia_code` @debug "running $cmd" run(cmd) end @@ -200,7 +204,8 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; filter_stdlibs=false, replace_default::Bool=false, cpu_target::String=NATIVE_CPU_TARGET, - base_sysimage::Union{Nothing, String}=nothing) + base_sysimage::Union{Nothing, String}=nothing, + compiled_modules=true) if sysimage_path === nothing if replace_default == false error("`sysimage_path` cannot be `nothing` if `replace_default` is `false`") @@ -241,7 +246,8 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; base_sysimage=base_sysimage, precompile_execution_file=precompile_execution_file, precompile_statements_file=precompile_statements_file, - cpu_target=cpu_target) + cpu_target=cpu_target, + compiled_modules=compiled_modules) create_sysimg_from_object_file(object_file, sysimage_path) if replace_default if !isfile(backup_default_sysimg_path()) @@ -391,13 +397,15 @@ function create_app(package_dir::String, precompile_execution_file=precompile_execution_file, precompile_statements_file=precompile_statements_file, cpu_target=APP_CPU_TARGET, - base_sysimage=tmp_base_sysimage) + base_sysimage=tmp_base_sysimage, + compiled_modules=false #= workaround julia#34076=#) else create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=project_path, incremental=incremental, filter_stdlibs=filter_stdlibs, precompile_execution_file=precompile_execution_file, precompile_statements_file=precompile_statements_file, - cpu_target=APP_CPU_TARGET) + cpu_target=APP_CPU_TARGET, + compiled_modules=false #= workaround julia#34076=#) end create_executable_from_sysimg(; sysimage_path=sysimg_file, executable_path=app_name) diff --git a/test/runtests.jl b/test/runtests.jl index e0d1f7cb..8889b9f6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -26,7 +26,8 @@ ENV["JULIA_DEBUG"] = "PackageCompilerX" filter_stdlibs = (false,) end for filter in filter_stdlibs - create_app(app_source_dir, app_compiled_dir; incremental=incremental, force=true, filter_stdlibs=filter) + create_app(app_source_dir, app_compiled_dir; incremental=incremental, force=true, filter_stdlibs=filter, + precompile_execution_file=joinpath(app_source_dir, "precompile_app.jl")) app_path = abspath(app_compiled_dir, "bin", "MyApp" * (Sys.iswindows() ? ".exe" : "")) app_output = read(`$app_path`, String) From c1aad5300a034282a20fe2ab746729593c8cfa0a Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 12 Dec 2019 10:32:33 +0100 Subject: [PATCH 32/80] Windows ci (#56) * enable windows CI --- .travis.yml | 7 +++---- src/PackageCompilerX.jl | 25 ++++++++++++------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2c8eb3d1..264b32b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,15 +2,14 @@ language: julia os: - linux - osx -# - windows -# - arm64 + - windows +julia: + - 1.3 branches: only: - master - /^release-.*/ - /^v[0-9]+\.[0-9]+\.[0-9]+$/ # version tags -julia: - - 1.3 notifications: email: false jobs: diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index fc203e84..f286ed88 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -1,6 +1,5 @@ module PackageCompilerX - # TODO: Add good debugging statements # TODO: sysimage or sysimg... @@ -23,22 +22,22 @@ all_stdlibs() = readdir(Sys.STDLIB) yesno(b::Bool) = b ? "yes" : "no" -# TODO: Check more carefully how to just use mingw on windows without using cygwin. function get_compiler() + cc = get(ENV, "JULIA_CC", nothing) + if cc !== nothing + return `$cc` + end + if Sys.which("gcc") !== nothing + return `gcc` + elseif Sys.which("clang") !== nothing + return `clang` + end if Sys.iswindows() - if Sys.which("gcc") !== nothing - return `gcc` - elseif Sys.which("x86_64-w64-mingw32-gcc") !== nothing + if Sys.which("x86_64-w64-mingw32-gcc") !== nothing return `x86_64-w64-mingw32-gcc` end - else - if Sys.which("gcc") !== nothing - return `gcc` - elseif Sys.which("clang") !== nothing - return `clang` - end - error("could not find a compiler, looked for `gcc` and `clang`") end + error("could not find a compiler, looked for `gcc` and `clang`") end function get_julia_cmd() @@ -193,7 +192,7 @@ function gather_stdlibs_project(project::String) end """ - $SIGNATURES + SIGNATURES """ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; sysimage_path::Union{String,Nothing}=nothing, From 4aca7e041b49afb126915cbd03f9672a312a69e9 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 12 Dec 2019 11:42:00 +0100 Subject: [PATCH 33/80] Update Project.toml --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index 5e5c1513..fd3d714a 100644 --- a/Project.toml +++ b/Project.toml @@ -10,6 +10,7 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] +DocStringExtensions = "0.8" julia = "1.3" [extras] From 48c6aebf207827925e356f994013a1a96352291e Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 12 Dec 2019 12:05:30 +0100 Subject: [PATCH 34/80] Developer docs (#57) * import the blog posts * wip * move blogposts to docs * some tweaks --- docs/make.jl | 13 +- docs/src/devdocs/app.png | Bin 0 -> 15066 bytes docs/src/devdocs/appexe.png | Bin 0 -> 23608 bytes docs/src/devdocs/binaries_part_2.md | 239 +++++++++++++ docs/src/devdocs/intro.md | 25 ++ docs/src/devdocs/relocatable_part_3.md | 357 +++++++++++++++++++ docs/src/devdocs/sysimages_part_1.md | 473 +++++++++++++++++++++++++ 7 files changed, 1105 insertions(+), 2 deletions(-) create mode 100644 docs/src/devdocs/app.png create mode 100644 docs/src/devdocs/appexe.png create mode 100644 docs/src/devdocs/binaries_part_2.md create mode 100644 docs/src/devdocs/intro.md create mode 100644 docs/src/devdocs/relocatable_part_3.md create mode 100644 docs/src/devdocs/sysimages_part_1.md diff --git a/docs/make.jl b/docs/make.jl index 74a08a53..742234aa 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -6,17 +6,26 @@ makedocs( prettyurls = haskey(ENV, "HAS_JOSH_K_SEAL_OF_APPROVAL"), ), sitename = "PackageCompilerX", - pages = Any[ + pages = [ "Home" => "index.md", + "Manual" => [ "prereq.md" "sysimages.md" "apps.md" ], - "Examples" => Any[ + "Examples" => [ "examples/ohmyrepl.md", ], + + "PackageCompilerX - the manual way" => [ + "devdocs/intro.md", + "devdocs/sysimages_part_1.md", + "devdocs/binaries_part_2.md", + "devdocs/relocatable_part_3.md", + ], + "References" => "refs.md", ] ) diff --git a/docs/src/devdocs/app.png b/docs/src/devdocs/app.png new file mode 100644 index 0000000000000000000000000000000000000000..fdcc7e40b49fe520ce9d3af73245cf1a8481b0f7 GIT binary patch literal 15066 zcmeIZRa_ih+vZCWENFrTCn0!n_dt-~?i$?PHCPB9f_s8{wm3qB?T!=G-5Oa1O!a!&*CZw2+xs#?Qf{az;Ail zTVMwQqKlZc8Y=MPi)#8E_)O#~q2;RTVD9Si#n}wO!rsBojLF5s+04w|#nQp`_(i)g za1hheL7$w>zPMUB*i))m*_k1zxtUS2@lh%`xKpyT0voIx{H*N!Ts)LY@|2$>)NBGu z`wiRKxPCwdz8Mp0 zW9eaqow}>pYqzRrK^O0y%UFE;Sc2u2(t%fCVQv2N0gj4jQt+PRclPM{Q110|{B_Hk zUwp>mVot5?(sa}-G1RyAUL`h5&)$Z-64Rr|qkGzeNGTEc^q2W(&PP<>A7Am1fKMTf zpCg_>9fqKS_q2EY9X%TG3H4cHC7qI*wGPaM@AtJF;;(u1$KRbnh|j$-YI}AUu5*BG zowl|-1-=}LEQxzB0n{d!494aUz^UI?oV1l+jhRqdJ21`bw$)W|m=@xWos*}(`Q+}M zxu7{k>`LYx-|F?!Kl6ST-2QYAmHEZ)kvU)Is~9O68SsAxldpGRuNW~fR#mX@w-MUi z8#Lacm@u$NKnh5j1ZIEPtPK8KoXwtb0Ie-PXwu(!MP53=PhWh`;!ocg#Vn|%qd-`h z(Pv9cj;mWk*J*FXYWVt!;fz^}0$0e5n^>PR5sB}ncK5r?QBP>JVSL@1Y@zYfed{K| zvYt!)Uh5*XdOUScd)Q)9%UZ7EUSHW#v~&$Lb*P4E|FPv(J~=mk0563aJ|JB5;@C!f z8_4lR!?LHYoNeWfMc?I02{NqLMt_d*cmv;!Jzke~CwC~imLn1nXoH6#_{^2nDhWv1mbqeT03w&*lz1srD(kS zGXG@P9^oR)V~xV$5jW`VyNx#eq=bS9-~;Xh(5KOC#q{;kszGKC!Y^jltbZc{+M1rz@ciJT%+QM z9qG<5`4ocq^JJ<&IIq2iB&iVay$C%WwH7*WTDxM&9S=6)3T#D*uY(34UJ#2zW7LhfwLd+q-{%%h)!ts;t!!zm z&EC2@`|G)TB`#d;(J^ohP)zXQPyr9V7;zZ!JpT8hdDG42+ZaVKVf4%jbm&=oqp)X) zgv)kj`&3FE-RZ|@h>J%VjqNTK;tCt`==YvYsFsUYZcYbi!_Krq=i~Bn^vv{HC;8uY zmw@z~KOEKc5i)j6!Yw(*^rO{F9IKz7-bg1#=iLNxRWGZ=LY!|{S#l#YTWSQ`)1|oh zxAVK$$M2ml;&)@ImEzK^mbOVq!Xybl!`mq66cwxC_~dgDijfMIzOeEtYeSKDy+XlK zo>b58u_PdEwDawZ#H7*xhR}5OPc3x)`z%i`TgeHONxg}bc8nhmVxXd1d-8`vp(z70eK$}MJb3lel@nIi({Rx6F5m2+ zB1S)I72DXr;G^_}L!RGdfh<3c>-|$)44m#Dn^bV}!ftkpYm~h_<2kQyy}SQ8v4sDK zUg>^!jAF?I_RWBTLD69`myeL2 z<)ak4Mq`2e)1br)8WEv*y4?Eqy9NaU;GF*+#hTwn1W9rIL7=(r*8Bzb?(XVH8N`i{);gOEe9O#URdd#B*9DQ8yPeltx-JzB z_hXjs4g6cpA{yvU=Q=_in^KhZy0~_yqd@~T*a~=sxa$@9J|8g>&8awWKp^bA^w4<> z$=sLm#djt`WZ_NYOn2M6@~N^@hvsCFWgy%GxDiK6v^gvPmG`KQYYw!@-Z7F8ROZ|| zZ`)(ZY0Q~gvdC<$aYP@^fs^ZbpI;O3?Zd1R+ z&sWu|+-&xuS>nR;&)TzI7x^yHi4@YpHgY!~6RdFwPc3!DBzwO;pC43kfp>N$^Dw*; zZsPh&mA*l+DeYzbdha|ou3=3t^X>@k8Y9ANJ&%`Ia__SUXy*VFbF!pzKTzL>heRX4 z*Y!QF92M<~hO6#};`Y(bJ!YYGKqYRijz_(1x~abZ?KH&mR()`kAJkr-h(8!m6CSWx z`nO7}4J+@>{z`%-H|}Z`5rCGjq}=D8{LXD z6RykoaVHNK)zib;tVKUX44U}V?rGoMS#AQ*P)qc8R3?j&O}5n-nnrMGX@MWj~0Qwc9Lqt1el zXYvZGwk=cayJsVl0n>x>eWP-gig?dXZ@+%}rdn2GW4&A!XG}^;!Oe|le42L_OWRG- zB}u)OiTQAbq>&NAJYSRR{v1}cWG7E#()qOJU`B+~RDC(4ql?Q6qs#s17PcLsLWpCC zPWkp;VFA12DngVbRU|>V(bz8mtu|jWLqAw1m;KF&kLN|)%OTH83c=gtFsmuLh?W*# zgA_P%czDw2aG=Mv#Y1rJck4IO-rh7TA9Vgk;2+qXO%t3?L@K@}Cm)#4Gre0`CgI`X z(J0gE?L8-GzVOo2#mh_DI)lz%woLibf$H9LkNF)dnU^@0oUVQuw&ZQaEq)aI_(cra zpbE~JHjU}h6qmu3G)Pb?INdcu@wKWkh-1>jxR35HSONk1dYRkqwkK4{yW}e_wu2;u zE2@Z)U$t^WusiGncqHf>C-wSTea8L0K}0}5&1!{Z+CwYKR=>AQ%ntwJ4pKI$(fskU zn#Jft_<+m~4C1Wb>~8Z%b=1}3?p(z2MhRIoaQOnN+W^rsEZ2*v z-RSddYAT+)fzd0xH$R`M5lVsW$J#DBrtPs@Z6B~;o3N+qbN5;w)Z}gmTglKuTT1y@ z#*k$##gPr`GTvX z6B+bXAlJ!ZwX^E24XbIA*7^t`q)uJilJW@l$%tLmz5Yk#%kmQ|5>qBLvEHs9VBvn9 zmlnR>r0xEgsARaZ6GK8+*S8n+kmu`tYeNhhS(!8O7KN9!+8)gGD|e5o6wy1SELgfL zf|HY%zRjx$*a}fDUdpWIwz}M90{zW(N$w)Ljz1uGT;@ZK^+Rc29;0up=|0eH1YhH@ zrVeEYrS}Ho3IIb=ioCK?Ix?nPLZW)Lhav+tHU8#yg7cZ_2>u?Z)R}K0hHAr%yR#yb z-qcLn09~M-b_4pm2hxivB7%K(HlWAdt9Q956>Z zO{FhQJ!$C#<*G0~O~r^KfYdK@x#WzS2J*pzMM$FWyZNrS1iw0-?LzIax!3KP8wLgU z5AvlAyNICk9S-f*1!qD?SJ$NNQhj;s)43EvAR^NCg|~MLp8WZIfYL_AhEQC*SEdhs z`860N_D5QpLOM57g@LjWdwQyyF;Qu=VDBeoKkc7wVVf$w%O5xcvw-0h7npd~;&mUZ zz9`Wfv2QHB`+8PuY$K|adFJ9hxWQa9#P})KrGeQ;+^XNet7n@nzkO>BpJY`jAoFso z<5QWT$3(N>W!wm-m32StW6FSrR6;7V(@LCv|kh%dSgthoi?nTU1ao^^pm%o z^T?!nN^?5C_nY=8**A4J{57hzKfiFLVpVdQCw1v|nYCVeiF&fx6+U>3t)1G9hnK*& z*mIm;cwm{MP<@9S#`qF$0MN7q=K-mO&AgR&gE=#i{x}`b2Bd6P|3_!LV`Lx zoc3r21@Q)#n0Tlw+$4o}{OaR{EJ#&Eq+Q2^xiyJ{z1p^(kTChz`n?})_s0*>M1s8vKcx7D*5H)4%c%SQpc#wX=Ya} zInBtjbcN=%-lLlpODI`YKa{PRw#^#&Jj+xVL~#*$%f&^OJKm@3Fj9TcAZc>1oTE{y zjJIP_;W6#jke656!zbV}lWx0sqZ(~DgXa?76fw%tD=bQ;x#5PUb0^8pb$1nMLFpUW zAC5g2vB=I)EpuwEea>_FbH&-oVK>qzB(~E|Gha|5)&m`)w_Nosp%8Dnbg8*-n0`x4 zgN;cCSpd&a#guT$bCt(_oa-mZ3l(^P357>SHdeSy&%|(SsFPXjX!+gyO1o_ssF_l-fQGjD ziFL{a0t208&Z&Dk(59B{lQoO*7CTx>W0;C-L#9r$l$a8O>Sf;$Ws^y@i$q2Y;V__M zHcIhdlb5Q{cX>enbcD+CIYtQ2m?1z6xt%_Q?BAx|rL|3y*d?Wc*?dC4g2;ZO1FhsGQnk$oe~U%LFtU!tA$jE+tgr#KK& zY*hpf6ZtxPP#*V}^KZm0hlEs`jJr1NQFH%aDyTlckgWBmL7h+iQZ4(E+>9?>W@D!d zCB0t`$gLOb*M6xeaWW7Kl7&egEZQuavd1SSDJ0+Usxcq|Ss<>WoM>rX`>&tb*)fIL zX)>;^pT1!XM3WZxOhoo%FpPZJj5O$Z9*;cza6A9a{c4NElxYq=L zUGa0Juz!01tt!Q{+Oz|Y8rRx$6%mGA*g#3RQj~L{)cnoQ*oMlHL zzR21sF#|s{^xl{SI6QB>2o>`jh`g#t3kceh_HjEnjL>kebN>>16YYSujNoa5mWy}AA<0DlP3cXCmUbLGz7oAjr=&DdBe0=Y9wEjYQh zSZ@rGmO5m!6k9C}QvAfcD{r&m|o z0om<}($e%4qq==pM=uq68-Mnj8ef$)q;MZOeZvH!SbdVY6hvqbi`f{|o~Wj&hjxXKO!^{&I^ z0Ey&>d*!vcj7bQA+lJHlFj*W}3KSo8Ta7cgJfNfsS-hP-$D0INMoVIssx(5EY zm-*h^9Jl7WznInPgRP>i-4DBt zr4(uZnte@C0{hET$q*SNQxhkXTU8>zj?5jf?7uw<2`C4@?UFAXFSEbF{@zi<^P}br z!z*2-!Yz4!cpQ7Qw5f14n#W-M%DmS&G* zMui39ut?)@v#;$OF*x!iAC60F@xHwUm(`U!1w$l;y9YZMr^1~1nVkj}jn5uEdwZn_ znQ9N9cI8$)Tuu<7sC6bh}y`y%do{whN0d;aw zYETSC^l^@6gdYzLelxls4RQWo!UH!XJzx&oZk82K;;qHeJc9>SnDAK8 zk8!rfFfWrSk}-x3ao@dg>O_|> zu^?`5Q|il2zihcF!;St2hSzfD$-`9(cZa);VT$&^-9+==&+5OKh_pJ9BPCZ~G%cvF zLO40Xk%8X5S6Siz>QJa;ymQ%*E&>Egow_a_r*8!nW_C@hqR)KO;4E9LXC9`uA&t(8 zASR$2ec+Xx3rS*T`LCKOe6da%!Sw%8H8Brw6%B~JusWuA9-mrW6B;6K8L9}OEL@%v zBEB(Z;802V5m=n}6UeK8slqT0-pr6pb;QeQU`{ue7_%lukNGu-Y)4ses;HIA@?o)f zh|vScpQl~0n^?Y~M=B^aLY!SHH+lREmKNRuHIC6|`gay}TYPv&qd9^S#}^NP0IOMA zPuKUb&f`35xGY4$R)?7-&(sh-h3LIbuq}VHITNC2Z+HJOe%^W1p3uN-00`Er0S@j1 znSjAg#`f&IR{q6*~w+p36h9RN=3pRfxQ;1VsHBxOvh3I8va5QjOEy-iJ1S zc)EFZgKae2l%bSCCe0B}qNtf%1NAIo&cKMlZ6J>Y>m0<;{0W}GO^yuzCkq=Ov){NL z;E1qGitff&7^Qd{;#g-L85B{bk}lv$gwAZZa5^!p6mPIBUlAEF3rac*)2%GQV>JW^ z;|@#4!v;jIstE9#%t$Hfub>(%sgUkQ36Y11GE$<~mZ!LWTXM@&ON>oY!*IpEABbjxEK`IZ z>3~EKl?XRjii!zPh!A6K6>dJ29~v{V%NF%9Q+p$5;i zjHX>is>$WiIW$j##kJdq5@yM1~OWZrVfRD0Q^;2HgYPzEp0bk;vqa%xGwuJ9jR=|=l6 zzC`$!Pddf@i!TxWg_~Ux|KiK%KmLWAgwo2C!1Yl-{tGwt-v3`bnVAs{FELHyQBPr4 z2=|%gRY$;XC1v1O@)zZ7%~!2Y+esjnV%3=4FpF{jWh^l$=}rVQ`V-Xf(9^&}ym<3{ zL09%F$)WrJV}1|uqNSIn=SQYN;T3)zvVPdIse?g~;ST%~3VNECZLLa{a_Ee>1*X2| zsJ!OiearlC0uWD}gS(s!pYi(RP=EmzoT=2>HOGNnFoCcC(C-Jcf?W}h3`;h&1z-&ye9?#@^6c4(-4ed47m+qe+pkN=fg?KV?{~*Rmu%yz7P;d{D%wAS6(eJc zrtJ0RZn%JBTp9l1b?WoS^h$Wd3chhy6b^;`hRb4Gd-^t1aD8|Pz^$%mAlXPm=D0{A z4n6G<{hMy=j{zV47FZU27${s_)p+i+WOLs5)gYMu#w@*#65%BFI2PW=%z%DQC$G>2 znmlrcjnzYCl-Hm6`}xyZEc<+`e2blXFb3Is^VDVa-TAF+LFIQJs`p#m0$CF;>@Qf8+{muQ1>m>#Q;6=^3V(03p_jTuu_i$B!&P^e(~!R zT?|q$&;-$dGlvrE>lvcOLXkfPcoM=-uSDLysUD37`{5(x@8Hn$p5Xp%h;{uBs>Da~ zCv%`O=?)mDn3tGX+&Q(wR|GoV2|MP!?;O<|*+|=Un7~Smj}q^^h_9+y(1c6-Z_C9K zREMZ&16lr3&_pti z=jvi<6N!+KPY(b+>f7V{XFAaC^qM7W7TaaVaVX+f3lanY*IcN+QfO8lg_?IQO zOOQ5?mMrzjgb!ao2wXSNqX|-_=M+uNtamapB%OVL2)&5;Fa8*&AkCSWtl`D*Wp}lI z(^g->?2ADwzj9o5H2g&2os>v8T1J5qr9~iC$mIIJKq-S=8 zxb*6h%6XB)Glb_|<*`!jk@k0e3*xh~D?&vXH1__@^}%h#kew z$A3|;icURrf36!9PPx5>l<*!VxuEILoJS?v5DUaeE@^;0>$XhrWNd5}KS^R3573Dq z&3{fTEt$-EUKfi68yA{!*uDIHcF`lm8#Lxa^8@T>mWlT7kZ710@!Z7eEfM1mIPV2e z2T0Dg(IVXkyf+E8S-=DOC**wp;rt(~hds0Nf$0-R{g1gLha`v2k~5cacxTyN-MRHN zR$#8)Cc@2Y?~Kj&Y<`UKXQ|XE-4&Vd)Ey!ImS$nf!7w9-!v+ldxLD{Ur(wA@G8Htx zvbrzRnAU{&!B8mX6qjGEsa)8^iQ1qLD7&`{U;LW)@w4L0waFweJs=V-gqf5F zkSn)w?RbN+qs=`^RVzrAbRCA*^9i%7byr$_EZmQEY+yEXtknufqn=yZ%icB`(QFk1 zZbHS?bl)$-R}IU1*b2Aa)DnYAmkJNkbjqK&E)h^Kr(Zttvi!kb-szdFC3@G`>`|CW z(hTFa&fLYM_%}?4y`S~|%jrG8gRcmCn}I^*-?~CGUifW%&L=`^>@u-6s4JGc{t#$` zn>WeEkfs}@<{;rfvcX|R6pKRGzCrly8ba$m#+Qx|+6qq!-~vE7`hSU`LS-~LW~WvD ze~$Y9RgRh&Qg~&M^ zJl1T#x&$M7T&2q)mi(EO)z1^DQEA+LSCZC}Q^JI%GU)7eedujqEu1i`6J@;H=0V4y z&O7aSQfbbau^c`XK>myM?F*1r0P*ntx2DC*V{IN054xAqg%Ts27NHsleotLyQSdu zr3?P`(ZTz=9AZ}i(s)wWOj{Ng)odT&c(gb*UitIMWpnrh3oA=7uOV zTNU*8jAE(3Zlwe<9!)*MO#3h0vl$w|NIr4xT!~X2*v5dEa2b<#dI)`^5=qCuc{7Un z_ivyrq%G`P|?R%CGnAN1GCNFcea^7tvUzXHi$SW404(S9;X{Xt-a0wL^GyxB& z8$-`)KL-+IrGsNw#Ta!P)e5tIcQ9_&FTeYlmW%cH_l0>?UNtr>(ITdO`hwSoCM~d$ z-oquIA1Ns`EoWyOZ!nN&${~jndUeI8>k@|fN1MnCjVw$`QjPq{Je(TpTcxj?7>YcxEP_bGVP=8; zGJOm!yIyoAhtgFJqv;^NEYT6D7ETXlzmBn0y&qoOaLiWBu4bZ)Ass1Wht69GdAaK- zrn~L@JS%?`xg7UwyIZfk9Oe+xA1y@3I$7=FO(sM#>8UsCY>qF}wXwRcK|*ONH#F&q zvKcHRx?1@T`qmM|F6_zjEWUqezQ#+Cx9Wu*By=7Rsm^LlNlIIr+UGh?MMXu_UV)xg zaa->{VgS2=Imh8bs+dO7R$Zlu4?C3Hy2Wqt_Gs!f|z1$b}- z@urih_k2DxyB!XT;qHd(rtknL6Cj$G6eP%r&uKW8nV!=rGS(9yPScANHV8s<23puT z`8^J*r4o`36D@4XRqu4k4XU1DUCwG7G7EMAkzGEhO0g#wUy!617-$gk?UvO_MhbA+ zSf2C7_eaU(H(tC6{HDyWQBeuoPP)VuWn%&1WsY__?d=;CyR!saB6zVP5a+hk%1v9Ma~>!D@oUv1Vi$GTTzkc`3wByVKz-c zkMfz!uLz0?8I2F{ABoy>qN1stb`J*L&CBLZMRThc4^4?yuFw#nDKtR1TMTOXnEPwI zN!4|%NhpjzGYO?P_MNbB%&%VvQ`TJknJn!O&Osuo1Ti>Dl=q_1DCDq32y;?l-^Sr>-{PlLL4B57E=$jQ13o3n1loAo8mohyA9)4q`iJSwF1~lW?vqOE6{1SU+6?A}39#8;G%&MZ z=p_!0yYF)upfP8fRCq#@4a|E)S`-eNW!p=5L@jnkFMIU55g1Dt&+~{!_rd|Vy8v!i zO4310q!}C~R@nAAsv{iRtMz5Q##K?BWFwAlx!nE$_P61zr=Yg3BbIOy-u5bQFBkmQcT<=K!zvBHGmY;)rnc;|)PP?&BLk7g94#T%WGq zcG#zG-&$WDBfLhX2!m9}ctwyO6Vx}gn7p7(pYn)3xQzIhTDa zw7BHbk!J(mA@?R@UBvjGDn^9&f2D%8+PncBv$*(4{`qSpp4)Vm+*DM=8dpi1=tXg~ z@z-f(?f6()Pp``@v$Wnlg9xApb#+Y zbp*bU>Khm+s8muE7f;c*ounBrnzlZ0mOC%V%ggZKFVL>(`Jl&}3=m74+27HsFj($S z99hwWJ?(2Dxo8rp`HLpt2w?P`@19PO1+jEM?#~WB=BuE-Y$dPk01;&GkF2W*O0o$uflj*7_irMwaA=~^BC9I06*0+``&;Yy4)a?3rY z7{wBtex*DM%wIe?Pr5;dFT{F!_!AF0eySoaFMS83oRjY}IKuf{%XZJ3DsN1Wu-l$* zKYEDd_STzS%+-tYoKewme}A2;b3T^G3!WPqdRJW|`^L%8CMc-o^!(hFVH?sd)nsGj zg1kZ^t5!=9PXQG_;C}l-+jXcS^E;{`G56Hk#)f^vFwxFWwB6~#*TBdCnAVt9T=i}L zkjXbW$dRORVZZ;dn7l@%`-MB5&u!qc7r*fz?UloT%gH=ux?wbViW4fEOlmTsH1s#Q2DMVB;rsdNJ@L8VB&`F^shO) zYL-q&y)JJN9$s_=vSRP=asKNxp46ed{QOC3VXQm!z;&7Hyxz^sNt zw)6wA1?5Bx(XaK+cHR<7S_9KWfP}CqVpWtktQJ2@9bFct2S`x6Xut?^H!$cq&yCdF@ zOF}I-G87u100a*Yx5UEtGkJn3q6zR=rFp=!SqP{#A0_?jTmXwv?uI&v#r8i21Nja3e9#L-MDHrkc}q0(!TEq(>q+5@@#7iEOx2rM=Ctl>?6C`o8~8VH}>RA$hnH zpEz8px8Ma7+vk7!rRsmJWJ^p$g1>vSWx11lGiK)aStA^tW0=w2Hy)+=*nK`KGE$Og zsAOPo)OoGT>pSP~emxhBO`O`xhz)ipy+K-=Aa~>K%R)}J*eMUJ<|2#P z25pYQwXLJ>i@ktS&8V8~k<8A1Tno=**e$b-s#;AUWfNx@xq!u&4)N6SCL7%E|662p zrww)ufX#fr#(6buJhilFH8_+L!^tH@#6jeI=cs*V>D79{6S8_h5&tqp(x>h9two-G zwpsRadpB%ndQtVHhlIt+r`-S#AHQbpp2F*TDb`_M_KzXDI@}jfZyVB1H>9fk?#V|o zWy^)MzNaVrm*i{Rr0}o2L8e(tH&%Zn8iw?8MkIFZ+mp>(T_^>OEZd&YVa?jwcW}>B{4d#$n6-r??mi&Khdu+Llxu+dr zE>DPZKAm7Cp-Ft$6j%jU5PhqE{$Dwb&c~RXDET@c0a`8&E z0aI#&Fyt*`V$?KxpW@u^1Sha`#!h6;R6i!|21gm!@HypHfWr^4UB0~U%~&zwB?S`Q zroFzxoyTL<{PM;8^{L%7l7w>KQfo{C5@>!4qsfI7P%Jx`STRhbUrW$pIYLEP_?E6q zQ%h&ut%+EU3+$|fVJ2tSc4D(_goNBo9bjo)-?RoQmthON8#tql>VG`Snw<9HF@9-) zuo^tCcNFGtXY%@TY_Jh2RMj8;!Tt0@oKG zz;TnwewLc3dh588fz`E>8SviRoBDmRV3Y!7PfyzJLM&aAk(TrK{}go#Gy37Au=mmw zt)EYeI|1=;m;a(db>%-b9bPoatxv!&&aL-xx!Lc{{@QFFFMaMjWpVtn!DGbTWE(R* zVz^4NZiz}@sCkl8x9)Ji5EDR}{+5w)YX}<|& zN%-gCuq?3Z;>E}Pj)1YdVf80m+4w^SI>LnjtUmfB$T+?C^b{2ucFK-x_n|LZ|cjm9_P{!+q< zU{pfaKz>=`eC@dRWKq=gu^G2~K_}nj%K?gL`M?mF3H~xc{Vf9{-Aq-xYVu!jxRCSf z9ZTfIChm-xiPPF~y^eiY+QW)wzw8Y>!foVR&Ky(v@GP`hcAJ6a#nGajF(bOmtzD0n zuZh4qE{W91*YOa{H1YMJdwqj9K=#pjG-c6(hom{emMnN3>G!zNm>!+zcA%DsGBaTJ zE%eo_MrMVol6H!$#4=I~ah)UGvYH0Q*LA3NVanMMUQ3Rhid z*-bCDWKIn^Xtt5O9XmTPfKEIT*}(~;C_1Q z8}=WVa#XUzq~^8`EVZu45u5V@030qMD0ZA` zNvIo_EFmGv{TC+r#4t7vcGTpqVr7A4VzhxmC~L0?B<5D7J$bC;ueKckPKk}yD^&ZI zfhGGtoYIelvl#)19Dq}T&Nm5HvRC1`C$?i44S!gHHAS7a+^R|5cg@29;2Vk3l>eWA zZzOgQwb>8P-v2Kh1OD2lLW4LjKw6%a$!tg4vDFXkS5oN9yvF6*mX_NxODM>C=Tiei ziW#E8?LW7eI0@XbnsnebH=zy#Qg=r==#_foEfTKqIvcPc41;juGu)Jb+9Gzo_ID9i zBT|SlHxa!|*~(GiD2rG2H(a>PK|r~flMCp{Kn1+x1^w|3kTUcZ&)KTU$2$&*a1_== zytT)|su1h|42ca%uRFdGzYER5Q#1uYs3x`Q9X~CJ=6GuKw!8wTQN4RUVmkyTcpIvT z^0V9f2BSVOqJU~UyGT<<~H$z4JNho;dh+%*5; tmm~kpCp}b;{;|%8Fa4ijj|gxI@6_wlY4MN|VBIN#w1k3qg_u#${{S@}B**{& literal 0 HcmV?d00001 diff --git a/docs/src/devdocs/appexe.png b/docs/src/devdocs/appexe.png new file mode 100644 index 0000000000000000000000000000000000000000..949f7b6d11367c4bcf0d7cbd72131da512033483 GIT binary patch literal 23608 zcmeFZWmH_jyY2}|2*KTgh2ZXP2~Kc#hv4oW+}+(FxO)S^9YSzv2=4CI+|6Iky>rgY znptbsthpZ!Uv|sh-M!zcdaLUBJuhMMvf@baAK{^(ppYabL=>T*-oQdZLF0UQ54^&i zv&#=0-Z=_MDt`d}ygnF*0?+ZBMAe;?Y`;6X8aS9hncCP|o6tKNIhdH(IGWiyoxbf7 z08XNRJxSQX#K6hi*5;G4xwQ$Dnu+lz7N$?Kwl2VdnfntHGY=CJ4=d{@d6`ebqRM2l z%VtnepP(c~zAC$A9)VrmFr6X4&To%wJUB9}JS6ghX>{5o6wv&C_L1&Yd46W^%%7N9#%U zyTEjktj+^Bs?Z0Lz`LLw2hK0R15`C^N#sD_wY<;6ey?vSll}9Ia5C7}6XoN!{&5-& zkrXm!RPuzcj!<8!IaGt^!P-@7j`Yy4sfpCwlYo6@!8F-w1`9>Pb5`B_$CdHGHMUs{ z85c>Z;}qai6^LWVVM*SK7hKOL-1e$Xm?wjzh2BRg^s8U6WGHv$&!Wv3eQ-a-G1z0x zUp#hJ`S(>i9gdhV1XUfxl2c}(5^XUnr=5Me3Okv)4rZ_qCjsa1$i2E6(VdpQ(pswM zGv0R2+kV|n?vCW9I~`x9CIz`t5KegG@6sH(uI|(;$b+EXx$K8@)cI#Nz7&mSl%K=H zm6Uy&ZC0t*a}H z1Es@nV7_Zg$iKP+O(!-EbUIryONh)XDp*a~@V<>HH5aPeTU>v&3(L|TdfPO{aE~9P%6p~rpy1%d8InUVS=vlQ?y!RB?}2E~ zTU>N21vG(~MCH~erv>YGE_=iKuGvC31Y#=FpY(4OT;`KPv^JVYY4f!NrOH{nGjG@VPGrFG z;1q>jU2=E|j5NV%t^0NBP9qS^1hcj`m&#FHkydKPbk&x9w0%jvP&>iR4MfWSF;f0m z#M@{o->hLGt5Hw;a}xIfsfC)Mrp0TdKd7}_Q5AI5fhNSSKMx3efdrGQM%7#G^@x+;G!=6b<8UH$T9aCPrZKvwP2m$%GH#Ql^s z(3`){_u5fDpMtU~%9-gK^^jbJbfPc+dR*|9t)`g#=g={5olZ$%ZDu871S14Xmb$X4Q;hjJScIfm6#q=y;4N>5!f2iqUR%uD|< zMzo8Z4*pCwrr>Gk_BvitOK}T5>{fdglH~PLqO-4mMdla44w>D!xj1Lt%{@P-1o}`0 z*0kb3LtG|lirfJGH5J9mm1vP8WMH27w8rf7z)FPqbREU<7yhE6o7aJ0C!P@hR?D1O zV|2&85c-^T2X)39RkYoQ%(B#U#HuWLi4n+WJ%loy6`x*vZL*}mBHe3HvHq3e98}0) z;p!i!AX$fuk~+SSQ6#Gs9`L>Ac|5h(-T7UF;{30#_D3t1xtDs{nR$}TYU#UrpzP)~ z!fCg^jH`EpRJvZRH?SoP!BQG2Kgel|9d0}L4>IR7Yg=%AODyN<^!I-I$!KyI$oejI zu+MDYx!CH8_hdq3$`0qPZZ2ca2k;r$aar}h5I0%tY(L@P<2#yvobQ=QBOrS(a@jng zb@5O5`f5u~Lyby8f6` z%)CA)OGJIROSpdoR(d4Y!s#ulC?{{z9DM#}w3;TfpUoGOSY(FPM&A|m^O5Im>|OT< zkJGldMv9iEkp>co9j@T7DYo(H%HoG9u)i%IjX$3p?1fS2DGv{~<2iu2PC9N;QrfB2 z8hLV5xm437Tbi?>NeF9rti}X9Opa4E;X~e9aqzl$KHmnmp*uEeoo)MJYowShRJ+qT zU=v)bYYZ?~P91a%={T)$kH;&7GncX#t5DA}OIFqzCvuc}iI#LTu?1(s41U1F6Q9@> z=bt)?7+AN{uV1WLJdT2m@R)t1+B_ z8B=r+5d5|}PpQnTo7dULxAL|(*Nn|iS2>$G;T{4F@B2-uUsP}*>TH(q z3Yaav9=0|a3u(WZeE<9jzqdPK=+yDdKMw-u&U;?=4m@Uc<#?`pWP9d4I82r^+UoDe zla}3SbnTBGE4AfQosDnwf^H{}XPwa=t) znKNh?J7^P3vmYjooAgH7KGK%dp119!OYBS#58fWivbag`>G{;=*vyOk?hwZ$_HN0T z6aXL6jVc|g3tPK5uXKRe_-^_>*$_=bw}ZQTJX|1AQT>;Ux^hkp*-9nwxbDLEppHi8 zDnYf6q9r8j{gA*F+>nlyZxcP6#@=r7Skn-&!QP{2J{A0Zxvb-k=rxWG#`oJ^UY;Kp z)1rZqk;3P5Gplo1dL?2c2Yw7&UiO$;X|voNZsOR%-OO%5j|PnO`NXI}=J)3QRpkQx zRR}lh>5ZG*&rcliUDfyRrG}8~8+=SYJ=*F9H`YAx@;*S~7Dla&eLVt0Z3j28*2Pt1 zTOHHpsuBA}_oP&Gx&6F9%Y)dqyj%j0VrzWRi))a z*d?o5pDKI$1ZAt&7jHJ*TQ|uDPM>(OZfKg}Q8IoqgE_as6Ezy2>hkWv?gZP;D$kUIQ^izf~p<=P;FT zYkru~-fL+RncN#lQ*)7j=;9L+>!039@>9-cS8^HJa>&^AZON#fTyB|gKc}ovy=DCk z*X*(}{DR+e=({L=apS~87JRXhX`oJ1S)sk&TJZjd)_|+w!`Le5;X>@@WPGDo;rMPk zN8I7Y3M~X>+oOW0ezB*qW9Q9mNy?f#ZL6w~o$RFUR3@K+;oRHN)+S%tY%dR{49+Zr zwbk@fDZZj5G#` zj6zJ0wyL%kYR}Ue(NdLo_q5tJPwmufHHTBS5#n}mI+Ni4AR1qP#uaj^F7;)S3H|$R z!9`@6|LFvek?a$MGWQ`n;NhJvzMr`ayt(ZWMUCsbFh}v9P0qUrt<*$1KN-0ZqRUMe z2r#{J!tIsc%%LOfSZgedSTwE8^?U=;0I8EUzKVlB!F+HJ(*8sr)F)$O0#Pm2nsgc} zLB%g;pL7AQ>&u_u) zuIr1YL_1a%Jcsk1H_qpq15q32gFd|(zP>qxyf?q|gz>X|PZa2|5z5u7Z_RYn^55G^ zJn?xWv_~k0-Xd(GRN6YUpY3Vr4BsD7xl73z4$~Ex3G--NqM*CmjVErnk zP#gP)D4iYSmyDak9uE*Ar@bzTGDv21@aU+mY#e)V5WnGdMJN;^)fSE4Cq?wi) zde}oP3b_bNjo(wvkjrMOJ={%Ca(?MN;&H8s;b)d`F8F%~cW_w4N_;fB`vM;PLoPlv z##V1H3{O11HqJlo(Kbt@Cf!dIXdR=BzSb}wV{=`qfliHhD3g**IDG!gJxtrG- z&J@u3l@9|uG(wZN%pPZxG)2P^8V*m(+f-sO!nk{P?j?}jwVU5Cuk^3pWA|&0zVv7z z4*A!PjOqRTuG7<75EHH!)MTYm_Q7M}u#QH{U#R7#;`n&QAQ@|6{g$c+y8QLI39D>q^lO`OIyg8DdO*L2VHK{b4&jIWd`agKc&5dza}@+ zXS119h4lq&~L&Bz^C^tx5!` zlVs||?(9!(_3PBOg2S08nzqcXm3^w&Ev5(h{2b`0Wn$}^on2y&aK>Ok>-SrYwi>kQAfxp4u((@sabIc@tCctJR?H%V+FUsuzz_XewSSq4Pg~mo??i zZ=|2v@l-CydFyJ8lhM;>(ktYUjerJKalz~7op|)HTKmx8x0;$zo)EAxjwd2_s^}f| zOPRF^tMuT!jL(wd6SXw^coME!ra)Hsxa#Gt-A13eO~>qYNr5&RI=Zxo2<~*TjKLDy zb<_@y5xx((#ntt20cjbrFprrmWo6Ix^yO|ej$-z5aB1ElB;22N5F1&Gumq8jkEZpk zS*%pxUHb;m9-r|o@j@B7i`aNwB+*ycM0G?1ZT`JTOto_bV}U7{Gn{y zDXjNRL02aj!jr30(%DN@{E>ayN(jeAhB`wRGL92Yhca#OfeES5xUk0%Q;?A*KSF(tm3&BR5e3w1<(5G zzjl`|jymU>)-sP1v*#O9)%7-Dxf70BQM0o;p^{&?-v>}$w!?y|YucHiREput97fI$ zXU^h07tt_QYyI5BVOz9abZC^u?_8DVw{viXOe`epN37wMe7IY=i=2+OG*z`#HNEM7 z&o@J~EAvttW;QgP$z2dh*&BbV9(Aos84ib+uGkrAr7WouNAw4u^a$QW)iCmVis?-`v#Un)ZmJNw&e!fV-Df@wbQJapef=7Az)#@O!6~*`y{llh z#_^SEDV;yXHbkFhA{}2ne;;waxrg$r+>I%cAh-T%lJL~dL-qbHqhOS)_9rwYjB{%Y z5y-`*07K^mmOax8N>erX-vzFwmtMYzRYC%bg)M z^HuDoO4FA8Nl7Y{O`V;MlU!-zHNlhnszuYAv2g>dLkn<<%>eKI`XmS;ZL6EOGp-XN+U+QFj z_uLqF*8tzgRDreb+TvRMp&R9t+BT)Xp#@Y+{W81iUDnd_-a0fOwKaQP@BXRk zh#UI(#U^V3FQm6S45LbCsg2-u65!pOMRlF~M;iqXc%SD7NT;s^)iC+8eGCGY%AN{y z5wuAh3n2@G9_tCYlvl|d?@%m@c-n3X_x5#$A0OVSDoryiXDTIW$(|P@YChLe_Al2f zr?|fL4B9lgD$)nz(I|#zy@dNnlh(++Yx|p?RrQZg^W4nHrL+>_^$+fwIcTS4u42OV zciUP|>l- zllj6NWt-FO=Uxh&kk;}3qXqTE?0YH+tkO@|^(0w(s< z!&TPb?xV6BSUXZn2%`SaV_#iO!}*7S{IU#s1;-D;!$^!Brxxw(5PBjGCi^j~XhZ5r&>NA$CB>$cyk z^yeq>5);)^fz9dD&F%Bq#h+M*w%C%KyXbmTZNZ+-nb&qM-<-w$k9L0Eyl;nNeyBY= z9LB^^tAn&F85fqvAnM>nlqYd>BlN3D?M{jtFgC|?-~Kp|gz9x>Zg?ztpbrP~6_lOV z97?KdD+dJFUEDRI>}(ljxAF?wE>Z(yS22ejSQG}AtvJxgVwD%;P@3jZ!FTr>4L@%0 z^lwAM% zotVvx{t&lgl}Ujc(!N=n<8+v(`^M&l7W?IGn{NR;J`SWNAu@#!=M?b~gq)e|y)~U) zNJh$fO^a`g#G%^z{Of7^i}QzO>Z-m_TB$5k10!~D!z52TSRkn!g(iLlV< zdqJ)3shsny;`lZY9$4w|`#N`J)OwEvec)RH zYhv%w?YlW&x2%?ZM(sXqo$R)EPCmz$J>p=MT|PVnlD^q8Rfk-&hF7KeF3R<1@gQ(j zmafj0>iSN=@*<}r>Y#30y9RTL@O<-}0Rhh{sYw#SecGQ*Pka>6$bm&z)to(!&<8n- zt7Z7{4ZFW~3w8_mZXyFospZ<%xQ>j#?*l~XVVzs@ zd;Da#GHZ6sprOLY4;>IyKt848HaD(Jl{2{DiBSc#?CE2@HD2FA8PDVv^T{`{4>^KH zOD)n^#Oyv4d88E3@Wn_{Lr*(vW%t&^o*BO;=7BABv30p*){8?WPp@tIw}mrFH#}ey znI!LE#c2w8u2{D2N_1w|lqKEVUTeZX&hg3V$y&ph@n&jWWBk=6!}(@OPxxjeX88HW z(NrN@bMT`@efeh99w@YRGbRwR1&<)w!xh-=i>`Q2u48=^H5o7m)ad*@(1EsZAnJyL zxdX8K;mKa$&G~d3!AZrP8fmKyyFu&*EuI83J*SdwMLN)J6b9OcRusHvEuy?ZTR9a& z|K1r%&YM)1*h^GCsQUf;=@sP5_5Su)^;%^TY9_|-JXoxKO78m~ zD*oC@yjN}X?d*K^VgBi#nABmEgo{)BnNFgM$ui-_2^S7 zIeyu(tq>qMEJ3_;Xr8x9TUC$f1TMfjGOBTL);>nMEhg57(|+f9w`R)}4NyWsy+#)O z>>SuoepwHSEzeqiaGnBw_31WvUUX<4#-IvmGh9=^GQ0A98zYP*c|1v4ZF@ z@4dbfs(DdAqsB%=L>5m~x zrw1&73k{3X2V!qdh;@5gqx~&%Ot(#GpBEFJrwR9*AqJK?&aNP?eBGrVXdwYmwv}kh29T1ZJ^GTV+Sa@R`b-R zig&qNdp0zVe^zu~+sv?(lY0ngZuq8ZXGk>2gr}2vmKbgdCg?8~`d5)Z-Xdh+65Zbh zu%b%OdMM{%k-ZI)9cT=Ko`_v7$VBD6I!TMBD&7rg2vhIhwW^3sWk-puc%fvyw7t8` zBTUTVkpLoJgPp+#){dFo{>EJunbNBYgUowFg8+3?_H7qj`_Yy-rs{PiKc^QXODyDW zF)rR)2JcXl+;(eF>_~U7QMPsyuGlH$eog>`(8IFei#8Wy1dPfp>SUb*@I zEp5|gDB<2LqH=Q}yp&k;HGd4+JgL#hnET0N`|J7IYgPS&y@}-?PvnYz3gBDMrS|p| zK^^&+Q2oozlFUhqrFZ@1hZhl2eep_<>?N-waH85<#%?W+p|Mf`*%B-KW;cNFpK#dihb zOOyaQi>K&%NA1}k5*XEzRfE>NiGdfRrwuDq-IWBe5sQ9RtSxz!MDc~1<==QhQ!pP_ z!iOL2nMvN-^l(wZZ*6YtAEM=-OP|8ek0>)(bVhJ)$9 zu^p9v{`8m+jAK*|L-fwdrFE0YJ}##>$RPpY4LiQ}$A+&BcM_>4yrMCfzP#i3@#F0i zaTkTt9T^DGfa)WX!ltQi;x&+m8Ja1GKknM9hl3R*?_^xjS+@<2lCTm>vOd{*ALEd7 zAGu6Sc*ZTXXBQsUXxT4R9C@Dcq!{=0 z(vn0AYWMjSpOUA!np6wO`?NX{4`+&B4CR&t`BR>dq$D1%7*pE0fg9~pDHYAaKsuVi z5l;#_!n$O1ReI$C<*E@pj{hJ7JC+Kj|A3xTUhxC~SU?pLjeq-k_$l$vx$ED=;Qxk+ z{(pG!M(x&@DYN~jd2*4+wD(`yOCU19^cjdqa2yGDAdpIQnXrF#kWVp-Cb&u7HVKx# zlTqTIGEQlj@qq(w0n*8#(!NkHOnim@N3x-Mxe$S5_fYlB zY^@kR;LBfahL%%_4bzGB-*Lfr6^i}MlQ!Pi;mFo!e8I=7OQowXY7`93nhR)HRe z#f?t%WqEJK@6EegLO=Hx)rx+Y#2o(cyxTL)-t5jdaM{=^ZTGkVtaVYu^uenSth)4t5J!B*C4e@@$%I5XW(MXG^9lJXuJL z-n{gmvi1VA1}Yro-EYzn|8Nbg`)dySffY+1bUeHmL$X_7vIJV@2=9Zl=hrdtCQG{n zXY|R*#a%hM)&RmAw`q#z(u?U;pk=fKs{3A?-~Id0qb_B#M1PQHcMFKrhZ9zzMvUty z^N9Q@9@jY^`!MC5pD{mm{sUmH|0#&P8M)vgo{hkLgaTH7`t%HDs-yr8Wjb9J0@fzX z#L~qhiN~2ZsS6kY_Xn0{3TbH@Pii2~x6p*11qg^P0~T+Q`lP)vbAG;y6do)%6$k}g zy3|z-{A*ZN{d`dv5PRoL9H?i-s|dLc6#p3CWsdiizgme{=WVb+Zz;`K!RO5ei?;jU z$e5ztzddmQe6K3UTSFLCEf~h=6W);@Sz!y2Ie+0OO|c{^{udWb`co<;IIsKN&JTHV zyo7H1!+uWJNnCIsa*o7IAS^dvejredSRNgm%nTS9%mMn4m8$eVEf(u#6j|KA-@h{u zr~$hhFB5#!^9A+hzpjIeWk%B*-d-pE2UY!sY*o~6OrNnE_r7m3RapM^E9UV-Tt!hf zb$#i-=(Q``{p*(C1BoWZy>b2PgMYh8MpWVm$i_~@A6=`u$ef?=898h051n>T`=yG{ z)(1Bkzv;Y%?XPv544Ck>w-2W1*EY}gJ@;7kah2Zm>vG#9E^s^FULwwH!(}khd6Qg` zIs(4E`RS(4(@MY0yw&-Tu|QNUuSYZe zZ79<<*1yp-ea1!(Hz9GT;n5QvA4lYSa2lh)+9YQLiaUr%)bFw6cBd&U3l8a$lD;w( zW%p!bxsmdt-U1S!*$G;K*6tr$ajJG(ampBP0nqh6nSnb$Jm)&2b48$tHvG*Roxqsq z>5=^S%ep9*QeKbr2CsN-vlW}o@4HV7V->NS!> z+}L3z1Oyhk3e(-S1$Dj`Kw=X=AkK~4;>=#BWx9Ip} z=R#mrn2*;8gFE%e!t-Oh4%kd}m3^n!4O|341#hVYbwq%+!L*e*wg%rjTVUoB?Fmib zh}R}*WKBo78XMpe1gNzT->}d=Kr5ss=TPjm;X*le3uxE7@Rj7OiLKwVuj#|Jl0*Zz z)c9q5v<6$%*HnB>5Z$vi3ZSK^=f`fK&rd%`s9hPRACmms@VrY$Xej|@EIcJRb22*i zc^|FWP}}*i(#Xwju4Bw+ossY9HzGRv1{D?6R%wae&fe6)^UPj`xZN#(LTE%32`g&` zBMzO({-1P|6?gZ2x2-sh$To9aU^OZlk=zcBPOkdjl&E^Xm71~csmyY_cnLmx(Bnt| z@n-=dD2U4xH~AYat(x1(Pw2nN=UJ%&^_ZAc zva_ykeck`Mu70Zm_P^8hYHXBpsj1h$9IB4j!YOlz#sILj(A>Q9h>r4XgC*~M+{*C$ z(7X{R4tsW$uvTkpi=8Lt8;oeZWcrg)<(~k>-Q8UM_82ajaC|dv@I4D)2+(N0A(Ini z4P+Ep4gsH>E%3~#XXp^#zK#}v^Sa;cpAL^PT@D$&4GWu^54W_UgL=@#?@gZ_ynj5@ z+5;4(+ubOoO2O<0-s{3#pE#f&POS6Jq5o!$3|xHv$!Apjx4nKM6{G6~?q}z2u7KaX zn7(F$&=P(-+!5i|jTN*qpfd{Dgo^{RQUVR)>bBGni@FOvpsC=KJR}dz=_8!F@cI0>^+zogk zSwa#|c4Pp&q+Uu~SB)(9^7c+;erA9?EiGpXcmq1@lA@|=R!t;6XZRrTBa$uT+1ur4 zK{y22+6~ewUaL-5q1!?9!~9lueE>iB;_0X>HYurB7-jqu56{T4OM@}eYV4i+12!PH zp`PLl5Nob&xvWpsx5ZyBDqJLI_r+^`n+kTS+aV^qSWUM~CgJSg1k?OblCvr{c4Yuk zD7>#U>8{lEh$($jEaXO)MiwWb{fsNQg5^zH5YMi-^!^V3J_8mO<>Hjs>nW0NDBBp! zrK>~IN!YY;`5ypHLor?a3~^B8yH>xQK_Z4SW8OyV?1Nx z;&O8T4%W?p=?*IUAKHUtLVKNEH*}M=3^{pG?7S{GG@tFbcip_Sze3vwGvuwy=?EEp zqeLU5<-yg1S_6Sr@@0?+$mdLz(s^ekPCYiZr!L&dGw2s;neSl}k4ExXW-FHFxXm&1 zi(ScGyX~!ZmTdLeE5CK8&ZPe?H-jJb8N>q&(@8Um6hv! zUkntL%I8D&QC5ai^cW28FKmZLMhvEljRo!}g-+Tm^BwQQFcR(#%CX2OPj|-B*dRB} zu4Wr$ONQ)s!6NOw-BN>{-|rh^9Q9V@u}y4tt=u~;Veq}r!>=d-M+;yK+V1iHTJQtx z=R1fB(DYj46n1spZ9Km?p#IH z@SN$yfpA9CtTSGekBG>7yO%3n0KKMIaJPf~#0uQ5i2N5?J|)wu5Dydb$BTzVJ6p*! z#MPTdD0|aUQBhJ24OX&nJSFyY++G{+kMK&aW)E~fFIH%|qAU5hZz8l0|BI!osF?KH zxayZyK;AX?8#+#0USIc^Df2Z&yXT(*vnL5jCjqZ&z4y>CL($KlJ=-i3DQV`a&kWR! zEgcYTi|vvK`vnU~P6{z2nOFYf@K=-MhgB#o^CSx`SJ|OZbx^_6VFzsPQ4L(bRDi7| zfB!P;`RRA%an94B0bmT7DO%PEa7O}mE|opTci?Al?@ets7Ka4B!$3nDXfRhw<&g0XxNGe~@b&fe z4?rRiRH*M78XhkAIpk$v-(?ZiDd6rTa&065JHB72(&FO*(MC_cSmBUH(z#W`)RtTz zXyTE@(@j0lxzj38Cid5P$IV?tz^hqLLPt}@jpSM-T%V&exx}8`aiW)l@Iq|C2CqHf ziF?S?e`xdFW`xo7!Wv;8Oc|WrQIN*hh<`h|H$F40=g#-_3;^88(-r56XL~$5VVyaT zto1r)u1hmY&(7o=Y7cSXpmC*K^W~hj_%_XDp0df0Ud{xT3MR*QdWD@gtRDrlXw2;$ zzCh}f0;4UZ<|sqp2$zGOOiGlmLjV)i6pho%$z1w28+oFRN0#%Uk3~(S=3&{mrrcJwQa}n856l1)tIr*y9GzDXI0~y{MER?V zEVobAM*#;$Uc(+<4%E{zGd`F0I_3eP!(={Lb@iN5+QLXys%VSjO z{>2DKHj9={ECD!=Izh|3Vd%>$f^d#`Ul~E)dvnm{o3PaMP}_`87b!WnaPCq525xj| zm&guo2b#NG{-;WConO7pyQ?ePP9cQ;rJ&r?sy?=q|G=pSQPkAbMKv{{3k&Jjyi80` zMiMpR%MOj@u7_1BY@j8379P=2P`#i-#mw_=9P2LOC$Dd?Jta=RD8I(&88R7QEc50J z!ceKZVoPblS+C*xLzX1GG+_*J=fcl;SK8G+xlX}Fa#Eu5G$9F`djKh~yQ@5IC8$x3 zUkCnDV4;7Dw~hHe+NO^ZcP|!q#5$qPZtc0T8rIY_B{oHEz`EI$gl&d6pnJYDwHlj9 ze`fC{vV%OX#qBN1*q%&E)jFM~U5zUD)RbS}zs*pmv-NLGiWN%7nX_GN&ZnlEM$TI}^ zYCa%biWexP0!wEcla3Ze^8oKq?+P5C1%zzPh0khK%L**z!n0SC0Dq)DJNt3nIJZjc zHm2=J?kusVB8Xqg*`2dSjZ0Td4}LBy?{m?@ef8cT+nYy`T!gCDu>zzUlAH6@JBmFx zz%-Xj!rb_9@-33m>;DpwW-;sec7%sDlcQlj!Pb4aeO{KkvJmuG$^l8^a05?_BN?40FEAhW|Z69am#?;@AiK3 z=|~S~-)8$wXt|(QGwB)nxFWT^?nR^|(>k*L1Z1zpwHOZ1hm`)Tak%B@ciWnRR!-F7 z_TlTAAp8M7`5UzB^S9JA*lxLRLcP-E{q4)|?Nk$~7*y0qUqafsm7wP4rU`L8d=B zUsgw$cAtsd!oHrF_Nw*fuG5f|l;FhaAin25*G&G+2eg}_s#hXHSUu2EAI;u8zRSjT zLOi)96FLS;Qa9dpR)j~o-YPVZ=yYTvFlW~|Eo#!x(6nAoEW6$1 z?I)(F*=>lxN8nW34Lt2m2L%S=-fcwqDrsBogb+8!#>IiZly(U?`y|y?`o5ET5C~5Z z8wGo=8o4Trn3$!XHRyH}-0MTQIyJmP=XI~gf<;4<4gdE*p{f~`RT>*K9JVJ6JLA`5 zQ^BYke*oXzkke(nrMKpb`KPbK`P5G&QIV~W{EEe9+y|gn&v_Sz>l%NUOh%LAj6ZAvX^^Azo{U*vFEtL5e zx^i0Pl-{NjIBEFuXVKPT367Nd@&H;h3RWk` zIV^Zs9e}l}8LH!GkHgCvJ){ z{IQA>^=9-!%56vbSZ+i%g2~RbCELuFu%CmMcN~k)W2u94Tj$`Pw&w%1zrDT~1sY27 zBiCOcUg=sQD89rVFb@|#swBiF^PsfJ|1dpc2ALX0MY6x&$c1G28X28`6T)R)<&jNP zW2Pa4vMoBf76%AO3I!!>LeCW+G5!dx`MniALrFpor?2dXl0WPE=AEXlOU*$C_ts8K z^^;q*B@O<}Bz~Y(icHod6#}aAmk8q%mgOU-Rh?(}DQhr>8*8sJEeThNTrT6_2>ps9 zT>`X+V0H|GiLe5<=LByQ-=@g5sqC6Lj)p1T+iM>_EX%DprNtxaN;2sZlhp1mN$akE zduofQDj2A?C0c_@H1XVbq#4TLzqykzB+AuA)NHmoSJZN}^PIfKo`NiPP-sB9qJ0<$|$HbaQ@H5KK|aQFM5Gxr6AB_T4yo{_%QHCSZnjj%u8r3R+H}@ z_w)sZsU2)YSm{((Cuxj53cELlyTNE*E0~gMR+M#4Rw{(VaGzRvKZm|Zm3jwyQDbUY zgVJnzkTWZO8hRA^oj?oNSg&k)IVmq3ThUkLj@eLljSNgVjQ%g14`?eI8q(zrq)L4t z4g%pqKU9TlWFv+xE*E%qN;~C{<}WpDm@^{E1&yBmEPDC06q^rgmlcJRcU>8nO2U}Z zmj6%gc}LYAidCGBbaq-GY__ZAorqaBjlNy9+5JA2;YXQFP;c-{|RgH zs%b*r^6KAjv;IwT3Awyc*NfViavsC*p0{Mjb=HhdoX;#BDXUX7vtCj+Q7-Eu)HAGa z?|=Z9sbA67Ay&s^d&pf}O_T%wY?OLNue0aD_!|SQOg6uid=7%(Q(4}_-laqBoI-0m zamjdf9rKMm&-{V+KK|cP@cLIiT&%IX-hCoDR6doI@@+O#P~s*0Ss%~N@XKTq0*}JH z$KUWa^bEbp!KffqwKfXEvMsg6p{ly6#eULTMXl@N-_?G$?VOm7O@F?b`a0`mJ#}zv zX*?Fl0|RYixvF-B0R8Xl0?wZ6OpyVy(>AiMZ)wkMYl_WuF7cj(hu-^*SM?q+)CRN8 z%VOI9N!`3RZJHLDHLS~S?J{{YfUHI@lQ3!2nWFJaXZ`C67LkZ0F(c2tJCFz_#}H(p zg|9V2S7b8%6m^e?TJZ9*kHMfpLuLLdn^k>(t^K&=Lp5;6SJE zWaabgDD+={^$9#^xHH3Sz#);t#Ul8Xwf@!V?2=+cYjBH+A5fHxaMnBD%KcZo$>=z~ zf>%YkA{C)p>Z7@BSOIuE>|5FI&4WbUztbEcpTw}8DObcfWCRp!C-b;MalrB=#g87f z$QK8~8aQ+90iLn#)auKzKu>dbGKxT^tmK(*v~nd|b=Atxh5oV%p%-%oj$8{Ge?a){ zx^kT7ie6NQ?fvTo3I|?^wIscWkf%rg_F2ln_ivbv-6(j+uu_j*PLSX>Vwg$XsNqX! z%G|b;k*RG64D=5E3(*wdEJrPn?HysL#niN}prs=n&+l~o^}hhkoG*F;Qjy;cveQx~ zc63g5Uh$+_5w^4E6;uafk_+t1L}JSD>x(k8`4j(um-+XaGim?8muxxzsc7@x)6f6! zbkmKF8$dohFV1jiM#a#-PfC5KjeJ;uicnr70CX1*J{j|LwEdKH?wfQAJ8|U1@R?SY zV_PKuXUUp~H(xRVfIg$<^yF4i^tekjE?D63*u=1mhDWUw@55Ea0nofvsTY_x8T!3> zUgh6cIbBaEVA^qM(}?_c_2_?BkN$V{=zmv_{=cdoRY(Z<&wNp@4;#*v{y+F4B9N9- z&3E9V>Q{%N8F|Y=8$RuL;d~XF*Fm>AwTYFa4NH8(9aK$bwhfPjW{&C1uJA zu1DYQH41;59)p6ycYpoA0NyNUuFh8?2|=&67on`-?tijE>Cl5wrQ`IbO{;LAf^ta9 z;}59`-e7}R`R!-dDf?YEDX~%-ZU8hp4;mvoKn*2P64y&Xjt}Q`tH$geCa&X#|J-XZQd*8(^fE-W*1BB zGbdxOruy?SocD{w(Kbh&Aiq#w^X7T@SpiMSN~<(rf!zSy-Za&jzXZ^zbU!)>@Cx0O zh_8g(&sH`^uXk(QvOETLot+7@N=Sqj_5DtzKQm&VuTPVVO zeg6Df`F9uHSlDqbm~ZMrz9n{D5-11MsNHC{$L1sLUngU@S-;yBhhGr%z$?jK5t@0{ z`LKtkYHBqkbG|3!J*;7;&ELC|`}a{fvH<-=^!rE0T=1NPQbE3qfi zh){qwuF$~#w!G9I(3{`I{Pe#00zkgY69Qy1`rWNw{Lt)Kq+Z6bP+mYRzrHpYxN5(< zw6t}c{yEGi_{f+Dlp)Z))~CHdPD!vCv)m$2F?Ywd#mwKwBz7;RPqKQVI1#t+ z7kv$ODhBswTHj7hDacqvyaqunfRoH8yE6zZO5OJubRIIt$zJ-#(tv#bWkJ|P6RtRk z8WWNzyKsCAW*Cgu6oQ0L|NRlK!>wGL4!4J`$tz!g%HBcSiV$tYI`2c9XRZPd6z>L7 zG$SL7I;zp^tU3Xvu<-6~YA}M?&$I~!G%k=@xZKF~H3e$SH;_{C_`Ttq@ z)%~5@fC*SH>wUu8f3ShFt|M1~3~W3fEZ}kGKfNrsPoMof zxO~|>Wo2EQ9~S%Z1=S?(Q!X7=f7?7wsPY#~g@!+WAT<6`);d3xE6pl(pE@d}Dpj@R zo9fg>x{EAekc0wj0-%M4z6m8uey6B^CKiE3X8npT^yMB@MT@eN`M7e)f4Q}CpM6?a zp8VtVFOSp&;jD|lBoTr}+bc5?Yg z(LLD&=T;N83U~zhOI_xuiZtqL57naMHlPYxBCt0S^LX2ma$lofGv1(oK z_`BwZaue^Bc0Vg5?RwcUcq1|3z0g4{GgezxRt8iP#Aox%*>Vi5|14>13$Cb$?lw9< zw=Y~sSZ<*I{Mq5cvWctaBR1sEA&*sXQVP7_cTaCKoTsT83h_AS`9%>E%2)=YZzypKk>0WO(dW@n4V7ef`}m{<9xGvJDR$A-KlM%)sYhc8-x*=_PnHhbVIf? z=AON@#3S%<=NKL_8XcIk2@AE9cP~bQ*mLUmaakMtc9z66I9Ls3OsHsR_aq2f1a5ByKmvN6 z%|)}I#;8P4ORbE9v!^jtSTB#_vjSQ_<4&`rEE-xIaePfRmIyGgW{UFmK7amfn7{D_ zcq+YA$>p{YwlD2_R+T^z6B-&CIB(P6y;MC{@6%iSeX^_h$yzKLw2^uVyfsvNzuJMd zGy(!JY8fB!A?Uh8Rzdv{y4$mhjv630cZoCNla8!NWOJZy0};ySJfb}H=`bS=QZm?T zG(uKa-){H!3;Qi(-Jr?ZQqxg?u0kA_&kQQqpvLQ7obO_9y4wrr(G}Xvj8?th1t5r0 zDV@YmUWI&6w`-h9V`GRe#|&HX5g}-3Xx$tArx!gD)i6dCsdM?=do&+yWiO3IQcTFj7o4I;oiw=r!Tmwcg|e@ z{W$>o?Zd^bsp-Xi8T;0*E%e{wjDa(N2lff0JSTOC_CMWlHUlexDJuaW{z?i%sR2lb zoN)=$2tRDVZD(h_zJRYFd~c7c&2{p7ugB#h&nqCWyJKShTO zDk&(yjV614`Goq3kue8I1b;iAA_-aJ-7iyKzZ(<#WYboTKg7VSCm3j?JfZB!!+G2P zQOH$BMH#JYP)dP8L=Z_40g*0YXlam;l$4ZEqX(Xg;Xiyr7p@$f9 zU}!jd&RKV@d(Jv{-9IzG_Wt&}-{*av=l#~V_dN-T=%XVs-ufL$URo^G0JM&|0hSqz zWVXO0Gv@ikb@o=hmVjuv%OxeJi8>UPINuczz(4XjolmmyLW|WT=)73QFZ}-M@6Y#O z3pwiI$FC!Aq4OFWgEOQX#11)7J0}|$=;3s55*%LPxdEZ0r`OmeCkJQg6Ri_EWbalo z5+Nu{1Y-S-0p^mZhTP{4iNP5pt`K61W+yG%*|S_Sx{+*YXoF$X#gPWfzK`UDW2f_8 zXq&c{zP?CMy0~FbVvJbV?>$;0M7fRXeJTPUf#hoC3*-9C>?-Tf_fw%GA)qSBR%np(exUhCp@ zfue%K-(JoJuCM*Jiy?4~?_6-_k^g8K)eHkbMWS3?QM`}f>NIDfI}w&uRFtA*y!U8# zT*u6;?b)mG?A5%;h=>ef&4LBIwz=DYP6r$kcaQxYXX`U1woj;>ycZA6xd8b`iRGGe zfeKZ5z(er>Wb5E8Qpg z67k$0w6M71--)$-moTYVl{Sm53u9YqW)g%UdB|zGP7OO7lR2L4$8r9LB3ZSk8$P+} zbwW`Q5wa;+A)W_LC!LD#cp%`n>`b;{&9Jg#4jbF8j;zRLELIwbiTpC_T{@dmS#isr zj?1M+JUd%xT;%oVdJwG^^|RFLZ{-;|2ff4FJ9jkUA*qURvZy#)ufbSM@L_o+g~Ng66XoAkA7U)@k!Z( z;1%P_;rSn$kk07BE@zj;J>stK-esKuR@jFVQ>&KRxgmI^i1{zSH?Pf-HizyjJTJ+g zPfR27s*{qeuf8b{dlVfJkrH}~c!PoWm$Gua#ZMO_H-?Y*I|T2ugl9}VIBP}_Rddc# zk(92LT63tCO@S|I+fW?cV;<4h3`h$o4XVT`io%9n4rO*-dXy~1QvAj37cR*6hqvS_aO<9AE`$v{i{NwW>CXoAL$; zI=c~OhO)?P>!_(DFc^mxV2%s+PH2;FPzla{;@Ncbt>G64*Gq>#hz>pw8@cAJH{zzN zbniAt#}LoMw`*oPyF`^g!8I#gpIafLmp$`VFF7Q#$As~AR5h-KEPOQ@Z|!);s}{9< z+-7?_M(sVZ#X{^&w++Y&%*~t{;B(Is$*W*QHmrw8UV?rIN*;}8vRp2i8QLY9%PoI- z(ykO%{}5;g6rMR$C-|2vMC808wd#9@RwGX+2wcWKK)Bz^%y{PMMOqj1&99qK!|;P>3!!~Qd=@o-w~x)O2(=(NoWQdQ18&mwUAER?~;mQCYCJ6f!3IRz2I&(*75uQGn0E zXtl5VKvPI<@!2t(dW{2_S-H5jjt;^Eh~e(Sa_4Rp;Gi6+WpqeO2r>UMR(1Na*}Q8I zDW?{di5XvrSm z{O%^z|#;QM|f+p4KHR=Z7-3$1rAmei+s?(66O7 zh7l%0A+slp7cBc`)ZAauSCI|xyFn3)px}pz3EG=fR0YMw^Ht`tsG@3`GLN-Wyrkr0 z%sRSu+Go+8ni!{8vJ(q)!-I_$t)T*Eu=(Dlm6bAw;YZR(JG#J4hX@!MF&do=Tp#wF zst#(qBc(|ii1lkYI>!e(hVN1*)#46%)QQb*8=D*f*99?lO$NK3+WKgw^a zFDy)pit35SIWD(_&`oH~(t>49&ix8rzU-3|BHp_FD_8JkfI34E+=fTYTeYXvOUuiN z*-c?j_!`~_4Whe|S$>U*M#1DN|n2#ZL2HMqQ`2H4ktS+KGnS2Q{$;?!8bjR;&{Q{16dopaRH=lIov1`X3Ma z-%8|P0;bRr3cWt++37vl+K1x9m$lti=rC?5Wg`o(~;vdkPy}zB&Fx^uEfp;BZ3W29=>AW-D7Yn zuIzL*aJ&__Rph9BHH!<{d~Q^5OVsbUeoPfecaZ6=$VXg!a0BM48Q`^4qCqCs&jOvQ zI0TClGfCP$H7^4gKJ<<$Msv9_{J>S;9vd!#(d69bO<5lvitpF z2{~*lk|R&(yru4Mts<`a^R0OyvnCghSe2jup?Ph&`E0|e$x=6c=_aV%v1@fCLy>~c z7eiDt1h|glc|}Ld!FZfi-zp`bu|*fLuHLDxs}utSccrCc;an?(*4w9xjMImm5_l=c zM?&0YC(?M7l#lW>&qbb{q3>M@#>1?H1sx5h9gu_&lQ;8|CG%ZIC^H58^|-h=&HIp= z+EP3dX6m-f4_0Pae`NODXZNRd9Snqgjj~FNR?-Jg*D1UGoZ>7Dv@$t46VS^umKF>| zSxSD89wOXe;InlcL`eEpUZdWZ;iEZLmnq{yEH`&qJykF=mQhH4Ax|S6BK#g|KhPX} z?i19}*_U=eK}5FNi_FgKMtvWP&u+?n;mr*mu6)a@?=eaQ^QLgCy5EJ;P372aXx&=!tw3d#s?e% zDq3lXvjee&`ud|d;W+a}Y}Y!WRdjSdm?VjTom*PE?}g?jVdCOLYH4VgeWx=vHg-BO zHAl)?S;>8O!%cw!Q-I;q{1>8R7jTt;-w%;XIWI_Jtj zXQzYa;o}G6a?U|b^Va&%S}VnUf*m@Fd#Korh{$l<*uiMDY{gGVg(+ru&sw0dHuqRr z%M2lTrZp~izLqd*8o#_w6CI|S=p>wZ8`q0yO-x>oJ@G@luM6)^bVk@#l-X%f9wIMk z!@^GGtg-{JFTjYBQC0xU%$GN7-v}`{CKj`_|)2{qyIr zFX=dv=>c!XQc@DykM;?~mHy@*92@DP76!~TBv{%1)tMlqheX~h|H{MXZ*Y+z#fom$ z(^}F_e}uP2B>7wszTCM)-AT3@`4Lf^LU?%iFo9i9(6BJxqgUbv>L;`)ZUpeaUr-Jg!^Z4c3193q-~`*Z*g!I&-X#a?|%$FHQ>d&h`UR#UghE}*olM>(|EvLkNO=|lSW%|m-uyP>J zDwwqM@hQ;56pnQZu@TcY{G_ex?`f`Cjjut-z9C`V7cfI%be9kQ9K;I{ArIKWkgjen zM^_xn1La?%q*ArbdA~7qlZ4=kqc`yBTmRNj#EZ`tMTsN6aRA%j@v?c7aWfACLUh}^ zCSmeju%yEiO88Ocv}-IOVhCaGd-^IBxUe|8SHpL|y>Gdj}B<}PfIHqaA&I%3Zx z;i9d>J;?m%iKX64Njxg=qW!zWsHspaXYuwQSBOBS literal 0 HcmV?d00001 diff --git a/docs/src/devdocs/binaries_part_2.md b/docs/src/devdocs/binaries_part_2.md new file mode 100644 index 00000000..229b64c2 --- /dev/null +++ b/docs/src/devdocs/binaries_part_2.md @@ -0,0 +1,239 @@ +# [Creating a binary from Julia code](@id man-tutorial-binary) + +This section targets how to build an executable based on the custom sysimage so +that it can be run without having to explicitly start a Julia session. + +## Interacting with Julia through `libjulia`. + +The way to interact with Julia without using the Julia executable itself is by +calling into the Julia runtime library (`libjulia`) from a C program. A quite +detail set of docs for how this is done can be found at the [embedding chapter +in the Julia manual](https://docs.julialang.org/en/v1/manual/embedding/) and it +is recommended to read before reading on. Since this is supposed to highlight +the interals of PackageCompilerX, will not use the conveniences shown in that +section (e.g. the `julia-config.jl` script) but it is good to know they exist. + +A rough outline of the steps we will take to create an executable are: + +- Create our Julia app with a `Base.@ccallable` entry-point which means the Julia + function can be called directly from C. +- Create a custom sysimage to reduce latency (this is pretty much just doing + part 1) and to hold the C-callable function from the first step. +- Write an embedding wrapper in C that loads our custom sysimage, does some + initialization and calls the entry point in the script. + +## A toy application + +To have something concrete to work with we will create a very simple +application. Keeping with the spirit of CSV parsing, we will create a small +app that parses a list of CSV files given as arguments to the app and prints +the size of the parsed result. The code for the app (`MyApp.jl`) is shown +below: + +```jl +module MyApp + +using CSV + +Base.@ccallable function julia_main()::Cint + try + real_main() + catch + Base.invokelatest(Base.display_error, Base.catch_stack()) + return 1 + end + return 0 +end + +function real_main() + for file in ARGS + if !isfile(file) + error("could not find file $file") + end + df = CSV.read(file) + println(file, ": ", size(df, 1), "x", size(df, 2)) + end +end + +if abspath(PROGRAM_FILE) == @__FILE__ + real_main() +end + +end # module +``` + +The function `julia_main` has been annotated with `Base.@ccallable` which means +that a function with the unmangled name will appear in the sysimage. This +function is just a small wrapper function that calls out to `real_main` which +does the actual work. All the code that is executed is put inside a try-catch +block since the error will otherwise happen in the C-code where the backtrace +is not very good + +To facilitate testing, we [check if the file was directly +executed](https://docs.julialang.org/en/v1/manual/faq/#How-do-I-check-if-the-current-file-is-being-run-as-the-main-script?-1) +and in that case, run the main function. We can test (and time) the script on +the sample CSV file [from the first tutorial](@ref man-tutorial-sysimage) + +``` +❯ time julia MyApp.jl FL_insurance_sample.csv +FL_insurance_sample.csv: 36634x18 +julia MyApp.jl FL_insurance_sample.csv 12.51s user 0.38s system 104% cpu 12.385 total +``` + +## Create the sysimage + +As in the previous tutorial, we do a "sample run" of our app to record what +functions end up getting compiled. Here, we simply run the app on the sample +CSV file since that should give good "coverage": + +```jl +julia --startup-file=no --trace-compile=app_precompile.jl MyApp.jl "FL_insurance_sample.csv" +``` + +The `create_sysimage.jl` script look similar to before with the exception that +we added an include of the app file inside the anonymous module where the +precompiliation statements are evaluated in: + +```jl +Base.init_depot_path() +Base.init_load_path() + +@eval Module() begin + Base.include(@__MODULE__, "MyApp.jl") + for (pkgid, mod) in Base.loaded_modules + if !(pkgid.name in ("Main", "Core", "Base")) + eval(@__MODULE__, :(const $(Symbol(mod)) = $mod)) + end + end + for statement in readlines("app_precompile.jl") + try + Base.include_string(@__MODULE__, statement) + catch + # See julia issue #28808 + Core.println("failed to compile statement: ", statement) + end + end +end # module + +empty!(LOAD_PATH) +empty!(DEPOT_PATH) +``` + +The sysimage is then created as before: + +``` +❯ julia --startup-file=no -J"/home/kc/julia/lib/julia/sys.so" --output-o sys.o custom_sysimage.jl + +❯ gcc -shared -o sys.so -fPIC -Wl,--whole-archive sys.o -Wl,--no-whole-archive -L"/home/kc/julia/lib" -ljulia +``` + +### Windows-specific flags + +For Windows we need to tell the linker to export all symbols via the flag `-Wl,--export-all-symbols`. +Otherwise, the linker will fail to find `julia_main` when we build the executable. + +## Creating the executable + +### Embedding code + +The embedding script is the "driver" of the app. It initializes the julia +runtime, does some other initialization, calls into our `julia_main` and then +does some cleanup when it returns. We can borrow a lot for this embedding +script from the embedding manual there are however some things we ne +ed to set up +"manually" that Julia usually does by itself when starting Julia. This +includes assigning the `PROGRAM_FILE` variable as well as updating `Base.ARGS` +to contain the correct values. The script `MyApp.c` ends up looking like: + +```c +// Standard headers +#include +#include + +// Julia headers (for initialization and gc commands) +#include "uv.h" +#include "julia.h" + +JULIA_DEFINE_FAST_TLS() + +// Forward declare C prototype of the C entry point in our application +int julia_main(); + +int main(int argc, char *argv[]) +{ + uv_setup_args(argc, argv); + + // initialization + libsupport_init(); + + // JULIAC_PROGRAM_LIBNAME defined on command-line for compilation + jl_options.image_file = JULIAC_PROGRAM_LIBNAME; + julia_init(JL_IMAGE_JULIA_HOME); + + // Initialize Core.ARGS with the full argv. + jl_set_ARGS(argc, argv); + + // Set PROGRAM_FILE to argv[0]. + jl_set_global(jl_base_module, + jl_symbol("PROGRAM_FILE"), (jl_value_t*)jl_cstr_to_string(argv[0])); + + // Set Base.ARGS to `String[ unsafe_string(argv[i]) for i = 1:argc ]` + jl_array_t *ARGS = (jl_array_t*)jl_get_global(jl_base_module, jl_symbol("ARGS")); + jl_array_grow_end(ARGS, argc - 1); + for (int i = 1; i < argc; i++) { + jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]); + jl_arrayset(ARGS, s, i - 1); + } + + // call the work function, and get back a value + int ret = julia_main(); + + // Cleanup and gracefully exit + jl_atexit_hook(ret); + return ret; +} +``` + +## Building the executable + +We now have all the pieces needed to build the executable; a sysimage and a driver script. +It is compiled as: + +``` +❯ gcc -DJULIAC_PROGRAM_LIBNAME=\"sys.so\" -o MyApp MyApp.c sys.so -O2 -fPIE \ + -I'/home/kc/julia/include/julia' \ + -L'/home/kc/julia/lib' \ + -ljulia \ + -Wl,-rpath,'/home/kc/julia/lib:$ORIGIN' +``` + +where we have added an `rpath` entry into the executable so that the julia +library can be found at runtime as well as the `sys.so` library ($ORIGIN means +to look in the same folder as the binary for shared libraries). + +``` +❯ time ./MyApp FL_insurance_sample.csv +FL_insurance_sample.csv: 36634x18 +./MyApp FL_insurance_sample.csv 0.19s user 0.09s system 242% cpu 0.115 total + +❯ ./MyApp non_existing.csv +ERROR: could not find file non_existing.csv +Stacktrace: + [1] error(::String) at ./error.jl:33 + [2] real_main() at /home/kc/MyApp/MyApp.jl:21 + [3] julia_main() at /home/kc/MyApp/MyApp.jl:7 +``` + +### macOS considerations + +On macOS, instead of `$ORIGIN` for the `rpath`, use `@executable_path`. + +### Windows considerations + +On Windows, it is recommended to increase the size of the stack from the +default 1 MB to 8MB which can be done by passing the `-Wl,--stack,8388608` +flag. Windows doesn't have (at least in an as simple way as Linux and macOS) +the concept of `rpath`. The goto solution is to either set the `PATH` +environment variable to the Julia `bin` folder or alternatively copy paste all +the libraries in the Julia `bin` folder so they sit next to the executable. + diff --git a/docs/src/devdocs/intro.md b/docs/src/devdocs/intro.md new file mode 100644 index 00000000..0fe72bdb --- /dev/null +++ b/docs/src/devdocs/intro.md @@ -0,0 +1,25 @@ +# Introduction + +This part of the documentation contains a set of tutorial to teach how +PackageCompilerX works internally by going through the steps for the f. By knowing the internals of +PackageCompilerX you can more easily figure out root causes of problems and +help other. + +The inner functionality of PackageCompilerX is actually quite simple. +There are a few julia commands and compiler invocations that everything +is built around, the rest is just scaffolding. + +[Part 1](@ref man-tutorial-sysimage) focuses on how to build a local system +image to reduce package load times and reduce the latency that can occur when +calling a function for the first time. [Part 2](@ref man-tutorial-binary) +targets how to build an executable based on the custom sysimage so that it can +be run without having to explicitly start a Julia session. [Part 3](@ref man-tutorial-reloc) +details how to bundle that executable together with the Julia libraries and +other files needed so that the bundle can be sent to and run on a different +system where Julia might not be installed. These functionalities are exposed +from PackageCompilerX as [`create_sysimage`](@ref) and [`create_app`](@ref). + +It should be noted that there is some usage of non-documented Julia functions +and flags. They have not been changed for quite a long time (and are unlikely +to change too much in the future) but some care should be taken. + diff --git a/docs/src/devdocs/relocatable_part_3.md b/docs/src/devdocs/relocatable_part_3.md new file mode 100644 index 00000000..0a627956 --- /dev/null +++ b/docs/src/devdocs/relocatable_part_3.md @@ -0,0 +1,357 @@ +# [Relocatable apps](@id man-tutorial-reloc) + +In the previous tutorials, we created a custom sysimage and a binary (app) that +did some simple CSV parsing with an (depending on the exact demands) acceptable +latency (time until the app starts doing real work). However, trying to send +this executable to another machine will fail spectacularly. This tutorial +outlines how to create and package a bundle of files into an app that we can +send to other machines and have them run, without for example, requiring Julia +itself to be installed, and without having to ship the source code of the app. + +The tutorial will not deal with any kind of file size optimization or "tree +shaking" as it is sometimes called. + +## Why is the built executable in the previous tutorial non-relocatable? + +With relocatability, we mean the ability of being able to send e.g. an +executable (or a bundle of files including an executable, here called an app) +to another machine and have it run there without too many assumptions of the +state of the other machine. Relocatability is not an absolute measure, most +apps assume some properties of the machine they will run on (like graphics +drivers if one want to show graphics) but other (implicit) assumptions, like +embedding absolute paths into source code would make the app almost completely +non-relocatable since that absolute path is unlikely to exist on another +machine. The goal here is to make our app relocatable enough such that if we +could install and run the same Julia as we use to build the app on the other +machine, then the app should also run on that machine (with exceptions if some +of our dependencies impose extra requirements on the machine). + +So what is causing our executable that we built in the previous tutorial to not +be relocatable? Firstly, our sysimage relies on `libjulia` which we currently +load from the Julia directory and, in addition, `libjulia` itself relies on +other libraries (like LLVM) to work. And secondly, the packages we embedded in +the sysimage might have encoded assumptions about the current system into their +code. + +The first problem is quite easy to fix while the second one is harder since +some popular packages that we might want to use as dependencies are inherently +non-relocatable. There is nothing to do about that except try to fix these +packages. + +For now, we will ignore the problem of packages not being relocatable by only +using a small dependency that we know does not have a relocatability problem. +Later in the blog post, we will revisit this and discuss more in-depth what +makes a package non-relocatable and how to fix this, even if the package needs +things like external libraries or binaries (spoiler alert: it is using the +artifact system presented in [the blog about +artifacts](https://julialang.org/blog/2019/11/artifacts). + +## A toy app + +The package we used in the previous examples to create a sysimage and +executable was CSV.jl. Now, to simplify things, we will only use a very simple +package with no relocatability problems that also has no dependencies. The +app will take some input on stdin and print it out with color to the terminal +using the [Crayons.jl](https://github.com/KristofferC/Crayons.jl) package. + +When we add the Crayons.jl package we use a separate project to encapsulate +things better by creating a new project in the app directory: + +``` +~/MyApp +❯ julia -q --project=. + +julia> using Pkg; Pkg.add("Crayons") + Updating registry at `~/.julia/registries/General` + Updating git-repo `https://github.com/JuliaRegistries/General.git` + Resolving package versions... + Updating `~/MyApp/Project.toml` + [a8cc5b0e] + Crayons v4.0.1 + Updating `~/MyApp/Manifest.toml` + [a8cc5b0e] + Crayons v4.0.1 +``` + +The code for the app itself is quite simple: + +```jl +module MyApp +using Crayons + +Base.@ccallable function julia_main()::Cint + try + real_main() + catch + Base.invokelatest(Base.display_error, Base.catch_stack()) + return 1 + end + return 0 +end + +function real_main() + Crayons.FORCE_COLOR[] = true + color = :red + for arg in ARGS + if !(arg in ["red", "green", "blue"]) + error("invalid color $arg") + end + color = Symbol(arg) + end + c = Crayon(foreground=color) + r = Crayon(reset=true) + while !eof(stdin) + txt = String(readavailable(stdin)) + print(r, c, txt, r) + end + return 0 +end +if abspath(PROGRAM_FILE) == @__FILE__ + real_main() +end +end # module +``` + +It got the same high-level structure as the previous app in the earlier parts. +The exact details are not so interesting but here a color is set based on the +command-line arguments and the `stdin` is written to `stdout` with that color. +We can see some usage of it: + +![](app.png) + + +## Precompilation and sysimage + +As in part 1 we generate precompilation statements and create a system image. +When recording precompilation statements and creating the sysimage, we make +sure to use the `--project` flag to use the packages declared in the local +project: + +``` +~/MyApp +❯ echo "Hello, this is some stdin" | julia --project --startup-file=no --trace-compile=app_precompile.jl MyApp.jl green +``` + +The `.o` file is then created with the same `generate_sysimage.jl` file as in part 2: + +``` +~/MyApp +❯ gcc -shared -o sys.so -Wl,--whole-archive sys.o -Wl,--no-whole-archive -L"/home/kc/julia/lib" -ljulia +``` + +And then the sysimage is linked: + +``` +~/MyApp +❯ gcc -shared -o sys.so -Wl,--whole-archive sys.o -Wl,--no-whole-archive -L"/home/kc/julia/lib" -ljulia +``` + +Before moving on and creating the executable, we need to think about what other +files we need for the app and the file structure we want. + +## File structure for our app bundle + +We already mentioned that `libjulia` has some dependencies. Using `ldd`, we +can see the dependencies and where the dynamic linker would load them from: + +``` +~/julia/lib +❯ ldd libjulia.so + linux-vdso.so.1 (0x00007ffec63c3000) + libLLVM-6.0.so => /home/kc/julia/lib/./julia/libLLVM-6.0.so (0x00007f925ef13000) + libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f925eeea000) + librt.so.1 => /lib/x86_64-linux-gnu/librt.so.1 (0x00007f925eedf000) + libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f925eebc000) + libstdc++.so.6 => /home/kc/julia/lib/./julia/libstdc++.so.6 (0x00007f925eb3e000) + libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f925e9ef000) + libgcc_s.so.1 => /home/kc/julia/lib/./julia/libgcc_s.so.1 (0x00007f925e7d5000) + libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f925e5e4000) + /lib64/ld-linux-x86-64.so.2 (0x00007f9262356000) +``` + +So some libraries would be loaded from the (`libdl`, `librt`) system itself, +and some are bundled with Julia (`libLLVM`, `libstdc++` etc) in the `julia` +folder inside `lib`. The reason the dynamic linker finds the libraries in the +subfolder is due to the `rpath` which can be seen with `objdump`: + +``` +❯ objdump -x libjulia.so |grep RPATH + RPATH $ORIGIN/julia:$ORIGIN +``` + +However, these are not the only libraries Julia (and its standard libraries) +need. Libraries can also be dynamically opened at runtime (with +[dlopen](https://linux.die.net/man/3/dlopen)). For now, we will just bring all +the libraries in `lib/julia` along (excluding the sysimage since we will use +our sysimage). + +The plan is that on macOS and Linux the files are structured as: + +``` +├── bin +│   └── MyApp [executable] +│   └── sys.so +└── lib + ├── julia + │   ├── libamd.so -> libamd.so.2.4.6 + │   ├── libamd.so.2 -> libamd.so.2.4.6 + │   ├── libamd.so.2.4.6 + │   ├── libcamd.so -> libcamd.so.2.4.6 + ... ... + │   └── libz.so.1.2.11 + ├── libjulia.so -> libjulia.so.1.3 + ├── libjulia.so.1 -> libjulia.so.1.3 + └── libjulia.so.1.3 +``` + +On Windows, we will just store everything in `bin` due to no convenient way of using `RPATH`. + +We create a new folder `lib` and copy the libraries into it (and remove the +sysimage, since we will create cusom sysimage anyway): + +``` +~/MyApp +❯ mkdir lib + +~/MyApp +❯ cp -r ~/julia/lib/ . + +~/MyApp +❯ rm lib/julia/sys.so +``` + +## Creating the binary and the bundle + +With some tweaks to the `rpath` entry so that the executable can find +`libjulia` the executable is created in the same way as in the previous tutorial. + +``` +~/MyApp +❯ gcc -DJULIAC_PROGRAM_LIBNAME=\"sys.so\" -o MyApp MyApp.c sys.so -O2 -I'/home/kc/julia/include/julia' -L'/home/kc/julia/lib' -fpie -Wl,-rpath,'$ORIGIN:$ORIGIN/../lib' -ljulia +``` + +We then finally move the executable and the sysimage to the `bin` folder: + +``` +~/MyApp +❯ mkdir bin + +~/MyApp +❯ cp MyApp sys.so bin/ +``` + +![](appexe.png) + +The final bundle of our relocatable app is then created by putting the `bin` +and `lib` folders into an archive: + + +``` +~/MyApp +❯ mkdir MyApp + +~/MyApp +❯ cp MyApp sys.so bin/ + +~/MyApp +❯ tar czvf MyApp.tar.gz MyApp +MyApp/ +MyApp/bin/ +MyApp/bin/MyApp +MyApp/bin/sys.so +MyApp/lib/ +MyApp/lib/julia/ +... +``` + +### macOS consideration + +On macOS we need to run `install_name_tool` to make it use the `rpath` entries +which is done by executing: + +``` +install_name_tool -change sys.so @rpath/sys.so MyApp` +``` + + +## Information about source code and build machine state stored in resulting app + +It should be noted that there is some state from the machine where the sysimage +and binary is built that can be observed and the original source code. Using +the [`strings`](https://linux.die.net/man/1/strings) application we can see what strings are embedded in +an executable or library. Running it and grepping for some relevant substrings +we can see that a bunch of absolute paths are stored inside the sysimage: + +``` +~/MyApp/MyApp/lib/julia +❯ strings sys.so | grep /home/kc +/home/kc/.julia/packages/Crayons/P4fls/src/downcasts.jl +/home/kc/.julia/packages/Crayons/P4fls/src/crayon.jl +/home/kc/.julia/packages/Crayons/P4fls/src/crayon_stack.jl +/home/kc/MyApp/MyApp.jl +/home/kc/.julia/packages/Crayons/P4fls/src/Crayons.jl +/home/kc/.julia/packages/Crayons/P4fls/src/crayon_wrapper.jl +/home/kc/.julia/packages/Crayons/P4fls/src/test_prints.jl +/home/kc/.julia/packages/Crayons/P4fls/src/macro.jl +``` + +In addition, when we print the stacktrace upon failure in the main function, +we also leak absolute paths of the build machine: + +``` +~/MyApp +❯ MyApp/bin/MyApp purple +ERROR: invalid color purple +Stacktrace: + [1] error(::String) at ./error.jl:33 + [2] real_main() at /home/kc/MyApp/MyApp.jl:20 + [3] julia_main() at /home/kc/MyApp/MyApp.jl:6 +``` + +This could be avoided by not printing stacktraces and perhaps even binary +patching out the paths in the sysimage (not covered in this blog post). + +The lowered code can also be read by loading the sysimage and using e.g. `@code_lowered` +on methods. + +## Relocatability of Julia packages + +The main problem with relocatability of Julia packages is that many packages +are encoding fundamentally non-relocatable information *into the source code*. +As an example, many packages tend to use a `build.jl` file (which runs when the +package is installed) that looks something like: + +``` +lib_path = find_library("libfoo") +write("deps.jl", "const LIBFOO_PATH = $(repr(lib_path))") +``` + +The main package file then contains + +``` +if !isfile("../build/deps.jl") + error("run Pkg.build(\"Package\") to re-build Package") +end +include("../build/deps.jl") + +function __init__() + libfoo = Libdl.dlopen(LIBFOO_PATH) +end +``` + +The problem here is that `deps.jl` contains an absolute path to the library and +this gets encoded into the source code of the package. If we would store the +package in the sysimage and try use it on another system, it would error when +initialized since the `LIBFOO_PATH` variable is not valid on the other system. +However, sometimes we need to bundle libraries and data files since the package +uses them. Fortunately, there is a plan for that which can be seen in the [blog +post about artifacts](https://julialang.org/blog/2019/11/artifacts). + +The idea is that with the new artifact system a file (`Artifacts.toml`), a +package can declaratively list external libraries and files that it needs. In +addition, the artifact system provides a way to find these files at runtime in +a deterministic way. It is then possible to make sure that all artifacts needed +for the package is bundled in the app and can also be found by the package +during runtime. + +The details are left out here since they become a bit technical but it should +give some incentive to switch to the artifact system. + diff --git a/docs/src/devdocs/sysimages_part_1.md b/docs/src/devdocs/sysimages_part_1.md new file mode 100644 index 00000000..d875a94d --- /dev/null +++ b/docs/src/devdocs/sysimages_part_1.md @@ -0,0 +1,473 @@ +# [Creating a sysimage](@id man-tutorial-sysimage) + +## Julia's compilation model and sysimages + +Julia is a JIT-compiled language. More specifically, functions are compiled +just before getting executed. A more suitable description of the Julia +compilation model might, therefore, be Just-Ahead-of-Time (JAOT) compilation. +The term JIT is sometimes used to describe the compilation model where code is +dynamically recompiled based on runtime performance data, which Julia does not +do. At the same time, Julia comes with a lot of built-in functionality +including several standard libraries. If all this built-in functionality would +need to be parsed, type inferred and compiled every time Julia started, the +startup-time would be longer than reasonable. Therefore, Julia bundles +something called a "sysimage" which is a [shared +library](https://en.wikipedia.org/wiki/Library_(computing)#Shared_libraries) +where (roughly) the state of a running Julia session has been stored +(serialized). When Julia starts, this sysimage gets loaded, which is a quite +quick process (50ms on the author's machine), and all the cached compiled code +can immediately be used, without requiring any compilation. + +## Custom sysimages + +There are cases where one wants to generate a custom sysimage for a similar +reason as to why Julia bundles one, to reduce time from Julia start until the +program is executing. The time from startup to execution is here denoted as +"latency" and we want to minimize the latency of our program. A drawback of +putting a package inside the sysimage is that it becomes "frozen" at the +particular version it was when it got put into the sysimage. In addition, all +the dependencies of the package put into the sysimage will be frozen in the +same manner. In particular, it will no longer be updated like normal packages +when using the package manager. In some cases, other ways of reducing latency +might be preferable, for example, using [Revise.jl](https://github.com/timholy/Revise.jl) + +## Example workload + +To have something concrete to work with, let's assume we have a small script +that reads a CSV-file and computes some statistics on it. As an example, we +will use a sample CSV file containing Florida insurance data, which can be +downloaded [from +here](http://spatialkeydocs.s3.amazonaws.com/FL_insurance_sample.csv.zip). + +One way of loading this file into Julia is by using the `CSV.jl` package. We +can install `CSV.jl` using the Julia package manager `Pkg` as: + +```jl-repl +julia> import Pkg; Pkg.add("CSV") + Resolving package versions... + Updating `~/.julia/environments/v1.3/Project.toml` + [336ed68f] + CSV v0.5.13 + Updating `~/.julia/environments/v1.3/Manifest.toml` + [no changes] +``` + +When a package is loaded for the first time it gets "precompiled": + +```jl-repl +julia> @time using CSV +[ Info: Precompiling CSV [336ed68f-0bac-5ca0-87d4-7b16caf5d00b] + 13.321758 seconds (2.69 M allocations: 151.302 MiB, 0.05% gc time) +``` + +The term "precompiled" can be a bit misleading since there is no native +compiled code cached in the precompilation file. Julia is dynamically typed so +it is not obvious what types to compile the different methods for. + +Even with `CSV` "precompiled", there is a still some loading time but it is +significantly lower: + +```jl-repl +julia> @time using CSV + 0.694224 seconds (1.90 M allocations: 114.210 MiB) +``` + +Let's load the sample CSV file: + + +```jl-repl +julia> @time CSV.read("FL_insurance_sample.csv"); +9.264898 seconds (37.17 M allocations: 2.278 GiB, 3.90% gc time)1 +``` + +That's is quite a long time to read a smallish CSV file. One way to check +the compilation overhead is by running the function again: + +```jl-repl +julia> @time CSV.read("FL_insurance_sample.csv"); + 0.083543 seconds (423 allocations: 34.695 KiB) +``` + +So clearly, the first call to the function is dominated by compilation time. +In many cases, this is not a problem in practice since often one wants to parse +multiple CSV files such that the overhead will become negligible or one keeps a +Julia session open for a longer time so that the compiled version of the +function is still in memory. + +However, since the end goal of this blog series is to create an executable that +can be distributed we want to try to avoid as much runtime compilation +(latency) as possible + +## Creating a custom sysimage + +If we time the loading of a standard library it is clear that it is "cached" +somehow since the time to load it is so short: + +```jl-repl +julia> @time using Dates + 0.000816 seconds (1.25 k allocations: 65.625 KiB) +``` + +Since `Dates` is a standard library it comes bundled in the system image. In +fact, `Dates` is already "loaded" when starting Julia, the effect of running +`using Dates` just makes the module available in the `Main` module namespace +which is what the REPL evaluates in. + +Delving into some internals, there is a dictionary in `Base` that keeps track +of all loaded modules: + +``` +julia> Base.loaded_modules +Dict{Base.PkgId,Module} with 33 entries: + SHA [ea8e919c-243c-51af-8825-aaa63cd721ce] => SHA + Profile [9abbd945-dff8-562f-b5e8-e1ebf5ef1b79] => Profile + Dates [ade2ca70-3891-5945-98fb-dc099432e06a] => Dates + Mmap [a63ad114-7e13-5084-954f-fe012c677804] => Mmap +... +``` + +and we can here see the `Dates` module is there even after restarting Julia +and not explicitly loading `Dates`, showing that `Dates` is loaded together with +the start-up of Julia. + +Creating and using a custom sysimage is done in three steps: + +1. Start Julia with the `--output-o=sys.o custom_sysimage.jl` where + `custom_sysimage.jl` is a file that creates the state that we want the + sysimage to contain and `sys.o` is the resulting [object + file](https://en.wikipedia.org/wiki/Object_file) that we will turn into a + sysimage. +2. Create a shared library from the object file by linking it with `libjulia`. + This is the actual sysimage. +3. Use the custom sysimage in Julia with the `-Jpath/to/sysimage` (or the + longer, more descriptive `--sysimage`) flag. + +### 1. Creating the object file + +For now, the goal is to put `CSV` in the sysimage (in the same way as the +standard library `Dates` is in it). We therefore initially simply create a file +called `custom_sysimage.jl` with the content. + +```jl +using CSV +``` + +in a `custom_sysimage.jl` file. Let's try using the flag `--output-o` (and +disabling using the startup file) and running the file: + +``` +julia --startup-file=no --output-o=sys.o -- custom_sysimage.jl +ERROR: could not open file boot.jl +``` + +That didn't work well. It turns out that when using the `--output-o` option one +has to explicitly give a sysimage path ([due to this +line](https://github.com/JuliaLang/julia/blob/49fb7924498e9fe813444cc684a24002e75b2ac9/src/jloptions.c#L533)). Since we don't have a custom sysimage yet we +just want to give the path to the default sysimage which we can get the path to +via: + +``` +julia> unsafe_string(Base.JLOptions().image_file) +"/home/kc/julia/lib/julia/sys.so" +``` + +Let's try again, specifying the default sysimage path with the `-J` flag: + +``` +julia --startup-file=no --output-o sys.o -J"/home/kc/julia/lib/julia/sys.so" custom_sysimage.jl +signal (11): Segmentation fault +in expression starting at none:0 +uv_write2 at /workspace/srcdir/libuv/src/unix/stream.c:1397 +uv_write at /workspace/srcdir/libuv/src/unix/stream.c:1492 +jl_uv_write at /buildworker/worker/package_linux64/build/src/jl_uv.c:476 +uv_write_async at ./stream.jl:967 +uv_write at ./stream.jl:924 +``` + +Failure again! Another caveat when using `--output-o` is that modules +`__init__()` functions do not end up getting called which is what normally +happens when a module is loaded. The reason for this is that often the state +that gets defined in `__init__` is not something that you want to serialize to +a file. In this particular case, some parts of the IO system has not been +initialized so Julia crashes while trying to print an error. The magic +incantation to make IO work properly is `Base.reinit_stdio()` so to figure out +the actual problem we modify the `custom_sysimage.jl` file to look like: + +``` +Base.reinit_stdio() +using CSV +``` + + +and rerun the julia-command: + +``` +julia --startup-file=no --output-o sys.o -J"/home/kc/julia/lib/julia/sys.so" custom_sysimage.jl +ERROR: LoadError: ArgumentError: Package CSV not found in current path: +- Run `import Pkg; Pkg.add("CSV")` to install the CSV package. + +Stacktrace: + [1] require(::Module, ::Symbol) at ./loading.jl:887 + [2] include at ./boot.jl:328 [inlined] + [3] include_relative(::Module, ::String) at ./loading.jl:1105 + [4] include(::Module, ::String) at ./Base.jl:31 + [5] exec_options(::Base.JLOptions) at ./client.jl:295 + [6] _start() at ./client.jl:468 +in expression starting at /home/kc/custom_sysimage.jl:2 +``` + +Okay, now we can see the error. Julia can for some reason not find the `CSV` +package. Package-loading in Julia is based on the two arrays `LOAD_PATH` and +`DEPOT_PATH`. Adding `@show LOAD_PATH` and `@show DEPOT_PATH` to the +`custom_sysimage.jl` file and rerunning the command above prints: + +```jl +LOAD_PATH = String[] +DEPOT_PATH = String[] +``` + +So again, we have an initialization problem. Looking at [what Julia itself does +before including the standard libraries][sysimage-path-init], we can see that +the functions initializing these variables are explicitly called. Let's do the +same by updating the `custom_sysimage.jl` file to: + +```jl +Base.init_depot_path() +Base.init_load_path() + +using CSV + +empty!(LOAD_PATH) +empty!(DEPOT_PATH) +``` + +and running + +``` +julia --startup-file=no --output-o sys.o -J"/home/kc/julia/lib/julia/sys.so" custom_sysimage.jl +``` + +This time, after some waiting (2 min on the authors quite beefy computer) we do +end up with a `sys.o` file. + +### 2. Creating the sysimage shared library from the object file + +The goal in this part is to take the object file, link it with `libjulia` to +finally produce a shared library which is our sysimage. For this, we need to +use a C-compiler e.g. `gcc`. We need to link with `libjulia` so we need to give +the compiler the path to where the julia library resides which can be gotten +by: + +```jl-repl +julia> abspath(Sys.BINDIR, Base.LIBDIR) +"/home/kc/julia/lib" +``` + +We tell `gcc` that we want a shared library with the `-shared` flag and to keep +all symbols into the library by passing the `--whole-archive` to the linker +(this is on Linux, see the later section for platform differences). The final +`gcc` invocation ends up as: + +``` +gcc -shared -o sys.so -Wl,--whole-archive sys.o -Wl,--no-whole-archive -L"/home/kc/julia/lib" -ljulia +``` + +which creates the sysimage `sys.so`. + +We can compare the size of the new sysimage vs the default one and see that the +new is a bit larger due to the extra packages it contains: + +``` +julia> stat("sys.so").size / (1024*1024) +162.16205596923828 + +julia> stat(unsafe_string(Base.JLOptions().image_file)).size / (1024*1024) +147.0646743774414 +``` + +#### Platform differences + +##### macOS + +On `macOS` the linker flag `-Wl,--whole-archive` is instead written as +`-Wl,-all_load` so the command would be + +``` +gcc -shared -o sys.dylib -Wl,-all_load sys.o -L"/home/kc/Applications/julia-1.3.0-rc4/lib" -ljulia +``` + +Note that the extension has been changed from `so` to `dylib` which is the +convention for shared libraries on macOS. + +##### Windows + +Getting a compiler toolchain on Windows that works well with Julia is a bit +trickier than on Linux or macOS. One quite simple way is to follow the same +process as needed to compile Julia on windows as outlined +[here](https://github.com/JuliaLang/julia/blob/master/doc/build/windows.md#cygwin-to-mingw-cross-compiling) and then use the `x86_64-w64-mingw32-gcc` +compiler in Cygwin instead of `gcc`. Alternatively, a mingw compiler can be +downloaded [from +here](https://sourceforge.net/projects/mingw-w64/files/mingw-w64/) The +`libjulia` is also in a different location on Windows. Instead of the `lib` +folder it is in the `bin` folder. Other than that, the same flags as for Linux +should work to produce the sysimage shared library. + +### 3. Running Julia with the new sysimage + +We start Julia with the `-Jsys.so` flag to load the new custom `sys.so` sysimage (or `sys.dylib`, `sys.dll` on macOS and Windows respecitively) +and indeed loading CSV is now very fast: + +```jl-repl +julia> @time using CSV + 0.000432 seconds (665 allocations: 32.656 KiB) +``` + +In fact, restarting Julia and looking at `Base.loaded_modules` we can see that, just like the standard libraries, CSV and +its dependencies are already loaded when Julia is started: + +```jl-repl +julia> Base.loaded_modules +Dict{Base.PkgId,Module} with 52 entries: + Parsers [69de0a69-1ddd-5017-9359-2bf0b02dc9f0] => Parsers +... + CSV [336ed68f-0bac-5ca0-87d4-7b16caf5d00b] => CSV +... +``` + +However, remember that a large part of the latency was not loading the package +but to compile the functions used by CSV the first time. Let's try it with the +custom sysimage: + + +```jl-repl +julia> @time using CSV + 0.001487 seconds (711 allocations: 35.203 KiB) + +julia> @time CSV.read("FL_insurance_sample.csv"); + 3.609626 seconds (16.34 M allocations: 795.619 MiB, 5.88% gc time) + +julia> @time CSV.read("FL_insurance_sample.csv"); + 0.026917 seconds (423 allocations: 34.695 KiB) +``` + +Reading the CSV file is significantly faster than before but still a lot slower +than the second time. As previously mentioned, the native code for the +functions in CSV is not compiled just by loading the package. This means that +even though CSV is in the sysimage the functions in CSV still need to be +compiled. The reason why the first call is faster at all is likely that +loading packages can invalidate other methods and they thus have to be +recompiled. With CSV in the sysimage, these invalidations have already been +resolved. + + +## Recording precompile statements + +We are now at the stage where we have CSV in the sysimage, but we still suffer +some latency because of compilation. +Note that Julia is a dynamically typed language, it is therefore not known statically +what types will be used in functions. Therefore, in order to be able to compile code +one needs to know what types functions should be compiled for. One way to do this is to run +some representative workload and record what types functions end up getting called with. +This is a little bit like [Profile Guide Optimization (PGO)](https://en.wikipedia.org/wiki/Profile-guided_optimization) +while it here being something more like Profile Guided Compilation.. + +There is indeed a way for Julia to record what functions are getting compiled. +We can save these and then when building the sysimage tell Julia to compile and store +the native code for these functions. + +We create a file called `generate_csv_precompile.jl` containing some "training +code" that we will use as a base to figure out what functions end up getting +compiled: + +```jl +using CSV +CSV.read("FL_insurance_sample.csv") +``` + +We then make julia run this code but we add the `--trace-compile` flag to +output "precompilation statements" to a file: + +``` +julia --startup-file=no --trace-compile=csv_precompile.jl generate_csv_precompile.jl +``` + +Looking at `csv_precompile.jl` we can see hundreds of functions that end up getting compiled. +For example, the line + +```jl +precompile(Tuple{typeof(CSV.getsource), String, Bool}) +``` + +instructs julia to compile the function `CSV.getsource` for the arguments of +type `String` and `Bool`. + +Note that some of the symbols in the list of precompile statements have a bit +of a weird syntax containing `Symbol(#...)`, e.g: + +``` +precompile(Tuple{typeof(Base.map), getfield(CSV, Symbol("##4#5")), Base.SubString{String}}) +``` + +These are symbols that were not explicitly named in the source code but that +Julia automatically gave an internal name to refer to. These symbols are not +necessarily consistent between different Julia versions or even Julia built for +different operating systems. It is possible to make the precompile statements +more portable by filtering out any symbols starting with `#` but that naturally +leaves some latency on the table since these now have to be compiled during runtime. + +The way we make Julia cache the compilation of the functions in the list is +simply by executing the statement on each line when the sysimage is created. It +, unfortunately, isn't as simple as just adding an `include("csv_precompile")` +to our `custom_precompile.jl` file. Firstly, all the modules used in the +precompilation statements (like `DataFrames`) are not defined in the Main +namespace. Secondly, due to [some bugs in the way Julia export precompile +statements](https://github.com/JuliaLang/julia/issues/28808) running a +precompile statement can fail. The solution to these issues is to load all +modules in the sysimage by looping through `Base.loaded_modules` and to use a +`try-catch` for each precompile statement. In addition, we evaluate everything +in an anonymous module to not pollute the `Main` module which a bunch of +symbols. + +The end result is a `custom_sysimage.jl` file looking like: + +```jl +Base.init_depot_path() +Base.init_load_path() + +using CSV + +@eval Module() begin + for (pkgid, mod) in Base.loaded_modules + if !(pkgid.name in ("Main", "Core", "Base")) + eval(@__MODULE__, :(const $(Symbol(mod)) = $_mod)) + end + end + for statement in readlines("csv_precompile.jl") + try + Base.include_string(@__MODULE__, statement) + catch + # See julia issue #28808 + @info "failed to compile statement: $statement" + end + end +end # module + +empty!(LOAD_PATH) +empty!(DEPOT_PATH) +``` + +After repeating the process of creating the object file and using a compiler to +create the shared library sysimage, we are in a position to time again: + +```jl +julia> @time using CSV + 0.000408 seconds (665 allocations: 32.656 KiB) + +julia> @time CSV.read("FL_insurance_sample.csv"); + 0.031504 seconds (441 allocations: 37.383 KiB) + +julia> @time CSV.read("FL_insurance_sample.csv"); + 0.021355 seconds (423 allocations: 34.695 KiB) +``` + +And finally, our first time for parsing the CSV-file is close to the second time. + From aaa574f99eeb3008b1450807f727548e8722eb73 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 12 Dec 2019 17:01:20 +0100 Subject: [PATCH 35/80] add docstrings (#59) * add docstrings * add some more --- Project.toml | 2 - docs/src/index.md | 17 ++++++ docs/src/refs.md | 2 +- docs/src/sysimages.md | 2 +- src/PackageCompilerX.jl | 132 ++++++++++++++++++++++++++++++---------- 5 files changed, 118 insertions(+), 37 deletions(-) diff --git a/Project.toml b/Project.toml index fd3d714a..a54dfc0c 100644 --- a/Project.toml +++ b/Project.toml @@ -4,13 +4,11 @@ authors = ["KristofferC "] version = "0.1.0" [deps] -DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] -DocStringExtensions = "0.8" julia = "1.3" [extras] diff --git a/docs/src/index.md b/docs/src/index.md index dd5f783e..f5e75fd3 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -11,3 +11,20 @@ PackageCompilerX is a Julia package with two main purposes: The manual contains some uses of Linux commands like `ls` (`dir` in Windows) and `cat` but hopefully these commands are common enough that the points still come across. + +## Installation instructions + +To use PackageCompilerX, a C compiler need to be available: + +### macOS, Linux + +Having a recently modern `gcc` or `clang` available should be enough to use PackageCompilerX on Linux or macOS. +For macOS, using something like `homebrew` and for Linux the system package manager should work fine. + +### Windows + +For Windows, the minGW compiler toolchain is needed. It can be downloaded from e.g. +https://sourceforge.net/projects/mingw-w64/files/ or by following the +instructions for setting up a toolchain capable of compiling Julia itself on Windows at +https://github.com/JuliaLang/julia/blob/master/doc/build/windows.md#cygwin-to-mingw-cross-compiling. +and running PackageCompilerX from the cygwin terminal. diff --git a/docs/src/refs.md b/docs/src/refs.md index edfc6056..d8b0039b 100644 --- a/docs/src/refs.md +++ b/docs/src/refs.md @@ -2,7 +2,7 @@ ```@docs PackageCompilerX.create_sysimage -PackageCompilerX.restore_default_sysimg +PackageCompilerX.restore_default_sysimage PackageCompilerX.create_app PackageCompilerX.audit_app ``` diff --git a/docs/src/sysimages.md b/docs/src/sysimages.md index 8723c452..9bc1bf22 100644 --- a/docs/src/sysimages.md +++ b/docs/src/sysimages.md @@ -105,7 +105,7 @@ create_sysimage([:Debugger, :OhMyREPL]; replace_default=true) If this is the first time `create_sysimage` is called with `replace_default`, a backup of the default sysimage is created. The default sysimage can then be -restored with [`restore_default_sysimg()`](@ref). +restored with [`restore_default_sysimage()`](@ref). Note that sysimages are created "incrementally" in the sense that they add to the sysimage of the process running PackageCompilerX. If the default sysimage diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index f286ed88..77aa93d0 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -1,15 +1,11 @@ module PackageCompilerX -# TODO: Add good debugging statements -# TODO: sysimage or sysimg... - using Base: active_project using Libdl: Libdl using Pkg: Pkg using UUIDs: UUID -using DocStringExtensions: SIGNATURES, TYPEDEF -export create_sysimage, create_app, audit_app, restore_default_sysimg +export create_sysimage, create_app, audit_app, restore_default_sysimage include("juliaconfig.jl") @@ -87,9 +83,7 @@ function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String) return tmp_sys_ji end -# TODO: Add output file? function run_precompilation_script(project::String, precompile_file::Union{String, Nothing}) - # TODO: Audit tempname usage tracefile = tempname() if precompile_file == nothing arg = `-e ''` @@ -107,8 +101,8 @@ end function create_sysimg_object_file(object_file::String, packages::Vector{Symbol}; project::String, base_sysimage::String, - precompile_execution_file::Union{Vector{String}, Nothing}, - precompile_statements_file::Union{Vector{String}, Nothing}, + precompile_execution_file::Vector{String}, + precompile_statements_file::Vector{String}, cpu_target::String, compiled_modules::Bool) # include all packages into the sysimg @@ -126,15 +120,13 @@ function create_sysimg_object_file(object_file::String, packages::Vector{Symbol} precompile_statements = "" @debug "running precompilation execution script..." tracefiles = String[] - for file in (precompile_execution_file === nothing ? (nothing,) : precompile_execution_file) + for file in (isempty(precompile_execution_file) ? (nothing,) : precompile_execution_file) tracefile = run_precompilation_script(project, file) precompile_statements *= " append!(precompile_statements, readlines($(repr(tracefile))))\n" end - if precompile_statements_file != nothing - for file in precompile_statements_file - precompile_statements *= - " append!(precompile_statements, readlines($(repr(file))))\n" - end + for file in precompile_statements_file + precompile_statements *= + " append!(precompile_statements, readlines($(repr(file))))\n" end precompile_code = """ @@ -192,13 +184,41 @@ function gather_stdlibs_project(project::String) end """ - SIGNATURES + create_sysimage(packages::Union{Symbol, Vector{Symbol}}; kwargs...) + +Create a system image that includes the package(s) in `packages`. An attempt +to automatically find a compiler will be done but can also be given explicitly +by setting the envirnment variable `JULIA_CC` to a path to a compiler + +### Keyword arguments: + +- `sysimage_path::Union{String,Nothing}`: The path to where + the resulting sysimage should be saved. If set to `nothing` the keyword argument + `replace_defalt` needs to be set to `true`. + +- `project::String`: The project that should be active when the sysmage is created, + defaults to the current active project. + +- `precompile_execution_file::Union{String, Vector{String}}`: A file or list of + files that contain code which precompilation statements should be recorded from. + +- `precompile_statements_file::Union{String, Vector{String}}`: A file or list of + files that contains precompilation statements that should be included in the sysimage. + +- `incremental::Bool`: If `true`, build the new sysimage on top of the sysimage + of the current process otherwise build a new sysimage from scratch. Defaults to `true`. + +- `filter_stdlibs::Bool`: If `true`, only include stdlibs that are in the project file. + Defaults to `false`, only set to `true` if you know the potential pitfalls. + +- `replace_default::Bool`: If `true`, replaces the default system image which is automatically + used when Julia starts. To replace with the one Julia ships with, use [`restore_default_sysimage()`](@ref) """ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; sysimage_path::Union{String,Nothing}=nothing, project::String=active_project(), - precompile_execution_file::Union{String, Vector{String}, Nothing}=nothing, - precompile_statements_file::Union{String, Vector{String}, Nothing}=nothing, + precompile_execution_file::Union{String, Vector{String}}=String[], + precompile_statements_file::Union{String, Vector{String}}=String[], incremental::Bool=true, filter_stdlibs=false, replace_default::Bool=false, @@ -213,15 +233,20 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; tmp = mktempdir() sysimage_path = joinpath(tmp, string("sys.", Libdl.dlext)) end + if replace_default==true + if sysimage_path !== nothing + error("cannot specify `sysimage_path` when `replace_default` is `true`") + end + end if filter_stdlibs && incremental error("must use `incremental=false` to use `filter_stdlibs=true`") end - # Functions lower down handles precompilation file as arrays so convert here + # Functions lower down handles `packages` and precompilation file as arrays so convert here packages = vcat(packages) - precompile_execution_file !== nothing && (precompile_execution_file = vcat(precompile_execution_file)) - precompile_statements_file !== nothing && (precompile_statements = vcat(precompile_statements_file)) + precompile_execution_file = vcat(precompile_execution_file) + precompile_statements_file = vcat(precompile_statements_file) if !incremental if base_sysimage !== nothing @@ -270,16 +295,19 @@ function create_sysimg_from_object_file(input_object::String, sysimage_path::Str o_file = `-Wl,--whole-archive $input_object -Wl,--no-whole-archive` end extra = Sys.iswindows() ? `-Wl,--export-all-symbols` : `` - run(`$(get_compiler()) -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia $extra`) + cmd = `$(get_compiler()) -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia $extra` + @debug "running $cmd" + run(cmd) return nothing end """ - $SIGNATURES + restore_default_sysimage() -lalala +Restores the default system image to the one that Julia shipped with. +Useful after running [`create_sysimage`](@ref) with `replace_default=true`. """ -function restore_default_sysimg() +function restore_default_sysimage() if !isfile(backup_default_sysimg_path()) error("did not find a backup sysimg") end @@ -291,11 +319,10 @@ end const REQUIRES = "Requires" => UUID("ae029012-a4dd-5104-9daa-d747884805df") -# Check for things that might indicate that the app or dependencies """ - $SIGNATURES + audit_app(app_dir::String) -Check for possible problems with regfards to relocatability at +Check for possible problems with regards to relocatability for the project at `app_dir`. !!! warning @@ -335,18 +362,58 @@ function audit_app(ctx::Pkg.Types.Context) end """ - $SIGNATURES + create_app(app_source::String, compiled_app::String) + +Compile an app with the source in `app_source` to the folder `compiled_app`. +The folder `app_source` needs to contain a package where the package include a +function with the signature + +``` +Base.@ccallable julia_main()::Cint + # Perhaps do something based on ARGS + ... +end +``` + +The executable will be placed in a folder called `bin` in `compiled_app` and +when the executabl run the `julia_main` function is called. + +An attempt to automatically find a compiler will be done but can also be given +explicitly by setting the envirnment variable `JULIA_CC` to a path to a +compiler. + +### Keyword arguments: + +- `precompile_execution_file::Union{String, Vector{String}}`: A file or list of + files that contain code which precompilation statements should be recorded from. + +- `precompile_statements_file::Union{String, Vector{String}}`: A file or list of + files that contains precompilation statements that should be included in the sysimage + for the app. + +- `incremental::Bool`: If `true`, build the new sysimage on top of the sysimage + of the current process otherwise build a new sysimage from scratch. Defaults to `false`. + +- `filter_stdlibs::Bool`: If `true`, only include stdlibs that are in the project file. + Defaults to `false`, only set to `true` if you know the potential pitfalls. + +- `audit::Bool`: Warn about eventual relocatability problems with the app, defaults + to `true`. + +- `force::Bool`: Remove the folder `compiled_app` if it exists before creating the app. """ function create_app(package_dir::String, app_dir::String; - precompile_execution_file::Union{String, Vector{String}, Nothing}=nothing, - precompile_statements_file::Union{String, Vector{String}, Nothing}=nothing, + precompile_execution_file::Union{String, Vector{String}}=String[], + precompile_statements_file::Union{String, Vector{String}}=String[], incremental=false, filter_stdlibs=false, audit=true, force=false) - # Need to check if nothing was returned below project_toml_path = abspath(Pkg.Types.projectfile_path(package_dir; strict=true)) + if project_toml_path === nothing + error("no project found in $(repr(package_dir))") + end manifest_toml_path = abspath(Pkg.Types.manifestfile_path(package_dir)) if manifest_toml_path === nothing @warn "it is not recommended to create an app without a preexisting manifest" @@ -406,7 +473,6 @@ function create_app(package_dir::String, cpu_target=APP_CPU_TARGET, compiled_modules=false #= workaround julia#34076=#) end - create_executable_from_sysimg(; sysimage_path=sysimg_file, executable_path=app_name) if Sys.isapple() cmd = `install_name_tool -change $sysimg_file @rpath/$sysimg_file $app_name` From 6f5fcf2bacb82e795d97455ebd8ebd7aeee5c6df Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 12 Dec 2019 17:36:43 +0100 Subject: [PATCH 36/80] Update README.md --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index eee5056f..6f04137c 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,13 @@ [![Codecov](https://codecov.io/gh/KristofferC/PackageCompilerX.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/KristofferC/PackageCompilerX.jl) [![][docs-stable-img]][docs-stable-url] -**This package is a work in progress. Its current state of functionality / docs should not be indicative of the quality of the finished package** +PackageCompilerX is a Julia package with two main purposes: -On Linux or Mac this package requires `gcc` to be installed and be on the `PATH`. -On Windows, a cygwin setup needs to be installed as decribed [here](https://github.com/JuliaLang/julia/blob/master/doc/build/windows.md#cygwin-to-mingw-cross-compiling) and Julia need to be run in the cygwin environement. + 1. Creating custom sysimages for reduced latency when working locally with packages that has a high startup time. + 2. Creating "apps" which are a bundle of files including an executable that can be sent and run on other machines without Julia being installed on that machine. + +For installation and usage instructions, see the [documentation][docs-stable-url]. [docs-stable-img]: https://img.shields.io/badge/docs-stable-blue.svg [docs-stable-url]: https://kristofferc.github.io/PackageCompilerX.jl/dev From 85dbf5a55d37c1c60e47bf012f27b61328e6c447 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 12 Dec 2019 17:39:21 +0100 Subject: [PATCH 37/80] Update index.md --- docs/src/index.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/src/index.md b/docs/src/index.md index f5e75fd3..7b2bc365 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -14,7 +14,7 @@ come across. ## Installation instructions -To use PackageCompilerX, a C compiler need to be available: +To use PackageCompilerX a C-compiler needs to be available: ### macOS, Linux @@ -24,7 +24,7 @@ For macOS, using something like `homebrew` and for Linux the system package mana ### Windows For Windows, the minGW compiler toolchain is needed. It can be downloaded from e.g. -https://sourceforge.net/projects/mingw-w64/files/ or by following the +[https://sourceforge.net/projects/mingw-w64/files/](https://sourceforge.net/projects/mingw-w64/files/) or by following the instructions for setting up a toolchain capable of compiling Julia itself on Windows at -https://github.com/JuliaLang/julia/blob/master/doc/build/windows.md#cygwin-to-mingw-cross-compiling. -and running PackageCompilerX from the cygwin terminal. +[https://github.com/JuliaLang/julia/blob/master/doc/build/windows.md#cygwin-to-mingw-cross-compiling](https://github.com/JuliaLang/julia/blob/master/doc/build/windows.md#cygwin-to-mingw-cross-compiling) +and then run PackageCompilerX from the cygwin terminal. From 78babba7246a653ea68003b122330da6c1eca9bc Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 12 Dec 2019 17:44:43 +0100 Subject: [PATCH 38/80] Update intro.md --- docs/src/devdocs/intro.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/src/devdocs/intro.md b/docs/src/devdocs/intro.md index 0fe72bdb..790e72a1 100644 --- a/docs/src/devdocs/intro.md +++ b/docs/src/devdocs/intro.md @@ -1,13 +1,12 @@ # Introduction -This part of the documentation contains a set of tutorial to teach how -PackageCompilerX works internally by going through the steps for the f. By knowing the internals of -PackageCompilerX you can more easily figure out root causes of problems and -help other. - -The inner functionality of PackageCompilerX is actually quite simple. -There are a few julia commands and compiler invocations that everything -is built around, the rest is just scaffolding. +This part of the documentation contains a set of tutorials aimed to teach how +PackageCompilerX works internally. This is done by going through some examples +of manually creating sysimages and apps mostly from the command line. +By knowing the internals of PackageCompilerX you can more easily figure out +root causes of problems and help others. The inner functionality of PackageCompilerX +is actually quite simple. There are a few julia commands and compiler invocations +that everything is built around, the rest is mostly scaffolding. [Part 1](@ref man-tutorial-sysimage) focuses on how to build a local system image to reduce package load times and reduce the latency that can occur when From 0f4b55a69a2a17419860aed3783dcf379ddc599f Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 13 Dec 2019 17:20:28 +0100 Subject: [PATCH 39/80] some proof reading (#61) --- docs/make.jl | 1 - docs/src/apps.md | 108 +++++++++++++++------------ docs/src/devdocs/intro.md | 4 +- docs/src/devdocs/sysimages_part_1.md | 38 +++++----- docs/src/examples/ohmyrepl.md | 62 ++++++++------- docs/src/index.md | 5 +- docs/src/prereq.md | 7 -- docs/src/sysimages.md | 49 ++++++------ 8 files changed, 142 insertions(+), 132 deletions(-) delete mode 100644 docs/src/prereq.md diff --git a/docs/make.jl b/docs/make.jl index 742234aa..f853da28 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -10,7 +10,6 @@ makedocs( "Home" => "index.md", "Manual" => [ - "prereq.md" "sysimages.md" "apps.md" ], diff --git a/docs/src/apps.md b/docs/src/apps.md index 1295b563..7e142b79 100644 --- a/docs/src/apps.md +++ b/docs/src/apps.md @@ -1,46 +1,50 @@ # Apps -With an "app" we here mean a bundle of files where one of these files is an -executable and where the bundle can be sent to another machine while still allowing +With an "app" we here mean a "bundle" of files where one of these files is an +executable and where this bundle can be sent to another machine while still allowing the executable to run. -Use cases for Julia-apps is for example when one wants to provide some kind of -functionality where the fact that it was written in Julia is just an -implementation detail and forcing the user to download and use Julia to run the -code would be a distraction. There is also no need to provide the original -Julia source code for apps since everything gets baked into the sysimage. +Use-cases for Julia-apps are for example when one wants to provide some kind of +functionality where the fact that the code was written in Julia is just an +implementation detail and where requiring the user to download and use Julia to +run the code would be a distraction. There is also no need to provide the +original Julia source code for apps since everything gets baked into the +sysimage. ## Relocatability Since we want to send the app to other machines the app we create must be -"relocatable". With an app being "relocatable" we mean it does not rely on +"relocatable". With an app being relocatable we mean it does not rely on specifics of the machine where the app was created. Relocatability is not an absolute measure, most apps assume some properties of the machine they will run on, like what operating system is installed and the presence of graphics -drivers if one want to show graphics. On the other hand, embedding things into -the app that is most likely unique to the machine, like absolute paths, means -that the application almost surely will not run properly on another machine. +drivers if one wants to show graphics. On the other hand, embedding things into +the app that is most likely unique to the machine, such as absolute paths to +libraries, means that the application almost surely will not run properly on +another machine. For something to be relocatable, everything that it depends on must also be relocatable. In the case of an app, the app itself and all the Julia packages it depends on must also relocatable. This is a bit of an issue because the -Julia package ecosystem has not thought much about relocatability since app -making has not been common in the Julia community. +Julia package ecosystem has not rarely given much thought to relocatability +since creating "apps" has not been common. The main problem with relocatability of Julia packages is that many packages are encoding fundamentally non-relocatable *into the source code*. As an example, many packages tend to use a `build.jl` file (which runs when the package is first installed) that looks something like: -``` +```jl lib_path = find_library("libfoo") write("deps.jl", "const LIBFOO_PATH = $(repr(lib_path))") ``` -The main package file then contains +The main package file then contains: + +```jl +module Package -``` if !isfile("../build/deps.jl") error("run Pkg.build(\"Package\") to re-build Package") end @@ -49,21 +53,25 @@ function __init__() libfoo = Libdl.dlopen(LIBFOO_PATH) end +... + +end # module + ``` The absolute path to `lib_path` that `find_library` found is thus effectively included into the source code of the package. Arguably, the whole build system in Julia is inherently non-relocatable because it runs when the package is -being installed which is a concept that doesn't make sense when distributing an -app. +being installed which is a concept that does not make sense when distributing +an app. Some packages do need to call into external libraries and use external binaries -so how are these packages supposed to do this in a relocatable way? The answer -is to use the "artifact system" which was described in the following [blog +so the question then aris: "how are these packages supposed to do this in a +relocatable way? The answer is to use the "artifact system" which is +described in the following [blog post](https://julialang.org/blog/2019/11/artifacts). The artifact system is a declarative way of downloading and using "external files" like binaries and -libraries. How this is used in practice is described a bit later in this -document. +libraries. How this is used in practice is described later. ## Creating an app @@ -89,8 +97,8 @@ relocatability in the app. The app is then compiled using the [`create_app`](@ref) function that takes a path to the source code of the app and the destination where the app should be compiled to. This will bundle all required libraries for the app to run on -another machine where the same Julia that created the app could run. As an -example, below the example app linked above is compiled and run: +another machine where the same Julia that created the app can run. As an +example, in the code snippet below, the example app linked above is compiled and run: ``` ~/PackageCompilerX.jl/examples @@ -98,7 +106,7 @@ example, below the example app linked above is compiled and run: julia> using PackageCompilerX -julia> create_app("MyApp/", "MyAppCompiled") +julia> create_app("MyApp", "MyAppCompiled") [ Info: PackageCompilerX: creating base system image (incremental=false), this might take a while... [ Info: PackageCompilerX: creating system image object file, this might take a while... @@ -109,10 +117,12 @@ julia> exit() ARGS = ["foo", "bar"] Base.PROGRAM_FILE = "MyAppCompiled/bin/MyApp" ... -ἔοικα γοῦν τούτου γε σμικρῷ τινι αὐτῷ τούτῳ σοφώτερος εἶναι, ὅτι ἃ μὴ οἶδα οὐδὲ οἴομαι εἰδέναι. -unsafe_string((Base.JLOptions()).image_file) = "/home/kc/PackageCompilerX.jl/examples/MyAppCompiled/bin/MyApp.so" +Hello, World! + +Running the artifact +The result of 2*5^2 - 10 == 40.000000 +unsafe_string((Base.JLOptions()).image_file) = "/Users/kristoffer/PackageCompilerX.jl/examples/MyAppCompiled/bin/MyApp.dylib" Example.domath(5) = 10 -sin(0.0) = 0.0 ``` The resulting executable is found in the `bin` folder in the compiled app @@ -130,9 +140,9 @@ sysimages, the same keyword arguments are used to add precompilation to apps. In the section about creating sysimages, there was a short discussion about incremental vs non-incremental sysimages. In short, an incremental sysimage is -built on top of another sysimage while a non-incremental is created from -scratch. For sysimages, it made sense to use an incremental sysimage built on -top of Julia's default sysimage since we wanted the benefit of having a snappy +built on top of another sysimage, while a non-incremental is created from +scratch. For sysimages, it makes sense to use an incremental sysimage built on +top of Julia's default sysimage since we wanted the benefit of having a responsive REPL that it provides. For apps, this is no longer the case, the sysimage is not meant to be used when working interactively, it only needs to be specialized for the specific app. Therefore, by default, `incremental=true` is @@ -143,19 +153,19 @@ non-incremental sysimage is about 70MB smaller than the default sysimage. ### Filtering stdlibs By default, all standard libraries are included in the sysimage. It is -possible to only include those standard libraries that the project needs by -passing the keyword argument `filter_stdlibs=true` to `create_app`. This -causes the sysimage to be smaller, and possibly load faster. The reason this -is not the default is that it is possible to "accidentally" depend on a +possible to only include those standard libraries that the project needs. This +is done by passing the keyword argument `filter_stdlibs=true` to `create_app`. +This causes the sysimage to be smaller, and possibly load faster. The reason +this is not the default is that it is possible to "accidentally" depend on a standard library without it being reflected in the Project file. For example, it is possible to call `rand()` from a package without depending on Random, -even though that is where it is defined. If Random was excluded from the -sysimage that call would then error. Same applies to matrix multiplication, -`rand(3,3) * rand(3,3)` requires both `LinearAlgebra` and `Random` This is -because these standard libraries do "type piracy" so just loading them can -cause code to change behavior. +even though that is where the method is defined. If Random was excluded from +the sysimage that call would then error. The aame thing is true for e.g. matrix +multiplication, `rand(3,3) * rand(3,3)` requires both the standard libraries +`LinearAlgebra` and `Random` This is because these standard libraries do +"type-piracy" so just loading those packages can cause code to change behavior. -Nevertheless, the option is there to use, just make sure to properly test the +Nevertheless, the option is there to use. Just make sure to properly test the app with the resulting sysimage. @@ -167,15 +177,15 @@ PackageCompilerX will bundle all artifacts needed by the project, and set up things so that they can be found during runtime on other machines. The example app uses the artifact system to depend on a very simple toy binary -that prints some greek text. It is instructive to see how the [artifact -file](https://github.com/KristofferC/PackageCompilerX.jl/blob/d12d8d9b2286bb7d57ca28ad0f2a8cd130c70c81/examples/MyApp/Artifacts.toml) -is [used in the source -code](https://github.com/KristofferC/PackageCompilerX.jl/blob/d12d8d9b2286bb7d57ca28ad0f2a8cd130c70c81/examples/MyApp/src/MyApp.jl#L4-L7) +that does some simple arithmetic. It is instructive to see how the [artifact +file](https://github.com/KristofferC/PackageCompilerX.jl/blob/8aa31d60ace6dd278daed9bef62fd5a01258da1e/examples/MyApp/Artifacts.toml) +is [used in the source](https://github.com/KristofferC/PackageCompilerX.jl/blob/8aa31d60ace6dd278daed9bef62fd5a01258da1e/examples/MyApp/src/MyApp.jl#L7-L8). -### What things are being leaked about the build machine and the source code +### Reverse engineering the compiled app While the created app is relocatable and no source code is bundled with it, -there are still some things about the build machine that can be observed. +there are still some things about the build machine and the source code that +can be "reverse engineered". #### Absolute paths of build machine @@ -195,7 +205,7 @@ MyApp.jl /home/kc/PackageCompilerX.jl/examples/MyApp/src/MyApp.jl ``` -This is a problem that Julia itself has: +This is a problem that the Julia standard libraries themselves have: ``` julia> @which rand() @@ -209,7 +219,7 @@ comes with the app. And while the source code is not available one can read the "lowered code" and use reflection to find things like the name of fields in structs and global variables etc: -``` +```jl-repl ~/PackageCompilerX.jl/examples/MyAppCompiled/bin kc/docs_apps* ❯ julia -q -JMyApp.so julia> MyApp = Base.loaded_modules[Base.PkgId(Base.UUID("f943f3d7-887a-4ed5-b0c0-a1d6899aa8f5"), "MyApp")] diff --git a/docs/src/devdocs/intro.md b/docs/src/devdocs/intro.md index 790e72a1..98ef2949 100644 --- a/docs/src/devdocs/intro.md +++ b/docs/src/devdocs/intro.md @@ -2,7 +2,7 @@ This part of the documentation contains a set of tutorials aimed to teach how PackageCompilerX works internally. This is done by going through some examples -of manually creating sysimages and apps mostly from the command line. +of manually creating sysimages and apps, mostly from the command line. By knowing the internals of PackageCompilerX you can more easily figure out root causes of problems and help others. The inner functionality of PackageCompilerX is actually quite simple. There are a few julia commands and compiler invocations @@ -20,5 +20,5 @@ from PackageCompilerX as [`create_sysimage`](@ref) and [`create_app`](@ref). It should be noted that there is some usage of non-documented Julia functions and flags. They have not been changed for quite a long time (and are unlikely -to change too much in the future) but some care should be taken. +to change too much in the future), but some care should be taken. diff --git a/docs/src/devdocs/sysimages_part_1.md b/docs/src/devdocs/sysimages_part_1.md index d875a94d..4836638c 100644 --- a/docs/src/devdocs/sysimages_part_1.md +++ b/docs/src/devdocs/sysimages_part_1.md @@ -21,11 +21,11 @@ can immediately be used, without requiring any compilation. ## Custom sysimages There are cases where one wants to generate a custom sysimage for a similar -reason as to why Julia bundles one, to reduce time from Julia start until the +reason as to why Julia bundles one: to reduce time from Julia start until the program is executing. The time from startup to execution is here denoted as "latency" and we want to minimize the latency of our program. A drawback of putting a package inside the sysimage is that it becomes "frozen" at the -particular version it was when it got put into the sysimage. In addition, all +particular version it was, when it got put into the sysimage. In addition, all the dependencies of the package put into the sysimage will be frozen in the same manner. In particular, it will no longer be updated like normal packages when using the package manager. In some cases, other ways of reducing latency @@ -63,7 +63,7 @@ The term "precompiled" can be a bit misleading since there is no native compiled code cached in the precompilation file. Julia is dynamically typed so it is not obvious what types to compile the different methods for. -Even with `CSV` "precompiled", there is a still some loading time but it is +Even with `CSV` "precompiled", there is a still some loading time, but it is significantly lower: ```jl-repl @@ -95,11 +95,11 @@ function is still in memory. However, since the end goal of this blog series is to create an executable that can be distributed we want to try to avoid as much runtime compilation -(latency) as possible +(latency) as possible. ## Creating a custom sysimage -If we time the loading of a standard library it is clear that it is "cached" +If we time the loading of a standard library, it is clear that it is "cached" somehow since the time to load it is so short: ```jl-repl @@ -108,7 +108,7 @@ julia> @time using Dates ``` Since `Dates` is a standard library it comes bundled in the system image. In -fact, `Dates` is already "loaded" when starting Julia, the effect of running +fact, `Dates` is already "loaded" when starting Julia. The effect of running `using Dates` just makes the module available in the `Main` module namespace which is what the REPL evaluates in. @@ -125,9 +125,9 @@ Dict{Base.PkgId,Module} with 33 entries: ... ``` -and we can here see the `Dates` module is there even after restarting Julia -and not explicitly loading `Dates`, showing that `Dates` is loaded together with -the start-up of Julia. +and we can here see the `Dates` module is there, even after restarting Julia. +This means that `Dates` is in the sysimage itself and does not have to be loaded +from amywhere external. Creating and using a custom sysimage is done in three steps: @@ -159,9 +159,9 @@ julia --startup-file=no --output-o=sys.o -- custom_sysimage.jl ERROR: could not open file boot.jl ``` -That didn't work well. It turns out that when using the `--output-o` option one +That did not work well. It turns out that when using the `--output-o` option one has to explicitly give a sysimage path ([due to this -line](https://github.com/JuliaLang/julia/blob/49fb7924498e9fe813444cc684a24002e75b2ac9/src/jloptions.c#L533)). Since we don't have a custom sysimage yet we +line](https://github.com/JuliaLang/julia/blob/49fb7924498e9fe813444cc684a24002e75b2ac9/src/jloptions.c#L533)). Since we do not have a custom sysimage yet we just want to give the path to the default sysimage which we can get the path to via: @@ -184,12 +184,12 @@ uv_write at ./stream.jl:924 ``` Failure again! Another caveat when using `--output-o` is that modules -`__init__()` functions do not end up getting called which is what normally +`__init__()` functions do not end up getting called, which is what normally happens when a module is loaded. The reason for this is that often the state that gets defined in `__init__` is not something that you want to serialize to -a file. In this particular case, some parts of the IO system has not been +a file. In this particular case, some parts of the IO system have not been initialized so Julia crashes while trying to print an error. The magic -incantation to make IO work properly is `Base.reinit_stdio()` so to figure out +incantation to make IO work properly is `Base.reinit_stdio()`. To figure out the actual problem we modify the `custom_sysimage.jl` file to look like: ``` @@ -215,7 +215,7 @@ Stacktrace: in expression starting at /home/kc/custom_sysimage.jl:2 ``` -Okay, now we can see the error. Julia can for some reason not find the `CSV` +Okay, now we can see the error. Julia can not find the `CSV` package. Package-loading in Julia is based on the two arrays `LOAD_PATH` and `DEPOT_PATH`. Adding `@show LOAD_PATH` and `@show DEPOT_PATH` to the `custom_sysimage.jl` file and rerunning the command above prints: @@ -225,9 +225,9 @@ LOAD_PATH = String[] DEPOT_PATH = String[] ``` -So again, we have an initialization problem. Looking at [what Julia itself does -before including the standard libraries][sysimage-path-init], we can see that -the functions initializing these variables are explicitly called. Let's do the +Again, we have an initialization problem. Looking at [what Julia itself does +before including the standard libraries](https://github.com/JuliaLang/julia/blob/88c34fc51d962aaef973935942b2e073e2e2f398/base/sysimg.jl#L13-L14), we can see that +the functions initializing these variables are explicitly called. Let us do the same by updating the `custom_sysimage.jl` file to: ```jl @@ -273,7 +273,7 @@ gcc -shared -o sys.so -Wl,--whole-archive sys.o -Wl,--no-whole-archive -L"/home/ which creates the sysimage `sys.so`. -We can compare the size of the new sysimage vs the default one and see that the +We can compare the size of the new sysimage versus the default one and see that the new is a bit larger due to the extra packages it contains: ``` diff --git a/docs/src/examples/ohmyrepl.md b/docs/src/examples/ohmyrepl.md index 8131f70e..61bad2e3 100644 --- a/docs/src/examples/ohmyrepl.md +++ b/docs/src/examples/ohmyrepl.md @@ -1,44 +1,50 @@ -# Creating a sysimage with OhMyREPL - -[OhMyREPL.jl](https://github.com/KristofferC/OhMyREPL.jl) is a package that enhances the REPL with, for example, syntax highlighting. -It does, however, come with a bit of a startup time increase -so compiling a new system image with OhMyREPL included is useful. -Importing the OhMyREPL package is not the only factor that contributes to the extra load time from using OhMyREPL -In addition, the time of compiling functions that OhMyREPL uses is also a factor. -Therefore, we also want to -do Profile Guided Precompilation (PGP) where we record what functions gets compiled when using -OhMyREPL so they can be cached into the system image. OhMyREPL is a bit different from -most other packages in that is used interactive. Normally to do PGP with PackageCompilerX we pass a script to -to execute as the `precompile_exectution_file` which is used to collect compilation data, -but in this case, we will use Julia to manually collect this data. - -First install `OhMyREPL` in the global environement using `import Pkg; Pkg.add("OhMyREPL")`. -Run `using OhMyREPL` and write something (like `1+1`). It should be syntax highlighted but you might -have noticed that there was a bit of a delay before the characters appeared. This is the extra latency -from using the package that we want to get rid off. +# [Creating a sysimage with OhMyREPL](@id manual-omr) + +[OhMyREPL.jl](https://github.com/KristofferC/OhMyREPL.jl) is a package that +enhances the REPL with, for example, syntax highlighting. It does, however, +come with a bit of a startup time increase, so compiling a new system image +with OhMyREPL included is useful. Importing the OhMyREPL package is not the +only factor that contributes to the extra load time from using OhMyREPL In +addition, the time of compiling functions that OhMyREPL uses is also a factor. +Therefore, we also want to do "Profile Guided Compilation" (PGC), where we +record what functions gets compiled when using OhMyREPL, so they can be cached +into the system image. OhMyREPL is a bit different from most other packages in +that is used interactive. Normally to do PGC with PackageCompilerX we pass a +script to to execute as the `precompile_exectution_file` which is used to +collect compilation data, but in this case, we will use Julia to manually +collect this data. + +First install `OhMyREPL` in the global environement using `import Pkg; +Pkg.add("OhMyREPL")`. Run `using OhMyREPL` and write something (like `1+1`). +It should be syntax highlighted, but you might have noticed that there was a bit +of a delay before the characters appeared. This is the extra latency from using +the package that we want to get rid off. ![OhMyREPL installation](omr_install.png) -The first goal is to have Julia emit the functions it compiles when running OhMyREPL. -To this end, start Julia with the `--trace-compile=ohmyrepl_precompile` flag. This will -start a normal-looking Julia session but all functions that get compiled are output -to the file `ohmyrepl_precompile`. In the Julia session, load OhMyREPL, use the REPL a bit -so that the functionality of OhMyREPL is exercised. Quit Julia and look into the file `ohmyrepl_precompile`. -It should be filled with lines like: +The first goal is to have Julia emit the functions it compiles when running +OhMyREPL. To this end, start Julia with the +`--trace-compile=ohmyrepl_precompile.jl` flag. This will start a standard +Julia session but all functions that get compiled are output to the file +`ohmyrepl_precompile.jl`. In the Julia session, load OhMyREPL, use the REPL a bit +so that the functionality of OhMyREPL is exercised. Quit Julia and look into +the file `ohmyrepl_precompile`. It should be filled with lines like: ``` precompile(Tuple{typeof(OhMyREPL.Prompt.insert_keybindings), Any}) precompile(Tuple{typeof(OhMyREPL.__init__)}) ``` -These are functions that Julia compiled. We now just tell `create_sysimage` to use these precompile statements -when creating the system image: +These are functions that Julia compiled. We now just tell `create_sysimage` to +use these precompile statements when creating the system image: ```jl -PackageCompilerX.create_sysimage(:OhMyREPL; precompile_statements_file="ohmyrepl_precompile", replace_default=true) +PackageCompilerX.create_sysimage(:OhMyREPL; precompile_statements_file="ohmyrepl_precompile.jl", replace_default=true) ``` -Restart julia and start typing something. If everything went well you should see the type text become highlighted with a significantly smaller delay than before creating the new system image +Restart julia and start typing something. If everything went well you should +see the type text become highlighted with a significantly smaller delay than +before creating the new system image !!! note diff --git a/docs/src/index.md b/docs/src/index.md index 7b2bc365..096d42f2 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -18,7 +18,7 @@ To use PackageCompilerX a C-compiler needs to be available: ### macOS, Linux -Having a recently modern `gcc` or `clang` available should be enough to use PackageCompilerX on Linux or macOS. +Having a decently modern `gcc` or `clang` available should be enough to use PackageCompilerX on Linux or macOS. For macOS, using something like `homebrew` and for Linux the system package manager should work fine. ### Windows @@ -27,4 +27,5 @@ For Windows, the minGW compiler toolchain is needed. It can be downloaded from e [https://sourceforge.net/projects/mingw-w64/files/](https://sourceforge.net/projects/mingw-w64/files/) or by following the instructions for setting up a toolchain capable of compiling Julia itself on Windows at [https://github.com/JuliaLang/julia/blob/master/doc/build/windows.md#cygwin-to-mingw-cross-compiling](https://github.com/JuliaLang/julia/blob/master/doc/build/windows.md#cygwin-to-mingw-cross-compiling) -and then run PackageCompilerX from the cygwin terminal. +and then run PackageCompilerX from the cygwin terminal. Alternatively, the package manager +[chocolatey](https://chocolatey.org/) can be used to get mingw on Windows. diff --git a/docs/src/prereq.md b/docs/src/prereq.md deleted file mode 100644 index 82c6165e..00000000 --- a/docs/src/prereq.md +++ /dev/null @@ -1,7 +0,0 @@ -# Prerequisites - -In order to use this package you need either `gcc` or `clang` installed and on the path. - -The package currently only works on Linux and macOS with Windows support being a work in progress. - -In the future, we hope to avoid this by automatically providing a working compiler with the package. diff --git a/docs/src/sysimages.md b/docs/src/sysimages.md index 9bc1bf22..b36ed0c5 100644 --- a/docs/src/sysimages.md +++ b/docs/src/sysimages.md @@ -3,7 +3,7 @@ ## What is a sysimage A sysimage is a file which, in a loose sense, contains a Julia session -serialized to a file. A "Julia session" include things like loaded packages, +serialized to a file. A "Julia session" includes things like loaded packages, global variables, inferred and compiled code, etc. By starting Julia with a sysimage, the stored Julia session is deserialized and loaded. The idea behind the sysimage is that this deserialization is faster than having to reload @@ -11,14 +11,14 @@ packages and recompile code from scratch. Julia ships with a sysimage that is used by default when Julia is started. That sysimage contains the Julia compiler itself, the standard libraries and also -compiled code (precompile statements) that has been put there to reduce the -time required to do common operations, like working in the REPL. +compiled code that has been put there to reduce the time required to do common +operations, like working in the REPL. -Sometimes, it is desirable to create a custom sysimage with custom precompile -statements. This is the case if one has some dependencies that take a -significant time to load or where the compilation time for the first call is -uncomfortably long. This section of the documentation is intended to document -how to use PackageCompilerX to create such sysimages. +Sometimes it is desirable to create a custom sysimage with custom precompiled +code. This is the case if one has some dependencies that take a significant +time to load or where the compilation time for the first call is uncomfortably +long. This section of the documentation is intended to document how to use +PackageCompilerX to create such sysimages. ### Drawbacks to custom sysimages @@ -26,16 +26,16 @@ It should be clearly stated that there are some drawbacks to using a custom sysimage, thereby sidestepping the standard Julia package precompilation system. The biggest drawback is that packages that are compiled into a sysimage (including their dependencies!) are "locked" to the version they where -at when the sysimage was ( created. This means that no matter what package +at when the sysimage was created. This means that no matter what package version you have installed in your current project, the one in the sysimage will take precedence. This can lead to bugs where you start with a project that needs a specific version of a package, but you have another one compiled into the sysimage. Putting packages in the sysimage is therefore only recommended if the load time -of the packages getting put in there is a significant problem and that these +of those packages is a significant problem and when these packages are not frequently updated. In addition, compiling "workflow packages" like -Revise.jl and OhMyREPL.jl might make sense. +Revise.jl and OhMyREPL.jl and using that as a default sysimage might make sense. ## Creating a sysimage using PackageCompilerX @@ -48,12 +48,13 @@ sysimage is given by the `sysimage_path` keyword. After the sysimage is created, giving the command flag `-Jpath/to/sysimage` will start Julia with the given sysimage. -As an example, below, a new sysimage in a separate project is created with the -package Example.jl in it. Using `Base.loaded_modules` it can be seen that the -package is loaded without having to explicitly `import` it. +Below is an example of a new sysimage, from a separate project, being created +with the package Example.jl in it. Using `Base.loaded_modules` it can be seen +that the package is loaded without having to explicitly `import` it. + ``` ~ -❯ mkdir NewSysImageEnv +❯mkdir NewSysImageEnv ~ ❯ cd NewSysImageEnv @@ -112,11 +113,11 @@ the sysimage of the process running PackageCompilerX. If the default sysimage has been replaced, the next `create_sysimage` call will create a new sysimage based on the replaced sysimage. It is possible to create a sysimage non-incrementally by passing the `incremental=false` keyword. This will create -a new system image from scratch, however, it will lose the special +a new system image from scratch. However, it will lose the special precompilation that the Julia bundled sysimage provides which is what make the -REPL and package manager snappy. It is therefore unlikely that -`incremental=false` is of much use unless in special cases for sysimage -creation (for apps it is a different story though). +REPL and package manager not require compilation after a Julia restart.. It is +therefore unlikely that `incremental=false` is of much use unless in special +cases for sysimage creation (for apps it is a different story though). ### Precompilation @@ -146,11 +147,11 @@ using Example Example.hello("friend") ``` -We now create a new system image called `ExampleSysimagePrecompile.so` where -the `precompile_execution_file` keyword argument has been giving, pointing to +We now create a new system image called `ExampleSysimagePrecompile.so`, where +the `precompile_execution_file` keyword argument has been given, pointing to the file just shown above: -``` +```julia-repl ~/NewSysImageEnv ❯ julia-q @@ -166,7 +167,7 @@ julia> PackageCompilerX.create_sysimage(:Example; sysimage_path="ExampleSysimage julia> exit() ``` -Using the just created system image, we can see that the `hello` function no longer needs to get compiled_: +Using the just created system image, we can see that the `hello` function no longer needs to get compiled: ``` ~/NewSysImageEnv @@ -183,7 +184,7 @@ statements to `file.jl` for the duration of the started Julia process. This can be useful in cases where it is difficult to give a script that executes the code (like with interactive use). A file with a list of such precompile statements can be used when creating a sysimage by passing the keyword argument -`precompile_statements_file`. See the OhMyREPL.jl example in the docs for more +`precompile_statements_file`. See the [OhMyREPL.jl example](@ref manual-omr) in the docs for more details on how to use `--trace-compile` with PackageCompilerX. It is also possible to use From f0b14d98cc7ce377f7eb9d49dcc7eb12c8c293f4 Mon Sep 17 00:00:00 2001 From: KristofferC Date: Fri, 13 Dec 2019 18:37:56 +0100 Subject: [PATCH 40/80] remove github documenter workflow --- .github/workflows/documenter-workflow.yml | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 .github/workflows/documenter-workflow.yml diff --git a/.github/workflows/documenter-workflow.yml b/.github/workflows/documenter-workflow.yml deleted file mode 100644 index 6d433383..00000000 --- a/.github/workflows/documenter-workflow.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Documentation - -on: [push] - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - matrix: - julia-version: [1.2.0] - julia-arch: [x86] - os: [ubuntu-latest] - steps: - - uses: actions/checkout@v1.0.0 - - uses: julia-actions/setup-julia@latest - with: - version: ${{ matrix.julia-version }} - - name: Install dependencies - run: julia --project=docs/ -e 'using Pkg; Pkg.instantiate()' - - name: Build and deploy - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: julia --project=docs/ docs/make.jl From 93733103c67daa7e16bdb235de9f915c7f50af70 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 13 Dec 2019 19:30:11 +0100 Subject: [PATCH 41/80] Deploy docs with GitHub actions (#62) * Revert "remove github documenter workflow" This reverts commit dacb45b00005325fc5a04efa62a7646e3bc2c054. * do not run documenter on travis * pretty urls --- .github/workflows/documenter-workflow.yml | 23 +++++++++++++++++++++++ .travis.yml | 11 ----------- docs/make.jl | 3 +-- 3 files changed, 24 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/documenter-workflow.yml diff --git a/.github/workflows/documenter-workflow.yml b/.github/workflows/documenter-workflow.yml new file mode 100644 index 00000000..9f1e0dca --- /dev/null +++ b/.github/workflows/documenter-workflow.yml @@ -0,0 +1,23 @@ +name: Documentation + +on: [push] + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + julia-version: [1.3.0] + julia-arch: [x86] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v1.0.0 + - uses: julia-actions/setup-julia@latest + with: + version: ${{ matrix.julia-version }} + - name: Install dependencies + run: julia --project=docs/ -e 'using Pkg; Pkg.instantiate()' + - name: Build and deploy + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: julia --project=docs/ docs/make.jl deploy diff --git a/.travis.yml b/.travis.yml index 264b32b3..7900ab4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,14 +12,3 @@ branches: - /^v[0-9]+\.[0-9]+\.[0-9]+$/ # version tags notifications: email: false -jobs: - include: - - stage: "Documentation" - julia: 1.3 - os: linux - script: - - julia --project=docs/ -e 'using Pkg; Pkg.instantiate()' - - julia --project=docs/ docs/make.jl - after_success: skip -after_success: - - julia -e 'using Pkg; Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder())' diff --git a/docs/make.jl b/docs/make.jl index f853da28..2171db58 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -2,8 +2,7 @@ using Documenter, PackageCompilerX makedocs( format = Documenter.HTML( - # prettyurls on travis - prettyurls = haskey(ENV, "HAS_JOSH_K_SEAL_OF_APPROVAL"), + prettyurls = "deploy" in ARGS, ), sitename = "PackageCompilerX", pages = [ From 17cbe46f989b312f81ac9ab04f2d29b89a5371a0 Mon Sep 17 00:00:00 2001 From: KristofferC Date: Fri, 13 Dec 2019 20:25:39 +0100 Subject: [PATCH 42/80] update Documenter --- docs/Manifest.toml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 6564d5df..1f2b1a59 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -19,11 +19,9 @@ version = "0.8.1" [[Documenter]] deps = ["Base64", "Dates", "DocStringExtensions", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] -git-tree-sha1 = "2b45ba82d7de1083af18f09af4ebdeb4dd99b9f3" -repo-rev = "master" -repo-url = "https://github.com/JuliaDocs/Documenter.jl.git" +git-tree-sha1 = "0be9bf63e854a2408c2ecd3c600d68d4d87d8a73" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "0.24.0-DEV" +version = "0.24.2" [[InteractiveUtils]] deps = ["Markdown"] @@ -52,7 +50,7 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[PackageCompilerX]] -deps = ["DocStringExtensions", "Libdl", "Pkg", "UUIDs"] +deps = ["Libdl", "Pkg", "UUIDs"] path = ".." uuid = "dffaa6cc-da53-48e5-b007-4292dfcc27f1" version = "0.1.0" From 796ec53c58be20b80313b6639b4d090677489ca5 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 13 Dec 2019 23:06:47 +0100 Subject: [PATCH 43/80] fix order of check (#64) --- src/PackageCompilerX.jl | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 77aa93d0..a7c9c71f 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -225,6 +225,11 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; cpu_target::String=NATIVE_CPU_TARGET, base_sysimage::Union{Nothing, String}=nothing, compiled_modules=true) + if replace_default==true + if sysimage_path !== nothing + error("cannot specify `sysimage_path` when `replace_default` is `true`") + end + end if sysimage_path === nothing if replace_default == false error("`sysimage_path` cannot be `nothing` if `replace_default` is `false`") @@ -233,12 +238,6 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; tmp = mktempdir() sysimage_path = joinpath(tmp, string("sys.", Libdl.dlext)) end - if replace_default==true - if sysimage_path !== nothing - error("cannot specify `sysimage_path` when `replace_default` is `true`") - end - end - if filter_stdlibs && incremental error("must use `incremental=false` to use `filter_stdlibs=true`") end From ed036e1dc83094f6e2f3b72210f1fd0542eb34e8 Mon Sep 17 00:00:00 2001 From: Simon Christ Date: Mon, 16 Dec 2019 16:08:25 +0100 Subject: [PATCH 44/80] fix typos (#65) --- docs/src/apps.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/apps.md b/docs/src/apps.md index 7e142b79..0bcaca50 100644 --- a/docs/src/apps.md +++ b/docs/src/apps.md @@ -145,7 +145,7 @@ scratch. For sysimages, it makes sense to use an incremental sysimage built on top of Julia's default sysimage since we wanted the benefit of having a responsive REPL that it provides. For apps, this is no longer the case, the sysimage is not meant to be used when working interactively, it only needs to be -specialized for the specific app. Therefore, by default, `incremental=true` is +specialized for the specific app. Therefore, by default, `incremental=false` is used for `create_app`. If, for some reason, one wants an incremental sysimage, `incremental=true` could be passed to `create_app`. With the example app, a non-incremental sysimage is about 70MB smaller than the default sysimage. @@ -160,7 +160,7 @@ this is not the default is that it is possible to "accidentally" depend on a standard library without it being reflected in the Project file. For example, it is possible to call `rand()` from a package without depending on Random, even though that is where the method is defined. If Random was excluded from -the sysimage that call would then error. The aame thing is true for e.g. matrix +the sysimage that call would then error. The same thing is true for e.g. matrix multiplication, `rand(3,3) * rand(3,3)` requires both the standard libraries `LinearAlgebra` and `Random` This is because these standard libraries do "type-piracy" so just loading those packages can cause code to change behavior. From 538809154bfd39d6ac53554c5a03913226b0bf5c Mon Sep 17 00:00:00 2001 From: Nathan Daly Date: Wed, 1 Jan 2020 06:29:10 -0500 Subject: [PATCH 45/80] Documentation: typos and wording in apps.md (#66) Fixes some typos and clarifies some wording in docs/src/apps.md. --- docs/src/apps.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/src/apps.md b/docs/src/apps.md index 0bcaca50..a92158da 100644 --- a/docs/src/apps.md +++ b/docs/src/apps.md @@ -27,12 +27,12 @@ another machine. For something to be relocatable, everything that it depends on must also be relocatable. In the case of an app, the app itself and all the Julia packages it depends on must also relocatable. This is a bit of an issue because the -Julia package ecosystem has not rarely given much thought to relocatability +Julia package ecosystem has rarely given much thought to relocatability since creating "apps" has not been common. The main problem with relocatability of Julia packages is that many packages -are encoding fundamentally non-relocatable *into the source code*. As an -example, many packages tend to use a `build.jl` file (which runs when the +are encoding fundamentally non-relocatable information *into the source code*. +As an example, many packages tend to use a `build.jl` file (which runs when the package is first installed) that looks something like: ```jl @@ -66,9 +66,9 @@ being installed which is a concept that does not make sense when distributing an app. Some packages do need to call into external libraries and use external binaries -so the question then aris: "how are these packages supposed to do this in a -relocatable way? The answer is to use the "artifact system" which is -described in the following [blog +so the question then arises: "how are these packages supposed to do this in a +relocatable way?" The answer is to use the "artifact system" introduced in +Julia 1.3, and described in the following [blog post](https://julialang.org/blog/2019/11/artifacts). The artifact system is a declarative way of downloading and using "external files" like binaries and libraries. How this is used in practice is described later. From b626fd81c38627a45cf736efd3bb2460a57db6cb Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 2 Jan 2020 19:42:25 +0000 Subject: [PATCH 46/80] improve testing (#60) * improve testing --- src/PackageCompilerX.jl | 62 ++++++++++++++++++++--------------------- test/runtests.jl | 19 +++++++++++-- 2 files changed, 46 insertions(+), 35 deletions(-) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index a7c9c71f..e7c71288 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -169,9 +169,8 @@ backup_default_sysimg_path() = default_sysimg_path() * ".backup" backup_default_sysimg_name() = basename(backup_default_sysimg_path()) # TODO: Also check UUIDs for stdlibs, not only names -function gather_stdlibs_project(project::String) - project_toml_path = abspath(Pkg.Types.projectfile_path(project; strict=true)) - ctx = Pkg.Types.Context(env=Pkg.Types.EnvCache(project_toml_path)) +gather_stdlibs_project(project::String) = gather_stdlibs_project(create_pkg_context(project)) +function gather_stdlibs_project(ctx) @assert ctx.env.manifest !== nothing stdlibs = all_stdlibs() stdlibs_project = String[] @@ -216,7 +215,7 @@ by setting the envirnment variable `JULIA_CC` to a path to a compiler """ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; sysimage_path::Union{String,Nothing}=nothing, - project::String=active_project(), + project::String=dirname(active_project()), precompile_execution_file::Union{String, Vector{String}}=String[], precompile_statements_file::Union{String, Vector{String}}=String[], incremental::Bool=true, @@ -247,12 +246,17 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; precompile_execution_file = vcat(precompile_execution_file) precompile_statements_file = vcat(precompile_statements_file) + # Instantiate the project + ctx = create_pkg_context(project) + @debug "instantiating project at $(repr(project))" + Pkg.instantiate(ctx) + if !incremental if base_sysimage !== nothing error("cannot specify `base_sysimage` when `incremental=false`") end if filter_stdlibs - stdlibs = gather_stdlibs_project(project) + stdlibs = gather_stdlibs_project(ctx) else stdlibs= all_stdlibs() end @@ -318,6 +322,14 @@ end const REQUIRES = "Requires" => UUID("ae029012-a4dd-5104-9daa-d747884805df") +function create_pkg_context(project) + project_toml_path = Pkg.Types.projectfile_path(project; strict=true) + if project_toml_path === nothing + error("could not find project at $(repr(project))") + end + return Pkg.Types.Context(env=Pkg.Types.EnvCache(project_toml_path)) +end + """ audit_app(app_dir::String) @@ -328,11 +340,7 @@ the project at `app_dir`. This cannot guarantee that the project is free of relocatability problems, it can only detect some known bad cases and warn about those. """ -function audit_app(app_dir::String) - project_toml_path = abspath(Pkg.Types.projectfile_path(app_dir; strict=true)) - ctx = Pkg.Types.Context(env=Pkg.Types.EnvCache(project_toml_path)) - return audit_app(ctx) -end +audit_app(app_dir::String) = audit_app(create_pkg_context(app_dir)) function audit_app(ctx::Pkg.Types.Context) # Check for Requires.jl usage if REQUIRES in ctx.env.project.deps @@ -409,25 +417,16 @@ function create_app(package_dir::String, filter_stdlibs=false, audit=true, force=false) - project_toml_path = abspath(Pkg.Types.projectfile_path(package_dir; strict=true)) - if project_toml_path === nothing - error("no project found in $(repr(package_dir))") - end - manifest_toml_path = abspath(Pkg.Types.manifestfile_path(package_dir)) - if manifest_toml_path === nothing + package_dir = abspath(package_dir) + ctx = create_pkg_context(package_dir) + if isempty(ctx.env.manifest) @warn "it is not recommended to create an app without a preexisting manifest" end - project_toml = Pkg.TOML.parsefile(project_toml_path) - project_path = abspath(package_dir) - app_name = get(project_toml, "name") do + if ctx.env.pkg === nothing error("expected package to have a `name`-entry") end + app_name = ctx.env.pkg.name sysimg_file = app_name * "." * Libdl.dlext - - ctx = Pkg.Types.Context(env=Pkg.Types.EnvCache(project_toml_path)) - @debug "instantiating project at \"$project_toml_path\"" - Pkg.instantiate(ctx) - if isdir(app_dir) if !force error("directory $(repr(app_dir)) already exists, use `force=true` to overwrite (will completely", @@ -453,11 +452,11 @@ function create_app(package_dir::String, # by first creating a normal "empty" sysimage and then use that to finally create the one # with the @ccallable function tmp_base_sysimage = joinpath(tmp, "tmp_sys.so") - create_sysimage(Symbol[]; sysimage_path=tmp_base_sysimage, project=project_path, + create_sysimage(Symbol[]; sysimage_path=tmp_base_sysimage, project=package_dir, incremental=false, filter_stdlibs=filter_stdlibs, cpu_target=APP_CPU_TARGET) - create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=project_path, + create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=package_dir, incremental=true, precompile_execution_file=precompile_execution_file, precompile_statements_file=precompile_statements_file, @@ -465,7 +464,7 @@ function create_app(package_dir::String, base_sysimage=tmp_base_sysimage, compiled_modules=false #= workaround julia#34076=#) else - create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=project_path, + create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=package_dir, incremental=incremental, filter_stdlibs=filter_stdlibs, precompile_execution_file=precompile_execution_file, precompile_statements_file=precompile_statements_file, @@ -518,11 +517,10 @@ function bundle_artifacts(ctx, app_dir) Pkg.Operations.load_all_deps!(ctx, pkgs) # Also want artifacts for the project itself - if ctx.env.pkg !== nothing - # This is kinda ugly... - ctx.env.pkg.path = dirname(ctx.env.project_file) - push!(pkgs, ctx.env.pkg) - end + @assert ctx.env.pkg !== nothing + # This is kinda ugly... + ctx.env.pkg.path = dirname(ctx.env.project_file) + push!(pkgs, ctx.env.pkg) # Collect all artifacts needed for the project artifact_paths = String[] diff --git a/test/runtests.jl b/test/runtests.jl index 8889b9f6..c76edf8f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,6 +4,14 @@ using Libdl ENV["JULIA_DEBUG"] = "PackageCompilerX" +# Make a new depot +new_depot = mktempdir() +@show DEPOT_PATH +mkpath(joinpath(new_depot, "registries")) +cp(joinpath(DEPOT_PATH[1], "registries", "General"), joinpath(new_depot, "registries", "General")) +ENV["JULIA_DEPOT_PATH"] = new_depot +Base.init_depot_path() + @testset "PackageCompilerX.jl" begin tmp = mktempdir() sysimage_path = joinpath(tmp, "sys." * Libdl.dlext) @@ -15,10 +23,9 @@ ENV["JULIA_DEBUG"] = "PackageCompilerX" # Test creating an app app_source_dir = joinpath(@__DIR__, "..", "examples/MyApp/") - # TODO: Also test something that actually gives audit warnings @test_logs PackageCompilerX.audit_app(app_source_dir) - app_compiled_dir = joinpath(tmp, "MyApp") + app_compiled_dir = joinpath(tmp, "MyAppCompiled") for incremental in (true, false) if incremental == false filter_stdlibs = (true, false) @@ -26,8 +33,14 @@ ENV["JULIA_DEBUG"] = "PackageCompilerX" filter_stdlibs = (false,) end for filter in filter_stdlibs - create_app(app_source_dir, app_compiled_dir; incremental=incremental, force=true, filter_stdlibs=filter, + tmp_app_source_dir = joinpath(tmp, "MyApp") + cp(app_source_dir, tmp_app_source_dir) + create_app(tmp_app_source_dir, app_compiled_dir; incremental=incremental, force=true, filter_stdlibs=filter, precompile_execution_file=joinpath(app_source_dir, "precompile_app.jl")) + rm(tmp_app_source_dir; recursive=true) + # Get rid of some local state + rm(joinpath(new_depot, "packages"); recursive=true) + rm(joinpath(new_depot, "compiled"); recursive=true) app_path = abspath(app_compiled_dir, "bin", "MyApp" * (Sys.iswindows() ? ".exe" : "")) app_output = read(`$app_path`, String) From 11d7343fb95da5666eafde7cf4490955a26037ce Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sat, 4 Jan 2020 21:33:13 +0000 Subject: [PATCH 47/80] do not try bundle artifacts that dont have an entry for our system (#69) --- src/PackageCompilerX.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index e7c71288..7a610b68 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -534,6 +534,8 @@ function bundle_artifacts(ctx, app_dir) @debug "bundling artifacts for $(pkg.name)" artifact_dict = Pkg.Artifacts.load_artifacts_toml(artifacts_toml_path) for name in keys(artifact_dict) + meta = Pkg.Artifacts.artifact_meta(name, artifacts_toml_path) + meta == nothing && continue @debug " \"$name\"" push!(artifact_paths, Pkg.Artifacts.ensure_artifact_installed(name, artifacts_toml_path)) end From 6707a64b308f39a5c01f31ed7c87f8827e85a2c8 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sat, 4 Jan 2020 21:33:24 +0000 Subject: [PATCH 48/80] dont run full Base init and do some cleanup (#70) --- src/PackageCompilerX.jl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 7a610b68..ab04f577 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -107,10 +107,9 @@ function create_sysimg_object_file(object_file::String, packages::Vector{Symbol} compiled_modules::Bool) # include all packages into the sysimg julia_code = """ - if !isdefined(Base, :uv_eventloop) - Base.reinit_stdio() - end - Base.__init__(); + Base.reinit_stdio() + Base.init_load_path() + Base.init_depot_path() """ for package in packages julia_code *= "using $package\n" @@ -153,6 +152,11 @@ function create_sysimg_object_file(object_file::String, packages::Vector{Symbol} """ julia_code *= precompile_code + julia_code *= """ + empty!(LOAD_PATH) + empty!(DEPOT_PATH) + """ + # finally, make julia output the resulting object file @debug "creating object file at $object_file" @info "PackageCompilerX: creating system image object file, this might take a while..." From b48365a18f11c4b8e688e32cc41de7c7a78c356e Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 7 Jan 2020 15:40:52 +0100 Subject: [PATCH 49/80] use compilecache instead of using in output process (#71) * use compilecache instead of using in output process --- examples/MyApp/src/MyApp.jl | 2 +- src/PackageCompilerX.jl | 132 ++++++++++++++++++++++++++++-------- test/runtests.jl | 5 +- 3 files changed, 108 insertions(+), 31 deletions(-) diff --git a/examples/MyApp/src/MyApp.jl b/examples/MyApp/src/MyApp.jl index b0d55ed9..cc3d5780 100644 --- a/examples/MyApp/src/MyApp.jl +++ b/examples/MyApp/src/MyApp.jl @@ -7,7 +7,7 @@ using Pkg.Artifacts const fooifier = joinpath(ensure_artifact_installed("fooifier", joinpath(@__DIR__, "..", "Artifacts.toml")), "bin", "fooifier" * (Sys.iswindows() ? ".exe" : "")) -Base.@ccallable function julia_main()::Cint +function julia_main() try real_main() catch diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index ab04f577..c0d128d2 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -46,11 +46,12 @@ function rewrite_sysimg_jl_only_needed_stdlibs(stdlibs::Vector{String}) sysimg_content = read(sysimg_source_path, String) # replaces the hardcoded list of stdlibs in sysimg.jl with # the stdlibs that is given as argument - return replace(sysimg_content, r"stdlibs = \[(.*?)\]"s => string("stdlibs = [", join(":" .* string.(stdlibs), ",\n"), "]")) + return replace(sysimg_content, + r"stdlibs = \[(.*?)\]"s => string("stdlibs = [", join(":" .* stdlibs, ",\n"), "]")) end function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String) - tmp = mktempdir(cleanup=false) + tmp = mktempdir() sysimg_source_path = Base.find_source_file("sysimg.jl") base_dir = dirname(sysimg_source_path) tmp_corecompiler_ji = joinpath(tmp, "corecompiler.ji") @@ -83,7 +84,7 @@ function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String) return tmp_sys_ji end -function run_precompilation_script(project::String, precompile_file::Union{String, Nothing}) +function run_precompilation_script(project::String, sysimg::String, precompile_file::Union{String, Nothing}) tracefile = tempname() if precompile_file == nothing arg = `-e ''` @@ -91,36 +92,40 @@ function run_precompilation_script(project::String, precompile_file::Union{Strin arg = `$precompile_file` end touch(tracefile) - cmd = `$(get_julia_cmd()) --sysimage=$(current_process_sysimage_path()) --project=$project + cmd = `$(get_julia_cmd()) --sysimage=$(sysimg) --project=$project --compile=all --trace-compile=$tracefile $arg` @debug "run_precompilation_script: running $cmd" read(cmd) return tracefile end -function create_sysimg_object_file(object_file::String, packages::Vector{Symbol}; +function do_compilecache(project, packages, sysimage) + # TODO: Only call compilecache on packages with stale cachefiles + compilecache = "" + for (pkg, uuid) in packages + compilecache *= """println(Base.compilecache(Base.PkgId(Base.UUID("$(uuid)"), $(repr(pkg)))))\n""" + end + + cmd = `$(get_julia_cmd()) --sysimage=$sysimage --project=$project -e $compilecache` + @debug "running $cmd" + paths = read(cmd, String) + return String.(split(paths)) +end + +function create_sysimg_object_file(object_file::String, packages::Vector{String}; project::String, base_sysimage::String, precompile_execution_file::Vector{String}, precompile_statements_file::Vector{String}, cpu_target::String, - compiled_modules::Bool) - # include all packages into the sysimg - julia_code = """ - Base.reinit_stdio() - Base.init_load_path() - Base.init_depot_path() - """ - for package in packages - julia_code *= "using $package\n" - end + isapp::Bool) - # handle precompilation + # Handle precompilation precompile_statements = "" @debug "running precompilation execution script..." tracefiles = String[] for file in (isempty(precompile_execution_file) ? (nothing,) : precompile_execution_file) - tracefile = run_precompilation_script(project, file) + tracefile = run_precompilation_script(project, base_sysimage, file) precompile_statements *= " append!(precompile_statements, readlines($(repr(tracefile))))\n" end for file in precompile_statements_file @@ -150,18 +155,74 @@ function create_sysimg_object_file(object_file::String, packages::Vector{Symbol} end end # module """ + + # include all packages into the sysimg + julia_code = """ + Base.reinit_stdio() + Base.init_load_path() + Base.init_depot_path() + """ + + # Create package => uuid map + ctx = create_pkg_context(project) + pkg_uuid_map = copy(ctx.env.project.deps) + if ctx.env.pkg !== nothing + pkg_uuid_map[ctx.env.pkg.name] = ctx.env.pkg.uuid + end + + # Run compilecache on packages to be put into sysimage + if !isempty(packages) + ji_paths = do_compilecache(project, [pkg => pkg_uuid_map[pkg] for pkg in packages], base_sysimage) + else + ji_paths = String[] + end + + for path in ji_paths + julia_code *= """ + @eval Module() begin + m = Base._require_from_serialized($(repr(path))) + m isa Exception && throw(m) + end + """ + end + + # Load the packages into Main + for pkg in packages + julia_code *= """ + const $(pkg) = Base.loaded_modules[ + Base.PkgId(Base.UUID("$(pkg_uuid_map[pkg])"), $(repr(pkg)))] + """ + end + julia_code *= precompile_code + if isapp + # If it is an app, there is only one packages + @assert length(packages) == 1 + packages[1] + app_start_code = """ + Base.@ccallable function julia_main()::Cint + try + $(packages[1]).julia_main() + catch + Core.print("julia_main() threw an unhandled exception") + return 1 + end + end + """ + julia_code *= app_start_code + end + julia_code *= """ empty!(LOAD_PATH) empty!(DEPOT_PATH) - """ + """ # finally, make julia output the resulting object file @debug "creating object file at $object_file" @info "PackageCompilerX: creating system image object file, this might take a while..." - cmd = `$(get_julia_cmd()) --compiled-modules=$(yesno(compiled_modules)) --cpu-target=$cpu_target + cmd = `$(get_julia_cmd()) --cpu-target=$cpu_target --sysimage=$base_sysimage --project=$project --output-o=$(object_file) -e $julia_code` @debug "running $cmd" run(cmd) @@ -186,6 +247,17 @@ function gather_stdlibs_project(ctx) return stdlibs_project end +function check_packages_in_project(ctx, packages) + packages_in_project = collect(keys(ctx.env.project.deps)) + if ctx.env.pkg !== nothing + push!(packages_in_project, ctx.env.pkg.name) + end + packages_not_in_project = setdiff(string.(packages), packages_in_project) + if !isempty(packages_not_in_project) + error("package(s) $(join(packages_not_in_project, ", ")) not in project") + end +end + """ create_sysimage(packages::Union{Symbol, Vector{Symbol}}; kwargs...) @@ -227,7 +299,7 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; replace_default::Bool=false, cpu_target::String=NATIVE_CPU_TARGET, base_sysimage::Union{Nothing, String}=nothing, - compiled_modules=true) + isapp::Bool=false) if replace_default==true if sysimage_path !== nothing error("cannot specify `sysimage_path` when `replace_default` is `true`") @@ -246,7 +318,7 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; end # Functions lower down handles `packages` and precompilation file as arrays so convert here - packages = vcat(packages) + packages = string.(vcat(packages)) # Package names are often used as string inside Julia precompile_execution_file = vcat(precompile_execution_file) precompile_statements_file = vcat(precompile_statements_file) @@ -255,6 +327,8 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; @debug "instantiating project at $(repr(project))" Pkg.instantiate(ctx) + check_packages_in_project(ctx, packages) + if !incremental if base_sysimage !== nothing error("cannot specify `base_sysimage` when `incremental=false`") @@ -271,6 +345,7 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; end end + # Create the sysimage object_file = tempname() * ".o" create_sysimg_object_file(object_file, packages; project=project, @@ -278,8 +353,10 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; precompile_execution_file=precompile_execution_file, precompile_statements_file=precompile_statements_file, cpu_target=cpu_target, - compiled_modules=compiled_modules) + isapp=isapp) create_sysimg_from_object_file(object_file, sysimage_path) + + # Maybe replace default sysimage if replace_default if !isfile(backup_default_sysimg_path()) @debug "making a backup of default sysimg" @@ -288,7 +365,8 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; @info "PackageCompilerX: default sysimg replaced, restart Julia for the new sysimg to be in effect" mv(sysimage_path, default_sysimg_path(); force=true) end - # TODO: Remove object file + rm(object_file; force=true) + return nothing end function create_sysimg_from_object_file(input_object::String, sysimage_path::String) @@ -380,7 +458,7 @@ The folder `app_source` needs to contain a package where the package include a function with the signature ``` -Base.@ccallable julia_main()::Cint +julia_main()::Cint # Perhaps do something based on ARGS ... end @@ -466,14 +544,14 @@ function create_app(package_dir::String, precompile_statements_file=precompile_statements_file, cpu_target=APP_CPU_TARGET, base_sysimage=tmp_base_sysimage, - compiled_modules=false #= workaround julia#34076=#) + isapp=true) else create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=package_dir, incremental=incremental, filter_stdlibs=filter_stdlibs, precompile_execution_file=precompile_execution_file, precompile_statements_file=precompile_statements_file, cpu_target=APP_CPU_TARGET, - compiled_modules=false #= workaround julia#34076=#) + isapp=true) end create_executable_from_sysimg(; sysimage_path=sysimg_file, executable_path=app_name) if Sys.isapple() @@ -485,8 +563,6 @@ function create_app(package_dir::String, return end -# This requires that the sysimg have been built so that there is a ccallable `julia_main` -# in Main. function create_executable_from_sysimg(;sysimage_path::String, executable_path::String) flags = join((cflags(), ldflags(), ldlibs()), " ") diff --git a/test/runtests.jl b/test/runtests.jl index c76edf8f..77348d18 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,7 +6,6 @@ ENV["JULIA_DEBUG"] = "PackageCompilerX" # Make a new depot new_depot = mktempdir() -@show DEPOT_PATH mkpath(joinpath(new_depot, "registries")) cp(joinpath(DEPOT_PATH[1], "registries", "General"), joinpath(new_depot, "registries", "General")) ENV["JULIA_DEPOT_PATH"] = new_depot @@ -19,7 +18,9 @@ Base.init_depot_path() precompile_execution_file="precompile_execution.jl", precompile_statements_file=["precompile_statements.jl", "precompile_statements2.jl"]) - run(`$(Base.julia_cmd()) -J $(sysimage_path) -e 'println(1337)'`) + # Check we can load sysimage and that Example is available in Main + str = read(`$(Base.julia_cmd()) -J $(sysimage_path) -e 'print(Example.hello("foo"))'`, String) + @test occursin("Hello, foo", str) # Test creating an app app_source_dir = joinpath(@__DIR__, "..", "examples/MyApp/") From d38edb01cf05eb8d3e39376ae315e81c65b20ef7 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 7 Jan 2020 16:34:10 +0100 Subject: [PATCH 50/80] fix precompilation path (#73) --- src/PackageCompilerX.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index c0d128d2..0acaa5db 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -300,6 +300,7 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; cpu_target::String=NATIVE_CPU_TARGET, base_sysimage::Union{Nothing, String}=nothing, isapp::Bool=false) + precompile_statements_file = abspath.(precompile_statements_file) if replace_default==true if sysimage_path !== nothing error("cannot specify `sysimage_path` when `replace_default` is `true`") @@ -499,6 +500,7 @@ function create_app(package_dir::String, filter_stdlibs=false, audit=true, force=false) + precompile_statements_file = abspath.(precompile_statements_file) package_dir = abspath(package_dir) ctx = create_pkg_context(package_dir) if isempty(ctx.env.manifest) From 6a6f7c42bf88f85fb10bb84fac4248745286fec6 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 7 Jan 2020 18:27:21 +0100 Subject: [PATCH 51/80] fix depot path (#75) --- src/embedding_wrapper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/embedding_wrapper.c b/src/embedding_wrapper.c index 627b876b..6c447d2c 100644 --- a/src/embedding_wrapper.c +++ b/src/embedding_wrapper.c @@ -34,7 +34,7 @@ int main(int argc, char *argv[]) } char buf[PATH_MAX]; - snprintf(buf, sizeof(buf), "JULIA_DEPOT_PATH=%s/../", free_path); + snprintf(buf, sizeof(buf), "JULIA_DEPOT_PATH=%s/../../", free_path); putenv(buf); putenv("JULIA_LOAD_PATH=@"); From 5ea685421a80077d1b61fc1e0d960aefc596e37c Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 8 Jan 2020 19:05:58 +0100 Subject: [PATCH 52/80] fix path to sysimg (#74) --- src/PackageCompilerX.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 0acaa5db..ad9873a3 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -228,7 +228,7 @@ function create_sysimg_object_file(object_file::String, packages::Vector{String} run(cmd) end -default_sysimg_path() = joinpath(julia_private_libdir(), "sys." * Libdl.dlext) +default_sysimg_path() = abspath(Sys.BINDIR, "..", "lib", "julia", "sys." * Libdl.dlext) default_sysimg_name() = basename(default_sysimg_path()) backup_default_sysimg_path() = default_sysimg_path() * ".backup" backup_default_sysimg_name() = basename(backup_default_sysimg_path()) From 96b2a948e38eae3b66d7361347bedb999f11804d Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 9 Jan 2020 13:43:21 +0100 Subject: [PATCH 53/80] change to just use using (#77) --- src/PackageCompilerX.jl | 40 ++++++++-------------------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index ad9873a3..f0c3d095 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -99,17 +99,13 @@ function run_precompilation_script(project::String, sysimg::String, precompile_f return tracefile end -function do_compilecache(project, packages, sysimage) - # TODO: Only call compilecache on packages with stale cachefiles - compilecache = "" - for (pkg, uuid) in packages - compilecache *= """println(Base.compilecache(Base.PkgId(Base.UUID("$(uuid)"), $(repr(pkg)))))\n""" - end - - cmd = `$(get_julia_cmd()) --sysimage=$sysimage --project=$project -e $compilecache` +# Load packages in a normal julia process to make them precompile "normally" +function do_ensurecompiled(project, packages, sysimage) + use = join("using " .* packages, '\n') + cmd = `$(get_julia_cmd()) --sysimage=$sysimage --project=$project -e $use` @debug "running $cmd" - paths = read(cmd, String) - return String.(split(paths)) + read(cmd, String) + return nothing end function create_sysimg_object_file(object_file::String, packages::Vector{String}; @@ -163,34 +159,14 @@ function create_sysimg_object_file(object_file::String, packages::Vector{String} Base.init_depot_path() """ - # Create package => uuid map - ctx = create_pkg_context(project) - pkg_uuid_map = copy(ctx.env.project.deps) - if ctx.env.pkg !== nothing - pkg_uuid_map[ctx.env.pkg.name] = ctx.env.pkg.uuid - end - # Run compilecache on packages to be put into sysimage if !isempty(packages) - ji_paths = do_compilecache(project, [pkg => pkg_uuid_map[pkg] for pkg in packages], base_sysimage) - else - ji_paths = String[] - end - - for path in ji_paths - julia_code *= """ - @eval Module() begin - m = Base._require_from_serialized($(repr(path))) - m isa Exception && throw(m) - end - """ + do_ensurecompiled(project, packages, base_sysimage) end - # Load the packages into Main for pkg in packages julia_code *= """ - const $(pkg) = Base.loaded_modules[ - Base.PkgId(Base.UUID("$(pkg_uuid_map[pkg])"), $(repr(pkg)))] + using $pkg """ end From b0471977627d40bf3690fa9d8dde7328d936fb38 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 16 Jan 2020 07:33:34 -0500 Subject: [PATCH 54/80] Don't show `failed to execute` messages by default (#79) --- src/PackageCompilerX.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index f0c3d095..54cfd165 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -146,7 +146,7 @@ function create_sysimg_object_file(object_file::String, packages::Vector{String} Base.include_string(PrecompileStagingArea, statement) catch # See julia issue #28808 - @error "failed to execute \$statement" + @debug "failed to execute \$statement" end end end # module From dcd6922cfa16ec34f926d93d2314f36411c9ccba Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sun, 19 Jan 2020 14:39:33 +0100 Subject: [PATCH 55/80] require 1.3.1 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index a54dfc0c..32bcfba3 100644 --- a/Project.toml +++ b/Project.toml @@ -9,7 +9,7 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] -julia = "1.3" +julia = "1.3.1" [extras] Example = "7876af07-990d-54b4-ab0e-23690620f79a" From d48d7e65355155ea202f9ac50bf44eb49568ca09 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 20 Jan 2020 14:42:28 +0100 Subject: [PATCH 56/80] add a Plots.jl example (#82) * add a Plots.jl example * more --- docs/make.jl | 1 + docs/src/examples/plots.md | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 docs/src/examples/plots.md diff --git a/docs/make.jl b/docs/make.jl index 2171db58..28a2f93a 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -15,6 +15,7 @@ makedocs( "Examples" => [ "examples/ohmyrepl.md", + "examples/plots.md", ], "PackageCompilerX - the manual way" => [ diff --git a/docs/src/examples/plots.md b/docs/src/examples/plots.md new file mode 100644 index 00000000..0a052f3c --- /dev/null +++ b/docs/src/examples/plots.md @@ -0,0 +1,59 @@ +# Creating a sysimage for fast plotting with Plots.jl + +A common complaint about Julia is that the "time to first plot" is a bit +longer than desired. In this example, we will create a sysimage that is made +to specifically improve this. + +To get a reference, we measure the time it takes to create the first plot with +the default sysimage: + +```jl-repl +julia> @time using Plots + 5.284989 seconds (5.22 M allocations: 308.954 MiB, 1.41% gc time) + +julia> @time (p = plot(rand(5), rand(5)); display(p)) + 13.769197 seconds (18.42 M allocations: 909.963 MiB, 1.75% gc time) +``` + +This is approximately 19 seconds from start of Julia to the first plot. + +We now create a precompilation file with exactly this workload in `precompile_plots.jl`: + + +```jl +using Plots +p = plot(rand(5), rand(5)) +display(p) +``` + +The custom sysimage is then created as: + +``` +using PackageCompilerX +create_sysimage(:Plots, sysimage_path="sys_plots.so", precompile_execution_file="precompile_plots.jl") +``` + +If we now start Julia with the flag `-Jsys_plots.so` and re-time our previous commands: + +``` +julia> @time using Plots + 0.000826 seconds (852 allocations: 42.125 KiB) + +julia> @time (p = plot(rand(5), rand(5)); display(p)) + 0.139642 seconds (468.42 k allocations: 12.176 MiB) +``` + +which is a sizeable speedup. + +Note that since we have more stuff in our sysimage, Julia is slightly slower to +start (0.04 seconds on this machine): + +``` +# Default sysimage +➜ time julia -e '' + 0.13s user 0.08s system 88% cpu 0.232 total + +# Custom sysimage +➜ time julia -Jsys_plots.so -e '' + 0.17s user 0.10s system 94% cpu 0.284 total +``` From 8720aa9e26417245503f8b0a5b7f9fca3e8bf801 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 20 Jan 2020 15:23:14 +0100 Subject: [PATCH 57/80] bump Documenter version --- docs/Manifest.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 1f2b1a59..4546464f 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -19,9 +19,9 @@ version = "0.8.1" [[Documenter]] deps = ["Base64", "Dates", "DocStringExtensions", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] -git-tree-sha1 = "0be9bf63e854a2408c2ecd3c600d68d4d87d8a73" +git-tree-sha1 = "51f0c7df46abb9c07d80e529718951e634670afb" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "0.24.2" +version = "0.24.4" [[InteractiveUtils]] deps = ["Markdown"] @@ -62,7 +62,7 @@ uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" version = "0.3.10" [[Pkg]] -deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Test", "UUIDs"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [[Printf]] From fd3764d35ce48dbd63e55c6365f51839c76b829d Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 20 Jan 2020 16:35:09 +0100 Subject: [PATCH 58/80] Add some flags to be more correct on 32 bit (#58) --- .travis.yml | 12 ++++++++++++ examples/MyApp/Artifacts.toml | 10 ++++++++++ src/PackageCompilerX.jl | 6 ++++-- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7900ab4c..844fb080 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,18 @@ os: - windows julia: - 1.3 +before_install: + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update; fi + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install gcc-multilib; fi +arch: + - x64 + - x86 +matrix: + allow_failures: + - arch: x86 + exclude: + - os: osx + arch: x86 branches: only: - master diff --git a/examples/MyApp/Artifacts.toml b/examples/MyApp/Artifacts.toml index 2b831cc3..9bc2f15f 100644 --- a/examples/MyApp/Artifacts.toml +++ b/examples/MyApp/Artifacts.toml @@ -8,6 +8,16 @@ os = "linux" sha256 = "5208c63a9d07e592c78f541fc13caa8cd191b11e7e77b31d407237c2b13ec391" url = "https://github.com/staticfloat/small_bin/raw/master/libfoo/libfoo.x86_64-linux-gnu.tar.gz" +[[fooifier]] +arch = "i686" +git-tree-sha1 = "c3a9f27382862092e064bcf4aeb3cb7190578338" +libc = "glibc" +os = "linux" + + [[fooifier.download]] + sha256 = "97655b6a218d61284723b6923d7c96e6a256fa68b9419d723c588aa24404b102" + url = "https://github.com/staticfloat/small_bin/raw/master/libfoo/libfoo.i686-linux-gnu.tar.gz" + [[fooifier]] arch = "x86_64" git-tree-sha1 = "f413ff2438a4e9e9dd69b23c35ca30de6af069cc" diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 54cfd165..2e85ba2f 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -17,6 +17,8 @@ current_process_sysimage_path() = unsafe_string(Base.JLOptions().image_file) all_stdlibs() = readdir(Sys.STDLIB) yesno(b::Bool) = b ? "yes" : "no" +bitflag() = Int == Int32 ? `-m32` : `-m64` +march() = (Int == Int32 ? `-march=pentium4` : ``) function get_compiler() cc = get(ENV, "JULIA_CC", nothing) @@ -357,7 +359,7 @@ function create_sysimg_from_object_file(input_object::String, sysimage_path::Str o_file = `-Wl,--whole-archive $input_object -Wl,--no-whole-archive` end extra = Sys.iswindows() ? `-Wl,--export-all-symbols` : `` - cmd = `$(get_compiler()) -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia $extra` + cmd = `$(get_compiler()) $(bitflag()) $(march()) -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia $extra` @debug "running $cmd" run(cmd) return nothing @@ -553,7 +555,7 @@ function create_executable_from_sysimg(;sysimage_path::String, else rpath = `-Wl,-rpath,\$ORIGIN:\$ORIGIN/../lib` end - cmd = `$(get_compiler()) -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) -o $(executable_path) $(wrapper) $(sysimage_path) -O2 $rpath $flags` + cmd = `$(get_compiler()) -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) $(bitflag()) $(march()) -o $(executable_path) $(wrapper) $(sysimage_path) -O2 $rpath $flags` @debug "running $cmd" run(cmd) return nothing From 9045f53bc556de1b8891444599463698c3a83c13 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 21 Jan 2020 16:22:11 +0100 Subject: [PATCH 59/80] add a DOCUMENTER_KEY to github workflow --- .github/workflows/documenter-workflow.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/documenter-workflow.yml b/.github/workflows/documenter-workflow.yml index 9f1e0dca..7a86a177 100644 --- a/.github/workflows/documenter-workflow.yml +++ b/.github/workflows/documenter-workflow.yml @@ -20,4 +20,5 @@ jobs: - name: Build and deploy env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} run: julia --project=docs/ docs/make.jl deploy From 95f3e2e1e4d23d4eadf907ab0c87f4d455106f4d Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 21 Jan 2020 16:35:44 +0100 Subject: [PATCH 60/80] add some langauge keys to code blocks --- docs/src/apps.md | 3 +-- docs/src/devdocs/binaries_part_2.md | 2 +- docs/src/devdocs/relocatable_part_3.md | 4 ++-- docs/src/devdocs/sysimages_part_1.md | 10 +++++----- docs/src/examples/plots.md | 4 ++-- docs/src/sysimages.md | 2 +- 6 files changed, 12 insertions(+), 13 deletions(-) diff --git a/docs/src/apps.md b/docs/src/apps.md index a92158da..7435ff0e 100644 --- a/docs/src/apps.md +++ b/docs/src/apps.md @@ -56,7 +56,6 @@ end ... end # module - ``` The absolute path to `lib_path` that `find_library` found is thus effectively @@ -207,7 +206,7 @@ MyApp.jl This is a problem that the Julia standard libraries themselves have: -``` +```jl-repl julia> @which rand() rand() in Random at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.3/Random/src/Random.jl:256 ``` diff --git a/docs/src/devdocs/binaries_part_2.md b/docs/src/devdocs/binaries_part_2.md index 229b64c2..b1e85e81 100644 --- a/docs/src/devdocs/binaries_part_2.md +++ b/docs/src/devdocs/binaries_part_2.md @@ -86,7 +86,7 @@ As in the previous tutorial, we do a "sample run" of our app to record what functions end up getting compiled. Here, we simply run the app on the sample CSV file since that should give good "coverage": -```jl +``` julia --startup-file=no --trace-compile=app_precompile.jl MyApp.jl "FL_insurance_sample.csv" ``` diff --git a/docs/src/devdocs/relocatable_part_3.md b/docs/src/devdocs/relocatable_part_3.md index 0a627956..a7755124 100644 --- a/docs/src/devdocs/relocatable_part_3.md +++ b/docs/src/devdocs/relocatable_part_3.md @@ -319,14 +319,14 @@ are encoding fundamentally non-relocatable information *into the source code*. As an example, many packages tend to use a `build.jl` file (which runs when the package is installed) that looks something like: -``` +```jl lib_path = find_library("libfoo") write("deps.jl", "const LIBFOO_PATH = $(repr(lib_path))") ``` The main package file then contains -``` +```jl if !isfile("../build/deps.jl") error("run Pkg.build(\"Package\") to re-build Package") end diff --git a/docs/src/devdocs/sysimages_part_1.md b/docs/src/devdocs/sysimages_part_1.md index 4836638c..106dc3f2 100644 --- a/docs/src/devdocs/sysimages_part_1.md +++ b/docs/src/devdocs/sysimages_part_1.md @@ -115,7 +115,7 @@ which is what the REPL evaluates in. Delving into some internals, there is a dictionary in `Base` that keeps track of all loaded modules: -``` +```jl-repl julia> Base.loaded_modules Dict{Base.PkgId,Module} with 33 entries: SHA [ea8e919c-243c-51af-8825-aaa63cd721ce] => SHA @@ -165,7 +165,7 @@ line](https://github.com/JuliaLang/julia/blob/49fb7924498e9fe813444cc684a24002e7 just want to give the path to the default sysimage which we can get the path to via: -``` +```jl-repl julia> unsafe_string(Base.JLOptions().image_file) "/home/kc/julia/lib/julia/sys.so" ``` @@ -192,7 +192,7 @@ initialized so Julia crashes while trying to print an error. The magic incantation to make IO work properly is `Base.reinit_stdio()`. To figure out the actual problem we modify the `custom_sysimage.jl` file to look like: -``` +```jl Base.reinit_stdio() using CSV ``` @@ -276,7 +276,7 @@ which creates the sysimage `sys.so`. We can compare the size of the new sysimage versus the default one and see that the new is a bit larger due to the extra packages it contains: -``` +```jl-repl julia> stat("sys.so").size / (1024*1024) 162.16205596923828 @@ -403,7 +403,7 @@ type `String` and `Bool`. Note that some of the symbols in the list of precompile statements have a bit of a weird syntax containing `Symbol(#...)`, e.g: -``` +```jl precompile(Tuple{typeof(Base.map), getfield(CSV, Symbol("##4#5")), Base.SubString{String}}) ``` diff --git a/docs/src/examples/plots.md b/docs/src/examples/plots.md index 0a052f3c..22a0a365 100644 --- a/docs/src/examples/plots.md +++ b/docs/src/examples/plots.md @@ -28,14 +28,14 @@ display(p) The custom sysimage is then created as: -``` +```jl using PackageCompilerX create_sysimage(:Plots, sysimage_path="sys_plots.so", precompile_execution_file="precompile_plots.jl") ``` If we now start Julia with the flag `-Jsys_plots.so` and re-time our previous commands: -``` +```jl-repl julia> @time using Plots 0.000826 seconds (852 allocations: 42.125 KiB) diff --git a/docs/src/sysimages.md b/docs/src/sysimages.md index b36ed0c5..7b0094d3 100644 --- a/docs/src/sysimages.md +++ b/docs/src/sysimages.md @@ -100,7 +100,7 @@ Alternatively, instead of giving a path to where the new sysimage should appear, can choose to replace the default sysimage. This is done by omitting the `sysimage_path` keyword and instead adding `replace_default=true`, for example: -``` +```jl create_sysimage([:Debugger, :OhMyREPL]; replace_default=true) ``` From ac1af6484631f50f2ed7e5efa92e27c08940da63 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 21 Jan 2020 17:11:13 +0100 Subject: [PATCH 61/80] fixup docs for create_app --- docs/src/apps.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/src/apps.md b/docs/src/apps.md index 7435ff0e..538215c0 100644 --- a/docs/src/apps.md +++ b/docs/src/apps.md @@ -79,8 +79,9 @@ The source of an app is a package with a project and manifest file. It should define a function with the signature ```jl -Base.@ccallable function julia_main()::Cint - ... +function julia_main()::Cint + # do something based on ARGS? + return 0 # if things finished successfully end ``` From 1338c5ea83a97bd111cea8cf1277d355d4e4c269 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 22 Jan 2020 10:28:24 +0100 Subject: [PATCH 62/80] fix highlighting --- docs/src/apps.md | 14 +++---- docs/src/devdocs/binaries_part_2.md | 12 +++--- docs/src/devdocs/relocatable_part_3.md | 14 +++---- docs/src/devdocs/sysimages_part_1.md | 54 +++++++++++++------------- docs/src/examples/ohmyrepl.md | 8 ++-- docs/src/examples/plots.md | 8 ++-- docs/src/sysimages.md | 4 +- 7 files changed, 57 insertions(+), 57 deletions(-) diff --git a/docs/src/apps.md b/docs/src/apps.md index 538215c0..d0118733 100644 --- a/docs/src/apps.md +++ b/docs/src/apps.md @@ -28,21 +28,21 @@ For something to be relocatable, everything that it depends on must also be relocatable. In the case of an app, the app itself and all the Julia packages it depends on must also relocatable. This is a bit of an issue because the Julia package ecosystem has rarely given much thought to relocatability -since creating "apps" has not been common. +since creating "apps" has not been common. The main problem with relocatability of Julia packages is that many packages are encoding fundamentally non-relocatable information *into the source code*. As an example, many packages tend to use a `build.jl` file (which runs when the package is first installed) that looks something like: -```jl +```julia lib_path = find_library("libfoo") write("deps.jl", "const LIBFOO_PATH = $(repr(lib_path))") ``` The main package file then contains: -```jl +```julia module Package if !isfile("../build/deps.jl") @@ -78,7 +78,7 @@ libraries. How this is used in practice is described later. The source of an app is a package with a project and manifest file. It should define a function with the signature -```jl +```julia function julia_main()::Cint # do something based on ARGS? return 0 # if things finished successfully @@ -92,7 +92,7 @@ https://github.com/KristofferC/PackageCompilerX.jl/tree/master/examples/MyApp. Regarding relocatability, PackageCompilerX provides a function [`audit_app(app_dir::String)`](@ref) that tries to find common problems with -relocatability in the app. +relocatability in the app. The app is then compiled using the [`create_app`](@ref) function that takes a path to the source code of the app and the destination where the app should be @@ -207,7 +207,7 @@ MyApp.jl This is a problem that the Julia standard libraries themselves have: -```jl-repl +```julia-repl julia> @which rand() rand() in Random at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.3/Random/src/Random.jl:256 ``` @@ -219,7 +219,7 @@ comes with the app. And while the source code is not available one can read the "lowered code" and use reflection to find things like the name of fields in structs and global variables etc: -```jl-repl +```julia-repl ~/PackageCompilerX.jl/examples/MyAppCompiled/bin kc/docs_apps* ❯ julia -q -JMyApp.so julia> MyApp = Base.loaded_modules[Base.PkgId(Base.UUID("f943f3d7-887a-4ed5-b0c0-a1d6899aa8f5"), "MyApp")] diff --git a/docs/src/devdocs/binaries_part_2.md b/docs/src/devdocs/binaries_part_2.md index b1e85e81..d4815d6d 100644 --- a/docs/src/devdocs/binaries_part_2.md +++ b/docs/src/devdocs/binaries_part_2.md @@ -1,7 +1,7 @@ # [Creating a binary from Julia code](@id man-tutorial-binary) This section targets how to build an executable based on the custom sysimage so -that it can be run without having to explicitly start a Julia session. +that it can be run without having to explicitly start a Julia session. ## Interacting with Julia through `libjulia`. @@ -30,7 +30,7 @@ app that parses a list of CSV files given as arguments to the app and prints the size of the parsed result. The code for the app (`MyApp.jl`) is shown below: -```jl +```julia module MyApp using CSV @@ -78,7 +78,7 @@ the sample CSV file [from the first tutorial](@ref man-tutorial-sysimage) ❯ time julia MyApp.jl FL_insurance_sample.csv FL_insurance_sample.csv: 36634x18 julia MyApp.jl FL_insurance_sample.csv 12.51s user 0.38s system 104% cpu 12.385 total -``` +``` ## Create the sysimage @@ -94,7 +94,7 @@ The `create_sysimage.jl` script look similar to before with the exception that we added an include of the app file inside the anonymous module where the precompiliation statements are evaluated in: -```jl +```julia Base.init_depot_path() Base.init_load_path() @@ -192,7 +192,7 @@ int main(int argc, char *argv[]) jl_atexit_hook(ret); return ret; } -``` +``` ## Building the executable @@ -221,7 +221,7 @@ ERROR: could not find file non_existing.csv Stacktrace: [1] error(::String) at ./error.jl:33 [2] real_main() at /home/kc/MyApp/MyApp.jl:21 - [3] julia_main() at /home/kc/MyApp/MyApp.jl:7 + [3] julia_main() at /home/kc/MyApp/MyApp.jl:7 ``` ### macOS considerations diff --git a/docs/src/devdocs/relocatable_part_3.md b/docs/src/devdocs/relocatable_part_3.md index a7755124..3670eb3a 100644 --- a/docs/src/devdocs/relocatable_part_3.md +++ b/docs/src/devdocs/relocatable_part_3.md @@ -52,7 +52,7 @@ The package we used in the previous examples to create a sysimage and executable was CSV.jl. Now, to simplify things, we will only use a very simple package with no relocatability problems that also has no dependencies. The app will take some input on stdin and print it out with color to the terminal -using the [Crayons.jl](https://github.com/KristofferC/Crayons.jl) package. +using the [Crayons.jl](https://github.com/KristofferC/Crayons.jl) package. When we add the Crayons.jl package we use a separate project to encapsulate things better by creating a new project in the app directory: @@ -73,7 +73,7 @@ julia> using Pkg; Pkg.add("Crayons") The code for the app itself is quite simple: -```jl +```julia module MyApp using Crayons @@ -128,7 +128,7 @@ project: ``` ~/MyApp ❯ echo "Hello, this is some stdin" | julia --project --startup-file=no --trace-compile=app_precompile.jl MyApp.jl green -``` +``` The `.o` file is then created with the same `generate_sysimage.jl` file as in part 2: @@ -280,7 +280,7 @@ the [`strings`](https://linux.die.net/man/1/strings) application we can see what an executable or library. Running it and grepping for some relevant substrings we can see that a bunch of absolute paths are stored inside the sysimage: -``` +``` ~/MyApp/MyApp/lib/julia ❯ strings sys.so | grep /home/kc /home/kc/.julia/packages/Crayons/P4fls/src/downcasts.jl @@ -304,7 +304,7 @@ Stacktrace: [1] error(::String) at ./error.jl:33 [2] real_main() at /home/kc/MyApp/MyApp.jl:20 [3] julia_main() at /home/kc/MyApp/MyApp.jl:6 -``` +``` This could be avoided by not printing stacktraces and perhaps even binary patching out the paths in the sysimage (not covered in this blog post). @@ -319,14 +319,14 @@ are encoding fundamentally non-relocatable information *into the source code*. As an example, many packages tend to use a `build.jl` file (which runs when the package is installed) that looks something like: -```jl +```julia lib_path = find_library("libfoo") write("deps.jl", "const LIBFOO_PATH = $(repr(lib_path))") ``` The main package file then contains -```jl +```julia if !isfile("../build/deps.jl") error("run Pkg.build(\"Package\") to re-build Package") end diff --git a/docs/src/devdocs/sysimages_part_1.md b/docs/src/devdocs/sysimages_part_1.md index 106dc3f2..40a2022f 100644 --- a/docs/src/devdocs/sysimages_part_1.md +++ b/docs/src/devdocs/sysimages_part_1.md @@ -42,7 +42,7 @@ here](http://spatialkeydocs.s3.amazonaws.com/FL_insurance_sample.csv.zip). One way of loading this file into Julia is by using the `CSV.jl` package. We can install `CSV.jl` using the Julia package manager `Pkg` as: -```jl-repl +```julia-repl julia> import Pkg; Pkg.add("CSV") Resolving package versions... Updating `~/.julia/environments/v1.3/Project.toml` @@ -53,7 +53,7 @@ julia> import Pkg; Pkg.add("CSV") When a package is loaded for the first time it gets "precompiled": -```jl-repl +```julia-repl julia> @time using CSV [ Info: Precompiling CSV [336ed68f-0bac-5ca0-87d4-7b16caf5d00b] 13.321758 seconds (2.69 M allocations: 151.302 MiB, 0.05% gc time) @@ -66,7 +66,7 @@ it is not obvious what types to compile the different methods for. Even with `CSV` "precompiled", there is a still some loading time, but it is significantly lower: -```jl-repl +```julia-repl julia> @time using CSV 0.694224 seconds (1.90 M allocations: 114.210 MiB) ``` @@ -74,7 +74,7 @@ julia> @time using CSV Let's load the sample CSV file: -```jl-repl +```julia-repl julia> @time CSV.read("FL_insurance_sample.csv"); 9.264898 seconds (37.17 M allocations: 2.278 GiB, 3.90% gc time)1 ``` @@ -82,7 +82,7 @@ julia> @time CSV.read("FL_insurance_sample.csv"); That's is quite a long time to read a smallish CSV file. One way to check the compilation overhead is by running the function again: -```jl-repl +```julia-repl julia> @time CSV.read("FL_insurance_sample.csv"); 0.083543 seconds (423 allocations: 34.695 KiB) ``` @@ -102,20 +102,20 @@ can be distributed we want to try to avoid as much runtime compilation If we time the loading of a standard library, it is clear that it is "cached" somehow since the time to load it is so short: -```jl-repl +```julia-repl julia> @time using Dates 0.000816 seconds (1.25 k allocations: 65.625 KiB) -``` +``` Since `Dates` is a standard library it comes bundled in the system image. In fact, `Dates` is already "loaded" when starting Julia. The effect of running `using Dates` just makes the module available in the `Main` module namespace -which is what the REPL evaluates in. +which is what the REPL evaluates in. Delving into some internals, there is a dictionary in `Base` that keeps track of all loaded modules: -```jl-repl +```julia-repl julia> Base.loaded_modules Dict{Base.PkgId,Module} with 33 entries: SHA [ea8e919c-243c-51af-8825-aaa63cd721ce] => SHA @@ -147,7 +147,7 @@ For now, the goal is to put `CSV` in the sysimage (in the same way as the standard library `Dates` is in it). We therefore initially simply create a file called `custom_sysimage.jl` with the content. -```jl +```julia using CSV ``` @@ -165,7 +165,7 @@ line](https://github.com/JuliaLang/julia/blob/49fb7924498e9fe813444cc684a24002e7 just want to give the path to the default sysimage which we can get the path to via: -```jl-repl +```julia-repl julia> unsafe_string(Base.JLOptions().image_file) "/home/kc/julia/lib/julia/sys.so" ``` @@ -192,7 +192,7 @@ initialized so Julia crashes while trying to print an error. The magic incantation to make IO work properly is `Base.reinit_stdio()`. To figure out the actual problem we modify the `custom_sysimage.jl` file to look like: -```jl +```julia Base.reinit_stdio() using CSV ``` @@ -200,7 +200,7 @@ using CSV and rerun the julia-command: -``` +``` julia --startup-file=no --output-o sys.o -J"/home/kc/julia/lib/julia/sys.so" custom_sysimage.jl ERROR: LoadError: ArgumentError: Package CSV not found in current path: - Run `import Pkg; Pkg.add("CSV")` to install the CSV package. @@ -220,7 +220,7 @@ package. Package-loading in Julia is based on the two arrays `LOAD_PATH` and `DEPOT_PATH`. Adding `@show LOAD_PATH` and `@show DEPOT_PATH` to the `custom_sysimage.jl` file and rerunning the command above prints: -```jl +```julia LOAD_PATH = String[] DEPOT_PATH = String[] ``` @@ -230,7 +230,7 @@ before including the standard libraries](https://github.com/JuliaLang/julia/blob the functions initializing these variables are explicitly called. Let us do the same by updating the `custom_sysimage.jl` file to: -```jl +```julia Base.init_depot_path() Base.init_load_path() @@ -257,7 +257,7 @@ use a C-compiler e.g. `gcc`. We need to link with `libjulia` so we need to give the compiler the path to where the julia library resides which can be gotten by: -```jl-repl +```julia-repl julia> abspath(Sys.BINDIR, Base.LIBDIR) "/home/kc/julia/lib" ``` @@ -276,7 +276,7 @@ which creates the sysimage `sys.so`. We can compare the size of the new sysimage versus the default one and see that the new is a bit larger due to the extra packages it contains: -```jl-repl +```julia-repl julia> stat("sys.so").size / (1024*1024) 162.16205596923828 @@ -293,7 +293,7 @@ On `macOS` the linker flag `-Wl,--whole-archive` is instead written as ``` gcc -shared -o sys.dylib -Wl,-all_load sys.o -L"/home/kc/Applications/julia-1.3.0-rc4/lib" -ljulia -``` +``` Note that the extension has been changed from `so` to `dylib` which is the convention for shared libraries on macOS. @@ -313,10 +313,10 @@ should work to produce the sysimage shared library. ### 3. Running Julia with the new sysimage -We start Julia with the `-Jsys.so` flag to load the new custom `sys.so` sysimage (or `sys.dylib`, `sys.dll` on macOS and Windows respecitively) +We start Julia with the `-Jsys.so` flag to load the new custom `sys.so` sysimage (or `sys.dylib`, `sys.dll` on macOS and Windows respecitively) and indeed loading CSV is now very fast: -```jl-repl +```julia-repl julia> @time using CSV 0.000432 seconds (665 allocations: 32.656 KiB) ``` @@ -324,7 +324,7 @@ julia> @time using CSV In fact, restarting Julia and looking at `Base.loaded_modules` we can see that, just like the standard libraries, CSV and its dependencies are already loaded when Julia is started: -```jl-repl +```julia-repl julia> Base.loaded_modules Dict{Base.PkgId,Module} with 52 entries: Parsers [69de0a69-1ddd-5017-9359-2bf0b02dc9f0] => Parsers @@ -338,7 +338,7 @@ but to compile the functions used by CSV the first time. Let's try it with the custom sysimage: -```jl-repl +```julia-repl julia> @time using CSV 0.001487 seconds (711 allocations: 35.203 KiB) @@ -378,7 +378,7 @@ We create a file called `generate_csv_precompile.jl` containing some "training code" that we will use as a base to figure out what functions end up getting compiled: -```jl +```julia using CSV CSV.read("FL_insurance_sample.csv") ``` @@ -393,7 +393,7 @@ julia --startup-file=no --trace-compile=csv_precompile.jl generate_csv_precompil Looking at `csv_precompile.jl` we can see hundreds of functions that end up getting compiled. For example, the line -```jl +```julia precompile(Tuple{typeof(CSV.getsource), String, Bool}) ``` @@ -403,7 +403,7 @@ type `String` and `Bool`. Note that some of the symbols in the list of precompile statements have a bit of a weird syntax containing `Symbol(#...)`, e.g: -```jl +```julia precompile(Tuple{typeof(Base.map), getfield(CSV, Symbol("##4#5")), Base.SubString{String}}) ``` @@ -429,7 +429,7 @@ symbols. The end result is a `custom_sysimage.jl` file looking like: -```jl +```julia Base.init_depot_path() Base.init_load_path() @@ -458,7 +458,7 @@ empty!(DEPOT_PATH) After repeating the process of creating the object file and using a compiler to create the shared library sysimage, we are in a position to time again: -```jl +```julia julia> @time using CSV 0.000408 seconds (665 allocations: 32.656 KiB) diff --git a/docs/src/examples/ohmyrepl.md b/docs/src/examples/ohmyrepl.md index 61bad2e3..b92b5aa3 100644 --- a/docs/src/examples/ohmyrepl.md +++ b/docs/src/examples/ohmyrepl.md @@ -33,12 +33,12 @@ the file `ohmyrepl_precompile`. It should be filled with lines like: ``` precompile(Tuple{typeof(OhMyREPL.Prompt.insert_keybindings), Any}) precompile(Tuple{typeof(OhMyREPL.__init__)}) -``` +``` These are functions that Julia compiled. We now just tell `create_sysimage` to use these precompile statements when creating the system image: -```jl +```julia PackageCompilerX.create_sysimage(:OhMyREPL; precompile_statements_file="ohmyrepl_precompile.jl", replace_default=true) ``` @@ -47,9 +47,9 @@ see the type text become highlighted with a significantly smaller delay than before creating the new system image -!!! note +!!! note If you want to go back to the default sysimage you can run - ```jl + ```julia PackageCompilerX.restore_default_sysimage() ``` diff --git a/docs/src/examples/plots.md b/docs/src/examples/plots.md index 22a0a365..e9d226fa 100644 --- a/docs/src/examples/plots.md +++ b/docs/src/examples/plots.md @@ -7,7 +7,7 @@ to specifically improve this. To get a reference, we measure the time it takes to create the first plot with the default sysimage: -```jl-repl +```julia-repl julia> @time using Plots 5.284989 seconds (5.22 M allocations: 308.954 MiB, 1.41% gc time) @@ -20,7 +20,7 @@ This is approximately 19 seconds from start of Julia to the first plot. We now create a precompilation file with exactly this workload in `precompile_plots.jl`: -```jl +```julia using Plots p = plot(rand(5), rand(5)) display(p) @@ -28,14 +28,14 @@ display(p) The custom sysimage is then created as: -```jl +```julia using PackageCompilerX create_sysimage(:Plots, sysimage_path="sys_plots.so", precompile_execution_file="precompile_plots.jl") ``` If we now start Julia with the flag `-Jsys_plots.so` and re-time our previous commands: -```jl-repl +```julia-repl julia> @time using Plots 0.000826 seconds (852 allocations: 42.125 KiB) diff --git a/docs/src/sysimages.md b/docs/src/sysimages.md index 7b0094d3..14d2c1c6 100644 --- a/docs/src/sysimages.md +++ b/docs/src/sysimages.md @@ -30,7 +30,7 @@ at when the sysimage was created. This means that no matter what package version you have installed in your current project, the one in the sysimage will take precedence. This can lead to bugs where you start with a project that needs a specific version of a package, but you have another one compiled into -the sysimage. +the sysimage. Putting packages in the sysimage is therefore only recommended if the load time of those packages is a significant problem and when these packages @@ -100,7 +100,7 @@ Alternatively, instead of giving a path to where the new sysimage should appear, can choose to replace the default sysimage. This is done by omitting the `sysimage_path` keyword and instead adding `replace_default=true`, for example: -```jl +```julia create_sysimage([:Debugger, :OhMyREPL]; replace_default=true) ``` From 30164fb4fc62f3170b0c9d711b638cd0577596a5 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 3 Feb 2020 10:18:33 +0100 Subject: [PATCH 63/80] fixup links after repo move (#87) --- LICENSE | 2 +- Project.toml | 2 +- README.md | 6 +++--- docs/make.jl | 2 +- docs/src/apps.md | 6 +++--- examples/MyApp/Project.toml | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/LICENSE b/LICENSE index 9adf483e..d3d225a9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2019 KristofferC +Copyright (c) 2019 Kristoffer Carlsson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Project.toml b/Project.toml index 32bcfba3..7649fc6a 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "PackageCompilerX" uuid = "dffaa6cc-da53-48e5-b007-4292dfcc27f1" -authors = ["KristofferC "] +authors = ["Kristoffer Carlsson "] version = "0.1.0" [deps] diff --git a/README.md b/README.md index 6f04137c..931c392f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # PackageCompilerX -[![Build Status](https://travis-ci.com/KristofferC/PackageCompilerX.jl.svg?branch=master)](https://travis-ci.com/KristofferC/PackageCompilerX.jl) -[![Codecov](https://codecov.io/gh/KristofferC/PackageCompilerX.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/KristofferC/PackageCompilerX.jl) +[![Build Status](https://travis-ci.com/JuliaComputing/PackageCompilerX.jl.svg?branch=master)](https://travis-ci.com/JuliaComputing/PackageCompilerX.jl) +[![Codecov](https://codecov.io/gh/JuliaComputing/PackageCompilerX.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/JuliaComputing/PackageCompilerX.jl) [![][docs-stable-img]][docs-stable-url] PackageCompilerX is a Julia package with two main purposes: @@ -13,4 +13,4 @@ PackageCompilerX is a Julia package with two main purposes: For installation and usage instructions, see the [documentation][docs-stable-url]. [docs-stable-img]: https://img.shields.io/badge/docs-stable-blue.svg -[docs-stable-url]: https://kristofferc.github.io/PackageCompilerX.jl/dev +[docs-stable-url]: https://JuliaComputing.github.io/PackageCompilerX.jl/dev diff --git a/docs/make.jl b/docs/make.jl index 28a2f93a..adba4898 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -30,5 +30,5 @@ makedocs( ) deploydocs( - repo = "github.com/KristofferC/PackageCompilerX.jl.git", + repo = "github.com/JuliaComputing/PackageCompilerX.jl.git", ) diff --git a/docs/src/apps.md b/docs/src/apps.md index d0118733..417582ab 100644 --- a/docs/src/apps.md +++ b/docs/src/apps.md @@ -88,7 +88,7 @@ end which will be the entry point of the app (the function that runs when the executable in the app is run). A skeleton of an app to start working from can be found at -https://github.com/KristofferC/PackageCompilerX.jl/tree/master/examples/MyApp. +https://github.com/JuliaComputing/PackageCompilerX.jl/tree/master/examples/MyApp. Regarding relocatability, PackageCompilerX provides a function [`audit_app(app_dir::String)`](@ref) that tries to find common problems with @@ -178,8 +178,8 @@ things so that they can be found during runtime on other machines. The example app uses the artifact system to depend on a very simple toy binary that does some simple arithmetic. It is instructive to see how the [artifact -file](https://github.com/KristofferC/PackageCompilerX.jl/blob/8aa31d60ace6dd278daed9bef62fd5a01258da1e/examples/MyApp/Artifacts.toml) -is [used in the source](https://github.com/KristofferC/PackageCompilerX.jl/blob/8aa31d60ace6dd278daed9bef62fd5a01258da1e/examples/MyApp/src/MyApp.jl#L7-L8). +file](https://github.com/JuliaComputing/PackageCompilerX.jl/blob/master/examples/MyApp/Artifacts.toml) +is [used in the source](https://github.com/JuliaComputing/PackageCompilerX.jl/blob/d722a3d91abe328ebd239e2f45660be35263ebe1/examples/MyApp/src/MyApp.jl#L7-L8). ### Reverse engineering the compiled app diff --git a/examples/MyApp/Project.toml b/examples/MyApp/Project.toml index db2daf45..246f67e9 100644 --- a/examples/MyApp/Project.toml +++ b/examples/MyApp/Project.toml @@ -1,6 +1,6 @@ name = "MyApp" uuid = "f943f3d7-887a-4ed5-b0c0-a1d6899aa8f5" -authors = ["KristofferC "] +authors = ["Kristoffer Carlsson "] version = "0.1.0" [deps] From f51cb261d745d248c2f275009dbd2441e77dda2a Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Mon, 27 Jan 2020 07:07:21 -0500 Subject: [PATCH 64/80] Add `cpu_target` to the docstrings of `create_app` and `create_sysimage` tweak docstring --- src/PackageCompilerX.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 2e85ba2f..d46dbbec 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -266,6 +266,8 @@ by setting the envirnment variable `JULIA_CC` to a path to a compiler - `replace_default::Bool`: If `true`, replaces the default system image which is automatically used when Julia starts. To replace with the one Julia ships with, use [`restore_default_sysimage()`](@ref) + +- `cpu_target::String`: The value to use for `JULIA_CPU_TARGET` when building the system image. """ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; sysimage_path::Union{String,Nothing}=nothing, @@ -469,6 +471,8 @@ compiler. to `true`. - `force::Bool`: Remove the folder `compiled_app` if it exists before creating the app. + +- `cpu_target::String`: The value to use for `JULIA_CPU_TARGET` when building the system image. """ function create_app(package_dir::String, app_dir::String; From b8b3cec90f62a1d5fe72030bd0ca2ae9b4e7f866 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Fri, 24 Jan 2020 16:31:31 -0500 Subject: [PATCH 65/80] Docs: Document how to use a package's test suite to generate precompile statements --- docs/src/sysimages.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/src/sysimages.md b/docs/src/sysimages.md index 14d2c1c6..1dcd036b 100644 --- a/docs/src/sysimages.md +++ b/docs/src/sysimages.md @@ -177,7 +177,7 @@ Using the just created system image, we can see that the `hello` function no lon ❯ ``` -### Using a manually generated list of precompile statements +#### Using a manually generated list of precompile statements Starting Julia with `--trace-compile=file.jl` will emit precompilation statements to `file.jl` for the duration of the started Julia process. This @@ -191,3 +191,16 @@ It is also possible to use [SnoopCompile.jl](https://timholy.github.io/SnoopCompile.jl/stable/snoopi/#auto-1) to create files with precompilation statements. + +#### Using a package's test suite to generate precompile statements + +It is also possible to use a package's test suite to generate a list of +precompile statements by including the content: + +```julia +import Example +include(joinpath(pkgdir(Example), "test", "runtests.jl")) +``` + +in the precompile file. Note that you need to have any test dependencies installed +in your current project. From 9101f3fe4ba6ebd4b30341774e4daca3d26cf5e3 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 4 Feb 2020 17:57:42 +0100 Subject: [PATCH 66/80] use travis for building docs (#88) --- .travis.yml | 10 +++++++++- docs/Manifest.toml | 11 ++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 844fb080..4f4c27e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,12 +11,20 @@ before_install: arch: - x64 - x86 -matrix: +jobs: allow_failures: - arch: x86 exclude: - os: osx arch: x86 + include: + - stage: "Documentation" + julia: 1.3 + os: linux + script: + - julia --project=docs/ -e 'using Pkg; Pkg.instantiate()' + - julia --project=docs/ docs/make.jl + after_success: skip branches: only: - master diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 4546464f..90948056 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -19,9 +19,9 @@ version = "0.8.1" [[Documenter]] deps = ["Base64", "Dates", "DocStringExtensions", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] -git-tree-sha1 = "51f0c7df46abb9c07d80e529718951e634670afb" +git-tree-sha1 = "d497bcc45bb98a1fbe19445a774cfafeabc6c6df" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "0.24.4" +version = "0.24.5" [[InteractiveUtils]] deps = ["Markdown"] @@ -34,6 +34,7 @@ uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" version = "0.21.0" [[LibGit2]] +deps = ["Printf"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" [[Libdl]] @@ -57,12 +58,12 @@ version = "0.1.0" [[Parsers]] deps = ["Dates", "Test"] -git-tree-sha1 = "0139ba59ce9bc680e2925aec5b7db79065d60556" +git-tree-sha1 = "d112c19ccca00924d5d3a38b11ae2b4b268dda39" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "0.3.10" +version = "0.3.11" [[Pkg]] -deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Test", "UUIDs"] +deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [[Printf]] From 92c270a542b9bf55582cbc5e3ad7d4504cb8b3cf Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 5 Feb 2020 14:22:11 +0100 Subject: [PATCH 67/80] add a note to say to use official binaries --- docs/src/index.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/src/index.md b/docs/src/index.md index 096d42f2..f01bf062 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -14,6 +14,12 @@ come across. ## Installation instructions +!!! note + + It is strongly recommended to use the official binaries that are downloaded from + https://julialang.org/downloads/. Distribution-provided Julia installations are + unlikely to work properly with this package. + To use PackageCompilerX a C-compiler needs to be available: ### macOS, Linux From 0bc3fbc8a198d16483109e86614bc82c27180f52 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 5 Feb 2020 22:41:50 +0100 Subject: [PATCH 68/80] set BINDIR in output process (#92) --- src/PackageCompilerX.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index d46dbbec..be1d7932 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -157,6 +157,7 @@ function create_sysimg_object_file(object_file::String, packages::Vector{String} # include all packages into the sysimg julia_code = """ Base.reinit_stdio() + @eval Sys BINDIR = ccall(:jl_get_julia_bindir, Any, ())::String Base.init_load_path() Base.init_depot_path() """ From 49e21a3983646c7848b6d2a5088ba6ff3e404249 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 6 Feb 2020 10:41:02 +0100 Subject: [PATCH 69/80] Use artifacts on windows to get the compiler (#91) * artifacts for windows compiler --- Artifacts.toml | 9 +++++++++ docs/src/index.md | 7 +------ src/PackageCompilerX.jl | 37 ++++++++++++++++++++++++++----------- 3 files changed, 36 insertions(+), 17 deletions(-) create mode 100644 Artifacts.toml diff --git a/Artifacts.toml b/Artifacts.toml new file mode 100644 index 00000000..3f86a9c3 --- /dev/null +++ b/Artifacts.toml @@ -0,0 +1,9 @@ +[[x86_64-w64-mingw32]] +git-tree-sha1 = "572b61b5075459e3ed62317e674398166ca98dd4" +os = "windows" +arch = "x86_64" +lazy = true + + [[x86_64-w64-mingw32.download]] + sha256 = "fe3f401bc936fbe6af940b26c5e0f266f762a3416f979c706e599b24082dc5c7" + url = "https://github.com/JuliaComputing/PackageCompilerX.jl/releases/download/v0.1/x86_64-8.1.0-release-posix-seh-rt_v6-rev0.tar.gz" diff --git a/docs/src/index.md b/docs/src/index.md index f01bf062..790ff470 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -29,9 +29,4 @@ For macOS, using something like `homebrew` and for Linux the system package mana ### Windows -For Windows, the minGW compiler toolchain is needed. It can be downloaded from e.g. -[https://sourceforge.net/projects/mingw-w64/files/](https://sourceforge.net/projects/mingw-w64/files/) or by following the -instructions for setting up a toolchain capable of compiling Julia itself on Windows at -[https://github.com/JuliaLang/julia/blob/master/doc/build/windows.md#cygwin-to-mingw-cross-compiling](https://github.com/JuliaLang/julia/blob/master/doc/build/windows.md#cygwin-to-mingw-cross-compiling) -and then run PackageCompilerX from the cygwin terminal. Alternatively, the package manager -[chocolatey](https://chocolatey.org/) can be used to get mingw on Windows. +A suitable compiler will be automatically installed the first time it is neeed. diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index be1d7932..7bbe0edd 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -20,20 +20,28 @@ yesno(b::Bool) = b ? "yes" : "no" bitflag() = Int == Int32 ? `-m32` : `-m64` march() = (Int == Int32 ? `-march=pentium4` : ``) +function windows_compiler_artifact_path(f, compiler) + if Sys.iswindows() + withenv("PATH" => string(ENV["PATH"], ";", dirname(compiler))) do + f() + end + else + f() + end +end + function get_compiler() cc = get(ENV, "JULIA_CC", nothing) if cc !== nothing - return `$cc` + return cc + end + if Sys.iswindows() + return joinpath(Pkg.Artifacts.artifact"x86_64-w64-mingw32", "mingw64", "bin", "gcc.exe") end if Sys.which("gcc") !== nothing - return `gcc` + return "gcc" elseif Sys.which("clang") !== nothing - return `clang` - end - if Sys.iswindows() - if Sys.which("x86_64-w64-mingw32-gcc") !== nothing - return `x86_64-w64-mingw32-gcc` - end + return "clang" end error("could not find a compiler, looked for `gcc` and `clang`") end @@ -362,9 +370,12 @@ function create_sysimg_from_object_file(input_object::String, sysimage_path::Str o_file = `-Wl,--whole-archive $input_object -Wl,--no-whole-archive` end extra = Sys.iswindows() ? `-Wl,--export-all-symbols` : `` - cmd = `$(get_compiler()) $(bitflag()) $(march()) -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia $extra` + compiler = get_compiler() + cmd = `$compiler $(bitflag()) $(march()) -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia $extra` @debug "running $cmd" - run(cmd) + windows_compiler_artifact_path(compiler) do + run(cmd) + end return nothing end @@ -560,9 +571,13 @@ function create_executable_from_sysimg(;sysimage_path::String, else rpath = `-Wl,-rpath,\$ORIGIN:\$ORIGIN/../lib` end - cmd = `$(get_compiler()) -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) $(bitflag()) $(march()) -o $(executable_path) $(wrapper) $(sysimage_path) -O2 $rpath $flags` + compiler = get_compiler() + cmd = `$compiler -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) $(bitflag()) $(march()) -o $(executable_path) $(wrapper) $(sysimage_path) -O2 $rpath $flags` @debug "running $cmd" run(cmd) + windows_compiler_artifact_path(compiler) do + run(cmd) + end return nothing end From 4d62801c4b0f3c4f0f94a7f758bb460679e3a550 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 6 Feb 2020 13:07:14 +0100 Subject: [PATCH 70/80] tweak how default sysimage is replaced on Windows (#95) * tweak how default sysimage is replaced * tweak * mv instead of cp and rm --- src/PackageCompilerX.jl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 7bbe0edd..15f5d3c5 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -20,6 +20,14 @@ yesno(b::Bool) = b ? "yes" : "no" bitflag() = Int == Int32 ? `-m32` : `-m64` march() = (Int == Int32 ? `-march=pentium4` : ``) +# Overwriting an open file is problematic in Windows +# so move it out of the way first +function move_default_sysimage_if_windows() + if Sys.iswindows() && isfile(default_sysimg_path()) + mv(default_sysimg_path(), tempname()) + end +end + function windows_compiler_artifact_path(f, compiler) if Sys.iswindows() withenv("PATH" => string(ENV["PATH"], ";", dirname(compiler))) do @@ -352,8 +360,9 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; @debug "making a backup of default sysimg" cp(default_sysimg_path(), backup_default_sysimg_path()) end - @info "PackageCompilerX: default sysimg replaced, restart Julia for the new sysimg to be in effect" + move_default_sysimage_if_windows() mv(sysimage_path, default_sysimg_path(); force=true) + @info "PackageCompilerX: default sysimg replaced, restart Julia for the new sysimg to be in effect" end rm(object_file; force=true) return nothing @@ -389,8 +398,8 @@ function restore_default_sysimage() if !isfile(backup_default_sysimg_path()) error("did not find a backup sysimg") end - cp(backup_default_sysimg_path(), default_sysimg_path(); force=true) - rm(backup_default_sysimg_path()) + move_default_sysimage_if_windows() + mv(backup_default_sysimg_path(), default_sysimg_path(); force=true) @info "PackageCompilerX: default sysimg restored, restart Julia for the new sysimg to be in effect" return nothing end From 155f9772c1261433ab2a0c13817e0662d777a5cb Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sat, 8 Feb 2020 13:11:35 +0100 Subject: [PATCH 71/80] Allow user to customize `cpu_target` in `create_app` (#101) Co-authored-by: Dilum Aluthge --- src/PackageCompilerX.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 15f5d3c5..76cd87a6 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -502,7 +502,8 @@ function create_app(package_dir::String, incremental=false, filter_stdlibs=false, audit=true, - force=false) + force=false, + cpu_target::String=APP_CPU_TARGET) precompile_statements_file = abspath.(precompile_statements_file) package_dir = abspath(package_dir) ctx = create_pkg_context(package_dir) @@ -541,13 +542,13 @@ function create_app(package_dir::String, tmp_base_sysimage = joinpath(tmp, "tmp_sys.so") create_sysimage(Symbol[]; sysimage_path=tmp_base_sysimage, project=package_dir, incremental=false, filter_stdlibs=filter_stdlibs, - cpu_target=APP_CPU_TARGET) + cpu_target=cpu_target) create_sysimage(Symbol(app_name); sysimage_path=sysimg_file, project=package_dir, incremental=true, precompile_execution_file=precompile_execution_file, precompile_statements_file=precompile_statements_file, - cpu_target=APP_CPU_TARGET, + cpu_target=cpu_target, base_sysimage=tmp_base_sysimage, isapp=true) else @@ -555,7 +556,7 @@ function create_app(package_dir::String, incremental=incremental, filter_stdlibs=filter_stdlibs, precompile_execution_file=precompile_execution_file, precompile_statements_file=precompile_statements_file, - cpu_target=APP_CPU_TARGET, + cpu_target=cpu_target, isapp=true) end create_executable_from_sysimg(; sysimage_path=sysimg_file, executable_path=app_name) From b144cbf358022632ec8ef0946f254b51065b64e7 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sat, 8 Feb 2020 15:37:53 +0100 Subject: [PATCH 72/80] add an option to include a script into the sysimage (#103) --- src/PackageCompilerX.jl | 16 +++++++++++++++- test/runtests.jl | 8 ++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 76cd87a6..5936a709 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -132,6 +132,7 @@ function create_sysimg_object_file(object_file::String, packages::Vector{String} precompile_execution_file::Vector{String}, precompile_statements_file::Vector{String}, cpu_target::String, + script::Union{Nothing, String}, isapp::Bool) # Handle precompilation @@ -178,7 +179,8 @@ function create_sysimg_object_file(object_file::String, packages::Vector{String} Base.init_depot_path() """ - # Run compilecache on packages to be put into sysimage + # Ensure packages to be put into sysimage are precompiled by loading them in a + # separate process first. if !isempty(packages) do_ensurecompiled(project, packages, base_sysimage) end @@ -191,6 +193,12 @@ function create_sysimg_object_file(object_file::String, packages::Vector{String} julia_code *= precompile_code + if script !== nothing + julia_code *= """ + include($(repr(abspath(script)))) + """ + end + if isapp # If it is an app, there is only one packages @assert length(packages) == 1 @@ -284,7 +292,11 @@ by setting the envirnment variable `JULIA_CC` to a path to a compiler - `replace_default::Bool`: If `true`, replaces the default system image which is automatically used when Julia starts. To replace with the one Julia ships with, use [`restore_default_sysimage()`](@ref) +### Advanced keyword arguments + - `cpu_target::String`: The value to use for `JULIA_CPU_TARGET` when building the system image. + +- `script::String`: Path to a file that gets executed in the `--output-o` process. """ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; sysimage_path::Union{String,Nothing}=nothing, @@ -295,6 +307,7 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; filter_stdlibs=false, replace_default::Bool=false, cpu_target::String=NATIVE_CPU_TARGET, + script::Union{Nothing, String}=nothing, base_sysimage::Union{Nothing, String}=nothing, isapp::Bool=false) precompile_statements_file = abspath.(precompile_statements_file) @@ -351,6 +364,7 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; precompile_execution_file=precompile_execution_file, precompile_statements_file=precompile_statements_file, cpu_target=cpu_target, + script=script, isapp=isapp) create_sysimg_from_object_file(object_file, sysimage_path) diff --git a/test/runtests.jl b/test/runtests.jl index 77348d18..a255432a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -14,13 +14,17 @@ Base.init_depot_path() @testset "PackageCompilerX.jl" begin tmp = mktempdir() sysimage_path = joinpath(tmp, "sys." * Libdl.dlext) + script = tempname() + write(script, "script_func() = println(\"I am a script\")") create_sysimage(:Example; sysimage_path=sysimage_path, precompile_execution_file="precompile_execution.jl", precompile_statements_file=["precompile_statements.jl", - "precompile_statements2.jl"]) + "precompile_statements2.jl"], + script=script) # Check we can load sysimage and that Example is available in Main - str = read(`$(Base.julia_cmd()) -J $(sysimage_path) -e 'print(Example.hello("foo"))'`, String) + str = read(`$(Base.julia_cmd()) -J $(sysimage_path) -e 'println(Example.hello("foo")); script_func()'`, String) @test occursin("Hello, foo", str) + @test occursin("I am a script", str) # Test creating an app app_source_dir = joinpath(@__DIR__, "..", "examples/MyApp/") From f8d93ba3b73823877de2f67d2f37286650bcf0e7 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 8 Feb 2020 12:07:39 -0500 Subject: [PATCH 73/80] Try fix arm64 (#100) * add arm64 to travis --- .travis.yml | 9 +++++++-- examples/MyApp/Artifacts.toml | 9 +++++++++ src/PackageCompilerX.jl | 18 ++++++++++++++++-- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4f4c27e4..7ed91e20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,16 +7,21 @@ julia: - 1.3 before_install: - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update; fi - - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install gcc-multilib; fi + - if [ "$TRAVIS_OS_NAME" = "linux" ] && [ $TRAVIS_CPU_ARCH != arm64 ]; then sudo apt-get install gcc-multilib; fi arch: - - x64 + - amd64 - x86 + - arm64 jobs: allow_failures: - arch: x86 exclude: - os: osx arch: x86 + - os: osx + arch: arm64 + - os: windows + arch: arm64 include: - stage: "Documentation" julia: 1.3 diff --git a/examples/MyApp/Artifacts.toml b/examples/MyApp/Artifacts.toml index 9bc2f15f..d33e2118 100644 --- a/examples/MyApp/Artifacts.toml +++ b/examples/MyApp/Artifacts.toml @@ -35,3 +35,12 @@ os = "windows" [[fooifier.download]] sha256 = "7f8939e9529835b83810d3ae7e2556f6e002d571f619894e54ece42ea5262b7f" url = "https://github.com/staticfloat/small_bin/raw/master/libfoo/libfoo.x86_64-w64-mingw32.tar.gz" + +[[fooifier]] +arch = "aarch64" +git-tree-sha1 = "281cbe3dd65aa4bdb887bfb29651da500c81e242" +os = "linux" + + [[fooifier.download]] + sha256 = "36886ac25cf5678c01fe20630b413f9354b7a3721c6a2c2043162f7ebd147ff5" + url = "https://github.com/staticfloat/small_bin/raw/master/libfoo/libfoo.aarch64-linux-gnu.tar.gz" diff --git a/src/PackageCompilerX.jl b/src/PackageCompilerX.jl index 5936a709..a298c852 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompilerX.jl @@ -17,8 +17,22 @@ current_process_sysimage_path() = unsafe_string(Base.JLOptions().image_file) all_stdlibs() = readdir(Sys.STDLIB) yesno(b::Bool) = b ? "yes" : "no" -bitflag() = Int == Int32 ? `-m32` : `-m64` -march() = (Int == Int32 ? `-march=pentium4` : ``) + +function bitflag() + if Sys.ARCH == :aarch64 || Sys.ARCH == :arm + return `` + else + return Int == Int32 ? `-m32` : `-m64` + end +end + +function march() + if Sys.ARCH == :aarch64 || Sys.ARCH == :arm + return (Int == Int32 ? `-march=armv7-a` : `-march=armv8-a+crypto+simd`) + else + return (Int == Int32 ? `-march=pentium4` : ``) + end +end # Overwriting an open file is problematic in Windows # so move it out of the way first From 6965e6846427e58c10c6894debfcb68daa812b5d Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sun, 9 Feb 2020 14:48:39 +0100 Subject: [PATCH 74/80] update Project uuid and version --- Project.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Project.toml b/Project.toml index 7649fc6a..0a025bd7 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,6 @@ name = "PackageCompilerX" -uuid = "dffaa6cc-da53-48e5-b007-4292dfcc27f1" -authors = ["Kristoffer Carlsson "] -version = "0.1.0" +uuid = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" +version = "1.0.0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" From 5df4211f691b9e2fc10aaa948da9c52ff19ba0d1 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sun, 9 Feb 2020 14:50:42 +0100 Subject: [PATCH 75/80] rename PackageCompilerX to PackageCompiler --- Project.toml | 2 +- README.md | 10 +++--- docs/Manifest.toml | 2 +- docs/Project.toml | 2 +- docs/make.jl | 8 ++--- docs/src/apps.md | 34 +++++++++---------- docs/src/devdocs/binaries_part_2.md | 2 +- docs/src/devdocs/intro.md | 8 ++--- docs/src/examples/ohmyrepl.md | 6 ++-- docs/src/examples/plots.md | 2 +- docs/src/index.md | 8 ++--- docs/src/refs.md | 8 ++--- docs/src/sysimages.md | 22 ++++++------ ...PackageCompilerX.jl => PackageCompiler.jl} | 10 +++--- test/runtests.jl | 8 ++--- 15 files changed, 66 insertions(+), 66 deletions(-) rename src/{PackageCompilerX.jl => PackageCompiler.jl} (98%) diff --git a/Project.toml b/Project.toml index 0a025bd7..c244d5ca 100644 --- a/Project.toml +++ b/Project.toml @@ -1,4 +1,4 @@ -name = "PackageCompilerX" +name = "PackageCompiler" uuid = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" version = "1.0.0" diff --git a/README.md b/README.md index 931c392f..c70d9f4f 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# PackageCompilerX +# PackageCompiler -[![Build Status](https://travis-ci.com/JuliaComputing/PackageCompilerX.jl.svg?branch=master)](https://travis-ci.com/JuliaComputing/PackageCompilerX.jl) -[![Codecov](https://codecov.io/gh/JuliaComputing/PackageCompilerX.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/JuliaComputing/PackageCompilerX.jl) +[![Build Status](https://travis-ci.com/JuliaComputing/PackageCompiler.jl.svg?branch=master)](https://travis-ci.com/JuliaComputing/PackageCompiler.jl) +[![Codecov](https://codecov.io/gh/JuliaComputing/PackageCompiler.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/JuliaComputing/PackageCompiler.jl) [![][docs-stable-img]][docs-stable-url] -PackageCompilerX is a Julia package with two main purposes: +PackageCompiler is a Julia package with two main purposes: 1. Creating custom sysimages for reduced latency when working locally with packages that has a high startup time. @@ -13,4 +13,4 @@ PackageCompilerX is a Julia package with two main purposes: For installation and usage instructions, see the [documentation][docs-stable-url]. [docs-stable-img]: https://img.shields.io/badge/docs-stable-blue.svg -[docs-stable-url]: https://JuliaComputing.github.io/PackageCompilerX.jl/dev +[docs-stable-url]: https://JuliaComputing.github.io/PackageCompiler.jl/dev diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 90948056..4e5e74bc 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -50,7 +50,7 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" -[[PackageCompilerX]] +[[PackageCompiler]] deps = ["Libdl", "Pkg", "UUIDs"] path = ".." uuid = "dffaa6cc-da53-48e5-b007-4292dfcc27f1" diff --git a/docs/Project.toml b/docs/Project.toml index 57aa2754..e8d80518 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,3 +1,3 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -PackageCompilerX = "dffaa6cc-da53-48e5-b007-4292dfcc27f1" +PackageCompiler = "dffaa6cc-da53-48e5-b007-4292dfcc27f1" diff --git a/docs/make.jl b/docs/make.jl index adba4898..b55d06dc 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,10 +1,10 @@ -using Documenter, PackageCompilerX +using Documenter, PackageCompiler makedocs( format = Documenter.HTML( prettyurls = "deploy" in ARGS, ), - sitename = "PackageCompilerX", + sitename = "PackageCompiler", pages = [ "Home" => "index.md", @@ -18,7 +18,7 @@ makedocs( "examples/plots.md", ], - "PackageCompilerX - the manual way" => [ + "PackageCompiler - the manual way" => [ "devdocs/intro.md", "devdocs/sysimages_part_1.md", "devdocs/binaries_part_2.md", @@ -30,5 +30,5 @@ makedocs( ) deploydocs( - repo = "github.com/JuliaComputing/PackageCompilerX.jl.git", + repo = "github.com/JuliaComputing/PackageCompiler.jl.git", ) diff --git a/docs/src/apps.md b/docs/src/apps.md index 417582ab..01a62bd5 100644 --- a/docs/src/apps.md +++ b/docs/src/apps.md @@ -88,9 +88,9 @@ end which will be the entry point of the app (the function that runs when the executable in the app is run). A skeleton of an app to start working from can be found at -https://github.com/JuliaComputing/PackageCompilerX.jl/tree/master/examples/MyApp. +https://github.com/JuliaLang/PackageCompiler.jl/tree/master/examples/MyApp. -Regarding relocatability, PackageCompilerX provides a function +Regarding relocatability, PackageCompiler provides a function [`audit_app(app_dir::String)`](@ref) that tries to find common problems with relocatability in the app. @@ -101,18 +101,18 @@ another machine where the same Julia that created the app can run. As an example, in the code snippet below, the example app linked above is compiled and run: ``` -~/PackageCompilerX.jl/examples +~/PackageCompiler.jl/examples ❯ julia -q --project -julia> using PackageCompilerX +julia> using PackageCompiler julia> create_app("MyApp", "MyAppCompiled") -[ Info: PackageCompilerX: creating base system image (incremental=false), this might take a while... -[ Info: PackageCompilerX: creating system image object file, this might take a while... +[ Info: PackageCompiler: creating base system image (incremental=false), this might take a while... +[ Info: PackageCompiler: creating system image object file, this might take a while... julia> exit() -~/PackageCompilerX.jl/examples +~/PackageCompiler.jl/examples ❯ MyAppCompiled/bin/MyApp ARGS = ["foo", "bar"] Base.PROGRAM_FILE = "MyAppCompiled/bin/MyApp" @@ -121,7 +121,7 @@ Hello, World! Running the artifact The result of 2*5^2 - 10 == 40.000000 -unsafe_string((Base.JLOptions()).image_file) = "/Users/kristoffer/PackageCompilerX.jl/examples/MyAppCompiled/bin/MyApp.dylib" +unsafe_string((Base.JLOptions()).image_file) = "/Users/kristoffer/PackageCompiler.jl/examples/MyAppCompiled/bin/MyApp.dylib" Example.domath(5) = 10 ``` @@ -173,13 +173,13 @@ app with the resulting sysimage. The way to depend on external libraries or binaries when creating apps is by using the [artifact system](https://julialang.github.io/Pkg.jl/v1/artifacts/). -PackageCompilerX will bundle all artifacts needed by the project, and set up +PackageCompiler will bundle all artifacts needed by the project, and set up things so that they can be found during runtime on other machines. The example app uses the artifact system to depend on a very simple toy binary that does some simple arithmetic. It is instructive to see how the [artifact -file](https://github.com/JuliaComputing/PackageCompilerX.jl/blob/master/examples/MyApp/Artifacts.toml) -is [used in the source](https://github.com/JuliaComputing/PackageCompilerX.jl/blob/d722a3d91abe328ebd239e2f45660be35263ebe1/examples/MyApp/src/MyApp.jl#L7-L8). +file](https://github.com/JuliaLang/PackageCompiler.jl/blob/master/examples/MyApp/Artifacts.toml) +is [used in the source](https://github.com/JuliaLang/PackageCompiler.jl/blob/d722a3d91abe328ebd239e2f45660be35263ebe1/examples/MyApp/src/MyApp.jl#L7-L8). ### Reverse engineering the compiled app @@ -194,15 +194,15 @@ compiled. These get cached into the sysimage and can be found e.g. by dumping all strings in the sysimage: ``` -~/PackageCompilerX.jl/examples/MyAppCompiled/bin +~/PackageCompiler.jl/examples/MyAppCompiled/bin ❯ strings MyApp.so | grep MyApp MyApp -/home/kc/PackageCompilerX.jl/examples/MyApp/ +/home/kc/PackageCompiler.jl/examples/MyApp/ MyApp -/home/kc/PackageCompilerX.jl/examples/MyApp/src/MyApp.jl -/home/kc/PackageCompilerX.jl/examples/MyApp/src +/home/kc/PackageCompiler.jl/examples/MyApp/src/MyApp.jl +/home/kc/PackageCompiler.jl/examples/MyApp/src MyApp.jl -/home/kc/PackageCompilerX.jl/examples/MyApp/src/MyApp.jl +/home/kc/PackageCompiler.jl/examples/MyApp/src/MyApp.jl ``` This is a problem that the Julia standard libraries themselves have: @@ -220,7 +220,7 @@ the "lowered code" and use reflection to find things like the name of fields in structs and global variables etc: ```julia-repl -~/PackageCompilerX.jl/examples/MyAppCompiled/bin kc/docs_apps* +~/PackageCompiler.jl/examples/MyAppCompiled/bin kc/docs_apps* ❯ julia -q -JMyApp.so julia> MyApp = Base.loaded_modules[Base.PkgId(Base.UUID("f943f3d7-887a-4ed5-b0c0-a1d6899aa8f5"), "MyApp")] MyApp diff --git a/docs/src/devdocs/binaries_part_2.md b/docs/src/devdocs/binaries_part_2.md index d4815d6d..889b03dd 100644 --- a/docs/src/devdocs/binaries_part_2.md +++ b/docs/src/devdocs/binaries_part_2.md @@ -10,7 +10,7 @@ calling into the Julia runtime library (`libjulia`) from a C program. A quite detail set of docs for how this is done can be found at the [embedding chapter in the Julia manual](https://docs.julialang.org/en/v1/manual/embedding/) and it is recommended to read before reading on. Since this is supposed to highlight -the interals of PackageCompilerX, will not use the conveniences shown in that +the interals of PackageCompiler, will not use the conveniences shown in that section (e.g. the `julia-config.jl` script) but it is good to know they exist. A rough outline of the steps we will take to create an executable are: diff --git a/docs/src/devdocs/intro.md b/docs/src/devdocs/intro.md index 98ef2949..629cb31d 100644 --- a/docs/src/devdocs/intro.md +++ b/docs/src/devdocs/intro.md @@ -1,10 +1,10 @@ # Introduction This part of the documentation contains a set of tutorials aimed to teach how -PackageCompilerX works internally. This is done by going through some examples +PackageCompiler works internally. This is done by going through some examples of manually creating sysimages and apps, mostly from the command line. -By knowing the internals of PackageCompilerX you can more easily figure out -root causes of problems and help others. The inner functionality of PackageCompilerX +By knowing the internals of PackageCompiler you can more easily figure out +root causes of problems and help others. The inner functionality of PackageCompiler is actually quite simple. There are a few julia commands and compiler invocations that everything is built around, the rest is mostly scaffolding. @@ -16,7 +16,7 @@ be run without having to explicitly start a Julia session. [Part 3](@ref man-tut details how to bundle that executable together with the Julia libraries and other files needed so that the bundle can be sent to and run on a different system where Julia might not be installed. These functionalities are exposed -from PackageCompilerX as [`create_sysimage`](@ref) and [`create_app`](@ref). +from PackageCompiler as [`create_sysimage`](@ref) and [`create_app`](@ref). It should be noted that there is some usage of non-documented Julia functions and flags. They have not been changed for quite a long time (and are unlikely diff --git a/docs/src/examples/ohmyrepl.md b/docs/src/examples/ohmyrepl.md index b92b5aa3..df989ffc 100644 --- a/docs/src/examples/ohmyrepl.md +++ b/docs/src/examples/ohmyrepl.md @@ -9,7 +9,7 @@ addition, the time of compiling functions that OhMyREPL uses is also a factor. Therefore, we also want to do "Profile Guided Compilation" (PGC), where we record what functions gets compiled when using OhMyREPL, so they can be cached into the system image. OhMyREPL is a bit different from most other packages in -that is used interactive. Normally to do PGC with PackageCompilerX we pass a +that is used interactive. Normally to do PGC with PackageCompiler we pass a script to to execute as the `precompile_exectution_file` which is used to collect compilation data, but in this case, we will use Julia to manually collect this data. @@ -39,7 +39,7 @@ These are functions that Julia compiled. We now just tell `create_sysimage` to use these precompile statements when creating the system image: ```julia -PackageCompilerX.create_sysimage(:OhMyREPL; precompile_statements_file="ohmyrepl_precompile.jl", replace_default=true) +PackageCompiler.create_sysimage(:OhMyREPL; precompile_statements_file="ohmyrepl_precompile.jl", replace_default=true) ``` Restart julia and start typing something. If everything went well you should @@ -51,5 +51,5 @@ before creating the new system image If you want to go back to the default sysimage you can run ```julia - PackageCompilerX.restore_default_sysimage() + PackageCompiler.restore_default_sysimage() ``` diff --git a/docs/src/examples/plots.md b/docs/src/examples/plots.md index e9d226fa..f2a96118 100644 --- a/docs/src/examples/plots.md +++ b/docs/src/examples/plots.md @@ -29,7 +29,7 @@ display(p) The custom sysimage is then created as: ```julia -using PackageCompilerX +using PackageCompiler create_sysimage(:Plots, sysimage_path="sys_plots.so", precompile_execution_file="precompile_plots.jl") ``` diff --git a/docs/src/index.md b/docs/src/index.md index 790ff470..7664d5be 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,6 +1,6 @@ -# PackageCompilerX +# PackageCompiler -PackageCompilerX is a Julia package with two main purposes: +PackageCompiler is a Julia package with two main purposes: 1. Creating custom sysimages for reduced latency when working locally with packages that has a high startup time. @@ -20,11 +20,11 @@ come across. https://julialang.org/downloads/. Distribution-provided Julia installations are unlikely to work properly with this package. -To use PackageCompilerX a C-compiler needs to be available: +To use PackageCompiler a C-compiler needs to be available: ### macOS, Linux -Having a decently modern `gcc` or `clang` available should be enough to use PackageCompilerX on Linux or macOS. +Having a decently modern `gcc` or `clang` available should be enough to use PackageCompiler on Linux or macOS. For macOS, using something like `homebrew` and for Linux the system package manager should work fine. ### Windows diff --git a/docs/src/refs.md b/docs/src/refs.md index d8b0039b..5c45c57c 100644 --- a/docs/src/refs.md +++ b/docs/src/refs.md @@ -1,8 +1,8 @@ # References ```@docs -PackageCompilerX.create_sysimage -PackageCompilerX.restore_default_sysimage -PackageCompilerX.create_app -PackageCompilerX.audit_app +PackageCompiler.create_sysimage +PackageCompiler.restore_default_sysimage +PackageCompiler.create_app +PackageCompiler.audit_app ``` diff --git a/docs/src/sysimages.md b/docs/src/sysimages.md index 1dcd036b..b25d3d08 100644 --- a/docs/src/sysimages.md +++ b/docs/src/sysimages.md @@ -18,7 +18,7 @@ Sometimes it is desirable to create a custom sysimage with custom precompiled code. This is the case if one has some dependencies that take a significant time to load or where the compilation time for the first call is uncomfortably long. This section of the documentation is intended to document how to use -PackageCompilerX to create such sysimages. +PackageCompiler to create such sysimages. ### Drawbacks to custom sysimages @@ -37,9 +37,9 @@ of those packages is a significant problem and when these packages are not frequently updated. In addition, compiling "workflow packages" like Revise.jl and OhMyREPL.jl and using that as a default sysimage might make sense. -## Creating a sysimage using PackageCompilerX +## Creating a sysimage using PackageCompiler -PackageCompilerX provides the function [`create_sysimage`](@ref) to create a +PackageCompiler provides the function [`create_sysimage`](@ref) to create a sysimage. It takes as the first argument a package or a list of packages that should be embedded in the resulting sysimage. By default, the given packages are loaded from the active project but a specific project can be specified by @@ -62,8 +62,8 @@ that the package is loaded without having to explicitly `import` it. ~/NewSysImageEnv 29s ❯ julia -q -julia> using PackageCompilerX -[ Info: Precompiling PackageCompilerX [dffaa6cc-da53-48e5-b007-4292dfcc27f1] +julia> using PackageCompiler +[ Info: Precompiling PackageCompiler [dffaa6cc-da53-48e5-b007-4292dfcc27f1] (v1.3) pkg> activate . Activating new environment at `~/NewSysImageEnv/Project.toml` @@ -78,7 +78,7 @@ Activating new environment at `~/NewSysImageEnv/Project.toml` [7876af07] + Example v0.5.3 julia> create_sysimage(:Example; sysimage_path="ExampleSysimage.so") -[ Info: PackageCompilerX: creating system image object file, this might take a while... +[ Info: PackageCompiler: creating system image object file, this might take a while... julia> exit() @@ -109,7 +109,7 @@ backup of the default sysimage is created. The default sysimage can then be restored with [`restore_default_sysimage()`](@ref). Note that sysimages are created "incrementally" in the sense that they add to -the sysimage of the process running PackageCompilerX. If the default sysimage +the sysimage of the process running PackageCompiler. If the default sysimage has been replaced, the next `create_sysimage` call will create a new sysimage based on the replaced sysimage. It is possible to create a sysimage non-incrementally by passing the `incremental=false` keyword. This will create @@ -155,14 +155,14 @@ the file just shown above: ~/NewSysImageEnv ❯ julia-q -julia> using PackageCompilerX +julia> using PackageCompiler (v1.3) pkg> activate . Activating environment at `~/NewSysImageEnv/Project.toml` -julia> PackageCompilerX.create_sysimage(:Example; sysimage_path="ExampleSysimagePrecompile.so", +julia> PackageCompiler.create_sysimage(:Example; sysimage_path="ExampleSysimagePrecompile.so", precompile_execution_file="precompile_example.jl") -[ Info: PackageCompilerX: creating system image object file, this might take a while... +[ Info: PackageCompiler: creating system image object file, this might take a while... julia> exit() ``` @@ -185,7 +185,7 @@ can be useful in cases where it is difficult to give a script that executes the code (like with interactive use). A file with a list of such precompile statements can be used when creating a sysimage by passing the keyword argument `precompile_statements_file`. See the [OhMyREPL.jl example](@ref manual-omr) in the docs for more -details on how to use `--trace-compile` with PackageCompilerX. +details on how to use `--trace-compile` with PackageCompiler. It is also possible to use [SnoopCompile.jl](https://timholy.github.io/SnoopCompile.jl/stable/snoopi/#auto-1) diff --git a/src/PackageCompilerX.jl b/src/PackageCompiler.jl similarity index 98% rename from src/PackageCompilerX.jl rename to src/PackageCompiler.jl index a298c852..9e078480 100644 --- a/src/PackageCompilerX.jl +++ b/src/PackageCompiler.jl @@ -1,4 +1,4 @@ -module PackageCompilerX +module PackageCompiler using Base: active_project using Libdl: Libdl @@ -90,7 +90,7 @@ function create_fresh_base_sysimage(stdlibs::Vector{String}; cpu_target::String) tmp_sys_ji = joinpath(tmp, "sys.ji") compiler_source_path = joinpath(base_dir, "compiler", "compiler.jl") - @info "PackageCompilerX: creating base system image (incremental=false)..." + @info "PackageCompiler: creating base system image (incremental=false)..." cd(base_dir) do # Create corecompiler.ji cmd = `$(get_julia_cmd()) --cpu-target $cpu_target --output-ji $tmp_corecompiler_ji @@ -237,7 +237,7 @@ function create_sysimg_object_file(object_file::String, packages::Vector{String} # finally, make julia output the resulting object file @debug "creating object file at $object_file" - @info "PackageCompilerX: creating system image object file, this might take a while..." + @info "PackageCompiler: creating system image object file, this might take a while..." cmd = `$(get_julia_cmd()) --cpu-target=$cpu_target --sysimage=$base_sysimage --project=$project --output-o=$(object_file) -e $julia_code` @@ -390,7 +390,7 @@ function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; end move_default_sysimage_if_windows() mv(sysimage_path, default_sysimg_path(); force=true) - @info "PackageCompilerX: default sysimg replaced, restart Julia for the new sysimg to be in effect" + @info "PackageCompiler: default sysimg replaced, restart Julia for the new sysimg to be in effect" end rm(object_file; force=true) return nothing @@ -428,7 +428,7 @@ function restore_default_sysimage() end move_default_sysimage_if_windows() mv(backup_default_sysimg_path(), default_sysimg_path(); force=true) - @info "PackageCompilerX: default sysimg restored, restart Julia for the new sysimg to be in effect" + @info "PackageCompiler: default sysimg restored, restart Julia for the new sysimg to be in effect" return nothing end diff --git a/test/runtests.jl b/test/runtests.jl index a255432a..5d8b8c7c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,8 +1,8 @@ -using PackageCompilerX: PackageCompilerX, create_sysimage, create_app +using PackageCompiler: PackageCompiler, create_sysimage, create_app using Test using Libdl -ENV["JULIA_DEBUG"] = "PackageCompilerX" +ENV["JULIA_DEBUG"] = "PackageCompiler" # Make a new depot new_depot = mktempdir() @@ -11,7 +11,7 @@ cp(joinpath(DEPOT_PATH[1], "registries", "General"), joinpath(new_depot, "regist ENV["JULIA_DEPOT_PATH"] = new_depot Base.init_depot_path() -@testset "PackageCompilerX.jl" begin +@testset "PackageCompiler.jl" begin tmp = mktempdir() sysimage_path = joinpath(tmp, "sys." * Libdl.dlext) script = tempname() @@ -29,7 +29,7 @@ Base.init_depot_path() # Test creating an app app_source_dir = joinpath(@__DIR__, "..", "examples/MyApp/") # TODO: Also test something that actually gives audit warnings - @test_logs PackageCompilerX.audit_app(app_source_dir) + @test_logs PackageCompiler.audit_app(app_source_dir) app_compiled_dir = joinpath(tmp, "MyAppCompiled") for incremental in (true, false) if incremental == false From 817a366745fa4503828b08c589317540e7ec6735 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sun, 9 Feb 2020 16:01:51 +0100 Subject: [PATCH 76/80] use better defaults for march and cpu target (#105) --- src/PackageCompiler.jl | 47 +++++++++++++++++++++++++++++++++--------- test/runtests.jl | 10 +++++++-- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/src/PackageCompiler.jl b/src/PackageCompiler.jl index 9e078480..022609f4 100644 --- a/src/PackageCompiler.jl +++ b/src/PackageCompiler.jl @@ -10,7 +10,22 @@ export create_sysimage, create_app, audit_app, restore_default_sysimage include("juliaconfig.jl") const NATIVE_CPU_TARGET = "native" -const APP_CPU_TARGET = "generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)" +# See https://github.com/JuliaCI/julia-buildbot/blob/489ad6dee5f1e8f2ad341397dc15bb4fce436b26/master/inventory.py +function default_app_cpu_target() + if Sys.ARCH === :i686 + return "pentium4;sandybridge,-xsaveopt,clone_all" + elseif Sys.ARCH === :x86_64 + return "generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)" + elseif Sys.ARCH === :arm + return "armv7-a;armv7-a,neon;armv7-a,neon,vfp4" + elseif Sys.ARCH === :aarch64 + return "generic" # is this really the best here? + elseif Sys.ARCH === :powerpc64le + return "pwr8" + else + return "generic" + end +end current_process_sysimage_path() = unsafe_string(Base.JLOptions().image_file) @@ -19,18 +34,28 @@ all_stdlibs() = readdir(Sys.STDLIB) yesno(b::Bool) = b ? "yes" : "no" function bitflag() - if Sys.ARCH == :aarch64 || Sys.ARCH == :arm - return `` + if Sys.ARCH == :i686 + return `-m32` + elseif Sys.ARCH == :x86_64 + return `-m64` else - return Int == Int32 ? `-m32` : `-m64` + return `` end end function march() - if Sys.ARCH == :aarch64 || Sys.ARCH == :arm - return (Int == Int32 ? `-march=armv7-a` : `-march=armv8-a+crypto+simd`) + if Sys.ARCH === :i686 + return "-march=pentium4" + elseif Sys.ARCH === :x86_64 + return "-march=x86-64" + elseif Sys.ARCH === :arm + return "-march=armv7-a+simd" + elseif Sys.ARCH === :aarch64 + return "-march=armv8-a+crypto+simd" + elseif Sys.ARCH === :powerpc64le + return nothing else - return (Int == Int32 ? `-march=pentium4` : ``) + return nothing end end @@ -408,7 +433,8 @@ function create_sysimg_from_object_file(input_object::String, sysimage_path::Str end extra = Sys.iswindows() ? `-Wl,--export-all-symbols` : `` compiler = get_compiler() - cmd = `$compiler $(bitflag()) $(march()) -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia $extra` + m = something(march(), ``) + cmd = `$compiler $(bitflag()) $m -shared -L$(julia_libdir) -o $sysimage_path $o_file -ljulia $extra` @debug "running $cmd" windows_compiler_artifact_path(compiler) do run(cmd) @@ -531,7 +557,7 @@ function create_app(package_dir::String, filter_stdlibs=false, audit=true, force=false, - cpu_target::String=APP_CPU_TARGET) + cpu_target::String=default_app_cpu_target()) precompile_statements_file = abspath.(precompile_statements_file) package_dir = abspath(package_dir) ctx = create_pkg_context(package_dir) @@ -610,7 +636,8 @@ function create_executable_from_sysimg(;sysimage_path::String, rpath = `-Wl,-rpath,\$ORIGIN:\$ORIGIN/../lib` end compiler = get_compiler() - cmd = `$compiler -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) $(bitflag()) $(march()) -o $(executable_path) $(wrapper) $(sysimage_path) -O2 $rpath $flags` + m = something(march(), ``) + cmd = `$compiler -DJULIAC_PROGRAM_LIBNAME=$(repr(sysimage_path)) $(bitflag()) $m -o $(executable_path) $(wrapper) $(sysimage_path) -O2 $rpath $flags` @debug "running $cmd" run(cmd) windows_compiler_artifact_path(compiler) do diff --git a/test/runtests.jl b/test/runtests.jl index 5d8b8c7c..7fc2a1d2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -11,6 +11,12 @@ cp(joinpath(DEPOT_PATH[1], "registries", "General"), joinpath(new_depot, "regist ENV["JULIA_DEPOT_PATH"] = new_depot Base.init_depot_path() +is_slow_ci = haskey(ENV, "CI") && Sys.ARCH == :aarch64 + +if haskey(ENV, "CI") + @show Sys.ARCH +end + @testset "PackageCompiler.jl" begin tmp = mktempdir() sysimage_path = joinpath(tmp, "sys." * Libdl.dlext) @@ -31,9 +37,9 @@ Base.init_depot_path() # TODO: Also test something that actually gives audit warnings @test_logs PackageCompiler.audit_app(app_source_dir) app_compiled_dir = joinpath(tmp, "MyAppCompiled") - for incremental in (true, false) + for incremental in (is_slow_ci ? (false,) : (true, false)) if incremental == false - filter_stdlibs = (true, false) + filter_stdlibs = (is_slow_ci ? (true, ) : (true, false)) else filter_stdlibs = (false,) end From 54f89dc906a8b8800e217dbbaafb22801a44acce Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Sun, 9 Feb 2020 13:47:25 -0500 Subject: [PATCH 77/80] Allow PRs to push preview documentation (#305) --- docs/make.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/make.jl b/docs/make.jl index b55d06dc..796e78b8 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -31,4 +31,5 @@ makedocs( deploydocs( repo = "github.com/JuliaComputing/PackageCompiler.jl.git", + push_preview = true, ) From c02c4e7fd0a51d2f221140e60d146126616612d4 Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Mon, 10 Feb 2020 12:50:31 +0100 Subject: [PATCH 78/80] Add default value (no packages) to create_sysimage. (#306) --- src/PackageCompiler.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/PackageCompiler.jl b/src/PackageCompiler.jl index 022609f4..7e3e2433 100644 --- a/src/PackageCompiler.jl +++ b/src/PackageCompiler.jl @@ -173,7 +173,7 @@ function create_sysimg_object_file(object_file::String, packages::Vector{String} cpu_target::String, script::Union{Nothing, String}, isapp::Bool) - + # Handle precompilation precompile_statements = "" @debug "running precompilation execution script..." @@ -305,7 +305,7 @@ end Create a system image that includes the package(s) in `packages`. An attempt to automatically find a compiler will be done but can also be given explicitly -by setting the envirnment variable `JULIA_CC` to a path to a compiler +by setting the environment variable `JULIA_CC` to a path to a compiler ### Keyword arguments: @@ -313,10 +313,10 @@ by setting the envirnment variable `JULIA_CC` to a path to a compiler the resulting sysimage should be saved. If set to `nothing` the keyword argument `replace_defalt` needs to be set to `true`. -- `project::String`: The project that should be active when the sysmage is created, +- `project::String`: The project that should be active when the sysimage is created, defaults to the current active project. -- `precompile_execution_file::Union{String, Vector{String}}`: A file or list of +- `precompile_execution_file::Union{String, Vector{String}}`: A file or list of files that contain code which precompilation statements should be recorded from. - `precompile_statements_file::Union{String, Vector{String}}`: A file or list of @@ -337,7 +337,7 @@ by setting the envirnment variable `JULIA_CC` to a path to a compiler - `script::String`: Path to a file that gets executed in the `--output-o` process. """ -function create_sysimage(packages::Union{Symbol, Vector{Symbol}}; +function create_sysimage(packages::Union{Symbol, Vector{Symbol}}=Symbol[]; sysimage_path::Union{String,Nothing}=nothing, project::String=dirname(active_project()), precompile_execution_file::Union{String, Vector{String}}=String[], @@ -578,7 +578,7 @@ function create_app(package_dir::String, end audit && audit_app(ctx) - + mkpath(app_dir) bundle_julia_libraries(app_dir) From 00462eedd841bb0bd378eca3b28f2daed8daa35c Mon Sep 17 00:00:00 2001 From: Anshul Singhvi Date: Mon, 10 Feb 2020 13:52:25 -0500 Subject: [PATCH 79/80] Some fixes for documentation build and deployment (#307) * JuliaComputing -> JuliaLang * Build documentation on correct conditions push to master and pull request based on master --- .github/workflows/documenter-workflow.yml | 7 ++++++- docs/make.jl | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/documenter-workflow.yml b/.github/workflows/documenter-workflow.yml index 7a86a177..26d15524 100644 --- a/.github/workflows/documenter-workflow.yml +++ b/.github/workflows/documenter-workflow.yml @@ -1,6 +1,11 @@ name: Documentation -on: [push] +on: + push: + branches: master + + pull_request: + branches: master jobs: build: diff --git a/docs/make.jl b/docs/make.jl index 796e78b8..2e0b76c5 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -30,6 +30,6 @@ makedocs( ) deploydocs( - repo = "github.com/JuliaComputing/PackageCompiler.jl.git", + repo = "github.com/JuliaLang/PackageCompiler.jl.git", push_preview = true, ) From 2742c7166ae1da6f22f63beda8082e154d03d44d Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 10 Feb 2020 23:08:35 +0100 Subject: [PATCH 80/80] add back authors to LICENSE file Co-Authored-By: Dilum Aluthge --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index d3d225a9..cc1ebfbf 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2019 Kristoffer Carlsson +Copyright (c) 2015-2020 Kristoffer Carlsson, Simon Danisch, Luca Trevisani, Julia Computing, and contributors. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal