Build process and caching improvements

.circleci/config.yml

@@ -5,30 +5,43 @@ jobs:
     working_directory: /app
     docker:
       - image: ubuntu:16.04
+    environment:
+      CACHE_VERSION: v1
     steps:
       - checkout
       # Install any system package dependencies.
-      - run: env INSTALL_TEST_DEPENDENCIES=true ./build/scripts/install_build_dependencies
+      - run: env INSTALL_TEST_DEPENDENCIES=true ./tasks/install-system-build-dependencies
       - restore_cache:
           # CircleCI's prefix based matching will mean the cache from the last
           # cache for this branch will be restored.
           key: cache-build-deps-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ arch }}-
+      # The bootstrap's "task" file will be cached on subsequent runs, but due
+      # to the reliance on timestamps, this fact might not get picked up in the
+      # CI environment. So if a stamp file exists for the initial bootstrap
+      # process, re-touch it to prevent re-downloads.
+      - run: touch -c build/work/stamp/bootstrap-*
       # Build all the API Umbrella software dependencies.
       - run: ./configure
-      - run: make all test-deps test-bundle
+      - run: make all test-deps
+      # Cache the staged build data and task checksums of what's been
+      # completed. But explicitly don't cache most of the build/work/tasks/*
+      # data, since that contains a lot of the intermediate build files, which
+      # can  significantly increase the cache size.
       - save_cache:
           key: cache-build-deps-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ arch }}-{{ epoch }}
           paths:
-            - build/work
-            - CMakeCache.txt
-            - CMakeFiles
-            - Makefile
-            - cmake_install.cmake
-      - type: shell
-        command: |
-          groupadd -r api-umbrella
-          useradd -r -g api-umbrella -s /sbin/nologin -d /opt/api-umbrella -c "API Umbrella user" api-umbrella
-          ./test/scripts/circle-ci
+            - .task
+            - build/work/dev-env
+            - build/work/stage
+            - build/work/stamp
+            - build/work/task
+            - build/work/tasks/app-deps/admin-ui/yarn/_persist
+            - build/work/tasks/app-deps/web-app/bundle/_persist
+            - build/work/tasks/app/admin-ui/build/_persist/dist
+            - build/work/tasks/app/web-app/precompile/_persist/public/web-assets
+            - build/work/tasks/test-deps/bundle/_persist
+            - build/work/test-env
+      - run: make test:circle-ci
       - store_test_results:
           path: test/tmp/reports
       - store_artifacts:

.dockerignore

@@ -1,33 +1,20 @@
-# CMake
-CMakeCache.txt
-CMakeFiles
-Makefile
-cmake_install.cmake
-install_manifest.txt
-install_manifest_core.txt
-install_manifest_hadoop-analytics.txt
-
+.task
 build/work
 build/package/work
+build/package/verify/.rnd
 
 .vagrant
 workspace
 chef/cookbooks
 chef/tmp
 .bundle
 .ruby-version
-CMakeCache.txt
-CMakeFiles
 Dockerfile*
 Makefile
-cmake_install.cmake
 docker-compose.yml
 install_manifest.txt
 install_manifest_core.txt
 install_manifest_hadoop-analytics.txt
-build/work
-build/package/work
-build/package/verify/.rnd
 deploy/.env
 deploy/log
 deploy/tmp

.gitignore

@@ -17,18 +17,13 @@
 # Ignore bundler config
 /.bundle
 
-# Ignore Circle CI items, so the GitHub pages publish task can work. 
+# Ignore Circle CI items, so the GitHub pages publish task can work.
 /.ruby-version
 
-# CMake
-/CMakeCache.txt
-/CMakeFiles
-/Makefile
-/cmake_install.cmake
-/install_manifest.txt
-/install_manifest_core.txt
-/install_manifest_hadoop-analytics.txt
+# Build task checksums
+/.task
 
+/Makefile
 /build/work
 /build/package/work
 /build/package/verify/.rnd

Dockerfile-dev-build

@@ -1,54 +1,59 @@
-FROM ubuntu:16.04
+FROM ubuntu:16.04 AS build
+
+RUN mkdir -p /app/build /build/build/work
+RUN ln -snf /build/build/work /app/build/work
+WORKDIR /app
 
-RUN mkdir /app
-RUN mkdir /build
-WORKDIR /build
 ENV DOCKER_DEV true
 ENV NOKOGIRI_USE_SYSTEM_LIBRARIES 1
 
-COPY build/scripts/install_build_dependencies /app/build/scripts/install_build_dependencies
+COPY tasks/install-system-build-dependencies /app/tasks/install-system-build-dependencies
 COPY build/package_dependencies.sh /app/build/package_dependencies.sh
-RUN env INSTALL_TEST_DEPENDENCIES=true /app/build/scripts/install_build_dependencies
-
-COPY build/scripts/download_cmake /app/build/scripts/download_cmake
-RUN /app/build/scripts/download_cmake
+RUN env INSTALL_TEST_DEPENDENCIES=true /app/tasks/install-system-build-dependencies
 
-COPY CMakeLists.txt /app/CMakeLists.txt
-COPY build/cmake/functions /app/build/cmake/functions
-COPY build/cmake/deps /app/build/cmake/deps
-COPY build/cmake/deps.cmake /app/build/cmake/deps.cmake
-COPY build/patches /app/build/patches
+COPY Makefile.in /app/Makefile.in
+COPY Taskfile.yml /app/Taskfile.yml
 COPY configure /app/configure
-RUN /app/configure && make deps
+COPY tasks/bootstrap-* /app/tasks/
+COPY tasks/helpers.sh /app/tasks/helpers.sh
+RUN ./configure
 
-COPY build/cmake/build-deps /app/build/cmake/build-deps
-COPY build/cmake/build-deps.cmake /app/build/cmake/build-deps.cmake
-RUN /app/configure && make build-deps
+COPY build/patches /app/build/patches
+COPY tasks/deps /app/tasks/deps
+RUN make deps
 
-COPY build/cmake/test-deps /app/build/cmake/test-deps
-COPY build/cmake/test-deps.cmake /app/build/cmake/test-deps.cmake
-RUN /app/configure && make test-deps
+COPY tasks/build-deps /app/tasks/build-deps
+RUN make build-deps
 
 COPY Gemfile /app/Gemfile
 COPY Gemfile.lock /app/Gemfile.lock
-COPY build/cmake/test-bundle.cmake /app/build/cmake/test-bundle.cmake
-RUN /app/configure && make test-bundle
+COPY tasks/test-deps /app/tasks/test-deps
+RUN make test-deps
 
-COPY build/cmake/app-deps /app/build/cmake/app-deps
-COPY build/cmake/app-deps.cmake /app/build/cmake/app-deps.cmake
-RUN /app/configure && make app-deps
+COPY src/api-umbrella/admin-ui/package.json /app/src/api-umbrella/admin-ui/package.json
+COPY src/api-umbrella/admin-ui/yarn.lock /app/src/api-umbrella/admin-ui/yarn.lock
+COPY src/api-umbrella/web-app/Gemfile /app/src/api-umbrella/web-app/Gemfile
+COPY src/api-umbrella/web-app/Gemfile.lock /app/src/api-umbrella/web-app/Gemfile.lock
+COPY tasks/app-deps /app/tasks/app-deps
+RUN make app-deps
 
 COPY . /app
-RUN /app/configure && make && \
-  rm -rf /build/build/work/src/api-umbrella-core/tmp/admin-ui-build/node_modules/*/.node_modules.ember-try
+RUN make
+
+RUN rm -rf build/work/tasks
+
+FROM ubuntu:16.04
+
+COPY --from=build /app /app
+COPY --from=build /build /build
+RUN env INSTALL_TEST_DEPENDENCIES=true /app/tasks/install-system-build-dependencies
+WORKDIR /app
 
 RUN groupadd -r api-umbrella && \
   useradd -r -g api-umbrella -s /sbin/nologin -d /opt/api-umbrella -c "API Umbrella user" api-umbrella
 
 ENV PATH "/app/bin:/build/build/work/dev-env/sbin:/build/build/work/dev-env/bin:/build/build/work/test-env/sbin:/build/build/work/test-env/bin:/build/build/work/stage/opt/api-umbrella/sbin:/build/build/work/stage/opt/api-umbrella/bin:/build/build/work/stage/opt/api-umbrella/embedded/sbin:/build/build/work/stage/opt/api-umbrella/embedded/bin:${PATH}"
 ENV API_UMBRELLA_ROOT /build/build/work/stage/opt/api-umbrella
 
-WORKDIR /app
-
 ENTRYPOINT ["/app/docker/dev/docker-entrypoint"]
 CMD ["/app/docker/dev/docker-start"]

Makefile.in

@@ -0,0 +1,25 @@
+# A minimal Makefile that downloads the "task" tool and passes things off to
+# that tool for executing (so "make all" actually becomes "task all"). See
+# Taskfile.yml for more detail.
+
+ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+WORK_DIR:=$(ROOT_DIR)/build/work
+
+prefix = @prefix@
+export PREFIX=$(prefix)
+
+.PHONY: default task-passthrough
+default: all
+
+# Download and locally install the "task" tool.
+TASK_VERSION:=2.0.2
+$(ROOT_DIR)/tasks/bootstrap-$(TASK_VERSION): ;
+$(WORK_DIR)/stamp/bootstrap-$(TASK_VERSION): $(ROOT_DIR)/tasks/bootstrap-$(TASK_VERSION)
+	$(ROOT_DIR)/tasks/bootstrap-$(TASK_VERSION)
+
+task-passthrough: $(WORK_DIR)/stamp/bootstrap-$(TASK_VERSION)
+	$(WORK_DIR)/task $(MAKECMDGOALS)
+
+# Match all commands sent to "make" and send them to "task" instead.
+%: task-passthrough
+	@true

Rakefile

@@ -15,11 +15,10 @@ ENV["PATH"] = [
   "#{API_UMBRELLA_SRC_ROOT}/build/work/test-env/sbin",
   "#{API_UMBRELLA_SRC_ROOT}/build/work/dev-env/bin",
   "#{API_UMBRELLA_SRC_ROOT}/build/work/dev-env/sbin",
-  "#{API_UMBRELLA_SRC_ROOT}/build/work/cmake/bin",
   ENV["PATH"],
 ].join(":")
 
 Dir.glob(File.join(API_UMBRELLA_SRC_ROOT, "scripts/rake/*.rake")).each { |r| import r }
 
 task(:default).clear
-task(:default => [:lint, :test])
+task(:default => [:test])