Adopt single msi-builder in circle and release script (open-telemetry#210)

rmfitzpatrick · web-flow · commit 13b7c05331e3 · 2021-03-31T15:39:31.000-04:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -237,6 +237,12 @@ workflows:
           filters:
             tags:
               only: /^v[0-9]+\.[0-9]+\.[0-9]+.*/
+      - windows-msi-validation:
+          requires:
+            - windows-msi
+          filters:
+            tags:
+              only: /^v[0-9]+\.[0-9]+\.[0-9]+.*/
 
 jobs:
   setup-environment:
@@ -404,6 +410,7 @@ jobs:
   installer-script-test:
     machine:
       image: ubuntu-1604:202007-01
+      docker_layer_caching: true
     parallelism: 2
     steps:
       - checkout
@@ -433,30 +440,35 @@ jobs:
       - save_pytest_results
 
   windows-msi:
-    executor:
-      name: win/default
-      shell: powershell.exe
+    machine:
+      image: ubuntu-1604:202007-01
+      docker_layer_caching: true
     steps:
       - attach_to_workspace
       - run:
-          command: mkdir -p dist
-      - run:
-          name: Install Wix Toolset
-          command: .\internal\buildscripts\packaging\msi\make.ps1 Install-Tools
+          name: Build msi-builder image
+          command: docker build -t msi-builder internal/buildscripts/packaging/msi/msi-builder
       - run:
           name: Build MSI
           command: |
-            $Version = if ($env:CIRCLE_TAG -match '^v(\d+\.\d+\.\d+)') { $Matches[1] } else { "0.0.1.$env:CIRCLE_BUILD_NUM" }
-            .\internal\buildscripts\packaging\msi\make.ps1 New-MSI -Version $Version
-            Remove-Item -Force -Recurse .\dist -Exclude *.msi
+            mkdir dist
+            export VERSION_TAG="${CIRCLE_TAG/v/}"
+            docker run --rm -v $(pwd):/project -u 0 msi-builder "${VERSION_TAG:-0.0.1.$CIRCLE_BUILD_NUM}"
+      - persist_to_workspace:
+          root: ~/
+          paths: project/dist/*.msi
+      - store_artifacts:
+          path: dist
+
+  windows-msi-validation:
+    executor:
+      name: win/default
+      shell: powershell.exe
+    steps:
+      - attach_to_workspace
       - run:
           name: Installation test
           command: |
             $msi_path = Resolve-Path .\dist\splunk-otel-collector*.msi
             $env:VERIFY_ACCESS_TOKEN = "false"
             .\internal\buildscripts\packaging\installer\install.ps1 -access_token "testing123" -msi_path "$msi_path"
-      - persist_to_workspace:
-          root: ~/
-          paths: project/dist/*.msi
-      - store_artifacts:
-          path: dist
diff --git a/.gitignore b/.gitignore
@@ -2,7 +2,7 @@ bin/
 dist/
 
 # GoLand IDEA
-/.idea/
+.idea
 *.iml
 
 # VS Code
diff --git a/internal/buildscripts/packaging/msi/msi-builder/Dockerfile b/internal/buildscripts/packaging/msi/msi-builder/Dockerfile
@@ -0,0 +1,8 @@
+FROM felfert/wix:latest
+
+COPY ./build.sh /work/build.sh
+
+ENV OUTPUT_DIR=project/dist
+
+# `wix` user cannot write to anything outside /work, so run w/ `-u 0` if necessary (circle configuration)
+ENTRYPOINT ["bash", "-c", "su wix -c \"/work/build.sh $0 $@ --output /work/build/stage \" && cp /work/build/stage/*.msi $OUTPUT_DIR/"]
diff --git a/internal/buildscripts/packaging/msi/msi-builder/build.sh b/internal/buildscripts/packaging/msi/msi-builder/build.sh
@@ -16,39 +16,35 @@
 
 set -euo pipefail
 
-SCRIPT_DIR="$( cd "$( dirname ${BASH_SOURCE[0]} )" && pwd )"
-REPO_DIR="$( cd $SCRIPT_DIR/../../../../ && pwd )"
-
-IMAGE_NAME="felfert/wix:latest"
-WXS_PATH="./internal/buildscripts/packaging/msi/splunk-otel-collector.wxs"
-OTELCOL="./bin/otelcol_windows_amd64.exe"
-CONFIG="./cmd/otelcol/config/collector/agent_config.yaml"
-FLUENTD_CONFIG="./internal/buildscripts/packaging/fpm/etc/otel/collector/fluentd/fluent.conf"
-FLUENTD_CONFD="./internal/buildscripts/packaging/msi/fluentd/conf.d"
-OUTPUT_DIR="./dist"
+WXS_PATH="/project/internal/buildscripts/packaging/msi/splunk-otel-collector.wxs"
+OTELCOL="/project/bin/otelcol_windows_amd64.exe"
+CONFIG="/project/cmd/otelcol/config/collector/agent_config.yaml"
+FLUENTD_CONFIG="/project/internal/buildscripts/packaging/fpm/etc/otel/collector/fluentd/fluent.conf"
+FLUENTD_CONFD="/project/internal/buildscripts/packaging/msi/fluentd/conf.d"
+SPLUNK_ICON="/project/internal/buildscripts/packaging/msi/splunk.ico"
+OUTPUT_DIR="/project/dist"
 
 usage() {
     cat <<EOH >&2
 usage: ${BASH_SOURCE[0]} [OPTIONS] VERSION
 
 Description:
-    Build the MSI with the '$IMAGE_NAME' docker image.
+    Build the Splunk OpenTelemetry MSI from the project available at /project.
     By default, the MSI is saved as '${OUTPUT_DIR}/splunk-otel-collector-VERSION-amd64.msi'.
 
-Required Arguments:
-    VERSION:        The version for the MSI. The version should be in the form "N.N.N" or "N.N.N.N".
-
 OPTIONS:
-    --otelcol PATH:     Relative path from the repo base directory to the otelcol exe.
-                        Defaults to '$OTELCOL'.
-    --config PATH:      Relative path from the repo base directory to the agent config.
-                        Defaults to '$CONFIG'.
-    --fluentd PATH:     Relative path from the repo base directory to the fluentd config.
-                        Defaults to '$FLUENTD_CONFIG'.
-    --eventlog PATH:    Relative path from the repo base directory to the eventlog config.
-                        Defaults to '$EVENTLOG_CONFIG'.
-    --output DIR:       Directory to save the MSI.
-                        Defaults to '$OUTPUT_DIR'.
+    --otelcol PATH:          Absolute path to the otelcol exe.
+                             Defaults to '$OTELCOL'.
+    --config PATH:           Absolute path to the agent config.
+                             Defaults to '$CONFIG'.
+    --fluentd PATH:          Absolute path to the fluentd config.
+                             Defaults to '$FLUENTD_CONFIG'.
+    --fluentd-confd PATH:    Absolute path to the conf.d.
+                             Defaults to '$FLUENTD_CONFD'.
+    --splunk-icon PATH:      Absolute path to the splunk.ico.
+                             Defaults to '$SPLUNK_ICON'.
+    --output DIR:            Directory to save the MSI.
+                             Defaults to '$OUTPUT_DIR'.
 
 EOH
 }
@@ -59,6 +55,7 @@ parse_args_and_build() {
     local fluentd_config="$FLUENTD_CONFIG"
     local fluentd_confd="$FLUENTD_CONFD"
     local output="$OUTPUT_DIR"
+    local splunk_icon="$SPLUNK_ICON"
     local version=
 
     while [ -n "${1-}" ]; do
@@ -79,6 +76,10 @@ parse_args_and_build() {
                 fluentd_confd="$2"
                 shift 1
                 ;;
+            --splunk-icon)
+                splunk_icon="$2"
+                shift 1
+                ;;
             --output)
                 output="$2"
                 shift 1
@@ -107,8 +108,8 @@ parse_args_and_build() {
         exit 1
     fi
 
-    docker_run="docker run --rm -v ${REPO_DIR}:/work -w /work $IMAGE_NAME"
-    build_dir="${output}/build"
+    set -x
+    build_dir="/work/build"
     files_dir="${build_dir}/msi"
     msi_name="splunk-otel-collector-${version}-amd64.msi"
 
@@ -121,10 +122,27 @@ parse_args_and_build() {
     cp "$fluentd_config" "${files_dir}/fluentd/td-agent.conf"
     cp "${fluentd_confd}"/*.conf "${files_dir}/fluentd/conf.d/"
 
-    $docker_run heat dir "$files_dir" -srd -sreg -gg -template fragment -cg ConfigFiles -dr INSTALLDIR -out "${build_dir}/configfiles.wsx"
-    $docker_run candle -arch x64 -out "${build_dir}/configfiles.wixobj" "${build_dir}/configfiles.wsx"
-    $docker_run candle -arch x64 -out "${build_dir}/splunk-otel-collector.wixobj" -dVersion="$version" -dOtelcol="$otelcol" "$WXS_PATH"
-    $docker_run light -ext WixUtilExtension.dll -sval -out "${output}/${msi_name}" -b "${files_dir}" "${build_dir}/splunk-otel-collector.wixobj" "${build_dir}/configfiles.wixobj"
+    # kludge to satisfy relative path in splunk-otel-collector.wxs
+    mkdir -p /work/internal/buildscripts/packaging/msi
+    cp "${splunk_icon}" "/work/internal/buildscripts/packaging/msi/splunk.ico"
+
+    cd /work
+    configFilesWsx="${build_dir}/configfiles.wsx"
+    heat dir "$files_dir" -srd -sreg -gg -template fragment -cg ConfigFiles -dr INSTALLDIR -out "${configFilesWsx//\//\\}"
+
+    configFilesWixObj="${build_dir}/configfiles.wixobj"
+    candle -arch x64 -out "${configFilesWixObj//\//\\}" "${configFilesWsx//\//\\}"
+
+    collectorWixObj="${build_dir}/splunk-otel-collector.wixobj"
+    candle -arch x64 -out "${collectorWixObj//\//\\}" -dVersion="$version" -dOtelcol="$otelcol" "${WXS_PATH//\//\\}"
+
+    msi="${build_dir}/${msi_name}"
+    light -ext WixUtilExtension.dll -sval -out "${msi//\//\\}" -b "${files_dir//\//\\}" "${collectorWixObj//\//\\}" "${configFilesWixObj//\//\\}"
+
+    mkdir -p $output
+    cp "${msi}" "${output}/${msi_name}"
+    { set +x; } 2>/dev/null
+
     echo "MSI saved to ${output}/${msi_name}"
 }
 
diff --git a/internal/buildscripts/packaging/release/helpers/util.py b/internal/buildscripts/packaging/release/helpers/util.py
@@ -438,59 +438,31 @@ def build_msi(exe_path, args):
             sys.exit(1)
         os.remove(msi_path)
 
-    client = docker.from_env()
+    os.makedirs(args.assets_dir, exist_ok=True)
 
-    container_options = {
-        "remove": True,
-        "volumes": {
-            REPO_DIR: {"bind": "/work", "mode": "rw"},
-        },
-        "working_dir": "/work",
-    }
+    client = docker.from_env()
+    msi_builder_path = os.path.join(REPO_DIR, "internal", "buildscripts", "packaging", "msi", "msi-builder")
+    msi_builder_image, _ = client.images.build(path=msi_builder_path)
 
     with tempfile.TemporaryDirectory(dir=str(REPO_DIR)) as build_dir:
-        base_dir = os.path.basename(build_dir)
-        config_dir = os.path.join(build_dir, "config")
-        fluentd_dir = os.path.join(config_dir, "fluentd")
-        os.makedirs(fluentd_dir, exist_ok=True)
-        shutil.copy(exe_path, os.path.join(build_dir, "otelcol.exe"))
-        shutil.copy(os.path.join(str(REPO_DIR), MSI_CONFIG), os.path.join(config_dir, "config.yaml"))
-        shutil.copy(os.path.join(str(REPO_DIR), FLUENTD_CONFIG), os.path.join(fluentd_dir, "td-agent.conf"))
-        shutil.copytree(os.path.join(str(REPO_DIR), FLUENTD_CONFD), os.path.join(fluentd_dir, "conf.d"))
-        cont_config_dir = os.path.join(base_dir, "config")
-        cmd = (
-            f"heat dir {cont_config_dir} -srd -sreg -gg -template fragment "
-            f"-cg ConfigFiles -dr INSTALLDIR -out {os.path.join(base_dir, 'configfiles.wsx')}"
-        )
-        output = client.containers.run(WIX_IMAGE, command=cmd, **container_options)
-        print(output.decode("utf-8"))
-        assert os.path.isfile(os.path.join(build_dir, "configfiles.wsx")), "configfiles.wsx not found!"
-        cmd = (
-            f"candle -arch x64 -out {os.path.join(base_dir, 'configfiles.wixobj')} "
-            f"{os.path.join(base_dir, 'configfiles.wsx')}"
-        )
-        output = client.containers.run(WIX_IMAGE, command=cmd, **container_options)
-        print(output.decode("utf-8"))
-        assert os.path.isfile(os.path.join(build_dir, "configfiles.wixobj")), "configfiles.wixobj not found!"
-        cmd = (
-            f"candle -arch x64 -out {os.path.join(base_dir, 'splunk-otel-collector.wixobj')} "
-            f'-dVersion="{msi_version}" -dOtelcol="{os.path.join(base_dir, "otelcol.exe")}" {WXS_PATH}'
-        )
-        output = client.containers.run(WIX_IMAGE, command=cmd, **container_options)
-        print(output.decode("utf-8"))
-        assert os.path.isfile(
-            os.path.join(build_dir, "splunk-otel-collector.wixobj")
-        ), "splunk-otel-collector.wixobj not found!"
-        cmd = (
-            f"light -ext WixUtilExtension.dll -sval -spdb -out {os.path.join(base_dir, msi_name)} "
-            f"-b {cont_config_dir} {os.path.join(base_dir, 'splunk-otel-collector.wixobj')} "
-            f"{os.path.join(base_dir, 'configfiles.wixobj')}"
-        )
-        output = client.containers.run(WIX_IMAGE, command=cmd, **container_options)
+        container_options = {
+            "remove": True,
+            "volumes": {
+                REPO_DIR: {"bind": "/project", "mode": "ro"},
+                build_dir: {"bind": "/work/stage", "mode": "rw"},
+            },
+            "user": 0,
+            "working_dir": "/work",
+            "environment": {'OUTPUT_DIR': "/work/stage"},
+            "command": f"{msi_version}",
+        }
+        output = client.containers.run(msi_builder_image, **container_options)
         print(output.decode("utf-8"))
         assert os.path.isfile(os.path.join(build_dir, msi_name)), f"{msi_name} not found!"
         os.makedirs(args.assets_dir, exist_ok=True)
         os.rename(os.path.join(build_dir, msi_name), msi_path)
+        assert os.path.isfile(msi_path), f"{msi_name} not found in {args.assets_dir}!"
+        print(f"Successfully built {msi_path}.")
 
     return msi_path
 
