Fetching Upstream

.github/ISSUE_TEMPLATE/config.yml

@@ -1,8 +1,8 @@
 blank_issues_enabled: true
 contact_links: 
   - about: "Ask questions or see past answers on Stack Overflow"
-    name: "Stack Overflow"
+    name: "💬 Stack Overflow"
     url: "https://stackoverflow.com/questions/tagged/vowpalwabbit"
   - about: "Chat with the community on Gitter"
-    name: "Gitter"
+    name: "💬 Gitter"
     url: "https://gitter.im/VowpalWabbit/community/"

.github/ISSUE_TEMPLATE/deprecation.md

@@ -0,0 +1,47 @@
+---
+name: Propose Deprecation
+about: Propose a feature to be deprecated and removed
+title: ''
+labels: 'Deprecation'
+assignees: ''
+
+---
+
+#### Feature to be deprecated
+
+<!--
+Describe the feature to be deprecated, focus on the user visible aspects of it.
+
+For example:
+- Are we removing a command line option?
+- Are we changing the semantics of an input format field?
+- Are we removing a C++ API method?
+-->
+
+#### Value proposition
+
+<!--
+What is the value we'll derive from removing it? A deprecation must be necessary or helpful based on these principles.
+
+- Command line usability
+- API line usability
+- Maintenance burden
+- Incompatible with new feature
+- Better alternative readily available
+- Other, please describe
+-->
+
+#### User impact and mitigation
+
+<!--
+What's the impact of deprecating and removing this feature to current users?
+How would they detect deprecation/removal?
+How can they mitigate this feature being missing? How much effort would that be?
+-->
+
+#### Schedule
+- Deprecated in release: 
+- Removed in release: 
+
+#### Additional context
+<!-- Add any other context about the problem here. -->

.github/workflows/build_docs.yml

@@ -7,6 +7,10 @@ on:
   push:
     branches:
       - master
+      - 'releases/**'
+  pull_request:
+    branches:
+      - '*'
   release:
     types:
       - created
@@ -32,41 +36,70 @@ jobs:
         with:
           name: cxx_docs
           path: doc/html/
-  python:
+  python-build:
     container:
-      image: vowpalwabbit/ubuntu1604-build:latest
+      image: vowpalwabbit/manylinux2010-build:latest
     runs-on: ubuntu-latest
+
     steps:
       - uses: actions/checkout@v1
-      - name: Install Python package
+      - run: echo "/opt/python/cp39-cp39/bin" >> $GITHUB_PATH
+      - name: Build wheel
+        shell: bash
         run: |
-          /usr/local/miniconda/bin/activate test-python36
-          python setup.py install --user
-          /usr/local/miniconda/bin/deactivate
-      - name: Build docs
+          pip wheel . -w wheel_output/ --global-option --cmake-options="-DSTATIC_LINK_VW_JAVA=On;-DPython_INCLUDE_DIR='/opt/python/cp39-cp39/include/python3.9/'" --verbose
+          auditwheel repair wheel_output/*whl -w audit_output/
+      - name: Upload built wheel
+        uses: actions/upload-artifact@v1
+        with:
+          name: python_wheel
+          path: audit_output/
+  python-doc:
+    needs: python-build
+    container:
+      image: python:3.9
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: If this is a push build then set version to latest
+        if: ${{ github.event_name == 'push' }}
+        run: echo "VW_SPHINX_VERSION_OVERRIDE=latest" >> $GITHUB_ENV
+      - name: Download Wheel
+        uses: actions/download-artifact@v1
+        with:
+          name: python_wheel
+      - name: Install dependencies
+        shell: bash
         run: |
-          /usr/local/miniconda/bin/activate test-python36
-          pip install sphinx numpydoc
+          pip install -r python/docs/build-requirements.txt
           pip install -r requirements.txt
+      - name: Install wheel
+        shell: bash
+        run: |
+          export wheel_files=(python_wheel/*)
+          export wheel_file="${wheel_files[0]}"
+          echo Installing ${wheel_file}...
+          pip install ${wheel_file}
+      - name: Build docs
+        run: |
           cd python/docs
           make html
-          /usr/local/miniconda/bin/deactivate
       - name: Upload built docs
         uses: actions/upload-artifact@v1
         with:
           name: python_docs
           path: python/docs/build/
   upload:
-    needs: [cpp, python]
+    needs: [cpp, python-doc]
     runs-on: ubuntu-latest
-    # The upload step should only be run on the main repository.
-    if: github.repository == 'VowpalWabbit/vowpal_wabbit'
+    # The upload step should only be run on the main repository and only for pushes or releases (not pull requests).
+    if: ${{ github.repository == 'VowpalWabbit/vowpal_wabbit' && (github.event_name == 'push' || github.event_name == 'release') }}
     steps:
       - name: Set folder name to latest if push
-        if: github.event_name == 'push'
+        if: ${{ github.event_name == 'push' }}
         run: echo "FOLDER_NAME=latest" >> $GITHUB_ENV
       - name: Set folder name to version if release
-        if: github.event_name == 'release'
+        if: ${{ github.event_name == 'release' }}
         run: echo "FOLDER_NAME=$(echo ${GITHUB_REF:10})" >> $GITHUB_ENV
       - name: Download c++ Docs
         uses: actions/download-artifact@v1

.github/workflows/build_python_wheels.yml

@@ -7,8 +7,7 @@ on:
       - 'releases/**'
   pull_request:
     branches:
-      - master
-
+      - '*'
 
 jobs:
   build:
@@ -19,7 +18,6 @@ jobs:
     strategy:
       matrix:
         config:
-        - { version: 3.5, base_path: /opt/python/cp35-cp35m/, include_dir_name: python3.5m/, experimental: false }
         - { version: 3.6, base_path: /opt/python/cp36-cp36m/, include_dir_name: python3.6m/, experimental: false }
         - { version: 3.7, base_path: /opt/python/cp37-cp37m/, include_dir_name: python3.7m/, experimental: false }
         - { version: 3.8, base_path: /opt/python/cp38-cp38/, include_dir_name: python3.8/, experimental: false }
@@ -46,7 +44,6 @@ jobs:
     strategy:
       matrix:
         config:
-        - { version: 3.5, experimental: false }
         - { version: 3.6, experimental: false }
         - { version: 3.7, experimental: false }
         - { version: 3.8, experimental: false }
@@ -57,14 +54,23 @@ jobs:
         uses: actions/download-artifact@v1
         with:
           name: wheel_${{ matrix.config.version }}
-      - name: Test wheel
+      - name: Install dependencies
+        shell: bash
+        run: |
+          pip install -r requirements.txt
+          pip install pytest
+      - name: Install wheel
         shell: bash
         run: |
           export wheel_files=(wheel_${{ matrix.config.version }}/*)
           export wheel_file="${wheel_files[0]}"
           echo Installing ${wheel_file}...
-          pip install -r requirements.txt
-          pip install pytest
           pip install ${wheel_file}
+      - name: Run unit tests
+        shell: bash
+        run: |
           python -m pytest ./python/tests/
+      - name: Run doctest
+        shell: bash
+        run: |
           python ./python/tests/run_doctest.py

.github/workflows/build_python_wheels_macos.yml

@@ -7,7 +7,7 @@ on:
       - 'releases/**'
   pull_request:
     branches:
-      - master
+      - '*'
 
 jobs:
   build:

.github/workflows/build_python_wheels_windows.yml

@@ -7,7 +7,7 @@ on:
       - 'releases/**'
   pull_request:
     branches:
-      - master
+      - '*'
 
 jobs:
   build:
@@ -35,6 +35,11 @@ jobs:
         with:
           path: ${{github.workspace}}\\vcpkg
           key: ${{ runner.os }}-${{ matrix.config.version }}-${{ matrix.config.vcpkg_commit }}-2
+      # Downgrade CMake to mitigate crashing issue introduced in 3.21 that only affects vcpkg. See here: https://github.com/microsoft/vcpkg/issues/18718
+      - name: Setup cmake
+        uses: jwlawson/[email protected]
+        with:
+          cmake-version: '3.20.x'
       - name: Bootstrap and install vcpkg packages
         shell: powershell
         if: steps.vcpkg-cache.outputs.cache-hit != 'true'

.github/workflows/build_vw_slim.yml

@@ -1,12 +1,13 @@
 name: Linux / C++ - VW Slim
+
 on:
   push:
     branches:
       - master
       - 'releases/**'
   pull_request:
     branches:
-      - master
+      - '*'
 
 jobs:
   check:

.github/workflows/build_windows_cmake.yml

@@ -7,7 +7,7 @@ on:
       - 'releases/**'
   pull_request:
     branches:
-      - master
+      - '*'
 
 jobs:
   check:
@@ -43,6 +43,11 @@ jobs:
             !${{ env.VCPKG_ROOT }}/downloads
           key: |
             ${{ env.VCPKG_REF }}-vcpkg-cache-invalidate-0
+      # Downgrade CMake to mitigate crashing issue introduced in 3.21 that only affects vcpkg. See here: https://github.com/microsoft/vcpkg/issues/18718
+      - name: Setup cmake
+        uses: jwlawson/[email protected]
+        with:
+          cmake-version: '3.20.x'
       - uses: ilammy/msvc-dev-cmd@v1
       - name: Bootstrap vcpkg
         run: ${{ env.VCPKG_ROOT }}/bootstrap-vcpkg.bat

.github/workflows/format_check.yml

@@ -1,12 +1,13 @@
 name: Formatting
+
 on:
   push:
     branches:
       - master
       - 'releases/**'
   pull_request:
     branches:
-      - master
+      - '*'
 
 jobs:
   check:

.github/workflows/python_typecheck.yml

@@ -0,0 +1,35 @@
+name: Python typecheck
+
+on:
+  push:
+    branches:
+      - master
+      - 'releases/**'
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  build:
+    container:
+      image: vowpalwabbit/manylinux2010-build:latest
+    runs-on: ubuntu-latest
+    continue-on-error: ${{ matrix.config.experimental }}
+    strategy:
+      matrix:
+        config:
+        - { version: 3.8, base_path: /opt/python/cp38-cp38/, include_dir_name: python3.8/, experimental: false }
+    steps:
+      # v1 must be used because newer versions require a node.js version that will not run on this old image.
+      - uses: actions/checkout@v1
+      - name: add to path
+        shell: bash
+        run: |
+          echo "${{ matrix.config.base_path }}bin" >> $GITHUB_PATH
+      - name: Build + check
+        shell: bash
+        run: |
+          ${{ matrix.config.base_path }}bin/python setup.py --cmake-options="-DSTATIC_LINK_VW_JAVA=On;-DPython_INCLUDE_DIR='${{ matrix.config.base_path }}include/${{ matrix.config.include_dir_name }}'" install
+          ${{ matrix.config.base_path }}bin/pip install -r requirements.txt
+          ${{ matrix.config.base_path }}bin/pip install pytype
+          ${{ matrix.config.base_path }}bin/python -m pytype ./python/vowpalwabbit/ --verbosity=2

.github/workflows/run_benchmarks.yml

@@ -1,4 +1,5 @@
 name: Run Benchmarks
+
 on:
   pull_request:
     branches:

.github/workflows/upload_coverage.yml

@@ -1,13 +1,13 @@
 name: Coverage
+
 on:
   push:
     branches:
       - master
       - 'releases/**'
   pull_request:
     branches:
-      - master
-
+      - '*'
 jobs:
   upload:
     container:

.github/workflows/valgrind.yml

@@ -0,0 +1,28 @@
+name: Linux / Valgrind
+
+on:
+  push:
+    branches:
+      - master
+      - 'releases/**'
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  check_valgrind:
+    container:
+      image: vowpalwabbit/ubuntu1604-build:latest
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Install Valgrind
+        run: |
+          sudo apt update -y
+          sudo apt install -y valgrind
+      - name: Build C++ VW binary
+        run: ./.scripts/linux/build-minimal.sh Release
+      - name: Run unit tests with Valgrind
+        run: ./.scripts/linux/unit-tests-valgrind.sh
+      - name: Run test suite with Valgrind
+        run: ./.scripts/linux/test-valgrind.sh

.pipelines/build-linux-java.yml

@@ -1,5 +1,9 @@
-pr:
+trigger:
 - master
+- 'releases/**'
+
+pr:
+- '*'
 
 pool:
   vmImage: 'ubuntu-16.04'
@@ -20,7 +24,7 @@ steps:
     ossrh_username: $(ossrh_username)
     ossrh_password: $(ossrh_password)
 - task: PublishBuildArtifacts@1
-  inputs: 
+  inputs:
     pathtoPublish: build/java/libvw_jni.so
     artifactName: 'vw-static-linux-64'
   displayName: 'java artifact linux 64'