test: add windows and C++ coverage

Collect Windows and C++ coverage. Configure codecov so that
comments are more concise and are only left when coverage
varies.

PR-URL: #35670
Fixes: #35696
Refs: #35653
Refs: #35646
Reviewed-By: Rich Trott <[email protected]>
Reviewed-By: Antoine du Hamel <[email protected]>
Reviewed-By: Richard Lau <[email protected]>
Reviewed-By: Jiawen Geng <[email protected]>
Reviewed-By: Michael Dawson <[email protected]>

Author: Benjamin Coe

.github/workflows/coverage-linux.yml

@@ -1,18 +1,26 @@
-# TODO(bcoe): add similar job for Windows coverage.
 name: coverage-linux
 
 on:
   pull_request:
+    paths-ignore:
+      - 'doc/**'
+      - 'deps/**'
+      - 'benchmark/**'
+      - 'tools/**'
   push:
     branches:
       - master
+    paths-ignore:
+      - 'doc/**'
+      - 'deps/**'
+      - 'benchmark/**'
+      - 'tools/**'
 
 env:
   PYTHON_VERSION: 3.9
   FLAKY_TESTS: dontcare
 
 jobs:
-  # TODO(bcoe): add support for C++ coverage.
   coverage-linux:
     runs-on: ubuntu-latest
     steps:
@@ -23,18 +31,21 @@ jobs:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Environment Information
         run: npx envinfo
+      - name: Install gcovr
+        run: pip install gcovr==4.2
       - name: Build
-        run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn"
+        run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn --coverage"
       # TODO(bcoe): fix the couple tests that fail with the inspector enabled.
       # The cause is most likely coverage's use of the inspector.
       - name: Test
-        run: NODE_V8_COVERAGE=coverage/tmp make run-ci -j2 V=1 TEST_CI_ARGS="-p dots" || exit 0
-      - name: Report
+        run: NODE_V8_COVERAGE=coverage/tmp make test-cov -j2 V=1 TEST_CI_ARGS="-p dots" || exit 0
+      - name: Report JS
         run: npx c8 report --check-coverage
-      - name: Output file count
-        run: ls -l coverage/tmp/ | wc -l
+      - name: Report C++
+        run: cd out && gcovr --gcov-exclude='.*\b(deps|usr|out|obj|cctest|embedding)\b' -v -r Release/obj.target --xml -o ../coverage/coverage-cxx.xml --root=$(cd ../ && pwd)
+      # Clean temporary output from gcov and c8, so that it's not uploaded:
       - name: Clean tmp
-        run: rm -rf coverage/tmp
+        run: rm -rf coverage/tmp && rm -rf out
       - name: Upload
         uses: codecov/codecov-action@v1
         with:

.github/workflows/coverage-windows.yml

@@ -0,0 +1,51 @@
+name: coverage-windows
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'doc/**'
+      - 'deps/**'
+      - 'benchmark/**'
+      - 'tools/**'
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - 'doc/**'
+      - 'deps/**'
+      - 'benchmark/**'
+      - 'tools/**'
+
+env:
+  PYTHON_VERSION: 3.9
+  FLAKY_TESTS: dontcare
+
+jobs:
+  coverage-windows:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install deps
+        run: choco install nasm
+      - name: Environment Information
+        run: npx envinfo
+      - name: Build
+        run: ./vcbuild.bat
+      # TODO(bcoe): investigate tests that fail with coverage enabled
+      # on Windows.
+      - name: Test
+        run: ./vcbuild.bat test-ci-js; node -e 'process.exit(0)'
+        env:
+          NODE_V8_COVERAGE: ./coverage/tmp
+      - name: Report
+        run: npx c8 report
+      - name: Clean tmp
+        run: npx rimraf ./coverage/tmp
+      - name: Upload
+        uses: codecov/codecov-action@v1
+        with:
+          directory: ./coverage

.nycrc

@@ -9,7 +9,7 @@
   "reporter": [
     "html",
     "text",
-    "lcov"
+    "cobertura"
   ],
   "lines": 95,
   "branches": "93",

Makefile

@@ -294,6 +294,7 @@ v8:
 .PHONY: jstest
 jstest: build-addons build-js-native-api-tests build-node-api-tests ## Runs addon tests and JS tests
 	$(PYTHON) tools/test.py $(PARALLEL_ARGS) --mode=$(BUILDTYPE_LOWER) \
+		$(TEST_CI_ARGS) \
 		--skip-tests=$(CI_SKIP_TESTS) \
 		$(JS_SUITES) \
 		$(NATIVE_SUITES)

codecov.yml

@@ -0,0 +1,10 @@
+comment:
+  # Only show diff and files changed:
+  layout: "diff, files"
+  # Don't post if no changes in coverage:
+  require_changes: true
+
+codecov:
+  notify:
+    # Wait for all coverage builds:
+    after_n_builds: 2