Tools update

.codecov.yml

@@ -1,42 +1,46 @@
 #see https://github.com/codecov/support/wiki/Codecov-Yaml
 codecov:
-  notify:
-    require_ci_to_pass: yes
+  require_ci_to_pass: yes
 
 coverage:
-  precision: 2  # 2 = xx.xx%, 0 = xx%
-  round: nearest # how coverage is rounded: down/up/nearest
-  range: 10...90 # custom range of coverage colors from red -> yellow -> green
+
+  # 2 = xx.xx%, 0 = xx%
+  precision: 2
+
+  # https://docs.codecov.com/docs/commit-status
   status:
-    # https://codecov.readme.io/v1.0/docs/commit-status
+
+    # We want our total main project to always remain above 87% coverage, a
+    # drop of 0.20% is allowed. It should fail if coverage couldn't be uploaded
+    # of the CI fails otherwise
     project:
       default:
-        against: auto
-        target: 70% # specify the target coverage for each commit status
-        threshold: 50% # allow this little decrease on project
-        # https://github.com/codecov/support/wiki/Filtering-Branches
-        # branches: master
+        target: 87%
+        threshold: 0.20%
+        if_not_found: failure
         if_ci_failed: error
-    # https://github.com/codecov/support/wiki/Patch-Status
+
+    # The code changed by a PR should have 90% coverage. This is different from the
+    # overall number shown above.
+    # This encourages small PR's as they are easier to test.
     patch:
       default:
-        against: auto
-        target: 30% # specify the target "X%" coverage to hit
-        threshold: 50% # allow this much decrease on patch
-    changes: false
+        target: 90%
+        if_not_found: failure
+        if_ci_failed: error
 
+# We upload additional information on branching with pytest-cov `--cov-branch`
+# This information can be used by codecov.com to increase analysis of code
 parsers:
   gcov:
     branch_detection:
       conditional: true
       loop: true
+      method: true
       macro: false
-      method: false
-  javascript:
-    enable_partials: false
+
 
 comment:
-  layout: header, diff
+  layout: diff, reach
+  behavior: default
   require_changes: false
-  behavior: default  # update if exists else create new
-  branches: *

.flake8

@@ -1,7 +1,10 @@
 [flake8]
-max-line-length = 100
 show-source = True
-application-import-names = autosklearn
-exclude =
+max-line-length = 88
+extend-exclude =
     venv
+    .venv
     build
+extend-ignore =
+    # No lambdas — too strict
+    E731

.github/workflows/pytest.yml

@@ -33,6 +33,7 @@ env:
   # Arguments used for code-cov which is later used to annotate PR's on github
   code-cov-args: >-
     --cov=autosklearn
+    --cov-branch
     --cov-report=xml
 
 jobs:

.pre-commit-config.yaml

@@ -1,57 +1,61 @@
+# If you see me, please update my `rev` field using the provided links
+# Click the repo and update to latest tags.
+# If things break on update, raise an issue
 repos:
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        name: isort imports autosklearn
+        files: autosklearn/.*
+        args: [--check]
+
+      - id: isort
+        name: isort imports test
+        files: test/.*
+        args: [--check]
+
+  - repo: https://github.com/ambv/black
+    rev: 21.12b0
+    hooks:
+      - id: black
+        name: black formatter autosklearn
+        files: autosklearn/.*
+        args: [--check]
+
+      - id: black
+        name: black formatter test
+        files: test/.*
+        args: [--check]
+
+      - id: black
+        name: black formatter examples
+        files: examples/.*
+        args: [--check]
+
+  # This is disabled as most modules fail this
+  - repo: https://github.com/pycqa/pydocstyle
+    rev: 6.1.1
+    hooks:
+      - id: pydocstyle
+        files: DISABLED # autosklearn/.*
+        always_run: false
+
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.761
+    rev: v0.930
     hooks:
       - id: mypy
-        args: [--show-error-codes]
-        name: mypy auto-sklearn-ensembles
-        files: autosklearn/ensembles
-      - id: mypy
-        args: [--show-error-codes]
-        name: mypy auto-sklearn-metrics
-        files: autosklearn/metrics
-      - id: mypy
-        args: [--show-error-codes]
-        name: mypy auto-sklearn-data
-        files: autosklearn/data
-      - id: mypy
-        args: [--show-error-codes]
-        name: mypy auto-sklearn-util
-        files: autosklearn/util
-      - id: mypy
-        args: [--show-error-codes]
-        name: mypy auto-sklearn-evaluation
-        files: autosklearn/evaluation
-      - id: mypy
-        args: [--show-error-codes]
-        name: mypy auto-sklearn-datapreprocessing
-        files: autosklearn/pipeline/components/data_preprocessing/
+        name: mypy auto-sklearn
+        files: autosklearn/.*
+
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.3
+    rev: 4.0.1
     hooks:
       - id: flake8
-        name: flake8 auto-sklearn
+        name: flake8 autosklearn
         files: autosklearn/.*
+
       - id: flake8
-        name: flake8 file-order-data
-        files: autosklearn/data
-        additional_dependencies:
-          - flake8-import-order
-      - id: flake8
-        name: flake8 file-order-ensemble
-        files: autosklearn/ensembles
-        additional_dependencies:
-          - flake8-import-order
-      - id: flake8
-        name: flake8 file-order-metrics
-        files: autosklearn/metrics
-        additional_dependencies:
-          - flake8-import-order
-      - id: flake8
-        name: flake8 file-order-util
-        files:  autosklearn/util
-        additional_dependencies:
-          - flake8-import-order
-      - id: flake8
-        name: flake8 autosklearn-test
+        name: flake8 test
         files: test/.*

Makefile

@@ -1,32 +1,114 @@
-# simple makefile to simplify repetitive build env management tasks under posix
+# NOTE: Used on linux, limited support outside of Linux
+#
+# A simple makefile to help with small tasks related to development of autosklearn
+# These have been configured to only really run short tasks. Longer form tasks
+# are usually completed in github actions.
+
+.PHONY: help install-dev clean clean-doc clean-build build doc links examples publish
+
+help:
+	@echo "Makefile autosklearn"
+	@echo "* install-dev      to install all dev requirements and install pre-commit"
+	@echo "* check            to check the source code for issues"
+	@echo "* format           to format the code with black and isort"
+	@echo "* pre-commit       to run the pre-commit check"
+	@echo "* clean            to clean the dist and doc build files"
+	@echo "* build            to build a dist"
+	@echo "* doc              to generate and view the html files"
+	@echo "* linkcheck        to check the documentation links"
+	@echo "* examples         to run and generate the examples"
+	@echo "* publish          to help publish the current branch to pypi"
 
 PYTHON ?= python
 CYTHON ?= cython
 PYTEST ?= python -m pytest
 CTAGS ?= ctags
+PIP ?= python -m pip
+MAKE ?= make
+BLACK ?= black
+ISORT ?= isort
+PYDOCSTYLE ?= pydocstyle
+MYPY ?= mypy
+PRECOMMIT ?= pre-commit
+FLAKE8 ?= flake8
+
+DIR := ${CURDIR}
+DIST := ${CURDIR}/dist
+DOCDIR := ${DIR}/doc
+INDEX_HTML := file://${DOCDIR}/html/build/index.html
+
+install-dev:
+	$(PIP) install -e ".[test,examples,docs]"
+	pre-commit install
+
+check-black:
+	$(BLACK) autosklearn examples test --check || :
+
+check-isort:
+	$(ISORT) autosklearn test --check || :
+
+check-pydocstyle:
+	$(PYDOCSTYLE) autosklearn || :
+
+check-mypy:
+	$(MYPY) autosklearn || :
 
-all: clean inplace test
+check-flake8:
+	$(FLAKE8) autosklearn || :
+	$(FLAKE8) test || :
 
-clean:
+# pydocstyle does not have easy ignore rules, instead, we include as they are covered
+check: check-black check-isort check-mypy check-flake8 # check-pydocstyle
+
+pre-commit:
+	$(PRECOMMIT) run --all-files
+
+format-black:
+	$(BLACK) autosklearn test examples
+
+format-isort:
+	$(ISORT) autosklearn test
+
+format: format-black format-isort
+
+clean-doc:
+	$(MAKE) -C ${DOCDIR} clean
+
+clean-build:
 	$(PYTHON) setup.py clean
-	rm -rf dist
+	rm -rf ${DIST}
+
+# Clean up any builds in ./dist as well as doc
+clean: clean-doc clean-build
 
-in: inplace # just a shortcut
-inplace:
-	$(PYTHON) setup.py build_ext -i
+# Build a distribution in ./dist
+build:
+	$(PYTHON) setup.py bdist
 
 doc:
-	cd ./doc
-	make html
-	cd ..
+	$(MAKE) -C ${DOCDIR} html-noexamples
+	@echo
+	@echo "View docs at:"
+	@echo ${INDEX_HTML}
 
-test-code: in
-	$(PYTEST) -s -v test
-test-doc:
-	$(PYTEST) -s -v doc/*.rst
+links:
+	$(MAKE) -C ${DOCDIR} linkcheck
 
-test-coverage:
-	rm -rf coverage .coverage
-	$(PYTEST) -s -v --with-coverage test
+examples:
+	$(MAKE) -C ${DOCDIR} html
+	@echo
+	@echo "View docs at:"
+	@echo ${INDEX_HTML}
 
-test: test-code test-sphinxext test-doc
+# Publish to testpypi
+# Will echo the commands to actually publish to be run to publish to actual PyPi
+# This is done to prevent accidental publishing but provide the same conveniences
+publish: clean-build build
+	$(PIP) install twine
+	$(PYTHON) -m twine upload --repository testpypi ${DIST}/*
+	@echo
+	@echo "Test with the following line:"
+	@echo "pip install --index-url https://test.pypi.org/simple/ auto-sklearn"
+	@echo
+	@echo "Once you have decided it works, publish to actual pypi with"
+	@echo "python -m twine upload dist/*"