diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 00000000..840560a9 --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,74 @@ +name: Test against different Python version + +on: + push: + branches: master + pull_request: + branches: [ master ] + +jobs: + python_tests: + runs-on: ubuntu-latest + + strategy: + # Do not fail if one the tests did not pass + fail-fast: false + + matrix: + # Python version(s) to use when running the tests + python: + - "3.6" + - "3.8" + - "3.9-dev" + + # Docker images of MySQL-compliant databases to run the tests suite on + database: + - "mysql:8.0.20" + + services: + mysql: + image: ${{ matrix.database }} + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_DATABASE: index_digest + ports: + - "3306:3306" + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python }} + + # https://github.com/actions/cache/blob/main/examples.md#using-pip-to-get-cache-location + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt', '**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependencies + run: make install + + - name: Linter + run: make lint + + - name: Set up the database + run: | + docker ps + mysql --protocol=tcp -u root -v < setup.sql + # import the test schema files + "./sql/populate.sh" + mysql --protocol=tcp -uindex_digest -pqwerty index_digest -v -e '\s; SHOW TABLES; SHOW DATABASES;' + + - name: Tests + run: make test diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 97d9aad3..4f22b7db 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,11 +2,11 @@ name: Integration tests on: push: - branches: [ master ] + branches: master pull_request: branches: [ master ] -jobs: +jobs: integrations_tests: runs-on: ubuntu-latest @@ -16,7 +16,8 @@ jobs: matrix: # Python version(s) to use when running the tests - python: [ 3.8, 3.9-dev ] + python: + - "3.8" # Docker images of MySQL-compliant databases to run the tests suite on database: diff --git a/Makefile b/Makefile index ad146103..163b47ff 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ install: pip install -U -e .[dev] test: - pytest -x $(project_name) + pytest -x $(project_name) -o log_cli=true -o log_cli_level=warning coverage: rm -f .coverage* diff --git a/indexdigest/test/__init__.py b/indexdigest/test/__init__.py index 72403b56..fbed1291 100644 --- a/indexdigest/test/__init__.py +++ b/indexdigest/test/__init__.py @@ -1,3 +1,5 @@ +import logging + from ..database import Database from unittest import TestCase @@ -17,6 +19,7 @@ def read_queries_from_log(log_file): class DatabaseTestMixin(object): DSN = 'mysql://index_digest:qwerty@127.0.0.1/index_digest' + DBNAME = 'index_digest' @property def connection(self): @@ -31,6 +34,7 @@ class BigTableTest(TestCase, DatabaseTestMixin): ROWS = 100000 # how many rows to generate BATCH = 5000 # perform INSERT in batches + BIG_TABLE_NAME = '0020_big_table' PREPARED = False def setUp(self): @@ -85,7 +89,7 @@ def _prepare_big_table(self): # no? populate it for row in self._rows(): # Report low cardinality indices, use only a few distinct values (#31) - num = row % 3 + num = row % 2 values.append((row, val, '{:05x}'.format(row)[:5], num)) @@ -107,6 +111,14 @@ def _prepare_big_table(self): # update key distribution statistics (#31) self.connection.query('ANALYZE TABLE 0020_big_table') + cardinality_stats = self.connection.query_dict_rows( + "select TABLE_NAME, INDEX_NAME, COLUMN_NAME, CARDINALITY from" + " INFORMATION_SCHEMA.STATISTICS where" + " TABLE_NAME = '{table_name}' AND TABLE_SCHEMA = '{database_name}'".format( + table_name=self.BIG_TABLE_NAME, database_name=self.DBNAME) + ) + logging.warning('Big table initialized, cardinality: %r', list(cardinality_stats)) + def table_populated(self): """ :rtype: bool diff --git a/indexdigest/test/linters/test_0031_low_cardinality_index.py b/indexdigest/test/linters/test_0031_low_cardinality_index.py index f0970b7d..a74ba41d 100644 --- a/indexdigest/test/linters/test_0031_low_cardinality_index.py +++ b/indexdigest/test/linters/test_0031_low_cardinality_index.py @@ -20,7 +20,7 @@ def test_get_low_cardinality_indices(self): assert index[0] == '0020_big_table' assert index[2]['INDEX_NAME'] == 'num_idx' assert index[2]['COLUMN_NAME'] == 'num' - assert index[2]['CARDINALITY'] > 1 + assert index[2]['CARDINALITY'] >= 1 assert index[2]['CARDINALITY'] <= INDEX_CARDINALITY_THRESHOLD def test_low_cardinality_index(self): @@ -37,4 +37,5 @@ def test_low_cardinality_index(self): assert reports[0].context['column_name'] == 'num' assert reports[0].context['index_name'] == 'num_idx' assert isinstance(reports[0].context['index_cardinality'], int) - assert 33 <= int(reports[0].context['value_usage']) <= 35 + + self.assertAlmostEqual(int(reports[0].context['value_usage']), 50, delta=5)