Prepare release 0.9.0

.github/workflows/ci.yaml

@@ -25,7 +25,8 @@ jobs:
 #          - Windows
 #          - MacOs
         py:
-          - "3.12.0"
+          - "3.13"
+          - "3.12"
           - "3.11"
           - "3.10"
           - "3.9"
@@ -43,7 +44,7 @@ jobs:
           version: '2.14.2' # the full version of `docker-compose` command
 
       - name: Docker compose up
-        run: make docker-compose-up
+        run: make docker-compose-up-deamon
 
       - name: Setup
         run: make setup

Makefile

@@ -9,8 +9,9 @@ setup-dev:
 	pyenv install -s 3.9.9
 	pyenv install -s 3.10.7
 	pyenv install -s 3.11.0
-	pyenv install -s 3.12-dev
-	pyenv local 3.7.12 3.8.6 3.9.9 3.10.7 3.11.0 3.12-dev
+	pyenv install -s 3.12.0
+	pyenv install -s 3.13.0
+	pyenv local 3.7.12 3.8.6 3.9.9 3.10.7 3.11.0 3.12.0 3.13.0
 	python3 -m pip install tox==4.8.0
 
 .PHONY: test
@@ -38,6 +39,10 @@ open-coverage:
 docker-compose-up:
 	cd dev && make configure up
 
+.PHONY: docker-compose-up-deamon
+docker-compose-up-deamon:
+	cd dev && make configure up-deamon
+
 .PHONY: docker-compose-down
 docker-compose-down:
 	cd dev && make down clean

dev/CH-TEMPLATE/config.d/database_atomic_delay_before_drop_table.xml

@@ -0,0 +1,3 @@
+<clickhouse>
+    <database_atomic_delay_before_drop_table_sec>1</database_atomic_delay_before_drop_table_sec>
+</clickhouse>

dev/config.xml → dev/CH-TEMPLATE/config.xml

@@ -28,12 +28,17 @@
     <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
     <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
 
-    <users_config>users.xml</users_config>
     <default_profile>default</default_profile>
     <default_database>default</default_database>
     <timezone>Europe/Moscow</timezone>
     <mlock_executable>false</mlock_executable>
 
+    <user_directories>
+        <users_xml>
+            <path>/etc/clickhouse-server/default-users.xml</path>
+        </users_xml>
+    </user_directories>
+
     <remote_servers>
         <company_cluster>
             <shard>

dev/CH-TEMPLATE/users.d/default-user.xml

@@ -0,0 +1,12 @@
+<clickhouse>
+  <!-- Docs: <https://clickhouse.com/docs/operations/settings/settings_users/> -->
+  <users>
+    <default>
+      <!-- User default is available only locally -->
+      <networks>
+        <ip>::1</ip>
+        <ip>127.0.0.1</ip>
+      </networks>
+    </default>
+  </users>
+</clickhouse>

dev/Makefile

@@ -1,23 +1,26 @@
 configure:
 	make clean
-	mkdir -p clickhouse01 clickhouse02 clickhouse03 clickhouse04
-	REPLICA=01 SHARD=01 envsubst < config.xml > clickhouse01/config.xml
-	REPLICA=02 SHARD=01 envsubst < config.xml > clickhouse02/config.xml
-	REPLICA=03 SHARD=02 envsubst < config.xml > clickhouse03/config.xml
-	REPLICA=04 SHARD=02 envsubst < config.xml > clickhouse04/config.xml
-	cp users.xml clickhouse01/users.xml
-	cp users.xml clickhouse02/users.xml
-	cp users.xml clickhouse03/users.xml
-	cp users.xml clickhouse04/users.xml
+	cp -r CH-TEMPLATE clickhouse01
+	cp -r CH-TEMPLATE clickhouse02
+	cp -r CH-TEMPLATE clickhouse03
+	cp -r CH-TEMPLATE clickhouse04
+	REPLICA=01 SHARD=01 envsubst < CH-TEMPLATE/config.xml > clickhouse01/config.xml
+	REPLICA=02 SHARD=01 envsubst < CH-TEMPLATE/config.xml > clickhouse02/config.xml
+	REPLICA=03 SHARD=02 envsubst < CH-TEMPLATE/config.xml > clickhouse03/config.xml
+	REPLICA=04 SHARD=02 envsubst < CH-TEMPLATE/config.xml > clickhouse04/config.xml
 
 up:
+	docker-compose up
+
+up-deamon:
 	docker-compose up -d
 
 start:
 	docker-compose start
 
 down:
-	docker-compose down
+	docker-compose down -v
+	docker volume prune -f
 
 clean:
 	rm -rf clickhouse01 clickhouse02 clickhouse03 clickhouse04

dev/docker-compose.yaml

@@ -1,5 +1,3 @@
-version: '3.5'
-
 networks:
   clickhouse-network:
     name: clickhouse-network
@@ -17,8 +15,7 @@ services:
         ipv4_address: 172.23.0.10
 
   clickhouse01:
-    image: yandex/clickhouse-server:20.4
-    # image: altinity/clickhouse-server:23.8.11.29.altinitystable
+    image: clickhouse/clickhouse-server:25.7.4
     container_name: clickhouse01
     hostname: clickhouse01
     networks:
@@ -33,8 +30,7 @@ services:
       - zookeeper
 
   clickhouse02:
-    image: yandex/clickhouse-server:20.4
-    # image: altinity/clickhouse-server:23.8.11.29.altinitystable
+    image: clickhouse/clickhouse-server:25.7.4
     container_name: clickhouse02
     hostname: clickhouse02
     networks:
@@ -46,8 +42,7 @@ services:
       - zookeeper
 
   clickhouse03:
-    image: yandex/clickhouse-server:20.4
-    # image: altinity/clickhouse-server:23.8.11.29.altinitystable
+    image: clickhouse/clickhouse-server:25.7.4
     container_name: clickhouse03
     hostname: clickhouse03
     networks:
@@ -59,8 +54,7 @@ services:
       - zookeeper
 
   clickhouse04:
-    image: yandex/clickhouse-server:20.4
-    # image: altinity/clickhouse-server:23.8.11.29.altinitystable
+    image: clickhouse/clickhouse-server:25.7.4
     container_name: clickhouse04
     hostname: clickhouse04
     networks:

pyproject.toml

@@ -23,14 +23,14 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
     "clickhouse-driver>=0.2.2",
 ]
 optional-dependencies.testing = [
-    "pytest==7.4.1",
-    "pytest-cov==4.1.0",
-    "parameterized==0.9.0",
+    "pytest==8.4.1",
+    "pytest-cov==6.2.1",
 ]
 dynamic =["version", "readme"]
 

src/clickhouse_migrations/__init__.py

@@ -1,4 +1,5 @@
 """
 Simple file-based migrations for clickhouse
 """
-__version__ = "0.8.1"
+
+__version__ = "0.9.0"

src/tests/conftest.py

@@ -1,15 +1,38 @@
 import pytest
 
 from clickhouse_migrations.clickhouse_cluster import ClickhouseCluster
+from clickhouse_migrations.migrator import Migrator
 
 
 @pytest.fixture
 def cluster() -> ClickhouseCluster:
-    return ClickhouseCluster(db_host="localhost", db_user="default", db_password="")
+    return ClickhouseCluster(
+        db_host="localhost",
+        db_user="default",
+        db_password="",
+        db_name="pytest",
+    )
 
 
 @pytest.fixture(autouse=True)
 def before(cluster: ClickhouseCluster):
     with cluster.connection("") as conn:
-        conn.execute("DROP DATABASE IF EXISTS pytest")
+        conn.execute("DROP DATABASE IF EXISTS pytest SYNC")
         conn.execute("CREATE DATABASE pytest")
+
+
+@pytest.fixture(name="_clean_slate")
+def clean_slate(cluster):
+    with cluster.connection("") as conn:
+        conn.execute("DROP DATABASE IF EXISTS pytest ON CLUSTER company_cluster SYNC")
+
+
+@pytest.fixture(name="_schema")
+def schema(cluster, _clean_slate):
+    conn = cluster.connection("")
+    conn.execute("CREATE DATABASE pytest ON CLUSTER company_cluster")
+    conn = cluster.connection("pytest")
+    migrator = Migrator(conn)
+    migrator.init_schema("company_cluster")
+
+    return conn

src/tests/test_cluster_support.py

@@ -1,9 +1,6 @@
 from pathlib import Path
 
-import pytest
-
 from clickhouse_migrations.clickhouse_cluster import ClickhouseCluster
-from clickhouse_migrations.migrator import Migrator
 
 TESTS_DIR = Path(__file__).parent
 
@@ -15,28 +12,6 @@
 )
 
 
-@pytest.fixture
-def cluster():
-    return ClickhouseCluster(db_host="localhost", db_user="default", db_password="")
-
-
-@pytest.fixture(name="_clean_slate")
-def clean_slate(cluster):
-    with cluster.connection("") as conn:
-        conn.execute("DROP DATABASE IF EXISTS pytest ON CLUSTER company_cluster")
-
-
-@pytest.fixture(name="_schema")
-def schema(cluster, _clean_slate):
-    conn = cluster.connection("")
-    conn.execute("CREATE DATABASE pytest ON CLUSTER company_cluster")
-    conn = cluster.connection("pytest")
-    migrator = Migrator(conn)
-    migrator.init_schema("company_cluster")
-
-    return conn
-
-
 def test_replicated_schema(_schema):
     with _schema:
         for server in CLICKHOUSE_SERVERS:
@@ -45,7 +20,7 @@ def test_replicated_schema(_schema):
             )[0][0]
             assert (
                 table_engine
-                == "ReplicatedMergeTree('/clickhouse/tables/{database}/{table}', '{replica}') ORDER BY tuple(created_at) SETTINGS index_granularity = 8192"  # pylint: disable=C0301 # noqa: E501
+                == "ReplicatedMergeTree('/clickhouse/tables/pytest/schema_versions', '{replica}') ORDER BY tuple(created_at) SETTINGS index_granularity = 8192"  # pylint: disable=C0301 # noqa: E501
             )
 
 

src/tests/test_dryrun.py

@@ -1,36 +1,12 @@
 import logging
 from pathlib import Path
 
-import pytest
-
-from clickhouse_migrations.clickhouse_cluster import ClickhouseCluster
 from clickhouse_migrations.migration import MigrationStorage
 
 TESTS_DIR = Path(__file__).parent
 MIGRATIONS = MigrationStorage(TESTS_DIR / "migrations").migrations()
 
 
-@pytest.fixture
-def cluster():
-    return ClickhouseCluster(db_host="localhost", db_user="default", db_password="")
-
-
-@pytest.fixture(name="_clean_slate")
-def clean_slate(cluster):
-    with cluster.connection("") as conn:
-        conn.execute("DROP DATABASE IF EXISTS pytest ON CLUSTER company_cluster")
-
-
-@pytest.fixture(name="_schema")
-def schema(cluster, _clean_slate):
-    conn = cluster.connection("")
-    conn.execute("CREATE DATABASE pytest ON CLUSTER company_cluster")
-    conn = cluster.connection("pytest")
-    cluster.init_schema("pytest")
-
-    return conn
-
-
 def test_dryrun(_schema, cluster, caplog):
     with caplog.at_level(logging.INFO):
         cluster.migrate("pytest", TESTS_DIR / "migrations", dryrun=True)

src/tests/test_init_clickhouse_cluster.py

@@ -1,19 +1,11 @@
 from pathlib import Path
 
-import pytest
-
-from clickhouse_migrations.clickhouse_cluster import ClickhouseCluster
 from clickhouse_migrations.migration import Migration, MigrationStorage
 
 TESTS_DIR = Path(__file__).parent
 MIGRATIONS = MigrationStorage(TESTS_DIR / "migrations").migrations()
 
 
-@pytest.fixture
-def cluster():
-    return ClickhouseCluster(db_url="clickhouse://default:@localhost:9000/pytest")
-
-
 def test_apply_new_migration_ok(cluster):
     cluster.init_schema()