diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14a4aae2fb6..374c04d3efd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -216,7 +216,7 @@ jobs: docker load --input /tmp/docker-deps/deps.tar fi images=( - "ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.6.10034.altinitystable" + "ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.8.10041.altinitystable" "ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:6.2.0" "ghcr.io/getsentry/image-mirror-confluentinc-cp-zookeeper:6.2.0" "ghcr.io/getsentry/docker-redis-cluster:7.0.10" @@ -283,7 +283,7 @@ jobs: docker load --input /tmp/docker-deps/deps.tar fi images=( - "ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.6.10034.altinitystable" + "ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.8.10041.altinitystable" "ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:6.2.0" "ghcr.io/getsentry/image-mirror-confluentinc-cp-zookeeper:6.2.0" "ghcr.io/getsentry/docker-redis-cluster:7.0.10" @@ -458,7 +458,8 @@ jobs: matrix: version: [ - "25.3.6.10034.altinitystable", + "25.3.8.10041.altinitystable", + "25.8.16.10001.altinitystable", ] steps: diff --git a/devservices/config.yml b/devservices/config.yml index 16aa137158b..e94ca78fb3b 100644 --- a/devservices/config.yml +++ b/devservices/config.yml @@ -47,7 +47,7 @@ x-programs: services: clickhouse: - image: ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.6.10034.altinitystable + image: ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.8.10041.altinitystable ulimits: nofile: soft: 262144 diff --git a/docker-compose.gcb.yml b/docker-compose.gcb.yml index 7a89c1e3e4f..e97455e11f4 100644 --- a/docker-compose.gcb.yml +++ b/docker-compose.gcb.yml @@ -70,7 +70,7 @@ services: KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN" KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN" clickhouse: - image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.6.10034.altinitystable}" + image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.8.10041.altinitystable}" hostname: clickhouse.local extra_hosts: - "clickhouse.local:127.0.0.1" # Add entry to /etc/hosts file @@ -92,7 +92,7 @@ services: clickhouse-query: depends_on: - zookeeper - image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.6.10034.altinitystable}" + image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.8.10041.altinitystable}" hostname: clickhouse-query.local extra_hosts: - "clickhouse-query.local:127.0.0.1" # Add entry to /etc/hosts file @@ -108,7 +108,7 @@ services: clickhouse-01: depends_on: - zookeeper - image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.6.10034.altinitystable}" + image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.8.10041.altinitystable}" hostname: clickhouse-01.local extra_hosts: - "clickhouse-01.local:127.0.0.1" # Add entry to /etc/hosts file @@ -125,7 +125,7 @@ services: clickhouse-02: depends_on: - zookeeper - image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.6.10034.altinitystable}" + image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.8.10041.altinitystable}" hostname: clickhouse-02.local extra_hosts: - "clickhouse-02.local:127.0.0.1" # Add entry to /etc/hosts file @@ -142,7 +142,7 @@ services: clickhouse-03: depends_on: - zookeeper - image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.6.10034.altinitystable}" + image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.8.10041.altinitystable}" hostname: clickhouse-03.local extra_hosts: - "clickhouse-03.local:127.0.0.1" # Add entry to /etc/hosts file @@ -159,7 +159,7 @@ services: clickhouse-04: depends_on: - zookeeper - image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.6.10034.altinitystable}" + image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.8.10041.altinitystable}" hostname: clickhouse-04.local extra_hosts: - "clickhouse-04.local:127.0.0.1" # Add entry to /etc/hosts file diff --git a/docs/source/clickhouse/supported_versions.rst b/docs/source/clickhouse/supported_versions.rst index 00ce3b3d37a..ba8bed7fbaa 100644 --- a/docs/source/clickhouse/supported_versions.rst +++ b/docs/source/clickhouse/supported_versions.rst @@ -4,7 +4,8 @@ ClickHouse supported versions The following version(s) of Clickhouse have been tested and are known to work with Snuba: -- 23.8.11.29 (Altinity Stable Build) +- 25.3.8.10041 (Altinity Stable Build) +- 25.8.16.10001 (Altinity Stable Build) Any version of Clikhouse used outside of this list could potentially work, but is not guaranteed to work. Some functionality might be broken. Use a diff --git a/snuba/migrations/clickhouse.py b/snuba/migrations/clickhouse.py index 86e356a18f9..b187d8396ac 100644 --- a/snuba/migrations/clickhouse.py +++ b/snuba/migrations/clickhouse.py @@ -1,4 +1,2 @@ -CLICKHOUSE_SERVER_MIN_VERSION = "23.8.11.29" -# Note: SaaS, self-hosted, and sentry dev -# environements should all be on 23.8.11.29 -CLICKHOUSE_SERVER_MAX_VERSION = "25.3.6.10034" +CLICKHOUSE_SERVER_MIN_VERSION = "25.3.6.10034" +CLICKHOUSE_SERVER_MAX_VERSION = "25.8.16.10001" diff --git a/snuba/snuba_migrations/events/0012_errors_make_level_nullable.py b/snuba/snuba_migrations/events/0012_errors_make_level_nullable.py index b4dcfa6c774..8d7f5e7cee2 100644 --- a/snuba/snuba_migrations/events/0012_errors_make_level_nullable.py +++ b/snuba/snuba_migrations/events/0012_errors_make_level_nullable.py @@ -9,27 +9,24 @@ class Migration(migration.ClickhouseNodeMigrationLegacy): blocking = False - def __forward_migrations( - self, table_name: str - ) -> Sequence[operations.SqlOperation]: + def __forward_migrations(self, table_name: str) -> Sequence[operations.SqlOperation]: return [ operations.ModifyColumn( storage_set=StorageSetKey.EVENTS, table_name=table_name, - column=Column( - "level", String(Modifiers(low_cardinality=True, nullable=True)) - ), + column=Column("level", String(Modifiers(low_cardinality=True, nullable=True))), ) ] - def __backwards_migrations( - self, table_name: str - ) -> Sequence[operations.SqlOperation]: + def __backwards_migrations(self, table_name: str) -> Sequence[operations.SqlOperation]: return [ operations.ModifyColumn( storage_set=StorageSetKey.EVENTS, table_name=table_name, - column=Column("level", String(Modifiers(low_cardinality=True))), + column=Column( + "level", + String(Modifiers(low_cardinality=True, default="''")), + ), ) ] diff --git a/snuba/snuba_migrations/events_analytics_platform/0023_smart_autocomplete_mv.py b/snuba/snuba_migrations/events_analytics_platform/0023_smart_autocomplete_mv.py index fa7b7636cf1..931619ddc96 100644 --- a/snuba/snuba_migrations/events_analytics_platform/0023_smart_autocomplete_mv.py +++ b/snuba/snuba_migrations/events_analytics_platform/0023_smart_autocomplete_mv.py @@ -19,9 +19,7 @@ } -_attr_columns = [ - Column(f"attrs_{type_name}", type_spec) for type_name, type_spec in _TYPES.items() -] +_attr_columns = [Column(f"attrs_{type_name}", type_spec) for type_name, type_spec in _TYPES.items()] columns: List[Column[Modifiers]] = [ @@ -40,7 +38,7 @@ MV_QUERY = f""" SELECT project_id, - 'span', + 'span' AS item_type, toDate(_sort_timestamp) AS date, retention_days as retention_days, mapConcat(attr_str_0, attr_str_1, attr_str_2, attr_str_3, attr_str_4, attr_str_5, attr_str_6, attr_str_7, attr_str_8, attr_str_9, attr_str_10, attr_str_11, attr_str_12, attr_str_13, attr_str_14, attr_str_15, attr_str_16, attr_str_17, attr_str_18, attr_str_19) AS attrs_string, -- `attrs_string` Map(String, String), @@ -100,7 +98,6 @@ class Migration(migration.ClickhouseNodeMigration): - blocking = False storage_set_key = StorageSetKey.EVENTS_ANALYTICS_PLATFORM granularity = "8192" diff --git a/snuba/snuba_migrations/events_analytics_platform/0029_remove_smart_autocomplete_experimental.py b/snuba/snuba_migrations/events_analytics_platform/0029_remove_smart_autocomplete_experimental.py index fb974f255f7..fb49569af0a 100644 --- a/snuba/snuba_migrations/events_analytics_platform/0029_remove_smart_autocomplete_experimental.py +++ b/snuba/snuba_migrations/events_analytics_platform/0029_remove_smart_autocomplete_experimental.py @@ -19,9 +19,7 @@ } -_attr_columns = [ - Column(f"attrs_{type_name}", type_spec) for type_name, type_spec in _TYPES.items() -] +_attr_columns = [Column(f"attrs_{type_name}", type_spec) for type_name, type_spec in _TYPES.items()] columns: List[Column[Modifiers]] = [ @@ -40,7 +38,7 @@ MV_QUERY = f""" SELECT project_id, - 'span', + 'span' AS item_type, toDate(_sort_timestamp) AS date, retention_days as retention_days, mapConcat(attr_str_0, attr_str_1, attr_str_2, attr_str_3, attr_str_4, attr_str_5, attr_str_6, attr_str_7, attr_str_8, attr_str_9, attr_str_10, attr_str_11, attr_str_12, attr_str_13, attr_str_14, attr_str_15, attr_str_16, attr_str_17, attr_str_18, attr_str_19) AS attrs_string, -- `attrs_string` Map(String, String), @@ -100,7 +98,6 @@ class Migration(migration.ClickhouseNodeMigration): - blocking = False storage_set_key = StorageSetKey.EVENTS_ANALYTICS_PLATFORM granularity = "8192" diff --git a/snuba/snuba_migrations/events_analytics_platform/0053_alter_deletes_workload_max_threads.py b/snuba/snuba_migrations/events_analytics_platform/0053_alter_deletes_workload_max_threads.py index 30d25f6bae9..3f1906b82a4 100644 --- a/snuba/snuba_migrations/events_analytics_platform/0053_alter_deletes_workload_max_threads.py +++ b/snuba/snuba_migrations/events_analytics_platform/0053_alter_deletes_workload_max_threads.py @@ -2,33 +2,57 @@ from snuba.clusters.storage_sets import StorageSetKey from snuba.migrations import migration, operations +from snuba.migrations.migration_utilities import get_clickhouse_version_for_storage_set from snuba.migrations.operations import OperationTarget -class Migration(migration.ClickhouseNodeMigration): - blocking = False - storage_set_key = StorageSetKey.EVENTS_ANALYTICS_PLATFORM +class _AlterWorkloadOp(operations.RunSql): + """Version-aware RunSql that picks the correct thread setting at execution time.""" - def forwards_ops(self) -> Sequence[operations.SqlOperation]: - alter_workload = """ + def __init__(self) -> None: + # Placeholder statement for validation; actual SQL is generated in execute() + super().__init__( + storage_set=StorageSetKey.EVENTS_ANALYTICS_PLATFORM, + statement="SELECT 1", + target=OperationTarget.LOCAL, + ) + + def _build_statement(self, thread_setting: str) -> str: + return f""" CREATE OR REPLACE WORKLOAD low_priority_deletes IN all SETTINGS priority = 100, max_requests = 2, - max_threads = 4; + {thread_setting} = 4; """ - return [ - operations.RunSql( - storage_set=self.storage_set_key, - statement=alter_workload, - target=OperationTarget.LOCAL, - ), - ] + def execute(self) -> None: + ch_version = get_clickhouse_version_for_storage_set(self._storage_set, None) + + # max_threads was renamed to max_concurrent_threads in ClickHouse 25.8+ + if ch_version >= (25, 8): + thread_setting = "max_concurrent_threads" + else: + thread_setting = "max_threads" + + resolved = operations.RunSql( + storage_set=self._storage_set, + statement=self._build_statement(thread_setting), + target=OperationTarget.LOCAL, + ) + resolved.execute() + + +class Migration(migration.ClickhouseNodeMigration): + blocking = False + storage_set_key = StorageSetKey.EVENTS_ANALYTICS_PLATFORM + + def forwards_ops(self) -> Sequence[operations.SqlOperation]: + return [_AlterWorkloadOp()] def backwards_ops(self) -> Sequence[operations.SqlOperation]: - # Restore original settings without max_threads + # Restore original settings without thread limit alter_workload = """ CREATE OR REPLACE WORKLOAD low_priority_deletes IN all diff --git a/snuba/snuba_migrations/generic_metrics/0024_gauges_mv.py b/snuba/snuba_migrations/generic_metrics/0024_gauges_mv.py index 18b788434be..a5147690e83 100644 --- a/snuba/snuba_migrations/generic_metrics/0024_gauges_mv.py +++ b/snuba/snuba_migrations/generic_metrics/0024_gauges_mv.py @@ -50,9 +50,9 @@ def forwards_ops(self) -> Sequence[operations.SqlOperation]: tags.key, tags.indexed_value, tags.raw_value, - maxState(timestamp as raw_timestamp) as last_timestamp, + maxState(timestamp) as last_timestamp, toDateTime(multiIf(granularity=0,10,granularity=1,60,granularity=2,3600,granularity=3,86400,-1) * - intDiv(toUnixTimestamp(raw_timestamp), + intDiv(toUnixTimestamp(timestamp), multiIf(granularity=0,10,granularity=1,60,granularity=2,3600,granularity=3,86400,-1))) as rounded_timestamp, least(retention_days, multiIf(granularity=0,decasecond_retention_days, @@ -64,7 +64,7 @@ def forwards_ops(self) -> Sequence[operations.SqlOperation]: maxState(arrayJoin(gauges_values.max)) as max, sumState(arrayJoin(gauges_values.sum)) as sum, sumState(arrayJoin(gauges_values.count)) as count, - argMaxState(arrayJoin(gauges_values.last), raw_timestamp) as last + argMaxState(arrayJoin(gauges_values.last), timestamp) as last FROM generic_metric_gauges_raw_local WHERE materialization_version = 2 AND metric_type = 'gauge' @@ -76,7 +76,7 @@ def forwards_ops(self) -> Sequence[operations.SqlOperation]: tags.key, tags.indexed_value, tags.raw_value, - raw_timestamp, + timestamp, granularity, retention_days """, diff --git a/snuba/snuba_migrations/generic_metrics/0032_counters_meta_table_mv.py b/snuba/snuba_migrations/generic_metrics/0032_counters_meta_table_mv.py index a49291bdb50..03da680694d 100644 --- a/snuba/snuba_migrations/generic_metrics/0032_counters_meta_table_mv.py +++ b/snuba/snuba_migrations/generic_metrics/0032_counters_meta_table_mv.py @@ -21,7 +21,7 @@ class Migration(migration.ClickhouseNodeMigration): Column("timestamp", DateTime(modifiers=Modifiers(codecs=["DoubleDelta"]))), Column("retention_days", UInt(16)), Column("tag_values", AggregateFunction("groupUniqArray", [String()])), - Column("value", AggregateFunction("sum", [Float(64)])), + Column("count", AggregateFunction("sum", [Float(64)])), ] storage_set_key = StorageSetKey.GENERIC_METRICS_COUNTERS diff --git a/snuba/snuba_migrations/generic_metrics/0040_remove_counters_meta_tables.py b/snuba/snuba_migrations/generic_metrics/0040_remove_counters_meta_tables.py index f2b4d4b23c3..eff1a415ac3 100644 --- a/snuba/snuba_migrations/generic_metrics/0040_remove_counters_meta_tables.py +++ b/snuba/snuba_migrations/generic_metrics/0040_remove_counters_meta_tables.py @@ -12,9 +12,7 @@ class Migration(migration.ClickhouseNodeMigration): blocking = False granularity = "2048" tag_value_view_name = "generic_metric_counters_meta_tag_value_aggregation_mv" - tag_value_local_table_name = ( - "generic_metric_counters_meta_tag_value_aggregated_local" - ) + tag_value_local_table_name = "generic_metric_counters_meta_tag_value_aggregated_local" tag_value_dist_table_name = "generic_metric_counters_meta_tag_value_aggregated_dist" tag_value_table_columns: Sequence[Column[Modifiers]] = [ Column("project_id", UInt(64)), @@ -38,7 +36,7 @@ class Migration(migration.ClickhouseNodeMigration): Column("timestamp", DateTime(modifiers=Modifiers(codecs=["DoubleDelta"]))), Column("retention_days", UInt(16)), Column("tag_values", AggregateFunction("groupUniqArray", [String()])), - Column("value", AggregateFunction("sum", [Float(64)])), + Column("count", AggregateFunction("sum", [Float(64)])), ] storage_set_key = StorageSetKey.GENERIC_METRICS_COUNTERS diff --git a/snuba/snuba_migrations/generic_metrics/0041_adjust_partitioning_meta_tables.py b/snuba/snuba_migrations/generic_metrics/0041_adjust_partitioning_meta_tables.py index 2a0e0460f89..d0e2276ee61 100644 --- a/snuba/snuba_migrations/generic_metrics/0041_adjust_partitioning_meta_tables.py +++ b/snuba/snuba_migrations/generic_metrics/0041_adjust_partitioning_meta_tables.py @@ -250,7 +250,7 @@ def backwards_ops(self) -> Sequence[operations.SqlOperation]: storage_set=self.storage_set_key, table_name=self.old_meta_dist_table_name, engine=table_engines.Distributed( - local_table_name=self.meta_local_table_name, sharding_key=None + local_table_name=self.old_meta_local_table_name, sharding_key=None ), columns=self.meta_table_columns, target=OperationTarget.DISTRIBUTED, @@ -259,7 +259,7 @@ def backwards_ops(self) -> Sequence[operations.SqlOperation]: storage_set=self.storage_set_key, view_name=self.old_meta_view_name, columns=self.meta_table_columns, - destination_table_name=self.meta_local_table_name, + destination_table_name=self.old_meta_local_table_name, target=OperationTarget.LOCAL, query=""" SELECT @@ -302,7 +302,7 @@ def backwards_ops(self) -> Sequence[operations.SqlOperation]: storage_set=self.storage_set_key, table_name=self.old_tag_value_dist_table_name, engine=table_engines.Distributed( - local_table_name=self.tag_value_local_table_name, sharding_key=None + local_table_name=self.old_tag_value_local_table_name, sharding_key=None ), columns=self.tag_value_table_columns, target=OperationTarget.DISTRIBUTED, @@ -311,7 +311,7 @@ def backwards_ops(self) -> Sequence[operations.SqlOperation]: storage_set=self.storage_set_key, view_name=self.old_tag_value_view_name, columns=self.tag_value_table_columns, - destination_table_name=self.tag_value_local_table_name, + destination_table_name=self.old_tag_value_local_table_name, target=OperationTarget.LOCAL, query=""" SELECT diff --git a/snuba/snuba_migrations/generic_metrics/0057_gauges_mv3.py b/snuba/snuba_migrations/generic_metrics/0057_gauges_mv3.py index ba03ae9ba4c..4f08a4cda4a 100644 --- a/snuba/snuba_migrations/generic_metrics/0057_gauges_mv3.py +++ b/snuba/snuba_migrations/generic_metrics/0057_gauges_mv3.py @@ -52,9 +52,9 @@ def forwards_ops(self) -> Sequence[operations.SqlOperation]: tags.key, tags.indexed_value, tags.raw_value, - maxState(timestamp as raw_timestamp) as last_timestamp, + maxState(timestamp) as last_timestamp, toDateTime(multiIf(granularity=0,10,granularity=1,60,granularity=2,3600,granularity=3,86400,-1) * - intDiv(toUnixTimestamp(raw_timestamp), + intDiv(toUnixTimestamp(timestamp), multiIf(granularity=0,10,granularity=1,60,granularity=2,3600,granularity=3,86400,-1))) as rounded_timestamp, least(retention_days, multiIf(granularity=0,decasecond_retention_days, @@ -68,7 +68,7 @@ def forwards_ops(self) -> Sequence[operations.SqlOperation]: sumState(sampling_weight * arrayJoin(gauges_values.sum)) AS sum_weighted, sumState(arrayJoin(gauges_values.count)) as count, sumState(sampling_weight * arrayJoin(gauges_values.count)) AS count_weighted, - argMaxState(arrayJoin(gauges_values.last), raw_timestamp) as last + argMaxState(arrayJoin(gauges_values.last), timestamp) as last FROM generic_metric_gauges_raw_local WHERE materialization_version = 3 AND metric_type = 'gauge' @@ -80,7 +80,7 @@ def forwards_ops(self) -> Sequence[operations.SqlOperation]: tags.key, tags.indexed_value, tags.raw_value, - raw_timestamp, + timestamp, granularity, retention_days """, diff --git a/snuba/snuba_migrations/transactions/0010_transactions_nullable_trace_id.py b/snuba/snuba_migrations/transactions/0010_transactions_nullable_trace_id.py index 8755b14d715..b2e983a55e4 100644 --- a/snuba/snuba_migrations/transactions/0010_transactions_nullable_trace_id.py +++ b/snuba/snuba_migrations/transactions/0010_transactions_nullable_trace_id.py @@ -13,9 +13,7 @@ class Migration(migration.ClickhouseNodeMigrationLegacy): blocking = False - def __forward_migrations( - self, table_name: str - ) -> Sequence[operations.SqlOperation]: + def __forward_migrations(self, table_name: str) -> Sequence[operations.SqlOperation]: return [ operations.ModifyColumn( storage_set=StorageSetKey.TRANSACTIONS, @@ -24,14 +22,15 @@ def __forward_migrations( ), ] - def __backward_migrations( - self, table_name: str - ) -> Sequence[operations.SqlOperation]: + def __backward_migrations(self, table_name: str) -> Sequence[operations.SqlOperation]: return [ operations.ModifyColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name=table_name, - column=Column("trace_id", UUID()), + column=Column( + "trace_id", + UUID(Modifiers(default="toUUID('00000000-0000-0000-0000-000000000000')")), + ), ), ]