From 582b042eb80bc212d0d7eef5eb86c2f9deaddb09 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 3 Apr 2025 21:00:31 +0200 Subject: [PATCH 1/2] Cleanup setting for secondary queries --- .../ObjectStorage/StorageObjectStorageCluster.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp index 3a2bfe8a990b..d4ec795abcb4 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp @@ -268,6 +268,20 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded( if (cluster_name_in_settings) { configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context, /*with_structure=*/true); + + auto * select_query = query->as(); + if (select_query) + { + auto settings = select_query->settings(); + if (settings) + { + auto & settings_ast = settings->as(); + if (settings_ast.changes.removeSetting("object_storage_cluster") && settings_ast.changes.empty()) + { + select_query->setExpression(ASTSelectQuery::Expression::SETTINGS, {}); + } + } + } } else { From 480ca5ab9675174576b8e8f6ddf05a921703a301 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 4 Apr 2025 13:06:00 +0200 Subject: [PATCH 2/2] Convert function with setting to cluster function --- .../StorageObjectStorageCluster.cpp | 51 ++++++++++++++++--- ...tTableFunctionArgumentsFromSelectQuery.cpp | 10 +++- ...actTableFunctionArgumentsFromSelectQuery.h | 2 + 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp index d4ec795abcb4..33784fddabd5 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp @@ -269,18 +269,53 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded( { configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context, /*with_structure=*/true); + /// Convert to old-stype *Cluster table function. + /// This allows to use old clickhouse versions in cluster. + static std::unordered_map function_to_cluster_function = { + {"s3", "s3Cluster"}, + {"azureBlobStorage", "azureBlobStorageCluster"}, + {"hdfs", "hdfsCluster"}, + {"iceberg", "icebergS3Cluster"}, + {"icebergS3", "icebergS3Cluster"}, + {"icebergAzure", "icebergAzureCluster"}, + {"icebergHDFS", "icebergHDFSCluster"}, + {"deltaLake", "deltaLakeCluster"}, + {"hudi", "hudiCluster"}, + }; + + ASTFunction * table_function = extractTableFunctionFromSelectQuery(query); + + auto p = function_to_cluster_function.find(table_function->name); + if (p == function_to_cluster_function.end()) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Can't find cluster name for table function {}", + table_function->name); + } + + table_function->name = p->second; + + auto cluster_name = getClusterName(context); + auto cluster_name_arg = std::make_shared(cluster_name); + args.insert(args.begin(), cluster_name_arg); + auto * select_query = query->as(); - if (select_query) + if (!select_query) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Expected SELECT query from table function {}", + configuration->getEngineName()); + + auto settings = select_query->settings(); + if (settings) { - auto settings = select_query->settings(); - if (settings) + auto & settings_ast = settings->as(); + if (settings_ast.changes.removeSetting("object_storage_cluster") && settings_ast.changes.empty()) { - auto & settings_ast = settings->as(); - if (settings_ast.changes.removeSetting("object_storage_cluster") && settings_ast.changes.empty()) - { - select_query->setExpression(ASTSelectQuery::Expression::SETTINGS, {}); - } + select_query->setExpression(ASTSelectQuery::Expression::SETTINGS, {}); } + /// No throw if not found - `object_storage_cluster` can be global setting. } } else diff --git a/src/Storages/extractTableFunctionArgumentsFromSelectQuery.cpp b/src/Storages/extractTableFunctionArgumentsFromSelectQuery.cpp index 382964d9fe17..711cd62a5ff3 100644 --- a/src/Storages/extractTableFunctionArgumentsFromSelectQuery.cpp +++ b/src/Storages/extractTableFunctionArgumentsFromSelectQuery.cpp @@ -11,7 +11,7 @@ namespace DB { -ASTExpressionList * extractTableFunctionArgumentsFromSelectQuery(ASTPtr & query) +ASTFunction * extractTableFunctionFromSelectQuery(ASTPtr & query) { auto * select_query = query->as(); if (!select_query || !select_query->tables()) @@ -23,6 +23,14 @@ ASTExpressionList * extractTableFunctionArgumentsFromSelectQuery(ASTPtr & query) return nullptr; auto * table_function = table_expression->table_function->as(); + return table_function; +} + +ASTExpressionList * extractTableFunctionArgumentsFromSelectQuery(ASTPtr & query) +{ + auto * table_function = extractTableFunctionFromSelectQuery(query); + if (!table_function) + return nullptr; return table_function->arguments->as(); } diff --git a/src/Storages/extractTableFunctionArgumentsFromSelectQuery.h b/src/Storages/extractTableFunctionArgumentsFromSelectQuery.h index af19ef656cc6..87edf01c1c82 100644 --- a/src/Storages/extractTableFunctionArgumentsFromSelectQuery.h +++ b/src/Storages/extractTableFunctionArgumentsFromSelectQuery.h @@ -1,11 +1,13 @@ #pragma once #include +#include #include namespace DB { +ASTFunction * extractTableFunctionFromSelectQuery(ASTPtr & query); ASTExpressionList * extractTableFunctionArgumentsFromSelectQuery(ASTPtr & query); }