diff --git a/src/Storages/ObjectStorage/Local/Configuration.cpp b/src/Storages/ObjectStorage/Local/Configuration.cpp index 9ff67e836342..58eeac06b0e2 100644 --- a/src/Storages/ObjectStorage/Local/Configuration.cpp +++ b/src/Storages/ObjectStorage/Local/Configuration.cpp @@ -81,4 +81,20 @@ StorageObjectStorageQuerySettings StorageLocalConfiguration::getQuerySettings(co .ignore_non_existent_file = false}; } +ASTPtr StorageLocalConfiguration::createArgsWithAccessData() const +{ + auto arguments = std::make_shared(); + + arguments->children.push_back(std::make_shared(path.path)); + if (getFormat() != "auto") + arguments->children.push_back(std::make_shared(getFormat())); + if (getStructure() != "auto") + arguments->children.push_back(std::make_shared(getStructure())); + if (getCompressionMethod() != "auto") + arguments->children.push_back(std::make_shared(getCompressionMethod())); + + return arguments; +} + + } diff --git a/src/Storages/ObjectStorage/Local/Configuration.h b/src/Storages/ObjectStorage/Local/Configuration.h index 231e33f84d35..207d297147c1 100644 --- a/src/Storages/ObjectStorage/Local/Configuration.h +++ b/src/Storages/ObjectStorage/Local/Configuration.h @@ -60,6 +60,8 @@ class StorageLocalConfiguration : public StorageObjectStorageConfiguration void addStructureAndFormatToArgsIfNeeded(ASTs &, const String &, const String &, ContextPtr, bool) override { } + ASTPtr createArgsWithAccessData() const override; + private: void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override; void fromAST(ASTs & args, ContextPtr context, bool with_structure) override; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp index efae7b129d7a..b317d4dd8826 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp @@ -295,6 +295,7 @@ void StorageObjectStorageCluster::updateQueryForDistributedEngineIfNeeded(ASTPtr {"IcebergS3", "icebergS3"}, {"IcebergAzure", "icebergAzure"}, {"IcebergHDFS", "icebergHDFS"}, + {"IcebergLocal", "icebergLocal"}, {"DeltaLake", "deltaLake"}, {"DeltaLakeS3", "deltaLakeS3"}, {"DeltaLakeAzure", "deltaLakeAzure"}, @@ -416,6 +417,7 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded( {"icebergS3", "icebergS3Cluster"}, {"icebergAzure", "icebergAzureCluster"}, {"icebergHDFS", "icebergHDFSCluster"}, + {"icebergLocal", "icebergLocalCluster"}, {"deltaLake", "deltaLakeCluster"}, {"deltaLakeS3", "deltaLakeS3Cluster"}, {"deltaLakeAzure", "deltaLakeAzureCluster"}, diff --git a/src/Storages/ObjectStorage/StorageObjectStorageDefinitions.h b/src/Storages/ObjectStorage/StorageObjectStorageDefinitions.h index 3163e517542b..ef00c15750ab 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageDefinitions.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageDefinitions.h @@ -155,6 +155,13 @@ struct IcebergHDFSClusterDefinition static constexpr auto non_clustered_storage_engine_name = IcebergHDFSDefinition::storage_engine_name; }; +struct IcebergLocalClusterDefinition +{ + static constexpr auto name = "icebergLocalCluster"; + static constexpr auto storage_engine_name = "IcebergLocalCluster"; + static constexpr auto non_clustered_storage_engine_name = IcebergLocalDefinition::storage_engine_name; +}; + struct DeltaLakeClusterDefinition { static constexpr auto name = "deltaLakeCluster"; diff --git a/src/TableFunctions/TableFunctionObjectStorage.cpp b/src/TableFunctions/TableFunctionObjectStorage.cpp index a7b8762a7e6b..1d04e20debb6 100644 --- a/src/TableFunctions/TableFunctionObjectStorage.cpp +++ b/src/TableFunctions/TableFunctionObjectStorage.cpp @@ -296,6 +296,7 @@ template class TableFunctionObjectStorage; +template class TableFunctionObjectStorage; #endif #if USE_AVRO && USE_AWS_S3 @@ -334,13 +335,4 @@ void registerTableFunctionIceberg(TableFunctionFactory & factory) .allow_readonly = false}); } #endif - - -void registerDataLakeTableFunctions(TableFunctionFactory & factory) -{ - UNUSED(factory); -#if USE_AVRO - registerTableFunctionIceberg(factory); -#endif -} } diff --git a/src/TableFunctions/TableFunctionObjectStorageCluster.cpp b/src/TableFunctions/TableFunctionObjectStorageCluster.cpp index 5d5a6fa15134..89892f45bf1b 100644 --- a/src/TableFunctions/TableFunctionObjectStorageCluster.cpp +++ b/src/TableFunctions/TableFunctionObjectStorageCluster.cpp @@ -155,6 +155,13 @@ void registerTableFunctionIcebergCluster(TableFunctionFactory & factory) .category = FunctionDocumentation::Category::TableFunction}, .allow_readonly = false}); + factory.registerFunction( + {.documentation + = {.description = R"(The table function can be used to read the Iceberg table stored on shared storage in parallel for many nodes in a specified cluster.)", + .examples{{IcebergLocalClusterDefinition::name, "SELECT * FROM icebergLocalCluster(cluster, filename, format, [,compression])", ""}}, + .category = FunctionDocumentation::Category::TableFunction}, + .allow_readonly = false}); + # if USE_AWS_S3 factory.registerFunction( {.documentation diff --git a/src/TableFunctions/TableFunctionObjectStorageCluster.h b/src/TableFunctions/TableFunctionObjectStorageCluster.h index 2529c36da62b..06044e480590 100644 --- a/src/TableFunctions/TableFunctionObjectStorageCluster.h +++ b/src/TableFunctions/TableFunctionObjectStorageCluster.h @@ -62,6 +62,7 @@ using TableFunctionHDFSCluster = TableFunctionObjectStorageCluster; +using TableFunctionIcebergLocalCluster = TableFunctionObjectStorageCluster; #endif #if USE_AVRO && USE_AWS_S3 diff --git a/src/TableFunctions/TableFunctionObjectStorageClusterFallback.cpp b/src/TableFunctions/TableFunctionObjectStorageClusterFallback.cpp index c0fbea05a3f5..0ec8b178f6a3 100644 --- a/src/TableFunctions/TableFunctionObjectStorageClusterFallback.cpp +++ b/src/TableFunctions/TableFunctionObjectStorageClusterFallback.cpp @@ -67,6 +67,13 @@ struct IcebergHDFSClusterFallbackDefinition static constexpr auto storage_engine_cluster_name = "IcebergHDFSCluster"; }; +struct IcebergLocalClusterFallbackDefinition +{ + static constexpr auto name = "icebergLocal"; + static constexpr auto storage_engine_name = "Local"; + static constexpr auto storage_engine_cluster_name = "IcebergLocalCluster"; +}; + struct DeltaLakeClusterFallbackDefinition { static constexpr auto name = "deltaLake"; @@ -163,6 +170,7 @@ using TableFunctionHDFSClusterFallback = TableFunctionObjectStorageClusterFallba #if USE_AVRO using TableFunctionIcebergClusterFallback = TableFunctionObjectStorageClusterFallback; +using TableFunctionIcebergLocalClusterFallback = TableFunctionObjectStorageClusterFallback; #endif #if USE_AVRO && USE_AWS_S3 @@ -286,6 +294,27 @@ void registerTableFunctionObjectStorageClusterFallback(TableFunctionFactory & fa .allow_readonly = false } ); + + factory.registerFunction( + { + .documentation = { + .description=R"(The table function can be used to read the Iceberg table stored on shared disk in parallel for many nodes in a specified cluster or from single node.)", + .examples{ + { + "icebergLocal", + "SELECT * FROM icebergLocal(filename)", "" + }, + { + "icebergLocal", + "SELECT * FROM icebergLocal(filename) " + "SETTINGS object_storage_cluster='cluster'", "" + }, + }, + .category = FunctionDocumentation::Category::TableFunction + }, + .allow_readonly = false + } + ); #endif #if USE_AVRO && USE_AWS_S3 diff --git a/src/TableFunctions/registerTableFunctions.cpp b/src/TableFunctions/registerTableFunctions.cpp index 2f2709ed3a93..c8497fbfdda2 100644 --- a/src/TableFunctions/registerTableFunctions.cpp +++ b/src/TableFunctions/registerTableFunctions.cpp @@ -69,7 +69,6 @@ void registerTableFunctions() registerTableFunctionObjectStorage(factory); registerTableFunctionObjectStorageCluster(factory); registerTableFunctionObjectStorageClusterFallback(factory); - registerDataLakeTableFunctions(factory); registerDataLakeClusterTableFunctions(factory); #if USE_YTSAURUS diff --git a/src/TableFunctions/registerTableFunctions.h b/src/TableFunctions/registerTableFunctions.h index c1dcb14568d7..84f0418bc4e1 100644 --- a/src/TableFunctions/registerTableFunctions.h +++ b/src/TableFunctions/registerTableFunctions.h @@ -70,7 +70,6 @@ void registerTableFunctionExplain(TableFunctionFactory & factory); void registerTableFunctionObjectStorage(TableFunctionFactory & factory); void registerTableFunctionObjectStorageCluster(TableFunctionFactory & factory); void registerTableFunctionObjectStorageClusterFallback(TableFunctionFactory & factory); -void registerDataLakeTableFunctions(TableFunctionFactory & factory); void registerDataLakeClusterTableFunctions(TableFunctionFactory & factory); void registerTableFunctionTimeSeries(TableFunctionFactory & factory);