forked from ClickHouse/ClickHouse
-
Notifications
You must be signed in to change notification settings - Fork 18
Expand file tree
/
Copy pathInterpreterCreateQuery.cpp
More file actions
2526 lines (2139 loc) · 113 KB
/
InterpreterCreateQuery.cpp
File metadata and controls
2526 lines (2139 loc) · 113 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#include <memory>
#include <filesystem>
#include <Access/AccessControl.h>
#include <Access/User.h>
#include <Core/Settings.h>
#include <Interpreters/InterpreterAlterQuery.h>
#include <Parsers/ASTPartition.h>
#include <Parsers/ASTSetQuery.h>
#include <Parsers/queryToString.h>
#include <Common/Exception.h>
#include <Common/Macros.h>
#include <Common/PoolId.h>
#include <Common/SipHash.h>
#include <Common/StringUtils.h>
#include <Common/atomicRename.h>
#include <Common/escapeForFileName.h>
#include <Common/getRandomASCIIString.h>
#include <Common/logger_useful.h>
#include <Common/typeid_cast.h>
#include <Common/thread_local_rng.h>
#include <Core/Defines.h>
#include <Core/SettingsEnums.h>
#include <Core/ServerSettings.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/WriteHelpers.h>
#include <Parsers/ASTColumnDeclaration.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTInsertQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/formatAST.h>
#include <Parsers/parseQuery.h>
#include <Storages/MergeTree/MergeTreeSettings.h>
#include <Storages/StorageFactory.h>
#include <Storages/StorageInMemoryMetadata.h>
#include <Storages/StorageReplicatedMergeTree.h>
#include <Storages/StorageTimeSeries.h>
#include <Storages/WindowView/StorageWindowView.h>
#include <Interpreters/Context.h>
#include <Interpreters/executeDDLQueryOnCluster.h>
#include <Interpreters/executeQuery.h>
#include <Interpreters/DDLTask.h>
#include <Interpreters/ExpressionAnalyzer.h>
#include <Interpreters/InterpreterFactory.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Interpreters/InterpreterSelectWithUnionQuery.h>
#include <Interpreters/InterpreterSelectQueryAnalyzer.h>
#include <Interpreters/InterpreterInsertQuery.h>
#include <Interpreters/InterpreterRenameQuery.h>
#include <Interpreters/AddDefaultDatabaseVisitor.h>
#include <Interpreters/GinFilter.h>
#include <Interpreters/parseColumnsListForTableFunction.h>
#include <Interpreters/TemporaryReplaceTableName.h>
#include <Access/Common/AccessRightsElement.h>
#include <DataTypes/DataTypeFactory.h>
#include <DataTypes/NestedUtils.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeLowCardinality.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeAggregateFunction.h>
#include <DataTypes/ObjectUtils.h>
#include <DataTypes/hasNullable.h>
#include <Databases/DatabaseFactory.h>
#include <Databases/DatabaseReplicated.h>
#include <Databases/DatabaseOnDisk.h>
#include <Databases/DatabaseOrdinary.h>
#include <Databases/TablesLoader.h>
#include <Databases/DDLDependencyVisitor.h>
#include <Databases/NormalizeAndEvaluateConstantsVisitor.h>
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
#include <Compression/CompressionFactory.h>
#include <Interpreters/InterpreterDropQuery.h>
#include <Interpreters/QueryLog.h>
#include <Interpreters/addTypeConversionToAST.h>
#include <Interpreters/FunctionNameNormalizer.h>
#include <Interpreters/ApplyWithSubqueryVisitor.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Functions/UserDefined/UserDefinedSQLFunctionFactory.h>
#include <Functions/UserDefined/UserDefinedSQLFunctionVisitor.h>
#include <Interpreters/ReplaceQueryParameterVisitor.h>
#include <Parsers/QueryParameterVisitor.h>
namespace CurrentMetrics
{
extern const Metric AttachedTable;
extern const Metric AttachedReplicatedTable;
extern const Metric AttachedDictionary;
extern const Metric AttachedView;
}
namespace DB
{
namespace Setting
{
extern const SettingsBool allow_experimental_analyzer;
extern const SettingsBool allow_experimental_codecs;
extern const SettingsBool allow_experimental_database_materialized_postgresql;
extern const SettingsBool allow_experimental_full_text_index;
extern const SettingsBool allow_experimental_inverted_index;
extern const SettingsBool allow_experimental_statistics;
extern const SettingsBool allow_experimental_vector_similarity_index;
extern const SettingsBool allow_materialized_view_with_bad_select;
extern const SettingsBool allow_suspicious_codecs;
extern const SettingsBool compatibility_ignore_collation_in_create_table;
extern const SettingsBool compatibility_ignore_auto_increment_in_create_table;
extern const SettingsBool create_if_not_exists;
extern const SettingsFloat create_replicated_merge_tree_fault_injection_probability;
extern const SettingsBool database_atomic_wait_for_drop_and_detach_synchronously;
extern const SettingsUInt64 database_replicated_allow_explicit_uuid;
extern const SettingsBool database_replicated_allow_heavy_create;
extern const SettingsBool database_replicated_allow_only_replicated_engine;
extern const SettingsBool data_type_default_nullable;
extern const SettingsSQLSecurityType default_materialized_view_sql_security;
extern const SettingsSQLSecurityType default_normal_view_sql_security;
extern const SettingsDefaultTableEngine default_table_engine;
extern const SettingsDefaultTableEngine default_temporary_table_engine;
extern const SettingsString default_view_definer;
extern const SettingsUInt64 distributed_ddl_entry_format_version;
extern const SettingsBool enable_deflate_qpl_codec;
extern const SettingsBool enable_zstd_qat_codec;
extern const SettingsBool flatten_nested;
extern const SettingsBool fsync_metadata;
extern const SettingsBool insert_allow_materialized_columns;
extern const SettingsSeconds lock_acquire_timeout;
extern const SettingsUInt64 max_parser_backtracks;
extern const SettingsUInt64 max_parser_depth;
extern const SettingsBool restore_replace_external_engines_to_null;
extern const SettingsBool restore_replace_external_table_functions_to_null;
extern const SettingsBool restore_replace_external_dictionary_source_to_null;
}
namespace ServerSetting
{
extern const ServerSettingsBool ignore_empty_sql_security_in_create_view_query;
extern const ServerSettingsUInt64 max_database_num_to_throw;
extern const ServerSettingsUInt64 max_dictionary_num_to_throw;
extern const ServerSettingsUInt64 max_table_num_to_throw;
extern const ServerSettingsUInt64 max_replicated_table_num_to_throw;
extern const ServerSettingsUInt64 max_view_num_to_throw;
}
namespace ErrorCodes
{
extern const int TABLE_ALREADY_EXISTS;
extern const int DICTIONARY_ALREADY_EXISTS;
extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
extern const int INCORRECT_QUERY;
extern const int UNKNOWN_DATABASE_ENGINE;
extern const int DUPLICATE_COLUMN;
extern const int DATABASE_ALREADY_EXISTS;
extern const int BAD_ARGUMENTS;
extern const int BAD_DATABASE_FOR_TEMPORARY_TABLE;
extern const int ILLEGAL_SYNTAX_FOR_DATA_TYPE;
extern const int ILLEGAL_COLUMN;
extern const int ILLEGAL_INDEX;
extern const int LOGICAL_ERROR;
extern const int UNKNOWN_DATABASE;
extern const int PATH_ACCESS_DENIED;
extern const int NOT_IMPLEMENTED;
extern const int ENGINE_REQUIRED;
extern const int UNKNOWN_STORAGE;
extern const int SYNTAX_ERROR;
extern const int SUPPORT_IS_DISABLED;
extern const int TOO_MANY_TABLES;
extern const int TOO_MANY_DATABASES;
extern const int THERE_IS_NO_COLUMN;
}
namespace fs = std::filesystem;
InterpreterCreateQuery::InterpreterCreateQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_)
: WithMutableContext(context_), query_ptr(query_ptr_)
{
}
BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
{
String database_name = create.getDatabase();
auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, "");
/// Database can be created before or it can be created concurrently in another thread, while we were waiting in DDLGuard
if (DatabaseCatalog::instance().isDatabaseExist(database_name))
{
if (create.if_not_exists)
return {};
throw Exception(ErrorCodes::DATABASE_ALREADY_EXISTS, "Database {} already exists.", database_name);
}
auto db_num_limit = getContext()->getGlobalContext()->getServerSettings()[ServerSetting::max_database_num_to_throw];
if (db_num_limit > 0 && !internal)
{
size_t db_count = DatabaseCatalog::instance().getDatabases().size();
std::initializer_list<std::string_view> system_databases =
{
DatabaseCatalog::TEMPORARY_DATABASE,
DatabaseCatalog::SYSTEM_DATABASE,
DatabaseCatalog::INFORMATION_SCHEMA,
DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE,
};
for (const auto & system_database : system_databases)
{
if (db_count > 0 && DatabaseCatalog::instance().isDatabaseExist(std::string(system_database)))
--db_count;
}
if (db_count >= db_num_limit)
throw Exception(ErrorCodes::TOO_MANY_DATABASES,
"Too many databases. "
"The limit (server configuration parameter `max_database_num_to_throw`) is set to {}, the current number of databases is {}",
db_num_limit, db_count);
}
auto db_disk = getContext()->getDatabaseDisk();
/// Will write file with database metadata, if needed.
String database_name_escaped = escapeForFileName(database_name);
fs::path metadata_dir_path("metadata");
fs::path store_dir_path("store");
db_disk->createDirectories(metadata_dir_path);
fs::path metadata_file_tmp_path = metadata_dir_path / (database_name_escaped + ".sql.tmp");
fs::path metadata_file_path = metadata_dir_path / (database_name_escaped + ".sql");
fs::path metadata_path;
if (!create.storage && create.attach)
{
if (!db_disk->existsFile(metadata_file_path))
throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, "Database engine must be specified for ATTACH DATABASE query");
/// Short syntax: try read database definition from file
auto ast = DatabaseOnDisk::parseQueryFromMetadata(nullptr, getContext(), metadata_file_path);
create = ast->as<ASTCreateQuery &>();
if (create.table || !create.storage)
throw Exception(ErrorCodes::INCORRECT_QUERY, "Metadata file {} contains incorrect CREATE DATABASE query", metadata_file_path.string());
create.attach = true;
create.attach_short_syntax = true;
create.setDatabase(database_name);
}
else if (!create.storage || !create.storage->engine)
{
/// For new-style databases engine is explicitly specified in .sql
/// When attaching old-style database during server startup, we must always use Ordinary engine
if (create.attach)
throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, "Database engine must be specified for ATTACH DATABASE query");
if (!create.storage)
{
auto storage = std::make_shared<ASTStorage>();
create.set(create.storage, storage);
}
auto engine = std::make_shared<ASTFunction>();
engine->name = "Atomic";
engine->no_empty_args = true;
create.storage->set(create.storage->engine, engine);
}
else if ((create.columns_list
&& ((create.columns_list->indices && !create.columns_list->indices->children.empty())
|| (create.columns_list->projections && !create.columns_list->projections->children.empty()))))
{
/// Currently, there are no database engines, that support any arguments.
throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, "Unknown database engine: {}", serializeAST(*create.storage));
}
if (create.storage && !create.storage->engine)
throw Exception(ErrorCodes::INCORRECT_QUERY, "Database engine must be specified");
if (create.storage->engine->name == "Atomic"
|| create.storage->engine->name == "Replicated"
|| create.storage->engine->name == "MaterializedPostgreSQL")
{
if (create.attach && create.uuid == UUIDHelpers::Nil)
throw Exception(ErrorCodes::INCORRECT_QUERY, "UUID must be specified for ATTACH. "
"If you want to attach existing database, use just ATTACH DATABASE {};", create.getDatabase());
if (create.uuid == UUIDHelpers::Nil)
create.uuid = UUIDHelpers::generateV4();
metadata_path = store_dir_path / DatabaseCatalog::getPathForUUID(create.uuid);
if (!create.attach && db_disk->existsDirectory(metadata_path) && !db_disk->isDirectoryEmpty(metadata_path))
throw Exception(ErrorCodes::DATABASE_ALREADY_EXISTS, "Metadata directory {} already exists and is not empty", metadata_path.string());
}
else
{
bool is_on_cluster = getContext()->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY;
if (create.uuid != UUIDHelpers::Nil && !is_on_cluster && !internal)
throw Exception(ErrorCodes::INCORRECT_QUERY, "Ordinary database engine does not support UUID");
/// The database doesn't support UUID so we'll ignore it. The UUID could be set here because of either
/// a) the initiator of `ON CLUSTER` query generated it to ensure the same UUIDs are used on different hosts; or
/// b) `RESTORE from backup` query generated it to ensure the same UUIDs are used on different hosts.
create.uuid = UUIDHelpers::Nil;
metadata_path = metadata_dir_path / database_name_escaped;
}
if (create.storage->engine->name == "Replicated" && !internal && !create.attach && create.storage->engine->arguments)
{
/// Fill in default parameters
if (create.storage->engine->arguments->children.size() == 1)
create.storage->engine->arguments->children.push_back(std::make_shared<ASTLiteral>("{shard}"));
if (create.storage->engine->arguments->children.size() == 2)
create.storage->engine->arguments->children.push_back(std::make_shared<ASTLiteral>("{replica}"));
}
if (create.storage->engine->name == "MaterializedPostgreSQL"
&& !getContext()->getSettingsRef()[Setting::allow_experimental_database_materialized_postgresql] && !internal && !create.attach)
{
throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE,
"MaterializedPostgreSQL is an experimental database engine. "
"Enable allow_experimental_database_materialized_postgresql to use it");
}
bool need_write_metadata = !create.attach || !db_disk->existsFile(metadata_file_path);
bool need_lock_uuid = internal || need_write_metadata;
auto mode = getLoadingStrictnessLevel(create.attach, force_attach, has_force_restore_data_flag, /*secondary*/ false);
/// Lock uuid, so we will known it's already in use.
/// We do it when attaching databases on server startup (internal) and on CREATE query (!create.attach);
TemporaryLockForUUIDDirectory uuid_lock;
if (need_lock_uuid)
uuid_lock = TemporaryLockForUUIDDirectory{create.uuid};
else if (create.uuid != UUIDHelpers::Nil && !DatabaseCatalog::instance().hasUUIDMapping(create.uuid))
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create.uuid);
DatabasePtr database = DatabaseFactory::instance().get(create, metadata_path / "", getContext());
if (create.uuid != UUIDHelpers::Nil)
create.setDatabase(TABLE_WITH_UUID_NAME_PLACEHOLDER);
if (need_write_metadata)
{
create.attach = true;
create.if_not_exists = false;
WriteBufferFromOwnString statement_buf;
formatAST(create, statement_buf, false);
writeChar('\n', statement_buf);
String statement = statement_buf.str();
/// Needed to make database creation retriable if it fails after the file is created
db_disk->removeFileIfExists(metadata_file_tmp_path);
/// Exclusive flag guarantees, that database is not created right now in another thread.
writeMetadataFile(
db_disk,
/*file_path=*/metadata_file_tmp_path,
/*content=*/statement,
/*fsync_metadata=*/getContext()->getSettingsRef()[Setting::fsync_metadata]);
}
/// We attach database before loading it's tables, so do not allow concurrent DDL queries
auto db_guard = DatabaseCatalog::instance().getExclusiveDDLGuardForDatabase(database_name);
bool added = false;
bool renamed = false;
try
{
/// TODO Attach db only after it was loaded. Now it's not possible because of view dependencies
DatabaseCatalog::instance().attachDatabase(database_name, database);
added = true;
if (!load_database_without_tables)
{
/// We use global context here, because storages lifetime is bigger than query context lifetime
TablesLoader loader{getContext()->getGlobalContext(), {{database_name, database}}, mode};
auto load_tasks = loader.loadTablesAsync();
auto startup_tasks = loader.startupTablesAsync();
/// First prioritize, schedule and wait all the load table tasks
waitLoad(currentPoolOr(TablesLoaderForegroundPoolId), load_tasks);
/// Only then prioritize, schedule and wait all the startup tasks
waitLoad(currentPoolOr(TablesLoaderForegroundPoolId), startup_tasks);
}
if (need_write_metadata)
{
/// Prevents from overwriting metadata of detached database
db_disk->moveFile(metadata_file_tmp_path, metadata_file_path);
renamed = true;
}
}
catch (...)
{
if (renamed)
{
assert(db_disk->existsFile(metadata_file_path));
db_disk->removeFileIfExists(metadata_file_path);
}
if (added)
DatabaseCatalog::instance().detachDatabase(getContext(), database_name, false, false);
throw;
}
return {};
}
ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns)
{
auto columns_list = std::make_shared<ASTExpressionList>();
for (const auto & column : columns)
{
const auto column_declaration = std::make_shared<ASTColumnDeclaration>();
column_declaration->name = column.name;
ParserDataType type_parser;
String type_name = column.type->getName();
const char * pos = type_name.data();
const char * end = pos + type_name.size();
column_declaration->type = parseQuery(type_parser, pos, end, "data type", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
columns_list->children.emplace_back(column_declaration);
}
return columns_list;
}
ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns, const NamesAndAliases & alias_columns)
{
std::shared_ptr<ASTExpressionList> columns_list = std::static_pointer_cast<ASTExpressionList>(formatColumns(columns));
for (const auto & alias_column : alias_columns)
{
const auto column_declaration = std::make_shared<ASTColumnDeclaration>();
column_declaration->name = alias_column.name;
ParserDataType type_parser;
String type_name = alias_column.type->getName();
const char * type_pos = type_name.data();
const char * type_end = type_pos + type_name.size();
column_declaration->type = parseQuery(type_parser, type_pos, type_end, "data type", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
column_declaration->default_specifier = "ALIAS";
const auto & alias = alias_column.expression;
const char * alias_pos = alias.data();
const char * alias_end = alias_pos + alias.size();
ParserExpression expression_parser;
column_declaration->default_expression = parseQuery(expression_parser, alias_pos, alias_end, "expression", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
column_declaration->children.push_back(column_declaration->default_expression);
columns_list->children.emplace_back(column_declaration);
}
return columns_list;
}
ASTPtr InterpreterCreateQuery::formatColumns(const ColumnsDescription & columns)
{
auto columns_list = std::make_shared<ASTExpressionList>();
for (const auto & column : columns)
{
const auto column_declaration = std::make_shared<ASTColumnDeclaration>();
ASTPtr column_declaration_ptr{column_declaration};
column_declaration->name = column.name;
ParserDataType type_parser;
String type_name = column.type->getName();
const char * type_name_pos = type_name.data();
const char * type_name_end = type_name_pos + type_name.size();
column_declaration->type = parseQuery(type_parser, type_name_pos, type_name_end, "data type", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
if (column.default_desc.expression)
{
column_declaration->default_specifier = toString(column.default_desc.kind);
column_declaration->default_expression = column.default_desc.expression->clone();
column_declaration->children.push_back(column_declaration->default_expression);
}
column_declaration->ephemeral_default = column.default_desc.ephemeral_default;
if (!column.comment.empty())
{
column_declaration->comment = std::make_shared<ASTLiteral>(Field(column.comment));
column_declaration->children.push_back(column_declaration->comment);
}
if (column.codec)
{
column_declaration->codec = column.codec;
column_declaration->children.push_back(column_declaration->codec);
}
if (!column.statistics.empty())
{
column_declaration->statistics_desc = column.statistics.getAST();
column_declaration->children.push_back(column_declaration->statistics_desc);
}
if (column.ttl)
{
column_declaration->ttl = column.ttl;
column_declaration->children.push_back(column_declaration->ttl);
}
if (!column.settings.empty())
{
auto settings = std::make_shared<ASTSetQuery>();
settings->is_standalone = false;
settings->changes = column.settings;
column_declaration->settings = std::move(settings);
}
columns_list->children.push_back(column_declaration_ptr);
}
return columns_list;
}
ASTPtr InterpreterCreateQuery::formatIndices(const IndicesDescription & indices)
{
auto res = std::make_shared<ASTExpressionList>();
for (const auto & index : indices)
res->children.push_back(index.definition_ast->clone());
return res;
}
ASTPtr InterpreterCreateQuery::formatConstraints(const ConstraintsDescription & constraints)
{
auto res = std::make_shared<ASTExpressionList>();
for (const auto & constraint : constraints.getConstraints())
res->children.push_back(constraint->clone());
return res;
}
ASTPtr InterpreterCreateQuery::formatProjections(const ProjectionsDescription & projections)
{
auto res = std::make_shared<ASTExpressionList>();
for (const auto & projection : projections)
res->children.push_back(projection.definition_ast->clone());
return res;
}
DataTypePtr InterpreterCreateQuery::getColumnType(
const ASTColumnDeclaration & col_decl, const LoadingStrictnessLevel mode, const bool make_columns_nullable)
{
if (!col_decl.type)
{
/// we're creating dummy DataTypeUInt8 in order to prevent the NullPointerException in ExpressionActions
return std::make_shared<DataTypeUInt8>();
}
DataTypePtr column_type = DataTypeFactory::instance().get(col_decl.type);
if (LoadingStrictnessLevel::ATTACH <= mode)
setVersionToAggregateFunctions(column_type, true);
if (col_decl.null_modifier)
{
if (column_type->isNullable())
throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE, "Can't use [NOT] NULL modifier with Nullable type");
if (*col_decl.null_modifier)
column_type = makeNullable(column_type);
}
else if (make_columns_nullable)
{
column_type = makeNullable(column_type);
}
else if (
!hasNullable(column_type) && col_decl.default_specifier == "DEFAULT" && col_decl.default_expression
&& col_decl.default_expression->as<ASTLiteral>() && col_decl.default_expression->as<ASTLiteral>()->value.isNull())
{
if (column_type->lowCardinality())
{
const auto * low_cardinality_type = typeid_cast<const DataTypeLowCardinality *>(column_type.get());
assert(low_cardinality_type);
column_type = std::make_shared<DataTypeLowCardinality>(makeNullable(low_cardinality_type->getDictionaryType()));
}
else
column_type = makeNullable(column_type);
}
return column_type;
}
ColumnsDescription InterpreterCreateQuery::getColumnsDescription(
const ASTExpressionList & columns_ast, ContextPtr context_, LoadingStrictnessLevel mode, bool is_restore_from_backup)
{
/// First, deduce implicit types.
/** all default_expressions as a single expression list,
* mixed with conversion-columns for each explicitly specified type */
DefaultExpressionsInfo default_expr_info{std::make_shared<ASTExpressionList>()};
NamesAndTypesList column_names_and_types;
bool make_columns_nullable = mode <= LoadingStrictnessLevel::SECONDARY_CREATE && !is_restore_from_backup
&& context_->getSettingsRef()[Setting::data_type_default_nullable];
for (const auto & ast : columns_ast.children)
{
const auto & col_decl = ast->as<ASTColumnDeclaration &>();
if (col_decl.collation && !context_->getSettingsRef()[Setting::compatibility_ignore_collation_in_create_table])
{
throw Exception(
ErrorCodes::NOT_IMPLEMENTED, "Cannot support collation, please set compatibility_ignore_collation_in_create_table=true");
}
column_names_and_types.emplace_back(col_decl.name, getColumnType(col_decl, mode, make_columns_nullable));
/// add column to postprocessing if there is a default_expression specified
getDefaultExpressionInfoInto(col_decl, column_names_and_types.back().type, default_expr_info);
}
Block defaults_sample_block;
/// Set missing types and wrap default_expression's in a conversion-function if necessary.
/// We try to avoid that validation while restoring from a backup because it might be slow or troublesome
/// (for example, a default expression can contain dictGet() and that dictionary can access remote servers or
/// require different users to authenticate).
if (!default_expr_info.expr_list->children.empty()
&& (default_expr_info.has_columns_with_default_without_type || (mode <= LoadingStrictnessLevel::CREATE)))
{
defaults_sample_block = validateColumnsDefaultsAndGetSampleBlock(default_expr_info.expr_list, column_names_and_types, context_);
}
bool skip_checks = LoadingStrictnessLevel::SECONDARY_CREATE <= mode;
bool sanity_check_compression_codecs = !skip_checks && !context_->getSettingsRef()[Setting::allow_suspicious_codecs];
bool allow_experimental_codecs = skip_checks || context_->getSettingsRef()[Setting::allow_experimental_codecs];
bool enable_deflate_qpl_codec = skip_checks || context_->getSettingsRef()[Setting::enable_deflate_qpl_codec];
bool enable_zstd_qat_codec = skip_checks || context_->getSettingsRef()[Setting::enable_zstd_qat_codec];
ColumnsDescription res;
auto name_type_it = column_names_and_types.begin();
for (const auto * ast_it = columns_ast.children.begin(); ast_it != columns_ast.children.end(); ++ast_it, ++name_type_it)
{
ColumnDescription column;
auto & col_decl = (*ast_it)->as<ASTColumnDeclaration &>();
column.name = col_decl.name;
/// ignore or not other database extensions depending on compatibility settings
if (col_decl.default_specifier == "AUTO_INCREMENT"
&& !context_->getSettingsRef()[Setting::compatibility_ignore_auto_increment_in_create_table])
{
throw Exception(ErrorCodes::SYNTAX_ERROR,
"AUTO_INCREMENT is not supported. To ignore the keyword "
"in column declaration, set `compatibility_ignore_auto_increment_in_create_table` to true");
}
if (col_decl.default_expression)
{
if (context_->hasQueryContext() && context_->getQueryContext().get() == context_.get())
{
/// Normalize query only for original CREATE query, not on metadata loading.
/// And for CREATE query we can pass local context, because result will not change after restart.
NormalizeAndEvaluateConstantsVisitor::Data visitor_data{context_};
NormalizeAndEvaluateConstantsVisitor visitor(visitor_data);
visitor.visit(col_decl.default_expression);
}
ASTPtr default_expr = col_decl.default_expression->clone();
if (col_decl.type)
column.type = name_type_it->type;
else
{
column.type = defaults_sample_block.getByName(column.name).type;
/// set nullability for case of column declaration w/o type but with default expression
if ((col_decl.null_modifier && *col_decl.null_modifier) || make_columns_nullable)
column.type = makeNullable(column.type);
}
column.default_desc.kind = columnDefaultKindFromString(col_decl.default_specifier);
column.default_desc.expression = default_expr;
column.default_desc.ephemeral_default = col_decl.ephemeral_default;
}
else if (col_decl.type)
column.type = name_type_it->type;
else
throw Exception(ErrorCodes::LOGICAL_ERROR, "Neither default value expression nor type is provided for a column");
if (col_decl.comment)
column.comment = col_decl.comment->as<ASTLiteral &>().value.safeGet<String>();
if (col_decl.codec)
{
if (col_decl.default_specifier == "ALIAS")
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS");
column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(
col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs, enable_deflate_qpl_codec, enable_zstd_qat_codec);
}
if (col_decl.statistics_desc)
{
if (!skip_checks && !context_->getSettingsRef()[Setting::allow_experimental_statistics])
throw Exception(
ErrorCodes::INCORRECT_QUERY, "Create table with statistics is now disabled. Turn on allow_experimental_statistics");
column.statistics = ColumnStatisticsDescription::fromColumnDeclaration(col_decl, column.type);
}
if (col_decl.ttl)
column.ttl = col_decl.ttl;
if (col_decl.settings)
{
column.settings = col_decl.settings->as<ASTSetQuery &>().changes;
MergeTreeColumnSettings::validate(column.settings);
}
res.add(std::move(column));
}
if (mode <= LoadingStrictnessLevel::SECONDARY_CREATE && !is_restore_from_backup && context_->getSettingsRef()[Setting::flatten_nested])
res.flattenNested();
if (res.getAllPhysical().empty())
throw Exception(ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED, "Cannot CREATE table without physical columns");
return res;
}
ConstraintsDescription InterpreterCreateQuery::getConstraintsDescription(
const ASTExpressionList * constraints, const ColumnsDescription & columns, ContextPtr local_context)
{
ASTs constraints_data;
const auto column_names_and_types = columns.getAllPhysical();
if (constraints)
for (const auto & constraint : constraints->children)
{
auto clone = constraint->clone();
TreeRewriter(local_context).analyze(clone, column_names_and_types);
constraints_data.push_back(constraint->clone());
}
return ConstraintsDescription{constraints_data};
}
InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTablePropertiesAndNormalizeCreateQuery(
ASTCreateQuery & create, LoadingStrictnessLevel mode) const
{
/// Set the table engine if it was not specified explicitly.
setEngine(create);
/// We have to check access rights again (in case engine was changed).
if (create.storage && create.storage->engine)
getContext()->checkAccess(AccessType::TABLE_ENGINE, create.storage->engine->name);
/// If this is a TimeSeries table then we need to normalize list of columns (add missing columns and reorder), and also set inner table engines.
if (create.is_time_series_table && (mode < LoadingStrictnessLevel::ATTACH))
StorageTimeSeries::normalizeTableDefinition(create, getContext());
TableProperties properties;
TableLockHolder as_storage_lock;
if (create.columns_list)
{
if (create.as_table_function && (create.columns_list->indices || create.columns_list->constraints))
throw Exception(ErrorCodes::INCORRECT_QUERY, "Indexes and constraints are not supported for table functions");
/// Dictionaries have dictionary_attributes_list instead of columns_list
assert(!create.is_dictionary);
if (create.columns_list->columns)
{
properties.columns = getColumnsDescription(*create.columns_list->columns, getContext(), mode, is_restore_from_backup);
}
if (create.columns_list->indices)
for (const auto & index : create.columns_list->indices->children)
{
IndexDescription index_desc = IndexDescription::getIndexFromAST(index->clone(), properties.columns, getContext());
if (properties.indices.has(index_desc.name))
throw Exception(ErrorCodes::ILLEGAL_INDEX, "Duplicated index name {} is not allowed. Please use a different index name", backQuoteIfNeed(index_desc.name));
const auto & settings = getContext()->getSettingsRef();
if (index_desc.type == FULL_TEXT_INDEX_NAME && !settings[Setting::allow_experimental_full_text_index])
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "The experimental full-text index feature is disabled. Enable the setting 'allow_experimental_full_text_index' to use it");
/// ----
/// Temporary check during a transition period. Please remove at the end of 2024.
if (index_desc.type == INVERTED_INDEX_NAME && !settings[Setting::allow_experimental_inverted_index])
throw Exception(ErrorCodes::ILLEGAL_INDEX, "The 'inverted' index type is deprecated. Please use the 'full_text' index type instead");
/// ----
if (index_desc.type == "vector_similarity" && !settings[Setting::allow_experimental_vector_similarity_index])
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "The experimental vector similarity index feature is disabled. Enable the setting 'allow_experimental_vector_similarity_index' to use it");
properties.indices.push_back(index_desc);
}
if (create.columns_list->projections)
for (const auto & projection_ast : create.columns_list->projections->children)
{
auto projection = ProjectionDescription::getProjectionFromAST(projection_ast, properties.columns, getContext());
properties.projections.add(std::move(projection));
}
properties.constraints = getConstraintsDescription(create.columns_list->constraints, properties.columns, getContext());
}
else if (!create.as_table.empty())
{
String as_database_name = getContext()->resolveDatabase(create.as_database);
StoragePtr as_storage = DatabaseCatalog::instance().getTable({as_database_name, create.as_table}, getContext());
/// as_storage->getColumns() and setEngine(...) must be called under structure lock of other_table for CREATE ... AS other_table.
as_storage_lock = as_storage->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef()[Setting::lock_acquire_timeout]);
auto as_storage_metadata = as_storage->getInMemoryMetadataPtr();
properties.columns = as_storage_metadata->getColumns();
if (!create.comment && !as_storage_metadata->comment.empty())
create.set(create.comment, std::make_shared<ASTLiteral>(as_storage_metadata->comment));
/// Secondary indices and projections make sense only for MergeTree family of storage engines.
/// We should not copy them for other storages.
if (create.storage && endsWith(create.storage->engine->name, "MergeTree"))
{
/// Copy secondary indexes but only the ones which were not implicitly created. These will be re-generated later again and need
/// not be copied.
const auto & indices = as_storage_metadata->getSecondaryIndices();
for (const auto & index : indices)
if (!index.isImplicitlyCreated())
properties.indices.push_back(index);
/// Copy projections.
properties.projections = as_storage_metadata->getProjections().clone();
/// CREATE TABLE AS should copy PRIMARY KEY, ORDER BY, and similar clauses.
/// Note: only supports the source table engine is using the new syntax.
if (const auto * merge_tree_data = dynamic_cast<const MergeTreeData *>(as_storage.get()))
{
if (merge_tree_data->format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
{
if (!create.storage->primary_key && as_storage_metadata->isPrimaryKeyDefined() && as_storage_metadata->hasPrimaryKey())
create.storage->set(create.storage->primary_key, as_storage_metadata->getPrimaryKeyAST()->clone());
if (!create.storage->partition_by && as_storage_metadata->isPartitionKeyDefined() && as_storage_metadata->hasPartitionKey())
create.storage->set(create.storage->partition_by, as_storage_metadata->getPartitionKeyAST()->clone());
if (!create.storage->order_by && as_storage_metadata->isSortingKeyDefined() && as_storage_metadata->hasSortingKey())
create.storage->set(create.storage->order_by, as_storage_metadata->getSortingKeyAST()->clone());
if (!create.storage->sample_by && as_storage_metadata->isSamplingKeyDefined() && as_storage_metadata->hasSamplingKey())
create.storage->set(create.storage->sample_by, as_storage_metadata->getSamplingKeyAST()->clone());
}
}
}
else
{
/// Only MergeTree support TTL
properties.columns.resetColumnTTLs();
}
properties.constraints = as_storage_metadata->getConstraints();
if (create.is_clone_as)
{
if (!endsWith(as_storage->getName(), "MergeTree"))
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support CLONE AS from tables of the MergeTree family");
if (create.storage)
{
if (!endsWith(create.storage->engine->name, "MergeTree"))
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support CLONE AS with tables of the MergeTree family");
/// Ensure that as_storage and the new storage has the same primary key, sorting key and partition key
auto query_to_string = [](const IAST * ast) { return ast ? queryToString(*ast) : ""; };
const String as_storage_sorting_key_str = query_to_string(as_storage_metadata->getSortingKeyAST().get());
const String as_storage_primary_key_str = query_to_string(as_storage_metadata->getPrimaryKeyAST().get());
const String as_storage_partition_key_str = query_to_string(as_storage_metadata->getPartitionKeyAST().get());
const String storage_sorting_key_str = query_to_string(create.storage->order_by);
const String storage_primary_key_str = query_to_string(create.storage->primary_key);
const String storage_partition_key_str = query_to_string(create.storage->partition_by);
if (as_storage_sorting_key_str != storage_sorting_key_str)
{
/// It is possible that the storage only has primary key and an empty sorting key, and as_storage has both primary key and sorting key with the same value.
if (as_storage_sorting_key_str != as_storage_primary_key_str || as_storage_sorting_key_str != storage_primary_key_str)
{
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different ordering");
}
}
if (as_storage_partition_key_str != storage_partition_key_str)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different partition key");
if (as_storage_primary_key_str != storage_primary_key_str)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different primary key");
}
}
}
else if (create.select)
{
if (create.isParameterizedView())
return properties;
if (create.aliases_list)
{
auto & aliases_children = create.aliases_list->children;
const auto * select_with_union_query = create.select->as<ASTSelectWithUnionQuery>();
if (!select_with_union_query)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected ASTSelectWithUnionQuery");
const auto & selects = select_with_union_query->list_of_selects->children;
for (const auto & select : selects)
{
const auto * select_query = select->as<ASTSelectQuery>();
if (!select_query)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected ASTSelectQuery inside ASTSelectWithUnionQuery");
auto select_expression_list = select_query->select();
if (!select_expression_list)
throw Exception(ErrorCodes::LOGICAL_ERROR, "No select expressions in SELECT query");
auto & select_expressions = select_expression_list->children;
if (select_expressions.size() != aliases_children.size())
{
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Number of aliases does not match number of expressions in SELECT list");
}
for (size_t i = 0; i < select_expressions.size(); ++i)
{
auto & expr = select_expressions[i];
const auto & alias_ast = aliases_children[i]->as<ASTIdentifier &>();
expr->setAlias(alias_ast.name());
}
}
}
Block as_select_sample;
if (getContext()->getSettingsRef()[Setting::allow_experimental_analyzer])
{
as_select_sample = InterpreterSelectQueryAnalyzer::getSampleBlock(create.select->clone(), getContext());
}
else
{
as_select_sample = InterpreterSelectWithUnionQuery::getSampleBlock(create.select->clone(), getContext());
}
properties.columns = ColumnsDescription(as_select_sample.getNamesAndTypesList());
properties.columns_inferred_from_select_query = true;
}
else if (create.as_table_function)
{
/// Table function without columns list.
auto table_function_ast = create.as_table_function->ptr();
auto table_function = TableFunctionFactory::instance().get(table_function_ast, getContext());
properties.columns = table_function->getActualTableStructure(getContext(), /*is_insert_query*/ true);
}
else if (create.is_dictionary)
{
if (!create.dictionary || !create.dictionary->source)
return {};
/// Evaluate expressions (like currentDatabase() or tcpPort()) in dictionary source definition.
NormalizeAndEvaluateConstantsVisitor::Data visitor_data{getContext()};
NormalizeAndEvaluateConstantsVisitor visitor(visitor_data);
visitor.visit(create.dictionary->source->ptr());
return {};
}
else if (!create.storage || !create.storage->engine)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected application state. CREATE query is missing either its storage or engine.");
/// We can have queries like "CREATE TABLE <table> ENGINE=<engine>" if <engine>
/// supports schema inference (will determine table structure in it's constructor).
else if (!StorageFactory::instance().getStorageFeatures(create.storage->engine->name).supports_schema_inference)
throw Exception(ErrorCodes::INCORRECT_QUERY, "Incorrect CREATE query: required list of column descriptions or AS section or SELECT.");
/// Even if query has list of columns, canonicalize it (unfold Nested columns).
if (!create.columns_list)
create.set(create.columns_list, std::make_shared<ASTColumns>());
ASTPtr new_columns = formatColumns(properties.columns);
ASTPtr new_indices = formatIndices(properties.indices);
ASTPtr new_constraints = formatConstraints(properties.constraints);