diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index dc3a94257d02..c55bd407e159 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1933,7 +1933,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( bool first_stage_, bool second_stage_, bool only_types, - const FilterDAGInfoPtr & filter_info_, + const FilterDAGInfoPtr & row_policy_info_, const FilterDAGInfoPtr & additional_filter, const Block & source_header) : first_stage(first_stage_) @@ -2031,10 +2031,10 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( columns_for_additional_filter.begin(), columns_for_additional_filter.end()); } - if (storage && filter_info_) + if (storage && row_policy_info_) { - filter_info = filter_info_; - filter_info->do_remove_column = true; + row_policy_info = row_policy_info_; + row_policy_info->do_remove_column = true; } if (prewhere_dag_and_flags = query_analyzer.appendPrewhere(chain, !first_stage); prewhere_dag_and_flags) @@ -2373,9 +2373,9 @@ std::string ExpressionAnalysisResult::dump() const ss << "prewhere_info " << prewhere_info->dump() << "\n"; } - if (filter_info) + if (row_policy_info) { - ss << "filter_info " << filter_info->dump() << "\n"; + ss << "filter_info " << row_policy_info->dump() << "\n"; } if (before_aggregation) diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 881bc31071f5..b0a5d67a440a 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -270,7 +270,7 @@ struct ExpressionAnalysisResult NameSet columns_to_remove_after_prewhere; PrewhereInfoPtr prewhere_info; - FilterDAGInfoPtr filter_info; + FilterDAGInfoPtr row_policy_info; ConstantFilterDescription prewhere_constant_filter_description; ConstantFilterDescription where_constant_filter_description; /// Actions by every element of ORDER BY @@ -285,12 +285,12 @@ struct ExpressionAnalysisResult bool first_stage, bool second_stage, bool only_types, - const FilterDAGInfoPtr & filter_info, + const FilterDAGInfoPtr & row_policy_info, const FilterDAGInfoPtr & additional_filter, /// for setting additional_filters const Block & source_header); /// Filter for row-level security. - bool hasFilter() const { return filter_info.get(); } + bool hasRowPolicyFilter() const { return row_policy_info.get(); } bool hasJoin() const { return join.get(); } bool hasPrewhere() const { return prewhere_info.get(); } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 32f05f89a9d6..2c96ced02581 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -885,7 +885,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( /// Fix source_header for filter actions. if (row_policy_filter && !row_policy_filter->empty()) { - filter_info = generateFilterActions( + row_policy_info = generateFilterActions( table_id, row_policy_filter->expression, context, storage, storage_snapshot, metadata_snapshot, required_columns, prepared_sets); @@ -1052,8 +1052,6 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis() max_rows = max_rows ? std::min(max_rows, settings[Setting::max_rows_to_read].value) : settings[Setting::max_rows_to_read]; query_info_copy.trivial_limit = max_rows; - /// Apply filters to prewhere and add them to the query_info so we can filter out parts efficiently during row estimation - applyFiltersToPrewhereInAnalysis(analysis_copy); if (analysis_copy.prewhere_info) { query_info_copy.prewhere_info = analysis_copy.prewhere_info; @@ -1069,13 +1067,13 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis() = query_info_copy.prewhere_info->prewhere_actions.findInOutputs(query_info_copy.prewhere_info->prewhere_column_name); added_filter_nodes.nodes.push_back(&node); } + } - if (query_info_copy.prewhere_info->row_level_filter) - { - const auto & node - = query_info_copy.prewhere_info->row_level_filter->findInOutputs(query_info_copy.prewhere_info->row_level_column_name); - added_filter_nodes.nodes.push_back(&node); - } + if (query_info_copy.row_level_filter) + { + const auto & node + = query_info_copy.row_level_filter->actions.findInOutputs(query_info_copy.row_level_filter->column_name); + added_filter_nodes.nodes.push_back(&node); } if (auto filter_actions_dag = ActionsDAG::buildFilterActionsDAG(added_filter_nodes.nodes)) @@ -1178,7 +1176,7 @@ Block InterpreterSelectQuery::getSampleBlockImpl() && options.to_stage > QueryProcessingStage::WithMergeableState; analysis_result = ExpressionAnalysisResult( - *query_analyzer, metadata_snapshot, first_stage, second_stage, options.only_analyze, filter_info, additional_filter_info, source_header); + *query_analyzer, metadata_snapshot, first_stage, second_stage, options.only_analyze, row_policy_info, additional_filter_info, source_header); if (options.to_stage == QueryProcessingStage::Enum::FetchColumns) { @@ -1621,13 +1619,13 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

(source_header); query_plan.addStep(std::move(read_nothing)); - if (expressions.filter_info) + if (expressions.row_policy_info) { auto row_level_security_step = std::make_unique( query_plan.getCurrentHeader(), - expressions.filter_info->actions.clone(), - expressions.filter_info->column_name, - expressions.filter_info->do_remove_column); + expressions.row_policy_info->actions.clone(), + expressions.row_policy_info->column_name, + expressions.row_policy_info->do_remove_column); row_level_security_step->setStepDescription("Row-level security filter"); query_plan.addStep(std::move(row_level_security_step)); @@ -1635,18 +1633,6 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

row_level_filter) - { - auto row_level_filter_step = std::make_unique( - query_plan.getCurrentHeader(), - expressions.prewhere_info->row_level_filter->clone(), - expressions.prewhere_info->row_level_column_name, - true); - - row_level_filter_step->setStepDescription("Row-level security filter (PREWHERE)"); - query_plan.addStep(std::move(row_level_filter_step)); - } - auto prewhere_step = std::make_unique( query_plan.getCurrentHeader(), expressions.prewhere_info->prewhere_actions.clone(), @@ -1748,13 +1734,13 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

supportsPrewhere())) { auto row_level_security_step = std::make_unique( query_plan.getCurrentHeader(), - expressions.filter_info->actions.clone(), - expressions.filter_info->column_name, - expressions.filter_info->do_remove_column); + expressions.row_policy_info->actions.clone(), + expressions.row_policy_info->column_name, + expressions.row_policy_info->do_remove_column); row_level_security_step->setStepDescription("Row-level security filter"); query_plan.addStep(std::move(row_level_security_step)); @@ -2194,21 +2180,20 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan(QueryPlan & query_plan, c { Pipe pipe(std::make_shared(source_header)); + if (query_info.row_level_filter) + { + auto row_level_actions = std::make_shared(query_info.row_level_filter->actions.clone()); + pipe.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, + row_level_actions, + query_info.row_level_filter->column_name, + query_info.row_level_filter->do_remove_column); + }); + } if (query_info.prewhere_info) { auto & prewhere_info = *query_info.prewhere_info; - - if (prewhere_info.row_level_filter) - { - auto row_level_actions = std::make_shared(prewhere_info.row_level_filter->clone()); - pipe.addSimpleTransform([&](const Block & header) - { - return std::make_shared(header, - row_level_actions, - prewhere_info.row_level_column_name, true); - }); - } - auto filter_actions = std::make_shared(prewhere_info.prewhere_actions.clone()); pipe.addSimpleTransform([&](const Block & header) { @@ -2247,38 +2232,9 @@ bool InterpreterSelectQuery::shouldMoveToPrewhere() const return settings[Setting::optimize_move_to_prewhere] && (!query.final() || settings[Setting::optimize_move_to_prewhere_if_final]); } -/// Note that this is const and accepts the analysis ref to be able to use it to do analysis for parallel replicas -/// without affecting the final analysis multiple times -void InterpreterSelectQuery::applyFiltersToPrewhereInAnalysis(ExpressionAnalysisResult & analysis) const -{ - if (!analysis.filter_info) - return; - - if (!analysis.prewhere_info) - { - const bool does_storage_support_prewhere = !input_pipe && storage && storage->supportsPrewhere(); - if (does_storage_support_prewhere && shouldMoveToPrewhere()) - { - /// Execute row level filter in prewhere as a part of "move to prewhere" optimization. - analysis.prewhere_info = std::make_shared(std::move(analysis.filter_info->actions), analysis.filter_info->column_name); - analysis.prewhere_info->remove_prewhere_column = std::move(analysis.filter_info->do_remove_column); - analysis.prewhere_info->need_filter = true; - analysis.filter_info = nullptr; - } - } - else - { - /// Add row level security actions to prewhere. - analysis.prewhere_info->row_level_filter = std::move(analysis.filter_info->actions); - analysis.prewhere_info->row_level_column_name = std::move(analysis.filter_info->column_name); - analysis.filter_info = nullptr; - } -} - - void InterpreterSelectQuery::addPrewhereAliasActions() { - applyFiltersToPrewhereInAnalysis(analysis_result); + auto & row_level_filter = analysis_result.row_policy_info; auto & prewhere_info = analysis_result.prewhere_info; auto & columns_to_remove_after_prewhere = analysis_result.columns_to_remove_after_prewhere; @@ -2305,12 +2261,11 @@ void InterpreterSelectQuery::addPrewhereAliasActions() /// Get some columns directly from PREWHERE expression actions auto prewhere_required_columns = prewhere_info->prewhere_actions.getRequiredColumns().getNames(); columns.insert(prewhere_required_columns.begin(), prewhere_required_columns.end()); - - if (prewhere_info->row_level_filter) - { - auto row_level_required_columns = prewhere_info->row_level_filter->getRequiredColumns().getNames(); - columns.insert(row_level_required_columns.begin(), row_level_required_columns.end()); - } + } + if (row_level_filter) + { + auto row_level_required_columns = row_level_filter->actions.getRequiredColumns().getNames(); + columns.insert(row_level_required_columns.begin(), row_level_required_columns.end()); } return columns; @@ -2468,13 +2423,15 @@ std::optional InterpreterSelectQuery::getTrivialCount(UInt64 allow_exper // It's possible to optimize count() given only partition predicates ActionsDAG::NodeRawConstPtrs filter_nodes; + if (analysis_result.hasRowPolicyFilter()) + { + auto & row_level_filter = analysis_result.row_policy_info; + filter_nodes.push_back(&row_level_filter->actions.findInOutputs(row_level_filter->column_name)); + } if (analysis_result.hasPrewhere()) { auto & prewhere_info = analysis_result.prewhere_info; filter_nodes.push_back(&prewhere_info->prewhere_actions.findInOutputs(prewhere_info->prewhere_column_name)); - - if (prewhere_info->row_level_filter) - filter_nodes.push_back(&prewhere_info->row_level_filter->findInOutputs(prewhere_info->row_level_column_name)); } if (analysis_result.hasWhere()) { @@ -2665,10 +2622,11 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc if (max_streams == 0) max_streams = 1; - auto & prewhere_info = analysis_result.prewhere_info; + if (analysis_result.row_policy_info && (!input_pipe && storage && storage->supportsPrewhere())) + query_info.row_level_filter = analysis_result.row_policy_info; - if (prewhere_info) - query_info.prewhere_info = prewhere_info; + if (analysis_result.prewhere_info) + query_info.prewhere_info = analysis_result.prewhere_info; bool optimize_read_in_order = analysis_result.optimize_read_in_order; bool optimize_aggregation_in_order = analysis_result.optimize_read_in_order && !query_analyzer->useGroupingSetKey(); diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 96cedb322a5a..f5e4e509e09d 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -219,7 +219,7 @@ class InterpreterSelectQuery : public IInterpreterUnionOrSelectQuery ExpressionAnalysisResult analysis_result; /// For row-level security. RowPolicyFilterPtr row_policy_filter; - FilterDAGInfoPtr filter_info; + FilterDAGInfoPtr row_policy_info; /// For additional_filter setting. FilterDAGInfoPtr additional_filter_info; diff --git a/src/Interpreters/getHeaderForProcessingStage.cpp b/src/Interpreters/getHeaderForProcessingStage.cpp index c0b691591e8b..6166e4930eab 100644 --- a/src/Interpreters/getHeaderForProcessingStage.cpp +++ b/src/Interpreters/getHeaderForProcessingStage.cpp @@ -103,7 +103,7 @@ Block getHeaderForProcessingStage( case QueryProcessingStage::FetchColumns: { Block header = storage_snapshot->getSampleBlockForColumns(column_names); - header = SourceStepWithFilter::applyPrewhereActions(header, query_info.prewhere_info); + header = SourceStepWithFilter::applyPrewhereActions(header, query_info.row_level_filter, query_info.prewhere_info); return header; } case QueryProcessingStage::WithMergeableState: diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index b97d0f07696a..eb0d6544ae24 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -572,6 +572,7 @@ std::optional buildCustomKeyFilterIfNeeded(const StoragePtr & sto std::optional buildAdditionalFiltersIfNeeded(const StoragePtr & storage, const String & table_expression_alias, SelectQueryInfo & table_expression_query_info, + const PrewhereInfoPtr & prewhere_info, PlannerContextPtr & planner_context) { const auto & query_context = planner_context->getQueryContext(); @@ -611,7 +612,13 @@ std::optional buildAdditionalFiltersIfNeeded(const StoragePtr & s return {}; table_expression_query_info.additional_filter_ast = additional_filter_ast; - return buildFilterInfo(additional_filter_ast, table_expression_query_info.table_expression, planner_context); + auto filter_info = buildFilterInfo(additional_filter_ast, table_expression_query_info.table_expression, planner_context); + if (prewhere_info) + { + for (const auto * input : filter_info.actions.getInputs()) + prewhere_info->prewhere_actions.tryRestoreColumn(input->result_name); + } + return filter_info; } UInt64 mainQueryNodeBlockSizeByLimit(const SelectQueryInfo & select_query_info) @@ -834,6 +841,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres { if (!select_query_options.only_analyze) { + auto & row_level_filter = table_expression_query_info.row_level_filter; auto & prewhere_info = table_expression_query_info.prewhere_info; const auto & prewhere_actions = table_expression_data.getPrewhereFilterActions(); const auto & columns_names = table_expression_data.getColumnNames(); @@ -851,49 +859,17 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres updatePrewhereOutputsIfNeeded(table_expression_query_info, table_expression_data.getColumnNames(), storage_snapshot); std::vector> where_filters; - const auto add_filter = [&](FilterDAGInfo & filter_info, std::string description) - { - bool is_final = table_expression_query_info.table_expression_modifiers - && table_expression_query_info.table_expression_modifiers->hasFinal(); - bool optimize_move_to_prewhere - = settings[Setting::optimize_move_to_prewhere] && (!is_final || settings[Setting::optimize_move_to_prewhere_if_final]); - - auto supported_prewhere_columns = storage->supportedPrewhereColumns(); - if (storage->canMoveConditionsToPrewhere() && optimize_move_to_prewhere - && (!supported_prewhere_columns || supported_prewhere_columns->contains(filter_info.column_name))) - { - if (!prewhere_info) - { - prewhere_info = std::make_shared(); - prewhere_info->prewhere_actions = std::move(filter_info.actions); - prewhere_info->prewhere_column_name = filter_info.column_name; - prewhere_info->remove_prewhere_column = filter_info.do_remove_column; - prewhere_info->need_filter = true; - } - else if (!prewhere_info->row_level_filter) - { - prewhere_info->row_level_filter = std::move(filter_info.actions); - prewhere_info->row_level_column_name = filter_info.column_name; - prewhere_info->need_filter = true; - } - else - { - where_filters.emplace_back(std::move(filter_info), std::move(description)); - } - - } - else - { - where_filters.emplace_back(std::move(filter_info), std::move(description)); - } - }; auto row_policy_filter_info = buildRowPolicyFilterIfNeeded(storage, table_expression_query_info, planner_context, used_row_policies); if (row_policy_filter_info) { table_expression_data.setRowLevelFilterActions(row_policy_filter_info->actions.clone()); - add_filter(*row_policy_filter_info, "Row-level security filter"); + /// TODO: Never put row-level security filter in WHERE clause for storages that do not support PREWHERE to avoid merging of filters. + if (storage->supportsPrewhere()) + row_level_filter = std::make_shared(std::move(*row_policy_filter_info)); + else + where_filters.emplace_back(std::move(*row_policy_filter_info), "Row-level security filter"); } if (query_context->canUseParallelReplicasCustomKey()) @@ -901,7 +877,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres if (settings[Setting::parallel_replicas_count] > 1) { if (auto parallel_replicas_custom_key_filter_info= buildCustomKeyFilterIfNeeded(storage, table_expression_query_info, planner_context)) - add_filter(*parallel_replicas_custom_key_filter_info, "Parallel replicas custom key filter"); + where_filters.emplace_back(std::move(*parallel_replicas_custom_key_filter_info), "Parallel replicas custom key filter"); } else if (auto * distributed = typeid_cast(storage.get()); distributed && query_context->canUseParallelReplicasCustomKeyForCluster(*distributed->getCluster())) @@ -916,10 +892,10 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres } const auto & table_expression_alias = table_expression->getOriginalAlias(); - if (auto additional_filters_info = buildAdditionalFiltersIfNeeded(storage, table_expression_alias, table_expression_query_info, planner_context)) + if (auto additional_filters_info = buildAdditionalFiltersIfNeeded(storage, table_expression_alias, table_expression_query_info, prewhere_info, planner_context)) { appendSetsFromActionsDAG(additional_filters_info->actions, useful_sets); - add_filter(*additional_filters_info, "additional filter"); + where_filters.emplace_back(std::move(*additional_filters_info), "additional filter"); } from_stage = storage->getQueryProcessingStage( diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp index ec5ed5b7d867..473edd641ba2 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp @@ -61,8 +61,7 @@ void optimizePrewhere(Stack & stack, QueryPlan::Nodes &) if (!storage.canMoveConditionsToPrewhere()) return; - const auto & storage_prewhere_info = source_step_with_filter->getPrewhereInfo(); - if (storage_prewhere_info) + if (source_step_with_filter->getPrewhereInfo()) return; /// TODO: We can also check for UnionStep, such as StorageBuffer and local distributed plans. @@ -108,11 +107,7 @@ void optimizePrewhere(Stack & stack, QueryPlan::Nodes &) if (optimize_result.prewhere_nodes.empty()) return; - PrewhereInfoPtr prewhere_info; - if (storage_prewhere_info) - prewhere_info = storage_prewhere_info->clone(); - else - prewhere_info = std::make_shared(); + PrewhereInfoPtr prewhere_info = std::make_shared(); prewhere_info->need_filter = true; prewhere_info->remove_prewhere_column = optimize_result.fully_moved_to_prewhere && filter_step->removesFilterColumn(); diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp index ce36c7bddb43..f8c8c83171f5 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp @@ -16,12 +16,11 @@ void optimizePrimaryKeyConditionAndLimit(const Stack & stack) return; const auto & storage_prewhere_info = source_step_with_filter->getPrewhereInfo(); + const auto & storage_row_level_filter = source_step_with_filter->getRowLevelFilter(); + if (storage_row_level_filter) + source_step_with_filter->addFilter(storage_row_level_filter->actions.clone(), storage_row_level_filter->column_name); if (storage_prewhere_info) - { source_step_with_filter->addFilter(storage_prewhere_info->prewhere_actions.clone(), storage_prewhere_info->prewhere_column_name); - if (storage_prewhere_info->row_level_filter) - source_step_with_filter->addFilter(storage_prewhere_info->row_level_filter->clone(), storage_prewhere_info->row_level_column_name); - } for (auto iter = stack.rbegin() + 1; iter != stack.rend(); ++iter) { diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp index 703b9ff7350d..af7af50c1d0a 100644 --- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp +++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp @@ -133,17 +133,16 @@ bool QueryDAG::buildImpl(QueryPlan::Node & node, ActionsDAG::NodeRawConstPtrs & IQueryPlanStep * step = node.step.get(); if (auto * reading = typeid_cast(step)) { + if (const auto & row_level_filter = reading->getRowLevelFilter()) + { + appendExpression(row_level_filter->actions); + if (const auto * filter_expression = findInOutputs(*dag, row_level_filter->column_name, row_level_filter->do_remove_column)) + filter_nodes.push_back(filter_expression); + else + return false; + } if (const auto & prewhere_info = reading->getPrewhereInfo()) { - if (prewhere_info->row_level_filter) - { - appendExpression(*prewhere_info->row_level_filter); - if (const auto * filter_expression = findInOutputs(*dag, prewhere_info->row_level_column_name, false)) - filter_nodes.push_back(filter_expression); - else - return false; - } - appendExpression(prewhere_info->prewhere_actions); if (const auto * filter_expression = findInOutputs(*dag, prewhere_info->prewhere_column_name, prewhere_info->remove_prewhere_column)) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 42a6b316a8bf..b2e526941cae 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -109,13 +109,14 @@ bool restoreDAGInputs(ActionsDAG & dag, const NameSet & inputs) return added; } -bool restorePrewhereInputs(PrewhereInfo & info, const NameSet & inputs) +bool restorePrewhereInputs(FilterDAGInfo * row_level_filter, PrewhereInfo * info, const NameSet & inputs) { bool added = false; - if (info.row_level_filter) - added = added || restoreDAGInputs(*info.row_level_filter, inputs); + if (row_level_filter) + added = added || restoreDAGInputs(row_level_filter->actions, inputs); - added = added || restoreDAGInputs(info.prewhere_actions, inputs); + if (info) + added = added || restoreDAGInputs(info->prewhere_actions, inputs); return added; } @@ -239,7 +240,8 @@ static SortDescription getSortDescriptionForOutputHeader( const Names & sorting_key_columns, const int sort_direction, InputOrderInfoPtr input_order_info, - PrewhereInfoPtr prewhere_info, + const FilterDAGInfoPtr & row_level_filter, + const PrewhereInfoPtr & prewhere_info, bool enable_vertical_final) { /// Updating sort description can be done after PREWHERE actions are applied to the header. @@ -258,16 +260,16 @@ static SortDescription getSortDescriptionForOutputHeader( column.name = original_node->result_name; } } + } - if (prewhere_info->row_level_filter) + if (row_level_filter) + { + FindOriginalNodeForOutputName original_column_finder(row_level_filter->actions); + for (auto & column : original_header) { - FindOriginalNodeForOutputName original_column_finder(*prewhere_info->row_level_filter); - for (auto & column : original_header) - { - const auto * original_node = original_column_finder.find(column.name); - if (original_node) - column.name = original_node->result_name; - } + const auto * original_node = original_column_finder.find(column.name); + if (original_node) + column.name = original_node->result_name; } } @@ -347,6 +349,7 @@ ReadFromMergeTree::ReadFromMergeTree( std::optional number_of_current_replica_) : SourceStepWithFilter(MergeTreeSelectProcessor::transformHeader( storage_snapshot_->getSampleBlockForColumns(all_column_names_), + query_info_.row_level_filter, query_info_.prewhere_info), all_column_names_, query_info_, storage_snapshot_, context_) , reader_settings(getMergeTreeReaderSettings(context_, query_info_)) , prepared_parts(std::move(parts_)) @@ -442,7 +445,8 @@ Pipe ReadFromMergeTree::readFromPoolParallelReplicas(RangesInDataParts parts_wit mutations_snapshot, shared_virtual_fields, storage_snapshot, - prewhere_info, + query_info.row_level_filter, + query_info.prewhere_info, actions_settings, reader_settings, required_columns, @@ -457,7 +461,13 @@ Pipe ReadFromMergeTree::readFromPoolParallelReplicas(RangesInDataParts parts_wit auto algorithm = std::make_unique(i); auto processor - = std::make_unique(pool, std::move(algorithm), prewhere_info, actions_settings, reader_settings); + = std::make_unique( + pool, + std::move(algorithm), + query_info.row_level_filter, + query_info.prewhere_info, + actions_settings, + reader_settings); auto source = std::make_shared(std::move(processor), data.getLogName()); pipes.emplace_back(std::move(source)); @@ -519,7 +529,8 @@ Pipe ReadFromMergeTree::readFromPool( mutations_snapshot, shared_virtual_fields, storage_snapshot, - prewhere_info, + query_info.row_level_filter, + query_info.prewhere_info, actions_settings, reader_settings, required_columns, @@ -534,7 +545,8 @@ Pipe ReadFromMergeTree::readFromPool( mutations_snapshot, shared_virtual_fields, storage_snapshot, - prewhere_info, + query_info.row_level_filter, + query_info.prewhere_info, actions_settings, reader_settings, required_columns, @@ -551,7 +563,13 @@ Pipe ReadFromMergeTree::readFromPool( auto algorithm = std::make_unique(i); auto processor - = std::make_unique(pool, std::move(algorithm), prewhere_info, actions_settings, reader_settings); + = std::make_unique( + pool, + std::move(algorithm), + query_info.row_level_filter, + query_info.prewhere_info, + actions_settings, + reader_settings); auto source = std::make_shared(std::move(processor), data.getLogName()); @@ -599,7 +617,8 @@ Pipe ReadFromMergeTree::readInOrder( shared_virtual_fields, has_limit_below_one_block, storage_snapshot, - prewhere_info, + query_info.row_level_filter, + query_info.prewhere_info, actions_settings, reader_settings, required_columns, @@ -616,7 +635,8 @@ Pipe ReadFromMergeTree::readInOrder( mutations_snapshot, shared_virtual_fields, storage_snapshot, - prewhere_info, + query_info.row_level_filter, + query_info.prewhere_info, actions_settings, reader_settings, required_columns, @@ -656,7 +676,13 @@ Pipe ReadFromMergeTree::readInOrder( algorithm = std::make_unique(i); auto processor - = std::make_unique(pool, std::move(algorithm), prewhere_info, actions_settings, reader_settings); + = std::make_unique( + pool, + std::move(algorithm), + query_info.row_level_filter, + query_info.prewhere_info, + actions_settings, + reader_settings); processor->addPartLevelToChunk(isQueryWithFinal()); @@ -866,7 +892,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_ fault(thread_local_rng) && !isQueryWithFinal() && data.merging_params.is_deleted_column.empty() && - !prewhere_info && + !query_info.row_level_filter && + !query_info.prewhere_info && !reader_settings.use_query_condition_cache) /// the query condition cache produces incorrect results with intersecting ranges { NameSet column_names_set(column_names.begin(), column_names.end()); @@ -972,13 +999,13 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( /// To fix this, we prohibit removing any input in prewhere actions. Instead, projection actions will be added after sorting. /// See 02354_read_in_order_prewhere.sql as an example. bool have_input_columns_removed_after_prewhere = false; - if (prewhere_info) + if (query_info.prewhere_info || query_info.row_level_filter) { NameSet sorting_columns; for (const auto & column : storage_snapshot->metadata->getSortingKey().expression->getRequiredColumnsWithTypes()) sorting_columns.insert(column.name); - have_input_columns_removed_after_prewhere = restorePrewhereInputs(*prewhere_info, sorting_columns); + have_input_columns_removed_after_prewhere = restorePrewhereInputs(query_info.row_level_filter.get(), query_info.prewhere_info.get(), sorting_columns); } /// Let's split ranges to avoid reading much data. @@ -1350,12 +1377,12 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( auto sorting_expr = storage_snapshot->metadata->getSortingKey().expression; - if (prewhere_info) + if (query_info.prewhere_info || query_info.row_level_filter) { NameSet sorting_columns; for (const auto & column : storage_snapshot->metadata->getSortingKey().expression->getRequiredColumnsWithTypes()) sorting_columns.insert(column.name); - restorePrewhereInputs(*prewhere_info, sorting_columns); + restorePrewhereInputs(query_info.row_level_filter.get(), query_info.prewhere_info.get(), sorting_columns); } for (size_t range_index = 0; range_index < parts_to_merge_ranges.size() - 1; ++range_index) @@ -1902,7 +1929,8 @@ void ReadFromMergeTree::updateSortDescription() storage_snapshot->metadata->getSortingKeyColumns(), getSortDirection(), query_info.input_order_info, - prewhere_info, + query_info.row_level_filter, + query_info.prewhere_info, enable_vertical_final); } @@ -1946,10 +1974,10 @@ bool ReadFromMergeTree::readsInOrder() const void ReadFromMergeTree::updatePrewhereInfo(const PrewhereInfoPtr & prewhere_info_value) { query_info.prewhere_info = prewhere_info_value; - prewhere_info = prewhere_info_value; output_header = MergeTreeSelectProcessor::transformHeader( storage_snapshot->getSampleBlockForColumns(all_column_names), + query_info.row_level_filter, prewhere_info_value); updateSortDescription(); @@ -2061,8 +2089,8 @@ Pipe ReadFromMergeTree::spreadMarkRanges( sampling_columns.insert(column); } - if (prewhere_info) - restorePrewhereInputs(*prewhere_info, sampling_columns); + if (query_info.prewhere_info || query_info.row_level_filter) + restorePrewhereInputs(query_info.row_level_filter.get(), query_info.prewhere_info.get(), sampling_columns); } if (final) @@ -2325,33 +2353,38 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const format_settings.out << prefix << "Granules: " << result.index_stats.back().num_granules_after << '\n'; } - if (prewhere_info) + if (query_info.prewhere_info || query_info.row_level_filter) { format_settings.out << prefix << "Prewhere info" << '\n'; - format_settings.out << prefix << "Need filter: " << prewhere_info->need_filter << '\n'; + if (query_info.prewhere_info) + format_settings.out << prefix << "Need filter: " << query_info.prewhere_info->need_filter << '\n'; prefix.push_back(format_settings.indent_char); prefix.push_back(format_settings.indent_char); + } - { - format_settings.out << prefix << "Prewhere filter" << '\n'; - format_settings.out << prefix << "Prewhere filter column: " << prewhere_info->prewhere_column_name; - if (prewhere_info->remove_prewhere_column) - format_settings.out << " (removed)"; - format_settings.out << '\n'; + if (query_info.prewhere_info) + { + format_settings.out << prefix << "Prewhere filter" << '\n'; + format_settings.out << prefix << "Prewhere filter column: " << query_info.prewhere_info->prewhere_column_name; + if (query_info.prewhere_info->remove_prewhere_column) + format_settings.out << " (removed)"; + format_settings.out << '\n'; - auto expression = std::make_shared(prewhere_info->prewhere_actions.clone()); - expression->describeActions(format_settings.out, prefix); - } + auto expression = std::make_shared(query_info.prewhere_info->prewhere_actions.clone()); + expression->describeActions(format_settings.out, prefix); + } - if (prewhere_info->row_level_filter) - { - format_settings.out << prefix << "Row level filter" << '\n'; - format_settings.out << prefix << "Row level filter column: " << prewhere_info->row_level_column_name << '\n'; + if (query_info.row_level_filter) + { + format_settings.out << prefix << "Row level filter" << '\n'; + format_settings.out << prefix << "Row level filter column: " << query_info.row_level_filter->column_name; + if (query_info.row_level_filter->do_remove_column) + format_settings.out << " (removed)"; + format_settings.out << '\n'; - auto expression = std::make_shared(prewhere_info->row_level_filter->clone()); - expression->describeActions(format_settings.out, prefix); - } + auto expression = std::make_shared(query_info.row_level_filter->actions.clone()); + expression->describeActions(format_settings.out, prefix); } if (virtual_row_conversion) @@ -2371,33 +2404,37 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const map.add("Granules", result.index_stats.back().num_granules_after); } - if (prewhere_info) + std::unique_ptr prewhere_info_map; + if (query_info.prewhere_info || query_info.row_level_filter) { - std::unique_ptr prewhere_info_map = std::make_unique(); - prewhere_info_map->add("Need filter", prewhere_info->need_filter); + prewhere_info_map = std::make_unique(); + if (query_info.prewhere_info) + prewhere_info_map->add("Need filter", query_info.prewhere_info->need_filter); + } - { - std::unique_ptr prewhere_filter_map = std::make_unique(); - prewhere_filter_map->add("Prewhere filter column", prewhere_info->prewhere_column_name); - prewhere_filter_map->add("Prewhere filter remove filter column", prewhere_info->remove_prewhere_column); - auto expression = std::make_shared(prewhere_info->prewhere_actions.clone()); - prewhere_filter_map->add("Prewhere filter expression", expression->toTree()); + if (query_info.prewhere_info) + { + std::unique_ptr prewhere_filter_map = std::make_unique(); + prewhere_filter_map->add("Prewhere filter column", query_info.prewhere_info->prewhere_column_name); + prewhere_filter_map->add("Prewhere filter remove filter column", query_info.prewhere_info->remove_prewhere_column); + auto expression = std::make_shared(query_info.prewhere_info->prewhere_actions.clone()); + prewhere_filter_map->add("Prewhere filter expression", expression->toTree()); - prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map)); - } + prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map)); + } - if (prewhere_info->row_level_filter) - { - std::unique_ptr row_level_filter_map = std::make_unique(); - row_level_filter_map->add("Row level filter column", prewhere_info->row_level_column_name); - auto expression = std::make_shared(prewhere_info->row_level_filter->clone()); - row_level_filter_map->add("Row level filter expression", expression->toTree()); + if (query_info.row_level_filter) + { + std::unique_ptr row_level_filter_map = std::make_unique(); + row_level_filter_map->add("Row level filter column", query_info.row_level_filter->column_name); + auto expression = std::make_shared(query_info.row_level_filter->actions.clone()); + row_level_filter_map->add("Row level filter expression", expression->toTree()); - prewhere_info_map->add("Row level filter", std::move(row_level_filter_map)); - } + prewhere_info_map->add("Row level filter", std::move(row_level_filter_map)); + } + if (prewhere_info_map) map.add("Prewhere info", std::move(prewhere_info_map)); - } if (virtual_row_conversion) map.add("Virtual row conversions", virtual_row_conversion->toTree()); diff --git a/src/Processors/QueryPlan/SourceStepWithFilter.cpp b/src/Processors/QueryPlan/SourceStepWithFilter.cpp index 46df96ba6ac4..21555d69c017 100644 --- a/src/Processors/QueryPlan/SourceStepWithFilter.cpp +++ b/src/Processors/QueryPlan/SourceStepWithFilter.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -16,25 +17,26 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER; } -Block SourceStepWithFilter::applyPrewhereActions(Block block, const PrewhereInfoPtr & prewhere_info) +Block SourceStepWithFilter::applyPrewhereActions(Block block, const FilterDAGInfoPtr & row_level_filter, const PrewhereInfoPtr & prewhere_info) { - if (prewhere_info) + if (row_level_filter) { - if (prewhere_info->row_level_filter) + block = row_level_filter->actions.updateHeader(block); + auto & row_level_column = block.getByName(row_level_filter->column_name); + if (!row_level_column.type->canBeUsedInBooleanContext()) { - block = prewhere_info->row_level_filter->updateHeader(block); - auto & row_level_column = block.getByName(prewhere_info->row_level_column_name); - if (!row_level_column.type->canBeUsedInBooleanContext()) - { - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER, - "Invalid type for filter in PREWHERE: {}", - row_level_column.type->getName()); - } - - block.erase(prewhere_info->row_level_column_name); + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER, + "Invalid type for filter in PREWHERE: {}", + row_level_column.type->getName()); } + if (row_level_filter->do_remove_column) + block.erase(row_level_filter->column_name); + } + + if (prewhere_info) + { { block = prewhere_info->prewhere_actions.updateHeader(block); @@ -91,73 +93,81 @@ void SourceStepWithFilter::applyFilters(ActionDAGNodes added_filter_nodes) void SourceStepWithFilter::updatePrewhereInfo(const PrewhereInfoPtr & prewhere_info_value) { query_info.prewhere_info = prewhere_info_value; - prewhere_info = prewhere_info_value; - output_header = applyPrewhereActions(*output_header, prewhere_info); + output_header = applyPrewhereActions(*output_header, query_info.row_level_filter, query_info.prewhere_info); } void SourceStepWithFilter::describeActions(FormatSettings & format_settings) const { std::string prefix(format_settings.offset, format_settings.indent_char); - if (prewhere_info) + if (query_info.prewhere_info || query_info.row_level_filter) { format_settings.out << prefix << "Prewhere info" << '\n'; - format_settings.out << prefix << "Need filter: " << prewhere_info->need_filter << '\n'; + if (query_info.prewhere_info) + format_settings.out << prefix << "Need filter: " << query_info.prewhere_info->need_filter << '\n'; prefix.push_back(format_settings.indent_char); prefix.push_back(format_settings.indent_char); + } - { - format_settings.out << prefix << "Prewhere filter" << '\n'; - format_settings.out << prefix << "Prewhere filter column: " << prewhere_info->prewhere_column_name; - if (prewhere_info->remove_prewhere_column) - format_settings.out << " (removed)"; - format_settings.out << '\n'; - - auto expression = std::make_shared(prewhere_info->prewhere_actions.clone()); - expression->describeActions(format_settings.out, prefix); - } - - if (prewhere_info->row_level_filter) - { - format_settings.out << prefix << "Row level filter" << '\n'; - format_settings.out << prefix << "Row level filter column: " << prewhere_info->row_level_column_name << '\n'; + if (query_info.prewhere_info) + { + format_settings.out << prefix << "Prewhere filter" << '\n'; + format_settings.out << prefix << "Prewhere filter column: " << query_info.prewhere_info->prewhere_column_name; + if (query_info.prewhere_info->remove_prewhere_column) + format_settings.out << " (removed)"; + format_settings.out << '\n'; + + auto expression = std::make_shared(query_info.prewhere_info->prewhere_actions.clone()); + expression->describeActions(format_settings.out, prefix); + } - auto expression = std::make_shared(prewhere_info->row_level_filter->clone()); - expression->describeActions(format_settings.out, prefix); - } + if (query_info.row_level_filter) + { + format_settings.out << prefix << "Row level filter" << '\n'; + format_settings.out << prefix << "Row level filter column: " << query_info.row_level_filter->column_name; + if (query_info.row_level_filter->do_remove_column) + format_settings.out << " (removed)"; + format_settings.out << '\n'; + + auto expression = std::make_shared(query_info.row_level_filter->actions.clone()); + expression->describeActions(format_settings.out, prefix); } } void SourceStepWithFilter::describeActions(JSONBuilder::JSONMap & map) const { - if (prewhere_info) + std::unique_ptr prewhere_info_map; + if (query_info.prewhere_info || query_info.row_level_filter) { - std::unique_ptr prewhere_info_map = std::make_unique(); - prewhere_info_map->add("Need filter", prewhere_info->need_filter); + prewhere_info_map = std::make_unique(); + if (query_info.prewhere_info) + prewhere_info_map->add("Need filter", query_info.prewhere_info->need_filter); + } - { - std::unique_ptr prewhere_filter_map = std::make_unique(); - prewhere_filter_map->add("Prewhere filter column", prewhere_info->prewhere_column_name); - prewhere_filter_map->add("Prewhere filter remove filter column", prewhere_info->remove_prewhere_column); - auto expression = std::make_shared(prewhere_info->prewhere_actions.clone()); - prewhere_filter_map->add("Prewhere filter expression", expression->toTree()); + if (query_info.prewhere_info) + { + std::unique_ptr prewhere_filter_map = std::make_unique(); + prewhere_filter_map->add("Prewhere filter column", query_info.prewhere_info->prewhere_column_name); + prewhere_filter_map->add("Prewhere filter remove filter column", query_info.prewhere_info->remove_prewhere_column); + auto expression = std::make_shared(query_info.prewhere_info->prewhere_actions.clone()); + prewhere_filter_map->add("Prewhere filter expression", expression->toTree()); - prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map)); - } + prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map)); + } - if (prewhere_info->row_level_filter) - { - std::unique_ptr row_level_filter_map = std::make_unique(); - row_level_filter_map->add("Row level filter column", prewhere_info->row_level_column_name); - auto expression = std::make_shared(prewhere_info->row_level_filter->clone()); - row_level_filter_map->add("Row level filter expression", expression->toTree()); + if (query_info.row_level_filter) + { + std::unique_ptr row_level_filter_map = std::make_unique(); + row_level_filter_map->add("Row level filter column", query_info.row_level_filter->column_name); + auto expression = std::make_shared(query_info.row_level_filter->actions.clone()); + row_level_filter_map->add("Row level filter expression", expression->toTree()); - prewhere_info_map->add("Row level filter", std::move(row_level_filter_map)); - } + prewhere_info_map->add("Row level filter", std::move(row_level_filter_map)); + } + if (prewhere_info_map) map.add("Prewhere info", std::move(prewhere_info_map)); - } } } diff --git a/src/Processors/QueryPlan/SourceStepWithFilter.h b/src/Processors/QueryPlan/SourceStepWithFilter.h index 4718ad78947f..294e2d3396b0 100644 --- a/src/Processors/QueryPlan/SourceStepWithFilter.h +++ b/src/Processors/QueryPlan/SourceStepWithFilter.h @@ -41,6 +41,7 @@ class SourceStepWithFilterBase : public ISourceStep } virtual void applyFilters(ActionDAGNodes added_filter_nodes); + virtual FilterDAGInfoPtr getRowLevelFilter() const { return nullptr; } virtual PrewhereInfoPtr getPrewhereInfo() const { return nullptr; } const std::optional & getFilterActionsDAG() const { return filter_actions_dag; } @@ -75,14 +76,14 @@ class SourceStepWithFilter : public SourceStepWithFilterBase : SourceStepWithFilterBase(std::move(output_header_)) , required_source_columns(column_names_) , query_info(query_info_) - , prewhere_info(query_info.prewhere_info) , storage_snapshot(storage_snapshot_) , context(context_) { } const SelectQueryInfo & getQueryInfo() const { return query_info; } - PrewhereInfoPtr getPrewhereInfo() const override { return prewhere_info; } + FilterDAGInfoPtr getRowLevelFilter() const override { return query_info.row_level_filter; } + PrewhereInfoPtr getPrewhereInfo() const override { return query_info.prewhere_info; } ContextPtr getContext() const { return context; } const StorageSnapshotPtr & getStorageSnapshot() const { return storage_snapshot; } @@ -98,12 +99,11 @@ class SourceStepWithFilter : public SourceStepWithFilterBase void describeActions(JSONBuilder::JSONMap & map) const override; - static Block applyPrewhereActions(Block block, const PrewhereInfoPtr & prewhere_info); + static Block applyPrewhereActions(Block block, const FilterDAGInfoPtr & row_level_filter, const PrewhereInfoPtr & prewhere_info); protected: Names required_source_columns; SelectQueryInfo query_info; - PrewhereInfoPtr prewhere_info; StorageSnapshotPtr storage_snapshot; ContextPtr context; }; diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 7b25434aa827..2722b793b4ca 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -395,11 +395,6 @@ std::string PrewhereInfo::dump() const WriteBufferFromOwnString ss; ss << "PrewhereDagInfo\n"; - if (row_level_filter) - { - ss << "row_level_filter " << row_level_filter->dumpDAG() << "\n"; - } - { ss << "prewhere_actions " << prewhere_actions.dumpDAG() << "\n"; } diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 82b44fad5ab2..166d2caf4ad1 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -281,6 +281,7 @@ MergeTreeReadTaskColumns getReadTaskColumns( const IMergeTreeDataPartInfoForReader & data_part_info_for_reader, const StorageSnapshotPtr & storage_snapshot, const Names & required_columns, + const FilterDAGInfoPtr & row_level_filter, const PrewhereInfoPtr & prewhere_info, const PrewhereExprSteps & mutation_steps, const ExpressionActionsSettings & actions_settings, @@ -366,9 +367,10 @@ MergeTreeReadTaskColumns getReadTaskColumns( for (const auto & step : mutation_steps) add_step(*step); - if (prewhere_info) + if (prewhere_info || row_level_filter) { auto prewhere_actions = MergeTreeSelectProcessor::getPrewhereActions( + row_level_filter, prewhere_info, actions_settings, reader_settings.enable_multiple_prewhere_read_steps, reader_settings.force_short_circuit_execution); @@ -399,6 +401,7 @@ MergeTreeReadTaskColumns getReadTaskColumnsForMerge( data_part_info_for_reader, storage_snapshot, required_columns, + /*row_level_filter=*/ nullptr, /*prewhere_info=*/ nullptr, mutation_steps, /*actions_settings=*/ {}, diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.h b/src/Storages/MergeTree/MergeTreeBlockReadUtils.h index 26d915d164f2..a1c4f8f25c8f 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.h +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.h @@ -18,6 +18,7 @@ MergeTreeReadTaskColumns getReadTaskColumns( const IMergeTreeDataPartInfoForReader & data_part_info_for_reader, const StorageSnapshotPtr & storage_snapshot, const Names & required_columns, + const FilterDAGInfoPtr & row_level_filter, const PrewhereInfoPtr & prewhere_info, const PrewhereExprSteps & mutation_steps, const ExpressionActionsSettings & actions_settings, diff --git a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp index 5ce079f48359..b44a3ffc02a4 100644 --- a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp @@ -97,6 +97,7 @@ MergeTreePrefetchedReadPool::MergeTreePrefetchedReadPool( MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, + const FilterDAGInfoPtr & row_level_filter_, const PrewhereInfoPtr & prewhere_info_, const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, @@ -109,6 +110,7 @@ MergeTreePrefetchedReadPool::MergeTreePrefetchedReadPool( std::move(mutations_snapshot_), std::move(shared_virtual_fields_), storage_snapshot_, + row_level_filter_, prewhere_info_, actions_settings_, reader_settings_, diff --git a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h index b94d4ea113ab..129a1b0bc9c8 100644 --- a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h +++ b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.h @@ -22,6 +22,7 @@ class MergeTreePrefetchedReadPool : public MergeTreeReadPoolBase MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, + const FilterDAGInfoPtr & row_level_filter_, const PrewhereInfoPtr & prewhere_info_, const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index 54498b4f1e10..ee04a0dde134 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -40,6 +40,7 @@ MergeTreeReadPool::MergeTreeReadPool( MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, + const FilterDAGInfoPtr & row_level_filter_, const PrewhereInfoPtr & prewhere_info_, const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, @@ -52,6 +53,7 @@ MergeTreeReadPool::MergeTreeReadPool( std::move(mutations_snapshot_), std::move(shared_virtual_fields_), storage_snapshot_, + row_level_filter_, prewhere_info_, actions_settings_, reader_settings_, diff --git a/src/Storages/MergeTree/MergeTreeReadPool.h b/src/Storages/MergeTree/MergeTreeReadPool.h index 1b55284c592b..8891158d2c14 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.h +++ b/src/Storages/MergeTree/MergeTreeReadPool.h @@ -29,6 +29,7 @@ class MergeTreeReadPool : public MergeTreeReadPoolBase MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, + const FilterDAGInfoPtr & row_level_filter_, const PrewhereInfoPtr & prewhere_info_, const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, diff --git a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp index cc44256e6395..07baba6278fc 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp @@ -25,6 +25,7 @@ MergeTreeReadPoolBase::MergeTreeReadPoolBase( MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, + const FilterDAGInfoPtr & row_level_filter_, const PrewhereInfoPtr & prewhere_info_, const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, @@ -37,6 +38,7 @@ MergeTreeReadPoolBase::MergeTreeReadPoolBase( , mutations_snapshot(std::move(mutations_snapshot_)) , shared_virtual_fields(std::move(shared_virtual_fields_)) , storage_snapshot(storage_snapshot_) + , row_level_filter(row_level_filter_) , prewhere_info(prewhere_info_) , actions_settings(actions_settings_) , reader_settings(reader_settings_) @@ -179,6 +181,7 @@ void MergeTreeReadPoolBase::fillPerPartInfos(const Settings & settings) part_info, storage_snapshot, column_names, + row_level_filter, prewhere_info, read_task_info.mutation_steps, actions_settings, diff --git a/src/Storages/MergeTree/MergeTreeReadPoolBase.h b/src/Storages/MergeTree/MergeTreeReadPoolBase.h index 19b261564337..50d5444ebb1e 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolBase.h +++ b/src/Storages/MergeTree/MergeTreeReadPoolBase.h @@ -28,6 +28,7 @@ class MergeTreeReadPoolBase : public IMergeTreeReadPool, protected WithContext MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, + const FilterDAGInfoPtr & row_level_filter_, const PrewhereInfoPtr & prewhere_info_, const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, @@ -44,6 +45,7 @@ class MergeTreeReadPoolBase : public IMergeTreeReadPool, protected WithContext const MutationsSnapshotPtr mutations_snapshot; const VirtualFields shared_virtual_fields; const StorageSnapshotPtr storage_snapshot; + const FilterDAGInfoPtr row_level_filter; const PrewhereInfoPtr prewhere_info; const ExpressionActionsSettings actions_settings; const MergeTreeReaderSettings reader_settings; diff --git a/src/Storages/MergeTree/MergeTreeReadPoolInOrder.cpp b/src/Storages/MergeTree/MergeTreeReadPoolInOrder.cpp index c4244ecd9820..1aa6b5eedc19 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolInOrder.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPoolInOrder.cpp @@ -15,6 +15,7 @@ MergeTreeReadPoolInOrder::MergeTreeReadPoolInOrder( MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, + const FilterDAGInfoPtr & row_level_filter_, const PrewhereInfoPtr & prewhere_info_, const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, @@ -27,6 +28,7 @@ MergeTreeReadPoolInOrder::MergeTreeReadPoolInOrder( std::move(mutations_snapshot_), std::move(shared_virtual_fields_), storage_snapshot_, + row_level_filter_, prewhere_info_, actions_settings_, reader_settings_, diff --git a/src/Storages/MergeTree/MergeTreeReadPoolInOrder.h b/src/Storages/MergeTree/MergeTreeReadPoolInOrder.h index 41f3ab1061c1..35c968acb28c 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolInOrder.h +++ b/src/Storages/MergeTree/MergeTreeReadPoolInOrder.h @@ -14,6 +14,7 @@ class MergeTreeReadPoolInOrder : public MergeTreeReadPoolBase MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, + const FilterDAGInfoPtr & row_level_filter_, const PrewhereInfoPtr & prewhere_info_, const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp index 8f06fc312c2f..a376e2237700 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp @@ -107,6 +107,7 @@ MergeTreeReadPoolParallelReplicas::MergeTreeReadPoolParallelReplicas( MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, + const FilterDAGInfoPtr & row_level_filter_, const PrewhereInfoPtr & prewhere_info_, const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, @@ -119,6 +120,7 @@ MergeTreeReadPoolParallelReplicas::MergeTreeReadPoolParallelReplicas( std::move(mutations_snapshot_), std::move(shared_virtual_fields_), storage_snapshot_, + row_level_filter_, prewhere_info_, actions_settings_, reader_settings_, diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h index 63816340eb1d..b5fbd4efc1ba 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h +++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h @@ -14,6 +14,7 @@ class MergeTreeReadPoolParallelReplicas : public MergeTreeReadPoolBase MutationsSnapshotPtr mutations_snapshot_, VirtualFields shared_virtual_fields_, const StorageSnapshotPtr & storage_snapshot_, + const FilterDAGInfoPtr & row_level_filter_, const PrewhereInfoPtr & prewhere_info_, const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp index 551ae3dce4ff..7b14cc1b40bf 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp @@ -28,6 +28,7 @@ MergeTreeReadPoolParallelReplicasInOrder::MergeTreeReadPoolParallelReplicasInOrd VirtualFields shared_virtual_fields_, bool has_limit_below_one_block_, const StorageSnapshotPtr & storage_snapshot_, + const FilterDAGInfoPtr & row_level_filter_, const PrewhereInfoPtr & prewhere_info_, const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, @@ -40,6 +41,7 @@ MergeTreeReadPoolParallelReplicasInOrder::MergeTreeReadPoolParallelReplicasInOrd std::move(mutations_snapshot_), std::move(shared_virtual_fields_), storage_snapshot_, + row_level_filter_, prewhere_info_, actions_settings_, reader_settings_, diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.h b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.h index c557f579c87d..aa00a14a0161 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.h +++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.h @@ -16,6 +16,7 @@ class MergeTreeReadPoolParallelReplicasInOrder : public MergeTreeReadPoolBase VirtualFields shared_virtual_fields_, bool has_limit_below_one_block_, const StorageSnapshotPtr & storage_snapshot_, + const FilterDAGInfoPtr & row_level_filter_, const PrewhereInfoPtr & prewhere_info_, const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 9021c48ad972..bc219cded808 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -88,16 +88,23 @@ std::optional ParallelReadingExtension::sendReadRequest( MergeTreeSelectProcessor::MergeTreeSelectProcessor( MergeTreeReadPoolPtr pool_, MergeTreeSelectAlgorithmPtr algorithm_, + const FilterDAGInfoPtr & row_level_filter_, const PrewhereInfoPtr & prewhere_info_, const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_) : pool(std::move(pool_)) , algorithm(std::move(algorithm_)) + , row_level_filter(row_level_filter_) , prewhere_info(prewhere_info_) , actions_settings(actions_settings_) - , prewhere_actions(getPrewhereActions(prewhere_info, actions_settings, reader_settings_.enable_multiple_prewhere_read_steps, reader_settings_.force_short_circuit_execution)) + , prewhere_actions(getPrewhereActions( + row_level_filter, + prewhere_info, + actions_settings, + reader_settings_.enable_multiple_prewhere_read_steps, + reader_settings_.force_short_circuit_execution)) , reader_settings(reader_settings_) - , result_header(transformHeader(pool->getHeader(), prewhere_info)) + , result_header(transformHeader(pool->getHeader(), row_level_filter, prewhere_info)) { bool has_prewhere_actions_steps = !prewhere_actions.steps.empty(); if (has_prewhere_actions_steps) @@ -117,28 +124,32 @@ String MergeTreeSelectProcessor::getName() const bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere, bool force_short_circuit_execution); -PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps, bool force_short_circuit_execution) +PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions( + const FilterDAGInfoPtr & row_level_filter, + const PrewhereInfoPtr & prewhere_info, + const ExpressionActionsSettings & actions_settings, + bool enable_multiple_prewhere_read_steps, + bool force_short_circuit_execution) { PrewhereExprInfo prewhere_actions; - if (prewhere_info) + if (row_level_filter) { - if (prewhere_info->row_level_filter) + PrewhereExprStep row_level_filter_step { - PrewhereExprStep row_level_filter_step - { - .type = PrewhereExprStep::Filter, - .actions = std::make_shared(prewhere_info->row_level_filter->clone(), actions_settings), - .filter_column_name = prewhere_info->row_level_column_name, - .remove_filter_column = true, - .need_filter = true, - .perform_alter_conversions = true, - }; - - prewhere_actions.steps.emplace_back(std::make_shared(std::move(row_level_filter_step))); - } + .type = PrewhereExprStep::Filter, + .actions = std::make_shared(row_level_filter->actions.clone(), actions_settings), + .filter_column_name = row_level_filter->column_name, + .remove_filter_column = row_level_filter->do_remove_column, + .need_filter = true, + .perform_alter_conversions = true, + }; + + prewhere_actions.steps.emplace_back(std::make_shared(std::move(row_level_filter_step))); + } - if (!enable_multiple_prewhere_read_steps || - !tryBuildPrewhereSteps(prewhere_info, actions_settings, prewhere_actions, force_short_circuit_execution)) + if (prewhere_info && + (!enable_multiple_prewhere_read_steps || + !tryBuildPrewhereSteps(prewhere_info, actions_settings, prewhere_actions, force_short_circuit_execution))) { PrewhereExprStep prewhere_step { @@ -152,7 +163,6 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr pr prewhere_actions.steps.emplace_back(std::make_shared(std::move(prewhere_step))); } - } return prewhere_actions; } @@ -260,9 +270,9 @@ void MergeTreeSelectProcessor::initializeReadersChain() task->initializeReadersChain(all_prewhere_actions, read_steps_performance_counters); } -Block MergeTreeSelectProcessor::transformHeader(Block block, const PrewhereInfoPtr & prewhere_info) +Block MergeTreeSelectProcessor::transformHeader(Block block, const FilterDAGInfoPtr & row_level_filter, const PrewhereInfoPtr & prewhere_info) { - return SourceStepWithFilter::applyPrewhereActions(std::move(block), prewhere_info); + return SourceStepWithFilter::applyPrewhereActions(std::move(block), row_level_filter, prewhere_info); } static String dumpStatistics(const ReadStepsPerformanceCounters & counters) diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 15974c7f916e..30697fd39a8f 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -57,13 +57,14 @@ class MergeTreeSelectProcessor : private boost::noncopyable MergeTreeSelectProcessor( MergeTreeReadPoolPtr pool_, MergeTreeSelectAlgorithmPtr algorithm_, + const FilterDAGInfoPtr & row_level_filter_, const PrewhereInfoPtr & prewhere_info_, const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_); String getName() const; - static Block transformHeader(Block block, const PrewhereInfoPtr & prewhere_info); + static Block transformHeader(Block block, const FilterDAGInfoPtr & row_level_filter, const PrewhereInfoPtr & prewhere_info); Block getHeader() const { return result_header; } ChunkAndProgress read(); @@ -73,7 +74,8 @@ class MergeTreeSelectProcessor : private boost::noncopyable const MergeTreeReaderSettings & getSettings() const { return reader_settings; } static PrewhereExprInfo getPrewhereActions( - PrewhereInfoPtr prewhere_info, + const FilterDAGInfoPtr & row_level_filter, + const PrewhereInfoPtr & prewhere_info, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps, bool force_short_circuit_execution); @@ -89,6 +91,7 @@ class MergeTreeSelectProcessor : private boost::noncopyable const MergeTreeReadPoolPtr pool; const MergeTreeSelectAlgorithmPtr algorithm; + const FilterDAGInfoPtr row_level_filter; const PrewhereInfoPtr prewhere_info; const ExpressionActionsSettings actions_settings; const PrewhereExprInfo prewhere_actions; diff --git a/src/Storages/SelectQueryInfo.cpp b/src/Storages/SelectQueryInfo.cpp index afba148189e2..f4b3fc2df56d 100644 --- a/src/Storages/SelectQueryInfo.cpp +++ b/src/Storages/SelectQueryInfo.cpp @@ -39,4 +39,16 @@ std::unordered_map SelectQueryInfo::buildNod return node_name_to_input_node_column; } +PrewhereInfo PrewhereInfo::clone() const +{ + PrewhereInfo prewhere_info; + + prewhere_info.prewhere_actions = prewhere_actions.clone(); + + prewhere_info.prewhere_column_name = prewhere_column_name; + prewhere_info.remove_prewhere_column = remove_prewhere_column; + prewhere_info.need_filter = need_filter; + + return prewhere_info; +} } diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 07c84f0cf2ce..4a531551a5d4 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -41,16 +41,11 @@ using PreparedSetsPtr = std::shared_ptr; struct PrewhereInfo { - /// Actions for row level security filter. Applied separately before prewhere_actions. - /// This actions are separate because prewhere condition should not be executed over filtered rows. - std::optional row_level_filter; /// Actions which are executed on block in order to get filter column for prewhere step. ActionsDAG prewhere_actions; - String row_level_column_name; String prewhere_column_name; bool remove_prewhere_column = false; bool need_filter = false; - bool generated_by_optimizer = false; PrewhereInfo() = default; explicit PrewhereInfo(ActionsDAG prewhere_actions_, String prewhere_column_name_) @@ -58,23 +53,7 @@ struct PrewhereInfo std::string dump() const; - PrewhereInfoPtr clone() const - { - PrewhereInfoPtr prewhere_info = std::make_shared(); - - if (row_level_filter) - prewhere_info->row_level_filter = row_level_filter->clone(); - - prewhere_info->prewhere_actions = prewhere_actions.clone(); - - prewhere_info->row_level_column_name = row_level_column_name; - prewhere_info->prewhere_column_name = prewhere_column_name; - prewhere_info->remove_prewhere_column = remove_prewhere_column; - prewhere_info->need_filter = need_filter; - prewhere_info->generated_by_optimizer = generated_by_optimizer; - - return prewhere_info; - } + PrewhereInfo clone() const; }; /// Same as FilterInfo, but with ActionsDAG. @@ -197,6 +176,10 @@ struct SelectQueryInfo bool has_window = false; bool has_order_by = false; bool need_aggregate = false; + + /// Actions for row level security filter. Applied separately before prewhere. + /// These actions are separate because prewhere condition should not be executed over filtered rows. + FilterDAGInfoPtr row_level_filter; PrewhereInfoPtr prewhere_info; /// If query has aggregate functions diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 22d6ad635a31..4cf1fc71eb3d 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -348,27 +348,33 @@ void StorageBuffer::read( else { auto src_table_query_info = query_info; - if (src_table_query_info.prewhere_info) + ActionsDAG converting_dag; + if (src_table_query_info.prewhere_info || src_table_query_info.row_level_filter) { - src_table_query_info.prewhere_info = src_table_query_info.prewhere_info->clone(); - - auto actions_dag = ActionsDAG::makeConvertingActions( - header_after_adding_defaults.getColumnsWithTypeAndName(), - header.getColumnsWithTypeAndName(), - ActionsDAG::MatchColumnsMode::Name); - - if (src_table_query_info.prewhere_info->row_level_filter) - { - src_table_query_info.prewhere_info->row_level_filter = ActionsDAG::merge( - actions_dag.clone(), - std::move(*src_table_query_info.prewhere_info->row_level_filter)); + converting_dag = ActionsDAG::makeConvertingActions( + header_after_adding_defaults.getColumnsWithTypeAndName(), + header.getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Name); + } - src_table_query_info.prewhere_info->row_level_filter->removeUnusedActions(); - } + if (src_table_query_info.row_level_filter) + { + auto row_level_filter = std::make_shared(); + row_level_filter->column_name = src_table_query_info.row_level_filter->column_name; + row_level_filter->do_remove_column = src_table_query_info.row_level_filter->do_remove_column; + row_level_filter->actions = ActionsDAG::merge( + converting_dag.clone(), + src_table_query_info.row_level_filter->actions.clone()); + row_level_filter->actions.removeUnusedActions(); + src_table_query_info.row_level_filter = std::move(row_level_filter); + } + if (src_table_query_info.prewhere_info) + { + src_table_query_info.prewhere_info = std::make_shared(src_table_query_info.prewhere_info->clone()); { src_table_query_info.prewhere_info->prewhere_actions = ActionsDAG::merge( - actions_dag.clone(), + converting_dag.clone(), std::move(src_table_query_info.prewhere_info->prewhere_actions)); src_table_query_info.prewhere_info->prewhere_actions.removeUnusedActions(); @@ -473,23 +479,24 @@ void StorageBuffer::read( } else { - if (query_info.prewhere_info) + if (query_info.row_level_filter) { ExpressionActionsSettings actions_settings(local_context); + auto actions = std::make_shared(query_info.row_level_filter->actions.clone(), actions_settings); - if (query_info.prewhere_info->row_level_filter) + pipe_from_buffers.addSimpleTransform([&](const Block & header) { - auto actions = std::make_shared(query_info.prewhere_info->row_level_filter->clone(), actions_settings); - pipe_from_buffers.addSimpleTransform([&](const Block & header) - { - return std::make_shared( - header, - actions, - query_info.prewhere_info->row_level_column_name, - false); - }); - } + return std::make_shared( + header, + actions, + query_info.row_level_filter->column_name, + query_info.row_level_filter->do_remove_column); + }); + } + if (query_info.prewhere_info) + { + ExpressionActionsSettings actions_settings(local_context); auto actions = std::make_shared(query_info.prewhere_info->prewhere_actions.clone(), actions_settings); pipe_from_buffers.addSimpleTransform([&](const Block & header) { diff --git a/src/Storages/StorageDummy.cpp b/src/Storages/StorageDummy.cpp index 0f7c1047fbc0..52c4feeeec5e 100644 --- a/src/Storages/StorageDummy.cpp +++ b/src/Storages/StorageDummy.cpp @@ -57,7 +57,7 @@ ReadFromDummy::ReadFromDummy( const ContextPtr & context_, const StorageDummy & storage_) : SourceStepWithFilter(SourceStepWithFilter::applyPrewhereActions( - storage_snapshot_->getSampleBlockForColumns(column_names_), query_info_.prewhere_info), + storage_snapshot_->getSampleBlockForColumns(column_names_), query_info_.row_level_filter, query_info_.prewhere_info), column_names_, query_info_, storage_snapshot_, diff --git a/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.reference b/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.reference index 4a4e338438b2..669d0fe248d3 100644 --- a/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.reference +++ b/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.reference @@ -19,7 +19,7 @@ Positions: 0 1 FUNCTION equals(id : 0, 5 :: 1) -> equals(id, 5) UInt8 : 2 Positions: 2 0 Row level filter - Row level filter column: greaterOrEquals(id, 5) + Row level filter column: greaterOrEquals(id, 5) (removed) Actions: INPUT : 0 -> id UInt64 : 0 COLUMN Const(UInt8) -> 5 UInt8 : 1 FUNCTION greaterOrEquals(id : 0, 5 :: 1) -> greaterOrEquals(id, 5) UInt8 : 2 @@ -49,7 +49,7 @@ Positions: 1 2 FUNCTION equals(id : 0, 5_UInt8 :: 1) -> equals(id, 5_UInt8) UInt8 : 2 Positions: 2 0 Row level filter - Row level filter column: greaterOrEquals(id, 5_UInt8) + Row level filter column: greaterOrEquals(id, 5_UInt8) (removed) Actions: INPUT : 0 -> id UInt64 : 0 COLUMN Const(UInt8) -> 5_UInt8 UInt8 : 1 FUNCTION greaterOrEquals(id : 0, 5_UInt8 :: 1) -> greaterOrEquals(id, 5_UInt8) UInt8 : 2 diff --git a/tests/queries/0_stateless/03591_optimize_prewhere_row_policy.reference b/tests/queries/0_stateless/03591_optimize_prewhere_row_policy.reference new file mode 100644 index 000000000000..9a79cbcb823a --- /dev/null +++ b/tests/queries/0_stateless/03591_optimize_prewhere_row_policy.reference @@ -0,0 +1,144 @@ +-- {echoOn} + +SET use_query_condition_cache = 0; +SET enable_parallel_replicas = 0; +DROP TABLE IF EXISTS 03591_test; +DROP ROW POLICY IF EXISTS 03591_rp ON 03591_test; +CREATE TABLE 03591_test (a Int32, b Int32) ENGINE=MergeTree ORDER BY tuple(); +INSERT INTO 03591_test VALUES (3, 1), (2, 2), (3, 2); +SELECT * FROM 03591_test; +3 1 +2 2 +3 2 +SELECT * FROM 03591_test WHERE throwIf(b=1, 'Should throw') SETTINGS optimize_move_to_prewhere = 1; -- {serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO} +CREATE ROW POLICY 03591_rp ON 03591_test USING b=2 TO CURRENT_USER; +SELECT * FROM 03591_test; +2 2 +3 2 +-- Print plan with actions to make sure both a > 0 and b=2 are present in the prewhere section +EXPLAIN PLAN actions=1 SELECT * FROM 03591_test WHERE a > 0 SETTINGS optimize_move_to_prewhere = 1, allow_experimental_analyzer = 1; +Expression ((Project names + Projection)) +Actions: INPUT : 0 -> __table1.b Int32 : 0 + INPUT : 1 -> __table1.a Int32 : 1 + ALIAS __table1.b :: 0 -> b Int32 : 2 + ALIAS __table1.a :: 1 -> a Int32 : 0 +Positions: 0 2 + Expression + Actions: INPUT : 0 -> b Int32 : 0 + INPUT : 1 -> a Int32 : 1 + ALIAS b :: 0 -> __table1.b Int32 : 2 + ALIAS a :: 1 -> __table1.a Int32 : 0 + Positions: 2 0 + ReadFromMergeTree (default.03591_test) + ReadType: Default + Parts: 1 + Granules: 1 + Prewhere info + Need filter: 1 + Prewhere filter + Prewhere filter column: greater(__table1.a, 0_UInt8) (removed) + Actions: INPUT : 0 -> a Int32 : 0 + COLUMN Const(UInt8) -> 0_UInt8 UInt8 : 1 + FUNCTION greater(a : 0, 0_UInt8 :: 1) -> greater(__table1.a, 0_UInt8) UInt8 : 2 + Positions: 0 2 + Row level filter + Row level filter column: equals(b, 2_UInt8) (removed) + Actions: INPUT : 0 -> b Int32 : 0 + COLUMN Const(UInt8) -> 2_UInt8 UInt8 : 1 + FUNCTION equals(b : 0, 2_UInt8 :: 1) -> equals(b, 2_UInt8) UInt8 : 2 + Positions: 2 0 +EXPLAIN PLAN actions=1 SELECT * FROM 03591_test WHERE a > 0 SETTINGS optimize_move_to_prewhere = 1, allow_experimental_analyzer = 0; +Expression ((Projection + Before ORDER BY)) +Actions: INPUT :: 0 -> a Int32 : 0 + INPUT :: 1 -> b Int32 : 1 +Positions: 0 1 + Expression + Actions: INPUT :: 0 -> a Int32 : 0 + INPUT :: 1 -> b Int32 : 1 + Positions: 0 1 + ReadFromMergeTree (default.03591_test) + ReadType: Default + Parts: 1 + Granules: 1 + Prewhere info + Need filter: 1 + Prewhere filter + Prewhere filter column: greater(a, 0) (removed) + Actions: INPUT : 0 -> a Int32 : 0 + COLUMN Const(UInt8) -> 0 UInt8 : 1 + FUNCTION greater(a : 0, 0 :: 1) -> greater(a, 0) UInt8 : 2 + Positions: 0 2 + Row level filter + Row level filter column: equals(b, 2) (removed) + Actions: INPUT : 0 -> b Int32 : 0 + COLUMN Const(UInt8) -> 2 UInt8 : 1 + FUNCTION equals(b : 0, 2 :: 1) -> equals(b, 2) UInt8 : 2 + Positions: 2 0 +SELECT * FROM 03591_test WHERE throwIf(b=1, 'Should not throw because b=1 is not visible to this user due to the b=2 row policy') SETTINGS optimize_move_to_prewhere = 1; +-- Print plan with actions to make sure a > 0, b = 2 and a = 3 are present in the prewhere section +EXPLAIN PLAN actions=1 SELECT * FROM 03591_test WHERE a > 0 SETTINGS optimize_move_to_prewhere = 1, additional_table_filters={'03591_test': 'a=3'}, allow_experimental_analyzer = 1; +Expression ((Project names + Projection)) +Actions: INPUT : 0 -> __table1.a Int32 : 0 + INPUT : 1 -> __table1.b Int32 : 1 + ALIAS __table1.a :: 0 -> a Int32 : 2 + ALIAS __table1.b :: 1 -> b Int32 : 0 +Positions: 2 0 + Expression + Actions: INPUT : 1 -> b Int32 : 0 + INPUT : 0 -> a Int32 : 1 + ALIAS b :: 0 -> __table1.b Int32 : 2 + ALIAS a :: 1 -> __table1.a Int32 : 0 + Positions: 0 2 + ReadFromMergeTree (default.03591_test) + ReadType: Default + Parts: 1 + Granules: 1 + Prewhere info + Need filter: 1 + Prewhere filter + Prewhere filter column: and(equals(a, 3_UInt8), greater(__table1.a, 0_UInt8)) (removed) + Actions: INPUT : 0 -> a Int32 : 0 + COLUMN Const(UInt8) -> 3_UInt8 UInt8 : 1 + COLUMN Const(UInt8) -> 0_UInt8 UInt8 : 2 + FUNCTION equals(a : 0, 3_UInt8 :: 1) -> equals(a, 3_UInt8) UInt8 : 3 + FUNCTION greater(a : 0, 0_UInt8 :: 2) -> greater(__table1.a, 0_UInt8) UInt8 : 1 + FUNCTION and(equals(a, 3_UInt8) :: 3, greater(__table1.a, 0_UInt8) :: 1) -> and(equals(a, 3_UInt8), greater(__table1.a, 0_UInt8)) UInt8 : 2 + Positions: 0 2 + Row level filter + Row level filter column: equals(b, 2_UInt8) (removed) + Actions: INPUT : 0 -> b Int32 : 0 + COLUMN Const(UInt8) -> 2_UInt8 UInt8 : 1 + FUNCTION equals(b : 0, 2_UInt8 :: 1) -> equals(b, 2_UInt8) UInt8 : 2 + Positions: 2 0 +EXPLAIN PLAN actions=1 SELECT * FROM 03591_test WHERE a > 0 SETTINGS optimize_move_to_prewhere = 1, additional_table_filters={'03591_test': 'a=3'}, allow_experimental_analyzer = 0; +Expression ((Projection + Before ORDER BY)) +Actions: INPUT :: 0 -> a Int32 : 0 + INPUT :: 1 -> b Int32 : 1 +Positions: 0 1 + Expression + Actions: INPUT :: 0 -> a Int32 : 0 + INPUT :: 1 -> b Int32 : 1 + Positions: 0 1 + ReadFromMergeTree (default.03591_test) + ReadType: Default + Parts: 1 + Granules: 1 + Prewhere info + Need filter: 1 + Prewhere filter + Prewhere filter column: and(equals(a, 3), greater(a, 0)) (removed) + Actions: INPUT : 0 -> a Int32 : 0 + COLUMN Const(UInt8) -> 3 UInt8 : 1 + COLUMN Const(UInt8) -> 0 UInt8 : 2 + FUNCTION equals(a : 0, 3 :: 1) -> equals(a, 3) UInt8 : 3 + FUNCTION greater(a : 0, 0 :: 2) -> greater(a, 0) UInt8 : 1 + FUNCTION and(equals(a, 3) :: 3, greater(a, 0) :: 1) -> and(equals(a, 3), greater(a, 0)) UInt8 : 2 + Positions: 0 2 + Row level filter + Row level filter column: equals(b, 2) (removed) + Actions: INPUT : 0 -> b Int32 : 0 + COLUMN Const(UInt8) -> 2 UInt8 : 1 + FUNCTION equals(b : 0, 2 :: 1) -> equals(b, 2) UInt8 : 2 + Positions: 2 0 +DROP ROW POLICY 03591_rp ON 03591_test; +SELECT * FROM 03591_test WHERE throwIf(b=2, 'Should throw') SETTINGS optimize_move_to_prewhere = 1; -- {serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO} diff --git a/tests/queries/0_stateless/03591_optimize_prewhere_row_policy.sql b/tests/queries/0_stateless/03591_optimize_prewhere_row_policy.sql new file mode 100644 index 000000000000..b010fadff97b --- /dev/null +++ b/tests/queries/0_stateless/03591_optimize_prewhere_row_policy.sql @@ -0,0 +1,34 @@ +-- {echoOn} + +SET use_query_condition_cache = 0; +SET enable_parallel_replicas = 0; + +DROP TABLE IF EXISTS 03591_test; + +DROP ROW POLICY IF EXISTS 03591_rp ON 03591_test; + +CREATE TABLE 03591_test (a Int32, b Int32) ENGINE=MergeTree ORDER BY tuple(); + +INSERT INTO 03591_test VALUES (3, 1), (2, 2), (3, 2); + +SELECT * FROM 03591_test; + +SELECT * FROM 03591_test WHERE throwIf(b=1, 'Should throw') SETTINGS optimize_move_to_prewhere = 1; -- {serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO} + +CREATE ROW POLICY 03591_rp ON 03591_test USING b=2 TO CURRENT_USER; + +SELECT * FROM 03591_test; + +-- Print plan with actions to make sure both a > 0 and b=2 are present in the prewhere section +EXPLAIN PLAN actions=1 SELECT * FROM 03591_test WHERE a > 0 SETTINGS optimize_move_to_prewhere = 1, allow_experimental_analyzer = 1; +EXPLAIN PLAN actions=1 SELECT * FROM 03591_test WHERE a > 0 SETTINGS optimize_move_to_prewhere = 1, allow_experimental_analyzer = 0; + +SELECT * FROM 03591_test WHERE throwIf(b=1, 'Should not throw because b=1 is not visible to this user due to the b=2 row policy') SETTINGS optimize_move_to_prewhere = 1; + +-- Print plan with actions to make sure a > 0, b = 2 and a = 3 are present in the prewhere section +EXPLAIN PLAN actions=1 SELECT * FROM 03591_test WHERE a > 0 SETTINGS optimize_move_to_prewhere = 1, additional_table_filters={'03591_test': 'a=3'}, allow_experimental_analyzer = 1; +EXPLAIN PLAN actions=1 SELECT * FROM 03591_test WHERE a > 0 SETTINGS optimize_move_to_prewhere = 1, additional_table_filters={'03591_test': 'a=3'}, allow_experimental_analyzer = 0; + +DROP ROW POLICY 03591_rp ON 03591_test; + +SELECT * FROM 03591_test WHERE throwIf(b=2, 'Should throw') SETTINGS optimize_move_to_prewhere = 1; -- {serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO} diff --git a/tests/queries/0_stateless/03641_analyzer_issue_85834.reference b/tests/queries/0_stateless/03641_analyzer_issue_85834.reference new file mode 100644 index 000000000000..d81cc0710eb6 --- /dev/null +++ b/tests/queries/0_stateless/03641_analyzer_issue_85834.reference @@ -0,0 +1 @@ +42 diff --git a/tests/queries/0_stateless/03641_analyzer_issue_85834.sql b/tests/queries/0_stateless/03641_analyzer_issue_85834.sql new file mode 100644 index 000000000000..a8903f82508b --- /dev/null +++ b/tests/queries/0_stateless/03641_analyzer_issue_85834.sql @@ -0,0 +1,14 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/85834 + +DROP TABLE IF EXISTS test_generic_events_all; + +CREATE TABLE test_generic_events_all (APIKey UInt8, SessionType UInt8) ENGINE = MergeTree() PARTITION BY APIKey ORDER BY tuple(); +INSERT INTO test_generic_events_all VALUES( 42, 42 ); +ALTER TABLE test_generic_events_all ADD COLUMN OperatingSystem UInt64 DEFAULT 42; + +CREATE ROW POLICY rp ON test_generic_events_all USING APIKey>35 TO CURRENT_USER; + +SELECT OperatingSystem +FROM test_generic_events_all +PREWHERE APIKey = 42 +SETTINGS additional_table_filters = {'test_generic_events_all':'APIKey > 40'};