Skip to content

Commit 5570606

Browse files
ClaireLyttClaireLytt
authored
Support Oracle SQL parser correctly extract REGEXP_SUBSTR parameters. (#37924)
* support Oracle SQL parser * support new version * update * update * support test * realese-notes * realese-notes * realese-notes
1 parent 1650c1b commit 5570606

7 files changed

Lines changed: 231 additions & 2 deletions

File tree

‎RELEASE-NOTES.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66

77
### Bug Fixes
88

9+
1. SQL Parser: Support Oracle SQL parser correctly extract REGEXP_SUBSTR parameters - [#37924](https://github.com/apache/shardingsphere/pull/37924)
10+
911
## Release 5.5.3
1012

1113
### CVE

‎parser/sql/engine/dialect/oracle/src/main/java/org/apache/shardingsphere/sql/parser/engine/oracle/visitor/statement/OracleStatementVisitor.java‎

Lines changed: 109 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,96 @@ public abstract class OracleStatementVisitor extends OracleStatementBaseVisitor<
224224

225225
@Override
226226
public final ASTNode visitParameterMarker(final ParameterMarkerContext ctx) {
227-
return new ParameterMarkerValue(globalParameterMarkerSegments.size(), ParameterMarkerType.QUESTION);
227+
return new ParameterMarkerValue(getParameterMarkerIndex(ctx), ParameterMarkerType.QUESTION);
228+
}
229+
230+
private int getParameterMarkerIndex(final ParameterMarkerContext ctx) {
231+
int startIndex = ctx.getStart().getStartIndex();
232+
if (startIndex <= 0) {
233+
return 0;
234+
}
235+
String sql = ctx.getStart().getInputStream().getText(new Interval(0, startIndex - 1));
236+
return countParameterMarkers(sql);
237+
}
238+
239+
private int countParameterMarkers(final String sql) {
240+
ParameterMarkerScanState state = new ParameterMarkerScanState();
241+
while (state.index < sql.length()) {
242+
if (advanceInLineComment(sql, state)) {
243+
continue;
244+
}
245+
if (advanceInBlockComment(sql, state)) {
246+
continue;
247+
}
248+
if (advanceInOrToggleStringLiteral(sql, state)) {
249+
continue;
250+
}
251+
if (enterLineOrBlockComment(sql, state)) {
252+
continue;
253+
}
254+
if (!state.inStringLiteral && sql.charAt(state.index) == '?') {
255+
state.result++;
256+
}
257+
state.index++;
258+
}
259+
return state.result;
260+
}
261+
262+
private boolean advanceInLineComment(final String sql, final ParameterMarkerScanState state) {
263+
if (!state.inLineComment) {
264+
return false;
265+
}
266+
char ch = sql.charAt(state.index);
267+
if ('\n' == ch || '\r' == ch) {
268+
state.inLineComment = false;
269+
}
270+
state.index++;
271+
return true;
272+
}
273+
274+
private boolean advanceInBlockComment(final String sql, final ParameterMarkerScanState state) {
275+
if (!state.inBlockComment) {
276+
return false;
277+
}
278+
char ch = sql.charAt(state.index);
279+
if ('*' == ch && state.index + 1 < sql.length() && '/' == sql.charAt(state.index + 1)) {
280+
state.inBlockComment = false;
281+
state.index += 2;
282+
} else {
283+
state.index++;
284+
}
285+
return true;
286+
}
287+
288+
private boolean advanceInOrToggleStringLiteral(final String sql, final ParameterMarkerScanState state) {
289+
if (sql.charAt(state.index) != '\'') {
290+
return false;
291+
}
292+
if (state.inStringLiteral && state.index + 1 < sql.length() && '\'' == sql.charAt(state.index + 1)) {
293+
state.index += 2;
294+
} else {
295+
state.inStringLiteral = !state.inStringLiteral;
296+
state.index++;
297+
}
298+
return true;
299+
}
300+
301+
private boolean enterLineOrBlockComment(final String sql, final ParameterMarkerScanState state) {
302+
if (state.inStringLiteral) {
303+
return false;
304+
}
305+
char ch = sql.charAt(state.index);
306+
if ('-' == ch && state.index + 1 < sql.length() && '-' == sql.charAt(state.index + 1)) {
307+
state.inLineComment = true;
308+
state.index += 2;
309+
return true;
310+
}
311+
if ('/' == ch && state.index + 1 < sql.length() && '*' == sql.charAt(state.index + 1)) {
312+
state.inBlockComment = true;
313+
state.index += 2;
314+
return true;
315+
}
316+
return false;
228317
}
229318

230319
@Override
@@ -533,6 +622,12 @@ public final ASTNode visitPredicate(final PredicateContext ctx) {
533622
if (null != ctx.LIKE()) {
534623
return createBinaryOperationExpressionFromLike(ctx);
535624
}
625+
if (null != ctx.PRIOR()) {
626+
return null == ctx.predicate() ? new CommonExpressionSegment(ctx.start.getStartIndex(), ctx.stop.getStopIndex(), getOriginalText(ctx)) : visit(ctx.predicate());
627+
}
628+
if (null == ctx.bitExpr(0)) {
629+
return new CommonExpressionSegment(ctx.start.getStartIndex(), ctx.stop.getStopIndex(), getOriginalText(ctx));
630+
}
536631
return visit(ctx.bitExpr(0));
537632
}
538633

@@ -1317,4 +1412,17 @@ protected void increaseCursorForLoopLevel() {
13171412
protected void decreaseCursorForLoopLevel() {
13181413
--cursorForLoopLevel;
13191414
}
1415+
1416+
private static final class ParameterMarkerScanState {
1417+
1418+
private int index;
1419+
1420+
private int result;
1421+
1422+
private boolean inStringLiteral;
1423+
1424+
private boolean inLineComment;
1425+
1426+
private boolean inBlockComment;
1427+
}
13201428
}

‎parser/sql/engine/dialect/oracle/src/main/java/org/apache/shardingsphere/sql/parser/engine/oracle/visitor/statement/type/OracleDMLStatementVisitor.java‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@
107107
import org.apache.shardingsphere.sql.parser.autogen.OracleStatementParser.UsingClauseContext;
108108
import org.apache.shardingsphere.sql.parser.autogen.OracleStatementParser.WhereClauseContext;
109109
import org.apache.shardingsphere.sql.parser.autogen.OracleStatementParser.WithClauseContext;
110+
import org.apache.shardingsphere.sql.parser.autogen.OracleStatementParser.HierarchicalQueryClauseContext;
110111
import org.apache.shardingsphere.sql.parser.engine.oracle.visitor.statement.OracleStatementVisitor;
111112
import org.apache.shardingsphere.sql.parser.statement.core.enums.CombineType;
112113
import org.apache.shardingsphere.sql.parser.statement.core.enums.JoinType;
@@ -150,6 +151,7 @@
150151
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.order.item.IndexOrderByItemSegment;
151152
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.order.item.OrderByItemSegment;
152153
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.predicate.HavingSegment;
154+
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.predicate.HierarchicalQuerySegment;
153155
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.predicate.LockSegment;
154156
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.predicate.WhereSegment;
155157
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.table.MultiTableConditionalIntoElseSegment;
@@ -441,10 +443,12 @@ public SelectStatement visitSelectIntoStatement(final SelectIntoStatementContext
441443
if (null != ctx.whereClause()) {
442444
result.setWhere((WhereSegment) visit(ctx.whereClause()));
443445
}
446+
if (null != ctx.hierarchicalQueryClause()) {
447+
result.setHierarchicalQuery((HierarchicalQuerySegment) visit(ctx.hierarchicalQueryClause()));
448+
}
444449
if (null != ctx.groupByClause()) {
445450
result.setGroupBy((GroupBySegment) visit(ctx.groupByClause()));
446451
}
447-
// TODO Visit hierarchicalQueryClause
448452
if (null != ctx.modelClause()) {
449453
result.setModel((ModelSegment) visit(ctx.modelClause()));
450454
}
@@ -649,6 +653,9 @@ public ASTNode visitQueryBlock(final QueryBlockContext ctx) {
649653
if (null != ctx.whereClause()) {
650654
result.setWhere((WhereSegment) visit(ctx.whereClause()));
651655
}
656+
if (null != ctx.hierarchicalQueryClause()) {
657+
result.setHierarchicalQuery((HierarchicalQuerySegment) visit(ctx.hierarchicalQueryClause()));
658+
}
652659
if (null != ctx.groupByClause()) {
653660
result.setGroupBy((GroupBySegment) visit(ctx.groupByClause()));
654661
if (null != ctx.groupByClause().havingClause()) {
@@ -661,6 +668,28 @@ public ASTNode visitQueryBlock(final QueryBlockContext ctx) {
661668
return result;
662669
}
663670

671+
@Override
672+
public ASTNode visitHierarchicalQueryClause(final HierarchicalQueryClauseContext ctx) {
673+
HierarchicalQuerySegment result = new HierarchicalQuerySegment(ctx.getStart().getStartIndex(), ctx.getStop().getStopIndex());
674+
result.setNoCycle(null != ctx.NOCYCLE());
675+
if (ctx.expr().isEmpty()) {
676+
return result;
677+
}
678+
boolean connectByFirst = "CONNECT".equalsIgnoreCase(ctx.getStart().getText());
679+
if (connectByFirst) {
680+
result.setConnectBy((ExpressionSegment) visit(ctx.expr(0)));
681+
if (ctx.expr().size() > 1) {
682+
result.setStartWith((ExpressionSegment) visit(ctx.expr(1)));
683+
}
684+
} else {
685+
result.setStartWith((ExpressionSegment) visit(ctx.expr(0)));
686+
if (ctx.expr().size() > 1) {
687+
result.setConnectBy((ExpressionSegment) visit(ctx.expr(1)));
688+
}
689+
}
690+
return result;
691+
}
692+
664693
@Override
665694
public ASTNode visitHavingClause(final HavingClauseContext ctx) {
666695
ExpressionSegment expr = (ExpressionSegment) visit(ctx.expr());

‎parser/sql/statement/core/src/main/java/org/apache/shardingsphere/sql/parser/statement/core/segment/dml/predicate/HierarchicalQuerySegment.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.shardingsphere.sql.parser.statement.core.segment.dml.predicate;
19+
20+
import lombok.Getter;
21+
import lombok.RequiredArgsConstructor;
22+
import lombok.Setter;
23+
import org.apache.shardingsphere.sql.parser.statement.core.segment.SQLSegment;
24+
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.ExpressionSegment;
25+
26+
/**
27+
* Hierarchical query segment for Oracle START WITH / CONNECT BY.
28+
*/
29+
@RequiredArgsConstructor
30+
@Getter
31+
@Setter
32+
public final class HierarchicalQuerySegment implements SQLSegment {
33+
34+
private final int startIndex;
35+
36+
private final int stopIndex;
37+
38+
private boolean noCycle;
39+
40+
private ExpressionSegment startWith;
41+
42+
private ExpressionSegment connectBy;
43+
}

‎parser/sql/statement/core/src/main/java/org/apache/shardingsphere/sql/parser/statement/core/statement/type/dml/SelectStatement.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.order.OrderBySegment;
2929
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.pagination.limit.LimitSegment;
3030
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.predicate.HavingSegment;
31+
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.predicate.HierarchicalQuerySegment;
3132
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.predicate.LockSegment;
3233
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.predicate.WhereSegment;
3334
import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.ModelSegment;
@@ -54,6 +55,8 @@ public final class SelectStatement extends DMLStatement {
5455

5556
private WhereSegment where;
5657

58+
private HierarchicalQuerySegment hierarchicalQuery;
59+
5760
private GroupBySegment groupBy;
5861

5962
private HavingSegment having;
@@ -104,6 +107,15 @@ public Optional<WhereSegment> getWhere() {
104107
return Optional.ofNullable(where);
105108
}
106109

110+
/**
111+
* Get hierarchical query.
112+
*
113+
* @return hierarchical query segment
114+
*/
115+
public Optional<HierarchicalQuerySegment> getHierarchicalQuery() {
116+
return Optional.ofNullable(hierarchicalQuery);
117+
}
118+
107119
/**
108120
* Get group by segment.
109121
*

‎test/it/parser/src/main/resources/case/dml/select-start-with-connect-by.xml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,4 +70,38 @@
7070
</expr>
7171
</where>
7272
</select>
73+
74+
<select sql-case-id="select_with_connect_by" parameters="2, 1">
75+
<projections start-index="7" stop-index="7">
76+
<shorthand-projection start-index="7" stop-index="7" />
77+
</projections>
78+
<from>
79+
<subquery-table start-index="14" stop-index="167">
80+
<subquery>
81+
<select>
82+
<projections start-index="22" stop-index="97">
83+
<expression-projection alias="order_id" text="REGEXP_SUBSTR('5214|1521|5152|1616|218|8226', '[^|]+', 1, LEVEL)" start-index="22" stop-index="97" />
84+
</projections>
85+
<from>
86+
<simple-table name="dual" start-index="104" stop-index="107" />
87+
</from>
88+
</select>
89+
</subquery>
90+
</subquery-table>
91+
</from>
92+
<where start-index="169" stop-index="185">
93+
<expr>
94+
<binary-operation-expression start-index="175" stop-index="185">
95+
<left>
96+
<column name="ROWNUM" start-index="175" stop-index="180" />
97+
</left>
98+
<operator>&lt;=</operator>
99+
<right>
100+
<literal-expression value="1" start-index="185" stop-index="185" />
101+
<parameter-marker-expression parameter-index="1" start-index="185" stop-index="185" />
102+
</right>
103+
</binary-operation-expression>
104+
</expr>
105+
</where>
106+
</select>
73107
</sql-parser-test-cases>

‎test/it/parser/src/main/resources/sql/supported/dml/select-start-with-connect-by.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@
1818

1919
<sql-cases>
2020
<sql-case id="select_start_with_connect_by" value="SELECT * FROM (SELECT level, o.* FROM t_order o WHERE o.order_id = 1 START WITH o.user_id = 1 CONNECT BY PRIOR user_id = user_id ORDER BY level) temp WHERE ROWNUM = 1" db-types="Oracle" />
21+
<sql-case id="select_with_connect_by" value="SELECT * FROM (SELECT REGEXP_SUBSTR('5214|1521|5152|1616|218|8226', '[^|]+', 1, LEVEL) AS order_id FROM dual CONNECT BY REGEXP_SUBSTR(?, '[^|]+', 1, LEVEL) IS NOT NULL) WHERE ROWNUM &lt;= ?" db-types="Oracle"/>
2122
</sql-cases>

0 commit comments

Comments
 (0)