Skip to content

fix(schema): loading table detail costs too much time #2626

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yhilmare merged 1 commit into from
Jun 4, 2024
Merged

fix(schema): loading table detail costs too much time #2626

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions libs/db-browser/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@
<maven-source-plugin.version>2.2.1</maven-source-plugin.version>
<license-maven-plugin.version>3.0</license-maven-plugin.version>
<oracle.jdbc.version>21.1.0.0</oracle.jdbc.version>
<caffeine.version>2.8.0</caffeine.version>
</properties>

<dependencyManagement>
Expand Down Expand Up @@ -201,6 +202,11 @@
<artifactId>ojdbc8</artifactId>
<version>${oracle.jdbc.version}</version>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
<version>${caffeine.version}</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
Expand Down Expand Up @@ -307,6 +313,10 @@
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ojdbc8</artifactId>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
</dependency>
</dependencies>

<build>
Expand Down
46 changes: 46 additions & 0 deletions libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/CacheElement.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Copyright (c) 2023 OceanBase.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.oceanbase.tools.dbbrowser.parser;

import lombok.NonNull;

class CacheElement<T> {

private final T value;
private final Exception exception;

public CacheElement(@NonNull T value) {
this.exception = null;
this.value = value;
}

public CacheElement(@NonNull Exception exception) {
this.value = null;
this.exception = exception;
}

public T get() {
if (this.exception != null) {
if (this.exception instanceof RuntimeException) {
throw (RuntimeException) this.exception;
} else {
throw new RuntimeException(this.exception);
}
}
return this.value;
}

}
53 changes: 53 additions & 0 deletions libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/PLParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,16 @@
*/
package com.oceanbase.tools.dbbrowser.parser;

import java.util.concurrent.TimeUnit;

import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeWalker;

import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.oceanbase.tools.dbbrowser.parser.listener.LogErrorListener;
import com.oceanbase.tools.dbbrowser.parser.listener.MysqlModePLParserListener;
import com.oceanbase.tools.dbbrowser.parser.listener.OracleModePLParserListener;
Expand All @@ -46,11 +50,38 @@
@Slf4j
public class PLParser {

private final static Cache<String, CacheElement<ParseMysqlPLResult>> OB_MYSQL_PARSE_CACHE =
Caffeine.newBuilder()
.maximumSize(1000)
.expireAfterWrite(10, TimeUnit.MINUTES)
.build();
private final static Cache<String, CacheElement<ParseOraclePLResult>> OB_ORACLE_PARSE_CACHE =
Caffeine.newBuilder()
.maximumSize(1000)
.expireAfterWrite(10, TimeUnit.MINUTES)
.build();
private final static Cache<String, CacheElement<ParseOraclePLResult>> ORACLE_PARSE_CACHE =
Comment thread
yhilmare marked this conversation as resolved.
Caffeine.newBuilder()
.maximumSize(1000)
.expireAfterWrite(10, TimeUnit.MINUTES)
.build();

public static ParseMysqlPLResult parseObMysql(final String pl) {
return parseObMysql(pl, 0);
}

public static ParseMysqlPLResult parseObMysql(final String pl, long timeoutMillis) {
CacheElement<ParseMysqlPLResult> value = OB_MYSQL_PARSE_CACHE.get(pl, s -> {
try {
return new CacheElement<>(doParseObMysql(pl, timeoutMillis));
} catch (Exception e) {
return new CacheElement<>(e);
}
});
return value == null ? null : value.get();
}

private static ParseMysqlPLResult doParseObMysql(final String pl, long timeoutMillis) {
long startTime = System.currentTimeMillis();
CharStream input = CharStreams.fromString(pl);
// Lexer-Lexical analysis
Expand Down Expand Up @@ -89,6 +120,17 @@ public static ParseOraclePLResult parseObOracle(final String pl) {
}

public static ParseOraclePLResult parseObOracle(final String pl, long timeoutMillis) {
CacheElement<ParseOraclePLResult> value = OB_ORACLE_PARSE_CACHE.get(pl, s -> {
try {
return new CacheElement<>(doParseObOracle(pl, timeoutMillis));
} catch (Exception e) {
return new CacheElement<>(e);
}
});
return value == null ? null : value.get();
}

private static ParseOraclePLResult doParseObOracle(final String pl, long timeoutMillis) {
long startTime = System.currentTimeMillis();
ParseOraclePLResult result =
parseByRule(pl, com.oceanbase.tools.sqlparser.oboracle.PLParser.RULE_pl_entry_stmt_list, timeoutMillis);
Expand Down Expand Up @@ -137,6 +179,17 @@ public static ParseOraclePLResult parseOracle(final String pl) {
}

public static ParseOraclePLResult parseOracle(final String pl, long timeoutMillis) {
CacheElement<ParseOraclePLResult> value = ORACLE_PARSE_CACHE.get(pl, s -> {
try {
return new CacheElement<>(doParseOracle(pl, timeoutMillis));
} catch (Exception e) {
return new CacheElement<>(e);
}
});
return value == null ? null : value.get();
}

private static ParseOraclePLResult doParseOracle(final String pl, long timeoutMillis) {
CharStream input = CharStreams.fromString(pl);
CaseChangingCharStream caseChangingCharStream = new CaseChangingCharStream(input, true);
PlSqlLexer lexer = new PlSqlLexer(caseChangingCharStream);
Expand Down
73 changes: 73 additions & 0 deletions libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/SqlParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,26 @@
*/
package com.oceanbase.tools.dbbrowser.parser;

import java.io.StringReader;
import java.util.concurrent.TimeUnit;

import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeWalker;

import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.oceanbase.tools.dbbrowser.parser.listener.CustomErrorListener;
import com.oceanbase.tools.dbbrowser.parser.listener.MysqlModeSqlParserListener;
import com.oceanbase.tools.dbbrowser.parser.listener.OracleModeSqlParserListener;
import com.oceanbase.tools.dbbrowser.parser.result.ParseSqlResult;
import com.oceanbase.tools.sqlparser.OBMySQLParser;
import com.oceanbase.tools.sqlparser.OBOracleSQLParser;
import com.oceanbase.tools.sqlparser.obmysql.OBLexer;
import com.oceanbase.tools.sqlparser.obmysql.OBParser;
import com.oceanbase.tools.sqlparser.statement.Statement;
import com.oceanbase.tools.sqlparser.util.TimeoutTokenStream;

import lombok.NonNull;
Expand All @@ -40,11 +48,65 @@
@Slf4j
public class SqlParser {

private final static Cache<String, CacheElement<ParseSqlResult>> OB_MYSQL_PARSE_CACHE =
Caffeine.newBuilder()
.maximumSize(1000)
.expireAfterWrite(10, TimeUnit.MINUTES)
.build();
private final static Cache<String, CacheElement<ParseSqlResult>> OB_ORACLE_PARSE_CACHE =
Caffeine.newBuilder()
.maximumSize(1000)
.expireAfterWrite(10, TimeUnit.MINUTES)
.build();
private final static Cache<String, CacheElement<Statement>> OB_MYSQL_STMT_PARSE_CACHE =
Caffeine.newBuilder()
.maximumSize(1000)
.expireAfterWrite(10, TimeUnit.MINUTES)
.build();
private final static Cache<String, CacheElement<Statement>> OB_ORACLE_STMT_PARSE_CACHE =
Caffeine.newBuilder()
.maximumSize(1000)
.expireAfterWrite(10, TimeUnit.MINUTES)
.build();

public static Statement parseMysqlStatement(@NonNull String sql) {
CacheElement<Statement> value = OB_MYSQL_STMT_PARSE_CACHE.get(sql, s -> {
try {
return new CacheElement<>(new OBMySQLParser().parse(new StringReader(sql)));
} catch (Exception e) {
return new CacheElement<>(e);
}
});
return value == null ? null : value.get();
}

public static Statement parseOracleStatement(@NonNull String sql) {
CacheElement<Statement> value = OB_ORACLE_STMT_PARSE_CACHE.get(sql, s -> {
try {
return new CacheElement<>(new OBOracleSQLParser().parse(new StringReader(sql)));
} catch (Exception e) {
return new CacheElement<>(e);
}
});
return value == null ? null : value.get();
}

public static ParseSqlResult parseMysql(final String sql) {
return parseMysql(sql, 0);
}

public static ParseSqlResult parseMysql(final String sql, long timeoutMillis) {
CacheElement<ParseSqlResult> value = OB_MYSQL_PARSE_CACHE.get(sql, s -> {
try {
return new CacheElement<>(doParseMysql(sql, timeoutMillis));
} catch (Exception e) {
return new CacheElement<>(e);
}
});
return value == null ? null : value.get();
}

private static ParseSqlResult doParseMysql(final String sql, long timeoutMillis) {
long startTime = System.currentTimeMillis();
CharStream input = CharStreams.fromString(sql);
// Lexer-Lexical analysis
Expand Down Expand Up @@ -79,6 +141,17 @@ public static ParseSqlResult parseOracle(final String sql) {
}

public static ParseSqlResult parseOracle(final String sql, long timeoutMillis) {
CacheElement<ParseSqlResult> value = OB_ORACLE_PARSE_CACHE.get(sql, s -> {
try {
return new CacheElement<>(doParseOracle(sql, timeoutMillis));
} catch (Exception e) {
return new CacheElement<>(e);
}
});
return value == null ? null : value.get();
}

private static ParseSqlResult doParseOracle(final String sql, long timeoutMillis) {
long startTime = System.currentTimeMillis();
CharStream input = CharStreams.fromString(sql);
// Lexer-Lexical analysis
Expand Down
32 changes: 16 additions & 16 deletions ...browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.io.StringWriter;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
Expand Down Expand Up @@ -81,6 +80,7 @@
import com.oceanbase.tools.dbbrowser.model.DBView;
import com.oceanbase.tools.dbbrowser.model.MySQLConstants;
import com.oceanbase.tools.dbbrowser.parser.PLParser;
import com.oceanbase.tools.dbbrowser.parser.SqlParser;
import com.oceanbase.tools.dbbrowser.parser.result.ParseMysqlPLResult;
import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessor;
import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessorSqlMapper;
Expand All @@ -91,8 +91,7 @@
import com.oceanbase.tools.dbbrowser.util.MySQLSqlBuilder;
import com.oceanbase.tools.dbbrowser.util.SqlBuilder;
import com.oceanbase.tools.dbbrowser.util.StringUtils;
import com.oceanbase.tools.sqlparser.OBMySQLParser;
import com.oceanbase.tools.sqlparser.SQLParser;
import com.oceanbase.tools.sqlparser.statement.Statement;
import com.oceanbase.tools.sqlparser.statement.createtable.CreateTable;
import com.oceanbase.tools.sqlparser.statement.createtable.TableOptions;

Expand Down Expand Up @@ -1040,19 +1039,20 @@ private void obtainOptionsByQuery(String schemaName, String tableName, DBTable.D
}

private void obtainOptionsByParser(DBTable.DBTableOptions dbTableOptions, String ddl) {
SQLParser sqlParser = new OBMySQLParser();
CreateTable stmt = (CreateTable) sqlParser.parse(new StringReader(ddl));
TableOptions options = stmt.getTableOptions();
if (Objects.nonNull(options)) {
dbTableOptions.setCharsetName(options.getCharset());
dbTableOptions.setRowFormat(options.getRowFormat());
dbTableOptions.setCompressionOption(options.getCompression());
dbTableOptions.setReplicaNum(options.getReplicaNum());
dbTableOptions.setBlockSize(options.getBlockSize());
dbTableOptions.setUseBloomFilter(options.getUseBloomFilter());
dbTableOptions
.setTabletSize(
Objects.nonNull(options.getTabletSize()) ? options.getTabletSize().longValue() : null);
Statement statement = SqlParser.parseMysqlStatement(ddl);
if (statement instanceof CreateTable) {
CreateTable stmt = (CreateTable) statement;
TableOptions options = stmt.getTableOptions();
if (Objects.nonNull(options)) {
dbTableOptions.setCharsetName(options.getCharset());
dbTableOptions.setRowFormat(options.getRowFormat());
dbTableOptions.setCompressionOption(options.getCompression());
dbTableOptions.setReplicaNum(options.getReplicaNum());
dbTableOptions.setBlockSize(options.getBlockSize());
dbTableOptions.setUseBloomFilter(options.getUseBloomFilter());
dbTableOptions.setTabletSize(
Objects.nonNull(options.getTabletSize()) ? options.getTabletSize().longValue() : null);
}
}
}

Expand Down
31 changes: 16 additions & 15 deletions ...in/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.io.StringWriter;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
Expand Down Expand Up @@ -84,6 +83,7 @@
import com.oceanbase.tools.dbbrowser.model.DBView;
import com.oceanbase.tools.dbbrowser.model.MySQLConstants;
import com.oceanbase.tools.dbbrowser.parser.PLParser;
import com.oceanbase.tools.dbbrowser.parser.SqlParser;
import com.oceanbase.tools.dbbrowser.parser.result.ParseMysqlPLResult;
import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessor;
import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessorSqlMapper;
Expand All @@ -94,8 +94,7 @@
import com.oceanbase.tools.dbbrowser.util.MySQLSqlBuilder;
import com.oceanbase.tools.dbbrowser.util.SqlBuilder;
import com.oceanbase.tools.dbbrowser.util.StringUtils;
import com.oceanbase.tools.sqlparser.OBMySQLParser;
import com.oceanbase.tools.sqlparser.SQLParser;
import com.oceanbase.tools.sqlparser.statement.Statement;
import com.oceanbase.tools.sqlparser.statement.createtable.CreateTable;
import com.oceanbase.tools.sqlparser.statement.createtable.TableOptions;

Expand Down Expand Up @@ -1097,18 +1096,20 @@ private void obtainOptionsByQuery(String schemaName, String tableName, DBTableOp
}

private void obtainOptionsByParser(DBTableOptions dbTableOptions, String ddl) {
SQLParser sqlParser = new OBMySQLParser();
CreateTable stmt = (CreateTable) sqlParser.parse(new StringReader(ddl));
TableOptions options = stmt.getTableOptions();
if (Objects.nonNull(options)) {
dbTableOptions.setCharsetName(options.getCharset());
dbTableOptions.setRowFormat(options.getRowFormat());
dbTableOptions.setCompressionOption(options.getCompression());
dbTableOptions.setReplicaNum(options.getReplicaNum());
dbTableOptions.setBlockSize(options.getBlockSize());
dbTableOptions.setUseBloomFilter(options.getUseBloomFilter());
dbTableOptions.setTabletSize(
Objects.nonNull(options.getTabletSize()) ? options.getTabletSize().longValue() : null);
Statement statement = SqlParser.parseMysqlStatement(ddl);
if (statement instanceof CreateTable) {
CreateTable stmt = (CreateTable) statement;
TableOptions options = stmt.getTableOptions();
if (Objects.nonNull(options)) {
dbTableOptions.setCharsetName(options.getCharset());
dbTableOptions.setRowFormat(options.getRowFormat());
dbTableOptions.setCompressionOption(options.getCompression());
dbTableOptions.setReplicaNum(options.getReplicaNum());
dbTableOptions.setBlockSize(options.getBlockSize());
dbTableOptions.setUseBloomFilter(options.getUseBloomFilter());
dbTableOptions.setTabletSize(
Objects.nonNull(options.getTabletSize()) ? options.getTabletSize().longValue() : null);
}
}
}

Expand Down
Loading