From 7dbd95d243e5f61529ea785b358450d848144cf6 Mon Sep 17 00:00:00 2001 From: lvca Date: Sun, 24 Sep 2023 18:10:17 -0400 Subject: [PATCH 01/19] First version with basic functionalities working Still missing tests with conditions, parenthesis, support for embedded object and graph syntax --- .../java/com/arcadedb/database/Database.java | 14 +- .../arcadedb/database/DatabaseInternal.java | 1 + .../arcadedb/database/EmbeddedDatabase.java | 107 +++++--- .../arcadedb/query/nativ/NativeOperator.java | 104 +++++++ .../query/nativ/NativeParameterValue.java | 28 ++ .../query/nativ/NativePropertyValue.java | 21 ++ .../query/nativ/NativeRecordProperty.java | 36 +++ .../query/nativ/NativeRuntimeValue.java | 11 + .../arcadedb/query/nativ/NativeSelect.java | 259 ++++++++++++++++++ .../arcadedb/query/nativ/NativeTreeNode.java | 80 ++++++ .../arcadedb/query/nativ/QueryIterator.java | 35 +++ .../com/arcadedb/utility/VariableParser.java | 38 +-- .../query/nativ/NativeSelectExecutionIT.java | 183 +++++++++++++ .../collection/SQLMethodTransformTest.java | 6 + .../com/arcadedb/server/ServerDatabase.java | 37 ++- .../server/ha/ReplicatedDatabase.java | 47 ++-- 16 files changed, 919 insertions(+), 88 deletions(-) create mode 100644 engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java create mode 100644 engine/src/main/java/com/arcadedb/query/nativ/NativeParameterValue.java create mode 100644 engine/src/main/java/com/arcadedb/query/nativ/NativePropertyValue.java create mode 100644 engine/src/main/java/com/arcadedb/query/nativ/NativeRecordProperty.java create mode 100644 engine/src/main/java/com/arcadedb/query/nativ/NativeRuntimeValue.java create mode 100644 engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java create mode 100644 engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java create mode 100644 engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java create mode 100644 engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java diff --git a/engine/src/main/java/com/arcadedb/database/Database.java b/engine/src/main/java/com/arcadedb/database/Database.java index 7d9ef67995..b12cf44fba 100644 --- a/engine/src/main/java/com/arcadedb/database/Database.java +++ b/engine/src/main/java/com/arcadedb/database/Database.java @@ -28,6 +28,7 @@ import com.arcadedb.graph.Vertex; import com.arcadedb.index.IndexCursor; import com.arcadedb.query.QueryEngine; +import com.arcadedb.query.nativ.NativeSelect; import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.schema.Schema; @@ -55,6 +56,8 @@ enum TRANSACTION_ISOLATION_LEVEL { */ String getCurrentUserName(); + NativeSelect select(); + /** * Executes a command by specifying the language and arguments in a map. * @@ -218,9 +221,9 @@ enum TRANSACTION_ISOLATION_LEVEL { * @see DatabaseAsyncExecutor#newEdgeByKeys(String, String, Object, String, String, Object, boolean, String, boolean, boolean, NewEdgeCallback, Object...) * @see #newEdgeByKeys(Vertex, String, String[], Object[], boolean, String, boolean, Object...) */ - Edge newEdgeByKeys(String sourceVertexType, String[] sourceVertexKeyNames, Object[] sourceVertexKeyValues, String destinationVertexType, - String[] destinationVertexKeyNames, Object[] destinationVertexKeyValues, boolean createVertexIfNotExist, String edgeType, boolean bidirectional, - Object... properties); + Edge newEdgeByKeys(String sourceVertexType, String[] sourceVertexKeyNames, Object[] sourceVertexKeyValues, + String destinationVertexType, String[] destinationVertexKeyNames, Object[] destinationVertexKeyValues, + boolean createVertexIfNotExist, String edgeType, boolean bidirectional, Object... properties); /** * Creates a new edge between two vertices specifying the source vertex instance and the key/value pairs to lookup for the destination vertices. The direction @@ -242,8 +245,9 @@ Edge newEdgeByKeys(String sourceVertexType, String[] sourceVertexKeyNames, Objec * @see DatabaseAsyncExecutor#newEdgeByKeys(String, String, Object, String, String, Object, boolean, String, boolean, boolean, NewEdgeCallback, Object...) * @see #newEdgeByKeys(String, String[], Object[], String, String[], Object[], boolean, String, boolean, Object...) */ - Edge newEdgeByKeys(Vertex sourceVertex, String destinationVertexType, String[] destinationVertexKeyNames, Object[] destinationVertexKeyValues, - boolean createVertexIfNotExist, String edgeType, boolean bidirectional, Object... properties); + Edge newEdgeByKeys(Vertex sourceVertex, String destinationVertexType, String[] destinationVertexKeyNames, + Object[] destinationVertexKeyValues, boolean createVertexIfNotExist, String edgeType, boolean bidirectional, + Object... properties); /** * Returns the query engine by language name. diff --git a/engine/src/main/java/com/arcadedb/database/DatabaseInternal.java b/engine/src/main/java/com/arcadedb/database/DatabaseInternal.java index d197e9881c..eca18f30ed 100644 --- a/engine/src/main/java/com/arcadedb/database/DatabaseInternal.java +++ b/engine/src/main/java/com/arcadedb/database/DatabaseInternal.java @@ -24,6 +24,7 @@ import com.arcadedb.engine.WALFileFactory; import com.arcadedb.exception.TransactionException; import com.arcadedb.graph.GraphEngine; +import com.arcadedb.query.nativ.NativeSelect; import com.arcadedb.query.sql.parser.ExecutionPlanCache; import com.arcadedb.query.sql.parser.StatementCache; import com.arcadedb.security.SecurityDatabaseUser; diff --git a/engine/src/main/java/com/arcadedb/database/EmbeddedDatabase.java b/engine/src/main/java/com/arcadedb/database/EmbeddedDatabase.java index 5c5b7f8931..3c13fe8cd8 100644 --- a/engine/src/main/java/com/arcadedb/database/EmbeddedDatabase.java +++ b/engine/src/main/java/com/arcadedb/database/EmbeddedDatabase.java @@ -59,6 +59,7 @@ import com.arcadedb.log.LogManager; import com.arcadedb.query.QueryEngine; import com.arcadedb.query.QueryEngineManager; +import com.arcadedb.query.nativ.NativeSelect; import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.query.sql.parser.ExecutionPlanCache; import com.arcadedb.query.sql.parser.StatementCache; @@ -87,9 +88,9 @@ public class EmbeddedDatabase extends RWLockContext implements DatabaseInternal { public static final int EDGE_LIST_INITIAL_CHUNK_SIZE = 64; public static final int MAX_RECOMMENDED_EDGE_LIST_CHUNK_SIZE = 8192; - private static final Set SUPPORTED_FILE_EXT = Set.of(Dictionary.DICT_EXT, Bucket.BUCKET_EXT, - LSMTreeIndexMutable.NOTUNIQUE_INDEX_EXT, LSMTreeIndexMutable.UNIQUE_INDEX_EXT, LSMTreeIndexCompacted.NOTUNIQUE_INDEX_EXT, - LSMTreeIndexCompacted.UNIQUE_INDEX_EXT, HnswVectorIndex.FILE_EXT); + private static final Set SUPPORTED_FILE_EXT = Set.of(Dictionary.DICT_EXT, + Bucket.BUCKET_EXT, LSMTreeIndexMutable.NOTUNIQUE_INDEX_EXT, LSMTreeIndexMutable.UNIQUE_INDEX_EXT, + LSMTreeIndexCompacted.NOTUNIQUE_INDEX_EXT, LSMTreeIndexCompacted.UNIQUE_INDEX_EXT, HnswVectorIndex.FILE_EXT); public final AtomicLong indexCompactions = new AtomicLong(); protected final String name; protected final ComponentFile.MODE mode; @@ -127,8 +128,8 @@ public class EmbeddedDatabase extends RWLockContext implements DatabaseInternal private final ConcurrentHashMap reusableQueryEngines = new ConcurrentHashMap<>(); private TRANSACTION_ISOLATION_LEVEL transactionIsolationLevel = TRANSACTION_ISOLATION_LEVEL.READ_COMMITTED; - protected EmbeddedDatabase(final String path, final ComponentFile.MODE mode, final ContextConfiguration configuration, final SecurityManager security, - final Map>> callbacks) { + protected EmbeddedDatabase(final String path, final ComponentFile.MODE mode, final ContextConfiguration configuration, + final SecurityManager security, final Map>> callbacks) { try { this.mode = mode; this.configuration = configuration; @@ -137,7 +138,8 @@ protected EmbeddedDatabase(final String path, final ComponentFile.MODE mode, fin this.serializer = new BinarySerializer(configuration); this.walFactory = mode == ComponentFile.MODE.READ_WRITE ? new WALFileFactoryEmbedded() : null; this.statementCache = new StatementCache(this, configuration.getValueAsInteger(GlobalConfiguration.SQL_STATEMENT_CACHE)); - this.executionPlanCache = new ExecutionPlanCache(this, configuration.getValueAsInteger(GlobalConfiguration.SQL_STATEMENT_CACHE)); + this.executionPlanCache = new ExecutionPlanCache(this, + configuration.getValueAsInteger(GlobalConfiguration.SQL_STATEMENT_CACHE)); if (path.endsWith(File.separator)) databasePath = path.substring(0, path.length() - 1); @@ -185,7 +187,8 @@ protected void open() { protected void create() { final File databaseDirectory = new File(databasePath); - if (new File(databaseDirectory, EmbeddedSchema.SCHEMA_FILE_NAME).exists() || new File(databaseDirectory, EmbeddedSchema.SCHEMA_PREV_FILE_NAME).exists()) + if (new File(databaseDirectory, EmbeddedSchema.SCHEMA_FILE_NAME).exists() || new File(databaseDirectory, + EmbeddedSchema.SCHEMA_PREV_FILE_NAME).exists()) throw new DatabaseOperationException("Database '" + databasePath + "' already exists"); if (!databaseDirectory.exists() && !databaseDirectory.mkdirs()) @@ -371,7 +374,8 @@ public void commit() { executeInReadLock(() -> { checkTransactionIsActive(false); - final DatabaseContext.DatabaseContextTL current = DatabaseContext.INSTANCE.getContext(EmbeddedDatabase.this.getDatabasePath()); + final DatabaseContext.DatabaseContextTL current = DatabaseContext.INSTANCE.getContext( + EmbeddedDatabase.this.getDatabasePath()); try { current.getLastTransaction().commit(); } finally { @@ -390,7 +394,8 @@ public void rollback() { try { checkTransactionIsActive(false); - final DatabaseContext.DatabaseContextTL current = DatabaseContext.INSTANCE.getContext(EmbeddedDatabase.this.getDatabasePath()); + final DatabaseContext.DatabaseContextTL current = DatabaseContext.INSTANCE.getContext( + EmbeddedDatabase.this.getDatabasePath()); current.popIfNotLastTransaction().rollback(); } catch (final TransactionException e) { @@ -408,7 +413,8 @@ public void rollbackAllNested() { stats.txRollbacks.incrementAndGet(); executeInReadLock(() -> { - final DatabaseContext.DatabaseContextTL current = DatabaseContext.INSTANCE.getContext(EmbeddedDatabase.this.getDatabasePath()); + final DatabaseContext.DatabaseContextTL current = DatabaseContext.INSTANCE.getContext( + EmbeddedDatabase.this.getDatabasePath()); TransactionContext tx; while ((tx = current.popIfNotLastTransaction()) != null) { @@ -455,7 +461,8 @@ public void scanType(final String typeName, final boolean polymorphic, final Doc } @Override - public void scanType(final String typeName, final boolean polymorphic, final DocumentCallback callback, final ErrorRecordCallback errorRecordCallback) { + public void scanType(final String typeName, final boolean polymorphic, final DocumentCallback callback, + final ErrorRecordCallback errorRecordCallback) { stats.scanType.incrementAndGet(); executeInReadLock(() -> { @@ -690,7 +697,8 @@ public IndexCursor lookupByKey(final String type, final String[] keyNames, final final TypeIndex idx = t.getPolymorphicIndexByProperties(keyNames); if (idx == null) - throw new IllegalArgumentException("No index has been created on type '" + type + "' properties " + Arrays.toString(keyNames)); + throw new IllegalArgumentException( + "No index has been created on type '" + type + "' properties " + Arrays.toString(keyNames)); return idx.get(keyValues); }); @@ -902,10 +910,11 @@ public void updateRecord(final Record record) { public Document getOriginalDocument(final Record record) { final Binary originalBuffer = ((RecordInternal) record).getBuffer(); if (originalBuffer == null) - throw new IllegalStateException( - "Cannot read original buffer for record " + record.getIdentity() + ". In case of tx retry check the record is created inside the transaction"); + throw new IllegalStateException("Cannot read original buffer for record " + record.getIdentity() + + ". In case of tx retry check the record is created inside the transaction"); originalBuffer.rewind(); - return (Document) recordFactory.newImmutableRecord(this, ((Document) record).getType(), record.getIdentity(), originalBuffer, null); + return (Document) recordFactory.newImmutableRecord(this, ((Document) record).getType(), record.getIdentity(), originalBuffer, + null); } @Override @@ -914,7 +923,9 @@ public void updateRecordNoLock(final Record record, final boolean discardRecordA final boolean implicitTransaction = checkTransactionIsActive(autoTransaction); try { - final List indexes = record instanceof Document ? indexer.getInvolvedIndexes((Document) record) : Collections.emptyList(); + final List indexes = record instanceof Document ? + indexer.getInvolvedIndexes((Document) record) : + Collections.emptyList(); if (!indexes.isEmpty()) { // UPDATE THE INDEXES TOO @@ -1023,7 +1034,8 @@ public boolean transaction(final TransactionScope txBlock, final boolean joinCur } @Override - public boolean transaction(final TransactionScope txBlock, final boolean joinCurrentTx, int attempts, final OkCallback ok, final ErrorCallback error) { + public boolean transaction(final TransactionScope txBlock, final boolean joinCurrentTx, int attempts, final OkCallback ok, + final ErrorCallback error) { if (txBlock == null) throw new IllegalArgumentException("Transaction block is null"); @@ -1127,7 +1139,8 @@ public MutableEmbeddedDocument newEmbeddedDocument(final EmbeddedModifier modifi final DocumentType type = schema.getType(typeName); if (!type.getClass().equals(DocumentType.class)) throw new IllegalArgumentException( - "Cannot create an embedded document of type '" + typeName + "' because it is a " + type.getClass().getName() + " instead of a document type "); + "Cannot create an embedded document of type '" + typeName + "' because it is a " + type.getClass().getName() + + " instead of a document type "); return new MutableEmbeddedDocument(wrappedDatabaseInstance, type, modifier); } @@ -1146,9 +1159,10 @@ public MutableVertex newVertex(final String typeName) { return new MutableVertex(wrappedDatabaseInstance, (VertexType) type, null); } - public Edge newEdgeByKeys(final String sourceVertexType, final String[] sourceVertexKeyNames, final Object[] sourceVertexKeyValues, - final String destinationVertexType, final String[] destinationVertexKeyNames, final Object[] destinationVertexKeyValues, - final boolean createVertexIfNotExist, final String edgeType, final boolean bidirectional, final Object... properties) { + public Edge newEdgeByKeys(final String sourceVertexType, final String[] sourceVertexKeyNames, + final Object[] sourceVertexKeyValues, final String destinationVertexType, final String[] destinationVertexKeyNames, + final Object[] destinationVertexKeyValues, final boolean createVertexIfNotExist, final String edgeType, + final boolean bidirectional, final Object... properties) { if (sourceVertexKeyNames == null) throw new IllegalArgumentException("Source vertex key is null"); @@ -1172,11 +1186,13 @@ public Edge newEdgeByKeys(final String sourceVertexType, final String[] sourceVe ((MutableVertex) sourceVertex).save(); } else throw new IllegalArgumentException( - "Cannot find source vertex with key " + Arrays.toString(sourceVertexKeyNames) + "=" + Arrays.toString(sourceVertexKeyValues)); + "Cannot find source vertex with key " + Arrays.toString(sourceVertexKeyNames) + "=" + Arrays.toString( + sourceVertexKeyValues)); } else sourceVertex = v1Result.next().getIdentity().asVertex(); - final Iterator v2Result = lookupByKey(destinationVertexType, destinationVertexKeyNames, destinationVertexKeyValues); + final Iterator v2Result = lookupByKey(destinationVertexType, destinationVertexKeyNames, + destinationVertexKeyValues); final Vertex destinationVertex; if (!v2Result.hasNext()) { if (createVertexIfNotExist) { @@ -1186,7 +1202,8 @@ public Edge newEdgeByKeys(final String sourceVertexType, final String[] sourceVe ((MutableVertex) destinationVertex).save(); } else throw new IllegalArgumentException( - "Cannot find destination vertex with key " + Arrays.toString(destinationVertexKeyNames) + "=" + Arrays.toString(destinationVertexKeyValues)); + "Cannot find destination vertex with key " + Arrays.toString(destinationVertexKeyNames) + "=" + Arrays.toString( + destinationVertexKeyValues)); } else destinationVertex = v2Result.next().getIdentity().asVertex(); @@ -1196,8 +1213,8 @@ public Edge newEdgeByKeys(final String sourceVertexType, final String[] sourceVe } public Edge newEdgeByKeys(final Vertex sourceVertex, final String destinationVertexType, final String[] destinationVertexKeyNames, - final Object[] destinationVertexKeyValues, final boolean createVertexIfNotExist, final String edgeType, final boolean bidirectional, - final Object... properties) { + final Object[] destinationVertexKeyValues, final boolean createVertexIfNotExist, final String edgeType, + final boolean bidirectional, final Object... properties) { if (sourceVertex == null) throw new IllegalArgumentException("Source vertex is null"); @@ -1207,7 +1224,8 @@ public Edge newEdgeByKeys(final Vertex sourceVertex, final String destinationVer if (destinationVertexKeyNames.length != destinationVertexKeyValues.length) throw new IllegalArgumentException("Destination vertex key and value arrays have different sizes"); - final Iterator v2Result = lookupByKey(destinationVertexType, destinationVertexKeyNames, destinationVertexKeyValues); + final Iterator v2Result = lookupByKey(destinationVertexType, destinationVertexKeyNames, + destinationVertexKeyValues); final Vertex destinationVertex; if (!v2Result.hasNext()) { if (createVertexIfNotExist) { @@ -1217,7 +1235,8 @@ public Edge newEdgeByKeys(final Vertex sourceVertex, final String destinationVer ((MutableVertex) destinationVertex).save(); } else throw new IllegalArgumentException( - "Cannot find destination vertex with key " + Arrays.toString(destinationVertexKeyNames) + "=" + Arrays.toString(destinationVertexKeyValues)); + "Cannot find destination vertex with key " + Arrays.toString(destinationVertexKeyNames) + "=" + Arrays.toString( + destinationVertexKeyValues)); } else destinationVertex = v2Result.next().getIdentity().asVertex(); @@ -1295,7 +1314,8 @@ public ResultSet command(final String language, final String query, final Object } @Override - public ResultSet command(final String language, final String query, final ContextConfiguration configuration, final Object... parameters) { + public ResultSet command(final String language, final String query, final ContextConfiguration configuration, + final Object... parameters) { checkDatabaseIsOpen(); stats.commands.incrementAndGet(); return getQueryEngine(language).command(query, configuration, parameters); @@ -1307,7 +1327,8 @@ public ResultSet command(final String language, final String query, final Map parameters) { + public ResultSet command(final String language, final String query, final ContextConfiguration configuration, + final Map parameters) { checkDatabaseIsOpen(); stats.commands.incrementAndGet(); return getQueryEngine(language).command(query, configuration, parameters); @@ -1343,6 +1364,11 @@ public ResultSet query(final String language, final String query, final Map 0) { LogManager.instance() - .log(this, Level.FINE, "Wait %d ms before the next retry for transaction commit (threadId=%d)", retryDelay, Thread.currentThread().getId()); + .log(this, Level.FINE, "Wait %d ms before the next retry for transaction commit (threadId=%d)", retryDelay, + Thread.currentThread().getId()); try { Thread.sleep(1 + new Random().nextInt(retryDelay)); diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java new file mode 100644 index 0000000000..9ded4aa938 --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java @@ -0,0 +1,104 @@ +package com.arcadedb.query.nativ;/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.arcadedb.serializer.BinaryComparator; + +import java.util.*; + +/** + * Native condition with support for simple operators through inheritance. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public enum NativeOperator { + or("or", 0) { + @Override + Boolean eval(final Object left, final Object right) { + return left == Boolean.TRUE || right == Boolean.TRUE; + } + }, + + and("and", 2) { + @Override + Boolean eval(final Object left, final Object right) { + return left == Boolean.TRUE && right == Boolean.TRUE; + } + }, + + not("not", 2) { + @Override + Boolean eval(final Object left, final Object right) { + return left == Boolean.FALSE; + } + }, + + eq("==", 1) { + @Override + Object eval(final Object left, final Object right) { + return BinaryComparator.equals(left, right); + } + }, + + lt("<", 1) { + @Override + Object eval(final Object left, final Object right) { + return BinaryComparator.compareTo(left, right) < 0; + } + }, + + le("<=", 1) { + @Override + Object eval(final Object left, final Object right) { + return BinaryComparator.compareTo(left, right) <= 0; + } + }, + + gt(">", 1) { + @Override + Object eval(final Object left, final Object right) { + return BinaryComparator.compareTo(left, right) > 0; + } + }, + + ge(">=", 1) { + @Override + Object eval(final Object left, final Object right) { + return BinaryComparator.compareTo(left, right) >= 0; + } + }, + + run("!", -1) { + @Override + Object eval(final Object left, final Object right) { + return left; + } + }; + + NativeOperator(final String name, final int precedence) { + this.name = name; + this.precedence = precedence; + } + + public final String name; + public final int precedence; + + abstract Object eval(Object left, Object right); + + @Override + public String toString() { + return name; + } +} diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeParameterValue.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeParameterValue.java new file mode 100644 index 0000000000..30aac4bdb2 --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeParameterValue.java @@ -0,0 +1,28 @@ +package com.arcadedb.query.nativ; + +import com.arcadedb.database.Document; + +public class NativeParameterValue implements NativeRuntimeValue { + public final String parameterName; + private final NativeSelect select; + + public NativeParameterValue(final NativeSelect select, final String parameterName) { + this.select = select; + this.parameterName = parameterName; + } + + @Override + public Object eval(final Document record) { + if (select.parameters == null) + throw new IllegalArgumentException("Missing parameter '" + parameterName + "'"); + + if (!select.parameters.containsKey(parameterName)) + throw new IllegalArgumentException("Missing parameter '" + parameterName + "'"); + return select.parameters.get(parameterName); + } + + @Override + public String toString() { + return "#" + parameterName; + } +} diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativePropertyValue.java b/engine/src/main/java/com/arcadedb/query/nativ/NativePropertyValue.java new file mode 100644 index 0000000000..67bcd31bc9 --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativePropertyValue.java @@ -0,0 +1,21 @@ +package com.arcadedb.query.nativ; + +import com.arcadedb.database.Document; + +public class NativePropertyValue implements NativeRuntimeValue { + public final String propertyName; + + public NativePropertyValue(final String propertyName) { + this.propertyName = propertyName; + } + + @Override + public Object eval(final Document record) { + return record.get(propertyName); + } + + @Override + public String toString() { + return ":" + propertyName; + } +} diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeRecordProperty.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeRecordProperty.java new file mode 100644 index 0000000000..dced1dd2ec --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeRecordProperty.java @@ -0,0 +1,36 @@ +package com.arcadedb.query.nativ;/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.arcadedb.database.Document; + +import java.util.*; + +/** + * Native condition with support for simple operators through inheritance. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class NativeRecordProperty { + private final String name; + + public NativeRecordProperty(final String name) { + this.name = name; + } + + public Object eval(final Document record) { + return record.get(name); + } +} diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeRuntimeValue.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeRuntimeValue.java new file mode 100644 index 0000000000..74ee81448a --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeRuntimeValue.java @@ -0,0 +1,11 @@ +package com.arcadedb.query.nativ; + +import com.arcadedb.database.Document; + +/** + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public interface NativeRuntimeValue { + + Object eval(final Document record); +} diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java new file mode 100644 index 0000000000..4658e255da --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java @@ -0,0 +1,259 @@ +package com.arcadedb.query.nativ;/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.arcadedb.database.DatabaseInternal; +import com.arcadedb.database.Document; +import com.arcadedb.database.Record; +import com.arcadedb.graph.Edge; +import com.arcadedb.graph.Vertex; +import com.arcadedb.schema.DocumentType; + +import java.util.*; + +/** + * Native Query engine is a simple query engine that covers most of the classic use cases, such as the retrieval of records + * with a where condition. It could be much faster than the same SQL query because it does not use any parser and it is very + * JIT friendly. Future versions could translate the query into bytecode to have an even faster execution. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class NativeSelect { + private final DatabaseInternal database; + + enum STATE {DEFAULT, WHERE, COMPILED} + + Map parameters; + + private NativeTreeNode rootTreeElement; + private NativeTreeNode treeElement; + private DocumentType from; + private NativeOperator operator; + private NativeRuntimeValue property; + private Object propertyValue; + private boolean polymorphic = true; + private int limit = -1; + private STATE state = STATE.DEFAULT; + + public NativeSelect(final DatabaseInternal database) { + this.database = database; + } + + public NativeSelect from(final String from) { + checkNotCompiled(); + if (this.from != null) + throw new IllegalArgumentException("From has already been set"); + + this.from = database.getSchema().getType(from); + return this; + } + + public NativeSelect eq() { + return setOperator(NativeOperator.eq); + } + + public NativeSelect lt() { + return setOperator(NativeOperator.lt); + } + + public NativeSelect le() { + return setOperator(NativeOperator.le); + } + + public NativeSelect gt() { + return setOperator(NativeOperator.gt); + } + + public NativeSelect ge() { + return setOperator(NativeOperator.ge); + } + + public NativeSelect and() { + return setLogic(NativeOperator.and); + } + + public NativeSelect or() { + return setLogic(NativeOperator.or); + } + + public NativeSelect property(final String name) { + checkNotCompiled(); + if (property != null) + throw new IllegalArgumentException("Property has already been set"); + if (state != STATE.WHERE) + throw new IllegalArgumentException("No context was provided for the parameter"); + this.property = new NativePropertyValue(name); + return this; + } + + public NativeSelect value(final Object value) { + checkNotCompiled(); + if (property == null) + throw new IllegalArgumentException("Property has not been set"); + + switch (state) { + case WHERE: + if (operator == null) + throw new IllegalArgumentException("No operator has been set"); + if (propertyValue != null) + throw new IllegalArgumentException("Property value has already been set"); + this.propertyValue = value; + break; + } + + return this; + } + + public NativeSelect where() { + checkNotCompiled(); + if (rootTreeElement != null) + throw new IllegalArgumentException("Where has already been set"); + state = STATE.WHERE; + return this; + } + + public NativeSelect parameter(final String parameterName) { + checkNotCompiled(); + this.propertyValue = new NativeParameterValue(this, parameterName); + return this; + } + + public NativeSelect parameter(final String paramName, final Object paramValue) { + if (parameters == null) + parameters = new HashMap<>(); + parameters.put(paramName, paramValue); + return this; + } + + public NativeSelect limit(final int limit) { + checkNotCompiled(); + this.limit = limit; + return this; + } + + public NativeSelect polymorphic(final boolean polymorphic) { + checkNotCompiled(); + this.polymorphic = polymorphic; + return this; + } + + public QueryIterator vertices() { + return run(); + } + + public QueryIterator edges() { + return run(); + } + + public QueryIterator documents() { + return run(); + } + + private QueryIterator run() { + if (from == null) + throw new IllegalArgumentException("from has not been set"); + if (state != STATE.COMPILED) { + setLogic(NativeOperator.run); + state = STATE.COMPILED; + } + + final int[] returned = new int[] { 0 }; + final Iterator iterator = database.iterateType(from.getName(), polymorphic); + return new QueryIterator<>() { + private T next = null; + + @Override + public boolean hasNext() { + if (limit > -1 && returned[0] >= limit) + return false; + if (next != null) + return true; + if (!iterator.hasNext()) + return false; + + next = fetchNext(); + return next != null; + } + + @Override + public T next() { + if (next == null && !hasNext()) + throw new NoSuchElementException(); + try { + return next; + } finally { + next = null; + } + } + + private T fetchNext() { + do { + final Document record = iterator.next().asDocument(); + if (evaluateWhere(record)) { + ++returned[0]; + return (T) record; + } + + } while (iterator.hasNext()); + + // NOT FOUND + return null; + } + }; + } + + private boolean evaluateWhere(final Document record) { + final Object result = rootTreeElement.eval(record); + if (result instanceof Boolean) + return (Boolean) result; + throw new IllegalArgumentException("A boolean result was expected but '" + result + "' was returned"); + } + + private NativeSelect setLogic(final NativeOperator newLogicOperator) { + checkNotCompiled(); + if (operator == null) + throw new IllegalArgumentException("Missing condition"); + + final NativeTreeNode newTreeElement = new NativeTreeNode(property, operator, propertyValue); + if (rootTreeElement == null) { + // 1ST TIME ONLY + rootTreeElement = new NativeTreeNode(newTreeElement, newLogicOperator, null); + newTreeElement.setParent(rootTreeElement); + } else { + final NativeTreeNode parent = new NativeTreeNode(newTreeElement, newLogicOperator, null); + // REPLACE THE NODE + treeElement.getParent().setRight(parent); + } + treeElement = newTreeElement; + + operator = null; + property = null; + propertyValue = null; + return this; + } + + private NativeSelect setOperator(final NativeOperator nativeOperator) { + checkNotCompiled(); + if (operator != null) + throw new IllegalArgumentException("Operator has already been set (" + operator + ")"); + operator = nativeOperator; + return this; + } + + private void checkNotCompiled() { + if (state == STATE.COMPILED) + throw new IllegalArgumentException("Cannot modify the structure of a select what has been already compiled"); + } +} diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java new file mode 100644 index 0000000000..9854a40dc4 --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java @@ -0,0 +1,80 @@ +package com.arcadedb.query.nativ;/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.arcadedb.database.Document; + +/** + * Native condition representation in a tree. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class NativeTreeNode { + private final Object left; + private final NativeOperator operator; + private Object right; + private NativeTreeNode parent; + + public NativeTreeNode(final Object left, final NativeOperator operator, final Object right) { + this.left = left; + this.operator = operator; + this.right = right; + } + + public Object eval(final Document record) { + final Object leftValue; + if (left instanceof NativeTreeNode) + leftValue = ((NativeTreeNode) left).eval(record); + else if (left instanceof NativeRuntimeValue) + leftValue = ((NativeRuntimeValue) left).eval(record); + else + leftValue = left; + + final Object rightValue; + if (right instanceof NativeTreeNode) + rightValue = ((NativeTreeNode) right).eval(record); + else if (right instanceof NativeRuntimeValue) + rightValue = ((NativeRuntimeValue) right).eval(record); + else + rightValue = right; + + return operator.eval(leftValue, rightValue); + } + + public void setRight(final NativeTreeNode right) { + this.right = right; + } + + public NativeTreeNode getParent() { + return parent; + } + + public void setParent(final NativeTreeNode parent) { + this.parent = parent; + } + + @Override + public String toString() { + final StringBuilder buffer = new StringBuilder(); + buffer.append("( "); + buffer.append(left); + buffer.append(" "); + buffer.append(operator); + buffer.append(" "); + buffer.append(right); + buffer.append(" )"); + return buffer.toString(); + } +} diff --git a/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java b/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java new file mode 100644 index 0000000000..f81c9b3a6f --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java @@ -0,0 +1,35 @@ +package com.arcadedb.query.nativ;/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.arcadedb.database.DatabaseInternal; +import com.arcadedb.database.Document; +import com.arcadedb.database.Record; +import com.arcadedb.graph.Edge; +import com.arcadedb.graph.Vertex; +import com.arcadedb.schema.DocumentType; + +import java.util.*; + +/** + * Query iterator returned from queries. Extends the base Java iterator with convenient methods. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public abstract class QueryIterator implements Iterator { + public T nextOrNull() { + return hasNext() ? next() : null; + } +} diff --git a/engine/src/main/java/com/arcadedb/utility/VariableParser.java b/engine/src/main/java/com/arcadedb/utility/VariableParser.java index 424b8486ef..886fae74d5 100644 --- a/engine/src/main/java/com/arcadedb/utility/VariableParser.java +++ b/engine/src/main/java/com/arcadedb/utility/VariableParser.java @@ -19,6 +19,7 @@ package com.arcadedb.utility; import com.arcadedb.log.LogManager; +import org.apache.lucene.codecs.CodecUtil; import java.util.logging.*; @@ -28,39 +29,44 @@ * @author Luca Garulli (luca.garulli--at--assetdata.it) */ public class VariableParser { - public static Object resolveVariables(final String iText, final String iBegin, final String iEnd, final VariableParserListener iListener) { + public static Object resolveVariables(final String iText, final String iBegin, final String iEnd, + final VariableParserListener iListener) { return resolveVariables(iText, iBegin, iEnd, iListener, null); } - public static Object resolveVariables(final String iText, final String iBegin, final String iEnd, final VariableParserListener iListener, - final Object iDefaultValue) { - if (iListener == null) + public static Object resolveVariables(final String text, final String beginPattern, final String endPattern, + final VariableParserListener listener, final Object defaultValue) { + if (listener == null) throw new IllegalArgumentException("Missed VariableParserListener listener"); - final int beginPos = iText.lastIndexOf(iBegin); + final int beginPos = text.lastIndexOf(beginPattern); if (beginPos == -1) - return iText; + return text; - final int endPos = iText.indexOf(iEnd, beginPos + 1); + final int endPos = text.indexOf(endPattern, beginPos + 1); if (endPos == -1) - return iText; + return text; - final String pre = iText.substring(0, beginPos); - final String var = iText.substring(beginPos + iBegin.length(), endPos); - final String post = iText.substring(endPos + iEnd.length()); + final String pre = text.substring(0, beginPos); + String var = text.substring(beginPos + beginPattern.length(), endPos); + final String post = text.substring(endPos + endPattern.length()); - Object resolved = iListener.resolve(var); + // DECODE INTERNAL + var = var.replace("$\\{", "${"); + var = var.replace("\\}", "}"); + + Object resolved = listener.resolve(var); if (resolved == null) { - if (iDefaultValue == null) - LogManager.instance().log(null, Level.INFO, "[VariableParser.resolveVariables] Error on resolving property: %s", var); + if (defaultValue == null) + LogManager.instance().log(null, Level.INFO, "Error on resolving property: %s", var); else - resolved = iDefaultValue; + resolved = defaultValue; } if (pre.length() > 0 || post.length() > 0) { final String path = pre + (resolved != null ? resolved.toString() : "") + post; - return resolveVariables(path, iBegin, iEnd, iListener); + return resolveVariables(path, beginPattern, endPattern, listener); } return resolved; diff --git a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java new file mode 100644 index 0000000000..e113892721 --- /dev/null +++ b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java @@ -0,0 +1,183 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.query.nativ; + +import com.arcadedb.TestHelper; +import com.arcadedb.graph.Vertex; +import com.arcadedb.utility.Callable; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class NativeSelectExecutionIT extends TestHelper { + + public NativeSelectExecutionIT() { + autoStartTx = true; + } + + @Override + protected void beginTest() { + database.getSchema().createDocumentType("Document"); + database.getSchema().createVertexType("Vertex"); + database.getSchema().createEdgeType("Edge"); + + database.transaction(() -> { + for (int i = 0; i < 100; i++) { + database.newDocument("Document").set("id", i, "float", 3.14F, "name", "Elon").save(); + database.newVertex("Vertex").set("id", i, "float", 3.14F, "name", "Elon").save(); + } + + for (int i = 1; i < 100; i++) { + final Vertex root = database.select().from("Vertex").where().property("id").eq().value(0).vertices().nextOrNull(); + Assertions.assertNotNull(root); + Assertions.assertEquals(0, root.getInteger("id")); + + root.newEdge("Edge", database.select().from("Vertex").where().property("id").eq().value(i).vertices().nextOrNull(), true) + .save(); + } + }); + } + + @Test + public void okReuse() { + final NativeSelect select = database.select().from("Vertex").where().property("id").eq().parameter("value"); + for (int i = 0; i < 100; i++) + Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); + } + + @Test + public void okAnd() { + final NativeSelect select = database.select().from("Vertex")// + .where().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon"); + + for (int i = 0; i < 100; i++) + Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); + + final NativeSelect select2 = database.select().from("Vertex")// + .where().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon2"); + + Assertions.assertFalse(select2.parameter("value", 3).vertices().hasNext()); + + final NativeSelect select3 = database.select().from("Vertex")// + .where().property("id").eq().value(-1)// + .and().property("name").eq().value("Elon"); + + Assertions.assertFalse(select3.vertices().hasNext()); + } + + @Test + public void okOr() { + final NativeSelect select = database.select().from("Vertex")// + .where().property("id").eq().parameter("value")// + .or().property("name").eq().value("Elon2"); + + Assertions.assertEquals(3, select.parameter("value", 3).vertices().nextOrNull().getInteger("id")); + + final NativeSelect select2 = database.select().from("Vertex")// + .where().property("id").eq().parameter("value")// + .or().property("name").eq().value("Elon2"); + + Assertions.assertEquals(3, select2.parameter("value", 3).vertices().nextOrNull().getInteger("id")); + + final NativeSelect select3 = database.select().from("Vertex")// + .where().property("id").eq().value(-1)// + .or().property("name").eq().value("Elon"); + + Assertions.assertEquals(0, select3.parameter("value", 3).vertices().nextOrNull().getInteger("id")); + } + + @Test + public void okAndOr() { + final NativeSelect select = database.select().from("Vertex")// + .where().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon")// + .and().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon"); + + for (int i = 0; i < 100; i++) + Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); + + final NativeSelect select2 = database.select().from("Vertex")// + .where().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon2"); + + Assertions.assertFalse(select2.parameter("value", 3).vertices().hasNext()); + + final NativeSelect select3 = database.select().from("Vertex")// + .where().property("id").eq().value(-1)// + .and().property("name").eq().value("Elon"); + + Assertions.assertFalse(select3.vertices().hasNext()); + + } + + @Test + public void okLimit() { + final NativeSelect select = database.select().from("Vertex")// + .where().property("id").lt().value(10)// + .and().property("name").eq().value("Elon").limit(10); + + final QueryIterator iter = select.vertices(); + while (iter.hasNext()) { + Assertions.assertTrue(iter.next().getInteger("id") < 10); + } + Assertions.assertFalse(iter.hasNext()); + } + + @Test + public void errorMissingParameter() { + expectingException(() -> { + database.select().from("Vertex")// + .where().property("id").eq().parameter("value")// + .vertices().nextOrNull(); + }, IllegalArgumentException.class, "Missing parameter 'value'"); + } + + @Test + public void errorMissingCondition() { + expectingException(() -> { + database.select().from("Vertex")// + .where().property("id").eq().parameter("value")// + .or().vertices().nextOrNull(); + }, IllegalArgumentException.class, "Missing condition"); + } + + private void expectingException(final Runnable callback, final Class expectedException, + final String mustContains) { + boolean failed = true; + try { + callback.run(); + failed = false; + } catch (Throwable e) { + if (!expectedException.equals(e.getClass())) + e.printStackTrace(); + + Assertions.assertEquals(expectedException, e.getClass()); + Assertions.assertTrue(e.getMessage().contains(mustContains), + "Expected '" + mustContains + "' in the error message. Error message is: " + e.getMessage()); + } + + if (!failed) + Assertions.fail("Expected exception " + expectedException); + } +} diff --git a/engine/src/test/java/com/arcadedb/query/sql/method/collection/SQLMethodTransformTest.java b/engine/src/test/java/com/arcadedb/query/sql/method/collection/SQLMethodTransformTest.java index d0fba1feae..4dfc84cc2b 100644 --- a/engine/src/test/java/com/arcadedb/query/sql/method/collection/SQLMethodTransformTest.java +++ b/engine/src/test/java/com/arcadedb/query/sql/method/collection/SQLMethodTransformTest.java @@ -49,6 +49,7 @@ import com.arcadedb.graph.Vertex; import com.arcadedb.index.IndexCursor; import com.arcadedb.query.QueryEngine; +import com.arcadedb.query.nativ.NativeSelect; import com.arcadedb.query.sql.SQLQueryEngine; import com.arcadedb.query.sql.executor.BasicCommandContext; import com.arcadedb.query.sql.executor.ResultSet; @@ -351,6 +352,11 @@ public String getCurrentUserName() { return null; } + @Override + public NativeSelect select() { + return null; + } + @Override public ResultSet command(String language, String query, Map args) { return null; diff --git a/server/src/main/java/com/arcadedb/server/ServerDatabase.java b/server/src/main/java/com/arcadedb/server/ServerDatabase.java index fae43b35d1..1beb4a8a6c 100644 --- a/server/src/main/java/com/arcadedb/server/ServerDatabase.java +++ b/server/src/main/java/com/arcadedb/server/ServerDatabase.java @@ -49,6 +49,7 @@ import com.arcadedb.graph.Vertex; import com.arcadedb.index.IndexCursor; import com.arcadedb.query.QueryEngine; +import com.arcadedb.query.nativ.NativeSelect; import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.query.sql.parser.ExecutionPlanCache; import com.arcadedb.query.sql.parser.StatementCache; @@ -109,6 +110,11 @@ public String getCurrentUserName() { return wrapped.getCurrentUserName(); } + @Override + public NativeSelect select() { + return wrapped.select(); + } + @Override public Map alignToReplicas() { throw new UnsupportedOperationException("Align Database not supported"); @@ -164,7 +170,8 @@ public void scanType(final String typeName, final boolean polymorphic, final Doc } @Override - public void scanType(final String typeName, final boolean polymorphic, final DocumentCallback callback, final ErrorRecordCallback errorRecordCallback) { + public void scanType(final String typeName, final boolean polymorphic, final DocumentCallback callback, + final ErrorRecordCallback errorRecordCallback) { wrapped.scanType(typeName, polymorphic, callback, errorRecordCallback); } @@ -337,7 +344,8 @@ public boolean transaction(final TransactionScope txBlock, final boolean joinCur } @Override - public boolean transaction(final TransactionScope txBlock, final boolean joinCurrentTx, final int attempts, final OkCallback ok, final ErrorCallback error) { + public boolean transaction(final TransactionScope txBlock, final boolean joinCurrentTx, final int attempts, final OkCallback ok, + final ErrorCallback error) { return wrapped.transaction(txBlock, joinCurrentTx, attempts, ok, error); } @@ -378,19 +386,20 @@ public MutableVertex newVertex(final String typeName) { } @Override - public Edge newEdgeByKeys(final String sourceVertexType, final String[] sourceVertexKeyNames, final Object[] sourceVertexKeyValues, - final String destinationVertexType, final String[] destinationVertexKeyNames, final Object[] destinationVertexKeyValues, - final boolean createVertexIfNotExist, final String edgeType, final boolean bidirectional, final Object... properties) { - return wrapped.newEdgeByKeys(sourceVertexType, sourceVertexKeyNames, sourceVertexKeyValues, destinationVertexType, destinationVertexKeyNames, - destinationVertexKeyValues, createVertexIfNotExist, edgeType, bidirectional, properties); + public Edge newEdgeByKeys(final String sourceVertexType, final String[] sourceVertexKeyNames, + final Object[] sourceVertexKeyValues, final String destinationVertexType, final String[] destinationVertexKeyNames, + final Object[] destinationVertexKeyValues, final boolean createVertexIfNotExist, final String edgeType, + final boolean bidirectional, final Object... properties) { + return wrapped.newEdgeByKeys(sourceVertexType, sourceVertexKeyNames, sourceVertexKeyValues, destinationVertexType, + destinationVertexKeyNames, destinationVertexKeyValues, createVertexIfNotExist, edgeType, bidirectional, properties); } @Override public Edge newEdgeByKeys(final Vertex sourceVertex, final String destinationVertexType, final String[] destinationVertexKeyNames, - final Object[] destinationVertexKeyValues, final boolean createVertexIfNotExist, final String edgeType, final boolean bidirectional, - final Object... properties) { - return wrapped.newEdgeByKeys(sourceVertex, destinationVertexType, destinationVertexKeyNames, destinationVertexKeyValues, createVertexIfNotExist, edgeType, - bidirectional, properties); + final Object[] destinationVertexKeyValues, final boolean createVertexIfNotExist, final String edgeType, + final boolean bidirectional, final Object... properties) { + return wrapped.newEdgeByKeys(sourceVertex, destinationVertexType, destinationVertexKeyNames, destinationVertexKeyValues, + createVertexIfNotExist, edgeType, bidirectional, properties); } @Override @@ -437,7 +446,8 @@ public DocumentIndexer getIndexer() { } @Override - public ResultSet command(final String language, final String query, final ContextConfiguration configuration, final Object... args) { + public ResultSet command(final String language, final String query, final ContextConfiguration configuration, + final Object... args) { return wrapped.command(language, query, configuration, args); } @@ -452,7 +462,8 @@ public ResultSet command(final String language, final String query, final Map args) { + public ResultSet command(final String language, final String query, final ContextConfiguration configuration, + final Map args) { return wrapped.command(language, query, configuration, args); } diff --git a/server/src/main/java/com/arcadedb/server/ha/ReplicatedDatabase.java b/server/src/main/java/com/arcadedb/server/ha/ReplicatedDatabase.java index 7f053c4e45..c6182e33d9 100644 --- a/server/src/main/java/com/arcadedb/server/ha/ReplicatedDatabase.java +++ b/server/src/main/java/com/arcadedb/server/ha/ReplicatedDatabase.java @@ -56,6 +56,7 @@ import com.arcadedb.index.IndexCursor; import com.arcadedb.network.binary.ServerIsNotTheLeaderException; import com.arcadedb.query.QueryEngine; +import com.arcadedb.query.nativ.NativeSelect; import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.query.sql.parser.ExecutionPlanCache; import com.arcadedb.query.sql.parser.StatementCache; @@ -117,8 +118,8 @@ public void commit() { replicateTx(tx, phase1, bufferChanges); else { // USE A BIGGER TIMEOUT CONSIDERING THE DOUBLE LATENCY - final TxForwardRequest command = new TxForwardRequest(ReplicatedDatabase.this, getTransactionIsolationLevel(), tx.getBucketRecordDelta(), - bufferChanges, tx.getIndexChanges().toMap()); + final TxForwardRequest command = new TxForwardRequest(ReplicatedDatabase.this, getTransactionIsolationLevel(), + tx.getBucketRecordDelta(), bufferChanges, tx.getIndexChanges().toMap()); server.getHA().forwardCommandToLeader(command, timeout * 2); tx.reset(); } @@ -144,7 +145,8 @@ public void commit() { }); } - public void replicateTx(final TransactionContext tx, final TransactionContext.TransactionPhase1 phase1, final Binary bufferChanges) { + public void replicateTx(final TransactionContext tx, final TransactionContext.TransactionPhase1 phase1, + final Binary bufferChanges) { final int configuredServers = server.getHA().getConfiguredServers(); final int reqQuorum; @@ -357,6 +359,11 @@ public String getCurrentUserName() { return proxied.getCurrentUserName(); } + @Override + public NativeSelect select() { + return proxied.select(); + } + @Override public ContextConfiguration getConfiguration() { return proxied.getConfiguration(); @@ -428,7 +435,8 @@ public void scanType(final String typeName, final boolean polymorphic, final Doc } @Override - public void scanType(final String typeName, final boolean polymorphic, final DocumentCallback callback, final ErrorRecordCallback errorRecordCallback) { + public void scanType(final String typeName, final boolean polymorphic, final DocumentCallback callback, + final ErrorRecordCallback errorRecordCallback) { proxied.scanType(typeName, polymorphic, callback, errorRecordCallback); } @@ -504,11 +512,11 @@ public MutableVertex newVertex(final String typeName) { @Override public Edge newEdgeByKeys(final Vertex sourceVertex, final String destinationVertexType, final String[] destinationVertexKeyNames, - final Object[] destinationVertexKeyValues, final boolean createVertexIfNotExist, final String edgeType, final boolean bidirectional, - final Object... properties) { + final Object[] destinationVertexKeyValues, final boolean createVertexIfNotExist, final String edgeType, + final boolean bidirectional, final Object... properties) { - return proxied.newEdgeByKeys(sourceVertex, destinationVertexType, destinationVertexKeyNames, destinationVertexKeyValues, createVertexIfNotExist, edgeType, - bidirectional, properties); + return proxied.newEdgeByKeys(sourceVertex, destinationVertexType, destinationVertexKeyNames, destinationVertexKeyValues, + createVertexIfNotExist, edgeType, bidirectional, properties); } @Override @@ -517,12 +525,13 @@ public QueryEngine getQueryEngine(final String language) { } @Override - public Edge newEdgeByKeys(final String sourceVertexType, final String[] sourceVertexKeyNames, final Object[] sourceVertexKeyValues, - final String destinationVertexType, final String[] destinationVertexKeyNames, final Object[] destinationVertexKeyValues, - final boolean createVertexIfNotExist, final String edgeType, final boolean bidirectional, final Object... properties) { + public Edge newEdgeByKeys(final String sourceVertexType, final String[] sourceVertexKeyNames, + final Object[] sourceVertexKeyValues, final String destinationVertexType, final String[] destinationVertexKeyNames, + final Object[] destinationVertexKeyValues, final boolean createVertexIfNotExist, final String edgeType, + final boolean bidirectional, final Object... properties) { - return proxied.newEdgeByKeys(sourceVertexType, sourceVertexKeyNames, sourceVertexKeyValues, destinationVertexType, destinationVertexKeyNames, - destinationVertexKeyValues, createVertexIfNotExist, edgeType, bidirectional, properties); + return proxied.newEdgeByKeys(sourceVertexType, sourceVertexKeyNames, sourceVertexKeyValues, destinationVertexType, + destinationVertexKeyNames, destinationVertexKeyValues, createVertexIfNotExist, edgeType, bidirectional, properties); } @Override @@ -551,7 +560,8 @@ public boolean transaction(final TransactionScope txBlock, final boolean joinCur } @Override - public boolean transaction(final TransactionScope txBlock, final boolean joinCurrentTx, final int retries, final OkCallback ok, final ErrorCallback error) { + public boolean transaction(final TransactionScope txBlock, final boolean joinCurrentTx, final int retries, final OkCallback ok, + final ErrorCallback error) { return proxied.transaction(txBlock, joinCurrentTx, retries, ok, error); } @@ -586,7 +596,8 @@ public boolean equals(final Object o) { } @Override - public ResultSet command(final String language, final String query, final ContextConfiguration configuration, final Object... args) { + public ResultSet command(final String language, final String query, final ContextConfiguration configuration, + final Object... args) { if (!isLeader()) { final QueryEngine queryEngine = proxied.getQueryEngineManager().getInstance(language, this); if (queryEngine.isExecutedByTheLeader() || queryEngine.analyze(query).isDDL()) { @@ -611,7 +622,8 @@ public ResultSet command(final String language, final String query, final Map args) { + public ResultSet command(final String language, final String query, final ContextConfiguration configuration, + final Map args) { if (!isLeader()) { final QueryEngine queryEngine = proxied.getQueryEngineManager().getInstance(language, this); if (queryEngine.isExecutedByTheLeader() || queryEngine.analyze(query).isDDL()) { @@ -811,7 +823,8 @@ public Map alignToReplicas() { fileSizes.put(file.getFileId(), file.getSize()); } - final DatabaseAlignRequest request = new DatabaseAlignRequest(getName(), getSchema().getEmbedded().toJSON().toString(), fileChecksums, fileSizes); + final DatabaseAlignRequest request = new DatabaseAlignRequest(getName(), getSchema().getEmbedded().toJSON().toString(), + fileChecksums, fileSizes); final List responsePayloads = ha.sendCommandToReplicasWithQuorum(request, quorum, 120_000); if (responsePayloads != null) { From 84d4525b93c96fc6756527b14c299eeae93b85fe Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 5 Oct 2023 13:29:40 -0400 Subject: [PATCH 02/19] fix: fixed native select expression tree, added from buckets --- .../arcadedb/query/nativ/NativeSelect.java | 83 ++++++++++++++++--- .../arcadedb/query/nativ/NativeTreeNode.java | 44 ++++++++-- .../query/nativ/NativeSelectExecutionIT.java | 64 +++++++------- 3 files changed, 144 insertions(+), 47 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java index 4658e255da..021d49a2f8 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java @@ -17,11 +17,14 @@ import com.arcadedb.database.DatabaseInternal; import com.arcadedb.database.Document; import com.arcadedb.database.Record; +import com.arcadedb.engine.Bucket; import com.arcadedb.graph.Edge; import com.arcadedb.graph.Vertex; import com.arcadedb.schema.DocumentType; +import com.arcadedb.utility.MultiIterator; import java.util.*; +import java.util.stream.*; /** * Native Query engine is a simple query engine that covers most of the classic use cases, such as the retrieval of records @@ -38,8 +41,9 @@ enum STATE {DEFAULT, WHERE, COMPILED} Map parameters; private NativeTreeNode rootTreeElement; - private NativeTreeNode treeElement; - private DocumentType from; + private NativeTreeNode lastTreeElement; + private DocumentType fromType; + private List fromBuckets; private NativeOperator operator; private NativeRuntimeValue property; private Object propertyValue; @@ -51,12 +55,37 @@ public NativeSelect(final DatabaseInternal database) { this.database = database; } - public NativeSelect from(final String from) { + public NativeSelect fromType(final String fromType) { checkNotCompiled(); - if (this.from != null) - throw new IllegalArgumentException("From has already been set"); + if (this.fromType != null) + throw new IllegalArgumentException("From type has already been set"); + if (this.fromBuckets != null) + throw new IllegalArgumentException("From bucket(s) has already been set"); - this.from = database.getSchema().getType(from); + this.fromType = database.getSchema().getType(fromType); + return this; + } + + public NativeSelect fromBuckets(final String... fromBucketNames) { + checkNotCompiled(); + if (this.fromType != null) + throw new IllegalArgumentException("From type has already been set"); + if (this.fromBuckets != null) + throw new IllegalArgumentException("From bucket(s) has already been set"); + + this.fromBuckets = Arrays.stream(fromBucketNames).map(b -> database.getSchema().getBucketByName(b)) + .collect(Collectors.toList()); + return this; + } + + public NativeSelect fromBuckets(final Integer... fromBucketIds) { + checkNotCompiled(); + if (this.fromType != null) + throw new IllegalArgumentException("From type has already been set"); + if (this.fromBuckets != null) + throw new IllegalArgumentException("From bucket(s) has already been set"); + + this.fromBuckets = Arrays.stream(fromBucketIds).map(b -> database.getSchema().getBucketById(b)).collect(Collectors.toList()); return this; } @@ -162,15 +191,27 @@ public QueryIterator documents() { } private QueryIterator run() { - if (from == null) - throw new IllegalArgumentException("from has not been set"); + if (fromType == null && fromBuckets == null) + throw new IllegalArgumentException("from (type or buckets) has not been set"); if (state != STATE.COMPILED) { setLogic(NativeOperator.run); state = STATE.COMPILED; } final int[] returned = new int[] { 0 }; - final Iterator iterator = database.iterateType(from.getName(), polymorphic); + final Iterator iterator; + + if (fromType != null) + iterator = database.iterateType(fromType.getName(), polymorphic); + else if (fromBuckets.size() == 1) + iterator = database.iterateBucket(fromBuckets.get(0).getName()); + else { + final MultiIterator multiIterator = new MultiIterator<>(); + for (Bucket b : fromBuckets) + multiIterator.addIterator(b.iterator()); + iterator = multiIterator; + } + return new QueryIterator<>() { private T next = null; @@ -231,12 +272,28 @@ private NativeSelect setLogic(final NativeOperator newLogicOperator) { // 1ST TIME ONLY rootTreeElement = new NativeTreeNode(newTreeElement, newLogicOperator, null); newTreeElement.setParent(rootTreeElement); + lastTreeElement = newTreeElement; } else { - final NativeTreeNode parent = new NativeTreeNode(newTreeElement, newLogicOperator, null); - // REPLACE THE NODE - treeElement.getParent().setRight(parent); + if (newLogicOperator.equals(NativeOperator.run)) { + // EXECUTION = LAST NODE: APPEND TO THE RIGHT OF THE LATEST + lastTreeElement.getParent().setRight(newTreeElement); + } else if (lastTreeElement.getParent().operator.precedence > newLogicOperator.precedence) { + // AND+ OPERATOR + final NativeTreeNode newNode = new NativeTreeNode(newTreeElement, newLogicOperator, null); + lastTreeElement.getParent().setRight(newNode); + lastTreeElement = newTreeElement; + } else { + // OR+ OPERATOR + final NativeTreeNode currentParent = lastTreeElement.getParent(); + currentParent.setRight(newTreeElement); + final NativeTreeNode newNode = new NativeTreeNode(currentParent, newLogicOperator, null); + if (rootTreeElement.equals(currentParent)) + rootTreeElement = newNode; + else + newNode.setParent(currentParent.getParent()); + lastTreeElement = currentParent; + } } - treeElement = newTreeElement; operator = null; property = null; diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java index 9854a40dc4..8e142484a9 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java @@ -15,6 +15,9 @@ */ import com.arcadedb.database.Document; +import com.arcadedb.query.sql.parser.BooleanExpression; + +import java.util.*; /** * Native condition representation in a tree. @@ -22,15 +25,22 @@ * @author Luca Garulli (l.garulli@arcadedata.com) */ public class NativeTreeNode { - private final Object left; - private final NativeOperator operator; - private Object right; - private NativeTreeNode parent; + public Object left; + public final NativeOperator operator; + private Object right; + private NativeTreeNode parent; + private List children; // TODO: REMOVE IT? public NativeTreeNode(final Object left, final NativeOperator operator, final Object right) { this.left = left; + if (left instanceof NativeTreeNode) + ((NativeTreeNode) left).setParent(this); + this.operator = operator; + this.right = right; + if (right instanceof NativeTreeNode) + ((NativeTreeNode) right).setParent(this); } public Object eval(final Document record) { @@ -53,16 +63,38 @@ else if (right instanceof NativeRuntimeValue) return operator.eval(leftValue, rightValue); } + public void addChild(final NativeTreeNode child) { + if (children == null) + children = new ArrayList<>(); + children.add(child); + } + public void setRight(final NativeTreeNode right) { + if (this.right != null) + throw new IllegalArgumentException("Cannot assign the right node because already assigned to " + this.right); this.right = right; + if (right.parent != null) + throw new IllegalArgumentException("Cannot assign the parent to the right node " + right); + right.parent = this; } public NativeTreeNode getParent() { return parent; } - public void setParent(final NativeTreeNode parent) { - this.parent = parent; + public void setParent(final NativeTreeNode newParent) { + if (this.parent == newParent) + return; + + if (this.parent != null) { + if (this.parent.left == this) { + this.parent.left = newParent; + } else if (this.parent.right == this) { + this.parent.right = newParent; + } + } + + this.parent = newParent; } @Override diff --git a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java index e113892721..d84bbe81fb 100644 --- a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java +++ b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java @@ -20,7 +20,6 @@ import com.arcadedb.TestHelper; import com.arcadedb.graph.Vertex; -import com.arcadedb.utility.Callable; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -46,11 +45,11 @@ protected void beginTest() { } for (int i = 1; i < 100; i++) { - final Vertex root = database.select().from("Vertex").where().property("id").eq().value(0).vertices().nextOrNull(); + final Vertex root = database.select().fromType("Vertex").where().property("id").eq().value(0).vertices().nextOrNull(); Assertions.assertNotNull(root); Assertions.assertEquals(0, root.getInteger("id")); - root.newEdge("Edge", database.select().from("Vertex").where().property("id").eq().value(i).vertices().nextOrNull(), true) + root.newEdge("Edge", database.select().fromType("Vertex").where().property("id").eq().value(i).vertices().nextOrNull(), true) .save(); } }); @@ -58,72 +57,81 @@ protected void beginTest() { @Test public void okReuse() { - final NativeSelect select = database.select().from("Vertex").where().property("id").eq().parameter("value"); + final NativeSelect select = database.select().fromType("Vertex").where().property("id").eq().parameter("value"); for (int i = 0; i < 100; i++) Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); } @Test public void okAnd() { - final NativeSelect select = database.select().from("Vertex")// + final NativeSelect select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// .and().property("name").eq().value("Elon"); for (int i = 0; i < 100; i++) Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); - final NativeSelect select2 = database.select().from("Vertex")// + final NativeSelect select2 = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// .and().property("name").eq().value("Elon2"); Assertions.assertFalse(select2.parameter("value", 3).vertices().hasNext()); - final NativeSelect select3 = database.select().from("Vertex")// + final NativeSelect select3 = database.select().fromType("Vertex")// .where().property("id").eq().value(-1)// .and().property("name").eq().value("Elon"); Assertions.assertFalse(select3.vertices().hasNext()); + + final NativeSelect select4 = database.select().fromType("Vertex")// + .where().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon")// + .and().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon"); + + for (int i = 0; i < 100; i++) + Assertions.assertEquals(i, select4.parameter("value", i).vertices().nextOrNull().getInteger("id")); } @Test public void okOr() { - final NativeSelect select = database.select().from("Vertex")// + final NativeSelect select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// - .or().property("name").eq().value("Elon2"); + .or().property("name").eq().value("Elon"); - Assertions.assertEquals(3, select.parameter("value", 3).vertices().nextOrNull().getInteger("id")); + for (QueryIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { + final Vertex v = result.next(); + Assertions.assertTrue(v.getInteger("id").equals(3) || v.getString("name").equals("Elon")); + } - final NativeSelect select2 = database.select().from("Vertex")// + final NativeSelect select2 = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// .or().property("name").eq().value("Elon2"); - Assertions.assertEquals(3, select2.parameter("value", 3).vertices().nextOrNull().getInteger("id")); + for (QueryIterator result = select2.parameter("value", 3).vertices(); result.hasNext(); ) { + final Vertex v = result.next(); + Assertions.assertTrue(v.getInteger("id").equals(3) || v.getString("name").equals("Elon2")); + } - final NativeSelect select3 = database.select().from("Vertex")// + final NativeSelect select3 = database.select().fromType("Vertex")// .where().property("id").eq().value(-1)// .or().property("name").eq().value("Elon"); - Assertions.assertEquals(0, select3.parameter("value", 3).vertices().nextOrNull().getInteger("id")); + for (QueryIterator result = select3.parameter("value", 3).vertices(); result.hasNext(); ) { + final Vertex v = result.next(); + Assertions.assertTrue(v.getInteger("id").equals(-1) || v.getString("name").equals("Elon")); + } } @Test public void okAndOr() { - final NativeSelect select = database.select().from("Vertex")// - .where().property("id").eq().parameter("value")// - .and().property("name").eq().value("Elon")// - .and().property("id").eq().parameter("value")// - .and().property("name").eq().value("Elon"); - - for (int i = 0; i < 100; i++) - Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); - - final NativeSelect select2 = database.select().from("Vertex")// + final NativeSelect select2 = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// .and().property("name").eq().value("Elon2"); Assertions.assertFalse(select2.parameter("value", 3).vertices().hasNext()); - final NativeSelect select3 = database.select().from("Vertex")// + final NativeSelect select3 = database.select().fromType("Vertex")// .where().property("id").eq().value(-1)// .and().property("name").eq().value("Elon"); @@ -133,7 +141,7 @@ public void okAndOr() { @Test public void okLimit() { - final NativeSelect select = database.select().from("Vertex")// + final NativeSelect select = database.select().fromType("Vertex")// .where().property("id").lt().value(10)// .and().property("name").eq().value("Elon").limit(10); @@ -147,7 +155,7 @@ public void okLimit() { @Test public void errorMissingParameter() { expectingException(() -> { - database.select().from("Vertex")// + database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// .vertices().nextOrNull(); }, IllegalArgumentException.class, "Missing parameter 'value'"); @@ -156,7 +164,7 @@ public void errorMissingParameter() { @Test public void errorMissingCondition() { expectingException(() -> { - database.select().from("Vertex")// + database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// .or().vertices().nextOrNull(); }, IllegalArgumentException.class, "Missing condition"); From e4a670b64d4ae5d165416d9457c72b4cb18e9c08 Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 5 Oct 2023 13:36:04 -0400 Subject: [PATCH 03/19] test: fixed test cases --- .../query/nativ/NativeSelectExecutionIT.java | 139 +++++++++++------- 1 file changed, 83 insertions(+), 56 deletions(-) diff --git a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java index d84bbe81fb..7fe58ce636 100644 --- a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java +++ b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java @@ -49,94 +49,114 @@ protected void beginTest() { Assertions.assertNotNull(root); Assertions.assertEquals(0, root.getInteger("id")); - root.newEdge("Edge", database.select().fromType("Vertex").where().property("id").eq().value(i).vertices().nextOrNull(), true) - .save(); + root.newEdge("Edge", database.select().fromType("Vertex").where().property("id").eq().value(i).vertices().nextOrNull(), + true).save(); } }); } - @Test - public void okReuse() { - final NativeSelect select = database.select().fromType("Vertex").where().property("id").eq().parameter("value"); - for (int i = 0; i < 100; i++) - Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); - } - @Test public void okAnd() { - final NativeSelect select = database.select().fromType("Vertex")// - .where().property("id").eq().parameter("value")// - .and().property("name").eq().value("Elon"); + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon"); - for (int i = 0; i < 100; i++) - Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); + for (int i = 0; i < 100; i++) + Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); + } - final NativeSelect select2 = database.select().fromType("Vertex")// - .where().property("id").eq().parameter("value")// - .and().property("name").eq().value("Elon2"); + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon2"); - Assertions.assertFalse(select2.parameter("value", 3).vertices().hasNext()); + Assertions.assertFalse(select.parameter("value", 3).vertices().hasNext()); + } - final NativeSelect select3 = database.select().fromType("Vertex")// - .where().property("id").eq().value(-1)// - .and().property("name").eq().value("Elon"); + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").eq().value(-1)// + .and().property("name").eq().value("Elon"); - Assertions.assertFalse(select3.vertices().hasNext()); + Assertions.assertFalse(select.vertices().hasNext()); + } - final NativeSelect select4 = database.select().fromType("Vertex")// - .where().property("id").eq().parameter("value")// - .and().property("name").eq().value("Elon")// - .and().property("id").eq().parameter("value")// - .and().property("name").eq().value("Elon"); + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon")// + .and().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon"); - for (int i = 0; i < 100; i++) - Assertions.assertEquals(i, select4.parameter("value", i).vertices().nextOrNull().getInteger("id")); + for (int i = 0; i < 100; i++) + Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); + } } @Test public void okOr() { - final NativeSelect select = database.select().fromType("Vertex")// - .where().property("id").eq().parameter("value")// - .or().property("name").eq().value("Elon"); + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").eq().parameter("value")// + .or().property("name").eq().value("Elon"); - for (QueryIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { - final Vertex v = result.next(); - Assertions.assertTrue(v.getInteger("id").equals(3) || v.getString("name").equals("Elon")); + for (QueryIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { + final Vertex v = result.next(); + Assertions.assertTrue(v.getInteger("id").equals(3) || v.getString("name").equals("Elon")); + } } - final NativeSelect select2 = database.select().fromType("Vertex")// - .where().property("id").eq().parameter("value")// - .or().property("name").eq().value("Elon2"); + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").eq().parameter("value")// + .or().property("name").eq().value("Elon2"); - for (QueryIterator result = select2.parameter("value", 3).vertices(); result.hasNext(); ) { - final Vertex v = result.next(); - Assertions.assertTrue(v.getInteger("id").equals(3) || v.getString("name").equals("Elon2")); + for (QueryIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { + final Vertex v = result.next(); + Assertions.assertTrue(v.getInteger("id").equals(3) || v.getString("name").equals("Elon2")); + } } - final NativeSelect select3 = database.select().fromType("Vertex")// - .where().property("id").eq().value(-1)// - .or().property("name").eq().value("Elon"); + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").eq().value(-1)// + .or().property("name").eq().value("Elon"); - for (QueryIterator result = select3.parameter("value", 3).vertices(); result.hasNext(); ) { - final Vertex v = result.next(); - Assertions.assertTrue(v.getInteger("id").equals(-1) || v.getString("name").equals("Elon")); + for (QueryIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { + final Vertex v = result.next(); + Assertions.assertTrue(v.getInteger("id").equals(-1) || v.getString("name").equals("Elon")); + } } } @Test public void okAndOr() { - final NativeSelect select2 = database.select().fromType("Vertex")// - .where().property("id").eq().parameter("value")// - .and().property("name").eq().value("Elon2"); - - Assertions.assertFalse(select2.parameter("value", 3).vertices().hasNext()); + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon2")// + .or().property("name").eq().value("Elon"); - final NativeSelect select3 = database.select().fromType("Vertex")// - .where().property("id").eq().value(-1)// - .and().property("name").eq().value("Elon"); + for (QueryIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { + final Vertex v = result.next(); + Assertions.assertTrue(v.getInteger("id").equals(3) && v.getString("name").equals("Elon2") ||// + v.getString("name").equals("Elon")); + } + } - Assertions.assertFalse(select3.vertices().hasNext()); + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").eq().parameter("value")// + .or().property("name").eq().value("Elon2")// + .and().property("name").eq().value("Elon"); + for (QueryIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { + final Vertex v = result.next(); + Assertions.assertTrue(v.getInteger("id").equals(3) ||// + v.getString("name").equals("Elon2") && v.getString("name").equals("Elon")); + } + } } @Test @@ -161,6 +181,13 @@ public void errorMissingParameter() { }, IllegalArgumentException.class, "Missing parameter 'value'"); } + @Test + public void okReuse() { + final NativeSelect select = database.select().fromType("Vertex").where().property("id").eq().parameter("value"); + for (int i = 0; i < 100; i++) + Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); + } + @Test public void errorMissingCondition() { expectingException(() -> { From 4067018b7ec4a6207d4df43bcf2fbfd66ab8fedd Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 5 Oct 2023 15:10:10 -0400 Subject: [PATCH 04/19] feat: native select, implemented to and from JSON --- .../arcadedb/query/nativ/NativeOperator.java | 40 ++-- .../query/nativ/NativePropertyValue.java | 1 + .../arcadedb/query/nativ/NativeSelect.java | 178 ++++++++++-------- .../query/nativ/NativeSelectExecutor.java | 107 +++++++++++ .../arcadedb/query/nativ/NativeTreeNode.java | 41 ++-- .../query/nativ/NativeSelectExecutionIT.java | 20 ++ 6 files changed, 284 insertions(+), 103 deletions(-) create mode 100644 engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java index 9ded4aa938..6284a1415b 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java @@ -14,9 +14,11 @@ * limitations under the License. */ +import com.arcadedb.database.Document; import com.arcadedb.serializer.BinaryComparator; import java.util.*; +import java.util.concurrent.*; /** * Native condition with support for simple operators through inheritance. @@ -24,79 +26,91 @@ * @author Luca Garulli (l.garulli@arcadedata.com) */ public enum NativeOperator { - or("or", 0) { + or("or", true, 0) { @Override Boolean eval(final Object left, final Object right) { return left == Boolean.TRUE || right == Boolean.TRUE; } }, - and("and", 2) { + and("and", true, 2) { @Override Boolean eval(final Object left, final Object right) { return left == Boolean.TRUE && right == Boolean.TRUE; } }, - not("not", 2) { + not("not", true, 2) { @Override Boolean eval(final Object left, final Object right) { return left == Boolean.FALSE; } }, - eq("==", 1) { + eq("=", false, 1) { @Override Object eval(final Object left, final Object right) { return BinaryComparator.equals(left, right); } }, - lt("<", 1) { + lt("<", false, 1) { @Override Object eval(final Object left, final Object right) { return BinaryComparator.compareTo(left, right) < 0; } }, - le("<=", 1) { + le("<=", false, 1) { @Override Object eval(final Object left, final Object right) { return BinaryComparator.compareTo(left, right) <= 0; } }, - gt(">", 1) { + gt(">", false, 1) { @Override Object eval(final Object left, final Object right) { return BinaryComparator.compareTo(left, right) > 0; } }, - ge(">=", 1) { + ge(">=", false, 1) { @Override Object eval(final Object left, final Object right) { return BinaryComparator.compareTo(left, right) >= 0; } }, - run("!", -1) { + run("!", true, -1) { @Override Object eval(final Object left, final Object right) { return left; } }; - NativeOperator(final String name, final int precedence) { + public final String name; + public final boolean logicOperator; + public final int precedence; + private static Map NAMES = new ConcurrentHashMap<>(); + + NativeOperator(final String name, final boolean logicOperator, final int precedence) { this.name = name; + this.logicOperator = logicOperator; this.precedence = precedence; } - public final String name; - public final int precedence; - abstract Object eval(Object left, Object right); + public static NativeOperator byName(final String name) { + if (NAMES.isEmpty()) { + for (NativeOperator v : values()) + NAMES.put(v.name, v); + } + + return NAMES.get(name); + } + @Override public String toString() { return name; diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativePropertyValue.java b/engine/src/main/java/com/arcadedb/query/nativ/NativePropertyValue.java index 67bcd31bc9..ddea336982 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativePropertyValue.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativePropertyValue.java @@ -1,6 +1,7 @@ package com.arcadedb.query.nativ; import com.arcadedb.database.Document; +import com.arcadedb.serializer.json.JSONArray; public class NativePropertyValue implements NativeRuntimeValue { public final String propertyName; diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java index 021d49a2f8..a14bb8509b 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java @@ -16,12 +16,12 @@ import com.arcadedb.database.DatabaseInternal; import com.arcadedb.database.Document; -import com.arcadedb.database.Record; import com.arcadedb.engine.Bucket; import com.arcadedb.graph.Edge; import com.arcadedb.graph.Vertex; import com.arcadedb.schema.DocumentType; -import com.arcadedb.utility.MultiIterator; +import com.arcadedb.serializer.json.JSONArray; +import com.arcadedb.serializer.json.JSONObject; import java.util.*; import java.util.stream.*; @@ -34,22 +34,22 @@ * @author Luca Garulli (l.garulli@arcadedata.com) */ public class NativeSelect { - private final DatabaseInternal database; + final DatabaseInternal database; enum STATE {DEFAULT, WHERE, COMPILED} Map parameters; - private NativeTreeNode rootTreeElement; - private NativeTreeNode lastTreeElement; - private DocumentType fromType; - private List fromBuckets; - private NativeOperator operator; - private NativeRuntimeValue property; - private Object propertyValue; - private boolean polymorphic = true; - private int limit = -1; - private STATE state = STATE.DEFAULT; + NativeTreeNode rootTreeElement; + DocumentType fromType; + List fromBuckets; + NativeOperator operator; + NativeRuntimeValue property; + Object propertyValue; + boolean polymorphic = true; + int limit = -1; + private STATE state = STATE.DEFAULT; + private NativeTreeNode lastTreeElement; public NativeSelect(final DatabaseInternal database) { this.database = database; @@ -174,92 +174,112 @@ public NativeSelect limit(final int limit) { public NativeSelect polymorphic(final boolean polymorphic) { checkNotCompiled(); + if (fromType == null) + throw new IllegalArgumentException("FromType was not set"); this.polymorphic = polymorphic; return this; } - public QueryIterator vertices() { - return run(); + public NativeSelect json(final JSONObject json) { + checkNotCompiled(); + if (json.has("fromType")) { + fromType(json.getString("fromType")); + if (json.has("polymorphic")) + polymorphic(json.getBoolean("polymorphic")); + } else if (json.has("fromBuckets")) { + final JSONArray buckets = json.getJSONArray("fromBuckets"); + fromBuckets( + buckets.toList().stream().map(b -> database.getSchema().getBucketByName(b.toString())).collect(Collectors.toList()) + .toArray(new String[buckets.length()])); + } + + if (json.has("where")) { + where(); + parseJsonCondition(json.getJSONArray("where")); + } + + if (json.has("limit")) + limit(json.getInt("limit")); + + return null; } - public QueryIterator edges() { - return run(); + private void parseJsonCondition(final JSONArray condition) { + if (condition.length() != 3) + throw new IllegalArgumentException("Invalid condition " + condition); + + final Object parsedLeft = condition.get(0); + if (parsedLeft instanceof JSONArray) + parseJsonCondition((JSONArray) parsedLeft); + else if (parsedLeft instanceof String && ((String) parsedLeft).startsWith(":")) + property(((String) parsedLeft).substring(1)); + else if (parsedLeft instanceof String && ((String) parsedLeft).startsWith("#")) + parameter(((String) parsedLeft).substring(1)); + else + throw new IllegalArgumentException("Unsupported value " + parsedLeft); + + final NativeOperator parsedOperator = NativeOperator.byName(condition.getString(1)); + + if (parsedOperator.logicOperator) + setLogic(parsedOperator); + else + setOperator(parsedOperator); + + final Object parsedRight = condition.get(2); + if (parsedRight instanceof JSONArray) + parseJsonCondition((JSONArray) parsedRight); + else if (parsedRight instanceof String && ((String) parsedRight).startsWith(":")) + property(((String) parsedRight).substring(1)); + else if (parsedRight instanceof String && ((String) parsedRight).startsWith("#")) + parameter(((String) parsedRight).substring(1)); + else + value(parsedRight); } - public QueryIterator documents() { - return run(); + public JSONObject json() { + final JSONObject json = new JSONObject(); + + if (fromType != null) { + json.put("fromType", fromType.getName()); + if (!polymorphic) + json.put("polymorphic", polymorphic); + } else if (fromBuckets != null) + json.put("fromBuckets", fromBuckets.stream().map(b -> b.getName()).collect(Collectors.toList())); + + if (rootTreeElement != null) + json.put("where", rootTreeElement.toJSON()); + + if (limit > -1) + json.put("limit", limit); + + return json; } - private QueryIterator run() { + public NativeSelect parse() { if (fromType == null && fromBuckets == null) throw new IllegalArgumentException("from (type or buckets) has not been set"); if (state != STATE.COMPILED) { setLogic(NativeOperator.run); state = STATE.COMPILED; } + return this; + } - final int[] returned = new int[] { 0 }; - final Iterator iterator; - - if (fromType != null) - iterator = database.iterateType(fromType.getName(), polymorphic); - else if (fromBuckets.size() == 1) - iterator = database.iterateBucket(fromBuckets.get(0).getName()); - else { - final MultiIterator multiIterator = new MultiIterator<>(); - for (Bucket b : fromBuckets) - multiIterator.addIterator(b.iterator()); - iterator = multiIterator; - } - - return new QueryIterator<>() { - private T next = null; - - @Override - public boolean hasNext() { - if (limit > -1 && returned[0] >= limit) - return false; - if (next != null) - return true; - if (!iterator.hasNext()) - return false; - - next = fetchNext(); - return next != null; - } - - @Override - public T next() { - if (next == null && !hasNext()) - throw new NoSuchElementException(); - try { - return next; - } finally { - next = null; - } - } - - private T fetchNext() { - do { - final Document record = iterator.next().asDocument(); - if (evaluateWhere(record)) { - ++returned[0]; - return (T) record; - } + public QueryIterator vertices() { + return run(); + } - } while (iterator.hasNext()); + public QueryIterator edges() { + return run(); + } - // NOT FOUND - return null; - } - }; + public QueryIterator documents() { + return run(); } - private boolean evaluateWhere(final Document record) { - final Object result = rootTreeElement.eval(record); - if (result instanceof Boolean) - return (Boolean) result; - throw new IllegalArgumentException("A boolean result was expected but '" + result + "' was returned"); + private QueryIterator run() { + parse(); + return new NativeSelectExecutor(this).execute(); } private NativeSelect setLogic(final NativeOperator newLogicOperator) { @@ -277,7 +297,7 @@ private NativeSelect setLogic(final NativeOperator newLogicOperator) { if (newLogicOperator.equals(NativeOperator.run)) { // EXECUTION = LAST NODE: APPEND TO THE RIGHT OF THE LATEST lastTreeElement.getParent().setRight(newTreeElement); - } else if (lastTreeElement.getParent().operator.precedence > newLogicOperator.precedence) { + } else if (lastTreeElement.getParent().operator.precedence < newLogicOperator.precedence) { // AND+ OPERATOR final NativeTreeNode newNode = new NativeTreeNode(newTreeElement, newLogicOperator, null); lastTreeElement.getParent().setRight(newNode); diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java new file mode 100644 index 0000000000..9dce9907fe --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java @@ -0,0 +1,107 @@ +package com.arcadedb.query.nativ;/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.arcadedb.database.DatabaseInternal; +import com.arcadedb.database.Document; +import com.arcadedb.database.Record; +import com.arcadedb.engine.Bucket; +import com.arcadedb.graph.Edge; +import com.arcadedb.graph.Vertex; +import com.arcadedb.schema.DocumentType; +import com.arcadedb.utility.MultiIterator; + +import java.util.*; +import java.util.stream.*; + +/** + * Native Query engine is a simple query engine that covers most of the classic use cases, such as the retrieval of records + * with a where condition. It could be much faster than the same SQL query because it does not use any parser and it is very + * JIT friendly. Future versions could translate the query into bytecode to have an even faster execution. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class NativeSelectExecutor { + private final NativeSelect select; + + public NativeSelectExecutor(final NativeSelect select) { + this.select = select; + } + + QueryIterator execute() { + final int[] returned = new int[] { 0 }; + final Iterator iterator; + + if (select.fromType != null) + iterator = select.database.iterateType(select.fromType.getName(), select.polymorphic); + else if (select.fromBuckets.size() == 1) + iterator = select.database.iterateBucket(select.fromBuckets.get(0).getName()); + else { + final MultiIterator multiIterator = new MultiIterator<>(); + for (Bucket b : select.fromBuckets) + multiIterator.addIterator(b.iterator()); + iterator = multiIterator; + } + + return new QueryIterator<>() { + private T next = null; + + @Override + public boolean hasNext() { + if (select.limit > -1 && returned[0] >= select.limit) + return false; + if (next != null) + return true; + if (!iterator.hasNext()) + return false; + + next = fetchNext(); + return next != null; + } + + @Override + public T next() { + if (next == null && !hasNext()) + throw new NoSuchElementException(); + try { + return next; + } finally { + next = null; + } + } + + private T fetchNext() { + do { + final Document record = iterator.next().asDocument(); + if (evaluateWhere(record)) { + ++returned[0]; + return (T) record; + } + + } while (iterator.hasNext()); + + // NOT FOUND + return null; + } + }; + } + + private boolean evaluateWhere(final Document record) { + final Object result = select.rootTreeElement.eval(record); + if (result instanceof Boolean) + return (Boolean) result; + throw new IllegalArgumentException("A boolean result was expected but '" + result + "' was returned"); + } +} diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java index 8e142484a9..96c9f96dcd 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java @@ -16,6 +16,8 @@ import com.arcadedb.database.Document; import com.arcadedb.query.sql.parser.BooleanExpression; +import com.arcadedb.serializer.json.JSONArray; +import com.arcadedb.serializer.json.JSONObject; import java.util.*; @@ -25,11 +27,10 @@ * @author Luca Garulli (l.garulli@arcadedata.com) */ public class NativeTreeNode { - public Object left; - public final NativeOperator operator; - private Object right; - private NativeTreeNode parent; - private List children; // TODO: REMOVE IT? + public Object left; + public final NativeOperator operator; + private Object right; + private NativeTreeNode parent; public NativeTreeNode(final Object left, final NativeOperator operator, final Object right) { this.left = left; @@ -63,12 +64,6 @@ else if (right instanceof NativeRuntimeValue) return operator.eval(leftValue, rightValue); } - public void addChild(final NativeTreeNode child) { - if (children == null) - children = new ArrayList<>(); - children.add(child); - } - public void setRight(final NativeTreeNode right) { if (this.right != null) throw new IllegalArgumentException("Cannot assign the right node because already assigned to " + this.right); @@ -109,4 +104,28 @@ public String toString() { buffer.append(" )"); return buffer.toString(); } + + public JSONArray toJSON() { + final JSONArray json = new JSONArray(); + + if (left instanceof NativeTreeNode) + json.put(((NativeTreeNode) left).toJSON()); + else if (left instanceof NativePropertyValue || left instanceof NativeParameterValue) + json.put(left.toString()); + else + json.put(left); + + if (operator != NativeOperator.run) + json.put(operator.name); + + if (right != null) { + if (right instanceof NativeTreeNode) + json.put(((NativeTreeNode) right).toJSON()); + else if (right instanceof NativePropertyValue || right instanceof NativeParameterValue) + json.put(right.toString()); + else + json.put(right); + } + return json; + } } diff --git a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java index 7fe58ce636..289ccbd833 100644 --- a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java +++ b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java @@ -20,6 +20,7 @@ import com.arcadedb.TestHelper; import com.arcadedb.graph.Vertex; +import com.arcadedb.serializer.json.JSONObject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -188,6 +189,25 @@ public void okReuse() { Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); } + @Test + public void okJSON() { + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon2")// + .or().property("name").eq().value("Elon").parse(); + + final JSONObject json = select.json(); + System.out.println(json.toString(2)); + + final NativeSelect select2 = database.select().json(json); + final JSONObject json2 = select.json(); + System.out.println(json.toString(2)); + + Assertions.assertEquals(json, json2); + } + } + @Test public void errorMissingCondition() { expectingException(() -> { From d07d06e9461364cfd5a7d8ba2e67c9e9745e2b0e Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 5 Oct 2023 15:12:41 -0400 Subject: [PATCH 05/19] fix: fixed native select json --- .../src/main/java/com/arcadedb/query/nativ/NativeSelect.java | 5 ++++- .../com/arcadedb/query/nativ/NativeSelectExecutionIT.java | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java index a14bb8509b..3916426c2c 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java @@ -201,7 +201,7 @@ public NativeSelect json(final JSONObject json) { if (json.has("limit")) limit(json.getInt("limit")); - return null; + return this; } private void parseJsonCondition(final JSONArray condition) { @@ -237,6 +237,9 @@ else if (parsedRight instanceof String && ((String) parsedRight).startsWith("#") } public JSONObject json() { + if (state != STATE.COMPILED) + parse(); + final JSONObject json = new JSONObject(); if (fromType != null) { diff --git a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java index 289ccbd833..5fba688d68 100644 --- a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java +++ b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java @@ -195,13 +195,13 @@ public void okJSON() { final NativeSelect select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// .and().property("name").eq().value("Elon2")// - .or().property("name").eq().value("Elon").parse(); + .or().property("name").eq().value("Elon"); final JSONObject json = select.json(); System.out.println(json.toString(2)); final NativeSelect select2 = database.select().json(json); - final JSONObject json2 = select.json(); + final JSONObject json2 = select2.json(); System.out.println(json.toString(2)); Assertions.assertEquals(json, json2); From c4e48c5cf885b52eef7974a6f486d15c776b7ef3 Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 5 Oct 2023 17:01:01 -0400 Subject: [PATCH 06/19] perf: speeded up logic for comparison in SQL engine --- .../sql/executor/QueryOperatorEquals.java | 53 ++++++++++--------- .../sql/function/math/SQLFunctionMax.java | 6 ++- .../sql/function/math/SQLFunctionMin.java | 6 ++- .../arcadedb/query/sql/parser/GeOperator.java | 14 ++--- .../arcadedb/query/sql/parser/GtOperator.java | 14 ++--- .../arcadedb/query/sql/parser/LeOperator.java | 24 +++++---- .../arcadedb/query/sql/parser/LtOperator.java | 14 ++--- .../main/java/com/arcadedb/schema/Type.java | 40 +++++++------- .../arcadedb/serializer/BinaryComparator.java | 12 ++++- 9 files changed, 101 insertions(+), 82 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/query/sql/executor/QueryOperatorEquals.java b/engine/src/main/java/com/arcadedb/query/sql/executor/QueryOperatorEquals.java index c5fc19ee8f..888dbd40ab 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/executor/QueryOperatorEquals.java +++ b/engine/src/main/java/com/arcadedb/query/sql/executor/QueryOperatorEquals.java @@ -22,58 +22,61 @@ import com.arcadedb.database.Identifiable; import com.arcadedb.database.RID; import com.arcadedb.schema.Type; +import com.arcadedb.serializer.BinaryComparator; import java.util.*; public class QueryOperatorEquals { - public static boolean equals(Object iLeft, Object iRight) { - if (iLeft == null || iRight == null) + public static boolean equals(Object left, Object right) { + if (left == null || right == null) return false; - if (iLeft == iRight) { + if (left == right) return true; - } - if (iLeft instanceof Result && !(iRight instanceof Result)) { - if (((Result) iLeft).isElement()) { - iLeft = ((Result) iLeft).toElement(); + if (left.getClass().equals(right.getClass())) + // SAME TYPE, NO CONVERSION + BinaryComparator.equals(left, right); + + if (left instanceof Result && !(right instanceof Result)) { + if (((Result) left).isElement()) { + left = ((Result) left).toElement(); } else { - return comparesValues(iRight, (Result) iLeft, true); + return comparesValues(right, (Result) left, true); } } - if (iRight instanceof Result && !(iLeft instanceof Result)) { - if (((Result) iRight).isElement()) { - iRight = ((Result) iRight).toElement(); + if (right instanceof Result && !(left instanceof Result)) { + if (((Result) right).isElement()) { + right = ((Result) right).toElement(); } else { - return comparesValues(iLeft, (Result) iRight, true); + return comparesValues(left, (Result) right, true); } } // RECORD & ORID - if (iLeft instanceof Identifiable) - return comparesValues(iRight, (Identifiable) iLeft, true); - else if (iRight instanceof Identifiable) - return comparesValues(iLeft, (Identifiable) iRight, true); - else if (iRight instanceof Result) - return comparesValues(iLeft, (Result) iRight, true); + if (left instanceof Identifiable) + return comparesValues(right, (Identifiable) left, true); + else if (right instanceof Identifiable) + return comparesValues(left, (Identifiable) right, true); + else if (right instanceof Result) + return comparesValues(left, (Result) right, true); // NUMBERS - if (iLeft instanceof Number && iRight instanceof Number) { - final Number[] couple = Type.castComparableNumber((Number) iLeft, (Number) iRight); + if (left instanceof Number && right instanceof Number) { + final Number[] couple = Type.castComparableNumber((Number) left, (Number) right); return couple[0].equals(couple[1]); } // ALL OTHER CASES try { - final Object right = Type.convert(null, iRight, iLeft.getClass()); - + right = Type.convert(null, right, left.getClass()); if (right == null) return false; - if (iLeft instanceof byte[] && iRight instanceof byte[]) { - return Arrays.equals((byte[]) iLeft, (byte[]) iRight); + if (left instanceof byte[] && right instanceof byte[]) { + return Arrays.equals((byte[]) left, (byte[]) right); } - return iLeft.equals(right); + return BinaryComparator.equals(left, right); } catch (final Exception ignore) { return false; } diff --git a/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMax.java b/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMax.java index 15560b2650..3442a6883b 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMax.java +++ b/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMax.java @@ -52,7 +52,8 @@ public Object execute(final Object iThis, final Identifiable iCurrentRecord, fin max = subitem; } } else { - if ((item instanceof Number) && (max instanceof Number)) { + if (!item.getClass().equals(max.getClass()) &&// + item instanceof Number && max instanceof Number) { final Number[] converted = Type.castComparableNumber((Number) item, (Number) max); item = converted[0]; max = converted[1]; @@ -88,7 +89,8 @@ public Object execute(final Object iThis, final Identifiable iCurrentRecord, fin public boolean aggregateResults() { // LET definitions (contain $current) does not require results aggregation - return configuredParameters != null && ((configuredParameters.length == 1) && !configuredParameters[0].toString().contains("$current")); + return configuredParameters != null && ((configuredParameters.length == 1) && !configuredParameters[0].toString() + .contains("$current")); } public String getSyntax() { diff --git a/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMin.java b/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMin.java index 1188ac3212..a6e2b6fd84 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMin.java +++ b/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMin.java @@ -53,7 +53,8 @@ public Object execute(final Object iThis, final Identifiable iCurrentRecord, fin min = subitem; } } else { - if ((item instanceof Number) && (min instanceof Number)) { + if (!item.getClass().equals(min.getClass()) &&// + item instanceof Number && min instanceof Number) { final Number[] converted = Type.castComparableNumber((Number) item, (Number) min); item = converted[0]; min = converted[1]; @@ -90,7 +91,8 @@ public Object execute(final Object iThis, final Identifiable iCurrentRecord, fin public boolean aggregateResults() { // LET definitions (contain $current) does not require results aggregation - return configuredParameters != null && ((configuredParameters.length == 1) && !configuredParameters[0].toString().contains("$current")); + return configuredParameters != null && ((configuredParameters.length == 1) && !configuredParameters[0].toString() + .contains("$current")); } public String getSyntax() { diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/GeOperator.java b/engine/src/main/java/com/arcadedb/query/sql/parser/GeOperator.java index 9af3b883c5..3bf0778fcb 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/GeOperator.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/GeOperator.java @@ -38,12 +38,14 @@ public boolean execute(final DatabaseInternal database, Object left, Object righ if (left == null || right == null) return false; - if (left.getClass() != right.getClass() && left instanceof Number && right instanceof Number) { - final Number[] couple = Type.castComparableNumber((Number) left, (Number) right); - left = couple[0]; - right = couple[1]; - } else - right = Type.convert(database, right, left.getClass()); + if (!left.getClass().equals(right.getClass())) { + if (left instanceof Number && right instanceof Number) { + final Number[] couple = Type.castComparableNumber((Number) left, (Number) right); + left = couple[0]; + right = couple[1]; + } else + right = Type.convert(database, right, left.getClass()); + } if (right == null) return false; diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/GtOperator.java b/engine/src/main/java/com/arcadedb/query/sql/parser/GtOperator.java index c136f9c639..9cc23292c7 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/GtOperator.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/GtOperator.java @@ -35,12 +35,14 @@ public boolean execute(final DatabaseInternal database, Object left, Object righ if (left == null || right == null) return false; - if (left.getClass() != right.getClass() && left instanceof Number && right instanceof Number) { - final Number[] couple = Type.castComparableNumber((Number) left, (Number) right); - left = couple[0]; - right = couple[1]; - } else - right = Type.convert(database, right, left.getClass()); + if (!left.getClass().equals(right.getClass())) { + if (left instanceof Number && right instanceof Number) { + final Number[] couple = Type.castComparableNumber((Number) left, (Number) right); + left = couple[0]; + right = couple[1]; + } else + right = Type.convert(database, right, left.getClass()); + } if (right == null) return false; diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/LeOperator.java b/engine/src/main/java/com/arcadedb/query/sql/parser/LeOperator.java index 2635beccd0..3cd73abbda 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/LeOperator.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/LeOperator.java @@ -30,24 +30,26 @@ public LeOperator(final int id) { } @Override - public boolean execute(final DatabaseInternal database, Object iLeft, Object iRight) { - if (iLeft == iRight) + public boolean execute(final DatabaseInternal database, Object left, Object right) { + if (left == right) return true; - if (iLeft == null || iRight == null) + if (left == null || right == null) return false; - if (iLeft.getClass() != iRight.getClass() && iLeft instanceof Number && iRight instanceof Number) { - final Number[] couple = Type.castComparableNumber((Number) iLeft, (Number) iRight); - iLeft = couple[0]; - iRight = couple[1]; - } else { - iRight = Type.convert(database, iRight, iLeft.getClass()); + if (!left.getClass().equals(right.getClass())) { + if (left instanceof Number && right instanceof Number) { + final Number[] couple = Type.castComparableNumber((Number) left, (Number) right); + left = couple[0]; + right = couple[1]; + } else { + right = Type.convert(database, right, left.getClass()); + } } - if (iRight == null) + if (right == null) return false; - return BinaryComparator.compareTo(iLeft, iRight) <= 0; + return BinaryComparator.compareTo(left, right) <= 0; } @Override diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/LtOperator.java b/engine/src/main/java/com/arcadedb/query/sql/parser/LtOperator.java index 5da35a86f9..c2066b5228 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/LtOperator.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/LtOperator.java @@ -34,12 +34,14 @@ public boolean execute(final DatabaseInternal database, Object left, Object righ if (left == null || right == null) return false; - if (left instanceof Number && right instanceof Number && left.getClass() != right.getClass()) { - final Number[] couple = Type.castComparableNumber((Number) left, (Number) right); - left = couple[0]; - right = couple[1]; - } else { - right = Type.convert(database, right, left.getClass()); + if (!left.getClass().equals(right.getClass())) { + if (left instanceof Number && right instanceof Number) { + final Number[] couple = Type.castComparableNumber((Number) left, (Number) right); + left = couple[0]; + right = couple[1]; + } else { + right = Type.convert(database, right, left.getClass()); + } } if (right == null) diff --git a/engine/src/main/java/com/arcadedb/schema/Type.java b/engine/src/main/java/com/arcadedb/schema/Type.java index a67adce761..a46ec9d5f5 100644 --- a/engine/src/main/java/com/arcadedb/schema/Type.java +++ b/engine/src/main/java/com/arcadedb/schema/Type.java @@ -94,13 +94,15 @@ public enum Type { ARRAY_OF_SHORTS("Short[]", 19, BinaryTypes.TYPE_ARRAY_OF_SHORTS, short[].class, new Class[] { short[].class, Short[].class }), - ARRAY_OF_INTEGERS("Integer[]", 20, BinaryTypes.TYPE_ARRAY_OF_INTEGERS, int[].class, new Class[] { int[].class, Integer[].class }), + ARRAY_OF_INTEGERS("Integer[]", 20, BinaryTypes.TYPE_ARRAY_OF_INTEGERS, int[].class, + new Class[] { int[].class, Integer[].class }), ARRAY_OF_LONGS("Long[]", 21, BinaryTypes.TYPE_ARRAY_OF_LONGS, long[].class, new Class[] { long[].class, Long[].class }), ARRAY_OF_FLOATS("Float[]", 22, BinaryTypes.TYPE_ARRAY_OF_FLOATS, float[].class, new Class[] { float[].class, Float[].class }), - ARRAY_OF_DOUBLES("Double[]", 23, BinaryTypes.TYPE_ARRAY_OF_DOUBLES, double[].class, new Class[] { double[].class, Double[].class }), + ARRAY_OF_DOUBLES("Double[]", 23, BinaryTypes.TYPE_ARRAY_OF_DOUBLES, double[].class, + new Class[] { double[].class, Double[].class }), ; // Don't change the order, the type discover get broken if you change the order. @@ -171,7 +173,8 @@ public enum Type { protected final Class[] allowAssignmentFrom; protected final Set castable; - Type(final String iName, final int iId, final byte binaryType, final Class iJavaDefaultType, final Class[] iAllowAssignmentBy) { + Type(final String iName, final int iId, final byte binaryType, final Class iJavaDefaultType, + final Class[] iAllowAssignmentBy) { this.name = iName.toUpperCase(); this.id = iId; this.binaryType = binaryType; @@ -280,19 +283,6 @@ public static Type getTypeByName(final String name) { return TYPES_BY_NAME.get(name.toLowerCase()); } - public static boolean isSimpleType(final Object iObject) { - if (iObject == null) - return false; - - final Class iType = iObject.getClass(); - - return iType.isPrimitive() || Number.class.isAssignableFrom(iType) || String.class.isAssignableFrom(iType) || Boolean.class.isAssignableFrom(iType) - || Date.class.isAssignableFrom(iType) || (iType.isArray() && (iType.equals(byte[].class) || iType.equals(char[].class) || iType.equals(int[].class) - || iType.equals(long[].class) || iType.equals(double[].class) || iType.equals(float[].class) || iType.equals(short[].class) || iType.equals( - Integer[].class) || iType.equals(String[].class) || iType.equals(Long[].class) || iType.equals(Short[].class) || iType.equals(Double[].class))); - - } - /** * Convert types based on the iTargetClass parameter. * @@ -487,7 +477,8 @@ else if (iValue instanceof String) { return LocalDateTime.parse(valueAsString); } catch (Exception e) { try { - return LocalDateTime.parse(valueAsString, DateTimeFormatter.ofPattern((database.getSchema().getDateTimeFormat()))); + return LocalDateTime.parse(valueAsString, + DateTimeFormatter.ofPattern((database.getSchema().getDateTimeFormat()))); } catch (final DateTimeParseException ignore) { return LocalDateTime.parse(valueAsString, DateTimeFormatter.ofPattern((database.getSchema().getDateFormat()))); } @@ -553,7 +544,8 @@ else if (iValue instanceof Calendar) try { result.add(new RID(database, iValue.toString())); } catch (final Exception e) { - LogManager.instance().log(Type.class, Level.FINE, "Error in conversion of value '%s' to type '%s'", e, iValue, targetClass); + LogManager.instance() + .log(Type.class, Level.FINE, "Error in conversion of value '%s' to type '%s'", e, iValue, targetClass); } } } @@ -562,7 +554,8 @@ else if (iValue instanceof Calendar) try { return new RID(database, (String) iValue); } catch (final Exception e) { - LogManager.instance().log(Type.class, Level.FINE, "Error in conversion of value '%s' to type '%s'", e, iValue, targetClass); + LogManager.instance() + .log(Type.class, Level.FINE, "Error in conversion of value '%s' to type '%s'", e, iValue, targetClass); } } } @@ -683,7 +676,8 @@ else if (b instanceof BigDecimal) } - throw new IllegalArgumentException("Cannot increment value '" + a + "' (" + a.getClass() + ") with '" + b + "' (" + b.getClass() + ")"); + throw new IllegalArgumentException( + "Cannot increment value '" + a + "' (" + a.getClass() + ") with '" + b + "' (" + b.getClass() + ")"); } public static Number decrement(final Number a, final Number b) { @@ -792,7 +786,8 @@ else if (b instanceof BigDecimal) } - throw new IllegalArgumentException("Cannot decrement value '" + a + "' (" + a.getClass() + ") with '" + b + "' (" + b.getClass() + ")"); + throw new IllegalArgumentException( + "Cannot decrement value '" + a + "' (" + a.getClass() + ") with '" + b + "' (" + b.getClass() + ")"); } public static Number[] castComparableNumber(Number left, Number right) { @@ -851,7 +846,8 @@ else if (right instanceof Byte || right instanceof Short || right instanceof Int // DOUBLE if (right instanceof BigDecimal) left = BigDecimal.valueOf(left.doubleValue()); - else if (right instanceof Byte || right instanceof Short || right instanceof Integer || right instanceof Long || right instanceof Float) + else if (right instanceof Byte || right instanceof Short || right instanceof Integer || right instanceof Long + || right instanceof Float) right = right.doubleValue(); } else if (left instanceof BigDecimal) { diff --git a/engine/src/main/java/com/arcadedb/serializer/BinaryComparator.java b/engine/src/main/java/com/arcadedb/serializer/BinaryComparator.java index 927e1affd6..161dcc7ebf 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinaryComparator.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinaryComparator.java @@ -21,6 +21,7 @@ import com.arcadedb.database.Binary; import com.arcadedb.database.DatabaseFactory; import com.arcadedb.database.Identifiable; +import com.arcadedb.schema.Type; import com.arcadedb.utility.CollectionUtils; import com.arcadedb.utility.DateUtils; @@ -140,7 +141,8 @@ public int compare(final Object value1, final byte type1, final Object value2, f if (value2 instanceof byte[]) return UnsignedBytesComparator.BEST_COMPARATOR.compare((byte[]) value1, (byte[]) value2); else - return UnsignedBytesComparator.BEST_COMPARATOR.compare((byte[]) value1, ((String) value2).getBytes(DatabaseFactory.getDefaultCharset())); + return UnsignedBytesComparator.BEST_COMPARATOR.compare((byte[]) value1, + ((String) value2).getBytes(DatabaseFactory.getDefaultCharset())); } return ((String) value1).compareTo(value2.toString()); @@ -378,6 +380,11 @@ else if (a instanceof byte[] && b instanceof byte[]) return equalsBytes((byte[]) a, (byte[]) b); else if (a instanceof Binary && b instanceof Binary) return equalsBinary((Binary) a, (Binary) b); + else if (!a.getClass().equals(b.getClass()) &&// + a instanceof Number && b instanceof Number) { + final Number[] pair = Type.castComparableNumber((Number) a, (Number) b); + return pair[0].equals(pair[1]); + } return a.equals(b); } @@ -385,7 +392,8 @@ public static boolean equalsString(final String buffer1, final String buffer2) { if (buffer1 == null || buffer2 == null) return false; - return equalsBytes(buffer1.getBytes(DatabaseFactory.getDefaultCharset()), buffer2.getBytes(DatabaseFactory.getDefaultCharset())); + return equalsBytes(buffer1.getBytes(DatabaseFactory.getDefaultCharset()), + buffer2.getBytes(DatabaseFactory.getDefaultCharset())); } public static boolean equalsBytes(final byte[] buffer1, final byte[] buffer2) { From 2c860b6113fba77ebe1d9f65944c8523887cf9d0 Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 5 Oct 2023 17:21:07 -0400 Subject: [PATCH 07/19] feat: native query now supports the timeout --- .../query/nativ/NativePropertyValue.java | 1 - .../arcadedb/query/nativ/NativeSelect.java | 10 +++++++++ .../query/nativ/NativeSelectExecutor.java | 21 +++++++++++-------- .../arcadedb/query/nativ/QueryIterator.java | 7 +++++++ .../query/nativ/NativeSelectExecutionIT.java | 2 -- 5 files changed, 29 insertions(+), 12 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativePropertyValue.java b/engine/src/main/java/com/arcadedb/query/nativ/NativePropertyValue.java index ddea336982..67bcd31bc9 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativePropertyValue.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativePropertyValue.java @@ -1,7 +1,6 @@ package com.arcadedb.query.nativ; import com.arcadedb.database.Document; -import com.arcadedb.serializer.json.JSONArray; public class NativePropertyValue implements NativeRuntimeValue { public final String propertyName; diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java index 3916426c2c..37f1e5d32e 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java @@ -24,6 +24,7 @@ import com.arcadedb.serializer.json.JSONObject; import java.util.*; +import java.util.concurrent.*; import java.util.stream.*; /** @@ -48,6 +49,8 @@ enum STATE {DEFAULT, WHERE, COMPILED} Object propertyValue; boolean polymorphic = true; int limit = -1; + long timeoutValue; + TimeUnit timeoutUnit; private STATE state = STATE.DEFAULT; private NativeTreeNode lastTreeElement; @@ -172,6 +175,13 @@ public NativeSelect limit(final int limit) { return this; } + public NativeSelect timeout(final long timeoutValue, final TimeUnit timeoutUnit) { + checkNotCompiled(); + this.timeoutValue = timeoutValue; + this.timeoutUnit = timeoutUnit; + return this; + } + public NativeSelect polymorphic(final boolean polymorphic) { checkNotCompiled(); if (fromType == null) diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java index 9dce9907fe..fa318b32c8 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java @@ -14,17 +14,11 @@ * limitations under the License. */ -import com.arcadedb.database.DatabaseInternal; import com.arcadedb.database.Document; -import com.arcadedb.database.Record; import com.arcadedb.engine.Bucket; -import com.arcadedb.graph.Edge; -import com.arcadedb.graph.Vertex; -import com.arcadedb.schema.DocumentType; import com.arcadedb.utility.MultiIterator; import java.util.*; -import java.util.stream.*; /** * Native Query engine is a simple query engine that covers most of the classic use cases, such as the retrieval of records @@ -35,6 +29,7 @@ */ public class NativeSelectExecutor { private final NativeSelect select; + private long evaluatedRecords = 0; public NativeSelectExecutor(final NativeSelect select) { this.select = select; @@ -42,12 +37,12 @@ public NativeSelectExecutor(final NativeSelect select) { QueryIterator execute() { final int[] returned = new int[] { 0 }; - final Iterator iterator; + final MultiIterator iterator; if (select.fromType != null) - iterator = select.database.iterateType(select.fromType.getName(), select.polymorphic); + iterator = (MultiIterator) select.database.iterateType(select.fromType.getName(), select.polymorphic); else if (select.fromBuckets.size() == 1) - iterator = select.database.iterateBucket(select.fromBuckets.get(0).getName()); + iterator = (MultiIterator) select.database.iterateBucket(select.fromBuckets.get(0).getName()); else { final MultiIterator multiIterator = new MultiIterator<>(); for (Bucket b : select.fromBuckets) @@ -55,6 +50,9 @@ else if (select.fromBuckets.size() == 1) iterator = multiIterator; } + if (select.timeoutUnit != null) + iterator.setTimeout(select.timeoutUnit.toMillis(select.timeoutValue)); + return new QueryIterator<>() { private T next = null; @@ -98,8 +96,13 @@ private T fetchNext() { }; } + public long getEvaluatedRecords() { + return evaluatedRecords; + } + private boolean evaluateWhere(final Document record) { final Object result = select.rootTreeElement.eval(record); + evaluatedRecords++; if (result instanceof Boolean) return (Boolean) result; throw new IllegalArgumentException("A boolean result was expected but '" + result + "' was returned"); diff --git a/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java b/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java index f81c9b3a6f..ed9e577afc 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java @@ -32,4 +32,11 @@ public abstract class QueryIterator implements Iterator { public T nextOrNull() { return hasNext() ? next() : null; } + + public List toList() { + final List result = new ArrayList<>(); + while (hasNext()) + result.add(next()); + return result; + } } diff --git a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java index 5fba688d68..c3d65fc909 100644 --- a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java +++ b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java @@ -198,11 +198,9 @@ public void okJSON() { .or().property("name").eq().value("Elon"); final JSONObject json = select.json(); - System.out.println(json.toString(2)); final NativeSelect select2 = database.select().json(json); final JSONObject json2 = select2.json(); - System.out.println(json.toString(2)); Assertions.assertEquals(json, json2); } From 11784d767c7ab6cdf75620b5ce396f1e1784b3ad Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 5 Oct 2023 18:09:18 -0400 Subject: [PATCH 08/19] perf: implemented lazy evaluation (speedup only when the operator need only one value) --- .../arcadedb/query/nativ/NativeOperator.java | 49 ++++++++++++------- .../query/nativ/NativeSelectExecutor.java | 10 ++++ .../arcadedb/query/nativ/NativeTreeNode.java | 18 +------ 3 files changed, 42 insertions(+), 35 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java index 6284a1415b..ce3c4c8078 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java @@ -28,64 +28,77 @@ public enum NativeOperator { or("or", true, 0) { @Override - Boolean eval(final Object left, final Object right) { - return left == Boolean.TRUE || right == Boolean.TRUE; + Object eval(final Document record, final Object left, final Object right) { + final Boolean leftValue = (Boolean) NativeSelectExecutor.evaluateValue(record, left); + if (leftValue) + return true; + + return NativeSelectExecutor.evaluateValue(record, right); } }, and("and", true, 2) { @Override - Boolean eval(final Object left, final Object right) { - return left == Boolean.TRUE && right == Boolean.TRUE; + Object eval(final Document record, final Object left, final Object right) { + final Boolean leftValue = (Boolean) NativeSelectExecutor.evaluateValue(record, left); + if (!leftValue) + return false; + + return NativeSelectExecutor.evaluateValue(record, right); } }, not("not", true, 2) { @Override - Boolean eval(final Object left, final Object right) { + Object eval(final Document record, final Object left, final Object right) { return left == Boolean.FALSE; } }, eq("=", false, 1) { @Override - Object eval(final Object left, final Object right) { - return BinaryComparator.equals(left, right); + Object eval(final Document record, final Object left, final Object right) { + return BinaryComparator.equals(NativeSelectExecutor.evaluateValue(record, left), + NativeSelectExecutor.evaluateValue(record, right)); } }, lt("<", false, 1) { @Override - Object eval(final Object left, final Object right) { - return BinaryComparator.compareTo(left, right) < 0; + Object eval(final Document record, final Object left, final Object right) { + return BinaryComparator.compareTo(NativeSelectExecutor.evaluateValue(record, left), + NativeSelectExecutor.evaluateValue(record, right)) < 0; } }, le("<=", false, 1) { @Override - Object eval(final Object left, final Object right) { - return BinaryComparator.compareTo(left, right) <= 0; + Object eval(final Document record, final Object left, final Object right) { + return BinaryComparator.compareTo(NativeSelectExecutor.evaluateValue(record, left), + NativeSelectExecutor.evaluateValue(record, right)) <= 0; } }, gt(">", false, 1) { @Override - Object eval(final Object left, final Object right) { - return BinaryComparator.compareTo(left, right) > 0; + Object eval(final Document record, final Object left, final Object right) { + return BinaryComparator.compareTo(NativeSelectExecutor.evaluateValue(record, left), + NativeSelectExecutor.evaluateValue(record, right)) > 0; } }, ge(">=", false, 1) { @Override - Object eval(final Object left, final Object right) { - return BinaryComparator.compareTo(left, right) >= 0; + Object eval(final Document record, final Object left, final Object right) { + return BinaryComparator.compareTo(NativeSelectExecutor.evaluateValue(record, left), + NativeSelectExecutor.evaluateValue(record, right)) >= 0; } }, run("!", true, -1) { @Override - Object eval(final Object left, final Object right) { - return left; + Object eval(final Document record, final Object left, final Object right) { + return NativeSelectExecutor.evaluateValue(record, NativeSelectExecutor.evaluateValue(record, right)); } }; @@ -100,7 +113,7 @@ Object eval(final Object left, final Object right) { this.precedence = precedence; } - abstract Object eval(Object left, Object right); + abstract Object eval(final Document record, Object left, Object right); public static NativeOperator byName(final String name) { if (NAMES.isEmpty()) { diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java index fa318b32c8..2008402006 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java @@ -96,6 +96,16 @@ private T fetchNext() { }; } + public static Object evaluateValue(final Document record, final Object value) { + if (value == null) + return null; + else if (value instanceof NativeTreeNode) + return ((NativeTreeNode) value).eval(record); + else if (value instanceof NativeRuntimeValue) + return ((NativeRuntimeValue) value).eval(record); + return value; + } + public long getEvaluatedRecords() { return evaluatedRecords; } diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java index 96c9f96dcd..f2e9700a2c 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java @@ -45,23 +45,7 @@ public NativeTreeNode(final Object left, final NativeOperator operator, final Ob } public Object eval(final Document record) { - final Object leftValue; - if (left instanceof NativeTreeNode) - leftValue = ((NativeTreeNode) left).eval(record); - else if (left instanceof NativeRuntimeValue) - leftValue = ((NativeRuntimeValue) left).eval(record); - else - leftValue = left; - - final Object rightValue; - if (right instanceof NativeTreeNode) - rightValue = ((NativeTreeNode) right).eval(record); - else if (right instanceof NativeRuntimeValue) - rightValue = ((NativeRuntimeValue) right).eval(record); - else - rightValue = right; - - return operator.eval(leftValue, rightValue); + return operator.eval(record, left, right); } public void setRight(final NativeTreeNode right) { From 1da0a0eb7123d594b8743bc1c448c311b4dbfdfb Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 5 Oct 2023 18:10:27 -0400 Subject: [PATCH 09/19] test: added some benchmark to activate on demand --- .../performance/LocalDatabaseBenchmark.java | 54 ++++++++++++++++--- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/engine/src/test/java/performance/LocalDatabaseBenchmark.java b/engine/src/test/java/performance/LocalDatabaseBenchmark.java index 8bd26538eb..80f13f2fa1 100644 --- a/engine/src/test/java/performance/LocalDatabaseBenchmark.java +++ b/engine/src/test/java/performance/LocalDatabaseBenchmark.java @@ -26,14 +26,18 @@ import com.arcadedb.database.DatabaseInternal; import com.arcadedb.exception.ConcurrentModificationException; import com.arcadedb.graph.MutableVertex; +import com.arcadedb.query.nativ.NativeSelect; +import com.arcadedb.query.sql.executor.ResultSet; import org.junit.jupiter.api.Assertions; import java.util.*; +import java.util.concurrent.*; import java.util.concurrent.atomic.*; public class LocalDatabaseBenchmark { - private static final int TOTAL = 10_000_000; - private static final int BATCH_TX = 200; + private static final int TOTAL = 1_000; + private static final int BATCH_TX = 200; + private static final int PRINT_EVERY_MS = 1_000; private static final int BUCKETS = 7; private static final int CONCURRENT_THREADS = BUCKETS; @@ -120,9 +124,43 @@ public void run() { Assertions.assertEquals(TOTAL * CONCURRENT_THREADS, database.countType("User", true)); +// queryNative(); +// querySQL(); +// queryNative(); + database.close(); } + private void queryNative() { + final long begin = System.currentTimeMillis(); + final NativeSelect cached = database.select().fromType("User").where()// + .property("id").eq().parameter("id").and()// + .property("id").eq().parameter("id").and()// + .property("id").eq().parameter("id").and()// + .property("id").eq().parameter("id").and()// + .property("id").eq().parameter("id").and()// + .property("id").eq().parameter("id").and()// + .property("id").eq().parameter("id").and()// + .property("id").eq().parameter("id").and()// + .property("id").eq().parameter("id").and()// + .property("id").eq().parameter("id")// + ; + for (int i = 0; i < TOTAL * CONCURRENT_THREADS; i++) { + Assertions.assertEquals(1, cached.parameter("id", i).vertices().toList().size()); + } + System.out.println("NATIVE " + (System.currentTimeMillis() - begin) + "ms"); + } + + private void querySQL() { + long begin = System.currentTimeMillis(); + for (int i = 0; i < TOTAL * CONCURRENT_THREADS; i++) { + Assertions.assertEquals(1, database.query("sql", + "select from User where id = ? and id = ? and id = ? and id = ? and id = ? and id = ? and id = ? and id = ? and id = ? and id = ?", + i, i, i, i, i, i, i, i, i, i).toVertices().size()); + } + System.out.println("SQL " + (System.currentTimeMillis() - begin) + "ms"); + } + private List checkRecordSequence(final Database database) { final List allIds = new ArrayList<>(); database.iterateType("User", true).forEachRemaining((a) -> allIds.add(a.getRecord().asVertex().getLong("id"))); @@ -185,7 +223,7 @@ private void executeInThread(final int threadId) { } catch (Throwable t) { incrementError(t); } finally { - mergeStats(((DatabaseInternal) database).getStats()); + mergeStats(database.getStats()); } } @@ -201,12 +239,12 @@ private long printStats(long beginTime) { final long delta = now - beginTime; beginTime = System.currentTimeMillis(); System.out.println( - ((globalCounter.get() - lastCounter.get()) * PRINT_EVERY_MS / (float) delta) + " req/sec (counter=" + globalCounter.get() + "/" + (CONCURRENT_THREADS - * TOTAL) + ", conflicts=" + concurrentExceptions.get() + ", errors=" + errors.get() + ")"); + ((globalCounter.get() - lastCounter.get()) * PRINT_EVERY_MS / (float) delta) + " req/sec (counter=" + globalCounter.get() + + "/" + (CONCURRENT_THREADS * TOTAL) + ", conflicts=" + concurrentExceptions.get() + ", errors=" + errors.get() + + ")"); } else { - System.out.println( - "COMPLETED (counter=" + globalCounter.get() + "/" + (CONCURRENT_THREADS * TOTAL) + ", conflicts=" + concurrentExceptions.get() + ", errors=" - + errors.get() + ")"); + System.out.println("COMPLETED (counter=" + globalCounter.get() + "/" + (CONCURRENT_THREADS * TOTAL) + ", conflicts=" + + concurrentExceptions.get() + ", errors=" + errors.get() + ")"); } lastCounter.set(globalCounter.get()); From 4406da4559d979f45fc01410508c5325e37c14b5 Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 5 Oct 2023 18:17:51 -0400 Subject: [PATCH 10/19] test: native select, added test case for timeout --- .../arcadedb/query/nativ/NativeOperator.java | 2 +- .../query/nativ/NativeSelectExecutionIT.java | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java index ce3c4c8078..ad5ebc56fe 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java @@ -98,7 +98,7 @@ Object eval(final Document record, final Object left, final Object right) { run("!", true, -1) { @Override Object eval(final Document record, final Object left, final Object right) { - return NativeSelectExecutor.evaluateValue(record, NativeSelectExecutor.evaluateValue(record, right)); + return NativeSelectExecutor.evaluateValue(record, left); } }; diff --git a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java index c3d65fc909..36dbf0e8db 100644 --- a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java +++ b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java @@ -19,11 +19,14 @@ package com.arcadedb.query.nativ; import com.arcadedb.TestHelper; +import com.arcadedb.exception.TimeoutException; import com.arcadedb.graph.Vertex; import com.arcadedb.serializer.json.JSONObject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.concurrent.*; + /** * @author Luca Garulli (l.garulli@arcadedata.com) */ @@ -173,6 +176,25 @@ public void okLimit() { Assertions.assertFalse(iter.hasNext()); } + @Test + public void errorTimeout() { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").lt().value(10)// + .and().property("name").eq().value("Elon").timeout(1, TimeUnit.MILLISECONDS); + + expectingException(() -> { + final QueryIterator iter = select.vertices(); + while (iter.hasNext()) { + Assertions.assertTrue(iter.next().getInteger("id") < 10); + try { + Thread.sleep(2); + } catch (InterruptedException e) { + // IGNORE IT + } + } + }, TimeoutException.class, "Timeout on iteration"); + } + @Test public void errorMissingParameter() { expectingException(() -> { From edaf4ac9aebbe3bbf7d958728ce460755cad7a5c Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 5 Oct 2023 18:26:14 -0400 Subject: [PATCH 11/19] feat: native select, supported timeout with cutting of results (no exceptions) --- .../arcadedb/database/EmbeddedDatabase.java | 2 +- .../arcadedb/query/nativ/NativeSelect.java | 5 ++- .../query/nativ/NativeSelectExecutor.java | 2 +- .../com/arcadedb/utility/MultiIterator.java | 38 ++++++++++++------- .../query/nativ/NativeSelectExecutionIT.java | 35 ++++++++++------- 5 files changed, 52 insertions(+), 30 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/database/EmbeddedDatabase.java b/engine/src/main/java/com/arcadedb/database/EmbeddedDatabase.java index 3c13fe8cd8..05fac000bf 100644 --- a/engine/src/main/java/com/arcadedb/database/EmbeddedDatabase.java +++ b/engine/src/main/java/com/arcadedb/database/EmbeddedDatabase.java @@ -530,7 +530,7 @@ public Iterator iterateType(final String typeName, final boolean polymor // SET THE PROFILED LIMITS IF ANY iter.setLimit(getResultSetLimit()); - iter.setTimeout(getReadTimeout()); + iter.setTimeout(getReadTimeout(), true); for (final Bucket b : type.getBuckets(polymorphic)) iter.addIterator(b.iterator()); diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java index 37f1e5d32e..f532ce4605 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java @@ -51,6 +51,8 @@ enum STATE {DEFAULT, WHERE, COMPILED} int limit = -1; long timeoutValue; TimeUnit timeoutUnit; + boolean exceptionOnTimeout; + private STATE state = STATE.DEFAULT; private NativeTreeNode lastTreeElement; @@ -175,10 +177,11 @@ public NativeSelect limit(final int limit) { return this; } - public NativeSelect timeout(final long timeoutValue, final TimeUnit timeoutUnit) { + public NativeSelect timeout(final long timeoutValue, final TimeUnit timeoutUnit, final boolean exceptionOnTimeout) { checkNotCompiled(); this.timeoutValue = timeoutValue; this.timeoutUnit = timeoutUnit; + this.exceptionOnTimeout = exceptionOnTimeout; return this; } diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java index 2008402006..47f17f0c80 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java @@ -51,7 +51,7 @@ else if (select.fromBuckets.size() == 1) } if (select.timeoutUnit != null) - iterator.setTimeout(select.timeoutUnit.toMillis(select.timeoutValue)); + iterator.setTimeout(select.timeoutUnit.toMillis(select.timeoutValue), select.exceptionOnTimeout); return new QueryIterator<>() { private T next = null; diff --git a/engine/src/main/java/com/arcadedb/utility/MultiIterator.java b/engine/src/main/java/com/arcadedb/utility/MultiIterator.java index 29f1b3e5b9..4d806538d9 100755 --- a/engine/src/main/java/com/arcadedb/utility/MultiIterator.java +++ b/engine/src/main/java/com/arcadedb/utility/MultiIterator.java @@ -31,13 +31,14 @@ public class MultiIterator implements ResettableIterator, Iterable { private Iterator sourcesIterator; private Iterator partialIterator; - private long browsed = 0L; - private long skip = -1L; - private long limit = -1L; - private long timeout = -1L; - private boolean embedded = false; - private int skipped = 0; - private final long beginTime = System.currentTimeMillis(); + private long browsed = 0L; + private long skip = -1L; + private long limit = -1L; + private long timeout = -1L; + private boolean exceptionOnTimeout = false; + private boolean embedded = false; + private int skipped = 0; + private final long beginTime = System.currentTimeMillis(); public MultiIterator() { sources = new ArrayList<>(); @@ -61,8 +62,9 @@ public boolean hasNext() { } private boolean hasNextInternal() { - if (timeout > -1L && System.currentTimeMillis() - beginTime > timeout) + if (timeout > -1L && System.currentTimeMillis() - beginTime > timeout) { throw new TimeoutException("Timeout on iteration"); + } if (sourcesIterator == null) { if (sources == null || sources.isEmpty()) @@ -125,8 +127,8 @@ public long countEntries() { long size = 0; final int totSources = sources.size(); for (int i = 0; i < totSources; ++i) { - if (timeout > -1L && System.currentTimeMillis() - beginTime > timeout) - throw new TimeoutException("Timeout on iteration"); + if (checkForTimeout()) + break; final Object o = sources.get(i); @@ -163,8 +165,9 @@ public void setLimit(final long limit) { this.limit = limit; } - public void setTimeout(final long readTimeout) { + public void setTimeout(final long readTimeout, final boolean exceptionOnTimeout) { this.timeout = readTimeout; + this.exceptionOnTimeout = exceptionOnTimeout; } public long getSkip() { @@ -193,8 +196,8 @@ public boolean contains(final Object value) { @SuppressWarnings("unchecked") protected boolean getNextPartial() { - if (timeout > -1L && System.currentTimeMillis() - beginTime > timeout) - throw new TimeoutException("Timeout on iteration"); + if (checkForTimeout()) + return false; if (sourcesIterator != null) while (sourcesIterator.hasNext()) { @@ -233,6 +236,15 @@ protected boolean getNextPartial() { return false; } + private boolean checkForTimeout() { + if (timeout > -1L && System.currentTimeMillis() - beginTime > timeout) + if (exceptionOnTimeout) + throw new TimeoutException("Timeout on iteration"); + else + return true; + return false; + } + public boolean isEmbedded() { return embedded; } diff --git a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java index 36dbf0e8db..96e6aa1ed3 100644 --- a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java +++ b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java @@ -178,21 +178,28 @@ public void okLimit() { @Test public void errorTimeout() { - final NativeSelect select = database.select().fromType("Vertex")// - .where().property("id").lt().value(10)// - .and().property("name").eq().value("Elon").timeout(1, TimeUnit.MILLISECONDS); - - expectingException(() -> { - final QueryIterator iter = select.vertices(); - while (iter.hasNext()) { - Assertions.assertTrue(iter.next().getInteger("id") < 10); - try { - Thread.sleep(2); - } catch (InterruptedException e) { - // IGNORE IT + { + expectingException(() -> { + final QueryIterator iter = database.select().fromType("Vertex")// + .where().property("id").lt().value(10)// + .and().property("name").eq().value("Elon").timeout(1, TimeUnit.MILLISECONDS, true).vertices(); + + while (iter.hasNext()) { + Assertions.assertTrue(iter.next().getInteger("id") < 10); + try { + Thread.sleep(2); + } catch (InterruptedException e) { + // IGNORE IT + } } - } - }, TimeoutException.class, "Timeout on iteration"); + }, TimeoutException.class, "Timeout on iteration"); + } + + { + final QueryIterator iter = database.select().fromType("Vertex")// + .where().property("id").lt().value(10)// + .and().property("name").eq().value("Elon").timeout(1, TimeUnit.MILLISECONDS, false).vertices(); + } } @Test From 6b13f8c3c2690aa5a67e86a260f8daabbd554294 Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 5 Oct 2023 18:43:58 -0400 Subject: [PATCH 12/19] test: added test case with update --- .../test/java/com/arcadedb/TestHelper.java | 2 -- .../query/nativ/NativeSelectExecutionIT.java | 21 ++++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/engine/src/test/java/com/arcadedb/TestHelper.java b/engine/src/test/java/com/arcadedb/TestHelper.java index 0b24bbda35..c02cd939fe 100644 --- a/engine/src/test/java/com/arcadedb/TestHelper.java +++ b/engine/src/test/java/com/arcadedb/TestHelper.java @@ -62,8 +62,6 @@ protected TestHelper(final boolean cleanBeforeTest) { database = factory.exists() ? factory.open() : factory.create(); Assertions.assertEquals(database, DatabaseFactory.getActiveDatabaseInstance(database.getDatabasePath())); - database.async().setParallelLevel(PARALLEL_LEVEL); - if (autoStartTx) database.begin(); } diff --git a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java index 96e6aa1ed3..6cd2a93f10 100644 --- a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java +++ b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java @@ -21,6 +21,9 @@ import com.arcadedb.TestHelper; import com.arcadedb.exception.TimeoutException; import com.arcadedb.graph.Vertex; +import com.arcadedb.schema.Schema; +import com.arcadedb.schema.Type; +import com.arcadedb.schema.VertexType; import com.arcadedb.serializer.json.JSONObject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -39,7 +42,9 @@ public NativeSelectExecutionIT() { @Override protected void beginTest() { database.getSchema().createDocumentType("Document"); - database.getSchema().createVertexType("Vertex"); + database.getSchema().createVertexType("Vertex")// + .createProperty("id", Type.INTEGER)// + .createIndex(Schema.INDEX_TYPE.LSM_TREE, true); database.getSchema().createEdgeType("Edge"); database.transaction(() -> { @@ -176,6 +181,20 @@ public void okLimit() { Assertions.assertFalse(iter.hasNext()); } + @Test + public void okUpdate() { + database.select().fromType("Vertex")// + .where().property("id").lt().value(10)// + .and().property("name").eq().value("Elon")// + .limit(10).vertices()// + .forEachRemaining(a -> a.modify().set("modified", true).save()); + + database.select().fromType("Vertex")// + .where().property("id").lt().value(10)// + .and().property("name").eq().value("Elon").limit(10).vertices() + .forEachRemaining(r -> Assertions.assertTrue(r.getInteger("id") < 10 && r.getBoolean("modified"))); + } + @Test public void errorTimeout() { { From a689debd4e9635985b0889cfed9226bf06b6bfae Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 5 Oct 2023 20:16:33 -0400 Subject: [PATCH 13/19] feat: first version of native select optimizer Now it's able to use the defined indexes, if any. When indexes are found, the index cursors are summed in a multi iterator. If an indexed property is in OR with another property that is not indexed, then the index cannot be used. --- .../query/nativ/NativeParameterValue.java | 1 - .../query/nativ/NativeRecordProperty.java | 36 ------ .../query/nativ/NativeSelectExecutor.java | 105 ++++++++++++++++-- .../arcadedb/query/nativ/NativeTreeNode.java | 2 +- .../arcadedb/query/nativ/QueryIterator.java | 3 +- .../query/nativ/NativeSelectExecutionIT.java | 7 +- 6 files changed, 103 insertions(+), 51 deletions(-) delete mode 100644 engine/src/main/java/com/arcadedb/query/nativ/NativeRecordProperty.java diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeParameterValue.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeParameterValue.java index 30aac4bdb2..734bf74b91 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeParameterValue.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeParameterValue.java @@ -15,7 +15,6 @@ public NativeParameterValue(final NativeSelect select, final String parameterNam public Object eval(final Document record) { if (select.parameters == null) throw new IllegalArgumentException("Missing parameter '" + parameterName + "'"); - if (!select.parameters.containsKey(parameterName)) throw new IllegalArgumentException("Missing parameter '" + parameterName + "'"); return select.parameters.get(parameterName); diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeRecordProperty.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeRecordProperty.java deleted file mode 100644 index dced1dd2ec..0000000000 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeRecordProperty.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.arcadedb.query.nativ;/* - * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import com.arcadedb.database.Document; - -import java.util.*; - -/** - * Native condition with support for simple operators through inheritance. - * - * @author Luca Garulli (l.garulli@arcadedata.com) - */ -public class NativeRecordProperty { - private final String name; - - public NativeRecordProperty(final String name) { - this.name = name; - } - - public Object eval(final Document record) { - return record.get(name); - } -} diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java index 47f17f0c80..5e2f9ad003 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java @@ -15,8 +15,13 @@ */ import com.arcadedb.database.Document; +import com.arcadedb.database.Identifiable; import com.arcadedb.engine.Bucket; +import com.arcadedb.index.IndexCursor; +import com.arcadedb.index.MultiIndexCursor; +import com.arcadedb.index.TypeIndex; import com.arcadedb.utility.MultiIterator; +import org.eclipse.collections.impl.lazy.parallel.set.sorted.SelectSortedSetBatch; import java.util.*; @@ -29,17 +34,23 @@ */ public class NativeSelectExecutor { private final NativeSelect select; - private long evaluatedRecords = 0; + private long browsed = 0; + private int indexesUsed = 0; public NativeSelectExecutor(final NativeSelect select) { this.select = select; } - QueryIterator execute() { + QueryIterator execute() { final int[] returned = new int[] { 0 }; - final MultiIterator iterator; - if (select.fromType != null) + final Iterator iteratorFromIndexes = lookForIndexes(); + + final Iterator iterator; + + if (iteratorFromIndexes != null) + iterator = iteratorFromIndexes; + else if (select.fromType != null) iterator = (MultiIterator) select.database.iterateType(select.fromType.getName(), select.polymorphic); else if (select.fromBuckets.size() == 1) iterator = (MultiIterator) select.database.iterateBucket(select.fromBuckets.get(0).getName()); @@ -50,8 +61,9 @@ else if (select.fromBuckets.size() == 1) iterator = multiIterator; } - if (select.timeoutUnit != null) - iterator.setTimeout(select.timeoutUnit.toMillis(select.timeoutValue), select.exceptionOnTimeout); + if (select.timeoutUnit != null && iterator instanceof MultiIterator) + ((MultiIterator) iterator).setTimeout(select.timeoutUnit.toMillis(select.timeoutValue), + select.exceptionOnTimeout); return new QueryIterator<>() { private T next = null; @@ -96,6 +108,81 @@ private T fetchNext() { }; } + private Iterator lookForIndexes() { + if (select.fromType != null && select.rootTreeElement != null) { + final List indexes = new ArrayList<>(); + lookForIndexes(select.rootTreeElement, indexes); + if (!indexes.isEmpty()) { + indexesUsed = indexes.size(); + return new MultiIndexCursor(indexes, select.limit, true); + } + } + return null; + } + + private void lookForIndexes(final NativeTreeNode node, final List indexes) { + if (!(node.left instanceof NativeTreeNode)) + lookForIndexesFinalNode(node, indexes); + else { + lookForIndexes((NativeTreeNode) node.left, indexes); + if (node.right != null) + lookForIndexes((NativeTreeNode) node.right, indexes); + } + } + + private void lookForIndexesFinalNode(final NativeTreeNode node, final List indexes) { + if (node.left instanceof NativePropertyValue) { + if (!(node.right instanceof NativePropertyValue)) { + final List propertyIndexes = select.fromType.getIndexesByProperties( + ((NativePropertyValue) node.left).propertyName); + if (!propertyIndexes.isEmpty()) { + if (node.getParent().operator == NativeOperator.or) { + if (!isLeftPropertyIndexed((NativeTreeNode) node.getParent().right)) + // UNDER AN 'OR' OPERATOR BUT THE OTHER RIGHT VALUE IS NOT INDEXED: CANNOT USE THE CURRENT INDEX + return; + } + + final Object rightValue; + if (node.right instanceof NativeParameterValue) + rightValue = ((NativeParameterValue) node.right).eval(null); + else + rightValue = node.right; + + for (TypeIndex idx : propertyIndexes) { + final IndexCursor cursor; + if (node.operator == NativeOperator.eq) + cursor = idx.get(new Object[] { rightValue }); + else if (node.operator == NativeOperator.gt) + cursor = idx.range(true, new Object[] { rightValue }, false, null, false); + else if (node.operator == NativeOperator.ge) + cursor = idx.range(true, new Object[] { rightValue }, true, null, false); + else if (node.operator == NativeOperator.lt) + cursor = idx.range(true, null, false, new Object[] { rightValue }, false); + else if (node.operator == NativeOperator.le) + cursor = idx.range(true, null, false, new Object[] { rightValue }, true); + else + continue; + + indexes.add(cursor); + } + } + } + } + } + + private boolean isLeftPropertyIndexed(final NativeTreeNode node) { + if (node.left instanceof NativePropertyValue) { + if (!(node.right instanceof NativePropertyValue)) { + final List propertyIndexes = select.fromType.getIndexesByProperties( + ((NativePropertyValue) node.left).propertyName); + if (!propertyIndexes.isEmpty()) { + return true; + } + } + } + return false; + } + public static Object evaluateValue(final Document record, final Object value) { if (value == null) return null; @@ -106,13 +193,13 @@ else if (value instanceof NativeRuntimeValue) return value; } - public long getEvaluatedRecords() { - return evaluatedRecords; + public Map metrics() { + return Map.of("browsed", browsed, "indexesUsed", indexesUsed); } private boolean evaluateWhere(final Document record) { + ++browsed; final Object result = select.rootTreeElement.eval(record); - evaluatedRecords++; if (result instanceof Boolean) return (Boolean) result; throw new IllegalArgumentException("A boolean result was expected but '" + result + "' was returned"); diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java index f2e9700a2c..4b2fab3219 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java @@ -29,7 +29,7 @@ public class NativeTreeNode { public Object left; public final NativeOperator operator; - private Object right; + public Object right; private NativeTreeNode parent; public NativeTreeNode(final Object left, final NativeOperator operator, final Object right) { diff --git a/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java b/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java index ed9e577afc..ce61596a99 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java @@ -16,6 +16,7 @@ import com.arcadedb.database.DatabaseInternal; import com.arcadedb.database.Document; +import com.arcadedb.database.Identifiable; import com.arcadedb.database.Record; import com.arcadedb.graph.Edge; import com.arcadedb.graph.Vertex; @@ -28,7 +29,7 @@ * * @author Luca Garulli (l.garulli@arcadedata.com) */ -public abstract class QueryIterator implements Iterator { +public abstract class QueryIterator implements Iterator { public T nextOrNull() { return hasNext() ? next() : null; } diff --git a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java index 6cd2a93f10..33eaeaad17 100644 --- a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java +++ b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java @@ -175,10 +175,12 @@ public void okLimit() { .and().property("name").eq().value("Elon").limit(10); final QueryIterator iter = select.vertices(); + int browsed = 0; while (iter.hasNext()) { Assertions.assertTrue(iter.next().getInteger("id") < 10); + ++browsed; } - Assertions.assertFalse(iter.hasNext()); + Assertions.assertEquals(10, browsed); } @Test @@ -200,8 +202,7 @@ public void errorTimeout() { { expectingException(() -> { final QueryIterator iter = database.select().fromType("Vertex")// - .where().property("id").lt().value(10)// - .and().property("name").eq().value("Elon").timeout(1, TimeUnit.MILLISECONDS, true).vertices(); + .where().property("name").eq().value("Elon").timeout(1, TimeUnit.MILLISECONDS, true).vertices(); while (iter.hasNext()) { Assertions.assertTrue(iter.next().getInteger("id") < 10); From a68f7b38b528185e6e5ad2cf72d4eb8d14505487 Mon Sep 17 00:00:00 2001 From: lvca Date: Thu, 5 Oct 2023 23:30:19 -0400 Subject: [PATCH 14/19] fix: usage of indexes only when possible --- .../query/nativ/NativeSelectExecutor.java | 120 ++++------- .../arcadedb/query/nativ/QueryIterator.java | 72 ++++++- .../arcadedb/serializer/BinaryComparator.java | 6 +- .../nativ/NativeSelectIndexExecutionIT.java | 198 ++++++++++++++++++ 4 files changed, 313 insertions(+), 83 deletions(-) create mode 100644 engine/src/test/java/com/arcadedb/query/nativ/NativeSelectIndexExecutionIT.java diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java index 5e2f9ad003..3c9692cf10 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java @@ -21,7 +21,6 @@ import com.arcadedb.index.MultiIndexCursor; import com.arcadedb.index.TypeIndex; import com.arcadedb.utility.MultiIterator; -import org.eclipse.collections.impl.lazy.parallel.set.sorted.SelectSortedSetBatch; import java.util.*; @@ -33,17 +32,15 @@ * @author Luca Garulli (l.garulli@arcadedata.com) */ public class NativeSelectExecutor { - private final NativeSelect select; - private long browsed = 0; - private int indexesUsed = 0; + final NativeSelect select; + private long evaluatedRecords = 0; + private int indexesUsed = 0; public NativeSelectExecutor(final NativeSelect select) { this.select = select; } QueryIterator execute() { - final int[] returned = new int[] { 0 }; - final Iterator iteratorFromIndexes = lookForIndexes(); final Iterator iterator; @@ -65,47 +62,7 @@ else if (select.fromBuckets.size() == 1) ((MultiIterator) iterator).setTimeout(select.timeoutUnit.toMillis(select.timeoutValue), select.exceptionOnTimeout); - return new QueryIterator<>() { - private T next = null; - - @Override - public boolean hasNext() { - if (select.limit > -1 && returned[0] >= select.limit) - return false; - if (next != null) - return true; - if (!iterator.hasNext()) - return false; - - next = fetchNext(); - return next != null; - } - - @Override - public T next() { - if (next == null && !hasNext()) - throw new NoSuchElementException(); - try { - return next; - } finally { - next = null; - } - } - - private T fetchNext() { - do { - final Document record = iterator.next().asDocument(); - if (evaluateWhere(record)) { - ++returned[0]; - return (T) record; - } - - } while (iterator.hasNext()); - - // NOT FOUND - return null; - } - }; + return new QueryIterator<>(this, iterator, iteratorFromIndexes != null); } private Iterator lookForIndexes() { @@ -133,11 +90,12 @@ private void lookForIndexes(final NativeTreeNode node, final List i private void lookForIndexesFinalNode(final NativeTreeNode node, final List indexes) { if (node.left instanceof NativePropertyValue) { if (!(node.right instanceof NativePropertyValue)) { - final List propertyIndexes = select.fromType.getIndexesByProperties( + final TypeIndex propertyIndex = select.fromType.getPolymorphicIndexByProperties( ((NativePropertyValue) node.left).propertyName); - if (!propertyIndexes.isEmpty()) { + if (propertyIndex != null) { if (node.getParent().operator == NativeOperator.or) { - if (!isLeftPropertyIndexed((NativeTreeNode) node.getParent().right)) + if (node != node.getParent().right &&// + !isTheNodeFullyIndexed((NativeTreeNode) node.getParent().right)) // UNDER AN 'OR' OPERATOR BUT THE OTHER RIGHT VALUE IS NOT INDEXED: CANNOT USE THE CURRENT INDEX return; } @@ -148,37 +106,47 @@ private void lookForIndexesFinalNode(final NativeTreeNode node, final List propertyIndexes = select.fromType.getIndexesByProperties( + final TypeIndex propertyIndex = select.fromType.getPolymorphicIndexByProperties( ((NativePropertyValue) node.left).propertyName); - if (!propertyIndexes.isEmpty()) { - return true; - } + return propertyIndex != null; } + } else { + final boolean leftIsIndexed = isTheNodeFullyIndexed((NativeTreeNode) node.left); + final boolean rightIsIndexed = isTheNodeFullyIndexed((NativeTreeNode) node.right); + + if (node.operator.equals(NativeOperator.and)) + // AND: ONE OR BOTH MEANS INDEXED + return leftIsIndexed || rightIsIndexed; + else if (node.operator.equals(NativeOperator.or)) + return leftIsIndexed || rightIsIndexed; + else if (node.operator.equals(NativeOperator.not)) + return leftIsIndexed; } return false; } @@ -194,11 +162,11 @@ else if (value instanceof NativeRuntimeValue) } public Map metrics() { - return Map.of("browsed", browsed, "indexesUsed", indexesUsed); + return Map.of("evaluatedRecords", evaluatedRecords, "indexesUsed", indexesUsed); } - private boolean evaluateWhere(final Document record) { - ++browsed; + boolean evaluateWhere(final Document record) { + ++evaluatedRecords; final Object result = select.rootTreeElement.eval(record); if (result instanceof Boolean) return (Boolean) result; diff --git a/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java b/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java index ce61596a99..fa57e52ea0 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java @@ -14,13 +14,9 @@ * limitations under the License. */ -import com.arcadedb.database.DatabaseInternal; import com.arcadedb.database.Document; import com.arcadedb.database.Identifiable; -import com.arcadedb.database.Record; -import com.arcadedb.graph.Edge; -import com.arcadedb.graph.Vertex; -import com.arcadedb.schema.DocumentType; +import com.arcadedb.database.RID; import java.util.*; @@ -29,7 +25,67 @@ * * @author Luca Garulli (l.garulli@arcadedata.com) */ -public abstract class QueryIterator implements Iterator { +public class QueryIterator implements Iterator { + private final NativeSelectExecutor executor; + private final Iterator iterator; + private HashSet filterOutRecords; + private T next = null; + private long returned = 0; + + protected QueryIterator(final NativeSelectExecutor executor, final Iterator iterator, final boolean uniqueResult) { + this.executor = executor; + this.iterator = iterator; + if (uniqueResult) + this.filterOutRecords = new HashSet<>(); + } + + @Override + public boolean hasNext() { + if (executor.select.limit > -1 && returned >= executor.select.limit) + return false; + if (next != null) + return true; + if (!iterator.hasNext()) + return false; + + next = fetchNext(); + return next != null; + } + + @Override + public T next() { + if (next == null && !hasNext()) + throw new NoSuchElementException(); + try { + return next; + } finally { + next = null; + } + } + + private T fetchNext() { + do { + final Document record = iterator.next().asDocument(); + + if (filterOutRecords != null && filterOutRecords.contains(record.getIdentity())) + // ALREADY RETURNED, AVOID DUPLICATES IN THE RESULTSET + continue; + + if (executor.evaluateWhere(record)) { + ++returned; + + if (filterOutRecords != null) + filterOutRecords.add(record.getIdentity()); + + return (T) record; + } + + } while (iterator.hasNext()); + + // NOT FOUND + return null; + } + public T nextOrNull() { return hasNext() ? next() : null; } @@ -40,4 +96,8 @@ public List toList() { result.add(next()); return result; } + + public Map getMetrics() { + return executor.metrics(); + } } diff --git a/engine/src/main/java/com/arcadedb/serializer/BinaryComparator.java b/engine/src/main/java/com/arcadedb/serializer/BinaryComparator.java index 161dcc7ebf..02e6f6d2ec 100644 --- a/engine/src/main/java/com/arcadedb/serializer/BinaryComparator.java +++ b/engine/src/main/java/com/arcadedb/serializer/BinaryComparator.java @@ -374,7 +374,11 @@ else if (b1 < b2) } public static boolean equals(final Object a, final Object b) { - if (a instanceof String && b instanceof String) + if (a == b) + return true; + else if (a == null || b == null) + return false; + else if (a instanceof String && b instanceof String) return equalsString((String) a, (String) b); else if (a instanceof byte[] && b instanceof byte[]) return equalsBytes((byte[]) a, (byte[]) b); diff --git a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectIndexExecutionIT.java b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectIndexExecutionIT.java new file mode 100644 index 0000000000..a438be0c93 --- /dev/null +++ b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectIndexExecutionIT.java @@ -0,0 +1,198 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.query.nativ; + +import com.arcadedb.TestHelper; +import com.arcadedb.exception.TimeoutException; +import com.arcadedb.graph.Vertex; +import com.arcadedb.index.Index; +import com.arcadedb.schema.Schema; +import com.arcadedb.schema.Type; +import com.arcadedb.schema.VertexType; +import com.arcadedb.serializer.json.JSONObject; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.*; +import java.util.concurrent.*; + +/** + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class NativeSelectIndexExecutionIT extends TestHelper { + + public NativeSelectIndexExecutionIT() { + autoStartTx = true; + } + + @Override + protected void beginTest() { + final VertexType v = database.getSchema().createVertexType("Vertex"); + v.createProperty("id", Type.INTEGER)// + .createIndex(Schema.INDEX_TYPE.LSM_TREE, true); + v.createProperty("name", Type.STRING)// + .createIndex(Schema.INDEX_TYPE.LSM_TREE, false); + + database.transaction(() -> { + for (int i = 0; i < 100; i++) + database.newVertex("Vertex").set("id", i, "float", 3.14F, "name", "Elon").save(); + for (int i = 100; i < 110; i++) + database.newVertex("Vertex").set("id", i, "name", "Jay").save(); + }); + } + + @Test + public void okBothIndexUsed() { + // EXPECTED TO USE BOTH INDEXES BECAUSE OF THE AND LOGIC OPERATOR + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon"); + + for (int i = 0; i < 110; i++) { + final int finalI = i; + final QueryIterator result = select.parameter("value", i).vertices(); + + final List list = result.toList(); + Assertions.assertEquals(i < 100 ? 1 : 0, list.size()); + + list.forEach(r -> Assertions.assertTrue(r.getInteger("id") == finalI && r.getString("name").equals("Elon"))); + + // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) + Assertions.assertEquals(100L, result.getMetrics().get("evaluatedRecords"), "With id " + i); + Assertions.assertEquals(2, result.getMetrics().get("indexesUsed")); + } + } + + // EXPECTED TO USE BOTH INDEXES BECAUSE OF THE OR LOGIC OPERATOR AND EACH PROPERTY IS INDEXED + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").eq().parameter("value")// + .or().property("name").eq().value("Elon"); + + for (int i = 0; i < 110; i++) { + final int finalI = i; + final QueryIterator result = select.parameter("value", i).vertices(); + + result.forEachRemaining(r -> Assertions.assertTrue(r.getInteger("id") == finalI || r.getString("name").equals("Elon"))); + + // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) + Assertions.assertEquals(100L, result.getMetrics().get("evaluatedRecords")); + Assertions.assertEquals(2, result.getMetrics().get("indexesUsed")); + } + } + } + + @Test + public void okOneIndexUsed() { + // EXPECTED TO USE ONLY ONE INDEX + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").eq().parameter("value")// + .and().property("unknown").eq().value(null); + + for (int i = 0; i < 110; i++) { + final int finalI = i; + final QueryIterator result = select.parameter("value", i).vertices(); + + result.forEachRemaining(r -> Assertions.assertEquals((int) r.getInteger("id"), finalI)); + + // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) + Assertions.assertEquals(1L, result.getMetrics().get("evaluatedRecords")); + Assertions.assertEquals(1, result.getMetrics().get("indexesUsed")); + } + } + + // EXPECTED TO USE ONLY ONE INDEX + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("unknown").eq().value(null)// + .and().property("id").eq().parameter("value"); + + for (int i = 0; i < 110; i++) { + final int finalI = i; + final QueryIterator result = select.parameter("value", i).vertices(); + + result.forEachRemaining(r -> Assertions.assertEquals((int) r.getInteger("id"), finalI)); + + // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) + Assertions.assertEquals(1L, result.getMetrics().get("evaluatedRecords")); + Assertions.assertEquals(1, result.getMetrics().get("indexesUsed")); + } + } + } + + @Test + public void okNoIndexUsed() { + // EXPECTED NO INDEXES IS USED BECAUSE NO INDEXES WERE DEFINED ON ANY OF THE PROPERTIES + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("unknown").eq().value(null)// + .and().property("unknown").eq().value(null); + + for (int i = 0; i < 110; i++) { + final QueryIterator result = select.parameter("value", i).vertices(); + result.toList(); + + // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) + Assertions.assertEquals(110L, result.getMetrics().get("evaluatedRecords")); + Assertions.assertEquals(0, result.getMetrics().get("indexesUsed")); + } + } + + // EXPECTED NO INDEXES IS USED BECAUSE THE OR OPERATOR ONLY ONE ONE PROPERTY + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").eq().parameter("value")// + .or().property("unknown").eq().value(null); + + for (int i = 0; i < 110; i++) { + final QueryIterator result = select.parameter("value", i).vertices(); + result.toList(); + + // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) + Assertions.assertEquals(110L, result.getMetrics().get("evaluatedRecords")); + Assertions.assertEquals(0, result.getMetrics().get("indexesUsed")); + } + } + + // EXPECTED NO INDEXES IS USED BECAUSE THE OR OPERATOR ONLY ONE ONE PROPERTY + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").eq().parameter("value")// + .or().property("unknown").eq().value(null).and().property("id").eq().parameter("value"); + + for (int i = 0; i < 110; i++) { + final int finalI = i; + + final QueryIterator result = select.parameter("value", i).vertices(); + + final List list = result.toList(); + Assertions.assertEquals(1, list.size()); + + list.forEach(r -> Assertions.assertEquals((int) r.getInteger("id"), finalI)); + + // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) + Assertions.assertEquals(1L, result.getMetrics().get("evaluatedRecords")); + Assertions.assertEquals(2, result.getMetrics().get("indexesUsed")); + } + } + + } +} From 86ce969d3fc7ab078708143d4a3bb830321d4fa8 Mon Sep 17 00:00:00 2001 From: lvca Date: Fri, 6 Oct 2023 00:50:08 -0400 Subject: [PATCH 15/19] feat: native select, supported like() and new() --- .../arcadedb/query/nativ/NativeOperator.java | 17 ++- .../arcadedb/query/nativ/NativeSelect.java | 8 ++ .../query/nativ/NativeSelectExecutor.java | 109 +++++++++++------- .../arcadedb/query/nativ/NativeTreeNode.java | 3 + .../query/nativ/NativeSelectExecutionIT.java | 56 +++++++++ .../nativ/NativeSelectIndexExecutionIT.java | 84 +++++++++++++- 6 files changed, 229 insertions(+), 48 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java index ad5ebc56fe..c34556bebe 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java @@ -15,6 +15,7 @@ */ import com.arcadedb.database.Document; +import com.arcadedb.query.sql.executor.QueryHelper; import com.arcadedb.serializer.BinaryComparator; import java.util.*; @@ -63,6 +64,14 @@ Object eval(final Document record, final Object left, final Object right) { } }, + neq("<>", false, 1) { + @Override + Object eval(final Document record, final Object left, final Object right) { + return !BinaryComparator.equals(NativeSelectExecutor.evaluateValue(record, left), + NativeSelectExecutor.evaluateValue(record, right)); + } + }, + lt("<", false, 1) { @Override Object eval(final Document record, final Object left, final Object right) { @@ -95,7 +104,13 @@ Object eval(final Document record, final Object left, final Object right) { } }, - run("!", true, -1) { + like("like", false, 1) { + @Override + Object eval(final Document record, final Object left, final Object right) { + return QueryHelper.like((String) NativeSelectExecutor.evaluateValue(record, left), + (String) NativeSelectExecutor.evaluateValue(record, right)); + } + }, run("!", true, -1) { @Override Object eval(final Document record, final Object left, final Object right) { return NativeSelectExecutor.evaluateValue(record, left); diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java index f532ce4605..56fb1eaf33 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelect.java @@ -98,6 +98,10 @@ public NativeSelect eq() { return setOperator(NativeOperator.eq); } + public NativeSelect neq() { + return setOperator(NativeOperator.neq); + } + public NativeSelect lt() { return setOperator(NativeOperator.lt); } @@ -114,6 +118,10 @@ public NativeSelect ge() { return setOperator(NativeOperator.ge); } + public NativeSelect like() { + return setOperator(NativeOperator.like); + } + public NativeSelect and() { return setLogic(NativeOperator.and); } diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java index 3c9692cf10..c76bedc6dc 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java @@ -67,73 +67,96 @@ else if (select.fromBuckets.size() == 1) private Iterator lookForIndexes() { if (select.fromType != null && select.rootTreeElement != null) { - final List indexes = new ArrayList<>(); - lookForIndexes(select.rootTreeElement, indexes); - if (!indexes.isEmpty()) { - indexesUsed = indexes.size(); - return new MultiIndexCursor(indexes, select.limit, true); + final List cursors = new ArrayList<>(); + + // FIND AVAILABLE INDEXES + boolean canUseIndexes = isTheNodeFullyIndexed(select.rootTreeElement); + + filterWithIndexes(select.rootTreeElement, cursors); + if (!cursors.isEmpty()) { + indexesUsed = cursors.size(); + return new MultiIndexCursor(cursors, select.limit, true); } } return null; } - private void lookForIndexes(final NativeTreeNode node, final List indexes) { + private void filterWithIndexes(final NativeTreeNode node, final List cursors) { if (!(node.left instanceof NativeTreeNode)) - lookForIndexesFinalNode(node, indexes); + filterWithIndexesFinalNode(node, cursors); else { - lookForIndexes((NativeTreeNode) node.left, indexes); + filterWithIndexes((NativeTreeNode) node.left, cursors); if (node.right != null) - lookForIndexes((NativeTreeNode) node.right, indexes); + filterWithIndexes((NativeTreeNode) node.right, cursors); } } - private void lookForIndexesFinalNode(final NativeTreeNode node, final List indexes) { - if (node.left instanceof NativePropertyValue) { - if (!(node.right instanceof NativePropertyValue)) { - final TypeIndex propertyIndex = select.fromType.getPolymorphicIndexByProperties( - ((NativePropertyValue) node.left).propertyName); - if (propertyIndex != null) { - if (node.getParent().operator == NativeOperator.or) { - if (node != node.getParent().right &&// - !isTheNodeFullyIndexed((NativeTreeNode) node.getParent().right)) - // UNDER AN 'OR' OPERATOR BUT THE OTHER RIGHT VALUE IS NOT INDEXED: CANNOT USE THE CURRENT INDEX - return; - } - - final Object rightValue; - if (node.right instanceof NativeParameterValue) - rightValue = ((NativeParameterValue) node.right).eval(null); - else - rightValue = node.right; - - final IndexCursor cursor; - if (node.operator == NativeOperator.eq) - cursor = propertyIndex.get(new Object[] { rightValue }); - else if (node.operator == NativeOperator.gt) - cursor = propertyIndex.range(true, new Object[] { rightValue }, false, null, false); - else if (node.operator == NativeOperator.ge) - cursor = propertyIndex.range(true, new Object[] { rightValue }, true, null, false); - else if (node.operator == NativeOperator.lt) - cursor = propertyIndex.range(true, null, false, new Object[] { rightValue }, false); - else if (node.operator == NativeOperator.le) - cursor = propertyIndex.range(true, null, false, new Object[] { rightValue }, true); - else - return; - - indexes.add(cursor); + private void filterWithIndexesFinalNode(final NativeTreeNode node, final List cursors) { + if (node.index == null) + return; + + if (node.getParent().operator == NativeOperator.or) { + if (node != node.getParent().right &&// + !isTheNodeFullyIndexed((NativeTreeNode) node.getParent().right)) + // UNDER AN 'OR' OPERATOR BUT THE OTHER RIGHT VALUE IS NOT INDEXED: CANNOT USE THE CURRENT INDEX + return; + } + + final Object rightValue; + if (node.right instanceof NativeParameterValue) + rightValue = ((NativeParameterValue) node.right).eval(null); + else + rightValue = node.right; + + final IndexCursor cursor; + if (node.operator == NativeOperator.eq) + cursor = node.index.get(new Object[] { rightValue }); + else if (node.operator == NativeOperator.gt) + cursor = node.index.range(true, new Object[] { rightValue }, false, null, false); + else if (node.operator == NativeOperator.ge) + cursor = node.index.range(true, new Object[] { rightValue }, true, null, false); + else if (node.operator == NativeOperator.lt) + cursor = node.index.range(true, null, false, new Object[] { rightValue }, false); + else if (node.operator == NativeOperator.le) + cursor = node.index.range(true, null, false, new Object[] { rightValue }, true); + else + return; + + final NativeTreeNode parentNode = node.getParent(); + if (parentNode.operator == NativeOperator.and && parentNode.left == node) { + if (!node.index.isUnique()) { + // CHECK IF THERE IS ANOTHER INDEXED NODE ON THE SIBLING THAT IS UNIQUE (TO PREFER TO THIS) + final TypeIndex rightIndex = ((NativeTreeNode) parentNode.right).index; + if (rightIndex != null && rightIndex.isUnique()) { + // DO NOT USE THIS INDEX (NOT UNIQUE), NOT WORTH IT + node.index = null; + return; } + } else { + // REMOVE THE INDEX ON THE SIBLING NODE + // TODO CALCULATE WHICH ONE IS FASTER AND REMOVE THE SLOWER ONE + ((NativeTreeNode) parentNode.right).index = null; } } + + cursors.add(cursor); } /** * Considers a fully indexed node when both properties are indexed or only one with an AND operator. */ private boolean isTheNodeFullyIndexed(final NativeTreeNode node) { + if (node == null) + return true; + if (!(node.left instanceof NativeTreeNode)) { if (!(node.right instanceof NativePropertyValue)) { final TypeIndex propertyIndex = select.fromType.getPolymorphicIndexByProperties( ((NativePropertyValue) node.left).propertyName); + + if (propertyIndex != null) + node.index = propertyIndex; + return propertyIndex != null; } } else { diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java b/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java index 4b2fab3219..5bd68233f2 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java +++ b/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java @@ -15,6 +15,8 @@ */ import com.arcadedb.database.Document; +import com.arcadedb.index.IndexCursor; +import com.arcadedb.index.TypeIndex; import com.arcadedb.query.sql.parser.BooleanExpression; import com.arcadedb.serializer.json.JSONArray; import com.arcadedb.serializer.json.JSONObject; @@ -31,6 +33,7 @@ public class NativeTreeNode { public final NativeOperator operator; public Object right; private NativeTreeNode parent; + public TypeIndex index; public NativeTreeNode(final Object left, final NativeOperator operator, final Object right) { this.left = left; diff --git a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java index 33eaeaad17..7658083a66 100644 --- a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java +++ b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java @@ -19,6 +19,8 @@ package com.arcadedb.query.nativ; import com.arcadedb.TestHelper; +import com.arcadedb.engine.Bucket; +import com.arcadedb.engine.Component; import com.arcadedb.exception.TimeoutException; import com.arcadedb.graph.Vertex; import com.arcadedb.schema.Schema; @@ -28,7 +30,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.*; import java.util.concurrent.*; +import java.util.stream.*; /** * @author Luca Garulli (l.garulli@arcadedata.com) @@ -64,6 +68,31 @@ protected void beginTest() { }); } + @Test + public void okFromBuckets() { + { + final NativeSelect select = database.select().fromBuckets( + database.getSchema().getType("Vertex").getBuckets(true).stream().map(Component::getName).collect(Collectors.toList()) + .toArray(new String[database.getSchema().getType("Vertex").getBuckets(true).size()]))// + .where().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon"); + + for (int i = 0; i < 100; i++) + Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); + } + + { + final NativeSelect select = database.select().fromBuckets( + database.getSchema().getType("Vertex").getBuckets(true).stream().map(Bucket::getFileId).collect(Collectors.toList()) + .toArray(new Integer[database.getSchema().getType("Vertex").getBuckets(true).size()]))// + .where().property("id").eq().parameter("value")// + .and().property("name").eq().value("Elon"); + + for (int i = 0; i < 100; i++) + Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); + } + } + @Test public void okAnd() { { @@ -222,6 +251,33 @@ public void errorTimeout() { } } + @Test + public void okNeq() { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").neq().parameter("value"); + + for (int i = 0; i < 100; i++) { + final int finalI = i; + final QueryIterator result = select.parameter("value", i).vertices(); + final List list = result.toList(); + Assertions.assertEquals(99, list.size()); + list.forEach(r -> Assertions.assertTrue(r.getInteger("id") != finalI)); + } + } + + @Test + public void okLike() { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("name").like().value("E%"); + + for (int i = 0; i < 100; i++) { + final QueryIterator result = select.parameter("value", i).vertices(); + final List list = result.toList(); + Assertions.assertEquals(100, list.size()); + list.forEach(r -> Assertions.assertTrue(r.getString("name").startsWith("E"))); + } + } + @Test public void errorMissingParameter() { expectingException(() -> { diff --git a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectIndexExecutionIT.java b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectIndexExecutionIT.java index a438be0c93..22e4c9bf84 100644 --- a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectIndexExecutionIT.java +++ b/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectIndexExecutionIT.java @@ -58,7 +58,7 @@ protected void beginTest() { } @Test - public void okBothIndexUsed() { + public void okOneOfTwoAvailableIndexes() { // EXPECTED TO USE BOTH INDEXES BECAUSE OF THE AND LOGIC OPERATOR { final NativeSelect select = database.select().fromType("Vertex")// @@ -75,11 +75,14 @@ public void okBothIndexUsed() { list.forEach(r -> Assertions.assertTrue(r.getInteger("id") == finalI && r.getString("name").equals("Elon"))); // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) - Assertions.assertEquals(100L, result.getMetrics().get("evaluatedRecords"), "With id " + i); - Assertions.assertEquals(2, result.getMetrics().get("indexesUsed")); + Assertions.assertEquals(1L, result.getMetrics().get("evaluatedRecords"), "With id " + i); + Assertions.assertEquals(1, result.getMetrics().get("indexesUsed")); } } + } + @Test + public void okBothAvailableIndexes() { // EXPECTED TO USE BOTH INDEXES BECAUSE OF THE OR LOGIC OPERATOR AND EACH PROPERTY IS INDEXED { final NativeSelect select = database.select().fromType("Vertex")// @@ -93,7 +96,7 @@ public void okBothIndexUsed() { result.forEachRemaining(r -> Assertions.assertTrue(r.getInteger("id") == finalI || r.getString("name").equals("Elon"))); // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) - Assertions.assertEquals(100L, result.getMetrics().get("evaluatedRecords")); + Assertions.assertEquals(i < 100 ? 100L : 101L, result.getMetrics().get("evaluatedRecords"), "" + finalI); Assertions.assertEquals(2, result.getMetrics().get("indexesUsed")); } } @@ -195,4 +198,77 @@ public void okNoIndexUsed() { } } + + @Test + public void okRanges() { + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").gt().parameter("value"); + + for (int i = 0; i < 110; i++) { + final int finalI = i; + final QueryIterator result = select.parameter("value", i).vertices(); + final List list = result.toList(); + Assertions.assertEquals(109 - i, list.size()); + list.forEach(r -> Assertions.assertTrue(r.getInteger("id") > finalI)); + Assertions.assertEquals(1, result.getMetrics().get("indexesUsed")); + } + } + + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").ge().parameter("value"); + + for (int i = 0; i < 110; i++) { + final int finalI = i; + final QueryIterator result = select.parameter("value", i).vertices(); + final List list = result.toList(); + Assertions.assertEquals(110 - i, list.size()); + list.forEach(r -> Assertions.assertTrue(r.getInteger("id") >= finalI)); + Assertions.assertEquals(1, result.getMetrics().get("indexesUsed")); + } + } + + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").lt().parameter("value"); + + for (int i = 0; i < 110; i++) { + final int finalI = i; + final QueryIterator result = select.parameter("value", i).vertices(); + final List list = result.toList(); + Assertions.assertEquals(i, list.size()); + list.forEach(r -> Assertions.assertTrue(r.getInteger("id") < finalI)); + Assertions.assertEquals(1, result.getMetrics().get("indexesUsed")); + } + } + + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").le().parameter("value"); + + for (int i = 0; i < 110; i++) { + final int finalI = i; + final QueryIterator result = select.parameter("value", i).vertices(); + final List list = result.toList(); + Assertions.assertEquals(i + 1, list.size()); + list.forEach(r -> Assertions.assertTrue(r.getInteger("id") <= finalI)); + Assertions.assertEquals(1, result.getMetrics().get("indexesUsed")); + } + } + + { + final NativeSelect select = database.select().fromType("Vertex")// + .where().property("id").neq().parameter("value"); + + for (int i = 0; i < 110; i++) { + final int finalI = i; + final QueryIterator result = select.parameter("value", i).vertices(); + final List list = result.toList(); + Assertions.assertEquals(109, list.size()); + list.forEach(r -> Assertions.assertTrue(r.getInteger("id") != finalI)); + Assertions.assertEquals(0, result.getMetrics().get("indexesUsed")); + } + } + } } From 791ed5bee42139f1b7cdf04927c6da93c0d8a67f Mon Sep 17 00:00:00 2001 From: lvca Date: Fri, 6 Oct 2023 11:58:25 -0400 Subject: [PATCH 16/19] chore: refactoring to statically limit choices while building the native query --- .../java/com/arcadedb/database/Database.java | 4 +- .../arcadedb/database/DatabaseInternal.java | 1 - .../arcadedb/database/EmbeddedDatabase.java | 6 +- .../NativeSelect.java => select/Select.java} | 162 ++++++------------ .../arcadedb/query/select/SelectCompiled.java | 80 +++++++++ .../SelectIterator.java} | 8 +- .../SelectOperator.java} | 50 +++--- .../SelectParameterValue.java} | 10 +- .../SelectPropertyValue.java} | 6 +- .../SelectRuntimeValue.java} | 4 +- .../SelectSelectExecutor.java} | 81 +++++---- .../SelectTreeNode.java} | 43 ++--- .../select/SelectWhereAfterFirstBlock.java | 68 ++++++++ .../query/select/SelectWhereBaseBlock.java | 44 +++++ .../query/select/SelectWhereLeftBlock.java | 38 ++++ .../select/SelectWhereOperatorBlock.java | 58 +++++++ .../query/select/SelectWhereRightBlock.java | 57 ++++++ .../SelectExecutionIT.java} | 102 +++++------ .../SelectIndexExecutionIT.java} | 82 +++++---- .../collection/SQLMethodTransformTest.java | 4 +- .../performance/LocalDatabaseBenchmark.java | 9 +- .../com/arcadedb/server/ServerDatabase.java | 4 +- .../server/ha/ReplicatedDatabase.java | 4 +- 23 files changed, 593 insertions(+), 332 deletions(-) rename engine/src/main/java/com/arcadedb/query/{nativ/NativeSelect.java => select/Select.java} (68%) create mode 100644 engine/src/main/java/com/arcadedb/query/select/SelectCompiled.java rename engine/src/main/java/com/arcadedb/query/{nativ/QueryIterator.java => select/SelectIterator.java} (89%) rename engine/src/main/java/com/arcadedb/query/{nativ/NativeOperator.java => select/SelectOperator.java} (68%) rename engine/src/main/java/com/arcadedb/query/{nativ/NativeParameterValue.java => select/SelectParameterValue.java} (68%) rename engine/src/main/java/com/arcadedb/query/{nativ/NativePropertyValue.java => select/SelectPropertyValue.java} (65%) rename engine/src/main/java/com/arcadedb/query/{nativ/NativeRuntimeValue.java => select/SelectRuntimeValue.java} (65%) rename engine/src/main/java/com/arcadedb/query/{nativ/NativeSelectExecutor.java => select/SelectSelectExecutor.java} (69%) rename engine/src/main/java/com/arcadedb/query/{nativ/NativeTreeNode.java => select/SelectTreeNode.java} (68%) create mode 100644 engine/src/main/java/com/arcadedb/query/select/SelectWhereAfterFirstBlock.java create mode 100644 engine/src/main/java/com/arcadedb/query/select/SelectWhereBaseBlock.java create mode 100644 engine/src/main/java/com/arcadedb/query/select/SelectWhereLeftBlock.java create mode 100644 engine/src/main/java/com/arcadedb/query/select/SelectWhereOperatorBlock.java create mode 100644 engine/src/main/java/com/arcadedb/query/select/SelectWhereRightBlock.java rename engine/src/test/java/com/arcadedb/query/{nativ/NativeSelectExecutionIT.java => select/SelectExecutionIT.java} (72%) rename engine/src/test/java/com/arcadedb/query/{nativ/NativeSelectIndexExecutionIT.java => select/SelectIndexExecutionIT.java} (72%) diff --git a/engine/src/main/java/com/arcadedb/database/Database.java b/engine/src/main/java/com/arcadedb/database/Database.java index b12cf44fba..b9723bb676 100644 --- a/engine/src/main/java/com/arcadedb/database/Database.java +++ b/engine/src/main/java/com/arcadedb/database/Database.java @@ -28,7 +28,7 @@ import com.arcadedb.graph.Vertex; import com.arcadedb.index.IndexCursor; import com.arcadedb.query.QueryEngine; -import com.arcadedb.query.nativ.NativeSelect; +import com.arcadedb.query.select.Select; import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.schema.Schema; @@ -56,7 +56,7 @@ enum TRANSACTION_ISOLATION_LEVEL { */ String getCurrentUserName(); - NativeSelect select(); + Select select(); /** * Executes a command by specifying the language and arguments in a map. diff --git a/engine/src/main/java/com/arcadedb/database/DatabaseInternal.java b/engine/src/main/java/com/arcadedb/database/DatabaseInternal.java index eca18f30ed..d197e9881c 100644 --- a/engine/src/main/java/com/arcadedb/database/DatabaseInternal.java +++ b/engine/src/main/java/com/arcadedb/database/DatabaseInternal.java @@ -24,7 +24,6 @@ import com.arcadedb.engine.WALFileFactory; import com.arcadedb.exception.TransactionException; import com.arcadedb.graph.GraphEngine; -import com.arcadedb.query.nativ.NativeSelect; import com.arcadedb.query.sql.parser.ExecutionPlanCache; import com.arcadedb.query.sql.parser.StatementCache; import com.arcadedb.security.SecurityDatabaseUser; diff --git a/engine/src/main/java/com/arcadedb/database/EmbeddedDatabase.java b/engine/src/main/java/com/arcadedb/database/EmbeddedDatabase.java index 05fac000bf..d4a7efb70c 100644 --- a/engine/src/main/java/com/arcadedb/database/EmbeddedDatabase.java +++ b/engine/src/main/java/com/arcadedb/database/EmbeddedDatabase.java @@ -59,7 +59,7 @@ import com.arcadedb.log.LogManager; import com.arcadedb.query.QueryEngine; import com.arcadedb.query.QueryEngineManager; -import com.arcadedb.query.nativ.NativeSelect; +import com.arcadedb.query.select.Select; import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.query.sql.parser.ExecutionPlanCache; import com.arcadedb.query.sql.parser.StatementCache; @@ -1365,8 +1365,8 @@ public ResultSet query(final String language, final String query, final Map parameters; - NativeTreeNode rootTreeElement; + SelectTreeNode rootTreeElement; DocumentType fromType; List fromBuckets; - NativeOperator operator; - NativeRuntimeValue property; + SelectOperator operator; + SelectRuntimeValue property; Object propertyValue; boolean polymorphic = true; int limit = -1; - long timeoutValue; - TimeUnit timeoutUnit; + long timeoutInMs = 0; boolean exceptionOnTimeout; - private STATE state = STATE.DEFAULT; - private NativeTreeNode lastTreeElement; + STATE state = STATE.DEFAULT; + private SelectTreeNode lastTreeElement; - public NativeSelect(final DatabaseInternal database) { + public Select(final DatabaseInternal database) { this.database = database; } - public NativeSelect fromType(final String fromType) { + public Select fromType(final String fromType) { checkNotCompiled(); if (this.fromType != null) throw new IllegalArgumentException("From type has already been set"); @@ -71,7 +70,7 @@ public NativeSelect fromType(final String fromType) { return this; } - public NativeSelect fromBuckets(final String... fromBucketNames) { + public Select fromBuckets(final String... fromBucketNames) { checkNotCompiled(); if (this.fromType != null) throw new IllegalArgumentException("From type has already been set"); @@ -83,7 +82,7 @@ public NativeSelect fromBuckets(final String... fromBucketNames) { return this; } - public NativeSelect fromBuckets(final Integer... fromBucketIds) { + public Select fromBuckets(final Integer... fromBucketIds) { checkNotCompiled(); if (this.fromType != null) throw new IllegalArgumentException("From type has already been set"); @@ -94,53 +93,17 @@ public NativeSelect fromBuckets(final Integer... fromBucketIds) { return this; } - public NativeSelect eq() { - return setOperator(NativeOperator.eq); - } - - public NativeSelect neq() { - return setOperator(NativeOperator.neq); - } - - public NativeSelect lt() { - return setOperator(NativeOperator.lt); - } - - public NativeSelect le() { - return setOperator(NativeOperator.le); - } - - public NativeSelect gt() { - return setOperator(NativeOperator.gt); - } - - public NativeSelect ge() { - return setOperator(NativeOperator.ge); - } - - public NativeSelect like() { - return setOperator(NativeOperator.like); - } - - public NativeSelect and() { - return setLogic(NativeOperator.and); - } - - public NativeSelect or() { - return setLogic(NativeOperator.or); - } - - public NativeSelect property(final String name) { + public Select property(final String name) { checkNotCompiled(); if (property != null) throw new IllegalArgumentException("Property has already been set"); if (state != STATE.WHERE) throw new IllegalArgumentException("No context was provided for the parameter"); - this.property = new NativePropertyValue(name); + this.property = new SelectPropertyValue(name); return this; } - public NativeSelect value(final Object value) { + public Select value(final Object value) { checkNotCompiled(); if (property == null) throw new IllegalArgumentException("Property has not been set"); @@ -158,42 +121,41 @@ public NativeSelect value(final Object value) { return this; } - public NativeSelect where() { + public SelectWhereLeftBlock where() { checkNotCompiled(); if (rootTreeElement != null) throw new IllegalArgumentException("Where has already been set"); state = STATE.WHERE; - return this; + return new SelectWhereLeftBlock(this); } - public NativeSelect parameter(final String parameterName) { + public Select parameter(final String parameterName) { checkNotCompiled(); - this.propertyValue = new NativeParameterValue(this, parameterName); + this.propertyValue = new SelectParameterValue(this, parameterName); return this; } - public NativeSelect parameter(final String paramName, final Object paramValue) { + public Select parameter(final String paramName, final Object paramValue) { if (parameters == null) parameters = new HashMap<>(); parameters.put(paramName, paramValue); return this; } - public NativeSelect limit(final int limit) { + public Select limit(final int limit) { checkNotCompiled(); this.limit = limit; return this; } - public NativeSelect timeout(final long timeoutValue, final TimeUnit timeoutUnit, final boolean exceptionOnTimeout) { + public Select timeout(final long timeoutValue, final TimeUnit timeoutUnit, final boolean exceptionOnTimeout) { checkNotCompiled(); - this.timeoutValue = timeoutValue; - this.timeoutUnit = timeoutUnit; + this.timeoutInMs = timeoutUnit.toMillis(timeoutValue); this.exceptionOnTimeout = exceptionOnTimeout; return this; } - public NativeSelect polymorphic(final boolean polymorphic) { + public Select polymorphic(final boolean polymorphic) { checkNotCompiled(); if (fromType == null) throw new IllegalArgumentException("FromType was not set"); @@ -201,7 +163,7 @@ public NativeSelect polymorphic(final boolean polymorphic) { return this; } - public NativeSelect json(final JSONObject json) { + public Select json(final JSONObject json) { checkNotCompiled(); if (json.has("fromType")) { fromType(json.getString("fromType")); @@ -239,7 +201,7 @@ else if (parsedLeft instanceof String && ((String) parsedLeft).startsWith("#")) else throw new IllegalArgumentException("Unsupported value " + parsedLeft); - final NativeOperator parsedOperator = NativeOperator.byName(condition.getString(1)); + final SelectOperator parsedOperator = SelectOperator.byName(condition.getString(1)); if (parsedOperator.logicOperator) setLogic(parsedOperator); @@ -257,80 +219,58 @@ else if (parsedRight instanceof String && ((String) parsedRight).startsWith("#") value(parsedRight); } - public JSONObject json() { - if (state != STATE.COMPILED) - parse(); - - final JSONObject json = new JSONObject(); - - if (fromType != null) { - json.put("fromType", fromType.getName()); - if (!polymorphic) - json.put("polymorphic", polymorphic); - } else if (fromBuckets != null) - json.put("fromBuckets", fromBuckets.stream().map(b -> b.getName()).collect(Collectors.toList())); - - if (rootTreeElement != null) - json.put("where", rootTreeElement.toJSON()); - - if (limit > -1) - json.put("limit", limit); - - return json; - } - - public NativeSelect parse() { + public SelectCompiled compile() { if (fromType == null && fromBuckets == null) throw new IllegalArgumentException("from (type or buckets) has not been set"); if (state != STATE.COMPILED) { - setLogic(NativeOperator.run); + setLogic(SelectOperator.run); state = STATE.COMPILED; } - return this; + return new SelectCompiled(this); } - public QueryIterator vertices() { + public SelectIterator vertices() { return run(); } - public QueryIterator edges() { + public SelectIterator edges() { return run(); } - public QueryIterator documents() { + public SelectIterator documents() { return run(); } - private QueryIterator run() { - parse(); - return new NativeSelectExecutor(this).execute(); + SelectIterator run() { + compile(); + return new SelectSelectExecutor(this).execute(); } - private NativeSelect setLogic(final NativeOperator newLogicOperator) { + SelectWhereLeftBlock setLogic(final SelectOperator newLogicOperator) { checkNotCompiled(); if (operator == null) throw new IllegalArgumentException("Missing condition"); - final NativeTreeNode newTreeElement = new NativeTreeNode(property, operator, propertyValue); + final SelectTreeNode newTreeElement = new SelectTreeNode(property, operator, propertyValue); if (rootTreeElement == null) { // 1ST TIME ONLY - rootTreeElement = new NativeTreeNode(newTreeElement, newLogicOperator, null); + rootTreeElement = new SelectTreeNode(newTreeElement, newLogicOperator, null); newTreeElement.setParent(rootTreeElement); lastTreeElement = newTreeElement; } else { - if (newLogicOperator.equals(NativeOperator.run)) { + if (newLogicOperator.equals(SelectOperator.run)) { // EXECUTION = LAST NODE: APPEND TO THE RIGHT OF THE LATEST lastTreeElement.getParent().setRight(newTreeElement); } else if (lastTreeElement.getParent().operator.precedence < newLogicOperator.precedence) { // AND+ OPERATOR - final NativeTreeNode newNode = new NativeTreeNode(newTreeElement, newLogicOperator, null); + final SelectTreeNode newNode = new SelectTreeNode(newTreeElement, newLogicOperator, null); lastTreeElement.getParent().setRight(newNode); lastTreeElement = newTreeElement; } else { // OR+ OPERATOR - final NativeTreeNode currentParent = lastTreeElement.getParent(); + final SelectTreeNode currentParent = lastTreeElement.getParent(); currentParent.setRight(newTreeElement); - final NativeTreeNode newNode = new NativeTreeNode(currentParent, newLogicOperator, null); + final SelectTreeNode newNode = new SelectTreeNode(currentParent, newLogicOperator, null); if (rootTreeElement.equals(currentParent)) rootTreeElement = newNode; else @@ -342,19 +282,19 @@ private NativeSelect setLogic(final NativeOperator newLogicOperator) { operator = null; property = null; propertyValue = null; - return this; + return new SelectWhereLeftBlock(this); } - private NativeSelect setOperator(final NativeOperator nativeOperator) { + void checkNotCompiled() { + if (state == STATE.COMPILED) + throw new IllegalArgumentException("Cannot modify the structure of a select what has been already compiled"); + } + + SelectWhereRightBlock setOperator(final SelectOperator selectOperator) { checkNotCompiled(); if (operator != null) throw new IllegalArgumentException("Operator has already been set (" + operator + ")"); - operator = nativeOperator; - return this; - } - - private void checkNotCompiled() { - if (state == STATE.COMPILED) - throw new IllegalArgumentException("Cannot modify the structure of a select what has been already compiled"); + operator = selectOperator; + return new SelectWhereRightBlock(this); } } diff --git a/engine/src/main/java/com/arcadedb/query/select/SelectCompiled.java b/engine/src/main/java/com/arcadedb/query/select/SelectCompiled.java new file mode 100644 index 0000000000..4822aefda0 --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/select/SelectCompiled.java @@ -0,0 +1,80 @@ +package com.arcadedb.query.select;/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.arcadedb.database.Document; +import com.arcadedb.graph.Edge; +import com.arcadedb.graph.Vertex; +import com.arcadedb.serializer.json.JSONObject; + +import java.util.*; +import java.util.stream.*; + +/** + * Native Query engine is a simple query engine that covers most of the classic use cases, such as the retrieval of records + * with a where condition. It could be much faster than the same SQL query because it does not use any parser and it is very + * JIT friendly. Future versions could translate the query into bytecode to have an even faster execution. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class SelectCompiled { + private final Select select; + + public SelectCompiled(final Select select) { + this.select = select; + } + + public SelectCompiled parameter(final String paramName, final Object paramValue) { + if (select.parameters == null) + select.parameters = new HashMap<>(); + select.parameters.put(paramName, paramValue); + return this; + } + + public JSONObject json() { + final JSONObject json = new JSONObject(); + + if (select.fromType != null) { + json.put("fromType", select.fromType.getName()); + if (!select.polymorphic) + json.put("polymorphic", select.polymorphic); + } else if (select.fromBuckets != null) + json.put("fromBuckets", select.fromBuckets.stream().map(b -> b.getName()).collect(Collectors.toList())); + + if (select.rootTreeElement != null) + json.put("where", select.rootTreeElement.toJSON()); + + if (select.limit > -1) + json.put("limit", select.limit); + if (select.timeoutInMs > 0) { + json.put("timeoutInMs", select.timeoutInMs); + json.put("exceptionOnTimeout", select.exceptionOnTimeout); + } + + return json; + } + + public SelectIterator vertices() { + return select.vertices(); + } + + public SelectIterator edges() { + return select.edges(); + } + + public SelectIterator documents() { + return select.documents(); + } +} diff --git a/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java b/engine/src/main/java/com/arcadedb/query/select/SelectIterator.java similarity index 89% rename from engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java rename to engine/src/main/java/com/arcadedb/query/select/SelectIterator.java index fa57e52ea0..833962a002 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/QueryIterator.java +++ b/engine/src/main/java/com/arcadedb/query/select/SelectIterator.java @@ -1,4 +1,4 @@ -package com.arcadedb.query.nativ;/* +package com.arcadedb.query.select;/* * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,14 +25,14 @@ * * @author Luca Garulli (l.garulli@arcadedata.com) */ -public class QueryIterator implements Iterator { - private final NativeSelectExecutor executor; +public class SelectIterator implements Iterator { + private final SelectSelectExecutor executor; private final Iterator iterator; private HashSet filterOutRecords; private T next = null; private long returned = 0; - protected QueryIterator(final NativeSelectExecutor executor, final Iterator iterator, final boolean uniqueResult) { + protected SelectIterator(final SelectSelectExecutor executor, final Iterator iterator, final boolean uniqueResult) { this.executor = executor; this.iterator = iterator; if (uniqueResult) diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java b/engine/src/main/java/com/arcadedb/query/select/SelectOperator.java similarity index 68% rename from engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java rename to engine/src/main/java/com/arcadedb/query/select/SelectOperator.java index c34556bebe..3066d2e307 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeOperator.java +++ b/engine/src/main/java/com/arcadedb/query/select/SelectOperator.java @@ -1,4 +1,4 @@ -package com.arcadedb.query.nativ;/* +package com.arcadedb.query.select;/* * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,26 +26,26 @@ * * @author Luca Garulli (l.garulli@arcadedata.com) */ -public enum NativeOperator { +public enum SelectOperator { or("or", true, 0) { @Override Object eval(final Document record, final Object left, final Object right) { - final Boolean leftValue = (Boolean) NativeSelectExecutor.evaluateValue(record, left); + final Boolean leftValue = (Boolean) SelectSelectExecutor.evaluateValue(record, left); if (leftValue) return true; - return NativeSelectExecutor.evaluateValue(record, right); + return SelectSelectExecutor.evaluateValue(record, right); } }, and("and", true, 2) { @Override Object eval(final Document record, final Object left, final Object right) { - final Boolean leftValue = (Boolean) NativeSelectExecutor.evaluateValue(record, left); + final Boolean leftValue = (Boolean) SelectSelectExecutor.evaluateValue(record, left); if (!leftValue) return false; - return NativeSelectExecutor.evaluateValue(record, right); + return SelectSelectExecutor.evaluateValue(record, right); } }, @@ -59,70 +59,70 @@ Object eval(final Document record, final Object left, final Object right) { eq("=", false, 1) { @Override Object eval(final Document record, final Object left, final Object right) { - return BinaryComparator.equals(NativeSelectExecutor.evaluateValue(record, left), - NativeSelectExecutor.evaluateValue(record, right)); + return BinaryComparator.equals(SelectSelectExecutor.evaluateValue(record, left), + SelectSelectExecutor.evaluateValue(record, right)); } }, neq("<>", false, 1) { @Override Object eval(final Document record, final Object left, final Object right) { - return !BinaryComparator.equals(NativeSelectExecutor.evaluateValue(record, left), - NativeSelectExecutor.evaluateValue(record, right)); + return !BinaryComparator.equals(SelectSelectExecutor.evaluateValue(record, left), + SelectSelectExecutor.evaluateValue(record, right)); } }, lt("<", false, 1) { @Override Object eval(final Document record, final Object left, final Object right) { - return BinaryComparator.compareTo(NativeSelectExecutor.evaluateValue(record, left), - NativeSelectExecutor.evaluateValue(record, right)) < 0; + return BinaryComparator.compareTo(SelectSelectExecutor.evaluateValue(record, left), + SelectSelectExecutor.evaluateValue(record, right)) < 0; } }, le("<=", false, 1) { @Override Object eval(final Document record, final Object left, final Object right) { - return BinaryComparator.compareTo(NativeSelectExecutor.evaluateValue(record, left), - NativeSelectExecutor.evaluateValue(record, right)) <= 0; + return BinaryComparator.compareTo(SelectSelectExecutor.evaluateValue(record, left), + SelectSelectExecutor.evaluateValue(record, right)) <= 0; } }, gt(">", false, 1) { @Override Object eval(final Document record, final Object left, final Object right) { - return BinaryComparator.compareTo(NativeSelectExecutor.evaluateValue(record, left), - NativeSelectExecutor.evaluateValue(record, right)) > 0; + return BinaryComparator.compareTo(SelectSelectExecutor.evaluateValue(record, left), + SelectSelectExecutor.evaluateValue(record, right)) > 0; } }, ge(">=", false, 1) { @Override Object eval(final Document record, final Object left, final Object right) { - return BinaryComparator.compareTo(NativeSelectExecutor.evaluateValue(record, left), - NativeSelectExecutor.evaluateValue(record, right)) >= 0; + return BinaryComparator.compareTo(SelectSelectExecutor.evaluateValue(record, left), + SelectSelectExecutor.evaluateValue(record, right)) >= 0; } }, like("like", false, 1) { @Override Object eval(final Document record, final Object left, final Object right) { - return QueryHelper.like((String) NativeSelectExecutor.evaluateValue(record, left), - (String) NativeSelectExecutor.evaluateValue(record, right)); + return QueryHelper.like((String) SelectSelectExecutor.evaluateValue(record, left), + (String) SelectSelectExecutor.evaluateValue(record, right)); } }, run("!", true, -1) { @Override Object eval(final Document record, final Object left, final Object right) { - return NativeSelectExecutor.evaluateValue(record, left); + return SelectSelectExecutor.evaluateValue(record, left); } }; public final String name; public final boolean logicOperator; public final int precedence; - private static Map NAMES = new ConcurrentHashMap<>(); + private static Map NAMES = new ConcurrentHashMap<>(); - NativeOperator(final String name, final boolean logicOperator, final int precedence) { + SelectOperator(final String name, final boolean logicOperator, final int precedence) { this.name = name; this.logicOperator = logicOperator; this.precedence = precedence; @@ -130,9 +130,9 @@ Object eval(final Document record, final Object left, final Object right) { abstract Object eval(final Document record, Object left, Object right); - public static NativeOperator byName(final String name) { + public static SelectOperator byName(final String name) { if (NAMES.isEmpty()) { - for (NativeOperator v : values()) + for (SelectOperator v : values()) NAMES.put(v.name, v); } diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeParameterValue.java b/engine/src/main/java/com/arcadedb/query/select/SelectParameterValue.java similarity index 68% rename from engine/src/main/java/com/arcadedb/query/nativ/NativeParameterValue.java rename to engine/src/main/java/com/arcadedb/query/select/SelectParameterValue.java index 734bf74b91..cc1ab23b15 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeParameterValue.java +++ b/engine/src/main/java/com/arcadedb/query/select/SelectParameterValue.java @@ -1,12 +1,12 @@ -package com.arcadedb.query.nativ; +package com.arcadedb.query.select; import com.arcadedb.database.Document; -public class NativeParameterValue implements NativeRuntimeValue { - public final String parameterName; - private final NativeSelect select; +public class SelectParameterValue implements SelectRuntimeValue { + public final String parameterName; + private final Select select; - public NativeParameterValue(final NativeSelect select, final String parameterName) { + public SelectParameterValue(final Select select, final String parameterName) { this.select = select; this.parameterName = parameterName; } diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativePropertyValue.java b/engine/src/main/java/com/arcadedb/query/select/SelectPropertyValue.java similarity index 65% rename from engine/src/main/java/com/arcadedb/query/nativ/NativePropertyValue.java rename to engine/src/main/java/com/arcadedb/query/select/SelectPropertyValue.java index 67bcd31bc9..590ec0a22c 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativePropertyValue.java +++ b/engine/src/main/java/com/arcadedb/query/select/SelectPropertyValue.java @@ -1,11 +1,11 @@ -package com.arcadedb.query.nativ; +package com.arcadedb.query.select; import com.arcadedb.database.Document; -public class NativePropertyValue implements NativeRuntimeValue { +public class SelectPropertyValue implements SelectRuntimeValue { public final String propertyName; - public NativePropertyValue(final String propertyName) { + public SelectPropertyValue(final String propertyName) { this.propertyName = propertyName; } diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeRuntimeValue.java b/engine/src/main/java/com/arcadedb/query/select/SelectRuntimeValue.java similarity index 65% rename from engine/src/main/java/com/arcadedb/query/nativ/NativeRuntimeValue.java rename to engine/src/main/java/com/arcadedb/query/select/SelectRuntimeValue.java index 74ee81448a..e704053210 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeRuntimeValue.java +++ b/engine/src/main/java/com/arcadedb/query/select/SelectRuntimeValue.java @@ -1,11 +1,11 @@ -package com.arcadedb.query.nativ; +package com.arcadedb.query.select; import com.arcadedb.database.Document; /** * @author Luca Garulli (l.garulli@arcadedata.com) */ -public interface NativeRuntimeValue { +public interface SelectRuntimeValue { Object eval(final Document record); } diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java b/engine/src/main/java/com/arcadedb/query/select/SelectSelectExecutor.java similarity index 69% rename from engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java rename to engine/src/main/java/com/arcadedb/query/select/SelectSelectExecutor.java index c76bedc6dc..dba6f383f4 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeSelectExecutor.java +++ b/engine/src/main/java/com/arcadedb/query/select/SelectSelectExecutor.java @@ -1,4 +1,4 @@ -package com.arcadedb.query.nativ;/* +package com.arcadedb.query.select;/* * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,16 +31,16 @@ * * @author Luca Garulli (l.garulli@arcadedata.com) */ -public class NativeSelectExecutor { - final NativeSelect select; - private long evaluatedRecords = 0; +public class SelectSelectExecutor { + final Select select; + private long evaluatedRecords = 0; private int indexesUsed = 0; - public NativeSelectExecutor(final NativeSelect select) { + public SelectSelectExecutor(final Select select) { this.select = select; } - QueryIterator execute() { + SelectIterator execute() { final Iterator iteratorFromIndexes = lookForIndexes(); final Iterator iterator; @@ -58,11 +58,10 @@ else if (select.fromBuckets.size() == 1) iterator = multiIterator; } - if (select.timeoutUnit != null && iterator instanceof MultiIterator) - ((MultiIterator) iterator).setTimeout(select.timeoutUnit.toMillis(select.timeoutValue), - select.exceptionOnTimeout); + if (select.timeoutInMs > 0 && iterator instanceof MultiIterator) + ((MultiIterator) iterator).setTimeout(select.timeoutInMs, select.exceptionOnTimeout); - return new QueryIterator<>(this, iterator, iteratorFromIndexes != null); + return new SelectIterator<>(this, iterator, iteratorFromIndexes != null); } private Iterator lookForIndexes() { @@ -81,52 +80,52 @@ private Iterator lookForIndexes() { return null; } - private void filterWithIndexes(final NativeTreeNode node, final List cursors) { - if (!(node.left instanceof NativeTreeNode)) + private void filterWithIndexes(final SelectTreeNode node, final List cursors) { + if (!(node.left instanceof SelectTreeNode)) filterWithIndexesFinalNode(node, cursors); else { - filterWithIndexes((NativeTreeNode) node.left, cursors); + filterWithIndexes((SelectTreeNode) node.left, cursors); if (node.right != null) - filterWithIndexes((NativeTreeNode) node.right, cursors); + filterWithIndexes((SelectTreeNode) node.right, cursors); } } - private void filterWithIndexesFinalNode(final NativeTreeNode node, final List cursors) { + private void filterWithIndexesFinalNode(final SelectTreeNode node, final List cursors) { if (node.index == null) return; - if (node.getParent().operator == NativeOperator.or) { + if (node.getParent().operator == SelectOperator.or) { if (node != node.getParent().right &&// - !isTheNodeFullyIndexed((NativeTreeNode) node.getParent().right)) + !isTheNodeFullyIndexed((SelectTreeNode) node.getParent().right)) // UNDER AN 'OR' OPERATOR BUT THE OTHER RIGHT VALUE IS NOT INDEXED: CANNOT USE THE CURRENT INDEX return; } final Object rightValue; - if (node.right instanceof NativeParameterValue) - rightValue = ((NativeParameterValue) node.right).eval(null); + if (node.right instanceof SelectParameterValue) + rightValue = ((SelectParameterValue) node.right).eval(null); else rightValue = node.right; final IndexCursor cursor; - if (node.operator == NativeOperator.eq) + if (node.operator == SelectOperator.eq) cursor = node.index.get(new Object[] { rightValue }); - else if (node.operator == NativeOperator.gt) + else if (node.operator == SelectOperator.gt) cursor = node.index.range(true, new Object[] { rightValue }, false, null, false); - else if (node.operator == NativeOperator.ge) + else if (node.operator == SelectOperator.ge) cursor = node.index.range(true, new Object[] { rightValue }, true, null, false); - else if (node.operator == NativeOperator.lt) + else if (node.operator == SelectOperator.lt) cursor = node.index.range(true, null, false, new Object[] { rightValue }, false); - else if (node.operator == NativeOperator.le) + else if (node.operator == SelectOperator.le) cursor = node.index.range(true, null, false, new Object[] { rightValue }, true); else return; - final NativeTreeNode parentNode = node.getParent(); - if (parentNode.operator == NativeOperator.and && parentNode.left == node) { + final SelectTreeNode parentNode = node.getParent(); + if (parentNode.operator == SelectOperator.and && parentNode.left == node) { if (!node.index.isUnique()) { // CHECK IF THERE IS ANOTHER INDEXED NODE ON THE SIBLING THAT IS UNIQUE (TO PREFER TO THIS) - final TypeIndex rightIndex = ((NativeTreeNode) parentNode.right).index; + final TypeIndex rightIndex = ((SelectTreeNode) parentNode.right).index; if (rightIndex != null && rightIndex.isUnique()) { // DO NOT USE THIS INDEX (NOT UNIQUE), NOT WORTH IT node.index = null; @@ -135,7 +134,7 @@ else if (node.operator == NativeOperator.le) } else { // REMOVE THE INDEX ON THE SIBLING NODE // TODO CALCULATE WHICH ONE IS FASTER AND REMOVE THE SLOWER ONE - ((NativeTreeNode) parentNode.right).index = null; + ((SelectTreeNode) parentNode.right).index = null; } } @@ -145,14 +144,14 @@ else if (node.operator == NativeOperator.le) /** * Considers a fully indexed node when both properties are indexed or only one with an AND operator. */ - private boolean isTheNodeFullyIndexed(final NativeTreeNode node) { + private boolean isTheNodeFullyIndexed(final SelectTreeNode node) { if (node == null) return true; - if (!(node.left instanceof NativeTreeNode)) { - if (!(node.right instanceof NativePropertyValue)) { + if (!(node.left instanceof SelectTreeNode)) { + if (!(node.right instanceof SelectPropertyValue)) { final TypeIndex propertyIndex = select.fromType.getPolymorphicIndexByProperties( - ((NativePropertyValue) node.left).propertyName); + ((SelectPropertyValue) node.left).propertyName); if (propertyIndex != null) node.index = propertyIndex; @@ -160,15 +159,15 @@ private boolean isTheNodeFullyIndexed(final NativeTreeNode node) { return propertyIndex != null; } } else { - final boolean leftIsIndexed = isTheNodeFullyIndexed((NativeTreeNode) node.left); - final boolean rightIsIndexed = isTheNodeFullyIndexed((NativeTreeNode) node.right); + final boolean leftIsIndexed = isTheNodeFullyIndexed((SelectTreeNode) node.left); + final boolean rightIsIndexed = isTheNodeFullyIndexed((SelectTreeNode) node.right); - if (node.operator.equals(NativeOperator.and)) + if (node.operator.equals(SelectOperator.and)) // AND: ONE OR BOTH MEANS INDEXED return leftIsIndexed || rightIsIndexed; - else if (node.operator.equals(NativeOperator.or)) + else if (node.operator.equals(SelectOperator.or)) return leftIsIndexed || rightIsIndexed; - else if (node.operator.equals(NativeOperator.not)) + else if (node.operator.equals(SelectOperator.not)) return leftIsIndexed; } return false; @@ -177,10 +176,10 @@ else if (node.operator.equals(NativeOperator.not)) public static Object evaluateValue(final Document record, final Object value) { if (value == null) return null; - else if (value instanceof NativeTreeNode) - return ((NativeTreeNode) value).eval(record); - else if (value instanceof NativeRuntimeValue) - return ((NativeRuntimeValue) value).eval(record); + else if (value instanceof SelectTreeNode) + return ((SelectTreeNode) value).eval(record); + else if (value instanceof SelectRuntimeValue) + return ((SelectRuntimeValue) value).eval(record); return value; } diff --git a/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java b/engine/src/main/java/com/arcadedb/query/select/SelectTreeNode.java similarity index 68% rename from engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java rename to engine/src/main/java/com/arcadedb/query/select/SelectTreeNode.java index 5bd68233f2..d9c4c72c5c 100644 --- a/engine/src/main/java/com/arcadedb/query/nativ/NativeTreeNode.java +++ b/engine/src/main/java/com/arcadedb/query/select/SelectTreeNode.java @@ -1,4 +1,4 @@ -package com.arcadedb.query.nativ;/* +package com.arcadedb.query.select;/* * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,43 +15,38 @@ */ import com.arcadedb.database.Document; -import com.arcadedb.index.IndexCursor; import com.arcadedb.index.TypeIndex; -import com.arcadedb.query.sql.parser.BooleanExpression; import com.arcadedb.serializer.json.JSONArray; -import com.arcadedb.serializer.json.JSONObject; - -import java.util.*; /** * Native condition representation in a tree. * * @author Luca Garulli (l.garulli@arcadedata.com) */ -public class NativeTreeNode { +public class SelectTreeNode { public Object left; - public final NativeOperator operator; + public final SelectOperator operator; public Object right; - private NativeTreeNode parent; + private SelectTreeNode parent; public TypeIndex index; - public NativeTreeNode(final Object left, final NativeOperator operator, final Object right) { + public SelectTreeNode(final Object left, final SelectOperator operator, final Object right) { this.left = left; - if (left instanceof NativeTreeNode) - ((NativeTreeNode) left).setParent(this); + if (left instanceof SelectTreeNode) + ((SelectTreeNode) left).setParent(this); this.operator = operator; this.right = right; - if (right instanceof NativeTreeNode) - ((NativeTreeNode) right).setParent(this); + if (right instanceof SelectTreeNode) + ((SelectTreeNode) right).setParent(this); } public Object eval(final Document record) { return operator.eval(record, left, right); } - public void setRight(final NativeTreeNode right) { + public void setRight(final SelectTreeNode right) { if (this.right != null) throw new IllegalArgumentException("Cannot assign the right node because already assigned to " + this.right); this.right = right; @@ -60,11 +55,11 @@ public void setRight(final NativeTreeNode right) { right.parent = this; } - public NativeTreeNode getParent() { + public SelectTreeNode getParent() { return parent; } - public void setParent(final NativeTreeNode newParent) { + public void setParent(final SelectTreeNode newParent) { if (this.parent == newParent) return; @@ -95,20 +90,20 @@ public String toString() { public JSONArray toJSON() { final JSONArray json = new JSONArray(); - if (left instanceof NativeTreeNode) - json.put(((NativeTreeNode) left).toJSON()); - else if (left instanceof NativePropertyValue || left instanceof NativeParameterValue) + if (left instanceof SelectTreeNode) + json.put(((SelectTreeNode) left).toJSON()); + else if (left instanceof SelectPropertyValue || left instanceof SelectParameterValue) json.put(left.toString()); else json.put(left); - if (operator != NativeOperator.run) + if (operator != SelectOperator.run) json.put(operator.name); if (right != null) { - if (right instanceof NativeTreeNode) - json.put(((NativeTreeNode) right).toJSON()); - else if (right instanceof NativePropertyValue || right instanceof NativeParameterValue) + if (right instanceof SelectTreeNode) + json.put(((SelectTreeNode) right).toJSON()); + else if (right instanceof SelectPropertyValue || right instanceof SelectParameterValue) json.put(right.toString()); else json.put(right); diff --git a/engine/src/main/java/com/arcadedb/query/select/SelectWhereAfterFirstBlock.java b/engine/src/main/java/com/arcadedb/query/select/SelectWhereAfterFirstBlock.java new file mode 100644 index 0000000000..cd6609ebb4 --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/select/SelectWhereAfterFirstBlock.java @@ -0,0 +1,68 @@ +package com.arcadedb.query.select;/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.arcadedb.database.Document; +import com.arcadedb.graph.Edge; +import com.arcadedb.graph.Vertex; + +import java.util.concurrent.*; + +/** + * Native Query engine is a simple query engine that covers most of the classic use cases, such as the retrieval of records + * with a where condition. It could be much faster than the same SQL query because it does not use any parser and it is very + * JIT friendly. Future versions could translate the query into bytecode to have an even faster execution. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class SelectWhereAfterFirstBlock { + private final Select select; + + public SelectWhereAfterFirstBlock(final Select select) { + this.select = select; + } + + public SelectWhereLeftBlock and() { + return select.setLogic(SelectOperator.and); + } + + public SelectWhereLeftBlock or() { + return select.setLogic(SelectOperator.or); + } + + public SelectIterator vertices() { + return select.run(); + } + + public SelectIterator edges() { + return select.run(); + } + + public SelectIterator documents() { + return select.run(); + } + + public SelectCompiled compile() { + return select.compile(); + } + + public Select limit(final int limit) { + return select.limit(limit); + } + + public Select timeout(final long timeoutValue, final TimeUnit timeoutUnit, final boolean exceptionOnTimeout) { + return select.timeout(timeoutValue, timeoutUnit, exceptionOnTimeout); + } +} diff --git a/engine/src/main/java/com/arcadedb/query/select/SelectWhereBaseBlock.java b/engine/src/main/java/com/arcadedb/query/select/SelectWhereBaseBlock.java new file mode 100644 index 0000000000..cc48d1abe8 --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/select/SelectWhereBaseBlock.java @@ -0,0 +1,44 @@ +package com.arcadedb.query.select;/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Native Query engine is a simple query engine that covers most of the classic use cases, such as the retrieval of records + * with a where condition. It could be much faster than the same SQL query because it does not use any parser and it is very + * JIT friendly. Future versions could translate the query into bytecode to have an even faster execution. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public abstract class SelectWhereBaseBlock { + protected final Select select; + + public SelectWhereBaseBlock(final Select select) { + this.select = select; + } + + protected void setParameter(final String parameterName) { + select.checkNotCompiled(); + select.propertyValue = new SelectParameterValue(select, parameterName); + } + + protected void setProperty(final String name) { + select.checkNotCompiled(); + if (select.property != null) + throw new IllegalArgumentException("Property has already been set"); + if (select.state != Select.STATE.WHERE) + throw new IllegalArgumentException("No context was provided for the parameter"); + select.property = new SelectPropertyValue(name); + } +} diff --git a/engine/src/main/java/com/arcadedb/query/select/SelectWhereLeftBlock.java b/engine/src/main/java/com/arcadedb/query/select/SelectWhereLeftBlock.java new file mode 100644 index 0000000000..2f2f1dae6c --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/select/SelectWhereLeftBlock.java @@ -0,0 +1,38 @@ +package com.arcadedb.query.select;/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Native Query engine is a simple query engine that covers most of the classic use cases, such as the retrieval of records + * with a where condition. It could be much faster than the same SQL query because it does not use any parser and it is very + * JIT friendly. Future versions could translate the query into bytecode to have an even faster execution. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class SelectWhereLeftBlock extends SelectWhereBaseBlock { + public SelectWhereLeftBlock(final Select select) { + super(select); + } + + public SelectWhereOperatorBlock property(final String name) { + setProperty(name); + return new SelectWhereOperatorBlock(select); + } + + public SelectWhereOperatorBlock parameter(final String parameterName) { + setParameter(parameterName); + return new SelectWhereOperatorBlock(select); + } +} diff --git a/engine/src/main/java/com/arcadedb/query/select/SelectWhereOperatorBlock.java b/engine/src/main/java/com/arcadedb/query/select/SelectWhereOperatorBlock.java new file mode 100644 index 0000000000..aefa9d94a5 --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/select/SelectWhereOperatorBlock.java @@ -0,0 +1,58 @@ +package com.arcadedb.query.select;/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Native Query engine is a simple query engine that covers most of the classic use cases, such as the retrieval of records + * with a where condition. It could be much faster than the same SQL query because it does not use any parser and it is very + * JIT friendly. Future versions could translate the query into bytecode to have an even faster execution. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class SelectWhereOperatorBlock { + private final Select select; + + public SelectWhereOperatorBlock(final Select select) { + this.select = select; + } + + public SelectWhereRightBlock eq() { + return select.setOperator(SelectOperator.eq); + } + + public SelectWhereRightBlock neq() { + return select.setOperator(SelectOperator.neq); + } + + public SelectWhereRightBlock lt() { + return select.setOperator(SelectOperator.lt); + } + + public SelectWhereRightBlock le() { + return select.setOperator(SelectOperator.le); + } + + public SelectWhereRightBlock gt() { + return select.setOperator(SelectOperator.gt); + } + + public SelectWhereRightBlock ge() { + return select.setOperator(SelectOperator.ge); + } + + public SelectWhereRightBlock like() { + return select.setOperator(SelectOperator.like); + } +} diff --git a/engine/src/main/java/com/arcadedb/query/select/SelectWhereRightBlock.java b/engine/src/main/java/com/arcadedb/query/select/SelectWhereRightBlock.java new file mode 100644 index 0000000000..519297a7ec --- /dev/null +++ b/engine/src/main/java/com/arcadedb/query/select/SelectWhereRightBlock.java @@ -0,0 +1,57 @@ +package com.arcadedb.query.select;/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Native Query engine is a simple query engine that covers most of the classic use cases, such as the retrieval of records + * with a where condition. It could be much faster than the same SQL query because it does not use any parser and it is very + * JIT friendly. Future versions could translate the query into bytecode to have an even faster execution. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class SelectWhereRightBlock extends SelectWhereBaseBlock { + + public SelectWhereRightBlock(final Select select) { + super(select); + } + + public SelectWhereAfterFirstBlock property(final String name) { + setProperty(name); + return new SelectWhereAfterFirstBlock(select); + } + + public SelectWhereAfterFirstBlock parameter(final String parameterName) { + setParameter(parameterName); + return new SelectWhereAfterFirstBlock(select); + } + + public SelectWhereAfterFirstBlock value(final Object value) { + select.checkNotCompiled(); + if (select.property == null) + throw new IllegalArgumentException("Property has not been set"); + + switch (select.state) { + case WHERE: + if (select.operator == null) + throw new IllegalArgumentException("No operator has been set"); + if (select.propertyValue != null) + throw new IllegalArgumentException("Property value has already been set"); + select.propertyValue = value; + break; + } + + return new SelectWhereAfterFirstBlock(select); + } +} diff --git a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java b/engine/src/test/java/com/arcadedb/query/select/SelectExecutionIT.java similarity index 72% rename from engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java rename to engine/src/test/java/com/arcadedb/query/select/SelectExecutionIT.java index 7658083a66..d80fc46e50 100644 --- a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectExecutionIT.java +++ b/engine/src/test/java/com/arcadedb/query/select/SelectExecutionIT.java @@ -16,7 +16,7 @@ * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) * SPDX-License-Identifier: Apache-2.0 */ -package com.arcadedb.query.nativ; +package com.arcadedb.query.select; import com.arcadedb.TestHelper; import com.arcadedb.engine.Bucket; @@ -25,7 +25,6 @@ import com.arcadedb.graph.Vertex; import com.arcadedb.schema.Schema; import com.arcadedb.schema.Type; -import com.arcadedb.schema.VertexType; import com.arcadedb.serializer.json.JSONObject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -37,9 +36,9 @@ /** * @author Luca Garulli (l.garulli@arcadedata.com) */ -public class NativeSelectExecutionIT extends TestHelper { +public class SelectExecutionIT extends TestHelper { - public NativeSelectExecutionIT() { + public SelectExecutionIT() { autoStartTx = true; } @@ -71,22 +70,22 @@ protected void beginTest() { @Test public void okFromBuckets() { { - final NativeSelect select = database.select().fromBuckets( + final SelectCompiled select = database.select().fromBuckets( database.getSchema().getType("Vertex").getBuckets(true).stream().map(Component::getName).collect(Collectors.toList()) .toArray(new String[database.getSchema().getType("Vertex").getBuckets(true).size()]))// .where().property("id").eq().parameter("value")// - .and().property("name").eq().value("Elon"); + .and().property("name").eq().value("Elon").compile(); for (int i = 0; i < 100; i++) Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); } { - final NativeSelect select = database.select().fromBuckets( + final SelectCompiled select = database.select().fromBuckets( database.getSchema().getType("Vertex").getBuckets(true).stream().map(Bucket::getFileId).collect(Collectors.toList()) .toArray(new Integer[database.getSchema().getType("Vertex").getBuckets(true).size()]))// .where().property("id").eq().parameter("value")// - .and().property("name").eq().value("Elon"); + .and().property("name").eq().value("Elon").compile(); for (int i = 0; i < 100; i++) Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); @@ -96,36 +95,36 @@ public void okFromBuckets() { @Test public void okAnd() { { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// - .and().property("name").eq().value("Elon"); + .and().property("name").eq().value("Elon").compile(); for (int i = 0; i < 100; i++) Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); } { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// - .and().property("name").eq().value("Elon2"); + .and().property("name").eq().value("Elon2").compile(); Assertions.assertFalse(select.parameter("value", 3).vertices().hasNext()); } { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").eq().value(-1)// - .and().property("name").eq().value("Elon"); + .and().property("name").eq().value("Elon").compile(); Assertions.assertFalse(select.vertices().hasNext()); } { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// .and().property("name").eq().value("Elon")// .and().property("id").eq().parameter("value")// - .and().property("name").eq().value("Elon"); + .and().property("name").eq().value("Elon").compile(); for (int i = 0; i < 100; i++) Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); @@ -135,33 +134,33 @@ public void okAnd() { @Test public void okOr() { { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// - .or().property("name").eq().value("Elon"); + .or().property("name").eq().value("Elon").compile(); - for (QueryIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { + for (SelectIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { final Vertex v = result.next(); Assertions.assertTrue(v.getInteger("id").equals(3) || v.getString("name").equals("Elon")); } } { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// - .or().property("name").eq().value("Elon2"); + .or().property("name").eq().value("Elon2").compile(); - for (QueryIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { + for (SelectIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { final Vertex v = result.next(); Assertions.assertTrue(v.getInteger("id").equals(3) || v.getString("name").equals("Elon2")); } } { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").eq().value(-1)// - .or().property("name").eq().value("Elon"); + .or().property("name").eq().value("Elon").compile(); - for (QueryIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { + for (SelectIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { final Vertex v = result.next(); Assertions.assertTrue(v.getInteger("id").equals(-1) || v.getString("name").equals("Elon")); } @@ -171,12 +170,12 @@ public void okOr() { @Test public void okAndOr() { { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// .and().property("name").eq().value("Elon2")// - .or().property("name").eq().value("Elon"); + .or().property("name").eq().value("Elon").compile(); - for (QueryIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { + for (SelectIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { final Vertex v = result.next(); Assertions.assertTrue(v.getInteger("id").equals(3) && v.getString("name").equals("Elon2") ||// v.getString("name").equals("Elon")); @@ -184,12 +183,12 @@ public void okAndOr() { } { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// .or().property("name").eq().value("Elon2")// - .and().property("name").eq().value("Elon"); + .and().property("name").eq().value("Elon").compile(); - for (QueryIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { + for (SelectIterator result = select.parameter("value", 3).vertices(); result.hasNext(); ) { final Vertex v = result.next(); Assertions.assertTrue(v.getInteger("id").equals(3) ||// v.getString("name").equals("Elon2") && v.getString("name").equals("Elon")); @@ -199,11 +198,11 @@ public void okAndOr() { @Test public void okLimit() { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").lt().value(10)// - .and().property("name").eq().value("Elon").limit(10); + .and().property("name").eq().value("Elon").limit(10).compile(); - final QueryIterator iter = select.vertices(); + final SelectIterator iter = select.vertices(); int browsed = 0; while (iter.hasNext()) { Assertions.assertTrue(iter.next().getInteger("id") < 10); @@ -230,7 +229,7 @@ public void okUpdate() { public void errorTimeout() { { expectingException(() -> { - final QueryIterator iter = database.select().fromType("Vertex")// + final SelectIterator iter = database.select().fromType("Vertex")// .where().property("name").eq().value("Elon").timeout(1, TimeUnit.MILLISECONDS, true).vertices(); while (iter.hasNext()) { @@ -245,7 +244,7 @@ public void errorTimeout() { } { - final QueryIterator iter = database.select().fromType("Vertex")// + final SelectIterator iter = database.select().fromType("Vertex")// .where().property("id").lt().value(10)// .and().property("name").eq().value("Elon").timeout(1, TimeUnit.MILLISECONDS, false).vertices(); } @@ -253,12 +252,12 @@ public void errorTimeout() { @Test public void okNeq() { - final NativeSelect select = database.select().fromType("Vertex")// - .where().property("id").neq().parameter("value"); + final SelectCompiled select = database.select().fromType("Vertex")// + .where().property("id").neq().parameter("value").compile(); for (int i = 0; i < 100; i++) { final int finalI = i; - final QueryIterator result = select.parameter("value", i).vertices(); + final SelectIterator result = select.parameter("value", i).vertices(); final List list = result.toList(); Assertions.assertEquals(99, list.size()); list.forEach(r -> Assertions.assertTrue(r.getInteger("id") != finalI)); @@ -267,11 +266,11 @@ public void okNeq() { @Test public void okLike() { - final NativeSelect select = database.select().fromType("Vertex")// - .where().property("name").like().value("E%"); + final SelectCompiled select = database.select().fromType("Vertex")// + .where().property("name").like().value("E%").compile(); for (int i = 0; i < 100; i++) { - final QueryIterator result = select.parameter("value", i).vertices(); + final SelectIterator result = select.parameter("value", i).vertices(); final List list = result.toList(); Assertions.assertEquals(100, list.size()); list.forEach(r -> Assertions.assertTrue(r.getString("name").startsWith("E"))); @@ -289,7 +288,8 @@ public void errorMissingParameter() { @Test public void okReuse() { - final NativeSelect select = database.select().fromType("Vertex").where().property("id").eq().parameter("value"); + final SelectCompiled select = database.select().fromType("Vertex").where().property("id").eq().parameter("value") + .compile(); for (int i = 0; i < 100; i++) Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); } @@ -297,29 +297,19 @@ public void okReuse() { @Test public void okJSON() { { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// .and().property("name").eq().value("Elon2")// - .or().property("name").eq().value("Elon"); + .or().property("name").eq().value("Elon").compile(); final JSONObject json = select.json(); - final NativeSelect select2 = database.select().json(json); - final JSONObject json2 = select2.json(); + final JSONObject json2 = database.select().json(json).compile().json(); Assertions.assertEquals(json, json2); } } - @Test - public void errorMissingCondition() { - expectingException(() -> { - database.select().fromType("Vertex")// - .where().property("id").eq().parameter("value")// - .or().vertices().nextOrNull(); - }, IllegalArgumentException.class, "Missing condition"); - } - private void expectingException(final Runnable callback, final Class expectedException, final String mustContains) { boolean failed = true; diff --git a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectIndexExecutionIT.java b/engine/src/test/java/com/arcadedb/query/select/SelectIndexExecutionIT.java similarity index 72% rename from engine/src/test/java/com/arcadedb/query/nativ/NativeSelectIndexExecutionIT.java rename to engine/src/test/java/com/arcadedb/query/select/SelectIndexExecutionIT.java index 22e4c9bf84..020b987636 100644 --- a/engine/src/test/java/com/arcadedb/query/nativ/NativeSelectIndexExecutionIT.java +++ b/engine/src/test/java/com/arcadedb/query/select/SelectIndexExecutionIT.java @@ -16,28 +16,24 @@ * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) * SPDX-License-Identifier: Apache-2.0 */ -package com.arcadedb.query.nativ; +package com.arcadedb.query.select; import com.arcadedb.TestHelper; -import com.arcadedb.exception.TimeoutException; import com.arcadedb.graph.Vertex; -import com.arcadedb.index.Index; import com.arcadedb.schema.Schema; import com.arcadedb.schema.Type; import com.arcadedb.schema.VertexType; -import com.arcadedb.serializer.json.JSONObject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.*; -import java.util.concurrent.*; /** * @author Luca Garulli (l.garulli@arcadedata.com) */ -public class NativeSelectIndexExecutionIT extends TestHelper { +public class SelectIndexExecutionIT extends TestHelper { - public NativeSelectIndexExecutionIT() { + public SelectIndexExecutionIT() { autoStartTx = true; } @@ -61,13 +57,13 @@ protected void beginTest() { public void okOneOfTwoAvailableIndexes() { // EXPECTED TO USE BOTH INDEXES BECAUSE OF THE AND LOGIC OPERATOR { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// - .and().property("name").eq().value("Elon"); + .and().property("name").eq().value("Elon").compile(); for (int i = 0; i < 110; i++) { final int finalI = i; - final QueryIterator result = select.parameter("value", i).vertices(); + final SelectIterator result = select.parameter("value", i).vertices(); final List list = result.toList(); Assertions.assertEquals(i < 100 ? 1 : 0, list.size()); @@ -85,13 +81,13 @@ public void okOneOfTwoAvailableIndexes() { public void okBothAvailableIndexes() { // EXPECTED TO USE BOTH INDEXES BECAUSE OF THE OR LOGIC OPERATOR AND EACH PROPERTY IS INDEXED { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// - .or().property("name").eq().value("Elon"); + .or().property("name").eq().value("Elon").compile(); for (int i = 0; i < 110; i++) { final int finalI = i; - final QueryIterator result = select.parameter("value", i).vertices(); + final SelectIterator result = select.parameter("value", i).vertices(); result.forEachRemaining(r -> Assertions.assertTrue(r.getInteger("id") == finalI || r.getString("name").equals("Elon"))); @@ -106,13 +102,13 @@ public void okBothAvailableIndexes() { public void okOneIndexUsed() { // EXPECTED TO USE ONLY ONE INDEX { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// - .and().property("unknown").eq().value(null); + .and().property("unknown").eq().value(null).compile(); for (int i = 0; i < 110; i++) { final int finalI = i; - final QueryIterator result = select.parameter("value", i).vertices(); + final SelectIterator result = select.parameter("value", i).vertices(); result.forEachRemaining(r -> Assertions.assertEquals((int) r.getInteger("id"), finalI)); @@ -124,13 +120,13 @@ public void okOneIndexUsed() { // EXPECTED TO USE ONLY ONE INDEX { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("unknown").eq().value(null)// - .and().property("id").eq().parameter("value"); + .and().property("id").eq().parameter("value").compile(); for (int i = 0; i < 110; i++) { final int finalI = i; - final QueryIterator result = select.parameter("value", i).vertices(); + final SelectIterator result = select.parameter("value", i).vertices(); result.forEachRemaining(r -> Assertions.assertEquals((int) r.getInteger("id"), finalI)); @@ -145,12 +141,12 @@ public void okOneIndexUsed() { public void okNoIndexUsed() { // EXPECTED NO INDEXES IS USED BECAUSE NO INDEXES WERE DEFINED ON ANY OF THE PROPERTIES { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("unknown").eq().value(null)// - .and().property("unknown").eq().value(null); + .and().property("unknown").eq().value(null).compile(); for (int i = 0; i < 110; i++) { - final QueryIterator result = select.parameter("value", i).vertices(); + final SelectIterator result = select.parameter("value", i).vertices(); result.toList(); // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) @@ -161,12 +157,12 @@ public void okNoIndexUsed() { // EXPECTED NO INDEXES IS USED BECAUSE THE OR OPERATOR ONLY ONE ONE PROPERTY { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// - .or().property("unknown").eq().value(null); + .or().property("unknown").eq().value(null).compile(); for (int i = 0; i < 110; i++) { - final QueryIterator result = select.parameter("value", i).vertices(); + final SelectIterator result = select.parameter("value", i).vertices(); result.toList(); // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) @@ -177,14 +173,14 @@ public void okNoIndexUsed() { // EXPECTED NO INDEXES IS USED BECAUSE THE OR OPERATOR ONLY ONE ONE PROPERTY { - final NativeSelect select = database.select().fromType("Vertex")// + final SelectCompiled select = database.select().fromType("Vertex")// .where().property("id").eq().parameter("value")// - .or().property("unknown").eq().value(null).and().property("id").eq().parameter("value"); + .or().property("unknown").eq().value(null).and().property("id").eq().parameter("value").compile(); for (int i = 0; i < 110; i++) { final int finalI = i; - final QueryIterator result = select.parameter("value", i).vertices(); + final SelectIterator result = select.parameter("value", i).vertices(); final List list = result.toList(); Assertions.assertEquals(1, list.size()); @@ -202,12 +198,12 @@ public void okNoIndexUsed() { @Test public void okRanges() { { - final NativeSelect select = database.select().fromType("Vertex")// - .where().property("id").gt().parameter("value"); + final SelectCompiled select = database.select().fromType("Vertex")// + .where().property("id").gt().parameter("value").compile(); for (int i = 0; i < 110; i++) { final int finalI = i; - final QueryIterator result = select.parameter("value", i).vertices(); + final SelectIterator result = select.parameter("value", i).vertices(); final List list = result.toList(); Assertions.assertEquals(109 - i, list.size()); list.forEach(r -> Assertions.assertTrue(r.getInteger("id") > finalI)); @@ -216,12 +212,12 @@ public void okRanges() { } { - final NativeSelect select = database.select().fromType("Vertex")// - .where().property("id").ge().parameter("value"); + final SelectCompiled select = database.select().fromType("Vertex")// + .where().property("id").ge().parameter("value").compile(); for (int i = 0; i < 110; i++) { final int finalI = i; - final QueryIterator result = select.parameter("value", i).vertices(); + final SelectIterator result = select.parameter("value", i).vertices(); final List list = result.toList(); Assertions.assertEquals(110 - i, list.size()); list.forEach(r -> Assertions.assertTrue(r.getInteger("id") >= finalI)); @@ -230,12 +226,12 @@ public void okRanges() { } { - final NativeSelect select = database.select().fromType("Vertex")// - .where().property("id").lt().parameter("value"); + final SelectCompiled select = database.select().fromType("Vertex")// + .where().property("id").lt().parameter("value").compile(); for (int i = 0; i < 110; i++) { final int finalI = i; - final QueryIterator result = select.parameter("value", i).vertices(); + final SelectIterator result = select.parameter("value", i).vertices(); final List list = result.toList(); Assertions.assertEquals(i, list.size()); list.forEach(r -> Assertions.assertTrue(r.getInteger("id") < finalI)); @@ -244,12 +240,12 @@ public void okRanges() { } { - final NativeSelect select = database.select().fromType("Vertex")// - .where().property("id").le().parameter("value"); + final SelectCompiled select = database.select().fromType("Vertex")// + .where().property("id").le().parameter("value").compile(); for (int i = 0; i < 110; i++) { final int finalI = i; - final QueryIterator result = select.parameter("value", i).vertices(); + final SelectIterator result = select.parameter("value", i).vertices(); final List list = result.toList(); Assertions.assertEquals(i + 1, list.size()); list.forEach(r -> Assertions.assertTrue(r.getInteger("id") <= finalI)); @@ -258,12 +254,12 @@ public void okRanges() { } { - final NativeSelect select = database.select().fromType("Vertex")// - .where().property("id").neq().parameter("value"); + final SelectCompiled select = database.select().fromType("Vertex")// + .where().property("id").neq().parameter("value").compile(); for (int i = 0; i < 110; i++) { final int finalI = i; - final QueryIterator result = select.parameter("value", i).vertices(); + final SelectIterator result = select.parameter("value", i).vertices(); final List list = result.toList(); Assertions.assertEquals(109, list.size()); list.forEach(r -> Assertions.assertTrue(r.getInteger("id") != finalI)); diff --git a/engine/src/test/java/com/arcadedb/query/sql/method/collection/SQLMethodTransformTest.java b/engine/src/test/java/com/arcadedb/query/sql/method/collection/SQLMethodTransformTest.java index 4dfc84cc2b..a7891087d1 100644 --- a/engine/src/test/java/com/arcadedb/query/sql/method/collection/SQLMethodTransformTest.java +++ b/engine/src/test/java/com/arcadedb/query/sql/method/collection/SQLMethodTransformTest.java @@ -49,7 +49,7 @@ import com.arcadedb.graph.Vertex; import com.arcadedb.index.IndexCursor; import com.arcadedb.query.QueryEngine; -import com.arcadedb.query.nativ.NativeSelect; +import com.arcadedb.query.select.Select; import com.arcadedb.query.sql.SQLQueryEngine; import com.arcadedb.query.sql.executor.BasicCommandContext; import com.arcadedb.query.sql.executor.ResultSet; @@ -353,7 +353,7 @@ public String getCurrentUserName() { } @Override - public NativeSelect select() { + public Select select() { return null; } diff --git a/engine/src/test/java/performance/LocalDatabaseBenchmark.java b/engine/src/test/java/performance/LocalDatabaseBenchmark.java index 80f13f2fa1..fad727fd12 100644 --- a/engine/src/test/java/performance/LocalDatabaseBenchmark.java +++ b/engine/src/test/java/performance/LocalDatabaseBenchmark.java @@ -23,15 +23,12 @@ import com.arcadedb.GlobalConfiguration; import com.arcadedb.database.Database; import com.arcadedb.database.DatabaseFactory; -import com.arcadedb.database.DatabaseInternal; import com.arcadedb.exception.ConcurrentModificationException; import com.arcadedb.graph.MutableVertex; -import com.arcadedb.query.nativ.NativeSelect; -import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.query.select.SelectCompiled; import org.junit.jupiter.api.Assertions; import java.util.*; -import java.util.concurrent.*; import java.util.concurrent.atomic.*; public class LocalDatabaseBenchmark { @@ -133,7 +130,7 @@ public void run() { private void queryNative() { final long begin = System.currentTimeMillis(); - final NativeSelect cached = database.select().fromType("User").where()// + final SelectCompiled cached = database.select().fromType("User").where()// .property("id").eq().parameter("id").and()// .property("id").eq().parameter("id").and()// .property("id").eq().parameter("id").and()// @@ -144,7 +141,7 @@ private void queryNative() { .property("id").eq().parameter("id").and()// .property("id").eq().parameter("id").and()// .property("id").eq().parameter("id")// - ; + .compile(); for (int i = 0; i < TOTAL * CONCURRENT_THREADS; i++) { Assertions.assertEquals(1, cached.parameter("id", i).vertices().toList().size()); } diff --git a/server/src/main/java/com/arcadedb/server/ServerDatabase.java b/server/src/main/java/com/arcadedb/server/ServerDatabase.java index 1beb4a8a6c..a0ad9a7e4f 100644 --- a/server/src/main/java/com/arcadedb/server/ServerDatabase.java +++ b/server/src/main/java/com/arcadedb/server/ServerDatabase.java @@ -49,7 +49,7 @@ import com.arcadedb.graph.Vertex; import com.arcadedb.index.IndexCursor; import com.arcadedb.query.QueryEngine; -import com.arcadedb.query.nativ.NativeSelect; +import com.arcadedb.query.select.Select; import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.query.sql.parser.ExecutionPlanCache; import com.arcadedb.query.sql.parser.StatementCache; @@ -111,7 +111,7 @@ public String getCurrentUserName() { } @Override - public NativeSelect select() { + public Select select() { return wrapped.select(); } diff --git a/server/src/main/java/com/arcadedb/server/ha/ReplicatedDatabase.java b/server/src/main/java/com/arcadedb/server/ha/ReplicatedDatabase.java index c6182e33d9..210c98ff03 100644 --- a/server/src/main/java/com/arcadedb/server/ha/ReplicatedDatabase.java +++ b/server/src/main/java/com/arcadedb/server/ha/ReplicatedDatabase.java @@ -56,7 +56,7 @@ import com.arcadedb.index.IndexCursor; import com.arcadedb.network.binary.ServerIsNotTheLeaderException; import com.arcadedb.query.QueryEngine; -import com.arcadedb.query.nativ.NativeSelect; +import com.arcadedb.query.select.Select; import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.query.sql.parser.ExecutionPlanCache; import com.arcadedb.query.sql.parser.StatementCache; @@ -360,7 +360,7 @@ public String getCurrentUserName() { } @Override - public NativeSelect select() { + public Select select() { return proxied.select(); } From 34e76c294d02ace68bd8d4250a1128b744953e39 Mon Sep 17 00:00:00 2001 From: lvca Date: Fri, 6 Oct 2023 19:42:20 -0400 Subject: [PATCH 17/19] feat: supported orderBy() with multiple properties asc/desc. If 1 index was used and the order by is by 1 property, then the resultset is already ordered and avoid to prefetch and sort in RAM --- .../database/IndexCursorCollection.java | 5 - .../com/arcadedb/index/EmptyIndexCursor.java | 7 +- .../java/com/arcadedb/index/IndexCursor.java | 2 - .../com/arcadedb/index/MultiIndexCursor.java | 18 +-- .../com/arcadedb/index/TempIndexCursor.java | 5 - .../index/lsm/LSMTreeIndexCursor.java | 39 +++--- .../com/arcadedb/query/select/Select.java | 39 +++--- ...electExecutor.java => SelectExecutor.java} | 76 +++++++---- .../arcadedb/query/select/SelectIterator.java | 74 ++++++++++- .../arcadedb/query/select/SelectOperator.java | 38 +++--- .../select/SelectWhereAfterFirstBlock.java | 4 + .../query/select/SelectIndexExecutionIT.java | 24 ++-- .../query/select/SelectOrderByIT.java | 121 ++++++++++++++++++ 13 files changed, 333 insertions(+), 119 deletions(-) rename engine/src/main/java/com/arcadedb/query/select/{SelectSelectExecutor.java => SelectExecutor.java} (76%) create mode 100644 engine/src/test/java/com/arcadedb/query/select/SelectOrderByIT.java diff --git a/engine/src/main/java/com/arcadedb/database/IndexCursorCollection.java b/engine/src/main/java/com/arcadedb/database/IndexCursorCollection.java index 595c6e7935..e6dd6a28c2 100644 --- a/engine/src/main/java/com/arcadedb/database/IndexCursorCollection.java +++ b/engine/src/main/java/com/arcadedb/database/IndexCursorCollection.java @@ -43,11 +43,6 @@ public Identifiable getRecord() { return last; } - @Override - public int getScore() { - return 0; - } - @Override public BinaryComparator getComparator() { return null; diff --git a/engine/src/main/java/com/arcadedb/index/EmptyIndexCursor.java b/engine/src/main/java/com/arcadedb/index/EmptyIndexCursor.java index 289e1f323c..3d08fd62b3 100644 --- a/engine/src/main/java/com/arcadedb/index/EmptyIndexCursor.java +++ b/engine/src/main/java/com/arcadedb/index/EmptyIndexCursor.java @@ -45,11 +45,6 @@ public Identifiable next() { throw new NoSuchElementException(); } - @Override - public int getScore() { - return 0; - } - @Override public BinaryComparator getComparator() { return null; @@ -62,7 +57,7 @@ public byte[] getBinaryKeyTypes() { @Override public long estimateSize() { - return 0l; + return 0L; } @Override diff --git a/engine/src/main/java/com/arcadedb/index/IndexCursor.java b/engine/src/main/java/com/arcadedb/index/IndexCursor.java index 2b49a32dcf..847a5c0bbc 100644 --- a/engine/src/main/java/com/arcadedb/index/IndexCursor.java +++ b/engine/src/main/java/com/arcadedb/index/IndexCursor.java @@ -32,8 +32,6 @@ public interface IndexCursor extends Cursor { Identifiable getRecord(); - int getScore(); - default void close() { // NO ACTIONS } diff --git a/engine/src/main/java/com/arcadedb/index/MultiIndexCursor.java b/engine/src/main/java/com/arcadedb/index/MultiIndexCursor.java index 8c5524cb46..28aad217f8 100644 --- a/engine/src/main/java/com/arcadedb/index/MultiIndexCursor.java +++ b/engine/src/main/java/com/arcadedb/index/MultiIndexCursor.java @@ -56,8 +56,8 @@ public MultiIndexCursor(final List indexes, final boolean ascendi initCursors(); } - public MultiIndexCursor(final List indexes, final Object[] fromKeys, final boolean ascendingOrder, final boolean includeFrom, - final int limit) { + public MultiIndexCursor(final List indexes, final Object[] fromKeys, final boolean ascendingOrder, + final boolean includeFrom, final int limit) { this.cursors = new ArrayList<>(indexes.size()); this.limit = limit; for (final Index i : indexes) { @@ -148,11 +148,6 @@ public Identifiable next() { return nextValue; } - @Override - public int getScore() { - return -1; - } - @Override public void close() { for (final IndexCursor cursor : cursors) @@ -162,11 +157,18 @@ public void close() { @Override public long estimateSize() { long tot = 0L; - for (final IndexCursor cursor : cursors) + for (final IndexCursor cursor : cursors) { + if (cursor.estimateSize() == -1) + return -1; tot += cursor.estimateSize(); + } return tot; } + public int getCursors() { + return cursors.size(); + } + @Override public Iterator iterator() { return this; diff --git a/engine/src/main/java/com/arcadedb/index/TempIndexCursor.java b/engine/src/main/java/com/arcadedb/index/TempIndexCursor.java index 9597f8a1e2..9f9fb0f2b6 100644 --- a/engine/src/main/java/com/arcadedb/index/TempIndexCursor.java +++ b/engine/src/main/java/com/arcadedb/index/TempIndexCursor.java @@ -54,11 +54,6 @@ public Identifiable next() { return current.record; } - @Override - public int getScore() { - return current.score; - } - @Override public BinaryComparator getComparator() { return null; diff --git a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexCursor.java b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexCursor.java index dbacb07b5e..580a356383 100644 --- a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexCursor.java +++ b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexCursor.java @@ -59,8 +59,8 @@ public LSMTreeIndexCursor(final LSMTreeIndexMutable index, final boolean ascendi this(index, ascendingOrder, null, true, null, true); } - public LSMTreeIndexCursor(final LSMTreeIndexMutable index, final boolean ascendingOrder, final Object[] fromKeys, final boolean beginKeysInclusive, - final Object[] toKeys, final boolean endKeysInclusive) throws IOException { + public LSMTreeIndexCursor(final LSMTreeIndexMutable index, final boolean ascendingOrder, final Object[] fromKeys, + final boolean beginKeysInclusive, final Object[] toKeys, final boolean endKeysInclusive) throws IOException { this.index = index; this.ascendingOrder = ascendingOrder; this.binaryKeyTypes = index.getBinaryKeyTypes(); @@ -118,13 +118,14 @@ public LSMTreeIndexCursor(final LSMTreeIndexMutable index, final boolean ascendi if (serializedFromKeys != null) { // SEEK FOR THE FROM RANGE - final BasePage currentPage = index.getDatabase().getTransaction().getPage(new PageId(index.getFileId(), pageId), index.getPageSize()); + final BasePage currentPage = index.getDatabase().getTransaction() + .getPage(new PageId(index.getFileId(), pageId), index.getPageSize()); final Binary currentPageBuffer = new Binary(currentPage.slice()); final int count = index.getCount(currentPage); if (count > 0) { - final LSMTreeIndexMutable.LookupResult lookupResult = index.lookupInPage(currentPage.getPageId().getPageNumber(), count, currentPageBuffer, - serializedFromKeys, ascendingOrder ? 2 : 3); + final LSMTreeIndexMutable.LookupResult lookupResult = index.lookupInPage(currentPage.getPageId().getPageNumber(), count, + currentPageBuffer, serializedFromKeys, ascendingOrder ? 2 : 3); if (!lookupResult.outside) { pageCursors[cursorIdx] = index.newPageIterator(pageId, lookupResult.keyIndex, ascendingOrder); @@ -148,7 +149,8 @@ public LSMTreeIndexCursor(final LSMTreeIndexMutable index, final boolean ascendi if (ascendingOrder) { pageCursors[cursorIdx] = index.newPageIterator(pageId, -1, true); } else { - final BasePage currentPage = index.getDatabase().getTransaction().getPage(new PageId(index.getFileId(), pageId), index.getPageSize()); + final BasePage currentPage = index.getDatabase().getTransaction() + .getPage(new PageId(index.getFileId(), pageId), index.getPageSize()); pageCursors[cursorIdx] = index.newPageIterator(pageId, index.getCount(currentPage), false); } @@ -249,9 +251,11 @@ public String dumpStats() { if (cursor == null) buffer.append(String.format("%n- Cursor[%d] = null", i)); else { - buffer.append(String.format("%n- Cursor[%d] %s=%s index=%s compacted=%s totalKeys=%d ascending=%s keyTypes=%s currentPageId=%s currentPosInPage=%d", i, - Arrays.toString(cursorKeys[i]), Arrays.toString(cursor.getValue()), cursor.index, cursor instanceof LSMTreeIndexUnderlyingCompactedSeriesCursor, - cursor.totalKeys, cursor.ascendingOrder, Arrays.toString(cursor.keyTypes), cursor.getCurrentPageId(), cursor.getCurrentPositionInPage())); + buffer.append(String.format( + "%n- Cursor[%d] %s=%s index=%s compacted=%s totalKeys=%d ascending=%s keyTypes=%s currentPageId=%s currentPosInPage=%d", + i, Arrays.toString(cursorKeys[i]), Arrays.toString(cursor.getValue()), cursor.index, + cursor instanceof LSMTreeIndexUnderlyingCompactedSeriesCursor, cursor.totalKeys, cursor.ascendingOrder, + Arrays.toString(cursor.keyTypes), cursor.getCurrentPageId(), cursor.getCurrentPositionInPage())); } } @@ -391,8 +395,8 @@ else if (tempCurrentValues.length > 0) { if (serializedToKeys != null) { final int compare = LSMTreeIndexMutable.compareKeys(comparator, binaryKeyTypes, cursorKeys[minorKeyIndex], toKeys); - if ((ascendingOrder && ((toKeysInclusive && compare > 0) || (!toKeysInclusive && compare >= 0))) || (!ascendingOrder && ( - (toKeysInclusive && compare < 0) || (!toKeysInclusive && compare <= 0)))) { + if ((ascendingOrder && ((toKeysInclusive && compare > 0) || (!toKeysInclusive && compare >= 0))) || (!ascendingOrder + && ((toKeysInclusive && compare < 0) || (!toKeysInclusive && compare <= 0)))) { currentCursor.close(); pageCursors[minorKeyIndex] = null; cursorKeys[minorKeyIndex] = null; @@ -420,8 +424,9 @@ else if (tempCurrentValues.length > 0) { if (txCursor == null || !txCursor.hasNext()) getClosestEntryInTx(currentKeys != null ? currentKeys : fromKeys, false); - } while ((currentValues == null || currentValues.length == 0 || (currentValueIndex < currentValues.length && index.isDeletedEntry( - currentValues[currentValueIndex]))) && hasNext()); + } while ( + (currentValues == null || currentValues.length == 0 || (currentValueIndex < currentValues.length && index.isDeletedEntry( + currentValues[currentValueIndex]))) && hasNext()); return currentValues == null || currentValueIndex >= currentValues.length ? null : currentValues[currentValueIndex++]; } @@ -451,7 +456,8 @@ else if (inclusive) entry = indexChanges.lowerEntry(new TransactionIndexContext.ComparableKey(keys)); } - final Map values = entry != null ? entry.getValue() : null; + final Map values = + entry != null ? entry.getValue() : null; if (values != null) { for (final TransactionIndexContext.IndexKey value : values.values()) { if (value != null) { @@ -502,11 +508,6 @@ public Identifiable getRecord() { return null; } - @Override - public int getScore() { - return 1; - } - @Override public void close() { for (final LSMTreeIndexUnderlyingAbstractCursor it : pageCursors) diff --git a/engine/src/main/java/com/arcadedb/query/select/Select.java b/engine/src/main/java/com/arcadedb/query/select/Select.java index 77c2ddd658..dd6dbb4e15 100644 --- a/engine/src/main/java/com/arcadedb/query/select/Select.java +++ b/engine/src/main/java/com/arcadedb/query/select/Select.java @@ -22,6 +22,7 @@ import com.arcadedb.schema.DocumentType; import com.arcadedb.serializer.json.JSONArray; import com.arcadedb.serializer.json.JSONObject; +import com.arcadedb.utility.Pair; import java.util.*; import java.util.concurrent.*; @@ -39,18 +40,18 @@ public class Select { enum STATE {DEFAULT, WHERE, COMPILED} - Map parameters; - - SelectTreeNode rootTreeElement; - DocumentType fromType; - List fromBuckets; - SelectOperator operator; - SelectRuntimeValue property; - Object propertyValue; - boolean polymorphic = true; - int limit = -1; - long timeoutInMs = 0; - boolean exceptionOnTimeout; + Map parameters; + SelectTreeNode rootTreeElement; + DocumentType fromType; + List fromBuckets; + SelectOperator operator; + SelectRuntimeValue property; + Object propertyValue; + boolean polymorphic = true; + int limit = -1; + long timeoutInMs = 0; + boolean exceptionOnTimeout; + ArrayList> orderBy; STATE state = STATE.DEFAULT; private SelectTreeNode lastTreeElement; @@ -163,6 +164,14 @@ public Select polymorphic(final boolean polymorphic) { return this; } + public Select orderBy(final String property, final boolean ascending) { + checkNotCompiled(); + if (this.orderBy == null) + this.orderBy = new ArrayList<>(); + this.orderBy.add(new Pair<>(property, ascending)); + return this; + } + public Select json(final JSONObject json) { checkNotCompiled(); if (json.has("fromType")) { @@ -222,10 +231,10 @@ else if (parsedRight instanceof String && ((String) parsedRight).startsWith("#") public SelectCompiled compile() { if (fromType == null && fromBuckets == null) throw new IllegalArgumentException("from (type or buckets) has not been set"); - if (state != STATE.COMPILED) { + if (state == STATE.WHERE) { setLogic(SelectOperator.run); - state = STATE.COMPILED; } + state = STATE.COMPILED; return new SelectCompiled(this); } @@ -243,7 +252,7 @@ public SelectIterator documents() { SelectIterator run() { compile(); - return new SelectSelectExecutor(this).execute(); + return new SelectExecutor(this).execute(); } SelectWhereLeftBlock setLogic(final SelectOperator newLogicOperator) { diff --git a/engine/src/main/java/com/arcadedb/query/select/SelectSelectExecutor.java b/engine/src/main/java/com/arcadedb/query/select/SelectExecutor.java similarity index 76% rename from engine/src/main/java/com/arcadedb/query/select/SelectSelectExecutor.java rename to engine/src/main/java/com/arcadedb/query/select/SelectExecutor.java index dba6f383f4..86ff02cfe4 100644 --- a/engine/src/main/java/com/arcadedb/query/select/SelectSelectExecutor.java +++ b/engine/src/main/java/com/arcadedb/query/select/SelectExecutor.java @@ -17,10 +17,12 @@ import com.arcadedb.database.Document; import com.arcadedb.database.Identifiable; import com.arcadedb.engine.Bucket; +import com.arcadedb.index.Index; import com.arcadedb.index.IndexCursor; import com.arcadedb.index.MultiIndexCursor; import com.arcadedb.index.TypeIndex; import com.arcadedb.utility.MultiIterator; +import com.arcadedb.utility.Pair; import java.util.*; @@ -31,17 +33,29 @@ * * @author Luca Garulli (l.garulli@arcadedata.com) */ -public class SelectSelectExecutor { - final Select select; - private long evaluatedRecords = 0; - private int indexesUsed = 0; +public class SelectExecutor { + final Select select; + long evaluatedRecords = 0; + List usedIndexes = null; + + class IndexInfo { + public final Index index; + public final String property; + public final boolean order; + + IndexInfo(final Index index, final String property, final boolean order) { + this.index = index; + this.property = property; + this.order = order; + } + } - public SelectSelectExecutor(final Select select) { + public SelectExecutor(final Select select) { this.select = select; } SelectIterator execute() { - final Iterator iteratorFromIndexes = lookForIndexes(); + final MultiIndexCursor iteratorFromIndexes = lookForIndexes(); final Iterator iterator; @@ -61,10 +75,10 @@ else if (select.fromBuckets.size() == 1) if (select.timeoutInMs > 0 && iterator instanceof MultiIterator) ((MultiIterator) iterator).setTimeout(select.timeoutInMs, select.exceptionOnTimeout); - return new SelectIterator<>(this, iterator, iteratorFromIndexes != null); + return new SelectIterator<>(this, iterator, iteratorFromIndexes != null && iteratorFromIndexes.getCursors() > 1); } - private Iterator lookForIndexes() { + private MultiIndexCursor lookForIndexes() { if (select.fromType != null && select.rootTreeElement != null) { final List cursors = new ArrayList<>(); @@ -72,10 +86,8 @@ private Iterator lookForIndexes() { boolean canUseIndexes = isTheNodeFullyIndexed(select.rootTreeElement); filterWithIndexes(select.rootTreeElement, cursors); - if (!cursors.isEmpty()) { - indexesUsed = cursors.size(); + if (!cursors.isEmpty()) return new MultiIndexCursor(cursors, select.limit, true); - } } return null; } @@ -107,19 +119,35 @@ private void filterWithIndexesFinalNode(final SelectTreeNode node, final List entry : select.orderBy) { + if (propertyName.equals(entry.getFirst())) { + if (!entry.getSecond()) + ascendingOrder = false; + break; + } + } + + if (node.operator == SelectOperator.gt) + cursor = node.index.range(ascendingOrder, new Object[] { rightValue }, false, null, false); + else if (node.operator == SelectOperator.ge) + cursor = node.index.range(ascendingOrder, new Object[] { rightValue }, true, null, false); + else if (node.operator == SelectOperator.lt) + cursor = node.index.range(ascendingOrder, null, false, new Object[] { rightValue }, false); + else if (node.operator == SelectOperator.le) + cursor = node.index.range(ascendingOrder, null, false, new Object[] { rightValue }, true); + else + return; + } final SelectTreeNode parentNode = node.getParent(); if (parentNode.operator == SelectOperator.and && parentNode.left == node) { @@ -138,6 +166,10 @@ else if (node.operator == SelectOperator.le) } } + if (usedIndexes == null) + usedIndexes = new ArrayList<>(); + + usedIndexes.add(new IndexInfo(node.index, propertyName, ascendingOrder)); cursors.add(cursor); } @@ -184,7 +216,7 @@ else if (value instanceof SelectRuntimeValue) } public Map metrics() { - return Map.of("evaluatedRecords", evaluatedRecords, "indexesUsed", indexesUsed); + return Map.of("evaluatedRecords", evaluatedRecords, "usedIndexes", usedIndexes != null ? usedIndexes.size() : 0); } boolean evaluateWhere(final Document record) { diff --git a/engine/src/main/java/com/arcadedb/query/select/SelectIterator.java b/engine/src/main/java/com/arcadedb/query/select/SelectIterator.java index 833962a002..e184679131 100644 --- a/engine/src/main/java/com/arcadedb/query/select/SelectIterator.java +++ b/engine/src/main/java/com/arcadedb/query/select/SelectIterator.java @@ -17,30 +17,52 @@ import com.arcadedb.database.Document; import com.arcadedb.database.Identifiable; import com.arcadedb.database.RID; +import com.arcadedb.serializer.BinaryComparator; +import com.arcadedb.utility.Pair; import java.util.*; /** * Query iterator returned from queries. Extends the base Java iterator with convenient methods. * + *

Implementation details

+ *

+ * The iterator keeps track for the returned records in case multiple indexes have being used. In fact, in case multiple + * indexes are used, it's much simpler to just return index cursor tha could overlap. In this case, the property + * `filterOutRecords` keeps track of the returning RIDs to avoid returning duplicates.

+ * * @author Luca Garulli (l.garulli@arcadedata.com) */ public class SelectIterator implements Iterator { - private final SelectSelectExecutor executor; + private final SelectExecutor executor; private final Iterator iterator; private HashSet filterOutRecords; - private T next = null; - private long returned = 0; + private T next = null; + private long returned = 0; + private List sortedResultSet; + private int orderIndex = 0; - protected SelectIterator(final SelectSelectExecutor executor, final Iterator iterator, final boolean uniqueResult) { + protected SelectIterator(final SelectExecutor executor, final Iterator iterator, + final boolean enforceUniqueReturn) { this.executor = executor; this.iterator = iterator; - if (uniqueResult) + if (enforceUniqueReturn) this.filterOutRecords = new HashSet<>(); + + fetchResultInCaseOfOrderBy(); } @Override public boolean hasNext() { + if (sortedResultSet != null) { + // RETURN FROM THE ORDERED RESULT SET + final boolean more = orderIndex < sortedResultSet.size(); + if (!more) + // EARLY DISPOSAL OF SORTED RESULT SET + sortedResultSet = null; + return more; + } + if (executor.select.limit > -1 && returned >= executor.select.limit) return false; if (next != null) @@ -54,6 +76,10 @@ public boolean hasNext() { @Override public T next() { + if (sortedResultSet != null) + // RETURN FROM THE ORDERED RESULT SET + return (T) sortedResultSet.get(orderIndex++); + if (next == null && !hasNext()) throw new NoSuchElementException(); try { @@ -71,7 +97,7 @@ private T fetchNext() { // ALREADY RETURNED, AVOID DUPLICATES IN THE RESULTSET continue; - if (executor.evaluateWhere(record)) { + if (executor.select.rootTreeElement == null || executor.evaluateWhere(record)) { ++returned; if (filterOutRecords != null) @@ -100,4 +126,40 @@ public List toList() { public Map getMetrics() { return executor.metrics(); } + + private void fetchResultInCaseOfOrderBy() { + if (executor.select.orderBy == null) + return; + + // CHECK ONLY THE CASE WITH ONE INDEX USED AND ONE ORDER BY + if (executor.select.orderBy.size() == 1 && executor.usedIndexes != null && executor.usedIndexes.size() == 1) { + final Pair orderBy = executor.select.orderBy.get(0); + final SelectExecutor.IndexInfo usedIndex = executor.usedIndexes.get(0); + + if (orderBy.getFirst().equals(usedIndex.property) &&// + orderBy.getSecond() == usedIndex.order) { + // ORDER BY THE INDEX USED, RESULTSET IS ALREADY ORDERED + return; + } + } + + final List resultSet = new ArrayList<>(); + while (hasNext()) + resultSet.add(next().asDocument(true)); + sortedResultSet = resultSet; + + Collections.sort(sortedResultSet, (a, b) -> { + for (Pair orderBy : executor.select.orderBy) { + final Object aVal = a.get(orderBy.getFirst()); + final Object bVal = b.get(orderBy.getFirst()); + int comp = BinaryComparator.compareTo(aVal, bVal); + if (comp != 0) { + if (!orderBy.getSecond()) + comp *= -1; + return comp; + } + } + return 0; + }); + } } diff --git a/engine/src/main/java/com/arcadedb/query/select/SelectOperator.java b/engine/src/main/java/com/arcadedb/query/select/SelectOperator.java index 3066d2e307..7a6ce33ac3 100644 --- a/engine/src/main/java/com/arcadedb/query/select/SelectOperator.java +++ b/engine/src/main/java/com/arcadedb/query/select/SelectOperator.java @@ -30,22 +30,22 @@ public enum SelectOperator { or("or", true, 0) { @Override Object eval(final Document record, final Object left, final Object right) { - final Boolean leftValue = (Boolean) SelectSelectExecutor.evaluateValue(record, left); + final Boolean leftValue = (Boolean) SelectExecutor.evaluateValue(record, left); if (leftValue) return true; - return SelectSelectExecutor.evaluateValue(record, right); + return SelectExecutor.evaluateValue(record, right); } }, and("and", true, 2) { @Override Object eval(final Document record, final Object left, final Object right) { - final Boolean leftValue = (Boolean) SelectSelectExecutor.evaluateValue(record, left); + final Boolean leftValue = (Boolean) SelectExecutor.evaluateValue(record, left); if (!leftValue) return false; - return SelectSelectExecutor.evaluateValue(record, right); + return SelectExecutor.evaluateValue(record, right); } }, @@ -59,61 +59,61 @@ Object eval(final Document record, final Object left, final Object right) { eq("=", false, 1) { @Override Object eval(final Document record, final Object left, final Object right) { - return BinaryComparator.equals(SelectSelectExecutor.evaluateValue(record, left), - SelectSelectExecutor.evaluateValue(record, right)); + return BinaryComparator.equals(SelectExecutor.evaluateValue(record, left), + SelectExecutor.evaluateValue(record, right)); } }, neq("<>", false, 1) { @Override Object eval(final Document record, final Object left, final Object right) { - return !BinaryComparator.equals(SelectSelectExecutor.evaluateValue(record, left), - SelectSelectExecutor.evaluateValue(record, right)); + return !BinaryComparator.equals(SelectExecutor.evaluateValue(record, left), + SelectExecutor.evaluateValue(record, right)); } }, lt("<", false, 1) { @Override Object eval(final Document record, final Object left, final Object right) { - return BinaryComparator.compareTo(SelectSelectExecutor.evaluateValue(record, left), - SelectSelectExecutor.evaluateValue(record, right)) < 0; + return BinaryComparator.compareTo(SelectExecutor.evaluateValue(record, left), + SelectExecutor.evaluateValue(record, right)) < 0; } }, le("<=", false, 1) { @Override Object eval(final Document record, final Object left, final Object right) { - return BinaryComparator.compareTo(SelectSelectExecutor.evaluateValue(record, left), - SelectSelectExecutor.evaluateValue(record, right)) <= 0; + return BinaryComparator.compareTo(SelectExecutor.evaluateValue(record, left), + SelectExecutor.evaluateValue(record, right)) <= 0; } }, gt(">", false, 1) { @Override Object eval(final Document record, final Object left, final Object right) { - return BinaryComparator.compareTo(SelectSelectExecutor.evaluateValue(record, left), - SelectSelectExecutor.evaluateValue(record, right)) > 0; + return BinaryComparator.compareTo(SelectExecutor.evaluateValue(record, left), + SelectExecutor.evaluateValue(record, right)) > 0; } }, ge(">=", false, 1) { @Override Object eval(final Document record, final Object left, final Object right) { - return BinaryComparator.compareTo(SelectSelectExecutor.evaluateValue(record, left), - SelectSelectExecutor.evaluateValue(record, right)) >= 0; + return BinaryComparator.compareTo(SelectExecutor.evaluateValue(record, left), + SelectExecutor.evaluateValue(record, right)) >= 0; } }, like("like", false, 1) { @Override Object eval(final Document record, final Object left, final Object right) { - return QueryHelper.like((String) SelectSelectExecutor.evaluateValue(record, left), - (String) SelectSelectExecutor.evaluateValue(record, right)); + return QueryHelper.like((String) SelectExecutor.evaluateValue(record, left), + (String) SelectExecutor.evaluateValue(record, right)); } }, run("!", true, -1) { @Override Object eval(final Document record, final Object left, final Object right) { - return SelectSelectExecutor.evaluateValue(record, left); + return SelectExecutor.evaluateValue(record, left); } }; diff --git a/engine/src/main/java/com/arcadedb/query/select/SelectWhereAfterFirstBlock.java b/engine/src/main/java/com/arcadedb/query/select/SelectWhereAfterFirstBlock.java index cd6609ebb4..fad7d5b38b 100644 --- a/engine/src/main/java/com/arcadedb/query/select/SelectWhereAfterFirstBlock.java +++ b/engine/src/main/java/com/arcadedb/query/select/SelectWhereAfterFirstBlock.java @@ -65,4 +65,8 @@ public Select limit(final int limit) { public Select timeout(final long timeoutValue, final TimeUnit timeoutUnit, final boolean exceptionOnTimeout) { return select.timeout(timeoutValue, timeoutUnit, exceptionOnTimeout); } + + public Select orderBy(final String property, final boolean order) { + return select.orderBy(property, order); + } } diff --git a/engine/src/test/java/com/arcadedb/query/select/SelectIndexExecutionIT.java b/engine/src/test/java/com/arcadedb/query/select/SelectIndexExecutionIT.java index 020b987636..c3f562861c 100644 --- a/engine/src/test/java/com/arcadedb/query/select/SelectIndexExecutionIT.java +++ b/engine/src/test/java/com/arcadedb/query/select/SelectIndexExecutionIT.java @@ -72,7 +72,7 @@ public void okOneOfTwoAvailableIndexes() { // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) Assertions.assertEquals(1L, result.getMetrics().get("evaluatedRecords"), "With id " + i); - Assertions.assertEquals(1, result.getMetrics().get("indexesUsed")); + Assertions.assertEquals(1, result.getMetrics().get("usedIndexes")); } } } @@ -93,7 +93,7 @@ public void okBothAvailableIndexes() { // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) Assertions.assertEquals(i < 100 ? 100L : 101L, result.getMetrics().get("evaluatedRecords"), "" + finalI); - Assertions.assertEquals(2, result.getMetrics().get("indexesUsed")); + Assertions.assertEquals(2, result.getMetrics().get("usedIndexes")); } } } @@ -114,7 +114,7 @@ public void okOneIndexUsed() { // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) Assertions.assertEquals(1L, result.getMetrics().get("evaluatedRecords")); - Assertions.assertEquals(1, result.getMetrics().get("indexesUsed")); + Assertions.assertEquals(1, result.getMetrics().get("usedIndexes")); } } @@ -132,7 +132,7 @@ public void okOneIndexUsed() { // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) Assertions.assertEquals(1L, result.getMetrics().get("evaluatedRecords")); - Assertions.assertEquals(1, result.getMetrics().get("indexesUsed")); + Assertions.assertEquals(1, result.getMetrics().get("usedIndexes")); } } } @@ -151,7 +151,7 @@ public void okNoIndexUsed() { // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) Assertions.assertEquals(110L, result.getMetrics().get("evaluatedRecords")); - Assertions.assertEquals(0, result.getMetrics().get("indexesUsed")); + Assertions.assertEquals(0, result.getMetrics().get("usedIndexes")); } } @@ -167,7 +167,7 @@ public void okNoIndexUsed() { // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) Assertions.assertEquals(110L, result.getMetrics().get("evaluatedRecords")); - Assertions.assertEquals(0, result.getMetrics().get("indexesUsed")); + Assertions.assertEquals(0, result.getMetrics().get("usedIndexes")); } } @@ -189,7 +189,7 @@ public void okNoIndexUsed() { // CHECK 1 FOR ID = I + 100 FOR NAME = ELON (ALL OF THEM) Assertions.assertEquals(1L, result.getMetrics().get("evaluatedRecords")); - Assertions.assertEquals(2, result.getMetrics().get("indexesUsed")); + Assertions.assertEquals(2, result.getMetrics().get("usedIndexes")); } } @@ -207,7 +207,7 @@ public void okRanges() { final List list = result.toList(); Assertions.assertEquals(109 - i, list.size()); list.forEach(r -> Assertions.assertTrue(r.getInteger("id") > finalI)); - Assertions.assertEquals(1, result.getMetrics().get("indexesUsed")); + Assertions.assertEquals(1, result.getMetrics().get("usedIndexes")); } } @@ -221,7 +221,7 @@ public void okRanges() { final List list = result.toList(); Assertions.assertEquals(110 - i, list.size()); list.forEach(r -> Assertions.assertTrue(r.getInteger("id") >= finalI)); - Assertions.assertEquals(1, result.getMetrics().get("indexesUsed")); + Assertions.assertEquals(1, result.getMetrics().get("usedIndexes")); } } @@ -235,7 +235,7 @@ public void okRanges() { final List list = result.toList(); Assertions.assertEquals(i, list.size()); list.forEach(r -> Assertions.assertTrue(r.getInteger("id") < finalI)); - Assertions.assertEquals(1, result.getMetrics().get("indexesUsed")); + Assertions.assertEquals(1, result.getMetrics().get("usedIndexes")); } } @@ -249,7 +249,7 @@ public void okRanges() { final List list = result.toList(); Assertions.assertEquals(i + 1, list.size()); list.forEach(r -> Assertions.assertTrue(r.getInteger("id") <= finalI)); - Assertions.assertEquals(1, result.getMetrics().get("indexesUsed")); + Assertions.assertEquals(1, result.getMetrics().get("usedIndexes")); } } @@ -263,7 +263,7 @@ public void okRanges() { final List list = result.toList(); Assertions.assertEquals(109, list.size()); list.forEach(r -> Assertions.assertTrue(r.getInteger("id") != finalI)); - Assertions.assertEquals(0, result.getMetrics().get("indexesUsed")); + Assertions.assertEquals(0, result.getMetrics().get("usedIndexes")); } } } diff --git a/engine/src/test/java/com/arcadedb/query/select/SelectOrderByIT.java b/engine/src/test/java/com/arcadedb/query/select/SelectOrderByIT.java new file mode 100644 index 0000000000..c87f2c4018 --- /dev/null +++ b/engine/src/test/java/com/arcadedb/query/select/SelectOrderByIT.java @@ -0,0 +1,121 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +package com.arcadedb.query.select; + +import com.arcadedb.TestHelper; +import com.arcadedb.graph.Vertex; +import com.arcadedb.schema.Schema; +import com.arcadedb.schema.Type; +import com.arcadedb.schema.VertexType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class SelectOrderByIT extends TestHelper { + + public SelectOrderByIT() { + autoStartTx = true; + } + + @Override + protected void beginTest() { + final VertexType v = database.getSchema().createVertexType("Vertex"); + v.createProperty("id", Type.INTEGER)// + .createIndex(Schema.INDEX_TYPE.LSM_TREE, true); + v.createProperty("name", Type.STRING)// + .createIndex(Schema.INDEX_TYPE.LSM_TREE, false); + + database.transaction(() -> { + for (int i = 0; i < 100; i++) + database.newVertex("Vertex").set("id", i, "notIndexedId", i, "float", i + 3.14F, "name", "Elon").save(); + for (int i = 100; i < 110; i++) + database.newVertex("Vertex").set("id", i, "notIndexedId", i, "name", "Jay").save(); + }); + } + + @Test + public void okOrderByNoIndex() { + // ASCENDING + { + final SelectCompiled select = database.select().fromType("Vertex").orderBy("notIndexedId", true).compile(); + int lastId = -1; + final SelectIterator result = select.vertices(); + + while (result.hasNext()) { + final Integer id = result.next().getInteger("notIndexedId"); + Assertions.assertTrue(id > lastId); + lastId = id; + } + + Assertions.assertEquals(0, result.getMetrics().get("usedIndexes")); + } + + // DESCENDING + { + final SelectCompiled select = database.select().fromType("Vertex").orderBy("notIndexedId", false).compile(); + int lastId = Integer.MAX_VALUE; + final SelectIterator result = select.vertices(); + + while (result.hasNext()) { + final Integer id = result.next().getInteger("notIndexedId"); + Assertions.assertTrue(id < lastId); + lastId = id; + } + + Assertions.assertEquals(0, result.getMetrics().get("usedIndexes")); + + } + } + + @Test + public void okOrderBy1Index() { + // ASCENDING + { + final SelectCompiled select = database.select().fromType("Vertex").where().property("id").gt().value(-1).orderBy("id", true).compile(); + int lastId = -1; + final SelectIterator result = select.vertices(); + + while (result.hasNext()) { + final Integer id = result.next().getInteger("id"); + Assertions.assertTrue(id > lastId); + lastId = id; + } + + Assertions.assertEquals(1, result.getMetrics().get("usedIndexes")); + } + + // DESCENDING + { + final SelectCompiled select = database.select().fromType("Vertex").where().property("id").gt().value(-1).orderBy("id", false).compile(); + int lastId = Integer.MAX_VALUE; + final SelectIterator result = select.vertices(); + + while (result.hasNext()) { + final Integer id = result.next().getInteger("id"); + Assertions.assertTrue(id < lastId); + lastId = id; + } + + Assertions.assertEquals(1, result.getMetrics().get("usedIndexes")); + + } + } +} From ab43a31db4b8c4ab457f46fda52f98b27dec1b52 Mon Sep 17 00:00:00 2001 From: lvca Date: Sat, 7 Oct 2023 00:31:58 -0400 Subject: [PATCH 18/19] fix: fixed NPE introduced by recent optimization with SQL operators --- .../com/arcadedb/query/sql/function/math/SQLFunctionMax.java | 4 ++-- .../com/arcadedb/query/sql/function/math/SQLFunctionMin.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMax.java b/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMax.java index 3442a6883b..d23b323ee2 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMax.java +++ b/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMax.java @@ -52,8 +52,8 @@ public Object execute(final Object iThis, final Identifiable iCurrentRecord, fin max = subitem; } } else { - if (!item.getClass().equals(max.getClass()) &&// - item instanceof Number && max instanceof Number) { + if (item instanceof Number && max instanceof Number &&// + !item.getClass().equals(max.getClass())) { final Number[] converted = Type.castComparableNumber((Number) item, (Number) max); item = converted[0]; max = converted[1]; diff --git a/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMin.java b/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMin.java index a6e2b6fd84..e11a278eb8 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMin.java +++ b/engine/src/main/java/com/arcadedb/query/sql/function/math/SQLFunctionMin.java @@ -53,8 +53,8 @@ public Object execute(final Object iThis, final Identifiable iCurrentRecord, fin min = subitem; } } else { - if (!item.getClass().equals(min.getClass()) &&// - item instanceof Number && min instanceof Number) { + if (item instanceof Number && min instanceof Number &&// + !item.getClass().equals(min.getClass())) { final Number[] converted = Type.castComparableNumber((Number) item, (Number) min); item = converted[0]; min = converted[1]; From ab0f80086820ddb196dd6746cad0c6933fb7f207 Mon Sep 17 00:00:00 2001 From: lvca Date: Mon, 16 Oct 2023 15:15:17 -0400 Subject: [PATCH 19/19] feat: supported SKIP in native select --- .../com/arcadedb/query/select/Select.java | 9 ++++ .../arcadedb/query/select/SelectCompiled.java | 2 + .../arcadedb/query/select/SelectIterator.java | 6 +++ .../select/SelectWhereAfterFirstBlock.java | 4 ++ .../query/select/SelectExecutionIT.java | 42 ++++++++++++++++++- 5 files changed, 61 insertions(+), 2 deletions(-) diff --git a/engine/src/main/java/com/arcadedb/query/select/Select.java b/engine/src/main/java/com/arcadedb/query/select/Select.java index dd6dbb4e15..98f6c9b1ba 100644 --- a/engine/src/main/java/com/arcadedb/query/select/Select.java +++ b/engine/src/main/java/com/arcadedb/query/select/Select.java @@ -49,6 +49,7 @@ enum STATE {DEFAULT, WHERE, COMPILED} Object propertyValue; boolean polymorphic = true; int limit = -1; + int skip = 0; long timeoutInMs = 0; boolean exceptionOnTimeout; ArrayList> orderBy; @@ -149,6 +150,12 @@ public Select limit(final int limit) { return this; } + public Select skip(final int skip) { + checkNotCompiled(); + this.skip = skip; + return this; + } + public Select timeout(final long timeoutValue, final TimeUnit timeoutUnit, final boolean exceptionOnTimeout) { checkNotCompiled(); this.timeoutInMs = timeoutUnit.toMillis(timeoutValue); @@ -192,6 +199,8 @@ public Select json(final JSONObject json) { if (json.has("limit")) limit(json.getInt("limit")); + if (json.has("skip")) + skip(json.getInt("skip")); return this; } diff --git a/engine/src/main/java/com/arcadedb/query/select/SelectCompiled.java b/engine/src/main/java/com/arcadedb/query/select/SelectCompiled.java index 4822aefda0..fd79d75a3f 100644 --- a/engine/src/main/java/com/arcadedb/query/select/SelectCompiled.java +++ b/engine/src/main/java/com/arcadedb/query/select/SelectCompiled.java @@ -58,6 +58,8 @@ public JSONObject json() { if (select.limit > -1) json.put("limit", select.limit); + if (select.skip > -1) + json.put("skip", select.skip); if (select.timeoutInMs > 0) { json.put("timeoutInMs", select.timeoutInMs); json.put("exceptionOnTimeout", select.exceptionOnTimeout); diff --git a/engine/src/main/java/com/arcadedb/query/select/SelectIterator.java b/engine/src/main/java/com/arcadedb/query/select/SelectIterator.java index e184679131..54637e897f 100644 --- a/engine/src/main/java/com/arcadedb/query/select/SelectIterator.java +++ b/engine/src/main/java/com/arcadedb/query/select/SelectIterator.java @@ -50,6 +50,12 @@ protected SelectIterator(final SelectExecutor executor, final Iterator(); fetchResultInCaseOfOrderBy(); + + for (int i = 0; i < executor.select.skip; i++) { + // CONSUME UNTIL THE SKIP THRESHOLD IS LIMIT + if (hasNext()) + next(); + } } @Override diff --git a/engine/src/main/java/com/arcadedb/query/select/SelectWhereAfterFirstBlock.java b/engine/src/main/java/com/arcadedb/query/select/SelectWhereAfterFirstBlock.java index fad7d5b38b..f4765f033d 100644 --- a/engine/src/main/java/com/arcadedb/query/select/SelectWhereAfterFirstBlock.java +++ b/engine/src/main/java/com/arcadedb/query/select/SelectWhereAfterFirstBlock.java @@ -62,6 +62,10 @@ public Select limit(final int limit) { return select.limit(limit); } + public Select skip(final int skip) { + return select.skip(skip); + } + public Select timeout(final long timeoutValue, final TimeUnit timeoutUnit, final boolean exceptionOnTimeout) { return select.timeout(timeoutValue, timeoutUnit, exceptionOnTimeout); } diff --git a/engine/src/test/java/com/arcadedb/query/select/SelectExecutionIT.java b/engine/src/test/java/com/arcadedb/query/select/SelectExecutionIT.java index d80fc46e50..1008dc61ab 100644 --- a/engine/src/test/java/com/arcadedb/query/select/SelectExecutionIT.java +++ b/engine/src/test/java/com/arcadedb/query/select/SelectExecutionIT.java @@ -211,6 +211,45 @@ public void okLimit() { Assertions.assertEquals(10, browsed); } + @Test + public void okSkip() { + SelectCompiled select = database.select().fromType("Vertex")// + .where().property("id").lt().value(10)// + .and().property("name").eq().value("Elon").skip(10).compile(); + + SelectIterator iter = select.vertices(); + int browsed = 0; + while (iter.hasNext()) { + Assertions.assertTrue(iter.next().getInteger("id") < 10); + ++browsed; + } + Assertions.assertEquals(0, browsed); + + select = database.select().fromType("Vertex")// + .where().property("id").lt().value(10)// + .and().property("name").eq().value("Elon").skip(0).compile(); + + iter = select.vertices(); + browsed = 0; + while (iter.hasNext()) { + Assertions.assertTrue(iter.next().getInteger("id") < 10); + ++browsed; + } + Assertions.assertEquals(10, browsed); + + select = database.select().fromType("Vertex")// + .where().property("id").lt().value(10)// + .and().property("name").eq().value("Elon").skip(2).compile(); + + iter = select.vertices(); + browsed = 0; + while (iter.hasNext()) { + Assertions.assertTrue(iter.next().getInteger("id") < 10); + ++browsed; + } + Assertions.assertEquals(8, browsed); + } + @Test public void okUpdate() { database.select().fromType("Vertex")// @@ -288,8 +327,7 @@ public void errorMissingParameter() { @Test public void okReuse() { - final SelectCompiled select = database.select().fromType("Vertex").where().property("id").eq().parameter("value") - .compile(); + final SelectCompiled select = database.select().fromType("Vertex").where().property("id").eq().parameter("value").compile(); for (int i = 0; i < 100; i++) Assertions.assertEquals(i, select.parameter("value", i).vertices().nextOrNull().getInteger("id")); }