From 53f7660e1d96f935c8fc59970b56dbc9ed03aa9e Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Fri, 4 Sep 2015 07:10:15 -0400 Subject: [PATCH 01/32] Basic compilation against TinkerPop 3.1.0-SNAPSHOT Three tests not passing right now. --- pom.xml | 2 +- .../titan/graphdb/tinkerpop/TitanBlueprintsGraph.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index c37b374420..bdd3707db3 100644 --- a/pom.xml +++ b/pom.xml @@ -59,7 +59,7 @@ - 3.0.0-incubating + 3.1.0-SNAPSHOT 4.12 1.1.0 2.1.9 diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/TitanBlueprintsGraph.java b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/TitanBlueprintsGraph.java index ce871b7efc..63a793f867 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/TitanBlueprintsGraph.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/TitanBlueprintsGraph.java @@ -15,7 +15,7 @@ import org.apache.tinkerpop.gremlin.structure.Transaction; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.io.Io; -import org.apache.tinkerpop.gremlin.structure.util.AbstractTransaction; +import org.apache.tinkerpop.gremlin.structure.util.AbstractThreadLocalTransaction; import org.apache.tinkerpop.gremlin.structure.util.StringFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -247,7 +247,7 @@ public VertexLabel getOrCreateVertexLabel(String name) { - class GraphTransaction extends AbstractTransaction { + class GraphTransaction extends AbstractThreadLocalTransaction { public GraphTransaction() { super(TitanBlueprintsGraph.this); From a23bfa582611dba5c6e61133d9b59bd5c2b19ef8 Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Thu, 10 Sep 2015 07:51:27 -0400 Subject: [PATCH 02/32] Remove OptOuts for tests that are no longer present in TinkerPop test suite. These tests were related to consistency of behavior around deleted elements. This proved a bit hard to enforce properly on the TinkerPop side and so this requirement was removed for 3.1.x. --- .../thinkaurelius/titan/core/TitanGraph.java | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraph.java b/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraph.java index 3a7e5934e9..c8818a4fe6 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraph.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraph.java @@ -23,26 +23,6 @@ @Graph.OptIn(Graph.OptIn.SUITE_GROOVY_ENVIRONMENT_INTEGRATE) @Graph.OptIn(Graph.OptIn.SUITE_GROOVY_ENVIRONMENT_PERFORMANCE) //------------------------ -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.structure.EdgeTest$ExceptionConsistencyWhenEdgeRemovedTest", - method = "shouldThrowExceptionIfEdgeWasRemoved", - specific = "e.remove()", - reason = "Titan cannot currently throw an exception on access to removed relations due to internal use.") -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.structure.EdgeTest$ExceptionConsistencyWhenEdgeRemovedTest", - method = "shouldThrowExceptionIfEdgeWasRemoved", - specific = "property(k)", - reason = "Titan cannot currently throw an exception on access to removed relations due to internal use.") -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.structure.EdgeTest$ExceptionConsistencyWhenEdgeRemovedTest", - method = "shouldThrowExceptionIfEdgeWasRemoved", - specific = "remove()", - reason = "Titan cannot currently throw an exception on access to removed relations due to internal use.") -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.structure.VertexPropertyTest$ExceptionConsistencyWhenVertexPropertyRemovedTest", - method = "shouldThrowExceptionIfVertexPropertyWasRemoved", - specific = "property(k)", - reason = "Titan cannot currently throw an exception on access to removed relations due to internal use.") @Graph.OptOut( test = "org.apache.tinkerpop.gremlin.structure.VertexPropertyTest$VertexPropertyAddition", method = "shouldHandleSetVertexProperties", From ab0128495affb657e1d0d7f6347ee6264755de0f Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Thu, 10 Sep 2015 12:59:21 -0400 Subject: [PATCH 03/32] Deal with shading of jackson dependencies in TinkerPop 3.1.0. Currently opted to reference the shaded jackson classes rather than depend on jackson directly in titan. Seems like it is better to handle it that way as it prevents jackson version divergence. --- .../titan/core/attribute/Geoshape.java | 8 +++---- .../io/graphson/TitanGraphSONModule.java | 21 +++++++++---------- .../titan/graphdb/attribute/GeoshapeTest.java | 5 ++--- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Geoshape.java b/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Geoshape.java index 41515991d9..d86ed07de2 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Geoshape.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/core/attribute/Geoshape.java @@ -1,9 +1,5 @@ package com.thinkaurelius.titan.core.attribute; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.ser.std.StdSerializer; import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; import com.spatial4j.core.context.SpatialContext; @@ -14,6 +10,10 @@ import com.thinkaurelius.titan.diskstorage.WriteBuffer; import com.thinkaurelius.titan.graphdb.database.idhandling.VariableLong; import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.tinkerpop.shaded.jackson.core.JsonGenerator; +import org.apache.tinkerpop.shaded.jackson.core.JsonProcessingException; +import org.apache.tinkerpop.shaded.jackson.databind.SerializerProvider; +import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdSerializer; import java.io.IOException; import java.lang.reflect.Array; diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/io/graphson/TitanGraphSONModule.java b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/io/graphson/TitanGraphSONModule.java index 0b430641ce..16083f1804 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/io/graphson/TitanGraphSONModule.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/io/graphson/TitanGraphSONModule.java @@ -1,19 +1,18 @@ package com.thinkaurelius.titan.graphdb.tinkerpop.io.graphson; -import com.fasterxml.jackson.core.JsonGenerationException; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import com.fasterxml.jackson.databind.jsontype.TypeDeserializer; -import com.fasterxml.jackson.databind.jsontype.TypeSerializer; -import com.fasterxml.jackson.databind.module.SimpleModule; -import com.fasterxml.jackson.databind.ser.std.StdSerializer; import com.thinkaurelius.titan.core.attribute.Geoshape; import com.thinkaurelius.titan.graphdb.relations.RelationIdentifier; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONTokens; +import org.apache.tinkerpop.shaded.jackson.core.JsonGenerationException; +import org.apache.tinkerpop.shaded.jackson.core.JsonGenerator; +import org.apache.tinkerpop.shaded.jackson.core.JsonParser; +import org.apache.tinkerpop.shaded.jackson.core.JsonProcessingException; +import org.apache.tinkerpop.shaded.jackson.databind.DeserializationContext; +import org.apache.tinkerpop.shaded.jackson.databind.SerializerProvider; +import org.apache.tinkerpop.shaded.jackson.databind.deser.std.StdDeserializer; +import org.apache.tinkerpop.shaded.jackson.databind.jsontype.TypeSerializer; +import org.apache.tinkerpop.shaded.jackson.databind.module.SimpleModule; +import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdSerializer; import java.io.IOException; diff --git a/titan-test/src/test/java/com/thinkaurelius/titan/graphdb/attribute/GeoshapeTest.java b/titan-test/src/test/java/com/thinkaurelius/titan/graphdb/attribute/GeoshapeTest.java index e2964a3955..30328d1240 100644 --- a/titan-test/src/test/java/com/thinkaurelius/titan/graphdb/attribute/GeoshapeTest.java +++ b/titan-test/src/test/java/com/thinkaurelius/titan/graphdb/attribute/GeoshapeTest.java @@ -1,9 +1,8 @@ package com.thinkaurelius.titan.graphdb.attribute; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.module.SimpleModule; import com.thinkaurelius.titan.core.attribute.Geoshape; +import org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper; +import org.apache.tinkerpop.shaded.jackson.databind.module.SimpleModule; import org.junit.Test; import java.util.Arrays; From 9e43e49dbce36742d95c69cbab4b61341b2ebb5c Mon Sep 17 00:00:00 2001 From: Jason Plurad Date: Tue, 22 Sep 2015 23:37:05 -0700 Subject: [PATCH 04/32] add CompactionStrategy options for Cassandra storage configuration --- .../AbstractCassandraStoreManager.java | 29 +++++++++++++++++++ .../astyanax/AstyanaxStoreManager.java | 2 ++ .../CassandraEmbeddedStoreManager.java | 6 ++++ .../thrift/CassandraThriftStoreManager.java | 2 ++ 4 files changed, 39 insertions(+) diff --git a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/AbstractCassandraStoreManager.java b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/AbstractCassandraStoreManager.java index 7d3869bedb..e73ba59f87 100644 --- a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/AbstractCassandraStoreManager.java +++ b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/AbstractCassandraStoreManager.java @@ -92,6 +92,17 @@ public static Partitioner getPartitioner(String className) { "here takes precedence over one set with " + ConfigElement.getPath(REPLICATION_FACTOR), ConfigOption.Type.FIXED, String[].class); + public static final ConfigOption COMPACTION_STRATEGY = + new ConfigOption(CASSANDRA_NS, "compaction-strategy-class", + "The compaction strategy to use for Titan tables", + ConfigOption.Type.FIXED, "org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy"); + + public static final ConfigOption COMPACTION_OPTIONS = + new ConfigOption(CASSANDRA_NS, "compaction-strategy-options", + "Compaction strategy options. This list is interpreted as a " + + "map. It must have an even number of elements in [key,val,key,val,...] form.", + ConfigOption.Type.FIXED, String[].class); + // Compression public static final ConfigOption CF_COMPRESSION = new ConfigOption(CASSANDRA_NS, "compression", @@ -144,6 +155,7 @@ public static Partitioner getPartitioner(String className) { protected final String keySpaceName; protected final Map strategyOptions; + protected final Map compactionOptions; protected final boolean compressionEnabled; protected final int compressionChunkSizeKB; @@ -189,6 +201,23 @@ public AbstractCassandraStoreManager(Configuration config) { } else { this.strategyOptions = ImmutableMap.of("replication_factor", String.valueOf(config.get(REPLICATION_FACTOR))); } + + if (config.has(COMPACTION_OPTIONS)) { + String[] options = config.get(COMPACTION_OPTIONS); + + if (options.length % 2 != 0) + throw new IllegalArgumentException(COMPACTION_OPTIONS.getName() + " should have even number of elements."); + + Map converted = new HashMap(options.length / 2); + + for (int i = 0; i < options.length; i += 2) { + converted.put(options[i], options[i + 1]); + } + + this.compactionOptions = ImmutableMap.copyOf(converted); + } else { + this.compactionOptions = ImmutableMap.of(); + } } public final Partitioner getPartitioner() { diff --git a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/astyanax/AstyanaxStoreManager.java b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/astyanax/AstyanaxStoreManager.java index f73ec15b5a..b8332746e5 100644 --- a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/astyanax/AstyanaxStoreManager.java +++ b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/astyanax/AstyanaxStoreManager.java @@ -439,6 +439,8 @@ private void ensureColumnFamilyExists(String name, String comparator) throws Bac cl.makeColumnFamilyDefinition() .setName(name) .setKeyspace(keySpaceName) + .setCompactionStrategy(storageConfig.get(COMPACTION_STRATEGY)) + .setCompactionStrategyOptions(compactionOptions) .setComparatorType(comparator); ImmutableMap.Builder compressionOptions = new ImmutableMap.Builder(); diff --git a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/embedded/CassandraEmbeddedStoreManager.java b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/embedded/CassandraEmbeddedStoreManager.java index cecc92fb45..408b1d62c9 100644 --- a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/embedded/CassandraEmbeddedStoreManager.java +++ b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/embedded/CassandraEmbeddedStoreManager.java @@ -285,6 +285,12 @@ private void ensureColumnFamilyExists(String keyspaceName, String columnfamilyNa // Column Family not found; create it CFMetaData cfm = new CFMetaData(keyspaceName, columnfamilyName, ColumnFamilyType.Standard, CellNames.fromAbstractType(comparator, true)); + try { + cfm.compactionStrategyClass(CFMetaData.createCompactionStrategy(storageConfig.get(COMPACTION_STRATEGY))); + cfm.compactionStrategyOptions(compactionOptions); + } catch (ConfigurationException e) { + throw new PermanentBackendException("Failed to create column family metadata for " + keyspaceName + ":" + columnfamilyName, e); + } // Hard-coded caching settings if (columnfamilyName.startsWith(Backend.EDGESTORE_NAME)) { diff --git a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/thrift/CassandraThriftStoreManager.java b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/thrift/CassandraThriftStoreManager.java index d90486097d..9c2d0151df 100644 --- a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/thrift/CassandraThriftStoreManager.java +++ b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/thrift/CassandraThriftStoreManager.java @@ -540,6 +540,8 @@ private void createColumnFamily(Cassandra.Client client, createColumnFamily.setName(cfName); createColumnFamily.setKeyspace(ksName); createColumnFamily.setComparator_type(comparator); + createColumnFamily.setCompaction_strategy(storageConfig.get(COMPACTION_STRATEGY)); + createColumnFamily.setCompaction_strategy_options(this.compactionOptions); ImmutableMap.Builder compressionOptions = new ImmutableMap.Builder(); From 92ce7ef48cd65d538fc66f8725a7c8ce7d2ff96f Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Thu, 1 Oct 2015 15:42:34 -0400 Subject: [PATCH 05/32] Get Titan building given change to TP3 around hadoop support. TP3 at 3.1.0 no longer supports hadoop 1.x. It supports 2.x. Had to make a package name change and specifically include spark-gremlin. No guarantees that what i've done packages things properly as I had to make some exclusions given hadoop 1.x dependencies. There is likely a bigger set of chagnes to the project structure and poms that what was done here to just get this to build. --- titan-dist/pom.xml | 18 ++++++++++++++++++ titan-hadoop-parent/pom.xml | 11 +++++++++++ .../titan/hadoop/CassandraInputFormatIT.java | 2 +- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/titan-dist/pom.xml b/titan-dist/pom.xml index 2ebd548de3..660f9b0ab6 100644 --- a/titan-dist/pom.xml +++ b/titan-dist/pom.xml @@ -73,6 +73,24 @@ org.apache.tinkerpop hadoop-gremlin ${tinkerpop.version} + + + + org.apache.hadoop + hadoop-client + + + + + org.apache.tinkerpop + spark-gremlin + ${tinkerpop.version} + + + net.jpountz.lz4 + lz4 + + org.apache.tinkerpop diff --git a/titan-hadoop-parent/pom.xml b/titan-hadoop-parent/pom.xml index c05586c53e..363c4f0c50 100644 --- a/titan-hadoop-parent/pom.xml +++ b/titan-hadoop-parent/pom.xml @@ -68,6 +68,17 @@ + + org.apache.tinkerpop + spark-gremlin + ${tinkerpop.version} + + + net.jpountz.lz4 + lz4 + + + diff --git a/titan-hadoop-parent/titan-hadoop-core/src/test/java/com/thinkaurelius/titan/hadoop/CassandraInputFormatIT.java b/titan-hadoop-parent/titan-hadoop-core/src/test/java/com/thinkaurelius/titan/hadoop/CassandraInputFormatIT.java index 93972c7fb5..1ffb13ae33 100644 --- a/titan-hadoop-parent/titan-hadoop-core/src/test/java/com/thinkaurelius/titan/hadoop/CassandraInputFormatIT.java +++ b/titan-hadoop-parent/titan-hadoop-core/src/test/java/com/thinkaurelius/titan/hadoop/CassandraInputFormatIT.java @@ -7,10 +7,10 @@ import com.thinkaurelius.titan.example.GraphOfTheGodsFactory; import com.thinkaurelius.titan.graphdb.TitanGraphBaseTest; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.util.GraphFactory; -import org.apache.tinkerpop.gremlin.process.spa import org.junit.Test; import java.util.List; From b8c6f36e89f25bdcb7a6de91ce24032874582581 Mon Sep 17 00:00:00 2001 From: "Marko A. Rodriguez" Date: Mon, 12 Oct 2015 12:19:18 -0600 Subject: [PATCH 06/32] renamed VendorOptimizationStrategy to ProviderOptimizationStrategy. --- .../optimize/AdjacentVertexFilterOptimizerStrategy.java | 2 +- .../graphdb/tinkerpop/optimize/TitanGraphStepStrategy.java | 2 +- .../optimize/TitanLocalQueryOptimizerStrategy.java | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/AdjacentVertexFilterOptimizerStrategy.java b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/AdjacentVertexFilterOptimizerStrategy.java index a170ff60e8..7fcc4a8f3e 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/AdjacentVertexFilterOptimizerStrategy.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/AdjacentVertexFilterOptimizerStrategy.java @@ -22,7 +22,7 @@ /** * @author Matthias Broecheler (me@matthiasb.com) */ -public class AdjacentVertexFilterOptimizerStrategy extends AbstractTraversalStrategy implements TraversalStrategy.VendorOptimizationStrategy { +public class AdjacentVertexFilterOptimizerStrategy extends AbstractTraversalStrategy implements TraversalStrategy.ProviderOptimizationStrategy { private static final AdjacentVertexFilterOptimizerStrategy INSTANCE = new AdjacentVertexFilterOptimizerStrategy(); diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanGraphStepStrategy.java b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanGraphStepStrategy.java index 647d260cde..5e9c2b6aa4 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanGraphStepStrategy.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanGraphStepStrategy.java @@ -14,7 +14,7 @@ /** * @author Matthias Broecheler (me@matthiasb.com) */ -public class TitanGraphStepStrategy extends AbstractTraversalStrategy implements TraversalStrategy.VendorOptimizationStrategy { +public class TitanGraphStepStrategy extends AbstractTraversalStrategy implements TraversalStrategy.ProviderOptimizationStrategy { private static final TitanGraphStepStrategy INSTANCE = new TitanGraphStepStrategy(); diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanLocalQueryOptimizerStrategy.java b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanLocalQueryOptimizerStrategy.java index 61a76f48a9..293d39703c 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanLocalQueryOptimizerStrategy.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanLocalQueryOptimizerStrategy.java @@ -21,7 +21,7 @@ * @author Marko A. Rodriguez (http://markorodriguez.com) * @author Matthias Broecheler (http://matthiasb.com) */ -public class TitanLocalQueryOptimizerStrategy extends AbstractTraversalStrategy implements TraversalStrategy.VendorOptimizationStrategy { +public class TitanLocalQueryOptimizerStrategy extends AbstractTraversalStrategy implements TraversalStrategy.ProviderOptimizationStrategy { private static final TitanLocalQueryOptimizerStrategy INSTANCE = new TitanLocalQueryOptimizerStrategy(); @@ -141,11 +141,11 @@ private static void unfoldLocalTraversal(final Traversal.Admin traversal, } } - private static final Set> PRIORS = Collections.singleton(AdjacentVertexFilterOptimizerStrategy.class); + private static final Set> PRIORS = Collections.singleton(AdjacentVertexFilterOptimizerStrategy.class); @Override - public Set> applyPrior() { + public Set> applyPrior() { return PRIORS; } From c9ebb1c676110cb6ebfa50e4d412ebea9aac54a4 Mon Sep 17 00:00:00 2001 From: "Marko A. Rodriguez" Date: Thu, 29 Oct 2015 17:09:37 -0600 Subject: [PATCH 07/32] TitanGraphStep is now a able to be used mid-traversal. TinkerPop3.1 update. Added a test case to make sure that TitanGraphStep folds in HasContainers even in mid-traversal (@dalaro -- I don't know where this should have gone. So please feel free to put somewhere else if you want. I put it in the standard pattern we use in TinkerPop). Added GraphComputer.config(key,value) as a default method to TitanGraphComputer. It simply does nothing. I don't know if you guys want to have it set configurations. If you do, please do what is appropriate instead of it just doing nothing. Again, that was a TinkerPop3.1 update. --- .../thinkaurelius/titan/core/TitanGraph.java | 1 + .../titan/core/TitanGraphComputer.java | 5 +- ...AdjacentVertexFilterOptimizerStrategy.java | 23 ++++--- .../tinkerpop/optimize/TitanGraphStep.java | 20 ++++-- .../optimize/TitanGraphStepStrategy.java | 17 +++-- .../titan/graphdb/TitanGraphTest.java | 13 ++-- .../strategy/TitanStrategySuite.java | 26 ++++++++ .../traversal/strategy/TitanStrategyTest.java | 14 +++++ .../TitanGraphStepStrategyTest.java | 62 +++++++++++++++++++ 9 files changed, 150 insertions(+), 31 deletions(-) create mode 100644 titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/TitanStrategySuite.java create mode 100644 titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/TitanStrategyTest.java create mode 100644 titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/optimization/TitanGraphStepStrategyTest.java diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraph.java b/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraph.java index c8818a4fe6..30595a3f62 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraph.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraph.java @@ -22,6 +22,7 @@ @Graph.OptIn(Graph.OptIn.SUITE_GROOVY_ENVIRONMENT) @Graph.OptIn(Graph.OptIn.SUITE_GROOVY_ENVIRONMENT_INTEGRATE) @Graph.OptIn(Graph.OptIn.SUITE_GROOVY_ENVIRONMENT_PERFORMANCE) +@Graph.OptIn("com.thinkaurelius.titan.blueprints.process.traversal.strategy.TitanStrategySuite") //------------------------ @Graph.OptOut( test = "org.apache.tinkerpop.gremlin.structure.VertexPropertyTest$VertexPropertyAddition", diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraphComputer.java b/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraphComputer.java index 1903be110a..a92a4297b2 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraphComputer.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraphComputer.java @@ -38,6 +38,7 @@ public default TitanGraphComputer resultMode(ResultMode mode) { return this; } - - + public default TitanGraphComputer config(final String key, final Object value) { + return this; + } } diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/AdjacentVertexFilterOptimizerStrategy.java b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/AdjacentVertexFilterOptimizerStrategy.java index 7fcc4a8f3e..4faeb1ff3e 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/AdjacentVertexFilterOptimizerStrategy.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/AdjacentVertexFilterOptimizerStrategy.java @@ -2,14 +2,17 @@ import com.thinkaurelius.titan.core.TitanVertex; import com.thinkaurelius.titan.graphdb.types.system.ImplicitKey; -import org.apache.tinkerpop.gremlin.process.traversal.*; +import org.apache.tinkerpop.gremlin.process.traversal.Compare; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; import org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasStep; import org.apache.tinkerpop.gremlin.process.traversal.step.filter.IsStep; import org.apache.tinkerpop.gremlin.process.traversal.step.filter.TraversalFilterStep; import org.apache.tinkerpop.gremlin.process.traversal.step.map.EdgeOtherVertexStep; import org.apache.tinkerpop.gremlin.process.traversal.step.map.EdgeVertexStep; import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep; -import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GraphStep; import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IdentityStep; import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer; import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy; @@ -41,22 +44,22 @@ public void apply(final Traversal.Admin traversal) { // Check if this filter traversal matches the pattern: _.inV/outV/otherV.is(x) Traversal.Admin filterTraversal = (Traversal.Admin) originalStep.getLocalChildren().get(0); List steps = filterTraversal.getSteps(); - if (steps.size()==2 && + if (steps.size() == 2 && (steps.get(0) instanceof EdgeVertexStep || steps.get(0) instanceof EdgeOtherVertexStep) && (steps.get(1) instanceof IsStep)) { //Get the direction in which we filter on the adjacent vertex (or null if not a valid adjacency filter) Direction direction = null; if (steps.get(0) instanceof EdgeVertexStep) { - EdgeVertexStep evs = (EdgeVertexStep)steps.get(0); - if (evs.getDirection()!=Direction.BOTH) direction = evs.getDirection(); + EdgeVertexStep evs = (EdgeVertexStep) steps.get(0); + if (evs.getDirection() != Direction.BOTH) direction = evs.getDirection(); } else { assert steps.get(0) instanceof EdgeOtherVertexStep; direction = Direction.BOTH; } - P predicate = ((IsStep)steps.get(1)).getPredicate(); + P predicate = ((IsStep) steps.get(1)).getPredicate(); //Check that we have a valid direction and a valid vertex filter predicate - if (direction!=null && predicate.getBiPredicate()== Compare.eq && predicate.getValue() instanceof Vertex) { - TitanVertex vertex = TitanTraversalUtil.getTitanVertex((Vertex)predicate.getValue()); + if (direction != null && predicate.getBiPredicate() == Compare.eq && predicate.getValue() instanceof Vertex) { + TitanVertex vertex = TitanTraversalUtil.getTitanVertex((Vertex) predicate.getValue()); //Now, check that this step is preceeded by VertexStep that returns edges Step currentStep = originalStep.getPreviousStep(); @@ -66,9 +69,9 @@ public void apply(final Traversal.Admin traversal) { } else break; } if (currentStep instanceof VertexStep) { - VertexStep vstep = (VertexStep)currentStep; + VertexStep vstep = (VertexStep) currentStep; if (vstep.returnsEdge() - && (direction==Direction.BOTH || direction.equals(vstep.getDirection().opposite()))) { + && (direction == Direction.BOTH || direction.equals(vstep.getDirection().opposite()))) { //Now replace the step with a has condition TraversalHelper.replaceStep(originalStep, new HasStep(traversal, diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanGraphStep.java b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanGraphStep.java index ef782ef8f2..d16382d534 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanGraphStep.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanGraphStep.java @@ -9,8 +9,9 @@ import com.thinkaurelius.titan.graphdb.query.profile.QueryProfiler; import com.thinkaurelius.titan.graphdb.tinkerpop.profile.TP3ProfileWrapper; import org.apache.tinkerpop.gremlin.process.traversal.Order; +import org.apache.tinkerpop.gremlin.process.traversal.step.HasContainerHolder; import org.apache.tinkerpop.gremlin.process.traversal.step.Profiling; -import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GraphStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep; import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer; import org.apache.tinkerpop.gremlin.process.traversal.util.MutableMetrics; import org.apache.tinkerpop.gremlin.structure.Element; @@ -18,12 +19,13 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory; import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** * @author Matthias Broecheler (me@matthiasb.com) */ -public class TitanGraphStep extends GraphStep implements HasStepFolder, Profiling { +public class TitanGraphStep extends GraphStep implements HasStepFolder, Profiling, HasContainerHolder { private final List hasContainers = new ArrayList<>(); private int limit = BaseQuery.NO_LIMIT; @@ -31,8 +33,8 @@ public class TitanGraphStep extends GraphStep implements H private QueryProfiler queryProfiler = QueryProfiler.NO_OP; - public TitanGraphStep(final GraphStep originalStep) { - super(originalStep.getTraversal(), originalStep.getReturnClass(), originalStep.getIds()); + public TitanGraphStep(final GraphStep originalStep) { + super(originalStep.getTraversal(), originalStep.getReturnClass(), originalStep.isStartStep(), originalStep.getIds()); originalStep.getLabels().forEach(this::addLabel); this.setIteratorSupplier(() -> { TitanTransaction tx = TitanTraversalUtil.getTx(traversal); @@ -76,5 +78,15 @@ public int getLimit() { public void setMetrics(MutableMetrics metrics) { queryProfiler = new TP3ProfileWrapper(metrics); } + + @Override + public List getHasContainers() { + return this.hasContainers; + } + + @Override + public void addHasContainer(final HasContainer hasContainer) { + this.addAll(Collections.singleton(hasContainer)); + } } diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanGraphStepStrategy.java b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanGraphStepStrategy.java index 5e9c2b6aa4..33647de7f3 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanGraphStepStrategy.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/optimize/TitanGraphStepStrategy.java @@ -4,10 +4,11 @@ import org.apache.tinkerpop.gremlin.process.traversal.Step; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; -import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GraphStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep; import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy; import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper; import org.apache.tinkerpop.gremlin.structure.Element; +import org.apache.tinkerpop.gremlin.structure.Graph; import java.util.Iterator; @@ -26,13 +27,11 @@ public void apply(final Traversal.Admin traversal) { if (traversal.getEngine().isComputer()) return; - final Step startStep = traversal.getStartStep(); - if (startStep instanceof GraphStep) { - final GraphStep originalGraphStep = (GraphStep) startStep; + TraversalHelper.getStepsOfClass(GraphStep.class, traversal).forEach(originalGraphStep -> { if (originalGraphStep.getIds() == null || originalGraphStep.getIds().length == 0) { //Try to optimize for index calls - final TitanGraphStep titanGraphStep = new TitanGraphStep<>(originalGraphStep); - TraversalHelper.replaceStep(startStep, (Step) titanGraphStep, traversal); + final TitanGraphStep titanGraphStep = new TitanGraphStep<>(originalGraphStep); + TraversalHelper.replaceStep(originalGraphStep, (Step) titanGraphStep, traversal); HasStepFolder.foldInHasContainer(titanGraphStep, traversal); HasStepFolder.foldInOrder(titanGraphStep, traversal, traversal, titanGraphStep.returnsVertex()); @@ -48,11 +47,11 @@ public void apply(final Traversal.Admin traversal) { elementIds[i] = ((Element) ids[i]).id(); } originalGraphStep.setIteratorSupplier(() -> (Iterator) (originalGraphStep.returnsVertex() ? - originalGraphStep.getTraversal().getGraph().get().vertices(elementIds) : - originalGraphStep.getTraversal().getGraph().get().edges(elementIds))); + ((Graph) originalGraphStep.getTraversal().getGraph().get()).vertices(elementIds) : + ((Graph) originalGraphStep.getTraversal().getGraph().get()).edges(elementIds))); } } - } + }); } public static TitanGraphStepStrategy instance() { diff --git a/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanGraphTest.java b/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanGraphTest.java index 7bfd98c5d7..3665a4f5a7 100644 --- a/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanGraphTest.java +++ b/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanGraphTest.java @@ -104,8 +104,8 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; import org.apache.tinkerpop.gremlin.process.traversal.step.branch.LocalStep; import org.apache.tinkerpop.gremlin.process.traversal.step.filter.TraversalFilterStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep; import org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderGlobalStep; -import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GraphStep; import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.StartStep; import org.apache.tinkerpop.gremlin.process.traversal.util.Metrics; import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalMetrics; @@ -3377,7 +3377,7 @@ public void testTinkerPopOptimizationStrategies() { sv[i] = graph.addVertex("id", sid); for (int j = 0; j < numV; j++) { sv[i].addEdge("knows", vs[j], "weight", j % 5); - sv[i].property(VertexProperty.Cardinality.list, "names", "n"+j, "weight", j % 5); + sv[i].property(VertexProperty.Cardinality.list, "names", "n" + j, "weight", j % 5); } } @@ -3453,7 +3453,7 @@ public void testTinkerPopOptimizationStrategies() { gts = graph.traversal(); //Verify traversal metrics when having to read from backend [same query as above] - t = gts.V().has("id", sid).local(__.outE("knows").has("weight", P.gte(1)).has("weight",P.lt(3)).order().by("weight", decr).limit(10)).profile(); + t = gts.V().has("id", sid).local(__.outE("knows").has("weight", P.gte(1)).has("weight", P.lt(3)).order().by("weight", decr).limit(10)).profile(); assertCount(superV * 10, t); metrics = (TraversalMetrics) t.asAdmin().getSideEffects().get("~metrics").get(); // System.out.println(metrics); @@ -3497,7 +3497,7 @@ private static void verifyMetrics(Metrics metric, boolean fromCache, boolean mul assertTrue(metric.getDuration(TimeUnit.MICROSECONDS) > 0); assertTrue(metric.getCount(TraversalMetrics.ELEMENT_COUNT_ID) > 0); String hasMultiQuery = (String) metric.getAnnotation(QueryProfiler.MULTIQUERY_ANNOTATION); - assertTrue(multiQuery?hasMultiQuery.equalsIgnoreCase("true"):hasMultiQuery==null); + assertTrue(multiQuery ? hasMultiQuery.equalsIgnoreCase("true") : hasMultiQuery == null); for (Metrics submetric : metric.getNested()) { assertTrue(submetric.getDuration(TimeUnit.MICROSECONDS) > 0); switch (submetric.getName()) { @@ -3505,10 +3505,11 @@ private static void verifyMetrics(Metrics metric, boolean fromCache, boolean mul assertNull(submetric.getCount(TraversalMetrics.ELEMENT_COUNT_ID)); break; case "backend-query": - if (fromCache) assertFalse("Should not execute backend-queries when cached",true); + if (fromCache) assertFalse("Should not execute backend-queries when cached", true); assertTrue(submetric.getCount(TraversalMetrics.ELEMENT_COUNT_ID) > 0); break; - default: assertFalse("Unrecognized nested query: " + submetric.getName(),true); + default: + assertFalse("Unrecognized nested query: " + submetric.getName(), true); } } diff --git a/titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/TitanStrategySuite.java b/titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/TitanStrategySuite.java new file mode 100644 index 0000000000..5617dc606f --- /dev/null +++ b/titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/TitanStrategySuite.java @@ -0,0 +1,26 @@ +package com.thinkaurelius.titan.blueprints.process.traversal.strategy; + +import com.thinkaurelius.titan.blueprints.process.traversal.strategy.optimization.TitanGraphStepStrategyTest; +import org.apache.tinkerpop.gremlin.AbstractGremlinSuite; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine; +import org.junit.runners.model.InitializationError; +import org.junit.runners.model.RunnerBuilder; + +/** + * @author Marko A. Rodriguez (http://markorodriguez.com) + */ +public class TitanStrategySuite extends AbstractGremlinSuite { + + public TitanStrategySuite(final Class klass, final RunnerBuilder builder) throws InitializationError { + + super(klass, builder, + new Class[]{ + TitanGraphStepStrategyTest.class + }, new Class[]{ + TitanGraphStepStrategyTest.class + }, + false, + TraversalEngine.Type.STANDARD); + } + +} \ No newline at end of file diff --git a/titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/TitanStrategyTest.java b/titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/TitanStrategyTest.java new file mode 100644 index 0000000000..6ea5b0d175 --- /dev/null +++ b/titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/TitanStrategyTest.java @@ -0,0 +1,14 @@ +package com.thinkaurelius.titan.blueprints.process.traversal.strategy; + +import com.thinkaurelius.titan.blueprints.InMemoryGraphProvider; +import com.thinkaurelius.titan.core.TitanGraph; +import org.apache.tinkerpop.gremlin.GraphProviderClass; +import org.junit.runner.RunWith; + +/** + * @author Marko A. Rodriguez (http://markorodriguez.com) + */ +@RunWith(TitanStrategySuite.class) +@GraphProviderClass(provider = InMemoryGraphProvider.class, graph = TitanGraph.class) +public class TitanStrategyTest { +} diff --git a/titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/optimization/TitanGraphStepStrategyTest.java b/titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/optimization/TitanGraphStepStrategyTest.java new file mode 100644 index 0000000000..11c259e9fb --- /dev/null +++ b/titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/optimization/TitanGraphStepStrategyTest.java @@ -0,0 +1,62 @@ +package com.thinkaurelius.titan.blueprints.process.traversal.strategy.optimization; + +import com.thinkaurelius.titan.graphdb.tinkerpop.optimize.TitanGraphStep; +import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest; +import org.apache.tinkerpop.gremlin.process.IgnoreEngine; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasStep; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author Marko A. Rodriguez (http://markorodriguez.com) + */ +public class TitanGraphStepStrategyTest extends AbstractGremlinProcessTest { + + @Test + @IgnoreEngine(TraversalEngine.Type.COMPUTER) + public void shouldFoldInHasContainers() { + GraphTraversal.Admin traversal = g.V().has("name", "marko").asAdmin(); + assertEquals(2, traversal.getSteps().size()); + assertEquals(HasStep.class, traversal.getEndStep().getClass()); + traversal.applyStrategies(); + assertEquals(1, traversal.getSteps().size()); + assertEquals(TitanGraphStep.class, traversal.getStartStep().getClass()); + assertEquals(TitanGraphStep.class, traversal.getEndStep().getClass()); + assertEquals(1, ((TitanGraphStep) traversal.getStartStep()).getHasContainers().size()); + assertEquals("name", ((TitanGraphStep) traversal.getStartStep()).getHasContainers().get(0).getKey()); + assertEquals("marko", ((TitanGraphStep) traversal.getStartStep()).getHasContainers().get(0).getValue()); + //// + traversal = g.V().has("name", "marko").has("age", P.gt(20)).asAdmin(); + traversal.applyStrategies(); + assertEquals(1, traversal.getSteps().size()); + assertEquals(TitanGraphStep.class, traversal.getStartStep().getClass()); + assertEquals(2, ((TitanGraphStep) traversal.getStartStep()).getHasContainers().size()); + //// + traversal = g.V().has("name", "marko").out().has("name", "daniel").asAdmin(); + traversal.applyStrategies(); + assertEquals(3, traversal.getSteps().size()); + assertEquals(TitanGraphStep.class, traversal.getStartStep().getClass()); + assertEquals(1, ((TitanGraphStep) traversal.getStartStep()).getHasContainers().size()); + assertEquals("name", ((TitanGraphStep) traversal.getStartStep()).getHasContainers().get(0).getKey()); + assertEquals("marko", ((TitanGraphStep) traversal.getStartStep()).getHasContainers().get(0).getValue()); + assertEquals(HasStep.class, traversal.getEndStep().getClass()); + //// + traversal = g.V().has("name", "marko").out().V().has("name", "daniel").asAdmin(); + traversal.applyStrategies(); + assertEquals(3, traversal.getSteps().size()); + assertEquals(TitanGraphStep.class, traversal.getStartStep().getClass()); + assertEquals(1, ((TitanGraphStep) traversal.getStartStep()).getHasContainers().size()); + assertEquals("name", ((TitanGraphStep) traversal.getStartStep()).getHasContainers().get(0).getKey()); + assertEquals("marko", ((TitanGraphStep) traversal.getStartStep()).getHasContainers().get(0).getValue()); + assertEquals(TitanGraphStep.class, traversal.getSteps().get(2).getClass()); + assertEquals(1, ((TitanGraphStep) traversal.getSteps().get(2)).getHasContainers().size()); + assertEquals("name", ((TitanGraphStep) traversal.getSteps().get(2)).getHasContainers().get(0).getKey()); + assertEquals("daniel", ((TitanGraphStep) traversal.getSteps().get(2)).getHasContainers().get(0).getValue()); + assertEquals(TitanGraphStep.class, traversal.getEndStep().getClass()); + } + +} From 6d2f7e10d07d4bede3cd4c721968e3bbbb68c131 Mon Sep 17 00:00:00 2001 From: "Marko A. Rodriguez" Date: Mon, 2 Nov 2015 09:02:08 -0700 Subject: [PATCH 08/32] GraphComputer.configure() now has a default implementation. Thus, backwards compatible. Given that Titan just returned this, removed the method. --- .../com/thinkaurelius/titan/core/TitanGraphComputer.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraphComputer.java b/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraphComputer.java index a92a4297b2..cfab15648f 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraphComputer.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/core/TitanGraphComputer.java @@ -30,6 +30,7 @@ public Persist toPersist() { } + @Override public TitanGraphComputer workers(int threads); public default TitanGraphComputer resultMode(ResultMode mode) { @@ -37,8 +38,4 @@ public default TitanGraphComputer resultMode(ResultMode mode) { persist(mode.toPersist()); return this; } - - public default TitanGraphComputer config(final String key, final Object value) { - return this; - } } From daf76820db9e064dd91be1674f985db3dd3d0df7 Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Sat, 7 Nov 2015 13:51:15 -0500 Subject: [PATCH 09/32] Added fix to be applied to after TINKERPOP3-885 is merged. --- .../tinkerpop/TitanBlueprintsGraph.java | 4 +- .../tinkerpop/TitanBlueprintsTransaction.java | 42 ++++--------------- 2 files changed, 10 insertions(+), 36 deletions(-) diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/TitanBlueprintsGraph.java b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/TitanBlueprintsGraph.java index 63a793f867..a44d591bc0 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/TitanBlueprintsGraph.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/TitanBlueprintsGraph.java @@ -285,8 +285,8 @@ public void close() { } void close(Transaction tx) { - closeConsumer.accept(tx); - Preconditions.checkState(!tx.isOpen(),"Invalid close behavior configured: Should close transaction. [%s]",closeConsumer); + closeConsumerInternal.get().accept(tx); + Preconditions.checkState(!tx.isOpen(),"Invalid close behavior configured: Should close transaction. [%s]", closeConsumerInternal); } @Override diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/TitanBlueprintsTransaction.java b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/TitanBlueprintsTransaction.java index 4f71dfca09..761fcd561d 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/TitanBlueprintsTransaction.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/graphdb/tinkerpop/TitanBlueprintsTransaction.java @@ -11,6 +11,7 @@ import org.apache.tinkerpop.gremlin.process.computer.GraphComputer; import org.apache.tinkerpop.gremlin.structure.*; import org.apache.tinkerpop.gremlin.structure.io.Io; +import org.apache.tinkerpop.gremlin.structure.util.AbstractThreadedTransaction; import org.apache.tinkerpop.gremlin.structure.util.ElementHelper; import org.apache.tinkerpop.gremlin.structure.util.StringFactory; import org.apache.commons.configuration.Configuration; @@ -161,19 +162,19 @@ public String toString() { @Override public Transaction tx() { - return new Transaction() { + return new AbstractThreadedTransaction(getGraph()) { @Override - public void open() { + public void doOpen() { if (isClosed()) throw new IllegalStateException("Cannot re-open a closed transaction."); } @Override - public void commit() { + public void doCommit() { TitanBlueprintsTransaction.this.commit(); } @Override - public void rollback() { + public void doRollback() { TitanBlueprintsTransaction.this.rollback(); } @@ -194,38 +195,11 @@ public boolean isOpen() { } @Override - public void readWrite() { - //Does not apply to thread-independent transactions - } - - @Override - public void close() { + public void doClose() { getGraph().tinkerpopTxContainer.close(this); - } - @Override - public Transaction onReadWrite(Consumer transactionConsumer) { - throw new UnsupportedOperationException("Transaction consumer can only be configured at the graph and not the transaction level."); - } - - @Override - public Transaction onClose(Consumer transactionConsumer) { - throw new UnsupportedOperationException("Transaction consumer can only be configured at the graph and not the transaction level."); - } - - @Override - public void addTransactionListener(Consumer listener) { - throw new UnsupportedOperationException("Transaction consumer can only be configured at the graph and not the transaction level."); - } - - @Override - public void removeTransactionListener(Consumer listener) { - throw new UnsupportedOperationException("Transaction consumer can only be configured at the graph and not the transaction level."); - } - - @Override - public void clearTransactionListeners() { - // Could issue a warning here + // calling super will clear listeners + super.doClose(); } }; } From 04c578b575568fa1ab0c9a5931b3e4167587af2e Mon Sep 17 00:00:00 2001 From: Christian Bellina Date: Fri, 20 Nov 2015 17:43:42 -0500 Subject: [PATCH 10/32] Upgrade tinkerpop 3.1.0-incubating --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0a476d3246..a7806470a1 100644 --- a/pom.xml +++ b/pom.xml @@ -59,7 +59,7 @@ - 3.1.0-SNAPSHOT + 3.1.0-incubating 4.12 1.1.0 2.1.9 From d3006dc939c1b640bb263806abd3fd6bee630d12 Mon Sep 17 00:00:00 2001 From: Anderson de Andrade Date: Wed, 25 Nov 2015 16:35:21 -0500 Subject: [PATCH 11/32] Handle self-loop edges in CassandraInputFormat. --- .../formats/util/TitanVertexDeserializer.java | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/titan-hadoop-parent/titan-hadoop-core/src/main/java/com/thinkaurelius/titan/hadoop/formats/util/TitanVertexDeserializer.java b/titan-hadoop-parent/titan-hadoop-core/src/main/java/com/thinkaurelius/titan/hadoop/formats/util/TitanVertexDeserializer.java index 3d09f457ec..83db253682 100644 --- a/titan-hadoop-parent/titan-hadoop-core/src/main/java/com/thinkaurelius/titan/hadoop/formats/util/TitanVertexDeserializer.java +++ b/titan-hadoop-parent/titan-hadoop-core/src/main/java/com/thinkaurelius/titan/hadoop/formats/util/TitanVertexDeserializer.java @@ -14,6 +14,7 @@ import com.thinkaurelius.titan.hadoop.formats.util.input.TitanHadoopSetup; import org.apache.tinkerpop.gremlin.structure.T; import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerEdge; import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph; @@ -21,6 +22,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Iterator; import java.util.NoSuchElementException; public class TitanVertexDeserializer implements AutoCloseable { @@ -41,6 +43,20 @@ public TitanVertexDeserializer(final TitanHadoopSetup setup) { this.idManager = setup.getIDManager(); } + private static Boolean isLoopAdded(Vertex vertex, String label) { + Iterator adjacentVertices = vertex.vertices(Direction.BOTH, label); + + while (adjacentVertices.hasNext()) { + Vertex adjacentVertex = adjacentVertices.next(); + + if(adjacentVertex.equals(vertex)){ + return true; + } + } + + return false; + } + // Read a single row from the edgestore and create a TinkerVertex corresponding to the row // The neighboring vertices are represented by DetachedVertex instances public TinkerVertex readHadoopVertex(final StaticBuffer key, Iterable entries) { @@ -123,14 +139,18 @@ public TinkerVertex readHadoopVertex(final StaticBuffer key, Iterable ent // Decode edge TinkerEdge te; + // We don't know the label of the other vertex, but one must be provided + TinkerVertex adjacentVertex = getOrCreateVertex(relation.getOtherVertexId(), null, tg); + + // handle self-loop edges + if (tv.equals(adjacentVertex) && isLoopAdded(tv, type.name())) { + continue; + } + if (relation.direction.equals(Direction.IN)) { - // We don't know the label of the other vertex, but one must be provided - TinkerVertex outV = getOrCreateVertex(relation.getOtherVertexId(), null, tg); - te = (TinkerEdge)outV.addEdge(type.name(), tv, T.id, relation.relationId); + te = (TinkerEdge)adjacentVertex.addEdge(type.name(), tv, T.id, relation.relationId); } else if (relation.direction.equals(Direction.OUT)) { - // We don't know the label of the other vertex, but one must be provided - TinkerVertex inV = getOrCreateVertex(relation.getOtherVertexId(), null, tg); - te = (TinkerEdge)tv.addEdge(type.name(), inV, T.id, relation.relationId); + te = (TinkerEdge)tv.addEdge(type.name(), adjacentVertex, T.id, relation.relationId); } else { throw new RuntimeException("Direction.BOTH is not supported"); } From f26d0de058da52febfb2b74434b9ee3993c303b0 Mon Sep 17 00:00:00 2001 From: Anderson de Andrade Date: Thu, 26 Nov 2015 12:35:53 -0500 Subject: [PATCH 12/32] Make TitanVertexDeserializer (refCounter) a static object. --- .../hadoop/formats/util/GiraphInputFormat.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/titan-hadoop-parent/titan-hadoop-core/src/main/java/com/thinkaurelius/titan/hadoop/formats/util/GiraphInputFormat.java b/titan-hadoop-parent/titan-hadoop-core/src/main/java/com/thinkaurelius/titan/hadoop/formats/util/GiraphInputFormat.java index f3d5c7d12d..41d8d50c16 100644 --- a/titan-hadoop-parent/titan-hadoop-core/src/main/java/com/thinkaurelius/titan/hadoop/formats/util/GiraphInputFormat.java +++ b/titan-hadoop-parent/titan-hadoop-core/src/main/java/com/thinkaurelius/titan/hadoop/formats/util/GiraphInputFormat.java @@ -21,7 +21,7 @@ public abstract class GiraphInputFormat extends InputFormat implements Configurable { private final InputFormat> inputFormat; - private RefCountedCloseable refCounter; + private static RefCountedCloseable refCounter; public GiraphInputFormat(InputFormat> inputFormat) { this.inputFormat = inputFormat; @@ -38,17 +38,27 @@ public RecordReader createRecordReader(InputSplit return new GiraphRecordReader(refCounter, inputFormat.createRecordReader(split, context)); } - @Override - public void setConf(Configuration conf) { + public void setConf(Configuration conf, Boolean reset) { ((Configurable)inputFormat).setConf(conf); + + if(refCounter != null && !reset) return; + refCounter = new RefCountedCloseable<>(() -> { final String titanVersion = "current"; + String className = SETUP_PACKAGE_PREFIX + titanVersion + SETUP_CLASS_NAME; + TitanHadoopSetup ts = ConfigurationUtil.instantiate(className, new Object[]{conf}, new Class[]{Configuration.class}); + return new TitanVertexDeserializer(ts); }); } + @Override + public void setConf(Configuration conf) { + setConf(conf, false); + } + @Override public Configuration getConf() { return ((Configurable)inputFormat).getConf(); From b8d1e60abced01ceec5a981de9fa837518d5b149 Mon Sep 17 00:00:00 2001 From: David Robinson Date: Fri, 11 Dec 2015 17:07:53 -0500 Subject: [PATCH 13/32] remove rexster dir and reference in pom.xml --- pom.xml | 2 - titan-rexster/pom.xml | 36 --------- .../rexster/TitanGraphConfiguration.java | 79 ------------------- 3 files changed, 117 deletions(-) delete mode 100644 titan-rexster/pom.xml delete mode 100644 titan-rexster/src/main/java/com/thinkaurelius/titan/tinkerpop/rexster/TitanGraphConfiguration.java diff --git a/pom.xml b/pom.xml index a7806470a1..a8b5a3af67 100644 --- a/pom.xml +++ b/pom.xml @@ -123,8 +123,6 @@ titan-hbase-parent titan-es titan-lucene - - titan-all titan-dist titan-doc diff --git a/titan-rexster/pom.xml b/titan-rexster/pom.xml deleted file mode 100644 index 45a9f8c674..0000000000 --- a/titan-rexster/pom.xml +++ /dev/null @@ -1,36 +0,0 @@ - - 4.0.0 - - com.thinkaurelius.titan - titan - 0.9.0-SNAPSHOT - ../pom.xml - - titan-rexster - Titan-Rexster: Support for Titan in Rexster - http://thinkaurelius.github.com/titan/ - - - ${basedir}/.. - - - - - com.thinkaurelius.titan - titan-core - ${project.version} - - - org.apache.tinkerpop - rexster-core - ${tinkerpop.version} - - - junit - junit - ${junit.version} - test - - - - diff --git a/titan-rexster/src/main/java/com/thinkaurelius/titan/tinkerpop/rexster/TitanGraphConfiguration.java b/titan-rexster/src/main/java/com/thinkaurelius/titan/tinkerpop/rexster/TitanGraphConfiguration.java deleted file mode 100644 index 524185beb0..0000000000 --- a/titan-rexster/src/main/java/com/thinkaurelius/titan/tinkerpop/rexster/TitanGraphConfiguration.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.thinkaurelius.titan.tinkerpop.rexster; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Implements a Rexster GraphConfiguration for Titan - * - * @author Matthias Broecheler (http://www.matthiasb.com) - * @author Stephen Mallette (http://stephen.genoprime.com) - */ - -public class TitanGraphConfiguration { - - private static final Logger log = - LoggerFactory.getLogger(TitanGraphConfiguration.class); - -// @Override -// public Graph configureGraphInstance(final GraphConfigurationContext context) throws GraphConfigurationException { -// log.debug("Calling TitanFactory.open({})", context); -// final Graph g = TitanFactory.open(convertConfiguration(context)); -// log.debug("Returning new graph {}", g); -// return g; -// } -// -// public Configuration convertConfiguration(final GraphConfigurationContext context) throws GraphConfigurationException { -// Configuration properties = context.getProperties(); -// try { -// final Configuration titanConfig = new BaseConfiguration(); -// try { -// final Configuration titanConfigProperties = ((HierarchicalConfiguration) properties).configurationAt(Tokens.REXSTER_GRAPH_PROPERTIES); -// -// final Iterator titanConfigPropertiesKeys = titanConfigProperties.getKeys(); -// while (titanConfigPropertiesKeys.hasNext()) { -// String doubleDotKey = titanConfigPropertiesKeys.next(); -// -// // replace the ".." put in play by apache commons configuration. that's expected behavior -// // due to parsing key names to xml. -// String singleDotKey = doubleDotKey.replace("..", "."); -// -// // Combine multiple values into a comma-separated string -// String listVal = Joiner.on(AbstractConfiguration.getDefaultListDelimiter()).join(titanConfigProperties.getStringArray(doubleDotKey)); -// -// titanConfig.setProperty(singleDotKey, listVal); -// } -// } catch (IllegalArgumentException iae) { -// throw new GraphConfigurationException("Check graph configuration. Missing or empty configuration element: " + Tokens.REXSTER_GRAPH_PROPERTIES); -// } -// -// final Configuration rewriteConfig = new BaseConfiguration(); -// final String location = properties.getString(Tokens.REXSTER_GRAPH_LOCATION, ""); -// if (titanConfig.getString("storage.backend").equals("berkeleyje") && location.trim().length() > 0) { -// final File directory = new File(properties.getString(Tokens.REXSTER_GRAPH_LOCATION)); -// if (!directory.isDirectory()) { -// if (!directory.mkdirs()) { -// throw new IllegalArgumentException("Could not create directory: " + directory); -// } -// } -// rewriteConfig.setProperty("storage.directory", directory.toString()); -// } -// -// if (properties.containsKey(Tokens.REXSTER_GRAPH_READ_ONLY)) { -// rewriteConfig.setProperty("storage.read-only", properties.getBoolean(Tokens.REXSTER_GRAPH_READ_ONLY)); -// } -// -// final CompositeConfiguration jointConfig = new CompositeConfiguration(); -// jointConfig.addConfiguration(rewriteConfig); -// jointConfig.addConfiguration(titanConfig); -// return jointConfig; -// } catch (Exception e) { -// //Unroll exceptions so that Rexster prints root cause -// Throwable cause = e; -// while (cause.getCause() != null) { -// cause = cause.getCause(); -// } -// throw new GraphConfigurationException(cause); -// } -// } -} From 3e93f5c5014ec9aa40422c6c4d2ae8ae90c758a4 Mon Sep 17 00:00:00 2001 From: David Robinson Date: Fri, 11 Dec 2015 17:21:58 -0500 Subject: [PATCH 14/32] version update slf4j --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a7806470a1..ce4eec0b80 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,7 @@ 2.1.2 3.0.1 2.7.10 - 1.7.5 + 1.7.12 4.4.1 1.2.1 2.7.1 From 63173d9c1ab539e172df09050a000f948b3e32fd Mon Sep 17 00:00:00 2001 From: David Robinson Date: Tue, 15 Dec 2015 07:58:06 -0500 Subject: [PATCH 15/32] fix nullpointer in TitanGraphStepStrategyTest testcase --- .../optimization/TitanGraphStepStrategyTest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/optimization/TitanGraphStepStrategyTest.java b/titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/optimization/TitanGraphStepStrategyTest.java index 11c259e9fb..6df7f75b93 100644 --- a/titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/optimization/TitanGraphStepStrategyTest.java +++ b/titan-test/src/test/java/com/thinkaurelius/titan/blueprints/process/traversal/strategy/optimization/TitanGraphStepStrategyTest.java @@ -1,19 +1,28 @@ package com.thinkaurelius.titan.blueprints.process.traversal.strategy.optimization; +import com.thinkaurelius.titan.blueprints.InMemoryGraphProvider; +import com.thinkaurelius.titan.core.TitanGraph; import com.thinkaurelius.titan.graphdb.tinkerpop.optimize.TitanGraphStep; + import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest; import org.apache.tinkerpop.gremlin.process.IgnoreEngine; import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.GraphProviderClass; +import org.apache.tinkerpop.gremlin.process.ProcessStandardSuite; import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasStep; + import org.junit.Test; +import org.junit.runner.RunWith; import static org.junit.Assert.assertEquals; /** * @author Marko A. Rodriguez (http://markorodriguez.com) */ +@RunWith(ProcessStandardSuite.class) +@GraphProviderClass(provider = InMemoryGraphProvider.class, graph = TitanGraph.class) public class TitanGraphStepStrategyTest extends AbstractGremlinProcessTest { @Test From 1eacc0a77691dc3e78d054eca1bdbc9159873f8e Mon Sep 17 00:00:00 2001 From: Dan LaRocque Date: Tue, 15 Dec 2015 11:33:43 -0500 Subject: [PATCH 16/32] Fixing TitanIndexTest Assertions after calling clopen() in BerkeleyElasticsearchTest have been failing since the upgrade from TP 3.0 to 3.1. I can't tell precisely when it broke, since many of the recent predecessor commits that bisect would touch have compile errors. However, squinting at the failure stack traces and playing with tweaks eventually pointed to a change in the default on-close behavior of transactions between TP 3.0 and 3.1. TitanIndexTest relies on commit. However, this was changed to rollback in 3.1: http://tinkerpop.apache.org/docs/3.1.0-incubating/upgrade.html#_transaction_close_default_behavior I overrode TitanIndexTest's clopen() to commit the graph's transaction before invoking super.clopen(). --- .../java/com/thinkaurelius/titan/BerkeleyStorageSetup.java | 2 +- .../src/main/java/com/thinkaurelius/titan/StorageSetup.java | 4 ---- .../com/thinkaurelius/titan/graphdb/TitanIndexTest.java | 6 ++++++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/titan-berkeleyje/src/test/java/com/thinkaurelius/titan/BerkeleyStorageSetup.java b/titan-berkeleyje/src/test/java/com/thinkaurelius/titan/BerkeleyStorageSetup.java index 486f5da073..94c7c5829e 100644 --- a/titan-berkeleyje/src/test/java/com/thinkaurelius/titan/BerkeleyStorageSetup.java +++ b/titan-berkeleyje/src/test/java/com/thinkaurelius/titan/BerkeleyStorageSetup.java @@ -14,7 +14,7 @@ public static ModifiableConfiguration getBerkeleyJEConfiguration(String dir) { } public static ModifiableConfiguration getBerkeleyJEConfiguration() { - return getBerkeleyJEConfiguration(getHomeDir()); + return getBerkeleyJEConfiguration(getHomeDir("berkeleyje")); } public static WriteConfiguration getBerkeleyJEGraphConfiguration() { diff --git a/titan-test/src/main/java/com/thinkaurelius/titan/StorageSetup.java b/titan-test/src/main/java/com/thinkaurelius/titan/StorageSetup.java index 0e6dfbf5ab..c33cc61a39 100644 --- a/titan-test/src/main/java/com/thinkaurelius/titan/StorageSetup.java +++ b/titan-test/src/main/java/com/thinkaurelius/titan/StorageSetup.java @@ -30,10 +30,6 @@ public static final String getHomeDir(String subdir) { return homedir; } - public static final String getHomeDir() { - return getHomeDir(null); - } - public static final File getHomeDirFile() { return getHomeDirFile(null); } diff --git a/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanIndexTest.java b/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanIndexTest.java index 6ed83a6e18..b27d9bb359 100644 --- a/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanIndexTest.java +++ b/titan-test/src/main/java/com/thinkaurelius/titan/graphdb/TitanIndexTest.java @@ -125,6 +125,12 @@ public void open(WriteConfiguration config) { indexFeatures = graph.getBackend().getIndexFeatures().get(INDEX); } + @Override + public void clopen(Object... settings) { + graph.tx().commit(); + super.clopen(settings); + } + @Rule public TestName methodName = new TestName(); From 931cc0dadeebd24907256d4d85dd868122c24c99 Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Wed, 16 Dec 2015 12:57:18 -0500 Subject: [PATCH 17/32] Clean up some mentions of Blueprints in titanbasics #1165 --- docs/titanbasics.txt | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/docs/titanbasics.txt b/docs/titanbasics.txt index a77d98bb9d..f2c43f9a0a 100644 --- a/docs/titanbasics.txt +++ b/docs/titanbasics.txt @@ -306,11 +306,6 @@ Defining Vertex Labels Like edges, vertices have labels. Unlike edge labels, vertex labels are optional. Vertex labels are useful to distinguish different types of vertices, e.g. _user_ vertices and _product_ vertices. -For compatibility with Blueprints, Titan provides differently-named methods for adding labeled and unlabeled vertices: - -* `addVertexWithLabel` -* `addVertex` - Although labels are optional at the conceptual and data model level, Titan assigns all vertices a label as an internal implementation detail. Vertices created by the `addVertex` methods use Titan's default label. To create a label, call `makeVertexLabel(String).make()` on an open graph or management transaction and provide the name of the vertex label as the argument. Vertex label names must be unique in the graph. @@ -851,7 +846,7 @@ This section describes Titan's transactional semantics and API. Transaction Handling ~~~~~~~~~~~~~~~~~~~~ -Every graph operation in Titan occurs within the context of a transaction. According to the Blueprints' specification, each thread opens its own transaction against the graph database with the first operation (i.e. retrieval or mutation) on the graph:: +Every graph operation in Titan occurs within the context of a transaction. According to the TinkerPop's transactional specification, each thread opens its own transaction against the graph database with the first operation (i.e. retrieval or mutation) on the graph: [source, gremlin] ---- @@ -866,7 +861,7 @@ In this example, a local Titan graph database is opened. Adding the vertex "juno Transactional Scope ~~~~~~~~~~~~~~~~~~~ -All graph elements (vertices, edges, and types) are associated with the transactional scope in which they were retrieved or created. Under Blueprint's default transactional semantics, transactions are automatically created with the first operation on the graph and closed explicitly using `commit()` or `rollback()`. Once the transaction is closed, all graph elements associated with that transaction become stale and unavailable. However, Titan will automatically transition vertices and types into the new transactional scope as shown in this example:: +All graph elements (vertices, edges, and types) are associated with the transactional scope in which they were retrieved or created. Under TinkerPop's default transactional semantics, transactions are automatically created with the first operation on the graph and closed explicitly using `commit()` or `rollback()`. Once the transaction is closed, all graph elements associated with that transaction become stale and unavailable. However, Titan will automatically transition vertices and types into the new transactional scope as shown in this example: [source, gremlin] graph = TitanFactory.open("berkeleyje:/tmp/titan") @@ -874,7 +869,7 @@ juno = graph.addVertex() //Automatically opens a new transaction graph.tx().commit() //Ends transaction juno.property("name", "juno") //Vertex is automatically transitioned -Edges, on the other hand, are not automatically transitioned and cannot be accessed outside their original transaction. They must be explicitly transitioned. +Edges, on the other hand, are not automatically transitioned and cannot be accessed outside their original transaction. They must be explicitly transitioned: [source, gremlin] e = juno.addEdge("knows", graph.addVertex()) @@ -885,7 +880,7 @@ e.property("time", 99) Transaction Failures ~~~~~~~~~~~~~~~~~~~~ -When committing a transaction, Titan will attempt to persist all changes to the storage backend. This might not always be successful due to IO exceptions, network errors, machine crashes or resource unavailability. Hence, transactions can fail. In fact, transactions *will eventually fail* in sufficiently large systems. Therefore, we highly recommend that your code expects and accommodates such failures. +When committing a transaction, Titan will attempt to persist all changes to the storage backend. This might not always be successful due to IO exceptions, network errors, machine crashes or resource unavailability. Hence, transactions can fail. In fact, transactions *will eventually fail* in sufficiently large systems. Therefore, we highly recommend that your code expects and accommodates such failures: [source, gremlin] try { @@ -903,7 +898,7 @@ The example above demonstrates a simplified user signup implementation where `na If the transaction fails, a `TitanException` is thrown. There are a variety of reasons why a transaction may fail. Titan differentiates between _potentially temporary_ and _permanent_ failures. -Potentially temporary failures are those related to resource unavailability and IO hickups (e.g. network timeouts). Titan automatically tries to recover from temporary failures by retrying to persist the transactional state after some delay. The number of retry attempts and the retry delay are configurable (see <>). +Potentially temporary failures are those related to resource unavailability and IO hiccups (e.g. network timeouts). Titan automatically tries to recover from temporary failures by retrying to persist the transactional state after some delay. The number of retry attempts and the retry delay are configurable (see <>). Permanent failures can be caused by complete connection loss, hardware failure or lock contention. To understand the cause of lock contention, consider the signup example above and suppose a user tries to signup with username "juno". That username may still be available at the beginning of the transaction but by the time the transaction is committed, another user might have concurrently registered with "juno" as well and that transaction holds the lock on the username therefore causing the other transaction to fail. Depending on the transaction semantics one can recover from a lock contention failure by re-running the entire transaction. @@ -916,30 +911,30 @@ Permanent exceptions that can fail a transaction include: Multi-Threaded Transactions ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Titan supports multi-threaded transactions through Blueprint's http://tinkerpop.incubator.apache.org/docs/{tinkerpop_version}/#transactions[ThreadedTransactionalGraph] interface. Hence, to speed up transaction processing and utilize multi-core architectures multiple threads can run concurrently in a single transaction. +Titan supports multi-threaded transactions through TinkerPop's http://tinkerpop.incubator.apache.org/docs/{tinkerpop_version}/#_threaded_transactions[threaded transactions]. Hence, to speed up transaction processing and utilize multi-core architectures multiple threads can run concurrently in a single transaction. -With Blueprints' default transaction handling, each thread automatically opens its own transaction against the graph database. To open a thread-independent transaction, use the `newTransaction()` method. +With TinkerPop's default transaction handling, each thread automatically opens its own transaction against the graph database. To open a thread-independent transaction, use the `newThreadedTx()` method. [source, gremlin] -tx = graph.newTransaction(); +threadedGraph = graph.tx().newThreadedTx(); threads = new Thread[10]; for (int i=0; i> only. -Transactions are automatically started under the Blueprints semantics but *not* automatically terminated. Transactions have to be terminated manually with `g.commit()` if successful or `g.rollback()` if not. Manual termination of transactions is necessary because only the user knows the transactional boundary. +Transactions are automatically started under the TinkerPop semantics but *not* automatically terminated. Transactions have to be terminated manually with `g.commit()` if successful or `g.rollback()` if not. Manual termination of transactions is necessary because only the user knows the transactional boundary. A transaction will attempt to maintain its state from the beginning of the transaction. This might lead to unexpected behavior in multi-threaded applications as illustrated in the following artificial example:: [source, gremlin] From 80d5993ad1a06b57b072a1daec1236e42b8f1ec2 Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Wed, 16 Dec 2015 13:53:46 -0500 Subject: [PATCH 18/32] Removed as many rexster references as made sense. Didn't do too much replacement with "Gremlin Server" stuff in the debian packaging stuff as it is generally out of date and expected to see removal as a whole. --- docs/hbase.txt | 4 +- docs/titanbasics.txt | 4 +- .../titan/util/system/CheckSocket.java | 4 +- titan-dist/src/assembly/static/bin/titan.sh | 9 -- .../pkgcommon/payload/etc/titan/rexster.xml | 118 ------------------ .../src/pkg/static/debian/titan.install | 2 - .../pkgcommon/bin/quick-setup-for-debian.sh | 2 +- .../pkgcommon/bin/quick-setup-for-redhat.sh | 2 +- .../payload/usr/bin/rexster-console.sh | 22 ---- .../static/pkgcommon/payload/usr/sbin/titan | 8 +- .../payload/usr/share/titan/titan.in.sh | 1 - .../src/pkg/static/redhat/titan.spec.base | 3 - .../titan/pkgtest/TitanScriptIT.java | 2 +- .../titan-hadoop-core/README.textile | 1 - 14 files changed, 13 insertions(+), 169 deletions(-) delete mode 100644 titan-dist/src/pkg/resources/pkgcommon/payload/etc/titan/rexster.xml delete mode 100644 titan-dist/src/pkg/static/pkgcommon/payload/usr/bin/rexster-console.sh diff --git a/docs/hbase.txt b/docs/hbase.txt index ba7518d38e..e30ee43803 100644 --- a/docs/hbase.txt +++ b/docs/hbase.txt @@ -61,8 +61,8 @@ image:titan-modes-rexster.png[] Finally, Gremlin server can be wrapped around each Titan instance defined in the previous subsection. In this way, the end-user application need not be a Java-based application as it can communicate with Gremlin server as a client. This type of deployment is great for polyglot architectures where various components written in different languages need to reference and compute on the graph. ---- -http://rexster.titan.machine1/mygraph/vertices/1 -http://rexster.titan.machine2/mygraph/tp/gremlin?script=g.v(1).out('follows').out('created') +http://gremlin-server.titan.machine1/mygraph/vertices/1 +http://gremlin-server.titan.machine2/mygraph/tp/gremlin?script=g.v(1).out('follows').out('created') ---- In this case, each Gremlin server server would be configured to connect to the HBase cluster. The following shows the graph specific fragment of the Gremlin server configuration. Refer to <> for a complete example and more information on how to configure the server. diff --git a/docs/titanbasics.txt b/docs/titanbasics.txt index f2c43f9a0a..cb042e1235 100644 --- a/docs/titanbasics.txt +++ b/docs/titanbasics.txt @@ -1077,7 +1077,7 @@ The configuration option `cache.db-cache-size` controls how much heap space Tita The cache size can be configured as a percentage (expressed as a decimal between 0 and 1) of the total heap space available to the JVM running Titan or as an absolute number of bytes. -Note, that the cache size refers to the amount of heap space that is exclusively occupied by the cache. Titan's other data structures and each open transaction will occupy additional heap space. If additional software layers are running in the same JVM, those may occupy a significant amount of heap space as well (e.g. Rexster, embedded Cassandra, etc). Be conservative in your heap memory estimation. Configuring a cache that is too large can lead to out-of-memory exceptions and excessive GC. +Note, that the cache size refers to the amount of heap space that is exclusively occupied by the cache. Titan's other data structures and each open transaction will occupy additional heap space. If additional software layers are running in the same JVM, those may occupy a significant amount of heap space as well (e.g. Gremlin Server, embedded Cassandra, etc). Be conservative in your heap memory estimation. Configuring a cache that is too large can lead to out-of-memory exceptions and excessive GC. Clean Up Wait Time ^^^^^^^^^^^^^^^^^^ @@ -1262,7 +1262,7 @@ When launching Titan with embedded Cassandra, the following warnings may be disp `958 [MutationStage:25] WARN org.apache.cassandra.db.Memtable - MemoryMeter uninitialized (jamm not specified as java agent); assuming liveRatio of 10.0. Usually this means cassandra-env.sh disabled jamm because you are using a buggy JRE; upgrade to the Sun JRE instead` -Cassandra uses a Java agent called `MemoryMeter` which allows it to measure the actual memory use of an object, including JVM overhead. To use https://github.com/jbellis/jamm[JAMM] (Java Agent for Memory Measurements), the path to the JAMM jar must be specific in the Java javaagent parameter when launching the JVM (e.g. `-javaagent:path/to/jamm.jar`) through either titan.sh, gremlin.sh, or Rexster: +Cassandra uses a Java agent called `MemoryMeter` which allows it to measure the actual memory use of an object, including JVM overhead. To use https://github.com/jbellis/jamm[JAMM] (Java Agent for Memory Measurements), the path to the JAMM jar must be specific in the Java javaagent parameter when launching the JVM (e.g. `-javaagent:path/to/jamm.jar`) through either titan.sh, gremlin.sh, or Gremlin Server: [source, bash] export TITAN_JAVA_OPTS=-javaagent:$TITAN_HOME/lib/jamm-$MAVEN{jamm.version}.jar diff --git a/titan-core/src/main/java/com/thinkaurelius/titan/util/system/CheckSocket.java b/titan-core/src/main/java/com/thinkaurelius/titan/util/system/CheckSocket.java index acee6bb401..3b9da3bf08 100644 --- a/titan-core/src/main/java/com/thinkaurelius/titan/util/system/CheckSocket.java +++ b/titan-core/src/main/java/com/thinkaurelius/titan/util/system/CheckSocket.java @@ -5,9 +5,9 @@ /* * This doesn't really belong here. It's only used in the zipfile - * distribution to check whether Rexster or ES are listening on + * distribution to check whether Gremlin Server or ES are listening on * their respective TCP ports. But it's so tiny that I don't want - * to reorganize the repo to accomodate it (yet). + * to reorganize the repo to accommodate it (yet). * * Many widely available *NIX programs do this task better (e.g. * netcat, telnet, nmap, socat, ... we could even use netstat since diff --git a/titan-dist/src/assembly/static/bin/titan.sh b/titan-dist/src/assembly/static/bin/titan.sh index 7fa445a71f..269dcc840d 100755 --- a/titan-dist/src/assembly/static/bin/titan.sh +++ b/titan-dist/src/assembly/static/bin/titan.sh @@ -227,15 +227,6 @@ usage() { echo " clean: permanently delete all graph data (run when stopped)" >&2 echo "Options:" >&2 echo " -v enable logging to console in addition to logfiles" >&2 -# echo " -c str configure gremlin-server with conf/rexster-.xml" >&3 -# echo " recognized arguments to -c:" >&2 -# shopt -s nullglob -# for f in "$BIN"/../conf/rexster-*.xml; do -# f="`basename $f`" -# f="${f#rexster-}" -# f="${f%.xml}" -# echo " $f" >&2 -# done } find_verb() { diff --git a/titan-dist/src/pkg/resources/pkgcommon/payload/etc/titan/rexster.xml b/titan-dist/src/pkg/resources/pkgcommon/payload/etc/titan/rexster.xml deleted file mode 100644 index 76a2be3af7..0000000000 --- a/titan-dist/src/pkg/resources/pkgcommon/payload/etc/titan/rexster.xml +++ /dev/null @@ -1,118 +0,0 @@ - - - - 8182 - 0.0.0.0 - http://localhost - public - UTF-8 - false - true - 2097152 - 8192 - 30000 - - - 8 - 8 - - - 4 - 4 - - - leader-follower - - - 8184 - 0.0.0.0 - 1790000 - 3000000 - 180000 - 3000000 - 65536 - false - - - 8 - 8 - - - 4 - 4 - - - leader-follower - - 8183 - 127.0.0.1 - 10000 - - - gremlin-groovy - 500 - /etc/titan/rexster-init.groovy - ${imports} - ${staticimports} - - - - - none - - - - rexster - rexster - - - - - - - - jmx - - - http - - - console - - SECONDS - SECONDS - 10 - MINUTES - http.rest.* - http.rest.*.delete - - - - - - emptygraph - tinkergraph - true - - - tp:gremlin - - - - - graph - com.thinkaurelius.titan.tinkerpop.rexster.TitanGraphConfiguration - - cassandra - 127.0.0.1 - elasticsearch - 127.0.0.1 - true - true - 50 - 300000 - 0.25 - - - - diff --git a/titan-dist/src/pkg/static/debian/titan.install b/titan-dist/src/pkg/static/debian/titan.install index c60d0ee69e..199c8cbe13 100644 --- a/titan-dist/src/pkg/static/debian/titan.install +++ b/titan-dist/src/pkg/static/debian/titan.install @@ -1,9 +1,7 @@ /usr/bin/gremlin.sh -/usr/bin/rexster-console.sh /usr/sbin/titan /usr/share/titan/titan.in.sh /etc/titan/titan-env.sh /etc/titan/config.properties /etc/titan/log4j.properties -/etc/titan/rexster.xml ../payload/cassandra.yaml /etc/titan/ diff --git a/titan-dist/src/pkg/static/pkgcommon/bin/quick-setup-for-debian.sh b/titan-dist/src/pkg/static/pkgcommon/bin/quick-setup-for-debian.sh index 2d1e1d3a46..99532ae715 100755 --- a/titan-dist/src/pkg/static/pkgcommon/bin/quick-setup-for-debian.sh +++ b/titan-dist/src/pkg/static/pkgcommon/bin/quick-setup-for-debian.sh @@ -23,7 +23,7 @@ echo "Installing Cassandra, ES, and Titan with apt-get" apt-get update apt-get install cassandra=2.0.7 elasticsearch=1.0.3 titan -# Reduce Cassandra and Rexster/Titan heapsizes +# Reduce Cassandra and Gremlin Server/Titan heapsizes echo 'export MAX_HEAP_SIZE=512M export HEAP_NEWSIZE=128M' > /etc/default/cassandra diff --git a/titan-dist/src/pkg/static/pkgcommon/bin/quick-setup-for-redhat.sh b/titan-dist/src/pkg/static/pkgcommon/bin/quick-setup-for-redhat.sh index 9ef8deca61..54d05bc6c1 100755 --- a/titan-dist/src/pkg/static/pkgcommon/bin/quick-setup-for-redhat.sh +++ b/titan-dist/src/pkg/static/pkgcommon/bin/quick-setup-for-redhat.sh @@ -41,7 +41,7 @@ echo "Installing Cassandra, ES, and Titan with yum" yum update yum install cassandra20-2.0.7 elasticsearch-1.0.3 titan -# Reduce Cassandra and Rexster/Titan heapsizes +# Reduce Cassandra and Gremlin Server/Titan heapsizes echo 'export MAX_HEAP_SIZE=512M export HEAP_NEWSIZE=128M' > /etc/default/cassandra diff --git a/titan-dist/src/pkg/static/pkgcommon/payload/usr/bin/rexster-console.sh b/titan-dist/src/pkg/static/pkgcommon/payload/usr/bin/rexster-console.sh deleted file mode 100644 index 9b53c23491..0000000000 --- a/titan-dist/src/pkg/static/pkgcommon/payload/usr/bin/rexster-console.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -CP=$( echo /usr/share/titan/lib/*.jar . | sed 's/ /:/g') -#echo $CP - -# Find Java -if [ "$JAVA_HOME" = "" ] ; then - JAVA="java -server" -else - JAVA="$JAVA_HOME/bin/java -server" -fi - -# Set Java options -if [ "$JAVA_OPTIONS" = "" ] ; then - JAVA_OPTIONS="-Xms32m -Xmx512m" -fi - -# Launch the application -$JAVA $JAVA_OPTIONS -cp $CP org.apache.tinkerpop.rexster.console.RexsterConsole $@ - -# Return the program's exit code -exit $? diff --git a/titan-dist/src/pkg/static/pkgcommon/payload/usr/sbin/titan b/titan-dist/src/pkg/static/pkgcommon/payload/usr/sbin/titan index 218420c5ee..dd5bca7959 100755 --- a/titan-dist/src/pkg/static/pkgcommon/payload/usr/sbin/titan +++ b/titan-dist/src/pkg/static/pkgcommon/payload/usr/sbin/titan @@ -109,7 +109,7 @@ launch_service() #parms="-Dlog4j.configuration=file://"$TITAN_CONF"/log4j.properties -Dlog4j.defaultInitOverride=true" parms='' - # Rexster is hardcoded to use either log4j.properties in the current + # Gremlin Server is hardcoded to use either log4j.properties in the current # working directory or to use the one shipped in its own jar. # It always overrides the default log4j configuration mechanism (!) # Switch to the directory containing our log4j.properties as a workaround @@ -117,10 +117,10 @@ launch_service() if [ "x$foreground" != "x" ]; then export CLASSPATH - exec "$JAVA" $JVM_OPTS $parms $props "$class" -s -c "$REXSTER_CFG" + exec "$JAVA" $JVM_OPTS $parms $props "$class" else export CLASSPATH - exec "$JAVA" $JVM_OPTS $parms $props "$class" -s -c "$REXSTER_CFG" <&- & + exec "$JAVA" $JVM_OPTS $parms $props "$class" <&- & [ ! -z "$pidpath" ] && printf "%d" $! > "$pidpath" true fi @@ -132,7 +132,7 @@ launch_service() args=`getopt fhp:bD: "$@"` eval set -- "$args" -classname="org.apache.tinkerpop.rexster.Application" +classname="org.apache.tinkerpop.gremlin.server.GremlinServer" while true; do case "$1" in -p) diff --git a/titan-dist/src/pkg/static/pkgcommon/payload/usr/share/titan/titan.in.sh b/titan-dist/src/pkg/static/pkgcommon/payload/usr/share/titan/titan.in.sh index 097a11374c..de2ae1c674 100755 --- a/titan-dist/src/pkg/static/pkgcommon/payload/usr/share/titan/titan.in.sh +++ b/titan-dist/src/pkg/static/pkgcommon/payload/usr/share/titan/titan.in.sh @@ -22,7 +22,6 @@ TITAN_HOME="/usr/share/titan" TITAN_CONF="/etc/titan" TITAN_CFG="$TITAN_CONF/config.properties" -REXSTER_CFG="$TITAN_CONF/rexster.xml" # JAVA_HOME can optionally be set here #JAVA_HOME=/usr/local/jdk6 diff --git a/titan-dist/src/pkg/static/redhat/titan.spec.base b/titan-dist/src/pkg/static/redhat/titan.spec.base index 6d9d6462ae..c552a83966 100644 --- a/titan-dist/src/pkg/static/redhat/titan.spec.base +++ b/titan-dist/src/pkg/static/redhat/titan.spec.base @@ -114,8 +114,6 @@ rm -rf $RPM_BUILD_ROOT %attr(0644, root, root) %config /etc/titan/cassandra.yaml %attr(0644, root, root) %config /etc/titan/config.properties %attr(0644, root, root) %config /etc/titan/log4j.properties -%attr(0644, root, root) %config /etc/titan/rexster.xml -%attr(0644, root, root) %config /etc/titan/rexster-init.groovy %attr(0644, root, root) %config /etc/titan/titan-env.sh %attr(0644, root, root) %config /etc/sysconfig/titan %attr(0755, root, root) %config /etc/rc.d/init.d/titan @@ -123,7 +121,6 @@ rm -rf $RPM_BUILD_ROOT %attr(0755, titan, titan) %dir /var/lib/titan/ %attr(0755, titan, titan) %dir /var/run/titan/ %attr(0755, root, root) /usr/bin/gremlin.sh -%attr(0755, root, root) /usr/bin/rexster-console.sh %attr(0755, root, root) /usr/sbin/titan %attr(0444, root, root) /usr/share/titan/lib/*.jar %attr(0644, root, root) /usr/share/titan/titan.in.sh diff --git a/titan-dist/src/test/java/com/thinkaurelius/titan/pkgtest/TitanScriptIT.java b/titan-dist/src/test/java/com/thinkaurelius/titan/pkgtest/TitanScriptIT.java index 8ba4d57a4d..7839cc00c3 100644 --- a/titan-dist/src/test/java/com/thinkaurelius/titan/pkgtest/TitanScriptIT.java +++ b/titan-dist/src/test/java/com/thinkaurelius/titan/pkgtest/TitanScriptIT.java @@ -3,7 +3,7 @@ import org.junit.Test; /** - * Test the titan.sh script that starts and stops Cassandra, ES, and Rexster. + * Test the titan.sh script that starts and stops Cassandra, ES, and Gremlin Server. */ public class TitanScriptIT extends AbstractTitanAssemblyIT { diff --git a/titan-hadoop-parent/titan-hadoop-core/README.textile b/titan-hadoop-parent/titan-hadoop-core/README.textile index deaab8de3b..05d96a5780 100644 --- a/titan-hadoop-parent/titan-hadoop-core/README.textile +++ b/titan-hadoop-parent/titan-hadoop-core/README.textile @@ -8,7 +8,6 @@ h2. Features ** "Titan":http://thinkaurelius.github.com/titan/ distributed graph database *** "Apache Cassandra":http://cassandra.apache.org/ *** "Apache HBase":http://hbase.apache.org/ - ** "Rexster":http://rexster.tinkerpop.com fronted graph databases ** "GraphSON":https://github.com/tinkerpop/blueprints/wiki/GraphSON-Reader-and-Writer-Library text format stored in HDFS ** EdgeList multi-relational text format stored in HDFS *** "RDF":http://www.w3.org/RDF/ text formats stored in HDFS From 63d821bd9494dfd3d8d9d81be5eebe97e0b4ce23 Mon Sep 17 00:00:00 2001 From: Dan LaRocque Date: Thu, 17 Dec 2015 03:02:56 -0500 Subject: [PATCH 19/32] Switch back to Hadoop 2 --- titan-all/pom.xml | 6 +++--- titan-dist/pom.xml | 6 +++--- titan-dist/titan-dist-hadoop-1/pom.xml | 7 ++++--- titan-dist/titan-dist-hadoop-2/pom.xml | 7 +++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/titan-all/pom.xml b/titan-all/pom.xml index 962e924e1e..68513e86a3 100644 --- a/titan-all/pom.xml +++ b/titan-all/pom.xml @@ -46,7 +46,7 @@ org.apache.hbase hbase-client - ${hbase098.core.version}-hadoop1 + ${hbase098.core.version}-hadoop2 ${project.groupId} @@ -55,8 +55,8 @@ org.apache.hadoop - hadoop-core - ${hadoop1.version} + hadoop-common + ${hadoop2.version} com.thinkaurelius.titan diff --git a/titan-dist/pom.xml b/titan-dist/pom.xml index 660f9b0ab6..32cdbae0f1 100644 --- a/titan-dist/pom.xml +++ b/titan-dist/pom.xml @@ -41,9 +41,9 @@ - titan-dist-hadoop-1 - - + + + titan-dist-hadoop-2 diff --git a/titan-dist/titan-dist-hadoop-1/pom.xml b/titan-dist/titan-dist-hadoop-1/pom.xml index b91030b798..4b049f6ce2 100644 --- a/titan-dist/titan-dist-hadoop-1/pom.xml +++ b/titan-dist/titan-dist-hadoop-1/pom.xml @@ -177,12 +177,13 @@ - dev-install-hadoop1-by-default + dev-install-hadoop1 - + - !dev.hadoop + dev.hadoop + 2 diff --git a/titan-dist/titan-dist-hadoop-2/pom.xml b/titan-dist/titan-dist-hadoop-2/pom.xml index fa5fdaa041..3b366d0f0f 100644 --- a/titan-dist/titan-dist-hadoop-2/pom.xml +++ b/titan-dist/titan-dist-hadoop-2/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-dist - 0.9.0-SNAPSHOT + 1.0.1-SNAPSHOT ../pom.xml pom @@ -180,11 +180,10 @@ dev-install-hadoop2 - + - dev.hadoop - 2 + !dev.hadoop From 948a2795ad6ca2bae4fd48895706173099b2acd7 Mon Sep 17 00:00:00 2001 From: Dan LaRocque Date: Thu, 17 Dec 2015 03:05:56 -0500 Subject: [PATCH 20/32] Set version to 1.1.0-SNAPSHOT --- pom.xml | 2 +- titan-all/pom.xml | 2 +- titan-berkeleyje/pom.xml | 2 +- titan-cassandra/pom.xml | 2 +- titan-core/pom.xml | 2 +- titan-dist/pom.xml | 2 +- titan-dist/titan-dist-hadoop-2/pom.xml | 2 +- titan-doc/pom.xml | 2 +- titan-es/pom.xml | 2 +- titan-hadoop-parent/pom.xml | 2 +- titan-hadoop-parent/titan-hadoop-1/pom.xml | 2 +- titan-hadoop-parent/titan-hadoop-2/pom.xml | 2 +- titan-hadoop-parent/titan-hadoop-core/pom.xml | 2 +- titan-hadoop-parent/titan-hadoop/pom.xml | 2 +- titan-hbase-parent/pom.xml | 2 +- titan-hbase-parent/titan-hbase-094/pom.xml | 2 +- titan-hbase-parent/titan-hbase-096/pom.xml | 2 +- titan-hbase-parent/titan-hbase-098/pom.xml | 2 +- titan-hbase-parent/titan-hbase-10/pom.xml | 2 +- titan-hbase-parent/titan-hbase-core/pom.xml | 2 +- titan-hbase-parent/titan-hbase/pom.xml | 2 +- titan-lucene/pom.xml | 2 +- titan-solr/pom.xml | 2 +- titan-test/pom.xml | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pom.xml b/pom.xml index a8b5a3af67..c037cd259e 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ com.thinkaurelius.titan titan - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT pom 2.2.1 diff --git a/titan-all/pom.xml b/titan-all/pom.xml index 68513e86a3..062697aea7 100644 --- a/titan-all/pom.xml +++ b/titan-all/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-all diff --git a/titan-berkeleyje/pom.xml b/titan-berkeleyje/pom.xml index 32e2d36777..773b93c6dc 100644 --- a/titan-berkeleyje/pom.xml +++ b/titan-berkeleyje/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-berkeleyje diff --git a/titan-cassandra/pom.xml b/titan-cassandra/pom.xml index 0a0e047931..0df1a0da48 100644 --- a/titan-cassandra/pom.xml +++ b/titan-cassandra/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-cassandra diff --git a/titan-core/pom.xml b/titan-core/pom.xml index 77f5d2f206..b6307fb1cf 100644 --- a/titan-core/pom.xml +++ b/titan-core/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-core diff --git a/titan-dist/pom.xml b/titan-dist/pom.xml index 32cdbae0f1..fec727b814 100644 --- a/titan-dist/pom.xml +++ b/titan-dist/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml pom diff --git a/titan-dist/titan-dist-hadoop-2/pom.xml b/titan-dist/titan-dist-hadoop-2/pom.xml index 3b366d0f0f..7a29037c07 100644 --- a/titan-dist/titan-dist-hadoop-2/pom.xml +++ b/titan-dist/titan-dist-hadoop-2/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-dist - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml pom diff --git a/titan-doc/pom.xml b/titan-doc/pom.xml index 2d9c199316..3d3e312f94 100644 --- a/titan-doc/pom.xml +++ b/titan-doc/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml pom diff --git a/titan-es/pom.xml b/titan-es/pom.xml index 8eba4162f8..f91d02d46f 100644 --- a/titan-es/pom.xml +++ b/titan-es/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-es diff --git a/titan-hadoop-parent/pom.xml b/titan-hadoop-parent/pom.xml index 363c4f0c50..b97a17b544 100644 --- a/titan-hadoop-parent/pom.xml +++ b/titan-hadoop-parent/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-hadoop-parent diff --git a/titan-hadoop-parent/titan-hadoop-1/pom.xml b/titan-hadoop-parent/titan-hadoop-1/pom.xml index 69cb61d6b9..d114ddbc38 100644 --- a/titan-hadoop-parent/titan-hadoop-1/pom.xml +++ b/titan-hadoop-parent/titan-hadoop-1/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-hadoop-parent - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-hadoop-1 diff --git a/titan-hadoop-parent/titan-hadoop-2/pom.xml b/titan-hadoop-parent/titan-hadoop-2/pom.xml index 65fe8feffa..f2f65bc519 100644 --- a/titan-hadoop-parent/titan-hadoop-2/pom.xml +++ b/titan-hadoop-parent/titan-hadoop-2/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-hadoop-parent - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-hadoop-2 diff --git a/titan-hadoop-parent/titan-hadoop-core/pom.xml b/titan-hadoop-parent/titan-hadoop-core/pom.xml index dcf832fa58..b2287abc28 100644 --- a/titan-hadoop-parent/titan-hadoop-core/pom.xml +++ b/titan-hadoop-parent/titan-hadoop-core/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-hadoop-parent - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-hadoop-core diff --git a/titan-hadoop-parent/titan-hadoop/pom.xml b/titan-hadoop-parent/titan-hadoop/pom.xml index c0a5e831f0..fdd395a802 100644 --- a/titan-hadoop-parent/titan-hadoop/pom.xml +++ b/titan-hadoop-parent/titan-hadoop/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-hadoop-parent - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-hadoop diff --git a/titan-hbase-parent/pom.xml b/titan-hbase-parent/pom.xml index 0b0010d6da..a65ca4e4e4 100644 --- a/titan-hbase-parent/pom.xml +++ b/titan-hbase-parent/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-hbase-parent diff --git a/titan-hbase-parent/titan-hbase-094/pom.xml b/titan-hbase-parent/titan-hbase-094/pom.xml index f176e66a86..4125afac15 100644 --- a/titan-hbase-parent/titan-hbase-094/pom.xml +++ b/titan-hbase-parent/titan-hbase-094/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-hbase-parent - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-hbase-094 diff --git a/titan-hbase-parent/titan-hbase-096/pom.xml b/titan-hbase-parent/titan-hbase-096/pom.xml index 2ea2f7a92d..2eb38f0070 100644 --- a/titan-hbase-parent/titan-hbase-096/pom.xml +++ b/titan-hbase-parent/titan-hbase-096/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-hbase-parent - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-hbase-096 diff --git a/titan-hbase-parent/titan-hbase-098/pom.xml b/titan-hbase-parent/titan-hbase-098/pom.xml index 6c0dda29e9..664fe226b5 100644 --- a/titan-hbase-parent/titan-hbase-098/pom.xml +++ b/titan-hbase-parent/titan-hbase-098/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-hbase-parent - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-hbase-098 diff --git a/titan-hbase-parent/titan-hbase-10/pom.xml b/titan-hbase-parent/titan-hbase-10/pom.xml index 1dcb241e53..6d5a87d8fd 100644 --- a/titan-hbase-parent/titan-hbase-10/pom.xml +++ b/titan-hbase-parent/titan-hbase-10/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-hbase-parent - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-hbase-10 diff --git a/titan-hbase-parent/titan-hbase-core/pom.xml b/titan-hbase-parent/titan-hbase-core/pom.xml index 6c3b13c268..64b810f282 100644 --- a/titan-hbase-parent/titan-hbase-core/pom.xml +++ b/titan-hbase-parent/titan-hbase-core/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-hbase-parent - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-hbase-core diff --git a/titan-hbase-parent/titan-hbase/pom.xml b/titan-hbase-parent/titan-hbase/pom.xml index 007b018e56..dc1a637343 100644 --- a/titan-hbase-parent/titan-hbase/pom.xml +++ b/titan-hbase-parent/titan-hbase/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-hbase-parent - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-hbase diff --git a/titan-lucene/pom.xml b/titan-lucene/pom.xml index ada9087c47..34cf406f8a 100644 --- a/titan-lucene/pom.xml +++ b/titan-lucene/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-lucene diff --git a/titan-solr/pom.xml b/titan-solr/pom.xml index 1d69185333..e8badfdbaf 100644 --- a/titan-solr/pom.xml +++ b/titan-solr/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-solr diff --git a/titan-test/pom.xml b/titan-test/pom.xml index 0ac191bb21..dcf3357953 100644 --- a/titan-test/pom.xml +++ b/titan-test/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan - 1.0.1-SNAPSHOT + 1.1.0-SNAPSHOT ../pom.xml titan-test From d647a6925658a3709899b521fca3fb23e082e1c8 Mon Sep 17 00:00:00 2001 From: Dan LaRocque Date: Thu, 17 Dec 2015 03:10:33 -0500 Subject: [PATCH 21/32] Apply C* compaction options only when present This changes the default behavior of #1156. Instead of hardcoding SizeTieredCompactionStrategy as the default and applying it in the absence of an explicit user override, this commit removes the default value for COMPACTION_STRATEGY. It also updates the three use sites to check whether the config contains COMPACTION_STRATEGY before attempting to read the option's value. Similarly, if COMPACTION_OPTIONS was not set, or if it was set to an empty map, then its use sites will not call the associated builder method, allowing the implementation to use whatever default it might have (probably empty anyway). --- .../cassandra/AbstractCassandraStoreManager.java | 2 +- .../cassandra/astyanax/AstyanaxStoreManager.java | 10 ++++++++-- .../embedded/CassandraEmbeddedStoreManager.java | 8 ++++++-- .../cassandra/thrift/CassandraThriftStoreManager.java | 9 ++++++--- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/AbstractCassandraStoreManager.java b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/AbstractCassandraStoreManager.java index e73ba59f87..fe8214918e 100644 --- a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/AbstractCassandraStoreManager.java +++ b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/AbstractCassandraStoreManager.java @@ -95,7 +95,7 @@ public static Partitioner getPartitioner(String className) { public static final ConfigOption COMPACTION_STRATEGY = new ConfigOption(CASSANDRA_NS, "compaction-strategy-class", "The compaction strategy to use for Titan tables", - ConfigOption.Type.FIXED, "org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy"); + ConfigOption.Type.FIXED, String.class); public static final ConfigOption COMPACTION_OPTIONS = new ConfigOption(CASSANDRA_NS, "compaction-strategy-options", diff --git a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/astyanax/AstyanaxStoreManager.java b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/astyanax/AstyanaxStoreManager.java index b8332746e5..b3e5daa439 100644 --- a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/astyanax/AstyanaxStoreManager.java +++ b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/astyanax/AstyanaxStoreManager.java @@ -439,12 +439,18 @@ private void ensureColumnFamilyExists(String name, String comparator) throws Bac cl.makeColumnFamilyDefinition() .setName(name) .setKeyspace(keySpaceName) - .setCompactionStrategy(storageConfig.get(COMPACTION_STRATEGY)) - .setCompactionStrategyOptions(compactionOptions) .setComparatorType(comparator); ImmutableMap.Builder compressionOptions = new ImmutableMap.Builder(); + if (storageConfig.has(COMPACTION_STRATEGY)) { + cfDef.setCompactionStrategy(storageConfig.get(COMPACTION_STRATEGY)) + } + + if (!compactionOptions.isEmpty()) { + cfDef.setCompactionStrategyOptions(compactionOptions); + } + if (compressionEnabled) { compressionOptions.put("sstable_compression", compressionClass) .put("chunk_length_kb", Integer.toString(compressionChunkSizeKB)); diff --git a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/embedded/CassandraEmbeddedStoreManager.java b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/embedded/CassandraEmbeddedStoreManager.java index 408b1d62c9..06fd6003d1 100644 --- a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/embedded/CassandraEmbeddedStoreManager.java +++ b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/embedded/CassandraEmbeddedStoreManager.java @@ -286,8 +286,12 @@ private void ensureColumnFamilyExists(String keyspaceName, String columnfamilyNa // Column Family not found; create it CFMetaData cfm = new CFMetaData(keyspaceName, columnfamilyName, ColumnFamilyType.Standard, CellNames.fromAbstractType(comparator, true)); try { - cfm.compactionStrategyClass(CFMetaData.createCompactionStrategy(storageConfig.get(COMPACTION_STRATEGY))); - cfm.compactionStrategyOptions(compactionOptions); + if (storageConfig.has(COMPACTION_STRATEGY)) { + cfm.compactionStrategyClass(CFMetaData.createCompactionStrategy(storageConfig.get(COMPACTION_STRATEGY))); + } + if (!compactionOptions.isEmpty()) { + cfm.compactionStrategyOptions(compactionOptions); + } } catch (ConfigurationException e) { throw new PermanentBackendException("Failed to create column family metadata for " + keyspaceName + ":" + columnfamilyName, e); } diff --git a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/thrift/CassandraThriftStoreManager.java b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/thrift/CassandraThriftStoreManager.java index 9c2d0151df..1d1ecabfeb 100644 --- a/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/thrift/CassandraThriftStoreManager.java +++ b/titan-cassandra/src/main/java/com/thinkaurelius/titan/diskstorage/cassandra/thrift/CassandraThriftStoreManager.java @@ -540,9 +540,12 @@ private void createColumnFamily(Cassandra.Client client, createColumnFamily.setName(cfName); createColumnFamily.setKeyspace(ksName); createColumnFamily.setComparator_type(comparator); - createColumnFamily.setCompaction_strategy(storageConfig.get(COMPACTION_STRATEGY)); - createColumnFamily.setCompaction_strategy_options(this.compactionOptions); - + if (storageConfig.has(COMPACTION_STRATEGY)) { + createColumnFamily.setCompaction_strategy(storageConfig.get(COMPACTION_STRATEGY)); + } + if (!compactionOptions.isEmpty()) { + createColumnFamily.setCompaction_strategy_options(compactionOptions); + } ImmutableMap.Builder compressionOptions = new ImmutableMap.Builder(); if (compressionEnabled) { From 94a6ff6279dfa9c385937fb3ae50aeafcb4f8ee5 Mon Sep 17 00:00:00 2001 From: Dan LaRocque Date: Thu, 17 Dec 2015 03:54:39 -0500 Subject: [PATCH 22/32] Update versions to sate Spark & Gremlin-Server This upgrades Jackson from 2.3.0 to 2.4.4 (for compatibility with the version of Spark preferred by TP 3.1). This downgrades Jersey from 1.18.2 to 1.9 to match Gremlin-Server. This is probably going to break something else. Independent of the version changes, this commit also puts jersey-client and jersey-guice under version management. Before, they were unmanaged, so they could get out of sync with the versions of the rest of the (managed) jersey artifacts. Hence jersey-client mismatched to jersey-server, etc. --- pom.xml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index c037cd259e..58aa0013b2 100644 --- a/pom.xml +++ b/pom.xml @@ -79,7 +79,7 @@ 1.0.2 ${hbase100.core.version} 1.9.2 - 2.3.0 + 2.4.4