emissions grid using jersey

.gitignore

@@ -1,3 +1,12 @@
 /.idea/*
 *.iml
-*/target/*
+*/target/*
+
+.settings
+.classpath
+.factorypath
+.project
+
+*.swp
+fileUploads
+

frame-animation/src/main/java/org/matsim/viz/frameAnimation/inputProcessing/VisualizationGenerator.java

@@ -151,7 +151,6 @@ private boolean isValidInput(org.matsim.viz.filesApi.Visualization visualization
                 && visualization.getInputFiles().containsKey(NETWORK_KEY)
                 && visualization.getInputFiles().containsKey(EVENTS_KEY)
                 && visualization.getInputFiles().containsKey(PLANS_KEY)
-                && visualization.getParameters().size() == 1
                 && visualization.getParameters().containsKey(SNAPSHOT_INTERVAL_KEY)
                 && !visualization.getPermissions().isEmpty();
     }

postprocessing-bundle/src/main/java/org/matsim/viz/postprocessing/bundle/VisualizationFetcher.java

@@ -84,7 +84,10 @@ private void generateVisualization(Visualization inputVisualization) {
                 );
                 persistProgress(visualization, PersistentVisualization.Progress.Done, session);
             } catch (Exception e) {
-                log.severe("something went wrong. Setting processing status to failed");
+                e.printStackTrace();
+                log.severe(e.getMessage());
+                log.severe(e.getStackTrace().toString());
+                log.severe("### Something went wrong. Setting processing status to failed");
                 persistProgress(visualization, PersistentVisualization.Progress.Failed, session);
                 // don't throw here, to let other processing continue
             }

postprocessing-emissions/pom.xml

@@ -47,7 +47,13 @@
         <dependency>
             <groupId>org.matsim.contrib</groupId>
             <artifactId>emissions</artifactId>
-            <version>12.0-SNAPSHOT</version>
+            <version>12.0-2020w02-SNAPSHOT</version><!-- use a matsim version with java 8 -->
+            <exclusions>
+                <exclusion>
+                    <groupId>org.hibernate.validator</groupId>
+                    <artifactId>hibernate-validator</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <!-- put jts onto the classpath since the contrib doesn't do it itself... -->
         <dependency>

postprocessing-emissions/src/main/java/org/matsim/viz/postprocessing/emissions/App.java

@@ -13,7 +13,7 @@
 public class App extends Application<PostprocessingConfiguration> {
 
     private HibernateBundle<PostprocessingConfiguration> hibernate = new HibernateBundle<PostprocessingConfiguration>(
-            Agent.class, Permission.class, Visualization.class, FetchInformation.class
+            Agent.class, Permission.class, Bin.class, Visualization.class, FetchInformation.class
     ) {
         @Override
         public PooledDataSourceFactory getDataSourceFactory(PostprocessingConfiguration appConfiguration) {

postprocessing-emissions/src/main/java/org/matsim/viz/postprocessing/emissions/Bin.java

@@ -0,0 +1,26 @@
+package org.matsim.viz.postprocessing.emissions;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import org.matsim.viz.database.AbstractEntity;
+import org.matsim.viz.postprocessing.bundle.PersistentVisualization;
+
+import javax.persistence.Entity;
+import javax.persistence.Lob;
+import javax.persistence.ManyToOne;
+
+@Getter
+@Setter
+@NoArgsConstructor
+@Entity
+public class Bin extends AbstractEntity {
+
+    private double startTime;
+
+    @ManyToOne(optional = false)
+    private Visualization visualization;
+
+    @Lob
+    private String data = "";
+}

postprocessing-emissions/src/main/java/org/matsim/viz/postprocessing/emissions/DataGenerator.java

@@ -6,6 +6,7 @@
 import org.matsim.contrib.emissions.analysis.EmissionGridAnalyzer;
 import org.matsim.core.network.NetworkUtils;
 import org.matsim.core.network.io.MatsimNetworkReader;
+import org.matsim.core.utils.collections.Tuple;
 import org.matsim.viz.postprocessing.bundle.VisualizationGenerator;
 
 @Log
@@ -46,12 +47,24 @@ public void generate(Input<Visualization> input) {
                 .build();
 
         log.info("Start processing emissions. This may take a while.");
-        val json = analyzer.processToJsonString(input.getInputFiles().get(EVENTS_KEY).getPath().toString());
 
-        log.info("Finished processing emissions. Write result to database");
+        analyzer.processTimeBinsWithEmissions(input.getInputFiles().get(EVENTS_KEY).getPath().toString());
 
+        // all of the bins need to be successful, or the transaction should fail.
         session.beginTransaction();
-        mergedViz.setData(json);
+
+        while (analyzer.hasNextTimeBin()) {
+            Tuple<Double, String> result = analyzer.processNextTimeBin();
+            log.info("-- JSON size: " + result.getSecond().length());
+
+            Bin bin = new Bin();
+            bin.setStartTime(result.getFirst());
+            bin.setData(result.getSecond());
+            mergedViz.addBin(bin);
+        }
+
         session.getTransaction().commit();
+
+        log.info("Finished committing database transactions");
     }
 }

postprocessing-emissions/src/main/java/org/matsim/viz/postprocessing/emissions/Visualization.java

@@ -5,19 +5,29 @@
 import lombok.Setter;
 import org.matsim.viz.postprocessing.bundle.PersistentVisualization;
 
-import javax.persistence.Entity;
-import javax.persistence.Lob;
+import javax.persistence.*;
+import java.util.HashSet;
+import java.util.Set;
 
 @Getter
 @Setter
 @NoArgsConstructor
 @Entity
 public class Visualization extends PersistentVisualization {
 
-    @Lob
-    private String data = "";
+    @OneToMany(mappedBy = "visualization", fetch = FetchType.LAZY, orphanRemoval = true, cascade = CascadeType.ALL)
+    private Set<Bin> bins = new HashSet();
 
     private double cellSize;
     private double smoothingRadius;
     private double timeBinSize;
+
+    /**
+     * Add a time bin to the visualization, and link that viz to the bin itself
+     * @param bin
+     */
+    void addBin(Bin bin) {
+        bin.setVisualization(this);
+        bins.add(bin);
+    }
 }

postprocessing-emissions/src/main/java/org/matsim/viz/postprocessing/emissions/VisualizationResource.java

@@ -3,21 +3,23 @@
 import com.querydsl.jpa.impl.JPAQueryFactory;
 import io.dropwizard.auth.Auth;
 import io.dropwizard.hibernate.UnitOfWork;
-import lombok.RequiredArgsConstructor;
+import lombok.*;
 import lombok.extern.java.Log;
-import lombok.val;
 import org.matsim.viz.error.ForbiddenException;
 import org.matsim.viz.error.InvalidInputException;
 import org.matsim.viz.postprocessing.bundle.Agent;
 import org.matsim.viz.postprocessing.bundle.QPermission;
 
 import javax.persistence.EntityManagerFactory;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
+import javax.ws.rs.*;
 import javax.ws.rs.core.MediaType;
 
+import java.util.ArrayList;
+import java.util.List;
+
+import static java.lang.Double.parseDouble;
+
+@Data
 @Log
 @RequiredArgsConstructor
 @Path("{id}")
@@ -26,28 +28,54 @@ public class VisualizationResource {
     private final EntityManagerFactory emFactory;
 
     @GET
-    @Path("/data")
+    @Path("/bin")
     @Produces(MediaType.APPLICATION_JSON)
     @UnitOfWork
-    public String data(@Auth Agent agent, @PathParam("id") String vizId) {
+    public String bin(@Auth Agent agent,
+                       @PathParam("id") String vizId,
+                       @QueryParam("startTime") String startTime) {
+
+        if (hasNoPermission(agent, vizId)) {
+            throw new ForbiddenException("user doesn't have permission");
+        }
+
+        QBin binTable = QBin.bin;
+        val bin = new JPAQueryFactory(emFactory.createEntityManager()).selectFrom(binTable)
+                .where(binTable.visualization.id.eq(vizId)
+                        .and(binTable.startTime.eq(parseDouble(startTime))))
+                .fetchFirst();
 
-        return findVisualization(agent, vizId).getData();
+        if (bin == null) throw new InvalidInputException("Could not find startTime " + startTime);
+
+        return bin.getData();
     }
 
-    private Visualization findVisualization(Agent agent, String vizId) {
+    @GET
+    @Path("/startTimes")
+    @Produces(MediaType.APPLICATION_JSON)
+    @UnitOfWork
+    public List<Double> startTimes(@Auth Agent agent,
+                             @PathParam("id") String vizId) {
 
         if (hasNoPermission(agent, vizId)) {
             throw new ForbiddenException("user doesn't have permission");
         }
 
-        val visualizationTable = QVisualization.visualization;
-        val visualization = new JPAQueryFactory(emFactory.createEntityManager()).selectFrom(visualizationTable)
-                .where(visualizationTable.id.eq(vizId))
-                .fetchOne();
+        QBin binTable = QBin.bin;
+        val startTimes = new JPAQueryFactory(emFactory.createEntityManager()).selectFrom(binTable)
+                .where(binTable.visualization.id.eq(vizId))
+                .select(binTable.startTime)
+                .fetch();
+
+        if (startTimes == null)
+            throw new InvalidInputException("No time bins found");
+
+        ArrayList<Double> result = new ArrayList<>();
+        for (Double startTime : startTimes) result.add(startTime.doubleValue());
+
+        log.info("time bins: " + result.toString());
 
-        if (visualization == null)
-            throw new InvalidInputException("Could not find visualization with id: " + vizId);
-        return visualization;
+        return result;
     }
 
     private boolean hasNoPermission(Agent agent, String vizId) {

postprocessing-emissions/src/main/resources/db/migration/V0.2.0__add_bin_table.sql

@@ -0,0 +1,10 @@
+CREATE TABLE `Bin` (
+  `id` varchar(255) NOT NULL,
+  `starttime` double DEFAULT NULL,
+  `data` longtext DEFAULT NULL,
+  `visualization_id` varchar(255) NOT NULL,
+  PRIMARY KEY (`id`),
+  CONSTRAINT `fk_visualization_id` FOREIGN KEY (`visualization_id`) REFERENCES `Visualization` (`id`)
+)
+CHARACTER SET utf8,
+COLLATE utf8_general_ci;