diff --git a/.gitignore b/.gitignore index 0678262..c8908f1 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ s3credentials *.class conf/application.conf /bin +/cache/ +*~ \ No newline at end of file diff --git a/app/Global.java b/app/Global.java index f37520a..a331246 100644 --- a/app/Global.java +++ b/app/Global.java @@ -1,6 +1,7 @@ import java.io.IOException; import models.Scenario; +import models.Shapefile; import models.User; import controllers.Api; import play.Application; @@ -17,6 +18,7 @@ public void onStart(Application app) { try { Scenario.buildAll(); + Shapefile.writeAllToClusterCache(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); diff --git a/app/controllers/Api.java b/app/controllers/Api.java index fa43c5f..4f23139 100644 --- a/app/controllers/Api.java +++ b/app/controllers/Api.java @@ -155,9 +155,7 @@ public Result apply() throws Throwable { promise = Promise.promise( new Function0() { public TimeSurfaceShort apply() { - LatLon latLon = new LatLon(String.format("%s,%s", lat, lon)); - - ProfileRequest request = analyst.buildProfileRequest(mode, jodaDate, fromTime, toTime, latLon);; + ProfileRequest request = analyst.buildProfileRequest(mode, jodaDate, fromTime, toTime, lat, lon); if(request == null) return null; @@ -284,20 +282,20 @@ public static Result isochrone(Integer surfaceId, List cutoffs) throws * @param shapefileId * @return */ - public static Result result(Integer surfaceId, String shapefileId, String attributeName) { + public static Result result(Integer surfaceId, String shapefileId) { final Shapefile shp = Shapefile.getShapefile(shapefileId); ResultSet result; // it could be a profile request, or not // The IDs are unique; they come from inside OTP. try { - result = AnalystProfileRequest.getResult(surfaceId, shapefileId, attributeName); + result = AnalystProfileRequest.getResult(surfaceId, shapefileId); } catch (NullPointerException e) { - result = AnalystRequest.getResult(surfaceId, shapefileId, attributeName); + result = AnalystRequest.getResult(surfaceId, shapefileId); } ByteArrayOutputStream baos = new ByteArrayOutputStream(); - result.writeJson(baos, shp.getPointSet(attributeName)); + result.writeJson(baos, shp.getPointSet()); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); response().setContentType("application/json"); return ok(bais); @@ -330,7 +328,7 @@ public static List getIsochronesAccumulative(TimeSurface surf, Li } public static Result queryBins(String queryId, Integer timeLimit, String weightByShapefile, String weightByAttribute, String groupBy, - String which, String compareTo) { + String which, String attributeName, String compareTo) { response().setHeader(CACHE_CONTROL, "no-cache, no-store, must-revalidate"); response().setHeader(PRAGMA, "no-cache"); @@ -360,13 +358,13 @@ public static Result queryBins(String queryId, Integer timeLimit, String weightB try { - String queryKey = queryId + "_" + timeLimit + "_" + which; + String queryKey = queryId + "_" + timeLimit + "_" + which + "_" + attributeName; QueryResults qr = null; synchronized(QueryResults.queryResultsCache) { if(!QueryResults.queryResultsCache.containsKey(queryKey)) { - qr = new QueryResults(query, timeLimit, whichEnum); + qr = new QueryResults(query, timeLimit, whichEnum, attributeName); QueryResults.queryResultsCache.put(queryKey, qr); } else @@ -376,9 +374,9 @@ public static Result queryBins(String queryId, Integer timeLimit, String weightB if (otherQuery != null) { QueryResults otherQr = null; - queryKey = compareTo + "_" + timeLimit + "_" + which; + queryKey = compareTo + "_" + timeLimit + "_" + which + "_" + attributeName; if (!QueryResults.queryResultsCache.containsKey(queryKey)) { - otherQr = new QueryResults(otherQuery, timeLimit, whichEnum); + otherQr = new QueryResults(otherQuery, timeLimit, whichEnum, attributeName); QueryResults.queryResultsCache.put(queryKey, otherQr); } else { @@ -625,9 +623,10 @@ public static Result createShapefile() throws ZipException, IOException { if (file != null && file.getFile() != null) { - Shapefile s = Shapefile.create(file.getFile(), body.asFormUrlEncoded().get("projectId")[0]); + String projectId = body.asFormUrlEncoded().get("projectId")[0]; + String name = body.asFormUrlEncoded().get("name")[0]; + Shapefile s = Shapefile.create(file.getFile(), projectId, name); - s.name = body.asFormUrlEncoded().get("name")[0]; s.description = body.asFormUrlEncoded().get("description")[0]; s.save(); diff --git a/app/controllers/Gis.java b/app/controllers/Gis.java index 243d855..53a79a9 100644 --- a/app/controllers/Gis.java +++ b/app/controllers/Gis.java @@ -83,7 +83,7 @@ public class Gis extends Controller { static File TMP_PATH = new File(Application.tmpPath); public static Result query(String queryId, Integer timeLimit, String weightByShapefile, String weightByAttribute, - String groupBy, String which, String compareTo) { + String groupBy, String which, String attributeName, String compareTo) { response().setHeader(CACHE_CONTROL, "no-cache, no-store, must-revalidate"); response().setHeader(PRAGMA, "no-cache"); @@ -114,7 +114,7 @@ public static Result query(String queryId, Integer timeLimit, String weightBySha synchronized(QueryResults.queryResultsCache) { if(!QueryResults.queryResultsCache.containsKey(queryKey)) { - qr = new QueryResults(query, timeLimit, whichEnum); + qr = new QueryResults(query, timeLimit, whichEnum, attributeName); QueryResults.queryResultsCache.put(queryKey, qr); } else @@ -124,7 +124,7 @@ public static Result query(String queryId, Integer timeLimit, String weightBySha String q2key = compareTo + "_" + timeLimit + "_" + which; if(!QueryResults.queryResultsCache.containsKey(q2key)) { - qr2 = new QueryResults(query2, timeLimit, whichEnum); + qr2 = new QueryResults(query2, timeLimit, whichEnum, attributeName); QueryResults.queryResultsCache.put(q2key, qr2); } else { @@ -218,14 +218,14 @@ public static Result query(String queryId, Integer timeLimit, String weightBySha } - public static Result surface(Integer surfaceId, String shapefileId, String attributeName, Integer timeLimit, String compareTo) { + public static Result surface(Integer surfaceId, String shapefileId, Integer timeLimit, String compareTo) { response().setHeader(CACHE_CONTROL, "no-cache, no-store, must-revalidate"); response().setHeader(PRAGMA, "no-cache"); response().setHeader(EXPIRES, "0"); - String shapeName = (timeLimit / 60) + "_mins_" + shapefileId.toString().toLowerCase() + "_" + surfaceId.toString() + "_" + attributeName; + String shapeName = (timeLimit / 60) + "_mins_" + shapefileId.toString().toLowerCase() + "_" + surfaceId.toString(); try { @@ -239,33 +239,41 @@ public static Result surface(Integer surfaceId, String shapefileId, String attri ResultSetWithTimes result; try { - result = AnalystProfileRequest.getResultWithTimes(surfaceId, shapefileId, attributeName); + result = AnalystProfileRequest.getResultWithTimes(surfaceId, shapefileId); } catch (NullPointerException e) { // not a profile request - result = AnalystRequest.getResultWithTimes(surfaceId, shapefileId, attributeName); + result = AnalystRequest.getResultWithTimes(surfaceId, shapefileId); } Collection features = shp.getShapeFeatureStore().getAll(); ArrayList fields = new ArrayList(); - - fields.add(shp.name.replaceAll("\\W+", "")); - + + for (Attribute a : shp.attributes.values()) { + if (a.numeric) { + fields.add(a.name); + } + } + ArrayList gisFeatures = new ArrayList(); - PointSet ps = Shapefile.getShapefile(shapefileId).getPointSet(attributeName); + PointSet ps = Shapefile.getShapefile(shapefileId).getPointSet(); for (ShapeFeature feature : features) { Integer sampleTime = result.times[ps.getIndexForFeature(feature.id)]; - GisShapeFeature gf = new GisShapeFeature(); gf.geom = feature.geom; gf.id = feature.id; gf.time = sampleTime; - gf.fields.add(feature.getAttribute(attributeName)); + // TODO: handle non-integer attributes + for (Attribute a : shp.attributes.values()) { + if (a.numeric) { + gf.fields.add(feature.getAttribute(a.name)); + } + } gisFeatures.add(gf); @@ -342,7 +350,7 @@ static File generateZippedShapefile(String fileName, ArrayList fieldName featureDefinition += "String"; if(features.get(0).fields.get(fieldPosition) instanceof Number) featureDefinition += "Double"; - fieldPosition++; + fieldPosition++; } SimpleFeatureType featureType = DataUtilities.createType("Analyst", featureDefinition); diff --git a/app/controllers/Tiles.java b/app/controllers/Tiles.java index 8d7ba5c..1d59d9c 100644 --- a/app/controllers/Tiles.java +++ b/app/controllers/Tiles.java @@ -108,23 +108,24 @@ public static Promise shape(String shapefileId, Integer x, Integer y, In return tileBuilder(tileRequest); } - public static Promise surface(Integer surfaceId, String shapefileId, String attributeName, Integer x, Integer y, Integer z, + public static Promise surface(Integer surfaceId, String shapefileId, Integer x, Integer y, Integer z, Boolean showIso, Boolean showPoints, Integer timeLimit, Integer minTime) { - AnalystTileRequest tileRequest = new SurfaceTile( surfaceId, shapefileId, attributeName, x, y, z, showIso, showPoints, timeLimit, minTime); + AnalystTileRequest tileRequest = new SurfaceTile( surfaceId, shapefileId, x, y, z, showIso, showPoints, timeLimit, minTime); return tileBuilder(tileRequest); } - public static Promise surfaceComparison(Integer surfaceId1, Integer surfaceId2, String shapefileId, String attributeName, + public static Promise surfaceComparison(Integer surfaceId1, Integer surfaceId2, String shapefileId, Integer x, Integer y, Integer z, Boolean showIso, Boolean showPoints, Integer timeLimit, Integer minTime) { - AnalystTileRequest tileRequest = new SurfaceComparisonTile(surfaceId1, surfaceId2, shapefileId, attributeName, x, y, z, showIso, showPoints, timeLimit, minTime); + AnalystTileRequest tileRequest = new SurfaceComparisonTile(surfaceId1, surfaceId2, shapefileId, x, y, z, showIso, showPoints, timeLimit, minTime); return tileBuilder(tileRequest); } public static Promise query(String queryId, Integer x, Integer y, Integer z, - Integer timeLimit, String weightByShapefile, String weightByAttribute, String groupBy, String which, String compareTo) { + Integer timeLimit, String weightByShapefile, String weightByAttribute, String groupBy, + String which, String attributeName, String compareTo) { ResultEnvelope.Which whichEnum; try { @@ -142,9 +143,9 @@ public Result apply() throws Throwable { AnalystTileRequest tileRequest; if (compareTo == null) - tileRequest = new QueryTile(queryId, x, y, z, timeLimit, weightByShapefile, weightByAttribute, groupBy, whichEnum); + tileRequest = new QueryTile(queryId, x, y, z, timeLimit, weightByShapefile, weightByAttribute, groupBy, whichEnum, attributeName); else - tileRequest = new QueryComparisonTile(queryId, compareTo, x, y, z, timeLimit, weightByShapefile, weightByAttribute, groupBy, whichEnum); + tileRequest = new QueryComparisonTile(queryId, compareTo, x, y, z, timeLimit, weightByShapefile, weightByAttribute, groupBy, whichEnum, attributeName); return tileBuilder(tileRequest); } diff --git a/app/migrations/MoveToJavaSerialization.java b/app/migrations/MoveToJavaSerialization.java new file mode 100644 index 0000000..ac1bf67 --- /dev/null +++ b/app/migrations/MoveToJavaSerialization.java @@ -0,0 +1,95 @@ +package migrations; + +import java.io.File; +import java.util.Map.Entry; + +import com.google.common.io.Files; + +import models.Attribute; +import models.Project; +import models.Shapefile; +import models.User; +import utils.DataStore; + +/** + * Move the storage of most of the data to Java Serialization format; see discussion in issue 68. + * + * This is a java main class and runs outside the Play framework; I just run it inside Eclipse. + * + * @author mattwigway + */ +public class MoveToJavaSerialization { + public static void main(String... args) { + File inDir = new File(args[0]); + File outDir = new File(args[1]); + + if (args.length != 2 || !inDir.isDirectory() || !outDir.isDirectory() || outDir.list().length != 0) { + System.out.println("usage: ... old_data_directory new_data_directory"); + System.out.println("both directories must exist. new directory must be empty."); + return; + } + + // we don't want to use the built-in datastores of models, so point them at nowhere + DataStore.dataPath = Files.createTempDir().getAbsolutePath(); + + // for each of the models, get a datastore, read the data, and then write it out + System.out.println("Processing shapefiles"); + int migrated = new Migrator(inDir, outDir, "shapes").migrate(); + System.out.println("Done processing " + migrated + " shapefiles"); + + System.out.println("Processing users"); + migrated = new Migrator(inDir, outDir, "users").migrate(); + System.out.println("Done processing " + migrated + " users"); + + System.out.println("Processing projects"); + migrated = new Migrator(inDir, outDir, "projects").migrate(); + System.out.println("Done processing " + migrated + " projects"); + + System.out.println("Processing scenarios"); + migrated = new Migrator(inDir, outDir, "scenario").migrate(); + System.out.println("Done processing " + migrated + " scenarios"); + + System.out.println("Processing queries"); + migrated = new Migrator(inDir, outDir, "queries").migrate(); + System.out.println("Done processing " + migrated + " queries"); + } + + /** Migrate a datastore to Java serialization */ + private static class Migrator { + private File inDir; + private File outDir; + private String name; + + public Migrator(File inDir, File outDir, String name) { + this.inDir = inDir; + this.outDir = outDir; + this.name = name; + } + + public int migrate() { + // note: hard wired to transactional and hard-ref default cache. these things are tiny anyhow. + DataStore in = new DataStore(inDir, name, true, false, false); + DataStore out = new DataStore(outDir, name, true, false, true); + + int migrated = 0; + + for (Entry kv : in.getEntries()) { + + // set category IDs on old shapefiles while we're at it. + if (kv instanceof Shapefile) { + Shapefile shp = (Shapefile) kv; + if (shp.categoryId == null) { + shp.categoryId = Attribute.convertNameToId(shp.name); + } + } + + out.saveWithoutCommit(kv.getKey(), kv.getValue()); + migrated++; + } + + out.commit(); + + return migrated; + } + } +} diff --git a/app/models/Attribute.java b/app/models/Attribute.java index 6fe59c7..38c7d80 100644 --- a/app/models/Attribute.java +++ b/app/models/Attribute.java @@ -53,7 +53,11 @@ else if(f instanceof Long) count++; } - static String convertNameToId(String name) { + /** + * Sanitize a name for use as a category ID. Also implemented in the client: + * A.models.Shapefile.getCategoryName() + */ + public static String convertNameToId(String name) { return name.toLowerCase().trim().replaceAll(" ", "_").replaceAll("\\W",""); } diff --git a/app/models/Project.java b/app/models/Project.java index d83d611..8f6f7c9 100644 --- a/app/models/Project.java +++ b/app/models/Project.java @@ -16,7 +16,7 @@ public class Project implements Serializable { private static final long serialVersionUID = 1L; - static private DataStore projectData = new DataStore("projects"); + static private DataStore projectData = new DataStore("projects", true); public String id; public String name; diff --git a/app/models/Query.java b/app/models/Query.java index c196649..7600949 100644 --- a/app/models/Query.java +++ b/app/models/Query.java @@ -57,7 +57,7 @@ public class Query implements Serializable { private static final long serialVersionUID = 1L; - static DataStore queryData = new DataStore("queries"); + static DataStore queryData = new DataStore("queries", true); public String id; public String projectId; @@ -69,8 +69,6 @@ public class Query implements Serializable { public String mode; public String shapefileId; - - public String attributeName; public String scenarioId; public String status; @@ -123,18 +121,6 @@ public Boolean isTransit () { return new TraverseModeSet(this.mode).isTransit(); } - /** - * What attribute is this associated with? - */ - public Attribute getAttribute () { - Shapefile l = Shapefile.getShapefile(shapefileId); - - if (l == null) - return null; - - return l.attributes.get(attributeName); - } - public void save() { // assign id at save @@ -153,7 +139,7 @@ public void save() { public void run() { - ActorRef queryActor = Akka.system().actorOf(Props.create(QueryActor.class)); + ActorRef queryActor = Cluster.getActorSystem().actorOf(Props.create(QueryActor.class)); System.out.println(queryActor.path()); queryActor.tell(this, null); @@ -174,7 +160,11 @@ public void delete() throws IOException { public synchronized DataStore getResults() { if(results == null) { - results = new DataStore(new File(Application.dataPath, "results"), "r_" + id); + // use a non-transactional store to save disk space and increase performance. + // if the query dies we need to throw away the query anyhow. + // we use mapdb serialization because we're more concerned about speed than data durability. + // (this data can be easily reconstructed; this is basically a persistent cache) + results = new DataStore(new File(Application.dataPath, "results"), "r_" + id, false, true, false); } return results; @@ -215,51 +205,10 @@ static void saveQueryResult(String id, ResultEnvelope resultEnvelope) { Query q = getQuery(id); - if(q == null) - return; - - ArrayList writeList = null; - - synchronized(resultsQueue) { - if(!resultsQueue.containsKey(id)) - resultsQueue.put(id, new ArrayList()); - resultsQueue.get(id).add(resultEnvelope); - - if(resultsQueue.get(id).size() > 10) { - writeList = new ArrayList(resultsQueue.get(id)); - resultsQueue.get(id).clear(); - Logger.info("flushing queue..."); - } - - } - - if(writeList != null){ - for(ResultEnvelope rf1 : writeList) - q.getResults().saveWithoutCommit(rf1.id, rf1); - - q.getResults().commit(); - - Tiles.resetQueryCache(id); - } - - - } - - static void updateStatus(String id, JobStatus js) { - - Query q = getQuery(id); - if(q == null) return; - synchronized(q) { - q.totalPoints = (int)js.total; - q.completePoints = (int)js.complete; - q.jobId = js.curJobId; - q.save(); - } - - Logger.info("status update: " + js.complete + "/" + js.total); + q.getResults().saveWithoutCommit(resultEnvelope.id, resultEnvelope); } public static class QueryActor extends UntypedActor { @@ -276,53 +225,35 @@ public void onReceive(Object message) throws Exception { if (workOffline == null) workOffline = true; - String pointSetCachedName = sl.writeToClusterCache(workOffline, q.attributeName); - ActorSystem system = Cluster.getActorSystem(); ActorRef executive = Cluster.getExecutive(); JobSpec js; + String pointSetId = sl.id + ".json"; + + q.totalPoints = sl.getFeatureCount(); + q.completePoints = 0; + if (q.isTransit()) { // create a profile request - ProfileRequest pr = Api.analyst.buildProfileRequest(q.mode, q.date, q.fromTime, q.toTime, null); - js = new JobSpec(q.scenarioId, pointSetCachedName, pointSetCachedName, pr); + ProfileRequest pr = Api.analyst.buildProfileRequest(q.mode, q.date, q.fromTime, q.toTime, 0, 0); + // the pointset is already in the cluster cache, from when it was uploaded. + // every pointset has all shapefile attributes. + js = new JobSpec(q.scenarioId, pointSetId, pointSetId, pr); } else { // this is not a transit request, no need for computationally-expensive profile routing RoutingRequest rr = Api.analyst.buildRequest(q.scenarioId, q.date, q.fromTime, null, q.mode, 120); - js = new JobSpec(q.scenarioId, pointSetCachedName, pointSetCachedName, rr); + js = new JobSpec(q.scenarioId, pointSetId, pointSetId, rr); } // plus a callback that registers how many work items have returned - ActorRef callback = system.actorOf(Props.create(SaveQueryCallback.class, q.id)); + ActorRef callback = system.actorOf(Props.create(SaveQueryCallback.class, q.id, q.totalPoints)); js.setCallback(callback); // start the job - Timeout timeout = new Timeout(Duration.create(60, "seconds")); - Future future = Patterns.ask(executive, js, timeout); - - int jobId = ((JobId) Await.result(future, timeout.duration())).jobId; - - JobStatus status = null; - - // wait for job to complete - do { - Thread.sleep(500); - try { - status = Cluster.getStatus(jobId); - - if(status != null) - Query.updateStatus(q.id, status); - else - Logger.debug("waiting for job status messages, incomplete"); - } - catch (Exception e) { - Logger.debug("waiting for job status messages"); - } - - - } while(status == null || !status.isComplete()); + executive.tell(js, ActorRef.noSender()); } } } @@ -351,17 +282,43 @@ public static class SaveQueryCallback extends JobItemActor { /** the query ID */ public final String id; + /** the number of points completed so far */ + public int complete; + + /** the number of points we expect to complete */ + public final int totalPoints; + /** * Create a new save query callback. * @param id the query ID. */ - public SaveQueryCallback(String id) { + public SaveQueryCallback(String id, int totalPoints) { this.id = id; + this.totalPoints = totalPoints; + complete = 0; } @Override - public synchronized void onWorkResult(WorkResult res) { - Query.saveQueryResult(id, new ResultEnvelope(res)); + public synchronized void onWorkResult(WorkResult res) { + if (res.success) { + Query.saveQueryResult(id, new ResultEnvelope(res)); + } + + // update complete after query has been saved + complete++; + + // only update client every 200 points or when the query is done + if (complete % 200 == 0 || complete == totalPoints) { + Query query = Query.getQuery(id); + + // flush to disk before saying the query is done + // transactional support is off, so this is important + if (complete == totalPoints) + query.getResults().commit(); + + query.completePoints = complete; + query.save(); + } } } } diff --git a/app/models/Scenario.java b/app/models/Scenario.java index 7add2b4..984e7a3 100644 --- a/app/models/Scenario.java +++ b/app/models/Scenario.java @@ -62,7 +62,7 @@ public class Scenario implements Serializable { private static final long serialVersionUID = 1L; - static DataStore scenarioData = new DataStore("scenario"); + static DataStore scenarioData = new DataStore("scenario", true); public String id; public String projectId; diff --git a/app/models/Shapefile.java b/app/models/Shapefile.java index 57a727a..86e0bf8 100644 --- a/app/models/Shapefile.java +++ b/app/models/Shapefile.java @@ -49,6 +49,7 @@ import play.Logger; import play.Play; +import play.libs.Akka; import com.conveyal.otpac.PointSetDatastore; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -63,6 +64,8 @@ import controllers.Api; import controllers.Application; +import scala.concurrent.ExecutionContext; +import scala.concurrent.duration.Duration; import utils.Bounds; import utils.DataStore; import utils.HaltonPoints; @@ -74,14 +77,21 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class Shapefile implements Serializable { - - private static final long serialVersionUID = 1L; + // this should remain constant unless we make a change where we explicitly want to break deserialization + // so that users have to start fresh. + private static final long serialVersionUID = 2L; @JsonIgnore - static private DataStore shapefilesData = new DataStore("shapes"); + static private DataStore shapefilesData = new DataStore("shapes", true); public String id; public String name; + + /** + * The name of this shapefile in the pointset. Don't change. + */ + public String categoryId; + public String description; public String filename; @@ -95,7 +105,8 @@ public class Shapefile implements Serializable { @JsonIgnore public HashMap attributes = new HashMap(); - public static Map pointSetCache = new ConcurrentHashMap(); + /** the pointset for this shapefile */ + private transient PointSet pointSet; @JsonIgnore public File file; @@ -109,7 +120,7 @@ public class Shapefile implements Serializable { public Shapefile() { } - static public class ShapeFeature implements Serializable, Comparable { + static public class ShapeFeature implements Serializable, Comparable { private static final long serialVersionUID = 1L; public String id; @@ -237,64 +248,57 @@ public synchronized STRtree getSpatialIndex() { return spatialIndex; } + /** + * Get the pointset. + */ @JsonIgnore - public PointSet getPointSet(String attrName) { + public synchronized PointSet getPointSet() { + if (pointSet != null) + return pointSet; - String psId = this.id + "_" + attrName; - - synchronized (pointSetCache) { - if(pointSetCache.containsKey(psId)) - return pointSetCache.get(psId); + pointSet = new PointSet(getFeatureCount()); - PointSet ps = new PointSet(getFeatureCount()); + pointSet.id = categoryId; + pointSet.label = this.name; + pointSet.description = this.description; - String categoryId = Attribute.convertNameToId(this.name); + int index = 0; + for(ShapeFeature sf : this.getShapeFeatureStore().getAll()) { - ps.id = categoryId; - ps.label = this.name; - ps.description = this.description; + HashMap propertyData = new HashMap(); - int index = 0; - for(ShapeFeature sf : this.getShapeFeatureStore().getAll()) { - - HashMap propertyData = new HashMap(); - - Attribute a = this.attributes.get(attrName); - String propertyId = categoryId + "." + Attribute.convertNameToId(a.name); + for (Attribute a : this.attributes.values()) { + String propertyId = categoryId + "." + a.fieldName; propertyData.put(propertyId, sf.getAttribute(a.fieldName)); + // TODO: update names when attribute name is edited. + pointSet.setLabel(propertyId, a.name); + } - PointFeature pf; - try { - pf = new PointFeature(sf.id.toString(), sf.geom, propertyData); - ps.addFeature(pf, index); - } catch (EmptyPolygonException | UnsupportedGeometryException e) { - e.printStackTrace(); - } - - - index++; + PointFeature pf; + try { + pf = new PointFeature(sf.id.toString(), sf.geom, propertyData); + pointSet.addFeature(pf, index); + } catch (EmptyPolygonException | UnsupportedGeometryException e) { + e.printStackTrace(); } - ps.setLabel(categoryId, this.name); - Attribute attr = this.attributes.get(attrName); - String propertyId = categoryId + "." + Attribute.convertNameToId(attr.name); - ps.setLabel(propertyId, attr.name); - - if (attr.color != null) - ps.setStyle(propertyId, "color", attr.color); + index++; + } - pointSetCache.put(psId, ps); + pointSet.setLabel(categoryId, this.name); - return ps; - } + return pointSet; } - public String writeToClusterCache(Boolean workOffline, String attrName) throws IOException { + /** + * Write the shapefile to the cluster cache and to S3. + */ + public String writeToClusterCache() throws IOException { - PointSet ps = this.getPointSet(attrName); - String cachePointSetId = id + "_" + Attribute.convertNameToId(attrName) + ".json"; + PointSet ps = this.getPointSet(); + String cachePointSetId = id + ".json"; File f = new File(cachePointSetId); @@ -304,6 +308,7 @@ public String writeToClusterCache(Boolean workOffline, String attrName) throws I String s3credentials = Play.application().configuration().getString("cluster.s3credentials"); String bucket = Play.application().configuration().getString("cluster.pointsets-bucket"); + boolean workOffline = Play.application().configuration().getBoolean("cluster.work-offline"); PointSetDatastore datastore = new PointSetDatastore(10, s3credentials, workOffline, bucket); @@ -361,7 +366,9 @@ public Integer getFeatureCount() { private void buildIndex() { Logger.info("building index for shapefile " + this.id); - spatialIndex = new STRtree(getShapeFeatureStore().size()); + // it's not possible to make an R-tree with only one node, so we make an r-tree with two + // nodes and leave one empty. + spatialIndex = new STRtree(Math.max(getShapeFeatureStore().size(), 2)); for(ShapeFeature feature : getShapeFeatureStore().getAll()) { spatialIndex.insert(feature.geom.getEnvelopeInternal(), feature); @@ -390,7 +397,10 @@ public Collection queryAll() { } - public static Shapefile create(File originalShapefileZip, String projectId) throws ZipException, IOException { + /** + * Create a new shapefile with the given name. + */ + public static Shapefile create(File originalShapefileZip, String projectId, String name) throws ZipException, IOException { String shapefileHash = HashUtils.hashFile(originalShapefileZip); @@ -410,6 +420,10 @@ public static Shapefile create(File originalShapefileZip, String projectId) thro shapefile.id = shapefileId; shapefile.projectId = projectId; + + shapefile.name = name; + shapefile.categoryId = Attribute.convertNameToId(name); + ZipFile zipFile = new ZipFile(originalShapefileZip); @@ -445,7 +459,7 @@ public static Shapefile create(File originalShapefileZip, String projectId) thro shapefile.setShapeFeatureStore(features); shapefile.save(); - + Logger.info("done loading shapefile " + shapefileId); } else @@ -533,26 +547,26 @@ private List> getShapeFeatures() throws ZipExcep for(Object attr : sFeature.getProperties()) { if(attr instanceof Property) { Property p = ((Property)attr); - String name = p.getName().toString(); + String name = Attribute.convertNameToId(p.getName().toString()); PropertyType pt = p.getType(); Object value = p.getValue(); updateAttributeStats(name, value); if(value != null && (value instanceof Long)) { - feature.attributes.put(p.getName().toString(), (int)(long)p.getValue()); + feature.attributes.put(name, (int)(long)p.getValue()); - fieldnamesFound.add(p.getName().toString()); + fieldnamesFound.add(name); } else if( value instanceof Integer) { - feature.attributes.put(p.getName().toString(), (int)p.getValue()); + feature.attributes.put(name, (int)p.getValue()); - fieldnamesFound.add(p.getName().toString()); + fieldnamesFound.add(name); } else if(value != null && (value instanceof Double )) { - feature.attributes.put(p.getName().toString(), (int)(long)Math.round((Double)p.getValue())); + feature.attributes.put(name, (int)(long)Math.round((Double)p.getValue())); - fieldnamesFound.add(p.getName().toString()); + fieldnamesFound.add(name); } } @@ -655,7 +669,6 @@ static public Shapefile getShapefile(String id) { } static public Collection getShapfiles(String projectId) { - if(projectId == null) return shapefilesData.getAll(); @@ -674,4 +687,22 @@ else if(sf.projectId.equals(projectId)) } } + + public static void writeAllToClusterCache() { + ExecutionContext ctx = Akka.system().dispatchers().defaultGlobalDispatcher(); + + for (final Shapefile shapefile : getShapfiles(null)) { + Akka.system().scheduler().scheduleOnce(Duration.create(10, "milliseconds"), new Runnable() { + + @Override + public void run() { + try { + shapefile.writeToClusterCache(); + } catch (Exception e) { + Logger.error("Exception writing " + shapefile + " to cluster cache: " + e); + } + } + }, ctx); + } + } } diff --git a/app/models/User.java b/app/models/User.java index 3b9e133..fd525a5 100644 --- a/app/models/User.java +++ b/app/models/User.java @@ -23,7 +23,7 @@ public class User implements Serializable { private static final long serialVersionUID = 1L; - static private DataStore userData = new DataStore("users"); + static private DataStore userData = new DataStore("users", true); public String id; public String username; diff --git a/app/otp/Analyst.java b/app/otp/Analyst.java index c1328c3..6b3609b 100644 --- a/app/otp/Analyst.java +++ b/app/otp/Analyst.java @@ -1,53 +1,22 @@ package otp; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TimeZone; -import java.util.concurrent.ConcurrentHashMap; - -import models.SpatialLayer; - +import com.conveyal.otpac.PrototypeAnalystProfileRequest; +import com.conveyal.otpac.PrototypeAnalystRequest; import org.joda.time.DateTimeZone; import org.joda.time.LocalDate; -import org.joda.time.LocalDateTime; -import org.opentripplanner.analyst.core.Sample; -import org.opentripplanner.api.model.TimeSurfaceShort; -import org.opentripplanner.api.param.LatLon; -import org.opentripplanner.api.param.YearMonthDay; +import org.opentripplanner.analyst.core.Sample; import org.opentripplanner.common.model.GenericLocation; import org.opentripplanner.profile.Option; import org.opentripplanner.profile.ProfileRequest; -import org.opentripplanner.profile.ProfileResponse; -import org.opentripplanner.profile.ProfileRouter; -import org.opentripplanner.routing.algorithm.EarliestArrivalSPTService; import org.opentripplanner.routing.core.RoutingRequest; import org.opentripplanner.routing.core.TraverseModeSet; -import org.opentripplanner.routing.error.VertexNotFoundException; import org.opentripplanner.routing.graph.Graph; -import org.opentripplanner.routing.services.GraphSource; -import org.opentripplanner.routing.spt.ShortestPathTree; - -import com.conveyal.otpac.PrototypeAnalystProfileRequest; -import com.conveyal.otpac.PrototypeAnalystRequest; -import com.conveyal.otpac.message.JobSpec; -import com.conveyal.otpac.message.WorkResult; -import com.conveyal.otpac.standalone.StandaloneCluster; -import com.conveyal.otpac.standalone.StandaloneExecutive; -import com.conveyal.otpac.standalone.StandaloneWorker; -import com.google.common.collect.Lists; -import com.vividsolutions.jts.geom.Coordinate; - import play.Logger; +import java.io.File; +import java.io.IOException; +import java.util.TimeZone; + public class Analyst { @@ -90,7 +59,7 @@ public RoutingRequest buildRequest(String graphId, LocalDate date, int time, Gen return req; } - public ProfileRequest buildProfileRequest(String mode, LocalDate date, int fromTime, int toTime, LatLon latLon) { + public ProfileRequest buildProfileRequest(String mode, LocalDate date, int fromTime, int toTime, double lat, double lon) { ProfileRequest req = new PrototypeAnalystProfileRequest(); // split the modeset into two modes @@ -99,14 +68,16 @@ public ProfileRequest buildProfileRequest(String mode, LocalDate date, int fromT TraverseModeSet transitModes = new TraverseModeSet(mode); transitModes.setBicycle(false); - transitModes.setDriving(false); + transitModes.setCar(false); transitModes.setWalk(false); req.accessModes = req.egressModes = req.directModes = modes; req.transitModes = transitModes; - req.from = latLon; - req.to = latLon; // not used but required + req.fromLat = lat; + req.fromLon = lon; + req.toLat = lat; // not used but required + req.toLon = lon; req.analyst = true; req.fromTime = fromTime; req.toTime = toTime; diff --git a/app/otp/AnalystGraphBuilder.java b/app/otp/AnalystGraphBuilder.java index 4016a5b..bfbb6b3 100644 --- a/app/otp/AnalystGraphBuilder.java +++ b/app/otp/AnalystGraphBuilder.java @@ -1,115 +1,21 @@ package otp; import java.io.File; -import java.util.ArrayList; -import java.util.List; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; -import org.opentripplanner.graph_builder.GraphBuilderTask; -import org.opentripplanner.graph_builder.impl.EmbeddedConfigGraphBuilderImpl; -import org.opentripplanner.graph_builder.impl.GtfsGraphBuilderImpl; -import org.opentripplanner.graph_builder.impl.DirectTransferGenerator; -import org.opentripplanner.graph_builder.impl.PruneFloatingIslands; -import org.opentripplanner.graph_builder.impl.TransitToStreetNetworkGraphBuilderImpl; -import org.opentripplanner.graph_builder.impl.ned.ElevationGraphBuilderImpl; -import org.opentripplanner.graph_builder.impl.ned.NEDGridCoverageFactoryImpl; -import org.opentripplanner.graph_builder.impl.osm.DefaultWayPropertySetSource; -import org.opentripplanner.graph_builder.impl.osm.OpenStreetMapGraphBuilderImpl; -import org.opentripplanner.graph_builder.model.GtfsBundle; -import org.opentripplanner.graph_builder.services.GraphBuilder; -import org.opentripplanner.graph_builder.services.ned.ElevationGridCoverageFactory; -import org.opentripplanner.openstreetmap.impl.AnyFileBasedOpenStreetMapProviderImpl; -import org.opentripplanner.openstreetmap.services.OpenStreetMapProvider; -import org.opentripplanner.routing.impl.DefaultFareServiceFactory; - +import org.opentripplanner.graph_builder.GraphBuilder; import org.opentripplanner.standalone.CommandLineParameters; -import org.opentripplanner.standalone.OTPConfigurator; -import play.Logger; - -import com.google.common.collect.Lists; public class AnalystGraphBuilder { - public static GraphBuilderTask createBuilder(File dir) { - - GraphBuilderTask graphBuilder = new GraphBuilderTask(); - List gtfsFiles = Lists.newArrayList(); - List osmFiles = Lists.newArrayList(); - File configFile = null; - /* For now this is adding files from all directories listed, rather than building multiple graphs. */ - - if ( !dir.isDirectory() && dir.canRead()) { - return null; - } - - graphBuilder.setPath(dir); - for (File file : dir.listFiles()) { - switch (InputFileType.forFile(file)) { - case GTFS: - Logger.info("Found GTFS file {}", file); - gtfsFiles.add(file); - break; - case OSM: - Logger.info("Found OSM file {}", file); - osmFiles.add(file); - break; - case OTHER: - Logger.debug("Skipping file '{}'", file); - } - } - - boolean hasOSM = ! (osmFiles.isEmpty()); - boolean hasGTFS = ! (gtfsFiles.isEmpty()); - - if ( ! (hasOSM || hasGTFS )) { - Logger.error("Found no input files from which to build a graph in {}", dir.toString()); - return null; - } - - if ( hasOSM ) { - List osmProviders = Lists.newArrayList(); - for (File osmFile : osmFiles) { - OpenStreetMapProvider osmProvider = new AnyFileBasedOpenStreetMapProviderImpl(osmFile); - osmProviders.add(osmProvider); - } - OpenStreetMapGraphBuilderImpl osmBuilder = new OpenStreetMapGraphBuilderImpl(osmProviders); - DefaultWayPropertySetSource defaultWayPropertySetSource = new DefaultWayPropertySetSource(); - osmBuilder.setDefaultWayPropertySetSource(defaultWayPropertySetSource); - osmBuilder.skipVisibility = false; - graphBuilder.addGraphBuilder(osmBuilder); - graphBuilder.addGraphBuilder(new PruneFloatingIslands()); - graphBuilder.addGraphBuilder(new DirectTransferGenerator()); - - } - if ( hasGTFS ) { - List gtfsBundles = Lists.newArrayList(); - for (File gtfsFile : gtfsFiles) { - GtfsBundle gtfsBundle = new GtfsBundle(gtfsFile); - gtfsBundle.setTransfersTxtDefinesStationPaths(false); - gtfsBundle.linkStopsToParentStations = false; - gtfsBundle.parentStationTransfers =false; - gtfsBundles.add(gtfsBundle); - } - GtfsGraphBuilderImpl gtfsBuilder = new GtfsGraphBuilderImpl(gtfsBundles); - graphBuilder.addGraphBuilder(gtfsBuilder); - - graphBuilder.addGraphBuilder(new TransitToStreetNetworkGraphBuilderImpl()); - - - - } - graphBuilder.serializeGraph = false; + public static GraphBuilder createBuilder(File dir) { CommandLineParameters params = new CommandLineParameters(); - params.build = new ArrayList(1); - params.build.add(dir); + params.build = dir; params.inMemory = true; - params.longDistance = true; - graphBuilder = new OTPConfigurator(params).builderFromParameters(); - - return graphBuilder; + return GraphBuilder.forDirectory(params, dir); } private static enum InputFileType { diff --git a/app/otp/AnalystGraphCache.java b/app/otp/AnalystGraphCache.java index 40fc8b4..85937a7 100644 --- a/app/otp/AnalystGraphCache.java +++ b/app/otp/AnalystGraphCache.java @@ -7,21 +7,13 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import org.opentripplanner.api.model.TimeSurfaceShort; -import org.opentripplanner.common.model.GenericLocation; -import org.opentripplanner.graph_builder.GraphBuilderTask; +import org.opentripplanner.graph_builder.GraphBuilder; import org.opentripplanner.routing.graph.Graph; import org.opentripplanner.routing.impl.DefaultStreetVertexIndexFactory; import play.Logger; import play.Play; import play.libs.Akka; -import play.libs.Json; -import play.libs.F.Function; -import play.libs.F.Function0; -import play.libs.F.Promise; -import play.mvc.Result; -import scala.concurrent.ExecutionContext; import scala.concurrent.duration.Duration; import com.conveyal.otpac.ClusterGraphService; @@ -151,10 +143,10 @@ public Graph load(final String graphId) throws Exception { Graph g; try { - GraphBuilderTask gbt = AnalystGraphBuilder.createBuilder(new File(new File(Application.dataPath,"graphs"), graphId)); - gbt.run(); + GraphBuilder gb = AnalystGraphBuilder.createBuilder(new File(new File(Application.dataPath,"graphs"), graphId)); + gb.run(); - g = gbt.getGraph(); + g = gb.getGraph(); g.routerId = graphId; g.index(new DefaultStreetVertexIndexFactory()); diff --git a/app/otp/AnalystProfileRequest.java b/app/otp/AnalystProfileRequest.java index 74f0f44..8e1fa91 100644 --- a/app/otp/AnalystProfileRequest.java +++ b/app/otp/AnalystProfileRequest.java @@ -8,8 +8,10 @@ import models.Shapefile; +import org.opentripplanner.analyst.PointSet; import org.opentripplanner.analyst.ResultSet; import org.opentripplanner.analyst.ResultSetWithTimes; +import org.opentripplanner.analyst.SampleSet; import org.opentripplanner.analyst.SurfaceCache; import org.opentripplanner.analyst.TimeSurface; import org.opentripplanner.api.model.TimeSurfaceShort; @@ -79,9 +81,9 @@ else if (which == ResultEnvelope.Which.AVERAGE) { * Get the ResultSet for the given ID. Note that no ResultEnvelope.Which need be specified as each surface ID is unique to a particular * statistic. */ - public static ResultSet getResult(Integer surfaceId, String shapefileId, String attributeName) { + public static ResultSet getResult(Integer surfaceId, String shapefileId) { - String resultId = "resultId_" + surfaceId + "_" + shapefileId + "_" + attributeName; + String resultId = "resultId_" + surfaceId + "_" + shapefileId; ResultSet result; @@ -91,7 +93,10 @@ public static ResultSet getResult(Integer surfaceId, String shapefileId, String else { TimeSurface surf =getSurface(surfaceId); - result = new ResultSet(Shapefile.getShapefile(shapefileId).getPointSet(attributeName).getSampleSet(Api.analyst.getGraph(surf.routerId)), surf); + PointSet ps = Shapefile.getShapefile(shapefileId).getPointSet(); + SampleSet ss = ps.getSampleSet(Api.analyst.getGraph(surf.routerId)); + result = new ResultSet(ss, surf); + resultCache.put(resultId, result); } } @@ -103,9 +108,9 @@ public static ResultSet getResult(Integer surfaceId, String shapefileId, String * Get the ResultSet for the given ID. Note that no min/max need be specified as each surface ID is unique to a particular * statistic. */ - public static ResultSetWithTimes getResultWithTimes(Integer surfaceId, String shapefileId, String attributeName) { + public static ResultSetWithTimes getResultWithTimes(Integer surfaceId, String shapefileId) { - String resultId = "resultWithTimesId_" + surfaceId + "_" + shapefileId + "_" + attributeName; + String resultId = "resultWithTimesId_" + surfaceId + "_" + shapefileId; ResultSetWithTimes resultWithTimes; @@ -114,8 +119,10 @@ public static ResultSetWithTimes getResultWithTimes(Integer surfaceId, String sh resultWithTimes = (ResultSetWithTimes)resultCache.get(resultId); else { TimeSurface surf = getSurface(surfaceId); - - resultWithTimes = new ResultSetWithTimes(Shapefile.getShapefile(shapefileId).getPointSet(attributeName).getSampleSet(Api.analyst.getGraph(surf.routerId)), surf); + + PointSet ps = Shapefile.getShapefile(shapefileId).getPointSet(); + SampleSet ss = ps.getSampleSet(Api.analyst.getGraph(surf.routerId)); + resultWithTimes = new ResultSetWithTimes(ss, surf); resultCache.put(resultId, resultWithTimes); } } diff --git a/app/otp/AnalystRequest.java b/app/otp/AnalystRequest.java index f27d782..39e1de4 100644 --- a/app/otp/AnalystRequest.java +++ b/app/otp/AnalystRequest.java @@ -1,20 +1,9 @@ package otp; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectOutputStream; -import java.math.BigInteger; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.List; import java.util.Map; -import java.util.TimeZone; import java.util.concurrent.ConcurrentHashMap; -import javax.ws.rs.core.Response; - import models.Shapefile; -import models.SpatialLayer; import org.opentripplanner.analyst.PointSet; import org.opentripplanner.analyst.ResultSet; @@ -24,13 +13,10 @@ import org.opentripplanner.analyst.TimeSurface; import org.opentripplanner.api.model.TimeSurfaceShort; import org.opentripplanner.common.model.GenericLocation; -import org.opentripplanner.routing.algorithm.EarliestArrivalSPTService; +import org.opentripplanner.routing.algorithm.EarliestArrivalSearch; import org.opentripplanner.routing.core.RoutingRequest; import org.opentripplanner.routing.spt.ShortestPathTree; -import com.amazonaws.services.elasticmapreduce.model.Application; -import com.google.common.collect.Maps; - import controllers.Api; public class AnalystRequest extends RoutingRequest{ @@ -44,7 +30,7 @@ public class AnalystRequest extends RoutingRequest{ public int cutoffMinutes; - static public AnalystRequest create(String graphId, GenericLocation latLon, int cutoffMinutes) throws IOException, NoSuchAlgorithmException { + static public AnalystRequest create(String graphId, GenericLocation latLon, int cutoffMinutes) { AnalystRequest request = new PrototypeAnalystRequest(); @@ -61,7 +47,7 @@ static public AnalystRequest create(String graphId, GenericLocation latLon, int public static TimeSurfaceShort createSurface(RoutingRequest req, int cutoffMinutes) { - EarliestArrivalSPTService sptService = new EarliestArrivalSPTService(); + EarliestArrivalSearch sptService = new EarliestArrivalSearch(); sptService.maxDuration = 60 * cutoffMinutes; ShortestPathTree spt = sptService.getShortestPathTree(req); @@ -81,9 +67,9 @@ public static TimeSurfaceShort createSurface(RoutingRequest req, int cutoffMinut } - public static ResultSet getResult(Integer surfaceId, String shapefileId, String attributeName) { + public static ResultSet getResult(Integer surfaceId, String shapefileId) { - String resultId = "resultId_" + surfaceId + "_" + shapefileId + "_" + attributeName; + String resultId = "resultId_" + surfaceId + "_" + shapefileId; ResultSet result; @@ -92,7 +78,7 @@ public static ResultSet getResult(Integer surfaceId, String shapefileId, String result = resultCache.get(resultId); else { TimeSurface surf =getSurface(surfaceId); - PointSet ps = Shapefile.getShapefile(shapefileId).getPointSet(attributeName); + PointSet ps = Shapefile.getShapefile(shapefileId).getPointSet(); SampleSet ss = ps.getSampleSet(Api.analyst.getGraph(surf.routerId)); result = new ResultSet(ss, surf); resultCache.put(resultId, result); @@ -102,9 +88,9 @@ public static ResultSet getResult(Integer surfaceId, String shapefileId, String return result; } - public static ResultSetWithTimes getResultWithTimes(Integer surfaceId, String shapefileId, String attributeName) { + public static ResultSetWithTimes getResultWithTimes(Integer surfaceId, String shapefileId) { - String resultId = "resultWIthTimesId_" + surfaceId + "_" + shapefileId + "_" + attributeName; + String resultId = "resultWithTimesId_" + surfaceId + "_" + shapefileId; ResultSetWithTimes resultWithTimes; @@ -113,7 +99,7 @@ public static ResultSetWithTimes getResultWithTimes(Integer surfaceId, String sh resultWithTimes = (ResultSetWithTimes)resultCache.get(resultId); else { TimeSurface surf =getSurface(surfaceId); - PointSet ps = Shapefile.getShapefile(shapefileId).getPointSet(attributeName); + PointSet ps = Shapefile.getShapefile(shapefileId).getPointSet(); SampleSet ss = ps.getSampleSet(Api.analyst.getGraph(surf.routerId)); resultWithTimes = new ResultSetWithTimes(ss, surf); resultCache.put(resultId, resultWithTimes); diff --git a/app/tiles/AnalystTileRequest.java b/app/tiles/AnalystTileRequest.java index 0da39d2..10ba90c 100644 --- a/app/tiles/AnalystTileRequest.java +++ b/app/tiles/AnalystTileRequest.java @@ -20,6 +20,7 @@ import org.opentripplanner.analyst.PointSet; import org.opentripplanner.analyst.ResultSetDelta; import org.opentripplanner.analyst.ResultSetWithTimes; +import org.opentripplanner.analyst.SampleSet; import org.opentripplanner.analyst.TimeSurface; import otp.AnalystProfileRequest; @@ -41,6 +42,8 @@ import com.google.common.hash.Hashing; import com.vividsolutions.jts.index.strtree.STRtree; +import controllers.Api; + public abstract class AnalystTileRequest { private static TransportIndex transitIndex = new TransportIndex(); @@ -346,7 +349,6 @@ public byte[] render(){ public static class SurfaceComparisonTile extends AnalystTileRequest { final String shapefileId; - final String attributeName; final Integer surfaceId1; final Integer surfaceId2; final Boolean showIso; @@ -354,12 +356,11 @@ public static class SurfaceComparisonTile extends AnalystTileRequest { final Integer timeLimit; final Integer minTime; - public SurfaceComparisonTile(Integer surfaceId1, Integer surfaceId2, String shapefileId, String attributeName, + public SurfaceComparisonTile(Integer surfaceId1, Integer surfaceId2, String shapefileId, Integer x, Integer y, Integer z, Boolean showIso, Boolean showPoints, Integer timeLimit, Integer minTime) { super(x, y, z, "surface"); this.shapefileId = shapefileId; - this.attributeName = attributeName; this.surfaceId1 = surfaceId1; this.surfaceId2 = surfaceId2; this.showIso = showIso; @@ -369,7 +370,7 @@ public SurfaceComparisonTile(Integer surfaceId1, Integer surfaceId2, String shap } public String getId() { - return super.getId() + "_" + shapefileId + "_" + attributeName + "_" + surfaceId1 + "_" + surfaceId2 + "_" + showIso + "_" + showPoints + "_" + timeLimit + "_" + minTime; + return super.getId() + "_" + shapefileId + "_" + surfaceId1 + "_" + surfaceId2 + "_" + showIso + "_" + showPoints + "_" + timeLimit + "_" + minTime; } public byte[] render(){ @@ -389,8 +390,13 @@ public byte[] render(){ if (surf2 == null) surf2 = AnalystRequest.getSurface(surfaceId2); - PointSet ps = shp.getPointSet(attributeName); - ResultSetDelta resultDelta = new ResultSetDelta(ps.getSampleSet(Api.analyst.getGraph(surf1.routerId)), ps.getSampleSet(Api.analyst.getGraph(surf2.routerId)), surf1, surf2); + PointSet ps = shp.getPointSet(); + + // TODO: cache samples on multiple tile requests (should be a performance win) + SampleSet ss1 = ps.getSampleSet(Api.analyst.getGraph(surf1.routerId)); + SampleSet ss2 = ps.getSampleSet(Api.analyst.getGraph(surf2.routerId)); + ResultSetDelta resultDelta = new ResultSetDelta(ss1, ss2, surf1, surf2); + List features = shp.query(tile.envelope); @@ -476,14 +482,12 @@ public static class SurfaceTile extends AnalystTileRequest { final Boolean showPoints; final Integer timeLimit; final Integer minTime; - final String attributeName; - public SurfaceTile(Integer surfaceId, String shapefileId, String attributeName, Integer x, Integer y, Integer z, + public SurfaceTile(Integer surfaceId, String shapefileId, Integer x, Integer y, Integer z, Boolean showIso, Boolean showPoints, Integer timeLimit, Integer minTime) { super(x, y, z, "surface"); this.shapefileId = shapefileId; - this.attributeName = attributeName; this.surfaceId = surfaceId; this.showIso = showIso; this.showPoints = showPoints; @@ -492,7 +496,7 @@ public SurfaceTile(Integer surfaceId, String shapefileId, String attributeName, } public String getId() { - return super.getId() + "_" + shapefileId + "_" + attributeName + "_" + surfaceId + "_" + showIso + "_" + showPoints + "_" + timeLimit + "_" + minTime; + return super.getId() + "_" + shapefileId + "_" + surfaceId + "_" + showIso + "_" + showPoints + "_" + timeLimit + "_" + minTime; } public byte[] render(){ @@ -508,16 +512,16 @@ public byte[] render(){ ResultSetWithTimes result; try { - result = AnalystProfileRequest.getResultWithTimes(surfaceId, shapefileId, attributeName); + result = AnalystProfileRequest.getResultWithTimes(surfaceId, shapefileId); } catch (NullPointerException e) { // not a profile request - result = AnalystRequest.getResultWithTimes(surfaceId, shapefileId, attributeName); + result = AnalystRequest.getResultWithTimes(surfaceId, shapefileId); } List features = shp.query(tile.envelope); - PointSet ps = Shapefile.getShapefile(shapefileId).getPointSet(attributeName); + PointSet ps = Shapefile.getShapefile(shapefileId).getPointSet(); for(ShapeFeature feature : features) { @@ -588,11 +592,13 @@ public static class QueryTile extends AnalystTileRequest { final Integer timeLimit; final String weightByShapefile; final String weightByAttribute; + final String attributeName; final String groupBy; final ResultEnvelope.Which which; public QueryTile(String queryId, Integer x, Integer y, Integer z, Integer timeLimit, - String weightByShapefile, String weightByAttribute, String groupBy, ResultEnvelope.Which which) { + String weightByShapefile, String weightByAttribute, String groupBy, + ResultEnvelope.Which which, String attributeName) { super(x, y, z, "transit"); this.queryId = queryId; @@ -601,10 +607,11 @@ public QueryTile(String queryId, Integer x, Integer y, Integer z, Integer timeLi this.weightByAttribute = weightByAttribute; this.groupBy = groupBy; this.which = which; + this.attributeName = attributeName; } public String getId() { - return super.getId() + "_" + queryId + "_" + timeLimit + "_" + which + "_" + weightByShapefile + "_" + groupBy + "_" + weightByAttribute; + return super.getId() + "_" + queryId + "_" + timeLimit + "_" + which + "_" + weightByShapefile + "_" + groupBy + "_" + weightByAttribute + "_" + attributeName; } public byte[] render(){ @@ -615,13 +622,13 @@ public byte[] render(){ return null; - String queryKey = queryId + "_" + timeLimit + "_" + which; + String queryKey = queryId + "_" + timeLimit + "_" + which + "_" + attributeName; QueryResults qr = null; synchronized(QueryResults.queryResultsCache) { if(!QueryResults.queryResultsCache.containsKey(queryKey)) { - qr = new QueryResults(query, timeLimit, which); + qr = new QueryResults(query, timeLimit, which, attributeName); QueryResults.queryResultsCache.put(queryKey, qr); } else @@ -724,8 +731,9 @@ public static class QueryComparisonTile extends QueryTile { public final String compareTo; public QueryComparisonTile(String queryId, String compareTo, Integer x, Integer y, Integer z, Integer timeLimit, - String weightByShapefile, String weightByAttribute, String groupBy, ResultEnvelope.Which which) { - super(queryId, x, y, z, timeLimit, weightByShapefile, weightByAttribute, groupBy, which); + String weightByShapefile, String weightByAttribute, String groupBy, ResultEnvelope.Which which, + String attributeName) { + super(queryId, x, y, z, timeLimit, weightByShapefile, weightByAttribute, groupBy, which, attributeName); this.compareTo = compareTo; } @@ -743,13 +751,13 @@ public byte[] render () { if (q1 == null || q2 == null || !q1.shapefileId.equals(q2.shapefileId)) return null; - String q1key = queryId + "_" + timeLimit + "_" + which; - String q2key = compareTo + "_" + timeLimit + "_" + which; + String q1key = queryId + "_" + timeLimit + "_" + which + "_" + attributeName; + String q2key = compareTo + "_" + timeLimit + "_" + which + "_" + attributeName; QueryResults qr1, qr2; if (!QueryResults.queryResultsCache.containsKey(q1key)) { - qr1 = new QueryResults(q1, timeLimit, which); + qr1 = new QueryResults(q1, timeLimit, which, attributeName); QueryResults.queryResultsCache.put(q1key, qr1); } else { @@ -757,7 +765,7 @@ public byte[] render () { } if (!QueryResults.queryResultsCache.containsKey(q2key)) { - qr2 = new QueryResults(q2, timeLimit, which); + qr2 = new QueryResults(q2, timeLimit, which, attributeName); QueryResults.queryResultsCache.put(q2key, qr2); } else { diff --git a/app/utils/Bounds.java b/app/utils/Bounds.java index bae1780..501ea18 100644 --- a/app/utils/Bounds.java +++ b/app/utils/Bounds.java @@ -7,7 +7,8 @@ import com.vividsolutions.jts.geom.Envelope; public class Bounds implements Serializable { - + private static final long serialVersionUID = 1L; + public Double west, east, south, north; public Bounds() { diff --git a/app/utils/ClassLoaderSerializer.java b/app/utils/ClassLoaderSerializer.java new file mode 100644 index 0000000..2c5a793 --- /dev/null +++ b/app/utils/ClassLoaderSerializer.java @@ -0,0 +1,41 @@ +package utils; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.io.Serializable; + +import org.apache.commons.io.input.ClassLoaderObjectInputStream; +import org.mapdb.Serializer; + +/** + * Deserialize using the thread's class loader, not the root class loader. + */ +public class ClassLoaderSerializer implements Serializer, Serializable { + + @Override + public void serialize(DataOutput out, Object value) throws IOException { + ObjectOutputStream out2 = new ObjectOutputStream((OutputStream) out); + out2.writeObject(value); + out2.flush(); + } + + @Override + public Object deserialize(DataInput in, int available) throws IOException { + try { + ObjectInputStream in2 = new ClassLoaderObjectInputStream(Thread.currentThread().getContextClassLoader(), (InputStream) in); + return in2.readObject(); + } catch (ClassNotFoundException e) { + throw new IOException(e); + } + } + + @Override + public int fixedSize() { + return -1; + } +} diff --git a/app/utils/Cluster.java b/app/utils/Cluster.java index 90936b3..f9e83d2 100644 --- a/app/utils/Cluster.java +++ b/app/utils/Cluster.java @@ -98,20 +98,11 @@ public static synchronized ActorRef getExecutive () { */ public static JobStatus getStatus(int jobId) { Timeout timeout = new Timeout(Duration.create(5, "seconds")); - Future future = Patterns.ask(executive, new JobStatusQuery(), timeout); - ArrayList result; + Future future = Patterns.ask(executive, new JobStatusQuery(jobId), timeout); try { - result = (ArrayList) Await.result(future, timeout.duration()); + return (JobStatus) Await.result(future, timeout.duration()); } catch (Exception e) { return null; } - - for (JobStatus s : result) { - if (s.curJobId == jobId) { - return s; - } - } - - return null; } } diff --git a/app/utils/DataStore.java b/app/utils/DataStore.java index 9b32a1b..b62ac96 100644 --- a/app/utils/DataStore.java +++ b/app/utils/DataStore.java @@ -8,12 +8,14 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import models.Shapefile.ShapeFeature; import org.mapdb.BTreeKeySerializer; import org.mapdb.BTreeMap; import org.mapdb.DB; +import org.mapdb.DB.BTreeMapMaker; import org.mapdb.DBMaker; import org.mapdb.Fun; import org.mapdb.Pump; @@ -28,12 +30,37 @@ public class DataStore { DB db; Map map; - public DataStore(String dataFile) { + public static String dataPath = null; - this(new File(Application.dataPath), dataFile); + /** Create a new data store in the default location with transactional support enabled and the default cache and serializer */ + public DataStore(String dataFile) { + + // allow models to be used outside of the application by specifying a data path directly + this(new File(dataPath != null ? dataPath : Application.dataPath), dataFile, true, false, false); } + /** Create a new data store in the default location with transactional support enabled and the default cache */ + public DataStore(String dataFile, boolean useJavaSerialization) { + + this(new File(dataPath != null ? dataPath : Application.dataPath), dataFile, true, false, useJavaSerialization); + } + + /** + * Create a new datastore with transactional support enabled and a default cache. + */ public DataStore(File directory, String dataFile) { + this(directory, dataFile, true, false, false); + } + + /** + * Create a new DataStore. + * @param directory Where should it be created? + * @param dataFile What should it be called? + * @param transactional Should MapDB's transactional support be enabled? + * @param weakRefCache Should we use a weak reference cache instead of the default fixed-size cache? + * @param useJavaSerialization Should java serialization be used instead of mapdb serialization (more tolerant to class version changes)? + */ + public DataStore(File directory, String dataFile, boolean transactional, boolean weakRefCache, boolean useJavaSerialization) { if(!directory.exists()) directory.mkdirs(); @@ -45,13 +72,27 @@ public DataStore(File directory, String dataFile) { e.printStackTrace(); } - db = DBMaker.newFileDB(new File(directory, dataFile + ".db")) - .closeOnJvmShutdown() - .make(); + DBMaker dbm = DBMaker.newFileDB(new File(directory, dataFile + ".db")) + .closeOnJvmShutdown(); - map = db.getTreeMap(dataFile); + if (!transactional) + dbm = dbm.transactionDisable(); + + if (weakRefCache) + dbm = dbm.cacheWeakRefEnable(); + + db = dbm.make(); + + BTreeMapMaker maker = db.createTreeMap(dataFile); + + // this probably ought to cache the serializer. + if (useJavaSerialization) + maker = maker.valueSerializer(new ClassLoaderSerializer()); + + map = maker.makeOrGet(); } + // TODO: add all the other arguments about what kind of serialization, transactions, etc. public DataStore(File directory, String dataFile, List>inputData) { if(!directory.exists()) @@ -131,6 +172,10 @@ public Collection getAll() { return map.values(); } + public Collection> getEntries () { + return map.entrySet(); + } + public Integer size() { return map.keySet().size(); } diff --git a/app/utils/QueryResults.java b/app/utils/QueryResults.java index 5368de0..99e5cb5 100644 --- a/app/utils/QueryResults.java +++ b/app/utils/QueryResults.java @@ -25,6 +25,7 @@ import com.vividsolutions.jts.index.SpatialIndex; import com.vividsolutions.jts.index.strtree.STRtree; +import models.Attribute; import models.Query; import models.Shapefile; import models.SpatialLayer; @@ -88,7 +89,7 @@ public QueryResults() { } - public QueryResults(Query q, Integer timeLimit, ResultEnvelope.Which which) { + public QueryResults(Query q, Integer timeLimit, ResultEnvelope.Which which, String attributeId) { Shapefile sd = Shapefile.getShapefile(q.shapefileId); this.which = which; @@ -119,7 +120,7 @@ public QueryResults(Query q, Integer timeLimit, ResultEnvelope.Which which) { throw new RuntimeException("Unhandled envelope type"); } - value = (double) feature.sum(timeLimit); + value = (double) feature.sum(timeLimit, sd.categoryId + "." + attributeId); if(maxValue == null || value > maxValue) maxValue = value; @@ -137,7 +138,7 @@ public QueryResults(Query q, Integer timeLimit, ResultEnvelope.Which which) { shapeFileId = sd.id; - attributeId = q.attributeName; + this.attributeId = attributeId; this.maxPossible = sd.attributes.get(attributeId).sum; @@ -382,7 +383,9 @@ public SpatialIndex getSpatialIndex () { */ public SpatialIndex getSpatialIndex (boolean forceRebuild) { if (forceRebuild || spIdx == null) { - spIdx = new STRtree(items.size()); + // we can't build an STRtree with only one node, so we make sure we make a minimum of + // two nodes even if we leave one empty + spIdx = new STRtree(Math.max(items.size(), 2)); for (QueryResultItem i : this.items.values()) { spIdx.insert(i.feature.geom.getEnvelopeInternal(), i); diff --git a/app/utils/ResultEnvelope.java b/app/utils/ResultEnvelope.java index e5078f4..1343879 100644 --- a/app/utils/ResultEnvelope.java +++ b/app/utils/ResultEnvelope.java @@ -61,6 +61,7 @@ public ResultEnvelope (WorkResult res) { this.avgCase = res.getAvgCase(); this.pointEstimate = null; this.spread = null; + // the surface will never be null, because it is only created if the workresult was successful this.id = this.bestCase.id; } else { diff --git a/app/utils/TransportIndex.java b/app/utils/TransportIndex.java index a54d84c..baa7651 100644 --- a/app/utils/TransportIndex.java +++ b/app/utils/TransportIndex.java @@ -57,7 +57,8 @@ public STRtree getIndexForGraph(String graphId) { if (Edge instanceof PlainStreetEdge.class) }*/ - STRtree spatialIndex = new STRtree(segments.size()); + // R-trees have to have a minimum of two nodes + STRtree spatialIndex = new STRtree(Math.max(segments.size(), 2)); for(TransitSegment ts : segments) { spatialIndex.insert(ts.geom.getEnvelopeInternal(), ts); diff --git a/app/views/main.scala.html b/app/views/main.scala.html index fadde64..ee01ef1 100644 --- a/app/views/main.scala.html +++ b/app/views/main.scala.html @@ -11,7 +11,7 @@ - + @@ -37,8 +37,6 @@ - - diff --git a/conf/application.conf.template b/conf/application.conf.template index 8a273f4..9a2f38f 100644 --- a/conf/application.conf.template +++ b/conf/application.conf.template @@ -34,6 +34,7 @@ cluster.work-offline=true # akka { # actor { # provider = "akka.remote.RemoteActorRefProvider" +# guardian-supervisor-strategy = "com.conveyal.otpac.RootSupervisionStrategy" # } # remote { # watch-failure-detector.acceptable-heartbeat-pause = 60 s @@ -41,6 +42,9 @@ cluster.work-offline=true # netty.tcp { # hostname = "127.0.0.1" # port = 2552 +# send-buffer-size = 256m +# receive-buffer-size = 256m +# maximum-frame-size = 32m # } # # # secure akka settings diff --git a/conf/routes b/conf/routes index 4145c65..c56eea0 100644 --- a/conf/routes +++ b/conf/routes @@ -57,23 +57,23 @@ DELETE /api/query/:id controllers.Api.deleteQuery(id GET /tile/shapefile controllers.Tiles.shape(shapefileId:String, x:Integer, y:Integer, z:Integer, attributeName:String ?= null) -GET /tile/spatial controllers.Tiles.spatial(shapefileId:String, x:Integer, y:Integer, z:Integer, selectedAttributes:String ) -GET /tile/surface controllers.Tiles.surface(surfaceId:Integer, shapefileId:String, attributeName:String, x:Integer, y:Integer, z:Integer, showIso:Boolean ?= false, showPoints:Boolean ?= true, timeLimit:Integer ?= 3600, minTime:Integer ?= null) -GET /tile/query controllers.Tiles.query(queryId:String, x:Integer, y:Integer, z:Integer, timeLimit:Integer ?= 3600, weightByShapefile:String ?= null, weightByAttribute:String ?= null, groupBy:String ?= null, which: String, compareTo:String ?= null) +GET /tile/spatial controllers.Tiles.spatial(shapefileId:String, x:Integer, y:Integer, z:Integer, selectedAttributes:String ) +GET /tile/surface controllers.Tiles.surface(surfaceId:Integer, shapefileId:String, x:Integer, y:Integer, z:Integer, showIso:Boolean ?= false, showPoints:Boolean ?= true, timeLimit:Integer ?= 3600, minTime:Integer ?= null) +GET /tile/query controllers.Tiles.query(queryId:String, x:Integer, y:Integer, z:Integer, timeLimit:Integer ?= 3600, weightByShapefile:String ?= null, weightByAttribute:String ?= null, groupBy:String ?= null, which: String, attributeName:String, compareTo:String ?= null) GET /tile/transit controllers.Tiles.transit(scenarioId:String, x:Integer, y:Integer, z:Integer) GET /tile/transitComparison controllers.Tiles.transitComparison(scenarioId1:String, scenarioId2:String, x:Integer, y:Integer, z:Integer) -GET /tile/surfaceComparison controllers.Tiles.surfaceComparison(surfaceId1:Integer, surfaceId2:Integer, shapefileId:String, attributeName:String, x:Integer, y:Integer, z:Integer, showIso:Boolean ?= false, showPoints:Boolean ?= false, timeLimit:Integer ?= 3600, minTime:Integer ?= null) +GET /tile/surfaceComparison controllers.Tiles.surfaceComparison(surfaceId1:Integer, surfaceId2:Integer, shapefileId:String, x:Integer, y:Integer, z:Integer, showIso:Boolean ?= false, showPoints:Boolean ?= false, timeLimit:Integer ?= 3600, minTime:Integer ?= null) -GET /gis/query controllers.Gis.query(queryId:String, timeLimit:Integer ?= 3600, weightByShapefile:String ?= null, weightByAttribute:String ?= null, groupBy:String ?= null, which: String, compareTo:String ?= null) -GET /gis/surface controllers.Gis.surface(surfaceId:Integer, shapefileId:String, attributeName:String, timeLimit:Integer ?= 3600, compareTo:String ?= null) +GET /gis/query controllers.Gis.query(queryId:String, timeLimit:Integer ?= 3600, weightByShapefile:String ?= null, weightByAttribute:String ?= null, groupBy:String ?= null, which: String, attributeName: String, compareTo:String ?= null) +GET /gis/surface controllers.Gis.surface(surfaceId:Integer, shapefileId:String, timeLimit:Integer ?= 3600, compareTo:String ?= null) -GET /api/queryBins controllers.Api.queryBins(queryId:String, timeLimit:Integer ?= 3600, weightByShapefile:String ?= null, weightByAttribute:String ?= null, groupBy:String ?= null, which: String, compareTo: String ?= null) +GET /api/queryBins controllers.Api.queryBins(queryId:String, timeLimit:Integer ?= 3600, weightByShapefile:String ?= null, weightByAttribute:String ?= null, groupBy:String ?= null, which: String, attributeName: String, compareTo: String ?= null) -GET /api/surface controllers.Api.surface(graphId:String, lat:Double, lon:Double, mode:String, bikeSpeed:Double, walkSpeed:Double, which:String, date:String, fromTime:Integer, toTime:Integer ?= -1) -GET /api/isochrone controllers.Api.isochrone(surfaceId:Integer, cutoffs:java.util.List[Integer]) -GET /api/result controllers.Api.result(surfaceId:Integer, shapefileId:String, attributeName:String) +GET /api/surface controllers.Api.surface(graphId:String, lat:Double, lon:Double, mode:String, bikeSpeed:Double, walkSpeed:Double, which:String, date:String, fromTime:Integer, toTime:Integer ?= -1) +GET /api/isochrone controllers.Api.isochrone(surfaceId:Integer, cutoffs:java.util.List[Integer]) +GET /api/result controllers.Api.result(surfaceId:Integer, shapefileId:String) # Map static resources from the /public folder to the /assets URL path diff --git a/lib/otpa-cluster.jar b/lib/otpa-cluster.jar index b4c040e..b527d7a 100644 Binary files a/lib/otpa-cluster.jar and b/lib/otpa-cluster.jar differ diff --git a/public/javascripts/analysis-multi.js b/public/javascripts/analysis-multi.js index 7e18cfa..d368c95 100644 --- a/public/javascripts/analysis-multi.js +++ b/public/javascripts/analysis-multi.js @@ -9,8 +9,7 @@ var Analyst = Analyst || {}; events: { 'click #createQuery': 'createQuery', 'click #cancelQuery': 'cancelQuery', - 'click #newQuery': 'newQuery', - 'change #shapefile': 'updateAttributes' + 'click #newQuery': 'newQuery' }, regions: { @@ -18,7 +17,7 @@ var Analyst = Analyst || {}; }, initialize: function(options) { - _.bindAll(this, 'createQuery', 'cancelQuery', 'updateAttributes'); + _.bindAll(this, 'createQuery', 'cancelQuery'); }, onRender: function() { @@ -93,10 +92,8 @@ var Analyst = Analyst || {}; .attr('value', shp.id) .text(shp.get('name')) .appendTo(this.$('#shapefile')); - }); - - _this.updateAttributes(); }); + }); // pick a reasonable default date $.get('api/project/' + A.app.selectedProject + '/exemplarDay') @@ -130,26 +127,6 @@ var Analyst = Analyst || {}; this.main.show(queryListLayout); }, - /** - * Update the attributes select to show the attributes of the current shapefile - */ - updateAttributes: function () { - var shpId = this.$('#shapefile').val(); - var shp = this.shapefiles.get(shpId); - var _this = this; - - this.$('#shapefileColumn').empty(); - - shp.getNumericAttributes().forEach(function (attr) { - var atName = A.models.Shapefile.attributeName(attr); - - $('