Persist patterns

src/main/java/com/conveyal/gtfs/PatternBuilder.java

@@ -40,14 +40,10 @@ public PatternBuilder(Feed feed) throws SQLException {
         connection = feed.getConnection();
     }
 
-    private String getTableName(String tableName) {
-        return feed.tablePrefix + tableName;
-    }
-
     public void create(Map<TripPatternKey, Pattern> patterns, Set<String> patternIdsLoadedFromFile) {
-        String patternsTableName = getTableName("patterns");
-        String tripsTableName = getTableName("trips");
-        String patternStopsTableName = getTableName("pattern_stops");
+        String patternsTableName = feed.getTableName("patterns");
+        String tripsTableName = feed.getTableName("trips");
+        String patternStopsTableName = feed.getTableName("pattern_stops");
 
         Table patternsTable = new Table(patternsTableName, Pattern.class, Requirement.EDITOR, Table.PATTERNS.fields);
         Table patternStopsTable = new Table(patternStopsTableName, PatternStop.class, Requirement.EDITOR, Table.PATTERN_STOP.fields);
@@ -62,9 +58,10 @@ public void create(Map<TripPatternKey, Pattern> patterns, Set<String> patternIds
                 patternsTable.createSqlTable(connection, null, true);
             }
             patternStopsTable.createSqlTable(connection, null, true);
-            PrintStream patternForTripsFileStream = createTempPatternForTripsTable(tempPatternForTripsTextFile, statement);
-            processPatternAndPatternStops(patternsTable, patternStopsTable, patternForTripsFileStream, patterns, patternIdsLoadedFromFile);
-            updateTripPatternIds(tempPatternForTripsTextFile, patternForTripsFileStream, statement, tripsTableName);
+            try (PrintStream patternForTripsFileStream = createTempPatternForTripsTable(tempPatternForTripsTextFile, statement)) {
+                processPatternAndPatternStops(patternsTable, patternStopsTable, patternForTripsFileStream, patterns, patternIdsLoadedFromFile);
+            }
+            updateTripPatternIds(tempPatternForTripsTextFile, statement, tripsTableName);
             createIndexes(statement, patternsTableName, patternStopsTableName, tripsTableName);
             connection.commit();
         } catch (SQLException | IOException e) {
@@ -143,13 +140,11 @@ private void updateTripPatternReferences(PrintStream patternForTripsFileStream,
      */
     private void updateTripPatternIds(
         File tempPatternForTripsTextFile,
-        PrintStream patternForTripsFileStream,
         Statement statement,
         String tripsTableName
     ) throws SQLException, IOException {
 
         LOG.info("Updating trips with pattern IDs.");
-        patternForTripsFileStream.close();
         // Copy file contents into temp pattern for trips table.
         copyFromFile(connection, tempPatternForTripsTextFile, TEMP_FILE_NAME);
         // Before updating the trips with pattern IDs, index the table on trip_id.

src/main/java/com/conveyal/gtfs/loader/Feed.java

@@ -26,9 +26,9 @@ public class Feed {
 
     private final DataSource dataSource;
 
-    // The unique database schema name for this particular feed, including the separator charater (dot).
+    // The unique database schema name for this particular feed, including the separator character (dot).
     // This may be the empty string if the feed is stored in the root ("public") schema.
-    public final String tablePrefix;
+    private final String tablePrefix;
 
     public final TableReader<Agency>        agencies;
     public final TableReader<Calendar>      calendars;
@@ -154,4 +154,10 @@ public Connection getConnection() throws SQLException {
         return dataSource.getConnection();
     }
 
+    /**
+     * @return the table name prefixed with this feed's database schema.
+     */
+    public String getTableName(String tableName) {
+        return tablePrefix + tableName;
+    }
 }

src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java

@@ -408,7 +408,7 @@ private TableLoadResult export (Table table, String filterSql) {
             }
 
             // Create entry for table
-            String textFileName = table.getFileName();
+            String textFileName = Table.getTableFileName(table.name);
             ZipEntry zipEntry = new ZipEntry(textFileName);
             zipOutputStream.putNextEntry(zipEntry);
 

src/main/java/com/conveyal/gtfs/loader/JdbcGtfsLoader.java

@@ -323,7 +323,7 @@ private int getTableSize(Table table) {
     private int loadInternal(Table table) throws Exception {
         CsvReader csvReader = table.getCsvReader(zip, errorStorage);
         if (csvReader == null) {
-            LOG.info("File {} not found in gtfs zip file.", table.getFileName());
+            LOG.info("File {} not found in gtfs zip file.", Table.getTableFileName(table.name));
             // This GTFS table could not be opened in the zip, even in a subdirectory.
             if (table.isRequired()) errorStorage.storeError(NewGTFSError.forTable(table, MISSING_TABLE));
             return 0;

src/main/java/com/conveyal/gtfs/loader/Table.java

@@ -104,10 +104,7 @@ public class Table {
     private String[] primaryKeyNames;
 
     /** Prefixed to exported tables/files that are not part of the GTFS spec. */
-    public static String proprietaryFilePrefix = "datatools_";
-
-    /** Identifies which tables on export are proprietary and should have the proprietary prefix applied. */
-    public boolean isProprietaryTable = false;
+    public static final String PROPRIETARY_FILE_PREFIX = "datatools_";
 
     public Table (String name, Class<? extends Entity> entityClass, Requirement required, Field... fields) {
         // TODO: verify table name is OK for use in constructing dynamic SQL queries
@@ -260,8 +257,7 @@ public Table (String name, Class<? extends Entity> entityClass, Requirement requ
             new StringField("shape_id", OPTIONAL).isReferenceTo(SHAPES)
     )
     .addPrimaryKey()
-    .addPrimaryKeyNames("pattern_id")
-    .hasProprietaryFilePrefix();
+    .addPrimaryKeyNames("pattern_id");
 
     public static final Table STOPS = new Table("stops", Stop.class, REQUIRED,
         new StringField("stop_id", REQUIRED),
@@ -495,15 +491,6 @@ public Table addPrimaryKeyNames(String ...keys) {
         return this;
     }
 
-    /**
-     * Fluent method that defines tables that on export should have the proprietary prefixed value applied to the file
-     * name.
-     */
-    public Table hasProprietaryFilePrefix() {
-        this.isProprietaryTable = true;
-        return this;
-    }
-
     /**
      * Get field names that are primary keys for a table.
      */
@@ -635,11 +622,13 @@ public String generateInsertSql (String namespace, boolean setDefaultId) {
         );
     }
 
-    public String getFileName() {
-        String fileName = this.name + ".txt";
-        return this.isProprietaryTable
-            ? Table.proprietaryFilePrefix + fileName
-            : fileName;
+    /**
+     * Proprietary table file names are prefix with "datatools_" to distinguish them from GTFS spec files.
+     */
+    public static String getTableFileName(String tableName) {
+        return (tableName.equals("patterns"))
+            ? String.format("%s%s.txt", PROPRIETARY_FILE_PREFIX, tableName)
+            : tableName + ".txt";
     }
 
     /**
@@ -649,7 +638,7 @@ public String getFileName() {
      * It then creates a CSV reader for that table if it's found.
      */
     public CsvReader getCsvReader(ZipFile zipFile, SQLErrorStorage sqlErrorStorage) {
-        final String tableFileName = getFileName();
+        final String tableFileName = getTableFileName(this.name);
         ZipEntry entry = zipFile.getEntry(tableFileName);
         if (entry == null) {
             // Table was not found, check if it is in a subdirectory.

src/main/java/com/conveyal/gtfs/model/Pattern.java

@@ -1,7 +1,6 @@
 package com.conveyal.gtfs.model;
 
 import com.conveyal.gtfs.GTFSFeed;
-import com.conveyal.gtfs.error.NoAgencyInFeedError;
 import com.conveyal.gtfs.loader.Table;
 import com.google.common.base.Joiner;
 import org.locationtech.jts.geom.LineString;
@@ -114,7 +113,7 @@ public Pattern () {}
     public static class Loader extends Entity.Loader<Pattern> {
 
         public Loader(GTFSFeed feed) {
-            super(feed, Table.proprietaryFilePrefix + "patterns");
+            super(feed, Table.PROPRIETARY_FILE_PREFIX + "patterns");
         }
 
         @Override
@@ -142,7 +141,7 @@ public void loadOneRow() throws IOException {
 
     public static class Writer extends Entity.Writer<Pattern> {
         public Writer (GTFSFeed feed) {
-            super(feed, Table.proprietaryFilePrefix + "patterns");
+            super(feed, Table.PROPRIETARY_FILE_PREFIX + "patterns");
         }
 
         @Override

src/main/java/com/conveyal/gtfs/validator/ServiceValidator.java

@@ -268,7 +268,7 @@ select durations.service_id, duration_seconds, days_active from (
             // Except that some service IDs may have no trips on them, or may not be referenced in any calendar or
             // calendar exception, which would keep them from appearing in either of those tables. So we just create
             // this somewhat redundant materialized view to serve as a master list of all services.
-            String servicesTableName = feed.tablePrefix + "services";
+            String servicesTableName = feed.getTableName("services");
             String sql = String.format("create table %s (service_id varchar, n_days_active integer, duration_seconds integer, n_trips integer)", servicesTableName);
             LOG.info(sql);
             statement.execute(sql);
@@ -285,7 +285,7 @@ select durations.service_id, duration_seconds, days_active from (
             serviceTracker.executeRemaining();
 
             // Create a table that shows on which dates each service is active.
-            String serviceDatesTableName = feed.tablePrefix + "service_dates";
+            String serviceDatesTableName = feed.getTableName("service_dates");
             sql = String.format("create table %s (service_date varchar, service_id varchar)", serviceDatesTableName);
             LOG.info(sql);
             statement.execute(sql);
@@ -318,7 +318,7 @@ select durations.service_id, duration_seconds, days_active from (
             // where dates.service_id = durations.service_id
             // group by service_date, route_type order by service_date, route_type;
 
-            String serviceDurationsTableName = feed.tablePrefix + "service_durations";
+            String serviceDurationsTableName = feed.getTableName("service_durations");
             sql = String.format("create table %s (service_id varchar, route_type integer, " +
                                     "duration_seconds integer, primary key (service_id, route_type))", serviceDurationsTableName);
             LOG.info(sql);

src/test/java/com/conveyal/gtfs/GTFSTest.java

@@ -5,6 +5,7 @@
 import com.conveyal.gtfs.loader.FeedLoadResult;
 import com.conveyal.gtfs.loader.JdbcGtfsExporter;
 import com.conveyal.gtfs.loader.SnapshotResult;
+import com.conveyal.gtfs.loader.Table;
 import com.conveyal.gtfs.storage.ErrorExpectation;
 import com.conveyal.gtfs.storage.ExpectedFieldType;
 import com.conveyal.gtfs.storage.PersistenceExpectation;
@@ -997,15 +998,6 @@ private static int countValidationErrorsOfType(
         return errorCount;
     }
 
-    /**
-     * Proprietary table file names are prefix with "datatools_" to distinguish them from GTFS spec files.
-     */
-    private String getTableFileName(String tableName) {
-        return (tableName.equals("patterns"))
-            ? String.format("datatools_%s.txt", tableName)
-            : tableName + ".txt";
-    }
-
     /**
      * Helper to assert that the GTFS that was exported to a zip file matches all data expectations defined in the
      * persistence expectations.
@@ -1024,7 +1016,7 @@ private void assertThatExportedGtfsMeetsExpectations(
             if (persistenceExpectation.appliesToEditorDatabaseOnly) continue;
             // No need to check that errors were exported because it is an internal table only.
             if ("errors".equals(persistenceExpectation.tableName)) continue;
-            final String tableFileName = getTableFileName(persistenceExpectation.tableName);
+            final String tableFileName = Table.getTableFileName(persistenceExpectation.tableName);
             LOG.info(String.format("reading table: %s", tableFileName));
 
             ZipEntry entry = gtfsZipfile.getEntry(tableFileName);