Merge pull request #133 from linkedconnections/development

v2.1.0

bin/gtfs2lc.js

@@ -14,7 +14,6 @@ program
   .option('-s, --stream', 'Get the connections as a stream on the standard output')
   .option('-S, --store <store>', 'Store type: LevelStore (uses your disk to avoid that you run out of RAM) or MemStore (default)')
   .option('--fresh', 'Make sure to convert all Connection and ignore existing Historic records (which will be deleted)')
-  .option('-t --test', 'Flag that this is a test...nevermind this')
   .arguments('<path>', 'Path to sorted GTFS files')
   .action(function (path) {
     program.path = path;
@@ -44,8 +43,7 @@ var mapper = new gtfs2lc.Connections({
   store: !program.store || program.store === 'undefined' ? 'MemStore' : program.store,
   format: !program.format || program.format === 'undefined' ? 'json' : program.format,
   fresh: program.fresh,
-  baseUris: baseUris,
-  isTest: program.test
+  baseUris: baseUris
 });
 
 var resultStream = null;

lib/gtfs2connections.js

@@ -47,7 +47,7 @@ Mapper.prototype.resultStream = async function (path, output, done) {
 
    // Step 2: Read all the required GTFS files and create reusable indexes
    console.error('Creating index stores...');
-   const stores = await StoreManager(output, this._options.store, this._options.isTest);
+   const stores = await StoreManager(output, this._options.store);
 
    // Step 3: Produce (diff) connection rules based on available CPU cores
    console.error('Creating Connection rules...');
@@ -94,7 +94,7 @@ Mapper.prototype.resultStream = async function (path, output, done) {
             try {
                console.error('Merging final Linked Connections file...');
                // Join all resulting files into one
-               await exec(`for i in ${raws.map(r => {return `${r}.${ext}`}).join(" ")} ; do cat "$i" >> linkedConnections.${ext} && rm "$i" || break ; done`, { cwd: output });
+               await exec(`for i in ${raws.map(r => { return `${r}.${ext}` }).join(" ")} ; do cat "$i" >> linkedConnections.${ext} && rm "$i" || break ; done`, { cwd: output });
                let t1 = new Date();
                console.error('linkedConnections.' + ext + ' File created in ' + (t1.getTime() - t0.getTime()) + ' ms');
                await del(
@@ -125,7 +125,7 @@ Mapper.prototype.resultStream = async function (path, output, done) {
 async function cleanUpSources(sources) {
    try {
       await exec(`${path.resolve(`${__dirname}/../bin/gtfs2lc-clean.sh`)} ${sources}`);
-   } catch(err) {
+   } catch (err) {
       console.error(err);
       throw new Error('Process gtfs2lc-clean.sh exit with code: ' + code);
    }

lib/services/CalendarExpander.js

@@ -0,0 +1,70 @@
+/**
+ * Pieter Colpaert and Julián Rojas © Ghent University - imec 
+ * Make sure that the stop_times.txt is ordered by trip_id and stop_sequence before piping it to this library
+ */
+const Transform = require('stream').Transform;
+const { format, eachDayOfInterval } = require('date-fns');
+
+class CalendarExpander extends Transform {
+    constructor(calendarDates) {
+        super({ objectMode: true });
+        this._calendarDates = calendarDates;
+    }
+
+    _transform(calendar, encoding, done) {
+        // Parse and expand the calendar in memory
+        // GTFS specification declares a date as yyyyMMdd. No other formats possible. 
+        // Parsing with substr should be safe. Mind that timezones don’t matter here.
+        const startDate = this.createDate(calendar['start_date']);
+        const endDate = this.createDate(calendar['end_date']);
+        const days = eachDayOfInterval({ start: startDate, end: endDate });
+        const calDates = this.calendarDates.get(calendar['service_id']);
+        const expanded = new Set();
+
+        if (calDates) {
+            // Add already all added service dates
+            calDates.added.forEach(d => expanded.add(format(this.createDate(d), 'yyyyMMdd')));
+
+            for (const d of days) {
+                // Check this date is an actual service date and it hasn't been removed
+                if (calendar[format(d, 'iiii').toLowerCase()] === '1' 
+                    && !calDates.removed.has(d)) {
+                    expanded.add(format(d, 'yyyyMMdd'));
+                }
+            }
+            // Delete calendar_dates rule since is no longer needed
+            this.calendarDates.delete(calendar['service_id']);
+        } else {
+            // There are not additional service date rules for this calendar
+            for (const d of days) {
+                if (calendar[format(d, 'iiii').toLowerCase()] === '1') {
+                    expanded.add(format(d, 'yyyyMMdd'));
+                }
+            }
+        }
+
+        this.push({ 'service_id': calendar['service_id'], dates: Array.from(expanded) });
+        done();
+    }
+
+    _flush(done) {
+        // Deal with all the calendar_dates that didn't have a corresponding calendar rule
+        for(const [service_id, obj] of this.calendarDates) {
+            const dates = []
+            obj.added.forEach(d => dates.push(format(this.createDate(d), 'yyyyMMdd')));
+
+            this.push({ service_id, dates });
+        }
+        done();
+    }
+
+    createDate(dateString) {
+        return new Date(dateString.substr(0, 4), parseInt(dateString.substr(4, 2)) - 1, dateString.substr(6, 2));
+    }
+
+    get calendarDates() {
+        return this._calendarDates;
+    }
+}
+
+module.exports = CalendarExpander;

lib/stoptimes/StopTimes2Cxs.js

@@ -38,7 +38,7 @@ module.exports = function (sourcePath, outPath, stores, fresh) {
       ));
 
       connectionRules.on('error', err => {
-         console.error(err.message);
+         console.error(err);
          process.exit(-1);
       })
 

lib/stoptimes/st2c.js

@@ -107,7 +107,7 @@
 		 const arrivalStop = await this.stopsDB.get(arrival);
 		 const trip = await this.tripsDB.get(this.previousStopTime['trip_id']);
 		 const route = await this.routesDB.get(trip['route_id']);
-		 const serviceDates = await this.servicesDB.get(trip['service_id']);
+		 const serviceDates = await this.servicesDB.get(trip['service_id']) || [];
 		 return {
 			 departureStop,
 			 arrivalStop,