Merge pull request #141 from linkedconnections/development

v2.1.8

bin/gtfs2lc.js

@@ -11,6 +11,7 @@ program
   .option('-f, --format <format>', 'Format of the output. Possibilities: csv, n-triples, turtle, json, jsonld, mongo (extended JSON format to be used with mongoimport) or mongold (default: json)')
   .option('-b, --baseUris <baseUris>', 'Path to a file that describes the baseUris in json')
   .option('-o, --output <output>', 'Path to the folder where the result file will be stored')
+  .option('-c, --compressed', 'Compress resulting connections file using gzip')
   .option('-s, --stream', 'Get the connections as a stream on the standard output')
   .option('-S, --store <store>', 'Store type: LevelStore (uses your disk to avoid that you run out of RAM) or MemStore (default)')
   .option('--fresh', 'Make sure to convert all Connection and ignore existing Historic records (which will be deleted)')
@@ -42,6 +43,7 @@ if (program.baseUris) {
 var mapper = new gtfs2lc.Connections({
   store: !program.store || program.store === 'undefined' ? 'MemStore' : program.store,
   format: !program.format || program.format === 'undefined' ? 'json' : program.format,
+  compressed: program.compressed,
   fresh: program.fresh,
   baseUris: baseUris
 });

lib/gtfs2connections.js

@@ -53,7 +53,7 @@ Mapper.prototype.resultStream = async function (path, output, done) {
    console.error('Creating Connection rules...');
    await StopTimes2Cxs(path, output, stores, this._options.fresh);
 
-   // Step 4: Create connections in parallel using worker threads
+   // Step 4: Materialize connections in parallel using worker threads
    let w = 0;
    const raws = [];
    // Create as many worker threads as there are available CPUs
@@ -78,6 +78,8 @@ Mapper.prototype.resultStream = async function (path, output, done) {
             // Step 5: Merge all the created files into one
             const format = this._options.format;
             let ext = null;
+            let mergeCommand = 'zcat';
+
             if (!format || ['json', 'mongo', 'jsonld', 'mongold'].indexOf(format) >= 0) {
                await appendLineBreaks(output);
                ext = 'json';
@@ -92,9 +94,13 @@ Mapper.prototype.resultStream = async function (path, output, done) {
 
             try {
                console.error('Merging final Linked Connections file...');
-
+               if(this._options.compressed) {
+                  mergeCommand = 'cat';
+                  ext += '.gz';
+               }
+
                // Join all resulting files into one
-               await exec(`for i in ${raws.map(r => { return `${r}.${ext}.gz` }).join(" ")} ; do zcat "$i" >> linkedConnections.${ext} && rm "$i" || break ; done`, { cwd: output });
+               await exec(`for i in ${raws.map(r => { return `${r}.${ext}.gz` }).join(" ")} ; do ${mergeCommand} "$i" >> linkedConnections.${ext} && rm "$i" || break ; done`, { cwd: output });
                let t1 = new Date();
                console.error('linkedConnections.' + ext + ' File created in ' + (t1.getTime() - t0.getTime()) + ' ms');
                await del(

package.json

@@ -1,6 +1,6 @@
 {
   "name": "gtfs2lc",
-  "version": "2.1.7",
+  "version": "2.1.8",
   "description": "Mapping script from gtfs to (linked) connections",
   "main": "lib/gtfs2lc.js",
   "bin": {