forked from mapbox/tile-reduce
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
added a readme and described example use case
- Loading branch information
Jennings Anderson
committed
Oct 9, 2018
1 parent
ee45fc7
commit 67cc4aa
Showing
20 changed files
with
129 additions
and
1,930 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
# Stream-Reduce | ||
|
||
This is a simplified framework built off Mapbox's [tile-reduce](//github.com/mapbox/tile-reduce) to perform map-reduce functions against large files of line-delimited JSON. This simply removes all of the `tile` processing and instead passes each new line in the file to the map script. | ||
|
||
This is currently used for geometry-reconstruction of historical OSM objects as output from the [OSM-Wayback](//github.com/osmlab/osm-wayback) utility | ||
|
||
### Example Implementation | ||
|
||
#### File: `example/index.js` | ||
|
||
var count = 0; | ||
|
||
streamReduce({ | ||
map: path.join(__dirname, 'map.js'), //Map function | ||
file: path.join(__dirname, 'file.jsonseq'), //Input file (lines of JSON) | ||
maxWorkers:10 // The number of cpus you'd like to use | ||
}) | ||
.on('reduce', function(res) { | ||
count+=res | ||
}) | ||
.on('end', function() { | ||
console.log("Finished with count value: "+count) | ||
}); | ||
|
||
#### File: `example/map.js` | ||
|
||
module.exports = function(line, writeData, done) { | ||
var object = JSON.parse(line.toString()); | ||
done(object.count) | ||
}) | ||
|
||
#### file.jsonseq | ||
|
||
{"count":10} | ||
{"count":10} | ||
{"count":10} | ||
{"count":10} | ||
{"count":10} | ||
|
||
#### Use | ||
$ node examples/index.js | ||
|
||
>>Starting up 2 workers... Job started. | ||
>>Processing lines from file: examples/file.jsonseq | ||
>>5 lines processed in 0s. | ||
>>Finished, value of count: 50 | ||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
{"count":10} | ||
{"count":10} | ||
{"count":10} | ||
{"count":10} | ||
{"count":10} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
var streamReduce = require('../src'); | ||
var path = require('path') | ||
|
||
var count = 0; | ||
|
||
streamReduce({ | ||
map: path.join(__dirname, 'map.js'), | ||
file: path.join(__dirname, 'file.jsonseq'), | ||
maxWorkers:5 // The number of cpus you'd like to use | ||
}) | ||
.on('reduce', function(res) { | ||
count+=res | ||
}) | ||
.on('end', function() { | ||
console.log("Finished, value of count: "+count) | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
'use strict'; | ||
|
||
module.exports = function(line, writeData, done) { | ||
|
||
var object = JSON.parse(line.toString()); | ||
|
||
done(null, object.count) | ||
} |
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.