Skip to content

Commit

Permalink
it works, and it's wayyy fast
Browse files Browse the repository at this point in the history
  • Loading branch information
Jennings Anderson committed Oct 9, 2018
1 parent 15c63cf commit ee45fc7
Show file tree
Hide file tree
Showing 6 changed files with 90 additions and 62 deletions.
2 changes: 1 addition & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ node_modules
*.log
*.mbtiles
test/
out
out*
77 changes: 37 additions & 40 deletions geometry-reconstruction/map-geom-reconstruction.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,31 @@ var WayGeometryBuilder = require('./way-history-builder.js')
var NodeGeometryBuilder = require('./node-history-builder.js')
var RelationGeometryBuilder = require('./relation-history-builder.js')

/*
* Helper function to reconstruct properties between major Versions from diffs
*/
function reconstructMajorOSMTags(baseObject,newObject){
if (newObject.hasOwnProperty('aA') && newObject.aA){
Object.keys(newObject.aA).forEach(function(key){
baseObject[key] = newObject.aA[key]
})
}
if (newObject.hasOwnProperty('aM') && newObject.aM){
Object.keys(newObject.aM).forEach(function(key){
baseObject[key] = newObject.aM[key][1]
})
}
if (newObject.hasOwnProperty('aD') && newObject.aD){
Object.keys(newObject.aD).forEach(function(key){
delete baseObject[key]
})
}
return baseObject
}

const DEBUG = true;

//TODO: abstract this out to INDEX
const CONFIG = {
//Choose this...
'GEOMETRY_ONLY' : false, //Only @validSince, @validUntil on ALL objects
Expand All @@ -25,24 +48,22 @@ const CONFIG = {
}

module.exports = function(line, writeData, done) {

var status = {
linesProcessed : false,
lineProcessed : false,
noHistory : false,
jsonParsingError : false,
noNodeLocations : 0,
geometryBuilderFailedToDefine : false,
totalGeometries : false,
emptyLines : false,
totalGeometries : 0,
processLineFailures : false,
topoJSONEncodingError : false,
allGeometriesByteSize :0,
historyCompleteSingleObjectByteSize :0,
topojsonHistoryByteSize :0,
string : ""
topojsonHistoryByteSize :0
}

var geometryBuilder;
var string

try{
var object = JSON.parse(line.toString());
Expand Down Expand Up @@ -73,9 +94,9 @@ module.exports = function(line, writeData, done) {
})
}else{
status.noNodeLocations++;
return false
}

//if Geometry Builder was defined, keep going!
if (geometryBuilder){
/* Populates geometryBuilder.historicalGeometries object:
historicalGeometries = {
Expand Down Expand Up @@ -180,15 +201,15 @@ module.exports = function(line, writeData, done) {
if (CONFIG.WRITE_EVERY_GEOMETRY){
string = JSON.stringify(thisVersion)
allGeometriesByteSize += string.length;
console.log(string)
writeData(string+"\n")
}

newHistoryObject.push(thisVersion)

}
})//End @history.forEach();

totalGeometries += newHistoryObject.length;
status.totalGeometries += newHistoryObject.length;

//Fix up the history of the original object?
object.properties['@history'] = newHistoryObject;
Expand All @@ -212,8 +233,8 @@ module.exports = function(line, writeData, done) {

if(CONFIG.WRITE_HISTORY_COMPLETE_OBJECT){
string = JSON.stringify(object)
historyCompleteSingleObjectByteSize += string.length;
console.log(string)
status.historyCompleteSingleObjectByteSize += string.length;
writeData(string+"\n")
}

//Encode TopoJSON
Expand All @@ -230,11 +251,10 @@ module.exports = function(line, writeData, done) {
// console.log(hopeItWorked)

string = JSON.stringify(object)
console.log(string)
writeData(string+"\n")
status.topojsonHistoryByteSize += string.length;
}catch(e){
status.topoJSONEncodingError = true;
return false
}
}
}else{
Expand All @@ -244,40 +264,17 @@ module.exports = function(line, writeData, done) {
//TODO: add geometries even if there is no history?
}else{
//There was no history? log it and write the object back out.
noHistory++;
console.log(line)
status.noHistory = true;
writeData(line+"\n")
}
//If we got here, it all ran :)
// return true;
}

/**
* Helper function to reconstruct properties between major Versions from diffs
*/
function reconstructMajorOSMTags(baseObject,newObject){

if (newObject.hasOwnProperty('aA') && newObject.aA){
Object.keys(newObject.aA).forEach(function(key){
baseObject[key] = newObject.aA[key]
})
}
if (newObject.hasOwnProperty('aM') && newObject.aM){
Object.keys(newObject.aM).forEach(function(key){
baseObject[key] = newObject.aM[key][1]
})
}
if (newObject.hasOwnProperty('aD') && newObject.aD){
Object.keys(newObject.aD).forEach(function(key){
delete baseObject[key]
})
}
return baseObject
}

status.lineProcessed = true
}catch(err){
status.jsonParsingError = true
console.warn(err)
}


done(null, status);
}
0 geometry-reconstruction/index.js → geometry-reconstruction/old-index.js
View file
File renamed without changes.
35 changes: 30 additions & 5 deletions index-geom.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,40 @@ var fs = require('fs');
var split = require('binary-split');

var lines = 0;

var status = {
linesProcessed : 0,
noHistory : 0,
jsonParsingError : 0,
noNodeLocations : 0,
geometryBuilderFailedToDefine : 0,
totalGeometries : 0,
processLineFailures : 0,
topoJSONEncodingError : 0,
allGeometriesByteSize :0,
historyCompleteSingleObjectByteSize :0,
topojsonHistoryByteSize :0
}

streamReduce({
map: path.join(__dirname, 'geometry-reconstruction/map-geom-reconstruction.js'),
// sources: [{name: 'osm', mbtiles: path.join(__dirname, 'tmp.mbtiles'), raw: true}],
maxWorkers:2,
lineStream: fs.createReadStream(path.join(__dirname, 'test/tampa-test.history.geometries.head')).pipe(split())
// maxWorkers:2,
lineStream: fs.createReadStream(path.join(__dirname, 'test/tampa-test.history.geometries')).pipe(split())
})
.on('reduce', function(num) {
lines += num;
.on('reduce', function(res) {
if(res.lineProcessed) status.linesProcessed ++
if(res.noHistory) status.noHistory ++
if(res.jsonParsingError) status.jsonParsingError ++
if(res.geometryBuilderFailedToDefine) status.geometryBuilderFailedToDefine ++
if(res.processLineFailures) status.processLineFailures ++
if(res.topoJSONEncodingError) status.topoJSONEncodingError ++
status.noNodeLocations += res.noNodeLocations;
status.totalGeometries += res.totalGeometries;
status.allGeometriesByteSize += res.allGeometriesByteSize;
status.historyCompleteSingleObjectByteSize += res.historyCompleteSingleObjectByteSize;
status.topojsonHistoryByteSize += res.topojsonHistoryByteSize;
})
.on('end', function() {
console.log("DONE", lines)
console.warn(JSON.stringify(status,null,2))
});
37 changes: 21 additions & 16 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
"events": "^3.0.0",
"fork": "^1.3.1",
"fs": "0.0.1-security",
"lodash": "^4.17.11",
"path": "^0.12.7",
"queue-async": "^1.2.1",
"split": "^1.0.1",
Expand Down

0 comments on commit ee45fc7

Please sign in to comment.