-
Notifications
You must be signed in to change notification settings - Fork 156
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Make composite tile #4
Changes from all commits
7cb47ac
180365a
e19651f
69ba157
dc410fc
5554957
36cb38e
5fb2426
d3f8ae8
68fd781
31bde95
6e0f9c7
434ed9a
ca69ae6
7187ece
1634430
aad3cdb
2eb4dea
b63c911
dec3c9e
d7a15aa
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,5 @@ | ||
module.exports = { | ||
gzipTileset : require('./lib/gzipTileset'), | ||
makeCompositeTile : require('./lib/makeCompositeTile'), | ||
pipeline : require('./lib/runPipeline') | ||
}; |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,18 +1,13 @@ | ||
'use strict'; | ||
var fsExtra = require('fs-extra'); | ||
var path = require('path'); | ||
var Promise = require('bluebird'); | ||
|
||
var fsExtraOutputFile = Promise.promisify(fsExtra.outputFile); | ||
var writeTile = require('./writeTile'); | ||
|
||
module.exports = getDefaultWriteCallback; | ||
|
||
/** | ||
* @private | ||
*/ | ||
function getDefaultWriteCallback(outputDirectory) { | ||
return function(file, data) { | ||
var outputFile = path.join(outputDirectory, file); | ||
return fsExtraOutputFile(outputFile, data); | ||
function getDefaultWriteCallback() { | ||
return function(file, data, options) { | ||
return writeTile(file, data, options); | ||
}; | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
'use strict'; | ||
var Cesium = require('cesium'); | ||
var Promise = require('bluebird'); | ||
var fsExtra = require('fs-extra'); | ||
var path = require('path'); | ||
|
||
var defaultValue = Cesium.defaultValue; | ||
|
||
var fsExtraReaddir = Promise.promisify(fsExtra.readdir); | ||
var fsExtraStat = Promise.promisify(fsExtra.stat); | ||
|
||
module.exports = getFilesInDirectory; | ||
|
||
function getFilesInDirectory(directory, options) { | ||
var files = []; | ||
options = defaultValue(options, defaultValue); | ||
var recursive = defaultValue(options.recursive, false); | ||
var filter = defaultValue(options.filter, function() { | ||
return true; | ||
}); | ||
return findFiles(directory, files, recursive, filter); | ||
} | ||
|
||
function findFiles(directory, files, recursive, filter) { | ||
return fsExtraReaddir(directory).map(function(fileName) { | ||
var fullPath = path.join(directory, fileName); | ||
return fsExtraStat(fullPath) | ||
.then(function(stats) { | ||
if (stats.isFile() && filter(fullPath)) { | ||
files.push(fullPath); | ||
} else if (recursive && stats.isDirectory()) { | ||
return findFiles(fullPath, files, recursive, filter); | ||
} | ||
}); | ||
}) | ||
.then(function() { | ||
return files; | ||
}); | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,18 +1,19 @@ | ||
'use strict'; | ||
var Cesium = require('cesium'); | ||
var fsExtra = require('fs-extra'); | ||
var path = require('path'); | ||
var Promise = require('bluebird'); | ||
var path = require('path'); | ||
var zlib = require('zlib'); | ||
var getDefaultWriteCallback = require('./getDefaultWriteCallback'); | ||
var isGzipped = require('./isGzipped'); | ||
|
||
var fsExtraReadFile = Promise.promisify(fsExtra.readFile); | ||
var getFilesInDirectory = require('./getFilesInDirectory'); | ||
var isTileFile = require('./isTileFile'); | ||
var readTile = require('./readTile'); | ||
|
||
var defaultValue = Cesium.defaultValue; | ||
var defined = Cesium.defined; | ||
var DeveloperError = Cesium.DeveloperError; | ||
|
||
var zlibGzip = Promise.promisify(zlib.gzip); | ||
|
||
module.exports = gzipTileset; | ||
|
||
/** | ||
|
@@ -40,100 +41,26 @@ function gzipTileset(options) { | |
outputDirectory = path.normalize(defaultValue(outputDirectory, | ||
path.join(path.dirname(inputDirectory), path.basename(inputDirectory) + '-' + (gzip ? 'gzipped' : 'ungzipped')))); | ||
|
||
var writeCallback = defaultValue(options.writeCallback, getDefaultWriteCallback(outputDirectory)); | ||
var writeCallback = defaultValue(options.writeCallback, getDefaultWriteCallback()); | ||
var logCallback = options.logCallback; | ||
|
||
if (defined(logCallback)) { | ||
logCallback((gzip ? 'Compressing' : 'Uncompressing') + ' files...'); | ||
} | ||
|
||
var operation = gzip ? zlib.gzipSync : zlib.gunzipSync; | ||
|
||
return new Promise(function(resolve, reject) { | ||
getNumberOfFilesInDirectory(inputDirectory) | ||
.then(function(numberOfFiles) { | ||
var writeFile = getWriteFile(writeCallback, numberOfFiles, resolve, reject); | ||
fsExtra.walk(inputDirectory) | ||
.on('data', function (item) { | ||
if (!item.stats.isDirectory()) { | ||
var inputFile = item.path; | ||
var file = path.relative(inputDirectory, item.path); | ||
|
||
if (gzip && tilesOnly && !isTile(inputFile)) { | ||
copyFile(inputFile, file, writeFile); | ||
} else { | ||
isGzipped(inputFile) | ||
.then(function(fileIsGzipped) { | ||
if (fileIsGzipped === gzip) { | ||
// File is already in the correct state | ||
copyFile(inputFile, file, writeFile); | ||
} else { | ||
fsExtraReadFile(inputFile) | ||
.then(function(data) { | ||
data = operation(data); | ||
writeFile(file, data); | ||
}) | ||
.catch(reject); | ||
} | ||
}) | ||
.catch(reject); | ||
} | ||
} | ||
}) | ||
.on('error', reject); | ||
}) | ||
.catch(reject); | ||
}); | ||
} | ||
|
||
function isTile(file) { | ||
var extension = path.extname(file); | ||
return extension === '.b3dm' || | ||
extension === '.i3dm' || | ||
extension === '.pnts' || | ||
extension === '.cmpt' || | ||
extension === '.vctr'; | ||
} | ||
|
||
function getNumberOfFilesInDirectory(directory) { | ||
return new Promise(function(resolve, reject) { | ||
var numberOfFiles = 0; | ||
fsExtra.walk(directory) | ||
.on('data', function (item) { | ||
if (!item.stats.isDirectory()) { | ||
++numberOfFiles; | ||
return getFilesInDirectory(inputDirectory, { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I used to do a similar approach here, but changed it for this comment: #3 (comment). I'm fine with merging the changes in this file in though. I have a branch that cleans up |
||
recursive: true | ||
}).map(function (filename) { | ||
var writeFile = path.join(outputDirectory, path.relative(inputDirectory, filename)); | ||
return readTile(filename) | ||
.then(function(data) { | ||
if (gzip && (!tilesOnly || isTileFile(writeFile))) { | ||
return zlibGzip(data); | ||
} | ||
return Promise.resolve(data); | ||
}) | ||
.on('end', function () { | ||
resolve(numberOfFiles); | ||
}) | ||
.on('error', reject); | ||
.then(function(data) { | ||
return writeCallback(writeFile, data); | ||
}); | ||
}); | ||
} | ||
|
||
function getWriteFile(writeCallback, numberOfFiles, resolve, reject) { | ||
var numberComplete = 0; | ||
function complete() { | ||
++numberComplete; | ||
if (numberComplete === numberOfFiles) { | ||
resolve(); | ||
} | ||
} | ||
return function(file, data) { | ||
var promise = writeCallback(file, data); | ||
if (defined(promise)) { | ||
promise | ||
.then(complete) | ||
.catch(reject); | ||
} else { | ||
complete(); | ||
} | ||
}; | ||
} | ||
|
||
function copyFile(inputFile, file, writeFile) { | ||
return fsExtraReadFile(inputFile) | ||
.then(function(data) { | ||
return writeFile(file, data); | ||
}); | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,17 +1,20 @@ | ||
'use strict'; | ||
var fsExtra = require('fs-extra'); | ||
var Promise = require('bluebird'); | ||
var Cesium = require('cesium'); | ||
|
||
var fsExtraReadFile = Promise.promisify(fsExtra.readFile); | ||
var DeveloperError = Cesium.DeveloperError; | ||
var defined = Cesium.defined; | ||
|
||
module.exports = isGzipped; | ||
|
||
/** | ||
* @private | ||
* Test if the provided data is gzipped. | ||
* | ||
* @param {Buffer} data A buffer containing the data to test. | ||
* @returns {Boolean} True if the data is gzipped, False if not. | ||
*/ | ||
function isGzipped(file) { | ||
return fsExtraReadFile(file) | ||
.then(function (data) { | ||
return (data[0] === 0x1f) && (data[1] === 0x8b); | ||
}); | ||
} | ||
function isGzipped(data) { | ||
if (!defined(data)) { | ||
throw new DeveloperError('data must be defined.'); | ||
} | ||
return data[0] === 0x1f && data[1] === 0x8b; | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
'use strict'; | ||
var Promise = require('bluebird'); | ||
var fsExtra = require('fs-extra'); | ||
var isGzipped = require('./isGzipped'); | ||
|
||
var fsExtraReadFile = Promise.promisify(fsExtra.readFile); | ||
|
||
module.exports = isGzippedFile; | ||
|
||
/** | ||
* @private | ||
*/ | ||
function isGzippedFile(file) { | ||
return fsExtraReadFile(file) | ||
.then(function (data) { | ||
return isGzipped(data); | ||
}); | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
'use strict'; | ||
var path = require('path'); | ||
|
||
module.exports = isTileFile; | ||
|
||
function isTileFile(file) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Mark as @Private |
||
var extension = path.extname(file); | ||
return extension === '.b3dm' || | ||
extension === '.i3dm' || | ||
extension === '.pnts' || | ||
extension === '.cmpt' || | ||
extension === '.vctr'; | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
'use strict'; | ||
module.exports = makeCompositeTile; | ||
|
||
/** | ||
* Combines an array of tile buffers into a single composite tile. | ||
* | ||
* @param {Buffer[]} tileBuffers An array of buffers holding tile data. | ||
* @returns {Buffer} A single buffer holding the composite tile. | ||
*/ | ||
function makeCompositeTile(tileBuffers) { | ||
var header = new Buffer(16); | ||
var buffers = []; | ||
buffers.push(header); | ||
var byteLength = header.length; | ||
for (var i = 0; i < tileBuffers.length; i++) { | ||
var tile = tileBuffers[i]; | ||
// Byte align all tiles to 4 bytes | ||
var tilePadding = tile.length % 4; | ||
if (tilePadding !== 0) { | ||
tile = Buffer.concat([tile, new Buffer(4 - tilePadding)]); | ||
} | ||
tile.writeUInt32LE(tile.length, 8); // byteLength | ||
byteLength += tile.length; | ||
buffers.push(tile); | ||
} | ||
header.write('cmpt', 0); // magic | ||
header.writeUInt32LE(1, 4); // version | ||
header.writeUInt32LE(byteLength, 8); // byteLength | ||
header.writeUInt32LE(tileBuffers.length, 12); // tilesLength | ||
return Buffer.concat(buffers); | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
'use strict'; | ||
var Cesium = require('cesium'); | ||
var Promise = require('bluebird'); | ||
var fsExtra = require('fs-extra'); | ||
var zlib = require('zlib'); | ||
var isGzipped = require('./isGzipped'); | ||
|
||
var DeveloperError = Cesium.DeveloperError; | ||
var defined = Cesium.defined; | ||
|
||
var fsExtraReadFile = Promise.promisify(fsExtra.readFile); | ||
var zlibGunzip = Promise.promisify(zlib.gunzip); | ||
|
||
module.exports = readTile; | ||
|
||
/** | ||
* Reads tile data from a file. | ||
* | ||
* @param {String} filePath The file path to read from. | ||
* @returns {Promise} A promise that resolves with the data when the read operation completes. | ||
*/ | ||
function readTile(filePath) { | ||
if (!defined(filePath)) { | ||
throw new DeveloperError('filePath must be defined'); | ||
} | ||
return fsExtraReadFile(filePath) | ||
.then(function(buffer) { | ||
if (isGzipped(buffer)) { | ||
return zlibGunzip(buffer); | ||
} | ||
return buffer; | ||
}); | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think it's best to just keep this really simple like it was before where it just gets data and write to a file. Gzipping or anything else should be handled by the calling code. Maybe this makes
writeTile
not needed anymore.