Make composite tile

tools/index.js

@@ -1,4 +1,5 @@
 module.exports = {
     gzipTileset : require('./lib/gzipTileset'),
+    makeCompositeTile : require('./lib/makeCompositeTile'),
     pipeline : require('./lib/runPipeline')
 };

tools/lib/getDefaultWriteCallback.js

@@ -1,18 +1,13 @@
 'use strict';
-var fsExtra = require('fs-extra');
-var path = require('path');
-var Promise = require('bluebird');
-
-var fsExtraOutputFile = Promise.promisify(fsExtra.outputFile);
+var writeTile = require('./writeTile');
 
 module.exports = getDefaultWriteCallback;
 
 /**
  * @private
  */
-function getDefaultWriteCallback(outputDirectory) {
-    return function(file, data) {
-        var outputFile = path.join(outputDirectory, file);
-        return fsExtraOutputFile(outputFile, data);
+function getDefaultWriteCallback() {
+    return function(file, data, options) {
+        return writeTile(file, data, options);
     };
 }

tools/lib/getFilesInDirectory.js

@@ -0,0 +1,39 @@
+'use strict';
+var Cesium = require('cesium');
+var Promise = require('bluebird');
+var fsExtra = require('fs-extra');
+var path = require('path');
+
+var defaultValue = Cesium.defaultValue;
+
+var fsExtraReaddir = Promise.promisify(fsExtra.readdir);
+var fsExtraStat = Promise.promisify(fsExtra.stat);
+
+module.exports = getFilesInDirectory;
+
+function getFilesInDirectory(directory, options) {
+    var files = [];
+    options = defaultValue(options, defaultValue);
+    var recursive = defaultValue(options.recursive, false);
+    var filter = defaultValue(options.filter, function() {
+        return true;
+    });
+    return findFiles(directory, files, recursive, filter);
+}
+
+function findFiles(directory, files, recursive, filter) {
+    return fsExtraReaddir(directory).map(function(fileName) {
+        var fullPath = path.join(directory, fileName);
+        return fsExtraStat(fullPath)
+            .then(function(stats) {
+                if (stats.isFile() && filter(fullPath)) {
+                    files.push(fullPath);
+                } else if (recursive && stats.isDirectory()) {
+                    return findFiles(fullPath, files, recursive, filter);
+                }
+            });
+    })
+        .then(function() {
+            return files;
+        });
+}

tools/lib/gzipTileset.js

@@ -1,18 +1,19 @@
 'use strict';
 var Cesium = require('cesium');
-var fsExtra = require('fs-extra');
-var path = require('path');
 var Promise = require('bluebird');
+var path = require('path');
 var zlib = require('zlib');
 var getDefaultWriteCallback = require('./getDefaultWriteCallback');
-var isGzipped = require('./isGzipped');
-
-var fsExtraReadFile = Promise.promisify(fsExtra.readFile);
+var getFilesInDirectory = require('./getFilesInDirectory');
+var isTileFile = require('./isTileFile');
+var readTile = require('./readTile');
 
 var defaultValue = Cesium.defaultValue;
 var defined = Cesium.defined;
 var DeveloperError = Cesium.DeveloperError;
 
+var zlibGzip = Promise.promisify(zlib.gzip);
+
 module.exports = gzipTileset;
 
 /**
@@ -40,100 +41,26 @@ function gzipTileset(options) {
     outputDirectory = path.normalize(defaultValue(outputDirectory,
         path.join(path.dirname(inputDirectory), path.basename(inputDirectory) + '-' + (gzip ? 'gzipped' : 'ungzipped'))));
 
-    var writeCallback = defaultValue(options.writeCallback, getDefaultWriteCallback(outputDirectory));
+    var writeCallback = defaultValue(options.writeCallback, getDefaultWriteCallback());
     var logCallback = options.logCallback;
 
     if (defined(logCallback)) {
         logCallback((gzip ? 'Compressing' : 'Uncompressing') + ' files...');
     }
 
-    var operation = gzip ? zlib.gzipSync : zlib.gunzipSync;
-
-    return new Promise(function(resolve, reject) {
-        getNumberOfFilesInDirectory(inputDirectory)
-            .then(function(numberOfFiles) {
-                var writeFile = getWriteFile(writeCallback, numberOfFiles, resolve, reject);
-                fsExtra.walk(inputDirectory)
-                    .on('data', function (item) {
-                        if (!item.stats.isDirectory()) {
-                            var inputFile = item.path;
-                            var file = path.relative(inputDirectory, item.path);
-
-                            if (gzip && tilesOnly && !isTile(inputFile)) {
-                                copyFile(inputFile, file, writeFile);
-                            } else {
-                                isGzipped(inputFile)
-                                    .then(function(fileIsGzipped) {
-                                        if (fileIsGzipped === gzip) {
-                                            // File is already in the correct state
-                                            copyFile(inputFile, file, writeFile);
-                                        } else {
-                                            fsExtraReadFile(inputFile)
-                                                .then(function(data) {
-                                                    data = operation(data);
-                                                    writeFile(file, data);
-                                                })
-                                                .catch(reject);
-                                        }
-                                    })
-                                    .catch(reject);
-                            }
-                        }
-                    })
-                    .on('error', reject);
-            })
-            .catch(reject);
-    });
-}
-
-function isTile(file) {
-    var extension = path.extname(file);
-    return extension === '.b3dm' ||
-           extension === '.i3dm' ||
-           extension === '.pnts' ||
-           extension === '.cmpt' ||
-           extension === '.vctr';
-}
-
-function getNumberOfFilesInDirectory(directory) {
-    return new Promise(function(resolve, reject) {
-        var numberOfFiles = 0;
-        fsExtra.walk(directory)
-            .on('data', function (item) {
-                if (!item.stats.isDirectory()) {
-                    ++numberOfFiles;
+    return getFilesInDirectory(inputDirectory, {
+        recursive: true
+    }).map(function (filename) {
+        var writeFile = path.join(outputDirectory, path.relative(inputDirectory, filename));
+        return readTile(filename)
+            .then(function(data) {
+                if (gzip && (!tilesOnly || isTileFile(writeFile))) {
+                    return zlibGzip(data);
                 }
+                return Promise.resolve(data);
             })
-            .on('end', function () {
-                resolve(numberOfFiles);
-            })
-            .on('error', reject);
+            .then(function(data) {
+                return writeCallback(writeFile, data);
+            });
     });
 }
-
-function getWriteFile(writeCallback, numberOfFiles, resolve, reject) {
-    var numberComplete = 0;
-    function complete() {
-        ++numberComplete;
-        if (numberComplete === numberOfFiles) {
-            resolve();
-        }
-    }
-    return function(file, data) {
-        var promise = writeCallback(file, data);
-        if (defined(promise)) {
-            promise
-                .then(complete)
-                .catch(reject);
-        } else {
-            complete();
-        }
-    };
-}
-
-function copyFile(inputFile, file, writeFile) {
-    return fsExtraReadFile(inputFile)
-        .then(function(data) {
-            return writeFile(file, data);
-        });
-}

tools/lib/isGzipped.js

@@ -1,17 +1,20 @@
 'use strict';
-var fsExtra = require('fs-extra');
-var Promise = require('bluebird');
+var Cesium = require('cesium');
 
-var fsExtraReadFile = Promise.promisify(fsExtra.readFile);
+var DeveloperError = Cesium.DeveloperError;
+var defined = Cesium.defined;
 
 module.exports = isGzipped;
 
 /**
- * @private
+ * Test if the provided data is gzipped.
+ *
+ * @param {Buffer} data A buffer containing the data to test.
+ * @returns {Boolean} True if the data is gzipped, False if not.
  */
-function isGzipped(file) {
-    return fsExtraReadFile(file)
-        .then(function (data) {
-            return (data[0] === 0x1f) && (data[1] === 0x8b);
-        });
-}
+function isGzipped(data) {
+    if (!defined(data)) {
+        throw new DeveloperError('data must be defined.');
+    }
+    return data[0] === 0x1f && data[1] === 0x8b;
+}

tools/lib/isGzippedFile.js

@@ -0,0 +1,18 @@
+'use strict';
+var Promise = require('bluebird');
+var fsExtra = require('fs-extra');
+var isGzipped = require('./isGzipped');
+
+var fsExtraReadFile = Promise.promisify(fsExtra.readFile);
+
+module.exports = isGzippedFile;
+
+/**
+ * @private
+ */
+function isGzippedFile(file) {
+    return fsExtraReadFile(file)
+        .then(function (data) {
+            return isGzipped(data);
+        });
+}

tools/lib/isTileFile.js

@@ -0,0 +1,13 @@
+'use strict';
+var path = require('path');
+
+module.exports = isTileFile;
+
+function isTileFile(file) {
+    var extension = path.extname(file);
+    return extension === '.b3dm' ||
+        extension === '.i3dm' ||
+        extension === '.pnts' ||
+        extension === '.cmpt' ||
+        extension === '.vctr';
+}

tools/lib/makeCompositeTile.js

@@ -0,0 +1,31 @@
+'use strict';
+module.exports = makeCompositeTile;
+
+/**
+ * Combines an array of tile buffers into a single composite tile.
+ *
+ * @param {Buffer[]} tileBuffers An array of buffers holding tile data.
+ * @returns {Buffer} A single buffer holding the composite tile.
+ */
+function makeCompositeTile(tileBuffers) {
+    var header = new Buffer(16);
+    var buffers = [];
+    buffers.push(header);
+    var byteLength = header.length;
+    for (var i = 0; i < tileBuffers.length; i++) {
+        var tile = tileBuffers[i];
+        // Byte align all tiles to 4 bytes
+        var tilePadding = tile.length % 4;
+        if (tilePadding !== 0) {
+            tile = Buffer.concat([tile, new Buffer(4 - tilePadding)]);
+        }
+        tile.writeUInt32LE(tile.length, 8);       // byteLength
+        byteLength += tile.length;
+        buffers.push(tile);
+    }
+    header.write('cmpt', 0);                      // magic
+    header.writeUInt32LE(1, 4);                   // version
+    header.writeUInt32LE(byteLength, 8);          // byteLength
+    header.writeUInt32LE(tileBuffers.length, 12); // tilesLength
+    return Buffer.concat(buffers);
+}

tools/lib/readTile.js

@@ -0,0 +1,33 @@
+'use strict';
+var Cesium = require('cesium');
+var Promise = require('bluebird');
+var fsExtra = require('fs-extra');
+var zlib = require('zlib');
+var isGzipped = require('./isGzipped');
+
+var DeveloperError = Cesium.DeveloperError;
+var defined = Cesium.defined;
+
+var fsExtraReadFile = Promise.promisify(fsExtra.readFile);
+var zlibGunzip = Promise.promisify(zlib.gunzip);
+
+module.exports = readTile;
+
+/**
+ * Reads tile data from a file.
+ *
+ * @param {String} filePath The file path to read from.
+ * @returns {Promise} A promise that resolves with the data when the read operation completes.
+ */
+function readTile(filePath) {
+    if (!defined(filePath)) {
+        throw new DeveloperError('filePath must be defined');
+    }
+    return fsExtraReadFile(filePath)
+        .then(function(buffer) {
+            if (isGzipped(buffer)) {
+                return zlibGunzip(buffer);
+            }
+            return buffer;
+        });
+}

tools/lib/runPipeline.js

@@ -1,8 +1,8 @@
 'use strict';
 var Cesium = require('cesium');
+var Promise = require('bluebird');
 var fsExtra = require('fs-extra');
 var path = require('path');
-var Promise = require('bluebird');
 var getWorkingDirectory = require('./getWorkingDirectory');
 var gzipTileset = require('./gzipTileset');
 
@@ -90,22 +90,22 @@ function runPipeline(pipeline, options) {
         }
 
         stageObjects.push({
-            options : stageOptions,
-            stageFunction : stageFunction,
-            name : stageName
+            options: stageOptions,
+            stageFunction: stageFunction,
+            name: stageName
         });
     }
 
     // Run the stages in sequence
-    return Promise.each(stageObjects, function(stage) {
+    return Promise.each(stageObjects, function (stage) {
         return fsExtraEmptyDir(stage.options.outputDirectory)
-            .then(function() {
+            .then(function () {
                 if (defined(logCallback)) {
                     logCallback('Running ' + stage.name);
                 }
                 return stage.stageFunction(stage.options);
             });
-    }).finally(function() {
+    }).finally(function () {
         return Promise.all([
             fsExtraRemove(workingDirectory1),
             fsExtraRemove(workingDirectory2)