From b4d539cc75a6f8be7b8f20c5c5cc51f6dfb70971 Mon Sep 17 00:00:00 2001 From: animetosho Date: Wed, 1 Feb 2017 19:32:07 +1000 Subject: [PATCH] Add nexe build script --- README.md | 42 ++++++++ nexe/build.js | 254 +++++++++++++++++++++++++++++++++++++++++++ nexe/ncp.js | 291 ++++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 3 +- 4 files changed, 589 insertions(+), 1 deletion(-) create mode 100644 nexe/build.js create mode 100644 nexe/ncp.js diff --git a/README.md b/README.md index d85bf08..49ea5f4 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,9 @@ The following modules have been marked optional: - [xz](): enables NZBs to be compressed using xz via the `--nzb-compress xz` option +Development +=========== + Running Tests ------------- @@ -210,6 +213,45 @@ Note that some test cases test functionality of timeouts; to reduce time it takes to run these tests, timeouts are set relatively small, which means that a slow computer may not be able to service them as expected. +Building Binary +--------------- + +Compiling Nyuu into a single binary can be done via +[nexe](). There is a little complication with +bundling the *yencode *module, but a rather fragile script has been supplied in +*nexe/build.js* to help with the process. The following general steps need to be +taken: + +1. Ensure that *nexe* is installed (doesn’t need to be globally installed) and + [its requirements]() met + +2. Download a Node.js source package. The script has mostly been tested with + Node 4.7.x, it may work with other versions + +3. The required Nyuu libraries need to be installed into the *node\_modules* + folder + +4. Inside the *nexe* folder (the one containing *build.js*), create the + following two folders: *node* and *yencode-src* + +5. Inside the *node* folder, create a folder with the version number of the + package you downloaded in step 2, for example “4.7.2”. Inside *this* folder, + create one named “\_” and place the downloaded sources in this folder. After + doing this, the file *nexe/node/x.x.x/\_/node.gyp* should exist, where + *x.x.x* is the node version number + +6. Inside the *yencode-src* folder, copy the source code for the *yencode* + module + +7. Edit *nexe/build.js*; options that are likely to be edited are at the top of + the file. You’ll likely need to change *nodeVer* to be the version of node + you’re using + +8. In the *nexe* folder, run *build.js*. This script patches node to embed the + yencode module, and customises a few compiler options, then calls nexe to + build the final executable. If it worked, you should get a binary named + *nyuu* or *nyuu.exe* in the nexe folder + Usage ===== diff --git a/nexe/build.js b/nexe/build.js new file mode 100644 index 0000000..fa6b156 --- /dev/null +++ b/nexe/build.js @@ -0,0 +1,254 @@ +var nodeVer = '4.7.3'; +var nexeBase = '.'; +var nodeSrc = nexeBase + '/node/' + nodeVer + '/_/'; // TODO: auto search folder +var yencSrc = './yencode-src/'; +var python = 'python'; +var makeArgs = ["-j", "4"]; +var vcBuildArch = "x86"; // x86 or x64 + +var fs = require('fs'); +var ncp = require('./ncp').ncp; +var nexe = require('nexe'); + +var isNode010 = !!nodeVer.match(/^0\.10\./); +var modulePref = isNode010?'node_':''; +var yencodeCC = fs.readFileSync(yencSrc + 'yencode.cc').toString(); // trigger error if it doesn't exist + +var gypParse = function(gyp) { + // very hacky fixes for Python's flexibility + gyp = gyp.replace(/'(\s*\n\s*')/g, "' +$1"); + gyp = gyp.replace(/#[^'"]*?(\r?\n)/g, "$1"); + gyp = gyp.replace(/(\n\s*)#.*?(\r?\n)/g, "$1$2"); + gyp = gyp.replace(/(\n\s*)#.*?(\r?\n)/g, "$1$2"); + gyp = gyp.replace(/(\n\s*)#.*?(\r?\n)/g, "$1$2"); + gyp = gyp.replace(/(\n\s*)#.*?(\r?\n)/g, "$1$2"); + return eval('(' + gyp + ')'); +}; +// monkey patch node.gyp +var gypData = fs.readFileSync(nodeSrc + 'node.gyp').toString(); +var gyp = gypParse(gypData); + + +var findGypTarget = function(targ) { + for(var i in gyp.targets) + if(gyp.targets[i].target_name == targ) + return gyp.targets[i]; + return false; +}; + +// changing the GYP too much breaks nexe, so resort to monkey-patching it + +var doPatch = function(r, s) { + var m = gypData.match(r); + if(!m) throw new Error('Could not match ' + r); + if(gypData.substr(m.index+1).match(r)) + throw new Error('Expression matched >1 times: ' + r); + gypData = gypData.replace(r, '$1 ' + s); +}; +if(!findGypTarget('crcutil')) { + doPatch(/(\},\s*['"]targets['"]: \[)/, JSON.stringify({ + "target_name": "crcutil", + "type": "static_library", + "sources": [ + "crcutil-1.0/code/crc32c_sse4.cc", + "crcutil-1.0/code/multiword_64_64_cl_i386_mmx.cc", + "crcutil-1.0/code/multiword_64_64_gcc_amd64_asm.cc", + "crcutil-1.0/code/multiword_64_64_gcc_i386_mmx.cc", + "crcutil-1.0/code/multiword_64_64_intrinsic_i386_mmx.cc", + "crcutil-1.0/code/multiword_128_64_gcc_amd64_sse2.cc", + "crcutil-1.0/examples/interface.cc" + ], + "conditions": [ + ['OS=="win"', { + "msvs_settings": {"VCCLCompilerTool": {"EnableEnhancedInstructionSet": "2"}} + }, { + "cxxflags": ["-msse2", "-O3", "-fomit-frame-pointer"] + }] + ], + "include_dirs": ["crcutil-1.0/code", "crcutil-1.0/tests"], + "defines": ["CRCUTIL_USE_MM_CRC32=0"] + })+','); +} + +var tNode = findGypTarget('<(node_core_target_name)'); +var tNodeM = "['\"]target_name['\"]:\\s*['\"]<\\(node_core_target_name\\)['\"],"; +if(!tNode) { + tNode = findGypTarget('node'); + tNodeM = "['\"]target_name['\"]:\\s*['\"]node['\"],"; +} +var tNodeMatch = new RegExp('('+tNodeM+')'); +if(tNode.sources.indexOf('src/'+modulePref+'yencode.cc') < 0) + doPatch(/(['"]src\/node_file\.cc['"],)/, "'src/"+modulePref+"yencode.cc',"); +if(tNode.dependencies.indexOf('crcutil') < 0) + doPatch(/(['"]node_js2c#host['"],)/, "'crcutil',"); +if(tNode.include_dirs.indexOf('crcutil-1.0/code') < 0) + doPatch(/(['"]deps\/uv\/src\/ares['"],)/, "'crcutil-1.0/code', 'crcutil-1.0/examples',"); + +if(gyp.variables.library_files.indexOf('lib/yencode.js') < 0) + doPatch(/(['"]lib\/fs\.js['"],)/, "'lib/yencode.js',"); + + + +// urgh, copy+paste :/ +if(!tNode.msvs_settings) { + doPatch(tNodeMatch, "'msvs_settings': {'VCCLCompilerTool': {'EnableEnhancedInstructionSet': '2', 'FavorSizeOrSpeed': '2'}, 'VCLinkerTool': {'GenerateDebugInformation': 'false'}},"); +} else { + if(!tNode.msvs_settings.VCCLCompilerTool) { + doPatch(new RegExp("(" + tNodeM + "[^]*?['\"]msvs_settings['\"]:\\s*\\{)"), "'VCCLCompilerTool': {'EnableEnhancedInstructionSet': '2', 'FavorSizeOrSpeed': '2'},"); + } else if(!tNode.msvs_settings.VCCLCompilerTool.EnableEnhancedInstructionSet) { + doPatch(/(['"]VCCLCompilerTool['"]:\s*\{)/, "'EnableEnhancedInstructionSet': '2', 'FavorSizeOrSpeed': '2',"); + } + + if(!tNode.msvs_settings.VCLinkerTool) { + doPatch(new RegExp("(" + tNodeM + "[^]*?['\"]msvs_settings['\"]:\\s*\\{)"), "'VCLinkerTool': {'GenerateDebugInformation': 'false'},"); + } else if(!tNode.msvs_settings.VCLinkerTool.GenerateDebugInformation) { + doPatch(/(['"]VCLinkerTool['"]:\s*\{)/, "'GenerateDebugInformation': 'false',"); + } +} +if(!tNode.cxxflags) { + doPatch(tNodeMatch, "'cxxflags': ['-Os','-msse2','-static','-flto'],"); +} else if(tNode.cxxflags.indexOf('-Os') < 0) { + doPatch(new RegExp("(" + tNodeM + "[^]*?['\"]cxxflags['\"]:\\s*\\[)"), "'-Os','-msse2','-static','-flto',"); +} + +if(!tNode.ldflags) { + doPatch(tNodeMatch, "'ldflags': ['-s','-static','-flto'],"); +} else if(tNode.ldflags.indexOf('-s') < 0) { + doPatch(new RegExp("(" + tNodeM + "[^]*?['\"]ldflags['\"]:\\s*\\[)"), "'-s','-static','-flto',"); +} + + + +fs.writeFileSync(nodeSrc + 'node.gyp', gypData); + + +// patch manifest +var pkg = require('../package.json'); +var manif = fs.readFileSync(nodeSrc + 'src/res/node.rc').toString(); +manif = manif +.replace(/1 ICON node\.ico/, '') +.replace(/VALUE "CompanyName", "[^"]+"/, '') +.replace(/VALUE "ProductName", "[^"]+"/, 'VALUE "ProductName", "' + pkg.name + '"') +.replace(/VALUE "FileDescription", "[^"]+"/, 'VALUE "FileDescription", "' + pkg.description + '"') +.replace(/VALUE "FileVersion", NODE_EXE_VERSION/, 'VALUE "FileVersion", "' + pkg.version + '"') +.replace(/VALUE "ProductVersion", NODE_EXE_VERSION/, 'VALUE "ProductVersion", "' + pkg.version + '"') +.replace(/VALUE "InternalName", "[^"]+"/, 'VALUE "InternalName", "nyuu"'); +fs.writeFileSync(nodeSrc + 'src/res/node.rc', manif); + + +var patchGypCompiler = function(file, targets) { + // require SSE2; TODO: tweak this? + var gypData = fs.readFileSync(nodeSrc + file).toString(); + var gyp = gypParse(gypData); + + if(!gyp.target_defaults) { + targets = targets || 'targets'; + gypData = gypData.replace("'"+targets+"':", "'target_defaults': {'msvs_settings': {'VCCLCompilerTool': {'EnableEnhancedInstructionSet': '2', 'FavorSizeOrSpeed': '2'}, 'VCLinkerTool': {'GenerateDebugInformation': 'false'}}, 'cxxflags': ['-Os','-msse2','-static','-flto'], 'ldflags': ['-s','-static','-flto']}, '"+targets+"':"); + } else { + // TODO: other possibilities + if(!gyp.target_defaults.msvs_settings) + gypData = gypData.replace("'target_defaults': {", "'target_defaults': {'msvs_settings': {'VCCLCompilerTool': {'EnableEnhancedInstructionSet': '2', 'FavorSizeOrSpeed': '2'}, 'VCLinkerTool': {'GenerateDebugInformation': 'false'}},"); + else if(!gyp.target_defaults.msvs_settings.VCCLCompilerTool || !gyp.target_defaults.msvs_settings.VCLinkerTool || !gyp.target_defaults.msvs_settings.VCCLCompilerTool.EnableEnhancedInstructionSet) + throw new Error('To be implemented'); + if(!gyp.target_defaults.cxxflags) + gypData = gypData.replace("'target_defaults': {", "'target_defaults': {'cxxflags': ['-Os','-msse2','-static','-flto'],"); + else if(gyp.target_defaults.cxxflags.indexOf('-flto') < 0) + throw new Error('To be implemented'); + if(!gyp.target_defaults.ldflags) + gypData = gypData.replace("'target_defaults': {", "'target_defaults': {'ldflags': ['-s','-static','-flto'],"); + else if(gyp.target_defaults.ldflags.indexOf('-flto') < 0) + throw new Error('To be implemented'); + } + + fs.writeFileSync(nodeSrc + file, gypData); +}; +//patchGypCompiler('node.gyp'); +patchGypCompiler('deps/cares/cares.gyp'); +//patchGypCompiler('deps/http_parser/http_parser.gyp'); +patchGypCompiler('deps/openssl/openssl.gyp'); +patchGypCompiler('deps/uv/uv.gyp'); +patchGypCompiler('deps/zlib/zlib.gyp', 'conditions'); +if(fs.existsSync(nodeSrc + 'deps/v8/src/v8.gyp')) + patchGypCompiler('deps/v8/src/v8.gyp'); +else + patchGypCompiler('deps/v8/tools/gyp/v8.gyp'); + + + + +if(fs.existsSync(nodeSrc + 'src/node_extensions.h')) { // node 0.10.x + var ext = fs.readFileSync(nodeSrc + 'src/node_extensions.h').toString(); + ext = ext.replace('\nNODE_EXT_LIST_START', '\nNODE_EXT_LIST_START\nNODE_EXT_LIST_ITEM('+modulePref+'yencode)'); + fs.writeFileSync(nodeSrc + 'src/node_extensions.h', ext); +} + +// create embeddable help +fs.writeFileSync('../bin/help.json', JSON.stringify({ + full: fs.readFileSync('../help.txt').toString(), + short: fs.readFileSync('../help-short.txt').toString() +})); + + +// copy yencode sources across +if(isNode010) + yencodeCC = yencodeCC.replace('NODE_MODULE(yencode', 'NODE_MODULE('+modulePref+'yencode'); +else + yencodeCC = yencodeCC.replace('NODE_MODULE(', 'NODE_MODULE_CONTEXT_AWARE_BUILTIN('); +fs.writeFileSync(nodeSrc + 'src/'+modulePref+'yencode.cc', yencodeCC); + +var yencodeJs = fs.readFileSync(yencSrc + 'index.js').toString(); +yencodeJs = yencodeJs.replace(/require\(['"][^'"]*yencode\.node'\)/g, "process.binding('yencode')"); +fs.writeFileSync(nodeSrc + 'lib/yencode.js', yencodeJs); + +fs.readdirSync(yencSrc).forEach(function(f) { + if(f == 'yencode.cc' || f == 'index.js' || f.match(/^test/)) return; + + var dst; + if(f.match(/\.(c|cpp|cc|h)$/)) + dst = 'src/' + f; + else if(f.match(/\.js$/)) + dst = 'lib/' + f; + + if(dst) { + fs.writeFileSync(nodeSrc + dst, fs.readFileSync(yencSrc + f)); + } +}); +ncp(yencSrc + 'crcutil-1.0', nodeSrc + 'crcutil-1.0', function() { + + + // now run nexe + // TODO: consider building startup snapshot? + + nexe.compile({ + input: '../bin/nyuu.js', // where the input file is + output: './nyuu' + (require('os').platform() == 'win32' ? '.exe':''), // where to output the compiled binary + nodeVersion: nodeVer, // node version + nodeTempDir: nexeBase, // where to store node source. + nodeConfigureArgs: ['--fully-static', '--without-dtrace', '--without-etw', '--without-perfctr', '--without-npm', '--with-intl=none'], // for all your configure arg needs. + nodeMakeArgs: makeArgs, // when you want to control the make process. + nodeVCBuildArgs: ["nosign", vcBuildArch], // when you want to control the make process for windows. + // By default "nosign" option will be specified + // You can check all available options and its default values here: + // https://github.com/nodejs/node/blob/master/vcbuild.bat + python: python, // for non-standard python setups. Or python 3.x forced ones. + resourceFiles: [ ], // array of files to embed. + resourceRoot: [ ], // where to embed the resourceFiles. + flags: true, // use this for applications that need command line flags. + jsFlags: "", // v8 flags + startupSnapshot: '', // when you want to specify a script to be + // added to V8's startup snapshot. This V8 + // feature deserializes a heap to save startup time. + // More information in this blog post: + // http://v8project.blogspot.de/2015/09/custom-startup-snapshots.html + framework: "node", // node, nodejs, or iojs + + browserifyExcludes: ['yencode','xz','../node_modules/xz/package.json'] + }, function(err) { + if(err) { + return console.log(err); + } + + console.log('done'); + fs.unlinkSync('../bin/help.json'); + }); +}); diff --git a/nexe/ncp.js b/nexe/ncp.js new file mode 100644 index 0000000..17424af --- /dev/null +++ b/nexe/ncp.js @@ -0,0 +1,291 @@ +/** + * Shamefully stolen from ncp module, placed here for convenience + * License follows + +# MIT License + +###Copyright (C) 2011 by Charlie McConnell + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + + */ + + +var fs = require('fs'), + path = require('path'); + +module.exports = ncp; +ncp.ncp = ncp; + +function ncp (source, dest, options, callback) { + var cback = callback; + + if (!callback) { + cback = options; + options = {}; + } + + var basePath = process.cwd(), + currentPath = path.resolve(basePath, source), + targetPath = path.resolve(basePath, dest), + filter = options.filter, + rename = options.rename, + transform = options.transform, + clobber = options.clobber !== false, + modified = options.modified, + dereference = options.dereference, + errs = null, + started = 0, + finished = 0, + running = 0, + limit = options.limit || ncp.limit || 16; + + limit = (limit < 1) ? 1 : (limit > 512) ? 512 : limit; + + startCopy(currentPath); + + function startCopy(source) { + started++; + if (filter) { + if (filter instanceof RegExp) { + if (!filter.test(source)) { + return cb(true); + } + } + else if (typeof filter === 'function') { + if (!filter(source)) { + return cb(true); + } + } + } + return getStats(source); + } + + function getStats(source) { + var stat = dereference ? fs.stat : fs.lstat; + if (running >= limit) { + return setImmediate(function () { + getStats(source); + }); + } + running++; + stat(source, function (err, stats) { + var item = {}; + if (err) { + return onError(err); + } + + // We need to get the mode from the stats object and preserve it. + item.name = source; + item.mode = stats.mode; + item.mtime = stats.mtime; //modified time + item.atime = stats.atime; //access time + + if (stats.isDirectory()) { + return onDir(item); + } + else if (stats.isFile()) { + return onFile(item); + } + else if (stats.isSymbolicLink()) { + // Symlinks don't really need to know about the mode. + return onLink(source); + } + }); + } + + function onFile(file) { + var target = file.name.replace(currentPath, targetPath); + if(rename) { + target = rename(target); + } + isWritable(target, function (writable) { + if (writable) { + return copyFile(file, target); + } + if(clobber) { + rmFile(target, function () { + copyFile(file, target); + }); + } + if (modified) { + var stat = dereference ? fs.stat : fs.lstat; + stat(target, function(err, stats) { + //if souce modified time greater to target modified time copy file + if (file.mtime.getTime()>stats.mtime.getTime()) + copyFile(file, target); + else return cb(); + }); + } + else { + return cb(); + } + }); + } + + function copyFile(file, target) { + var readStream = fs.createReadStream(file.name), + writeStream = fs.createWriteStream(target, { mode: file.mode }); + + readStream.on('error', onError); + writeStream.on('error', onError); + + if(transform) { + transform(readStream, writeStream, file); + } else { + writeStream.on('open', function() { + readStream.pipe(writeStream); + }); + } + writeStream.once('finish', function() { + if (modified) { + //target file modified date sync. + fs.utimesSync(target, file.atime, file.mtime); + cb(); + } + else cb(); + }); + } + + function rmFile(file, done) { + fs.unlink(file, function (err) { + if (err) { + return onError(err); + } + return done(); + }); + } + + function onDir(dir) { + var target = dir.name.replace(currentPath, targetPath); + isWritable(target, function (writable) { + if (writable) { + return mkDir(dir, target); + } + copyDir(dir.name); + }); + } + + function mkDir(dir, target) { + fs.mkdir(target, dir.mode, function (err) { + if (err) { + return onError(err); + } + copyDir(dir.name); + }); + } + + function copyDir(dir) { + fs.readdir(dir, function (err, items) { + if (err) { + return onError(err); + } + items.forEach(function (item) { + startCopy(path.join(dir, item)); + }); + return cb(); + }); + } + + function onLink(link) { + var target = link.replace(currentPath, targetPath); + fs.readlink(link, function (err, resolvedPath) { + if (err) { + return onError(err); + } + checkLink(resolvedPath, target); + }); + } + + function checkLink(resolvedPath, target) { + if (dereference) { + resolvedPath = path.resolve(basePath, resolvedPath); + } + isWritable(target, function (writable) { + if (writable) { + return makeLink(resolvedPath, target); + } + fs.readlink(target, function (err, targetDest) { + if (err) { + return onError(err); + } + if (dereference) { + targetDest = path.resolve(basePath, targetDest); + } + if (targetDest === resolvedPath) { + return cb(); + } + return rmFile(target, function () { + makeLink(resolvedPath, target); + }); + }); + }); + } + + function makeLink(linkPath, target) { + fs.symlink(linkPath, target, function (err) { + if (err) { + return onError(err); + } + return cb(); + }); + } + + function isWritable(path, done) { + fs.lstat(path, function (err) { + if (err) { + if (err.code === 'ENOENT') return done(true); + return done(false); + } + return done(false); + }); + } + + function onError(err) { + if (options.stopOnError) { + return cback(err); + } + else if (!errs && options.errs) { + errs = fs.createWriteStream(options.errs); + } + else if (!errs) { + errs = []; + } + if (typeof errs.write === 'undefined') { + errs.push(err); + } + else { + errs.write(err.stack + '\n\n'); + } + return cb(); + } + + function cb(skipped) { + if (!skipped) running--; + finished++; + if ((started === finished) && (running === 0)) { + if (cback !== undefined ) { + return errs ? cback(errs) : cback(null); + } + } + } +} + + diff --git a/package.json b/package.json index 6707410..57ef26c 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "yencode" : ">=1.0.6" }, "devDependencies" : { - "mocha" : "*" + "mocha" : "*", + "nexe" : "*" }, "optionalDependencies" : { "xz" : "*"