Skip to content

Commit

Permalink
Merge pull request #216 from elastic-coders/ignore-minor-npm-ls-warnings
Browse files Browse the repository at this point in the history 
Ignore irrelevant NPM problems when building the dependency graph
  • Loading branch information
@HyperBrain
HyperBrain authored Sep 8, 2017
2 parents 8d4a98a + 51ad4b1 commit 68c3b28
Show file tree
Hide file tree
Showing 2 changed files with 328 additions and 113 deletions.
172 changes: 103 additions & 69 deletions lib/packExternalModules.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,81 +116,115 @@ module.exports = {
this.options.verbose && this.serverless.cli.log(`Fetch dependency graph from ${packageJsonPath}`);
// Get first level dependency graph
const command = 'npm ls -prod -json -depth=1'; // Only prod dependencies
let dependencyGraph = {};
try {
const depJson = childProcess.execSync(command, {

const ignoredNpmErrors = [
{ npmError: 'extraneous', log: false },
{ npmError: 'missing', log: false },
{ npmError: 'peer dep missing', log: true },
];

return BbPromise.fromCallback(cb => {
childProcess.exec(command, {
cwd: path.dirname(packageJsonPath),
maxBuffer: this.serverless.service.custom.packExternalModulesMaxBuffer || 200 * 1024,
encoding: 'utf8'
}, (err, stdout, stderr) => {
if (err) {
// Only exit with an error if we have critical npm errors for 2nd level inside
const errors = _.split(stderr, '\n');
const failed = _.reduce(errors, (failed, error) => {
if (failed) {
return true;
}
return !_.isEmpty(error) && !_.some(ignoredNpmErrors, ignoredError => _.startsWith(error, `npm ERR! ${ignoredError.npmError}`));
}, false);

if (failed) {
return cb(err);
}
}
return cb(null, stdout);
});
dependencyGraph = JSON.parse(depJson);
} catch (e) {
// We rethrow here. It's not recoverable.
throw e;
}

// (1) Generate dependency composition
const compositeModules = _.uniq(_.flatMap(stats.stats, compileStats => {
const externalModules = getExternalModules.call(this, compileStats);
return getProdModules.call(this, externalModules, packagePath, dependencyGraph);
}));
})
.then(depJson => BbPromise.try(() => JSON.parse(depJson)))
.then(dependencyGraph => {
const problems = _.get(dependencyGraph, 'problems', []);
if (this.options.verbose && !_.isEmpty(problems)) {
this.serverless.cli.log(`Ignoring ${_.size(problems)} NPM errors:`);
_.forEach(problems, problem => {
this.serverless.cli.log(`=> ${problem}`);
});
}

// (1.a) Install all needed modules
const compositeModulePath = path.join(this.webpackOutputPath, 'dependencies');
const compositePackageJson = path.join(compositeModulePath, 'package.json');
this.serverless.utils.writeFileSync(compositePackageJson, '{}');
// (1) Generate dependency composition
const compositeModules = _.uniq(_.flatMap(stats.stats, compileStats => {
const externalModules = getExternalModules.call(this, compileStats);
return getProdModules.call(this, externalModules, packagePath, dependencyGraph);
}));

this.serverless.cli.log('Packing external modules: ' + compositeModules.join(', '));
if (_.isEmpty(compositeModules)) {
// The compiled code does not reference any external modules at all
this.serverless.cli.log('No external modules needed');
return BbPromise.resolve(stats);
}

return new BbPromise((resolve, reject) => {
const start = _.now();
npm.install(compositeModules, {
cwd: compositeModulePath,
maxBuffer: this.serverless.service.custom.packExternalModulesMaxBuffer || 200 * 1024,
save: true
}).then(() => {
this.options.verbose && this.serverless.cli.log(`Package took [${_.now() - start} ms]`); // eslint-disable-line promise/always-return
resolve(stats.stats);
}).catch(e => {
reject(e);
});
})
.mapSeries(compileStats => {
const modulePath = compileStats.compilation.compiler.outputPath;

// Create package.json
const modulePackageJson = path.join(modulePath, 'package.json');
const modulePackage = {
dependencies: {}
};
const prodModules = getProdModules.call(this, getExternalModules.call(this, compileStats), packagePath, dependencyGraph);
_.forEach(prodModules, prodModule => {
const splitModule = _.split(prodModule, '@');
// If we have a scoped module we have to re-add the @
if (_.startsWith(prodModule, '@')) {
splitModule.splice(0, 1);
splitModule[0] = '@' + splitModule[0];
}
const moduleVersion = _.join(_.tail(splitModule), '@');
modulePackage.dependencies[_.first(splitModule)] = moduleVersion;
});
this.serverless.utils.writeFileSync(modulePackageJson, JSON.stringify(modulePackage, null, 2));

// Copy modules
const startCopy = _.now();
return BbPromise.fromCallback(callback => fse.copy(path.join(compositeModulePath, 'node_modules'), path.join(modulePath, 'node_modules'), callback))
.tap(() => this.options.verbose && this.serverless.cli.log(`Copy modules: ${modulePath} [${_.now() - startCopy} ms]`))
.then(() => {
// Prune extraneous packages - removes not needed ones
const startPrune = _.now();
return BbPromise.fromCallback(callback => {
childProcess.exec('npm prune', {
cwd: modulePath
}, callback);
})
.tap(() => this.options.verbose && this.serverless.cli.log(`Prune: ${modulePath} [${_.now() - startPrune} ms]`));
});
})
.return(stats);
// (1.a) Install all needed modules
const compositeModulePath = path.join(this.webpackOutputPath, 'dependencies');
const compositePackageJson = path.join(compositeModulePath, 'package.json');
this.serverless.utils.writeFileSync(compositePackageJson, '{}');

this.serverless.cli.log('Packing external modules: ' + compositeModules.join(', '));

return new BbPromise((resolve, reject) => {
const start = _.now();
npm.install(compositeModules, {
cwd: compositeModulePath,
maxBuffer: this.serverless.service.custom.packExternalModulesMaxBuffer || 200 * 1024,
save: true
}).then(() => {
this.options.verbose && this.serverless.cli.log(`Package took [${_.now() - start} ms]`); // eslint-disable-line promise/always-return
resolve(stats.stats);
}).catch(e => {
reject(e);
});
})
.mapSeries(compileStats => {
const modulePath = compileStats.compilation.compiler.outputPath;

// Create package.json
const modulePackageJson = path.join(modulePath, 'package.json');
const modulePackage = {
dependencies: {}
};
const prodModules = getProdModules.call(this, getExternalModules.call(this, compileStats), packagePath, dependencyGraph);
_.forEach(prodModules, prodModule => {
const splitModule = _.split(prodModule, '@');
// If we have a scoped module we have to re-add the @
if (_.startsWith(prodModule, '@')) {
splitModule.splice(0, 1);
splitModule[0] = '@' + splitModule[0];
}
const moduleVersion = _.join(_.tail(splitModule), '@');
modulePackage.dependencies[_.first(splitModule)] = moduleVersion;
});
this.serverless.utils.writeFileSync(modulePackageJson, JSON.stringify(modulePackage, null, 2));

// Copy modules
const startCopy = _.now();
return BbPromise.fromCallback(callback => fse.copy(path.join(compositeModulePath, 'node_modules'), path.join(modulePath, 'node_modules'), callback))
.tap(() => this.options.verbose && this.serverless.cli.log(`Copy modules: ${modulePath} [${_.now() - startCopy} ms]`))
.then(() => {
// Prune extraneous packages - removes not needed ones
const startPrune = _.now();
return BbPromise.fromCallback(callback => {
childProcess.exec('npm prune', {
cwd: modulePath
}, callback);
})
.tap(() => this.options.verbose && this.serverless.cli.log(`Prune: ${modulePath} [${_.now() - startPrune} ms]`));
});
})
.return(stats);
});
}
};
Loading

0 comments on commit 68c3b28

Please sign in to comment.