123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306 |
- const { fork: cpFork } = require('child_process');
- const { cpus } = require('os');
- const { resolve } = require('path');
- const logMessages = require('./util/log-messages');
- const pluginCompat = require('./util/plugin-compat');
- const webpackBin = () => {
- try {
- return require.resolve('webpack-cli');
- } catch (e) {}
- try {
- return require.resolve('webpack-command');
- } catch (e) {}
- throw new Error('webpack cli tool not installed or discoverable');
- };
- const configPath = compiler => {
- try {
- return require.resolve(
- resolve(compiler.options.context || process.cwd(), 'webpack.config'),
- );
- } catch (e) {}
- try {
- return require.resolve(resolve(process.cwd(), 'webpack.config'));
- } catch (e) {}
- throw new Error('config not in obvious location');
- };
- class ParallelModulePlugin {
- constructor(options) {
- this.options = options;
- }
- apply(compiler) {
- try {
- require('webpack/lib/JavascriptGenerator');
- } catch (e) {
- logMessages.parallelRequireWebpack4(compiler);
- return;
- }
- const options = this.options || {};
- const fork =
- options.fork ||
- ((fork, compiler, webpackBin) =>
- fork(webpackBin(compiler), ['--config', configPath(compiler)], {
- silent: true,
- }));
- const numWorkers = options.numWorkers
- ? typeof options.numWorkers === 'function'
- ? options.numWorkers
- : () => options.numWorkers
- : () => cpus().length;
- const minModules =
- typeof options.minModules === 'number' ? options.minModules : 10;
- const compilerHooks = pluginCompat.hooks(compiler);
- let freeze, thaw;
- compilerHooks._hardSourceMethods.tap('ParallelModulePlugin', methods => {
- freeze = methods.freeze;
- thaw = methods.thaw;
- });
- compilerHooks.thisCompilation.tap(
- 'ParallelModulePlugin',
- (compilation, params) => {
- const compilationHooks = pluginCompat.hooks(compilation);
- const nmfHooks = pluginCompat.hooks(params.normalModuleFactory);
- const doMaster = () => {
- const jobs = {};
- const readyJobs = {};
- const workers = [];
- let nextWorkerIndex = 0;
- let start = 0;
- let started = false;
- let configMismatch = false;
- let modules = 0;
- const startWorkers = () => {
- const _numWorkers = numWorkers();
- logMessages.parallelStartWorkers(compiler, {
- numWorkers: _numWorkers,
- });
- for (let i = 0; i < _numWorkers; i++) {
- const worker = fork(cpFork, compiler, webpackBin);
- workers.push(worker);
- worker.on('message', _result => {
- if (configMismatch) {
- return;
- }
- if (_result.startsWith('ready:')) {
- const configHash = _result.split(':')[1];
- if (configHash !== compiler.__hardSource_configHash) {
- logMessages.parallelConfigMismatch(compiler, {
- outHash: compiler.__hardSource_configHash,
- theirHash: configHash,
- });
- configMismatch = true;
- killWorkers();
- for (const id in jobs) {
- jobs[id].cb({ error: true });
- delete readyJobs[id];
- delete jobs[id];
- }
- return;
- }
- }
- if (Object.values(readyJobs).length) {
- const id = Object.keys(readyJobs)[0];
- worker.send(
- JSON.stringify({
- id,
- data: readyJobs[id].data,
- }),
- );
- delete readyJobs[id];
- } else {
- worker.ready = true;
- }
- if (_result.startsWith('ready:')) {
- start = Date.now();
- return;
- }
- const result = JSON.parse(_result);
- jobs[result.id].cb(result);
- delete [result.id];
- });
- }
- };
- const killWorkers = () => {
- Object.values(workers).forEach(worker => worker.kill());
- };
- const doJob = (module, cb) => {
- if (configMismatch) {
- cb({ error: new Error('config mismatch') });
- return;
- }
- const id = 'xxxxxxxx-xxxxxxxx'.replace(/x/g, () =>
- Math.random()
- .toString(16)
- .substring(2, 3),
- );
- jobs[id] = {
- id,
- data: freeze('Module', null, module, {
- id: module.identifier(),
- compilation,
- }),
- cb,
- };
- const worker = Object.values(workers).find(worker => worker.ready);
- if (worker) {
- worker.ready = false;
- worker.send(
- JSON.stringify({
- id,
- data: jobs[id].data,
- }),
- );
- } else {
- readyJobs[id] = jobs[id];
- }
- if (!started) {
- started = true;
- startWorkers();
- }
- };
- const _create = params.normalModuleFactory.create;
- params.normalModuleFactory.create = (data, cb) => {
- _create.call(params.normalModuleFactory, data, (err, module) => {
- if (err) {
- return cb(err);
- }
- if (module.constructor.name === 'NormalModule') {
- const build = module.build;
- module.build = (
- options,
- compilation,
- resolver,
- fs,
- callback,
- ) => {
- if (modules < minModules) {
- build.call(
- module,
- options,
- compilation,
- resolver,
- fs,
- callback,
- );
- modules += 1;
- return;
- }
- try {
- doJob(module, result => {
- if (result.error) {
- build.call(
- module,
- options,
- compilation,
- resolver,
- fs,
- callback,
- );
- } else {
- thaw('Module', module, result.module, {
- compilation,
- normalModuleFactory: params.normalModuleFactory,
- contextModuleFactory: params.contextModuleFactory,
- });
- callback();
- }
- });
- } catch (e) {
- logMessages.parallelErrorSendingJob(compiler, e);
- build.call(
- module,
- options,
- compilation,
- resolver,
- fs,
- callback,
- );
- }
- };
- cb(null, module);
- } else {
- cb(err, module);
- }
- });
- };
- compilationHooks.seal.tap('ParallelModulePlugin', () => {
- killWorkers();
- });
- };
- const doChild = () => {
- const _create = params.normalModuleFactory.create;
- params.normalModuleFactory.create = (data, cb) => {};
- process.send('ready:' + compiler.__hardSource_configHash);
- process.on('message', _job => {
- const job = JSON.parse(_job);
- const module = thaw('Module', null, job.data, {
- compilation,
- normalModuleFactory: params.normalModuleFactory,
- contextModuleFactory: params.contextModuleFactory,
- });
- module.build(
- compilation.options,
- compilation,
- compilation.resolverFactory.get('normal', module.resolveOptions),
- compilation.inputFileSystem,
- error => {
- process.send(
- JSON.stringify({
- id: job.id,
- error: error,
- module:
- module &&
- freeze('Module', null, module, {
- id: module.identifier(),
- compilation,
- }),
- }),
- );
- },
- );
- });
- };
- if (!process.send) {
- doMaster();
- } else {
- doChild();
- }
- },
- );
- }
- }
- module.exports = ParallelModulePlugin;
|