const { fork: cpFork } = require('child_process'); const { cpus } = require('os'); const { resolve } = require('path'); const logMessages = require('./util/log-messages'); const pluginCompat = require('./util/plugin-compat'); const webpackBin = () => { try { return require.resolve('webpack-cli'); } catch (e) {} try { return require.resolve('webpack-command'); } catch (e) {} throw new Error('webpack cli tool not installed or discoverable'); }; const configPath = compiler => { try { return require.resolve( resolve(compiler.options.context || process.cwd(), 'webpack.config'), ); } catch (e) {} try { return require.resolve(resolve(process.cwd(), 'webpack.config')); } catch (e) {} throw new Error('config not in obvious location'); }; class ParallelModulePlugin { constructor(options) { this.options = options; } apply(compiler) { try { require('webpack/lib/JavascriptGenerator'); } catch (e) { logMessages.parallelRequireWebpack4(compiler); return; } const options = this.options || {}; const fork = options.fork || ((fork, compiler, webpackBin) => fork(webpackBin(compiler), ['--config', configPath(compiler)], { silent: true, })); const numWorkers = options.numWorkers ? typeof options.numWorkers === 'function' ? options.numWorkers : () => options.numWorkers : () => cpus().length; const minModules = typeof options.minModules === 'number' ? options.minModules : 10; const compilerHooks = pluginCompat.hooks(compiler); let freeze, thaw; compilerHooks._hardSourceMethods.tap('ParallelModulePlugin', methods => { freeze = methods.freeze; thaw = methods.thaw; }); compilerHooks.thisCompilation.tap( 'ParallelModulePlugin', (compilation, params) => { const compilationHooks = pluginCompat.hooks(compilation); const nmfHooks = pluginCompat.hooks(params.normalModuleFactory); const doMaster = () => { const jobs = {}; const readyJobs = {}; const workers = []; let nextWorkerIndex = 0; let start = 0; let started = false; let configMismatch = false; let modules = 0; const startWorkers = () => { const _numWorkers = numWorkers(); logMessages.parallelStartWorkers(compiler, { numWorkers: _numWorkers, }); for (let i = 0; i < _numWorkers; i++) { const worker = fork(cpFork, compiler, webpackBin); workers.push(worker); worker.on('message', _result => { if (configMismatch) { return; } if (_result.startsWith('ready:')) { const configHash = _result.split(':')[1]; if (configHash !== compiler.__hardSource_configHash) { logMessages.parallelConfigMismatch(compiler, { outHash: compiler.__hardSource_configHash, theirHash: configHash, }); configMismatch = true; killWorkers(); for (const id in jobs) { jobs[id].cb({ error: true }); delete readyJobs[id]; delete jobs[id]; } return; } } if (Object.values(readyJobs).length) { const id = Object.keys(readyJobs)[0]; worker.send( JSON.stringify({ id, data: readyJobs[id].data, }), ); delete readyJobs[id]; } else { worker.ready = true; } if (_result.startsWith('ready:')) { start = Date.now(); return; } const result = JSON.parse(_result); jobs[result.id].cb(result); delete [result.id]; }); } }; const killWorkers = () => { Object.values(workers).forEach(worker => worker.kill()); }; const doJob = (module, cb) => { if (configMismatch) { cb({ error: new Error('config mismatch') }); return; } const id = 'xxxxxxxx-xxxxxxxx'.replace(/x/g, () => Math.random() .toString(16) .substring(2, 3), ); jobs[id] = { id, data: freeze('Module', null, module, { id: module.identifier(), compilation, }), cb, }; const worker = Object.values(workers).find(worker => worker.ready); if (worker) { worker.ready = false; worker.send( JSON.stringify({ id, data: jobs[id].data, }), ); } else { readyJobs[id] = jobs[id]; } if (!started) { started = true; startWorkers(); } }; const _create = params.normalModuleFactory.create; params.normalModuleFactory.create = (data, cb) => { _create.call(params.normalModuleFactory, data, (err, module) => { if (err) { return cb(err); } if (module.constructor.name === 'NormalModule') { const build = module.build; module.build = ( options, compilation, resolver, fs, callback, ) => { if (modules < minModules) { build.call( module, options, compilation, resolver, fs, callback, ); modules += 1; return; } try { doJob(module, result => { if (result.error) { build.call( module, options, compilation, resolver, fs, callback, ); } else { thaw('Module', module, result.module, { compilation, normalModuleFactory: params.normalModuleFactory, contextModuleFactory: params.contextModuleFactory, }); callback(); } }); } catch (e) { logMessages.parallelErrorSendingJob(compiler, e); build.call( module, options, compilation, resolver, fs, callback, ); } }; cb(null, module); } else { cb(err, module); } }); }; compilationHooks.seal.tap('ParallelModulePlugin', () => { killWorkers(); }); }; const doChild = () => { const _create = params.normalModuleFactory.create; params.normalModuleFactory.create = (data, cb) => {}; process.send('ready:' + compiler.__hardSource_configHash); process.on('message', _job => { const job = JSON.parse(_job); const module = thaw('Module', null, job.data, { compilation, normalModuleFactory: params.normalModuleFactory, contextModuleFactory: params.contextModuleFactory, }); module.build( compilation.options, compilation, compilation.resolverFactory.get('normal', module.resolveOptions), compilation.inputFileSystem, error => { process.send( JSON.stringify({ id: job.id, error: error, module: module && freeze('Module', null, module, { id: module.identifier(), compilation, }), }), ); }, ); }); }; if (!process.send) { doMaster(); } else { doChild(); } }, ); } } module.exports = ParallelModulePlugin;