ParallelModulePlugin.js 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306
  1. const { fork: cpFork } = require('child_process');
  2. const { cpus } = require('os');
  3. const { resolve } = require('path');
  4. const logMessages = require('./util/log-messages');
  5. const pluginCompat = require('./util/plugin-compat');
  6. const webpackBin = () => {
  7. try {
  8. return require.resolve('webpack-cli');
  9. } catch (e) {}
  10. try {
  11. return require.resolve('webpack-command');
  12. } catch (e) {}
  13. throw new Error('webpack cli tool not installed or discoverable');
  14. };
  15. const configPath = compiler => {
  16. try {
  17. return require.resolve(
  18. resolve(compiler.options.context || process.cwd(), 'webpack.config'),
  19. );
  20. } catch (e) {}
  21. try {
  22. return require.resolve(resolve(process.cwd(), 'webpack.config'));
  23. } catch (e) {}
  24. throw new Error('config not in obvious location');
  25. };
  26. class ParallelModulePlugin {
  27. constructor(options) {
  28. this.options = options;
  29. }
  30. apply(compiler) {
  31. try {
  32. require('webpack/lib/JavascriptGenerator');
  33. } catch (e) {
  34. logMessages.parallelRequireWebpack4(compiler);
  35. return;
  36. }
  37. const options = this.options || {};
  38. const fork =
  39. options.fork ||
  40. ((fork, compiler, webpackBin) =>
  41. fork(webpackBin(compiler), ['--config', configPath(compiler)], {
  42. silent: true,
  43. }));
  44. const numWorkers = options.numWorkers
  45. ? typeof options.numWorkers === 'function'
  46. ? options.numWorkers
  47. : () => options.numWorkers
  48. : () => cpus().length;
  49. const minModules =
  50. typeof options.minModules === 'number' ? options.minModules : 10;
  51. const compilerHooks = pluginCompat.hooks(compiler);
  52. let freeze, thaw;
  53. compilerHooks._hardSourceMethods.tap('ParallelModulePlugin', methods => {
  54. freeze = methods.freeze;
  55. thaw = methods.thaw;
  56. });
  57. compilerHooks.thisCompilation.tap(
  58. 'ParallelModulePlugin',
  59. (compilation, params) => {
  60. const compilationHooks = pluginCompat.hooks(compilation);
  61. const nmfHooks = pluginCompat.hooks(params.normalModuleFactory);
  62. const doMaster = () => {
  63. const jobs = {};
  64. const readyJobs = {};
  65. const workers = [];
  66. let nextWorkerIndex = 0;
  67. let start = 0;
  68. let started = false;
  69. let configMismatch = false;
  70. let modules = 0;
  71. const startWorkers = () => {
  72. const _numWorkers = numWorkers();
  73. logMessages.parallelStartWorkers(compiler, {
  74. numWorkers: _numWorkers,
  75. });
  76. for (let i = 0; i < _numWorkers; i++) {
  77. const worker = fork(cpFork, compiler, webpackBin);
  78. workers.push(worker);
  79. worker.on('message', _result => {
  80. if (configMismatch) {
  81. return;
  82. }
  83. if (_result.startsWith('ready:')) {
  84. const configHash = _result.split(':')[1];
  85. if (configHash !== compiler.__hardSource_configHash) {
  86. logMessages.parallelConfigMismatch(compiler, {
  87. outHash: compiler.__hardSource_configHash,
  88. theirHash: configHash,
  89. });
  90. configMismatch = true;
  91. killWorkers();
  92. for (const id in jobs) {
  93. jobs[id].cb({ error: true });
  94. delete readyJobs[id];
  95. delete jobs[id];
  96. }
  97. return;
  98. }
  99. }
  100. if (Object.values(readyJobs).length) {
  101. const id = Object.keys(readyJobs)[0];
  102. worker.send(
  103. JSON.stringify({
  104. id,
  105. data: readyJobs[id].data,
  106. }),
  107. );
  108. delete readyJobs[id];
  109. } else {
  110. worker.ready = true;
  111. }
  112. if (_result.startsWith('ready:')) {
  113. start = Date.now();
  114. return;
  115. }
  116. const result = JSON.parse(_result);
  117. jobs[result.id].cb(result);
  118. delete [result.id];
  119. });
  120. }
  121. };
  122. const killWorkers = () => {
  123. Object.values(workers).forEach(worker => worker.kill());
  124. };
  125. const doJob = (module, cb) => {
  126. if (configMismatch) {
  127. cb({ error: new Error('config mismatch') });
  128. return;
  129. }
  130. const id = 'xxxxxxxx-xxxxxxxx'.replace(/x/g, () =>
  131. Math.random()
  132. .toString(16)
  133. .substring(2, 3),
  134. );
  135. jobs[id] = {
  136. id,
  137. data: freeze('Module', null, module, {
  138. id: module.identifier(),
  139. compilation,
  140. }),
  141. cb,
  142. };
  143. const worker = Object.values(workers).find(worker => worker.ready);
  144. if (worker) {
  145. worker.ready = false;
  146. worker.send(
  147. JSON.stringify({
  148. id,
  149. data: jobs[id].data,
  150. }),
  151. );
  152. } else {
  153. readyJobs[id] = jobs[id];
  154. }
  155. if (!started) {
  156. started = true;
  157. startWorkers();
  158. }
  159. };
  160. const _create = params.normalModuleFactory.create;
  161. params.normalModuleFactory.create = (data, cb) => {
  162. _create.call(params.normalModuleFactory, data, (err, module) => {
  163. if (err) {
  164. return cb(err);
  165. }
  166. if (module.constructor.name === 'NormalModule') {
  167. const build = module.build;
  168. module.build = (
  169. options,
  170. compilation,
  171. resolver,
  172. fs,
  173. callback,
  174. ) => {
  175. if (modules < minModules) {
  176. build.call(
  177. module,
  178. options,
  179. compilation,
  180. resolver,
  181. fs,
  182. callback,
  183. );
  184. modules += 1;
  185. return;
  186. }
  187. try {
  188. doJob(module, result => {
  189. if (result.error) {
  190. build.call(
  191. module,
  192. options,
  193. compilation,
  194. resolver,
  195. fs,
  196. callback,
  197. );
  198. } else {
  199. thaw('Module', module, result.module, {
  200. compilation,
  201. normalModuleFactory: params.normalModuleFactory,
  202. contextModuleFactory: params.contextModuleFactory,
  203. });
  204. callback();
  205. }
  206. });
  207. } catch (e) {
  208. logMessages.parallelErrorSendingJob(compiler, e);
  209. build.call(
  210. module,
  211. options,
  212. compilation,
  213. resolver,
  214. fs,
  215. callback,
  216. );
  217. }
  218. };
  219. cb(null, module);
  220. } else {
  221. cb(err, module);
  222. }
  223. });
  224. };
  225. compilationHooks.seal.tap('ParallelModulePlugin', () => {
  226. killWorkers();
  227. });
  228. };
  229. const doChild = () => {
  230. const _create = params.normalModuleFactory.create;
  231. params.normalModuleFactory.create = (data, cb) => {};
  232. process.send('ready:' + compiler.__hardSource_configHash);
  233. process.on('message', _job => {
  234. const job = JSON.parse(_job);
  235. const module = thaw('Module', null, job.data, {
  236. compilation,
  237. normalModuleFactory: params.normalModuleFactory,
  238. contextModuleFactory: params.contextModuleFactory,
  239. });
  240. module.build(
  241. compilation.options,
  242. compilation,
  243. compilation.resolverFactory.get('normal', module.resolveOptions),
  244. compilation.inputFileSystem,
  245. error => {
  246. process.send(
  247. JSON.stringify({
  248. id: job.id,
  249. error: error,
  250. module:
  251. module &&
  252. freeze('Module', null, module, {
  253. id: module.identifier(),
  254. compilation,
  255. }),
  256. }),
  257. );
  258. },
  259. );
  260. });
  261. };
  262. if (!process.send) {
  263. doMaster();
  264. } else {
  265. doChild();
  266. }
  267. },
  268. );
  269. }
  270. }
  271. module.exports = ParallelModulePlugin;