worker.js 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419
  1. "use strict";
  2. var _fs = _interopRequireDefault(require("fs"));
  3. var _module = _interopRequireDefault(require("module"));
  4. var _querystring = _interopRequireDefault(require("querystring"));
  5. var _loaderRunner = _interopRequireDefault(require("loader-runner"));
  6. var _queue = _interopRequireDefault(require("neo-async/queue"));
  7. var _jsonParseBetterErrors = _interopRequireDefault(require("json-parse-better-errors"));
  8. var _schemaUtils = require("schema-utils");
  9. var _readBuffer = _interopRequireDefault(require("./readBuffer"));
  10. var _serializer = require("./serializer");
  11. function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
  12. /* eslint-disable no-console */
  13. const writePipe = _fs.default.createWriteStream(null, {
  14. fd: 3
  15. });
  16. const readPipe = _fs.default.createReadStream(null, {
  17. fd: 4
  18. });
  19. writePipe.on('finish', onTerminateWrite);
  20. readPipe.on('end', onTerminateRead);
  21. writePipe.on('close', onTerminateWrite);
  22. readPipe.on('close', onTerminateRead);
  23. readPipe.on('error', onError);
  24. writePipe.on('error', onError);
  25. const PARALLEL_JOBS = +process.argv[2] || 20;
  26. let terminated = false;
  27. let nextQuestionId = 0;
  28. const callbackMap = Object.create(null);
  29. function onError(error) {
  30. console.error(error);
  31. }
  32. function onTerminateRead() {
  33. terminateRead();
  34. }
  35. function onTerminateWrite() {
  36. terminateWrite();
  37. }
  38. function writePipeWrite(...args) {
  39. if (!terminated) {
  40. writePipe.write(...args);
  41. }
  42. }
  43. function writePipeCork() {
  44. if (!terminated) {
  45. writePipe.cork();
  46. }
  47. }
  48. function writePipeUncork() {
  49. if (!terminated) {
  50. writePipe.uncork();
  51. }
  52. }
  53. function terminateRead() {
  54. terminated = true;
  55. readPipe.removeAllListeners();
  56. }
  57. function terminateWrite() {
  58. terminated = true;
  59. writePipe.removeAllListeners();
  60. }
  61. function terminate() {
  62. terminateRead();
  63. terminateWrite();
  64. }
  65. function toErrorObj(err) {
  66. return {
  67. message: err.message,
  68. details: err.details,
  69. stack: err.stack,
  70. hideStack: err.hideStack
  71. };
  72. }
  73. function toNativeError(obj) {
  74. if (!obj) return null;
  75. const err = new Error(obj.message);
  76. err.details = obj.details;
  77. err.missing = obj.missing;
  78. return err;
  79. }
  80. function writeJson(data) {
  81. writePipeCork();
  82. process.nextTick(() => {
  83. writePipeUncork();
  84. });
  85. const lengthBuffer = Buffer.alloc(4);
  86. const messageBuffer = Buffer.from(JSON.stringify(data, _serializer.replacer), 'utf-8');
  87. lengthBuffer.writeInt32BE(messageBuffer.length, 0);
  88. writePipeWrite(lengthBuffer);
  89. writePipeWrite(messageBuffer);
  90. }
  91. const queue = (0, _queue.default)(({
  92. id,
  93. data
  94. }, taskCallback) => {
  95. try {
  96. const resolveWithOptions = (context, request, callback, options) => {
  97. callbackMap[nextQuestionId] = callback;
  98. writeJson({
  99. type: 'resolve',
  100. id,
  101. questionId: nextQuestionId,
  102. context,
  103. request,
  104. options
  105. });
  106. nextQuestionId += 1;
  107. };
  108. const buildDependencies = [];
  109. _loaderRunner.default.runLoaders({
  110. loaders: data.loaders,
  111. resource: data.resource,
  112. readResource: _fs.default.readFile.bind(_fs.default),
  113. context: {
  114. version: 2,
  115. fs: _fs.default,
  116. loadModule: (request, callback) => {
  117. callbackMap[nextQuestionId] = (error, result) => callback(error, ...result);
  118. writeJson({
  119. type: 'loadModule',
  120. id,
  121. questionId: nextQuestionId,
  122. request
  123. });
  124. nextQuestionId += 1;
  125. },
  126. resolve: (context, request, callback) => {
  127. resolveWithOptions(context, request, callback);
  128. },
  129. // eslint-disable-next-line consistent-return
  130. getResolve: options => (context, request, callback) => {
  131. if (callback) {
  132. resolveWithOptions(context, request, callback, options);
  133. } else {
  134. return new Promise((resolve, reject) => {
  135. resolveWithOptions(context, request, (err, result) => {
  136. if (err) {
  137. reject(err);
  138. } else {
  139. resolve(result);
  140. }
  141. }, options);
  142. });
  143. }
  144. },
  145. // Not an arrow function because it uses this
  146. getOptions(schema) {
  147. // loaders, loaderIndex will be defined by runLoaders
  148. const loader = this.loaders[this.loaderIndex]; // Verbatim copy from
  149. // https://github.com/webpack/webpack/blob/v5.31.2/lib/NormalModule.js#L471-L508
  150. // except eslint/prettier differences
  151. // -- unfortunate result of getOptions being synchronous functions.
  152. let {
  153. options
  154. } = loader;
  155. if (typeof options === 'string') {
  156. if (options.substr(0, 1) === '{' && options.substr(-1) === '}') {
  157. try {
  158. options = (0, _jsonParseBetterErrors.default)(options);
  159. } catch (e) {
  160. throw new Error(`Cannot parse string options: ${e.message}`);
  161. }
  162. } else {
  163. options = _querystring.default.parse(options, '&', '=', {
  164. maxKeys: 0
  165. });
  166. }
  167. } // eslint-disable-next-line no-undefined
  168. if (options === null || options === undefined) {
  169. options = {};
  170. }
  171. if (schema) {
  172. let name = 'Loader';
  173. let baseDataPath = 'options';
  174. let match; // eslint-disable-next-line no-cond-assign
  175. if (schema.title && (match = /^(.+) (.+)$/.exec(schema.title))) {
  176. [, name, baseDataPath] = match;
  177. }
  178. (0, _schemaUtils.validate)(schema, options, {
  179. name,
  180. baseDataPath
  181. });
  182. }
  183. return options;
  184. },
  185. emitWarning: warning => {
  186. writeJson({
  187. type: 'emitWarning',
  188. id,
  189. data: toErrorObj(warning)
  190. });
  191. },
  192. emitError: error => {
  193. writeJson({
  194. type: 'emitError',
  195. id,
  196. data: toErrorObj(error)
  197. });
  198. },
  199. exec: (code, filename) => {
  200. const module = new _module.default(filename, void 0);
  201. module.paths = _module.default._nodeModulePaths((void 0).context); // eslint-disable-line no-underscore-dangle
  202. module.filename = filename;
  203. module._compile(code, filename); // eslint-disable-line no-underscore-dangle
  204. return module.exports;
  205. },
  206. addBuildDependency: filename => {
  207. buildDependencies.push(filename);
  208. },
  209. options: {
  210. context: data.optionsContext
  211. },
  212. webpack: true,
  213. 'thread-loader': true,
  214. sourceMap: data.sourceMap,
  215. target: data.target,
  216. minimize: data.minimize,
  217. resourceQuery: data.resourceQuery,
  218. rootContext: data.rootContext
  219. }
  220. }, (err, lrResult) => {
  221. const {
  222. result,
  223. cacheable,
  224. fileDependencies,
  225. contextDependencies,
  226. missingDependencies
  227. } = lrResult;
  228. const buffersToSend = [];
  229. const convertedResult = Array.isArray(result) && result.map(item => {
  230. const isBuffer = Buffer.isBuffer(item);
  231. if (isBuffer) {
  232. buffersToSend.push(item);
  233. return {
  234. buffer: true
  235. };
  236. }
  237. if (typeof item === 'string') {
  238. const stringBuffer = Buffer.from(item, 'utf-8');
  239. buffersToSend.push(stringBuffer);
  240. return {
  241. buffer: true,
  242. string: true
  243. };
  244. }
  245. return {
  246. data: item
  247. };
  248. });
  249. writeJson({
  250. type: 'job',
  251. id,
  252. error: err && toErrorObj(err),
  253. result: {
  254. result: convertedResult,
  255. cacheable,
  256. fileDependencies,
  257. contextDependencies,
  258. missingDependencies,
  259. buildDependencies
  260. },
  261. data: buffersToSend.map(buffer => buffer.length)
  262. });
  263. buffersToSend.forEach(buffer => {
  264. writePipeWrite(buffer);
  265. });
  266. setImmediate(taskCallback);
  267. });
  268. } catch (e) {
  269. writeJson({
  270. type: 'job',
  271. id,
  272. error: toErrorObj(e)
  273. });
  274. taskCallback();
  275. }
  276. }, PARALLEL_JOBS);
  277. function dispose() {
  278. terminate();
  279. queue.kill();
  280. process.exit(0);
  281. }
  282. function onMessage(message) {
  283. try {
  284. const {
  285. type,
  286. id
  287. } = message;
  288. switch (type) {
  289. case 'job':
  290. {
  291. queue.push(message);
  292. break;
  293. }
  294. case 'result':
  295. {
  296. const {
  297. error,
  298. result
  299. } = message;
  300. const callback = callbackMap[id];
  301. if (callback) {
  302. const nativeError = toNativeError(error);
  303. callback(nativeError, result);
  304. } else {
  305. console.error(`Worker got unexpected result id ${id}`);
  306. }
  307. delete callbackMap[id];
  308. break;
  309. }
  310. case 'warmup':
  311. {
  312. const {
  313. requires
  314. } = message; // load modules into process
  315. requires.forEach(r => require(r)); // eslint-disable-line import/no-dynamic-require, global-require
  316. break;
  317. }
  318. default:
  319. {
  320. console.error(`Worker got unexpected job type ${type}`);
  321. break;
  322. }
  323. }
  324. } catch (e) {
  325. console.error(`Error in worker ${e}`);
  326. }
  327. }
  328. function readNextMessage() {
  329. (0, _readBuffer.default)(readPipe, 4, (lengthReadError, lengthBuffer) => {
  330. if (lengthReadError) {
  331. console.error(`Failed to communicate with main process (read length) ${lengthReadError}`);
  332. return;
  333. }
  334. const length = lengthBuffer.length && lengthBuffer.readInt32BE(0);
  335. if (length === 0) {
  336. // worker should dispose and exit
  337. dispose();
  338. return;
  339. }
  340. (0, _readBuffer.default)(readPipe, length, (messageError, messageBuffer) => {
  341. if (terminated) {
  342. return;
  343. }
  344. if (messageError) {
  345. console.error(`Failed to communicate with main process (read message) ${messageError}`);
  346. return;
  347. }
  348. const messageString = messageBuffer.toString('utf-8');
  349. const message = JSON.parse(messageString, _serializer.reviver);
  350. onMessage(message);
  351. setImmediate(() => readNextMessage());
  352. });
  353. });
  354. } // start reading messages from main process
  355. readNextMessage();