CacheEnhancedResolve.js 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517
  1. const path = require('path');
  2. const lodash = require('lodash');
  3. const nodeObjectHash = require('node-object-hash');
  4. const parseJson = require('parse-json');
  5. const pluginCompat = require('./util/plugin-compat');
  6. const promisify = require('./util/promisify');
  7. const relateContext = require('./util/relate-context');
  8. const serial = require('./util/serial');
  9. const values = require('./util/Object.values');
  10. const bulkFsTask = require('./util/bulk-fs-task');
  11. const { parityCacheFromCache, pushParityWriteOps } = require('./util/parity');
  12. const serialNormalResolved = serial.created({
  13. result: serial.path,
  14. resourceResolveData: serial.objectAssign({
  15. context: serial.created({
  16. issuer: serial.request,
  17. resolveOptions: serial.identity,
  18. }),
  19. path: serial.path,
  20. descriptionFilePath: serial.path,
  21. descriptionFileRoot: serial.path,
  22. }),
  23. });
  24. class EnhancedResolveCache {
  25. apply(compiler) {
  26. let missingCacheSerializer;
  27. let resolverCacheSerializer;
  28. let missingCache = { normal: {}, loader: {}, context: {} };
  29. let resolverCache = { normal: {}, loader: {}, context: {} };
  30. let parityCache = {};
  31. const compilerHooks = pluginCompat.hooks(compiler);
  32. compilerHooks._hardSourceCreateSerializer.tap(
  33. 'HardSource - EnhancedResolveCache',
  34. (cacheSerializerFactory, cacheDirPath) => {
  35. missingCacheSerializer = cacheSerializerFactory.create({
  36. name: 'missing-resolve',
  37. type: 'data',
  38. autoParse: true,
  39. cacheDirPath,
  40. });
  41. resolverCacheSerializer = cacheSerializerFactory.create({
  42. name: 'resolver',
  43. type: 'data',
  44. autoParse: true,
  45. cacheDirPath,
  46. });
  47. },
  48. );
  49. compilerHooks._hardSourceResetCache.tap(
  50. 'HardSource - EnhancedResolveCache',
  51. () => {
  52. missingCache = { normal: {}, loader: {}, context: {} };
  53. resolverCache = { normal: {}, loader: {}, context: {} };
  54. parityCache = {};
  55. compiler.__hardSource_missingCache = missingCache;
  56. },
  57. );
  58. compilerHooks._hardSourceReadCache.tapPromise(
  59. 'HardSource - EnhancedResolveCache',
  60. ({ contextNormalPath, contextNormalRequest }) => {
  61. return Promise.all([
  62. missingCacheSerializer.read().then(_missingCache => {
  63. missingCache = { normal: {}, loader: {}, context: {} };
  64. compiler.__hardSource_missingCache = missingCache;
  65. function contextNormalMissingKey(compiler, key) {
  66. const parsed = parseJson(key);
  67. return JSON.stringify([
  68. contextNormalPath(compiler, parsed[0]),
  69. contextNormalPath(compiler, parsed[1]),
  70. ]);
  71. }
  72. function contextNormalMissing(compiler, missing) {
  73. return missing.map(missed =>
  74. contextNormalRequest(compiler, missed),
  75. );
  76. }
  77. Object.keys(_missingCache).forEach(key => {
  78. let item = _missingCache[key];
  79. if (typeof item === 'string') {
  80. item = parseJson(item);
  81. }
  82. const splitIndex = key.indexOf('/');
  83. const group = key.substring(0, splitIndex);
  84. const keyName = contextNormalMissingKey(
  85. compiler,
  86. key.substring(splitIndex + 1),
  87. );
  88. missingCache[group] = missingCache[group] || {};
  89. missingCache[group][keyName] = contextNormalMissing(
  90. compiler,
  91. item,
  92. );
  93. });
  94. }),
  95. resolverCacheSerializer.read().then(_resolverCache => {
  96. resolverCache = { normal: {}, loader: {}, context: {} };
  97. parityCache = {};
  98. function contextNormalResolvedKey(compiler, key) {
  99. const parsed = parseJson(key);
  100. return JSON.stringify([
  101. contextNormalPath(compiler, parsed[0]),
  102. parsed[1],
  103. ]);
  104. }
  105. function contextNormalResolved(compiler, resolved) {
  106. return serialNormalResolved.thaw(resolved, resolved, {
  107. compiler,
  108. });
  109. }
  110. Object.keys(_resolverCache).forEach(key => {
  111. let item = _resolverCache[key];
  112. if (typeof item === 'string') {
  113. item = parseJson(item);
  114. }
  115. if (key.startsWith('__hardSource_parityToken')) {
  116. parityCache[key] = item;
  117. return;
  118. }
  119. const splitIndex = key.indexOf('/');
  120. const group = key.substring(0, splitIndex);
  121. const keyName = contextNormalResolvedKey(
  122. compiler,
  123. key.substring(splitIndex + 1),
  124. );
  125. resolverCache[group] = resolverCache[group] || {};
  126. resolverCache[group][keyName] = contextNormalResolved(
  127. compiler,
  128. item,
  129. );
  130. });
  131. }),
  132. ]);
  133. },
  134. );
  135. compilerHooks._hardSourceParityCache.tap(
  136. 'HardSource - EnhancedResolveCache',
  137. parityRoot => {
  138. parityCacheFromCache('EnhancedResolve', parityRoot, parityCache);
  139. },
  140. );
  141. let missingVerifyResolve;
  142. compiler.__hardSource_missingVerify = new Promise(resolve => {
  143. missingVerifyResolve = resolve;
  144. });
  145. compilerHooks._hardSourceVerifyCache.tapPromise(
  146. 'HardSource - EnhancedResolveCache',
  147. () =>
  148. (() => {
  149. compiler.__hardSource_missingVerify = new Promise(resolve => {
  150. missingVerifyResolve = resolve;
  151. });
  152. const bulk = lodash.flatten(
  153. Object.keys(missingCache).map(group =>
  154. lodash.flatten(
  155. Object.keys(missingCache[group])
  156. .map(key => {
  157. const missingItem = missingCache[group][key];
  158. if (!missingItem) {
  159. return;
  160. }
  161. return missingItem.map((missed, index) => [
  162. group,
  163. key,
  164. missed,
  165. index,
  166. ]);
  167. })
  168. .filter(Boolean),
  169. ),
  170. ),
  171. );
  172. return bulkFsTask(bulk, (item, task) => {
  173. const group = item[0];
  174. const key = item[1];
  175. const missingItem = missingCache[group][key];
  176. const missed = item[2];
  177. const missedPath = missed.split('?')[0];
  178. const missedIndex = item[3];
  179. // The missed index is the resolved item. Invalidate if it does not
  180. // exist.
  181. if (missedIndex === missingItem.length - 1) {
  182. compiler.inputFileSystem.stat(
  183. missed,
  184. task((err, stat) => {
  185. if (err) {
  186. missingItem.invalid = true;
  187. missingItem.invalidReason = 'resolved now missing';
  188. }
  189. }),
  190. );
  191. } else {
  192. compiler.inputFileSystem.stat(
  193. missed,
  194. task((err, stat) => {
  195. if (err) {
  196. return;
  197. }
  198. if (stat.isDirectory()) {
  199. if (group === 'context') {
  200. missingItem.invalid = true;
  201. }
  202. }
  203. if (stat.isFile()) {
  204. if (group === 'loader' || group.startsWith('normal')) {
  205. missingItem.invalid = true;
  206. missingItem.invalidReason = 'missing now found';
  207. }
  208. }
  209. }),
  210. );
  211. }
  212. });
  213. })().then(missingVerifyResolve),
  214. );
  215. function bindResolvers() {
  216. function configureMissing(key, resolver) {
  217. // missingCache[key] = missingCache[key] || {};
  218. // resolverCache[key] = resolverCache[key] || {};
  219. const _resolve = resolver.resolve;
  220. resolver.resolve = function(info, context, request, cb, cb2) {
  221. let numArgs = 4;
  222. if (!cb) {
  223. numArgs = 3;
  224. cb = request;
  225. request = context;
  226. context = info;
  227. }
  228. let resolveContext;
  229. if (cb2) {
  230. numArgs = 5;
  231. resolveContext = cb;
  232. cb = cb2;
  233. }
  234. if (info && info.resolveOptions) {
  235. key = `normal-${new nodeObjectHash({ sort: false }).hash(
  236. info.resolveOptions,
  237. )}`;
  238. resolverCache[key] = resolverCache[key] || {};
  239. missingCache[key] = missingCache[key] || {};
  240. }
  241. const resolveId = JSON.stringify([context, request]);
  242. const absResolveId = JSON.stringify([
  243. context,
  244. relateContext.relateAbsolutePath(context, request),
  245. ]);
  246. const resolve =
  247. resolverCache[key][resolveId] || resolverCache[key][absResolveId];
  248. if (resolve && !resolve.invalid) {
  249. const missingId = JSON.stringify([context, resolve.result]);
  250. const missing = missingCache[key][missingId];
  251. if (missing && !missing.invalid) {
  252. return cb(
  253. null,
  254. [resolve.result].concat(request.split('?').slice(1)).join('?'),
  255. resolve.resourceResolveData,
  256. );
  257. } else {
  258. resolve.invalid = true;
  259. resolve.invalidReason = 'out of date';
  260. }
  261. }
  262. let localMissing = [];
  263. const callback = (err, result, result2) => {
  264. if (result) {
  265. const inverseId = JSON.stringify([context, result.split('?')[0]]);
  266. const resolveId = JSON.stringify([context, request]);
  267. // Skip recording missing for any dependency in node_modules.
  268. // Changes to them will be handled by the environment hash. If we
  269. // tracked the stuff in node_modules too, we'd be adding a whole
  270. // bunch of reduntant work.
  271. if (result.includes('node_modules')) {
  272. localMissing = localMissing.filter(
  273. missed => !missed.includes('node_modules'),
  274. );
  275. }
  276. // In case of other cache layers, if we already have missing
  277. // recorded and we get a new empty array of missing, keep the old
  278. // value.
  279. if (localMissing.length === 0 && missingCache[key][inverseId]) {
  280. return cb(err, result, result2);
  281. }
  282. missingCache[key][inverseId] = localMissing
  283. .filter((missed, missedIndex) => {
  284. const index = localMissing.indexOf(missed);
  285. if (index === -1 || index < missedIndex) {
  286. return false;
  287. }
  288. if (missed === result) {
  289. return false;
  290. }
  291. return true;
  292. })
  293. .concat(result.split('?')[0]);
  294. missingCache[key][inverseId].new = true;
  295. resolverCache[key][resolveId] = {
  296. result: result.split('?')[0],
  297. resourceResolveData: result2,
  298. new: true,
  299. };
  300. }
  301. cb(err, result, result2);
  302. };
  303. const _missing =
  304. cb.missing || (resolveContext && resolveContext.missing);
  305. if (_missing) {
  306. callback.missing = {
  307. push(path) {
  308. localMissing.push(path);
  309. _missing.push(path);
  310. },
  311. add(path) {
  312. localMissing.push(path);
  313. _missing.add(path);
  314. },
  315. };
  316. if (resolveContext) {
  317. resolveContext.missing = callback.missing;
  318. }
  319. } else {
  320. callback.missing = Object.assign(localMissing, {
  321. add(path) {
  322. localMissing.push(path);
  323. },
  324. });
  325. if (resolveContext) {
  326. resolveContext.missing = callback.missing;
  327. }
  328. }
  329. if (numArgs === 3) {
  330. _resolve.call(this, context, request, callback);
  331. } else if (numArgs === 5) {
  332. _resolve.call(
  333. this,
  334. info,
  335. context,
  336. request,
  337. resolveContext,
  338. callback,
  339. );
  340. } else {
  341. _resolve.call(this, info, context, request, callback);
  342. }
  343. };
  344. }
  345. if (compiler.resolverFactory) {
  346. compiler.resolverFactory.hooks.resolver
  347. .for('normal')
  348. .tap('HardSource resolve cache', (resolver, options) => {
  349. const normalCacheId = `normal-${new nodeObjectHash({
  350. sort: false,
  351. }).hash(Object.assign({}, options, { fileSystem: null }))}`;
  352. resolverCache[normalCacheId] = resolverCache[normalCacheId] || {};
  353. missingCache[normalCacheId] = missingCache[normalCacheId] || {};
  354. configureMissing(normalCacheId, resolver);
  355. return resolver;
  356. });
  357. compiler.resolverFactory.hooks.resolver
  358. .for('loader')
  359. .tap('HardSource resolve cache', resolver => {
  360. configureMissing('loader', resolver);
  361. return resolver;
  362. });
  363. compiler.resolverFactory.hooks.resolver
  364. .for('context')
  365. .tap('HardSource resolve cache', resolver => {
  366. configureMissing('context', resolver);
  367. return resolver;
  368. });
  369. } else {
  370. configureMissing('normal', compiler.resolvers.normal);
  371. configureMissing('loader', compiler.resolvers.loader);
  372. configureMissing('context', compiler.resolvers.context);
  373. }
  374. }
  375. compilerHooks.afterPlugins.tap('HardSource - EnhancedResolveCache', () => {
  376. if (compiler.resolvers.normal) {
  377. bindResolvers();
  378. } else {
  379. compilerHooks.afterResolvers.tap(
  380. 'HardSource - EnhancedResolveCache',
  381. bindResolvers,
  382. );
  383. }
  384. });
  385. compilerHooks._hardSourceWriteCache.tapPromise(
  386. 'HardSource - EnhancedResolveCache',
  387. (compilation, { relateNormalPath, relateNormalRequest }) => {
  388. if (compilation.compiler.parentCompilation) {
  389. const resolverOps = [];
  390. pushParityWriteOps(compilation, resolverOps);
  391. return resolverCacheSerializer.write(resolverOps);
  392. }
  393. const missingOps = [];
  394. const resolverOps = [];
  395. function relateNormalMissingKey(compiler, key) {
  396. const parsed = parseJson(key);
  397. return JSON.stringify([
  398. relateNormalPath(compiler, parsed[0]),
  399. relateNormalPath(compiler, parsed[1]),
  400. ]);
  401. }
  402. function relateNormalMissing(compiler, missing) {
  403. return missing.map(missed => relateNormalRequest(compiler, missed));
  404. }
  405. Object.keys(missingCache).forEach(group => {
  406. Object.keys(missingCache[group]).forEach(key => {
  407. if (!missingCache[group][key]) {
  408. return;
  409. }
  410. if (missingCache[group][key].new) {
  411. missingCache[group][key].new = false;
  412. missingOps.push({
  413. key: `${group}/${relateNormalMissingKey(compiler, key)}`,
  414. value: JSON.stringify(
  415. relateNormalMissing(compiler, missingCache[group][key]),
  416. ),
  417. });
  418. } else if (missingCache[group][key].invalid) {
  419. missingCache[group][key] = null;
  420. missingOps.push({
  421. key: `${group}/${relateNormalMissingKey(compiler, key)}`,
  422. value: null,
  423. });
  424. }
  425. });
  426. });
  427. function relateNormalResolvedKey(compiler, key) {
  428. const parsed = parseJson(key);
  429. return JSON.stringify([
  430. relateNormalPath(compiler, parsed[0]),
  431. relateContext.relateAbsolutePath(parsed[0], parsed[1]),
  432. ]);
  433. }
  434. function relateNormalResolved(compiler, resolved) {
  435. return serialNormalResolved.freeze(resolved, resolved, {
  436. compiler,
  437. });
  438. }
  439. Object.keys(resolverCache).forEach(group => {
  440. Object.keys(resolverCache[group]).forEach(key => {
  441. if (!resolverCache[group][key]) {
  442. return;
  443. }
  444. if (resolverCache[group][key].new) {
  445. resolverCache[group][key].new = false;
  446. resolverOps.push({
  447. key: `${group}/${relateNormalResolvedKey(compiler, key)}`,
  448. value: JSON.stringify(
  449. relateNormalResolved(compiler, resolverCache[group][key]),
  450. ),
  451. });
  452. } else if (resolverCache[group][key].invalid) {
  453. resolverCache[group][key] = null;
  454. resolverOps.push({
  455. key: `${group}/${relateNormalResolvedKey(compiler, key)}`,
  456. value: null,
  457. });
  458. }
  459. });
  460. });
  461. pushParityWriteOps(compilation, resolverOps);
  462. return Promise.all([
  463. missingCacheSerializer.write(missingOps),
  464. resolverCacheSerializer.write(resolverOps),
  465. ]);
  466. },
  467. );
  468. }
  469. }
  470. module.exports = EnhancedResolveCache;