index.js 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582
  1. const crypto = require('crypto');
  2. const fs = require('graceful-fs');
  3. const path = require('path');
  4. const lodash = require('lodash');
  5. const _mkdirp = require('mkdirp');
  6. const _rimraf = require('rimraf');
  7. const nodeObjectHash = require('node-object-hash');
  8. const findCacheDir = require('find-cache-dir');
  9. const envHash = require('./lib/envHash');
  10. const defaultConfigHash = require('./lib/defaultConfigHash');
  11. const promisify = require('./lib/util/promisify');
  12. const relateContext = require('./lib/util/relate-context');
  13. const pluginCompat = require('./lib/util/plugin-compat');
  14. const logMessages = require('./lib/util/log-messages');
  15. const LoggerFactory = require('./lib/loggerFactory');
  16. const cachePrefix = require('./lib/util').cachePrefix;
  17. const CacheSerializerFactory = require('./lib/CacheSerializerFactory');
  18. const ExcludeModulePlugin = require('./lib/ExcludeModulePlugin');
  19. const HardSourceLevelDbSerializerPlugin = require('./lib/SerializerLeveldbPlugin');
  20. const SerializerAppend2Plugin = require('./lib/SerializerAppend2Plugin');
  21. const SerializerAppendPlugin = require('./lib/SerializerAppendPlugin');
  22. const SerializerCacachePlugin = require('./lib/SerializerCacachePlugin');
  23. const SerializerJsonPlugin = require('./lib/SerializerJsonPlugin');
  24. const hardSourceVersion = require('./package.json').version;
  25. function requestHash(request) {
  26. return crypto
  27. .createHash('sha1')
  28. .update(request)
  29. .digest()
  30. .hexSlice();
  31. }
  32. const mkdirp = promisify(_mkdirp, { context: _mkdirp });
  33. mkdirp.sync = _mkdirp.sync.bind(_mkdirp);
  34. const rimraf = promisify(_rimraf);
  35. rimraf.sync = _rimraf.sync.bind(_rimraf);
  36. const fsReadFile = promisify(fs.readFile, { context: fs });
  37. const fsWriteFile = promisify(fs.writeFile, { context: fs });
  38. const bulkFsTask = (array, each) =>
  39. new Promise((resolve, reject) => {
  40. let ops = 0;
  41. const out = [];
  42. array.forEach((item, i) => {
  43. out[i] = each(item, (back, callback) => {
  44. ops++;
  45. return (err, value) => {
  46. try {
  47. out[i] = back(err, value, out[i]);
  48. } catch (e) {
  49. return reject(e);
  50. }
  51. ops--;
  52. if (ops === 0) {
  53. resolve(out);
  54. }
  55. };
  56. });
  57. });
  58. if (ops === 0) {
  59. resolve(out);
  60. }
  61. });
  62. const compilerContext = relateContext.compilerContext;
  63. const relateNormalPath = relateContext.relateNormalPath;
  64. const contextNormalPath = relateContext.contextNormalPath;
  65. const contextNormalPathSet = relateContext.contextNormalPathSet;
  66. function relateNormalRequest(compiler, key) {
  67. return key
  68. .split('!')
  69. .map(subkey => relateNormalPath(compiler, subkey))
  70. .join('!');
  71. }
  72. function relateNormalModuleId(compiler, id) {
  73. return id.substring(0, 24) + relateNormalRequest(compiler, id.substring(24));
  74. }
  75. function contextNormalRequest(compiler, key) {
  76. return key
  77. .split('!')
  78. .map(subkey => contextNormalPath(compiler, subkey))
  79. .join('!');
  80. }
  81. function contextNormalModuleId(compiler, id) {
  82. return id.substring(0, 24) + contextNormalRequest(compiler, id.substring(24));
  83. }
  84. function contextNormalLoaders(compiler, loaders) {
  85. return loaders.map(loader =>
  86. Object.assign({}, loader, {
  87. loader: contextNormalPath(compiler, loader.loader),
  88. }),
  89. );
  90. }
  91. function contextNormalPathArray(compiler, paths) {
  92. return paths.map(subpath => contextNormalPath(compiler, subpath));
  93. }
  94. class HardSourceWebpackPlugin {
  95. constructor(options) {
  96. this.options = options || {};
  97. }
  98. getPath(dirName, suffix) {
  99. const confighashIndex = dirName.search(/\[confighash\]/);
  100. if (confighashIndex !== -1) {
  101. dirName = dirName.replace(/\[confighash\]/, this.configHash);
  102. }
  103. let cachePath = path.resolve(
  104. process.cwd(),
  105. this.compilerOutputOptions.path,
  106. dirName,
  107. );
  108. if (suffix) {
  109. cachePath = path.join(cachePath, suffix);
  110. }
  111. return cachePath;
  112. }
  113. getCachePath(suffix) {
  114. return this.getPath(this.options.cacheDirectory, suffix);
  115. }
  116. apply(compiler) {
  117. const options = this.options;
  118. let active = true;
  119. const logger = new LoggerFactory(compiler).create();
  120. const loggerCore = logger.from('core');
  121. logger.lock();
  122. const compilerHooks = pluginCompat.hooks(compiler);
  123. if (!compiler.options.cache) {
  124. compiler.options.cache = true;
  125. }
  126. if (!options.cacheDirectory) {
  127. options.cacheDirectory = path.resolve(
  128. findCacheDir({
  129. name: 'hard-source',
  130. cwd: compiler.options.context || process.cwd(),
  131. }),
  132. '[confighash]',
  133. );
  134. }
  135. this.compilerOutputOptions = compiler.options.output;
  136. if (!options.configHash) {
  137. options.configHash = defaultConfigHash;
  138. }
  139. if (options.configHash) {
  140. if (typeof options.configHash === 'string') {
  141. this.configHash = options.configHash;
  142. } else if (typeof options.configHash === 'function') {
  143. this.configHash = options.configHash(compiler.options);
  144. }
  145. compiler.__hardSource_configHash = this.configHash;
  146. compiler.__hardSource_shortConfigHash = this.configHash.substring(0, 8);
  147. }
  148. const configHashInDirectory =
  149. options.cacheDirectory.search(/\[confighash\]/) !== -1;
  150. if (configHashInDirectory && !this.configHash) {
  151. logMessages.configHashSetButNotUsed(compiler, {
  152. cacheDirectory: options.cacheDirectory,
  153. });
  154. active = false;
  155. function unlockLogger() {
  156. logger.unlock();
  157. }
  158. compilerHooks.watchRun.tap('HardSource - index', unlockLogger);
  159. compilerHooks.run.tap('HardSource - index', unlockLogger);
  160. return;
  161. }
  162. let environmentHasher = null;
  163. if (typeof options.environmentHash !== 'undefined') {
  164. if (options.environmentHash === false) {
  165. environmentHasher = () => Promise.resolve('');
  166. } else if (typeof options.environmentHash === 'string') {
  167. environmentHasher = () => Promise.resolve(options.environmentHash);
  168. } else if (typeof options.environmentHash === 'object') {
  169. environmentHasher = () => envHash(options.environmentHash);
  170. environmentHasher.inputs = () =>
  171. envHash.inputs(options.environmentHash);
  172. } else if (typeof options.environmentHash === 'function') {
  173. environmentHasher = () => Promise.resolve(options.environmentHash());
  174. if (options.environmentHash.inputs) {
  175. environmentHasher.inputs = () =>
  176. Promise.resolve(options.environmentHasher.inputs());
  177. }
  178. }
  179. }
  180. if (!environmentHasher) {
  181. environmentHasher = envHash;
  182. }
  183. const cacheDirPath = this.getCachePath();
  184. const cacheAssetDirPath = path.join(cacheDirPath, 'assets');
  185. const resolveCachePath = path.join(cacheDirPath, 'resolve.json');
  186. let currentStamp = '';
  187. const cacheSerializerFactory = new CacheSerializerFactory(compiler);
  188. let createSerializers = true;
  189. let cacheRead = false;
  190. const _this = this;
  191. pluginCompat.register(compiler, '_hardSourceCreateSerializer', 'sync', [
  192. 'cacheSerializerFactory',
  193. 'cacheDirPath',
  194. ]);
  195. pluginCompat.register(compiler, '_hardSourceResetCache', 'sync', []);
  196. pluginCompat.register(compiler, '_hardSourceReadCache', 'asyncParallel', [
  197. 'relativeHelpers',
  198. ]);
  199. pluginCompat.register(
  200. compiler,
  201. '_hardSourceVerifyCache',
  202. 'asyncParallel',
  203. [],
  204. );
  205. pluginCompat.register(compiler, '_hardSourceWriteCache', 'asyncParallel', [
  206. 'compilation',
  207. 'relativeHelpers',
  208. ]);
  209. if (configHashInDirectory) {
  210. const PruneCachesSystem = require('./lib/SystemPruneCaches');
  211. new PruneCachesSystem(
  212. path.dirname(cacheDirPath),
  213. options.cachePrune,
  214. ).apply(compiler);
  215. }
  216. function runReadOrReset(_compiler) {
  217. logger.unlock();
  218. if (!active) {
  219. return Promise.resolve();
  220. }
  221. try {
  222. fs.statSync(cacheAssetDirPath);
  223. } catch (_) {
  224. mkdirp.sync(cacheAssetDirPath);
  225. logMessages.configHashFirstBuild(compiler, {
  226. cacheDirPath,
  227. configHash: compiler.__hardSource_configHash,
  228. });
  229. }
  230. const start = Date.now();
  231. if (createSerializers) {
  232. createSerializers = false;
  233. try {
  234. compilerHooks._hardSourceCreateSerializer.call(
  235. cacheSerializerFactory,
  236. cacheDirPath,
  237. );
  238. } catch (err) {
  239. return Promise.reject(err);
  240. }
  241. }
  242. return Promise.all([
  243. fsReadFile(path.join(cacheDirPath, 'stamp'), 'utf8').catch(() => ''),
  244. environmentHasher(),
  245. fsReadFile(path.join(cacheDirPath, 'version'), 'utf8').catch(() => ''),
  246. environmentHasher.inputs ? environmentHasher.inputs() : null,
  247. ]).then(([stamp, hash, versionStamp, hashInputs]) => {
  248. if (!configHashInDirectory && options.configHash) {
  249. hash += `_${_this.configHash}`;
  250. }
  251. if (hashInputs && !cacheRead) {
  252. logMessages.environmentInputs(compiler, { inputs: hashInputs });
  253. }
  254. currentStamp = hash;
  255. if (!hash || hash !== stamp || hardSourceVersion !== versionStamp) {
  256. if (hash && stamp) {
  257. if (configHashInDirectory) {
  258. logMessages.environmentHashChanged(compiler);
  259. } else {
  260. logMessages.configHashChanged(compiler);
  261. }
  262. } else if (versionStamp && hardSourceVersion !== versionStamp) {
  263. logMessages.hardSourceVersionChanged(compiler);
  264. }
  265. // Reset the cache, we can't use it do to an environment change.
  266. pluginCompat.call(compiler, '_hardSourceResetCache', []);
  267. return rimraf(cacheDirPath);
  268. }
  269. if (cacheRead) {
  270. return Promise.resolve();
  271. }
  272. cacheRead = true;
  273. logMessages.configHashBuildWith(compiler, {
  274. cacheDirPath,
  275. configHash: compiler.__hardSource_configHash,
  276. });
  277. function contextKeys(compiler, fn) {
  278. return source => {
  279. const dest = {};
  280. Object.keys(source).forEach(key => {
  281. dest[fn(compiler, key)] = source[key];
  282. });
  283. return dest;
  284. };
  285. }
  286. function contextValues(compiler, fn) {
  287. return source => {
  288. const dest = {};
  289. Object.keys(source).forEach(key => {
  290. const value = fn(compiler, source[key], key);
  291. if (value) {
  292. dest[key] = value;
  293. } else {
  294. delete dest[key];
  295. }
  296. });
  297. return dest;
  298. };
  299. }
  300. function copyWithDeser(dest, source) {
  301. Object.keys(source).forEach(key => {
  302. const item = source[key];
  303. dest[key] = typeof item === 'string' ? JSON.parse(item) : item;
  304. });
  305. }
  306. return Promise.all([
  307. compilerHooks._hardSourceReadCache.promise({
  308. contextKeys,
  309. contextValues,
  310. contextNormalPath,
  311. contextNormalRequest,
  312. contextNormalModuleId,
  313. copyWithDeser,
  314. }),
  315. ])
  316. .catch(error => {
  317. logMessages.serialBadCache(compiler, error);
  318. return rimraf(cacheDirPath);
  319. })
  320. .then(() => {
  321. // console.log('cache in', Date.now() - start);
  322. });
  323. });
  324. }
  325. compilerHooks.watchRun.tapPromise(
  326. 'HardSource - index - readOrReset',
  327. runReadOrReset,
  328. );
  329. compilerHooks.run.tapPromise(
  330. 'HardSource - index - readOrReset',
  331. runReadOrReset,
  332. );
  333. const detectModule = path => {
  334. try {
  335. require(path);
  336. return true;
  337. } catch (_) {
  338. return false;
  339. }
  340. };
  341. const webpackFeatures = {
  342. concatenatedModule: detectModule(
  343. 'webpack/lib/optimize/ConcatenatedModule',
  344. ),
  345. generator: detectModule('webpack/lib/JavascriptGenerator'),
  346. };
  347. let schemasVersion = 2;
  348. if (webpackFeatures.concatenatedModule) {
  349. schemasVersion = 3;
  350. }
  351. if (webpackFeatures.generator) {
  352. schemasVersion = 4;
  353. }
  354. const ArchetypeSystem = require('./lib/SystemArchetype');
  355. const ParitySystem = require('./lib/SystemParity');
  356. const AssetCache = require('./lib/CacheAsset');
  357. const ModuleCache = require('./lib/CacheModule');
  358. const EnhancedResolveCache = require('./lib/CacheEnhancedResolve');
  359. const Md5Cache = require('./lib/CacheMd5');
  360. const ModuleResolverCache = require('./lib/CacheModuleResolver');
  361. const TransformCompilationPlugin = require('./lib/TransformCompilationPlugin');
  362. const TransformAssetPlugin = require('./lib/TransformAssetPlugin');
  363. let TransformConcatenationModulePlugin;
  364. if (webpackFeatures.concatenatedModule) {
  365. TransformConcatenationModulePlugin = require('./lib/TransformConcatenationModulePlugin');
  366. }
  367. const TransformNormalModulePlugin = require('./lib/TransformNormalModulePlugin');
  368. const TransformNormalModuleFactoryPlugin = require('./lib/TransformNormalModuleFactoryPlugin');
  369. const TransformModuleAssetsPlugin = require('./lib/TransformModuleAssetsPlugin');
  370. const TransformModuleErrorsPlugin = require('./lib/TransformModuleErrorsPlugin');
  371. const SupportExtractTextPlugin = require('./lib/SupportExtractTextPlugin');
  372. let SupportMiniCssExtractPlugin;
  373. if (webpackFeatures.generator) {
  374. SupportMiniCssExtractPlugin = require('./lib/SupportMiniCssExtractPlugin');
  375. }
  376. const TransformDependencyBlockPlugin = require('./lib/TransformDependencyBlockPlugin');
  377. const TransformBasicDependencyPlugin = require('./lib/TransformBasicDependencyPlugin');
  378. let HardHarmonyDependencyPlugin;
  379. const TransformSourcePlugin = require('./lib/TransformSourcePlugin');
  380. const TransformParserPlugin = require('./lib/TransformParserPlugin');
  381. let TransformGeneratorPlugin;
  382. if (webpackFeatures.generator) {
  383. TransformGeneratorPlugin = require('./lib/TransformGeneratorPlugin');
  384. }
  385. const ChalkLoggerPlugin = require('./lib/ChalkLoggerPlugin');
  386. new ArchetypeSystem().apply(compiler);
  387. new ParitySystem().apply(compiler);
  388. new AssetCache().apply(compiler);
  389. new ModuleCache().apply(compiler);
  390. new EnhancedResolveCache().apply(compiler);
  391. new Md5Cache().apply(compiler);
  392. new ModuleResolverCache().apply(compiler);
  393. new TransformCompilationPlugin().apply(compiler);
  394. new TransformAssetPlugin().apply(compiler);
  395. new TransformNormalModulePlugin({
  396. schema: schemasVersion,
  397. }).apply(compiler);
  398. new TransformNormalModuleFactoryPlugin().apply(compiler);
  399. if (TransformConcatenationModulePlugin) {
  400. new TransformConcatenationModulePlugin().apply(compiler);
  401. }
  402. new TransformModuleAssetsPlugin().apply(compiler);
  403. new TransformModuleErrorsPlugin().apply(compiler);
  404. new SupportExtractTextPlugin().apply(compiler);
  405. if (SupportMiniCssExtractPlugin) {
  406. new SupportMiniCssExtractPlugin().apply(compiler);
  407. }
  408. new TransformDependencyBlockPlugin({
  409. schema: schemasVersion,
  410. }).apply(compiler);
  411. new TransformBasicDependencyPlugin({
  412. schema: schemasVersion,
  413. }).apply(compiler);
  414. new TransformSourcePlugin({
  415. schema: schemasVersion,
  416. }).apply(compiler);
  417. new TransformParserPlugin({
  418. schema: schemasVersion,
  419. }).apply(compiler);
  420. if (TransformGeneratorPlugin) {
  421. new TransformGeneratorPlugin({
  422. schema: schemasVersion,
  423. }).apply(compiler);
  424. }
  425. new ChalkLoggerPlugin(this.options.info).apply(compiler);
  426. function runVerify(_compiler) {
  427. if (!active) {
  428. return Promise.resolve();
  429. }
  430. const stats = {};
  431. return pluginCompat.promise(compiler, '_hardSourceVerifyCache', []);
  432. }
  433. compilerHooks.watchRun.tapPromise('HardSource - index - verify', runVerify);
  434. compilerHooks.run.tapPromise('HardSource - index - verify', runVerify);
  435. let freeze;
  436. compilerHooks._hardSourceMethods.tap('HardSource - index', methods => {
  437. freeze = methods.freeze;
  438. });
  439. compilerHooks.afterCompile.tapPromise('HardSource - index', compilation => {
  440. if (!active) {
  441. return Promise.resolve();
  442. }
  443. const startCacheTime = Date.now();
  444. const identifierPrefix = cachePrefix(compilation);
  445. if (identifierPrefix !== null) {
  446. freeze('Compilation', null, compilation, {
  447. compilation,
  448. });
  449. }
  450. return Promise.all([
  451. mkdirp(cacheDirPath).then(() =>
  452. Promise.all([
  453. fsWriteFile(path.join(cacheDirPath, 'stamp'), currentStamp, 'utf8'),
  454. fsWriteFile(
  455. path.join(cacheDirPath, 'version'),
  456. hardSourceVersion,
  457. 'utf8',
  458. ),
  459. ]),
  460. ),
  461. pluginCompat.promise(compiler, '_hardSourceWriteCache', [
  462. compilation,
  463. {
  464. relateNormalPath,
  465. relateNormalRequest,
  466. relateNormalModuleId,
  467. contextNormalPath,
  468. contextNormalRequest,
  469. contextNormalModuleId,
  470. },
  471. ]),
  472. ]).then(() => {
  473. // console.log('cache out', Date.now() - startCacheTime);
  474. });
  475. });
  476. }
  477. }
  478. module.exports = HardSourceWebpackPlugin;
  479. HardSourceWebpackPlugin.ExcludeModulePlugin = ExcludeModulePlugin;
  480. HardSourceWebpackPlugin.HardSourceLevelDbSerializerPlugin = HardSourceLevelDbSerializerPlugin;
  481. HardSourceWebpackPlugin.LevelDbSerializerPlugin = HardSourceLevelDbSerializerPlugin;
  482. HardSourceWebpackPlugin.SerializerAppend2Plugin = SerializerAppend2Plugin;
  483. HardSourceWebpackPlugin.SerializerAppendPlugin = SerializerAppendPlugin;
  484. HardSourceWebpackPlugin.SerializerCacachePlugin = SerializerCacachePlugin;
  485. HardSourceWebpackPlugin.SerializerJsonPlugin = SerializerJsonPlugin;
  486. Object.defineProperty(HardSourceWebpackPlugin, 'ParallelModulePlugin', {
  487. get() {
  488. return require('./lib/ParallelModulePlugin');
  489. },
  490. });