cache.js 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291
  1. var path = require('path');
  2. var crypto = require('crypto');
  3. module.exports = {
  4. createFromFile: function (filePath, useChecksum) {
  5. var fname = path.basename(filePath);
  6. var dir = path.dirname(filePath);
  7. return this.create(fname, dir, useChecksum);
  8. },
  9. create: function (cacheId, _path, useChecksum) {
  10. var fs = require('fs');
  11. var flatCache = require('flat-cache');
  12. var cache = flatCache.load(cacheId, _path);
  13. var normalizedEntries = {};
  14. var removeNotFoundFiles = function removeNotFoundFiles() {
  15. const cachedEntries = cache.keys();
  16. // remove not found entries
  17. cachedEntries.forEach(function remover(fPath) {
  18. try {
  19. fs.statSync(fPath);
  20. } catch (err) {
  21. if (err.code === 'ENOENT') {
  22. cache.removeKey(fPath);
  23. }
  24. }
  25. });
  26. };
  27. removeNotFoundFiles();
  28. return {
  29. /**
  30. * the flat cache storage used to persist the metadata of the `files
  31. * @type {Object}
  32. */
  33. cache: cache,
  34. /**
  35. * Given a buffer, calculate md5 hash of its content.
  36. * @method getHash
  37. * @param {Buffer} buffer buffer to calculate hash on
  38. * @return {String} content hash digest
  39. */
  40. getHash: function (buffer) {
  41. return crypto.createHash('md5').update(buffer).digest('hex');
  42. },
  43. /**
  44. * Return whether or not a file has changed since last time reconcile was called.
  45. * @method hasFileChanged
  46. * @param {String} file the filepath to check
  47. * @return {Boolean} wheter or not the file has changed
  48. */
  49. hasFileChanged: function (file) {
  50. return this.getFileDescriptor(file).changed;
  51. },
  52. /**
  53. * given an array of file paths it return and object with three arrays:
  54. * - changedFiles: Files that changed since previous run
  55. * - notChangedFiles: Files that haven't change
  56. * - notFoundFiles: Files that were not found, probably deleted
  57. *
  58. * @param {Array} files the files to analyze and compare to the previous seen files
  59. * @return {[type]} [description]
  60. */
  61. analyzeFiles: function (files) {
  62. var me = this;
  63. files = files || [];
  64. var res = {
  65. changedFiles: [],
  66. notFoundFiles: [],
  67. notChangedFiles: [],
  68. };
  69. me.normalizeEntries(files).forEach(function (entry) {
  70. if (entry.changed) {
  71. res.changedFiles.push(entry.key);
  72. return;
  73. }
  74. if (entry.notFound) {
  75. res.notFoundFiles.push(entry.key);
  76. return;
  77. }
  78. res.notChangedFiles.push(entry.key);
  79. });
  80. return res;
  81. },
  82. getFileDescriptor: function (file) {
  83. var fstat;
  84. try {
  85. fstat = fs.statSync(file);
  86. } catch (ex) {
  87. this.removeEntry(file);
  88. return { key: file, notFound: true, err: ex };
  89. }
  90. if (useChecksum) {
  91. return this._getFileDescriptorUsingChecksum(file);
  92. }
  93. return this._getFileDescriptorUsingMtimeAndSize(file, fstat);
  94. },
  95. _getFileDescriptorUsingMtimeAndSize: function (file, fstat) {
  96. var meta = cache.getKey(file);
  97. var cacheExists = !!meta;
  98. var cSize = fstat.size;
  99. var cTime = fstat.mtime.getTime();
  100. var isDifferentDate;
  101. var isDifferentSize;
  102. if (!meta) {
  103. meta = { size: cSize, mtime: cTime };
  104. } else {
  105. isDifferentDate = cTime !== meta.mtime;
  106. isDifferentSize = cSize !== meta.size;
  107. }
  108. var nEntry = (normalizedEntries[file] = {
  109. key: file,
  110. changed: !cacheExists || isDifferentDate || isDifferentSize,
  111. meta: meta,
  112. });
  113. return nEntry;
  114. },
  115. _getFileDescriptorUsingChecksum: function (file) {
  116. var meta = cache.getKey(file);
  117. var cacheExists = !!meta;
  118. var contentBuffer;
  119. try {
  120. contentBuffer = fs.readFileSync(file);
  121. } catch (ex) {
  122. contentBuffer = '';
  123. }
  124. var isDifferent = true;
  125. var hash = this.getHash(contentBuffer);
  126. if (!meta) {
  127. meta = { hash: hash };
  128. } else {
  129. isDifferent = hash !== meta.hash;
  130. }
  131. var nEntry = (normalizedEntries[file] = {
  132. key: file,
  133. changed: !cacheExists || isDifferent,
  134. meta: meta,
  135. });
  136. return nEntry;
  137. },
  138. /**
  139. * Return the list o the files that changed compared
  140. * against the ones stored in the cache
  141. *
  142. * @method getUpdated
  143. * @param files {Array} the array of files to compare against the ones in the cache
  144. * @returns {Array}
  145. */
  146. getUpdatedFiles: function (files) {
  147. var me = this;
  148. files = files || [];
  149. return me
  150. .normalizeEntries(files)
  151. .filter(function (entry) {
  152. return entry.changed;
  153. })
  154. .map(function (entry) {
  155. return entry.key;
  156. });
  157. },
  158. /**
  159. * return the list of files
  160. * @method normalizeEntries
  161. * @param files
  162. * @returns {*}
  163. */
  164. normalizeEntries: function (files) {
  165. files = files || [];
  166. var me = this;
  167. var nEntries = files.map(function (file) {
  168. return me.getFileDescriptor(file);
  169. });
  170. //normalizeEntries = nEntries;
  171. return nEntries;
  172. },
  173. /**
  174. * Remove an entry from the file-entry-cache. Useful to force the file to still be considered
  175. * modified the next time the process is run
  176. *
  177. * @method removeEntry
  178. * @param entryName
  179. */
  180. removeEntry: function (entryName) {
  181. delete normalizedEntries[entryName];
  182. cache.removeKey(entryName);
  183. },
  184. /**
  185. * Delete the cache file from the disk
  186. * @method deleteCacheFile
  187. */
  188. deleteCacheFile: function () {
  189. cache.removeCacheFile();
  190. },
  191. /**
  192. * remove the cache from the file and clear the memory cache
  193. */
  194. destroy: function () {
  195. normalizedEntries = {};
  196. cache.destroy();
  197. },
  198. _getMetaForFileUsingCheckSum: function (cacheEntry) {
  199. var contentBuffer = fs.readFileSync(cacheEntry.key);
  200. var hash = this.getHash(contentBuffer);
  201. var meta = Object.assign(cacheEntry.meta, { hash: hash });
  202. delete meta.size;
  203. delete meta.mtime;
  204. return meta;
  205. },
  206. _getMetaForFileUsingMtimeAndSize: function (cacheEntry) {
  207. var stat = fs.statSync(cacheEntry.key);
  208. var meta = Object.assign(cacheEntry.meta, {
  209. size: stat.size,
  210. mtime: stat.mtime.getTime(),
  211. });
  212. delete meta.hash;
  213. return meta;
  214. },
  215. /**
  216. * Sync the files and persist them to the cache
  217. * @method reconcile
  218. */
  219. reconcile: function (noPrune) {
  220. removeNotFoundFiles();
  221. noPrune = typeof noPrune === 'undefined' ? true : noPrune;
  222. var entries = normalizedEntries;
  223. var keys = Object.keys(entries);
  224. if (keys.length === 0) {
  225. return;
  226. }
  227. var me = this;
  228. keys.forEach(function (entryName) {
  229. var cacheEntry = entries[entryName];
  230. try {
  231. var meta = useChecksum
  232. ? me._getMetaForFileUsingCheckSum(cacheEntry)
  233. : me._getMetaForFileUsingMtimeAndSize(cacheEntry);
  234. cache.setKey(entryName, meta);
  235. } catch (err) {
  236. // if the file does not exists we don't save it
  237. // other errors are just thrown
  238. if (err.code !== 'ENOENT') {
  239. throw err;
  240. }
  241. }
  242. });
  243. cache.save(noPrune);
  244. },
  245. };
  246. },
  247. };