xxhash64.js 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444
  1. /**
  2. xxHash64 implementation in pure Javascript
  3. Copyright (C) 2016, Pierre Curto
  4. MIT license
  5. */
  6. var UINT64 = require('cuint').UINT64
  7. /*
  8. * Constants
  9. */
  10. var PRIME64_1 = UINT64( '11400714785074694791' )
  11. var PRIME64_2 = UINT64( '14029467366897019727' )
  12. var PRIME64_3 = UINT64( '1609587929392839161' )
  13. var PRIME64_4 = UINT64( '9650029242287828579' )
  14. var PRIME64_5 = UINT64( '2870177450012600261' )
  15. /**
  16. * Convert string to proper UTF-8 array
  17. * @param str Input string
  18. * @returns {Uint8Array} UTF8 array is returned as uint8 array
  19. */
  20. function toUTF8Array (str) {
  21. var utf8 = []
  22. for (var i=0, n=str.length; i < n; i++) {
  23. var charcode = str.charCodeAt(i)
  24. if (charcode < 0x80) utf8.push(charcode)
  25. else if (charcode < 0x800) {
  26. utf8.push(0xc0 | (charcode >> 6),
  27. 0x80 | (charcode & 0x3f))
  28. }
  29. else if (charcode < 0xd800 || charcode >= 0xe000) {
  30. utf8.push(0xe0 | (charcode >> 12),
  31. 0x80 | ((charcode>>6) & 0x3f),
  32. 0x80 | (charcode & 0x3f))
  33. }
  34. // surrogate pair
  35. else {
  36. i++;
  37. // UTF-16 encodes 0x10000-0x10FFFF by
  38. // subtracting 0x10000 and splitting the
  39. // 20 bits of 0x0-0xFFFFF into two halves
  40. charcode = 0x10000 + (((charcode & 0x3ff)<<10)
  41. | (str.charCodeAt(i) & 0x3ff))
  42. utf8.push(0xf0 | (charcode >>18),
  43. 0x80 | ((charcode>>12) & 0x3f),
  44. 0x80 | ((charcode>>6) & 0x3f),
  45. 0x80 | (charcode & 0x3f))
  46. }
  47. }
  48. return new Uint8Array(utf8)
  49. }
  50. /**
  51. * XXH64 object used as a constructor or a function
  52. * @constructor
  53. * or
  54. * @param {Object|String} input data
  55. * @param {Number|UINT64} seed
  56. * @return ThisExpression
  57. * or
  58. * @return {UINT64} xxHash
  59. */
  60. function XXH64 () {
  61. if (arguments.length == 2)
  62. return new XXH64( arguments[1] ).update( arguments[0] ).digest()
  63. if (!(this instanceof XXH64))
  64. return new XXH64( arguments[0] )
  65. init.call(this, arguments[0])
  66. }
  67. /**
  68. * Initialize the XXH64 instance with the given seed
  69. * @method init
  70. * @param {Number|Object} seed as a number or an unsigned 32 bits integer
  71. * @return ThisExpression
  72. */
  73. function init (seed) {
  74. this.seed = seed instanceof UINT64 ? seed.clone() : UINT64(seed)
  75. this.v1 = this.seed.clone().add(PRIME64_1).add(PRIME64_2)
  76. this.v2 = this.seed.clone().add(PRIME64_2)
  77. this.v3 = this.seed.clone()
  78. this.v4 = this.seed.clone().subtract(PRIME64_1)
  79. this.total_len = 0
  80. this.memsize = 0
  81. this.memory = null
  82. return this
  83. }
  84. XXH64.prototype.init = init
  85. /**
  86. * Add data to be computed for the XXH64 hash
  87. * @method update
  88. * @param {String|Buffer|ArrayBuffer} input as a string or nodejs Buffer or ArrayBuffer
  89. * @return ThisExpression
  90. */
  91. XXH64.prototype.update = function (input) {
  92. var isString = typeof input == 'string'
  93. var isArrayBuffer
  94. // Convert all strings to utf-8 first (issue #5)
  95. if (isString) {
  96. input = toUTF8Array(input)
  97. isString = false
  98. isArrayBuffer = true
  99. }
  100. if (typeof ArrayBuffer !== "undefined" && input instanceof ArrayBuffer)
  101. {
  102. isArrayBuffer = true
  103. input = new Uint8Array(input);
  104. }
  105. var p = 0
  106. var len = input.length
  107. var bEnd = p + len
  108. if (len == 0) return this
  109. this.total_len += len
  110. if (this.memsize == 0)
  111. {
  112. if (isString) {
  113. this.memory = ''
  114. } else if (isArrayBuffer) {
  115. this.memory = new Uint8Array(32)
  116. } else {
  117. this.memory = new Buffer(32)
  118. }
  119. }
  120. if (this.memsize + len < 32) // fill in tmp buffer
  121. {
  122. // XXH64_memcpy(this.memory + this.memsize, input, len)
  123. if (isString) {
  124. this.memory += input
  125. } else if (isArrayBuffer) {
  126. this.memory.set( input.subarray(0, len), this.memsize )
  127. } else {
  128. input.copy( this.memory, this.memsize, 0, len )
  129. }
  130. this.memsize += len
  131. return this
  132. }
  133. if (this.memsize > 0) // some data left from previous update
  134. {
  135. // XXH64_memcpy(this.memory + this.memsize, input, 16-this.memsize);
  136. if (isString) {
  137. this.memory += input.slice(0, 32 - this.memsize)
  138. } else if (isArrayBuffer) {
  139. this.memory.set( input.subarray(0, 32 - this.memsize), this.memsize )
  140. } else {
  141. input.copy( this.memory, this.memsize, 0, 32 - this.memsize )
  142. }
  143. var p64 = 0
  144. if (isString) {
  145. var other
  146. other = UINT64(
  147. (this.memory.charCodeAt(p64+1) << 8) | this.memory.charCodeAt(p64)
  148. , (this.memory.charCodeAt(p64+3) << 8) | this.memory.charCodeAt(p64+2)
  149. , (this.memory.charCodeAt(p64+5) << 8) | this.memory.charCodeAt(p64+4)
  150. , (this.memory.charCodeAt(p64+7) << 8) | this.memory.charCodeAt(p64+6)
  151. )
  152. this.v1.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  153. p64 += 8
  154. other = UINT64(
  155. (this.memory.charCodeAt(p64+1) << 8) | this.memory.charCodeAt(p64)
  156. , (this.memory.charCodeAt(p64+3) << 8) | this.memory.charCodeAt(p64+2)
  157. , (this.memory.charCodeAt(p64+5) << 8) | this.memory.charCodeAt(p64+4)
  158. , (this.memory.charCodeAt(p64+7) << 8) | this.memory.charCodeAt(p64+6)
  159. )
  160. this.v2.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  161. p64 += 8
  162. other = UINT64(
  163. (this.memory.charCodeAt(p64+1) << 8) | this.memory.charCodeAt(p64)
  164. , (this.memory.charCodeAt(p64+3) << 8) | this.memory.charCodeAt(p64+2)
  165. , (this.memory.charCodeAt(p64+5) << 8) | this.memory.charCodeAt(p64+4)
  166. , (this.memory.charCodeAt(p64+7) << 8) | this.memory.charCodeAt(p64+6)
  167. )
  168. this.v3.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  169. p64 += 8
  170. other = UINT64(
  171. (this.memory.charCodeAt(p64+1) << 8) | this.memory.charCodeAt(p64)
  172. , (this.memory.charCodeAt(p64+3) << 8) | this.memory.charCodeAt(p64+2)
  173. , (this.memory.charCodeAt(p64+5) << 8) | this.memory.charCodeAt(p64+4)
  174. , (this.memory.charCodeAt(p64+7) << 8) | this.memory.charCodeAt(p64+6)
  175. )
  176. this.v4.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  177. } else {
  178. var other
  179. other = UINT64(
  180. (this.memory[p64+1] << 8) | this.memory[p64]
  181. , (this.memory[p64+3] << 8) | this.memory[p64+2]
  182. , (this.memory[p64+5] << 8) | this.memory[p64+4]
  183. , (this.memory[p64+7] << 8) | this.memory[p64+6]
  184. )
  185. this.v1.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  186. p64 += 8
  187. other = UINT64(
  188. (this.memory[p64+1] << 8) | this.memory[p64]
  189. , (this.memory[p64+3] << 8) | this.memory[p64+2]
  190. , (this.memory[p64+5] << 8) | this.memory[p64+4]
  191. , (this.memory[p64+7] << 8) | this.memory[p64+6]
  192. )
  193. this.v2.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  194. p64 += 8
  195. other = UINT64(
  196. (this.memory[p64+1] << 8) | this.memory[p64]
  197. , (this.memory[p64+3] << 8) | this.memory[p64+2]
  198. , (this.memory[p64+5] << 8) | this.memory[p64+4]
  199. , (this.memory[p64+7] << 8) | this.memory[p64+6]
  200. )
  201. this.v3.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  202. p64 += 8
  203. other = UINT64(
  204. (this.memory[p64+1] << 8) | this.memory[p64]
  205. , (this.memory[p64+3] << 8) | this.memory[p64+2]
  206. , (this.memory[p64+5] << 8) | this.memory[p64+4]
  207. , (this.memory[p64+7] << 8) | this.memory[p64+6]
  208. )
  209. this.v4.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  210. }
  211. p += 32 - this.memsize
  212. this.memsize = 0
  213. if (isString) this.memory = ''
  214. }
  215. if (p <= bEnd - 32)
  216. {
  217. var limit = bEnd - 32
  218. do
  219. {
  220. if (isString) {
  221. var other
  222. other = UINT64(
  223. (input.charCodeAt(p+1) << 8) | input.charCodeAt(p)
  224. , (input.charCodeAt(p+3) << 8) | input.charCodeAt(p+2)
  225. , (input.charCodeAt(p+5) << 8) | input.charCodeAt(p+4)
  226. , (input.charCodeAt(p+7) << 8) | input.charCodeAt(p+6)
  227. )
  228. this.v1.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  229. p += 8
  230. other = UINT64(
  231. (input.charCodeAt(p+1) << 8) | input.charCodeAt(p)
  232. , (input.charCodeAt(p+3) << 8) | input.charCodeAt(p+2)
  233. , (input.charCodeAt(p+5) << 8) | input.charCodeAt(p+4)
  234. , (input.charCodeAt(p+7) << 8) | input.charCodeAt(p+6)
  235. )
  236. this.v2.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  237. p += 8
  238. other = UINT64(
  239. (input.charCodeAt(p+1) << 8) | input.charCodeAt(p)
  240. , (input.charCodeAt(p+3) << 8) | input.charCodeAt(p+2)
  241. , (input.charCodeAt(p+5) << 8) | input.charCodeAt(p+4)
  242. , (input.charCodeAt(p+7) << 8) | input.charCodeAt(p+6)
  243. )
  244. this.v3.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  245. p += 8
  246. other = UINT64(
  247. (input.charCodeAt(p+1) << 8) | input.charCodeAt(p)
  248. , (input.charCodeAt(p+3) << 8) | input.charCodeAt(p+2)
  249. , (input.charCodeAt(p+5) << 8) | input.charCodeAt(p+4)
  250. , (input.charCodeAt(p+7) << 8) | input.charCodeAt(p+6)
  251. )
  252. this.v4.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  253. } else {
  254. var other
  255. other = UINT64(
  256. (input[p+1] << 8) | input[p]
  257. , (input[p+3] << 8) | input[p+2]
  258. , (input[p+5] << 8) | input[p+4]
  259. , (input[p+7] << 8) | input[p+6]
  260. )
  261. this.v1.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  262. p += 8
  263. other = UINT64(
  264. (input[p+1] << 8) | input[p]
  265. , (input[p+3] << 8) | input[p+2]
  266. , (input[p+5] << 8) | input[p+4]
  267. , (input[p+7] << 8) | input[p+6]
  268. )
  269. this.v2.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  270. p += 8
  271. other = UINT64(
  272. (input[p+1] << 8) | input[p]
  273. , (input[p+3] << 8) | input[p+2]
  274. , (input[p+5] << 8) | input[p+4]
  275. , (input[p+7] << 8) | input[p+6]
  276. )
  277. this.v3.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  278. p += 8
  279. other = UINT64(
  280. (input[p+1] << 8) | input[p]
  281. , (input[p+3] << 8) | input[p+2]
  282. , (input[p+5] << 8) | input[p+4]
  283. , (input[p+7] << 8) | input[p+6]
  284. )
  285. this.v4.add( other.multiply(PRIME64_2) ).rotl(31).multiply(PRIME64_1);
  286. }
  287. p += 8
  288. } while (p <= limit)
  289. }
  290. if (p < bEnd)
  291. {
  292. // XXH64_memcpy(this.memory, p, bEnd-p);
  293. if (isString) {
  294. this.memory += input.slice(p)
  295. } else if (isArrayBuffer) {
  296. this.memory.set( input.subarray(p, bEnd), this.memsize )
  297. } else {
  298. input.copy( this.memory, this.memsize, p, bEnd )
  299. }
  300. this.memsize = bEnd - p
  301. }
  302. return this
  303. }
  304. /**
  305. * Finalize the XXH64 computation. The XXH64 instance is ready for reuse for the given seed
  306. * @method digest
  307. * @return {UINT64} xxHash
  308. */
  309. XXH64.prototype.digest = function () {
  310. var input = this.memory
  311. var isString = typeof input == 'string'
  312. var p = 0
  313. var bEnd = this.memsize
  314. var h64, h
  315. var u = new UINT64
  316. if (this.total_len >= 32)
  317. {
  318. h64 = this.v1.clone().rotl(1)
  319. h64.add( this.v2.clone().rotl(7) )
  320. h64.add( this.v3.clone().rotl(12) )
  321. h64.add( this.v4.clone().rotl(18) )
  322. h64.xor( this.v1.multiply(PRIME64_2).rotl(31).multiply(PRIME64_1) )
  323. h64.multiply(PRIME64_1).add(PRIME64_4)
  324. h64.xor( this.v2.multiply(PRIME64_2).rotl(31).multiply(PRIME64_1) )
  325. h64.multiply(PRIME64_1).add(PRIME64_4)
  326. h64.xor( this.v3.multiply(PRIME64_2).rotl(31).multiply(PRIME64_1) )
  327. h64.multiply(PRIME64_1).add(PRIME64_4)
  328. h64.xor( this.v4.multiply(PRIME64_2).rotl(31).multiply(PRIME64_1) )
  329. h64.multiply(PRIME64_1).add(PRIME64_4)
  330. }
  331. else
  332. {
  333. h64 = this.seed.clone().add( PRIME64_5 )
  334. }
  335. h64.add( u.fromNumber(this.total_len) )
  336. while (p <= bEnd - 8)
  337. {
  338. if (isString) {
  339. u.fromBits(
  340. (input.charCodeAt(p+1) << 8) | input.charCodeAt(p)
  341. , (input.charCodeAt(p+3) << 8) | input.charCodeAt(p+2)
  342. , (input.charCodeAt(p+5) << 8) | input.charCodeAt(p+4)
  343. , (input.charCodeAt(p+7) << 8) | input.charCodeAt(p+6)
  344. )
  345. } else {
  346. u.fromBits(
  347. (input[p+1] << 8) | input[p]
  348. , (input[p+3] << 8) | input[p+2]
  349. , (input[p+5] << 8) | input[p+4]
  350. , (input[p+7] << 8) | input[p+6]
  351. )
  352. }
  353. u.multiply(PRIME64_2).rotl(31).multiply(PRIME64_1)
  354. h64
  355. .xor(u)
  356. .rotl(27)
  357. .multiply( PRIME64_1 )
  358. .add( PRIME64_4 )
  359. p += 8
  360. }
  361. if (p + 4 <= bEnd) {
  362. if (isString) {
  363. u.fromBits(
  364. (input.charCodeAt(p+1) << 8) | input.charCodeAt(p)
  365. , (input.charCodeAt(p+3) << 8) | input.charCodeAt(p+2)
  366. , 0
  367. , 0
  368. )
  369. } else {
  370. u.fromBits(
  371. (input[p+1] << 8) | input[p]
  372. , (input[p+3] << 8) | input[p+2]
  373. , 0
  374. , 0
  375. )
  376. }
  377. h64
  378. .xor( u.multiply(PRIME64_1) )
  379. .rotl(23)
  380. .multiply( PRIME64_2 )
  381. .add( PRIME64_3 )
  382. p += 4
  383. }
  384. while (p < bEnd)
  385. {
  386. u.fromBits( isString ? input.charCodeAt(p++) : input[p++], 0, 0, 0 )
  387. h64
  388. .xor( u.multiply(PRIME64_5) )
  389. .rotl(11)
  390. .multiply(PRIME64_1)
  391. }
  392. h = h64.clone().shiftRight(33)
  393. h64.xor(h).multiply(PRIME64_2)
  394. h = h64.clone().shiftRight(29)
  395. h64.xor(h).multiply(PRIME64_3)
  396. h = h64.clone().shiftRight(32)
  397. h64.xor(h)
  398. // Reset the state
  399. this.init( this.seed )
  400. return h64
  401. }
  402. module.exports = XXH64