xxhash.go 5.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235
  1. // Package xxhash implements the 64-bit variant of xxHash (XXH64) as described
  2. // at http://cyan4973.github.io/xxHash/.
  3. package xxhash
  4. import (
  5. "encoding/binary"
  6. "errors"
  7. "math/bits"
  8. )
  9. const (
  10. prime1 uint64 = 11400714785074694791
  11. prime2 uint64 = 14029467366897019727
  12. prime3 uint64 = 1609587929392839161
  13. prime4 uint64 = 9650029242287828579
  14. prime5 uint64 = 2870177450012600261
  15. )
  16. // NOTE(caleb): I'm using both consts and vars of the primes. Using consts where
  17. // possible in the Go code is worth a small (but measurable) performance boost
  18. // by avoiding some MOVQs. Vars are needed for the asm and also are useful for
  19. // convenience in the Go code in a few places where we need to intentionally
  20. // avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the
  21. // result overflows a uint64).
  22. var (
  23. prime1v = prime1
  24. prime2v = prime2
  25. prime3v = prime3
  26. prime4v = prime4
  27. prime5v = prime5
  28. )
  29. // Digest implements hash.Hash64.
  30. type Digest struct {
  31. v1 uint64
  32. v2 uint64
  33. v3 uint64
  34. v4 uint64
  35. total uint64
  36. mem [32]byte
  37. n int // how much of mem is used
  38. }
  39. // New creates a new Digest that computes the 64-bit xxHash algorithm.
  40. func New() *Digest {
  41. var d Digest
  42. d.Reset()
  43. return &d
  44. }
  45. // Reset clears the Digest's state so that it can be reused.
  46. func (d *Digest) Reset() {
  47. d.v1 = prime1v + prime2
  48. d.v2 = prime2
  49. d.v3 = 0
  50. d.v4 = -prime1v
  51. d.total = 0
  52. d.n = 0
  53. }
  54. // Size always returns 8 bytes.
  55. func (d *Digest) Size() int { return 8 }
  56. // BlockSize always returns 32 bytes.
  57. func (d *Digest) BlockSize() int { return 32 }
  58. // Write adds more data to d. It always returns len(b), nil.
  59. func (d *Digest) Write(b []byte) (n int, err error) {
  60. n = len(b)
  61. d.total += uint64(n)
  62. if d.n+n < 32 {
  63. // This new data doesn't even fill the current block.
  64. copy(d.mem[d.n:], b)
  65. d.n += n
  66. return
  67. }
  68. if d.n > 0 {
  69. // Finish off the partial block.
  70. copy(d.mem[d.n:], b)
  71. d.v1 = round(d.v1, u64(d.mem[0:8]))
  72. d.v2 = round(d.v2, u64(d.mem[8:16]))
  73. d.v3 = round(d.v3, u64(d.mem[16:24]))
  74. d.v4 = round(d.v4, u64(d.mem[24:32]))
  75. b = b[32-d.n:]
  76. d.n = 0
  77. }
  78. if len(b) >= 32 {
  79. // One or more full blocks left.
  80. nw := writeBlocks(d, b)
  81. b = b[nw:]
  82. }
  83. // Store any remaining partial block.
  84. copy(d.mem[:], b)
  85. d.n = len(b)
  86. return
  87. }
  88. // Sum appends the current hash to b and returns the resulting slice.
  89. func (d *Digest) Sum(b []byte) []byte {
  90. s := d.Sum64()
  91. return append(
  92. b,
  93. byte(s>>56),
  94. byte(s>>48),
  95. byte(s>>40),
  96. byte(s>>32),
  97. byte(s>>24),
  98. byte(s>>16),
  99. byte(s>>8),
  100. byte(s),
  101. )
  102. }
  103. // Sum64 returns the current hash.
  104. func (d *Digest) Sum64() uint64 {
  105. var h uint64
  106. if d.total >= 32 {
  107. v1, v2, v3, v4 := d.v1, d.v2, d.v3, d.v4
  108. h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
  109. h = mergeRound(h, v1)
  110. h = mergeRound(h, v2)
  111. h = mergeRound(h, v3)
  112. h = mergeRound(h, v4)
  113. } else {
  114. h = d.v3 + prime5
  115. }
  116. h += d.total
  117. i, end := 0, d.n
  118. for ; i+8 <= end; i += 8 {
  119. k1 := round(0, u64(d.mem[i:i+8]))
  120. h ^= k1
  121. h = rol27(h)*prime1 + prime4
  122. }
  123. if i+4 <= end {
  124. h ^= uint64(u32(d.mem[i:i+4])) * prime1
  125. h = rol23(h)*prime2 + prime3
  126. i += 4
  127. }
  128. for i < end {
  129. h ^= uint64(d.mem[i]) * prime5
  130. h = rol11(h) * prime1
  131. i++
  132. }
  133. h ^= h >> 33
  134. h *= prime2
  135. h ^= h >> 29
  136. h *= prime3
  137. h ^= h >> 32
  138. return h
  139. }
  140. const (
  141. magic = "xxh\x06"
  142. marshaledSize = len(magic) + 8*5 + 32
  143. )
  144. // MarshalBinary implements the encoding.BinaryMarshaler interface.
  145. func (d *Digest) MarshalBinary() ([]byte, error) {
  146. b := make([]byte, 0, marshaledSize)
  147. b = append(b, magic...)
  148. b = appendUint64(b, d.v1)
  149. b = appendUint64(b, d.v2)
  150. b = appendUint64(b, d.v3)
  151. b = appendUint64(b, d.v4)
  152. b = appendUint64(b, d.total)
  153. b = append(b, d.mem[:d.n]...)
  154. b = b[:len(b)+len(d.mem)-d.n]
  155. return b, nil
  156. }
  157. // UnmarshalBinary implements the encoding.BinaryUnmarshaler interface.
  158. func (d *Digest) UnmarshalBinary(b []byte) error {
  159. if len(b) < len(magic) || string(b[:len(magic)]) != magic {
  160. return errors.New("xxhash: invalid hash state identifier")
  161. }
  162. if len(b) != marshaledSize {
  163. return errors.New("xxhash: invalid hash state size")
  164. }
  165. b = b[len(magic):]
  166. b, d.v1 = consumeUint64(b)
  167. b, d.v2 = consumeUint64(b)
  168. b, d.v3 = consumeUint64(b)
  169. b, d.v4 = consumeUint64(b)
  170. b, d.total = consumeUint64(b)
  171. copy(d.mem[:], b)
  172. d.n = int(d.total % uint64(len(d.mem)))
  173. return nil
  174. }
  175. func appendUint64(b []byte, x uint64) []byte {
  176. var a [8]byte
  177. binary.LittleEndian.PutUint64(a[:], x)
  178. return append(b, a[:]...)
  179. }
  180. func consumeUint64(b []byte) ([]byte, uint64) {
  181. x := u64(b)
  182. return b[8:], x
  183. }
  184. func u64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) }
  185. func u32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) }
  186. func round(acc, input uint64) uint64 {
  187. acc += input * prime2
  188. acc = rol31(acc)
  189. acc *= prime1
  190. return acc
  191. }
  192. func mergeRound(acc, val uint64) uint64 {
  193. val = round(0, val)
  194. acc ^= val
  195. acc = acc*prime1 + prime4
  196. return acc
  197. }
  198. func rol1(x uint64) uint64 { return bits.RotateLeft64(x, 1) }
  199. func rol7(x uint64) uint64 { return bits.RotateLeft64(x, 7) }
  200. func rol11(x uint64) uint64 { return bits.RotateLeft64(x, 11) }
  201. func rol12(x uint64) uint64 { return bits.RotateLeft64(x, 12) }
  202. func rol18(x uint64) uint64 { return bits.RotateLeft64(x, 18) }
  203. func rol23(x uint64) uint64 { return bits.RotateLeft64(x, 23) }
  204. func rol27(x uint64) uint64 { return bits.RotateLeft64(x, 27) }
  205. func rol31(x uint64) uint64 { return bits.RotateLeft64(x, 31) }