snappy.go 3.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141
  1. package snappy
  2. import (
  3. "bytes"
  4. "encoding/binary"
  5. "errors"
  6. master "github.com/golang/snappy"
  7. )
  8. const (
  9. sizeOffset = 16
  10. sizeBytes = 4
  11. )
  12. var (
  13. xerialHeader = []byte{130, 83, 78, 65, 80, 80, 89, 0}
  14. // This is xerial version 1 and minimally compatible with version 1
  15. xerialVersionInfo = []byte{0, 0, 0, 1, 0, 0, 0, 1}
  16. // ErrMalformed is returned by the decoder when the xerial framing
  17. // is malformed
  18. ErrMalformed = errors.New("malformed xerial framing")
  19. )
  20. func min(x, y int) int {
  21. if x < y {
  22. return x
  23. }
  24. return y
  25. }
  26. // Encode encodes data as snappy with no framing header.
  27. func Encode(src []byte) []byte {
  28. return master.Encode(nil, src)
  29. }
  30. // EncodeStream *appends* to the specified 'dst' the compressed
  31. // 'src' in xerial framing format. If 'dst' does not have enough
  32. // capacity, then a new slice will be allocated. If 'dst' has
  33. // non-zero length, then if *must* have been built using this function.
  34. func EncodeStream(dst, src []byte) []byte {
  35. if len(dst) == 0 {
  36. dst = append(dst, xerialHeader...)
  37. dst = append(dst, xerialVersionInfo...)
  38. }
  39. // Snappy encode in blocks of maximum 32KB
  40. var (
  41. max = len(src)
  42. blockSize = 32 * 1024
  43. pos = 0
  44. chunk []byte
  45. )
  46. for pos < max {
  47. newPos := min(pos+blockSize, max)
  48. chunk = master.Encode(chunk[:cap(chunk)], src[pos:newPos])
  49. // First encode the compressed size (big-endian)
  50. // Put* panics if the buffer is too small, so pad 4 bytes first
  51. origLen := len(dst)
  52. dst = append(dst, dst[0:4]...)
  53. binary.BigEndian.PutUint32(dst[origLen:], uint32(len(chunk)))
  54. // And now the compressed data
  55. dst = append(dst, chunk...)
  56. pos = newPos
  57. }
  58. return dst
  59. }
  60. // Decode decodes snappy data whether it is traditional unframed
  61. // or includes the xerial framing format.
  62. func Decode(src []byte) ([]byte, error) {
  63. return DecodeInto(nil, src)
  64. }
  65. // DecodeInto decodes snappy data whether it is traditional unframed
  66. // or includes the xerial framing format into the specified `dst`.
  67. // It is assumed that the entirety of `dst` including all capacity is available
  68. // for use by this function. If `dst` is nil *or* insufficiently large to hold
  69. // the decoded `src`, new space will be allocated.
  70. func DecodeInto(dst, src []byte) ([]byte, error) {
  71. if len(src) < 8 || !bytes.Equal(src[:8], xerialHeader) {
  72. dst, err := master.Decode(dst[:cap(dst)], src)
  73. if err != nil && len(src) < len(xerialHeader) {
  74. // Keep compatibility and return ErrMalformed when there is a
  75. // short or truncated header.
  76. return nil, ErrMalformed
  77. }
  78. return dst, err
  79. }
  80. var max = len(src)
  81. if max < len(xerialHeader) {
  82. return nil, ErrMalformed
  83. }
  84. if max == sizeOffset {
  85. return []byte{}, nil
  86. }
  87. if max < sizeOffset+sizeBytes {
  88. return nil, ErrMalformed
  89. }
  90. if dst == nil {
  91. dst = make([]byte, 0, len(src))
  92. }
  93. dst = dst[:0]
  94. var (
  95. pos = sizeOffset
  96. chunk []byte
  97. err error
  98. )
  99. for pos+sizeBytes <= max {
  100. size := int(binary.BigEndian.Uint32(src[pos : pos+sizeBytes]))
  101. pos += sizeBytes
  102. nextPos := pos + size
  103. // On architectures where int is 32-bytes wide size + pos could
  104. // overflow so we need to check the low bound as well as the
  105. // high
  106. if nextPos < pos || nextPos > max {
  107. return nil, ErrMalformed
  108. }
  109. chunk, err = master.Decode(chunk[:cap(chunk)], src[pos:nextPos])
  110. if err != nil {
  111. return nil, err
  112. }
  113. pos = nextPos
  114. dst = append(dst, chunk...)
  115. }
  116. return dst, nil
  117. }