hash/fnv: add 128-bit FNV hash support

The 128bit FNV hash will be used e.g. in QUIC.

The algorithm is described at
https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function

Change-Id: I13f3ec39b0e12b7a5008824a6619dff2e708ee81
Reviewed-on: https://go-review.googlesource.com/38356
Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>

diff --git a/src/hash/fnv/fnv.go b/src/hash/fnv/fnv.go
index f1fbb25bdf5899a32fc61a24406f4860ed1075ac..3d2df73d3dc330275df9f40d4654322acbf323e7 100644 (file)
--- a/src/hash/fnv/fnv.go
+++ b/src/hash/fnv/fnv.go
@@ -13,17 +13,23 @@ import (
 )
 
 type (
-       sum32  uint32
-       sum32a uint32
-       sum64  uint64
-       sum64a uint64
+       sum32   uint32
+       sum32a  uint32
+       sum64   uint64
+       sum64a  uint64
+       sum128  [2]uint64
+       sum128a [2]uint64
 )
 
 const (
-       offset32 = 2166136261
-       offset64 = 14695981039346656037
-       prime32  = 16777619
-       prime64  = 1099511628211
+       offset32        = 2166136261
+       offset64        = 14695981039346656037
+       offset128Lower  = 0x62b821756295c58d
+       offset128Higher = 0x6c62272e07bb0142
+       prime32         = 16777619
+       prime64         = 1099511628211
+       prime128Lower   = 0x13b
+       prime128Shift   = 24
 )
 
 // New32 returns a new 32-bit FNV-1 hash.Hash.
@@ -54,10 +60,30 @@ func New64a() hash.Hash64 {
        return &s
 }
 
-func (s *sum32) Reset()  { *s = offset32 }
-func (s *sum32a) Reset() { *s = offset32 }
-func (s *sum64) Reset()  { *s = offset64 }
-func (s *sum64a) Reset() { *s = offset64 }
+// New128 returns a new 128-bit FNV-1 hash.Hash.
+// Its Sum method will lay the value out in big-endian byte order.
+func New128() hash.Hash {
+       var s sum128
+       s[0] = offset128Higher
+       s[1] = offset128Lower
+       return &s
+}
+
+// New128a returns a new 128-bit FNV-1a hash.Hash.
+// Its Sum method will lay the value out in big-endian byte order.
+func New128a() hash.Hash {
+       var s sum128a
+       s[0] = offset128Higher
+       s[1] = offset128Lower
+       return &s
+}
+
+func (s *sum32) Reset()   { *s = offset32 }
+func (s *sum32a) Reset()  { *s = offset32 }
+func (s *sum64) Reset()   { *s = offset64 }
+func (s *sum64a) Reset()  { *s = offset64 }
+func (s *sum128) Reset()  { s[0] = offset128Higher; s[1] = offset128Lower }
+func (s *sum128a) Reset() { s[0] = offset128Higher; s[1] = offset128Lower }
 
 func (s *sum32) Sum32() uint32  { return uint32(*s) }
 func (s *sum32a) Sum32() uint32 { return uint32(*s) }
@@ -104,15 +130,57 @@ func (s *sum64a) Write(data []byte) (int, error) {
        return len(data), nil
 }
 
-func (s *sum32) Size() int  { return 4 }
-func (s *sum32a) Size() int { return 4 }
-func (s *sum64) Size() int  { return 8 }
-func (s *sum64a) Size() int { return 8 }
+func (s *sum128) Write(data []byte) (int, error) {
+       for _, c := range data {
+               // Compute the multiplication in 4 parts to simplify carrying
+               s1l := (s[1] & 0xffffffff) * prime128Lower
+               s1h := (s[1] >> 32) * prime128Lower
+               s0l := (s[0]&0xffffffff)*prime128Lower + (s[1]&0xffffffff)<<prime128Shift
+               s0h := (s[0]>>32)*prime128Lower + (s[1]>>32)<<prime128Shift
+               // Carries
+               s1h += s1l >> 32
+               s0l += s1h >> 32
+               s0h += s0l >> 32
+               // Update the values
+               s[1] = (s1l & 0xffffffff) + (s1h << 32)
+               s[0] = (s0l & 0xffffffff) + (s0h << 32)
+               s[1] ^= uint64(c)
+       }
+       return len(data), nil
+}
+
+func (s *sum128a) Write(data []byte) (int, error) {
+       for _, c := range data {
+               s[1] ^= uint64(c)
+               // Compute the multiplication in 4 parts to simplify carrying
+               s1l := (s[1] & 0xffffffff) * prime128Lower
+               s1h := (s[1] >> 32) * prime128Lower
+               s0l := (s[0]&0xffffffff)*prime128Lower + (s[1]&0xffffffff)<<prime128Shift
+               s0h := (s[0]>>32)*prime128Lower + (s[1]>>32)<<prime128Shift
+               // Carries
+               s1h += s1l >> 32
+               s0l += s1h >> 32
+               s0h += s0l >> 32
+               // Update the values
+               s[1] = (s1l & 0xffffffff) + (s1h << 32)
+               s[0] = (s0l & 0xffffffff) + (s0h << 32)
+       }
+       return len(data), nil
+}
 
-func (s *sum32) BlockSize() int  { return 1 }
-func (s *sum32a) BlockSize() int { return 1 }
-func (s *sum64) BlockSize() int  { return 1 }
-func (s *sum64a) BlockSize() int { return 1 }
+func (s *sum32) Size() int   { return 4 }
+func (s *sum32a) Size() int  { return 4 }
+func (s *sum64) Size() int   { return 8 }
+func (s *sum64a) Size() int  { return 8 }
+func (s *sum128) Size() int  { return 16 }
+func (s *sum128a) Size() int { return 16 }
+
+func (s *sum32) BlockSize() int   { return 1 }
+func (s *sum32a) BlockSize() int  { return 1 }
+func (s *sum64) BlockSize() int   { return 1 }
+func (s *sum64a) BlockSize() int  { return 1 }
+func (s *sum128) BlockSize() int  { return 1 }
+func (s *sum128a) BlockSize() int { return 1 }
 
 func (s *sum32) Sum(in []byte) []byte {
        v := uint32(*s)
@@ -133,3 +201,17 @@ func (s *sum64a) Sum(in []byte) []byte {
        v := uint64(*s)
        return append(in, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), byte(v))
 }
+
+func (s *sum128) Sum(in []byte) []byte {
+       return append(in,
+               byte(s[0]>>56), byte(s[0]>>48), byte(s[0]>>40), byte(s[0]>>32), byte(s[0]>>24), byte(s[0]>>16), byte(s[0]>>8), byte(s[0]),
+               byte(s[1]>>56), byte(s[1]>>48), byte(s[1]>>40), byte(s[1]>>32), byte(s[1]>>24), byte(s[1]>>16), byte(s[1]>>8), byte(s[1]),
+       )
+}
+
+func (s *sum128a) Sum(in []byte) []byte {
+       return append(in,
+               byte(s[0]>>56), byte(s[0]>>48), byte(s[0]>>40), byte(s[0]>>32), byte(s[0]>>24), byte(s[0]>>16), byte(s[0]>>8), byte(s[0]),
+               byte(s[1]>>56), byte(s[1]>>48), byte(s[1]>>40), byte(s[1]>>32), byte(s[1]>>24), byte(s[1]>>16), byte(s[1]>>8), byte(s[1]),
+       )
+}

diff --git a/src/hash/fnv/fnv_test.go b/src/hash/fnv/fnv_test.go
index 89d39b38ad3922403e03fb49b52fa174cc9a7e44..7da15ba9e8224c40568b8ae4e9c234a390777ad5 100644 (file)
--- a/src/hash/fnv/fnv_test.go
+++ b/src/hash/fnv/fnv_test.go
@@ -44,6 +44,20 @@ var golden64a = []golden{
        {[]byte{0xe7, 0x1f, 0xa2, 0x19, 0x05, 0x41, 0x57, 0x4b}, "abc"},
 }
 
+var golden128 = []golden{
+       {[]byte{0x6c, 0x62, 0x27, 0x2e, 0x07, 0xbb, 0x01, 0x42, 0x62, 0xb8, 0x21, 0x75, 0x62, 0x95, 0xc5, 0x8d}, ""},
+       {[]byte{0xd2, 0x28, 0xcb, 0x69, 0x10, 0x1a, 0x8c, 0xaf, 0x78, 0x91, 0x2b, 0x70, 0x4e, 0x4a, 0x14, 0x1e}, "a"},
+       {[]byte{0x8, 0x80, 0x94, 0x5a, 0xee, 0xab, 0x1b, 0xe9, 0x5a, 0xa0, 0x73, 0x30, 0x55, 0x26, 0xc0, 0x88}, "ab"},
+       {[]byte{0xa6, 0x8b, 0xb2, 0xa4, 0x34, 0x8b, 0x58, 0x22, 0x83, 0x6d, 0xbc, 0x78, 0xc6, 0xae, 0xe7, 0x3b}, "abc"},
+}
+
+var golden128a = []golden{
+       {[]byte{0x6c, 0x62, 0x27, 0x2e, 0x07, 0xbb, 0x01, 0x42, 0x62, 0xb8, 0x21, 0x75, 0x62, 0x95, 0xc5, 0x8d}, ""},
+       {[]byte{0xd2, 0x28, 0xcb, 0x69, 0x6f, 0x1a, 0x8c, 0xaf, 0x78, 0x91, 0x2b, 0x70, 0x4e, 0x4a, 0x89, 0x64}, "a"},
+       {[]byte{0x08, 0x80, 0x95, 0x44, 0xbb, 0xab, 0x1b, 0xe9, 0x5a, 0xa0, 0x73, 0x30, 0x55, 0xb6, 0x9a, 0x62}, "ab"},
+       {[]byte{0xa6, 0x8d, 0x62, 0x2c, 0xec, 0x8b, 0x58, 0x22, 0x83, 0x6d, 0xbc, 0x79, 0x77, 0xaf, 0x7f, 0x3b}, "abc"},
+}
+
 func TestGolden32(t *testing.T) {
        testGolden(t, New32(), golden32)
 }
@@ -60,6 +74,14 @@ func TestGolden64a(t *testing.T) {
        testGolden(t, New64a(), golden64a)
 }
 
+func TestGolden128(t *testing.T) {
+       testGolden(t, New128(), golden128)
+}
+
+func TestGolden128a(t *testing.T) {
+       testGolden(t, New128a(), golden128a)
+}
+
 func testGolden(t *testing.T, hash hash.Hash, gold []golden) {
        for _, g := range gold {
                hash.Reset()
@@ -91,6 +113,13 @@ func TestIntegrity64(t *testing.T) {
 func TestIntegrity64a(t *testing.T) {
        testIntegrity(t, New64a())
 }
+func TestIntegrity128(t *testing.T) {
+       testIntegrity(t, New128())
+}
+
+func TestIntegrity128a(t *testing.T) {
+       testIntegrity(t, New128a())
+}
 
 func testIntegrity(t *testing.T, h hash.Hash) {
        data := []byte{'1', '2', 3, 4, 5}
@@ -129,6 +158,8 @@ func testIntegrity(t *testing.T, h hash.Hash) {
                if sum64 != binary.BigEndian.Uint64(sum) {
                        t.Fatalf("Sum()=0x%x, but Sum64()=0x%x", sum, sum64)
                }
+       case 16:
+               // There's no Sum128 function, so we don't need to test anything here.
        }
 }
 
@@ -148,6 +179,14 @@ func BenchmarkFnv64aKB(b *testing.B) {
        benchmarkKB(b, New64a())
 }
 
+func BenchmarkFnv128KB(b *testing.B) {
+       benchmarkKB(b, New128())
+}
+
+func BenchmarkFnv128aKB(b *testing.B) {
+       benchmarkKB(b, New128a())
+}
+
 func benchmarkKB(b *testing.B, h hash.Hash) {
        b.SetBytes(1024)
        data := make([]byte, 1024)