From 2e6fe912c2a105b4431f331e7bed85b94567f879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=BA=90=E6=96=87=E9=9B=A8?= <41315874+fumiama@users.noreply.github.com> Date: Fri, 22 Apr 2022 21:05:19 +0800 Subject: [PATCH] =?UTF-8?q?speed=20up=20encoder=20&=20decoder=20name=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20old=20time/op=20=20=20=20new=20time/?= =?UTF-8?q?op=20=20=20=20delta=20Encoder/16-8=20=20=20=20=20=20136ns=20?= =?UTF-8?q?=C2=B1=202%=20=20=20=20=20102ns=20=C2=B1=201%=20=20-25.00%=20?= =?UTF-8?q?=20(p=3D0.008=20n=3D5+5)=20Encoder/256-8=20=20=20=20=20490ns=20?= =?UTF-8?q?=C2=B1=201%=20=20=20=20=20410ns=20=C2=B1=200%=20=20-16.24%=20?= =?UTF-8?q?=20(p=3D0.008=20n=3D5+5)=20Encoder/4K-8=20=20=20=20=204.47?= =?UTF-8?q?=C2=B5s=20=C2=B1=201%=20=20=20=203.52=C2=B5s=20=C2=B1=201%=20?= =?UTF-8?q?=20-21.10%=20=20(p=3D0.008=20n=3D5+5)=20Encoder/32K-8=20=20=20?= =?UTF-8?q?=2038.9=C2=B5s=20=C2=B1=200%=20=20=20=2033.6=C2=B5s=20=C2=B1=20?= =?UTF-8?q?1%=20=20-13.72%=20=20(p=3D0.008=20n=3D5+5)=20Decoder/16-8=20=20?= =?UTF-8?q?=20=20=20=20269ns=20=C2=B1=201%=20=20=20=20=20253ns=20=C2=B1=20?= =?UTF-8?q?1%=20=20=20-5.95%=20=20(p=3D0.008=20n=3D5+5)=20Decoder/256-8=20?= =?UTF-8?q?=20=20=20=20421ns=20=C2=B1=201%=20=20=20=20=20404ns=20=C2=B1=20?= =?UTF-8?q?2%=20=20=20-4.22%=20=20(p=3D0.008=20n=3D5+5)=20Decoder/4K-8=20?= =?UTF-8?q?=20=20=20=201.68=C2=B5s=20=C2=B1=201%=20=20=20=201.66=C2=B5s=20?= =?UTF-8?q?=C2=B1=203%=20=20=20=20=20~=20=20=20=20=20(p=3D0.190=20n=3D5+5)?= =?UTF-8?q?=20Decoder/32K-8=20=20=20=2012.9=C2=B5s=20=C2=B1=201%=20=20=20?= =?UTF-8?q?=2012.5=C2=B5s=20=C2=B1=201%=20=20=20-2.68%=20=20(p=3D0.008=20n?= =?UTF-8?q?=3D5+5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit name old speed new speed delta Encoder/16-8 118MB/s ± 2% 157MB/s ± 1% +33.34% (p=0.008 n=5+5) Encoder/256-8 523MB/s ± 1% 624MB/s ± 0% +19.38% (p=0.008 n=5+5) Encoder/4K-8 917MB/s ± 1% 1162MB/s ± 1% +26.73% (p=0.008 n=5+5) Encoder/32K-8 841MB/s ± 0% 975MB/s ± 1% +15.90% (p=0.008 n=5+5) Decoder/16-8 59.5MB/s ± 1% 63.2MB/s ± 1% +6.34% (p=0.008 n=5+5) Decoder/256-8 607MB/s ± 1% 634MB/s ± 2% +4.42% (p=0.008 n=5+5) Decoder/4K-8 2.44GB/s ± 1% 2.46GB/s ± 3% ~ (p=0.222 n=5+5) Decoder/32K-8 2.54GB/s ± 1% 2.61GB/s ± 1% +2.76% (p=0.008 n=5+5) name old alloc/op new alloc/op delta Encoder/16-8 40.0B ± 0% 24.0B ± 0% -40.00% (p=0.008 n=5+5) Encoder/256-8 696B ± 0% 472B ± 0% -32.18% (p=0.008 n=5+5) Encoder/4K-8 4.12kB ± 0% 0.02kB ± 0% -99.42% (p=0.008 n=5+5) Encoder/32K-8 69.7kB ± 0% 41.0kB ± 0% -41.16% (p=0.000 n=5+4) Decoder/16-8 752B ± 0% 752B ± 0% ~ (all equal) Decoder/256-8 1.39kB ± 0% 1.39kB ± 0% ~ (all equal) Decoder/4K-8 4.98kB ± 0% 4.98kB ± 0% ~ (all equal) Decoder/32K-8 41.1kB ± 0% 41.1kB ± 0% ~ (all equal) name old allocs/op new allocs/op delta Encoder/16-8 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.008 n=5+5) Encoder/256-8 3.00 ± 0% 2.00 ± 0% -33.33% (p=0.008 n=5+5) Encoder/4K-8 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.008 n=5+5) Encoder/32K-8 3.00 ± 0% 2.00 ± 0% -33.33% (p=0.008 n=5+5) Decoder/16-8 3.00 ± 0% 3.00 ± 0% ~ (all equal) Decoder/256-8 3.00 ± 0% 3.00 ± 0% ~ (all equal) Decoder/4K-8 3.00 ± 0% 3.00 ± 0% ~ (all equal) Decoder/32K-8 3.00 ± 0% 3.00 ± 0% ~ (all equal) --- base14_amd64.s | 4 +- base14_asm.go | 2 + base14_test.go | 160 ++++++++++++++++++++++++++++++++++--------------- decoder.go | 60 ++++++++++++------- encoder.go | 7 ++- go.mod | 8 +-- go.sum | 1 + 7 files changed, 163 insertions(+), 79 deletions(-) diff --git a/base14_amd64.s b/base14_amd64.s index 14441ba..fb2beb0 100644 --- a/base14_amd64.s +++ b/base14_amd64.s @@ -4,7 +4,7 @@ #include "textflag.h" // func encode(offset, outlen int, b, encd []byte) -TEXT ·encode(SB), NOSPLIT, $56-64 +TEXT ·encode(SB), NOSPLIT, $0-64 MOVQ ·offset+0(FP), R10 MOVQ ·outlen+8(FP), AX MOVQ ·data+16(FP), DI @@ -125,7 +125,7 @@ encend: // func decode(offset, outlen int, b, decd []byte) -TEXT ·decode(SB), NOSPLIT, $56-64 +TEXT ·decode(SB), NOSPLIT, $0-64 MOVQ ·offset+0(FP), BX MOVQ ·outlen+8(FP), R8 MOVQ ·data+16(FP), DI diff --git a/base14_asm.go b/base14_asm.go index befed37..4ba6ec1 100644 --- a/base14_asm.go +++ b/base14_asm.go @@ -4,7 +4,9 @@ package base14 //go:noescape +//go:nosplit func encode(offset, outlen int, b, encd []byte) //go:noescape +//go:nosplit func decode(offset, outlen int, b, decd []byte) diff --git a/base14_test.go b/base14_test.go index 230d5ba..583eb3c 100644 --- a/base14_test.go +++ b/base14_test.go @@ -29,22 +29,23 @@ func TestBase14(t *testing.T) { } func TestEncoder(t *testing.T) { - buf := make([]byte, 42242141) + buf := make([]byte, 1024*1024+1) _, err := rand.Read(buf) if err != nil { t.Fatal(err) } - e := NewEncoder(bytes.NewReader(buf)) - w := bytes.NewBuffer(make([]byte, 0, 42242150)) - _, err = io.Copy(w, e) - if err != nil { - t.Fatal(err) + w := bytes.NewBuffer(make([]byte, 0, 1024*1024+1)) + for i := 0; i <= 1024*1024; i += rand.Intn(128) * 7 { + e := NewEncoder(bytes.NewReader(buf[:i])) + _, err = io.Copy(w, e) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(Encode(buf[:i]), w.Bytes()) { + t.Fail() + } + w.Reset() } - out := w.Bytes() - assert.Equal(t, 48276736, w.Len()) - d := Decode(out) - t.Log(len(out)) - assert.Equal(t, buf, d) } func TestBufferedEncoder(t *testing.T) { @@ -53,17 +54,17 @@ func TestBufferedEncoder(t *testing.T) { if err != nil { t.Fatal(err) } - e := NewBufferedEncoder(buf) - w := bytes.NewBuffer(make([]byte, 0, 1024*1024+16)) - _, err = io.Copy(w, e) - if err != nil { - t.Fatal(err) - } - out := w.Bytes() - t.Log(w.Len()) - d := Decode(out) - if !bytes.Equal(buf, d) { - t.Fail() + w := bytes.NewBuffer(make([]byte, 0, 1024*1024+1)) + for i := 0; i <= 1024*1024; i += rand.Intn(128) * 7 { + e := NewBufferedEncoder(buf[:i]) + _, err = io.Copy(w, e) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(Encode(buf[:i]), w.Bytes()) { + t.Fail() + } + w.Reset() } } @@ -73,15 +74,18 @@ func TestDecoder(t *testing.T) { if err != nil { t.Fatal(err) } + encd := Encode(buf) w := bytes.NewBuffer(make([]byte, 0, 1024*1024+1)) - d := NewDecoder(bytes.NewReader(Encode(buf))) - _, err = io.Copy(w, d) - if err != nil { - t.Fatal(err) - } - t.Log(w.Len()) - if !bytes.Equal(buf, w.Bytes()) { - t.Fail() + for i := 0; i <= 1024*1024; i += rand.Intn(128) * 8 { + d := NewDecoder(bytes.NewReader(encd[:i])) + _, err = io.Copy(w, d) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf[:i/8*7], w.Bytes()) { + t.Fail() + } + w.Reset() } } @@ -91,19 +95,22 @@ func TestBufferedDecoder(t *testing.T) { if err != nil { t.Fatal(err) } + encd := Encode(buf) w := bytes.NewBuffer(make([]byte, 0, 1024*1024+1)) - d := NewBufferedDecoder(Encode(buf)) - _, err = io.Copy(w, d) - if err != nil { - t.Fatal(err) - } - t.Log(w.Len()) - if !bytes.Equal(buf, w.Bytes()) { - t.Fail() + for i := 0; i <= 1024*1024; i += rand.Intn(128) * 8 { + d := NewBufferedDecoder(encd[:i]) + _, err = io.Copy(w, d) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf[:i/8*7], w.Bytes()) { + t.Fail() + } + w.Reset() } } -func benchEncrypt(b *testing.B, data []byte) { +func benchEncode(b *testing.B, data []byte) { _, err := rand.Read(data) if err != nil { panic(err) @@ -116,7 +123,7 @@ func benchEncrypt(b *testing.B, data []byte) { } } -func benchDecrypt(b *testing.B, data []byte) { +func benchDecode(b *testing.B, data []byte) { _, err := rand.Read(data) if err != nil { panic(err) @@ -136,37 +143,94 @@ func benchDecrypt(b *testing.B, data []byte) { func BenchmarkEncodeTo(b *testing.B) { b.Run("16", func(b *testing.B) { data := make([]byte, 16) - benchEncrypt(b, data) + benchEncode(b, data) }) b.Run("256", func(b *testing.B) { data := make([]byte, 256) - benchEncrypt(b, data) + benchEncode(b, data) }) b.Run("4K", func(b *testing.B) { data := make([]byte, 1024*4) - benchEncrypt(b, data) + benchEncode(b, data) }) b.Run("32K", func(b *testing.B) { data := make([]byte, 1024*32) - benchEncrypt(b, data) + benchEncode(b, data) }) } func BenchmarkDecodeTo(b *testing.B) { b.Run("16", func(b *testing.B) { data := make([]byte, 16) - benchDecrypt(b, data) + benchDecode(b, data) }) b.Run("256", func(b *testing.B) { data := make([]byte, 256) - benchDecrypt(b, data) + benchDecode(b, data) }) b.Run("4K", func(b *testing.B) { data := make([]byte, 4096) - benchDecrypt(b, data) + benchDecode(b, data) }) b.Run("32K", func(b *testing.B) { data := make([]byte, 1024*32) - benchDecrypt(b, data) + benchDecode(b, data) + }) +} + +func benchEncoder(b *testing.B, cnt int64) { + enc := NewEncoder(rand.New(rand.NewSource(0))) + buf := bytes.NewBuffer(make([]byte, 0, cnt)) + b.SetBytes(cnt) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = io.CopyN(buf, enc, cnt) + buf.Reset() + } +} + +func benchDecoder(b *testing.B, cnt int64) { + enc := NewEncoder(rand.New(rand.NewSource(0))) + buf := bytes.NewBuffer(make([]byte, 0, cnt)) + _, err := io.CopyN(buf, enc, cnt) + if err != nil { + panic(err) + } + buf2 := bytes.NewBuffer(make([]byte, 0, cnt)) + b.SetBytes(cnt) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = io.Copy(buf2, NewDecoder(bytes.NewReader(buf.Bytes()))) + buf2.Reset() + } +} + +func BenchmarkEncoder(b *testing.B) { + b.Run("16", func(b *testing.B) { + benchEncoder(b, 16) + }) + b.Run("256", func(b *testing.B) { + benchEncoder(b, 256) + }) + b.Run("4K", func(b *testing.B) { + benchEncoder(b, 1024*4) + }) + b.Run("32K", func(b *testing.B) { + benchEncoder(b, 1024*32) + }) +} + +func BenchmarkDecoder(b *testing.B) { + b.Run("16", func(b *testing.B) { + benchDecoder(b, 16) + }) + b.Run("256", func(b *testing.B) { + benchDecoder(b, 256) + }) + b.Run("4K", func(b *testing.B) { + benchDecoder(b, 1024*4) + }) + b.Run("32K", func(b *testing.B) { + benchDecoder(b, 1024*32) }) } diff --git a/decoder.go b/decoder.go index 363f135..a0d678f 100644 --- a/decoder.go +++ b/decoder.go @@ -24,43 +24,63 @@ func (d *Decoder) Read(p []byte) (n int, err error) { err = io.EOF return } - inlen := len(p)/7*8 + 2 + inlen := len(p) / 7 * 8 if d.r != nil { d.b = append(d.b, make([]byte, inlen)...) n, err = d.r.Read(d.b[i:]) + if n <= 0 { + return + } inlen = i + n - d.b = d.b[:inlen] if err != nil { - if len(d.b) > 0 { + if inlen > 0 { offset := 0 - if d.b[len(d.b)-2] == '=' { - offset = int(d.b[len(d.b)-1]) + if d.b[inlen-2] == '=' { + offset = int(d.b[inlen-1]) } - n = DecodeLen(len(d.b), offset) - _ = DecodeTo(d.b, p) + n = DecodeLen(inlen, offset) + _ = DecodeTo(d.b[:inlen], p) d.b = nil d.r = nil } return } + if inlen%2 != 0 { + d.b = d.b[:inlen] + n = 0 + return + } + offset := 0 + if d.b[inlen-2] == '=' { + offset = int(d.b[inlen-1]) + } + if offset > 0 { + n = DecodeLen(len(d.b[:inlen]), offset) + _ = DecodeTo(d.b[:inlen], p) + d.b = nil + d.r = nil + } else { + n = DecodeLen(inlen, 0) + _ = DecodeTo(d.b[:inlen], p) + d.b = d.b[:0] + } + return } else if inlen > len(d.b) { inlen = len(d.b) } - if inlen >= 2 { - inlen -= 2 + if inlen <= 2 { + err = io.EOF + return + } + if len(d.b[inlen:]) == 2 { + inlen += 2 } offset := 0 - if d.b[len(d.b)-2] == '=' { - offset = int(d.b[len(d.b)-1]) - } - if offset > 0 { - n = DecodeLen(len(d.b), offset) - _ = DecodeTo(d.b, p) - d.b = nil - } else { - n = DecodeLen(inlen, 0) - _ = DecodeTo(d.b[:inlen], p) - d.b = d.b[inlen:] + if d.b[inlen-2] == '=' { + offset = int(d.b[inlen-1]) } + n = DecodeLen(inlen, offset) + _ = DecodeTo(d.b[:inlen], p) + d.b = d.b[inlen:] return } diff --git a/encoder.go b/encoder.go index 5073f7c..b91cd53 100644 --- a/encoder.go +++ b/encoder.go @@ -29,7 +29,6 @@ func (e *Encoder) Read(p []byte) (n int, err error) { e.b = append(e.b, make([]byte, inlen)...) n, err = e.r.Read(e.b[i:]) inlen = i + n - e.b = e.b[:inlen] if err != nil { if len(e.b) > 0 { n = EncodeLen(inlen) @@ -39,11 +38,15 @@ func (e *Encoder) Read(p []byte) (n int, err error) { e.r = nil return } + n = EncodeLen(inlen) + err = EncodeTo(e.b[:inlen], p) + e.b = e.b[:0] + return } else if inlen > len(e.b) { inlen = len(e.b) } n = EncodeLen(inlen) - _ = EncodeTo(e.b[:inlen], p) + err = EncodeTo(e.b[:inlen], p) e.b = e.b[inlen:] return } diff --git a/go.mod b/go.mod index 5d7fd30..6eb139b 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,8 @@ module github.com/fumiama/go-base16384 -go 1.18 +go 1.16 require ( github.com/stretchr/testify v1.7.1 golang.org/x/text v0.3.7 ) - -require ( - github.com/davecgh/go-spew v1.1.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect -) diff --git a/go.sum b/go.sum index 9885ff7..eedcb75 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,7 @@ github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMT github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=