From: Sergey Matveev Date: Mon, 13 Jan 2020 10:58:30 +0000 (+0300) Subject: Refactoring, io.Reader/Writer friendliness, performance optimization X-Git-Tag: v2.0.0^0 X-Git-Url: http://www.git.cypherpunks.ru/?p=netstring.git;a=commitdiff_plain;h=b8868fafef88a7b8320d6e68a3c4c4697950d1e1 Refactoring, io.Reader/Writer friendliness, performance optimization --- diff --git a/cmd/netstring/main.go b/cmd/netstring/main.go new file mode 100644 index 0000000..306f46e --- /dev/null +++ b/cmd/netstring/main.go @@ -0,0 +1,110 @@ +/* +netstring -- netstring format serialization library +Copyright (C) 2015-2020 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package main + +import ( + "flag" + "fmt" + "io" + "os" + "strconv" + + "go.cypherpunks.ru/netstring/v2" +) + +func usage() { + fmt.Fprintf(os.Stderr, "ns -- Work with netstring encoded files\n\n") + fmt.Fprintf(os.Stderr, "Usage: %s list FILE\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s read FILE CHUNK > data\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s write FILE ... > data\n", os.Args[0]) + os.Exit(1) +} + +func main() { + flag.Usage = usage + flag.Parse() + if len(os.Args) < 3 { + usage() + } + switch os.Args[1] { + case "list": + fd, err := os.Open(os.Args[2]) + if err != nil { + panic(err) + } + r := netstring.NewReader(fd) + for i := 0; ; i++ { + size, err := r.Next() + if err == io.EOF { + break + } + if err != nil { + panic(err) + } + fmt.Printf("%d\t%d\n", i, size) + r.Discard() + } + case "read": + if len(os.Args) != 4 { + usage() + } + chunk, err := strconv.Atoi(os.Args[3]) + if err != nil { + panic(err) + } + fd, err := os.Open(os.Args[2]) + if err != nil { + panic(err) + } + r := netstring.NewReader(fd) + for i := 0; i < chunk; i++ { + _, err = r.Next() + if err != nil { + panic(err) + } + r.Discard() + } + _, err = r.Next() + if err != nil { + panic(err) + } + if _, err = io.Copy(os.Stdout, r); err != nil { + panic(err) + } + case "write": + w := netstring.NewWriter(os.Stdout) + for i, fn := range os.Args[2:] { + fd, err := os.Open(fn) + if err != nil { + panic(err) + } + fi, err := fd.Stat() + if err != nil { + panic(err) + } + size := uint64(fi.Size()) + if _, err = w.WriteSize(size); err != nil { + panic(err) + } + if _, err = io.Copy(w, fd); err != nil { + panic(err) + } + fmt.Fprintf(os.Stderr, "%d\t%d\t%s\n", i, size, fn) + } + } +} diff --git a/go.mod b/go.mod index 206a9aa..cdefced 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ -module go.cypherpunks.ru/netstring +module go.cypherpunks.ru/netstring/v2 go 1.12 diff --git a/netstring_test.go b/netstring_test.go deleted file mode 100644 index 702e79d..0000000 --- a/netstring_test.go +++ /dev/null @@ -1,139 +0,0 @@ -/* -netstring -- netstring format serialization library -Copyright (C) 2015-2020 Sergey Matveev - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, version 3 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - -package netstring - -import ( - "bytes" - "testing" - "testing/quick" -) - -func TestTrivial(t *testing.T) { - buf := bytes.NewBuffer(nil) - w := NewWriter(buf) - if n, err := w.Write([]byte{}); err != nil || n != 3 || string(buf.Next(3)) != "0:," { - t.Fail() - } - if n, err := w.Write([]byte("foo")); err != nil || n != 6 { - t.Fail() - } - if n, err := w.Write([]byte("barz")); err != nil || n != 7 { - t.Fail() - } - if string(buf.Bytes()) != "3:foo,4:barz," { - t.Fail() - } - r := NewReader(buf) - if n, err := r.Iter(); err != nil || n != 3 { - t.Fail() - } - if r.Discard() != nil { - t.Fail() - } - if n, err := r.Iter(); err != nil || n != 4 { - t.Fail() - } - m := make([]byte, int(4)) - if r.Read(m) != nil { - t.Fail() - } - if bytes.Compare(m, []byte("barz")) != 0 { - t.Fail() - } -} - -func TestSymmetric(t *testing.T) { - buf := bytes.NewBuffer(nil) - w := NewWriter(buf) - r := NewReader(buf) - f := func(i uint8, data []byte) bool { - var n uint8 - for n = 0; n < i; n++ { - if _, err := w.Write(data); err != nil { - return false - } - } - for n = 0; n < i; n++ { - size, err := r.Iter() - if err != nil || int(size) != len(data) { - return false - } - read := make([]byte, size) - if r.Read(read) != nil { - return false - } - if bytes.Compare(read, data) != 0 { - return false - } - } - return true - } - if err := quick.Check(f, nil); err != nil { - t.Error(err) - } -} - -func TestErrors(t *testing.T) { - b := bytes.NewBufferString("junk") - r := NewReader(b) - if _, err := r.Iter(); err == nil { - t.FailNow() - } - - b = bytes.NewBufferString("1111111111111111111111:") - r = NewReader(b) - if _, err := r.Iter(); err == nil { - t.FailNow() - } - - b = bytes.NewBufferString("6foobar") - r = NewReader(b) - if _, err := r.Iter(); err == nil { - t.FailNow() - } - - data := make([]byte, 1<<10) - - b = bytes.NewBufferString("0:foobar,") - r = NewReader(b) - if _, err := r.Iter(); err != nil { - t.FailNow() - } - if r.Read(data) == nil { - t.FailNow() - } - - b = bytes.NewBufferString("0:foobar") - r = NewReader(b) - if _, err := r.Iter(); err != nil { - t.FailNow() - } - if r.Read(data) == nil { - t.FailNow() - } - - b = bytes.NewBufferString("6:foobar") - r = NewReader(b) - if _, err := r.Iter(); err != nil { - t.FailNow() - } - if r.Read(data) == nil { - t.FailNow() - } - -} diff --git a/netstring.go b/ns.go similarity index 53% rename from netstring.go rename to ns.go index 8345556..af1159f 100644 --- a/netstring.go +++ b/ns.go @@ -21,30 +21,15 @@ along with this program. If not, see . // (http://cr.yp.to/proto/netstrings.txt) format for binary string // serialization. // -// buf := bytes.NewBuffer(nil) -// w := netstring.NewWriter(buf) -// w.Write([]byte("hello")) // buf contains "5:hello," -// w.Write([]byte("world!")) // buf contains "5:hello,6:world!," -// r := netstring.NewReader(buf) -// size, err := r.Iter() // size is 5 -// r.Discard() // discard (skip) current netstring ("hello") -// size, err = r.Iter() // size is 6 -// out := make([]byte, size) -// r.Read(out) // out contains "world!" bytes -// -// Pay attention that netstring uses bufio for Reader and Writer. +// var b bytes.Buffer +// w := netstring.NewWriter(&b) +// n, _ = w.WriteChunk([]byte("hello")) // n is 8, "5:hello," +// n, _ = w.WriteSize(6) // n is 2 +// n, _ = w.Write([]byte("wor")) // n is 3 +// n, _ = w.Write([]byte("ld!")) // n is 3, "5:hello,6:world!," +// r := netstring.NewReader(&b) +// size, err := r.Next() // size is 5 +// r.Discard() // skip that chunk +// size, err = r.Next() // size is 6 +// data, _ := ioutil.ReadAll(r) // data contains "world!" package netstring - -import ( - "errors" -) - -const ( - MaxPrefixSize = 21 -) - -var ( - ErrBufSize error = errors.New("Invalid destination buffer size") - ErrState error = errors.New("Invalid state") - ErrTerminator error = errors.New("Invalid terminator") -) diff --git a/ns_test.go b/ns_test.go new file mode 100644 index 0000000..084cd6d --- /dev/null +++ b/ns_test.go @@ -0,0 +1,164 @@ +/* +netstring -- netstring format serialization library +Copyright (C) 2015-2020 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package netstring + +import ( + "bytes" + "io/ioutil" + "testing" + "testing/quick" +) + +func TestTrivial(t *testing.T) { + var buf bytes.Buffer + w := NewWriter(&buf) + if n, err := w.WriteChunk([]byte{}); err != nil || n != 3 || string(buf.Next(3)) != "0:," { + t.FailNow() + } + if n, err := w.WriteChunk([]byte("foo")); err != nil || n != 6 { + t.FailNow() + } + if n, err := w.WriteChunk([]byte("barz")); err != nil || n != 7 { + t.FailNow() + } + if string(buf.Bytes()) != "3:foo,4:barz," { + t.FailNow() + } + r := NewReader(&buf) + if n, err := r.Next(); err != nil || n != 3 { + t.FailNow() + } + if r.Discard() != nil { + t.FailNow() + } + if n, err := r.Next(); err != nil || n != 4 { + t.FailNow() + } + m := make([]byte, 4) + if n, err := r.Read(m); err != nil || n != 4 { + t.FailNow() + } + if bytes.Compare(m, []byte("barz")) != 0 { + t.FailNow() + } +} + +func TestSymmetric(t *testing.T) { + var buf bytes.Buffer + w := NewWriter(&buf) + r := NewReader(&buf) + f := func(datum [][]byte) bool { + for _, data := range datum { + if n, err := w.WriteChunk(data); err != nil || n <= len(data) { + return false + } + } + for _, data := range datum { + if n, err := r.Next(); err != nil || n != uint64(len(data)) { + return false + } + got, err := ioutil.ReadAll(r) + if err != nil || bytes.Compare(got, data) != 0 { + return false + } + } + return true + } + if err := quick.Check(f, nil); err != nil { + t.Error(err) + } +} + +func TestErrors(t *testing.T) { + b := bytes.NewBufferString("junk") + r := NewReader(b) + if _, err := r.Next(); err == nil { + t.FailNow() + } + + b = bytes.NewBufferString("1111111111111111111111:") + r = NewReader(b) + if _, err := r.Next(); err == nil { + t.FailNow() + } + + b = bytes.NewBufferString("6foobar") + r = NewReader(b) + if _, err := r.Next(); err == nil { + t.FailNow() + } + + data := make([]byte, 1<<10) + + b = bytes.NewBufferString("0:foobar,") + r = NewReader(b) + if _, err := r.Next(); err != nil { + t.FailNow() + } + if _, err := r.Read(data); err == nil { + t.FailNow() + } + + b = bytes.NewBufferString("0:foobar") + r = NewReader(b) + if _, err := r.Next(); err != nil { + t.FailNow() + } + if _, err := r.Read(data); err == nil { + t.FailNow() + } + + b = bytes.NewBufferString("6:foobar") + r = NewReader(b) + if _, err := r.Next(); err != nil { + t.FailNow() + } + if _, err := r.Read(data); err == nil { + t.FailNow() + } +} + +func TestExample(t *testing.T) { + var b bytes.Buffer + w := NewWriter(&b) + if n, err := w.WriteChunk([]byte("hello")); err != nil || n != 8 || b.String() != "5:hello," { + t.FailNow() + } + if n, err := w.WriteSize(6); err != nil || n != 2 { + t.FailNow() + } + if n, err := w.Write([]byte("wor")); err != nil || n != 3 { + t.FailNow() + } + if n, err := w.Write([]byte("ld!")); err != nil || n != 3 || b.String() != "5:hello,6:world!," { + t.FailNow() + } + r := NewReader(&b) + if size, err := r.Next(); err != nil || size != 5 { + t.FailNow() + } + if err := r.Discard(); err != nil { + t.FailNow() + } + if size, err := r.Next(); err != nil || size != 6 { + t.FailNow() + } + if data, err := ioutil.ReadAll(r); err != nil || string(data) != "world!" { + t.FailNow() + } +} diff --git a/r.go b/r.go new file mode 100644 index 0000000..f5f74dd --- /dev/null +++ b/r.go @@ -0,0 +1,112 @@ +/* +netstring -- netstring format serialization library +Copyright (C) 2015-2020 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package netstring + +import ( + "bufio" + "bytes" + "errors" + "io" + "strconv" +) + +type Reader struct { + r *bufio.Reader + left uint64 + eof bool +} + +// Create new Reader. +// Pay attention that bufio.Reader is used to read from it. +func NewReader(r io.Reader) *Reader { + return &Reader{r: bufio.NewReader(r), eof: true} +} + +// Parse netstring prefix. It returns data length. +// After this method you have to call either Read() or Discard(). +// io.EOF is returned when underlying reader has no data anymore. +func (r *Reader) Next() (uint64, error) { + if !r.eof { + return 0, errors.New("current chunk is unread") + } + p, _ := r.r.Peek(21) + if len(p) == 0 { + return 0, io.EOF + } + idx := bytes.Index(p, []byte{':'}) + if idx == -1 { + return 0, errors.New("no length separator found") + } + size, err := strconv.ParseUint(string(p[:idx]), 10, 64) + if err != nil { + return 0, err + } + if _, err = r.r.Discard(idx + 1); err != nil { + return 0, err + } + r.left = size + r.eof = false + return size, nil +} + +func (r *Reader) checkTerminator() error { + b, err := r.r.ReadByte() + if err != nil { + return err + } + if b != ',' { + return errors.New("no terminator found") + } + return nil +} + +// Read current netstring chunk. +// This method must be called after Next(). +// Terminator check and error raising are performed only at the end. +func (r *Reader) Read(buf []byte) (n int, err error) { + if r.eof { + return 0, io.EOF + } + if uint64(len(buf)) > r.left { + buf = buf[:r.left] + } + n, err = r.r.Read(buf) + r.left -= uint64(n) + if r.left == 0 { + err = r.checkTerminator() + if err == nil { + r.eof = true + } + } + return +} + +// Discard remaining bytes in the chunk, possibly fully skipping it. +// This method must be called after Next(). +func (r *Reader) Discard() (err error) { + _, err = r.r.Discard(int(r.left)) + if err != nil { + return + } + err = r.checkTerminator() + if err != nil { + return + } + r.eof = true + return +} diff --git a/reader.go b/reader.go deleted file mode 100644 index 547a7dc..0000000 --- a/reader.go +++ /dev/null @@ -1,114 +0,0 @@ -/* -netstring -- netstring format serialization library -Copyright (C) 2015-2020 Sergey Matveev - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, version 3 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - -package netstring - -import ( - "bufio" - "io" - "strconv" -) - -type Reader struct { - reader *bufio.Reader - prefix []byte - err error - size uint64 - n int -} - -func NewReader(r io.Reader) *Reader { - return &Reader{ - reader: bufio.NewReader(r), - prefix: make([]byte, MaxPrefixSize), - } -} - -// Parse incoming netstring prefix. It returns netstring's incoming -// data length. After using this method you can call either Read() -// or Discard() methods. User can check if incoming data length is -// too big. -func (self *Reader) Iter() (size uint64, err error) { - self.n = 0 - for self.n < MaxPrefixSize { - self.prefix[self.n], self.err = self.reader.ReadByte() - if self.err != nil { - return 0, self.err - } - if self.prefix[self.n] == ':' { - break - } - self.n++ - } - self.size, self.err = strconv.ParseUint( - string(self.prefix[:self.n]), 10, 64, - ) - if self.err != nil { - return 0, self.err - } - return self.size, nil -} - -func (self *Reader) terminator() bool { - self.prefix[0], self.err = self.reader.ReadByte() - if self.err != nil { - return false - } - if self.prefix[0] != ',' { - self.err = ErrTerminator - return false - } - return true -} - -// Receive the full netstring message. This method is called after -// Iter() and user must preallocate buf. If buf size is smaller than -// incoming data size, then function will return an error. Also it -// checks the final terminator character and will return an error if -// it won't find it. -func (self *Reader) Read(buf []byte) error { - if self.err != nil { - return ErrState - } - if uint64(cap(buf)) < self.size { - return ErrBufSize - } - _, self.err = io.ReadAtLeast(self.reader, buf, int(self.size)) - if self.err != nil { - return self.err - } - if !self.terminator() { - return self.err - } - return nil -} - -// Discard (skip) netstring message. This method is called after Iter(). -// It reads and ignores data from the reader and checks that terminator -// character is valid. -func (self *Reader) Discard() error { - if self.err != nil { - return ErrState - } - if _, self.err = self.reader.Discard(int(self.size)); self.err != nil { - return self.err - } - if !self.terminator() { - return self.err - } - return nil -} diff --git a/w.go b/w.go new file mode 100644 index 0000000..aaf33d5 --- /dev/null +++ b/w.go @@ -0,0 +1,87 @@ +/* +netstring -- netstring format serialization library +Copyright (C) 2015-2020 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package netstring + +import ( + "bufio" + "errors" + "io" + "strconv" +) + +type Writer struct { + w *bufio.Writer + left uint64 +} + +// Create new Writer. +// Pay attention that bufio.Writer is used to write to it. +func NewWriter(w io.Writer) *Writer { + return &Writer{w: bufio.NewWriter(w)} +} + +// Write size of the data going to be supplied to Write method. +// It returns number of length prefixed written (possibly just +// buffered). +func (w *Writer) WriteSize(size uint64) (n int, err error) { + if w.left > 0 { + return 0, errors.New("current chunk in not written") + } + w.left = size + return w.w.WriteString(strconv.FormatUint(size, 10) + ":") +} + +// Write the chunk data. WriteSize must preceed this call. +// Write could be buffered for writing. Only when the last bytes are +// written, then terminator is appended and all the data flushed. +// Terminator is not taken in account of written bytes count! +func (w *Writer) Write(buf []byte) (written int, err error) { + if w.left == 0 && len(buf) > 0 { + return 0, errors.New("chunk in already written") + } + written, err = w.w.Write(buf) + if err != nil { + return + } + w.left -= uint64(written) + if w.left == 0 { + _, err = w.w.Write([]byte{','}) + if err != nil { + return + } + err = w.w.Flush() + } + return +} + +// Write the whole chunk at once. It could be convenient to use instead +// of WriteSize/Write invocations +func (w *Writer) WriteChunk(buf []byte) (n int, err error) { + n, err = w.WriteSize(uint64(len(buf))) + if err != nil { + return + } + var nw int + nw, err = w.Write(buf) + n += nw + if err != nil { + return + } + n++ + return +} diff --git a/writer.go b/writer.go deleted file mode 100644 index fbae39f..0000000 --- a/writer.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -netstring -- netstring format serialization library -Copyright (C) 2015-2020 Sergey Matveev - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, version 3 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - -package netstring - -import ( - "bufio" - "io" - "strconv" -) - -type Writer struct { - writer *bufio.Writer - prefix []byte - err error -} - -func NewWriter(w io.Writer) *Writer { - return &Writer{ - writer: bufio.NewWriter(w), - prefix: make([]byte, 0, MaxPrefixSize), - } -} - -// Write serialized representation of data to the underlying writer. -// It returns number of serialized data bytes written. -func (self *Writer) Write(data []byte) (bytesWritten int, err error) { - self.prefix = strconv.AppendUint(self.prefix[:0], uint64(len(data)), 10) - self.prefix = append(self.prefix, ':') - if _, self.err = self.writer.Write(self.prefix); self.err != nil { - return 0, self.err - } - if _, self.err = self.writer.Write(data); self.err != nil { - return 0, self.err - } - if _, self.err = self.writer.Write([]byte{','}); self.err != nil { - return 0, self.err - } - if self.err = self.writer.Flush(); self.err != nil { - return 0, self.err - } - return len(self.prefix) + len(data) + 1, nil -}