diff --git a/README.md b/README.md index 38349af..4e99523 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,25 @@ # fwd - import "github.com/philhofer/fwd" -The `fwd` package provides a buffered reader +[![Go Reference](https://pkg.go.dev/badge/github.com/philhofer/fwd.svg)](https://pkg.go.dev/github.com/philhofer/fwd) + + +`import "github.com/philhofer/fwd"` + +* [Overview](#pkg-overview) +* [Index](#pkg-index) + +## Overview +Package fwd provides a buffered reader and writer. Each has methods that help improve the encoding/decoding performance of some binary protocols. -The `fwd.Writer` and `fwd.Reader` type provide similar +The `Writer` and `Reader` type provide similar functionality to their counterparts in `bufio`, plus a few extra utility methods that simplify read-ahead and write-ahead. I wrote this package to improve serialization -performance for http://github.com/tinylib/msgp, +performance for [github.com/tinylib/msgp](https://github.com/tinylib/msgp), where it provided about a 2x speedup over `bufio` for certain workloads. However, care must be taken to understand the semantics of the extra methods provided by this package, as they allow @@ -37,9 +45,48 @@ the write position by the length of the returned slice. This allows users to write directly to the end of the buffer. +## Portability + +Because it uses the unsafe package, there are theoretically +no promises about forward or backward portability. + +To stay compatible with tinygo 0.32, unsafestr() has been updated +to use unsafe.Slice() as suggested by +https://tinygo.org/docs/guides/compatibility, which also required +bumping go.mod to require at least go 1.20. -## Constants +## Index +* [Constants](#pkg-constants) +* [type Reader](#Reader) + * [func NewReader(r io.Reader) *Reader](#NewReader) + * [func NewReaderBuf(r io.Reader, buf []byte) *Reader](#NewReaderBuf) + * [func NewReaderSize(r io.Reader, n int) *Reader](#NewReaderSize) + * [func (r *Reader) BufferSize() int](#Reader.BufferSize) + * [func (r *Reader) Buffered() int](#Reader.Buffered) + * [func (r *Reader) Next(n int) ([]byte, error)](#Reader.Next) + * [func (r *Reader) Peek(n int) ([]byte, error)](#Reader.Peek) + * [func (r *Reader) Read(b []byte) (int, error)](#Reader.Read) + * [func (r *Reader) ReadByte() (byte, error)](#Reader.ReadByte) + * [func (r *Reader) ReadFull(b []byte) (int, error)](#Reader.ReadFull) + * [func (r *Reader) Reset(rd io.Reader)](#Reader.Reset) + * [func (r *Reader) Skip(n int) (int, error)](#Reader.Skip) + * [func (r *Reader) WriteTo(w io.Writer) (int64, error)](#Reader.WriteTo) +* [type Writer](#Writer) + * [func NewWriter(w io.Writer) *Writer](#NewWriter) + * [func NewWriterBuf(w io.Writer, buf []byte) *Writer](#NewWriterBuf) + * [func NewWriterSize(w io.Writer, n int) *Writer](#NewWriterSize) + * [func (w *Writer) BufferSize() int](#Writer.BufferSize) + * [func (w *Writer) Buffered() int](#Writer.Buffered) + * [func (w *Writer) Flush() error](#Writer.Flush) + * [func (w *Writer) Next(n int) ([]byte, error)](#Writer.Next) + * [func (w *Writer) ReadFrom(r io.Reader) (int64, error)](#Writer.ReadFrom) + * [func (w *Writer) Write(p []byte) (int, error)](#Writer.Write) + * [func (w *Writer) WriteByte(b byte) error](#Writer.WriteByte) + * [func (w *Writer) WriteString(s string) (int, error)](#Writer.WriteString) + + +## Constants ``` go const ( // DefaultReaderSize is the default size of the read buffer @@ -121,7 +168,7 @@ and the reader position will not be incremented. -### func (\*Reader) Peek +### func (\*Reader) Peek ``` go func (r *Reader) Peek(n int) ([]byte, error) ``` @@ -134,23 +181,23 @@ io.ErrUnexpectedEOF. -### func (\*Reader) Read +### func (\*Reader) Read ``` go func (r *Reader) Read(b []byte) (int, error) ``` -Read implements `io.Reader` +Read implements `io.Reader`. -### func (\*Reader) ReadByte +### func (\*Reader) ReadByte ``` go func (r *Reader) ReadByte() (byte, error) ``` -ReadByte implements `io.ByteReader` +ReadByte implements `io.ByteReader`. -### func (\*Reader) ReadFull +### func (\*Reader) ReadFull ``` go func (r *Reader) ReadFull(b []byte) (int, error) ``` @@ -161,7 +208,7 @@ EOF is considered an unexpected error. -### func (\*Reader) Reset +### func (\*Reader) Reset ``` go func (r *Reader) Reset(rd io.Reader) ``` @@ -170,7 +217,7 @@ and the read buffer. -### func (\*Reader) Skip +### func (\*Reader) Skip ``` go func (r *Reader) Skip(n int) (int, error) ``` @@ -182,27 +229,30 @@ that method will be used to skip forward. If the reader encounters an EOF before skipping 'n' bytes, it -returns io.ErrUnexpectedEOF. If the -underlying reader implements io.Seeker, then +returns `io.ErrUnexpectedEOF`. If the +underlying reader implements `io.Seeker`, then those rules apply instead. (Many implementations will not return `io.EOF` until the next call -to Read.) +to Read). -### func (\*Reader) WriteTo + +### func (\*Reader) WriteTo ``` go func (r *Reader) WriteTo(w io.Writer) (int64, error) ``` -WriteTo implements `io.WriterTo` +WriteTo implements `io.WriterTo`. -## type Writer + +## type Writer ``` go type Writer struct { // contains filtered or unexported fields } + ``` Writer is a buffered writer @@ -212,9 +262,7 @@ Writer is a buffered writer - - -### func NewWriter +### func NewWriter ``` go func NewWriter(w io.Writer) *Writer ``` @@ -223,18 +271,24 @@ that writes to 'w' and has a buffer that is `DefaultWriterSize` bytes. -### func NewWriterSize +### func NewWriterBuf ``` go -func NewWriterSize(w io.Writer, size int) *Writer +func NewWriterBuf(w io.Writer, buf []byte) *Writer ``` -NewWriterSize returns a new writer -that writes to 'w' and has a buffer -that is 'size' bytes. +NewWriterBuf returns a new writer +that writes to 'w' and has 'buf' as a buffer. +'buf' is not used when has smaller capacity than 18, +custom buffer is allocated instead. +### func NewWriterSize +``` go +func NewWriterSize(w io.Writer, n int) *Writer +``` +NewWriterSize returns a new writer that +writes to 'w' and has a buffer size 'n'. - -### func (\*Writer) BufferSize +### func (\*Writer) BufferSize ``` go func (w *Writer) BufferSize() int ``` @@ -242,7 +296,7 @@ BufferSize returns the maximum size of the buffer. -### func (\*Writer) Buffered +### func (\*Writer) Buffered ``` go func (w *Writer) Buffered() int ``` @@ -251,7 +305,7 @@ in the reader. -### func (\*Writer) Flush +### func (\*Writer) Flush ``` go func (w *Writer) Flush() error ``` @@ -260,7 +314,7 @@ to the underlying writer. -### func (\*Writer) Next +### func (\*Writer) Next ``` go func (w *Writer) Next(n int) ([]byte, error) ``` @@ -273,7 +327,7 @@ the size of the returned buffer. -### func (\*Writer) ReadFrom +### func (\*Writer) ReadFrom ``` go func (w *Writer) ReadFrom(r io.Reader) (int64, error) ``` @@ -281,7 +335,7 @@ ReadFrom implements `io.ReaderFrom` -### func (\*Writer) Write +### func (\*Writer) Write ``` go func (w *Writer) Write(p []byte) (int, error) ``` @@ -289,7 +343,7 @@ Write implements `io.Writer` -### func (\*Writer) WriteByte +### func (\*Writer) WriteByte ``` go func (w *Writer) WriteByte(b byte) error ``` @@ -297,7 +351,7 @@ WriteByte implements `io.ByteWriter` -### func (\*Writer) WriteString +### func (\*Writer) WriteString ``` go func (w *Writer) WriteString(s string) (int, error) ``` @@ -310,6 +364,5 @@ WriteString is analogous to Write, but it takes a string. - - - - -Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) \ No newline at end of file +Generated by [godoc2md](https://github.com/davecheney/godoc2md) diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..77ccd8f --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/philhofer/fwd + +go 1.20 // unsafe.StringData requires go1.20 or later diff --git a/reader.go b/reader.go index 6918d3e..4b31385 100644 --- a/reader.go +++ b/reader.go @@ -1,10 +1,10 @@ -// The `fwd` package provides a buffered reader +// Package fwd provides a buffered reader // and writer. Each has methods that help improve // the encoding/decoding performance of some binary // protocols. // -// The `fwd.Writer` and `fwd.Reader` type provide similar -// functionality to their counterparts in `bufio`, plus +// The [Writer] and [Reader] type provide similar +// functionality to their counterparts in [bufio], plus // a few extra utility methods that simplify read-ahead // and write-ahead. I wrote this package to improve serialization // performance for http://github.com/tinylib/msgp, @@ -14,27 +14,31 @@ // the user to access and manipulate the buffer memory // directly. // -// The extra methods for `fwd.Reader` are `Peek`, `Skip` -// and `Next`. `(*fwd.Reader).Peek`, unlike `(*bufio.Reader).Peek`, +// The extra methods for [Reader] are [Reader.Peek], [Reader.Skip] +// and [Reader.Next]. (*fwd.Reader).Peek, unlike (*bufio.Reader).Peek, // will re-allocate the read buffer in order to accommodate arbitrarily -// large read-ahead. `(*fwd.Reader).Skip` skips the next `n` bytes -// in the stream, and uses the `io.Seeker` interface if the underlying -// stream implements it. `(*fwd.Reader).Next` returns a slice pointing -// to the next `n` bytes in the read buffer (like `Peek`), but also +// large read-ahead. (*fwd.Reader).Skip skips the next 'n' bytes +// in the stream, and uses the [io.Seeker] interface if the underlying +// stream implements it. (*fwd.Reader).Next returns a slice pointing +// to the next 'n' bytes in the read buffer (like Reader.Peek), but also // increments the read position. This allows users to process streams // in arbitrary block sizes without having to manage appropriately-sized // slices. Additionally, obviating the need to copy the data from the // buffer to another location in memory can improve performance dramatically // in CPU-bound applications. // -// `fwd.Writer` only has one extra method, which is `(*fwd.Writer).Next`, which -// returns a slice pointing to the next `n` bytes of the writer, and increments +// [Writer] only has one extra method, which is (*fwd.Writer).Next, which +// returns a slice pointing to the next 'n' bytes of the writer, and increments // the write position by the length of the returned slice. This allows users // to write directly to the end of the buffer. -// package fwd -import "io" +import ( + "errors" + "io" + "os" + "syscall" +) const ( // DefaultReaderSize is the default size of the read buffer @@ -80,9 +84,10 @@ type Reader struct { r io.Reader // underlying reader // data[n:len(data)] is buffered data; data[len(data):cap(data)] is free buffer space - data []byte // data - n int // read offset - state error // last read error + data []byte // data + n int // read offset + inputOffset int64 // offset in the input stream + state error // last read error // if the reader past to NewReader was // also an io.Seeker, this is non-nil @@ -95,6 +100,7 @@ func (r *Reader) Reset(rd io.Reader) { r.r = rd r.data = r.data[0:0] r.n = 0 + r.inputOffset = 0 r.state = nil if s, ok := rd.(io.Seeker); ok { r.rs = s @@ -122,12 +128,17 @@ func (r *Reader) more() { if a == 0 && r.state == nil { r.state = io.ErrNoProgress return - } else if a > 0 && r.state == io.EOF { - // discard the io.EOF if we read more than 0 bytes. - // the next call to Read should return io.EOF again. - r.state = nil } - r.data = r.data[:len(r.data)+a] + // io.Reader requires the caller to process n > 0 before the error. + // A non-EOF error used to return here and drop those bytes. + if a > 0 { + r.data = r.data[:len(r.data)+a] + if r.state == io.EOF { + // discard the io.EOF if we read more than 0 bytes. + // the next call to Read should return io.EOF again. + r.state = nil + } + } } // pop error @@ -154,6 +165,9 @@ func (r *Reader) Buffered() int { return len(r.data) - r.n } // BufferSize returns the total size of the buffer func (r *Reader) BufferSize() int { return cap(r.data) } +// InputOffset returns the input stream byte offset of the current reader position +func (r *Reader) InputOffset() int64 { return r.inputOffset } + // Peek returns the next 'n' buffered bytes, // reading from the underlying reader if necessary. // It will only return a slice shorter than 'n' bytes @@ -187,6 +201,53 @@ func (r *Reader) Peek(n int) ([]byte, error) { return r.data[r.n : r.n+n], nil } +func (r *Reader) PeekByte() (b byte, err error) { + if len(r.data)-r.n >= 1 { + b = r.data[r.n] + } else { + b, err = r.peekByte() + } + return +} + +func (r *Reader) peekByte() (byte, error) { + const n = 1 + if cap(r.data) < n { + old := r.data[r.n:] + r.data = make([]byte, n+r.buffered()) + r.data = r.data[:copy(r.data, old)] + r.n = 0 + } + + // keep filling until + // we hit an error or + // read enough bytes + for r.buffered() < n && r.state == nil { + r.more() + } + + // we must have hit an error + if r.buffered() < n { + return 0, r.err() + } + return r.data[r.n], nil +} + +// discard(n) discards up to 'n' buffered bytes, and +// and returns the number of bytes discarded +func (r *Reader) discard(n int) int { + inbuf := r.buffered() + if inbuf <= n { + r.n = 0 + r.inputOffset += int64(inbuf) + r.data = r.data[:0] + return inbuf + } + r.n += n + r.inputOffset += int64(n) + return n +} + // Skip moves the reader forward 'n' bytes. // Returns the number of bytes skipped and any // errors encountered. It is analogous to Seek(n, 1). @@ -195,39 +256,41 @@ func (r *Reader) Peek(n int) ([]byte, error) { // // If the reader encounters // an EOF before skipping 'n' bytes, it -// returns io.ErrUnexpectedEOF. If the -// underlying reader implements io.Seeker, then +// returns [io.ErrUnexpectedEOF]. If the +// underlying reader implements [io.Seeker], then // those rules apply instead. (Many implementations -// will not return `io.EOF` until the next call -// to Read.) +// will not return [io.EOF] until the next call +// to Read). func (r *Reader) Skip(n int) (int, error) { - - // fast path - if r.buffered() >= n { - r.n += n - return n, nil + if n < 0 { + return 0, os.ErrInvalid } - // use seeker implementation - // if we can - if r.rs != nil { - return r.skipSeek(n) - } + // discard some or all of the current buffer + skipped := r.discard(n) - // loop on filling - // and then erasing - o := n - for r.buffered() < n && r.state == nil { + // if we can Seek() through the remaining bytes, do that + if n > skipped && r.rs != nil { + offs, err := r.rs.Seek(int64(n-skipped), io.SeekCurrent) + if err == nil { + r.inputOffset = offs + return n, nil + } + // if the io.Seeker is not actually usable, + // just swallow the error and don't try + // seeking again + if !errors.Is(err, syscall.ESPIPE) { + return skipped, err + } + r.rs = nil + } + // otherwise, keep filling the buffer + // and discarding it up to 'n' + for skipped < n && r.state == nil { r.more() - // we can skip forward - // up to r.buffered() bytes - step := min(r.buffered(), n) - r.n += step - n -= step + skipped += r.discard(n - skipped) } - // at this point, n should be - // 0 if everything went smoothly - return o - n, r.noEOF() + return skipped, r.noEOF() } // Next returns the next 'n' bytes in the stream. @@ -239,8 +302,18 @@ func (r *Reader) Skip(n int) (int, error) { // If an the returned slice is less than the // length asked for, an error will be returned, // and the reader position will not be incremented. -func (r *Reader) Next(n int) ([]byte, error) { +func (r *Reader) Next(n int) (b []byte, err error) { + if r.state == nil && len(r.data)-r.n >= n { + b = r.data[r.n : r.n+n] + r.n += n + r.inputOffset += int64(n) + } else { + b, err = r.next(n) + } + return +} +func (r *Reader) next(n int) ([]byte, error) { // in case the buffer is too small if cap(r.data) < n { old := r.data[r.n:] @@ -259,32 +332,25 @@ func (r *Reader) Next(n int) ([]byte, error) { } out := r.data[r.n : r.n+n] r.n += n + r.inputOffset += int64(n) return out, nil } -// skipSeek uses the io.Seeker to seek forward. -// only call this function when n > r.buffered() -func (r *Reader) skipSeek(n int) (int, error) { - o := r.buffered() - // first, clear buffer - n -= o - r.n = 0 - r.data = r.data[:0] - - // then seek forward remaning bytes - i, err := r.rs.Seek(int64(n), 1) - return int(i) + o, err -} - -// Read implements `io.Reader` +// Read implements [io.Reader]. func (r *Reader) Read(b []byte) (int, error) { // if we have data in the buffer, just // return that. if r.buffered() != 0 { x := copy(b, r.data[r.n:]) r.n += x + r.inputOffset += int64(x) return x, nil } + // The buffer was already returned. Surface the error from that read + // before asking the underlying reader again. + if r.state != nil { + return 0, r.err() + } var n int // we have no buffered data; determine // whether or not to buffer or call @@ -299,6 +365,9 @@ func (r *Reader) Read(b []byte) (int, error) { if n == 0 { return 0, r.err() } + + r.inputOffset += int64(n) + return n, nil } @@ -313,17 +382,25 @@ func (r *Reader) ReadFull(b []byte) (int, error) { // either read buffered data, // or read directly for the underlying // buffer, or fetch more buffered data. - for n < l && r.state == nil { + // Buffered bytes are copied even when the fill also returned an error. + for n < l { if r.buffered() != 0 { nn = copy(b[n:], r.data[r.n:]) n += nn r.n += nn - } else if l-n > cap(r.data) { + r.inputOffset += int64(nn) + continue + } + if r.state != nil { + break + } + if l-n > cap(r.data) { nn, r.state = r.r.Read(b[n:]) n += nn - } else { - r.more() + r.inputOffset += int64(nn) + continue } + r.more() } if n < l { return n, r.noEOF() @@ -331,7 +408,7 @@ func (r *Reader) ReadFull(b []byte) (int, error) { return n, nil } -// ReadByte implements `io.ByteReader` +// ReadByte implements [io.ByteReader]. func (r *Reader) ReadByte() (byte, error) { for r.buffered() < 1 && r.state == nil { r.more() @@ -341,10 +418,12 @@ func (r *Reader) ReadByte() (byte, error) { } b := r.data[r.n] r.n++ + r.inputOffset++ + return b, nil } -// WriteTo implements `io.WriterTo` +// WriteTo implements [io.WriterTo]. func (r *Reader) WriteTo(w io.Writer) (int64, error) { var ( i int64 @@ -360,6 +439,7 @@ func (r *Reader) WriteTo(w io.Writer) (int64, error) { } r.data = r.data[0:0] r.n = 0 + r.inputOffset += int64(ii) } for r.state == nil { // here we just do @@ -373,6 +453,7 @@ func (r *Reader) WriteTo(w io.Writer) (int64, error) { } r.data = r.data[0:0] r.n = 0 + r.inputOffset += int64(ii) } } if r.state != io.EOF { @@ -381,13 +462,6 @@ func (r *Reader) WriteTo(w io.Writer) (int64, error) { return i, nil } -func min(a int, b int) int { - if a < b { - return a - } - return b -} - func max(a int, b int) int { if a < b { return b diff --git a/reader_test.go b/reader_test.go index c9a2d0a..378fd26 100644 --- a/reader_test.go +++ b/reader_test.go @@ -5,6 +5,7 @@ import ( "io" "io/ioutil" "math/rand" + "os" "testing" "unsafe" ) @@ -133,6 +134,10 @@ func TestReadByte(t *testing.T) { } } +func remaining(r *Reader) int { + return r.Buffered() + r.r.(partialReader).r.(*bytes.Reader).Len() +} + func TestSkipNoSeek(t *testing.T) { bts := randomBts(1024) rd := NewReaderSize(partialReader{bytes.NewReader(bts)}, 200) @@ -145,6 +150,10 @@ func TestSkipNoSeek(t *testing.T) { t.Fatalf("Skip() returned a nil error, but skipped %d bytes instead of %d", n, 512) } + if remaining(rd) != 512 { + t.Errorf("expected 512 remaining; got %d", remaining(rd)) + } + var b byte b, err = rd.ReadByte() if err != nil { @@ -152,7 +161,7 @@ func TestSkipNoSeek(t *testing.T) { } if b != bts[512] { - t.Fatalf("at index %d: %d in; %d out", 512, bts[512], b) + t.Errorf("at index %d: %d in; %d out", 512, bts[512], b) } n, err = rd.Skip(10) @@ -162,16 +171,38 @@ func TestSkipNoSeek(t *testing.T) { if n != 10 { t.Fatalf("Skip() returned a nil error, but skipped %d bytes instead of %d", n, 10) } + // the number of bytes remaining in the buffer needs + // to comport with the number of bytes we expect to have skipped + if want := 1024 - 512 - 10 - 1; remaining(rd) != want { + t.Errorf("only %d bytes remaining (want %d)?", remaining(rd), want) + } + n, err = rd.Skip(10) + if err != nil { + t.Fatal(err) + } + if n != 10 { + t.Fatalf("Skip(10) a second time returned %d", n) + } + if want := 1024 - 512 - 10 - 10 - 1; remaining(rd) != want { + t.Errorf("only %d bytes remaining (want %d)?", remaining(rd), want) + } + b, err = rd.ReadByte() + if err != nil { + t.Fatalf("second ReadByte(): %s", err) + } + if b != bts[512+10+10+1] { + t.Errorf("expected %d but got %d", bts[512+10+10], b) + } - // now try to skip past the end - rd = NewReaderSize(partialReader{bytes.NewReader(bts)}, 200) - + // now try to skip past the end; we expect + // only to skip the number of bytes remaining + want := remaining(rd) n, err = rd.Skip(2000) if err != io.ErrUnexpectedEOF { t.Fatalf("expected error %q; got %q", io.EOF, err) } - if n != 1024 { - t.Fatalf("expected to skip only 1024 bytes; skipped %d", n) + if n != want { + t.Fatalf("expected to skip only %d bytes; skipped %d", want, n) } } @@ -207,6 +238,10 @@ func TestSkipSeek(t *testing.T) { t.Fatalf("Skip() returned a nil error, but skipped %d bytes instead of %d", n, 10) } + if rd.InputOffset() != 512+1+10 { + t.Errorf("input offset = %d", rd.InputOffset()) + } + // now try to skip past the end rd.Reset(bytes.NewReader(bts)) @@ -221,6 +256,10 @@ func TestSkipSeek(t *testing.T) { t.Fatalf("should have returned %d bytes; returned %d", 2000, n) } + if rd.InputOffset() != 2000 { + t.Fatalf("input offset: %d", rd.InputOffset()) + } + // the next call to Read() // should return io.EOF n, err = rd.Read([]byte{0, 0, 0}) @@ -230,7 +269,6 @@ func TestSkipSeek(t *testing.T) { if n != 0 { t.Errorf("expected 0 bytes read; got %d", n) } - } func TestPeek(t *testing.T) { @@ -275,6 +313,35 @@ func TestPeek(t *testing.T) { } } +func TestPeekByte(t *testing.T) { + bts := randomBts(1024) + rd := NewReaderSize(partialReader{bytes.NewReader(bts)}, 200) + + // first, a peek < buffer size + var ( + peek byte + err error + ) + rd.Skip(100) + peek, err = rd.PeekByte() + if err != nil { + t.Fatal(err) + } + if peek != bts[100] { + t.Fatalf("peeked byte not equal: want %d got %d", bts[100], peek) + } + + // now, a peek > buffer size + rd.Skip(156) + peek, err = rd.PeekByte() + if err != nil { + t.Fatal(err) + } + if peek != bts[256] { + t.Fatalf("peeked byte not equal: want %d got %d", bts[256], peek) + } +} + func TestNext(t *testing.T) { size := 1024 bts := randomBts(size) @@ -353,6 +420,88 @@ func TestReadFull(t *testing.T) { } } +func TestInputOffset(t *testing.T) { + bts := randomBts(1024) + rd := NewReaderSize(partialReader{bytes.NewReader(bts)}, 128) + + if rd.InputOffset() != 0 { + t.Errorf("expected offset 0; got %d", rd.InputOffset()) + } + + // read a few bytes + rd.ReadFull(make([]byte, 10)) + + if rd.InputOffset() != 10 { + t.Errorf("expected offset 10; got %d", rd.InputOffset()) + } + + rd.Peek(384) + + // peeking doesn't advance the offset + if rd.InputOffset() != 10 { + t.Errorf("expected offset 10; got %d", rd.InputOffset()) + } + + rd.Next(246) + + if rd.InputOffset() != 256 { + t.Errorf("expected offset 256; got %d", rd.InputOffset()) + } + + rd.Skip(128) + + if rd.InputOffset() != 384 { + t.Errorf("expected offset 384; got %d", rd.InputOffset()) + } + + rd.ReadByte() + + if rd.InputOffset() != 385 { + t.Errorf("expected offset 385; got %d", rd.InputOffset()) + } + + n, _ := rd.Read(make([]byte, 128)) + + if rd.InputOffset() != int64(385+n) { + t.Errorf("expected offset %d; got %d", 385+n, rd.InputOffset()) + } + + rd.WriteTo(ioutil.Discard) + + if rd.InputOffset() != 1024 { + t.Errorf("expected offset 1024; got %d", rd.InputOffset()) + } + + // try to read more + _, err := rd.Read(make([]byte, 32)) + if err != io.EOF { + t.Fatalf("expected error %q; got %q", io.EOF, err) + } + + if rd.InputOffset() != 1024 { + t.Errorf("expected offset 1024; got %d", rd.InputOffset()) + } + + // reset the reader + rd.Reset(bytes.NewReader(bts)) + + if rd.InputOffset() != 0 { + t.Errorf("expected offset 0; got %d", rd.InputOffset()) + } + + rd.Skip(768 + 32) + + if rd.InputOffset() != 800 { + t.Errorf("expected offset 800; got %d", rd.InputOffset()) + } + + rd.WriteTo(ioutil.Discard) + + if rd.InputOffset() != 1024 { + t.Errorf("expected offset 1024; got %d", rd.InputOffset()) + } +} + type readCounter struct { r io.Reader count int @@ -421,3 +570,74 @@ func TestReaderBufCreation(t *testing.T) { } } } + +// bytesWithError returns its bytes and a non-EOF error once, then io.EOF. +type bytesWithError struct { + b []byte + err error + did bool +} + +func (r *bytesWithError) Read(p []byte) (int, error) { + if r.did { + return 0, io.EOF + } + r.did = true + return copy(p, r.b), r.err +} + +func TestReadKeepsBytesReturnedWithError(t *testing.T) { + want := []byte{1, 2, 3, 4} + + rd := NewReaderSize(&bytesWithError{b: want, err: io.ErrClosedPipe}, 32) + got, err := rd.Next(len(want)) + if err != nil { + t.Fatalf("Next: %v", err) + } + if !bytes.Equal(got, want) { + t.Fatalf("Next got %x, want %x", got, want) + } + if _, err = rd.Next(1); err != io.ErrClosedPipe { + t.Fatalf("Next after bytes: got %v, want %v", err, io.ErrClosedPipe) + } + + rd = NewReaderSize(&bytesWithError{b: want, err: io.ErrClosedPipe}, 32) + buf := make([]byte, len(want)) + n, err := rd.Read(buf) + if err != nil || n != len(want) || !bytes.Equal(buf[:n], want) { + t.Fatalf("Read: n=%d err=%v buf=%x", n, err, buf[:n]) + } + n, err = rd.Read(buf) + if n != 0 || err != io.ErrClosedPipe { + t.Fatalf("Read after bytes: n=%d err=%v, want 0, %v", n, err, io.ErrClosedPipe) + } + + rd = NewReaderSize(&bytesWithError{b: want, err: io.ErrClosedPipe}, 32) + out := make([]byte, 8) + n, err = rd.ReadFull(out) + if n != len(want) || err != io.ErrClosedPipe || !bytes.Equal(out[:n], want) { + t.Fatalf("ReadFull: n=%d err=%v out=%x", n, err, out[:n]) + } +} + +func TestPipeSeek(t *testing.T) { + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer r.Close() + body := "hello, world!\n" + fr := NewReader(r) + go func() { + defer w.Close() + io.WriteString(w, body) + }() + + n, err := fr.Skip(len(body)) + if err != nil { + t.Fatalf("Skip: %s", err) + } + if n != len(body) { + t.Fatalf("skipped %d instead of %d bytes?", n, len(body)) + } +} diff --git a/writer_appengine.go b/writer_appengine.go index e367f39..a978e3b 100644 --- a/writer_appengine.go +++ b/writer_appengine.go @@ -1,3 +1,4 @@ +//go:build appengine // +build appengine package fwd diff --git a/writer_tinygo.go b/writer_tinygo.go new file mode 100644 index 0000000..c98cd57 --- /dev/null +++ b/writer_tinygo.go @@ -0,0 +1,13 @@ +//go:build tinygo +// +build tinygo + +package fwd + +import ( + "unsafe" +) + +// unsafe cast string as []byte +func unsafestr(b string) []byte { + return unsafe.Slice(unsafe.StringData(b), len(b)) +} diff --git a/writer_unsafe.go b/writer_unsafe.go index a0bf453..dcb7ebf 100644 --- a/writer_unsafe.go +++ b/writer_unsafe.go @@ -1,18 +1,13 @@ -// +build !appengine +//go:build !appengine && !tinygo +// +build !appengine,!tinygo package fwd import ( - "reflect" "unsafe" ) // unsafe cast string as []byte -func unsafestr(b string) []byte { - l := len(b) - return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ - Len: l, - Cap: l, - Data: (*reflect.StringHeader)(unsafe.Pointer(&b)).Data, - })) +func unsafestr(s string) []byte { + return unsafe.Slice(unsafe.StringData(s), len(s)) }