diff --git a/LICENSE.md b/LICENSE.md deleted file mode 100644 index 1ac6a81..0000000 --- a/LICENSE.md +++ /dev/null @@ -1,7 +0,0 @@ -Copyright (c) 2014-2015, Philip Hofer - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index 4e99523..8c7d25a 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,19 @@ # fwd + import "github.com/philhofer/fwd" -[![Go Reference](https://pkg.go.dev/badge/github.com/philhofer/fwd.svg)](https://pkg.go.dev/github.com/philhofer/fwd) - - -`import "github.com/philhofer/fwd"` - -* [Overview](#pkg-overview) -* [Index](#pkg-index) - -## Overview -Package fwd provides a buffered reader +The `fwd` package provides a buffered reader and writer. Each has methods that help improve the encoding/decoding performance of some binary protocols. -The `Writer` and `Reader` type provide similar +The `fwd.Writer` and `fwd.Reader` type provide similar functionality to their counterparts in `bufio`, plus a few extra utility methods that simplify read-ahead and write-ahead. I wrote this package to improve serialization -performance for [github.com/tinylib/msgp](https://github.com/tinylib/msgp), -where it provided about a 2x speedup over `bufio` for certain -workloads. However, care must be taken to understand the semantics of the +performance for http://github.com/philhofer/msgp, +where it provided about a 2x speedup over `bufio`. However, +care must be taken to understand the semantics of the extra methods provided by this package, as they allow the user to access and manipulate the buffer memory directly. @@ -34,7 +26,7 @@ in the stream, and uses the `io.Seeker` interface if the underlying stream implements it. `(*fwd.Reader).Next` returns a slice pointing to the next `n` bytes in the read buffer (like `Peek`), but also increments the read position. This allows users to process streams -in arbitrary block sizes without having to manage appropriately-sized +in aribtrary block sizes without having to manage appropriately-sized slices. Additionally, obviating the need to copy the data from the buffer to another location in memory can improve performance dramatically in CPU-bound applications. @@ -45,48 +37,9 @@ the write position by the length of the returned slice. This allows users to write directly to the end of the buffer. -## Portability - -Because it uses the unsafe package, there are theoretically -no promises about forward or backward portability. - -To stay compatible with tinygo 0.32, unsafestr() has been updated -to use unsafe.Slice() as suggested by -https://tinygo.org/docs/guides/compatibility, which also required -bumping go.mod to require at least go 1.20. -## Index -* [Constants](#pkg-constants) -* [type Reader](#Reader) - * [func NewReader(r io.Reader) *Reader](#NewReader) - * [func NewReaderBuf(r io.Reader, buf []byte) *Reader](#NewReaderBuf) - * [func NewReaderSize(r io.Reader, n int) *Reader](#NewReaderSize) - * [func (r *Reader) BufferSize() int](#Reader.BufferSize) - * [func (r *Reader) Buffered() int](#Reader.Buffered) - * [func (r *Reader) Next(n int) ([]byte, error)](#Reader.Next) - * [func (r *Reader) Peek(n int) ([]byte, error)](#Reader.Peek) - * [func (r *Reader) Read(b []byte) (int, error)](#Reader.Read) - * [func (r *Reader) ReadByte() (byte, error)](#Reader.ReadByte) - * [func (r *Reader) ReadFull(b []byte) (int, error)](#Reader.ReadFull) - * [func (r *Reader) Reset(rd io.Reader)](#Reader.Reset) - * [func (r *Reader) Skip(n int) (int, error)](#Reader.Skip) - * [func (r *Reader) WriteTo(w io.Writer) (int64, error)](#Reader.WriteTo) -* [type Writer](#Writer) - * [func NewWriter(w io.Writer) *Writer](#NewWriter) - * [func NewWriterBuf(w io.Writer, buf []byte) *Writer](#NewWriterBuf) - * [func NewWriterSize(w io.Writer, n int) *Writer](#NewWriterSize) - * [func (w *Writer) BufferSize() int](#Writer.BufferSize) - * [func (w *Writer) Buffered() int](#Writer.Buffered) - * [func (w *Writer) Flush() error](#Writer.Flush) - * [func (w *Writer) Next(n int) ([]byte, error)](#Writer.Next) - * [func (w *Writer) ReadFrom(r io.Reader) (int64, error)](#Writer.ReadFrom) - * [func (w *Writer) Write(p []byte) (int, error)](#Writer.Write) - * [func (w *Writer) WriteByte(b byte) error](#Writer.WriteByte) - * [func (w *Writer) WriteString(s string) (int, error)](#Writer.WriteString) - - -## Constants +## Constants ``` go const ( // DefaultReaderSize is the default size of the read buffer @@ -157,18 +110,17 @@ Buffered returns the number of bytes currently in the buffer func (r *Reader) Next(n int) ([]byte, error) ``` Next returns the next 'n' bytes in the stream. +If the returned slice has a length less than 'n', +an error will also be returned. Unlike Peek, Next advances the reader position. The returned bytes point to the same data as the buffer, so the slice is only valid until the next reader method call. An EOF is considered an unexpected error. -If an the returned slice is less than the -length asked for, an error will be returned, -and the reader position will not be incremented. -### func (\*Reader) Peek +### func (\*Reader) Peek ``` go func (r *Reader) Peek(n int) ([]byte, error) ``` @@ -181,34 +133,33 @@ io.ErrUnexpectedEOF. -### func (\*Reader) Read +### func (\*Reader) Read ``` go func (r *Reader) Read(b []byte) (int, error) ``` -Read implements `io.Reader`. +Read implements `io.Reader` -### func (\*Reader) ReadByte +### func (\*Reader) ReadByte ``` go func (r *Reader) ReadByte() (byte, error) ``` -ReadByte implements `io.ByteReader`. +ReadByte implements `io.ByteReader` -### func (\*Reader) ReadFull +### func (\*Reader) ReadFull ``` go func (r *Reader) ReadFull(b []byte) (int, error) ``` ReadFull attempts to read len(b) bytes into 'b'. It returns the number of bytes read into 'b', and an error if it does not return len(b). -EOF is considered an unexpected error. -### func (\*Reader) Reset +### func (\*Reader) Reset ``` go func (r *Reader) Reset(rd io.Reader) ``` @@ -217,42 +168,39 @@ and the read buffer. -### func (\*Reader) Skip +### func (\*Reader) Skip ``` go func (r *Reader) Skip(n int) (int, error) ``` Skip moves the reader forward 'n' bytes. Returns the number of bytes skipped and any -errors encountered. It is analogous to Seek(n, 1). +errors encountered. It is analagous to Seek(n, 1). If the underlying reader implements io.Seeker, then that method will be used to skip forward. If the reader encounters an EOF before skipping 'n' bytes, it -returns `io.ErrUnexpectedEOF`. If the -underlying reader implements `io.Seeker`, then +returns io.ErrUnexpectedEOF. If the +underlying reader implements io.Seeker, then those rules apply instead. (Many implementations will not return `io.EOF` until the next call -to Read). +to Read.) - -### func (\*Reader) WriteTo +### func (\*Reader) WriteTo ``` go func (r *Reader) WriteTo(w io.Writer) (int64, error) ``` -WriteTo implements `io.WriterTo`. +WriteTo implements `io.WriterTo` - -## type Writer +## type Writer ``` go type Writer struct { // contains filtered or unexported fields } - ``` Writer is a buffered writer @@ -262,7 +210,9 @@ Writer is a buffered writer -### func NewWriter + + +### func NewWriter ``` go func NewWriter(w io.Writer) *Writer ``` @@ -271,24 +221,18 @@ that writes to 'w' and has a buffer that is `DefaultWriterSize` bytes. -### func NewWriterBuf +### func NewWriterSize ``` go -func NewWriterBuf(w io.Writer, buf []byte) *Writer +func NewWriterSize(w io.Writer, size int) *Writer ``` -NewWriterBuf returns a new writer -that writes to 'w' and has 'buf' as a buffer. -'buf' is not used when has smaller capacity than 18, -custom buffer is allocated instead. +NewWriterSize returns a new writer +that writes to 'w' and has a buffer +that is 'size' bytes. -### func NewWriterSize -``` go -func NewWriterSize(w io.Writer, n int) *Writer -``` -NewWriterSize returns a new writer that -writes to 'w' and has a buffer size 'n'. -### func (\*Writer) BufferSize + +### func (\*Writer) BufferSize ``` go func (w *Writer) BufferSize() int ``` @@ -296,7 +240,7 @@ BufferSize returns the maximum size of the buffer. -### func (\*Writer) Buffered +### func (\*Writer) Buffered ``` go func (w *Writer) Buffered() int ``` @@ -305,7 +249,7 @@ in the reader. -### func (\*Writer) Flush +### func (\*Writer) Flush ``` go func (w *Writer) Flush() error ``` @@ -314,7 +258,7 @@ to the underlying writer. -### func (\*Writer) Next +### func (\*Writer) Next ``` go func (w *Writer) Next(n int) ([]byte, error) ``` @@ -322,12 +266,10 @@ Next returns the next 'n' free bytes in the write buffer, flushing the writer as necessary. Next will return `io.ErrShortBuffer` if 'n' is greater than the size of the write buffer. -Calls to 'next' increment the write position by -the size of the returned buffer. -### func (\*Writer) ReadFrom +### func (\*Writer) ReadFrom ``` go func (w *Writer) ReadFrom(r io.Reader) (int64, error) ``` @@ -335,7 +277,7 @@ ReadFrom implements `io.ReaderFrom` -### func (\*Writer) Write +### func (\*Writer) Write ``` go func (w *Writer) Write(p []byte) (int, error) ``` @@ -343,7 +285,7 @@ Write implements `io.Writer` -### func (\*Writer) WriteByte +### func (\*Writer) WriteByte ``` go func (w *Writer) WriteByte(b byte) error ``` @@ -351,11 +293,12 @@ WriteByte implements `io.ByteWriter` -### func (\*Writer) WriteString +### func (\*Writer) WriteString ``` go func (w *Writer) WriteString(s string) (int, error) ``` -WriteString is analogous to Write, but it takes a string. +WriteString is analagous to Write, but it takes a string. + @@ -365,4 +308,4 @@ WriteString is analogous to Write, but it takes a string. - - - -Generated by [godoc2md](https://github.com/davecheney/godoc2md) +Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) \ No newline at end of file diff --git a/go.mod b/go.mod deleted file mode 100644 index 77ccd8f..0000000 --- a/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/philhofer/fwd - -go 1.20 // unsafe.StringData requires go1.20 or later diff --git a/reader.go b/reader.go index 4b31385..b8f0dc5 100644 --- a/reader.go +++ b/reader.go @@ -1,10 +1,10 @@ -// Package fwd provides a buffered reader +// The `fwd` package provides a buffered reader // and writer. Each has methods that help improve // the encoding/decoding performance of some binary // protocols. // -// The [Writer] and [Reader] type provide similar -// functionality to their counterparts in [bufio], plus +// The `fwd.Writer` and `fwd.Reader` type provide similar +// functionality to their counterparts in `bufio`, plus // a few extra utility methods that simplify read-ahead // and write-ahead. I wrote this package to improve serialization // performance for http://github.com/tinylib/msgp, @@ -14,31 +14,27 @@ // the user to access and manipulate the buffer memory // directly. // -// The extra methods for [Reader] are [Reader.Peek], [Reader.Skip] -// and [Reader.Next]. (*fwd.Reader).Peek, unlike (*bufio.Reader).Peek, +// The extra methods for `fwd.Reader` are `Peek`, `Skip` +// and `Next`. `(*fwd.Reader).Peek`, unlike `(*bufio.Reader).Peek`, // will re-allocate the read buffer in order to accommodate arbitrarily -// large read-ahead. (*fwd.Reader).Skip skips the next 'n' bytes -// in the stream, and uses the [io.Seeker] interface if the underlying -// stream implements it. (*fwd.Reader).Next returns a slice pointing -// to the next 'n' bytes in the read buffer (like Reader.Peek), but also +// large read-ahead. `(*fwd.Reader).Skip` skips the next `n` bytes +// in the stream, and uses the `io.Seeker` interface if the underlying +// stream implements it. `(*fwd.Reader).Next` returns a slice pointing +// to the next `n` bytes in the read buffer (like `Peek`), but also // increments the read position. This allows users to process streams -// in arbitrary block sizes without having to manage appropriately-sized +// in aribtrary block sizes without having to manage appropriately-sized // slices. Additionally, obviating the need to copy the data from the // buffer to another location in memory can improve performance dramatically // in CPU-bound applications. // -// [Writer] only has one extra method, which is (*fwd.Writer).Next, which -// returns a slice pointing to the next 'n' bytes of the writer, and increments +// `fwd.Writer` only has one extra method, which is `(*fwd.Writer).Next`, which +// returns a slice pointing to the next `n` bytes of the writer, and increments // the write position by the length of the returned slice. This allows users // to write directly to the end of the buffer. +// package fwd -import ( - "errors" - "io" - "os" - "syscall" -) +import "io" const ( // DefaultReaderSize is the default size of the read buffer @@ -54,24 +50,11 @@ func NewReader(r io.Reader) *Reader { } // NewReaderSize returns a new *Reader that -// reads from 'r' and has a buffer size 'n'. +// reads from 'r' and has a buffer size 'n' func NewReaderSize(r io.Reader, n int) *Reader { - buf := make([]byte, 0, max(n, minReaderSize)) - return NewReaderBuf(r, buf) -} - -// NewReaderBuf returns a new *Reader that -// reads from 'r' and uses 'buf' as a buffer. -// 'buf' is not used when has smaller capacity than 16, -// custom buffer is allocated instead. -func NewReaderBuf(r io.Reader, buf []byte) *Reader { - if cap(buf) < minReaderSize { - buf = make([]byte, 0, minReaderSize) - } - buf = buf[:0] rd := &Reader{ r: r, - data: buf, + data: make([]byte, 0, max(minReaderSize, n)), } if s, ok := r.(io.Seeker); ok { rd.rs = s @@ -84,10 +67,9 @@ type Reader struct { r io.Reader // underlying reader // data[n:len(data)] is buffered data; data[len(data):cap(data)] is free buffer space - data []byte // data - n int // read offset - inputOffset int64 // offset in the input stream - state error // last read error + data []byte // data + n int // read offset + state error // last read error // if the reader past to NewReader was // also an io.Seeker, this is non-nil @@ -100,7 +82,6 @@ func (r *Reader) Reset(rd io.Reader) { r.r = rd r.data = r.data[0:0] r.n = 0 - r.inputOffset = 0 r.state = nil if s, ok := rd.(io.Seeker); ok { r.rs = s @@ -116,11 +97,7 @@ func (r *Reader) more() { // we can supply the maximum number of // bytes to the reader if r.n != 0 { - if r.n < len(r.data) { - r.data = r.data[:copy(r.data[0:], r.data[r.n:])] - } else { - r.data = r.data[:0] - } + r.data = r.data[:copy(r.data[0:], r.data[r.n:])] r.n = 0 } var a int @@ -129,16 +106,7 @@ func (r *Reader) more() { r.state = io.ErrNoProgress return } - // io.Reader requires the caller to process n > 0 before the error. - // A non-EOF error used to return here and drop those bytes. - if a > 0 { - r.data = r.data[:len(r.data)+a] - if r.state == io.EOF { - // discard the io.EOF if we read more than 0 bytes. - // the next call to Read should return io.EOF again. - r.state = nil - } - } + r.data = r.data[:len(r.data)+a] } // pop error @@ -165,9 +133,6 @@ func (r *Reader) Buffered() int { return len(r.data) - r.n } // BufferSize returns the total size of the buffer func (r *Reader) BufferSize() int { return cap(r.data) } -// InputOffset returns the input stream byte offset of the current reader position -func (r *Reader) InputOffset() int64 { return r.inputOffset } - // Peek returns the next 'n' buffered bytes, // reading from the underlying reader if necessary. // It will only return a slice shorter than 'n' bytes @@ -183,7 +148,6 @@ func (r *Reader) Peek(n int) ([]byte, error) { old := r.data[r.n:] r.data = make([]byte, n+r.buffered()) r.data = r.data[:copy(r.data, old)] - r.n = 0 } // keep filling until @@ -201,96 +165,47 @@ func (r *Reader) Peek(n int) ([]byte, error) { return r.data[r.n : r.n+n], nil } -func (r *Reader) PeekByte() (b byte, err error) { - if len(r.data)-r.n >= 1 { - b = r.data[r.n] - } else { - b, err = r.peekByte() - } - return -} - -func (r *Reader) peekByte() (byte, error) { - const n = 1 - if cap(r.data) < n { - old := r.data[r.n:] - r.data = make([]byte, n+r.buffered()) - r.data = r.data[:copy(r.data, old)] - r.n = 0 - } - - // keep filling until - // we hit an error or - // read enough bytes - for r.buffered() < n && r.state == nil { - r.more() - } - - // we must have hit an error - if r.buffered() < n { - return 0, r.err() - } - return r.data[r.n], nil -} - -// discard(n) discards up to 'n' buffered bytes, and -// and returns the number of bytes discarded -func (r *Reader) discard(n int) int { - inbuf := r.buffered() - if inbuf <= n { - r.n = 0 - r.inputOffset += int64(inbuf) - r.data = r.data[:0] - return inbuf - } - r.n += n - r.inputOffset += int64(n) - return n -} - // Skip moves the reader forward 'n' bytes. // Returns the number of bytes skipped and any -// errors encountered. It is analogous to Seek(n, 1). +// errors encountered. It is analagous to Seek(n, 1). // If the underlying reader implements io.Seeker, then // that method will be used to skip forward. // // If the reader encounters // an EOF before skipping 'n' bytes, it -// returns [io.ErrUnexpectedEOF]. If the -// underlying reader implements [io.Seeker], then +// returns io.ErrUnexpectedEOF. If the +// underlying reader implements io.Seeker, then // those rules apply instead. (Many implementations -// will not return [io.EOF] until the next call -// to Read). +// will not return `io.EOF` until the next call +// to Read.) func (r *Reader) Skip(n int) (int, error) { - if n < 0 { - return 0, os.ErrInvalid + + // fast path + if r.buffered() >= n { + r.n += n + return n, nil } - // discard some or all of the current buffer - skipped := r.discard(n) - - // if we can Seek() through the remaining bytes, do that - if n > skipped && r.rs != nil { - offs, err := r.rs.Seek(int64(n-skipped), io.SeekCurrent) - if err == nil { - r.inputOffset = offs - return n, nil - } - // if the io.Seeker is not actually usable, - // just swallow the error and don't try - // seeking again - if !errors.Is(err, syscall.ESPIPE) { - return skipped, err - } - r.rs = nil + // use seeker implementation + // if we can + if r.rs != nil { + return r.skipSeek(n) } - // otherwise, keep filling the buffer - // and discarding it up to 'n' - for skipped < n && r.state == nil { + + // loop on filling + // and then erasing + o := n + for r.buffered() < n && r.state == nil { r.more() - skipped += r.discard(n - skipped) + // we can skip forward + // up to r.buffered() bytes + step := min(r.buffered(), n) + r.n += step + n -= step } - return skipped, r.noEOF() + // at this point, n should be + // 0 if everything went smoothly + return o - n, r.noEOF() } // Next returns the next 'n' bytes in the stream. @@ -302,24 +217,13 @@ func (r *Reader) Skip(n int) (int, error) { // If an the returned slice is less than the // length asked for, an error will be returned, // and the reader position will not be incremented. -func (r *Reader) Next(n int) (b []byte, err error) { - if r.state == nil && len(r.data)-r.n >= n { - b = r.data[r.n : r.n+n] - r.n += n - r.inputOffset += int64(n) - } else { - b, err = r.next(n) - } - return -} +func (r *Reader) Next(n int) ([]byte, error) { -func (r *Reader) next(n int) ([]byte, error) { // in case the buffer is too small if cap(r.data) < n { old := r.data[r.n:] r.data = make([]byte, n+r.buffered()) r.data = r.data[:copy(r.data, old)] - r.n = 0 } // fill at least 'n' bytes @@ -332,43 +236,40 @@ func (r *Reader) next(n int) ([]byte, error) { } out := r.data[r.n : r.n+n] r.n += n - r.inputOffset += int64(n) return out, nil } -// Read implements [io.Reader]. +// skipSeek uses the io.Seeker to seek forward. +// only call this function when n > r.buffered() +func (r *Reader) skipSeek(n int) (int, error) { + o := r.buffered() + // first, clear buffer + n -= o + r.n = 0 + r.data = r.data[:0] + + // then seek forward remaning bytes + i, err := r.rs.Seek(int64(n), 1) + return int(i) + o, err +} + +// Read implements `io.Reader` func (r *Reader) Read(b []byte) (int, error) { - // if we have data in the buffer, just - // return that. - if r.buffered() != 0 { + if len(b) <= r.buffered() { x := copy(b, r.data[r.n:]) r.n += x - r.inputOffset += int64(x) return x, nil } - // The buffer was already returned. Surface the error from that read - // before asking the underlying reader again. - if r.state != nil { - return 0, r.err() - } - var n int - // we have no buffered data; determine - // whether or not to buffer or call - // the underlying reader directly - if len(b) >= cap(r.data) { - n, r.state = r.r.Read(b) - } else { - r.more() - n = copy(b, r.data) - r.n = n - } - if n == 0 { - return 0, r.err() + r.more() + if r.buffered() > 0 { + x := copy(b, r.data[r.n:]) + r.n += x + return x, nil } - r.inputOffset += int64(n) - - return n, nil + // io.Reader is supposed to return + // 0 read bytes on error + return 0, r.err() } // ReadFull attempts to read len(b) bytes into @@ -376,39 +277,23 @@ func (r *Reader) Read(b []byte) (int, error) { // 'b', and an error if it does not return len(b). // EOF is considered an unexpected error. func (r *Reader) ReadFull(b []byte) (int, error) { - var n int // read into b - var nn int // scratch + var x int l := len(b) - // either read buffered data, - // or read directly for the underlying - // buffer, or fetch more buffered data. - // Buffered bytes are copied even when the fill also returned an error. - for n < l { - if r.buffered() != 0 { - nn = copy(b[n:], r.data[r.n:]) - n += nn - r.n += nn - r.inputOffset += int64(nn) - continue + for x < l { + if r.buffered() == 0 { + r.more() } + c := copy(b[x:], r.data[r.n:]) + x += c + r.n += c if r.state != nil { - break + return x, r.noEOF() } - if l-n > cap(r.data) { - nn, r.state = r.r.Read(b[n:]) - n += nn - r.inputOffset += int64(nn) - continue - } - r.more() } - if n < l { - return n, r.noEOF() - } - return n, nil + return x, nil } -// ReadByte implements [io.ByteReader]. +// ReadByte implements `io.ByteReader` func (r *Reader) ReadByte() (byte, error) { for r.buffered() < 1 && r.state == nil { r.more() @@ -418,12 +303,10 @@ func (r *Reader) ReadByte() (byte, error) { } b := r.data[r.n] r.n++ - r.inputOffset++ - return b, nil } -// WriteTo implements [io.WriterTo]. +// WriteTo implements `io.WriterTo` func (r *Reader) WriteTo(w io.Writer) (int64, error) { var ( i int64 @@ -439,7 +322,6 @@ func (r *Reader) WriteTo(w io.Writer) (int64, error) { } r.data = r.data[0:0] r.n = 0 - r.inputOffset += int64(ii) } for r.state == nil { // here we just do @@ -453,7 +335,6 @@ func (r *Reader) WriteTo(w io.Writer) (int64, error) { } r.data = r.data[0:0] r.n = 0 - r.inputOffset += int64(ii) } } if r.state != io.EOF { @@ -462,6 +343,13 @@ func (r *Reader) WriteTo(w io.Writer) (int64, error) { return i, nil } +func min(a int, b int) int { + if a < b { + return a + } + return b +} + func max(a int, b int) int { if a < b { return b diff --git a/reader_test.go b/reader_test.go index 378fd26..7cf89da 100644 --- a/reader_test.go +++ b/reader_test.go @@ -5,7 +5,6 @@ import ( "io" "io/ioutil" "math/rand" - "os" "testing" "unsafe" ) @@ -79,33 +78,6 @@ func TestRead(t *testing.T) { if !bytes.Equal(bts, all) { t.Errorf("bytes not equal; %d bytes in and %d bytes out", len(bts), len(out)) } - - // test filling out of the underlying reader - big := randomBts(1 << 21) - rd = NewReaderSize(partialReader{bytes.NewReader(big)}, 2048) - buf := make([]byte, 3100) - - n, err = rd.ReadFull(buf) - if err != nil { - t.Fatal(err) - } - if n != 3100 { - t.Errorf("expected 3100 bytes read by ReadFull; got %d", n) - } - if !bytes.Equal(buf[:n], big[:n]) { - t.Error("data parity") - } - rest := make([]byte, (1<<21)-3100) - n, err = io.ReadFull(rd, rest) - if err != nil { - t.Fatal(err) - } - if n != len(rest) { - t.Errorf("expected %d bytes read by io.ReadFull; got %d", len(rest), n) - } - if !bytes.Equal(append(buf, rest...), big) { - t.Fatal("data parity") - } } func TestReadByte(t *testing.T) { @@ -134,10 +106,6 @@ func TestReadByte(t *testing.T) { } } -func remaining(r *Reader) int { - return r.Buffered() + r.r.(partialReader).r.(*bytes.Reader).Len() -} - func TestSkipNoSeek(t *testing.T) { bts := randomBts(1024) rd := NewReaderSize(partialReader{bytes.NewReader(bts)}, 200) @@ -150,10 +118,6 @@ func TestSkipNoSeek(t *testing.T) { t.Fatalf("Skip() returned a nil error, but skipped %d bytes instead of %d", n, 512) } - if remaining(rd) != 512 { - t.Errorf("expected 512 remaining; got %d", remaining(rd)) - } - var b byte b, err = rd.ReadByte() if err != nil { @@ -161,7 +125,7 @@ func TestSkipNoSeek(t *testing.T) { } if b != bts[512] { - t.Errorf("at index %d: %d in; %d out", 512, bts[512], b) + t.Fatalf("at index %d: %d in; %d out", 512, bts[512], b) } n, err = rd.Skip(10) @@ -171,38 +135,16 @@ func TestSkipNoSeek(t *testing.T) { if n != 10 { t.Fatalf("Skip() returned a nil error, but skipped %d bytes instead of %d", n, 10) } - // the number of bytes remaining in the buffer needs - // to comport with the number of bytes we expect to have skipped - if want := 1024 - 512 - 10 - 1; remaining(rd) != want { - t.Errorf("only %d bytes remaining (want %d)?", remaining(rd), want) - } - n, err = rd.Skip(10) - if err != nil { - t.Fatal(err) - } - if n != 10 { - t.Fatalf("Skip(10) a second time returned %d", n) - } - if want := 1024 - 512 - 10 - 10 - 1; remaining(rd) != want { - t.Errorf("only %d bytes remaining (want %d)?", remaining(rd), want) - } - b, err = rd.ReadByte() - if err != nil { - t.Fatalf("second ReadByte(): %s", err) - } - if b != bts[512+10+10+1] { - t.Errorf("expected %d but got %d", bts[512+10+10], b) - } - // now try to skip past the end; we expect - // only to skip the number of bytes remaining - want := remaining(rd) + // now try to skip past the end + rd = NewReaderSize(partialReader{bytes.NewReader(bts)}, 200) + n, err = rd.Skip(2000) if err != io.ErrUnexpectedEOF { t.Fatalf("expected error %q; got %q", io.EOF, err) } - if n != want { - t.Fatalf("expected to skip only %d bytes; skipped %d", want, n) + if n != 1024 { + t.Fatalf("expected to skip only 1024 bytes; skipped %d", n) } } @@ -238,10 +180,6 @@ func TestSkipSeek(t *testing.T) { t.Fatalf("Skip() returned a nil error, but skipped %d bytes instead of %d", n, 10) } - if rd.InputOffset() != 512+1+10 { - t.Errorf("input offset = %d", rd.InputOffset()) - } - // now try to skip past the end rd.Reset(bytes.NewReader(bts)) @@ -256,10 +194,6 @@ func TestSkipSeek(t *testing.T) { t.Fatalf("should have returned %d bytes; returned %d", 2000, n) } - if rd.InputOffset() != 2000 { - t.Fatalf("input offset: %d", rd.InputOffset()) - } - // the next call to Read() // should return io.EOF n, err = rd.Read([]byte{0, 0, 0}) @@ -269,6 +203,7 @@ func TestSkipSeek(t *testing.T) { if n != 0 { t.Errorf("expected 0 bytes read; got %d", n) } + } func TestPeek(t *testing.T) { @@ -313,35 +248,6 @@ func TestPeek(t *testing.T) { } } -func TestPeekByte(t *testing.T) { - bts := randomBts(1024) - rd := NewReaderSize(partialReader{bytes.NewReader(bts)}, 200) - - // first, a peek < buffer size - var ( - peek byte - err error - ) - rd.Skip(100) - peek, err = rd.PeekByte() - if err != nil { - t.Fatal(err) - } - if peek != bts[100] { - t.Fatalf("peeked byte not equal: want %d got %d", bts[100], peek) - } - - // now, a peek > buffer size - rd.Skip(156) - peek, err = rd.PeekByte() - if err != nil { - t.Fatal(err) - } - if peek != bts[256] { - t.Fatalf("peeked byte not equal: want %d got %d", bts[256], peek) - } -} - func TestNext(t *testing.T) { size := 1024 bts := randomBts(size) @@ -401,12 +307,6 @@ func TestReadFull(t *testing.T) { t.Fatal("bytes not equal") } - // we've read everything; this should EOF - n, err = rd.Read(out) - if err != io.EOF { - t.Fatalf("expected %q; got %q", io.EOF, err) - } - rd.Reset(partialReader{bytes.NewReader(bts)}) // now try to read *past* EOF @@ -419,225 +319,3 @@ func TestReadFull(t *testing.T) { t.Fatalf("expected to read %d bytes; read %d", 1024, n) } } - -func TestInputOffset(t *testing.T) { - bts := randomBts(1024) - rd := NewReaderSize(partialReader{bytes.NewReader(bts)}, 128) - - if rd.InputOffset() != 0 { - t.Errorf("expected offset 0; got %d", rd.InputOffset()) - } - - // read a few bytes - rd.ReadFull(make([]byte, 10)) - - if rd.InputOffset() != 10 { - t.Errorf("expected offset 10; got %d", rd.InputOffset()) - } - - rd.Peek(384) - - // peeking doesn't advance the offset - if rd.InputOffset() != 10 { - t.Errorf("expected offset 10; got %d", rd.InputOffset()) - } - - rd.Next(246) - - if rd.InputOffset() != 256 { - t.Errorf("expected offset 256; got %d", rd.InputOffset()) - } - - rd.Skip(128) - - if rd.InputOffset() != 384 { - t.Errorf("expected offset 384; got %d", rd.InputOffset()) - } - - rd.ReadByte() - - if rd.InputOffset() != 385 { - t.Errorf("expected offset 385; got %d", rd.InputOffset()) - } - - n, _ := rd.Read(make([]byte, 128)) - - if rd.InputOffset() != int64(385+n) { - t.Errorf("expected offset %d; got %d", 385+n, rd.InputOffset()) - } - - rd.WriteTo(ioutil.Discard) - - if rd.InputOffset() != 1024 { - t.Errorf("expected offset 1024; got %d", rd.InputOffset()) - } - - // try to read more - _, err := rd.Read(make([]byte, 32)) - if err != io.EOF { - t.Fatalf("expected error %q; got %q", io.EOF, err) - } - - if rd.InputOffset() != 1024 { - t.Errorf("expected offset 1024; got %d", rd.InputOffset()) - } - - // reset the reader - rd.Reset(bytes.NewReader(bts)) - - if rd.InputOffset() != 0 { - t.Errorf("expected offset 0; got %d", rd.InputOffset()) - } - - rd.Skip(768 + 32) - - if rd.InputOffset() != 800 { - t.Errorf("expected offset 800; got %d", rd.InputOffset()) - } - - rd.WriteTo(ioutil.Discard) - - if rd.InputOffset() != 1024 { - t.Errorf("expected offset 1024; got %d", rd.InputOffset()) - } -} - -type readCounter struct { - r io.Reader - count int -} - -func (r *readCounter) Read(p []byte) (int, error) { - r.count++ - return r.r.Read(p) -} - -func TestReadFullPerf(t *testing.T) { - const size = 1 << 22 - data := randomBts(size) - - c := readCounter{ - r: &partialReader{ - r: bytes.NewReader(data), - }, - } - - r := NewReader(&c) - - const segments = 4 - out := make([]byte, size/segments) - - for i := 0; i < segments; i++ { - // force an unaligned read - _, err := r.Peek(5) - if err != nil { - t.Fatal(err) - } - - n, err := r.ReadFull(out) - if err != nil { - t.Fatal(err) - } - if n != size/segments { - t.Fatalf("read %d bytes, not %d", n, size/segments) - } - } - - t.Logf("called Read() on the underlying reader %d times to fill %d buffers", c.count, size/r.BufferSize()) -} - -func TestReaderBufCreation(t *testing.T) { - tests := []struct { - name string - buffer []byte - size int - }{ - {name: "nil", buffer: nil, size: minReaderSize}, - {name: "empty", buffer: []byte{}, size: minReaderSize}, - {name: "allocated", buffer: make([]byte, 0, 200), size: 200}, - {name: "filled", buffer: make([]byte, 200), size: 200}, - } - - for _, test := range tests { - var b bytes.Buffer - r := NewReaderBuf(&b, test.buffer) - - if r.BufferSize() != test.size { - t.Errorf("%s: unequal buffer size (got: %d, expected: %d)", test.name, r.BufferSize(), test.size) - } - if r.Buffered() != 0 { - t.Errorf("%s: unequal buffered bytes (got: %d, expected: 0)", test.name, r.Buffered()) - } - } -} - -// bytesWithError returns its bytes and a non-EOF error once, then io.EOF. -type bytesWithError struct { - b []byte - err error - did bool -} - -func (r *bytesWithError) Read(p []byte) (int, error) { - if r.did { - return 0, io.EOF - } - r.did = true - return copy(p, r.b), r.err -} - -func TestReadKeepsBytesReturnedWithError(t *testing.T) { - want := []byte{1, 2, 3, 4} - - rd := NewReaderSize(&bytesWithError{b: want, err: io.ErrClosedPipe}, 32) - got, err := rd.Next(len(want)) - if err != nil { - t.Fatalf("Next: %v", err) - } - if !bytes.Equal(got, want) { - t.Fatalf("Next got %x, want %x", got, want) - } - if _, err = rd.Next(1); err != io.ErrClosedPipe { - t.Fatalf("Next after bytes: got %v, want %v", err, io.ErrClosedPipe) - } - - rd = NewReaderSize(&bytesWithError{b: want, err: io.ErrClosedPipe}, 32) - buf := make([]byte, len(want)) - n, err := rd.Read(buf) - if err != nil || n != len(want) || !bytes.Equal(buf[:n], want) { - t.Fatalf("Read: n=%d err=%v buf=%x", n, err, buf[:n]) - } - n, err = rd.Read(buf) - if n != 0 || err != io.ErrClosedPipe { - t.Fatalf("Read after bytes: n=%d err=%v, want 0, %v", n, err, io.ErrClosedPipe) - } - - rd = NewReaderSize(&bytesWithError{b: want, err: io.ErrClosedPipe}, 32) - out := make([]byte, 8) - n, err = rd.ReadFull(out) - if n != len(want) || err != io.ErrClosedPipe || !bytes.Equal(out[:n], want) { - t.Fatalf("ReadFull: n=%d err=%v out=%x", n, err, out[:n]) - } -} - -func TestPipeSeek(t *testing.T) { - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - defer r.Close() - body := "hello, world!\n" - fr := NewReader(r) - go func() { - defer w.Close() - io.WriteString(w, body) - }() - - n, err := fr.Skip(len(body)) - if err != nil { - t.Fatalf("Skip: %s", err) - } - if n != len(body) { - t.Fatalf("skipped %d instead of %d bytes?", n, len(body)) - } -} diff --git a/writer.go b/writer.go index 4d6ea15..d6f44f5 100644 --- a/writer.go +++ b/writer.go @@ -29,28 +29,16 @@ func NewWriter(w io.Writer) *Writer { } } -// NewWriterSize returns a new writer that -// writes to 'w' and has a buffer size 'n'. -func NewWriterSize(w io.Writer, n int) *Writer { - if wr, ok := w.(*Writer); ok && cap(wr.buf) >= n { +// NewWriterSize returns a new writer +// that writes to 'w' and has a buffer +// that is 'size' bytes. +func NewWriterSize(w io.Writer, size int) *Writer { + if wr, ok := w.(*Writer); ok && cap(wr.buf) >= size { return wr } - buf := make([]byte, 0, max(n, minWriterSize)) - return NewWriterBuf(w, buf) -} - -// NewWriterBuf returns a new writer -// that writes to 'w' and has 'buf' as a buffer. -// 'buf' is not used when has smaller capacity than 18, -// custom buffer is allocated instead. -func NewWriterBuf(w io.Writer, buf []byte) *Writer { - if cap(buf) < minWriterSize { - buf = make([]byte, 0, minWriterSize) - } - buf = buf[:0] return &Writer{ w: w, - buf: buf, + buf: make([]byte, 0, max(size, minWriterSize)), } } @@ -110,7 +98,7 @@ func (w *Writer) Write(p []byte) (int, error) { return copy(w.buf[l:], p), nil } -// WriteString is analogous to Write, but it takes a string. +// WriteString is analagous to Write, but it takes a string. func (w *Writer) WriteString(s string) (int, error) { c, l, ln := cap(w.buf), len(w.buf), len(s) avail := c - l @@ -227,7 +215,7 @@ func (w *Writer) ReadFrom(r io.Reader) (int64, error) { // we only clear here // because we are sure // the writes have - // succeeded. otherwise, + // suceeded. otherwise, // we retain the data in case // future writes succeed. w.buf = w.buf[0:0] diff --git a/writer_appengine.go b/writer_appengine.go index a978e3b..e367f39 100644 --- a/writer_appengine.go +++ b/writer_appengine.go @@ -1,4 +1,3 @@ -//go:build appengine // +build appengine package fwd diff --git a/writer_test.go b/writer_test.go index 5868d29..8bab07c 100644 --- a/writer_test.go +++ b/writer_test.go @@ -29,7 +29,7 @@ func (c chunkedWriter) Write(p []byte) (int, error) { return n, nil } -// analogous to Write(), but w/ str +// analagous to Write(), but w/ str func (c chunkedWriter) WriteString(s string) (int, error) { l := len(s) n := 0 @@ -235,29 +235,5 @@ func TestReadFrom(t *testing.T) { if !bytes.Equal(buf.Bytes(), bts) { t.Fatal("buf.Bytes() and input are not equal") } -} - -func TestWriterBufCreation(t *testing.T) { - tests := []struct { - name string - buffer []byte - size int - }{ - {name: "nil", buffer: nil, size: minWriterSize}, - {name: "empty", buffer: []byte{}, size: minWriterSize}, - {name: "allocated", buffer: make([]byte, 0, 200), size: 200}, - {name: "filled", buffer: make([]byte, 200), size: 200}, - } - - for _, test := range tests { - var b bytes.Buffer - w := NewWriterBuf(&b, test.buffer) - - if w.BufferSize() != test.size { - t.Errorf("%s: unequal buffer size (got: %d, expected: %d)", test.name, w.BufferSize(), test.size) - } - if w.Buffered() != 0 { - t.Errorf("%s: unequal buffered bytes (got: %d, expected: 0)", test.name, w.Buffered()) - } - } + } diff --git a/writer_tinygo.go b/writer_tinygo.go deleted file mode 100644 index c98cd57..0000000 --- a/writer_tinygo.go +++ /dev/null @@ -1,13 +0,0 @@ -//go:build tinygo -// +build tinygo - -package fwd - -import ( - "unsafe" -) - -// unsafe cast string as []byte -func unsafestr(b string) []byte { - return unsafe.Slice(unsafe.StringData(b), len(b)) -} diff --git a/writer_unsafe.go b/writer_unsafe.go index dcb7ebf..a0bf453 100644 --- a/writer_unsafe.go +++ b/writer_unsafe.go @@ -1,13 +1,18 @@ -//go:build !appengine && !tinygo -// +build !appengine,!tinygo +// +build !appengine package fwd import ( + "reflect" "unsafe" ) // unsafe cast string as []byte -func unsafestr(s string) []byte { - return unsafe.Slice(unsafe.StringData(s), len(s)) +func unsafestr(b string) []byte { + l := len(b) + return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ + Len: l, + Cap: l, + Data: (*reflect.StringHeader)(unsafe.Pointer(&b)).Data, + })) }