diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..638fff8 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,18 @@ +on: [push, pull_request] +name: Test +jobs: + test: + strategy: + matrix: + go-version: [1.16.x, 1.17.x] + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + - name: Checkout code + uses: actions/checkout@v2 + - name: Test all + run: go test ./... diff --git a/README.md b/README.md index 8016fa9..2c89316 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,21 @@ # ZTYP Another approach to SSZ, focused on typing around merkle-tree-representations of state. -For a super-fast but non-tree approach, try [ZSSZ](https://github.com/protolambda/zssz). -Work in progress. No serialization/deserialization for types yet. +ZTYP mirrors most features of my Python +implementation [`remerkleable`](https://github.com/protolambda/remerkleable), + +In addition to tree structures and views, +ZTYP also provides encoding/decoding utils for flat native Go structures, in the `codec` package. + +[ZRNT](https://github.com/protolambda/zrnt) uses both the ZTYP tree structures (state) and flat utils (messages) +to implement the Eth2 API spec. + ## Design Construction: -- Ztyp describes types as interfaces to a binary merkle tree: views over a backing tree. +- ZTYP describes types as interfaces to a binary merkle tree: views over a backing tree. - Every node in the backing tree is referred to as a `Node`, build with: - `Commit`: combines two subtrees - `Root`: a bytes32. Can be a summary root of an unexpanded subtree, or a just a leaf node. @@ -21,20 +28,21 @@ Construction: - Convert a `Root` and sub-index into a typed sub-view by attaching a basic type definition. - a `View` is used to interact with a subtree. A general view only tracks its backing. - Typed views allow you to interact with a backing in typed ways: - - Basic types: `Uint64Type`, `Uint32Type`, `Uint16Type`, `Uint8Type`, `BoolType` - - Composite types: `Container`, `List`, `Vector` + - Basic types: `Uint256Type`, `Uint64Type`, `Uint32Type`, `Uint16Type`, `Uint8Type`, `BoolType` (`uint128` is not supported) + - Composite types: `Container`, `ComplexList`, `ComplexVector` + - Union type: `UnionType` - Basic composite types (to enable packing of consecutive elements): `BasicList`, `BasicVector` - - Bitfields: `Bitvector`, `Bitlist` + - Bitfields: `BitVector`, `BitList` - Optimized small byte vectors: `SmallByteVecMeta`: to derive any `BytesN` (`N <= 32`) from. - - `Root` itself is also a valid view. + - `RootView` for an efficient 32 byte (single node) immutable view. - Semi-typed views are useful to build your own types: `SubtreeView` - `ReadProp`/`WriteProp` functions can be used to describe reusable `Getter/Setter -> View -> *my-type*` pipelines. - Take a typed view that has getters/setters, create closure to fix it to a property index - Type the property function and add a receiver func to return the typed view instead. - Type `ReadPropFn` and `WritePropFn` with your own function types to not repeat the boilerplate. -- `ViewHook`s enable you to create smaller views attached to their parent views. - Program like you are mutating references, and have the view-hook propagate up the changes. -- The backing tree can be partial, and summarised/expanded dynamically. The typedefinition will safely handle a tree, +- `BackingHook`s enable you to create smaller views attached to their parent views. + Program like you are mutating references, and have the backing-hook propagate up the changes. +- The backing tree can be partial, and summarised/expanded dynamically. The type-definition will safely handle a tree, and return an error when the expected data for an operation is inconsistent or missing. - The hash-function for hash-tree-root is: - passed by reference, to reuse a single state during hashing. diff --git a/bitfields/bitfield.go b/bitfields/bitfield.go new file mode 100644 index 0000000..d7b7570 --- /dev/null +++ b/bitfields/bitfield.go @@ -0,0 +1,103 @@ +package bitfields + +import "fmt" + +// Note: bitfield indices and lengths are generally all uint32, as this is used in SSZ for lengths too. + +// General base interface for Bitlists and Bitvectors +// Note for Bitfields to work with the SSZ functionality: +// - Bitlists need to be of kind []byte (packed bits, incl delimiter bit) +// - Bitvectors need to be of kind [N]byte (packed bits) +type Bitfield interface { + Get(i uint64) bool + Set(i uint64, v bool) +} + +// bitfields implementing this can be checked to be of a valid or not. Useful for untrusted bitfields. +// See BitlistCheck and BitvectorCheck to easily implement the validity checks. +type CheckedBitfield interface { + Check() error +} + +// the exact bitlength can be determined for bitfields implementing this method. +type SizedBits interface { + BitLen() uint64 +} + +// Get index of left-most 1 bit. +// 0 (incl.) to 8 (excl.) +func BitIndex(v byte) (out uint64) { + // going to be prettier with new Go 1.13 binary constant syntax + if v&0xf0 != 0 { // 11110000 + out |= 4 + v >>= 4 + } + if v&0x0c != 0 { // 00001100 + out |= 2 + v >>= 2 + } + if v&0x02 != 0 { // 00000010 + out |= 1 + v >>= 1 + } + return +} + +// Helper function to implement Bitfields with. +// Assumes i is a valid bit-index to retrieve a bit from bytes b. +func GetBit(b []byte, i uint64) bool { + return (b[i>>3]>>(i&7))&1 == 1 +} + +// Helper function to implement Bitfields with. +// Assumes i is a valid bit-index to set a bit within bytes b. +func SetBit(b []byte, i uint64, v bool) { + if bit := byte(1) << (i & 7); v { + b[i>>3] |= bit + } else { + b[i>>3] &^= bit + } +} + +// Checks if the bitList is fully zeroed (except the leading bit) +func IsZeroBitlist(b []byte) bool { + end := len(b) + if end == 0 { + return true + } + end -= 1 + if end > 0 { + // bytes up to the end byte can be checked efficiently. No need for bit-magic. + for i := 0; i < end; i++ { + if b[i] != 0 { + return false + } + } + } + last := b[end] + if last == 0 { + // invalid bitlist, but reasonably zero + return true + } + // now check the last bit, but ignore the delimiter 1 bit. + last ^= byte(1) << BitIndex(last) + return last == 0 +} + +// Returns true if bf only has bits set to 1 that bitfield af also has set to 1 +// For bitlists and bitvectors, the trailing part is always the same if the bitlength is the same. +// So the result is independent of having a delimiting bit or not. +func Covers(af []byte, bf []byte) (bool, error) { + if a, b := len(af), len(bf); a != b { + return false, fmt.Errorf("bitfield byte-length mismatch: %d <> %d", a, b) + } + // both bitfields have the same delimiter bit set (and same zero padding), if any. + // We can ignore it, it won't change the outcome. + for i := 0; i < len(bf); i++ { + // if there are any bits set in b, that are not also set in a, then a does not cover. + if bf[i]&^af[i] != 0 { + return false, nil + } + } + return true, nil +} diff --git a/bitfields/bitfield_test.go b/bitfields/bitfield_test.go new file mode 100644 index 0000000..352cc4c --- /dev/null +++ b/bitfields/bitfield_test.go @@ -0,0 +1,220 @@ +package bitfields + +import ( + "fmt" + "testing" +) + +func TestBitIndex(t *testing.T) { + cases := []struct { + v byte + index uint64 + }{ + {0, 0}, + {1, 0}, + {2, 1}, + {3, 1}, + {4, 2}, + {34, 5}, + {127, 6}, + {128, 7}, + {255, 7}, + } + for _, testCase := range cases { + t.Run(fmt.Sprintf("v %b (bin) index %d", testCase.v, testCase.index), func(t *testing.T) { + if res := BitIndex(testCase.v); res != testCase.index { + t.Errorf("unexpected bit index: %d for value %b (bin), expected index: %d", + res, testCase.v, testCase.index) + } + }) + } +} + +func BenchmarkBitIndex(b *testing.B) { + // sum results for fun, and verify it has the same result with different benched solutions with same N. + out := uint64(0) + b.StartTimer() + for i := 0; i < b.N; i++ { + out += BitIndex(byte(i)) + } + b.StopTimer() + b.Logf("result after %d runs: %d", b.N, out) +} + +// For speed performance comparison. 2x faster, but dependent on global 256 bytes var. +// (Also consider heap/stack location, take 2x with a grain of salt) +var lookup = [256]byte{ + 0, + 0, + 1, 1, + 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, +} + +func BenchmarkLookupBitIndex(b *testing.B) { + // sum results for fun, and verify it has the same result with different benched solutions with same N. + out := uint64(0) + b.StartTimer() + for i := 0; i < b.N; i++ { + out += uint64(lookup[byte(i)]) + } + b.StopTimer() + b.Logf("result after %d runs: %d", b.N, out) +} + +func TestSetBit(t *testing.T) { + cases := []struct { + v []byte + index uint64 + expected1 string + expected0 string + }{ + {[]byte{0}, 0, "00000001 ", "00000000 "}, + {[]byte{1}, 1, "00000011 ", "00000001 "}, + {[]byte{1}, 2, "00000101 ", "00000001 "}, + {[]byte{8}, 1, "00001010 ", "00001000 "}, + {[]byte{10}, 1, "00001010 ", "00001000 "}, + {[]byte{255}, 5, "11111111 ", "11011111 "}, + {[]byte{0x00, 0x00}, 0, "00000001 00000000 ", "00000000 00000000 "}, + {[]byte{0x00, 0x00}, 3, "00001000 00000000 ", "00000000 00000000 "}, + {[]byte{0x00, 0x00}, 15, "00000000 10000000 ", "00000000 00000000 "}, + {[]byte{0xff, 0xff}, 0, "11111111 11111111 ", "11111110 11111111 "}, + {[]byte{0xff, 0xff}, 5, "11111111 11111111 ", "11011111 11111111 "}, + {[]byte{0x13, 0x37}, 5, "00110011 00110111 ", "00010011 00110111 "}, + {[]byte{0x13, 0x37}, 8, "00010011 00110111 ", "00010011 00110110 "}, + {[]byte{0xff, 0xff}, 10, "11111111 11111111 ", "11111111 11111011 "}, + {[]byte{0x13, 0x37}, 10, "00010011 00110111 ", "00010011 00110011 "}, + {[]byte{0xff, 0xff}, 15, "11111111 11111111 ", "11111111 01111111 "}, + } + for _, testCase := range cases { + t.Run(fmt.Sprintf("v %b (bin) index %d", testCase.v, testCase.index), func(t *testing.T) { + t.Run("set to 1", func(t *testing.T) { + a := make([]byte, len(testCase.v)) + copy(a, testCase.v) + SetBit(a, testCase.index, true) + res := "" + for _, b := range a { + res += fmt.Sprintf("%08b ", b) + } + if res != testCase.expected1 { + t.Errorf("expected %s but got %s", testCase.expected1, res) + } + }) + t.Run("set to 0", func(t *testing.T) { + a := make([]byte, len(testCase.v)) + copy(a, testCase.v) + SetBit(a, testCase.index, false) + res := "" + for _, b := range a { + res += fmt.Sprintf("%08b ", b) + } + if res != testCase.expected0 { + t.Errorf("expected %s but got %s", testCase.expected1, res) + } + }) + t.Run("get bit", func(t *testing.T) { + data := "" + for _, b := range testCase.v { + data += fmt.Sprintf("%08b", b) + } + bit := GetBit(testCase.v, testCase.index) + expected := data[(testCase.index/8)*8+7-(testCase.index%8)] == '1' + if bit != expected { + t.Errorf("expected %v but got %v, data: %s", bit, expected, data) + } + }) + }) + } +} + +func TestIsZeroBitlist(t *testing.T) { + cases := []struct { + v []byte + isZero bool + }{ + {[]byte{0}, true}, + {[]byte{1}, true}, + {[]byte{2}, true}, + {[]byte{3}, false}, + {[]byte{5}, false}, + {[]byte{8}, true}, + {[]byte{0x80}, true}, + {[]byte{0x81}, false}, + {[]byte{0x90}, false}, + {[]byte{0, 0}, true}, + {[]byte{0, 1}, true}, + {[]byte{0, 2}, true}, + {[]byte{2, 2}, false}, + {[]byte{0, 3}, false}, + {[]byte{0, 5}, false}, + {[]byte{0, 8}, true}, + {[]byte{0, 0x80}, true}, + {[]byte{3, 0x80}, false}, + {[]byte{0, 0x81}, false}, + {[]byte{0, 0x90}, false}, + {[]byte{0, 0, 0}, true}, + {[]byte{0, 0, 1}, true}, + {[]byte{0, 0, 2}, true}, + {[]byte{0, 0, 3}, false}, + {[]byte{0, 0, 5}, false}, + {[]byte{0, 0, 8}, true}, + {[]byte{0, 0, 0x80}, true}, + {[]byte{0, 0, 0x81}, false}, + {[]byte{0, 0, 0x90}, false}, + {[]byte{3, 0, 0x80}, false}, + } + for _, testCase := range cases { + t.Run(fmt.Sprintf("is-zero checking %b (bin)", testCase.v), func(t *testing.T) { + if res := IsZeroBitlist(testCase.v); res != testCase.isZero { + t.Errorf("unexpected is-zero result for %b (bin). Expected: %v, got: %v", + testCase.v, testCase.isZero, res) + } + }) + } +} + +func TestCovers(t *testing.T) { + cases := []struct { + a []byte + b []byte + covers bool + err bool + }{ + {[]byte{0}, []byte{}, false, true}, + {[]byte{}, []byte{0}, false, true}, + {[]byte{0}, []byte{1}, false, false}, + {[]byte{1}, []byte{0}, true, false}, + {[]byte{0xff}, []byte{0}, true, false}, + {[]byte{2}, []byte{0xff}, false, false}, + {[]byte{0xf0}, []byte{0x10}, true, false}, + {[]byte{0xf0}, []byte{0x0f}, false, false}, + {[]byte{0xff, 0xff}, []byte{0, 0}, true, false}, + {[]byte{0xff, 0xf0}, []byte{0, 0}, true, false}, + {[]byte{0xff, 0xf0}, []byte{0, 3}, false, false}, + {[]byte{0xff, 0xf0}, []byte{1, 3}, false, false}, + {[]byte{0xff, 0xff, 0xff}, []byte{0xff, 0xff}, false, true}, + {[]byte{0xff, 0xff, 0xff}, []byte{0xff, 0x7f, 0xff}, true, false}, + } + for _, testCase := range cases { + t.Run(fmt.Sprintf("covers check %b <> %b (%v)", testCase.a, testCase.b, testCase.covers), func(t *testing.T) { + if res, err := Covers(testCase.a, testCase.b); (err != nil) != testCase.err { + if err == nil { + t.Errorf("expected covers error result between %b <> %b", testCase.a, testCase.b) + } else { + t.Errorf("expected covers error %q between %b <> %b", err.Error(), testCase.a, testCase.b) + } + } else if res != testCase.covers { + t.Errorf("unexpected covers result between %b <> %b. Expected: %v, got: %v", + testCase.a, testCase.b, testCase.covers, res) + } + }) + } +} diff --git a/bitfields/bitlist.go b/bitfields/bitlist.go new file mode 100644 index 0000000..5a6bc69 --- /dev/null +++ b/bitfields/bitlist.go @@ -0,0 +1,76 @@ +package bitfields + +import ( + "errors" + "fmt" + "math/bits" +) + +// Returns the length of the bitlist. +// And although strictly speaking invalid. a sane default is returned for: +// - an empty raw bitlist: a default 0 bitlist will be of length 0 too. +// - a bitlist with a leading 0 byte: return the bitlist raw bit length, +// excluding the last byte (As if it was full 0 padding). +func BitlistLen(b []byte) uint64 { + byteLen := uint64(len(b)) + if byteLen == 0 { + return 0 + } + last := b[byteLen-1] + return ((byteLen - 1) << 3) | BitIndex(last) +} + +// Helper function to implement Bitlist with. +// It checks if: +// 0. the raw bitlist is not empty, there must be a 1 bit to determine the length. +// 1. the bitlist has a leading 1 bit in the last byte to determine the length with. +// 2. if b has no more than given limit in bits. +func BitlistCheck(b []byte, limit uint64) error { + byteLen := uint64(len(b)) + if err := BitlistCheckByteLen(byteLen, limit); err != nil { + return err + } + last := b[byteLen-1] + return BitlistCheckLastByte(last, limit-((byteLen-1)<<3)) +} + +func BitlistCheckByteLen(byteLen uint64, bitLimit uint64) error { + if byteLen == 0 { + return errors.New("bitlist is missing length delimit bit") + } + // there may not be more bytes than necessary for the N bits, +1 for the delimiting bit. (also rounds up) + if byteLimitWithDelimiter := (bitLimit >> 3) + 1; byteLen > byteLimitWithDelimiter { + return fmt.Errorf("got %d bytes, expected no more than %d bytes for bitlist", + byteLen, byteLimitWithDelimiter) + } + return nil +} + +func BitlistCheckLastByte(last byte, limit uint64) error { + if last == 0 { + return errors.New("bitlist is invalid, trailing 0 byte") + } + if BitIndex(last) > limit { + return errors.New("bitlist is invalid, too many bits") + } + return nil +} + +// Counts the bits set to 1, excluding the delimiter bit. +func BitlistOnesCount(v []byte) uint64 { + if len(v) == 0 { + return 0 + } + count := uint64(0) + for i := 0; i < len(v)-1; i++ { + count += uint64(bits.OnesCount8(v[i])) + } + last := v[len(v)-1] + if last == 0 { // trailing zero byte is technically an invalid bitlist, but we can ignore it. + return count + } + // ignore the delimiter bit. + last ^= uint8(1) << BitIndex(last) + count += uint64(bits.OnesCount8(last)) + return count +} diff --git a/bitfields/bitlist_test.go b/bitfields/bitlist_test.go new file mode 100644 index 0000000..802ed92 --- /dev/null +++ b/bitfields/bitlist_test.go @@ -0,0 +1,55 @@ +package bitfields + +import ( + "fmt" + "testing" +) + +func TestBitlistLen(t *testing.T) { + cases := []struct { + v []byte + n uint64 + valid bool + ones uint64 + }{ + {[]byte{0}, 0, false, 0}, + {[]byte{0, 0}, 8, false, 0}, + {[]byte{0, 0, 0}, 16, false, 0}, + {[]byte{0xff, 0, 0, 0}, 24, false, 0}, + {[]byte{1, 2, 3, 0xff, 0}, 32, false, 0}, + {[]byte{1}, 0, true, 0}, + {[]byte{2}, 1, true, 0}, + {[]byte{3}, 1, true, 1}, + {[]byte{0x1a}, 4, true, 2}, + {[]byte{0x2b}, 5, true, 3}, + {[]byte{0xab}, 7, true, 4}, + {[]byte{0, 0x9b}, 8 + 7, true, 4}, + {[]byte{0, 0, 0x9b}, 8 + 8 + 7, true, 4}, + {[]byte{0xff, 0xff, 0x9b}, 8 + 8 + 7, true, 8 + 8 + 4}, + {[]byte{0xff, 0xff, 0x04}, 8 + 8 + 2, true, 8 + 8}, + {[]byte{0, 0, 0, 0, 0, 4}, 5*8 + 2, true, 0}, + } + for _, testCase := range cases { + t.Run(fmt.Sprintf("v %b (bin) len %d", testCase.v, testCase.n), func(t *testing.T) { + t.Run("get length", func(t *testing.T) { + if x := BitlistLen(testCase.v); x != testCase.n { + t.Errorf("expected bitlist to be of length: %d but got %d", testCase.n, x) + } + }) + t.Run(fmt.Sprintf("check valid %v", testCase.valid), func(t *testing.T) { + if err := BitlistCheck(testCase.v, testCase.n); err != nil && testCase.valid { + t.Errorf("expected bitlist to be valid but got error: %v", err) + } else if err == nil && !testCase.valid { + t.Error("expected bitlist to be invalid but got no error") + } + }) + if testCase.valid { + t.Run("check ones count", func(t *testing.T) { + if res := BitlistOnesCount(testCase.v); res != testCase.ones { + t.Errorf("expected %d one bits, but got %d", testCase.ones, res) + } + }) + } + }) + } +} diff --git a/bitfields/bitvector.go b/bitfields/bitvector.go new file mode 100644 index 0000000..4ccf269 --- /dev/null +++ b/bitfields/bitvector.go @@ -0,0 +1,54 @@ +package bitfields + +import ( + "fmt" + "math/bits" +) + +// Helper function to implement Bitvector with. +// It checks if: +// 1. b has the same amount of bytes as necessary for n bits. +// 2. unused bits in b are 0 +func BitvectorCheck(b []byte, n uint64) error { + byteLen := uint64(len(b)) + if err := BitvectorCheckByteLen(byteLen, n); err != nil { + return err + } + if byteLen == 0 { + // empty bitvector + return nil + } + last := b[byteLen-1] + return BitvectorCheckLastByte(last, n) +} + +func BitvectorCheckByteLen(byteLen uint64, bitLength uint64) error { + if expected := (bitLength + 7) >> 3; byteLen != expected { + return fmt.Errorf("bitvector of %d bytes has not expected length in bytes %d", byteLen, expected) + } + return nil +} + +func BitvectorCheckLastByte(last byte, n uint64) error { + if n == 0 { + return fmt.Errorf("empty bitvector can not have a last byte") + } + if n&7 == 0 { + // n is a multiple of 8, so last byte fits + return nil + } + // check if it is big enough to hold any non-zero contents in the last byte. + if expectedBitsLen := n & 7; (last >> expectedBitsLen) != 0 { + return fmt.Errorf("bitvector last byte 0b%b has not expected %d bits in last byte", last, expectedBitsLen) + } + return nil +} + +// Counts the bits set to 1. Assumes the bitvector with length not a multiple of 8 has trailing zero bits. +func BitvectorOnesCount(v []byte) uint64 { + count := uint64(0) + for i := 0; i < len(v); i++ { + count += uint64(bits.OnesCount8(v[i])) + } + return count +} diff --git a/bitfields/bitvector_test.go b/bitfields/bitvector_test.go new file mode 100644 index 0000000..3e79606 --- /dev/null +++ b/bitfields/bitvector_test.go @@ -0,0 +1,53 @@ +package bitfields + +import ( + "fmt" + "testing" +) + +func TestBitvectorCheck(t *testing.T) { + cases := []struct { + v []byte + n uint64 + valid bool + ones uint64 + }{ + {[]byte{}, 0, true, 0}, + {[]byte{0}, 0, false, 0}, + {[]byte{0}, 1, true, 0}, + {[]byte{0}, 8, true, 0}, + {[]byte{1}, 1, true, 1}, + {[]byte{0x31}, 8, true, 3}, + {[]byte{0x14}, 5, true, 2}, + {[]byte{0}, 9, false, 0}, + {[]byte{0, 0}, 9, true, 0}, + {[]byte{0, 0, 0}, 16, false, 0}, + {[]byte{0, 0, 0}, 17, true, 0}, + {[]byte{0, 0, 0}, 24, true, 0}, + {[]byte{0, 0, 0, 0, 0, 0}, 48, true, 0}, + {[]byte{0xff}, 8, true, 8}, + {[]byte{0xff}, 7, false, 0}, + {[]byte{0x7f}, 7, true, 7}, + {[]byte{0xff, 0x80}, 16, true, 8 + 1}, + {[]byte{0xff, 0x80}, 15, false, 0}, + {[]byte{0xff, 0x40}, 15, true, 8 + 1}, + } + for _, testCase := range cases { + t.Run(fmt.Sprintf("v %b (bin) len %d", testCase.v, testCase.n), func(t *testing.T) { + t.Run(fmt.Sprintf("check valid %v", testCase.valid), func(t *testing.T) { + if err := BitvectorCheck(testCase.v, testCase.n); err != nil && testCase.valid { + t.Errorf("expected bitvector to be valid but got error: %v", err) + } else if err == nil && !testCase.valid { + t.Error("expected bitvector to be invalid but got no error") + } + }) + if testCase.valid { + t.Run(fmt.Sprintf("check ones count %v", testCase.valid), func(t *testing.T) { + if res := BitvectorOnesCount(testCase.v); res != testCase.ones { + t.Errorf("expected %d ones, but got %d", testCase.ones, res) + } + }) + } + }) + } +} diff --git a/codec/decoder.go b/codec/decoder.go new file mode 100644 index 0000000..b4ca96b --- /dev/null +++ b/codec/decoder.go @@ -0,0 +1,411 @@ +package codec + +import ( + "encoding/binary" + "fmt" + "github.com/protolambda/ztyp/bitfields" + "io" + "io/ioutil" +) + +type Deserializable interface { + Deserialize(dr *DecodingReader) error + FixedLength +} + +type FixedLength interface { + FixedLength() uint64 +} + +type Decodable interface { + Decode(x []byte) error +} + +type DecodingReader struct { + input io.Reader + i uint64 + max uint64 + scratch [32]byte +} + +func NewDecodingReader(input io.Reader, scope uint64) *DecodingReader { + return &DecodingReader{input: io.LimitReader(input, int64(scope)), i: 0, max: scope} +} + +// SubScope returns a scope of the SSZ reader. Re-uses same scratchpad. +func (dr *DecodingReader) SubScope(count uint64) (*DecodingReader, error) { + // TODO: based on scope, read a buffer ahead of time. + if span := dr.Scope(); span < count { + return nil, fmt.Errorf("cannot create scoped decoding reader, scope of %d bytes is bigger than parent scope has available space %d", count, span) + } + // TODO: don't nest readers, instead just limit input reads ourselves + return &DecodingReader{input: io.LimitReader(dr.input, int64(count)), i: 0, max: count}, nil +} + +func (dr *DecodingReader) UpdateIndexFromScoped(other *DecodingReader) { + dr.i += other.i +} + +// how far we have read so far (scoped per container) +func (dr *DecodingReader) Index() uint64 { + return dr.i +} + +// How far we can read (max - i = remaining bytes to read without error). +// Note: when a child element is not fixed length, +// the parent should set the scope, so that the child can infer its size from it. +func (dr *DecodingReader) Max() uint64 { + return dr.max +} + +func (dr *DecodingReader) hasScope(x uint64) error { + if ^uint64(0)-dr.i < x { + return fmt.Errorf("overflow: x: %d, i: %d, max: %d", x, dr.i, dr.max) + } + v := dr.i + x + if v > dr.max { + return fmt.Errorf("cannot read %d bytes, %d beyond scope", x, v-dr.max) + } + return nil +} + +func (dr *DecodingReader) checkedIndexUpdate(x uint64) (n int, err error) { + if ^uint64(0)-dr.i < x { + return 0, fmt.Errorf("overflow: x: %d, i: %d, max: %d", x, dr.i, dr.max) + } + v := dr.i + x + if v > dr.max { + return int(dr.i), fmt.Errorf("cannot read %d bytes, %d beyond scope", x, v-dr.max) + } + dr.i = v + return int(x), nil +} + +func (dr *DecodingReader) Skip(count uint64) (int, error) { + if n, err := dr.checkedIndexUpdate(count); err != nil { + return n, err + } + switch r := dr.input.(type) { + case io.Seeker: + n, err := r.Seek(int64(count), io.SeekCurrent) + return int(n), err + default: + n, err := io.CopyN(ioutil.Discard, dr.input, int64(count)) + return int(n), err + } +} + +// Read p fully, returns n read bytes. len(p) == n always if err == nil +func (dr *DecodingReader) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if n, err := dr.checkedIndexUpdate(uint64(len(p))); err != nil { + return n, err + } + n := 0 + for n < len(p) { + v, err := dr.input.Read(p[n:]) + n += v + if err != nil { + return n, err + } + } + return n, nil +} + +func (dr *DecodingReader) ReadByte() (byte, error) { + _, err := dr.Read(dr.scratch[0:1]) + return dr.scratch[0], err +} + +func (dr *DecodingReader) ReadUint16() (uint16, error) { + _, err := dr.Read(dr.scratch[0:2]) + return binary.LittleEndian.Uint16(dr.scratch[0:2]), err +} + +func (dr *DecodingReader) ReadUint32() (uint32, error) { + _, err := dr.Read(dr.scratch[0:4]) + return binary.LittleEndian.Uint32(dr.scratch[0:4]), err +} + +func (dr *DecodingReader) ReadUint64() (uint64, error) { + _, err := dr.Read(dr.scratch[0:8]) + return binary.LittleEndian.Uint64(dr.scratch[0:8]), err +} + +// returns the remaining scope (amount of bytes that can be read) +func (dr *DecodingReader) Scope() uint64 { + return dr.Max() - dr.Index() +} + +func (dr *DecodingReader) ReadOffset() (uint32, error) { + return dr.ReadUint32() +} + +// Deserialize vector. If fixedElemSize == 0, the item is regarded as dynamic length +func (dr *DecodingReader) Vector(item func(i uint64) Deserializable, fixedElemSize uint64, length uint64) error { + if fixedElemSize != 0 { + for i := uint64(0); i < length; i++ { + sub, err := dr.SubScope(fixedElemSize) + if err != nil { + return err + } + if err := item(i).Deserialize(sub); err != nil { + return fmt.Errorf("failed to deserialize item %d: %v", i, err) + } + } + return nil + } else { + scope := dr.Scope() + // TODO could optimize this + offsets := make([]uint64, length, length) + for i := uint64(0); i < length; i++ { + off, err := dr.ReadOffset() + if err != nil { + return err + } + offsets[i] = uint64(off) + } + var prev uint64 + for i, off := range offsets { + if prev > off { + return fmt.Errorf("offset %d is too low, previous was %d", off, prev) + } + item := item(uint64(0)) + next := scope + if len(offsets) > i+1 { + next = offsets[i+1] + } + sub, err := dr.SubScope(next - off) + if err != nil { + return err + } + if err := item.Deserialize(sub); err != nil { + return fmt.Errorf("failed to serialize item %d: %v", i, err) + } + prev = next + } + return nil + } +} + +func (dr *DecodingReader) List(add func() Deserializable, fixedElemSize uint64, limit uint64) error { + scope := dr.Scope() + // no items to decode + if scope == 0 { + return nil + } + if fixedElemSize != 0 { + if scope%fixedElemSize != 0 { + return fmt.Errorf("scope %d is not a multiple of expected element size: %d", scope, fixedElemSize) + } + length := scope / fixedElemSize + if length > limit { + return fmt.Errorf("too many items in list: %d > %d", length, limit) + } + for i := uint64(0); i < length; i++ { + item := add() + sub, err := dr.SubScope(fixedElemSize) + if err != nil { + return err + } + if err := item.Deserialize(sub); err != nil { + return err + } + } + return nil + } else { + firstOffset, err := dr.ReadOffset() + if err != nil { + return err + } + if firstOffset%4 != 0 { + return fmt.Errorf("first offset of list is invalid, not a multiple of 4: %d", firstOffset) + } + length := uint64(firstOffset / 4) + if length > limit { + return fmt.Errorf("too many items in list: %d > %d", length, limit) + } + // TODO could optimize this + offsets := make([]uint64, 0, length) + offsets = append(offsets, uint64(firstOffset)) + for i := uint64(1); i < length; i++ { + off, err := dr.ReadOffset() + if err != nil { + return err + } + offsets = append(offsets, uint64(off)) + } + var prev uint64 + for i, off := range offsets { + if prev > off { + return fmt.Errorf("offset %d is too low, previous was %d", off, prev) + } + item := add() + next := scope + if len(offsets) > i+1 { + next = offsets[i+1] + } + if next == off { + prev = off + continue + } + sub, err := dr.SubScope(next - off) + if err != nil { + return err + } + if err := item.Deserialize(sub); err != nil { + return fmt.Errorf("failed to deserialize item %d: %v", i, err) + } + prev = off + } + return nil + } +} + +func (dr *DecodingReader) BitVector(dst *[]byte, bitLength uint64) error { + if dst == nil { + return fmt.Errorf("bitvector destination is nil") + } + // grow the destination if necessary + byteLen := (bitLength + 7) >> 3 + if uint64(cap(*dst)) < byteLen { + *dst = make([]byte, byteLen, byteLen) + } else { + *dst = (*dst)[:byteLen] + } + if _, err := dr.Read(*dst); err != nil { + return err + } + return bitfields.BitvectorCheck(*dst, bitLength) +} + +func (dr *DecodingReader) BitList(dst *[]byte, bitLimit uint64) error { + byteLen := dr.Scope() + if byteLimit := (bitLimit + 7) >> 3; byteLen > byteLimit { + return fmt.Errorf("bitlist is too big: %d bytes, limit is %d (bitlimit %d)", byteLen, byteLimit, bitLimit) + } + // grow the destination if necessary + if uint64(cap(*dst)) < byteLen { + *dst = make([]byte, byteLen, byteLen) + } else { + *dst = (*dst)[:byteLen] + } + if _, err := dr.Read(*dst); err != nil { + return err + } + return bitfields.BitlistCheck(*dst, bitLimit) +} + +func (dr *DecodingReader) ByteVector(dst *[]byte, byteLength uint64) error { + if dst == nil { + return fmt.Errorf("byte vector destination is nil") + } + // grow the destination if necessary + if uint64(cap(*dst)) < byteLength { + *dst = make([]byte, byteLength, byteLength) + } else { + *dst = (*dst)[:byteLength] + } + _, err := dr.Read(*dst) + return err +} + +func (dr *DecodingReader) ByteList(dst *[]byte, byteLimit uint64) error { + byteLen := dr.Scope() + if byteLen > byteLimit { + return fmt.Errorf("byte list is too big: %d bytes, limit is %d", byteLen, byteLimit) + } + // grow the destination if necessary + if uint64(cap(*dst)) < byteLen { + *dst = make([]byte, byteLen, byteLen) + } else if uint64(len(*dst)) < byteLen { + *dst = (*dst)[:byteLen] + } + _, err := dr.Read(*dst) + return err +} + +// FixedLenContainer just reads the concatednated fields, without opening smaller scopes. +// Much faster for simple structs, but use with caution. +func (dr *DecodingReader) FixedLenContainer(fields ...Deserializable) error { + for i, f := range fields { + if err := f.Deserialize(dr); err != nil { + return fmt.Errorf("failed to deserialize fixed-length field %d: %v", i, err) + } + } + return nil +} + +func (dr *DecodingReader) Container(fields ...Deserializable) error { + scope := dr.Scope() + var offsets []uint64 + var dynFields []Deserializable + var prev uint64 + for i, f := range fields { + if fix := f.FixedLength(); fix != 0 { + sub, err := dr.SubScope(fix) + if err != nil { + return err + } + if err := f.Deserialize(sub); err != nil { + return fmt.Errorf("failed to deserialize fixed-length field %d: %v", i, err) + } + prev += fix + } else { + off, err := dr.ReadOffset() + if err != nil { + return fmt.Errorf("failed to read offset for field %d: %v", i, err) + } + offsets = append(offsets, uint64(off)) + dynFields = append(dynFields, f) + prev += 4 + } + } + if len(dynFields) == 0 || len(offsets) == 0 { + return nil + } + if prev != offsets[0] { + return fmt.Errorf("offset 0 is incorrect, expected %d, got %d", prev, offsets[0]) + } + for i, off := range offsets { + f := dynFields[i] + next := scope + if len(offsets) > i+1 { + next = offsets[i+1] + } + if next < off { + return fmt.Errorf("scope cannot be negative, got offset %d after %d, at index %d", next, off, i) + } + sub, err := dr.SubScope(next - off) + if err != nil { + return err + } + if err := f.Deserialize(sub); err != nil { + return fmt.Errorf("failed to deserialize dynamic-length field %d: %v", i, err) + } + prev = next + } + return nil +} + +// Deserialize a Union, the selectFn is called to retrieve a destination to deserialize into, and remember the selector. +// The selectFn can return a nil when there is nothing to decode (when Union is used as an Optional) +func (dr *DecodingReader) Union(selectFn func(selector uint8) (Deserializable, error)) error { + selector, err := dr.ReadByte() + if err != nil { + return fmt.Errorf("fialed to read union selector: %v", err) + } + dest, err := selectFn(selector) + if err != nil { + return fmt.Errorf("failed to select union option, with selector %d: %v", selector, err) + } + if dest == nil { + if selector != 0 { + return fmt.Errorf("only 0 the selector can indicate a None value") + } + return nil + } + return dest.Deserialize(dr) +} diff --git a/codec/encoder.go b/codec/encoder.go new file mode 100644 index 0000000..1ccfd72 --- /dev/null +++ b/codec/encoder.go @@ -0,0 +1,223 @@ +package codec + +import ( + "encoding/binary" + "fmt" + "io" +) + +type Serializable interface { + Serialize(w *EncodingWriter) error + ByteLength + FixedLength +} + +type ByteLength interface { + ByteLength() uint64 +} + +const OFFSET_SIZE = 4 + +func Sum(values ...ByteLength) (out uint64) { + for _, v := range values { + out += v.ByteLength() + } + return +} + +func ContainerLength(values ...Serializable) (out uint64) { + for _, v := range values { + if size := v.FixedLength(); size == 0 { + out += v.ByteLength() + OFFSET_SIZE + } else { + out += size + } + } + return +} + +type Encodable interface { + Encode() ([]byte, error) +} + +type EncodingWriter struct { + w io.Writer + n int + Scratch [32]byte +} + +func NewEncodingWriter(w io.Writer) *EncodingWriter { + return &EncodingWriter{w: w, n: 0} +} + +// How many bytes were written to the underlying io.Writer before ending encoding (for handling errors) +func (ew *EncodingWriter) Written() int { + return ew.n +} + +// Write writes len(p) bytes from p fully to the underlying accumulated buffer. +func (ew *EncodingWriter) Write(p []byte) error { + n := 0 + for n < len(p) { + d, err := ew.w.Write(p[n:]) + ew.n += d + if err != nil { + return err + } + n += d + } + return nil +} + +// Write a single byte to the buffer. +func (ew *EncodingWriter) WriteByte(v byte) error { + ew.Scratch[0] = v + return ew.Write(ew.Scratch[0:1]) +} + +// Writes an offset for an element +func (ew *EncodingWriter) WriteOffset(prevOffset uint64, elemLen uint64) (offset uint64, err error) { + if prevOffset >= (uint64(1) << 32) { + panic("cannot write offset with invalid previous offset") + } + if elemLen >= (uint64(1) << 32) { + panic("cannot write offset with invalid element size") + } + offset = prevOffset + elemLen + if offset >= (uint64(1) << 32) { + panic("offset too large, not uint32") + } + binary.LittleEndian.PutUint32(ew.Scratch[0:4], uint32(offset)) + err = ew.Write(ew.Scratch[0:4]) + return +} + +func (ew *EncodingWriter) WriteUint16(v uint16) error { + binary.LittleEndian.PutUint16(ew.Scratch[0:2], v) + return ew.Write(ew.Scratch[0:2]) +} + +func (ew *EncodingWriter) WriteUint32(v uint32) error { + binary.LittleEndian.PutUint32(ew.Scratch[0:4], v) + return ew.Write(ew.Scratch[0:4]) +} + +func (ew *EncodingWriter) WriteUint64(v uint64) error { + binary.LittleEndian.PutUint64(ew.Scratch[0:8], v) + return ew.Write(ew.Scratch[0:8]) +} + +// List serialization. +// If fixedElemSize == 0, then items are considered dynamic length, and will be encoded with offsets. +func (ew *EncodingWriter) List(item func(i uint64) Serializable, fixedElemSize uint64, length uint64) error { + if fixedElemSize == 0 { + prevOffset := 4 * length + prevSize := uint64(0) + // write offsets + for i := uint64(0); i < length; i++ { + size := item(i).ByteLength() + offset, err := ew.WriteOffset(prevOffset, prevSize) + if err != nil { + return fmt.Errorf("failed to serialize list item %d: %v", i, err) + } + prevOffset = offset + prevSize = size + } + } + // write elements + for i := uint64(0); i < length; i++ { + if err := item(i).Serialize(ew); err != nil { + return fmt.Errorf("failed to serialize list item %d: %v", i, err) + } + } + return nil +} + +// Vector serialization, works the same as encoding a List. +func (ew *EncodingWriter) Vector(item func(i uint64) Serializable, fixedElemSize uint64, length uint64) error { + return ew.List(item, fixedElemSize, length) +} + +func (ew *EncodingWriter) BitList(bits []byte) error { + if len(bits) == 0 || bits[len(bits)-1] == 0 { + return fmt.Errorf("missing delimiter bit, invalid bitlist") + } + return ew.Write(bits) +} + +func (ew *EncodingWriter) BitVector(bits []byte) error { + if len(bits) == 0 { + return fmt.Errorf("bitvector must not be empty") + } + return ew.Write(bits) +} + +func (ew *EncodingWriter) FixedLenContainer(fields ...Serializable) error { + for i, f := range fields { + if err := f.Serialize(ew); err != nil { + return fmt.Errorf("failed to serialize fixed-length field %d: %v", i, err) + } + } + return nil +} + +// Container serialization. Fields with a non-zero .FixedLength() are considered fixed-length. +func (ew *EncodingWriter) Container(fields ...Serializable) error { + fixedLen := uint64(0) + isFixedLen := true + for _, f := range fields { + if fix := f.FixedLength(); fix != 0 { + fixedLen += fix + } else { + // length of an offset + fixedLen += 4 + isFixedLen = false + } + } + // the previous offset, to calculate a new offset from, starting after the fixed data. + prevOffset := fixedLen + // span of the previous var-size element + prevSize := uint64(0) + for i, f := range fields { + if f.FixedLength() != 0 { + if err := f.Serialize(ew); err != nil { + return fmt.Errorf("failed to serialize fixed-length field %d: %v", i, err) + } + } else { + if offset, err := ew.WriteOffset(prevOffset, prevSize); err != nil { + return err + } else { + prevOffset = offset + } + prevSize = f.ByteLength() + } + } + // Only iterate over and write dynamic parts if we need to. + if !isFixedLen { + for i, f := range fields { + if f.FixedLength() == 0 { + if err := f.Serialize(ew); err != nil { + return fmt.Errorf("failed to serialize dynamic-length field %d: %v", i, err) + } + } + } + } + return nil +} + +// Serialize a Union. The value +func (ew *EncodingWriter) Union(selector uint8, value Serializable) error { + if err := ew.WriteByte(selector); err != nil { + return fmt.Errorf("failed to write union selector (%d): %v", selector, err) + } + if value == nil { + if selector != 0 { + return fmt.Errorf("cannot serialize nil value, only 0 selector can be None, got %d", selector) + } + return nil + } + if err := value.Serialize(ew); err != nil { + return fmt.Errorf("failed to write union value (selcector %v, value type %T): ", selector, value) + } + return nil +} diff --git a/conv/bytes.go b/conv/bytes.go new file mode 100644 index 0000000..abae8ca --- /dev/null +++ b/conv/bytes.go @@ -0,0 +1,60 @@ +package conv + +import ( + "encoding/hex" + "errors" + "fmt" +) + +func BytesString(b []byte) string { + l := 2 + (len(b) * 2) + res := make([]byte, l, l) + res[0] = '0' + res[1] = 'x' + hex.Encode(res[2:], b) + return string(res) +} + +func BytesMarshalText(b []byte) ([]byte, error) { + l := 2 + (len(b) * 2) + res := make([]byte, l, l) + res[0] = '0' + res[1] = 'x' + hex.Encode(res[2:], b) + return res, nil +} + +// Decodes hex data, with optional 0x prefix, and lower/upper/mixed case. +// The decoded byte length must match the dst length. +func FixedBytesUnmarshalText(dst []byte, text []byte) error { + if dst == nil { + return DestNilErr + } + if len(text) >= 2 && text[0] == '0' && (text[1] == 'x' || text[1] == 'X') { + text = text[2:] + } + if len(text) != 2*len(dst) { + return fmt.Errorf("unexpected length %d, expected %d, got string '%s'", len(text), len(dst), string(text)) + } + _, err := hex.Decode(dst, text) + return err +} + +// Decodes hex data, with optional 0x prefix, and lower/upper/mixed case. +// Reuses the dst space if possible. +// If dst is too small, a new dst is allocated and the old header is overwritten +func DynamicBytesUnmarshalText(dst *[]byte, text []byte) error { + if dst == nil { + return errors.New("nil dst, cannot decode") + } + if len(text) >= 2 && text[0] == '0' && (text[1] == 'x' || text[1] == 'X') { + text = text[2:] + } + size := len(text) / 2 + if cap(*dst) < size { + *dst = make([]byte, size, size) + } + *dst = (*dst)[:size] + _, err := hex.Decode(*dst, text) + return err +} diff --git a/conv/numbers.go b/conv/numbers.go new file mode 100644 index 0000000..4ed7eb4 --- /dev/null +++ b/conv/numbers.go @@ -0,0 +1,123 @@ +package conv + +import ( + "errors" + "fmt" + "github.com/holiman/uint256" + "math/big" + "strconv" +) + +var DestNilErr = errors.New("destination is nil") +var EmptyInputErr = errors.New("input is empty") +var MissingQuoteErr = errors.New("input has quote open without close") + +func Uint256Unmarshal(v *uint256.Int, b []byte) error { + if v == nil { + return DestNilErr + } + if len(b) == 0 { + return EmptyInputErr + } + if b[0] == '"' { + if len(b) == 1 || b[len(b)-1] != b[0] { + return MissingQuoteErr + } + b = b[1 : len(b)-1] + } + x := new(big.Int) + err := x.UnmarshalText(b) + if err != nil { + return fmt.Errorf("failed to unmarshal Uint256View: %w", err) + } + if x.Sign() < 0 { + return strconv.ErrRange + } + if v.SetFromBig(x) { + return strconv.ErrRange + } + return nil +} + +// Parse a uint of bitSize bits into a uint64, with or without quotes, in any base, +// with common prefixes accepted to change base. +func uintUnmarshal(v *uint64, b []byte, bitSize int) error { + if v == nil { + return DestNilErr + } + if len(b) == 0 { + return EmptyInputErr + } + if b[0] == '"' { + if len(b) == 1 || b[len(b)-1] != b[0] { + return MissingQuoteErr + } + b = b[1 : len(b)-1] + } + n, err := strconv.ParseUint(string(b), 0, bitSize) + if err != nil { + return err + } + *v = n + return nil +} + +func Uint64Unmarshal(v *uint64, b []byte) error { + return uintUnmarshal(v, b, 64) +} + +func Uint32Unmarshal(v *uint32, b []byte) error { + var x uint64 + if err := uintUnmarshal(&x, b, 32); err != nil { + return err + } + *v = uint32(x) + return nil +} + +func Uint16Unmarshal(v *uint16, b []byte) error { + var x uint64 + if err := uintUnmarshal(&x, b, 16); err != nil { + return err + } + *v = uint16(x) + return nil +} + +func Uint8Unmarshal(v *uint8, b []byte) error { + var x uint64 + if err := uintUnmarshal(&x, b, 8); err != nil { + return err + } + *v = uint8(x) + return nil +} + +// Uint256Marshal to decimal number, with quotes +func Uint256Marshal(v *uint256.Int) ([]byte, error) { + return []byte(fmt.Sprintf("\"%d\"", v)), nil +} + +// Uint64Marshal to decimal number, with quotes +func Uint64Marshal(v uint64) ([]byte, error) { + var dest [22]byte // ceil(log10(2**64)) + 2 = 22 + dest[0] = '"' + res := strconv.AppendUint(dest[0:1], v, 10) + res = append(res, '"') + return res, nil +} + +// Uint32Marshal to decimal number, with quotes +func Uint32Marshal(v uint32) ([]byte, error) { + return Uint64Marshal(uint64(v)) +} + +// Uint16Marshal to decimal number, with quotes +func Uint16Marshal(v uint16) ([]byte, error) { + return Uint64Marshal(uint64(v)) +} + +// Uint8Marshal to decimal number, with quotes +func Uint8Marshal(v uint8) ([]byte, error) { + return Uint64Marshal(uint64(v)) +} diff --git a/conv/numbers_test.go b/conv/numbers_test.go new file mode 100644 index 0000000..017f5cf --- /dev/null +++ b/conv/numbers_test.go @@ -0,0 +1,135 @@ +package conv + +import ( + "errors" + "fmt" + "math/big" + "strconv" + "testing" +) + +type testCase struct { + str string + num uint64 + bitSize int + err error +} + +func (tc *testCase) testUnmarshal(t *testing.T) { + var err error + var out uint64 + switch tc.bitSize { + case 8: + var y uint8 + err = Uint8Unmarshal(&y, []byte(tc.str)) + out = uint64(y) + case 16: + var y uint16 + err = Uint16Unmarshal(&y, []byte(tc.str)) + out = uint64(y) + case 32: + var y uint32 + err = Uint32Unmarshal(&y, []byte(tc.str)) + out = uint64(y) + case 64: + var y uint64 + err = Uint64Unmarshal(&y, []byte(tc.str)) + out = y + default: + panic("bad bit size") + } + if tc.err != nil { + if !errors.Is(err, tc.err) { + t.Errorf("expected err %v, but got %v", tc.err, err) + } + } else { + if err != nil { + t.Errorf("Unexpected error: %v", err) + } else if out != tc.num { + t.Errorf("Unexpected unmarshal result %d, expected %d", out, tc.num) + } + } +} + +func (tc *testCase) testMarshal(t *testing.T) { + var out []byte + var err error + switch tc.bitSize { + case 8: + out, err = Uint8Marshal(uint8(tc.num)) + case 16: + out, err = Uint16Marshal(uint16(tc.num)) + case 32: + out, err = Uint32Marshal(uint32(tc.num)) + case 64: + out, err = Uint64Marshal(tc.num) + default: + panic("bad bit size") + } + + if tc.err != nil { + if !errors.Is(err, tc.err) { + t.Errorf("expected err %v, but got %v", tc.err, err) + } + } else { + if err != nil { + t.Errorf("Unexpected error: %v", err) + } else if string(out) != tc.str { + t.Errorf("Unexpected marshal result %s, expected %s", string(out), tc.str) + } + } +} + +func TestUintUnmarshal(t *testing.T) { + var testCases []testCase + for bitSize := 8; bitSize <= 64; bitSize *= 2 { + max := (uint64(1) << bitSize) - 1 + outOfRange := new(big.Int).Add(new(big.Int).SetUint64(max), big.NewInt(1)) + + for _, form := range []string{"%d", "%#b", "%#o", "%#O", "%#x", "%#X"} { + testCases = append(testCases, + testCase{fmt.Sprintf(form, 0), 0, bitSize, nil}, + testCase{fmt.Sprintf(form, 1), 1, bitSize, nil}, + testCase{fmt.Sprintf(form, max/3), max / 3, bitSize, nil}, + testCase{fmt.Sprintf(form, max), max, bitSize, nil}, + testCase{fmt.Sprintf(form, outOfRange), 0, bitSize, strconv.ErrRange}) + } + } + for _, tc := range testCases { + testCases = append(testCases, testCase{"-" + tc.str, 0, tc.bitSize, strconv.ErrSyntax}) + } + for _, tc := range testCases { + testCases = append(testCases, + testCase{"\"" + tc.str + "\"", tc.num, tc.bitSize, tc.err}, + testCase{"\"" + tc.str, 0, tc.bitSize, MissingQuoteErr}, + testCase{"'" + tc.str + "'", tc.num, tc.bitSize, strconv.ErrSyntax}) + if tc.err == nil { + testCases = append(testCases, testCase{tc.str + "\"", 0, tc.bitSize, strconv.ErrSyntax}) + } + } + for bitSize := 8; bitSize <= 64; bitSize *= 2 { + testCases = append(testCases, testCase{"", 0, bitSize, EmptyInputErr}) + } + for _, tc := range testCases { + t.Run(fmt.Sprintf("unmarshal_uint%d_%s", tc.bitSize, tc.str), func(t *testing.T) { + tc.testUnmarshal(t) + }) + } +} + +func TestUintMarshal(t *testing.T) { + var testCases []testCase + for bitSize := 8; bitSize <= 64; bitSize *= 2 { + max := (uint64(1) << bitSize) - 1 + testCases = append(testCases, + testCase{fmt.Sprintf("\"%d\"", 0), 0, bitSize, nil}, + testCase{fmt.Sprintf("\"%d\"", 1), 1, bitSize, nil}, + testCase{fmt.Sprintf("\"%d\"", max/3), max / 3, bitSize, nil}, + testCase{fmt.Sprintf("\"%d\"", max), max, bitSize, nil}) + } + for _, tc := range testCases { + t.Run(fmt.Sprintf("marshal_uint%d_%d", tc.bitSize, tc.num), func(t *testing.T) { + tc.testMarshal(t) + }) + } +} diff --git a/experiment_test.go b/experiment_test.go index d7b8e4b..5bbf61b 100644 --- a/experiment_test.go +++ b/experiment_test.go @@ -6,7 +6,7 @@ import ( "testing" ) -var BLSPubkeyType = BasicVectorType("BLSPubkey", ByteType, 48) +var BLSPubkeyType = BasicVectorType(ByteType, 48) var ValidatorType = ContainerType("Validator", []FieldDef{ {"pubkey", BLSPubkeyType}, @@ -24,42 +24,53 @@ const VALIDATOR_REGISTRY_LIMIT uint64 = 1 << 40 var RegistryBalancesType = BasicListType(Uint64Type, VALIDATOR_REGISTRY_LIMIT) -var RegistryValidatorsType = ListType(ValidatorType, VALIDATOR_REGISTRY_LIMIT) +var RegistryValidatorsType = ComplexListType(ValidatorType, VALIDATOR_REGISTRY_LIMIT) func BenchmarkRegInitHash(t *testing.B) { startCount := 100000 regView := RegistryValidatorsType.New() for i := 0; i < startCount; i++ { if err := regView.Append(ValidatorType.New()); err != nil { - t.Error(err) + t.Fatal(err) } } + // after initial cost, hashing is cached, and it's essentially free. + hFn := tree.GetHashFn() + res := byte(0) + t.ResetTimer() for i := 0; i < t.N; i++ { - r := regView.ViewRoot(tree.Hash) - t.Logf("x; %x", r) - ll, err := regView.Length() - t.Logf("length: %d %v", ll, err) + r := regView.HashTreeRoot(hFn) + res ^= r[0] // do something with the output, don't ignore it. + //t.Logf("x; %x", r) } + t.Logf("res %d", res) + ll, err := regView.Length() + t.Logf("length: %d %v, N: %d", ll, err, t.N) } func BenchmarkRegHash(t *testing.B) { startCount := 100000 regView := RegistryValidatorsType.New() - r := regView.ViewRoot(tree.Hash) + hFn := tree.GetHashFn() + r := regView.HashTreeRoot(hFn) t.Logf("r; %x", r) for i := 0; i < startCount; i++ { if err := regView.Append(ValidatorType.New()); err != nil { - t.Error(err) + t.Fatal(err) } } ll, err := regView.Length() t.Logf("length: %d %v", ll, err) + t.Logf("N: %d", t.N) t.ResetTimer() + res := byte(0) for i := 0; i < t.N; i++ { if err := regView.Append(ValidatorType.New()); err != nil { t.Error(err) } - r := regView.ViewRoot(tree.Hash) - t.Logf("x; %x", r) + r := regView.HashTreeRoot(hFn) + res ^= r[0] // do something with the output, don't ignore it. + //t.Logf("x; %x", r) } + t.Logf("res %d", res) } diff --git a/go.mod b/go.mod index f476b57..fce0223 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/protolambda/ztyp -go 1.13 +go 1.16 + +require github.com/holiman/uint256 v1.2.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..6252fa8 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= +github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= diff --git a/props/basic.go b/props/basic.go deleted file mode 100644 index 642d5a2..0000000 --- a/props/basic.go +++ /dev/null @@ -1,67 +0,0 @@ -package props - -import ( - "fmt" - . "github.com/protolambda/ztyp/tree" - . "github.com/protolambda/ztyp/view" -) - -type RootReadProp ReadPropFn - -func (p RootReadProp) Root() (Root, error) { - v, err := p() - if err != nil { - return Root{}, err - } - r, ok := v.(*Root) - if ok { - return Root{}, fmt.Errorf("not a root view: %v", v) - } - return *r, nil -} - -type RootWriteProp WritePropFn - -func (p RootWriteProp) SetRoot(v Root) error { - return p(&v) -} - -type Uint64ReadProp ReadPropFn - -func (p Uint64ReadProp) Uint64() (uint64, error) { - v, err := p() - if err != nil { - return 0, err - } - n, ok := v.(Uint64View) - if ok { - return 0, fmt.Errorf("not a uint64 view: %v", v) - } - return uint64(n), nil -} - -type Uint64WriteProp WritePropFn - -func (p Uint64WriteProp) SetUint64(v uint64) error { - return p(Uint64View(v)) -} - -type BoolReadProp ReadPropFn - -func (p BoolReadProp) Bool() (bool, error) { - v, err := p() - if err != nil { - return false, err - } - b, ok := v.(BoolView) - if ok { - return false, fmt.Errorf("not a bool view: %v", v) - } - return bool(b), nil -} - -type BoolWriteProp WritePropFn - -func (p BoolWriteProp) SetBool(v BoolView) error { - return p(v) -} diff --git a/props/complex.go b/props/complex.go deleted file mode 100644 index e0dd4e1..0000000 --- a/props/complex.go +++ /dev/null @@ -1,92 +0,0 @@ -package props - -import ( - "fmt" - . "github.com/protolambda/ztyp/view" -) - -// This file is why we need generics in Go. Could have been ~10 lines. - -type ContainerReadProp ReadPropFn - -func (p ContainerReadProp) Container() (*ContainerView, error) { - v, err := p() - if err != nil { - return nil, err - } - c, ok := v.(*ContainerView) - if ok { - return nil, fmt.Errorf("view is not a container: %v", v) - } - return c, nil -} - -type VectorReadProp ReadPropFn - -func (p VectorReadProp) Vector() (*ComplexVectorView, error) { - v, err := p() - if err != nil { - return nil, err - } - c, ok := v.(*ComplexVectorView) - if ok { - return nil, fmt.Errorf("view is not a vector: %v", v) - } - return c, nil -} - -type BasicVectorReadProp ReadPropFn - -func (p BasicVectorReadProp) BasicVector() (*PackedVectorView, error) { - v, err := p() - if err != nil { - return nil, err - } - bv, ok := v.(*PackedVectorView) - if ok { - return nil, fmt.Errorf("view is not a basic vector: %v", v) - } - return bv, nil -} - -type BitVectorReadProp ReadPropFn - -func (p BitVectorReadProp) BitVector() (*BitVectorView, error) { - v, err := p() - if err != nil { - return nil, err - } - bv, ok := v.(*BitVectorView) - if ok { - return nil, fmt.Errorf("view is not a bitvector: %v", v) - } - return bv, nil -} - -type ListReadProp ReadPropFn - -func (p ListReadProp) List() (*ListView, error) { - v, err := p() - if err != nil { - return nil, err - } - c, ok := v.(*ListView) - if ok { - return nil, fmt.Errorf("view is not a list: %v", v) - } - return c, nil -} - -type BitListReadProp ReadPropFn - -func (p BitListReadProp) BitList() (*BitListView, error) { - v, err := p() - if err != nil { - return nil, err - } - bv, ok := v.(*BitListView) - if ok { - return nil, fmt.Errorf("view is not a bitlist: %v", v) - } - return bv, nil -} diff --git a/props/interaction.go b/props/interaction.go deleted file mode 100644 index c077a96..0000000 --- a/props/interaction.go +++ /dev/null @@ -1,62 +0,0 @@ -package props - -import ( - "errors" - . "github.com/protolambda/ztyp/view" -) - -type ReadProp interface { - Read() (View, error) -} - -type ReadPropFn func() (View, error) - -func (f ReadPropFn) Read() (View, error) { - if f == nil { - return nil, errors.New("property is not available to read") - } - return f() -} - -type WriteProp interface { - Write(v View) error -} - -type WritePropFn func(v View) error - -func (f WritePropFn) Write(v View) error { - if f == nil { - return errors.New("property is not available to write") - } - return f(v) -} - -type MutProp interface { - ReadProp - WriteProp -} - -type MutPropFns struct { - ReadPropFn - WritePropFn -} - -type ReadablePropView interface { - Get(i uint64) (View, error) -} - -type WritablePropView interface { - Set(i uint64, v View) error -} - -func PropReader(rv ReadablePropView, i uint64) ReadPropFn { - return func() (View, error) { - return rv.Get(i) - } -} - -func PropWriter(wv WritablePropView, i uint64) WritePropFn { - return func(v View) error { - return wv.Set(i, v) - } -} diff --git a/tree/bitlen.go b/tree/bitlen.go index edf25c5..7d12a42 100644 --- a/tree/bitlen.go +++ b/tree/bitlen.go @@ -26,7 +26,7 @@ func CoverDepth(v uint64) (out uint8) { if v == 0 || v == 1 { return 0 } - return BitIndex(v - 1) + 1 + return BitIndex(v-1) + 1 } // bitmagic: binary search through a uint64 to find the bit-length diff --git a/tree/bitlen_test.go b/tree/bitlen_test.go index 5778608..d129a2a 100644 --- a/tree/bitlen_test.go +++ b/tree/bitlen_test.go @@ -29,9 +29,9 @@ var testCases = []testCase{ func init() { for i := uint8(4); i < 64; i++ { testCases = append(testCases, - testCase{v: (1 << i) - 1, d: i, l: i, i: i-1}, - testCase{v: 1 << i , d: i, l: i+1, i: i}, - testCase{v: (1 << i) + 1, d: i+1, l: i+1, i: i}, + testCase{v: (1 << i) - 1, d: i, l: i, i: i - 1}, + testCase{v: 1 << i, d: i, l: i + 1, i: i}, + testCase{v: (1 << i) + 1, d: i + 1, l: i + 1, i: i}, ) } } diff --git a/tree/gindex.go b/tree/gindex.go index a2b9cf2..555a233 100644 --- a/tree/gindex.go +++ b/tree/gindex.go @@ -1,6 +1,9 @@ package tree -import "fmt" +import ( + "encoding/binary" + "fmt" +) type Gindex interface { // Subtree returns the same gindex, but with the anchor moved one bit to the right, to represent the subtree position. @@ -24,6 +27,12 @@ type Gindex interface { // Iterate over the bits of the gindex // The depth is excl. the "root" bit BitIter() (iter GindexBitIter, depth uint32) + // Encode the gindex as little-endian byte array. An invalid gindex (<= 0) must return nil. + LittleEndian() []byte + // Encode the gindex as big-endian byte array. An invalid gindex (<= 0) must return nil. + BigEndian() []byte + // LeftAlignedBigEndian returns the bits shifted such that the anchor bit is the left-most + LeftAlignedBigEndian() (data []byte, bitLen uint32) } type GindexBitIter interface { @@ -89,6 +98,61 @@ func (v Gindex64) BitIter() (iter GindexBitIter, depth uint32) { }, uint32(d) } +func (v Gindex64) LittleEndian() []byte { + if v == 0 { + return nil + } + var out [8]byte + binary.LittleEndian.PutUint64(out[:], uint64(v)) + s := 1 + if v >= 1<<32 { + v >>= 32 + s += 4 + } + if v >= 1<<16 { + v >>= 16 + s += 2 + } + if v >= 1<<8 { + v >>= 8 + s += 1 + } + return out[:s] +} + +func (v Gindex64) BigEndian() []byte { + if v == 0 { + return nil + } + var out [8]byte + binary.BigEndian.PutUint64(out[:], uint64(v)) + s := 7 + if v >= 1<<32 { + v >>= 32 + s -= 4 + } + if v >= 1<<16 { + v >>= 16 + s -= 2 + } + if v >= 1<<8 { + v >>= 8 + s -= 1 + } + return out[s:] +} + +func (v Gindex64) LeftAlignedBigEndian() (data []byte, bitLen uint32) { + if v == 0 { + return nil, 0 + } + bitLen8 := BitLength(uint64(v)) + leftAligned := uint64(v) << (64 - bitLen8) + var out [8]byte + binary.BigEndian.PutUint64(out[:], leftAligned) + return out[:(bitLen8+7)>>3], uint32(bitLen8) +} + func ToGindex64(index uint64, depth uint8) (Gindex64, error) { if depth >= 64 { return 0, fmt.Errorf("depth %d is too deep for Gindex64", depth) @@ -107,5 +171,5 @@ type Gindex64BitIter struct { func (iter *Gindex64BitIter) Next() (right bool, ok bool) { iter.Marker >>= 1 - return iter.Gindex & iter.Marker != 0, iter.Marker != 0 + return iter.Gindex&iter.Marker != 0, iter.Marker != 0 } diff --git a/tree/gindex_test.go b/tree/gindex_test.go new file mode 100644 index 0000000..077d69f --- /dev/null +++ b/tree/gindex_test.go @@ -0,0 +1,46 @@ +package tree + +import ( + "encoding/hex" + "testing" +) + +func TestGindex64_Encoding(t *testing.T) { + cases := map[Gindex64]string{ + 0: "", + 1: "01", + 2: "02", + 0xff: "ff", + 0x01ff: "01ff", + 0xaaaaaa: "aaaaaa", + 0xabcdef12345: "0abcdef12345", + 0xaffffffffffffffb: "affffffffffffffb", + } + for k, v := range cases { + t.Run(v, func(t *testing.T) { + t.Run("little", func(t *testing.T) { + data := k.LittleEndian() + got := hex.EncodeToString(data) + + decoded, err := hex.DecodeString(v) + if err != nil { + t.Fatal(err) + } + for i, j := 0, len(decoded)-1; i < j; i, j = i+1, j-1 { + decoded[i], decoded[j] = decoded[j], decoded[i] + } + littleEndianV := hex.EncodeToString(decoded) + if got != littleEndianV { + t.Errorf("got %s, expected %s", got, littleEndianV) + } + }) + t.Run("big", func(t *testing.T) { + data := k.BigEndian() + got := hex.EncodeToString(data) + if got != v { + t.Errorf("got %s, expected %s", got, v) + } + }) + }) + } +} diff --git a/tree/hashing.go b/tree/hashing.go index 872b685..3c095a0 100644 --- a/tree/hashing.go +++ b/tree/hashing.go @@ -2,11 +2,174 @@ package tree import ( "crypto/sha256" + "encoding/binary" "fmt" + "github.com/protolambda/ztyp/bitfields" ) +type HTR interface { + HashTreeRoot(h HashFn) Root +} + +type SeriesHTR func(i uint64) HTR + +type ChunksHTR func(i uint64) Root + type HashFn func(a Root, b Root) Root +// HashTreeRoot is a small utility function to implement HTR for custom compound types easily. +// E.g. for a struct `x` with 3 fields, call hFn.HashTreeRoot(x.A, x.B, x.C) +func (h HashFn) HashTreeRoot(fields ...HTR) Root { + // TODO; benchmark, may be worth hard-coding a few more common short-paths + n := uint64(len(fields)) + switch n { + case 0: + return Root{} + case 1: + return fields[0].HashTreeRoot(h) + case 2: + return h(fields[0].HashTreeRoot(h), fields[1].HashTreeRoot(h)) + default: + return Merkleize(h, uint64(len(fields)), uint64(len(fields)), func(i uint64) Root { + return fields[i].HashTreeRoot(h) + }) + } +} + +func (h HashFn) ComplexVectorHTR(series SeriesHTR, length uint64) Root { + // length is alos limit for vectors + return Merkleize(h, length, length, func(i uint64) Root { + htr := series(i) + if htr == nil { // missing element? Fine, just like an empty node then + return Root{} + } + return htr.HashTreeRoot(h) + }) +} + +func (h HashFn) ComplexListHTR(series SeriesHTR, length uint64, limit uint64) Root { + return h.Mixin(Merkleize(h, length, limit, func(i uint64) Root { + htr := series(i) + if htr == nil { // missing element? Fine, just like an empty node then + return Root{} + } + return htr.HashTreeRoot(h) + }), length) +} + +func (h HashFn) Mixin(v Root, length uint64) Root { + var mixin Root + binary.LittleEndian.PutUint64(mixin[:], length) + return h(v, mixin) +} + +// ChunksHTR is like SeriesHTR, except that the items are chunked by the input, +// and chunks are merely merkleized to get the hash-tree-root. +// No length mixin is performed (required for a list/basic-list/bitlist hash-tree-root). +func (h HashFn) ChunksHTR(chunks ChunksHTR, length uint64, limit uint64) Root { + return Merkleize(h, length, limit, chunks) +} + +func (h HashFn) Uint8VectorHTR(v func(i uint64) uint8, length uint64) Root { + // 32 items per chunk + chunks := (length + 31) >> 5 + return h.ChunksHTR(func(i uint64) (out Root) { + for x, j := 0, i<<5; x < 32 && j < length; j, x = j+1, x+1 { + out[x] = v(j) + } + return + }, chunks, chunks) +} + +func (h HashFn) Uint8ListHTR(v func(i uint64) uint8, length uint64, limit uint64) Root { + // 32 items per chunk + chunks := (length + 31) >> 5 + return h.Mixin(h.ChunksHTR(func(i uint64) (out Root) { + for x, j := 0, i<<5; x < 32 && j < length; j, x = j+1, x+1 { + out[x] = v(j) + } + return + }, chunks, (limit+31)>>5), length) +} + +func (h HashFn) Uint64VectorHTR(v func(i uint64) uint64, length uint64) Root { + // 4 items per chunk + chunks := (length + 3) >> 2 + return h.ChunksHTR(func(i uint64) (out Root) { + for x, j := 0, i<<2; x < 32 && j < length; j, x = j+1, x+8 { + binary.LittleEndian.PutUint64(out[x:], v(j)) + } + return + }, chunks, chunks) +} + +func (h HashFn) Uint64ListHTR(v func(i uint64) uint64, length uint64, limit uint64) Root { + // 4 items per chunk + chunks := (length + 3) >> 2 + return h.Mixin(h.ChunksHTR(func(i uint64) (out Root) { + for x, j := 0, i<<2; x < 32 && j < length; j, x = j+1, x+8 { + binary.LittleEndian.PutUint64(out[x:], v(j)) + } + return + }, chunks, (limit+3)>>2), length) +} + +func (h HashFn) ByteVectorHTR(values []byte) Root { + chunks := (uint64(len(values)) + 31) / 32 + return h.ChunksHTR(func(i uint64) (out Root) { + copy(out[:], values[i<<5:]) + return + }, chunks, chunks) +} + +func (h HashFn) ByteListHTR(values []byte, limit uint64) Root { + chunks := (uint64(len(values)) + 31) / 32 + chunkLimit := (limit + 31) / 32 + return h.Mixin(h.ChunksHTR(func(i uint64) (out Root) { + copy(out[:], values[i<<5:]) + return + }, chunks, chunkLimit), uint64(len(values))) +} + +func (h HashFn) BitVectorHTR(bits []byte) Root { + // it's a vector, chunks is also chunkLimit. + // The bits are already packed in bytes, just divide by 32 (rounding up). + chunks := (uint64(len(bits)) + 31) / 32 + return h.ChunksHTR(func(i uint64) (out Root) { + if i < chunks { + copy(out[:], bits[i<<5:]) + // no delimiter bits in bit vectors + } + return + }, chunks, chunks) +} + +func (h HashFn) BitListHTR(bits []byte, bitlimit uint64) Root { + bitLen := bitfields.BitlistLen(bits) + chunks := (bitLen + 0xff) >> 8 + chunkLimit := (bitlimit + 0xff) >> 8 + return h.Mixin(h.ChunksHTR(func(i uint64) (out Root) { + if i < chunks { + copy(out[:], bits[i<<5:]) + // mask out delimit bit if necessary + if ((i + 1) << 8) > bitLen { + out[(bitLen&0xff)>>3] &^= 1 << (bitLen & 0x7) + } + } + return + }, chunks, chunkLimit), bitLen) +} + +func (h HashFn) Union(selector uint8, value HTR) Root { + var selectorNode Root + selectorNode[0] = selector + if value == nil { + return h(Root{}, selectorNode) + } else { + return h(value.HashTreeRoot(h), selectorNode) + } +} + func sha256Combi(a Root, b Root) Root { v := [64]byte{} copy(v[:32], a[:]) @@ -16,9 +179,9 @@ func sha256Combi(a Root, b Root) Root { func sha256CombiRepeat() HashFn { hash := sha256.New() + v := [64]byte{} hashFn := func(a Root, b Root) (out Root) { hash.Reset() - v := [64]byte{} copy(v[:32], a[:]) copy(v[32:], b[:]) hash.Write(v[:]) diff --git a/tree/merkle.go b/tree/merkle.go new file mode 100644 index 0000000..3bbc816 --- /dev/null +++ b/tree/merkle.go @@ -0,0 +1,64 @@ +package tree + +// Merkleize with log(N) space allocation +func Merkleize(hasher HashFn, count uint64, limit uint64, leaf func(i uint64) Root) (out Root) { + if count > limit { + // merkleizing list that is too large, over limit + count = limit + } + if limit == 0 { + return + } + if limit == 1 { + if count == 1 { + out = leaf(0) + } + return + } + depth := CoverDepth(count) + limitDepth := CoverDepth(limit) + tmp := make([]Root, limitDepth+1, limitDepth+1) + + j := uint8(0) + hArr := Root{} + + merge := func(i uint64) { + // merge back up from bottom to top, as far as we can + for j = 0; ; j++ { + // stop merging when we are in the left side of the next combi + if i&(uint64(1)<= length { + *roots = (*roots)[:length] + } else { + *roots = make([]Root, length, length) + } + } + dst := *roots + for i := uint64(0); i < length; i++ { + if _, err := dr.Read(dst[i][:]); err != nil { + return err + } + } + return nil +} + +func ReadRootsLimited(dr *codec.DecodingReader, roots *[]Root, limit uint64) error { + scope := dr.Scope() + if scope%32 != 0 { + return fmt.Errorf("bad deserialization scope, cannot decode roots list") + } + length := scope / 32 + if length > limit { + return fmt.Errorf("too many roots: %d > %d", length, limit) + } + return ReadRoots(dr, roots, length) +} + +// WriteRoots serialization, efficient version of List/Vector serialization +func WriteRoots(ew *codec.EncodingWriter, roots []Root) error { + for i := range roots { + if err := ew.Write(roots[i][:]); err != nil { + return err + } + } + return nil +} diff --git a/tree/root_test.go b/tree/root_test.go new file mode 100644 index 0000000..8697077 --- /dev/null +++ b/tree/root_test.go @@ -0,0 +1,103 @@ +package tree + +import ( + "encoding/json" + "fmt" + "testing" +) + +type rootTestCase struct { + root Root + valid bool + input []byte +} + +var unmarshalTestCases = []rootTestCase{ + {Root{0: 0xaa, 31: 0xbb}, true, []byte("aa000000000000000000000000000000000000000000000000000000000000bb")}, + {Root{0: 0xaa, 31: 0xbb}, true, []byte("0xaa000000000000000000000000000000000000000000000000000000000000bb")}, + {Root{0: 0xaa, 31: 0xbb}, true, []byte("0Xaa000000000000000000000000000000000000000000000000000000000000bb")}, + {Root{0: 0xaa, 31: 0xbb}, true, []byte("0XaA000000000000000000000000000000000000000000000000000000000000bb")}, + {Root{0: 0xaa, 31: 0xbb}, true, []byte("AA000000000000000000000000000000000000000000000000000000000000BB")}, + {Root{0: 0xaa, 31: 0xbb}, true, []byte("0XAA000000000000000000000000000000000000000000000000000000000000BB")}, + {Root{}, false, []byte("aa000000000000000000000000000000000000000000000000000000000000bb0x")}, + {Root{}, false, []byte("0xaa000000000000000000000000000000000000000000000000000000000000bb0")}, + {Root{}, false, []byte("0xaa000000000000000000000000000000000000000000000000000000000000b")}, + {Root{}, false, []byte("xaa000000000000000000000000000000000000000000000000000000000000bb")}, + {Root{}, false, []byte("a000000000000000000000000000000000000000000000000000000000000bb")}, + {Root{}, false, []byte("aa00000000000000000000000000000000000000y000000000000000000000bb")}, + {Root{}, false, []byte("aa00")}, + {Root{}, false, []byte("a")}, + {Root{}, false, []byte{}}, +} + +func TestRoot_UnmarshalText(t *testing.T) { + for i, testCase := range unmarshalTestCases { + t.Run(fmt.Sprintf("case%d", i), func(t *testing.T) { + var x Root + err := x.UnmarshalText(testCase.input) + if testCase.valid { + if err != nil { + t.Error(err) + return + } + if x != testCase.root { + t.Errorf("Expected %x but got %x", testCase.root[:], x[:]) + } + } else { + if err == nil { + t.Error("expected error but did not get any") + } + } + }) + } +} + +func TestRoot_MarshalText(t *testing.T) { + x := Root{0: 0xaa, 31: 0xbb} + data, err := x.MarshalText() + if err != nil { + t.Error(t) + return + } + if string(data) != "0xaa000000000000000000000000000000000000000000000000000000000000bb" { + t.Errorf("unexpected string: %s", string(data)) + return + } +} + +func TestRoot_String(t *testing.T) { + x := Root{0: 0xaa, 31: 0xbb} + if res := fmt.Sprintf("fmt %s", x); res != "fmt 0xaa000000000000000000000000000000000000000000000000000000000000bb" { + t.Errorf("unexpected direct fmt result: %s", res) + return + } + if res := fmt.Sprintf("fmt %s", &x); res != "fmt 0xaa000000000000000000000000000000000000000000000000000000000000bb" { + t.Errorf("unexpected pointer fmt result: %s", res) + return + } +} + +func TestRoot_JSON_Marshal(t *testing.T) { + x := Root{0: 0xaa, 31: 0xbb} + out, err := json.Marshal(x) + if err != nil { + t.Error(err) + return + } + if string(out) != "\"0xaa000000000000000000000000000000000000000000000000000000000000bb\"" { + t.Errorf("unexpected json: %s", string(out)) + return + } +} + +func TestRoot_JSON_Unmarshal(t *testing.T) { + var x Root + if err := json.Unmarshal([]byte("\"0xaa000000000000000000000000000000000000000000000000000000000000bb\""), &x); err != nil { + t.Error(err) + return + } + if x != (Root{0: 0xaa, 31: 0xbb}) { + t.Errorf("unexpected value: %s", x) + return + } +} diff --git a/view/backed.go b/view/backed.go index 37bf203..a6b4fb2 100644 --- a/view/backed.go +++ b/view/backed.go @@ -4,12 +4,12 @@ import . "github.com/protolambda/ztyp/tree" type BackedView struct { ViewBase - Hook BackingHook + Hook BackingHook BackingNode Node } func (v *BackedView) Copy() (View, error) { - return v.TypeDef.ViewFromBacking(v.BackingNode, v.Hook) + return v.TypeDef.ViewFromBacking(v.BackingNode, nil) // copy does not propagate changes to the same hook. } func (v *BackedView) Default(hook BackingHook) (View, error) { diff --git a/view/basic.go b/view/basic.go index b97cb7b..519db53 100644 --- a/view/basic.go +++ b/view/basic.go @@ -4,8 +4,10 @@ import ( "encoding/binary" "errors" "fmt" + "github.com/protolambda/ztyp/codec" + "github.com/protolambda/ztyp/conv" . "github.com/protolambda/ztyp/tree" - "io" + "strconv" ) // A uint type, identified by its size in bytes. @@ -21,6 +23,10 @@ func (td UintMeta) Default(_ BackingHook) View { return Uint32View(0) case Uint64Type: return Uint64View(0) + case Uint128Type: + return nil + case Uint256Type: + return Uint256View{} default: // unsupported uint type return nil @@ -41,6 +47,10 @@ func (td UintMeta) New() BasicView { return Uint32View(0) case Uint64Type: return Uint64View(0) + case Uint128Type: + return nil + case Uint256Type: + return Uint256View{} default: return nil } @@ -62,6 +72,12 @@ func (td UintMeta) ViewFromBacking(node Node, _ BackingHook) (View, error) { return Uint32View(binary.LittleEndian.Uint32(v[:4])), nil case Uint64Type: return Uint64View(binary.LittleEndian.Uint64(v[:8])), nil + case Uint128Type: + return nil, UnsupportedUintType + case Uint256Type: + var out Uint256View + out.setBytes32(v[:]) + return out, nil default: return nil, UnsupportedUintType } @@ -80,6 +96,12 @@ func (td UintMeta) BasicViewFromBacking(v *Root, i uint8) (BasicView, error) { return Uint32View(binary.LittleEndian.Uint32(v[4*i : 4*i+4])), nil case Uint64Type: return Uint64View(binary.LittleEndian.Uint64(v[8*i : 8*i+8])), nil + case Uint128Type: + return nil, UnsupportedUintType + case Uint256Type: + var out Uint256View + out.setBytes32(v[:]) + return out, nil default: return nil, UnsupportedUintType } @@ -97,7 +119,7 @@ func (td UintMeta) PackViews(views []BasicView) ([]Node, error) { root = views[i].BackingFromBase(root, j) i += 1 } - chunks = append(chunks, root) + chunks[chunk] = root } return chunks, nil } @@ -118,24 +140,26 @@ func (td UintMeta) MaxByteLength() uint64 { return uint64(td) } -func (td UintMeta) Deserialize(r io.Reader, scope uint64) (View, error) { - if scope < uint64(td) { - return nil, fmt.Errorf("scope of %d bytes not enough for %s", scope, td.String()) - } - tmp := make([]byte, td, td) - _, err := r.Read(tmp) - if err != nil { - return nil, err - } +func (td UintMeta) Deserialize(dr *codec.DecodingReader) (View, error) { switch td { case Uint8Type: - return Uint8View(tmp[0]), nil + v, err := dr.ReadByte() + return Uint8View(v), err case Uint16Type: - return Uint16View(binary.LittleEndian.Uint16(tmp)), nil + v, err := dr.ReadUint16() + return Uint16View(v), err case Uint32Type: - return Uint32View(binary.LittleEndian.Uint32(tmp)), nil + v, err := dr.ReadUint32() + return Uint32View(v), err case Uint64Type: - return Uint64View(binary.LittleEndian.Uint64(tmp)), nil + v, err := dr.ReadUint64() + return Uint64View(v), err + case Uint128Type: + return nil, UnsupportedUintType + case Uint256Type: + var out Uint256View + err := out.Deserialize(dr) + return out, err default: return nil, UnsupportedUintType } @@ -146,16 +170,31 @@ func (td UintMeta) String() string { } const ( - Uint8Type UintMeta = 1 - Uint16Type UintMeta = 2 - Uint32Type UintMeta = 4 - Uint64Type UintMeta = 8 + Uint8Type UintMeta = 1 + Uint16Type UintMeta = 2 + Uint32Type UintMeta = 4 + Uint64Type UintMeta = 8 + Uint128Type UintMeta = 16 + Uint256Type UintMeta = 32 ) var BasicViewNoSetBackingError = errors.New("basic views cannot set new backing") +var BadLengthError = errors.New("scope is wrong") + type Uint8View uint8 +func AsUint8(v View, err error) (Uint8View, error) { + if err != nil { + return 0, err + } + n, ok := v.(Uint8View) + if !ok { + return 0, fmt.Errorf("not a uint8 view: %v", v) + } + return n, nil +} + func (v Uint8View) SetBacking(b Node) error { return BasicViewNoSetBackingError } @@ -183,9 +222,37 @@ func (v Uint8View) ValueByteLength() (uint64, error) { return 1, nil } -func (v Uint8View) Serialize(w io.Writer) error { - _, err := w.Write([]byte{byte(v)}) - return err +func (v Uint8View) ByteLength() uint64 { + return 1 +} + +func (v Uint8View) FixedLength() uint64 { + return 1 +} + +func (v Uint8View) Serialize(w *codec.EncodingWriter) error { + return w.WriteByte(byte(v)) +} + +func (v Uint8View) Encode() ([]byte, error) { + return []byte{byte(v)}, nil +} + +func (v *Uint8View) Deserialize(r *codec.DecodingReader) error { + b, err := r.ReadByte() + if err != nil { + return err + } + *v = Uint8View(b) + return nil +} + +func (v *Uint8View) Decode(x []byte) error { + if len(x) != 1 { + return BadLengthError + } + *v = Uint8View(x[0]) + return nil } func (v Uint8View) HashTreeRoot(h HashFn) Root { @@ -198,14 +265,55 @@ func (v Uint8View) Type() TypeDef { return Uint8Type } +func (v Uint8View) MarshalText() (out []byte, err error) { + out = strconv.AppendUint(out, uint64(v), 10) + return +} + +func (v *Uint8View) UnmarshalText(b []byte) error { + n, err := strconv.ParseUint(string(b), 0, 8) + if err != nil { + return err + } + *v = Uint8View(n) + return nil +} + +func (v Uint8View) MarshalJSON() ([]byte, error) { + return conv.Uint8Marshal(uint8(v)) +} + +func (v *Uint8View) UnmarshalJSON(b []byte) error { + return conv.Uint8Unmarshal((*uint8)(v), b) +} + +func (v Uint8View) String() string { + return strconv.FormatUint(uint64(v), 10) +} + // Alias to Uint8Type const ByteType = Uint8Type // Alias to Uint8View type ByteView = Uint8View +func AsByte(v View, err error) (ByteView, error) { + return AsUint8(v, err) +} + type Uint16View uint16 +func AsUint16(v View, err error) (Uint16View, error) { + if err != nil { + return 0, err + } + n, ok := v.(Uint16View) + if !ok { + return 0, fmt.Errorf("not a uint16 view: %v", v) + } + return n, nil +} + func (v Uint16View) SetBacking(b Node) error { return BasicViewNoSetBackingError } @@ -233,9 +341,39 @@ func (v Uint16View) ValueByteLength() (uint64, error) { return 2, nil } -func (v Uint16View) Serialize(w io.Writer) error { - _, err := w.Write([]byte{byte(v), byte(v >> 8)}) - return err +func (v Uint16View) ByteLength() uint64 { + return 2 +} + +func (v Uint16View) FixedLength() uint64 { + return 2 +} + +func (v Uint16View) Serialize(w *codec.EncodingWriter) error { + return w.WriteUint16(uint16(v)) +} + +func (v Uint16View) Encode() ([]byte, error) { + var out [2]byte + binary.LittleEndian.PutUint16(out[:], uint16(v)) + return out[:], nil +} + +func (v *Uint16View) Deserialize(r *codec.DecodingReader) error { + d, err := r.ReadUint16() + if err != nil { + return err + } + *v = Uint16View(d) + return nil +} + +func (v *Uint16View) Decode(x []byte) error { + if len(x) != 2 { + return BadLengthError + } + *v = Uint16View(binary.LittleEndian.Uint16(x[:])) + return nil } func (v Uint16View) HashTreeRoot(h HashFn) Root { @@ -248,8 +386,45 @@ func (v Uint16View) Type() TypeDef { return Uint16Type } +func (v Uint16View) MarshalText() (out []byte, err error) { + out = strconv.AppendUint(out, uint64(v), 10) + return +} + +func (v *Uint16View) UnmarshalText(b []byte) error { + n, err := strconv.ParseUint(string(b), 0, 16) + if err != nil { + return err + } + *v = Uint16View(n) + return nil +} + +func (v Uint16View) MarshalJSON() ([]byte, error) { + return conv.Uint16Marshal(uint16(v)) +} + +func (v *Uint16View) UnmarshalJSON(b []byte) error { + return conv.Uint16Unmarshal((*uint16)(v), b) +} + +func (v Uint16View) String() string { + return strconv.FormatUint(uint64(v), 10) +} + type Uint32View uint32 +func AsUint32(v View, err error) (Uint32View, error) { + if err != nil { + return 0, err + } + n, ok := v.(Uint32View) + if !ok { + return 0, fmt.Errorf("not a uint32 view: %v", v) + } + return n, nil +} + func (v Uint32View) SetBacking(b Node) error { return BasicViewNoSetBackingError } @@ -277,9 +452,39 @@ func (v Uint32View) ValueByteLength() (uint64, error) { return 4, nil } -func (v Uint32View) Serialize(w io.Writer) error { - _, err := w.Write([]byte{byte(v), byte(v >> 8), byte(v >> 16), byte(v >> 24)}) - return err +func (v Uint32View) ByteLength() uint64 { + return 4 +} + +func (v Uint32View) FixedLength() uint64 { + return 4 +} + +func (v Uint32View) Serialize(w *codec.EncodingWriter) error { + return w.WriteUint32(uint32(v)) +} + +func (v Uint32View) Encode() ([]byte, error) { + var out [4]byte + binary.LittleEndian.PutUint32(out[:], uint32(v)) + return out[:], nil +} + +func (v *Uint32View) Deserialize(r *codec.DecodingReader) error { + d, err := r.ReadUint32() + if err != nil { + return err + } + *v = Uint32View(d) + return nil +} + +func (v *Uint32View) Decode(x []byte) error { + if len(x) != 4 { + return BadLengthError + } + *v = Uint32View(binary.LittleEndian.Uint32(x[:])) + return nil } func (v Uint32View) HashTreeRoot(h HashFn) Root { @@ -292,8 +497,45 @@ func (v Uint32View) Type() TypeDef { return Uint32Type } +func (v Uint32View) MarshalText() (out []byte, err error) { + out = strconv.AppendUint(out, uint64(v), 10) + return +} + +func (v *Uint32View) UnmarshalText(b []byte) error { + n, err := strconv.ParseUint(string(b), 0, 32) + if err != nil { + return err + } + *v = Uint32View(n) + return nil +} + +func (v Uint32View) MarshalJSON() ([]byte, error) { + return conv.Uint32Marshal(uint32(v)) +} + +func (v *Uint32View) UnmarshalJSON(b []byte) error { + return conv.Uint32Unmarshal((*uint32)(v), b) +} + +func (v Uint32View) String() string { + return strconv.FormatUint(uint64(v), 10) +} + type Uint64View uint64 +func AsUint64(v View, err error) (Uint64View, error) { + if err != nil { + return 0, err + } + n, ok := v.(Uint64View) + if !ok { + return 0, fmt.Errorf("not a uint64 view: %v", v) + } + return n, nil +} + func (v Uint64View) SetBacking(b Node) error { return BasicViewNoSetBackingError } @@ -321,11 +563,39 @@ func (v Uint64View) ValueByteLength() (uint64, error) { return 8, nil } -func (v Uint64View) Serialize(w io.Writer) error { - b := [8]byte{} - binary.LittleEndian.PutUint64(b[:], uint64(v)) - _, err := w.Write(b[:]) - return err +func (v Uint64View) ByteLength() uint64 { + return 8 +} + +func (v Uint64View) FixedLength() uint64 { + return 8 +} + +func (v Uint64View) Serialize(w *codec.EncodingWriter) error { + return w.WriteUint64(uint64(v)) +} + +func (v Uint64View) Encode() ([]byte, error) { + var out [8]byte + binary.LittleEndian.PutUint64(out[:], uint64(v)) + return out[:], nil +} + +func (v *Uint64View) Deserialize(r *codec.DecodingReader) error { + d, err := r.ReadUint64() + if err != nil { + return err + } + *v = Uint64View(d) + return nil +} + +func (v *Uint64View) Decode(x []byte) error { + if len(x) != 8 { + return BadLengthError + } + *v = Uint64View(binary.LittleEndian.Uint64(x[:])) + return nil } func (v Uint64View) HashTreeRoot(h HashFn) Root { @@ -338,6 +608,32 @@ func (v Uint64View) Type() TypeDef { return Uint64Type } +func (v Uint64View) MarshalText() (out []byte, err error) { + out = strconv.AppendUint(out, uint64(v), 10) + return +} + +func (v *Uint64View) UnmarshalText(b []byte) error { + n, err := strconv.ParseUint(string(b), 0, 64) + if err != nil { + return err + } + *v = Uint64View(n) + return nil +} + +func (v Uint64View) MarshalJSON() ([]byte, error) { + return conv.Uint64Marshal(uint64(v)) +} + +func (v *Uint64View) UnmarshalJSON(b []byte) error { + return conv.Uint64Unmarshal((*uint64)(v), b) +} + +func (v Uint64View) String() string { + return strconv.FormatUint(uint64(v), 10) +} + type BoolMeta uint8 func (td BoolMeta) SubViewFromBacking(v *Root, i uint8) BasicView { @@ -393,19 +689,15 @@ func (td BoolMeta) MaxByteLength() uint64 { return 1 } -func (td BoolMeta) Deserialize(r io.Reader, scope uint64) (View, error) { - if scope == 0 { - return nil, errors.New("cannot read bool, scope is 0") - } - b := [1]byte{} - _, err := r.Read(b[:]) +func (td BoolMeta) Deserialize(dr *codec.DecodingReader) (View, error) { + b, err := dr.ReadByte() if err != nil { return nil, err } - if b[0] > 1 { - return nil, fmt.Errorf("invalid bool value: 0x%x", b[0]) + if b > 1 { + return nil, fmt.Errorf("invalid bool value: 0x%x", b) } - return BoolView(b[0] == 1), nil + return BoolView(b == 1), nil } func (td BoolMeta) String() string { @@ -416,6 +708,17 @@ const BoolType BoolMeta = 0 type BoolView bool +func AsBool(v View, err error) (BoolView, error) { + if err != nil { + return false, err + } + b, ok := v.(BoolView) + if !ok { + return false, fmt.Errorf("not a bool view: %v", v) + } + return b, nil +} + var trueRoot = &Root{1} func (v BoolView) SetBacking(b Node) error { @@ -465,13 +768,47 @@ func (v BoolView) Copy() (View, error) { return v, nil } +func (v BoolView) ByteLength() uint64 { + return 1 +} + func (v BoolView) ValueByteLength() (uint64, error) { return 1, nil } -func (v BoolView) Serialize(w io.Writer) error { - _, err := w.Write([]byte{v.byte()}) - return err +func (v BoolView) FixedLength() uint64 { + return 1 +} + +func (v BoolView) Serialize(w *codec.EncodingWriter) error { + return w.WriteByte(v.byte()) +} + +func (v BoolView) Encode() ([]byte, error) { + return []byte{v.byte()}, nil +} + +func (v *BoolView) Deserialize(r *codec.DecodingReader) error { + d, err := r.ReadByte() + if err != nil { + return err + } + if d > 1 { + return fmt.Errorf("invalid bool value: 0x%x", d) + } + *v = BoolView(d > 0) + return nil +} + +func (v *BoolView) Decode(x []byte) error { + if len(x) != 1 { + return BadLengthError + } + if x[0] > 1 { + return fmt.Errorf("invalid bool value: 0x%x", x[0]) + } + *v = BoolView(x[0] > 0) + return nil } func (v BoolView) HashTreeRoot(h HashFn) Root { @@ -483,3 +820,11 @@ func (v BoolView) HashTreeRoot(h HashFn) Root { func (v BoolView) Type() TypeDef { return BoolType } + +func (v BoolView) String() string { + if v { + return "true" + } else { + return "false" + } +} diff --git a/view/basic_list.go b/view/basic_list.go index 5ea26f9..f415f70 100644 --- a/view/basic_list.go +++ b/view/basic_list.go @@ -3,8 +3,8 @@ package view import ( "encoding/binary" "fmt" + "github.com/protolambda/ztyp/codec" . "github.com/protolambda/ztyp/tree" - "io" ) type BasicListTypeDef struct { @@ -26,7 +26,7 @@ func BasicListType(elemType BasicTypeDef, limit uint64) *BasicListTypeDef { } } -func (td *BasicListTypeDef) FromElements(v... BasicView) (*BasicListView, error) { +func (td *BasicListTypeDef) FromElements(v ...BasicView) (*BasicListView, error) { length := uint64(len(v)) if length > td.ListLimit { return nil, fmt.Errorf("expected no more than %d elements, got %d", td.ListLimit, len(v)) @@ -63,10 +63,10 @@ func (td *BasicListTypeDef) ViewFromBacking(node Node, hook BackingHook) (View, ViewBase: ViewBase{ TypeDef: td, }, - Hook: hook, + Hook: hook, BackingNode: node, }, - depth: depth + 1, // +1 for length mix-in + depth: depth + 1, // +1 for length mix-in }, BasicListTypeDef: td, }, nil @@ -95,17 +95,21 @@ func (td *BasicListTypeDef) New() *BasicListView { return td.Default(nil).(*BasicListView) } -func (td *BasicListTypeDef) Deserialize(r io.Reader, scope uint64) (View, error) { +func (td *BasicListTypeDef) Deserialize(dr *codec.DecodingReader) (View, error) { elemSize := td.ElemType.TypeByteLength() + scope := dr.Scope() length := scope / elemSize if length > td.ListLimit { return nil, fmt.Errorf("too many items, limit %d but got %d", td.ListLimit, length) } - if expected := length*elemSize; expected != scope { + if expected := length * elemSize; expected != scope { return nil, fmt.Errorf("scope %d does not align to elem size %d", scope, elemSize) } + if length == 0 { + return td.New(), nil + } contents := make([]byte, scope, scope) - if _, err := r.Read(contents); err != nil { + if _, err := dr.Read(contents); err != nil { return nil, err } bottomNodes, err := BytesIntoNodes(contents) @@ -128,6 +132,17 @@ type BasicListView struct { *BasicListTypeDef } +func AsBasicList(v View, err error) (*BasicListView, error) { + if err != nil { + return nil, err + } + bv, ok := v.(*BasicListView) + if !ok { + return nil, fmt.Errorf("view is not a basic list: %v", v) + } + return bv, nil +} + func (tv *BasicListView) ViewRoot(h HashFn) Root { return tv.BackingNode.MerkleRoot(h) } @@ -326,7 +341,7 @@ func (tv *BasicListView) ReadonlyIter() ElemIter { return ErrElemIter{err} } // ignore length mixin in stack - return basicElemReadonlyIter(node, length, tv.depth - 1, tv.ElemType) + return basicElemReadonlyIter(node, length, tv.depth-1, tv.ElemType) } func (tv *BasicListView) ValueByteLength() (uint64, error) { @@ -337,7 +352,7 @@ func (tv *BasicListView) ValueByteLength() (uint64, error) { return length * tv.ElemType.TypeByteLength(), nil } -func (tv *BasicListView) Serialize(w io.Writer) error { +func (tv *BasicListView) Serialize(w *codec.EncodingWriter) error { contentsAnchor, err := tv.BackingNode.Getter(LeftGindex) if err != nil { return err @@ -346,14 +361,14 @@ func (tv *BasicListView) Serialize(w io.Writer) error { if err != nil { return err } - contents := make([]byte, length, length) - // one less depth, ignore length mix-in elemSize := tv.ElemType.TypeByteLength() - nodeCount := (length + elemSize - 1) / elemSize - if err := SubtreeIntoBytes(contentsAnchor, tv.depth - 1, nodeCount, contents); err != nil { + byteLength := length * elemSize + contents := make([]byte, byteLength, byteLength) + // one less depth, ignore length mix-in + perNode := 32 / elemSize + nodeCount := (length + perNode - 1) / perNode + if err := SubtreeIntoBytes(contentsAnchor, tv.depth-1, nodeCount, contents); err != nil { return err } - _, err = w.Write(contents) - return err + return w.Write(contents) } - diff --git a/view/basic_test.go b/view/basic_test.go new file mode 100644 index 0000000..890c458 --- /dev/null +++ b/view/basic_test.go @@ -0,0 +1,89 @@ +package view + +import "testing" + +func TestUint64View_MarshalJSON(t *testing.T) { + cases := []struct { + n uint64 + s string + }{ + {0, "0"}, + {1, "1"}, + {1234, "1234"}, + {uint64(1) << 63, "9223372036854775808"}, + {^uint64(0), "18446744073709551615"}, + } + for _, c := range cases { + t.Run(c.s, func(t *testing.T) { + v, err := Uint64View(c.n).MarshalJSON() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if string(v) != `"`+c.s+`"` { + t.Errorf("unexpected value: %s", string(v)) + } + }) + } +} + +func TestUint64View_UnmarshalJSON(t *testing.T) { + cases := []struct { + n uint64 + s string + }{ + {0, "0"}, + {0, "0b0"}, + {0, "0x0"}, + {0, "0x00"}, + {0, "0x0000"}, + {0, "000"}, + {1, "1"}, + {1234, "1234"}, + {0xc0fFeE, "0xc0fFeE"}, + {0b10101011, "0b10101011"}, + {uint64(1) << 63, "9223372036854775808"}, + {^uint64(0), "18446744073709551615"}, + {^uint64(0), "0xFFFFFFFFFFFFFFFF"}, + {^uint64(0), "0x0FFFFFFFFFFFFFFFF"}, + {^uint64(0), "0x00FFFFFFFFFFFFFFFF"}, + } + for _, c := range cases { + t.Run(c.s+"_unquoted", func(t *testing.T) { + expected := Uint64View(c.n) + var res Uint64View + err := res.UnmarshalJSON([]byte(c.s)) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if res != expected { + t.Errorf("unexpected value: %v", res) + } + }) + t.Run(c.s+"_double_quoted", func(t *testing.T) { + expected := Uint64View(c.n) + var res Uint64View + b := []byte{'"'} + b = append(b, c.s...) + b = append(b, '"') + err := res.UnmarshalJSON(b) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if res != expected { + t.Errorf("unexpected value: %v", res) + } + }) + } + bad := []string{ + ``, `"`, `""`, `''`, `""0""`, `"0""`, `""0"`, `"0'`, `'0'"`, `""0''`, + "00x0", "00x0", "0x", "0b", "-0", "-123", "0x1FFFFFFFFFFFFFFFF", "FFFFFFFFFFFFFFFF", "-a", "-0xab", + } + for _, b := range bad { + t.Run(b, func(t *testing.T) { + var res Uint64View + if err := res.UnmarshalJSON([]byte(b)); err == nil { + t.Errorf("expected error, but got value %d", uint64(res)) + } + }) + } +} diff --git a/view/basic_vector.go b/view/basic_vector.go index 2e42543..fc83316 100644 --- a/view/basic_vector.go +++ b/view/basic_vector.go @@ -2,8 +2,8 @@ package view import ( "fmt" + "github.com/protolambda/ztyp/codec" . "github.com/protolambda/ztyp/tree" - "io" ) type BasicVectorTypeDef struct { @@ -18,15 +18,15 @@ func BasicVectorType(elemType BasicTypeDef, length uint64) *BasicVectorTypeDef { ElemType: elemType, VectorLength: length, ComplexTypeBase: ComplexTypeBase{ - MinSize: size, - MaxSize: size, - Size: size, + MinSize: size, + MaxSize: size, + Size: size, IsFixedSize: true, }, } } -func (td *BasicVectorTypeDef) FromElements(v... BasicView) (*BasicVectorView, error) { +func (td *BasicVectorTypeDef) FromElements(v ...BasicView) (*BasicVectorView, error) { length := uint64(len(v)) if length > td.VectorLength { return nil, fmt.Errorf("expected no more than %d elements, got %d", td.VectorLength, length) @@ -62,10 +62,10 @@ func (td *BasicVectorTypeDef) ViewFromBacking(node Node, hook BackingHook) (View ViewBase: ViewBase{ TypeDef: td, }, - Hook: hook, + Hook: hook, BackingNode: node, }, - depth: depth, + depth: depth, }, BasicVectorTypeDef: td, }, nil @@ -94,12 +94,13 @@ func (td *BasicVectorTypeDef) New() *BasicVectorView { return td.Default(nil).(*BasicVectorView) } -func (td *BasicVectorTypeDef) Deserialize(r io.Reader, scope uint64) (View, error) { +func (td *BasicVectorTypeDef) Deserialize(dr *codec.DecodingReader) (View, error) { + scope := dr.Scope() if td.Size != scope { return nil, fmt.Errorf("expected size %d does not match scope %d", td.Size, scope) } contents := make([]byte, scope, scope) - if _, err := r.Read(contents); err != nil { + if _, err := dr.Read(contents); err != nil { return nil, err } bottomNodes, err := BytesIntoNodes(contents) @@ -121,6 +122,17 @@ type BasicVectorView struct { *BasicVectorTypeDef } +func AsBasicVector(v View, err error) (*BasicVectorView, error) { + if err != nil { + return nil, err + } + bv, ok := v.(*BasicVectorView) + if !ok { + return nil, fmt.Errorf("view is not a basic vector: %v", v) + } + return bv, nil +} + func (tv *BasicVectorView) subviewNode(i uint64) (r *Root, bottomIndex uint64, subIndex uint8, err error) { bottomIndex, subIndex = tv.TranslateIndex(i) v, err := tv.SubtreeView.GetNode(bottomIndex) @@ -184,11 +196,10 @@ func (tv *BasicVectorView) ValueByteLength() (uint64, error) { return tv.Size, nil } -func (tv *BasicVectorView) Serialize(w io.Writer) error { +func (tv *BasicVectorView) Serialize(w *codec.EncodingWriter) error { contents := make([]byte, tv.Size, tv.Size) if err := SubtreeIntoBytes(tv.BackingNode, tv.depth, tv.BottomNodeLength(), contents); err != nil { return err } - _, err := w.Write(contents) - return err + return w.Write(contents) } diff --git a/view/bitfield_iter.go b/view/bitfield_iter.go index 5f889c2..fe2927d 100644 --- a/view/bitfield_iter.go +++ b/view/bitfield_iter.go @@ -5,7 +5,6 @@ import ( . "github.com/protolambda/ztyp/tree" ) - type ErrBitIter struct { error } @@ -14,7 +13,7 @@ func (e ErrBitIter) Next() (elem bool, ok bool, err error) { return false, false, e.error } -type BitIterFn func() (elem bool, ok bool, err error) +type BitIterFn func() (elem bool, ok bool, err error) func (f BitIterFn) Next() (elem bool, ok bool, err error) { return f() @@ -26,7 +25,10 @@ type BitIter interface { Next() (elem bool, ok bool, err error) } -func bitReadonlyIter(node Node, length uint64, depth uint8) BitIter { +func bitReadonlyIter(anchor Node, length uint64, depth uint8) BitIter { + if limit := (uint64(1) << depth) << 8; limit < length { + return ErrBitIter{fmt.Errorf("cannot handle iterate length %d bits in subtree of depth %d deep (limit %d bits)", length, depth, limit)} + } stack := make([]Node, depth, depth) i := uint64(0) @@ -41,19 +43,23 @@ func bitReadonlyIter(node Node, length uint64, depth uint8) BitIter { } // in the middle of a node currently? finish that first if j > 0 { - elByte := currentRoot[j >> 3] + elByte := currentRoot[j>>3] elem = ((elByte >> (j & 7)) & 1) == 1 + // overflow is a feature here: no more than 256 bits + // (255 here, 1 at index 0 at the entry of the bottom node) j += 1 + i += 1 return elem, true, nil } + var node Node stackIndex := uint8(0) if rootIndex != 0 { // XOR current index with previous index // Result: highest bit matches amount we have to backtrack up the stack s := rootIndex ^ (rootIndex - 1) - stackIndex = depth - 1 + stackIndex = depth for s != 0 { - s >>= 0 + s >>= 1 stackIndex -= 1 } // then move to the right from that upper previously remembered left-hand node @@ -64,19 +70,17 @@ func bitReadonlyIter(node Node, length uint64, depth uint8) BitIter { } stackIndex += 1 } else { - if depth != 0 { - stack[0] = node - } - stackIndex = 1 + node = anchor } // and move down left into this new subtree for ; stackIndex < depth; stackIndex++ { + // remember left-hand nodes, we may revisit them + stack[stackIndex] = node + node, err = node.Left() if err != nil { return false, false, err } - // remember left-hand nodes, we may revisit them - stack[stackIndex] = node } // Get leaf node as a root @@ -88,9 +92,14 @@ func bitReadonlyIter(node Node, length uint64, depth uint8) BitIter { currentRoot = *r // get the first bit - el := currentRoot[0] & 1 == 1 + el := currentRoot[0]&1 == 1 // indicate that we have done one bit, and need to read more j = 1 + // And that the next root will be 1 after + rootIndex += 1 + // And we progress the general element counter + i += 1 + // Return the actual element return el, true, nil }) diff --git a/view/bitlist.go b/view/bitlist.go index 2c639cf..23d3ffd 100644 --- a/view/bitlist.go +++ b/view/bitlist.go @@ -3,8 +3,8 @@ package view import ( "encoding/binary" "fmt" + "github.com/protolambda/ztyp/codec" . "github.com/protolambda/ztyp/tree" - "io" ) type BitListTypeDef struct { @@ -16,7 +16,7 @@ func BitListType(limit uint64) *BitListTypeDef { return &BitListTypeDef{ BitLimit: limit, ComplexTypeBase: ComplexTypeBase{ - MinSize: 1, // 1 byte, to do the 1 delimiting bit + MinSize: 1, // 1 byte, to do the 1 delimiting bit MaxSize: (limit + 7 + 1) / 8, Size: 0, IsFixedSize: false, @@ -57,10 +57,10 @@ func (td *BitListTypeDef) ViewFromBacking(node Node, hook BackingHook) (View, er ViewBase: ViewBase{ TypeDef: td, }, - Hook: hook, + Hook: hook, BackingNode: node, }, - depth: depth + 1, // +1 for length mix-in + depth: depth + 1, // +1 for length mix-in }, BitListTypeDef: td, }, nil @@ -79,21 +79,26 @@ func (td *BitListTypeDef) New() *BitListView { return td.Default(nil).(*BitListView) } -func (td *BitListTypeDef) Deserialize(r io.Reader, scope uint64) (View, error) { +func (td *BitListTypeDef) Deserialize(dr *codec.DecodingReader) (View, error) { + scope := dr.Scope() if scope == 0 { - return td.New(), nil + return nil, fmt.Errorf("expected at least a delimit bit, bitlist scope cannot be 0") } - if scope > td.Size { - return nil, fmt.Errorf("bitlist has too many bytes, bitlimit %d (byte size %d) but got scope %d", td.BitLimit, td.Size, scope) + if scope > td.MaxSize { + return nil, fmt.Errorf("bitlist has too many bytes, bitlimit %d (byte size %d) but got scope %d", td.BitLimit, td.MaxSize, scope) } contents := make([]byte, scope, scope) - if _, err := r.Read(contents); err != nil { + if _, err := dr.Read(contents); err != nil { return nil, err } lastByte := contents[scope-1] if lastByte == 0 { return nil, fmt.Errorf("bitlist last byte must not be zero, delimit bit is missing") } + if scope == 1 && lastByte == 1 { + // only a delimit bit, return empty bitlist + return td.New(), nil + } delimitBitIndex := ByteBitIndex(lastByte) bitLen := ((scope - 1) << 3) + delimitBitIndex if bitLen > td.BitLimit { @@ -140,12 +145,22 @@ func (td *BitListTypeDef) String() string { return fmt.Sprintf("Bitist[%d]", td.BitLimit) } - type BitListView struct { SubtreeView *BitListTypeDef } +func AsBitList(v View, err error) (*BitListView, error) { + if err != nil { + return nil, err + } + bv, ok := v.(*BitListView) + if !ok { + return nil, fmt.Errorf("view is not a bitlist: %v", v) + } + return bv, nil +} + func (tv *BitListView) Append(view BoolView) error { ll, err := tv.Length() if err != nil { @@ -339,7 +354,7 @@ func (tv *BitListView) ReadonlyIter() BitIter { return ErrBitIter{err} } // ignore length mixin in stack - return bitReadonlyIter(node, length, tv.depth - 1) + return bitReadonlyIter(node, length, tv.depth-1) } func (tv *BitListView) ValueByteLength() (uint64, error) { @@ -350,7 +365,7 @@ func (tv *BitListView) ValueByteLength() (uint64, error) { return (length + 7 + 1) / 8, nil } -func (tv *BitListView) Serialize(w io.Writer) error { +func (tv *BitListView) Serialize(w *codec.EncodingWriter) error { contentsAnchor, err := tv.BackingNode.Getter(LeftGindex) if err != nil { return err @@ -363,11 +378,10 @@ func (tv *BitListView) Serialize(w io.Writer) error { byteLength := (bitLength + 7 + 1) / 8 contents := make([]byte, byteLength, byteLength) // one less depth, ignore length mix-in. Iteration length without bitlist delimit bit. - if err := SubtreeIntoBytes(contentsAnchor, tv.depth - 1, (bitLength + 0xff) >> 8, contents); err != nil { + if err := SubtreeIntoBytes(contentsAnchor, tv.depth-1, (bitLength+0xff)>>8, contents); err != nil { return err } // Add delimit bit contents[byteLength-1] |= 1 << (bitLength & 7) - _, err = w.Write(contents) - return err + return w.Write(contents) } diff --git a/view/bitvector.go b/view/bitvector.go index 14d74fc..bcedac3 100644 --- a/view/bitvector.go +++ b/view/bitvector.go @@ -2,8 +2,8 @@ package view import ( "fmt" + "github.com/protolambda/ztyp/codec" . "github.com/protolambda/ztyp/tree" - "io" ) type BitVectorTypeDef struct { @@ -56,10 +56,10 @@ func (td *BitVectorTypeDef) ViewFromBacking(node Node, hook BackingHook) (View, ViewBase: ViewBase{ TypeDef: td, }, - Hook: hook, + Hook: hook, BackingNode: node, }, - depth: depth, + depth: depth, }, BitVectorTypeDef: td, }, nil @@ -78,17 +78,18 @@ func (td *BitVectorTypeDef) New() *BitVectorView { return td.Default(nil).(*BitVectorView) } -func (td *BitVectorTypeDef) Deserialize(r io.Reader, scope uint64) (View, error) { +func (td *BitVectorTypeDef) Deserialize(dr *codec.DecodingReader) (View, error) { + scope := dr.Scope() if td.Size != scope { return nil, fmt.Errorf("expected size %d does not match scope %d", td.Size, scope) } contents := make([]byte, scope, scope) - if _, err := r.Read(contents); err != nil { + if _, err := dr.Read(contents); err != nil { return nil, err } - if scope != 0 && td.BitLength & 7 != 0 { + if scope != 0 && td.BitLength&7 != 0 { last := contents[scope-1] - if last & byte((uint16(1) << (td.BitLength & 7)) - 1) != last { + if last&byte((uint16(1)<<(td.BitLength&7))-1) != last { return nil, fmt.Errorf("last bitvector byte %d has out of bounds bits set", last) } } @@ -111,6 +112,17 @@ type BitVectorView struct { *BitVectorTypeDef } +func AsBitVector(v View, err error) (*BitVectorView, error) { + if err != nil { + return nil, err + } + bv, ok := v.(*BitVectorView) + if !ok { + return nil, fmt.Errorf("view is not a bitvector: %v", v) + } + return bv, nil +} + func (tv *BitVectorView) subviewNode(i uint64) (r *Root, bottomIndex uint64, subIndex uint8, err error) { bottomIndex, subIndex = i>>8, uint8(i) v, err := tv.SubtreeView.GetNode(bottomIndex) @@ -146,25 +158,6 @@ func (tv *BitVectorView) Set(i uint64, v BoolView) error { return tv.SubtreeView.SetNode(bottomIndex, v.BackingFromBitfieldBase(r, subIndex)) } -// Shifts the bitvector contents to the right, clipping off the overflow. Only supported for small BitVectors. -func (tv *BitVectorView) ShRight(sh uint8) error { - if tv.BitLength > 8 { - return fmt.Errorf("shifting large bitvectors is unsupported") - } - v, err := tv.SubtreeView.GetNode(0) - if err != nil { - return err - } - r, ok := v.(*Root) - if !ok { - return fmt.Errorf("bitvector bottom node is not a root, cannot perform bitshift") - } - newRoot := *r - // Mask to clip off bits. - newRoot[0] = (newRoot[0] << sh) & ((1 << tv.BitLength) - 1) - return tv.SubtreeView.SetNode(0, &newRoot) -} - func (tv *BitVectorView) Copy() (View, error) { tvCopy := *tv tvCopy.Hook = nil @@ -195,11 +188,10 @@ func (tv *BitVectorView) ValueByteLength() (uint64, error) { return tv.Size, nil } -func (tv *BitVectorView) Serialize(w io.Writer) error { +func (tv *BitVectorView) Serialize(w *codec.EncodingWriter) error { contents := make([]byte, tv.Size, tv.Size) if err := SubtreeIntoBytes(tv.BackingNode, tv.depth, tv.BottomNodeLength(), contents); err != nil { return err } - _, err := w.Write(contents) - return err + return w.Write(contents) } diff --git a/view/complex.go b/view/complex.go index c8cabb0..92c2e6c 100644 --- a/view/complex.go +++ b/view/complex.go @@ -1,18 +1,17 @@ package view import ( - "encoding/binary" "fmt" + "github.com/protolambda/ztyp/codec" . "github.com/protolambda/ztyp/tree" - "io" ) const OffsetByteLength = 4 type ComplexTypeBase struct { - MinSize uint64 - MaxSize uint64 - Size uint64 + MinSize uint64 + MaxSize uint64 + Size uint64 IsFixedSize bool } @@ -80,7 +79,7 @@ func (e ErrNodeIter) Next() (chunk Node, ok bool, err error) { return nil, false, e.error } -type NodeIterFn func() (chunk Node, ok bool, err error) +type NodeIterFn func() (chunk Node, ok bool, err error) func (f NodeIterFn) Next() (chunk Node, ok bool, err error) { return f() @@ -100,7 +99,7 @@ func (e ErrElemIter) Next() (elem View, ok bool, err error) { return nil, false, e.error } -type ElemIterFn func() (elem View, ok bool, err error) +type ElemIterFn func() (elem View, ok bool, err error) func (f ElemIterFn) Next() (elem View, ok bool, err error) { return f() @@ -112,30 +111,7 @@ type ElemIter interface { Next() (elem View, ok bool, err error) } -func WriteOffset(w io.Writer, prevOffset uint64, elemLen uint64) (offset uint64, err error) { - if prevOffset >= (uint64(1) << 32) { - panic("cannot write offset with invalid previous offset") - } - if elemLen >= (uint64(1) << 32) { - panic("cannot write offset with invalid element size") - } - offset = prevOffset + elemLen - if offset >= (uint64(1) << 32) { - panic("offset too large, not uint32") - } - tmp := make([]byte, 4, 4) - binary.LittleEndian.PutUint32(tmp, uint32(offset)) - _, err = w.Write(tmp) - return -} - -func ReadOffset(r io.Reader) (uint32, error) { - tmp := make([]byte, 4, 4) - _, err := r.Read(tmp) - return binary.LittleEndian.Uint32(tmp), err -} - -func serializeComplexFixElemSeries(iter ElemIter, w io.Writer) error { +func serializeComplexFixElemSeries(iter ElemIter, w *codec.EncodingWriter) error { for { el, ok, err := iter.Next() if err != nil { @@ -151,15 +127,14 @@ func serializeComplexFixElemSeries(iter ElemIter, w io.Writer) error { return nil } -func serializeComplexVarElemSeries(length uint64, iter ElemIter, w io.Writer) error { - elements := make([]View, length, length) - +func serializeComplexVarElemSeries(length uint64, iterFn func() ElemIter, w *codec.EncodingWriter) error { // the previous offset, to calculate a new offset from, starting after the fixed data. prevOffset := length * OffsetByteLength // span of the previous var-size element prevSize := uint64(0) - // write all offsets, remember the elements + iter := iterFn() + // write all offsets for { el, ok, err := iter.Next() if err != nil { @@ -172,17 +147,23 @@ func serializeComplexVarElemSeries(length uint64, iter ElemIter, w io.Writer) er if err != nil { return err } - prevOffset, err = WriteOffset(w, prevOffset, prevSize) + prevOffset, err = w.WriteOffset(prevOffset, prevSize) if err != nil { return err } prevSize = elValSize - // Queue the actual element to be encoded after the fixed part of the container is encoded. - elements = append(elements, el) } + iter = iterFn() // now write all elements - for _, v := range elements { - if err := v.Serialize(w); err != nil { + for { + el, ok, err := iter.Next() + if err != nil { + return err + } + if !ok { + break + } + if err := el.Serialize(w); err != nil { return err } } diff --git a/view/complex_list.go b/view/complex_list.go index e9add3a..55f3880 100644 --- a/view/complex_list.go +++ b/view/complex_list.go @@ -3,8 +3,8 @@ package view import ( "encoding/binary" "fmt" + "github.com/protolambda/ztyp/codec" . "github.com/protolambda/ztyp/tree" - "io" ) type ComplexListTypeDef struct { @@ -32,7 +32,7 @@ func ComplexListType(elemType TypeDef, limit uint64) *ComplexListTypeDef { } } -func (td *ComplexListTypeDef) FromElements(v... View) (*ComplexListView, error) { +func (td *ComplexListTypeDef) FromElements(v ...View) (*ComplexListView, error) { if uint64(len(v)) > td.ListLimit { return nil, fmt.Errorf("expected no more than %d elements, got %d", td.ListLimit, len(v)) } @@ -87,7 +87,8 @@ func (td *ComplexListTypeDef) New() *ComplexListView { return td.Default(nil).(*ComplexListView) } -func (td *ComplexListTypeDef) Deserialize(r io.Reader, scope uint64) (View, error) { +func (td *ComplexListTypeDef) Deserialize(dr *codec.DecodingReader) (View, error) { + scope := dr.Scope() if scope == 0 { return td.New(), nil } @@ -97,12 +98,16 @@ func (td *ComplexListTypeDef) Deserialize(r io.Reader, scope uint64) (View, erro if length > td.ListLimit { return nil, fmt.Errorf("too many items, limit %d but got %d", td.ListLimit, length) } - if expected := length*elemSize; expected != scope { + if expected := length * elemSize; expected != scope { return nil, fmt.Errorf("scope %d does not align to elem size %d", scope, elemSize) } elements := make([]View, length, length) for i := uint64(0); i < length; i++ { - el, err := td.ElemType.Deserialize(r, elemSize) + sub, err := dr.SubScope(elemSize) + if err != nil { + return nil, err + } + el, err := td.ElemType.Deserialize(sub) if err != nil { return nil, err } @@ -110,11 +115,11 @@ func (td *ComplexListTypeDef) Deserialize(r io.Reader, scope uint64) (View, erro } return td.FromElements(elements...) } else { - firstOffset, err := ReadOffset(r) + firstOffset, err := dr.ReadOffset() if err != nil { return nil, err } - if firstOffset % OffsetByteLength != 0 { + if firstOffset%OffsetByteLength != 0 { return nil, fmt.Errorf("first offset %d does not align to offset length %d", firstOffset, OffsetByteLength) } length := uint64(firstOffset) / OffsetByteLength @@ -125,7 +130,7 @@ func (td *ComplexListTypeDef) Deserialize(r io.Reader, scope uint64) (View, erro offsets[0] = firstOffset prevOffset := firstOffset for i := uint64(1); i < length; i++ { - offset, err := ReadOffset(r) + offset, err := dr.ReadOffset() if err != nil { return nil, err } @@ -139,13 +144,21 @@ func (td *ComplexListTypeDef) Deserialize(r io.Reader, scope uint64) (View, erro lastIndex := uint32(len(elements) - 1) for i := uint32(0); i < lastIndex; i++ { size := offsets[i+1] - offsets[i] - el, err := td.ElemType.Deserialize(r, uint64(size)) + sub, err := dr.SubScope(uint64(size)) + if err != nil { + return nil, err + } + el, err := td.ElemType.Deserialize(sub) if err != nil { return nil, err } elements[i] = el } - el, err := td.ElemType.Deserialize(r, scope - uint64(offsets[lastIndex])) + sub, err := dr.SubScope(scope - uint64(offsets[lastIndex])) + if err != nil { + return nil, err + } + el, err := td.ElemType.Deserialize(sub) if err != nil { return nil, err } @@ -163,6 +176,17 @@ type ComplexListView struct { *ComplexListTypeDef } +func AsComplexList(v View, err error) (*ComplexListView, error) { + if err != nil { + return nil, err + } + c, ok := v.(*ComplexListView) + if !ok { + return nil, fmt.Errorf("view is not a list: %v", v) + } + return c, nil +} + func (tv *ComplexListView) Append(v View) error { ll, err := tv.Length() if err != nil { @@ -327,7 +351,7 @@ func (tv *ComplexListView) ReadonlyIter() ElemIter { return ErrElemIter{err} } // ignore length mixin in stack - return elemReadonlyIter(node, length, tv.depth - 1, tv.ElemType) + return elemReadonlyIter(node, length, tv.depth-1, tv.ElemType) } func (tv *ComplexListView) ValueByteLength() (uint64, error) { @@ -358,16 +382,14 @@ func (tv *ComplexListView) ValueByteLength() (uint64, error) { } } -func (tv *ComplexListView) Serialize(w io.Writer) error { - iter := tv.ReadonlyIter() +func (tv *ComplexListView) Serialize(w *codec.EncodingWriter) error { if tv.ElemType.IsFixedByteLength() { - return serializeComplexFixElemSeries(iter, w) + return serializeComplexFixElemSeries(tv.ReadonlyIter(), w) } else { length, err := tv.Length() if err != nil { return err } - return serializeComplexVarElemSeries(length, iter, w) + return serializeComplexVarElemSeries(length, tv.ReadonlyIter, w) } } - diff --git a/view/complex_vector.go b/view/complex_vector.go index 45760da..39a9f64 100644 --- a/view/complex_vector.go +++ b/view/complex_vector.go @@ -2,8 +2,8 @@ package view import ( "fmt" + "github.com/protolambda/ztyp/codec" . "github.com/protolambda/ztyp/tree" - "io" ) type ComplexVectorTypeDef struct { @@ -29,15 +29,15 @@ func ComplexVectorType(elemType TypeDef, length uint64) *ComplexVectorTypeDef { ElemType: elemType, VectorLength: length, ComplexTypeBase: ComplexTypeBase{ - MinSize: minSize, - MaxSize: maxSize, - Size: size, + MinSize: minSize, + MaxSize: maxSize, + Size: size, IsFixedSize: isFixedSize, }, } } -func (td *ComplexVectorTypeDef) FromElements(v... View) (*ComplexVectorView, error) { +func (td *ComplexVectorTypeDef) FromElements(v ...View) (*ComplexVectorView, error) { if td.VectorLength != uint64(len(v)) { return nil, fmt.Errorf("expected %d elements, got %d", td.VectorLength, len(v)) } @@ -76,10 +76,10 @@ func (td *ComplexVectorTypeDef) ViewFromBacking(node Node, hook BackingHook) (Vi ViewBase: ViewBase{ TypeDef: td, }, - Hook: hook, + Hook: hook, BackingNode: node, }, - depth: depth, + depth: depth, }, ComplexVectorTypeDef: td, }, nil @@ -94,7 +94,8 @@ func (td *ComplexVectorTypeDef) New() *ComplexVectorView { return td.Default(nil).(*ComplexVectorView) } -func (td *ComplexVectorTypeDef) Deserialize(r io.Reader, scope uint64) (View, error) { +func (td *ComplexVectorTypeDef) Deserialize(dr *codec.DecodingReader) (View, error) { + scope := dr.Scope() if td.IsFixedSize { elemSize := td.ElemType.TypeByteLength() if td.Size != scope { @@ -102,7 +103,11 @@ func (td *ComplexVectorTypeDef) Deserialize(r io.Reader, scope uint64) (View, er } elements := make([]View, td.VectorLength, td.VectorLength) for i := uint64(0); i < td.VectorLength; i++ { - el, err := td.ElemType.Deserialize(r, elemSize) + sub, err := dr.SubScope(elemSize) + if err != nil { + return nil, err + } + el, err := td.ElemType.Deserialize(sub) if err != nil { return nil, err } @@ -113,7 +118,7 @@ func (td *ComplexVectorTypeDef) Deserialize(r io.Reader, scope uint64) (View, er offsets := make([]uint32, td.VectorLength, td.VectorLength) prevOffset := uint32(0) for i := uint64(0); i < td.VectorLength; i++ { - offset, err := ReadOffset(r) + offset, err := dr.ReadOffset() if err != nil { return nil, err } @@ -127,13 +132,21 @@ func (td *ComplexVectorTypeDef) Deserialize(r io.Reader, scope uint64) (View, er lastIndex := uint32(len(elements) - 1) for i := uint32(0); i < lastIndex; i++ { size := offsets[i+1] - offsets[i] - el, err := td.ElemType.Deserialize(r, uint64(size)) + sub, err := dr.SubScope(uint64(size)) + if err != nil { + return nil, err + } + el, err := td.ElemType.Deserialize(sub) if err != nil { return nil, err } elements[i] = el } - el, err := td.ElemType.Deserialize(r, scope - uint64(offsets[lastIndex])) + sub, err := dr.SubScope(scope - uint64(offsets[lastIndex])) + if err != nil { + return nil, err + } + el, err := td.ElemType.Deserialize(sub) if err != nil { return nil, err } @@ -151,6 +164,17 @@ type ComplexVectorView struct { *ComplexVectorTypeDef } +func AsComplexVector(v View, err error) (*ComplexVectorView, error) { + if err != nil { + return nil, err + } + c, ok := v.(*ComplexVectorView) + if !ok { + return nil, fmt.Errorf("view is not a vector: %v", v) + } + return c, nil +} + func (tv *ComplexVectorView) Get(i uint64) (View, error) { if i >= tv.ComplexVectorTypeDef.VectorLength { return nil, fmt.Errorf("cannot get item at element index %d, vector only has %d elements", i, tv.ComplexVectorTypeDef.VectorLength) @@ -206,16 +230,31 @@ func (tv *ComplexVectorView) ReadonlyIter() ElemIter { func (tv *ComplexVectorView) ValueByteLength() (uint64, error) { if tv.IsFixedSize { return tv.Size, nil + } else { + size := tv.VectorLength * OffsetByteLength + iter := tv.ReadonlyIter() + for { + elem, ok, err := iter.Next() + if err != nil { + return 0, err + } + if !ok { + break + } + valSize, err := elem.ValueByteLength() + if err != nil { + return 0, err + } + size += valSize + } + return size, nil } - // TODO - return 0, nil } -func (tv *ComplexVectorView) Serialize(w io.Writer) error { - iter := tv.ReadonlyIter() +func (tv *ComplexVectorView) Serialize(w *codec.EncodingWriter) error { if tv.IsFixedSize { - return serializeComplexFixElemSeries(iter, w) + return serializeComplexFixElemSeries(tv.ReadonlyIter(), w) } else { - return serializeComplexVarElemSeries(tv.VectorLength, iter, w) + return serializeComplexVarElemSeries(tv.VectorLength, tv.ReadonlyIter, w) } } diff --git a/view/container.go b/view/container.go index d6fbfce..f59ed7d 100644 --- a/view/container.go +++ b/view/container.go @@ -3,8 +3,8 @@ package view import ( "bytes" "fmt" + "github.com/protolambda/ztyp/codec" . "github.com/protolambda/ztyp/tree" - "io" ) type FieldDef struct { @@ -15,8 +15,8 @@ type FieldDef struct { type ContainerTypeDef struct { ComplexTypeBase ContainerName string - Fields []FieldDef - OffsetsCount uint64 + Fields []FieldDef + OffsetsCount uint64 FixedPartSize uint64 } @@ -27,7 +27,10 @@ func ContainerType(name string, fields []FieldDef) *ContainerTypeDef { offsetsCount := uint64(0) for _, f := range fields { if f.Type.IsFixedByteLength() { - fixedPart += f.Type.TypeByteLength() + size := f.Type.TypeByteLength() + fixedPart += size + minSize += size + maxSize += size } else { offsetsCount += 1 fixedPart += OffsetByteLength @@ -38,25 +41,23 @@ func ContainerType(name string, fields []FieldDef) *ContainerTypeDef { size := uint64(0) isFixedSize := offsetsCount == 0 if isFixedSize { - minSize = fixedPart - maxSize = fixedPart size = fixedPart } return &ContainerTypeDef{ ComplexTypeBase: ComplexTypeBase{ - MinSize: minSize, - MaxSize: maxSize, - Size: size, + MinSize: minSize, + MaxSize: maxSize, + Size: size, IsFixedSize: isFixedSize, }, ContainerName: name, - Fields: fields, - OffsetsCount: offsetsCount, + Fields: fields, + OffsetsCount: offsetsCount, FixedPartSize: fixedPart, } } -func (td *ContainerTypeDef) FromFields(v... View) (*ContainerView, error) { +func (td *ContainerTypeDef) FromFields(v ...View) (*ContainerView, error) { if len(td.Fields) != len(v) { return nil, fmt.Errorf("expected %d fields, got %d", len(td.Fields), len(v)) } @@ -97,10 +98,10 @@ func (td *ContainerTypeDef) ViewFromBacking(node Node, hook BackingHook) (View, ViewBase: ViewBase{ TypeDef: td, }, - Hook: hook, + Hook: hook, BackingNode: node, }, - depth: depth, + depth: depth, }, ContainerTypeDef: td, }, nil @@ -120,28 +121,32 @@ func (td *ContainerTypeDef) FieldCount() uint64 { } type offsetField struct { - index int + index int offset uint32 } -func (td *ContainerTypeDef) Deserialize(r io.Reader, scope uint64) (View, error) { +func (td *ContainerTypeDef) Deserialize(dr *codec.DecodingReader) (View, error) { fields := make([]View, len(td.Fields), len(td.Fields)) - offsets := make([]offsetField, td.OffsetsCount, td.OffsetsCount) + offsets := make([]offsetField, 0, td.OffsetsCount) prevOffset := uint32(td.FixedPartSize) + scope := dr.Scope() if err := td.checkScope(scope); err != nil { return nil, err } // Deserialize the fixed part: fixed-size fields and offsets to dynamic fields for i, f := range td.Fields { if f.Type.IsFixedByteLength() { - // No need to redefine the scope for fixed-length SSZ objects. - v, err := f.Type.Deserialize(r, scope) + sub, err := dr.SubScope(f.Type.TypeByteLength()) + if err != nil { + return nil, err + } + v, err := f.Type.Deserialize(sub) if err != nil { return nil, err } fields[i] = v } else { - offset, err := ReadOffset(r) + offset, err := dr.ReadOffset() if err != nil { return nil, err } @@ -151,18 +156,23 @@ func (td *ContainerTypeDef) Deserialize(r io.Reader, scope uint64) (View, error) if uint64(offset) > scope { return nil, fmt.Errorf("offset %d of field %d is too big for scope %d", offset, i, scope) } + prevOffset = offset offsets = append(offsets, offsetField{index: i, offset: offset}) } } // Deserialize the dynamic part: for each offset, get the size and deserialize the element for i, item := range offsets { var size uint32 - if i + 1 == len(offsets) { + if i+1 == len(offsets) { size = uint32(scope) - item.offset } else { size = offsets[i+1].offset - item.offset } - v, err := td.Fields[item.index].Type.Deserialize(r, uint64(size)) + sub, err := dr.SubScope(uint64(size)) + if err != nil { + return nil, err + } + v, err := td.Fields[item.index].Type.Deserialize(sub) if err != nil { return nil, err } @@ -190,12 +200,22 @@ func (td *ContainerTypeDef) TypeRepr() string { return buf.String() } - type ContainerView struct { SubtreeView *ContainerTypeDef } +func AsContainer(v View, err error) (*ContainerView, error) { + if err != nil { + return nil, err + } + c, ok := v.(*ContainerView) + if !ok { + return nil, fmt.Errorf("view is not a container: %v", v) + } + return c, nil +} + func (tv *ContainerView) Copy() (View, error) { tvCopy := *tv tvCopy.Hook = nil @@ -225,9 +245,9 @@ func (tv *ContainerView) ValueByteLength() (uint64, error) { return out, nil } -func (tv *ContainerView) Serialize(w io.Writer) error { +func (tv *ContainerView) Serialize(w *codec.EncodingWriter) error { fieldIter := tv.ReadonlyIter() - var dynFields []View + var dynFields []View if !tv.IsFixedSize { dynFields = make([]View, 0, tv.OffsetsCount) } @@ -254,7 +274,7 @@ func (tv *ContainerView) Serialize(w io.Writer) error { if err != nil { return err } - prevOffset, err = WriteOffset(w, prevOffset, prevSize) + prevOffset, err = w.WriteOffset(prevOffset, prevSize) if err != nil { return err } @@ -288,6 +308,24 @@ func (tv *ContainerView) Iter() ElemIter { }) } +func (tv *ContainerView) FieldValues() ([]View, error) { + values := make([]View, len(tv.Fields), len(tv.Fields)) + iter := tv.ReadonlyIter() + i := 0 + for { + el, ok, err := iter.Next() + if err != nil { + return nil, err + } + if !ok { + break + } + values[i] = el + i++ + } + return values, nil +} + func (tv *ContainerView) ReadonlyIter() ElemIter { return fieldReadonlyIter(tv.BackingNode, tv.depth, tv.Fields) } diff --git a/view/defs.go b/view/defs.go index 0dbd37b..4f7294c 100644 --- a/view/defs.go +++ b/view/defs.go @@ -1,8 +1,8 @@ package view import ( + "github.com/protolambda/ztyp/codec" . "github.com/protolambda/ztyp/tree" - "io" ) type View interface { @@ -10,7 +10,7 @@ type View interface { SetBacking(b Node) error Copy() (View, error) ValueByteLength() (uint64, error) - Serialize(w io.Writer) error + Serialize(w *codec.EncodingWriter) error HashTreeRoot(h HashFn) Root Type() TypeDef } @@ -42,7 +42,7 @@ type TypeDef interface { TypeByteLength() uint64 MinByteLength() uint64 MaxByteLength() uint64 - Deserialize(r io.Reader, scope uint64) (View, error) + Deserialize(dr *codec.DecodingReader) (View, error) String() string // TODO: could add navigation by key/index into subtypes } diff --git a/view/iter.go b/view/iter.go index b272417..1ffda35 100644 --- a/view/iter.go +++ b/view/iter.go @@ -5,16 +5,17 @@ import ( . "github.com/protolambda/ztyp/tree" ) -func basicElemReadonlyIter(node Node, length uint64, depth uint8, elemType BasicTypeDef) ElemIter { - stack := make([]Node, depth, depth) - +func basicElemReadonlyIter(anchor Node, length uint64, depth uint8, elemType BasicTypeDef) ElemIter { i := uint64(0) // max 32 elements per bottom nodes, uint8 is safe. perNode := 32 / uint8(elemType.TypeByteLength()) - j := uint8(i) % perNode - if j == 0 { - j = perNode + j := perNode + + if limit := (uint64(1) << depth) * uint64(perNode); limit < length { + return ErrElemIter{fmt.Errorf("cannot handle iterate length %d bottom subviews (%d per node) in subtree of depth %d deep (limit %d subviews)", length, perNode, depth, limit)} } + + stack := make([]Node, depth, depth) var currentRoot *Root rootIndex := uint64(0) return ElemIterFn(func() (elem View, ok bool, err error) { @@ -25,20 +26,24 @@ func basicElemReadonlyIter(node Node, length uint64, depth uint8, elemType Basic // in the middle of a node currently? finish that first if j < perNode { el, err := elemType.BasicViewFromBacking(currentRoot, j) - j += 1 if err != nil { return nil, false, err } + // progress how many nodes we have seen so far in this bottom chunk + j += 1 + // progress the general element counter, + i += 1 return el, true, nil } + var node Node stackIndex := uint8(0) if rootIndex != 0 { // XOR current index with previous index // Result: highest bit matches amount we have to backtrack up the stack s := rootIndex ^ (rootIndex - 1) - stackIndex = depth - 1 + stackIndex = depth for s != 0 { - s >>= 0 + s >>= 1 stackIndex -= 1 } // then move to the right from that upper previously remembered left-hand node @@ -49,19 +54,17 @@ func basicElemReadonlyIter(node Node, length uint64, depth uint8, elemType Basic } stackIndex += 1 } else { - if depth != 0 { - stack[0] = node - } - stackIndex = 1 + node = anchor } // and move down left into this new subtree for ; stackIndex < depth; stackIndex++ { + // remember left-hand nodes, we may revisit them + stack[stackIndex] = node + node, err = node.Left() if err != nil { return nil, false, err } - // remember left-hand nodes, we may revisit them - stack[stackIndex] = node } // Get leaf node as a root @@ -79,29 +82,37 @@ func basicElemReadonlyIter(node Node, length uint64, depth uint8, elemType Basic } // indicate that we have done one subview, and may need more to be read. Next one would be index 1, if any. j = 1 + // And that the next root will be 1 after + rootIndex += 1 + // And we progress the general element counter + i += 1 + // Return the actual element return el, true, nil }) } -func nodeReadonlyIter(node Node, length uint64, depth uint8) NodeIter { +func nodeReadonlyIter(anchor Node, length uint64, depth uint8) NodeIter { + if limit := uint64(1) << depth; limit < length { + return ErrNodeIter{fmt.Errorf("cannot handle iterate length %d nodes in subtree of depth %d deep (limit %d)", length, depth, limit)} + } stack := make([]Node, depth, depth) i := uint64(0) - rootIndex := uint64(0) return NodeIterFn(func() (chunk Node, ok bool, err error) { // done yet? if i >= length { return nil, false, nil } + var node Node stackIndex := uint8(0) - if rootIndex != 0 { + if i != 0 { // XOR current index with previous index // Result: highest bit matches amount we have to backtrack up the stack - s := rootIndex ^ (rootIndex - 1) - stackIndex = depth - 1 + s := i ^ (i - 1) + stackIndex = depth for s != 0 { - s >>= 0 + s >>= 1 stackIndex -= 1 } // then move to the right from that upper previously remembered left-hand node @@ -112,20 +123,19 @@ func nodeReadonlyIter(node Node, length uint64, depth uint8) NodeIter { } stackIndex += 1 } else { - if depth != 0 { - stack[0] = node - } - stackIndex = 1 + node = anchor } // and move down left into this new subtree for ; stackIndex < depth; stackIndex++ { + // remember left-hand nodes, we may revisit them + stack[stackIndex] = node + node, err = node.Left() if err != nil { return nil, false, err } - // remember left-hand nodes, we may revisit them - stack[stackIndex] = node } + i += 1 // Return the actual element return node, true, nil @@ -134,6 +144,8 @@ func nodeReadonlyIter(node Node, length uint64, depth uint8) NodeIter { func elemReadonlyIter(node Node, length uint64, depth uint8, elemType TypeDef) ElemIter { nodeIter := nodeReadonlyIter(node, length, depth) + // Re-use a typed view by changing its backing each iteration step. + elemView := elemType.Default(nil) return ElemIterFn(func() (elem View, ok bool, err error) { node, ok, err := nodeIter.Next() if err != nil { @@ -142,12 +154,17 @@ func elemReadonlyIter(node Node, length uint64, depth uint8, elemType TypeDef) E if !ok { return nil, false, nil } - el, err := elemType.ViewFromBacking(node, nil) - if err != nil { - return nil, false, err + if err := elemView.SetBacking(node); err != nil { + // Create a new view if we can't re-use the existing view. + el, err := elemType.ViewFromBacking(node, nil) + if err != nil { + return nil, false, err + } + // Return the actual element + return el, true, nil + } else { + return elemView, true, nil } - // Return the actual element - return el, true, nil }) } diff --git a/view/iter_test.go b/view/iter_test.go new file mode 100644 index 0000000..7b3059f --- /dev/null +++ b/view/iter_test.go @@ -0,0 +1,70 @@ +package view + +import ( + "fmt" + . "github.com/protolambda/ztyp/tree" + "testing" +) + +type iterTestCase struct { + node Node + dOffset uint8 + length uint64 + depth uint8 + expected []Node +} + +func TestNodeReadonlyIter(t *testing.T) { + maxLength := 100 + treeEnd := maxLength * 2 + nodes := make([]Node, treeEnd, treeEnd) + for i := maxLength; i < treeEnd; i++ { + nodes[i] = &Root{uint8(i)} + } + for i := maxLength - 1; i > 0; i-- { + nodes[i] = NewPairNode(nodes[i*2], nodes[i*2+1]) + } + testCases := make([]iterTestCase, 0) + depth := CoverDepth(uint64(maxLength)) + + for d := uint8(0); d < depth; d++ { + for dOffset := uint8(0); dOffset < (depth - d); dOffset++ { + top := uint64(1) << dOffset + bottomStart := uint64(1) << (dOffset + d) + maxLenAtBottom := uint64(1) << (d - dOffset) + for l := uint64(0); l <= maxLenAtBottom; l++ { + testCases = append(testCases, iterTestCase{ + node: nodes[top], + dOffset: dOffset, + length: l, + depth: d, + expected: nodes[bottomStart : bottomStart+l], + }) + } + } + } + hFn := GetHashFn() + nodes[1].MerkleRoot(hFn) + + for _, testCase := range testCases { + t.Run(fmt.Sprintf("case_dep_%d_len_%d_off_%d", testCase.depth, testCase.length, testCase.dOffset), func(t *testing.T) { + iter := nodeReadonlyIter(testCase.node, testCase.length, testCase.depth) + for i := uint64(0); i < testCase.length; i++ { + el, ok, err := iter.Next() + if err != nil { + t.Fatalf("unexpected err on %d: %v", i, err) + } + if !ok { + t.Fatalf("unexpected stop on %d: %v", i, err) + } + if el.MerkleRoot(hFn) != testCase.expected[i].MerkleRoot(hFn) { + t.Fatalf("expected node is different on %d", i) + } + } + el, ok, err := iter.Next() + if el != nil || ok || err != nil { + t.Fatal("expected end") + } + }) + } +} diff --git a/view/root.go b/view/root.go index 63b585e..1f7128a 100644 --- a/view/root.go +++ b/view/root.go @@ -2,8 +2,9 @@ package view import ( "fmt" + "github.com/protolambda/ztyp/codec" + "github.com/protolambda/ztyp/conv" . "github.com/protolambda/ztyp/tree" - "io" ) type RootMeta uint8 @@ -28,12 +29,9 @@ func (m RootMeta) MaxByteLength() uint64 { return 32 } -func (m RootMeta) Deserialize(r io.Reader, scope uint64) (View, error) { - if scope < 32 { - return nil, fmt.Errorf("scope of %d not enough for root", scope) - } +func (m RootMeta) Deserialize(dr *codec.DecodingReader) (View, error) { v := RootView{} - _, err := r.Read(v[:]) + _, err := dr.Read(v[:]) return &v, err } @@ -62,6 +60,17 @@ const RootType RootMeta = 0 type RootView Root +func AsRoot(v View, err error) (Root, error) { + if err != nil { + return Root{}, err + } + r, ok := v.(*RootView) + if !ok { + return Root{}, fmt.Errorf("not a root view: %v", v) + } + return Root(*r), nil +} + func (r *RootView) Type() TypeDef { return RootType } @@ -72,7 +81,12 @@ func (r *RootView) Backing() Node { } func (r *RootView) SetBacking(b Node) error { - return NavigationError + if br, ok := b.(*Root); ok { + *r = RootView(*br) + return nil + } else { + return NavigationError + } } func (r *RootView) Copy() (View, error) { @@ -83,11 +97,22 @@ func (r *RootView) ValueByteLength() (uint64, error) { return 32, nil } -func (r *RootView) Serialize(w io.Writer) error { - _, err := w.Write(r[:]) - return err +func (r *RootView) Serialize(w *codec.EncodingWriter) error { + return w.Write(r[:]) } func (r *RootView) HashTreeRoot(h HashFn) Root { return Root(*r) } + +func (r *RootView) MarshalText() ([]byte, error) { + return conv.BytesMarshalText(r[:]) +} + +func (r *RootView) UnmarshalText(text []byte) error { + return conv.FixedBytesUnmarshalText(r[:], text) +} + +func (r *RootView) String() string { + return conv.BytesString(r[:]) +} diff --git a/view/root_test.go b/view/root_test.go new file mode 100644 index 0000000..4040735 --- /dev/null +++ b/view/root_test.go @@ -0,0 +1,56 @@ +package view + +import "testing" + +func TestRootView_MarshalText(t *testing.T) { + r := RootView{0: 0xab, 31: 0x12} + s, err := r.MarshalText() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if string(s) != "0xab00000000000000000000000000000000000000000000000000000000000012" { + t.Errorf("unexpected string: %s", string(s)) + } +} + +func TestRootView_UnmarshalText(t *testing.T) { + cases := []struct { + r RootView + s string + }{ + {RootView{0: 0xab, 31: 0x12}, "ab00000000000000000000000000000000000000000000000000000000000012"}, + {RootView{0: 0xab, 31: 0x12}, "0Xab00000000000000000000000000000000000000000000000000000000000012"}, + {RootView{0: 0xab, 31: 0x12}, "0xab00000000000000000000000000000000000000000000000000000000000012"}, + {RootView{}, "0000000000000000000000000000000000000000000000000000000000000000"}, + {RootView{}, "0x0000000000000000000000000000000000000000000000000000000000000000"}, + } + for _, c := range cases { + t.Run(c.s, func(t *testing.T) { + var root RootView + if err := root.UnmarshalText([]byte(c.s)); err != nil { + t.Errorf("unexpected error: %v", err) + } + if root != c.r { + t.Errorf("unexpected root: %x", root[:]) + } + }) + } + bad := []string{ + // no quotes. It's marshalText, not raw JSON. + `""`, `"0x0000000000000000000000000000000000000000000000000000000000000000"`, + "00000000000000000000000000000000000000000000000000000000000000000", // 1 zero extra + "00x00000000000000000000000000000000000000000000000000000000000000000", + "00x0000000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000000000000000000000000000000000000x", + "0x000x00000000000000000000000000000000000000000000000000000000000000", + "0x0g00000000000000000000000000000000000000000000000000000000000012", + } + for _, b := range bad { + t.Run(b, func(t *testing.T) { + var root RootView + if err := root.UnmarshalText([]byte(b)); err == nil { + t.Errorf("expected error, but got root %x", root[:]) + } + }) + } +} diff --git a/view/small_byte_vec.go b/view/small_byte_vec.go index e1709d6..1327a78 100644 --- a/view/small_byte_vec.go +++ b/view/small_byte_vec.go @@ -3,8 +3,9 @@ package view import ( "errors" "fmt" + "github.com/protolambda/ztyp/codec" + "github.com/protolambda/ztyp/conv" . "github.com/protolambda/ztyp/tree" - "io" ) // To represent views of < 32 bytes efficiently as just a slice of those bytes. @@ -53,12 +54,9 @@ func (td SmallByteVecMeta) MaxByteLength() uint64 { return uint64(td) } -func (td SmallByteVecMeta) Deserialize(r io.Reader, scope uint64) (View, error) { - if scope < uint64(td) { - return nil, fmt.Errorf("scope of %d not enough for small byte vec of %d bytes", scope, td) - } +func (td SmallByteVecMeta) Deserialize(dr *codec.DecodingReader) (View, error) { v := make(SmallByteVecView, td, td) - _, err := r.Read(v) + _, err := dr.Read(v) return v, err } @@ -68,6 +66,17 @@ func (td SmallByteVecMeta) String() string { type SmallByteVecView []byte +func AsSmallByteVec(v View, err error) (SmallByteVecView, error) { + if err != nil { + return nil, err + } + data, ok := v.(SmallByteVecView) + if !ok { + return nil, fmt.Errorf("not a small byte vec view: %v", v) + } + return data, nil +} + func (v SmallByteVecView) SetBacking(b Node) error { return errors.New("cannot set backing of SmallByteVecView") } @@ -86,9 +95,8 @@ func (v SmallByteVecView) ValueByteLength() (uint64, error) { return uint64(len(v)), nil } -func (v SmallByteVecView) Serialize(w io.Writer) error { - _, err := w.Write(v) - return err +func (v SmallByteVecView) Serialize(w *codec.EncodingWriter) error { + return w.Write(v) } func (v SmallByteVecView) HashTreeRoot(h HashFn) Root { @@ -101,6 +109,62 @@ func (v SmallByteVecView) Type() TypeDef { return SmallByteVecMeta(len(v)) } +func (v SmallByteVecView) MarshalText() ([]byte, error) { + return conv.BytesMarshalText(v) +} + +func (v SmallByteVecView) UnmarshalText(text []byte) error { + return conv.FixedBytesUnmarshalText(v, text) +} + +func (v SmallByteVecView) String() string { + return conv.BytesString(v) +} + const Bytes4Type SmallByteVecMeta = 4 const Bytes8Type SmallByteVecMeta = 8 const Bytes16Type SmallByteVecMeta = 16 + +// Go could use generics... + +func AsBytes4(v View, err error) ([4]byte, error) { + const byteLen = 4 + data, err := AsSmallByteVec(v, err) + if err != nil { + return [byteLen]byte{}, err + } + if len(data) != byteLen { + return [byteLen]byte{}, fmt.Errorf("expected %d byte long small byte vec, got %d byte long", byteLen, len(data)) + } + var out [byteLen]byte + copy(out[:], data) + return out, nil +} + +func AsBytes8(v View, err error) ([8]byte, error) { + const byteLen = 8 + data, err := AsSmallByteVec(v, err) + if err != nil { + return [byteLen]byte{}, err + } + if len(data) != byteLen { + return [byteLen]byte{}, fmt.Errorf("expected %d byte long small byte vec, got %d byte long", byteLen, len(data)) + } + var out [byteLen]byte + copy(out[:], data) + return out, nil +} + +func AsBytes16(v View, err error) ([16]byte, error) { + const byteLen = 16 + data, err := AsSmallByteVec(v, err) + if err != nil { + return [byteLen]byte{}, err + } + if len(data) != byteLen { + return [byteLen]byte{}, fmt.Errorf("expected %d byte long small byte vec, got %d byte long", byteLen, len(data)) + } + var out [byteLen]byte + copy(out[:], data) + return out, nil +} diff --git a/view/subtree.go b/view/subtree.go index 1c18394..ba7f6e2 100644 --- a/view/subtree.go +++ b/view/subtree.go @@ -7,7 +7,7 @@ import ( type SubtreeView struct { BackedView - depth uint8 + depth uint8 } func (stv *SubtreeView) GetNode(i uint64) (Node, error) { diff --git a/view/u256.go b/view/u256.go new file mode 100644 index 0000000..6674c2d --- /dev/null +++ b/view/u256.go @@ -0,0 +1,164 @@ +package view + +import ( + "encoding/binary" + "fmt" + "github.com/holiman/uint256" + "github.com/protolambda/ztyp/codec" + "github.com/protolambda/ztyp/conv" + . "github.com/protolambda/ztyp/tree" + "math/big" + "strconv" +) + +type Uint256View uint256.Int + +func AsUint256(v View, err error) (Uint256View, error) { + if err != nil { + return Uint256View{}, err + } + n, ok := v.(Uint256View) + if !ok { + return Uint256View{}, fmt.Errorf("not a uint256 view: %v", v) + } + return n, nil +} + +func (v Uint256View) SetBacking(b Node) error { + return BasicViewNoSetBackingError +} + +// Bytes32 returns little endian encoding +func (v Uint256View) Bytes32() (out [32]byte) { + // uint64 3 is most significant internally in uint256.Int + binary.LittleEndian.PutUint64(out[0:8], v[0]) + binary.LittleEndian.PutUint64(out[8:16], v[1]) + binary.LittleEndian.PutUint64(out[16:24], v[2]) + binary.LittleEndian.PutUint64(out[24:32], v[3]) + return +} + +// SetBytes32 sets view from little endian encoding +func (v *Uint256View) SetBytes32(data [32]byte) { + v.setBytes32(data[:]) +} + +func (v *Uint256View) setBytes32(data []byte) { + v[0] = binary.LittleEndian.Uint64(data[0:8]) + v[1] = binary.LittleEndian.Uint64(data[8:16]) + v[2] = binary.LittleEndian.Uint64(data[16:24]) + v[3] = binary.LittleEndian.Uint64(data[24:32]) +} + +// Bytes returns little endian encoding (always 32 bytes) +func (v Uint256View) Bytes() []byte { + out := v.Bytes32() + return out[:] +} + +func (v Uint256View) Backing() Node { + out := Root(v.Bytes32()) + return &out +} + +func (v Uint256View) BackingFromBase(base *Root, i uint8) *Root { + if i != 0 { // must always be aligned, we overwrite full 32 bytes here, nothing to pack along in same root. + return nil + } + newRoot := Root(v.Bytes32()) + return &newRoot +} + +func (v Uint256View) Copy() (View, error) { + return v, nil +} + +func (v Uint256View) ValueByteLength() (uint64, error) { + return 32, nil +} + +func (v Uint256View) ByteLength() uint64 { + return 32 +} + +func (v Uint256View) FixedLength() uint64 { + return 32 +} + +func (v Uint256View) Serialize(w *codec.EncodingWriter) error { + return w.Write(v.Bytes()) +} + +func (v Uint256View) Encode() ([]byte, error) { + return v.Bytes(), nil +} + +func (v *Uint256View) Deserialize(r *codec.DecodingReader) error { + var data [32]byte + _, err := r.Read(data[:]) + if err != nil { + return err + } + v.SetBytes32(data) + return nil +} + +func (v *Uint256View) Decode(x []byte) error { + if len(x) != 32 { + return BadLengthError + } + v.setBytes32(x) + return nil +} + +func (v Uint256View) HashTreeRoot(h HashFn) Root { + return v.Bytes32() +} + +func (v Uint256View) Type() TypeDef { + return Uint256Type +} + +func (v Uint256View) MarshalText() (out []byte, err error) { + return []byte(fmt.Sprintf("%d", (*uint256.Int)(&v))), nil +} + +func (v *Uint256View) SetFromBig(x *big.Int) (overflow bool) { + if x.Sign() < 0 { + return true + } + return (*uint256.Int)(v).SetFromBig(x) +} + +func (v *Uint256View) UnmarshalText(b []byte) error { + x := new(big.Int) + err := x.UnmarshalText(b) + if err != nil { + return fmt.Errorf("failed to unmarshal Uint256View: %w", err) + } + if v.SetFromBig(x) { + return strconv.ErrRange + } + return nil +} + +func (v Uint256View) MarshalJSON() ([]byte, error) { + return conv.Uint256Marshal((*uint256.Int)(&v)) +} + +func (v *Uint256View) UnmarshalJSON(b []byte) error { + return conv.Uint256Unmarshal((*uint256.Int)(v), b) +} + +func (v Uint256View) String() string { + return fmt.Sprintf("%d", (*uint256.Int)(&v)) +} + +func MustUint256(v string) Uint256View { + var out Uint256View + err := out.UnmarshalText([]byte(v)) + if err != nil { + panic(err) + } + return out +} diff --git a/view/union.go b/view/union.go new file mode 100644 index 0000000..b70b6a0 --- /dev/null +++ b/view/union.go @@ -0,0 +1,246 @@ +package view + +import ( + "bytes" + "fmt" + "github.com/protolambda/ztyp/codec" + . "github.com/protolambda/ztyp/tree" +) + +type UnionTypeDef struct { + ComplexTypeBase + // a "None" option is just a nil. + Options []TypeDef +} + +func UnionType(options []TypeDef) *UnionTypeDef { + if len(options) == 0 { + panic("union requires at least 1 option") + } + minSize := uint64(0) + maxSize := uint64(0) + + if options[0] != nil { + minSize = options[0].MinByteLength() + maxSize = options[0].MaxByteLength() + } + + for i, t := range options[1:] { + if t == nil { + panic(fmt.Errorf("union type option %d should not be nil, only option 0 can be nil", i)) + } + min, max := t.MinByteLength(), t.MaxByteLength() + if min < minSize { + minSize = min + } + if max > maxSize { + maxSize = max + } + } + return &UnionTypeDef{ + ComplexTypeBase: ComplexTypeBase{ + // Add the selector length + MinSize: minSize + 1, + MaxSize: maxSize + 1, + Size: 0, + IsFixedSize: false, + }, + Options: options, + } +} + +func (td *UnionTypeDef) FromView(selector uint8, v View) (*UnionView, error) { + var selectorNode Root + selectorNode[0] = selector + var content Node + if v == nil { + content = new(Root) + } else { + content = v.Backing() + } + rootNode := NewPairNode(content, &selectorNode) + conView, err := td.ViewFromBacking(rootNode, nil) + if err != nil { + return nil, err + } + return conView.(*UnionView), nil +} + +func (td *UnionTypeDef) DefaultNode() Node { + return NewPairNode(td.Options[0].DefaultNode(), new(Root)) +} + +func (td *UnionTypeDef) ViewFromBacking(node Node, hook BackingHook) (View, error) { + return &UnionView{ + BackedView: BackedView{ + ViewBase: ViewBase{ + TypeDef: td, + }, + Hook: hook, + BackingNode: node, + }, + UnionTypeDef: td, + }, nil +} + +func (td *UnionTypeDef) Default(hook BackingHook) View { + v, _ := td.ViewFromBacking(td.DefaultNode(), hook) + return v +} + +func (td *UnionTypeDef) New() *UnionView { + return td.Default(nil).(*UnionView) +} + +func (td *UnionTypeDef) Deserialize(dr *codec.DecodingReader) (View, error) { + scope := dr.Scope() + if scope == 0 { + return nil, fmt.Errorf("scope must be non-zero to deserialize union") + } + selector, err := dr.ReadByte() + if err != nil { + return nil, fmt.Errorf("failed to read selector: %v", err) + } + if selector >= uint8(len(td.Options)) { + return nil, fmt.Errorf("type selector is too large: %d (%d options)", selector, len(td.Options)) + } + option := td.Options[selector] + if option == nil { + return td.FromView(selector, nil) + } + subView, err := option.Deserialize(dr) + if err != nil { + return nil, fmt.Errorf("failed to decode union element (selector %d, scope %d): %v", selector, scope, err) + } + return td.FromView(selector, subView) +} + +func (td *UnionTypeDef) String() string { + return td.TypeRepr() +} + +func (td *UnionTypeDef) TypeRepr() string { + var buf bytes.Buffer + buf.WriteString("Union[") + for _, f := range td.Options { + buf.WriteString(f.String()) + buf.WriteString(", ") + } + buf.WriteRune(']') + return buf.String() +} + +type UnionView struct { + BackedView + *UnionTypeDef +} + +func AsUnion(v View, err error) (*UnionView, error) { + if err != nil { + return nil, err + } + c, ok := v.(*UnionView) + if !ok { + return nil, fmt.Errorf("view is not a union: %v", v) + } + return c, nil +} + +func (tv *UnionView) Copy() (View, error) { + tvCopy := *tv + tvCopy.Hook = nil + return &tvCopy, nil +} + +func (tv *UnionView) Selector() (uint8, error) { + selectorNode, err := tv.BackingNode.Right() + if err != nil { + return 0, fmt.Errorf("union selector could not be read: %v", err) + } + root, ok := selectorNode.(*Root) + if !ok { + return 0, fmt.Errorf("expected Root node for union selector, but got type %T", selectorNode) + } + for i := 1; i < 32; i++ { + if root[i] != 0 { + return 0, fmt.Errorf("union selector node has invalid value: %x", root[:]) + } + } + if root[0] >= uint8(len(tv.Options)) { + return 0, fmt.Errorf("out of range selector: %v (%d options)", root[0], len(tv.Options)) + } + return root[0], nil +} + +// Return the value. May be nil if it's a Union[None, T...] +func (tv *UnionView) Value() (View, error) { + selector, err := tv.Selector() + if err != nil { + return nil, err + } + content, err := tv.BackingNode.Left() + if err != nil { + return nil, fmt.Errorf("could not access union content node: %v", err) + } + option := tv.Options[selector] + if option == nil { + return nil, nil + } + return option.ViewFromBacking(content, nil) +} + +func (tv *UnionView) ValueByteLength() (uint64, error) { + contentView, err := tv.Value() + if err != nil { + return 0, fmt.Errorf("failed to interpret content") + } + if contentView == nil { + return 1, nil + } + contentSize, err := contentView.ValueByteLength() + // add 1 for the selector byte + return contentSize + 1, err +} + +func (tv *UnionView) Serialize(w *codec.EncodingWriter) error { + selector, err := tv.Selector() + if err != nil { + return err + } + if err := w.WriteByte(selector); err != nil { + return fmt.Errorf("failed to write selector: %v", err) + } + content, err := tv.BackingNode.Left() + if err != nil { + return fmt.Errorf("could not access union content node: %v", err) + } + option := tv.Options[selector] + if option == nil { + return nil + } + v, err := option.ViewFromBacking(content, nil) + if err != nil { + return fmt.Errorf("invalid value content node: %v", err) + } + return v.Serialize(w) +} + +// Changes the union selector and value. Does not check the view type. +// The value may be nil in the Union[None, T...] case. +func (tv *UnionView) Change(selector uint8, value View) error { + if selector >= uint8(len(tv.Options)) { + return fmt.Errorf("out of range selector: %v (%d options)", selector, len(tv.Options)) + } + var selectorNode Root + selectorNode[0] = selector + var contentNode Node + if value == nil { + if selector != 0 { + return fmt.Errorf("only the 0 selector can be used for nil values, got %d", selector) + } + contentNode = new(Root) + } else { + contentNode = value.Backing() + } + return tv.BackedView.SetBacking(NewPairNode(&selectorNode, contentNode)) +} diff --git a/view/view_test.go b/view/view_test.go index e024e7d..b9410f8 100644 --- a/view/view_test.go +++ b/view/view_test.go @@ -5,7 +5,9 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "github.com/protolambda/ztyp/codec" "github.com/protolambda/ztyp/tree" + "reflect" "strings" "testing" ) @@ -35,7 +37,7 @@ var ComplexTestStructType = ContainerType("ComplexTestStruct", []FieldDef{ {Name: "A", Type: Uint16Type}, {Name: "B", Type: ListType(Uint16Type, 128)}, {Name: "C", Type: Uint8Type}, - {Name: "D", Type: ListType(ByteType, 64)}, + {Name: "D", Type: ListType(ByteType, 256)}, {Name: "E", Type: VarTestStructType}, {Name: "F", Type: VectorType(FixedTestStructType, 4)}, {Name: "G", Type: VectorType(VarTestStructType, 2)}, @@ -51,8 +53,7 @@ var ListStructType = ContainerType("ComplexTestStruct", []FieldDef{ var SigType = BasicVectorType(ByteType, 96) var sigBytes = [96]byte{0: 1, 32: 2, 64: 3, 95: 0xff} -var SigView, _ = SigType.Deserialize(bytes.NewReader(sigBytes[:]), 96) - +var SigView, _ = SigType.Deserialize(codec.NewDecodingReader(bytes.NewReader(sigBytes[:]), 96)) func chunk(v string) string { res := [32]byte{} @@ -145,7 +146,7 @@ func init() { testCases = []sszTestCase{ {"bool F", BoolView(false), "00", chunk("00")}, {"bool T", BoolView(true), "01", chunk("01")}, - {"bitlist empty", BitListType(8).New(), "01", h(chunk(""), chunk("00"))}, + {"bitlist empty", BitListType(8).New(), "01", h(chunk(""), chunk("00"))}, {"bitvector TTFTFTFF", bitvec("TTFTFTFF"), "2b", chunk("2b")}, {"bitlist TTFTFTFF", bitlist("TTFTFTFF", 8), "2b01", h(chunk("2b"), chunk("08"))}, {"bitvector FTFT", bitvec("FTFT"), "0a", chunk("0a")}, @@ -178,50 +179,52 @@ func init() { {"uint32 01234567", Uint32View(0x01234567), "67452301", chunk("67452301")}, {"uint64 0000000000000000", Uint64View(0), "0000000000000000", chunk("0000000000000000")}, {"uint64 0123456789abcdef", Uint64View(0x0123456789abcdef), "efcdab8967452301", chunk("efcdab8967452301")}, - // TODO: bytelist type/view that is not backed by a tree, but makes the tree on demand, possible optimization. - // ("bytes48", Vector[byte, 48], Vector[byte, 48](*range(48)), "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f", - // h("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "202122232425262728292a2b2c2d2e2f00000000000000000000000000000000")), - //("raw bytes48", ByteVector[48], ByteVector[48](*range(48)), "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f", - // h("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "202122232425262728292a2b2c2d2e2f00000000000000000000000000000000")), - //("small empty bytelist", List[byte, 10], List[byte, 10](), "", h(chunk(""), chunk("00"))), - //("big empty bytelist", List[byte, 2048], List[byte, 2048](), "", h(zero_hashes[6], chunk("00"))), - //("raw small empty bytelist", ByteList[10], ByteList[10](), "", h(chunk(""), chunk("00"))), - //("raw big empty bytelist", ByteList[2048], ByteList[2048](), "", h(zero_hashes[6], chunk("00"))), - //("bytelist 7", List[byte, 7], List[byte, 7](*range(7)), "00010203040506", - // h(chunk("00010203040506"), chunk("07"))), - //("raw bytelist 7", ByteList[7], ByteList[7](*range(7)), "00010203040506", - // h(chunk("00010203040506"), chunk("07"))), - //("bytelist 50", List[byte, 50], List[byte, 50](*range(50)), "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031", - // h(h("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "202122232425262728292a2b2c2d2e2f30310000000000000000000000000000"), chunk("32"))), - //("raw bytelist 50", ByteList[50], ByteList[50](*range(50)), "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031", - // h(h("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "202122232425262728292a2b2c2d2e2f30310000000000000000000000000000"), chunk("32"))), - //("bytelist 6/256", List[byte, 256], List[byte, 256](*range(6)), "000102030405", - // h(h(h(h(chunk("000102030405"), zero_hashes[0]), zero_hashes[1]), zero_hashes[2]), chunk("06"))), - //("raw bytelist 6/256", ByteList[256], List[byte, 256](*range(6)), "000102030405", - // h(h(h(h(chunk("000102030405"), zero_hashes[0]), zero_hashes[1]), zero_hashes[2]), chunk("06"))), - //("sig", Vector[byte, 96], Vector[byte, 96](*sig_test_data), - // "0100000000000000000000000000000000000000000000000000000000000000" - // "0200000000000000000000000000000000000000000000000000000000000000" - // "03000000000000000000000000000000000000000000000000000000000000ff", - // h(h(chunk("01"), chunk("02")), - // h("03000000000000000000000000000000000000000000000000000000000000ff", chunk("")))), - //{"sig", [96]byte{0: 1, 32: 2, 64: 3, 95: 0xff}, - //"01" + repeat("00", 31) + "02" + repeat("00", 31) + "03" + repeat("00", 30) + "ff", - //h(h(chunk("01"), chunk("02")), h("03"+repeat("00", 30)+"ff", chunk("")))}, - //("raw sig", ByteVector[96], ByteVector[96](*sig_test_data), - // "0100000000000000000000000000000000000000000000000000000000000000" - // "0200000000000000000000000000000000000000000000000000000000000000" - // "03000000000000000000000000000000000000000000000000000000000000ff", - // h(h(chunk("01"), chunk("02")), - // h("03000000000000000000000000000000000000000000000000000000000000ff", chunk("")))), - //("3 sigs", Vector[ByteVector[96], 3], Vector[ByteVector[96], 3]( - // [1] + [0 for i in range(95)], - // [2] + [0 for i in range(95)], - // [3] + [0 for i in range(95)] - //), - // "01" + ("00" * 95) + "02" + ("00" * 95) + "03" + ("00" * 95), - // h(h(h(h(chunk("01"), chunk("")), zero_hashes[1]), h(h(chunk("02"), chunk("")), zero_hashes[1])), - // h(h(h(chunk("03"), chunk("")), zero_hashes[1]), chunk("")))), + {"uint256 0000000000000000000000000000000000000000000000000000000000000000", Uint256View{}, "0000000000000000000000000000000000000000000000000000000000000000", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"uint256 f1f0e1e0d1d0c1c0b1b0a1a09190818071706160515041403130212011100100", MustUint256("0xf1f0e1e0d1d0c1c0b1b0a1a09190818071706160515041403130212011100100"), "0001101120213031404150516061707180819091a0a1b0b1c0c1d0d1e0e1f0f1", "0001101120213031404150516061707180819091a0a1b0b1c0c1d0d1e0e1f0f1"}, + // TODO: bytelist type/view that is not backed by a tree, but makes the tree on demand, possible optimization. + // ("bytes48", Vector[byte, 48], Vector[byte, 48](*range(48)), "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f", + // h("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "202122232425262728292a2b2c2d2e2f00000000000000000000000000000000")), + //("raw bytes48", ByteVector[48], ByteVector[48](*range(48)), "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f", + // h("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "202122232425262728292a2b2c2d2e2f00000000000000000000000000000000")), + //("small empty bytelist", List[byte, 10], List[byte, 10](), "", h(chunk(""), chunk("00"))), + //("big empty bytelist", List[byte, 2048], List[byte, 2048](), "", h(zero_hashes[6], chunk("00"))), + //("raw small empty bytelist", ByteList[10], ByteList[10](), "", h(chunk(""), chunk("00"))), + //("raw big empty bytelist", ByteList[2048], ByteList[2048](), "", h(zero_hashes[6], chunk("00"))), + //("bytelist 7", List[byte, 7], List[byte, 7](*range(7)), "00010203040506", + // h(chunk("00010203040506"), chunk("07"))), + //("raw bytelist 7", ByteList[7], ByteList[7](*range(7)), "00010203040506", + // h(chunk("00010203040506"), chunk("07"))), + //("bytelist 50", List[byte, 50], List[byte, 50](*range(50)), "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031", + // h(h("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "202122232425262728292a2b2c2d2e2f30310000000000000000000000000000"), chunk("32"))), + //("raw bytelist 50", ByteList[50], ByteList[50](*range(50)), "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031", + // h(h("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "202122232425262728292a2b2c2d2e2f30310000000000000000000000000000"), chunk("32"))), + //("bytelist 6/256", List[byte, 256], List[byte, 256](*range(6)), "000102030405", + // h(h(h(h(chunk("000102030405"), zero_hashes[0]), zero_hashes[1]), zero_hashes[2]), chunk("06"))), + //("raw bytelist 6/256", ByteList[256], List[byte, 256](*range(6)), "000102030405", + // h(h(h(h(chunk("000102030405"), zero_hashes[0]), zero_hashes[1]), zero_hashes[2]), chunk("06"))), + //("sig", Vector[byte, 96], Vector[byte, 96](*sig_test_data), + // "0100000000000000000000000000000000000000000000000000000000000000" + // "0200000000000000000000000000000000000000000000000000000000000000" + // "03000000000000000000000000000000000000000000000000000000000000ff", + // h(h(chunk("01"), chunk("02")), + // h("03000000000000000000000000000000000000000000000000000000000000ff", chunk("")))), + //{"sig", [96]byte{0: 1, 32: 2, 64: 3, 95: 0xff}, + //"01" + repeat("00", 31) + "02" + repeat("00", 31) + "03" + repeat("00", 30) + "ff", + //h(h(chunk("01"), chunk("02")), h("03"+repeat("00", 30)+"ff", chunk("")))}, + //("raw sig", ByteVector[96], ByteVector[96](*sig_test_data), + // "0100000000000000000000000000000000000000000000000000000000000000" + // "0200000000000000000000000000000000000000000000000000000000000000" + // "03000000000000000000000000000000000000000000000000000000000000ff", + // h(h(chunk("01"), chunk("02")), + // h("03000000000000000000000000000000000000000000000000000000000000ff", chunk("")))), + //("3 sigs", Vector[ByteVector[96], 3], Vector[ByteVector[96], 3]( + // [1] + [0 for i in range(95)], + // [2] + [0 for i in range(95)], + // [3] + [0 for i in range(95)] + //), + // "01" + ("00" * 95) + "02" + ("00" * 95) + "03" + ("00" * 95), + // h(h(h(h(chunk("01"), chunk("")), zero_hashes[1]), h(h(chunk("02"), chunk("")), zero_hashes[1])), + // h(h(h(chunk("03"), chunk("")), zero_hashes[1]), chunk("")))), {"singleFieldTestStruct", viewMust(SingleFieldTestStructType.FromFields(Uint16View(0xab))), "ab", chunk("ab")}, {"uint16 list", viewMust(BasicListType(Uint16Type, 32).FromElements(Uint16View(0xaabb), Uint16View(0xc0ad), Uint16View(0xeeff))), "bbaaadc0ffee", @@ -237,6 +240,12 @@ func init() { "ffee000000000000000000000000000000000000000000000000000000000000", h(merge(h(h(chunk("bbaa"), chunk("adc0")), h(chunk("ffee"), chunk(""))), zeroHashes[2:6]), chunk("03000000")), }, + {"uint256 list", viewMust(ComplexListType(Uint256Type, 64).FromElements(&Uint256View{0: 0xaabb}, &Uint256View{0: 0xc0ad}, &Uint256View{0: 0xeeff})), + "bbaa000000000000000000000000000000000000000000000000000000000000" + + "adc0000000000000000000000000000000000000000000000000000000000000" + + "ffee000000000000000000000000000000000000000000000000000000000000", + h(merge(h(h(chunk("bbaa"), chunk("adc0")), h(chunk("ffee"), chunk(""))), zeroHashes[2:6]), chunk("03000000")), + }, {"bytes32 list long", viewMust(ComplexListType(RootType, 128).FromElements( &RootView{1}, &RootView{2}, &RootView{3}, &RootView{4}, &RootView{5}, &RootView{6}, &RootView{7}, &RootView{8}, &RootView{9}, &RootView{10}, &RootView{11}, &RootView{12}, &RootView{13}, &RootView{14}, &RootView{15}, &RootView{16}, &RootView{17}, &RootView{18}, &RootView{19}, @@ -304,7 +313,7 @@ func init() { {"var element list", viewMust(ComplexListType(VarTestStructType, 8).FromElements( viewMust(VarTestStructType.FromFields(Uint16View(0xdead), viewMust(BasicListType(Uint16Type, 1024).FromElements(Uint16View(1), Uint16View(2), Uint16View(3))), Uint8View(0x11))), viewMust(VarTestStructType.FromFields(Uint16View(0xbeef), viewMust(BasicListType(Uint16Type, 1024).FromElements(Uint16View(4), Uint16View(5), Uint16View(6))), Uint8View(0x22))), - )), + )), "08000000" + "15000000" + "adde0700000011010002000300" + "efbe0700000022040005000600", @@ -348,7 +357,7 @@ func init() { Uint16View(0xaabb), viewMust(BasicListType(Uint16Type, 128).FromElements(Uint16View(0x1122), Uint16View(0x3344))), Uint8View(0xff), - viewMust(BasicListType(ByteType, 64).FromElements(ByteView('f'), ByteView('o'), ByteView('o'), ByteView('b'), ByteView('a'), ByteView('r'))), + viewMust(BasicListType(ByteType, 256).FromElements(ByteView('f'), ByteView('o'), ByteView('o'), ByteView('b'), ByteView('a'), ByteView('r'))), viewMust( VarTestStructType.FromFields( Uint16View(0xabcd), @@ -427,14 +436,13 @@ func init() { } } - -func TestEncode(t *testing.T) { +func TestSerializeView(t *testing.T) { var buf bytes.Buffer for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { buf.Reset() - if err := tt.value.Serialize(&buf); err != nil { + if err := tt.value.Serialize(codec.NewEncodingWriter(&buf)); err != nil { t.Fatalf("encoding failed, err: %v", err) } data := buf.Bytes() @@ -445,6 +453,69 @@ func TestEncode(t *testing.T) { } } +func TestDeserializeSerialize(t *testing.T) { + for _, tt := range testCases { + v := reflect.New(reflect.TypeOf(tt.value)) + if _, ok := v.Interface().(codec.Deserializable); ok { + t.Run(tt.name, func(t *testing.T) { + d := v.Interface().(codec.Deserializable) + data, err := hex.DecodeString(tt.hex) + if err != nil { + t.Fatal(err) + } + r := bytes.NewReader(data) + // For dynamic types, we need to pass the length of the message to the decoder. + bytesLen := uint64(len(tt.hex)) / 2 + if err := d.Deserialize(codec.NewDecodingReader(r, bytesLen)); err != nil { + t.Fatalf("decoding failed, err: %v", err) + } + s, ok := d.(codec.Serializable) + if !ok { + t.Fatal("implemented Deserializable, but not Serializable") + } + var buf bytes.Buffer + if err := s.Serialize(codec.NewEncodingWriter(&buf)); err != nil { + t.Fatal(err) + } + got := hex.EncodeToString(buf.Bytes()) + if got != tt.hex { + t.Fatalf("got %s, expected %s", got, tt.hex) + } + }) + } + } +} + +func TestDecodeEncode(t *testing.T) { + for _, tt := range testCases { + v := reflect.New(reflect.TypeOf(tt.value)) + if _, ok := v.Interface().(codec.Decodable); ok { + t.Run(tt.name, func(t *testing.T) { + d := v.Interface().(codec.Decodable) + data, err := hex.DecodeString(tt.hex) + if err != nil { + t.Fatal(err) + } + if err := d.Decode(data); err != nil { + t.Fatalf("decoding failed, err: %v", err) + } + s, ok := d.(codec.Encodable) + if !ok { + t.Fatal("implemented Decodable, but not Encodable") + } + out, err := s.Encode() + if err != nil { + t.Fatal(err) + } + got := hex.EncodeToString(out) + if got != tt.hex { + t.Fatalf("got %s, expected %s", got, tt.hex) + } + }) + } + } +} + func TestValueByteLength(t *testing.T) { for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { @@ -458,9 +529,9 @@ func TestValueByteLength(t *testing.T) { } } -func TestDecode(t *testing.T) { +func TestDeserializeView(t *testing.T) { + hFn := tree.GetHashFn() for _, tt := range testCases { - hFn := tree.GetHashFn() t.Run(tt.name, func(t *testing.T) { sszTyp := tt.value.Type() data, err := hex.DecodeString(tt.hex) @@ -471,14 +542,22 @@ func TestDecode(t *testing.T) { // For dynamic types, we need to pass the length of the message to the decoder. bytesLen := uint64(len(tt.hex)) / 2 - dest, err := sszTyp.Deserialize(r, bytesLen) + dest, err := sszTyp.Deserialize(codec.NewDecodingReader(r, bytesLen)) if err != nil { t.Fatal(err) } root := dest.HashTreeRoot(hFn) hexRoot := hex.EncodeToString(root[:]) if hexRoot != tt.root { - t.Errorf("Hash tree root of deserialized object does not match expected root. Got: %s, expected: %s", hexRoot, tt.root) + t.Errorf("Hash tree root of deserialized object does not match expected root. Got: %s, expected: %s.", hexRoot, tt.root) + } + var buf bytes.Buffer + if err := dest.Serialize(codec.NewEncodingWriter(&buf)); err != nil { + t.Errorf("failed to serialize object that was just deserialized: %v", err) + } + out := hex.EncodeToString(buf.Bytes()) + if out != tt.hex { + t.Errorf("original bytes do not match bytes after deserialize/serialize round-trip:\nout: %s\nin: %s", out, tt.hex) } }) }