Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

rlp: support for uint256 #26898

Merged
merged 17 commits into from
Mar 17, 2023
Merged

rlp: support for uint256 #26898

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
4c5c9b5
rlp: support for uint256
holiman Mar 15, 2023
eb8c057
rlp: return error for too large uint256
fjl Mar 15, 2023
56c659e
rlp: fix decoding of non-pointer uint256
fjl Mar 15, 2023
0d92715
rlp: add uint256 tests
fjl Mar 15, 2023
08d450e
rlp: handle encoding u256 both pointer and non-pointer
holiman Mar 16, 2023
5084e85
rlp: implement encoderlp for uint256.Int
holiman Mar 16, 2023
4712879
rlp: benchmark tests for u256
holiman Mar 16, 2023
6f9fcc4
rlp: add EncoderBuffer method for Uint256
fjl Mar 16, 2023
c5306d3
rlp: update comment
fjl Mar 16, 2023
87b7fdb
Update rlp/encbuffer.go
holiman Mar 16, 2023
b774ecc
rlp: rename to ReadUint256
fjl Mar 16, 2023
6e9d483
rlp/rlpgen: add support for uint256
fjl Mar 16, 2023
0a2fdfb
rlp: add failing test for array encoding
fjl Mar 16, 2023
89298f7
rlp: fix array test
fjl Mar 16, 2023
8a6cbf8
rlp: check for uint256 more strictly
fjl Mar 16, 2023
aa376f8
rlp: check for uint256 more strictly in decoder
fjl Mar 16, 2023
4ad1faa
rlp/rlpgen: update uint256 output after renaming of method
holiman Mar 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions rlp/decode.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"sync"

"github.com/ethereum/go-ethereum/rlp/internal/rlpstruct"
"github.com/holiman/uint256"
)

//lint:ignore ST1012 EOL is not an error.
Expand All @@ -52,6 +53,7 @@ var (
errUintOverflow = errors.New("rlp: uint overflow")
errNoPointer = errors.New("rlp: interface given to Decode must be a pointer")
errDecodeIntoNil = errors.New("rlp: pointer given to Decode must not be nil")
errUint256Large = errors.New("rlp: value too large for uint256")

streamPool = sync.Pool{
New: func() interface{} { return new(Stream) },
Expand Down Expand Up @@ -148,6 +150,7 @@ func addErrorContext(err error, ctx string) error {
var (
decoderInterface = reflect.TypeOf(new(Decoder)).Elem()
bigInt = reflect.TypeOf(big.Int{})
u256Int = reflect.TypeOf(uint256.Int{})
)

func makeDecoder(typ reflect.Type, tags rlpstruct.Tags) (dec decoder, err error) {
Expand All @@ -159,6 +162,10 @@ func makeDecoder(typ reflect.Type, tags rlpstruct.Tags) (dec decoder, err error)
return decodeBigInt, nil
case typ.AssignableTo(bigInt):
return decodeBigIntNoPtr, nil
case typ.AssignableTo(reflect.PtrTo(u256Int)):
return decodeU256, nil
case typ.AssignableTo(u256Int):
return decodeU256NoPtr, nil
case kind == reflect.Ptr:
return makePtrDecoder(typ, tags)
case reflect.PtrTo(typ).Implements(decoderInterface):
Expand Down Expand Up @@ -235,6 +242,24 @@ func decodeBigInt(s *Stream, val reflect.Value) error {
return nil
}

func decodeU256NoPtr(s *Stream, val reflect.Value) error {
return decodeU256(s, val.Addr())
}

func decodeU256(s *Stream, val reflect.Value) error {
i := val.Interface().(*uint256.Int)
if i == nil {
i = new(uint256.Int)
val.Set(reflect.ValueOf(i))
}

err := s.ReadUint256(i)
if err != nil {
return wrapStreamError(err, val.Type())
}
return nil
}

func makeListDecoder(typ reflect.Type, tag rlpstruct.Tags) (decoder, error) {
etype := typ.Elem()
if etype.Kind() == reflect.Uint8 && !reflect.PtrTo(etype).Implements(decoderInterface) {
Expand Down Expand Up @@ -863,6 +888,45 @@ func (s *Stream) decodeBigInt(dst *big.Int) error {
return nil
}

// ReadUint256 decodes the next value as a uint256.
func (s *Stream) ReadUint256(dst *uint256.Int) error {
var buffer []byte
kind, size, err := s.Kind()
switch {
case err != nil:
return err
case kind == List:
return ErrExpectedString
case kind == Byte:
buffer = s.uintbuf[:1]
buffer[0] = s.byteval
s.kind = -1 // re-arm Kind
case size == 0:
// Avoid zero-length read.
s.kind = -1
case size <= uint64(len(s.uintbuf)):
// All possible uint256 values fit into s.uintbuf.
buffer = s.uintbuf[:size]
if err := s.readFull(buffer); err != nil {
return err
}
// Reject inputs where single byte encoding should have been used.
if size == 1 && buffer[0] < 128 {
return ErrCanonSize
}
default:
return errUint256Large
}

// Reject leading zero bytes.
if len(buffer) > 0 && buffer[0] == 0 {
return ErrCanonInt
}
// Set the integer bytes.
dst.SetBytes(buffer)
return nil
}

// Decode decodes a value and stores the result in the value pointed
// to by val. Please see the documentation for the Decode function
// to learn about the decoding rules.
Expand Down
42 changes: 42 additions & 0 deletions rlp/decode_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"testing"

"github.com/ethereum/go-ethereum/common/math"
"github.com/holiman/uint256"
)

func TestStreamKind(t *testing.T) {
Expand Down Expand Up @@ -468,6 +469,10 @@ var (
veryVeryBigInt = new(big.Int).Exp(veryBigInt, big.NewInt(8), nil)
)

var (
veryBigInt256, _ = uint256.FromBig(veryBigInt)
)

var decodeTests = []decodeTest{
// booleans
{input: "01", ptr: new(bool), value: true},
Expand Down Expand Up @@ -541,11 +546,27 @@ var decodeTests = []decodeTest{
{input: "89FFFFFFFFFFFFFFFFFF", ptr: new(*big.Int), value: veryBigInt},
{input: "B848FFFFFFFFFFFFFFFFF800000000000000001BFFFFFFFFFFFFFFFFC8000000000000000045FFFFFFFFFFFFFFFFC800000000000000001BFFFFFFFFFFFFFFFFF8000000000000000001", ptr: new(*big.Int), value: veryVeryBigInt},
{input: "10", ptr: new(big.Int), value: *big.NewInt(16)}, // non-pointer also works

// big int errors
{input: "C0", ptr: new(*big.Int), error: "rlp: expected input string or byte for *big.Int"},
{input: "00", ptr: new(*big.Int), error: "rlp: non-canonical integer (leading zero bytes) for *big.Int"},
{input: "820001", ptr: new(*big.Int), error: "rlp: non-canonical integer (leading zero bytes) for *big.Int"},
{input: "8105", ptr: new(*big.Int), error: "rlp: non-canonical size information for *big.Int"},

// uint256
{input: "80", ptr: new(*uint256.Int), value: uint256.NewInt(0)},
{input: "01", ptr: new(*uint256.Int), value: uint256.NewInt(1)},
{input: "88FFFFFFFFFFFFFFFF", ptr: new(*uint256.Int), value: uint256.NewInt(math.MaxUint64)},
{input: "89FFFFFFFFFFFFFFFFFF", ptr: new(*uint256.Int), value: veryBigInt256},
{input: "10", ptr: new(uint256.Int), value: *uint256.NewInt(16)}, // non-pointer also works

// uint256 errors
{input: "C0", ptr: new(*uint256.Int), error: "rlp: expected input string or byte for *uint256.Int"},
{input: "00", ptr: new(*uint256.Int), error: "rlp: non-canonical integer (leading zero bytes) for *uint256.Int"},
{input: "820001", ptr: new(*uint256.Int), error: "rlp: non-canonical integer (leading zero bytes) for *uint256.Int"},
{input: "8105", ptr: new(*uint256.Int), error: "rlp: non-canonical size information for *uint256.Int"},
{input: "A1FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00", ptr: new(*uint256.Int), error: "rlp: value too large for uint256"},

// structs
{
input: "C50583343434",
Expand Down Expand Up @@ -1223,6 +1244,27 @@ func BenchmarkDecodeBigInts(b *testing.B) {
}
}

func BenchmarkDecodeU256Ints(b *testing.B) {
ints := make([]*uint256.Int, 200)
for i := range ints {
ints[i], _ = uint256.FromBig(math.BigPow(2, int64(i)))
}
enc, err := EncodeToBytes(ints)
if err != nil {
b.Fatal(err)
}
b.SetBytes(int64(len(enc)))
b.ReportAllocs()
b.ResetTimer()

var out []*uint256.Int
for i := 0; i < b.N; i++ {
if err := DecodeBytes(enc, &out); err != nil {
b.Fatal(err)
}
}
}

func encodeTestSlice(n uint) []byte {
s := make([]uint, n)
for i := uint(0); i < n; i++ {
Expand Down
25 changes: 25 additions & 0 deletions rlp/encbuffer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@
package rlp

import (
"encoding/binary"
"io"
"math/big"
"reflect"
"sync"

"github.com/holiman/uint256"
)

type encBuffer struct {
Expand Down Expand Up @@ -169,6 +172,23 @@ func (w *encBuffer) writeBigInt(i *big.Int) {
}
}

// writeUint256 writes z as an integer.
func (w *encBuffer) writeUint256(z *uint256.Int) {
bitlen := z.BitLen()
if bitlen <= 64 {
w.writeUint64(z.Uint64())
return
}
nBytes := byte((bitlen + 7) / 8)
var b [33]byte
binary.BigEndian.PutUint64(b[1:9], z[3])
binary.BigEndian.PutUint64(b[9:17], z[2])
binary.BigEndian.PutUint64(b[17:25], z[1])
binary.BigEndian.PutUint64(b[25:33], z[0])
b[32-nBytes] = 0x80 + nBytes
w.str = append(w.str, b[32-nBytes:]...)
}

// list adds a new list header to the header stack. It returns the index of the header.
// Call listEnd with this index after encoding the content of the list.
func (buf *encBuffer) list() int {
Expand Down Expand Up @@ -376,6 +396,11 @@ func (w EncoderBuffer) WriteBigInt(i *big.Int) {
w.buf.writeBigInt(i)
}

// WriteUint256 encodes uint256.Int as an RLP string.
func (w EncoderBuffer) WriteUint256(i *uint256.Int) {
w.buf.writeUint256(i)
}

// WriteBytes encodes b as an RLP string.
func (w EncoderBuffer) WriteBytes(b []byte) {
w.buf.writeBytes(b)
Expand Down
21 changes: 21 additions & 0 deletions rlp/encode.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"reflect"

"github.com/ethereum/go-ethereum/rlp/internal/rlpstruct"
"github.com/holiman/uint256"
)

var (
Expand Down Expand Up @@ -144,6 +145,10 @@ func makeWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) {
return writeBigIntPtr, nil
case typ.AssignableTo(bigInt):
return writeBigIntNoPtr, nil
case typ.AssignableTo(reflect.PtrTo(u256Int)):
return writeU256IntPtr, nil
case typ.AssignableTo(u256Int):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think there is a problem with this check. As written, this will also trigger for values of type [4]uint64, because those values are also 'assignable' to uint256.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm. Ouch

Copy link
Contributor

@fjl fjl Mar 16, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We check for big.Int in the same way, but it's less of a problem because (1) big.Int is a struct with private fields, and (2) it is usually handled as a pointer, and named types can't be pointers (type X *big.Int is not allowed by compiler).

return writeU256IntNoPtr, nil
case kind == reflect.Ptr:
return makePtrWriter(typ, ts)
case reflect.PtrTo(typ).Implements(encoderInterface):
Expand Down Expand Up @@ -206,6 +211,22 @@ func writeBigIntNoPtr(val reflect.Value, w *encBuffer) error {
return nil
}

func writeU256IntPtr(val reflect.Value, w *encBuffer) error {
ptr := val.Interface().(*uint256.Int)
if ptr == nil {
w.str = append(w.str, 0x80)
return nil
}
w.writeUint256(ptr)
return nil
}

func writeU256IntNoPtr(val reflect.Value, w *encBuffer) error {
i := val.Interface().(uint256.Int)
w.writeUint256(&i)
return nil
}

func writeBytes(val reflect.Value, w *encBuffer) error {
w.writeBytes(val.Bytes())
return nil
Expand Down
47 changes: 47 additions & 0 deletions rlp/encode_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"testing"

"github.com/ethereum/go-ethereum/common/math"
"github.com/holiman/uint256"
)

type testEncoder struct {
Expand Down Expand Up @@ -147,6 +148,30 @@ var encTests = []encTest{
{val: big.NewInt(-1), error: "rlp: cannot encode negative big.Int"},
{val: *big.NewInt(-1), error: "rlp: cannot encode negative big.Int"},

// uint256
{val: uint256.NewInt(0), output: "80"},
{val: uint256.NewInt(1), output: "01"},
{val: uint256.NewInt(127), output: "7F"},
{val: uint256.NewInt(128), output: "8180"},
{val: uint256.NewInt(256), output: "820100"},
{val: uint256.NewInt(1024), output: "820400"},
{val: uint256.NewInt(0xFFFFFF), output: "83FFFFFF"},
{val: uint256.NewInt(0xFFFFFFFF), output: "84FFFFFFFF"},
{val: uint256.NewInt(0xFFFFFFFFFF), output: "85FFFFFFFFFF"},
{val: uint256.NewInt(0xFFFFFFFFFFFF), output: "86FFFFFFFFFFFF"},
{val: uint256.NewInt(0xFFFFFFFFFFFFFF), output: "87FFFFFFFFFFFFFF"},
{
val: new(uint256.Int).SetBytes(unhex("102030405060708090A0B0C0D0E0F2")),
output: "8F102030405060708090A0B0C0D0E0F2",
},
{
val: new(uint256.Int).SetBytes(unhex("0100020003000400050006000700080009000A000B000C000D000E01")),
output: "9C0100020003000400050006000700080009000A000B000C000D000E01",
},
// non-pointer uint256.Int
{val: *uint256.NewInt(0), output: "80"},
{val: *uint256.NewInt(0xFFFFFF), output: "83FFFFFF"},

// byte arrays
{val: [0]byte{}, output: "80"},
{val: [1]byte{0}, output: "00"},
Expand Down Expand Up @@ -256,6 +281,10 @@ var encTests = []encTest{
output: "F90200CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376CF84617364668471776572847A786376",
},

// arrays
{val: [4]uint32{1, 2, 3, 4}, output: "8401020304"},
Copy link
Contributor Author

@holiman holiman Mar 16, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't the want C401020304 ?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I made a mistake. It's fixed now though!

{val: [4]uint64{1, 2, 3, 4}, output: "8401020304"},

// RawValue
{val: RawValue(unhex("01")), output: "01"},
{val: RawValue(unhex("82FFFF")), output: "82FFFF"},
Expand Down Expand Up @@ -301,6 +330,7 @@ var encTests = []encTest{
{val: (*[]byte)(nil), output: "80"},
{val: (*[10]byte)(nil), output: "80"},
{val: (*big.Int)(nil), output: "80"},
{val: (*uint256.Int)(nil), output: "80"},
{val: (*[]string)(nil), output: "C0"},
{val: (*[10]string)(nil), output: "C0"},
{val: (*[]interface{})(nil), output: "C0"},
Expand Down Expand Up @@ -509,6 +539,23 @@ func BenchmarkEncodeBigInts(b *testing.B) {
}
}

func BenchmarkEncodeU256Ints(b *testing.B) {
ints := make([]*uint256.Int, 200)
for i := range ints {
ints[i], _ = uint256.FromBig(math.BigPow(2, int64(i)))
}
out := bytes.NewBuffer(make([]byte, 0, 4096))
b.ResetTimer()
b.ReportAllocs()

for i := 0; i < b.N; i++ {
out.Reset()
if err := Encode(out, ints); err != nil {
b.Fatal(err)
}
}
}

func BenchmarkEncodeConcurrentInterface(b *testing.B) {
type struct1 struct {
A string
Expand Down
Loading