From b2fc93449d3e091cdf91a5b268285c659407f8c2 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 11 Jun 2023 17:36:47 -0500 Subject: [PATCH] feat: generate JSON for arbitrary CBOR Fixes #305 --- cbor/cbor.go | 4 + cbor/decode.go | 2 +- cbor/value.go | 260 +++++++++++++++++++++++++++++++-------- cbor/value_test.go | 205 ++++++++++++++++++++++++++++++ cmd/block-fetch/main.go | 9 +- internal/test/helpers.go | 23 ++++ ledger/error.go | 18 +-- 7 files changed, 458 insertions(+), 63 deletions(-) create mode 100644 cbor/value_test.go diff --git a/cbor/cbor.go b/cbor/cbor.go index 1e518e10..35079dca 100644 --- a/cbor/cbor.go +++ b/cbor/cbor.go @@ -30,6 +30,10 @@ const ( // Max value able to be stored in a single byte without type prefix CBOR_MAX_UINT_SIMPLE uint8 = 0x17 + + // Useful tag numbers + CborTagSet = 258 + CborTagMap = 259 ) // Create an alias for RawMessage for convenience diff --git a/cbor/decode.go b/cbor/decode.go index 1988dc91..177c730d 100644 --- a/cbor/decode.go +++ b/cbor/decode.go @@ -64,7 +64,7 @@ func DecodeIdFromList(cborData []byte) (int, error) { return 0, err } // Make sure that the value is actually numeric - switch v := tmp.Value.([]interface{})[0].(type) { + switch v := tmp.Value().([]interface{})[0].(type) { // The upstream CBOR library uses uint64 by default for numeric values case uint64: return int(v), nil diff --git a/cbor/value.go b/cbor/value.go index 2ef5f7be..6c08e74d 100644 --- a/cbor/value.go +++ b/cbor/value.go @@ -15,88 +15,88 @@ package cbor import ( + "encoding/hex" + "encoding/json" "fmt" + "sort" + "strings" ) // Helpful wrapper for parsing arbitrary CBOR data which may contain types that // cannot be easily represented in Go (such as maps with bytestring keys) type Value struct { - Value interface{} + value interface{} // We store this as a string so that the type is still hashable for use as map keys cborData string } -func (v *Value) UnmarshalCBOR(data []byte) (err error) { +func (v *Value) UnmarshalCBOR(data []byte) error { // Save the original CBOR v.cborData = string(data[:]) cborType := data[0] & CBOR_TYPE_MASK switch cborType { case CBOR_TYPE_MAP: - // There are certain types that cannot be used as map keys in Go but are valid in CBOR. Trying to - // parse CBOR containing a map with keys of one of those types will cause a panic. We setup this - // deferred function to recover from a possible panic and return an error - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("decode failure, probably due to type unsupported by Go: %v", r) - } - }() - tmpValue := map[Value]Value{} - if _, err := Decode(data, &tmpValue); err != nil { - return err - } - // Extract actual value from each child value - newValue := map[interface{}]interface{}{} - for key, value := range tmpValue { - newValue[key.Value] = value.Value - } - v.Value = newValue + return v.processMap(data) case CBOR_TYPE_ARRAY: - tmpValue := []Value{} - if _, err := Decode(data, &tmpValue); err != nil { - return err - } - // Extract actual value from each child value - newValue := []interface{}{} - for _, value := range tmpValue { - newValue = append(newValue, value.Value) - } - v.Value = newValue + return v.processArray(data) case CBOR_TYPE_TEXT_STRING: var tmpValue string if _, err := Decode(data, &tmpValue); err != nil { return err } - v.Value = tmpValue + v.value = tmpValue case CBOR_TYPE_BYTE_STRING: // Use our custom type which stores the bytestring in a way that allows it to be used as a map key var tmpValue ByteString if _, err := Decode(data, &tmpValue); err != nil { return err } - v.Value = tmpValue + v.value = tmpValue case CBOR_TYPE_TAG: // Parse as a raw tag to get number and nested CBOR data tmpTag := RawTag{} if _, err := Decode(data, &tmpTag); err != nil { return err } - // Parse the tag value via our custom Value object to handle problem types - tmpValue := Value{} - if _, err := Decode(tmpTag.Content, &tmpValue); err != nil { - return err - } - // Create new tag object with decoded value - newValue := Tag{ - Number: tmpTag.Number, - Content: tmpValue.Value, + switch tmpTag.Number { + case CborTagSet: + return v.processArray(tmpTag.Content) + case CborTagMap: + return v.processMap(tmpTag.Content) + default: + // Parse the tag value via our custom Value object to handle problem types + tmpValue := Value{} + if _, err := Decode(tmpTag.Content, &tmpValue); err != nil { + return err + } + if tmpTag.Number >= 121 && tmpTag.Number <= 127 { + v.value = Constructor{ + constructor: uint(tmpTag.Number - 121), + value: &tmpValue, + } + } else if tmpTag.Number >= 1280 && tmpTag.Number <= 1400 { + v.value = Constructor{ + constructor: uint(tmpTag.Number - 1280 + 7), + value: &tmpValue, + } + } else if tmpTag.Number == 101 { + newValue := Value{ + value: tmpValue.Value().([]interface{})[1], + } + v.value = Constructor{ + constructor: tmpValue.Value().([]interface{})[0].(uint), + value: &newValue, + } + } else { + return fmt.Errorf("unsupported CBOR tag: %d", tmpTag.Number) + } } - v.Value = newValue default: var tmpValue interface{} if _, err := Decode(data, &tmpValue); err != nil { return err } - v.Value = tmpValue + v.value = tmpValue } return nil } @@ -105,19 +105,179 @@ func (v Value) Cbor() []byte { return []byte(v.cborData) } +func (v Value) Value() interface{} { + return v.value +} + +func (v Value) MarshalJSON() ([]byte, error) { + var tmpJson string + if v.value != nil { + astJson, err := generateAstJson(v.value) + if err != nil { + return nil, err + } + tmpJson = fmt.Sprintf( + `{"cbor":"%s","json":%s}`, + hex.EncodeToString([]byte(v.cborData)), + astJson, + ) + } else { + tmpJson = fmt.Sprintf( + `{"cbor":"%s"}`, + hex.EncodeToString([]byte(v.cborData)), + ) + } + return []byte(tmpJson), nil +} + +func (v *Value) processMap(data []byte) (err error) { + // There are certain types that cannot be used as map keys in Go but are valid in CBOR. Trying to + // parse CBOR containing a map with keys of one of those types will cause a panic. We setup this + // deferred function to recover from a possible panic and return an error + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("decode failure, probably due to type unsupported by Go: %v", r) + } + }() + tmpValue := map[Value]Value{} + if _, err = Decode(data, &tmpValue); err != nil { + return err + } + // Extract actual value from each child value + newValue := map[interface{}]interface{}{} + for key, value := range tmpValue { + newValue[key.Value()] = value.Value() + } + v.value = newValue + return nil +} + +func (v *Value) processArray(data []byte) error { + tmpValue := []Value{} + if _, err := Decode(data, &tmpValue); err != nil { + return err + } + // Extract actual value from each child value + newValue := []interface{}{} + for _, value := range tmpValue { + newValue = append(newValue, value.Value()) + } + v.value = newValue + return nil +} + +func generateAstJson(obj interface{}) ([]byte, error) { + tmpJsonObj := map[string]interface{}{} + switch v := obj.(type) { + case ByteString: + tmpJsonObj["bytes"] = hex.EncodeToString(v.Bytes()) + case []interface{}: + tmpJson := `{"list":[` + for idx, val := range v { + tmpVal, err := generateAstJson(val) + if err != nil { + return nil, err + } + tmpJson += string(tmpVal) + if idx != (len(v) - 1) { + tmpJson += `,` + } + } + tmpJson += `]}` + return []byte(tmpJson), nil + case map[interface{}]interface{}: + tmpItems := []string{} + for key, val := range v { + keyAstJson, err := generateAstJson(key) + if err != nil { + return nil, err + } + valAstJson, err := generateAstJson(val) + if err != nil { + return nil, err + } + tmpJson := fmt.Sprintf( + `{"k":%s,"v":%s}`, + keyAstJson, + valAstJson, + ) + tmpItems = append(tmpItems, string(tmpJson)) + } + // We naively sort the rendered map items to give consistent ordering + sort.Strings(tmpItems) + tmpJson := fmt.Sprintf( + `{"map":[%s]}`, + strings.Join(tmpItems, ","), + ) + return []byte(tmpJson), nil + case Constructor: + return json.Marshal(obj) + case int, uint, uint64, int64: + tmpJsonObj["int"] = v + case bool: + tmpJsonObj["bool"] = v + case string: + tmpJsonObj["string"] = v + default: + return nil, fmt.Errorf("unknown data type (%T) for value: %#v", obj, obj) + } + return json.Marshal(&tmpJsonObj) +} + +type Constructor struct { + constructor uint + value *Value +} + +func (v Constructor) MarshalJSON() ([]byte, error) { + tmpJson := fmt.Sprintf(`{"constructor":%d,"fields":[`, v.constructor) + tmpList := [][]byte{} + for _, val := range v.value.Value().([]any) { + tmpVal, err := generateAstJson(val) + if err != nil { + return nil, err + } + tmpList = append(tmpList, tmpVal) + } + for idx, val := range tmpList { + tmpJson += string(val) + if idx != (len(tmpList) - 1) { + tmpJson += `,` + } + } + tmpJson += `]}` + return []byte(tmpJson), nil +} + type LazyValue struct { - *Value + value *Value } func (l *LazyValue) UnmarshalCBOR(data []byte) error { - if l.Value == nil { - l.Value = &Value{} + if l.value == nil { + l.value = &Value{} } - l.cborData = string(data[:]) + l.value.cborData = string(data[:]) return nil } -func (l *LazyValue) Decode() (*Value, error) { - err := l.Value.UnmarshalCBOR([]byte(l.cborData)) - return l.Value, err +func (l *LazyValue) MarshalJSON() ([]byte, error) { + if l.Value() == nil { + // Try to decode if we can, but don't blow up if we can't + _, _ = l.Decode() + } + return l.value.MarshalJSON() +} + +func (l *LazyValue) Decode() (interface{}, error) { + err := l.value.UnmarshalCBOR([]byte(l.value.cborData)) + return l.Value(), err +} + +func (l *LazyValue) Value() interface{} { + return l.value.Value() +} + +func (l *LazyValue) Cbor() []byte { + return l.value.Cbor() } diff --git a/cbor/value_test.go b/cbor/value_test.go new file mode 100644 index 00000000..ecb19d7a --- /dev/null +++ b/cbor/value_test.go @@ -0,0 +1,205 @@ +// Copyright 2023 Blink Labs, LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cbor_test + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/blinklabs-io/gouroboros/cbor" + "github.com/blinklabs-io/gouroboros/internal/test" +) + +var testDefs = []struct { + cborHex string + expectedObject interface{} + expectedAstJson string + expectedDecodeError error +}{ + // [] + { + cborHex: "80", + expectedObject: []any{}, + expectedAstJson: `{"list":[]}`, + }, + // Invalid CBOR + { + cborHex: "81", + expectedObject: nil, + expectedDecodeError: fmt.Errorf("EOF"), + }, + // Invalid map key type + { + cborHex: "A1810000", + expectedDecodeError: fmt.Errorf("decode failure, probably due to type unsupported by Go: runtime error: hash of unhashable type []interface {}"), + }, + // [1, 2, 3] + { + cborHex: "83010203", + expectedObject: []any{uint64(1), uint64(2), uint64(3)}, + expectedAstJson: `{"list":[{"int":1},{"int":2},{"int":3}]}`, + }, + // {1: 2, 3: 4} + { + cborHex: "A201020304", + expectedObject: map[any]any{uint64(1): uint64(2), uint64(3): uint64(4)}, + expectedAstJson: `{"map":[{"k":{"int":1},"v":{"int":2}},{"k":{"int":3},"v":{"int":4}}]}`, + }, + // {1: [2], 3: [4]} + { + cborHex: "A2018102038104", + expectedObject: map[any]any{ + uint64(1): []any{uint64(2)}, + uint64(3): []any{uint64(4)}, + }, + expectedAstJson: `{"map":[{"k":{"int":1},"v":{"list":[{"int":2}]}},{"k":{"int":3},"v":{"list":[{"int":4}]}}]}`, + }, +} + +func TestValueDecode(t *testing.T) { + for _, testDef := range testDefs { + cborData, err := hex.DecodeString(testDef.cborHex) + if err != nil { + t.Fatalf("failed to decode CBOR hex: %s", err) + } + var tmpValue cbor.Value + if _, err := cbor.Decode(cborData, &tmpValue); err != nil { + if testDef.expectedDecodeError != nil { + if err.Error() != testDef.expectedDecodeError.Error() { + t.Fatalf("did not receive expected decode error, got: %s, wanted: %s", err, testDef.expectedDecodeError) + } + continue + } else { + t.Fatalf("failed to decode CBOR data: %s", err) + } + } else { + if testDef.expectedDecodeError != nil { + t.Fatalf("did not receive expected decode error: %s", testDef.expectedDecodeError) + } + } + newObj := tmpValue.Value() + if !reflect.DeepEqual(newObj, testDef.expectedObject) { + t.Fatalf("CBOR did not decode to expected object\n got: %#v\n wanted: %#v", newObj, testDef.expectedObject) + } + } +} + +func TestValueMarshalJSON(t *testing.T) { + for _, testDef := range testDefs { + // Skip test if the CBOR decode is expected to fail + if testDef.expectedDecodeError != nil { + continue + } + cborData, err := hex.DecodeString(testDef.cborHex) + if err != nil { + t.Fatalf("failed to decode CBOR hex: %s", err) + } + var tmpValue cbor.Value + if _, err := cbor.Decode(cborData, &tmpValue); err != nil { + t.Fatalf("failed to decode CBOR data: %s", err) + } + jsonData, err := json.Marshal(&tmpValue) + if err != nil { + t.Fatalf("failed to marshal Value as JSON: %s", err) + } + // We create the wrapper JSON here, since it would otherwise result in the CBOR hex being duplicated in each test definition + fullExpectedJson := fmt.Sprintf( + `{"cbor":"%s","json":%s}`, + strings.ToLower(testDef.cborHex), + testDef.expectedAstJson, + ) + if testDef.expectedObject == nil { + fullExpectedJson = fmt.Sprintf( + `{"cbor":"%s"}`, + strings.ToLower(testDef.cborHex), + ) + } + if !test.JsonStringsEqual(jsonData, []byte(fullExpectedJson)) { + t.Fatalf("CBOR did not marshal to expected JSON\n got: %s\n wanted: %s", jsonData, fullExpectedJson) + } + } +} + +func TestLazyValueDecode(t *testing.T) { + for _, testDef := range testDefs { + cborData, err := hex.DecodeString(testDef.cborHex) + if err != nil { + t.Fatalf("failed to decode CBOR hex: %s", err) + } + var tmpValue cbor.LazyValue + if _, err := cbor.Decode(cborData, &tmpValue); err != nil { + if testDef.expectedDecodeError != nil { + if err.Error() != testDef.expectedDecodeError.Error() { + t.Fatalf("did not receive expected decode error, got: %s, wanted: %s", err, testDef.expectedDecodeError) + } + continue + } else { + t.Fatalf("failed to decode CBOR data: %s", err) + } + } + newObj, err := tmpValue.Decode() + if err != nil { + if testDef.expectedDecodeError != nil { + if err.Error() != testDef.expectedDecodeError.Error() { + t.Fatalf("did not receive expected decode error, got: %s, wanted: %s", err, testDef.expectedDecodeError) + } + continue + } else { + t.Fatalf("failed to decode CBOR data: %s", err) + } + } else { + if testDef.expectedDecodeError != nil { + t.Fatalf("did not receive expected decode error: %s", testDef.expectedDecodeError) + } + } + if !reflect.DeepEqual(newObj, testDef.expectedObject) { + t.Fatalf("CBOR did not decode to expected object\n got: %#v\n wanted: %#v", newObj, testDef.expectedObject) + } + } +} + +func TestLazyValueMarshalJSON(t *testing.T) { + for _, testDef := range testDefs { + // Skip test if the CBOR decode is expected to fail + if testDef.expectedDecodeError != nil { + continue + } + cborData, err := hex.DecodeString(testDef.cborHex) + if err != nil { + t.Fatalf("failed to decode CBOR hex: %s", err) + } + var tmpValue cbor.LazyValue + if _, err := cbor.Decode(cborData, &tmpValue); err != nil { + t.Fatalf("failed to decode CBOR data: %s", err) + } + jsonData, err := json.Marshal(&tmpValue) + if err != nil { + t.Fatalf("failed to marshal Value as JSON: %s", err) + } + // We create the wrapper JSON here, since it would otherwise result in the CBOR hex being duplicated in each test definition + fullExpectedJson := fmt.Sprintf( + `{"cbor":"%s","json":%s}`, + strings.ToLower(testDef.cborHex), + testDef.expectedAstJson, + ) + if !test.JsonStringsEqual(jsonData, []byte(fullExpectedJson)) { + t.Fatalf("CBOR did not marshal to expected JSON\n got: %s\n wanted: %s", jsonData, fullExpectedJson) + } + } +} diff --git a/cmd/block-fetch/main.go b/cmd/block-fetch/main.go index c677c971..a8c8fa35 100644 --- a/cmd/block-fetch/main.go +++ b/cmd/block-fetch/main.go @@ -16,6 +16,7 @@ package main import ( "encoding/hex" + "encoding/json" "fmt" "os" @@ -88,8 +89,8 @@ func main() { fmt.Printf("\nTransactions:\n") for _, tx := range block.Transactions() { fmt.Printf(" Hash: %s\n", tx.Hash()) - if tx.Metadata().Value != nil { - fmt.Printf(" Metadata:\n %#v (%x)\n", tx.Metadata().Value, tx.Metadata().Cbor()) + if tx.Metadata().Value() != nil { + fmt.Printf(" Metadata:\n %#v (%x)\n", tx.Metadata().Value(), tx.Metadata().Cbor()) } fmt.Printf(" Inputs:\n") for _, input := range tx.Inputs() { @@ -115,11 +116,11 @@ func main() { } } if output.Datum() != nil { - datumValue, err := output.Datum().Decode() + jsonData, err := json.Marshal(output.Datum()) if err != nil { fmt.Printf(" Datum (hex): %x\n", output.Datum().Cbor()) } else { - fmt.Printf(" Datum: %#v\n", datumValue.Value) + fmt.Printf(" Datum: %s\n", jsonData) } } fmt.Println("") diff --git a/internal/test/helpers.go b/internal/test/helpers.go index d10daeff..eddc2b3b 100644 --- a/internal/test/helpers.go +++ b/internal/test/helpers.go @@ -1,8 +1,11 @@ package test import ( + "bytes" "encoding/hex" + "encoding/json" "fmt" + "reflect" "strings" ) @@ -17,3 +20,23 @@ func DecodeHexString(hexData string) []byte { } return decoded } + +// JsonStringsEqual is a helper function for tests that compares JSON strings. To account for +// differences in whitespace, map key ordering, etc., we unmarshal the JSON strings into +// objects and then compare the objects +func JsonStringsEqual(jsonData1 []byte, jsonData2 []byte) bool { + // Short-circuit for the happy path where they match exactly + if bytes.Compare(jsonData1, jsonData2) == 0 { + return true + } + // Decode provided JSON strings + var tmpObj1 interface{} + if err := json.Unmarshal(jsonData1, &tmpObj1); err != nil { + return false + } + var tmpObj2 interface{} + if err := json.Unmarshal(jsonData2, &tmpObj2); err != nil { + return false + } + return reflect.DeepEqual(tmpObj1, tmpObj2) +} diff --git a/ledger/error.go b/ledger/error.go index 8db36bf1..26eea59f 100644 --- a/ledger/error.go +++ b/ledger/error.go @@ -71,7 +71,7 @@ func (e *GenericError) UnmarshalCBOR(data []byte) error { if _, err := cbor.Decode(data, &tmpValue); err != nil { return err } - e.Value = tmpValue.Value + e.Value = tmpValue.Value() e.Cbor = data return nil } @@ -330,7 +330,7 @@ type OutsideValidityIntervalUtxo struct { } func (e *OutsideValidityIntervalUtxo) Error() string { - validityInterval := e.ValidityInterval.Value.([]interface{}) + validityInterval := e.ValidityInterval.Value().([]interface{}) return fmt.Sprintf("OutsideValidityIntervalUtxo (ValidityInterval { invalidBefore = %v, invalidHereafter = %v }, Slot %d)", validityInterval[0], validityInterval[1], e.Slot) } @@ -389,10 +389,12 @@ func (e *OutputTooSmallUtxo) Error() string { return ret } -type TxOut cbor.Value +type TxOut struct { + cbor.Value +} func (t *TxOut) String() string { - return fmt.Sprintf("TxOut (%v)", t.Value) + return fmt.Sprintf("TxOut (%v)", t.Value.Value()) } type UtxosFailure struct { @@ -411,7 +413,7 @@ type WrongNetwork struct { } func (e *WrongNetwork) Error() string { - return fmt.Sprintf("WrongNetwork (ExpectedNetworkId %d, Addresses (%v))", e.ExpectedNetworkId, e.Addresses.Value) + return fmt.Sprintf("WrongNetwork (ExpectedNetworkId %d, Addresses (%v))", e.ExpectedNetworkId, e.Addresses.Value()) } type WrongNetworkWithdrawal struct { @@ -421,7 +423,7 @@ type WrongNetworkWithdrawal struct { } func (e *WrongNetworkWithdrawal) Error() string { - return fmt.Sprintf("WrongNetworkWithdrawal (ExpectedNetworkId %d, RewardAccounts (%v))", e.ExpectedNetworkId, e.RewardAccounts.Value) + return fmt.Sprintf("WrongNetworkWithdrawal (ExpectedNetworkId %d, RewardAccounts (%v))", e.ExpectedNetworkId, e.RewardAccounts.Value()) } type OutputBootAddrAttrsTooBig struct { @@ -488,7 +490,7 @@ type ScriptsNotPaidUtxo struct { } func (e *ScriptsNotPaidUtxo) Error() string { - return fmt.Sprintf("ScriptsNotPaidUtxo (%v)", e.Value.Value) + return fmt.Sprintf("ScriptsNotPaidUtxo (%v)", e.Value.Value()) } type ExUnitsTooBigUtxo struct { @@ -508,7 +510,7 @@ type CollateralContainsNonADA struct { } func (e *CollateralContainsNonADA) Error() string { - return fmt.Sprintf("CollateralContainsNonADA (%v)", e.Value.Value) + return fmt.Sprintf("CollateralContainsNonADA (%v)", e.Value.Value()) } type WrongNetworkInTxBody struct {