Skip to content

Commit

Permalink
TANGO-2130 : Fixes after review
Browse files Browse the repository at this point in the history
  • Loading branch information
@adamallegro
adamallegro committed Sep 27, 2024
1 parent 03c5d90 commit cb8147e
Show file tree
Hide file tree
Showing 5 changed files with 63 additions and 109 deletions.
28 changes: 1 addition & 27 deletions ...mes/consumers/consumer/sender/googlebigquery/avro/GoogleBigQueryAvroToProtoConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,32 +54,6 @@ private void convertSimpleField(GenericRecord inputObject, Descriptors.FieldDesc
}
}

private Integer toInteger(Object object) {
if (object instanceof Number) {
return ((Number) object).intValue();
} else if (object instanceof String) {
return Integer.parseInt((String) object);
} else if (object instanceof Boolean) {
return (Boolean) object ? 1 : 0;
} else {
return (int) object;
}

}

private Long toLong(Object object) {
if (object instanceof Number) {
return ((Number) object).longValue();
} else if (object instanceof String) {
return Long.parseLong((String) object);
} else if (object instanceof Boolean) {
return (Boolean) object ? 1l : 0l;
} else {
return (long) object;
}

}


private <T extends Number> T toNumber(Object object, Class<T> clazz) {
if (object instanceof Number) {
Expand Down Expand Up @@ -127,7 +101,7 @@ private Object toProtobufValue(Descriptors.FieldDescriptor fieldDescriptor, Obje
if (value instanceof Boolean) {
return value;
} else if (value instanceof Number) {
return ((Number) value).intValue() > 0;
return ((Number) value).intValue() != 0;
} else
return Boolean.parseBoolean(value.toString());

Expand Down
19 changes: 0 additions & 19 deletions ...ovy/pl/allegro/tech/hermes/consumers/consumer/sender/googlebigquery/avro/AvroTrait.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,7 @@ import org.apache.avro.io.DatumReader
import org.apache.avro.io.DecoderFactory

trait AvroTrait {

Schema getSchemaFromResources(String fileName) {
return new Schema.Parser().parse(AvroTrait.class.getClassLoader().getResourceAsStream("schemas/${fileName}.avsc"))
}

GenericRecord getGenericRecordFromFile(String payloadFile, Schema schema) {
def uri = this.class.getClassLoader().getResource(payloadFile).toURI()
def avroFile = new File(uri)
DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema, schema)

for (int i = 0; i <= 5; i++) { // don't ask
try {
byte[] bs = Arrays.copyOfRange(avroFile.bytes, i, avroFile.bytes.length)
def binaryDecoder = DecoderFactory.get().binaryDecoder(bs, null)

return reader.read(null, binaryDecoder)
} catch (ArrayIndexOutOfBoundsException|AvroRuntimeException|IOException ignored) {
}
}

throw new RuntimeException("Cannot decode")
}
}
88 changes: 44 additions & 44 deletions ...umers/consumer/sender/googlebigquery/avro/conversions/ConvertAndCastPrimitivesTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import pl.allegro.tech.hermes.consumers.consumer.sender.googlebigquery.avro.desc
import pl.allegro.tech.hermes.consumers.consumer.sender.googlebigquery.avro.descriptor.StringProto
import spock.lang.Specification

class ConvertAndCastPrimitivesTest extends Specification implements AvroTrait{
class ConvertAndCastPrimitivesTest extends Specification implements AvroTrait {

@Test

Expand All @@ -39,56 +39,56 @@ class ConvertAndCastPrimitivesTest extends Specification implements AvroTrait{
if (field.getDefaultValue() != avroValue) {
record.hasField(fieldName) == message.hasField(field)
}
transformResult(converted_value) == expectedProtoValue
converted_value == expectedProtoValue


where:
suite | avroType | protoDescriptor | avroValue | expectedProtoValue | transformResult
"primitives" | "string" | StringProto.PrimitivesString.getDescriptor() | "value" | "value" | { c -> c }
"primitives" | "string" | StringProto.PrimitivesString.getDescriptor() | 12 | "12" | { c -> c }
"primitives" | "string" | StringProto.PrimitivesString.getDescriptor() | 12l | "12" | { c -> c }
"primitives" | "string" | StringProto.PrimitivesString.getDescriptor() | 12f | "12.0" | { c -> c }
"primitives" | "string" | StringProto.PrimitivesString.getDescriptor() | 12d | "12.0" | { c -> c }
suite | avroType | protoDescriptor | avroValue | expectedProtoValue
"primitives" | "string" | StringProto.PrimitivesString.getDescriptor() | "value" | "value"
"primitives" | "string" | StringProto.PrimitivesString.getDescriptor() | 12 | "12"
"primitives" | "string" | StringProto.PrimitivesString.getDescriptor() | 12l | "12"
"primitives" | "string" | StringProto.PrimitivesString.getDescriptor() | 12f | "12.0"
"primitives" | "string" | StringProto.PrimitivesString.getDescriptor() | 12d | "12.0"

"primitives" | "int" | Int32Proto.PrimitivesInt32.getDescriptor() | 12 | 12 | { c -> c }
"primitives" | "long" | Int32Proto.PrimitivesInt32.getDescriptor() | 12l | 12 | { c -> c }
"primitives" | "float" | Int32Proto.PrimitivesInt32.getDescriptor() | 12f | 12 | { c -> c }
"primitives" | "double" | Int32Proto.PrimitivesInt32.getDescriptor() | 12d | 12 | { c -> c }
"primitives" | "string" | Int32Proto.PrimitivesInt32.getDescriptor() | "12" | 12 | { c -> c }
"primitives" | "boolean" | Int32Proto.PrimitivesInt32.getDescriptor() | true | 1 | { c -> c }
"primitives" | "boolean" | Int32Proto.PrimitivesInt32.getDescriptor() | false | 0 | { c -> c }
"primitives" | "int" | Int32Proto.PrimitivesInt32.getDescriptor() | 12 | 12
"primitives" | "long" | Int32Proto.PrimitivesInt32.getDescriptor() | 12l | 12
"primitives" | "float" | Int32Proto.PrimitivesInt32.getDescriptor() | 12f | 12
"primitives" | "double" | Int32Proto.PrimitivesInt32.getDescriptor() | 12d | 12
"primitives" | "string" | Int32Proto.PrimitivesInt32.getDescriptor() | "12" | 12
"primitives" | "boolean" | Int32Proto.PrimitivesInt32.getDescriptor() | true | 1
"primitives" | "boolean" | Int32Proto.PrimitivesInt32.getDescriptor() | false | 0

"primitives" | "int" | Int64Proto.PrimitivesInt64.getDescriptor() | 12 | 12l | { c -> c }
"primitives" | "long" | Int64Proto.PrimitivesInt64.getDescriptor() | 12l | 12l | { c -> c }
"primitives" | "float" | Int64Proto.PrimitivesInt64.getDescriptor() | 12f | 12l | { c -> c }
"primitives" | "double" | Int64Proto.PrimitivesInt64.getDescriptor() | 12d | 12l | { c -> c }
"primitives" | "string" | Int64Proto.PrimitivesInt64.getDescriptor() | "12.0" | 12l | { c -> c }
"primitives" | "boolean" | Int64Proto.PrimitivesInt64.getDescriptor() | true | 1l | { c -> c }
"primitives" | "boolean" | Int64Proto.PrimitivesInt64.getDescriptor() | false | 0l | { c -> c }
"primitives" | "int" | Int64Proto.PrimitivesInt64.getDescriptor() | 12 | 12l
"primitives" | "long" | Int64Proto.PrimitivesInt64.getDescriptor() | 12l | 12l
"primitives" | "float" | Int64Proto.PrimitivesInt64.getDescriptor() | 12f | 12l
"primitives" | "double" | Int64Proto.PrimitivesInt64.getDescriptor() | 12d | 12l
"primitives" | "string" | Int64Proto.PrimitivesInt64.getDescriptor() | "12.0" | 12l
"primitives" | "boolean" | Int64Proto.PrimitivesInt64.getDescriptor() | true | 1l
"primitives" | "boolean" | Int64Proto.PrimitivesInt64.getDescriptor() | false | 0l

"primitives" | "int" | FloatProto.PrimitivesFloat.getDescriptor() | 12 | 12.0f | { c -> c }
"primitives" | "long" | FloatProto.PrimitivesFloat.getDescriptor() | 12l | 12.0f | { c -> c }
"primitives" | "float" | FloatProto.PrimitivesFloat.getDescriptor() | 12.234f | 12.234f | { c -> c }
"primitives" | "double" | FloatProto.PrimitivesFloat.getDescriptor() | 12.234d | 12.234f | { c -> c }
"primitives" | "string" | FloatProto.PrimitivesFloat.getDescriptor() | "12.234" | 12.234f | { c -> c }
"primitives" | "boolean" | FloatProto.PrimitivesFloat.getDescriptor() | true | 1f | { c -> c }
"primitives" | "boolean" | FloatProto.PrimitivesFloat.getDescriptor() | false | 0f | { c -> c }
"primitives" | "int" | FloatProto.PrimitivesFloat.getDescriptor() | 12 | 12.0f
"primitives" | "long" | FloatProto.PrimitivesFloat.getDescriptor() | 12l | 12.0f
"primitives" | "float" | FloatProto.PrimitivesFloat.getDescriptor() | 12.234f | 12.234f
"primitives" | "double" | FloatProto.PrimitivesFloat.getDescriptor() | 12.234d | 12.234f
"primitives" | "string" | FloatProto.PrimitivesFloat.getDescriptor() | "12.234" | 12.234f
"primitives" | "boolean" | FloatProto.PrimitivesFloat.getDescriptor() | true | 1f
"primitives" | "boolean" | FloatProto.PrimitivesFloat.getDescriptor() | false | 0f

"primitives" | "int" | DoubleProto.PrimitivesDouble.getDescriptor() | 12 | 12.0d | { c -> c }
"primitives" | "long" | DoubleProto.PrimitivesDouble.getDescriptor() | 12l | 12.0d | { c -> c }
"primitives" | "float" | DoubleProto.PrimitivesDouble.getDescriptor() | 12.5f | 12.5d | { c -> c }
"primitives" | "double" | DoubleProto.PrimitivesDouble.getDescriptor() | 12.234d | 12.234d | { c -> c }
"primitives" | "string" | DoubleProto.PrimitivesDouble.getDescriptor() | "12.234" | 12.234d | { c -> c }
"primitives" | "boolean" | DoubleProto.PrimitivesDouble.getDescriptor() | true | 1d | { c -> c }
"primitives" | "boolean" | DoubleProto.PrimitivesDouble.getDescriptor() | false | 0d | { c -> c }
"primitives" | "int" | DoubleProto.PrimitivesDouble.getDescriptor() | 12 | 12.0d
"primitives" | "long" | DoubleProto.PrimitivesDouble.getDescriptor() | 12l | 12.0d
"primitives" | "float" | DoubleProto.PrimitivesDouble.getDescriptor() | 12.5f | 12.5d
"primitives" | "double" | DoubleProto.PrimitivesDouble.getDescriptor() | 12.234d | 12.234d
"primitives" | "string" | DoubleProto.PrimitivesDouble.getDescriptor() | "12.234" | 12.234d
"primitives" | "boolean" | DoubleProto.PrimitivesDouble.getDescriptor() | true | 1d
"primitives" | "boolean" | DoubleProto.PrimitivesDouble.getDescriptor() | false | 0d

"primitives" | "boolean" | BoolProto.PrimitivesBool.getDescriptor() | true | true | { c -> c }
"primitives" | "boolean" | BoolProto.PrimitivesBool.getDescriptor() | false | false | { c -> c }
"primitives" | "string" | BoolProto.PrimitivesBool.getDescriptor() | "true" | true | { c -> c }
"primitives" | "string" | BoolProto.PrimitivesBool.getDescriptor() | "false" | false | { c -> c }
"primitives" | "string" | BoolProto.PrimitivesBool.getDescriptor() | "True" | true | { c -> c }
"primitives" | "string" | BoolProto.PrimitivesBool.getDescriptor() | "False" | false | { c -> c }
"primitives" | "boolean" | BoolProto.PrimitivesBool.getDescriptor() | 1 | true | { c -> c }
"primitives" | "boolean" | BoolProto.PrimitivesBool.getDescriptor() | 0 | false | { c -> c }
"primitives" | "boolean" | BoolProto.PrimitivesBool.getDescriptor() | true | true
"primitives" | "boolean" | BoolProto.PrimitivesBool.getDescriptor() | false | false
"primitives" | "string" | BoolProto.PrimitivesBool.getDescriptor() | "true" | true
"primitives" | "string" | BoolProto.PrimitivesBool.getDescriptor() | "false" | false
"primitives" | "string" | BoolProto.PrimitivesBool.getDescriptor() | "True" | true
"primitives" | "string" | BoolProto.PrimitivesBool.getDescriptor() | "False" | false
"primitives" | "boolean" | BoolProto.PrimitivesBool.getDescriptor() | 1 | true
"primitives" | "boolean" | BoolProto.PrimitivesBool.getDescriptor() | 0 | false
}
}
16 changes: 8 additions & 8 deletions ...umers/consumer/sender/googlebigquery/avro/conversions/ConvertArrayOfPrimitivesTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import spock.lang.Specification

import java.nio.ByteBuffer

class ConvertArrayOfPrimitivesTest extends Specification implements AvroTrait {
class ConvertArrayOfPrimitivesTest extends Specification implements AvroTrait {

@Test
void 'should convert array primitive types'() {
Expand Down Expand Up @@ -90,14 +90,14 @@ class ConvertArrayOfPrimitivesTest extends Specification implements AvroTrait {

where:
suite | avroType | protoDescriptor | avroValue | expectedProtoValue | transformResult
"map-primitives" | "string" | StringToStringProto.PrimitivesStringToString.getDescriptor() | [:] | "[]" | { c -> c }
"map-primitives" | "int" | StringToInt32Proto.PrimitivesStringToInt32.getDescriptor() | [:] | "[]" | { c -> c }
"map-primitives" | "long" | StringToInt64Proto.PrimitivesStringToInt64.getDescriptor() | [:] | "[]" | { c -> c }
"map-primitives" | "string" | StringToStringProto.PrimitivesStringToString.getDescriptor() | [:] | "[]" | { c -> c }
"map-primitives" | "int" | StringToInt32Proto.PrimitivesStringToInt32.getDescriptor() | [:] | "[]" | { c -> c }
"map-primitives" | "long" | StringToInt64Proto.PrimitivesStringToInt64.getDescriptor() | [:] | "[]" | { c -> c }
"map-primitives" | "boolean" | StringToInt64Proto.PrimitivesStringToInt64.getDescriptor() | [:] | "[]" | { c -> c }
"map-primitives" | "boolean" | StringToBoolProto.PrimitivesStringToBool.getDescriptor() | [:] | "[]" | { c -> c }
"map-primitives" | "bytes" | StringToBytesProto.PrimitivesStringToBytes.getDescriptor() | [:] | "[]" | { c -> c }
"map-primitives" | "float" | StringToFloatProto.PrimitivesStringToFloat.getDescriptor() | [:] | "[]" | { c -> c }
"map-primitives" | "double" | StringToDoubleProto.PrimitivesStringToDouble.getDescriptor() | [:] | "[]" | { c -> c }
"map-primitives" | "boolean" | StringToBoolProto.PrimitivesStringToBool.getDescriptor() | [:] | "[]" | { c -> c }
"map-primitives" | "bytes" | StringToBytesProto.PrimitivesStringToBytes.getDescriptor() | [:] | "[]" | { c -> c }
"map-primitives" | "float" | StringToFloatProto.PrimitivesStringToFloat.getDescriptor() | [:] | "[]" | { c -> c }
"map-primitives" | "double" | StringToDoubleProto.PrimitivesStringToDouble.getDescriptor() | [:] | "[]" | { c -> c }
"map-primitives" | "string" | StringToStringProto.PrimitivesStringToString.getDescriptor() | ["a": "a", "b": "b", "c": "c"] | "[key: \"a\" value: \"a\" , key: \"b\" value: \"b\" , key: \"c\" value: \"c\" ]" | { c -> c }
"map-primitives" | "int" | StringToInt32Proto.PrimitivesStringToInt32.getDescriptor() | ["a": 1, "b": 2, "c": 3] | "[key: \"a\" value: 1 , key: \"b\" value: 2 , key: \"c\" value: 3 ]" | { c -> c }
"map-primitives" | "long" | StringToInt64Proto.PrimitivesStringToInt64.getDescriptor() | ["a": 4l, "b": 5l, "c": 6l] | "[key: \"a\" value: 4 , key: \"b\" value: 5 , key: \"c\" value: 6 ]" | { c -> c }
Expand Down
Loading

0 comments on commit cb8147e

Please sign in to comment.