Add support for message-level extra_fields

bq_table.proto

@@ -50,4 +50,10 @@ message BigQueryMessageOptions {
   // If true, BigQuery field names will default to a field's JSON name,
   // not its original/proto field name.
   bool use_json_names = 2;
+
+  // If set, adds defined extra fields to a JSON representation of the message.
+  // Value format: "<field name>:<BigQuery field type>" for basic types
+  // or "<field name>:RECORD:<protobuf type>" for message types.
+  // "NULLABLE" by default, different mode may be set via optional suffix ":<mode>"
+  repeated string extra_fields = 3;
 }

main.go

@@ -284,26 +284,85 @@ func convertField(
 		return field, nil
 	}
 
-	recordType, ok, comments, path := curPkg.lookupType(desc.GetTypeName())
-	if !ok {
-		return nil, fmt.Errorf("no such message type named %s", desc.GetTypeName())
-	}
-	fieldMsgOpts, err := getBigqueryMessageOptions(recordType)
+	fields, err := convertFieldsForType(curPkg, desc.GetTypeName(), parentMessages)
 	if err != nil {
 		return nil, err
 	}
-	field.Fields, err = convertMessageType(curPkg, recordType, fieldMsgOpts, parentMessages, comments, path)
+
+	if len(fields) == 0 { // discard RECORDs that would have zero fields
+		return nil, nil
+	}
+
+	field.Fields = fields
+
+	return field, nil
+}
+
+func convertExtraField(curPkg *ProtoPackage, extraFieldDefinition string, parentMessages map[*descriptor.DescriptorProto]bool) (*Field, error) {
+	parts := strings.Split(extraFieldDefinition, ":")
+	if len(parts) < 2 {
+		return nil, fmt.Errorf("expecting at least 2 parts in extra field definition separated by colon, got %d", len(parts))
+	}
+
+	field := &Field{
+		Name: parts[0],
+		Type: parts[1],
+		Mode: "NULLABLE",
+	}
+
+	modeIndex := 2
+	if field.Type == "RECORD" {
+		modeIndex = 3
+	}
+	if len(parts) > modeIndex {
+		field.Mode = parts[modeIndex]
+	}
+
+	if field.Type != "RECORD" {
+		return field, nil
+	}
+
+	if len(parts) < 3 {
+		return nil, fmt.Errorf("extra field %s has no type defined", field.Type)
+	}
+
+	typeName := parts[2]
+
+	if t, ok := typeFromWKT[typeName]; ok {
+		field.Type = t
+		return field, nil
+	}
+
+	fields, err := convertFieldsForType(curPkg, typeName, parentMessages)
 	if err != nil {
 		return nil, err
 	}
 
-	if len(field.Fields) == 0 { // discard RECORDs that would have zero fields
+	if len(fields) == 0 { // discard RECORDs that would have zero fields
 		return nil, nil
 	}
 
+	field.Fields = fields
+
 	return field, nil
 }
 
+func convertFieldsForType(curPkg *ProtoPackage,
+	typeName string,
+	parentMessages map[*descriptor.DescriptorProto]bool) ([]*Field, error) {
+	recordType, ok, comments, path := curPkg.lookupType(typeName)
+	if !ok {
+		return nil, fmt.Errorf("no such message type named %s", typeName)
+	}
+
+	fieldMsgOpts, err := getBigqueryMessageOptions(recordType)
+	if err != nil {
+		return nil, err
+	}
+
+	return convertMessageType(curPkg, recordType, fieldMsgOpts, parentMessages, comments, path)
+}
+
 func convertMessageType(
 	curPkg *ProtoPackage,
 	msg *descriptor.DescriptorProto,
@@ -335,7 +394,19 @@ func convertMessageType(
 			schema = append(schema, field)
 		}
 	}
+
+	for _, extraField := range opts.GetExtraFields() {
+		field, err := convertExtraField(curPkg, extraField, parentMessages)
+		if err != nil {
+			glog.Errorf("Failed to convert extra field %s in %s: %v", extraField, msg.GetName(), err)
+			return nil, err
+		}
+
+		schema = append(schema, field)
+	}
+
 	parentMessages[msg] = false
+
 	return
 }
 

plugin_test.go

@@ -446,3 +446,61 @@ func TestModes(t *testing.T) {
 			]`,
 		})
 }
+
+func TestExtraFields(t *testing.T) {
+	testConvert(t, `
+			file_to_generate: "foo.proto"
+			proto_file <
+				name: "foo.proto"
+				package: "example_package"
+				message_type <
+					name: "FooProto"
+					field <
+						name: "i1"
+						number: 1
+						type: TYPE_INT32
+						label: LABEL_OPTIONAL
+					>
+					options <
+						[gen_bq_schema.bigquery_opts]: <
+							table_name: "foo_table"
+							extra_fields: [
+								"i2:INTEGER",
+								"i3:STRING:REPEATED",
+								"i4:TIMESTAMP:REQUIRED",
+								"i5:RECORD:example_package.nested2.BarProto",
+								"i6:RECORD:.google.protobuf.DoubleValue:REQUIRED"
+							]
+						>
+					>
+				>
+			>
+			proto_file <
+				name: "bar.proto"
+				package: "example_package.nested2"
+				message_type <
+					name: "BarProto"
+					field < name: "i1" number: 1 type: TYPE_INT32 label: LABEL_OPTIONAL >
+					field < name: "i2" number: 2 type: TYPE_INT32 label: LABEL_OPTIONAL >
+					field < name: "i3" number: 3 type: TYPE_INT32 label: LABEL_OPTIONAL >
+				>
+			>
+		`,
+		map[string]string{
+			"example_package/foo_table.schema": `[
+				{ "name": "i1", "type": "INTEGER", "mode": "NULLABLE" },
+				{ "name": "i2", "type": "INTEGER", "mode": "NULLABLE" },
+				{ "name": "i3", "type": "STRING", "mode": "REPEATED" },
+				{ "name": "i4", "type": "TIMESTAMP", "mode": "REQUIRED" },
+				{
+					"name": "i5", "type": "RECORD", "mode": "NULLABLE",
+					"fields": [
+						{ "name": "i1", "type": "INTEGER", "mode": "NULLABLE" },
+						{ "name": "i2", "type": "INTEGER", "mode": "NULLABLE" },
+						{ "name": "i3", "type": "INTEGER", "mode": "NULLABLE" }
+					]
+				},
+				{ "name": "i6", "type": "FLOAT", "mode": "REQUIRED" }
+			]`,
+		})
+}