feat!: standarize refs across all docs (#199)

**NOTE**: some fields in schemas have changed in name and shape (new shape is always `proto/versionId/v1.proto` for reference): * Preset: `fieldIds` -> `fieldRefs` * Preset: `iconId` -> `iconRef` * Track: `refs` -> `observationRefs` * Translation: `docIdRef` and `schemaNameRef` now are inside `docRef` (`schemaNameRef` now being called `type`) * Translation: `fieldRef` -> `propertyRef` (to avoid confusion since its not related to `Field` at all) * standarize refs accross all docs * * add `versionId` proto, use it on every ref.versionId * update ref fields and use a more specific name * use ref.type only on translation * remove unnecessary checks * fix types and encode/decode * add checks for repeated fields * fix tests * add versionId/v1.proto * use similar error messages when missing `versionId` * remove bad-docs tests since they don't make sense anymore --------- Co-authored-by: Tomás Ciccola <tciccola@digital-democracy.com>
digidem · Jul 31, 2024 · d4e000f · d4e000f
1 parent 37bc87d
commit d4e000f
Show file tree

Hide file tree

Showing 14 changed files with 196 additions and 107 deletions.
diff --git a/proto/common/v1.proto b/proto/common/v1.proto
@@ -3,15 +3,12 @@ package mapeo;
 
 import "google/protobuf/timestamp.proto";
 import "options.proto";
+import "versionId/v1.proto";
 
 message Common_1 {
   // 32-byte random generated number
   optional bytes docId = 1 [(required) = true];
-  message Link {
-    bytes coreDiscoveryKey = 1;
-    int32 index = 2;
-  }
-  repeated Link links = 2;
+  repeated VersionId_1 links = 2;
   google.protobuf.Timestamp createdAt = 3 [(required) = true];
   google.protobuf.Timestamp updatedAt = 4 [(required) = true];
   // 32-byte hash of the discovery key of a core

diff --git a/proto/preset/v1.proto b/proto/preset/v1.proto
@@ -5,6 +5,7 @@ import "google/protobuf/struct.proto";
 import "google/protobuf/timestamp.proto";
 import "tags/v1.proto";
 import "common/v1.proto";
+import "versionId/v1.proto";
 import "options.proto";
 
 message Preset_1 {
@@ -15,6 +16,15 @@ message Preset_1 {
   Common_1 common = 1;
 
   string name = 5;
+  repeated Geometry geometry = 6;
+  map<string, TagValue_1> tags = 7;
+  map<string, TagValue_1> addTags = 8;
+  map<string, TagValue_1> removeTags = 9;
+  repeated FieldRef fieldRefs = 10;
+  optional IconRef iconRef = 11;
+  repeated string terms = 12;
+  string color = 13;
+
   enum Geometry {
     geometry_unspecified = 0;
     point = 1;
@@ -23,12 +33,15 @@ message Preset_1 {
     area = 4;
     relation = 5;
   }
-  repeated Geometry geometry = 6;
-  map<string, TagValue_1> tags = 7;
-  map<string, TagValue_1> addTags = 8;
-  map<string, TagValue_1> removeTags = 9;
-  repeated bytes fieldIds = 10;
-  optional bytes iconId = 11;
-  repeated string terms = 12;
-  string color = 13;
+
+
+  message FieldRef {
+    bytes docId = 1;
+    VersionId_1 versionId = 2;
+  }
+
+  message IconRef {
+    bytes docId = 1;
+    VersionId_1 versionId = 2;
+  }
 }
diff --git a/proto/track/v1.proto b/proto/track/v1.proto
@@ -6,6 +6,7 @@ import "google/protobuf/struct.proto";
 import "tags/v1.proto";
 import "common/v1.proto";
 import "options.proto";
+import "versionId/v1.proto";
 
 message Track_1 {
   // **DO NOT CHANGE dataTypeId** generated with `openssl rand -hex 6`
@@ -14,25 +15,22 @@ message Track_1 {
 
   Common_1 common = 1;
   repeated Position locations = 2;
-  repeated Ref refs = 3;
+  repeated ObservationRef observationRefs = 3;
   repeated Attachment attachments = 4;
   map<string, TagValue_1> tags = 5;
 
-  enum RefType {
-    ref_type_unspecified = 0;
-    observation = 1;
-  }
-  message Ref {
-    bytes id = 1;
-    RefType type = 2;
-  }
-
   enum AttachmentType {
     attachment_type_unspecified = 0;
     photo = 1;
     video = 2;
     audio = 3;
   }
+
+  message ObservationRef {
+    bytes docId = 1;
+    VersionId_1 versionId = 2;
+  }
+
   message Attachment {
     bytes driveDiscoveryId = 1;
     string name = 2;

diff --git a/proto/translation/v1.proto b/proto/translation/v1.proto
@@ -2,6 +2,7 @@ syntax = "proto3";
 package mapeo;
 
 import "common/v1.proto";
+import "versionId/v1.proto";
 import "options.proto";
 
 message Translation_1 {
@@ -10,12 +11,17 @@ message Translation_1 {
   option (schemaName) = "translation";
 
   Common_1 common = 1;
+  DocRef docRef = 2;
 
-  string schemaNameRef = 2;
-  bytes docIdRef = 3;
-  string fieldRef = 4;
-  string languageCode = 5;
-  string regionCode = 6;
-  string message = 7;
+  string propertyRef = 3;
+  string languageCode = 4;
+  string regionCode = 5;
+  string message = 6;
+
+  message DocRef {
+    bytes docId = 1;
+    VersionId_1 versionId = 2;
+    string type = 3;
+  }
 
 }
diff --git a/proto/versionId/v1.proto b/proto/versionId/v1.proto
@@ -0,0 +1,7 @@
+syntax = "proto3";
+package mapeo;
+
+message VersionId_1 {
+  bytes coreDiscoveryKey = 1;
+  int32 index = 2;
+}
diff --git a/schema/preset/v1.json b/schema/preset/v1.json
@@ -76,17 +76,24 @@
       "description": "Tags that are removed when changing to another preset (default is the same value as 'addTags' which in turn defaults to 'tags')",
       "$ref": "#/definitions/tags"
     },
-    "fieldIds": {
-      "description": "hex-encoded string. IDs of fields to displayed to the user when the preset is created or edited",
+    "fieldRefs": {
       "type": "array",
+      "description": "References to any fields that this preset is related to.",
       "items": {
-        "type": "string"
+        "type": "object",
+        "properties": {
+          "docId": {
+            "description": "hex-encoded id of the element that this observation references",
+            "type": "string"
+          },
+          "versionId": {
+            "description": "core discovery id (hex-encoded 32-byte buffer) and core index number, separated by '/'",
+            "type": "string"
+          }
+        },
+        "required": ["docId", "versionId"]
       }
     },
-    "iconId": {
-      "description": "hex-encoded string. ID of preset icon which represents this preset",
-      "type": "string"
-    },
     "terms": {
       "description": "Synonyms or related terms (used for search)",
       "type": "array",
@@ -99,17 +106,17 @@
       "type": "string",
       "pattern": "^#[a-fA-F0-9]{6}$"
     }
-},
-"required": [
-  "name",
-  "geometry",
-  "tags",
-  "addTags",
-  "removeTags",
-  "fieldIds",
-  "schemaName",
-  "terms",
-  "color"
-],
-"additionalProperties": false
+  },
+  "required": [
+    "name",
+    "geometry",
+    "tags",
+    "addTags",
+    "removeTags",
+    "fieldRefs",
+    "schemaName",
+    "terms",
+    "color"
+  ],
+  "additionalProperties": false
 }
diff --git a/schema/track/v1.json b/schema/track/v1.json
@@ -61,26 +61,22 @@
         "$ref": "#/definitions/position"
       }
     },
-    "refs": {
+    "observationRefs": {
       "type": "array",
       "description": "References to any observations that this track is related to.",
       "items": {
         "type": "object",
         "properties": {
-          "id": {
+          "docId": {
             "description": "hex-encoded id of the element that this track references",
             "type": "string"
           },
-          "type": {
-            "description": "type of the element that this track references",
-            "type": "string",
-            "enum": ["ref_type_unspecified", "observation", "UNRECOGNIZED"],
-            "meta:enum": {
-              "UNRECOGNIZED": "future reference type"
-            }
+          "versionId": {
+            "description": "core discovery id (hex-encoded 32-byte buffer) and core index number, separated by '/'",
+            "type": "string"
           }
         },
-        "required": ["id", "type"]
+        "required": ["docId", "versionId"]
       }
     },
     "attachments": {
@@ -160,6 +156,12 @@
       }
     }
   },
-  "required": ["schemaName", "locations", "tags", "refs", "attachments"],
+  "required": [
+    "schemaName",
+    "locations",
+    "tags",
+    "observationRefs",
+    "attachments"
+  ],
   "additionalProperties": false
 }
diff --git a/schema/translation/v1.json b/schema/translation/v1.json
@@ -4,20 +4,30 @@
   "description": "A translation is a translated message in a single language for a string property/field of any Mapeo record type",
   "title": "translation",
   "type": "object",
-  "properties":{
-    "schemaName":{
-      "type":"string",
-      "const": "translation"
-    },
-    "schemaNameRef": {
+  "properties": {
+    "schemaName": {
       "type": "string",
-      "description": "schema name of record related to the field being translated"
+      "const": "translation"
     },
-    "docIdRef": {
-      "type": "string",
-      "description": "the docId of the record that this field is translated for"
+    "docRef": {
+      "type": "object",
+      "properties": {
+        "docId": {
+          "description": "hex-encoded id of the element that this observation references",
+          "type": "string"
+        },
+        "versionId": {
+          "description": "core discovery id (hex-encoded 32-byte buffer) and core index number, separated by '/'",
+          "type": "string"
+        },
+        "type": {
+          "description": "type of the element that this translation references",
+          "type": "string"
+        }
+      },
+      "required": ["docId", "versionId", "type"]
     },
-    "fieldRef": {
+    "propertyRef": {
       "type": "string",
       "description": "identifier for translated field/property in dot-prop notation"
     },
@@ -26,14 +36,21 @@
       "description": "three-letter ISO 169-3 language code"
     },
     "regionCode": {
-        "type": "string",
-        "description": "two-letter country code from ISO 3166-1 alpha-2 or a three-digit code from UN M.49 for geographical regions"
-},
+      "type": "string",
+      "description": "two-letter country code from ISO 3166-1 alpha-2 or a three-digit code from UN M.49 for geographical regions"
+    },
     "message": {
       "type": "string",
       "description": "the translated string"
     }
   },
-  "required": ["schemaName", "schemaNameRef", "docIdRef", "fieldRef", "languageCode", "regionCode", "message"],
+  "required": [
+    "schemaName",
+    "docRef",
+    "propertyRef",
+    "languageCode",
+    "regionCode",
+    "message"
+  ],
   "additionalProperties": false
 }
diff --git a/scripts/lib/parse-config.js b/scripts/lib/parse-config.js
@@ -6,7 +6,7 @@ import schema from 'protocol-buffers-schema'
 import { capitalize, PROJECT_ROOT } from './utils.js'
 
 // These messages are embedded in others and do not define Mapeo data types
-const EMBEDDED_MESSAGES = ['tags', 'common']
+const EMBEDDED_MESSAGES = ['tags', 'common', 'versionId']
 
 // We avoid creating data type IDs that match these, since blobs (e.g. icons)
 // can be stored in Mapeo hypercores, and we want to avoid trying to parse a

diff --git a/src/lib/decode-conversions.ts b/src/lib/decode-conversions.ts
@@ -122,11 +122,16 @@ export const convertPreset: ConvertFunction<'preset'> = (
     ...jsonSchemaCommon,
     ...rest,
     geometry,
-    iconId: rest.iconId ? rest.iconId.toString('hex') : undefined,
     tags: convertTags(rest.tags),
     addTags: convertTags(rest.addTags),
     removeTags: convertTags(rest.removeTags),
-    fieldIds: rest.fieldIds.map((id) => id.toString('hex')),
+    fieldRefs: rest.fieldRefs.map(({ docId, versionId }) => {
+      if (!versionId) throw new Error('missing fieldRef.versionId for preset')
+      return {
+        docId: docId.toString('hex'),
+        versionId: getVersionId(versionId),
+      }
+    }),
   }
 }
 
@@ -201,25 +206,38 @@ export const convertTranslation: ConvertFunction<'translation'> = (
 ) => {
   const { common, schemaVersion, ...rest } = message
   const jsonSchemaCommon = convertCommon(common, versionObj)
+  if (!message.docRef) throw new Error('missing docRef for translation')
+  if (!message.docRef.versionId)
+    throw new Error('missing docRef.versionId for translation')
   return {
     ...jsonSchemaCommon,
     ...rest,
-    docIdRef: message.docIdRef.toString('hex'),
+    docRef: {
+      docId: message.docRef.docId.toString('hex'),
+      versionId: getVersionId(message.docRef.versionId),
+      type: message.docRef.type,
+    },
   }
 }
 
 export const convertTrack: ConvertFunction<'track'> = (message, versionObj) => {
   const { common, schemaVersion, ...rest } = message
   const jsonSchemaCommon = convertCommon(common, versionObj)
   const locations = message.locations.map(convertTrackPosition)
-  const refs = message.refs.map(({ id, type }) => ({
-    id: id.toString('hex'),
-    type,
-  }))
+  const observationRefs = message.observationRefs.map(
+    ({ docId, versionId }) => {
+      if (!versionId)
+        throw new Error('missing observationRef.versionId from track')
+      return {
+        docId: docId.toString('hex'),
+        versionId: getVersionId(versionId),
+      }
+    }
+  )
   return {
     ...jsonSchemaCommon,
     ...rest,
-    refs,
+    observationRefs,
     locations,
     attachments: message.attachments.map(convertAttachment),
     tags: convertTags(message.tags),