Merge branch 'feat/replication-state' into feat/integrate-replication…

…-state

* feat/replication-state:
  chore: update @mapeo/schema to 3.0.0-next.10 (#290)

drizzle/client/0000_fearless_green_goblin.sql → drizzle/client/0000_foamy_madame_hydra.sql

@@ -22,6 +22,7 @@ CREATE TABLE `projectSettings` (
 	`createdBy` text NOT NULL,
 	`updatedAt` text NOT NULL,
 	`links` text NOT NULL,
+	`deleted` integer NOT NULL,
 	`name` text,
 	`defaultPresets` text,
 	`forks` text NOT NULL

drizzle/client/meta/0000_snapshot.json

@@ -1,7 +1,7 @@
 {
   "version": "5",
   "dialect": "sqlite",
-  "id": "e6899762-16e4-4571-92f5-e0e5fdb87e86",
+  "id": "fb87fd88-8355-492e-9515-9aea8f994567",
   "prevId": "00000000-0000-0000-0000-000000000000",
   "tables": {
     "localDeviceInfo": {
@@ -141,6 +141,13 @@
           "notNull": true,
           "autoincrement": false
         },
+        "deleted": {
+          "name": "deleted",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
         "name": {
           "name": "name",
           "type": "text",

drizzle/client/meta/_journal.json

@@ -5,8 +5,8 @@
     {
       "idx": 0,
       "version": "5",
-      "when": 1695224550073,
-      "tag": "0000_fearless_green_goblin",
+      "when": 1695927276027,
+      "tag": "0000_foamy_madame_hydra",
       "breakpoints": true
     }
   ]

drizzle/project/0000_huge_mikhail_rasputin.sql → drizzle/project/0000_brown_living_mummy.sql

@@ -10,6 +10,7 @@ CREATE TABLE `coreOwnership` (
 	`createdBy` text NOT NULL,
 	`updatedAt` text NOT NULL,
 	`links` text NOT NULL,
+	`deleted` integer NOT NULL,
 	`authCoreId` text NOT NULL,
 	`configCoreId` text NOT NULL,
 	`dataCoreId` text NOT NULL,
@@ -30,6 +31,7 @@ CREATE TABLE `deviceInfo` (
 	`createdBy` text NOT NULL,
 	`updatedAt` text NOT NULL,
 	`links` text NOT NULL,
+	`deleted` integer NOT NULL,
 	`name` text NOT NULL,
 	`forks` text NOT NULL
 );
@@ -46,6 +48,7 @@ CREATE TABLE `field` (
 	`createdBy` text NOT NULL,
 	`updatedAt` text NOT NULL,
 	`links` text NOT NULL,
+	`deleted` integer NOT NULL,
 	`tagKey` text NOT NULL,
 	`type` text NOT NULL,
 	`label` text NOT NULL,
@@ -70,6 +73,7 @@ CREATE TABLE `observation` (
 	`createdBy` text NOT NULL,
 	`updatedAt` text NOT NULL,
 	`links` text NOT NULL,
+	`deleted` integer NOT NULL,
 	`lat` real,
 	`lon` real,
 	`refs` text NOT NULL,
@@ -91,6 +95,7 @@ CREATE TABLE `preset` (
 	`createdBy` text NOT NULL,
 	`updatedAt` text NOT NULL,
 	`links` text NOT NULL,
+	`deleted` integer NOT NULL,
 	`name` text NOT NULL,
 	`geometry` text NOT NULL,
 	`tags` text NOT NULL,
@@ -114,6 +119,7 @@ CREATE TABLE `role` (
 	`createdBy` text NOT NULL,
 	`updatedAt` text NOT NULL,
 	`links` text NOT NULL,
+	`deleted` integer NOT NULL,
 	`roleId` text NOT NULL,
 	`fromIndex` real NOT NULL,
 	`forks` text NOT NULL

drizzle/project/meta/0000_snapshot.json

@@ -1,7 +1,7 @@
 {
   "version": "5",
   "dialect": "sqlite",
-  "id": "5078e2b0-6319-4f3c-aee3-b024e8fe3f9a",
+  "id": "ee8a89aa-fd1d-47d8-88db-86f467de1c18",
   "prevId": "00000000-0000-0000-0000-000000000000",
   "tables": {
     "coreOwnership_backlink": {
@@ -72,6 +72,13 @@
           "notNull": true,
           "autoincrement": false
         },
+        "deleted": {
+          "name": "deleted",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
         "authCoreId": {
           "name": "authCoreId",
           "type": "text",
@@ -188,6 +195,13 @@
           "notNull": true,
           "autoincrement": false
         },
+        "deleted": {
+          "name": "deleted",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
         "name": {
           "name": "name",
           "type": "text",
@@ -276,6 +290,13 @@
           "notNull": true,
           "autoincrement": false
         },
+        "deleted": {
+          "name": "deleted",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
         "tagKey": {
           "name": "tagKey",
           "type": "text",
@@ -420,6 +441,13 @@
           "notNull": true,
           "autoincrement": false
         },
+        "deleted": {
+          "name": "deleted",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
         "lat": {
           "name": "lat",
           "type": "real",
@@ -543,6 +571,13 @@
           "notNull": true,
           "autoincrement": false
         },
+        "deleted": {
+          "name": "deleted",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
         "name": {
           "name": "name",
           "type": "text",
@@ -680,6 +715,13 @@
           "notNull": true,
           "autoincrement": false
         },
+        "deleted": {
+          "name": "deleted",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
         "roleId": {
           "name": "roleId",
           "type": "text",

drizzle/project/meta/_journal.json

@@ -5,8 +5,8 @@
     {
       "idx": 0,
       "version": "5",
-      "when": 1695224585287,
-      "tag": "0000_huge_mikhail_rasputin",
+      "when": 1695927172554,
+      "tag": "0000_brown_living_mummy",
       "breakpoints": true
     }
   ]

package.json

@@ -103,7 +103,7 @@
     "@fastify/type-provider-typebox": "^3.3.0",
     "@hyperswarm/secret-stream": "^6.1.2",
     "@mapeo/crypto": "^1.0.0-alpha.8",
-    "@mapeo/schema": "^3.0.0-next.9",
+    "@mapeo/schema": "^3.0.0-next.10",
     "@mapeo/sqlite-indexer": "^1.0.0-alpha.6",
     "@sinclair/typebox": "^0.29.6",
     "b4a": "^1.6.3",

src/blob-store/index.js

@@ -1,6 +1,7 @@
 import Hyperdrive from 'hyperdrive'
 import b4a from 'b4a'
 import util from 'node:util'
+import { discoveryKey } from 'hypercore-crypto'
 import { TypedEmitter } from 'tiny-typed-emitter'
 import { LiveDownload } from './live-download.js'
 
@@ -28,7 +29,7 @@ class ErrNotFound extends Error {
 }
 
 export class BlobStore {
-  /** @type {Map<string, Hyperdrive>} Indexed by hex-encoded key */
+  /** @type {Map<string, Hyperdrive>} Indexed by hex-encoded discovery key */
   #hyperdrives = new Map()
   #writer
   /**
@@ -48,17 +49,20 @@ export class BlobStore {
     for (const { key } of blobIndexCores) {
       // @ts-ignore - we know pretendCorestore is not actually a Corestore
       const drive = new Hyperdrive(corestore, key)
-      this.#hyperdrives.set(key.toString('hex'), drive)
+      // We use the discovery key to derive the id for a drive
+      this.#hyperdrives.set(getDiscoveryId(key), drive)
       if (key.equals(writerKey)) {
         this.#writer = proxyProps(drive, { key: writerKey })
       }
     }
     coreManager.on('add-core', ({ key, namespace }) => {
       if (namespace !== 'blobIndex') return
-      if (this.#hyperdrives.has(key.toString('hex'))) return
+      // We use the discovery key to derive the id for a drive
+      const driveId = getDiscoveryId(key)
+      if (this.#hyperdrives.has(driveId)) return
       // @ts-ignore - we know pretendCorestore is not actually a Corestore
       const drive = new Hyperdrive(corestore, key)
-      this.#hyperdrives.set(key.toString('hex'), drive)
+      this.#hyperdrives.set(driveId, drive)
       this.#driveEmitter.emit('add-drive', drive)
     })
     // This shouldn't happen, but this check ensures this.#writer is typed to exist
@@ -67,11 +71,11 @@ export class BlobStore {
   }
 
   get writerDriveId() {
-    return this.#writer.key.toString('hex')
+    return getDiscoveryId(this.#writer.key)
   }
 
   /**
-   * @param {string} driveId
+   * @param {string} driveId hex-encoded discovery key
    */
   #getDrive(driveId) {
     const drive = this.#hyperdrives.get(driveId)
@@ -134,7 +138,7 @@ export class BlobStore {
   /**
    * Optimization for creating the blobs read stream when you have
    * previously read the entry from Hyperdrive using `drive.entry`
-   * @param {BlobId['driveId']} driveId Hyperdrive drive id
+   * @param {BlobId['driveId']} driveId Hyperdrive drive discovery id
    * @param {import('hyperdrive').HyperdriveEntry} entry Hyperdrive entry
    * @param {object} [options]
    * @param {boolean} [options.wait=false] Set to `true` to wait for a blob to download, otherwise will throw if blob is not available locally
@@ -177,12 +181,12 @@ export class BlobStore {
    * @param {Buffer} blob
    * @param {object} [options]
    * @param {{mimeType: string}} [options.metadata] Metadata to store with the blob
-   * @returns {Promise<string>} public key as hex string of hyperdrive where blob is stored
+   * @returns {Promise<string>} discovery key as hex string of hyperdrive where blob is stored
    */
   async put({ type, variant, name }, blob, options) {
     const path = makePath({ type, variant, name })
     await this.#writer.put(path, blob, options)
-    return this.#writer.key.toString('hex')
+    return this.writerDriveId
   }
 
   /**
@@ -193,7 +197,9 @@ export class BlobStore {
   createWriteStream({ type, variant, name }, options) {
     const path = makePath({ type, variant, name })
     const stream = this.#writer.createWriteStream(path, options)
-    return proxyProps(stream, { driveId: this.#writer.key.toString('hex') })
+    return proxyProps(stream, {
+      driveId: this.writerDriveId,
+    })
   }
 
   /**
@@ -304,3 +310,11 @@ class PretendCorestore {
   /** no-op */
   close() {}
 }
+
+/**
+ * @param {Buffer} key Public key of hypercore
+ * @returns {string} Hex-encoded string of derived discovery key
+ */
+function getDiscoveryId(key) {
+  return discoveryKey(key).toString('hex')
+}

src/core-manager/index.js

@@ -214,6 +214,19 @@ export class CoreManager extends TypedEmitter {
     return coreRecord && coreRecord.core
   }
 
+  /**
+   * Get a core by its discovery key
+   *
+   * @param {Buffer} discoveryKey
+   * @returns {Core | undefined}
+   */
+  getCoreByDiscoveryKey(discoveryKey) {
+    const coreRecord = this.#coreIndex.getByDiscoveryId(
+      discoveryKey.toString('hex')
+    )
+    return coreRecord && coreRecord.core
+  }
+
   /**
    * Close all open cores and end any replication streams
    * TODO: gracefully close replication streams

src/core-ownership.js

@@ -7,6 +7,7 @@ import sodium from 'sodium-universal'
 import { kTable, kSelect, kCreateWithDocId } from './datatype/index.js'
 import { eq, or } from 'drizzle-orm'
 import mapObject from 'map-obj'
+import { discoveryKey } from 'hypercore-crypto'
 
 /**
  * @typedef {import('./types.js').CoreOwnershipWithSignatures} CoreOwnershipWithSignatures
@@ -97,8 +98,10 @@ export class CoreOwnership {
  * @param {import('@mapeo/schema').VersionIdObject} version
  * @returns {import('@mapeo/schema').CoreOwnership}
  */
-export function mapAndValidateCoreOwnership(doc, { coreKey }) {
-  if (doc.authCoreId !== coreKey.toString('hex')) {
+export function mapAndValidateCoreOwnership(doc, { coreDiscoveryKey }) {
+  if (
+    !coreDiscoveryKey.equals(discoveryKey(Buffer.from(doc.authCoreId, 'hex')))
+  ) {
     throw new Error('Invalid coreOwnership record: mismatched authCoreId')
   }
   if (!verifyCoreOwnership(doc)) {