Protocol definitions for DwnDidStore, DwnKeyStore and `DwnIdentit…

…yStore` (#743) * protocol defined key/identity/did stores * skipped tech-preview for test coverage --------- Co-authored-by: Henry Tsai <[email protected]>
decentralized-identity · Jul 12, 2024 · 44604a4 · 44604a4
1 parent 60db3dc
commit 44604a4
Show file tree

Hide file tree

Showing 13 changed files with 351 additions and 36 deletions.
diff --git a/.changeset/shiny-students-enjoy.md b/.changeset/shiny-students-enjoy.md
@@ -0,0 +1,8 @@
+---
+"@web5/agent": minor
+"@web5/identity-agent": minor
+"@web5/proxy-agent": minor
+"@web5/user-agent": minor
+---
+
+Protocol driven identity, key and did storage
diff --git a/packages/agent/src/store-data-protocols.ts b/packages/agent/src/store-data-protocols.ts
@@ -0,0 +1,40 @@
+import type { ProtocolDefinition } from '@tbd54566975/dwn-sdk-js';
+
+export const IdentityProtocolDefinition: ProtocolDefinition = {
+  protocol  : 'http://identity.foundation/protocols/web5/identity-store',
+  published : false,
+  types     : {
+    portableDid: {
+      schema      : 'https://identity.foundation/schemas/web5/portable-did',
+      dataFormats : [
+        'application/json'
+      ]
+    },
+    identityMetadata: {
+      schema      : 'https://identity.foundation/schemas/web5/identity-metadata',
+      dataFormats : [
+        'application/json'
+      ]
+    }
+  },
+  structure: {
+    portableDid      : {},
+    identityMetadata : {}
+  }
+};
+
+export const JwkProtocolDefinition: ProtocolDefinition = {
+  protocol  : 'http://identity.foundation/protocols/web5/jwk-store',
+  published : false,
+  types     : {
+    privateJwk: {
+      schema      : 'https://identity.foundation/schemas/web5/private-jwk',
+      dataFormats : [
+        'application/json'
+      ]
+    },
+  },
+  structure: {
+    privateJwk: {}
+  }
+};
diff --git a/packages/agent/src/store-data.ts b/packages/agent/src/store-data.ts
@@ -8,6 +8,7 @@ import type { Web5PlatformAgent } from './types/agent.js';
 import { TENANT_SEPARATOR } from './utils-internal.js';
 import { getDataStoreTenant } from './utils-internal.js';
 import { DwnInterface } from './types/dwn.js';
+import { ProtocolDefinition } from '@tbd54566975/dwn-sdk-js';
 
 export type DataStoreTenantParams = {
   agent: Web5PlatformAgent;
@@ -59,12 +60,22 @@ export class DwnDataStore<TStoreObject extends Record<string, any> = Jwk> implem
    */
   protected _index = new TtlCache<string, string>({ ttl: ms('2 hours'), max: 1000 });
 
+  /**
+   * Cache of tenant DIDs that have been initialized with the protocol.
+   * This is used to avoid redundant protocol initialization requests.
+   */
+  protected _protocolInitializedCache: TtlCache<string, boolean> = new TtlCache({ ttl: ms('1 hour'), max: 1000 });
+
+  /**
+   * The protocol assigned to this storage instance.
+   */
+  protected _recordProtocolDefinition!: ProtocolDefinition;
+
   /**
    * Properties to use when writing and querying records with the DWN store.
    */
   protected _recordProperties = {
-    dataFormat : 'application/json',
-    schema     : 'https://identity.foundation/schemas/web5/private-jwk'
+    dataFormat: 'application/json',
   };
 
   public async delete({ id, agent, tenant }: DataStoreDeleteParams): Promise<boolean> {
@@ -128,6 +139,9 @@ export class DwnDataStore<TStoreObject extends Record<string, any> = Jwk> implem
     // Determine the tenant identifier (DID) for the set operation.
     const tenantDid = await getDataStoreTenant({ agent, tenant, didUri: id });
 
+    // initialize the storage protocol if not already done
+    await this.initialize({ tenant: tenantDid, agent });
+
     // If enabled, check if a record with the given `id` is already present in the store.
     if (preventDuplicates) {
       // Look up the DWN record ID of the object in the store with the given `id`.
@@ -163,6 +177,39 @@ export class DwnDataStore<TStoreObject extends Record<string, any> = Jwk> implem
     }
   }
 
+  /**
+   * Initialize the relevant protocol for the given tenant.
+   * This confirms that the storage protocol is configured, otherwise it will be installed.
+   */
+  public async initialize({ tenant, agent }: DataStoreTenantParams) {
+    const tenantDid = await getDataStoreTenant({ agent, tenant });
+    if (this._protocolInitializedCache.has(tenantDid)) {
+      return;
+    }
+
+    const { reply: { status, entries }} = await agent.dwn.processRequest({
+      author        : tenantDid,
+      target        : tenantDid,
+      messageType   : DwnInterface.ProtocolsQuery,
+      messageParams : {
+        filter: {
+          protocol: this._recordProtocolDefinition.protocol
+        }
+      },
+    });
+
+    if (status.code !== 200) {
+      throw new Error(`Failed to query for protocols: ${status.code} - ${status.detail}`);
+    }
+
+    if (entries?.length === 0) {
+      // protocol is not installed, install it
+      await this.installProtocol(tenantDid, agent);
+    }
+
+    this._protocolInitializedCache.set(tenantDid, true);
+  }
+
   protected async getAllRecords(_params: {
     agent: Web5PlatformAgent;
     tenantDid: string;
@@ -207,6 +254,24 @@ export class DwnDataStore<TStoreObject extends Record<string, any> = Jwk> implem
     return storeObject;
   }
 
+  /**
+   * Install the protocol for the given tenant using a `ProtocolsConfigure` message.
+   */
+  private async installProtocol(tenant: string, agent: Web5PlatformAgent) {
+    const { reply : { status } } = await agent.dwn.processRequest({
+      author        : tenant,
+      target        : tenant,
+      messageType   : DwnInterface.ProtocolsConfigure,
+      messageParams : {
+        definition: this._recordProtocolDefinition
+      },
+    });
+
+    if (status.code !== 202) {
+      throw new Error(`Failed to install protocol: ${status.code} - ${status.detail}`);
+    }
+  }
+
   private async lookupRecordId({ id, tenantDid, agent }: {
     id: string;
     tenantDid: string;

diff --git a/packages/agent/src/store-did.ts b/packages/agent/src/store-did.ts
@@ -5,20 +5,25 @@ import { Convert } from '@web5/common';
 import type { Web5PlatformAgent } from './types/agent.js';
 import type { AgentDataStore, DataStoreDeleteParams, DataStoreGetParams, DataStoreListParams, DataStoreSetParams } from './store-data.js';
 
-import { TENANT_SEPARATOR } from './utils-internal.js';
 import { DwnInterface } from './types/dwn.js';
+import { IdentityProtocolDefinition } from './store-data-protocols.js';
 import { isPortableDid } from './prototyping/dids/utils.js';
+import { TENANT_SEPARATOR } from './utils-internal.js';
 import { DwnDataStore, InMemoryDataStore } from './store-data.js';
 
 export class DwnDidStore extends DwnDataStore<PortableDid> implements AgentDataStore<PortableDid> {
   protected name = 'DwnDidStore';
 
+  protected _recordProtocolDefinition = IdentityProtocolDefinition;
+
   /**
    * Properties to use when writing and querying DID records with the DWN store.
    */
   protected _recordProperties = {
-    dataFormat : 'application/json',
-    schema     : 'https://identity.foundation/schemas/web5/portable-did'
+    dataFormat   : 'application/json',
+    protocol     : this._recordProtocolDefinition.protocol,
+    protocolPath : 'portableDid',
+    schema       : this._recordProtocolDefinition.types.portableDid.schema,
   };
 
   public async delete(params: DataStoreDeleteParams): Promise<boolean> {

diff --git a/packages/agent/src/store-identity.ts b/packages/agent/src/store-identity.ts
@@ -4,8 +4,9 @@ import type { Web5PlatformAgent } from './types/agent.js';
 import type { IdentityMetadata } from './types/identity.js';
 import type { AgentDataStore, DataStoreDeleteParams, DataStoreGetParams, DataStoreListParams, DataStoreSetParams } from './store-data.js';
 
-import { TENANT_SEPARATOR } from './utils-internal.js';
 import { DwnInterface } from './types/dwn.js';
+import { IdentityProtocolDefinition } from './store-data-protocols.js';
+import { TENANT_SEPARATOR } from './utils-internal.js';
 import { DwnDataStore, InMemoryDataStore } from './store-data.js';
 
 export function isIdentityMetadata(obj: unknown): obj is IdentityMetadata {
@@ -17,12 +18,16 @@ export function isIdentityMetadata(obj: unknown): obj is IdentityMetadata {
 export class DwnIdentityStore extends DwnDataStore<IdentityMetadata> implements AgentDataStore<IdentityMetadata> {
   protected name = 'DwnIdentityStore';
 
+  protected _recordProtocolDefinition = IdentityProtocolDefinition;
+
   /**
    * Properties to use when writing and querying Identity records with the DWN store.
    */
   protected _recordProperties = {
-    dataFormat : 'application/json',
-    schema     : 'https://identity.foundation/schemas/web5/identity-metadata'
+    dataFormat   : 'application/json',
+    protocol     : this._recordProtocolDefinition.protocol,
+    protocolPath : 'identityMetadata',
+    schema       : this._recordProtocolDefinition.types.identityMetadata.schema,
   };
 
   public async delete(params: DataStoreDeleteParams): Promise<boolean> {

diff --git a/packages/agent/src/store-key.ts b/packages/agent/src/store-key.ts
@@ -5,19 +5,24 @@ import { Convert } from '@web5/common';
 
 import type { Web5PlatformAgent } from './types/agent.js';
 
-import { TENANT_SEPARATOR } from './utils-internal.js';
 import { DwnInterface } from './types/dwn.js';
+import { JwkProtocolDefinition } from './store-data-protocols.js';
+import { TENANT_SEPARATOR } from './utils-internal.js';
 import { AgentDataStore, DataStoreDeleteParams, DataStoreGetParams, DataStoreListParams, DataStoreSetParams, DwnDataStore, InMemoryDataStore } from './store-data.js';
 
 export class DwnKeyStore extends DwnDataStore<Jwk> implements AgentDataStore<Jwk> {
   protected name = 'DwnKeyStore';
 
+  protected _recordProtocolDefinition = JwkProtocolDefinition;
+
   /**
    * Properties to use when writing and querying Private Key records with the DWN store.
    */
   protected _recordProperties = {
-    dataFormat : 'application/json',
-    schema     : 'https://identity.foundation/schemas/web5/private-jwk'
+    dataFormat   : 'application/json',
+    protocol     : this._recordProtocolDefinition.protocol,
+    protocolPath : 'privateJwk',
+    schema       : this._recordProtocolDefinition.types.privateJwk.schema,
   };
 
   public async delete(params: DataStoreDeleteParams): Promise<boolean> {

diff --git a/packages/agent/src/test-harness.ts b/packages/agent/src/test-harness.ts
@@ -35,6 +35,12 @@ type PlatformAgentTestHarnessParams = {
   dwnResumableTaskStore: ResumableTaskStoreLevel;
   syncStore: AbstractLevel<string | Buffer | Uint8Array>;
   vaultStore: KeyValueStore<string, string>;
+  dwnStores: {
+    keyStore: DwnKeyStore;
+    identityStore: DwnIdentityStore;
+    didStore: DwnDidStore;
+    clear: () => void;
+  }
 }
 
 type PlatformAgentTestHarnessSetupParams = {
@@ -56,6 +62,18 @@ export class PlatformAgentTestHarness {
   public syncStore: AbstractLevel<string | Buffer | Uint8Array>;
   public vaultStore: KeyValueStore<string, string>;
 
+  /**
+   * Custom DWN Stores for `keyStore`, `identityStore` and `didStore`.
+   * This allows us to clear the store cache between tests
+   */
+  public dwnStores: {
+    keyStore: DwnKeyStore;
+    identityStore: DwnIdentityStore;
+    didStore: DwnDidStore;
+    /** clears the protocol initialization caches */
+    clear: () => void;
+  };
+
   constructor(params: PlatformAgentTestHarnessParams) {
     this.agent = params.agent;
     this.agentStores = params.agentStores;
@@ -67,6 +85,7 @@ export class PlatformAgentTestHarness {
     this.syncStore = params.syncStore;
     this.vaultStore = params.vaultStore;
     this.dwnResumableTaskStore = params.dwnResumableTaskStore;
+    this.dwnStores = params.dwnStores;
   }
 
   public async clearStorage(): Promise<void> {
@@ -170,6 +189,17 @@ export class PlatformAgentTestHarness {
     // Instantiate Agent's RPC Client.
     const rpcClient = new Web5RpcClient();
 
+    const dwnStores = {
+      keyStore      : new DwnKeyStore(),
+      identityStore : new DwnIdentityStore(),
+      didStore      : new DwnDidStore(),
+      clear         : ():void => {
+        dwnStores.keyStore['_protocolInitializedCache']?.clear();
+        dwnStores.identityStore['_protocolInitializedCache']?.clear();
+        dwnStores.didStore['_protocolInitializedCache']?.clear();
+      }
+    };
+
     const {
       agentVault,
       didApi,
@@ -179,7 +209,7 @@ export class PlatformAgentTestHarness {
       vaultStore
     } = (agentStores === 'memory')
       ? PlatformAgentTestHarness.useMemoryStores()
-      : PlatformAgentTestHarness.useDiskStores({ testDataLocation });
+      : PlatformAgentTestHarness.useDiskStores({ testDataLocation, stores: dwnStores });
 
     // Instantiate custom stores to use with DWN instance.
     // Note: There is no in-memory store for DWN, so we always use LevelDB-based disk stores.
@@ -233,20 +263,28 @@ export class PlatformAgentTestHarness {
       dwnEventLog,
       dwnMessageStore,
       dwnResumableTaskStore,
+      dwnStores,
       syncStore,
       vaultStore
     });
   }
 
-  private static useDiskStores({ agent, testDataLocation }: {
+  private static useDiskStores({ agent, testDataLocation, stores }: {
     agent?: Web5PlatformAgent;
+    stores: {
+      keyStore: DwnKeyStore;
+      identityStore: DwnIdentityStore;
+      didStore: DwnDidStore;
+    }
     testDataLocation: string;
   }) {
     const testDataPath = (path: string) => `${testDataLocation}/${path}`;
 
     const vaultStore = new LevelStore<string, string>({ location: testDataPath('VAULT_STORE') });
     const agentVault = new HdIdentityVault({ keyDerivationWorkFactor: 1, store: vaultStore });
 
+    const { didStore, identityStore, keyStore } = stores;
+
     // Setup DID Resolver Cache
     const didResolverCache = new DidResolverCacheLevel({
       location: testDataPath('DID_RESOLVERCACHE')
@@ -256,12 +294,12 @@ export class PlatformAgentTestHarness {
       agent         : agent,
       didMethods    : [DidDht, DidJwk],
       resolverCache : didResolverCache,
-      store         : new DwnDidStore()
+      store         : didStore
     });
 
-    const identityApi = new AgentIdentityApi({ agent, store: new DwnIdentityStore() });
+    const identityApi = new AgentIdentityApi({ agent, store: identityStore });
 
-    const keyManager = new LocalKeyManager({ agent, keyStore: new DwnKeyStore() });
+    const keyManager = new LocalKeyManager({ agent, keyStore: keyStore });
 
     return { agentVault, didApi, didResolverCache, identityApi, keyManager, vaultStore };
   }