Skip to content

Commit

Permalink
build(deps): replace ipfs-http-client with kubo-rpc-client
Browse files Browse the repository at this point in the history
- Replace deprecated ipfs-http-client with kubo-rpc-client.
- kubo-rpc-client must be imported dynamically since it's ESM-only and we
    still use CJS.

Depends on: hyperledger#2821

Signed-off-by: Michal Bajer <michal.bajer@fujitsu.com>
  • Loading branch information
outSH authored and petermetz committed Nov 9, 2023
1 parent 9043e59 commit acde2a5
Show file tree
Hide file tree
Showing 21 changed files with 438 additions and 263 deletions.
4 changes: 4 additions & 0 deletions .cspell.json
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@
"ipaddress",
"ipfs",
"IPFSHTTP",
"IPLD",
"ipld",
"Iroha",
"Irohad",
"isready",
Expand All @@ -84,6 +86,7 @@
"KEYUTIL",
"KJUR",
"Knetic",
"kubo",
"LEDGERBLOCKACK",
"leveldb",
"lmify",
Expand Down Expand Up @@ -147,6 +150,7 @@
"txqueue",
"Uisrs",
"undici",
"unixfs",
"Unmarshal",
"uuidv",
"vscc",
Expand Down
6 changes: 4 additions & 2 deletions extensions/cactus-plugin-object-store-ipfs/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -55,21 +55,23 @@
"webpack:dev:web": "webpack --env=dev --target=web --config ../../webpack.config.js"
},
"dependencies": {
"@hyperledger/cacti-esm-compat-hacks": "2.0.0-alpha.2",
"@hyperledger/cactus-common": "2.0.0-alpha.2",
"@hyperledger/cactus-core": "2.0.0-alpha.2",
"@hyperledger/cactus-core-api": "2.0.0-alpha.2",
"axios": "1.5.1",
"ipfs-http-client": "60.0.1",
"run-time-error": "1.4.0",
"typescript-optional": "2.0.1",
"uuid": "8.3.2"
},
"devDependencies": {
"@hyperledger/cactus-test-tooling": "2.0.0-alpha.2",
"@multiformats/multiaddr": "11.6.1",
"@types/express": "4.17.19",
"express": "4.18.2",
"ipfs-core-types": "0.14.1",
"multiformats": "9.4.9"
"ipfs-unixfs": "9.0.1",
"multiformats": "11.0.2"
},
"engines": {
"node": ">=10",
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
/**
* Since kubo-rpc-client uses ESM only, we can't import it to get types (since we use CJS).
* To fix this we define required types here, based on their counterparts in kubo-rpc-client.
*/

import type { Multiaddr } from "@multiformats/multiaddr";
import type { MultihashHasher } from "multiformats/hashes/interface";
import type { Agent as HttpAgent } from "http";
import type { Agent as HttpsAgent } from "https";
import type { CID } from "multiformats/cid";
import type { Mtime } from "ipfs-unixfs";

/////////////////////////////////////
// Types from kubo-rpc-client
/////////////////////////////////////
// Some are simplified when details are not needed

export type MultibaseCodec<Prefix extends string = any> =
import("multiformats/bases/interface").MultibaseCodec<Prefix>;
export type BlockCodec<
T1 = any,
T2 = any,
> = import("multiformats/codecs/interface").BlockCodec<T1, T2>;

export interface LoadBaseFn {
(codeOrName: number | string): Promise<MultibaseCodec<any>>;
}
export interface LoadCodecFn {
(codeOrName: number | string): Promise<BlockCodec<any, any>>;
}
export interface LoadHasherFn {
(codeOrName: number | string): Promise<MultihashHasher>;
}

export interface IPLDOptions {
loadBase: LoadBaseFn;
loadCodec: LoadCodecFn;
loadHasher: LoadHasherFn;
bases: Array<MultibaseCodec<any>>;
codecs: Array<BlockCodec<any, any>>;
hashers: MultihashHasher[];
}

export interface Options {
host?: string;
port?: number;
protocol?: string;
headers?: Headers | Record<string, string>;
timeout?: number | string;
apiPath?: string;
url?: URL | string | Multiaddr;
ipld?: Partial<IPLDOptions>;
agent?: HttpAgent | HttpsAgent;
}

export type IPFSPath = CID | string;

export interface StatResult {
cid: CID;
size: number;
cumulativeSize: number;
type: "directory" | "file";
blocks: number;
withLocality: boolean;
local?: boolean;
sizeLocal?: number;
mode?: number;
mtime?: Mtime;
}

/////////////////////////////////////////////////////////
// LikeIpfsHttpClient instead of full IpfsHttpClient
/////////////////////////////////////////////////////////

/**
* Connector only needs these methods to work.
* More methods can be added in the future.
*/
export interface LikeIpfsHttpClientFile {
read: (
ipfsPath: IPFSPath,
options?: Record<string, unknown>,
) => AsyncIterable<Uint8Array>;

write: (
ipfsPath: string,
content:
| string
| Uint8Array
| Blob
| AsyncIterable<Uint8Array>
| Iterable<Uint8Array>,
options?: Record<string, unknown>,
) => Promise<void>;

stat: (
ipfsPath: IPFSPath,
options?: Record<string, unknown>,
) => Promise<StatResult>;
}

export function isLikeIpfsHttpClientFile(
x: unknown,
): x is LikeIpfsHttpClientFile {
if (!x) {
return false;
}
return (
typeof (x as LikeIpfsHttpClientFile).read === "function" &&
typeof (x as LikeIpfsHttpClientFile).write === "function" &&
typeof (x as LikeIpfsHttpClientFile).stat === "function"
);
}

/**
* Only files API is used
*/
export interface LikeIpfsHttpClient {
files: LikeIpfsHttpClientFile;
}

export function isLikeIpfsHttpClient(x: unknown): x is LikeIpfsHttpClient {
if (!x) {
return false;
}
return isLikeIpfsHttpClientFile((x as LikeIpfsHttpClient).files);
}
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import path from "path";
import type { Express } from "express";
import { create, IPFSHTTPClient } from "ipfs-http-client";
import type { Options } from "ipfs-http-client";
import { RuntimeError } from "run-time-error";
import { Logger, Checks, LoggerProvider } from "@hyperledger/cactus-common";
import type { LogLevelDesc } from "@hyperledger/cactus-common";
import {
Logger,
Checks,
LoggerProvider,
LogLevelDesc,
} from "@hyperledger/cactus-common";
import type {
IPluginObjectStore,
ICactusPluginOptions,
Expand All @@ -16,27 +18,32 @@ import type {
SetObjectRequestV1,
SetObjectResponseV1,
} from "@hyperledger/cactus-core-api";
import { dynamicImportKuboRpcClientESMWorkaround } from "@hyperledger/cacti-esm-compat-hacks";

import OAS from "../json/openapi.json";

import { GetObjectEndpointV1 } from "./web-services/get-object-endpoint-v1";
import { SetObjectEndpointV1 } from "./web-services/set-object-endpoint-v1";
import { HasObjectEndpointV1 } from "./web-services/has-object-endpoint-v1";
import { isIpfsHttpClientOptions } from "./i-ipfs-http-client";
import {
LikeIpfsHttpClient,
isLikeIpfsHttpClient,
Options,
} from "./kubo-rpc-client-types";

export const K_IPFS_JS_HTTP_ERROR_FILE_DOES_NOT_EXIST =
"HTTPError: file does not exist";

export interface IPluginObjectStoreIpfsOptions extends ICactusPluginOptions {
readonly logLevel?: LogLevelDesc;
readonly parentDir: string;
readonly ipfsClientOrOptions: Options | IPFSHTTPClient;
readonly ipfsClientOrOptions: Options | LikeIpfsHttpClient;
}

export class PluginObjectStoreIpfs implements IPluginObjectStore {
public static readonly CLASS_NAME = "PluginObjectStoreIpfs";

private readonly ipfs: IPFSHTTPClient;
private ipfs: LikeIpfsHttpClient | undefined;
private readonly log: Logger;
private readonly instanceId: string;
private readonly parentDir: string;
Expand All @@ -45,25 +52,45 @@ export class PluginObjectStoreIpfs implements IPluginObjectStore {
return PluginObjectStoreIpfs.CLASS_NAME;
}

/**
* We use dynamic import for kubo-rpc-client since it's ESM and we can't import it normally.
* This methods will load the module and initialize local IPFS client based on ctor arguments.
*/
private async initIpfs(): Promise<void> {
if (isLikeIpfsHttpClient(this.opts.ipfsClientOrOptions)) {
this.ipfs = this.opts.ipfsClientOrOptions;
} else if (this.opts.ipfsClientOrOptions) {
const kuboRpcModule = await dynamicImportKuboRpcClientESMWorkaround();
this.ipfs = kuboRpcModule.create(this.opts.ipfsClientOrOptions);
} else {
const errorMessage = `initIpfs Need either "ipfsClient" or "ipfsClientOptions" to construct ${this.className} Neither was provided.`;
throw new RuntimeError(errorMessage);
}
}

/**
* Get IPFS client or initialize it from constructor args.
* @returns `LikeIpfsHttpClient` or exception
*/
private async getIpfs(): Promise<LikeIpfsHttpClient> {
if (!this.ipfs) {
await this.initIpfs();
}

if (!this.ipfs) {
throw new Error("Could not instantiate ipfs http client");
}

return this.ipfs;
}

constructor(public readonly opts: IPluginObjectStoreIpfsOptions) {
const fnTag = `${this.className}#constructor()`;
Checks.truthy(opts, `${fnTag} arg options`);
Checks.nonBlankString(opts.instanceId, `${fnTag} options.instanceId`);
Checks.nonBlankString(opts.parentDir, `${fnTag} options.parentDir`);
Checks.truthy(opts.ipfsClientOrOptions, `${fnTag} ipfsClientOrOptions`);

if (isIpfsHttpClientOptions(opts.ipfsClientOrOptions)) {
this.ipfs = opts.ipfsClientOrOptions;
} else if (opts.ipfsClientOrOptions) {
this.ipfs = create({
...(this.opts.ipfsClientOrOptions as Options),
});
} else {
const errorMessage = `${fnTag} Need either "ipfsClient" or "ipfsClientOptions" to construct ${this.className} Neither was provided.`;
throw new RuntimeError(errorMessage);
}
Checks.truthy(this.ipfs, `${fnTag} arg options.backend`);

const level = this.opts.logLevel || "INFO";
const label = this.className;
this.log = LoggerProvider.getOrCreate({ level, label });
Expand All @@ -79,7 +106,7 @@ export class PluginObjectStoreIpfs implements IPluginObjectStore {
}

public async onPluginInit(): Promise<unknown> {
return; // no-op
return this.initIpfs();
}

public async registerWebServices(
Expand Down Expand Up @@ -130,7 +157,8 @@ export class PluginObjectStoreIpfs implements IPluginObjectStore {

public async get(req: GetObjectRequestV1): Promise<GetObjectResponseV1> {
const keyPath = this.getKeyPath(req);
const chunksIterable = this.ipfs.files.read(keyPath);
const ipfs = await this.getIpfs();
const chunksIterable = ipfs.files.read(keyPath);
const chunks = [];
for await (const chunk of chunksIterable) {
chunks.push(chunk);
Expand All @@ -151,7 +179,8 @@ export class PluginObjectStoreIpfs implements IPluginObjectStore {
const checkedAt = new Date().toJSON();
const keyPath = this.getKeyPath(req);
try {
const statResult = await this.ipfs.files.stat(keyPath);
const ipfs = await this.getIpfs();
const statResult = await ipfs.files.stat(keyPath);
this.log.debug(`StatResult for ${req.key}: %o`, statResult);
return { key: req.key, checkedAt, isPresent: true };
} catch (ex) {
Expand All @@ -170,7 +199,8 @@ export class PluginObjectStoreIpfs implements IPluginObjectStore {
try {
this.log.debug(`Seting object ${keyPath} in IPFS...`);
const buffer = Buffer.from(req.value, "base64");
await this.ipfs.files.write(keyPath, buffer, {
const ipfs = await this.getIpfs();
await ipfs.files.write(keyPath, buffer, {
create: true,
parents: true,
});
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
export * from "./generated/openapi/typescript-axios/index";
export { IIpfsHttpClient } from "./i-ipfs-http-client";
export {
Options,
LikeIpfsHttpClientFile,
LikeIpfsHttpClient,
} from "./kubo-rpc-client-types";
import { IPluginFactoryOptions } from "@hyperledger/cactus-core-api";
export {
PluginObjectStoreIpfs,
Expand Down
Loading

0 comments on commit acde2a5

Please sign in to comment.