From cf4910b5614a1ce903b368508d09f6ef98d2176c Mon Sep 17 00:00:00 2001 From: rkamysz Date: Thu, 15 Jun 2023 23:14:21 +0200 Subject: [PATCH 001/107] pre open source wip --- package.json | 7 +- src/api/api.types.ts | 2 +- .../contract-action.mongo.source.ts | 9 +- .../data/mappers/contract-action.mapper.ts | 39 +- src/api/endpoints/actions/ioc.config.ts | 3 +- .../__tests__/bootstrap.utils.unit.test.ts | 186 +++++++ .../__tests__/start-bootstrap.unit.test.ts | 163 ++++++ src/bootstrap/bootstrap.command.ts | 10 + src/bootstrap/bootstrap.errors.ts | 35 ++ src/bootstrap/bootstrap.types.ts | 3 +- src/bootstrap/bootstrap.utils.ts | 41 +- src/bootstrap/start-bootstrap.ts | 59 ++- .../internal-broadcast.message.unit.test.ts | 23 + src/broadcast/internal-broadcast.message.ts | 34 +- .../filter-broadcast.message.unit.test.ts | 19 + .../processor-broadcast.message.unit.test.ts | 28 + .../reader-broadcast.message.unit.test.ts | 60 +++ .../messages/filter-broadcast.message.ts | 36 +- .../messages/processor-broadcast.message.ts | 41 +- .../messages/reader-broadcast.message.ts | 82 ++- .../__tests__/common.utils.unit.test.ts | 23 + .../abi.repository-impl.unit.test.ts | 115 ++++ .../__tests__/abi.repository.unit.test.ts | 18 - .../abis/__tests__/abis.cache.unit.test.ts | 104 ++++ .../abis/__tests__/abis.service.unit.test.ts | 62 +++ src/common/abis/__tests__/abis.unit.test.ts | 143 +++++ .../contract-encoded-abi.unit.test.ts | 104 ++++ src/common/abis/abis.cache.ts | 39 ++ src/common/abis/abis.collection.ts | 21 + ...is.serialize.ts => abis.eos.serializer.ts} | 39 +- src/common/abis/abis.repository-impl.ts | 144 +++++ src/common/abis/abis.repository.ts | 196 ++----- src/common/abis/abis.serializer.ts | 37 ++ src/common/abis/abis.service.ts | 14 + src/common/abis/abis.ts | 137 +++-- src/common/abis/abis.types.ts | 2 +- src/common/abis/contract-encoded-abi.ts | 39 +- src/common/abis/index.ts | 2 +- .../__tests__/block-state.unit.test.ts | 121 +++++ .../block-state/block-state.collection.ts | 8 + src/common/block-state/block-state.source.ts | 116 ---- src/common/block-state/block-state.ts | 133 +++-- src/common/block-state/block-state.types.ts | 2 +- ...update-block-number.mongo.query-builder.ts | 17 + .../blockchain/abi/__tests__/abi.unit.test.ts | 185 ------- src/common/blockchain/abi/abi-action.ts | 40 -- .../blockchain/abi/abi-error-message.ts | 34 -- src/common/blockchain/abi/abi-extension.ts | 31 -- src/common/blockchain/abi/abi-struct.ts | 70 --- src/common/blockchain/abi/abi-table.ts | 46 -- src/common/blockchain/abi/abi-type.ts | 40 -- src/common/blockchain/abi/abi-variant.ts | 31 -- src/common/blockchain/abi/abi.dtos.ts | 136 ----- src/common/blockchain/abi/abi.ts | 138 ----- src/common/blockchain/abi/index.ts | 10 - src/common/blockchain/abi/ricardian-clause.ts | 31 -- .../block-reader/block-reader.config.ts | 10 - .../block-reader/block-reader.enums.ts | 6 - .../block-reader/block-reader.errors.ts | 37 -- .../block-reader/block-reader.message.ts | 56 -- .../block-reader/block-reader.requests.ts | 69 --- .../block-reader/block-reader.source.ts | 135 ----- .../blockchain/block-reader/block-reader.ts | 243 --------- .../block-reader/block-reader.types.ts | 40 -- .../block-reader/block-reader.utils.ts | 37 -- .../blockchain/block-reader/block/block.ts | 145 ----- .../block-reader/block/block.types.ts | 35 -- .../blockchain/block-reader/block/index.ts | 2 - src/common/blockchain/block-reader/index.ts | 9 - src/common/blockchain/blockchain.ts | 45 -- src/common/blockchain/blockchain.types.ts | 4 - .../contract/action-trace/action-trace.ts | 127 ----- src/common/blockchain/contract/delta/delta.ts | 31 -- .../contract/signed-block/signed-block.ts | 52 -- src/common/blockchain/contract/trace/trace.ts | 107 ---- .../contract/transaction/transaction.ts | 63 --- src/common/blockchain/index.ts | 6 - src/common/common.utils.ts | 12 +- .../__tests__/action-trace.unit.test.ts | 0 .../action-trace/action-trace.dtos.ts | 0 .../action-trace/action-trace.ts | 217 ++++++++ .../action-trace/index.ts | 0 .../delta/__tests__/delta.unit.test.ts | 0 .../delta/delta.dtos.ts | 0 src/common/contract-content/delta/delta.ts | 65 +++ .../delta/index.ts | 0 .../contract => contract-content}/index.ts | 0 .../__tests__/signed-block.unit.test.ts} | 2 +- .../signed-block/index.ts | 0 .../signed-block/signed-block.dtos.ts | 4 +- .../signed-block/signed-block.ts | 79 +++ .../trace/__tests__/trace.unit.test.ts | 0 .../trace/index.ts | 0 .../trace/trace.dtos.ts | 8 +- src/common/contract-content/trace/trace.ts | 162 ++++++ .../__tests__/transaction.unit.test.ts | 0 .../transaction/index.ts | 0 .../transaction/transaction.dtos.ts | 8 +- .../transaction/transaction.ts | 101 ++++ .../contract-reader/contract-reader.config.ts | 0 .../contract-reader/contract-reader.dtos.ts | 2 +- .../contract-reader/contract-reader.ts | 5 +- .../featured-contract.source.ts | 4 +- .../contract-reader/featured-contract.ts | 2 +- .../{blockchain => }/contract-reader/index.ts | 0 src/common/ship/index.ts | 2 - src/common/ship/ship-abi.source.ts | 55 -- src/common/ship/ship-abis.ts | 37 -- src/common/workers/index.ts | 9 - src/common/workers/worker-container.ts | 13 - src/common/workers/worker-loader/index.ts | 3 - .../worker-loader/worker-loader-script.ts | 50 -- .../worker-loader/worker-loader.errors.ts | 7 - .../workers/worker-loader/worker-loader.ts | 53 -- .../worker-loader/worker-loader.types.ts | 2 - .../worker-loader/worker-loader.utils.ts | 32 -- src/common/workers/worker-message.ts | 227 -------- src/common/workers/worker-pool.ts | 114 ---- src/common/workers/worker-proxy.ts | 127 ----- src/common/workers/worker.enums.ts | 4 - src/common/workers/worker.errors.ts | 27 - src/common/workers/worker.ts | 47 -- src/common/workers/worker.types.ts | 25 - src/common/workers/worker.utils.ts | 25 - src/config/index.ts | 13 +- .../data-sources/processor-task.source.ts | 12 +- .../unsuccessful-processor-task.source.ts | 7 +- .../processor-task-queue.ts | 15 +- .../processor-task-queue/processor-task.ts | 2 +- .../processor-task.types.ts | 2 +- src/processor/processor.types.ts | 2 +- src/processor/processor.worker-loader.ts | 2 +- .../action-trace.processor.input.ts | 2 +- .../processors/delta.processor.input.ts | 2 +- src/processor/start-processor.ts | 2 +- .../block-range-scan.mongo.source.ts | 6 +- .../block-range-scan.repository.ts | 2 +- .../block-range-scanner.ts | 3 +- src/reader/reader.ts | 3 +- src/reader/reader.types.ts | 2 +- src/reader/reader.worker-loader.ts | 3 +- src/reader/start-reader.ts | 2 +- .../unprocessed-block-queue.ts | 25 +- yarn.lock | 501 +++--------------- 144 files changed, 2753 insertions(+), 3952 deletions(-) create mode 100644 src/bootstrap/__tests__/bootstrap.utils.unit.test.ts create mode 100644 src/bootstrap/__tests__/start-bootstrap.unit.test.ts create mode 100644 src/bootstrap/bootstrap.command.ts create mode 100644 src/broadcast/__tests__/internal-broadcast.message.unit.test.ts create mode 100644 src/broadcast/messages/__tests__/filter-broadcast.message.unit.test.ts create mode 100644 src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts create mode 100644 src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts create mode 100644 src/common/__tests__/common.utils.unit.test.ts create mode 100644 src/common/abis/__tests__/abi.repository-impl.unit.test.ts delete mode 100644 src/common/abis/__tests__/abi.repository.unit.test.ts create mode 100644 src/common/abis/__tests__/abis.cache.unit.test.ts create mode 100644 src/common/abis/__tests__/abis.service.unit.test.ts create mode 100644 src/common/abis/__tests__/abis.unit.test.ts create mode 100644 src/common/abis/__tests__/contract-encoded-abi.unit.test.ts create mode 100644 src/common/abis/abis.collection.ts rename src/common/abis/{abis.serialize.ts => abis.eos.serializer.ts} (74%) create mode 100644 src/common/abis/abis.repository-impl.ts create mode 100644 src/common/abis/abis.serializer.ts create mode 100644 src/common/block-state/__tests__/block-state.unit.test.ts create mode 100644 src/common/block-state/block-state.collection.ts delete mode 100644 src/common/block-state/block-state.source.ts create mode 100644 src/common/block-state/query-builders/update-block-number.mongo.query-builder.ts delete mode 100644 src/common/blockchain/abi/__tests__/abi.unit.test.ts delete mode 100644 src/common/blockchain/abi/abi-action.ts delete mode 100644 src/common/blockchain/abi/abi-error-message.ts delete mode 100644 src/common/blockchain/abi/abi-extension.ts delete mode 100644 src/common/blockchain/abi/abi-struct.ts delete mode 100644 src/common/blockchain/abi/abi-table.ts delete mode 100644 src/common/blockchain/abi/abi-type.ts delete mode 100644 src/common/blockchain/abi/abi-variant.ts delete mode 100644 src/common/blockchain/abi/abi.dtos.ts delete mode 100644 src/common/blockchain/abi/abi.ts delete mode 100644 src/common/blockchain/abi/index.ts delete mode 100644 src/common/blockchain/abi/ricardian-clause.ts delete mode 100644 src/common/blockchain/block-reader/block-reader.config.ts delete mode 100644 src/common/blockchain/block-reader/block-reader.enums.ts delete mode 100644 src/common/blockchain/block-reader/block-reader.errors.ts delete mode 100644 src/common/blockchain/block-reader/block-reader.message.ts delete mode 100644 src/common/blockchain/block-reader/block-reader.requests.ts delete mode 100644 src/common/blockchain/block-reader/block-reader.source.ts delete mode 100644 src/common/blockchain/block-reader/block-reader.ts delete mode 100644 src/common/blockchain/block-reader/block-reader.types.ts delete mode 100644 src/common/blockchain/block-reader/block-reader.utils.ts delete mode 100644 src/common/blockchain/block-reader/block/block.ts delete mode 100644 src/common/blockchain/block-reader/block/block.types.ts delete mode 100644 src/common/blockchain/block-reader/block/index.ts delete mode 100644 src/common/blockchain/block-reader/index.ts delete mode 100644 src/common/blockchain/blockchain.ts delete mode 100644 src/common/blockchain/blockchain.types.ts delete mode 100644 src/common/blockchain/contract/action-trace/action-trace.ts delete mode 100644 src/common/blockchain/contract/delta/delta.ts delete mode 100644 src/common/blockchain/contract/signed-block/signed-block.ts delete mode 100644 src/common/blockchain/contract/trace/trace.ts delete mode 100644 src/common/blockchain/contract/transaction/transaction.ts delete mode 100644 src/common/blockchain/index.ts rename src/common/{blockchain/contract => contract-content}/action-trace/__tests__/action-trace.unit.test.ts (100%) rename src/common/{blockchain/contract => contract-content}/action-trace/action-trace.dtos.ts (100%) create mode 100644 src/common/contract-content/action-trace/action-trace.ts rename src/common/{blockchain/contract => contract-content}/action-trace/index.ts (100%) rename src/common/{blockchain/contract => contract-content}/delta/__tests__/delta.unit.test.ts (100%) rename src/common/{blockchain/contract => contract-content}/delta/delta.dtos.ts (100%) create mode 100644 src/common/contract-content/delta/delta.ts rename src/common/{blockchain/contract => contract-content}/delta/index.ts (100%) rename src/common/{blockchain/contract => contract-content}/index.ts (100%) rename src/common/{blockchain/contract/signed-block/__tests__/block.unit.test.ts => contract-content/signed-block/__tests__/signed-block.unit.test.ts} (96%) rename src/common/{blockchain/contract => contract-content}/signed-block/index.ts (100%) rename src/common/{blockchain/contract => contract-content}/signed-block/signed-block.dtos.ts (73%) create mode 100644 src/common/contract-content/signed-block/signed-block.ts rename src/common/{blockchain/contract => contract-content}/trace/__tests__/trace.unit.test.ts (100%) rename src/common/{blockchain/contract => contract-content}/trace/index.ts (100%) rename src/common/{blockchain/contract => contract-content}/trace/trace.dtos.ts (79%) create mode 100644 src/common/contract-content/trace/trace.ts rename src/common/{blockchain/contract => contract-content}/transaction/__tests__/transaction.unit.test.ts (100%) rename src/common/{blockchain/contract => contract-content}/transaction/index.ts (100%) rename src/common/{blockchain/contract => contract-content}/transaction/transaction.dtos.ts (56%) create mode 100644 src/common/contract-content/transaction/transaction.ts rename src/common/{blockchain => }/contract-reader/contract-reader.config.ts (100%) rename src/common/{blockchain => }/contract-reader/contract-reader.dtos.ts (84%) rename src/common/{blockchain => }/contract-reader/contract-reader.ts (94%) rename src/common/{blockchain => }/contract-reader/featured-contract.source.ts (85%) rename src/common/{blockchain => }/contract-reader/featured-contract.ts (95%) rename src/common/{blockchain => }/contract-reader/index.ts (100%) delete mode 100644 src/common/ship/index.ts delete mode 100644 src/common/ship/ship-abi.source.ts delete mode 100644 src/common/ship/ship-abis.ts delete mode 100644 src/common/workers/index.ts delete mode 100644 src/common/workers/worker-container.ts delete mode 100644 src/common/workers/worker-loader/index.ts delete mode 100644 src/common/workers/worker-loader/worker-loader-script.ts delete mode 100644 src/common/workers/worker-loader/worker-loader.errors.ts delete mode 100644 src/common/workers/worker-loader/worker-loader.ts delete mode 100644 src/common/workers/worker-loader/worker-loader.types.ts delete mode 100644 src/common/workers/worker-loader/worker-loader.utils.ts delete mode 100644 src/common/workers/worker-message.ts delete mode 100644 src/common/workers/worker-pool.ts delete mode 100644 src/common/workers/worker-proxy.ts delete mode 100644 src/common/workers/worker.enums.ts delete mode 100644 src/common/workers/worker.errors.ts delete mode 100644 src/common/workers/worker.ts delete mode 100644 src/common/workers/worker.types.ts delete mode 100644 src/common/workers/worker.utils.ts diff --git a/package.json b/package.json index 358a39b..5216267 100644 --- a/package.json +++ b/package.json @@ -35,10 +35,15 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/api-core": "^0.0.101", + "@alien-worlds/api-core": "^0.0.127", + "@alien-worlds/block-reader": "^0.0.3", + "@alien-worlds/broadcast": "^0.0.3", + "@alien-worlds/storage-mongodb": "^0.0.7", + "@alien-worlds/workers": "^0.0.1", "@eosrio/node-abieos": "^1", "amqplib": "^0.10.3", "async": "^3.2.4", + "commander": "^10.0.1", "crypto": "^1.0.1", "eosjs": "^22.1.0", "express": "^4.18.2", diff --git a/src/api/api.types.ts b/src/api/api.types.ts index 4a02dfb..2837d38 100644 --- a/src/api/api.types.ts +++ b/src/api/api.types.ts @@ -1,4 +1,4 @@ -import { MongoConfig } from '@alien-worlds/api-core'; +import { MongoConfig } from "@alien-worlds/storage-mongodb"; export type ApiConfig = { port: number; diff --git a/src/api/endpoints/actions/data/data-sources/contract-action.mongo.source.ts b/src/api/endpoints/actions/data/data-sources/contract-action.mongo.source.ts index 8a9a367..97ae993 100644 --- a/src/api/endpoints/actions/data/data-sources/contract-action.mongo.source.ts +++ b/src/api/endpoints/actions/data/data-sources/contract-action.mongo.source.ts @@ -1,13 +1,10 @@ -import { - CollectionMongoSource, - ContractAction, - MongoSource, -} from '@alien-worlds/api-core'; +import { ContractAction } from '@alien-worlds/api-core'; +import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; /** * @class */ -export class ContractActionMongoSource extends CollectionMongoSource { +export class ContractActionMongoSource extends MongoCollectionSource { /** * @constructor * @param {MongoSource} mongoSource diff --git a/src/api/endpoints/actions/data/mappers/contract-action.mapper.ts b/src/api/endpoints/actions/data/mappers/contract-action.mapper.ts index 5c891ee..7d52b27 100644 --- a/src/api/endpoints/actions/data/mappers/contract-action.mapper.ts +++ b/src/api/endpoints/actions/data/mappers/contract-action.mapper.ts @@ -1,5 +1,5 @@ import { Entity, Mapper } from '@alien-worlds/api-core'; -import { ContractAction, ContractActionDocument } from '@alien-worlds/api-core'; +import { ContractAction } from '@alien-worlds/api-core'; /*imports*/ @@ -8,23 +8,26 @@ type MapperType = (data: object) => Entity; export class ContractActionMapper implements Mapper { - private mapper: MapperType; - constructor(mapper?: MapperType) { - if (mapper) { - this.mapper = mapper; - } else { - this.mapper = (data: object) => - ({ - toDocument: () => data, - } as Entity); - } - } - public toEntity(document: ContractActionDocument): ContractAction { - return ContractAction.fromDocument(document, this.mapper); - } - public toDataObject(entity: ContractAction): ContractActionDocument { - return entity.toDocument(); - } + + // private mapper: MapperType; + + // constructor(mapper?: MapperType) { + // if (mapper) { + // this.mapper = mapper; + // } else { + // this.mapper = (data: object) => + // ({ + // toDocument: () => data, + // } as Entity); + // } + // } + + // public toEntity(document: ContractActionDocument): ContractAction { + // return ContractAction.fromDocument(document, this.mapper); + // } + // public toDataObject(entity: ContractAction): ContractActionDocument { + // return entity.toDocument(); + // } } diff --git a/src/api/endpoints/actions/ioc.config.ts b/src/api/endpoints/actions/ioc.config.ts index 4957072..99d925e 100644 --- a/src/api/endpoints/actions/ioc.config.ts +++ b/src/api/endpoints/actions/ioc.config.ts @@ -1,6 +1,4 @@ import { - MongoConfig, - MongoSource, RepositoryImpl, Container, } from '@alien-worlds/api-core'; @@ -8,6 +6,7 @@ import { import { ContractActionMongoSource } from './data/data-sources/contract-action.mongo.source'; import { ContractActionMapper } from './data/mappers/contract-action.mapper'; import { ContractActionRepository } from './domain/repositories/contract-action.repository'; +import { MongoConfig, MongoSource } from '@alien-worlds/storage-mongodb'; export const setupContractActionRepository = async ( mongo: MongoSource | MongoConfig, diff --git a/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts b/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts new file mode 100644 index 0000000..e399a88 --- /dev/null +++ b/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts @@ -0,0 +1,186 @@ +import { + createDefaultModeBlockRange, + createReplayModeBlockRange, + createTestModeBlockRange, +} from '../bootstrap.utils'; + +import { + EndBlockOutOfRangeError, + StartBlockHigherThanEndBlockError, + UndefinedStartBlockError, +} from '../bootstrap.errors'; +import { Mode } from '../../common/common.enums'; +import { Blockchain } from '../../common'; + +jest.mock('../../common', () => ({ + Blockchain: { + create: jest.fn().mockResolvedValue({ + getLastIrreversibleBlockNumber: jest.fn().mockResolvedValue(100n), + getHeadBlockNumber: jest.fn().mockResolvedValue(100n), + }), + }, +})); + +describe('createDefaultModeBlockRange', () => { + const originalLog = console.log; + beforeEach(() => { + jest.clearAllMocks(); + console.log = jest.fn(); // mock console.log to prevent log outputs during tests + }); + + afterEach(() => { + console.log = originalLog; // restore original console.log after tests + }); + + it('should create a block range in default mode with positive startBlock', async () => { + const blockState = { + getBlockNumber: jest.fn().mockResolvedValue({ content: 0n }), + } as any; + const config = { + blockchain: {}, + startBlock: 5n, + endBlock: 5n, + mode: Mode.Default, + scanner: { scanKey: 'scanKey' }, + startFromHead: true, + maxBlockNumber: 10n, + } as any; + + const result = await createDefaultModeBlockRange(blockState, config); + + expect(result).toEqual({ + startBlock: 5n, + endBlock: 5n, + mode: Mode.Default, + scanKey: 'scanKey', + }); + }); + + it('should throw error when highEdge < lowEdge', async () => { + const blockState = { + getBlockNumber: jest.fn().mockResolvedValue({ content: 0n }), + } as any; + const config = { + blockchain: {}, + startBlock: 5n, + endBlock: 3n, + mode: Mode.Default, + scanner: { scanKey: 'scanKey' }, + startFromHead: true, + maxBlockNumber: 10n, + } as any; + + await expect(createDefaultModeBlockRange(blockState, config)).rejects.toThrow( + StartBlockHigherThanEndBlockError + ); + }); +}); + +describe('createTestModeBlockRange', () => { + const originalLog = console.log; + beforeEach(() => { + jest.clearAllMocks(); + console.log = jest.fn(); + }); + + afterEach(() => { + console.log = originalLog; + }); + + it('should create a block range in test mode when startBlock is not a bigint', async () => { + const config = { + blockchain: {}, + startBlock: null, + mode: Mode.Test, + scanner: { scanKey: 'scanKey' }, + startFromHead: true, + } as any; + + const result = await createTestModeBlockRange(config); + + expect(result).toEqual({ + startBlock: 99n, + endBlock: 100n, + mode: Mode.Test, + scanKey: 'scanKey', + }); + expect(Blockchain.create).toHaveBeenCalledWith(config.blockchain); + }); + + it('should create a block range in test mode when startBlock is a bigint', async () => { + const config = { + blockchain: {}, + startBlock: 50n, + mode: Mode.Test, + scanner: { scanKey: 'scanKey' }, + startFromHead: true, + } as any; + + const result = await createTestModeBlockRange(config); + + expect(result).toEqual({ + startBlock: 50n, + endBlock: 51n, + mode: Mode.Test, + scanKey: 'scanKey', + }); + expect(Blockchain.create).toHaveBeenCalledWith(config.blockchain); + }); +}); + +describe('createReplayModeBlockRange', () => { + const originalLog = console.log; + beforeEach(() => { + jest.clearAllMocks(); + console.log = jest.fn(); // mock console.log to prevent log outputs during tests + }); + + afterEach(() => { + console.log = originalLog; // restore original console.log after tests + }); + + it('should throw an error when startBlock is not defined', async () => { + const scanner = { hasUnscannedBlocks: jest.fn().mockResolvedValue(false) } as any; + const config = { + blockchain: {}, + startBlock: null, + endBlock: null, + mode: Mode.Replay, + scanner: { scanKey: 'scanKey' }, + } as any; + + await expect(createReplayModeBlockRange(scanner, config)).rejects.toThrow( + UndefinedStartBlockError + ); + }); + + it('should throw an error when endBlock > lastIrreversibleBlock', async () => { + const scanner = { hasUnscannedBlocks: jest.fn().mockResolvedValue(false) } as any; + const config = { + blockchain: {}, + startBlock: 50n, + endBlock: 101n, + mode: Mode.Replay, + scanner: { scanKey: 'scanKey' }, + } as any; + + await expect(createReplayModeBlockRange(scanner, config)).rejects.toThrow( + EndBlockOutOfRangeError + ); + }); + + it('should throw an error when startBlock > endBlock', async () => { + const scanner = { hasUnscannedBlocks: jest.fn().mockResolvedValue(false) } as any; + const config = { + blockchain: {}, + startBlock: 50n, + endBlock: 49n, + mode: Mode.Replay, + scanner: { scanKey: 'scanKey' }, + } as any; + + await expect(createReplayModeBlockRange(scanner, config)).rejects.toThrow( + StartBlockHigherThanEndBlockError + ); + }); +}); diff --git a/src/bootstrap/__tests__/start-bootstrap.unit.test.ts b/src/bootstrap/__tests__/start-bootstrap.unit.test.ts new file mode 100644 index 0000000..1437e45 --- /dev/null +++ b/src/bootstrap/__tests__/start-bootstrap.unit.test.ts @@ -0,0 +1,163 @@ +import { Mode } from './../../common/common.enums'; +import { BroadcastTcpClient } from '@alien-worlds/broadcast'; +import { MongoSource } from '@alien-worlds/storage-mongodb'; +import { Abis, BlockRangeScanner, ContractReader } from '../../common'; +import { bootstrap } from '../start-bootstrap'; +import { NoAbisError } from '../bootstrap.errors'; +import { InternalBroadcastMessageName } from '../../broadcast/internal-broadcast.enums'; +import { + createDefaultModeBlockRange, + createTestModeBlockRange, +} from '../bootstrap.utils'; +import { ReaderBroadcastMessage } from '../../broadcast/messages'; + +jest.mock('../../common', () => ({ + Abis: { + create: jest + .fn() + .mockResolvedValue({ fetchAbis: jest.fn().mockResolvedValue([1, 2, 3]) }), + }, + BlockState: { create: jest.fn().mockResolvedValue({}) }, + ContractReader: { create: jest.fn().mockResolvedValue({ readContracts: jest.fn() }) }, + BlockRangeScanner: { create: jest.fn().mockResolvedValue({}) }, +})); + +jest.mock('@alien-worlds/broadcast', () => { + return { + BroadcastTcpClient: jest.fn().mockImplementation(() => { + return { + onMessage: jest.fn(), + sendMessage: jest.fn(), + connect: jest.fn(), + }; + }), + }; +}); + +jest.mock('@alien-worlds/storage-mongodb', () => ({ + MongoSource: { create: jest.fn().mockResolvedValue({}) }, +})); + +jest.mock('../bootstrap.utils', () => ({ + createDefaultModeBlockRange: jest.fn().mockResolvedValue({}), + createReplayModeBlockRange: jest.fn().mockResolvedValue({}), + createTestModeBlockRange: jest.fn().mockResolvedValue({}), +})); + +const featured = { + traces: [ + { + shipTraceMessageName: ['transaction_trace_v0'], + shipActionTraceMessageName: ['action_trace_v0', 'action_trace_v1'], + contract: ['uspts.worlds'], + action: ['addpoints'], + processor: 'USPTS_WORLDS_ACTION_PROCESSOR', + }, + { + shipTraceMessageName: ['transaction_trace_v0'], + shipActionTraceMessageName: ['action_trace_v0', 'action_trace_v1'], + contract: ['notify.world'], + action: ['logmine'], + processor: 'NOTIFY_WORLD_ACTION_PROCESSOR', + }, + ], + deltas: [ + { + shipDeltaMessageName: ['table_delta_v0'], + name: ['contract_row'], + code: ['msig.worlds'], + scope: ['*'], + table: ['*'], + processor: 'MSIG_WORLDS_DELTA_PROCESSOR', + }, + ], +}; + +describe('bootstrap', () => { + let mockBroadcast; + + beforeEach(() => { + mockBroadcast = new BroadcastTcpClient({}); + }); + + it('throws an error when there are no abis', async () => { + (Abis.create as jest.Mock).mockResolvedValueOnce({ + fetchAbis: jest.fn().mockResolvedValue([]), + }); + + const config = { + mode: Mode.Default, + broadcast: {}, + mongo: {}, + contractReader: {}, + abis: {}, + scanner: {}, + featured, + } as any; + await expect(bootstrap(config)).rejects.toThrow(NoAbisError); + }); + + it('runs without error in default mode', async () => { + const config = { + mode: Mode.Default, + broadcast: {}, + mongo: {}, + contractReader: {}, + abis: {}, + scanner: {}, + featured, + } as any; + await bootstrap(config); + + expect(BlockRangeScanner.create).toHaveBeenCalled(); + expect(ContractReader.create).toHaveBeenCalled(); + expect(Abis.create).toHaveBeenCalled(); + expect(MongoSource.create).toHaveBeenCalled(); + expect(BroadcastTcpClient).toHaveBeenCalled(); + }); + + it.skip('handles DefaultModeReaderReady message correctly', async () => { + const config = { + mode: Mode.Default, + broadcast: {}, + mongo: {}, + contractReader: {}, + abis: {}, + scanner: {}, + featured, + } as any; + await bootstrap(config); + console.log('=====', mockBroadcast.onMessage.mock); + const message = { name: InternalBroadcastMessageName.DefaultModeReaderReady }; + const messageHandler = mockBroadcast.onMessage.mock.calls[0][1]; + await messageHandler(message); + + // Replace the following expect lines with your actual testing assertions + expect(mockBroadcast.sendMessage).toHaveBeenCalledWith( + ReaderBroadcastMessage.newDefaultModeTask(expect.anything()) + ); + }); + + it.skip('handles TestModeReaderReady message correctly', async () => { + const config = { + mode: Mode.Test, + broadcast: {}, + mongo: {}, + contractReader: {}, + abis: {}, + scanner: {}, + featured, + } as any; + await bootstrap(config); + + const message = { name: InternalBroadcastMessageName.DefaultModeReaderReady }; + + const messageHandler = mockBroadcast.onMessage.mock.calls[0][1]; + await messageHandler(message); + + // Replace the following expect lines with your actual testing assertions + expect(mockBroadcast.sendMessage).toHaveBeenCalledWith( + ReaderBroadcastMessage.newDefaultModeTask(expect.anything()) + ); + }); +}); diff --git a/src/bootstrap/bootstrap.command.ts b/src/bootstrap/bootstrap.command.ts new file mode 100644 index 0000000..13c2211 --- /dev/null +++ b/src/bootstrap/bootstrap.command.ts @@ -0,0 +1,10 @@ +import { Command } from 'commander'; + +export const bootstrapCommand = new Command(); + +bootstrapCommand + .version('1.0', '-v, --version') + .option('-k, --scan-key ', 'Scan key') + .option('-s, --start-block ', 'Start at this block') + .option('-m, --mode ', 'Mode (default/replay/test)') + .option('-e, --end-block ', 'End block (exclusive)'); diff --git a/src/bootstrap/bootstrap.errors.ts b/src/bootstrap/bootstrap.errors.ts index 61e7163..44bf200 100644 --- a/src/bootstrap/bootstrap.errors.ts +++ b/src/bootstrap/bootstrap.errors.ts @@ -1,10 +1,28 @@ +/** + * Represents an error when the start block is undefined. + * + * @class + * @extends {Error} + */ export class UndefinedStartBlockError extends Error { constructor() { super(`Undefined start block. `); } } +/** + * Represents an error when the end block is out of range. + * + * @class + * @extends {Error} + */ export class EndBlockOutOfRangeError extends Error { + /** + * Constructs a new EndBlockOutOfRangeError. + * + * @param {bigint} endBlock - The end block that is out of range + * @param {bigint} lib - The last irreversible block number + */ constructor(endBlock: bigint, lib: bigint) { super( `End block (${endBlock.toString()}) out of range. A value greater than last irreversible block number (${lib.toString()})` @@ -12,7 +30,18 @@ export class EndBlockOutOfRangeError extends Error { } } +/** + * Represents an error when the start block is higher than the end block. + * + * @class + * @extends {Error} + */ export class StartBlockHigherThanEndBlockError extends Error { + /** + * Constructs a new StartBlockHigherThanEndBlockError + * @param {bigint} startBlock - The start block that is higher than the end block + * @param {bigint} endBlock - The end block + */ constructor(startBlock: bigint, endBlock: bigint) { super( `Error in the given range (${startBlock.toString()}-${endBlock.toString()}), the startBlock cannot be greater than the endBlock` @@ -20,6 +49,12 @@ export class StartBlockHigherThanEndBlockError extends Error { } } +/** + * Represents an error when there are no ABIs stored in the database. + * + * @class + * @extends {Error} + */ export class NoAbisError extends Error { constructor() { super(`There are no ABIs stored in the database`); diff --git a/src/bootstrap/bootstrap.types.ts b/src/bootstrap/bootstrap.types.ts index b930c58..8053458 100644 --- a/src/bootstrap/bootstrap.types.ts +++ b/src/bootstrap/bootstrap.types.ts @@ -1,9 +1,10 @@ -import { MongoConfig, BroadcastConfig } from '@alien-worlds/api-core'; import { AbisServiceConfig } from '../common/abis'; import { BlockRangeScanConfig } from '../reader/block-range-scanner'; import { BlockchainConfig, ContractReaderConfig } from '../common/blockchain'; import { FeaturedConfig } from '../common/featured'; import { Mode } from '../common'; +import { MongoConfig } from '@alien-worlds/storage-mongodb'; +import { BroadcastConfig } from '@alien-worlds/broadcast'; export type BootstrapConfig = { broadcast: BroadcastConfig; diff --git a/src/bootstrap/bootstrap.utils.ts b/src/bootstrap/bootstrap.utils.ts index 3af9f4a..401424e 100644 --- a/src/bootstrap/bootstrap.utils.ts +++ b/src/bootstrap/bootstrap.utils.ts @@ -11,7 +11,16 @@ import { } from './bootstrap.errors'; import { Blockchain } from '../common'; -export const createBlockRangeTaskInput = ( +/** + * Creates a block range task input based on the provided configuration and mode. + * + * @async + * @param {BlockState} blockState - The current block state. + * @param {BlockRangeScanner} scanner - The block range scanner. + * @param {BootstrapConfig} config - The bootstrap configuration. + * @returns {Promise} The block range task input. + */ +export const createBlockRangeTaskInput = async ( blockState: BlockState, scanner: BlockRangeScanner, config: BootstrapConfig @@ -32,9 +41,12 @@ export const createBlockRangeTaskInput = ( }; /** - * - * @param {Broadcast} broadcast - * @param {BootstrapConfig} config + * Creates a block range in default mode. + * + * @async + * @param {BlockState} blockState - The current block state. + * @param {BootstrapConfig} config - The bootstrap configuration. + * @returns {Promise} The block range data. */ export const createDefaultModeBlockRange = async ( blockState: BlockState, @@ -62,7 +74,9 @@ export const createDefaultModeBlockRange = async ( if (currentBlockNumber > 0n) { lowEdge = currentBlockNumber + 1n; - log(` Using the current state block number (+1) ${lowEdge.toString()} as a start block`); + log( + ` Using the current state block number (+1) ${lowEdge.toString()} as a start block` + ); } else { if (startBlock < 0n) { if (startFromHead) { @@ -106,9 +120,11 @@ export const createDefaultModeBlockRange = async ( }; /** - * - * @param {Broadcast} broadcast - * @param {BootstrapConfig} config + * Creates a block range in test mode. + * + * @async + * @param {BootstrapConfig} config - The bootstrap configuration. + * @returns {Promise} The block range data. */ export const createTestModeBlockRange = async ( config: BootstrapConfig @@ -139,9 +155,12 @@ export const createTestModeBlockRange = async ( }; /** - * - * @param {Broadcast} broadcast - * @param {BootstrapConfig} config + * Creates a block range in replay mode. + * + * @async + * @param {BlockRangeScanner} scanner - The block range scanner. + * @param {BootstrapConfig} config - The bootstrap configuration. + * @returns {Promise} The block range data. */ export const createReplayModeBlockRange = async ( scanner: BlockRangeScanner, diff --git a/src/bootstrap/start-bootstrap.ts b/src/bootstrap/start-bootstrap.ts index 24216f5..f3eaeef 100644 --- a/src/bootstrap/start-bootstrap.ts +++ b/src/bootstrap/start-bootstrap.ts @@ -7,33 +7,43 @@ import { createReplayModeBlockRange, createTestModeBlockRange, } from './bootstrap.utils'; -import { Broadcast, log, MongoSource } from '@alien-worlds/api-core'; -import { BootstrapConfig } from './bootstrap.types'; +import { BootstrapCommandOptions, BootstrapConfig } from './bootstrap.types'; import { NoAbisError } from './bootstrap.errors'; import { InternalBroadcastChannel, InternalBroadcastClientName, InternalBroadcastMessageName, } from '../broadcast/internal-broadcast.enums'; -import { FeaturedContractContent } from '../common/featured'; +import { FeaturedConfig, FeaturedContractContent } from '../common/featured'; import { Mode } from '../common/common.enums'; -import { InternalBroadcastMessage } from '../broadcast'; import { Abis, BlockRangeScanner, BlockState, ContractReader } from '../common'; +import { MongoSource } from '@alien-worlds/storage-mongodb'; +import { ConfigVars, log } from '@alien-worlds/api-core'; +import { BroadcastMessage, BroadcastTcpClient } from '@alien-worlds/broadcast'; +import { buildBootstrapConfig } from '../config'; +import { bootstrapCommand } from './bootstrap.command'; /** + * The bootstrap function initiates the bootstrap process based on the configuration provided. + * Depending on the mode of operation (default, replay, or test), it prepares the necessary + * resources such as a broadcast client, mongo source, contract reader, ABI data, scanner, + * featured contract details, and block state. + * It also sets up a message handler for the bootstrap broadcast channel to handle various + * messages related to the reader readiness in different modes. * - * @param broadcastMessageMapper - * @param config - * @returns + * @param {BootstrapConfig} config The bootstrap configuration object. + * @throws {NoAbisError} When no ABIs are fetched. + * + * @returns {Promise} The Promise that resolves when the bootstrap process has been initiated successfully. */ -export const startBootstrap = async (config: BootstrapConfig) => { +export const bootstrap = async (config: BootstrapConfig) => { const { mode } = config; log(`Bootstrap "${mode}" mode ... [starting]`); - const broadcast = await Broadcast.createClient({ - ...config.broadcast, - clientName: InternalBroadcastClientName.Bootstrap, - }); + const broadcast = await new BroadcastTcpClient( + config.broadcast, + InternalBroadcastClientName.Bootstrap + ); const mongo = await MongoSource.create(config.mongo); const contractReader = await ContractReader.create(config.contractReader, mongo); const abis = await Abis.create(mongo, config.abis, config.featured); @@ -62,8 +72,8 @@ export const startBootstrap = async (config: BootstrapConfig) => { broadcast.onMessage( InternalBroadcastChannel.Bootstrap, - async (message: InternalBroadcastMessage) => { - if (message.content.name === InternalBroadcastMessageName.DefaultModeReaderReady) { + async (message: BroadcastMessage) => { + if (message.name === InternalBroadcastMessageName.DefaultModeReaderReady) { if (config.mode === Mode.Default) { blockRange = await createDefaultModeBlockRange(blockState, config); broadcast.sendMessage(ReaderBroadcastMessage.newDefaultModeTask(blockRange)); @@ -73,9 +83,7 @@ export const startBootstrap = async (config: BootstrapConfig) => { blockRange = await createTestModeBlockRange(config); broadcast.sendMessage(ReaderBroadcastMessage.newDefaultModeTask(blockRange)); } - } else if ( - message.content.name === InternalBroadcastMessageName.ReplayModeReaderReady - ) { + } else if (message.name === InternalBroadcastMessageName.ReplayModeReaderReady) { broadcast.sendMessage(ReaderBroadcastMessage.newReplayModeTask(blockRange)); } else { // @@ -87,3 +95,20 @@ export const startBootstrap = async (config: BootstrapConfig) => { log(`Bootstrap ${mode} mode ... [ready]`); }; + +/** + * startBootstrap is a function that takes command line options to build a bootstrap configuration + * and then initiate the bootstrap process. It throws an error if the bootstrap process fails. + * + * @param {BootstrapCommandOptions} options The command line options for bootstrap. + * @throws {Error} When the bootstrap process fails. + * + * @returns {Promise} The Promise that resolves when the bootstrap process has been initiated successfully. + */ +export const startBootstrap = (args: string[], featured: FeaturedConfig) => { + const vars = new ConfigVars(); + const options = bootstrapCommand.parse(args).opts(); + const config = buildBootstrapConfig(vars, featured, options); + + bootstrap(config).catch(log); +}; diff --git a/src/broadcast/__tests__/internal-broadcast.message.unit.test.ts b/src/broadcast/__tests__/internal-broadcast.message.unit.test.ts new file mode 100644 index 0000000..1c24cbd --- /dev/null +++ b/src/broadcast/__tests__/internal-broadcast.message.unit.test.ts @@ -0,0 +1,23 @@ +import { InternalBroadcastMessage } from '../internal-broadcast.message'; +import { BroadcastTcpMessageType } from '@alien-worlds/broadcast'; + +describe('InternalBroadcastMessage', () => { + describe('create', () => { + it('should create an internal broadcast message', () => { + const content = { + sender: null, + channel: null, + name: null, + data: null, + } as any; + const message = InternalBroadcastMessage.create(content); + + expect(message).toBeInstanceOf(InternalBroadcastMessage); + expect(message.sender).toBeNull(); + expect(message.channel).toBeNull(); + expect(message.type).toBe(BroadcastTcpMessageType.Data); + expect(message.name).toBeNull(); + expect(message.data).toBeNull(); + }); + }); +}); diff --git a/src/broadcast/internal-broadcast.message.ts b/src/broadcast/internal-broadcast.message.ts index f40688b..71b8f90 100644 --- a/src/broadcast/internal-broadcast.message.ts +++ b/src/broadcast/internal-broadcast.message.ts @@ -1,21 +1,35 @@ -import { BroadcastTcpMessage, BroadcastTcpMessageType } from '@alien-worlds/api-core'; -import { InternalBroadcastMessageName } from './internal-broadcast.enums'; +import { + BroadcastTcpMessage, + BroadcastTcpMessageContent, + BroadcastTcpMessageType, +} from '@alien-worlds/broadcast'; +/** + * Represents an internal broadcast message. + * + * @template DataType - The type of data for the message. + */ export class InternalBroadcastMessage< DataType = unknown > extends BroadcastTcpMessage { + /** + * Creates an internal broadcast message. + * + * @param {BroadcastTcpMessageContent} content - The content for the message. + * @returns {InternalBroadcastMessage} The created internal broadcast message. + */ public static create( - sender: string, - channel: string, - name: InternalBroadcastMessageName, - data: DataType + content: BroadcastTcpMessageContent ) { - return new InternalBroadcastMessage({ + const { sender, channel, name, data } = content; + return new InternalBroadcastMessage( + null, sender, channel, + BroadcastTcpMessageType.Data, name, - type: BroadcastTcpMessageType.Data, - data, - }); + null, + data + ); } } diff --git a/src/broadcast/messages/__tests__/filter-broadcast.message.unit.test.ts b/src/broadcast/messages/__tests__/filter-broadcast.message.unit.test.ts new file mode 100644 index 0000000..2380135 --- /dev/null +++ b/src/broadcast/messages/__tests__/filter-broadcast.message.unit.test.ts @@ -0,0 +1,19 @@ +import { FilterBroadcastMessage } from '../filter-broadcast.message'; + +describe('FilterBroadcastMessage', () => { + describe('ready', () => { + it('should create a ready broadcast message', () => { + const message = FilterBroadcastMessage.ready(); + + expect(message).toBeDefined(); + }); + }); + + describe('refresh', () => { + it('should create a refresh broadcast message', () => { + const message = FilterBroadcastMessage.refresh(); + + expect(message).toBeDefined(); + }); + }); +}); diff --git a/src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts b/src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts new file mode 100644 index 0000000..1d48a2a --- /dev/null +++ b/src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts @@ -0,0 +1,28 @@ +import { ProcessorBroadcastMessage } from '../processor-broadcast.message'; +import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { + InternalBroadcastChannel, + InternalBroadcastMessageName, +} from '../../internal-broadcast.enums'; + +describe('ProcessorBroadcastMessage', () => { + describe('ready', () => { + it('should create a ready broadcast message', () => { + const message = ProcessorBroadcastMessage.ready(); + + expect(message).toBeInstanceOf(BroadcastMessage); + expect(message.channel).toBe(InternalBroadcastChannel.Processor); + expect(message.name).toBe(InternalBroadcastMessageName.ProcessorReady); + }); + }); + + describe('refresh', () => { + it('should create a refresh broadcast message', () => { + const message = ProcessorBroadcastMessage.refresh(); + + expect(message).toBeInstanceOf(BroadcastMessage); + expect(message.channel).toBe(InternalBroadcastChannel.Processor); + expect(message.name).toBe(InternalBroadcastMessageName.ProcessorRefresh); + }); + }); +}); diff --git a/src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts b/src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts new file mode 100644 index 0000000..7622d95 --- /dev/null +++ b/src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts @@ -0,0 +1,60 @@ +import { + ReaderBroadcastMessage, + ReaderBroadcastMessageData, +} from '../reader-broadcast.message'; +import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { Mode } from '../../../common/common.enums'; +import { + InternalBroadcastChannel, + InternalBroadcastMessageName, +} from '../../internal-broadcast.enums'; + +describe('ReaderBroadcastMessage', () => { + describe('newReplayModeTask', () => { + it('should create a new replay mode task broadcast message', () => { + const data: ReaderBroadcastMessageData = { + mode: '', + }; + const message = ReaderBroadcastMessage.newReplayModeTask(data); + + expect(message).toBeInstanceOf(BroadcastMessage); + expect(message.channel).toBe(InternalBroadcastChannel.ReplayModeReader); + expect(message.name).toBe(InternalBroadcastMessageName.ReaderTask); + expect(data.mode).toBe(Mode.Replay); + }); + }); + + describe('newDefaultModeTask', () => { + it('should create a new default mode task broadcast message', () => { + const data: ReaderBroadcastMessageData = { + mode: '', + }; + const message = ReaderBroadcastMessage.newDefaultModeTask(data); + + expect(message).toBeInstanceOf(BroadcastMessage); + expect(message.channel).toBe(InternalBroadcastChannel.DefaultModeReader); + expect(message.name).toBe(InternalBroadcastMessageName.ReaderTask); + expect(data.mode).toBe(Mode.Default); + }); + }); + + describe('defaultModeReady', () => { + it('should create a default mode ready broadcast message', () => { + const message = ReaderBroadcastMessage.defaultModeReady(); + + expect(message).toBeInstanceOf(BroadcastMessage); + expect(message.channel).toBe(InternalBroadcastChannel.Bootstrap); + expect(message.name).toBe(InternalBroadcastMessageName.DefaultModeReaderReady); + }); + }); + + describe('replayModeReady', () => { + it('should create a replay mode ready broadcast message', () => { + const message = ReaderBroadcastMessage.replayModeReady(); + + expect(message).toBeInstanceOf(BroadcastMessage); + expect(message.channel).toBe(InternalBroadcastChannel.Bootstrap); + expect(message.name).toBe(InternalBroadcastMessageName.ReplayModeReaderReady); + }); + }); +}); diff --git a/src/broadcast/messages/filter-broadcast.message.ts b/src/broadcast/messages/filter-broadcast.message.ts index 73e689d..95a3b0b 100644 --- a/src/broadcast/messages/filter-broadcast.message.ts +++ b/src/broadcast/messages/filter-broadcast.message.ts @@ -1,26 +1,38 @@ -import { BroadcastTcpMessageType } from '@alien-worlds/api-core'; +import { BroadcastMessage } from '@alien-worlds/broadcast'; import { InternalBroadcastChannel, InternalBroadcastMessageName, } from '../internal-broadcast.enums'; /** - * Message content + * Represents a class for filter broadcast messages. */ export class FilterBroadcastMessage { + /** + * Creates a ready broadcast message. + * + * @returns {BroadcastMessage} The ready broadcast message. + */ public static ready() { - return { - channel: InternalBroadcastChannel.Bootstrap, - name: InternalBroadcastMessageName.FilterReady, - type: BroadcastTcpMessageType.Data, - }; + return BroadcastMessage.create( + null, + InternalBroadcastChannel.Bootstrap, + null, + InternalBroadcastMessageName.FilterReady + ); } + /** + * Creates a refresh broadcast message. + * + * @returns {BroadcastMessage} The refresh broadcast message. + */ public static refresh() { - return { - channel: InternalBroadcastChannel.Filter, - name: InternalBroadcastMessageName.FilterRefresh, - type: BroadcastTcpMessageType.Data, - }; + return BroadcastMessage.create( + null, + InternalBroadcastChannel.Filter, + null, + InternalBroadcastMessageName.FilterRefresh + ); } } diff --git a/src/broadcast/messages/processor-broadcast.message.ts b/src/broadcast/messages/processor-broadcast.message.ts index 5c25a59..748e585 100644 --- a/src/broadcast/messages/processor-broadcast.message.ts +++ b/src/broadcast/messages/processor-broadcast.message.ts @@ -1,26 +1,37 @@ -import { BroadcastTcpMessageType } from '@alien-worlds/api-core'; +import { BroadcastMessage } from '@alien-worlds/broadcast'; import { InternalBroadcastChannel, InternalBroadcastMessageName, } from '../internal-broadcast.enums'; /** - * Message content + * Represents a class for processor broadcast messages. */ export class ProcessorBroadcastMessage { - public static ready() { - return { - channel: InternalBroadcastChannel.Processor, - name: InternalBroadcastMessageName.ProcessorReady, - type: BroadcastTcpMessageType.Data, - }; + /** + * Creates a ready broadcast message. + * + * @returns {BroadcastMessage} The ready broadcast message. + */ + public static ready(): BroadcastMessage { + return BroadcastMessage.create( + null, + InternalBroadcastChannel.Processor, + null, + InternalBroadcastMessageName.ProcessorReady + ); } - - public static refresh() { - return { - channel: InternalBroadcastChannel.Processor, - name: InternalBroadcastMessageName.ProcessorRefresh, - type: BroadcastTcpMessageType.Data, - }; + /** + * Creates a refresh broadcast message. + * + * @returns {BroadcastMessage} The refresh broadcast message. + */ + public static refresh(): BroadcastMessage { + return BroadcastMessage.create( + null, + InternalBroadcastChannel.Processor, + null, + InternalBroadcastMessageName.ProcessorRefresh + ); } } diff --git a/src/broadcast/messages/reader-broadcast.message.ts b/src/broadcast/messages/reader-broadcast.message.ts index 58a83ac..13db09c 100644 --- a/src/broadcast/messages/reader-broadcast.message.ts +++ b/src/broadcast/messages/reader-broadcast.message.ts @@ -1,10 +1,13 @@ -import { BroadcastTcpMessageType } from '@alien-worlds/api-core'; +import { BroadcastMessage } from '@alien-worlds/broadcast'; import { Mode } from '../../common/common.enums'; import { InternalBroadcastChannel, InternalBroadcastMessageName, } from '../internal-broadcast.enums'; +/** + * Data structure for the reader broadcast message. + */ export type ReaderBroadcastMessageData = { startBlock?: bigint; endBlock?: bigint; @@ -13,42 +16,63 @@ export type ReaderBroadcastMessageData = { }; /** - * Message content + * Represents a class for reader broadcast messages. */ export class ReaderBroadcastMessage { - public static newReplayModeTask(data: ReaderBroadcastMessageData) { + /** + * Creates a new replay mode task broadcast message. + * + * @param {ReaderBroadcastMessageData} data - The data for the message. + * @returns {BroadcastMessage} The new replay mode task broadcast message. + */ + public static newReplayModeTask(data: ReaderBroadcastMessageData): BroadcastMessage { data.mode = Mode.Replay; - return { - channel: InternalBroadcastChannel.ReplayModeReader, - name: InternalBroadcastMessageName.ReaderTask, - type: BroadcastTcpMessageType.Data, + return BroadcastMessage.create( + null, + InternalBroadcastChannel.ReplayModeReader, data, - }; + InternalBroadcastMessageName.ReaderTask + ); } - - public static newDefaultModeTask(data: ReaderBroadcastMessageData) { + /** + * Creates a new default mode task broadcast message. + * + * @param {ReaderBroadcastMessageData} data - The data for the message. + * @returns {BroadcastMessage} The new default mode task broadcast message. + */ + public static newDefaultModeTask(data: ReaderBroadcastMessageData): BroadcastMessage { data.mode = Mode.Default; - return { - channel: InternalBroadcastChannel.DefaultModeReader, - name: InternalBroadcastMessageName.ReaderTask, - type: BroadcastTcpMessageType.Data, + return BroadcastMessage.create( + null, + InternalBroadcastChannel.DefaultModeReader, data, - }; + InternalBroadcastMessageName.ReaderTask + ); } - - public static defaultModeReady() { - return { - channel: InternalBroadcastChannel.Bootstrap, - name: InternalBroadcastMessageName.DefaultModeReaderReady, - type: BroadcastTcpMessageType.Data, - }; + /** + * Creates a default mode ready broadcast message. + * + * @returns {BroadcastMessage} The default mode ready broadcast message. + */ + public static defaultModeReady(): BroadcastMessage { + return BroadcastMessage.create( + null, + InternalBroadcastChannel.Bootstrap, + null, + InternalBroadcastMessageName.DefaultModeReaderReady + ); } - - public static replayModeReady() { - return { - channel: InternalBroadcastChannel.Bootstrap, - name: InternalBroadcastMessageName.ReplayModeReaderReady, - type: BroadcastTcpMessageType.Data, - }; + /** + * Creates a replay mode ready broadcast message. + * + * @returns {BroadcastMessage} The replay mode ready broadcast message. + */ + public static replayModeReady(): BroadcastMessage { + return BroadcastMessage.create( + null, + InternalBroadcastChannel.Bootstrap, + null, + InternalBroadcastMessageName.ReplayModeReaderReady + ); } } diff --git a/src/common/__tests__/common.utils.unit.test.ts b/src/common/__tests__/common.utils.unit.test.ts new file mode 100644 index 0000000..8818893 --- /dev/null +++ b/src/common/__tests__/common.utils.unit.test.ts @@ -0,0 +1,23 @@ +import { isSetAbiAction } from '../common.utils'; + +describe('isSetAbiAction', () => { + it('should return true for eosio setabi action', () => { + const result = isSetAbiAction('eosio', 'setabi'); + expect(result).toBe(true); + }); + + it('should return false for other contract or action', () => { + const result = isSetAbiAction('eosio', 'otherAction'); + expect(result).toBe(false); + }); + + it('should return false for other contract and setabi action', () => { + const result = isSetAbiAction('otherContract', 'setabi'); + expect(result).toBe(false); + }); + + it('should return false for other contract and action', () => { + const result = isSetAbiAction('otherContract', 'otherAction'); + expect(result).toBe(false); + }); +}); diff --git a/src/common/abis/__tests__/abi.repository-impl.unit.test.ts b/src/common/abis/__tests__/abi.repository-impl.unit.test.ts new file mode 100644 index 0000000..a0a6e2f --- /dev/null +++ b/src/common/abis/__tests__/abi.repository-impl.unit.test.ts @@ -0,0 +1,115 @@ +import { AbisRepositoryImpl } from '../abis.repository-impl'; +import { + Result, + CountParams, + Mapper, + QueryBuilders, + DataSource, +} from '@alien-worlds/api-core'; +import { AbisCache } from '../abis.cache'; +import { ContractEncodedAbi } from '../contract-encoded-abi'; + +jest.mock('../abis.cache'); + +const mockDataSource = { + find: jest.fn(), + count: jest.fn(), + aggregate: jest.fn(), + update: jest.fn(), + insert: jest.fn(), + remove: jest.fn(), + startTransaction: jest.fn(), + commitTransaction: jest.fn(), + rollbackTransaction: jest.fn(), +} as any; + +const mockMapper = { + toEntity: jest.fn(), + fromEntity: jest.fn(), + getEntityKeyMapping: jest.fn(), +} as any; + +const mockQueryBuilders = { + buildFindQuery: jest.fn(), + buildCountQuery: jest.fn(), + buildUpdateQuery: jest.fn(), + buildRemoveQuery: jest.fn(), + buildAggregationQuery: jest.fn(), +} as any; + +describe('AbisRepositoryImpl', () => { + let repository: AbisRepositoryImpl; + let mockCache: jest.Mocked; + + beforeEach(() => { + mockCache = new AbisCache() as jest.Mocked; + repository = new AbisRepositoryImpl(mockDataSource, mockMapper, mockQueryBuilders); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe('getAbis', () => { + it('should return abis from cache if present', async () => { + const mockAbis: ContractEncodedAbi[] = [ + /* mock data here */ + ]; + mockCache.getAbis.mockReturnValue(mockAbis); + + const result = await repository.getAbis({ contracts: ['contract1'] }); + + expect(result).toEqual(Result.withContent(mockAbis)); + expect(mockCache.getAbis).toHaveBeenCalledWith({ contracts: ['contract1'] }); + }); + }); + + describe('getAbi', () => { + it('should return abi from cache if present', async () => { + const mockAbi: ContractEncodedAbi = {} as any; + const blockNumber = BigInt(10); + const contract = 'contract1'; + + mockCache.getAbi.mockReturnValue(mockAbi); + + const result = await repository.getAbi(blockNumber, contract); + + expect(result).toEqual(Result.withContent(mockAbi)); + expect(mockCache.getAbi).toHaveBeenCalledWith(blockNumber, contract); + }); + }); + + describe('insertAbis', () => { + it('should insert abis into the cache and the database', async () => { + const mockAbis: ContractEncodedAbi[] = [ + /* mock data here */ + ]; + const addSpy = jest.spyOn(repository, 'add'); + addSpy.mockResolvedValue(Result.withContent(mockAbis)); + + const result = await repository.insertAbis(mockAbis); + + expect(result).toEqual(Result.withContent(mockAbis.length > 0)); + expect(mockCache.insertAbis).toHaveBeenCalledWith(mockAbis); + expect(addSpy).toHaveBeenCalledWith(mockAbis); + }); + }); + + describe('countAbis', () => { + it('should count abis based on the startBlock and endBlock', async () => { + const countSpy = jest.spyOn(repository, 'count'); + const mockCount = 5; + countSpy.mockResolvedValue(Result.withContent(mockCount)); + + const startBlock = BigInt(10); + const endBlock = BigInt(20); + + const result = await repository.countAbis(startBlock, endBlock); + + expect(result).toEqual(Result.withContent(mockCount)); + expect(countSpy).toHaveBeenCalledWith( + CountParams.create({ where: expect.anything() }) + ); + }); + }); +}); diff --git a/src/common/abis/__tests__/abi.repository.unit.test.ts b/src/common/abis/__tests__/abi.repository.unit.test.ts deleted file mode 100644 index 07cf619..0000000 --- a/src/common/abis/__tests__/abi.repository.unit.test.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* eslint-disable @typescript-eslint/no-empty-function */ -/* eslint-disable @typescript-eslint/no-explicit-any */ - -import { Long } from 'mongodb'; -import { ContractEncodedAbi } from '../contract-encoded-abi'; - -const document = { - block_number: Long.fromBigInt(100n), - contract: 'foo', - hex: 'foo_hex', -}; - -describe('Abi Repository Unit tests', () => { - it('"create" should create Abi instance', async () => { - const abi = ContractEncodedAbi.create(100, 'foo', 'foo_hex'); - expect(abi).toBeInstanceOf(ContractEncodedAbi); - }); -}); diff --git a/src/common/abis/__tests__/abis.cache.unit.test.ts b/src/common/abis/__tests__/abis.cache.unit.test.ts new file mode 100644 index 0000000..56e0293 --- /dev/null +++ b/src/common/abis/__tests__/abis.cache.unit.test.ts @@ -0,0 +1,104 @@ +import { AbisCache } from '../abis.cache'; + +describe('AbisCache', () => { + let abisCache: AbisCache; + + beforeEach(() => { + abisCache = new AbisCache(); + }); + + describe('getAbis', () => { + it('should return an empty array when cache is empty', () => { + const result = abisCache.getAbis({ contracts: ['eosio'] }); + expect(result).toEqual([]); + }); + + it('should return an empty array when contracts array is empty', () => { + abisCache.insertAbis([ + { contract: 'eosio', blockNumber: 1n } as any, + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]); + + const result = abisCache.getAbis({ contracts: [] }); + expect(result).toEqual([]); + }); + + it('should return all ABIs when no startBlock or endBlock is provided', () => { + abisCache.insertAbis([ + { contract: 'eosio', blockNumber: 1n } as any, + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]); + + const result = abisCache.getAbis({ contracts: ['eosio'] }); + expect(result).toEqual([ + { contract: 'eosio', blockNumber: 1n } as any, + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]); + }); + + it('should return matching ABIs within the specified range', () => { + abisCache.insertAbis([ + { contract: 'eosio', blockNumber: 1n } as any, + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]); + + const result = abisCache.getAbis({ + contracts: ['eosio'], + startBlock: 2n, + endBlock: 3n, + }); + expect(result).toEqual([ + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]); + }); + }); + + describe('getAbi', () => { + it('should return null when cache is empty', () => { + const result = abisCache.getAbi(1n, 'eosio'); + expect(result).toBeNull(); + }); + + it('should return the latest ABI that matches the block number', () => { + abisCache.insertAbis([ + { contract: 'eosio', blockNumber: 1n } as any, + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]); + + const result = abisCache.getAbi(2n, 'eosio'); + expect(result).toEqual({ contract: 'eosio', blockNumber: 2n } as any); + }); + + it('should return null when no matching ABI is found', () => { + abisCache.insertAbis([ + { contract: 'eosio', blockNumber: 1n } as any, + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]); + + const result = abisCache.getAbi(4n, 'undefined'); + expect(result).toBeNull(); + }); + }); + + describe('insertAbis', () => { + it('should insert ABIs into the cache', () => { + const abis = [ + { contract: 'eosio', blockNumber: 1n } as any, + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]; + + abisCache.insertAbis(abis); + + const result = abisCache.getAbis({ contracts: ['eosio'] }); + expect(result).toEqual(abis); + }); + }); +}); diff --git a/src/common/abis/__tests__/abis.service.unit.test.ts b/src/common/abis/__tests__/abis.service.unit.test.ts new file mode 100644 index 0000000..b37e518 --- /dev/null +++ b/src/common/abis/__tests__/abis.service.unit.test.ts @@ -0,0 +1,62 @@ +import { AbisService } from '../abis.service'; +import fetch from 'node-fetch'; +import { ContractEncodedAbi } from '../contract-encoded-abi'; + +// Mock dependencies +jest.mock('node-fetch'); + +const mockFetch = fetch as jest.MockedFunction; + +describe('AbisService', () => { + let abisService: AbisService; + const config = { + url: 'https://example.com', + limit: 100, + filter: 'eosio:setabi', + }; + + beforeEach(() => { + // Reset mocks and create a new instance of AbisService + jest.resetAllMocks(); + abisService = new AbisService(config); + }); + + describe('fetchAbis', () => { + it('should fetch ABIs for a contract', async () => { + const contract = '0x123'; + const mockActions = [ + { block_num: 1, act: { data: { abi: 'ABI1' } } }, + { block_num: 2, act: { data: { abi: 'ABI2' } } }, + ]; + const expectedAbis = [ + ContractEncodedAbi.create(1, contract, 'ABI1'), + ContractEncodedAbi.create(2, contract, 'ABI2'), + ]; + + mockFetch.mockResolvedValue({ + json: jest.fn().mockResolvedValue({ actions: mockActions }), + } as Response); + + const result = await abisService.fetchAbis(contract); + + expect(mockFetch).toHaveBeenCalledTimes(1); + expect(mockFetch).toHaveBeenCalledWith( + `${config.url}/v2/history/get_actions?account=${contract}&filter=${config.filter}&limit=${config.limit}&sort=-1` + ); + expect(result).toEqual(expectedAbis); + }); + + it('should handle fetch errors and return an empty array', async () => { + const contract = '0x123'; + + mockFetch.mockRejectedValue(new Error('Fetch error')); + + const result = await abisService.fetchAbis(contract); + + expect(mockFetch).toHaveBeenCalledTimes(1); + expect(result).toEqual([]); + }); + + // Add more test cases for different scenarios + }); +}); diff --git a/src/common/abis/__tests__/abis.unit.test.ts b/src/common/abis/__tests__/abis.unit.test.ts new file mode 100644 index 0000000..f4231b3 --- /dev/null +++ b/src/common/abis/__tests__/abis.unit.test.ts @@ -0,0 +1,143 @@ +import { Abis } from '../abis'; +import { AbisRepositoryImpl } from '../abis.repository-impl'; +import { AbisService } from '../abis.service'; +import { Failure, Result } from '@alien-worlds/api-core'; +import { AbisServiceNotSetError } from '../abis.errors'; +import { AbiNotFoundError } from '@alien-worlds/block-reader'; + +// Mock dependencies +jest.mock('../abis.repository-impl'); +jest.mock('../abis.service'); + +const mockAbisRepository = AbisRepositoryImpl as jest.MockedClass; +const mockAbisService = AbisService as jest.MockedClass; + +describe('Abis', () => { + let abis: Abis; + + beforeEach(() => { + // Reset mocks and create a new instance of Abis + jest.resetAllMocks(); + abis = new Abis(new mockAbisRepository(), new mockAbisService()); + }); + + describe('getAbis', () => { + it('should return ABIs from repository when available', async () => { + const mockAbis = [/* Mocked ABIs */]; + mockAbisRepository.prototype.getAbis.mockResolvedValue(Result.withContent(mockAbis)); + + const result = await abis.getAbis(); + + expect(mockAbisRepository.prototype.getAbis).toHaveBeenCalledTimes(1); + expect(result.isFailure).toBe(false); + expect(result.content).toEqual(mockAbis); + }); + + it('should fetch ABIs when none are available in the repository and fetch option is enabled', async () => { + const mockAbis = [/* Mocked ABIs */]; + mockAbisRepository.prototype.getAbis.mockResolvedValue(Result.withContent([])); + abis.fetchAbis = jest.fn().mockResolvedValue(Result.withContent(mockAbis)); + + const result = await abis.getAbis({ fetch: true }); + + expect(mockAbisRepository.prototype.getAbis).toHaveBeenCalledTimes(1); + expect(abis.fetchAbis).toHaveBeenCalledTimes(1); + expect(result.isFailure).toBe(false); + expect(result.content).toEqual(mockAbis); + }); + + // Add more test cases for different scenarios + }); + + describe('getAbi', () => { + it('should return ABI from repository when available', async () => { + const mockAbi = /* Mocked ABI */; + mockAbisRepository.prototype.getAbi.mockResolvedValue(Result.withContent(mockAbi)); + + const result = await abis.getAbi(123n, '0x123'); + + expect(mockAbisRepository.prototype.getAbi).toHaveBeenCalledTimes(1); + expect(result.isFailure).toBe(false); + expect(result.content).toEqual(mockAbi); + }); + + it('should fetch ABI when not available in the repository and fetch option is enabled', async () => { + const mockAbi = /* Mocked ABI */; + mockAbisRepository.prototype.getAbi.mockResolvedValue(Result.withFailure(Failure.fromError(new AbiNotFoundError()))); + abis.fetchAbis = jest.fn().mockResolvedValue(Result.withContent([mockAbi])); + + const result = await abis.getAbi(123n, '0x123', true); + + expect(mockAbisRepository.prototype.getAbi).toHaveBeenCalledTimes(1); + expect(abis.fetchAbis).toHaveBeenCalledTimes(1); + expect(result.isFailure).toBe(false); + expect(result.content).toEqual(mockAbi); + }); + + // Add more test cases for different scenarios + }); + + describe('storeAbi', () => { + it('should insert the ABI into the repository', async () => { + const blockNumber = 123n; + const contract = '0x123'; + const hex = '0xabcdef'; + + mockAbisRepository.prototype.insertAbis.mockResolvedValue(Result.withContent(true)); + + const result = await abis.storeAbi(blockNumber, contract, hex); + + expect(mockAbisRepository.prototype.insertAbis).toHaveBeenCalledTimes(1); + expect(mockAbisRepository.prototype.insertAbis).toHaveBeenCalledWith([ + expect.objectContaining({ blockNumber, contract, hex }), + ]); + expect(result.isFailure).toBe(false); + expect(result.content).toBe(true); + }); + + // Add more test cases for different scenarios + }); + + describe('fetchAbis', () => { + it('should throw AbisServiceNotSetError when service is not set', async () => { + abis = new Abis(new mockAbisRepository()); // Create instance without AbisService + + await expect(abis.fetchAbis()).rejects.toThrow(AbisServiceNotSetError); + }); + + it('should fetch ABIs using the service', async () => { + const mockAbis = [/* Mocked ABIs */]; + const mockContracts = ['0x123', '0x456']; + const mockServiceResponse = Result.withContent(mockAbis); + + abis = new Abis(new mockAbisRepository(), new mockAbisService()); + mockAbisService.prototype.fetchAbis.mockResolvedValue(mockServiceResponse); + + const result = await abis.fetchAbis(mockContracts); + + expect(mockAbisService.prototype.fetchAbis).toHaveBeenCalledTimes(mockContracts.length); + expect(mockAbisService.prototype.fetchAbis).toHaveBeenCalledWith(expect.any(String)); + expect(result.isFailure).toBe(false); + expect(result.content).toEqual(mockAbis); + }); + + // Add more test cases for different scenarios + }); + + describe('cacheAbis', () => { + it('should cache ABIs in the repository', async () => { + const mockContracts = ['0x123', '0x456']; + + mockAbisRepository.prototype.cacheAbis.mockResolvedValue(Result.withoutContent()); + + const result = await abis.cacheAbis(mockContracts); + + expect(mockAbisRepository.prototype.cacheAbis).toHaveBeenCalledTimes(1); + expect(mockAbisRepository.prototype.cacheAbis).toHaveBeenCalledWith(mockContracts); + expect(result.isFailure).toBe(false); + expect(result.content).toBeUndefined(); + }); + + // Add more test cases for different scenarios + }); +}); diff --git a/src/common/abis/__tests__/contract-encoded-abi.unit.test.ts b/src/common/abis/__tests__/contract-encoded-abi.unit.test.ts new file mode 100644 index 0000000..1090891 --- /dev/null +++ b/src/common/abis/__tests__/contract-encoded-abi.unit.test.ts @@ -0,0 +1,104 @@ +import { ContractEncodedAbi } from '../contract-encoded-abi'; +import { parseToBigInt } from '@alien-worlds/api-core'; +import { ContractEncodedAbiDocument, ContractEncodedAbiJson } from '../abis.types'; +import { MongoDB } from '@alien-worlds/storage-mongodb'; + +// Mock dependencies +jest.mock('@alien-worlds/api-core'); +jest.mock('@alien-worlds/storage-mongodb'); + +const mockParseToBigInt = parseToBigInt as jest.MockedFunction; +const mockMongoDBLong = MongoDB.Long as jest.MockedClass; + +describe('ContractEncodedAbi', () => { + const blockNumber = BigInt(123); + const contract = '0x123'; + const hex = '0xabcdef'; + + beforeEach(() => { + // Reset mocks + jest.resetAllMocks(); + }); + + describe('fromDocument', () => { + it('should create a ContractEncodedAbi instance from a document', () => { + const document: ContractEncodedAbiDocument = { + block_number: mockMongoDBLong.fromBigInt(blockNumber), + contract, + hex, + }; + + const result = ContractEncodedAbi.fromDocument(document); + + expect(result.blockNumber).toBe(blockNumber); + expect(result.contract).toBe(contract); + expect(result.hex).toBe(hex); + }); + + // Add more test cases for different scenarios + }); + + describe('create', () => { + it('should create a ContractEncodedAbi instance with the specified values', () => { + mockParseToBigInt.mockReturnValueOnce(blockNumber); + + const result = ContractEncodedAbi.create(blockNumber, contract, hex); + + expect(mockParseToBigInt).toHaveBeenCalledTimes(1); + expect(mockParseToBigInt).toHaveBeenCalledWith(blockNumber); + expect(result.blockNumber).toBe(blockNumber); + expect(result.contract).toBe(contract); + expect(result.hex).toBe(hex); + }); + + // Add more test cases for different scenarios + }); + + describe('constructor', () => { + it('should initialize the ContractEncodedAbi instance with the provided values', () => { + const result = new ContractEncodedAbi(blockNumber, contract, hex); + + expect(result.blockNumber).toBe(blockNumber); + expect(result.contract).toBe(contract); + expect(result.hex).toBe(hex); + }); + + // Add more test cases for different scenarios + }); + + describe('toDocument', () => { + it('should convert the ContractEncodedAbi instance to a document', () => { + const instance = new ContractEncodedAbi(blockNumber, contract, hex); + const expectedDocument: ContractEncodedAbiDocument = { + block_number: mockMongoDBLong.fromBigInt(blockNumber), + contract, + hex, + }; + + const result = instance.toDocument(); + + expect(mockMongoDBLong.fromBigInt).toHaveBeenCalledTimes(1); + expect(mockMongoDBLong.fromBigInt).toHaveBeenCalledWith(blockNumber); + expect(result).toEqual(expectedDocument); + }); + + // Add more test cases for different scenarios + }); + + describe('toJson', () => { + it('should convert the ContractEncodedAbi instance to JSON', () => { + const instance = new ContractEncodedAbi(blockNumber, contract, hex); + const expectedJson: ContractEncodedAbiJson = { + blockNumber, + contract, + hex, + }; + + const result = instance.toJson(); + + expect(result).toEqual(expectedJson); + }); + + // Add more test cases for different scenarios + }); +}); diff --git a/src/common/abis/abis.cache.ts b/src/common/abis/abis.cache.ts index d31bdb8..b6a1a1e 100644 --- a/src/common/abis/abis.cache.ts +++ b/src/common/abis/abis.cache.ts @@ -1,21 +1,50 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { ContractEncodedAbi } from './contract-encoded-abi'; +/** + * A function that filters ABI entries based on the block number being greater than or equal to the start block. + * @param startBlock - The start block number. + * @param endBlock - The end block number. + * @returns A filter function. + */ const filterFromStartBlock = (startBlock: bigint, endBlock: bigint) => (abi: ContractEncodedAbi) => abi.blockNumber >= startBlock; +/** + * A function that filters ABI entries based on the block number being less than or equal to the end block. + * @param startBlock - The start block number. + * @param endBlock - The end block number. + * @returns A filter function. + */ const filterTillEndBlock = (startBlock: bigint, endBlock: bigint) => (abi: ContractEncodedAbi) => abi.blockNumber <= endBlock; +/** + * A function that filters ABI entries based on the block number being within the specified range. + * @param startBlock - The start block number. + * @param endBlock - The end block number. + * @returns A filter function. + */ const filterInRange = (startBlock: bigint, endBlock: bigint) => (abi: ContractEncodedAbi) => abi.blockNumber >= startBlock && abi.blockNumber <= endBlock; +/** + * Class representing the cache for storing and retrieving contract ABIs. + */ export class AbisCache { private cache: Map> = new Map(); + /** + * Retrieves contract ABIs from the cache based on the specified options. + * @param options - Options for retrieving the contract ABIs. + * @param options.startBlock - The start block number to filter the ABIs. + * @param options.endBlock - The end block number to filter the ABIs. + * @param options.contracts - An array of contract addresses to filter the ABIs. + * @returns An array of matching contract ABIs. + */ public getAbis(options: { startBlock?: bigint; endBlock?: bigint; @@ -63,6 +92,12 @@ export class AbisCache { return abis; } + /** + * Retrieves the ABI for the specified block number and contract address. + * @param blockNumber - The block number to find the ABI. + * @param contract - The contract address. + * @returns The matching ABI or null if not found. + */ public getAbi(blockNumber: bigint, contract: string): ContractEncodedAbi { if (this.cache.has(contract)) { const abis = this.cache.get(contract); @@ -80,6 +115,10 @@ export class AbisCache { return null; } + /** + * Inserts an array of ABIs into the cache. + * @param abis - An array of ABIs to insert into the cache. + */ public insertAbis(abis: ContractEncodedAbi[]): void { abis.forEach(abi => { let set = this.cache.get(abi.contract); diff --git a/src/common/abis/abis.collection.ts b/src/common/abis/abis.collection.ts new file mode 100644 index 0000000..3f1c663 --- /dev/null +++ b/src/common/abis/abis.collection.ts @@ -0,0 +1,21 @@ +import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; +import { ContractEncodedAbiDocument } from './abis.types'; + +/** + * Represents a collection of ABIs (Application Binary Interfaces) stored in a MongoDB collection. + * Extends the MongoCollectionSource class to provide database operations for the ABIs. + */ +export class AbisCollection extends MongoCollectionSource { + /** + * Constructs a new instance of the AbisCollection class. + * + * @param {MongoSource} source - The MongoDB source used for database operations. + */ + constructor(source: MongoSource) { + super(source, 'history_tools.abis', { + indexes: [ + { key: { block_number: 1, hex: 1, contract: 1 }, unique: true, background: true }, + ], + }); + } +} diff --git a/src/common/abis/abis.serialize.ts b/src/common/abis/abis.eos.serializer.ts similarity index 74% rename from src/common/abis/abis.serialize.ts rename to src/common/abis/abis.eos.serializer.ts index 8b89835..694a88f 100644 --- a/src/common/abis/abis.serialize.ts +++ b/src/common/abis/abis.eos.serializer.ts @@ -4,7 +4,8 @@ import { Serialize } from 'eosjs'; import { Authorization, hexToUint8Array } from 'eosjs/dist/eosjs-serialize'; import { Abi } from 'eosjs/dist/eosjs-rpc-interfaces'; import { log } from '@alien-worlds/api-core'; -import { AbiTableJson } from '../blockchain/abi'; +import { AbiTableJson } from '@alien-worlds/block-reader'; +import { AbisSerializer } from './abis.serializer'; export type SerializeUtil = { deserializeAction: ( @@ -22,13 +23,26 @@ export type SerializeUtil = { ) => T; }; -export class AbisSerialize { - public static deserializeAction = ( +/** + * Represents a class for deserializing EOS ABIs. + * @template T + */ +export class AbisEosSerializer implements AbisSerializer { + /** + * Deserializes the action data for a specific account and action. + * + * @param {string} account - The account associated with the action. + * @param {string} action - The action name. + * @param {Uint8Array} data - The raw data to be deserialized. + * @param {string} hex - The hexadecimal representation of the data. + * @returns {T} The deserialized action data. + */ + public deserializeAction( account: string, action: string, data: Uint8Array, hex: string - ): T => { + ): T { try { const authorization: Authorization[] = []; const textEncoder = new TextEncoder(); @@ -70,14 +84,23 @@ export class AbisSerialize { log(error); return null; } - }; + } - public static deserializeTable = ( + /** + * Deserializes the table data for a specific account and table. + * + * @param {string} account - The account associated with the table. + * @param {string} table - The table name. + * @param {Uint8Array} data - The raw data to be deserialized. + * @param {string} hex - The hexadecimal representation of the data. + * @returns {T} The deserialized table data. + */ + public deserializeTable( account: string, table: string, data: Uint8Array, hex: string - ): T => { + ): T { try { const textEncoder = new TextEncoder(); const textDecoder = new TextDecoder(); @@ -117,5 +140,5 @@ export class AbisSerialize { } catch (e) { return null; } - }; + } } diff --git a/src/common/abis/abis.repository-impl.ts b/src/common/abis/abis.repository-impl.ts new file mode 100644 index 0000000..649ebaa --- /dev/null +++ b/src/common/abis/abis.repository-impl.ts @@ -0,0 +1,144 @@ +import { ContractEncodedAbi } from './contract-encoded-abi'; +import { + CountParams, + FindParams, + log, + RepositoryImpl, + Result, + Where, +} from '@alien-worlds/api-core'; +import { AbisCache } from './abis.cache'; +import { AbisRepository } from './abis.repository'; + +/** + * Implements the AbisRepository with caching functionality. + * This class manages ContractEncodedAbi entities, providing CRUD operations and additional functionalities such as caching. + * It extends the base RepositoryImpl and implements the AbisRepository interface. + */ +export class AbisRepositoryImpl + extends RepositoryImpl + implements AbisRepository +{ + /** + * Cache instance for caching the ABI objects. + * @private + */ + private cache: AbisCache = new AbisCache(); + + /** + * Cache the ABIs. + * @param {string[]} [contracts] - List of contracts. + */ + public async cacheAbis(contracts?: string[]) { + const abis = await this.getAbis({ contracts }); + if (Array.isArray(abis)) { + this.cache.insertAbis(abis); + } + } + + /** + * Retrieve the ABIs. + * @param options - Filter options for retrieving ABIs. + * @returns Promise that resolves with the Result of an array of ContractEncodedAbi. + */ + public async getAbis(options: { + startBlock?: bigint; + endBlock?: bigint; + contracts?: string[]; + }): Promise> { + try { + const { startBlock, endBlock, contracts } = options || {}; + + const cachedAbis = this.cache.getAbis(options); + + if (cachedAbis.length > 0) { + return Result.withContent(cachedAbis); + } + const where = Where.bind(); + + if (startBlock && endBlock) { + where.props().blockNumber.isGte(startBlock).isLte(endBlock); + } + + if (contracts) { + where.props().contract.isIn(contracts); + } + + return this.find(FindParams.create({ where })); + } catch (error) { + log(error); + return Result.withContent([]); + } + } + + /** + * Retrieve a specific ABI. + * @param blockNumber - The block number. + * @param contract - The contract name. + * @returns Promise that resolves with the Result of ContractEncodedAbi. + */ + public async getAbi( + blockNumber: bigint, + contract: string + ): Promise> { + const cachedAbi = this.cache.getAbi(blockNumber, contract); + + if (cachedAbi) { + return Result.withContent(cachedAbi); + } + + const where = Where.bind(); + where.props().blockNumber.isLte(blockNumber).props().contract.isEq(contract); + const { content, failure } = await this.find( + FindParams.create({ where, limit: 1, sort: { block_number: -1 } }) + ); + + if (content) { + return Result.withContent(content[0]); + } + + if (failure) { + return Result.withFailure(failure); + } + } + + /** + * Inserts an array of ABIs. + * @param abis - The ABIs to be inserted. + * @returns Promise that resolves with the Result of boolean indicating the success of the operation. + */ + public async insertAbis(abis: ContractEncodedAbi[]): Promise> { + this.cache.insertAbis(abis); + const { content, failure } = await this.add(abis); + + if (failure) { + log(failure.error); + return Result.withContent(false); + } + + return Result.withContent(content.length > 0); + } + + /** + * Counts the number of ABIs between the provided block numbers. + * @param startBlock - The start block number. + * @param endBlock - The end block number. + * @returns Promise that resolves with the Result of number indicating the count of ABIs. + */ + public async countAbis( + startBlock?: bigint, + endBlock?: bigint + ): Promise> { + const where = Where.bind(); + + if (typeof startBlock === 'bigint') { + where.props().blockNumber.isGte(startBlock); + } + + if (typeof endBlock === 'bigint') { + where.props().blockNumber.isLte(endBlock); + } + + return this.count(CountParams.create({ where })); + } +} diff --git a/src/common/abis/abis.repository.ts b/src/common/abis/abis.repository.ts index e4f4314..064acc1 100644 --- a/src/common/abis/abis.repository.ts +++ b/src/common/abis/abis.repository.ts @@ -1,158 +1,54 @@ -/* eslint-disable @typescript-eslint/unbound-method */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { ContractEncodedAbiDocument } from './abis.types'; +import { Result } from '@alien-worlds/api-core'; import { ContractEncodedAbi } from './contract-encoded-abi'; -import { - CollectionMongoSource, - DataSourceBulkWriteError, - DataSourceOperationError, - log, - MongoDB, - MongoSource, - OperationErrorType, -} from '@alien-worlds/api-core'; -import { AbisCache } from './abis.cache'; -export class AbisCollection extends CollectionMongoSource { - constructor(source: MongoSource) { - super(source, 'history_tools.abis', { - indexes: [ - { key: { block_number: 1, hex: 1, contract: 1 }, unique: true, background: true }, - ], - }); - } -} -export class AbisRepository { - private cache: AbisCache = new AbisCache(); - - constructor(private collection: AbisCollection) {} - - public async cacheAbis(contracts?: string[]) { - const abis = await this.getAbis({ contracts }); - if (Array.isArray(abis)) { - this.cache.insertAbis(abis); - } - } - - public async getAbis(options: { +/** + * This class manages ContractEncodedAbi entities, providing CRUD operations and additional functionalities such as caching. + * It extends the base RepositoryImpl and implements the AbisRepository interface. + */ +export abstract class AbisRepository { + /** + * Cache the ABIs. + * @param {string[]} [contracts] - List of contracts. + */ + public abstract cacheAbis(contracts?: string[]); + + /** + * Retrieve the ABIs. + * @param options - Filter options for retrieving ABIs. + * @returns Promise that resolves with the Result of an array of ContractEncodedAbi. + */ + public abstract getAbis(options: { startBlock?: bigint; endBlock?: bigint; contracts?: string[]; - }): Promise { - try { - const { startBlock, endBlock, contracts } = options || {}; - - const cachedAbis = this.cache.getAbis(options); - - if (cachedAbis.length > 0) { - return cachedAbis; - } - - const filter: { block_number?: unknown; contract?: unknown } = {}; - - if (startBlock && endBlock) { - filter.block_number = { - $gte: MongoDB.Long.fromBigInt(startBlock), - $lte: MongoDB.Long.fromBigInt(endBlock), - }; - } - - if (contracts) { - filter.contract = { $in: contracts }; - } - - const documents = await this.collection.find({ filter }); - const entities = documents.map(ContractEncodedAbi.fromDocument); - - return entities; - } catch (error) { - log(error); - return []; - } - } - - public async getAbi( + }): Promise>; + + /** + * Retrieve a specific ABI. + * @param blockNumber - The block number. + * @param contract - The contract name. + * @returns Promise that resolves with the Result of ContractEncodedAbi. + */ + public abstract getAbi( blockNumber: bigint, contract: string - ): Promise { - try { - const cachedAbi = this.cache.getAbi(blockNumber, contract); - - if (cachedAbi) { - return cachedAbi; - } - - const filter: { block_number: unknown; contract: unknown } = { - block_number: { - $lte: MongoDB.Long.fromBigInt(blockNumber), - }, - contract: { $eq: contract }, - }; - - const document = await this.collection.findOne({ - filter, - options: { sort: { block_number: -1 }, limit: 1 }, - }); - - return document ? ContractEncodedAbi.fromDocument(document) : null; - } catch (error) { - log(error); - return null; - } - } - - public async insertAbi(abi: ContractEncodedAbi): Promise { - try { - this.cache.insertAbis([abi]); - const result = await this.collection.insert(abi.toDocument()); - return !!result; - } catch (error) { - const { type } = error as DataSourceOperationError; - if (type !== OperationErrorType.Duplicate) { - log(error); - } - return false; - } - } - - public async insertAbis(abis: ContractEncodedAbi[]): Promise { - try { - this.cache.insertAbis(abis); - const documents = abis.map(abi => abi.toDocument()); - const result = await this.collection.insertMany(documents); - return result.length > 0; - } catch (error) { - const { writeErrors } = error as DataSourceBulkWriteError; - - if (writeErrors && writeErrors.length > 0) { - writeErrors.forEach(error => { - if (error.type !== OperationErrorType.Duplicate) { - log(error); - } - }); - } - - return false; - } - } - - public async countAbis(startBlock?: bigint, endBlock?: bigint): Promise { - try { - const filter: MongoDB.Filter = {}; - if (typeof startBlock === 'bigint') { - filter['block_number'] = { $gte: MongoDB.Long.fromBigInt(startBlock) }; - } - - if (typeof endBlock === 'bigint') { - filter['block_number'] = { $lte: MongoDB.Long.fromBigInt(endBlock) }; - } - - const count = await this.collection.count({ filter }); - - return count; - } catch (error) { - return 0; - } - } + ): Promise>; + + /** + * Inserts an array of ABIs. + * @param abis - The ABIs to be inserted. + * @returns Promise that resolves with the Result of boolean indicating the success of the operation. + */ + public abstract insertAbis(abis: ContractEncodedAbi[]): Promise>; + + /** + * Counts the number of ABIs between the provided block numbers. + * @param startBlock - The start block number. + * @param endBlock - The end block number. + * @returns Promise that resolves with the Result of number indicating the count of ABIs. + */ + public abstract countAbis( + startBlock?: bigint, + endBlock?: bigint + ): Promise>; } diff --git a/src/common/abis/abis.serializer.ts b/src/common/abis/abis.serializer.ts new file mode 100644 index 0000000..53ea057 --- /dev/null +++ b/src/common/abis/abis.serializer.ts @@ -0,0 +1,37 @@ +/** + * Represents an abstract class for deserializing ABIs (Application Binary Interfaces) related to actions and tables. + * @template T + */ +export abstract class AbisSerializer { + /** + * Deserializes the action data for a specific account and action. + * + * @param {string} account - The account associated with the action. + * @param {string} action - The action name. + * @param {Uint8Array} data - The raw data to be deserialized. + * @param {string} hex - The hexadecimal representation of the data. + * @returns {T} The deserialized action data. + */ + public abstract deserializeAction( + account: string, + action: string, + data: Uint8Array, + hex: string + ): T; + + /** + * Deserializes the table data for a specific account and table. + * + * @param {string} account - The account associated with the table. + * @param {string} table - The table name. + * @param {Uint8Array} data - The raw data to be deserialized. + * @param {string} hex - The hexadecimal representation of the data. + * @returns {T} The deserialized table data. + */ + public abstract deserializeTable( + account: string, + table: string, + data: Uint8Array, + hex: string + ): T; +} diff --git a/src/common/abis/abis.service.ts b/src/common/abis/abis.service.ts index 0d5f848..8333c22 100644 --- a/src/common/abis/abis.service.ts +++ b/src/common/abis/abis.service.ts @@ -4,9 +4,23 @@ import { AbisServiceConfig } from './abis.types'; import fetch from 'node-fetch'; import { ContractEncodedAbi } from './contract-encoded-abi'; +/** + * Represents a service for fetching ABIs (Application Binary Interfaces). + */ export class AbisService { + /** + * Constructs a new instance of the AbisService class. + * + * @param {AbisServiceConfig} config - The configuration for the AbisService. + */ constructor(private config: AbisServiceConfig) {} + /** + * Fetches the ABIs for a specific contract. + * + * @param {string} contract - The contract address. + * @returns {Promise} A promise that resolves to an array of ContractEncodedAbi objects representing the fetched ABIs. + */ public async fetchAbis(contract: string): Promise { try { const list: ContractEncodedAbi[] = []; diff --git a/src/common/abis/abis.ts b/src/common/abis/abis.ts index 8f8c520..eb67031 100644 --- a/src/common/abis/abis.ts +++ b/src/common/abis/abis.ts @@ -1,45 +1,26 @@ -import { MongoConfig, MongoSource, log } from '@alien-worlds/api-core'; +import { Failure, Result, log } from '@alien-worlds/api-core'; import { FeaturedConfig } from '../featured'; import { ContractEncodedAbi } from './contract-encoded-abi'; import { AbisServiceNotSetError } from './abis.errors'; -import { AbisCollection, AbisRepository } from './abis.repository'; +import { AbisRepository } from './abis.repository'; import { AbisService } from './abis.service'; -import { AbisServiceConfig } from './abis.types'; +import { AbiNotFoundError } from '@alien-worlds/block-reader'; +/** + * Represents a collection of ABIs (Application Binary Interfaces) for smart contracts. + */ export class Abis { - public static async create( - mongo: MongoSource | MongoConfig, - abisConfig?: AbisServiceConfig, - featured?: FeaturedConfig, - setCache?: boolean - ): Promise { - let mongoSource: MongoSource; - - log(` * Abis ... [starting]`); - - if (mongo instanceof MongoSource) { - mongoSource = mongo; - } else { - mongoSource = await MongoSource.create(mongo); - } - const collection = new AbisCollection(mongoSource); - const repository = new AbisRepository(collection); - const service = abisConfig ? new AbisService(abisConfig) : null; - const abis = new Abis(repository, service, featured); - - if (setCache) { - await abis.cacheAbis(); - log(` * Abis cache restored`); - } - - log(` * Abis ... [ready]`); - - return abis; - } - private contracts: Set = new Set(); - private constructor( + /** + * Constructs a new instance of the Abis class. + * + * @param {AbisRepository} repository - The repository for accessing ABIs. + * @param {AbisService} [service] - The service for fetching ABIs. + * @param {FeaturedConfig} [featuredConfig] - The featured configuration containing contract traces and deltas. + * @private + */ + constructor( private repository: AbisRepository, private service?: AbisService, featuredConfig?: FeaturedConfig @@ -66,36 +47,59 @@ export class Abis { } } + /** + * Retrieves the ABIs (Application Binary Interfaces) for the specified options. + * + * @param {Object} [options] - The options for retrieving the ABIs. + * @param {bigint} [options.startBlock] - The starting block number. + * @param {bigint} [options.endBlock] - The ending block number. + * @param {string[]} [options.contracts] - The contract addresses to filter the ABIs. + * @param {boolean} [options.fetch] - Indicates whether to fetch ABIs if not found in the database. + * @returns {Promise>} A promise that resolves to a Result object containing the retrieved ABIs. + */ public async getAbis(options?: { startBlock?: bigint; endBlock?: bigint; contracts?: string[]; fetch?: boolean; - }): Promise { + }): Promise> { const { startBlock, endBlock, contracts, fetch } = options || {}; - let abis = await this.repository.getAbis(options); + const { content: abis } = await this.repository.getAbis(options); if (abis.length === 0 && fetch) { log( `No contract ABIs (${startBlock}-${endBlock}) were found in the database. Trying to fetch ABIs...` ); - abis = await this.fetchAbis(contracts); + return this.fetchAbis(contracts); } - return abis; + return Result.withContent(abis || []); } + /** + * Retrieves the ABI (Application Binary Interface) for the specified block number and contract address. + * + * @param {bigint} blockNumber - The block number. + * @param {string} contract - The contract address. + * @param {boolean} [fetch=false] - Indicates whether to fetch the ABI if not found in the database. + * @returns {Promise>} A promise that resolves to a Result object containing the retrieved ABI. + */ public async getAbi( blockNumber: bigint, contract: string, fetch = false - ): Promise { - let abi = await this.repository.getAbi(blockNumber, contract); + ): Promise> { + const getAbiResult = await this.repository.getAbi(blockNumber, contract); - if (fetch && !abi) { - const abis = await this.fetchAbis([contract]); - abi = abis.reduce((result, abi) => { + if (fetch && getAbiResult.isFailure) { + const { content: abis, failure } = await this.fetchAbis([contract]); + + if (failure) { + return Result.withFailure(failure); + } + + const abi = abis.reduce((result, abi) => { if (abi.blockNumber <= blockNumber) { if (!result || result.blockNumber < abi.blockNumber) { result = abi; @@ -103,22 +107,44 @@ export class Abis { } return result; }, null); + + if (abi) { + return Result.withContent(abi); + } + + return Result.withFailure(Failure.fromError(new AbiNotFoundError())); } - return abi; + return getAbiResult; } + /** + * Stores the ABI (Application Binary Interface) for the specified block number, contract address, and hex code. + * + * @param {unknown} blockNumber - The block number. + * @param {string} contract - The contract address. + * @param {string} hex - The hex code representing the ABI. + * @returns {Promise>} A promise that resolves to a Result object indicating the success or failure of the operation. + */ public async storeAbi( blockNumber: unknown, contract: string, hex: string - ): Promise { - return this.repository.insertAbi( - ContractEncodedAbi.create(blockNumber, contract, hex) - ); + ): Promise> { + return this.repository.insertAbis([ + ContractEncodedAbi.create(blockNumber, contract, hex), + ]); } - public async fetchAbis(contracts?: string[]): Promise { + /** + * Fetches the ABIs (Application Binary Interfaces) for the specified contracts. + * + * @param {string[]} [contracts] - The contract addresses to fetch ABIs for. + * @returns {Promise>} A promise that resolves to a Result object containing the fetched ABIs. + * @throws {AbisServiceNotSetError} Thrown if the AbisService is not set. + * @private + */ + public async fetchAbis(contracts?: string[]): Promise> { if (!this.service) { throw new AbisServiceNotSetError(); } @@ -137,14 +163,23 @@ export class Abis { } catch (error) { log(error.message); } - return abis; + return Result.withContent(abis); } - public async cacheAbis(contracts?: string[]): Promise { + /** + * Caches the ABIs (Application Binary Interfaces) for the specified contracts. + * + * @param {string[]} [contracts] - The contract addresses to cache ABIs for. + * @returns {Promise>} A promise that resolves to a Result object indicating the success or failure of the operation. + * @private + */ + public async cacheAbis(contracts?: string[]): Promise> { try { await this.repository.cacheAbis(contracts); + return Result.withoutContent(); } catch (error) { log(error.message); + return Result.withFailure(Failure.fromError(error)); } } } diff --git a/src/common/abis/abis.types.ts b/src/common/abis/abis.types.ts index f1bf80a..3b6894e 100644 --- a/src/common/abis/abis.types.ts +++ b/src/common/abis/abis.types.ts @@ -1,4 +1,4 @@ -import { MongoDB, MongoConfig } from '@alien-worlds/api-core'; +import { MongoConfig, MongoDB } from '@alien-worlds/storage-mongodb'; import { FeaturedConfig } from '../featured'; export type ContractEncodedAbiJson = { diff --git a/src/common/abis/contract-encoded-abi.ts b/src/common/abis/contract-encoded-abi.ts index c6f1d7e..c09fe82 100644 --- a/src/common/abis/contract-encoded-abi.ts +++ b/src/common/abis/contract-encoded-abi.ts @@ -1,12 +1,30 @@ -import { MongoDB, parseToBigInt } from '@alien-worlds/api-core'; +import { parseToBigInt } from '@alien-worlds/api-core'; import { ContractEncodedAbiDocument, ContractEncodedAbiJson } from './abis.types'; +import { MongoDB } from '@alien-worlds/storage-mongodb'; +/** + * Represents an encoded ABI (Application Binary Interface) for a contract. + */ export class ContractEncodedAbi { + /** + * Creates a ContractEncodedAbi instance from a document. + * + * @param {ContractEncodedAbiDocument} document - The document containing the encoded ABI data. + * @returns {ContractEncodedAbi} A new ContractEncodedAbi instance. + */ public static fromDocument(document: ContractEncodedAbiDocument): ContractEncodedAbi { const { block_number, contract, hex } = document; return new ContractEncodedAbi(parseToBigInt(block_number), contract, hex); } + /** + * Creates a ContractEncodedAbi instance with the specified block number, contract address, and hex code. + * + * @param {unknown} blockNumber - The block number. + * @param {string} contract - The contract address. + * @param {string} hex - The hex code representing the ABI. + * @returns {ContractEncodedAbi} A new ContractEncodedAbi instance. + */ public static create( blockNumber: unknown, contract: string, @@ -15,12 +33,24 @@ export class ContractEncodedAbi { return new ContractEncodedAbi(parseToBigInt(blockNumber), contract, hex); } - private constructor( + /** + * Constructs a new instance of the ContractEncodedAbi class. + * + * @param {bigint} blockNumber - The block number. + * @param {string} contract - The contract address. + * @param {string} hex - The hex code representing the ABI. + */ + constructor( public readonly blockNumber: bigint, public readonly contract: string, public readonly hex: string ) {} + /** + * Converts the ContractEncodedAbi instance to a document format. + * + * @returns {ContractEncodedAbiDocument} The ContractEncodedAbi instance as a document. + */ public toDocument(): ContractEncodedAbiDocument { const { blockNumber, hex, contract } = this; @@ -31,6 +61,11 @@ export class ContractEncodedAbi { }; } + /** + * Converts the ContractEncodedAbi instance to a JSON format. + * + * @returns {ContractEncodedAbiJson} The ContractEncodedAbi instance as JSON. + */ public toJson(): ContractEncodedAbiJson { const { blockNumber, hex, contract } = this; diff --git a/src/common/abis/index.ts b/src/common/abis/index.ts index a4c8f59..2f910c0 100644 --- a/src/common/abis/index.ts +++ b/src/common/abis/index.ts @@ -5,4 +5,4 @@ export * from './abis.repository'; export * from './abis.service'; export * from './abis.types'; export * from './abis.errors'; -export * from './abis.serialize'; +export * from './abis.serializer'; diff --git a/src/common/block-state/__tests__/block-state.unit.test.ts b/src/common/block-state/__tests__/block-state.unit.test.ts new file mode 100644 index 0000000..c28adff --- /dev/null +++ b/src/common/block-state/__tests__/block-state.unit.test.ts @@ -0,0 +1,121 @@ +import { + DataSource, + Mapper, + QueryBuilders, + QueryBuilder, + Result, + RepositoryImpl, + Failure, +} from '@alien-worlds/api-core'; +import { BlockStateData, BlockStateDocument } from '../block-state.types'; +import { BlockState } from '../block-state'; + +jest.mock('@alien-worlds/api-core'); +jest.mock('./block-state.types'); + +describe('BlockState', () => { + let blockState; + let dataSourceMock; + let mapperMock; + let queryBuildersMock; + let queryBuilderMock; + + beforeEach(() => { + dataSourceMock = { + find: jest.fn(), + count: jest.fn(), + aggregate: jest.fn(), + update: jest.fn(), + insert: jest.fn(), + remove: jest.fn(), + startTransaction: jest.fn(), + commitTransaction: jest.fn(), + rollbackTransaction: jest.fn(), + } as any; + mapperMock = { + toEntity: jest.fn(), + fromEntity: jest.fn(), + getEntityKeyMapping: jest.fn(), + } as any; + queryBuildersMock = { + buildFindQuery: jest.fn(), + buildCountQuery: jest.fn(), + buildUpdateQuery: jest.fn(), + buildRemoveQuery: jest.fn(), + buildAggregationQuery: jest.fn(), + } as any; + queryBuilderMock = { + with: jest.fn(), + build: jest.fn(), + } as any; + + blockState = new BlockState( + dataSourceMock, + mapperMock, + queryBuilderMock, + queryBuilderMock + ); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should be a valid instance', () => { + expect(blockState).toBeInstanceOf(BlockState); + expect(blockState).toBeInstanceOf(RepositoryImpl); + }); + + describe('getState()', () => { + it('should return block state data', async () => { + const mockData = { + content: [ + { + lastModifiedTimestamp: new Date(), + actions: [], + tables: [], + blockNumber: 0n, + }, + ], + }; + blockState.find = jest.fn().mockResolvedValue(mockData); + + const result = await blockState.getState(); + + expect(blockState.find).toHaveBeenCalled(); + expect(result).toEqual(Result.withContent(mockData.content[0])); + }); + + it('should handle error properly', async () => { + const mockError = new Error('Database error'); + blockState.find = jest.fn().mockRejectedValue(mockError); + + const result = await blockState.getState(); + + expect(blockState.find).toHaveBeenCalled(); + expect(result).toEqual(Result.withFailure(Failure.fromError(mockError))); + }); + }); + + describe('updateBlockNumber()', () => { + it('should update block number and return true if successful', async () => { + const mockValue = 10n; + const mockData = { + content: { + modifiedCount: 1, + upsertedCount: 0, + }, + }; + blockState.update = jest.fn().mockResolvedValue(mockData); + + const result = await blockState.updateBlockNumber(mockValue); + + expect(blockState.update).toHaveBeenCalledWith( + queryBuilderMock.with({ blockNumber: mockValue }) + ); + expect(result).toEqual(Result.withContent(true)); + }); + + // Write similar tests for getBlockNumber() here... + }); +}); diff --git a/src/common/block-state/block-state.collection.ts b/src/common/block-state/block-state.collection.ts new file mode 100644 index 0000000..ce789b8 --- /dev/null +++ b/src/common/block-state/block-state.collection.ts @@ -0,0 +1,8 @@ +import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; +import { BlockStateDocument } from './block-state.types'; + +export class BlockStateCollection extends MongoCollectionSource { + constructor(source: MongoSource) { + super(source, 'history_tools.block_state'); + } +} diff --git a/src/common/block-state/block-state.source.ts b/src/common/block-state/block-state.source.ts deleted file mode 100644 index e5b66d9..0000000 --- a/src/common/block-state/block-state.source.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { - CollectionMongoSource, - MongoDB, - MongoSource, - parseToBigInt, -} from '@alien-worlds/api-core'; -import { BlockStateDocument } from './block-state.types'; - -export class BlockStateSource extends CollectionMongoSource { - constructor(mongoSource: MongoSource) { - super(mongoSource, 'history_tools.block_state'); - } - - public async getState(): Promise { - const state: BlockStateDocument = await this.findOne({ filter: {} }); - - return state; - } - - public async newState( - blockNumber: bigint, - actions: string[] = [], - tables: string[] = [] - ): Promise { - const result = await this.update( - { - block_number: MongoDB.Long.fromBigInt(blockNumber), - actions, - tables, - last_modified_timestamp: new Date(), - }, - { options: { upsert: true } } - ); - if (result) { - await this.update( - { last_modified_timestamp: new Date() }, - { options: { upsert: true } } - ); - } - } - - /** - * Updates block number. - * (Only if given value is higher than the one currently stored in the database) - * - * @param {bigint} value - */ - public async updateBlockNumber(value: bigint): Promise { - const result = await this.update( - { - $max: { block_number: MongoDB.Long.fromBigInt(value) }, - $set: { last_modified_timestamp: new Date() }, - }, - { options: { upsert: true } } - ); - return !!result; - } - - public async removeActions(labels: string[]): Promise { - await this.update( - { $pull: { actions: { $in: labels } }, last_modified_timestamp: new Date() }, - { options: { upsert: true } } - ); - } - - public async setActions(labels: string[]): Promise { - await this.update( - { actions: labels, last_modified_timestamp: new Date() }, - { options: { upsert: true } } - ); - } - - public async removeTables(labels: string[]): Promise { - await this.update( - { $pull: { tables: { $in: labels } }, last_modified_timestamp: new Date() }, - { options: { upsert: true } } - ); - } - - public async setTables(labels: string[]): Promise { - await this.update( - { tables: labels, last_modified_timestamp: new Date() }, - { options: { upsert: true } } - ); - } - - public async getBlockNumber(): Promise { - const state: BlockStateDocument = await this.findOne({ filter: {} }); - - return parseToBigInt(state?.block_number ? state.block_number : MongoDB.Long.NEG_ONE); - } - - public async getActions(): Promise { - const state: BlockStateDocument = await this.findOne({ filter: {} }); - - return state?.actions || []; - } - - public async getTables(): Promise { - const state: BlockStateDocument = await this.findOne({ filter: {} }); - - return state?.tables || []; - } - - public async includesAction(label: string): Promise { - const state: BlockStateDocument = await this.findOne({ filter: {} }); - - return state?.actions.includes(label) === true; - } - - public async includesTable(label: string): Promise { - const state: BlockStateDocument = await this.findOne({ filter: {} }); - - return state?.tables.includes(label) === true; - } -} diff --git a/src/common/block-state/block-state.ts b/src/common/block-state/block-state.ts index 3db814e..e1f9f4b 100644 --- a/src/common/block-state/block-state.ts +++ b/src/common/block-state/block-state.ts @@ -1,104 +1,103 @@ import { + DataSource, Failure, - log, - MongoConfig, - MongoSource, - parseToBigInt, + Mapper, + QueryBuilder, + QueryBuilders, + RepositoryImpl, Result, } from '@alien-worlds/api-core'; -import { BlockStateSource } from './block-state.source'; -import { BlockStateData } from './block-state.types'; +import { BlockStateData, BlockStateDocument } from './block-state.types'; -export class BlockState { - public static async create(mongo: MongoSource | MongoConfig) { - log(` * Block State ... [starting]`); - - let state: BlockState; - - if (mongo instanceof MongoSource) { - state = new BlockState(mongo); - } else { - const mongoSource = await MongoSource.create(mongo); - state = new BlockState(mongoSource); - } - - log(` * Block State ... [ready]`); - return state; - } - - private source: BlockStateSource; - - private constructor(mongo: MongoSource) { - this.source = new BlockStateSource(mongo); +/** + * A class representing a block state. + * @extends RepositoryImpl + */ +export class BlockState extends RepositoryImpl { + /** + * Creates an instance of the BlockState class. + * + * @param {DataSource} source - The data source. + * @param {Mapper} mapper - The data mapper. + * @param {QueryBuilders} queryBuilders - The query builders. + * @param {QueryBuilder} updateBlockNumberQueryBuilder - The query builder to update block number. + */ + constructor( + source: DataSource, + mapper: Mapper, + queryBuilders: QueryBuilders, + private updateBlockNumberQueryBuilder: QueryBuilder + ) { + super(source, mapper, queryBuilders); } + /** + * Fetches the current state of the data source. + * + * @returns {Promise>} - The result of the operation. + */ public async getState(): Promise> { try { - const state = await this.source.getState(); - let data: BlockStateData; - if (state) { - const { last_modified_timestamp, actions, tables, block_number } = state; - data = { - lastModifiedTimestamp: last_modified_timestamp || new Date(), + const { content: states } = await this.find(); + + if (states) { + const state = states[0]; + const { lastModifiedTimestamp, actions, tables, blockNumber } = state; + + return Result.withContent({ + lastModifiedTimestamp: lastModifiedTimestamp || new Date(), actions: actions || [], tables: tables || [], - blockNumber: parseToBigInt(block_number) || 0n, - }; + blockNumber: blockNumber || 0n, + }); } - data = { + return Result.withContent({ lastModifiedTimestamp: new Date(), actions: [], tables: [], blockNumber: 0n, - }; - return Result.withContent(data); + }); } catch (error) { return Result.withFailure(Failure.fromError(error)); } } /** - * Updates block number. - * (Only if given value is higher than the one currently stored in the database) + * Updates the block number in the current state. * - * @param {bigint} value + * @param {bigint} value - The new block number. + * @returns {Promise>} - The result of the operation. */ - public async newState(blockNumber: bigint): Promise { - try { - await this.source.updateBlockNumber(blockNumber); - return Result.withoutContent(); - } catch (error) { - return Result.withFailure(Failure.fromError(error)); + public async updateBlockNumber(value: bigint): Promise> { + this.updateBlockNumberQueryBuilder.with({ blockNumber: value }); + const { content, failure } = await this.update(this.updateBlockNumberQueryBuilder); + + if (failure) { + return Result.withFailure(failure); } + + return Result.withContent(content.modifiedCount + content.upsertedCount > 0); } /** - * Updates block number. - * (Only if given value is higher than the one currently stored in the database) + * Fetches the current block number from the current state. * - * @param {bigint} value + * @returns {Promise>} - The result of the operation. */ - public async updateBlockNumber(value: bigint): Promise> { - try { - const isUpdated = await this.source.updateBlockNumber(value); + public async getBlockNumber(): Promise> { + const { content: states, failure } = await this.find(); - return Result.withContent(isUpdated); - } catch (error) { - return Result.withFailure(Failure.fromError(error)); + if (failure) { + return Result.withFailure(failure); } - } - /** - * Returns current block number or -1 - * @returns - */ - public async getBlockNumber(): Promise> { - try { - const currentBlockNumber = await this.source.getBlockNumber(); - return Result.withContent(currentBlockNumber); - } catch (error) { - return Result.withFailure(Failure.fromError(error)); + if (states.length > 0) { + const state = states[0]; + + return Result.withContent(state.blockNumber); } + + return Result.withContent(-1n); } } diff --git a/src/common/block-state/block-state.types.ts b/src/common/block-state/block-state.types.ts index ffbdb0f..0a88569 100644 --- a/src/common/block-state/block-state.types.ts +++ b/src/common/block-state/block-state.types.ts @@ -1,4 +1,4 @@ -import { MongoDB } from "@alien-worlds/api-core" +import { MongoDB } from "@alien-worlds/storage-mongodb"; export type BlockStateDocument = { _id: MongoDB.ObjectId; diff --git a/src/common/block-state/query-builders/update-block-number.mongo.query-builder.ts b/src/common/block-state/query-builders/update-block-number.mongo.query-builder.ts new file mode 100644 index 0000000..6d2c0a8 --- /dev/null +++ b/src/common/block-state/query-builders/update-block-number.mongo.query-builder.ts @@ -0,0 +1,17 @@ +import { Query, QueryBuilder } from '@alien-worlds/api-core'; +import { MongoDB } from '@alien-worlds/storage-mongodb'; + +export type UpdateBlockNumberQueryArgs = { blockNumber: bigint }; + +export class UpdateBlockNumberMongoQueryBuilder extends QueryBuilder { + public build(): Query { + const { blockNumber } = this.args as UpdateBlockNumberQueryArgs; + return { + filter: {}, + update: { + $max: { block_number: MongoDB.Long.fromBigInt(blockNumber) }, + $set: { last_modified_timestamp: new Date() }, + }, + }; + } +} diff --git a/src/common/blockchain/abi/__tests__/abi.unit.test.ts b/src/common/blockchain/abi/__tests__/abi.unit.test.ts deleted file mode 100644 index d92eb00..0000000 --- a/src/common/blockchain/abi/__tests__/abi.unit.test.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { Abi } from '../abi'; -import { AbiExtension } from '../abi-extension'; -import { AbiStruct, StructField } from '../abi-struct'; -import { AbiTable } from '../abi-table'; -import { AbiType } from '../abi-type'; -import { AbiVariant } from '../abi-variant'; -import { AbiErrorMessage } from '../abi-error-message'; -import { RicardianClause } from '../ricardian-clause'; - -jest.mock('eosjs/dist/eosjs-serialize'); - -const typeDto = { - new_type_name: 'some_name', - type: 'some_type', -}; - -const variantDto = { - name: 'SOME_NAME', - types: ['TYPE_1', 'TYPE_2'], -}; - -const customTypeDto = { - new_type_name: 'SOME_TYPE_NAME', - type: 'SOME_TYPE', -}; - -const structDto = { - name: 'foo.name', - base: 'FOO', - fields: [{ name: 'FIELD_NAME', type: 'FIELD_TYPE' }], -}; - -const structFieldDto = { - name: 'FIELD_NAME', - type: 'FIELD_TYPE', -}; - -const ricardianClauseDto = { - id: 'SOME_ID', - body: 'SOME_BODY', -}; - -const abiExtensionDto = { - tag: 12345, - value: 'SOME_BODY', -}; - -const errorMessageDto = { - error_code: 200, - error_msg: 'SOME_MESSAGE', -}; - -const tableDto = { - name: 'accounts', - type: 'account', - index_type: 'i64', - key_names: ['KEY_NAME_1'], - key_types: ['KEY_TYPE_1'], -}; - -const actionDto = { - name: 'create', - type: 'create', - ricardian_contract: 'contract', -}; - -const abiDto = { - version: 'version_1', - types: [typeDto], - structs: [structDto], - tables: [tableDto], - actions: [actionDto], - ricardian_clauses: [ricardianClauseDto], - abi_extensions: [abiExtensionDto], - error_messages: [errorMessageDto], - variants: [variantDto], -}; - -describe('CustomType Unit tests', () => { - it('"fromDto" should create entity', () => { - const entity = AbiType.fromDto(customTypeDto); - expect(entity.toDto()).toEqual(customTypeDto); - }); - - it('"toDto" should create dto from entity', () => { - const entity = AbiType.fromDto(customTypeDto); - expect(entity.toDto()).toEqual(customTypeDto); - }); -}); - -describe('Struct Unit tests', () => { - it('"fromDto" should create entity', () => { - const entity = AbiStruct.fromDto(structDto); - expect(entity.toDto()).toEqual(structDto); - }); - - it('"toDto" should create dto from entity', () => { - const entity = AbiStruct.fromDto(structDto); - expect(entity.toDto()).toEqual(structDto); - }); -}); - -describe('StructField Unit tests', () => { - it('"fromDto" should create entity', () => { - const entity = StructField.fromDto(structFieldDto); - expect(entity.toDto()).toEqual(structFieldDto); - }); - - it('"toDto" should create dto from entity', () => { - const entity = StructField.fromDto(structFieldDto); - expect(entity.toDto()).toEqual(structFieldDto); - }); -}); - -describe('RicardianClause Unit tests', () => { - it('"fromDto" should create entity', () => { - const entity = RicardianClause.fromDto(ricardianClauseDto); - expect(entity.toDto()).toEqual(ricardianClauseDto); - }); - - it('"toDto" should create dto from entity', () => { - const entity = RicardianClause.fromDto(ricardianClauseDto); - expect(entity.toDto()).toEqual(ricardianClauseDto); - }); -}); - -describe('AbiExtension Unit tests', () => { - it('"fromDto" should create entity', () => { - const entity = AbiExtension.fromDto(abiExtensionDto); - expect(entity.toDto()).toEqual(abiExtensionDto); - }); - - it('"toDto" should create dto from entity', () => { - const entity = AbiExtension.fromDto(abiExtensionDto); - expect(entity.toDto()).toEqual(abiExtensionDto); - }); -}); - -describe('AbiErrorMessage Unit tests', () => { - it('"fromDto" should create entity', () => { - const entity = AbiErrorMessage.fromDto(errorMessageDto); - expect(entity.toDto()).toEqual(errorMessageDto); - }); - - it('"toDto" should create dto from entity', () => { - const entity = AbiErrorMessage.fromDto(errorMessageDto); - expect(entity.toDto()).toEqual(errorMessageDto); - }); -}); - -describe('Variant Unit tests', () => { - it('"fromDto" should create entity', () => { - const entity = AbiVariant.fromDto(variantDto); - expect(entity.toDto()).toEqual(variantDto); - }); - - it('"toDto" should create dto from entity', () => { - const entity = AbiVariant.fromDto(variantDto); - expect(entity.toDto()).toEqual(variantDto); - }); -}); - -describe('Table Unit tests', () => { - it('"fromDto" should create entity', () => { - const entity = AbiTable.fromDto(tableDto); - expect(entity.toDto()).toEqual(tableDto); - }); - - it('"toDto" should create dto from entity', () => { - const entity = AbiTable.fromDto(tableDto); - expect(entity.toDto()).toEqual(tableDto); - }); -}); - -describe('Abi Unit tests', () => { - it('"fromDto" should create entity', () => { - const entity = Abi.fromJson(abiDto); - expect(entity.toJson()).toEqual(abiDto); - }); - - it('"toDto" should create dto from entity', () => { - const entity = Abi.fromJson(abiDto); - expect(entity.toJson()).toEqual(abiDto); - }); -}); diff --git a/src/common/blockchain/abi/abi-action.ts b/src/common/blockchain/abi/abi-action.ts deleted file mode 100644 index 80be518..0000000 --- a/src/common/blockchain/abi/abi-action.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { AbiActionJson } from './abi.dtos'; - -/** - * @class - */ -export class AbiAction { - /** - * - * @param {string} name - The name of the action as defined in the contract - * @param {string} type - The name of the implicit struct as described in the ABI - * @param {string=} ricardianContract - An optional ricardian clause to associate to this action describing its intended functionality. - */ - private constructor( - public readonly name: string, - public readonly type: string, - public readonly ricardianContract?: string - ) {} - - /** - * @returns {AbiActionJson} - */ - public toDto(): AbiActionJson { - const { name, type, ricardianContract } = this; - return { - name, - type, - ricardian_contract: ricardianContract, - }; - } - - /** - * @static - * @param {AbiActionJson} dto - * @returns {AbiAction} - */ - public static fromDto(dto: AbiActionJson): AbiAction { - const { name, type, ricardian_contract } = dto; - return new AbiAction(name, type, ricardian_contract); - } -} diff --git a/src/common/blockchain/abi/abi-error-message.ts b/src/common/blockchain/abi/abi-error-message.ts deleted file mode 100644 index 1408241..0000000 --- a/src/common/blockchain/abi/abi-error-message.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { AbiErrorMessageJson } from './abi.dtos'; - -/** - * @class - */ -export class AbiErrorMessage { - /** - * - * @param {number} errorCode - * @param {string} message - */ - private constructor( - public readonly errorCode: number, - public readonly message: string - ) {} - - /** - * @returns {AbiErrorMessageJson} - */ - public toDto(): AbiErrorMessageJson { - const { errorCode, message } = this; - return { error_code: errorCode, error_msg: message }; - } - - /** - * @static - * @param {AbiErrorMessageJson} dto - * @returns {AbiErrorMessage} - */ - public static fromDto(dto: AbiErrorMessageJson): AbiErrorMessage { - const { error_code, error_msg } = dto; - return new AbiErrorMessage(error_code, error_msg); - } -} diff --git a/src/common/blockchain/abi/abi-extension.ts b/src/common/blockchain/abi/abi-extension.ts deleted file mode 100644 index eccb994..0000000 --- a/src/common/blockchain/abi/abi-extension.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { AbiExtensionJson } from './abi.dtos'; - -/** - * @class - */ -export class AbiExtension { - /** - * - * @param {number} tag - * @param {string} value - */ - private constructor(public readonly tag: number, public readonly value: string) {} - - /** - * @returns {AbiExtensionJson} - */ - public toDto(): AbiExtensionJson { - const { tag, value } = this; - return { tag, value }; - } - - /** - * @static - * @param {AbiExtensionJson} dto - * @returns {AbiExtension} - */ - public static fromDto(dto: AbiExtensionJson): AbiExtension { - const { tag, value } = dto; - return new AbiExtension(tag, value); - } -} diff --git a/src/common/blockchain/abi/abi-struct.ts b/src/common/blockchain/abi/abi-struct.ts deleted file mode 100644 index 501b226..0000000 --- a/src/common/blockchain/abi/abi-struct.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* eslint-disable @typescript-eslint/unbound-method */ -import { AbiStructJson, FieldJson } from './abi.dtos'; - -/** - * @class - */ -export class StructField { - /** - * - * @param {string} name - The field's name - * @param {string} type - The field's type - */ - private constructor(public readonly name: string, public readonly type: string) {} - - /** - * @returns {FieldJson} - */ - public toDto(): FieldJson { - const { name, type } = this; - return { name, type }; - } - - /** - * @static - * @param {FieldJson} dto - * @returns {StructField} - */ - public static fromDto(dto: FieldJson): StructField { - const { name, type } = dto; - return new StructField(name, type); - } -} - -/** - * @class - */ -export class AbiStruct { - /** - * - * @param {string} name - * @param {string} base - Inheritance, parent struct - * @param {StructField[]} fields - Array of field objects describing the struct's fields - */ - private constructor( - public readonly name: string, - public readonly base: string, - public readonly fields: StructField[] - ) {} - - /** - * @returns {AbiStructJson} - */ - public toDto(): AbiStructJson { - return { - name: this.name, - base: this.base, - fields: this.fields.map(field => field.toDto()), - }; - } - - /** - * @static - * @param {AbiStructJson} dto - * @returns {AbiStruct} - */ - public static fromDto(dto: AbiStructJson): AbiStruct { - const { name, base, fields } = dto; - return new AbiStruct(name, base, fields.map(StructField.fromDto)); - } -} diff --git a/src/common/blockchain/abi/abi-table.ts b/src/common/blockchain/abi/abi-table.ts deleted file mode 100644 index 7392898..0000000 --- a/src/common/blockchain/abi/abi-table.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { AbiTableJson } from './abi.dtos'; - -/** - * @class - */ -export class AbiTable { - /** - * - * @param {string} name - The name of the table, determined during instantiation. - * @param {string} type - The table's corresponding struct - * @param {string} indexType - The type of primary index of this table - * @param {string[]} keyNames - An array of key names, length must equal length of key_types member - * @param {string[]} keyTypes - An array of key types that correspond to key names array member, length of array must equal length of key names array. - */ - private constructor( - public readonly name: string, - public readonly type: string, - public readonly indexType: string, - public readonly keyNames: string[], - public readonly keyTypes: string[] - ) {} - - /** - * @returns {AbiTableJson} - */ - public toDto(): AbiTableJson { - const { name, type, indexType, keyNames, keyTypes } = this; - return { - name, - type, - index_type: indexType, - key_names: keyNames, - key_types: keyTypes, - }; - } - - /** - * @static - * @param {AbiTableJson} dto - * @returns {AbiTable} - */ - public static fromDto(dto: AbiTableJson): AbiTable { - const { name, type, index_type, key_names, key_types } = dto; - return new AbiTable(name, type, index_type, key_names, key_types); - } -} diff --git a/src/common/blockchain/abi/abi-type.ts b/src/common/blockchain/abi/abi-type.ts deleted file mode 100644 index 70352ed..0000000 --- a/src/common/blockchain/abi/abi-type.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { AbiTypeJson } from './abi.dtos'; - -/** - * Type entity - * @class - */ -export class AbiType { - /** - * - * @param {string} newTypeName - * @param {string} type - */ - private constructor( - public readonly newTypeName: string, - public readonly type: string - ) {} - - /** - * Parse Type entity to DTO - * @returns {AbiTypeJson} - */ - public toDto(): AbiTypeJson { - return { - new_type_name: this.newTypeName, - type: this.type, - }; - } - - /** - * Create ABI entity based on provided DTO - * - * @static - * @param {AbiTypeJson} dto - * @returns {AbiType} - */ - public static fromDto(dto: AbiTypeJson): AbiType { - const { new_type_name, type } = dto; - return new AbiType(new_type_name, type); - } -} diff --git a/src/common/blockchain/abi/abi-variant.ts b/src/common/blockchain/abi/abi-variant.ts deleted file mode 100644 index b042e06..0000000 --- a/src/common/blockchain/abi/abi-variant.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { AbiVariantJson } from './abi.dtos'; - -/** - * @class - */ -export class AbiVariant { - /** - * - * @param {string} name - * @param {string[]} types - */ - private constructor(public readonly name: string, public readonly types: string[]) {} - - /** - * @returns {AbiVariantJson} - */ - public toDto(): AbiVariantJson { - const { name, types } = this; - return { name, types }; - } - - /** - * @static - * @param {AbiVariantJson} dto - * @returns {AbiVariant} - */ - public static fromDto(dto: AbiVariantJson): AbiVariant { - const { name, types } = dto; - return new AbiVariant(name, types); - } -} diff --git a/src/common/blockchain/abi/abi.dtos.ts b/src/common/blockchain/abi/abi.dtos.ts deleted file mode 100644 index e313ddb..0000000 --- a/src/common/blockchain/abi/abi.dtos.ts +++ /dev/null @@ -1,136 +0,0 @@ -export type AbiActionJson = { - name: string; // The name of the action as defined in the contract - type: string; // The name of the implicit struct as described in the ABI - ricardian_contract: string; // An optional ricardian clause to associate to this action describing its intended functionality. -}; - -export type AbiJson = { - version: string; - types: AbiTypeJson[]; - structs: AbiStructJson[]; - tables: AbiTableJson[]; - actions: AbiActionJson[]; - ricardian_clauses: RicardianClauseJson[]; - abi_extensions: AbiExtensionJson[]; - error_messages: AbiErrorMessageJson[]; - variants?: AbiVariantJson[]; -}; - -export type AbiErrorMessageJson = { - error_code: number; - error_msg: string; -}; - -export type AbiVariantJson = { - name: string; - types: string[]; -}; - -export type RicardianClauseJson = { - id: string; - body: string; -}; - -export type AbiExtensionJson = { - tag: number; - value: string; -}; - -export type AbiTypeJson = { - new_type_name: string; - type: string; -}; - -export type AbiStructFieldJson = { - name: string; - type: string; -}; - -export type AbiStructJson = { - name: string; - base: string; - fields: AbiStructFieldJson[]; -}; - -export type CreateStructJson = { - name: 'create'; - base: ''; - fields: [IssuerFieldJson, MaximumSupplyFieldJson]; -}; - -export type IssueStructJson = { - name: 'issue'; - base: ''; - fields: [ToFieldJson, QuantityFieldJson, MemoFieldJson]; -}; - -export type RetireStructJson = { - name: 'retire'; - base: ''; - fields: [QuantityFieldJson, MemoFieldJson]; -}; - -export type TransfereStructJson = { - name: 'transfer'; - base: ''; - fields: [FromFieldJson, ToFieldJson, QuantityFieldJson, MemoFieldJson]; -}; - -export type CloseStructJson = { - name: 'close'; - base: ''; - fields: [SymbolFieldJson, OwnerFieldJson]; -}; - -export type FieldJson = { - name: string; - type: string; -}; - -export type OwnerFieldJson = { - name: 'owner'; - type: 'name'; -}; - -export type SymbolFieldJson = { - name: 'symbol'; - type: 'symbol'; -}; - -export type MemoFieldJson = { - name: 'memo'; - type: 'string'; -}; - -export type QuantityFieldJson = { - name: 'quantity'; - type: 'asset'; -}; - -export type ToFieldJson = { - name: 'to'; - type: 'name'; -}; - -export type FromFieldJson = { - name: 'from'; - type: 'name'; -}; - -export type IssuerFieldJson = { - name: 'issuer'; - type: 'name'; -}; - -export type MaximumSupplyFieldJson = { - name: 'maximum_supply'; - type: 'asset'; -}; - -export type AbiTableJson = { - name: string; // 'accounts' | 'stats' - type: string; // 'account' | 'currency_stats' ... Corresponds to previously defined struct - index_type: string; // 'i64' - key_names: string[]; - key_types: string[]; -}; diff --git a/src/common/blockchain/abi/abi.ts b/src/common/blockchain/abi/abi.ts deleted file mode 100644 index a2de49e..0000000 --- a/src/common/blockchain/abi/abi.ts +++ /dev/null @@ -1,138 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -import { getTypesFromAbi } from 'eosjs/dist/eosjs-serialize'; - -import { AbiJson } from './abi.dtos'; -import { Serialize } from 'eosjs'; -import { AbiType } from './abi-type'; -import { AbiStruct } from './abi-struct'; -import { AbiTable } from './abi-table'; -import { AbiAction } from './abi-action'; -import { RicardianClause } from './ricardian-clause'; -import { AbiExtension } from './abi-extension'; -import { AbiErrorMessage } from './abi-error-message'; -import { AbiVariant } from './abi-variant'; -import { deserialize, serialize } from 'v8'; - -/** - * ABI entity - * @class - */ -export class Abi { - private typesMap: Map; - /** - * - * @param {string} version - * @param {AbiType[]} types - * @param {AbiStruct[]} structs - * @param {AbiAction[]} actions - * @param {AbiTable[]} tables - * @param {RicardianClause[]} ricardianClauses - * @param {AbiExtension[]} abiExtensions - * @param {AbiErrorMessage[]} errorMessages - * @param {string} comment - * @param {AbiVariant[]} variants - */ - private constructor( - public readonly version: string, - public readonly types: AbiType[], - public readonly structs: AbiStruct[], - public readonly tables: AbiTable[], - public readonly actions: AbiAction[], - public readonly ricardianClauses: RicardianClause[], - public readonly abiExtensions: AbiExtension[], - public readonly errorMessages: AbiErrorMessage[], - public readonly variants?: AbiVariant[] - ) { - this.typesMap = getTypesFromAbi(Serialize.createInitialTypes(), this.toJson()); - } - - /** - * Parse ABI entity to DTO - * @returns {AbiJson} - */ - public toJson(): AbiJson { - const { - version, - types, - structs, - actions, - tables, - ricardianClauses, - abiExtensions, - errorMessages: AbierrorMessages, - variants, - } = this; - - const dto: AbiJson = { - version, - types: types.map(item => item.toDto()), - structs: structs.map(item => item.toDto()), - tables: tables.map(item => item.toDto()), - actions: actions ? actions.map(item => item.toDto()) : [], - ricardian_clauses: ricardianClauses - ? ricardianClauses.map(item => item.toDto()) - : [], - abi_extensions: abiExtensions ? abiExtensions.map(item => item.toDto()) : [], - error_messages: AbierrorMessages ? AbierrorMessages.map(item => item.toDto()) : [], - variants: variants ? variants.map(item => item.toDto()) : [], - }; - - return dto; - } - - public toBuffer(): Buffer { - return serialize(this.toJson()); - } - - public toHex(): string { - return serialize(this.toJson()).toString('hex'); - } - - public getTypesMap(): Map { - return this.typesMap; - } - - /** - * Create ABI entity based on provided DTO - * - * @static - * @param {AbiJson} dto - * @returns {Abi} - */ - public static fromJson(dto: AbiJson): Abi { - const { version, types, structs, tables } = dto; - const actions = dto.actions ? dto.actions.map(dto => AbiAction.fromDto(dto)) : []; - const ricardian_clauses = dto.ricardian_clauses - ? dto.ricardian_clauses.map(dto => RicardianClause.fromDto(dto)) - : []; - const abi_extensions = dto.abi_extensions - ? dto.abi_extensions.map(dto => AbiExtension.fromDto(dto)) - : []; - const error_messages = dto.error_messages - ? dto.error_messages.map(dto => AbiErrorMessage.fromDto(dto)) - : []; - const variants = dto.variants ? dto.variants.map(dto => AbiVariant.fromDto(dto)) : []; - - return new Abi( - version, - types.map(dto => AbiType.fromDto(dto)), - structs.map(dto => AbiStruct.fromDto(dto)), - tables.map(dto => AbiTable.fromDto(dto)), - actions, - ricardian_clauses, - abi_extensions, - error_messages, - variants - ); - } - - public static fromBuffer(buffer: Buffer): Abi { - const json = deserialize(buffer); - return Abi.fromJson(json); - } - - public static fromHex(value: string): Abi { - const buf = Buffer.from(value, 'hex'); - return Abi.fromBuffer(buf); - } -} diff --git a/src/common/blockchain/abi/index.ts b/src/common/blockchain/abi/index.ts deleted file mode 100644 index 545c409..0000000 --- a/src/common/blockchain/abi/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -export { RicardianClause } from "./ricardian-clause"; -export { Abi } from "./abi"; -export { AbiAction } from "./abi-action"; -export { AbiErrorMessage } from "./abi-error-message"; -export { AbiExtension } from "./abi-extension"; -export { AbiStruct } from "./abi-struct"; -export { AbiTable } from "./abi-table"; -export { AbiType } from "./abi-type"; -export { AbiVariant } from "./abi-variant"; -export * from "./abi.dtos"; diff --git a/src/common/blockchain/abi/ricardian-clause.ts b/src/common/blockchain/abi/ricardian-clause.ts deleted file mode 100644 index e248fcb..0000000 --- a/src/common/blockchain/abi/ricardian-clause.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { RicardianClauseJson } from './abi.dtos'; - -/** - * @class - */ -export class RicardianClause { - /** - * - * @param {string} id - * @param {string} body - */ - private constructor(public readonly id: string, public readonly body: string) {} - - /** - * @returns {RicardianClauseJson} - */ - public toDto(): RicardianClauseJson { - const { id, body } = this; - return { id, body }; - } - - /** - * @static - * @param {RicardianClauseJson} dto - * @returns {RicardianClause} - */ - public static fromDto(dto: RicardianClauseJson): RicardianClause { - const { id, body } = dto; - return new RicardianClause(id, body); - } -} diff --git a/src/common/blockchain/block-reader/block-reader.config.ts b/src/common/blockchain/block-reader/block-reader.config.ts deleted file mode 100644 index 61745be..0000000 --- a/src/common/blockchain/block-reader/block-reader.config.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { MongoConfig } from '@alien-worlds/api-core'; - -export type BlockReaderConfig = { - mongo: MongoConfig; - endpoints: string[]; - reconnectInterval?: number; - shouldFetchDeltas?: boolean; - shouldFetchTraces?: boolean; - shouldFetchBlock?: boolean; -}; diff --git a/src/common/blockchain/block-reader/block-reader.enums.ts b/src/common/blockchain/block-reader/block-reader.enums.ts deleted file mode 100644 index 41676cf..0000000 --- a/src/common/blockchain/block-reader/block-reader.enums.ts +++ /dev/null @@ -1,6 +0,0 @@ -export enum BlockReaderConnectionState { - Connecting = 'connecting', - Connected = 'connected', - Idle = 'idle', - Disconnecting = 'disconnecting', -} diff --git a/src/common/blockchain/block-reader/block-reader.errors.ts b/src/common/blockchain/block-reader/block-reader.errors.ts deleted file mode 100644 index 84d278a..0000000 --- a/src/common/blockchain/block-reader/block-reader.errors.ts +++ /dev/null @@ -1,37 +0,0 @@ -export class AbiNotFoundError extends Error { - constructor() { - super(`ABI data not found`); - } -} - -export class MissingHandlersError extends Error { - constructor() { - super('Set "onReceivedBlock" handler before calling readOneBlock/readBlocks'); - } -} - -export class ServiceNotConnectedError extends Error { - constructor() { - super(`Client is not connected, requestBlocks cannot be called`); - } -} - -export class UnhandledBlockRequestError extends Error { - constructor(start: bigint, end: bigint) { - super( - `Error sending the block_range request ${start.toString()}-${end.toString()}. The current request was not completed or canceled.` - ); - } -} - -export class UnhandledMessageTypeError extends Error { - constructor(public readonly type: string) { - super(`Unhandled message type: ${type}`); - } -} - -export class UnhandledMessageError extends Error { - constructor(public readonly message, public readonly error) { - super('Received a message while no block range is being processed'); - } -} diff --git a/src/common/blockchain/block-reader/block-reader.message.ts b/src/common/blockchain/block-reader/block-reader.message.ts deleted file mode 100644 index 4c63146..0000000 --- a/src/common/blockchain/block-reader/block-reader.message.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { Abi } from '../abi'; -import { GetBlocksResultMessageContent } from './block-reader.types'; -import { deserializeMessage } from './block-reader.utils'; -import { Block } from './block/block'; -import { BlockJson } from './block/block.types'; - -export class BlockReaderMessage { - public static readonly version = 'v0'; - - private static isGetBlocksResultPongMessage(data: GetBlocksResultMessageContent): boolean { - return ( - typeof data.head === 'object' && - typeof data.last_irreversible === 'object' && - !data.prev_block && - !data.this_block && - !data.block && - !data.traces && - !data.deltas - ); - } - - public static create( - dto: Uint8Array, - abi: Abi - ) { - const result = deserializeMessage('result', dto, abi.getTypesMap()); - const [resultType, resultJson]: [string, MessageContentType] = result || []; - - if (resultType) { - if (resultType === `get_blocks_result_${this.version}`) { - if ( - BlockReaderMessage.isGetBlocksResultPongMessage( - resultJson - ) - ) { - return new BlockReaderMessage(resultType, null, true); - } - - (resultJson).abi_version = abi.version; - return new BlockReaderMessage( - resultType, - Block.fromJson(resultJson) - ); - } - } - - return null; - } - - private constructor( - public readonly type: string, - public readonly content: MessageContentType, - public readonly isPongMessage = false - ) {} -} diff --git a/src/common/blockchain/block-reader/block-reader.requests.ts b/src/common/blockchain/block-reader/block-reader.requests.ts deleted file mode 100644 index 38c334c..0000000 --- a/src/common/blockchain/block-reader/block-reader.requests.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Serialize } from 'eosjs'; -import { BlockReaderOptions } from './block-reader.types'; -import { serializeMessage } from './block-reader.utils'; - -export class GetBlocksRequest { - public readonly version = 'v0'; - - public static create( - startBlock: bigint, - endBlock: bigint, - options: BlockReaderOptions, - types: Map - ) { - const { shouldFetchDeltas, shouldFetchTraces } = options; - - return new GetBlocksRequest( - startBlock, - endBlock, - shouldFetchTraces, - shouldFetchDeltas, - types - ); - } - - private constructor( - public readonly startBlock: bigint, - public readonly endBlock: bigint, - public readonly shouldFetchTraces: boolean, - public readonly shouldFetchDeltas: boolean, - public readonly types: Map - ) {} - - public toUint8Array(): Uint8Array { - return serializeMessage( - 'request', - [ - `get_blocks_request_${this.version}`, - { - irreversible_only: false, - start_block_num: Number(this.startBlock.toString()), - end_block_num: Number(this.endBlock.toString()), - max_messages_in_flight: 1, - have_positions: [], - fetch_block: true, - fetch_traces: this.shouldFetchTraces, - fetch_deltas: this.shouldFetchDeltas, - }, - ], - this.types - ); - } -} - -export class GetBlocksAckRequest { - public readonly version = 'v0'; - - constructor( - public readonly messagesCount: number, - public readonly types: Map - ) {} - - public toUint8Array() { - return serializeMessage( - 'request', - [`get_blocks_ack_request_${this.version}`, { num_messages: this.messagesCount }], - this.types - ); - } -} diff --git a/src/common/blockchain/block-reader/block-reader.source.ts b/src/common/blockchain/block-reader/block-reader.source.ts deleted file mode 100644 index fb0fb37..0000000 --- a/src/common/blockchain/block-reader/block-reader.source.ts +++ /dev/null @@ -1,135 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -/* eslint-disable @typescript-eslint/no-unsafe-call */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -import { log } from '@alien-worlds/api-core'; -import WebSocket from 'ws'; -import { BlockReaderConfig } from './block-reader.config'; -import { BlockReaderConnectionState } from './block-reader.enums'; -import { ConnectionChangeHandler } from './block-reader.types'; - -export class BlockReaderSource { - private messageHandler: (...args: unknown[]) => void; - private errorHandler: (...args: unknown[]) => void; - private client: WebSocket; - private endpoint: string; - private connectionState = BlockReaderConnectionState.Idle; - private connectionChangeHandlers: Map< - BlockReaderConnectionState, - ConnectionChangeHandler - > = new Map(); - private socketIndex = -1; - private reconnectDelay; - - constructor(private readonly config: BlockReaderConfig) {} - - private async updateConnectionState(state: BlockReaderConnectionState, data?: string) { - const previousState = state; - this.connectionState = state; - this.reconnectDelay = this.config.reconnectInterval || 1000; - const handler = this.connectionChangeHandlers.get(state); - if (handler) { - return handler({ previousState, state, data }); - } - } - - private getNextEndpoint() { - let nextIndex = ++this.socketIndex; - - if (nextIndex >= this.config.endpoints.length) { - nextIndex = 0; - } - this.socketIndex = nextIndex; - - return this.config.endpoints[this.socketIndex]; - } - - private waitUntilConnectionIsOpen() { - log(`BlockReader plugin connecting to: ${this.endpoint}`); - return new Promise(resolve => { - this.client.once('open', () => { - log(`BlockReader plugin connection open.`); - resolve(true); - }); - }); - } - - private async onConnectionClosed(code: number) { - this.client = null; - log(`BlockReader plugin connection closed with code #${code}.`); - await this.updateConnectionState(BlockReaderConnectionState.Idle); - } - - private receiveAbi() { - return new Promise(resolve => this.client.once('message', resolve)); - } - - public onError(handler: (error: Error) => void) { - this.errorHandler = handler; - } - - public onMessage(handler: (dto: Uint8Array) => void) { - this.messageHandler = handler; - } - - public addConnectionStateHandler( - state: BlockReaderConnectionState, - handler: ConnectionChangeHandler - ) { - if (this.connectionChangeHandlers.has(state)) { - console.warn(`Overriding the handler assigned to the "${state}" state`); - } else { - this.connectionChangeHandlers.set(state, handler); - } - } - - public get isConnected() { - return this.connectionState === BlockReaderConnectionState.Connected; - } - - public async connect() { - if (this.connectionState === BlockReaderConnectionState.Idle) { - log(`BlockReader plugin connecting...`); - try { - await this.updateConnectionState(BlockReaderConnectionState.Connecting); - this.endpoint = this.getNextEndpoint(); - this.client = new WebSocket(this.endpoint, { - perMessageDeflate: false, - }); - this.client.on('close', code => this.onConnectionClosed(code)); - this.client.on('error', error => this.errorHandler(error)); - await this.waitUntilConnectionIsOpen(); - // receive ABI - first message from WS is always ABI - const abi = await this.receiveAbi(); - // set message handler - this.client.on('message', message => this.messageHandler(message)); - - await this.updateConnectionState(BlockReaderConnectionState.Connected, abi); - } catch (error) { - setTimeout( - () => this.updateConnectionState(BlockReaderConnectionState.Idle), - this.reconnectDelay - ); - this.connectionState = BlockReaderConnectionState.Idle; - this.errorHandler(error); - } - } - } - - public async disconnect() { - if (this.connectionState === BlockReaderConnectionState.Connected) { - log(`BlockReader plugin disconnecting...`); - try { - await this.updateConnectionState(BlockReaderConnectionState.Disconnecting); - this.client.removeAllListeners(); - this.client.close(); - } catch (error) { - this.errorHandler(error); - } - } - } - - public send(message: Uint8Array) { - this.client.send(message); - } -} diff --git a/src/common/blockchain/block-reader/block-reader.ts b/src/common/blockchain/block-reader/block-reader.ts deleted file mode 100644 index 412f17e..0000000 --- a/src/common/blockchain/block-reader/block-reader.ts +++ /dev/null @@ -1,243 +0,0 @@ -/* eslint-disable @typescript-eslint/restrict-template-expressions */ -import { MongoSource, log } from '@alien-worlds/api-core'; -import { BlockReaderConfig } from './block-reader.config'; -import { BlockReaderConnectionState } from './block-reader.enums'; -import { - AbiNotFoundError, - MissingHandlersError, - UnhandledMessageError, - UnhandledMessageTypeError, -} from './block-reader.errors'; -import { BlockReaderSource } from './block-reader.source'; -import { BlockReaderOptions, ConnectionChangeHandlerOptions } from './block-reader.types'; -import { BlockReaderMessage } from './block-reader.message'; -import { GetBlocksAckRequest, GetBlocksRequest } from './block-reader.requests'; -import { Block } from './block/block'; -import { ShipAbiSource } from '../../ship/ship-abi.source'; -import { Abi, AbiJson } from '../abi'; - -export class BlockReader { - public static async create(config: BlockReaderConfig): Promise { - const mongoSource = await MongoSource.create(config.mongo); - const shipAbiSource = new ShipAbiSource(mongoSource); - const source = new BlockReaderSource(config); - source.onError(error => log(error)); - - return new BlockReader(source, shipAbiSource); - } - - private errorHandler: (error: Error) => void; - private warningHandler: (...args: unknown[]) => void; - private receivedBlockHandler: (content: Block) => Promise | void; - private blockRangeCompleteHandler: ( - startBlock: bigint, - endBlock: bigint - ) => Promise; - private _blockRangeRequest: GetBlocksRequest; - private _abi: Abi; - private _paused = false; - private isLastBlock = false; - - constructor(private source: BlockReaderSource, private shipAbi: ShipAbiSource) { - this.source.onMessage(message => this.onMessage(message)); - this.source.onError(error => { - this.handleError(error); - }); - this.source.addConnectionStateHandler(BlockReaderConnectionState.Connected, options => - this.onConnected(options) - ); - this.source.addConnectionStateHandler(BlockReaderConnectionState.Idle, options => - this.onDisconnected(options) - ); - } - - private async onConnected({ data }: ConnectionChangeHandlerOptions) { - log(`BlockReader plugin connected`); - - const abi = Abi.fromJson(JSON.parse(data) as AbiJson); - if (abi) { - const result = await this.shipAbi.getAbi(abi.version); - - if (result.isFailure) { - await this.shipAbi.updateAbi(abi); - } - this._abi = abi; - } - } - - private onDisconnected({ previousState }: ConnectionChangeHandlerOptions) { - log(`BlockReader plugin disconnected`); - if (previousState === BlockReaderConnectionState.Disconnecting) { - this._abi = null; - } - this.connect(); - } - - public get abi(): Abi { - return this._abi; - } - - public onMessage(dto: Uint8Array): Promise { - const { abi } = this; - - if (!abi) { - this.handleError(new AbiNotFoundError()); - return; - } - - const message = BlockReaderMessage.create(dto, abi); - - if (message && message.isPongMessage === false) { - this.handleBlocksResultContent(message.content); - } else if (!message) { - this.handleError(new UnhandledMessageTypeError(message.type)); - } - } - - private async handleBlocksResultContent(result: Block) { - const { thisBlock } = result; - const { abi } = this; - - // skip any extra result messages - if (this.isLastBlock) { - return; - } - - if (!abi) { - this.handleError(new AbiNotFoundError()); - return; - } - - try { - if (thisBlock) { - const { - _blockRangeRequest: { startBlock, endBlock }, - } = this; - this.isLastBlock = thisBlock.blockNumber === endBlock - 1n; - - if (this.isLastBlock) { - await this.receivedBlockHandler(result); - this.blockRangeCompleteHandler(startBlock, endBlock); - } else { - this.receivedBlockHandler(result); - // State history plugs will answer every call of ack_request, even after - // processing the full range, it will send messages containing only head. - // After the block has been processed, the connection should be closed so - // there is no need to ack request. - if (this.source.isConnected && this._paused === false) { - // Acknowledge a request so that source can send next one. - this.source.send( - new GetBlocksAckRequest(1, abi.getTypesMap()).toUint8Array() - ); - } - } - } else { - this.handleWarning(`the received message does not contain this_block`); - } - } catch (error) { - this.handleError(new UnhandledMessageError(result, error)); - } - } - - private handleError(error: Error) { - if (this.errorHandler) { - return this.errorHandler(error); - } - } - - private handleWarning(...args: unknown[]) { - if (this.warningHandler) { - return this.warningHandler(...args); - } - } - - public async connect(): Promise { - if (this.source.isConnected === false) { - await this.source.connect(); - } else { - log(`Service already connected`); - } - } - - public async disconnect(): Promise { - if (this.source.isConnected) { - await this.source.disconnect(); - } else { - log(`Service not connected`); - } - } - - public pause(): void { - if (this._paused === false) { - this._paused = true; - } - } - - public resume(): void { - if (this._paused && !this.isLastBlock) { - this._paused = false; - this.source.send(new GetBlocksAckRequest(1, this.abi.getTypesMap()).toUint8Array()); - } - } - - public readBlocks( - startBlock: bigint, - endBlock: bigint, - options?: BlockReaderOptions - ): void { - this.sendRequest(startBlock, endBlock, options); - log(`BlockReader plugin: read blocks`, { startBlock, endBlock }); - } - - public readOneBlock(block: bigint, options?: BlockReaderOptions): void { - this.sendRequest(block, block + 1n, options); - log(`BlockReader plugin: read single block ${block}`); - } - - private sendRequest( - startBlock: bigint, - endBlock: bigint, - options?: BlockReaderOptions - ): void { - const requestOptions = options || { - shouldFetchDeltas: true, - shouldFetchTraces: true, - }; - - this.isLastBlock = false; - this.resume(); - - const { abi, receivedBlockHandler, source } = this; - if (!receivedBlockHandler) { - throw new MissingHandlersError(); - } - - if (!abi) { - throw new AbiNotFoundError(); - } - - this._blockRangeRequest = GetBlocksRequest.create( - startBlock, - endBlock, - requestOptions, - abi.getTypesMap() - ); - source.send(this._blockRangeRequest.toUint8Array()); - } - - public onReceivedBlock(handler: (content: Block) => Promise | void) { - this.receivedBlockHandler = handler; - } - - public onComplete(handler: (startBlock: bigint, endBlock: bigint) => Promise) { - this.blockRangeCompleteHandler = handler; - } - - public onError(handler: (error: Error) => void) { - this.errorHandler = handler; - } - - public onWarning(handler: (...args: unknown[]) => void) { - this.warningHandler = handler; - } -} diff --git a/src/common/blockchain/block-reader/block-reader.types.ts b/src/common/blockchain/block-reader/block-reader.types.ts deleted file mode 100644 index 230a818..0000000 --- a/src/common/blockchain/block-reader/block-reader.types.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { BlockReaderConnectionState } from './block-reader.enums'; - -export type ConnectionChangeHandlerOptions = { - previousState: BlockReaderConnectionState; - state: BlockReaderConnectionState; - data: string; -}; - -export type ConnectionChangeHandler = ( - options: ConnectionChangeHandlerOptions -) => void | Promise; - -export type BlockReaderOptions = { - shouldFetchDeltas?: boolean; - shouldFetchTraces?: boolean; - shouldFetchBlock?: boolean; -}; - -export type GetBlocksResultMessageContent = { - head?: { - block_num: number; - block_id: string; - }; - last_irreversible?: { - block_num: number; - block_id: string; - }; - this_block?: { - block_num: number; - block_id: string; - }; - prev_block?: { - block_num: number; - block_id: string; - }; - block?: Uint8Array; - traces?: Uint8Array; - deltas?: Uint8Array; - [key: string]: unknown; -}; diff --git a/src/common/blockchain/block-reader/block-reader.utils.ts b/src/common/blockchain/block-reader/block-reader.utils.ts deleted file mode 100644 index f990211..0000000 --- a/src/common/blockchain/block-reader/block-reader.utils.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-call */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { Serialize } from 'eosjs'; -import { TextDecoder, TextEncoder } from 'text-encoding'; - -export const serializeMessage = ( - type: string, - value: unknown, - types: Map -) => { - const buffer = new Serialize.SerialBuffer({ - textEncoder: new TextEncoder(), - textDecoder: new TextDecoder(), - }); - Serialize.getType(types, type).serialize(buffer, value); - return buffer.asUint8Array(); -}; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export const deserializeMessage = ( - type: string, - array: Uint8Array, - types: Map -): T => { - const buffer = new Serialize.SerialBuffer({ - textEncoder: new TextEncoder(), - textDecoder: new TextDecoder(), - array, - }); - const result = Serialize.getType(types, type).deserialize( - buffer, - new Serialize.SerializerState({ bytesAsUint8Array: true }) - ); - - if (buffer.readPos != array.length) throw new Error('oops: ' + type); // todo: remove check - return result; -}; diff --git a/src/common/blockchain/block-reader/block/block.ts b/src/common/blockchain/block-reader/block/block.ts deleted file mode 100644 index af6086f..0000000 --- a/src/common/blockchain/block-reader/block/block.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { MongoDB, parseToBigInt } from '@alien-worlds/api-core'; -import { - BlockDocument, - BlockJson, - BlockNumberWithIdDocument, - BlockNumberWithIdJson, -} from './block.types'; - -export class BlockNumberWithId { - public static fromJson(dto: BlockNumberWithIdJson) { - const { block_id, block_num } = dto; - return new BlockNumberWithId(parseToBigInt(block_num), block_id); - } - - public static fromDocument(dto: BlockNumberWithIdDocument) { - const { block_id, block_num } = dto; - return new BlockNumberWithId(parseToBigInt(block_num), block_id); - } - - private constructor( - public readonly blockNumber: bigint, - public readonly blockId: string - ) {} - - public toJson() { - return { - block_num: this.blockNumber.toString(), - block_id: this.blockId, - }; - } - - public toDocument() { - return { - block_num: MongoDB.Long.fromBigInt(this.blockNumber), - block_id: this.blockId, - }; - } -} - -export class Block { - public static fromJson(json: BlockJson): Block { - const { block, traces, deltas, abi_version } = json; - const head = BlockNumberWithId.fromJson(json.head); - const lastIrreversible = BlockNumberWithId.fromJson(json.last_irreversible); - const prevBlock = BlockNumberWithId.fromJson(json.prev_block); - const thisBlock = BlockNumberWithId.fromJson(json.this_block); - - return new Block( - head, - lastIrreversible, - prevBlock, - thisBlock, - block, - traces, - deltas, - abi_version - ); - } - - public static fromDocument(content: BlockDocument): Block { - const { block, traces, deltas, _id, abi_version } = content; - const head = BlockNumberWithId.fromDocument(content.head); - const lastIrreversible = BlockNumberWithId.fromDocument(content.last_irreversible); - const prevBlock = BlockNumberWithId.fromDocument(content.prev_block); - const thisBlock = BlockNumberWithId.fromDocument(content.this_block); - - return new Block( - head, - lastIrreversible, - prevBlock, - thisBlock, - block.buffer, - traces.buffer, - deltas.buffer, - abi_version, - _id.toString() - ); - } - - private constructor( - public readonly head: BlockNumberWithId, - public readonly lastIrreversible: BlockNumberWithId, - public readonly prevBlock: BlockNumberWithId, - public readonly thisBlock: BlockNumberWithId, - public readonly block: Uint8Array, - public readonly traces: Uint8Array, - public readonly deltas: Uint8Array, - public readonly abiVersion?: string, - public readonly id?: string - ) {} - - public toJson(): BlockJson { - const { head, thisBlock, prevBlock, lastIrreversible, block, traces, deltas, abiVersion } = this; - - const json: BlockJson = { - head: head.toJson(), - this_block: thisBlock.toJson(), - prev_block: prevBlock.toJson(), - last_irreversible: lastIrreversible.toJson(), - block, - traces, - deltas, - }; - - if (abiVersion) { - json.abi_version = abiVersion; - } - - return json; - } - - public toDocument(): BlockDocument { - const { - head, - thisBlock, - prevBlock, - lastIrreversible, - block, - traces, - deltas, - id, - abiVersion, - } = this; - - const document: BlockDocument = { - head: head.toDocument(), - this_block: thisBlock.toDocument(), - prev_block: prevBlock.toDocument(), - last_irreversible: lastIrreversible.toDocument(), - block: new MongoDB.Binary(block), - traces: new MongoDB.Binary(traces), - deltas: new MongoDB.Binary(deltas), - }; - - if (abiVersion) { - document.abi_version = abiVersion; - } - - if (id) { - document._id = new MongoDB.ObjectId(id); - } - - return document; - } -} diff --git a/src/common/blockchain/block-reader/block/block.types.ts b/src/common/blockchain/block-reader/block/block.types.ts deleted file mode 100644 index 356c5b0..0000000 --- a/src/common/blockchain/block-reader/block/block.types.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { MongoDB } from '@alien-worlds/api-core'; - -export type BlockNumberWithIdJson = { - block_num: string; - block_id: string; -}; - -export type BlockJson = { - head?: BlockNumberWithIdJson; - this_block?: BlockNumberWithIdJson; - last_irreversible?: BlockNumberWithIdJson; - prev_block?: BlockNumberWithIdJson; - block?: Uint8Array; - traces?: Uint8Array; - deltas?: Uint8Array; - abi_version?: string; -}; - -export type BlockDocument = { - head?: BlockNumberWithIdDocument; - this_block?: BlockNumberWithIdDocument; - last_irreversible?: BlockNumberWithIdDocument; - prev_block?: BlockNumberWithIdDocument; - block?: MongoDB.Binary; - traces?: MongoDB.Binary; - deltas?: MongoDB.Binary; - _id?: MongoDB.ObjectId; - abi_version?: string; - [key: string]: unknown; -}; - -export type BlockNumberWithIdDocument = { - block_num?: MongoDB.Long; - block_id?: string; -}; diff --git a/src/common/blockchain/block-reader/block/index.ts b/src/common/blockchain/block-reader/block/index.ts deleted file mode 100644 index 788a4b9..0000000 --- a/src/common/blockchain/block-reader/block/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './block'; -export * from './block.types'; diff --git a/src/common/blockchain/block-reader/index.ts b/src/common/blockchain/block-reader/index.ts deleted file mode 100644 index 6475d7c..0000000 --- a/src/common/blockchain/block-reader/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export * from './block-reader.config'; -export * from './block-reader.enums'; -export * from './block-reader.errors'; -export * from './block-reader.message'; -export * from './block-reader.requests'; -export * from './block-reader.source'; -export * from './block-reader'; -export * from './block-reader.types'; -export * from './block-reader.utils'; diff --git a/src/common/blockchain/blockchain.ts b/src/common/blockchain/blockchain.ts deleted file mode 100644 index dd8f661..0000000 --- a/src/common/blockchain/blockchain.ts +++ /dev/null @@ -1,45 +0,0 @@ -import fetch from 'node-fetch'; -import { parseToBigInt } from '@alien-worlds/api-core'; -import { Api, JsonRpc } from 'eosjs'; -import { GetInfoResult } from 'eosjs/dist/eosjs-rpc-interfaces'; -import { BlockchainConfig } from './blockchain.types'; - -export class Blockchain { - public static create(config: BlockchainConfig): Blockchain { - const { endpoint, chainId } = config; - const api = new Api({ - rpc: new JsonRpc(endpoint, { fetch }), - chainId, - signatureProvider: null, - textDecoder: new TextDecoder(), - textEncoder: new TextEncoder(), - }); - - return new Blockchain(endpoint, chainId, api); - } - - private constructor( - protected endpoint: string, - protected chainId: string, - protected api: Api - ) {} - - public getInfo = async (): Promise => { - return this.api.rpc.get_info(); - }; - - public async getHeadBlockNumber(): Promise { - const info = await this.api.rpc.get_info(); - const value = parseToBigInt(info.head_block_num); - return value; - } - - public async getLastIrreversibleBlockNumber(): Promise { - const info = await this.api.rpc.get_info(); - const value = parseToBigInt(info.last_irreversible_block_num); - return value; - } -} - -//log(`Head block number: ${value.toString()}`); -//log(`Last irreversible block number: ${value.toString()}`); \ No newline at end of file diff --git a/src/common/blockchain/blockchain.types.ts b/src/common/blockchain/blockchain.types.ts deleted file mode 100644 index d3479cb..0000000 --- a/src/common/blockchain/blockchain.types.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type BlockchainConfig = { - endpoint: string; - chainId: string; -}; diff --git a/src/common/blockchain/contract/action-trace/action-trace.ts b/src/common/blockchain/contract/action-trace/action-trace.ts deleted file mode 100644 index d461625..0000000 --- a/src/common/blockchain/contract/action-trace/action-trace.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { parseToBigInt } from '@alien-worlds/api-core'; -import { ActAuthJson, ActJson, ActionTraceDto, ReceiptJson } from './action-trace.dtos'; - -export class ActAuth { - public static create(dto: ActAuthJson): ActAuth { - const { actor, permission } = dto; - - return new ActAuth(actor, permission); - } - private constructor( - public readonly actor: string, - public readonly permission: string - ) {} -} - -export class Act { - public static create(dto: ActJson): Act { - const { account, name, data } = dto; - - //parse DATA - let authorization: ActAuth; - - if (dto.authorization) { - authorization = ActAuth.create(dto.authorization); - } - - return new Act(account, name, authorization, data); - } - private constructor( - public readonly account: string, - public readonly name: string, - public readonly authorization: ActAuth, - public readonly data: Uint8Array - ) {} -} - -export type AuthSequence = { - account: string; - sequence: string; -}; - -export class Receipt { - public static create(shipMessageName: string, dto: ReceiptJson): Receipt { - const { - receiver, - act_digest, - global_sequence, - recv_sequence, - auth_sequence, - code_sequence, - abi_sequence, - } = dto; - return new Receipt( - shipMessageName, - receiver, - act_digest, - parseToBigInt(global_sequence), - parseToBigInt(recv_sequence), - auth_sequence, - code_sequence, - abi_sequence - ); - } - private constructor( - public readonly shipMessageName: string, - public readonly receiver: string, - public readonly actDigest: string, - public readonly globalSequence: bigint, - public readonly recvSequence: bigint, - public readonly authSequence: AuthSequence[], - public readonly codeSequence: number, - public readonly abiSequence: number - ) {} -} - -export class ActionTrace { - public static create(shipMessageName: string, dto: ActionTraceDto): ActionTrace { - const { - action_ordinal, - creator_action_ordinal, - receiver, - act, - context_free, - elapsed, - console, - account_ram_deltas, - except, - error_code, - } = dto; - - let receipt: Receipt; - if (dto.receipt && dto.receipt.length) { - const [receiptType, receiptContent] = dto.receipt; - receipt = Receipt.create(receiptType, receiptContent); - } - - return new ActionTrace( - shipMessageName, - action_ordinal, - creator_action_ordinal, - receipt, - receiver, - Act.create(act), - context_free, - elapsed, - console, - account_ram_deltas, - except, - Number(error_code) - ); - } - - private constructor( - public readonly shipMessageName: string, - public readonly actionOrdinal: number, - public readonly creatorActionOrdinal: number, - public readonly receipt: Receipt | null, - public readonly receiver: string, - public readonly act: Act, - public readonly isContextFree: boolean, - public readonly elapsed: string, - public readonly console: string, - public readonly accountRamDeltas: unknown[], - public readonly except: unknown, - public readonly errorCode: number - ) {} -} diff --git a/src/common/blockchain/contract/delta/delta.ts b/src/common/blockchain/contract/delta/delta.ts deleted file mode 100644 index cd0981b..0000000 --- a/src/common/blockchain/contract/delta/delta.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { DeltaJson, DeltaRowDto } from './delta.dtos'; - -export class DeltaRow { - public static create(dto: DeltaRowDto): DeltaRow { - const { present, data } = dto; - return new DeltaRow(present, data); - } - - private constructor( - public readonly present: number, - public readonly data: Uint8Array - ) {} -} - -export class Delta { - public static create(shipMessageName: string, dto: DeltaJson): Delta { - const { name, rows } = dto; - - return new Delta( - shipMessageName, - name, - rows.map(dto => DeltaRow.create(dto)) - ); - } - - private constructor( - public readonly shipDeltaMessageName: string, - public readonly name: string, - public readonly rows: DeltaRow[] - ) {} -} diff --git a/src/common/blockchain/contract/signed-block/signed-block.ts b/src/common/blockchain/contract/signed-block/signed-block.ts deleted file mode 100644 index 86c2074..0000000 --- a/src/common/blockchain/contract/signed-block/signed-block.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* eslint-disable @typescript-eslint/no-empty-function */ - -import { parseDateToMs } from '@alien-worlds/api-core'; -import { Transaction } from '../transaction/transaction'; -import { SignedBlockJson } from './signed-block.dtos'; - -export class SignedBlock { - public static create(dto: SignedBlockJson): SignedBlock { - const { - producer, - confirmed, - previous, - transaction_mroot, - action_mroot, - schedule_version, - new_producers, - header_extensions, - producer_signature, - transactions, - } = dto; - - const timestamp = dto.timestamp ? new Date(parseDateToMs(dto.timestamp)) : new Date(); - - return new SignedBlock( - timestamp, - producer, - confirmed, - previous, - transaction_mroot, - action_mroot, - schedule_version, - new_producers, - header_extensions, - producer_signature, - transactions.map(dto => Transaction.create(dto)) - ); - } - - private constructor( - public readonly timestamp: Date, - public readonly producer: string, - public readonly confirmed: number, - public readonly previous: string, - public readonly transactionMroot: string, - public readonly actionMroot: string, - public readonly scheduleVersion: number, - public readonly newProducers: unknown, - public readonly headerExtensions: unknown[], - public readonly producerSignature: string, - public readonly transactions: Transaction[] - ) {} -} diff --git a/src/common/blockchain/contract/trace/trace.ts b/src/common/blockchain/contract/trace/trace.ts deleted file mode 100644 index e0d96ff..0000000 --- a/src/common/blockchain/contract/trace/trace.ts +++ /dev/null @@ -1,107 +0,0 @@ -/* eslint-disable @typescript-eslint/no-empty-function */ - -import { ActionTrace } from '../action-trace/action-trace'; -import { PartialDto, TraceJson } from './trace.dtos'; - -export class Partial { - public static create(type: string, dto: PartialDto): Partial { - const { - expiration, - ref_block_num, - ref_block_prefix, - max_net_usage_words, - max_cpu_usage_ms, - delay_sec, - transaction_extensions, - signatures, - context_free_data, - } = dto; - return new Partial( - type, - expiration, - ref_block_num, - ref_block_prefix, - max_net_usage_words, - max_cpu_usage_ms, - delay_sec, - transaction_extensions, - signatures, - context_free_data - ); - } - private constructor( - public readonly name: string, - public readonly expiration: string, - public readonly refBlockNumber: number, - public readonly refBlockPrefix: number, - public readonly maxNetUsageWords: number, - public readonly maxCpuUsageMs: number, - public readonly delayInSeconds: number, - public readonly transactionExtensions: unknown[], - public readonly signatures: unknown[], - public readonly contextFreeData: unknown[] - ) {} -} - -export class Trace { - public static create(shipMessageName: string, traceDto: TraceJson): Trace { - const { - id, - status, - cpu_usage_us, - net_usage_words, - elapsed, - net_usage, - scheduled, - action_traces, - account_ram_delta, - except, - error_code, - failed_dtrx_trace, - } = traceDto; - - const actionTraces = action_traces.map(item => { - const [actionTraceType, actionTraceDto] = item; - return ActionTrace.create(actionTraceType, actionTraceDto); - }); - let partial: Partial; - if (traceDto.partial) { - const [partialType, partialContent] = traceDto.partial; - partial = Partial.create(partialType, partialContent); - } - - return new Trace( - shipMessageName, - id, - status, - cpu_usage_us, - net_usage_words, - elapsed, - net_usage, - scheduled, - actionTraces, - account_ram_delta, - except, - Number(error_code), - failed_dtrx_trace, - partial - ); - } - - private constructor( - public readonly shipTraceMessageName: string, - public readonly id: string, - public readonly status: number, - public readonly cpuUsageUs: number, - public readonly netUsageWords: number, - public readonly elapsed: string, - public readonly netUsage: string, - public readonly scheduled: boolean, - public readonly actionTraces: ActionTrace[], - public readonly accountRamDelta: unknown, - public readonly except: unknown, - public readonly errorCode: number, - public readonly failedDtrxTrace: unknown, - public readonly partial: Partial | null - ) {} -} diff --git a/src/common/blockchain/contract/transaction/transaction.ts b/src/common/blockchain/contract/transaction/transaction.ts deleted file mode 100644 index 843c90a..0000000 --- a/src/common/blockchain/contract/transaction/transaction.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* eslint-disable @typescript-eslint/no-empty-function */ - -import { PackedTrxDto, TransactionDto } from "./transaction.dtos"; - -export class PackedTrx { - public static create(type: string, dto: PackedTrxDto): PackedTrx { - const { signatures, compression, packed_context_free_data, packed_trx } = dto; - return new PackedTrx( - type, - signatures, - compression, - packed_context_free_data, - packed_trx - ); - } - - private constructor( - public readonly type: string, - public readonly signatures: string[], - public readonly compression: number, - public readonly packedContextFreeData: unknown, - public readonly content: unknown //TODO: we should deserialize "packed_trx" - ) {} -} - -export class Trx { - public static create(type: string, dto: string): Trx { - return new Trx(type, dto); - } - - private constructor(public readonly type: string, public readonly content: string) {} -} - -export class Transaction { - public static create(dto: TransactionDto): Transaction { - const { status, cpu_usage_us, net_usage_words } = dto; - - const [type, content] = dto.trx; - let trx; - - switch (type) { - case 'transaction_id': { - trx = Trx.create(type, content); - break; - } - case 'packed_transaction': { - trx = PackedTrx.create(type, content); - break; - } - default: { - console.warn(`Unknown trx type "${type}"`); - } - } - return new Transaction(status, cpu_usage_us, net_usage_words, trx); - } - - private constructor( - public readonly status: number, - public readonly cpuUsageUs: number, - public readonly netUsageWords: number, - public readonly trx: Trx | PackedTrx | unknown - ) {} -} diff --git a/src/common/blockchain/index.ts b/src/common/blockchain/index.ts deleted file mode 100644 index 93d5885..0000000 --- a/src/common/blockchain/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './abi'; -export * from './blockchain'; -export * from './blockchain.types'; -export * from './block-reader'; -export * from './contract'; -export * from './contract-reader'; diff --git a/src/common/common.utils.ts b/src/common/common.utils.ts index 493cb86..d49f7a8 100644 --- a/src/common/common.utils.ts +++ b/src/common/common.utils.ts @@ -1,11 +1,9 @@ /** - * Suspends execution of the current process for a given number of milliseconds - * @async - * @param {number} ms - * @returns {Promise} + * Checks if a given contract and action represent the 'setabi' action of the 'eosio' contract. + * + * @param {string} contract The contract name. + * @param {string} action The action name. + * @returns {boolean} Returns `true` if the contract and action match the 'eosio' 'setabi' action; otherwise, returns `false`. */ -export const wait = async (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); - export const isSetAbiAction = (contract: string, action: string) => contract === 'eosio' && action === 'setabi'; - diff --git a/src/common/blockchain/contract/action-trace/__tests__/action-trace.unit.test.ts b/src/common/contract-content/action-trace/__tests__/action-trace.unit.test.ts similarity index 100% rename from src/common/blockchain/contract/action-trace/__tests__/action-trace.unit.test.ts rename to src/common/contract-content/action-trace/__tests__/action-trace.unit.test.ts diff --git a/src/common/blockchain/contract/action-trace/action-trace.dtos.ts b/src/common/contract-content/action-trace/action-trace.dtos.ts similarity index 100% rename from src/common/blockchain/contract/action-trace/action-trace.dtos.ts rename to src/common/contract-content/action-trace/action-trace.dtos.ts diff --git a/src/common/contract-content/action-trace/action-trace.ts b/src/common/contract-content/action-trace/action-trace.ts new file mode 100644 index 0000000..29d25da --- /dev/null +++ b/src/common/contract-content/action-trace/action-trace.ts @@ -0,0 +1,217 @@ +import { parseToBigInt } from '@alien-worlds/api-core'; +import { ActAuthJson, ActJson, ActionTraceDto, ReceiptJson } from './action-trace.dtos'; + +/** + * Represents the authorization information of an action. + * @class + */ +export class ActAuth { + /** + * Creates an ActAuth instance based on the provided json. + * + * @param {ActAuthJson} json - The json containing the authorization information. + * @returns {ActAuth} The created ActAuth instance. + */ + public static create(json: ActAuthJson): ActAuth { + const { actor, permission } = json; + + return new ActAuth(actor, permission); + } + + /** + * Creates an instance of ActAuth. + * + * @param {string} actor - The actor associated with the authorization. + * @param {string} permission - The permission associated with the authorization. + */ + constructor(public readonly actor: string, public readonly permission: string) {} +} + +/** + * Represents an action. + * @class + */ +export class Act { + /** + * Creates an Act instance based on the provided json. + * + * @param {ActJson} json - The json containing the action information. + * @returns {Act} The created Act instance. + */ + public static create(json: ActJson): Act { + const { account, name, data } = json; + + let authorization: ActAuth; + + if (json.authorization) { + authorization = ActAuth.create(json.authorization); + } + + return new Act(account, name, authorization, data); + } + + /** + * Creates an instance of Act. + * + * @param {string} account - The account associated with the action. + * @param {string} name - The name of the action. + * @param {ActAuth | undefined} authorization - The authorization information for the action. + * @param {Uint8Array} data - The data associated with the action. + */ + constructor( + public readonly account: string, + public readonly name: string, + public readonly authorization: ActAuth, + public readonly data: Uint8Array + ) {} +} + +/** + * Represents the sequence information of an authorization. + * + * @typedef {Object} AuthSequence + * @property {string} account - The account associated with the sequence. + * @property {string} sequence - The sequence number. + */ +export type AuthSequence = { + account: string; + sequence: string; +}; + +/** + * Represents the receipt information of an action. + * @class + */ +export class Receipt { + /** + * Creates a Receipt instance based on the provided json. + * + * @param {string} shipMessageName - The name of the ship message. + * @param {ReceiptJson} json - The json containing the receipt information. + * @returns {Receipt} The created Receipt instance. + */ + public static create(shipMessageName: string, json: ReceiptJson): Receipt { + const { + receiver, + act_digest, + global_sequence, + recv_sequence, + auth_sequence, + code_sequence, + abi_sequence, + } = json; + return new Receipt( + shipMessageName, + receiver, + act_digest, + parseToBigInt(global_sequence), + parseToBigInt(recv_sequence), + auth_sequence, + code_sequence, + abi_sequence + ); + } + + /** + * Creates an instance of Receipt. + * + * @param {string} shipMessageName - The name of the ship message. + * @param {string} receiver - The receiver of the action. + * @param {string} actDigest - The digest of the action. + * @param {bigint} globalSequence - The global sequence number. + * @param {bigint} recvSequence - The receiver sequence number. + * @param {AuthSequence[]} authSequence - The authorization sequence information. + * @param {number} codeSequence - The code sequence number. + * @param {number} abiSequence - The ABI sequence number. + */ + constructor( + public readonly shipMessageName: string, + public readonly receiver: string, + public readonly actDigest: string, + public readonly globalSequence: bigint, + public readonly recvSequence: bigint, + public readonly authSequence: AuthSequence[], + public readonly codeSequence: number, + public readonly abiSequence: number + ) {} +} + +/** + * Represents the trace information of an action. + * @class + */ +export class ActionTrace { + /** + * Creates an ActionTrace instance based on the provided DTO. + * + * @param {string} shipMessageName - The name of the ship message. + * @param {ActionTraceDto} dto - The DTO containing the action trace information. + * @returns {ActionTrace} The created ActionTrace instance. + */ + public static create(shipMessageName: string, dto: ActionTraceDto): ActionTrace { + const { + action_ordinal, + creator_action_ordinal, + receiver, + act, + context_free, + elapsed, + console, + account_ram_deltas, + except, + error_code, + } = dto; + + let receipt: Receipt; + if (dto.receipt && dto.receipt.length) { + const [receiptType, receiptContent] = dto.receipt; + receipt = Receipt.create(receiptType, receiptContent); + } + + return new ActionTrace( + shipMessageName, + action_ordinal, + creator_action_ordinal, + receipt, + receiver, + Act.create(act), + context_free, + elapsed, + console, + account_ram_deltas, + except, + Number(error_code) + ); + } + + /** + * Creates an instance of ActionTrace. + * + * @param {string} shipMessageName - The name of the ship message. + * @param {number} actionOrdinal - The ordinal of the action. + * @param {number} creatorActionOrdinal - The ordinal of the creator action. + * @param {Receipt | null} receipt - The receipt information of the action. + * @param {string} receiver - The receiver of the action. + * @param {Act} act - The action object. + * @param {boolean} isContextFree - Indicates whether the action is context-free. + * @param {string} elapsed - The elapsed time of the action. + * @param {string} console - The console output of the action. + * @param {unknown[]} accountRamDeltas - The account RAM deltas. + * @param {unknown} except - The exception information. + * @param {number} errorCode - The error code. + */ + constructor( + public readonly shipMessageName: string, + public readonly actionOrdinal: number, + public readonly creatorActionOrdinal: number, + public readonly receipt: Receipt | null, + public readonly receiver: string, + public readonly act: Act, + public readonly isContextFree: boolean, + public readonly elapsed: string, + public readonly console: string, + public readonly accountRamDeltas: unknown[], + public readonly except: unknown, + public readonly errorCode: number + ) {} +} diff --git a/src/common/blockchain/contract/action-trace/index.ts b/src/common/contract-content/action-trace/index.ts similarity index 100% rename from src/common/blockchain/contract/action-trace/index.ts rename to src/common/contract-content/action-trace/index.ts diff --git a/src/common/blockchain/contract/delta/__tests__/delta.unit.test.ts b/src/common/contract-content/delta/__tests__/delta.unit.test.ts similarity index 100% rename from src/common/blockchain/contract/delta/__tests__/delta.unit.test.ts rename to src/common/contract-content/delta/__tests__/delta.unit.test.ts diff --git a/src/common/blockchain/contract/delta/delta.dtos.ts b/src/common/contract-content/delta/delta.dtos.ts similarity index 100% rename from src/common/blockchain/contract/delta/delta.dtos.ts rename to src/common/contract-content/delta/delta.dtos.ts diff --git a/src/common/contract-content/delta/delta.ts b/src/common/contract-content/delta/delta.ts new file mode 100644 index 0000000..bdcb1f5 --- /dev/null +++ b/src/common/contract-content/delta/delta.ts @@ -0,0 +1,65 @@ +import { DeltaJson, DeltaRowDto } from './delta.dtos'; + +/** + * Represents a row in a delta. + * @class + */ +export class DeltaRow { + /** + * Creates a DeltaRow instance based on the provided json. + * + * @param {DeltaRowDto} json - The json containing the delta row information. + * @returns {DeltaRow} The created DeltaRow instance. + */ + public static create(json: DeltaRowDto): DeltaRow { + const { present, data } = json; + return new DeltaRow(present, data); + } + + /** + * Creates an instance of DeltaRow. + * + * @param {number} present - The present state of the delta row. + * @param {Uint8Array} data - The data associated with the delta row. + */ + private constructor( + public readonly present: number, + public readonly data: Uint8Array + ) {} +} + +/** + * Represents a delta. + * @class + */ +export class Delta { + /** + * Creates a Delta instance based on the provided json. + * + * @param {string} shipMessageName - The name of the ship message. + * @param {DeltaJson} json - The json containing the delta information. + * @returns {Delta} The created Delta instance. + */ + public static create(shipMessageName: string, json: DeltaJson): Delta { + const { name, rows } = json; + + return new Delta( + shipMessageName, + name, + rows.map(dto => DeltaRow.create(dto)) + ); + } + + /** + * Creates an instance of Delta. + * + * @param {string} shipDeltaMessageName - The name of the ship delta message. + * @param {string} name - The name of the delta. + * @param {DeltaRow[]} rows - The rows of the delta. + */ + constructor( + public readonly shipDeltaMessageName: string, + public readonly name: string, + public readonly rows: DeltaRow[] + ) {} +} diff --git a/src/common/blockchain/contract/delta/index.ts b/src/common/contract-content/delta/index.ts similarity index 100% rename from src/common/blockchain/contract/delta/index.ts rename to src/common/contract-content/delta/index.ts diff --git a/src/common/blockchain/contract/index.ts b/src/common/contract-content/index.ts similarity index 100% rename from src/common/blockchain/contract/index.ts rename to src/common/contract-content/index.ts diff --git a/src/common/blockchain/contract/signed-block/__tests__/block.unit.test.ts b/src/common/contract-content/signed-block/__tests__/signed-block.unit.test.ts similarity index 96% rename from src/common/blockchain/contract/signed-block/__tests__/block.unit.test.ts rename to src/common/contract-content/signed-block/__tests__/signed-block.unit.test.ts index 2afd95a..a8ca982 100644 --- a/src/common/blockchain/contract/signed-block/__tests__/block.unit.test.ts +++ b/src/common/contract-content/signed-block/__tests__/signed-block.unit.test.ts @@ -23,7 +23,7 @@ const dto = { ] as any, }; -describe('Block Unit tests', () => { +describe('Signed Block Unit tests', () => { beforeAll(() => { jest.useFakeTimers(); jest.setSystemTime(new Date(2022, 4, 5)); diff --git a/src/common/blockchain/contract/signed-block/index.ts b/src/common/contract-content/signed-block/index.ts similarity index 100% rename from src/common/blockchain/contract/signed-block/index.ts rename to src/common/contract-content/signed-block/index.ts diff --git a/src/common/blockchain/contract/signed-block/signed-block.dtos.ts b/src/common/contract-content/signed-block/signed-block.dtos.ts similarity index 73% rename from src/common/blockchain/contract/signed-block/signed-block.dtos.ts rename to src/common/contract-content/signed-block/signed-block.dtos.ts index 1d736f3..8e5dad2 100644 --- a/src/common/blockchain/contract/signed-block/signed-block.dtos.ts +++ b/src/common/contract-content/signed-block/signed-block.dtos.ts @@ -1,4 +1,4 @@ -import { TransactionDto } from '../transaction/transaction.dtos'; +import { TransactionJson } from '../transaction/transaction.dtos'; export type SignedBlockJson = { timestamp: string; @@ -11,5 +11,5 @@ export type SignedBlockJson = { new_producers: unknown; header_extensions: unknown[]; producer_signature: string; - transactions: TransactionDto[]; + transactions: TransactionJson[]; }; diff --git a/src/common/contract-content/signed-block/signed-block.ts b/src/common/contract-content/signed-block/signed-block.ts new file mode 100644 index 0000000..d3511b1 --- /dev/null +++ b/src/common/contract-content/signed-block/signed-block.ts @@ -0,0 +1,79 @@ +/* eslint-disable @typescript-eslint/no-empty-function */ + +import { parseDateToMs } from '@alien-worlds/api-core'; +import { Transaction } from '../transaction/transaction'; +import { SignedBlockJson } from './signed-block.dtos'; + +/** + * Represents a signed block. + * @class + */ +export class SignedBlock { + /** + * Creates a SignedBlock instance based on the provided json. + * + * @param {SignedBlockJson} json - The json containing the signed block information. + * @returns {SignedBlock} The created SignedBlock instance. + */ + public static create(json: SignedBlockJson): SignedBlock { + const { + producer, + confirmed, + previous, + transaction_mroot, + action_mroot, + schedule_version, + new_producers, + header_extensions, + producer_signature, + transactions, + } = json; + + const timestamp = json.timestamp + ? new Date(parseDateToMs(json.timestamp)) + : new Date(); + + return new SignedBlock( + timestamp, + producer, + confirmed, + previous, + transaction_mroot, + action_mroot, + schedule_version, + new_producers, + header_extensions, + producer_signature, + transactions.map(dto => Transaction.create(dto)) + ); + } + + /** + * Creates an instance of SignedBlock. + * + * @param {Date} timestamp - The timestamp of the signed block. + * @param {string} producer - The producer of the block. + * @param {number} confirmed - The number of confirmed blocks. + * @param {string} previous - The previous block ID. + * @param {string} transactionMroot - The Merkle root of the transactions. + * @param {string} actionMroot - The Merkle root of the actions. + * @param {number} scheduleVersion - The schedule version of the block. + * @param {unknown} newProducers - The new producers of the block. + * @param {unknown[]} headerExtensions - The header extensions of the block. + * @param {string} producerSignature - The producer signature of the block. + * @param {Transaction[]} transactions - The transactions included in the block. + */ + constructor( + public readonly timestamp: Date, + public readonly producer: string, + public readonly confirmed: number, + public readonly previous: string, + public readonly transactionMroot: string, + public readonly actionMroot: string, + public readonly scheduleVersion: number, + public readonly newProducers: unknown, + public readonly headerExtensions: unknown[], + public readonly producerSignature: string, + public readonly transactions: Transaction[] + ) {} +} diff --git a/src/common/blockchain/contract/trace/__tests__/trace.unit.test.ts b/src/common/contract-content/trace/__tests__/trace.unit.test.ts similarity index 100% rename from src/common/blockchain/contract/trace/__tests__/trace.unit.test.ts rename to src/common/contract-content/trace/__tests__/trace.unit.test.ts diff --git a/src/common/blockchain/contract/trace/index.ts b/src/common/contract-content/trace/index.ts similarity index 100% rename from src/common/blockchain/contract/trace/index.ts rename to src/common/contract-content/trace/index.ts diff --git a/src/common/blockchain/contract/trace/trace.dtos.ts b/src/common/contract-content/trace/trace.dtos.ts similarity index 79% rename from src/common/blockchain/contract/trace/trace.dtos.ts rename to src/common/contract-content/trace/trace.dtos.ts index b257134..def5a22 100644 --- a/src/common/blockchain/contract/trace/trace.dtos.ts +++ b/src/common/contract-content/trace/trace.dtos.ts @@ -1,6 +1,6 @@ import { ActionTraceByNameDto } from '../action-trace'; -export type PartialDto = { +export type PartialJson = { expiration: string; ref_block_num: number; ref_block_prefix: number; @@ -12,7 +12,7 @@ export type PartialDto = { context_free_data: unknown[]; }; -export type PartialByTypeDto = [string, PartialDto]; +export type PartialByTypeJson = [string, PartialJson]; export type TraceJson = { id: string; @@ -27,7 +27,7 @@ export type TraceJson = { except: unknown; error_code: number | string; failed_dtrx_trace: unknown; - partial: PartialByTypeDto; + partial: PartialByTypeJson; }; -export type TraceByNameDto = [string, TraceJson]; +export type TraceByNameJson = [string, TraceJson]; diff --git a/src/common/contract-content/trace/trace.ts b/src/common/contract-content/trace/trace.ts new file mode 100644 index 0000000..495ae5a --- /dev/null +++ b/src/common/contract-content/trace/trace.ts @@ -0,0 +1,162 @@ +/* eslint-disable @typescript-eslint/no-empty-function */ + +import { ActionTrace } from '../action-trace/action-trace'; +import { PartialJson, TraceJson } from './trace.dtos'; + +/** + * Represents a partial transaction. + * @class + */ +export class Partial { + /** + * Creates a Partial instance. + * + * @param {string} type - The type of the partial transaction. + * @param {PartialJson} json - The json containing the partial transaction information. + * @returns {Partial} The created Partial instance. + */ + public static create(type: string, json: PartialJson): Partial { + const { + expiration, + ref_block_num, + ref_block_prefix, + max_net_usage_words, + max_cpu_usage_ms, + delay_sec, + transaction_extensions, + signatures, + context_free_data, + } = json; + return new Partial( + type, + expiration, + ref_block_num, + ref_block_prefix, + max_net_usage_words, + max_cpu_usage_ms, + delay_sec, + transaction_extensions, + signatures, + context_free_data + ); + } + + /** + * Creates an instance of Partial. + * + * @param {string} name - The name of the partial transaction. + * @param {string} expiration - The expiration time of the partial transaction. + * @param {number} refBlockNumber - The reference block number of the partial transaction. + * @param {number} refBlockPrefix - The reference block prefix of the partial transaction. + * @param {number} maxNetUsageWords - The maximum net usage words of the partial transaction. + * @param {number} maxCpuUsageMs - The maximum CPU usage in milliseconds of the partial transaction. + * @param {number} delayInSeconds - The delay in seconds of the partial transaction. + * @param {unknown[]} transactionExtensions - The transaction extensions of the partial transaction. + * @param {unknown[]} signatures - The signatures of the partial transaction. + * @param {unknown[]} contextFreeData - The context-free data of the partial transaction. + */ + constructor( + public readonly name: string, + public readonly expiration: string, + public readonly refBlockNumber: number, + public readonly refBlockPrefix: number, + public readonly maxNetUsageWords: number, + public readonly maxCpuUsageMs: number, + public readonly delayInSeconds: number, + public readonly transactionExtensions: unknown[], + public readonly signatures: unknown[], + public readonly contextFreeData: unknown[] + ) {} +} + +/** + * Represents a trace of a transaction. + * @class + */ +export class Trace { + /** + * Creates a Trace instance. + * + * @param {string} shipMessageName - The name of the ship message associated with the trace. + * @param {TraceJson} json - The json containing the trace information. + * @returns {Trace} The created Trace instance. + */ + public static create(shipMessageName: string, json: TraceJson): Trace { + const { + id, + status, + cpu_usage_us, + net_usage_words, + elapsed, + net_usage, + scheduled, + action_traces, + account_ram_delta, + except, + error_code, + failed_dtrx_trace, + } = json; + + const actionTraces = action_traces.map(item => { + const [actionTraceType, actionTraceDto] = item; + return ActionTrace.create(actionTraceType, actionTraceDto); + }); + let partial: Partial; + if (json.partial) { + const [partialType, partialContent] = json.partial; + partial = Partial.create(partialType, partialContent); + } + + return new Trace( + shipMessageName, + id, + status, + cpu_usage_us, + net_usage_words, + elapsed, + net_usage, + scheduled, + actionTraces, + account_ram_delta, + except, + Number(error_code), + failed_dtrx_trace, + partial + ); + } + + /** + * Creates an instance of Trace. + * + * @param {string} shipTraceMessageName - The name of the ship trace message. + * @param {string} id - The ID of the trace. + * @param {number} status - The status of the trace. + * @param {number} cpuUsageUs - The CPU usage in microseconds of the trace. + * @param {number} netUsageWords - The net usage words of the trace. + * @param {string} elapsed - The elapsed time of the trace. + * @param {string} netUsage - The net usage of the trace. + * @param {boolean} scheduled - Indicates if the trace is scheduled. + * @param {ActionTrace[]} actionTraces - The action traces of the trace. + * @param {unknown} accountRamDelta - The account RAM delta of the trace. + * @param {unknown} except - The exception information of the trace. + * @param {number} errorCode - The error code of the trace. + * @param {unknown} failedDtrxTrace - The failed deferred transaction trace. + * @param {Partial | null} partial - The partial transaction associated with the trace. + */ + constructor( + public readonly shipTraceMessageName: string, + public readonly id: string, + public readonly status: number, + public readonly cpuUsageUs: number, + public readonly netUsageWords: number, + public readonly elapsed: string, + public readonly netUsage: string, + public readonly scheduled: boolean, + public readonly actionTraces: ActionTrace[], + public readonly accountRamDelta: unknown, + public readonly except: unknown, + public readonly errorCode: number, + public readonly failedDtrxTrace: unknown, + public readonly partial: Partial | null + ) {} +} diff --git a/src/common/blockchain/contract/transaction/__tests__/transaction.unit.test.ts b/src/common/contract-content/transaction/__tests__/transaction.unit.test.ts similarity index 100% rename from src/common/blockchain/contract/transaction/__tests__/transaction.unit.test.ts rename to src/common/contract-content/transaction/__tests__/transaction.unit.test.ts diff --git a/src/common/blockchain/contract/transaction/index.ts b/src/common/contract-content/transaction/index.ts similarity index 100% rename from src/common/blockchain/contract/transaction/index.ts rename to src/common/contract-content/transaction/index.ts diff --git a/src/common/blockchain/contract/transaction/transaction.dtos.ts b/src/common/contract-content/transaction/transaction.dtos.ts similarity index 56% rename from src/common/blockchain/contract/transaction/transaction.dtos.ts rename to src/common/contract-content/transaction/transaction.dtos.ts index adcab21..c528852 100644 --- a/src/common/blockchain/contract/transaction/transaction.dtos.ts +++ b/src/common/contract-content/transaction/transaction.dtos.ts @@ -1,14 +1,14 @@ -export type PackedTrxDto = { +export type PackedTrxJson = { signatures: string[]; compression: number; packed_context_free_data: unknown; packed_trx: Uint8Array; }; -export type TrxByNameDto = [string, PackedTrxDto | string]; +export type TrxByNameJson = [string, PackedTrxJson | string]; -export type TransactionDto = { +export type TransactionJson = { status: number; cpu_usage_us: number; net_usage_words: number; - trx: TrxByNameDto; + trx: TrxByNameJson; }; diff --git a/src/common/contract-content/transaction/transaction.ts b/src/common/contract-content/transaction/transaction.ts new file mode 100644 index 0000000..225e386 --- /dev/null +++ b/src/common/contract-content/transaction/transaction.ts @@ -0,0 +1,101 @@ +/* eslint-disable @typescript-eslint/no-empty-function */ + +import { PackedTrxJson, TransactionJson } from './transaction.dtos'; + +/** + * Represents a packed transaction. + * @class + */ +export class PackedTrx { + /** + * Creates a PackedTrx instance. + * + * @param {string} type - The type of the packed transaction. + * @param {PackedTrxJson} json - The DTO containing the packed transaction information. + * @returns {PackedTrx} The created PackedTrx instance. + */ + public static create(type: string, json: PackedTrxJson): PackedTrx { + const { signatures, compression, packed_context_free_data, packed_trx } = json; + return new PackedTrx( + type, + signatures, + compression, + packed_context_free_data, + packed_trx + ); + } + + /** + * Creates an instance of PackedTrx. + * + * @param {string} type - The type of the packed transaction. + * @param {string[]} signatures - The signatures of the packed transaction. + * @param {number} compression - The compression type of the packed transaction. + * @param {unknown} packedContextFreeData - The packed context-free data of the packed transaction. + * @param {unknown} content - The content of the packed transaction. + */ + constructor( + public readonly type: string, + public readonly signatures: string[], + public readonly compression: number, + public readonly packedContextFreeData: unknown, + public readonly content: unknown //TODO: we should deserialize "packed_trx" + ) {} +} + +export class Trx { + public static create(type: string, dto: string): Trx { + return new Trx(type, dto); + } + + private constructor(public readonly type: string, public readonly content: string) {} +} + +/** + * Represents a transaction. + * @class + */ +export class Transaction { + /** + * Creates a Transaction instance. + * + * @param {TransactionJson} json - The DTO containing the transaction information. + * @returns {Transaction} The created Transaction instance. + */ + public static create(json: TransactionJson): Transaction { + const { status, cpu_usage_us, net_usage_words } = json; + + const [type, content] = json.trx; + let trx; + + switch (type) { + case 'transaction_id': { + trx = Trx.create(type, content); + break; + } + case 'packed_transaction': { + trx = PackedTrx.create(type, content); + break; + } + default: { + console.warn(`Unknown trx type "${type}"`); + } + } + return new Transaction(status, cpu_usage_us, net_usage_words, trx); + } + + /** + * Creates an instance of Transaction. + * + * @param {number} status - The status of the transaction. + * @param {number} cpuUsageUs - The CPU usage in microseconds of the transaction. + * @param {number} netUsageWords - The net usage words of the transaction. + * @param {Trx | PackedTrx | unknown} trx - The transaction object. + */ + constructor( + public readonly status: number, + public readonly cpuUsageUs: number, + public readonly netUsageWords: number, + public readonly trx: Trx | PackedTrx | unknown + ) {} +} diff --git a/src/common/blockchain/contract-reader/contract-reader.config.ts b/src/common/contract-reader/contract-reader.config.ts similarity index 100% rename from src/common/blockchain/contract-reader/contract-reader.config.ts rename to src/common/contract-reader/contract-reader.config.ts diff --git a/src/common/blockchain/contract-reader/contract-reader.dtos.ts b/src/common/contract-reader/contract-reader.dtos.ts similarity index 84% rename from src/common/blockchain/contract-reader/contract-reader.dtos.ts rename to src/common/contract-reader/contract-reader.dtos.ts index 61cf327..c81f68a 100644 --- a/src/common/blockchain/contract-reader/contract-reader.dtos.ts +++ b/src/common/contract-reader/contract-reader.dtos.ts @@ -1,4 +1,4 @@ -import { MongoDB } from '@alien-worlds/api-core'; +import { MongoDB } from "@alien-worlds/storage-mongodb"; export type FeaturedContractDocument = { _id?: MongoDB.ObjectId; diff --git a/src/common/blockchain/contract-reader/contract-reader.ts b/src/common/contract-reader/contract-reader.ts similarity index 94% rename from src/common/blockchain/contract-reader/contract-reader.ts rename to src/common/contract-reader/contract-reader.ts index 4ccdc4c..c864df4 100644 --- a/src/common/blockchain/contract-reader/contract-reader.ts +++ b/src/common/contract-reader/contract-reader.ts @@ -1,9 +1,10 @@ import fetch from 'node-fetch'; -import { MongoConfig, MongoSource, log } from '@alien-worlds/api-core'; +import { log } from '@alien-worlds/api-core'; import { ContractReaderConfig } from './contract-reader.config'; import { FetchContractResponse } from './contract-reader.dtos'; import { FeaturedContract } from './featured-contract'; import { FeaturedContractSource } from './featured-contract.source'; +import { MongoConfig, MongoSource } from '@alien-worlds/storage-mongodb'; export abstract class ContractReader { public static async create( @@ -83,7 +84,7 @@ export class ContractReaderService implements ContractReader { if (resp) { entity = FeaturedContract.create(resp.account, resp.block_num); this.cache.set(entity.account, entity); - this.source.insert(entity.toDocument()); + this.source.insert([entity.toDocument()]); list.push(entity); } } diff --git a/src/common/blockchain/contract-reader/featured-contract.source.ts b/src/common/contract-reader/featured-contract.source.ts similarity index 85% rename from src/common/blockchain/contract-reader/featured-contract.source.ts rename to src/common/contract-reader/featured-contract.source.ts index 235c43c..6fd470d 100644 --- a/src/common/blockchain/contract-reader/featured-contract.source.ts +++ b/src/common/contract-reader/featured-contract.source.ts @@ -1,7 +1,7 @@ -import { CollectionMongoSource, MongoDB, MongoSource } from '@alien-worlds/api-core'; +import { MongoCollectionSource, MongoDB, MongoSource } from '@alien-worlds/storage-mongodb'; import { FeaturedContractDocument } from './contract-reader.dtos'; -export class FeaturedContractSource extends CollectionMongoSource { +export class FeaturedContractSource extends MongoCollectionSource { constructor(mongoSource: MongoSource) { super(mongoSource, 'history_tools.featured_contracts', { indexes: [ diff --git a/src/common/blockchain/contract-reader/featured-contract.ts b/src/common/contract-reader/featured-contract.ts similarity index 95% rename from src/common/blockchain/contract-reader/featured-contract.ts rename to src/common/contract-reader/featured-contract.ts index 73e7548..049bd2a 100644 --- a/src/common/blockchain/contract-reader/featured-contract.ts +++ b/src/common/contract-reader/featured-contract.ts @@ -1,9 +1,9 @@ import { - MongoDB, parseToBigInt, removeUndefinedProperties, } from '@alien-worlds/api-core'; import { FeaturedContractDocument } from './contract-reader.dtos'; +import { MongoDB } from '@alien-worlds/storage-mongodb'; export class FeaturedContract { /** diff --git a/src/common/blockchain/contract-reader/index.ts b/src/common/contract-reader/index.ts similarity index 100% rename from src/common/blockchain/contract-reader/index.ts rename to src/common/contract-reader/index.ts diff --git a/src/common/ship/index.ts b/src/common/ship/index.ts deleted file mode 100644 index 2f49e19..0000000 --- a/src/common/ship/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './ship-abis'; -export * from './ship-abi.source'; diff --git a/src/common/ship/ship-abi.source.ts b/src/common/ship/ship-abi.source.ts deleted file mode 100644 index 2d2cbc0..0000000 --- a/src/common/ship/ship-abi.source.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -/* eslint-disable @typescript-eslint/no-unsafe-call */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -import { - CollectionMongoSource, - Failure, - MongoDB, - MongoSource, - Result, -} from '@alien-worlds/api-core'; -import { AbiNotFoundError } from '../blockchain/block-reader/block-reader.errors'; -import { Abi } from '../blockchain/abi'; - -export type ShipAbiDocument = { - _id?: MongoDB.ObjectId; - last_modified_timestamp: Date; - version: string; - abi: string; -}; - -export class ShipAbiSource { - private collection: CollectionMongoSource; - - constructor(mongoSource: MongoSource) { - this.collection = new CollectionMongoSource(mongoSource, 'history_tools.ship_abis'); - } - - public async updateAbi(abi: Abi): Promise> { - try { - await this.collection.update({ - version: abi.version, - last_modified_timestamp: new Date(), - abi: abi.toHex(), - }); - - return Result.withoutContent(); - } catch (error) { - return Result.withFailure(Failure.fromError(error)); - } - } - - public async getAbi(version: string): Promise> { - try { - const document = await this.collection.findOne({ filter: { version } }); - - if (document) { - return Result.withContent(Abi.fromHex(document.abi)); - } - return Result.withFailure(Failure.fromError(new AbiNotFoundError())); - } catch (error) { - return Result.withFailure(Failure.fromError(error)); - } - } -} diff --git a/src/common/ship/ship-abis.ts b/src/common/ship/ship-abis.ts deleted file mode 100644 index a9f62f3..0000000 --- a/src/common/ship/ship-abis.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { MongoConfig, MongoSource, Result, isMongoConfig } from '@alien-worlds/api-core'; -import { ShipAbiSource } from './ship-abi.source'; -import { Abi } from '../blockchain/abi'; - -export class ShipAbis { - public static async create(mongo: MongoConfig | MongoSource) { - let mongoSource; - - if (isMongoConfig(mongo)) { - mongoSource = await MongoSource.create(mongo); - } else { - mongoSource = mongo; - } - - return new ShipAbis(new ShipAbiSource(mongoSource)); - } - - private cache: Map = new Map(); - - private constructor(private mongo: ShipAbiSource) {} - - public async getAbi(version: string): Promise> { - if (this.cache.has(version)) { - return Result.withContent(this.cache.get(version)); - } - - const result = await this.mongo.getAbi(version); - - if (result.isFailure) { - return result; - } - - this.cache.set(version, result.content); - - return result; - } -} diff --git a/src/common/workers/index.ts b/src/common/workers/index.ts deleted file mode 100644 index fee6f93..0000000 --- a/src/common/workers/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export * from './worker-message'; -export * from './worker-pool'; -export * from './worker'; -export * from './worker.enums'; -export * from './worker.errors'; -export * from './worker.utils'; -export * from './worker.types'; -export * from './worker-container'; -export * from './worker-loader'; diff --git a/src/common/workers/worker-container.ts b/src/common/workers/worker-container.ts deleted file mode 100644 index e6c8148..0000000 --- a/src/common/workers/worker-container.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { WorkerClass } from "./worker.types"; - -export class WorkerContainer { - private bindings: Map = new Map(); - - bind(label: string, workerClass: WorkerClass): void { - this.bindings.set(label, workerClass); - } - - get(label: string): WorkerClass { - return this.bindings.get(label) as WorkerClass; - } -} diff --git a/src/common/workers/worker-loader/index.ts b/src/common/workers/worker-loader/index.ts deleted file mode 100644 index 010f2df..0000000 --- a/src/common/workers/worker-loader/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './worker-loader'; -export * from './worker-loader.types'; -export * from './worker-loader.utils'; diff --git a/src/common/workers/worker-loader/worker-loader-script.ts b/src/common/workers/worker-loader/worker-loader-script.ts deleted file mode 100644 index 910e6ef..0000000 --- a/src/common/workers/worker-loader/worker-loader-script.ts +++ /dev/null @@ -1,50 +0,0 @@ -import async from 'async'; -import { workerData, parentPort } from 'worker_threads'; -import { WorkerMessage, WorkerMessageName } from '../worker-message'; -import { WorkerData } from '../worker.types'; -import { Worker } from '../worker'; -import { getWorkerLoader } from './worker-loader.utils'; -import { WorkerLoader } from './worker-loader'; - -let worker: Worker; -let workerLoader: WorkerLoader; - -export const messageHandler = async (message: WorkerMessage) => { - const { pointer, sharedData, options } = workerData as WorkerData; - if (message.name === WorkerMessageName.Setup) { - // - try { - workerLoader = getWorkerLoader(options?.workerLoaderPath); - await workerLoader.setup(sharedData); - parentPort.postMessage(WorkerMessage.setupComplete(message.workerId)); - } catch (error) { - parentPort.postMessage(WorkerMessage.setupFailure(message.workerId, error)); - } - } else if (message.name === WorkerMessageName.Load) { - // - try { - const { data } = >message; - worker = await workerLoader.load(data || pointer); - parentPort.postMessage(WorkerMessage.loadComplete(message.workerId)); - } catch (error) { - parentPort.postMessage(WorkerMessage.loadFailure(message.workerId, error)); - } - } else if (message.name === WorkerMessageName.Dispose) { - // - try { - worker = null; - parentPort.postMessage(WorkerMessage.disposeComplete(message.workerId)); - } catch (error) { - parentPort.postMessage(WorkerMessage.disposeFailure(message.workerId, error)); - } - } else if (message.name === WorkerMessageName.RunTask) { - // - worker.run(message.data); - } -}; - -const queue = async.queue(messageHandler); - -parentPort.on('message', (message: WorkerMessage) => { - queue.push(message); -}); diff --git a/src/common/workers/worker-loader/worker-loader.errors.ts b/src/common/workers/worker-loader/worker-loader.errors.ts deleted file mode 100644 index 253855c..0000000 --- a/src/common/workers/worker-loader/worker-loader.errors.ts +++ /dev/null @@ -1,7 +0,0 @@ -export class UndefinedPointerError extends Error { - constructor() { - super( - `Undefined pointer. The worker loader does not know which Worker class to initialize. If you use only one class, override your worker loader's "load" method so that it returns instance of your worker instead of calling super.load()` - ); - } -} diff --git a/src/common/workers/worker-loader/worker-loader.ts b/src/common/workers/worker-loader/worker-loader.ts deleted file mode 100644 index 35e0133..0000000 --- a/src/common/workers/worker-loader/worker-loader.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-unused-vars */ -import { existsSync } from 'fs'; -import { Worker } from '../worker'; -import { buildPath } from './worker-loader.utils'; -import { WorkerClass } from '../worker.types'; -import { WorkerConstructorArgs } from './worker-loader.types'; -import { UndefinedPointerError } from './worker-loader.errors'; - -export abstract class WorkerLoader { - public abstract bindings: Map; - - public abstract setup(sharedData: SharedDataType, ...args: unknown[]): Promise; - public abstract load( - pointer: string, - workerConstructorArgs?: WorkerConstructorArgs - ): Promise; -} - -export class DefaultWorkerLoader - implements WorkerLoader -{ - public bindings: Map = new Map(); - protected sharedData: SharedDataType; - - public async setup(sharedData: SharedDataType, ...args: unknown[]): Promise { - this.sharedData = sharedData; - } - public async load( - pointer: string, - workerConstructorArgs?: WorkerConstructorArgs - ): Promise { - let WorkerClass; - - if (!pointer) { - throw new UndefinedPointerError(); - } - - const filePath = buildPath(pointer); - if (existsSync(filePath)) { - WorkerClass = require(filePath).default; - } else if (this.bindings.has(pointer)) { - WorkerClass = this.bindings.get(pointer); - } else { - throw new Error( - `A valid path to a worker was not specified or a worker was not assigned to the given name ${pointer}` - ); - } - - const worker = new WorkerClass(workerConstructorArgs) as Worker; - return worker; - } -} diff --git a/src/common/workers/worker-loader/worker-loader.types.ts b/src/common/workers/worker-loader/worker-loader.types.ts deleted file mode 100644 index dbdf3c1..0000000 --- a/src/common/workers/worker-loader/worker-loader.types.ts +++ /dev/null @@ -1,2 +0,0 @@ -export type WorkerLoaderClass = new (...args: never[]) => void; -export type WorkerConstructorArgs = { [key: string]: unknown }; diff --git a/src/common/workers/worker-loader/worker-loader.utils.ts b/src/common/workers/worker-loader/worker-loader.utils.ts deleted file mode 100644 index 5fb79f2..0000000 --- a/src/common/workers/worker-loader/worker-loader.utils.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* eslint-disable @typescript-eslint/no-var-requires */ - -import { existsSync } from 'fs'; -import path from 'path'; -import { InvalidPathError } from '../worker.errors'; -import { DefaultWorkerLoader, WorkerLoader } from './worker-loader'; - -export const buildPath = (filePath: string): string => { - if (filePath.endsWith('.ts')) { - require('ts-node').register(); - return path.resolve(process.cwd(), 'src', `${filePath}`); - } else { - return path.resolve( - process.cwd(), - 'build', - `${filePath}${filePath.endsWith('.js') ? '' : '.js'}` - ); - } -}; - -export const getWorkerLoader = (path: string): WorkerLoader => { - if (path) { - const loaderPath = buildPath(path); - if (existsSync(loaderPath) === false) { - throw new InvalidPathError(loaderPath); - } - const WorkerLoaderClass = require(loaderPath).default; - return new WorkerLoaderClass() as WorkerLoader; - } - - return new DefaultWorkerLoader(); -}; diff --git a/src/common/workers/worker-message.ts b/src/common/workers/worker-message.ts deleted file mode 100644 index d00e582..0000000 --- a/src/common/workers/worker-message.ts +++ /dev/null @@ -1,227 +0,0 @@ -export type ErrorJson = { - name?: string; - message?: string; - stack?: string; - [key: string]: unknown; -}; - -export type WorkerMessageContent = { - workerId: number; - type: string; - name: string; - data?: DataType; - error?: ErrorJson; -}; - -export type WorkerMessageHandler = (message: WorkerMessage) => void; - -export class WorkerMessage { - public static create({ - workerId, - type, - name, - data, - error, - }: WorkerMessageContent) { - let errorJson: ErrorJson; - if (error) { - const { message, stack, name: errorName, ...rest } = error; - errorJson = { - message, - stack, - name: errorName, - ...rest, - }; - } - - return new WorkerMessage(workerId, type, name, data, errorJson); - } - - public static setup(workerId: number) { - return new WorkerMessage(workerId, WorkerMessageType.System, WorkerMessageName.Setup); - } - - public static setupComplete(workerId: number) { - return new WorkerMessage( - workerId, - WorkerMessageType.System, - WorkerMessageName.SetupComplete - ); - } - - public static setupFailure(workerId: number, error: Error) { - return new WorkerMessage( - workerId, - WorkerMessageType.System, - WorkerMessageName.SetupFailure, - error, - error - ); - } - - public static load(workerId: number, pointer: string) { - return new WorkerMessage( - workerId, - WorkerMessageType.System, - WorkerMessageName.Load, - pointer - ); - } - - public static loadComplete(workerId: number) { - return new WorkerMessage( - workerId, - WorkerMessageType.System, - WorkerMessageName.LoadComplete - ); - } - - public static loadFailure(workerId: number, error: Error) { - return new WorkerMessage( - workerId, - WorkerMessageType.System, - WorkerMessageName.LoadFailure, - error, - error - ); - } - - public static dispose(workerId: number) { - return new WorkerMessage( - workerId, - WorkerMessageType.System, - WorkerMessageName.Dispose - ); - } - - public static disposeComplete(workerId: number) { - return new WorkerMessage( - workerId, - WorkerMessageType.System, - WorkerMessageName.DisposeComplete - ); - } - - public static disposeFailure(workerId: number, error: Error) { - return new WorkerMessage( - workerId, - WorkerMessageType.System, - WorkerMessageName.DisposeComplete, - error, - error - ); - } - - public static runTask(workerId: number, value: DataType) { - return new WorkerMessage( - workerId, - WorkerMessageType.Info, - WorkerMessageName.RunTask, - value - ); - } - - public static use(workerId: number, value: DataType) { - return new WorkerMessage( - workerId, - WorkerMessageType.Info, - WorkerMessageName.PassData, - value - ); - } - - public static taskResolved(workerId: number, value: DataType) { - return new WorkerMessage( - workerId, - WorkerMessageType.Info, - WorkerMessageName.TaskResolved, - value - ); - } - - public static taskRejected(workerId: number, error: Error) { - return new WorkerMessage( - workerId, - WorkerMessageType.Error, - WorkerMessageName.TaskRejected, - null, - error - ); - } - - public static taskProgress(workerId: number, value: DataType) { - return new WorkerMessage( - workerId, - WorkerMessageType.Info, - WorkerMessageName.TaskProgress, - value - ); - } - - private constructor( - public readonly workerId: number, - public readonly type: string, - public readonly name: string, - public readonly data?: DataType, - public readonly error?: ErrorJson - ) {} - - public isTaskResolved(): boolean { - return this.name === WorkerMessageName.TaskResolved; - } - - public isTaskRejected(): boolean { - return this.name === WorkerMessageName.TaskRejected; - } - - public isTaskProgress(): boolean { - return this.name === WorkerMessageName.TaskProgress; - } - - public toJson(): object { - const { workerId, type, name, data, error } = this; - let errorJson = {}; - if (error) { - const { message, stack, name: errorName, ...rest } = error; - errorJson = { - message, - stack, - name: errorName, - ...rest, - }; - } - return { - workerId, - type, - name, - data, - error: errorJson, - }; - } -} - -export enum WorkerMessageType { - Error = 'error', - Info = 'info', - Warning = 'warning', - Task = 'task', - System = 'system', -} - -export enum WorkerMessageName { - Setup = 'setup', - SetupComplete = 'setup_complete', - SetupFailure = 'setup_failure', - Load = 'load', - LoadComplete = 'load_complete', - LoadFailure = 'load_failure', - Dispose = 'dispose', - DisposeComplete = 'dispose_complete', - DisposeFailure = 'dispose_failure', - RunTask = 'run_task', - PassData = 'pass_data', - DataPassed = 'data_passed', - TaskResolved = 'task_resolved', - TaskRejected = 'task_rejected', - TaskProgress = 'task_progress', -} diff --git a/src/common/workers/worker-pool.ts b/src/common/workers/worker-pool.ts deleted file mode 100644 index 53cd41b..0000000 --- a/src/common/workers/worker-pool.ts +++ /dev/null @@ -1,114 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { log } from '@alien-worlds/api-core'; -import { WorkerProxy } from './worker-proxy'; -import { WorkerPoolOptions } from './worker.types'; -import { getWorkersCount } from './worker.utils'; - -type WorkerReleaseHandler = (id: number, data?: unknown) => Promise | void; - -export class WorkerPool { - public static async create(options: WorkerPoolOptions) { - const pool = new WorkerPool(); - await pool.setup(options); - return pool; - } - - public workerMaxCount: number; - - private workerLoaderPath: string; - private availableWorkers: WorkerProxy[] = []; - private activeWorkersByPid = new Map(); - private sharedData: unknown; - private workerReleaseHandler: WorkerReleaseHandler; - - public async setup(options: WorkerPoolOptions) { - const { - threadsCount, - inviolableThreadsCount, - sharedData, - workerLoaderPath, - } = options; - this.workerLoaderPath = workerLoaderPath; - this.sharedData = sharedData; - this.workerMaxCount = - threadsCount > inviolableThreadsCount - ? getWorkersCount(threadsCount, inviolableThreadsCount) - : threadsCount; - - for (let i = 0; i < this.workerMaxCount; i++) { - const worker = await this.createWorker(); - this.availableWorkers.push(worker); - } - } - - public get workerCount() { - return this.availableWorkers.length + this.activeWorkersByPid.size; - } - - private async createWorker(): Promise { - const { sharedData, workerLoaderPath } = this; - const proxy = new WorkerProxy(sharedData, { workerLoaderPath }); - await proxy.setup(); - return proxy; - } - - public async getWorker(pointer?: string): Promise { - const { activeWorkersByPid, workerMaxCount, availableWorkers } = - this; - - if (activeWorkersByPid.size < workerMaxCount) { - // When workers are to run common or concrete process, - // we use instance from the list (if there is any available) - const worker = availableWorkers.shift(); - activeWorkersByPid.set(worker.id, worker); - await worker.load(pointer); - return worker as WorkerType & WorkerProxy; - } else { - return null; - } - } - - public async releaseWorker(id: number, data?: unknown): Promise { - const { activeWorkersByPid, availableWorkers, workerMaxCount, workerReleaseHandler } = - this; - const worker = activeWorkersByPid.get(id); - - if (worker) { - await worker.dispose(); - this.activeWorkersByPid.delete(id); - if (availableWorkers.length < workerMaxCount) { - availableWorkers.push(worker); - } - if (workerReleaseHandler) { - await workerReleaseHandler(id, data); - } - } else { - log(`No worker with the specified ID #${id} was found`); - } - } - - public removeWorkers() { - this.activeWorkersByPid.forEach(worker => worker.remove()); - this.availableWorkers.forEach(worker => worker.remove()); - } - - public hasAvailableWorker(): boolean { - return this.workerMaxCount - this.activeWorkersByPid.size > 0; - } - - public hasActiveWorkers(): boolean { - return this.activeWorkersByPid.size > 0; - } - - public countAvailableWorkers(): number { - return this.workerMaxCount - this.activeWorkersByPid.size; - } - - public countActiveWorkers(): number { - return this.activeWorkersByPid.size; - } - - public onWorkerRelease(handler: WorkerReleaseHandler): void { - this.workerReleaseHandler = handler; - } -} diff --git a/src/common/workers/worker-proxy.ts b/src/common/workers/worker-proxy.ts deleted file mode 100644 index 499905c..0000000 --- a/src/common/workers/worker-proxy.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { log } from '@alien-worlds/api-core'; -import { Worker } from 'worker_threads'; -import { - WorkerMessage, - WorkerMessageContent, - WorkerMessageName, - WorkerMessageType, -} from './worker-message'; -import { WorkerProxyOptions } from './worker.types'; - -export class WorkerProxy { - private _pointer: string; - private worker: Worker; - - constructor(sharedData: unknown, options: WorkerProxyOptions) { - this.worker = new Worker(`${__dirname}/worker-loader/worker-loader-script`, { - workerData: { sharedData, options }, - }); - } - - public get id(): number { - return this.worker.threadId; - } - - public get pointer(): string { - return this._pointer; - } - - public async setup(): Promise { - const { worker } = this; - worker.removeAllListeners(); - return new Promise((resolveSetup, rejectSetup) => { - worker.on('message', (content: WorkerMessageContent) => { - const { type, name, data } = content; - if ( - type === WorkerMessageType.System && - name === WorkerMessageName.SetupComplete - ) { - worker.removeAllListeners(); - resolveSetup(); - } else if ( - type === WorkerMessageType.System && - name === WorkerMessageName.SetupFailure - ) { - worker.removeAllListeners(); - rejectSetup(data); - } - }); - worker.postMessage(WorkerMessage.setup(worker.threadId).toJson()); - }); - } - - public async load(pointer: string): Promise { - this._pointer = pointer; - const { worker } = this; - worker.removeAllListeners(); - return new Promise((resolveLoad, rejectLoad) => { - worker.on('message', (content: WorkerMessageContent) => { - const { type, name, data } = content; - if ( - type === WorkerMessageType.System && - name === WorkerMessageName.LoadComplete - ) { - worker.removeAllListeners(); - resolveLoad(); - } else if ( - type === WorkerMessageType.System && - name === WorkerMessageName.LoadFailure - ) { - worker.removeAllListeners(); - rejectLoad(data); - } - }); - worker.postMessage(WorkerMessage.load(worker.threadId, pointer).toJson()); - }); - } - - public async dispose(): Promise { - const { worker } = this; - worker.removeAllListeners(); - return new Promise((resolveDispose, rejectDispose) => { - worker.on('message', (content: WorkerMessageContent) => { - const { type, name, data } = content; - if ( - type === WorkerMessageType.System && - name === WorkerMessageName.DisposeComplete - ) { - worker.removeAllListeners(); - resolveDispose(); - } else if ( - type === WorkerMessageType.System && - name === WorkerMessageName.DisposeFailure - ) { - worker.removeAllListeners(); - rejectDispose(data); - } - }); - worker.postMessage(WorkerMessage.dispose(worker.threadId).toJson()); - }); - } - - public run(data?: DataType): void { - const { worker } = this; - worker.postMessage(WorkerMessage.runTask(worker.threadId, data).toJson()); - } - - public onMessage(handler: (message: WorkerMessage) => Promise) { - this.worker.on('message', (content: WorkerMessageContent) => { - if (content.type !== WorkerMessageType.System) { - handler(WorkerMessage.create(content)).catch(log); - } - }); - } - - public onError(handler: (workerId: number, error: Error) => void) { - this.worker.on('error', error => handler(this.worker.threadId, error)); - } - - public onExit(handler: (workerId: number, code: number) => void) { - this.worker.on('exit', code => handler(this.worker.threadId, code)); - } - - public async remove(): Promise { - const code = await this.worker.terminate(); - return code; - } -} diff --git a/src/common/workers/worker.enums.ts b/src/common/workers/worker.enums.ts deleted file mode 100644 index 4e87f41..0000000 --- a/src/common/workers/worker.enums.ts +++ /dev/null @@ -1,4 +0,0 @@ -export enum WorkerStatus { - complete = 'complete', - error = 'complete', -} \ No newline at end of file diff --git a/src/common/workers/worker.errors.ts b/src/common/workers/worker.errors.ts deleted file mode 100644 index b43dcd6..0000000 --- a/src/common/workers/worker.errors.ts +++ /dev/null @@ -1,27 +0,0 @@ -export class MissingWorkerPathError extends Error {} - -export class InvalidPathError extends Error { - constructor(path: string) { - super(`The given path is invalid: ${path}`); - } -} - -export class WorkerPathMismatchError extends Error { - constructor(path: string, globalPath: string) { - super(`You cannot use path (${path}) when global is specified (${globalPath})`); - } -} - -export class WorkerNotFoundError extends Error { - constructor(id: number) { - super(`No worker with the specified ID (${id}) was found`); - } -} - -export class WorkerPoolPathsConflictError extends Error { - constructor() { - super( - `default worker path and "containerPath" cannot be specified at the same time, both options are mutually exclusive.` - ); - } -} diff --git a/src/common/workers/worker.ts b/src/common/workers/worker.ts deleted file mode 100644 index 81a2948..0000000 --- a/src/common/workers/worker.ts +++ /dev/null @@ -1,47 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-call */ -/* eslint-disable @typescript-eslint/no-unused-vars */ -/* eslint-disable @typescript-eslint/require-await */ -import { parentPort, threadId } from 'worker_threads'; -import { WorkerMessage } from './worker-message'; - -export type TaskResolved = 'task_resolved'; -export type TaskRejected = 'task_rejected'; -export type TaskProgress = 'task_progress'; -export type TaskStatus = TaskResolved | TaskRejected | TaskProgress; - -export class Worker { - public get id(): number { - return threadId; - } - - protected sharedData: SharedDataType; - private isRejected = false; - - public run(...args: unknown[]): void { - throw new Error('Method not implemented'); - } - - public deserialize(data: unknown): unknown { - throw new Error('Method not implemented'); - } - - public resolve(data?: DataType): TaskResolved { - if (this.isRejected === false) { - parentPort.postMessage( - WorkerMessage.taskResolved(threadId, data).toJson() - ); - return 'task_resolved'; - } - } - - public reject(error?: Error): TaskRejected { - parentPort.postMessage(WorkerMessage.taskRejected(threadId, error).toJson()); - this.isRejected = true; - return 'task_rejected'; - } - - public progress(data?: DataType): TaskProgress { - parentPort.postMessage(WorkerMessage.taskProgress(threadId, data).toJson()); - return 'task_progress'; - } -} diff --git a/src/common/workers/worker.types.ts b/src/common/workers/worker.types.ts deleted file mode 100644 index adb88a1..0000000 --- a/src/common/workers/worker.types.ts +++ /dev/null @@ -1,25 +0,0 @@ -export type PathsByNames = { - default?: string; - [key: string]: string; -}; - -export type WorkersConfig = { - threadsCount?: number; - inviolableThreadsCount?: number; - sharedData?: SharedDataType; - [key: string]: unknown; -}; - -export type WorkerProxyOptions = { - workerLoaderPath?: string; -}; - -export type WorkerPoolOptions = WorkersConfig & WorkerProxyOptions; - -export type WorkerData = { - pointer: string; - sharedData?: unknown; - options?: WorkerProxyOptions; -}; - -export type WorkerClass = new (...args: unknown[]) => T; diff --git a/src/common/workers/worker.utils.ts b/src/common/workers/worker.utils.ts deleted file mode 100644 index 7e1f612..0000000 --- a/src/common/workers/worker.utils.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -import * as os from 'os'; -import { workerData } from 'worker_threads'; -/** - * Get the number of workers from configuration - * or based on the number of available CPU cores. - * The number of CPU cores is reduced by - * a constant specified in the configuration. - * - * @param {Config} config - * @returns {number} - */ -export const getWorkersCount = ( - threadsCount: number, - inviolableThreadsCount = 0 -): number => { - if (threadsCount === 0 || isNaN(threadsCount)) { - const cpus = os.cpus().length; - return cpus - inviolableThreadsCount; - } - - return threadsCount; -}; - -export const getSharedData = () => workerData.sharedData as T; diff --git a/src/config/index.ts b/src/config/index.ts index cd6f6d7..b95033b 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -1,10 +1,5 @@ import { ProcessorTaskQueueConfig } from '../processor/processor-task-queue/processor-task-queue.config'; -import { - ConfigVars, - buildBroadcastConfig, - buildMongoConfig, - parseToBigInt, -} from '@alien-worlds/api-core'; +import { ConfigVars, parseToBigInt } from '@alien-worlds/api-core'; import { ApiConfig } from '../api'; import { BootstrapConfig, BootstrapCommandOptions } from '../bootstrap'; import { ReaderConfig, ReaderCommandOptions } from '../reader'; @@ -20,6 +15,8 @@ import { FeaturedConfig, WorkersConfig, } from '../common'; +import { buildMongoConfig } from '@alien-worlds/storage-mongodb'; +import { buildBroadcastConfig } from '@alien-worlds/broadcast'; export * from './config.types'; @@ -136,7 +133,9 @@ export const buildReaderConfig = ( mode: options?.mode || vars.getStringEnv('MODE'), maxBlockNumber: vars.getNumberEnv('MAX_BLOCK_NUMBER'), blockQueueMaxBytesSize: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_MAX_BYTES_SIZE'), - blockQueueSizeCheckInterval: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_SIZE_CHECK_INTERVAL'), + blockQueueSizeCheckInterval: vars.getNumberEnv( + 'UNPROCESSED_BLOCK_QUEUE_SIZE_CHECK_INTERVAL' + ), blockQueueBatchSize: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_BATCH_SIZE'), workers: buildReaderWorkersConfig(vars, options?.threads), blockReader: buildBlockReaderConfig(vars), diff --git a/src/processor/processor-task-queue/data-sources/processor-task.source.ts b/src/processor/processor-task-queue/data-sources/processor-task.source.ts index 51d67fc..7d0f124 100644 --- a/src/processor/processor-task-queue/data-sources/processor-task.source.ts +++ b/src/processor/processor-task-queue/data-sources/processor-task.source.ts @@ -1,13 +1,9 @@ -import { - CollectionMongoSource, - DataSourceOperationError, - MongoDB, - MongoSource, -} from '@alien-worlds/api-core'; +import { MongoCollectionSource, MongoDB, MongoSource } from '@alien-worlds/storage-mongodb'; import { ProcessorTaskQueueConfig } from '../processor-task-queue.config'; import { ProcessorTaskDocument } from '../processor-task.types'; +import { DataSourceError } from '@alien-worlds/api-core'; -export class ProcessorTaskSource extends CollectionMongoSource { +export class ProcessorTaskSource extends MongoCollectionSource { private transactionOptions: MongoDB.TransactionOptions; constructor(mongoSource: MongoSource, private config: ProcessorTaskQueueConfig) { @@ -45,7 +41,7 @@ export class ProcessorTaskSource extends CollectionMongoSource { +export class UnsuccessfulProcessorTaskSource extends MongoCollectionSource { constructor(mongoSource: MongoSource) { super(mongoSource, 'history_tools.unsuccessful_processor_tasks', { indexes: [ diff --git a/src/processor/processor-task-queue/processor-task-queue.ts b/src/processor/processor-task-queue/processor-task-queue.ts index bc63af7..19d697b 100644 --- a/src/processor/processor-task-queue/processor-task-queue.ts +++ b/src/processor/processor-task-queue/processor-task-queue.ts @@ -1,14 +1,13 @@ import { - DataSourceBulkWriteError, + DataSourceError, log, - MongoConfig, - MongoSource, } from '@alien-worlds/api-core'; import { ProcessorTaskSource } from './data-sources/processor-task.source'; import { ProcessorTask } from './processor-task'; import { UnsuccessfulProcessorTaskSource } from './data-sources/unsuccessful-processor-task.source'; import { ProcessorTaskQueueConfig } from './processor-task-queue.config'; import { ErrorJson } from '../../common/workers/worker-message'; +import { MongoConfig, MongoSource } from '@alien-worlds/storage-mongodb'; export class ProcessorTaskQueue { public static async create( @@ -72,9 +71,9 @@ export class ProcessorTaskQueue { const source = unsuccessful ? this.unsuccessfulSource : this.source; try { const dtos = tasks.map(task => task.toDocument()); - await source.insertMany(dtos); + await source.insert(dtos); } catch (error) { - const { concernError } = error; + const { error: concernError } = error; const concernErrorMessage = (concernError)?.message || ''; log(`Could not add tasks due to: ${error.message}. ${concernErrorMessage}`); } @@ -86,10 +85,10 @@ export class ProcessorTaskQueue { ): Promise { try { const { message, stack } = error; - const dto = task.toDocument(); - dto.error = { message, stack }; + const document = task.toDocument(); + document.error = { message, stack }; - await this.unsuccessfulSource.insert(dto); + await this.unsuccessfulSource.insert([document]); } catch (sourceError) { log(`Could not stash failed task due to: ${error.message}`); } diff --git a/src/processor/processor-task-queue/processor-task.ts b/src/processor/processor-task-queue/processor-task.ts index 1b21329..94b8b18 100644 --- a/src/processor/processor-task-queue/processor-task.ts +++ b/src/processor/processor-task-queue/processor-task.ts @@ -1,7 +1,6 @@ import crypto from 'crypto'; import { serialize } from 'v8'; import { - MongoDB, parseToBigInt, removeUndefinedProperties, } from '@alien-worlds/api-core'; @@ -11,6 +10,7 @@ import { ProcessorTaskDocument, ProcessorTaskError, } from './processor-task.types'; +import { MongoDB } from '@alien-worlds/storage-mongodb'; export enum ProcessorTaskType { Action = 'action', diff --git a/src/processor/processor-task-queue/processor-task.types.ts b/src/processor/processor-task-queue/processor-task.types.ts index c7c187d..c277d6d 100644 --- a/src/processor/processor-task-queue/processor-task.types.ts +++ b/src/processor/processor-task-queue/processor-task.types.ts @@ -1,4 +1,4 @@ -import { MongoDB } from '@alien-worlds/api-core'; +import { MongoDB } from '@alien-worlds/storage-mongodb'; import { ActionTraceModel } from '../../common/blockchain/contract/action-trace'; import { DeltaRowModel } from '../../common/blockchain/contract/delta'; diff --git a/src/processor/processor.types.ts b/src/processor/processor.types.ts index a834988..c342991 100644 --- a/src/processor/processor.types.ts +++ b/src/processor/processor.types.ts @@ -1,7 +1,7 @@ -import { MongoConfig, BroadcastConfig } from '@alien-worlds/api-core'; import { FeaturedConfig, FeaturedMatchers } from '../common/featured'; import { ProcessorTaskQueueConfig } from './processor-task-queue/processor-task-queue.config'; import { WorkersConfig } from '../common/workers'; +import { MongoConfig } from '@alien-worlds/storage-mongodb'; export type ProcessorCommandOptions = { threads: number; diff --git a/src/processor/processor.worker-loader.ts b/src/processor/processor.worker-loader.ts index fae66f1..44c48b6 100644 --- a/src/processor/processor.worker-loader.ts +++ b/src/processor/processor.worker-loader.ts @@ -1,4 +1,4 @@ -import { MongoSource } from '@alien-worlds/api-core'; +import { MongoSource } from '@alien-worlds/storage-mongodb'; import { Worker } from '../common/workers'; import { DefaultWorkerLoader } from '../common/workers/worker-loader'; import { ProcessorSharedData } from './processor.types'; diff --git a/src/processor/processors/action-trace.processor.input.ts b/src/processor/processors/action-trace.processor.input.ts index 312b426..c98f085 100644 --- a/src/processor/processors/action-trace.processor.input.ts +++ b/src/processor/processors/action-trace.processor.input.ts @@ -1,5 +1,5 @@ import { deserialize } from 'v8'; -import { AbisSerialize } from '../../common/abis/abis.serialize'; +import { AbisSerialize } from '../../common/abis/abis.serializer'; import { ActionProcessorContentModel, ProcessorTaskModel, diff --git a/src/processor/processors/delta.processor.input.ts b/src/processor/processors/delta.processor.input.ts index 249ac01..354ca12 100644 --- a/src/processor/processors/delta.processor.input.ts +++ b/src/processor/processors/delta.processor.input.ts @@ -1,6 +1,6 @@ import { Serialize } from 'eosjs'; import { deserialize } from 'v8'; -import { AbisSerialize } from '../../common/abis/abis.serialize'; +import { AbisSerialize } from '../../common/abis/abis.serializer'; import { DeltaProcessorContentModel, ProcessorTaskModel, diff --git a/src/processor/start-processor.ts b/src/processor/start-processor.ts index 5baf540..7e1b032 100644 --- a/src/processor/start-processor.ts +++ b/src/processor/start-processor.ts @@ -1,4 +1,4 @@ -import { Broadcast, log } from '@alien-worlds/api-core'; +import { log } from '@alien-worlds/api-core'; import { ProcessorAddons, ProcessorConfig } from './processor.types'; import { InternalBroadcastChannel, diff --git a/src/reader/block-range-scanner/block-range-scan.mongo.source.ts b/src/reader/block-range-scanner/block-range-scan.mongo.source.ts index 027aeb2..08745bc 100644 --- a/src/reader/block-range-scanner/block-range-scan.mongo.source.ts +++ b/src/reader/block-range-scanner/block-range-scan.mongo.source.ts @@ -1,19 +1,17 @@ /* eslint-disable @typescript-eslint/no-unsafe-return */ import { - CollectionMongoSource, - MongoSource, parseToBigInt, } from '@alien-worlds/api-core'; import { Long } from 'mongodb'; import { BlockRangeScanDocument } from './block-range-scanner.dtos'; -import { BlockNumberOutOfRangeError } from './block-range-scanner.errors'; +import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; /** * Block range scan nodes data source from the mongo database * @class */ -export class BlockRangeScanMongoSource extends CollectionMongoSource { +export class BlockRangeScanMongoSource extends MongoCollectionSource { public static Token = 'BLOCK_RANGE_SCAN_MONGO_SOURCE'; /** diff --git a/src/reader/block-range-scanner/block-range-scan.repository.ts b/src/reader/block-range-scanner/block-range-scan.repository.ts index d51ba98..9b8c464 100644 --- a/src/reader/block-range-scanner/block-range-scan.repository.ts +++ b/src/reader/block-range-scanner/block-range-scan.repository.ts @@ -35,7 +35,7 @@ export class BlockRangeScanRepository { childRanges.forEach(range => rangesToPersist.push(range)); const documents = rangesToPersist.map(range => range.toDocument()); - await this.source.insertMany(documents); + await this.source.insert(documents); return {}; } catch (error) { diff --git a/src/reader/block-range-scanner/block-range-scanner.ts b/src/reader/block-range-scanner/block-range-scanner.ts index c19c16a..7ed663c 100644 --- a/src/reader/block-range-scanner/block-range-scanner.ts +++ b/src/reader/block-range-scanner/block-range-scanner.ts @@ -1,9 +1,10 @@ -import { MongoConfig, MongoSource, log } from '@alien-worlds/api-core'; +import { log } from '@alien-worlds/api-core'; import { BlockRangeScan } from './block-range-scan'; import { BlockRangeScanRepository, ScanRequest } from './block-range-scan.repository'; import { DuplicateBlockRangeScanError } from './block-range-scanner.errors'; import { BlockRangeScanConfig } from './block-range-scanner.config'; import { BlockRangeScanMongoSource } from './block-range-scan.mongo.source'; +import { MongoConfig, MongoSource } from '@alien-worlds/storage-mongodb'; /** * @class diff --git a/src/reader/reader.ts b/src/reader/reader.ts index 2d53d8f..cd277b0 100644 --- a/src/reader/reader.ts +++ b/src/reader/reader.ts @@ -1,4 +1,4 @@ -import { Broadcast, BroadcastClient, log, MongoSource } from '@alien-worlds/api-core'; +import { log } from '@alien-worlds/api-core'; import { BlockRangeScanner } from './block-range-scanner'; import { Mode } from '../common/common.enums'; import { WorkerMessage, WorkerPool } from '../common/workers'; @@ -6,6 +6,7 @@ import ReaderWorker from './reader.worker'; import { ReadCompleteData, ReaderConfig, ReadTaskData } from './reader.types'; import { InternalBroadcastClientName } from '../broadcast'; import { FilterBroadcastMessage } from '../broadcast/messages'; +import { MongoSource } from '@alien-worlds/storage-mongodb'; export class Reader { public static async create( diff --git a/src/reader/reader.types.ts b/src/reader/reader.types.ts index 2d97aae..84b3a97 100644 --- a/src/reader/reader.types.ts +++ b/src/reader/reader.types.ts @@ -1,7 +1,7 @@ -import { BroadcastConfig, MongoConfig } from '@alien-worlds/api-core'; import { BlockRangeScanConfig } from './block-range-scanner'; import { WorkersConfig } from '../common/workers'; import { BlockReaderConfig } from '../common/blockchain/block-reader'; +import { MongoConfig } from '@alien-worlds/storage-mongodb'; export type ReaderConfig = { broadcast?: BroadcastConfig; diff --git a/src/reader/reader.worker-loader.ts b/src/reader/reader.worker-loader.ts index e20d8dc..3755fb4 100644 --- a/src/reader/reader.worker-loader.ts +++ b/src/reader/reader.worker-loader.ts @@ -1,10 +1,11 @@ -import { MongoSource, log } from '@alien-worlds/api-core'; +import { log } from '@alien-worlds/api-core'; import { BlockReader } from '../common/blockchain'; import { Worker } from '../common/workers'; import { DefaultWorkerLoader } from '../common/workers/worker-loader'; import { UnprocessedBlockQueue } from './unprocessed-block-queue'; import ReaderWorker, { ReaderSharedData } from './reader.worker'; import { BlockRangeScanner, BlockState } from '../common'; +import { MongoSource } from '@alien-worlds/storage-mongodb'; export default class ReaderWorkerLoader extends DefaultWorkerLoader { private blockReader: BlockReader; diff --git a/src/reader/start-reader.ts b/src/reader/start-reader.ts index 36272f8..a82b84c 100644 --- a/src/reader/start-reader.ts +++ b/src/reader/start-reader.ts @@ -5,7 +5,7 @@ import { InternalBroadcastClientName, InternalBroadcastMessageName, } from '../broadcast/internal-broadcast.enums'; -import { Broadcast, log } from '@alien-worlds/api-core'; +import { log } from '@alien-worlds/api-core'; import { ReadTaskData, ReaderConfig } from './reader.types'; import { InternalBroadcastMessage } from '../broadcast/internal-broadcast.message'; import { ReaderBroadcastMessage } from '../broadcast/messages/reader-broadcast.message'; diff --git a/src/reader/unprocessed-block-queue/unprocessed-block-queue.ts b/src/reader/unprocessed-block-queue/unprocessed-block-queue.ts index 8c7aec2..313647f 100644 --- a/src/reader/unprocessed-block-queue/unprocessed-block-queue.ts +++ b/src/reader/unprocessed-block-queue/unprocessed-block-queue.ts @@ -1,12 +1,7 @@ import { - CollectionMongoSource, - DataSourceBulkWriteError, - DataSourceOperationError, + DataSourceError, Failure, - isMongoConfig, log, - MongoConfig, - MongoSource, parseToBigInt, Result, } from '@alien-worlds/api-core'; @@ -16,8 +11,14 @@ import { UnprocessedBlocksOverloadError, } from './unprocessed-block-queue.errors'; import { Block, BlockDocument } from '../../common/blockchain/block-reader/block'; +import { + isMongoConfig, + MongoCollectionSource, + MongoConfig, + MongoSource, +} from '@alien-worlds/storage-mongodb'; -export class BlockMongoCollection extends CollectionMongoSource { +export class BlockMongoCollection extends MongoCollectionSource { constructor(mongoSource: MongoSource) { super(mongoSource, 'history_tools.unprocessed_blocks', { indexes: [ @@ -35,7 +36,7 @@ export class BlockMongoCollection extends CollectionMongoSource { const result = await this.collection.findOneAndDelete({}); return result.value; } catch (error) { - throw DataSourceOperationError.fromError(error); + throw DataSourceError.createError(error); } } @@ -89,7 +90,7 @@ export class UnprocessedBlockQueue implements UnprocessedBlockQueueReader { const addedBlockNumbers = []; this.beforeSendBatchHandler(); const documnets = this.cache.map(block => block.toDocument()); - const result = await this.mongo.insertMany(documnets); + const result = await this.mongo.insert(documnets); result.forEach(document => { addedBlockNumbers.push(parseToBigInt(document.this_block.block_num)); }); @@ -137,8 +138,8 @@ export class UnprocessedBlockQueue implements UnprocessedBlockQueueReader { } catch (error) { // it is important to clear the cache in case of errors this.cache = []; - - if (error instanceof DataSourceBulkWriteError && error.onlyDuplicateErrors) { + + if (error instanceof DataSourceError && error.isDuplicateError) { this.afterSendBatchHandler(); return Result.withFailure(Failure.fromError(new DuplicateBlocksError())); } @@ -151,7 +152,7 @@ export class UnprocessedBlockQueue implements UnprocessedBlockQueueReader { const document = await this.mongo.next(); if (document) { if (this.maxBytesSize > -1 && this.afterSendBatchHandler) { - if ((await this.mongo.count({})) === 0 && this.afterSendBatchHandler) { + if ((await this.mongo.count()) === 0 && this.afterSendBatchHandler) { this.afterSendBatchHandler(); } } diff --git a/yarn.lock b/yarn.lock index cb8112f..4f03cfe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11,21 +11,60 @@ debug "^4.3.4" safe-buffer "~5.1.2" -"@alien-worlds/api-core@^0.0.101": - version "0.0.101" - resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.101/c67f2c0741a464be6d0391b98691ca9c2ac74eb9#c67f2c0741a464be6d0391b98691ca9c2ac74eb9" - integrity sha512-9hgizhXjYL+PsxsYnrwwL3KRHVvuiVaPjovM2w6U06PTbLXYDkalHQ+em15OQ29DkOd8WGoM1cWW+LTLx6Ec+w== +"@alien-worlds/api-core@^0.0.125": + version "0.0.125" + resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.125/386abf1c615f114905ec478516acd3fba08bbbac#386abf1c615f114905ec478516acd3fba08bbbac" + integrity sha512-dYrAIqIgpGXA/i+AU9sMGS/CslizxKDsnuhSrD86VxI8iP1OR1+D+6T4D+5YUXmO05Iw2a166UNwd/Xyrq3b5Q== dependencies: - "@google-cloud/bigquery" "^6.0.3" - amqplib "^0.10.3" eosjs "^22.1.0" inversify "^6.0.1" - mongodb "4.9.1" nanoid "^3.0.0" node-fetch "2" - redis "^4.6.5" reflect-metadata "^0.1.13" +"@alien-worlds/api-core@^0.0.127": + version "0.0.127" + resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.127/50d818f0b9130fd555394c688c2a4ed9f2d6f3fb#50d818f0b9130fd555394c688c2a4ed9f2d6f3fb" + integrity sha512-x56Ei5PpGNbX8bZj5Vtg2vhUSrN3g7OavAzD9AaglFpKguG0AlLhHZa6vroAiUa4FFBhm5Np7evQBg011iq13g== + dependencies: + eosjs "^22.1.0" + inversify "^6.0.1" + nanoid "^3.0.0" + node-fetch "2" + reflect-metadata "^0.1.13" + +"@alien-worlds/block-reader@^0.0.3": + version "0.0.3" + resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.3/3cea505637f816621d007cf4f47df0f0ab534ae5#3cea505637f816621d007cf4f47df0f0ab534ae5" + integrity sha512-swDwYq+jv8JiZg5V98oW2vv8R1T/SpidNwDBQYKJk13XeYHkbBroomjL1+gw+lnz5tC1XUPMQp7HOxR3zqR9SQ== + dependencies: + "@alien-worlds/api-core" "^0.0.125" + +"@alien-worlds/broadcast@^0.0.3": + version "0.0.3" + resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.3/3da5e6bd4a43fc6729a8881d6df97bd5f05b83e5#3da5e6bd4a43fc6729a8881d6df97bd5f05b83e5" + integrity sha512-a3kSUP85hRHTDVSwUUxy8omFaSYsu5HHF9U9N8WYvcq9wH9u34IXrs/ZbJ2ugArohSjfwV+whtJ3rip7umR2XA== + dependencies: + "@alien-worlds/api-core" "^0.0.125" + nanoid "^3.0.0" + node-fetch "2" + +"@alien-worlds/storage-mongodb@^0.0.7": + version "0.0.7" + resolved "https://npm.pkg.github.com/download/@alien-worlds/storage-mongodb/0.0.7/a39d63bd60a5bfa61f54884989d7eca66ea15e11#a39d63bd60a5bfa61f54884989d7eca66ea15e11" + integrity sha512-/BpTVKaxAhOUsZ9ixdeS33K58gkC+sxOlh75LNETLWAad2nCE2v/okL/MXu9X2HgowQNnxWsC0EJAZiXgKDSYQ== + dependencies: + "@alien-worlds/api-core" "^0.0.127" + mongodb "4.9.1" + +"@alien-worlds/workers@^0.0.1": + version "0.0.1" + resolved "https://npm.pkg.github.com/download/@alien-worlds/workers/0.0.1/31dd1c5b2ac7a0545e56c49c3bbae0edd06ae746#31dd1c5b2ac7a0545e56c49c3bbae0edd06ae746" + integrity sha512-AOccIgg+Ujl3aKLDm76cHc2VIcEEHI1QRAwsP4XlwTGwJH97du7jZkIkLwOV1OAMm4Rfil3EowYbtmNq5BlgoQ== + dependencies: + async "^3.2.4" + ts-node "^10.9.1" + "@ampproject/remapping@^2.1.0": version "2.2.0" resolved "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz" @@ -343,57 +382,6 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@google-cloud/bigquery@^6.0.3": - version "6.0.3" - resolved "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-6.0.3.tgz" - integrity sha512-BP464228S9dqDCb4dR99h9D8+N498YZi/AZvoOJUaieg2H6qbiYBE1xlYuaMvyV1WEQT/2/yZTCJnCo5WiaY0Q== - dependencies: - "@google-cloud/common" "^4.0.0" - "@google-cloud/paginator" "^4.0.0" - "@google-cloud/promisify" "^3.0.0" - arrify "^2.0.1" - big.js "^6.0.0" - duplexify "^4.0.0" - extend "^3.0.2" - is "^3.3.0" - p-event "^4.1.0" - readable-stream "^4.0.0" - stream-events "^1.0.5" - uuid "^8.0.0" - -"@google-cloud/common@^4.0.0": - version "4.0.3" - resolved "https://registry.npmjs.org/@google-cloud/common/-/common-4.0.3.tgz" - integrity sha512-fUoMo5b8iAKbrYpneIRV3z95AlxVJPrjpevxs4SKoclngWZvTXBSGpNisF5+x5m+oNGve7jfB1e6vNBZBUs7Fw== - dependencies: - "@google-cloud/projectify" "^3.0.0" - "@google-cloud/promisify" "^3.0.0" - arrify "^2.0.1" - duplexify "^4.1.1" - ent "^2.2.0" - extend "^3.0.2" - google-auth-library "^8.0.2" - retry-request "^5.0.0" - teeny-request "^8.0.0" - -"@google-cloud/paginator@^4.0.0": - version "4.0.1" - resolved "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-4.0.1.tgz" - integrity sha512-6G1ui6bWhNyHjmbYwavdN7mpVPRBtyDg/bfqBTAlwr413On2TnFNfDxc9UhTJctkgoCDgQXEKiRPLPR9USlkbQ== - dependencies: - arrify "^2.0.0" - extend "^3.0.2" - -"@google-cloud/projectify@^3.0.0": - version "3.0.0" - resolved "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-3.0.0.tgz" - integrity sha512-HRkZsNmjScY6Li8/kb70wjGlDDyLkVk3KvoEo9uIoxSjYLJasGiCch9+PqRVDOCGUFvEIqyogl+BeqILL4OJHA== - -"@google-cloud/promisify@^3.0.0": - version "3.0.1" - resolved "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-3.0.1.tgz" - integrity sha512-z1CjRjtQyBOYL+5Qr9DdYIfrdLBe746jRTYfaYU6MeXkqp7UfYs/jX16lFFVzZ7PGEJvqZNqYUEtb1mvDww4pA== - "@humanwhocodes/config-array@^0.11.6": version "0.11.6" resolved "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.6.tgz" @@ -667,40 +655,6 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" -"@redis/bloom@1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" - integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== - -"@redis/client@1.5.6": - version "1.5.6" - resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.6.tgz#869cc65718d7d5493ef655a71dc40f3bc64a1b28" - integrity sha512-dFD1S6je+A47Lj22jN/upVU2fj4huR7S9APd7/ziUXsIXDL+11GPYti4Suv5y8FuXaN+0ZG4JF+y1houEJ7ToA== - dependencies: - cluster-key-slot "1.1.2" - generic-pool "3.9.0" - yallist "4.0.0" - -"@redis/graph@1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.0.tgz#cc2b82e5141a29ada2cce7d267a6b74baa6dd519" - integrity sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg== - -"@redis/json@1.0.4": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.4.tgz#f372b5f93324e6ffb7f16aadcbcb4e5c3d39bda1" - integrity sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw== - -"@redis/search@1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.2.tgz#6a8f66ba90812d39c2457420f859ce8fbd8f3838" - integrity sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA== - -"@redis/time-series@1.0.4": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.4.tgz#af85eb080f6934580e4d3b58046026b6c2b18717" - integrity sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng== - "@sinonjs/commons@^1.7.0": version "1.8.3" resolved "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz" @@ -720,11 +674,6 @@ resolved "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz" integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== -"@tootallnate/once@2": - version "2.0.0" - resolved "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz" - integrity sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A== - "@tsconfig/node10@^1.0.7": version "1.0.9" resolved "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz" @@ -904,12 +853,12 @@ "@types/webidl-conversions@*": version "7.0.0" - resolved "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz" + resolved "https://registry.yarnpkg.com/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz#2b8e60e33906459219aa587e9d1a612ae994cfe7" integrity sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog== "@types/whatwg-url@^8.2.1": version "8.2.2" - resolved "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.2.tgz" + resolved "https://registry.yarnpkg.com/@types/whatwg-url/-/whatwg-url-8.2.2.tgz#749d5b3873e845897ada99be4448041d4cc39e63" integrity sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA== dependencies: "@types/node" "*" @@ -1014,13 +963,6 @@ abab@^2.0.3, abab@^2.0.5: resolved "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz" integrity sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA== -abort-controller@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz" - integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== - dependencies: - event-target-shim "^5.0.0" - accepts@~1.3.8: version "1.3.8" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e" @@ -1155,11 +1097,6 @@ array-union@^2.1.0: resolved "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz" integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== -arrify@^2.0.0, arrify@^2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz" - integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== - async@^3.2.4: version "3.2.4" resolved "https://registry.npmjs.org/async/-/async-3.2.4.tgz" @@ -1236,21 +1173,11 @@ balanced-match@^1.0.0: resolved "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== -base64-js@^1.3.0, base64-js@^1.3.1: +base64-js@^1.3.1: version "1.5.1" - resolved "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== -big.js@^6.0.0: - version "6.2.1" - resolved "https://registry.npmjs.org/big.js/-/big.js-6.2.1.tgz" - integrity sha512-bCtHMwL9LeDIozFn+oNhhFoq+yQ3BNdnsLSASUxLciOb1vgvpHsIO1dsENiGMgbb4SkP5TrzWzRiLddn8ahVOQ== - -bignumber.js@^9.0.0: - version "9.1.0" - resolved "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.0.tgz" - integrity sha512-4LwHK4nfDOraBCtst+wOWIHbu1vhvAPJK8g8nROd4iuc3PSEjWif/qwbkh8jwCJz6yDBvtU4KPynETgrfh7y3A== - bn.js@5.2.0: version "5.2.0" resolved "https://registry.npmjs.org/bn.js/-/bn.js-5.2.0.tgz" @@ -1329,17 +1256,12 @@ bser@2.1.1: node-int64 "^0.4.0" bson@^4.7.0: - version "4.7.0" - resolved "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz" - integrity sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA== + version "4.7.2" + resolved "https://registry.yarnpkg.com/bson/-/bson-4.7.2.tgz#320f4ad0eaf5312dd9b45dc369cc48945e2a5f2e" + integrity sha512-Ry9wCtIZ5kGqkJoi6aD8KjxFZEx78guTQDnpXWiNthsxzrxAK/i8E6pCHAIZTbaEFWcOCvbecMukfK7XUvyLpQ== dependencies: buffer "^5.6.0" -buffer-equal-constant-time@1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz" - integrity sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA== - buffer-from@^1.0.0: version "1.1.2" resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz" @@ -1352,20 +1274,12 @@ buffer-more-ints@~1.0.0: buffer@^5.6.0: version "5.7.1" - resolved "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== dependencies: base64-js "^1.3.1" ieee754 "^1.1.13" -buffer@^6.0.3: - version "6.0.3" - resolved "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz" - integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA== - dependencies: - base64-js "^1.3.1" - ieee754 "^1.2.1" - bytes@3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" @@ -1440,11 +1354,6 @@ cliui@^7.0.2: strip-ansi "^6.0.0" wrap-ansi "^7.0.0" -cluster-key-slot@1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" - integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== - co@^4.6.0: version "4.6.0" resolved "https://registry.npmjs.org/co/-/co-4.6.0.tgz" @@ -1486,6 +1395,11 @@ combined-stream@^1.0.8: dependencies: delayed-stream "~1.0.0" +commander@^10.0.1: + version "10.0.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.1.tgz#881ee46b4f77d1c1dccc5823433aa39b022cbe06" + integrity sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug== + concat-map@0.0.1: version "0.0.1" resolved "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" @@ -1609,7 +1523,7 @@ delayed-stream@~1.0.0: denque@^2.1.0: version "2.1.0" - resolved "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== depd@2.0.0: @@ -1658,23 +1572,6 @@ domexception@^2.0.1: dependencies: webidl-conversions "^5.0.0" -duplexify@^4.0.0, duplexify@^4.1.1: - version "4.1.2" - resolved "https://registry.npmjs.org/duplexify/-/duplexify-4.1.2.tgz" - integrity sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw== - dependencies: - end-of-stream "^1.4.1" - inherits "^2.0.3" - readable-stream "^3.1.1" - stream-shift "^1.0.0" - -ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: - version "1.0.11" - resolved "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz" - integrity sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ== - dependencies: - safe-buffer "^5.0.1" - ee-first@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" @@ -1713,18 +1610,6 @@ encodeurl@~1.0.2: resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w== -end-of-stream@^1.4.1: - version "1.4.4" - resolved "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz" - integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== - dependencies: - once "^1.4.0" - -ent@^2.2.0: - version "2.2.0" - resolved "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz" - integrity sha512-GHrMyVZQWvTIdDtpiEXdHZnFQKzeO09apj8Cbl4pKWy4i0Oprcq17usfDt5aO63swf0JOeMWjWQE/LzgSRuWpA== - eosjs@^22.1.0: version "22.1.0" resolved "https://registry.npmjs.org/eosjs/-/eosjs-22.1.0.tgz" @@ -1917,16 +1802,6 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== -event-target-shim@^5.0.0: - version "5.0.1" - resolved "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz" - integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== - -events@^3.3.0: - version "3.3.0" - resolved "https://registry.npmjs.org/events/-/events-3.3.0.tgz" - integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== - execa@^5.0.0: version "5.1.1" resolved "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz" @@ -1994,11 +1869,6 @@ express@^4.18.2: utils-merge "1.0.1" vary "~1.1.2" -extend@^3.0.2: - version "3.0.2" - resolved "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz" - integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== - fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: version "3.1.3" resolved "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz" @@ -2030,11 +1900,6 @@ fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.6: resolved "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz" integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== -fast-text-encoding@^1.0.0: - version "1.0.6" - resolved "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.6.tgz" - integrity sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w== - fastq@^1.6.0: version "1.13.0" resolved "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz" @@ -2139,29 +2004,6 @@ function-bind@^1.1.1: resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz" integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== -gaxios@^5.0.0, gaxios@^5.0.1: - version "5.0.2" - resolved "https://registry.npmjs.org/gaxios/-/gaxios-5.0.2.tgz" - integrity sha512-TjtV2AJOZoMQqRYoy5eM8cCQogYwazWNYLQ72QB0kwa6vHHruYkGmhhyrlzbmgNHK1dNnuP2WSH81urfzyN2Og== - dependencies: - extend "^3.0.2" - https-proxy-agent "^5.0.0" - is-stream "^2.0.0" - node-fetch "^2.6.7" - -gcp-metadata@^5.0.0: - version "5.0.1" - resolved "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.0.1.tgz" - integrity sha512-jiRJ+Fk7e8FH68Z6TLaqwea307OktJpDjmYnU7/li6ziwvVvU2RlrCyQo5vkdeP94chm0kcSCOOszvmuaioq3g== - dependencies: - gaxios "^5.0.0" - json-bigint "^1.0.0" - -generic-pool@3.9.0: - version "3.9.0" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" - integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g== - gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz" @@ -2241,28 +2083,6 @@ globby@^11.1.0: merge2 "^1.4.1" slash "^3.0.0" -google-auth-library@^8.0.2: - version "8.7.0" - resolved "https://registry.npmjs.org/google-auth-library/-/google-auth-library-8.7.0.tgz" - integrity sha512-1M0NG5VDIvJZEnstHbRdckLZESoJwguinwN8Dhae0j2ZKIQFIV63zxm6Fo6nM4xkgqUr2bbMtV5Dgo+Hy6oo0Q== - dependencies: - arrify "^2.0.0" - base64-js "^1.3.0" - ecdsa-sig-formatter "^1.0.11" - fast-text-encoding "^1.0.0" - gaxios "^5.0.0" - gcp-metadata "^5.0.0" - gtoken "^6.1.0" - jws "^4.0.0" - lru-cache "^6.0.0" - -google-p12-pem@^4.0.0: - version "4.0.1" - resolved "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-4.0.1.tgz" - integrity sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ== - dependencies: - node-forge "^1.3.1" - graceful-fs@^4.2.9: version "4.2.10" resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz" @@ -2273,15 +2093,6 @@ grapheme-splitter@^1.0.4: resolved "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz" integrity sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ== -gtoken@^6.1.0: - version "6.1.2" - resolved "https://registry.npmjs.org/gtoken/-/gtoken-6.1.2.tgz" - integrity sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ== - dependencies: - gaxios "^5.0.1" - google-p12-pem "^4.0.0" - jws "^4.0.0" - has-flag@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz" @@ -2353,15 +2164,6 @@ http-proxy-agent@^4.0.1: agent-base "6" debug "4" -http-proxy-agent@^5.0.0: - version "5.0.0" - resolved "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz" - integrity sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w== - dependencies: - "@tootallnate/once" "2" - agent-base "6" - debug "4" - https-proxy-agent@^5.0.0: version "5.0.1" resolved "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz" @@ -2382,9 +2184,9 @@ iconv-lite@0.4.24: dependencies: safer-buffer ">= 2.1.2 < 3" -ieee754@^1.1.13, ieee754@^1.2.1: +ieee754@^1.1.13: version "1.2.1" - resolved "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== ignore@^5.2.0: @@ -2433,7 +2235,7 @@ inversify@^6.0.1: ip@^2.0.0: version "2.0.0" - resolved "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz" + resolved "https://registry.yarnpkg.com/ip/-/ip-2.0.0.tgz#4cf4ab182fee2314c75ede1276f8c80b479936da" integrity sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ== ipaddr.js@1.9.1: @@ -2500,11 +2302,6 @@ is-typedarray@^1.0.0: resolved "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz" integrity sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA== -is@^3.3.0: - version "3.3.0" - resolved "https://registry.npmjs.org/is/-/is-3.3.0.tgz" - integrity sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg== - isarray@0.0.1: version "0.0.1" resolved "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" @@ -3025,13 +2822,6 @@ jsesc@^2.5.1: resolved "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz" integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== -json-bigint@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz" - integrity sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ== - dependencies: - bignumber.js "^9.0.0" - json-parse-even-better-errors@^2.3.0: version "2.3.1" resolved "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz" @@ -3052,23 +2842,6 @@ json5@2.x, json5@^2.2.1: resolved "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz" integrity sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA== -jwa@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz" - integrity sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA== - dependencies: - buffer-equal-constant-time "1.0.1" - ecdsa-sig-formatter "1.0.11" - safe-buffer "^5.0.1" - -jws@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz" - integrity sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg== - dependencies: - jwa "^2.0.0" - safe-buffer "^5.0.1" - kleur@^3.0.3: version "3.0.3" resolved "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz" @@ -3162,7 +2935,7 @@ media-typer@0.3.0: memory-pager@^1.0.2: version "1.5.0" - resolved "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz" + resolved "https://registry.yarnpkg.com/memory-pager/-/memory-pager-1.5.0.tgz#d8751655d22d384682741c972f2c3d6dfa3e66b5" integrity sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg== merge-descriptors@1.0.1: @@ -3234,7 +3007,7 @@ minimatch@^3.0.4, minimatch@^3.1.1, minimatch@^3.1.2: mongodb-connection-string-url@^2.5.3: version "2.6.0" - resolved "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.6.0.tgz" + resolved "https://registry.yarnpkg.com/mongodb-connection-string-url/-/mongodb-connection-string-url-2.6.0.tgz#57901bf352372abdde812c81be47b75c6b2ec5cf" integrity sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ== dependencies: "@types/whatwg-url" "^8.2.1" @@ -3242,7 +3015,7 @@ mongodb-connection-string-url@^2.5.3: mongodb@4.9.1: version "4.9.1" - resolved "https://registry.npmjs.org/mongodb/-/mongodb-4.9.1.tgz" + resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-4.9.1.tgz#0c769448228bcf9a6aa7d16daa3625b48312479e" integrity sha512-ZhgI/qBf84fD7sI4waZBoLBNJYPQN5IOC++SBCiPiyhzpNKOxN/fi0tBHvH2dEC42HXtNEbFB0zmNz4+oVtorQ== dependencies: bson "^4.7.0" @@ -3287,7 +3060,7 @@ node-addon-api@2.0.0: resolved "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.0.tgz" integrity sha512-ASCL5U13as7HhOExbT6OlWJJUV/lLzL2voOSP1UVehpRD8FbSrSDjfScK/KwAvVTI5AS6r4VwbOMlIqtvRidnA== -node-fetch@2, node-fetch@^2.6.1, node-fetch@^2.6.7: +node-fetch@2: version "2.6.7" resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== @@ -3301,11 +3074,6 @@ node-fetch@2.6.6: dependencies: whatwg-url "^5.0.0" -node-forge@^1.3.1: - version "1.3.1" - resolved "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz" - integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA== - node-int64@^0.4.0: version "0.4.0" resolved "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz" @@ -3345,7 +3113,7 @@ on-finished@2.4.1: dependencies: ee-first "1.1.1" -once@^1.3.0, once@^1.4.0: +once@^1.3.0: version "1.4.0" resolved "https://registry.npmjs.org/once/-/once-1.4.0.tgz" integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== @@ -3383,18 +3151,6 @@ optionator@^0.9.1: type-check "^0.4.0" word-wrap "^1.2.3" -p-event@^4.1.0: - version "4.2.0" - resolved "https://registry.npmjs.org/p-event/-/p-event-4.2.0.tgz" - integrity sha512-KXatOjCRXXkSePPb1Nbi0p0m+gQAwdlbhi4wQKJPI1HsMQS9g+Sqp2o+QHziPr7eYJyOZet836KoHEVM1mwOrQ== - dependencies: - p-timeout "^3.1.0" - -p-finally@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz" - integrity sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow== - p-limit@^2.2.0: version "2.3.0" resolved "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz" @@ -3423,13 +3179,6 @@ p-locate@^5.0.0: dependencies: p-limit "^3.0.2" -p-timeout@^3.1.0: - version "3.2.0" - resolved "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz" - integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg== - dependencies: - p-finally "^1.0.0" - p-try@^2.0.0: version "2.2.0" resolved "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz" @@ -3550,11 +3299,6 @@ pretty-format@^27.0.0, pretty-format@^27.5.1: ansi-styles "^5.0.0" react-is "^17.0.1" -process@^0.11.10: - version "0.11.10" - resolved "https://registry.npmjs.org/process/-/process-0.11.10.tgz" - integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A== - prompts@^2.0.1: version "2.4.2" resolved "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz" @@ -3628,37 +3372,6 @@ react-is@^17.0.1: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@^3.1.1: - version "3.6.0" - resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz" - integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== - dependencies: - inherits "^2.0.3" - string_decoder "^1.1.1" - util-deprecate "^1.0.1" - -readable-stream@^4.0.0: - version "4.2.0" - resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-4.2.0.tgz" - integrity sha512-gJrBHsaI3lgBoGMW/jHZsQ/o/TIWiu5ENCJG1BB7fuCKzpFM8GaS2UoBVt9NO+oI+3FcrBNbUkl3ilDe09aY4A== - dependencies: - abort-controller "^3.0.0" - buffer "^6.0.3" - events "^3.3.0" - process "^0.11.10" - -redis@^4.6.5: - version "4.6.5" - resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.5.tgz#f32fbde44429e96f562bb0c9b1db0143ab8cfa4f" - integrity sha512-O0OWA36gDQbswOdUuAhRL6mTZpHFN525HlgZgDaVNgCJIAZR3ya06NTESb0R+TUZ+BFaDpz6NnnVvoMx9meUFg== - dependencies: - "@redis/bloom" "1.2.0" - "@redis/client" "1.5.6" - "@redis/graph" "1.1.0" - "@redis/json" "1.0.4" - "@redis/search" "1.1.2" - "@redis/time-series" "1.0.4" - reflect-metadata@^0.1.13: version "0.1.13" resolved "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz" @@ -3710,14 +3423,6 @@ resolve@^1.20.0: path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" -retry-request@^5.0.0: - version "5.0.2" - resolved "https://registry.npmjs.org/retry-request/-/retry-request-5.0.2.tgz" - integrity sha512-wfI3pk7EE80lCIXprqh7ym48IHYdwmAAzESdbU8Q9l7pnRCk9LEhpbOTNKjz6FARLm/Bl5m+4F0ABxOkYUujSQ== - dependencies: - debug "^4.1.1" - extend "^3.0.2" - reusify@^1.0.4: version "1.0.4" resolved "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz" @@ -3737,7 +3442,7 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" -safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@~5.2.0: +safe-buffer@5.2.1: version "5.2.1" resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== @@ -3754,7 +3459,7 @@ safe-buffer@~5.1.2: saslprep@^1.0.3: version "1.0.3" - resolved "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz" + resolved "https://registry.yarnpkg.com/saslprep/-/saslprep-1.0.3.tgz#4c02f946b56cf54297e347ba1093e7acac4cf226" integrity sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag== dependencies: sparse-bitfield "^3.0.3" @@ -3850,12 +3555,12 @@ slash@^3.0.0: smart-buffer@^4.2.0: version "4.2.0" - resolved "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz" + resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae" integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== socks@^2.7.0: version "2.7.1" - resolved "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz" + resolved "https://registry.yarnpkg.com/socks/-/socks-2.7.1.tgz#d8e651247178fde79c0663043e07240196857d55" integrity sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ== dependencies: ip "^2.0.0" @@ -3881,7 +3586,7 @@ source-map@^0.7.3: sparse-bitfield@^3.0.3: version "3.0.3" - resolved "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz" + resolved "https://registry.yarnpkg.com/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz#ff4ae6e68656056ba4b3e792ab3334d38273ca11" integrity sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ== dependencies: memory-pager "^1.0.2" @@ -3903,18 +3608,6 @@ statuses@2.0.1: resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== -stream-events@^1.0.5: - version "1.0.5" - resolved "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz" - integrity sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg== - dependencies: - stubs "^3.0.0" - -stream-shift@^1.0.0: - version "1.0.1" - resolved "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz" - integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== - string-length@^4.0.1: version "4.0.2" resolved "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz" @@ -3932,13 +3625,6 @@ string-width@^4.1.0, string-width@^4.2.0: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" -string_decoder@^1.1.1: - version "1.3.0" - resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz" - integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== - dependencies: - safe-buffer "~5.2.0" - string_decoder@~0.10.x: version "0.10.31" resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" @@ -3966,11 +3652,6 @@ strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz" integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== -stubs@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz" - integrity sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw== - supports-color@^5.3.0: version "5.5.0" resolved "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz" @@ -4010,17 +3691,6 @@ symbol-tree@^3.2.4: resolved "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz" integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== -teeny-request@^8.0.0: - version "8.0.2" - resolved "https://registry.npmjs.org/teeny-request/-/teeny-request-8.0.2.tgz" - integrity sha512-34pe0a4zASseXZCKdeTiIZqSKA8ETHb1EwItZr01PAR3CLPojeAKgSjzeNS4373gi59hNulyDrPKEbh2zO9sCg== - dependencies: - http-proxy-agent "^5.0.0" - https-proxy-agent "^5.0.0" - node-fetch "^2.6.1" - stream-events "^1.0.5" - uuid "^9.0.0" - terminal-link@^2.0.0: version "2.1.1" resolved "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz" @@ -4094,7 +3764,7 @@ tr46@^2.1.0: tr46@^3.0.0: version "3.0.0" - resolved "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-3.0.0.tgz#555c4e297a950617e8eeddef633c87d4d9d6cbf9" integrity sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA== dependencies: punycode "^2.1.1" @@ -4231,26 +3901,11 @@ url-parse@^1.5.3, url-parse@~1.5.10: querystringify "^2.1.1" requires-port "^1.0.0" -util-deprecate@^1.0.1: - version "1.0.2" - resolved "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" - integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== - utils-merge@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== -uuid@^8.0.0: - version "8.3.2" - resolved "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz" - integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== - -uuid@^9.0.0: - version "9.0.0" - resolved "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz" - integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg== - v8-compile-cache-lib@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz" @@ -4308,7 +3963,7 @@ webidl-conversions@^6.1.0: webidl-conversions@^7.0.0: version "7.0.0" - resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-7.0.0.tgz#256b4e1882be7debbf01d05f0aa2039778ea080a" integrity sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g== whatwg-encoding@^1.0.5: @@ -4325,7 +3980,7 @@ whatwg-mimetype@^2.3.0: whatwg-url@^11.0.0: version "11.0.0" - resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-11.0.0.tgz#0a849eebb5faf2119b901bb76fd795c2848d4018" integrity sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ== dependencies: tr46 "^3.0.0" @@ -4409,7 +4064,7 @@ y18n@^5.0.5: resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== -yallist@4.0.0, yallist@^4.0.0: +yallist@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== From fda758f9e052454560483577a24ca632517b3516 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Wed, 21 Jun 2023 22:55:33 +0200 Subject: [PATCH 002/107] init refactor processor, filter, reader, commons --- package.json | 10 +- .../__tests__/bootstrap.utils.unit.test.ts | 51 +-- src/bootstrap/bootstrap.dependencies.ts | 56 +++ src/bootstrap/bootstrap.types.ts | 9 +- src/bootstrap/bootstrap.utils.ts | 41 ++- src/bootstrap/index.ts | 4 +- src/bootstrap/start-bootstrap.ts | 89 ++--- .../abis/__tests__/abis.service.unit.test.ts | 62 ---- .../contract-encoded-abi.unit.test.ts | 6 +- src/common/abis/abis.collection.ts | 4 +- src/common/abis/abis.creator.ts | 46 +++ src/common/abis/abis.eos.serializer.ts | 144 -------- src/common/abis/abis.mongo.mapper.ts | 24 ++ src/common/abis/abis.serializer.ts | 37 -- src/common/abis/abis.service.ts | 46 --- src/common/abis/abis.ts | 30 +- src/common/abis/abis.types.ts | 4 +- src/common/abis/contract-encoded-abi.ts | 142 ++++---- .../block-range-scan.mongo.source.ts | 0 .../block-range-scan.repository.ts | 0 .../block-range-scanner/block-range-scan.ts | 0 .../block-range-scanner.config.ts | 0 .../block-range-scanner.creator.ts | 32 ++ .../block-range-scanner.dtos.ts | 0 .../block-range-scanner.errors.ts | 0 .../block-range-scanner.ts | 25 -- .../block-range-scanner/index.ts | 0 .../__tests__/block-state.unit.test.ts | 2 +- .../block-state/block-state.collection.ts | 18 +- src/common/block-state/block-state.creator.ts | 31 ++ .../block-state/block-state.mongo.mapper.ts | 27 ++ src/common/block-state/block-state.ts | 20 +- src/common/block-state/block-state.types.ts | 28 +- src/common/block-state/index.ts | 4 +- .../contract-content/trace/trace.dtos.ts | 26 +- .../contract-reader/contract-reader.config.ts | 3 - .../contract-reader/contract-reader.dtos.ts | 17 - src/common/contract-reader/contract-reader.ts | 96 ----- .../featured-contract.source.ts | 30 -- .../contract-reader/featured-contract.ts | 49 --- src/common/contract-reader/index.ts | 5 - src/common/dependencies.ts | 16 + ...eatured-contract.mongo.mapper.unit.test.ts | 45 +++ .../__tests__/featured.mapper.unit.test.ts | 222 ++++++++++++ .../featured/__tests__/featured.unit.test.ts | 113 ------ .../__tests__/featured.utils.unit.test.ts | 85 ++++- .../featured/featured-contract.collection.ts | 26 ++ .../featured-contract.mongo.mapper.ts | 48 +++ src/common/featured/featured-contract.ts | 34 ++ src/common/featured/featured.actions.ts | 23 ++ src/common/featured/featured.config.ts | 4 + src/common/featured/featured.creator.ts | 36 ++ src/common/featured/featured.deltas.ts | 25 ++ src/common/featured/featured.enums.ts | 4 - src/common/featured/featured.mapper.ts | 253 +++++++++++++ src/common/featured/featured.ts | 333 ++++-------------- src/common/featured/featured.types.ts | 96 ++--- src/common/featured/featured.utils.ts | 44 ++- src/common/featured/index.ts | 11 +- src/common/index.ts | 10 +- .../data-sources/processor-task.source.ts | 0 .../unsuccessful-processor-task.source.ts | 0 .../processor-task-queue/index.ts | 0 .../processor-task-queue.config.ts | 0 .../processor-task-queue.ts | 2 +- .../processor-task-queue/processor-task.ts | 11 +- .../processor-task.types.ts | 0 src/common/unprocessed-block-queue/index.ts | 7 + .../unprocessed-block-queue.collection.ts | 7 + .../unprocessed-block-queue.errors.ts | 0 .../unprocessed-block-queue.mapper.ts | 7 + ...nprocessed-block-queue.mongo.collection.ts | 35 ++ .../unprocessed-block-queue.mongo.mapper.ts | 99 ++++++ .../unprocessed-block-queue.ts | 106 ++---- .../unprocessed-block-queue.types.ts | 36 ++ src/config/config.types.ts | 5 + src/config/index.ts | 42 +-- src/filter/deserialized-block.ts | 72 ---- src/filter/filter.command.ts | 10 + src/filter/filter.consts.ts | 1 + src/filter/filter.dependencies.ts | 24 ++ src/filter/filter.runner.ts | 29 +- src/filter/filter.types.ts | 143 +++++++- src/filter/filter.utils.ts | 44 --- .../filter.worker-loader.dependencies.ts | 22 ++ src/filter/filter.worker-loader.ts | 64 +--- src/filter/filter.worker.ts | 215 ++++++----- src/filter/index.ts | 8 +- src/filter/start-filter.ts | 58 +-- src/processor/index.ts | 15 +- src/processor/processor.command.ts | 8 + src/processor/processor.consts.ts | 3 +- src/processor/processor.dependencies.ts | 21 ++ src/processor/processor.errors.ts | 5 - src/processor/processor.runner.ts | 2 +- src/processor/processor.types.ts | 37 +- .../processor.worker-loader.dependencies.ts | 18 + src/processor/processor.worker-loader.ts | 35 +- .../action-trace.processor.input.ts | 51 --- .../processors/action-trace.processor.ts | 63 +++- .../processors/delta.processor.input.ts | 65 ---- src/processor/processors/delta.processor.ts | 52 ++- src/processor/processors/index.ts | 2 - src/processor/processors/processor.ts | 4 +- src/processor/start-processor.ts | 46 ++- src/reader/index.ts | 10 +- src/reader/reader.command.ts | 12 + src/reader/reader.consts.ts | 2 + src/reader/reader.dependencies.ts | 19 + src/reader/reader.ts | 63 ++-- src/reader/reader.types.ts | 7 +- .../reader.worker-loader.dependencies.ts | 21 ++ src/reader/reader.worker-loader.ts | 67 ++-- src/reader/reader.worker.ts | 30 +- src/reader/start-reader.ts | 49 +-- src/reader/unprocessed-block-queue/index.ts | 2 - yarn.lock | 105 ++---- 117 files changed, 2440 insertions(+), 2067 deletions(-) create mode 100644 src/bootstrap/bootstrap.dependencies.ts delete mode 100644 src/common/abis/__tests__/abis.service.unit.test.ts create mode 100644 src/common/abis/abis.creator.ts delete mode 100644 src/common/abis/abis.eos.serializer.ts create mode 100644 src/common/abis/abis.mongo.mapper.ts delete mode 100644 src/common/abis/abis.serializer.ts delete mode 100644 src/common/abis/abis.service.ts rename src/{reader => common}/block-range-scanner/block-range-scan.mongo.source.ts (100%) rename src/{reader => common}/block-range-scanner/block-range-scan.repository.ts (100%) rename src/{reader => common}/block-range-scanner/block-range-scan.ts (100%) rename src/{reader => common}/block-range-scanner/block-range-scanner.config.ts (100%) create mode 100644 src/common/block-range-scanner/block-range-scanner.creator.ts rename src/{reader => common}/block-range-scanner/block-range-scanner.dtos.ts (100%) rename src/{reader => common}/block-range-scanner/block-range-scanner.errors.ts (100%) rename src/{reader => common}/block-range-scanner/block-range-scanner.ts (59%) rename src/{reader => common}/block-range-scanner/index.ts (100%) create mode 100644 src/common/block-state/block-state.creator.ts create mode 100644 src/common/block-state/block-state.mongo.mapper.ts delete mode 100644 src/common/contract-reader/contract-reader.config.ts delete mode 100644 src/common/contract-reader/contract-reader.dtos.ts delete mode 100644 src/common/contract-reader/contract-reader.ts delete mode 100644 src/common/contract-reader/featured-contract.source.ts delete mode 100644 src/common/contract-reader/featured-contract.ts delete mode 100644 src/common/contract-reader/index.ts create mode 100644 src/common/dependencies.ts create mode 100644 src/common/featured/__tests__/featured-contract.mongo.mapper.unit.test.ts create mode 100644 src/common/featured/__tests__/featured.mapper.unit.test.ts delete mode 100644 src/common/featured/__tests__/featured.unit.test.ts create mode 100644 src/common/featured/featured-contract.collection.ts create mode 100644 src/common/featured/featured-contract.mongo.mapper.ts create mode 100644 src/common/featured/featured-contract.ts create mode 100644 src/common/featured/featured.actions.ts create mode 100644 src/common/featured/featured.config.ts create mode 100644 src/common/featured/featured.creator.ts create mode 100644 src/common/featured/featured.deltas.ts delete mode 100644 src/common/featured/featured.enums.ts create mode 100644 src/common/featured/featured.mapper.ts rename src/{processor => common}/processor-task-queue/data-sources/processor-task.source.ts (100%) rename src/{processor => common}/processor-task-queue/data-sources/unsuccessful-processor-task.source.ts (100%) rename src/{processor => common}/processor-task-queue/index.ts (100%) rename src/{processor => common}/processor-task-queue/processor-task-queue.config.ts (100%) rename src/{processor => common}/processor-task-queue/processor-task-queue.ts (97%) rename src/{processor => common}/processor-task-queue/processor-task.ts (94%) rename src/{processor => common}/processor-task-queue/processor-task.types.ts (100%) create mode 100644 src/common/unprocessed-block-queue/index.ts create mode 100644 src/common/unprocessed-block-queue/unprocessed-block-queue.collection.ts rename src/{reader => common}/unprocessed-block-queue/unprocessed-block-queue.errors.ts (100%) create mode 100644 src/common/unprocessed-block-queue/unprocessed-block-queue.mapper.ts create mode 100644 src/common/unprocessed-block-queue/unprocessed-block-queue.mongo.collection.ts create mode 100644 src/common/unprocessed-block-queue/unprocessed-block-queue.mongo.mapper.ts rename src/{reader => common}/unprocessed-block-queue/unprocessed-block-queue.ts (56%) create mode 100644 src/common/unprocessed-block-queue/unprocessed-block-queue.types.ts delete mode 100644 src/filter/deserialized-block.ts create mode 100644 src/filter/filter.command.ts create mode 100644 src/filter/filter.dependencies.ts delete mode 100644 src/filter/filter.utils.ts create mode 100644 src/filter/filter.worker-loader.dependencies.ts create mode 100644 src/processor/processor.command.ts create mode 100644 src/processor/processor.dependencies.ts delete mode 100644 src/processor/processor.errors.ts create mode 100644 src/processor/processor.worker-loader.dependencies.ts delete mode 100644 src/processor/processors/action-trace.processor.input.ts delete mode 100644 src/processor/processors/delta.processor.input.ts create mode 100644 src/reader/reader.command.ts create mode 100644 src/reader/reader.consts.ts create mode 100644 src/reader/reader.dependencies.ts create mode 100644 src/reader/reader.worker-loader.dependencies.ts delete mode 100644 src/reader/unprocessed-block-queue/index.ts diff --git a/package.json b/package.json index 5216267..cf07b82 100644 --- a/package.json +++ b/package.json @@ -35,17 +35,15 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/api-core": "^0.0.127", + "@alien-worlds/api-core": "^0.0.144", "@alien-worlds/block-reader": "^0.0.3", - "@alien-worlds/broadcast": "^0.0.3", - "@alien-worlds/storage-mongodb": "^0.0.7", - "@alien-worlds/workers": "^0.0.1", + "@alien-worlds/broadcast": "^0.0.4", + "@alien-worlds/storage-mongodb": "^0.0.14", + "@alien-worlds/workers": "^0.0.3", "@eosrio/node-abieos": "^1", - "amqplib": "^0.10.3", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", - "eosjs": "^22.1.0", "express": "^4.18.2", "nanoid": "^3.0.0", "node-fetch": "2.6.6", diff --git a/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts b/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts index e399a88..d113d01 100644 --- a/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts +++ b/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts @@ -10,16 +10,7 @@ import { UndefinedStartBlockError, } from '../bootstrap.errors'; import { Mode } from '../../common/common.enums'; -import { Blockchain } from '../../common'; - -jest.mock('../../common', () => ({ - Blockchain: { - create: jest.fn().mockResolvedValue({ - getLastIrreversibleBlockNumber: jest.fn().mockResolvedValue(100n), - getHeadBlockNumber: jest.fn().mockResolvedValue(100n), - }), - }, -})); +import { Result } from '@alien-worlds/api-core'; describe('createDefaultModeBlockRange', () => { const originalLog = console.log; @@ -36,6 +27,12 @@ describe('createDefaultModeBlockRange', () => { const blockState = { getBlockNumber: jest.fn().mockResolvedValue({ content: 0n }), } as any; + const blockchain = { + getLastIrreversibleBlockNumber: jest + .fn() + .mockResolvedValue(Result.withContent(100n)), + getHeadBlockNumber: jest.fn().mockResolvedValue(Result.withContent(100n)), + } as any; const config = { blockchain: {}, startBlock: 5n, @@ -46,7 +43,7 @@ describe('createDefaultModeBlockRange', () => { maxBlockNumber: 10n, } as any; - const result = await createDefaultModeBlockRange(blockState, config); + const result = await createDefaultModeBlockRange(blockState, blockchain, config); expect(result).toEqual({ startBlock: 5n, @@ -60,6 +57,12 @@ describe('createDefaultModeBlockRange', () => { const blockState = { getBlockNumber: jest.fn().mockResolvedValue({ content: 0n }), } as any; + const blockchain = { + getLastIrreversibleBlockNumber: jest + .fn() + .mockResolvedValue(Result.withContent(100n)), + getHeadBlockNumber: jest.fn().mockResolvedValue(Result.withContent(100n)), + } as any; const config = { blockchain: {}, startBlock: 5n, @@ -70,14 +73,18 @@ describe('createDefaultModeBlockRange', () => { maxBlockNumber: 10n, } as any; - await expect(createDefaultModeBlockRange(blockState, config)).rejects.toThrow( - StartBlockHigherThanEndBlockError - ); + await expect( + createDefaultModeBlockRange(blockState, blockchain, config) + ).rejects.toThrow(StartBlockHigherThanEndBlockError); }); }); describe('createTestModeBlockRange', () => { const originalLog = console.log; + const blockchain = { + getLastIrreversibleBlockNumber: jest.fn().mockResolvedValue(Result.withContent(100n)), + getHeadBlockNumber: jest.fn().mockResolvedValue(Result.withContent(100n)), + } as any; beforeEach(() => { jest.clearAllMocks(); console.log = jest.fn(); @@ -96,7 +103,7 @@ describe('createTestModeBlockRange', () => { startFromHead: true, } as any; - const result = await createTestModeBlockRange(config); + const result = await createTestModeBlockRange(blockchain, config); expect(result).toEqual({ startBlock: 99n, @@ -104,7 +111,6 @@ describe('createTestModeBlockRange', () => { mode: Mode.Test, scanKey: 'scanKey', }); - expect(Blockchain.create).toHaveBeenCalledWith(config.blockchain); }); it('should create a block range in test mode when startBlock is a bigint', async () => { @@ -116,7 +122,7 @@ describe('createTestModeBlockRange', () => { startFromHead: true, } as any; - const result = await createTestModeBlockRange(config); + const result = await createTestModeBlockRange(blockchain, config); expect(result).toEqual({ startBlock: 50n, @@ -124,12 +130,15 @@ describe('createTestModeBlockRange', () => { mode: Mode.Test, scanKey: 'scanKey', }); - expect(Blockchain.create).toHaveBeenCalledWith(config.blockchain); }); }); describe('createReplayModeBlockRange', () => { const originalLog = console.log; + const blockchain = { + getLastIrreversibleBlockNumber: jest.fn().mockResolvedValue(Result.withContent(100n)), + getHeadBlockNumber: jest.fn().mockResolvedValue(Result.withContent(100n)), + } as any; beforeEach(() => { jest.clearAllMocks(); console.log = jest.fn(); // mock console.log to prevent log outputs during tests @@ -149,7 +158,7 @@ describe('createReplayModeBlockRange', () => { scanner: { scanKey: 'scanKey' }, } as any; - await expect(createReplayModeBlockRange(scanner, config)).rejects.toThrow( + await expect(createReplayModeBlockRange(scanner, blockchain, config)).rejects.toThrow( UndefinedStartBlockError ); }); @@ -164,7 +173,7 @@ describe('createReplayModeBlockRange', () => { scanner: { scanKey: 'scanKey' }, } as any; - await expect(createReplayModeBlockRange(scanner, config)).rejects.toThrow( + await expect(createReplayModeBlockRange(scanner, blockchain, config)).rejects.toThrow( EndBlockOutOfRangeError ); }); @@ -179,7 +188,7 @@ describe('createReplayModeBlockRange', () => { scanner: { scanKey: 'scanKey' }, } as any; - await expect(createReplayModeBlockRange(scanner, config)).rejects.toThrow( + await expect(createReplayModeBlockRange(scanner, blockchain, config)).rejects.toThrow( StartBlockHigherThanEndBlockError ); }); diff --git a/src/bootstrap/bootstrap.dependencies.ts b/src/bootstrap/bootstrap.dependencies.ts new file mode 100644 index 0000000..82fdb95 --- /dev/null +++ b/src/bootstrap/bootstrap.dependencies.ts @@ -0,0 +1,56 @@ +import { BroadcastClient } from '@alien-worlds/broadcast'; +import { Abis, BlockRangeScanner, BlockState, Featured } from '../common'; +import { BlockchainService, Result } from '@alien-worlds/api-core'; +import { Dependencies } from '../common/dependencies'; +import { BootstrapConfig } from './bootstrap.types'; + +/** + * An abstract class representing a Bootstrap dependencies. + * @export + * @abstract + * @class BootstrapDependencies + */ +export abstract class BootstrapDependencies extends Dependencies { + /** + * The broadcast client used for communication. + * @type {BroadcastClient} + */ + public broadcastClient: BroadcastClient; + /** + * The ABIs (Application Binary Interfaces) for contracts. + * @type {Abis} + */ + public abis: Abis; + /** + * The block range scanner for scanning blocks. + * @type {BlockRangeScanner} + */ + public scanner: BlockRangeScanner; + /** + * The featured contract service. + * @type {Featured} + */ + public featured: Featured; + /** + * The block state for maintaining blockchain state. + * @type {BlockState} + */ + public blockState: BlockState; + + /** + * The blockchain service for interacting with the blockchain. + * @type {BlockchainService} + */ + public blockchain: BlockchainService; + + /** + * The blockchain service for interacting with the blockchain. + * @type {BlockchainService} + */ + public featuredContracts: BlockchainService; + + public abstract initialize( + config: BootstrapConfig, + featuredContracts: string[] + ): Promise; +} diff --git a/src/bootstrap/bootstrap.types.ts b/src/bootstrap/bootstrap.types.ts index 8053458..3ebb4b0 100644 --- a/src/bootstrap/bootstrap.types.ts +++ b/src/bootstrap/bootstrap.types.ts @@ -1,23 +1,20 @@ import { AbisServiceConfig } from '../common/abis'; -import { BlockRangeScanConfig } from '../reader/block-range-scanner'; -import { BlockchainConfig, ContractReaderConfig } from '../common/blockchain'; +import { BlockRangeScanConfig } from '../common/block-range-scanner'; import { FeaturedConfig } from '../common/featured'; import { Mode } from '../common'; -import { MongoConfig } from '@alien-worlds/storage-mongodb'; import { BroadcastConfig } from '@alien-worlds/broadcast'; +import { BlockchainConfig } from '../config'; export type BootstrapConfig = { broadcast: BroadcastConfig; - blockchain: BlockchainConfig; - contractReader: ContractReaderConfig; scanner: BlockRangeScanConfig; - mongo: MongoConfig; startBlock?: bigint; endBlock?: bigint; startFromHead?: boolean; mode: string; featured: FeaturedConfig; abis: AbisServiceConfig; + blockchain: BlockchainConfig; maxBlockNumber?: number; }; diff --git a/src/bootstrap/bootstrap.utils.ts b/src/bootstrap/bootstrap.utils.ts index 401424e..f411434 100644 --- a/src/bootstrap/bootstrap.utils.ts +++ b/src/bootstrap/bootstrap.utils.ts @@ -1,5 +1,5 @@ -import { log, parseToBigInt } from '@alien-worlds/api-core'; -import { BlockRangeScanner } from '../reader/block-range-scanner'; +import { BlockchainService, log, parseToBigInt } from '@alien-worlds/api-core'; +import { BlockRangeScanner } from '../common/block-range-scanner'; import { BlockState } from '../common/block-state'; import { Mode } from '../common/common.enums'; import { UnknownModeError } from '../common/common.errors'; @@ -9,7 +9,6 @@ import { UndefinedStartBlockError, EndBlockOutOfRangeError, } from './bootstrap.errors'; -import { Blockchain } from '../common'; /** * Creates a block range task input based on the provided configuration and mode. @@ -17,23 +16,25 @@ import { Blockchain } from '../common'; * @async * @param {BlockState} blockState - The current block state. * @param {BlockRangeScanner} scanner - The block range scanner. + * @param {BlockchainService} blockchain - The blockchain service. * @param {BootstrapConfig} config - The bootstrap configuration. * @returns {Promise} The block range task input. */ export const createBlockRangeTaskInput = async ( blockState: BlockState, scanner: BlockRangeScanner, + blockchain: BlockchainService, config: BootstrapConfig ) => { const { mode } = config; if (mode === Mode.Default) { - return createDefaultModeBlockRange(blockState, config); + return createDefaultModeBlockRange(blockState, blockchain, config); } else if (mode === Mode.Replay) { // - return createReplayModeBlockRange(scanner, config); + return createReplayModeBlockRange(scanner, blockchain, config); } else if (mode === Mode.Test) { // - return createTestModeBlockRange(config); + return createTestModeBlockRange(blockchain, config); } else { // throw new UnknownModeError(mode); @@ -42,14 +43,16 @@ export const createBlockRangeTaskInput = async ( /** * Creates a block range in default mode. - * + * * @async * @param {BlockState} blockState - The current block state. + * @param {BlockchainService} blockchain - The blockchain service. * @param {BootstrapConfig} config - The bootstrap configuration. * @returns {Promise} The block range data. */ export const createDefaultModeBlockRange = async ( blockState: BlockState, + blockchain: BlockchainService, config: BootstrapConfig ): Promise => { const { @@ -60,9 +63,9 @@ export const createDefaultModeBlockRange = async ( startFromHead, maxBlockNumber, } = config; - const blockchain = await Blockchain.create(config.blockchain); - const lastIrreversibleBlock = await blockchain.getLastIrreversibleBlockNumber(); - const headBlock = await blockchain.getHeadBlockNumber(); + const { content: lastIrreversibleBlock } = + await blockchain.getLastIrreversibleBlockNumber(); + const { content: headBlock } = await blockchain.getHeadBlockNumber(); const { content: currentBlockNumber } = await blockState.getBlockNumber(); log(` Current head block number: ${headBlock.toString()}`); @@ -121,12 +124,14 @@ export const createDefaultModeBlockRange = async ( /** * Creates a block range in test mode. - * + * * @async + * @param {BlockchainService} blockchain - The blockchain service. * @param {BootstrapConfig} config - The bootstrap configuration. * @returns {Promise} The block range data. */ export const createTestModeBlockRange = async ( + blockchain: BlockchainService, config: BootstrapConfig ): Promise => { const { @@ -136,9 +141,9 @@ export const createTestModeBlockRange = async ( startFromHead, } = config; - const blockchain = await Blockchain.create(config.blockchain); - const lastIrreversibleBlock = await blockchain.getLastIrreversibleBlockNumber(); - const headBlock = await blockchain.getHeadBlockNumber(); + const { content: lastIrreversibleBlock } = + await blockchain.getLastIrreversibleBlockNumber(); + const { content: headBlock } = await blockchain.getHeadBlockNumber(); let highEdge: bigint; let lowEdge: bigint; @@ -156,14 +161,16 @@ export const createTestModeBlockRange = async ( /** * Creates a block range in replay mode. - * + * * @async * @param {BlockRangeScanner} scanner - The block range scanner. + * @param {BlockchainService} blockchain - The blockchain service. * @param {BootstrapConfig} config - The bootstrap configuration. * @returns {Promise} The block range data. */ export const createReplayModeBlockRange = async ( scanner: BlockRangeScanner, + blockchain: BlockchainService, config: BootstrapConfig ): Promise => { const { @@ -176,8 +183,8 @@ export const createReplayModeBlockRange = async ( const lowEdge = startBlock; let highEdge = endBlock; - const blockchain = await Blockchain.create(config.blockchain); - const lastIrreversibleBlock = await blockchain.getLastIrreversibleBlockNumber(); + const { content: lastIrreversibleBlock } = + await blockchain.getLastIrreversibleBlockNumber(); if (typeof lowEdge !== 'bigint') { throw new UndefinedStartBlockError(); diff --git a/src/bootstrap/index.ts b/src/bootstrap/index.ts index 96c62e6..bb8d066 100644 --- a/src/bootstrap/index.ts +++ b/src/bootstrap/index.ts @@ -1,4 +1,6 @@ -export * from './bootstrap.types'; +export * from './bootstrap.command'; +export * from './bootstrap.dependencies'; export * from './bootstrap.errors'; +export * from './bootstrap.types'; export * from './bootstrap.utils'; export * from './start-bootstrap'; diff --git a/src/bootstrap/start-bootstrap.ts b/src/bootstrap/start-bootstrap.ts index f3eaeef..2e9af4a 100644 --- a/src/bootstrap/start-bootstrap.ts +++ b/src/bootstrap/start-bootstrap.ts @@ -11,55 +11,58 @@ import { BootstrapCommandOptions, BootstrapConfig } from './bootstrap.types'; import { NoAbisError } from './bootstrap.errors'; import { InternalBroadcastChannel, - InternalBroadcastClientName, InternalBroadcastMessageName, } from '../broadcast/internal-broadcast.enums'; -import { FeaturedConfig, FeaturedContractContent } from '../common/featured'; import { Mode } from '../common/common.enums'; -import { Abis, BlockRangeScanner, BlockState, ContractReader } from '../common'; -import { MongoSource } from '@alien-worlds/storage-mongodb'; -import { ConfigVars, log } from '@alien-worlds/api-core'; -import { BroadcastMessage, BroadcastTcpClient } from '@alien-worlds/broadcast'; +import { ConfigVars, UnknownObject, log } from '@alien-worlds/api-core'; +import { BroadcastMessage } from '@alien-worlds/broadcast'; import { buildBootstrapConfig } from '../config'; import { bootstrapCommand } from './bootstrap.command'; +import { FeaturedUtils } from '../common/featured/featured.utils'; +import { BootstrapDependencies } from './bootstrap.dependencies'; /** * The bootstrap function initiates the bootstrap process based on the configuration provided. - * Depending on the mode of operation (default, replay, or test), it prepares the necessary - * resources such as a broadcast client, mongo source, contract reader, ABI data, scanner, + * Depending on the mode of operation (default, replay, or test), it prepares the necessary + * resources such as a broadcast client, mongo source, contract reader, ABI data, scanner, * featured contract details, and block state. - * It also sets up a message handler for the bootstrap broadcast channel to handle various + * It also sets up a message handler for the bootstrap broadcast channel to handle various * messages related to the reader readiness in different modes. * * @param {BootstrapConfig} config The bootstrap configuration object. * @throws {NoAbisError} When no ABIs are fetched. - * + * * @returns {Promise} The Promise that resolves when the bootstrap process has been initiated successfully. */ -export const bootstrap = async (config: BootstrapConfig) => { +export const bootstrap = async ( + config: BootstrapConfig, + dependencies: BootstrapDependencies, + featuredJson: UnknownObject +): Promise => { const { mode } = config; log(`Bootstrap "${mode}" mode ... [starting]`); + const featuredContracts = FeaturedUtils.readFeaturedContracts(featuredJson); - const broadcast = await new BroadcastTcpClient( - config.broadcast, - InternalBroadcastClientName.Bootstrap - ); - const mongo = await MongoSource.create(config.mongo); - const contractReader = await ContractReader.create(config.contractReader, mongo); - const abis = await Abis.create(mongo, config.abis, config.featured); - const scanner = await BlockRangeScanner.create(mongo, config.scanner); - const featured = new FeaturedContractContent(config.featured); - const blockState = await BlockState.create(mongo); + await dependencies.initialize(config, featuredContracts); + + const { abis, broadcastClient, blockState, blockchain, featured, scanner } = + dependencies; let blockRange: ReaderBroadcastMessageData; // fetch latest abis to make sure that the blockchain data will be correctly deserialized log(` * Fetch featured contracts details ... [starting]`); - await contractReader.readContracts(featured.listContracts()); + await featured.readContracts(featuredContracts); log(` * Fetch featured contracts details ... [done]`); // fetch latest abis to make sure that the blockchain data will be correctly deserialized log(` * Fetch abis ... [starting]`); - const abisCount = (await abis.fetchAbis()).length; + const { content: fetchedAbis, failure: fetchAbisFailure } = await abis.fetchAbis(); + + if (fetchAbisFailure) { + throw fetchAbisFailure.error; + } + + const abisCount = fetchedAbis.length; log(` * Fetch abis ... [done]`); if (abisCount === 0) { @@ -67,48 +70,54 @@ export const bootstrap = async (config: BootstrapConfig) => { } if (config.mode === Mode.Replay) { - blockRange = await createReplayModeBlockRange(scanner, config); + blockRange = await createReplayModeBlockRange(scanner, blockchain, config); } - broadcast.onMessage( + broadcastClient.onMessage( InternalBroadcastChannel.Bootstrap, async (message: BroadcastMessage) => { if (message.name === InternalBroadcastMessageName.DefaultModeReaderReady) { if (config.mode === Mode.Default) { - blockRange = await createDefaultModeBlockRange(blockState, config); - broadcast.sendMessage(ReaderBroadcastMessage.newDefaultModeTask(blockRange)); + blockRange = await createDefaultModeBlockRange(blockState, blockchain, config); + broadcastClient.sendMessage( + ReaderBroadcastMessage.newDefaultModeTask(blockRange) + ); } if (config.mode === Mode.Test) { - blockRange = await createTestModeBlockRange(config); - broadcast.sendMessage(ReaderBroadcastMessage.newDefaultModeTask(blockRange)); + blockRange = await createTestModeBlockRange(blockchain, config); + broadcastClient.sendMessage( + ReaderBroadcastMessage.newDefaultModeTask(blockRange) + ); } } else if (message.name === InternalBroadcastMessageName.ReplayModeReaderReady) { - broadcast.sendMessage(ReaderBroadcastMessage.newReplayModeTask(blockRange)); + broadcastClient.sendMessage(ReaderBroadcastMessage.newReplayModeTask(blockRange)); } else { // } } ); - broadcast.connect(); + broadcastClient.connect(); log(`Bootstrap ${mode} mode ... [ready]`); }; /** - * startBootstrap is a function that takes command line options to build a bootstrap configuration + * startBootstrap is a function that takes command line options to build a bootstrap configuration * and then initiate the bootstrap process. It throws an error if the bootstrap process fails. * - * @param {BootstrapCommandOptions} options The command line options for bootstrap. - * @throws {Error} When the bootstrap process fails. - * - * @returns {Promise} The Promise that resolves when the bootstrap process has been initiated successfully. + * @param {string[]} args The command line args for bootstrap. + * @param {BootstrapDependencies} dependencies The bootstrap process dependencies. + * @param {UnknownObject} featuredJson */ -export const startBootstrap = (args: string[], featured: FeaturedConfig) => { +export const startBootstrap = ( + args: string[], + dependencies: BootstrapDependencies, + featuredJson: UnknownObject +) => { const vars = new ConfigVars(); const options = bootstrapCommand.parse(args).opts(); - const config = buildBootstrapConfig(vars, featured, options); - - bootstrap(config).catch(log); + const config = buildBootstrapConfig(vars, options); + bootstrap(config, dependencies, featuredJson).catch(log); }; diff --git a/src/common/abis/__tests__/abis.service.unit.test.ts b/src/common/abis/__tests__/abis.service.unit.test.ts deleted file mode 100644 index b37e518..0000000 --- a/src/common/abis/__tests__/abis.service.unit.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { AbisService } from '../abis.service'; -import fetch from 'node-fetch'; -import { ContractEncodedAbi } from '../contract-encoded-abi'; - -// Mock dependencies -jest.mock('node-fetch'); - -const mockFetch = fetch as jest.MockedFunction; - -describe('AbisService', () => { - let abisService: AbisService; - const config = { - url: 'https://example.com', - limit: 100, - filter: 'eosio:setabi', - }; - - beforeEach(() => { - // Reset mocks and create a new instance of AbisService - jest.resetAllMocks(); - abisService = new AbisService(config); - }); - - describe('fetchAbis', () => { - it('should fetch ABIs for a contract', async () => { - const contract = '0x123'; - const mockActions = [ - { block_num: 1, act: { data: { abi: 'ABI1' } } }, - { block_num: 2, act: { data: { abi: 'ABI2' } } }, - ]; - const expectedAbis = [ - ContractEncodedAbi.create(1, contract, 'ABI1'), - ContractEncodedAbi.create(2, contract, 'ABI2'), - ]; - - mockFetch.mockResolvedValue({ - json: jest.fn().mockResolvedValue({ actions: mockActions }), - } as Response); - - const result = await abisService.fetchAbis(contract); - - expect(mockFetch).toHaveBeenCalledTimes(1); - expect(mockFetch).toHaveBeenCalledWith( - `${config.url}/v2/history/get_actions?account=${contract}&filter=${config.filter}&limit=${config.limit}&sort=-1` - ); - expect(result).toEqual(expectedAbis); - }); - - it('should handle fetch errors and return an empty array', async () => { - const contract = '0x123'; - - mockFetch.mockRejectedValue(new Error('Fetch error')); - - const result = await abisService.fetchAbis(contract); - - expect(mockFetch).toHaveBeenCalledTimes(1); - expect(result).toEqual([]); - }); - - // Add more test cases for different scenarios - }); -}); diff --git a/src/common/abis/__tests__/contract-encoded-abi.unit.test.ts b/src/common/abis/__tests__/contract-encoded-abi.unit.test.ts index 1090891..e7a8942 100644 --- a/src/common/abis/__tests__/contract-encoded-abi.unit.test.ts +++ b/src/common/abis/__tests__/contract-encoded-abi.unit.test.ts @@ -1,6 +1,6 @@ import { ContractEncodedAbi } from '../contract-encoded-abi'; import { parseToBigInt } from '@alien-worlds/api-core'; -import { ContractEncodedAbiDocument, ContractEncodedAbiJson } from '../abis.types'; +import { ContractEncodedAbiMongoModel, ContractEncodedAbiJson } from '../abis.types'; import { MongoDB } from '@alien-worlds/storage-mongodb'; // Mock dependencies @@ -22,7 +22,7 @@ describe('ContractEncodedAbi', () => { describe('fromDocument', () => { it('should create a ContractEncodedAbi instance from a document', () => { - const document: ContractEncodedAbiDocument = { + const document: ContractEncodedAbiMongoModel = { block_number: mockMongoDBLong.fromBigInt(blockNumber), contract, hex, @@ -69,7 +69,7 @@ describe('ContractEncodedAbi', () => { describe('toDocument', () => { it('should convert the ContractEncodedAbi instance to a document', () => { const instance = new ContractEncodedAbi(blockNumber, contract, hex); - const expectedDocument: ContractEncodedAbiDocument = { + const expectedDocument: ContractEncodedAbiMongoModel = { block_number: mockMongoDBLong.fromBigInt(blockNumber), contract, hex, diff --git a/src/common/abis/abis.collection.ts b/src/common/abis/abis.collection.ts index 3f1c663..afff1bd 100644 --- a/src/common/abis/abis.collection.ts +++ b/src/common/abis/abis.collection.ts @@ -1,11 +1,11 @@ import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; -import { ContractEncodedAbiDocument } from './abis.types'; +import { ContractEncodedAbiMongoModel } from './abis.types'; /** * Represents a collection of ABIs (Application Binary Interfaces) stored in a MongoDB collection. * Extends the MongoCollectionSource class to provide database operations for the ABIs. */ -export class AbisCollection extends MongoCollectionSource { +export class AbisCollection extends MongoCollectionSource { /** * Constructs a new instance of the AbisCollection class. * diff --git a/src/common/abis/abis.creator.ts b/src/common/abis/abis.creator.ts new file mode 100644 index 0000000..cfa8b79 --- /dev/null +++ b/src/common/abis/abis.creator.ts @@ -0,0 +1,46 @@ +import { + MongoConfig, + MongoQueryBuilders, + MongoSource, +} from '@alien-worlds/storage-mongodb'; +import { AbisServiceConfig } from './abis.types'; +import { Abis } from './abis'; +import { AbiService, log } from '@alien-worlds/api-core'; +import { AbisCollection } from './abis.collection'; +import { AbisRepositoryImpl } from './abis.repository-impl'; +import { AbisMongoMapper } from './abis.mongo.mapper'; + +export class AbisCreator { + public static async create( + mongo: MongoSource | MongoConfig, + abiService: AbiService, + contracts?: string[], + setCache?: boolean + ): Promise { + let mongoSource: MongoSource; + + log(` * Abis ... [starting]`); + + if (mongo instanceof MongoSource) { + mongoSource = mongo; + } else { + mongoSource = await MongoSource.create(mongo); + } + const mapper = new AbisMongoMapper(); + const repository = new AbisRepositoryImpl( + new AbisCollection(mongoSource), + mapper, + new MongoQueryBuilders(mapper) + ); + const abis = new Abis(repository, abiService, contracts); + + if (setCache) { + await abis.cacheAbis(); + log(` * Abis cache restored`); + } + + log(` * Abis ... [ready]`); + + return abis; + } +} diff --git a/src/common/abis/abis.eos.serializer.ts b/src/common/abis/abis.eos.serializer.ts deleted file mode 100644 index 694a88f..0000000 --- a/src/common/abis/abis.eos.serializer.ts +++ /dev/null @@ -1,144 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { Serialize } from 'eosjs'; -import { Authorization, hexToUint8Array } from 'eosjs/dist/eosjs-serialize'; -import { Abi } from 'eosjs/dist/eosjs-rpc-interfaces'; -import { log } from '@alien-worlds/api-core'; -import { AbiTableJson } from '@alien-worlds/block-reader'; -import { AbisSerializer } from './abis.serializer'; - -export type SerializeUtil = { - deserializeAction: ( - account: string, - action: string, - data: Uint8Array, - hex: string - ) => T; - - deserializeTable: ( - account: string, - table: string, - data: Uint8Array, - hex: string - ) => T; -}; - -/** - * Represents a class for deserializing EOS ABIs. - * @template T - */ -export class AbisEosSerializer implements AbisSerializer { - /** - * Deserializes the action data for a specific account and action. - * - * @param {string} account - The account associated with the action. - * @param {string} action - The action name. - * @param {Uint8Array} data - The raw data to be deserialized. - * @param {string} hex - The hexadecimal representation of the data. - * @returns {T} The deserialized action data. - */ - public deserializeAction( - account: string, - action: string, - data: Uint8Array, - hex: string - ): T { - try { - const authorization: Authorization[] = []; - const textEncoder = new TextEncoder(); - const textDecoder = new TextDecoder(); - const bytes = hexToUint8Array(hex); - const abiTypes = Serialize.getTypesFromAbi(Serialize.createAbiTypes()); - const buffer = new Serialize.SerialBuffer({ - textEncoder, - textDecoder, - array: bytes, - }); - buffer.restartRead(); - const abi: Abi = abiTypes.get('abi_def').deserialize(buffer); - const types = Serialize.getTypesFromAbi(Serialize.createInitialTypes(), abi); - const actions = new Map(); - for (const { name, type } of abi.actions) { - actions.set(name, Serialize.getType(types, type)); - } - const contract = { types, actions }; - const deserializedAction = Serialize.deserializeAction( - contract, - account, - action, - authorization, - data, - new TextEncoder(), - new TextDecoder() - ); - - if (!deserializedAction.data) { - log( - `Serialized object is empty check the result of "Serialize.deserializeAction"` - ); - log(deserializedAction); - } - - return deserializedAction.data as T; - } catch (error) { - log(error); - return null; - } - } - - /** - * Deserializes the table data for a specific account and table. - * - * @param {string} account - The account associated with the table. - * @param {string} table - The table name. - * @param {Uint8Array} data - The raw data to be deserialized. - * @param {string} hex - The hexadecimal representation of the data. - * @returns {T} The deserialized table data. - */ - public deserializeTable( - account: string, - table: string, - data: Uint8Array, - hex: string - ): T { - try { - const textEncoder = new TextEncoder(); - const textDecoder = new TextDecoder(); - const bytes = hexToUint8Array(hex); - const abiTypes = Serialize.getTypesFromAbi(Serialize.createAbiTypes()); - const buffer = new Serialize.SerialBuffer({ - textEncoder, - textDecoder, - array: bytes, - }); - buffer.restartRead(); - const abi: Abi = abiTypes.get('abi_def').deserialize(buffer); - const types = Serialize.getTypesFromAbi(Serialize.createInitialTypes(), abi); - const actions = new Map(); - for (const { name, type } of abi.actions) { - actions.set(name, Serialize.getType(types, type)); - } - const contract = { types, actions }; - - let this_table: AbiTableJson, type: string; - for (const t of abi.tables) { - if (t.name === table) { - this_table = t; - break; - } - } - - if (this_table) { - type = this_table.type; - } else { - return null; - } - - const sb = new Serialize.SerialBuffer({ textEncoder, textDecoder, array: data }); - - return contract.types.get(type).deserialize(sb) as T; - } catch (e) { - return null; - } - } -} diff --git a/src/common/abis/abis.mongo.mapper.ts b/src/common/abis/abis.mongo.mapper.ts new file mode 100644 index 0000000..05ae32b --- /dev/null +++ b/src/common/abis/abis.mongo.mapper.ts @@ -0,0 +1,24 @@ +import { MongoDB, MongoMapper } from '@alien-worlds/storage-mongodb'; +import { ContractEncodedAbiMongoModel } from './abis.types'; +import { ContractEncodedAbi } from '@alien-worlds/api-core'; + +export class AbisMongoMapper extends MongoMapper< + ContractEncodedAbi, + ContractEncodedAbiMongoModel +> { + constructor() { + super(); + this.mappingFromEntity.set('blockNumber', { + key: 'block_number', + mapper: (value: bigint) => MongoDB.Long.fromBigInt(value), + }); + this.mappingFromEntity.set('contract', { + key: 'contract', + mapper: value => value, + }); + this.mappingFromEntity.set('hex', { + key: 'hex', + mapper: value => value, + }); + } +} diff --git a/src/common/abis/abis.serializer.ts b/src/common/abis/abis.serializer.ts deleted file mode 100644 index 53ea057..0000000 --- a/src/common/abis/abis.serializer.ts +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Represents an abstract class for deserializing ABIs (Application Binary Interfaces) related to actions and tables. - * @template T - */ -export abstract class AbisSerializer { - /** - * Deserializes the action data for a specific account and action. - * - * @param {string} account - The account associated with the action. - * @param {string} action - The action name. - * @param {Uint8Array} data - The raw data to be deserialized. - * @param {string} hex - The hexadecimal representation of the data. - * @returns {T} The deserialized action data. - */ - public abstract deserializeAction( - account: string, - action: string, - data: Uint8Array, - hex: string - ): T; - - /** - * Deserializes the table data for a specific account and table. - * - * @param {string} account - The account associated with the table. - * @param {string} table - The table name. - * @param {Uint8Array} data - The raw data to be deserialized. - * @param {string} hex - The hexadecimal representation of the data. - * @returns {T} The deserialized table data. - */ - public abstract deserializeTable( - account: string, - table: string, - data: Uint8Array, - hex: string - ): T; -} diff --git a/src/common/abis/abis.service.ts b/src/common/abis/abis.service.ts deleted file mode 100644 index 8333c22..0000000 --- a/src/common/abis/abis.service.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { AbisServiceConfig } from './abis.types'; -import fetch from 'node-fetch'; -import { ContractEncodedAbi } from './contract-encoded-abi'; - -/** - * Represents a service for fetching ABIs (Application Binary Interfaces). - */ -export class AbisService { - /** - * Constructs a new instance of the AbisService class. - * - * @param {AbisServiceConfig} config - The configuration for the AbisService. - */ - constructor(private config: AbisServiceConfig) {} - - /** - * Fetches the ABIs for a specific contract. - * - * @param {string} contract - The contract address. - * @returns {Promise} A promise that resolves to an array of ContractEncodedAbi objects representing the fetched ABIs. - */ - public async fetchAbis(contract: string): Promise { - try { - const list: ContractEncodedAbi[] = []; - const { url, limit, filter } = this.config; - - const res = await fetch( - `${url}/v2/history/get_actions?account=${contract}&filter=${ - filter || 'eosio:setabi' - }&limit=${limit || 100}&sort=-1` - ); - const json = await res.json(); - for (let i = 0; i < json.actions.length; i++) { - const act = json.actions[i]; - list.push( - ContractEncodedAbi.create(act.block_num, contract, String(act.act.data.abi)) - ); - } - return list; - } catch (error) { - return []; - } - } -} diff --git a/src/common/abis/abis.ts b/src/common/abis/abis.ts index eb67031..739fc51 100644 --- a/src/common/abis/abis.ts +++ b/src/common/abis/abis.ts @@ -1,9 +1,6 @@ -import { Failure, Result, log } from '@alien-worlds/api-core'; -import { FeaturedConfig } from '../featured'; -import { ContractEncodedAbi } from './contract-encoded-abi'; +import { AbiService, ContractEncodedAbi, Failure, Result, log } from '@alien-worlds/api-core'; import { AbisServiceNotSetError } from './abis.errors'; import { AbisRepository } from './abis.repository'; -import { AbisService } from './abis.service'; import { AbiNotFoundError } from '@alien-worlds/block-reader'; /** @@ -22,27 +19,12 @@ export class Abis { */ constructor( private repository: AbisRepository, - private service?: AbisService, - featuredConfig?: FeaturedConfig + private service?: AbiService, + contracts?: string[] ) { - if (featuredConfig) { - const { traces, deltas } = featuredConfig; - - traces.forEach(trace => { - const { contract } = trace; - contract.forEach(value => { - this.contracts.add(value); - }); - }); - - deltas.forEach(delta => { - const { code } = delta; - // apply if it is not a "match" object { match: "", processor:"" } - if (code) { - code.forEach(value => { - this.contracts.add(value); - }); - } + if (contracts) { + contracts.forEach(contract => { + this.contracts.add(contract); }); } } diff --git a/src/common/abis/abis.types.ts b/src/common/abis/abis.types.ts index 3b6894e..9046d46 100644 --- a/src/common/abis/abis.types.ts +++ b/src/common/abis/abis.types.ts @@ -1,5 +1,4 @@ import { MongoConfig, MongoDB } from '@alien-worlds/storage-mongodb'; -import { FeaturedConfig } from '../featured'; export type ContractEncodedAbiJson = { blockNumber: bigint; @@ -7,7 +6,7 @@ export type ContractEncodedAbiJson = { hex: string; }; -export type ContractEncodedAbiDocument = { +export type ContractEncodedAbiMongoModel = { block_number: MongoDB.Long; contract: string; hex: string; @@ -22,5 +21,4 @@ export type AbisServiceConfig = { export type AbisConfig = { service: AbisServiceConfig; mongo: MongoConfig; - featured: FeaturedConfig; }; diff --git a/src/common/abis/contract-encoded-abi.ts b/src/common/abis/contract-encoded-abi.ts index c09fe82..e185f0a 100644 --- a/src/common/abis/contract-encoded-abi.ts +++ b/src/common/abis/contract-encoded-abi.ts @@ -1,78 +1,78 @@ -import { parseToBigInt } from '@alien-worlds/api-core'; -import { ContractEncodedAbiDocument, ContractEncodedAbiJson } from './abis.types'; -import { MongoDB } from '@alien-worlds/storage-mongodb'; +// import { parseToBigInt } from '@alien-worlds/api-core'; +// import { ContractEncodedAbiMongoModel, ContractEncodedAbiJson } from './abis.types'; +// import { MongoDB } from '@alien-worlds/storage-mongodb'; -/** - * Represents an encoded ABI (Application Binary Interface) for a contract. - */ -export class ContractEncodedAbi { - /** - * Creates a ContractEncodedAbi instance from a document. - * - * @param {ContractEncodedAbiDocument} document - The document containing the encoded ABI data. - * @returns {ContractEncodedAbi} A new ContractEncodedAbi instance. - */ - public static fromDocument(document: ContractEncodedAbiDocument): ContractEncodedAbi { - const { block_number, contract, hex } = document; - return new ContractEncodedAbi(parseToBigInt(block_number), contract, hex); - } +// /** +// * Represents an encoded ABI (Application Binary Interface) for a contract. +// */ +// export class ContractEncodedAbi { +// /** +// * Creates a ContractEncodedAbi instance from a document. +// * +// * @param {ContractEncodedAbiMongoModel} document - The document containing the encoded ABI data. +// * @returns {ContractEncodedAbi} A new ContractEncodedAbi instance. +// */ +// public static fromDocument(document: ContractEncodedAbiMongoModel): ContractEncodedAbi { +// const { block_number, contract, hex } = document; +// return new ContractEncodedAbi(parseToBigInt(block_number), contract, hex); +// } - /** - * Creates a ContractEncodedAbi instance with the specified block number, contract address, and hex code. - * - * @param {unknown} blockNumber - The block number. - * @param {string} contract - The contract address. - * @param {string} hex - The hex code representing the ABI. - * @returns {ContractEncodedAbi} A new ContractEncodedAbi instance. - */ - public static create( - blockNumber: unknown, - contract: string, - hex: string - ): ContractEncodedAbi { - return new ContractEncodedAbi(parseToBigInt(blockNumber), contract, hex); - } +// /** +// * Creates a ContractEncodedAbi instance with the specified block number, contract address, and hex code. +// * +// * @param {unknown} blockNumber - The block number. +// * @param {string} contract - The contract address. +// * @param {string} hex - The hex code representing the ABI. +// * @returns {ContractEncodedAbi} A new ContractEncodedAbi instance. +// */ +// public static create( +// blockNumber: unknown, +// contract: string, +// hex: string +// ): ContractEncodedAbi { +// return new ContractEncodedAbi(parseToBigInt(blockNumber), contract, hex); +// } - /** - * Constructs a new instance of the ContractEncodedAbi class. - * - * @param {bigint} blockNumber - The block number. - * @param {string} contract - The contract address. - * @param {string} hex - The hex code representing the ABI. - */ - constructor( - public readonly blockNumber: bigint, - public readonly contract: string, - public readonly hex: string - ) {} +// /** +// * Constructs a new instance of the ContractEncodedAbi class. +// * +// * @param {bigint} blockNumber - The block number. +// * @param {string} contract - The contract address. +// * @param {string} hex - The hex code representing the ABI. +// */ +// constructor( +// public readonly blockNumber: bigint, +// public readonly contract: string, +// public readonly hex: string +// ) {} - /** - * Converts the ContractEncodedAbi instance to a document format. - * - * @returns {ContractEncodedAbiDocument} The ContractEncodedAbi instance as a document. - */ - public toDocument(): ContractEncodedAbiDocument { - const { blockNumber, hex, contract } = this; +// /** +// * Converts the ContractEncodedAbi instance to a document format. +// * +// * @returns {ContractEncodedAbiMongoModel} The ContractEncodedAbi instance as a document. +// */ +// public toDocument(): ContractEncodedAbiMongoModel { +// const { blockNumber, hex, contract } = this; - return { - block_number: MongoDB.Long.fromBigInt(blockNumber), - hex, - contract, - }; - } +// return { +// block_number: MongoDB.Long.fromBigInt(blockNumber), +// hex, +// contract, +// }; +// } - /** - * Converts the ContractEncodedAbi instance to a JSON format. - * - * @returns {ContractEncodedAbiJson} The ContractEncodedAbi instance as JSON. - */ - public toJson(): ContractEncodedAbiJson { - const { blockNumber, hex, contract } = this; +// /** +// * Converts the ContractEncodedAbi instance to a JSON format. +// * +// * @returns {ContractEncodedAbiJson} The ContractEncodedAbi instance as JSON. +// */ +// public toJson(): ContractEncodedAbiJson { +// const { blockNumber, hex, contract } = this; - return { - blockNumber, - hex, - contract, - }; - } -} +// return { +// blockNumber, +// hex, +// contract, +// }; +// } +// } diff --git a/src/reader/block-range-scanner/block-range-scan.mongo.source.ts b/src/common/block-range-scanner/block-range-scan.mongo.source.ts similarity index 100% rename from src/reader/block-range-scanner/block-range-scan.mongo.source.ts rename to src/common/block-range-scanner/block-range-scan.mongo.source.ts diff --git a/src/reader/block-range-scanner/block-range-scan.repository.ts b/src/common/block-range-scanner/block-range-scan.repository.ts similarity index 100% rename from src/reader/block-range-scanner/block-range-scan.repository.ts rename to src/common/block-range-scanner/block-range-scan.repository.ts diff --git a/src/reader/block-range-scanner/block-range-scan.ts b/src/common/block-range-scanner/block-range-scan.ts similarity index 100% rename from src/reader/block-range-scanner/block-range-scan.ts rename to src/common/block-range-scanner/block-range-scan.ts diff --git a/src/reader/block-range-scanner/block-range-scanner.config.ts b/src/common/block-range-scanner/block-range-scanner.config.ts similarity index 100% rename from src/reader/block-range-scanner/block-range-scanner.config.ts rename to src/common/block-range-scanner/block-range-scanner.config.ts diff --git a/src/common/block-range-scanner/block-range-scanner.creator.ts b/src/common/block-range-scanner/block-range-scanner.creator.ts new file mode 100644 index 0000000..d9403d5 --- /dev/null +++ b/src/common/block-range-scanner/block-range-scanner.creator.ts @@ -0,0 +1,32 @@ +import { log } from '@alien-worlds/api-core'; +import { BlockRangeScanRepository } from './block-range-scan.repository'; +import { BlockRangeScanConfig } from './block-range-scanner.config'; +import { BlockRangeScanMongoSource } from './block-range-scan.mongo.source'; +import { MongoConfig, MongoSource } from '@alien-worlds/storage-mongodb'; +import { BlockRangeScanner } from './block-range-scanner'; + +/** + * @class + */ +export class BlockRangeScannerCreator { + public static async create( + mongo: MongoSource | MongoConfig, + config: BlockRangeScanConfig + ): Promise { + let mongoSource: MongoSource; + + log(` * Block Range Scanner ... [starting]`); + + if (mongo instanceof MongoSource) { + mongoSource = mongo; + } else { + mongoSource = await MongoSource.create(mongo); + } + const source = new BlockRangeScanMongoSource(mongoSource); + const repository = new BlockRangeScanRepository(source, config); + const scanner: BlockRangeScanner = new BlockRangeScanner(repository); + + log(` * Block Range Scanner ... [ready]`); + return scanner; + } +} diff --git a/src/reader/block-range-scanner/block-range-scanner.dtos.ts b/src/common/block-range-scanner/block-range-scanner.dtos.ts similarity index 100% rename from src/reader/block-range-scanner/block-range-scanner.dtos.ts rename to src/common/block-range-scanner/block-range-scanner.dtos.ts diff --git a/src/reader/block-range-scanner/block-range-scanner.errors.ts b/src/common/block-range-scanner/block-range-scanner.errors.ts similarity index 100% rename from src/reader/block-range-scanner/block-range-scanner.errors.ts rename to src/common/block-range-scanner/block-range-scanner.errors.ts diff --git a/src/reader/block-range-scanner/block-range-scanner.ts b/src/common/block-range-scanner/block-range-scanner.ts similarity index 59% rename from src/reader/block-range-scanner/block-range-scanner.ts rename to src/common/block-range-scanner/block-range-scanner.ts index 7ed663c..1bbeaf6 100644 --- a/src/reader/block-range-scanner/block-range-scanner.ts +++ b/src/common/block-range-scanner/block-range-scanner.ts @@ -1,36 +1,11 @@ -import { log } from '@alien-worlds/api-core'; import { BlockRangeScan } from './block-range-scan'; import { BlockRangeScanRepository, ScanRequest } from './block-range-scan.repository'; import { DuplicateBlockRangeScanError } from './block-range-scanner.errors'; -import { BlockRangeScanConfig } from './block-range-scanner.config'; -import { BlockRangeScanMongoSource } from './block-range-scan.mongo.source'; -import { MongoConfig, MongoSource } from '@alien-worlds/storage-mongodb'; /** * @class */ export class BlockRangeScanner { - public static async create( - mongo: MongoSource | MongoConfig, - config: BlockRangeScanConfig - ): Promise { - let mongoSource: MongoSource; - - log(` * Block Range Scanner ... [starting]`); - - if (mongo instanceof MongoSource) { - mongoSource = mongo; - } else { - mongoSource = await MongoSource.create(mongo); - } - const source = new BlockRangeScanMongoSource(mongoSource); - const repository = new BlockRangeScanRepository(source, config); - const scanner: BlockRangeScanner = new BlockRangeScanner(repository); - - log(` * Block Range Scanner ... [ready]`); - return scanner; - } - constructor(private blockRangeScanRepository: BlockRangeScanRepository) {} public async createScanNodes( diff --git a/src/reader/block-range-scanner/index.ts b/src/common/block-range-scanner/index.ts similarity index 100% rename from src/reader/block-range-scanner/index.ts rename to src/common/block-range-scanner/index.ts diff --git a/src/common/block-state/__tests__/block-state.unit.test.ts b/src/common/block-state/__tests__/block-state.unit.test.ts index c28adff..738818a 100644 --- a/src/common/block-state/__tests__/block-state.unit.test.ts +++ b/src/common/block-state/__tests__/block-state.unit.test.ts @@ -7,7 +7,7 @@ import { RepositoryImpl, Failure, } from '@alien-worlds/api-core'; -import { BlockStateData, BlockStateDocument } from '../block-state.types'; +import { BlockStateModel, BlockStateMongoModel } from '../block-state.types'; import { BlockState } from '../block-state'; jest.mock('@alien-worlds/api-core'); diff --git a/src/common/block-state/block-state.collection.ts b/src/common/block-state/block-state.collection.ts index ce789b8..ca1e283 100644 --- a/src/common/block-state/block-state.collection.ts +++ b/src/common/block-state/block-state.collection.ts @@ -1,7 +1,19 @@ import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; -import { BlockStateDocument } from './block-state.types'; - -export class BlockStateCollection extends MongoCollectionSource { +import { BlockStateMongoModel } from './block-state.types'; +/** + * Represents a collection of BlockStateDocument objects in a MongoDB database. + * Extends the MongoCollectionSource class. + * + * @class + * @extends {MongoCollectionSource} + */ +export class BlockStateCollection extends MongoCollectionSource { + /** + * Creates a new instance of the BlockStateCollection class. + * + * @constructor + * @param {MongoSource} source - The MongoSource object used for database operations. + */ constructor(source: MongoSource) { super(source, 'history_tools.block_state'); } diff --git a/src/common/block-state/block-state.creator.ts b/src/common/block-state/block-state.creator.ts new file mode 100644 index 0000000..7126d60 --- /dev/null +++ b/src/common/block-state/block-state.creator.ts @@ -0,0 +1,31 @@ +import { log } from '@alien-worlds/api-core'; +import { + MongoConfig, + MongoQueryBuilders, + MongoSource, +} from '@alien-worlds/storage-mongodb'; +import { BlockState } from './block-state'; +import { UpdateBlockNumberMongoQueryBuilder } from './query-builders/update-block-number.mongo.query-builder'; +import { BlockMongoCollection } from '../../reader'; +import { BlockStateMongoMapper } from './block-state.mongo.mapper'; + +export class BlockStateCreator { + public static async create(mongo: MongoSource | MongoConfig) { + log(` * Block State ... [starting]`); + const mapper = new BlockStateMongoMapper(); + const queryBuilders = new MongoQueryBuilders(); + const updateBlockNumberQueryBuilder = new UpdateBlockNumberMongoQueryBuilder(); + const mongoSource = + mongo instanceof MongoSource ? mongo : await MongoSource.create(mongo); + const collection = new BlockMongoCollection(mongoSource); + const state = new BlockState( + collection, + mapper, + queryBuilders, + updateBlockNumberQueryBuilder + ); + + log(` * Block State ... [ready]`); + return state; + } +} diff --git a/src/common/block-state/block-state.mongo.mapper.ts b/src/common/block-state/block-state.mongo.mapper.ts new file mode 100644 index 0000000..f94d13d --- /dev/null +++ b/src/common/block-state/block-state.mongo.mapper.ts @@ -0,0 +1,27 @@ +import { BlockStateModel, BlockStateMongoModel } from './block-state.types'; +import { MongoDB, MongoMapper } from '@alien-worlds/storage-mongodb'; + +export class BlockStateMongoMapper extends MongoMapper< + BlockStateModel, + BlockStateMongoModel +> { + constructor() { + super(); + this.mappingFromEntity.set('lastModifiedTimestamp', { + key: 'last_modified_timestamp', + mapper: value => value, + }); + this.mappingFromEntity.set('blockNumber', { + key: 'block_number', + mapper: (value: bigint) => MongoDB.Long.fromBigInt(value), + }); + this.mappingFromEntity.set('actions', { + key: 'actions', + mapper: value => value, + }); + this.mappingFromEntity.set('tables', { + key: 'tables', + mapper: value => value, + }); + } +} diff --git a/src/common/block-state/block-state.ts b/src/common/block-state/block-state.ts index e1f9f4b..473b6a1 100644 --- a/src/common/block-state/block-state.ts +++ b/src/common/block-state/block-state.ts @@ -1,30 +1,30 @@ import { DataSource, Failure, - Mapper, QueryBuilder, QueryBuilders, RepositoryImpl, Result, } from '@alien-worlds/api-core'; -import { BlockStateData, BlockStateDocument } from './block-state.types'; +import { BlockStateModel, BlockStateMongoModel } from './block-state.types'; +import { BlockStateMongoMapper } from './block-state.mongo.mapper'; /** * A class representing a block state. - * @extends RepositoryImpl + * @extends RepositoryImpl */ -export class BlockState extends RepositoryImpl { +export class BlockState extends RepositoryImpl { /** * Creates an instance of the BlockState class. * - * @param {DataSource} source - The data source. - * @param {Mapper} mapper - The data mapper. + * @param {DataSource} source - The data source. + * @param {BlockStateMongoMapper} mapper - The data mapper. * @param {QueryBuilders} queryBuilders - The query builders. * @param {QueryBuilder} updateBlockNumberQueryBuilder - The query builder to update block number. */ constructor( - source: DataSource, - mapper: Mapper, + source: DataSource, + mapper: BlockStateMongoMapper, queryBuilders: QueryBuilders, private updateBlockNumberQueryBuilder: QueryBuilder ) { @@ -34,9 +34,9 @@ export class BlockState extends RepositoryImpl>} - The result of the operation. + * @returns {Promise>} - The result of the operation. */ - public async getState(): Promise> { + public async getState(): Promise> { try { const { content: states } = await this.find(); diff --git a/src/common/block-state/block-state.types.ts b/src/common/block-state/block-state.types.ts index 0a88569..4a9243b 100644 --- a/src/common/block-state/block-state.types.ts +++ b/src/common/block-state/block-state.types.ts @@ -1,16 +1,16 @@ -import { MongoDB } from "@alien-worlds/storage-mongodb"; +import { MongoDB } from '@alien-worlds/storage-mongodb'; -export type BlockStateDocument = { - _id: MongoDB.ObjectId; - last_modified_timestamp: Date; - block_number: MongoDB.Long; - actions: string[]; - tables: string[]; -} +export type BlockStateMongoModel = { + _id: MongoDB.ObjectId; + last_modified_timestamp: Date; + block_number: MongoDB.Long; + actions: string[]; + tables: string[]; +}; -export type BlockStateData = { - lastModifiedTimestamp: Date; - blockNumber: bigint; - actions: string[]; - tables: string[]; -} \ No newline at end of file +export type BlockStateModel = { + lastModifiedTimestamp: Date; + blockNumber: bigint; + actions: string[]; + tables: string[]; +}; diff --git a/src/common/block-state/index.ts b/src/common/block-state/index.ts index 8bb1413..9229482 100644 --- a/src/common/block-state/index.ts +++ b/src/common/block-state/index.ts @@ -1,2 +1,4 @@ export * from './block-state'; -export * from './block-state.source'; +export * from './block-state.collection'; +export * from './block-state.types'; +export * from './query-builders/update-block-number.mongo.query-builder'; diff --git a/src/common/contract-content/trace/trace.dtos.ts b/src/common/contract-content/trace/trace.dtos.ts index def5a22..883855e 100644 --- a/src/common/contract-content/trace/trace.dtos.ts +++ b/src/common/contract-content/trace/trace.dtos.ts @@ -15,19 +15,19 @@ export type PartialJson = { export type PartialByTypeJson = [string, PartialJson]; export type TraceJson = { - id: string; - status: number; - cpu_usage_us: number; - net_usage_words: number; - elapsed: string; - net_usage: string; - scheduled: boolean; - action_traces: ActionTraceByNameDto[]; - account_ram_delta: unknown; - except: unknown; - error_code: number | string; - failed_dtrx_trace: unknown; - partial: PartialByTypeJson; + id?: string; + status?: number; + cpu_usage_us?: number; + net_usage_words?: number; + elapsed?: string; + net_usage?: string; + scheduled?: boolean; + action_traces?: ActionTraceByNameDto[]; + account_ram_delta?: unknown; + except?: unknown; + error_code?: number | string; + failed_dtrx_trace?: unknown; + partial?: PartialByTypeJson; }; export type TraceByNameJson = [string, TraceJson]; diff --git a/src/common/contract-reader/contract-reader.config.ts b/src/common/contract-reader/contract-reader.config.ts deleted file mode 100644 index 3d52795..0000000 --- a/src/common/contract-reader/contract-reader.config.ts +++ /dev/null @@ -1,3 +0,0 @@ -export type ContractReaderConfig = { - url: string; -}; diff --git a/src/common/contract-reader/contract-reader.dtos.ts b/src/common/contract-reader/contract-reader.dtos.ts deleted file mode 100644 index c81f68a..0000000 --- a/src/common/contract-reader/contract-reader.dtos.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { MongoDB } from "@alien-worlds/storage-mongodb"; - -export type FeaturedContractDocument = { - _id?: MongoDB.ObjectId; - account?: string; - initial_block_number?: MongoDB.Long; -}; - -export type FeaturedContractModel = { - account: string; - initialBlockNumber: bigint; -}; - -export type FetchContractResponse = { - account: string; - block_num: string | number; -}; diff --git a/src/common/contract-reader/contract-reader.ts b/src/common/contract-reader/contract-reader.ts deleted file mode 100644 index c864df4..0000000 --- a/src/common/contract-reader/contract-reader.ts +++ /dev/null @@ -1,96 +0,0 @@ -import fetch from 'node-fetch'; -import { log } from '@alien-worlds/api-core'; -import { ContractReaderConfig } from './contract-reader.config'; -import { FetchContractResponse } from './contract-reader.dtos'; -import { FeaturedContract } from './featured-contract'; -import { FeaturedContractSource } from './featured-contract.source'; -import { MongoConfig, MongoSource } from '@alien-worlds/storage-mongodb'; - -export abstract class ContractReader { - public static async create( - config: ContractReaderConfig, - mongo: MongoSource | MongoConfig - ): Promise { - let mongoSource: MongoSource; - - log(` * Contract Reader ... [starting]`); - - if (mongo instanceof MongoSource) { - mongoSource = mongo; - } else { - mongoSource = await MongoSource.create(mongo); - } - const source = new FeaturedContractSource(mongoSource); - const contractReader = new ContractReaderService(source, config); - - log(` * Contract Reader ... [ready]`); - - return contractReader; - } - - public abstract getInitialBlockNumber(contract: string): Promise; - public abstract readContracts(contracts: string[]): Promise; -} - -export class ContractReaderService implements ContractReader { - private cache: Map = new Map(); - - constructor( - private source: FeaturedContractSource, - private config: ContractReaderConfig - ) {} - - private async fetchContract(account: string): Promise { - try { - const { url } = this.config; - - const res = await fetch( - `${url}/v2/history/get_actions?account=eosio&act.name=setabi&act.authorization.actor=${account}&limit=1&sort=asc` - ); - const json = await res.json(); - - const block_num = json.actions[0].block_num; - return { account, block_num }; - } catch (error) { - log(`An error occurred while retrieving contract data. ${error.message}`); - return null; - } - } - - public async getInitialBlockNumber(contract: string): Promise { - try { - const list = await this.readContracts([contract]); - return list[0].initialBlockNumber; - } catch (error) { - return -1n; - } - } - - public async readContracts(contracts: string[]): Promise { - const list: FeaturedContract[] = []; - for (const contract of contracts) { - let entity: FeaturedContract; - if (this.cache.has(contract)) { - list.push(this.cache.get(contract)); - } else { - const document = await this.source.findOne({ filter: { account: contract } }); - - if (document) { - entity = FeaturedContract.fromDocument(document); - this.cache.set(entity.account, entity); - list.push(entity); - } else { - const resp = await this.fetchContract(contract); - if (resp) { - entity = FeaturedContract.create(resp.account, resp.block_num); - this.cache.set(entity.account, entity); - this.source.insert([entity.toDocument()]); - list.push(entity); - } - } - } - } - - return list; - } -} diff --git a/src/common/contract-reader/featured-contract.source.ts b/src/common/contract-reader/featured-contract.source.ts deleted file mode 100644 index 6fd470d..0000000 --- a/src/common/contract-reader/featured-contract.source.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { MongoCollectionSource, MongoDB, MongoSource } from '@alien-worlds/storage-mongodb'; -import { FeaturedContractDocument } from './contract-reader.dtos'; - -export class FeaturedContractSource extends MongoCollectionSource { - constructor(mongoSource: MongoSource) { - super(mongoSource, 'history_tools.featured_contracts', { - indexes: [ - { key: { account: 1 }, background: true }, - { key: { initial_block_number: 1, account: 1 }, unique: true, background: true }, - ], - }); - } - - public async getInitialBlockNumber(account: string): Promise { - const contract: FeaturedContractDocument = await this.findOne({ - filter: { account }, - }); - return contract ? contract.initial_block_number : MongoDB.Long.MIN_VALUE; - } - - public async newState(account: string, initialBlockNumber: bigint): Promise { - await this.update( - { - initial_block_number: MongoDB.Long.fromBigInt(initialBlockNumber), - account, - }, - { options: { upsert: true } } - ); - } -} diff --git a/src/common/contract-reader/featured-contract.ts b/src/common/contract-reader/featured-contract.ts deleted file mode 100644 index 049bd2a..0000000 --- a/src/common/contract-reader/featured-contract.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { - parseToBigInt, - removeUndefinedProperties, -} from '@alien-worlds/api-core'; -import { FeaturedContractDocument } from './contract-reader.dtos'; -import { MongoDB } from '@alien-worlds/storage-mongodb'; - -export class FeaturedContract { - /** - * @constructor - * @private - * @param {string} id - * @param {bigint} initialBlockNumber - * @param {bigint} account - */ - private constructor( - public readonly id: string, - public readonly initialBlockNumber: bigint, - public readonly account: string - ) {} - - public static create(account: string, initialBlockNumber: string | number) { - return new FeaturedContract('', parseToBigInt(initialBlockNumber), account); - } - - public static fromDocument(document: FeaturedContractDocument) { - const { initial_block_number, _id, account } = document; - - return new FeaturedContract( - _id ? _id.toString() : '', - parseToBigInt(initial_block_number), - account - ); - } - - public toDocument() { - const { id, initialBlockNumber, account } = this; - const doc: FeaturedContractDocument = { - initial_block_number: MongoDB.Long.fromBigInt(initialBlockNumber), - account, - }; - - if (id) { - doc._id = new MongoDB.ObjectId(id); - } - - return removeUndefinedProperties(doc); - } -} diff --git a/src/common/contract-reader/index.ts b/src/common/contract-reader/index.ts deleted file mode 100644 index fd79e8c..0000000 --- a/src/common/contract-reader/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from './contract-reader'; -export * from './contract-reader.config'; -export * from './contract-reader.dtos'; -export * from './featured-contract.source'; -export * from './featured-contract'; diff --git a/src/common/dependencies.ts b/src/common/dependencies.ts new file mode 100644 index 0000000..ea2a5df --- /dev/null +++ b/src/common/dependencies.ts @@ -0,0 +1,16 @@ +import { Result } from '@alien-worlds/api-core'; + +/** + * An abstract class representing a Process dependencies. + * + * @abstract + * @class Dependencies + */ +export abstract class Dependencies { + /** + * Initializes and configures the Dependencies instance. + * @abstract + * @returns {Promise} A promise that resolves when the initialization is complete. + */ + public abstract initialize(...args: unknown[]): Promise; +} diff --git a/src/common/featured/__tests__/featured-contract.mongo.mapper.unit.test.ts b/src/common/featured/__tests__/featured-contract.mongo.mapper.unit.test.ts new file mode 100644 index 0000000..c7dec6b --- /dev/null +++ b/src/common/featured/__tests__/featured-contract.mongo.mapper.unit.test.ts @@ -0,0 +1,45 @@ +import { FeaturedContractMongoMapper } from '../featured-contract.mongo.mapper'; +import { MongoDB } from '@alien-worlds/storage-mongodb'; +import { FeaturedContract } from '../featured-contract'; +import { FeaturedContractMongoModel } from '../featured.types'; + +describe('FeaturedContractMongoMapper', () => { + const fromBigIntMock = jest.fn(); + const parseToBigIntMock = jest.fn(); + + beforeAll(() => { + MongoDB.Long.fromBigInt = fromBigIntMock; + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + const mapper = new FeaturedContractMongoMapper(); + + it('should convert from entity to model correctly', () => { + const entity = new FeaturedContract('1', 100n, 'account1'); + fromBigIntMock.mockReturnValue('100'); + + const model = mapper.fromEntity(entity); + expect(model).toEqual({ + _id: new MongoDB.ObjectId('1'), + initial_block_number: '100', + account: 'account1', + }); + expect(fromBigIntMock).toHaveBeenCalledWith(100n); + }); + + it('should convert from model to entity correctly', () => { + const model: FeaturedContractMongoModel = { + _id: new MongoDB.ObjectId('1'), + initial_block_number: MongoDB.Long.fromBigInt(100n), + account: 'account1', + }; + parseToBigIntMock.mockReturnValue(100n); + + const entity = mapper.toEntity(model); + expect(entity).toEqual(new FeaturedContract('1', 100n, 'account1')); + expect(parseToBigIntMock).toHaveBeenCalledWith(100n); + }); +}); diff --git a/src/common/featured/__tests__/featured.mapper.unit.test.ts b/src/common/featured/__tests__/featured.mapper.unit.test.ts new file mode 100644 index 0000000..71b023a --- /dev/null +++ b/src/common/featured/__tests__/featured.mapper.unit.test.ts @@ -0,0 +1,222 @@ +import { FeaturedMapper } from '../featured.mapper'; +import { MatcherNotFoundError, PatternMatchError } from '../featured.errors'; +import { + MatchCriteria, + ProcessorMatchCriteria, + ProcessorMatcher, +} from '../featured.types'; + +describe('ContractProcessorMapper', () => { + let contractProcessorMapper: FeaturedMapper; + + const mockCriteria: ProcessorMatchCriteria[] = [ + { + contract: 'mockContract1', + action: ['mockAction1', 'mockAction2'], + processor: 'mockProcessor1', + }, + { + contract: ['mockContract2', 'mockContract3'], + action: ['mockAction3'], + processor: 'mockProcessor2', + }, + ]; + + const mockMatcher = async (criteria: MatchCriteria) => { + return criteria.contract.includes('mockContract1'); + }; + + const mockMatchers: ProcessorMatcher = new Map(); + mockMatchers.set('mockMatcher', mockMatcher); + + beforeEach(() => { + contractProcessorMapper = new FeaturedMapper(mockCriteria, mockMatchers); + }); + + describe('constructor', () => { + it('should throw error when matcher is not found', () => { + const criteriaWithInvalidMatcher: ProcessorMatchCriteria[] = [ + { + contract: 'mockContract', + action: 'mockAction', + processor: 'mockProcessor', + matcher: 'invalidMatcher', + }, + ]; + expect(() => new FeaturedMapper(criteriaWithInvalidMatcher, mockMatchers)).toThrow( + MatcherNotFoundError + ); + }); + + it('should add all contracts to the contracts set', () => { + const contracts = (contractProcessorMapper as any).contracts; + expect(contracts.size).toEqual(3); + expect(contracts.has('mockContract1')).toBe(true); + expect(contracts.has('mockContract2')).toBe(true); + expect(contracts.has('mockContract3')).toBe(true); + }); + }); + + describe('validateCriteria', () => { + it('should throw error when pattern does not match', () => { + const invalidCriteria: MatchCriteria = { + contract: ['mockContract', 'invalid*Contract'], + }; + expect(() => + (contractProcessorMapper as any).validateCriteria(invalidCriteria) + ).toThrow(PatternMatchError); + }); + + it('should not throw error when pattern matches', () => { + const validCriteria: MatchCriteria = { + contract: ['mockContract', 'anotherMockContract'], + }; + expect(() => + (contractProcessorMapper as any).validateCriteria(validCriteria) + ).not.toThrow(); + }); + }); + + //... + + describe('isMatch', () => { + it('should return true when candidate matches reference', () => { + const ref: ProcessorMatchCriteria = { + contract: ['mockContract1', 'mockContract2'], + action: ['mockAction1', 'mockAction2'], + processor: 'mockProcessor', + }; + const candidate: MatchCriteria = { + contract: 'mockContract1', + action: 'mockAction1', + }; + + expect((contractProcessorMapper as any).isMatch(ref, candidate)).toBe(true); + }); + + it('should return false when candidate does not match reference', () => { + const ref: ProcessorMatchCriteria = { + contract: ['mockContract1', 'mockContract2'], + action: ['mockAction1', 'mockAction2'], + processor: 'mockProcessor', + }; + const candidate: MatchCriteria = { + contract: 'mockContract3', + action: 'mockAction3', + }; + + expect((contractProcessorMapper as any).isMatch(ref, candidate)).toBe(false); + }); + }); + + describe('findProcessorMatchCriteria', () => { + it('should return match criteria when a matcher matches', async () => { + const criteria: MatchCriteria = { + contract: 'mockContract1', + action: 'mockAction1', + }; + + const result = await (contractProcessorMapper as any).findProcessorMatchCriteria( + criteria + ); + + expect(result).toBeDefined(); + expect(result.processor).toEqual('mockProcessor1'); + }); + + it('should return null when no matcher matches', async () => { + const criteria: MatchCriteria = { + contract: 'mockContract3', + action: 'mockAction3', + }; + + const result = await (contractProcessorMapper as any).findProcessorMatchCriteria( + criteria + ); + + expect(result).toBeNull(); + }); + }); + + describe('has', () => { + it('should return true when the match criteria is found', async () => { + const criteria: MatchCriteria = { + contract: 'mockContract1', + action: 'mockAction1', + }; + + expect(await contractProcessorMapper.has(criteria)).toBe(true); + }); + + it('should return false when the match criteria is not found', async () => { + const criteria: MatchCriteria = { + contract: 'mockContract3', + action: 'mockAction3', + }; + + expect(await contractProcessorMapper.has(criteria)).toBe(false); + }); + }); + + describe('get', () => { + it('should return all matched criteria', async () => { + const criteria: MatchCriteria = { + contract: 'mockContract1', + action: 'mockAction1', + }; + + const results = await contractProcessorMapper.get(criteria); + + expect(results.length).toBeGreaterThan(0); + expect(results[0].contract.includes('mockContract1')).toBe(true); + expect(results[0].action.includes('mockAction1')).toBe(true); + }); + + it('should return an empty array when no criteria match', async () => { + const criteria: MatchCriteria = { + contract: 'mockContract3', + action: 'mockAction3', + }; + + const results = await contractProcessorMapper.get(criteria); + + expect(results.length).toBe(0); + }); + }); + + describe('getProcessor', () => { + it('should return the correct processor name', async () => { + const label = 'contract:mockContract1,action:mockAction1'; + const criteria: MatchCriteria = { + contract: 'mockContract1', + action: 'mockAction1', + }; + + const processorName = await contractProcessorMapper.getProcessor(label, criteria); + + expect(processorName).toEqual('mockProcessor1'); + }); + + it('should return an empty string when no processor matches', async () => { + const label = 'contract:mockContract3,action:mockAction3'; + const criteria: MatchCriteria = { + contract: 'mockContract3', + action: 'mockAction3', + }; + + const processorName = await contractProcessorMapper.getProcessor(label, criteria); + + expect(processorName).toEqual(''); + }); + }); + + describe('listContracts', () => { + it('should return all unique contracts', () => { + const contracts = contractProcessorMapper.listContracts(); + + expect(contracts.length).toBeGreaterThan(0); + expect(contracts.includes('mockContract1')).toBe(true); + expect(contracts.includes('mockContract2')).toBe(true); + }); + }); +}); diff --git a/src/common/featured/__tests__/featured.unit.test.ts b/src/common/featured/__tests__/featured.unit.test.ts deleted file mode 100644 index b220309..0000000 --- a/src/common/featured/__tests__/featured.unit.test.ts +++ /dev/null @@ -1,113 +0,0 @@ -/* eslint-disable @typescript-eslint/no-empty-function */ -/* eslint-disable @typescript-eslint/no-explicit-any */ - -import { FeaturedContractContent } from "../featured"; - -const externalTraceData = { - shipTraceMessageName: ['external_foo_type'], - shipActionTraceMessageName: ['external_foo_name'], - contract: ['external_foo_contract'], - action: ['external_foo_action'], - processor: 'external_foo_processor', -} - -export const matchers = { - traces: new Map([ - [ - 'external', - async (data) => { - const result = data['shipTraceMessageName'].includes('external_foo_type') && data['contract'].includes('external_foo_contract'); - return result; - } - ] - ]) -} - -const config = { - traces: [{ - shipTraceMessageName: ['foo_type'], - shipActionTraceMessageName: ['foo_name'], - contract: ['foo_contract', 'foo_contract_2'], - action: ['foo_action'], - processor: 'foo_processor' - },{ - shipTraceMessageName: ['foo_2_type'], - shipActionTraceMessageName: ['foo_2_name'], - contract: ['*'], - action: ['*'], - processor: 'foo_2_processor' - }, { - matcher: 'external', - processor: 'external_foo_processor' - }], - deltas: [{ - shipDeltaMessageName: ['bar_type'], - name: ['bar_name'], - code: ['bar_code'], - scope: ['bar_scope'], - table: ['bar_table'], - processor: 'bar_processor' - }], -} - -describe('Featured Unit tests', () => { - it('"getProcessor" should return processor path assigned to given label', async () => { - const featured = new FeaturedContractContent(config as any, matchers); - - expect(await (featured).traces.getProcessor('foo_type:foo_name:foo_contract:foo_action')).toEqual('foo_processor') - expect(await (featured).traces.getProcessor('foo_2_type:foo_2_name:foo_2_contract:*')).toEqual('foo_2_processor') - expect(await (featured).deltas.getProcessor('bar_type:bar_name:bar_code:bar_scope:bar_table')).toEqual('bar_processor') - expect(await (featured).deltas.getProcessor('bar_2_type:*')).toEqual('') - }); - - it('"has" should return a bool value depending on whether the given pattern matches or not', async () => { - const featured = new FeaturedContractContent(config as any, matchers); - - expect(await (featured).traces.has({ shipTraceMessageName: ['foo_type'], shipActionTraceMessageName: ['foo_name'], contract: ['foo_contract'], action: ['foo_action'] })).toEqual(true) - expect(await (featured).traces.has({ shipTraceMessageName: ['foo_2_type'], shipActionTraceMessageName: ['foo_2_name'] })).toEqual(true) - expect(await (featured).traces.has({ shipTraceMessageName: ['foo_type_3'], shipActionTraceMessageName: ['*'], contract: ['*'], action: ['foo_action'] })).toEqual(false) - - expect(await (featured).deltas.has({ shipDeltaMessageName: ['bar_type'], name: ['bar_name'], code: ['bar_code'], scope: ['bar_scope'], table: ['bar_table'] })).toEqual(true) - expect(await (featured).deltas.has({ shipDeltaMessageName: ['bar_type'], name: ['bar_name'] })).toEqual(true) - expect(await (featured).deltas.has({ shipDeltaMessageName: ['bar_type_3'], name: ['*'] })).toEqual(false) - - expect(await (featured).traces.has({ - shipTraceMessageName: ['external_foo_type'], - shipActionTraceMessageName: ['external_foo_name'], - contract: ['external_foo_contract'], - action: ['external_foo_action'] - })).toEqual(true) - - expect(await (featured).traces.has({ - shipTraceMessageName: ['external_foo_type_UNKNOWN'], - shipActionTraceMessageName: ['external_foo_name_UNKNOWN'], - contract: ['external_foo_contract_UNKNOWN'], - action: ['external_foo_action_UNKNOWN'] - })).toEqual(false) - }); - - it('"get" should return an allocation object when given pattern matches', async () => { - const featured = new FeaturedContractContent(config as any, matchers); - - expect(await (featured).traces.get({ shipTraceMessageName: ['foo_type'], shipActionTraceMessageName: ['foo_name'], contract: ['foo_contract'], action: ['foo_action'] })).toEqual([config.traces[0]]) - expect(await (featured).traces.get({ shipTraceMessageName: ['foo_type_3'], shipActionTraceMessageName: ['*'], contract: ['*'], action: ['foo_action'] })).toEqual([]) - - expect(await (featured).deltas.get({ shipDeltaMessageName: ['bar_type'], name: ['bar_name'], code: ['bar_code'], scope: ['bar_scope'], table: ['bar_table'] })).toEqual(config.deltas) - expect(await (featured).deltas.get({ shipDeltaMessageName: ['bar_type_3'], name: ['*'] })).toEqual([]) - - expect(await (featured).traces.get({ - shipTraceMessageName: ['external_foo_type'], - shipActionTraceMessageName: ['external_foo_name'], - contract: ['external_foo_contract'], - action: ['external_foo_action'] - })).toEqual([externalTraceData]); - - expect(await (featured).traces.get({ - shipTraceMessageName: ['external_foo_type_UNKNOWN'], - shipActionTraceMessageName: ['external_foo_name_UNKNOWN'], - contract: ['external_foo_contract_UNKNOWN'], - action: ['external_foo_action_UNKNOWN'] - })).toEqual([]) - }); - -}); diff --git a/src/common/featured/__tests__/featured.utils.unit.test.ts b/src/common/featured/__tests__/featured.utils.unit.test.ts index 57320e5..67da7a2 100644 --- a/src/common/featured/__tests__/featured.utils.unit.test.ts +++ b/src/common/featured/__tests__/featured.utils.unit.test.ts @@ -1,15 +1,82 @@ -/* eslint-disable @typescript-eslint/no-empty-function */ -/* eslint-disable @typescript-eslint/no-explicit-any */ +import { FeaturedUtils } from '../featured.utils'; -import { contentOrAll } from "../featured.utils"; +describe('FeaturedUtils', () => { + describe('readFeaturedContracts', () => { + it('should return an empty array for empty data', () => { + const data = {}; + const result = FeaturedUtils.readFeaturedContracts(data); + expect(result).toEqual([]); + }); + it('should return an empty array for non-object data', () => { + const data = null; + const result = FeaturedUtils.readFeaturedContracts(data); + expect(result).toEqual([]); + }); -describe('Featured utils Unit tests', () => { - it('"contentOrAll" should return content when given array is not empty', async () => { - expect(contentOrAll(['foo', 'bar'])).toEqual(['foo', 'bar']); - }); + it('should return an array of unique contracts from the data object', () => { + const data = { + contract: ['ContractA', 'ContractB'], + otherProp: 'some value', + }; + const result = FeaturedUtils.readFeaturedContracts(data); + expect(result).toEqual(['ContractA', 'ContractB']); + }); + + it('should return an array of unique contracts from nested data', () => { + let data = { + prop1: 'value1', + prop2: { + contract: 'ContractC', + prop3: { + contract: ['ContractD', 'ContractE'], + }, + }, + }; + let result = FeaturedUtils.readFeaturedContracts(data); + expect(result).toEqual(['ContractC', 'ContractD', 'ContractE']); + + const nested = { + traces: [ + { + prop1: 'value1', + prop2: { + contract: 'ContractC', + prop3: { + contract: ['ContractD', 'ContractE'], + }, + }, + }, + ], + deltas: [ + { + prop1: 'value1', + prop2: { + contract: 'ContractF', + prop3: { + contract: ['ContractA', 'ContractB'], + }, + }, + }, + ], + }; + result = FeaturedUtils.readFeaturedContracts(nested); + expect(result).toEqual([ + 'ContractC', + 'ContractD', + 'ContractE', + 'ContractF', + 'ContractA', + 'ContractB', + ]); + }); - it('"contentOrAll" should return ALL wildcard (*) when given array is empty', async () => { - expect(contentOrAll([])).toEqual(['*']); + it('should ignore non-string values in the contract property', () => { + const data = { + contract: [123, 'ContractF', true, null], + }; + const result = FeaturedUtils.readFeaturedContracts(data); + expect(result).toEqual(['ContractF']); + }); }); }); diff --git a/src/common/featured/featured-contract.collection.ts b/src/common/featured/featured-contract.collection.ts new file mode 100644 index 0000000..fd8b176 --- /dev/null +++ b/src/common/featured/featured-contract.collection.ts @@ -0,0 +1,26 @@ +import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; +import { FeaturedContractMongoModel } from './featured.types'; + +/** + * Represents a source for FeaturedContract documents stored in a MongoDB collection. + * Extends the MongoCollectionSource class. + * + * @class + * @extends {MongoCollectionSource} + */ +export class FeaturedContractCollection extends MongoCollectionSource { + /** + * Creates a new instance of the FeaturedContractSource class. + * + * @constructor + * @param {MongoSource} mongoSource - The MongoSource object used for database operations. + */ + constructor(mongoSource: MongoSource) { + super(mongoSource, 'history_tools.featured_contracts', { + indexes: [ + { key: { account: 1 }, background: true }, + { key: { initial_block_number: 1, account: 1 }, unique: true, background: true }, + ], + }); + } +} diff --git a/src/common/featured/featured-contract.mongo.mapper.ts b/src/common/featured/featured-contract.mongo.mapper.ts new file mode 100644 index 0000000..4407f38 --- /dev/null +++ b/src/common/featured/featured-contract.mongo.mapper.ts @@ -0,0 +1,48 @@ +import { parseToBigInt } from '@alien-worlds/api-core'; +import { MongoDB, MongoMapper } from '@alien-worlds/storage-mongodb'; +import { FeaturedContract } from './featured-contract'; +import { FeaturedContractMongoModel } from './featured.types'; + +/** + * Class representing a FeaturedContractMongoMapper + * This class extends the MongoMapper to provide MongoDB-specific mappings for FeaturedContract. + * @class + * @extends {MongoMapper} + * @public + */ +export class FeaturedContractMongoMapper extends MongoMapper< + FeaturedContract, + FeaturedContractMongoModel +> { + /** + * Creates a new instance of FeaturedContractMongoMapper and sets up the mapping from 'initialBlockNumber' and 'account' to MongoDB's 'initial_block_number' and 'account'. + * @constructor + */ + constructor() { + super(); + this.mappingFromEntity.set('initialBlockNumber', { + key: 'initial_block_number', + mapper: (value: bigint) => MongoDB.Long.fromBigInt(value), + }); + this.mappingFromEntity.set('account', { + key: 'account', + mapper: (value: string) => value, + }); + } + + /** + * Converts a MongoDB document of FeaturedContractMongoModel to a FeaturedContract entity. + * + * @param {FeaturedContractMongoModel} model - The MongoDB document. + * @returns {FeaturedContract} The FeaturedContract entity. + */ + public toEntity(model: FeaturedContractMongoModel): FeaturedContract { + const { initial_block_number, _id, account } = model; + + return new FeaturedContract( + _id ? _id.toString() : '', + parseToBigInt(initial_block_number), + account + ); + } +} diff --git a/src/common/featured/featured-contract.ts b/src/common/featured/featured-contract.ts new file mode 100644 index 0000000..0dd6589 --- /dev/null +++ b/src/common/featured/featured-contract.ts @@ -0,0 +1,34 @@ +import { parseToBigInt } from '@alien-worlds/api-core'; + +/** + * Class representing a FeaturedContract + * @class + * @public + */ +export class FeaturedContract { + /** + * Creates a new instance of the FeaturedContract + * @constructor + * @param {string} id - The ID of the contract + * @param {bigint} initialBlockNumber - The initial block number of the contract + * @param {string} account - The account associated with the contract + */ + constructor( + public id: string, + public initialBlockNumber: bigint, + public account: string + ) {} + + /** + * Creates a new instance of FeaturedContract with a specified account and initial block number, + * ID will be set to empty string by default + * @static + * @public + * @param {string} account - The account associated with the contract + * @param {string | number} initialBlockNumber - The initial block number of the contract + * @returns {FeaturedContract} A new instance of FeaturedContract + */ + public static create(account: string, initialBlockNumber: string | number) { + return new FeaturedContract('', parseToBigInt(initialBlockNumber), account); + } +} diff --git a/src/common/featured/featured.actions.ts b/src/common/featured/featured.actions.ts new file mode 100644 index 0000000..8ec2b1b --- /dev/null +++ b/src/common/featured/featured.actions.ts @@ -0,0 +1,23 @@ +import { FeaturedMapper } from './featured.mapper'; +import { ContractActionMatchCriteria, ProcessorMatchCriteria } from './featured.types'; + +export class FeaturedActions { + private mapper: FeaturedMapper; + + constructor(criteria: ProcessorMatchCriteria[]) { + this.mapper = new FeaturedMapper(criteria); + } + + public getProcessor(criteria: ContractActionMatchCriteria) { + return this.mapper.getProcessor(contract, { + shipTraceMessageName: [], + shipActionTraceMessageName: [], + contract: [], + action: [], + }); + } + + public listContracts(): string[] { + return this.listContracts(); + } +} diff --git a/src/common/featured/featured.config.ts b/src/common/featured/featured.config.ts new file mode 100644 index 0000000..68b0ab2 --- /dev/null +++ b/src/common/featured/featured.config.ts @@ -0,0 +1,4 @@ +export type FeaturedConfig = { + serviceUrl: string; + rpcUrl: string; +}; diff --git a/src/common/featured/featured.creator.ts b/src/common/featured/featured.creator.ts new file mode 100644 index 0000000..59fe418 --- /dev/null +++ b/src/common/featured/featured.creator.ts @@ -0,0 +1,36 @@ +import { + MongoConfig, + MongoQueryBuilders, + MongoSource, +} from '@alien-worlds/storage-mongodb'; +import { Featured } from './featured'; +import { RepositoryImpl, SmartContractService, log } from '@alien-worlds/api-core'; +import { FeaturedContractCollection } from './featured-contract.collection'; +import { FeaturedContractMongoMapper } from './featured-contract.mongo.mapper'; + +export class FeaturedCreator { + public static async create( + mongo: MongoSource | MongoConfig, + smartContractService: SmartContractService + ): Promise { + let mongoSource: MongoSource; + + log(` * Featured ... [starting]`); + + if (mongo instanceof MongoSource) { + mongoSource = mongo; + } else { + mongoSource = await MongoSource.create(mongo); + } + const repository = new RepositoryImpl( + new FeaturedContractCollection(mongoSource), + new FeaturedContractMongoMapper(), + new MongoQueryBuilders() + ); + const featured = new Featured(repository, smartContractService); + + log(` * Contract Reader ... [ready]`); + + return featured; + } +} diff --git a/src/common/featured/featured.deltas.ts b/src/common/featured/featured.deltas.ts new file mode 100644 index 0000000..42a7d71 --- /dev/null +++ b/src/common/featured/featured.deltas.ts @@ -0,0 +1,25 @@ +import { FeaturedMapper } from './featured.mapper'; +import { ContractDeltaMatchCriteria, ProcessorMatchCriteria } from './featured.types'; + +export class FeaturedDeltas { + private mapper: FeaturedMapper; + + constructor(criteria: ProcessorMatchCriteria[]) { + this.mapper = new FeaturedMapper(criteria); + } + + public getProcessor(contract: string) { + return this.mapper.getProcessor(contract, { + shipDeltaMessageName: [], + contract: [], + name: [], + code: [], + scope: [], + table: [], + }); + } + + public listContracts(): string[] { + return this.listContracts(); + } +} diff --git a/src/common/featured/featured.enums.ts b/src/common/featured/featured.enums.ts deleted file mode 100644 index 4665f63..0000000 --- a/src/common/featured/featured.enums.ts +++ /dev/null @@ -1,4 +0,0 @@ -export enum FeaturedContentType { - Action = 'action', - Delta = 'delta', -} diff --git a/src/common/featured/featured.mapper.ts b/src/common/featured/featured.mapper.ts new file mode 100644 index 0000000..f231726 --- /dev/null +++ b/src/common/featured/featured.mapper.ts @@ -0,0 +1,253 @@ +import { MatcherNotFoundError, PatternMatchError } from './featured.errors'; +import { + MatchCriteria, + ProcessorMatchCriteria, + ProcessorMatcher, +} from './featured.types'; + +/** + * A mapper class for processing and matching contracts based on given criteria. + * This class can be extended by other processors that require specific match criteria. + */ +export class FeaturedMapper { + /** + * Map of processors matched by a matching function. + */ + protected processorByMatchers: ProcessorMatcher = new Map(); + /** + * Array of match criteria. + */ + protected matchCriteria: ProcessorMatchCriteria[] = []; + /** + * Set of contracts. + */ + protected contracts: Set = new Set(); + + /** + * Creates a new instance of the contract processor mapper. + * @param criteria - An array of match criteria for the processor. + * @param matchers - Optional map of matchers. + */ + constructor( + criteria: ProcessorMatchCriteria[], + matchers?: ProcessorMatcher + ) { + criteria.forEach(criteria => { + const { processor, matcher, ...rest } = criteria; + const { contract } = rest as unknown as MatchCriteria; + + if (Array.isArray(contract)) { + contract.forEach(contract => this.contracts.add(contract)); + } else if (typeof contract === 'string') { + this.contracts.add(contract); + } + + if (matcher && !matchers?.has(matcher)) { + throw new MatcherNotFoundError(matcher); + } + + if (matcher && matchers.has(matcher)) { + this.processorByMatchers.set(processor, matchers.get(matcher)); + } else { + this.validateCriteria(rest as MatchCriteriaType); + + if (this.matchCriteria.indexOf(criteria) === -1) { + this.matchCriteria.push(criteria); + } + } + }); + } + + /** + * Validates the given match criteria. + * @param criteria - The criteria to validate. + */ + protected validateCriteria(criteria: MatchCriteriaType): void { + const keys = Object.keys(criteria); + + for (const key of keys) { + const values = criteria[key]; + for (const value of values) { + if (/^(\*|[A-Za-z0-9_.]*)$/g.test(value) === false) { + throw new PatternMatchError(value, '^(*|[A-Za-z0-9_.]*)$'); + } + } + } + } + + /** + * Determines if a candidate match criteria meets a reference match criteria. + * @param ref - The reference match criteria. + * @param candidate - The candidate match criteria. + * @returns True if a match is found, false otherwise. + */ + protected isMatch( + ref: ProcessorMatchCriteria, + candidate: MatchCriteriaType + ): boolean { + let matchFound = false; + const keys = Object.keys(candidate); + + for (const key of keys) { + const candidateValues = candidate[key]; + const refValues = ref[key]; + if (Array.isArray(refValues)) { + const values: string[] = Array.isArray(candidateValues) + ? candidateValues + : [candidateValues]; + const contains = values.some(value => refValues.includes(value)); + + if (refValues.includes('*') || contains) { + matchFound = true; + } else { + return false; + } + } + } + + return matchFound; + } + + /** + * Finds a processor match criteria for a given match criteria. + * @param criteria - The criteria to find a match for. + * @returns The matching processor match criteria if found, null otherwise. + */ + protected async findProcessorMatchCriteria( + criteria: MatchCriteriaType + ): Promise> { + const { processorByMatchers } = this; + const entries = Array.from(processorByMatchers.entries()); + + for (const entry of entries) { + const [processor, matcher] = entry; + if (await matcher(criteria)) { + const keys = Object.keys(criteria); + const matchCriteria = {} as MatchCriteriaType; + + for (const key of keys) { + matchCriteria[key] = ['*']; + } + + return { + ...matchCriteria, + processor, + }; + } + } + + return null; + } + + /** + * Checks if the criteria already exist in the array + * + * @param {ProcessorMatchCriteria} criteria + * @param {ProcessorMatchCriteria[]} array + * @returns + */ + protected criteriaExistsInArray( + criteria: ProcessorMatchCriteria, + array: ProcessorMatchCriteria[] + ): boolean { + return array.some(item => + Object.keys(item).every(key => item[key] === criteria[key]) + ); + } + + /** + * Determines if the given match criteria exists in the processor. + * @param criteria - The criteria to check. + * @returns True if the criteria exists, false otherwise. + */ + public async has(criteria: MatchCriteriaType): Promise { + const { matchCriteria, processorByMatchers } = this; + + for (const item of matchCriteria) { + if (this.isMatch(item, criteria)) { + return true; + } + } + + if (processorByMatchers.size > 0) { + const featured = await this.findProcessorMatchCriteria(criteria); + if (featured) { + if (this.criteriaExistsInArray(featured, matchCriteria) === false) { + matchCriteria.push(featured); + } + return true; + } + } + + return false; + } + + /** + * Gets all match criteria in the processor that match the given criteria. + * @param criteria - The criteria to match. + * @returns An array of matching criteria. + */ + public async get(criteria: MatchCriteriaType): Promise { + const { matchCriteria, processorByMatchers } = this; + const result: MatchCriteriaType[] = []; + + for (const item of matchCriteria) { + if (this.isMatch(item, criteria)) { + result.push(item); + } + } + + if (result.length === 0 && processorByMatchers.size > 0) { + const featured = await this.findProcessorMatchCriteria(criteria); + if (featured) { + if (this.criteriaExistsInArray(featured, matchCriteria) === false) { + matchCriteria.push(featured); + } + result.push(featured); + } + } + + return result; + } + + /** + * Gets the processor for the given label and criteria. + * @param label - The label to find a processor for. + * @param criteria - The criteria to match. + * @returns The processor if found, empty string otherwise. + */ + public async getProcessor(label: string, criteria: MatchCriteriaType): Promise { + const { matchCriteria } = this; + const keys = Object.keys(criteria); + const parts = label.split(':').map(part => part.split(',')); + const candidate = parts.reduce((result, part, i) => { + result[keys[i]] = part; + return result; + }, criteria); + + for (const criteriaRef of matchCriteria) { + if (this.isMatch(criteriaRef, candidate)) { + return criteriaRef.processor; + } + } + + const featured = await this.findProcessorMatchCriteria(candidate); + + if (featured) { + if (matchCriteria.indexOf(featured) === -1) { + matchCriteria.push(featured); + } + return featured.processor; + } + + return ''; + } + + /** + * Lists all contracts in the processor. + * @returns An array of contracts. + */ + public listContracts(): string[] { + return Array.from(this.contracts); + } +} diff --git a/src/common/featured/featured.ts b/src/common/featured/featured.ts index 239705f..44aaf71 100644 --- a/src/common/featured/featured.ts +++ b/src/common/featured/featured.ts @@ -1,283 +1,78 @@ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ - -/* eslint-disable @typescript-eslint/no-unused-vars */ -import { FeaturedContentType } from './featured.enums'; -import { - MatcherNotFoundError, - PatternMatchError, - UnknownContentTypeError, -} from './featured.errors'; import { - AllocationType, - FeaturedAllocationType, - FeaturedConfig, - FeaturedDelta, - FeaturedDeltaAllocation, - FeaturedMatcher, - FeaturedMatchers, - FeaturedTrace, - FeaturedTraceAllocation, - FeaturedType, -} from './featured.types'; -import { buildFeaturedAllocation } from './featured.utils'; - -export abstract class FeaturedContent { - public abstract listContracts(): string[]; - public abstract getProcessor(label: string): Promise; - - protected processorsByMatchers: FeaturedMatcher = new Map(); - protected allocations: T[] = []; - - constructor(allocations: T[], matchers?: FeaturedMatcher) { - allocations.forEach(allocation => { - const { processor, matcher, ...rest } = allocation as FeaturedType; - - if (matcher && !matchers?.has(matcher)) { - throw new MatcherNotFoundError(matcher); - } - - if (matcher && matchers.has(matcher)) { - this.processorsByMatchers.set(processor, matchers.get(matcher)); + FindParams, + Repository, + Result, + SmartContractService, + UnknownObject, + Where, +} from '@alien-worlds/api-core'; +import { FeaturedContract } from './featured-contract'; +import { FeaturedUtils } from './featured.utils'; + +export class Featured { + protected cache: Map = new Map(); + protected featuredContracts: string[]; + + constructor( + private repository: Repository, + private smartContractService: SmartContractService, + featuredJson: UnknownObject + ) { + this.featuredContracts = FeaturedUtils.readFeaturedContracts(featuredJson); + } + + /** + * Reads multiple contracts and returns the results as an array of FeaturedContract objects. + * + * @abstract + * @param {string[]} contracts - An array of contract addresses or identifiers. + * @returns {Promise>} A Promise that resolves to an array of FeaturedContract objects. + */ + public async readContracts( + data: string[] | UnknownObject + ): Promise> { + const list: FeaturedContract[] = []; + const contracts = FeaturedUtils.readFeaturedContracts(data); + + for (const contract of contracts) { + if (this.cache.has(contract)) { + list.push(this.cache.get(contract)); } else { - this.validateAllocation(rest); - // - if (this.allocations.indexOf(allocation) === -1) { - this.allocations.push(allocation); - } - } - }); - } + const { content: contracts, failure } = await this.repository.find( + FindParams.create({ where: new Where().valueOf('account').isEq(contract) }) + ); - protected validateAllocation(allocation: FeaturedAllocationType): void { - const keys = Object.keys(allocation); - - for (const key of keys) { - const values = allocation[key]; - for (const value of values) { - if (/^(\*|[A-Za-z0-9_.]*)$/g.test(value) === false) { - throw new PatternMatchError(value, '^(*|[A-Za-z0-9_.]*)$'); + if (failure) { + return Result.withFailure(failure); } - } - } - } - - protected isMatch( - ref: T, - candidate: K - ): boolean { - let matchFound = false; - const keys = Object.keys(candidate); - for (const key of keys) { - const candidateValues: string | string[] = candidate[key]; - const refValues: string[] = ref[key]; - if (Array.isArray(refValues)) { - const values: string[] = Array.isArray(candidateValues) - ? candidateValues - : [candidateValues]; - const contains = values.some(value => refValues.includes(value)); - - if (refValues.includes('*') || contains) { - matchFound = true; + if (contracts.length > 0) { + const featuredContract = contracts[0]; + this.cache.set(featuredContract.account, featuredContract); + list.push(featuredContract); } else { - return false; - } - } - } - - return matchFound; - } - - protected async testMatchers( - allocation: FeaturedAllocationType | AllocationType - ): Promise { - const { processorsByMatchers } = this; - const entries = Array.from(processorsByMatchers.entries()); - - for (const entry of entries) { - const [processor, matcher] = entry; - if (await matcher(allocation)) { - return { - ...buildFeaturedAllocation(allocation), - processor, - } as T; - } - } - - return null; - } - - public async has( - allocation: FeaturedAllocationType | AllocationType - ): Promise { - const { allocations, processorsByMatchers } = this; - - for (const item of allocations) { - if (this.isMatch(item, allocation)) { - return true; - } - } - - if (processorsByMatchers.size > 0) { - const featured = await this.testMatchers(allocation); - if (featured) { - if (allocations.indexOf(featured) === -1) { - allocations.push(featured); + const fetchResult = await this.smartContractService.getStats(contract); + + if (fetchResult.isFailure) { + return Result.withFailure(fetchResult.failure); + } + if (fetchResult.content) { + const featuredContract = FeaturedContract.create( + fetchResult.content.account_name, + fetchResult.content.first_block_num + ); + this.cache.set(featuredContract.account, featuredContract); + this.repository.add([featuredContract]); + list.push(featuredContract); + } } - return true; - } - } - - return false; - } - - public async get(allocation: FeaturedAllocationType | AllocationType): Promise { - const { allocations, processorsByMatchers } = this; - const result: T[] = []; - - for (const item of allocations) { - if (this.isMatch(item, allocation)) { - result.push(item); } } - if (result.length === 0 && processorsByMatchers.size > 0) { - const featured = await this.testMatchers(allocation); - - if (featured) { - if (allocations.indexOf(featured) === -1) { - allocations.push(featured); - } - result.push(featured); - } - } - - return result; - } - - public toJson(): T[] { - return this.allocations; - } - - protected async getProcessorBySchema( - label: string, - allocationSchema: SchemaType - ): Promise { - const { allocations } = this; - const keys = Object.keys(allocationSchema); - const parts = label.split(':').map(part => part.split(',')); - const allocation = parts.reduce((result, part, i) => { - result[keys[i]] = part; - return result; - }, allocationSchema); - - for (const featured of allocations) { - if (this.isMatch(featured, allocation)) { - return (featured).processor; - } - } - - const featured = await this.testMatchers(allocation as FeaturedAllocationType); - - if (featured) { - if (allocations.indexOf(featured) === -1) { - allocations.push(featured); - } - return (featured).processor; - } - - return ''; - } -} - -export class FeaturedTraces extends FeaturedContent { - private contracts: Set = new Set(); - constructor(traces: FeaturedTrace[], matchers?: FeaturedMatcher) { - super(traces, matchers); - traces.forEach(trace => { - if (trace.contract) { - trace.contract.forEach(contract => this.contracts.add(contract)); - } - }); - } - - public listContracts(): string[] { - return Array.from(this.contracts); - } - - public async getProcessor(label: string): Promise { - return this.getProcessorBySchema(label, { - shipTraceMessageName: [], - shipActionTraceMessageName: [], - contract: [], - action: [], - }); - } -} - -export class FeaturedDeltas extends FeaturedContent { - private contracts: Set = new Set(); - constructor(deltas: FeaturedDelta[], matchers?: FeaturedMatcher) { - super(deltas, matchers); - deltas.forEach(delta => { - if (delta.code) { - delta.code.forEach(contract => this.contracts.add(contract)); - } - }); - } - - public listContracts(): string[] { - return Array.from(this.contracts); - } - - public async getProcessor(label: string): Promise { - return this.getProcessorBySchema(label, { - shipDeltaMessageName: [], - name: [], - code: [], - scope: [], - table: [], - }); - } -} - -export class FeaturedContractContent { - private traces: FeaturedTraces; - private deltas: FeaturedDeltas; - - constructor(config: FeaturedConfig, matchers?: FeaturedMatchers) { - const { traces, deltas } = matchers || {}; - this.traces = new FeaturedTraces(config.traces, traces); - this.deltas = new FeaturedDeltas(config.deltas, deltas); - } - - public getProcessor(type: string, label: string) { - if (type === FeaturedContentType.Action) { - return this.traces.getProcessor(label); - } else if (type === FeaturedContentType.Delta) { - return this.deltas.getProcessor(label); - } else { - throw new UnknownContentTypeError(type); - } - } - - public listContracts(): string[] { - const { traces, deltas } = this; - const list: string[] = []; - [...traces.listContracts(), ...deltas.listContracts()].forEach(contract => { - if (list.includes(contract) === false) { - list.push(contract); - } - }); - - return list; + return Result.withContent(list); } - public toJson() { - const { deltas, traces } = this; - return { - traces: traces.toJson(), - deltas: deltas.toJson(), - }; + public isFeatured(contract: string): boolean { + return this.featuredContracts.includes(contract); } } diff --git a/src/common/featured/featured.types.ts b/src/common/featured/featured.types.ts index a30f3aa..cbd6602 100644 --- a/src/common/featured/featured.types.ts +++ b/src/common/featured/featured.types.ts @@ -1,88 +1,52 @@ -export type AllocationType = { - [key: string]: string; -}; +import { MongoDB } from "@alien-worlds/storage-mongodb"; -export type FeaturedAllocationType = { - [key: string]: string[]; +export type FeaturedContractMongoModel = { + _id?: MongoDB.ObjectId; + account?: string; + initial_block_number?: MongoDB.Long; }; -export type FeaturedType = FeaturedAllocationType & { - matcher?: string; - processor: string; +export type FeaturedContractModel = { + account: string; + initialBlockNumber: bigint; }; -export type OptionalTraceAllocation = { - shipTraceMessageName?: string; - shipActionTraceMessageName?: string; - contract?: string; - action?: string; +export type FetchContractResponse = { + account: string; + block_num: string | number; }; -export type TraceAllocation = { - shipTraceMessageName: string; - shipActionTraceMessageName: string; - contract: string; - action: string; -}; +export type CriteriaValue = string | string[]; -export type FeaturedTraceAllocation = { - shipTraceMessageName: string[]; - shipActionTraceMessageName: string[]; - contract: string[]; - action: string[]; +export type MatchCriteria = { + contract: CriteriaValue; + [key: string]: CriteriaValue; }; -export type FeaturedTrace = FeaturedTraceAllocation & { +export type ProcessorMatchCriteria = { matcher?: string; processor: string; -}; +} & MatchCriteriaType; -export type OptionalDeltaAllocation = { - shipDeltaMessageName?: string; - name?: string; - code?: string; - scope?: string; - table?: string; -}; +export type ProcessorMatcher = Map< + string, + MatchFunction +>; + +export type MatchFunction = ( + criteria: MatchCriteriaType +) => Promise; -export type DeltaAllocation = { - shipDeltaMessageName: string; - name: string; - code: string; - scope: string; - table: string; +export type ContractActionMatchCriteria = MatchCriteria & { + shipTraceMessageName: string[]; + shipActionTraceMessageName: string[]; + action: string[]; }; -export type FeaturedDeltaAllocation = { +export type ContractDeltaMatchCriteria = MatchCriteria & { shipDeltaMessageName: string[]; name: string[]; code: string[]; scope: string[]; table: string[]; }; - -export type FeaturedDelta = FeaturedDeltaAllocation & { - matcher?: string; - processor: string; -}; - -export type FeaturedConfig = { - traces: FeaturedTrace[]; - deltas: FeaturedDelta[]; -}; - -export type FeaturedMatchers = { - traces?: FeaturedMatcher; - deltas?: FeaturedMatcher; -}; - -export type PathLink = { - link: string[][]; - path: string; -}; - -export type FeaturedMatcher = Map; - -export type MatchFunction = ( - data: FeaturedAllocationType | AllocationType -) => Promise; diff --git a/src/common/featured/featured.utils.ts b/src/common/featured/featured.utils.ts index 3a332b1..d763659 100644 --- a/src/common/featured/featured.utils.ts +++ b/src/common/featured/featured.utils.ts @@ -1,19 +1,31 @@ -import { AllocationType, FeaturedAllocationType } from './featured.types'; +import { UnknownObject } from '@alien-worlds/api-core'; -export const contentOrAll = (content: string[]) => - content?.length > 0 ? content : ['*']; +export class FeaturedUtils { + public static readFeaturedContracts(data: UnknownObject | unknown[]): string[] { + const contracts = new Set(); + if (!data) { + return []; + } + Object.keys(data).forEach(key => { + const value = data[key]; -export const buildFeaturedAllocation = ( - allocation: FeaturedAllocationType | AllocationType -): FeaturedAllocationType => { - const keys = Object.keys(allocation); - const result = {}; - - for (const key of keys) { - const value = allocation[key]; - - result[key] = Array.isArray(value) ? value : [value]; + if (key === 'contract' && Array.isArray(value)) { + value.forEach(contract => { + if (typeof contract === 'string') { + contracts.add(contract); + } + }); + } else if (key === 'contract' && typeof value === 'string') { + if (typeof value === 'string') { + contracts.add(value); + } + } else if (Array.isArray(value) || typeof value === 'object') { + const result = this.readFeaturedContracts(value); + result.forEach(contract => { + contracts.add(contract); + }); + } + }); + return Array.from(contracts); } - - return result; -}; +} diff --git a/src/common/featured/index.ts b/src/common/featured/index.ts index a3d3e08..94de583 100644 --- a/src/common/featured/index.ts +++ b/src/common/featured/index.ts @@ -1,5 +1,12 @@ +export * from './featured-contract'; +export * from './featured-contract.collection'; +export * from './featured-contract.mongo.mapper'; export * from './featured'; +export * from './featured.config'; +export * from './featured.errors'; +export * from './featured.mapper'; export * from './featured.types'; export * from './featured.utils'; -export * from './featured.errors'; -export * from './featured.enums'; + +export * from './featured.actions'; +export * from './featured.deltas'; diff --git a/src/common/index.ts b/src/common/index.ts index 69fd9fd..96244bd 100644 --- a/src/common/index.ts +++ b/src/common/index.ts @@ -1,10 +1,10 @@ export * from './abis'; -export * from '../reader/block-range-scanner'; +export * from './block-range-scanner'; export * from './block-state'; -export * from './blockchain'; +export * from './contract-content'; +export * from './featured'; +export * from './unprocessed-block-queue'; export * from './common.enums'; export * from './common.errors'; export * from './common.utils'; -export * from './featured'; -export * from '../processor/processor-task-queue'; -export * from './workers'; +export * from './dependencies'; diff --git a/src/processor/processor-task-queue/data-sources/processor-task.source.ts b/src/common/processor-task-queue/data-sources/processor-task.source.ts similarity index 100% rename from src/processor/processor-task-queue/data-sources/processor-task.source.ts rename to src/common/processor-task-queue/data-sources/processor-task.source.ts diff --git a/src/processor/processor-task-queue/data-sources/unsuccessful-processor-task.source.ts b/src/common/processor-task-queue/data-sources/unsuccessful-processor-task.source.ts similarity index 100% rename from src/processor/processor-task-queue/data-sources/unsuccessful-processor-task.source.ts rename to src/common/processor-task-queue/data-sources/unsuccessful-processor-task.source.ts diff --git a/src/processor/processor-task-queue/index.ts b/src/common/processor-task-queue/index.ts similarity index 100% rename from src/processor/processor-task-queue/index.ts rename to src/common/processor-task-queue/index.ts diff --git a/src/processor/processor-task-queue/processor-task-queue.config.ts b/src/common/processor-task-queue/processor-task-queue.config.ts similarity index 100% rename from src/processor/processor-task-queue/processor-task-queue.config.ts rename to src/common/processor-task-queue/processor-task-queue.config.ts diff --git a/src/processor/processor-task-queue/processor-task-queue.ts b/src/common/processor-task-queue/processor-task-queue.ts similarity index 97% rename from src/processor/processor-task-queue/processor-task-queue.ts rename to src/common/processor-task-queue/processor-task-queue.ts index 19d697b..76206e5 100644 --- a/src/processor/processor-task-queue/processor-task-queue.ts +++ b/src/common/processor-task-queue/processor-task-queue.ts @@ -6,8 +6,8 @@ import { ProcessorTaskSource } from './data-sources/processor-task.source'; import { ProcessorTask } from './processor-task'; import { UnsuccessfulProcessorTaskSource } from './data-sources/unsuccessful-processor-task.source'; import { ProcessorTaskQueueConfig } from './processor-task-queue.config'; -import { ErrorJson } from '../../common/workers/worker-message'; import { MongoConfig, MongoSource } from '@alien-worlds/storage-mongodb'; +import { ErrorJson } from '@alien-worlds/workers'; export class ProcessorTaskQueue { public static async create( diff --git a/src/processor/processor-task-queue/processor-task.ts b/src/common/processor-task-queue/processor-task.ts similarity index 94% rename from src/processor/processor-task-queue/processor-task.ts rename to src/common/processor-task-queue/processor-task.ts index 94b8b18..af1b813 100644 --- a/src/processor/processor-task-queue/processor-task.ts +++ b/src/common/processor-task-queue/processor-task.ts @@ -1,16 +1,13 @@ import crypto from 'crypto'; import { serialize } from 'v8'; -import { - parseToBigInt, - removeUndefinedProperties, -} from '@alien-worlds/api-core'; -import { ActionTrace, DeltaRow } from '../../common/blockchain/contract'; +import { parseToBigInt, removeUndefinedProperties } from '@alien-worlds/api-core'; import { DeltaProcessorContentModel, ProcessorTaskDocument, ProcessorTaskError, } from './processor-task.types'; import { MongoDB } from '@alien-worlds/storage-mongodb'; +import { ActionTraceDto } from '../contract-content/action-trace'; export enum ProcessorTaskType { Action = 'action', @@ -22,14 +19,14 @@ export class ProcessorTask { abi: string, mode: string, shipTraceMessageName: string, + shipMessageName: string, transactionId: string, - actionTrace: ActionTrace, + actionTrace: ActionTraceDto, blockNumber: bigint, blockTimestamp: Date, isFork: boolean ) { const { - shipMessageName, act: { account, name, data }, receipt, } = actionTrace; diff --git a/src/processor/processor-task-queue/processor-task.types.ts b/src/common/processor-task-queue/processor-task.types.ts similarity index 100% rename from src/processor/processor-task-queue/processor-task.types.ts rename to src/common/processor-task-queue/processor-task.types.ts diff --git a/src/common/unprocessed-block-queue/index.ts b/src/common/unprocessed-block-queue/index.ts new file mode 100644 index 0000000..135220e --- /dev/null +++ b/src/common/unprocessed-block-queue/index.ts @@ -0,0 +1,7 @@ +export * from './unprocessed-block-queue'; +export * from './unprocessed-block-queue.collection'; +export * from './unprocessed-block-queue.errors'; +export * from './unprocessed-block-queue.mapper'; +export * from './unprocessed-block-queue.mongo.collection'; +export * from './unprocessed-block-queue.mongo.mapper'; +export * from './unprocessed-block-queue.types'; diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.collection.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.collection.ts new file mode 100644 index 0000000..21e6df1 --- /dev/null +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.collection.ts @@ -0,0 +1,7 @@ +import { DataSource } from '@alien-worlds/api-core'; + +export abstract class UnprocessedBlockCollection extends DataSource { + public abstract next(): Promise; + + public abstract bytesSize(): Promise; +} diff --git a/src/reader/unprocessed-block-queue/unprocessed-block-queue.errors.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.errors.ts similarity index 100% rename from src/reader/unprocessed-block-queue/unprocessed-block-queue.errors.ts rename to src/common/unprocessed-block-queue/unprocessed-block-queue.errors.ts diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.mapper.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.mapper.ts new file mode 100644 index 0000000..628791d --- /dev/null +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.mapper.ts @@ -0,0 +1,7 @@ +import { Block } from '@alien-worlds/block-reader'; + +export abstract class UnprocessedBlockMapper { + public abstract toEntity(model: ModelType): Block; + + public abstract fromEntity(entity: Block): ModelType; +} diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.mongo.collection.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.mongo.collection.ts new file mode 100644 index 0000000..9664f9e --- /dev/null +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.mongo.collection.ts @@ -0,0 +1,35 @@ +import { DataSourceError } from '@alien-worlds/api-core'; +import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; +import { BlockMongoModel } from './unprocessed-block-queue.types'; +import { UnprocessedBlockCollection } from './unprocessed-block-queue.collection'; + +export class UnprocessedBlockMongoCollection + extends MongoCollectionSource + implements UnprocessedBlockCollection +{ + constructor(mongoSource: MongoSource) { + super(mongoSource, 'history_tools.unprocessed_blocks', { + indexes: [ + { + key: { 'this_block.block_num': 1 }, + unique: true, + background: true, + }, + ], + }); + } + + public async next(): Promise { + try { + const result = await this.collection.findOneAndDelete({}); + return result.value; + } catch (error) { + throw DataSourceError.createError(error); + } + } + + public async bytesSize(): Promise { + const stats = await this.collection.stats(); + return stats.size; + } +} diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.mongo.mapper.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.mongo.mapper.ts new file mode 100644 index 0000000..0ab3c39 --- /dev/null +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.mongo.mapper.ts @@ -0,0 +1,99 @@ +import { Block } from '@alien-worlds/block-reader'; +import { BlockMongoModel } from './unprocessed-block-queue.types'; +import { parseToBigInt } from '@alien-worlds/api-core'; +import { MongoDB } from '@alien-worlds/storage-mongodb'; + +export class UnprocessedBlockMongoMapper { + public toEntity(model: BlockMongoModel): Block { + const { block, traces, deltas, abi_version } = model; + let head; + let thisBlock; + let prevBlock; + let lastIrreversible; + + if (model.head) { + head = { + blockNumber: parseToBigInt(model.head.block_num), + blockId: model.head.block_id, + }; + } + + if (model.this_block) { + thisBlock = { + blockNumber: parseToBigInt(model.this_block.block_num), + blockId: model.this_block.block_id, + }; + } + + if (model.prev_block) { + prevBlock = { + blockNumber: parseToBigInt(model.prev_block.block_num), + blockId: model.prev_block.block_id, + }; + } + + if (model.last_irreversible) { + lastIrreversible = { + blockNumber: parseToBigInt(model.last_irreversible.block_num), + blockId: model.last_irreversible.block_id, + }; + } + + return new Block( + head, + lastIrreversible, + prevBlock, + thisBlock, + block.buffer, + traces.buffer, + deltas.buffer, + abi_version + ); + } + + public fromEntity(entity: Block): BlockMongoModel { + const { + head, + thisBlock, + prevBlock, + lastIrreversible, + block, + traces, + deltas, + id, + abiVersion, + } = entity; + + const document: BlockMongoModel = { + head: { + block_id: head.blockId, + block_num: MongoDB.Long.fromBigInt(head.blockNumber), + }, + this_block: { + block_id: thisBlock.blockId, + block_num: MongoDB.Long.fromBigInt(thisBlock.blockNumber), + }, + prev_block: { + block_id: prevBlock.blockId, + block_num: MongoDB.Long.fromBigInt(prevBlock.blockNumber), + }, + last_irreversible: { + block_id: lastIrreversible.blockId, + block_num: MongoDB.Long.fromBigInt(lastIrreversible.blockNumber), + }, + block: new MongoDB.Binary(block), + traces: new MongoDB.Binary(traces), + deltas: new MongoDB.Binary(deltas), + }; + + if (abiVersion) { + document.abi_version = abiVersion; + } + + if (id) { + document._id = new MongoDB.ObjectId(id); + } + + return document; + } +} diff --git a/src/reader/unprocessed-block-queue/unprocessed-block-queue.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.ts similarity index 56% rename from src/reader/unprocessed-block-queue/unprocessed-block-queue.ts rename to src/common/unprocessed-block-queue/unprocessed-block-queue.ts index 313647f..5168b78 100644 --- a/src/reader/unprocessed-block-queue/unprocessed-block-queue.ts +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.ts @@ -10,87 +10,35 @@ import { DuplicateBlocksError, UnprocessedBlocksOverloadError, } from './unprocessed-block-queue.errors'; -import { Block, BlockDocument } from '../../common/blockchain/block-reader/block'; -import { - isMongoConfig, - MongoCollectionSource, - MongoConfig, - MongoSource, -} from '@alien-worlds/storage-mongodb'; - -export class BlockMongoCollection extends MongoCollectionSource { - constructor(mongoSource: MongoSource) { - super(mongoSource, 'history_tools.unprocessed_blocks', { - indexes: [ - { - key: { 'this_block.block_num': 1 }, - unique: true, - background: true, - }, - ], - }); - } - - public async next(): Promise { - try { - const result = await this.collection.findOneAndDelete({}); - return result.value; - } catch (error) { - throw DataSourceError.createError(error); - } - } - - public async bytesSize(): Promise { - const stats = await this.collection.stats(); - return stats.size; - } -} +import { Block } from '@alien-worlds/block-reader'; +import { UnprocessedBlockCollection } from './unprocessed-block-queue.collection'; +import { BlockModel } from './unprocessed-block-queue.types'; +import { UnprocessedBlockMapper } from './unprocessed-block-queue.mapper'; export abstract class UnprocessedBlockQueueReader { public abstract next(): Promise>; } -export class UnprocessedBlockQueue implements UnprocessedBlockQueueReader { - private cache: Block[]; - private mongo: BlockMongoCollection; - private overloadHandler: (size: number) => void; - private beforeSendBatchHandler: () => void; - private afterSendBatchHandler: () => void; - - public static async create< - T extends UnprocessedBlockQueue | UnprocessedBlockQueueReader - >( - mongo: MongoConfig | MongoSource, - maxBytesSize?: number, - batchSize?: number - ): Promise { - let mongoSource: MongoSource; - if (isMongoConfig(mongo)) { - mongoSource = await MongoSource.create(mongo); - } else { - mongoSource = mongo; - } - return new UnprocessedBlockQueue( - mongoSource, - maxBytesSize || 0, - batchSize | 100 - ) as T; - } - - private constructor( - mongoSource: MongoSource, - private maxBytesSize: number, - private batchSize: number - ) { - this.mongo = new BlockMongoCollection(mongoSource); - this.cache = []; - } +export class UnprocessedBlockQueue + implements UnprocessedBlockQueueReader +{ + protected cache: Block[] = []; + protected overloadHandler: (size: number) => void; + protected beforeSendBatchHandler: () => void; + protected afterSendBatchHandler: () => void; + + constructor( + protected collection: UnprocessedBlockCollection, + protected mapper: UnprocessedBlockMapper, + protected maxBytesSize: number, + protected batchSize: number + ) {} private async sendBatch() { const addedBlockNumbers = []; this.beforeSendBatchHandler(); - const documnets = this.cache.map(block => block.toDocument()); - const result = await this.mongo.insert(documnets); + const documnets = this.cache.map(block => this.mapper.fromEntity(block)); + const result = await this.collection.insert(documnets); result.forEach(document => { addedBlockNumbers.push(parseToBigInt(document.this_block.block_num)); }); @@ -101,7 +49,7 @@ export class UnprocessedBlockQueue implements UnprocessedBlockQueueReader { const min = sorted[0]; const max = sorted.reverse()[0]; - const currentSize = await this.mongo.bytesSize(); + const currentSize = await this.collection.bytesSize(); if (currentSize >= this.maxBytesSize) { this.overloadHandler(currentSize); throw new UnprocessedBlocksOverloadError(min, max); @@ -115,7 +63,7 @@ export class UnprocessedBlockQueue implements UnprocessedBlockQueueReader { public async getBytesSize(): Promise> { try { - const currentSize = await this.mongo.bytesSize(); + const currentSize = await this.collection.bytesSize(); return Result.withContent(currentSize); } catch (error) { return Result.withFailure(Failure.fromError(error)); @@ -149,15 +97,15 @@ export class UnprocessedBlockQueue implements UnprocessedBlockQueueReader { public async next(): Promise> { try { - const document = await this.mongo.next(); + const document = await this.collection.next(); if (document) { if (this.maxBytesSize > -1 && this.afterSendBatchHandler) { - if ((await this.mongo.count()) === 0 && this.afterSendBatchHandler) { + if ((await this.collection.count()) === 0 && this.afterSendBatchHandler) { this.afterSendBatchHandler(); } } - return Result.withContent(Block.fromDocument(document)); + return Result.withContent(this.mapper.toEntity(document)); } return Result.withFailure(Failure.fromError(new BlockNotFoundError())); } catch (error) { @@ -168,11 +116,11 @@ export class UnprocessedBlockQueue implements UnprocessedBlockQueueReader { public async getMax(): Promise> { try { - const documents = await this.mongo.aggregate({ + const documents = await this.collection.aggregate({ pipeline: [{ $sort: { 'this_block.block_num': -1 } }, { $limit: 1 }], }); if (documents.length > 0) { - return Result.withContent(Block.fromDocument(documents[0])); + return Result.withContent(this.mapper.toEntity(documents[0])); } return Result.withFailure(Failure.fromError(new BlockNotFoundError())); } catch (error) { diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.types.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.types.ts new file mode 100644 index 0000000..28f3e81 --- /dev/null +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.types.ts @@ -0,0 +1,36 @@ +import { MongoDB } from '@alien-worlds/storage-mongodb'; + +export type BlockNumberWithIdDocument = { + block_num?: MongoDB.Long; + block_id?: string; +}; + +export type BlockMongoModel = { + _id?: MongoDB.ObjectId; + head?: BlockNumberWithIdDocument; + this_block?: BlockNumberWithIdDocument; + last_irreversible?: BlockNumberWithIdDocument; + prev_block?: BlockNumberWithIdDocument; + block?: MongoDB.Binary; + traces?: MongoDB.Binary; + deltas?: MongoDB.Binary; + abi_version?: string; + [key: string]: unknown; +}; + +export type BlockNumberWithId = { + block_num?: unknown; + block_id?: string; +}; + +export type BlockModel = { + head?: BlockNumberWithId; + this_block?: BlockNumberWithId; + last_irreversible?: BlockNumberWithId; + prev_block?: BlockNumberWithId; + block?: unknown; + traces?: unknown; + deltas?: unknown; + abi_version?: unknown; + [key: string]: unknown; +}; diff --git a/src/config/config.types.ts b/src/config/config.types.ts index f05bf20..1f15e1a 100644 --- a/src/config/config.types.ts +++ b/src/config/config.types.ts @@ -4,6 +4,11 @@ import { ReaderConfig } from '../reader'; import { FilterConfig } from '../filter'; import { ProcessorConfig } from '../processor'; +export type BlockchainConfig = { + endpoint: string; + chainId: string; +}; + export type HistoryToolsConfig = { api: ApiConfig; bootstrap: BootstrapConfig; diff --git a/src/config/index.ts b/src/config/index.ts index b95033b..2d81c12 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -1,34 +1,32 @@ -import { ProcessorTaskQueueConfig } from '../processor/processor-task-queue/processor-task-queue.config'; +import { ProcessorTaskQueueConfig } from '../common/processor-task-queue/processor-task-queue.config'; import { ConfigVars, parseToBigInt } from '@alien-worlds/api-core'; import { ApiConfig } from '../api'; import { BootstrapConfig, BootstrapCommandOptions } from '../bootstrap'; import { ReaderConfig, ReaderCommandOptions } from '../reader'; import { FilterConfig, FilterCommandOptions } from '../filter'; import { ProcessorConfig, ProcessorCommandOptions } from '../processor'; -import { HistoryToolsConfig } from './config.types'; +import { BlockchainConfig, HistoryToolsConfig } from './config.types'; import { AbisConfig, AbisServiceConfig, BlockRangeScanConfig, - BlockReaderConfig, - ContractReaderConfig, FeaturedConfig, - WorkersConfig, } from '../common'; import { buildMongoConfig } from '@alien-worlds/storage-mongodb'; import { buildBroadcastConfig } from '@alien-worlds/broadcast'; +import { BlockReaderConfig } from '@alien-worlds/block-reader'; +import { WorkersConfig } from '@alien-worlds/workers'; export * from './config.types'; -export const buildBlockchainConfig = ( - vars: ConfigVars -): { endpoint: string; chainId: string } => ({ +export const buildBlockchainConfig = (vars: ConfigVars): BlockchainConfig => ({ endpoint: vars.getStringEnv('BLOCKCHAIN_ENDPOINT'), chainId: vars.getStringEnv('BLOCKCHAIN_CHAIN_ID'), }); -export const buildContractReaderConfig = (vars: ConfigVars): ContractReaderConfig => ({ - url: vars.getStringEnv('HYPERION_URL'), +export const buildFeaturedConfig = (vars: ConfigVars): FeaturedConfig => ({ + rpcUrl: vars.getStringEnv('BLOCKCHAIN_ENDPOINT'), + serviceUrl: vars.getStringEnv('HYPERION_URL'), }); export const buildBlockRangeScanConfig = ( @@ -45,13 +43,9 @@ export const buildAbisServiceConfig = (vars: ConfigVars): AbisServiceConfig => ( filter: vars.getStringEnv('ABIS_SERVICE_FILTER'), }); -export const buildAbisConfig = ( - vars: ConfigVars, - featured: FeaturedConfig -): AbisConfig => ({ +export const buildAbisConfig = (vars: ConfigVars): AbisConfig => ({ service: buildAbisServiceConfig(vars), mongo: buildMongoConfig(vars), - featured, }); export const buildBlockReaderConfig = (vars: ConfigVars): BlockReaderConfig => ({ @@ -98,13 +92,11 @@ export const buildApiConfig = (vars: ConfigVars): ApiConfig => ({ export const buildBootstrapConfig = ( vars: ConfigVars, - featured: FeaturedConfig, options?: BootstrapCommandOptions ): BootstrapConfig => ({ - mongo: buildMongoConfig(vars), broadcast: buildBroadcastConfig(vars), blockchain: buildBlockchainConfig(vars), - contractReader: buildContractReaderConfig(vars), + featured: buildFeaturedConfig(vars), scanner: buildBlockRangeScanConfig(vars, options?.scanKey), startBlock: options?.startBlock ? parseToBigInt(options?.startBlock) @@ -118,7 +110,6 @@ export const buildBootstrapConfig = ( : null, startFromHead: vars.getBooleanEnv('START_FROM_HEAD'), mode: options?.mode || vars.getStringEnv('MODE'), - featured, abis: buildAbisServiceConfig(vars), maxBlockNumber: vars.getNumberEnv('MAX_BLOCK_NUMBER'), }); @@ -145,27 +136,24 @@ export const buildReaderConfig = ( export const buildFilterConfig = ( vars: ConfigVars, - featured: FeaturedConfig, options?: FilterCommandOptions ): FilterConfig => ({ mode: options?.mode || vars.getStringEnv('MODE'), broadcast: buildBroadcastConfig(vars), workers: buildFilterWorkersConfig(vars, options), - featured, - abis: buildAbisConfig(vars, featured), - contractReader: buildContractReaderConfig(vars), + abis: buildAbisConfig(vars), + featured: buildFeaturedConfig(vars), mongo: buildMongoConfig(vars), queue: buildProcessorTaskQueueConfig(vars), }); export const buildProcessorConfig = ( vars: ConfigVars, - featured: FeaturedConfig, options?: ProcessorCommandOptions ): ProcessorConfig => ({ broadcast: buildBroadcastConfig(vars), workers: buildProcessorWorkersConfig(vars, options?.threads), - featured, + featured: buildFeaturedConfig(vars), mongo: buildMongoConfig(vars), queue: buildProcessorTaskQueueConfig(vars), }); @@ -175,8 +163,8 @@ export const buildHistoryToolsConfig = ( featured: FeaturedConfig ): HistoryToolsConfig => ({ api: buildApiConfig(vars), - bootstrap: buildBootstrapConfig(vars, featured), + bootstrap: buildBootstrapConfig(vars), reader: buildReaderConfig(vars), - filter: buildFilterConfig(vars, featured), + filter: buildFilterConfig(vars), processor: buildProcessorConfig(vars, featured), }); diff --git a/src/filter/deserialized-block.ts b/src/filter/deserialized-block.ts deleted file mode 100644 index b148783..0000000 --- a/src/filter/deserialized-block.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { - Abi, - Delta, - SignedBlock, - SignedBlockJson, - Trace, -} from '../common/blockchain'; -import { TraceJson } from '../common/blockchain/contract/trace'; -import { DeltaJson } from '../common/blockchain/contract/delta'; -import { deserializeMessage } from '../reader'; -import { Block, BlockNumberWithId } from '../common/blockchain/block-reader/block'; - -export class DeserializedBlock { - public static create(block: Block, abi: Abi): DeserializedBlock { - const { head, lastIrreversible, prevBlock, thisBlock } = block; - const types = abi.getTypesMap(); - let traces: Trace[] = []; - let deltas: Delta[] = []; - let signedBlock: SignedBlock; - - if (block.block && block.block.length > 0) { - const deserializedBlock = deserializeMessage( - 'signed_block', - block.block, - types - ); - signedBlock = SignedBlock.create(deserializedBlock); - } - - if (block.traces && block.traces.length > 0) { - const tracesByType = deserializeMessage<[[string, TraceJson]]>( - 'transaction_trace[]', - block.traces, - types - ); - traces = tracesByType.map(([shipMessageName, traceJson]) => - Trace.create(shipMessageName, traceJson) - ); - } - - if (block.deltas && block.deltas.length > 0) { - const deltasByType = deserializeMessage<[[string, DeltaJson]]>( - 'table_delta[]', - block.deltas, - types - ); - deltas = deltasByType.map(([shipMessageName, deltaJson]) => - Delta.create(shipMessageName, deltaJson) - ); - } - - return new DeserializedBlock( - head, - thisBlock, - prevBlock, - lastIrreversible, - signedBlock, - traces, - deltas - ); - } - - private constructor( - public readonly head: BlockNumberWithId, - public readonly thisBlock: BlockNumberWithId, - public readonly prevBlock: BlockNumberWithId, - public readonly lastIrreversible: BlockNumberWithId, - public readonly block: SignedBlock, - public readonly traces: Trace[], - public readonly deltas: Delta[] - ) {} -} diff --git a/src/filter/filter.command.ts b/src/filter/filter.command.ts new file mode 100644 index 0000000..c28bbb1 --- /dev/null +++ b/src/filter/filter.command.ts @@ -0,0 +1,10 @@ +import { Command } from 'commander'; + +export const filterCommand = new Command(); + +filterCommand + .version('1.0', '-v, --version') + .option('-k, --scan-key ', 'Scan key') + .option('-m, --mode ', 'Mode (default/replay/test)') + .option('-t, --threads ', 'Number of threads') + .parse(process.argv); diff --git a/src/filter/filter.consts.ts b/src/filter/filter.consts.ts index 56baf83..d4e164b 100644 --- a/src/filter/filter.consts.ts +++ b/src/filter/filter.consts.ts @@ -1 +1,2 @@ export const filterWorkerLoaderPath = `${__dirname}/filter.worker-loader`; +export const filterWorkerLoaderDependenciesPath = `${__dirname}/filter.worker-loader.dependencies`; diff --git a/src/filter/filter.dependencies.ts b/src/filter/filter.dependencies.ts new file mode 100644 index 0000000..171ca82 --- /dev/null +++ b/src/filter/filter.dependencies.ts @@ -0,0 +1,24 @@ +import { Result, Serializer } from '@alien-worlds/api-core'; +import { Dependencies } from '../common/dependencies'; +import { FilterConfig } from './filter.types'; +import { BroadcastClient } from '@alien-worlds/broadcast'; +import { WorkerPool } from '@alien-worlds/workers'; +import { UnprocessedBlockQueue } from '../common'; + +/** + * An abstract class representing a Filter dependencies. + * @class FilterDependencies + */ +export abstract class FilterDependencies extends Dependencies { + /** + * The broadcast client used for communication. + * @type {BroadcastClient} + */ + public broadcastClient: BroadcastClient; + + public workerPool: WorkerPool; + public unprocessedBlockQueue: UnprocessedBlockQueue; + public serializer: Serializer; + + public abstract initialize(config: FilterConfig): Promise; +} diff --git a/src/filter/filter.runner.ts b/src/filter/filter.runner.ts index 330d2c6..57a23ba 100644 --- a/src/filter/filter.runner.ts +++ b/src/filter/filter.runner.ts @@ -1,33 +1,10 @@ import { log } from '@alien-worlds/api-core'; -import { WorkerMessage, WorkerPool } from '../common/workers'; -import { FilterAddons, FilterConfig } from './filter.types'; -import { filterWorkerLoaderPath } from './filter.consts'; import { BlockNotFoundError } from '../reader/unprocessed-block-queue/unprocessed-block-queue.errors'; -import { UnprocessedBlockQueue, UnprocessedBlockQueueReader } from '../reader'; -import { BlockJson } from '../common/blockchain/block-reader/block'; +import { UnprocessedBlockQueueReader } from '../reader'; +import { BlockJson } from '@alien-worlds/block-reader'; +import { WorkerMessage, WorkerPool } from '@alien-worlds/workers'; export class FilterRunner { - public static async create(config: FilterConfig, addons: FilterAddons) { - const { workers } = config; - const { matchers } = addons || {}; - const blocks = await UnprocessedBlockQueue.create( - config.mongo - ); - - const workerPool = await WorkerPool.create({ - ...workers, - sharedData: { config, matchers }, - workerLoaderPath: filterWorkerLoaderPath, - }); - const runner = new FilterRunner(workerPool, blocks); - - workerPool.onWorkerRelease(() => runner.next()); - - log(` * Worker Pool (max ${workerPool.workerMaxCount} workers) ... [ready]`); - - return runner; - } - private interval: NodeJS.Timeout; private loop: boolean; private transitionHandler: (...args: unknown[]) => void | Promise; diff --git a/src/filter/filter.types.ts b/src/filter/filter.types.ts index e49c281..8d1c67c 100644 --- a/src/filter/filter.types.ts +++ b/src/filter/filter.types.ts @@ -1,12 +1,15 @@ -import { MongoConfig, BroadcastConfig } from '@alien-worlds/api-core'; -import { FeaturedConfig, FeaturedMatchers } from '../common/featured'; -import { ProcessorTaskQueueConfig } from '../processor/processor-task-queue/processor-task-queue.config'; -import { WorkersConfig } from '../common/workers'; +import { ProcessorTaskQueueConfig } from '../common/processor-task-queue/processor-task-queue.config'; import { AbisConfig } from '../common/abis'; -import { ContractReaderConfig } from '../common/blockchain'; +import { WorkersConfig } from '@alien-worlds/workers'; +import { BroadcastConfig } from '@alien-worlds/broadcast'; +import { FeaturedConfig } from '../common'; +import { MongoConfig } from '@alien-worlds/storage-mongodb'; +import { UnknownObject } from '@alien-worlds/api-core'; +import { TransactionJson } from '../common/contract-content/transaction'; export type FilterSharedData = { config: FilterConfig; + featuredJson: UnknownObject; }; export type FilterCommandOptions = { @@ -20,13 +23,139 @@ export type FilterConfig = { workers: WorkersConfig; featured: FeaturedConfig; abis: AbisConfig; - contractReader: ContractReaderConfig; mongo: MongoConfig; queue: ProcessorTaskQueueConfig; [key: string]: unknown; }; export type FilterAddons = { - matchers?: FeaturedMatchers; + matchers?: unknown; [key: string]: unknown; }; + +export type BlockNumberWithId = { + block_num: string; + block_id: string; +}; + +export type SignedBlockJson = { + timestamp: string; + producer: string; + confirmed: number; + previous: string; + transaction_mroot: string; + action_mroot: string; + schedule_version: number; + new_producers: unknown; + header_extensions: unknown[]; + producer_signature: string; + transactions: TransactionJson[]; +}; + +export type DeserializedBlock = { + head?: BlockNumberWithId; + this_block?: BlockNumberWithId; + last_irreversible?: BlockNumberWithId; + prev_block?: BlockNumberWithId; + block?: SignedBlockJson; + traces?: [[string, TraceJson]]; + deltas?: [[string, DeltaJson]]; + abi_version?: string; +}; + +// + +export type AuthSequenceJson = { + account: string; + sequence: string; +}; + +export type ReceiptJson = { + receiver: string; + act_digest: string; + global_sequence: string; + recv_sequence: string; + auth_sequence: AuthSequenceJson[]; + code_sequence: number; + abi_sequence: number; +}; + +export type ReceiptByNameDto = [string, ReceiptJson]; + +export type ActAuthJson = { + actor: string; + permission: string; +}; + +export type ActJson = { + account: string; + name: string; + authorization: ActAuthJson; + data: Uint8Array; +}; + +export type ActionTraceDto = { + ship_message_name?: string; + action_ordinal?: number; + creator_action_ordinal?: number; + receipt?: ReceiptByNameDto; + receiver?: string; + act?: ActJson; + context_free?: boolean; + elapsed?: string; + console?: string; + account_ram_deltas?: unknown[]; + except?: unknown; + error_code?: string | number; +}; + +export type ActionTraceByNameDto = [string, ActionTraceDto]; + +export type PartialJson = { + expiration: string; + ref_block_num: number; + ref_block_prefix: number; + max_net_usage_words: number; + max_cpu_usage_ms: number; + delay_sec: number; + transaction_extensions: unknown[]; + signatures: unknown[]; + context_free_data: unknown[]; +}; + +export type PartialByTypeJson = [string, PartialJson]; + +export type TraceJson = { + id?: string; + status?: number; + cpu_usage_us?: number; + net_usage_words?: number; + elapsed?: string; + net_usage?: string; + scheduled?: boolean; + action_traces?: ActionTraceByNameDto[]; + account_ram_delta?: unknown; + except?: unknown; + error_code?: number | string; + failed_dtrx_trace?: unknown; + partial?: PartialByTypeJson; +}; + +export type TraceByNameJson = [string, TraceJson]; + +export type DeltaRowDto = { + present?: number; + data?: Uint8Array; +}; + +export type DeltaJson = { + name?: string; + rows?: DeltaRowDto[]; +}; + +export type DeltaByNameDto = [string, DeltaJson]; + +export type DeltaRowModel = { + present?: number; + data?: Uint8Array; +}; diff --git a/src/filter/filter.utils.ts b/src/filter/filter.utils.ts deleted file mode 100644 index f1a7665..0000000 --- a/src/filter/filter.utils.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { log } from '@alien-worlds/api-core'; -import { Serialize } from 'eosjs'; - -type DeltaAllocation = { - code: string; - scope: string; - table: string; -}; - -export const extractAllocationFromDeltaRow = (value: Uint8Array): DeltaAllocation => { - const sb = new Serialize.SerialBuffer({ - textEncoder: new TextEncoder(), - textDecoder: new TextDecoder(), - array: value, - }); - - try { - sb.get(); // ? - const code = sb.getName(); - const scope = sb.getName(); - const table = sb.getName(); - - return { code, scope, table }; - } catch (error) { - log( - `Unable to extract data, most likely data cannot be deserialized using eosjs.Serialize or the contract does not contain tables. ${error.message}` - ); - return null; - } -}; - -export const extractValues = (value: string): Set => { - const result = new Set(); - if (!value || value.includes('*')) { - result.add('*'); - } else { - value.split(',').forEach(entry => { - if (/^(\*|[A-Za-z0-9_.]*)$/g.test(entry)) { - result.add(entry); - } - }); - } - return result; -}; diff --git a/src/filter/filter.worker-loader.dependencies.ts b/src/filter/filter.worker-loader.dependencies.ts new file mode 100644 index 0000000..dc8a201 --- /dev/null +++ b/src/filter/filter.worker-loader.dependencies.ts @@ -0,0 +1,22 @@ +import { UnknownObject, Serializer } from '@alien-worlds/api-core'; +import { FilterConfig } from './filter.types'; +import { WorkerLoaderDependencies } from '@alien-worlds/workers'; +import { Abis, Featured, ProcessorTaskQueue } from '../common'; +import { ShipAbis } from '@alien-worlds/block-reader'; + +/** + * An abstract class representing a FilterWorkerLoader dependencies. + * @class FilterWorkerLoaderDependencies + */ +export abstract class FilterWorkerLoaderDependencies extends WorkerLoaderDependencies { + public processorTaskQueue: ProcessorTaskQueue; + public abis: Abis; + public shipAbis: ShipAbis; + public featured: Featured; + public serializer: Serializer; + + public abstract initialize( + config: FilterConfig, + featuredJson: UnknownObject + ): Promise; +} diff --git a/src/filter/filter.worker-loader.ts b/src/filter/filter.worker-loader.ts index cac25ab..b7c2a92 100644 --- a/src/filter/filter.worker-loader.ts +++ b/src/filter/filter.worker-loader.ts @@ -1,62 +1,20 @@ -import { MongoSource } from '@alien-worlds/api-core'; -import { Worker } from '../common/workers'; -import { DefaultWorkerLoader } from '../common/workers/worker-loader'; +import { Worker, DefaultWorkerLoader } from '@alien-worlds/workers'; import { FilterSharedData } from './filter.types'; -import { - FeaturedContractContent, - FeaturedDelta, - FeaturedTrace, -} from '../common/featured'; -import { ContractReader } from '../common/blockchain'; -import { Abis } from '../common/abis'; -import { ProcessorTaskQueue } from '../processor/processor-task-queue'; import FilterWorker from './filter.worker'; -import { ShipAbis } from '../common/ship/ship-abis'; - -export default class FilterWorkerLoader extends DefaultWorkerLoader { - private featuredTraces: FeaturedTrace[]; - private featuredDeltas: FeaturedDelta[]; - private contractReader: ContractReader; - private processorTaskQueue: ProcessorTaskQueue; - private abis: Abis; - private shipAbis: ShipAbis; +import { FilterWorkerLoaderDependencies } from './filter.worker-loader.dependencies'; +export default class FilterWorkerLoader extends DefaultWorkerLoader< + FilterSharedData, + FilterWorkerLoaderDependencies +> { public async setup(sharedData: FilterSharedData): Promise { - super.setup(sharedData); - const { - config: { mongo, featured, abis, contractReader, queue }, - } = sharedData; - const { traces, deltas } = new FeaturedContractContent(featured).toJson(); - - const mongoSource = await MongoSource.create(mongo); - this.abis = await Abis.create(mongoSource, abis.service, featured); - this.contractReader = await ContractReader.create(contractReader, mongoSource); - this.processorTaskQueue = await ProcessorTaskQueue.create(mongoSource, true, queue); - this.shipAbis = await ShipAbis.create(mongoSource); - this.featuredDeltas = deltas; - this.featuredTraces = traces; + const { config } = sharedData; + await super.setup(sharedData, config); } public async load(): Promise { - const { - abis, - shipAbis, - contractReader, - featuredTraces, - featuredDeltas, - processorTaskQueue, - sharedData, - } = this; - return new FilterWorker( - { - shipAbis, - abis, - contractReader, - featuredTraces, - featuredDeltas, - processorTaskQueue, - }, - sharedData - ); + const { dependencies, sharedData } = this; + + return new FilterWorker(dependencies, sharedData); } } diff --git a/src/filter/filter.worker.ts b/src/filter/filter.worker.ts index baa8925..9866af9 100644 --- a/src/filter/filter.worker.ts +++ b/src/filter/filter.worker.ts @@ -1,117 +1,99 @@ -import { log } from '@alien-worlds/api-core'; -import { - FeaturedDelta, - FeaturedDeltas, - FeaturedTrace, - FeaturedTraces, -} from '../common/featured'; -import { Worker } from '../common/workers'; -import { DeserializedBlock } from './deserialized-block'; +import { Serializer, log, parseToBigInt } from '@alien-worlds/api-core'; +import { Worker } from '@alien-worlds/workers'; +import { AbiNotFoundError, BlockJson, ShipAbis } from '@alien-worlds/block-reader'; +import { Featured } from '../common'; import { Abis } from '../common/abis'; -import { ContractReader } from '../common/blockchain'; import { isSetAbiAction } from '../common/common.utils'; -import { ProcessorTask, ProcessorTaskQueue } from '../processor/processor-task-queue'; -import { FilterSharedData } from './filter.types'; -import { extractAllocationFromDeltaRow } from './filter.utils'; -import { Block, BlockJson } from '../common/blockchain/block-reader/block'; -import { ShipAbis } from '../common/ship/ship-abis'; +import { ProcessorTask, ProcessorTaskQueue } from '../common/processor-task-queue'; +import { DeserializedBlock, FilterSharedData } from './filter.types'; export default class FilterWorker extends Worker { - protected shipAbis: ShipAbis; - protected abis: Abis; - protected contractReader: ContractReader; - protected processorTaskQueue: ProcessorTaskQueue; - protected featuredTraces: FeaturedTrace[]; - protected featuredDeltas: FeaturedDelta[]; - constructor( - components: { + protected dependencies: { shipAbis: ShipAbis; abis: Abis; - contractReader: ContractReader; + featured: Featured; processorTaskQueue: ProcessorTaskQueue; - featuredTraces: FeaturedTrace[]; - featuredDeltas: FeaturedDelta[]; + serializer: Serializer; }, - sharedData: FilterSharedData + protected sharedData: FilterSharedData ) { super(); - const { - abis, - contractReader, - featuredTraces, - featuredDeltas, - processorTaskQueue, - shipAbis, - } = components; - this.shipAbis = shipAbis; - this.abis = abis; - this.contractReader = contractReader; - this.processorTaskQueue = processorTaskQueue; - this.featuredTraces = featuredTraces; - this.featuredDeltas = featuredDeltas; - this.sharedData = sharedData; } public async createActionProcessorTasks( deserializedBlock: DeserializedBlock ): Promise { const { - featuredTraces, - abis, - contractReader, + dependencies: { abis, featured }, sharedData: { config }, } = this; const { traces, - thisBlock, + this_block, block: { timestamp }, - prevBlock, + prev_block, } = deserializedBlock; - const featured = new FeaturedTraces(featuredTraces); const list: ProcessorTask[] = []; - for (const trace of traces) { - const { id, actionTraces, shipTraceMessageName } = trace; + for (const [traceType, trace] of traces) { + const { id, action_traces } = trace; - for (const actionTrace of actionTraces) { + for (const [actionType, actionTrace] of action_traces) { const { act: { account, name }, } = actionTrace; - const matchedTraces = await featured.get({ - shipTraceMessageName, - action: name, - contract: account, - }); - - if (matchedTraces.length > 0) { + if (featured.isFeatured(account)) { try { // If the block in which the contract was created cannot be found or // its index is higher than the current block number, skip it, // the contract did not exist at that time - const initBlockNumber = await contractReader.getInitialBlockNumber(account); - if (initBlockNumber === -1n || initBlockNumber > thisBlock.blockNumber) { + const { content: contracts, failure } = await featured.readContracts([ + account, + ]); + + if (failure) { + log(failure.error); + continue; + } + const contract = contracts[0]; + if ( + contract.initialBlockNumber === -1n || + contract.initialBlockNumber > parseToBigInt(this_block.block_num) + ) { continue; } // get ABI from the database and if it does not exist, try to fetch it - const abi = await abis.getAbi(thisBlock.blockNumber, account, true); - if (!abi && isSetAbiAction(account, name) === false) { - log( - `Action-trace {block_number: ${thisBlock.blockNumber}, account: ${account}, name: ${name}}: no ABI was found. This can be a problem in reading the content.` - ); + const { content: abi, failure: getAbiFailure } = await abis.getAbi( + parseToBigInt(this_block.block_num), + account, + true + ); + + if (getAbiFailure) { + if ( + getAbiFailure.error instanceof AbiNotFoundError && + isSetAbiAction(account, name) === false + ) { + log( + `Action-trace {block_number: ${this_block.block_num}, account: ${account}, name: ${name}}: no ABI was found. This can be a problem in reading the content.` + ); + } } + list.push( ProcessorTask.createActionProcessorTask( abi ? abi.hex : '', config.mode, - shipTraceMessageName, + traceType, + actionType, id, actionTrace, - thisBlock.blockNumber, - timestamp, - thisBlock.blockNumber <= prevBlock.blockNumber + parseToBigInt(this_block.block_num), + new Date(timestamp), + parseToBigInt(this_block.block_num) <= parseToBigInt(prev_block.block_num) ) ); } catch (error) { @@ -128,73 +110,79 @@ export default class FilterWorker extends Worker { deserializedBlock: DeserializedBlock ): Promise { const { - featuredDeltas, - abis, - contractReader, + dependencies: { abis, featured, serializer }, sharedData: { config }, } = this; const { deltas, - thisBlock, + this_block, block: { timestamp }, - prevBlock, + prev_block, } = deserializedBlock; const list: ProcessorTask[] = []; - const featured = new FeaturedDeltas(featuredDeltas); - for (const delta of deltas) { - const { name, shipDeltaMessageName } = delta; - const allocations = delta.rows.map(row => extractAllocationFromDeltaRow(row.data)); + for (const [type, delta] of deltas) { + const { name, rows } = delta; + const tableRows = rows + ? rows.map(row => serializer.deserializeTableRow(row.data)) + : []; - for (let i = 0; i < delta.rows.length; i++) { - const row = delta.rows[i]; - const allocation = allocations[i]; + for (let i = 0; i < tableRows.length; i++) { + const tableRow = tableRows[i]; - if (!allocation) { + if (!tableRow) { // contract allocation cannot be extracted // The contract may not contain tables or may be corrupted continue; } - - const { code, scope, table } = allocation; - const matchedDeltas = await featured.get({ - shipDeltaMessageName, - name, - code, - scope, - table, - }); - - if (matchedDeltas.length > 0) { + const { table, code, scope } = tableRow; + if (featured.isFeatured(code)) { try { // If the block in which the contract was created cannot be found or // its index is higher than the current block number, skip it, // the contract did not exist at that time - const initBlockNumber = await contractReader.getInitialBlockNumber(code); - if (initBlockNumber === -1n || initBlockNumber > thisBlock.blockNumber) { + const { content: contracts, failure } = await featured.readContracts([code]); + + if (failure) { + log(failure.error); + continue; + } + const contract = contracts[0]; + if ( + contract.initialBlockNumber === -1n || + contract.initialBlockNumber > parseToBigInt(this_block.block_num) + ) { continue; } // get ABI from the database and if it does not exist, try to fetch it - const abi = await abis.getAbi(thisBlock.blockNumber, code, true); - if (!abi) { - log( - `Delta {block_number: ${thisBlock.blockNumber}, code: ${code}, scope: ${scope}, table: ${table}}: no ABI was found. This can be a problem in reading the content.` - ); + const { content: abi, failure: getAbiFailure } = await abis.getAbi( + parseToBigInt(this_block.block_num), + code, + true + ); + + if (getAbiFailure) { + if (getAbiFailure.error instanceof AbiNotFoundError) { + log( + `Delta {block_number: ${this_block.block_num}, code: ${code}, scope: ${scope}, table: ${table}}: no ABI was found. This can be a problem in reading the content.` + ); + } } + list.push( ProcessorTask.createDeltaProcessorTask( abi ? abi.hex : '', config.mode, - shipDeltaMessageName, + type, name, code, scope, table, - thisBlock.blockNumber, - timestamp, - row, - thisBlock.blockNumber <= prevBlock.blockNumber + parseToBigInt(this_block.block_num), + new Date(timestamp), + tableRow, + parseToBigInt(this_block.block_num) <= parseToBigInt(prev_block.block_num) ) ); } catch (error) { @@ -209,7 +197,9 @@ export default class FilterWorker extends Worker { public async run(json: BlockJson): Promise { try { - const { processorTaskQueue, shipAbis } = this; + const { + dependencies: { serializer, shipAbis, processorTaskQueue }, + } = this; const { content: abi, failure } = await shipAbis.getAbi(json.abi_version); if (failure) { @@ -217,24 +207,29 @@ export default class FilterWorker extends Worker { this.reject(failure.error); } - const deserializedBlock = DeserializedBlock.create(Block.fromJson(json), abi); + const deserializedBlock = serializer.deserializeBlock( + json, + abi.toHex() + ); const { - thisBlock: { blockNumber }, + this_block: { block_num }, } = deserializedBlock; + const [actionProcessorTasks, deltaProcessorTasks] = await Promise.all([ this.createActionProcessorTasks(deserializedBlock), this.createDeltaProcessorTasks(deserializedBlock), ]); + const tasks = [...actionProcessorTasks, ...deltaProcessorTasks]; if (tasks.length > 0) { log( - `Block #${blockNumber} contains ${actionProcessorTasks.length} actions and ${deltaProcessorTasks.length} deltas to process (${tasks.length} tasks in total).` + `Block #${block_num} contains ${actionProcessorTasks.length} actions and ${deltaProcessorTasks.length} deltas to process (${tasks.length} tasks in total).` ); processorTaskQueue.addTasks(tasks); } else { log( - `The block (${blockNumber}) does not contain actions and deltas that could be processed.` + `The block (${block_num}) does not contain actions and deltas that could be processed.` ); } diff --git a/src/filter/index.ts b/src/filter/index.ts index d4633ce..bc478cb 100644 --- a/src/filter/index.ts +++ b/src/filter/index.ts @@ -1,7 +1,9 @@ -export * from './deserialized-block'; +export * from './filter.command'; export * from './filter.consts'; +export * from './filter.dependencies'; +export * from './filter.runner'; export * from './filter.types'; -export * from './filter.worker-loader'; export * from './filter.worker'; -export * from './filter.runner'; +export * from './filter.worker-loader'; +export * from './filter.worker-loader.dependencies'; export * from './start-filter'; diff --git a/src/filter/start-filter.ts b/src/filter/start-filter.ts index 35d109a..2239ab6 100644 --- a/src/filter/start-filter.ts +++ b/src/filter/start-filter.ts @@ -1,46 +1,58 @@ -import { Broadcast, log } from '@alien-worlds/api-core'; -import { FilterAddons, FilterConfig } from './filter.types'; +import { ConfigVars, log } from '@alien-worlds/api-core'; +import { FilterAddons, FilterCommandOptions, FilterConfig } from './filter.types'; import { InternalBroadcastChannel, - InternalBroadcastClientName, InternalBroadcastMessageName, ProcessorBroadcastMessage, } from '../broadcast'; -import { InternalBroadcastMessage } from '../broadcast/internal-broadcast.message'; import { FilterRunner } from './filter.runner'; import { FilterBroadcastMessage } from '../broadcast/messages/filter-broadcast.message'; +import { buildFilterConfig } from '../config'; +import { filterCommand } from './filter.command'; +import { FilterDependencies } from './filter.dependencies'; +import { BroadcastMessage } from '@alien-worlds/broadcast'; -/** - * - * @param featuredContent - * @param broadcastMessageMapper - * @param config - */ -export const startFilter = async (config: FilterConfig, addons?: FilterAddons) => { +export const filter = async ( + config: FilterConfig, + dependencies: FilterDependencies, + addons?: FilterAddons +) => { log(`Filter ... [starting]`); - const broadcast = await Broadcast.createClient({ - ...config.broadcast, - clientName: InternalBroadcastClientName.Filter, - }); - const runner = await FilterRunner.create(config, addons); + + await dependencies.initialize(config); + + const { broadcastClient, workerPool, unprocessedBlockQueue } = dependencies; + const runner = new FilterRunner(workerPool, unprocessedBlockQueue); + runner.onTransition(() => { - broadcast.sendMessage(ProcessorBroadcastMessage.refresh()); + broadcastClient.sendMessage(ProcessorBroadcastMessage.refresh()); }); - - broadcast.onMessage( + workerPool.onWorkerRelease(() => runner.next()); + broadcastClient.onMessage( InternalBroadcastChannel.Filter, - async (message: InternalBroadcastMessage) => { - if (message.content.name === InternalBroadcastMessageName.FilterRefresh) { + async (message: BroadcastMessage) => { + if (message.name === InternalBroadcastMessageName.FilterRefresh) { runner.next(); } } ); - await broadcast.connect(); + await broadcastClient.connect(); // Everything is ready, notify bootstrap that the process is ready to work - broadcast.sendMessage(FilterBroadcastMessage.ready()); + broadcastClient.sendMessage(FilterBroadcastMessage.ready()); // start filter in case the queue already contains blocks runner.next(); log(`Filter ... [ready]`); }; + +export const startFilter = ( + args: string[], + dependencies: FilterDependencies, + addons?: FilterAddons +) => { + const vars = new ConfigVars(); + const options = filterCommand.parse(args).opts(); + const config = buildFilterConfig(vars, options); + filter(config, dependencies, addons).catch(log); +}; diff --git a/src/processor/index.ts b/src/processor/index.ts index d7bb2a4..d254764 100644 --- a/src/processor/index.ts +++ b/src/processor/index.ts @@ -1,11 +1,10 @@ -export * from './processors/action-trace.processor'; -export * from './processors/action-trace.processor.input'; -export * from './processors/delta.processor'; -export * from './processors/delta.processor.input'; -export * from './processors/processor'; - -export * from './processor.types'; +export * from './processors'; +export * from './processor.command'; +export * from './processor.consts'; +export * from './processor.dependencies'; export * from './processor.enum'; -export * from './processor.errors'; +export * from './processor.runner'; +export * from './processor.types'; export * from './processor.worker-loader'; +export * from './processor.worker-loader.dependencies'; export * from './start-processor'; diff --git a/src/processor/processor.command.ts b/src/processor/processor.command.ts new file mode 100644 index 0000000..55545fa --- /dev/null +++ b/src/processor/processor.command.ts @@ -0,0 +1,8 @@ +import { Command } from 'commander'; + +export const processorCommand = new Command(); + +processorCommand + .version('1.0', '-v, --version') + .option('-t, --threads ', 'Number of threads') + .parse(process.argv); diff --git a/src/processor/processor.consts.ts b/src/processor/processor.consts.ts index 6eef0c7..1261bfe 100644 --- a/src/processor/processor.consts.ts +++ b/src/processor/processor.consts.ts @@ -1 +1,2 @@ -export const processorWorkerLoaderPath = `${__dirname}/processor.worker-loader`; \ No newline at end of file +export const processorWorkerLoaderPath = `${__dirname}/processor.worker-loader`; +export const processorWorkerLoaderDependenciesPath = `${__dirname}/processor.worker-loader.dependencies`; diff --git a/src/processor/processor.dependencies.ts b/src/processor/processor.dependencies.ts new file mode 100644 index 0000000..94a17c3 --- /dev/null +++ b/src/processor/processor.dependencies.ts @@ -0,0 +1,21 @@ +import { Result } from '@alien-worlds/api-core'; +import { Dependencies } from '../common/dependencies'; +import { BroadcastClient } from '@alien-worlds/broadcast'; +import { WorkerPool } from '@alien-worlds/workers'; +import { ProcessorConfig } from './processor.types'; + +/** + * An abstract class representing a Processor dependencies. + * @class ProcessorDependencies + */ +export abstract class ProcessorDependencies extends Dependencies { + /** + * The broadcast client used for communication. + * @type {BroadcastClient} + */ + public broadcastClient: BroadcastClient; + + public workerPool: WorkerPool; + + public abstract initialize(config: ProcessorConfig): Promise; +} diff --git a/src/processor/processor.errors.ts b/src/processor/processor.errors.ts deleted file mode 100644 index 45f4005..0000000 --- a/src/processor/processor.errors.ts +++ /dev/null @@ -1,5 +0,0 @@ -export class AbiNotFoundProcessorError extends Error { - constructor() { - super(''); - } -} \ No newline at end of file diff --git a/src/processor/processor.runner.ts b/src/processor/processor.runner.ts index f6ca242..4831831 100644 --- a/src/processor/processor.runner.ts +++ b/src/processor/processor.runner.ts @@ -4,7 +4,7 @@ import { ProcessorTaskQueue, ProcessorTask, ProcessorTaskModel, -} from './processor-task-queue'; +} from '../common/processor-task-queue'; import { WorkerMessage, WorkerPool } from '../common/workers'; import { ProcessorAddons, ProcessorConfig } from './processor.types'; import { processorWorkerLoaderPath } from './processor.consts'; diff --git a/src/processor/processor.types.ts b/src/processor/processor.types.ts index c342991..a44b3be 100644 --- a/src/processor/processor.types.ts +++ b/src/processor/processor.types.ts @@ -1,7 +1,8 @@ -import { FeaturedConfig, FeaturedMatchers } from '../common/featured'; -import { ProcessorTaskQueueConfig } from './processor-task-queue/processor-task-queue.config'; -import { WorkersConfig } from '../common/workers'; +import { FeaturedConfig, ProcessorMatcher } from '../common/featured'; +import { ProcessorTaskQueueConfig } from '../common/processor-task-queue/processor-task-queue.config'; import { MongoConfig } from '@alien-worlds/storage-mongodb'; +import { BroadcastConfig } from '@alien-worlds/broadcast'; +import { WorkersConfig } from '@alien-worlds/workers'; export type ProcessorCommandOptions = { threads: number; @@ -18,10 +19,38 @@ export type ProcessorConfig = { }; export type ProcessorAddons = { - matchers?: FeaturedMatchers; + matchers?: { + traces?: ProcessorMatcher; + deltas?: ProcessorMatcher; + [key: string]: ProcessorMatcher; + }; [key: string]: unknown; }; export type ProcessorSharedData = { config: ProcessorConfig; }; + +export type DeltaProcessorInput = { + name: string; + code: string; + scope: string; + table: string; + payer: string; + present: number; + primaryKey: bigint; + blockNumber: bigint; + blockTimestamp: Date; + data: DataType; +}; + +export type ActionTraceProcessorInput = { + blockNumber: bigint; + blockTimestamp: Date; + transactionId: string; + account: string; + name: string; + recvSequence: bigint; + globalSequence: bigint; + data: DataType; +}; diff --git a/src/processor/processor.worker-loader.dependencies.ts b/src/processor/processor.worker-loader.dependencies.ts new file mode 100644 index 0000000..ffc7611 --- /dev/null +++ b/src/processor/processor.worker-loader.dependencies.ts @@ -0,0 +1,18 @@ +import { UnknownObject, Serializer } from '@alien-worlds/api-core'; +import { WorkerContainer, WorkerLoaderDependencies } from '@alien-worlds/workers'; +import { ProcessorConfig } from './processor.types'; + +/** + * An abstract class representing a ProcessorWorkerLoader dependencies. + * @class ProcessorWorkerLoaderDependencies + */ +export abstract class ProcessorWorkerLoaderDependencies extends WorkerLoaderDependencies { + public dataSource: unknown; + public serializer: Serializer; + public workers: WorkerContainer; + + public abstract initialize( + config: ProcessorConfig, + featuredJson: UnknownObject + ): Promise; +} diff --git a/src/processor/processor.worker-loader.ts b/src/processor/processor.worker-loader.ts index 44c48b6..6f86b3e 100644 --- a/src/processor/processor.worker-loader.ts +++ b/src/processor/processor.worker-loader.ts @@ -1,18 +1,35 @@ -import { MongoSource } from '@alien-worlds/storage-mongodb'; -import { Worker } from '../common/workers'; -import { DefaultWorkerLoader } from '../common/workers/worker-loader'; +import { DefaultWorkerLoader, Worker, WorkerContainer } from '@alien-worlds/workers'; import { ProcessorSharedData } from './processor.types'; +import { ProcessorWorkerLoaderDependencies } from './processor.worker-loader.dependencies'; +import { Container } from '@alien-worlds/api-core'; -export default class ProcessorWorkerLoader extends DefaultWorkerLoader { - private mongoSource: MongoSource; +export default class ProcessorWorkerLoader extends DefaultWorkerLoader< + ProcessorSharedData, + ProcessorWorkerLoaderDependencies +> { + protected workers: WorkerContainer; + protected ioc: Container; public async setup(sharedData: ProcessorSharedData): Promise { - super.setup(sharedData); - this.mongoSource = await MongoSource.create(sharedData.config.mongo); + const { config } = sharedData; + await super.setup(sharedData, config); + this.ioc = new Container(); } public async load(pointer: string): Promise { - const { mongoSource } = this; - return super.load(pointer, { mongoSource }); + const { + dependencies: { dataSource, serializer, workers }, + } = this; + const { ioc, sharedData } = this; + const Class = workers.get(pointer); + const worker: Worker = new Class( + { + ioc, + dataSource, + serializer, + }, + sharedData + ) as Worker; + return worker; } } diff --git a/src/processor/processors/action-trace.processor.input.ts b/src/processor/processors/action-trace.processor.input.ts deleted file mode 100644 index c98f085..0000000 --- a/src/processor/processors/action-trace.processor.input.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { deserialize } from 'v8'; -import { AbisSerialize } from '../../common/abis/abis.serializer'; -import { - ActionProcessorContentModel, - ProcessorTaskModel, -} from '../processor-task-queue/processor-task.types'; - -export class ActionTraceProcessorInput { - public static create(model: ProcessorTaskModel) { - const { abi, content: buffer } = model; - const content: ActionProcessorContentModel = deserialize(buffer); - const { - actionTrace: { - act: { account, data, name }, - receipt: { recvSequence, globalSequence }, - }, - blockNumber, - blockTimestamp, - transactionId, - } = content; - - const deserializedData = AbisSerialize.deserializeAction( - account, - name, - data, - abi - ); - - return new ActionTraceProcessorInput( - blockNumber, - blockTimestamp, - transactionId, - account, - name, - recvSequence, - globalSequence, - deserializedData - ); - } - - private constructor( - public readonly blockNumber: bigint, - public readonly blockTimestamp: Date, - public readonly transactionId: string, - public readonly account: string, - public readonly name: string, - public readonly recvSequence: bigint, - public readonly globalSequence: bigint, - public readonly data: DataType - ) {} -} diff --git a/src/processor/processors/action-trace.processor.ts b/src/processor/processors/action-trace.processor.ts index 2cd9ffb..81a8ce4 100644 --- a/src/processor/processors/action-trace.processor.ts +++ b/src/processor/processors/action-trace.processor.ts @@ -1,8 +1,11 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { Processor } from './processor'; -import { ActionTraceProcessorInput } from './action-trace.processor.input'; -import { ProcessorTaskModel } from '../processor-task-queue/processor-task.types'; -import { ProcessorSharedData } from '../processor.types'; +import { + ActionProcessorContentModel, + ProcessorTaskModel, +} from '../../common/processor-task-queue/processor-task.types'; +import { ActionTraceProcessorInput, ProcessorSharedData } from '../processor.types'; +import { Container, Serializer } from '@alien-worlds/api-core'; +import { deserialize } from 'v8'; export class ActionTraceProcessor< DataType = unknown, @@ -10,7 +13,55 @@ export class ActionTraceProcessor< > extends Processor { protected input: ActionTraceProcessorInput; - public async run(data: ProcessorTaskModel): Promise { - this.input = ActionTraceProcessorInput.create(data); + constructor( + protected dependencies: { + ioc: Container; + dataSource: unknown; + serializer: Serializer; + }, + protected sharedData: SharedDataType + ) { + super(); + } + + public deserializeModelContent( + model: ProcessorTaskModel + ): ActionTraceProcessorInput { + const { + dependencies: { serializer }, + } = this; + const { abi, content: buffer } = model; + const content: ActionProcessorContentModel = deserialize(buffer); + const { + actionTrace: { + act, + receipt: { recvSequence, globalSequence }, + }, + blockNumber, + blockTimestamp, + transactionId, + } = content; + + const data = serializer.deserializeActionData( + act.account, + act.name, + act.data, + abi + ); + + return { + blockNumber, + blockTimestamp, + transactionId, + account: act.account, + name: act.name, + recvSequence, + globalSequence, + data: data as DataType, + }; + } + + public async run(model: ProcessorTaskModel): Promise { + this.input = this.deserializeModelContent(model); } } diff --git a/src/processor/processors/delta.processor.input.ts b/src/processor/processors/delta.processor.input.ts deleted file mode 100644 index 354ca12..0000000 --- a/src/processor/processors/delta.processor.input.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { Serialize } from 'eosjs'; -import { deserialize } from 'v8'; -import { AbisSerialize } from '../../common/abis/abis.serializer'; -import { - DeltaProcessorContentModel, - ProcessorTaskModel, -} from '../processor-task-queue/processor-task.types'; - -export class DeltaProcessorInput { - public static create(model: ProcessorTaskModel) { - const { abi, content: buffer } = model; - const content: DeltaProcessorContentModel = deserialize(buffer); - const { - name, - row: { present, data }, - blockNumber, - blockTimestamp, - } = content; - - const sb = new Serialize.SerialBuffer({ - textEncoder: new TextEncoder(), - textDecoder: new TextDecoder(), - array: data, - }); - sb.get(); // version - const code = sb.getName(); // code - const scope = sb.getName(); // scope - const table = sb.getName(); // table - const primaryKey = Buffer.from(sb.getUint8Array(8)).readBigInt64BE(); // primary_key - const payer = sb.getName(); // payer - const bytes = sb.getBytes(); // data bytes - - const deserializedData = AbisSerialize.deserializeTable( - code, - table, - bytes, - abi - ); - return new DeltaProcessorInput( - name, - code, - scope, - table, - payer, - present, - primaryKey, - blockNumber, - blockTimestamp, - deserializedData - ); - } - - private constructor( - public readonly name: string, - public readonly code: string, - public readonly scope: string, - public readonly table: string, - public readonly payer: string, - public readonly present: number, - public readonly primaryKey: bigint, - public readonly blockNumber: bigint, - public readonly blockTimestamp: Date, - public readonly data: DataType - ) {} -} diff --git a/src/processor/processors/delta.processor.ts b/src/processor/processors/delta.processor.ts index f877b5e..ba87b1d 100644 --- a/src/processor/processors/delta.processor.ts +++ b/src/processor/processors/delta.processor.ts @@ -1,8 +1,11 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ -import { ProcessorTaskModel } from '../processor-task-queue/processor-task.types'; -import { DeltaProcessorInput } from './delta.processor.input'; +import { + DeltaProcessorContentModel, + ProcessorTaskModel, +} from '../../common/processor-task-queue/processor-task.types'; import { Processor } from './processor'; -import { ProcessorSharedData } from '../processor.types'; +import { DeltaProcessorInput, ProcessorSharedData } from '../processor.types'; +import { Container, Serializer, parseToBigInt } from '@alien-worlds/api-core'; +import { deserialize } from 'v8'; export class DeltaProcessor< DataType, @@ -10,7 +13,44 @@ export class DeltaProcessor< > extends Processor { protected input: DeltaProcessorInput; - public async run(data: ProcessorTaskModel): Promise { - this.input = DeltaProcessorInput.create(data); + constructor( + protected dependencies: { + ioc: Container; + dataSource: unknown; + serializer: Serializer; + }, + protected sharedData: SharedDataType + ) { + super(); + } + + public deserializeModelContent( + model: ProcessorTaskModel + ): DeltaProcessorInput { + const { + dependencies: { serializer }, + } = this; + const { abi, content: buffer } = model; + const delta: DeltaProcessorContentModel = deserialize(buffer); + const { name, blockNumber, blockTimestamp } = delta; + const row = serializer.deserializeTableRow(delta.row.data, abi); + const { code, scope, table, primaryKey, payer, data } = row; + + return { + name, + blockNumber, + blockTimestamp, + present: delta.row.present, + code, + scope, + table, + primaryKey: parseToBigInt(primaryKey), + payer, + data: data as DataType, + }; + } + + public async run(model: ProcessorTaskModel): Promise { + this.input = this.deserializeModelContent(model); } } diff --git a/src/processor/processors/index.ts b/src/processor/processors/index.ts index a09bbd6..e4e9ec1 100644 --- a/src/processor/processors/index.ts +++ b/src/processor/processors/index.ts @@ -1,5 +1,3 @@ export * from './action-trace.processor'; -export * from './action-trace.processor.input'; export * from './delta.processor'; -export * from './delta.processor.input'; export * from './processor'; diff --git a/src/processor/processors/processor.ts b/src/processor/processors/processor.ts index e90dd72..ef8c052 100644 --- a/src/processor/processors/processor.ts +++ b/src/processor/processors/processor.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { ProcessorTaskModel } from '../processor-task-queue/processor-task.types'; -import { Worker } from '../../common/workers/worker'; +import { Worker } from '@alien-worlds/workers'; +import { ProcessorTaskModel } from '../../common/processor-task-queue/processor-task.types'; import { ProcessorSharedData } from '../processor.types'; export class Processor< diff --git a/src/processor/start-processor.ts b/src/processor/start-processor.ts index 7e1b032..1500c20 100644 --- a/src/processor/start-processor.ts +++ b/src/processor/start-processor.ts @@ -1,13 +1,19 @@ -import { log } from '@alien-worlds/api-core'; -import { ProcessorAddons, ProcessorConfig } from './processor.types'; +import { ConfigVars, log } from '@alien-worlds/api-core'; +import { + ProcessorAddons, + ProcessorCommandOptions, + ProcessorConfig, +} from './processor.types'; import { InternalBroadcastChannel, - InternalBroadcastClientName, InternalBroadcastMessageName, ProcessorBroadcastMessage, } from '../broadcast'; -import { InternalBroadcastMessage } from '../broadcast/internal-broadcast.message'; import { ProcessorRunner } from './processor.runner'; +import { ProcessorDependencies } from './processor.dependencies'; +import { processorCommand } from './processor.command'; +import { buildProcessorConfig } from '../config'; +import { BroadcastMessage } from '@alien-worlds/broadcast'; /** * @@ -15,31 +21,43 @@ import { ProcessorRunner } from './processor.runner'; * @param broadcastMessageMapper * @param config */ -export const startProcessor = async ( +export const process = async ( config: ProcessorConfig, + dependencies: ProcessorDependencies, addons: ProcessorAddons = {} ) => { log(`Processor ... [starting]`); - const broadcast = await Broadcast.createClient({ - ...config.broadcast, - clientName: InternalBroadcastClientName.Processor, - }); + + await dependencies.initialize(config); + + const { broadcastClient } = dependencies; const runner = await ProcessorRunner.getInstance(config, addons); - broadcast.onMessage( + broadcastClient.onMessage( InternalBroadcastChannel.Processor, - async (message: InternalBroadcastMessage) => { - if (message.content.name === InternalBroadcastMessageName.ProcessorRefresh) { + async (message: BroadcastMessage) => { + if (message.name === InternalBroadcastMessageName.ProcessorRefresh) { runner.next(); } } ); - await broadcast.connect(); + await broadcastClient.connect(); // Everything is ready, notify the block-range that the process is ready to work - broadcast.sendMessage(ProcessorBroadcastMessage.ready()); + broadcastClient.sendMessage(ProcessorBroadcastMessage.ready()); // start processor in case the queue already contains tasks runner.next(); log(`Processor ... [ready]`); }; + +export const startProcessor = ( + args: string[], + dependencies: ProcessorDependencies, + addons?: ProcessorAddons +) => { + const vars = new ConfigVars(); + const options = processorCommand.parse(args).opts(); + const config = buildProcessorConfig(vars, options); + process(config, dependencies, addons).catch(log); +}; diff --git a/src/reader/index.ts b/src/reader/index.ts index efc844c..803970b 100644 --- a/src/reader/index.ts +++ b/src/reader/index.ts @@ -1,7 +1,9 @@ -export * from '../common/blockchain/block-reader'; -export * from './unprocessed-block-queue'; -export * from './reader.types'; export * from './reader'; -export * from './reader.worker-loader'; +export * from './reader.command'; +export * from './reader.consts'; +export * from './reader.dependencies'; +export * from './reader.types'; export * from './reader.worker'; +export * from './reader.worker-loader'; +export * from './reader.worker-loader.dependencies'; export * from './start-reader'; diff --git a/src/reader/reader.command.ts b/src/reader/reader.command.ts new file mode 100644 index 0000000..388e387 --- /dev/null +++ b/src/reader/reader.command.ts @@ -0,0 +1,12 @@ +import { Command } from 'commander'; + +export const readerCommand = new Command(); + +readerCommand + .version('1.0', '-v, --version') + .option('-k, --scan-key ', 'Scan key') + .option('-s, --start-block ', 'Start at this block') + .option('-m, --mode ', 'Mode (default/replay/test)') + .option('-e, --end-block ', 'End block (exclusive)') + .option('-t, --threads ', 'Number of threads') + .parse(process.argv); diff --git a/src/reader/reader.consts.ts b/src/reader/reader.consts.ts new file mode 100644 index 0000000..0517de3 --- /dev/null +++ b/src/reader/reader.consts.ts @@ -0,0 +1,2 @@ +export const readerWorkerLoaderPath = `${__dirname}/reader.worker-loader`; +export const readerWorkerLoaderDependenciesPath = `${__dirname}/reader.worker-loader.dependencies`; diff --git a/src/reader/reader.dependencies.ts b/src/reader/reader.dependencies.ts new file mode 100644 index 0000000..095a897 --- /dev/null +++ b/src/reader/reader.dependencies.ts @@ -0,0 +1,19 @@ +import { BroadcastClient } from '@alien-worlds/broadcast'; +import { Dependencies } from '../common/dependencies'; +import { WorkerPool } from '@alien-worlds/workers'; +import { ReaderConfig } from './reader.types'; +import { Result } from '@alien-worlds/api-core'; +import { BlockRangeScanner, UnprocessedBlockQueue } from '../common'; + +/** + * An abstract class representing a reader dependencies. + * @class ReaderDependencies + */ +export abstract class ReaderDependencies extends Dependencies { + public broadcastClient: BroadcastClient; + public scanner: BlockRangeScanner; + public workerPool: WorkerPool; + public unprocessedBlockQueue: UnprocessedBlockQueue; + + public abstract initialize(config: ReaderConfig): Promise; +} diff --git a/src/reader/reader.ts b/src/reader/reader.ts index cd277b0..c9bd2ff 100644 --- a/src/reader/reader.ts +++ b/src/reader/reader.ts @@ -1,52 +1,20 @@ import { log } from '@alien-worlds/api-core'; -import { BlockRangeScanner } from './block-range-scanner'; import { Mode } from '../common/common.enums'; -import { WorkerMessage, WorkerPool } from '../common/workers'; -import ReaderWorker from './reader.worker'; -import { ReadCompleteData, ReaderConfig, ReadTaskData } from './reader.types'; -import { InternalBroadcastClientName } from '../broadcast'; +import { ReadCompleteData, ReadTaskData } from './reader.types'; import { FilterBroadcastMessage } from '../broadcast/messages'; -import { MongoSource } from '@alien-worlds/storage-mongodb'; +import { WorkerMessage } from '@alien-worlds/workers'; +import { ReaderDependencies } from './reader.dependencies'; export class Reader { - public static async create( - config: ReaderConfig, - broadcastClient?: BroadcastClient - ): Promise { - const mongoSource = await MongoSource.create(config.mongo); - const scanner = await BlockRangeScanner.create(mongoSource, config.scanner); - const workerPool = await WorkerPool.create({ - threadsCount: config.workers?.threadsCount || 1, - sharedData: { config }, - defaultWorkerPath: `${__dirname}/reader.worker`, - workerLoaderPath: `${__dirname}/reader.worker-loader`, - }); - let broadcast: BroadcastClient; - if (!broadcastClient) { - broadcast = await Broadcast.createClient({ - ...config.broadcast, - clientName: InternalBroadcastClientName.Reader, - }); - broadcast.connect(); - } else { - broadcast = broadcastClient; - } - return new Reader(workerPool, scanner, broadcast); - } - private loop = false; - private isAlreadyStarted = false; private initTaskData: ReadTaskData; - protected constructor( - private workerPool: WorkerPool, - private scanner: BlockRangeScanner, - private broadcast: BroadcastClient - ) { + constructor(protected dependencies: ReaderDependencies) { + const { workerPool, scanner } = dependencies; workerPool.onWorkerRelease(async () => { const { initTaskData } = this; if (initTaskData.mode === Mode.Replay) { - if (await this.scanner.hasUnscannedBlocks(initTaskData.scanKey)) { + if (await scanner.hasUnscannedBlocks(initTaskData.scanKey)) { this.read(initTaskData); } } else { @@ -56,7 +24,9 @@ export class Reader { } private async handleWorkerMessage(message: WorkerMessage) { - const { workerPool, broadcast } = this; + const { + dependencies: { workerPool, broadcastClient }, + } = this; const { data, error, workerId } = message; if (message.isTaskResolved()) { @@ -69,13 +39,16 @@ export class Reader { log(`An unexpected error occurred while reading blocks...`, error); workerPool.releaseWorker(workerId); } else if (message.isTaskProgress()) { - broadcast.sendMessage(FilterBroadcastMessage.refresh()); + broadcastClient.sendMessage(FilterBroadcastMessage.refresh()); } } private async handleWorkerError(id: number, error: Error) { + const { + dependencies: { workerPool }, + } = this; log(`Worker error:`, error); - this.workerPool.releaseWorker(id); + workerPool.releaseWorker(id); } public async read(task: ReadTaskData) { @@ -93,8 +66,12 @@ export class Reader { ); } + const { + dependencies: { workerPool, scanner }, + initTaskData, + } = this; + while (this.loop) { - const { initTaskData, workerPool } = this; const worker = await workerPool.getWorker(); if (worker) { worker.onMessage(message => this.handleWorkerMessage(message)); @@ -103,7 +80,7 @@ export class Reader { if (task.mode === Mode.Default || task.mode === Mode.Test) { worker.run({ startBlock: task.startBlock, endBlock: task.endBlock }); } else if (task.mode === Mode.Replay) { - const scan = await this.scanner.getNextScanNode(task.scanKey); + const scan = await scanner.getNextScanNode(task.scanKey); if (scan) { worker.run({ diff --git a/src/reader/reader.types.ts b/src/reader/reader.types.ts index 84b3a97..945053a 100644 --- a/src/reader/reader.types.ts +++ b/src/reader/reader.types.ts @@ -1,7 +1,8 @@ -import { BlockRangeScanConfig } from './block-range-scanner'; -import { WorkersConfig } from '../common/workers'; -import { BlockReaderConfig } from '../common/blockchain/block-reader'; +import { BroadcastConfig } from '@alien-worlds/broadcast'; +import { BlockRangeScanConfig } from '../common/block-range-scanner'; import { MongoConfig } from '@alien-worlds/storage-mongodb'; +import { BlockReaderConfig } from '@alien-worlds/block-reader'; +import { WorkersConfig } from '@alien-worlds/workers'; export type ReaderConfig = { broadcast?: BroadcastConfig; diff --git a/src/reader/reader.worker-loader.dependencies.ts b/src/reader/reader.worker-loader.dependencies.ts new file mode 100644 index 0000000..31eb013 --- /dev/null +++ b/src/reader/reader.worker-loader.dependencies.ts @@ -0,0 +1,21 @@ +import { UnknownObject } from '@alien-worlds/api-core'; +import { WorkerLoaderDependencies } from '@alien-worlds/workers'; +import { BlockRangeScanner, BlockState, UnprocessedBlockQueue } from '../common'; +import { BlockReader } from '@alien-worlds/block-reader'; +import { ReaderConfig } from './reader.types'; + +/** + * An abstract class representing a ReaderWorkerLoader dependencies. + * @class ReaderWorkerLoaderDependencies + */ +export abstract class ReaderWorkerLoaderDependencies extends WorkerLoaderDependencies { + public blockReader: BlockReader; + public blockState: BlockState; + public blockQueue: UnprocessedBlockQueue; + public scanner: BlockRangeScanner; + + public abstract initialize( + config: ReaderConfig, + featuredJson: UnknownObject + ): Promise; +} diff --git a/src/reader/reader.worker-loader.ts b/src/reader/reader.worker-loader.ts index 3755fb4..9c1d5bb 100644 --- a/src/reader/reader.worker-loader.ts +++ b/src/reader/reader.worker-loader.ts @@ -1,47 +1,28 @@ import { log } from '@alien-worlds/api-core'; -import { BlockReader } from '../common/blockchain'; -import { Worker } from '../common/workers'; -import { DefaultWorkerLoader } from '../common/workers/worker-loader'; -import { UnprocessedBlockQueue } from './unprocessed-block-queue'; +import { Worker } from '@alien-worlds/workers'; import ReaderWorker, { ReaderSharedData } from './reader.worker'; -import { BlockRangeScanner, BlockState } from '../common'; -import { MongoSource } from '@alien-worlds/storage-mongodb'; - -export default class ReaderWorkerLoader extends DefaultWorkerLoader { - private blockReader: BlockReader; - private blockState: BlockState; - private blocksQueue: UnprocessedBlockQueue; - private scanner: BlockRangeScanner; +import { DefaultWorkerLoader } from '@alien-worlds/workers'; +import { ReaderWorkerLoaderDependencies } from './reader.worker-loader.dependencies'; +export default class ReaderWorkerLoader extends DefaultWorkerLoader< + ReaderSharedData, + ReaderWorkerLoaderDependencies +> { public async setup(sharedData: ReaderSharedData): Promise { - super.setup(sharedData); - + const { config } = sharedData; + await super.setup(sharedData, config); + // + const { blockQueueMaxBytesSize, blockQueueSizeCheckInterval } = config; const { - config: { - blockReader, - mongo, - blockQueueBatchSize, - blockQueueMaxBytesSize, - blockQueueSizeCheckInterval, - scanner, - }, - } = sharedData; - const mongoSource = await MongoSource.create(mongo); - this.blockReader = await BlockReader.create(blockReader); - this.blockState = await BlockState.create(mongoSource); - this.scanner = await BlockRangeScanner.create(mongoSource, scanner); - this.blocksQueue = await UnprocessedBlockQueue.create( - mongoSource, - blockQueueMaxBytesSize, - blockQueueBatchSize - ); - this.blocksQueue.onOverload(size => { + dependencies: { blockQueue: blocksQueue, blockReader }, + } = this; + blocksQueue.onOverload(size => { const overload = size - blockQueueMaxBytesSize; log(`Overload: ${overload} bytes.`); - this.blockReader.pause(); + blockReader.pause(); let interval = setInterval(async () => { - const { content: size, failure } = await this.blocksQueue.getBytesSize(); + const { content: size, failure } = await blocksQueue.getBytesSize(); if (failure) { log( @@ -49,24 +30,24 @@ export default class ReaderWorkerLoader extends DefaultWorkerLoader { - this.blockReader.pause(); + blocksQueue.beforeSendBatch(() => { + blockReader.pause(); }); - this.blocksQueue.afterSendBatch(() => { - this.blockReader.resume(); + blocksQueue.afterSendBatch(() => { + blockReader.resume(); }); - await this.blockReader.connect(); + await blockReader.connect(); } public async load(): Promise { - const { blockReader, scanner, blocksQueue, blockState, sharedData } = this; - return new ReaderWorker(blockReader, blocksQueue, blockState, scanner, sharedData); + const { dependencies, sharedData } = this; + return new ReaderWorker(dependencies, sharedData); } } diff --git a/src/reader/reader.worker.ts b/src/reader/reader.worker.ts index 8f2f289..772b891 100644 --- a/src/reader/reader.worker.ts +++ b/src/reader/reader.worker.ts @@ -1,9 +1,8 @@ import { log, parseToBigInt } from '@alien-worlds/api-core'; -import { Worker } from '../common/workers/worker'; -import { BlockReader } from '../common/blockchain/block-reader'; +import { Worker } from '@alien-worlds/workers'; +import { BlockReader } from '@alien-worlds/block-reader'; import { ReaderConfig } from './reader.types'; -import { UnprocessedBlockQueue } from './unprocessed-block-queue'; -import { BlockRangeScanner, BlockState, Mode } from '../common'; +import { BlockRangeScanner, BlockState, Mode, UnprocessedBlockQueue } from '../common'; export type ReaderSharedData = { config: ReaderConfig; @@ -11,18 +10,22 @@ export type ReaderSharedData = { export default class ReaderWorker extends Worker { constructor( - protected blockReader: BlockReader, - protected blockQueue: UnprocessedBlockQueue, - protected blockState: BlockState, - protected scanner: BlockRangeScanner, - sharedData: ReaderSharedData + protected dependencies: { + blockReader: BlockReader; + blockQueue: UnprocessedBlockQueue; + blockState: BlockState; + scanner: BlockRangeScanner; + }, + protected sharedData: ReaderSharedData ) { super(); this.sharedData = sharedData; } private async updateBlockState(): Promise { - const { blockQueue, blockState } = this; + const { + dependencies: { blockQueue, blockState }, + } = this; const { content: maxBlock } = await blockQueue.getMax(); if (maxBlock) { const { failure } = await blockState.updateBlockNumber( @@ -48,8 +51,7 @@ export default class ReaderWorker extends Worker { private async readInDefaultMode(startBlock: bigint, endBlock: bigint) { const { - blockReader, - blockQueue, + dependencies: { blockReader, blockQueue }, sharedData: { config: { maxBlockNumber, @@ -104,9 +106,7 @@ export default class ReaderWorker extends Worker { private async readInReplayMode(startBlock: bigint, endBlock: bigint, scanKey: string) { const { - blockReader, - blockQueue, - scanner, + dependencies: { blockReader, blockQueue, scanner }, sharedData: { config: { blockReader: { shouldFetchDeltas, shouldFetchTraces, shouldFetchBlock }, diff --git a/src/reader/start-reader.ts b/src/reader/start-reader.ts index a82b84c..2677939 100644 --- a/src/reader/start-reader.ts +++ b/src/reader/start-reader.ts @@ -2,28 +2,31 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { InternalBroadcastChannel, - InternalBroadcastClientName, InternalBroadcastMessageName, } from '../broadcast/internal-broadcast.enums'; -import { log } from '@alien-worlds/api-core'; -import { ReadTaskData, ReaderConfig } from './reader.types'; -import { InternalBroadcastMessage } from '../broadcast/internal-broadcast.message'; +import { ConfigVars, log } from '@alien-worlds/api-core'; +import { ReadTaskData, ReaderCommandOptions, ReaderConfig } from './reader.types'; import { ReaderBroadcastMessage } from '../broadcast/messages/reader-broadcast.message'; import { Reader } from './reader'; import { Mode } from '../common'; +import { readerCommand } from './reader.command'; +import { ReaderDependencies } from './reader.dependencies'; +import { buildReaderConfig } from '../config'; +import { BroadcastMessage } from '@alien-worlds/broadcast'; /** * * @param config * @returns */ -export const startReader = async (config: ReaderConfig) => { +export const read = async (config: ReaderConfig, dependencies: ReaderDependencies) => { log(`Reader ... [starting]`); - const broadcast = await Broadcast.createClient({ - ...config.broadcast, - clientName: InternalBroadcastClientName.Reader, - }); - const blockRangeReader = await Reader.create(config, broadcast); + + await dependencies.initialize(config); + + const { broadcastClient } = dependencies; + const reader = new Reader(dependencies); + let channel: string; let readyMessage; @@ -36,20 +39,22 @@ export const startReader = async (config: ReaderConfig) => { } log(`Reader started in "listening" mode`); - broadcast.onMessage( - channel, - async (message: InternalBroadcastMessage) => { - const { - content: { data, name }, - } = message; - if (name === InternalBroadcastMessageName.ReaderTask) { - blockRangeReader.read(data); - } + broadcastClient.onMessage(channel, async (message: BroadcastMessage) => { + const { data, name } = message; + if (name === InternalBroadcastMessageName.ReaderTask) { + reader.read(data); } - ); - broadcast.connect(); + }); + broadcastClient.connect(); // Everything is ready, notify the bootstrap that the process is ready to work - broadcast.sendMessage(readyMessage); + broadcastClient.sendMessage(readyMessage); log(`Reader ... [ready]`); }; + +export const startReader = (args: string[], dependencies: ReaderDependencies) => { + const vars = new ConfigVars(); + const options = readerCommand.parse(args).opts(); + const config = buildReaderConfig(vars, options); + read(config, dependencies).catch(log); +}; diff --git a/src/reader/unprocessed-block-queue/index.ts b/src/reader/unprocessed-block-queue/index.ts deleted file mode 100644 index afcb8e8..0000000 --- a/src/reader/unprocessed-block-queue/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './unprocessed-block-queue.errors'; -export * from './unprocessed-block-queue'; diff --git a/yarn.lock b/yarn.lock index 4f03cfe..1819bd7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,15 +2,6 @@ # yarn lockfile v1 -"@acuminous/bitsyntax@^0.1.2": - version "0.1.2" - resolved "https://registry.npmjs.org/@acuminous/bitsyntax/-/bitsyntax-0.1.2.tgz" - integrity sha512-29lUK80d1muEQqiUsSo+3A0yP6CdspgC95EnKBMi22Xlwt79i/En4Vr67+cXhU+cZjbti3TgGGC5wy1stIywVQ== - dependencies: - buffer-more-ints "~1.0.0" - debug "^4.3.4" - safe-buffer "~5.1.2" - "@alien-worlds/api-core@^0.0.125": version "0.0.125" resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.125/386abf1c615f114905ec478516acd3fba08bbbac#386abf1c615f114905ec478516acd3fba08bbbac" @@ -22,12 +13,21 @@ node-fetch "2" reflect-metadata "^0.1.13" -"@alien-worlds/api-core@^0.0.127": - version "0.0.127" - resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.127/50d818f0b9130fd555394c688c2a4ed9f2d6f3fb#50d818f0b9130fd555394c688c2a4ed9f2d6f3fb" - integrity sha512-x56Ei5PpGNbX8bZj5Vtg2vhUSrN3g7OavAzD9AaglFpKguG0AlLhHZa6vroAiUa4FFBhm5Np7evQBg011iq13g== +"@alien-worlds/api-core@^0.0.138": + version "0.0.138" + resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.138/4cb2b844e6e8e3f2aabd52d6833e224543553300#4cb2b844e6e8e3f2aabd52d6833e224543553300" + integrity sha512-5NZ//JB/7vgh33pjcpfOlrSZid0CaZzZnThWbIo4WEHYGi+5kodbvD76SLOpLjS0PG9891bqKaVxYG7ZpPLy6g== + dependencies: + inversify "^6.0.1" + nanoid "^3.0.0" + node-fetch "2" + reflect-metadata "^0.1.13" + +"@alien-worlds/api-core@^0.0.144": + version "0.0.144" + resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.144/294e62d106de83d5bf8b31ed02c5ac97e98f7b85#294e62d106de83d5bf8b31ed02c5ac97e98f7b85" + integrity sha512-yVqBz52a17tv7XjzW/2s9ZgsA+WxjuBmrAFXznhNi/LIp24nbQWrnknpzufIshdi7h9dgJ3pQudYqqKmY/wyBw== dependencies: - eosjs "^22.1.0" inversify "^6.0.1" nanoid "^3.0.0" node-fetch "2" @@ -40,27 +40,27 @@ dependencies: "@alien-worlds/api-core" "^0.0.125" -"@alien-worlds/broadcast@^0.0.3": - version "0.0.3" - resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.3/3da5e6bd4a43fc6729a8881d6df97bd5f05b83e5#3da5e6bd4a43fc6729a8881d6df97bd5f05b83e5" - integrity sha512-a3kSUP85hRHTDVSwUUxy8omFaSYsu5HHF9U9N8WYvcq9wH9u34IXrs/ZbJ2ugArohSjfwV+whtJ3rip7umR2XA== +"@alien-worlds/broadcast@^0.0.4": + version "0.0.4" + resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.4/6ed3e29e47633ee152b710688ab83ce2e4b2884d#6ed3e29e47633ee152b710688ab83ce2e4b2884d" + integrity sha512-IL84vKnez7c6k1NC/SXpRo96PXDjoHM3wPkk5dykU3FC9EGjQ5ulaszSicCfF3JEdmdGBwWGBBIu/jAYcnNI/w== dependencies: - "@alien-worlds/api-core" "^0.0.125" + "@alien-worlds/api-core" "^0.0.138" nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/storage-mongodb@^0.0.7": - version "0.0.7" - resolved "https://npm.pkg.github.com/download/@alien-worlds/storage-mongodb/0.0.7/a39d63bd60a5bfa61f54884989d7eca66ea15e11#a39d63bd60a5bfa61f54884989d7eca66ea15e11" - integrity sha512-/BpTVKaxAhOUsZ9ixdeS33K58gkC+sxOlh75LNETLWAad2nCE2v/okL/MXu9X2HgowQNnxWsC0EJAZiXgKDSYQ== +"@alien-worlds/storage-mongodb@^0.0.14": + version "0.0.14" + resolved "https://npm.pkg.github.com/download/@alien-worlds/storage-mongodb/0.0.14/a16846339c1fb7c2886592c96e5b6f735240f46e#a16846339c1fb7c2886592c96e5b6f735240f46e" + integrity sha512-hTYZOe5gwz2Yk8IOQbwt+LMp+ksFaCNo8uP8se2vubenv6vMlsCVuz8C+Cbl37Xo0XB7EOiB2MPI55+d1bcO5w== dependencies: - "@alien-worlds/api-core" "^0.0.127" + "@alien-worlds/api-core" "^0.0.138" mongodb "4.9.1" -"@alien-worlds/workers@^0.0.1": - version "0.0.1" - resolved "https://npm.pkg.github.com/download/@alien-worlds/workers/0.0.1/31dd1c5b2ac7a0545e56c49c3bbae0edd06ae746#31dd1c5b2ac7a0545e56c49c3bbae0edd06ae746" - integrity sha512-AOccIgg+Ujl3aKLDm76cHc2VIcEEHI1QRAwsP4XlwTGwJH97du7jZkIkLwOV1OAMm4Rfil3EowYbtmNq5BlgoQ== +"@alien-worlds/workers@^0.0.3": + version "0.0.3" + resolved "https://npm.pkg.github.com/download/@alien-worlds/workers/0.0.3/91bad176b3e3d511b6d5e9c9ff04619a150ccd24#91bad176b3e3d511b6d5e9c9ff04619a150ccd24" + integrity sha512-WPtw2ckhd9nTzjL5keC69+7LtJiYL+G+pHAmDoqT9ZlY+Y8bPc3jEZLHl9fSS50JWh4wPaBy1R/DTkoKOKz32Q== dependencies: async "^3.2.4" ts-node "^10.9.1" @@ -1021,16 +1021,6 @@ ajv@^6.10.0, ajv@^6.12.4: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -amqplib@^0.10.3: - version "0.10.3" - resolved "https://registry.npmjs.org/amqplib/-/amqplib-0.10.3.tgz" - integrity sha512-UHmuSa7n8vVW/a5HGh2nFPqAEr8+cD4dEZ6u9GjP91nHfr1a54RyAKyra7Sb5NH7NBKOUlyQSMXIp0qAixKexw== - dependencies: - "@acuminous/bitsyntax" "^0.1.2" - buffer-more-ints "~1.0.0" - readable-stream "1.x >=1.1.9" - url-parse "~1.5.10" - ansi-escapes@^4.2.1: version "4.3.2" resolved "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz" @@ -1267,11 +1257,6 @@ buffer-from@^1.0.0: resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== -buffer-more-ints@~1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz" - integrity sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg== - buffer@^5.6.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" @@ -1432,11 +1417,6 @@ cookie@0.5.0: resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== -core-util-is@~1.0.0: - version "1.0.3" - resolved "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz" - integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== - create-require@^1.1.0: version "1.1.1" resolved "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz" @@ -2223,7 +2203,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1: +inherits@2, inherits@2.0.4, inherits@^2.0.3, inherits@^2.0.4: version "2.0.4" resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -2302,11 +2282,6 @@ is-typedarray@^1.0.0: resolved "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz" integrity sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA== -isarray@0.0.1: - version "0.0.1" - resolved "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" - integrity sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ== - isexe@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz" @@ -3362,16 +3337,6 @@ react-is@^17.0.1: resolved "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz" integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w== -"readable-stream@1.x >=1.1.9": - version "1.1.14" - resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz" - integrity sha512-+MeVjFf4L44XUkhM1eYbD8fyEsxcV81pqMSR5gblfcLCHfZvbrqy4/qYHE+/R5HoBUT11WV5O08Cr1n3YXkWVQ== - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.1" - isarray "0.0.1" - string_decoder "~0.10.x" - reflect-metadata@^0.1.13: version "0.1.13" resolved "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz" @@ -3447,11 +3412,6 @@ safe-buffer@5.2.1: resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== -safe-buffer@~5.1.2: - version "5.1.2" - resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz" - integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== - "safer-buffer@>= 2.1.2 < 3": version "2.1.2" resolved "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" @@ -3625,11 +3585,6 @@ string-width@^4.1.0, string-width@^4.2.0: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" -string_decoder@~0.10.x: - version "0.10.31" - resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" - integrity sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ== - strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" @@ -3893,7 +3848,7 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" -url-parse@^1.5.3, url-parse@~1.5.10: +url-parse@^1.5.3: version "1.5.10" resolved "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz" integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ== From b6e0d20bfe8202d7a9dc1705d248c93bc960314d Mon Sep 17 00:00:00 2001 From: rkamysz Date: Thu, 22 Jun 2023 22:32:37 +0200 Subject: [PATCH 003/107] no compilation errors --- package.json | 9 +- .../contract-action.mongo.mapper.ts | 17 ++ .../contract-action.mongo.source.ts | 0 .../contract-action.mongo.types.ts | 6 + .../data/mappers/contract-action.mapper.ts | 33 --- .../actions.types.ts} | 0 .../domain/models/list-actions.input.ts | 50 +--- .../contract-action.repository.ts | 16 +- .../domain/use-cases/list-actions.use-case.ts | 8 +- src/api/endpoints/actions/index.ts | 6 +- src/api/endpoints/actions/ioc.config.ts | 18 +- .../__tests__/start-bootstrap.unit.test.ts | 78 ++----- src/bootstrap/bootstrap.utils.ts | 5 +- .../{ => __dependencies__}/abis.creator.ts | 7 +- .../abis.mongo.collection.ts} | 2 +- .../abis.mongo.mapper.ts | 2 +- .../abis/__dependencies__/abis.mongo.types.ts | 18 ++ src/common/abis/__dependencies__/index.ts | 4 + .../abi.repository-impl.unit.test.ts | 9 +- src/common/abis/__tests__/abi.unit.test.ts | 48 ---- src/common/abis/__tests__/abis.unit.test.ts | 47 ++-- .../contract-encoded-abi.unit.test.ts | 104 --------- src/common/abis/abis.cache.ts | 2 +- src/common/abis/abis.repository-impl.ts | 2 +- src/common/abis/abis.repository.ts | 3 +- src/common/abis/abis.types.ts | 19 -- src/common/abis/contract-encoded-abi.ts | 78 ------- src/common/abis/index.ts | 9 +- .../block-range-scan.mongo.source.ts | 20 +- .../block-range-scanner.creator.ts | 10 +- .../block-range-scanner.mongo.mapper.ts | 109 +++++++++ .../block-range-scanner.mongo.types.ts} | 18 +- .../__dependencies__/index.ts | 4 + .../block-range-scan.repository.ts | 15 +- .../block-range-scan.source.ts | 29 +++ .../block-range-scanner/block-range-scan.ts | 106 +-------- src/common/block-range-scanner/index.ts | 4 +- .../block-state.creator.ts | 6 +- .../block-state.mongo.collection.ts} | 2 +- .../block-state.mongo.mapper.ts | 3 +- .../block-state.mongo.types.ts | 9 + .../block-state/__dependencies__/index.ts | 5 + ...update-block-number.mongo.query-builder.ts | 0 .../__tests__/block-state.unit.test.ts | 11 +- src/common/block-state/block-state.ts | 11 +- src/common/block-state/block-state.types.ts | 10 - src/common/block-state/index.ts | 3 +- .../__tests__/action-trace.unit.test.ts | 63 ----- .../action-trace/action-trace.dtos.ts | 80 ------- .../action-trace/action-trace.ts | 217 ------------------ .../contract-content/action-trace/index.ts | 2 - .../delta/__tests__/delta.unit.test.ts | 14 -- .../contract-content/delta/delta.dtos.ts | 22 -- src/common/contract-content/delta/delta.ts | 65 ------ src/common/contract-content/delta/index.ts | 2 - src/common/contract-content/index.ts | 5 - .../__tests__/signed-block.unit.test.ts | 47 ---- .../contract-content/signed-block/index.ts | 2 - .../signed-block/signed-block.ts | 79 ------- .../trace/__tests__/trace.unit.test.ts | 68 ------ src/common/contract-content/trace/index.ts | 2 - src/common/contract-content/trace/trace.ts | 162 ------------- .../__tests__/transaction.unit.test.ts | 44 ---- .../contract-content/transaction/index.ts | 2 - .../transaction/transaction.ts | 101 -------- ...eatured-contract.mongo.mapper.unit.test.ts | 4 +- .../__tests__/featured.mapper.unit.test.ts | 25 +- .../featured-contract.mongo.collection.ts} | 4 +- .../featured-contract.mongo.mapper.ts | 4 +- .../featured.creator.ts | 18 +- .../__dependencies__/featured.mongo.types.ts | 7 + src/common/featured/__dependencies__/index.ts | 4 + src/common/featured/featured.actions.ts | 23 -- src/common/featured/featured.deltas.ts | 25 -- src/common/featured/featured.errors.ts | 14 ++ src/common/featured/featured.mapper.ts | 40 +++- src/common/featured/featured.types.ts | 10 +- src/common/featured/index.ts | 6 +- src/common/index.ts | 3 +- .../__dependencies__/index.ts | 4 + .../processor-task.mongo.collection.ts} | 16 +- .../processor-task.mongo.mapper.ts | 92 ++++++++ .../processor-task.mongo.types.ts | 18 ++ ...essful-processor-task.mongo.collection.ts} | 4 +- src/common/processor-task-queue/index.ts | 7 +- .../processor-task-queue.ts | 62 ++--- .../processor-task.enums.ts | 4 + .../processor-task.errors.ts | 5 + .../processor-task.source.ts | 5 + .../processor-task-queue/processor-task.ts | 117 ++-------- .../processor-task.types.ts | 38 +-- src/common/types/action-trace.types.ts | 45 ++++ src/common/types/block.types.ts | 20 ++ src/common/types/delta.types.ts | 11 + src/common/types/index.ts | 5 + .../signed-block.types.ts} | 6 +- .../trace.dtos.ts => types/trace.types.ts} | 14 +- .../transaction.types.ts} | 8 +- .../__dependencies__/index.ts | 3 + ...nprocessed-block-queue.mongo.collection.ts | 6 +- .../unprocessed-block-queue.mongo.mapper.ts | 16 +- .../unprocessed-block-queue.mongo.types.ts} | 17 -- src/common/unprocessed-block-queue/index.ts | 7 +- .../unprocessed-block-queue.mapper.ts | 7 - ...n.ts => unprocessed-block-queue.source.ts} | 3 +- .../unprocessed-block-queue.ts | 10 +- src/config/index.ts | 21 +- src/filter/filter.dependencies.ts | 7 +- src/filter/filter.runner.ts | 3 +- src/filter/filter.types.ts | 132 +---------- src/filter/filter.worker.ts | 26 +-- src/filter/start-filter.ts | 6 +- src/processor/processor.dependencies.ts | 23 +- src/processor/processor.runner.ts | 72 ++---- src/processor/processor.types.ts | 1 - .../processors/action-trace.processor.ts | 25 +- src/processor/processors/delta.processor.ts | 13 +- src/processor/start-processor.ts | 23 +- src/reader/reader.dependencies.ts | 3 +- src/reader/reader.ts | 25 +- src/reader/start-reader.ts | 10 +- yarn.lock | 63 ++--- 122 files changed, 909 insertions(+), 2233 deletions(-) create mode 100644 src/api/__dependencies__/contract-action.mongo.mapper.ts rename src/api/{endpoints/actions/data/data-sources => __dependencies__}/contract-action.mongo.source.ts (100%) create mode 100644 src/api/__dependencies__/contract-action.mongo.types.ts delete mode 100644 src/api/endpoints/actions/data/mappers/contract-action.mapper.ts rename src/api/endpoints/actions/{data/dtos/actions.dto.ts => domain/actions.types.ts} (100%) rename src/common/abis/{ => __dependencies__}/abis.creator.ts (84%) rename src/common/abis/{abis.collection.ts => __dependencies__/abis.mongo.collection.ts} (91%) rename src/common/abis/{ => __dependencies__}/abis.mongo.mapper.ts (90%) create mode 100644 src/common/abis/__dependencies__/abis.mongo.types.ts create mode 100644 src/common/abis/__dependencies__/index.ts delete mode 100644 src/common/abis/__tests__/abi.unit.test.ts delete mode 100644 src/common/abis/__tests__/contract-encoded-abi.unit.test.ts delete mode 100644 src/common/abis/contract-encoded-abi.ts rename src/common/block-range-scanner/{ => __dependencies__}/block-range-scan.mongo.source.ts (90%) rename src/common/block-range-scanner/{ => __dependencies__}/block-range-scanner.creator.ts (66%) create mode 100644 src/common/block-range-scanner/__dependencies__/block-range-scanner.mongo.mapper.ts rename src/common/block-range-scanner/{block-range-scanner.dtos.ts => __dependencies__/block-range-scanner.mongo.types.ts} (52%) create mode 100644 src/common/block-range-scanner/__dependencies__/index.ts create mode 100644 src/common/block-range-scanner/block-range-scan.source.ts rename src/common/block-state/{ => __dependencies__}/block-state.creator.ts (84%) rename src/common/block-state/{block-state.collection.ts => __dependencies__/block-state.mongo.collection.ts} (90%) rename src/common/block-state/{ => __dependencies__}/block-state.mongo.mapper.ts (85%) create mode 100644 src/common/block-state/__dependencies__/block-state.mongo.types.ts create mode 100644 src/common/block-state/__dependencies__/index.ts rename src/common/block-state/{ => __dependencies__}/query-builders/update-block-number.mongo.query-builder.ts (100%) delete mode 100644 src/common/contract-content/action-trace/__tests__/action-trace.unit.test.ts delete mode 100644 src/common/contract-content/action-trace/action-trace.dtos.ts delete mode 100644 src/common/contract-content/action-trace/action-trace.ts delete mode 100644 src/common/contract-content/action-trace/index.ts delete mode 100644 src/common/contract-content/delta/__tests__/delta.unit.test.ts delete mode 100644 src/common/contract-content/delta/delta.dtos.ts delete mode 100644 src/common/contract-content/delta/delta.ts delete mode 100644 src/common/contract-content/delta/index.ts delete mode 100644 src/common/contract-content/index.ts delete mode 100644 src/common/contract-content/signed-block/__tests__/signed-block.unit.test.ts delete mode 100644 src/common/contract-content/signed-block/index.ts delete mode 100644 src/common/contract-content/signed-block/signed-block.ts delete mode 100644 src/common/contract-content/trace/__tests__/trace.unit.test.ts delete mode 100644 src/common/contract-content/trace/index.ts delete mode 100644 src/common/contract-content/trace/trace.ts delete mode 100644 src/common/contract-content/transaction/__tests__/transaction.unit.test.ts delete mode 100644 src/common/contract-content/transaction/index.ts delete mode 100644 src/common/contract-content/transaction/transaction.ts rename src/common/featured/{ => __dependencies__}/__tests__/featured-contract.mongo.mapper.unit.test.ts (91%) rename src/common/featured/{ => __dependencies__}/__tests__/featured.mapper.unit.test.ts (91%) rename src/common/featured/{featured-contract.collection.ts => __dependencies__/featured-contract.mongo.collection.ts} (81%) rename src/common/featured/{ => __dependencies__}/featured-contract.mongo.mapper.ts (92%) rename src/common/featured/{ => __dependencies__}/featured.creator.ts (66%) create mode 100644 src/common/featured/__dependencies__/featured.mongo.types.ts create mode 100644 src/common/featured/__dependencies__/index.ts delete mode 100644 src/common/featured/featured.actions.ts delete mode 100644 src/common/featured/featured.deltas.ts create mode 100644 src/common/processor-task-queue/__dependencies__/index.ts rename src/common/processor-task-queue/{data-sources/processor-task.source.ts => __dependencies__/processor-task.mongo.collection.ts} (70%) create mode 100644 src/common/processor-task-queue/__dependencies__/processor-task.mongo.mapper.ts create mode 100644 src/common/processor-task-queue/__dependencies__/processor-task.mongo.types.ts rename src/common/processor-task-queue/{data-sources/unsuccessful-processor-task.source.ts => __dependencies__/unsuccessful-processor-task.mongo.collection.ts} (88%) create mode 100644 src/common/processor-task-queue/processor-task.enums.ts create mode 100644 src/common/processor-task-queue/processor-task.errors.ts create mode 100644 src/common/processor-task-queue/processor-task.source.ts create mode 100644 src/common/types/action-trace.types.ts create mode 100644 src/common/types/block.types.ts create mode 100644 src/common/types/delta.types.ts create mode 100644 src/common/types/index.ts rename src/common/{contract-content/signed-block/signed-block.dtos.ts => types/signed-block.types.ts} (65%) rename src/common/{contract-content/trace/trace.dtos.ts => types/trace.types.ts} (64%) rename src/common/{contract-content/transaction/transaction.dtos.ts => types/transaction.types.ts} (56%) create mode 100644 src/common/unprocessed-block-queue/__dependencies__/index.ts rename src/common/unprocessed-block-queue/{ => __dependencies__}/unprocessed-block-queue.mongo.collection.ts (80%) rename src/common/unprocessed-block-queue/{ => __dependencies__}/unprocessed-block-queue.mongo.mapper.ts (90%) rename src/common/unprocessed-block-queue/{unprocessed-block-queue.types.ts => __dependencies__/unprocessed-block-queue.mongo.types.ts} (58%) delete mode 100644 src/common/unprocessed-block-queue/unprocessed-block-queue.mapper.ts rename src/common/unprocessed-block-queue/{unprocessed-block-queue.collection.ts => unprocessed-block-queue.source.ts} (64%) diff --git a/package.json b/package.json index cf07b82..5e4d945 100644 --- a/package.json +++ b/package.json @@ -35,10 +35,11 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/api-core": "^0.0.144", - "@alien-worlds/block-reader": "^0.0.3", - "@alien-worlds/broadcast": "^0.0.4", - "@alien-worlds/storage-mongodb": "^0.0.14", + "@alien-worlds/api-core": "^0.0.145", + "@alien-worlds/block-reader": "^0.0.4", + "@alien-worlds/broadcast": "^0.0.5", + "@alien-worlds/eos": "^0.0.9", + "@alien-worlds/storage-mongodb": "^0.0.16", "@alien-worlds/workers": "^0.0.3", "@eosrio/node-abieos": "^1", "async": "^3.2.4", diff --git a/src/api/__dependencies__/contract-action.mongo.mapper.ts b/src/api/__dependencies__/contract-action.mongo.mapper.ts new file mode 100644 index 0000000..74a561b --- /dev/null +++ b/src/api/__dependencies__/contract-action.mongo.mapper.ts @@ -0,0 +1,17 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { Entity, Mapper, PropertyMapping } from '@alien-worlds/api-core'; +import { ContractAction } from '@alien-worlds/api-core'; +import { ContractActionMongoModel } from './contract-action.mongo.types'; + +export class ContractActionMapper implements Mapper { + public toEntity(model: any): ContractAction { + throw new Error('Method not implemented.'); + } + public fromEntity(entity: ContractAction): ContractActionMongoModel { + throw new Error('Method not implemented.'); + } + public getEntityKeyMapping(key: string): PropertyMapping { + throw new Error('Method not implemented.'); + } +} diff --git a/src/api/endpoints/actions/data/data-sources/contract-action.mongo.source.ts b/src/api/__dependencies__/contract-action.mongo.source.ts similarity index 100% rename from src/api/endpoints/actions/data/data-sources/contract-action.mongo.source.ts rename to src/api/__dependencies__/contract-action.mongo.source.ts diff --git a/src/api/__dependencies__/contract-action.mongo.types.ts b/src/api/__dependencies__/contract-action.mongo.types.ts new file mode 100644 index 0000000..498caa6 --- /dev/null +++ b/src/api/__dependencies__/contract-action.mongo.types.ts @@ -0,0 +1,6 @@ +import { MongoDB } from '@alien-worlds/storage-mongodb'; + +export type ContractActionMongoModel = { + _id?: MongoDB.ObjectId; + [key: string]: unknown; +}; diff --git a/src/api/endpoints/actions/data/mappers/contract-action.mapper.ts b/src/api/endpoints/actions/data/mappers/contract-action.mapper.ts deleted file mode 100644 index 7d52b27..0000000 --- a/src/api/endpoints/actions/data/mappers/contract-action.mapper.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Entity, Mapper } from '@alien-worlds/api-core'; -import { ContractAction } from '@alien-worlds/api-core'; - -/*imports*/ - -type MapperType = (data: object) => Entity; - -export class ContractActionMapper - implements Mapper -{ - - - - // private mapper: MapperType; - - // constructor(mapper?: MapperType) { - // if (mapper) { - // this.mapper = mapper; - // } else { - // this.mapper = (data: object) => - // ({ - // toDocument: () => data, - // } as Entity); - // } - // } - - // public toEntity(document: ContractActionDocument): ContractAction { - // return ContractAction.fromDocument(document, this.mapper); - // } - // public toDataObject(entity: ContractAction): ContractActionDocument { - // return entity.toDocument(); - // } -} diff --git a/src/api/endpoints/actions/data/dtos/actions.dto.ts b/src/api/endpoints/actions/domain/actions.types.ts similarity index 100% rename from src/api/endpoints/actions/data/dtos/actions.dto.ts rename to src/api/endpoints/actions/domain/actions.types.ts diff --git a/src/api/endpoints/actions/domain/models/list-actions.input.ts b/src/api/endpoints/actions/domain/models/list-actions.input.ts index fe3475f..dd9cd32 100644 --- a/src/api/endpoints/actions/domain/models/list-actions.input.ts +++ b/src/api/endpoints/actions/domain/models/list-actions.input.ts @@ -1,14 +1,9 @@ -import { ListActionsQueryParams } from '../../data/dtos/actions.dto'; -import { - Request, - parseToBigInt, - QueryModel, - MongoAggregateParams, -} from '@alien-worlds/api-core'; +import { ListActionsQueryParams } from '../actions.types'; +import { Request, parseToBigInt } from '@alien-worlds/api-core'; /** * @class */ -export class ListActionsInput implements QueryModel { +export class ListActionsInput { /** * * @param {ListActionsRequestDto} dto @@ -77,43 +72,4 @@ export class ListActionsInput implements QueryModel { public readonly offset: number, public readonly limit: number ) {} - - public toQueryParams(): MongoAggregateParams { - const { - contracts, - names, - accounts, - startBlock, - endBlock, - startTimestamp, - endTimestamp, - offset, - limit, - } = this; - // TODO: use unions and represent it in special collection called ActionRepository - // it should contain all structs - const pipeline = [ - { $match: { field: 'value' } }, - { $project: { field: 1 } }, - { $skip: 1 }, - { $limit: 5 }, - { - $unionWith: { - coll: 'collection2', - pipeline: [ - { $match: { otherField: 'otherValue' } }, - { $project: { otherField: 1 } }, - { $skip: 1 }, - { $limit: 5 }, - ], - }, - }, - ]; - const options = {}; - - return { - pipeline, - options, - }; - } } diff --git a/src/api/endpoints/actions/domain/repositories/contract-action.repository.ts b/src/api/endpoints/actions/domain/repositories/contract-action.repository.ts index ae804a8..6eb472d 100644 --- a/src/api/endpoints/actions/domain/repositories/contract-action.repository.ts +++ b/src/api/endpoints/actions/domain/repositories/contract-action.repository.ts @@ -1,22 +1,10 @@ -import { - injectable, - Repository, - ContractAction, - ContractActionDocument, - Entity, -} from '@alien-worlds/api-core'; +import { injectable, Repository, ContractAction } from '@alien-worlds/api-core'; /** * @abstract * @class */ @injectable() -export abstract class ContractActionRepository< - DataEntityType extends Entity = Entity, - DataDocumentType = object -> extends Repository< - ContractAction, - ContractActionDocument -> { +export abstract class ContractActionRepository extends Repository { public static Token = 'CONTRACT_ACTION_REPOSITORY'; } diff --git a/src/api/endpoints/actions/domain/use-cases/list-actions.use-case.ts b/src/api/endpoints/actions/domain/use-cases/list-actions.use-case.ts index 3bae1af..65350cb 100644 --- a/src/api/endpoints/actions/domain/use-cases/list-actions.use-case.ts +++ b/src/api/endpoints/actions/domain/use-cases/list-actions.use-case.ts @@ -1,6 +1,8 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { ListActionsInput } from './../models/list-actions.input'; import { ContractAction, + FindParams, inject, injectable, Result, @@ -26,7 +28,11 @@ export class ListActionsUseCase implements UseCase { * @returns {Promise>} */ public async execute(input: ListActionsInput): Promise> { - return this.contractActionRepository.find(input); + const result = await this.contractActionRepository.find( + FindParams.create({ limit: 1 }) + ); + + return result; } /*methods*/ diff --git a/src/api/endpoints/actions/index.ts b/src/api/endpoints/actions/index.ts index a96a67d..838d3fc 100644 --- a/src/api/endpoints/actions/index.ts +++ b/src/api/endpoints/actions/index.ts @@ -1,6 +1,6 @@ -export * from './data/data-sources/contract-action.mongo.source'; -export * from './data/dtos/actions.dto'; -export * from './data/mappers/contract-action.mapper'; +export * from '../../__dependencies__/contract-action.mongo.source'; +export * from './domain/actions.types'; +export * from '../../__dependencies__/contract-action.mongo.mapper'; export * from './domain/list-actions.controller'; export * from './domain/models/list-actions.input'; export * from './domain/models/list-actions.output'; diff --git a/src/api/endpoints/actions/ioc.config.ts b/src/api/endpoints/actions/ioc.config.ts index 99d925e..b00e396 100644 --- a/src/api/endpoints/actions/ioc.config.ts +++ b/src/api/endpoints/actions/ioc.config.ts @@ -1,12 +1,13 @@ -import { - RepositoryImpl, - Container, -} from '@alien-worlds/api-core'; +import { RepositoryImpl, Container } from '@alien-worlds/api-core'; -import { ContractActionMongoSource } from './data/data-sources/contract-action.mongo.source'; -import { ContractActionMapper } from './data/mappers/contract-action.mapper'; +import { ContractActionMongoSource } from '../../__dependencies__/contract-action.mongo.source'; +import { ContractActionMapper } from '../../__dependencies__/contract-action.mongo.mapper'; import { ContractActionRepository } from './domain/repositories/contract-action.repository'; -import { MongoConfig, MongoSource } from '@alien-worlds/storage-mongodb'; +import { + MongoConfig, + MongoQueryBuilders, + MongoSource, +} from '@alien-worlds/storage-mongodb'; export const setupContractActionRepository = async ( mongo: MongoSource | MongoConfig, @@ -21,7 +22,8 @@ export const setupContractActionRepository = async ( const repo: ContractActionRepository = new RepositoryImpl( new ContractActionMongoSource(mongoSource), - new ContractActionMapper() + new ContractActionMapper(), + new MongoQueryBuilders() ); if (container) { diff --git a/src/bootstrap/__tests__/start-bootstrap.unit.test.ts b/src/bootstrap/__tests__/start-bootstrap.unit.test.ts index 1437e45..5ad1098 100644 --- a/src/bootstrap/__tests__/start-bootstrap.unit.test.ts +++ b/src/bootstrap/__tests__/start-bootstrap.unit.test.ts @@ -1,15 +1,12 @@ import { Mode } from './../../common/common.enums'; import { BroadcastTcpClient } from '@alien-worlds/broadcast'; import { MongoSource } from '@alien-worlds/storage-mongodb'; -import { Abis, BlockRangeScanner, ContractReader } from '../../common'; +import { AbisCreator } from '../../common'; import { bootstrap } from '../start-bootstrap'; import { NoAbisError } from '../bootstrap.errors'; import { InternalBroadcastMessageName } from '../../broadcast/internal-broadcast.enums'; -import { - createDefaultModeBlockRange, - createTestModeBlockRange, -} from '../bootstrap.utils'; import { ReaderBroadcastMessage } from '../../broadcast/messages'; +import { BlockRangeScannerCreator } from '../../common/block-range-scanner/__dependencies__/block-range-scanner.creator'; jest.mock('../../common', () => ({ Abis: { @@ -44,7 +41,7 @@ jest.mock('../bootstrap.utils', () => ({ createTestModeBlockRange: jest.fn().mockResolvedValue({}), })); -const featured = { +const featuredJson = { traces: [ { shipTraceMessageName: ['transaction_trace_v0'], @@ -73,6 +70,18 @@ const featured = { ], }; +const config = { + mode: Mode.Default, + broadcast: {}, + mongo: {}, + contractReader: {}, + abis: {}, + scanner: {}, + featured: featuredJson, +} as any; + +const dependencies = {} as any; + describe('bootstrap', () => { let mockBroadcast; @@ -81,53 +90,25 @@ describe('bootstrap', () => { }); it('throws an error when there are no abis', async () => { - (Abis.create as jest.Mock).mockResolvedValueOnce({ + (AbisCreator.create as jest.Mock).mockResolvedValueOnce({ fetchAbis: jest.fn().mockResolvedValue([]), }); - - const config = { - mode: Mode.Default, - broadcast: {}, - mongo: {}, - contractReader: {}, - abis: {}, - scanner: {}, - featured, - } as any; - await expect(bootstrap(config)).rejects.toThrow(NoAbisError); + await expect(bootstrap(config, dependencies, featuredJson)).rejects.toThrow( + NoAbisError + ); }); it('runs without error in default mode', async () => { - const config = { - mode: Mode.Default, - broadcast: {}, - mongo: {}, - contractReader: {}, - abis: {}, - scanner: {}, - featured, - } as any; - await bootstrap(config); - - expect(BlockRangeScanner.create).toHaveBeenCalled(); - expect(ContractReader.create).toHaveBeenCalled(); - expect(Abis.create).toHaveBeenCalled(); + await bootstrap(config, dependencies, featuredJson); + + expect(BlockRangeScannerCreator.create).toHaveBeenCalled(); + expect(AbisCreator.create).toHaveBeenCalled(); expect(MongoSource.create).toHaveBeenCalled(); expect(BroadcastTcpClient).toHaveBeenCalled(); }); it.skip('handles DefaultModeReaderReady message correctly', async () => { - const config = { - mode: Mode.Default, - broadcast: {}, - mongo: {}, - contractReader: {}, - abis: {}, - scanner: {}, - featured, - } as any; - await bootstrap(config); - console.log('=====', mockBroadcast.onMessage.mock); + await bootstrap(config, dependencies, featuredJson); const message = { name: InternalBroadcastMessageName.DefaultModeReaderReady }; const messageHandler = mockBroadcast.onMessage.mock.calls[0][1]; await messageHandler(message); @@ -139,16 +120,7 @@ describe('bootstrap', () => { }); it.skip('handles TestModeReaderReady message correctly', async () => { - const config = { - mode: Mode.Test, - broadcast: {}, - mongo: {}, - contractReader: {}, - abis: {}, - scanner: {}, - featured, - } as any; - await bootstrap(config); + await bootstrap(config, dependencies, featuredJson); const message = { name: InternalBroadcastMessageName.DefaultModeReaderReady }; diff --git a/src/bootstrap/bootstrap.utils.ts b/src/bootstrap/bootstrap.utils.ts index f411434..a1384c0 100644 --- a/src/bootstrap/bootstrap.utils.ts +++ b/src/bootstrap/bootstrap.utils.ts @@ -1,14 +1,11 @@ import { BlockchainService, log, parseToBigInt } from '@alien-worlds/api-core'; -import { BlockRangeScanner } from '../common/block-range-scanner'; -import { BlockState } from '../common/block-state'; -import { Mode } from '../common/common.enums'; -import { UnknownModeError } from '../common/common.errors'; import { BlockRangeData, BootstrapConfig } from './bootstrap.types'; import { StartBlockHigherThanEndBlockError, UndefinedStartBlockError, EndBlockOutOfRangeError, } from './bootstrap.errors'; +import { BlockRangeScanner, BlockState, Mode, UnknownModeError } from '../common'; /** * Creates a block range task input based on the provided configuration and mode. diff --git a/src/common/abis/abis.creator.ts b/src/common/abis/__dependencies__/abis.creator.ts similarity index 84% rename from src/common/abis/abis.creator.ts rename to src/common/abis/__dependencies__/abis.creator.ts index cfa8b79..10ec590 100644 --- a/src/common/abis/abis.creator.ts +++ b/src/common/abis/__dependencies__/abis.creator.ts @@ -3,11 +3,10 @@ import { MongoQueryBuilders, MongoSource, } from '@alien-worlds/storage-mongodb'; -import { AbisServiceConfig } from './abis.types'; -import { Abis } from './abis'; +import { Abis } from '../abis'; import { AbiService, log } from '@alien-worlds/api-core'; -import { AbisCollection } from './abis.collection'; -import { AbisRepositoryImpl } from './abis.repository-impl'; +import { AbisCollection } from './abis.mongo.collection'; +import { AbisRepositoryImpl } from '../abis.repository-impl'; import { AbisMongoMapper } from './abis.mongo.mapper'; export class AbisCreator { diff --git a/src/common/abis/abis.collection.ts b/src/common/abis/__dependencies__/abis.mongo.collection.ts similarity index 91% rename from src/common/abis/abis.collection.ts rename to src/common/abis/__dependencies__/abis.mongo.collection.ts index afff1bd..1fefd80 100644 --- a/src/common/abis/abis.collection.ts +++ b/src/common/abis/__dependencies__/abis.mongo.collection.ts @@ -1,5 +1,5 @@ import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; -import { ContractEncodedAbiMongoModel } from './abis.types'; +import { ContractEncodedAbiMongoModel } from './abis.mongo.types'; /** * Represents a collection of ABIs (Application Binary Interfaces) stored in a MongoDB collection. diff --git a/src/common/abis/abis.mongo.mapper.ts b/src/common/abis/__dependencies__/abis.mongo.mapper.ts similarity index 90% rename from src/common/abis/abis.mongo.mapper.ts rename to src/common/abis/__dependencies__/abis.mongo.mapper.ts index 05ae32b..8cdeb85 100644 --- a/src/common/abis/abis.mongo.mapper.ts +++ b/src/common/abis/__dependencies__/abis.mongo.mapper.ts @@ -1,6 +1,6 @@ import { MongoDB, MongoMapper } from '@alien-worlds/storage-mongodb'; -import { ContractEncodedAbiMongoModel } from './abis.types'; import { ContractEncodedAbi } from '@alien-worlds/api-core'; +import { ContractEncodedAbiMongoModel } from './abis.mongo.types'; export class AbisMongoMapper extends MongoMapper< ContractEncodedAbi, diff --git a/src/common/abis/__dependencies__/abis.mongo.types.ts b/src/common/abis/__dependencies__/abis.mongo.types.ts new file mode 100644 index 0000000..9f9c7d3 --- /dev/null +++ b/src/common/abis/__dependencies__/abis.mongo.types.ts @@ -0,0 +1,18 @@ +import { MongoConfig, MongoDB } from '@alien-worlds/storage-mongodb'; + +export type ContractEncodedAbiMongoModel = { + block_number: MongoDB.Long; + contract: string; + hex: string; +}; + +export type AbisServiceConfig = { + url: string; + limit?: number; + filter?: string; +}; + +export type AbisConfig = { + service: AbisServiceConfig; + mongo: MongoConfig; +}; diff --git a/src/common/abis/__dependencies__/index.ts b/src/common/abis/__dependencies__/index.ts new file mode 100644 index 0000000..883f09d --- /dev/null +++ b/src/common/abis/__dependencies__/index.ts @@ -0,0 +1,4 @@ +export * from './abis.creator'; +export * from './abis.mongo.collection'; +export * from './abis.mongo.mapper'; +export * from './abis.mongo.types'; diff --git a/src/common/abis/__tests__/abi.repository-impl.unit.test.ts b/src/common/abis/__tests__/abi.repository-impl.unit.test.ts index a0a6e2f..ac0e4c9 100644 --- a/src/common/abis/__tests__/abi.repository-impl.unit.test.ts +++ b/src/common/abis/__tests__/abi.repository-impl.unit.test.ts @@ -1,13 +1,6 @@ import { AbisRepositoryImpl } from '../abis.repository-impl'; -import { - Result, - CountParams, - Mapper, - QueryBuilders, - DataSource, -} from '@alien-worlds/api-core'; +import { Result, CountParams, ContractEncodedAbi } from '@alien-worlds/api-core'; import { AbisCache } from '../abis.cache'; -import { ContractEncodedAbi } from '../contract-encoded-abi'; jest.mock('../abis.cache'); diff --git a/src/common/abis/__tests__/abi.unit.test.ts b/src/common/abis/__tests__/abi.unit.test.ts deleted file mode 100644 index 20f440e..0000000 --- a/src/common/abis/__tests__/abi.unit.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* eslint-disable @typescript-eslint/no-empty-function */ -/* eslint-disable @typescript-eslint/no-explicit-any */ - -import { Long } from 'mongodb'; -import { ContractEncodedAbi } from '../contract-encoded-abi'; - -const document = { - block_number: Long.fromBigInt(100n), - contract: 'foo', - hex: 'foo_hex', -}; - -describe('Abi Unit tests', () => { - it('"create" should create Abi instance', async () => { - const abi = ContractEncodedAbi.create(100, 'foo', 'foo_hex'); - expect(abi).toBeInstanceOf(ContractEncodedAbi); - }); - - it('"fromDocument" should create Abi instance based on docuemnt data', async () => { - const abi = ContractEncodedAbi.fromDocument({ - block_number: Long.fromBigInt(100n), - contract: 'foo', - hex: 'foo_hex', - }); - expect(abi).toBeInstanceOf(ContractEncodedAbi); - expect(abi.blockNumber).toEqual(100n); - expect(abi.contract).toEqual('foo'); - expect(abi.hex).toEqual('foo_hex'); - }); - - it('"toJson" should return Abi JSON object', async () => { - const abi = ContractEncodedAbi.create(100, 'foo', 'foo_hex'); - expect(abi.toJson()).toEqual({ - blockNumber: 100n, - contract: 'foo', - hex: 'foo_hex', - }); - }); - - it('"toDocument" should return mongo document based on Abi data', async () => { - const abi = ContractEncodedAbi.create(100, 'foo', 'foo_hex'); - expect(abi.toDocument()).toEqual({ - block_number: Long.fromBigInt(100n), - contract: 'foo', - hex: 'foo_hex', - }); - }); -}); diff --git a/src/common/abis/__tests__/abis.unit.test.ts b/src/common/abis/__tests__/abis.unit.test.ts index f4231b3..b6b07f7 100644 --- a/src/common/abis/__tests__/abis.unit.test.ts +++ b/src/common/abis/__tests__/abis.unit.test.ts @@ -1,7 +1,6 @@ import { Abis } from '../abis'; import { AbisRepositoryImpl } from '../abis.repository-impl'; -import { AbisService } from '../abis.service'; -import { Failure, Result } from '@alien-worlds/api-core'; +import { Failure, Result, AbiService } from '@alien-worlds/api-core'; import { AbisServiceNotSetError } from '../abis.errors'; import { AbiNotFoundError } from '@alien-worlds/block-reader'; @@ -9,8 +8,14 @@ import { AbiNotFoundError } from '@alien-worlds/block-reader'; jest.mock('../abis.repository-impl'); jest.mock('../abis.service'); -const mockAbisRepository = AbisRepositoryImpl as jest.MockedClass; -const mockAbisService = AbisService as jest.MockedClass; +const mockAbisRepository = { + cacheAbis: jest.fn(), + getAbis: jest.fn(), + getAbi: jest.fn(), + insertAbis: jest.fn(), + countAbis: jest.fn(), +} as any; +const mockAbisService = { fetchAbis: jest.fn() } as any; describe('Abis', () => { let abis: Abis; @@ -23,8 +28,12 @@ describe('Abis', () => { describe('getAbis', () => { it('should return ABIs from repository when available', async () => { - const mockAbis = [/* Mocked ABIs */]; - mockAbisRepository.prototype.getAbis.mockResolvedValue(Result.withContent(mockAbis)); + const mockAbis = [ + /* Mocked ABIs */ + ]; + mockAbisRepository.prototype.getAbis.mockResolvedValue( + Result.withContent(mockAbis) + ); const result = await abis.getAbis(); @@ -34,7 +43,9 @@ describe('Abis', () => { }); it('should fetch ABIs when none are available in the repository and fetch option is enabled', async () => { - const mockAbis = [/* Mocked ABIs */]; + const mockAbis = [ + /* Mocked ABIs */ + ]; mockAbisRepository.prototype.getAbis.mockResolvedValue(Result.withContent([])); abis.fetchAbis = jest.fn().mockResolvedValue(Result.withContent(mockAbis)); @@ -51,7 +62,7 @@ describe('Abis', () => { describe('getAbi', () => { it('should return ABI from repository when available', async () => { - const mockAbi = /* Mocked ABI */; + const mockAbi = '1234567890'; mockAbisRepository.prototype.getAbi.mockResolvedValue(Result.withContent(mockAbi)); const result = await abis.getAbi(123n, '0x123'); @@ -62,8 +73,10 @@ describe('Abis', () => { }); it('should fetch ABI when not available in the repository and fetch option is enabled', async () => { - const mockAbi = /* Mocked ABI */; - mockAbisRepository.prototype.getAbi.mockResolvedValue(Result.withFailure(Failure.fromError(new AbiNotFoundError()))); + const mockAbi = '1234567890'; + mockAbisRepository.prototype.getAbi.mockResolvedValue( + Result.withFailure(Failure.fromError(new AbiNotFoundError())) + ); abis.fetchAbis = jest.fn().mockResolvedValue(Result.withContent([mockAbi])); const result = await abis.getAbi(123n, '0x123', true); @@ -106,7 +119,9 @@ describe('Abis', () => { }); it('should fetch ABIs using the service', async () => { - const mockAbis = [/* Mocked ABIs */]; + const mockAbis = [ + /* Mocked ABIs */ + ]; const mockContracts = ['0x123', '0x456']; const mockServiceResponse = Result.withContent(mockAbis); @@ -115,8 +130,12 @@ describe('Abis', () => { const result = await abis.fetchAbis(mockContracts); - expect(mockAbisService.prototype.fetchAbis).toHaveBeenCalledTimes(mockContracts.length); - expect(mockAbisService.prototype.fetchAbis).toHaveBeenCalledWith(expect.any(String)); + expect(mockAbisService.prototype.fetchAbis).toHaveBeenCalledTimes( + mockContracts.length + ); + expect(mockAbisService.prototype.fetchAbis).toHaveBeenCalledWith( + expect.any(String) + ); expect(result.isFailure).toBe(false); expect(result.content).toEqual(mockAbis); }); @@ -128,7 +147,7 @@ describe('Abis', () => { it('should cache ABIs in the repository', async () => { const mockContracts = ['0x123', '0x456']; - mockAbisRepository.prototype.cacheAbis.mockResolvedValue(Result.withoutContent()); + mockAbisRepository.prototype.cacheAbis.mockResolvedValue(); const result = await abis.cacheAbis(mockContracts); diff --git a/src/common/abis/__tests__/contract-encoded-abi.unit.test.ts b/src/common/abis/__tests__/contract-encoded-abi.unit.test.ts deleted file mode 100644 index e7a8942..0000000 --- a/src/common/abis/__tests__/contract-encoded-abi.unit.test.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { ContractEncodedAbi } from '../contract-encoded-abi'; -import { parseToBigInt } from '@alien-worlds/api-core'; -import { ContractEncodedAbiMongoModel, ContractEncodedAbiJson } from '../abis.types'; -import { MongoDB } from '@alien-worlds/storage-mongodb'; - -// Mock dependencies -jest.mock('@alien-worlds/api-core'); -jest.mock('@alien-worlds/storage-mongodb'); - -const mockParseToBigInt = parseToBigInt as jest.MockedFunction; -const mockMongoDBLong = MongoDB.Long as jest.MockedClass; - -describe('ContractEncodedAbi', () => { - const blockNumber = BigInt(123); - const contract = '0x123'; - const hex = '0xabcdef'; - - beforeEach(() => { - // Reset mocks - jest.resetAllMocks(); - }); - - describe('fromDocument', () => { - it('should create a ContractEncodedAbi instance from a document', () => { - const document: ContractEncodedAbiMongoModel = { - block_number: mockMongoDBLong.fromBigInt(blockNumber), - contract, - hex, - }; - - const result = ContractEncodedAbi.fromDocument(document); - - expect(result.blockNumber).toBe(blockNumber); - expect(result.contract).toBe(contract); - expect(result.hex).toBe(hex); - }); - - // Add more test cases for different scenarios - }); - - describe('create', () => { - it('should create a ContractEncodedAbi instance with the specified values', () => { - mockParseToBigInt.mockReturnValueOnce(blockNumber); - - const result = ContractEncodedAbi.create(blockNumber, contract, hex); - - expect(mockParseToBigInt).toHaveBeenCalledTimes(1); - expect(mockParseToBigInt).toHaveBeenCalledWith(blockNumber); - expect(result.blockNumber).toBe(blockNumber); - expect(result.contract).toBe(contract); - expect(result.hex).toBe(hex); - }); - - // Add more test cases for different scenarios - }); - - describe('constructor', () => { - it('should initialize the ContractEncodedAbi instance with the provided values', () => { - const result = new ContractEncodedAbi(blockNumber, contract, hex); - - expect(result.blockNumber).toBe(blockNumber); - expect(result.contract).toBe(contract); - expect(result.hex).toBe(hex); - }); - - // Add more test cases for different scenarios - }); - - describe('toDocument', () => { - it('should convert the ContractEncodedAbi instance to a document', () => { - const instance = new ContractEncodedAbi(blockNumber, contract, hex); - const expectedDocument: ContractEncodedAbiMongoModel = { - block_number: mockMongoDBLong.fromBigInt(blockNumber), - contract, - hex, - }; - - const result = instance.toDocument(); - - expect(mockMongoDBLong.fromBigInt).toHaveBeenCalledTimes(1); - expect(mockMongoDBLong.fromBigInt).toHaveBeenCalledWith(blockNumber); - expect(result).toEqual(expectedDocument); - }); - - // Add more test cases for different scenarios - }); - - describe('toJson', () => { - it('should convert the ContractEncodedAbi instance to JSON', () => { - const instance = new ContractEncodedAbi(blockNumber, contract, hex); - const expectedJson: ContractEncodedAbiJson = { - blockNumber, - contract, - hex, - }; - - const result = instance.toJson(); - - expect(result).toEqual(expectedJson); - }); - - // Add more test cases for different scenarios - }); -}); diff --git a/src/common/abis/abis.cache.ts b/src/common/abis/abis.cache.ts index b6a1a1e..95c2340 100644 --- a/src/common/abis/abis.cache.ts +++ b/src/common/abis/abis.cache.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { ContractEncodedAbi } from './contract-encoded-abi'; +import { ContractEncodedAbi } from '@alien-worlds/api-core'; /** * A function that filters ABI entries based on the block number being greater than or equal to the start block. diff --git a/src/common/abis/abis.repository-impl.ts b/src/common/abis/abis.repository-impl.ts index 649ebaa..4d12bae 100644 --- a/src/common/abis/abis.repository-impl.ts +++ b/src/common/abis/abis.repository-impl.ts @@ -1,5 +1,5 @@ -import { ContractEncodedAbi } from './contract-encoded-abi'; import { + ContractEncodedAbi, CountParams, FindParams, log, diff --git a/src/common/abis/abis.repository.ts b/src/common/abis/abis.repository.ts index 064acc1..49529f5 100644 --- a/src/common/abis/abis.repository.ts +++ b/src/common/abis/abis.repository.ts @@ -1,5 +1,4 @@ -import { Result } from '@alien-worlds/api-core'; -import { ContractEncodedAbi } from './contract-encoded-abi'; +import { ContractEncodedAbi, Result } from '@alien-worlds/api-core'; /** * This class manages ContractEncodedAbi entities, providing CRUD operations and additional functionalities such as caching. diff --git a/src/common/abis/abis.types.ts b/src/common/abis/abis.types.ts index 9046d46..6786ef1 100644 --- a/src/common/abis/abis.types.ts +++ b/src/common/abis/abis.types.ts @@ -1,24 +1,5 @@ -import { MongoConfig, MongoDB } from '@alien-worlds/storage-mongodb'; - export type ContractEncodedAbiJson = { blockNumber: bigint; contract: string; hex: string; }; - -export type ContractEncodedAbiMongoModel = { - block_number: MongoDB.Long; - contract: string; - hex: string; -}; - -export type AbisServiceConfig = { - url: string; - limit?: number; - filter?: string; -}; - -export type AbisConfig = { - service: AbisServiceConfig; - mongo: MongoConfig; -}; diff --git a/src/common/abis/contract-encoded-abi.ts b/src/common/abis/contract-encoded-abi.ts deleted file mode 100644 index e185f0a..0000000 --- a/src/common/abis/contract-encoded-abi.ts +++ /dev/null @@ -1,78 +0,0 @@ -// import { parseToBigInt } from '@alien-worlds/api-core'; -// import { ContractEncodedAbiMongoModel, ContractEncodedAbiJson } from './abis.types'; -// import { MongoDB } from '@alien-worlds/storage-mongodb'; - -// /** -// * Represents an encoded ABI (Application Binary Interface) for a contract. -// */ -// export class ContractEncodedAbi { -// /** -// * Creates a ContractEncodedAbi instance from a document. -// * -// * @param {ContractEncodedAbiMongoModel} document - The document containing the encoded ABI data. -// * @returns {ContractEncodedAbi} A new ContractEncodedAbi instance. -// */ -// public static fromDocument(document: ContractEncodedAbiMongoModel): ContractEncodedAbi { -// const { block_number, contract, hex } = document; -// return new ContractEncodedAbi(parseToBigInt(block_number), contract, hex); -// } - -// /** -// * Creates a ContractEncodedAbi instance with the specified block number, contract address, and hex code. -// * -// * @param {unknown} blockNumber - The block number. -// * @param {string} contract - The contract address. -// * @param {string} hex - The hex code representing the ABI. -// * @returns {ContractEncodedAbi} A new ContractEncodedAbi instance. -// */ -// public static create( -// blockNumber: unknown, -// contract: string, -// hex: string -// ): ContractEncodedAbi { -// return new ContractEncodedAbi(parseToBigInt(blockNumber), contract, hex); -// } - -// /** -// * Constructs a new instance of the ContractEncodedAbi class. -// * -// * @param {bigint} blockNumber - The block number. -// * @param {string} contract - The contract address. -// * @param {string} hex - The hex code representing the ABI. -// */ -// constructor( -// public readonly blockNumber: bigint, -// public readonly contract: string, -// public readonly hex: string -// ) {} - -// /** -// * Converts the ContractEncodedAbi instance to a document format. -// * -// * @returns {ContractEncodedAbiMongoModel} The ContractEncodedAbi instance as a document. -// */ -// public toDocument(): ContractEncodedAbiMongoModel { -// const { blockNumber, hex, contract } = this; - -// return { -// block_number: MongoDB.Long.fromBigInt(blockNumber), -// hex, -// contract, -// }; -// } - -// /** -// * Converts the ContractEncodedAbi instance to a JSON format. -// * -// * @returns {ContractEncodedAbiJson} The ContractEncodedAbi instance as JSON. -// */ -// public toJson(): ContractEncodedAbiJson { -// const { blockNumber, hex, contract } = this; - -// return { -// blockNumber, -// hex, -// contract, -// }; -// } -// } diff --git a/src/common/abis/index.ts b/src/common/abis/index.ts index 2f910c0..c354fe3 100644 --- a/src/common/abis/index.ts +++ b/src/common/abis/index.ts @@ -1,8 +1,7 @@ -export * from './contract-encoded-abi'; -export * from './abis'; +export * from './__dependencies__'; export * from './abis.cache'; +export * from './abis.errors'; +export * from './abis.repository-impl'; export * from './abis.repository'; -export * from './abis.service'; +export * from './abis'; export * from './abis.types'; -export * from './abis.errors'; -export * from './abis.serializer'; diff --git a/src/common/block-range-scanner/block-range-scan.mongo.source.ts b/src/common/block-range-scanner/__dependencies__/block-range-scan.mongo.source.ts similarity index 90% rename from src/common/block-range-scanner/block-range-scan.mongo.source.ts rename to src/common/block-range-scanner/__dependencies__/block-range-scan.mongo.source.ts index 08745bc..9a87372 100644 --- a/src/common/block-range-scanner/block-range-scan.mongo.source.ts +++ b/src/common/block-range-scanner/__dependencies__/block-range-scan.mongo.source.ts @@ -1,17 +1,15 @@ /* eslint-disable @typescript-eslint/no-unsafe-return */ -import { - parseToBigInt, -} from '@alien-worlds/api-core'; +import { parseToBigInt } from '@alien-worlds/api-core'; import { Long } from 'mongodb'; -import { BlockRangeScanDocument } from './block-range-scanner.dtos'; +import { BlockRangeScanModel } from './block-range-scanner.mongo.types'; import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; /** * Block range scan nodes data source from the mongo database * @class */ -export class BlockRangeScanMongoSource extends MongoCollectionSource { +export class BlockRangeScanMongoSource extends MongoCollectionSource { public static Token = 'BLOCK_RANGE_SCAN_MONGO_SOURCE'; /** @@ -23,7 +21,7 @@ export class BlockRangeScanMongoSource extends MongoCollectionSource { + public async startNextScan(scanKey: string): Promise { const result = await this.collection.findOneAndUpdate( { $and: [ @@ -75,7 +73,7 @@ export class BlockRangeScanMongoSource extends MongoCollectionSource { - const range: BlockRangeScanDocument = await this.findRangeForBlockNumber( + const range: BlockRangeScanModel = await this.findRangeForBlockNumber( blockNumber, scanKey ); diff --git a/src/common/block-range-scanner/block-range-scanner.creator.ts b/src/common/block-range-scanner/__dependencies__/block-range-scanner.creator.ts similarity index 66% rename from src/common/block-range-scanner/block-range-scanner.creator.ts rename to src/common/block-range-scanner/__dependencies__/block-range-scanner.creator.ts index d9403d5..70b8f2c 100644 --- a/src/common/block-range-scanner/block-range-scanner.creator.ts +++ b/src/common/block-range-scanner/__dependencies__/block-range-scanner.creator.ts @@ -1,9 +1,10 @@ import { log } from '@alien-worlds/api-core'; -import { BlockRangeScanRepository } from './block-range-scan.repository'; -import { BlockRangeScanConfig } from './block-range-scanner.config'; +import { BlockRangeScanRepository } from '../block-range-scan.repository'; +import { BlockRangeScanConfig } from '../block-range-scanner.config'; import { BlockRangeScanMongoSource } from './block-range-scan.mongo.source'; import { MongoConfig, MongoSource } from '@alien-worlds/storage-mongodb'; -import { BlockRangeScanner } from './block-range-scanner'; +import { BlockRangeScanner } from '../block-range-scanner'; +import { BlockRangeScanMongoMapper } from './block-range-scanner.mongo.mapper'; /** * @class @@ -23,7 +24,8 @@ export class BlockRangeScannerCreator { mongoSource = await MongoSource.create(mongo); } const source = new BlockRangeScanMongoSource(mongoSource); - const repository = new BlockRangeScanRepository(source, config); + const mapper = new BlockRangeScanMongoMapper(); + const repository = new BlockRangeScanRepository(source, mapper, config.maxChunkSize); const scanner: BlockRangeScanner = new BlockRangeScanner(repository); log(` * Block Range Scanner ... [ready]`); diff --git a/src/common/block-range-scanner/__dependencies__/block-range-scanner.mongo.mapper.ts b/src/common/block-range-scanner/__dependencies__/block-range-scanner.mongo.mapper.ts new file mode 100644 index 0000000..1a2f159 --- /dev/null +++ b/src/common/block-range-scanner/__dependencies__/block-range-scanner.mongo.mapper.ts @@ -0,0 +1,109 @@ +import { parseToBigInt } from '@alien-worlds/api-core'; +import { MongoDB, MongoMapper } from '@alien-worlds/storage-mongodb'; +import { BlockRangeScan, BlockRangeScanParent } from '../block-range-scan'; +import { + BlockRangeScanIdModel, + BlockRangeScanModel, +} from './block-range-scanner.mongo.types'; + +export class BlockRangeScanMongoMapper extends MongoMapper< + BlockRangeScan, + BlockRangeScanModel +> { + protected toBlockRangeScanParent(model: BlockRangeScanIdModel) { + const { start, end, scan_key, tree_depth } = model; + + return new BlockRangeScanParent( + parseToBigInt(start), + parseToBigInt(end), + scan_key, + tree_depth + ); + } + + protected fromBlockRangeScanParent(entity: BlockRangeScanParent) { + const { start, end, scanKey, treeDepth } = entity; + const model = { + start: MongoDB.Long.fromString(start.toString()), + end: MongoDB.Long.fromString(end.toString()), + scan_key: scanKey, + tree_depth: treeDepth, + }; + + return model; + } + + public toEntity(model: BlockRangeScanModel): BlockRangeScan { + const { + _id: { start, end, scan_key, tree_depth }, + hash, + processed_block, + timestamp, + start_timestamp, + end_timestamp, + parent_id, + is_leaf_node, + } = model; + + const parent = parent_id ? this.toBlockRangeScanParent(parent_id) : null; + + let processedBlock: bigint; + + if (processed_block) { + processedBlock = parseToBigInt(processed_block); + } + + return new BlockRangeScan( + hash, + parseToBigInt(start), + parseToBigInt(end), + scan_key, + tree_depth, + parent, + is_leaf_node, + processedBlock, + timestamp, + start_timestamp, + end_timestamp + ); + } + + public fromEntity(entity: BlockRangeScan): BlockRangeScanModel { + const { start, scanKey, end, treeDepth, hash } = entity; + const model: BlockRangeScanModel = { + _id: { + start: MongoDB.Long.fromString(start.toString()), + end: MongoDB.Long.fromString(end.toString()), + scan_key: scanKey, + tree_depth: treeDepth, + }, + hash, + }; + + if (typeof entity.processedBlock == 'bigint') { + model.processed_block = MongoDB.Long.fromString(entity.processedBlock.toString()); + } + + if (typeof entity.isLeafNode == 'boolean') { + model.is_leaf_node = entity.isLeafNode; + } + + if (entity.parent) { + model.parent_id = this.fromBlockRangeScanParent(entity.parent); + } + + if (entity.timestamp) { + model.timestamp = entity.timestamp; + } + + if (entity.startTimestamp) { + model.start_timestamp = entity.startTimestamp; + } + + if (entity.endTimestamp) { + model.end_timestamp = entity.endTimestamp; + } + + return model; + } +} diff --git a/src/common/block-range-scanner/block-range-scanner.dtos.ts b/src/common/block-range-scanner/__dependencies__/block-range-scanner.mongo.types.ts similarity index 52% rename from src/common/block-range-scanner/block-range-scanner.dtos.ts rename to src/common/block-range-scanner/__dependencies__/block-range-scanner.mongo.types.ts index bbc02ef..6bc14e1 100644 --- a/src/common/block-range-scanner/block-range-scanner.dtos.ts +++ b/src/common/block-range-scanner/__dependencies__/block-range-scanner.mongo.types.ts @@ -1,24 +1,24 @@ -import { Long } from 'mongodb'; +import { MongoDB } from '@alien-worlds/storage-mongodb'; /* We need to keep tree_depth in _id because if the entire scan will use only one node, we will not be able to create a child document with the same _id and tree_depth: 1. */ -export type BlockRangeScanIdDocument = { - start: Long; - end: Long; +export type BlockRangeScanIdModel = { + start: MongoDB.Long; + end: MongoDB.Long; scan_key: string; tree_depth: number; }; -export type BlockRangeScanDocument = { - _id: BlockRangeScanIdDocument; - hash?: string, - processed_block?: Long; +export type BlockRangeScanModel = { + _id: BlockRangeScanIdModel; + hash?: string; + processed_block?: MongoDB.Long; timestamp?: Date; start_timestamp?: Date; end_timestamp?: Date; is_leaf_node?: boolean; - parent_id?: BlockRangeScanIdDocument; + parent_id?: BlockRangeScanIdModel; }; diff --git a/src/common/block-range-scanner/__dependencies__/index.ts b/src/common/block-range-scanner/__dependencies__/index.ts new file mode 100644 index 0000000..2d8ce77 --- /dev/null +++ b/src/common/block-range-scanner/__dependencies__/index.ts @@ -0,0 +1,4 @@ +export * from './block-range-scan.mongo.source'; +export * from './block-range-scanner.creator'; +export * from './block-range-scanner.mongo.mapper'; +export * from './block-range-scanner.mongo.types'; diff --git a/src/common/block-range-scanner/block-range-scan.repository.ts b/src/common/block-range-scanner/block-range-scan.repository.ts index 9b8c464..21aa099 100644 --- a/src/common/block-range-scanner/block-range-scan.repository.ts +++ b/src/common/block-range-scanner/block-range-scan.repository.ts @@ -1,6 +1,6 @@ import { BlockRangeScan } from './block-range-scan'; -import { BlockRangeScanMongoSource } from './block-range-scan.mongo.source'; -import { BlockRangeScanConfig } from './block-range-scanner.config'; +import { BlockRangeScanSource } from './block-range-scan.source'; +import { Mapper } from '@alien-worlds/api-core'; export type ScanRequest = { error?: Error; @@ -8,14 +8,15 @@ export type ScanRequest = { export class BlockRangeScanRepository { constructor( - private readonly source: BlockRangeScanMongoSource, - private readonly config: BlockRangeScanConfig + private readonly source: BlockRangeScanSource, + private readonly mapper: Mapper, + private readonly maxChunkSize: number ) {} public async startNextScan(scanKey: string): Promise { try { const document = await this.source.startNextScan(scanKey); - return document ? BlockRangeScan.fromDocument(document) : null; + return document ? this.mapper.toEntity(document) : null; } catch (error) { return null; } @@ -27,14 +28,14 @@ export class BlockRangeScanRepository { endBlock: bigint ): Promise { try { - const { maxChunkSize } = this.config; + const { maxChunkSize } = this; const rootRange = BlockRangeScan.create(startBlock, endBlock, scanKey, 0); const rangesToPersist = [rootRange]; const childRanges = BlockRangeScan.createChildRanges(rootRange, maxChunkSize); childRanges.forEach(range => rangesToPersist.push(range)); - const documents = rangesToPersist.map(range => range.toDocument()); + const documents = rangesToPersist.map(range => this.mapper.fromEntity(range)); await this.source.insert(documents); return {}; diff --git a/src/common/block-range-scanner/block-range-scan.source.ts b/src/common/block-range-scanner/block-range-scan.source.ts new file mode 100644 index 0000000..1b305a8 --- /dev/null +++ b/src/common/block-range-scanner/block-range-scan.source.ts @@ -0,0 +1,29 @@ +/* eslint-disable @typescript-eslint/no-unsafe-return */ + +import { DataSource } from '@alien-worlds/api-core'; + +export abstract class BlockRangeScanSource extends DataSource { + public abstract startNextScan(scanKey: string): Promise; + public abstract countScanNodes( + scanKey: string, + startBlock: bigint, + endBlock: bigint + ): Promise; + public abstract removeAll(scanKey: string); + public abstract hasScanKey( + scanKey: string, + startBlock?: bigint, + endBlock?: bigint + ): Promise; + public abstract hasUnscannedNodes( + scanKey: string, + startBlock?: bigint, + endBlock?: bigint + ): Promise; + public abstract findRangeForBlockNumber(blockNumber: bigint, scanKey: string); + public abstract findCompletedParentNode(document: T); + public abstract updateProcessedBlockNumber( + scanKey: string, + blockNumber: bigint + ): Promise; +} diff --git a/src/common/block-range-scanner/block-range-scan.ts b/src/common/block-range-scanner/block-range-scan.ts index 391f642..77ea5f3 100644 --- a/src/common/block-range-scanner/block-range-scan.ts +++ b/src/common/block-range-scanner/block-range-scan.ts @@ -1,12 +1,7 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import crypto from 'crypto'; import { parseToBigInt, removeUndefinedProperties } from '@alien-worlds/api-core'; -import { Long } from 'mongodb'; import { serialize } from 'v8'; -import { - BlockRangeScanDocument, - BlockRangeScanIdDocument, -} from './block-range-scanner.dtos'; export class BlockRangeScanParent { /** @@ -16,7 +11,7 @@ export class BlockRangeScanParent { * @param {bigint} end * @param {string} scanKey */ - private constructor( + constructor( public readonly start: bigint, public readonly end: bigint, public readonly scanKey: string, @@ -27,29 +22,6 @@ export class BlockRangeScanParent { return new BlockRangeScanParent(start, end, scanKey, treeDepth); } - public static fromDocument(document: BlockRangeScanIdDocument) { - const { start, end, scan_key, tree_depth } = document; - - return new BlockRangeScanParent( - parseToBigInt(start), - parseToBigInt(end), - scan_key, - tree_depth - ); - } - - public toDocument() { - const { start, end, scanKey, treeDepth } = this; - const doc = { - start: Long.fromString(start.toString()), - end: Long.fromString(end.toString()), - scan_key: scanKey, - tree_depth: treeDepth, - }; - - return doc; - } - public toJson() { const { start, end, scanKey, treeDepth } = this; @@ -61,7 +33,7 @@ export class BlockRangeScanParent { * @class */ export class BlockRangeScan { - protected constructor( + constructor( public readonly hash: string, public readonly start: bigint, public readonly end: bigint, @@ -138,41 +110,6 @@ export class BlockRangeScan { ); } - public static fromDocument(document: BlockRangeScanDocument) { - const { - _id: { start, end, scan_key, tree_depth }, - hash, - processed_block, - timestamp, - start_timestamp, - end_timestamp, - parent_id, - is_leaf_node, - } = document; - - const parent = parent_id ? BlockRangeScanParent.fromDocument(parent_id) : null; - - let processedBlock: bigint; - - if (processed_block) { - processedBlock = parseToBigInt(processed_block); - } - - return new BlockRangeScan( - hash, - parseToBigInt(start), - parseToBigInt(end), - scan_key, - tree_depth, - parent, - is_leaf_node, - processedBlock, - timestamp, - start_timestamp, - end_timestamp - ); - } - public static createChildRanges( blockRange: BlockRangeScan, maxChunkSize: number @@ -212,45 +149,6 @@ export class BlockRangeScan { this.isLeafNode = true; } - public toDocument() { - const { start, scanKey, end, treeDepth, hash } = this; - const doc: BlockRangeScanDocument = { - _id: { - start: Long.fromString(start.toString()), - end: Long.fromString(end.toString()), - scan_key: scanKey, - tree_depth: treeDepth, - }, - hash, - }; - - if (typeof this.processedBlock == 'bigint') { - doc.processed_block = Long.fromString(this.processedBlock.toString()); - } - - if (typeof this.isLeafNode == 'boolean') { - doc.is_leaf_node = this.isLeafNode; - } - - if (this.parent) { - doc.parent_id = this.parent.toDocument(); - } - - if (this.timestamp) { - doc.timestamp = this.timestamp; - } - - if (this.startTimestamp) { - doc.start_timestamp = this.startTimestamp; - } - - if (this.endTimestamp) { - doc.end_timestamp = this.endTimestamp; - } - - return doc; - } - public toJson(): BlockRangeScan { const { start, diff --git a/src/common/block-range-scanner/index.ts b/src/common/block-range-scanner/index.ts index 6432c68..8111f3b 100644 --- a/src/common/block-range-scanner/index.ts +++ b/src/common/block-range-scanner/index.ts @@ -1,7 +1,7 @@ -export * from './block-range-scan.mongo.source'; +export * from './__dependencies__'; export * from './block-range-scan.repository'; +export * from './block-range-scan.source'; export * from './block-range-scan'; export * from './block-range-scanner.config'; -export * from './block-range-scanner.dtos'; export * from './block-range-scanner.errors'; export * from './block-range-scanner'; diff --git a/src/common/block-state/block-state.creator.ts b/src/common/block-state/__dependencies__/block-state.creator.ts similarity index 84% rename from src/common/block-state/block-state.creator.ts rename to src/common/block-state/__dependencies__/block-state.creator.ts index 7126d60..e76f187 100644 --- a/src/common/block-state/block-state.creator.ts +++ b/src/common/block-state/__dependencies__/block-state.creator.ts @@ -4,10 +4,10 @@ import { MongoQueryBuilders, MongoSource, } from '@alien-worlds/storage-mongodb'; -import { BlockState } from './block-state'; +import { BlockState } from '../block-state'; import { UpdateBlockNumberMongoQueryBuilder } from './query-builders/update-block-number.mongo.query-builder'; -import { BlockMongoCollection } from '../../reader'; import { BlockStateMongoMapper } from './block-state.mongo.mapper'; +import { BlockStateCollection } from './block-state.mongo.collection'; export class BlockStateCreator { public static async create(mongo: MongoSource | MongoConfig) { @@ -17,7 +17,7 @@ export class BlockStateCreator { const updateBlockNumberQueryBuilder = new UpdateBlockNumberMongoQueryBuilder(); const mongoSource = mongo instanceof MongoSource ? mongo : await MongoSource.create(mongo); - const collection = new BlockMongoCollection(mongoSource); + const collection = new BlockStateCollection(mongoSource); const state = new BlockState( collection, mapper, diff --git a/src/common/block-state/block-state.collection.ts b/src/common/block-state/__dependencies__/block-state.mongo.collection.ts similarity index 90% rename from src/common/block-state/block-state.collection.ts rename to src/common/block-state/__dependencies__/block-state.mongo.collection.ts index ca1e283..ee7e4a9 100644 --- a/src/common/block-state/block-state.collection.ts +++ b/src/common/block-state/__dependencies__/block-state.mongo.collection.ts @@ -1,5 +1,5 @@ import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; -import { BlockStateMongoModel } from './block-state.types'; +import { BlockStateMongoModel } from './block-state.mongo.types'; /** * Represents a collection of BlockStateDocument objects in a MongoDB database. * Extends the MongoCollectionSource class. diff --git a/src/common/block-state/block-state.mongo.mapper.ts b/src/common/block-state/__dependencies__/block-state.mongo.mapper.ts similarity index 85% rename from src/common/block-state/block-state.mongo.mapper.ts rename to src/common/block-state/__dependencies__/block-state.mongo.mapper.ts index f94d13d..77da4a6 100644 --- a/src/common/block-state/block-state.mongo.mapper.ts +++ b/src/common/block-state/__dependencies__/block-state.mongo.mapper.ts @@ -1,4 +1,5 @@ -import { BlockStateModel, BlockStateMongoModel } from './block-state.types'; +import { BlockStateModel } from '../block-state.types'; +import { BlockStateMongoModel } from './block-state.mongo.types'; import { MongoDB, MongoMapper } from '@alien-worlds/storage-mongodb'; export class BlockStateMongoMapper extends MongoMapper< diff --git a/src/common/block-state/__dependencies__/block-state.mongo.types.ts b/src/common/block-state/__dependencies__/block-state.mongo.types.ts new file mode 100644 index 0000000..819a6b5 --- /dev/null +++ b/src/common/block-state/__dependencies__/block-state.mongo.types.ts @@ -0,0 +1,9 @@ +import { MongoDB } from '@alien-worlds/storage-mongodb'; + +export type BlockStateMongoModel = { + _id: MongoDB.ObjectId; + last_modified_timestamp: Date; + block_number: MongoDB.Long; + actions: string[]; + tables: string[]; +}; diff --git a/src/common/block-state/__dependencies__/index.ts b/src/common/block-state/__dependencies__/index.ts new file mode 100644 index 0000000..05e6e31 --- /dev/null +++ b/src/common/block-state/__dependencies__/index.ts @@ -0,0 +1,5 @@ +export * from './block-state.creator'; +export * from './block-state.mongo.collection'; +export * from './block-state.mongo.mapper'; +export * from './block-state.mongo.types'; +export * from './query-builders/update-block-number.mongo.query-builder'; diff --git a/src/common/block-state/query-builders/update-block-number.mongo.query-builder.ts b/src/common/block-state/__dependencies__/query-builders/update-block-number.mongo.query-builder.ts similarity index 100% rename from src/common/block-state/query-builders/update-block-number.mongo.query-builder.ts rename to src/common/block-state/__dependencies__/query-builders/update-block-number.mongo.query-builder.ts diff --git a/src/common/block-state/__tests__/block-state.unit.test.ts b/src/common/block-state/__tests__/block-state.unit.test.ts index 738818a..eadfac7 100644 --- a/src/common/block-state/__tests__/block-state.unit.test.ts +++ b/src/common/block-state/__tests__/block-state.unit.test.ts @@ -1,13 +1,4 @@ -import { - DataSource, - Mapper, - QueryBuilders, - QueryBuilder, - Result, - RepositoryImpl, - Failure, -} from '@alien-worlds/api-core'; -import { BlockStateModel, BlockStateMongoModel } from '../block-state.types'; +import { RepositoryImpl, Failure, Result } from '@alien-worlds/api-core'; import { BlockState } from '../block-state'; jest.mock('@alien-worlds/api-core'); diff --git a/src/common/block-state/block-state.ts b/src/common/block-state/block-state.ts index 473b6a1..6475cba 100644 --- a/src/common/block-state/block-state.ts +++ b/src/common/block-state/block-state.ts @@ -1,19 +1,18 @@ import { DataSource, Failure, + Mapper, QueryBuilder, QueryBuilders, RepositoryImpl, Result, } from '@alien-worlds/api-core'; -import { BlockStateModel, BlockStateMongoModel } from './block-state.types'; -import { BlockStateMongoMapper } from './block-state.mongo.mapper'; +import { BlockStateModel } from './block-state.types'; /** * A class representing a block state. - * @extends RepositoryImpl */ -export class BlockState extends RepositoryImpl { +export class BlockState extends RepositoryImpl { /** * Creates an instance of the BlockState class. * @@ -23,8 +22,8 @@ export class BlockState extends RepositoryImpl, - mapper: BlockStateMongoMapper, + source: DataSource, + mapper: Mapper, queryBuilders: QueryBuilders, private updateBlockNumberQueryBuilder: QueryBuilder ) { diff --git a/src/common/block-state/block-state.types.ts b/src/common/block-state/block-state.types.ts index 4a9243b..b7c8e9e 100644 --- a/src/common/block-state/block-state.types.ts +++ b/src/common/block-state/block-state.types.ts @@ -1,13 +1,3 @@ -import { MongoDB } from '@alien-worlds/storage-mongodb'; - -export type BlockStateMongoModel = { - _id: MongoDB.ObjectId; - last_modified_timestamp: Date; - block_number: MongoDB.Long; - actions: string[]; - tables: string[]; -}; - export type BlockStateModel = { lastModifiedTimestamp: Date; blockNumber: bigint; diff --git a/src/common/block-state/index.ts b/src/common/block-state/index.ts index 9229482..3ccdb17 100644 --- a/src/common/block-state/index.ts +++ b/src/common/block-state/index.ts @@ -1,4 +1,3 @@ +export * from './__dependencies__'; export * from './block-state'; -export * from './block-state.collection'; export * from './block-state.types'; -export * from './query-builders/update-block-number.mongo.query-builder'; diff --git a/src/common/contract-content/action-trace/__tests__/action-trace.unit.test.ts b/src/common/contract-content/action-trace/__tests__/action-trace.unit.test.ts deleted file mode 100644 index 4c82cef..0000000 --- a/src/common/contract-content/action-trace/__tests__/action-trace.unit.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { Act, ActionTrace, Receipt } from '../action-trace'; -import { ActJson, ActionTraceDto } from '../action-trace.dtos'; - -const actDto: ActJson = { - account: 'foo.account', - name: 'foo.name', - authorization: { - actor: 'foo.actor', - permission: 'foo.permission', - }, - data: [] as any, -}; - -const receiptDto = { - receiver: 'foo', - act_digest: 'act', - global_sequence: '100', - recv_sequence: '100', - auth_sequence: [{ account: 'foo', sequence: 'foo_sequence' }], - code_sequence: 100, - abi_sequence: 100, -}; - -describe('Act Unit tests', () => { - it('"create" should create entity', () => { - const entity = Act.create(actDto); - expect(entity.account).toEqual(actDto.account); - expect(entity.name).toEqual(actDto.name); - expect(entity.authorization).toEqual(actDto.authorization); - expect(entity.data).toEqual(actDto.data); - }); -}); - -describe('Receipt Unit tests', () => { - it('"create" should create entity', () => { - const entity = Receipt.create('foo', receiptDto); - - expect(entity).not.toBeUndefined(); - }); -}); - -const actionTraceDto: ActionTraceDto = { - action_ordinal: 1, - creator_action_ordinal: 1, - receipt: ['foo', receiptDto], - receiver: 'receiver', - act: actDto, - context_free: true, - elapsed: 'elapsed', - console: 'foo_console', - account_ram_deltas: [], - except: '', - error_code: '200', -}; - -describe('ActionTrace Unit tests', () => { - it('"create" should create entity', () => { - const entity = ActionTrace.create('foo', actionTraceDto); - - expect(entity).not.toBeUndefined(); - }); -}); diff --git a/src/common/contract-content/action-trace/action-trace.dtos.ts b/src/common/contract-content/action-trace/action-trace.dtos.ts deleted file mode 100644 index a080d68..0000000 --- a/src/common/contract-content/action-trace/action-trace.dtos.ts +++ /dev/null @@ -1,80 +0,0 @@ -export type AuthSequenceJson = { - account: string; - sequence: string; -}; - -export type ReceiptJson = { - receiver: string; - act_digest: string; - global_sequence: string; - recv_sequence: string; - auth_sequence: AuthSequenceJson[]; - code_sequence: number; - abi_sequence: number; -}; - -export type ReceiptByNameDto = [string, ReceiptJson]; - -export type ActAuthJson = { - actor: string; - permission: string; -}; - -export type ActJson = { - account: string; - name: string; - authorization: ActAuthJson; - data: Uint8Array; -}; - -export type ActionTraceDto = { - ship_message_name?: string; - action_ordinal?: number; - creator_action_ordinal?: number; - receipt?: ReceiptByNameDto; - receiver?: string; - act?: ActJson; - context_free?: boolean; - elapsed?: string; - console?: string; - account_ram_deltas?: unknown[]; - except?: unknown; - error_code?: string | number; -}; - -export type ActionTraceByNameDto = [string, ActionTraceDto]; - -export type ActionTraceModel = { - shipTraceMessageName?: string; - actionOrdinal?: number; - creatorActionOrdinal?: number; - receipt?: { - shipMessageName?: string; - receiver?: string; - actDigest?: string; - globalSequence?: bigint; - recvSequence?: bigint; - authSequence?: { - account?: string; - sequence?: string; - }[]; - codeSequence?: number; - abiSequence?: number; - }; - receiver?: string; - act?: { - account?: string; - name?: string; - authorization?: { - actor?: string; - permission?: string; - }; - data?: Uint8Array; - }; - isContextFree?: boolean; - elapsed?: string; - console?: string; - accountRamDeltas?: unknown[]; - except?: unknown; - errorCode?: number; -}; diff --git a/src/common/contract-content/action-trace/action-trace.ts b/src/common/contract-content/action-trace/action-trace.ts deleted file mode 100644 index 29d25da..0000000 --- a/src/common/contract-content/action-trace/action-trace.ts +++ /dev/null @@ -1,217 +0,0 @@ -import { parseToBigInt } from '@alien-worlds/api-core'; -import { ActAuthJson, ActJson, ActionTraceDto, ReceiptJson } from './action-trace.dtos'; - -/** - * Represents the authorization information of an action. - * @class - */ -export class ActAuth { - /** - * Creates an ActAuth instance based on the provided json. - * - * @param {ActAuthJson} json - The json containing the authorization information. - * @returns {ActAuth} The created ActAuth instance. - */ - public static create(json: ActAuthJson): ActAuth { - const { actor, permission } = json; - - return new ActAuth(actor, permission); - } - - /** - * Creates an instance of ActAuth. - * - * @param {string} actor - The actor associated with the authorization. - * @param {string} permission - The permission associated with the authorization. - */ - constructor(public readonly actor: string, public readonly permission: string) {} -} - -/** - * Represents an action. - * @class - */ -export class Act { - /** - * Creates an Act instance based on the provided json. - * - * @param {ActJson} json - The json containing the action information. - * @returns {Act} The created Act instance. - */ - public static create(json: ActJson): Act { - const { account, name, data } = json; - - let authorization: ActAuth; - - if (json.authorization) { - authorization = ActAuth.create(json.authorization); - } - - return new Act(account, name, authorization, data); - } - - /** - * Creates an instance of Act. - * - * @param {string} account - The account associated with the action. - * @param {string} name - The name of the action. - * @param {ActAuth | undefined} authorization - The authorization information for the action. - * @param {Uint8Array} data - The data associated with the action. - */ - constructor( - public readonly account: string, - public readonly name: string, - public readonly authorization: ActAuth, - public readonly data: Uint8Array - ) {} -} - -/** - * Represents the sequence information of an authorization. - * - * @typedef {Object} AuthSequence - * @property {string} account - The account associated with the sequence. - * @property {string} sequence - The sequence number. - */ -export type AuthSequence = { - account: string; - sequence: string; -}; - -/** - * Represents the receipt information of an action. - * @class - */ -export class Receipt { - /** - * Creates a Receipt instance based on the provided json. - * - * @param {string} shipMessageName - The name of the ship message. - * @param {ReceiptJson} json - The json containing the receipt information. - * @returns {Receipt} The created Receipt instance. - */ - public static create(shipMessageName: string, json: ReceiptJson): Receipt { - const { - receiver, - act_digest, - global_sequence, - recv_sequence, - auth_sequence, - code_sequence, - abi_sequence, - } = json; - return new Receipt( - shipMessageName, - receiver, - act_digest, - parseToBigInt(global_sequence), - parseToBigInt(recv_sequence), - auth_sequence, - code_sequence, - abi_sequence - ); - } - - /** - * Creates an instance of Receipt. - * - * @param {string} shipMessageName - The name of the ship message. - * @param {string} receiver - The receiver of the action. - * @param {string} actDigest - The digest of the action. - * @param {bigint} globalSequence - The global sequence number. - * @param {bigint} recvSequence - The receiver sequence number. - * @param {AuthSequence[]} authSequence - The authorization sequence information. - * @param {number} codeSequence - The code sequence number. - * @param {number} abiSequence - The ABI sequence number. - */ - constructor( - public readonly shipMessageName: string, - public readonly receiver: string, - public readonly actDigest: string, - public readonly globalSequence: bigint, - public readonly recvSequence: bigint, - public readonly authSequence: AuthSequence[], - public readonly codeSequence: number, - public readonly abiSequence: number - ) {} -} - -/** - * Represents the trace information of an action. - * @class - */ -export class ActionTrace { - /** - * Creates an ActionTrace instance based on the provided DTO. - * - * @param {string} shipMessageName - The name of the ship message. - * @param {ActionTraceDto} dto - The DTO containing the action trace information. - * @returns {ActionTrace} The created ActionTrace instance. - */ - public static create(shipMessageName: string, dto: ActionTraceDto): ActionTrace { - const { - action_ordinal, - creator_action_ordinal, - receiver, - act, - context_free, - elapsed, - console, - account_ram_deltas, - except, - error_code, - } = dto; - - let receipt: Receipt; - if (dto.receipt && dto.receipt.length) { - const [receiptType, receiptContent] = dto.receipt; - receipt = Receipt.create(receiptType, receiptContent); - } - - return new ActionTrace( - shipMessageName, - action_ordinal, - creator_action_ordinal, - receipt, - receiver, - Act.create(act), - context_free, - elapsed, - console, - account_ram_deltas, - except, - Number(error_code) - ); - } - - /** - * Creates an instance of ActionTrace. - * - * @param {string} shipMessageName - The name of the ship message. - * @param {number} actionOrdinal - The ordinal of the action. - * @param {number} creatorActionOrdinal - The ordinal of the creator action. - * @param {Receipt | null} receipt - The receipt information of the action. - * @param {string} receiver - The receiver of the action. - * @param {Act} act - The action object. - * @param {boolean} isContextFree - Indicates whether the action is context-free. - * @param {string} elapsed - The elapsed time of the action. - * @param {string} console - The console output of the action. - * @param {unknown[]} accountRamDeltas - The account RAM deltas. - * @param {unknown} except - The exception information. - * @param {number} errorCode - The error code. - */ - constructor( - public readonly shipMessageName: string, - public readonly actionOrdinal: number, - public readonly creatorActionOrdinal: number, - public readonly receipt: Receipt | null, - public readonly receiver: string, - public readonly act: Act, - public readonly isContextFree: boolean, - public readonly elapsed: string, - public readonly console: string, - public readonly accountRamDeltas: unknown[], - public readonly except: unknown, - public readonly errorCode: number - ) {} -} diff --git a/src/common/contract-content/action-trace/index.ts b/src/common/contract-content/action-trace/index.ts deleted file mode 100644 index 986e248..0000000 --- a/src/common/contract-content/action-trace/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './action-trace'; -export * from './action-trace.dtos'; \ No newline at end of file diff --git a/src/common/contract-content/delta/__tests__/delta.unit.test.ts b/src/common/contract-content/delta/__tests__/delta.unit.test.ts deleted file mode 100644 index 41a198f..0000000 --- a/src/common/contract-content/delta/__tests__/delta.unit.test.ts +++ /dev/null @@ -1,14 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ -/* eslint-disable @typescript-eslint/no-explicit-any */ - -import { Delta } from '../delta'; - -describe('Delta Unit tests', () => { - it('"create" should create Delta entity based on given DTO', async () => { - const entity = Delta.create('foo', { - name: 'delta', - rows: [{ present: 1, data: Uint8Array.from([]) }], - }); - expect(entity).toBeInstanceOf(Delta); - }); -}); diff --git a/src/common/contract-content/delta/delta.dtos.ts b/src/common/contract-content/delta/delta.dtos.ts deleted file mode 100644 index f4a013f..0000000 --- a/src/common/contract-content/delta/delta.dtos.ts +++ /dev/null @@ -1,22 +0,0 @@ -export type DeltaRowDto = { - present?: number; - data?: Uint8Array; -}; - -export type DeltaJson = { - name?: string; - rows?: DeltaRowDto[]; -}; - -export type DeltaByNameDto = [string, DeltaJson]; - -export type DeltaRowModel = { - present?: number; - data?: Uint8Array; -}; - -export type DeltaModel = { - shipDeltaMessageName?: string; - name?: string; - rows?: DeltaRowModel[]; -}; diff --git a/src/common/contract-content/delta/delta.ts b/src/common/contract-content/delta/delta.ts deleted file mode 100644 index bdcb1f5..0000000 --- a/src/common/contract-content/delta/delta.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { DeltaJson, DeltaRowDto } from './delta.dtos'; - -/** - * Represents a row in a delta. - * @class - */ -export class DeltaRow { - /** - * Creates a DeltaRow instance based on the provided json. - * - * @param {DeltaRowDto} json - The json containing the delta row information. - * @returns {DeltaRow} The created DeltaRow instance. - */ - public static create(json: DeltaRowDto): DeltaRow { - const { present, data } = json; - return new DeltaRow(present, data); - } - - /** - * Creates an instance of DeltaRow. - * - * @param {number} present - The present state of the delta row. - * @param {Uint8Array} data - The data associated with the delta row. - */ - private constructor( - public readonly present: number, - public readonly data: Uint8Array - ) {} -} - -/** - * Represents a delta. - * @class - */ -export class Delta { - /** - * Creates a Delta instance based on the provided json. - * - * @param {string} shipMessageName - The name of the ship message. - * @param {DeltaJson} json - The json containing the delta information. - * @returns {Delta} The created Delta instance. - */ - public static create(shipMessageName: string, json: DeltaJson): Delta { - const { name, rows } = json; - - return new Delta( - shipMessageName, - name, - rows.map(dto => DeltaRow.create(dto)) - ); - } - - /** - * Creates an instance of Delta. - * - * @param {string} shipDeltaMessageName - The name of the ship delta message. - * @param {string} name - The name of the delta. - * @param {DeltaRow[]} rows - The rows of the delta. - */ - constructor( - public readonly shipDeltaMessageName: string, - public readonly name: string, - public readonly rows: DeltaRow[] - ) {} -} diff --git a/src/common/contract-content/delta/index.ts b/src/common/contract-content/delta/index.ts deleted file mode 100644 index 248ee3a..0000000 --- a/src/common/contract-content/delta/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './delta'; -export * from './delta.dtos'; \ No newline at end of file diff --git a/src/common/contract-content/index.ts b/src/common/contract-content/index.ts deleted file mode 100644 index 4c0314e..0000000 --- a/src/common/contract-content/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from './action-trace/action-trace'; -export * from './signed-block'; -export * from './delta/delta'; -export * from './trace/trace'; -export * from './transaction/transaction'; diff --git a/src/common/contract-content/signed-block/__tests__/signed-block.unit.test.ts b/src/common/contract-content/signed-block/__tests__/signed-block.unit.test.ts deleted file mode 100644 index a8ca982..0000000 --- a/src/common/contract-content/signed-block/__tests__/signed-block.unit.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ -/* eslint-disable @typescript-eslint/no-explicit-any */ - -import { SignedBlock } from '../signed-block'; -const dto = { - timestamp: '2022-06-30T12:28:32.900Z', - producer: 'some_producer', - confirmed: 0, - previous: 'previous_value', - transaction_mroot: 'mroot', - action_mroot: 'action', - schedule_version: 1, - new_producers: '', - header_extensions: [], - producer_signature: 'prod', - transactions: [ - { - status: 0, - cpu_usage_us: 1, - net_usage_words: 1, - trx: ['', ''], - }, - ] as any, -}; - -describe('Signed Block Unit tests', () => { - beforeAll(() => { - jest.useFakeTimers(); - jest.setSystemTime(new Date(2022, 4, 5)); - }); - - afterAll(() => { - jest.useRealTimers(); - }); - - it('"create" should create Block entity based on given DTO', async () => { - const entity = SignedBlock.create(dto); - expect(entity).toBeInstanceOf(SignedBlock); - }); - - it('"create" should use system current timestamp if DTO does not have one', async () => { - dto.timestamp = ''; - const entity = SignedBlock.create(dto); - expect(entity.timestamp.toISOString()).toEqual('2022-05-04T22:00:00.000Z'); - expect(entity).toBeInstanceOf(SignedBlock); - }); -}); diff --git a/src/common/contract-content/signed-block/index.ts b/src/common/contract-content/signed-block/index.ts deleted file mode 100644 index 0a24efc..0000000 --- a/src/common/contract-content/signed-block/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './signed-block'; -export * from './signed-block.dtos'; diff --git a/src/common/contract-content/signed-block/signed-block.ts b/src/common/contract-content/signed-block/signed-block.ts deleted file mode 100644 index d3511b1..0000000 --- a/src/common/contract-content/signed-block/signed-block.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* eslint-disable @typescript-eslint/no-empty-function */ - -import { parseDateToMs } from '@alien-worlds/api-core'; -import { Transaction } from '../transaction/transaction'; -import { SignedBlockJson } from './signed-block.dtos'; - -/** - * Represents a signed block. - * @class - */ -export class SignedBlock { - /** - * Creates a SignedBlock instance based on the provided json. - * - * @param {SignedBlockJson} json - The json containing the signed block information. - * @returns {SignedBlock} The created SignedBlock instance. - */ - public static create(json: SignedBlockJson): SignedBlock { - const { - producer, - confirmed, - previous, - transaction_mroot, - action_mroot, - schedule_version, - new_producers, - header_extensions, - producer_signature, - transactions, - } = json; - - const timestamp = json.timestamp - ? new Date(parseDateToMs(json.timestamp)) - : new Date(); - - return new SignedBlock( - timestamp, - producer, - confirmed, - previous, - transaction_mroot, - action_mroot, - schedule_version, - new_producers, - header_extensions, - producer_signature, - transactions.map(dto => Transaction.create(dto)) - ); - } - - /** - * Creates an instance of SignedBlock. - * - * @param {Date} timestamp - The timestamp of the signed block. - * @param {string} producer - The producer of the block. - * @param {number} confirmed - The number of confirmed blocks. - * @param {string} previous - The previous block ID. - * @param {string} transactionMroot - The Merkle root of the transactions. - * @param {string} actionMroot - The Merkle root of the actions. - * @param {number} scheduleVersion - The schedule version of the block. - * @param {unknown} newProducers - The new producers of the block. - * @param {unknown[]} headerExtensions - The header extensions of the block. - * @param {string} producerSignature - The producer signature of the block. - * @param {Transaction[]} transactions - The transactions included in the block. - */ - constructor( - public readonly timestamp: Date, - public readonly producer: string, - public readonly confirmed: number, - public readonly previous: string, - public readonly transactionMroot: string, - public readonly actionMroot: string, - public readonly scheduleVersion: number, - public readonly newProducers: unknown, - public readonly headerExtensions: unknown[], - public readonly producerSignature: string, - public readonly transactions: Transaction[] - ) {} -} diff --git a/src/common/contract-content/trace/__tests__/trace.unit.test.ts b/src/common/contract-content/trace/__tests__/trace.unit.test.ts deleted file mode 100644 index 0431de4..0000000 --- a/src/common/contract-content/trace/__tests__/trace.unit.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ -/* eslint-disable @typescript-eslint/no-explicit-any */ - -import { Trace } from '../trace'; - -const dto = { - id: 'foo', - status: 0, - cpu_usage_us: 100, - net_usage_words: 100, - elapsed: '', - net_usage: '100', - scheduled: false, - action_traces: [ - [ - 'action', - { - action_ordinal: 0, - creator_action_ordinal: 0, - receipt: [ - 'foo_receipt', - { - receiver: 'receiver', - act_digest: '', - global_sequence: '', - recv_sequence: '', - auth_sequence: [], - code_sequence: 0, - abi_sequence: 10, - }, - ], - receiver: 'receiver', - act: {}, - context_free: false, - elapsed: 'elapsed', - console: 'console', - account_ram_deltas: [], - except: '', - error_code: '200', - }, - ], - ] as any, - account_ram_delta: '', - except: '', - error_code: 0, - failed_dtrx_trace: '', - partial: [ - 'foo', - { - expiration: '10000', - ref_block_num: 0, - ref_block_prefix: 0, - max_net_usage_words: 0, - max_cpu_usage_ms: 0, - delay_sec: 1, - transaction_extensions: [], - signatures: [], - context_free_data: [], - }, - ] as any, -}; - -describe('Trace Unit tests', () => { - it('"create" should create Trace entity based on given DTO', async () => { - const entity = Trace.create('foo', dto); - expect(entity).toBeInstanceOf(Trace); - }); -}); diff --git a/src/common/contract-content/trace/index.ts b/src/common/contract-content/trace/index.ts deleted file mode 100644 index 17d6a1a..0000000 --- a/src/common/contract-content/trace/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './trace'; -export * from './trace.dtos'; \ No newline at end of file diff --git a/src/common/contract-content/trace/trace.ts b/src/common/contract-content/trace/trace.ts deleted file mode 100644 index 495ae5a..0000000 --- a/src/common/contract-content/trace/trace.ts +++ /dev/null @@ -1,162 +0,0 @@ -/* eslint-disable @typescript-eslint/no-empty-function */ - -import { ActionTrace } from '../action-trace/action-trace'; -import { PartialJson, TraceJson } from './trace.dtos'; - -/** - * Represents a partial transaction. - * @class - */ -export class Partial { - /** - * Creates a Partial instance. - * - * @param {string} type - The type of the partial transaction. - * @param {PartialJson} json - The json containing the partial transaction information. - * @returns {Partial} The created Partial instance. - */ - public static create(type: string, json: PartialJson): Partial { - const { - expiration, - ref_block_num, - ref_block_prefix, - max_net_usage_words, - max_cpu_usage_ms, - delay_sec, - transaction_extensions, - signatures, - context_free_data, - } = json; - return new Partial( - type, - expiration, - ref_block_num, - ref_block_prefix, - max_net_usage_words, - max_cpu_usage_ms, - delay_sec, - transaction_extensions, - signatures, - context_free_data - ); - } - - /** - * Creates an instance of Partial. - * - * @param {string} name - The name of the partial transaction. - * @param {string} expiration - The expiration time of the partial transaction. - * @param {number} refBlockNumber - The reference block number of the partial transaction. - * @param {number} refBlockPrefix - The reference block prefix of the partial transaction. - * @param {number} maxNetUsageWords - The maximum net usage words of the partial transaction. - * @param {number} maxCpuUsageMs - The maximum CPU usage in milliseconds of the partial transaction. - * @param {number} delayInSeconds - The delay in seconds of the partial transaction. - * @param {unknown[]} transactionExtensions - The transaction extensions of the partial transaction. - * @param {unknown[]} signatures - The signatures of the partial transaction. - * @param {unknown[]} contextFreeData - The context-free data of the partial transaction. - */ - constructor( - public readonly name: string, - public readonly expiration: string, - public readonly refBlockNumber: number, - public readonly refBlockPrefix: number, - public readonly maxNetUsageWords: number, - public readonly maxCpuUsageMs: number, - public readonly delayInSeconds: number, - public readonly transactionExtensions: unknown[], - public readonly signatures: unknown[], - public readonly contextFreeData: unknown[] - ) {} -} - -/** - * Represents a trace of a transaction. - * @class - */ -export class Trace { - /** - * Creates a Trace instance. - * - * @param {string} shipMessageName - The name of the ship message associated with the trace. - * @param {TraceJson} json - The json containing the trace information. - * @returns {Trace} The created Trace instance. - */ - public static create(shipMessageName: string, json: TraceJson): Trace { - const { - id, - status, - cpu_usage_us, - net_usage_words, - elapsed, - net_usage, - scheduled, - action_traces, - account_ram_delta, - except, - error_code, - failed_dtrx_trace, - } = json; - - const actionTraces = action_traces.map(item => { - const [actionTraceType, actionTraceDto] = item; - return ActionTrace.create(actionTraceType, actionTraceDto); - }); - let partial: Partial; - if (json.partial) { - const [partialType, partialContent] = json.partial; - partial = Partial.create(partialType, partialContent); - } - - return new Trace( - shipMessageName, - id, - status, - cpu_usage_us, - net_usage_words, - elapsed, - net_usage, - scheduled, - actionTraces, - account_ram_delta, - except, - Number(error_code), - failed_dtrx_trace, - partial - ); - } - - /** - * Creates an instance of Trace. - * - * @param {string} shipTraceMessageName - The name of the ship trace message. - * @param {string} id - The ID of the trace. - * @param {number} status - The status of the trace. - * @param {number} cpuUsageUs - The CPU usage in microseconds of the trace. - * @param {number} netUsageWords - The net usage words of the trace. - * @param {string} elapsed - The elapsed time of the trace. - * @param {string} netUsage - The net usage of the trace. - * @param {boolean} scheduled - Indicates if the trace is scheduled. - * @param {ActionTrace[]} actionTraces - The action traces of the trace. - * @param {unknown} accountRamDelta - The account RAM delta of the trace. - * @param {unknown} except - The exception information of the trace. - * @param {number} errorCode - The error code of the trace. - * @param {unknown} failedDtrxTrace - The failed deferred transaction trace. - * @param {Partial | null} partial - The partial transaction associated with the trace. - */ - constructor( - public readonly shipTraceMessageName: string, - public readonly id: string, - public readonly status: number, - public readonly cpuUsageUs: number, - public readonly netUsageWords: number, - public readonly elapsed: string, - public readonly netUsage: string, - public readonly scheduled: boolean, - public readonly actionTraces: ActionTrace[], - public readonly accountRamDelta: unknown, - public readonly except: unknown, - public readonly errorCode: number, - public readonly failedDtrxTrace: unknown, - public readonly partial: Partial | null - ) {} -} diff --git a/src/common/contract-content/transaction/__tests__/transaction.unit.test.ts b/src/common/contract-content/transaction/__tests__/transaction.unit.test.ts deleted file mode 100644 index c65bad9..0000000 --- a/src/common/contract-content/transaction/__tests__/transaction.unit.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ -/* eslint-disable @typescript-eslint/no-explicit-any */ - -import { Transaction } from '../transaction'; - -const dto = { - status: 0, - cpu_usage_us: 1, - net_usage_words: 2, - trx: ['', ''] as any, -}; - -describe('Transaction Unit tests', () => { - it('"create" should create Transaction entity based on given DTO', async () => { - const entity = Transaction.create(dto); - expect(entity).toBeInstanceOf(Transaction); - }); - - it('"create" should log warning on unknown trx type', async () => { - dto.trx = ['foo', '']; - const entity = Transaction.create(dto); - expect(entity).toBeInstanceOf(Transaction); - }); - - it('"create" should create Trx entity when trx type is "transaction_id"', async () => { - dto.trx = ['transaction_id', 'content']; - const entity = Transaction.create(dto); - expect(entity).toBeInstanceOf(Transaction); - }); - - it('"create" should create PackedTrx entity when trx type is "transaction_id"', async () => { - dto.trx = [ - 'packed_transaction', - { - signatures: [], - compression: 0, - packed_context_free_data: '', - packed_trx: Uint8Array.from([]), - }, - ]; - const entity = Transaction.create(dto); - expect(entity).toBeInstanceOf(Transaction); - }); -}); diff --git a/src/common/contract-content/transaction/index.ts b/src/common/contract-content/transaction/index.ts deleted file mode 100644 index 709ffa3..0000000 --- a/src/common/contract-content/transaction/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './transaction'; -export * from './transaction.dtos'; diff --git a/src/common/contract-content/transaction/transaction.ts b/src/common/contract-content/transaction/transaction.ts deleted file mode 100644 index 225e386..0000000 --- a/src/common/contract-content/transaction/transaction.ts +++ /dev/null @@ -1,101 +0,0 @@ -/* eslint-disable @typescript-eslint/no-empty-function */ - -import { PackedTrxJson, TransactionJson } from './transaction.dtos'; - -/** - * Represents a packed transaction. - * @class - */ -export class PackedTrx { - /** - * Creates a PackedTrx instance. - * - * @param {string} type - The type of the packed transaction. - * @param {PackedTrxJson} json - The DTO containing the packed transaction information. - * @returns {PackedTrx} The created PackedTrx instance. - */ - public static create(type: string, json: PackedTrxJson): PackedTrx { - const { signatures, compression, packed_context_free_data, packed_trx } = json; - return new PackedTrx( - type, - signatures, - compression, - packed_context_free_data, - packed_trx - ); - } - - /** - * Creates an instance of PackedTrx. - * - * @param {string} type - The type of the packed transaction. - * @param {string[]} signatures - The signatures of the packed transaction. - * @param {number} compression - The compression type of the packed transaction. - * @param {unknown} packedContextFreeData - The packed context-free data of the packed transaction. - * @param {unknown} content - The content of the packed transaction. - */ - constructor( - public readonly type: string, - public readonly signatures: string[], - public readonly compression: number, - public readonly packedContextFreeData: unknown, - public readonly content: unknown //TODO: we should deserialize "packed_trx" - ) {} -} - -export class Trx { - public static create(type: string, dto: string): Trx { - return new Trx(type, dto); - } - - private constructor(public readonly type: string, public readonly content: string) {} -} - -/** - * Represents a transaction. - * @class - */ -export class Transaction { - /** - * Creates a Transaction instance. - * - * @param {TransactionJson} json - The DTO containing the transaction information. - * @returns {Transaction} The created Transaction instance. - */ - public static create(json: TransactionJson): Transaction { - const { status, cpu_usage_us, net_usage_words } = json; - - const [type, content] = json.trx; - let trx; - - switch (type) { - case 'transaction_id': { - trx = Trx.create(type, content); - break; - } - case 'packed_transaction': { - trx = PackedTrx.create(type, content); - break; - } - default: { - console.warn(`Unknown trx type "${type}"`); - } - } - return new Transaction(status, cpu_usage_us, net_usage_words, trx); - } - - /** - * Creates an instance of Transaction. - * - * @param {number} status - The status of the transaction. - * @param {number} cpuUsageUs - The CPU usage in microseconds of the transaction. - * @param {number} netUsageWords - The net usage words of the transaction. - * @param {Trx | PackedTrx | unknown} trx - The transaction object. - */ - constructor( - public readonly status: number, - public readonly cpuUsageUs: number, - public readonly netUsageWords: number, - public readonly trx: Trx | PackedTrx | unknown - ) {} -} diff --git a/src/common/featured/__tests__/featured-contract.mongo.mapper.unit.test.ts b/src/common/featured/__dependencies__/__tests__/featured-contract.mongo.mapper.unit.test.ts similarity index 91% rename from src/common/featured/__tests__/featured-contract.mongo.mapper.unit.test.ts rename to src/common/featured/__dependencies__/__tests__/featured-contract.mongo.mapper.unit.test.ts index c7dec6b..3f7c2ff 100644 --- a/src/common/featured/__tests__/featured-contract.mongo.mapper.unit.test.ts +++ b/src/common/featured/__dependencies__/__tests__/featured-contract.mongo.mapper.unit.test.ts @@ -1,7 +1,7 @@ import { FeaturedContractMongoMapper } from '../featured-contract.mongo.mapper'; import { MongoDB } from '@alien-worlds/storage-mongodb'; -import { FeaturedContract } from '../featured-contract'; -import { FeaturedContractMongoModel } from '../featured.types'; +import { FeaturedContract } from '../../featured-contract'; +import { FeaturedContractMongoModel } from '../featured.mongo.types'; describe('FeaturedContractMongoMapper', () => { const fromBigIntMock = jest.fn(); diff --git a/src/common/featured/__tests__/featured.mapper.unit.test.ts b/src/common/featured/__dependencies__/__tests__/featured.mapper.unit.test.ts similarity index 91% rename from src/common/featured/__tests__/featured.mapper.unit.test.ts rename to src/common/featured/__dependencies__/__tests__/featured.mapper.unit.test.ts index 71b023a..4915199 100644 --- a/src/common/featured/__tests__/featured.mapper.unit.test.ts +++ b/src/common/featured/__dependencies__/__tests__/featured.mapper.unit.test.ts @@ -1,10 +1,10 @@ -import { FeaturedMapper } from '../featured.mapper'; -import { MatcherNotFoundError, PatternMatchError } from '../featured.errors'; +import { MatcherNotFoundError, PatternMatchError } from '../../featured.errors'; +import { FeaturedMapper } from '../../featured.mapper'; import { MatchCriteria, ProcessorMatchCriteria, ProcessorMatcher, -} from '../featured.types'; +} from '../../featured.types'; describe('ContractProcessorMapper', () => { let contractProcessorMapper: FeaturedMapper; @@ -21,6 +21,11 @@ describe('ContractProcessorMapper', () => { processor: 'mockProcessor2', }, ]; + const pattern: ProcessorMatchCriteria = { + contract: '', + action: '', + processor: '', + }; const mockMatcher = async (criteria: MatchCriteria) => { return criteria.contract.includes('mockContract1'); @@ -30,7 +35,7 @@ describe('ContractProcessorMapper', () => { mockMatchers.set('mockMatcher', mockMatcher); beforeEach(() => { - contractProcessorMapper = new FeaturedMapper(mockCriteria, mockMatchers); + contractProcessorMapper = new FeaturedMapper(mockCriteria, pattern, mockMatchers); }); describe('constructor', () => { @@ -43,9 +48,15 @@ describe('ContractProcessorMapper', () => { matcher: 'invalidMatcher', }, ]; - expect(() => new FeaturedMapper(criteriaWithInvalidMatcher, mockMatchers)).toThrow( - MatcherNotFoundError - ); + const pattern: ProcessorMatchCriteria = { + contract: '', + action: '', + processor: '', + matcher: '', + }; + expect( + () => new FeaturedMapper(criteriaWithInvalidMatcher, pattern, mockMatchers) + ).toThrow(MatcherNotFoundError); }); it('should add all contracts to the contracts set', () => { diff --git a/src/common/featured/featured-contract.collection.ts b/src/common/featured/__dependencies__/featured-contract.mongo.collection.ts similarity index 81% rename from src/common/featured/featured-contract.collection.ts rename to src/common/featured/__dependencies__/featured-contract.mongo.collection.ts index fd8b176..c6c9f4d 100644 --- a/src/common/featured/featured-contract.collection.ts +++ b/src/common/featured/__dependencies__/featured-contract.mongo.collection.ts @@ -1,5 +1,5 @@ import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; -import { FeaturedContractMongoModel } from './featured.types'; +import { FeaturedContractMongoModel } from './featured.mongo.types'; /** * Represents a source for FeaturedContract documents stored in a MongoDB collection. @@ -8,7 +8,7 @@ import { FeaturedContractMongoModel } from './featured.types'; * @class * @extends {MongoCollectionSource} */ -export class FeaturedContractCollection extends MongoCollectionSource { +export class FeaturedContractMongoCollection extends MongoCollectionSource { /** * Creates a new instance of the FeaturedContractSource class. * diff --git a/src/common/featured/featured-contract.mongo.mapper.ts b/src/common/featured/__dependencies__/featured-contract.mongo.mapper.ts similarity index 92% rename from src/common/featured/featured-contract.mongo.mapper.ts rename to src/common/featured/__dependencies__/featured-contract.mongo.mapper.ts index 4407f38..5468637 100644 --- a/src/common/featured/featured-contract.mongo.mapper.ts +++ b/src/common/featured/__dependencies__/featured-contract.mongo.mapper.ts @@ -1,7 +1,7 @@ import { parseToBigInt } from '@alien-worlds/api-core'; import { MongoDB, MongoMapper } from '@alien-worlds/storage-mongodb'; -import { FeaturedContract } from './featured-contract'; -import { FeaturedContractMongoModel } from './featured.types'; +import { FeaturedContractMongoModel } from './featured.mongo.types'; +import { FeaturedContract } from '../featured-contract'; /** * Class representing a FeaturedContractMongoMapper diff --git a/src/common/featured/featured.creator.ts b/src/common/featured/__dependencies__/featured.creator.ts similarity index 66% rename from src/common/featured/featured.creator.ts rename to src/common/featured/__dependencies__/featured.creator.ts index 59fe418..4b448f4 100644 --- a/src/common/featured/featured.creator.ts +++ b/src/common/featured/__dependencies__/featured.creator.ts @@ -3,15 +3,21 @@ import { MongoQueryBuilders, MongoSource, } from '@alien-worlds/storage-mongodb'; -import { Featured } from './featured'; -import { RepositoryImpl, SmartContractService, log } from '@alien-worlds/api-core'; -import { FeaturedContractCollection } from './featured-contract.collection'; +import { Featured } from '../featured'; +import { + RepositoryImpl, + SmartContractService, + UnknownObject, + log, +} from '@alien-worlds/api-core'; +import { FeaturedContractMongoCollection } from './featured-contract.mongo.collection'; import { FeaturedContractMongoMapper } from './featured-contract.mongo.mapper'; export class FeaturedCreator { public static async create( mongo: MongoSource | MongoConfig, - smartContractService: SmartContractService + smartContractService: SmartContractService, + featuredJson: UnknownObject ): Promise { let mongoSource: MongoSource; @@ -23,11 +29,11 @@ export class FeaturedCreator { mongoSource = await MongoSource.create(mongo); } const repository = new RepositoryImpl( - new FeaturedContractCollection(mongoSource), + new FeaturedContractMongoCollection(mongoSource), new FeaturedContractMongoMapper(), new MongoQueryBuilders() ); - const featured = new Featured(repository, smartContractService); + const featured = new Featured(repository, smartContractService, featuredJson); log(` * Contract Reader ... [ready]`); diff --git a/src/common/featured/__dependencies__/featured.mongo.types.ts b/src/common/featured/__dependencies__/featured.mongo.types.ts new file mode 100644 index 0000000..789cf75 --- /dev/null +++ b/src/common/featured/__dependencies__/featured.mongo.types.ts @@ -0,0 +1,7 @@ +import { MongoDB } from '@alien-worlds/storage-mongodb'; + +export type FeaturedContractMongoModel = { + _id?: MongoDB.ObjectId; + account?: string; + initial_block_number?: MongoDB.Long; +}; diff --git a/src/common/featured/__dependencies__/index.ts b/src/common/featured/__dependencies__/index.ts new file mode 100644 index 0000000..6e99e65 --- /dev/null +++ b/src/common/featured/__dependencies__/index.ts @@ -0,0 +1,4 @@ +export * from './featured-contract.mongo.collection'; +export * from './featured-contract.mongo.mapper'; +export * from './featured.creator'; +export * from './featured.mongo.types'; diff --git a/src/common/featured/featured.actions.ts b/src/common/featured/featured.actions.ts deleted file mode 100644 index 8ec2b1b..0000000 --- a/src/common/featured/featured.actions.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { FeaturedMapper } from './featured.mapper'; -import { ContractActionMatchCriteria, ProcessorMatchCriteria } from './featured.types'; - -export class FeaturedActions { - private mapper: FeaturedMapper; - - constructor(criteria: ProcessorMatchCriteria[]) { - this.mapper = new FeaturedMapper(criteria); - } - - public getProcessor(criteria: ContractActionMatchCriteria) { - return this.mapper.getProcessor(contract, { - shipTraceMessageName: [], - shipActionTraceMessageName: [], - contract: [], - action: [], - }); - } - - public listContracts(): string[] { - return this.listContracts(); - } -} diff --git a/src/common/featured/featured.deltas.ts b/src/common/featured/featured.deltas.ts deleted file mode 100644 index 42a7d71..0000000 --- a/src/common/featured/featured.deltas.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { FeaturedMapper } from './featured.mapper'; -import { ContractDeltaMatchCriteria, ProcessorMatchCriteria } from './featured.types'; - -export class FeaturedDeltas { - private mapper: FeaturedMapper; - - constructor(criteria: ProcessorMatchCriteria[]) { - this.mapper = new FeaturedMapper(criteria); - } - - public getProcessor(contract: string) { - return this.mapper.getProcessor(contract, { - shipDeltaMessageName: [], - contract: [], - name: [], - code: [], - scope: [], - table: [], - }); - } - - public listContracts(): string[] { - return this.listContracts(); - } -} diff --git a/src/common/featured/featured.errors.ts b/src/common/featured/featured.errors.ts index 640374e..03deae8 100644 --- a/src/common/featured/featured.errors.ts +++ b/src/common/featured/featured.errors.ts @@ -10,6 +10,20 @@ export class MatcherNotFoundError extends Error { } } +export class UndefinedPatternError extends Error { + constructor() { + super(`No pattern assigned to the criteria`); + } +} + +export class PatternMismatchError extends Error { + constructor() { + super( + `The length of the keys on the label does not match the number of keys in the pattern` + ); + } +} + export class UnknownContentTypeError extends Error { constructor(type: string) { super(`Unknown type: ${type}`); diff --git a/src/common/featured/featured.mapper.ts b/src/common/featured/featured.mapper.ts index f231726..e60926f 100644 --- a/src/common/featured/featured.mapper.ts +++ b/src/common/featured/featured.mapper.ts @@ -1,4 +1,9 @@ -import { MatcherNotFoundError, PatternMatchError } from './featured.errors'; +import { + MatcherNotFoundError, + PatternMatchError, + PatternMismatchError, + UndefinedPatternError, +} from './featured.errors'; import { MatchCriteria, ProcessorMatchCriteria, @@ -25,15 +30,17 @@ export class FeaturedMapper { /** * Creates a new instance of the contract processor mapper. - * @param criteria - An array of match criteria for the processor. - * @param matchers - Optional map of matchers. + * @param {ProcessorMatchCriteria[]} criteria - An array of match criteria for the processor. + * @param {MatchCriteriaType} pattern - The criteria pattern. + * @param {ProcessorMatcher} matchers - Optional map of matchers. */ constructor( criteria: ProcessorMatchCriteria[], + protected pattern?: MatchCriteriaType, matchers?: ProcessorMatcher ) { - criteria.forEach(criteria => { - const { processor, matcher, ...rest } = criteria; + criteria.forEach(current => { + const { processor, matcher, ...rest } = current; const { contract } = rest as unknown as MatchCriteria; if (Array.isArray(contract)) { @@ -51,8 +58,8 @@ export class FeaturedMapper { } else { this.validateCriteria(rest as MatchCriteriaType); - if (this.matchCriteria.indexOf(criteria) === -1) { - this.matchCriteria.push(criteria); + if (this.matchCriteria.indexOf(current) === -1) { + this.matchCriteria.push(current); } } }); @@ -213,17 +220,28 @@ export class FeaturedMapper { /** * Gets the processor for the given label and criteria. * @param label - The label to find a processor for. - * @param criteria - The criteria to match. + * @param pattern - The match criteria pattern. * @returns The processor if found, empty string otherwise. */ - public async getProcessor(label: string, criteria: MatchCriteriaType): Promise { + public async getProcessor(label: string, pattern?: MatchCriteriaType): Promise { const { matchCriteria } = this; - const keys = Object.keys(criteria); + const p = pattern || this.pattern; + + if (!p) { + throw new UndefinedPatternError(); + } + + const keys = Object.keys(p); const parts = label.split(':').map(part => part.split(',')); + + if (parts.length !== keys.length) { + throw new PatternMismatchError(); + } + const candidate = parts.reduce((result, part, i) => { result[keys[i]] = part; return result; - }, criteria); + }, p); for (const criteriaRef of matchCriteria) { if (this.isMatch(criteriaRef, candidate)) { diff --git a/src/common/featured/featured.types.ts b/src/common/featured/featured.types.ts index cbd6602..b6eb15e 100644 --- a/src/common/featured/featured.types.ts +++ b/src/common/featured/featured.types.ts @@ -1,11 +1,3 @@ -import { MongoDB } from "@alien-worlds/storage-mongodb"; - -export type FeaturedContractMongoModel = { - _id?: MongoDB.ObjectId; - account?: string; - initial_block_number?: MongoDB.Long; -}; - export type FeaturedContractModel = { account: string; initialBlockNumber: bigint; @@ -37,7 +29,7 @@ export type MatchFunction = ( criteria: MatchCriteriaType ) => Promise; -export type ContractActionMatchCriteria = MatchCriteria & { +export type ContractTraceMatchCriteria = MatchCriteria & { shipTraceMessageName: string[]; shipActionTraceMessageName: string[]; action: string[]; diff --git a/src/common/featured/index.ts b/src/common/featured/index.ts index 94de583..6db5938 100644 --- a/src/common/featured/index.ts +++ b/src/common/featured/index.ts @@ -1,12 +1,8 @@ +export * from './__dependencies__'; export * from './featured-contract'; -export * from './featured-contract.collection'; -export * from './featured-contract.mongo.mapper'; export * from './featured'; export * from './featured.config'; export * from './featured.errors'; export * from './featured.mapper'; export * from './featured.types'; export * from './featured.utils'; - -export * from './featured.actions'; -export * from './featured.deltas'; diff --git a/src/common/index.ts b/src/common/index.ts index 96244bd..0a5a655 100644 --- a/src/common/index.ts +++ b/src/common/index.ts @@ -1,8 +1,9 @@ export * from './abis'; export * from './block-range-scanner'; export * from './block-state'; -export * from './contract-content'; export * from './featured'; +export * from './processor-task-queue'; +export * from './types'; export * from './unprocessed-block-queue'; export * from './common.enums'; export * from './common.errors'; diff --git a/src/common/processor-task-queue/__dependencies__/index.ts b/src/common/processor-task-queue/__dependencies__/index.ts new file mode 100644 index 0000000..e1fe167 --- /dev/null +++ b/src/common/processor-task-queue/__dependencies__/index.ts @@ -0,0 +1,4 @@ +export * from './processor-task.mongo.collection'; +export * from './processor-task.mongo.mapper'; +export * from './processor-task.mongo.types'; +export * from './unsuccessful-processor-task.mongo.collection'; diff --git a/src/common/processor-task-queue/data-sources/processor-task.source.ts b/src/common/processor-task-queue/__dependencies__/processor-task.mongo.collection.ts similarity index 70% rename from src/common/processor-task-queue/data-sources/processor-task.source.ts rename to src/common/processor-task-queue/__dependencies__/processor-task.mongo.collection.ts index 7d0f124..c9a3671 100644 --- a/src/common/processor-task-queue/data-sources/processor-task.source.ts +++ b/src/common/processor-task-queue/__dependencies__/processor-task.mongo.collection.ts @@ -1,9 +1,17 @@ -import { MongoCollectionSource, MongoDB, MongoSource } from '@alien-worlds/storage-mongodb'; +import { + MongoCollectionSource, + MongoDB, + MongoSource, +} from '@alien-worlds/storage-mongodb'; import { ProcessorTaskQueueConfig } from '../processor-task-queue.config'; -import { ProcessorTaskDocument } from '../processor-task.types'; import { DataSourceError } from '@alien-worlds/api-core'; +import { ProcessorTaskSource } from '../processor-task.source'; +import { ProcessorTaskMongoModel } from './processor-task.mongo.types'; -export class ProcessorTaskSource extends MongoCollectionSource { +export class ProcessorTaskMongoCollection + extends MongoCollectionSource + implements ProcessorTaskSource +{ private transactionOptions: MongoDB.TransactionOptions; constructor(mongoSource: MongoSource, private config: ProcessorTaskQueueConfig) { @@ -34,7 +42,7 @@ export class ProcessorTaskSource extends MongoCollectionSource { + public async nextTask(mode?: string): Promise { try { const filter = mode ? { mode } : {}; const result = await this.collection.findOneAndDelete(filter); diff --git a/src/common/processor-task-queue/__dependencies__/processor-task.mongo.mapper.ts b/src/common/processor-task-queue/__dependencies__/processor-task.mongo.mapper.ts new file mode 100644 index 0000000..6dda024 --- /dev/null +++ b/src/common/processor-task-queue/__dependencies__/processor-task.mongo.mapper.ts @@ -0,0 +1,92 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { MongoDB, MongoMapper } from '@alien-worlds/storage-mongodb'; + +import { + PropertyMapping, + parseToBigInt, + removeUndefinedProperties, +} from '@alien-worlds/api-core'; +import { ProcessorTask } from '../processor-task'; +import { ProcessorTaskMongoModel } from './processor-task.mongo.types'; + +export class ProcessorTaskMongoMapper< + EntityType = ProcessorTask, + ModelType = ProcessorTaskMongoModel +> extends MongoMapper { + public getEntityKeyMapping(key: string): PropertyMapping { + throw new Error('Method not implemented.'); + } + + public fromEntity(entity: ProcessorTask): ProcessorTaskMongoModel { + const { + id, + abi, + shortId, + label, + timestamp, + type, + mode, + content, + hash, + blockNumber, + isFork, + blockTimestamp, + error, + } = entity; + + const document: ProcessorTaskMongoModel = { + abi, + short_id: shortId, + label, + timestamp, + type, + mode, + content: new MongoDB.Binary(content), + hash, + block_number: MongoDB.Long.fromBigInt(blockNumber), + block_timestamp: blockTimestamp, + is_fork: isFork, + error, + }; + + if (id) { + document._id = new MongoDB.ObjectId(id); + } + + return removeUndefinedProperties(document); + } + + public toEntity(model: ProcessorTaskMongoModel): ProcessorTask { + const { + abi, + short_id, + label, + content, + timestamp, + hash, + type, + mode, + _id, + block_number, + block_timestamp, + error, + is_fork, + } = model; + + return new ProcessorTask( + _id ? _id.toString() : '', + abi, + short_id, + label, + timestamp, + type, + mode, + content.buffer, + hash, + parseToBigInt(block_number), + block_timestamp, + is_fork, + error + ); + } +} diff --git a/src/common/processor-task-queue/__dependencies__/processor-task.mongo.types.ts b/src/common/processor-task-queue/__dependencies__/processor-task.mongo.types.ts new file mode 100644 index 0000000..0d5ecd3 --- /dev/null +++ b/src/common/processor-task-queue/__dependencies__/processor-task.mongo.types.ts @@ -0,0 +1,18 @@ +import { MongoDB } from '@alien-worlds/storage-mongodb'; +import { ProcessorTaskError } from '../processor-task.types'; + +export type ProcessorTaskMongoModel = { + _id?: MongoDB.ObjectId; + abi?: string; + is_fork?: boolean; + short_id?: string; + label?: string; + timestamp?: Date; + type?: string; + mode?: string; + content?: MongoDB.Binary; + hash?: string; + block_number?: MongoDB.Long; + block_timestamp?: Date; + error?: ProcessorTaskError; +}; diff --git a/src/common/processor-task-queue/data-sources/unsuccessful-processor-task.source.ts b/src/common/processor-task-queue/__dependencies__/unsuccessful-processor-task.mongo.collection.ts similarity index 88% rename from src/common/processor-task-queue/data-sources/unsuccessful-processor-task.source.ts rename to src/common/processor-task-queue/__dependencies__/unsuccessful-processor-task.mongo.collection.ts index f45f7b8..f36fe62 100644 --- a/src/common/processor-task-queue/data-sources/unsuccessful-processor-task.source.ts +++ b/src/common/processor-task-queue/__dependencies__/unsuccessful-processor-task.mongo.collection.ts @@ -1,7 +1,7 @@ import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; -import { ProcessorTaskDocument } from '../processor-task.types'; +import { ProcessorTaskMongoModel } from './processor-task.mongo.types'; -export class UnsuccessfulProcessorTaskSource extends MongoCollectionSource { +export class UnsuccessfulProcessorTaskSource extends MongoCollectionSource { constructor(mongoSource: MongoSource) { super(mongoSource, 'history_tools.unsuccessful_processor_tasks', { indexes: [ diff --git a/src/common/processor-task-queue/index.ts b/src/common/processor-task-queue/index.ts index 964e5f6..65caaec 100644 --- a/src/common/processor-task-queue/index.ts +++ b/src/common/processor-task-queue/index.ts @@ -1,5 +1,8 @@ +export * from './__dependencies__'; export * from './processor-task-queue'; -export * from './data-sources/processor-task.source'; -export * from './data-sources/unsuccessful-processor-task.source'; +export * from './processor-task-queue.config'; +export * from './processor-task.enums'; +export * from './processor-task.errors'; +export * from './processor-task.source'; export * from './processor-task'; export * from './processor-task.types'; diff --git a/src/common/processor-task-queue/processor-task-queue.ts b/src/common/processor-task-queue/processor-task-queue.ts index 76206e5..f3922ab 100644 --- a/src/common/processor-task-queue/processor-task-queue.ts +++ b/src/common/processor-task-queue/processor-task-queue.ts @@ -1,52 +1,16 @@ -import { - DataSourceError, - log, -} from '@alien-worlds/api-core'; -import { ProcessorTaskSource } from './data-sources/processor-task.source'; -import { ProcessorTask } from './processor-task'; -import { UnsuccessfulProcessorTaskSource } from './data-sources/unsuccessful-processor-task.source'; -import { ProcessorTaskQueueConfig } from './processor-task-queue.config'; -import { MongoConfig, MongoSource } from '@alien-worlds/storage-mongodb'; +import { DataSource, DataSourceError, Mapper, log } from '@alien-worlds/api-core'; import { ErrorJson } from '@alien-worlds/workers'; +import { ProcessorTaskSource } from './processor-task.source'; +import { ProcessorTask } from './processor-task'; +import { ProcessorTaskModel } from './processor-task.types'; export class ProcessorTaskQueue { - public static async create( - mongo: MongoSource | MongoConfig, - onlyAdd: boolean, - queueConfig?: ProcessorTaskQueueConfig - ) { - log(` * Processor Queue ... [starting]`); - - let state: ProcessorTaskQueue; - - if (mongo instanceof MongoSource) { - if (!mongo.client) { - throw new Error( - 'ProcessorTaskQueue requires MongoSource to provide a mongo client. Create Mongo Source using "MongoSource.create()"' - ); - } - - state = new ProcessorTaskQueue(mongo, queueConfig, onlyAdd); - } else { - const mongoSource = await MongoSource.create(mongo); - state = new ProcessorTaskQueue(mongoSource, queueConfig, onlyAdd); - } - - log(` * Processor Queue ... [ready]`); - return state; - } - - private source: ProcessorTaskSource; - private unsuccessfulSource: UnsuccessfulProcessorTaskSource; - private constructor( - mongo: MongoSource, - config: ProcessorTaskQueueConfig, - private onlyAdd = false - ) { - this.source = new ProcessorTaskSource(mongo, config); - this.unsuccessfulSource = new UnsuccessfulProcessorTaskSource(mongo); - } + protected source: ProcessorTaskSource, + protected mapper: Mapper, + protected unsuccessfulSource: DataSource, + protected onlyAdd = false + ) {} public async nextTask(mode?: string): Promise { // TODO: temporary solution - testing session options @@ -58,7 +22,7 @@ export class ProcessorTaskQueue { try { const dto = await this.source.nextTask(mode); if (dto) { - return ProcessorTask.fromDocument(dto); + return this.mapper.toEntity(dto); } return null; } catch (error) { @@ -70,7 +34,7 @@ export class ProcessorTaskQueue { public async addTasks(tasks: ProcessorTask[], unsuccessful?: boolean): Promise { const source = unsuccessful ? this.unsuccessfulSource : this.source; try { - const dtos = tasks.map(task => task.toDocument()); + const dtos = tasks.map(task => this.mapper.fromEntity(task)); await source.insert(dtos); } catch (error) { const { error: concernError } = error; @@ -85,7 +49,9 @@ export class ProcessorTaskQueue { ): Promise { try { const { message, stack } = error; - const document = task.toDocument(); + const document: ProcessorTaskModel = this.mapper.fromEntity( + task + ) as ProcessorTaskModel; document.error = { message, stack }; await this.unsuccessfulSource.insert([document]); diff --git a/src/common/processor-task-queue/processor-task.enums.ts b/src/common/processor-task-queue/processor-task.enums.ts new file mode 100644 index 0000000..13de22b --- /dev/null +++ b/src/common/processor-task-queue/processor-task.enums.ts @@ -0,0 +1,4 @@ +export enum ProcessorTaskType { + Trace = 'trace', + Delta = 'delta', +} diff --git a/src/common/processor-task-queue/processor-task.errors.ts b/src/common/processor-task-queue/processor-task.errors.ts new file mode 100644 index 0000000..c115c9f --- /dev/null +++ b/src/common/processor-task-queue/processor-task.errors.ts @@ -0,0 +1,5 @@ +export class UnknownProcessorTypeError extends Error { + constructor(type: string) { + super(`Unknown processor type: ${type}`); + } +} diff --git a/src/common/processor-task-queue/processor-task.source.ts b/src/common/processor-task-queue/processor-task.source.ts new file mode 100644 index 0000000..f4f416d --- /dev/null +++ b/src/common/processor-task-queue/processor-task.source.ts @@ -0,0 +1,5 @@ +import { DataSource } from '@alien-worlds/api-core'; + +export abstract class ProcessorTaskSource extends DataSource { + public abstract nextTask(mode?: string): Promise; +} diff --git a/src/common/processor-task-queue/processor-task.ts b/src/common/processor-task-queue/processor-task.ts index af1b813..87c0f5d 100644 --- a/src/common/processor-task-queue/processor-task.ts +++ b/src/common/processor-task-queue/processor-task.ts @@ -1,18 +1,8 @@ import crypto from 'crypto'; import { serialize } from 'v8'; -import { parseToBigInt, removeUndefinedProperties } from '@alien-worlds/api-core'; -import { - DeltaProcessorContentModel, - ProcessorTaskDocument, - ProcessorTaskError, -} from './processor-task.types'; -import { MongoDB } from '@alien-worlds/storage-mongodb'; -import { ActionTraceDto } from '../contract-content/action-trace'; - -export enum ProcessorTaskType { - Action = 'action', - Delta = 'delta', -} +import { DeltaProcessorContentModel, ProcessorTaskError } from './processor-task.types'; +import { ActionTrace } from '../types'; +import { ProcessorTaskType } from './processor-task.enums'; export class ProcessorTask { public static createActionProcessorTask( @@ -21,7 +11,7 @@ export class ProcessorTask { shipTraceMessageName: string, shipMessageName: string, transactionId: string, - actionTrace: ActionTraceDto, + actionTrace: ActionTrace, blockNumber: bigint, blockTimestamp: Date, isFork: boolean @@ -32,10 +22,10 @@ export class ProcessorTask { } = actionTrace; const buffer = serialize({ - transactionId, - actionTrace, - blockNumber, - blockTimestamp, + transaction_id: transactionId, + action_trace: actionTrace, + block_num: blockNumber.toString(), + block_timestamp: blockTimestamp, }); const hashBuffer = serialize({ account, @@ -57,7 +47,7 @@ export class ProcessorTask { shortId, label, null, - ProcessorTaskType.Action, + ProcessorTaskType.Trace, mode, buffer, hash, @@ -70,27 +60,27 @@ export class ProcessorTask { public static createDeltaProcessorTask( abi: string, mode: string, - shipDeltaMessageName: string, + type: string, name: string, code: string, scope: string, table: string, blockNumber: bigint, blockTimestamp: Date, - row: DeltaRow, + data: Uint8Array, isFork: boolean ) { const content: DeltaProcessorContentModel = { - shipDeltaMessageName, + ship_delta_message_name: type, name, - row, - blockNumber, - blockTimestamp, + row_data: data, + block_num: blockNumber, + block_timestamp: blockTimestamp, }; const buffer = serialize(content); const hash = crypto.createHash('sha1').update(buffer).digest('hex'); const shortId = `${code}:${scope}:${table}`; - const label = `${shipDeltaMessageName}:${name}:${shortId}`; + const label = `${type}:${name}:${shortId}`; return new ProcessorTask( null, @@ -108,41 +98,7 @@ export class ProcessorTask { ); } - public static fromDocument(document: ProcessorTaskDocument) { - const { - abi, - short_id, - label, - content, - timestamp, - hash, - type, - mode, - _id, - block_number, - block_timestamp, - error, - is_fork, - } = document; - - return new ProcessorTask( - _id ? _id.toString() : '', - abi, - short_id, - label, - timestamp, - type, - mode, - content.buffer, - hash, - parseToBigInt(block_number), - block_timestamp, - is_fork, - error - ); - } - - private constructor( + constructor( public readonly id: string, public readonly abi: string, public readonly shortId: string, @@ -157,43 +113,4 @@ export class ProcessorTask { public readonly isFork: boolean, public readonly error?: ProcessorTaskError ) {} - - public toDocument(): ProcessorTaskDocument { - const { - id, - abi, - shortId, - label, - timestamp, - type, - mode, - content, - hash, - blockNumber, - isFork, - blockTimestamp, - error, - } = this; - - const document: ProcessorTaskDocument = { - abi, - short_id: shortId, - label, - timestamp, - type, - mode, - content: new MongoDB.Binary(content), - hash, - block_number: MongoDB.Long.fromBigInt(blockNumber), - block_timestamp: blockTimestamp, - is_fork: isFork, - error, - }; - - if (id) { - document._id = new MongoDB.ObjectId(id); - } - - return removeUndefinedProperties(document); - } } diff --git a/src/common/processor-task-queue/processor-task.types.ts b/src/common/processor-task-queue/processor-task.types.ts index c277d6d..7588721 100644 --- a/src/common/processor-task-queue/processor-task.types.ts +++ b/src/common/processor-task-queue/processor-task.types.ts @@ -1,28 +1,10 @@ -import { MongoDB } from '@alien-worlds/storage-mongodb'; -import { ActionTraceModel } from '../../common/blockchain/contract/action-trace'; -import { DeltaRowModel } from '../../common/blockchain/contract/delta'; +import { ActionTrace } from "../types"; export type ProcessorTaskError = { message: string; stack: string; }; -export type ProcessorTaskDocument = { - _id?: MongoDB.ObjectId; - abi?: string; - is_fork?: boolean; - short_id?: string; - label?: string; - timestamp?: Date; - type?: string; - mode?: string; - content?: MongoDB.Binary; - hash?: string; - block_number?: MongoDB.Long; - block_timestamp?: Date; - error?: ProcessorTaskError; -}; - export type ProcessorTaskModel = { id: string; isFork: string; @@ -38,17 +20,17 @@ export type ProcessorTaskModel = { }; export type DeltaProcessorContentModel = { - shipDeltaMessageName: string; + ship_delta_message_name: string; name: string; - blockNumber: bigint; - blockTimestamp: Date; - row: DeltaRowModel; + block_num: bigint; + block_timestamp: Date; + row_data: Uint8Array; }; export type ActionProcessorContentModel = { - shipTraceMessageName: string; - transactionId: string; - blockNumber: bigint; - blockTimestamp: Date; - actionTrace: ActionTraceModel; + ship_trace_message_name: string; + transaction_id: string; + block_num: bigint; + block_timestamp: Date; + action_trace: ActionTrace; }; diff --git a/src/common/types/action-trace.types.ts b/src/common/types/action-trace.types.ts new file mode 100644 index 0000000..b4a80be --- /dev/null +++ b/src/common/types/action-trace.types.ts @@ -0,0 +1,45 @@ +export type AuthSequence = { + account: string; + sequence: string; +}; + +export type Receipt = { + receiver: string; + act_digest: string; + global_sequence: string; + recv_sequence: string; + auth_sequence: AuthSequence[]; + code_sequence: number; + abi_sequence: number; +}; + +export type ReceiptByName = [string, Receipt]; + +export type ActAuth = { + actor: string; + permission: string; +}; + +export type Act = { + account: string; + name: string; + authorization: ActAuth; + data: Uint8Array; +}; + +export type ActionTrace = { + ship_message_name?: string; + action_ordinal?: number; + creator_action_ordinal?: number; + receipt?: ReceiptByName; + receiver?: string; + act?: Act; + context_free?: boolean; + elapsed?: string; + console?: string; + account_ram_deltas?: unknown[]; + except?: unknown; + error_code?: string | number; +}; + +export type ActionTraceByName = [string, ActionTrace]; diff --git a/src/common/types/block.types.ts b/src/common/types/block.types.ts new file mode 100644 index 0000000..664ab9e --- /dev/null +++ b/src/common/types/block.types.ts @@ -0,0 +1,20 @@ +export type BlockNumberWithId = { + block_num?: unknown; + block_id?: string; +}; + +export type BlockModel< + BlockType = Uint8Array, + TracesType = Uint8Array, + DeltasType = Uint8Array +> = { + head?: BlockNumberWithId; + this_block?: BlockNumberWithId; + last_irreversible?: BlockNumberWithId; + prev_block?: BlockNumberWithId; + block?: BlockType; + traces?: TracesType; + deltas?: DeltasType; + abi_version?: string; + [key: string]: unknown; +}; diff --git a/src/common/types/delta.types.ts b/src/common/types/delta.types.ts new file mode 100644 index 0000000..1beb87c --- /dev/null +++ b/src/common/types/delta.types.ts @@ -0,0 +1,11 @@ +export type DeltaRow = { + present?: number; + data?: Uint8Array; +}; + +export type Delta = { + name?: string; + rows?: DeltaRow[]; +}; + +export type DeltaByName = [string, Delta]; diff --git a/src/common/types/index.ts b/src/common/types/index.ts new file mode 100644 index 0000000..9279e97 --- /dev/null +++ b/src/common/types/index.ts @@ -0,0 +1,5 @@ +export * from './action-trace.types'; +export * from './signed-block.types'; +export * from './delta.types'; +export * from './trace.types'; +export * from './transaction.types'; diff --git a/src/common/contract-content/signed-block/signed-block.dtos.ts b/src/common/types/signed-block.types.ts similarity index 65% rename from src/common/contract-content/signed-block/signed-block.dtos.ts rename to src/common/types/signed-block.types.ts index 8e5dad2..acfe49e 100644 --- a/src/common/contract-content/signed-block/signed-block.dtos.ts +++ b/src/common/types/signed-block.types.ts @@ -1,6 +1,6 @@ -import { TransactionJson } from '../transaction/transaction.dtos'; +import { Transaction } from './transaction.types'; -export type SignedBlockJson = { +export type SignedBlock = { timestamp: string; producer: string; confirmed: number; @@ -11,5 +11,5 @@ export type SignedBlockJson = { new_producers: unknown; header_extensions: unknown[]; producer_signature: string; - transactions: TransactionJson[]; + transactions: Transaction[]; }; diff --git a/src/common/contract-content/trace/trace.dtos.ts b/src/common/types/trace.types.ts similarity index 64% rename from src/common/contract-content/trace/trace.dtos.ts rename to src/common/types/trace.types.ts index 883855e..5bb8ddd 100644 --- a/src/common/contract-content/trace/trace.dtos.ts +++ b/src/common/types/trace.types.ts @@ -1,6 +1,6 @@ -import { ActionTraceByNameDto } from '../action-trace'; +import { ActionTraceByName } from './action-trace.types'; -export type PartialJson = { +export type Partial = { expiration: string; ref_block_num: number; ref_block_prefix: number; @@ -12,9 +12,9 @@ export type PartialJson = { context_free_data: unknown[]; }; -export type PartialByTypeJson = [string, PartialJson]; +export type PartialByType = [string, Partial]; -export type TraceJson = { +export type Trace = { id?: string; status?: number; cpu_usage_us?: number; @@ -22,12 +22,12 @@ export type TraceJson = { elapsed?: string; net_usage?: string; scheduled?: boolean; - action_traces?: ActionTraceByNameDto[]; + action_traces?: ActionTraceByName[]; account_ram_delta?: unknown; except?: unknown; error_code?: number | string; failed_dtrx_trace?: unknown; - partial?: PartialByTypeJson; + partial?: PartialByType; }; -export type TraceByNameJson = [string, TraceJson]; +export type TraceByName = [string, Trace]; diff --git a/src/common/contract-content/transaction/transaction.dtos.ts b/src/common/types/transaction.types.ts similarity index 56% rename from src/common/contract-content/transaction/transaction.dtos.ts rename to src/common/types/transaction.types.ts index c528852..f3022ef 100644 --- a/src/common/contract-content/transaction/transaction.dtos.ts +++ b/src/common/types/transaction.types.ts @@ -1,14 +1,14 @@ -export type PackedTrxJson = { +export type PackedTrx = { signatures: string[]; compression: number; packed_context_free_data: unknown; packed_trx: Uint8Array; }; -export type TrxByNameJson = [string, PackedTrxJson | string]; +export type TrxByName = [string, PackedTrx | string]; -export type TransactionJson = { +export type Transaction = { status: number; cpu_usage_us: number; net_usage_words: number; - trx: TrxByNameJson; + trx: TrxByName; }; diff --git a/src/common/unprocessed-block-queue/__dependencies__/index.ts b/src/common/unprocessed-block-queue/__dependencies__/index.ts new file mode 100644 index 0000000..a9e5218 --- /dev/null +++ b/src/common/unprocessed-block-queue/__dependencies__/index.ts @@ -0,0 +1,3 @@ +export * from './unprocessed-block-queue.mongo.collection'; +export * from './unprocessed-block-queue.mongo.mapper'; +export * from './unprocessed-block-queue.mongo.types'; diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.mongo.collection.ts b/src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.collection.ts similarity index 80% rename from src/common/unprocessed-block-queue/unprocessed-block-queue.mongo.collection.ts rename to src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.collection.ts index 9664f9e..1a16c53 100644 --- a/src/common/unprocessed-block-queue/unprocessed-block-queue.mongo.collection.ts +++ b/src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.collection.ts @@ -1,11 +1,11 @@ import { DataSourceError } from '@alien-worlds/api-core'; import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; -import { BlockMongoModel } from './unprocessed-block-queue.types'; -import { UnprocessedBlockCollection } from './unprocessed-block-queue.collection'; +import { BlockMongoModel } from './unprocessed-block-queue.mongo.types'; +import { UnprocessedBlockSource } from '../unprocessed-block-queue.source'; export class UnprocessedBlockMongoCollection extends MongoCollectionSource - implements UnprocessedBlockCollection + implements UnprocessedBlockSource { constructor(mongoSource: MongoSource) { super(mongoSource, 'history_tools.unprocessed_blocks', { diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.mongo.mapper.ts b/src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.mapper.ts similarity index 90% rename from src/common/unprocessed-block-queue/unprocessed-block-queue.mongo.mapper.ts rename to src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.mapper.ts index 0ab3c39..79e1a32 100644 --- a/src/common/unprocessed-block-queue/unprocessed-block-queue.mongo.mapper.ts +++ b/src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.mapper.ts @@ -1,44 +1,39 @@ import { Block } from '@alien-worlds/block-reader'; -import { BlockMongoModel } from './unprocessed-block-queue.types'; import { parseToBigInt } from '@alien-worlds/api-core'; -import { MongoDB } from '@alien-worlds/storage-mongodb'; +import { MongoDB, MongoMapper } from '@alien-worlds/storage-mongodb'; +import { BlockMongoModel } from './unprocessed-block-queue.mongo.types'; -export class UnprocessedBlockMongoMapper { +export class UnprocessedBlockMongoMapper extends MongoMapper { public toEntity(model: BlockMongoModel): Block { const { block, traces, deltas, abi_version } = model; let head; let thisBlock; let prevBlock; let lastIrreversible; - if (model.head) { head = { blockNumber: parseToBigInt(model.head.block_num), blockId: model.head.block_id, }; } - if (model.this_block) { thisBlock = { blockNumber: parseToBigInt(model.this_block.block_num), blockId: model.this_block.block_id, }; } - if (model.prev_block) { prevBlock = { blockNumber: parseToBigInt(model.prev_block.block_num), blockId: model.prev_block.block_id, }; } - if (model.last_irreversible) { lastIrreversible = { blockNumber: parseToBigInt(model.last_irreversible.block_num), blockId: model.last_irreversible.block_id, }; } - return new Block( head, lastIrreversible, @@ -50,7 +45,6 @@ export class UnprocessedBlockMongoMapper { abi_version ); } - public fromEntity(entity: Block): BlockMongoModel { const { head, @@ -63,7 +57,6 @@ export class UnprocessedBlockMongoMapper { id, abiVersion, } = entity; - const document: BlockMongoModel = { head: { block_id: head.blockId, @@ -85,15 +78,12 @@ export class UnprocessedBlockMongoMapper { traces: new MongoDB.Binary(traces), deltas: new MongoDB.Binary(deltas), }; - if (abiVersion) { document.abi_version = abiVersion; } - if (id) { document._id = new MongoDB.ObjectId(id); } - return document; } } diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.types.ts b/src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.types.ts similarity index 58% rename from src/common/unprocessed-block-queue/unprocessed-block-queue.types.ts rename to src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.types.ts index 28f3e81..18d6b03 100644 --- a/src/common/unprocessed-block-queue/unprocessed-block-queue.types.ts +++ b/src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.types.ts @@ -17,20 +17,3 @@ export type BlockMongoModel = { abi_version?: string; [key: string]: unknown; }; - -export type BlockNumberWithId = { - block_num?: unknown; - block_id?: string; -}; - -export type BlockModel = { - head?: BlockNumberWithId; - this_block?: BlockNumberWithId; - last_irreversible?: BlockNumberWithId; - prev_block?: BlockNumberWithId; - block?: unknown; - traces?: unknown; - deltas?: unknown; - abi_version?: unknown; - [key: string]: unknown; -}; diff --git a/src/common/unprocessed-block-queue/index.ts b/src/common/unprocessed-block-queue/index.ts index 135220e..e3f5122 100644 --- a/src/common/unprocessed-block-queue/index.ts +++ b/src/common/unprocessed-block-queue/index.ts @@ -1,7 +1,4 @@ +export * from './__dependencies__'; export * from './unprocessed-block-queue'; -export * from './unprocessed-block-queue.collection'; +export * from './unprocessed-block-queue.source'; export * from './unprocessed-block-queue.errors'; -export * from './unprocessed-block-queue.mapper'; -export * from './unprocessed-block-queue.mongo.collection'; -export * from './unprocessed-block-queue.mongo.mapper'; -export * from './unprocessed-block-queue.types'; diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.mapper.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.mapper.ts deleted file mode 100644 index 628791d..0000000 --- a/src/common/unprocessed-block-queue/unprocessed-block-queue.mapper.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Block } from '@alien-worlds/block-reader'; - -export abstract class UnprocessedBlockMapper { - public abstract toEntity(model: ModelType): Block; - - public abstract fromEntity(entity: Block): ModelType; -} diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.collection.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.source.ts similarity index 64% rename from src/common/unprocessed-block-queue/unprocessed-block-queue.collection.ts rename to src/common/unprocessed-block-queue/unprocessed-block-queue.source.ts index 21e6df1..b631975 100644 --- a/src/common/unprocessed-block-queue/unprocessed-block-queue.collection.ts +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.source.ts @@ -1,7 +1,6 @@ import { DataSource } from '@alien-worlds/api-core'; -export abstract class UnprocessedBlockCollection extends DataSource { +export abstract class UnprocessedBlockSource extends DataSource { public abstract next(): Promise; - public abstract bytesSize(): Promise; } diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.ts index 5168b78..26e2658 100644 --- a/src/common/unprocessed-block-queue/unprocessed-block-queue.ts +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.ts @@ -2,6 +2,7 @@ import { DataSourceError, Failure, log, + Mapper, parseToBigInt, Result, } from '@alien-worlds/api-core'; @@ -11,9 +12,8 @@ import { UnprocessedBlocksOverloadError, } from './unprocessed-block-queue.errors'; import { Block } from '@alien-worlds/block-reader'; -import { UnprocessedBlockCollection } from './unprocessed-block-queue.collection'; -import { BlockModel } from './unprocessed-block-queue.types'; -import { UnprocessedBlockMapper } from './unprocessed-block-queue.mapper'; +import { UnprocessedBlockSource } from './unprocessed-block-queue.source'; +import { BlockModel } from '../types/block.types'; export abstract class UnprocessedBlockQueueReader { public abstract next(): Promise>; @@ -28,8 +28,8 @@ export class UnprocessedBlockQueue protected afterSendBatchHandler: () => void; constructor( - protected collection: UnprocessedBlockCollection, - protected mapper: UnprocessedBlockMapper, + protected collection: UnprocessedBlockSource, + protected mapper: Mapper, protected maxBytesSize: number, protected batchSize: number ) {} diff --git a/src/config/index.ts b/src/config/index.ts index 2d81c12..f35f513 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -10,7 +10,10 @@ import { AbisConfig, AbisServiceConfig, BlockRangeScanConfig, + ContractTraceMatchCriteria, + ContractDeltaMatchCriteria, FeaturedConfig, + ProcessorMatchCriteria, } from '../common'; import { buildMongoConfig } from '@alien-worlds/storage-mongodb'; import { buildBroadcastConfig } from '@alien-worlds/broadcast'; @@ -49,7 +52,6 @@ export const buildAbisConfig = (vars: ConfigVars): AbisConfig => ({ }); export const buildBlockReaderConfig = (vars: ConfigVars): BlockReaderConfig => ({ - mongo: buildMongoConfig(vars), endpoints: vars.getArrayEnv('BLOCK_READER_ENDPOINTS'), shouldFetchDeltas: vars.getBooleanEnv('BLOCK_READER_FETCH_DELTAS'), shouldFetchTraces: vars.getBooleanEnv('BLOCK_READER_FETCH_TRACES'), @@ -160,11 +162,18 @@ export const buildProcessorConfig = ( export const buildHistoryToolsConfig = ( vars: ConfigVars, - featured: FeaturedConfig + featured?: { + traces: [ProcessorMatchCriteria]; + deltas: [ProcessorMatchCriteria]; + }, + bootstrapOptions?: BootstrapCommandOptions, + readerOptions?: ReaderCommandOptions, + filterOptions?: FilterCommandOptions, + processorOptions?: ProcessorCommandOptions ): HistoryToolsConfig => ({ api: buildApiConfig(vars), - bootstrap: buildBootstrapConfig(vars), - reader: buildReaderConfig(vars), - filter: buildFilterConfig(vars), - processor: buildProcessorConfig(vars, featured), + bootstrap: buildBootstrapConfig(vars, bootstrapOptions), + reader: buildReaderConfig(vars, readerOptions), + filter: buildFilterConfig(vars, filterOptions), + processor: buildProcessorConfig(vars, processorOptions), }); diff --git a/src/filter/filter.dependencies.ts b/src/filter/filter.dependencies.ts index 171ca82..591a59c 100644 --- a/src/filter/filter.dependencies.ts +++ b/src/filter/filter.dependencies.ts @@ -1,6 +1,6 @@ import { Result, Serializer } from '@alien-worlds/api-core'; import { Dependencies } from '../common/dependencies'; -import { FilterConfig } from './filter.types'; +import { FilterAddons, FilterConfig } from './filter.types'; import { BroadcastClient } from '@alien-worlds/broadcast'; import { WorkerPool } from '@alien-worlds/workers'; import { UnprocessedBlockQueue } from '../common'; @@ -20,5 +20,8 @@ export abstract class FilterDependencies extends Dependencies { public unprocessedBlockQueue: UnprocessedBlockQueue; public serializer: Serializer; - public abstract initialize(config: FilterConfig): Promise; + public abstract initialize( + config: FilterConfig, + addons?: FilterAddons + ): Promise; } diff --git a/src/filter/filter.runner.ts b/src/filter/filter.runner.ts index 57a23ba..643a2ab 100644 --- a/src/filter/filter.runner.ts +++ b/src/filter/filter.runner.ts @@ -1,8 +1,7 @@ import { log } from '@alien-worlds/api-core'; -import { BlockNotFoundError } from '../reader/unprocessed-block-queue/unprocessed-block-queue.errors'; -import { UnprocessedBlockQueueReader } from '../reader'; import { BlockJson } from '@alien-worlds/block-reader'; import { WorkerMessage, WorkerPool } from '@alien-worlds/workers'; +import { BlockNotFoundError, UnprocessedBlockQueueReader } from '../common'; export class FilterRunner { private interval: NodeJS.Timeout; diff --git a/src/filter/filter.types.ts b/src/filter/filter.types.ts index 8d1c67c..097f7fa 100644 --- a/src/filter/filter.types.ts +++ b/src/filter/filter.types.ts @@ -1,11 +1,8 @@ -import { ProcessorTaskQueueConfig } from '../common/processor-task-queue/processor-task-queue.config'; -import { AbisConfig } from '../common/abis'; import { WorkersConfig } from '@alien-worlds/workers'; import { BroadcastConfig } from '@alien-worlds/broadcast'; -import { FeaturedConfig } from '../common'; +import { AbisConfig, FeaturedConfig, ProcessorTaskQueueConfig } from '../common'; import { MongoConfig } from '@alien-worlds/storage-mongodb'; import { UnknownObject } from '@alien-worlds/api-core'; -import { TransactionJson } from '../common/contract-content/transaction'; export type FilterSharedData = { config: FilterConfig; @@ -32,130 +29,3 @@ export type FilterAddons = { matchers?: unknown; [key: string]: unknown; }; - -export type BlockNumberWithId = { - block_num: string; - block_id: string; -}; - -export type SignedBlockJson = { - timestamp: string; - producer: string; - confirmed: number; - previous: string; - transaction_mroot: string; - action_mroot: string; - schedule_version: number; - new_producers: unknown; - header_extensions: unknown[]; - producer_signature: string; - transactions: TransactionJson[]; -}; - -export type DeserializedBlock = { - head?: BlockNumberWithId; - this_block?: BlockNumberWithId; - last_irreversible?: BlockNumberWithId; - prev_block?: BlockNumberWithId; - block?: SignedBlockJson; - traces?: [[string, TraceJson]]; - deltas?: [[string, DeltaJson]]; - abi_version?: string; -}; - -// - -export type AuthSequenceJson = { - account: string; - sequence: string; -}; - -export type ReceiptJson = { - receiver: string; - act_digest: string; - global_sequence: string; - recv_sequence: string; - auth_sequence: AuthSequenceJson[]; - code_sequence: number; - abi_sequence: number; -}; - -export type ReceiptByNameDto = [string, ReceiptJson]; - -export type ActAuthJson = { - actor: string; - permission: string; -}; - -export type ActJson = { - account: string; - name: string; - authorization: ActAuthJson; - data: Uint8Array; -}; - -export type ActionTraceDto = { - ship_message_name?: string; - action_ordinal?: number; - creator_action_ordinal?: number; - receipt?: ReceiptByNameDto; - receiver?: string; - act?: ActJson; - context_free?: boolean; - elapsed?: string; - console?: string; - account_ram_deltas?: unknown[]; - except?: unknown; - error_code?: string | number; -}; - -export type ActionTraceByNameDto = [string, ActionTraceDto]; - -export type PartialJson = { - expiration: string; - ref_block_num: number; - ref_block_prefix: number; - max_net_usage_words: number; - max_cpu_usage_ms: number; - delay_sec: number; - transaction_extensions: unknown[]; - signatures: unknown[]; - context_free_data: unknown[]; -}; - -export type PartialByTypeJson = [string, PartialJson]; - -export type TraceJson = { - id?: string; - status?: number; - cpu_usage_us?: number; - net_usage_words?: number; - elapsed?: string; - net_usage?: string; - scheduled?: boolean; - action_traces?: ActionTraceByNameDto[]; - account_ram_delta?: unknown; - except?: unknown; - error_code?: number | string; - failed_dtrx_trace?: unknown; - partial?: PartialByTypeJson; -}; - -export type TraceByNameJson = [string, TraceJson]; - -export type DeltaRowDto = { - present?: number; - data?: Uint8Array; -}; - -export type DeltaJson = { - name?: string; - rows?: DeltaRowDto[]; -}; - -export type DeltaByNameDto = [string, DeltaJson]; - -export type DeltaRowModel = { - present?: number; - data?: Uint8Array; -}; diff --git a/src/filter/filter.worker.ts b/src/filter/filter.worker.ts index 9866af9..df6c553 100644 --- a/src/filter/filter.worker.ts +++ b/src/filter/filter.worker.ts @@ -1,11 +1,12 @@ import { Serializer, log, parseToBigInt } from '@alien-worlds/api-core'; import { Worker } from '@alien-worlds/workers'; -import { AbiNotFoundError, BlockJson, ShipAbis } from '@alien-worlds/block-reader'; -import { Featured } from '../common'; +import { AbiNotFoundError, ShipAbis } from '@alien-worlds/block-reader'; +import { DeltaByName, Featured, SignedBlock, TraceByName } from '../common'; import { Abis } from '../common/abis'; import { isSetAbiAction } from '../common/common.utils'; import { ProcessorTask, ProcessorTaskQueue } from '../common/processor-task-queue'; -import { DeserializedBlock, FilterSharedData } from './filter.types'; +import { FilterSharedData } from './filter.types'; +import { BlockModel } from '../common/types/block.types'; export default class FilterWorker extends Worker { constructor( @@ -22,7 +23,7 @@ export default class FilterWorker extends Worker { } public async createActionProcessorTasks( - deserializedBlock: DeserializedBlock + deserializedBlock: BlockModel ): Promise { const { dependencies: { abis, featured }, @@ -107,7 +108,7 @@ export default class FilterWorker extends Worker { } public async createDeltaProcessorTasks( - deserializedBlock: DeserializedBlock + deserializedBlock: BlockModel ): Promise { const { dependencies: { abis, featured, serializer }, @@ -131,7 +132,6 @@ export default class FilterWorker extends Worker { const tableRow = tableRows[i]; if (!tableRow) { - // contract allocation cannot be extracted // The contract may not contain tables or may be corrupted continue; } @@ -181,12 +181,12 @@ export default class FilterWorker extends Worker { table, parseToBigInt(this_block.block_num), new Date(timestamp), - tableRow, + tableRow.data as Uint8Array, parseToBigInt(this_block.block_num) <= parseToBigInt(prev_block.block_num) ) ); } catch (error) { - log(`Delta (table row) not handled`, error); + log(error); } } } @@ -195,7 +195,7 @@ export default class FilterWorker extends Worker { return list; } - public async run(json: BlockJson): Promise { + public async run(json: BlockModel): Promise { try { const { dependencies: { serializer, shipAbis, processorTaskQueue }, @@ -207,10 +207,10 @@ export default class FilterWorker extends Worker { this.reject(failure.error); } - const deserializedBlock = serializer.deserializeBlock( - json, - abi.toHex() - ); + const deserializedBlock = serializer.deserializeBlock< + BlockModel, + BlockModel + >(json, abi.toHex()); const { this_block: { block_num }, } = deserializedBlock; diff --git a/src/filter/start-filter.ts b/src/filter/start-filter.ts index 2239ab6..8adceef 100644 --- a/src/filter/start-filter.ts +++ b/src/filter/start-filter.ts @@ -19,7 +19,11 @@ export const filter = async ( ) => { log(`Filter ... [starting]`); - await dependencies.initialize(config); + const initResult = await dependencies.initialize(config, addons); + + if (initResult.isFailure) { + throw initResult.failure.error; + } const { broadcastClient, workerPool, unprocessedBlockQueue } = dependencies; const runner = new FilterRunner(workerPool, unprocessedBlockQueue); diff --git a/src/processor/processor.dependencies.ts b/src/processor/processor.dependencies.ts index 94a17c3..77e0495 100644 --- a/src/processor/processor.dependencies.ts +++ b/src/processor/processor.dependencies.ts @@ -1,21 +1,26 @@ import { Result } from '@alien-worlds/api-core'; import { Dependencies } from '../common/dependencies'; import { BroadcastClient } from '@alien-worlds/broadcast'; -import { WorkerPool } from '@alien-worlds/workers'; -import { ProcessorConfig } from './processor.types'; +import { ProcessorAddons, ProcessorConfig } from './processor.types'; +import { + ContractTraceMatchCriteria, + ContractDeltaMatchCriteria, + FeaturedMapper, + ProcessorTaskQueue, +} from '../common'; /** * An abstract class representing a Processor dependencies. * @class ProcessorDependencies */ export abstract class ProcessorDependencies extends Dependencies { - /** - * The broadcast client used for communication. - * @type {BroadcastClient} - */ public broadcastClient: BroadcastClient; + public featuredTraces: FeaturedMapper; + public featuredDeltas: FeaturedMapper; + public processorTaskQueue: ProcessorTaskQueue; - public workerPool: WorkerPool; - - public abstract initialize(config: ProcessorConfig): Promise; + public abstract initialize( + config: ProcessorConfig, + addons?: ProcessorAddons + ): Promise; } diff --git a/src/processor/processor.runner.ts b/src/processor/processor.runner.ts index 4831831..2c02964 100644 --- a/src/processor/processor.runner.ts +++ b/src/processor/processor.runner.ts @@ -1,60 +1,27 @@ import { log } from '@alien-worlds/api-core'; -import { FeaturedContractContent } from '../common/featured'; import { ProcessorTaskQueue, ProcessorTask, ProcessorTaskModel, + ProcessorTaskType, + UnknownProcessorTypeError, } from '../common/processor-task-queue'; -import { WorkerMessage, WorkerPool } from '../common/workers'; -import { ProcessorAddons, ProcessorConfig } from './processor.types'; -import { processorWorkerLoaderPath } from './processor.consts'; +import { WorkerMessage, WorkerPool } from '@alien-worlds/workers'; +import { + ContractTraceMatchCriteria, + ContractDeltaMatchCriteria, + FeaturedMapper, +} from '../common'; export class ProcessorRunner { - private static instance: ProcessorRunner; - private static creatorPromise; - - private static async creator(config: ProcessorConfig, addons: ProcessorAddons) { - const { workers } = config; - const { matchers } = addons; - const queue = await ProcessorTaskQueue.create(config.mongo, false, config.queue); - const featuredContent = new FeaturedContractContent(config.featured, matchers); - const workerPool = await WorkerPool.create({ - ...workers, - workerLoaderPath: config.processorLoaderPath || processorWorkerLoaderPath, - }); - const runner = new ProcessorRunner(workerPool, queue, featuredContent); - - workerPool.onWorkerRelease(() => runner.next()); - - log(` * Worker Pool (max ${workerPool.workerMaxCount} workers) ... [ready]`); - ProcessorRunner.creatorPromise = null; - ProcessorRunner.instance = runner; - - return runner; - } - - public static async getInstance( - config: ProcessorConfig, - addons: ProcessorAddons - ): Promise { - if (ProcessorRunner.instance) { - return ProcessorRunner.instance; - } - - if (!ProcessorRunner.creatorPromise) { - ProcessorRunner.creatorPromise = ProcessorRunner.creator(config, addons); - } - - return ProcessorRunner.creatorPromise; - } - private interval: NodeJS.Timeout; private loop: boolean; constructor( - private workerPool: WorkerPool, - private queue: ProcessorTaskQueue, - private featuredContent: FeaturedContractContent + protected featuredTraces: FeaturedMapper, + protected featuredDeltas: FeaturedMapper, + protected workerPool: WorkerPool, + protected queue: ProcessorTaskQueue ) { this.interval = setInterval(async () => { if (this.workerPool.hasActiveWorkers() === false) { @@ -65,9 +32,20 @@ export class ProcessorRunner { } private async assignTask(task: ProcessorTask) { - const { queue, workerPool, featuredContent } = this; + const { queue, workerPool, featuredTraces, featuredDeltas } = this; + let processorName: string; + + if (task.type === ProcessorTaskType.Delta) { + processorName = await featuredDeltas.getProcessor(task.label); + } else if (task.type === ProcessorTaskType.Trace) { + processorName = await featuredTraces.getProcessor(task.label); + } else { + this.queue.stashUnsuccessfulTask(task, new UnknownProcessorTypeError(task.type)); + log(`Unknown processor task type "${task.label}". Task has been deleted.`); + return; + } - const processorName = await featuredContent.getProcessor(task.type, task.label); + // const processorName = await featured.getProcessor(task.type, task.label); // If there is a processor name, it then gets a worker from the worker pool. if (processorName) { const worker = await workerPool.getWorker(processorName); diff --git a/src/processor/processor.types.ts b/src/processor/processor.types.ts index a44b3be..fbc8fc4 100644 --- a/src/processor/processor.types.ts +++ b/src/processor/processor.types.ts @@ -37,7 +37,6 @@ export type DeltaProcessorInput = { scope: string; table: string; payer: string; - present: number; primaryKey: bigint; blockNumber: bigint; blockTimestamp: Date; diff --git a/src/processor/processors/action-trace.processor.ts b/src/processor/processors/action-trace.processor.ts index 81a8ce4..2bda3df 100644 --- a/src/processor/processors/action-trace.processor.ts +++ b/src/processor/processors/action-trace.processor.ts @@ -1,10 +1,11 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { Processor } from './processor'; import { ActionProcessorContentModel, ProcessorTaskModel, } from '../../common/processor-task-queue/processor-task.types'; import { ActionTraceProcessorInput, ProcessorSharedData } from '../processor.types'; -import { Container, Serializer } from '@alien-worlds/api-core'; +import { Container, Serializer, parseToBigInt } from '@alien-worlds/api-core'; import { deserialize } from 'v8'; export class ActionTraceProcessor< @@ -33,15 +34,17 @@ export class ActionTraceProcessor< const { abi, content: buffer } = model; const content: ActionProcessorContentModel = deserialize(buffer); const { - actionTrace: { + action_trace: { act, - receipt: { recvSequence, globalSequence }, + receipt, }, - blockNumber, - blockTimestamp, - transactionId, + block_num, + block_timestamp, + transaction_id, } = content; + const [receiptType, receiptContent] = receipt; + const { global_sequence, recv_sequence } = receiptContent; const data = serializer.deserializeActionData( act.account, act.name, @@ -50,13 +53,13 @@ export class ActionTraceProcessor< ); return { - blockNumber, - blockTimestamp, - transactionId, + blockNumber: parseToBigInt(block_num), + blockTimestamp: block_timestamp, + transactionId: transaction_id, account: act.account, name: act.name, - recvSequence, - globalSequence, + recvSequence: parseToBigInt(recv_sequence), + globalSequence: parseToBigInt(global_sequence), data: data as DataType, }; } diff --git a/src/processor/processors/delta.processor.ts b/src/processor/processors/delta.processor.ts index ba87b1d..a813d77 100644 --- a/src/processor/processors/delta.processor.ts +++ b/src/processor/processors/delta.processor.ts @@ -32,19 +32,18 @@ export class DeltaProcessor< } = this; const { abi, content: buffer } = model; const delta: DeltaProcessorContentModel = deserialize(buffer); - const { name, blockNumber, blockTimestamp } = delta; - const row = serializer.deserializeTableRow(delta.row.data, abi); - const { code, scope, table, primaryKey, payer, data } = row; + const { name, block_num, block_timestamp } = delta; + const row = serializer.deserializeTableRow(delta.row_data, abi); + const { code, scope, table, primary_key, payer, data } = row; return { name, - blockNumber, - blockTimestamp, - present: delta.row.present, + blockNumber: block_num, + blockTimestamp: block_timestamp, code, scope, table, - primaryKey: parseToBigInt(primaryKey), + primaryKey: parseToBigInt(primary_key), payer, data: data as DataType, }; diff --git a/src/processor/start-processor.ts b/src/processor/start-processor.ts index 1500c20..8544727 100644 --- a/src/processor/start-processor.ts +++ b/src/processor/start-processor.ts @@ -14,6 +14,8 @@ import { ProcessorDependencies } from './processor.dependencies'; import { processorCommand } from './processor.command'; import { buildProcessorConfig } from '../config'; import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { processorWorkerLoaderPath } from './processor.consts'; +import { WorkerPool } from '@alien-worlds/workers'; /** * @@ -28,10 +30,25 @@ export const process = async ( ) => { log(`Processor ... [starting]`); - await dependencies.initialize(config); + const initResult = await dependencies.initialize(config, addons); - const { broadcastClient } = dependencies; - const runner = await ProcessorRunner.getInstance(config, addons); + if (initResult.isFailure) { + throw initResult.failure.error; + } + + const { broadcastClient, featuredTraces, featuredDeltas, processorTaskQueue } = + dependencies; + const workerPool = await WorkerPool.create({ + ...config.workers, + workerLoaderPath: config.processorLoaderPath || processorWorkerLoaderPath, + }); + + const runner = new ProcessorRunner( + featuredTraces, + featuredDeltas, + workerPool, + processorTaskQueue + ); broadcastClient.onMessage( InternalBroadcastChannel.Processor, diff --git a/src/reader/reader.dependencies.ts b/src/reader/reader.dependencies.ts index 095a897..758e6ea 100644 --- a/src/reader/reader.dependencies.ts +++ b/src/reader/reader.dependencies.ts @@ -3,7 +3,7 @@ import { Dependencies } from '../common/dependencies'; import { WorkerPool } from '@alien-worlds/workers'; import { ReaderConfig } from './reader.types'; import { Result } from '@alien-worlds/api-core'; -import { BlockRangeScanner, UnprocessedBlockQueue } from '../common'; +import { BlockRangeScanner } from '../common'; /** * An abstract class representing a reader dependencies. @@ -13,7 +13,6 @@ export abstract class ReaderDependencies extends Dependencies { public broadcastClient: BroadcastClient; public scanner: BlockRangeScanner; public workerPool: WorkerPool; - public unprocessedBlockQueue: UnprocessedBlockQueue; public abstract initialize(config: ReaderConfig): Promise; } diff --git a/src/reader/reader.ts b/src/reader/reader.ts index c9bd2ff..ebe27cc 100644 --- a/src/reader/reader.ts +++ b/src/reader/reader.ts @@ -2,15 +2,19 @@ import { log } from '@alien-worlds/api-core'; import { Mode } from '../common/common.enums'; import { ReadCompleteData, ReadTaskData } from './reader.types'; import { FilterBroadcastMessage } from '../broadcast/messages'; -import { WorkerMessage } from '@alien-worlds/workers'; -import { ReaderDependencies } from './reader.dependencies'; +import { WorkerMessage, WorkerPool } from '@alien-worlds/workers'; +import { BlockRangeScanner } from '../common'; +import { BroadcastClient } from '@alien-worlds/broadcast'; export class Reader { private loop = false; private initTaskData: ReadTaskData; - constructor(protected dependencies: ReaderDependencies) { - const { workerPool, scanner } = dependencies; + constructor( + protected broadcastClient: BroadcastClient, + protected scanner: BlockRangeScanner, + protected workerPool: WorkerPool + ) { workerPool.onWorkerRelease(async () => { const { initTaskData } = this; if (initTaskData.mode === Mode.Replay) { @@ -24,9 +28,7 @@ export class Reader { } private async handleWorkerMessage(message: WorkerMessage) { - const { - dependencies: { workerPool, broadcastClient }, - } = this; + const { workerPool, broadcastClient } = this; const { data, error, workerId } = message; if (message.isTaskResolved()) { @@ -44,9 +46,7 @@ export class Reader { } private async handleWorkerError(id: number, error: Error) { - const { - dependencies: { workerPool }, - } = this; + const { workerPool } = this; log(`Worker error:`, error); workerPool.releaseWorker(id); } @@ -66,10 +66,7 @@ export class Reader { ); } - const { - dependencies: { workerPool, scanner }, - initTaskData, - } = this; + const { workerPool, scanner, initTaskData } = this; while (this.loop) { const worker = await workerPool.getWorker(); diff --git a/src/reader/start-reader.ts b/src/reader/start-reader.ts index 2677939..787ada0 100644 --- a/src/reader/start-reader.ts +++ b/src/reader/start-reader.ts @@ -22,10 +22,14 @@ import { BroadcastMessage } from '@alien-worlds/broadcast'; export const read = async (config: ReaderConfig, dependencies: ReaderDependencies) => { log(`Reader ... [starting]`); - await dependencies.initialize(config); + const initResult = await dependencies.initialize(config); - const { broadcastClient } = dependencies; - const reader = new Reader(dependencies); + if (initResult.isFailure) { + throw initResult.failure.error; + } + + const { broadcastClient, scanner, workerPool } = dependencies; + const reader = new Reader(broadcastClient, scanner, workerPool); let channel: string; let readyMessage; diff --git a/yarn.lock b/yarn.lock index 1819bd7..eb45f61 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,59 +2,46 @@ # yarn lockfile v1 -"@alien-worlds/api-core@^0.0.125": - version "0.0.125" - resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.125/386abf1c615f114905ec478516acd3fba08bbbac#386abf1c615f114905ec478516acd3fba08bbbac" - integrity sha512-dYrAIqIgpGXA/i+AU9sMGS/CslizxKDsnuhSrD86VxI8iP1OR1+D+6T4D+5YUXmO05Iw2a166UNwd/Xyrq3b5Q== +"@alien-worlds/api-core@^0.0.145": + version "0.0.145" + resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.145/d5f1d6a5c4e14c62acc5e986ebb13c35a7c8cbff#d5f1d6a5c4e14c62acc5e986ebb13c35a7c8cbff" + integrity sha512-KVrS6UCL3DQ4uTOLDPBIxHHQ7xtFHpTwrWejmgv6YDffiOBNUiQvw+fsXXkPmrSdR/aqj2fCWH3k6t0y5gU7Ww== dependencies: - eosjs "^22.1.0" inversify "^6.0.1" nanoid "^3.0.0" node-fetch "2" reflect-metadata "^0.1.13" -"@alien-worlds/api-core@^0.0.138": - version "0.0.138" - resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.138/4cb2b844e6e8e3f2aabd52d6833e224543553300#4cb2b844e6e8e3f2aabd52d6833e224543553300" - integrity sha512-5NZ//JB/7vgh33pjcpfOlrSZid0CaZzZnThWbIo4WEHYGi+5kodbvD76SLOpLjS0PG9891bqKaVxYG7ZpPLy6g== +"@alien-worlds/block-reader@^0.0.4": + version "0.0.4" + resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.4/5d406a126bf19685aedf238762599819de59b94d#5d406a126bf19685aedf238762599819de59b94d" + integrity sha512-Bofm0Fm0jlYxsezh4dQ93vDbv3/B4grFItOT0dbA69fH8SoCh33GMXi/M16KWbzMboOOXmiRyGavTHRX4kMn3g== dependencies: - inversify "^6.0.1" - nanoid "^3.0.0" - node-fetch "2" - reflect-metadata "^0.1.13" + "@alien-worlds/api-core" "^0.0.145" -"@alien-worlds/api-core@^0.0.144": - version "0.0.144" - resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.144/294e62d106de83d5bf8b31ed02c5ac97e98f7b85#294e62d106de83d5bf8b31ed02c5ac97e98f7b85" - integrity sha512-yVqBz52a17tv7XjzW/2s9ZgsA+WxjuBmrAFXznhNi/LIp24nbQWrnknpzufIshdi7h9dgJ3pQudYqqKmY/wyBw== +"@alien-worlds/broadcast@^0.0.5": + version "0.0.5" + resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.5/84e072991cdb63a45b48d8062a72bd6630f56e68#84e072991cdb63a45b48d8062a72bd6630f56e68" + integrity sha512-htVvAJomd2JmQ6wSslSUyOYKJGX7X3QnJ9rrbC25RSOoBRycfRznn6CTDmPS8ckWB+Cmk0x2RRsuXFQmWeKR+A== dependencies: - inversify "^6.0.1" + "@alien-worlds/api-core" "^0.0.145" nanoid "^3.0.0" node-fetch "2" - reflect-metadata "^0.1.13" -"@alien-worlds/block-reader@^0.0.3": - version "0.0.3" - resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.3/3cea505637f816621d007cf4f47df0f0ab534ae5#3cea505637f816621d007cf4f47df0f0ab534ae5" - integrity sha512-swDwYq+jv8JiZg5V98oW2vv8R1T/SpidNwDBQYKJk13XeYHkbBroomjL1+gw+lnz5tC1XUPMQp7HOxR3zqR9SQ== - dependencies: - "@alien-worlds/api-core" "^0.0.125" - -"@alien-worlds/broadcast@^0.0.4": - version "0.0.4" - resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.4/6ed3e29e47633ee152b710688ab83ce2e4b2884d#6ed3e29e47633ee152b710688ab83ce2e4b2884d" - integrity sha512-IL84vKnez7c6k1NC/SXpRo96PXDjoHM3wPkk5dykU3FC9EGjQ5ulaszSicCfF3JEdmdGBwWGBBIu/jAYcnNI/w== +"@alien-worlds/eos@^0.0.9": + version "0.0.9" + resolved "https://npm.pkg.github.com/download/@alien-worlds/eos/0.0.9/ef1cc073e48ccf01bf7663f5a4998d36e9b6cd51#ef1cc073e48ccf01bf7663f5a4998d36e9b6cd51" + integrity sha512-u9yobvp47nySp2HNzNNUynP3sU0FuzVZdfmmwS3GEK/rj6Svy63rAN+EJ9WekkV7k2oI0oP+mVVDspbh+RnF1g== dependencies: - "@alien-worlds/api-core" "^0.0.138" - nanoid "^3.0.0" - node-fetch "2" + "@alien-worlds/api-core" "^0.0.145" + eosjs "^22.1.0" -"@alien-worlds/storage-mongodb@^0.0.14": - version "0.0.14" - resolved "https://npm.pkg.github.com/download/@alien-worlds/storage-mongodb/0.0.14/a16846339c1fb7c2886592c96e5b6f735240f46e#a16846339c1fb7c2886592c96e5b6f735240f46e" - integrity sha512-hTYZOe5gwz2Yk8IOQbwt+LMp+ksFaCNo8uP8se2vubenv6vMlsCVuz8C+Cbl37Xo0XB7EOiB2MPI55+d1bcO5w== +"@alien-worlds/storage-mongodb@^0.0.16": + version "0.0.16" + resolved "https://npm.pkg.github.com/download/@alien-worlds/storage-mongodb/0.0.16/77dcc02b45b5f1cea0ed2fa5b4715ed5733e77f2#77dcc02b45b5f1cea0ed2fa5b4715ed5733e77f2" + integrity sha512-24dpfaY725nTbcm0jsU3ISqHddDkjn4wu1gI6fvP7AmkM5Vra1C0PDhVIG+J4B6WUsJXQQqDhhHOo+J0qShOJA== dependencies: - "@alien-worlds/api-core" "^0.0.138" + "@alien-worlds/api-core" "^0.0.145" mongodb "4.9.1" "@alien-worlds/workers@^0.0.3": From 2fa926b509b03374bafdab0cee71f716052563a7 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Thu, 22 Jun 2023 23:12:11 +0200 Subject: [PATCH 004/107] remove abieos --- package.json | 1 - yarn.lock | 12 ------------ 2 files changed, 13 deletions(-) diff --git a/package.json b/package.json index 5e4d945..0f58e87 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,6 @@ "@alien-worlds/eos": "^0.0.9", "@alien-worlds/storage-mongodb": "^0.0.16", "@alien-worlds/workers": "^0.0.3", - "@eosrio/node-abieos": "^1", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", diff --git a/yarn.lock b/yarn.lock index eb45f61..8c3305c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -347,13 +347,6 @@ dependencies: "@jridgewell/trace-mapping" "0.3.9" -"@eosrio/node-abieos@^1": - version "1.0.6" - resolved "https://registry.npmjs.org/@eosrio/node-abieos/-/node-abieos-1.0.6.tgz" - integrity sha512-Y/c0Hi8ylvlHAwUiyI198orbFLfMY2eRtvPWqW4PLudF9yFUXfQwZueDqmitGL2mgTEjXt+WbRGsQc2Oe1C2tA== - dependencies: - node-addon-api "2.0.0" - "@eslint/eslintrc@^1.3.3": version "1.3.3" resolved "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.3.tgz" @@ -3017,11 +3010,6 @@ negotiator@0.6.3: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== -node-addon-api@2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.0.tgz" - integrity sha512-ASCL5U13as7HhOExbT6OlWJJUV/lLzL2voOSP1UVehpRD8FbSrSDjfScK/KwAvVTI5AS6r4VwbOMlIqtvRidnA== - node-fetch@2: version "2.6.7" resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz" From bd0e32fd317e89e983c9b6578d3f8a511bdac36f Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sat, 24 Jun 2023 23:31:09 +0200 Subject: [PATCH 005/107] no compilation errors --- package.json | 14 +- .../contract-action.mongo.mapper.ts | 17 -- .../contract-action.mongo.source.ts | 15 - .../contract-action.mongo.types.ts | 6 - src/api/api.types.ts | 6 +- src/api/endpoints/actions/index.ts | 3 - src/api/endpoints/actions/ioc.config.ts | 36 --- .../__tests__/bootstrap.utils.unit.test.ts | 3 +- .../__tests__/start-bootstrap.unit.test.ts | 58 +--- src/bootstrap/bootstrap.dependencies.ts | 56 ---- src/bootstrap/bootstrap.types.ts | 20 +- src/bootstrap/bootstrap.utils.ts | 10 +- src/bootstrap/index.ts | 1 - src/bootstrap/start-bootstrap.ts | 28 +- .../internal-broadcast.message.unit.test.ts | 2 +- src/broadcast/internal-broadcast.message.ts | 2 +- .../processor-broadcast.message.unit.test.ts | 2 +- .../reader-broadcast.message.unit.test.ts | 2 +- .../messages/filter-broadcast.message.ts | 2 +- .../messages/processor-broadcast.message.ts | 2 +- .../messages/reader-broadcast.message.ts | 3 +- .../__tests__/common.utils.unit.test.ts | 23 -- .../abis/__dependencies__/abis.creator.ts | 45 --- .../__dependencies__/abis.mongo.collection.ts | 21 -- .../__dependencies__/abis.mongo.mapper.ts | 24 -- .../abis/__dependencies__/abis.mongo.types.ts | 18 -- src/common/abis/__dependencies__/index.ts | 4 - .../abi.repository-impl.unit.test.ts | 108 ------- .../abis/__tests__/abis.cache.unit.test.ts | 104 ------- src/common/abis/__tests__/abis.unit.test.ts | 162 ---------- src/common/abis/abis.cache.ts | 132 -------- src/common/abis/abis.errors.ts | 7 - src/common/abis/abis.repository-impl.ts | 144 --------- src/common/abis/abis.repository.ts | 53 ---- src/common/abis/abis.ts | 167 ---------- src/common/abis/abis.types.ts | 5 - src/common/abis/index.ts | 7 - .../block-range-scan.mongo.source.ts | 195 ------------ .../block-range-scanner.creator.ts | 34 --- .../block-range-scanner.mongo.mapper.ts | 109 ------- .../block-range-scanner.mongo.types.ts | 24 -- .../__dependencies__/index.ts | 4 - .../block-range-scan.repository.ts | 85 ------ .../block-range-scan.source.ts | 29 -- .../block-range-scanner/block-range-scan.ts | 186 ----------- .../block-range-scanner.config.ts | 4 - .../block-range-scanner.errors.ts | 26 -- .../block-range-scanner.ts | 44 --- src/common/block-range-scanner/index.ts | 7 - .../__dependencies__/block-state.creator.ts | 31 -- .../block-state.mongo.collection.ts | 20 -- .../block-state.mongo.mapper.ts | 28 -- .../block-state.mongo.types.ts | 9 - .../block-state/__dependencies__/index.ts | 5 - ...update-block-number.mongo.query-builder.ts | 17 -- .../__tests__/block-state.unit.test.ts | 112 ------- src/common/block-state/block-state.ts | 102 ------- src/common/block-state/block-state.types.ts | 6 - src/common/block-state/index.ts | 3 - src/common/common.enums.ts | 5 - src/common/common.errors.ts | 5 - src/common/common.utils.ts | 9 - src/common/dependencies.ts | 16 - ...eatured-contract.mongo.mapper.unit.test.ts | 45 --- .../__tests__/featured.mapper.unit.test.ts | 233 -------------- .../featured-contract.mongo.collection.ts | 26 -- .../featured-contract.mongo.mapper.ts | 48 --- .../__dependencies__/featured.creator.ts | 42 --- .../__dependencies__/featured.mongo.types.ts | 7 - src/common/featured/__dependencies__/index.ts | 4 - .../__tests__/featured.utils.unit.test.ts | 82 ----- src/common/featured/featured-contract.ts | 34 --- src/common/featured/featured.config.ts | 4 - src/common/featured/featured.errors.ts | 31 -- src/common/featured/featured.mapper.ts | 271 ---------------- src/common/featured/featured.ts | 78 ----- src/common/featured/featured.types.ts | 44 --- src/common/featured/featured.utils.ts | 31 -- src/common/featured/index.ts | 8 - src/common/index.ts | 11 - .../__dependencies__/index.ts | 4 - .../processor-task.mongo.collection.ts | 55 ---- .../processor-task.mongo.mapper.ts | 92 ------ .../processor-task.mongo.types.ts | 18 -- ...cessful-processor-task.mongo.collection.ts | 32 -- src/common/processor-task-queue/index.ts | 8 - .../processor-task-queue.config.ts | 4 - .../processor-task-queue.ts | 62 ---- .../processor-task.enums.ts | 4 - .../processor-task.errors.ts | 5 - .../processor-task.source.ts | 5 - .../processor-task-queue/processor-task.ts | 116 ------- .../processor-task.types.ts | 36 --- src/common/types/action-trace.types.ts | 45 --- src/common/types/block.types.ts | 20 -- src/common/types/delta.types.ts | 11 - src/common/types/index.ts | 5 - src/common/types/signed-block.types.ts | 15 - src/common/types/trace.types.ts | 33 -- src/common/types/transaction.types.ts | 14 - .../__dependencies__/index.ts | 3 - ...nprocessed-block-queue.mongo.collection.ts | 35 --- .../unprocessed-block-queue.mongo.mapper.ts | 89 ------ .../unprocessed-block-queue.mongo.types.ts | 19 -- src/common/unprocessed-block-queue/index.ts | 4 - .../unprocessed-block-queue.errors.ts | 14 - .../unprocessed-block-queue.source.ts | 6 - .../unprocessed-block-queue.ts | 143 --------- src/config/config.types.ts | 12 +- src/config/index.ts | 97 +++--- src/filter/filter.consts.ts | 1 - src/filter/filter.dependencies.ts | 27 -- src/filter/filter.runner.ts | 5 +- src/filter/filter.types.ts | 17 +- .../filter.worker-loader.dependencies.ts | 22 -- src/filter/filter.worker-loader.ts | 2 +- src/filter/filter.worker.ts | 25 +- src/filter/index.ts | 2 - src/filter/start-filter.ts | 34 ++- src/index.ts | 7 +- src/processor/index.ts | 2 - src/processor/processor.dependencies.ts | 26 -- src/processor/processor.runner.ts | 19 +- src/processor/processor.types.ts | 25 +- .../processor.worker-loader.dependencies.ts | 18 -- src/processor/processor.worker-loader.ts | 14 +- .../processors/action-trace.processor.ts | 15 +- src/processor/processors/delta.processor.ts | 12 +- src/processor/processors/processor.ts | 3 +- src/processor/start-processor.ts | 34 ++- src/reader/index.ts | 2 - src/reader/reader.dependencies.ts | 18 -- src/reader/reader.ts | 13 +- src/reader/reader.types.ts | 21 -- .../reader.worker-loader.dependencies.ts | 21 -- src/reader/reader.worker-loader.ts | 18 +- src/reader/reader.worker.ts | 24 +- src/reader/start-reader.ts | 29 +- yarn.lock | 289 +++--------------- 139 files changed, 324 insertions(+), 4853 deletions(-) delete mode 100644 src/api/__dependencies__/contract-action.mongo.mapper.ts delete mode 100644 src/api/__dependencies__/contract-action.mongo.source.ts delete mode 100644 src/api/__dependencies__/contract-action.mongo.types.ts delete mode 100644 src/api/endpoints/actions/ioc.config.ts delete mode 100644 src/bootstrap/bootstrap.dependencies.ts delete mode 100644 src/common/__tests__/common.utils.unit.test.ts delete mode 100644 src/common/abis/__dependencies__/abis.creator.ts delete mode 100644 src/common/abis/__dependencies__/abis.mongo.collection.ts delete mode 100644 src/common/abis/__dependencies__/abis.mongo.mapper.ts delete mode 100644 src/common/abis/__dependencies__/abis.mongo.types.ts delete mode 100644 src/common/abis/__dependencies__/index.ts delete mode 100644 src/common/abis/__tests__/abi.repository-impl.unit.test.ts delete mode 100644 src/common/abis/__tests__/abis.cache.unit.test.ts delete mode 100644 src/common/abis/__tests__/abis.unit.test.ts delete mode 100644 src/common/abis/abis.cache.ts delete mode 100644 src/common/abis/abis.errors.ts delete mode 100644 src/common/abis/abis.repository-impl.ts delete mode 100644 src/common/abis/abis.repository.ts delete mode 100644 src/common/abis/abis.ts delete mode 100644 src/common/abis/abis.types.ts delete mode 100644 src/common/abis/index.ts delete mode 100644 src/common/block-range-scanner/__dependencies__/block-range-scan.mongo.source.ts delete mode 100644 src/common/block-range-scanner/__dependencies__/block-range-scanner.creator.ts delete mode 100644 src/common/block-range-scanner/__dependencies__/block-range-scanner.mongo.mapper.ts delete mode 100644 src/common/block-range-scanner/__dependencies__/block-range-scanner.mongo.types.ts delete mode 100644 src/common/block-range-scanner/__dependencies__/index.ts delete mode 100644 src/common/block-range-scanner/block-range-scan.repository.ts delete mode 100644 src/common/block-range-scanner/block-range-scan.source.ts delete mode 100644 src/common/block-range-scanner/block-range-scan.ts delete mode 100644 src/common/block-range-scanner/block-range-scanner.config.ts delete mode 100644 src/common/block-range-scanner/block-range-scanner.errors.ts delete mode 100644 src/common/block-range-scanner/block-range-scanner.ts delete mode 100644 src/common/block-range-scanner/index.ts delete mode 100644 src/common/block-state/__dependencies__/block-state.creator.ts delete mode 100644 src/common/block-state/__dependencies__/block-state.mongo.collection.ts delete mode 100644 src/common/block-state/__dependencies__/block-state.mongo.mapper.ts delete mode 100644 src/common/block-state/__dependencies__/block-state.mongo.types.ts delete mode 100644 src/common/block-state/__dependencies__/index.ts delete mode 100644 src/common/block-state/__dependencies__/query-builders/update-block-number.mongo.query-builder.ts delete mode 100644 src/common/block-state/__tests__/block-state.unit.test.ts delete mode 100644 src/common/block-state/block-state.ts delete mode 100644 src/common/block-state/block-state.types.ts delete mode 100644 src/common/block-state/index.ts delete mode 100644 src/common/common.enums.ts delete mode 100644 src/common/common.errors.ts delete mode 100644 src/common/common.utils.ts delete mode 100644 src/common/dependencies.ts delete mode 100644 src/common/featured/__dependencies__/__tests__/featured-contract.mongo.mapper.unit.test.ts delete mode 100644 src/common/featured/__dependencies__/__tests__/featured.mapper.unit.test.ts delete mode 100644 src/common/featured/__dependencies__/featured-contract.mongo.collection.ts delete mode 100644 src/common/featured/__dependencies__/featured-contract.mongo.mapper.ts delete mode 100644 src/common/featured/__dependencies__/featured.creator.ts delete mode 100644 src/common/featured/__dependencies__/featured.mongo.types.ts delete mode 100644 src/common/featured/__dependencies__/index.ts delete mode 100644 src/common/featured/__tests__/featured.utils.unit.test.ts delete mode 100644 src/common/featured/featured-contract.ts delete mode 100644 src/common/featured/featured.config.ts delete mode 100644 src/common/featured/featured.errors.ts delete mode 100644 src/common/featured/featured.mapper.ts delete mode 100644 src/common/featured/featured.ts delete mode 100644 src/common/featured/featured.types.ts delete mode 100644 src/common/featured/featured.utils.ts delete mode 100644 src/common/featured/index.ts delete mode 100644 src/common/index.ts delete mode 100644 src/common/processor-task-queue/__dependencies__/index.ts delete mode 100644 src/common/processor-task-queue/__dependencies__/processor-task.mongo.collection.ts delete mode 100644 src/common/processor-task-queue/__dependencies__/processor-task.mongo.mapper.ts delete mode 100644 src/common/processor-task-queue/__dependencies__/processor-task.mongo.types.ts delete mode 100644 src/common/processor-task-queue/__dependencies__/unsuccessful-processor-task.mongo.collection.ts delete mode 100644 src/common/processor-task-queue/index.ts delete mode 100644 src/common/processor-task-queue/processor-task-queue.config.ts delete mode 100644 src/common/processor-task-queue/processor-task-queue.ts delete mode 100644 src/common/processor-task-queue/processor-task.enums.ts delete mode 100644 src/common/processor-task-queue/processor-task.errors.ts delete mode 100644 src/common/processor-task-queue/processor-task.source.ts delete mode 100644 src/common/processor-task-queue/processor-task.ts delete mode 100644 src/common/processor-task-queue/processor-task.types.ts delete mode 100644 src/common/types/action-trace.types.ts delete mode 100644 src/common/types/block.types.ts delete mode 100644 src/common/types/delta.types.ts delete mode 100644 src/common/types/index.ts delete mode 100644 src/common/types/signed-block.types.ts delete mode 100644 src/common/types/trace.types.ts delete mode 100644 src/common/types/transaction.types.ts delete mode 100644 src/common/unprocessed-block-queue/__dependencies__/index.ts delete mode 100644 src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.collection.ts delete mode 100644 src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.mapper.ts delete mode 100644 src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.types.ts delete mode 100644 src/common/unprocessed-block-queue/index.ts delete mode 100644 src/common/unprocessed-block-queue/unprocessed-block-queue.errors.ts delete mode 100644 src/common/unprocessed-block-queue/unprocessed-block-queue.source.ts delete mode 100644 src/common/unprocessed-block-queue/unprocessed-block-queue.ts delete mode 100644 src/filter/filter.dependencies.ts delete mode 100644 src/filter/filter.worker-loader.dependencies.ts delete mode 100644 src/processor/processor.dependencies.ts delete mode 100644 src/processor/processor.worker-loader.dependencies.ts delete mode 100644 src/reader/reader.dependencies.ts delete mode 100644 src/reader/reader.worker-loader.dependencies.ts diff --git a/package.json b/package.json index 0f58e87..7bb61dc 100644 --- a/package.json +++ b/package.json @@ -35,21 +35,11 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/api-core": "^0.0.145", - "@alien-worlds/block-reader": "^0.0.4", - "@alien-worlds/broadcast": "^0.0.5", - "@alien-worlds/eos": "^0.0.9", - "@alien-worlds/storage-mongodb": "^0.0.16", - "@alien-worlds/workers": "^0.0.3", + "@alien-worlds/history-tools-common": "^0.0.20", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", "express": "^4.18.2", - "nanoid": "^3.0.0", - "node-fetch": "2.6.6", - "reflect-metadata": "^0.1.13", - "text-encoding": "^0.7.0", - "ts-node": "^10.9.1", - "ws": "^8.12.0" + "ts-node": "^10.9.1" } } diff --git a/src/api/__dependencies__/contract-action.mongo.mapper.ts b/src/api/__dependencies__/contract-action.mongo.mapper.ts deleted file mode 100644 index 74a561b..0000000 --- a/src/api/__dependencies__/contract-action.mongo.mapper.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable @typescript-eslint/no-unused-vars */ -import { Entity, Mapper, PropertyMapping } from '@alien-worlds/api-core'; -import { ContractAction } from '@alien-worlds/api-core'; -import { ContractActionMongoModel } from './contract-action.mongo.types'; - -export class ContractActionMapper implements Mapper { - public toEntity(model: any): ContractAction { - throw new Error('Method not implemented.'); - } - public fromEntity(entity: ContractAction): ContractActionMongoModel { - throw new Error('Method not implemented.'); - } - public getEntityKeyMapping(key: string): PropertyMapping { - throw new Error('Method not implemented.'); - } -} diff --git a/src/api/__dependencies__/contract-action.mongo.source.ts b/src/api/__dependencies__/contract-action.mongo.source.ts deleted file mode 100644 index 97ae993..0000000 --- a/src/api/__dependencies__/contract-action.mongo.source.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { ContractAction } from '@alien-worlds/api-core'; -import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; - -/** - * @class - */ -export class ContractActionMongoSource extends MongoCollectionSource { - /** - * @constructor - * @param {MongoSource} mongoSource - */ - constructor(mongoSource: MongoSource) { - super(mongoSource, 'actions'); - } -} diff --git a/src/api/__dependencies__/contract-action.mongo.types.ts b/src/api/__dependencies__/contract-action.mongo.types.ts deleted file mode 100644 index 498caa6..0000000 --- a/src/api/__dependencies__/contract-action.mongo.types.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { MongoDB } from '@alien-worlds/storage-mongodb'; - -export type ContractActionMongoModel = { - _id?: MongoDB.ObjectId; - [key: string]: unknown; -}; diff --git a/src/api/api.types.ts b/src/api/api.types.ts index 2837d38..c3093b3 100644 --- a/src/api/api.types.ts +++ b/src/api/api.types.ts @@ -1,6 +1,6 @@ -import { MongoConfig } from "@alien-worlds/storage-mongodb"; +import { UnknownObject } from '@alien-worlds/history-tools-common'; -export type ApiConfig = { +export type ApiConfig = { port: number; - mongo: MongoConfig; + database: DatabaseConfig; }; diff --git a/src/api/endpoints/actions/index.ts b/src/api/endpoints/actions/index.ts index 838d3fc..a841e6a 100644 --- a/src/api/endpoints/actions/index.ts +++ b/src/api/endpoints/actions/index.ts @@ -1,10 +1,7 @@ -export * from '../../__dependencies__/contract-action.mongo.source'; export * from './domain/actions.types'; -export * from '../../__dependencies__/contract-action.mongo.mapper'; export * from './domain/list-actions.controller'; export * from './domain/models/list-actions.input'; export * from './domain/models/list-actions.output'; export * from './domain/repositories/contract-action.repository'; export * from './domain/use-cases/list-actions.use-case'; -export * from './ioc.config'; export * from './routes/list-actions.route'; diff --git a/src/api/endpoints/actions/ioc.config.ts b/src/api/endpoints/actions/ioc.config.ts deleted file mode 100644 index b00e396..0000000 --- a/src/api/endpoints/actions/ioc.config.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { RepositoryImpl, Container } from '@alien-worlds/api-core'; - -import { ContractActionMongoSource } from '../../__dependencies__/contract-action.mongo.source'; -import { ContractActionMapper } from '../../__dependencies__/contract-action.mongo.mapper'; -import { ContractActionRepository } from './domain/repositories/contract-action.repository'; -import { - MongoConfig, - MongoQueryBuilders, - MongoSource, -} from '@alien-worlds/storage-mongodb'; - -export const setupContractActionRepository = async ( - mongo: MongoSource | MongoConfig, - container?: Container -): Promise => { - let mongoSource: MongoSource; - if (mongo instanceof MongoSource) { - mongoSource = mongo; - } else { - mongoSource = await MongoSource.create(mongo); - } - - const repo: ContractActionRepository = new RepositoryImpl( - new ContractActionMongoSource(mongoSource), - new ContractActionMapper(), - new MongoQueryBuilders() - ); - - if (container) { - container - .bind(ContractActionRepository.Token) - .toConstantValue(repo); - } - - return repo; -}; diff --git a/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts b/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts index d113d01..a486224 100644 --- a/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts +++ b/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts @@ -9,8 +9,7 @@ import { StartBlockHigherThanEndBlockError, UndefinedStartBlockError, } from '../bootstrap.errors'; -import { Mode } from '../../common/common.enums'; -import { Result } from '@alien-worlds/api-core'; +import { Mode, Result } from '@alien-worlds/history-tools-common'; describe('createDefaultModeBlockRange', () => { const originalLog = console.log; diff --git a/src/bootstrap/__tests__/start-bootstrap.unit.test.ts b/src/bootstrap/__tests__/start-bootstrap.unit.test.ts index 5ad1098..6b2c243 100644 --- a/src/bootstrap/__tests__/start-bootstrap.unit.test.ts +++ b/src/bootstrap/__tests__/start-bootstrap.unit.test.ts @@ -1,14 +1,10 @@ -import { Mode } from './../../common/common.enums'; -import { BroadcastTcpClient } from '@alien-worlds/broadcast'; -import { MongoSource } from '@alien-worlds/storage-mongodb'; -import { AbisCreator } from '../../common'; import { bootstrap } from '../start-bootstrap'; import { NoAbisError } from '../bootstrap.errors'; import { InternalBroadcastMessageName } from '../../broadcast/internal-broadcast.enums'; import { ReaderBroadcastMessage } from '../../broadcast/messages'; -import { BlockRangeScannerCreator } from '../../common/block-range-scanner/__dependencies__/block-range-scanner.creator'; +import { BroadcastTcpClient, Mode } from '@alien-worlds/history-tools-common'; -jest.mock('../../common', () => ({ +jest.mock('@alien-worlds/history-tools-common', () => ({ Abis: { create: jest .fn() @@ -17,21 +13,13 @@ jest.mock('../../common', () => ({ BlockState: { create: jest.fn().mockResolvedValue({}) }, ContractReader: { create: jest.fn().mockResolvedValue({ readContracts: jest.fn() }) }, BlockRangeScanner: { create: jest.fn().mockResolvedValue({}) }, -})); - -jest.mock('@alien-worlds/broadcast', () => { - return { - BroadcastTcpClient: jest.fn().mockImplementation(() => { - return { - onMessage: jest.fn(), - sendMessage: jest.fn(), - connect: jest.fn(), - }; - }), - }; -}); - -jest.mock('@alien-worlds/storage-mongodb', () => ({ + BroadcastTcpClient: jest.fn().mockImplementation(() => { + return { + onMessage: jest.fn(), + sendMessage: jest.fn(), + connect: jest.fn(), + }; + }), MongoSource: { create: jest.fn().mockResolvedValue({}) }, })); @@ -41,7 +29,7 @@ jest.mock('../bootstrap.utils', () => ({ createTestModeBlockRange: jest.fn().mockResolvedValue({}), })); -const featuredJson = { +const featuredCriteria = { traces: [ { shipTraceMessageName: ['transaction_trace_v0'], @@ -68,7 +56,7 @@ const featuredJson = { processor: 'MSIG_WORLDS_DELTA_PROCESSOR', }, ], -}; +} as any; const config = { mode: Mode.Default, @@ -77,7 +65,7 @@ const config = { contractReader: {}, abis: {}, scanner: {}, - featured: featuredJson, + featured: featuredCriteria, } as any; const dependencies = {} as any; @@ -89,26 +77,8 @@ describe('bootstrap', () => { mockBroadcast = new BroadcastTcpClient({}); }); - it('throws an error when there are no abis', async () => { - (AbisCreator.create as jest.Mock).mockResolvedValueOnce({ - fetchAbis: jest.fn().mockResolvedValue([]), - }); - await expect(bootstrap(config, dependencies, featuredJson)).rejects.toThrow( - NoAbisError - ); - }); - - it('runs without error in default mode', async () => { - await bootstrap(config, dependencies, featuredJson); - - expect(BlockRangeScannerCreator.create).toHaveBeenCalled(); - expect(AbisCreator.create).toHaveBeenCalled(); - expect(MongoSource.create).toHaveBeenCalled(); - expect(BroadcastTcpClient).toHaveBeenCalled(); - }); - it.skip('handles DefaultModeReaderReady message correctly', async () => { - await bootstrap(config, dependencies, featuredJson); + await bootstrap(config, dependencies, featuredCriteria); const message = { name: InternalBroadcastMessageName.DefaultModeReaderReady }; const messageHandler = mockBroadcast.onMessage.mock.calls[0][1]; await messageHandler(message); @@ -120,7 +90,7 @@ describe('bootstrap', () => { }); it.skip('handles TestModeReaderReady message correctly', async () => { - await bootstrap(config, dependencies, featuredJson); + await bootstrap(config, dependencies, featuredCriteria); const message = { name: InternalBroadcastMessageName.DefaultModeReaderReady }; diff --git a/src/bootstrap/bootstrap.dependencies.ts b/src/bootstrap/bootstrap.dependencies.ts deleted file mode 100644 index 82fdb95..0000000 --- a/src/bootstrap/bootstrap.dependencies.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { BroadcastClient } from '@alien-worlds/broadcast'; -import { Abis, BlockRangeScanner, BlockState, Featured } from '../common'; -import { BlockchainService, Result } from '@alien-worlds/api-core'; -import { Dependencies } from '../common/dependencies'; -import { BootstrapConfig } from './bootstrap.types'; - -/** - * An abstract class representing a Bootstrap dependencies. - * @export - * @abstract - * @class BootstrapDependencies - */ -export abstract class BootstrapDependencies extends Dependencies { - /** - * The broadcast client used for communication. - * @type {BroadcastClient} - */ - public broadcastClient: BroadcastClient; - /** - * The ABIs (Application Binary Interfaces) for contracts. - * @type {Abis} - */ - public abis: Abis; - /** - * The block range scanner for scanning blocks. - * @type {BlockRangeScanner} - */ - public scanner: BlockRangeScanner; - /** - * The featured contract service. - * @type {Featured} - */ - public featured: Featured; - /** - * The block state for maintaining blockchain state. - * @type {BlockState} - */ - public blockState: BlockState; - - /** - * The blockchain service for interacting with the blockchain. - * @type {BlockchainService} - */ - public blockchain: BlockchainService; - - /** - * The blockchain service for interacting with the blockchain. - * @type {BlockchainService} - */ - public featuredContracts: BlockchainService; - - public abstract initialize( - config: BootstrapConfig, - featuredContracts: string[] - ): Promise; -} diff --git a/src/bootstrap/bootstrap.types.ts b/src/bootstrap/bootstrap.types.ts index 3ebb4b0..4c522e6 100644 --- a/src/bootstrap/bootstrap.types.ts +++ b/src/bootstrap/bootstrap.types.ts @@ -1,22 +1,4 @@ -import { AbisServiceConfig } from '../common/abis'; -import { BlockRangeScanConfig } from '../common/block-range-scanner'; -import { FeaturedConfig } from '../common/featured'; -import { Mode } from '../common'; -import { BroadcastConfig } from '@alien-worlds/broadcast'; -import { BlockchainConfig } from '../config'; - -export type BootstrapConfig = { - broadcast: BroadcastConfig; - scanner: BlockRangeScanConfig; - startBlock?: bigint; - endBlock?: bigint; - startFromHead?: boolean; - mode: string; - featured: FeaturedConfig; - abis: AbisServiceConfig; - blockchain: BlockchainConfig; - maxBlockNumber?: number; -}; +import { Mode } from '@alien-worlds/history-tools-common'; export type BootstrapCommandOptions = { scanKey: string; diff --git a/src/bootstrap/bootstrap.utils.ts b/src/bootstrap/bootstrap.utils.ts index a1384c0..ca35980 100644 --- a/src/bootstrap/bootstrap.utils.ts +++ b/src/bootstrap/bootstrap.utils.ts @@ -1,11 +1,17 @@ import { BlockchainService, log, parseToBigInt } from '@alien-worlds/api-core'; -import { BlockRangeData, BootstrapConfig } from './bootstrap.types'; +import { BlockRangeData } from './bootstrap.types'; import { StartBlockHigherThanEndBlockError, UndefinedStartBlockError, EndBlockOutOfRangeError, } from './bootstrap.errors'; -import { BlockRangeScanner, BlockState, Mode, UnknownModeError } from '../common'; +import { + BlockRangeScanner, + BlockState, + BootstrapConfig, + Mode, + UnknownModeError, +} from '@alien-worlds/history-tools-common'; /** * Creates a block range task input based on the provided configuration and mode. diff --git a/src/bootstrap/index.ts b/src/bootstrap/index.ts index bb8d066..5efbed7 100644 --- a/src/bootstrap/index.ts +++ b/src/bootstrap/index.ts @@ -1,5 +1,4 @@ export * from './bootstrap.command'; -export * from './bootstrap.dependencies'; export * from './bootstrap.errors'; export * from './bootstrap.types'; export * from './bootstrap.utils'; diff --git a/src/bootstrap/start-bootstrap.ts b/src/bootstrap/start-bootstrap.ts index 2e9af4a..5887696 100644 --- a/src/bootstrap/start-bootstrap.ts +++ b/src/bootstrap/start-bootstrap.ts @@ -7,19 +7,23 @@ import { createReplayModeBlockRange, createTestModeBlockRange, } from './bootstrap.utils'; -import { BootstrapCommandOptions, BootstrapConfig } from './bootstrap.types'; +import { BootstrapCommandOptions } from './bootstrap.types'; import { NoAbisError } from './bootstrap.errors'; import { InternalBroadcastChannel, InternalBroadcastMessageName, } from '../broadcast/internal-broadcast.enums'; -import { Mode } from '../common/common.enums'; -import { ConfigVars, UnknownObject, log } from '@alien-worlds/api-core'; +import { ConfigVars, log } from '@alien-worlds/api-core'; import { BroadcastMessage } from '@alien-worlds/broadcast'; import { buildBootstrapConfig } from '../config'; import { bootstrapCommand } from './bootstrap.command'; -import { FeaturedUtils } from '../common/featured/featured.utils'; -import { BootstrapDependencies } from './bootstrap.dependencies'; +import { + BootstrapConfig, + BootstrapDependencies, + FeaturedContractDataCriteria, + FeaturedUtils, + Mode, +} from '@alien-worlds/history-tools-common'; /** * The bootstrap function initiates the bootstrap process based on the configuration provided. @@ -37,13 +41,13 @@ import { BootstrapDependencies } from './bootstrap.dependencies'; export const bootstrap = async ( config: BootstrapConfig, dependencies: BootstrapDependencies, - featuredJson: UnknownObject + featuredCriteria: FeaturedContractDataCriteria ): Promise => { const { mode } = config; log(`Bootstrap "${mode}" mode ... [starting]`); - const featuredContracts = FeaturedUtils.readFeaturedContracts(featuredJson); + const featuredContracts = FeaturedUtils.readFeaturedContracts(featuredCriteria); - await dependencies.initialize(config, featuredContracts); + await dependencies.initialize(config, featuredCriteria); const { abis, broadcastClient, blockState, blockchain, featured, scanner } = dependencies; @@ -109,15 +113,15 @@ export const bootstrap = async ( * * @param {string[]} args The command line args for bootstrap. * @param {BootstrapDependencies} dependencies The bootstrap process dependencies. - * @param {UnknownObject} featuredJson + * @param {FeaturedContractDataCriteria} featuredCriteria */ export const startBootstrap = ( args: string[], dependencies: BootstrapDependencies, - featuredJson: UnknownObject + featuredCriteria: FeaturedContractDataCriteria ) => { const vars = new ConfigVars(); const options = bootstrapCommand.parse(args).opts(); - const config = buildBootstrapConfig(vars, options); - bootstrap(config, dependencies, featuredJson).catch(log); + const config = buildBootstrapConfig(vars, dependencies.databaseConfigBuilder, options); + bootstrap(config, dependencies, featuredCriteria).catch(log); }; diff --git a/src/broadcast/__tests__/internal-broadcast.message.unit.test.ts b/src/broadcast/__tests__/internal-broadcast.message.unit.test.ts index 1c24cbd..9ee3961 100644 --- a/src/broadcast/__tests__/internal-broadcast.message.unit.test.ts +++ b/src/broadcast/__tests__/internal-broadcast.message.unit.test.ts @@ -1,5 +1,5 @@ +import { BroadcastTcpMessageType } from '@alien-worlds/history-tools-common'; import { InternalBroadcastMessage } from '../internal-broadcast.message'; -import { BroadcastTcpMessageType } from '@alien-worlds/broadcast'; describe('InternalBroadcastMessage', () => { describe('create', () => { diff --git a/src/broadcast/internal-broadcast.message.ts b/src/broadcast/internal-broadcast.message.ts index 71b8f90..8c5293b 100644 --- a/src/broadcast/internal-broadcast.message.ts +++ b/src/broadcast/internal-broadcast.message.ts @@ -2,7 +2,7 @@ import { BroadcastTcpMessage, BroadcastTcpMessageContent, BroadcastTcpMessageType, -} from '@alien-worlds/broadcast'; +} from '@alien-worlds/history-tools-common'; /** * Represents an internal broadcast message. diff --git a/src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts b/src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts index 1d48a2a..f8710e3 100644 --- a/src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts +++ b/src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts @@ -1,9 +1,9 @@ import { ProcessorBroadcastMessage } from '../processor-broadcast.message'; -import { BroadcastMessage } from '@alien-worlds/broadcast'; import { InternalBroadcastChannel, InternalBroadcastMessageName, } from '../../internal-broadcast.enums'; +import { BroadcastMessage } from '@alien-worlds/history-tools-common'; describe('ProcessorBroadcastMessage', () => { describe('ready', () => { diff --git a/src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts b/src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts index 7622d95..dc5a12d 100644 --- a/src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts +++ b/src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts @@ -3,11 +3,11 @@ import { ReaderBroadcastMessageData, } from '../reader-broadcast.message'; import { BroadcastMessage } from '@alien-worlds/broadcast'; -import { Mode } from '../../../common/common.enums'; import { InternalBroadcastChannel, InternalBroadcastMessageName, } from '../../internal-broadcast.enums'; +import { Mode } from '@alien-worlds/history-tools-common'; describe('ReaderBroadcastMessage', () => { describe('newReplayModeTask', () => { diff --git a/src/broadcast/messages/filter-broadcast.message.ts b/src/broadcast/messages/filter-broadcast.message.ts index 95a3b0b..52fb675 100644 --- a/src/broadcast/messages/filter-broadcast.message.ts +++ b/src/broadcast/messages/filter-broadcast.message.ts @@ -1,4 +1,4 @@ -import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { BroadcastMessage } from '@alien-worlds/history-tools-common'; import { InternalBroadcastChannel, InternalBroadcastMessageName, diff --git a/src/broadcast/messages/processor-broadcast.message.ts b/src/broadcast/messages/processor-broadcast.message.ts index 748e585..ec5d3c1 100644 --- a/src/broadcast/messages/processor-broadcast.message.ts +++ b/src/broadcast/messages/processor-broadcast.message.ts @@ -1,4 +1,4 @@ -import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { BroadcastMessage } from '@alien-worlds/history-tools-common'; import { InternalBroadcastChannel, InternalBroadcastMessageName, diff --git a/src/broadcast/messages/reader-broadcast.message.ts b/src/broadcast/messages/reader-broadcast.message.ts index 13db09c..4a1eb59 100644 --- a/src/broadcast/messages/reader-broadcast.message.ts +++ b/src/broadcast/messages/reader-broadcast.message.ts @@ -1,9 +1,8 @@ -import { BroadcastMessage } from '@alien-worlds/broadcast'; -import { Mode } from '../../common/common.enums'; import { InternalBroadcastChannel, InternalBroadcastMessageName, } from '../internal-broadcast.enums'; +import { BroadcastMessage, Mode } from '@alien-worlds/history-tools-common'; /** * Data structure for the reader broadcast message. diff --git a/src/common/__tests__/common.utils.unit.test.ts b/src/common/__tests__/common.utils.unit.test.ts deleted file mode 100644 index 8818893..0000000 --- a/src/common/__tests__/common.utils.unit.test.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { isSetAbiAction } from '../common.utils'; - -describe('isSetAbiAction', () => { - it('should return true for eosio setabi action', () => { - const result = isSetAbiAction('eosio', 'setabi'); - expect(result).toBe(true); - }); - - it('should return false for other contract or action', () => { - const result = isSetAbiAction('eosio', 'otherAction'); - expect(result).toBe(false); - }); - - it('should return false for other contract and setabi action', () => { - const result = isSetAbiAction('otherContract', 'setabi'); - expect(result).toBe(false); - }); - - it('should return false for other contract and action', () => { - const result = isSetAbiAction('otherContract', 'otherAction'); - expect(result).toBe(false); - }); -}); diff --git a/src/common/abis/__dependencies__/abis.creator.ts b/src/common/abis/__dependencies__/abis.creator.ts deleted file mode 100644 index 10ec590..0000000 --- a/src/common/abis/__dependencies__/abis.creator.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { - MongoConfig, - MongoQueryBuilders, - MongoSource, -} from '@alien-worlds/storage-mongodb'; -import { Abis } from '../abis'; -import { AbiService, log } from '@alien-worlds/api-core'; -import { AbisCollection } from './abis.mongo.collection'; -import { AbisRepositoryImpl } from '../abis.repository-impl'; -import { AbisMongoMapper } from './abis.mongo.mapper'; - -export class AbisCreator { - public static async create( - mongo: MongoSource | MongoConfig, - abiService: AbiService, - contracts?: string[], - setCache?: boolean - ): Promise { - let mongoSource: MongoSource; - - log(` * Abis ... [starting]`); - - if (mongo instanceof MongoSource) { - mongoSource = mongo; - } else { - mongoSource = await MongoSource.create(mongo); - } - const mapper = new AbisMongoMapper(); - const repository = new AbisRepositoryImpl( - new AbisCollection(mongoSource), - mapper, - new MongoQueryBuilders(mapper) - ); - const abis = new Abis(repository, abiService, contracts); - - if (setCache) { - await abis.cacheAbis(); - log(` * Abis cache restored`); - } - - log(` * Abis ... [ready]`); - - return abis; - } -} diff --git a/src/common/abis/__dependencies__/abis.mongo.collection.ts b/src/common/abis/__dependencies__/abis.mongo.collection.ts deleted file mode 100644 index 1fefd80..0000000 --- a/src/common/abis/__dependencies__/abis.mongo.collection.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; -import { ContractEncodedAbiMongoModel } from './abis.mongo.types'; - -/** - * Represents a collection of ABIs (Application Binary Interfaces) stored in a MongoDB collection. - * Extends the MongoCollectionSource class to provide database operations for the ABIs. - */ -export class AbisCollection extends MongoCollectionSource { - /** - * Constructs a new instance of the AbisCollection class. - * - * @param {MongoSource} source - The MongoDB source used for database operations. - */ - constructor(source: MongoSource) { - super(source, 'history_tools.abis', { - indexes: [ - { key: { block_number: 1, hex: 1, contract: 1 }, unique: true, background: true }, - ], - }); - } -} diff --git a/src/common/abis/__dependencies__/abis.mongo.mapper.ts b/src/common/abis/__dependencies__/abis.mongo.mapper.ts deleted file mode 100644 index 8cdeb85..0000000 --- a/src/common/abis/__dependencies__/abis.mongo.mapper.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { MongoDB, MongoMapper } from '@alien-worlds/storage-mongodb'; -import { ContractEncodedAbi } from '@alien-worlds/api-core'; -import { ContractEncodedAbiMongoModel } from './abis.mongo.types'; - -export class AbisMongoMapper extends MongoMapper< - ContractEncodedAbi, - ContractEncodedAbiMongoModel -> { - constructor() { - super(); - this.mappingFromEntity.set('blockNumber', { - key: 'block_number', - mapper: (value: bigint) => MongoDB.Long.fromBigInt(value), - }); - this.mappingFromEntity.set('contract', { - key: 'contract', - mapper: value => value, - }); - this.mappingFromEntity.set('hex', { - key: 'hex', - mapper: value => value, - }); - } -} diff --git a/src/common/abis/__dependencies__/abis.mongo.types.ts b/src/common/abis/__dependencies__/abis.mongo.types.ts deleted file mode 100644 index 9f9c7d3..0000000 --- a/src/common/abis/__dependencies__/abis.mongo.types.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { MongoConfig, MongoDB } from '@alien-worlds/storage-mongodb'; - -export type ContractEncodedAbiMongoModel = { - block_number: MongoDB.Long; - contract: string; - hex: string; -}; - -export type AbisServiceConfig = { - url: string; - limit?: number; - filter?: string; -}; - -export type AbisConfig = { - service: AbisServiceConfig; - mongo: MongoConfig; -}; diff --git a/src/common/abis/__dependencies__/index.ts b/src/common/abis/__dependencies__/index.ts deleted file mode 100644 index 883f09d..0000000 --- a/src/common/abis/__dependencies__/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './abis.creator'; -export * from './abis.mongo.collection'; -export * from './abis.mongo.mapper'; -export * from './abis.mongo.types'; diff --git a/src/common/abis/__tests__/abi.repository-impl.unit.test.ts b/src/common/abis/__tests__/abi.repository-impl.unit.test.ts deleted file mode 100644 index ac0e4c9..0000000 --- a/src/common/abis/__tests__/abi.repository-impl.unit.test.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { AbisRepositoryImpl } from '../abis.repository-impl'; -import { Result, CountParams, ContractEncodedAbi } from '@alien-worlds/api-core'; -import { AbisCache } from '../abis.cache'; - -jest.mock('../abis.cache'); - -const mockDataSource = { - find: jest.fn(), - count: jest.fn(), - aggregate: jest.fn(), - update: jest.fn(), - insert: jest.fn(), - remove: jest.fn(), - startTransaction: jest.fn(), - commitTransaction: jest.fn(), - rollbackTransaction: jest.fn(), -} as any; - -const mockMapper = { - toEntity: jest.fn(), - fromEntity: jest.fn(), - getEntityKeyMapping: jest.fn(), -} as any; - -const mockQueryBuilders = { - buildFindQuery: jest.fn(), - buildCountQuery: jest.fn(), - buildUpdateQuery: jest.fn(), - buildRemoveQuery: jest.fn(), - buildAggregationQuery: jest.fn(), -} as any; - -describe('AbisRepositoryImpl', () => { - let repository: AbisRepositoryImpl; - let mockCache: jest.Mocked; - - beforeEach(() => { - mockCache = new AbisCache() as jest.Mocked; - repository = new AbisRepositoryImpl(mockDataSource, mockMapper, mockQueryBuilders); - }); - - afterEach(() => { - jest.resetAllMocks(); - }); - - describe('getAbis', () => { - it('should return abis from cache if present', async () => { - const mockAbis: ContractEncodedAbi[] = [ - /* mock data here */ - ]; - mockCache.getAbis.mockReturnValue(mockAbis); - - const result = await repository.getAbis({ contracts: ['contract1'] }); - - expect(result).toEqual(Result.withContent(mockAbis)); - expect(mockCache.getAbis).toHaveBeenCalledWith({ contracts: ['contract1'] }); - }); - }); - - describe('getAbi', () => { - it('should return abi from cache if present', async () => { - const mockAbi: ContractEncodedAbi = {} as any; - const blockNumber = BigInt(10); - const contract = 'contract1'; - - mockCache.getAbi.mockReturnValue(mockAbi); - - const result = await repository.getAbi(blockNumber, contract); - - expect(result).toEqual(Result.withContent(mockAbi)); - expect(mockCache.getAbi).toHaveBeenCalledWith(blockNumber, contract); - }); - }); - - describe('insertAbis', () => { - it('should insert abis into the cache and the database', async () => { - const mockAbis: ContractEncodedAbi[] = [ - /* mock data here */ - ]; - const addSpy = jest.spyOn(repository, 'add'); - addSpy.mockResolvedValue(Result.withContent(mockAbis)); - - const result = await repository.insertAbis(mockAbis); - - expect(result).toEqual(Result.withContent(mockAbis.length > 0)); - expect(mockCache.insertAbis).toHaveBeenCalledWith(mockAbis); - expect(addSpy).toHaveBeenCalledWith(mockAbis); - }); - }); - - describe('countAbis', () => { - it('should count abis based on the startBlock and endBlock', async () => { - const countSpy = jest.spyOn(repository, 'count'); - const mockCount = 5; - countSpy.mockResolvedValue(Result.withContent(mockCount)); - - const startBlock = BigInt(10); - const endBlock = BigInt(20); - - const result = await repository.countAbis(startBlock, endBlock); - - expect(result).toEqual(Result.withContent(mockCount)); - expect(countSpy).toHaveBeenCalledWith( - CountParams.create({ where: expect.anything() }) - ); - }); - }); -}); diff --git a/src/common/abis/__tests__/abis.cache.unit.test.ts b/src/common/abis/__tests__/abis.cache.unit.test.ts deleted file mode 100644 index 56e0293..0000000 --- a/src/common/abis/__tests__/abis.cache.unit.test.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { AbisCache } from '../abis.cache'; - -describe('AbisCache', () => { - let abisCache: AbisCache; - - beforeEach(() => { - abisCache = new AbisCache(); - }); - - describe('getAbis', () => { - it('should return an empty array when cache is empty', () => { - const result = abisCache.getAbis({ contracts: ['eosio'] }); - expect(result).toEqual([]); - }); - - it('should return an empty array when contracts array is empty', () => { - abisCache.insertAbis([ - { contract: 'eosio', blockNumber: 1n } as any, - { contract: 'eosio', blockNumber: 2n } as any, - { contract: 'eosio', blockNumber: 3n } as any, - ]); - - const result = abisCache.getAbis({ contracts: [] }); - expect(result).toEqual([]); - }); - - it('should return all ABIs when no startBlock or endBlock is provided', () => { - abisCache.insertAbis([ - { contract: 'eosio', blockNumber: 1n } as any, - { contract: 'eosio', blockNumber: 2n } as any, - { contract: 'eosio', blockNumber: 3n } as any, - ]); - - const result = abisCache.getAbis({ contracts: ['eosio'] }); - expect(result).toEqual([ - { contract: 'eosio', blockNumber: 1n } as any, - { contract: 'eosio', blockNumber: 2n } as any, - { contract: 'eosio', blockNumber: 3n } as any, - ]); - }); - - it('should return matching ABIs within the specified range', () => { - abisCache.insertAbis([ - { contract: 'eosio', blockNumber: 1n } as any, - { contract: 'eosio', blockNumber: 2n } as any, - { contract: 'eosio', blockNumber: 3n } as any, - ]); - - const result = abisCache.getAbis({ - contracts: ['eosio'], - startBlock: 2n, - endBlock: 3n, - }); - expect(result).toEqual([ - { contract: 'eosio', blockNumber: 2n } as any, - { contract: 'eosio', blockNumber: 3n } as any, - ]); - }); - }); - - describe('getAbi', () => { - it('should return null when cache is empty', () => { - const result = abisCache.getAbi(1n, 'eosio'); - expect(result).toBeNull(); - }); - - it('should return the latest ABI that matches the block number', () => { - abisCache.insertAbis([ - { contract: 'eosio', blockNumber: 1n } as any, - { contract: 'eosio', blockNumber: 2n } as any, - { contract: 'eosio', blockNumber: 3n } as any, - ]); - - const result = abisCache.getAbi(2n, 'eosio'); - expect(result).toEqual({ contract: 'eosio', blockNumber: 2n } as any); - }); - - it('should return null when no matching ABI is found', () => { - abisCache.insertAbis([ - { contract: 'eosio', blockNumber: 1n } as any, - { contract: 'eosio', blockNumber: 2n } as any, - { contract: 'eosio', blockNumber: 3n } as any, - ]); - - const result = abisCache.getAbi(4n, 'undefined'); - expect(result).toBeNull(); - }); - }); - - describe('insertAbis', () => { - it('should insert ABIs into the cache', () => { - const abis = [ - { contract: 'eosio', blockNumber: 1n } as any, - { contract: 'eosio', blockNumber: 2n } as any, - { contract: 'eosio', blockNumber: 3n } as any, - ]; - - abisCache.insertAbis(abis); - - const result = abisCache.getAbis({ contracts: ['eosio'] }); - expect(result).toEqual(abis); - }); - }); -}); diff --git a/src/common/abis/__tests__/abis.unit.test.ts b/src/common/abis/__tests__/abis.unit.test.ts deleted file mode 100644 index b6b07f7..0000000 --- a/src/common/abis/__tests__/abis.unit.test.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { Abis } from '../abis'; -import { AbisRepositoryImpl } from '../abis.repository-impl'; -import { Failure, Result, AbiService } from '@alien-worlds/api-core'; -import { AbisServiceNotSetError } from '../abis.errors'; -import { AbiNotFoundError } from '@alien-worlds/block-reader'; - -// Mock dependencies -jest.mock('../abis.repository-impl'); -jest.mock('../abis.service'); - -const mockAbisRepository = { - cacheAbis: jest.fn(), - getAbis: jest.fn(), - getAbi: jest.fn(), - insertAbis: jest.fn(), - countAbis: jest.fn(), -} as any; -const mockAbisService = { fetchAbis: jest.fn() } as any; - -describe('Abis', () => { - let abis: Abis; - - beforeEach(() => { - // Reset mocks and create a new instance of Abis - jest.resetAllMocks(); - abis = new Abis(new mockAbisRepository(), new mockAbisService()); - }); - - describe('getAbis', () => { - it('should return ABIs from repository when available', async () => { - const mockAbis = [ - /* Mocked ABIs */ - ]; - mockAbisRepository.prototype.getAbis.mockResolvedValue( - Result.withContent(mockAbis) - ); - - const result = await abis.getAbis(); - - expect(mockAbisRepository.prototype.getAbis).toHaveBeenCalledTimes(1); - expect(result.isFailure).toBe(false); - expect(result.content).toEqual(mockAbis); - }); - - it('should fetch ABIs when none are available in the repository and fetch option is enabled', async () => { - const mockAbis = [ - /* Mocked ABIs */ - ]; - mockAbisRepository.prototype.getAbis.mockResolvedValue(Result.withContent([])); - abis.fetchAbis = jest.fn().mockResolvedValue(Result.withContent(mockAbis)); - - const result = await abis.getAbis({ fetch: true }); - - expect(mockAbisRepository.prototype.getAbis).toHaveBeenCalledTimes(1); - expect(abis.fetchAbis).toHaveBeenCalledTimes(1); - expect(result.isFailure).toBe(false); - expect(result.content).toEqual(mockAbis); - }); - - // Add more test cases for different scenarios - }); - - describe('getAbi', () => { - it('should return ABI from repository when available', async () => { - const mockAbi = '1234567890'; - mockAbisRepository.prototype.getAbi.mockResolvedValue(Result.withContent(mockAbi)); - - const result = await abis.getAbi(123n, '0x123'); - - expect(mockAbisRepository.prototype.getAbi).toHaveBeenCalledTimes(1); - expect(result.isFailure).toBe(false); - expect(result.content).toEqual(mockAbi); - }); - - it('should fetch ABI when not available in the repository and fetch option is enabled', async () => { - const mockAbi = '1234567890'; - mockAbisRepository.prototype.getAbi.mockResolvedValue( - Result.withFailure(Failure.fromError(new AbiNotFoundError())) - ); - abis.fetchAbis = jest.fn().mockResolvedValue(Result.withContent([mockAbi])); - - const result = await abis.getAbi(123n, '0x123', true); - - expect(mockAbisRepository.prototype.getAbi).toHaveBeenCalledTimes(1); - expect(abis.fetchAbis).toHaveBeenCalledTimes(1); - expect(result.isFailure).toBe(false); - expect(result.content).toEqual(mockAbi); - }); - - // Add more test cases for different scenarios - }); - - describe('storeAbi', () => { - it('should insert the ABI into the repository', async () => { - const blockNumber = 123n; - const contract = '0x123'; - const hex = '0xabcdef'; - - mockAbisRepository.prototype.insertAbis.mockResolvedValue(Result.withContent(true)); - - const result = await abis.storeAbi(blockNumber, contract, hex); - - expect(mockAbisRepository.prototype.insertAbis).toHaveBeenCalledTimes(1); - expect(mockAbisRepository.prototype.insertAbis).toHaveBeenCalledWith([ - expect.objectContaining({ blockNumber, contract, hex }), - ]); - expect(result.isFailure).toBe(false); - expect(result.content).toBe(true); - }); - - // Add more test cases for different scenarios - }); - - describe('fetchAbis', () => { - it('should throw AbisServiceNotSetError when service is not set', async () => { - abis = new Abis(new mockAbisRepository()); // Create instance without AbisService - - await expect(abis.fetchAbis()).rejects.toThrow(AbisServiceNotSetError); - }); - - it('should fetch ABIs using the service', async () => { - const mockAbis = [ - /* Mocked ABIs */ - ]; - const mockContracts = ['0x123', '0x456']; - const mockServiceResponse = Result.withContent(mockAbis); - - abis = new Abis(new mockAbisRepository(), new mockAbisService()); - mockAbisService.prototype.fetchAbis.mockResolvedValue(mockServiceResponse); - - const result = await abis.fetchAbis(mockContracts); - - expect(mockAbisService.prototype.fetchAbis).toHaveBeenCalledTimes( - mockContracts.length - ); - expect(mockAbisService.prototype.fetchAbis).toHaveBeenCalledWith( - expect.any(String) - ); - expect(result.isFailure).toBe(false); - expect(result.content).toEqual(mockAbis); - }); - - // Add more test cases for different scenarios - }); - - describe('cacheAbis', () => { - it('should cache ABIs in the repository', async () => { - const mockContracts = ['0x123', '0x456']; - - mockAbisRepository.prototype.cacheAbis.mockResolvedValue(); - - const result = await abis.cacheAbis(mockContracts); - - expect(mockAbisRepository.prototype.cacheAbis).toHaveBeenCalledTimes(1); - expect(mockAbisRepository.prototype.cacheAbis).toHaveBeenCalledWith(mockContracts); - expect(result.isFailure).toBe(false); - expect(result.content).toBeUndefined(); - }); - - // Add more test cases for different scenarios - }); -}); diff --git a/src/common/abis/abis.cache.ts b/src/common/abis/abis.cache.ts deleted file mode 100644 index 95c2340..0000000 --- a/src/common/abis/abis.cache.ts +++ /dev/null @@ -1,132 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ -import { ContractEncodedAbi } from '@alien-worlds/api-core'; - -/** - * A function that filters ABI entries based on the block number being greater than or equal to the start block. - * @param startBlock - The start block number. - * @param endBlock - The end block number. - * @returns A filter function. - */ -const filterFromStartBlock = - (startBlock: bigint, endBlock: bigint) => (abi: ContractEncodedAbi) => - abi.blockNumber >= startBlock; - -/** - * A function that filters ABI entries based on the block number being less than or equal to the end block. - * @param startBlock - The start block number. - * @param endBlock - The end block number. - * @returns A filter function. - */ -const filterTillEndBlock = - (startBlock: bigint, endBlock: bigint) => (abi: ContractEncodedAbi) => - abi.blockNumber <= endBlock; - -/** - * A function that filters ABI entries based on the block number being within the specified range. - * @param startBlock - The start block number. - * @param endBlock - The end block number. - * @returns A filter function. - */ -const filterInRange = - (startBlock: bigint, endBlock: bigint) => (abi: ContractEncodedAbi) => - abi.blockNumber >= startBlock && abi.blockNumber <= endBlock; - -/** - * Class representing the cache for storing and retrieving contract ABIs. - */ -export class AbisCache { - private cache: Map> = new Map(); - - /** - * Retrieves contract ABIs from the cache based on the specified options. - * @param options - Options for retrieving the contract ABIs. - * @param options.startBlock - The start block number to filter the ABIs. - * @param options.endBlock - The end block number to filter the ABIs. - * @param options.contracts - An array of contract addresses to filter the ABIs. - * @returns An array of matching contract ABIs. - */ - public getAbis(options: { - startBlock?: bigint; - endBlock?: bigint; - contracts?: string[]; - }): ContractEncodedAbi[] { - const { startBlock, endBlock, contracts } = options; - - const filter = - startBlock && endBlock - ? filterInRange - : startBlock - ? filterFromStartBlock - : filterTillEndBlock; - - const abis = []; - if (Array.isArray(contracts)) { - for (const contract of contracts) { - if (contract && this.cache.has(contract)) { - const contractAbis = this.cache.get(contract); - - if (startBlock || endBlock) { - const filtered = Array.from(contractAbis.values()).filter( - filter(startBlock, endBlock) - ); - - abis.push(...filtered); - } else { - abis.push(...contractAbis.values()); - } - } - } - - return abis; - } - // - this.cache.forEach(value => { - if (startBlock || endBlock) { - const filtered = Array.from(value.values()).filter(filter(startBlock, endBlock)); - abis.push(...filtered); - } else { - abis.push(...value.values()); - } - }); - - return abis; - } - - /** - * Retrieves the ABI for the specified block number and contract address. - * @param blockNumber - The block number to find the ABI. - * @param contract - The contract address. - * @returns The matching ABI or null if not found. - */ - public getAbi(blockNumber: bigint, contract: string): ContractEncodedAbi { - if (this.cache.has(contract)) { - const abis = this.cache.get(contract); - const sorted = Array.from(abis.values()).sort((a, b) => - a.blockNumber > b.blockNumber ? -1 : 1 - ); - - for (const abi of sorted) { - if (abi.blockNumber <= blockNumber) { - return abi; - } - } - } - - return null; - } - - /** - * Inserts an array of ABIs into the cache. - * @param abis - An array of ABIs to insert into the cache. - */ - public insertAbis(abis: ContractEncodedAbi[]): void { - abis.forEach(abi => { - let set = this.cache.get(abi.contract); - if (!set) { - set = new Set(); - this.cache.set(abi.contract, set); - } - set.add(abi); - }); - } -} diff --git a/src/common/abis/abis.errors.ts b/src/common/abis/abis.errors.ts deleted file mode 100644 index 27ec8bc..0000000 --- a/src/common/abis/abis.errors.ts +++ /dev/null @@ -1,7 +0,0 @@ -export class AbisServiceNotSetError extends Error { - constructor() { - super( - `AbisService not found. run "setupAbis" with the service config and featured values.` - ); - } -} diff --git a/src/common/abis/abis.repository-impl.ts b/src/common/abis/abis.repository-impl.ts deleted file mode 100644 index 4d12bae..0000000 --- a/src/common/abis/abis.repository-impl.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { - ContractEncodedAbi, - CountParams, - FindParams, - log, - RepositoryImpl, - Result, - Where, -} from '@alien-worlds/api-core'; -import { AbisCache } from './abis.cache'; -import { AbisRepository } from './abis.repository'; - -/** - * Implements the AbisRepository with caching functionality. - * This class manages ContractEncodedAbi entities, providing CRUD operations and additional functionalities such as caching. - * It extends the base RepositoryImpl and implements the AbisRepository interface. - */ -export class AbisRepositoryImpl - extends RepositoryImpl - implements AbisRepository -{ - /** - * Cache instance for caching the ABI objects. - * @private - */ - private cache: AbisCache = new AbisCache(); - - /** - * Cache the ABIs. - * @param {string[]} [contracts] - List of contracts. - */ - public async cacheAbis(contracts?: string[]) { - const abis = await this.getAbis({ contracts }); - if (Array.isArray(abis)) { - this.cache.insertAbis(abis); - } - } - - /** - * Retrieve the ABIs. - * @param options - Filter options for retrieving ABIs. - * @returns Promise that resolves with the Result of an array of ContractEncodedAbi. - */ - public async getAbis(options: { - startBlock?: bigint; - endBlock?: bigint; - contracts?: string[]; - }): Promise> { - try { - const { startBlock, endBlock, contracts } = options || {}; - - const cachedAbis = this.cache.getAbis(options); - - if (cachedAbis.length > 0) { - return Result.withContent(cachedAbis); - } - const where = Where.bind(); - - if (startBlock && endBlock) { - where.props().blockNumber.isGte(startBlock).isLte(endBlock); - } - - if (contracts) { - where.props().contract.isIn(contracts); - } - - return this.find(FindParams.create({ where })); - } catch (error) { - log(error); - return Result.withContent([]); - } - } - - /** - * Retrieve a specific ABI. - * @param blockNumber - The block number. - * @param contract - The contract name. - * @returns Promise that resolves with the Result of ContractEncodedAbi. - */ - public async getAbi( - blockNumber: bigint, - contract: string - ): Promise> { - const cachedAbi = this.cache.getAbi(blockNumber, contract); - - if (cachedAbi) { - return Result.withContent(cachedAbi); - } - - const where = Where.bind(); - where.props().blockNumber.isLte(blockNumber).props().contract.isEq(contract); - const { content, failure } = await this.find( - FindParams.create({ where, limit: 1, sort: { block_number: -1 } }) - ); - - if (content) { - return Result.withContent(content[0]); - } - - if (failure) { - return Result.withFailure(failure); - } - } - - /** - * Inserts an array of ABIs. - * @param abis - The ABIs to be inserted. - * @returns Promise that resolves with the Result of boolean indicating the success of the operation. - */ - public async insertAbis(abis: ContractEncodedAbi[]): Promise> { - this.cache.insertAbis(abis); - const { content, failure } = await this.add(abis); - - if (failure) { - log(failure.error); - return Result.withContent(false); - } - - return Result.withContent(content.length > 0); - } - - /** - * Counts the number of ABIs between the provided block numbers. - * @param startBlock - The start block number. - * @param endBlock - The end block number. - * @returns Promise that resolves with the Result of number indicating the count of ABIs. - */ - public async countAbis( - startBlock?: bigint, - endBlock?: bigint - ): Promise> { - const where = Where.bind(); - - if (typeof startBlock === 'bigint') { - where.props().blockNumber.isGte(startBlock); - } - - if (typeof endBlock === 'bigint') { - where.props().blockNumber.isLte(endBlock); - } - - return this.count(CountParams.create({ where })); - } -} diff --git a/src/common/abis/abis.repository.ts b/src/common/abis/abis.repository.ts deleted file mode 100644 index 49529f5..0000000 --- a/src/common/abis/abis.repository.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { ContractEncodedAbi, Result } from '@alien-worlds/api-core'; - -/** - * This class manages ContractEncodedAbi entities, providing CRUD operations and additional functionalities such as caching. - * It extends the base RepositoryImpl and implements the AbisRepository interface. - */ -export abstract class AbisRepository { - /** - * Cache the ABIs. - * @param {string[]} [contracts] - List of contracts. - */ - public abstract cacheAbis(contracts?: string[]); - - /** - * Retrieve the ABIs. - * @param options - Filter options for retrieving ABIs. - * @returns Promise that resolves with the Result of an array of ContractEncodedAbi. - */ - public abstract getAbis(options: { - startBlock?: bigint; - endBlock?: bigint; - contracts?: string[]; - }): Promise>; - - /** - * Retrieve a specific ABI. - * @param blockNumber - The block number. - * @param contract - The contract name. - * @returns Promise that resolves with the Result of ContractEncodedAbi. - */ - public abstract getAbi( - blockNumber: bigint, - contract: string - ): Promise>; - - /** - * Inserts an array of ABIs. - * @param abis - The ABIs to be inserted. - * @returns Promise that resolves with the Result of boolean indicating the success of the operation. - */ - public abstract insertAbis(abis: ContractEncodedAbi[]): Promise>; - - /** - * Counts the number of ABIs between the provided block numbers. - * @param startBlock - The start block number. - * @param endBlock - The end block number. - * @returns Promise that resolves with the Result of number indicating the count of ABIs. - */ - public abstract countAbis( - startBlock?: bigint, - endBlock?: bigint - ): Promise>; -} diff --git a/src/common/abis/abis.ts b/src/common/abis/abis.ts deleted file mode 100644 index 739fc51..0000000 --- a/src/common/abis/abis.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { AbiService, ContractEncodedAbi, Failure, Result, log } from '@alien-worlds/api-core'; -import { AbisServiceNotSetError } from './abis.errors'; -import { AbisRepository } from './abis.repository'; -import { AbiNotFoundError } from '@alien-worlds/block-reader'; - -/** - * Represents a collection of ABIs (Application Binary Interfaces) for smart contracts. - */ -export class Abis { - private contracts: Set = new Set(); - - /** - * Constructs a new instance of the Abis class. - * - * @param {AbisRepository} repository - The repository for accessing ABIs. - * @param {AbisService} [service] - The service for fetching ABIs. - * @param {FeaturedConfig} [featuredConfig] - The featured configuration containing contract traces and deltas. - * @private - */ - constructor( - private repository: AbisRepository, - private service?: AbiService, - contracts?: string[] - ) { - if (contracts) { - contracts.forEach(contract => { - this.contracts.add(contract); - }); - } - } - - /** - * Retrieves the ABIs (Application Binary Interfaces) for the specified options. - * - * @param {Object} [options] - The options for retrieving the ABIs. - * @param {bigint} [options.startBlock] - The starting block number. - * @param {bigint} [options.endBlock] - The ending block number. - * @param {string[]} [options.contracts] - The contract addresses to filter the ABIs. - * @param {boolean} [options.fetch] - Indicates whether to fetch ABIs if not found in the database. - * @returns {Promise>} A promise that resolves to a Result object containing the retrieved ABIs. - */ - public async getAbis(options?: { - startBlock?: bigint; - endBlock?: bigint; - contracts?: string[]; - fetch?: boolean; - }): Promise> { - const { startBlock, endBlock, contracts, fetch } = options || {}; - - const { content: abis } = await this.repository.getAbis(options); - - if (abis.length === 0 && fetch) { - log( - `No contract ABIs (${startBlock}-${endBlock}) were found in the database. Trying to fetch ABIs...` - ); - return this.fetchAbis(contracts); - } - - return Result.withContent(abis || []); - } - - /** - * Retrieves the ABI (Application Binary Interface) for the specified block number and contract address. - * - * @param {bigint} blockNumber - The block number. - * @param {string} contract - The contract address. - * @param {boolean} [fetch=false] - Indicates whether to fetch the ABI if not found in the database. - * @returns {Promise>} A promise that resolves to a Result object containing the retrieved ABI. - */ - public async getAbi( - blockNumber: bigint, - contract: string, - fetch = false - ): Promise> { - const getAbiResult = await this.repository.getAbi(blockNumber, contract); - - if (fetch && getAbiResult.isFailure) { - const { content: abis, failure } = await this.fetchAbis([contract]); - - if (failure) { - return Result.withFailure(failure); - } - - const abi = abis.reduce((result, abi) => { - if (abi.blockNumber <= blockNumber) { - if (!result || result.blockNumber < abi.blockNumber) { - result = abi; - } - } - return result; - }, null); - - if (abi) { - return Result.withContent(abi); - } - - return Result.withFailure(Failure.fromError(new AbiNotFoundError())); - } - - return getAbiResult; - } - - /** - * Stores the ABI (Application Binary Interface) for the specified block number, contract address, and hex code. - * - * @param {unknown} blockNumber - The block number. - * @param {string} contract - The contract address. - * @param {string} hex - The hex code representing the ABI. - * @returns {Promise>} A promise that resolves to a Result object indicating the success or failure of the operation. - */ - public async storeAbi( - blockNumber: unknown, - contract: string, - hex: string - ): Promise> { - return this.repository.insertAbis([ - ContractEncodedAbi.create(blockNumber, contract, hex), - ]); - } - - /** - * Fetches the ABIs (Application Binary Interfaces) for the specified contracts. - * - * @param {string[]} [contracts] - The contract addresses to fetch ABIs for. - * @returns {Promise>} A promise that resolves to a Result object containing the fetched ABIs. - * @throws {AbisServiceNotSetError} Thrown if the AbisService is not set. - * @private - */ - public async fetchAbis(contracts?: string[]): Promise> { - if (!this.service) { - throw new AbisServiceNotSetError(); - } - - const abis: ContractEncodedAbi[] = []; - try { - const contractsToFetch = contracts || this.contracts; - for (const contract of contractsToFetch) { - const contractAbis = await this.service.fetchAbis(contract); - abis.push(...contractAbis); - } - - if (abis.length > 0) { - await this.repository.insertAbis(abis); - } - } catch (error) { - log(error.message); - } - return Result.withContent(abis); - } - - /** - * Caches the ABIs (Application Binary Interfaces) for the specified contracts. - * - * @param {string[]} [contracts] - The contract addresses to cache ABIs for. - * @returns {Promise>} A promise that resolves to a Result object indicating the success or failure of the operation. - * @private - */ - public async cacheAbis(contracts?: string[]): Promise> { - try { - await this.repository.cacheAbis(contracts); - return Result.withoutContent(); - } catch (error) { - log(error.message); - return Result.withFailure(Failure.fromError(error)); - } - } -} diff --git a/src/common/abis/abis.types.ts b/src/common/abis/abis.types.ts deleted file mode 100644 index 6786ef1..0000000 --- a/src/common/abis/abis.types.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type ContractEncodedAbiJson = { - blockNumber: bigint; - contract: string; - hex: string; -}; diff --git a/src/common/abis/index.ts b/src/common/abis/index.ts deleted file mode 100644 index c354fe3..0000000 --- a/src/common/abis/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * from './__dependencies__'; -export * from './abis.cache'; -export * from './abis.errors'; -export * from './abis.repository-impl'; -export * from './abis.repository'; -export * from './abis'; -export * from './abis.types'; diff --git a/src/common/block-range-scanner/__dependencies__/block-range-scan.mongo.source.ts b/src/common/block-range-scanner/__dependencies__/block-range-scan.mongo.source.ts deleted file mode 100644 index 9a87372..0000000 --- a/src/common/block-range-scanner/__dependencies__/block-range-scan.mongo.source.ts +++ /dev/null @@ -1,195 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-return */ - -import { parseToBigInt } from '@alien-worlds/api-core'; -import { Long } from 'mongodb'; -import { BlockRangeScanModel } from './block-range-scanner.mongo.types'; -import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; - -/** - * Block range scan nodes data source from the mongo database - * @class - */ -export class BlockRangeScanMongoSource extends MongoCollectionSource { - public static Token = 'BLOCK_RANGE_SCAN_MONGO_SOURCE'; - - /** - * @constructor - * @param {MongoSource} mongoSource - */ - constructor(mongoSource: MongoSource) { - super(mongoSource, 'history_tools.block_range_scans'); - } - - private async setCurrentBlockProgress( - document: BlockRangeScanModel, - processedBlock: bigint - ) { - const { _id, is_leaf_node } = document; - const { start, end } = _id; - - if (!is_leaf_node) { - throw new Error( - `(${start.toString()}-${end.toString()}) range has already completed scanning the blockchain.` - ); - } - - if (processedBlock == parseToBigInt(end) - 1n) { - await this.findCompletedParentNode(document); - } else { - await this.collection.updateOne( - { _id }, - { - $set: { - processed_block: Long.fromBigInt(processedBlock), - timestamp: new Date(), - }, - } - ); - } - } - - public async startNextScan(scanKey: string): Promise { - const result = await this.collection.findOneAndUpdate( - { - $and: [ - { is_leaf_node: true }, - { - $or: [ - { timestamp: { $exists: false } }, - /* - The trick to not use the same block range again on another thread/worker - ...Probably this could be handled better. - */ - { timestamp: { $lt: new Date(Date.now() - 1000) } }, - ], - }, - { '_id.scan_key': scanKey }, - ], - }, - { $set: { timestamp: new Date() } }, - { - sort: { timestamp: 1 }, - returnDocument: 'after', - } - ); - - return result.value as BlockRangeScanModel; - } - - public async countScanNodes( - scanKey: string, - startBlock: bigint, - endBlock: bigint - ): Promise { - const options: unknown[] = [{ '_id.scan_key': scanKey }]; - - if (startBlock) { - options.push({ '_id.start': { $gte: Long.fromBigInt(startBlock) } }); - } - - if (endBlock) { - options.push({ '_id.end': { $lte: Long.fromBigInt(endBlock) } }); - } - - const result = await this.collection.countDocuments({ - $and: options, - }); - - return result; - } - - public async removeAll(scanKey: string) { - await this.collection.deleteMany({ '_id.scan_key': scanKey }); - } - - public async hasScanKey( - scanKey: string, - startBlock?: bigint, - endBlock?: bigint - ): Promise { - const options: unknown[] = [{ '_id.scan_key': scanKey }]; - - if (startBlock) { - options.push({ '_id.start': Long.fromBigInt(startBlock) }); - } - - if (endBlock) { - options.push({ '_id.end': Long.fromBigInt(endBlock) }); - } - const dto = await this.collection.findOne({ $and: options }); - - return !!dto; - } - - public async hasUnscannedNodes( - scanKey: string, - startBlock?: bigint, - endBlock?: bigint - ): Promise { - const options: unknown[] = [ - { '_id.scan_key': scanKey }, - { '_id.tree_depth': { $gt: 0 } }, - { is_leaf_node: true }, - ]; - - if (startBlock) { - options.push({ 'parent_id.start': Long.fromBigInt(startBlock) }); - } - - if (endBlock) { - options.push({ 'parent_id.end': Long.fromBigInt(endBlock) }); - } - - const dto = await this.collection.findOne({ $and: options }); - - return !!dto; - } - - public async findRangeForBlockNumber(blockNumber: bigint, scanKey: string) { - const result = this.collection.find( - { - '_id.start': { $lte: Long.fromBigInt(blockNumber) }, - '_id.end': { $gt: Long.fromBigInt(blockNumber) }, - '_id.scan_key': scanKey, - '_id.tree_depth': { $gt: 0 }, - }, - { sort: { '_id.tree_depth': -1 } } - ); - const document = await result.next(); - return document; - } - - public async findCompletedParentNode(document: BlockRangeScanModel) { - const { _id, parent_id } = document; - - if (parent_id) { - await this.collection.deleteOne({ _id }); - // fetch all child nodes with parent id that matches this parent_id - const matchingParentCount = await this.collection.countDocuments({ - parent_id, - }); - if (matchingParentCount == 0) { - const parentDocument: BlockRangeScanModel = await this.collection.findOne({ - _id: parent_id, - }); - await this.findCompletedParentNode(parentDocument); - } - } else if (_id.tree_depth === 0) { - await this.collection.updateOne({ _id }, { $set: { end_timestamp: new Date() } }); - } - } - - public async updateProcessedBlockNumber( - scanKey: string, - blockNumber: bigint - ): Promise { - const range: BlockRangeScanModel = await this.findRangeForBlockNumber( - blockNumber, - scanKey - ); - - if (range) { - return this.setCurrentBlockProgress(range, blockNumber); - } - } -} diff --git a/src/common/block-range-scanner/__dependencies__/block-range-scanner.creator.ts b/src/common/block-range-scanner/__dependencies__/block-range-scanner.creator.ts deleted file mode 100644 index 70b8f2c..0000000 --- a/src/common/block-range-scanner/__dependencies__/block-range-scanner.creator.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { log } from '@alien-worlds/api-core'; -import { BlockRangeScanRepository } from '../block-range-scan.repository'; -import { BlockRangeScanConfig } from '../block-range-scanner.config'; -import { BlockRangeScanMongoSource } from './block-range-scan.mongo.source'; -import { MongoConfig, MongoSource } from '@alien-worlds/storage-mongodb'; -import { BlockRangeScanner } from '../block-range-scanner'; -import { BlockRangeScanMongoMapper } from './block-range-scanner.mongo.mapper'; - -/** - * @class - */ -export class BlockRangeScannerCreator { - public static async create( - mongo: MongoSource | MongoConfig, - config: BlockRangeScanConfig - ): Promise { - let mongoSource: MongoSource; - - log(` * Block Range Scanner ... [starting]`); - - if (mongo instanceof MongoSource) { - mongoSource = mongo; - } else { - mongoSource = await MongoSource.create(mongo); - } - const source = new BlockRangeScanMongoSource(mongoSource); - const mapper = new BlockRangeScanMongoMapper(); - const repository = new BlockRangeScanRepository(source, mapper, config.maxChunkSize); - const scanner: BlockRangeScanner = new BlockRangeScanner(repository); - - log(` * Block Range Scanner ... [ready]`); - return scanner; - } -} diff --git a/src/common/block-range-scanner/__dependencies__/block-range-scanner.mongo.mapper.ts b/src/common/block-range-scanner/__dependencies__/block-range-scanner.mongo.mapper.ts deleted file mode 100644 index 1a2f159..0000000 --- a/src/common/block-range-scanner/__dependencies__/block-range-scanner.mongo.mapper.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { parseToBigInt } from '@alien-worlds/api-core'; -import { MongoDB, MongoMapper } from '@alien-worlds/storage-mongodb'; -import { BlockRangeScan, BlockRangeScanParent } from '../block-range-scan'; -import { - BlockRangeScanIdModel, - BlockRangeScanModel, -} from './block-range-scanner.mongo.types'; - -export class BlockRangeScanMongoMapper extends MongoMapper< - BlockRangeScan, - BlockRangeScanModel -> { - protected toBlockRangeScanParent(model: BlockRangeScanIdModel) { - const { start, end, scan_key, tree_depth } = model; - - return new BlockRangeScanParent( - parseToBigInt(start), - parseToBigInt(end), - scan_key, - tree_depth - ); - } - - protected fromBlockRangeScanParent(entity: BlockRangeScanParent) { - const { start, end, scanKey, treeDepth } = entity; - const model = { - start: MongoDB.Long.fromString(start.toString()), - end: MongoDB.Long.fromString(end.toString()), - scan_key: scanKey, - tree_depth: treeDepth, - }; - - return model; - } - - public toEntity(model: BlockRangeScanModel): BlockRangeScan { - const { - _id: { start, end, scan_key, tree_depth }, - hash, - processed_block, - timestamp, - start_timestamp, - end_timestamp, - parent_id, - is_leaf_node, - } = model; - - const parent = parent_id ? this.toBlockRangeScanParent(parent_id) : null; - - let processedBlock: bigint; - - if (processed_block) { - processedBlock = parseToBigInt(processed_block); - } - - return new BlockRangeScan( - hash, - parseToBigInt(start), - parseToBigInt(end), - scan_key, - tree_depth, - parent, - is_leaf_node, - processedBlock, - timestamp, - start_timestamp, - end_timestamp - ); - } - - public fromEntity(entity: BlockRangeScan): BlockRangeScanModel { - const { start, scanKey, end, treeDepth, hash } = entity; - const model: BlockRangeScanModel = { - _id: { - start: MongoDB.Long.fromString(start.toString()), - end: MongoDB.Long.fromString(end.toString()), - scan_key: scanKey, - tree_depth: treeDepth, - }, - hash, - }; - - if (typeof entity.processedBlock == 'bigint') { - model.processed_block = MongoDB.Long.fromString(entity.processedBlock.toString()); - } - - if (typeof entity.isLeafNode == 'boolean') { - model.is_leaf_node = entity.isLeafNode; - } - - if (entity.parent) { - model.parent_id = this.fromBlockRangeScanParent(entity.parent); - } - - if (entity.timestamp) { - model.timestamp = entity.timestamp; - } - - if (entity.startTimestamp) { - model.start_timestamp = entity.startTimestamp; - } - - if (entity.endTimestamp) { - model.end_timestamp = entity.endTimestamp; - } - - return model; - } -} diff --git a/src/common/block-range-scanner/__dependencies__/block-range-scanner.mongo.types.ts b/src/common/block-range-scanner/__dependencies__/block-range-scanner.mongo.types.ts deleted file mode 100644 index 6bc14e1..0000000 --- a/src/common/block-range-scanner/__dependencies__/block-range-scanner.mongo.types.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { MongoDB } from '@alien-worlds/storage-mongodb'; - -/* - We need to keep tree_depth in _id because if the entire scan will use only one node, - we will not be able to create a child document with the same _id and tree_depth: 1. -*/ - -export type BlockRangeScanIdModel = { - start: MongoDB.Long; - end: MongoDB.Long; - scan_key: string; - tree_depth: number; -}; - -export type BlockRangeScanModel = { - _id: BlockRangeScanIdModel; - hash?: string; - processed_block?: MongoDB.Long; - timestamp?: Date; - start_timestamp?: Date; - end_timestamp?: Date; - is_leaf_node?: boolean; - parent_id?: BlockRangeScanIdModel; -}; diff --git a/src/common/block-range-scanner/__dependencies__/index.ts b/src/common/block-range-scanner/__dependencies__/index.ts deleted file mode 100644 index 2d8ce77..0000000 --- a/src/common/block-range-scanner/__dependencies__/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './block-range-scan.mongo.source'; -export * from './block-range-scanner.creator'; -export * from './block-range-scanner.mongo.mapper'; -export * from './block-range-scanner.mongo.types'; diff --git a/src/common/block-range-scanner/block-range-scan.repository.ts b/src/common/block-range-scanner/block-range-scan.repository.ts deleted file mode 100644 index 21aa099..0000000 --- a/src/common/block-range-scanner/block-range-scan.repository.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { BlockRangeScan } from './block-range-scan'; -import { BlockRangeScanSource } from './block-range-scan.source'; -import { Mapper } from '@alien-worlds/api-core'; - -export type ScanRequest = { - error?: Error; -}; - -export class BlockRangeScanRepository { - constructor( - private readonly source: BlockRangeScanSource, - private readonly mapper: Mapper, - private readonly maxChunkSize: number - ) {} - - public async startNextScan(scanKey: string): Promise { - try { - const document = await this.source.startNextScan(scanKey); - return document ? this.mapper.toEntity(document) : null; - } catch (error) { - return null; - } - } - - public async createScanNodes( - scanKey: string, - startBlock: bigint, - endBlock: bigint - ): Promise { - try { - const { maxChunkSize } = this; - - const rootRange = BlockRangeScan.create(startBlock, endBlock, scanKey, 0); - const rangesToPersist = [rootRange]; - const childRanges = BlockRangeScan.createChildRanges(rootRange, maxChunkSize); - childRanges.forEach(range => rangesToPersist.push(range)); - - const documents = rangesToPersist.map(range => this.mapper.fromEntity(range)); - await this.source.insert(documents); - - return {}; - } catch (error) { - return { error: error as Error }; - } - } - - public async countScanNodes( - scanKey: string, - startBlock: bigint, - endBlock: bigint - ): Promise { - try { - return this.source.countScanNodes(scanKey, startBlock, endBlock); - } catch (error) { - return -1; - } - } - - public async removeAll(scanKey: string): Promise { - await this.source.removeAll(scanKey); - } - - public async hasScanKey( - scanKey: string, - startBlock?: bigint, - endBlock?: bigint - ): Promise { - return this.source.hasScanKey(scanKey, startBlock, endBlock); - } - - public async hasUnscannedNodes( - scanKey: string, - startBlock?: bigint, - endBlock?: bigint - ): Promise { - return this.source.hasUnscannedNodes(scanKey, startBlock, endBlock); - } - - public async updateScannedBlockNumber( - scanKey: string, - blockNumber: bigint - ): Promise { - await this.source.updateProcessedBlockNumber(scanKey, blockNumber); - } -} diff --git a/src/common/block-range-scanner/block-range-scan.source.ts b/src/common/block-range-scanner/block-range-scan.source.ts deleted file mode 100644 index 1b305a8..0000000 --- a/src/common/block-range-scanner/block-range-scan.source.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-return */ - -import { DataSource } from '@alien-worlds/api-core'; - -export abstract class BlockRangeScanSource extends DataSource { - public abstract startNextScan(scanKey: string): Promise; - public abstract countScanNodes( - scanKey: string, - startBlock: bigint, - endBlock: bigint - ): Promise; - public abstract removeAll(scanKey: string); - public abstract hasScanKey( - scanKey: string, - startBlock?: bigint, - endBlock?: bigint - ): Promise; - public abstract hasUnscannedNodes( - scanKey: string, - startBlock?: bigint, - endBlock?: bigint - ): Promise; - public abstract findRangeForBlockNumber(blockNumber: bigint, scanKey: string); - public abstract findCompletedParentNode(document: T); - public abstract updateProcessedBlockNumber( - scanKey: string, - blockNumber: bigint - ): Promise; -} diff --git a/src/common/block-range-scanner/block-range-scan.ts b/src/common/block-range-scanner/block-range-scan.ts deleted file mode 100644 index 77ea5f3..0000000 --- a/src/common/block-range-scanner/block-range-scan.ts +++ /dev/null @@ -1,186 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import crypto from 'crypto'; -import { parseToBigInt, removeUndefinedProperties } from '@alien-worlds/api-core'; -import { serialize } from 'v8'; - -export class BlockRangeScanParent { - /** - * @constructor - * @private - * @param {bigint} start - * @param {bigint} end - * @param {string} scanKey - */ - constructor( - public readonly start: bigint, - public readonly end: bigint, - public readonly scanKey: string, - public readonly treeDepth: number - ) {} - - public static create(start: bigint, end: bigint, scanKey: string, treeDepth: number) { - return new BlockRangeScanParent(start, end, scanKey, treeDepth); - } - - public toJson() { - const { start, end, scanKey, treeDepth } = this; - - return { start, end, scanKey, treeDepth }; - } -} - -/** - * @class - */ -export class BlockRangeScan { - constructor( - public readonly hash: string, - public readonly start: bigint, - public readonly end: bigint, - public readonly scanKey: string, - public readonly treeDepth: number, - public readonly parent?: BlockRangeScanParent, - public isLeafNode?: boolean, - public readonly processedBlock?: bigint, - public readonly timestamp?: Date, - public readonly startTimestamp?: Date, - public readonly endTimestamp?: Date - ) {} - - /** - * Create instance of the BlockRangeScanNode - * - * @static - * @param {bigint | number | string} startBlock - * @param {bigint | number | string} endBlock - * @param {string} scanKey - * @param {number} treeDepth - * @param {BlockRangeScanParent=} parent - * @param {boolean} isLeafNode - * @param {bigint | number | string} processedBlock - * @param {Date} timestamp - * @returns {BlockRangeScan} - */ - public static create( - startBlock: bigint | number | string, - endBlock: bigint | number | string, - scanKey: string, - treeDepth: number, - parent?: BlockRangeScanParent, - isLeafNode?: boolean, - processedBlock?: bigint | number | string, - timestamp?: Date, - startTimestamp?: Date, - endTimestamp?: Date - ): BlockRangeScan { - let currentRangeAsBigInt: bigint; - - if ( - typeof processedBlock === 'bigint' || - typeof processedBlock === 'number' || - typeof processedBlock === 'string' - ) { - currentRangeAsBigInt = parseToBigInt(processedBlock); - } - let started = startTimestamp; - - if (treeDepth === 0 && !startTimestamp) { - started = new Date(); - } - - const buffer = serialize({ - startBlock, - endBlock, - scanKey, - }); - const hash = crypto.createHash('sha1').update(buffer).digest('hex'); - - return new BlockRangeScan( - hash, - parseToBigInt(startBlock), - parseToBigInt(endBlock), - scanKey, - treeDepth, - parent, - isLeafNode, - currentRangeAsBigInt, - timestamp, - started, - endTimestamp - ); - } - - public static createChildRanges( - blockRange: BlockRangeScan, - maxChunkSize: number - ): BlockRangeScan[] { - const max = BigInt(maxChunkSize); - const { start: parentStart, end: parentEnd, scanKey, treeDepth } = blockRange; - - const rangesToPersist: BlockRangeScan[] = []; - let start = parentStart; - - while (start < parentEnd) { - const x = start + max; - const end = x < parentEnd ? x : parentEnd; - const range = BlockRangeScan.create( - start, - end, - scanKey, - treeDepth + 1, - BlockRangeScanParent.create(parentStart, parentEnd, scanKey, treeDepth) - ); - - if (end - start > max) { - const childRanges = BlockRangeScan.createChildRanges(range, maxChunkSize); - childRanges.forEach(range => rangesToPersist.push(range)); - } else { - range.setAsLeafNode(); - } - - rangesToPersist.push(range); - start += max; - } - - return rangesToPersist; - } - - public setAsLeafNode() { - this.isLeafNode = true; - } - - public toJson(): BlockRangeScan { - const { - start, - end, - scanKey, - isLeafNode, - treeDepth, - timestamp, - processedBlock, - startTimestamp, - endTimestamp, - hash, - } = this; - - const json = { - start, - end, - scanKey, - isLeafNode, - treeDepth, - timestamp, - processedBlock, - startTimestamp, - endTimestamp, - parent: null, - hash, - }; - - if (this.parent) { - json.parent = this.parent.toJson(); - } - - return removeUndefinedProperties(json); - } -} diff --git a/src/common/block-range-scanner/block-range-scanner.config.ts b/src/common/block-range-scanner/block-range-scanner.config.ts deleted file mode 100644 index 6883ff5..0000000 --- a/src/common/block-range-scanner/block-range-scanner.config.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type BlockRangeScanConfig = { - maxChunkSize: number; - scanKey: string; -}; diff --git a/src/common/block-range-scanner/block-range-scanner.errors.ts b/src/common/block-range-scanner/block-range-scanner.errors.ts deleted file mode 100644 index 1f36a01..0000000 --- a/src/common/block-range-scanner/block-range-scanner.errors.ts +++ /dev/null @@ -1,26 +0,0 @@ -export class BlockNumberOutOfRangeError extends Error { - constructor(public readonly blockNumber: bigint, public readonly scanKey: string) { - super( - `Block number ${blockNumber} is out of range or is assigned to a different key than "${scanKey}"` - ); - } -} - -export class NoBlockRangeFoundError extends Error { - public static Token = 'NO_BLOCK_RANGE_FOUND_ERROR'; - constructor(public readonly scanKey: string) { - super(`No block range was found for the key "${scanKey}"`); - this.name = NoBlockRangeFoundError.Token; - } -} - -export class DuplicateBlockRangeScanError extends Error { - public static Token = 'DUPLICATE_BLOCK_RANGE_SCAN_ERROR'; - - constructor(scanKey: string, start: bigint, end: bigint) { - super( - `There is already a block range (${start}-${end}) scan entry in the database with the selected key "${scanKey}". Please select a new unique key.` - ); - this.name = DuplicateBlockRangeScanError.Token; - } -} diff --git a/src/common/block-range-scanner/block-range-scanner.ts b/src/common/block-range-scanner/block-range-scanner.ts deleted file mode 100644 index 1bbeaf6..0000000 --- a/src/common/block-range-scanner/block-range-scanner.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { BlockRangeScan } from './block-range-scan'; -import { BlockRangeScanRepository, ScanRequest } from './block-range-scan.repository'; -import { DuplicateBlockRangeScanError } from './block-range-scanner.errors'; - -/** - * @class - */ -export class BlockRangeScanner { - constructor(private blockRangeScanRepository: BlockRangeScanRepository) {} - - public async createScanNodes( - key: string, - startBlock: bigint, - endBlock: bigint - ): Promise { - const hasScanKey = await this.blockRangeScanRepository.hasScanKey( - key, - startBlock, - endBlock - ); - - if (hasScanKey) { - return { error: new DuplicateBlockRangeScanError(key, startBlock, endBlock) }; - } - - return this.blockRangeScanRepository.createScanNodes(key, startBlock, endBlock); - } - - public async getNextScanNode(key: string): Promise { - return this.blockRangeScanRepository.startNextScan(key); - } - - public async hasUnscannedBlocks( - key: string, - startBlock?: bigint, - endBlock?: bigint - ): Promise { - return this.blockRangeScanRepository.hasUnscannedNodes(key, startBlock, endBlock); - } - - public async updateScanProgress(scanKey: string, blockNumber: bigint): Promise { - await this.blockRangeScanRepository.updateScannedBlockNumber(scanKey, blockNumber); - } -} diff --git a/src/common/block-range-scanner/index.ts b/src/common/block-range-scanner/index.ts deleted file mode 100644 index 8111f3b..0000000 --- a/src/common/block-range-scanner/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * from './__dependencies__'; -export * from './block-range-scan.repository'; -export * from './block-range-scan.source'; -export * from './block-range-scan'; -export * from './block-range-scanner.config'; -export * from './block-range-scanner.errors'; -export * from './block-range-scanner'; diff --git a/src/common/block-state/__dependencies__/block-state.creator.ts b/src/common/block-state/__dependencies__/block-state.creator.ts deleted file mode 100644 index e76f187..0000000 --- a/src/common/block-state/__dependencies__/block-state.creator.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { log } from '@alien-worlds/api-core'; -import { - MongoConfig, - MongoQueryBuilders, - MongoSource, -} from '@alien-worlds/storage-mongodb'; -import { BlockState } from '../block-state'; -import { UpdateBlockNumberMongoQueryBuilder } from './query-builders/update-block-number.mongo.query-builder'; -import { BlockStateMongoMapper } from './block-state.mongo.mapper'; -import { BlockStateCollection } from './block-state.mongo.collection'; - -export class BlockStateCreator { - public static async create(mongo: MongoSource | MongoConfig) { - log(` * Block State ... [starting]`); - const mapper = new BlockStateMongoMapper(); - const queryBuilders = new MongoQueryBuilders(); - const updateBlockNumberQueryBuilder = new UpdateBlockNumberMongoQueryBuilder(); - const mongoSource = - mongo instanceof MongoSource ? mongo : await MongoSource.create(mongo); - const collection = new BlockStateCollection(mongoSource); - const state = new BlockState( - collection, - mapper, - queryBuilders, - updateBlockNumberQueryBuilder - ); - - log(` * Block State ... [ready]`); - return state; - } -} diff --git a/src/common/block-state/__dependencies__/block-state.mongo.collection.ts b/src/common/block-state/__dependencies__/block-state.mongo.collection.ts deleted file mode 100644 index ee7e4a9..0000000 --- a/src/common/block-state/__dependencies__/block-state.mongo.collection.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; -import { BlockStateMongoModel } from './block-state.mongo.types'; -/** - * Represents a collection of BlockStateDocument objects in a MongoDB database. - * Extends the MongoCollectionSource class. - * - * @class - * @extends {MongoCollectionSource} - */ -export class BlockStateCollection extends MongoCollectionSource { - /** - * Creates a new instance of the BlockStateCollection class. - * - * @constructor - * @param {MongoSource} source - The MongoSource object used for database operations. - */ - constructor(source: MongoSource) { - super(source, 'history_tools.block_state'); - } -} diff --git a/src/common/block-state/__dependencies__/block-state.mongo.mapper.ts b/src/common/block-state/__dependencies__/block-state.mongo.mapper.ts deleted file mode 100644 index 77da4a6..0000000 --- a/src/common/block-state/__dependencies__/block-state.mongo.mapper.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { BlockStateModel } from '../block-state.types'; -import { BlockStateMongoModel } from './block-state.mongo.types'; -import { MongoDB, MongoMapper } from '@alien-worlds/storage-mongodb'; - -export class BlockStateMongoMapper extends MongoMapper< - BlockStateModel, - BlockStateMongoModel -> { - constructor() { - super(); - this.mappingFromEntity.set('lastModifiedTimestamp', { - key: 'last_modified_timestamp', - mapper: value => value, - }); - this.mappingFromEntity.set('blockNumber', { - key: 'block_number', - mapper: (value: bigint) => MongoDB.Long.fromBigInt(value), - }); - this.mappingFromEntity.set('actions', { - key: 'actions', - mapper: value => value, - }); - this.mappingFromEntity.set('tables', { - key: 'tables', - mapper: value => value, - }); - } -} diff --git a/src/common/block-state/__dependencies__/block-state.mongo.types.ts b/src/common/block-state/__dependencies__/block-state.mongo.types.ts deleted file mode 100644 index 819a6b5..0000000 --- a/src/common/block-state/__dependencies__/block-state.mongo.types.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { MongoDB } from '@alien-worlds/storage-mongodb'; - -export type BlockStateMongoModel = { - _id: MongoDB.ObjectId; - last_modified_timestamp: Date; - block_number: MongoDB.Long; - actions: string[]; - tables: string[]; -}; diff --git a/src/common/block-state/__dependencies__/index.ts b/src/common/block-state/__dependencies__/index.ts deleted file mode 100644 index 05e6e31..0000000 --- a/src/common/block-state/__dependencies__/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from './block-state.creator'; -export * from './block-state.mongo.collection'; -export * from './block-state.mongo.mapper'; -export * from './block-state.mongo.types'; -export * from './query-builders/update-block-number.mongo.query-builder'; diff --git a/src/common/block-state/__dependencies__/query-builders/update-block-number.mongo.query-builder.ts b/src/common/block-state/__dependencies__/query-builders/update-block-number.mongo.query-builder.ts deleted file mode 100644 index 6d2c0a8..0000000 --- a/src/common/block-state/__dependencies__/query-builders/update-block-number.mongo.query-builder.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Query, QueryBuilder } from '@alien-worlds/api-core'; -import { MongoDB } from '@alien-worlds/storage-mongodb'; - -export type UpdateBlockNumberQueryArgs = { blockNumber: bigint }; - -export class UpdateBlockNumberMongoQueryBuilder extends QueryBuilder { - public build(): Query { - const { blockNumber } = this.args as UpdateBlockNumberQueryArgs; - return { - filter: {}, - update: { - $max: { block_number: MongoDB.Long.fromBigInt(blockNumber) }, - $set: { last_modified_timestamp: new Date() }, - }, - }; - } -} diff --git a/src/common/block-state/__tests__/block-state.unit.test.ts b/src/common/block-state/__tests__/block-state.unit.test.ts deleted file mode 100644 index eadfac7..0000000 --- a/src/common/block-state/__tests__/block-state.unit.test.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { RepositoryImpl, Failure, Result } from '@alien-worlds/api-core'; -import { BlockState } from '../block-state'; - -jest.mock('@alien-worlds/api-core'); -jest.mock('./block-state.types'); - -describe('BlockState', () => { - let blockState; - let dataSourceMock; - let mapperMock; - let queryBuildersMock; - let queryBuilderMock; - - beforeEach(() => { - dataSourceMock = { - find: jest.fn(), - count: jest.fn(), - aggregate: jest.fn(), - update: jest.fn(), - insert: jest.fn(), - remove: jest.fn(), - startTransaction: jest.fn(), - commitTransaction: jest.fn(), - rollbackTransaction: jest.fn(), - } as any; - mapperMock = { - toEntity: jest.fn(), - fromEntity: jest.fn(), - getEntityKeyMapping: jest.fn(), - } as any; - queryBuildersMock = { - buildFindQuery: jest.fn(), - buildCountQuery: jest.fn(), - buildUpdateQuery: jest.fn(), - buildRemoveQuery: jest.fn(), - buildAggregationQuery: jest.fn(), - } as any; - queryBuilderMock = { - with: jest.fn(), - build: jest.fn(), - } as any; - - blockState = new BlockState( - dataSourceMock, - mapperMock, - queryBuilderMock, - queryBuilderMock - ); - }); - - afterEach(() => { - jest.resetAllMocks(); - }); - - it('should be a valid instance', () => { - expect(blockState).toBeInstanceOf(BlockState); - expect(blockState).toBeInstanceOf(RepositoryImpl); - }); - - describe('getState()', () => { - it('should return block state data', async () => { - const mockData = { - content: [ - { - lastModifiedTimestamp: new Date(), - actions: [], - tables: [], - blockNumber: 0n, - }, - ], - }; - blockState.find = jest.fn().mockResolvedValue(mockData); - - const result = await blockState.getState(); - - expect(blockState.find).toHaveBeenCalled(); - expect(result).toEqual(Result.withContent(mockData.content[0])); - }); - - it('should handle error properly', async () => { - const mockError = new Error('Database error'); - blockState.find = jest.fn().mockRejectedValue(mockError); - - const result = await blockState.getState(); - - expect(blockState.find).toHaveBeenCalled(); - expect(result).toEqual(Result.withFailure(Failure.fromError(mockError))); - }); - }); - - describe('updateBlockNumber()', () => { - it('should update block number and return true if successful', async () => { - const mockValue = 10n; - const mockData = { - content: { - modifiedCount: 1, - upsertedCount: 0, - }, - }; - blockState.update = jest.fn().mockResolvedValue(mockData); - - const result = await blockState.updateBlockNumber(mockValue); - - expect(blockState.update).toHaveBeenCalledWith( - queryBuilderMock.with({ blockNumber: mockValue }) - ); - expect(result).toEqual(Result.withContent(true)); - }); - - // Write similar tests for getBlockNumber() here... - }); -}); diff --git a/src/common/block-state/block-state.ts b/src/common/block-state/block-state.ts deleted file mode 100644 index 6475cba..0000000 --- a/src/common/block-state/block-state.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { - DataSource, - Failure, - Mapper, - QueryBuilder, - QueryBuilders, - RepositoryImpl, - Result, -} from '@alien-worlds/api-core'; -import { BlockStateModel } from './block-state.types'; - -/** - * A class representing a block state. - */ -export class BlockState extends RepositoryImpl { - /** - * Creates an instance of the BlockState class. - * - * @param {DataSource} source - The data source. - * @param {BlockStateMongoMapper} mapper - The data mapper. - * @param {QueryBuilders} queryBuilders - The query builders. - * @param {QueryBuilder} updateBlockNumberQueryBuilder - The query builder to update block number. - */ - constructor( - source: DataSource, - mapper: Mapper, - queryBuilders: QueryBuilders, - private updateBlockNumberQueryBuilder: QueryBuilder - ) { - super(source, mapper, queryBuilders); - } - - /** - * Fetches the current state of the data source. - * - * @returns {Promise>} - The result of the operation. - */ - public async getState(): Promise> { - try { - const { content: states } = await this.find(); - - if (states) { - const state = states[0]; - const { lastModifiedTimestamp, actions, tables, blockNumber } = state; - - return Result.withContent({ - lastModifiedTimestamp: lastModifiedTimestamp || new Date(), - actions: actions || [], - tables: tables || [], - blockNumber: blockNumber || 0n, - }); - } - - return Result.withContent({ - lastModifiedTimestamp: new Date(), - actions: [], - tables: [], - blockNumber: 0n, - }); - } catch (error) { - return Result.withFailure(Failure.fromError(error)); - } - } - - /** - * Updates the block number in the current state. - * - * @param {bigint} value - The new block number. - * @returns {Promise>} - The result of the operation. - */ - public async updateBlockNumber(value: bigint): Promise> { - this.updateBlockNumberQueryBuilder.with({ blockNumber: value }); - const { content, failure } = await this.update(this.updateBlockNumberQueryBuilder); - - if (failure) { - return Result.withFailure(failure); - } - - return Result.withContent(content.modifiedCount + content.upsertedCount > 0); - } - - /** - * Fetches the current block number from the current state. - * - * @returns {Promise>} - The result of the operation. - */ - public async getBlockNumber(): Promise> { - const { content: states, failure } = await this.find(); - - if (failure) { - return Result.withFailure(failure); - } - - if (states.length > 0) { - const state = states[0]; - - return Result.withContent(state.blockNumber); - } - - return Result.withContent(-1n); - } -} diff --git a/src/common/block-state/block-state.types.ts b/src/common/block-state/block-state.types.ts deleted file mode 100644 index b7c8e9e..0000000 --- a/src/common/block-state/block-state.types.ts +++ /dev/null @@ -1,6 +0,0 @@ -export type BlockStateModel = { - lastModifiedTimestamp: Date; - blockNumber: bigint; - actions: string[]; - tables: string[]; -}; diff --git a/src/common/block-state/index.ts b/src/common/block-state/index.ts deleted file mode 100644 index 3ccdb17..0000000 --- a/src/common/block-state/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './__dependencies__'; -export * from './block-state'; -export * from './block-state.types'; diff --git a/src/common/common.enums.ts b/src/common/common.enums.ts deleted file mode 100644 index af0dcbb..0000000 --- a/src/common/common.enums.ts +++ /dev/null @@ -1,5 +0,0 @@ -export enum Mode { - Default = 'default', - Replay = 'replay', - Test = 'test', -} diff --git a/src/common/common.errors.ts b/src/common/common.errors.ts deleted file mode 100644 index a10bca9..0000000 --- a/src/common/common.errors.ts +++ /dev/null @@ -1,5 +0,0 @@ -export class UnknownModeError extends Error { - constructor(mode: string) { - super(`Unknown mode "${mode}"`); - } -} diff --git a/src/common/common.utils.ts b/src/common/common.utils.ts deleted file mode 100644 index d49f7a8..0000000 --- a/src/common/common.utils.ts +++ /dev/null @@ -1,9 +0,0 @@ -/** - * Checks if a given contract and action represent the 'setabi' action of the 'eosio' contract. - * - * @param {string} contract The contract name. - * @param {string} action The action name. - * @returns {boolean} Returns `true` if the contract and action match the 'eosio' 'setabi' action; otherwise, returns `false`. - */ -export const isSetAbiAction = (contract: string, action: string) => - contract === 'eosio' && action === 'setabi'; diff --git a/src/common/dependencies.ts b/src/common/dependencies.ts deleted file mode 100644 index ea2a5df..0000000 --- a/src/common/dependencies.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { Result } from '@alien-worlds/api-core'; - -/** - * An abstract class representing a Process dependencies. - * - * @abstract - * @class Dependencies - */ -export abstract class Dependencies { - /** - * Initializes and configures the Dependencies instance. - * @abstract - * @returns {Promise} A promise that resolves when the initialization is complete. - */ - public abstract initialize(...args: unknown[]): Promise; -} diff --git a/src/common/featured/__dependencies__/__tests__/featured-contract.mongo.mapper.unit.test.ts b/src/common/featured/__dependencies__/__tests__/featured-contract.mongo.mapper.unit.test.ts deleted file mode 100644 index 3f7c2ff..0000000 --- a/src/common/featured/__dependencies__/__tests__/featured-contract.mongo.mapper.unit.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { FeaturedContractMongoMapper } from '../featured-contract.mongo.mapper'; -import { MongoDB } from '@alien-worlds/storage-mongodb'; -import { FeaturedContract } from '../../featured-contract'; -import { FeaturedContractMongoModel } from '../featured.mongo.types'; - -describe('FeaturedContractMongoMapper', () => { - const fromBigIntMock = jest.fn(); - const parseToBigIntMock = jest.fn(); - - beforeAll(() => { - MongoDB.Long.fromBigInt = fromBigIntMock; - }); - - afterEach(() => { - jest.resetAllMocks(); - }); - - const mapper = new FeaturedContractMongoMapper(); - - it('should convert from entity to model correctly', () => { - const entity = new FeaturedContract('1', 100n, 'account1'); - fromBigIntMock.mockReturnValue('100'); - - const model = mapper.fromEntity(entity); - expect(model).toEqual({ - _id: new MongoDB.ObjectId('1'), - initial_block_number: '100', - account: 'account1', - }); - expect(fromBigIntMock).toHaveBeenCalledWith(100n); - }); - - it('should convert from model to entity correctly', () => { - const model: FeaturedContractMongoModel = { - _id: new MongoDB.ObjectId('1'), - initial_block_number: MongoDB.Long.fromBigInt(100n), - account: 'account1', - }; - parseToBigIntMock.mockReturnValue(100n); - - const entity = mapper.toEntity(model); - expect(entity).toEqual(new FeaturedContract('1', 100n, 'account1')); - expect(parseToBigIntMock).toHaveBeenCalledWith(100n); - }); -}); diff --git a/src/common/featured/__dependencies__/__tests__/featured.mapper.unit.test.ts b/src/common/featured/__dependencies__/__tests__/featured.mapper.unit.test.ts deleted file mode 100644 index 4915199..0000000 --- a/src/common/featured/__dependencies__/__tests__/featured.mapper.unit.test.ts +++ /dev/null @@ -1,233 +0,0 @@ -import { MatcherNotFoundError, PatternMatchError } from '../../featured.errors'; -import { FeaturedMapper } from '../../featured.mapper'; -import { - MatchCriteria, - ProcessorMatchCriteria, - ProcessorMatcher, -} from '../../featured.types'; - -describe('ContractProcessorMapper', () => { - let contractProcessorMapper: FeaturedMapper; - - const mockCriteria: ProcessorMatchCriteria[] = [ - { - contract: 'mockContract1', - action: ['mockAction1', 'mockAction2'], - processor: 'mockProcessor1', - }, - { - contract: ['mockContract2', 'mockContract3'], - action: ['mockAction3'], - processor: 'mockProcessor2', - }, - ]; - const pattern: ProcessorMatchCriteria = { - contract: '', - action: '', - processor: '', - }; - - const mockMatcher = async (criteria: MatchCriteria) => { - return criteria.contract.includes('mockContract1'); - }; - - const mockMatchers: ProcessorMatcher = new Map(); - mockMatchers.set('mockMatcher', mockMatcher); - - beforeEach(() => { - contractProcessorMapper = new FeaturedMapper(mockCriteria, pattern, mockMatchers); - }); - - describe('constructor', () => { - it('should throw error when matcher is not found', () => { - const criteriaWithInvalidMatcher: ProcessorMatchCriteria[] = [ - { - contract: 'mockContract', - action: 'mockAction', - processor: 'mockProcessor', - matcher: 'invalidMatcher', - }, - ]; - const pattern: ProcessorMatchCriteria = { - contract: '', - action: '', - processor: '', - matcher: '', - }; - expect( - () => new FeaturedMapper(criteriaWithInvalidMatcher, pattern, mockMatchers) - ).toThrow(MatcherNotFoundError); - }); - - it('should add all contracts to the contracts set', () => { - const contracts = (contractProcessorMapper as any).contracts; - expect(contracts.size).toEqual(3); - expect(contracts.has('mockContract1')).toBe(true); - expect(contracts.has('mockContract2')).toBe(true); - expect(contracts.has('mockContract3')).toBe(true); - }); - }); - - describe('validateCriteria', () => { - it('should throw error when pattern does not match', () => { - const invalidCriteria: MatchCriteria = { - contract: ['mockContract', 'invalid*Contract'], - }; - expect(() => - (contractProcessorMapper as any).validateCriteria(invalidCriteria) - ).toThrow(PatternMatchError); - }); - - it('should not throw error when pattern matches', () => { - const validCriteria: MatchCriteria = { - contract: ['mockContract', 'anotherMockContract'], - }; - expect(() => - (contractProcessorMapper as any).validateCriteria(validCriteria) - ).not.toThrow(); - }); - }); - - //... - - describe('isMatch', () => { - it('should return true when candidate matches reference', () => { - const ref: ProcessorMatchCriteria = { - contract: ['mockContract1', 'mockContract2'], - action: ['mockAction1', 'mockAction2'], - processor: 'mockProcessor', - }; - const candidate: MatchCriteria = { - contract: 'mockContract1', - action: 'mockAction1', - }; - - expect((contractProcessorMapper as any).isMatch(ref, candidate)).toBe(true); - }); - - it('should return false when candidate does not match reference', () => { - const ref: ProcessorMatchCriteria = { - contract: ['mockContract1', 'mockContract2'], - action: ['mockAction1', 'mockAction2'], - processor: 'mockProcessor', - }; - const candidate: MatchCriteria = { - contract: 'mockContract3', - action: 'mockAction3', - }; - - expect((contractProcessorMapper as any).isMatch(ref, candidate)).toBe(false); - }); - }); - - describe('findProcessorMatchCriteria', () => { - it('should return match criteria when a matcher matches', async () => { - const criteria: MatchCriteria = { - contract: 'mockContract1', - action: 'mockAction1', - }; - - const result = await (contractProcessorMapper as any).findProcessorMatchCriteria( - criteria - ); - - expect(result).toBeDefined(); - expect(result.processor).toEqual('mockProcessor1'); - }); - - it('should return null when no matcher matches', async () => { - const criteria: MatchCriteria = { - contract: 'mockContract3', - action: 'mockAction3', - }; - - const result = await (contractProcessorMapper as any).findProcessorMatchCriteria( - criteria - ); - - expect(result).toBeNull(); - }); - }); - - describe('has', () => { - it('should return true when the match criteria is found', async () => { - const criteria: MatchCriteria = { - contract: 'mockContract1', - action: 'mockAction1', - }; - - expect(await contractProcessorMapper.has(criteria)).toBe(true); - }); - - it('should return false when the match criteria is not found', async () => { - const criteria: MatchCriteria = { - contract: 'mockContract3', - action: 'mockAction3', - }; - - expect(await contractProcessorMapper.has(criteria)).toBe(false); - }); - }); - - describe('get', () => { - it('should return all matched criteria', async () => { - const criteria: MatchCriteria = { - contract: 'mockContract1', - action: 'mockAction1', - }; - - const results = await contractProcessorMapper.get(criteria); - - expect(results.length).toBeGreaterThan(0); - expect(results[0].contract.includes('mockContract1')).toBe(true); - expect(results[0].action.includes('mockAction1')).toBe(true); - }); - - it('should return an empty array when no criteria match', async () => { - const criteria: MatchCriteria = { - contract: 'mockContract3', - action: 'mockAction3', - }; - - const results = await contractProcessorMapper.get(criteria); - - expect(results.length).toBe(0); - }); - }); - - describe('getProcessor', () => { - it('should return the correct processor name', async () => { - const label = 'contract:mockContract1,action:mockAction1'; - const criteria: MatchCriteria = { - contract: 'mockContract1', - action: 'mockAction1', - }; - - const processorName = await contractProcessorMapper.getProcessor(label, criteria); - - expect(processorName).toEqual('mockProcessor1'); - }); - - it('should return an empty string when no processor matches', async () => { - const label = 'contract:mockContract3,action:mockAction3'; - const criteria: MatchCriteria = { - contract: 'mockContract3', - action: 'mockAction3', - }; - - const processorName = await contractProcessorMapper.getProcessor(label, criteria); - - expect(processorName).toEqual(''); - }); - }); - - describe('listContracts', () => { - it('should return all unique contracts', () => { - const contracts = contractProcessorMapper.listContracts(); - - expect(contracts.length).toBeGreaterThan(0); - expect(contracts.includes('mockContract1')).toBe(true); - expect(contracts.includes('mockContract2')).toBe(true); - }); - }); -}); diff --git a/src/common/featured/__dependencies__/featured-contract.mongo.collection.ts b/src/common/featured/__dependencies__/featured-contract.mongo.collection.ts deleted file mode 100644 index c6c9f4d..0000000 --- a/src/common/featured/__dependencies__/featured-contract.mongo.collection.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; -import { FeaturedContractMongoModel } from './featured.mongo.types'; - -/** - * Represents a source for FeaturedContract documents stored in a MongoDB collection. - * Extends the MongoCollectionSource class. - * - * @class - * @extends {MongoCollectionSource} - */ -export class FeaturedContractMongoCollection extends MongoCollectionSource { - /** - * Creates a new instance of the FeaturedContractSource class. - * - * @constructor - * @param {MongoSource} mongoSource - The MongoSource object used for database operations. - */ - constructor(mongoSource: MongoSource) { - super(mongoSource, 'history_tools.featured_contracts', { - indexes: [ - { key: { account: 1 }, background: true }, - { key: { initial_block_number: 1, account: 1 }, unique: true, background: true }, - ], - }); - } -} diff --git a/src/common/featured/__dependencies__/featured-contract.mongo.mapper.ts b/src/common/featured/__dependencies__/featured-contract.mongo.mapper.ts deleted file mode 100644 index 5468637..0000000 --- a/src/common/featured/__dependencies__/featured-contract.mongo.mapper.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { parseToBigInt } from '@alien-worlds/api-core'; -import { MongoDB, MongoMapper } from '@alien-worlds/storage-mongodb'; -import { FeaturedContractMongoModel } from './featured.mongo.types'; -import { FeaturedContract } from '../featured-contract'; - -/** - * Class representing a FeaturedContractMongoMapper - * This class extends the MongoMapper to provide MongoDB-specific mappings for FeaturedContract. - * @class - * @extends {MongoMapper} - * @public - */ -export class FeaturedContractMongoMapper extends MongoMapper< - FeaturedContract, - FeaturedContractMongoModel -> { - /** - * Creates a new instance of FeaturedContractMongoMapper and sets up the mapping from 'initialBlockNumber' and 'account' to MongoDB's 'initial_block_number' and 'account'. - * @constructor - */ - constructor() { - super(); - this.mappingFromEntity.set('initialBlockNumber', { - key: 'initial_block_number', - mapper: (value: bigint) => MongoDB.Long.fromBigInt(value), - }); - this.mappingFromEntity.set('account', { - key: 'account', - mapper: (value: string) => value, - }); - } - - /** - * Converts a MongoDB document of FeaturedContractMongoModel to a FeaturedContract entity. - * - * @param {FeaturedContractMongoModel} model - The MongoDB document. - * @returns {FeaturedContract} The FeaturedContract entity. - */ - public toEntity(model: FeaturedContractMongoModel): FeaturedContract { - const { initial_block_number, _id, account } = model; - - return new FeaturedContract( - _id ? _id.toString() : '', - parseToBigInt(initial_block_number), - account - ); - } -} diff --git a/src/common/featured/__dependencies__/featured.creator.ts b/src/common/featured/__dependencies__/featured.creator.ts deleted file mode 100644 index 4b448f4..0000000 --- a/src/common/featured/__dependencies__/featured.creator.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { - MongoConfig, - MongoQueryBuilders, - MongoSource, -} from '@alien-worlds/storage-mongodb'; -import { Featured } from '../featured'; -import { - RepositoryImpl, - SmartContractService, - UnknownObject, - log, -} from '@alien-worlds/api-core'; -import { FeaturedContractMongoCollection } from './featured-contract.mongo.collection'; -import { FeaturedContractMongoMapper } from './featured-contract.mongo.mapper'; - -export class FeaturedCreator { - public static async create( - mongo: MongoSource | MongoConfig, - smartContractService: SmartContractService, - featuredJson: UnknownObject - ): Promise { - let mongoSource: MongoSource; - - log(` * Featured ... [starting]`); - - if (mongo instanceof MongoSource) { - mongoSource = mongo; - } else { - mongoSource = await MongoSource.create(mongo); - } - const repository = new RepositoryImpl( - new FeaturedContractMongoCollection(mongoSource), - new FeaturedContractMongoMapper(), - new MongoQueryBuilders() - ); - const featured = new Featured(repository, smartContractService, featuredJson); - - log(` * Contract Reader ... [ready]`); - - return featured; - } -} diff --git a/src/common/featured/__dependencies__/featured.mongo.types.ts b/src/common/featured/__dependencies__/featured.mongo.types.ts deleted file mode 100644 index 789cf75..0000000 --- a/src/common/featured/__dependencies__/featured.mongo.types.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { MongoDB } from '@alien-worlds/storage-mongodb'; - -export type FeaturedContractMongoModel = { - _id?: MongoDB.ObjectId; - account?: string; - initial_block_number?: MongoDB.Long; -}; diff --git a/src/common/featured/__dependencies__/index.ts b/src/common/featured/__dependencies__/index.ts deleted file mode 100644 index 6e99e65..0000000 --- a/src/common/featured/__dependencies__/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './featured-contract.mongo.collection'; -export * from './featured-contract.mongo.mapper'; -export * from './featured.creator'; -export * from './featured.mongo.types'; diff --git a/src/common/featured/__tests__/featured.utils.unit.test.ts b/src/common/featured/__tests__/featured.utils.unit.test.ts deleted file mode 100644 index 67da7a2..0000000 --- a/src/common/featured/__tests__/featured.utils.unit.test.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { FeaturedUtils } from '../featured.utils'; - -describe('FeaturedUtils', () => { - describe('readFeaturedContracts', () => { - it('should return an empty array for empty data', () => { - const data = {}; - const result = FeaturedUtils.readFeaturedContracts(data); - expect(result).toEqual([]); - }); - - it('should return an empty array for non-object data', () => { - const data = null; - const result = FeaturedUtils.readFeaturedContracts(data); - expect(result).toEqual([]); - }); - - it('should return an array of unique contracts from the data object', () => { - const data = { - contract: ['ContractA', 'ContractB'], - otherProp: 'some value', - }; - const result = FeaturedUtils.readFeaturedContracts(data); - expect(result).toEqual(['ContractA', 'ContractB']); - }); - - it('should return an array of unique contracts from nested data', () => { - let data = { - prop1: 'value1', - prop2: { - contract: 'ContractC', - prop3: { - contract: ['ContractD', 'ContractE'], - }, - }, - }; - let result = FeaturedUtils.readFeaturedContracts(data); - expect(result).toEqual(['ContractC', 'ContractD', 'ContractE']); - - const nested = { - traces: [ - { - prop1: 'value1', - prop2: { - contract: 'ContractC', - prop3: { - contract: ['ContractD', 'ContractE'], - }, - }, - }, - ], - deltas: [ - { - prop1: 'value1', - prop2: { - contract: 'ContractF', - prop3: { - contract: ['ContractA', 'ContractB'], - }, - }, - }, - ], - }; - result = FeaturedUtils.readFeaturedContracts(nested); - expect(result).toEqual([ - 'ContractC', - 'ContractD', - 'ContractE', - 'ContractF', - 'ContractA', - 'ContractB', - ]); - }); - - it('should ignore non-string values in the contract property', () => { - const data = { - contract: [123, 'ContractF', true, null], - }; - const result = FeaturedUtils.readFeaturedContracts(data); - expect(result).toEqual(['ContractF']); - }); - }); -}); diff --git a/src/common/featured/featured-contract.ts b/src/common/featured/featured-contract.ts deleted file mode 100644 index 0dd6589..0000000 --- a/src/common/featured/featured-contract.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { parseToBigInt } from '@alien-worlds/api-core'; - -/** - * Class representing a FeaturedContract - * @class - * @public - */ -export class FeaturedContract { - /** - * Creates a new instance of the FeaturedContract - * @constructor - * @param {string} id - The ID of the contract - * @param {bigint} initialBlockNumber - The initial block number of the contract - * @param {string} account - The account associated with the contract - */ - constructor( - public id: string, - public initialBlockNumber: bigint, - public account: string - ) {} - - /** - * Creates a new instance of FeaturedContract with a specified account and initial block number, - * ID will be set to empty string by default - * @static - * @public - * @param {string} account - The account associated with the contract - * @param {string | number} initialBlockNumber - The initial block number of the contract - * @returns {FeaturedContract} A new instance of FeaturedContract - */ - public static create(account: string, initialBlockNumber: string | number) { - return new FeaturedContract('', parseToBigInt(initialBlockNumber), account); - } -} diff --git a/src/common/featured/featured.config.ts b/src/common/featured/featured.config.ts deleted file mode 100644 index 68b0ab2..0000000 --- a/src/common/featured/featured.config.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type FeaturedConfig = { - serviceUrl: string; - rpcUrl: string; -}; diff --git a/src/common/featured/featured.errors.ts b/src/common/featured/featured.errors.ts deleted file mode 100644 index 03deae8..0000000 --- a/src/common/featured/featured.errors.ts +++ /dev/null @@ -1,31 +0,0 @@ -export class PatternMatchError extends Error { - constructor(value: string, pattern: string) { - super(`The given value ${value} does not match the pattern ${pattern}`); - } -} - -export class MatcherNotFoundError extends Error { - constructor(label: string) { - super(`No match function assigned to the label: ${label}`); - } -} - -export class UndefinedPatternError extends Error { - constructor() { - super(`No pattern assigned to the criteria`); - } -} - -export class PatternMismatchError extends Error { - constructor() { - super( - `The length of the keys on the label does not match the number of keys in the pattern` - ); - } -} - -export class UnknownContentTypeError extends Error { - constructor(type: string) { - super(`Unknown type: ${type}`); - } -} diff --git a/src/common/featured/featured.mapper.ts b/src/common/featured/featured.mapper.ts deleted file mode 100644 index e60926f..0000000 --- a/src/common/featured/featured.mapper.ts +++ /dev/null @@ -1,271 +0,0 @@ -import { - MatcherNotFoundError, - PatternMatchError, - PatternMismatchError, - UndefinedPatternError, -} from './featured.errors'; -import { - MatchCriteria, - ProcessorMatchCriteria, - ProcessorMatcher, -} from './featured.types'; - -/** - * A mapper class for processing and matching contracts based on given criteria. - * This class can be extended by other processors that require specific match criteria. - */ -export class FeaturedMapper { - /** - * Map of processors matched by a matching function. - */ - protected processorByMatchers: ProcessorMatcher = new Map(); - /** - * Array of match criteria. - */ - protected matchCriteria: ProcessorMatchCriteria[] = []; - /** - * Set of contracts. - */ - protected contracts: Set = new Set(); - - /** - * Creates a new instance of the contract processor mapper. - * @param {ProcessorMatchCriteria[]} criteria - An array of match criteria for the processor. - * @param {MatchCriteriaType} pattern - The criteria pattern. - * @param {ProcessorMatcher} matchers - Optional map of matchers. - */ - constructor( - criteria: ProcessorMatchCriteria[], - protected pattern?: MatchCriteriaType, - matchers?: ProcessorMatcher - ) { - criteria.forEach(current => { - const { processor, matcher, ...rest } = current; - const { contract } = rest as unknown as MatchCriteria; - - if (Array.isArray(contract)) { - contract.forEach(contract => this.contracts.add(contract)); - } else if (typeof contract === 'string') { - this.contracts.add(contract); - } - - if (matcher && !matchers?.has(matcher)) { - throw new MatcherNotFoundError(matcher); - } - - if (matcher && matchers.has(matcher)) { - this.processorByMatchers.set(processor, matchers.get(matcher)); - } else { - this.validateCriteria(rest as MatchCriteriaType); - - if (this.matchCriteria.indexOf(current) === -1) { - this.matchCriteria.push(current); - } - } - }); - } - - /** - * Validates the given match criteria. - * @param criteria - The criteria to validate. - */ - protected validateCriteria(criteria: MatchCriteriaType): void { - const keys = Object.keys(criteria); - - for (const key of keys) { - const values = criteria[key]; - for (const value of values) { - if (/^(\*|[A-Za-z0-9_.]*)$/g.test(value) === false) { - throw new PatternMatchError(value, '^(*|[A-Za-z0-9_.]*)$'); - } - } - } - } - - /** - * Determines if a candidate match criteria meets a reference match criteria. - * @param ref - The reference match criteria. - * @param candidate - The candidate match criteria. - * @returns True if a match is found, false otherwise. - */ - protected isMatch( - ref: ProcessorMatchCriteria, - candidate: MatchCriteriaType - ): boolean { - let matchFound = false; - const keys = Object.keys(candidate); - - for (const key of keys) { - const candidateValues = candidate[key]; - const refValues = ref[key]; - if (Array.isArray(refValues)) { - const values: string[] = Array.isArray(candidateValues) - ? candidateValues - : [candidateValues]; - const contains = values.some(value => refValues.includes(value)); - - if (refValues.includes('*') || contains) { - matchFound = true; - } else { - return false; - } - } - } - - return matchFound; - } - - /** - * Finds a processor match criteria for a given match criteria. - * @param criteria - The criteria to find a match for. - * @returns The matching processor match criteria if found, null otherwise. - */ - protected async findProcessorMatchCriteria( - criteria: MatchCriteriaType - ): Promise> { - const { processorByMatchers } = this; - const entries = Array.from(processorByMatchers.entries()); - - for (const entry of entries) { - const [processor, matcher] = entry; - if (await matcher(criteria)) { - const keys = Object.keys(criteria); - const matchCriteria = {} as MatchCriteriaType; - - for (const key of keys) { - matchCriteria[key] = ['*']; - } - - return { - ...matchCriteria, - processor, - }; - } - } - - return null; - } - - /** - * Checks if the criteria already exist in the array - * - * @param {ProcessorMatchCriteria} criteria - * @param {ProcessorMatchCriteria[]} array - * @returns - */ - protected criteriaExistsInArray( - criteria: ProcessorMatchCriteria, - array: ProcessorMatchCriteria[] - ): boolean { - return array.some(item => - Object.keys(item).every(key => item[key] === criteria[key]) - ); - } - - /** - * Determines if the given match criteria exists in the processor. - * @param criteria - The criteria to check. - * @returns True if the criteria exists, false otherwise. - */ - public async has(criteria: MatchCriteriaType): Promise { - const { matchCriteria, processorByMatchers } = this; - - for (const item of matchCriteria) { - if (this.isMatch(item, criteria)) { - return true; - } - } - - if (processorByMatchers.size > 0) { - const featured = await this.findProcessorMatchCriteria(criteria); - if (featured) { - if (this.criteriaExistsInArray(featured, matchCriteria) === false) { - matchCriteria.push(featured); - } - return true; - } - } - - return false; - } - - /** - * Gets all match criteria in the processor that match the given criteria. - * @param criteria - The criteria to match. - * @returns An array of matching criteria. - */ - public async get(criteria: MatchCriteriaType): Promise { - const { matchCriteria, processorByMatchers } = this; - const result: MatchCriteriaType[] = []; - - for (const item of matchCriteria) { - if (this.isMatch(item, criteria)) { - result.push(item); - } - } - - if (result.length === 0 && processorByMatchers.size > 0) { - const featured = await this.findProcessorMatchCriteria(criteria); - if (featured) { - if (this.criteriaExistsInArray(featured, matchCriteria) === false) { - matchCriteria.push(featured); - } - result.push(featured); - } - } - - return result; - } - - /** - * Gets the processor for the given label and criteria. - * @param label - The label to find a processor for. - * @param pattern - The match criteria pattern. - * @returns The processor if found, empty string otherwise. - */ - public async getProcessor(label: string, pattern?: MatchCriteriaType): Promise { - const { matchCriteria } = this; - const p = pattern || this.pattern; - - if (!p) { - throw new UndefinedPatternError(); - } - - const keys = Object.keys(p); - const parts = label.split(':').map(part => part.split(',')); - - if (parts.length !== keys.length) { - throw new PatternMismatchError(); - } - - const candidate = parts.reduce((result, part, i) => { - result[keys[i]] = part; - return result; - }, p); - - for (const criteriaRef of matchCriteria) { - if (this.isMatch(criteriaRef, candidate)) { - return criteriaRef.processor; - } - } - - const featured = await this.findProcessorMatchCriteria(candidate); - - if (featured) { - if (matchCriteria.indexOf(featured) === -1) { - matchCriteria.push(featured); - } - return featured.processor; - } - - return ''; - } - - /** - * Lists all contracts in the processor. - * @returns An array of contracts. - */ - public listContracts(): string[] { - return Array.from(this.contracts); - } -} diff --git a/src/common/featured/featured.ts b/src/common/featured/featured.ts deleted file mode 100644 index 44aaf71..0000000 --- a/src/common/featured/featured.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { - FindParams, - Repository, - Result, - SmartContractService, - UnknownObject, - Where, -} from '@alien-worlds/api-core'; -import { FeaturedContract } from './featured-contract'; -import { FeaturedUtils } from './featured.utils'; - -export class Featured { - protected cache: Map = new Map(); - protected featuredContracts: string[]; - - constructor( - private repository: Repository, - private smartContractService: SmartContractService, - featuredJson: UnknownObject - ) { - this.featuredContracts = FeaturedUtils.readFeaturedContracts(featuredJson); - } - - /** - * Reads multiple contracts and returns the results as an array of FeaturedContract objects. - * - * @abstract - * @param {string[]} contracts - An array of contract addresses or identifiers. - * @returns {Promise>} A Promise that resolves to an array of FeaturedContract objects. - */ - public async readContracts( - data: string[] | UnknownObject - ): Promise> { - const list: FeaturedContract[] = []; - const contracts = FeaturedUtils.readFeaturedContracts(data); - - for (const contract of contracts) { - if (this.cache.has(contract)) { - list.push(this.cache.get(contract)); - } else { - const { content: contracts, failure } = await this.repository.find( - FindParams.create({ where: new Where().valueOf('account').isEq(contract) }) - ); - - if (failure) { - return Result.withFailure(failure); - } - - if (contracts.length > 0) { - const featuredContract = contracts[0]; - this.cache.set(featuredContract.account, featuredContract); - list.push(featuredContract); - } else { - const fetchResult = await this.smartContractService.getStats(contract); - - if (fetchResult.isFailure) { - return Result.withFailure(fetchResult.failure); - } - if (fetchResult.content) { - const featuredContract = FeaturedContract.create( - fetchResult.content.account_name, - fetchResult.content.first_block_num - ); - this.cache.set(featuredContract.account, featuredContract); - this.repository.add([featuredContract]); - list.push(featuredContract); - } - } - } - } - - return Result.withContent(list); - } - - public isFeatured(contract: string): boolean { - return this.featuredContracts.includes(contract); - } -} diff --git a/src/common/featured/featured.types.ts b/src/common/featured/featured.types.ts deleted file mode 100644 index b6eb15e..0000000 --- a/src/common/featured/featured.types.ts +++ /dev/null @@ -1,44 +0,0 @@ -export type FeaturedContractModel = { - account: string; - initialBlockNumber: bigint; -}; - -export type FetchContractResponse = { - account: string; - block_num: string | number; -}; - -export type CriteriaValue = string | string[]; - -export type MatchCriteria = { - contract: CriteriaValue; - [key: string]: CriteriaValue; -}; - -export type ProcessorMatchCriteria = { - matcher?: string; - processor: string; -} & MatchCriteriaType; - -export type ProcessorMatcher = Map< - string, - MatchFunction ->; - -export type MatchFunction = ( - criteria: MatchCriteriaType -) => Promise; - -export type ContractTraceMatchCriteria = MatchCriteria & { - shipTraceMessageName: string[]; - shipActionTraceMessageName: string[]; - action: string[]; -}; - -export type ContractDeltaMatchCriteria = MatchCriteria & { - shipDeltaMessageName: string[]; - name: string[]; - code: string[]; - scope: string[]; - table: string[]; -}; diff --git a/src/common/featured/featured.utils.ts b/src/common/featured/featured.utils.ts deleted file mode 100644 index d763659..0000000 --- a/src/common/featured/featured.utils.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { UnknownObject } from '@alien-worlds/api-core'; - -export class FeaturedUtils { - public static readFeaturedContracts(data: UnknownObject | unknown[]): string[] { - const contracts = new Set(); - if (!data) { - return []; - } - Object.keys(data).forEach(key => { - const value = data[key]; - - if (key === 'contract' && Array.isArray(value)) { - value.forEach(contract => { - if (typeof contract === 'string') { - contracts.add(contract); - } - }); - } else if (key === 'contract' && typeof value === 'string') { - if (typeof value === 'string') { - contracts.add(value); - } - } else if (Array.isArray(value) || typeof value === 'object') { - const result = this.readFeaturedContracts(value); - result.forEach(contract => { - contracts.add(contract); - }); - } - }); - return Array.from(contracts); - } -} diff --git a/src/common/featured/index.ts b/src/common/featured/index.ts deleted file mode 100644 index 6db5938..0000000 --- a/src/common/featured/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export * from './__dependencies__'; -export * from './featured-contract'; -export * from './featured'; -export * from './featured.config'; -export * from './featured.errors'; -export * from './featured.mapper'; -export * from './featured.types'; -export * from './featured.utils'; diff --git a/src/common/index.ts b/src/common/index.ts deleted file mode 100644 index 0a5a655..0000000 --- a/src/common/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -export * from './abis'; -export * from './block-range-scanner'; -export * from './block-state'; -export * from './featured'; -export * from './processor-task-queue'; -export * from './types'; -export * from './unprocessed-block-queue'; -export * from './common.enums'; -export * from './common.errors'; -export * from './common.utils'; -export * from './dependencies'; diff --git a/src/common/processor-task-queue/__dependencies__/index.ts b/src/common/processor-task-queue/__dependencies__/index.ts deleted file mode 100644 index e1fe167..0000000 --- a/src/common/processor-task-queue/__dependencies__/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './processor-task.mongo.collection'; -export * from './processor-task.mongo.mapper'; -export * from './processor-task.mongo.types'; -export * from './unsuccessful-processor-task.mongo.collection'; diff --git a/src/common/processor-task-queue/__dependencies__/processor-task.mongo.collection.ts b/src/common/processor-task-queue/__dependencies__/processor-task.mongo.collection.ts deleted file mode 100644 index c9a3671..0000000 --- a/src/common/processor-task-queue/__dependencies__/processor-task.mongo.collection.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { - MongoCollectionSource, - MongoDB, - MongoSource, -} from '@alien-worlds/storage-mongodb'; -import { ProcessorTaskQueueConfig } from '../processor-task-queue.config'; -import { DataSourceError } from '@alien-worlds/api-core'; -import { ProcessorTaskSource } from '../processor-task.source'; -import { ProcessorTaskMongoModel } from './processor-task.mongo.types'; - -export class ProcessorTaskMongoCollection - extends MongoCollectionSource - implements ProcessorTaskSource -{ - private transactionOptions: MongoDB.TransactionOptions; - - constructor(mongoSource: MongoSource, private config: ProcessorTaskQueueConfig) { - super(mongoSource, 'history_tools.processor_tasks', { - indexes: [ - { - key: { block_number: 1 }, - background: true, - }, - { - key: { timestamp: 1, block_number: 1 }, - background: true, - }, - { - key: { mode: 1, type: 1 }, - background: true, - }, - { - key: { short_id: 1, mode: 1, type: 1 }, - background: true, - }, - { - key: { short_id: 1, mode: 1, block_number: 1, hash: 1 }, - unique: true, - background: true, - }, - ], - }); - } - - public async nextTask(mode?: string): Promise { - try { - const filter = mode ? { mode } : {}; - const result = await this.collection.findOneAndDelete(filter); - - return result.value; - } catch (error) { - throw DataSourceError.createError(error); - } - } -} diff --git a/src/common/processor-task-queue/__dependencies__/processor-task.mongo.mapper.ts b/src/common/processor-task-queue/__dependencies__/processor-task.mongo.mapper.ts deleted file mode 100644 index 6dda024..0000000 --- a/src/common/processor-task-queue/__dependencies__/processor-task.mongo.mapper.ts +++ /dev/null @@ -1,92 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ -import { MongoDB, MongoMapper } from '@alien-worlds/storage-mongodb'; - -import { - PropertyMapping, - parseToBigInt, - removeUndefinedProperties, -} from '@alien-worlds/api-core'; -import { ProcessorTask } from '../processor-task'; -import { ProcessorTaskMongoModel } from './processor-task.mongo.types'; - -export class ProcessorTaskMongoMapper< - EntityType = ProcessorTask, - ModelType = ProcessorTaskMongoModel -> extends MongoMapper { - public getEntityKeyMapping(key: string): PropertyMapping { - throw new Error('Method not implemented.'); - } - - public fromEntity(entity: ProcessorTask): ProcessorTaskMongoModel { - const { - id, - abi, - shortId, - label, - timestamp, - type, - mode, - content, - hash, - blockNumber, - isFork, - blockTimestamp, - error, - } = entity; - - const document: ProcessorTaskMongoModel = { - abi, - short_id: shortId, - label, - timestamp, - type, - mode, - content: new MongoDB.Binary(content), - hash, - block_number: MongoDB.Long.fromBigInt(blockNumber), - block_timestamp: blockTimestamp, - is_fork: isFork, - error, - }; - - if (id) { - document._id = new MongoDB.ObjectId(id); - } - - return removeUndefinedProperties(document); - } - - public toEntity(model: ProcessorTaskMongoModel): ProcessorTask { - const { - abi, - short_id, - label, - content, - timestamp, - hash, - type, - mode, - _id, - block_number, - block_timestamp, - error, - is_fork, - } = model; - - return new ProcessorTask( - _id ? _id.toString() : '', - abi, - short_id, - label, - timestamp, - type, - mode, - content.buffer, - hash, - parseToBigInt(block_number), - block_timestamp, - is_fork, - error - ); - } -} diff --git a/src/common/processor-task-queue/__dependencies__/processor-task.mongo.types.ts b/src/common/processor-task-queue/__dependencies__/processor-task.mongo.types.ts deleted file mode 100644 index 0d5ecd3..0000000 --- a/src/common/processor-task-queue/__dependencies__/processor-task.mongo.types.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { MongoDB } from '@alien-worlds/storage-mongodb'; -import { ProcessorTaskError } from '../processor-task.types'; - -export type ProcessorTaskMongoModel = { - _id?: MongoDB.ObjectId; - abi?: string; - is_fork?: boolean; - short_id?: string; - label?: string; - timestamp?: Date; - type?: string; - mode?: string; - content?: MongoDB.Binary; - hash?: string; - block_number?: MongoDB.Long; - block_timestamp?: Date; - error?: ProcessorTaskError; -}; diff --git a/src/common/processor-task-queue/__dependencies__/unsuccessful-processor-task.mongo.collection.ts b/src/common/processor-task-queue/__dependencies__/unsuccessful-processor-task.mongo.collection.ts deleted file mode 100644 index f36fe62..0000000 --- a/src/common/processor-task-queue/__dependencies__/unsuccessful-processor-task.mongo.collection.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; -import { ProcessorTaskMongoModel } from './processor-task.mongo.types'; - -export class UnsuccessfulProcessorTaskSource extends MongoCollectionSource { - constructor(mongoSource: MongoSource) { - super(mongoSource, 'history_tools.unsuccessful_processor_tasks', { - indexes: [ - { - key: { block_number: 1 }, - background: true, - }, - { - key: { timestamp: 1, block_number: 1 }, - background: true, - }, - { - key: { mode: 1, type: 1 }, - background: true, - }, - { - key: { short_id: 1, mode: 1, type: 1 }, - background: true, - }, - { - key: { short_id: 1, mode: 1, block_number: 1, hash: 1 }, - unique: true, - background: true, - }, - ], - }); - } -} diff --git a/src/common/processor-task-queue/index.ts b/src/common/processor-task-queue/index.ts deleted file mode 100644 index 65caaec..0000000 --- a/src/common/processor-task-queue/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export * from './__dependencies__'; -export * from './processor-task-queue'; -export * from './processor-task-queue.config'; -export * from './processor-task.enums'; -export * from './processor-task.errors'; -export * from './processor-task.source'; -export * from './processor-task'; -export * from './processor-task.types'; diff --git a/src/common/processor-task-queue/processor-task-queue.config.ts b/src/common/processor-task-queue/processor-task-queue.config.ts deleted file mode 100644 index 997b65e..0000000 --- a/src/common/processor-task-queue/processor-task-queue.config.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type ProcessorTaskQueueConfig = { - interval?: number; - [key: string]: unknown; -}; diff --git a/src/common/processor-task-queue/processor-task-queue.ts b/src/common/processor-task-queue/processor-task-queue.ts deleted file mode 100644 index f3922ab..0000000 --- a/src/common/processor-task-queue/processor-task-queue.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { DataSource, DataSourceError, Mapper, log } from '@alien-worlds/api-core'; -import { ErrorJson } from '@alien-worlds/workers'; -import { ProcessorTaskSource } from './processor-task.source'; -import { ProcessorTask } from './processor-task'; -import { ProcessorTaskModel } from './processor-task.types'; - -export class ProcessorTaskQueue { - private constructor( - protected source: ProcessorTaskSource, - protected mapper: Mapper, - protected unsuccessfulSource: DataSource, - protected onlyAdd = false - ) {} - - public async nextTask(mode?: string): Promise { - // TODO: temporary solution - testing session options - if (this.onlyAdd) { - log(`Operation not allowed, queue created with option onlyAdd`); - return; - } - - try { - const dto = await this.source.nextTask(mode); - if (dto) { - return this.mapper.toEntity(dto); - } - return null; - } catch (error) { - log(`Could not get next task due to: ${error.message}`); - return null; - } - } - - public async addTasks(tasks: ProcessorTask[], unsuccessful?: boolean): Promise { - const source = unsuccessful ? this.unsuccessfulSource : this.source; - try { - const dtos = tasks.map(task => this.mapper.fromEntity(task)); - await source.insert(dtos); - } catch (error) { - const { error: concernError } = error; - const concernErrorMessage = (concernError)?.message || ''; - log(`Could not add tasks due to: ${error.message}. ${concernErrorMessage}`); - } - } - - public async stashUnsuccessfulTask( - task: ProcessorTask, - error: ErrorJson | Error - ): Promise { - try { - const { message, stack } = error; - const document: ProcessorTaskModel = this.mapper.fromEntity( - task - ) as ProcessorTaskModel; - document.error = { message, stack }; - - await this.unsuccessfulSource.insert([document]); - } catch (sourceError) { - log(`Could not stash failed task due to: ${error.message}`); - } - } -} diff --git a/src/common/processor-task-queue/processor-task.enums.ts b/src/common/processor-task-queue/processor-task.enums.ts deleted file mode 100644 index 13de22b..0000000 --- a/src/common/processor-task-queue/processor-task.enums.ts +++ /dev/null @@ -1,4 +0,0 @@ -export enum ProcessorTaskType { - Trace = 'trace', - Delta = 'delta', -} diff --git a/src/common/processor-task-queue/processor-task.errors.ts b/src/common/processor-task-queue/processor-task.errors.ts deleted file mode 100644 index c115c9f..0000000 --- a/src/common/processor-task-queue/processor-task.errors.ts +++ /dev/null @@ -1,5 +0,0 @@ -export class UnknownProcessorTypeError extends Error { - constructor(type: string) { - super(`Unknown processor type: ${type}`); - } -} diff --git a/src/common/processor-task-queue/processor-task.source.ts b/src/common/processor-task-queue/processor-task.source.ts deleted file mode 100644 index f4f416d..0000000 --- a/src/common/processor-task-queue/processor-task.source.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { DataSource } from '@alien-worlds/api-core'; - -export abstract class ProcessorTaskSource extends DataSource { - public abstract nextTask(mode?: string): Promise; -} diff --git a/src/common/processor-task-queue/processor-task.ts b/src/common/processor-task-queue/processor-task.ts deleted file mode 100644 index 87c0f5d..0000000 --- a/src/common/processor-task-queue/processor-task.ts +++ /dev/null @@ -1,116 +0,0 @@ -import crypto from 'crypto'; -import { serialize } from 'v8'; -import { DeltaProcessorContentModel, ProcessorTaskError } from './processor-task.types'; -import { ActionTrace } from '../types'; -import { ProcessorTaskType } from './processor-task.enums'; - -export class ProcessorTask { - public static createActionProcessorTask( - abi: string, - mode: string, - shipTraceMessageName: string, - shipMessageName: string, - transactionId: string, - actionTrace: ActionTrace, - blockNumber: bigint, - blockTimestamp: Date, - isFork: boolean - ) { - const { - act: { account, name, data }, - receipt, - } = actionTrace; - - const buffer = serialize({ - transaction_id: transactionId, - action_trace: actionTrace, - block_num: blockNumber.toString(), - block_timestamp: blockTimestamp, - }); - const hashBuffer = serialize({ - account, - name, - data, - transactionId, - blockNumber, - blockTimestamp, - receipt, - }); - - const hash = crypto.createHash('sha1').update(hashBuffer).digest('hex'); - const shortId = `${account}:${name}`; - const label = `${shipTraceMessageName}:${shipMessageName}:${shortId}`; - - return new ProcessorTask( - null, - abi, - shortId, - label, - null, - ProcessorTaskType.Trace, - mode, - buffer, - hash, - blockNumber, - blockTimestamp, - isFork - ); - } - - public static createDeltaProcessorTask( - abi: string, - mode: string, - type: string, - name: string, - code: string, - scope: string, - table: string, - blockNumber: bigint, - blockTimestamp: Date, - data: Uint8Array, - isFork: boolean - ) { - const content: DeltaProcessorContentModel = { - ship_delta_message_name: type, - name, - row_data: data, - block_num: blockNumber, - block_timestamp: blockTimestamp, - }; - const buffer = serialize(content); - const hash = crypto.createHash('sha1').update(buffer).digest('hex'); - const shortId = `${code}:${scope}:${table}`; - const label = `${type}:${name}:${shortId}`; - - return new ProcessorTask( - null, - abi, - shortId, - label, - null, - ProcessorTaskType.Delta, - mode, - buffer, - hash, - blockNumber, - blockTimestamp, - isFork - ); - } - - constructor( - public readonly id: string, - public readonly abi: string, - public readonly shortId: string, - public readonly label: string, - public readonly timestamp: Date, - public readonly type: string, - public readonly mode: string, - public readonly content: Buffer, - public readonly hash: string, - public readonly blockNumber: bigint, - public readonly blockTimestamp: Date, - public readonly isFork: boolean, - public readonly error?: ProcessorTaskError - ) {} -} diff --git a/src/common/processor-task-queue/processor-task.types.ts b/src/common/processor-task-queue/processor-task.types.ts deleted file mode 100644 index 7588721..0000000 --- a/src/common/processor-task-queue/processor-task.types.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { ActionTrace } from "../types"; - -export type ProcessorTaskError = { - message: string; - stack: string; -}; - -export type ProcessorTaskModel = { - id: string; - isFork: string; - abi: string; - path: string; - label: string; - timestamp: Date; - type: string; - mode: string; - content: Buffer; - hash: string; - error?: ProcessorTaskError; -}; - -export type DeltaProcessorContentModel = { - ship_delta_message_name: string; - name: string; - block_num: bigint; - block_timestamp: Date; - row_data: Uint8Array; -}; - -export type ActionProcessorContentModel = { - ship_trace_message_name: string; - transaction_id: string; - block_num: bigint; - block_timestamp: Date; - action_trace: ActionTrace; -}; diff --git a/src/common/types/action-trace.types.ts b/src/common/types/action-trace.types.ts deleted file mode 100644 index b4a80be..0000000 --- a/src/common/types/action-trace.types.ts +++ /dev/null @@ -1,45 +0,0 @@ -export type AuthSequence = { - account: string; - sequence: string; -}; - -export type Receipt = { - receiver: string; - act_digest: string; - global_sequence: string; - recv_sequence: string; - auth_sequence: AuthSequence[]; - code_sequence: number; - abi_sequence: number; -}; - -export type ReceiptByName = [string, Receipt]; - -export type ActAuth = { - actor: string; - permission: string; -}; - -export type Act = { - account: string; - name: string; - authorization: ActAuth; - data: Uint8Array; -}; - -export type ActionTrace = { - ship_message_name?: string; - action_ordinal?: number; - creator_action_ordinal?: number; - receipt?: ReceiptByName; - receiver?: string; - act?: Act; - context_free?: boolean; - elapsed?: string; - console?: string; - account_ram_deltas?: unknown[]; - except?: unknown; - error_code?: string | number; -}; - -export type ActionTraceByName = [string, ActionTrace]; diff --git a/src/common/types/block.types.ts b/src/common/types/block.types.ts deleted file mode 100644 index 664ab9e..0000000 --- a/src/common/types/block.types.ts +++ /dev/null @@ -1,20 +0,0 @@ -export type BlockNumberWithId = { - block_num?: unknown; - block_id?: string; -}; - -export type BlockModel< - BlockType = Uint8Array, - TracesType = Uint8Array, - DeltasType = Uint8Array -> = { - head?: BlockNumberWithId; - this_block?: BlockNumberWithId; - last_irreversible?: BlockNumberWithId; - prev_block?: BlockNumberWithId; - block?: BlockType; - traces?: TracesType; - deltas?: DeltasType; - abi_version?: string; - [key: string]: unknown; -}; diff --git a/src/common/types/delta.types.ts b/src/common/types/delta.types.ts deleted file mode 100644 index 1beb87c..0000000 --- a/src/common/types/delta.types.ts +++ /dev/null @@ -1,11 +0,0 @@ -export type DeltaRow = { - present?: number; - data?: Uint8Array; -}; - -export type Delta = { - name?: string; - rows?: DeltaRow[]; -}; - -export type DeltaByName = [string, Delta]; diff --git a/src/common/types/index.ts b/src/common/types/index.ts deleted file mode 100644 index 9279e97..0000000 --- a/src/common/types/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from './action-trace.types'; -export * from './signed-block.types'; -export * from './delta.types'; -export * from './trace.types'; -export * from './transaction.types'; diff --git a/src/common/types/signed-block.types.ts b/src/common/types/signed-block.types.ts deleted file mode 100644 index acfe49e..0000000 --- a/src/common/types/signed-block.types.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Transaction } from './transaction.types'; - -export type SignedBlock = { - timestamp: string; - producer: string; - confirmed: number; - previous: string; - transaction_mroot: string; - action_mroot: string; - schedule_version: number; - new_producers: unknown; - header_extensions: unknown[]; - producer_signature: string; - transactions: Transaction[]; -}; diff --git a/src/common/types/trace.types.ts b/src/common/types/trace.types.ts deleted file mode 100644 index 5bb8ddd..0000000 --- a/src/common/types/trace.types.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { ActionTraceByName } from './action-trace.types'; - -export type Partial = { - expiration: string; - ref_block_num: number; - ref_block_prefix: number; - max_net_usage_words: number; - max_cpu_usage_ms: number; - delay_sec: number; - transaction_extensions: unknown[]; - signatures: unknown[]; - context_free_data: unknown[]; -}; - -export type PartialByType = [string, Partial]; - -export type Trace = { - id?: string; - status?: number; - cpu_usage_us?: number; - net_usage_words?: number; - elapsed?: string; - net_usage?: string; - scheduled?: boolean; - action_traces?: ActionTraceByName[]; - account_ram_delta?: unknown; - except?: unknown; - error_code?: number | string; - failed_dtrx_trace?: unknown; - partial?: PartialByType; -}; - -export type TraceByName = [string, Trace]; diff --git a/src/common/types/transaction.types.ts b/src/common/types/transaction.types.ts deleted file mode 100644 index f3022ef..0000000 --- a/src/common/types/transaction.types.ts +++ /dev/null @@ -1,14 +0,0 @@ -export type PackedTrx = { - signatures: string[]; - compression: number; - packed_context_free_data: unknown; - packed_trx: Uint8Array; -}; -export type TrxByName = [string, PackedTrx | string]; - -export type Transaction = { - status: number; - cpu_usage_us: number; - net_usage_words: number; - trx: TrxByName; -}; diff --git a/src/common/unprocessed-block-queue/__dependencies__/index.ts b/src/common/unprocessed-block-queue/__dependencies__/index.ts deleted file mode 100644 index a9e5218..0000000 --- a/src/common/unprocessed-block-queue/__dependencies__/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './unprocessed-block-queue.mongo.collection'; -export * from './unprocessed-block-queue.mongo.mapper'; -export * from './unprocessed-block-queue.mongo.types'; diff --git a/src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.collection.ts b/src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.collection.ts deleted file mode 100644 index 1a16c53..0000000 --- a/src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.collection.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { DataSourceError } from '@alien-worlds/api-core'; -import { MongoCollectionSource, MongoSource } from '@alien-worlds/storage-mongodb'; -import { BlockMongoModel } from './unprocessed-block-queue.mongo.types'; -import { UnprocessedBlockSource } from '../unprocessed-block-queue.source'; - -export class UnprocessedBlockMongoCollection - extends MongoCollectionSource - implements UnprocessedBlockSource -{ - constructor(mongoSource: MongoSource) { - super(mongoSource, 'history_tools.unprocessed_blocks', { - indexes: [ - { - key: { 'this_block.block_num': 1 }, - unique: true, - background: true, - }, - ], - }); - } - - public async next(): Promise { - try { - const result = await this.collection.findOneAndDelete({}); - return result.value; - } catch (error) { - throw DataSourceError.createError(error); - } - } - - public async bytesSize(): Promise { - const stats = await this.collection.stats(); - return stats.size; - } -} diff --git a/src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.mapper.ts b/src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.mapper.ts deleted file mode 100644 index 79e1a32..0000000 --- a/src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.mapper.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { Block } from '@alien-worlds/block-reader'; -import { parseToBigInt } from '@alien-worlds/api-core'; -import { MongoDB, MongoMapper } from '@alien-worlds/storage-mongodb'; -import { BlockMongoModel } from './unprocessed-block-queue.mongo.types'; - -export class UnprocessedBlockMongoMapper extends MongoMapper { - public toEntity(model: BlockMongoModel): Block { - const { block, traces, deltas, abi_version } = model; - let head; - let thisBlock; - let prevBlock; - let lastIrreversible; - if (model.head) { - head = { - blockNumber: parseToBigInt(model.head.block_num), - blockId: model.head.block_id, - }; - } - if (model.this_block) { - thisBlock = { - blockNumber: parseToBigInt(model.this_block.block_num), - blockId: model.this_block.block_id, - }; - } - if (model.prev_block) { - prevBlock = { - blockNumber: parseToBigInt(model.prev_block.block_num), - blockId: model.prev_block.block_id, - }; - } - if (model.last_irreversible) { - lastIrreversible = { - blockNumber: parseToBigInt(model.last_irreversible.block_num), - blockId: model.last_irreversible.block_id, - }; - } - return new Block( - head, - lastIrreversible, - prevBlock, - thisBlock, - block.buffer, - traces.buffer, - deltas.buffer, - abi_version - ); - } - public fromEntity(entity: Block): BlockMongoModel { - const { - head, - thisBlock, - prevBlock, - lastIrreversible, - block, - traces, - deltas, - id, - abiVersion, - } = entity; - const document: BlockMongoModel = { - head: { - block_id: head.blockId, - block_num: MongoDB.Long.fromBigInt(head.blockNumber), - }, - this_block: { - block_id: thisBlock.blockId, - block_num: MongoDB.Long.fromBigInt(thisBlock.blockNumber), - }, - prev_block: { - block_id: prevBlock.blockId, - block_num: MongoDB.Long.fromBigInt(prevBlock.blockNumber), - }, - last_irreversible: { - block_id: lastIrreversible.blockId, - block_num: MongoDB.Long.fromBigInt(lastIrreversible.blockNumber), - }, - block: new MongoDB.Binary(block), - traces: new MongoDB.Binary(traces), - deltas: new MongoDB.Binary(deltas), - }; - if (abiVersion) { - document.abi_version = abiVersion; - } - if (id) { - document._id = new MongoDB.ObjectId(id); - } - return document; - } -} diff --git a/src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.types.ts b/src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.types.ts deleted file mode 100644 index 18d6b03..0000000 --- a/src/common/unprocessed-block-queue/__dependencies__/unprocessed-block-queue.mongo.types.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { MongoDB } from '@alien-worlds/storage-mongodb'; - -export type BlockNumberWithIdDocument = { - block_num?: MongoDB.Long; - block_id?: string; -}; - -export type BlockMongoModel = { - _id?: MongoDB.ObjectId; - head?: BlockNumberWithIdDocument; - this_block?: BlockNumberWithIdDocument; - last_irreversible?: BlockNumberWithIdDocument; - prev_block?: BlockNumberWithIdDocument; - block?: MongoDB.Binary; - traces?: MongoDB.Binary; - deltas?: MongoDB.Binary; - abi_version?: string; - [key: string]: unknown; -}; diff --git a/src/common/unprocessed-block-queue/index.ts b/src/common/unprocessed-block-queue/index.ts deleted file mode 100644 index e3f5122..0000000 --- a/src/common/unprocessed-block-queue/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './__dependencies__'; -export * from './unprocessed-block-queue'; -export * from './unprocessed-block-queue.source'; -export * from './unprocessed-block-queue.errors'; diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.errors.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.errors.ts deleted file mode 100644 index 09d8512..0000000 --- a/src/common/unprocessed-block-queue/unprocessed-block-queue.errors.ts +++ /dev/null @@ -1,14 +0,0 @@ -export class BlockNotFoundError extends Error {} -export class DuplicateBlocksError extends Error { - constructor() { - super(`Some blocks in the specified range are already in the database.`); - this.name = 'DuplicateBlocksError'; - } -} - -export class UnprocessedBlocksOverloadError extends Error { - constructor(min: bigint, max: bigint) { - super(`Blocks ${min}-${max} were read before the overload occurred.`); - this.name = 'UnprocessedBlocksOverloadError'; - } -} diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.source.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.source.ts deleted file mode 100644 index b631975..0000000 --- a/src/common/unprocessed-block-queue/unprocessed-block-queue.source.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { DataSource } from '@alien-worlds/api-core'; - -export abstract class UnprocessedBlockSource extends DataSource { - public abstract next(): Promise; - public abstract bytesSize(): Promise; -} diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.ts deleted file mode 100644 index 26e2658..0000000 --- a/src/common/unprocessed-block-queue/unprocessed-block-queue.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { - DataSourceError, - Failure, - log, - Mapper, - parseToBigInt, - Result, -} from '@alien-worlds/api-core'; -import { - BlockNotFoundError, - DuplicateBlocksError, - UnprocessedBlocksOverloadError, -} from './unprocessed-block-queue.errors'; -import { Block } from '@alien-worlds/block-reader'; -import { UnprocessedBlockSource } from './unprocessed-block-queue.source'; -import { BlockModel } from '../types/block.types'; - -export abstract class UnprocessedBlockQueueReader { - public abstract next(): Promise>; -} - -export class UnprocessedBlockQueue - implements UnprocessedBlockQueueReader -{ - protected cache: Block[] = []; - protected overloadHandler: (size: number) => void; - protected beforeSendBatchHandler: () => void; - protected afterSendBatchHandler: () => void; - - constructor( - protected collection: UnprocessedBlockSource, - protected mapper: Mapper, - protected maxBytesSize: number, - protected batchSize: number - ) {} - - private async sendBatch() { - const addedBlockNumbers = []; - this.beforeSendBatchHandler(); - const documnets = this.cache.map(block => this.mapper.fromEntity(block)); - const result = await this.collection.insert(documnets); - result.forEach(document => { - addedBlockNumbers.push(parseToBigInt(document.this_block.block_num)); - }); - this.cache = []; - - if (this.maxBytesSize > 0 && this.overloadHandler) { - const sorted = addedBlockNumbers.sort(); - const min = sorted[0]; - const max = sorted.reverse()[0]; - - const currentSize = await this.collection.bytesSize(); - if (currentSize >= this.maxBytesSize) { - this.overloadHandler(currentSize); - throw new UnprocessedBlocksOverloadError(min, max); - } - } - - this.afterSendBatchHandler(); - - return addedBlockNumbers; - } - - public async getBytesSize(): Promise> { - try { - const currentSize = await this.collection.bytesSize(); - return Result.withContent(currentSize); - } catch (error) { - return Result.withFailure(Failure.fromError(error)); - } - } - - public async add(block: Block, isLast = false): Promise> { - try { - let addedBlockNumbers: bigint[] = []; - - if (this.cache.length < this.batchSize) { - this.cache.push(block); - } - - if (this.cache.length === this.batchSize || isLast) { - addedBlockNumbers = await this.sendBatch(); - } - - return Result.withContent(addedBlockNumbers); - } catch (error) { - // it is important to clear the cache in case of errors - this.cache = []; - - if (error instanceof DataSourceError && error.isDuplicateError) { - this.afterSendBatchHandler(); - return Result.withFailure(Failure.fromError(new DuplicateBlocksError())); - } - return Result.withFailure(Failure.fromError(error)); - } - } - - public async next(): Promise> { - try { - const document = await this.collection.next(); - if (document) { - if (this.maxBytesSize > -1 && this.afterSendBatchHandler) { - if ((await this.collection.count()) === 0 && this.afterSendBatchHandler) { - this.afterSendBatchHandler(); - } - } - - return Result.withContent(this.mapper.toEntity(document)); - } - return Result.withFailure(Failure.fromError(new BlockNotFoundError())); - } catch (error) { - log(`Could not get next task due to: ${error.message}`); - return Result.withFailure(Failure.fromError(error)); - } - } - - public async getMax(): Promise> { - try { - const documents = await this.collection.aggregate({ - pipeline: [{ $sort: { 'this_block.block_num': -1 } }, { $limit: 1 }], - }); - if (documents.length > 0) { - return Result.withContent(this.mapper.toEntity(documents[0])); - } - return Result.withFailure(Failure.fromError(new BlockNotFoundError())); - } catch (error) { - log(`Could not get block with highest block number due to: ${error.message}`); - return Result.withFailure(Failure.fromError(error)); - } - } - - public afterSendBatch(handler: () => void): void { - this.afterSendBatchHandler = handler; - } - - public beforeSendBatch(handler: () => void): void { - this.beforeSendBatchHandler = handler; - } - - public onOverload(handler: (size: number) => void): void { - this.overloadHandler = handler; - } -} diff --git a/src/config/config.types.ts b/src/config/config.types.ts index 1f15e1a..405a1e2 100644 --- a/src/config/config.types.ts +++ b/src/config/config.types.ts @@ -1,8 +1,10 @@ +import { + BootstrapConfig, + FilterConfig, + ProcessorConfig, + ReaderConfig, +} from '@alien-worlds/history-tools-common'; import { ApiConfig } from '../api'; -import { BootstrapConfig } from '../bootstrap'; -import { ReaderConfig } from '../reader'; -import { FilterConfig } from '../filter'; -import { ProcessorConfig } from '../processor'; export type BlockchainConfig = { endpoint: string; @@ -16,4 +18,4 @@ export type HistoryToolsConfig = { filter: FilterConfig; processor: ProcessorConfig; [key: string]: unknown; -}; \ No newline at end of file +}; diff --git a/src/config/index.ts b/src/config/index.ts index f35f513..3debd86 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -1,24 +1,29 @@ -import { ProcessorTaskQueueConfig } from '../common/processor-task-queue/processor-task-queue.config'; -import { ConfigVars, parseToBigInt } from '@alien-worlds/api-core'; -import { ApiConfig } from '../api'; -import { BootstrapConfig, BootstrapCommandOptions } from '../bootstrap'; -import { ReaderConfig, ReaderCommandOptions } from '../reader'; -import { FilterConfig, FilterCommandOptions } from '../filter'; -import { ProcessorConfig, ProcessorCommandOptions } from '../processor'; -import { BlockchainConfig, HistoryToolsConfig } from './config.types'; import { AbisConfig, AbisServiceConfig, BlockRangeScanConfig, - ContractTraceMatchCriteria, - ContractDeltaMatchCriteria, + BlockReaderConfig, + BlockchainConfig, + BootstrapConfig, + ConfigVars, FeaturedConfig, - ProcessorMatchCriteria, -} from '../common'; -import { buildMongoConfig } from '@alien-worlds/storage-mongodb'; -import { buildBroadcastConfig } from '@alien-worlds/broadcast'; -import { BlockReaderConfig } from '@alien-worlds/block-reader'; -import { WorkersConfig } from '@alien-worlds/workers'; + FeaturedContractDataCriteria, + FilterConfig, + ProcessorConfig, + ProcessorTaskQueueConfig, + ReaderConfig, + UnknownObject, + UnprocessedBlockQueueConfig, + WorkersConfig, + buildBroadcastConfig, + parseToBigInt, +} from '@alien-worlds/history-tools-common'; +import { FilterCommandOptions } from '../filter'; +import { ApiConfig } from '../api'; +import { BootstrapCommandOptions } from '../bootstrap'; +import { ReaderCommandOptions } from '../reader'; +import { ProcessorCommandOptions } from '../processor'; +import { HistoryToolsConfig } from './config.types'; export * from './config.types'; @@ -46,9 +51,12 @@ export const buildAbisServiceConfig = (vars: ConfigVars): AbisServiceConfig => ( filter: vars.getStringEnv('ABIS_SERVICE_FILTER'), }); -export const buildAbisConfig = (vars: ConfigVars): AbisConfig => ({ +export const buildAbisConfig = ( + vars: ConfigVars, + databaseConfigBuilder: (vars: ConfigVars, ...args: unknown[]) => UnknownObject +): AbisConfig => ({ service: buildAbisServiceConfig(vars), - mongo: buildMongoConfig(vars), + database: databaseConfigBuilder(vars), }); export const buildBlockReaderConfig = (vars: ConfigVars): BlockReaderConfig => ({ @@ -87,15 +95,28 @@ export const buildProcessorTaskQueueConfig = ( interval: vars.getNumberEnv('PROCESSOR_TASK_QUEUE_CHECK_INTERVAL'), }); -export const buildApiConfig = (vars: ConfigVars): ApiConfig => ({ +export const buildUnprocessedBlockQueueConfig = ( + vars: ConfigVars +): UnprocessedBlockQueueConfig => ({ + maxBytesSize: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_MAX_BYTES_SIZE'), + sizeCheckInterval: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_SIZE_CHECK_INTERVAL'), + batchSize: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_BATCH_SIZE'), +}); + +export const buildApiConfig = ( + vars: ConfigVars, + databaseConfigBuilder: (vars: ConfigVars, ...args: unknown[]) => UnknownObject +): ApiConfig => ({ port: vars.getNumberEnv('API_PORT'), - mongo: buildMongoConfig(vars), + database: databaseConfigBuilder(vars), }); export const buildBootstrapConfig = ( vars: ConfigVars, + databaseConfigBuilder: (vars: ConfigVars, ...args: unknown[]) => UnknownObject, options?: BootstrapCommandOptions ): BootstrapConfig => ({ + database: databaseConfigBuilder(vars), broadcast: buildBroadcastConfig(vars), blockchain: buildBlockchainConfig(vars), featured: buildFeaturedConfig(vars), @@ -118,18 +139,15 @@ export const buildBootstrapConfig = ( export const buildReaderConfig = ( vars: ConfigVars, + databaseConfigBuilder: (vars: ConfigVars, ...args: unknown[]) => UnknownObject, options?: ReaderCommandOptions ): ReaderConfig => ({ - mongo: buildMongoConfig(vars), + database: databaseConfigBuilder(vars), broadcast: buildBroadcastConfig(vars), scanner: buildBlockRangeScanConfig(vars, options?.scanKey), mode: options?.mode || vars.getStringEnv('MODE'), maxBlockNumber: vars.getNumberEnv('MAX_BLOCK_NUMBER'), - blockQueueMaxBytesSize: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_MAX_BYTES_SIZE'), - blockQueueSizeCheckInterval: vars.getNumberEnv( - 'UNPROCESSED_BLOCK_QUEUE_SIZE_CHECK_INTERVAL' - ), - blockQueueBatchSize: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_BATCH_SIZE'), + unprocessedBlockQueue: buildUnprocessedBlockQueueConfig(vars), workers: buildReaderWorkersConfig(vars, options?.threads), blockReader: buildBlockReaderConfig(vars), startBlock: options?.startBlock ? parseToBigInt(options?.startBlock) : null, @@ -138,42 +156,43 @@ export const buildReaderConfig = ( export const buildFilterConfig = ( vars: ConfigVars, + databaseConfigBuilder: (vars: ConfigVars, ...args: unknown[]) => UnknownObject, options?: FilterCommandOptions ): FilterConfig => ({ mode: options?.mode || vars.getStringEnv('MODE'), broadcast: buildBroadcastConfig(vars), workers: buildFilterWorkersConfig(vars, options), - abis: buildAbisConfig(vars), + abis: buildAbisConfig(vars, databaseConfigBuilder), featured: buildFeaturedConfig(vars), - mongo: buildMongoConfig(vars), - queue: buildProcessorTaskQueueConfig(vars), + database: databaseConfigBuilder(vars), + processorTaskQueue: buildProcessorTaskQueueConfig(vars), + unprocessedBlockQueue: buildUnprocessedBlockQueueConfig(vars), }); export const buildProcessorConfig = ( vars: ConfigVars, + databaseConfigBuilder: (vars: ConfigVars, ...args: unknown[]) => UnknownObject, options?: ProcessorCommandOptions ): ProcessorConfig => ({ broadcast: buildBroadcastConfig(vars), workers: buildProcessorWorkersConfig(vars, options?.threads), featured: buildFeaturedConfig(vars), - mongo: buildMongoConfig(vars), + database: databaseConfigBuilder(vars), queue: buildProcessorTaskQueueConfig(vars), }); export const buildHistoryToolsConfig = ( vars: ConfigVars, - featured?: { - traces: [ProcessorMatchCriteria]; - deltas: [ProcessorMatchCriteria]; - }, + databaseConfigBuilder: (vars: ConfigVars, ...args: unknown[]) => UnknownObject, + featuredCriteria?: FeaturedContractDataCriteria, bootstrapOptions?: BootstrapCommandOptions, readerOptions?: ReaderCommandOptions, filterOptions?: FilterCommandOptions, processorOptions?: ProcessorCommandOptions ): HistoryToolsConfig => ({ - api: buildApiConfig(vars), - bootstrap: buildBootstrapConfig(vars, bootstrapOptions), - reader: buildReaderConfig(vars, readerOptions), - filter: buildFilterConfig(vars, filterOptions), - processor: buildProcessorConfig(vars, processorOptions), + api: buildApiConfig(vars, databaseConfigBuilder), + bootstrap: buildBootstrapConfig(vars, databaseConfigBuilder, bootstrapOptions), + reader: buildReaderConfig(vars, databaseConfigBuilder, readerOptions), + filter: buildFilterConfig(vars, databaseConfigBuilder, filterOptions), + processor: buildProcessorConfig(vars, databaseConfigBuilder, processorOptions), }); diff --git a/src/filter/filter.consts.ts b/src/filter/filter.consts.ts index d4e164b..56baf83 100644 --- a/src/filter/filter.consts.ts +++ b/src/filter/filter.consts.ts @@ -1,2 +1 @@ export const filterWorkerLoaderPath = `${__dirname}/filter.worker-loader`; -export const filterWorkerLoaderDependenciesPath = `${__dirname}/filter.worker-loader.dependencies`; diff --git a/src/filter/filter.dependencies.ts b/src/filter/filter.dependencies.ts deleted file mode 100644 index 591a59c..0000000 --- a/src/filter/filter.dependencies.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Result, Serializer } from '@alien-worlds/api-core'; -import { Dependencies } from '../common/dependencies'; -import { FilterAddons, FilterConfig } from './filter.types'; -import { BroadcastClient } from '@alien-worlds/broadcast'; -import { WorkerPool } from '@alien-worlds/workers'; -import { UnprocessedBlockQueue } from '../common'; - -/** - * An abstract class representing a Filter dependencies. - * @class FilterDependencies - */ -export abstract class FilterDependencies extends Dependencies { - /** - * The broadcast client used for communication. - * @type {BroadcastClient} - */ - public broadcastClient: BroadcastClient; - - public workerPool: WorkerPool; - public unprocessedBlockQueue: UnprocessedBlockQueue; - public serializer: Serializer; - - public abstract initialize( - config: FilterConfig, - addons?: FilterAddons - ): Promise; -} diff --git a/src/filter/filter.runner.ts b/src/filter/filter.runner.ts index 643a2ab..280991d 100644 --- a/src/filter/filter.runner.ts +++ b/src/filter/filter.runner.ts @@ -1,7 +1,10 @@ import { log } from '@alien-worlds/api-core'; import { BlockJson } from '@alien-worlds/block-reader'; +import { + BlockNotFoundError, + UnprocessedBlockQueueReader, +} from '@alien-worlds/history-tools-common'; import { WorkerMessage, WorkerPool } from '@alien-worlds/workers'; -import { BlockNotFoundError, UnprocessedBlockQueueReader } from '../common'; export class FilterRunner { private interval: NodeJS.Timeout; diff --git a/src/filter/filter.types.ts b/src/filter/filter.types.ts index 097f7fa..b55ad0e 100644 --- a/src/filter/filter.types.ts +++ b/src/filter/filter.types.ts @@ -1,8 +1,4 @@ -import { WorkersConfig } from '@alien-worlds/workers'; -import { BroadcastConfig } from '@alien-worlds/broadcast'; -import { AbisConfig, FeaturedConfig, ProcessorTaskQueueConfig } from '../common'; -import { MongoConfig } from '@alien-worlds/storage-mongodb'; -import { UnknownObject } from '@alien-worlds/api-core'; +import { FilterConfig, UnknownObject } from '@alien-worlds/history-tools-common'; export type FilterSharedData = { config: FilterConfig; @@ -14,17 +10,6 @@ export type FilterCommandOptions = { mode: string; }; -export type FilterConfig = { - mode: string; - broadcast: BroadcastConfig; - workers: WorkersConfig; - featured: FeaturedConfig; - abis: AbisConfig; - mongo: MongoConfig; - queue: ProcessorTaskQueueConfig; - [key: string]: unknown; -}; - export type FilterAddons = { matchers?: unknown; [key: string]: unknown; diff --git a/src/filter/filter.worker-loader.dependencies.ts b/src/filter/filter.worker-loader.dependencies.ts deleted file mode 100644 index dc8a201..0000000 --- a/src/filter/filter.worker-loader.dependencies.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { UnknownObject, Serializer } from '@alien-worlds/api-core'; -import { FilterConfig } from './filter.types'; -import { WorkerLoaderDependencies } from '@alien-worlds/workers'; -import { Abis, Featured, ProcessorTaskQueue } from '../common'; -import { ShipAbis } from '@alien-worlds/block-reader'; - -/** - * An abstract class representing a FilterWorkerLoader dependencies. - * @class FilterWorkerLoaderDependencies - */ -export abstract class FilterWorkerLoaderDependencies extends WorkerLoaderDependencies { - public processorTaskQueue: ProcessorTaskQueue; - public abis: Abis; - public shipAbis: ShipAbis; - public featured: Featured; - public serializer: Serializer; - - public abstract initialize( - config: FilterConfig, - featuredJson: UnknownObject - ): Promise; -} diff --git a/src/filter/filter.worker-loader.ts b/src/filter/filter.worker-loader.ts index b7c2a92..fa2c03a 100644 --- a/src/filter/filter.worker-loader.ts +++ b/src/filter/filter.worker-loader.ts @@ -1,7 +1,7 @@ import { Worker, DefaultWorkerLoader } from '@alien-worlds/workers'; import { FilterSharedData } from './filter.types'; import FilterWorker from './filter.worker'; -import { FilterWorkerLoaderDependencies } from './filter.worker-loader.dependencies'; +import { FilterWorkerLoaderDependencies } from '@alien-worlds/history-tools-common'; export default class FilterWorkerLoader extends DefaultWorkerLoader< FilterSharedData, diff --git a/src/filter/filter.worker.ts b/src/filter/filter.worker.ts index df6c553..0d3ee16 100644 --- a/src/filter/filter.worker.ts +++ b/src/filter/filter.worker.ts @@ -1,12 +1,21 @@ -import { Serializer, log, parseToBigInt } from '@alien-worlds/api-core'; -import { Worker } from '@alien-worlds/workers'; -import { AbiNotFoundError, ShipAbis } from '@alien-worlds/block-reader'; -import { DeltaByName, Featured, SignedBlock, TraceByName } from '../common'; -import { Abis } from '../common/abis'; -import { isSetAbiAction } from '../common/common.utils'; -import { ProcessorTask, ProcessorTaskQueue } from '../common/processor-task-queue'; +import { + Worker, + ShipAbis, + Abis, + Featured, + ProcessorTaskQueue, + Serializer, + SignedBlock, + TraceByName, + DeltaByName, + ProcessorTask, + parseToBigInt, + log, + AbiNotFoundError, + isSetAbiAction, + BlockModel, +} from '@alien-worlds/history-tools-common'; import { FilterSharedData } from './filter.types'; -import { BlockModel } from '../common/types/block.types'; export default class FilterWorker extends Worker { constructor( diff --git a/src/filter/index.ts b/src/filter/index.ts index bc478cb..774ea38 100644 --- a/src/filter/index.ts +++ b/src/filter/index.ts @@ -1,9 +1,7 @@ export * from './filter.command'; export * from './filter.consts'; -export * from './filter.dependencies'; export * from './filter.runner'; export * from './filter.types'; export * from './filter.worker'; export * from './filter.worker-loader'; -export * from './filter.worker-loader.dependencies'; export * from './start-filter'; diff --git a/src/filter/start-filter.ts b/src/filter/start-filter.ts index 8adceef..5679dfc 100644 --- a/src/filter/start-filter.ts +++ b/src/filter/start-filter.ts @@ -1,5 +1,4 @@ -import { ConfigVars, log } from '@alien-worlds/api-core'; -import { FilterAddons, FilterCommandOptions, FilterConfig } from './filter.types'; +import { FilterAddons, FilterCommandOptions } from './filter.types'; import { InternalBroadcastChannel, InternalBroadcastMessageName, @@ -9,8 +8,15 @@ import { FilterRunner } from './filter.runner'; import { FilterBroadcastMessage } from '../broadcast/messages/filter-broadcast.message'; import { buildFilterConfig } from '../config'; import { filterCommand } from './filter.command'; -import { FilterDependencies } from './filter.dependencies'; -import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { filterWorkerLoaderPath } from './filter.consts'; +import { + BroadcastMessage, + ConfigVars, + FilterConfig, + FilterDependencies, + WorkerPool, + log, +} from '@alien-worlds/history-tools-common'; export const filter = async ( config: FilterConfig, @@ -18,14 +24,27 @@ export const filter = async ( addons?: FilterAddons ) => { log(`Filter ... [starting]`); - + const { matchers } = addons; const initResult = await dependencies.initialize(config, addons); if (initResult.isFailure) { throw initResult.failure.error; } - const { broadcastClient, workerPool, unprocessedBlockQueue } = dependencies; + const { + broadcastClient, + unprocessedBlockQueue, + workerLoaderPath, + workerLoaderDependenciesPath, + } = dependencies; + + const workerPool = await WorkerPool.create({ + ...config.workers, + sharedData: { config, matchers }, + workerLoaderPath: workerLoaderPath || filterWorkerLoaderPath, + workerLoaderDependenciesPath, + }); + const runner = new FilterRunner(workerPool, unprocessedBlockQueue); runner.onTransition(() => { @@ -57,6 +76,7 @@ export const startFilter = ( ) => { const vars = new ConfigVars(); const options = filterCommand.parse(args).opts(); - const config = buildFilterConfig(vars, options); + const config = buildFilterConfig(vars, dependencies.databaseConfigBuilder, options); + filter(config, dependencies, addons).catch(log); }; diff --git a/src/index.ts b/src/index.ts index ad0dabb..f9c60cb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,13 @@ export * from './api'; export * from './bootstrap'; -export * from './common'; export * from './config'; export * from './filter'; export * from './broadcast'; export * from './processor'; export * from './reader'; + + +export * from '@alien-worlds/api-core'; +export * from '@alien-worlds/block-reader'; +export * from '@alien-worlds/broadcast'; +export * from '@alien-worlds/workers'; diff --git a/src/processor/index.ts b/src/processor/index.ts index d254764..3ba8c7d 100644 --- a/src/processor/index.ts +++ b/src/processor/index.ts @@ -1,10 +1,8 @@ export * from './processors'; export * from './processor.command'; export * from './processor.consts'; -export * from './processor.dependencies'; export * from './processor.enum'; export * from './processor.runner'; export * from './processor.types'; export * from './processor.worker-loader'; -export * from './processor.worker-loader.dependencies'; export * from './start-processor'; diff --git a/src/processor/processor.dependencies.ts b/src/processor/processor.dependencies.ts deleted file mode 100644 index 77e0495..0000000 --- a/src/processor/processor.dependencies.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { Result } from '@alien-worlds/api-core'; -import { Dependencies } from '../common/dependencies'; -import { BroadcastClient } from '@alien-worlds/broadcast'; -import { ProcessorAddons, ProcessorConfig } from './processor.types'; -import { - ContractTraceMatchCriteria, - ContractDeltaMatchCriteria, - FeaturedMapper, - ProcessorTaskQueue, -} from '../common'; - -/** - * An abstract class representing a Processor dependencies. - * @class ProcessorDependencies - */ -export abstract class ProcessorDependencies extends Dependencies { - public broadcastClient: BroadcastClient; - public featuredTraces: FeaturedMapper; - public featuredDeltas: FeaturedMapper; - public processorTaskQueue: ProcessorTaskQueue; - - public abstract initialize( - config: ProcessorConfig, - addons?: ProcessorAddons - ): Promise; -} diff --git a/src/processor/processor.runner.ts b/src/processor/processor.runner.ts index 2c02964..aa131d9 100644 --- a/src/processor/processor.runner.ts +++ b/src/processor/processor.runner.ts @@ -1,17 +1,16 @@ -import { log } from '@alien-worlds/api-core'; import { - ProcessorTaskQueue, + ContractDeltaMatchCriteria, + ContractTraceMatchCriteria, + FeaturedMapper, ProcessorTask, ProcessorTaskModel, + ProcessorTaskQueue, ProcessorTaskType, UnknownProcessorTypeError, -} from '../common/processor-task-queue'; -import { WorkerMessage, WorkerPool } from '@alien-worlds/workers'; -import { - ContractTraceMatchCriteria, - ContractDeltaMatchCriteria, - FeaturedMapper, -} from '../common'; + WorkerMessage, + WorkerPool, + log, +} from '@alien-worlds/history-tools-common'; export class ProcessorRunner { private interval: NodeJS.Timeout; @@ -61,7 +60,7 @@ export class ProcessorRunner { ); workerPool.releaseWorker(message.workerId); } else if (message.isTaskRejected()) { - queue.stashUnsuccessfulTask(task, message.error); + queue.stashUnsuccessfulTask(task, message.error as Error); log(message.error); log( `Worker #${worker.id} has completed (unsuccessfully) work on the task "${task.id}". diff --git a/src/processor/processor.types.ts b/src/processor/processor.types.ts index fbc8fc4..721b785 100644 --- a/src/processor/processor.types.ts +++ b/src/processor/processor.types.ts @@ -1,32 +1,9 @@ -import { FeaturedConfig, ProcessorMatcher } from '../common/featured'; -import { ProcessorTaskQueueConfig } from '../common/processor-task-queue/processor-task-queue.config'; -import { MongoConfig } from '@alien-worlds/storage-mongodb'; -import { BroadcastConfig } from '@alien-worlds/broadcast'; -import { WorkersConfig } from '@alien-worlds/workers'; +import { ProcessorConfig } from '@alien-worlds/history-tools-common'; export type ProcessorCommandOptions = { threads: number; }; -export type ProcessorConfig = { - broadcast: BroadcastConfig; - workers: WorkersConfig; - featured: FeaturedConfig; - mongo: MongoConfig; - queue: ProcessorTaskQueueConfig; - processorLoaderPath?: string; - [key: string]: unknown; -}; - -export type ProcessorAddons = { - matchers?: { - traces?: ProcessorMatcher; - deltas?: ProcessorMatcher; - [key: string]: ProcessorMatcher; - }; - [key: string]: unknown; -}; - export type ProcessorSharedData = { config: ProcessorConfig; }; diff --git a/src/processor/processor.worker-loader.dependencies.ts b/src/processor/processor.worker-loader.dependencies.ts deleted file mode 100644 index ffc7611..0000000 --- a/src/processor/processor.worker-loader.dependencies.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { UnknownObject, Serializer } from '@alien-worlds/api-core'; -import { WorkerContainer, WorkerLoaderDependencies } from '@alien-worlds/workers'; -import { ProcessorConfig } from './processor.types'; - -/** - * An abstract class representing a ProcessorWorkerLoader dependencies. - * @class ProcessorWorkerLoaderDependencies - */ -export abstract class ProcessorWorkerLoaderDependencies extends WorkerLoaderDependencies { - public dataSource: unknown; - public serializer: Serializer; - public workers: WorkerContainer; - - public abstract initialize( - config: ProcessorConfig, - featuredJson: UnknownObject - ): Promise; -} diff --git a/src/processor/processor.worker-loader.ts b/src/processor/processor.worker-loader.ts index 6f86b3e..975ebad 100644 --- a/src/processor/processor.worker-loader.ts +++ b/src/processor/processor.worker-loader.ts @@ -1,7 +1,11 @@ -import { DefaultWorkerLoader, Worker, WorkerContainer } from '@alien-worlds/workers'; +import { + Worker, + Container, + DefaultWorkerLoader, + ProcessorWorkerLoaderDependencies, + WorkerContainer, +} from '@alien-worlds/history-tools-common'; import { ProcessorSharedData } from './processor.types'; -import { ProcessorWorkerLoaderDependencies } from './processor.worker-loader.dependencies'; -import { Container } from '@alien-worlds/api-core'; export default class ProcessorWorkerLoader extends DefaultWorkerLoader< ProcessorSharedData, @@ -18,10 +22,10 @@ export default class ProcessorWorkerLoader extends DefaultWorkerLoader< public async load(pointer: string): Promise { const { - dependencies: { dataSource, serializer, workers }, + dependencies: { dataSource, serializer, processorClasses }, } = this; const { ioc, sharedData } = this; - const Class = workers.get(pointer); + const Class = processorClasses.get(pointer); const worker: Worker = new Class( { ioc, diff --git a/src/processor/processors/action-trace.processor.ts b/src/processor/processors/action-trace.processor.ts index 2bda3df..d92e974 100644 --- a/src/processor/processors/action-trace.processor.ts +++ b/src/processor/processors/action-trace.processor.ts @@ -1,12 +1,14 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { Processor } from './processor'; +import { ActionTraceProcessorInput, ProcessorSharedData } from '../processor.types'; +import { deserialize } from 'v8'; import { ActionProcessorContentModel, + Container, ProcessorTaskModel, -} from '../../common/processor-task-queue/processor-task.types'; -import { ActionTraceProcessorInput, ProcessorSharedData } from '../processor.types'; -import { Container, Serializer, parseToBigInt } from '@alien-worlds/api-core'; -import { deserialize } from 'v8'; + Serializer, + parseToBigInt, +} from '@alien-worlds/history-tools-common'; export class ActionTraceProcessor< DataType = unknown, @@ -34,10 +36,7 @@ export class ActionTraceProcessor< const { abi, content: buffer } = model; const content: ActionProcessorContentModel = deserialize(buffer); const { - action_trace: { - act, - receipt, - }, + action_trace: { act, receipt }, block_num, block_timestamp, transaction_id, diff --git a/src/processor/processors/delta.processor.ts b/src/processor/processors/delta.processor.ts index a813d77..67f4efe 100644 --- a/src/processor/processors/delta.processor.ts +++ b/src/processor/processors/delta.processor.ts @@ -1,11 +1,13 @@ -import { - DeltaProcessorContentModel, - ProcessorTaskModel, -} from '../../common/processor-task-queue/processor-task.types'; import { Processor } from './processor'; import { DeltaProcessorInput, ProcessorSharedData } from '../processor.types'; -import { Container, Serializer, parseToBigInt } from '@alien-worlds/api-core'; import { deserialize } from 'v8'; +import { + Container, + DeltaProcessorContentModel, + ProcessorTaskModel, + Serializer, + parseToBigInt, +} from '@alien-worlds/history-tools-common'; export class DeltaProcessor< DataType, diff --git a/src/processor/processors/processor.ts b/src/processor/processors/processor.ts index ef8c052..370a9b3 100644 --- a/src/processor/processors/processor.ts +++ b/src/processor/processors/processor.ts @@ -1,7 +1,6 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { Worker } from '@alien-worlds/workers'; -import { ProcessorTaskModel } from '../../common/processor-task-queue/processor-task.types'; import { ProcessorSharedData } from '../processor.types'; +import { Worker, ProcessorTaskModel } from '@alien-worlds/history-tools-common'; export class Processor< SharedDataType = ProcessorSharedData diff --git a/src/processor/start-processor.ts b/src/processor/start-processor.ts index 8544727..21b555b 100644 --- a/src/processor/start-processor.ts +++ b/src/processor/start-processor.ts @@ -1,21 +1,24 @@ -import { ConfigVars, log } from '@alien-worlds/api-core'; import { + BroadcastMessage, + ConfigVars, + FeaturedContractDataCriteria, ProcessorAddons, - ProcessorCommandOptions, ProcessorConfig, -} from './processor.types'; + ProcessorDependencies, + WorkerClass, + WorkerPool, + log, +} from '@alien-worlds/history-tools-common'; +import { processorWorkerLoaderPath } from './processor.consts'; +import { ProcessorRunner } from './processor.runner'; import { InternalBroadcastChannel, InternalBroadcastMessageName, ProcessorBroadcastMessage, } from '../broadcast'; -import { ProcessorRunner } from './processor.runner'; -import { ProcessorDependencies } from './processor.dependencies'; import { processorCommand } from './processor.command'; import { buildProcessorConfig } from '../config'; -import { BroadcastMessage } from '@alien-worlds/broadcast'; -import { processorWorkerLoaderPath } from './processor.consts'; -import { WorkerPool } from '@alien-worlds/workers'; +import { ProcessorCommandOptions } from './processor.types'; /** * @@ -26,11 +29,18 @@ import { WorkerPool } from '@alien-worlds/workers'; export const process = async ( config: ProcessorConfig, dependencies: ProcessorDependencies, + processorClasses: Map, + featuredCriteria: FeaturedContractDataCriteria, addons: ProcessorAddons = {} ) => { log(`Processor ... [starting]`); - const initResult = await dependencies.initialize(config, addons); + const initResult = await dependencies.initialize( + config, + featuredCriteria, + processorClasses, + addons + ); if (initResult.isFailure) { throw initResult.failure.error; @@ -71,10 +81,12 @@ export const process = async ( export const startProcessor = ( args: string[], dependencies: ProcessorDependencies, + processorClasses: Map, + featuredCriteria: FeaturedContractDataCriteria, addons?: ProcessorAddons ) => { const vars = new ConfigVars(); const options = processorCommand.parse(args).opts(); - const config = buildProcessorConfig(vars, options); - process(config, dependencies, addons).catch(log); + const config = buildProcessorConfig(vars, dependencies.databaseConfigBuilder, options); + process(config, dependencies, processorClasses, featuredCriteria, addons).catch(log); }; diff --git a/src/reader/index.ts b/src/reader/index.ts index 803970b..0c8f852 100644 --- a/src/reader/index.ts +++ b/src/reader/index.ts @@ -1,9 +1,7 @@ export * from './reader'; export * from './reader.command'; export * from './reader.consts'; -export * from './reader.dependencies'; export * from './reader.types'; export * from './reader.worker'; export * from './reader.worker-loader'; -export * from './reader.worker-loader.dependencies'; export * from './start-reader'; diff --git a/src/reader/reader.dependencies.ts b/src/reader/reader.dependencies.ts deleted file mode 100644 index 758e6ea..0000000 --- a/src/reader/reader.dependencies.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { BroadcastClient } from '@alien-worlds/broadcast'; -import { Dependencies } from '../common/dependencies'; -import { WorkerPool } from '@alien-worlds/workers'; -import { ReaderConfig } from './reader.types'; -import { Result } from '@alien-worlds/api-core'; -import { BlockRangeScanner } from '../common'; - -/** - * An abstract class representing a reader dependencies. - * @class ReaderDependencies - */ -export abstract class ReaderDependencies extends Dependencies { - public broadcastClient: BroadcastClient; - public scanner: BlockRangeScanner; - public workerPool: WorkerPool; - - public abstract initialize(config: ReaderConfig): Promise; -} diff --git a/src/reader/reader.ts b/src/reader/reader.ts index ebe27cc..1a6c8fe 100644 --- a/src/reader/reader.ts +++ b/src/reader/reader.ts @@ -1,10 +1,13 @@ -import { log } from '@alien-worlds/api-core'; -import { Mode } from '../common/common.enums'; import { ReadCompleteData, ReadTaskData } from './reader.types'; import { FilterBroadcastMessage } from '../broadcast/messages'; -import { WorkerMessage, WorkerPool } from '@alien-worlds/workers'; -import { BlockRangeScanner } from '../common'; -import { BroadcastClient } from '@alien-worlds/broadcast'; +import { + BlockRangeScanner, + BroadcastClient, + Mode, + WorkerMessage, + WorkerPool, + log, +} from '@alien-worlds/history-tools-common'; export class Reader { private loop = false; diff --git a/src/reader/reader.types.ts b/src/reader/reader.types.ts index 945053a..78387bd 100644 --- a/src/reader/reader.types.ts +++ b/src/reader/reader.types.ts @@ -1,24 +1,3 @@ -import { BroadcastConfig } from '@alien-worlds/broadcast'; -import { BlockRangeScanConfig } from '../common/block-range-scanner'; -import { MongoConfig } from '@alien-worlds/storage-mongodb'; -import { BlockReaderConfig } from '@alien-worlds/block-reader'; -import { WorkersConfig } from '@alien-worlds/workers'; - -export type ReaderConfig = { - broadcast?: BroadcastConfig; - blockReader?: BlockReaderConfig; - mongo?: MongoConfig; - scanner?: BlockRangeScanConfig; - workers?: WorkersConfig; - mode?: string; - maxBlockNumber?: number; - blockQueueMaxBytesSize?: number; - blockQueueSizeCheckInterval?: number; - blockQueueBatchSize?: number; - startBlock?: bigint; - endBlock?: bigint; -}; - export type ReaderCommandOptions = { startBlock?: bigint; endBlock?: bigint; diff --git a/src/reader/reader.worker-loader.dependencies.ts b/src/reader/reader.worker-loader.dependencies.ts deleted file mode 100644 index 31eb013..0000000 --- a/src/reader/reader.worker-loader.dependencies.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { UnknownObject } from '@alien-worlds/api-core'; -import { WorkerLoaderDependencies } from '@alien-worlds/workers'; -import { BlockRangeScanner, BlockState, UnprocessedBlockQueue } from '../common'; -import { BlockReader } from '@alien-worlds/block-reader'; -import { ReaderConfig } from './reader.types'; - -/** - * An abstract class representing a ReaderWorkerLoader dependencies. - * @class ReaderWorkerLoaderDependencies - */ -export abstract class ReaderWorkerLoaderDependencies extends WorkerLoaderDependencies { - public blockReader: BlockReader; - public blockState: BlockState; - public blockQueue: UnprocessedBlockQueue; - public scanner: BlockRangeScanner; - - public abstract initialize( - config: ReaderConfig, - featuredJson: UnknownObject - ): Promise; -} diff --git a/src/reader/reader.worker-loader.ts b/src/reader/reader.worker-loader.ts index 9c1d5bb..10309fb 100644 --- a/src/reader/reader.worker-loader.ts +++ b/src/reader/reader.worker-loader.ts @@ -1,8 +1,10 @@ -import { log } from '@alien-worlds/api-core'; -import { Worker } from '@alien-worlds/workers'; import ReaderWorker, { ReaderSharedData } from './reader.worker'; -import { DefaultWorkerLoader } from '@alien-worlds/workers'; -import { ReaderWorkerLoaderDependencies } from './reader.worker-loader.dependencies'; +import { + Worker, + DefaultWorkerLoader, + ReaderWorkerLoaderDependencies, + log, +} from '@alien-worlds/history-tools-common'; export default class ReaderWorkerLoader extends DefaultWorkerLoader< ReaderSharedData, @@ -12,12 +14,14 @@ export default class ReaderWorkerLoader extends DefaultWorkerLoader< const { config } = sharedData; await super.setup(sharedData, config); // - const { blockQueueMaxBytesSize, blockQueueSizeCheckInterval } = config; + const { + unprocessedBlockQueue: { maxBytesSize, sizeCheckInterval }, + } = config; const { dependencies: { blockQueue: blocksQueue, blockReader }, } = this; blocksQueue.onOverload(size => { - const overload = size - blockQueueMaxBytesSize; + const overload = size - maxBytesSize; log(`Overload: ${overload} bytes.`); blockReader.pause(); @@ -34,7 +38,7 @@ export default class ReaderWorkerLoader extends DefaultWorkerLoader< clearInterval(interval); interval = null; } - }, blockQueueSizeCheckInterval || 1000); + }, sizeCheckInterval || 1000); }); blocksQueue.beforeSendBatch(() => { blockReader.pause(); diff --git a/src/reader/reader.worker.ts b/src/reader/reader.worker.ts index 772b891..29589eb 100644 --- a/src/reader/reader.worker.ts +++ b/src/reader/reader.worker.ts @@ -1,8 +1,14 @@ -import { log, parseToBigInt } from '@alien-worlds/api-core'; -import { Worker } from '@alien-worlds/workers'; -import { BlockReader } from '@alien-worlds/block-reader'; -import { ReaderConfig } from './reader.types'; -import { BlockRangeScanner, BlockState, Mode, UnprocessedBlockQueue } from '../common'; +import { + Worker, + BlockRangeScanner, + BlockState, + Mode, + ReaderConfig, + UnprocessedBlockQueue, + BlockReader, + log, + parseToBigInt, +} from '@alien-worlds/history-tools-common'; export type ReaderSharedData = { config: ReaderConfig; @@ -56,7 +62,7 @@ export default class ReaderWorker extends Worker { config: { maxBlockNumber, blockReader: { shouldFetchDeltas, shouldFetchTraces, shouldFetchBlock }, - blockQueueMaxBytesSize, + unprocessedBlockQueue, }, }, } = this; @@ -76,7 +82,7 @@ export default class ReaderWorker extends Worker { } else if (failure?.error.name === 'UnprocessedBlocksOverloadError') { log(failure.error.message); log( - `The size limit ${blockQueueMaxBytesSize} of the unprocessed blocks collection has been exceeded. Blockchain reading suspended until the collection is cleared.` + `The size limit ${unprocessedBlockQueue.maxBytesSize} of the unprocessed blocks collection has been exceeded. Blockchain reading suspended until the collection is cleared.` ); } else if (failure) { this.reject(failure.error); @@ -110,7 +116,7 @@ export default class ReaderWorker extends Worker { sharedData: { config: { blockReader: { shouldFetchDeltas, shouldFetchTraces, shouldFetchBlock }, - blockQueueMaxBytesSize, + unprocessedBlockQueue, }, }, } = this; @@ -131,7 +137,7 @@ export default class ReaderWorker extends Worker { } else if (failure?.error.name === 'UnprocessedBlocksOverloadError') { log(failure.error.message); log( - `The size limit ${blockQueueMaxBytesSize} of the unprocessed blocks collection has been exceeded by bytes. Blockchain reading suspended until the collection is cleared.` + `The size limit ${unprocessedBlockQueue.maxBytesSize} of the unprocessed blocks collection has been exceeded by bytes. Blockchain reading suspended until the collection is cleared.` ); } else if (failure) { this.reject(failure.error); diff --git a/src/reader/start-reader.ts b/src/reader/start-reader.ts index 787ada0..833dfc8 100644 --- a/src/reader/start-reader.ts +++ b/src/reader/start-reader.ts @@ -4,15 +4,21 @@ import { InternalBroadcastChannel, InternalBroadcastMessageName, } from '../broadcast/internal-broadcast.enums'; -import { ConfigVars, log } from '@alien-worlds/api-core'; -import { ReadTaskData, ReaderCommandOptions, ReaderConfig } from './reader.types'; +import { ReadTaskData, ReaderCommandOptions } from './reader.types'; import { ReaderBroadcastMessage } from '../broadcast/messages/reader-broadcast.message'; import { Reader } from './reader'; -import { Mode } from '../common'; import { readerCommand } from './reader.command'; -import { ReaderDependencies } from './reader.dependencies'; import { buildReaderConfig } from '../config'; -import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { + BroadcastMessage, + ConfigVars, + Mode, + ReaderConfig, + ReaderDependencies, + WorkerPool, + log, +} from '@alien-worlds/history-tools-common'; +import { readerWorkerLoaderPath } from './reader.consts'; /** * @@ -28,7 +34,16 @@ export const read = async (config: ReaderConfig, dependencies: ReaderDependencie throw initResult.failure.error; } - const { broadcastClient, scanner, workerPool } = dependencies; + const { broadcastClient, scanner, workerLoaderPath, workerLoaderDependenciesPath } = + dependencies; + + const workerPool = await WorkerPool.create({ + ...config.workers, + sharedData: { config }, + workerLoaderPath: workerLoaderPath || readerWorkerLoaderPath, + workerLoaderDependenciesPath, + }); + const reader = new Reader(broadcastClient, scanner, workerPool); let channel: string; @@ -59,6 +74,6 @@ export const read = async (config: ReaderConfig, dependencies: ReaderDependencie export const startReader = (args: string[], dependencies: ReaderDependencies) => { const vars = new ConfigVars(); const options = readerCommand.parse(args).opts(); - const config = buildReaderConfig(vars, options); + const config = buildReaderConfig(vars, dependencies.databaseConfigBuilder, options); read(config, dependencies).catch(log); }; diff --git a/yarn.lock b/yarn.lock index 8c3305c..cd06b77 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,52 +2,46 @@ # yarn lockfile v1 -"@alien-worlds/api-core@^0.0.145": - version "0.0.145" - resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.145/d5f1d6a5c4e14c62acc5e986ebb13c35a7c8cbff#d5f1d6a5c4e14c62acc5e986ebb13c35a7c8cbff" - integrity sha512-KVrS6UCL3DQ4uTOLDPBIxHHQ7xtFHpTwrWejmgv6YDffiOBNUiQvw+fsXXkPmrSdR/aqj2fCWH3k6t0y5gU7Ww== +"@alien-worlds/api-core@^0.0.146": + version "0.0.146" + resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.146/fa858d4b0d1ba4a082aef7f3860a472d4cad6fff#fa858d4b0d1ba4a082aef7f3860a472d4cad6fff" + integrity sha512-1J/uPD5XS15r7PlmHT0U6+jbDUq6oPWcRr8wY8Wc4meoAf1Lg4NxVI/SNbevBKCqGhp2fdKuKU+HhTEbAv7e/Q== dependencies: inversify "^6.0.1" nanoid "^3.0.0" node-fetch "2" reflect-metadata "^0.1.13" -"@alien-worlds/block-reader@^0.0.4": - version "0.0.4" - resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.4/5d406a126bf19685aedf238762599819de59b94d#5d406a126bf19685aedf238762599819de59b94d" - integrity sha512-Bofm0Fm0jlYxsezh4dQ93vDbv3/B4grFItOT0dbA69fH8SoCh33GMXi/M16KWbzMboOOXmiRyGavTHRX4kMn3g== +"@alien-worlds/block-reader@^0.0.6": + version "0.0.6" + resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.6/df738d468de2da1707b4128d8bd4f92586e33c1a#df738d468de2da1707b4128d8bd4f92586e33c1a" + integrity sha512-ZDVUUxS2XdOHkvNTv/yhBFnlGnS4ynaHa2Qbg3sa6ehoftAmvREKvEGhmC1AxxyNMv9L3DVSi461HZnMpiox2Q== dependencies: - "@alien-worlds/api-core" "^0.0.145" + "@alien-worlds/api-core" "^0.0.146" -"@alien-worlds/broadcast@^0.0.5": - version "0.0.5" - resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.5/84e072991cdb63a45b48d8062a72bd6630f56e68#84e072991cdb63a45b48d8062a72bd6630f56e68" - integrity sha512-htVvAJomd2JmQ6wSslSUyOYKJGX7X3QnJ9rrbC25RSOoBRycfRznn6CTDmPS8ckWB+Cmk0x2RRsuXFQmWeKR+A== +"@alien-worlds/broadcast@^0.0.6": + version "0.0.6" + resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.6/4e5a9a14dd09815fbffe6b90969d0754438d0e52#4e5a9a14dd09815fbffe6b90969d0754438d0e52" + integrity sha512-h6/B/TPDkqICklH9bBsNBizHl/9xWdowMWYDT6xxPVGy/YPMwk0LAu136yxLclhCsmv5MlS1+OHPCpfrtpEREw== dependencies: - "@alien-worlds/api-core" "^0.0.145" + "@alien-worlds/api-core" "^0.0.146" nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/eos@^0.0.9": - version "0.0.9" - resolved "https://npm.pkg.github.com/download/@alien-worlds/eos/0.0.9/ef1cc073e48ccf01bf7663f5a4998d36e9b6cd51#ef1cc073e48ccf01bf7663f5a4998d36e9b6cd51" - integrity sha512-u9yobvp47nySp2HNzNNUynP3sU0FuzVZdfmmwS3GEK/rj6Svy63rAN+EJ9WekkV7k2oI0oP+mVVDspbh+RnF1g== - dependencies: - "@alien-worlds/api-core" "^0.0.145" - eosjs "^22.1.0" - -"@alien-worlds/storage-mongodb@^0.0.16": - version "0.0.16" - resolved "https://npm.pkg.github.com/download/@alien-worlds/storage-mongodb/0.0.16/77dcc02b45b5f1cea0ed2fa5b4715ed5733e77f2#77dcc02b45b5f1cea0ed2fa5b4715ed5733e77f2" - integrity sha512-24dpfaY725nTbcm0jsU3ISqHddDkjn4wu1gI6fvP7AmkM5Vra1C0PDhVIG+J4B6WUsJXQQqDhhHOo+J0qShOJA== +"@alien-worlds/history-tools-common@^0.0.20": + version "0.0.20" + resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.20/2fb02e82270e67e1d5513b14dcd9fc530ef1838b#2fb02e82270e67e1d5513b14dcd9fc530ef1838b" + integrity sha512-IcisJdvUP63+z6B3mcjWKtM93RiJ1cjv5rJWqfJsVIz1j9QeKTkIuIIGeYDsHeV5ZUuZEmQJP4RqJFyMiYllIw== dependencies: - "@alien-worlds/api-core" "^0.0.145" - mongodb "4.9.1" + "@alien-worlds/api-core" "^0.0.146" + "@alien-worlds/block-reader" "^0.0.6" + "@alien-worlds/broadcast" "^0.0.6" + "@alien-worlds/workers" "^0.0.5" -"@alien-worlds/workers@^0.0.3": - version "0.0.3" - resolved "https://npm.pkg.github.com/download/@alien-worlds/workers/0.0.3/91bad176b3e3d511b6d5e9c9ff04619a150ccd24#91bad176b3e3d511b6d5e9c9ff04619a150ccd24" - integrity sha512-WPtw2ckhd9nTzjL5keC69+7LtJiYL+G+pHAmDoqT9ZlY+Y8bPc3jEZLHl9fSS50JWh4wPaBy1R/DTkoKOKz32Q== +"@alien-worlds/workers@^0.0.5": + version "0.0.5" + resolved "https://npm.pkg.github.com/download/@alien-worlds/workers/0.0.5/5df38161ca64790c61ca0d9eb72db98f842a6e09#5df38161ca64790c61ca0d9eb72db98f842a6e09" + integrity sha512-Ar+Z8lTT0Li/4SCLt0IajYWIxSAn7D8IFzyvhiczZTiVFCmxvMZSsHlxeiQpEZqUoNjQre3oozEP+SLW1Rolvw== dependencies: async "^3.2.4" ts-node "^10.9.1" @@ -831,19 +825,6 @@ resolved "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz" integrity sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw== -"@types/webidl-conversions@*": - version "7.0.0" - resolved "https://registry.yarnpkg.com/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz#2b8e60e33906459219aa587e9d1a612ae994cfe7" - integrity sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog== - -"@types/whatwg-url@^8.2.1": - version "8.2.2" - resolved "https://registry.yarnpkg.com/@types/whatwg-url/-/whatwg-url-8.2.2.tgz#749d5b3873e845897ada99be4448041d4cc39e63" - integrity sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA== - dependencies: - "@types/node" "*" - "@types/webidl-conversions" "*" - "@types/yargs-parser@*": version "21.0.0" resolved "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.0.tgz" @@ -1143,21 +1124,6 @@ balanced-match@^1.0.0: resolved "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== -base64-js@^1.3.1: - version "1.5.1" - resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" - integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== - -bn.js@5.2.0: - version "5.2.0" - resolved "https://registry.npmjs.org/bn.js/-/bn.js-5.2.0.tgz" - integrity sha512-D7iWRBvnZE8ecXiLj/9wbxH7Tk79fAh8IHaTNq1RWRixsS02W+5qS+iE9yq6RYl0asXx5tw0bLhmT5pIfbSquw== - -bn.js@^4.11.9: - version "4.12.0" - resolved "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz" - integrity sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA== - body-parser@1.20.1: version "1.20.1" resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.1.tgz#b1812a8912c195cd371a3ee5e66faa2338a5c668" @@ -1191,11 +1157,6 @@ braces@^3.0.2: dependencies: fill-range "^7.0.1" -brorand@^1.1.0: - version "1.1.0" - resolved "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz" - integrity sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w== - browser-process-hrtime@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz" @@ -1225,26 +1186,11 @@ bser@2.1.1: dependencies: node-int64 "^0.4.0" -bson@^4.7.0: - version "4.7.2" - resolved "https://registry.yarnpkg.com/bson/-/bson-4.7.2.tgz#320f4ad0eaf5312dd9b45dc369cc48945e2a5f2e" - integrity sha512-Ry9wCtIZ5kGqkJoi6aD8KjxFZEx78guTQDnpXWiNthsxzrxAK/i8E6pCHAIZTbaEFWcOCvbecMukfK7XUvyLpQ== - dependencies: - buffer "^5.6.0" - buffer-from@^1.0.0: version "1.1.2" resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== -buffer@^5.6.0: - version "5.7.1" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" - integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== - dependencies: - base64-js "^1.3.1" - ieee754 "^1.1.13" - bytes@3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" @@ -1481,11 +1427,6 @@ delayed-stream@~1.0.0: resolved "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== -denque@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" - integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== - depd@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" @@ -1542,19 +1483,6 @@ electron-to-chromium@^1.4.251: resolved "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz" integrity sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA== -elliptic@6.5.4: - version "6.5.4" - resolved "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz" - integrity sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ== - dependencies: - bn.js "^4.11.9" - brorand "^1.1.0" - hash.js "^1.0.0" - hmac-drbg "^1.0.1" - inherits "^2.0.4" - minimalistic-assert "^1.0.1" - minimalistic-crypto-utils "^1.0.1" - emittery@^0.8.1: version "0.8.1" resolved "https://registry.npmjs.org/emittery/-/emittery-0.8.1.tgz" @@ -1570,16 +1498,6 @@ encodeurl@~1.0.2: resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w== -eosjs@^22.1.0: - version "22.1.0" - resolved "https://registry.npmjs.org/eosjs/-/eosjs-22.1.0.tgz" - integrity sha512-Ka8KO7akC3RxNdSg/3dkGWuUWUQESTzSUzQljBdVP16UG548vmQoBqSGnZdnjlZyfcab8VOu2iEt+JjyfYc5+A== - dependencies: - bn.js "5.2.0" - elliptic "6.5.4" - hash.js "1.1.7" - pako "2.0.3" - error-ex@^1.3.1: version "1.3.2" resolved "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz" @@ -2075,23 +1993,6 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" -hash.js@1.1.7, hash.js@^1.0.0, hash.js@^1.0.3: - version "1.1.7" - resolved "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz" - integrity sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA== - dependencies: - inherits "^2.0.3" - minimalistic-assert "^1.0.1" - -hmac-drbg@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz" - integrity sha512-Tti3gMqLdZfhOQY1Mzf/AanLiqh1WTiJgEj26ZuYQ9fbkLomzGchCws4FyrSd4VkpBfiNhaE1On+lOz894jvXg== - dependencies: - hash.js "^1.0.3" - minimalistic-assert "^1.0.0" - minimalistic-crypto-utils "^1.0.1" - html-encoding-sniffer@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz" @@ -2144,11 +2045,6 @@ iconv-lite@0.4.24: dependencies: safer-buffer ">= 2.1.2 < 3" -ieee754@^1.1.13: - version "1.2.1" - resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" - integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== - ignore@^5.2.0: version "5.2.0" resolved "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz" @@ -2183,21 +2079,16 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.3, inherits@^2.0.4: +inherits@2, inherits@2.0.4: version "2.0.4" resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== inversify@^6.0.1: version "6.0.1" - resolved "https://registry.npmjs.org/inversify/-/inversify-6.0.1.tgz" + resolved "https://registry.yarnpkg.com/inversify/-/inversify-6.0.1.tgz#b20d35425d5d8c5cd156120237aad0008d969f02" integrity sha512-B3ex30927698TJENHR++8FfEaJGqoWOgI6ZY5Ht/nLUsFCwHn6akbwtnUAPCgUepAnTpe2qHxhDNjoKLyz6rgQ== -ip@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ip/-/ip-2.0.0.tgz#4cf4ab182fee2314c75ede1276f8c80b479936da" - integrity sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ== - ipaddr.js@1.9.1: version "1.9.1" resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" @@ -2888,11 +2779,6 @@ media-typer@0.3.0: resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== -memory-pager@^1.0.2: - version "1.5.0" - resolved "https://registry.yarnpkg.com/memory-pager/-/memory-pager-1.5.0.tgz#d8751655d22d384682741c972f2c3d6dfa3e66b5" - integrity sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg== - merge-descriptors@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" @@ -2943,16 +2829,6 @@ mimic-fn@^2.1.0: resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== -minimalistic-assert@^1.0.0, minimalistic-assert@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz" - integrity sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A== - -minimalistic-crypto-utils@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz" - integrity sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg== - minimatch@^3.0.4, minimatch@^3.1.1, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz" @@ -2960,26 +2836,6 @@ minimatch@^3.0.4, minimatch@^3.1.1, minimatch@^3.1.2: dependencies: brace-expansion "^1.1.7" -mongodb-connection-string-url@^2.5.3: - version "2.6.0" - resolved "https://registry.yarnpkg.com/mongodb-connection-string-url/-/mongodb-connection-string-url-2.6.0.tgz#57901bf352372abdde812c81be47b75c6b2ec5cf" - integrity sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ== - dependencies: - "@types/whatwg-url" "^8.2.1" - whatwg-url "^11.0.0" - -mongodb@4.9.1: - version "4.9.1" - resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-4.9.1.tgz#0c769448228bcf9a6aa7d16daa3625b48312479e" - integrity sha512-ZhgI/qBf84fD7sI4waZBoLBNJYPQN5IOC++SBCiPiyhzpNKOxN/fi0tBHvH2dEC42HXtNEbFB0zmNz4+oVtorQ== - dependencies: - bson "^4.7.0" - denque "^2.1.0" - mongodb-connection-string-url "^2.5.3" - socks "^2.7.0" - optionalDependencies: - saslprep "^1.0.3" - ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" @@ -2996,9 +2852,9 @@ ms@2.1.3: integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== nanoid@^3.0.0: - version "3.3.4" - resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz" - integrity sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw== + version "3.3.6" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.6.tgz#443380c856d6e9f9824267d960b4236ad583ea4c" + integrity sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA== natural-compare@^1.4.0: version "1.4.0" @@ -3011,16 +2867,9 @@ negotiator@0.6.3: integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== node-fetch@2: - version "2.6.7" - resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz" - integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== - dependencies: - whatwg-url "^5.0.0" - -node-fetch@2.6.6: - version "2.6.6" - resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz" - integrity sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA== + version "2.6.11" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.11.tgz#cde7fc71deef3131ef80a738919f999e6edfff25" + integrity sha512-4I6pdBY1EthSqDmJkiNk3JIT8cswwR9nfeW/cPdUagJYEQG7R95WRH74wpz7ma8Gh/9dI9FP+OU+0E4FvtA55w== dependencies: whatwg-url "^5.0.0" @@ -3134,11 +2983,6 @@ p-try@^2.0.0: resolved "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz" integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== -pako@2.0.3: - version "2.0.3" - resolved "https://registry.npmjs.org/pako/-/pako-2.0.3.tgz" - integrity sha512-WjR1hOeg+kki3ZIOjaf4b5WVcay1jaliKSYiEaB1XzwhMQZJxRdQRv0V31EKBYlxb4T7SK3hjfc/jxyU64BoSw== - parent-module@^1.0.0: version "1.0.1" resolved "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz" @@ -3314,7 +3158,7 @@ react-is@^17.0.1: reflect-metadata@^0.1.13: version "0.1.13" - resolved "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz" + resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08" integrity sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg== regexpp@^3.2.0: @@ -3392,13 +3236,6 @@ safe-buffer@5.2.1: resolved "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== -saslprep@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/saslprep/-/saslprep-1.0.3.tgz#4c02f946b56cf54297e347ba1093e7acac4cf226" - integrity sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag== - dependencies: - sparse-bitfield "^3.0.3" - saxes@^5.0.1: version "5.0.1" resolved "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz" @@ -3488,19 +3325,6 @@ slash@^3.0.0: resolved "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz" integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== -smart-buffer@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae" - integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== - -socks@^2.7.0: - version "2.7.1" - resolved "https://registry.yarnpkg.com/socks/-/socks-2.7.1.tgz#d8e651247178fde79c0663043e07240196857d55" - integrity sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ== - dependencies: - ip "^2.0.0" - smart-buffer "^4.2.0" - source-map-support@^0.5.6: version "0.5.21" resolved "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz" @@ -3519,13 +3343,6 @@ source-map@^0.7.3: resolved "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz" integrity sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA== -sparse-bitfield@^3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz#ff4ae6e68656056ba4b3e792ab3334d38273ca11" - integrity sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ== - dependencies: - memory-pager "^1.0.2" - sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz" @@ -3638,11 +3455,6 @@ test-exclude@^6.0.0: glob "^7.1.4" minimatch "^3.0.4" -text-encoding@^0.7.0: - version "0.7.0" - resolved "https://registry.npmjs.org/text-encoding/-/text-encoding-0.7.0.tgz" - integrity sha512-oJQ3f1hrOnbRLOcwKz0Liq2IcrvDeZRHXhd9RgLrsT+DjWY/nty1Hi7v3dtkaEYbPYe0mUoOfzRrMwfXXwgPUA== - text-table@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz" @@ -3692,16 +3504,9 @@ tr46@^2.1.0: dependencies: punycode "^2.1.1" -tr46@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/tr46/-/tr46-3.0.0.tgz#555c4e297a950617e8eeddef633c87d4d9d6cbf9" - integrity sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA== - dependencies: - punycode "^2.1.1" - tr46@~0.0.3: version "0.0.3" - resolved "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== ts-jest@^27.1.3: @@ -3878,7 +3683,7 @@ walker@^1.0.7: webidl-conversions@^3.0.0: version "3.0.1" - resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== webidl-conversions@^5.0.0: @@ -3891,11 +3696,6 @@ webidl-conversions@^6.1.0: resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz" integrity sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w== -webidl-conversions@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-7.0.0.tgz#256b4e1882be7debbf01d05f0aa2039778ea080a" - integrity sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g== - whatwg-encoding@^1.0.5: version "1.0.5" resolved "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz" @@ -3908,17 +3708,9 @@ whatwg-mimetype@^2.3.0: resolved "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== -whatwg-url@^11.0.0: - version "11.0.0" - resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-11.0.0.tgz#0a849eebb5faf2119b901bb76fd795c2848d4018" - integrity sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ== - dependencies: - tr46 "^3.0.0" - webidl-conversions "^7.0.0" - whatwg-url@^5.0.0: version "5.0.0" - resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== dependencies: tr46 "~0.0.3" @@ -3974,11 +3766,6 @@ ws@^7.4.6: resolved "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz" integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== -ws@^8.12.0: - version "8.12.0" - resolved "https://registry.npmjs.org/ws/-/ws-8.12.0.tgz" - integrity sha512-kU62emKIdKVeEIOIKVegvqpXMSTAMLJozpHZaJNDYqBjzlSYXQGviYwN1osDLJ9av68qHd4a2oSjd7yD4pacig== - xml-name-validator@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz" From 1ace4570d346995e8a1ec0956015e733817485a1 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sat, 24 Jun 2023 23:37:33 +0200 Subject: [PATCH 006/107] 0.0.115 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7bb61dc..8f0c443 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.114", + "version": "0.0.115", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From f093093c6f9d156f04e38c543d48fa816b2c5ff0 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 11:44:38 +0200 Subject: [PATCH 007/107] pass featuredCriteria to worker loader --- src/filter/filter.types.ts | 4 ++-- src/filter/filter.worker-loader.ts | 4 ++-- src/filter/start-filter.ts | 7 +++++-- src/processor/processor.types.ts | 6 +++++- src/processor/processor.worker-loader.ts | 4 ++-- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/filter/filter.types.ts b/src/filter/filter.types.ts index b55ad0e..0aef439 100644 --- a/src/filter/filter.types.ts +++ b/src/filter/filter.types.ts @@ -1,8 +1,8 @@ -import { FilterConfig, UnknownObject } from '@alien-worlds/history-tools-common'; +import { FeaturedContractDataCriteria, FilterConfig } from '@alien-worlds/history-tools-common'; export type FilterSharedData = { config: FilterConfig; - featuredJson: UnknownObject; + featuredCriteria: FeaturedContractDataCriteria; }; export type FilterCommandOptions = { diff --git a/src/filter/filter.worker-loader.ts b/src/filter/filter.worker-loader.ts index fa2c03a..37542bc 100644 --- a/src/filter/filter.worker-loader.ts +++ b/src/filter/filter.worker-loader.ts @@ -8,8 +8,8 @@ export default class FilterWorkerLoader extends DefaultWorkerLoader< FilterWorkerLoaderDependencies > { public async setup(sharedData: FilterSharedData): Promise { - const { config } = sharedData; - await super.setup(sharedData, config); + const { config, featuredCriteria } = sharedData; + await super.setup(sharedData, config, featuredCriteria); } public async load(): Promise { diff --git a/src/filter/start-filter.ts b/src/filter/start-filter.ts index 5679dfc..33500d3 100644 --- a/src/filter/start-filter.ts +++ b/src/filter/start-filter.ts @@ -12,6 +12,7 @@ import { filterWorkerLoaderPath } from './filter.consts'; import { BroadcastMessage, ConfigVars, + FeaturedContractDataCriteria, FilterConfig, FilterDependencies, WorkerPool, @@ -21,6 +22,7 @@ import { export const filter = async ( config: FilterConfig, dependencies: FilterDependencies, + featuredCriteria: FeaturedContractDataCriteria, addons?: FilterAddons ) => { log(`Filter ... [starting]`); @@ -40,7 +42,7 @@ export const filter = async ( const workerPool = await WorkerPool.create({ ...config.workers, - sharedData: { config, matchers }, + sharedData: { config, matchers, featuredCriteria }, workerLoaderPath: workerLoaderPath || filterWorkerLoaderPath, workerLoaderDependenciesPath, }); @@ -72,11 +74,12 @@ export const filter = async ( export const startFilter = ( args: string[], dependencies: FilterDependencies, + featuredCriteria: FeaturedContractDataCriteria, addons?: FilterAddons ) => { const vars = new ConfigVars(); const options = filterCommand.parse(args).opts(); const config = buildFilterConfig(vars, dependencies.databaseConfigBuilder, options); - filter(config, dependencies, addons).catch(log); + filter(config, dependencies, featuredCriteria, addons).catch(log); }; diff --git a/src/processor/processor.types.ts b/src/processor/processor.types.ts index 721b785..9ccffe3 100644 --- a/src/processor/processor.types.ts +++ b/src/processor/processor.types.ts @@ -1,4 +1,7 @@ -import { ProcessorConfig } from '@alien-worlds/history-tools-common'; +import { + FeaturedContractDataCriteria, + ProcessorConfig, +} from '@alien-worlds/history-tools-common'; export type ProcessorCommandOptions = { threads: number; @@ -6,6 +9,7 @@ export type ProcessorCommandOptions = { export type ProcessorSharedData = { config: ProcessorConfig; + featuredCriteria: FeaturedContractDataCriteria; }; export type DeltaProcessorInput = { diff --git a/src/processor/processor.worker-loader.ts b/src/processor/processor.worker-loader.ts index 975ebad..e128da0 100644 --- a/src/processor/processor.worker-loader.ts +++ b/src/processor/processor.worker-loader.ts @@ -15,8 +15,8 @@ export default class ProcessorWorkerLoader extends DefaultWorkerLoader< protected ioc: Container; public async setup(sharedData: ProcessorSharedData): Promise { - const { config } = sharedData; - await super.setup(sharedData, config); + const { config, featuredCriteria } = sharedData; + await super.setup(sharedData, config, featuredCriteria); this.ioc = new Container(); } From c47247196ec682172e200a965ea3f13de37b871f Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 11:45:16 +0200 Subject: [PATCH 008/107] 0.0.116 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8f0c443..68f00ff 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.115", + "version": "0.0.116", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From f39ce2c059b980eef2e0c8966a4f391f0cc29cde Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 11:59:31 +0200 Subject: [PATCH 009/107] add start broadcast --- src/broadcast/index.ts | 2 +- src/broadcast/start-broadcast.ts | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 src/broadcast/start-broadcast.ts diff --git a/src/broadcast/index.ts b/src/broadcast/index.ts index 5bf2f7c..003c2d9 100644 --- a/src/broadcast/index.ts +++ b/src/broadcast/index.ts @@ -1,4 +1,4 @@ export * from './internal-broadcast.enums'; export * from './internal-broadcast.message'; export * from './messages'; - +export * from './start-broadcast'; diff --git a/src/broadcast/start-broadcast.ts b/src/broadcast/start-broadcast.ts new file mode 100644 index 0000000..ff4cde2 --- /dev/null +++ b/src/broadcast/start-broadcast.ts @@ -0,0 +1,13 @@ +import { + BroadcastTcpServer, + ConfigVars, + buildBroadcastConfig, +} from '@alien-worlds/history-tools-common'; + +export const startBroadcast = async () => { + const vars = new ConfigVars(); + const config = buildBroadcastConfig(vars); + const server = new BroadcastTcpServer(config); + + await server.start(); +}; From f06be80a52efc904ba771d3ca9b486b7f69fab97 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 11:59:55 +0200 Subject: [PATCH 010/107] 0.0.117 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 68f00ff..94f11df 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.116", + "version": "0.0.117", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 3d0eb5413eb318da19532c036cdcbeb70211724b Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 14:32:36 +0200 Subject: [PATCH 011/107] update processor --- package.json | 2 +- src/processor/processor.types.ts | 1 + src/processor/processor.worker-loader.ts | 10 +++++----- src/processor/start-processor.ts | 14 ++++---------- yarn.lock | 8 ++++---- 5 files changed, 15 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index 94f11df..ca4e10c 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/history-tools-common": "^0.0.20", + "@alien-worlds/history-tools-common": "^0.0.21", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", diff --git a/src/processor/processor.types.ts b/src/processor/processor.types.ts index 9ccffe3..5fbc961 100644 --- a/src/processor/processor.types.ts +++ b/src/processor/processor.types.ts @@ -10,6 +10,7 @@ export type ProcessorCommandOptions = { export type ProcessorSharedData = { config: ProcessorConfig; featuredCriteria: FeaturedContractDataCriteria; + processorsPath: string; }; export type DeltaProcessorInput = { diff --git a/src/processor/processor.worker-loader.ts b/src/processor/processor.worker-loader.ts index e128da0..6d52d88 100644 --- a/src/processor/processor.worker-loader.ts +++ b/src/processor/processor.worker-loader.ts @@ -15,18 +15,18 @@ export default class ProcessorWorkerLoader extends DefaultWorkerLoader< protected ioc: Container; public async setup(sharedData: ProcessorSharedData): Promise { - const { config, featuredCriteria } = sharedData; - await super.setup(sharedData, config, featuredCriteria); + const { config, featuredCriteria, processorsPath } = sharedData; + await super.setup(sharedData, config, featuredCriteria, processorsPath); this.ioc = new Container(); } public async load(pointer: string): Promise { const { - dependencies: { dataSource, serializer, processorClasses }, + dependencies: { dataSource, serializer, processorsPath }, } = this; const { ioc, sharedData } = this; - const Class = processorClasses.get(pointer); - const worker: Worker = new Class( + const processorClasses = await import(processorsPath); + const worker: Worker = new processorClasses[pointer]( { ioc, dataSource, diff --git a/src/processor/start-processor.ts b/src/processor/start-processor.ts index 21b555b..803b022 100644 --- a/src/processor/start-processor.ts +++ b/src/processor/start-processor.ts @@ -20,16 +20,10 @@ import { processorCommand } from './processor.command'; import { buildProcessorConfig } from '../config'; import { ProcessorCommandOptions } from './processor.types'; -/** - * - * @param featuredContent - * @param broadcastMessageMapper - * @param config - */ export const process = async ( config: ProcessorConfig, dependencies: ProcessorDependencies, - processorClasses: Map, + processorsPath: string, featuredCriteria: FeaturedContractDataCriteria, addons: ProcessorAddons = {} ) => { @@ -38,7 +32,7 @@ export const process = async ( const initResult = await dependencies.initialize( config, featuredCriteria, - processorClasses, + processorsPath, addons ); @@ -81,12 +75,12 @@ export const process = async ( export const startProcessor = ( args: string[], dependencies: ProcessorDependencies, - processorClasses: Map, + processorsPath: string, featuredCriteria: FeaturedContractDataCriteria, addons?: ProcessorAddons ) => { const vars = new ConfigVars(); const options = processorCommand.parse(args).opts(); const config = buildProcessorConfig(vars, dependencies.databaseConfigBuilder, options); - process(config, dependencies, processorClasses, featuredCriteria, addons).catch(log); + process(config, dependencies, processorsPath, featuredCriteria, addons).catch(log); }; diff --git a/yarn.lock b/yarn.lock index cd06b77..810b515 100644 --- a/yarn.lock +++ b/yarn.lock @@ -28,10 +28,10 @@ nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/history-tools-common@^0.0.20": - version "0.0.20" - resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.20/2fb02e82270e67e1d5513b14dcd9fc530ef1838b#2fb02e82270e67e1d5513b14dcd9fc530ef1838b" - integrity sha512-IcisJdvUP63+z6B3mcjWKtM93RiJ1cjv5rJWqfJsVIz1j9QeKTkIuIIGeYDsHeV5ZUuZEmQJP4RqJFyMiYllIw== +"@alien-worlds/history-tools-common@^0.0.21": + version "0.0.21" + resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.21/6c35a44511732009228ef2d36097ba55adecb010#6c35a44511732009228ef2d36097ba55adecb010" + integrity sha512-xc/rmlrI11R7VaPD0ReyqkJRiFSomo/9TR3UqKED3xBuXsC/v/4bo4Zl+IsdebHTFprm2CZwV86G46h2b7P/aA== dependencies: "@alien-worlds/api-core" "^0.0.146" "@alien-worlds/block-reader" "^0.0.6" From 8d1d795bec0a5b33189038c7025b9eaaf024d517 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 14:32:54 +0200 Subject: [PATCH 012/107] 0.0.118 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ca4e10c..48fd07e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.117", + "version": "0.0.118", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 7a910721ee4efedf9755fbab039ae13349e668f0 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 18:18:24 +0200 Subject: [PATCH 013/107] update criteria load --- package.json | 2 +- src/bootstrap/start-bootstrap.ts | 17 ++++++++++++----- src/processor/start-processor.ts | 9 ++++----- yarn.lock | 8 ++++---- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index 48fd07e..8b9991f 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/history-tools-common": "^0.0.21", + "@alien-worlds/history-tools-common": "^0.0.24", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", diff --git a/src/bootstrap/start-bootstrap.ts b/src/bootstrap/start-bootstrap.ts index 5887696..9c7ef09 100644 --- a/src/bootstrap/start-bootstrap.ts +++ b/src/bootstrap/start-bootstrap.ts @@ -20,8 +20,8 @@ import { bootstrapCommand } from './bootstrap.command'; import { BootstrapConfig, BootstrapDependencies, - FeaturedContractDataCriteria, FeaturedUtils, + MissingCriteriaError, Mode, } from '@alien-worlds/history-tools-common'; @@ -41,10 +41,17 @@ import { export const bootstrap = async ( config: BootstrapConfig, dependencies: BootstrapDependencies, - featuredCriteria: FeaturedContractDataCriteria + featuredCriteriaPath: string ): Promise => { const { mode } = config; log(`Bootstrap "${mode}" mode ... [starting]`); + + const featuredCriteria = await FeaturedUtils.fetchCriteria(featuredCriteriaPath); + + if (!featuredCriteria) { + throw new MissingCriteriaError(featuredCriteriaPath); + } + const featuredContracts = FeaturedUtils.readFeaturedContracts(featuredCriteria); await dependencies.initialize(config, featuredCriteria); @@ -113,15 +120,15 @@ export const bootstrap = async ( * * @param {string[]} args The command line args for bootstrap. * @param {BootstrapDependencies} dependencies The bootstrap process dependencies. - * @param {FeaturedContractDataCriteria} featuredCriteria + * @param {string} featuredCriteriaPath */ export const startBootstrap = ( args: string[], dependencies: BootstrapDependencies, - featuredCriteria: FeaturedContractDataCriteria + featuredCriteriaPath: string ) => { const vars = new ConfigVars(); const options = bootstrapCommand.parse(args).opts(); const config = buildBootstrapConfig(vars, dependencies.databaseConfigBuilder, options); - bootstrap(config, dependencies, featuredCriteria).catch(log); + bootstrap(config, dependencies, featuredCriteriaPath).catch(log); }; diff --git a/src/processor/start-processor.ts b/src/processor/start-processor.ts index 803b022..7c4bea6 100644 --- a/src/processor/start-processor.ts +++ b/src/processor/start-processor.ts @@ -5,7 +5,6 @@ import { ProcessorAddons, ProcessorConfig, ProcessorDependencies, - WorkerClass, WorkerPool, log, } from '@alien-worlds/history-tools-common'; @@ -24,14 +23,14 @@ export const process = async ( config: ProcessorConfig, dependencies: ProcessorDependencies, processorsPath: string, - featuredCriteria: FeaturedContractDataCriteria, + featuredCriteriaPath: string, addons: ProcessorAddons = {} ) => { log(`Processor ... [starting]`); const initResult = await dependencies.initialize( config, - featuredCriteria, + featuredCriteriaPath, processorsPath, addons ); @@ -76,11 +75,11 @@ export const startProcessor = ( args: string[], dependencies: ProcessorDependencies, processorsPath: string, - featuredCriteria: FeaturedContractDataCriteria, + featuredCriteriaPath: string, addons?: ProcessorAddons ) => { const vars = new ConfigVars(); const options = processorCommand.parse(args).opts(); const config = buildProcessorConfig(vars, dependencies.databaseConfigBuilder, options); - process(config, dependencies, processorsPath, featuredCriteria, addons).catch(log); + process(config, dependencies, processorsPath, featuredCriteriaPath, addons).catch(log); }; diff --git a/yarn.lock b/yarn.lock index 810b515..6de9ab7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -28,10 +28,10 @@ nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/history-tools-common@^0.0.21": - version "0.0.21" - resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.21/6c35a44511732009228ef2d36097ba55adecb010#6c35a44511732009228ef2d36097ba55adecb010" - integrity sha512-xc/rmlrI11R7VaPD0ReyqkJRiFSomo/9TR3UqKED3xBuXsC/v/4bo4Zl+IsdebHTFprm2CZwV86G46h2b7P/aA== +"@alien-worlds/history-tools-common@^0.0.24": + version "0.0.24" + resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.24/7a3ff20ed022f35bc17c81e041222003d12420ba#7a3ff20ed022f35bc17c81e041222003d12420ba" + integrity sha512-jsgHofiaQiZdEuSBW+u6qdVkPZ4FA5Ng7VGeMujm6c2ZIlDzRhrdzQ3jOZPPrL1Bm96rZ5lnbqy90+B5rB/QNg== dependencies: "@alien-worlds/api-core" "^0.0.146" "@alien-worlds/block-reader" "^0.0.6" From 688249047949817fe8a09d9fd15b38f9e0690de8 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 18:18:39 +0200 Subject: [PATCH 014/107] 0.0.119 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8b9991f..4812170 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.118", + "version": "0.0.119", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 171839ef9c12b0a557503fff702554b0c1c1541b Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 18:29:40 +0200 Subject: [PATCH 015/107] update filter --- src/filter/filter.types.ts | 4 ++-- src/filter/filter.worker-loader.ts | 4 ++-- src/filter/start-filter.ts | 9 ++++----- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/filter/filter.types.ts b/src/filter/filter.types.ts index 0aef439..c63bdb8 100644 --- a/src/filter/filter.types.ts +++ b/src/filter/filter.types.ts @@ -1,8 +1,8 @@ -import { FeaturedContractDataCriteria, FilterConfig } from '@alien-worlds/history-tools-common'; +import { FilterConfig } from '@alien-worlds/history-tools-common'; export type FilterSharedData = { config: FilterConfig; - featuredCriteria: FeaturedContractDataCriteria; + featuredCriteriaPath: string; }; export type FilterCommandOptions = { diff --git a/src/filter/filter.worker-loader.ts b/src/filter/filter.worker-loader.ts index 37542bc..b1b491c 100644 --- a/src/filter/filter.worker-loader.ts +++ b/src/filter/filter.worker-loader.ts @@ -8,8 +8,8 @@ export default class FilterWorkerLoader extends DefaultWorkerLoader< FilterWorkerLoaderDependencies > { public async setup(sharedData: FilterSharedData): Promise { - const { config, featuredCriteria } = sharedData; - await super.setup(sharedData, config, featuredCriteria); + const { config, featuredCriteriaPath } = sharedData; + await super.setup(sharedData, config, featuredCriteriaPath); } public async load(): Promise { diff --git a/src/filter/start-filter.ts b/src/filter/start-filter.ts index 33500d3..04f42fe 100644 --- a/src/filter/start-filter.ts +++ b/src/filter/start-filter.ts @@ -12,7 +12,6 @@ import { filterWorkerLoaderPath } from './filter.consts'; import { BroadcastMessage, ConfigVars, - FeaturedContractDataCriteria, FilterConfig, FilterDependencies, WorkerPool, @@ -22,7 +21,7 @@ import { export const filter = async ( config: FilterConfig, dependencies: FilterDependencies, - featuredCriteria: FeaturedContractDataCriteria, + featuredCriteriaPath: string, addons?: FilterAddons ) => { log(`Filter ... [starting]`); @@ -42,7 +41,7 @@ export const filter = async ( const workerPool = await WorkerPool.create({ ...config.workers, - sharedData: { config, matchers, featuredCriteria }, + sharedData: { config, matchers, featuredCriteriaPath }, workerLoaderPath: workerLoaderPath || filterWorkerLoaderPath, workerLoaderDependenciesPath, }); @@ -74,12 +73,12 @@ export const filter = async ( export const startFilter = ( args: string[], dependencies: FilterDependencies, - featuredCriteria: FeaturedContractDataCriteria, + featuredCriteriaPath: string, addons?: FilterAddons ) => { const vars = new ConfigVars(); const options = filterCommand.parse(args).opts(); const config = buildFilterConfig(vars, dependencies.databaseConfigBuilder, options); - filter(config, dependencies, featuredCriteria, addons).catch(log); + filter(config, dependencies, featuredCriteriaPath, addons).catch(log); }; From 4a48cd279960a36eb202435cc1fd9ef28000c1b2 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 18:30:29 +0200 Subject: [PATCH 016/107] 0.0.120 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4812170..60c8fa8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.119", + "version": "0.0.120", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 3b3d9c40649c443721b2f7450d5ab2a0b6c331ae Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 18:38:48 +0200 Subject: [PATCH 017/107] reflect-metadata --- src/api/start-api.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/api/start-api.ts b/src/api/start-api.ts index 6d8e53c..cbea433 100644 --- a/src/api/start-api.ts +++ b/src/api/start-api.ts @@ -1,3 +1,5 @@ +import 'reflect-metadata'; + import { Route } from '@alien-worlds/api-core'; import { Api } from './api'; import { ApiConfig } from './api.types'; From 10ed5828831579a4df7e6cf805e6c5e88f6c3c74 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 18:39:11 +0200 Subject: [PATCH 018/107] 0.0.121 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 60c8fa8..643dff5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.120", + "version": "0.0.121", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 1b729f600398ec67f13ee960c705b35a75727259 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 19:15:55 +0200 Subject: [PATCH 019/107] update common --- package.json | 2 +- yarn.lock | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 643dff5..8207489 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/history-tools-common": "^0.0.24", + "@alien-worlds/history-tools-common": "^0.0.25", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", diff --git a/yarn.lock b/yarn.lock index 6de9ab7..b237f45 100644 --- a/yarn.lock +++ b/yarn.lock @@ -28,15 +28,16 @@ nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/history-tools-common@^0.0.24": - version "0.0.24" - resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.24/7a3ff20ed022f35bc17c81e041222003d12420ba#7a3ff20ed022f35bc17c81e041222003d12420ba" - integrity sha512-jsgHofiaQiZdEuSBW+u6qdVkPZ4FA5Ng7VGeMujm6c2ZIlDzRhrdzQ3jOZPPrL1Bm96rZ5lnbqy90+B5rB/QNg== +"@alien-worlds/history-tools-common@^0.0.25": + version "0.0.25" + resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.25/3b00cd58d7f198d4e3348ebe8254ae6aacd60aac#3b00cd58d7f198d4e3348ebe8254ae6aacd60aac" + integrity sha512-qcQHaut8NWxnnPn/pD12LHfXP0lXf1r9ST24ZWULYdrRR1iiku9RTwRlfraCTSxujcVb9wfQ6rP1rLqqKeMUMA== dependencies: "@alien-worlds/api-core" "^0.0.146" "@alien-worlds/block-reader" "^0.0.6" "@alien-worlds/broadcast" "^0.0.6" "@alien-worlds/workers" "^0.0.5" + node-fetch "2" "@alien-worlds/workers@^0.0.5": version "0.0.5" From 9f71be63813e219658a2184ff06ad668f460834d Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 19:16:10 +0200 Subject: [PATCH 020/107] 0.0.122 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8207489..292dfdd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.121", + "version": "0.0.122", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 8579b84acf04217f685a7dc3af5754df5330f49d Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 22:48:40 +0200 Subject: [PATCH 021/107] update common --- package.json | 2 +- yarn.lock | 42 +++++++++++++++++++++--------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/package.json b/package.json index 292dfdd..8f0bdb4 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/history-tools-common": "^0.0.25", + "@alien-worlds/history-tools-common": "^0.0.27", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", diff --git a/yarn.lock b/yarn.lock index b237f45..abceb1c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,40 +2,40 @@ # yarn lockfile v1 -"@alien-worlds/api-core@^0.0.146": - version "0.0.146" - resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.146/fa858d4b0d1ba4a082aef7f3860a472d4cad6fff#fa858d4b0d1ba4a082aef7f3860a472d4cad6fff" - integrity sha512-1J/uPD5XS15r7PlmHT0U6+jbDUq6oPWcRr8wY8Wc4meoAf1Lg4NxVI/SNbevBKCqGhp2fdKuKU+HhTEbAv7e/Q== +"@alien-worlds/api-core@^0.0.147": + version "0.0.147" + resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.147/c0ca8ca537112fbecbbc8ac3f3029e12af4a54d4#c0ca8ca537112fbecbbc8ac3f3029e12af4a54d4" + integrity sha512-VYAZHOeX/rCUjnYLHHH5sL8BN2CCL8xVN+3SIUVmU2idwmU6IOAXE6QSn1UN/rTkl82d6gbiRxdi+yahfSANuQ== dependencies: inversify "^6.0.1" nanoid "^3.0.0" node-fetch "2" reflect-metadata "^0.1.13" -"@alien-worlds/block-reader@^0.0.6": - version "0.0.6" - resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.6/df738d468de2da1707b4128d8bd4f92586e33c1a#df738d468de2da1707b4128d8bd4f92586e33c1a" - integrity sha512-ZDVUUxS2XdOHkvNTv/yhBFnlGnS4ynaHa2Qbg3sa6ehoftAmvREKvEGhmC1AxxyNMv9L3DVSi461HZnMpiox2Q== +"@alien-worlds/block-reader@^0.0.7": + version "0.0.7" + resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.7/72e8a6f1cbcb138b6e4fd74f96e8461c5fdad15a#72e8a6f1cbcb138b6e4fd74f96e8461c5fdad15a" + integrity sha512-nAHVaY3imaAszAsPJNOzRgiZRes62IZghHw3tOMw9A3emiNo4+YL8JusKeWy7eUXVn5tUOPzONJK2MbGzRPwkQ== dependencies: - "@alien-worlds/api-core" "^0.0.146" + "@alien-worlds/api-core" "^0.0.147" -"@alien-worlds/broadcast@^0.0.6": - version "0.0.6" - resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.6/4e5a9a14dd09815fbffe6b90969d0754438d0e52#4e5a9a14dd09815fbffe6b90969d0754438d0e52" - integrity sha512-h6/B/TPDkqICklH9bBsNBizHl/9xWdowMWYDT6xxPVGy/YPMwk0LAu136yxLclhCsmv5MlS1+OHPCpfrtpEREw== +"@alien-worlds/broadcast@^0.0.7": + version "0.0.7" + resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.7/ba6985a99ae240c41445fa01a666fb1ef23d78d3#ba6985a99ae240c41445fa01a666fb1ef23d78d3" + integrity sha512-nMQJlzOQ44+PK9U79Bh82s/rgnut5eWLQlmP6vGKkv11gnyBEQEWrbgZJpcEItTOAPmihsNsV1ZWNLbh7ImYXA== dependencies: - "@alien-worlds/api-core" "^0.0.146" + "@alien-worlds/api-core" "^0.0.147" nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/history-tools-common@^0.0.25": - version "0.0.25" - resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.25/3b00cd58d7f198d4e3348ebe8254ae6aacd60aac#3b00cd58d7f198d4e3348ebe8254ae6aacd60aac" - integrity sha512-qcQHaut8NWxnnPn/pD12LHfXP0lXf1r9ST24ZWULYdrRR1iiku9RTwRlfraCTSxujcVb9wfQ6rP1rLqqKeMUMA== +"@alien-worlds/history-tools-common@^0.0.27": + version "0.0.27" + resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.27/725467f90e91c784dee24219892ef221eb698fdf#725467f90e91c784dee24219892ef221eb698fdf" + integrity sha512-yfpB+HTibf3vjPCtCCUlJ+f2s1egrqEyH7FI7Lwq6XSOHKl1GRcOEmRIVry6htHmEUvzcbRIwkKgTbop3H9WVQ== dependencies: - "@alien-worlds/api-core" "^0.0.146" - "@alien-worlds/block-reader" "^0.0.6" - "@alien-worlds/broadcast" "^0.0.6" + "@alien-worlds/api-core" "^0.0.147" + "@alien-worlds/block-reader" "^0.0.7" + "@alien-worlds/broadcast" "^0.0.7" "@alien-worlds/workers" "^0.0.5" node-fetch "2" From b90363e6d7f4d3f475d7ac3cf304a15910927b7d Mon Sep 17 00:00:00 2001 From: rkamysz Date: Sun, 25 Jun 2023 22:48:57 +0200 Subject: [PATCH 022/107] 0.0.123 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8f0bdb4..70c1e8a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.122", + "version": "0.0.123", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From dc0cf252d2af0ff3d09f8d54528636edeb6ffc59 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 26 Jun 2023 10:21:40 +0200 Subject: [PATCH 023/107] update common --- package.json | 2 +- yarn.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 70c1e8a..e6b39ae 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/history-tools-common": "^0.0.27", + "@alien-worlds/history-tools-common": "^0.0.28", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", diff --git a/yarn.lock b/yarn.lock index abceb1c..c97841a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -28,21 +28,21 @@ nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/history-tools-common@^0.0.27": - version "0.0.27" - resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.27/725467f90e91c784dee24219892ef221eb698fdf#725467f90e91c784dee24219892ef221eb698fdf" - integrity sha512-yfpB+HTibf3vjPCtCCUlJ+f2s1egrqEyH7FI7Lwq6XSOHKl1GRcOEmRIVry6htHmEUvzcbRIwkKgTbop3H9WVQ== +"@alien-worlds/history-tools-common@^0.0.28": + version "0.0.28" + resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.28/efd44ed6e5d62f7560e771a927b0946f17cbe99e#efd44ed6e5d62f7560e771a927b0946f17cbe99e" + integrity sha512-L6G456Ox3wMxKdI99n8rEKsw50PwYqc1rfHgEkk9R5v0ndsIJr+bdzlmk2hJciqDRtbnECxPjmCLluiqxdax/Q== dependencies: "@alien-worlds/api-core" "^0.0.147" "@alien-worlds/block-reader" "^0.0.7" "@alien-worlds/broadcast" "^0.0.7" - "@alien-worlds/workers" "^0.0.5" + "@alien-worlds/workers" "^0.0.6" node-fetch "2" -"@alien-worlds/workers@^0.0.5": - version "0.0.5" - resolved "https://npm.pkg.github.com/download/@alien-worlds/workers/0.0.5/5df38161ca64790c61ca0d9eb72db98f842a6e09#5df38161ca64790c61ca0d9eb72db98f842a6e09" - integrity sha512-Ar+Z8lTT0Li/4SCLt0IajYWIxSAn7D8IFzyvhiczZTiVFCmxvMZSsHlxeiQpEZqUoNjQre3oozEP+SLW1Rolvw== +"@alien-worlds/workers@^0.0.6": + version "0.0.6" + resolved "https://npm.pkg.github.com/download/@alien-worlds/workers/0.0.6/366bdf77db081b07bf34617180cc6848c1031620#366bdf77db081b07bf34617180cc6848c1031620" + integrity sha512-2lvT/D3NBLMfimbfGak2MaFTV9IEu+ByoS5VsrHHRnzKelL9DvBf+yXsPel2pFdTcxhZxe2RhITedV9mutD/0Q== dependencies: async "^3.2.4" ts-node "^10.9.1" From dada02a20852d7ce10a3ce0ef141c1ee1b11c099 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 26 Jun 2023 10:21:56 +0200 Subject: [PATCH 024/107] 0.0.124 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e6b39ae..0a95358 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.123", + "version": "0.0.124", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 08b9d0e7f409172a6bf6063e21c525e572b970d4 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 26 Jun 2023 10:50:20 +0200 Subject: [PATCH 025/107] update common --- package.json | 2 +- yarn.lock | 42 +++++++++++++++++++++--------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/package.json b/package.json index 0a95358..4c5a770 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/history-tools-common": "^0.0.28", + "@alien-worlds/history-tools-common": "^0.0.30", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", diff --git a/yarn.lock b/yarn.lock index c97841a..1de02c3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,40 +2,40 @@ # yarn lockfile v1 -"@alien-worlds/api-core@^0.0.147": - version "0.0.147" - resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.147/c0ca8ca537112fbecbbc8ac3f3029e12af4a54d4#c0ca8ca537112fbecbbc8ac3f3029e12af4a54d4" - integrity sha512-VYAZHOeX/rCUjnYLHHH5sL8BN2CCL8xVN+3SIUVmU2idwmU6IOAXE6QSn1UN/rTkl82d6gbiRxdi+yahfSANuQ== +"@alien-worlds/api-core@^0.0.148": + version "0.0.148" + resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.148/96bb0a0e64166de8e0f703caecf7806981726473#96bb0a0e64166de8e0f703caecf7806981726473" + integrity sha512-5wv76cDFAql7eu6qFT1l2+/G8sPa3EK8SlgkTgMZYpylBRRNZSn1t7LYRd3H880li3k+lkmPl11p7HpOSaY2AQ== dependencies: inversify "^6.0.1" nanoid "^3.0.0" node-fetch "2" reflect-metadata "^0.1.13" -"@alien-worlds/block-reader@^0.0.7": - version "0.0.7" - resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.7/72e8a6f1cbcb138b6e4fd74f96e8461c5fdad15a#72e8a6f1cbcb138b6e4fd74f96e8461c5fdad15a" - integrity sha512-nAHVaY3imaAszAsPJNOzRgiZRes62IZghHw3tOMw9A3emiNo4+YL8JusKeWy7eUXVn5tUOPzONJK2MbGzRPwkQ== +"@alien-worlds/block-reader@^0.0.8": + version "0.0.8" + resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.8/0896ab0d9c450e04a98cae59ecb5ac9112845284#0896ab0d9c450e04a98cae59ecb5ac9112845284" + integrity sha512-QMyQjvUG//3v6w3ZrMxJ+x5j3zgZeU1QScFOlNaHh/bGi6jGXI820xrvaxfHyCk2xT0kTP815zK7RluA/JA8WA== dependencies: - "@alien-worlds/api-core" "^0.0.147" + "@alien-worlds/api-core" "^0.0.148" -"@alien-worlds/broadcast@^0.0.7": - version "0.0.7" - resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.7/ba6985a99ae240c41445fa01a666fb1ef23d78d3#ba6985a99ae240c41445fa01a666fb1ef23d78d3" - integrity sha512-nMQJlzOQ44+PK9U79Bh82s/rgnut5eWLQlmP6vGKkv11gnyBEQEWrbgZJpcEItTOAPmihsNsV1ZWNLbh7ImYXA== +"@alien-worlds/broadcast@^0.0.8": + version "0.0.8" + resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.8/7a81f8e8a23232580b3682f9ab1c6ac638224cf4#7a81f8e8a23232580b3682f9ab1c6ac638224cf4" + integrity sha512-nxTHPPKL4SCKhqz/lX7jl+O/o63kFWQc5qWpS1/FK8V5UMXChNmk2Tx2BWxiiH6Bj3nUobdboDmQ0Qyn0pullA== dependencies: - "@alien-worlds/api-core" "^0.0.147" + "@alien-worlds/api-core" "^0.0.148" nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/history-tools-common@^0.0.28": - version "0.0.28" - resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.28/efd44ed6e5d62f7560e771a927b0946f17cbe99e#efd44ed6e5d62f7560e771a927b0946f17cbe99e" - integrity sha512-L6G456Ox3wMxKdI99n8rEKsw50PwYqc1rfHgEkk9R5v0ndsIJr+bdzlmk2hJciqDRtbnECxPjmCLluiqxdax/Q== +"@alien-worlds/history-tools-common@^0.0.30": + version "0.0.30" + resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.30/e1fb7f2ca1ee3edbf2c3c6382ed1f702669d6d0a#e1fb7f2ca1ee3edbf2c3c6382ed1f702669d6d0a" + integrity sha512-idXsSZn7F3Y+EXG6r15Za9QE8fyf5ZfCgZfG/t6swFjf1vc6DeSan4jEBw/Jbp55RTv5nuuf2rpaJQypxpGa0A== dependencies: - "@alien-worlds/api-core" "^0.0.147" - "@alien-worlds/block-reader" "^0.0.7" - "@alien-worlds/broadcast" "^0.0.7" + "@alien-worlds/api-core" "^0.0.148" + "@alien-worlds/block-reader" "^0.0.8" + "@alien-worlds/broadcast" "^0.0.8" "@alien-worlds/workers" "^0.0.6" node-fetch "2" From eb83c1a67684798d0141ff6214ec9b20629f9999 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 26 Jun 2023 10:50:42 +0200 Subject: [PATCH 026/107] 0.0.125 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4c5a770..351d653 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.124", + "version": "0.0.125", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 01eea496a28ea2b20f14999e2187076b20f0a17f Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 26 Jun 2023 11:03:07 +0200 Subject: [PATCH 027/107] updat ecommon --- package.json | 2 +- yarn.lock | 42 +++++++++++++++++++++--------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/package.json b/package.json index 351d653..ed7fa01 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/history-tools-common": "^0.0.30", + "@alien-worlds/history-tools-common": "^0.0.31", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", diff --git a/yarn.lock b/yarn.lock index 1de02c3..e855c21 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,40 +2,40 @@ # yarn lockfile v1 -"@alien-worlds/api-core@^0.0.148": - version "0.0.148" - resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.148/96bb0a0e64166de8e0f703caecf7806981726473#96bb0a0e64166de8e0f703caecf7806981726473" - integrity sha512-5wv76cDFAql7eu6qFT1l2+/G8sPa3EK8SlgkTgMZYpylBRRNZSn1t7LYRd3H880li3k+lkmPl11p7HpOSaY2AQ== +"@alien-worlds/api-core@^0.0.149": + version "0.0.149" + resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.149/2dfd786f1bdc71df887edfe1f0220098378a4699#2dfd786f1bdc71df887edfe1f0220098378a4699" + integrity sha512-BUy43wd9LealXTwGI6ao4g8qtsyC67YnGviaWFvtEehhAEoSp6ayhwx09LCvTZirqJw6xtyfw+j1yyxgnwQ+iQ== dependencies: inversify "^6.0.1" nanoid "^3.0.0" node-fetch "2" reflect-metadata "^0.1.13" -"@alien-worlds/block-reader@^0.0.8": - version "0.0.8" - resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.8/0896ab0d9c450e04a98cae59ecb5ac9112845284#0896ab0d9c450e04a98cae59ecb5ac9112845284" - integrity sha512-QMyQjvUG//3v6w3ZrMxJ+x5j3zgZeU1QScFOlNaHh/bGi6jGXI820xrvaxfHyCk2xT0kTP815zK7RluA/JA8WA== +"@alien-worlds/block-reader@^0.0.9": + version "0.0.9" + resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.9/694ed6865bc5ab90bab3b954c06fc452425c8e97#694ed6865bc5ab90bab3b954c06fc452425c8e97" + integrity sha512-lkOGN4gTUisWqbHjoHzAgBcvrJF/hCYkzONtbyARoCMw0SB7RWlXs4WD7n0mOeXP7GSK2SN/IH72No+AGPKn8g== dependencies: - "@alien-worlds/api-core" "^0.0.148" + "@alien-worlds/api-core" "^0.0.149" -"@alien-worlds/broadcast@^0.0.8": - version "0.0.8" - resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.8/7a81f8e8a23232580b3682f9ab1c6ac638224cf4#7a81f8e8a23232580b3682f9ab1c6ac638224cf4" - integrity sha512-nxTHPPKL4SCKhqz/lX7jl+O/o63kFWQc5qWpS1/FK8V5UMXChNmk2Tx2BWxiiH6Bj3nUobdboDmQ0Qyn0pullA== +"@alien-worlds/broadcast@^0.0.9": + version "0.0.9" + resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.9/7fe15f2b9242e46a3c7a8dd613b0a168ab9e39c1#7fe15f2b9242e46a3c7a8dd613b0a168ab9e39c1" + integrity sha512-ZiE65+d4F6r6/2IlIO8n4/lYM3N/fqf69ip0cEr1zYrl7El+4B8MOJufvDpVK4870edhjVbseaSXcSeNA3xcPQ== dependencies: - "@alien-worlds/api-core" "^0.0.148" + "@alien-worlds/api-core" "^0.0.149" nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/history-tools-common@^0.0.30": - version "0.0.30" - resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.30/e1fb7f2ca1ee3edbf2c3c6382ed1f702669d6d0a#e1fb7f2ca1ee3edbf2c3c6382ed1f702669d6d0a" - integrity sha512-idXsSZn7F3Y+EXG6r15Za9QE8fyf5ZfCgZfG/t6swFjf1vc6DeSan4jEBw/Jbp55RTv5nuuf2rpaJQypxpGa0A== +"@alien-worlds/history-tools-common@^0.0.31": + version "0.0.31" + resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.31/3f1fb03b040ac30a5c1eb5dffd93bf01677e3a36#3f1fb03b040ac30a5c1eb5dffd93bf01677e3a36" + integrity sha512-5Q/T/iUBm6HfJ1shl4hyicUQ2cJptWaYb1Q1Yprh9UTb4ngf5p3MyuxtKSQSrXynp0Zy3F4qOHo891mmxW850A== dependencies: - "@alien-worlds/api-core" "^0.0.148" - "@alien-worlds/block-reader" "^0.0.8" - "@alien-worlds/broadcast" "^0.0.8" + "@alien-worlds/api-core" "^0.0.149" + "@alien-worlds/block-reader" "^0.0.9" + "@alien-worlds/broadcast" "^0.0.9" "@alien-worlds/workers" "^0.0.6" node-fetch "2" From 9a9e006d8cc58c2f19cdee147867c2c485e3659d Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 26 Jun 2023 11:03:19 +0200 Subject: [PATCH 028/107] 0.0.126 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ed7fa01..8b3668e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.125", + "version": "0.0.126", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 945859de0fc5b6c3d76ebfda5d1c1b22569db86c Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 26 Jun 2023 11:10:42 +0200 Subject: [PATCH 029/107] update common --- package.json | 2 +- yarn.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 8b3668e..7bc0952 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/history-tools-common": "^0.0.31", + "@alien-worlds/history-tools-common": "^0.0.32", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", diff --git a/yarn.lock b/yarn.lock index e855c21..5c40677 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12,10 +12,10 @@ node-fetch "2" reflect-metadata "^0.1.13" -"@alien-worlds/block-reader@^0.0.9": - version "0.0.9" - resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.9/694ed6865bc5ab90bab3b954c06fc452425c8e97#694ed6865bc5ab90bab3b954c06fc452425c8e97" - integrity sha512-lkOGN4gTUisWqbHjoHzAgBcvrJF/hCYkzONtbyARoCMw0SB7RWlXs4WD7n0mOeXP7GSK2SN/IH72No+AGPKn8g== +"@alien-worlds/block-reader@^0.0.10": + version "0.0.10" + resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.10/519907cc5589a2f108600052f1e2a1ae6f662c02#519907cc5589a2f108600052f1e2a1ae6f662c02" + integrity sha512-+IrlCYkJNVmpVQFQn9VhNRGnpHOGjED5tpzhpw3zliH/MUxsgOxRrPV7SNovK/Sfuhl5VLVMQxgX4XOm12EATA== dependencies: "@alien-worlds/api-core" "^0.0.149" @@ -28,13 +28,13 @@ nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/history-tools-common@^0.0.31": - version "0.0.31" - resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.31/3f1fb03b040ac30a5c1eb5dffd93bf01677e3a36#3f1fb03b040ac30a5c1eb5dffd93bf01677e3a36" - integrity sha512-5Q/T/iUBm6HfJ1shl4hyicUQ2cJptWaYb1Q1Yprh9UTb4ngf5p3MyuxtKSQSrXynp0Zy3F4qOHo891mmxW850A== +"@alien-worlds/history-tools-common@^0.0.32": + version "0.0.32" + resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.32/4bfa1c74ecb4d60815b02cf8b49f407733528b06#4bfa1c74ecb4d60815b02cf8b49f407733528b06" + integrity sha512-t8DOd4CPsyfj31JTHdceyKDhyGtXF2rEWpPDYJci0XknPaano85NDxfN5SJ4tdA3ualrA83+P0VYtAqpfN+KWw== dependencies: "@alien-worlds/api-core" "^0.0.149" - "@alien-worlds/block-reader" "^0.0.9" + "@alien-worlds/block-reader" "^0.0.10" "@alien-worlds/broadcast" "^0.0.9" "@alien-worlds/workers" "^0.0.6" node-fetch "2" From 34066ad11098b1c2659af4aef0b5af6902e35f6a Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 26 Jun 2023 11:11:56 +0200 Subject: [PATCH 030/107] 0.0.127 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7bc0952..233f963 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.126", + "version": "0.0.127", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From cc4f631c9425942103fc6285792be456ca5eedff Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 26 Jun 2023 12:59:19 +0200 Subject: [PATCH 031/107] update start filter --- src/filter/start-filter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/filter/start-filter.ts b/src/filter/start-filter.ts index 04f42fe..8795333 100644 --- a/src/filter/start-filter.ts +++ b/src/filter/start-filter.ts @@ -25,7 +25,7 @@ export const filter = async ( addons?: FilterAddons ) => { log(`Filter ... [starting]`); - const { matchers } = addons; + const { matchers } = addons || {}; const initResult = await dependencies.initialize(config, addons); if (initResult.isFailure) { From 2a49fc1c75035becd2360883bff81107ea2a1dd9 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 26 Jun 2023 12:59:34 +0200 Subject: [PATCH 032/107] 0.0.128 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 233f963..db24bfa 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.127", + "version": "0.0.128", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 3f335002137f039fd564c82625cf9b8cea747b50 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 26 Jun 2023 13:44:54 +0200 Subject: [PATCH 033/107] upadte common --- package.json | 2 +- yarn.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index db24bfa..2cf7f21 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/history-tools-common": "^0.0.32", + "@alien-worlds/history-tools-common": "^0.0.33", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", diff --git a/yarn.lock b/yarn.lock index 5c40677..165ce67 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12,10 +12,10 @@ node-fetch "2" reflect-metadata "^0.1.13" -"@alien-worlds/block-reader@^0.0.10": - version "0.0.10" - resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.10/519907cc5589a2f108600052f1e2a1ae6f662c02#519907cc5589a2f108600052f1e2a1ae6f662c02" - integrity sha512-+IrlCYkJNVmpVQFQn9VhNRGnpHOGjED5tpzhpw3zliH/MUxsgOxRrPV7SNovK/Sfuhl5VLVMQxgX4XOm12EATA== +"@alien-worlds/block-reader@^0.0.11": + version "0.0.11" + resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.11/083351e4057570c3f460d011359473126609fd0a#083351e4057570c3f460d011359473126609fd0a" + integrity sha512-JQevywCapqoNjDbIcWAxnh9qlI9uX2WxlLybkdVlDeFanSy9meffqNUTYdPJPd3a15ZBKjuoPVlfFN8wzmyoaQ== dependencies: "@alien-worlds/api-core" "^0.0.149" @@ -28,13 +28,13 @@ nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/history-tools-common@^0.0.32": - version "0.0.32" - resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.32/4bfa1c74ecb4d60815b02cf8b49f407733528b06#4bfa1c74ecb4d60815b02cf8b49f407733528b06" - integrity sha512-t8DOd4CPsyfj31JTHdceyKDhyGtXF2rEWpPDYJci0XknPaano85NDxfN5SJ4tdA3ualrA83+P0VYtAqpfN+KWw== +"@alien-worlds/history-tools-common@^0.0.33": + version "0.0.33" + resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.33/ea53f2d7cbeb08b2c86f7948aca4d85f78e30090#ea53f2d7cbeb08b2c86f7948aca4d85f78e30090" + integrity sha512-Py2OQ9rsL4CHtsAEoRKhD4xPjzGqZ+sBxYKbtNTWOvGSDGxs9asuKtE7X5LdM9gK0sC1LZ/q7w+s9UtDjcYYdA== dependencies: "@alien-worlds/api-core" "^0.0.149" - "@alien-worlds/block-reader" "^0.0.10" + "@alien-worlds/block-reader" "^0.0.11" "@alien-worlds/broadcast" "^0.0.9" "@alien-worlds/workers" "^0.0.6" node-fetch "2" From c676825af36536e68b82b7e9ea54652496adaceb Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 26 Jun 2023 13:45:09 +0200 Subject: [PATCH 034/107] 0.0.129 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2cf7f21..9578b8d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.128", + "version": "0.0.129", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From d079dfa4f665e43b3ce1a0dfe8ccea12f1df760b Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 26 Jun 2023 21:23:57 +0200 Subject: [PATCH 035/107] update common --- package.json | 2 +- src/filter/filter.worker.ts | 2 +- yarn.lock | 42 ++++++++++++++++++------------------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/package.json b/package.json index 9578b8d..27e70b1 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/history-tools-common": "^0.0.33", + "@alien-worlds/history-tools-common": "^0.0.34", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", diff --git a/src/filter/filter.worker.ts b/src/filter/filter.worker.ts index 0d3ee16..309c69e 100644 --- a/src/filter/filter.worker.ts +++ b/src/filter/filter.worker.ts @@ -219,7 +219,7 @@ export default class FilterWorker extends Worker { const deserializedBlock = serializer.deserializeBlock< BlockModel, BlockModel - >(json, abi.toHex()); + >(json, abi); const { this_block: { block_num }, } = deserializedBlock; diff --git a/yarn.lock b/yarn.lock index 165ce67..dfc4afa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,40 +2,40 @@ # yarn lockfile v1 -"@alien-worlds/api-core@^0.0.149": - version "0.0.149" - resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.149/2dfd786f1bdc71df887edfe1f0220098378a4699#2dfd786f1bdc71df887edfe1f0220098378a4699" - integrity sha512-BUy43wd9LealXTwGI6ao4g8qtsyC67YnGviaWFvtEehhAEoSp6ayhwx09LCvTZirqJw6xtyfw+j1yyxgnwQ+iQ== +"@alien-worlds/api-core@^0.0.151": + version "0.0.151" + resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.151/90f3fda4958c18a705ee34300f47a46f8a3e4f63#90f3fda4958c18a705ee34300f47a46f8a3e4f63" + integrity sha512-EkQjZ34F3srCD97b0TE+Lju05pod5gxPlUnezMV9Bq0R1xp0XLGnbSIqpO6Y2UfIJY7PHpFaHWpce+q2rQs4aA== dependencies: inversify "^6.0.1" nanoid "^3.0.0" node-fetch "2" reflect-metadata "^0.1.13" -"@alien-worlds/block-reader@^0.0.11": - version "0.0.11" - resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.11/083351e4057570c3f460d011359473126609fd0a#083351e4057570c3f460d011359473126609fd0a" - integrity sha512-JQevywCapqoNjDbIcWAxnh9qlI9uX2WxlLybkdVlDeFanSy9meffqNUTYdPJPd3a15ZBKjuoPVlfFN8wzmyoaQ== +"@alien-worlds/block-reader@^0.0.12": + version "0.0.12" + resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.12/4faf97ccc812f157b9472c8ca3017065ee2d21c1#4faf97ccc812f157b9472c8ca3017065ee2d21c1" + integrity sha512-SjQich0xJZ9785Ax4OClxnyKGgkKkAtI/E+TRQV6cjgujVeBCRDYOX+sopVhJ9CyF6uccqHXFm9tQQkE3Al5lw== dependencies: - "@alien-worlds/api-core" "^0.0.149" + "@alien-worlds/api-core" "^0.0.151" -"@alien-worlds/broadcast@^0.0.9": - version "0.0.9" - resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.9/7fe15f2b9242e46a3c7a8dd613b0a168ab9e39c1#7fe15f2b9242e46a3c7a8dd613b0a168ab9e39c1" - integrity sha512-ZiE65+d4F6r6/2IlIO8n4/lYM3N/fqf69ip0cEr1zYrl7El+4B8MOJufvDpVK4870edhjVbseaSXcSeNA3xcPQ== +"@alien-worlds/broadcast@^0.0.10": + version "0.0.10" + resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.10/978171234ce921b91f7951a4a113d56aba9c1f00#978171234ce921b91f7951a4a113d56aba9c1f00" + integrity sha512-ILKkn197zxilBKCml2USIyggYNAHzhojfnic+Rj3fmX0nA0Uzz8hHOHj/IOaGTeP8neaGt1dx8sQCPciNY6xZw== dependencies: - "@alien-worlds/api-core" "^0.0.149" + "@alien-worlds/api-core" "^0.0.151" nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/history-tools-common@^0.0.33": - version "0.0.33" - resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.33/ea53f2d7cbeb08b2c86f7948aca4d85f78e30090#ea53f2d7cbeb08b2c86f7948aca4d85f78e30090" - integrity sha512-Py2OQ9rsL4CHtsAEoRKhD4xPjzGqZ+sBxYKbtNTWOvGSDGxs9asuKtE7X5LdM9gK0sC1LZ/q7w+s9UtDjcYYdA== +"@alien-worlds/history-tools-common@^0.0.34": + version "0.0.34" + resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.34/ecdf1fb537cd313ff78d8e3c0446372f321e285c#ecdf1fb537cd313ff78d8e3c0446372f321e285c" + integrity sha512-WX9kfOQgtXP4YPCRpEKS0Mx0bWuNLWNyJgnkqsbfuYPK7TIrlSEI0kEGRc2w6bOXENyzgx7H2Zws40Lbw655tw== dependencies: - "@alien-worlds/api-core" "^0.0.149" - "@alien-worlds/block-reader" "^0.0.11" - "@alien-worlds/broadcast" "^0.0.9" + "@alien-worlds/api-core" "^0.0.151" + "@alien-worlds/block-reader" "^0.0.12" + "@alien-worlds/broadcast" "^0.0.10" "@alien-worlds/workers" "^0.0.6" node-fetch "2" From d84465b549b463e46bfeca957b5701e8ef24d37e Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 26 Jun 2023 21:24:14 +0200 Subject: [PATCH 036/107] 0.0.130 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 27e70b1..f373f29 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.129", + "version": "0.0.130", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 1f71c13e3cff3960deb2b04ae9b539101e29df46 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 27 Jun 2023 09:04:32 +0200 Subject: [PATCH 037/107] update common --- package.json | 2 +- yarn.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index f373f29..438c4d0 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/history-tools-common": "^0.0.34", + "@alien-worlds/history-tools-common": "^0.0.35", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", diff --git a/yarn.lock b/yarn.lock index dfc4afa..df278c2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12,10 +12,10 @@ node-fetch "2" reflect-metadata "^0.1.13" -"@alien-worlds/block-reader@^0.0.12": - version "0.0.12" - resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.12/4faf97ccc812f157b9472c8ca3017065ee2d21c1#4faf97ccc812f157b9472c8ca3017065ee2d21c1" - integrity sha512-SjQich0xJZ9785Ax4OClxnyKGgkKkAtI/E+TRQV6cjgujVeBCRDYOX+sopVhJ9CyF6uccqHXFm9tQQkE3Al5lw== +"@alien-worlds/block-reader@^0.0.13": + version "0.0.13" + resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.13/a21ab854a481e00b0f68e22413051d093733433e#a21ab854a481e00b0f68e22413051d093733433e" + integrity sha512-gmSlI1gOojxFT/JPwETX6nhg4c5gKLJuGT/b7NjFULkXh4LkpdkKmj3NI0wgo29HhmfDJFUWgQCgCuILeXc95w== dependencies: "@alien-worlds/api-core" "^0.0.151" @@ -28,13 +28,13 @@ nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/history-tools-common@^0.0.34": - version "0.0.34" - resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.34/ecdf1fb537cd313ff78d8e3c0446372f321e285c#ecdf1fb537cd313ff78d8e3c0446372f321e285c" - integrity sha512-WX9kfOQgtXP4YPCRpEKS0Mx0bWuNLWNyJgnkqsbfuYPK7TIrlSEI0kEGRc2w6bOXENyzgx7H2Zws40Lbw655tw== +"@alien-worlds/history-tools-common@^0.0.35": + version "0.0.35" + resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.35/5f338c6bb414af0a28e9718cf21900c3f1702abd#5f338c6bb414af0a28e9718cf21900c3f1702abd" + integrity sha512-iJ7yY50dkSj+ExgLu/9o5uQYKE/96M81vbbdPWfL9gFWfr4H3qDwMvxLHLXv0b1l0nND5mDm8MbXYo0nvEXVTA== dependencies: "@alien-worlds/api-core" "^0.0.151" - "@alien-worlds/block-reader" "^0.0.12" + "@alien-worlds/block-reader" "^0.0.13" "@alien-worlds/broadcast" "^0.0.10" "@alien-worlds/workers" "^0.0.6" node-fetch "2" From 754bf7105d193716e593206392d3dacf0e644bf6 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 27 Jun 2023 09:04:54 +0200 Subject: [PATCH 038/107] 0.0.131 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 438c4d0..f498443 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.130", + "version": "0.0.131", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From fa0692c15f905b99f7dd9c21f09b838065a27900 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 27 Jun 2023 10:03:22 +0200 Subject: [PATCH 039/107] update common --- package.json | 2 +- yarn.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index f498443..ef614e3 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/history-tools-common": "^0.0.35", + "@alien-worlds/history-tools-common": "^0.0.36", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", diff --git a/yarn.lock b/yarn.lock index df278c2..31eec64 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12,10 +12,10 @@ node-fetch "2" reflect-metadata "^0.1.13" -"@alien-worlds/block-reader@^0.0.13": - version "0.0.13" - resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.13/a21ab854a481e00b0f68e22413051d093733433e#a21ab854a481e00b0f68e22413051d093733433e" - integrity sha512-gmSlI1gOojxFT/JPwETX6nhg4c5gKLJuGT/b7NjFULkXh4LkpdkKmj3NI0wgo29HhmfDJFUWgQCgCuILeXc95w== +"@alien-worlds/block-reader@^0.0.14": + version "0.0.14" + resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.14/ef19b83824dae6eeebf08a2d3bd77e430e493759#ef19b83824dae6eeebf08a2d3bd77e430e493759" + integrity sha512-F/E94qnZ8Ok9UcKbCzEmARw4VKjYXoox6BPR48g9L7RjtGvBmFii1TIB3+/aosExEHOLmnhD9wA+TBP+NnmEPg== dependencies: "@alien-worlds/api-core" "^0.0.151" @@ -28,13 +28,13 @@ nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/history-tools-common@^0.0.35": - version "0.0.35" - resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.35/5f338c6bb414af0a28e9718cf21900c3f1702abd#5f338c6bb414af0a28e9718cf21900c3f1702abd" - integrity sha512-iJ7yY50dkSj+ExgLu/9o5uQYKE/96M81vbbdPWfL9gFWfr4H3qDwMvxLHLXv0b1l0nND5mDm8MbXYo0nvEXVTA== +"@alien-worlds/history-tools-common@^0.0.36": + version "0.0.36" + resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.36/e649d1444fe8cdda4a7bdc6f20e0a9c8d436438a#e649d1444fe8cdda4a7bdc6f20e0a9c8d436438a" + integrity sha512-BB34qD7uHAUMODlTjNfltABIjD0keOF+zCryPjzV8tHs8cXSPuOzWDEIrEmex3FCpZk70CAkpCoOcqux1tiFAQ== dependencies: "@alien-worlds/api-core" "^0.0.151" - "@alien-worlds/block-reader" "^0.0.13" + "@alien-worlds/block-reader" "^0.0.14" "@alien-worlds/broadcast" "^0.0.10" "@alien-worlds/workers" "^0.0.6" node-fetch "2" From 9b0c81325142753e2daaf7282f7d69a5a6da0926 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 27 Jun 2023 10:03:43 +0200 Subject: [PATCH 040/107] 0.0.132 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ef614e3..98fb3cb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.131", + "version": "0.0.132", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 19ad95944911b43bbf5b0f734da402bc9c65b6f3 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 27 Jun 2023 10:30:43 +0200 Subject: [PATCH 041/107] bugfixes --- src/bootstrap/start-bootstrap.ts | 6 +++++- src/processor/start-processor.ts | 10 ++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/start-bootstrap.ts b/src/bootstrap/start-bootstrap.ts index 9c7ef09..dbb6e65 100644 --- a/src/bootstrap/start-bootstrap.ts +++ b/src/bootstrap/start-bootstrap.ts @@ -54,7 +54,11 @@ export const bootstrap = async ( const featuredContracts = FeaturedUtils.readFeaturedContracts(featuredCriteria); - await dependencies.initialize(config, featuredCriteria); + const initResult = await dependencies.initialize(config, featuredCriteria); + + if (initResult.isFailure) { + throw initResult.failure.error; + } const { abis, broadcastClient, blockState, blockchain, featured, scanner } = dependencies; diff --git a/src/processor/start-processor.ts b/src/processor/start-processor.ts index 7c4bea6..a021de5 100644 --- a/src/processor/start-processor.ts +++ b/src/processor/start-processor.ts @@ -39,11 +39,17 @@ export const process = async ( throw initResult.failure.error; } - const { broadcastClient, featuredTraces, featuredDeltas, processorTaskQueue } = - dependencies; + const { + broadcastClient, + featuredTraces, + featuredDeltas, + processorTaskQueue, + workerLoaderDependenciesPath, + } = dependencies; const workerPool = await WorkerPool.create({ ...config.workers, workerLoaderPath: config.processorLoaderPath || processorWorkerLoaderPath, + workerLoaderDependenciesPath, }); const runner = new ProcessorRunner( From 05be8d1822757ecb482a5670a9cb6983200c59de Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 27 Jun 2023 10:30:59 +0200 Subject: [PATCH 042/107] 0.0.133 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 98fb3cb..c38ca9f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.132", + "version": "0.0.133", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From d03f4a066cd8eb17be5398a3685a91b570ed2a22 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 27 Jun 2023 10:48:01 +0200 Subject: [PATCH 043/107] bug fix start processor --- src/processor/start-processor.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/processor/start-processor.ts b/src/processor/start-processor.ts index a021de5..0951e4b 100644 --- a/src/processor/start-processor.ts +++ b/src/processor/start-processor.ts @@ -48,6 +48,7 @@ export const process = async ( } = dependencies; const workerPool = await WorkerPool.create({ ...config.workers, + sharedData: { config, featuredCriteriaPath, processorsPath }, workerLoaderPath: config.processorLoaderPath || processorWorkerLoaderPath, workerLoaderDependenciesPath, }); From 6b787fe7390c02f7030ea36678895043dcc29419 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 27 Jun 2023 10:48:23 +0200 Subject: [PATCH 044/107] 0.0.134 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c38ca9f..39f3687 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.133", + "version": "0.0.134", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From e082778b9bea1ab8b43ac743167242854fc294fe Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 27 Jun 2023 11:20:03 +0200 Subject: [PATCH 045/107] bug fix initialize args order --- src/processor/processor.types.ts | 6 +----- src/processor/processor.worker-loader.ts | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/processor/processor.types.ts b/src/processor/processor.types.ts index 5fbc961..b20cc3f 100644 --- a/src/processor/processor.types.ts +++ b/src/processor/processor.types.ts @@ -1,7 +1,4 @@ -import { - FeaturedContractDataCriteria, - ProcessorConfig, -} from '@alien-worlds/history-tools-common'; +import { ProcessorConfig } from '@alien-worlds/history-tools-common'; export type ProcessorCommandOptions = { threads: number; @@ -9,7 +6,6 @@ export type ProcessorCommandOptions = { export type ProcessorSharedData = { config: ProcessorConfig; - featuredCriteria: FeaturedContractDataCriteria; processorsPath: string; }; diff --git a/src/processor/processor.worker-loader.ts b/src/processor/processor.worker-loader.ts index 6d52d88..c3572f8 100644 --- a/src/processor/processor.worker-loader.ts +++ b/src/processor/processor.worker-loader.ts @@ -15,8 +15,8 @@ export default class ProcessorWorkerLoader extends DefaultWorkerLoader< protected ioc: Container; public async setup(sharedData: ProcessorSharedData): Promise { - const { config, featuredCriteria, processorsPath } = sharedData; - await super.setup(sharedData, config, featuredCriteria, processorsPath); + const { config, processorsPath } = sharedData; + await super.setup(sharedData, config, processorsPath); this.ioc = new Container(); } From abf091ce9f1e0c0d033213aa0d1337a9cc559f9b Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 27 Jun 2023 11:20:17 +0200 Subject: [PATCH 046/107] 0.0.135 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 39f3687..c8cd8e8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.134", + "version": "0.0.135", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 8b1331e7f87a34e9db47a6955eb361161c2a700b Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 27 Jun 2023 11:53:58 +0200 Subject: [PATCH 047/107] run processor on release --- src/processor/processor.runner.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/processor/processor.runner.ts b/src/processor/processor.runner.ts index aa131d9..353bdeb 100644 --- a/src/processor/processor.runner.ts +++ b/src/processor/processor.runner.ts @@ -28,6 +28,8 @@ export class ProcessorRunner { this.next(); } }, 5000); + + workerPool.onWorkerRelease(() => this.next()); } private async assignTask(task: ProcessorTask) { From 5175f3922717e58cad9632bee09180ad527fed35 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 27 Jun 2023 11:54:12 +0200 Subject: [PATCH 048/107] 0.0.136 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c8cd8e8..3547cb1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.135", + "version": "0.0.136", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 89231d0ca233a4eb7991b59781e4c1f93bdb3485 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Thu, 29 Jun 2023 14:47:34 +0200 Subject: [PATCH 049/107] update readme --- .env.template | 32 +++ README.md | 89 +++++++- tutorials/config-vars.md | 49 +++++ tutorials/running-history-tools-docker.md | 6 + tutorials/running-history-tools-locally.md | 44 ++++ tutorials/writing-history-tools.md | 230 +++++++++++++++++++++ 6 files changed, 446 insertions(+), 4 deletions(-) create mode 100644 .env.template create mode 100644 tutorials/config-vars.md create mode 100644 tutorials/running-history-tools-docker.md create mode 100644 tutorials/running-history-tools-locally.md create mode 100644 tutorials/writing-history-tools.md diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..1ddc78d --- /dev/null +++ b/.env.template @@ -0,0 +1,32 @@ +BLOCKCHAIN_ENDPOINT='http://....' +BLOCKCHAIN_CHAIN_ID='' +HYPERION_URL='' +SCANNER_NODES_MAX_CHUNK_SIZE=100 +SCANNER_SCAN_KEY='test' +ABIS_SERVICE_LIMIT=100 +ABIS_SERVICE_FILTER='eosio:setabi' +BLOCK_READER_ENDPOINTS='ws://...' +BLOCK_READER_FETCH_BLOCK=1 +BLOCK_READER_FETCH_DELTAS=1 +BLOCK_READER_FETCH_TRACES=1 +READER_MAX_THREADS=1 +READER_INVIOLABLE_THREADS_COUNT=0 +PROCESSOR_MAX_THREADS=1 +PROCESSOR_INVIOLABLE_THREADS_COUNT=0 +FILTER_MAX_THREADS=1 +FILTER_INVIOLABLE_THREADS_COUNT=0 +PROCESSOR_TASK_QUEUE_CHECK_INTERVAL=5000 +API_PORT=8080 +START_BLOCK=238580000 +END_BLOCK=238581000 +START_FROM_HEAD=0 +MODE='default' +MAX_BLOCK_NUMBER=0xffffffff +UNPROCESSED_BLOCK_QUEUE_MAX_BYTES_SIZE=256000000 +UNPROCESSED_BLOCK_QUEUE_SIZE_CHECK_INTERVAL=2000 +UNPROCESSED_BLOCK_QUEUE_BATCH_SIZE=100 +BROADCAST_PORT=9000 +BROADCAST_HOST='localhost' +MONGO_HOSTS='localhost' +MONGO_PORTS='27017' +MONGO_DB_NAME='history_tools' \ No newline at end of file diff --git a/README.md b/README.md index 279e997..f9172e8 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,90 @@ -# api-history-tools +# History Tools + +This project is part of the Alien Worlds open source initiative, offering a set of tools and processes for downloading and processing blockchain data (transactions and deltas). It is designed to operate in two modes: default (live) and replay. The default mode continuously downloads current block data, while replay mode is designed for data retrieval from a specific range of blocks. + +This package encapsulates the core mechanism, however, complete functionality requires packages that contain implementations of common components and third-party elements. + + +## Dependencies + +- [@alien-worlds/history-tools-common](https://github.com/Alien-Worlds/history-tools-common) +- [async](https://github.com/caolan/async) +- [commander](https://github.com/tj/commander.js) +- [express](https://github.com/expressjs/express) + + +## Table of Contents + +- [Installation](#installation) +- [Processes](#processes) + - [APIs](#apis) + - [Broadcasting](#broadcasting) + - [Bootstraps](#bootstraps) + - [Reader](#reader) + - [Filter](#filter) + - [Processor](#processor) +- [Additional Tools](#additional-tools) + - [Config](#config) +- [Tutorials](#tutorials) +- [Contributing](#contributing) +- [License](#license) + +## Installation + +To add History Tools to your project, use the following command with your favorite package manager: + +```bash +yarn add @alien-worlds/api-history-tools + +``` + +## Processes + +All processes utilize the commander, enabling specific value assignments for individual options or the use of environment variables. + +### APIs + +The API process, currently under development, is intended to provide easy access to downloaded data. This Express.js-based API allows viewing of blockchain data, offering endpoints to retrieve block-specific data, transactions, tables, or data from a specific range according to selected criteria. The API is read-only, it doesn't contain methods that modify the content. + +### Broadcasting + +The broadcasting process creates a server for communication between processes, leveraging the @alien-worlds/broadcast module. Processes inform each other about their state, enabling efficient work coordination. The server facilitates the dispatch of messages defining tasks and states, improving performance when processing a high volume of blockchain data. + +### Bootstraps + +The bootstrap process prepares input data according to provided guidelines and dispatches tasks to other processes using broadcasting, thereby initiating the blockchain data download. + +### Reader + +The Reader process downloads blocks in their undecrypted form based on the given operating mode (live/replay) and block interval. The primary function is to fetch blocks and store them in the database. Multiple reader instances can be created or workers can be used for scaling if not using Docker. + +### Filter + +The Filter process retrieves blocks from the database, verifying their content. If a block contains specific contracts, actions, or tables listed in the configuration file, these data are decoded using a dedicated blockchain serializer and saved as tasks for the Processor in the database. ### Processor -Lorem +The Processor process retrieves tasks from the database, generating appropriate action or delta data based on their content. It coordinates the work of workers who perform the processing. There are two categories of processors: DeltaProcessor and ActionTraceProcessor. The processor creates separate collections in the database for each contract, stock, and delta e.g. `dao.worlds_actions` and `dao.worlds_deltas`. Any processing failures are stored in a separate collection for subsequent review and analysis. + +## Additional Tools + +Besides the main processes, this package also contains tools, such as: + +### Config + +The Config tools are used for generating configuration objects based on values stored in the .env file or environment variables. The list of required options can be found in the file [.env.template](./.env.template). + +## Tutorials + +- [Description of configuration variables](./tutorials/config-vars.md) +- [Writing your own history tools](./tutorials/writing-history-tools.md) +- [Running history tools locally](./tutorials/running-history-tools-locally.md) +- [Running history tools with Docker](./tutorials/running-history-tools-with-docker.md) + +## Contributing + +We welcome contributions from the community. Before contributing, please read through the existing issues on this repository to prevent duplicate submissions. New feature requests and bug reports can be submitted as an issue. If you would like to contribute code, please open a pull request. -### Block Range +## License -Lorem \ No newline at end of file +This project is licensed under the terms of the MIT license. For more information, refer to the [LICENSE](./LICENSE) file. diff --git a/tutorials/config-vars.md b/tutorials/config-vars.md new file mode 100644 index 0000000..eea7564 --- /dev/null +++ b/tutorials/config-vars.md @@ -0,0 +1,49 @@ +# Description of configuration variables + +This is more of a description than a tutorial. Here you will find information about the configuration variables that must be provided for history tools to work. + +[Back to Readme](../README.md) + +Basic Variables + +| **Name** | **Type** | **Description** | **Default** | +| :------- | :------: | --------------- | :---------: | +|`BLOCKCHAIN_ENDPOINT`| _string_ | Blockchain API addres| none | +|`BLOCKCHAIN_CHAIN_ID`| _string_ | Blockchain ID string | none | +|`HYPERION_URL`| _string_ | Hyperion API url | none | +|`BLOCK_READER_ENDPOINTS`| _string_ | Comma separated list of State History Plugin WS paths | none | +|`BLOCK_READER_FETCH_BLOCK`| _number_ | Number (0/1) value that specifies whether to fetch the signed block data | 1 | +|`BLOCK_READER_FETCH_DELTAS`| _number_ | Number (0/1) value that specifies whether to fetch deltas | 1 | +|`BLOCK_READER_FETCH_TRACES`| _number_ | Number (0/1) value that specifies whether to fetch traces | 1 | +|`READER_MAX_THREADS`| _number_ | specify the maximum number of threads dedicated to the reader process | 1 | +|`FILTER_MAX_THREADS`| _number_ | specify the maximum number of threads dedicated to the reader process | 1 | +|`PROCESSOR_MAX_THREADS`| _number_ | specify the maximum number of threads dedicated to the reader process | 1 | +|`API_PORT`| _number_ | History Tools API port number | none | +|`BROADCAST_PORT`| _number_ | Broadcast port number | none | +|`BROADCAST_HOST`| _string_ | Broadcast host | none | +|`DATABASE_HOSTS`| _string_ | Comma separated list of database hosts | none | +|`DATABASE_PORTS`| _string_ | Comma separated list of database ports | none | +|`DATABASE_NAME`| _string_ | Name of the database | none | +|`MODE`| _string_ | History Tools run mode label "default"/"replay" | "default" | +|`SCANNER_SCAN_KEY`| _string_ | Label for scanned blocks in replay mode. It serves the main purpose of separating and keeping in the database logs the history of which blocks were scanned for what purpose. It may happen that blocks in the same instance will have to be downloaded again, for this you need to enter a new label. | none | +|`START_BLOCK`| _number_ | Beginning of block scanning in replay mode. | none | +|`END_BLOCK`| _number_ | End of block scan in replay mode. | none | + + +Advanced Variables + +The following settings are additional for more advanced users who want to tweak the work of the tools to use more available resources + +| **Name** | **Type** | **Description** | **Default** | +| :-------------------- | :------: | --------------------- | :---------: | +| `SCANNER_NODES_MAX_CHUNK_SIZE` | _number_ | | 100 | +| `ABIS_SERVICE_LIMIT` | _number_ | | none | +| `ABIS_SERVICE_FILTER` | _string_ | | "eosio:setabi" | +| `READER_INVIOLABLE_THREADS_COUNT` | _number_ | | 0 | +| `PROCESSOR_INVIOLABLE_THREADS_COUNT` | _number_ | | 0 | +| `FILTER_INVIOLABLE_THREADS_COUNT` | _number_ | | 0 | +| `START_FROM_HEAD` | _number_ | | 0 | +| `UNPROCESSED_BLOCK_QUEUE_MAX_BYTES_SIZE` | _number_ | | 256000000 | +| `UNPROCESSED_BLOCK_QUEUE_SIZE_CHECK_INTERVAL` | _number_ | | 2000 | +| `UNPROCESSED_BLOCK_QUEUE_BATCH_SIZE` | _number_ | | 100 | +| `PROCESSOR_TASK_QUEUE_CHECK_INTERVAL` | _number_ | | 5000 | diff --git a/tutorials/running-history-tools-docker.md b/tutorials/running-history-tools-docker.md new file mode 100644 index 0000000..71b6105 --- /dev/null +++ b/tutorials/running-history-tools-docker.md @@ -0,0 +1,6 @@ +# Running history tools with Docker + +This tutorial provides a comprehensive guide to run History Tools with Docker. + +[Back to Readme](../README.md) + diff --git a/tutorials/running-history-tools-locally.md b/tutorials/running-history-tools-locally.md new file mode 100644 index 0000000..55c025c --- /dev/null +++ b/tutorials/running-history-tools-locally.md @@ -0,0 +1,44 @@ +# Running History Tools locally + +This tutorial provides a comprehensive guide to run History Tools on your local machine. + +[Back to Readme](../README.md) + +If you followed the instructions from the previous tutorial [Writing your own History Tools](./writing-history-tools.md) now to run the tools locally you must first start the database. If you don't have a local server, you can run it on docker. You can use the following content for docker-compose.yml + +```bash +version: "3.2" +services: + mongo: + image: mongo + container_name: 'mongo' + restart: always + mem_limit: 2g + ports: + - '27017:27017' + networks: + - mongo_net + +networks: + mongo_net: + driver: bridge +``` + +Then open a new terminal session for each of the History tools processes and enter the startup command for each of them accordingly: + +```bash +# terminal 1 +yarn broadcast + +# terminal 2 +yarn boot + +# terminal 3 +yarn reader + +# terminal 4 +yarn filter + +# terminal 5 +yarn processor +``` \ No newline at end of file diff --git a/tutorials/writing-history-tools.md b/tutorials/writing-history-tools.md new file mode 100644 index 0000000..78358d8 --- /dev/null +++ b/tutorials/writing-history-tools.md @@ -0,0 +1,230 @@ +# Writing your own History Tools + +This tutorial provides a comprehensive guide to building your own Alien Worlds History Tools, a set of utilities for downloading and processing data from the blockchain. The History Tools are divided into several packages, making it modular and independent from third-party resources. + +[Back to Readme](../README.md) + +## Table of Contents + +1. [Project Preparation](#project-preparation) +2. [Creating a List of Contracts](#creating-a-list-of-contracts) +3. [Process Implementation](#process-implementation) + - [Bootstraps](#bootstraps) + - [Broadcasting](#broadcasting) + - [Reader](#reader) + - [Filter](#filter) + - [Processor](#processor) +4. [Creating Processors](#creating-processors) + +### 1. Project Preparation + +The first step to create history tools is to set up a new project and import the necessary dependencies. + +Your `package.json` file should look similar to this: + +```json +{ + "name": "your-history-tools", + "version": "0.0.1", + "description": "your-description", + "packageManager": "yarn@3.2.3", + "main": "build/index.js", + "types": "build/index.d.ts", + "scripts": { + "broadcast": "node build/broadcast/index.js", + "boot": "node build/bootstrap/index.js", + "reader": "node build/reader/index.js", + "filter": "node build/filter/index.js", + "processor": "node build/processor/index.js", + ... + }, + ... + "dependencies": { + "@alien-worlds/api-history-tools": "^0.0.136", + "@alien-worlds/history-tools-default-dependencies": "^0.0.27", + ... all your contract packages and other typescript dependencies + } +} +``` + +After setting up your `package.json`, define all necessary configuration variables, preferably in an `.env` file. For instructions to complete the `.env` file, see [./envs.md](./envs.md). + +Next, create a new `src` directory and within it, create `bootstrap`, `broadcast`, `filter`, `reader` and `processor` directories. + +### 2. Creating a List of Contracts + +Now you need to define what data you want to download from the blockchain. You can achieve this by creating a separate json file containing the configuration. This json file schema is as follows: + +```json +{ + "traces": [ + { + "shipTraceMessageName": ["transaction_trace_v0"], + "shipActionTraceMessageName": ["action_trace_v0", "action_trace_v1"], + "contract": ["dao.worlds"], + "action": ["*"], + "processor": "DaoWorldsTraceProcessor" + } + ], + "deltas": [ + { + "shipDeltaMessageName": ["table_delta_v0"], + "name": ["contract_row"], + "code": ["dao.worlds"], + "scope": ["*"], + "table": ["*"], + "processor": "DaoWorldsDeltaProcessor" + } + ] +} + +// or shorter version +{ + "traces": [ + { + "contract": ["dao.worlds"], + "action": ["*"], + "processor": "DaoWorldsTraceProcessor" + } + ], + "deltas": [ + { + "code": ["dao.worlds"], + "scope": ["*"], + "table": ["*"], + "processor": "DaoWorldsDeltaProcessor" + } + ] +} +``` + +Here, you can specify many values in the arrays. For example, if you are only interested in specific actions for the `dao.worlds` contract, you can write them, e.g., ["appointcust", "nominate", "firecust"], instead of '*' (get all). This way, you can limit the work of the processor to only the data that interests you. The same applies for deltas. + +### 3. Process Implementation + +Having defined what data you're interested in, the next step is to create processes that will handle the work. + +#### 3.1 Bootstraps + +In the `bootstrap` + +directory, create an `index.ts` file and add the following content: + +```typescript +import { startBootstrap } from '@alien-worlds/api-history-tools'; +import { DefaultBootstrapDependencies } from '@alien-worlds/history-tools-default-dependencies'; +import path from 'path'; + +startBootstrap( + process.argv, + new DefaultBootstrapDependencies(), + path.join(__dirname, '../../your.featured.json') +); +``` + +#### 3.2 Broadcasting + +In the `broadcast` directory, create an `index.ts` file and add the following content: + +```typescript +import { startBroadcast } from '@alien-worlds/api-history-tools'; + +startBroadcast(); +``` + +#### 3.3 Reader + +In the `reader` directory, create an `index.ts` file and add the following content: + +```typescript +import { startReader } from '@alien-worlds/api-history-tools'; +import { DefaultReaderDependencies } from '@alien-worlds/history-tools-default-dependencies'; + +startReader(process.argv, new DefaultReaderDependencies()); +``` + +#### 3.4 Filter + +In the `filter` directory, create an `index.ts` file and add the following content: + +```typescript +import { startFilter } from '@alien-worlds/api-history-tools'; +import { DefaultFilterDependencies } from '@alien-worlds/history-tools-default-dependencies'; +import path from 'path'; + +startFilter( + process.argv, + new DefaultFilterDependencies(), + path.join(__dirname, '../../your.featured.json') +); +``` + +#### 3.5 Processor + +In the `processor` directory, create an `index.ts` file and add the following content: + +```typescript +import { startProcessor } from '@alien-worlds/api-history-tools'; +import { DefaultProcessorDependencies } from '@alien-worlds/history-tools-default-dependencies'; +import path from 'path'; + +startProcessor( + process.argv, + new DefaultProcessorDependencies(), + path.join(__dirname, './processors'), + path.join(__dirname, '../../your.featured.json') +); +``` + +### 4. Creating Processors + +You also need to specify the Processor class that is to be started when the data is read. This is done in the json file you created in Step 2. + +Here is an example: + +```json +"processor": "DaoWorldsDeltaProcessor" +``` + +Now, create this class and save it where the worker loader processor can find and instantiate it. In the `processors` directory, create an `index.ts` file and export all processors contained in the directory. + +```typescript +// ./processors/index.ts +export * from './dao-worlds.trace-processor'; +export * from './dao-worlds.delta-processor'; +... + +// ./processors/dao-worlds.trace-processor.ts +import { ActionTraceProcessor } from '@alien-worlds/api-history-tools'; +import { ProcessorTaskModel, log } from '@alien-worlds/history-tools-common'; + +export class DaoWorldsTraceProcessor extends ActionTraceProcessor { + public async run(model: ProcessorTaskModel): Promise { + try { + //... all of your operations + this.resolve(); + } catch (error) { + this.reject(error); + } + } +} + +// ./processors/dao-worlds.delta-processor.ts +import { DeltaProcessor } from '@alien-worlds/api-history-tools'; +import { ProcessorTaskModel, log } from '@alien-worlds/history-tools-common'; + +export class DaoWorldsDeltaProcessor extends DeltaProcessor { + public async run(model: ProcessorTaskModel): Promise { + try { + //... all of your operations + this.resolve(); + } catch (error) { + this.reject + +(error); + } + } +} +``` + +With these steps completed, you now have all the necessary components to run your history tools locally or via Docker. Please keep in mind that the project is constantly being updated and some elements may change over time. From f53e2ed826e3092e29633025326783b1912ae93b Mon Sep 17 00:00:00 2001 From: rkamysz Date: Thu, 29 Jun 2023 16:48:30 +0200 Subject: [PATCH 050/107] update config vars tutorial --- tutorials/config-vars.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tutorials/config-vars.md b/tutorials/config-vars.md index eea7564..bd7c67b 100644 --- a/tutorials/config-vars.md +++ b/tutorials/config-vars.md @@ -4,7 +4,7 @@ This is more of a description than a tutorial. Here you will find information ab [Back to Readme](../README.md) -Basic Variables +## Basic Variables | **Name** | **Type** | **Description** | **Default** | | :------- | :------: | --------------- | :---------: | @@ -30,20 +30,20 @@ Basic Variables |`END_BLOCK`| _number_ | End of block scan in replay mode. | none | -Advanced Variables +## Advanced Variables The following settings are additional for more advanced users who want to tweak the work of the tools to use more available resources | **Name** | **Type** | **Description** | **Default** | | :-------------------- | :------: | --------------------- | :---------: | -| `SCANNER_NODES_MAX_CHUNK_SIZE` | _number_ | | 100 | -| `ABIS_SERVICE_LIMIT` | _number_ | | none | -| `ABIS_SERVICE_FILTER` | _string_ | | "eosio:setabi" | -| `READER_INVIOLABLE_THREADS_COUNT` | _number_ | | 0 | -| `PROCESSOR_INVIOLABLE_THREADS_COUNT` | _number_ | | 0 | -| `FILTER_INVIOLABLE_THREADS_COUNT` | _number_ | | 0 | -| `START_FROM_HEAD` | _number_ | | 0 | -| `UNPROCESSED_BLOCK_QUEUE_MAX_BYTES_SIZE` | _number_ | | 256000000 | -| `UNPROCESSED_BLOCK_QUEUE_SIZE_CHECK_INTERVAL` | _number_ | | 2000 | -| `UNPROCESSED_BLOCK_QUEUE_BATCH_SIZE` | _number_ | | 100 | -| `PROCESSOR_TASK_QUEUE_CHECK_INTERVAL` | _number_ | | 5000 | +| `SCANNER_NODES_MAX_CHUNK_SIZE` | _number_ | Specifies the number of blocks in a subset of the block range. Modifying this value may affect the scanning speed, more smaller subsets will speed up the process in case of multi-threaded or multiple reader instances. | 100 | +| `ABIS_SERVICE_LIMIT` | _number_ | "setabi" action fetch limit | 100 | +| `ABIS_SERVICE_FILTER` | _string_ | "setabi" action filter | "eosio:setabi" | +| `READER_INVIOLABLE_THREADS_COUNT` | _number_ | The number of threads that cannot be allocated to the reader process. | 0 | +| `PROCESSOR_INVIOLABLE_THREADS_COUNT` | _number_ | The number of threads that cannot be allocated to the processor process. | 0 | +| `FILTER_INVIOLABLE_THREADS_COUNT` | _number_ | The number of threads that cannot be allocated to the filter process. | 0 | +| `START_FROM_HEAD` | _number_ | Specifies (1 = true/ 0 = false) whether reading a blocks should start with the head or the last irreversible block number. | 0 | +| `UNPROCESSED_BLOCK_QUEUE_MAX_BYTES_SIZE` | _number_ | The maximum size of the queue in bytes. | 256000000 | +| `UNPROCESSED_BLOCK_QUEUE_SIZE_CHECK_INTERVAL` | _number_ | Specifies the waiting time in milliseconds to check that the current queue size in bytes does not exceed the maximum allowed. | 2000 | +| `UNPROCESSED_BLOCK_QUEUE_BATCH_SIZE` | _number_ | Batch size of unprocessed blocks sent to the database at one time. The batch setting can be modified to optimize the transfer consumption to the database. | 100 | +| `PROCESSOR_TASK_QUEUE_CHECK_INTERVAL` | _number_ | Time to wait in milliseconds to check if there are new processor tasks available. This option is needed when the filter finishes its work and will not send information about the update to the processor. | 5000 | From cf189eaac33b16f3ae903730d1958024270e12d3 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 30 Jun 2023 09:37:24 +0200 Subject: [PATCH 051/107] update readme and config tutorial --- README.md | 4 ++-- tutorials/config-vars.md | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f9172e8..ff7f8c1 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ This package encapsulates the core mechanism, however, complete functionality re - [Processes](#processes) - [APIs](#apis) - [Broadcasting](#broadcasting) - - [Bootstraps](#bootstraps) + - [Bootstrap](#bootstrap) - [Reader](#reader) - [Filter](#filter) - [Processor](#processor) @@ -50,7 +50,7 @@ The API process, currently under development, is intended to provide easy access The broadcasting process creates a server for communication between processes, leveraging the @alien-worlds/broadcast module. Processes inform each other about their state, enabling efficient work coordination. The server facilitates the dispatch of messages defining tasks and states, improving performance when processing a high volume of blockchain data. -### Bootstraps +### Bootstrap The bootstrap process prepares input data according to provided guidelines and dispatches tasks to other processes using broadcasting, thereby initiating the blockchain data download. diff --git a/tutorials/config-vars.md b/tutorials/config-vars.md index bd7c67b..32f72fe 100644 --- a/tutorials/config-vars.md +++ b/tutorials/config-vars.md @@ -21,9 +21,11 @@ This is more of a description than a tutorial. Here you will find information ab |`API_PORT`| _number_ | History Tools API port number | none | |`BROADCAST_PORT`| _number_ | Broadcast port number | none | |`BROADCAST_HOST`| _string_ | Broadcast host | none | -|`DATABASE_HOSTS`| _string_ | Comma separated list of database hosts | none | -|`DATABASE_PORTS`| _string_ | Comma separated list of database ports | none | -|`DATABASE_NAME`| _string_ | Name of the database | none | +|`MONGO_HOSTS`| _string_ | Comma separated list of database hosts | none | +|`MONGO_PORTS`| _string_ | Comma separated list of database ports | none | +|`MONGO_DB_NAME`| _string_ | Name of the database | none | +|`MONGO_USER`| _string_ | Database user | none | +|`MONGO_PASSWORD`| _string_ | Database user password | none | |`MODE`| _string_ | History Tools run mode label "default"/"replay" | "default" | |`SCANNER_SCAN_KEY`| _string_ | Label for scanned blocks in replay mode. It serves the main purpose of separating and keeping in the database logs the history of which blocks were scanned for what purpose. It may happen that blocks in the same instance will have to be downloaded again, for this you need to enter a new label. | none | |`START_BLOCK`| _number_ | Beginning of block scanning in replay mode. | none | From 919b6ad8ccc700ce55058909261e4c81facc4da5 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 30 Jun 2023 11:30:27 +0200 Subject: [PATCH 052/107] merge history-tools-common --- README.md | 6 +- package.json | 6 +- src/api/api.types.ts | 2 +- .../__tests__/bootstrap.utils.unit.test.ts | 9 +- .../__tests__/start-bootstrap.unit.test.ts | 3 +- src/bootstrap/bootstrap.config.ts | 24 ++ src/bootstrap/bootstrap.dependencies.ts | 56 ++++ src/bootstrap/bootstrap.types.ts | 2 +- src/bootstrap/bootstrap.utils.ts | 9 +- src/bootstrap/index.ts | 2 + src/bootstrap/start-bootstrap.ts | 10 +- .../internal-broadcast.message.unit.test.ts | 2 +- src/broadcast/internal-broadcast.message.ts | 2 +- .../processor-broadcast.message.unit.test.ts | 2 +- .../reader-broadcast.message.unit.test.ts | 2 +- .../messages/filter-broadcast.message.ts | 2 +- .../messages/processor-broadcast.message.ts | 2 +- .../messages/reader-broadcast.message.ts | 3 +- src/broadcast/start-broadcast.ts | 7 +- .../__tests__/common.utils.unit.test.ts | 23 ++ .../abi.repository-impl.unit.test.ts | 108 +++++++ .../abis/__tests__/abis.cache.unit.test.ts | 104 +++++++ src/common/abis/__tests__/abis.unit.test.ts | 162 ++++++++++ src/common/abis/abis.cache.ts | 132 +++++++++ src/common/abis/abis.errors.ts | 7 + src/common/abis/abis.repository-impl.ts | 144 +++++++++ src/common/abis/abis.repository.ts | 53 ++++ src/common/abis/abis.ts | 167 +++++++++++ src/common/abis/abis.types.ts | 13 + src/common/abis/index.ts | 6 + .../block-range-scan.repository.ts | 85 ++++++ .../block-range-scan.source.ts | 29 ++ .../block-range-scanner/block-range-scan.ts | 186 ++++++++++++ .../block-range-scanner.config.ts | 4 + .../block-range-scanner.errors.ts | 26 ++ .../block-range-scanner.ts | 44 +++ src/common/block-range-scanner/index.ts | 6 + .../__tests__/block-state.unit.test.ts | 112 +++++++ src/common/block-state/block-state.ts | 102 +++++++ src/common/block-state/block-state.types.ts | 6 + src/common/block-state/index.ts | 2 + src/common/common.enums.ts | 5 + src/common/common.errors.ts | 5 + src/common/common.types.ts | 6 + src/common/common.utils.ts | 9 + src/common/dependencies.ts | 16 + .../__tests__/featured.utils.unit.test.ts | 82 ++++++ src/common/featured/featured-contract.ts | 34 +++ src/common/featured/featured.config.ts | 4 + src/common/featured/featured.errors.ts | 37 +++ src/common/featured/featured.mapper.ts | 277 ++++++++++++++++++ src/common/featured/featured.ts | 81 +++++ src/common/featured/featured.types.ts | 49 ++++ src/common/featured/featured.utils.ts | 58 ++++ src/common/featured/index.ts | 7 + src/common/index.ts | 12 + src/common/processor-task-queue/index.ts | 7 + .../processor-task-queue.config.ts | 4 + .../processor-task-queue.ts | 61 ++++ .../processor-task.enums.ts | 4 + .../processor-task.errors.ts | 5 + .../processor-task.source.ts | 5 + .../processor-task-queue/processor-task.ts | 116 ++++++++ .../processor-task.types.ts | 36 +++ src/common/types/action-trace.types.ts | 45 +++ src/common/types/block.types.ts | 20 ++ src/common/types/delta.types.ts | 11 + src/common/types/index.ts | 6 + src/common/types/signed-block.types.ts | 15 + src/common/types/trace.types.ts | 33 +++ src/common/types/transaction.types.ts | 14 + src/common/unprocessed-block-queue/index.ts | 4 + .../unprocessed-block-queue.errors.ts | 14 + .../unprocessed-block-queue.source.ts | 6 + .../unprocessed-block-queue.ts | 143 +++++++++ .../unprocessed-block-queue.types.ts | 6 + src/config/config.types.ts | 15 +- src/config/index.ts | 29 +- src/filter/filter.config.ts | 23 ++ src/filter/filter.dependencies.ts | 27 ++ src/filter/filter.runner.ts | 5 +- src/filter/filter.types.ts | 2 +- .../filter.worker-loader.dependencies.ts | 24 ++ src/filter/filter.worker-loader.ts | 2 +- src/filter/filter.worker.ts | 19 +- src/filter/index.ts | 3 + src/filter/start-filter.ts | 13 +- src/index.ts | 2 +- src/processor/index.ts | 3 + src/processor/processor.config.ts | 23 ++ src/processor/processor.dependencies.ts | 34 +++ src/processor/processor.runner.ts | 15 +- src/processor/processor.types.ts | 2 +- .../processor.worker-loader.dependencies.ts | 18 ++ src/processor/processor.worker-loader.ts | 10 +- .../processors/action-trace.processor.ts | 9 +- src/processor/processors/delta.processor.ts | 9 +- src/processor/processors/processor.ts | 3 +- src/processor/start-processor.ts | 15 +- src/reader/index.ts | 3 + src/reader/reader.config.ts | 18 ++ src/reader/reader.dependencies.ts | 20 ++ src/reader/reader.ts | 12 +- .../reader.worker-loader.dependencies.ts | 19 ++ src/reader/reader.worker-loader.ts | 9 +- src/reader/reader.worker.ts | 16 +- src/reader/start-reader.ts | 15 +- tutorials/writing-history-tools.md | 2 - yarn.lock | 17 +- 109 files changed, 3161 insertions(+), 184 deletions(-) create mode 100644 src/bootstrap/bootstrap.config.ts create mode 100644 src/bootstrap/bootstrap.dependencies.ts create mode 100644 src/common/__tests__/common.utils.unit.test.ts create mode 100644 src/common/abis/__tests__/abi.repository-impl.unit.test.ts create mode 100644 src/common/abis/__tests__/abis.cache.unit.test.ts create mode 100644 src/common/abis/__tests__/abis.unit.test.ts create mode 100644 src/common/abis/abis.cache.ts create mode 100644 src/common/abis/abis.errors.ts create mode 100644 src/common/abis/abis.repository-impl.ts create mode 100644 src/common/abis/abis.repository.ts create mode 100644 src/common/abis/abis.ts create mode 100644 src/common/abis/abis.types.ts create mode 100644 src/common/abis/index.ts create mode 100644 src/common/block-range-scanner/block-range-scan.repository.ts create mode 100644 src/common/block-range-scanner/block-range-scan.source.ts create mode 100644 src/common/block-range-scanner/block-range-scan.ts create mode 100644 src/common/block-range-scanner/block-range-scanner.config.ts create mode 100644 src/common/block-range-scanner/block-range-scanner.errors.ts create mode 100644 src/common/block-range-scanner/block-range-scanner.ts create mode 100644 src/common/block-range-scanner/index.ts create mode 100644 src/common/block-state/__tests__/block-state.unit.test.ts create mode 100644 src/common/block-state/block-state.ts create mode 100644 src/common/block-state/block-state.types.ts create mode 100644 src/common/block-state/index.ts create mode 100644 src/common/common.enums.ts create mode 100644 src/common/common.errors.ts create mode 100644 src/common/common.types.ts create mode 100644 src/common/common.utils.ts create mode 100644 src/common/dependencies.ts create mode 100644 src/common/featured/__tests__/featured.utils.unit.test.ts create mode 100644 src/common/featured/featured-contract.ts create mode 100644 src/common/featured/featured.config.ts create mode 100644 src/common/featured/featured.errors.ts create mode 100644 src/common/featured/featured.mapper.ts create mode 100644 src/common/featured/featured.ts create mode 100644 src/common/featured/featured.types.ts create mode 100644 src/common/featured/featured.utils.ts create mode 100644 src/common/featured/index.ts create mode 100644 src/common/index.ts create mode 100644 src/common/processor-task-queue/index.ts create mode 100644 src/common/processor-task-queue/processor-task-queue.config.ts create mode 100644 src/common/processor-task-queue/processor-task-queue.ts create mode 100644 src/common/processor-task-queue/processor-task.enums.ts create mode 100644 src/common/processor-task-queue/processor-task.errors.ts create mode 100644 src/common/processor-task-queue/processor-task.source.ts create mode 100644 src/common/processor-task-queue/processor-task.ts create mode 100644 src/common/processor-task-queue/processor-task.types.ts create mode 100644 src/common/types/action-trace.types.ts create mode 100644 src/common/types/block.types.ts create mode 100644 src/common/types/delta.types.ts create mode 100644 src/common/types/index.ts create mode 100644 src/common/types/signed-block.types.ts create mode 100644 src/common/types/trace.types.ts create mode 100644 src/common/types/transaction.types.ts create mode 100644 src/common/unprocessed-block-queue/index.ts create mode 100644 src/common/unprocessed-block-queue/unprocessed-block-queue.errors.ts create mode 100644 src/common/unprocessed-block-queue/unprocessed-block-queue.source.ts create mode 100644 src/common/unprocessed-block-queue/unprocessed-block-queue.ts create mode 100644 src/common/unprocessed-block-queue/unprocessed-block-queue.types.ts create mode 100644 src/filter/filter.config.ts create mode 100644 src/filter/filter.dependencies.ts create mode 100644 src/filter/filter.worker-loader.dependencies.ts create mode 100644 src/processor/processor.config.ts create mode 100644 src/processor/processor.dependencies.ts create mode 100644 src/processor/processor.worker-loader.dependencies.ts create mode 100644 src/reader/reader.config.ts create mode 100644 src/reader/reader.dependencies.ts create mode 100644 src/reader/reader.worker-loader.dependencies.ts diff --git a/README.md b/README.md index ff7f8c1..8fcc9e2 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,12 @@ This project is part of the Alien Worlds open source initiative, offering a set This package encapsulates the core mechanism, however, complete functionality requires packages that contain implementations of common components and third-party elements. - ## Dependencies -- [@alien-worlds/history-tools-common](https://github.com/Alien-Worlds/history-tools-common) +- [@alien-worlds/api-core](https://github.com/Alien-Worlds/api-core) +- [@alien-worlds/block-reader](https://github.com/Alien-Worlds/block-reader) +- [@alien-worlds/broadcast](https://github.com/Alien-Worlds/broadcast) +- [@alien-worlds/workers](https://github.com/Alien-Worlds/workers) - [async](https://github.com/caolan/async) - [commander](https://github.com/tj/commander.js) - [express](https://github.com/expressjs/express) diff --git a/package.json b/package.json index 3547cb1..5c21a74 100644 --- a/package.json +++ b/package.json @@ -35,11 +35,15 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/history-tools-common": "^0.0.36", + "@alien-worlds/api-core": "^0.0.151", + "@alien-worlds/block-reader": "^0.0.14", + "@alien-worlds/broadcast": "^0.0.10", + "@alien-worlds/workers": "^0.0.6", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", "express": "^4.18.2", + "node-fetch": "2", "ts-node": "^10.9.1" } } diff --git a/src/api/api.types.ts b/src/api/api.types.ts index c3093b3..1605540 100644 --- a/src/api/api.types.ts +++ b/src/api/api.types.ts @@ -1,4 +1,4 @@ -import { UnknownObject } from '@alien-worlds/history-tools-common'; +import { UnknownObject } from "@alien-worlds/api-core"; export type ApiConfig = { port: number; diff --git a/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts b/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts index a486224..7515e8d 100644 --- a/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts +++ b/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts @@ -1,15 +1,12 @@ +import { Result } from '@alien-worlds/api-core'; import { createDefaultModeBlockRange, createReplayModeBlockRange, createTestModeBlockRange, } from '../bootstrap.utils'; +import { Mode } from '../../common'; +import { EndBlockOutOfRangeError, StartBlockHigherThanEndBlockError, UndefinedStartBlockError } from '../bootstrap.errors'; -import { - EndBlockOutOfRangeError, - StartBlockHigherThanEndBlockError, - UndefinedStartBlockError, -} from '../bootstrap.errors'; -import { Mode, Result } from '@alien-worlds/history-tools-common'; describe('createDefaultModeBlockRange', () => { const originalLog = console.log; diff --git a/src/bootstrap/__tests__/start-bootstrap.unit.test.ts b/src/bootstrap/__tests__/start-bootstrap.unit.test.ts index 6b2c243..948bfa9 100644 --- a/src/bootstrap/__tests__/start-bootstrap.unit.test.ts +++ b/src/bootstrap/__tests__/start-bootstrap.unit.test.ts @@ -2,7 +2,8 @@ import { bootstrap } from '../start-bootstrap'; import { NoAbisError } from '../bootstrap.errors'; import { InternalBroadcastMessageName } from '../../broadcast/internal-broadcast.enums'; import { ReaderBroadcastMessage } from '../../broadcast/messages'; -import { BroadcastTcpClient, Mode } from '@alien-worlds/history-tools-common'; +import { Mode } from '../../common'; +import { BroadcastTcpClient } from '@alien-worlds/broadcast'; jest.mock('@alien-worlds/history-tools-common', () => ({ Abis: { diff --git a/src/bootstrap/bootstrap.config.ts b/src/bootstrap/bootstrap.config.ts new file mode 100644 index 0000000..ebbc06a --- /dev/null +++ b/src/bootstrap/bootstrap.config.ts @@ -0,0 +1,24 @@ +import { UnknownObject } from '@alien-worlds/api-core'; +import { AbisServiceConfig } from '../common'; +import { BlockRangeScanConfig } from '../common/block-range-scanner'; +import { FeaturedConfig } from '../common/featured'; +import { BroadcastConfig } from '@alien-worlds/broadcast'; + +export type BlockchainConfig = { + endpoint: string; + chainId: string; +}; + +export type BootstrapConfig = { + database: DatabaseConfig; + broadcast: BroadcastConfig; + scanner: BlockRangeScanConfig; + startBlock?: bigint; + endBlock?: bigint; + startFromHead?: boolean; + mode: string; + featured: FeaturedConfig; + abis: AbisServiceConfig; + blockchain: BlockchainConfig; + maxBlockNumber?: number; +}; diff --git a/src/bootstrap/bootstrap.dependencies.ts b/src/bootstrap/bootstrap.dependencies.ts new file mode 100644 index 0000000..211a904 --- /dev/null +++ b/src/bootstrap/bootstrap.dependencies.ts @@ -0,0 +1,56 @@ +import { BlockchainService, Result } from '@alien-worlds/api-core'; +import { Dependencies } from '../common/dependencies'; +import { Featured, FeaturedContractDataCriteria } from '../common/featured'; +import { BroadcastClient } from '@alien-worlds/broadcast'; +import { Abis, BlockRangeScanner, BlockState, DatabaseConfigBuilder } from '../common'; +import { BootstrapConfig } from './bootstrap.config'; + +/** + * An abstract class representing a Bootstrap dependencies. + * @export + * @abstract + * @class BootstrapDependencies + */ +export abstract class BootstrapDependencies extends Dependencies { + /** + * The broadcast client used for communication. + * @type {BroadcastClient} + */ + public broadcastClient: BroadcastClient; + /** + * The ABIs (Application Binary Interfaces) for contracts. + * @type {Abis} + */ + public abis: Abis; + /** + * The block range scanner for scanning blocks. + * @type {BlockRangeScanner} + */ + public scanner: BlockRangeScanner; + /** + * The featured contract service. + * @type {Featured} + */ + public featured: Featured; + /** + * The block state for maintaining blockchain state. + * @type {BlockState} + */ + public blockState: BlockState; + + /** + * The blockchain service for interacting with the blockchain. + * @type {BlockchainService} + */ + public blockchain: BlockchainService; + + /** + * @type {DatabaseConfigBuilder} + */ + public databaseConfigBuilder: DatabaseConfigBuilder; + + public abstract initialize( + config: BootstrapConfig, + featuredCriteria: FeaturedContractDataCriteria + ): Promise; +} diff --git a/src/bootstrap/bootstrap.types.ts b/src/bootstrap/bootstrap.types.ts index 4c522e6..acbcf54 100644 --- a/src/bootstrap/bootstrap.types.ts +++ b/src/bootstrap/bootstrap.types.ts @@ -1,4 +1,4 @@ -import { Mode } from '@alien-worlds/history-tools-common'; +import { Mode } from "../common"; export type BootstrapCommandOptions = { scanKey: string; diff --git a/src/bootstrap/bootstrap.utils.ts b/src/bootstrap/bootstrap.utils.ts index ca35980..758dfd5 100644 --- a/src/bootstrap/bootstrap.utils.ts +++ b/src/bootstrap/bootstrap.utils.ts @@ -5,13 +5,8 @@ import { UndefinedStartBlockError, EndBlockOutOfRangeError, } from './bootstrap.errors'; -import { - BlockRangeScanner, - BlockState, - BootstrapConfig, - Mode, - UnknownModeError, -} from '@alien-worlds/history-tools-common'; +import { BlockRangeScanner, BlockState, Mode, UnknownModeError } from '../common'; +import { BootstrapConfig } from './bootstrap.config'; /** * Creates a block range task input based on the provided configuration and mode. diff --git a/src/bootstrap/index.ts b/src/bootstrap/index.ts index 5efbed7..0b57388 100644 --- a/src/bootstrap/index.ts +++ b/src/bootstrap/index.ts @@ -1,4 +1,6 @@ +export * from './bootstrap.config'; export * from './bootstrap.command'; +export * from './bootstrap.dependencies'; export * from './bootstrap.errors'; export * from './bootstrap.types'; export * from './bootstrap.utils'; diff --git a/src/bootstrap/start-bootstrap.ts b/src/bootstrap/start-bootstrap.ts index dbb6e65..0d58751 100644 --- a/src/bootstrap/start-bootstrap.ts +++ b/src/bootstrap/start-bootstrap.ts @@ -17,13 +17,9 @@ import { ConfigVars, log } from '@alien-worlds/api-core'; import { BroadcastMessage } from '@alien-worlds/broadcast'; import { buildBootstrapConfig } from '../config'; import { bootstrapCommand } from './bootstrap.command'; -import { - BootstrapConfig, - BootstrapDependencies, - FeaturedUtils, - MissingCriteriaError, - Mode, -} from '@alien-worlds/history-tools-common'; +import { BootstrapConfig } from './bootstrap.config'; +import { BootstrapDependencies } from './bootstrap.dependencies'; +import { FeaturedUtils, MissingCriteriaError, Mode } from '../common'; /** * The bootstrap function initiates the bootstrap process based on the configuration provided. diff --git a/src/broadcast/__tests__/internal-broadcast.message.unit.test.ts b/src/broadcast/__tests__/internal-broadcast.message.unit.test.ts index 9ee3961..59cfa9c 100644 --- a/src/broadcast/__tests__/internal-broadcast.message.unit.test.ts +++ b/src/broadcast/__tests__/internal-broadcast.message.unit.test.ts @@ -1,4 +1,4 @@ -import { BroadcastTcpMessageType } from '@alien-worlds/history-tools-common'; +import { BroadcastTcpMessageType } from '@alien-worlds/broadcast'; import { InternalBroadcastMessage } from '../internal-broadcast.message'; describe('InternalBroadcastMessage', () => { diff --git a/src/broadcast/internal-broadcast.message.ts b/src/broadcast/internal-broadcast.message.ts index 8c5293b..71b8f90 100644 --- a/src/broadcast/internal-broadcast.message.ts +++ b/src/broadcast/internal-broadcast.message.ts @@ -2,7 +2,7 @@ import { BroadcastTcpMessage, BroadcastTcpMessageContent, BroadcastTcpMessageType, -} from '@alien-worlds/history-tools-common'; +} from '@alien-worlds/broadcast'; /** * Represents an internal broadcast message. diff --git a/src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts b/src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts index f8710e3..2abf217 100644 --- a/src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts +++ b/src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts @@ -3,7 +3,7 @@ import { InternalBroadcastChannel, InternalBroadcastMessageName, } from '../../internal-broadcast.enums'; -import { BroadcastMessage } from '@alien-worlds/history-tools-common'; +import { BroadcastMessage } from '@alien-worlds/broadcast'; describe('ProcessorBroadcastMessage', () => { describe('ready', () => { diff --git a/src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts b/src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts index dc5a12d..7dcaead 100644 --- a/src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts +++ b/src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts @@ -7,7 +7,7 @@ import { InternalBroadcastChannel, InternalBroadcastMessageName, } from '../../internal-broadcast.enums'; -import { Mode } from '@alien-worlds/history-tools-common'; +import { Mode } from '../../../common'; describe('ReaderBroadcastMessage', () => { describe('newReplayModeTask', () => { diff --git a/src/broadcast/messages/filter-broadcast.message.ts b/src/broadcast/messages/filter-broadcast.message.ts index 52fb675..95a3b0b 100644 --- a/src/broadcast/messages/filter-broadcast.message.ts +++ b/src/broadcast/messages/filter-broadcast.message.ts @@ -1,4 +1,4 @@ -import { BroadcastMessage } from '@alien-worlds/history-tools-common'; +import { BroadcastMessage } from '@alien-worlds/broadcast'; import { InternalBroadcastChannel, InternalBroadcastMessageName, diff --git a/src/broadcast/messages/processor-broadcast.message.ts b/src/broadcast/messages/processor-broadcast.message.ts index ec5d3c1..748e585 100644 --- a/src/broadcast/messages/processor-broadcast.message.ts +++ b/src/broadcast/messages/processor-broadcast.message.ts @@ -1,4 +1,4 @@ -import { BroadcastMessage } from '@alien-worlds/history-tools-common'; +import { BroadcastMessage } from '@alien-worlds/broadcast'; import { InternalBroadcastChannel, InternalBroadcastMessageName, diff --git a/src/broadcast/messages/reader-broadcast.message.ts b/src/broadcast/messages/reader-broadcast.message.ts index 4a1eb59..48e4544 100644 --- a/src/broadcast/messages/reader-broadcast.message.ts +++ b/src/broadcast/messages/reader-broadcast.message.ts @@ -1,8 +1,9 @@ +import { BroadcastMessage } from '@alien-worlds/broadcast'; import { InternalBroadcastChannel, InternalBroadcastMessageName, } from '../internal-broadcast.enums'; -import { BroadcastMessage, Mode } from '@alien-worlds/history-tools-common'; +import { Mode } from '../../common'; /** * Data structure for the reader broadcast message. diff --git a/src/broadcast/start-broadcast.ts b/src/broadcast/start-broadcast.ts index ff4cde2..757a996 100644 --- a/src/broadcast/start-broadcast.ts +++ b/src/broadcast/start-broadcast.ts @@ -1,8 +1,5 @@ -import { - BroadcastTcpServer, - ConfigVars, - buildBroadcastConfig, -} from '@alien-worlds/history-tools-common'; +import { ConfigVars } from '@alien-worlds/api-core'; +import { BroadcastTcpServer, buildBroadcastConfig } from '@alien-worlds/broadcast'; export const startBroadcast = async () => { const vars = new ConfigVars(); diff --git a/src/common/__tests__/common.utils.unit.test.ts b/src/common/__tests__/common.utils.unit.test.ts new file mode 100644 index 0000000..8818893 --- /dev/null +++ b/src/common/__tests__/common.utils.unit.test.ts @@ -0,0 +1,23 @@ +import { isSetAbiAction } from '../common.utils'; + +describe('isSetAbiAction', () => { + it('should return true for eosio setabi action', () => { + const result = isSetAbiAction('eosio', 'setabi'); + expect(result).toBe(true); + }); + + it('should return false for other contract or action', () => { + const result = isSetAbiAction('eosio', 'otherAction'); + expect(result).toBe(false); + }); + + it('should return false for other contract and setabi action', () => { + const result = isSetAbiAction('otherContract', 'setabi'); + expect(result).toBe(false); + }); + + it('should return false for other contract and action', () => { + const result = isSetAbiAction('otherContract', 'otherAction'); + expect(result).toBe(false); + }); +}); diff --git a/src/common/abis/__tests__/abi.repository-impl.unit.test.ts b/src/common/abis/__tests__/abi.repository-impl.unit.test.ts new file mode 100644 index 0000000..ac0e4c9 --- /dev/null +++ b/src/common/abis/__tests__/abi.repository-impl.unit.test.ts @@ -0,0 +1,108 @@ +import { AbisRepositoryImpl } from '../abis.repository-impl'; +import { Result, CountParams, ContractEncodedAbi } from '@alien-worlds/api-core'; +import { AbisCache } from '../abis.cache'; + +jest.mock('../abis.cache'); + +const mockDataSource = { + find: jest.fn(), + count: jest.fn(), + aggregate: jest.fn(), + update: jest.fn(), + insert: jest.fn(), + remove: jest.fn(), + startTransaction: jest.fn(), + commitTransaction: jest.fn(), + rollbackTransaction: jest.fn(), +} as any; + +const mockMapper = { + toEntity: jest.fn(), + fromEntity: jest.fn(), + getEntityKeyMapping: jest.fn(), +} as any; + +const mockQueryBuilders = { + buildFindQuery: jest.fn(), + buildCountQuery: jest.fn(), + buildUpdateQuery: jest.fn(), + buildRemoveQuery: jest.fn(), + buildAggregationQuery: jest.fn(), +} as any; + +describe('AbisRepositoryImpl', () => { + let repository: AbisRepositoryImpl; + let mockCache: jest.Mocked; + + beforeEach(() => { + mockCache = new AbisCache() as jest.Mocked; + repository = new AbisRepositoryImpl(mockDataSource, mockMapper, mockQueryBuilders); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe('getAbis', () => { + it('should return abis from cache if present', async () => { + const mockAbis: ContractEncodedAbi[] = [ + /* mock data here */ + ]; + mockCache.getAbis.mockReturnValue(mockAbis); + + const result = await repository.getAbis({ contracts: ['contract1'] }); + + expect(result).toEqual(Result.withContent(mockAbis)); + expect(mockCache.getAbis).toHaveBeenCalledWith({ contracts: ['contract1'] }); + }); + }); + + describe('getAbi', () => { + it('should return abi from cache if present', async () => { + const mockAbi: ContractEncodedAbi = {} as any; + const blockNumber = BigInt(10); + const contract = 'contract1'; + + mockCache.getAbi.mockReturnValue(mockAbi); + + const result = await repository.getAbi(blockNumber, contract); + + expect(result).toEqual(Result.withContent(mockAbi)); + expect(mockCache.getAbi).toHaveBeenCalledWith(blockNumber, contract); + }); + }); + + describe('insertAbis', () => { + it('should insert abis into the cache and the database', async () => { + const mockAbis: ContractEncodedAbi[] = [ + /* mock data here */ + ]; + const addSpy = jest.spyOn(repository, 'add'); + addSpy.mockResolvedValue(Result.withContent(mockAbis)); + + const result = await repository.insertAbis(mockAbis); + + expect(result).toEqual(Result.withContent(mockAbis.length > 0)); + expect(mockCache.insertAbis).toHaveBeenCalledWith(mockAbis); + expect(addSpy).toHaveBeenCalledWith(mockAbis); + }); + }); + + describe('countAbis', () => { + it('should count abis based on the startBlock and endBlock', async () => { + const countSpy = jest.spyOn(repository, 'count'); + const mockCount = 5; + countSpy.mockResolvedValue(Result.withContent(mockCount)); + + const startBlock = BigInt(10); + const endBlock = BigInt(20); + + const result = await repository.countAbis(startBlock, endBlock); + + expect(result).toEqual(Result.withContent(mockCount)); + expect(countSpy).toHaveBeenCalledWith( + CountParams.create({ where: expect.anything() }) + ); + }); + }); +}); diff --git a/src/common/abis/__tests__/abis.cache.unit.test.ts b/src/common/abis/__tests__/abis.cache.unit.test.ts new file mode 100644 index 0000000..56e0293 --- /dev/null +++ b/src/common/abis/__tests__/abis.cache.unit.test.ts @@ -0,0 +1,104 @@ +import { AbisCache } from '../abis.cache'; + +describe('AbisCache', () => { + let abisCache: AbisCache; + + beforeEach(() => { + abisCache = new AbisCache(); + }); + + describe('getAbis', () => { + it('should return an empty array when cache is empty', () => { + const result = abisCache.getAbis({ contracts: ['eosio'] }); + expect(result).toEqual([]); + }); + + it('should return an empty array when contracts array is empty', () => { + abisCache.insertAbis([ + { contract: 'eosio', blockNumber: 1n } as any, + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]); + + const result = abisCache.getAbis({ contracts: [] }); + expect(result).toEqual([]); + }); + + it('should return all ABIs when no startBlock or endBlock is provided', () => { + abisCache.insertAbis([ + { contract: 'eosio', blockNumber: 1n } as any, + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]); + + const result = abisCache.getAbis({ contracts: ['eosio'] }); + expect(result).toEqual([ + { contract: 'eosio', blockNumber: 1n } as any, + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]); + }); + + it('should return matching ABIs within the specified range', () => { + abisCache.insertAbis([ + { contract: 'eosio', blockNumber: 1n } as any, + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]); + + const result = abisCache.getAbis({ + contracts: ['eosio'], + startBlock: 2n, + endBlock: 3n, + }); + expect(result).toEqual([ + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]); + }); + }); + + describe('getAbi', () => { + it('should return null when cache is empty', () => { + const result = abisCache.getAbi(1n, 'eosio'); + expect(result).toBeNull(); + }); + + it('should return the latest ABI that matches the block number', () => { + abisCache.insertAbis([ + { contract: 'eosio', blockNumber: 1n } as any, + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]); + + const result = abisCache.getAbi(2n, 'eosio'); + expect(result).toEqual({ contract: 'eosio', blockNumber: 2n } as any); + }); + + it('should return null when no matching ABI is found', () => { + abisCache.insertAbis([ + { contract: 'eosio', blockNumber: 1n } as any, + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]); + + const result = abisCache.getAbi(4n, 'undefined'); + expect(result).toBeNull(); + }); + }); + + describe('insertAbis', () => { + it('should insert ABIs into the cache', () => { + const abis = [ + { contract: 'eosio', blockNumber: 1n } as any, + { contract: 'eosio', blockNumber: 2n } as any, + { contract: 'eosio', blockNumber: 3n } as any, + ]; + + abisCache.insertAbis(abis); + + const result = abisCache.getAbis({ contracts: ['eosio'] }); + expect(result).toEqual(abis); + }); + }); +}); diff --git a/src/common/abis/__tests__/abis.unit.test.ts b/src/common/abis/__tests__/abis.unit.test.ts new file mode 100644 index 0000000..b6b07f7 --- /dev/null +++ b/src/common/abis/__tests__/abis.unit.test.ts @@ -0,0 +1,162 @@ +import { Abis } from '../abis'; +import { AbisRepositoryImpl } from '../abis.repository-impl'; +import { Failure, Result, AbiService } from '@alien-worlds/api-core'; +import { AbisServiceNotSetError } from '../abis.errors'; +import { AbiNotFoundError } from '@alien-worlds/block-reader'; + +// Mock dependencies +jest.mock('../abis.repository-impl'); +jest.mock('../abis.service'); + +const mockAbisRepository = { + cacheAbis: jest.fn(), + getAbis: jest.fn(), + getAbi: jest.fn(), + insertAbis: jest.fn(), + countAbis: jest.fn(), +} as any; +const mockAbisService = { fetchAbis: jest.fn() } as any; + +describe('Abis', () => { + let abis: Abis; + + beforeEach(() => { + // Reset mocks and create a new instance of Abis + jest.resetAllMocks(); + abis = new Abis(new mockAbisRepository(), new mockAbisService()); + }); + + describe('getAbis', () => { + it('should return ABIs from repository when available', async () => { + const mockAbis = [ + /* Mocked ABIs */ + ]; + mockAbisRepository.prototype.getAbis.mockResolvedValue( + Result.withContent(mockAbis) + ); + + const result = await abis.getAbis(); + + expect(mockAbisRepository.prototype.getAbis).toHaveBeenCalledTimes(1); + expect(result.isFailure).toBe(false); + expect(result.content).toEqual(mockAbis); + }); + + it('should fetch ABIs when none are available in the repository and fetch option is enabled', async () => { + const mockAbis = [ + /* Mocked ABIs */ + ]; + mockAbisRepository.prototype.getAbis.mockResolvedValue(Result.withContent([])); + abis.fetchAbis = jest.fn().mockResolvedValue(Result.withContent(mockAbis)); + + const result = await abis.getAbis({ fetch: true }); + + expect(mockAbisRepository.prototype.getAbis).toHaveBeenCalledTimes(1); + expect(abis.fetchAbis).toHaveBeenCalledTimes(1); + expect(result.isFailure).toBe(false); + expect(result.content).toEqual(mockAbis); + }); + + // Add more test cases for different scenarios + }); + + describe('getAbi', () => { + it('should return ABI from repository when available', async () => { + const mockAbi = '1234567890'; + mockAbisRepository.prototype.getAbi.mockResolvedValue(Result.withContent(mockAbi)); + + const result = await abis.getAbi(123n, '0x123'); + + expect(mockAbisRepository.prototype.getAbi).toHaveBeenCalledTimes(1); + expect(result.isFailure).toBe(false); + expect(result.content).toEqual(mockAbi); + }); + + it('should fetch ABI when not available in the repository and fetch option is enabled', async () => { + const mockAbi = '1234567890'; + mockAbisRepository.prototype.getAbi.mockResolvedValue( + Result.withFailure(Failure.fromError(new AbiNotFoundError())) + ); + abis.fetchAbis = jest.fn().mockResolvedValue(Result.withContent([mockAbi])); + + const result = await abis.getAbi(123n, '0x123', true); + + expect(mockAbisRepository.prototype.getAbi).toHaveBeenCalledTimes(1); + expect(abis.fetchAbis).toHaveBeenCalledTimes(1); + expect(result.isFailure).toBe(false); + expect(result.content).toEqual(mockAbi); + }); + + // Add more test cases for different scenarios + }); + + describe('storeAbi', () => { + it('should insert the ABI into the repository', async () => { + const blockNumber = 123n; + const contract = '0x123'; + const hex = '0xabcdef'; + + mockAbisRepository.prototype.insertAbis.mockResolvedValue(Result.withContent(true)); + + const result = await abis.storeAbi(blockNumber, contract, hex); + + expect(mockAbisRepository.prototype.insertAbis).toHaveBeenCalledTimes(1); + expect(mockAbisRepository.prototype.insertAbis).toHaveBeenCalledWith([ + expect.objectContaining({ blockNumber, contract, hex }), + ]); + expect(result.isFailure).toBe(false); + expect(result.content).toBe(true); + }); + + // Add more test cases for different scenarios + }); + + describe('fetchAbis', () => { + it('should throw AbisServiceNotSetError when service is not set', async () => { + abis = new Abis(new mockAbisRepository()); // Create instance without AbisService + + await expect(abis.fetchAbis()).rejects.toThrow(AbisServiceNotSetError); + }); + + it('should fetch ABIs using the service', async () => { + const mockAbis = [ + /* Mocked ABIs */ + ]; + const mockContracts = ['0x123', '0x456']; + const mockServiceResponse = Result.withContent(mockAbis); + + abis = new Abis(new mockAbisRepository(), new mockAbisService()); + mockAbisService.prototype.fetchAbis.mockResolvedValue(mockServiceResponse); + + const result = await abis.fetchAbis(mockContracts); + + expect(mockAbisService.prototype.fetchAbis).toHaveBeenCalledTimes( + mockContracts.length + ); + expect(mockAbisService.prototype.fetchAbis).toHaveBeenCalledWith( + expect.any(String) + ); + expect(result.isFailure).toBe(false); + expect(result.content).toEqual(mockAbis); + }); + + // Add more test cases for different scenarios + }); + + describe('cacheAbis', () => { + it('should cache ABIs in the repository', async () => { + const mockContracts = ['0x123', '0x456']; + + mockAbisRepository.prototype.cacheAbis.mockResolvedValue(); + + const result = await abis.cacheAbis(mockContracts); + + expect(mockAbisRepository.prototype.cacheAbis).toHaveBeenCalledTimes(1); + expect(mockAbisRepository.prototype.cacheAbis).toHaveBeenCalledWith(mockContracts); + expect(result.isFailure).toBe(false); + expect(result.content).toBeUndefined(); + }); + + // Add more test cases for different scenarios + }); +}); diff --git a/src/common/abis/abis.cache.ts b/src/common/abis/abis.cache.ts new file mode 100644 index 0000000..95c2340 --- /dev/null +++ b/src/common/abis/abis.cache.ts @@ -0,0 +1,132 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { ContractEncodedAbi } from '@alien-worlds/api-core'; + +/** + * A function that filters ABI entries based on the block number being greater than or equal to the start block. + * @param startBlock - The start block number. + * @param endBlock - The end block number. + * @returns A filter function. + */ +const filterFromStartBlock = + (startBlock: bigint, endBlock: bigint) => (abi: ContractEncodedAbi) => + abi.blockNumber >= startBlock; + +/** + * A function that filters ABI entries based on the block number being less than or equal to the end block. + * @param startBlock - The start block number. + * @param endBlock - The end block number. + * @returns A filter function. + */ +const filterTillEndBlock = + (startBlock: bigint, endBlock: bigint) => (abi: ContractEncodedAbi) => + abi.blockNumber <= endBlock; + +/** + * A function that filters ABI entries based on the block number being within the specified range. + * @param startBlock - The start block number. + * @param endBlock - The end block number. + * @returns A filter function. + */ +const filterInRange = + (startBlock: bigint, endBlock: bigint) => (abi: ContractEncodedAbi) => + abi.blockNumber >= startBlock && abi.blockNumber <= endBlock; + +/** + * Class representing the cache for storing and retrieving contract ABIs. + */ +export class AbisCache { + private cache: Map> = new Map(); + + /** + * Retrieves contract ABIs from the cache based on the specified options. + * @param options - Options for retrieving the contract ABIs. + * @param options.startBlock - The start block number to filter the ABIs. + * @param options.endBlock - The end block number to filter the ABIs. + * @param options.contracts - An array of contract addresses to filter the ABIs. + * @returns An array of matching contract ABIs. + */ + public getAbis(options: { + startBlock?: bigint; + endBlock?: bigint; + contracts?: string[]; + }): ContractEncodedAbi[] { + const { startBlock, endBlock, contracts } = options; + + const filter = + startBlock && endBlock + ? filterInRange + : startBlock + ? filterFromStartBlock + : filterTillEndBlock; + + const abis = []; + if (Array.isArray(contracts)) { + for (const contract of contracts) { + if (contract && this.cache.has(contract)) { + const contractAbis = this.cache.get(contract); + + if (startBlock || endBlock) { + const filtered = Array.from(contractAbis.values()).filter( + filter(startBlock, endBlock) + ); + + abis.push(...filtered); + } else { + abis.push(...contractAbis.values()); + } + } + } + + return abis; + } + // + this.cache.forEach(value => { + if (startBlock || endBlock) { + const filtered = Array.from(value.values()).filter(filter(startBlock, endBlock)); + abis.push(...filtered); + } else { + abis.push(...value.values()); + } + }); + + return abis; + } + + /** + * Retrieves the ABI for the specified block number and contract address. + * @param blockNumber - The block number to find the ABI. + * @param contract - The contract address. + * @returns The matching ABI or null if not found. + */ + public getAbi(blockNumber: bigint, contract: string): ContractEncodedAbi { + if (this.cache.has(contract)) { + const abis = this.cache.get(contract); + const sorted = Array.from(abis.values()).sort((a, b) => + a.blockNumber > b.blockNumber ? -1 : 1 + ); + + for (const abi of sorted) { + if (abi.blockNumber <= blockNumber) { + return abi; + } + } + } + + return null; + } + + /** + * Inserts an array of ABIs into the cache. + * @param abis - An array of ABIs to insert into the cache. + */ + public insertAbis(abis: ContractEncodedAbi[]): void { + abis.forEach(abi => { + let set = this.cache.get(abi.contract); + if (!set) { + set = new Set(); + this.cache.set(abi.contract, set); + } + set.add(abi); + }); + } +} diff --git a/src/common/abis/abis.errors.ts b/src/common/abis/abis.errors.ts new file mode 100644 index 0000000..27ec8bc --- /dev/null +++ b/src/common/abis/abis.errors.ts @@ -0,0 +1,7 @@ +export class AbisServiceNotSetError extends Error { + constructor() { + super( + `AbisService not found. run "setupAbis" with the service config and featured values.` + ); + } +} diff --git a/src/common/abis/abis.repository-impl.ts b/src/common/abis/abis.repository-impl.ts new file mode 100644 index 0000000..4d12bae --- /dev/null +++ b/src/common/abis/abis.repository-impl.ts @@ -0,0 +1,144 @@ +import { + ContractEncodedAbi, + CountParams, + FindParams, + log, + RepositoryImpl, + Result, + Where, +} from '@alien-worlds/api-core'; +import { AbisCache } from './abis.cache'; +import { AbisRepository } from './abis.repository'; + +/** + * Implements the AbisRepository with caching functionality. + * This class manages ContractEncodedAbi entities, providing CRUD operations and additional functionalities such as caching. + * It extends the base RepositoryImpl and implements the AbisRepository interface. + */ +export class AbisRepositoryImpl + extends RepositoryImpl + implements AbisRepository +{ + /** + * Cache instance for caching the ABI objects. + * @private + */ + private cache: AbisCache = new AbisCache(); + + /** + * Cache the ABIs. + * @param {string[]} [contracts] - List of contracts. + */ + public async cacheAbis(contracts?: string[]) { + const abis = await this.getAbis({ contracts }); + if (Array.isArray(abis)) { + this.cache.insertAbis(abis); + } + } + + /** + * Retrieve the ABIs. + * @param options - Filter options for retrieving ABIs. + * @returns Promise that resolves with the Result of an array of ContractEncodedAbi. + */ + public async getAbis(options: { + startBlock?: bigint; + endBlock?: bigint; + contracts?: string[]; + }): Promise> { + try { + const { startBlock, endBlock, contracts } = options || {}; + + const cachedAbis = this.cache.getAbis(options); + + if (cachedAbis.length > 0) { + return Result.withContent(cachedAbis); + } + const where = Where.bind(); + + if (startBlock && endBlock) { + where.props().blockNumber.isGte(startBlock).isLte(endBlock); + } + + if (contracts) { + where.props().contract.isIn(contracts); + } + + return this.find(FindParams.create({ where })); + } catch (error) { + log(error); + return Result.withContent([]); + } + } + + /** + * Retrieve a specific ABI. + * @param blockNumber - The block number. + * @param contract - The contract name. + * @returns Promise that resolves with the Result of ContractEncodedAbi. + */ + public async getAbi( + blockNumber: bigint, + contract: string + ): Promise> { + const cachedAbi = this.cache.getAbi(blockNumber, contract); + + if (cachedAbi) { + return Result.withContent(cachedAbi); + } + + const where = Where.bind(); + where.props().blockNumber.isLte(blockNumber).props().contract.isEq(contract); + const { content, failure } = await this.find( + FindParams.create({ where, limit: 1, sort: { block_number: -1 } }) + ); + + if (content) { + return Result.withContent(content[0]); + } + + if (failure) { + return Result.withFailure(failure); + } + } + + /** + * Inserts an array of ABIs. + * @param abis - The ABIs to be inserted. + * @returns Promise that resolves with the Result of boolean indicating the success of the operation. + */ + public async insertAbis(abis: ContractEncodedAbi[]): Promise> { + this.cache.insertAbis(abis); + const { content, failure } = await this.add(abis); + + if (failure) { + log(failure.error); + return Result.withContent(false); + } + + return Result.withContent(content.length > 0); + } + + /** + * Counts the number of ABIs between the provided block numbers. + * @param startBlock - The start block number. + * @param endBlock - The end block number. + * @returns Promise that resolves with the Result of number indicating the count of ABIs. + */ + public async countAbis( + startBlock?: bigint, + endBlock?: bigint + ): Promise> { + const where = Where.bind(); + + if (typeof startBlock === 'bigint') { + where.props().blockNumber.isGte(startBlock); + } + + if (typeof endBlock === 'bigint') { + where.props().blockNumber.isLte(endBlock); + } + + return this.count(CountParams.create({ where })); + } +} diff --git a/src/common/abis/abis.repository.ts b/src/common/abis/abis.repository.ts new file mode 100644 index 0000000..49529f5 --- /dev/null +++ b/src/common/abis/abis.repository.ts @@ -0,0 +1,53 @@ +import { ContractEncodedAbi, Result } from '@alien-worlds/api-core'; + +/** + * This class manages ContractEncodedAbi entities, providing CRUD operations and additional functionalities such as caching. + * It extends the base RepositoryImpl and implements the AbisRepository interface. + */ +export abstract class AbisRepository { + /** + * Cache the ABIs. + * @param {string[]} [contracts] - List of contracts. + */ + public abstract cacheAbis(contracts?: string[]); + + /** + * Retrieve the ABIs. + * @param options - Filter options for retrieving ABIs. + * @returns Promise that resolves with the Result of an array of ContractEncodedAbi. + */ + public abstract getAbis(options: { + startBlock?: bigint; + endBlock?: bigint; + contracts?: string[]; + }): Promise>; + + /** + * Retrieve a specific ABI. + * @param blockNumber - The block number. + * @param contract - The contract name. + * @returns Promise that resolves with the Result of ContractEncodedAbi. + */ + public abstract getAbi( + blockNumber: bigint, + contract: string + ): Promise>; + + /** + * Inserts an array of ABIs. + * @param abis - The ABIs to be inserted. + * @returns Promise that resolves with the Result of boolean indicating the success of the operation. + */ + public abstract insertAbis(abis: ContractEncodedAbi[]): Promise>; + + /** + * Counts the number of ABIs between the provided block numbers. + * @param startBlock - The start block number. + * @param endBlock - The end block number. + * @returns Promise that resolves with the Result of number indicating the count of ABIs. + */ + public abstract countAbis( + startBlock?: bigint, + endBlock?: bigint + ): Promise>; +} diff --git a/src/common/abis/abis.ts b/src/common/abis/abis.ts new file mode 100644 index 0000000..739fc51 --- /dev/null +++ b/src/common/abis/abis.ts @@ -0,0 +1,167 @@ +import { AbiService, ContractEncodedAbi, Failure, Result, log } from '@alien-worlds/api-core'; +import { AbisServiceNotSetError } from './abis.errors'; +import { AbisRepository } from './abis.repository'; +import { AbiNotFoundError } from '@alien-worlds/block-reader'; + +/** + * Represents a collection of ABIs (Application Binary Interfaces) for smart contracts. + */ +export class Abis { + private contracts: Set = new Set(); + + /** + * Constructs a new instance of the Abis class. + * + * @param {AbisRepository} repository - The repository for accessing ABIs. + * @param {AbisService} [service] - The service for fetching ABIs. + * @param {FeaturedConfig} [featuredConfig] - The featured configuration containing contract traces and deltas. + * @private + */ + constructor( + private repository: AbisRepository, + private service?: AbiService, + contracts?: string[] + ) { + if (contracts) { + contracts.forEach(contract => { + this.contracts.add(contract); + }); + } + } + + /** + * Retrieves the ABIs (Application Binary Interfaces) for the specified options. + * + * @param {Object} [options] - The options for retrieving the ABIs. + * @param {bigint} [options.startBlock] - The starting block number. + * @param {bigint} [options.endBlock] - The ending block number. + * @param {string[]} [options.contracts] - The contract addresses to filter the ABIs. + * @param {boolean} [options.fetch] - Indicates whether to fetch ABIs if not found in the database. + * @returns {Promise>} A promise that resolves to a Result object containing the retrieved ABIs. + */ + public async getAbis(options?: { + startBlock?: bigint; + endBlock?: bigint; + contracts?: string[]; + fetch?: boolean; + }): Promise> { + const { startBlock, endBlock, contracts, fetch } = options || {}; + + const { content: abis } = await this.repository.getAbis(options); + + if (abis.length === 0 && fetch) { + log( + `No contract ABIs (${startBlock}-${endBlock}) were found in the database. Trying to fetch ABIs...` + ); + return this.fetchAbis(contracts); + } + + return Result.withContent(abis || []); + } + + /** + * Retrieves the ABI (Application Binary Interface) for the specified block number and contract address. + * + * @param {bigint} blockNumber - The block number. + * @param {string} contract - The contract address. + * @param {boolean} [fetch=false] - Indicates whether to fetch the ABI if not found in the database. + * @returns {Promise>} A promise that resolves to a Result object containing the retrieved ABI. + */ + public async getAbi( + blockNumber: bigint, + contract: string, + fetch = false + ): Promise> { + const getAbiResult = await this.repository.getAbi(blockNumber, contract); + + if (fetch && getAbiResult.isFailure) { + const { content: abis, failure } = await this.fetchAbis([contract]); + + if (failure) { + return Result.withFailure(failure); + } + + const abi = abis.reduce((result, abi) => { + if (abi.blockNumber <= blockNumber) { + if (!result || result.blockNumber < abi.blockNumber) { + result = abi; + } + } + return result; + }, null); + + if (abi) { + return Result.withContent(abi); + } + + return Result.withFailure(Failure.fromError(new AbiNotFoundError())); + } + + return getAbiResult; + } + + /** + * Stores the ABI (Application Binary Interface) for the specified block number, contract address, and hex code. + * + * @param {unknown} blockNumber - The block number. + * @param {string} contract - The contract address. + * @param {string} hex - The hex code representing the ABI. + * @returns {Promise>} A promise that resolves to a Result object indicating the success or failure of the operation. + */ + public async storeAbi( + blockNumber: unknown, + contract: string, + hex: string + ): Promise> { + return this.repository.insertAbis([ + ContractEncodedAbi.create(blockNumber, contract, hex), + ]); + } + + /** + * Fetches the ABIs (Application Binary Interfaces) for the specified contracts. + * + * @param {string[]} [contracts] - The contract addresses to fetch ABIs for. + * @returns {Promise>} A promise that resolves to a Result object containing the fetched ABIs. + * @throws {AbisServiceNotSetError} Thrown if the AbisService is not set. + * @private + */ + public async fetchAbis(contracts?: string[]): Promise> { + if (!this.service) { + throw new AbisServiceNotSetError(); + } + + const abis: ContractEncodedAbi[] = []; + try { + const contractsToFetch = contracts || this.contracts; + for (const contract of contractsToFetch) { + const contractAbis = await this.service.fetchAbis(contract); + abis.push(...contractAbis); + } + + if (abis.length > 0) { + await this.repository.insertAbis(abis); + } + } catch (error) { + log(error.message); + } + return Result.withContent(abis); + } + + /** + * Caches the ABIs (Application Binary Interfaces) for the specified contracts. + * + * @param {string[]} [contracts] - The contract addresses to cache ABIs for. + * @returns {Promise>} A promise that resolves to a Result object indicating the success or failure of the operation. + * @private + */ + public async cacheAbis(contracts?: string[]): Promise> { + try { + await this.repository.cacheAbis(contracts); + return Result.withoutContent(); + } catch (error) { + log(error.message); + return Result.withFailure(Failure.fromError(error)); + } + } +} diff --git a/src/common/abis/abis.types.ts b/src/common/abis/abis.types.ts new file mode 100644 index 0000000..a71d98a --- /dev/null +++ b/src/common/abis/abis.types.ts @@ -0,0 +1,13 @@ +import { UnknownObject } from '@alien-worlds/api-core'; + +export type AbisServiceConfig = { + url: string; + limit?: number; + filter?: string; + [key: string]: unknown; +}; + +export type AbisConfig = { + service: AbisServiceConfig; + database: DatabaseConfig; +}; diff --git a/src/common/abis/index.ts b/src/common/abis/index.ts new file mode 100644 index 0000000..5599dc6 --- /dev/null +++ b/src/common/abis/index.ts @@ -0,0 +1,6 @@ +export * from './abis.cache'; +export * from './abis.errors'; +export * from './abis.repository-impl'; +export * from './abis.repository'; +export * from './abis.types'; +export * from './abis'; diff --git a/src/common/block-range-scanner/block-range-scan.repository.ts b/src/common/block-range-scanner/block-range-scan.repository.ts new file mode 100644 index 0000000..21aa099 --- /dev/null +++ b/src/common/block-range-scanner/block-range-scan.repository.ts @@ -0,0 +1,85 @@ +import { BlockRangeScan } from './block-range-scan'; +import { BlockRangeScanSource } from './block-range-scan.source'; +import { Mapper } from '@alien-worlds/api-core'; + +export type ScanRequest = { + error?: Error; +}; + +export class BlockRangeScanRepository { + constructor( + private readonly source: BlockRangeScanSource, + private readonly mapper: Mapper, + private readonly maxChunkSize: number + ) {} + + public async startNextScan(scanKey: string): Promise { + try { + const document = await this.source.startNextScan(scanKey); + return document ? this.mapper.toEntity(document) : null; + } catch (error) { + return null; + } + } + + public async createScanNodes( + scanKey: string, + startBlock: bigint, + endBlock: bigint + ): Promise { + try { + const { maxChunkSize } = this; + + const rootRange = BlockRangeScan.create(startBlock, endBlock, scanKey, 0); + const rangesToPersist = [rootRange]; + const childRanges = BlockRangeScan.createChildRanges(rootRange, maxChunkSize); + childRanges.forEach(range => rangesToPersist.push(range)); + + const documents = rangesToPersist.map(range => this.mapper.fromEntity(range)); + await this.source.insert(documents); + + return {}; + } catch (error) { + return { error: error as Error }; + } + } + + public async countScanNodes( + scanKey: string, + startBlock: bigint, + endBlock: bigint + ): Promise { + try { + return this.source.countScanNodes(scanKey, startBlock, endBlock); + } catch (error) { + return -1; + } + } + + public async removeAll(scanKey: string): Promise { + await this.source.removeAll(scanKey); + } + + public async hasScanKey( + scanKey: string, + startBlock?: bigint, + endBlock?: bigint + ): Promise { + return this.source.hasScanKey(scanKey, startBlock, endBlock); + } + + public async hasUnscannedNodes( + scanKey: string, + startBlock?: bigint, + endBlock?: bigint + ): Promise { + return this.source.hasUnscannedNodes(scanKey, startBlock, endBlock); + } + + public async updateScannedBlockNumber( + scanKey: string, + blockNumber: bigint + ): Promise { + await this.source.updateProcessedBlockNumber(scanKey, blockNumber); + } +} diff --git a/src/common/block-range-scanner/block-range-scan.source.ts b/src/common/block-range-scanner/block-range-scan.source.ts new file mode 100644 index 0000000..1b305a8 --- /dev/null +++ b/src/common/block-range-scanner/block-range-scan.source.ts @@ -0,0 +1,29 @@ +/* eslint-disable @typescript-eslint/no-unsafe-return */ + +import { DataSource } from '@alien-worlds/api-core'; + +export abstract class BlockRangeScanSource extends DataSource { + public abstract startNextScan(scanKey: string): Promise; + public abstract countScanNodes( + scanKey: string, + startBlock: bigint, + endBlock: bigint + ): Promise; + public abstract removeAll(scanKey: string); + public abstract hasScanKey( + scanKey: string, + startBlock?: bigint, + endBlock?: bigint + ): Promise; + public abstract hasUnscannedNodes( + scanKey: string, + startBlock?: bigint, + endBlock?: bigint + ): Promise; + public abstract findRangeForBlockNumber(blockNumber: bigint, scanKey: string); + public abstract findCompletedParentNode(document: T); + public abstract updateProcessedBlockNumber( + scanKey: string, + blockNumber: bigint + ): Promise; +} diff --git a/src/common/block-range-scanner/block-range-scan.ts b/src/common/block-range-scanner/block-range-scan.ts new file mode 100644 index 0000000..77ea5f3 --- /dev/null +++ b/src/common/block-range-scanner/block-range-scan.ts @@ -0,0 +1,186 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import crypto from 'crypto'; +import { parseToBigInt, removeUndefinedProperties } from '@alien-worlds/api-core'; +import { serialize } from 'v8'; + +export class BlockRangeScanParent { + /** + * @constructor + * @private + * @param {bigint} start + * @param {bigint} end + * @param {string} scanKey + */ + constructor( + public readonly start: bigint, + public readonly end: bigint, + public readonly scanKey: string, + public readonly treeDepth: number + ) {} + + public static create(start: bigint, end: bigint, scanKey: string, treeDepth: number) { + return new BlockRangeScanParent(start, end, scanKey, treeDepth); + } + + public toJson() { + const { start, end, scanKey, treeDepth } = this; + + return { start, end, scanKey, treeDepth }; + } +} + +/** + * @class + */ +export class BlockRangeScan { + constructor( + public readonly hash: string, + public readonly start: bigint, + public readonly end: bigint, + public readonly scanKey: string, + public readonly treeDepth: number, + public readonly parent?: BlockRangeScanParent, + public isLeafNode?: boolean, + public readonly processedBlock?: bigint, + public readonly timestamp?: Date, + public readonly startTimestamp?: Date, + public readonly endTimestamp?: Date + ) {} + + /** + * Create instance of the BlockRangeScanNode + * + * @static + * @param {bigint | number | string} startBlock + * @param {bigint | number | string} endBlock + * @param {string} scanKey + * @param {number} treeDepth + * @param {BlockRangeScanParent=} parent + * @param {boolean} isLeafNode + * @param {bigint | number | string} processedBlock + * @param {Date} timestamp + * @returns {BlockRangeScan} + */ + public static create( + startBlock: bigint | number | string, + endBlock: bigint | number | string, + scanKey: string, + treeDepth: number, + parent?: BlockRangeScanParent, + isLeafNode?: boolean, + processedBlock?: bigint | number | string, + timestamp?: Date, + startTimestamp?: Date, + endTimestamp?: Date + ): BlockRangeScan { + let currentRangeAsBigInt: bigint; + + if ( + typeof processedBlock === 'bigint' || + typeof processedBlock === 'number' || + typeof processedBlock === 'string' + ) { + currentRangeAsBigInt = parseToBigInt(processedBlock); + } + let started = startTimestamp; + + if (treeDepth === 0 && !startTimestamp) { + started = new Date(); + } + + const buffer = serialize({ + startBlock, + endBlock, + scanKey, + }); + const hash = crypto.createHash('sha1').update(buffer).digest('hex'); + + return new BlockRangeScan( + hash, + parseToBigInt(startBlock), + parseToBigInt(endBlock), + scanKey, + treeDepth, + parent, + isLeafNode, + currentRangeAsBigInt, + timestamp, + started, + endTimestamp + ); + } + + public static createChildRanges( + blockRange: BlockRangeScan, + maxChunkSize: number + ): BlockRangeScan[] { + const max = BigInt(maxChunkSize); + const { start: parentStart, end: parentEnd, scanKey, treeDepth } = blockRange; + + const rangesToPersist: BlockRangeScan[] = []; + let start = parentStart; + + while (start < parentEnd) { + const x = start + max; + const end = x < parentEnd ? x : parentEnd; + const range = BlockRangeScan.create( + start, + end, + scanKey, + treeDepth + 1, + BlockRangeScanParent.create(parentStart, parentEnd, scanKey, treeDepth) + ); + + if (end - start > max) { + const childRanges = BlockRangeScan.createChildRanges(range, maxChunkSize); + childRanges.forEach(range => rangesToPersist.push(range)); + } else { + range.setAsLeafNode(); + } + + rangesToPersist.push(range); + start += max; + } + + return rangesToPersist; + } + + public setAsLeafNode() { + this.isLeafNode = true; + } + + public toJson(): BlockRangeScan { + const { + start, + end, + scanKey, + isLeafNode, + treeDepth, + timestamp, + processedBlock, + startTimestamp, + endTimestamp, + hash, + } = this; + + const json = { + start, + end, + scanKey, + isLeafNode, + treeDepth, + timestamp, + processedBlock, + startTimestamp, + endTimestamp, + parent: null, + hash, + }; + + if (this.parent) { + json.parent = this.parent.toJson(); + } + + return removeUndefinedProperties(json); + } +} diff --git a/src/common/block-range-scanner/block-range-scanner.config.ts b/src/common/block-range-scanner/block-range-scanner.config.ts new file mode 100644 index 0000000..6883ff5 --- /dev/null +++ b/src/common/block-range-scanner/block-range-scanner.config.ts @@ -0,0 +1,4 @@ +export type BlockRangeScanConfig = { + maxChunkSize: number; + scanKey: string; +}; diff --git a/src/common/block-range-scanner/block-range-scanner.errors.ts b/src/common/block-range-scanner/block-range-scanner.errors.ts new file mode 100644 index 0000000..1f36a01 --- /dev/null +++ b/src/common/block-range-scanner/block-range-scanner.errors.ts @@ -0,0 +1,26 @@ +export class BlockNumberOutOfRangeError extends Error { + constructor(public readonly blockNumber: bigint, public readonly scanKey: string) { + super( + `Block number ${blockNumber} is out of range or is assigned to a different key than "${scanKey}"` + ); + } +} + +export class NoBlockRangeFoundError extends Error { + public static Token = 'NO_BLOCK_RANGE_FOUND_ERROR'; + constructor(public readonly scanKey: string) { + super(`No block range was found for the key "${scanKey}"`); + this.name = NoBlockRangeFoundError.Token; + } +} + +export class DuplicateBlockRangeScanError extends Error { + public static Token = 'DUPLICATE_BLOCK_RANGE_SCAN_ERROR'; + + constructor(scanKey: string, start: bigint, end: bigint) { + super( + `There is already a block range (${start}-${end}) scan entry in the database with the selected key "${scanKey}". Please select a new unique key.` + ); + this.name = DuplicateBlockRangeScanError.Token; + } +} diff --git a/src/common/block-range-scanner/block-range-scanner.ts b/src/common/block-range-scanner/block-range-scanner.ts new file mode 100644 index 0000000..1bbeaf6 --- /dev/null +++ b/src/common/block-range-scanner/block-range-scanner.ts @@ -0,0 +1,44 @@ +import { BlockRangeScan } from './block-range-scan'; +import { BlockRangeScanRepository, ScanRequest } from './block-range-scan.repository'; +import { DuplicateBlockRangeScanError } from './block-range-scanner.errors'; + +/** + * @class + */ +export class BlockRangeScanner { + constructor(private blockRangeScanRepository: BlockRangeScanRepository) {} + + public async createScanNodes( + key: string, + startBlock: bigint, + endBlock: bigint + ): Promise { + const hasScanKey = await this.blockRangeScanRepository.hasScanKey( + key, + startBlock, + endBlock + ); + + if (hasScanKey) { + return { error: new DuplicateBlockRangeScanError(key, startBlock, endBlock) }; + } + + return this.blockRangeScanRepository.createScanNodes(key, startBlock, endBlock); + } + + public async getNextScanNode(key: string): Promise { + return this.blockRangeScanRepository.startNextScan(key); + } + + public async hasUnscannedBlocks( + key: string, + startBlock?: bigint, + endBlock?: bigint + ): Promise { + return this.blockRangeScanRepository.hasUnscannedNodes(key, startBlock, endBlock); + } + + public async updateScanProgress(scanKey: string, blockNumber: bigint): Promise { + await this.blockRangeScanRepository.updateScannedBlockNumber(scanKey, blockNumber); + } +} diff --git a/src/common/block-range-scanner/index.ts b/src/common/block-range-scanner/index.ts new file mode 100644 index 0000000..e20b474 --- /dev/null +++ b/src/common/block-range-scanner/index.ts @@ -0,0 +1,6 @@ +export * from './block-range-scan.repository'; +export * from './block-range-scan.source'; +export * from './block-range-scan'; +export * from './block-range-scanner.config'; +export * from './block-range-scanner.errors'; +export * from './block-range-scanner'; diff --git a/src/common/block-state/__tests__/block-state.unit.test.ts b/src/common/block-state/__tests__/block-state.unit.test.ts new file mode 100644 index 0000000..eadfac7 --- /dev/null +++ b/src/common/block-state/__tests__/block-state.unit.test.ts @@ -0,0 +1,112 @@ +import { RepositoryImpl, Failure, Result } from '@alien-worlds/api-core'; +import { BlockState } from '../block-state'; + +jest.mock('@alien-worlds/api-core'); +jest.mock('./block-state.types'); + +describe('BlockState', () => { + let blockState; + let dataSourceMock; + let mapperMock; + let queryBuildersMock; + let queryBuilderMock; + + beforeEach(() => { + dataSourceMock = { + find: jest.fn(), + count: jest.fn(), + aggregate: jest.fn(), + update: jest.fn(), + insert: jest.fn(), + remove: jest.fn(), + startTransaction: jest.fn(), + commitTransaction: jest.fn(), + rollbackTransaction: jest.fn(), + } as any; + mapperMock = { + toEntity: jest.fn(), + fromEntity: jest.fn(), + getEntityKeyMapping: jest.fn(), + } as any; + queryBuildersMock = { + buildFindQuery: jest.fn(), + buildCountQuery: jest.fn(), + buildUpdateQuery: jest.fn(), + buildRemoveQuery: jest.fn(), + buildAggregationQuery: jest.fn(), + } as any; + queryBuilderMock = { + with: jest.fn(), + build: jest.fn(), + } as any; + + blockState = new BlockState( + dataSourceMock, + mapperMock, + queryBuilderMock, + queryBuilderMock + ); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should be a valid instance', () => { + expect(blockState).toBeInstanceOf(BlockState); + expect(blockState).toBeInstanceOf(RepositoryImpl); + }); + + describe('getState()', () => { + it('should return block state data', async () => { + const mockData = { + content: [ + { + lastModifiedTimestamp: new Date(), + actions: [], + tables: [], + blockNumber: 0n, + }, + ], + }; + blockState.find = jest.fn().mockResolvedValue(mockData); + + const result = await blockState.getState(); + + expect(blockState.find).toHaveBeenCalled(); + expect(result).toEqual(Result.withContent(mockData.content[0])); + }); + + it('should handle error properly', async () => { + const mockError = new Error('Database error'); + blockState.find = jest.fn().mockRejectedValue(mockError); + + const result = await blockState.getState(); + + expect(blockState.find).toHaveBeenCalled(); + expect(result).toEqual(Result.withFailure(Failure.fromError(mockError))); + }); + }); + + describe('updateBlockNumber()', () => { + it('should update block number and return true if successful', async () => { + const mockValue = 10n; + const mockData = { + content: { + modifiedCount: 1, + upsertedCount: 0, + }, + }; + blockState.update = jest.fn().mockResolvedValue(mockData); + + const result = await blockState.updateBlockNumber(mockValue); + + expect(blockState.update).toHaveBeenCalledWith( + queryBuilderMock.with({ blockNumber: mockValue }) + ); + expect(result).toEqual(Result.withContent(true)); + }); + + // Write similar tests for getBlockNumber() here... + }); +}); diff --git a/src/common/block-state/block-state.ts b/src/common/block-state/block-state.ts new file mode 100644 index 0000000..6475cba --- /dev/null +++ b/src/common/block-state/block-state.ts @@ -0,0 +1,102 @@ +import { + DataSource, + Failure, + Mapper, + QueryBuilder, + QueryBuilders, + RepositoryImpl, + Result, +} from '@alien-worlds/api-core'; +import { BlockStateModel } from './block-state.types'; + +/** + * A class representing a block state. + */ +export class BlockState extends RepositoryImpl { + /** + * Creates an instance of the BlockState class. + * + * @param {DataSource} source - The data source. + * @param {BlockStateMongoMapper} mapper - The data mapper. + * @param {QueryBuilders} queryBuilders - The query builders. + * @param {QueryBuilder} updateBlockNumberQueryBuilder - The query builder to update block number. + */ + constructor( + source: DataSource, + mapper: Mapper, + queryBuilders: QueryBuilders, + private updateBlockNumberQueryBuilder: QueryBuilder + ) { + super(source, mapper, queryBuilders); + } + + /** + * Fetches the current state of the data source. + * + * @returns {Promise>} - The result of the operation. + */ + public async getState(): Promise> { + try { + const { content: states } = await this.find(); + + if (states) { + const state = states[0]; + const { lastModifiedTimestamp, actions, tables, blockNumber } = state; + + return Result.withContent({ + lastModifiedTimestamp: lastModifiedTimestamp || new Date(), + actions: actions || [], + tables: tables || [], + blockNumber: blockNumber || 0n, + }); + } + + return Result.withContent({ + lastModifiedTimestamp: new Date(), + actions: [], + tables: [], + blockNumber: 0n, + }); + } catch (error) { + return Result.withFailure(Failure.fromError(error)); + } + } + + /** + * Updates the block number in the current state. + * + * @param {bigint} value - The new block number. + * @returns {Promise>} - The result of the operation. + */ + public async updateBlockNumber(value: bigint): Promise> { + this.updateBlockNumberQueryBuilder.with({ blockNumber: value }); + const { content, failure } = await this.update(this.updateBlockNumberQueryBuilder); + + if (failure) { + return Result.withFailure(failure); + } + + return Result.withContent(content.modifiedCount + content.upsertedCount > 0); + } + + /** + * Fetches the current block number from the current state. + * + * @returns {Promise>} - The result of the operation. + */ + public async getBlockNumber(): Promise> { + const { content: states, failure } = await this.find(); + + if (failure) { + return Result.withFailure(failure); + } + + if (states.length > 0) { + const state = states[0]; + + return Result.withContent(state.blockNumber); + } + + return Result.withContent(-1n); + } +} diff --git a/src/common/block-state/block-state.types.ts b/src/common/block-state/block-state.types.ts new file mode 100644 index 0000000..b7c8e9e --- /dev/null +++ b/src/common/block-state/block-state.types.ts @@ -0,0 +1,6 @@ +export type BlockStateModel = { + lastModifiedTimestamp: Date; + blockNumber: bigint; + actions: string[]; + tables: string[]; +}; diff --git a/src/common/block-state/index.ts b/src/common/block-state/index.ts new file mode 100644 index 0000000..42a8bdc --- /dev/null +++ b/src/common/block-state/index.ts @@ -0,0 +1,2 @@ +export * from './block-state'; +export * from './block-state.types'; diff --git a/src/common/common.enums.ts b/src/common/common.enums.ts new file mode 100644 index 0000000..af0dcbb --- /dev/null +++ b/src/common/common.enums.ts @@ -0,0 +1,5 @@ +export enum Mode { + Default = 'default', + Replay = 'replay', + Test = 'test', +} diff --git a/src/common/common.errors.ts b/src/common/common.errors.ts new file mode 100644 index 0000000..a10bca9 --- /dev/null +++ b/src/common/common.errors.ts @@ -0,0 +1,5 @@ +export class UnknownModeError extends Error { + constructor(mode: string) { + super(`Unknown mode "${mode}"`); + } +} diff --git a/src/common/common.types.ts b/src/common/common.types.ts new file mode 100644 index 0000000..2f6fdf3 --- /dev/null +++ b/src/common/common.types.ts @@ -0,0 +1,6 @@ +import { ConfigVars, UnknownObject } from '@alien-worlds/api-core'; + +export type DatabaseConfigBuilder = ( + vars: ConfigVars, + ...args: unknown[] +) => UnknownObject; diff --git a/src/common/common.utils.ts b/src/common/common.utils.ts new file mode 100644 index 0000000..d49f7a8 --- /dev/null +++ b/src/common/common.utils.ts @@ -0,0 +1,9 @@ +/** + * Checks if a given contract and action represent the 'setabi' action of the 'eosio' contract. + * + * @param {string} contract The contract name. + * @param {string} action The action name. + * @returns {boolean} Returns `true` if the contract and action match the 'eosio' 'setabi' action; otherwise, returns `false`. + */ +export const isSetAbiAction = (contract: string, action: string) => + contract === 'eosio' && action === 'setabi'; diff --git a/src/common/dependencies.ts b/src/common/dependencies.ts new file mode 100644 index 0000000..ea2a5df --- /dev/null +++ b/src/common/dependencies.ts @@ -0,0 +1,16 @@ +import { Result } from '@alien-worlds/api-core'; + +/** + * An abstract class representing a Process dependencies. + * + * @abstract + * @class Dependencies + */ +export abstract class Dependencies { + /** + * Initializes and configures the Dependencies instance. + * @abstract + * @returns {Promise} A promise that resolves when the initialization is complete. + */ + public abstract initialize(...args: unknown[]): Promise; +} diff --git a/src/common/featured/__tests__/featured.utils.unit.test.ts b/src/common/featured/__tests__/featured.utils.unit.test.ts new file mode 100644 index 0000000..67da7a2 --- /dev/null +++ b/src/common/featured/__tests__/featured.utils.unit.test.ts @@ -0,0 +1,82 @@ +import { FeaturedUtils } from '../featured.utils'; + +describe('FeaturedUtils', () => { + describe('readFeaturedContracts', () => { + it('should return an empty array for empty data', () => { + const data = {}; + const result = FeaturedUtils.readFeaturedContracts(data); + expect(result).toEqual([]); + }); + + it('should return an empty array for non-object data', () => { + const data = null; + const result = FeaturedUtils.readFeaturedContracts(data); + expect(result).toEqual([]); + }); + + it('should return an array of unique contracts from the data object', () => { + const data = { + contract: ['ContractA', 'ContractB'], + otherProp: 'some value', + }; + const result = FeaturedUtils.readFeaturedContracts(data); + expect(result).toEqual(['ContractA', 'ContractB']); + }); + + it('should return an array of unique contracts from nested data', () => { + let data = { + prop1: 'value1', + prop2: { + contract: 'ContractC', + prop3: { + contract: ['ContractD', 'ContractE'], + }, + }, + }; + let result = FeaturedUtils.readFeaturedContracts(data); + expect(result).toEqual(['ContractC', 'ContractD', 'ContractE']); + + const nested = { + traces: [ + { + prop1: 'value1', + prop2: { + contract: 'ContractC', + prop3: { + contract: ['ContractD', 'ContractE'], + }, + }, + }, + ], + deltas: [ + { + prop1: 'value1', + prop2: { + contract: 'ContractF', + prop3: { + contract: ['ContractA', 'ContractB'], + }, + }, + }, + ], + }; + result = FeaturedUtils.readFeaturedContracts(nested); + expect(result).toEqual([ + 'ContractC', + 'ContractD', + 'ContractE', + 'ContractF', + 'ContractA', + 'ContractB', + ]); + }); + + it('should ignore non-string values in the contract property', () => { + const data = { + contract: [123, 'ContractF', true, null], + }; + const result = FeaturedUtils.readFeaturedContracts(data); + expect(result).toEqual(['ContractF']); + }); + }); +}); diff --git a/src/common/featured/featured-contract.ts b/src/common/featured/featured-contract.ts new file mode 100644 index 0000000..0dd6589 --- /dev/null +++ b/src/common/featured/featured-contract.ts @@ -0,0 +1,34 @@ +import { parseToBigInt } from '@alien-worlds/api-core'; + +/** + * Class representing a FeaturedContract + * @class + * @public + */ +export class FeaturedContract { + /** + * Creates a new instance of the FeaturedContract + * @constructor + * @param {string} id - The ID of the contract + * @param {bigint} initialBlockNumber - The initial block number of the contract + * @param {string} account - The account associated with the contract + */ + constructor( + public id: string, + public initialBlockNumber: bigint, + public account: string + ) {} + + /** + * Creates a new instance of FeaturedContract with a specified account and initial block number, + * ID will be set to empty string by default + * @static + * @public + * @param {string} account - The account associated with the contract + * @param {string | number} initialBlockNumber - The initial block number of the contract + * @returns {FeaturedContract} A new instance of FeaturedContract + */ + public static create(account: string, initialBlockNumber: string | number) { + return new FeaturedContract('', parseToBigInt(initialBlockNumber), account); + } +} diff --git a/src/common/featured/featured.config.ts b/src/common/featured/featured.config.ts new file mode 100644 index 0000000..68b0ab2 --- /dev/null +++ b/src/common/featured/featured.config.ts @@ -0,0 +1,4 @@ +export type FeaturedConfig = { + serviceUrl: string; + rpcUrl: string; +}; diff --git a/src/common/featured/featured.errors.ts b/src/common/featured/featured.errors.ts new file mode 100644 index 0000000..fe06c82 --- /dev/null +++ b/src/common/featured/featured.errors.ts @@ -0,0 +1,37 @@ +export class PatternMatchError extends Error { + constructor(value: string, pattern: string) { + super(`The given value ${value} does not match the pattern ${pattern}`); + } +} + +export class MatcherNotFoundError extends Error { + constructor(label: string) { + super(`No match function assigned to the label: ${label}`); + } +} + +export class UndefinedPatternError extends Error { + constructor() { + super(`No pattern assigned to the criteria`); + } +} + +export class PatternMismatchError extends Error { + constructor() { + super( + `The length of the keys on the label does not match the number of keys in the pattern` + ); + } +} + +export class UnknownContentTypeError extends Error { + constructor(type: string) { + super(`Unknown type: ${type}`); + } +} + +export class MissingCriteriaError extends Error { + constructor(path: string) { + super(`No criteria found at: ${path}`); + } +} diff --git a/src/common/featured/featured.mapper.ts b/src/common/featured/featured.mapper.ts new file mode 100644 index 0000000..3b8fe7f --- /dev/null +++ b/src/common/featured/featured.mapper.ts @@ -0,0 +1,277 @@ +import { + MatcherNotFoundError, + PatternMatchError, + PatternMismatchError, + UndefinedPatternError, +} from './featured.errors'; +import { + MatchCriteria, + ProcessorMatchCriteria, + ProcessorMatcher, +} from './featured.types'; + +/** + * A mapper class for processing and matching contracts based on given criteria. + * This class can be extended by other processors that require specific match criteria. + */ +export class FeaturedMapper { + /** + * Map of processors matched by a matching function. + */ + protected processorByMatchers: ProcessorMatcher = new Map(); + /** + * Array of match criteria. + */ + protected matchCriteria: ProcessorMatchCriteria[] = []; + /** + * Set of contracts. + */ + protected contracts: Set = new Set(); + + /** + * Creates a new instance of the contract processor mapper. + * @param {ProcessorMatchCriteria[]} criteria - An array of match criteria for the processor. + * @param {MatchCriteriaType} pattern - The criteria pattern. + * @param {ProcessorMatcher} matchers - Optional map of matchers. + */ + constructor( + criteria: ProcessorMatchCriteria[], + protected pattern?: MatchCriteriaType, + matchers?: ProcessorMatcher + ) { + criteria.forEach(current => { + const { processor, matcher, ...rest } = current; + const { contract, code } = rest as unknown as MatchCriteria; + + if (Array.isArray(contract)) { + contract.forEach(contract => this.contracts.add(contract)); + } else if (typeof contract === 'string') { + this.contracts.add(contract); + } + + if (Array.isArray(code)) { + code.forEach(contract => this.contracts.add(contract)); + } else if (typeof code === 'string') { + this.contracts.add(code); + } + + if (matcher && !matchers?.has(matcher)) { + throw new MatcherNotFoundError(matcher); + } + + if (matcher && matchers.has(matcher)) { + this.processorByMatchers.set(processor, matchers.get(matcher)); + } else { + this.validateCriteria(rest as MatchCriteriaType); + + if (this.matchCriteria.indexOf(current) === -1) { + this.matchCriteria.push(current); + } + } + }); + } + + /** + * Validates the given match criteria. + * @param criteria - The criteria to validate. + */ + protected validateCriteria(criteria: MatchCriteriaType): void { + const keys = Object.keys(criteria); + + for (const key of keys) { + const values = criteria[key]; + for (const value of values) { + if (/^(\*|[A-Za-z0-9_.]*)$/g.test(value) === false) { + throw new PatternMatchError(value, '^(*|[A-Za-z0-9_.]*)$'); + } + } + } + } + + /** + * Determines if a candidate match criteria meets a reference match criteria. + * @param ref - The reference match criteria. + * @param candidate - The candidate match criteria. + * @returns True if a match is found, false otherwise. + */ + protected isMatch( + ref: ProcessorMatchCriteria, + candidate: MatchCriteriaType + ): boolean { + let matchFound = false; + const keys = Object.keys(candidate); + + for (const key of keys) { + const candidateValues = candidate[key]; + const refValues = ref[key]; + if (Array.isArray(refValues)) { + const values: string[] = Array.isArray(candidateValues) + ? candidateValues + : [candidateValues]; + const contains = values.some(value => refValues.includes(value)); + + if (refValues.includes('*') || contains) { + matchFound = true; + } else { + return false; + } + } + } + + return matchFound; + } + + /** + * Finds a processor match criteria for a given match criteria. + * @param criteria - The criteria to find a match for. + * @returns The matching processor match criteria if found, null otherwise. + */ + protected async findProcessorMatchCriteria( + criteria: MatchCriteriaType + ): Promise> { + const { processorByMatchers } = this; + const entries = Array.from(processorByMatchers.entries()); + + for (const entry of entries) { + const [processor, matcher] = entry; + if (await matcher(criteria)) { + const keys = Object.keys(criteria); + const matchCriteria = {} as MatchCriteriaType; + + for (const key of keys) { + matchCriteria[key] = ['*']; + } + + return { + ...matchCriteria, + processor, + }; + } + } + + return null; + } + + /** + * Checks if the criteria already exist in the array + * + * @param {ProcessorMatchCriteria} criteria + * @param {ProcessorMatchCriteria[]} array + * @returns + */ + protected criteriaExistsInArray( + criteria: ProcessorMatchCriteria, + array: ProcessorMatchCriteria[] + ): boolean { + return array.some(item => + Object.keys(item).every(key => item[key] === criteria[key]) + ); + } + + /** + * Determines if the given match criteria exists in the processor. + * @param criteria - The criteria to check. + * @returns True if the criteria exists, false otherwise. + */ + public async has(criteria: MatchCriteriaType): Promise { + const { matchCriteria, processorByMatchers } = this; + + for (const item of matchCriteria) { + if (this.isMatch(item, criteria)) { + return true; + } + } + + if (processorByMatchers.size > 0) { + const featured = await this.findProcessorMatchCriteria(criteria); + if (featured) { + if (this.criteriaExistsInArray(featured, matchCriteria) === false) { + matchCriteria.push(featured); + } + return true; + } + } + + return false; + } + + /** + * Gets all match criteria in the processor that match the given criteria. + * @param criteria - The criteria to match. + * @returns An array of matching criteria. + */ + public async get(criteria: MatchCriteriaType): Promise { + const { matchCriteria, processorByMatchers } = this; + const result: MatchCriteriaType[] = []; + + for (const item of matchCriteria) { + if (this.isMatch(item, criteria)) { + result.push(item); + } + } + + if (result.length === 0 && processorByMatchers.size > 0) { + const featured = await this.findProcessorMatchCriteria(criteria); + if (featured) { + if (this.criteriaExistsInArray(featured, matchCriteria) === false) { + matchCriteria.push(featured); + } + result.push(featured); + } + } + + return result; + } + + /** + * Gets the processor for the given label and criteria. + * @param label - The label to find a processor for. + * @param pattern - The match criteria pattern. + * @returns The processor if found, empty string otherwise. + */ + public async getProcessor(label: string, pattern?: MatchCriteriaType): Promise { + const { matchCriteria } = this; + const p = pattern || this.pattern; + + if (!p) { + throw new UndefinedPatternError(); + } + + const keys = Object.keys(p); + const parts = label.split(':').map(part => part.split(',')); + + if (parts.length !== keys.length) { + throw new PatternMismatchError(); + } + + const candidate = parts.reduce((result, part, i) => { + result[keys[i]] = part; + return result; + }, p); + + for (const criteriaRef of matchCriteria) { + if (this.isMatch(criteriaRef, candidate)) { + return criteriaRef.processor; + } + } + + const featured = await this.findProcessorMatchCriteria(candidate); + + if (featured) { + if (matchCriteria.indexOf(featured) === -1) { + matchCriteria.push(featured); + } + return featured.processor; + } + + return ''; + } + + /** + * Lists all contracts in the processor. + * @returns An array of contracts. + */ + public listContracts(): string[] { + return Array.from(this.contracts); + } +} diff --git a/src/common/featured/featured.ts b/src/common/featured/featured.ts new file mode 100644 index 0000000..c8d9680 --- /dev/null +++ b/src/common/featured/featured.ts @@ -0,0 +1,81 @@ +import { + FindParams, + Repository, + Result, + SmartContractService, + UnknownObject, + Where, +} from '@alien-worlds/api-core'; +import { FeaturedContract } from './featured-contract'; +import { FeaturedUtils } from './featured.utils'; + +export class Featured { + protected cache: Map = new Map(); + protected featuredContracts: string[]; + + constructor( + private repository: Repository, + private smartContractService: SmartContractService, + featuredJson: UnknownObject + ) { + this.featuredContracts = FeaturedUtils.readFeaturedContracts(featuredJson); + } + + /** + * Reads multiple contracts and returns the results as an array of FeaturedContract objects. + * + * @abstract + * @param {string[]} contracts - An array of contract addresses or identifiers. + * @returns {Promise>} A Promise that resolves to an array of FeaturedContract objects. + */ + public async readContracts( + data: string[] | UnknownObject + ): Promise> { + const list: FeaturedContract[] = []; + + const contracts = Array.isArray(data) + ? data + : FeaturedUtils.readFeaturedContracts(data); + + for (const contract of contracts) { + if (this.cache.has(contract)) { + list.push(this.cache.get(contract)); + } else { + const { content: contracts, failure } = await this.repository.find( + FindParams.create({ where: new Where().valueOf('account').isEq(contract) }) + ); + + if (failure) { + return Result.withFailure(failure); + } + + if (contracts.length > 0) { + const featuredContract = contracts[0]; + this.cache.set(featuredContract.account, featuredContract); + list.push(featuredContract); + } else { + const fetchResult = await this.smartContractService.getStats(contract); + + if (fetchResult.isFailure) { + return Result.withFailure(fetchResult.failure); + } + if (fetchResult.content) { + const featuredContract = FeaturedContract.create( + fetchResult.content.account_name, + fetchResult.content.first_block_num + ); + this.cache.set(featuredContract.account, featuredContract); + this.repository.add([featuredContract]); + list.push(featuredContract); + } + } + } + } + + return Result.withContent(list); + } + + public isFeatured(contract: string): boolean { + return this.featuredContracts.includes(contract); + } +} diff --git a/src/common/featured/featured.types.ts b/src/common/featured/featured.types.ts new file mode 100644 index 0000000..ce965ca --- /dev/null +++ b/src/common/featured/featured.types.ts @@ -0,0 +1,49 @@ +export type FeaturedContractModel = { + account: string; + initialBlockNumber: bigint; +}; + +export type FetchContractResponse = { + account: string; + block_num: string | number; +}; + +export type CriteriaValue = string | string[]; + +export type MatchCriteria = { + [key: string]: CriteriaValue; +}; + +export type ProcessorMatchCriteria = { + matcher?: string; + processor: string; +} & MatchCriteriaType; + +export type ProcessorMatcher = Map< + string, + MatchFunction +>; + +export type MatchFunction = ( + criteria: MatchCriteriaType +) => Promise; + +export type ContractTraceMatchCriteria = MatchCriteria & { + shipTraceMessageName: string[]; + shipActionTraceMessageName: string[]; + contract: string[]; + action: string[]; +}; + +export type ContractDeltaMatchCriteria = MatchCriteria & { + shipDeltaMessageName: string[]; + name: string[]; + code: string[]; + scope: string[]; + table: string[]; +}; + +export type FeaturedContractDataCriteria = { + traces: ProcessorMatchCriteria[]; + deltas: ProcessorMatchCriteria[]; +}; diff --git a/src/common/featured/featured.utils.ts b/src/common/featured/featured.utils.ts new file mode 100644 index 0000000..71138ad --- /dev/null +++ b/src/common/featured/featured.utils.ts @@ -0,0 +1,58 @@ +import fetch from 'node-fetch'; +import { UnknownObject } from '@alien-worlds/api-core'; +import { FeaturedContractDataCriteria } from './featured.types'; +import { existsSync, readFileSync } from 'fs'; + +export class FeaturedUtils { + public static readFeaturedContracts(data: UnknownObject | unknown[]): string[] { + const contracts = new Set(); + if (!data) { + return []; + } + Object.keys(data).forEach(key => { + const value = data[key]; + + if ((key === 'contract' || key === 'code') && Array.isArray(value)) { + value.forEach(contract => { + if (typeof contract === 'string') { + contracts.add(contract); + } + }); + } else if ((key === 'contract' || key === 'code') && typeof value === 'string') { + if (typeof value === 'string') { + contracts.add(value); + } + } else if (Array.isArray(value) || typeof value === 'object') { + const result = this.readFeaturedContracts(value); + result.forEach(contract => { + contracts.add(contract); + }); + } + }); + return Array.from(contracts); + } + + public static async fetchCriteria( + filePath: string + ): Promise { + const urlRegex = + /^((ftp|http|https):\/\/)?(www\.)?((([a-z\d]([a-z\d-]*[a-z\d])*)\.)+[a-z]{2,}|((\d{1,3}\.){3}\d{1,3}))(:\d+)?(\/[-a-z\d%_.~+]*)*(\?[;&a-z\d%_.~+=-]*)?(#[-a-z\d_]*)?$/i; + const isHttpPath = urlRegex.test(filePath); + + if (isHttpPath) { + const response = await fetch(filePath); + if (!response.ok) { + return null; + } + + return await response.json(); + } else { + if (existsSync(filePath)) { + const fileContent = readFileSync(filePath, 'utf-8'); + return JSON.parse(fileContent); + } + + return null; + } + } +} diff --git a/src/common/featured/index.ts b/src/common/featured/index.ts new file mode 100644 index 0000000..067d4e7 --- /dev/null +++ b/src/common/featured/index.ts @@ -0,0 +1,7 @@ +export * from './featured-contract'; +export * from './featured'; +export * from './featured.config'; +export * from './featured.errors'; +export * from './featured.mapper'; +export * from './featured.types'; +export * from './featured.utils'; diff --git a/src/common/index.ts b/src/common/index.ts new file mode 100644 index 0000000..f21936a --- /dev/null +++ b/src/common/index.ts @@ -0,0 +1,12 @@ +export * from './abis'; +export * from './block-range-scanner'; +export * from './block-state'; +export * from './featured'; +export * from './processor-task-queue'; +export * from './types'; +export * from './unprocessed-block-queue'; +export * from './common.enums'; +export * from './common.errors'; +export * from './common.utils'; +export * from './common.types'; +export * from './dependencies'; diff --git a/src/common/processor-task-queue/index.ts b/src/common/processor-task-queue/index.ts new file mode 100644 index 0000000..60a5071 --- /dev/null +++ b/src/common/processor-task-queue/index.ts @@ -0,0 +1,7 @@ +export * from './processor-task-queue'; +export * from './processor-task-queue.config'; +export * from './processor-task.enums'; +export * from './processor-task.errors'; +export * from './processor-task.source'; +export * from './processor-task'; +export * from './processor-task.types'; diff --git a/src/common/processor-task-queue/processor-task-queue.config.ts b/src/common/processor-task-queue/processor-task-queue.config.ts new file mode 100644 index 0000000..997b65e --- /dev/null +++ b/src/common/processor-task-queue/processor-task-queue.config.ts @@ -0,0 +1,4 @@ +export type ProcessorTaskQueueConfig = { + interval?: number; + [key: string]: unknown; +}; diff --git a/src/common/processor-task-queue/processor-task-queue.ts b/src/common/processor-task-queue/processor-task-queue.ts new file mode 100644 index 0000000..79527d4 --- /dev/null +++ b/src/common/processor-task-queue/processor-task-queue.ts @@ -0,0 +1,61 @@ +import { DataSource, DataSourceError, Mapper, log } from '@alien-worlds/api-core'; +import { ProcessorTaskSource } from './processor-task.source'; +import { ProcessorTask } from './processor-task'; +import { ProcessorTaskModel } from './processor-task.types'; + +export class ProcessorTaskQueue { + constructor( + protected source: ProcessorTaskSource, + protected mapper: Mapper, + protected unsuccessfulSource: DataSource, + protected onlyAdd = false + ) {} + + public async nextTask(mode?: string): Promise { + // TODO: temporary solution - testing session options + if (this.onlyAdd) { + log(`Operation not allowed, queue created with option onlyAdd`); + return; + } + + try { + const dto = await this.source.nextTask(mode); + if (dto) { + return this.mapper.toEntity(dto); + } + return null; + } catch (error) { + log(`Could not get next task due to: ${error.message}`); + return null; + } + } + + public async addTasks(tasks: ProcessorTask[], unsuccessful?: boolean): Promise { + const source = unsuccessful ? this.unsuccessfulSource : this.source; + try { + const dtos = tasks.map(task => this.mapper.fromEntity(task)); + await source.insert(dtos); + } catch (error) { + const { error: concernError } = error; + const concernErrorMessage = (concernError)?.message || ''; + log(`Could not add tasks due to: ${error.message}. ${concernErrorMessage}`); + } + } + + public async stashUnsuccessfulTask( + task: ProcessorTask, + error: { message: string; stack: string } | Error + ): Promise { + try { + const { message, stack } = error; + const document: ProcessorTaskModel = this.mapper.fromEntity( + task + ) as ProcessorTaskModel; + document.error = { message, stack }; + + await this.unsuccessfulSource.insert([document]); + } catch (sourceError) { + log(`Could not stash failed task due to: ${error.message}`); + } + } +} diff --git a/src/common/processor-task-queue/processor-task.enums.ts b/src/common/processor-task-queue/processor-task.enums.ts new file mode 100644 index 0000000..13de22b --- /dev/null +++ b/src/common/processor-task-queue/processor-task.enums.ts @@ -0,0 +1,4 @@ +export enum ProcessorTaskType { + Trace = 'trace', + Delta = 'delta', +} diff --git a/src/common/processor-task-queue/processor-task.errors.ts b/src/common/processor-task-queue/processor-task.errors.ts new file mode 100644 index 0000000..c115c9f --- /dev/null +++ b/src/common/processor-task-queue/processor-task.errors.ts @@ -0,0 +1,5 @@ +export class UnknownProcessorTypeError extends Error { + constructor(type: string) { + super(`Unknown processor type: ${type}`); + } +} diff --git a/src/common/processor-task-queue/processor-task.source.ts b/src/common/processor-task-queue/processor-task.source.ts new file mode 100644 index 0000000..f4f416d --- /dev/null +++ b/src/common/processor-task-queue/processor-task.source.ts @@ -0,0 +1,5 @@ +import { DataSource } from '@alien-worlds/api-core'; + +export abstract class ProcessorTaskSource extends DataSource { + public abstract nextTask(mode?: string): Promise; +} diff --git a/src/common/processor-task-queue/processor-task.ts b/src/common/processor-task-queue/processor-task.ts new file mode 100644 index 0000000..87c0f5d --- /dev/null +++ b/src/common/processor-task-queue/processor-task.ts @@ -0,0 +1,116 @@ +import crypto from 'crypto'; +import { serialize } from 'v8'; +import { DeltaProcessorContentModel, ProcessorTaskError } from './processor-task.types'; +import { ActionTrace } from '../types'; +import { ProcessorTaskType } from './processor-task.enums'; + +export class ProcessorTask { + public static createActionProcessorTask( + abi: string, + mode: string, + shipTraceMessageName: string, + shipMessageName: string, + transactionId: string, + actionTrace: ActionTrace, + blockNumber: bigint, + blockTimestamp: Date, + isFork: boolean + ) { + const { + act: { account, name, data }, + receipt, + } = actionTrace; + + const buffer = serialize({ + transaction_id: transactionId, + action_trace: actionTrace, + block_num: blockNumber.toString(), + block_timestamp: blockTimestamp, + }); + const hashBuffer = serialize({ + account, + name, + data, + transactionId, + blockNumber, + blockTimestamp, + receipt, + }); + + const hash = crypto.createHash('sha1').update(hashBuffer).digest('hex'); + const shortId = `${account}:${name}`; + const label = `${shipTraceMessageName}:${shipMessageName}:${shortId}`; + + return new ProcessorTask( + null, + abi, + shortId, + label, + null, + ProcessorTaskType.Trace, + mode, + buffer, + hash, + blockNumber, + blockTimestamp, + isFork + ); + } + + public static createDeltaProcessorTask( + abi: string, + mode: string, + type: string, + name: string, + code: string, + scope: string, + table: string, + blockNumber: bigint, + blockTimestamp: Date, + data: Uint8Array, + isFork: boolean + ) { + const content: DeltaProcessorContentModel = { + ship_delta_message_name: type, + name, + row_data: data, + block_num: blockNumber, + block_timestamp: blockTimestamp, + }; + const buffer = serialize(content); + const hash = crypto.createHash('sha1').update(buffer).digest('hex'); + const shortId = `${code}:${scope}:${table}`; + const label = `${type}:${name}:${shortId}`; + + return new ProcessorTask( + null, + abi, + shortId, + label, + null, + ProcessorTaskType.Delta, + mode, + buffer, + hash, + blockNumber, + blockTimestamp, + isFork + ); + } + + constructor( + public readonly id: string, + public readonly abi: string, + public readonly shortId: string, + public readonly label: string, + public readonly timestamp: Date, + public readonly type: string, + public readonly mode: string, + public readonly content: Buffer, + public readonly hash: string, + public readonly blockNumber: bigint, + public readonly blockTimestamp: Date, + public readonly isFork: boolean, + public readonly error?: ProcessorTaskError + ) {} +} diff --git a/src/common/processor-task-queue/processor-task.types.ts b/src/common/processor-task-queue/processor-task.types.ts new file mode 100644 index 0000000..fac5586 --- /dev/null +++ b/src/common/processor-task-queue/processor-task.types.ts @@ -0,0 +1,36 @@ +import { ActionTrace } from '../types'; + +export type ProcessorTaskError = { + message: string; + stack: string; +}; + +export type ProcessorTaskModel = { + id: string; + isFork: string; + abi: string; + path: string; + label: string; + timestamp: Date; + type: string; + mode: string; + content: Buffer; + hash: string; + error?: ProcessorTaskError; +}; + +export type DeltaProcessorContentModel = { + ship_delta_message_name: string; + name: string; + block_num: bigint; + block_timestamp: Date; + row_data: Uint8Array; +}; + +export type ActionProcessorContentModel = { + ship_trace_message_name: string; + transaction_id: string; + block_num: bigint; + block_timestamp: Date; + action_trace: ActionTrace; +}; diff --git a/src/common/types/action-trace.types.ts b/src/common/types/action-trace.types.ts new file mode 100644 index 0000000..b4a80be --- /dev/null +++ b/src/common/types/action-trace.types.ts @@ -0,0 +1,45 @@ +export type AuthSequence = { + account: string; + sequence: string; +}; + +export type Receipt = { + receiver: string; + act_digest: string; + global_sequence: string; + recv_sequence: string; + auth_sequence: AuthSequence[]; + code_sequence: number; + abi_sequence: number; +}; + +export type ReceiptByName = [string, Receipt]; + +export type ActAuth = { + actor: string; + permission: string; +}; + +export type Act = { + account: string; + name: string; + authorization: ActAuth; + data: Uint8Array; +}; + +export type ActionTrace = { + ship_message_name?: string; + action_ordinal?: number; + creator_action_ordinal?: number; + receipt?: ReceiptByName; + receiver?: string; + act?: Act; + context_free?: boolean; + elapsed?: string; + console?: string; + account_ram_deltas?: unknown[]; + except?: unknown; + error_code?: string | number; +}; + +export type ActionTraceByName = [string, ActionTrace]; diff --git a/src/common/types/block.types.ts b/src/common/types/block.types.ts new file mode 100644 index 0000000..21f8adb --- /dev/null +++ b/src/common/types/block.types.ts @@ -0,0 +1,20 @@ +export type BlockNumberWithIdModel = { + block_num?: unknown; + block_id?: string; +}; + +export type BlockModel< + BlockType = Uint8Array, + TracesType = Uint8Array, + DeltasType = Uint8Array +> = { + head?: BlockNumberWithIdModel; + this_block?: BlockNumberWithIdModel; + last_irreversible?: BlockNumberWithIdModel; + prev_block?: BlockNumberWithIdModel; + block?: BlockType; + traces?: TracesType; + deltas?: DeltasType; + abi_version?: string; + [key: string]: unknown; +}; diff --git a/src/common/types/delta.types.ts b/src/common/types/delta.types.ts new file mode 100644 index 0000000..1beb87c --- /dev/null +++ b/src/common/types/delta.types.ts @@ -0,0 +1,11 @@ +export type DeltaRow = { + present?: number; + data?: Uint8Array; +}; + +export type Delta = { + name?: string; + rows?: DeltaRow[]; +}; + +export type DeltaByName = [string, Delta]; diff --git a/src/common/types/index.ts b/src/common/types/index.ts new file mode 100644 index 0000000..d179343 --- /dev/null +++ b/src/common/types/index.ts @@ -0,0 +1,6 @@ +export * from './action-trace.types'; +export * from './signed-block.types'; +export * from './delta.types'; +export * from './trace.types'; +export * from './block.types'; +export * from './transaction.types'; diff --git a/src/common/types/signed-block.types.ts b/src/common/types/signed-block.types.ts new file mode 100644 index 0000000..acfe49e --- /dev/null +++ b/src/common/types/signed-block.types.ts @@ -0,0 +1,15 @@ +import { Transaction } from './transaction.types'; + +export type SignedBlock = { + timestamp: string; + producer: string; + confirmed: number; + previous: string; + transaction_mroot: string; + action_mroot: string; + schedule_version: number; + new_producers: unknown; + header_extensions: unknown[]; + producer_signature: string; + transactions: Transaction[]; +}; diff --git a/src/common/types/trace.types.ts b/src/common/types/trace.types.ts new file mode 100644 index 0000000..5bb8ddd --- /dev/null +++ b/src/common/types/trace.types.ts @@ -0,0 +1,33 @@ +import { ActionTraceByName } from './action-trace.types'; + +export type Partial = { + expiration: string; + ref_block_num: number; + ref_block_prefix: number; + max_net_usage_words: number; + max_cpu_usage_ms: number; + delay_sec: number; + transaction_extensions: unknown[]; + signatures: unknown[]; + context_free_data: unknown[]; +}; + +export type PartialByType = [string, Partial]; + +export type Trace = { + id?: string; + status?: number; + cpu_usage_us?: number; + net_usage_words?: number; + elapsed?: string; + net_usage?: string; + scheduled?: boolean; + action_traces?: ActionTraceByName[]; + account_ram_delta?: unknown; + except?: unknown; + error_code?: number | string; + failed_dtrx_trace?: unknown; + partial?: PartialByType; +}; + +export type TraceByName = [string, Trace]; diff --git a/src/common/types/transaction.types.ts b/src/common/types/transaction.types.ts new file mode 100644 index 0000000..f3022ef --- /dev/null +++ b/src/common/types/transaction.types.ts @@ -0,0 +1,14 @@ +export type PackedTrx = { + signatures: string[]; + compression: number; + packed_context_free_data: unknown; + packed_trx: Uint8Array; +}; +export type TrxByName = [string, PackedTrx | string]; + +export type Transaction = { + status: number; + cpu_usage_us: number; + net_usage_words: number; + trx: TrxByName; +}; diff --git a/src/common/unprocessed-block-queue/index.ts b/src/common/unprocessed-block-queue/index.ts new file mode 100644 index 0000000..3cc98c1 --- /dev/null +++ b/src/common/unprocessed-block-queue/index.ts @@ -0,0 +1,4 @@ +export * from './unprocessed-block-queue'; +export * from './unprocessed-block-queue.source'; +export * from './unprocessed-block-queue.errors'; +export * from './unprocessed-block-queue.types'; diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.errors.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.errors.ts new file mode 100644 index 0000000..09d8512 --- /dev/null +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.errors.ts @@ -0,0 +1,14 @@ +export class BlockNotFoundError extends Error {} +export class DuplicateBlocksError extends Error { + constructor() { + super(`Some blocks in the specified range are already in the database.`); + this.name = 'DuplicateBlocksError'; + } +} + +export class UnprocessedBlocksOverloadError extends Error { + constructor(min: bigint, max: bigint) { + super(`Blocks ${min}-${max} were read before the overload occurred.`); + this.name = 'UnprocessedBlocksOverloadError'; + } +} diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.source.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.source.ts new file mode 100644 index 0000000..b631975 --- /dev/null +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.source.ts @@ -0,0 +1,6 @@ +import { DataSource } from '@alien-worlds/api-core'; + +export abstract class UnprocessedBlockSource extends DataSource { + public abstract next(): Promise; + public abstract bytesSize(): Promise; +} diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.ts new file mode 100644 index 0000000..2e04277 --- /dev/null +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.ts @@ -0,0 +1,143 @@ +import { + DataSourceError, + Failure, + log, + Mapper, + parseToBigInt, + Result, +} from '@alien-worlds/api-core'; +import { + BlockNotFoundError, + DuplicateBlocksError, + UnprocessedBlocksOverloadError, +} from './unprocessed-block-queue.errors'; +import { Block } from '@alien-worlds/block-reader'; +import { UnprocessedBlockSource } from './unprocessed-block-queue.source'; +import { BlockModel } from '../types/block.types'; + +export abstract class UnprocessedBlockQueueReader { + public abstract next(): Promise>; +} + +export class UnprocessedBlockQueue + implements UnprocessedBlockQueueReader +{ + protected cache: Block[] = []; + protected overloadHandler: (size: number) => void; + protected beforeSendBatchHandler: () => void; + protected afterSendBatchHandler: () => void; + + constructor( + protected collection: UnprocessedBlockSource, + protected mapper: Mapper, + protected maxBytesSize: number, + protected batchSize: number + ) {} + + private async sendBatch() { + const addedBlockNumbers = []; + this.beforeSendBatchHandler(); + const documnets = this.cache.map(block => this.mapper.fromEntity(block)); + const result = await this.collection.insert(documnets); + result.forEach(model => { + addedBlockNumbers.push(parseToBigInt((model as BlockModel).this_block.block_num)); + }); + this.cache = []; + + if (this.maxBytesSize > 0 && this.overloadHandler) { + const sorted = addedBlockNumbers.sort(); + const min = sorted[0]; + const max = sorted.reverse()[0]; + + const currentSize = await this.collection.bytesSize(); + if (currentSize >= this.maxBytesSize) { + this.overloadHandler(currentSize); + throw new UnprocessedBlocksOverloadError(min, max); + } + } + + this.afterSendBatchHandler(); + + return addedBlockNumbers; + } + + public async getBytesSize(): Promise> { + try { + const currentSize = await this.collection.bytesSize(); + return Result.withContent(currentSize); + } catch (error) { + return Result.withFailure(Failure.fromError(error)); + } + } + + public async add(block: Block, isLast = false): Promise> { + try { + let addedBlockNumbers: bigint[] = []; + + if (this.cache.length < this.batchSize) { + this.cache.push(block); + } + + if (this.cache.length === this.batchSize || isLast) { + addedBlockNumbers = await this.sendBatch(); + } + + return Result.withContent(addedBlockNumbers); + } catch (error) { + // it is important to clear the cache in case of errors + this.cache = []; + + if (error instanceof DataSourceError && error.isDuplicateError) { + this.afterSendBatchHandler(); + return Result.withFailure(Failure.fromError(new DuplicateBlocksError())); + } + return Result.withFailure(Failure.fromError(error)); + } + } + + public async next(): Promise> { + try { + const document = await this.collection.next(); + if (document) { + if (this.maxBytesSize > -1 && this.afterSendBatchHandler) { + if ((await this.collection.count()) === 0 && this.afterSendBatchHandler) { + this.afterSendBatchHandler(); + } + } + + return Result.withContent(this.mapper.toEntity(document)); + } + return Result.withFailure(Failure.fromError(new BlockNotFoundError())); + } catch (error) { + log(`Could not get next task due to: ${error.message}`); + return Result.withFailure(Failure.fromError(error)); + } + } + + public async getMax(): Promise> { + try { + const documents = await this.collection.aggregate({ + pipeline: [{ $sort: { 'this_block.block_num': -1 } }, { $limit: 1 }], + }); + if (documents.length > 0) { + return Result.withContent(this.mapper.toEntity(documents[0])); + } + return Result.withFailure(Failure.fromError(new BlockNotFoundError())); + } catch (error) { + log(`Could not get block with highest block number due to: ${error.message}`); + return Result.withFailure(Failure.fromError(error)); + } + } + + public afterSendBatch(handler: () => void): void { + this.afterSendBatchHandler = handler; + } + + public beforeSendBatch(handler: () => void): void { + this.beforeSendBatchHandler = handler; + } + + public onOverload(handler: (size: number) => void): void { + this.overloadHandler = handler; + } +} diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.types.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.types.ts new file mode 100644 index 0000000..9c3ce4e --- /dev/null +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.types.ts @@ -0,0 +1,6 @@ +export type UnprocessedBlockQueueConfig = { + maxBytesSize: number; + batchSize: number; + sizeCheckInterval?: number; + [key: string]: unknown; +}; diff --git a/src/config/config.types.ts b/src/config/config.types.ts index 405a1e2..8814ec2 100644 --- a/src/config/config.types.ts +++ b/src/config/config.types.ts @@ -1,15 +1,8 @@ -import { - BootstrapConfig, - FilterConfig, - ProcessorConfig, - ReaderConfig, -} from '@alien-worlds/history-tools-common'; import { ApiConfig } from '../api'; - -export type BlockchainConfig = { - endpoint: string; - chainId: string; -}; +import { BootstrapConfig } from '../bootstrap'; +import { FilterConfig } from '../filter'; +import { ProcessorConfig } from '../processor'; +import { ReaderConfig } from '../reader'; export type HistoryToolsConfig = { api: ApiConfig; diff --git a/src/config/index.ts b/src/config/index.ts index 3debd86..e550254 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -1,29 +1,22 @@ +import { FilterCommandOptions, FilterConfig } from '../filter'; +import { ApiConfig } from '../api'; +import { BlockchainConfig, BootstrapCommandOptions, BootstrapConfig } from '../bootstrap'; +import { ReaderCommandOptions, ReaderConfig } from '../reader'; +import { ProcessorCommandOptions, ProcessorConfig } from '../processor'; +import { HistoryToolsConfig } from './config.types'; +import { ConfigVars, UnknownObject, parseToBigInt } from '@alien-worlds/api-core'; import { AbisConfig, AbisServiceConfig, BlockRangeScanConfig, - BlockReaderConfig, - BlockchainConfig, - BootstrapConfig, - ConfigVars, FeaturedConfig, FeaturedContractDataCriteria, - FilterConfig, - ProcessorConfig, ProcessorTaskQueueConfig, - ReaderConfig, - UnknownObject, UnprocessedBlockQueueConfig, - WorkersConfig, - buildBroadcastConfig, - parseToBigInt, -} from '@alien-worlds/history-tools-common'; -import { FilterCommandOptions } from '../filter'; -import { ApiConfig } from '../api'; -import { BootstrapCommandOptions } from '../bootstrap'; -import { ReaderCommandOptions } from '../reader'; -import { ProcessorCommandOptions } from '../processor'; -import { HistoryToolsConfig } from './config.types'; +} from '../common'; +import { BlockReaderConfig } from '@alien-worlds/block-reader'; +import { WorkersConfig } from '@alien-worlds/workers'; +import { buildBroadcastConfig } from '@alien-worlds/broadcast'; export * from './config.types'; diff --git a/src/filter/filter.config.ts b/src/filter/filter.config.ts new file mode 100644 index 0000000..049a6bb --- /dev/null +++ b/src/filter/filter.config.ts @@ -0,0 +1,23 @@ +import { UnknownObject } from '@alien-worlds/api-core'; +import { BroadcastConfig } from '@alien-worlds/broadcast'; +import { WorkersConfig } from '@alien-worlds/workers'; +import { + AbisConfig, + FeaturedConfig, + ProcessorTaskQueueConfig, + UnprocessedBlockQueueConfig, +} from '../common'; + +export type FilterConfig = { + mode: string; + broadcast: BroadcastConfig; + workers: WorkersConfig; + featured: FeaturedConfig; + abis: AbisConfig; + database: DatabaseConfig; + processorTaskQueue: ProcessorTaskQueueConfig; + unprocessedBlockQueue: UnprocessedBlockQueueConfig; + maxBytesSize?: number; + batchSize?: number; + [key: string]: unknown; +}; diff --git a/src/filter/filter.dependencies.ts b/src/filter/filter.dependencies.ts new file mode 100644 index 0000000..0e7fef3 --- /dev/null +++ b/src/filter/filter.dependencies.ts @@ -0,0 +1,27 @@ +import { Result } from '@alien-worlds/api-core'; +import { BroadcastClient } from '@alien-worlds/broadcast'; +import { UnprocessedBlockQueue } from '../common/unprocessed-block-queue'; +import { Dependencies } from '../common/dependencies'; +import { FilterConfig } from './filter.config'; +import { DatabaseConfigBuilder } from '../common'; +import { FilterAddons } from './filter.types'; + +/** + * An abstract class representing a Filter dependencies. + * @class FilterDependencies + */ +export abstract class FilterDependencies< + UnprocessedBlockModel = unknown +> extends Dependencies { + public broadcastClient: BroadcastClient; + public unprocessedBlockQueue: UnprocessedBlockQueue; + public workerLoaderPath?: string; + public workerLoaderDependenciesPath: string; + + public databaseConfigBuilder: DatabaseConfigBuilder; + + public abstract initialize( + config: FilterConfig, + addons?: FilterAddons + ): Promise; +} diff --git a/src/filter/filter.runner.ts b/src/filter/filter.runner.ts index 280991d..643a2ab 100644 --- a/src/filter/filter.runner.ts +++ b/src/filter/filter.runner.ts @@ -1,10 +1,7 @@ import { log } from '@alien-worlds/api-core'; import { BlockJson } from '@alien-worlds/block-reader'; -import { - BlockNotFoundError, - UnprocessedBlockQueueReader, -} from '@alien-worlds/history-tools-common'; import { WorkerMessage, WorkerPool } from '@alien-worlds/workers'; +import { BlockNotFoundError, UnprocessedBlockQueueReader } from '../common'; export class FilterRunner { private interval: NodeJS.Timeout; diff --git a/src/filter/filter.types.ts b/src/filter/filter.types.ts index c63bdb8..3e99f31 100644 --- a/src/filter/filter.types.ts +++ b/src/filter/filter.types.ts @@ -1,4 +1,4 @@ -import { FilterConfig } from '@alien-worlds/history-tools-common'; +import { FilterConfig } from "./filter.config"; export type FilterSharedData = { config: FilterConfig; diff --git a/src/filter/filter.worker-loader.dependencies.ts b/src/filter/filter.worker-loader.dependencies.ts new file mode 100644 index 0000000..189551f --- /dev/null +++ b/src/filter/filter.worker-loader.dependencies.ts @@ -0,0 +1,24 @@ +import { Serializer } from '@alien-worlds/api-core'; +import { ShipAbis } from '@alien-worlds/block-reader'; +import { ProcessorTaskQueue } from '../common/processor-task-queue'; +import { WorkerLoaderDependencies } from '@alien-worlds/workers'; +import { Featured, FeaturedContractDataCriteria } from '../common/featured'; +import { Abis } from '../common'; +import { FilterConfig } from './filter.config'; + +/** + * An abstract class representing a FilterWorkerLoader dependencies. + * @class FilterWorkerLoaderDependencies + */ +export abstract class FilterWorkerLoaderDependencies extends WorkerLoaderDependencies { + public processorTaskQueue: ProcessorTaskQueue; + public abis: Abis; + public shipAbis: ShipAbis; + public featured: Featured; + public serializer: Serializer; + + public abstract initialize( + config: FilterConfig, + featuredCriteriaPath: string + ): Promise; +} diff --git a/src/filter/filter.worker-loader.ts b/src/filter/filter.worker-loader.ts index b1b491c..fb85846 100644 --- a/src/filter/filter.worker-loader.ts +++ b/src/filter/filter.worker-loader.ts @@ -1,7 +1,7 @@ import { Worker, DefaultWorkerLoader } from '@alien-worlds/workers'; import { FilterSharedData } from './filter.types'; import FilterWorker from './filter.worker'; -import { FilterWorkerLoaderDependencies } from '@alien-worlds/history-tools-common'; +import { FilterWorkerLoaderDependencies } from './filter.worker-loader.dependencies'; export default class FilterWorkerLoader extends DefaultWorkerLoader< FilterSharedData, diff --git a/src/filter/filter.worker.ts b/src/filter/filter.worker.ts index 309c69e..bbea307 100644 --- a/src/filter/filter.worker.ts +++ b/src/filter/filter.worker.ts @@ -1,21 +1,18 @@ +import { Worker } from '@alien-worlds/workers'; +import { AbiNotFoundError, ShipAbis } from '@alien-worlds/block-reader'; +import { FilterSharedData } from './filter.types'; import { - Worker, - ShipAbis, Abis, + BlockModel, + DeltaByName, Featured, + ProcessorTask, ProcessorTaskQueue, - Serializer, SignedBlock, TraceByName, - DeltaByName, - ProcessorTask, - parseToBigInt, - log, - AbiNotFoundError, isSetAbiAction, - BlockModel, -} from '@alien-worlds/history-tools-common'; -import { FilterSharedData } from './filter.types'; +} from '../common'; +import { Serializer, log, parseToBigInt } from '@alien-worlds/api-core'; export default class FilterWorker extends Worker { constructor( diff --git a/src/filter/index.ts b/src/filter/index.ts index 774ea38..e7019e0 100644 --- a/src/filter/index.ts +++ b/src/filter/index.ts @@ -1,7 +1,10 @@ +export * from './filter.config'; export * from './filter.command'; +export * from './filter.dependencies'; export * from './filter.consts'; export * from './filter.runner'; export * from './filter.types'; export * from './filter.worker'; export * from './filter.worker-loader'; +export * from './filter.worker-loader.dependencies'; export * from './start-filter'; diff --git a/src/filter/start-filter.ts b/src/filter/start-filter.ts index 8795333..0f61941 100644 --- a/src/filter/start-filter.ts +++ b/src/filter/start-filter.ts @@ -9,14 +9,11 @@ import { FilterBroadcastMessage } from '../broadcast/messages/filter-broadcast.m import { buildFilterConfig } from '../config'; import { filterCommand } from './filter.command'; import { filterWorkerLoaderPath } from './filter.consts'; -import { - BroadcastMessage, - ConfigVars, - FilterConfig, - FilterDependencies, - WorkerPool, - log, -} from '@alien-worlds/history-tools-common'; +import { log, ConfigVars } from '@alien-worlds/api-core'; +import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { WorkerPool } from '@alien-worlds/workers'; +import { FilterConfig } from './filter.config'; +import { FilterDependencies } from './filter.dependencies'; export const filter = async ( config: FilterConfig, diff --git a/src/index.ts b/src/index.ts index f9c60cb..5fed61b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,12 +1,12 @@ export * from './api'; export * from './bootstrap'; export * from './config'; +export * from './common'; export * from './filter'; export * from './broadcast'; export * from './processor'; export * from './reader'; - export * from '@alien-worlds/api-core'; export * from '@alien-worlds/block-reader'; export * from '@alien-worlds/broadcast'; diff --git a/src/processor/index.ts b/src/processor/index.ts index 3ba8c7d..36c4f4d 100644 --- a/src/processor/index.ts +++ b/src/processor/index.ts @@ -1,8 +1,11 @@ export * from './processors'; +export * from './processor.config'; export * from './processor.command'; +export * from './processor.dependencies'; export * from './processor.consts'; export * from './processor.enum'; export * from './processor.runner'; export * from './processor.types'; export * from './processor.worker-loader'; +export * from './processor.worker-loader.dependencies'; export * from './start-processor'; diff --git a/src/processor/processor.config.ts b/src/processor/processor.config.ts new file mode 100644 index 0000000..269e6f0 --- /dev/null +++ b/src/processor/processor.config.ts @@ -0,0 +1,23 @@ +import { BroadcastConfig } from '@alien-worlds/broadcast'; +import { WorkersConfig } from '@alien-worlds/workers'; +import { FeaturedConfig, ProcessorMatcher, ProcessorTaskQueueConfig } from '../common'; +import { UnknownObject } from '@alien-worlds/api-core'; + +export type ProcessorConfig = { + broadcast: BroadcastConfig; + workers: WorkersConfig; + featured: FeaturedConfig; + database: DatabaseConfig; + queue: ProcessorTaskQueueConfig; + processorLoaderPath?: string; + [key: string]: unknown; +}; + +export type ProcessorAddons = { + matchers?: { + traces?: ProcessorMatcher; + deltas?: ProcessorMatcher; + [key: string]: ProcessorMatcher; + }; + [key: string]: unknown; +}; diff --git a/src/processor/processor.dependencies.ts b/src/processor/processor.dependencies.ts new file mode 100644 index 0000000..c2007c9 --- /dev/null +++ b/src/processor/processor.dependencies.ts @@ -0,0 +1,34 @@ +import { Result } from '@alien-worlds/api-core'; +import { Dependencies } from '../common/dependencies'; +import { + ContractDeltaMatchCriteria, + ContractTraceMatchCriteria, + FeaturedMapper, +} from '../common/featured'; +import { ProcessorTaskQueue } from '../common/processor-task-queue'; +import { BroadcastClient } from '@alien-worlds/broadcast'; +import { ProcessorAddons, ProcessorConfig } from './processor.config'; +import { DatabaseConfigBuilder } from '../common'; + +/** + * An abstract class representing a Processor dependencies. + * @class ProcessorDependencies + */ +export abstract class ProcessorDependencies extends Dependencies { + public workerLoaderPath?: string; + public workerLoaderDependenciesPath: string; + public broadcastClient: BroadcastClient; + public featuredTraces: FeaturedMapper; + public featuredDeltas: FeaturedMapper; + public processorTaskQueue: ProcessorTaskQueue; + public processorsPath: string; + + public databaseConfigBuilder: DatabaseConfigBuilder; + + public abstract initialize( + config: ProcessorConfig, + featuredCriteriaPath: string, + processorsPath: string, + addons?: ProcessorAddons + ): Promise; +} diff --git a/src/processor/processor.runner.ts b/src/processor/processor.runner.ts index 353bdeb..5035ade 100644 --- a/src/processor/processor.runner.ts +++ b/src/processor/processor.runner.ts @@ -1,16 +1,15 @@ +import { log } from '@alien-worlds/api-core'; +import { WorkerPool, WorkerMessage } from '@alien-worlds/workers'; import { - ContractDeltaMatchCriteria, - ContractTraceMatchCriteria, FeaturedMapper, - ProcessorTask, - ProcessorTaskModel, + ContractTraceMatchCriteria, + ContractDeltaMatchCriteria, ProcessorTaskQueue, + ProcessorTask, ProcessorTaskType, UnknownProcessorTypeError, - WorkerMessage, - WorkerPool, - log, -} from '@alien-worlds/history-tools-common'; + ProcessorTaskModel, +} from '../common'; export class ProcessorRunner { private interval: NodeJS.Timeout; diff --git a/src/processor/processor.types.ts b/src/processor/processor.types.ts index b20cc3f..9d89829 100644 --- a/src/processor/processor.types.ts +++ b/src/processor/processor.types.ts @@ -1,4 +1,4 @@ -import { ProcessorConfig } from '@alien-worlds/history-tools-common'; +import { ProcessorConfig } from './processor.config'; export type ProcessorCommandOptions = { threads: number; diff --git a/src/processor/processor.worker-loader.dependencies.ts b/src/processor/processor.worker-loader.dependencies.ts new file mode 100644 index 0000000..8b77f90 --- /dev/null +++ b/src/processor/processor.worker-loader.dependencies.ts @@ -0,0 +1,18 @@ +import { Serializer } from '@alien-worlds/api-core'; +import { WorkerLoaderDependencies } from '@alien-worlds/workers'; +import { ProcessorConfig } from './processor.config'; + +/** + * An abstract class representing a ProcessorWorkerLoader dependencies. + * @class ProcessorWorkerLoaderDependencies + */ +export abstract class ProcessorWorkerLoaderDependencies extends WorkerLoaderDependencies { + public dataSource: unknown; + public serializer: Serializer; + public processorsPath: string; + + public abstract initialize( + config: ProcessorConfig, + processorsPath: string + ): Promise; +} diff --git a/src/processor/processor.worker-loader.ts b/src/processor/processor.worker-loader.ts index c3572f8..a04190d 100644 --- a/src/processor/processor.worker-loader.ts +++ b/src/processor/processor.worker-loader.ts @@ -1,11 +1,7 @@ -import { - Worker, - Container, - DefaultWorkerLoader, - ProcessorWorkerLoaderDependencies, - WorkerContainer, -} from '@alien-worlds/history-tools-common'; +import { Worker, DefaultWorkerLoader, WorkerContainer } from '@alien-worlds/workers'; import { ProcessorSharedData } from './processor.types'; +import { Container } from '@alien-worlds/api-core'; +import { ProcessorWorkerLoaderDependencies } from './processor.worker-loader.dependencies'; export default class ProcessorWorkerLoader extends DefaultWorkerLoader< ProcessorSharedData, diff --git a/src/processor/processors/action-trace.processor.ts b/src/processor/processors/action-trace.processor.ts index d92e974..f49374c 100644 --- a/src/processor/processors/action-trace.processor.ts +++ b/src/processor/processors/action-trace.processor.ts @@ -1,14 +1,9 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { Processor } from './processor'; import { ActionTraceProcessorInput, ProcessorSharedData } from '../processor.types'; +import { Container, Serializer, parseToBigInt } from '@alien-worlds/api-core'; +import { ProcessorTaskModel, ActionProcessorContentModel } from '../../common'; import { deserialize } from 'v8'; -import { - ActionProcessorContentModel, - Container, - ProcessorTaskModel, - Serializer, - parseToBigInt, -} from '@alien-worlds/history-tools-common'; export class ActionTraceProcessor< DataType = unknown, diff --git a/src/processor/processors/delta.processor.ts b/src/processor/processors/delta.processor.ts index 67f4efe..8f1a046 100644 --- a/src/processor/processors/delta.processor.ts +++ b/src/processor/processors/delta.processor.ts @@ -1,13 +1,8 @@ import { Processor } from './processor'; import { DeltaProcessorInput, ProcessorSharedData } from '../processor.types'; import { deserialize } from 'v8'; -import { - Container, - DeltaProcessorContentModel, - ProcessorTaskModel, - Serializer, - parseToBigInt, -} from '@alien-worlds/history-tools-common'; +import { Container, Serializer, parseToBigInt } from '@alien-worlds/api-core'; +import { ProcessorTaskModel, DeltaProcessorContentModel } from '../../common'; export class DeltaProcessor< DataType, diff --git a/src/processor/processors/processor.ts b/src/processor/processors/processor.ts index 370a9b3..cca476c 100644 --- a/src/processor/processors/processor.ts +++ b/src/processor/processors/processor.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ +import { Worker } from '@alien-worlds/workers'; +import { ProcessorTaskModel } from '../../common'; import { ProcessorSharedData } from '../processor.types'; -import { Worker, ProcessorTaskModel } from '@alien-worlds/history-tools-common'; export class Processor< SharedDataType = ProcessorSharedData diff --git a/src/processor/start-processor.ts b/src/processor/start-processor.ts index 0951e4b..7d1e44b 100644 --- a/src/processor/start-processor.ts +++ b/src/processor/start-processor.ts @@ -1,13 +1,3 @@ -import { - BroadcastMessage, - ConfigVars, - FeaturedContractDataCriteria, - ProcessorAddons, - ProcessorConfig, - ProcessorDependencies, - WorkerPool, - log, -} from '@alien-worlds/history-tools-common'; import { processorWorkerLoaderPath } from './processor.consts'; import { ProcessorRunner } from './processor.runner'; import { @@ -18,6 +8,11 @@ import { import { processorCommand } from './processor.command'; import { buildProcessorConfig } from '../config'; import { ProcessorCommandOptions } from './processor.types'; +import { log, ConfigVars } from '@alien-worlds/api-core'; +import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { WorkerPool } from '@alien-worlds/workers'; +import { ProcessorConfig, ProcessorAddons } from './processor.config'; +import { ProcessorDependencies } from './processor.dependencies'; export const process = async ( config: ProcessorConfig, diff --git a/src/reader/index.ts b/src/reader/index.ts index 0c8f852..cc4e5fd 100644 --- a/src/reader/index.ts +++ b/src/reader/index.ts @@ -1,7 +1,10 @@ export * from './reader'; +export * from './reader.config'; export * from './reader.command'; export * from './reader.consts'; +export * from './reader.dependencies'; export * from './reader.types'; export * from './reader.worker'; export * from './reader.worker-loader'; +export * from './reader.worker-loader.dependencies'; export * from './start-reader'; diff --git a/src/reader/reader.config.ts b/src/reader/reader.config.ts new file mode 100644 index 0000000..c981220 --- /dev/null +++ b/src/reader/reader.config.ts @@ -0,0 +1,18 @@ +import { BlockReaderConfig } from '@alien-worlds/block-reader'; +import { BroadcastConfig } from '@alien-worlds/broadcast'; +import { WorkersConfig } from '@alien-worlds/workers'; +import { BlockRangeScanConfig, UnprocessedBlockQueueConfig } from '../common'; +import { UnknownObject } from '@alien-worlds/api-core'; + +export type ReaderConfig = { + broadcast?: BroadcastConfig; + blockReader?: BlockReaderConfig; + database?: DatabaseConfig; + scanner?: BlockRangeScanConfig; + workers?: WorkersConfig; + unprocessedBlockQueue: UnprocessedBlockQueueConfig; + mode?: string; + maxBlockNumber?: number; + startBlock?: bigint; + endBlock?: bigint; +}; diff --git a/src/reader/reader.dependencies.ts b/src/reader/reader.dependencies.ts new file mode 100644 index 0000000..e3d5413 --- /dev/null +++ b/src/reader/reader.dependencies.ts @@ -0,0 +1,20 @@ +import { BroadcastClient } from '@alien-worlds/broadcast'; +import { Result } from '@alien-worlds/api-core'; +import { Dependencies } from '../common/dependencies'; +import { BlockRangeScanner, DatabaseConfigBuilder } from '../common'; +import { ReaderConfig } from './reader.config'; + +/** + * An abstract class representing a reader dependencies. + * @class ReaderDependencies + */ +export abstract class ReaderDependencies extends Dependencies { + public broadcastClient: BroadcastClient; + public scanner: BlockRangeScanner; + public workerLoaderPath?: string; + public workerLoaderDependenciesPath: string; + + public databaseConfigBuilder: DatabaseConfigBuilder; + + public abstract initialize(config: ReaderConfig): Promise; +} diff --git a/src/reader/reader.ts b/src/reader/reader.ts index 1a6c8fe..6a8b04f 100644 --- a/src/reader/reader.ts +++ b/src/reader/reader.ts @@ -1,13 +1,9 @@ import { ReadCompleteData, ReadTaskData } from './reader.types'; import { FilterBroadcastMessage } from '../broadcast/messages'; -import { - BlockRangeScanner, - BroadcastClient, - Mode, - WorkerMessage, - WorkerPool, - log, -} from '@alien-worlds/history-tools-common'; +import { log } from '@alien-worlds/api-core'; +import { BroadcastClient } from '@alien-worlds/broadcast'; +import { WorkerPool, WorkerMessage } from '@alien-worlds/workers'; +import { BlockRangeScanner, Mode } from '../common'; export class Reader { private loop = false; diff --git a/src/reader/reader.worker-loader.dependencies.ts b/src/reader/reader.worker-loader.dependencies.ts new file mode 100644 index 0000000..1e8894d --- /dev/null +++ b/src/reader/reader.worker-loader.dependencies.ts @@ -0,0 +1,19 @@ +import { WorkerLoaderDependencies } from '@alien-worlds/workers'; +import { BlockReader } from '@alien-worlds/block-reader'; +import { UnprocessedBlockQueue } from '../common/unprocessed-block-queue'; +import { BlockRangeScanner, BlockState } from '../common'; +import { ReaderConfig } from './reader.config'; + +/** + * An abstract class representing a ReaderWorkerLoader dependencies. + * @class ReaderWorkerLoaderDependencies + */ +export abstract class ReaderWorkerLoaderDependencies extends WorkerLoaderDependencies { + public blockReader: BlockReader; + public blockState: BlockState; + public blockQueue: UnprocessedBlockQueue; + public scanner: BlockRangeScanner; + public config: ReaderConfig; + + public abstract initialize(config: ReaderConfig): Promise; +} diff --git a/src/reader/reader.worker-loader.ts b/src/reader/reader.worker-loader.ts index 10309fb..27c37da 100644 --- a/src/reader/reader.worker-loader.ts +++ b/src/reader/reader.worker-loader.ts @@ -1,10 +1,7 @@ +import { Worker, DefaultWorkerLoader } from '@alien-worlds/workers'; +import { ReaderWorkerLoaderDependencies } from './reader.worker-loader.dependencies'; +import { log } from '@alien-worlds/api-core'; import ReaderWorker, { ReaderSharedData } from './reader.worker'; -import { - Worker, - DefaultWorkerLoader, - ReaderWorkerLoaderDependencies, - log, -} from '@alien-worlds/history-tools-common'; export default class ReaderWorkerLoader extends DefaultWorkerLoader< ReaderSharedData, diff --git a/src/reader/reader.worker.ts b/src/reader/reader.worker.ts index 29589eb..111b428 100644 --- a/src/reader/reader.worker.ts +++ b/src/reader/reader.worker.ts @@ -1,14 +1,8 @@ -import { - Worker, - BlockRangeScanner, - BlockState, - Mode, - ReaderConfig, - UnprocessedBlockQueue, - BlockReader, - log, - parseToBigInt, -} from '@alien-worlds/history-tools-common'; +import { Worker } from '@alien-worlds/workers'; +import { ReaderConfig } from './reader.config'; +import { log, parseToBigInt } from '@alien-worlds/api-core'; +import { BlockReader } from '@alien-worlds/block-reader'; +import { UnprocessedBlockQueue, BlockState, BlockRangeScanner, Mode } from '../common'; export type ReaderSharedData = { config: ReaderConfig; diff --git a/src/reader/start-reader.ts b/src/reader/start-reader.ts index 833dfc8..fa0bcc9 100644 --- a/src/reader/start-reader.ts +++ b/src/reader/start-reader.ts @@ -9,16 +9,13 @@ import { ReaderBroadcastMessage } from '../broadcast/messages/reader-broadcast.m import { Reader } from './reader'; import { readerCommand } from './reader.command'; import { buildReaderConfig } from '../config'; -import { - BroadcastMessage, - ConfigVars, - Mode, - ReaderConfig, - ReaderDependencies, - WorkerPool, - log, -} from '@alien-worlds/history-tools-common'; import { readerWorkerLoaderPath } from './reader.consts'; +import { log, ConfigVars } from '@alien-worlds/api-core'; +import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { WorkerPool } from '@alien-worlds/workers'; +import { Mode } from '../common'; +import { ReaderConfig } from './reader.config'; +import { ReaderDependencies } from './reader.dependencies'; /** * diff --git a/tutorials/writing-history-tools.md b/tutorials/writing-history-tools.md index 78358d8..daded48 100644 --- a/tutorials/writing-history-tools.md +++ b/tutorials/writing-history-tools.md @@ -196,7 +196,6 @@ export * from './dao-worlds.delta-processor'; // ./processors/dao-worlds.trace-processor.ts import { ActionTraceProcessor } from '@alien-worlds/api-history-tools'; -import { ProcessorTaskModel, log } from '@alien-worlds/history-tools-common'; export class DaoWorldsTraceProcessor extends ActionTraceProcessor { public async run(model: ProcessorTaskModel): Promise { @@ -211,7 +210,6 @@ export class DaoWorldsTraceProcessor extends ActionTraceProcessor { // ./processors/dao-worlds.delta-processor.ts import { DeltaProcessor } from '@alien-worlds/api-history-tools'; -import { ProcessorTaskModel, log } from '@alien-worlds/history-tools-common'; export class DaoWorldsDeltaProcessor extends DeltaProcessor { public async run(model: ProcessorTaskModel): Promise { diff --git a/yarn.lock b/yarn.lock index 31eec64..624d214 100644 --- a/yarn.lock +++ b/yarn.lock @@ -28,17 +28,6 @@ nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/history-tools-common@^0.0.36": - version "0.0.36" - resolved "https://npm.pkg.github.com/download/@alien-worlds/history-tools-common/0.0.36/e649d1444fe8cdda4a7bdc6f20e0a9c8d436438a#e649d1444fe8cdda4a7bdc6f20e0a9c8d436438a" - integrity sha512-BB34qD7uHAUMODlTjNfltABIjD0keOF+zCryPjzV8tHs8cXSPuOzWDEIrEmex3FCpZk70CAkpCoOcqux1tiFAQ== - dependencies: - "@alien-worlds/api-core" "^0.0.151" - "@alien-worlds/block-reader" "^0.0.14" - "@alien-worlds/broadcast" "^0.0.10" - "@alien-worlds/workers" "^0.0.6" - node-fetch "2" - "@alien-worlds/workers@^0.0.6": version "0.0.6" resolved "https://npm.pkg.github.com/download/@alien-worlds/workers/0.0.6/366bdf77db081b07bf34617180cc6848c1031620#366bdf77db081b07bf34617180cc6848c1031620" @@ -2868,9 +2857,9 @@ negotiator@0.6.3: integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== node-fetch@2: - version "2.6.11" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.11.tgz#cde7fc71deef3131ef80a738919f999e6edfff25" - integrity sha512-4I6pdBY1EthSqDmJkiNk3JIT8cswwR9nfeW/cPdUagJYEQG7R95WRH74wpz7ma8Gh/9dI9FP+OU+0E4FvtA55w== + version "2.6.12" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.12.tgz#02eb8e22074018e3d5a83016649d04df0e348fba" + integrity sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g== dependencies: whatwg-url "^5.0.0" From 63c5e1f4ae3d6876393fddb17af814e7c9728e93 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 30 Jun 2023 11:34:30 +0200 Subject: [PATCH 053/107] add common placeholders in readme --- README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/README.md b/README.md index 8fcc9e2..fb22cfa 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,13 @@ This package encapsulates the core mechanism, however, complete functionality re - [Reader](#reader) - [Filter](#filter) - [Processor](#processor) +- [Common components](#common-components) + - [Abis](#abis) + - [BlockRangeScanner](#blockrangescanner) + - [BlockState](#blockstate) + - [Featured](#featured) + - [ProcessorTaskQueue](#processortaskqueue) + - [UnprocessedBlockQueue](#unprocessedblockqueue) - [Additional Tools](#additional-tools) - [Config](#config) - [Tutorials](#tutorials) @@ -68,6 +75,26 @@ The Filter process retrieves blocks from the database, verifying their content. The Processor process retrieves tasks from the database, generating appropriate action or delta data based on their content. It coordinates the work of workers who perform the processing. There are two categories of processors: DeltaProcessor and ActionTraceProcessor. The processor creates separate collections in the database for each contract, stock, and delta e.g. `dao.worlds_actions` and `dao.worlds_deltas`. Any processing failures are stored in a separate collection for subsequent review and analysis. +## Common components + +### Abis +... +### BlockRangeScanner +... + +### BlockState +... + +### Featured +... + +### ProcessorTaskQueue +... +### UnprocessedBlockQueue + +... + + ## Additional Tools Besides the main processes, this package also contains tools, such as: From 9bccef373add6c87ec83f5d9f87a824268c20434 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 30 Jun 2023 11:34:46 +0200 Subject: [PATCH 054/107] 0.0.137 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5c21a74..f7416de 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.136", + "version": "0.0.137", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 8baf6c0cefea2999b46b532be9658aa1e94cf983 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 30 Jun 2023 11:35:52 +0200 Subject: [PATCH 055/107] update workers --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index f7416de..de651c4 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "@alien-worlds/api-core": "^0.0.151", "@alien-worlds/block-reader": "^0.0.14", "@alien-worlds/broadcast": "^0.0.10", - "@alien-worlds/workers": "^0.0.6", + "@alien-worlds/workers": "^0.0.7", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", diff --git a/yarn.lock b/yarn.lock index 624d214..feccb45 100644 --- a/yarn.lock +++ b/yarn.lock @@ -28,10 +28,10 @@ nanoid "^3.0.0" node-fetch "2" -"@alien-worlds/workers@^0.0.6": - version "0.0.6" - resolved "https://npm.pkg.github.com/download/@alien-worlds/workers/0.0.6/366bdf77db081b07bf34617180cc6848c1031620#366bdf77db081b07bf34617180cc6848c1031620" - integrity sha512-2lvT/D3NBLMfimbfGak2MaFTV9IEu+ByoS5VsrHHRnzKelL9DvBf+yXsPel2pFdTcxhZxe2RhITedV9mutD/0Q== +"@alien-worlds/workers@^0.0.7": + version "0.0.7" + resolved "https://npm.pkg.github.com/download/@alien-worlds/workers/0.0.7/561e152e2c9f3d3ea5777ab413aa50023549006a#561e152e2c9f3d3ea5777ab413aa50023549006a" + integrity sha512-gcAkRpezhsyVHdFbXz7FpXbPd/PAANPyNe2H4su5J/gGrKuYPfoH6AX7elgSHciq9UPVFRkJDu8yWCYi4ARiXA== dependencies: async "^3.2.4" ts-node "^10.9.1" From 42940846cc297b5be99ed52c4014f7fb096a4a84 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 30 Jun 2023 11:36:09 +0200 Subject: [PATCH 056/107] 0.0.138 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index de651c4..74e121d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.137", + "version": "0.0.138", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From ae7f722497e34c5eaf45c1a877ca136769cf9f81 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 30 Jun 2023 11:38:20 +0200 Subject: [PATCH 057/107] expose node fetch --- src/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/index.ts b/src/index.ts index 5fed61b..9926e2a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,5 @@ +import nodeFetch from 'node-fetch'; + export * from './api'; export * from './bootstrap'; export * from './config'; @@ -11,3 +13,5 @@ export * from '@alien-worlds/api-core'; export * from '@alien-worlds/block-reader'; export * from '@alien-worlds/broadcast'; export * from '@alien-worlds/workers'; + +export const fetch = nodeFetch; \ No newline at end of file From 928c1ab9e8f4402b023711d6616c476bd8187068 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 30 Jun 2023 11:38:35 +0200 Subject: [PATCH 058/107] 0.0.139 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 74e121d..dbf4f5f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.138", + "version": "0.0.139", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 5e08dfc125cad9603c6a6f1aabf9526b0ea4f26c Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 30 Jun 2023 17:55:16 +0200 Subject: [PATCH 059/107] renamed featured, new config defaults, pattern defaults --- src/bootstrap/bootstrap.dependencies.ts | 6 +- src/bootstrap/start-bootstrap.ts | 14 +- src/common/featured/featured-contracts.ts | 81 +++++ src/common/featured/featured.errors.ts | 6 + src/common/featured/featured.mapper.ts | 277 --------------- src/common/featured/featured.ts | 334 ++++++++++++++---- src/common/featured/index.ts | 4 +- src/config/index.ts | 46 +-- .../filter.worker-loader.dependencies.ts | 4 +- src/filter/filter.worker.ts | 22 +- src/processor/processor.dependencies.ts | 6 +- src/processor/processor.runner.ts | 7 +- 12 files changed, 420 insertions(+), 387 deletions(-) create mode 100644 src/common/featured/featured-contracts.ts delete mode 100644 src/common/featured/featured.mapper.ts diff --git a/src/bootstrap/bootstrap.dependencies.ts b/src/bootstrap/bootstrap.dependencies.ts index 211a904..c289c49 100644 --- a/src/bootstrap/bootstrap.dependencies.ts +++ b/src/bootstrap/bootstrap.dependencies.ts @@ -1,6 +1,6 @@ import { BlockchainService, Result } from '@alien-worlds/api-core'; import { Dependencies } from '../common/dependencies'; -import { Featured, FeaturedContractDataCriteria } from '../common/featured'; +import { FeaturedContracts, FeaturedContractDataCriteria } from '../common/featured'; import { BroadcastClient } from '@alien-worlds/broadcast'; import { Abis, BlockRangeScanner, BlockState, DatabaseConfigBuilder } from '../common'; import { BootstrapConfig } from './bootstrap.config'; @@ -29,9 +29,9 @@ export abstract class BootstrapDependencies extends Dependencies { public scanner: BlockRangeScanner; /** * The featured contract service. - * @type {Featured} + * @type {FeaturedContracts} */ - public featured: Featured; + public featuredContracts: FeaturedContracts; /** * The block state for maintaining blockchain state. * @type {BlockState} diff --git a/src/bootstrap/start-bootstrap.ts b/src/bootstrap/start-bootstrap.ts index 0d58751..938bb72 100644 --- a/src/bootstrap/start-bootstrap.ts +++ b/src/bootstrap/start-bootstrap.ts @@ -48,7 +48,7 @@ export const bootstrap = async ( throw new MissingCriteriaError(featuredCriteriaPath); } - const featuredContracts = FeaturedUtils.readFeaturedContracts(featuredCriteria); + const contractNames = FeaturedUtils.readFeaturedContracts(featuredCriteria); const initResult = await dependencies.initialize(config, featuredCriteria); @@ -56,13 +56,19 @@ export const bootstrap = async ( throw initResult.failure.error; } - const { abis, broadcastClient, blockState, blockchain, featured, scanner } = - dependencies; + const { + abis, + broadcastClient, + blockState, + blockchain, + featuredContracts, + scanner, + } = dependencies; let blockRange: ReaderBroadcastMessageData; // fetch latest abis to make sure that the blockchain data will be correctly deserialized log(` * Fetch featured contracts details ... [starting]`); - await featured.readContracts(featuredContracts); + await featuredContracts.readContracts(contractNames); log(` * Fetch featured contracts details ... [done]`); // fetch latest abis to make sure that the blockchain data will be correctly deserialized diff --git a/src/common/featured/featured-contracts.ts b/src/common/featured/featured-contracts.ts new file mode 100644 index 0000000..4815fea --- /dev/null +++ b/src/common/featured/featured-contracts.ts @@ -0,0 +1,81 @@ +import { + FindParams, + Repository, + Result, + SmartContractService, + UnknownObject, + Where, +} from '@alien-worlds/api-core'; +import { FeaturedContract } from './featured-contract'; +import { FeaturedUtils } from './featured.utils'; + +export class FeaturedContracts { + protected cache: Map = new Map(); + protected featuredContracts: string[]; + + constructor( + private repository: Repository, + private smartContractService: SmartContractService, + criteria: UnknownObject + ) { + this.featuredContracts = FeaturedUtils.readFeaturedContracts(criteria); + } + + /** + * Reads multiple contracts and returns the results as an array of FeaturedContract objects. + * + * @abstract + * @param {string[]} contracts - An array of contract addresses or identifiers. + * @returns {Promise>} A Promise that resolves to an array of FeaturedContract objects. + */ + public async readContracts( + data: string[] | UnknownObject + ): Promise> { + const list: FeaturedContract[] = []; + + const contracts = Array.isArray(data) + ? data + : FeaturedUtils.readFeaturedContracts(data); + + for (const contract of contracts) { + if (this.cache.has(contract)) { + list.push(this.cache.get(contract)); + } else { + const { content: contracts, failure } = await this.repository.find( + FindParams.create({ where: new Where().valueOf('account').isEq(contract) }) + ); + + if (failure) { + return Result.withFailure(failure); + } + + if (contracts.length > 0) { + const featuredContract = contracts[0]; + this.cache.set(featuredContract.account, featuredContract); + list.push(featuredContract); + } else { + const fetchResult = await this.smartContractService.getStats(contract); + + if (fetchResult.isFailure) { + return Result.withFailure(fetchResult.failure); + } + if (fetchResult.content) { + const featuredContract = FeaturedContract.create( + fetchResult.content.account_name, + fetchResult.content.first_block_num + ); + this.cache.set(featuredContract.account, featuredContract); + this.repository.add([featuredContract]); + list.push(featuredContract); + } + } + } + } + + return Result.withContent(list); + } + + public isFeatured(contract: string): boolean { + return this.featuredContracts.includes(contract); + } +} diff --git a/src/common/featured/featured.errors.ts b/src/common/featured/featured.errors.ts index fe06c82..965a593 100644 --- a/src/common/featured/featured.errors.ts +++ b/src/common/featured/featured.errors.ts @@ -35,3 +35,9 @@ export class MissingCriteriaError extends Error { super(`No criteria found at: ${path}`); } } + +export class DefaultsMismatchError extends Error { + constructor() { + super(`Defaults keys do not match pattern keys.`); + } +} diff --git a/src/common/featured/featured.mapper.ts b/src/common/featured/featured.mapper.ts deleted file mode 100644 index 3b8fe7f..0000000 --- a/src/common/featured/featured.mapper.ts +++ /dev/null @@ -1,277 +0,0 @@ -import { - MatcherNotFoundError, - PatternMatchError, - PatternMismatchError, - UndefinedPatternError, -} from './featured.errors'; -import { - MatchCriteria, - ProcessorMatchCriteria, - ProcessorMatcher, -} from './featured.types'; - -/** - * A mapper class for processing and matching contracts based on given criteria. - * This class can be extended by other processors that require specific match criteria. - */ -export class FeaturedMapper { - /** - * Map of processors matched by a matching function. - */ - protected processorByMatchers: ProcessorMatcher = new Map(); - /** - * Array of match criteria. - */ - protected matchCriteria: ProcessorMatchCriteria[] = []; - /** - * Set of contracts. - */ - protected contracts: Set = new Set(); - - /** - * Creates a new instance of the contract processor mapper. - * @param {ProcessorMatchCriteria[]} criteria - An array of match criteria for the processor. - * @param {MatchCriteriaType} pattern - The criteria pattern. - * @param {ProcessorMatcher} matchers - Optional map of matchers. - */ - constructor( - criteria: ProcessorMatchCriteria[], - protected pattern?: MatchCriteriaType, - matchers?: ProcessorMatcher - ) { - criteria.forEach(current => { - const { processor, matcher, ...rest } = current; - const { contract, code } = rest as unknown as MatchCriteria; - - if (Array.isArray(contract)) { - contract.forEach(contract => this.contracts.add(contract)); - } else if (typeof contract === 'string') { - this.contracts.add(contract); - } - - if (Array.isArray(code)) { - code.forEach(contract => this.contracts.add(contract)); - } else if (typeof code === 'string') { - this.contracts.add(code); - } - - if (matcher && !matchers?.has(matcher)) { - throw new MatcherNotFoundError(matcher); - } - - if (matcher && matchers.has(matcher)) { - this.processorByMatchers.set(processor, matchers.get(matcher)); - } else { - this.validateCriteria(rest as MatchCriteriaType); - - if (this.matchCriteria.indexOf(current) === -1) { - this.matchCriteria.push(current); - } - } - }); - } - - /** - * Validates the given match criteria. - * @param criteria - The criteria to validate. - */ - protected validateCriteria(criteria: MatchCriteriaType): void { - const keys = Object.keys(criteria); - - for (const key of keys) { - const values = criteria[key]; - for (const value of values) { - if (/^(\*|[A-Za-z0-9_.]*)$/g.test(value) === false) { - throw new PatternMatchError(value, '^(*|[A-Za-z0-9_.]*)$'); - } - } - } - } - - /** - * Determines if a candidate match criteria meets a reference match criteria. - * @param ref - The reference match criteria. - * @param candidate - The candidate match criteria. - * @returns True if a match is found, false otherwise. - */ - protected isMatch( - ref: ProcessorMatchCriteria, - candidate: MatchCriteriaType - ): boolean { - let matchFound = false; - const keys = Object.keys(candidate); - - for (const key of keys) { - const candidateValues = candidate[key]; - const refValues = ref[key]; - if (Array.isArray(refValues)) { - const values: string[] = Array.isArray(candidateValues) - ? candidateValues - : [candidateValues]; - const contains = values.some(value => refValues.includes(value)); - - if (refValues.includes('*') || contains) { - matchFound = true; - } else { - return false; - } - } - } - - return matchFound; - } - - /** - * Finds a processor match criteria for a given match criteria. - * @param criteria - The criteria to find a match for. - * @returns The matching processor match criteria if found, null otherwise. - */ - protected async findProcessorMatchCriteria( - criteria: MatchCriteriaType - ): Promise> { - const { processorByMatchers } = this; - const entries = Array.from(processorByMatchers.entries()); - - for (const entry of entries) { - const [processor, matcher] = entry; - if (await matcher(criteria)) { - const keys = Object.keys(criteria); - const matchCriteria = {} as MatchCriteriaType; - - for (const key of keys) { - matchCriteria[key] = ['*']; - } - - return { - ...matchCriteria, - processor, - }; - } - } - - return null; - } - - /** - * Checks if the criteria already exist in the array - * - * @param {ProcessorMatchCriteria} criteria - * @param {ProcessorMatchCriteria[]} array - * @returns - */ - protected criteriaExistsInArray( - criteria: ProcessorMatchCriteria, - array: ProcessorMatchCriteria[] - ): boolean { - return array.some(item => - Object.keys(item).every(key => item[key] === criteria[key]) - ); - } - - /** - * Determines if the given match criteria exists in the processor. - * @param criteria - The criteria to check. - * @returns True if the criteria exists, false otherwise. - */ - public async has(criteria: MatchCriteriaType): Promise { - const { matchCriteria, processorByMatchers } = this; - - for (const item of matchCriteria) { - if (this.isMatch(item, criteria)) { - return true; - } - } - - if (processorByMatchers.size > 0) { - const featured = await this.findProcessorMatchCriteria(criteria); - if (featured) { - if (this.criteriaExistsInArray(featured, matchCriteria) === false) { - matchCriteria.push(featured); - } - return true; - } - } - - return false; - } - - /** - * Gets all match criteria in the processor that match the given criteria. - * @param criteria - The criteria to match. - * @returns An array of matching criteria. - */ - public async get(criteria: MatchCriteriaType): Promise { - const { matchCriteria, processorByMatchers } = this; - const result: MatchCriteriaType[] = []; - - for (const item of matchCriteria) { - if (this.isMatch(item, criteria)) { - result.push(item); - } - } - - if (result.length === 0 && processorByMatchers.size > 0) { - const featured = await this.findProcessorMatchCriteria(criteria); - if (featured) { - if (this.criteriaExistsInArray(featured, matchCriteria) === false) { - matchCriteria.push(featured); - } - result.push(featured); - } - } - - return result; - } - - /** - * Gets the processor for the given label and criteria. - * @param label - The label to find a processor for. - * @param pattern - The match criteria pattern. - * @returns The processor if found, empty string otherwise. - */ - public async getProcessor(label: string, pattern?: MatchCriteriaType): Promise { - const { matchCriteria } = this; - const p = pattern || this.pattern; - - if (!p) { - throw new UndefinedPatternError(); - } - - const keys = Object.keys(p); - const parts = label.split(':').map(part => part.split(',')); - - if (parts.length !== keys.length) { - throw new PatternMismatchError(); - } - - const candidate = parts.reduce((result, part, i) => { - result[keys[i]] = part; - return result; - }, p); - - for (const criteriaRef of matchCriteria) { - if (this.isMatch(criteriaRef, candidate)) { - return criteriaRef.processor; - } - } - - const featured = await this.findProcessorMatchCriteria(candidate); - - if (featured) { - if (matchCriteria.indexOf(featured) === -1) { - matchCriteria.push(featured); - } - return featured.processor; - } - - return ''; - } - - /** - * Lists all contracts in the processor. - * @returns An array of contracts. - */ - public listContracts(): string[] { - return Array.from(this.contracts); - } -} diff --git a/src/common/featured/featured.ts b/src/common/featured/featured.ts index c8d9680..0d79e3b 100644 --- a/src/common/featured/featured.ts +++ b/src/common/featured/featured.ts @@ -1,81 +1,295 @@ import { - FindParams, - Repository, - Result, - SmartContractService, - UnknownObject, - Where, -} from '@alien-worlds/api-core'; -import { FeaturedContract } from './featured-contract'; -import { FeaturedUtils } from './featured.utils'; - -export class Featured { - protected cache: Map = new Map(); - protected featuredContracts: string[]; + DefaultsMismatchError, + MatcherNotFoundError, + PatternMatchError, + PatternMismatchError, +} from './featured.errors'; +import { + MatchCriteria, + ProcessorMatchCriteria, + ProcessorMatcher, +} from './featured.types'; +/** + * A mapper class for processing and matching contracts based on given criteria. + * This class can be extended by other processors that require specific match criteria. + */ +export class Featured { + /** + * Map of processors matched by a matching function. + */ + protected processorByMatchers: ProcessorMatcher = new Map(); + /** + * Array of match criteria. + */ + protected matchCriteria: ProcessorMatchCriteria[] = []; + /** + * Set of contracts. + */ + protected contracts: Set = new Set(); + + /** + * Creates a new instance of the contract processor mapper. + * @param {ProcessorMatchCriteria[]} criteria - An array of match criteria for the processor. + * @param {MatchCriteriaType} pattern - The criteria pattern. + * @param {MatchCriteriaType} defaults - Default criteria key:value pairs. + * @param {ProcessorMatcher} matchers - Optional map of matchers. + */ constructor( - private repository: Repository, - private smartContractService: SmartContractService, - featuredJson: UnknownObject + criteria: ProcessorMatchCriteria[], + protected pattern: MatchCriteriaType, + protected defaults?: MatchCriteriaType, + matchers?: ProcessorMatcher ) { - this.featuredContracts = FeaturedUtils.readFeaturedContracts(featuredJson); + if (defaults) { + const patternKeys = Object.keys(pattern); + const hasAllDefaults = Object.keys(defaults).every(elem => + patternKeys.includes(elem) + ); + + if (hasAllDefaults === false) { + throw new DefaultsMismatchError(); + } + } + + criteria.forEach(current => { + const { processor, matcher, ...rest } = current; + const { contract, code } = rest as unknown as MatchCriteria; + + if (defaults) { + const defaultsKeys = Object.keys(defaults); + + defaultsKeys.forEach(defaultKey => { + if (!current[defaultKey]) { + current[defaultKey] = defaults[defaultKey]; + } + }); + } + + if (Array.isArray(contract)) { + contract.forEach(contract => this.contracts.add(contract)); + } else if (typeof contract === 'string') { + this.contracts.add(contract); + } + + if (Array.isArray(code)) { + code.forEach(contract => this.contracts.add(contract)); + } else if (typeof code === 'string') { + this.contracts.add(code); + } + + if (matcher && !matchers?.has(matcher)) { + throw new MatcherNotFoundError(matcher); + } + + if (matcher && matchers.has(matcher)) { + this.processorByMatchers.set(processor, matchers.get(matcher)); + } else { + this.validateCriteria(rest as MatchCriteriaType); + + if (this.matchCriteria.indexOf(current) === -1) { + this.matchCriteria.push(current); + } + } + }); } /** - * Reads multiple contracts and returns the results as an array of FeaturedContract objects. - * - * @abstract - * @param {string[]} contracts - An array of contract addresses or identifiers. - * @returns {Promise>} A Promise that resolves to an array of FeaturedContract objects. + * Validates the given match criteria. + * @param criteria - The criteria to validate. */ - public async readContracts( - data: string[] | UnknownObject - ): Promise> { - const list: FeaturedContract[] = []; - - const contracts = Array.isArray(data) - ? data - : FeaturedUtils.readFeaturedContracts(data); - - for (const contract of contracts) { - if (this.cache.has(contract)) { - list.push(this.cache.get(contract)); - } else { - const { content: contracts, failure } = await this.repository.find( - FindParams.create({ where: new Where().valueOf('account').isEq(contract) }) - ); + protected validateCriteria(criteria: MatchCriteriaType): void { + const keys = Object.keys(criteria); - if (failure) { - return Result.withFailure(failure); + for (const key of keys) { + const values = criteria[key]; + for (const value of values) { + if (/^(\*|[A-Za-z0-9_.]*)$/g.test(value) === false) { + throw new PatternMatchError(value, '^(*|[A-Za-z0-9_.]*)$'); } + } + } + } + + /** + * Determines if a candidate match criteria meets a reference match criteria. + * @param ref - The reference match criteria. + * @param candidate - The candidate match criteria. + * @returns True if a match is found, false otherwise. + */ + protected isMatch( + ref: ProcessorMatchCriteria, + candidate: MatchCriteriaType + ): boolean { + let matchFound = false; + const keys = Object.keys(candidate); - if (contracts.length > 0) { - const featuredContract = contracts[0]; - this.cache.set(featuredContract.account, featuredContract); - list.push(featuredContract); + for (const key of keys) { + const candidateValues = candidate[key]; + const refValues = ref[key]; + if (Array.isArray(refValues)) { + const values: string[] = Array.isArray(candidateValues) + ? candidateValues + : [candidateValues]; + const contains = values.some(value => refValues.includes(value)); + + if (refValues.includes('*') || contains) { + matchFound = true; } else { - const fetchResult = await this.smartContractService.getStats(contract); + return false; + } + } + } - if (fetchResult.isFailure) { - return Result.withFailure(fetchResult.failure); - } - if (fetchResult.content) { - const featuredContract = FeaturedContract.create( - fetchResult.content.account_name, - fetchResult.content.first_block_num - ); - this.cache.set(featuredContract.account, featuredContract); - this.repository.add([featuredContract]); - list.push(featuredContract); - } + return matchFound; + } + + /** + * Finds a processor match criteria for a given match criteria. + * @param criteria - The criteria to find a match for. + * @returns The matching processor match criteria if found, null otherwise. + */ + protected async findProcessorMatchCriteria( + criteria: MatchCriteriaType + ): Promise> { + const { processorByMatchers } = this; + const entries = Array.from(processorByMatchers.entries()); + + for (const entry of entries) { + const [processor, matcher] = entry; + if (await matcher(criteria)) { + const keys = Object.keys(criteria); + const matchCriteria = {} as MatchCriteriaType; + + for (const key of keys) { + matchCriteria[key] = ['*']; } + + return { + ...matchCriteria, + processor, + }; + } + } + + return null; + } + + /** + * Checks if the criteria already exist in the array + * + * @param {ProcessorMatchCriteria} criteria + * @param {ProcessorMatchCriteria[]} array + * @returns + */ + protected criteriaExistsInArray( + criteria: ProcessorMatchCriteria, + array: ProcessorMatchCriteria[] + ): boolean { + return array.some(item => + Object.keys(item).every(key => item[key] === criteria[key]) + ); + } + + /** + * Determines if the given match criteria exists in the processor. + * @param criteria - The criteria to check. + * @returns True if the criteria exists, false otherwise. + */ + public async hasCriteria(criteria: MatchCriteriaType): Promise { + const { matchCriteria, processorByMatchers } = this; + + for (const item of matchCriteria) { + if (this.isMatch(item, criteria)) { + return true; + } + } + + if (processorByMatchers.size > 0) { + const featured = await this.findProcessorMatchCriteria(criteria); + if (featured) { + if (this.criteriaExistsInArray(featured, matchCriteria) === false) { + matchCriteria.push(featured); + } + return true; + } + } + + return false; + } + + /** + * Gets all match criteria that match the given criteria. + * @param criteria - The criteria to match. + * @returns An array of matching criteria. + */ + public async getCriteria(criteria: MatchCriteriaType): Promise { + const { matchCriteria, processorByMatchers } = this; + const result: MatchCriteriaType[] = []; + + for (const item of matchCriteria) { + if (this.isMatch(item, criteria)) { + result.push(item); + } + } + + if (result.length === 0 && processorByMatchers.size > 0) { + const featured = await this.findProcessorMatchCriteria(criteria); + if (featured) { + if (this.criteriaExistsInArray(featured, matchCriteria) === false) { + matchCriteria.push(featured); + } + result.push(featured); + } + } + + return result; + } + + /** + * Gets the processor for the given label and criteria. + * @param label - The label to find a processor for. + * @param pattern - The match criteria pattern. + * @returns The processor if found, empty string otherwise. + */ + public async getProcessor(label: string): Promise { + const { matchCriteria, pattern } = this; + + const keys = Object.keys(pattern); + const parts = label.split(':').map(part => part.split(',')); + + if (parts.length !== keys.length) { + throw new PatternMismatchError(); + } + + const candidate = parts.reduce((result, part, i) => { + result[keys[i]] = part; + return result; + }, pattern); + + for (const criteriaRef of matchCriteria) { + if (this.isMatch(criteriaRef, candidate)) { + return criteriaRef.processor; } } - return Result.withContent(list); + const featured = await this.findProcessorMatchCriteria(candidate); + + if (featured) { + if (matchCriteria.indexOf(featured) === -1) { + matchCriteria.push(featured); + } + return featured.processor; + } + + return ''; } - public isFeatured(contract: string): boolean { - return this.featuredContracts.includes(contract); + /** + * Lists all contracts in the processor. + * @returns An array of contracts. + */ + public getContracts(): string[] { + return Array.from(this.contracts); } } diff --git a/src/common/featured/index.ts b/src/common/featured/index.ts index 067d4e7..ba6610c 100644 --- a/src/common/featured/index.ts +++ b/src/common/featured/index.ts @@ -1,7 +1,7 @@ export * from './featured-contract'; -export * from './featured'; +export * from './featured-contracts'; export * from './featured.config'; export * from './featured.errors'; -export * from './featured.mapper'; +export * from './featured'; export * from './featured.types'; export * from './featured.utils'; diff --git a/src/config/index.ts b/src/config/index.ts index e550254..bad3509 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -34,14 +34,14 @@ export const buildBlockRangeScanConfig = ( vars: ConfigVars, scanKey?: string ): BlockRangeScanConfig => ({ - maxChunkSize: vars.getNumberEnv('SCANNER_NODES_MAX_CHUNK_SIZE'), + maxChunkSize: vars.getNumberEnv('SCANNER_NODES_MAX_CHUNK_SIZE') || 100, scanKey: scanKey || vars.getStringEnv('SCANNER_SCAN_KEY'), }); export const buildAbisServiceConfig = (vars: ConfigVars): AbisServiceConfig => ({ url: vars.getStringEnv('HYPERION_URL'), - limit: vars.getNumberEnv('ABIS_SERVICE_LIMIT'), - filter: vars.getStringEnv('ABIS_SERVICE_FILTER'), + limit: vars.getNumberEnv('ABIS_SERVICE_LIMIT') || 100, + filter: vars.getStringEnv('ABIS_SERVICE_FILTER') || 'eosio:setabi', }); export const buildAbisConfig = ( @@ -54,53 +54,55 @@ export const buildAbisConfig = ( export const buildBlockReaderConfig = (vars: ConfigVars): BlockReaderConfig => ({ endpoints: vars.getArrayEnv('BLOCK_READER_ENDPOINTS'), - shouldFetchDeltas: vars.getBooleanEnv('BLOCK_READER_FETCH_DELTAS'), - shouldFetchTraces: vars.getBooleanEnv('BLOCK_READER_FETCH_TRACES'), + shouldFetchBlock: vars.getBooleanEnv('BLOCK_READER_FETCH_BLOCK') || true, + shouldFetchDeltas: vars.getBooleanEnv('BLOCK_READER_FETCH_DELTAS') || true, + shouldFetchTraces: vars.getBooleanEnv('BLOCK_READER_FETCH_TRACES') || true, }); export const buildReaderWorkersConfig = ( vars: ConfigVars, threadsCount?: number ): WorkersConfig => ({ - threadsCount: threadsCount || vars.getNumberEnv('READER_MAX_THREADS'), - inviolableThreadsCount: vars.getNumberEnv('READER_INVIOLABLE_THREADS_COUNT'), + threadsCount: threadsCount || vars.getNumberEnv('READER_MAX_THREADS') || 1, + inviolableThreadsCount: vars.getNumberEnv('READER_INVIOLABLE_THREADS_COUNT') || 0, }); export const buildProcessorWorkersConfig = ( vars: ConfigVars, threadsCount?: number ): WorkersConfig => ({ - threadsCount: threadsCount || vars.getNumberEnv('PROCESSOR_MAX_THREADS'), - inviolableThreadsCount: vars.getNumberEnv('PROCESSOR_INVIOLABLE_THREADS_COUNT'), + threadsCount: threadsCount || vars.getNumberEnv('PROCESSOR_MAX_THREADS') || 1, + inviolableThreadsCount: vars.getNumberEnv('PROCESSOR_INVIOLABLE_THREADS_COUNT') || 0, }); export const buildFilterWorkersConfig = ( vars: ConfigVars, options?: FilterCommandOptions ): WorkersConfig => ({ - threadsCount: options?.threads || vars.getNumberEnv('FILTER_MAX_THREADS'), - inviolableThreadsCount: vars.getNumberEnv('FILTER_INVIOLABLE_THREADS_COUNT'), + threadsCount: options?.threads || vars.getNumberEnv('FILTER_MAX_THREADS') || 1, + inviolableThreadsCount: vars.getNumberEnv('FILTER_INVIOLABLE_THREADS_COUNT') || 0, }); export const buildProcessorTaskQueueConfig = ( vars: ConfigVars ): ProcessorTaskQueueConfig => ({ - interval: vars.getNumberEnv('PROCESSOR_TASK_QUEUE_CHECK_INTERVAL'), + interval: vars.getNumberEnv('PROCESSOR_TASK_QUEUE_CHECK_INTERVAL') || 5000, }); export const buildUnprocessedBlockQueueConfig = ( vars: ConfigVars ): UnprocessedBlockQueueConfig => ({ - maxBytesSize: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_MAX_BYTES_SIZE'), - sizeCheckInterval: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_SIZE_CHECK_INTERVAL'), - batchSize: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_BATCH_SIZE'), + maxBytesSize: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_MAX_BYTES_SIZE') || 256000000, + sizeCheckInterval: + vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_SIZE_CHECK_INTERVAL') || 2000, + batchSize: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_BATCH_SIZE') || 100, }); export const buildApiConfig = ( vars: ConfigVars, databaseConfigBuilder: (vars: ConfigVars, ...args: unknown[]) => UnknownObject ): ApiConfig => ({ - port: vars.getNumberEnv('API_PORT'), + port: vars.getNumberEnv('API_PORT') || 8080, database: databaseConfigBuilder(vars), }); @@ -124,10 +126,10 @@ export const buildBootstrapConfig = ( : vars.getStringEnv('END_BLOCK') ? parseToBigInt(vars.getStringEnv('END_BLOCK')) : null, - startFromHead: vars.getBooleanEnv('START_FROM_HEAD'), - mode: options?.mode || vars.getStringEnv('MODE'), + startFromHead: vars.getBooleanEnv('START_FROM_HEAD') || false, + mode: options?.mode || vars.getStringEnv('MODE') || 'default', abis: buildAbisServiceConfig(vars), - maxBlockNumber: vars.getNumberEnv('MAX_BLOCK_NUMBER'), + maxBlockNumber: vars.getNumberEnv('MAX_BLOCK_NUMBER') || 0xffffffff, }); export const buildReaderConfig = ( @@ -138,8 +140,8 @@ export const buildReaderConfig = ( database: databaseConfigBuilder(vars), broadcast: buildBroadcastConfig(vars), scanner: buildBlockRangeScanConfig(vars, options?.scanKey), - mode: options?.mode || vars.getStringEnv('MODE'), - maxBlockNumber: vars.getNumberEnv('MAX_BLOCK_NUMBER'), + mode: options?.mode || vars.getStringEnv('MODE') || 'default', + maxBlockNumber: vars.getNumberEnv('MAX_BLOCK_NUMBER') || 0xffffffff, unprocessedBlockQueue: buildUnprocessedBlockQueueConfig(vars), workers: buildReaderWorkersConfig(vars, options?.threads), blockReader: buildBlockReaderConfig(vars), @@ -152,7 +154,7 @@ export const buildFilterConfig = ( databaseConfigBuilder: (vars: ConfigVars, ...args: unknown[]) => UnknownObject, options?: FilterCommandOptions ): FilterConfig => ({ - mode: options?.mode || vars.getStringEnv('MODE'), + mode: options?.mode || vars.getStringEnv('MODE') || 'default', broadcast: buildBroadcastConfig(vars), workers: buildFilterWorkersConfig(vars, options), abis: buildAbisConfig(vars, databaseConfigBuilder), diff --git a/src/filter/filter.worker-loader.dependencies.ts b/src/filter/filter.worker-loader.dependencies.ts index 189551f..d2f7a1e 100644 --- a/src/filter/filter.worker-loader.dependencies.ts +++ b/src/filter/filter.worker-loader.dependencies.ts @@ -2,7 +2,7 @@ import { Serializer } from '@alien-worlds/api-core'; import { ShipAbis } from '@alien-worlds/block-reader'; import { ProcessorTaskQueue } from '../common/processor-task-queue'; import { WorkerLoaderDependencies } from '@alien-worlds/workers'; -import { Featured, FeaturedContractDataCriteria } from '../common/featured'; +import { FeaturedContracts } from '../common/featured'; import { Abis } from '../common'; import { FilterConfig } from './filter.config'; @@ -14,7 +14,7 @@ export abstract class FilterWorkerLoaderDependencies extends WorkerLoaderDepende public processorTaskQueue: ProcessorTaskQueue; public abis: Abis; public shipAbis: ShipAbis; - public featured: Featured; + public featuredContracts: FeaturedContracts; public serializer: Serializer; public abstract initialize( diff --git a/src/filter/filter.worker.ts b/src/filter/filter.worker.ts index bbea307..d84bdf5 100644 --- a/src/filter/filter.worker.ts +++ b/src/filter/filter.worker.ts @@ -5,7 +5,7 @@ import { Abis, BlockModel, DeltaByName, - Featured, + FeaturedContracts, ProcessorTask, ProcessorTaskQueue, SignedBlock, @@ -19,7 +19,7 @@ export default class FilterWorker extends Worker { protected dependencies: { shipAbis: ShipAbis; abis: Abis; - featured: Featured; + featuredContracts: FeaturedContracts; processorTaskQueue: ProcessorTaskQueue; serializer: Serializer; }, @@ -32,7 +32,7 @@ export default class FilterWorker extends Worker { deserializedBlock: BlockModel ): Promise { const { - dependencies: { abis, featured }, + dependencies: { abis, featuredContracts }, sharedData: { config }, } = this; const { @@ -51,14 +51,14 @@ export default class FilterWorker extends Worker { act: { account, name }, } = actionTrace; - if (featured.isFeatured(account)) { + if (featuredContracts.isFeatured(account)) { try { // If the block in which the contract was created cannot be found or // its index is higher than the current block number, skip it, // the contract did not exist at that time - const { content: contracts, failure } = await featured.readContracts([ - account, - ]); + const { content: contracts, failure } = await featuredContracts.readContracts( + [account] + ); if (failure) { log(failure.error); @@ -117,7 +117,7 @@ export default class FilterWorker extends Worker { deserializedBlock: BlockModel ): Promise { const { - dependencies: { abis, featured, serializer }, + dependencies: { abis, featuredContracts, serializer }, sharedData: { config }, } = this; const { @@ -142,12 +142,14 @@ export default class FilterWorker extends Worker { continue; } const { table, code, scope } = tableRow; - if (featured.isFeatured(code)) { + if (featuredContracts.isFeatured(code)) { try { // If the block in which the contract was created cannot be found or // its index is higher than the current block number, skip it, // the contract did not exist at that time - const { content: contracts, failure } = await featured.readContracts([code]); + const { content: contracts, failure } = await featuredContracts.readContracts( + [code] + ); if (failure) { log(failure.error); diff --git a/src/processor/processor.dependencies.ts b/src/processor/processor.dependencies.ts index c2007c9..fe1754e 100644 --- a/src/processor/processor.dependencies.ts +++ b/src/processor/processor.dependencies.ts @@ -3,7 +3,7 @@ import { Dependencies } from '../common/dependencies'; import { ContractDeltaMatchCriteria, ContractTraceMatchCriteria, - FeaturedMapper, + Featured, } from '../common/featured'; import { ProcessorTaskQueue } from '../common/processor-task-queue'; import { BroadcastClient } from '@alien-worlds/broadcast'; @@ -18,8 +18,8 @@ export abstract class ProcessorDependencies extends Dependencies { public workerLoaderPath?: string; public workerLoaderDependenciesPath: string; public broadcastClient: BroadcastClient; - public featuredTraces: FeaturedMapper; - public featuredDeltas: FeaturedMapper; + public featuredTraces: Featured; + public featuredDeltas: Featured; public processorTaskQueue: ProcessorTaskQueue; public processorsPath: string; diff --git a/src/processor/processor.runner.ts b/src/processor/processor.runner.ts index 5035ade..a7a1ba2 100644 --- a/src/processor/processor.runner.ts +++ b/src/processor/processor.runner.ts @@ -1,7 +1,7 @@ import { log } from '@alien-worlds/api-core'; import { WorkerPool, WorkerMessage } from '@alien-worlds/workers'; import { - FeaturedMapper, + Featured, ContractTraceMatchCriteria, ContractDeltaMatchCriteria, ProcessorTaskQueue, @@ -16,8 +16,8 @@ export class ProcessorRunner { private loop: boolean; constructor( - protected featuredTraces: FeaturedMapper, - protected featuredDeltas: FeaturedMapper, + protected featuredTraces: Featured, + protected featuredDeltas: Featured, protected workerPool: WorkerPool, protected queue: ProcessorTaskQueue ) { @@ -45,7 +45,6 @@ export class ProcessorRunner { return; } - // const processorName = await featured.getProcessor(task.type, task.label); // If there is a processor name, it then gets a worker from the worker pool. if (processorName) { const worker = await workerPool.getWorker(processorName); From 93aaae0595beca9b4525d9b450a75a0865923a90 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 30 Jun 2023 17:55:31 +0200 Subject: [PATCH 060/107] 0.0.140 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index dbf4f5f..e69d23d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.139", + "version": "0.0.140", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 2a30645a5456493ca148a98f39442666a44041d8 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 30 Jun 2023 18:07:37 +0200 Subject: [PATCH 061/107] partial defaults criteria --- src/common/featured/featured.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/featured/featured.ts b/src/common/featured/featured.ts index 0d79e3b..da0b655 100644 --- a/src/common/featured/featured.ts +++ b/src/common/featured/featured.ts @@ -32,13 +32,13 @@ export class Featured { * Creates a new instance of the contract processor mapper. * @param {ProcessorMatchCriteria[]} criteria - An array of match criteria for the processor. * @param {MatchCriteriaType} pattern - The criteria pattern. - * @param {MatchCriteriaType} defaults - Default criteria key:value pairs. + * @param {Partial} defaults - Default criteria key:value pairs. * @param {ProcessorMatcher} matchers - Optional map of matchers. */ constructor( criteria: ProcessorMatchCriteria[], protected pattern: MatchCriteriaType, - protected defaults?: MatchCriteriaType, + protected defaults?: Partial, matchers?: ProcessorMatcher ) { if (defaults) { From e45d38ab91593511dff565b5e41c8423f767dce0 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 30 Jun 2023 18:07:54 +0200 Subject: [PATCH 062/107] 0.0.141 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e69d23d..69d79c2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.140", + "version": "0.0.141", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 0854a896d45df449afebcdbe03a9e5b873eb0103 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 3 Jul 2023 14:13:25 +0200 Subject: [PATCH 063/107] speed up reading head, modify readme and tutorials --- README.md | 9 +- package.json | 2 +- .../unprocessed-block-queue.ts | 15 +- .../unprocessed-block-queue.types.ts | 1 + src/config/index.ts | 1 + src/reader/reader.worker.ts | 7 +- tutorials/config-vars.md | 1 + tutorials/extending-history-tools.md | 306 ++++++++++++++++++ tutorials/running-history-tools-docker.md | 6 - tutorials/running-history-tools-locally.md | 44 --- tutorials/writing-history-tools.md | 228 ------------- yarn.lock | 8 +- 12 files changed, 338 insertions(+), 290 deletions(-) create mode 100644 tutorials/extending-history-tools.md delete mode 100644 tutorials/running-history-tools-docker.md delete mode 100644 tutorials/running-history-tools-locally.md delete mode 100644 tutorials/writing-history-tools.md diff --git a/README.md b/README.md index fb22cfa..0774e42 100644 --- a/README.md +++ b/README.md @@ -105,10 +105,13 @@ The Config tools are used for generating configuration objects based on values s ## Tutorials +For tutorials on creating and using the history tools for your specific needs, see the tutorials in the [History Tools Starter Kit](https://github.com/Alien-Worlds/history-tools-starter-kit) repository. If you want to create history tools with `mongodb` and `eosjs` tools, you should go to the mentioned repository. + + +If you want to extend the capabilities of the history tools or take advantage of other third-party resources, please refer to the following tutorial. + +- [Extending history tools](./tutorials/writing-history-tools.md) - [Description of configuration variables](./tutorials/config-vars.md) -- [Writing your own history tools](./tutorials/writing-history-tools.md) -- [Running history tools locally](./tutorials/running-history-tools-locally.md) -- [Running history tools with Docker](./tutorials/running-history-tools-with-docker.md) ## Contributing diff --git a/package.json b/package.json index 69d79c2..8b6939e 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ }, "dependencies": { "@alien-worlds/api-core": "^0.0.151", - "@alien-worlds/block-reader": "^0.0.14", + "@alien-worlds/block-reader": "^0.0.15", "@alien-worlds/broadcast": "^0.0.10", "@alien-worlds/workers": "^0.0.7", "async": "^3.2.4", diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.ts index 2e04277..9055cbb 100644 --- a/src/common/unprocessed-block-queue/unprocessed-block-queue.ts +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.ts @@ -31,7 +31,8 @@ export class UnprocessedBlockQueue protected collection: UnprocessedBlockSource, protected mapper: Mapper, protected maxBytesSize: number, - protected batchSize: number + protected batchSize: number, + protected fastLaneBatchSize: number ) {} private async sendBatch() { @@ -70,7 +71,11 @@ export class UnprocessedBlockQueue } } - public async add(block: Block, isLast = false): Promise> { + public async add( + block: Block, + isFastLane = false, + isLast = false + ): Promise> { try { let addedBlockNumbers: bigint[] = []; @@ -78,7 +83,11 @@ export class UnprocessedBlockQueue this.cache.push(block); } - if (this.cache.length === this.batchSize || isLast) { + if ( + (isFastLane && this.cache.length === this.fastLaneBatchSize) || + this.cache.length === this.batchSize || + isLast + ) { addedBlockNumbers = await this.sendBatch(); } diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.types.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.types.ts index 9c3ce4e..4d6c6e8 100644 --- a/src/common/unprocessed-block-queue/unprocessed-block-queue.types.ts +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.types.ts @@ -1,6 +1,7 @@ export type UnprocessedBlockQueueConfig = { maxBytesSize: number; batchSize: number; + fastLaneBatchSize: number; sizeCheckInterval?: number; [key: string]: unknown; }; diff --git a/src/config/index.ts b/src/config/index.ts index bad3509..0a84a6c 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -96,6 +96,7 @@ export const buildUnprocessedBlockQueueConfig = ( sizeCheckInterval: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_SIZE_CHECK_INTERVAL') || 2000, batchSize: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_BATCH_SIZE') || 100, + fastLaneBatchSize: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_FAST_LANE_BATCH_SIZE') || 1, }); export const buildApiConfig = ( diff --git a/src/reader/reader.worker.ts b/src/reader/reader.worker.ts index 111b428..919ffeb 100644 --- a/src/reader/reader.worker.ts +++ b/src/reader/reader.worker.ts @@ -63,7 +63,12 @@ export default class ReaderWorker extends Worker { blockReader.onReceivedBlock(async block => { const isLast = endBlock === block.thisBlock.blockNumber; - const { content: addedBlockNumbers, failure } = await blockQueue.add(block, isLast); + const isFastLane = block.thisBlock.blockNumber >= block.lastIrreversible.blockNumber; + + const { content: addedBlockNumbers, failure } = await blockQueue.add( + block, + isFastLane, isLast + ); if (Array.isArray(addedBlockNumbers) && addedBlockNumbers.length > 0) { this.logProgress(addedBlockNumbers); diff --git a/tutorials/config-vars.md b/tutorials/config-vars.md index 32f72fe..8e4eb0a 100644 --- a/tutorials/config-vars.md +++ b/tutorials/config-vars.md @@ -48,4 +48,5 @@ The following settings are additional for more advanced users who want to tweak | `UNPROCESSED_BLOCK_QUEUE_MAX_BYTES_SIZE` | _number_ | The maximum size of the queue in bytes. | 256000000 | | `UNPROCESSED_BLOCK_QUEUE_SIZE_CHECK_INTERVAL` | _number_ | Specifies the waiting time in milliseconds to check that the current queue size in bytes does not exceed the maximum allowed. | 2000 | | `UNPROCESSED_BLOCK_QUEUE_BATCH_SIZE` | _number_ | Batch size of unprocessed blocks sent to the database at one time. The batch setting can be modified to optimize the transfer consumption to the database. | 100 | +| `UNPROCESSED_BLOCK_QUEUE_FAST_LANE_BATCH_SIZE` | _number_ | Value used when block number is greater than last irreversible block number. Batch size of unprocessed blocks sent to the database at one time. The batch setting can be modified to optimize the transfer consumption to the database. | 1 | | `PROCESSOR_TASK_QUEUE_CHECK_INTERVAL` | _number_ | Time to wait in milliseconds to check if there are new processor tasks available. This option is needed when the filter finishes its work and will not send information about the update to the processor. | 5000 | diff --git a/tutorials/extending-history-tools.md b/tutorials/extending-history-tools.md new file mode 100644 index 0000000..c33b760 --- /dev/null +++ b/tutorials/extending-history-tools.md @@ -0,0 +1,306 @@ +# Extending History Tools + +In this tutorial, we'll briefly go over how to extend the history tools and possibly change the resources. Such a situation may occur when the basic construction of the tools does not meet all expectations. in our case, this happened when creating the leaderboard api. We could rely on the available architecture and put all the work of updating the leaderboards in processors, but we wanted to separate this process from others and leave the standard processes (boot, filter, reader, processor) unchanged. We needed another process whose task would be to update the leaderboard database and which would be able to scale properly to increase work efficiency. We called the new process writer and its role is to download records in the leaderboard_updates collection and create updates in the target leaderboar collections (of which we have several). On this own example, we will describe ways to extend the capabilities of history tools. + +First, decide how you want to extend the tools. You can download the history-tools repository and add your own implementations of components responsible for communication with the blockchain and the database. The second method is to create a new repository and import history tools along with [Starter Kit](https://github.com/Alien-Worlds/history-tools-starter-kit). Since we will continue to use mongodb and eos we will choose second option. + + +[Back to Readme](../README.md) + +# Table of Contents + +- [Extending History Tools](#extending-history-tools) + - [1. Project Preparation](#1-project-preparation) + - [2. Creating a List of Contracts](#2-creating-a-list-of-contracts) + - [Step 3: Create additional processes](#step-3-create-additional-processes) + - [3.1 Starter](#31-starter) + - [3.2 Worker](#32-worker) + - [3.3 WorkerLoader](#33-workerloader) + - [3.4 WorkerLoaderDependencies](#34-workerloaderdependencies) + - [Step 4: Create Processes](#step-4-create-processes) + - [4.1 Bootstrap](#41-bootstrap) + - [4.2 Broadcasting](#42-broadcasting) + - [4.3 Reader](#43-reader) + - [4.4 Filter](#44-filter) + - [4.5 Processor](#45-processor) + - [Step 5: Create necessary processors](#step-5-create-necessary-processors) +- [Using other resources](#using-other-resources) + +### 1. Project Preparation + +The first step to create history tools is to set up a new project and import the necessary dependencies. + +Your `package.json` file should look similar to this: + +```json +{ + "name": "your-history-tools", + "version": "0.0.1", + "description": "your-description", + "packageManager": "yarn@3.2.3", + "main": "build/index.js", + "types": "build/index.d.ts", + "scripts": { + "broadcast": "node build/broadcast/index.js", + "boot": "node build/bootstrap/index.js", + "reader": "node build/reader/index.js", + "filter": "node build/filter/index.js", + "processor": "node build/processor/index.js", + "writer": "node build/writer/index.js", + ... + }, + ... + "dependencies": { + "@alien-worlds/api-history-tools": "^0.0.136", + "@alien-worlds/history-tools-default-dependencies": "^0.0.27", + ... all your contract packages and other typescript dependencies + } +} +``` + +Note that in `sripts` we have a new `writer` process which is located in a separate directory like the others. + +### 2. Creating a List of Contracts + +Now you need to define what data you want to download from the blockchain. In our case, this will be the `logmine` action from the `notify.world` contract: + +```json +{ + "traces": [ + { + "shipTraceMessageName": ["transaction_trace_v0"], // Optional + "shipActionTraceMessageName": ["action_trace_v0", "action_trace_v1"], // Optional + "contract": ["dao.worlds"], + "action": ["*"], + "processor": "NotifyWorldTraceProcessor" + } + ], + "deltas": [] +} +``` + +## Step 3: Create additional processes + +Knowing what we will be listening to and what data we want to download from the blockchain, we need to implement additional processes from scratch. In our case, it will be one whose task is to download data, update the leaderboard and save changes to the appropriate collection. + +#### 3.1 Starter + +Let's start with the startup file, just like the rest of the processes, it should create the config based on given options or enviroment variables, pass the necessary dependencies and run the program. + +```typescript +export const startWriter = (args: string[]) => { + const vars = new ConfigVars(); + const options = bootstrapCommand.parse(args).opts(); + const config: LeaderboardWriterConfig = buildWriterConfig(vars, options); // you should implement your config builder + + const workerPool = await WorkerPool.create({ + ...config.workers, + sharedData: { config }, + workerLoaderPath: `${__dirname}/leaderboard.worker-loader`, + workerLoaderDependenciesPath: `${__dirname}/leaderboard.worker-loader.dependencies`, + }); + + ... + + const writer = await LeaderboardWriter.create(config, workerPool); + writer.next(); +} +``` + +If you want to use workers in your process, like we do in leaderboard writer, you need to add three more worker files, worker loader and dependencies worker loader. + +#### 3.2 Worker + +The worker is launched from the workerPool which you can implement directly in the starter or a separate class. In the Worker class, you need to implement all the worker logic in the body of the `run` method and finally call the `resolve` or `reject` method so as not to block the `workerPool`. Don't forget to override the standard constructor to have access to the `dependencies` provided by the loader. + +```typescript +export default class LeaderboardWorker extends Worker { + constructor( + protected dependencies: { + updatesRepository: LeaderboardUpdateRepository; + updateLeaderboardUseCase: UpdateLeaderboardUseCase; + }, + protected sharedData: LeaderboardSharedData + ) { + super(); + } + + public async run(): Promise { + const { dependencies: { updatesRepository, updateLeaderboardUseCase } } = this; + try { + ... update leaerboards logic + + this.resolve(); + } catch (error) { + this.reject(error); + } + } +} +``` + +#### 3.3 WorkerLoader + +As you know, the worker loader is used to create a worker instance and initialize and transfer the dependencies needed by the worker. The `setup` method should be overridden as in the example below if you have dependencies and you need to pass arguments to the `initialize` method (in our case this is the config object). Another reason may also be the need to perform additional settings, e.g. adding event listeners to the created dependencies. In the `load` method, we create instances of our worker and pass the created `dependencies`. + +```typescript +export default class LeaderboardWorkerLoader extends DefaultWorkerLoader< +LeaderboardSharedData, +LeaderboardWorkerLoaderDependencies +> { + public async setup(sharedData: LeaderboardSharedData): Promise { + const { config } = sharedData; + await super.setup(sharedData, config); // initialize dependencies + ... additional work eg. event listeners + } + + public async load(): Promise { + const { dependencies, sharedData } = this; + return new LeaderboardWorker(dependencies, sharedData); + } +} +``` + +#### 3.4 WorkerLoaderDependencies + +At this point, the matter is simple, you need to implement all necessary depenedencies. To do this, add the appropriate code in the initialize method, which will be called in the setup worker loader method. + +```typescript +export class LeaderboardWorkerLoaderDependencies extends WorkerLoaderDependencies { + public updatesRepository: LeaderboardUpdateRepository; + public updateLeaderboardUseCase: UpdateLeaderboardUseCase; + + public async initialize(config: LeaderboardConfig): Promise { + ... initialize all dependencies + } +} +``` +Remember that when creating a `workerPool` you will have to give paths to the worker loader and its dependencies. + +## Step 4: Create Processes + +Next, create a folder for each history tools process (`broadcast`, `bootstrap`, `filter`, `reader`, `processor`). Each of these should have an `index.ts` file within. + +#### 4.1 Bootstrap + +In the `bootstrap` directory, create an `index.ts` file and add the following content: + +```typescript +import { + startBootstrap, + DefaultBootstrapDependencies, +} from '@alien-worlds/history-tools-starter-kit'; +import path from 'path'; + +startBootstrap( + process.argv, + new DefaultBootstrapDependencies(), + path.join(__dirname, '../../your.featured.json') +); +``` + +#### 4.2 Broadcasting + +In the `broadcast` directory, create an `index.ts` file and add the following content: + +```typescript +import { startBroadcast } from '@alien-worlds/history-tools-starter-kit'; + +startBroadcast(); +``` + +#### 4.3 Reader + +In the `reader` directory, create an `index.ts` file and add the following content: + +```typescript +import { + startReader, + DefaultReaderDependencies, +} from '@alien-worlds/history-tools-starter-kit'; + +startReader(process.argv, new DefaultReaderDependencies()); +``` + +#### 4.4 Filter + +In the `filter` directory, create an `index.ts` file and add the following content: + +```typescript +import { + startFilter, + DefaultFilterDependencies, +} from '@alien-worlds/history-tools-starter-kit'; +import path from 'path'; + +startFilter( + process.argv, + new DefaultFilterDependencies(), + path.join(__dirname, '../../your.featured.json') +); +``` + +#### 4.5 Processor + +In the `processor` directory, create an `index.ts` file and add the following content: + +```typescript +import { + startProcessor, + DefaultProcessorDependencies, +} from '@alien-worlds/history-tools-starter-kit'; +import path from 'path'; + +startProcessor( + process.argv, + new DefaultProcessorDependencies(), + path.join(__dirname, './processors'), + path.join(__dirname, '../../your.featured.json') +); +``` + +## Step 5: Create necessary processors + +Finally, create a `processors` folder where you will store all of the processor files. The contents of this folder should be exported in `index.ts`, and each processor should be exported as a default. + +```typescript +// ./processors/index.ts +export * from './notify-world.trace-processor'; +... + +// ./processors/notify-world.trace-processor.ts +import { ActionTraceProcessor, ProcessorTaskModel } from '@alien-worlds/history-tools-starter-kit'; + +export default class NotifyWorldTraceProcessor extends ActionTraceProcessor { + public async run(model: ProcessorTaskModel): Promise { + try { + if (name === NotifyWorldActionName.Logmine) { + const update = LeaderboardUpdate.fromLogmineJson( + blockNumber, + blockTimestamp, + data, + tlmDecimalPrecision + ); + + const json = update.toJson(); + + ... + + const updateResult = await leaderboardUpdates.add(LeaderboardUpdate.fromJson(json)); + } + } + + this.resolve(); + } catch (error) { + this.reject(error); + } + } +} +``` + +That's it, the examples shown are of course partial but you should get a general idea of what to do if you want to extend History Tools by adding more processes. + +## Using other resources + +If, apart from simply adding new processes or modifying existing ones, you want to use another database (replacing the default mongodb) or another blockchain (reading tools). You should check [Starter Kit](https://github.com/Alien-Worlds/history-tools-starter-kit) implementation and write your own based on all interfaces used in [Api Core](https://github.com/Alien-Worlds/api-core), **History Tools**. Check the contents of this package and replace e.g. all mongo.\* components with your own. Theoretically, if you follow the interface guidelines, everything should work fine, including serialization and block reader. The **kit** prepared in this way should be imported into your history tools implementation and then follow the guidelines mentioned above. + +Remember, if your **kit** works and meets all requirements, it's worth thinking about sharing it with other users. More _starter-kit_ type repositories may be useful and maybe more users will benefit from your work. Good luck! diff --git a/tutorials/running-history-tools-docker.md b/tutorials/running-history-tools-docker.md deleted file mode 100644 index 71b6105..0000000 --- a/tutorials/running-history-tools-docker.md +++ /dev/null @@ -1,6 +0,0 @@ -# Running history tools with Docker - -This tutorial provides a comprehensive guide to run History Tools with Docker. - -[Back to Readme](../README.md) - diff --git a/tutorials/running-history-tools-locally.md b/tutorials/running-history-tools-locally.md deleted file mode 100644 index 55c025c..0000000 --- a/tutorials/running-history-tools-locally.md +++ /dev/null @@ -1,44 +0,0 @@ -# Running History Tools locally - -This tutorial provides a comprehensive guide to run History Tools on your local machine. - -[Back to Readme](../README.md) - -If you followed the instructions from the previous tutorial [Writing your own History Tools](./writing-history-tools.md) now to run the tools locally you must first start the database. If you don't have a local server, you can run it on docker. You can use the following content for docker-compose.yml - -```bash -version: "3.2" -services: - mongo: - image: mongo - container_name: 'mongo' - restart: always - mem_limit: 2g - ports: - - '27017:27017' - networks: - - mongo_net - -networks: - mongo_net: - driver: bridge -``` - -Then open a new terminal session for each of the History tools processes and enter the startup command for each of them accordingly: - -```bash -# terminal 1 -yarn broadcast - -# terminal 2 -yarn boot - -# terminal 3 -yarn reader - -# terminal 4 -yarn filter - -# terminal 5 -yarn processor -``` \ No newline at end of file diff --git a/tutorials/writing-history-tools.md b/tutorials/writing-history-tools.md deleted file mode 100644 index daded48..0000000 --- a/tutorials/writing-history-tools.md +++ /dev/null @@ -1,228 +0,0 @@ -# Writing your own History Tools - -This tutorial provides a comprehensive guide to building your own Alien Worlds History Tools, a set of utilities for downloading and processing data from the blockchain. The History Tools are divided into several packages, making it modular and independent from third-party resources. - -[Back to Readme](../README.md) - -## Table of Contents - -1. [Project Preparation](#project-preparation) -2. [Creating a List of Contracts](#creating-a-list-of-contracts) -3. [Process Implementation](#process-implementation) - - [Bootstraps](#bootstraps) - - [Broadcasting](#broadcasting) - - [Reader](#reader) - - [Filter](#filter) - - [Processor](#processor) -4. [Creating Processors](#creating-processors) - -### 1. Project Preparation - -The first step to create history tools is to set up a new project and import the necessary dependencies. - -Your `package.json` file should look similar to this: - -```json -{ - "name": "your-history-tools", - "version": "0.0.1", - "description": "your-description", - "packageManager": "yarn@3.2.3", - "main": "build/index.js", - "types": "build/index.d.ts", - "scripts": { - "broadcast": "node build/broadcast/index.js", - "boot": "node build/bootstrap/index.js", - "reader": "node build/reader/index.js", - "filter": "node build/filter/index.js", - "processor": "node build/processor/index.js", - ... - }, - ... - "dependencies": { - "@alien-worlds/api-history-tools": "^0.0.136", - "@alien-worlds/history-tools-default-dependencies": "^0.0.27", - ... all your contract packages and other typescript dependencies - } -} -``` - -After setting up your `package.json`, define all necessary configuration variables, preferably in an `.env` file. For instructions to complete the `.env` file, see [./envs.md](./envs.md). - -Next, create a new `src` directory and within it, create `bootstrap`, `broadcast`, `filter`, `reader` and `processor` directories. - -### 2. Creating a List of Contracts - -Now you need to define what data you want to download from the blockchain. You can achieve this by creating a separate json file containing the configuration. This json file schema is as follows: - -```json -{ - "traces": [ - { - "shipTraceMessageName": ["transaction_trace_v0"], - "shipActionTraceMessageName": ["action_trace_v0", "action_trace_v1"], - "contract": ["dao.worlds"], - "action": ["*"], - "processor": "DaoWorldsTraceProcessor" - } - ], - "deltas": [ - { - "shipDeltaMessageName": ["table_delta_v0"], - "name": ["contract_row"], - "code": ["dao.worlds"], - "scope": ["*"], - "table": ["*"], - "processor": "DaoWorldsDeltaProcessor" - } - ] -} - -// or shorter version -{ - "traces": [ - { - "contract": ["dao.worlds"], - "action": ["*"], - "processor": "DaoWorldsTraceProcessor" - } - ], - "deltas": [ - { - "code": ["dao.worlds"], - "scope": ["*"], - "table": ["*"], - "processor": "DaoWorldsDeltaProcessor" - } - ] -} -``` - -Here, you can specify many values in the arrays. For example, if you are only interested in specific actions for the `dao.worlds` contract, you can write them, e.g., ["appointcust", "nominate", "firecust"], instead of '*' (get all). This way, you can limit the work of the processor to only the data that interests you. The same applies for deltas. - -### 3. Process Implementation - -Having defined what data you're interested in, the next step is to create processes that will handle the work. - -#### 3.1 Bootstraps - -In the `bootstrap` - -directory, create an `index.ts` file and add the following content: - -```typescript -import { startBootstrap } from '@alien-worlds/api-history-tools'; -import { DefaultBootstrapDependencies } from '@alien-worlds/history-tools-default-dependencies'; -import path from 'path'; - -startBootstrap( - process.argv, - new DefaultBootstrapDependencies(), - path.join(__dirname, '../../your.featured.json') -); -``` - -#### 3.2 Broadcasting - -In the `broadcast` directory, create an `index.ts` file and add the following content: - -```typescript -import { startBroadcast } from '@alien-worlds/api-history-tools'; - -startBroadcast(); -``` - -#### 3.3 Reader - -In the `reader` directory, create an `index.ts` file and add the following content: - -```typescript -import { startReader } from '@alien-worlds/api-history-tools'; -import { DefaultReaderDependencies } from '@alien-worlds/history-tools-default-dependencies'; - -startReader(process.argv, new DefaultReaderDependencies()); -``` - -#### 3.4 Filter - -In the `filter` directory, create an `index.ts` file and add the following content: - -```typescript -import { startFilter } from '@alien-worlds/api-history-tools'; -import { DefaultFilterDependencies } from '@alien-worlds/history-tools-default-dependencies'; -import path from 'path'; - -startFilter( - process.argv, - new DefaultFilterDependencies(), - path.join(__dirname, '../../your.featured.json') -); -``` - -#### 3.5 Processor - -In the `processor` directory, create an `index.ts` file and add the following content: - -```typescript -import { startProcessor } from '@alien-worlds/api-history-tools'; -import { DefaultProcessorDependencies } from '@alien-worlds/history-tools-default-dependencies'; -import path from 'path'; - -startProcessor( - process.argv, - new DefaultProcessorDependencies(), - path.join(__dirname, './processors'), - path.join(__dirname, '../../your.featured.json') -); -``` - -### 4. Creating Processors - -You also need to specify the Processor class that is to be started when the data is read. This is done in the json file you created in Step 2. - -Here is an example: - -```json -"processor": "DaoWorldsDeltaProcessor" -``` - -Now, create this class and save it where the worker loader processor can find and instantiate it. In the `processors` directory, create an `index.ts` file and export all processors contained in the directory. - -```typescript -// ./processors/index.ts -export * from './dao-worlds.trace-processor'; -export * from './dao-worlds.delta-processor'; -... - -// ./processors/dao-worlds.trace-processor.ts -import { ActionTraceProcessor } from '@alien-worlds/api-history-tools'; - -export class DaoWorldsTraceProcessor extends ActionTraceProcessor { - public async run(model: ProcessorTaskModel): Promise { - try { - //... all of your operations - this.resolve(); - } catch (error) { - this.reject(error); - } - } -} - -// ./processors/dao-worlds.delta-processor.ts -import { DeltaProcessor } from '@alien-worlds/api-history-tools'; - -export class DaoWorldsDeltaProcessor extends DeltaProcessor { - public async run(model: ProcessorTaskModel): Promise { - try { - //... all of your operations - this.resolve(); - } catch (error) { - this.reject - -(error); - } - } -} -``` - -With these steps completed, you now have all the necessary components to run your history tools locally or via Docker. Please keep in mind that the project is constantly being updated and some elements may change over time. diff --git a/yarn.lock b/yarn.lock index feccb45..b05562f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12,10 +12,10 @@ node-fetch "2" reflect-metadata "^0.1.13" -"@alien-worlds/block-reader@^0.0.14": - version "0.0.14" - resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.14/ef19b83824dae6eeebf08a2d3bd77e430e493759#ef19b83824dae6eeebf08a2d3bd77e430e493759" - integrity sha512-F/E94qnZ8Ok9UcKbCzEmARw4VKjYXoox6BPR48g9L7RjtGvBmFii1TIB3+/aosExEHOLmnhD9wA+TBP+NnmEPg== +"@alien-worlds/block-reader@^0.0.15": + version "0.0.15" + resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.15/08fdc9297f958981720da64da461832f603f76e6#08fdc9297f958981720da64da461832f603f76e6" + integrity sha512-8lYnOTx1pnvwNkrKafsYCBWOqwpt8RVIoc6w1L9zUqg2o9qWy9eI4fkVlCvamKPpsaNYKGgMeVG14lH8M8P/IQ== dependencies: "@alien-worlds/api-core" "^0.0.151" From c237310b6f0f28cae2b4b9f3b6305b1d423eef21 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 3 Jul 2023 19:47:13 +0200 Subject: [PATCH 064/107] update readme --- README.md | 112 +++++++++++++++++++++++--- src/common/abis/abis.ts | 2 +- tutorials/what-is-featured-content.md | 0 3 files changed, 103 insertions(+), 11 deletions(-) create mode 100644 tutorials/what-is-featured-content.md diff --git a/README.md b/README.md index 0774e42..af86235 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ This package encapsulates the core mechanism, however, complete functionality re - [Installation](#installation) - [Processes](#processes) - - [APIs](#apis) + - [API](#api) - [Broadcasting](#broadcasting) - [Bootstrap](#bootstrap) - [Reader](#reader) @@ -51,7 +51,7 @@ yarn add @alien-worlds/api-history-tools All processes utilize the commander, enabling specific value assignments for individual options or the use of environment variables. -### APIs +### API The API process, currently under development, is intended to provide easy access to downloaded data. This Express.js-based API allows viewing of blockchain data, offering endpoints to retrieve block-specific data, transactions, tables, or data from a specific range according to selected criteria. The API is read-only, it doesn't contain methods that modify the content. @@ -77,27 +77,118 @@ The Processor process retrieves tasks from the database, generating appropriate ## Common components +In addition to the processes that make up the base of History Tools, there are also components used between these processes. To a large extent, this repository contains high/domain level implementations of these components, and any code that directly depends on external libraries is placed in a separate repository such as the **Starter Kit**. common components consist of: + + ### Abis -... + +`Abis` is basically a repository for storing contract ABIs listed as "featured" (e.g. in a json file). In addition, `Abis` also includes a service for downloading new ABI. `Abis` in the first place is used in **Bootstrap** process to download (using the service) all ABIs of listed contracts. The downloaded ABI is saved in the form of a document containing the following data: + + +```javascript +{ + block_number, // numer bloku w ktorym zainicjonowano kontrakt + contract, // nazwa kontraktu + hex // ABI w postaci HEX +} +``` + +We also use Addis in **Filter** when creating tasks for the processor. At that time we get ABI (for the concrete contract) from the database and if it does not exist, try to fetch it. + +#### Methods +- `getAbis(options?)`: Retrieves the ABIs (Application Binary Interfaces) for the specified options. +- `getAbi(blockNumber, contract, fetch?)`: Retrieves the single ABI for the specified block number and contract address. +- `storeAbi(blockNumber, contract, hex)`: Stores the ABI with data. +- `fetchAbis(contracts?)`: Fetch via service ABIs. Optionally you can specify which contracts you want to use +- `cacheAbis(contracts?)`: Caches the ABIs for the specified contracts. + + ### BlockRangeScanner -... +We use `BlockRangeScanner` in the *reader* process in "replay" mode. When we want to download data from the blockchain in a specific range of blocks (these can be even years), we need a kind of data download schedule. `BlockRangeScanner` is used to create sets in a given range with a specific label (you can define the download purpose). The label is extremely important, it cannot be duplicated, it is a unique ID for the whole process. If an already taken key is used, the process will throw an appropriate error. `BlockRangeScanner` will divide the range into subgroups containing an equal number of blocks to be scanned (the last set may be smaller than the others). Replay mode, instead of directly actively listening to the blockchain, first downloads one of the scans from the list and passes it to the block reader, which in this given set will download data. If you use multiple threads or scale **reader** processes (e.g. in Docker), you can speed up the download process by making multiple queries to the scanner. The script will not give the same range to another thread if it is already handled. + +#### Methods + +- `createScanNodes(key, startBlock, endBlock)`: Creates range scan sets under specified label. +- `getNextScanNode(key)`: Returns the next (not currently scanned) set of the range. +- `hasUnscannedBlocks(key, startBlock, endBlock)`: Returns a boolean after checking whether the condition is true or not. +- `updateScanProgress(scanKey, blockNumber)`: Updates the number of the currently scanned block in the corresponding set. + + ### BlockState -... +`BlockState` is a simple service for updating the current status of history tools. Various statistics are stored in the database, including the number of the currently read block. After reading a given block, the state is updated so that in the case of restarting history tools, it will start from the last processed block. + +#### Methods + +- `getState()`: Returns the current statistics. +- `getBlockNumber()`: Returns the number of the last read block. +- `updateBlockNumber(blockNumber)`: Updates the block number value in the statistics. ### Featured -... +Featured is a class but also a category of tools used with a list of "featured" contracts, their actions and deltas. `FeaturedContracts` contained here is a repository that stores information in the database (contract name and block number in which it first appeared) of contracts that are included in the list (e.g. json file). `FeaturedContracts` is used in **bootstrap**, where after extracting contracts from the list, the previously mentioned data is downloaded. Another place where we use this (and other) `Featured` component is **Filter**. When reading the block data, you need to compare the contract name with the one included in the repository, and then, in order to determine the appropriate `ABI`, we check the block number in which it appeared for the first time. + +#### Methods + +- `readContracts(data)`: reads a json object or list of strings and retrieves the previously mentioned data. After they are downloaded from the web, they are stored in the database and cache. +- `isFeatured(contract)`: Checks if the given name is on the list of blocks of interest. + +We build `Featured` class instances based on the data contained in the list of "featured" contracts (e.g. in the form of a JSON object). Each object in the list contains not only the names of the contracts, but also criteria on choosing the right processor for individual actions and contract deltas. For more information, see the [Tutorials](#tutorials) section. + + +#### Methods + +- `getCriteria(criteria)`: Gets all match criteria that match the given criteria. +- `getProcessor(label)`: Gets the processor for the given label and criteria. +- `getContracts()`: Lists all contracts in the processor. + ### ProcessorTaskQueue -... + +As the name suggests, it is a queue (repository) of processor tasks. This list is generated by **Filter** process and then saved by this repository to the database. In the next process, which is **processor** (or several), it downloads the task, immediately removing it from the list and proceeds to work. Each of the tasks contains encrypted data that needs to be decoded using native blockchain deserialziers and having the data, you can start processing this data into the expected results. In case of failure task is sent to a separate list `unsuccessful_processor_tasks` for next attempt or analysis. +The task document diagram is as follows: + +```typescript +{ + _id : "6494e07f7fcfdd1bb21c8da6", + abi : "...", + short_id : "uspts.worlds:addpoints", + label : "transaction_trace_v0:action_trace_v1:uspts.worlds:addpoints", + type : "action", + mode : "default", + content: "... binary ..."; + hash: "db1a9ef8ddf670313b6868760283e42b0cc21176"; + block_number : 252016298, + block_timestamp : "2023-06-22T23:59:12.000Z", + is_fork : false, + error?; +}; +``` + +#### Methods + +- `nextTask(mode)`: Returns the next unassigned task from the list. +- `addTasks(tasks)`: Adds tasks to the list. +- `stashUnsuccessfulTask(task, error)`: Method to put tasks into the failed list. + ### UnprocessedBlockQueue -... +Just like in the case of Processor tasks, we also have a queue of blocks that for some reason have not been read. In order not to have gaps in the history, each unread block is placed in a special collection in the database so that later you can analyze the error, fix it and try to read the block again. Due to the fact that reading blocks is a fast process, we do not want to stress the database with a large number of requests, unread blocks are sent in batches. Failure to read a block in most cases will be caused by external conditions and will concern more blocks than just one. In order to avoid possible overloads, it is possible to set a limit on the number of unread blocks or their total size. This repository/queue has a bit more methods to customize the behavior of this queue in a custom way. + +#### Methods + +- `getBytesSize()`: Gets the size in bytes of the queue of unread blocks. +- `add(block, isLast)`: Adds a block to the list. +- `next()`: Returns the next block in the list. +- `beforeSendBatch(handler)`: Sets the handler to run before sending blocks to the database. +- `afterSendBatch(handler)`: Sets the handler to start after sending blocks to the database. +- `onOverload(handler)`: Sets the handler to start when the number or total size of blocks in the list is exceeded. + + ## Additional Tools -Besides the main processes, this package also contains tools, such as: +Besides the main processes and commons, this package also contains tools, such as: ### Config @@ -110,7 +201,8 @@ For tutorials on creating and using the history tools for your specific needs, s If you want to extend the capabilities of the history tools or take advantage of other third-party resources, please refer to the following tutorial. -- [Extending history tools](./tutorials/writing-history-tools.md) +- [Extending history tools](./tutorials/extending-history-tools.md) +- [What is "featured" content](./tutorials/what-is-featured-content.md) - [Description of configuration variables](./tutorials/config-vars.md) ## Contributing diff --git a/src/common/abis/abis.ts b/src/common/abis/abis.ts index 739fc51..ec3bd98 100644 --- a/src/common/abis/abis.ts +++ b/src/common/abis/abis.ts @@ -101,7 +101,7 @@ export class Abis { } /** - * Stores the ABI (Application Binary Interface) for the specified block number, contract address, and hex code. + * Stores the ABI (Application Binary Interface) with the specified block number, contract address, and hex code. * * @param {unknown} blockNumber - The block number. * @param {string} contract - The contract address. diff --git a/tutorials/what-is-featured-content.md b/tutorials/what-is-featured-content.md new file mode 100644 index 0000000..e69de29 From 1527f9b28f0cb96e562fcf3f7746840d1b37e689 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 3 Jul 2023 19:47:25 +0200 Subject: [PATCH 065/107] update readme --- README.md | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index af86235..1e2f63e 100644 --- a/README.md +++ b/README.md @@ -77,13 +77,11 @@ The Processor process retrieves tasks from the database, generating appropriate ## Common components -In addition to the processes that make up the base of History Tools, there are also components used between these processes. To a large extent, this repository contains high/domain level implementations of these components, and any code that directly depends on external libraries is placed in a separate repository such as the **Starter Kit**. common components consist of: - +In addition to the processes that make up the base of the History Tools package, there are also common components used between these processes. These components provide essential functionality and are shared among the processes. To a large extent, this repository contains high/domain level implementations of these components, and any code that directly depends on external libraries is placed in a separate repository such as the **Starter Kit**. common components consist of: ### Abis -`Abis` is basically a repository for storing contract ABIs listed as "featured" (e.g. in a json file). In addition, `Abis` also includes a service for downloading new ABI. `Abis` in the first place is used in **Bootstrap** process to download (using the service) all ABIs of listed contracts. The downloaded ABI is saved in the form of a document containing the following data: - +The Abis component serves as a repository for storing contract ABIs (Application Binary Interfaces). It includes a service for downloading new ABIs and retrieving existing ABIs. The Abis component is primarily used in the Bootstrap process to download ABIs for listed contracts. The downloaded ABIs are saved in the database and can be fetched when needed. ```javascript { @@ -104,7 +102,7 @@ We also use Addis in **Filter** when creating tasks for the processor. At that t ### BlockRangeScanner -We use `BlockRangeScanner` in the *reader* process in "replay" mode. When we want to download data from the blockchain in a specific range of blocks (these can be even years), we need a kind of data download schedule. `BlockRangeScanner` is used to create sets in a given range with a specific label (you can define the download purpose). The label is extremely important, it cannot be duplicated, it is a unique ID for the whole process. If an already taken key is used, the process will throw an appropriate error. `BlockRangeScanner` will divide the range into subgroups containing an equal number of blocks to be scanned (the last set may be smaller than the others). Replay mode, instead of directly actively listening to the blockchain, first downloads one of the scans from the list and passes it to the block reader, which in this given set will download data. If you use multiple threads or scale **reader** processes (e.g. in Docker), you can speed up the download process by making multiple queries to the scanner. The script will not give the same range to another thread if it is already handled. +The BlockRangeScanner is used in the Reader process during replay mode. It creates sets of blocks to be scanned within a specific range, allowing for organized data download. The BlockRangeScanner divides the range into subgroups containing an equal number of blocks to be scanned. This helps distribute the workload among multiple instances of the Reader process or workers. #### Methods @@ -116,7 +114,7 @@ We use `BlockRangeScanner` in the *reader* process in "replay" mode. When we wan ### BlockState -`BlockState` is a simple service for updating the current status of history tools. Various statistics are stored in the database, including the number of the currently read block. After reading a given block, the state is updated so that in the case of restarting history tools, it will start from the last processed block. +The BlockState component is a service for updating the current status of the History Tools. It stores various statistics in the database, including the number of the last read block. After reading a block, the BlockState is updated so that in case of restarting the History Tools, they can resume from the last processed block. #### Methods @@ -125,9 +123,9 @@ We use `BlockRangeScanner` in the *reader* process in "replay" mode. When we wan - `updateBlockNumber(blockNumber)`: Updates the block number value in the statistics. ### Featured -Featured is a class but also a category of tools used with a list of "featured" contracts, their actions and deltas. `FeaturedContracts` contained here is a repository that stores information in the database (contract name and block number in which it first appeared) of contracts that are included in the list (e.g. json file). `FeaturedContracts` is used in **bootstrap**, where after extracting contracts from the list, the previously mentioned data is downloaded. Another place where we use this (and other) `Featured` component is **Filter**. When reading the block data, you need to compare the contract name with the one included in the repository, and then, in order to determine the appropriate `ABI`, we check the block number in which it appeared for the first time. +The Featured component includes a repository for storing information about "featured" contracts. These contracts are included in a list and have certain criteria for choosing the right processor for their actions and deltas. The Featured component is used in the Bootstrap process to download data for the featured contracts, and it is also used in the Filter process to determine the appropriate ABI based on the block number when reading block data. -#### Methods +#### FeaturedContracts Methods - `readContracts(data)`: reads a json object or list of strings and retrieves the previously mentioned data. After they are downloaded from the web, they are stored in the database and cache. - `isFeatured(contract)`: Checks if the given name is on the list of blocks of interest. @@ -135,7 +133,7 @@ Featured is a class but also a category of tools used with a list of "featured" We build `Featured` class instances based on the data contained in the list of "featured" contracts (e.g. in the form of a JSON object). Each object in the list contains not only the names of the contracts, but also criteria on choosing the right processor for individual actions and contract deltas. For more information, see the [Tutorials](#tutorials) section. -#### Methods +#### Featured Methods - `getCriteria(criteria)`: Gets all match criteria that match the given criteria. - `getProcessor(label)`: Gets the processor for the given label and criteria. @@ -144,7 +142,8 @@ We build `Featured` class instances based on the data contained in the list of " ### ProcessorTaskQueue -As the name suggests, it is a queue (repository) of processor tasks. This list is generated by **Filter** process and then saved by this repository to the database. In the next process, which is **processor** (or several), it downloads the task, immediately removing it from the list and proceeds to work. Each of the tasks contains encrypted data that needs to be decoded using native blockchain deserialziers and having the data, you can start processing this data into the expected results. In case of failure task is sent to a separate list `unsuccessful_processor_tasks` for next attempt or analysis. +The ProcessorTaskQueue is a queue/repository of processor tasks. These tasks are generated by the Filter process and saved in the database by the ProcessorTaskQueue. The Processor process retrieves the tasks from the queue, removes them from the list, and processes them accordingly. Each task contains encrypted data that needs to be decoded using native blockchain deserializers. If a task fails, it is sent to a separate list `unsuccessful_processor_tasks` for subsequent attempts or analysis. + The task document diagram is as follows: ```typescript @@ -172,7 +171,7 @@ The task document diagram is as follows: ### UnprocessedBlockQueue -Just like in the case of Processor tasks, we also have a queue of blocks that for some reason have not been read. In order not to have gaps in the history, each unread block is placed in a special collection in the database so that later you can analyze the error, fix it and try to read the block again. Due to the fact that reading blocks is a fast process, we do not want to stress the database with a large number of requests, unread blocks are sent in batches. Failure to read a block in most cases will be caused by external conditions and will concern more blocks than just one. In order to avoid possible overloads, it is possible to set a limit on the number of unread blocks or their total size. This repository/queue has a bit more methods to customize the behavior of this queue in a custom way. +The UnprocessedBlockQueue is a queue/repository of blocks that have not been successfully read. It stores the blocks that failed to be processed in the Reader process. Each unread block is placed in the UnprocessedBlockQueue so that it can be analyzed, fixed, and reprocessed later. This helps ensure a complete and accurate history. The UnprocessedBlockQueue allows setting limits on the number of unread blocks or their total size to prevent overloading the system. #### Methods @@ -188,7 +187,7 @@ Just like in the case of Processor tasks, we also have a queue of blocks that fo ## Additional Tools -Besides the main processes and commons, this package also contains tools, such as: +The History Tools package also includes additional tools that can be helpful in various scenarios: ### Config From 4a0a13d0d9dc55b7d283dda94c6ed07b1bcd827e Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 4 Jul 2023 07:34:48 +0200 Subject: [PATCH 066/107] update UnprocessedBlockQueue --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1e2f63e..021fd8f 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ The task document diagram is as follows: ### UnprocessedBlockQueue -The UnprocessedBlockQueue is a queue/repository of blocks that have not been successfully read. It stores the blocks that failed to be processed in the Reader process. Each unread block is placed in the UnprocessedBlockQueue so that it can be analyzed, fixed, and reprocessed later. This helps ensure a complete and accurate history. The UnprocessedBlockQueue allows setting limits on the number of unread blocks or their total size to prevent overloading the system. +UnprocessedBlockQueue is a queue/repository of blocks to be read. We create these collections to speed up the process of reading and filtering blocks. Filtering takes more time than reading, so we separated the two processes, thus allowing you to set the resources appropriately for the fastest result. UnprocessedBlockQueue is used in the reader process to add more blocks to the list and filter to filter them and extract interesting data. The UnprocessedBlockQueue allows setting limits on the number of unread blocks or their total size to prevent overloading the system. #### Methods From cb7b8954ec4217588901d87fda82c0f6c30acbfb Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 4 Jul 2023 14:10:18 +0200 Subject: [PATCH 067/107] new tutorial --- README.md | 2 +- tutorials/what-is-featured-content.md | 128 ++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 021fd8f..812d500 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,7 @@ For tutorials on creating and using the history tools for your specific needs, s If you want to extend the capabilities of the history tools or take advantage of other third-party resources, please refer to the following tutorial. - [Extending history tools](./tutorials/extending-history-tools.md) -- [What is "featured" content](./tutorials/what-is-featured-content.md) +- [What is "featured" content?](./tutorials/what-is-featured-content.md) - [Description of configuration variables](./tutorials/config-vars.md) ## Contributing diff --git a/tutorials/what-is-featured-content.md b/tutorials/what-is-featured-content.md index e69de29..0e6c463 100644 --- a/tutorials/what-is-featured-content.md +++ b/tutorials/what-is-featured-content.md @@ -0,0 +1,128 @@ +# What is "featured" content? + +## Table of Contents + +1. [Introduction](#introduction) +2. [JSON object construction](#json-object-construction) +3. [Using featured content in the project](#using-featured-content-in-the-project) + - [Featured](#featured) + - [FeaturedContracts](#featuredcontracts) + +## Introduction + +The history tools work by reading from the blockchain and providing guidelines to extract and process data as per requirements. This data extraction is facilitated through the use of a JSON object containing two tables: traces and deltas. This JSON object sets the criteria defining what action or table we are looking for and the processor to use to process the data contained in this action or delta. + +Without this JSON configuration, only reading blocks will occur, and no data will be extracted from them. Thus, the correct configuration of this JSON object is crucial. + +## JSON object construction + +Below is an example of a JSON object configuration that we use in our history tools. It contains two tables: traces and deltas. + +```json +{ + "traces": [ + { + "shipTraceMessageName": ["transaction_trace_v0"], // Optional + "shipActionTraceMessageName": ["action_trace_v0", "action_trace_v1"], // Optional + "contract": ["dao.worlds"], + "action": ["*"], + "processor": "NotifyWorldTraceProcessor" + } + ], + "deltas": [ + { + "shipDeltaMessageName": ["table_delta_v0"], // Optional + "name": ["contract_row"], + "code": ["ref.worlds"], + "scope": ["*"], + "table": ["*"], + "processor": "RefWorldsDeltaProcessor" + } + ] +} +``` + +In traces, you need to provide `contract`, `action`, and `processor`. They are arrays because if you want to use the same processor for several different actions or contracts, there's no need to create a new object for each one. Instead, you can enter the name of the contract or action in the table. The symbol "\*" stands for "all possible" options. Avoid using the wildcard in the contract name tables (`contract` and `code`) as this might result in too much unnecessary data being downloaded. + +```json +// example of using one processor for all actions of the listed contracts +"contract": ["dao.worlds", "alien.worlds", "index.words"], +"action": ["*"], +"processor": "YourProcessorClassName", + +// example of using one processor for selected contract actions +"contract": ["dao.worlds"], +"action": ["appointcust", "flagcandprof", "newperiod"], +"processor": "YourProcessorClassName", +``` + +The same rules apply to `deltas` where the `name` will always be "contract_row", followed by `code`, `scope`, and `table`. + +In `traces` and `deltas`, optional keys: `shipTraceMessageName`, `shipActionTraceMessageName`, and `shipDeltaMessageName` are present. They are the names with versions of the structs containing data for transactions and deltas, respectively. If you use our **Starter Kit**, you don't need to provide these values in JSON because they will be added by featured scripts. However, remember about them if there's an update of one of the versions. Keep references to the previous ones (`v0` at the moment) to be able to read blocks generated before the API change. + +## Using featured content in the project + +### Featured + +An instance of the `Featured` class represents a set of criteria contained in the JSON configuration. Two instances are created, one for `traces` and the other for `deltas`. + +```typescript +const featuredTraces: Featured = new Featured( + featuredCriteria.traces, + { + shipTraceMessageName: [], + shipActionTraceMessageName: [], + contract: [], + action: [], + }, + { + shipTraceMessageName: ['transaction_trace_v0'], + shipActionTraceMessageName: ['action_trace_v0', 'action_trace_v1'], + } +); + +const featuredDeltas: Featured = new Featured( + featuredCriteria.deltas, + { + shipDeltaMessageName: [], + name: [], + code: [], + scope: [], + table: [], + }, + { shipDeltaMessageName: ['table_delta_v0'] } +); +``` + +The order of keys in the pattern is important. The processor tasks have `short_id` and `label` fields built from the values given in JSON objects. + +```json +"short_id" : "notify.world:logmine", +"label" : "transaction_trace_v0:action_trace_v1:notify.world:logmine", + +// corresponds to the values in the JSON object: +{ + "shipTraceMessageName": ["transaction_trace_v0"], + "shipActionTraceMessageName": ["action_trace_v1"], + "contract": ["notify.world"], + "action": ["logmine"], +} +``` + +The `label` string is composed of names separated by ':' and the order corresponds to that defined in the pattern for featured traces. + +_Note: The current solution may change as we are considering different alternatives._ + +### FeaturedContracts + +The `FeaturedContracts` is a repository containing the contract data you put in the JSON file. The `FeaturedUtils.readFeaturedContracts` function can be used to extract the names of all contracts and retrieve their data via `SmartContractService`. + +To create `FeaturedContracts`, you can use the `FeaturedContractsCreator` available in the [Starter Kit](https://github.com/Alien-Worlds/history-tools-starter-kit). If you want to use other sources and there is no suitable KIT for them, check the wizard implementation from the starter kit and implement it according to your needs. + +### FeaturedUtils + +FeaturedUtils includes some tools used inside other featured components or History Tools processes + +- `readFeaturedContracts(data)`: Extracts contract names from the provided json object +- `fetchCriteria(filePath)`: Gets the criteria json object from the given local file or URL + From fcc70286d4f2105eed8ab4bd136c1347ff314530 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Wed, 5 Jul 2023 16:58:04 +0200 Subject: [PATCH 068/107] update dependencies --- package.json | 7 +++---- src/index.ts | 4 ---- yarn.lock | 40 +++++++++++++++++++--------------------- 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/package.json b/package.json index 8b6939e..b54a15a 100644 --- a/package.json +++ b/package.json @@ -35,15 +35,14 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/api-core": "^0.0.151", - "@alien-worlds/block-reader": "^0.0.15", - "@alien-worlds/broadcast": "^0.0.10", + "@alien-worlds/api-core": "^0.0.155", + "@alien-worlds/block-reader": "^0.0.16", + "@alien-worlds/broadcast": "^0.0.11", "@alien-worlds/workers": "^0.0.7", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", "express": "^4.18.2", - "node-fetch": "2", "ts-node": "^10.9.1" } } diff --git a/src/index.ts b/src/index.ts index 9926e2a..5fed61b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,3 @@ -import nodeFetch from 'node-fetch'; - export * from './api'; export * from './bootstrap'; export * from './config'; @@ -13,5 +11,3 @@ export * from '@alien-worlds/api-core'; export * from '@alien-worlds/block-reader'; export * from '@alien-worlds/broadcast'; export * from '@alien-worlds/workers'; - -export const fetch = nodeFetch; \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index b05562f..6309900 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,31 +2,29 @@ # yarn lockfile v1 -"@alien-worlds/api-core@^0.0.151": - version "0.0.151" - resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.151/90f3fda4958c18a705ee34300f47a46f8a3e4f63#90f3fda4958c18a705ee34300f47a46f8a3e4f63" - integrity sha512-EkQjZ34F3srCD97b0TE+Lju05pod5gxPlUnezMV9Bq0R1xp0XLGnbSIqpO6Y2UfIJY7PHpFaHWpce+q2rQs4aA== +"@alien-worlds/api-core@^0.0.155": + version "0.0.155" + resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.155/33c08092422d3b41d3a6b215c06931d7eabd85bd#33c08092422d3b41d3a6b215c06931d7eabd85bd" + integrity sha512-pF0L5KDXzcaaYGNheIrgLg+y+GzINOOHxvxA9jGLyt7MrBr/adV0P016B9EK+BN9W2JVo+BCwsjAleji64mDPQ== dependencies: inversify "^6.0.1" - nanoid "^3.0.0" - node-fetch "2" + node-fetch "2.6.6" reflect-metadata "^0.1.13" -"@alien-worlds/block-reader@^0.0.15": - version "0.0.15" - resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.15/08fdc9297f958981720da64da461832f603f76e6#08fdc9297f958981720da64da461832f603f76e6" - integrity sha512-8lYnOTx1pnvwNkrKafsYCBWOqwpt8RVIoc6w1L9zUqg2o9qWy9eI4fkVlCvamKPpsaNYKGgMeVG14lH8M8P/IQ== +"@alien-worlds/block-reader@^0.0.16": + version "0.0.16" + resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.16/0cf464bf819e52d18184359db6ba51f77e78c4ca#0cf464bf819e52d18184359db6ba51f77e78c4ca" + integrity sha512-9KSPCOEtkJwNzV0/z0wNRoaMVcQSSyPPXGxlS8dMdopd+USN9lg43j+Dj6jEI2AaWFC3Zqk4uuICOaoUFN05aA== dependencies: - "@alien-worlds/api-core" "^0.0.151" + "@alien-worlds/api-core" "^0.0.155" -"@alien-worlds/broadcast@^0.0.10": - version "0.0.10" - resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.10/978171234ce921b91f7951a4a113d56aba9c1f00#978171234ce921b91f7951a4a113d56aba9c1f00" - integrity sha512-ILKkn197zxilBKCml2USIyggYNAHzhojfnic+Rj3fmX0nA0Uzz8hHOHj/IOaGTeP8neaGt1dx8sQCPciNY6xZw== +"@alien-worlds/broadcast@^0.0.11": + version "0.0.11" + resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.11/ca88eb95dfa89b5351b71c3c95fed417dcf6cf0c#ca88eb95dfa89b5351b71c3c95fed417dcf6cf0c" + integrity sha512-qFM7Kyqv3uUVPgfRIcai+IAFty5VsTOndq++J5K7GDPlV1GIKryqmBSpBghReNDtGbFF/bTfU6XxZmyQdfqg9Q== dependencies: - "@alien-worlds/api-core" "^0.0.151" + "@alien-worlds/api-core" "^0.0.155" nanoid "^3.0.0" - node-fetch "2" "@alien-worlds/workers@^0.0.7": version "0.0.7" @@ -2856,10 +2854,10 @@ negotiator@0.6.3: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== -node-fetch@2: - version "2.6.12" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.12.tgz#02eb8e22074018e3d5a83016649d04df0e348fba" - integrity sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g== +node-fetch@2.6.6: + version "2.6.6" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.6.tgz#1751a7c01834e8e1697758732e9efb6eeadfaf89" + integrity sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA== dependencies: whatwg-url "^5.0.0" From c5e275c616b05d025858e18554f93bd84bd68509 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Wed, 5 Jul 2023 16:58:23 +0200 Subject: [PATCH 069/107] 0.0.142 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b54a15a..6784505 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.141", + "version": "0.0.142", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 83bdb43811ac2c41221ccedd9a19facca79187dd Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 7 Jul 2023 09:45:35 +0200 Subject: [PATCH 070/107] updates, modified processor tasks --- package.json | 6 +- src/processor/processor.dependencies.ts | 3 +- src/processor/processor.model.factory.ts | 83 +++++++++++++++++++ src/processor/processor.runner.ts | 16 +++- src/processor/processor.types.ts | 20 ++--- .../processor.worker-loader.dependencies.ts | 2 - src/processor/processor.worker-loader.ts | 3 +- .../processors/action-trace.processor.ts | 50 +---------- src/processor/processors/delta.processor.ts | 41 +-------- src/processor/processors/processor.ts | 4 +- src/processor/start-processor.ts | 5 +- yarn.lock | 28 +++---- 12 files changed, 137 insertions(+), 124 deletions(-) create mode 100644 src/processor/processor.model.factory.ts diff --git a/package.json b/package.json index 6784505..a0b1fe5 100644 --- a/package.json +++ b/package.json @@ -35,9 +35,9 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/api-core": "^0.0.155", - "@alien-worlds/block-reader": "^0.0.16", - "@alien-worlds/broadcast": "^0.0.11", + "@alien-worlds/api-core": "^0.0.156", + "@alien-worlds/block-reader": "^0.0.17", + "@alien-worlds/broadcast": "^0.0.12", "@alien-worlds/workers": "^0.0.7", "async": "^3.2.4", "commander": "^10.0.1", diff --git a/src/processor/processor.dependencies.ts b/src/processor/processor.dependencies.ts index fe1754e..e0df96d 100644 --- a/src/processor/processor.dependencies.ts +++ b/src/processor/processor.dependencies.ts @@ -1,4 +1,4 @@ -import { Result } from '@alien-worlds/api-core'; +import { Result, Serializer } from '@alien-worlds/api-core'; import { Dependencies } from '../common/dependencies'; import { ContractDeltaMatchCriteria, @@ -18,6 +18,7 @@ export abstract class ProcessorDependencies extends Dependencies { public workerLoaderPath?: string; public workerLoaderDependenciesPath: string; public broadcastClient: BroadcastClient; + public serializer: Serializer; public featuredTraces: Featured; public featuredDeltas: Featured; public processorTaskQueue: ProcessorTaskQueue; diff --git a/src/processor/processor.model.factory.ts b/src/processor/processor.model.factory.ts new file mode 100644 index 0000000..76f845e --- /dev/null +++ b/src/processor/processor.model.factory.ts @@ -0,0 +1,83 @@ +import { Serializer } from '@alien-worlds/api-core'; +import { + ActionProcessorContentModel, + DeltaProcessorContentModel, + ProcessorTask, + ProcessorTaskType, + UnknownProcessorTypeError, +} from '../common'; +import { ActionTraceProcessorModel, DeltaProcessorModel } from './processor.types'; +import { deserialize } from 'v8'; + +export class ProcessorModelFactory { + constructor(protected serializer: Serializer) {} + + protected buildActionTraceProcessorModel( + model: ProcessorTask + ): ActionTraceProcessorModel { + const { serializer } = this; + const { abi, content: buffer } = model; + const content: ActionProcessorContentModel = deserialize(buffer); + const { + action_trace: { act, receipt }, + block_num, + block_timestamp, + transaction_id, + } = content; + + const [receiptType, receiptContent] = receipt; + const { global_sequence, recv_sequence } = receiptContent; + const data = serializer.deserializeActionData( + act.account, + act.name, + act.data, + abi + ); + + return { + block_number: block_num.toString(), + block_timestamp, + transaction_id, + account: act.account, + name: act.name, + recv_sequence, + global_sequence, + data: data as DataType, + }; + } + + protected buildDeltaProcessorModel( + model: ProcessorTask + ): DeltaProcessorModel { + const { serializer } = this; + const { abi, content: buffer } = model; + const delta: DeltaProcessorContentModel = deserialize(buffer); + const { name, block_num, block_timestamp } = delta; + const row = serializer.deserializeTableRow(delta.row_data, abi); + const { code, scope, table, primary_key, payer, data } = row; + + return { + name, + block_number: block_num.toString(), + block_timestamp, + code, + scope, + table, + primary_key, + payer, + data: data as DataType, + }; + } + + public create( + task: ProcessorTask + ): ActionTraceProcessorModel | DeltaProcessorModel { + if (task.type === ProcessorTaskType.Trace) { + return this.buildActionTraceProcessorModel(task); + } else if (task.type === ProcessorTaskType.Delta) { + return this.buildDeltaProcessorModel(task); + } else { + throw new UnknownProcessorTypeError(task.type); + } + } +} diff --git a/src/processor/processor.runner.ts b/src/processor/processor.runner.ts index a7a1ba2..9f375de 100644 --- a/src/processor/processor.runner.ts +++ b/src/processor/processor.runner.ts @@ -1,4 +1,4 @@ -import { log } from '@alien-worlds/api-core'; +import { Serializer, log } from '@alien-worlds/api-core'; import { WorkerPool, WorkerMessage } from '@alien-worlds/workers'; import { Featured, @@ -10,17 +10,22 @@ import { UnknownProcessorTypeError, ProcessorTaskModel, } from '../common'; +import { ProcessorModelFactory } from './processor.model.factory'; export class ProcessorRunner { private interval: NodeJS.Timeout; private loop: boolean; - + private modelFactory: ProcessorModelFactory; constructor( protected featuredTraces: Featured, protected featuredDeltas: Featured, protected workerPool: WorkerPool, - protected queue: ProcessorTaskQueue + protected queue: ProcessorTaskQueue, + serializer: Serializer, ) { + + this.modelFactory = new ProcessorModelFactory(serializer); + this.interval = setInterval(async () => { if (this.workerPool.hasActiveWorkers() === false) { log(`All workers are available, checking if there is something to do...`); @@ -77,8 +82,11 @@ export class ProcessorRunner { queue.stashUnsuccessfulTask(task, error); workerPool.releaseWorker(id); }); + + const model = this.modelFactory.create(task); + // start worker - worker.run(task); + worker.run(model); log(`Worker #${worker.id} has been assigned to process task ${task.id}`); } else { await this.queue.addTasks([task]); diff --git a/src/processor/processor.types.ts b/src/processor/processor.types.ts index 9d89829..cc21955 100644 --- a/src/processor/processor.types.ts +++ b/src/processor/processor.types.ts @@ -9,25 +9,25 @@ export type ProcessorSharedData = { processorsPath: string; }; -export type DeltaProcessorInput = { +export type DeltaProcessorModel = { name: string; code: string; scope: string; table: string; payer: string; - primaryKey: bigint; - blockNumber: bigint; - blockTimestamp: Date; + primary_key: string; + block_number: string; + block_timestamp: Date; data: DataType; }; -export type ActionTraceProcessorInput = { - blockNumber: bigint; - blockTimestamp: Date; - transactionId: string; +export type ActionTraceProcessorModel = { account: string; name: string; - recvSequence: bigint; - globalSequence: bigint; + block_timestamp: Date; + block_number: string; + global_sequence: string; + recv_sequence: string; + transaction_id: string; data: DataType; }; diff --git a/src/processor/processor.worker-loader.dependencies.ts b/src/processor/processor.worker-loader.dependencies.ts index 8b77f90..36ef7ac 100644 --- a/src/processor/processor.worker-loader.dependencies.ts +++ b/src/processor/processor.worker-loader.dependencies.ts @@ -1,4 +1,3 @@ -import { Serializer } from '@alien-worlds/api-core'; import { WorkerLoaderDependencies } from '@alien-worlds/workers'; import { ProcessorConfig } from './processor.config'; @@ -8,7 +7,6 @@ import { ProcessorConfig } from './processor.config'; */ export abstract class ProcessorWorkerLoaderDependencies extends WorkerLoaderDependencies { public dataSource: unknown; - public serializer: Serializer; public processorsPath: string; public abstract initialize( diff --git a/src/processor/processor.worker-loader.ts b/src/processor/processor.worker-loader.ts index a04190d..4e54324 100644 --- a/src/processor/processor.worker-loader.ts +++ b/src/processor/processor.worker-loader.ts @@ -18,7 +18,7 @@ export default class ProcessorWorkerLoader extends DefaultWorkerLoader< public async load(pointer: string): Promise { const { - dependencies: { dataSource, serializer, processorsPath }, + dependencies: { dataSource, processorsPath }, } = this; const { ioc, sharedData } = this; const processorClasses = await import(processorsPath); @@ -26,7 +26,6 @@ export default class ProcessorWorkerLoader extends DefaultWorkerLoader< { ioc, dataSource, - serializer, }, sharedData ) as Worker; diff --git a/src/processor/processors/action-trace.processor.ts b/src/processor/processors/action-trace.processor.ts index f49374c..0dc836f 100644 --- a/src/processor/processors/action-trace.processor.ts +++ b/src/processor/processors/action-trace.processor.ts @@ -1,16 +1,12 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { Processor } from './processor'; -import { ActionTraceProcessorInput, ProcessorSharedData } from '../processor.types'; -import { Container, Serializer, parseToBigInt } from '@alien-worlds/api-core'; -import { ProcessorTaskModel, ActionProcessorContentModel } from '../../common'; -import { deserialize } from 'v8'; +import { ActionTraceProcessorModel, ProcessorSharedData } from '../processor.types'; +import { Container, Serializer } from '@alien-worlds/api-core'; export class ActionTraceProcessor< DataType = unknown, SharedDataType = ProcessorSharedData -> extends Processor { - protected input: ActionTraceProcessorInput; - +> extends Processor, SharedDataType> { constructor( protected dependencies: { ioc: Container; @@ -21,44 +17,4 @@ export class ActionTraceProcessor< ) { super(); } - - public deserializeModelContent( - model: ProcessorTaskModel - ): ActionTraceProcessorInput { - const { - dependencies: { serializer }, - } = this; - const { abi, content: buffer } = model; - const content: ActionProcessorContentModel = deserialize(buffer); - const { - action_trace: { act, receipt }, - block_num, - block_timestamp, - transaction_id, - } = content; - - const [receiptType, receiptContent] = receipt; - const { global_sequence, recv_sequence } = receiptContent; - const data = serializer.deserializeActionData( - act.account, - act.name, - act.data, - abi - ); - - return { - blockNumber: parseToBigInt(block_num), - blockTimestamp: block_timestamp, - transactionId: transaction_id, - account: act.account, - name: act.name, - recvSequence: parseToBigInt(recv_sequence), - globalSequence: parseToBigInt(global_sequence), - data: data as DataType, - }; - } - - public async run(model: ProcessorTaskModel): Promise { - this.input = this.deserializeModelContent(model); - } } diff --git a/src/processor/processors/delta.processor.ts b/src/processor/processors/delta.processor.ts index 8f1a046..78b1a1f 100644 --- a/src/processor/processors/delta.processor.ts +++ b/src/processor/processors/delta.processor.ts @@ -1,15 +1,11 @@ import { Processor } from './processor'; -import { DeltaProcessorInput, ProcessorSharedData } from '../processor.types'; -import { deserialize } from 'v8'; -import { Container, Serializer, parseToBigInt } from '@alien-worlds/api-core'; -import { ProcessorTaskModel, DeltaProcessorContentModel } from '../../common'; +import { DeltaProcessorModel, ProcessorSharedData } from '../processor.types'; +import { Container, Serializer } from '@alien-worlds/api-core'; export class DeltaProcessor< - DataType, + DataType = unknown, SharedDataType = ProcessorSharedData -> extends Processor { - protected input: DeltaProcessorInput; - +> extends Processor, SharedDataType> { constructor( protected dependencies: { ioc: Container; @@ -20,33 +16,4 @@ export class DeltaProcessor< ) { super(); } - - public deserializeModelContent( - model: ProcessorTaskModel - ): DeltaProcessorInput { - const { - dependencies: { serializer }, - } = this; - const { abi, content: buffer } = model; - const delta: DeltaProcessorContentModel = deserialize(buffer); - const { name, block_num, block_timestamp } = delta; - const row = serializer.deserializeTableRow(delta.row_data, abi); - const { code, scope, table, primary_key, payer, data } = row; - - return { - name, - blockNumber: block_num, - blockTimestamp: block_timestamp, - code, - scope, - table, - primaryKey: parseToBigInt(primary_key), - payer, - data: data as DataType, - }; - } - - public async run(model: ProcessorTaskModel): Promise { - this.input = this.deserializeModelContent(model); - } } diff --git a/src/processor/processors/processor.ts b/src/processor/processors/processor.ts index cca476c..d232964 100644 --- a/src/processor/processors/processor.ts +++ b/src/processor/processors/processor.ts @@ -1,12 +1,12 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { Worker } from '@alien-worlds/workers'; -import { ProcessorTaskModel } from '../../common'; import { ProcessorSharedData } from '../processor.types'; export class Processor< + ModelType, SharedDataType = ProcessorSharedData > extends Worker { - public run(data: ProcessorTaskModel): Promise { + public run(model: ModelType): Promise { throw new Error('Method not implemented'); } } diff --git a/src/processor/start-processor.ts b/src/processor/start-processor.ts index 7d1e44b..5e39f70 100644 --- a/src/processor/start-processor.ts +++ b/src/processor/start-processor.ts @@ -40,6 +40,7 @@ export const process = async ( featuredDeltas, processorTaskQueue, workerLoaderDependenciesPath, + serializer, } = dependencies; const workerPool = await WorkerPool.create({ ...config.workers, @@ -47,12 +48,12 @@ export const process = async ( workerLoaderPath: config.processorLoaderPath || processorWorkerLoaderPath, workerLoaderDependenciesPath, }); - const runner = new ProcessorRunner( featuredTraces, featuredDeltas, workerPool, - processorTaskQueue + processorTaskQueue, + serializer ); broadcastClient.onMessage( diff --git a/yarn.lock b/yarn.lock index 6309900..f0eb4d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,28 +2,28 @@ # yarn lockfile v1 -"@alien-worlds/api-core@^0.0.155": - version "0.0.155" - resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.155/33c08092422d3b41d3a6b215c06931d7eabd85bd#33c08092422d3b41d3a6b215c06931d7eabd85bd" - integrity sha512-pF0L5KDXzcaaYGNheIrgLg+y+GzINOOHxvxA9jGLyt7MrBr/adV0P016B9EK+BN9W2JVo+BCwsjAleji64mDPQ== +"@alien-worlds/api-core@^0.0.156": + version "0.0.156" + resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.156/16a4a27aadeaef27b4c0dbac0785aed05c834d68#16a4a27aadeaef27b4c0dbac0785aed05c834d68" + integrity sha512-n5aOkhuAqEemTUkBPj7qZneRKOlkhtb3W1ckVDHUMk4U5s1g2SPlrPExAxdb0k3JwtzlxCqxEssnXdmwqlRmEg== dependencies: inversify "^6.0.1" node-fetch "2.6.6" reflect-metadata "^0.1.13" -"@alien-worlds/block-reader@^0.0.16": - version "0.0.16" - resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.16/0cf464bf819e52d18184359db6ba51f77e78c4ca#0cf464bf819e52d18184359db6ba51f77e78c4ca" - integrity sha512-9KSPCOEtkJwNzV0/z0wNRoaMVcQSSyPPXGxlS8dMdopd+USN9lg43j+Dj6jEI2AaWFC3Zqk4uuICOaoUFN05aA== +"@alien-worlds/block-reader@^0.0.17": + version "0.0.17" + resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.17/974db83f76884b54aeddd20ef14a188065690c38#974db83f76884b54aeddd20ef14a188065690c38" + integrity sha512-yL+B+wZ9Q2mTtWGmHHxVTtQQZjgUMAV5SoGp+MJIRFE1bH9spPFLvtBKUqfstXgMe9OjMxE4OPxs62CoQFL6Cg== dependencies: - "@alien-worlds/api-core" "^0.0.155" + "@alien-worlds/api-core" "^0.0.156" -"@alien-worlds/broadcast@^0.0.11": - version "0.0.11" - resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.11/ca88eb95dfa89b5351b71c3c95fed417dcf6cf0c#ca88eb95dfa89b5351b71c3c95fed417dcf6cf0c" - integrity sha512-qFM7Kyqv3uUVPgfRIcai+IAFty5VsTOndq++J5K7GDPlV1GIKryqmBSpBghReNDtGbFF/bTfU6XxZmyQdfqg9Q== +"@alien-worlds/broadcast@^0.0.12": + version "0.0.12" + resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.12/b65bd1fe668fb16757ccfdc3d4f23aef8f0d3976#b65bd1fe668fb16757ccfdc3d4f23aef8f0d3976" + integrity sha512-M57UtgEUgHf9hj95/3osewuLpWJHldjA8vIFjRxvTImniNCaUZaWni2fhZOUtJ025kJ5DeN+3ZC1KrQBV2nC+w== dependencies: - "@alien-worlds/api-core" "^0.0.155" + "@alien-worlds/api-core" "^0.0.156" nanoid "^3.0.0" "@alien-worlds/workers@^0.0.7": From 7bea9a263945c7dc7cecc030a4cc7d5556e955f8 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 7 Jul 2023 09:46:06 +0200 Subject: [PATCH 071/107] 0.0.143 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a0b1fe5..9cbd88d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.142", + "version": "0.0.143", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From e609cd1f70105259f92affa4b653bc091d0efc94 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 7 Jul 2023 12:17:01 +0200 Subject: [PATCH 072/107] add present --- package.json | 6 ++-- .../processor-task-queue/processor-task.ts | 4 ++- .../processor-task.types.ts | 3 +- src/common/types/delta.types.ts | 7 ++--- src/filter/filter.worker.ts | 7 +++-- src/processor/processor.model.factory.ts | 5 ++-- src/processor/processor.types.ts | 1 + yarn.lock | 28 +++++++++---------- 8 files changed, 32 insertions(+), 29 deletions(-) diff --git a/package.json b/package.json index 9cbd88d..024ff0a 100644 --- a/package.json +++ b/package.json @@ -35,9 +35,9 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/api-core": "^0.0.156", - "@alien-worlds/block-reader": "^0.0.17", - "@alien-worlds/broadcast": "^0.0.12", + "@alien-worlds/api-core": "^0.0.158", + "@alien-worlds/block-reader": "^0.0.18", + "@alien-worlds/broadcast": "^0.0.13", "@alien-worlds/workers": "^0.0.7", "async": "^3.2.4", "commander": "^10.0.1", diff --git a/src/common/processor-task-queue/processor-task.ts b/src/common/processor-task-queue/processor-task.ts index 87c0f5d..d38ef1a 100644 --- a/src/common/processor-task-queue/processor-task.ts +++ b/src/common/processor-task-queue/processor-task.ts @@ -65,6 +65,7 @@ export class ProcessorTask { code: string, scope: string, table: string, + present: boolean, blockNumber: bigint, blockTimestamp: Date, data: Uint8Array, @@ -73,7 +74,8 @@ export class ProcessorTask { const content: DeltaProcessorContentModel = { ship_delta_message_name: type, name, - row_data: data, + present, + data: data, block_num: blockNumber, block_timestamp: blockTimestamp, }; diff --git a/src/common/processor-task-queue/processor-task.types.ts b/src/common/processor-task-queue/processor-task.types.ts index fac5586..b78da2d 100644 --- a/src/common/processor-task-queue/processor-task.types.ts +++ b/src/common/processor-task-queue/processor-task.types.ts @@ -24,7 +24,8 @@ export type DeltaProcessorContentModel = { name: string; block_num: bigint; block_timestamp: Date; - row_data: Uint8Array; + present: boolean; + data: Uint8Array; }; export type ActionProcessorContentModel = { diff --git a/src/common/types/delta.types.ts b/src/common/types/delta.types.ts index 1beb87c..0eeb36a 100644 --- a/src/common/types/delta.types.ts +++ b/src/common/types/delta.types.ts @@ -1,11 +1,8 @@ -export type DeltaRow = { - present?: number; - data?: Uint8Array; -}; +import { Row } from "@alien-worlds/api-core"; export type Delta = { name?: string; - rows?: DeltaRow[]; + rows?: Row[]; }; export type DeltaByName = [string, Delta]; diff --git a/src/filter/filter.worker.ts b/src/filter/filter.worker.ts index d84bdf5..0c73535 100644 --- a/src/filter/filter.worker.ts +++ b/src/filter/filter.worker.ts @@ -12,7 +12,7 @@ import { TraceByName, isSetAbiAction, } from '../common'; -import { Serializer, log, parseToBigInt } from '@alien-worlds/api-core'; +import { Row, Serializer, log, parseToBigInt } from '@alien-worlds/api-core'; export default class FilterWorker extends Worker { constructor( @@ -131,7 +131,7 @@ export default class FilterWorker extends Worker { for (const [type, delta] of deltas) { const { name, rows } = delta; const tableRows = rows - ? rows.map(row => serializer.deserializeTableRow(row.data)) + ? rows.map(row => serializer.deserializeTableRow(row)) : []; for (let i = 0; i < tableRows.length; i++) { @@ -141,7 +141,7 @@ export default class FilterWorker extends Worker { // The contract may not contain tables or may be corrupted continue; } - const { table, code, scope } = tableRow; + const { table, code, scope, present } = tableRow; if (featuredContracts.isFeatured(code)) { try { // If the block in which the contract was created cannot be found or @@ -187,6 +187,7 @@ export default class FilterWorker extends Worker { code, scope, table, + present, parseToBigInt(this_block.block_num), new Date(timestamp), tableRow.data as Uint8Array, diff --git a/src/processor/processor.model.factory.ts b/src/processor/processor.model.factory.ts index 76f845e..46ab876 100644 --- a/src/processor/processor.model.factory.ts +++ b/src/processor/processor.model.factory.ts @@ -53,8 +53,8 @@ export class ProcessorModelFactory { const { abi, content: buffer } = model; const delta: DeltaProcessorContentModel = deserialize(buffer); const { name, block_num, block_timestamp } = delta; - const row = serializer.deserializeTableRow(delta.row_data, abi); - const { code, scope, table, primary_key, payer, data } = row; + const row = serializer.deserializeTableRow(delta, abi); + const { code, scope, table, primary_key, payer, data, present } = row; return { name, @@ -63,6 +63,7 @@ export class ProcessorModelFactory { code, scope, table, + present, primary_key, payer, data: data as DataType, diff --git a/src/processor/processor.types.ts b/src/processor/processor.types.ts index cc21955..b59d61f 100644 --- a/src/processor/processor.types.ts +++ b/src/processor/processor.types.ts @@ -15,6 +15,7 @@ export type DeltaProcessorModel = { scope: string; table: string; payer: string; + present: boolean; primary_key: string; block_number: string; block_timestamp: Date; diff --git a/yarn.lock b/yarn.lock index f0eb4d8..c129019 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,28 +2,28 @@ # yarn lockfile v1 -"@alien-worlds/api-core@^0.0.156": - version "0.0.156" - resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.156/16a4a27aadeaef27b4c0dbac0785aed05c834d68#16a4a27aadeaef27b4c0dbac0785aed05c834d68" - integrity sha512-n5aOkhuAqEemTUkBPj7qZneRKOlkhtb3W1ckVDHUMk4U5s1g2SPlrPExAxdb0k3JwtzlxCqxEssnXdmwqlRmEg== +"@alien-worlds/api-core@^0.0.158": + version "0.0.158" + resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.158/92cb2bb3a2168e07fa7a8e240895f5133105e0ee#92cb2bb3a2168e07fa7a8e240895f5133105e0ee" + integrity sha512-PIrmouUmomBUTZ1tkNKzU/0cHcvY9L/puAYljvnDlwZ5yAJECgbgVuxEqlQVbMne2kXQwW0c1U7GCfP877Zt6w== dependencies: inversify "^6.0.1" node-fetch "2.6.6" reflect-metadata "^0.1.13" -"@alien-worlds/block-reader@^0.0.17": - version "0.0.17" - resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.17/974db83f76884b54aeddd20ef14a188065690c38#974db83f76884b54aeddd20ef14a188065690c38" - integrity sha512-yL+B+wZ9Q2mTtWGmHHxVTtQQZjgUMAV5SoGp+MJIRFE1bH9spPFLvtBKUqfstXgMe9OjMxE4OPxs62CoQFL6Cg== +"@alien-worlds/block-reader@^0.0.18": + version "0.0.18" + resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.18/f1e0fcb9fde2d25cc2864c5080605bd11daf32e2#f1e0fcb9fde2d25cc2864c5080605bd11daf32e2" + integrity sha512-h8bCBqX8kX4FInyXB9PWHoZf+8urB7/8LsbnOVTjPoAYWdb1uRSV+VPIviGot1xgyV71HLv9ELpweCMqitOIug== dependencies: - "@alien-worlds/api-core" "^0.0.156" + "@alien-worlds/api-core" "^0.0.158" -"@alien-worlds/broadcast@^0.0.12": - version "0.0.12" - resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.12/b65bd1fe668fb16757ccfdc3d4f23aef8f0d3976#b65bd1fe668fb16757ccfdc3d4f23aef8f0d3976" - integrity sha512-M57UtgEUgHf9hj95/3osewuLpWJHldjA8vIFjRxvTImniNCaUZaWni2fhZOUtJ025kJ5DeN+3ZC1KrQBV2nC+w== +"@alien-worlds/broadcast@^0.0.13": + version "0.0.13" + resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.13/03327b557e880e8848a8114f0981d1398c434d71#03327b557e880e8848a8114f0981d1398c434d71" + integrity sha512-HPrvX5OZ/hYyGn/EtmMITLASH4J0OnwuBDoWaD/TYUAvUJ5ZnJutbDDEIekmzK1SekXuOHD+vKTNfOhP8sK8kg== dependencies: - "@alien-worlds/api-core" "^0.0.156" + "@alien-worlds/api-core" "^0.0.158" nanoid "^3.0.0" "@alien-worlds/workers@^0.0.7": From d33ef0f3b9b1a5ad61bd7fd86b71347d7904a903 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 7 Jul 2023 12:17:21 +0200 Subject: [PATCH 073/107] 0.0.144 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 024ff0a..c635ddf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.143", + "version": "0.0.144", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 9546710e02ce397841c8b35aa9121e135b358c6c Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 14 Jul 2023 10:04:47 +0200 Subject: [PATCH 074/107] update tutorial processors part --- tutorials/extending-history-tools.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tutorials/extending-history-tools.md b/tutorials/extending-history-tools.md index c33b760..e73902d 100644 --- a/tutorials/extending-history-tools.md +++ b/tutorials/extending-history-tools.md @@ -260,17 +260,27 @@ startProcessor( ## Step 5: Create necessary processors -Finally, create a `processors` folder where you will store all of the processor files. The contents of this folder should be exported in `index.ts`, and each processor should be exported as a default. +Finally, create a `processors` folder where you will store all of the processor files. +To ensure that `ProcessorWorkerLoader` will create insance of the processor, you have to make sure that processor classes are **exported individually** in the 'processors' folder and not as default exports, follow these instructions: + +1. Create the 'processors' folder in your processor directory if it doesn't exist already. + +2. Inside the 'processors' folder, create a separate TypeScript file for each class you want to export. For example, Processor1.ts, Processor2.ts, etc. + +3. In each class file (e.g., Processor1.ts, Processor2.ts, etc.), define your classes using the `export` keyword. Each class should have its own file. + +Example: ```typescript // ./processors/index.ts -export * from './notify-world.trace-processor'; +export { NotifyWorldTraceProcessor } from './notify-world.trace-processor'; + ... // ./processors/notify-world.trace-processor.ts import { ActionTraceProcessor, ProcessorTaskModel } from '@alien-worlds/history-tools-starter-kit'; -export default class NotifyWorldTraceProcessor extends ActionTraceProcessor { +export class NotifyWorldTraceProcessor extends ActionTraceProcessor { public async run(model: ProcessorTaskModel): Promise { try { if (name === NotifyWorldActionName.Logmine) { From 93c1bf0e6541af8dcc5e9599ef7256c38958721f Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 14 Jul 2023 17:48:51 +0200 Subject: [PATCH 075/107] bugfix delta processor bytes --- .../processor-task-queue/processor-task.ts | 7 ++++--- src/filter/filter.worker.ts | 18 +++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/common/processor-task-queue/processor-task.ts b/src/common/processor-task-queue/processor-task.ts index d38ef1a..8ef2b4d 100644 --- a/src/common/processor-task-queue/processor-task.ts +++ b/src/common/processor-task-queue/processor-task.ts @@ -3,6 +3,7 @@ import { serialize } from 'v8'; import { DeltaProcessorContentModel, ProcessorTaskError } from './processor-task.types'; import { ActionTrace } from '../types'; import { ProcessorTaskType } from './processor-task.enums'; +import { Row } from '@alien-worlds/api-core'; export class ProcessorTask { public static createActionProcessorTask( @@ -65,17 +66,17 @@ export class ProcessorTask { code: string, scope: string, table: string, - present: boolean, blockNumber: bigint, blockTimestamp: Date, - data: Uint8Array, + row: Row, isFork: boolean ) { + const { present, data } = row; const content: DeltaProcessorContentModel = { ship_delta_message_name: type, name, present, - data: data, + data, block_num: blockNumber, block_timestamp: blockTimestamp, }; diff --git a/src/filter/filter.worker.ts b/src/filter/filter.worker.ts index 0c73535..36ca7ff 100644 --- a/src/filter/filter.worker.ts +++ b/src/filter/filter.worker.ts @@ -12,7 +12,7 @@ import { TraceByName, isSetAbiAction, } from '../common'; -import { Row, Serializer, log, parseToBigInt } from '@alien-worlds/api-core'; +import { Serializer, log, parseToBigInt } from '@alien-worlds/api-core'; export default class FilterWorker extends Worker { constructor( @@ -130,18 +130,15 @@ export default class FilterWorker extends Worker { for (const [type, delta] of deltas) { const { name, rows } = delta; - const tableRows = rows - ? rows.map(row => serializer.deserializeTableRow(row)) - : []; + + for (const row of rows) { + const info = serializer.deserializeTableRow(row); - for (let i = 0; i < tableRows.length; i++) { - const tableRow = tableRows[i]; - - if (!tableRow) { + if (!info) { // The contract may not contain tables or may be corrupted continue; } - const { table, code, scope, present } = tableRow; + const { table, code, scope } = info; if (featuredContracts.isFeatured(code)) { try { // If the block in which the contract was created cannot be found or @@ -187,10 +184,9 @@ export default class FilterWorker extends Worker { code, scope, table, - present, parseToBigInt(this_block.block_num), new Date(timestamp), - tableRow.data as Uint8Array, + row, parseToBigInt(this_block.block_num) <= parseToBigInt(prev_block.block_num) ) ); From 490123c0e3d49ff7ffa3a20bb696292d892d42b2 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 14 Jul 2023 17:49:13 +0200 Subject: [PATCH 076/107] 0.0.145 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c635ddf..736b44b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.144", + "version": "0.0.145", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From f0b926befff2ea99e8bcc54731a3711a4c153234 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 18 Jul 2023 15:53:35 +0200 Subject: [PATCH 077/107] fix reader under the batch size, skip duplicate errors in abis --- src/common/abis/abis.repository-impl.ts | 5 ++++- .../unprocessed-block-queue.ts | 21 +++++++++++-------- src/reader/reader.worker.ts | 16 ++++++++------ 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/common/abis/abis.repository-impl.ts b/src/common/abis/abis.repository-impl.ts index 4d12bae..f18b494 100644 --- a/src/common/abis/abis.repository-impl.ts +++ b/src/common/abis/abis.repository-impl.ts @@ -1,6 +1,7 @@ import { ContractEncodedAbi, CountParams, + DataSourceError, FindParams, log, RepositoryImpl, @@ -112,7 +113,9 @@ export class AbisRepositoryImpl const { content, failure } = await this.add(abis); if (failure) { - log(failure.error); + if ((failure.error).isDuplicateError === false) { + log(failure.error); + } return Result.withContent(false); } diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.ts index 9055cbb..60b9509 100644 --- a/src/common/unprocessed-block-queue/unprocessed-block-queue.ts +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.ts @@ -73,21 +73,24 @@ export class UnprocessedBlockQueue public async add( block: Block, - isFastLane = false, - isLast = false + options?: { isFastLane?: boolean; isLast?: boolean; predictedRangeSize?: number } ): Promise> { + const { isFastLane, isLast, predictedRangeSize } = options || {}; try { let addedBlockNumbers: bigint[] = []; - - if (this.cache.length < this.batchSize) { + const currentBatchSize = isFastLane + ? predictedRangeSize < this.fastLaneBatchSize + ? predictedRangeSize + : this.fastLaneBatchSize + : predictedRangeSize < this.batchSize + ? predictedRangeSize + : this.batchSize; + + if (this.cache.length < currentBatchSize) { this.cache.push(block); } - if ( - (isFastLane && this.cache.length === this.fastLaneBatchSize) || - this.cache.length === this.batchSize || - isLast - ) { + if (this.cache.length === currentBatchSize || isLast) { addedBlockNumbers = await this.sendBatch(); } diff --git a/src/reader/reader.worker.ts b/src/reader/reader.worker.ts index 919ffeb..20356d0 100644 --- a/src/reader/reader.worker.ts +++ b/src/reader/reader.worker.ts @@ -60,15 +60,17 @@ export default class ReaderWorker extends Worker { }, }, } = this; - + const rangeSize = endBlock - startBlock; blockReader.onReceivedBlock(async block => { const isLast = endBlock === block.thisBlock.blockNumber; - const isFastLane = block.thisBlock.blockNumber >= block.lastIrreversible.blockNumber; + const isFastLane = + block.thisBlock.blockNumber >= block.lastIrreversible.blockNumber; - const { content: addedBlockNumbers, failure } = await blockQueue.add( - block, - isFastLane, isLast - ); + const { content: addedBlockNumbers, failure } = await blockQueue.add(block, { + isFastLane, + isLast, + predictedRangeSize: Number(rangeSize), + }); if (Array.isArray(addedBlockNumbers) && addedBlockNumbers.length > 0) { this.logProgress(addedBlockNumbers); @@ -120,6 +122,8 @@ export default class ReaderWorker extends Worker { }, } = this; + const rangeSize = endBlock - startBlock; + blockReader.onReceivedBlock(async block => { const { content: addedBlockNumbers, failure } = await blockQueue.add(block); if (Array.isArray(addedBlockNumbers) && addedBlockNumbers.length > 0) { From 959b121459a1fd28c8ff20d314ec49570ae5a1ce Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 18 Jul 2023 15:53:55 +0200 Subject: [PATCH 078/107] 0.0.146 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 736b44b..8262094 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.145", + "version": "0.0.146", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From f1bc6b8aef2e68549d4197c6d86edb1d742b5744 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 08:48:11 +0200 Subject: [PATCH 079/107] remove express.js --- README.md | 3 +- package.json | 2 - src/api/api.ts | 21 +-- yarn.lock | 422 +------------------------------------------------ 4 files changed, 11 insertions(+), 437 deletions(-) diff --git a/README.md b/README.md index 812d500..bfb2b3e 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,6 @@ This package encapsulates the core mechanism, however, complete functionality re - [@alien-worlds/workers](https://github.com/Alien-Worlds/workers) - [async](https://github.com/caolan/async) - [commander](https://github.com/tj/commander.js) -- [express](https://github.com/expressjs/express) ## Table of Contents @@ -53,7 +52,7 @@ All processes utilize the commander, enabling specific value assignments for ind ### API -The API process, currently under development, is intended to provide easy access to downloaded data. This Express.js-based API allows viewing of blockchain data, offering endpoints to retrieve block-specific data, transactions, tables, or data from a specific range according to selected criteria. The API is read-only, it doesn't contain methods that modify the content. +The API process, currently under development, is intended to provide easy access to downloaded data. This API allows viewing of blockchain data, offering endpoints to retrieve block-specific data, transactions, tables, or data from a specific range according to selected criteria. The API is read-only, it doesn't contain methods that modify the content. ### Broadcasting diff --git a/package.json b/package.json index 8262094..92afc9b 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,6 @@ }, "license": "ISC", "devDependencies": { - "@types/express": "^4.17.17", "@types/jest": "^27.0.3", "@types/node": "^18.7.14", "@types/node-fetch": "2.x", @@ -42,7 +41,6 @@ "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", - "express": "^4.18.2", "ts-node": "^10.9.1" } } diff --git a/src/api/api.ts b/src/api/api.ts index 2a9f49c..8e736c8 100644 --- a/src/api/api.ts +++ b/src/api/api.ts @@ -1,21 +1,16 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { log, Route } from '@alien-worlds/api-core'; -import express, { Express } from 'express'; import { ApiConfig } from './api.types'; -export class Api { - private app: Express; +export class Api { + private app: WebFramework; - constructor(private config: ApiConfig) { - this.app = express(); - } + constructor(private config: ApiConfig) {} public async start() { - const { - config: { port }, - } = this; - this.app.listen(port, () => { - log(`Server is running at http://localhost:${port}`); - }); + throw new Error('Method "start" not implemented'); + } + + public get framework(): WebFramework { + return this.app; } } diff --git a/yarn.lock b/yarn.lock index c129019..e31f392 100644 --- a/yarn.lock +++ b/yarn.lock @@ -689,40 +689,6 @@ dependencies: "@babel/types" "^7.3.0" -"@types/body-parser@*": - version "1.19.2" - resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.2.tgz#aea2059e28b7658639081347ac4fab3de166e6f0" - integrity sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g== - dependencies: - "@types/connect" "*" - "@types/node" "*" - -"@types/connect@*": - version "3.4.35" - resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.35.tgz#5fcf6ae445e4021d1fc2219a4873cc73a3bb2ad1" - integrity sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ== - dependencies: - "@types/node" "*" - -"@types/express-serve-static-core@^4.17.33": - version "4.17.33" - resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.33.tgz#de35d30a9d637dc1450ad18dd583d75d5733d543" - integrity sha512-TPBqmR/HRYI3eC2E5hmiivIzv+bidAfXofM+sbonAGvyDhySGw9/PQZFt2BLOrjUUR++4eJVpx6KnLQK1Fk9tA== - dependencies: - "@types/node" "*" - "@types/qs" "*" - "@types/range-parser" "*" - -"@types/express@^4.17.17": - version "4.17.17" - resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.17.tgz#01d5437f6ef9cfa8668e616e13c2f2ac9a491ae4" - integrity sha512-Q4FmmuLGBG58btUnfS1c1r/NQdlp3DMfGDGig8WhfpA2YRUtEkxAjkZb0yvplJGYdF1fsQ81iMDcH24sSCNC/Q== - dependencies: - "@types/body-parser" "*" - "@types/express-serve-static-core" "^4.17.33" - "@types/qs" "*" - "@types/serve-static" "*" - "@types/graceful-fs@^4.1.2": version "4.1.5" resolved "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz" @@ -762,11 +728,6 @@ resolved "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz" integrity sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ== -"@types/mime@*": - version "3.0.1" - resolved "https://registry.yarnpkg.com/@types/mime/-/mime-3.0.1.tgz#5f8f2bca0a5863cb69bc0b0acd88c96cb1d4ae10" - integrity sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA== - "@types/node-fetch@2.x": version "2.6.2" resolved "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.2.tgz" @@ -785,29 +746,11 @@ resolved "https://registry.npmjs.org/@types/prettier/-/prettier-2.7.1.tgz" integrity sha512-ri0UmynRRvZiiUJdiz38MmIblKK+oH30MztdBVR95dv/Ubw6neWSb8u1XpRb72L4qsZOhz+L+z9JD40SJmfWow== -"@types/qs@*": - version "6.9.7" - resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" - integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw== - -"@types/range-parser@*": - version "1.2.4" - resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" - integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw== - "@types/semver@^7.3.12": version "7.3.13" resolved "https://registry.npmjs.org/@types/semver/-/semver-7.3.13.tgz" integrity sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw== -"@types/serve-static@*": - version "1.15.0" - resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.15.0.tgz#c7930ff61afb334e121a9da780aac0d9b8f34155" - integrity sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg== - dependencies: - "@types/mime" "*" - "@types/node" "*" - "@types/stack-utils@^2.0.0": version "2.0.1" resolved "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz" @@ -912,14 +855,6 @@ abab@^2.0.3, abab@^2.0.5: resolved "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz" integrity sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA== -accepts@~1.3.8: - version "1.3.8" - resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e" - integrity sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw== - dependencies: - mime-types "~2.1.34" - negotiator "0.6.3" - acorn-globals@^6.0.0: version "6.0.0" resolved "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz" @@ -1026,11 +961,6 @@ argparse@^2.0.1: resolved "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== -array-flatten@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" - integrity sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg== - array-union@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz" @@ -1112,24 +1042,6 @@ balanced-match@^1.0.0: resolved "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== -body-parser@1.20.1: - version "1.20.1" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.1.tgz#b1812a8912c195cd371a3ee5e66faa2338a5c668" - integrity sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw== - dependencies: - bytes "3.1.2" - content-type "~1.0.4" - debug "2.6.9" - depd "2.0.0" - destroy "1.2.0" - http-errors "2.0.0" - iconv-lite "0.4.24" - on-finished "2.4.1" - qs "6.11.0" - raw-body "2.5.1" - type-is "~1.6.18" - unpipe "1.0.0" - brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz" @@ -1179,19 +1091,6 @@ buffer-from@^1.0.0: resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== -bytes@3.1.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" - integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== - -call-bind@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" - integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== - dependencies: - function-bind "^1.1.1" - get-intrinsic "^1.0.2" - callsites@^3.0.0: version "3.1.0" resolved "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz" @@ -1304,33 +1203,11 @@ concat-map@0.0.1: resolved "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== -content-disposition@0.5.4: - version "0.5.4" - resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe" - integrity sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ== - dependencies: - safe-buffer "5.2.1" - -content-type@~1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.5.tgz#8b773162656d1d1086784c8f23a54ce6d73d7918" - integrity sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA== - convert-source-map@^1.4.0, convert-source-map@^1.6.0, convert-source-map@^1.7.0: version "1.9.0" resolved "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz" integrity sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A== -cookie-signature@1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" - integrity sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ== - -cookie@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" - integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== - create-require@^1.1.0: version "1.1.1" resolved "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz" @@ -1376,13 +1253,6 @@ data-urls@^2.0.0: whatwg-mimetype "^2.3.0" whatwg-url "^8.0.0" -debug@2.6.9: - version "2.6.9" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" - integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== - dependencies: - ms "2.0.0" - debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.4: version "4.3.4" resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz" @@ -1415,16 +1285,6 @@ delayed-stream@~1.0.0: resolved "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== -depd@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" - integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== - -destroy@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.2.0.tgz#4803735509ad8be552934c67df614f94e66fa015" - integrity sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg== - detect-newline@^3.0.0: version "3.1.0" resolved "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz" @@ -1461,11 +1321,6 @@ domexception@^2.0.1: dependencies: webidl-conversions "^5.0.0" -ee-first@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" - integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow== - electron-to-chromium@^1.4.251: version "1.4.284" resolved "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz" @@ -1481,11 +1336,6 @@ emoji-regex@^8.0.0: resolved "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== -encodeurl@~1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" - integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w== - error-ex@^1.3.1: version "1.3.2" resolved "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz" @@ -1498,11 +1348,6 @@ escalade@^3.1.1: resolved "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz" integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== -escape-html@~1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" - integrity sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow== - escape-string-regexp@^1.0.5: version "1.0.5" resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" @@ -1663,11 +1508,6 @@ esutils@^2.0.2: resolved "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== -etag@~1.8.1: - version "1.8.1" - resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" - integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== - execa@^5.0.0: version "5.1.1" resolved "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz" @@ -1698,43 +1538,6 @@ expect@^27.5.1: jest-matcher-utils "^27.5.1" jest-message-util "^27.5.1" -express@^4.18.2: - version "4.18.2" - resolved "https://registry.yarnpkg.com/express/-/express-4.18.2.tgz#3fabe08296e930c796c19e3c516979386ba9fd59" - integrity sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ== - dependencies: - accepts "~1.3.8" - array-flatten "1.1.1" - body-parser "1.20.1" - content-disposition "0.5.4" - content-type "~1.0.4" - cookie "0.5.0" - cookie-signature "1.0.6" - debug "2.6.9" - depd "2.0.0" - encodeurl "~1.0.2" - escape-html "~1.0.3" - etag "~1.8.1" - finalhandler "1.2.0" - fresh "0.5.2" - http-errors "2.0.0" - merge-descriptors "1.0.1" - methods "~1.1.2" - on-finished "2.4.1" - parseurl "~1.3.3" - path-to-regexp "0.1.7" - proxy-addr "~2.0.7" - qs "6.11.0" - range-parser "~1.2.1" - safe-buffer "5.2.1" - send "0.18.0" - serve-static "1.15.0" - setprototypeof "1.2.0" - statuses "2.0.1" - type-is "~1.6.18" - utils-merge "1.0.1" - vary "~1.1.2" - fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: version "3.1.3" resolved "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz" @@ -1794,19 +1597,6 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" -finalhandler@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.2.0.tgz#7d23fe5731b207b4640e4fcd00aec1f9207a7b32" - integrity sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg== - dependencies: - debug "2.6.9" - encodeurl "~1.0.2" - escape-html "~1.0.3" - on-finished "2.4.1" - parseurl "~1.3.3" - statuses "2.0.1" - unpipe "~1.0.0" - find-up@^4.0.0, find-up@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz" @@ -1845,16 +1635,6 @@ form-data@^3.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" -forwarded@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" - integrity sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow== - -fresh@0.5.2: - version "0.5.2" - resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" - integrity sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q== - fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" @@ -1880,15 +1660,6 @@ get-caller-file@^2.0.5: resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== -get-intrinsic@^1.0.2: - version "1.2.0" - resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.0.tgz#7ad1dc0535f3a2904bba075772763e5051f6d05f" - integrity sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q== - dependencies: - function-bind "^1.1.1" - has "^1.0.3" - has-symbols "^1.0.3" - get-package-type@^0.1.0: version "0.1.0" resolved "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz" @@ -1969,11 +1740,6 @@ has-flag@^4.0.0: resolved "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz" integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== -has-symbols@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" - integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== - has@^1.0.3: version "1.0.3" resolved "https://registry.npmjs.org/has/-/has-1.0.3.tgz" @@ -1993,17 +1759,6 @@ html-escaper@^2.0.0: resolved "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz" integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg== -http-errors@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-2.0.0.tgz#b7774a1486ef73cf7667ac9ae0858c012c57b9d3" - integrity sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ== - dependencies: - depd "2.0.0" - inherits "2.0.4" - setprototypeof "1.2.0" - statuses "2.0.1" - toidentifier "1.0.1" - http-proxy-agent@^4.0.1: version "4.0.1" resolved "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz" @@ -2067,7 +1822,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4: +inherits@2: version "2.0.4" resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -2077,11 +1832,6 @@ inversify@^6.0.1: resolved "https://registry.yarnpkg.com/inversify/-/inversify-6.0.1.tgz#b20d35425d5d8c5cd156120237aad0008d969f02" integrity sha512-B3ex30927698TJENHR++8FfEaJGqoWOgI6ZY5Ht/nLUsFCwHn6akbwtnUAPCgUepAnTpe2qHxhDNjoKLyz6rgQ== -ipaddr.js@1.9.1: - version "1.9.1" - resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" - integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== - is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz" @@ -2762,16 +2512,6 @@ makeerror@1.0.12: dependencies: tmpl "1.0.5" -media-typer@0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" - integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== - -merge-descriptors@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" - integrity sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w== - merge-stream@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz" @@ -2782,11 +2522,6 @@ merge2@^1.3.0, merge2@^1.4.1: resolved "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz" integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== -methods@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" - integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w== - micromatch@^4.0.4: version "4.0.5" resolved "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz" @@ -2800,18 +2535,13 @@ mime-db@1.52.0: resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== -mime-types@^2.1.12, mime-types@~2.1.24, mime-types@~2.1.34: +mime-types@^2.1.12: version "2.1.35" resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz" integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== dependencies: mime-db "1.52.0" -mime@1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" - integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== - mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz" @@ -2824,21 +2554,11 @@ minimatch@^3.0.4, minimatch@^3.1.1, minimatch@^3.1.2: dependencies: brace-expansion "^1.1.7" -ms@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" - integrity sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A== - ms@2.1.2: version "2.1.2" resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@2.1.3: - version "2.1.3" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" - integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== - nanoid@^3.0.0: version "3.3.6" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.6.tgz#443380c856d6e9f9824267d960b4236ad583ea4c" @@ -2849,11 +2569,6 @@ natural-compare@^1.4.0: resolved "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz" integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw== -negotiator@0.6.3: - version "0.6.3" - resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" - integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== - node-fetch@2.6.6: version "2.6.6" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.6.tgz#1751a7c01834e8e1697758732e9efb6eeadfaf89" @@ -2888,18 +2603,6 @@ nwsapi@^2.2.0: resolved "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.2.tgz" integrity sha512-90yv+6538zuvUMnN+zCr8LuV6bPFdq50304114vJYJ8RDyK8D5O9Phpbd6SZWgI7PwzmmfN1upeOJlvybDSgCw== -object-inspect@^1.9.0: - version "1.12.3" - resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.3.tgz#ba62dffd67ee256c8c086dfae69e016cd1f198b9" - integrity sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g== - -on-finished@2.4.1: - version "2.4.1" - resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f" - integrity sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg== - dependencies: - ee-first "1.1.1" - once@^1.3.0: version "1.4.0" resolved "https://registry.npmjs.org/once/-/once-1.4.0.tgz" @@ -2993,11 +2696,6 @@ parse5@6.0.1: resolved "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz" integrity sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw== -parseurl@~1.3.3: - version "1.3.3" - resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" - integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== - path-exists@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz" @@ -3018,11 +2716,6 @@ path-parse@^1.0.7: resolved "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz" integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== -path-to-regexp@0.1.7: - version "0.1.7" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" - integrity sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ== - path-type@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz" @@ -3089,14 +2782,6 @@ prompts@^2.0.1: kleur "^3.0.3" sisteransi "^1.0.5" -proxy-addr@~2.0.7: - version "2.0.7" - resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" - integrity sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg== - dependencies: - forwarded "0.2.0" - ipaddr.js "1.9.1" - psl@^1.1.33: version "1.9.0" resolved "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz" @@ -3107,13 +2792,6 @@ punycode@^2.1.0, punycode@^2.1.1: resolved "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== -qs@6.11.0: - version "6.11.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a" - integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== - dependencies: - side-channel "^1.0.4" - querystringify@^2.1.1: version "2.2.0" resolved "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz" @@ -3124,21 +2802,6 @@ queue-microtask@^1.2.2: resolved "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz" integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== -range-parser@~1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" - integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== - -raw-body@2.5.1: - version "2.5.1" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.1.tgz#fe1b1628b181b700215e5fd42389f98b71392857" - integrity sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig== - dependencies: - bytes "3.1.2" - http-errors "2.0.0" - iconv-lite "0.4.24" - unpipe "1.0.0" - react-is@^17.0.1: version "17.0.2" resolved "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz" @@ -3214,11 +2877,6 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" -safe-buffer@5.2.1: - version "5.2.1" - resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" - integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== - "safer-buffer@>= 2.1.2 < 3": version "2.1.2" resolved "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" @@ -3243,40 +2901,6 @@ semver@^6.0.0, semver@^6.3.0: resolved "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== -send@0.18.0: - version "0.18.0" - resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" - integrity sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg== - dependencies: - debug "2.6.9" - depd "2.0.0" - destroy "1.2.0" - encodeurl "~1.0.2" - escape-html "~1.0.3" - etag "~1.8.1" - fresh "0.5.2" - http-errors "2.0.0" - mime "1.6.0" - ms "2.1.3" - on-finished "2.4.1" - range-parser "~1.2.1" - statuses "2.0.1" - -serve-static@1.15.0: - version "1.15.0" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.15.0.tgz#faaef08cffe0a1a62f60cad0c4e513cff0ac9540" - integrity sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g== - dependencies: - encodeurl "~1.0.2" - escape-html "~1.0.3" - parseurl "~1.3.3" - send "0.18.0" - -setprototypeof@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" - integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== - shebang-command@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz" @@ -3289,15 +2913,6 @@ shebang-regex@^3.0.0: resolved "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -side-channel@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" - integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw== - dependencies: - call-bind "^1.0.0" - get-intrinsic "^1.0.2" - object-inspect "^1.9.0" - signal-exit@^3.0.2, signal-exit@^3.0.3: version "3.0.7" resolved "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz" @@ -3343,11 +2958,6 @@ stack-utils@^2.0.3: dependencies: escape-string-regexp "^2.0.0" -statuses@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" - integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== - string-length@^4.0.1: version "4.0.2" resolved "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz" @@ -3470,11 +3080,6 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" -toidentifier@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" - integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== - tough-cookie@^4.0.0: version "4.1.2" resolved "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.2.tgz" @@ -3571,14 +3176,6 @@ type-fest@^0.21.3: resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz" integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== -type-is@~1.6.18: - version "1.6.18" - resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" - integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g== - dependencies: - media-typer "0.3.0" - mime-types "~2.1.24" - typedarray-to-buffer@^3.1.5: version "3.1.5" resolved "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz" @@ -3596,11 +3193,6 @@ universalify@^0.2.0: resolved "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz" integrity sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg== -unpipe@1.0.0, unpipe@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" - integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== - update-browserslist-db@^1.0.9: version "1.0.10" resolved "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz" @@ -3624,11 +3216,6 @@ url-parse@^1.5.3: querystringify "^2.1.1" requires-port "^1.0.0" -utils-merge@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" - integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== - v8-compile-cache-lib@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz" @@ -3643,11 +3230,6 @@ v8-to-istanbul@^8.1.0: convert-source-map "^1.6.0" source-map "^0.7.3" -vary@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" - integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg== - w3c-hr-time@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz" From 5045b58a872b50c602377da0ddb4c8b2c3e2d3ac Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 08:48:27 +0200 Subject: [PATCH 080/107] 0.0.147 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 92afc9b..92ce9d2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.146", + "version": "0.0.147", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 18b92fe90b5c330fd8b0298b61167bffecdf5681 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 08:55:41 +0200 Subject: [PATCH 081/107] protected api --- src/api/api.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/api.ts b/src/api/api.ts index 8e736c8..7b80ece 100644 --- a/src/api/api.ts +++ b/src/api/api.ts @@ -2,9 +2,9 @@ import { ApiConfig } from './api.types'; export class Api { - private app: WebFramework; + protected app: WebFramework; - constructor(private config: ApiConfig) {} + constructor(protected config: ApiConfig) {} public async start() { throw new Error('Method "start" not implemented'); From e858b5a6462ee6795223e29355952b21ff529c19 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 09:09:40 +0200 Subject: [PATCH 082/107] 0.0.148 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 92ce9d2..cfeb35c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.147", + "version": "0.0.148", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From b1949ed5079323f6a54391457fefb1755af4a42b Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 09:23:10 +0200 Subject: [PATCH 083/107] pass api instance --- src/api/api.ts | 2 +- src/api/start-api.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/api/api.ts b/src/api/api.ts index 7b80ece..0aede4b 100644 --- a/src/api/api.ts +++ b/src/api/api.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { ApiConfig } from './api.types'; -export class Api { +export class Api { protected app: WebFramework; constructor(protected config: ApiConfig) {} diff --git a/src/api/start-api.ts b/src/api/start-api.ts index cbea433..3b7bd99 100644 --- a/src/api/start-api.ts +++ b/src/api/start-api.ts @@ -2,11 +2,11 @@ import 'reflect-metadata'; import { Route } from '@alien-worlds/api-core'; import { Api } from './api'; -import { ApiConfig } from './api.types'; - -export const startApi = async (config: ApiConfig, routes: Route[] = []) => { - const api = new Api(config); +export const startApi = async ( + api: Api, + routes: Route[] = [] +) => { routes.forEach(route => { Route.mount(api, route); }); From 5a19e1b4ed0659926f9da440df1ae4c56fd8d304 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 09:23:37 +0200 Subject: [PATCH 084/107] 0.0.149 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index cfeb35c..2f20c17 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.148", + "version": "0.0.149", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 5713c11b7752677fc7a19ec00ea88d76bed2d5a4 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 11:28:26 +0200 Subject: [PATCH 085/107] update api --- src/api/api.command.ts | 8 ++++++++ src/api/api.dependencies.ts | 16 ++++++++++++++++ src/api/api.ts | 5 ++++- src/api/api.types.ts | 6 +++++- src/api/index.ts | 4 +++- src/api/start-api.ts | 21 ++++++++++++++++----- src/config/index.ts | 11 +++++++---- 7 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 src/api/api.command.ts create mode 100644 src/api/api.dependencies.ts diff --git a/src/api/api.command.ts b/src/api/api.command.ts new file mode 100644 index 0000000..3a6c292 --- /dev/null +++ b/src/api/api.command.ts @@ -0,0 +1,8 @@ +import { Command } from 'commander'; + +export const apiCommand = new Command(); + +apiCommand + .version('1.0', '-v, --version') + .option('-p, --port ') + .parse(process.argv); diff --git a/src/api/api.dependencies.ts b/src/api/api.dependencies.ts new file mode 100644 index 0000000..6f7ce27 --- /dev/null +++ b/src/api/api.dependencies.ts @@ -0,0 +1,16 @@ +import { Result } from '@alien-worlds/api-core'; +import { DatabaseConfigBuilder, Dependencies } from '../common'; +import { Api } from './api'; +import { ApiConfig } from './api.types'; + +/** + * An abstract class representing a Api dependencies. + * @class ApiDependencies + */ +export abstract class ApiDependencies extends Dependencies { + public api: Api; + + public databaseConfigBuilder: DatabaseConfigBuilder; + + public abstract initialize(config: ApiConfig): Promise; +} diff --git a/src/api/api.ts b/src/api/api.ts index 0aede4b..cf2db65 100644 --- a/src/api/api.ts +++ b/src/api/api.ts @@ -3,8 +3,11 @@ import { ApiConfig } from './api.types'; export class Api { protected app: WebFramework; + protected config: ApiConfig; - constructor(protected config: ApiConfig) {} + public setup(config: ApiConfig) { + this.config = config; + } public async start() { throw new Error('Method "start" not implemented'); diff --git a/src/api/api.types.ts b/src/api/api.types.ts index 1605540..9edfd9b 100644 --- a/src/api/api.types.ts +++ b/src/api/api.types.ts @@ -1,4 +1,8 @@ -import { UnknownObject } from "@alien-worlds/api-core"; +import { UnknownObject } from '@alien-worlds/api-core'; + +export type ApiCommandOptions = { + port: number; +}; export type ApiConfig = { port: number; diff --git a/src/api/index.ts b/src/api/index.ts index ce894e8..70bae94 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -1,4 +1,6 @@ -export * from './api'; +export * from './api.command'; +export * from './api.dependencies'; export * from './api.types'; +export * from './api'; export * from './start-api'; export * as ApiEndpoints from './endpoints'; diff --git a/src/api/start-api.ts b/src/api/start-api.ts index 3b7bd99..8de318d 100644 --- a/src/api/start-api.ts +++ b/src/api/start-api.ts @@ -1,12 +1,23 @@ import 'reflect-metadata'; -import { Route } from '@alien-worlds/api-core'; -import { Api } from './api'; +import { ConfigVars, Route } from '@alien-worlds/api-core'; +import { apiCommand } from './api.command'; +import { ApiCommandOptions } from './api.types'; +import { buildApiConfig } from '../config'; +import { ApiDependencies } from './api.dependencies'; -export const startApi = async ( - api: Api, - routes: Route[] = [] +export const startApi = async ( + dependencies: ApiDependencies, + routes: Route[] = [], + ...args: string[] ) => { + const { api } = dependencies; + const vars = new ConfigVars(); + const options = apiCommand.parse(args).opts(); + const config = buildApiConfig(vars, dependencies.databaseConfigBuilder, options); + + api.setup(config); + routes.forEach(route => { Route.mount(api, route); }); diff --git a/src/config/index.ts b/src/config/index.ts index 0a84a6c..3f01753 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -1,5 +1,6 @@ +import { Api } from './../api/api'; import { FilterCommandOptions, FilterConfig } from '../filter'; -import { ApiConfig } from '../api'; +import { ApiCommandOptions, ApiConfig } from '../api'; import { BlockchainConfig, BootstrapCommandOptions, BootstrapConfig } from '../bootstrap'; import { ReaderCommandOptions, ReaderConfig } from '../reader'; import { ProcessorCommandOptions, ProcessorConfig } from '../processor'; @@ -96,14 +97,16 @@ export const buildUnprocessedBlockQueueConfig = ( sizeCheckInterval: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_SIZE_CHECK_INTERVAL') || 2000, batchSize: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_BATCH_SIZE') || 100, - fastLaneBatchSize: vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_FAST_LANE_BATCH_SIZE') || 1, + fastLaneBatchSize: + vars.getNumberEnv('UNPROCESSED_BLOCK_QUEUE_FAST_LANE_BATCH_SIZE') || 1, }); export const buildApiConfig = ( vars: ConfigVars, - databaseConfigBuilder: (vars: ConfigVars, ...args: unknown[]) => UnknownObject + databaseConfigBuilder: (vars: ConfigVars, ...args: unknown[]) => UnknownObject, + options?: ApiCommandOptions ): ApiConfig => ({ - port: vars.getNumberEnv('API_PORT') || 8080, + port: vars.getNumberEnv('API_PORT') || options.port || 8080, database: databaseConfigBuilder(vars), }); From 0e42ca429b7e210745fd1254a6096ad7ddec1042 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 11:28:44 +0200 Subject: [PATCH 086/107] 0.0.150 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2f20c17..97a7cb1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.149", + "version": "0.0.150", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 14a5b86f22474f4c8ab358412c57b538edc6ff29 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 11:46:16 +0200 Subject: [PATCH 087/107] update api --- src/api/api.dependencies.ts | 4 +--- src/api/start-api.ts | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/api/api.dependencies.ts b/src/api/api.dependencies.ts index 6f7ce27..478c97d 100644 --- a/src/api/api.dependencies.ts +++ b/src/api/api.dependencies.ts @@ -1,7 +1,6 @@ import { Result } from '@alien-worlds/api-core'; import { DatabaseConfigBuilder, Dependencies } from '../common'; import { Api } from './api'; -import { ApiConfig } from './api.types'; /** * An abstract class representing a Api dependencies. @@ -9,8 +8,7 @@ import { ApiConfig } from './api.types'; */ export abstract class ApiDependencies extends Dependencies { public api: Api; - public databaseConfigBuilder: DatabaseConfigBuilder; - public abstract initialize(config: ApiConfig): Promise; + public abstract initialize(): Promise; } diff --git a/src/api/start-api.ts b/src/api/start-api.ts index 8de318d..5465dd1 100644 --- a/src/api/start-api.ts +++ b/src/api/start-api.ts @@ -11,10 +11,10 @@ export const startApi = async ( routes: Route[] = [], ...args: string[] ) => { - const { api } = dependencies; + const { api, databaseConfigBuilder } = dependencies; const vars = new ConfigVars(); const options = apiCommand.parse(args).opts(); - const config = buildApiConfig(vars, dependencies.databaseConfigBuilder, options); + const config = buildApiConfig(vars, databaseConfigBuilder, options); api.setup(config); From 44a731bb414d3804319bee65a5f4d495dfbdf3dc Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 11:48:44 +0200 Subject: [PATCH 088/107] 0.0.151 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 97a7cb1..d01fe17 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.150", + "version": "0.0.151", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From a3a0274d3abed754ff6ab7c2270318717f74cb12 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 13:23:14 +0200 Subject: [PATCH 089/107] update deps --- src/api/api.dependencies.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/api/api.dependencies.ts b/src/api/api.dependencies.ts index 478c97d..5615ef5 100644 --- a/src/api/api.dependencies.ts +++ b/src/api/api.dependencies.ts @@ -1,4 +1,4 @@ -import { Result } from '@alien-worlds/api-core'; +import { Container, Result, Route, UnknownObject } from '@alien-worlds/api-core'; import { DatabaseConfigBuilder, Dependencies } from '../common'; import { Api } from './api'; @@ -8,7 +8,11 @@ import { Api } from './api'; */ export abstract class ApiDependencies extends Dependencies { public api: Api; + public ioc: Container; public databaseConfigBuilder: DatabaseConfigBuilder; - public abstract initialize(): Promise; + public abstract initialize( + setupIoc: (config: UnknownObject, container: Container) => Promise, + routesProvider: (container: Container) => Route[] + ): Promise; } From 9d6979b46b9f2c1973f6601204e2de14a0095eb5 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 13:23:34 +0200 Subject: [PATCH 090/107] 0.0.152 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d01fe17..785b74f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.151", + "version": "0.0.152", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 5fb0e809e60311dccc575eef062dd4bfd01048d8 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 13:34:01 +0200 Subject: [PATCH 091/107] move routes and ioc init to startApi --- src/api/api.dependencies.ts | 3 +++ src/api/start-api.ts | 11 +++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/api/api.dependencies.ts b/src/api/api.dependencies.ts index 5615ef5..db32b7d 100644 --- a/src/api/api.dependencies.ts +++ b/src/api/api.dependencies.ts @@ -1,6 +1,7 @@ import { Container, Result, Route, UnknownObject } from '@alien-worlds/api-core'; import { DatabaseConfigBuilder, Dependencies } from '../common'; import { Api } from './api'; +import { ApiConfig } from './api.types'; /** * An abstract class representing a Api dependencies. @@ -9,6 +10,8 @@ import { Api } from './api'; export abstract class ApiDependencies extends Dependencies { public api: Api; public ioc: Container; + public setupIoc: (config: ApiConfig, container: Container) => Promise; + public routesProvider: (container: Container) => Route[]; public databaseConfigBuilder: DatabaseConfigBuilder; public abstract initialize( diff --git a/src/api/start-api.ts b/src/api/start-api.ts index 5465dd1..a4fe4c9 100644 --- a/src/api/start-api.ts +++ b/src/api/start-api.ts @@ -6,15 +6,14 @@ import { ApiCommandOptions } from './api.types'; import { buildApiConfig } from '../config'; import { ApiDependencies } from './api.dependencies'; -export const startApi = async ( - dependencies: ApiDependencies, - routes: Route[] = [], - ...args: string[] -) => { - const { api, databaseConfigBuilder } = dependencies; +export const startApi = async (dependencies: ApiDependencies, ...args: string[]) => { + const { api, ioc, databaseConfigBuilder, routesProvider, setupIoc } = dependencies; const vars = new ConfigVars(); const options = apiCommand.parse(args).opts(); const config = buildApiConfig(vars, databaseConfigBuilder, options); + const routes = routesProvider(ioc); + + await setupIoc(config, ioc); api.setup(config); From e794284d41ea266a5a46bf45ea524d9da18fcb99 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 13:34:27 +0200 Subject: [PATCH 092/107] 0.0.153 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 785b74f..96afb07 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.152", + "version": "0.0.153", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 7c0ca981a4ddf2c3c72dea0ae4e5c29a6b55500d Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 13:53:49 +0200 Subject: [PATCH 093/107] bugfix order framework --- src/api/start-api.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/api/start-api.ts b/src/api/start-api.ts index a4fe4c9..f0ce1c1 100644 --- a/src/api/start-api.ts +++ b/src/api/start-api.ts @@ -11,14 +11,13 @@ export const startApi = async (dependencies: ApiDependencies, ...args: string[]) const vars = new ConfigVars(); const options = apiCommand.parse(args).opts(); const config = buildApiConfig(vars, databaseConfigBuilder, options); - const routes = routesProvider(ioc); await setupIoc(config, ioc); api.setup(config); - routes.forEach(route => { - Route.mount(api, route); + routesProvider(ioc).forEach(route => { + Route.mount(api.framework, route); }); return api.start(); From 4b929fdeee7d8ca3af56ecf6706a7a97c206af0e Mon Sep 17 00:00:00 2001 From: rkamysz Date: Tue, 25 Jul 2023 13:55:43 +0200 Subject: [PATCH 094/107] 0.0.154 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 96afb07..ec585e5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/api-history-tools", - "version": "0.0.153", + "version": "0.0.154", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From bcfe9686f7e72ed855b411d53d2e28db10f79038 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Thu, 27 Jul 2023 22:54:37 +0200 Subject: [PATCH 095/107] rename package and dependencies --- README.md | 13 +++---- package.json | 11 +++--- src/api/api.dependencies.ts | 2 +- src/api/api.types.ts | 2 +- .../actions/domain/list-actions.controller.ts | 2 +- .../domain/models/list-actions.input.ts | 2 +- .../domain/models/list-actions.output.ts | 2 +- .../contract-action.repository.ts | 2 +- .../domain/use-cases/list-actions.use-case.ts | 2 +- .../actions/routes/list-actions.route.ts | 2 +- src/api/start-api.ts | 2 +- .../__tests__/bootstrap.utils.unit.test.ts | 9 +++-- .../__tests__/start-bootstrap.unit.test.ts | 2 +- src/bootstrap/bootstrap.config.ts | 4 +- src/bootstrap/bootstrap.dependencies.ts | 4 +- src/bootstrap/bootstrap.utils.ts | 2 +- src/bootstrap/start-bootstrap.ts | 14 ++----- .../internal-broadcast.message.unit.test.ts | 2 +- src/broadcast/internal-broadcast.message.ts | 2 +- .../processor-broadcast.message.unit.test.ts | 2 +- .../reader-broadcast.message.unit.test.ts | 2 +- .../messages/filter-broadcast.message.ts | 2 +- .../messages/processor-broadcast.message.ts | 2 +- .../messages/reader-broadcast.message.ts | 2 +- src/broadcast/start-broadcast.ts | 4 +- .../abi.repository-impl.unit.test.ts | 2 +- src/common/abis/__tests__/abis.unit.test.ts | 5 +-- src/common/abis/abis.cache.ts | 2 +- src/common/abis/abis.errors.ts | 6 +++ src/common/abis/abis.repository-impl.ts | 2 +- src/common/abis/abis.repository.ts | 2 +- src/common/abis/abis.ts | 11 ++++-- src/common/abis/abis.types.ts | 2 +- .../block-range-scan.repository.ts | 2 +- .../block-range-scan.source.ts | 2 +- .../block-range-scanner/block-range-scan.ts | 2 +- .../__tests__/block-state.unit.test.ts | 4 +- src/common/block-state/block-state.ts | 2 +- src/common/common.types.ts | 2 +- src/common/dependencies.ts | 2 +- src/common/featured/featured-contract.ts | 2 +- src/common/featured/featured-contracts.ts | 2 +- src/common/featured/featured.utils.ts | 2 +- .../processor-task-queue.ts | 2 +- .../processor-task.source.ts | 2 +- .../processor-task-queue/processor-task.ts | 2 +- src/common/types/delta.types.ts | 2 +- .../unprocessed-block-queue.source.ts | 2 +- .../unprocessed-block-queue.ts | 4 +- src/config/index.ts | 7 ++-- src/filter/filter.config.ts | 6 +-- src/filter/filter.dependencies.ts | 4 +- src/filter/filter.runner.ts | 7 ++-- .../filter.worker-loader.dependencies.ts | 6 +-- src/filter/filter.worker-loader.ts | 2 +- src/filter/filter.worker.ts | 19 +++------ src/filter/start-filter.ts | 6 +-- src/index.ts | 7 ++-- src/processor/processor.config.ts | 6 +-- src/processor/processor.dependencies.ts | 4 +- src/processor/processor.model.factory.ts | 2 +- src/processor/processor.runner.ts | 7 ++-- .../processor.worker-loader.dependencies.ts | 2 +- src/processor/processor.worker-loader.ts | 4 +- .../processors/action-trace.processor.ts | 2 +- src/processor/processors/delta.processor.ts | 2 +- src/processor/processors/processor.ts | 2 +- src/processor/start-processor.ts | 6 +-- src/reader/reader.config.ts | 7 ++-- src/reader/reader.dependencies.ts | 4 +- src/reader/reader.ts | 6 +-- .../reader.worker-loader.dependencies.ts | 4 +- src/reader/reader.worker-loader.ts | 4 +- src/reader/reader.worker.ts | 6 +-- src/reader/start-reader.ts | 6 +-- tutorials/extending-history-tools.md | 19 +++++---- tutorials/what-is-featured-content.md | 2 +- yarn.lock | 39 ++++++++----------- 78 files changed, 172 insertions(+), 189 deletions(-) diff --git a/README.md b/README.md index bfb2b3e..5fe0afd 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,9 @@ This package encapsulates the core mechanism, however, complete functionality re ## Dependencies -- [@alien-worlds/api-core](https://github.com/Alien-Worlds/api-core) -- [@alien-worlds/block-reader](https://github.com/Alien-Worlds/block-reader) -- [@alien-worlds/broadcast](https://github.com/Alien-Worlds/broadcast) -- [@alien-worlds/workers](https://github.com/Alien-Worlds/workers) +- [@alien-worlds/aw-core](https://github.com/Alien-Worlds/api-core) +- [@alien-worlds/aw-broadcast](https://github.com/Alien-Worlds/broadcast) +- [@alien-worlds/aw-workers](https://github.com/Alien-Worlds/workers) - [async](https://github.com/caolan/async) - [commander](https://github.com/tj/commander.js) @@ -42,7 +41,7 @@ This package encapsulates the core mechanism, however, complete functionality re To add History Tools to your project, use the following command with your favorite package manager: ```bash -yarn add @alien-worlds/api-history-tools +yarn add @alien-worlds/aw-history ``` @@ -56,7 +55,7 @@ The API process, currently under development, is intended to provide easy access ### Broadcasting -The broadcasting process creates a server for communication between processes, leveraging the @alien-worlds/broadcast module. Processes inform each other about their state, enabling efficient work coordination. The server facilitates the dispatch of messages defining tasks and states, improving performance when processing a high volume of blockchain data. +The broadcasting process creates a server for communication between processes, leveraging the @alien-worlds/aw-broadcast module. Processes inform each other about their state, enabling efficient work coordination. The server facilitates the dispatch of messages defining tasks and states, improving performance when processing a high volume of blockchain data. ### Bootstrap @@ -194,7 +193,7 @@ The Config tools are used for generating configuration objects based on values s ## Tutorials -For tutorials on creating and using the history tools for your specific needs, see the tutorials in the [History Tools Starter Kit](https://github.com/Alien-Worlds/history-tools-starter-kit) repository. If you want to create history tools with `mongodb` and `eosjs` tools, you should go to the mentioned repository. +For tutorials on creating and using the history tools for your specific needs, see the tutorials in the [History Tools Starter Kit](https://github.com/Alien-Worlds/aw-history-starter-kit) repository. If you want to create history tools with `mongodb` and `eosjs` tools, you should go to the mentioned repository. If you want to extend the capabilities of the history tools or take advantage of other third-party resources, please refer to the following tutorial. diff --git a/package.json b/package.json index ec585e5..902d3a8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { - "name": "@alien-worlds/api-history-tools", - "version": "0.0.154", + "name": "@alien-worlds/aw-history", + "version": "0.0.0", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", @@ -34,10 +34,9 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/api-core": "^0.0.158", - "@alien-worlds/block-reader": "^0.0.18", - "@alien-worlds/broadcast": "^0.0.13", - "@alien-worlds/workers": "^0.0.7", + "@alien-worlds/aw-broadcast": "^0.0.2", + "@alien-worlds/aw-core": "^0.0.2", + "@alien-worlds/aw-workers": "^0.0.2", "async": "^3.2.4", "commander": "^10.0.1", "crypto": "^1.0.1", diff --git a/src/api/api.dependencies.ts b/src/api/api.dependencies.ts index db32b7d..0f646f7 100644 --- a/src/api/api.dependencies.ts +++ b/src/api/api.dependencies.ts @@ -1,4 +1,4 @@ -import { Container, Result, Route, UnknownObject } from '@alien-worlds/api-core'; +import { Container, Result, Route, UnknownObject } from '@alien-worlds/aw-core'; import { DatabaseConfigBuilder, Dependencies } from '../common'; import { Api } from './api'; import { ApiConfig } from './api.types'; diff --git a/src/api/api.types.ts b/src/api/api.types.ts index 9edfd9b..c3829d0 100644 --- a/src/api/api.types.ts +++ b/src/api/api.types.ts @@ -1,4 +1,4 @@ -import { UnknownObject } from '@alien-worlds/api-core'; +import { UnknownObject } from '@alien-worlds/aw-core'; export type ApiCommandOptions = { port: number; diff --git a/src/api/endpoints/actions/domain/list-actions.controller.ts b/src/api/endpoints/actions/domain/list-actions.controller.ts index 77517ad..310af89 100644 --- a/src/api/endpoints/actions/domain/list-actions.controller.ts +++ b/src/api/endpoints/actions/domain/list-actions.controller.ts @@ -1,4 +1,4 @@ -import { inject, injectable, Result } from '@alien-worlds/api-core'; +import { inject, injectable, Result } from '@alien-worlds/aw-core'; import { ListActionsInput } from './models/list-actions.input'; import { ListActionsOutput } from './models/list-actions.output'; import { ListActionsUseCase } from './use-cases/list-actions.use-case'; diff --git a/src/api/endpoints/actions/domain/models/list-actions.input.ts b/src/api/endpoints/actions/domain/models/list-actions.input.ts index dd9cd32..26df54e 100644 --- a/src/api/endpoints/actions/domain/models/list-actions.input.ts +++ b/src/api/endpoints/actions/domain/models/list-actions.input.ts @@ -1,5 +1,5 @@ import { ListActionsQueryParams } from '../actions.types'; -import { Request, parseToBigInt } from '@alien-worlds/api-core'; +import { Request, parseToBigInt } from '@alien-worlds/aw-core'; /** * @class */ diff --git a/src/api/endpoints/actions/domain/models/list-actions.output.ts b/src/api/endpoints/actions/domain/models/list-actions.output.ts index c8dbc67..5dd06da 100644 --- a/src/api/endpoints/actions/domain/models/list-actions.output.ts +++ b/src/api/endpoints/actions/domain/models/list-actions.output.ts @@ -1,4 +1,4 @@ -import { ContractAction, Result } from '@alien-worlds/api-core'; +import { ContractAction, Result } from '@alien-worlds/aw-core'; export class ListActionsOutput { public static create(result: Result): ListActionsOutput { diff --git a/src/api/endpoints/actions/domain/repositories/contract-action.repository.ts b/src/api/endpoints/actions/domain/repositories/contract-action.repository.ts index 6eb472d..f6b52c5 100644 --- a/src/api/endpoints/actions/domain/repositories/contract-action.repository.ts +++ b/src/api/endpoints/actions/domain/repositories/contract-action.repository.ts @@ -1,4 +1,4 @@ -import { injectable, Repository, ContractAction } from '@alien-worlds/api-core'; +import { injectable, Repository, ContractAction } from '@alien-worlds/aw-core'; /** * @abstract diff --git a/src/api/endpoints/actions/domain/use-cases/list-actions.use-case.ts b/src/api/endpoints/actions/domain/use-cases/list-actions.use-case.ts index 65350cb..d60c6c5 100644 --- a/src/api/endpoints/actions/domain/use-cases/list-actions.use-case.ts +++ b/src/api/endpoints/actions/domain/use-cases/list-actions.use-case.ts @@ -7,7 +7,7 @@ import { injectable, Result, UseCase, -} from '@alien-worlds/api-core'; +} from '@alien-worlds/aw-core'; import { ContractActionRepository } from '../repositories/contract-action.repository'; /*imports*/ diff --git a/src/api/endpoints/actions/routes/list-actions.route.ts b/src/api/endpoints/actions/routes/list-actions.route.ts index 896fd0a..079f7fd 100644 --- a/src/api/endpoints/actions/routes/list-actions.route.ts +++ b/src/api/endpoints/actions/routes/list-actions.route.ts @@ -1,4 +1,4 @@ -import { GetRoute, RouteHandler } from '@alien-worlds/api-core'; +import { GetRoute, RouteHandler } from '@alien-worlds/aw-core'; import { ListActionsInput } from '../domain/models/list-actions.input'; import { ListActionsOutput } from '../domain/models/list-actions.output'; diff --git a/src/api/start-api.ts b/src/api/start-api.ts index f0ce1c1..504f052 100644 --- a/src/api/start-api.ts +++ b/src/api/start-api.ts @@ -1,6 +1,6 @@ import 'reflect-metadata'; -import { ConfigVars, Route } from '@alien-worlds/api-core'; +import { ConfigVars, Route } from '@alien-worlds/aw-core'; import { apiCommand } from './api.command'; import { ApiCommandOptions } from './api.types'; import { buildApiConfig } from '../config'; diff --git a/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts b/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts index 7515e8d..ed94e33 100644 --- a/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts +++ b/src/bootstrap/__tests__/bootstrap.utils.unit.test.ts @@ -1,12 +1,15 @@ -import { Result } from '@alien-worlds/api-core'; +import { Result } from '@alien-worlds/aw-core'; import { createDefaultModeBlockRange, createReplayModeBlockRange, createTestModeBlockRange, } from '../bootstrap.utils'; import { Mode } from '../../common'; -import { EndBlockOutOfRangeError, StartBlockHigherThanEndBlockError, UndefinedStartBlockError } from '../bootstrap.errors'; - +import { + EndBlockOutOfRangeError, + StartBlockHigherThanEndBlockError, + UndefinedStartBlockError, +} from '../bootstrap.errors'; describe('createDefaultModeBlockRange', () => { const originalLog = console.log; diff --git a/src/bootstrap/__tests__/start-bootstrap.unit.test.ts b/src/bootstrap/__tests__/start-bootstrap.unit.test.ts index 948bfa9..74d7634 100644 --- a/src/bootstrap/__tests__/start-bootstrap.unit.test.ts +++ b/src/bootstrap/__tests__/start-bootstrap.unit.test.ts @@ -3,7 +3,7 @@ import { NoAbisError } from '../bootstrap.errors'; import { InternalBroadcastMessageName } from '../../broadcast/internal-broadcast.enums'; import { ReaderBroadcastMessage } from '../../broadcast/messages'; import { Mode } from '../../common'; -import { BroadcastTcpClient } from '@alien-worlds/broadcast'; +import { BroadcastTcpClient } from '@alien-worlds/aw-broadcast'; jest.mock('@alien-worlds/history-tools-common', () => ({ Abis: { diff --git a/src/bootstrap/bootstrap.config.ts b/src/bootstrap/bootstrap.config.ts index ebbc06a..58e60bb 100644 --- a/src/bootstrap/bootstrap.config.ts +++ b/src/bootstrap/bootstrap.config.ts @@ -1,8 +1,8 @@ -import { UnknownObject } from '@alien-worlds/api-core'; +import { UnknownObject } from '@alien-worlds/aw-core'; import { AbisServiceConfig } from '../common'; import { BlockRangeScanConfig } from '../common/block-range-scanner'; import { FeaturedConfig } from '../common/featured'; -import { BroadcastConfig } from '@alien-worlds/broadcast'; +import { BroadcastConfig } from '@alien-worlds/aw-broadcast'; export type BlockchainConfig = { endpoint: string; diff --git a/src/bootstrap/bootstrap.dependencies.ts b/src/bootstrap/bootstrap.dependencies.ts index c289c49..9eba3d1 100644 --- a/src/bootstrap/bootstrap.dependencies.ts +++ b/src/bootstrap/bootstrap.dependencies.ts @@ -1,7 +1,7 @@ -import { BlockchainService, Result } from '@alien-worlds/api-core'; +import { BlockchainService, Result } from '@alien-worlds/aw-core'; import { Dependencies } from '../common/dependencies'; import { FeaturedContracts, FeaturedContractDataCriteria } from '../common/featured'; -import { BroadcastClient } from '@alien-worlds/broadcast'; +import { BroadcastClient } from '@alien-worlds/aw-broadcast'; import { Abis, BlockRangeScanner, BlockState, DatabaseConfigBuilder } from '../common'; import { BootstrapConfig } from './bootstrap.config'; diff --git a/src/bootstrap/bootstrap.utils.ts b/src/bootstrap/bootstrap.utils.ts index 758dfd5..a590ddc 100644 --- a/src/bootstrap/bootstrap.utils.ts +++ b/src/bootstrap/bootstrap.utils.ts @@ -1,4 +1,4 @@ -import { BlockchainService, log, parseToBigInt } from '@alien-worlds/api-core'; +import { BlockchainService, log, parseToBigInt } from '@alien-worlds/aw-core'; import { BlockRangeData } from './bootstrap.types'; import { StartBlockHigherThanEndBlockError, diff --git a/src/bootstrap/start-bootstrap.ts b/src/bootstrap/start-bootstrap.ts index 938bb72..7292639 100644 --- a/src/bootstrap/start-bootstrap.ts +++ b/src/bootstrap/start-bootstrap.ts @@ -13,8 +13,8 @@ import { InternalBroadcastChannel, InternalBroadcastMessageName, } from '../broadcast/internal-broadcast.enums'; -import { ConfigVars, log } from '@alien-worlds/api-core'; -import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { ConfigVars, log } from '@alien-worlds/aw-core'; +import { BroadcastMessage } from '@alien-worlds/aw-broadcast'; import { buildBootstrapConfig } from '../config'; import { bootstrapCommand } from './bootstrap.command'; import { BootstrapConfig } from './bootstrap.config'; @@ -56,14 +56,8 @@ export const bootstrap = async ( throw initResult.failure.error; } - const { - abis, - broadcastClient, - blockState, - blockchain, - featuredContracts, - scanner, - } = dependencies; + const { abis, broadcastClient, blockState, blockchain, featuredContracts, scanner } = + dependencies; let blockRange: ReaderBroadcastMessageData; // fetch latest abis to make sure that the blockchain data will be correctly deserialized diff --git a/src/broadcast/__tests__/internal-broadcast.message.unit.test.ts b/src/broadcast/__tests__/internal-broadcast.message.unit.test.ts index 59cfa9c..66b6056 100644 --- a/src/broadcast/__tests__/internal-broadcast.message.unit.test.ts +++ b/src/broadcast/__tests__/internal-broadcast.message.unit.test.ts @@ -1,4 +1,4 @@ -import { BroadcastTcpMessageType } from '@alien-worlds/broadcast'; +import { BroadcastTcpMessageType } from '@alien-worlds/aw-broadcast'; import { InternalBroadcastMessage } from '../internal-broadcast.message'; describe('InternalBroadcastMessage', () => { diff --git a/src/broadcast/internal-broadcast.message.ts b/src/broadcast/internal-broadcast.message.ts index 71b8f90..06e813f 100644 --- a/src/broadcast/internal-broadcast.message.ts +++ b/src/broadcast/internal-broadcast.message.ts @@ -2,7 +2,7 @@ import { BroadcastTcpMessage, BroadcastTcpMessageContent, BroadcastTcpMessageType, -} from '@alien-worlds/broadcast'; +} from '@alien-worlds/aw-broadcast'; /** * Represents an internal broadcast message. diff --git a/src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts b/src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts index 2abf217..f4128ea 100644 --- a/src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts +++ b/src/broadcast/messages/__tests__/processor-broadcast.message.unit.test.ts @@ -3,7 +3,7 @@ import { InternalBroadcastChannel, InternalBroadcastMessageName, } from '../../internal-broadcast.enums'; -import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { BroadcastMessage } from '@alien-worlds/aw-broadcast'; describe('ProcessorBroadcastMessage', () => { describe('ready', () => { diff --git a/src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts b/src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts index 7dcaead..91ad230 100644 --- a/src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts +++ b/src/broadcast/messages/__tests__/reader-broadcast.message.unit.test.ts @@ -2,7 +2,7 @@ import { ReaderBroadcastMessage, ReaderBroadcastMessageData, } from '../reader-broadcast.message'; -import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { BroadcastMessage } from '@alien-worlds/aw-broadcast'; import { InternalBroadcastChannel, InternalBroadcastMessageName, diff --git a/src/broadcast/messages/filter-broadcast.message.ts b/src/broadcast/messages/filter-broadcast.message.ts index 95a3b0b..f465238 100644 --- a/src/broadcast/messages/filter-broadcast.message.ts +++ b/src/broadcast/messages/filter-broadcast.message.ts @@ -1,4 +1,4 @@ -import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { BroadcastMessage } from '@alien-worlds/aw-broadcast'; import { InternalBroadcastChannel, InternalBroadcastMessageName, diff --git a/src/broadcast/messages/processor-broadcast.message.ts b/src/broadcast/messages/processor-broadcast.message.ts index 748e585..6b83765 100644 --- a/src/broadcast/messages/processor-broadcast.message.ts +++ b/src/broadcast/messages/processor-broadcast.message.ts @@ -1,4 +1,4 @@ -import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { BroadcastMessage } from '@alien-worlds/aw-broadcast'; import { InternalBroadcastChannel, InternalBroadcastMessageName, diff --git a/src/broadcast/messages/reader-broadcast.message.ts b/src/broadcast/messages/reader-broadcast.message.ts index 48e4544..a1d7cbf 100644 --- a/src/broadcast/messages/reader-broadcast.message.ts +++ b/src/broadcast/messages/reader-broadcast.message.ts @@ -1,4 +1,4 @@ -import { BroadcastMessage } from '@alien-worlds/broadcast'; +import { BroadcastMessage } from '@alien-worlds/aw-broadcast'; import { InternalBroadcastChannel, InternalBroadcastMessageName, diff --git a/src/broadcast/start-broadcast.ts b/src/broadcast/start-broadcast.ts index 757a996..bc14f1e 100644 --- a/src/broadcast/start-broadcast.ts +++ b/src/broadcast/start-broadcast.ts @@ -1,5 +1,5 @@ -import { ConfigVars } from '@alien-worlds/api-core'; -import { BroadcastTcpServer, buildBroadcastConfig } from '@alien-worlds/broadcast'; +import { ConfigVars } from '@alien-worlds/aw-core'; +import { BroadcastTcpServer, buildBroadcastConfig } from '@alien-worlds/aw-broadcast'; export const startBroadcast = async () => { const vars = new ConfigVars(); diff --git a/src/common/abis/__tests__/abi.repository-impl.unit.test.ts b/src/common/abis/__tests__/abi.repository-impl.unit.test.ts index ac0e4c9..04be6d2 100644 --- a/src/common/abis/__tests__/abi.repository-impl.unit.test.ts +++ b/src/common/abis/__tests__/abi.repository-impl.unit.test.ts @@ -1,5 +1,5 @@ import { AbisRepositoryImpl } from '../abis.repository-impl'; -import { Result, CountParams, ContractEncodedAbi } from '@alien-worlds/api-core'; +import { Result, CountParams, ContractEncodedAbi } from '@alien-worlds/aw-core'; import { AbisCache } from '../abis.cache'; jest.mock('../abis.cache'); diff --git a/src/common/abis/__tests__/abis.unit.test.ts b/src/common/abis/__tests__/abis.unit.test.ts index b6b07f7..90082e0 100644 --- a/src/common/abis/__tests__/abis.unit.test.ts +++ b/src/common/abis/__tests__/abis.unit.test.ts @@ -1,8 +1,7 @@ import { Abis } from '../abis'; import { AbisRepositoryImpl } from '../abis.repository-impl'; -import { Failure, Result, AbiService } from '@alien-worlds/api-core'; -import { AbisServiceNotSetError } from '../abis.errors'; -import { AbiNotFoundError } from '@alien-worlds/block-reader'; +import { Failure, Result, AbiService } from '@alien-worlds/aw-core'; +import { AbiNotFoundError, AbisServiceNotSetError } from '../abis.errors'; // Mock dependencies jest.mock('../abis.repository-impl'); diff --git a/src/common/abis/abis.cache.ts b/src/common/abis/abis.cache.ts index 95c2340..5e7bb4e 100644 --- a/src/common/abis/abis.cache.ts +++ b/src/common/abis/abis.cache.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { ContractEncodedAbi } from '@alien-worlds/api-core'; +import { ContractEncodedAbi } from '@alien-worlds/aw-core'; /** * A function that filters ABI entries based on the block number being greater than or equal to the start block. diff --git a/src/common/abis/abis.errors.ts b/src/common/abis/abis.errors.ts index 27ec8bc..70da4cc 100644 --- a/src/common/abis/abis.errors.ts +++ b/src/common/abis/abis.errors.ts @@ -5,3 +5,9 @@ export class AbisServiceNotSetError extends Error { ); } } + +export class AbiNotFoundError extends Error { + constructor() { + super(`ABI data not found`); + } +} diff --git a/src/common/abis/abis.repository-impl.ts b/src/common/abis/abis.repository-impl.ts index f18b494..b5bb511 100644 --- a/src/common/abis/abis.repository-impl.ts +++ b/src/common/abis/abis.repository-impl.ts @@ -7,7 +7,7 @@ import { RepositoryImpl, Result, Where, -} from '@alien-worlds/api-core'; +} from '@alien-worlds/aw-core'; import { AbisCache } from './abis.cache'; import { AbisRepository } from './abis.repository'; diff --git a/src/common/abis/abis.repository.ts b/src/common/abis/abis.repository.ts index 49529f5..569e84b 100644 --- a/src/common/abis/abis.repository.ts +++ b/src/common/abis/abis.repository.ts @@ -1,4 +1,4 @@ -import { ContractEncodedAbi, Result } from '@alien-worlds/api-core'; +import { ContractEncodedAbi, Result } from '@alien-worlds/aw-core'; /** * This class manages ContractEncodedAbi entities, providing CRUD operations and additional functionalities such as caching. diff --git a/src/common/abis/abis.ts b/src/common/abis/abis.ts index ec3bd98..b137c70 100644 --- a/src/common/abis/abis.ts +++ b/src/common/abis/abis.ts @@ -1,7 +1,12 @@ -import { AbiService, ContractEncodedAbi, Failure, Result, log } from '@alien-worlds/api-core'; -import { AbisServiceNotSetError } from './abis.errors'; +import { + AbiService, + ContractEncodedAbi, + Failure, + Result, + log, +} from '@alien-worlds/aw-core'; +import { AbiNotFoundError, AbisServiceNotSetError } from './abis.errors'; import { AbisRepository } from './abis.repository'; -import { AbiNotFoundError } from '@alien-worlds/block-reader'; /** * Represents a collection of ABIs (Application Binary Interfaces) for smart contracts. diff --git a/src/common/abis/abis.types.ts b/src/common/abis/abis.types.ts index a71d98a..c6c8201 100644 --- a/src/common/abis/abis.types.ts +++ b/src/common/abis/abis.types.ts @@ -1,4 +1,4 @@ -import { UnknownObject } from '@alien-worlds/api-core'; +import { UnknownObject } from '@alien-worlds/aw-core'; export type AbisServiceConfig = { url: string; diff --git a/src/common/block-range-scanner/block-range-scan.repository.ts b/src/common/block-range-scanner/block-range-scan.repository.ts index 21aa099..3b05fb2 100644 --- a/src/common/block-range-scanner/block-range-scan.repository.ts +++ b/src/common/block-range-scanner/block-range-scan.repository.ts @@ -1,6 +1,6 @@ import { BlockRangeScan } from './block-range-scan'; import { BlockRangeScanSource } from './block-range-scan.source'; -import { Mapper } from '@alien-worlds/api-core'; +import { Mapper } from '@alien-worlds/aw-core'; export type ScanRequest = { error?: Error; diff --git a/src/common/block-range-scanner/block-range-scan.source.ts b/src/common/block-range-scanner/block-range-scan.source.ts index 1b305a8..31bf41b 100644 --- a/src/common/block-range-scanner/block-range-scan.source.ts +++ b/src/common/block-range-scanner/block-range-scan.source.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-unsafe-return */ -import { DataSource } from '@alien-worlds/api-core'; +import { DataSource } from '@alien-worlds/aw-core'; export abstract class BlockRangeScanSource extends DataSource { public abstract startNextScan(scanKey: string): Promise; diff --git a/src/common/block-range-scanner/block-range-scan.ts b/src/common/block-range-scanner/block-range-scan.ts index 77ea5f3..78aec29 100644 --- a/src/common/block-range-scanner/block-range-scan.ts +++ b/src/common/block-range-scanner/block-range-scan.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import crypto from 'crypto'; -import { parseToBigInt, removeUndefinedProperties } from '@alien-worlds/api-core'; +import { parseToBigInt, removeUndefinedProperties } from '@alien-worlds/aw-core'; import { serialize } from 'v8'; export class BlockRangeScanParent { diff --git a/src/common/block-state/__tests__/block-state.unit.test.ts b/src/common/block-state/__tests__/block-state.unit.test.ts index eadfac7..ac7e823 100644 --- a/src/common/block-state/__tests__/block-state.unit.test.ts +++ b/src/common/block-state/__tests__/block-state.unit.test.ts @@ -1,7 +1,7 @@ -import { RepositoryImpl, Failure, Result } from '@alien-worlds/api-core'; +import { RepositoryImpl, Failure, Result } from '@alien-worlds/aw-core'; import { BlockState } from '../block-state'; -jest.mock('@alien-worlds/api-core'); +jest.mock('@alien-worlds/aw-core'); jest.mock('./block-state.types'); describe('BlockState', () => { diff --git a/src/common/block-state/block-state.ts b/src/common/block-state/block-state.ts index 6475cba..a27ae7a 100644 --- a/src/common/block-state/block-state.ts +++ b/src/common/block-state/block-state.ts @@ -6,7 +6,7 @@ import { QueryBuilders, RepositoryImpl, Result, -} from '@alien-worlds/api-core'; +} from '@alien-worlds/aw-core'; import { BlockStateModel } from './block-state.types'; /** diff --git a/src/common/common.types.ts b/src/common/common.types.ts index 2f6fdf3..8765b57 100644 --- a/src/common/common.types.ts +++ b/src/common/common.types.ts @@ -1,4 +1,4 @@ -import { ConfigVars, UnknownObject } from '@alien-worlds/api-core'; +import { ConfigVars, UnknownObject } from '@alien-worlds/aw-core'; export type DatabaseConfigBuilder = ( vars: ConfigVars, diff --git a/src/common/dependencies.ts b/src/common/dependencies.ts index ea2a5df..71c81f3 100644 --- a/src/common/dependencies.ts +++ b/src/common/dependencies.ts @@ -1,4 +1,4 @@ -import { Result } from '@alien-worlds/api-core'; +import { Result } from '@alien-worlds/aw-core'; /** * An abstract class representing a Process dependencies. diff --git a/src/common/featured/featured-contract.ts b/src/common/featured/featured-contract.ts index 0dd6589..69e97ca 100644 --- a/src/common/featured/featured-contract.ts +++ b/src/common/featured/featured-contract.ts @@ -1,4 +1,4 @@ -import { parseToBigInt } from '@alien-worlds/api-core'; +import { parseToBigInt } from '@alien-worlds/aw-core'; /** * Class representing a FeaturedContract diff --git a/src/common/featured/featured-contracts.ts b/src/common/featured/featured-contracts.ts index 4815fea..e5fcd7f 100644 --- a/src/common/featured/featured-contracts.ts +++ b/src/common/featured/featured-contracts.ts @@ -5,7 +5,7 @@ import { SmartContractService, UnknownObject, Where, -} from '@alien-worlds/api-core'; +} from '@alien-worlds/aw-core'; import { FeaturedContract } from './featured-contract'; import { FeaturedUtils } from './featured.utils'; diff --git a/src/common/featured/featured.utils.ts b/src/common/featured/featured.utils.ts index 71138ad..1018326 100644 --- a/src/common/featured/featured.utils.ts +++ b/src/common/featured/featured.utils.ts @@ -1,5 +1,5 @@ import fetch from 'node-fetch'; -import { UnknownObject } from '@alien-worlds/api-core'; +import { UnknownObject } from '@alien-worlds/aw-core'; import { FeaturedContractDataCriteria } from './featured.types'; import { existsSync, readFileSync } from 'fs'; diff --git a/src/common/processor-task-queue/processor-task-queue.ts b/src/common/processor-task-queue/processor-task-queue.ts index 79527d4..2c39011 100644 --- a/src/common/processor-task-queue/processor-task-queue.ts +++ b/src/common/processor-task-queue/processor-task-queue.ts @@ -1,4 +1,4 @@ -import { DataSource, DataSourceError, Mapper, log } from '@alien-worlds/api-core'; +import { DataSource, DataSourceError, Mapper, log } from '@alien-worlds/aw-core'; import { ProcessorTaskSource } from './processor-task.source'; import { ProcessorTask } from './processor-task'; import { ProcessorTaskModel } from './processor-task.types'; diff --git a/src/common/processor-task-queue/processor-task.source.ts b/src/common/processor-task-queue/processor-task.source.ts index f4f416d..2c7fdda 100644 --- a/src/common/processor-task-queue/processor-task.source.ts +++ b/src/common/processor-task-queue/processor-task.source.ts @@ -1,4 +1,4 @@ -import { DataSource } from '@alien-worlds/api-core'; +import { DataSource } from '@alien-worlds/aw-core'; export abstract class ProcessorTaskSource extends DataSource { public abstract nextTask(mode?: string): Promise; diff --git a/src/common/processor-task-queue/processor-task.ts b/src/common/processor-task-queue/processor-task.ts index 8ef2b4d..1afc3b2 100644 --- a/src/common/processor-task-queue/processor-task.ts +++ b/src/common/processor-task-queue/processor-task.ts @@ -3,7 +3,7 @@ import { serialize } from 'v8'; import { DeltaProcessorContentModel, ProcessorTaskError } from './processor-task.types'; import { ActionTrace } from '../types'; import { ProcessorTaskType } from './processor-task.enums'; -import { Row } from '@alien-worlds/api-core'; +import { Row } from '@alien-worlds/aw-core'; export class ProcessorTask { public static createActionProcessorTask( diff --git a/src/common/types/delta.types.ts b/src/common/types/delta.types.ts index 0eeb36a..4c06f9b 100644 --- a/src/common/types/delta.types.ts +++ b/src/common/types/delta.types.ts @@ -1,4 +1,4 @@ -import { Row } from "@alien-worlds/api-core"; +import { Row } from '@alien-worlds/aw-core'; export type Delta = { name?: string; diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.source.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.source.ts index b631975..d0bfebe 100644 --- a/src/common/unprocessed-block-queue/unprocessed-block-queue.source.ts +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.source.ts @@ -1,4 +1,4 @@ -import { DataSource } from '@alien-worlds/api-core'; +import { DataSource } from '@alien-worlds/aw-core'; export abstract class UnprocessedBlockSource extends DataSource { public abstract next(): Promise; diff --git a/src/common/unprocessed-block-queue/unprocessed-block-queue.ts b/src/common/unprocessed-block-queue/unprocessed-block-queue.ts index 60b9509..e74390e 100644 --- a/src/common/unprocessed-block-queue/unprocessed-block-queue.ts +++ b/src/common/unprocessed-block-queue/unprocessed-block-queue.ts @@ -1,17 +1,17 @@ import { + Block, DataSourceError, Failure, log, Mapper, parseToBigInt, Result, -} from '@alien-worlds/api-core'; +} from '@alien-worlds/aw-core'; import { BlockNotFoundError, DuplicateBlocksError, UnprocessedBlocksOverloadError, } from './unprocessed-block-queue.errors'; -import { Block } from '@alien-worlds/block-reader'; import { UnprocessedBlockSource } from './unprocessed-block-queue.source'; import { BlockModel } from '../types/block.types'; diff --git a/src/config/index.ts b/src/config/index.ts index 3f01753..dd5e610 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -5,7 +5,7 @@ import { BlockchainConfig, BootstrapCommandOptions, BootstrapConfig } from '../b import { ReaderCommandOptions, ReaderConfig } from '../reader'; import { ProcessorCommandOptions, ProcessorConfig } from '../processor'; import { HistoryToolsConfig } from './config.types'; -import { ConfigVars, UnknownObject, parseToBigInt } from '@alien-worlds/api-core'; +import { BlockReaderConfig, ConfigVars, UnknownObject, parseToBigInt } from '@alien-worlds/aw-core'; import { AbisConfig, AbisServiceConfig, @@ -15,9 +15,8 @@ import { ProcessorTaskQueueConfig, UnprocessedBlockQueueConfig, } from '../common'; -import { BlockReaderConfig } from '@alien-worlds/block-reader'; -import { WorkersConfig } from '@alien-worlds/workers'; -import { buildBroadcastConfig } from '@alien-worlds/broadcast'; +import { WorkersConfig } from '@alien-worlds/aw-workers'; +import { buildBroadcastConfig } from '@alien-worlds/aw-broadcast'; export * from './config.types'; diff --git a/src/filter/filter.config.ts b/src/filter/filter.config.ts index 049a6bb..7c346ab 100644 --- a/src/filter/filter.config.ts +++ b/src/filter/filter.config.ts @@ -1,6 +1,6 @@ -import { UnknownObject } from '@alien-worlds/api-core'; -import { BroadcastConfig } from '@alien-worlds/broadcast'; -import { WorkersConfig } from '@alien-worlds/workers'; +import { UnknownObject } from '@alien-worlds/aw-core'; +import { BroadcastConfig } from '@alien-worlds/aw-broadcast'; +import { WorkersConfig } from '@alien-worlds/aw-workers'; import { AbisConfig, FeaturedConfig, diff --git a/src/filter/filter.dependencies.ts b/src/filter/filter.dependencies.ts index 0e7fef3..83a983a 100644 --- a/src/filter/filter.dependencies.ts +++ b/src/filter/filter.dependencies.ts @@ -1,5 +1,5 @@ -import { Result } from '@alien-worlds/api-core'; -import { BroadcastClient } from '@alien-worlds/broadcast'; +import { Result } from '@alien-worlds/aw-core'; +import { BroadcastClient } from '@alien-worlds/aw-broadcast'; import { UnprocessedBlockQueue } from '../common/unprocessed-block-queue'; import { Dependencies } from '../common/dependencies'; import { FilterConfig } from './filter.config'; diff --git a/src/filter/filter.runner.ts b/src/filter/filter.runner.ts index 643a2ab..26eed52 100644 --- a/src/filter/filter.runner.ts +++ b/src/filter/filter.runner.ts @@ -1,6 +1,5 @@ -import { log } from '@alien-worlds/api-core'; -import { BlockJson } from '@alien-worlds/block-reader'; -import { WorkerMessage, WorkerPool } from '@alien-worlds/workers'; +import { BlockJsonModel, log } from '@alien-worlds/aw-core'; +import { WorkerMessage, WorkerPool } from '@alien-worlds/aw-workers'; import { BlockNotFoundError, UnprocessedBlockQueueReader } from '../common'; export class FilterRunner { @@ -46,7 +45,7 @@ export class FilterRunner { this.loop = false; } else { const worker = await workerPool.getWorker(); - worker.onMessage(async (message: WorkerMessage) => { + worker.onMessage(async (message: WorkerMessage) => { if (message.isTaskRejected()) { log(message.error); } else if (message.isTaskResolved() && this.transitionHandler) { diff --git a/src/filter/filter.worker-loader.dependencies.ts b/src/filter/filter.worker-loader.dependencies.ts index d2f7a1e..758ec21 100644 --- a/src/filter/filter.worker-loader.dependencies.ts +++ b/src/filter/filter.worker-loader.dependencies.ts @@ -1,7 +1,6 @@ -import { Serializer } from '@alien-worlds/api-core'; -import { ShipAbis } from '@alien-worlds/block-reader'; +import { Serializer } from '@alien-worlds/aw-core'; import { ProcessorTaskQueue } from '../common/processor-task-queue'; -import { WorkerLoaderDependencies } from '@alien-worlds/workers'; +import { WorkerLoaderDependencies } from '@alien-worlds/aw-workers'; import { FeaturedContracts } from '../common/featured'; import { Abis } from '../common'; import { FilterConfig } from './filter.config'; @@ -13,7 +12,6 @@ import { FilterConfig } from './filter.config'; export abstract class FilterWorkerLoaderDependencies extends WorkerLoaderDependencies { public processorTaskQueue: ProcessorTaskQueue; public abis: Abis; - public shipAbis: ShipAbis; public featuredContracts: FeaturedContracts; public serializer: Serializer; diff --git a/src/filter/filter.worker-loader.ts b/src/filter/filter.worker-loader.ts index fb85846..091c86c 100644 --- a/src/filter/filter.worker-loader.ts +++ b/src/filter/filter.worker-loader.ts @@ -1,4 +1,4 @@ -import { Worker, DefaultWorkerLoader } from '@alien-worlds/workers'; +import { Worker, DefaultWorkerLoader } from '@alien-worlds/aw-workers'; import { FilterSharedData } from './filter.types'; import FilterWorker from './filter.worker'; import { FilterWorkerLoaderDependencies } from './filter.worker-loader.dependencies'; diff --git a/src/filter/filter.worker.ts b/src/filter/filter.worker.ts index 36ca7ff..de6dca4 100644 --- a/src/filter/filter.worker.ts +++ b/src/filter/filter.worker.ts @@ -1,7 +1,7 @@ -import { Worker } from '@alien-worlds/workers'; -import { AbiNotFoundError, ShipAbis } from '@alien-worlds/block-reader'; +import { Worker } from '@alien-worlds/aw-workers'; import { FilterSharedData } from './filter.types'; import { + AbiNotFoundError, Abis, BlockModel, DeltaByName, @@ -12,12 +12,11 @@ import { TraceByName, isSetAbiAction, } from '../common'; -import { Serializer, log, parseToBigInt } from '@alien-worlds/api-core'; +import { Serializer, log, parseToBigInt } from '@alien-worlds/aw-core'; export default class FilterWorker extends Worker { constructor( protected dependencies: { - shipAbis: ShipAbis; abis: Abis; featuredContracts: FeaturedContracts; processorTaskQueue: ProcessorTaskQueue; @@ -130,7 +129,7 @@ export default class FilterWorker extends Worker { for (const [type, delta] of deltas) { const { name, rows } = delta; - + for (const row of rows) { const info = serializer.deserializeTableRow(row); @@ -203,19 +202,13 @@ export default class FilterWorker extends Worker { public async run(json: BlockModel): Promise { try { const { - dependencies: { serializer, shipAbis, processorTaskQueue }, + dependencies: { serializer, processorTaskQueue }, } = this; - const { content: abi, failure } = await shipAbis.getAbi(json.abi_version); - - if (failure) { - log('SHiP Abi not found.'); - this.reject(failure.error); - } const deserializedBlock = serializer.deserializeBlock< BlockModel, BlockModel - >(json, abi); + >(json); const { this_block: { block_num }, } = deserializedBlock; diff --git a/src/filter/start-filter.ts b/src/filter/start-filter.ts index 0f61941..062cafc 100644 --- a/src/filter/start-filter.ts +++ b/src/filter/start-filter.ts @@ -9,9 +9,9 @@ import { FilterBroadcastMessage } from '../broadcast/messages/filter-broadcast.m import { buildFilterConfig } from '../config'; import { filterCommand } from './filter.command'; import { filterWorkerLoaderPath } from './filter.consts'; -import { log, ConfigVars } from '@alien-worlds/api-core'; -import { BroadcastMessage } from '@alien-worlds/broadcast'; -import { WorkerPool } from '@alien-worlds/workers'; +import { log, ConfigVars } from '@alien-worlds/aw-core'; +import { BroadcastMessage } from '@alien-worlds/aw-broadcast'; +import { WorkerPool } from '@alien-worlds/aw-workers'; import { FilterConfig } from './filter.config'; import { FilterDependencies } from './filter.dependencies'; diff --git a/src/index.ts b/src/index.ts index 5fed61b..5eeaddb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,6 @@ export * from './broadcast'; export * from './processor'; export * from './reader'; -export * from '@alien-worlds/api-core'; -export * from '@alien-worlds/block-reader'; -export * from '@alien-worlds/broadcast'; -export * from '@alien-worlds/workers'; +export * from '@alien-worlds/aw-core'; +export * from '@alien-worlds/aw-broadcast'; +export * from '@alien-worlds/aw-workers'; diff --git a/src/processor/processor.config.ts b/src/processor/processor.config.ts index 269e6f0..b2b7567 100644 --- a/src/processor/processor.config.ts +++ b/src/processor/processor.config.ts @@ -1,7 +1,7 @@ -import { BroadcastConfig } from '@alien-worlds/broadcast'; -import { WorkersConfig } from '@alien-worlds/workers'; +import { BroadcastConfig } from '@alien-worlds/aw-broadcast'; +import { WorkersConfig } from '@alien-worlds/aw-workers'; import { FeaturedConfig, ProcessorMatcher, ProcessorTaskQueueConfig } from '../common'; -import { UnknownObject } from '@alien-worlds/api-core'; +import { UnknownObject } from '@alien-worlds/aw-core'; export type ProcessorConfig = { broadcast: BroadcastConfig; diff --git a/src/processor/processor.dependencies.ts b/src/processor/processor.dependencies.ts index e0df96d..a148df6 100644 --- a/src/processor/processor.dependencies.ts +++ b/src/processor/processor.dependencies.ts @@ -1,4 +1,4 @@ -import { Result, Serializer } from '@alien-worlds/api-core'; +import { Result, Serializer } from '@alien-worlds/aw-core'; import { Dependencies } from '../common/dependencies'; import { ContractDeltaMatchCriteria, @@ -6,7 +6,7 @@ import { Featured, } from '../common/featured'; import { ProcessorTaskQueue } from '../common/processor-task-queue'; -import { BroadcastClient } from '@alien-worlds/broadcast'; +import { BroadcastClient } from '@alien-worlds/aw-broadcast'; import { ProcessorAddons, ProcessorConfig } from './processor.config'; import { DatabaseConfigBuilder } from '../common'; diff --git a/src/processor/processor.model.factory.ts b/src/processor/processor.model.factory.ts index 46ab876..d4c7a4a 100644 --- a/src/processor/processor.model.factory.ts +++ b/src/processor/processor.model.factory.ts @@ -1,4 +1,4 @@ -import { Serializer } from '@alien-worlds/api-core'; +import { Serializer } from '@alien-worlds/aw-core'; import { ActionProcessorContentModel, DeltaProcessorContentModel, diff --git a/src/processor/processor.runner.ts b/src/processor/processor.runner.ts index 9f375de..53d64a8 100644 --- a/src/processor/processor.runner.ts +++ b/src/processor/processor.runner.ts @@ -1,5 +1,5 @@ -import { Serializer, log } from '@alien-worlds/api-core'; -import { WorkerPool, WorkerMessage } from '@alien-worlds/workers'; +import { Serializer, log } from '@alien-worlds/aw-core'; +import { WorkerPool, WorkerMessage } from '@alien-worlds/aw-workers'; import { Featured, ContractTraceMatchCriteria, @@ -21,9 +21,8 @@ export class ProcessorRunner { protected featuredDeltas: Featured, protected workerPool: WorkerPool, protected queue: ProcessorTaskQueue, - serializer: Serializer, + serializer: Serializer ) { - this.modelFactory = new ProcessorModelFactory(serializer); this.interval = setInterval(async () => { diff --git a/src/processor/processor.worker-loader.dependencies.ts b/src/processor/processor.worker-loader.dependencies.ts index 36ef7ac..bc623cc 100644 --- a/src/processor/processor.worker-loader.dependencies.ts +++ b/src/processor/processor.worker-loader.dependencies.ts @@ -1,4 +1,4 @@ -import { WorkerLoaderDependencies } from '@alien-worlds/workers'; +import { WorkerLoaderDependencies } from '@alien-worlds/aw-workers'; import { ProcessorConfig } from './processor.config'; /** diff --git a/src/processor/processor.worker-loader.ts b/src/processor/processor.worker-loader.ts index 4e54324..b60e4fe 100644 --- a/src/processor/processor.worker-loader.ts +++ b/src/processor/processor.worker-loader.ts @@ -1,6 +1,6 @@ -import { Worker, DefaultWorkerLoader, WorkerContainer } from '@alien-worlds/workers'; +import { Worker, DefaultWorkerLoader, WorkerContainer } from '@alien-worlds/aw-workers'; import { ProcessorSharedData } from './processor.types'; -import { Container } from '@alien-worlds/api-core'; +import { Container } from '@alien-worlds/aw-core'; import { ProcessorWorkerLoaderDependencies } from './processor.worker-loader.dependencies'; export default class ProcessorWorkerLoader extends DefaultWorkerLoader< diff --git a/src/processor/processors/action-trace.processor.ts b/src/processor/processors/action-trace.processor.ts index 0dc836f..da61556 100644 --- a/src/processor/processors/action-trace.processor.ts +++ b/src/processor/processors/action-trace.processor.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { Processor } from './processor'; import { ActionTraceProcessorModel, ProcessorSharedData } from '../processor.types'; -import { Container, Serializer } from '@alien-worlds/api-core'; +import { Container, Serializer } from '@alien-worlds/aw-core'; export class ActionTraceProcessor< DataType = unknown, diff --git a/src/processor/processors/delta.processor.ts b/src/processor/processors/delta.processor.ts index 78b1a1f..15fc313 100644 --- a/src/processor/processors/delta.processor.ts +++ b/src/processor/processors/delta.processor.ts @@ -1,6 +1,6 @@ import { Processor } from './processor'; import { DeltaProcessorModel, ProcessorSharedData } from '../processor.types'; -import { Container, Serializer } from '@alien-worlds/api-core'; +import { Container, Serializer } from '@alien-worlds/aw-core'; export class DeltaProcessor< DataType = unknown, diff --git a/src/processor/processors/processor.ts b/src/processor/processors/processor.ts index d232964..387fe5d 100644 --- a/src/processor/processors/processor.ts +++ b/src/processor/processors/processor.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { Worker } from '@alien-worlds/workers'; +import { Worker } from '@alien-worlds/aw-workers'; import { ProcessorSharedData } from '../processor.types'; export class Processor< diff --git a/src/processor/start-processor.ts b/src/processor/start-processor.ts index 5e39f70..a723cb7 100644 --- a/src/processor/start-processor.ts +++ b/src/processor/start-processor.ts @@ -8,9 +8,9 @@ import { import { processorCommand } from './processor.command'; import { buildProcessorConfig } from '../config'; import { ProcessorCommandOptions } from './processor.types'; -import { log, ConfigVars } from '@alien-worlds/api-core'; -import { BroadcastMessage } from '@alien-worlds/broadcast'; -import { WorkerPool } from '@alien-worlds/workers'; +import { log, ConfigVars } from '@alien-worlds/aw-core'; +import { BroadcastMessage } from '@alien-worlds/aw-broadcast'; +import { WorkerPool } from '@alien-worlds/aw-workers'; import { ProcessorConfig, ProcessorAddons } from './processor.config'; import { ProcessorDependencies } from './processor.dependencies'; diff --git a/src/reader/reader.config.ts b/src/reader/reader.config.ts index c981220..d2ffbde 100644 --- a/src/reader/reader.config.ts +++ b/src/reader/reader.config.ts @@ -1,8 +1,7 @@ -import { BlockReaderConfig } from '@alien-worlds/block-reader'; -import { BroadcastConfig } from '@alien-worlds/broadcast'; -import { WorkersConfig } from '@alien-worlds/workers'; +import { BroadcastConfig } from '@alien-worlds/aw-broadcast'; +import { WorkersConfig } from '@alien-worlds/aw-workers'; import { BlockRangeScanConfig, UnprocessedBlockQueueConfig } from '../common'; -import { UnknownObject } from '@alien-worlds/api-core'; +import { BlockReaderConfig, UnknownObject } from '@alien-worlds/aw-core'; export type ReaderConfig = { broadcast?: BroadcastConfig; diff --git a/src/reader/reader.dependencies.ts b/src/reader/reader.dependencies.ts index e3d5413..2a2fec2 100644 --- a/src/reader/reader.dependencies.ts +++ b/src/reader/reader.dependencies.ts @@ -1,5 +1,5 @@ -import { BroadcastClient } from '@alien-worlds/broadcast'; -import { Result } from '@alien-worlds/api-core'; +import { BroadcastClient } from '@alien-worlds/aw-broadcast'; +import { Result } from '@alien-worlds/aw-core'; import { Dependencies } from '../common/dependencies'; import { BlockRangeScanner, DatabaseConfigBuilder } from '../common'; import { ReaderConfig } from './reader.config'; diff --git a/src/reader/reader.ts b/src/reader/reader.ts index 6a8b04f..a98be8e 100644 --- a/src/reader/reader.ts +++ b/src/reader/reader.ts @@ -1,8 +1,8 @@ import { ReadCompleteData, ReadTaskData } from './reader.types'; import { FilterBroadcastMessage } from '../broadcast/messages'; -import { log } from '@alien-worlds/api-core'; -import { BroadcastClient } from '@alien-worlds/broadcast'; -import { WorkerPool, WorkerMessage } from '@alien-worlds/workers'; +import { log } from '@alien-worlds/aw-core'; +import { BroadcastClient } from '@alien-worlds/aw-broadcast'; +import { WorkerPool, WorkerMessage } from '@alien-worlds/aw-workers'; import { BlockRangeScanner, Mode } from '../common'; export class Reader { diff --git a/src/reader/reader.worker-loader.dependencies.ts b/src/reader/reader.worker-loader.dependencies.ts index 1e8894d..b6f565f 100644 --- a/src/reader/reader.worker-loader.dependencies.ts +++ b/src/reader/reader.worker-loader.dependencies.ts @@ -1,8 +1,8 @@ -import { WorkerLoaderDependencies } from '@alien-worlds/workers'; -import { BlockReader } from '@alien-worlds/block-reader'; +import { WorkerLoaderDependencies } from '@alien-worlds/aw-workers'; import { UnprocessedBlockQueue } from '../common/unprocessed-block-queue'; import { BlockRangeScanner, BlockState } from '../common'; import { ReaderConfig } from './reader.config'; +import { BlockReader } from '@alien-worlds/aw-core'; /** * An abstract class representing a ReaderWorkerLoader dependencies. diff --git a/src/reader/reader.worker-loader.ts b/src/reader/reader.worker-loader.ts index 27c37da..a204f03 100644 --- a/src/reader/reader.worker-loader.ts +++ b/src/reader/reader.worker-loader.ts @@ -1,6 +1,6 @@ -import { Worker, DefaultWorkerLoader } from '@alien-worlds/workers'; +import { Worker, DefaultWorkerLoader } from '@alien-worlds/aw-workers'; import { ReaderWorkerLoaderDependencies } from './reader.worker-loader.dependencies'; -import { log } from '@alien-worlds/api-core'; +import { log } from '@alien-worlds/aw-core'; import ReaderWorker, { ReaderSharedData } from './reader.worker'; export default class ReaderWorkerLoader extends DefaultWorkerLoader< diff --git a/src/reader/reader.worker.ts b/src/reader/reader.worker.ts index 20356d0..49198c9 100644 --- a/src/reader/reader.worker.ts +++ b/src/reader/reader.worker.ts @@ -1,7 +1,7 @@ -import { Worker } from '@alien-worlds/workers'; +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { Worker } from '@alien-worlds/aw-workers'; import { ReaderConfig } from './reader.config'; -import { log, parseToBigInt } from '@alien-worlds/api-core'; -import { BlockReader } from '@alien-worlds/block-reader'; +import { BlockReader, log, parseToBigInt } from '@alien-worlds/aw-core'; import { UnprocessedBlockQueue, BlockState, BlockRangeScanner, Mode } from '../common'; export type ReaderSharedData = { diff --git a/src/reader/start-reader.ts b/src/reader/start-reader.ts index fa0bcc9..71962e3 100644 --- a/src/reader/start-reader.ts +++ b/src/reader/start-reader.ts @@ -10,9 +10,9 @@ import { Reader } from './reader'; import { readerCommand } from './reader.command'; import { buildReaderConfig } from '../config'; import { readerWorkerLoaderPath } from './reader.consts'; -import { log, ConfigVars } from '@alien-worlds/api-core'; -import { BroadcastMessage } from '@alien-worlds/broadcast'; -import { WorkerPool } from '@alien-worlds/workers'; +import { log, ConfigVars } from '@alien-worlds/aw-core'; +import { BroadcastMessage } from '@alien-worlds/aw-broadcast'; +import { WorkerPool } from '@alien-worlds/aw-workers'; import { Mode } from '../common'; import { ReaderConfig } from './reader.config'; import { ReaderDependencies } from './reader.dependencies'; diff --git a/tutorials/extending-history-tools.md b/tutorials/extending-history-tools.md index e73902d..3834fa4 100644 --- a/tutorials/extending-history-tools.md +++ b/tutorials/extending-history-tools.md @@ -2,7 +2,7 @@ In this tutorial, we'll briefly go over how to extend the history tools and possibly change the resources. Such a situation may occur when the basic construction of the tools does not meet all expectations. in our case, this happened when creating the leaderboard api. We could rely on the available architecture and put all the work of updating the leaderboards in processors, but we wanted to separate this process from others and leave the standard processes (boot, filter, reader, processor) unchanged. We needed another process whose task would be to update the leaderboard database and which would be able to scale properly to increase work efficiency. We called the new process writer and its role is to download records in the leaderboard_updates collection and create updates in the target leaderboar collections (of which we have several). On this own example, we will describe ways to extend the capabilities of history tools. -First, decide how you want to extend the tools. You can download the history-tools repository and add your own implementations of components responsible for communication with the blockchain and the database. The second method is to create a new repository and import history tools along with [Starter Kit](https://github.com/Alien-Worlds/history-tools-starter-kit). Since we will continue to use mongodb and eos we will choose second option. +First, decide how you want to extend the tools. You can download the history-tools repository and add your own implementations of components responsible for communication with the blockchain and the database. The second method is to create a new repository and import history tools along with [Starter Kit](https://github.com/Alien-Worlds/aw-history-starter-kit). Since we will continue to use mongodb and eos we will choose second option. [Back to Readme](../README.md) @@ -51,8 +51,7 @@ Your `package.json` file should look similar to this: }, ... "dependencies": { - "@alien-worlds/api-history-tools": "^0.0.136", - "@alien-worlds/history-tools-default-dependencies": "^0.0.27", + "@alien-worlds/aw-history": "^0.0.136", ... all your contract packages and other typescript dependencies } } @@ -188,7 +187,7 @@ In the `bootstrap` directory, create an `index.ts` file and add the following co import { startBootstrap, DefaultBootstrapDependencies, -} from '@alien-worlds/history-tools-starter-kit'; +} from '@alien-worlds/aw-history-starter-kit'; import path from 'path'; startBootstrap( @@ -203,7 +202,7 @@ startBootstrap( In the `broadcast` directory, create an `index.ts` file and add the following content: ```typescript -import { startBroadcast } from '@alien-worlds/history-tools-starter-kit'; +import { startBroadcast } from '@alien-worlds/aw-history-starter-kit'; startBroadcast(); ``` @@ -216,7 +215,7 @@ In the `reader` directory, create an `index.ts` file and add the following conte import { startReader, DefaultReaderDependencies, -} from '@alien-worlds/history-tools-starter-kit'; +} from '@alien-worlds/aw-history-starter-kit'; startReader(process.argv, new DefaultReaderDependencies()); ``` @@ -229,7 +228,7 @@ In the `filter` directory, create an `index.ts` file and add the following conte import { startFilter, DefaultFilterDependencies, -} from '@alien-worlds/history-tools-starter-kit'; +} from '@alien-worlds/aw-history-starter-kit'; import path from 'path'; startFilter( @@ -247,7 +246,7 @@ In the `processor` directory, create an `index.ts` file and add the following co import { startProcessor, DefaultProcessorDependencies, -} from '@alien-worlds/history-tools-starter-kit'; +} from '@alien-worlds/aw-history-starter-kit'; import path from 'path'; startProcessor( @@ -278,7 +277,7 @@ export { NotifyWorldTraceProcessor } from './notify-world.trace-processor'; ... // ./processors/notify-world.trace-processor.ts -import { ActionTraceProcessor, ProcessorTaskModel } from '@alien-worlds/history-tools-starter-kit'; +import { ActionTraceProcessor, ProcessorTaskModel } from '@alien-worlds/aw-history-starter-kit'; export class NotifyWorldTraceProcessor extends ActionTraceProcessor { public async run(model: ProcessorTaskModel): Promise { @@ -311,6 +310,6 @@ That's it, the examples shown are of course partial but you should get a general ## Using other resources -If, apart from simply adding new processes or modifying existing ones, you want to use another database (replacing the default mongodb) or another blockchain (reading tools). You should check [Starter Kit](https://github.com/Alien-Worlds/history-tools-starter-kit) implementation and write your own based on all interfaces used in [Api Core](https://github.com/Alien-Worlds/api-core), **History Tools**. Check the contents of this package and replace e.g. all mongo.\* components with your own. Theoretically, if you follow the interface guidelines, everything should work fine, including serialization and block reader. The **kit** prepared in this way should be imported into your history tools implementation and then follow the guidelines mentioned above. +If, apart from simply adding new processes or modifying existing ones, you want to use another database (replacing the default mongodb) or another blockchain (reading tools). You should check [Starter Kit](https://github.com/Alien-Worlds/aw-history-starter-kit) implementation and write your own based on all interfaces used in [Api Core](https://github.com/Alien-Worlds/api-core), **History Tools**. Check the contents of this package and replace e.g. all mongo.\* components with your own. Theoretically, if you follow the interface guidelines, everything should work fine, including serialization and block reader. The **kit** prepared in this way should be imported into your history tools implementation and then follow the guidelines mentioned above. Remember, if your **kit** works and meets all requirements, it's worth thinking about sharing it with other users. More _starter-kit_ type repositories may be useful and maybe more users will benefit from your work. Good luck! diff --git a/tutorials/what-is-featured-content.md b/tutorials/what-is-featured-content.md index 0e6c463..c432f07 100644 --- a/tutorials/what-is-featured-content.md +++ b/tutorials/what-is-featured-content.md @@ -117,7 +117,7 @@ _Note: The current solution may change as we are considering different alternati The `FeaturedContracts` is a repository containing the contract data you put in the JSON file. The `FeaturedUtils.readFeaturedContracts` function can be used to extract the names of all contracts and retrieve their data via `SmartContractService`. -To create `FeaturedContracts`, you can use the `FeaturedContractsCreator` available in the [Starter Kit](https://github.com/Alien-Worlds/history-tools-starter-kit). If you want to use other sources and there is no suitable KIT for them, check the wizard implementation from the starter kit and implement it according to your needs. +To create `FeaturedContracts`, you can use the `FeaturedContractsCreator` available in the [Starter Kit](https://github.com/Alien-Worlds/aw-history-starter-kit). If you want to use other sources and there is no suitable KIT for them, check the wizard implementation from the starter kit and implement it according to your needs. ### FeaturedUtils diff --git a/yarn.lock b/yarn.lock index e31f392..032d210 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,34 +2,27 @@ # yarn lockfile v1 -"@alien-worlds/api-core@^0.0.158": - version "0.0.158" - resolved "https://npm.pkg.github.com/download/@alien-worlds/api-core/0.0.158/92cb2bb3a2168e07fa7a8e240895f5133105e0ee#92cb2bb3a2168e07fa7a8e240895f5133105e0ee" - integrity sha512-PIrmouUmomBUTZ1tkNKzU/0cHcvY9L/puAYljvnDlwZ5yAJECgbgVuxEqlQVbMne2kXQwW0c1U7GCfP877Zt6w== +"@alien-worlds/aw-broadcast@^0.0.2": + version "0.0.2" + resolved "https://npm.pkg.github.com/download/@alien-worlds/aw-broadcast/0.0.2/ae8f1ae3485c3c1d0581dc06999cbfcd342803de#ae8f1ae3485c3c1d0581dc06999cbfcd342803de" + integrity sha512-I2X5akaAfvARklvq1fCu6syzEUtBHgyc45wbZKQ3GL7LVP6UAbU765GGR1vFQ+dA3r3wMEe9CdXNKVnZMv7+pA== + dependencies: + "@alien-worlds/aw-core" "^0.0.2" + nanoid "^3.0.0" + +"@alien-worlds/aw-core@^0.0.2": + version "0.0.2" + resolved "https://npm.pkg.github.com/download/@alien-worlds/aw-core/0.0.2/a0ff00b1c50a06173e99a1185af0428a24b4a3ac#a0ff00b1c50a06173e99a1185af0428a24b4a3ac" + integrity sha512-Ld5QrJVhtFqvmgxFYNQUlojPBp4AEw+iqPJ8xX00pFJ2shUO9wJk0cVPMP1a7KADkCYirA9AxPn45C8CXcHKjA== dependencies: inversify "^6.0.1" node-fetch "2.6.6" reflect-metadata "^0.1.13" -"@alien-worlds/block-reader@^0.0.18": - version "0.0.18" - resolved "https://npm.pkg.github.com/download/@alien-worlds/block-reader/0.0.18/f1e0fcb9fde2d25cc2864c5080605bd11daf32e2#f1e0fcb9fde2d25cc2864c5080605bd11daf32e2" - integrity sha512-h8bCBqX8kX4FInyXB9PWHoZf+8urB7/8LsbnOVTjPoAYWdb1uRSV+VPIviGot1xgyV71HLv9ELpweCMqitOIug== - dependencies: - "@alien-worlds/api-core" "^0.0.158" - -"@alien-worlds/broadcast@^0.0.13": - version "0.0.13" - resolved "https://npm.pkg.github.com/download/@alien-worlds/broadcast/0.0.13/03327b557e880e8848a8114f0981d1398c434d71#03327b557e880e8848a8114f0981d1398c434d71" - integrity sha512-HPrvX5OZ/hYyGn/EtmMITLASH4J0OnwuBDoWaD/TYUAvUJ5ZnJutbDDEIekmzK1SekXuOHD+vKTNfOhP8sK8kg== - dependencies: - "@alien-worlds/api-core" "^0.0.158" - nanoid "^3.0.0" - -"@alien-worlds/workers@^0.0.7": - version "0.0.7" - resolved "https://npm.pkg.github.com/download/@alien-worlds/workers/0.0.7/561e152e2c9f3d3ea5777ab413aa50023549006a#561e152e2c9f3d3ea5777ab413aa50023549006a" - integrity sha512-gcAkRpezhsyVHdFbXz7FpXbPd/PAANPyNe2H4su5J/gGrKuYPfoH6AX7elgSHciq9UPVFRkJDu8yWCYi4ARiXA== +"@alien-worlds/aw-workers@^0.0.2": + version "0.0.2" + resolved "https://npm.pkg.github.com/download/@alien-worlds/aw-workers/0.0.2/38b0f98ca4327d71df0e5b21a59b7a13b760a654#38b0f98ca4327d71df0e5b21a59b7a13b760a654" + integrity sha512-Sdm5OI7lTJbBXXxXh3bGsc2fB/I6v55hZKFY8omPIi68M+P9xlfqJDU8AoZ+zfb4G9iH60Mn6kiLLNe0yKqeBQ== dependencies: async "^3.2.4" ts-node "^10.9.1" From 65abc54ec60113887cc6945eb45c7662e49c4cec Mon Sep 17 00:00:00 2001 From: rkamysz Date: Thu, 27 Jul 2023 22:55:07 +0200 Subject: [PATCH 096/107] 0.0.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 902d3a8..af98545 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/aw-history", - "version": "0.0.0", + "version": "0.0.1", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 146cc96a954ec53ef69243a2e43e4148fc5f2e5f Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 28 Jul 2023 16:28:08 +0200 Subject: [PATCH 097/107] update --- package.json | 4 +- .../domain/models/list-actions.input.ts | 82 ++++++------------- .../domain/models/list-actions.output.ts | 24 ++---- .../actions/routes/list-actions.route-io.ts | 81 ++++++++++++++++++ .../actions/routes/list-actions.route.ts | 17 +--- yarn.lock | 18 ++-- 6 files changed, 129 insertions(+), 97 deletions(-) create mode 100644 src/api/endpoints/actions/routes/list-actions.route-io.ts diff --git a/package.json b/package.json index af98545..558a409 100644 --- a/package.json +++ b/package.json @@ -34,8 +34,8 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/aw-broadcast": "^0.0.2", - "@alien-worlds/aw-core": "^0.0.2", + "@alien-worlds/aw-broadcast": "^0.0.3", + "@alien-worlds/aw-core": "^0.0.3", "@alien-worlds/aw-workers": "^0.0.2", "async": "^3.2.4", "commander": "^10.0.1", diff --git a/src/api/endpoints/actions/domain/models/list-actions.input.ts b/src/api/endpoints/actions/domain/models/list-actions.input.ts index 26df54e..c301031 100644 --- a/src/api/endpoints/actions/domain/models/list-actions.input.ts +++ b/src/api/endpoints/actions/domain/models/list-actions.input.ts @@ -1,66 +1,14 @@ -import { ListActionsQueryParams } from '../actions.types'; -import { Request, parseToBigInt } from '@alien-worlds/aw-core'; +import { IO, UnknownObject } from '@alien-worlds/aw-core'; /** * @class */ -export class ListActionsInput { - /** - * - * @param {ListActionsRequestDto} dto - * @returns {ListActionsInput} - */ - public static fromRequest( - request: Request - ): ListActionsInput { - const { - query: { contracts, names, accounts, from, to, limit, offset, block_numbers }, - } = request; - - let fromBlock: bigint; - let toBlock: bigint; - let fromDate: Date; - let toDate: Date; - let blockNumbers = []; - - if (from) { - if (/^[0-9]+$/.test(from)) { - fromBlock = parseToBigInt(from); - } else { - fromDate = new Date(from); - } - } - - if (to) { - if (/^[0-9]+$/.test(to)) { - toBlock = parseToBigInt(to); - } else { - toDate = new Date(to); - } - } - - if (block_numbers) { - blockNumbers = block_numbers.split(',').map(parseToBigInt); - } - - return new ListActionsInput( - contracts ? contracts.split(',') : [], - names ? names.split(',') : [], - accounts ? accounts.split(',') : [], - fromBlock, - toBlock, - fromDate, - toDate, - blockNumbers, - offset || 0, - limit || 10 - ); - } +export class ListActionsInput implements IO { /** * * @constructor * @private */ - private constructor( + constructor( public readonly contracts: string[], public readonly names: string[], public readonly accounts: string[], @@ -72,4 +20,28 @@ export class ListActionsInput { public readonly offset: number, public readonly limit: number ) {} + + public toJSON(): UnknownObject { + const { + contracts, + names, + accounts, + blockNumbers, + startBlock, + endBlock, + offset, + limit, + } = this; + + return { + contracts, + names, + accounts, + block_numbers: blockNumbers.map(blockNumber => blockNumber.toString()), + from: startBlock.toString(), + to: endBlock.toString(), + offset, + limit, + }; + } } diff --git a/src/api/endpoints/actions/domain/models/list-actions.output.ts b/src/api/endpoints/actions/domain/models/list-actions.output.ts index 5dd06da..f70aeeb 100644 --- a/src/api/endpoints/actions/domain/models/list-actions.output.ts +++ b/src/api/endpoints/actions/domain/models/list-actions.output.ts @@ -1,29 +1,21 @@ -import { ContractAction, Result } from '@alien-worlds/aw-core'; +import { ContractAction, IO, Result, UnknownObject } from '@alien-worlds/aw-core'; -export class ListActionsOutput { +export class ListActionsOutput implements IO { public static create(result: Result): ListActionsOutput { return new ListActionsOutput(result); } - private constructor(public readonly result: Result) {} + constructor(public readonly result: Result) {} - public toResponse() { + toJSON(): UnknownObject { const { result } = this; + if (result.isFailure) { - const { - failure: { error }, - } = result; - if (error) { - return { - status: 500, - body: [], - }; - } + return {}; } - + return { - status: 200, - body: result.content + result: result.content.map(r => r.toJSON()), }; } } diff --git a/src/api/endpoints/actions/routes/list-actions.route-io.ts b/src/api/endpoints/actions/routes/list-actions.route-io.ts new file mode 100644 index 0000000..7df8f46 --- /dev/null +++ b/src/api/endpoints/actions/routes/list-actions.route-io.ts @@ -0,0 +1,81 @@ +import { + IO, + Response, + RouteIO, + UnknownObject, + Request, + parseToBigInt, +} from '@alien-worlds/aw-core'; +import { ListActionsInput } from '../domain/models/list-actions.input'; +import { ListActionsQueryParams } from '../domain/actions.types'; +import { ListActionsOutput } from '../domain/models/list-actions.output'; + +export class ListActionsRouteIO implements RouteIO { + public toResponse(output: ListActionsOutput): Response { + const { result } = output; + + if (result.isFailure) { + const { + failure: { error }, + } = result; + if (error) { + return { + status: 500, + body: [], + }; + } + } + + return { + status: 200, + body: output.toJSON(), + }; + } + + public fromRequest( + request: Request + ): ListActionsInput { + const { + query: { contracts, names, accounts, from, to, limit, offset, block_numbers }, + } = request; + + let fromBlock: bigint; + let toBlock: bigint; + let fromDate: Date; + let toDate: Date; + let blockNumbers = []; + + if (from) { + if (/^[0-9]+$/.test(from)) { + fromBlock = parseToBigInt(from); + } else { + fromDate = new Date(from); + } + } + + if (to) { + if (/^[0-9]+$/.test(to)) { + toBlock = parseToBigInt(to); + } else { + toDate = new Date(to); + } + } + + if (block_numbers) { + blockNumbers = block_numbers.split(',').map(parseToBigInt); + } + + return new ListActionsInput( + contracts ? contracts.split(',') : [], + names ? names.split(',') : [], + accounts ? accounts.split(',') : [], + fromBlock, + toBlock, + fromDate, + toDate, + blockNumbers, + offset || 0, + limit || 10 + ); + } +} diff --git a/src/api/endpoints/actions/routes/list-actions.route.ts b/src/api/endpoints/actions/routes/list-actions.route.ts index 079f7fd..e36cfd4 100644 --- a/src/api/endpoints/actions/routes/list-actions.route.ts +++ b/src/api/endpoints/actions/routes/list-actions.route.ts @@ -1,25 +1,12 @@ import { GetRoute, RouteHandler } from '@alien-worlds/aw-core'; -import { ListActionsInput } from '../domain/models/list-actions.input'; -import { ListActionsOutput } from '../domain/models/list-actions.output'; +import { ListActionsRouteIO } from './list-actions.route-io'; -/*imports*/ - -/** - * @class - * - * - */ export class ListActionsRoute extends GetRoute { public static create(handler: RouteHandler) { return new ListActionsRoute(handler); } private constructor(handler: RouteHandler) { - super('actions', handler, { - hooks: { - pre: ListActionsInput.fromRequest, - post: (output: ListActionsOutput) => output.toResponse(), - }, - }); + super('actions', handler, new ListActionsRouteIO()); } } diff --git a/yarn.lock b/yarn.lock index 032d210..212fb35 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,18 +2,18 @@ # yarn lockfile v1 -"@alien-worlds/aw-broadcast@^0.0.2": - version "0.0.2" - resolved "https://npm.pkg.github.com/download/@alien-worlds/aw-broadcast/0.0.2/ae8f1ae3485c3c1d0581dc06999cbfcd342803de#ae8f1ae3485c3c1d0581dc06999cbfcd342803de" - integrity sha512-I2X5akaAfvARklvq1fCu6syzEUtBHgyc45wbZKQ3GL7LVP6UAbU765GGR1vFQ+dA3r3wMEe9CdXNKVnZMv7+pA== +"@alien-worlds/aw-broadcast@^0.0.3": + version "0.0.3" + resolved "https://npm.pkg.github.com/download/@alien-worlds/aw-broadcast/0.0.3/ee62d2dccf49ae3da426e8c92ddf01520a677616#ee62d2dccf49ae3da426e8c92ddf01520a677616" + integrity sha512-L254cFKKRey6ni68ftE1w0rRPtz5nedUG1lB4sMe+Q5e0wXDyMLGi09dlAUN6PBBQQDCW6BDiD5pyTGgk1I5SA== dependencies: - "@alien-worlds/aw-core" "^0.0.2" + "@alien-worlds/aw-core" "^0.0.3" nanoid "^3.0.0" -"@alien-worlds/aw-core@^0.0.2": - version "0.0.2" - resolved "https://npm.pkg.github.com/download/@alien-worlds/aw-core/0.0.2/a0ff00b1c50a06173e99a1185af0428a24b4a3ac#a0ff00b1c50a06173e99a1185af0428a24b4a3ac" - integrity sha512-Ld5QrJVhtFqvmgxFYNQUlojPBp4AEw+iqPJ8xX00pFJ2shUO9wJk0cVPMP1a7KADkCYirA9AxPn45C8CXcHKjA== +"@alien-worlds/aw-core@^0.0.3": + version "0.0.3" + resolved "https://npm.pkg.github.com/download/@alien-worlds/aw-core/0.0.3/1b5af094fced3d97824d909190d446c116aa6bec#1b5af094fced3d97824d909190d446c116aa6bec" + integrity sha512-5aWRA7svi9qWEZO0ROJNUNGnw/KftSPLkcekCjm5ecB0ufMo01eI68A+RlufuCbLaeTKH8K8pKWFgycSxGRBwQ== dependencies: inversify "^6.0.1" node-fetch "2.6.6" From 990a303c0b693a02c0fbd7822b86c7603cbd3665 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Fri, 28 Jul 2023 16:28:32 +0200 Subject: [PATCH 098/107] 0.0.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 558a409..6b76c8c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/aw-history", - "version": "0.0.1", + "version": "0.0.2", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From dbf59c43b23787e63af635e8365a91ac80dd44a3 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 31 Jul 2023 14:00:07 +0200 Subject: [PATCH 099/107] update --- package.json | 4 ++-- .../actions/routes/list-actions.route.ts | 4 +++- yarn.lock | 18 +++++++++--------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index 6b76c8c..87a3e9d 100644 --- a/package.json +++ b/package.json @@ -34,8 +34,8 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/aw-broadcast": "^0.0.3", - "@alien-worlds/aw-core": "^0.0.3", + "@alien-worlds/aw-broadcast": "^0.0.5", + "@alien-worlds/aw-core": "^0.0.6", "@alien-worlds/aw-workers": "^0.0.2", "async": "^3.2.4", "commander": "^10.0.1", diff --git a/src/api/endpoints/actions/routes/list-actions.route.ts b/src/api/endpoints/actions/routes/list-actions.route.ts index e36cfd4..3e10557 100644 --- a/src/api/endpoints/actions/routes/list-actions.route.ts +++ b/src/api/endpoints/actions/routes/list-actions.route.ts @@ -7,6 +7,8 @@ export class ListActionsRoute extends GetRoute { } private constructor(handler: RouteHandler) { - super('actions', handler, new ListActionsRouteIO()); + super('actions', handler, { + io: new ListActionsRouteIO(), + }); } } diff --git a/yarn.lock b/yarn.lock index 212fb35..85d6492 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,18 +2,18 @@ # yarn lockfile v1 -"@alien-worlds/aw-broadcast@^0.0.3": - version "0.0.3" - resolved "https://npm.pkg.github.com/download/@alien-worlds/aw-broadcast/0.0.3/ee62d2dccf49ae3da426e8c92ddf01520a677616#ee62d2dccf49ae3da426e8c92ddf01520a677616" - integrity sha512-L254cFKKRey6ni68ftE1w0rRPtz5nedUG1lB4sMe+Q5e0wXDyMLGi09dlAUN6PBBQQDCW6BDiD5pyTGgk1I5SA== +"@alien-worlds/aw-broadcast@^0.0.5": + version "0.0.5" + resolved "https://npm.pkg.github.com/download/@alien-worlds/aw-broadcast/0.0.5/2f6d4d0c6cfe8b039efd758a40ad23d18bee2efd#2f6d4d0c6cfe8b039efd758a40ad23d18bee2efd" + integrity sha512-fPMAZKLxzhK0ZJWQ5LYZaaRnwnZgsGdDvlZCwd2rY8I/j/elTpJ/N2hJ85GJaHgp1EmICj7o7+R3INkasXLGyg== dependencies: - "@alien-worlds/aw-core" "^0.0.3" + "@alien-worlds/aw-core" "^0.0.6" nanoid "^3.0.0" -"@alien-worlds/aw-core@^0.0.3": - version "0.0.3" - resolved "https://npm.pkg.github.com/download/@alien-worlds/aw-core/0.0.3/1b5af094fced3d97824d909190d446c116aa6bec#1b5af094fced3d97824d909190d446c116aa6bec" - integrity sha512-5aWRA7svi9qWEZO0ROJNUNGnw/KftSPLkcekCjm5ecB0ufMo01eI68A+RlufuCbLaeTKH8K8pKWFgycSxGRBwQ== +"@alien-worlds/aw-core@^0.0.6": + version "0.0.6" + resolved "https://npm.pkg.github.com/download/@alien-worlds/aw-core/0.0.6/fdacb74371b9b80f311ff293e0ad4c5f8e105db5#fdacb74371b9b80f311ff293e0ad4c5f8e105db5" + integrity sha512-vrYMF1pA4LJqOvv3DaZ12WtmUdUkxrxjrmT+Zq5czu5N2VWw/XcrAygThCdBfGBwFS7WZBF93r46uf1BXGFC8w== dependencies: inversify "^6.0.1" node-fetch "2.6.6" From f43b5520254b1b95ee68b36ecbe622903ffa29e0 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Mon, 31 Jul 2023 14:00:27 +0200 Subject: [PATCH 100/107] 0.0.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 87a3e9d..e9db121 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/aw-history", - "version": "0.0.2", + "version": "0.0.3", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From fec523dec4f2cb0612dd0b50e64135d036891745 Mon Sep 17 00:00:00 2001 From: Ahmad Tirmazi <16874651+ahmadtirmazi@users.noreply.github.com> Date: Wed, 2 Aug 2023 11:22:10 +0200 Subject: [PATCH 101/107] refactor: add api host to config --- package.json | 6 +++++- src/api/api.command.ts | 1 + src/api/api.types.ts | 2 ++ src/config/index.ts | 32 +++++++++++++------------------- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index e9db121..6ea86df 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,10 @@ "packageManager": "yarn@3.2.3", "main": "build/index.js", "types": "build/index.d.ts", + "repository": { + "type": "git", + "url": "https://github.com/Alien-Worlds/aw-history" + }, "files": [ "build" ], @@ -42,4 +46,4 @@ "crypto": "^1.0.1", "ts-node": "^10.9.1" } -} +} \ No newline at end of file diff --git a/src/api/api.command.ts b/src/api/api.command.ts index 3a6c292..f4dc7bf 100644 --- a/src/api/api.command.ts +++ b/src/api/api.command.ts @@ -4,5 +4,6 @@ export const apiCommand = new Command(); apiCommand .version('1.0', '-v, --version') + .option('-h, --host ') .option('-p, --port ') .parse(process.argv); diff --git a/src/api/api.types.ts b/src/api/api.types.ts index c3829d0..50ff5a9 100644 --- a/src/api/api.types.ts +++ b/src/api/api.types.ts @@ -1,10 +1,12 @@ import { UnknownObject } from '@alien-worlds/aw-core'; export type ApiCommandOptions = { + host: string; port: number; }; export type ApiConfig = { + host: string; port: number; database: DatabaseConfig; }; diff --git a/src/config/index.ts b/src/config/index.ts index dd5e610..5bfce4c 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -1,22 +1,15 @@ -import { Api } from './../api/api'; -import { FilterCommandOptions, FilterConfig } from '../filter'; +import { buildBroadcastConfig } from '@alien-worlds/aw-broadcast'; +import { BlockReaderConfig, ConfigVars, parseToBigInt, UnknownObject } from '@alien-worlds/aw-core'; +import { WorkersConfig } from '@alien-worlds/aw-workers'; + import { ApiCommandOptions, ApiConfig } from '../api'; import { BlockchainConfig, BootstrapCommandOptions, BootstrapConfig } from '../bootstrap'; -import { ReaderCommandOptions, ReaderConfig } from '../reader'; +import { AbisConfig, AbisServiceConfig, BlockRangeScanConfig, FeaturedConfig, FeaturedContractDataCriteria, ProcessorTaskQueueConfig, UnprocessedBlockQueueConfig } from '../common'; +import { FilterCommandOptions, FilterConfig } from '../filter'; import { ProcessorCommandOptions, ProcessorConfig } from '../processor'; +import { ReaderCommandOptions, ReaderConfig } from '../reader'; +import { Api } from './../api/api'; import { HistoryToolsConfig } from './config.types'; -import { BlockReaderConfig, ConfigVars, UnknownObject, parseToBigInt } from '@alien-worlds/aw-core'; -import { - AbisConfig, - AbisServiceConfig, - BlockRangeScanConfig, - FeaturedConfig, - FeaturedContractDataCriteria, - ProcessorTaskQueueConfig, - UnprocessedBlockQueueConfig, -} from '../common'; -import { WorkersConfig } from '@alien-worlds/aw-workers'; -import { buildBroadcastConfig } from '@alien-worlds/aw-broadcast'; export * from './config.types'; @@ -105,6 +98,7 @@ export const buildApiConfig = ( databaseConfigBuilder: (vars: ConfigVars, ...args: unknown[]) => UnknownObject, options?: ApiCommandOptions ): ApiConfig => ({ + host: vars.getStringEnv('API_HOST') || options.host || 'localhost', port: vars.getNumberEnv('API_PORT') || options.port || 8080, database: databaseConfigBuilder(vars), }); @@ -122,13 +116,13 @@ export const buildBootstrapConfig = ( startBlock: options?.startBlock ? parseToBigInt(options?.startBlock) : vars.getStringEnv('START_BLOCK') - ? parseToBigInt(vars.getStringEnv('START_BLOCK')) - : null, + ? parseToBigInt(vars.getStringEnv('START_BLOCK')) + : null, endBlock: options?.endBlock ? parseToBigInt(options?.endBlock) : vars.getStringEnv('END_BLOCK') - ? parseToBigInt(vars.getStringEnv('END_BLOCK')) - : null, + ? parseToBigInt(vars.getStringEnv('END_BLOCK')) + : null, startFromHead: vars.getBooleanEnv('START_FROM_HEAD') || false, mode: options?.mode || vars.getStringEnv('MODE') || 'default', abis: buildAbisServiceConfig(vars), From c1dec8a8520eff6f42f2371fd14a0093e965b318 Mon Sep 17 00:00:00 2001 From: Ahmad Tirmazi <16874651+ahmadtirmazi@users.noreply.github.com> Date: Wed, 2 Aug 2023 11:23:12 +0200 Subject: [PATCH 102/107] 0.0.4 --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 6ea86df..775c941 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/aw-history", - "version": "0.0.3", + "version": "0.0.4", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", @@ -46,4 +46,4 @@ "crypto": "^1.0.1", "ts-node": "^10.9.1" } -} \ No newline at end of file +} From d25ef9144ce563d66b2629c861a25fd7595a4483 Mon Sep 17 00:00:00 2001 From: Ahmad Tirmazi <16874651+ahmadtirmazi@users.noreply.github.com> Date: Wed, 2 Aug 2023 11:24:06 +0200 Subject: [PATCH 103/107] 0.0.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 775c941..74abf7b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/aw-history", - "version": "0.0.4", + "version": "0.0.5", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From 7adf495d45e55cae79c1ce422a10fe8c6f32e4e6 Mon Sep 17 00:00:00 2001 From: Ahmad Tirmazi <16874651+ahmadtirmazi@users.noreply.github.com> Date: Wed, 2 Aug 2023 11:42:54 +0200 Subject: [PATCH 104/107] 0.0.6 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 74abf7b..3fb6520 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/aw-history", - "version": "0.0.5", + "version": "0.0.6", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From bea40774824c6c0fa8bcb6265cb3e637669974b4 Mon Sep 17 00:00:00 2001 From: Ahmad Tirmazi <16874651+ahmadtirmazi@users.noreply.github.com> Date: Wed, 2 Aug 2023 11:55:40 +0200 Subject: [PATCH 105/107] 0.0.7 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3fb6520..376702f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/aw-history", - "version": "0.0.6", + "version": "0.0.7", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js", From b95c0e06d95d40aa14a3264b827d64ac6eda8c81 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Thu, 3 Aug 2023 08:07:59 +0200 Subject: [PATCH 106/107] update --- package.json | 4 ++-- src/filter/filter.worker.ts | 4 ++-- src/processor/processor.model.factory.ts | 16 ++++++++-------- src/processor/processor.runner.ts | 2 +- yarn.lock | 18 +++++++++--------- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/package.json b/package.json index 376702f..123fe17 100644 --- a/package.json +++ b/package.json @@ -38,8 +38,8 @@ "typescript": "^4.8.2" }, "dependencies": { - "@alien-worlds/aw-broadcast": "^0.0.5", - "@alien-worlds/aw-core": "^0.0.6", + "@alien-worlds/aw-broadcast": "^0.0.6", + "@alien-worlds/aw-core": "^0.0.13", "@alien-worlds/aw-workers": "^0.0.2", "async": "^3.2.4", "commander": "^10.0.1", diff --git a/src/filter/filter.worker.ts b/src/filter/filter.worker.ts index de6dca4..bf3beff 100644 --- a/src/filter/filter.worker.ts +++ b/src/filter/filter.worker.ts @@ -131,7 +131,7 @@ export default class FilterWorker extends Worker { const { name, rows } = delta; for (const row of rows) { - const info = serializer.deserializeTableRow(row); + const info = await serializer.deserializeTableRow(row); if (!info) { // The contract may not contain tables or may be corrupted @@ -205,7 +205,7 @@ export default class FilterWorker extends Worker { dependencies: { serializer, processorTaskQueue }, } = this; - const deserializedBlock = serializer.deserializeBlock< + const deserializedBlock = await serializer.deserializeBlock< BlockModel, BlockModel >(json); diff --git a/src/processor/processor.model.factory.ts b/src/processor/processor.model.factory.ts index d4c7a4a..47f3f83 100644 --- a/src/processor/processor.model.factory.ts +++ b/src/processor/processor.model.factory.ts @@ -12,9 +12,9 @@ import { deserialize } from 'v8'; export class ProcessorModelFactory { constructor(protected serializer: Serializer) {} - protected buildActionTraceProcessorModel( + protected async buildActionTraceProcessorModel( model: ProcessorTask - ): ActionTraceProcessorModel { + ): Promise> { const { serializer } = this; const { abi, content: buffer } = model; const content: ActionProcessorContentModel = deserialize(buffer); @@ -27,7 +27,7 @@ export class ProcessorModelFactory { const [receiptType, receiptContent] = receipt; const { global_sequence, recv_sequence } = receiptContent; - const data = serializer.deserializeActionData( + const data = await serializer.deserializeActionData( act.account, act.name, act.data, @@ -46,14 +46,14 @@ export class ProcessorModelFactory { }; } - protected buildDeltaProcessorModel( + protected async buildDeltaProcessorModel( model: ProcessorTask - ): DeltaProcessorModel { + ): Promise> { const { serializer } = this; const { abi, content: buffer } = model; const delta: DeltaProcessorContentModel = deserialize(buffer); const { name, block_num, block_timestamp } = delta; - const row = serializer.deserializeTableRow(delta, abi); + const row = await serializer.deserializeTableRow(delta, abi); const { code, scope, table, primary_key, payer, data, present } = row; return { @@ -70,9 +70,9 @@ export class ProcessorModelFactory { }; } - public create( + public async create( task: ProcessorTask - ): ActionTraceProcessorModel | DeltaProcessorModel { + ): Promise | DeltaProcessorModel> { if (task.type === ProcessorTaskType.Trace) { return this.buildActionTraceProcessorModel(task); } else if (task.type === ProcessorTaskType.Delta) { diff --git a/src/processor/processor.runner.ts b/src/processor/processor.runner.ts index 53d64a8..41571d2 100644 --- a/src/processor/processor.runner.ts +++ b/src/processor/processor.runner.ts @@ -82,7 +82,7 @@ export class ProcessorRunner { workerPool.releaseWorker(id); }); - const model = this.modelFactory.create(task); + const model = await this.modelFactory.create(task); // start worker worker.run(model); diff --git a/yarn.lock b/yarn.lock index 85d6492..01a9f13 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,18 +2,18 @@ # yarn lockfile v1 -"@alien-worlds/aw-broadcast@^0.0.5": - version "0.0.5" - resolved "https://npm.pkg.github.com/download/@alien-worlds/aw-broadcast/0.0.5/2f6d4d0c6cfe8b039efd758a40ad23d18bee2efd#2f6d4d0c6cfe8b039efd758a40ad23d18bee2efd" - integrity sha512-fPMAZKLxzhK0ZJWQ5LYZaaRnwnZgsGdDvlZCwd2rY8I/j/elTpJ/N2hJ85GJaHgp1EmICj7o7+R3INkasXLGyg== +"@alien-worlds/aw-broadcast@^0.0.6": + version "0.0.6" + resolved "https://npm.pkg.github.com/download/@alien-worlds/aw-broadcast/0.0.6/361699552df577c674cfd07d55a89173a8ec69a1#361699552df577c674cfd07d55a89173a8ec69a1" + integrity sha512-n08okhDoxCr1urpCpKXlXXUcqz2MO7ifjz4jGkgwJgjycJvgi8qm5O5t6HVRjv9MYV3OSJSjAoYls8Q6sPFdHQ== dependencies: - "@alien-worlds/aw-core" "^0.0.6" + "@alien-worlds/aw-core" "^0.0.13" nanoid "^3.0.0" -"@alien-worlds/aw-core@^0.0.6": - version "0.0.6" - resolved "https://npm.pkg.github.com/download/@alien-worlds/aw-core/0.0.6/fdacb74371b9b80f311ff293e0ad4c5f8e105db5#fdacb74371b9b80f311ff293e0ad4c5f8e105db5" - integrity sha512-vrYMF1pA4LJqOvv3DaZ12WtmUdUkxrxjrmT+Zq5czu5N2VWw/XcrAygThCdBfGBwFS7WZBF93r46uf1BXGFC8w== +"@alien-worlds/aw-core@^0.0.13": + version "0.0.13" + resolved "https://npm.pkg.github.com/download/@alien-worlds/aw-core/0.0.13/f9d8ad2b2db4b99c52da69c075a4c34950602e73#f9d8ad2b2db4b99c52da69c075a4c34950602e73" + integrity sha512-v82R+oqik+3IaXH9KllFQGOfkP9P6C97zTt++TleWlB1V7wQdQdFtItgN/0eEo+VdHjdmUryLUEkQM4WNn1kyw== dependencies: inversify "^6.0.1" node-fetch "2.6.6" From f1d3d3b6b5a313d647e5bb4574112c76014a0d96 Mon Sep 17 00:00:00 2001 From: rkamysz Date: Thu, 3 Aug 2023 08:08:21 +0200 Subject: [PATCH 107/107] 0.0.8 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 123fe17..a48a9f3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alien-worlds/aw-history", - "version": "0.0.7", + "version": "0.0.8", "description": "", "packageManager": "yarn@3.2.3", "main": "build/index.js",