From 13bd69132de611fde61d10c27a1e86a7f4cb5431 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Wed, 24 Jul 2024 13:55:13 +1200 Subject: [PATCH 1/8] WIP implement blockchain service --- packages/node-core/src/blockchain.service.ts | 68 +++++ packages/node-core/src/index.ts | 1 + .../blockDispatcher/block-dispatcher.ts | 64 +++-- .../src/indexer/ds-processor.service.ts | 25 +- .../src/indexer/dynamic-ds.service.spec.ts | 10 +- .../src/indexer/dynamic-ds.service.ts | 48 ++-- .../src/indexer/fetch.service.spec.ts | 76 +++--- .../node-core/src/indexer/fetch.service.ts | 72 +++-- .../node-core/src/indexer/indexer.manager.ts | 58 ++-- .../src/indexer/project.service.spec.ts | 77 ++++-- .../node-core/src/indexer/project.service.ts | 59 ++-- .../indexer/unfinalizedBlocks.service.spec.ts | 46 ++-- .../src/indexer/unfinalizedBlocks.service.ts | 46 ++-- .../src/indexer/worker/worker.core.module.ts | 21 +- packages/node-core/src/subcommands/index.ts | 1 + .../src/subcommands/reindex.service.ts | 8 +- .../src/subcommands/testing.core.module.ts | 48 ++++ packages/node/src/blockchain.service.ts | 165 +++++++++++ .../configure/SchemaMigration.service.test.ts | 5 +- packages/node/src/indexer/api.service.test.ts | 6 +- packages/node/src/indexer/api.service.ts | 7 +- .../block-dispatcher.service.ts | 113 -------- .../node/src/indexer/blockDispatcher/index.ts | 4 +- .../worker-block-dispatcher.service.ts | 25 +- .../substrateDictionary.service.spec.ts | 14 +- .../dictionary/substrateDictionary.service.ts | 10 +- .../v1/substrateDictionaryV1.spec.ts | 14 +- .../dictionary/v1/substrateDictionaryV1.ts | 9 +- .../src/indexer/ds-processor.service.spec.ts | 19 +- .../node/src/indexer/ds-processor.service.ts | 22 -- .../node/src/indexer/dynamic-ds.service.ts | 54 ---- packages/node/src/indexer/fetch.module.ts | 76 +++--- .../node/src/indexer/fetch.service.spec.ts | 6 +- .../node/src/indexer/fetch.service.test.ts | 9 +- packages/node/src/indexer/fetch.service.ts | 63 +---- .../node/src/indexer/indexer.manager.spec.ts | 258 ------------------ packages/node/src/indexer/indexer.manager.ts | 127 +++++---- .../node/src/indexer/project.service.spec.ts | 30 +- packages/node/src/indexer/project.service.ts | 82 ------ .../indexer/runtime/base-runtime.service.ts | 9 +- .../src/indexer/runtime/runtimeService.ts | 16 +- .../node/src/indexer/store.service.test.ts | 2 +- .../src/indexer/unfinalizedBlocks.service.ts | 55 ---- .../src/indexer/worker/worker-fetch.module.ts | 24 +- .../node/src/indexer/worker/worker.service.ts | 9 +- packages/node/src/indexer/worker/worker.ts | 2 +- packages/node/src/init.ts | 2 +- .../node/src/subcommands/reindex.module.ts | 8 +- .../node/src/subcommands/testing.module.ts | 43 +-- .../node/src/subcommands/testing.service.ts | 24 +- 50 files changed, 881 insertions(+), 1159 deletions(-) create mode 100644 packages/node-core/src/blockchain.service.ts create mode 100644 packages/node-core/src/subcommands/testing.core.module.ts create mode 100644 packages/node/src/blockchain.service.ts delete mode 100644 packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts delete mode 100644 packages/node/src/indexer/ds-processor.service.ts delete mode 100644 packages/node/src/indexer/dynamic-ds.service.ts delete mode 100644 packages/node/src/indexer/indexer.manager.spec.ts delete mode 100644 packages/node/src/indexer/project.service.ts delete mode 100644 packages/node/src/indexer/unfinalizedBlocks.service.ts diff --git a/packages/node-core/src/blockchain.service.ts b/packages/node-core/src/blockchain.service.ts new file mode 100644 index 0000000000..0a2f4e9f48 --- /dev/null +++ b/packages/node-core/src/blockchain.service.ts @@ -0,0 +1,68 @@ +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import { BaseCustomDataSource, BaseDataSource, IProjectNetworkConfig } from '@subql/types-core'; +import { DatasourceParams, Header, IBlock, ISubqueryProject } from './indexer'; + +// TODO probably need to split this in 2 to have a worker specific subset + +export interface IBlockchainService< + DS extends BaseDataSource = BaseDataSource, + CDS extends DS & BaseCustomDataSource = BaseCustomDataSource & DS, + SubQueryProject extends ISubqueryProject = ISubqueryProject, + SafeAPI = any, + LightBlock = any, + FullBlock = any, +> { + /* The semver of the node */ + packageVersion: string; + + blockHandlerKind: string; + // TODO SubqueryProject methods + + // Block dispatcher service + fetchBlocks(blockNums: number[]): Promise[] | IBlock[]>; // TODO this probably needs to change to get light block type correct + + // Project service + onProjectChange(project: SubQueryProject): Promise | void; + /* Not all networks have a block timestamp, e.g. Shiden */ + getBlockTimestamp(height: number): Promise; + + // Block dispatcher + /* Gets the size of the block, used to calculate a median */ + getBlockSize(block: IBlock): number; + + // Fetch service + /** + * The finalized header. If the chain doesn't have concrete finalization this could be a probablilistic finalization + * */ + getFinalizedHeader(): Promise
; + /** + * Gets the latest height of the chain, this should be unfinalized. + * Or if the chain has instant finalization this would be the same as the finalized height. + * */ + getBestHeight(): Promise; + /** + * The chain interval in milliseconds, if it is not consistent then provide a best estimate + * */ + getChainInterval(): Promise; + + // Unfinalized blocks + getHeaderForHash(hash: string): Promise
; + getHeaderForHeight(height: number): Promise
; + + // Dynamic Ds sevice + /** + * Applies and validates parameters to a template DS + * */ + updateDynamicDs(params: DatasourceParams, template: DS | CDS): Promise; + + isCustomDs: (x: DS | CDS) => x is CDS; + isRuntimeDs: (x: DS | CDS) => x is DS; + + // Indexer manager + /** + * Gets an API instance to a specific height so any state queries return data as represented at that height. + * */ + getSafeApi(block: LightBlock | FullBlock): Promise; +} diff --git a/packages/node-core/src/index.ts b/packages/node-core/src/index.ts index 47fe1649ba..804fc03930 100644 --- a/packages/node-core/src/index.ts +++ b/packages/node-core/src/index.ts @@ -17,3 +17,4 @@ export * from './indexer'; export * from './subcommands'; export * from './yargs'; export * from './admin'; +export * from './blockchain.service'; diff --git a/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts index 151aed8cb7..64634f7464 100644 --- a/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts @@ -1,21 +1,22 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {OnApplicationShutdown} from '@nestjs/common'; -import {EventEmitter2} from '@nestjs/event-emitter'; -import {Interval} from '@nestjs/schedule'; -import {NodeConfig} from '../../configure'; -import {IProjectUpgradeService} from '../../configure/ProjectUpgrade.service'; -import {IndexerEvent} from '../../events'; -import {getBlockHeight, IBlock, PoiSyncService} from '../../indexer'; -import {getLogger} from '../../logger'; -import {exitWithError, monitorWrite} from '../../process'; -import {profilerWrap} from '../../profiler'; -import {Queue, AutoQueue, RampQueue, delay, isTaskFlushedError} from '../../utils'; -import {StoreService} from '../store.service'; -import {IStoreModelProvider} from '../storeModelProvider'; -import {IProjectService, ISubqueryProject} from '../types'; -import {BaseBlockDispatcher, ProcessBlockResponse} from './base-block-dispatcher'; +import { OnApplicationShutdown } from '@nestjs/common'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { Interval } from '@nestjs/schedule'; +import { BaseDataSource } from '@subql/types-core'; +import { IBlockchainService } from '../../blockchain.service'; +import { NodeConfig } from '../../configure'; +import { IProjectUpgradeService } from '../../configure/ProjectUpgrade.service'; +import { IndexerEvent } from '../../events'; +import { getBlockHeight, IBlock, PoiSyncService, StoreService } from '../../indexer'; +import { getLogger } from '../../logger'; +import { exitWithError, monitorWrite } from '../../process'; +import { profilerWrap } from '../../profiler'; +import { Queue, AutoQueue, RampQueue, delay, isTaskFlushedError } from '../../utils'; +import { IStoreModelProvider } from '../storeModelProvider'; +import { IIndexerManager, IProjectService, ISubqueryProject } from '../types'; +import { BaseBlockDispatcher } from './base-block-dispatcher'; const logger = getLogger('BlockDispatcherService'); @@ -24,10 +25,9 @@ type BatchBlockFetcher = (heights: number[]) => Promise[]>; /** * @description Intended to behave the same as WorkerBlockDispatcherService but doesn't use worker threads or any parallel processing */ -export abstract class BlockDispatcher +export class BlockDispatcher extends BaseBlockDispatcher | number>, DS, B> - implements OnApplicationShutdown -{ + implements OnApplicationShutdown { private fetchQueue: AutoQueue>; private processQueue: AutoQueue; @@ -36,9 +36,6 @@ export abstract class BlockDispatcher private fetching = false; private isShutdown = false; - protected abstract getBlockSize(block: IBlock): number; - protected abstract indexBlock(block: IBlock): Promise; - constructor( nodeConfig: NodeConfig, eventEmitter: EventEmitter2, @@ -48,7 +45,8 @@ export abstract class BlockDispatcher storeModelProvider: IStoreModelProvider, poiSyncService: PoiSyncService, project: ISubqueryProject, - fetchBlocksBatches: BatchBlockFetcher + blockchainService: IBlockchainService, + private indexerManager: IIndexerManager ) { super( nodeConfig, @@ -63,16 +61,20 @@ export abstract class BlockDispatcher ); this.processQueue = new AutoQueue(nodeConfig.batchSize * 3, 1, nodeConfig.timeout, 'Process'); this.fetchQueue = new RampQueue( - this.getBlockSize.bind(this), + blockchainService.getBlockSize.bind(this), nodeConfig.batchSize, nodeConfig.batchSize * 3, nodeConfig.timeout, 'Fetch' ); if (this.nodeConfig.profiler) { - this.fetchBlocksBatches = profilerWrap(fetchBlocksBatches, 'BlockDispatcher', 'fetchBlocksBatches'); + this.fetchBlocksBatches = profilerWrap( + blockchainService.fetchBlocks.bind(blockchainService), + 'BlockDispatcher', + 'fetchBlocksBatches' + ); } else { - this.fetchBlocksBatches = fetchBlocksBatches; + this.fetchBlocksBatches = blockchainService.fetchBlocks.bind(blockchainService); } } @@ -161,7 +163,10 @@ export abstract class BlockDispatcher await this.preProcessBlock(header); monitorWrite(`Processing from main thread`); // Inject runtimeVersion here to enhance api.at preparation - const processBlockResponse = await this.indexBlock(block); + const processBlockResponse = await this.indexerManager.indexBlock( + block, + await this.projectService.getDataSources(block.getHeader().blockHeight) + ); await this.postProcessBlock(header, processBlockResponse); //set block to null for garbage collection (block as any) = null; @@ -172,8 +177,7 @@ export abstract class BlockDispatcher } logger.error( e, - `Failed to index block at height ${header.blockHeight} ${ - e.handler ? `${e.handler}(${e.stack ?? ''})` : '' + `Failed to index block at height ${header.blockHeight} ${e.handler ? `${e.handler}(${e.stack ?? ''})` : '' }` ); throw e; @@ -194,7 +198,7 @@ export abstract class BlockDispatcher // Do nothing, fetching the block was flushed, this could be caused by forked blocks or dynamic datasources return; } - exitWithError(new Error(`Failed to enqueue fetched block to process`, {cause: e}), logger); + exitWithError(new Error(`Failed to enqueue fetched block to process`, { cause: e }), logger); }); this.eventEmitter.emit(IndexerEvent.BlockQueueSize, { @@ -203,7 +207,7 @@ export abstract class BlockDispatcher } } catch (e: any) { if (!this.isShutdown) { - exitWithError(new Error(`Failed to process blocks from queue`, {cause: e}), logger); + exitWithError(new Error(`Failed to process blocks from queue`, { cause: e }), logger); } } finally { this.fetching = false; diff --git a/packages/node-core/src/indexer/ds-processor.service.ts b/packages/node-core/src/indexer/ds-processor.service.ts index ad57868c2b..6a384076ed 100644 --- a/packages/node-core/src/indexer/ds-processor.service.ts +++ b/packages/node-core/src/indexer/ds-processor.service.ts @@ -3,7 +3,7 @@ import fs from 'fs'; import path from 'path'; -import {Inject} from '@nestjs/common'; +import {Inject, Injectable} from '@nestjs/common'; import { BaseCustomDataSource, SecondLayerHandlerProcessor_0_0_0, @@ -13,6 +13,7 @@ import { IProjectNetworkConfig, } from '@subql/types-core'; import {VMScript} from 'vm2'; +import {IBlockchainService} from '../blockchain.service'; import {NodeConfig} from '../configure'; import {getLogger} from '../logger'; import {Sandbox, SandboxOption} from './sandbox'; @@ -27,7 +28,7 @@ function isSecondLayerHandlerProcessor_0_0_0< F extends Record, E, API, - DS extends BaseCustomDataSource = BaseCustomDataSource + DS extends BaseCustomDataSource = BaseCustomDataSource, >( processor: | SecondLayerHandlerProcessor_0_0_0 @@ -44,7 +45,7 @@ function isSecondLayerHandlerProcessor_1_0_0< F extends Record, E, API, - DS extends BaseCustomDataSource = BaseCustomDataSource + DS extends BaseCustomDataSource = BaseCustomDataSource, >( processor: | SecondLayerHandlerProcessor_0_0_0 @@ -60,7 +61,7 @@ export function asSecondLayerHandlerProcessor_1_0_0< F extends Record, E, API, - DS extends BaseCustomDataSource = BaseCustomDataSource + DS extends BaseCustomDataSource = BaseCustomDataSource, >( processor: | SecondLayerHandlerProcessor_0_0_0 @@ -101,7 +102,7 @@ class DsPluginSandbox

extends Sandbox { export function getDsProcessor< P, DS extends BaseDataSource = BaseDataSource, - CDS extends DS & BaseCustomDataSource = DS & BaseCustomDataSource + CDS extends DS & BaseCustomDataSource = DS & BaseCustomDataSource, >( ds: CDS, isCustomDs: (ds: any) => boolean, @@ -132,17 +133,17 @@ export function getDsProcessor< return processorCache[ds.processor.file] as unknown as P; } -export abstract class BaseDsProcessorService< +@Injectable() +export class DsProcessorService< DS extends BaseDataSource = BaseDataSource, CDS extends DS & BaseCustomDataSource = DS & BaseCustomDataSource, - P extends DsProcessor = DsProcessor + P extends DsProcessor = DsProcessor, > { private processorCache: Record = {}; - protected abstract isCustomDs(ds: DS): ds is CDS; - constructor( @Inject('ISubqueryProject') private readonly project: ISubqueryProject, + @Inject('IBlockchainService') private blockchainService: IBlockchainService, private readonly nodeConfig: NodeConfig ) {} @@ -168,11 +169,11 @@ export abstract class BaseDsProcessorService< } async validateProjectCustomDatasources(dataSources: DS[]): Promise { - await this.validateCustomDs(dataSources.filter((ds) => this.isCustomDs(ds)) as unknown as CDS[]); + await this.validateCustomDs(dataSources.filter((ds) => this.blockchainService.isCustomDs(ds)) as unknown as CDS[]); } getDsProcessor(ds: CDS): P { - if (!this.isCustomDs(ds)) { + if (!this.blockchainService.isCustomDs(ds)) { throw new Error(`data source is not a custom data source`); } if (!this.processorCache[ds.processor.file]) { @@ -196,7 +197,7 @@ export abstract class BaseDsProcessorService< // eslint-disable-next-line @typescript-eslint/require-await async getAssets(ds: CDS): Promise> { - if (!this.isCustomDs(ds)) { + if (!this.blockchainService.isCustomDs(ds)) { throw new Error(`data source is not a custom data source`); } diff --git a/packages/node-core/src/indexer/dynamic-ds.service.spec.ts b/packages/node-core/src/indexer/dynamic-ds.service.spec.ts index 6c7ca7be7d..50aa48a340 100644 --- a/packages/node-core/src/indexer/dynamic-ds.service.spec.ts +++ b/packages/node-core/src/indexer/dynamic-ds.service.spec.ts @@ -1,13 +1,17 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import {BaseDataSource} from '@subql/types-core'; +import {IBlockchainService} from '../blockchain.service'; import {DatasourceParams, DynamicDsService} from './dynamic-ds.service'; import {CacheMetadataModel} from './storeModelProvider'; import {ISubqueryProject} from './types'; -class TestDynamicDsService extends DynamicDsService { - protected async getDatasource(params: DatasourceParams): Promise { - return Promise.resolve(params); +class TestDynamicDsService extends DynamicDsService { + constructor(project: ISubqueryProject) { + super(project, { + updateDynamicDs: () => Promise.resolve(undefined), // Return the same value + } as unknown as IBlockchainService); } getTemplate[number], 'name'> & {startBlock?: number}>( diff --git a/packages/node-core/src/indexer/dynamic-ds.service.ts b/packages/node-core/src/indexer/dynamic-ds.service.ts index 9dae8d6245..e2cdf637da 100644 --- a/packages/node-core/src/indexer/dynamic-ds.service.ts +++ b/packages/node-core/src/indexer/dynamic-ds.service.ts @@ -1,12 +1,15 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {Transaction} from '@subql/x-sequelize'; -import {cloneDeep} from 'lodash'; -import {getLogger} from '../logger'; -import {exitWithError} from '../process'; -import {IMetadata} from './storeModelProvider'; -import {ISubqueryProject} from './types'; +import { Inject, Injectable } from '@nestjs/common'; +import { BaseDataSource } from '@subql/types-core'; +import { Transaction } from '@subql/x-sequelize'; +import { cloneDeep } from 'lodash'; +import { IBlockchainService } from '../blockchain.service'; +import { getLogger } from '../logger'; +import { exitWithError } from '../process'; +import { IMetadata } from './storeModelProvider'; +import { ISubqueryProject } from './types'; const logger = getLogger('dynamic-ds'); @@ -24,16 +27,17 @@ export interface IDynamicDsService { getDynamicDatasources(forceReload?: boolean): Promise; } -export abstract class DynamicDsService - implements IDynamicDsService -{ +@Injectable() +export class DynamicDsService + implements IDynamicDsService { private _metadata?: IMetadata; private _datasources?: DS[]; private _datasourceParams?: DatasourceParams[]; - protected abstract getDatasource(params: DatasourceParams): Promise; - - constructor(protected readonly project: P) {} + constructor( + @Inject('ISubqueryProject') private readonly project: P, + @Inject('IBlockchainService') private readonly blockchainService: IBlockchainService + ) {} async init(metadata: IMetadata): Promise { this._metadata = metadata; @@ -82,7 +86,7 @@ export abstract class DynamicDsService[number], 'name'> & {startBlock?: number}>( + protected getTemplate[number], 'name'> & { startBlock?: number }>( templateName: string, startBlock?: number ): T { @@ -122,7 +126,19 @@ export abstract class DynamicDsService { + const dsObj = this.getTemplate(params.templateName, params.startBlock); + + try { + await this.blockchainService.updateDynamicDs(params, dsObj); + + return dsObj; + } catch (e) { + throw new Error(`Unable to create dynamic datasource.\n ${(e as any).message}`); + } } } diff --git a/packages/node-core/src/indexer/fetch.service.spec.ts b/packages/node-core/src/indexer/fetch.service.spec.ts index 30ff1e2176..12aadaf519 100644 --- a/packages/node-core/src/indexer/fetch.service.spec.ts +++ b/packages/node-core/src/indexer/fetch.service.spec.ts @@ -1,11 +1,11 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {EventEmitter2} from '@nestjs/event-emitter'; -import {SchedulerRegistry} from '@nestjs/schedule'; -import {BaseDataSource, BaseHandler, BaseMapping, DictionaryQueryEntry} from '@subql/types-core'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { SchedulerRegistry } from '@nestjs/schedule'; +import { BaseDataSource, BaseHandler, BaseMapping, DictionaryQueryEntry } from '@subql/types-core'; import { - BaseUnfinalizedBlocksService, + UnfinalizedBlocksService, BlockDispatcher, delay, Header, @@ -13,10 +13,11 @@ import { IBlockDispatcher, IProjectService, NodeConfig, + IBlockchainService, } from '../'; -import {BlockHeightMap} from '../utils/blockHeightMap'; -import {DictionaryService} from './dictionary/dictionary.service'; -import {BaseFetchService} from './fetch.service'; +import { BlockHeightMap } from '../utils/blockHeightMap'; +import { DictionaryService } from './dictionary/dictionary.service'; +import { BaseFetchService } from './fetch.service'; const CHAIN_INTERVAL = 100; // 100ms const genesisHash = '0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3'; @@ -43,7 +44,7 @@ class TestFetchService extends BaseFetchService { return Promise.resolve(); } - async preLoopHook(data: {startHeight: number}): Promise { + async preLoopHook(data: { startHeight: number }): Promise { return Promise.resolve(); } @@ -112,7 +113,7 @@ function mockModuloDs(startBlock: number, endBlock: number, modulo: number): Bas { kind: 'mock/Handler', handler: 'mockFunction', - filter: {modulo: modulo}, + filter: { modulo: modulo }, }, ], }, @@ -163,8 +164,8 @@ describe('Fetch Service', () => { let blockDispatcher: IBlockDispatcher; let dictionaryService: DictionaryService; let dataSources: BaseDataSource[]; - let unfinalizedBlocksService: BaseUnfinalizedBlocksService; - let projectService: IProjectService; + let unfinalizedBlocksService: UnfinalizedBlocksService; + let blockchainService: IBlockchainService; let spyOnEnqueueSequential: jest.SpyInstance< void | Promise, @@ -181,7 +182,7 @@ describe('Fetch Service', () => { const eventEmitter = new EventEmitter2(); const schedulerRegistry = new SchedulerRegistry(); - projectService = { + const projectService = { getStartBlockFromDataSources: jest.fn(() => Math.min(...dataSources.map((ds) => ds.startBlock ?? 0))), getAllDataSources: jest.fn(() => dataSources), getDataSourcesMap: jest.fn(() => { @@ -213,7 +214,8 @@ describe('Fetch Service', () => { metadata: { set: jest.fn(), }, - } as any + } as any, + blockchainService ); spyOnEnqueueSequential = jest.spyOn(fetchService as any, 'enqueueSequential') as any; @@ -247,20 +249,20 @@ describe('Fetch Service', () => { const moduloBlockHeightMap = new BlockHeightMap( new Map([ - [1, [{...mockModuloDs(1, 100, 20), startBlock: 1, endBlock: 100}]], + [1, [{ ...mockModuloDs(1, 100, 20), startBlock: 1, endBlock: 100 }]], [ 101, // empty gap for discontinuous block [], ], - [201, [{...mockModuloDs(201, 500, 30), startBlock: 201, endBlock: 500}]], + [201, [{ ...mockModuloDs(201, 500, 30), startBlock: 201, endBlock: 500 }]], // to infinite - [500, [{...mockModuloDs(500, Number.MAX_SAFE_INTEGER, 99), startBlock: 500}]], + [500, [{ ...mockModuloDs(500, Number.MAX_SAFE_INTEGER, 99), startBlock: 500 }]], // multiple ds [ 600, [ - {...mockModuloDs(500, 800, 99), startBlock: 600, endBlock: 800}, - {...mockModuloDs(700, Number.MAX_SAFE_INTEGER, 101), startBlock: 700}, + { ...mockModuloDs(500, 800, 99), startBlock: 600, endBlock: 800 }, + { ...mockModuloDs(700, Number.MAX_SAFE_INTEGER, 101), startBlock: 700 }, ], ], ]) @@ -286,43 +288,43 @@ describe('Fetch Service', () => { [ 1, [ - {...mockDs, startBlock: 1, endBlock: 300}, - {...mockDs, startBlock: 1, endBlock: 100}, + { ...mockDs, startBlock: 1, endBlock: 300 }, + { ...mockDs, startBlock: 1, endBlock: 100 }, ], ], [ 10, [ - {...mockDs, startBlock: 1, endBlock: 300}, - {...mockDs, startBlock: 1, endBlock: 100}, - {...mockDs, startBlock: 10, endBlock: 20}, + { ...mockDs, startBlock: 1, endBlock: 300 }, + { ...mockDs, startBlock: 1, endBlock: 100 }, + { ...mockDs, startBlock: 10, endBlock: 20 }, ], ], [ 21, [ - {...mockDs, startBlock: 1, endBlock: 300}, - {...mockDs, startBlock: 1, endBlock: 100}, + { ...mockDs, startBlock: 1, endBlock: 300 }, + { ...mockDs, startBlock: 1, endBlock: 100 }, ], ], [ 50, [ - {...mockDs, startBlock: 1, endBlock: 300}, - {...mockDs, startBlock: 1, endBlock: 100}, - {...mockDs, startBlock: 50, endBlock: 200}, + { ...mockDs, startBlock: 1, endBlock: 300 }, + { ...mockDs, startBlock: 1, endBlock: 100 }, + { ...mockDs, startBlock: 50, endBlock: 200 }, ], ], [ 101, [ - {...mockDs, startBlock: 1, endBlock: 300}, - {...mockDs, startBlock: 50, endBlock: 200}, + { ...mockDs, startBlock: 1, endBlock: 300 }, + { ...mockDs, startBlock: 50, endBlock: 200 }, ], ], - [201, [{...mockDs, startBlock: 1, endBlock: 300}]], + [201, [{ ...mockDs, startBlock: 1, endBlock: 300 }]], [301, []], - [500, [{...mockDs, startBlock: 500}]], + [500, [{ ...mockDs, startBlock: 500 }]], ]) ) ); @@ -458,7 +460,7 @@ describe('Fetch Service', () => { { kind: 'mock/BlockHandler', handler: 'mockFunction', - filter: {modulo: 3}, + filter: { modulo: 3 }, }, { kind: 'mock/CallHandler', @@ -591,7 +593,7 @@ describe('Fetch Service', () => { it('enqueues modulo blocks with furture dataSources', async () => { fetchService.mockGetModulos([3]); - dataSources.push({...mockDs, startBlock: 20}); + dataSources.push({ ...mockDs, startBlock: 20 }); await fetchService.init(1); @@ -604,7 +606,7 @@ describe('Fetch Service', () => { it('at the end of modulo block filter, enqueue END should be min of data source range end height and api last height', async () => { // So this will skip next data source fetchService.mockGetModulos([10]); - dataSources.push({...mockDs, startBlock: 200}); + dataSources.push({ ...mockDs, startBlock: 200 }); await fetchService.init(191); expect((fetchService as any).useDictionary).toBeFalsy(); @@ -612,7 +614,7 @@ describe('Fetch Service', () => { }); it('skips bypassBlocks', async () => { - projectService.bypassBlocks = [3]; + (fetchService as any).networkConfig.bypassBlocks = [3]; await fetchService.init(1); @@ -623,7 +625,7 @@ describe('Fetch Service', () => { it('transforms bypassBlocks', async () => { // Set a range so on init its transformed - projectService.bypassBlocks = ['2-5']; + (fetchService as any).networkConfig.bypassBlocks = ['2-5']; await fetchService.init(1); diff --git a/packages/node-core/src/indexer/fetch.service.ts b/packages/node-core/src/indexer/fetch.service.ts index 42bb2d52ba..4aa45aa16f 100644 --- a/packages/node-core/src/indexer/fetch.service.ts +++ b/packages/node-core/src/indexer/fetch.service.ts @@ -2,28 +2,28 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import {OnApplicationShutdown} from '@nestjs/common'; -import {EventEmitter2} from '@nestjs/event-emitter'; -import {SchedulerRegistry} from '@nestjs/schedule'; -import {BaseDataSource} from '@subql/types-core'; -import {range} from 'lodash'; -import {NodeConfig} from '../configure'; -import {IndexerEvent} from '../events'; -import {getLogger} from '../logger'; -import {delay, filterBypassBlocks} from '../utils'; -import {IBlockDispatcher} from './blockDispatcher'; -import {mergeNumAndBlocksToNums} from './dictionary'; -import {DictionaryService} from './dictionary/dictionary.service'; -import {mergeNumAndBlocks} from './dictionary/utils'; -import {IStoreModelProvider} from './storeModelProvider'; -import {BypassBlocks, Header, IBlock, IProjectService} from './types'; -import {IUnfinalizedBlocksServiceUtil} from './unfinalizedBlocks.service'; +import { OnApplicationShutdown } from '@nestjs/common'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { SchedulerRegistry } from '@nestjs/schedule'; +import { BaseDataSource } from '@subql/types-core'; +import { range } from 'lodash'; +import { IBlockchainService } from '../blockchain.service'; +import { NodeConfig } from '../configure'; +import { IndexerEvent } from '../events'; +import { getLogger } from '../logger'; +import { delay, filterBypassBlocks, getModulos } from '../utils'; +import { IBlockDispatcher } from './blockDispatcher'; +import { mergeNumAndBlocksToNums } from './dictionary'; +import { DictionaryService } from './dictionary/dictionary.service'; +import { mergeNumAndBlocks } from './dictionary/utils'; +import { IStoreModelProvider } from './storeModelProvider'; +import { BypassBlocks, IBlock, IProjectService } from './types'; +import { IUnfinalizedBlocksServiceUtil } from './unfinalizedBlocks.service'; const logger = getLogger('FetchService'); export abstract class BaseFetchService, FB> - implements OnApplicationShutdown -{ + implements OnApplicationShutdown { private _latestBestHeight?: number; private _latestFinalizedHeight?: number; private isShutdown = false; @@ -31,20 +31,11 @@ export abstract class BaseFetchService; #pendingBestBlockHead?: Promise; - // If the chain doesn't have a distinction between the 2 it should return the same value for finalized and best - protected abstract getFinalizedHeader(): Promise

; - protected abstract getBestHeight(): Promise; - - // The rough interval at which new blocks are produced - protected abstract getChainInterval(): Promise; - // This return modulo numbers with given dataSources (number in the filters) - protected abstract getModulos(dataSources: DS[]): number[]; - protected abstract initBlockDispatcher(): Promise; // Gets called just before the loop is started // Used by substrate to init runtime service and get runtime version data from the dictionary - protected abstract preLoopHook(data: {startHeight: number}): Promise; + protected abstract preLoopHook(data: { startHeight: number }): Promise; constructor( private nodeConfig: NodeConfig, @@ -54,7 +45,8 @@ export abstract class BaseFetchService ) {} private get latestBestHeight(): number { @@ -70,6 +62,10 @@ export abstract class BaseFetchService { - const interval = await this.getChainInterval(); + const interval = await this.blockchainSevice.getChainInterval(); await Promise.all([this.memoGetFinalizedBlockHead(), this.memoGetBestBlockHead()]); @@ -129,7 +125,7 @@ export abstract class BaseFetchService { try { - const currentFinalizedHeader = await this.getFinalizedHeader(); + const currentFinalizedHeader = await this.blockchainSevice.getFinalizedHeader(); // Rpc could return finalized height below last finalized height due to unmatched nodes, and this could lead indexing stall // See how this could happen in https://gist.github.com/jiqiang90/ea640b07d298bca7cbeed4aee50776de if ( @@ -163,7 +159,7 @@ export abstract class BaseFetchService { try { - const currentBestHeight = await this.getBestHeight(); + const currentBestHeight = await this.blockchainSevice.getBestHeight(); if (this._latestBestHeight !== currentBestHeight) { this._latestBestHeight = currentBestHeight; this.eventEmitter.emit(IndexerEvent.BlockBest, { @@ -241,7 +237,7 @@ export abstract class BaseFetchService { // End height from current dataSource - const {endHeight, value: relevantDs} = this.getRelevantDsDetails(startBlockHeight); + const { endHeight, value: relevantDs } = this.getRelevantDsDetails(startBlockHeight); // Estimated range end height const estRangeEndHeight = Math.min( endHeight ?? Number.MAX_SAFE_INTEGER, diff --git a/packages/node-core/src/indexer/indexer.manager.ts b/packages/node-core/src/indexer/indexer.manager.ts index 269df29345..aed1910581 100644 --- a/packages/node-core/src/indexer/indexer.manager.ts +++ b/packages/node-core/src/indexer/indexer.manager.ts @@ -2,19 +2,20 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import {BaseCustomDataSource, BaseDataSource} from '@subql/types-core'; -import {IApi} from '../api.service'; -import {NodeConfig} from '../configure'; -import {getLogger} from '../logger'; -import {exitWithError, monitorWrite} from '../process'; -import {profilerWrap} from '../profiler'; -import {handledStringify} from './../utils'; -import {ProcessBlockResponse} from './blockDispatcher'; -import {asSecondLayerHandlerProcessor_1_0_0, BaseDsProcessorService} from './ds-processor.service'; -import {DynamicDsService} from './dynamic-ds.service'; -import {IndexerSandbox} from './sandbox'; -import {Header, IBlock, IIndexerManager} from './types'; -import {IUnfinalizedBlocksService} from './unfinalizedBlocks.service'; +import { BaseCustomDataSource, BaseDataSource, IProjectNetworkConfig } from '@subql/types-core'; +import { IApi } from '../api.service'; +import { IBlockchainService } from '../blockchain.service'; +import { NodeConfig } from '../configure'; +import { getLogger } from '../logger'; +import { exitWithError, monitorWrite } from '../process'; +import { profilerWrap } from '../profiler'; +import { handledStringify } from './../utils'; +import { ProcessBlockResponse } from './blockDispatcher'; +import { asSecondLayerHandlerProcessor_1_0_0, DsProcessorService } from './ds-processor.service'; +import { DynamicDsService } from './dynamic-ds.service'; +import { IndexerSandbox } from './sandbox'; +import { Header, IBlock, IIndexerManager, ISubqueryProject } from './types'; +import { IUnfinalizedBlocksService } from './unfinalizedBlocks.service'; const logger = getLogger('indexer'); @@ -26,7 +27,7 @@ export type FilterTypeMap = Record< export type ProcessorTypeMap> = { [K in keyof FM]: (data: any) => boolean; }; -export type HandlerInputTypeMap> = {[K in keyof FM]: any}; +export type HandlerInputTypeMap> = { [K in keyof FM]: any }; export interface CustomHandler> { handler: string; @@ -44,12 +45,8 @@ export abstract class BaseIndexerManager< FilterMap extends FilterTypeMap, ProcessorMap extends ProcessorTypeMap, HandlerInputMap extends HandlerInputTypeMap, -> implements IIndexerManager -{ - abstract indexBlock(block: IBlock, datasources: DS[], ...args: any[]): Promise; - - protected abstract isRuntimeDs(ds: DS): ds is DS; - protected abstract isCustomDs(ds: DS): ds is CDS; +> implements IIndexerManager { + abstract indexBlock(block: IBlock, datasources: DS[]): Promise; protected abstract indexBlockData( block: B, @@ -63,12 +60,13 @@ export abstract class BaseIndexerManager< constructor( protected readonly apiService: API, protected readonly nodeConfig: NodeConfig, - protected sandboxService: {getDsProcessor: (ds: DS, api: SA, unsafeApi: A) => IndexerSandbox}, - private dsProcessorService: BaseDsProcessorService, + protected sandboxService: { getDsProcessor: (ds: DS, api: SA, unsafeApi: A) => IndexerSandbox }, + private dsProcessorService: DsProcessorService, private dynamicDsService: DynamicDsService, private unfinalizedBlocksService: IUnfinalizedBlocksService, private filterMap: FilterMap, - private processorMap: ProcessorMap + private processorMap: ProcessorMap, + protected blockchainService: IBlockchainService, SA, B, B> ) { logger.info('indexer manager start'); } @@ -145,7 +143,7 @@ export abstract class BaseIndexerManager< // perform filter for custom ds filteredDs = filteredDs.filter((ds) => { - if (this.isCustomDs(ds)) { + if (this.blockchainService.isCustomDs(ds)) { return this.dsProcessorService.getDsProcessor(ds).dsFilterProcessor(ds, this.apiService.unsafeApi); } else { return true; @@ -179,7 +177,7 @@ export abstract class BaseIndexerManager< let vm: IndexerSandbox; assert(this.filterMap[kind], `Unsupported handler kind: ${kind.toString()}`); - if (this.isRuntimeDs(ds)) { + if (this.blockchainService.isRuntimeDs(ds)) { const handlers = ds.mapping.handlers.filter( (h) => h.kind === kind && this.filterMap[kind](data as any, h.filter, ds) ); @@ -193,13 +191,13 @@ export abstract class BaseIndexerManager< monitorWrite(() => `- Handler: ${handler.handler}, args:${handledStringify(data)}`); this.nodeConfig.profiler ? await profilerWrap( - vm.securedExec.bind(vm), - 'handlerPerformance', - handler.handler - )(handler.handler, [parsedData]) + vm.securedExec.bind(vm), + 'handlerPerformance', + handler.handler + )(handler.handler, [parsedData]) : await vm.securedExec(handler.handler, [parsedData]); } - } else if (this.isCustomDs(ds)) { + } else if (this.blockchainService.isCustomDs(ds)) { const handlers = this.filterCustomDsHandlers(ds, data, this.processorMap[kind], (data, baseFilter) => { if (!baseFilter.length) return true; diff --git a/packages/node-core/src/indexer/project.service.spec.ts b/packages/node-core/src/indexer/project.service.spec.ts index 49ae4bf431..9890542044 100644 --- a/packages/node-core/src/indexer/project.service.spec.ts +++ b/packages/node-core/src/indexer/project.service.spec.ts @@ -3,18 +3,15 @@ import { EventEmitter2 } from '@nestjs/event-emitter'; import { buildSchemaFromString } from '@subql/utils'; +import { IBlockchainService } from '../blockchain.service'; import { NodeConfig, ProjectUpgradeService } from '../configure'; -import { BaseDsProcessorService } from './ds-processor.service'; -import { DynamicDsService } from './dynamic-ds.service'; -import { BaseProjectService } from './project.service'; -import { Header, ISubqueryProject } from './types'; -import { - BaseUnfinalizedBlocksService, - METADATA_LAST_FINALIZED_PROCESSED_KEY, - METADATA_UNFINALIZED_BLOCKS_KEY, -} from './unfinalizedBlocks.service'; - -class TestProjectService extends BaseProjectService { +import { DsProcessorService } from './ds-processor.service'; +import { DatasourceParams, DynamicDsService } from './dynamic-ds.service'; +import { ProjectService } from './project.service'; +import { Header, IBlock, ISubqueryProject } from './types'; +import { METADATA_LAST_FINALIZED_PROCESSED_KEY, METADATA_UNFINALIZED_BLOCKS_KEY, UnfinalizedBlocksService } from './unfinalizedBlocks.service'; + +class TestProjectService extends ProjectService { packageVersion = '1.0.0'; async getBlockTimestamp(height: number): Promise { @@ -30,9 +27,49 @@ class TestProjectService extends BaseProjectService { } } -class TestUnfinalizedBlocksService extends BaseUnfinalizedBlocksService { +class TestBlockchainService implements IBlockchainService { + packageVersion = '0.0.0'; + blockHandlerKind = ''; + + // eslint-disable-next-line @typescript-eslint/promise-function-async + fetchBlocks(blockNums: number[]): Promise[]> { + throw new Error('Method not implemented.'); + } + + onProjectChange(project: ISubqueryProject): Promise | void { + throw new Error('Method not implemented.'); + } + // eslint-disable-next-line @typescript-eslint/promise-function-async + getBlockTimestamp(height: number): Promise { + throw new Error('Method not implemented.'); + } + getBlockSize(block: IBlock): number { + return 0; + } + // eslint-disable-next-line @typescript-eslint/promise-function-async + getBestHeight(): Promise { + throw new Error('Method not implemented.'); + } + // eslint-disable-next-line @typescript-eslint/promise-function-async + getChainInterval(): Promise { + throw new Error('Method not implemented.'); + } + // eslint-disable-next-line @typescript-eslint/promise-function-async + updateDynamicDs(params: DatasourceParams, template: any): Promise { + throw new Error('Method not implemented.'); + } + isCustomDs(x: any): x is any { + return false + } + isRuntimeDs(x: any): x is any { + return false; + } + // eslint-disable-next-line @typescript-eslint/promise-function-async + getSafeApi(block: any): Promise { + throw new Error('Method not implemented.'); + } // eslint-disable-next-line @typescript-eslint/require-await - protected async getFinalizedHead(): Promise
{ + async getFinalizedHeader(): Promise
{ return { blockHash: 'asdf', blockHeight: 1000, @@ -42,7 +79,7 @@ class TestUnfinalizedBlocksService extends BaseUnfinalizedBlocksService { } // eslint-disable-next-line @typescript-eslint/require-await - protected async getHeaderForHash(hash: string): Promise
{ + async getHeaderForHash(hash: string): Promise
{ const num = parseInt(hash.slice(1), 10); return { blockHeight: num, @@ -68,7 +105,7 @@ describe('BaseProjectService', () => { beforeEach(() => { service = new TestProjectService( - null as unknown as BaseDsProcessorService, + null as unknown as DsProcessorService, null as unknown as any, null as unknown as any, null as unknown as any, @@ -79,7 +116,8 @@ describe('BaseProjectService', () => { { unsafe: false } as unknown as NodeConfig, { getDynamicDatasources: jest.fn() } as unknown as DynamicDsService, null as unknown as any, - null as unknown as any + null as unknown as any, + new TestBlockchainService(), ); }); @@ -346,10 +384,12 @@ describe('BaseProjectService', () => { rewind: jest.fn(), } as unknown as any; + const blockchainService = new TestBlockchainService(); + service = new TestProjectService( { validateProjectCustomDatasources: jest.fn(), - } as unknown as BaseDsProcessorService, // dsProcessorService + } as unknown as DsProcessorService, // dsProcessorService { networkMeta: {} } as unknown as any, //apiService null as unknown as any, // poiService null as unknown as any, // poiSyncService @@ -369,7 +409,8 @@ describe('BaseProjectService', () => { resetDynamicDatasource: jest.fn(), } as unknown as DynamicDsService, // dynamicDsService new EventEmitter2(), // eventEmitter - new TestUnfinalizedBlocksService(nodeConfig, storeService.modelProvider) // unfinalizedBlocksService + new UnfinalizedBlocksService(nodeConfig, storeService.storeCache, blockchainService), // unfinalizedBlocksService + blockchainService, ); }; diff --git a/packages/node-core/src/indexer/project.service.ts b/packages/node-core/src/indexer/project.service.ts index 6e847091f0..fe2c4ac752 100644 --- a/packages/node-core/src/indexer/project.service.ts +++ b/packages/node-core/src/indexer/project.service.ts @@ -3,17 +3,19 @@ import assert from 'assert'; import { isMainThread } from 'worker_threads'; +import { Inject } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { BaseDataSource, IProjectNetworkConfig } from '@subql/types-core'; import { Sequelize } from '@subql/x-sequelize'; import { IApi } from '../api.service'; +import { IBlockchainService } from '../blockchain.service'; import { IProjectUpgradeService, NodeConfig } from '../configure'; import { IndexerEvent } from '../events'; import { getLogger } from '../logger'; import { exitWithError, monitorWrite } from '../process'; import { getExistingProjectSchema, getStartHeight, hasValue, initDbSchema, mainThreadOnly, reindex } from '../utils'; import { BlockHeightMap } from '../utils/blockHeightMap'; -import { BaseDsProcessorService } from './ds-processor.service'; +import { DsProcessorService } from './ds-processor.service'; import { DynamicDsService } from './dynamic-ds.service'; import { MetadataKeys } from './entities'; import { PoiSyncService } from './poi'; @@ -31,42 +33,38 @@ class NotInitError extends Error { } } -export abstract class BaseProjectService< - API extends IApi, - DS extends BaseDataSource, +export class ProjectService< + DS extends BaseDataSource = BaseDataSource, + API extends IApi = IApi, UnfinalizedBlocksService extends IUnfinalizedBlocksService = IUnfinalizedBlocksService, > implements IProjectService { private _schema?: string; private _startHeight?: number; private _blockOffset?: number; - protected abstract packageVersion: string; - protected abstract getBlockTimestamp(height: number): Promise; - protected abstract onProjectChange(project: ISubqueryProject): void | Promise; - constructor( - private readonly dsProcessorService: BaseDsProcessorService, - protected readonly apiService: API, - private readonly poiService: PoiService, - private readonly poiSyncService: PoiSyncService, - protected readonly sequelize: Sequelize, - protected readonly project: ISubqueryProject, - protected readonly projectUpgradeService: IProjectUpgradeService, - protected readonly storeService: StoreService, - protected readonly nodeConfig: NodeConfig, - protected readonly dynamicDsService: DynamicDsService, + private readonly dsProcessorService: DsProcessorService, + @Inject('APIService') protected readonly apiService: API, + @Inject(isMainThread ? PoiService : 'Null') private readonly poiService: PoiService, + @Inject(isMainThread ? PoiSyncService : 'Null') private readonly poiSyncService: PoiSyncService, + @Inject(isMainThread ? Sequelize : 'Null') private readonly sequelize: Sequelize, + @Inject('ISubqueryProject') private readonly project: ISubqueryProject, + @Inject('IProjectUpgradeService') private readonly projectUpgradeService: IProjectUpgradeService, + @Inject(isMainThread ? StoreService : 'Null') private readonly storeService: StoreService, + private readonly nodeConfig: NodeConfig, + private readonly dynamicDsService: DynamicDsService, private eventEmitter: EventEmitter2, - protected readonly unfinalizedBlockService: UnfinalizedBlocksService + @Inject('IUnfinalizedBlocksService') private readonly unfinalizedBlockService: UnfinalizedBlocksService, + @Inject('IBlockchainService') private blockchainService: IBlockchainService ) { + if (this.nodeConfig.unfinalizedBlocks && this.nodeConfig.allowSchemaMigration) { + throw new Error('Unfinalized Blocks and Schema Migration cannot be enabled at the same time'); + } if (this.nodeConfig.unsafe) { logger.warn( 'UNSAFE MODE IS ENABLED. This is not recommended for most projects and will not be supported by our hosted service' ); } - - if (this.nodeConfig.unfinalizedBlocks && this.nodeConfig.allowSchemaMigration) { - throw new Error('Unfinalized Blocks and Schema Migration cannot be enabled at the same time'); - } } protected get schema(): string { @@ -99,7 +97,7 @@ export abstract class BaseProjectService< this.ensureTimezone(); for await (const [, project] of this.projectUpgradeService.projects) { - await project.applyCronTimestamps(this.getBlockTimestamp.bind(this)); + await project.applyCronTimestamps(this.blockchainService.getBlockTimestamp.bind(this)); } // Do extra work on main thread to setup stuff @@ -155,7 +153,7 @@ export abstract class BaseProjectService< this.projectUpgradeService.initWorker(startHeight, this.handleProjectChange.bind(this)); // Called to allow handling the first project - await this.onProjectChange(this.project); + await this.blockchainService.onProjectChange(this.project); } // Used to load assets into DS-processor, has to be done in any thread @@ -251,9 +249,8 @@ export abstract class BaseProjectService< if (!existing.processedBlockCount && !existing.lastProcessedHeight) { await metadata.set('processedBlockCount', 0); } - - if (existing.indexerNodeVersion !== this.packageVersion) { - await metadata.set('indexerNodeVersion', this.packageVersion); + if (existing.indexerNodeVersion !== this.blockchainService.packageVersion) { + await metadata.set('indexerNodeVersion', this.blockchainService.packageVersion); } if (!existing.schemaMigrationCount) { await metadata.set('schemaMigrationCount', 0); @@ -405,7 +402,7 @@ export abstract class BaseProjectService< ); // Called to allow handling the first project - await this.onProjectChange(this.project); + await this.blockchainService.onProjectChange(this.project); if (isMainThread) { const lastProcessedHeight = await this.getLastProcessedHeight(); @@ -430,7 +427,7 @@ export abstract class BaseProjectService< logger.info(msg); monitorWrite(msg); - const timestamp = await this.getBlockTimestamp(upgradePoint); + const timestamp = await this.blockchainService.getBlockTimestamp(upgradePoint); // Only timestamp and blockHeight are used with reindexing so its safe to convert to a header await this.reindex({ blockHeight: upgradePoint, @@ -451,7 +448,7 @@ export abstract class BaseProjectService< // Reload the dynamic ds with new project await this.dynamicDsService.getDynamicDatasources(true); - await this.onProjectChange(this.project); + await this.blockchainService.onProjectChange(this.project); } async reindex(targetBlockHeader: Header): Promise { diff --git a/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts b/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts index 757acdf03d..913448665e 100644 --- a/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts +++ b/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts @@ -1,32 +1,31 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -// import { Header } from '@polkadot/types/interfaces'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { SchedulerRegistry } from '@nestjs/schedule'; +import { IBlockchainService } from '../blockchain.service'; import { Header, IBlock } from '../indexer'; import { StoreCacheService, CacheMetadataModel } from './storeModelProvider'; import { METADATA_LAST_FINALIZED_PROCESSED_KEY, METADATA_UNFINALIZED_BLOCKS_KEY, - BaseUnfinalizedBlocksService, + UnfinalizedBlocksService, } from './unfinalizedBlocks.service'; /* Notes: * Block hashes all have the format '0xabc' + block number * If they are forked they will have an `f` at the end */ -class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService> { - protected async getFinalizedHead(): Promise
{ +const BlockchainService = { + async getFinalizedHeader(): Promise
{ return Promise.resolve({ blockHeight: 91, blockHash: `0xabc91f`, parentHash: `0xabc90f`, timestamp: new Date(), }); - } - - protected async getHeaderForHash(hash: string): Promise
{ + }, + async getHeaderForHash(hash: string): Promise
{ const num = Number(hash.toString().replace('0xabc', '').replace('f', '')); return Promise.resolve({ blockHeight: num, @@ -34,8 +33,7 @@ class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService> parentHash: `0xabc${num - 1}f`, timestamp: new Date(), }); - } - + }, async getHeaderForHeight(height: number): Promise
{ return Promise.resolve({ blockHeight: height, @@ -43,8 +41,8 @@ class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService> parentHash: `0xabc${height - 1}f`, timestamp: new Date(), }); - } -} + }, +} as IBlockchainService; function getMockMetadata(): any { const data: Record = {}; @@ -81,7 +79,11 @@ describe('UnfinalizedBlocksService', () => { let unfinalizedBlocksService: UnfinalizedBlocksService; beforeEach(async () => { - unfinalizedBlocksService = new UnfinalizedBlocksService({ unfinalizedBlocks: true } as any, mockStoreCache()); + unfinalizedBlocksService = new UnfinalizedBlocksService( + { unfinalizedBlocks: true } as any, + mockStoreCache(), + BlockchainService + ); await unfinalizedBlocksService.init(() => Promise.resolve()); }); @@ -152,7 +154,7 @@ describe('UnfinalizedBlocksService', () => { const res = await unfinalizedBlocksService.processUnfinalizedBlocks(mockBlock(113, '0xabc113')); // Last valid block - expect(res).toMatchObject({blockHash: '0xabc111', blockHeight: 111, parentHash: ''}); + expect(res).toMatchObject({ blockHash: '0xabc111', blockHeight: 111, parentHash: '' }); // After this the call stack is something like: // indexerManager -> blockDispatcher -> project -> project -> reindex -> blockDispatcher.resetUnfinalizedBlocks @@ -177,7 +179,7 @@ describe('UnfinalizedBlocksService', () => { const res = await unfinalizedBlocksService.processUnfinalizedBlocks(mockBlock(117, '0xabc117')); // Last valid block - expect(res).toMatchObject({blockHash: '0xabc112', blockHeight: 112, parentHash: ''}); + expect(res).toMatchObject({ blockHash: '0xabc112', blockHeight: 112, parentHash: '' }); }); it('can handle a fork when all unfinalized blocks are invalid', async () => { @@ -192,7 +194,7 @@ describe('UnfinalizedBlocksService', () => { const res = await unfinalizedBlocksService.processUnfinalizedBlocks(mockBlock(113, '0xabc113')); // Last valid block - expect(res).toMatchObject({blockHash: '0xabc110f', blockHeight: 110, parentHash: '0xabc109f'}); + expect(res).toMatchObject({ blockHash: '0xabc110f', blockHeight: 110, parentHash: '0xabc109f' }); }); it('can handle a fork and when unfinalized blocks < finalized head', async () => { @@ -207,7 +209,7 @@ describe('UnfinalizedBlocksService', () => { const res = await unfinalizedBlocksService.processUnfinalizedBlocks(mockBlock(113, '0xabc113')); // Last valid block - expect(res).toMatchObject({blockHash: '0xabc110f', blockHeight: 110, parentHash: '0xabc109f'}); + expect(res).toMatchObject({ blockHash: '0xabc110f', blockHeight: 110, parentHash: '0xabc109f' }); }); it('can handle a fork and when unfinalized blocks < finalized head 2', async () => { @@ -228,7 +230,7 @@ describe('UnfinalizedBlocksService', () => { const res = await unfinalizedBlocksService.processUnfinalizedBlocks(mockBlock(113, '0xabc113')); // Last valid block - expect(res).toMatchObject({blockHash: '0xabc110f', blockHeight: 110, parentHash: '0xabc109f'}); + expect(res).toMatchObject({ blockHash: '0xabc110f', blockHeight: 110, parentHash: '0xabc109f' }); }); it('can handle a fork and when unfinalized blocks < finalized head with a large difference', async () => { @@ -243,7 +245,7 @@ describe('UnfinalizedBlocksService', () => { const res = await unfinalizedBlocksService.processUnfinalizedBlocks(mockBlock(113, '0xabc113')); // Last valid block - expect(res).toMatchObject({blockHash: '0xabc110f', blockHeight: 110, parentHash: '0xabc109f'}); + expect(res).toMatchObject({ blockHash: '0xabc110f', blockHeight: 110, parentHash: '0xabc109f' }); }); it('can rewind any unfinalized blocks when restarted and unfinalized blocks is disabled', async () => { @@ -265,14 +267,18 @@ describe('UnfinalizedBlocksService', () => { ]) ); await storeCache.metadata.set(METADATA_LAST_FINALIZED_PROCESSED_KEY, 90); - const unfinalizedBlocksService2 = new UnfinalizedBlocksService({ unfinalizedBlocks: false } as any, storeCache); + const unfinalizedBlocksService2 = new UnfinalizedBlocksService( + { unfinalizedBlocks: false } as any, + storeCache, + BlockchainService + ); const reindex = jest.fn().mockReturnValue(Promise.resolve()); await unfinalizedBlocksService2.init(reindex); expect(reindex).toHaveBeenCalledWith( - expect.objectContaining({blockHash: '0xabc90f', blockHeight: 90, parentHash: '0xabc89f'}) + expect.objectContaining({ blockHash: '0xabc90f', blockHeight: 90, parentHash: '0xabc89f' }) ); expect((unfinalizedBlocksService2 as any).lastCheckedBlockHeight).toBe(90); }); diff --git a/packages/node-core/src/indexer/unfinalizedBlocks.service.ts b/packages/node-core/src/indexer/unfinalizedBlocks.service.ts index b1724b2653..a33eb879d3 100644 --- a/packages/node-core/src/indexer/unfinalizedBlocks.service.ts +++ b/packages/node-core/src/indexer/unfinalizedBlocks.service.ts @@ -2,8 +2,10 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; +import { Inject, Injectable } from '@nestjs/common'; import { Transaction } from '@subql/x-sequelize'; import { isEqual, last } from 'lodash'; +import { IBlockchainService } from '../blockchain.service'; import { NodeConfig } from '../configure'; import { Header, IBlock } from '../indexer/types'; import { getLogger } from '../logger'; @@ -31,43 +33,28 @@ export interface IUnfinalizedBlocksService extends IUnfinalizedBlocksServiceU resetUnfinalizedBlocks(tx?: Transaction): void; resetLastFinalizedVerifiedHeight(tx?: Transaction): void; getMetadataUnfinalizedBlocks(): Promise; - - // Used by reindex service - getHeaderForHeight(height: number): Promise
; } export interface IUnfinalizedBlocksServiceUtil { registerFinalizedBlock(header: Header): void; } -export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlocksService { +@Injectable() +export class UnfinalizedBlocksService implements IUnfinalizedBlocksService { private _unfinalizedBlocks?: UnfinalizedBlocks; private _finalizedHeader?: Header; protected lastCheckedBlockHeight?: number; - // protected abstract blockToHeader(block: B): Header; - protected abstract getFinalizedHead(): Promise
; - protected abstract getHeaderForHash(hash: string): Promise
; - abstract getHeaderForHeight(height: number): Promise
; - @mainThreadOnly() - protected blockToHeader(block: IBlock): Header { + private blockToHeader(block: IBlock): Header { return block.getHeader(); } - private set unfinalizedBlocks(unfinalizedBlocks: UnfinalizedBlocks) { - this._unfinalizedBlocks = unfinalizedBlocks; - } - protected get unfinalizedBlocks(): UnfinalizedBlocks { assert(this._unfinalizedBlocks !== undefined, new Error('Unfinalized blocks service has not been initialized')); return this._unfinalizedBlocks; } - private set finalizedHeader(finalizedHeader: Header) { - this._finalizedHeader = finalizedHeader; - } - protected get finalizedHeader(): Header { assert(this._finalizedHeader !== undefined, new Error('Unfinalized blocks service has not been initialized')); return this._finalizedHeader; @@ -75,15 +62,16 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo constructor( protected readonly nodeConfig: NodeConfig, - protected readonly storeModelProvider: IStoreModelProvider + protected readonly storeModelProvider: IStoreModelProvider, + @Inject('IBlockchainService') private blockchainService: IBlockchainService ) {} async init(reindex: (tagetHeader: Header) => Promise): Promise
{ logger.info(`Unfinalized blocks is ${this.nodeConfig.unfinalizedBlocks ? 'enabled' : 'disabled'}`); - this.unfinalizedBlocks = await this.getMetadataUnfinalizedBlocks(); + this._unfinalizedBlocks = await this.getMetadataUnfinalizedBlocks(); this.lastCheckedBlockHeight = await this.getLastFinalizedVerifiedHeight(); - this.finalizedHeader = await this.getFinalizedHead(); + this._finalizedHeader = await this.blockchainService.getFinalizedHeader(); if (this.unfinalizedBlocks.length) { logger.info('Processing unfinalized blocks'); @@ -134,7 +122,7 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo if (this.finalizedHeader && this.finalizedBlockNumber >= header.blockHeight) { return; } - this.finalizedHeader = header; + this._finalizedHeader = header; } private async registerUnfinalizedBlock(header: Header): Promise { @@ -164,7 +152,7 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo // remove any records less and equal than input finalized blockHeight private removeFinalized(blockHeight: number): void { - this.unfinalizedBlocks = this.unfinalizedBlocks.filter(({ blockHeight: height }) => height > blockHeight); + this._unfinalizedBlocks = this.unfinalizedBlocks.filter(({ blockHeight: height }) => height > blockHeight); } // find closest record from block heights @@ -201,14 +189,14 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo * If we're off by a large number of blocks we can optimise by getting the block hash directly */ if (header.blockHeight - lastVerifiableBlock.blockHeight > UNFINALIZED_THRESHOLD) { - header = await this.getHeaderForHeight(lastVerifiableBlock.blockHeight); + header = await this.blockchainService.getHeaderForHeight(lastVerifiableBlock.blockHeight); } else { while (lastVerifiableBlock.blockHeight !== header.blockHeight) { assert( header.parentHash, 'When iterate back parent hashes to find matching height, we expect parentHash to be exist' ); - header = await this.getHeaderForHash(header.parentHash); + header = await this.blockchainService.getHeaderForHash(header.parentHash); } } @@ -238,14 +226,14 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo // Get the new parent assert(checkingHeader.parentHash, 'Expect checking header parentHash to be exist'); - checkingHeader = await this.getHeaderForHash(checkingHeader.parentHash); + checkingHeader = await this.blockchainService.getHeaderForHash(checkingHeader.parentHash); } if (!this.lastCheckedBlockHeight) { return undefined; } - return this.getHeaderForHeight(this.lastCheckedBlockHeight); + return this.blockchainService.getHeaderForHeight(this.lastCheckedBlockHeight); } // Finds the last POI that had a correct block hash, this is used with the Eth sdk @@ -267,7 +255,7 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo // Work backwards to find a block on chain that matches POI for (const indexedBlock of indexedBlocks) { - const chainHeader = await this.getHeaderForHeight(indexedBlock.id); + const chainHeader = await this.blockchainService.getHeaderForHeight(indexedBlock.id); // Need to convert to PoiBlock to encode block hash to Uint8Array properly const testPoiBlock = PoiBlock.create( @@ -300,7 +288,7 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo async resetUnfinalizedBlocks(tx?: Transaction): Promise { await this.storeModelProvider.metadata.set(METADATA_UNFINALIZED_BLOCKS_KEY, '[]', tx); - this.unfinalizedBlocks = []; + this._unfinalizedBlocks = []; } async resetLastFinalizedVerifiedHeight(tx?: Transaction): Promise { diff --git a/packages/node-core/src/indexer/worker/worker.core.module.ts b/packages/node-core/src/indexer/worker/worker.core.module.ts index 0aada5be69..b6d6790a12 100644 --- a/packages/node-core/src/indexer/worker/worker.core.module.ts +++ b/packages/node-core/src/indexer/worker/worker.core.module.ts @@ -4,12 +4,16 @@ import {Module} from '@nestjs/common'; import {ConnectionPoolService} from '../connectionPool.service'; import {ConnectionPoolStateManager} from '../connectionPoolState.manager'; +import {DynamicDsService} from '../dynamic-ds.service'; import {InMemoryCacheService} from '../inMemoryCache.service'; import {MonitorService} from '../monitor.service'; import {SandboxService} from '../sandbox.service'; +import {UnfinalizedBlocksService} from '../unfinalizedBlocks.service'; import {WorkerInMemoryCacheService} from './worker.cache.service'; import {WorkerConnectionPoolStateManager} from './worker.connectionPoolState.manager'; +import {WorkerDynamicDsService} from './worker.dynamic-ds.service'; import {WorkerMonitorService} from './worker.monitor.service'; +import {WorkerUnfinalizedBlocksService} from './worker.unfinalizedBlocks.service'; @Module({ providers: [ @@ -27,7 +31,22 @@ import {WorkerMonitorService} from './worker.monitor.service'; provide: InMemoryCacheService, useFactory: () => new WorkerInMemoryCacheService((global as any).host), }, + { + provide: 'IUnfinalizedBlocksService', + useFactory: () => new WorkerUnfinalizedBlocksService((global as any).host), + }, + { + provide: DynamicDsService, + useFactory: () => new WorkerDynamicDsService((global as any).host), + }, + ], + exports: [ + ConnectionPoolService, + SandboxService, + MonitorService, + InMemoryCacheService, + 'IUnfinalizedBlocksService', + DynamicDsService, ], - exports: [ConnectionPoolService, SandboxService, MonitorService, InMemoryCacheService], }) export class WorkerCoreModule {} diff --git a/packages/node-core/src/subcommands/index.ts b/packages/node-core/src/subcommands/index.ts index 9badc6fa70..62e8687bc4 100644 --- a/packages/node-core/src/subcommands/index.ts +++ b/packages/node-core/src/subcommands/index.ts @@ -6,3 +6,4 @@ export * from './forceClean.module'; export * from './foreceClean.init'; export * from './reindex.init'; export * from './reindex.service'; +export * from './testing.core.module'; diff --git a/packages/node-core/src/subcommands/reindex.service.ts b/packages/node-core/src/subcommands/reindex.service.ts index f117172155..7a2f1d1360 100644 --- a/packages/node-core/src/subcommands/reindex.service.ts +++ b/packages/node-core/src/subcommands/reindex.service.ts @@ -5,6 +5,7 @@ import assert from 'assert'; import { Inject, Injectable } from '@nestjs/common'; import { BaseDataSource } from '@subql/types-core'; import { Sequelize } from '@subql/x-sequelize'; +import { IBlockchainService } from '../blockchain.service'; import { NodeConfig, ProjectUpgradeService } from '../configure'; import { IUnfinalizedBlocksService, @@ -37,7 +38,8 @@ export class ReindexService

, - @Inject('DynamicDsService') private readonly dynamicDsService: DynamicDsService + @Inject('DynamicDsService') private readonly dynamicDsService: DynamicDsService, + @Inject('IBlockchainService') private readonly blockchainService: IBlockchainService, ) {} private get metadataRepo(): IMetadata { @@ -63,7 +65,7 @@ export class ReindexService

{ - const inputHeader = await this.unfinalizedBlocksService.getHeaderForHeight(inputHeight); + const inputHeader = await this.blockchainService.getHeaderForHeight(inputHeight); const unfinalizedBlocks = await this.unfinalizedBlocksService.getMetadataUnfinalizedBlocks(); const bestBlocks = unfinalizedBlocks.filter(({ blockHeight }) => blockHeight <= inputHeight); diff --git a/packages/node-core/src/subcommands/testing.core.module.ts b/packages/node-core/src/subcommands/testing.core.module.ts new file mode 100644 index 0000000000..673d595b1f --- /dev/null +++ b/packages/node-core/src/subcommands/testing.core.module.ts @@ -0,0 +1,48 @@ +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import {Module} from '@nestjs/common'; + +import {EventEmitter2} from '@nestjs/event-emitter'; +import {SchedulerRegistry} from '@nestjs/schedule'; +import { + ConnectionPoolService, + ConnectionPoolStateManager, + InMemoryCacheService, + PoiService, + PoiSyncService, + StoreCacheService, + StoreService, + SandboxService, + DsProcessorService, + UnfinalizedBlocksService, + DynamicDsService, +} from '@subql/node-core'; + +@Module({ + providers: [ + SchedulerRegistry, + InMemoryCacheService, + StoreService, + StoreCacheService, + EventEmitter2, + PoiService, + PoiSyncService, + SandboxService, + DsProcessorService, + DynamicDsService, + UnfinalizedBlocksService, + ConnectionPoolStateManager, + ConnectionPoolService, + ], + exports: [ + DsProcessorService, + PoiService, + PoiSyncService, + StoreService, + DynamicDsService, + UnfinalizedBlocksService, + SandboxService, + ], +}) +export class TestingCoreModule {} diff --git a/packages/node/src/blockchain.service.ts b/packages/node/src/blockchain.service.ts new file mode 100644 index 0000000000..9e22e17e64 --- /dev/null +++ b/packages/node/src/blockchain.service.ts @@ -0,0 +1,165 @@ +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import { isMainThread } from 'worker_threads'; +import { Inject, Injectable } from '@nestjs/common'; +import { isCustomDs, isRuntimeDs } from '@subql/common-substrate'; +import { + DatasourceParams, + Header, + IBlock, + IBlockchainService, + mainThreadOnly, +} from '@subql/node-core'; +import { + SubstrateCustomDatasource, + SubstrateCustomHandler, + SubstrateDatasource, + SubstrateHandlerKind, + SubstrateMapping, +} from '@subql/types'; +import { + SubqueryProject, + SubstrateProjectDs, +} from './configure/SubqueryProject'; +import { ApiService } from './indexer/api.service'; +import { RuntimeService } from './indexer/runtime/runtimeService'; +import { + ApiAt, + BlockContent, + getBlockSize, + isFullBlock, + LightBlockContent, +} from './indexer/types'; +import { + calcInterval, + getBlockByHeight, + getTimestamp, + substrateHeaderToHeader, +} from './utils/substrate'; + +const BLOCK_TIME_VARIANCE = 5000; //ms +const INTERVAL_PERCENT = 0.9; + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { version: packageVersion } = require('../package.json'); + +@Injectable() +export class BlockchainService + implements + IBlockchainService< + SubstrateDatasource, + SubstrateCustomDatasource< + string, + SubstrateMapping + >, + SubqueryProject, + ApiAt, + LightBlockContent, + BlockContent + > { + constructor( + @Inject('APIService') private apiService: ApiService, + @Inject(isMainThread ? RuntimeService : 'Null') + private runtimeService: RuntimeService, + ) {} + + isCustomDs = isCustomDs; + isRuntimeDs = isRuntimeDs; + blockHandlerKind = SubstrateHandlerKind.Block; + packageVersion = packageVersion; + + async fetchBlocks( + blockNums: number[], + ): Promise[] | IBlock[]> { + const specChanged = await this.runtimeService.specChanged( + blockNums[blockNums.length - 1], + ); + + // If specVersion not changed, a known overallSpecVer will be pass in + // Otherwise use api to fetch runtimes + return this.apiService.fetchBlocks( + blockNums, + specChanged ? undefined : this.runtimeService.parentSpecVersion, + ); + } + + async onProjectChange(project: SubqueryProject): Promise { + // Only network with chainTypes require to reload + await this.apiService.updateChainTypes(); + this.apiService.updateBlockFetching(); + } + + async getBlockTimestamp(height: number): Promise { + const block = await getBlockByHeight(this.apiService.api, height); + return getTimestamp(block); + } + + getBlockSize(block: IBlock): number { + return getBlockSize(block); + } + + async getFinalizedHeader(): Promise

{ + const finalizedHash = + await this.apiService.unsafeApi.rpc.chain.getFinalizedHead(); + const finalizedHeader = + await this.apiService.unsafeApi.rpc.chain.getHeader(finalizedHash); + return substrateHeaderToHeader(finalizedHeader); + } + + async getBestHeight(): Promise { + const bestHeader = await this.apiService.unsafeApi.rpc.chain.getHeader(); + return bestHeader.number.toNumber(); + } + // eslint-disable-next-line @typescript-eslint/require-await + async getChainInterval(): Promise { + const chainInterval = calcInterval(this.apiService.unsafeApi) + .muln(INTERVAL_PERCENT) + .toNumber(); + + return Math.min(BLOCK_TIME_VARIANCE, chainInterval); + } + + // TODO can this decorator be in unfinalizedBlocks Service? + @mainThreadOnly() + async getHeaderForHash(hash: string): Promise
{ + return substrateHeaderToHeader( + await this.apiService.unsafeApi.rpc.chain.getHeader(hash), + ); + } + + // TODO can this decorator be in unfinalizedBlocks Service? + @mainThreadOnly() + async getHeaderForHeight(height: number): Promise
{ + const hash = await this.apiService.unsafeApi.rpc.chain.getBlockHash(height); + return this.getHeaderForHash(hash.toHex()); + } + + // eslint-disable-next-line @typescript-eslint/require-await + async updateDynamicDs( + params: DatasourceParams, + dsObj: SubstrateProjectDs, + ): Promise { + if (isCustomDs(dsObj)) { + dsObj.processor.options = { + ...dsObj.processor.options, + ...params.args, + }; + // TODO needs dsProcessorService + // await this.dsProcessorService.validateCustomDs([dsObj]); + } else if (isRuntimeDs(dsObj)) { + // XXX add any modifications to the ds here + } + } + + async getSafeApi(block: LightBlockContent | BlockContent): Promise { + const runtimeVersion = !isFullBlock(block) + ? undefined + : await this.runtimeService.getRuntimeVersion(block.block); + + return this.apiService.getPatchedApi( + block.block.block.header, + runtimeVersion, + ); + } +} diff --git a/packages/node/src/configure/SchemaMigration.service.test.ts b/packages/node/src/configure/SchemaMigration.service.test.ts index 4ef4bb3c89..a8f6e4be40 100644 --- a/packages/node/src/configure/SchemaMigration.service.test.ts +++ b/packages/node/src/configure/SchemaMigration.service.test.ts @@ -2,10 +2,9 @@ // SPDX-License-Identifier: GPL-3.0 import { INestApplication } from '@nestjs/common'; -import { DbOption, StoreCacheService } from '@subql/node-core'; +import { DbOption, StoreCacheService, ProjectService } from '@subql/node-core'; import { QueryTypes, Sequelize } from '@subql/x-sequelize'; import { rimraf } from 'rimraf'; -import { ProjectService } from '../indexer/project.service'; import { prepareApp } from '../utils/test.utils'; const option: DbOption = { @@ -52,7 +51,6 @@ describe('SchemaMigration integration tests', () => { schemaName = 'test-migrations-6'; app = await prepareApp(schemaName, cid); - projectService = app.get('IProjectService'); await projectService.init(1); @@ -129,7 +127,6 @@ describe('SchemaMigration integration tests', () => { projectService = app.get('IProjectService'); const projectUpgradeService = app.get('IProjectUpgradeService'); const storeCache = app.get('IStoreModelProvider'); - await projectService.init(1); tempDir = (projectService as any).project.root; diff --git a/packages/node/src/indexer/api.service.test.ts b/packages/node/src/indexer/api.service.test.ts index 8d6099cef9..df0189a309 100644 --- a/packages/node/src/indexer/api.service.test.ts +++ b/packages/node/src/indexer/api.service.test.ts @@ -88,8 +88,7 @@ describe('ApiService', () => { app = module.createNestApplication(); await app.init(); - const apiService = app.get(ApiService); - return apiService; + return app.get(ApiService); }; it('can instantiate api', async () => { @@ -507,8 +506,7 @@ describe('Load chain type hasher', () => { app = module.createNestApplication(); await app.init(); - const apiService = app.get(ApiService); - return apiService; + return app.get(ApiService); }; it('should use new hasher function, types hasher string should be replaced with function', async () => { diff --git a/packages/node/src/indexer/api.service.ts b/packages/node/src/indexer/api.service.ts index 459a51de30..dd70a04d81 100644 --- a/packages/node/src/indexer/api.service.ts +++ b/packages/node/src/indexer/api.service.ts @@ -117,8 +117,7 @@ export class ApiService ApiPromiseConnection, IEndpointConfig > - implements OnApplicationShutdown -{ + implements OnApplicationShutdown { private _fetchBlocksFunction?: FetchFunc; private fetchBlocksBatches: GetFetchFunc = () => this.fetchBlocksFunction; private _currentBlockHash?: string; @@ -285,8 +284,8 @@ export class ApiService header: Header, runtimeVersion?: RuntimeVersion, ): Promise { - this.currentBlockHash = header.hash.toString(); - this.currentBlockNumber = header.number.toNumber(); + this._currentBlockHash = header.hash.toString(); + this._currentBlockNumber = header.number.toNumber(); const api = this.api; const apiAt = (await api.at( diff --git a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts deleted file mode 100644 index c3301de374..0000000000 --- a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import assert from 'assert'; -import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; -import { EventEmitter2 } from '@nestjs/event-emitter'; -import { - NodeConfig, - StoreService, - IProjectService, - BlockDispatcher, - ProcessBlockResponse, - IProjectUpgradeService, - PoiSyncService, - IBlock, - IStoreModelProvider, -} from '@subql/node-core'; -import { SubstrateBlock, SubstrateDatasource } from '@subql/types'; -import { SubqueryProject } from '../../configure/SubqueryProject'; -import { ApiService } from '../api.service'; -import { IndexerManager } from '../indexer.manager'; -import { RuntimeService } from '../runtime/runtimeService'; -import { BlockContent, isFullBlock, LightBlockContent } from '../types'; - -/** - * @description Intended to behave the same as WorkerBlockDispatcherService but doesn't use worker threads or any parallel processing - */ -@Injectable() -export class BlockDispatcherService - extends BlockDispatcher - implements OnApplicationShutdown -{ - private _runtimeService?: RuntimeService; - - constructor( - private apiService: ApiService, - nodeConfig: NodeConfig, - private indexerManager: IndexerManager, - eventEmitter: EventEmitter2, - @Inject('IProjectService') - projectService: IProjectService, - @Inject('IProjectUpgradeService') - projectUpgradeService: IProjectUpgradeService, - storeService: StoreService, - @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider, - poiSyncService: PoiSyncService, - @Inject('ISubqueryProject') project: SubqueryProject, - ) { - super( - nodeConfig, - eventEmitter, - projectService, - projectUpgradeService, - storeService, - storeModelProvider, - poiSyncService, - project, - async ( - blockNums: number[], - ): Promise[] | IBlock[]> => { - const specChanged = await this.runtimeService.specChanged( - blockNums[blockNums.length - 1], - ); - - // If specVersion not changed, a known overallSpecVer will be pass in - // Otherwise use api to fetch runtimes - return this.apiService.fetchBlocks( - blockNums, - specChanged ? undefined : this.runtimeService.parentSpecVersion, - ); - }, - ); - } - - private get runtimeService(): RuntimeService { - assert(this._runtimeService, 'Runtime service not initialized'); - return this._runtimeService; - } - private set runtimeService(value: RuntimeService) { - this._runtimeService = value; - } - - async init( - onDynamicDsCreated: (height: number) => void, - runtimeService?: RuntimeService, - ): Promise { - await super.init(onDynamicDsCreated); - if (runtimeService) this.runtimeService = runtimeService; - } - - protected async indexBlock( - block: IBlock | IBlock, - ): Promise { - const runtimeVersion = !isFullBlock(block.block) - ? undefined - : await this.runtimeService.getRuntimeVersion(block.block.block); - - return this.indexerManager.indexBlock( - block, - await this.projectService.getDataSources(block.getHeader().blockHeight), - runtimeVersion, - ); - } - - protected getBlockSize( - block: IBlock, - ): number { - return block.block.events.reduce( - (acc, evt) => acc + evt.encodedLength, - (block.block.block as SubstrateBlock)?.encodedLength ?? 0, - ); - } -} diff --git a/packages/node/src/indexer/blockDispatcher/index.ts b/packages/node/src/indexer/blockDispatcher/index.ts index 8b646d2a9c..0081a692eb 100644 --- a/packages/node/src/indexer/blockDispatcher/index.ts +++ b/packages/node/src/indexer/blockDispatcher/index.ts @@ -1,7 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { BlockDispatcherService } from './block-dispatcher.service'; +// import { BlockDispatcherService } from './block-dispatcher.service'; import { WorkerBlockDispatcherService } from './worker-block-dispatcher.service'; -export { BlockDispatcherService, WorkerBlockDispatcherService }; +export { WorkerBlockDispatcherService }; diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index fc92ebca9e..5ac0d589cf 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -17,14 +17,14 @@ import { createIndexerWorker as createIndexerWorkerCore, MonitorServiceInterface, IStoreModelProvider, + UnfinalizedBlocksService, + DynamicDsService, } from '@subql/node-core'; -import { SubstrateBlock, SubstrateDatasource } from '@subql/types'; +import { SubstrateDatasource } from '@subql/types'; import { SubqueryProject } from '../../configure/SubqueryProject'; import { ApiPromiseConnection } from '../apiPromise.connection'; -import { DynamicDsService } from '../dynamic-ds.service'; import { RuntimeService } from '../runtime/runtimeService'; -import { BlockContent } from '../types'; -import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service'; +import { BlockContent, LightBlockContent } from '../types'; import { IIndexerWorker } from '../worker/worker'; import { FetchBlockResponse } from '../worker/worker.service'; @@ -37,10 +37,9 @@ export class WorkerBlockDispatcherService extends WorkerBlockDispatcher< SubstrateDatasource, IndexerWorker, - SubstrateBlock + BlockContent | LightBlockContent > - implements OnApplicationShutdown -{ + implements OnApplicationShutdown { private _runtimeService?: RuntimeService; constructor( @@ -55,8 +54,10 @@ export class WorkerBlockDispatcherService @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider, poiSyncService: PoiSyncService, @Inject('ISubqueryProject') project: SubqueryProject, - dynamicDsService: DynamicDsService, - unfinalizedBlocksService: UnfinalizedBlocksService, + dynamicDsService: DynamicDsService, + unfinalizedBlocksService: UnfinalizedBlocksService< + BlockContent | LightBlockContent + >, connectionPoolState: ConnectionPoolStateManager, monitorService?: MonitorServiceInterface, ) { @@ -96,17 +97,13 @@ export class WorkerBlockDispatcherService return this._runtimeService; } - private set runtimeService(runtimeService: RuntimeService) { - this._runtimeService = runtimeService; - } - async init( onDynamicDsCreated: (height: number) => void, runtimeService?: RuntimeService, ): Promise { await super.init(onDynamicDsCreated); // Sync workers runtime from main - if (runtimeService) this.runtimeService = runtimeService; + if (runtimeService) this._runtimeService = runtimeService; this.syncWorkerRuntimes(); } diff --git a/packages/node/src/indexer/dictionary/substrateDictionary.service.spec.ts b/packages/node/src/indexer/dictionary/substrateDictionary.service.spec.ts index f0c29297a1..bbb3500c46 100644 --- a/packages/node/src/indexer/dictionary/substrateDictionary.service.spec.ts +++ b/packages/node/src/indexer/dictionary/substrateDictionary.service.spec.ts @@ -2,12 +2,16 @@ // SPDX-License-Identifier: GPL-3.0 import { EventEmitter2 } from '@nestjs/event-emitter'; +import { isCustomDs } from '@subql/common-substrate'; -import { NodeConfig } from '@subql/node-core'; +import { + NodeConfig, + DsProcessorService, + IBlockchainService, +} from '@subql/node-core'; import axios from 'axios'; import { GraphQLSchema } from 'graphql'; import { SubqueryProject } from '../../configure/SubqueryProject'; -import { DsProcessorService } from '../ds-processor.service'; import { SubstrateDictionaryService } from './substrateDictionary.service'; import { SubstrateDictionaryV1 } from './v1'; import { SubstrateDictionaryV2 } from './v2'; @@ -50,7 +54,11 @@ describe('Substrate Dictionary service', function () { ['wss://polkadot.api.onfinality.io/public-ws'], '0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3', ); - const dsProcessor = new DsProcessorService(project, nodeConfig); + const dsProcessor = new DsProcessorService( + project, + { isCustomDs } as IBlockchainService, + nodeConfig, + ); dictionaryService = new SubstrateDictionaryService( project, diff --git a/packages/node/src/indexer/dictionary/substrateDictionary.service.ts b/packages/node/src/indexer/dictionary/substrateDictionary.service.ts index 8c09f6f037..383b0b3db1 100644 --- a/packages/node/src/indexer/dictionary/substrateDictionary.service.ts +++ b/packages/node/src/indexer/dictionary/substrateDictionary.service.ts @@ -5,12 +5,14 @@ import assert from 'assert'; import { Inject, Injectable } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { NETWORK_FAMILY } from '@subql/common'; -import { isCustomDs } from '@subql/common-substrate'; -import { NodeConfig, DictionaryService, getLogger } from '@subql/node-core'; +import { + NodeConfig, + DictionaryService, + getLogger, + DsProcessorService, +} from '@subql/node-core'; import { SubstrateBlock, SubstrateDatasource } from '@subql/types'; -import { DsProcessor } from '@subql/types-core'; import { SubqueryProject } from '../../configure/SubqueryProject'; -import { DsProcessorService } from '../ds-processor.service'; import { SpecVersion } from './types'; import { SubstrateDictionaryV1 } from './v1'; import { SubstrateDictionaryV2 } from './v2'; diff --git a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts index b648e85730..c5055547d4 100644 --- a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts +++ b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts @@ -2,7 +2,12 @@ // SPDX-License-Identifier: GPL-3.0 import { EventEmitter2 } from '@nestjs/event-emitter'; -import { NodeConfig } from '@subql/node-core'; +import { isCustomDs } from '@subql/common-substrate'; +import { + NodeConfig, + DsProcessorService, + IBlockchainService, +} from '@subql/node-core'; import { SubstrateBlockHandler, SubstrateCallHandler, @@ -13,7 +18,6 @@ import { } from '@subql/types'; import { GraphQLSchema } from 'graphql'; import { SubqueryProject } from '../../../configure/SubqueryProject'; -import { DsProcessorService } from '../../ds-processor.service'; import { SubstrateDictionaryService } from '../substrateDictionary.service'; import { buildDictionaryV1QueryEntries } from './substrateDictionaryV1'; @@ -44,7 +48,11 @@ describe('Substrate DictionaryService', () => { project, nodeConfig, new EventEmitter2(), - new DsProcessorService(project, nodeConfig), + new DsProcessorService( + project, + { isCustomDs } as IBlockchainService, + nodeConfig, + ), ); // prepare dictionary service diff --git a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts index 15b0b94eb0..596e2f7400 100644 --- a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts +++ b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts @@ -14,7 +14,13 @@ import { SubstrateHandlerKind, SubstrateRuntimeHandlerFilter, } from '@subql/common-substrate'; -import { NodeConfig, DictionaryV1, timeout, getLogger } from '@subql/node-core'; +import { + NodeConfig, + DictionaryV1, + timeout, + getLogger, + DsProcessorService, +} from '@subql/node-core'; import { SubstrateBlockFilter, SubstrateDatasource } from '@subql/types'; import { DictionaryQueryCondition, @@ -24,7 +30,6 @@ import { buildQuery, GqlNode, GqlQuery } from '@subql/utils'; import { sortBy, uniqBy } from 'lodash'; import { SubqueryProject } from '../../../configure/SubqueryProject'; import { isBaseHandler, isCustomHandler } from '../../../utils/project'; -import { DsProcessorService } from '../../ds-processor.service'; import { SpecVersion, SpecVersionDictionary } from '../types'; type GetDsProcessor = DsProcessorService['getDsProcessor']; diff --git a/packages/node/src/indexer/ds-processor.service.spec.ts b/packages/node/src/indexer/ds-processor.service.spec.ts index 66edc8e82a..e6c0a40fde 100644 --- a/packages/node/src/indexer/ds-processor.service.spec.ts +++ b/packages/node/src/indexer/ds-processor.service.spec.ts @@ -3,11 +3,10 @@ import path from 'path'; import { isCustomDs } from '@subql/common-substrate'; -import { NodeConfig } from '@subql/node-core'; -import { SubstrateCustomDatasource } from '@subql/types'; +import { NodeConfig, DsProcessorService } from '@subql/node-core'; +import { SubstrateCustomDatasource, SubstrateDatasource } from '@subql/types'; import { GraphQLSchema } from 'graphql'; import { SubqueryProject } from '../configure/SubqueryProject'; -import { DsProcessorService } from './ds-processor.service'; function getTestProject( extraDataSources: SubstrateCustomDatasource[], @@ -40,12 +39,16 @@ const nodeConfig = new NodeConfig({ }); describe('DsProcessorService', () => { - let service: DsProcessorService; + let service: DsProcessorService; let project: SubqueryProject; beforeEach(() => { project = getTestProject([]); - service = new DsProcessorService(project, nodeConfig); + service = new DsProcessorService( + project, + { isCustomDs } as any, + nodeConfig, + ); }); it('can validate custom ds', async () => { @@ -66,7 +69,11 @@ describe('DsProcessorService', () => { }; project = getTestProject([badDs]); - service = new DsProcessorService(project, nodeConfig); + service = new DsProcessorService( + project, + { isCustomDs } as any, + nodeConfig, + ); await expect( service.validateProjectCustomDatasources(project.dataSources), diff --git a/packages/node/src/indexer/ds-processor.service.ts b/packages/node/src/indexer/ds-processor.service.ts deleted file mode 100644 index cead77c1bd..0000000000 --- a/packages/node/src/indexer/ds-processor.service.ts +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { Injectable } from '@nestjs/common'; -import { isCustomDs } from '@subql/common-substrate'; -import { BaseDsProcessorService } from '@subql/node-core'; -import { - SubstrateCustomDatasource, - SubstrateCustomHandler, - SubstrateDatasource, - SubstrateDatasourceProcessor, - SubstrateMapping, -} from '@subql/types'; - -@Injectable() -export class DsProcessorService extends BaseDsProcessorService< - SubstrateDatasource, - SubstrateCustomDatasource>, - SubstrateDatasourceProcessor> -> { - protected isCustomDs = isCustomDs; -} diff --git a/packages/node/src/indexer/dynamic-ds.service.ts b/packages/node/src/indexer/dynamic-ds.service.ts deleted file mode 100644 index 0db1c87b27..0000000000 --- a/packages/node/src/indexer/dynamic-ds.service.ts +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { Inject, Injectable } from '@nestjs/common'; -import { isCustomDs, isRuntimeDs } from '@subql/common-substrate'; -import { - DatasourceParams, - DynamicDsService as BaseDynamicDsService, -} from '@subql/node-core'; -import { - SubqueryProject, - SubstrateProjectDs, -} from '../configure/SubqueryProject'; -import { DsProcessorService } from './ds-processor.service'; - -@Injectable() -export class DynamicDsService extends BaseDynamicDsService< - SubstrateProjectDs, - SubqueryProject -> { - constructor( - private readonly dsProcessorService: DsProcessorService, - @Inject('ISubqueryProject') project: SubqueryProject, - ) { - super(project); - } - - protected async getDatasource( - params: DatasourceParams, - ): Promise { - const dsObj = this.getTemplate( - params.templateName, - params.startBlock, - ); - - try { - if (isCustomDs(dsObj)) { - dsObj.processor.options = { - ...dsObj.processor.options, - ...params.args, - }; - await this.dsProcessorService.validateCustomDs([dsObj]); - } else if (isRuntimeDs(dsObj)) { - // XXX add any modifications to the ds here - } - - return dsObj; - } catch (e) { - throw new Error( - `Unable to create dynamic datasource.\n ${(e as any).message}`, - ); - } - } -} diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts index 3d4e411961..1232c4e978 100644 --- a/packages/node/src/indexer/fetch.module.ts +++ b/packages/node/src/indexer/fetch.module.ts @@ -14,28 +14,28 @@ import { CoreModule, IStoreModelProvider, ConnectionPoolService, + UnfinalizedBlocksService, + BlockDispatcher, + DsProcessorService, + ProjectService, + DynamicDsService, } from '@subql/node-core'; +import { SubstrateDatasource } from '@subql/types'; +import { BlockchainService } from '../blockchain.service'; import { SubqueryProject } from '../configure/SubqueryProject'; import { ApiService } from './api.service'; import { ApiPromiseConnection } from './apiPromise.connection'; -import { - BlockDispatcherService, - WorkerBlockDispatcherService, -} from './blockDispatcher'; +import { WorkerBlockDispatcherService } from './blockDispatcher'; import { SubstrateDictionaryService } from './dictionary/substrateDictionary.service'; -import { DsProcessorService } from './ds-processor.service'; -import { DynamicDsService } from './dynamic-ds.service'; import { FetchService } from './fetch.service'; import { IndexerManager } from './indexer.manager'; -import { ProjectService } from './project.service'; import { RuntimeService } from './runtime/runtimeService'; -import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; @Module({ imports: [CoreModule], providers: [ { - provide: ApiService, + provide: 'APIService', useFactory: ApiService.init, inject: [ 'ISubqueryProject', @@ -44,27 +44,48 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; NodeConfig, ], }, + { + provide: RuntimeService, // TODO DOING this because of circular reference with dictionary service + useFactory: (apiService: ApiService) => new RuntimeService(apiService), + inject: ['APIService'], + }, + { + provide: 'IBlockchainService', + useClass: BlockchainService, + }, + /* START: Move to node core */ + DsProcessorService, + DynamicDsService, + { + provide: 'IUnfinalizedBlocksService', + useClass: UnfinalizedBlocksService, + }, + { + useClass: ProjectService, + provide: 'IProjectService', + }, + /* END: Move to node core */ IndexerManager, { provide: 'IBlockDispatcher', useFactory: ( nodeConfig: NodeConfig, eventEmitter: EventEmitter2, - projectService: ProjectService, + projectService: ProjectService, projectUpgradeService: IProjectUpgradeService, - apiService: ApiService, - indexerManager: IndexerManager, cacheService: InMemoryCacheService, storeService: StoreService, storeModelProvider: IStoreModelProvider, poiSyncService: PoiSyncService, project: SubqueryProject, - dynamicDsService: DynamicDsService, + dynamicDsService: DynamicDsService, unfinalizedBlocks: UnfinalizedBlocksService, connectionPoolState: ConnectionPoolStateManager, + blockchainService: BlockchainService, + indexerManager: IndexerManager, monitorService?: MonitorService, - ) => - nodeConfig.workers + ) => { + return nodeConfig.workers ? new WorkerBlockDispatcherService( nodeConfig, eventEmitter, @@ -80,10 +101,8 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; connectionPoolState, monitorService, ) - : new BlockDispatcherService( - apiService, + : new BlockDispatcher( nodeConfig, - indexerManager, eventEmitter, projectService, projectUpgradeService, @@ -91,35 +110,30 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; storeModelProvider, poiSyncService, project, - ), + blockchainService, + indexerManager, + ); + }, inject: [ NodeConfig, EventEmitter2, 'IProjectService', 'IProjectUpgradeService', - ApiService, - IndexerManager, InMemoryCacheService, StoreService, 'IStoreModelProvider', PoiSyncService, 'ISubqueryProject', DynamicDsService, - UnfinalizedBlocksService, + 'IUnfinalizedBlocksService', ConnectionPoolStateManager, + 'IBlockchainService', + IndexerManager, MonitorService, ], }, - FetchService, SubstrateDictionaryService, - DsProcessorService, - DynamicDsService, - { - useClass: ProjectService, - provide: 'IProjectService', - }, - UnfinalizedBlocksService, - RuntimeService, + FetchService, ], }) export class FetchModule {} diff --git a/packages/node/src/indexer/fetch.service.spec.ts b/packages/node/src/indexer/fetch.service.spec.ts index fdd7f6fe8b..64db340558 100644 --- a/packages/node/src/indexer/fetch.service.spec.ts +++ b/packages/node/src/indexer/fetch.service.spec.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import { ProjectService } from '@subql/node-core'; import { SubstrateBlockHandler, SubstrateCallHandler, @@ -12,9 +13,8 @@ import { } from '@subql/types'; import { DictionaryQueryEntry } from '@subql/types-core'; import { FetchService } from './fetch.service'; -import { ProjectService } from './project.service'; -const projectService: ProjectService = { +const projectService: ProjectService = { getAllDataSources: () => { return [ makeDs([{ ...blockHandler, filter: { modulo: 5 } }]), @@ -60,7 +60,6 @@ describe('FetchSevice', () => { beforeEach(() => { fetchService = new FetchService( - null as any, // ApiService null as any, // NodeConfig projectService, // ProjectService null as any, // BlockDispatcher, @@ -70,6 +69,7 @@ describe('FetchSevice', () => { null as any, // SchedulerRegistry null as any, // RuntimeService null as any, // StoreCacheService + null as any, // BlockchainService ) as any; }); diff --git a/packages/node/src/indexer/fetch.service.test.ts b/packages/node/src/indexer/fetch.service.test.ts index 4f2027ba9e..8054208743 100644 --- a/packages/node/src/indexer/fetch.service.test.ts +++ b/packages/node/src/indexer/fetch.service.test.ts @@ -2,12 +2,13 @@ // SPDX-License-Identifier: GPL-3.0 import { ApiPromise, HttpProvider } from '@polkadot/api'; -import { ApiService } from './api.service'; import { FetchService } from './fetch.service'; import { createCachedProvider } from './x-provider/cachedProvider'; const POLKADOT_ENDPOINT = 'https://rpc.polkadot.io'; +// TODO move these tests to Blockchain service + describe('FetchService', () => { let fetchService: FetchService; let api: ApiPromise; @@ -18,12 +19,7 @@ describe('FetchService', () => { provider: createCachedProvider(new HttpProvider(POLKADOT_ENDPOINT)), }); - const apiService = { - unsafeApi: api, - } as any as ApiService; - fetchService = new FetchService( - apiService, // ApiService null as any, // NodeConfig null as any, // ProjectService null as any, // BlockDispatcher, @@ -37,6 +33,7 @@ describe('FetchService', () => { null as any, // SchedulerRegistry null as any, // RuntimeService null as any, // StoreCacheService + null as any, // BlockchainService ) as any; }, 10000); diff --git a/packages/node/src/indexer/fetch.service.ts b/packages/node/src/indexer/fetch.service.ts index 9f2eacd163..784d6a07af 100644 --- a/packages/node/src/indexer/fetch.service.ts +++ b/packages/node/src/indexer/fetch.service.ts @@ -4,27 +4,18 @@ import { Inject, Injectable } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { SchedulerRegistry } from '@nestjs/schedule'; -import { ApiPromise } from '@polkadot/api'; - -import { isCustomDs, SubstrateHandlerKind } from '@subql/common-substrate'; import { NodeConfig, BaseFetchService, - getModulos, - Header, IStoreModelProvider, + UnfinalizedBlocksService, + ProjectService, } from '@subql/node-core'; import { SubstrateDatasource, SubstrateBlock } from '@subql/types'; -import { calcInterval, substrateHeaderToHeader } from '../utils/substrate'; -import { ApiService } from './api.service'; +import { BlockchainService } from '../blockchain.service'; import { ISubstrateBlockDispatcher } from './blockDispatcher/substrate-block-dispatcher'; import { SubstrateDictionaryService } from './dictionary/substrateDictionary.service'; -import { ProjectService } from './project.service'; import { RuntimeService } from './runtime/runtimeService'; -import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; - -const BLOCK_TIME_VARIANCE = 5000; //ms -const INTERVAL_PERCENT = 0.9; @Injectable() export class FetchService extends BaseFetchService< @@ -33,17 +24,19 @@ export class FetchService extends BaseFetchService< SubstrateBlock > { constructor( - private apiService: ApiService, nodeConfig: NodeConfig, - @Inject('IProjectService') projectService: ProjectService, + @Inject('IProjectService') + projectService: ProjectService, @Inject('IBlockDispatcher') blockDispatcher: ISubstrateBlockDispatcher, dictionaryService: SubstrateDictionaryService, - unfinalizedBlocksService: UnfinalizedBlocksService, + @Inject('IUnfinalizedBlocksService') + unfinalizedBlocksService: UnfinalizedBlocksService, eventEmitter: EventEmitter2, schedulerRegistry: SchedulerRegistry, private runtimeService: RuntimeService, @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider, + @Inject('IBlockchainService') blockchainService: BlockchainService, ) { super( nodeConfig, @@ -54,37 +47,10 @@ export class FetchService extends BaseFetchService< schedulerRegistry, unfinalizedBlocksService, storeModelProvider, + blockchainService, ); } - get api(): ApiPromise { - return this.apiService.unsafeApi; - } - - protected async getFinalizedHeader(): Promise
{ - const finalizedHash = await this.api.rpc.chain.getFinalizedHead(); - const finalizedHeader = await this.api.rpc.chain.getHeader(finalizedHash); - return substrateHeaderToHeader(finalizedHeader); - } - - protected async getBestHeight(): Promise { - const bestHeader = await this.api.rpc.chain.getHeader(); - return bestHeader.number.toNumber(); - } - - // eslint-disable-next-line @typescript-eslint/require-await - protected async getChainInterval(): Promise { - const chainInterval = calcInterval(this.api) - .muln(INTERVAL_PERCENT) - .toNumber(); - - return Math.min(BLOCK_TIME_VARIANCE, chainInterval); - } - - protected getModulos(dataSources: SubstrateDatasource[]): number[] { - return getModulos(dataSources, isCustomDs, SubstrateHandlerKind.Block); - } - protected async initBlockDispatcher(): Promise { await this.blockDispatcher.init( this.resetForNewDs.bind(this), @@ -97,12 +63,9 @@ export class FetchService extends BaseFetchService< }: { startHeight: number; }): Promise { - this.runtimeService.init(this.getLatestFinalizedHeight.bind(this)); - - await this.runtimeService.syncDictionarySpecVersions(startHeight); - - // setup parentSpecVersion - await this.runtimeService.specChanged(startHeight); - await this.runtimeService.prefetchMeta(startHeight); + await this.runtimeService.init( + startHeight, + this.getLatestFinalizedHeight(), + ); } } diff --git a/packages/node/src/indexer/indexer.manager.spec.ts b/packages/node/src/indexer/indexer.manager.spec.ts deleted file mode 100644 index 2662b939ab..0000000000 --- a/packages/node/src/indexer/indexer.manager.spec.ts +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { EventEmitter2 } from '@nestjs/event-emitter'; -import { SchedulerRegistry } from '@nestjs/schedule'; -import { - SubstrateDatasourceKind, - SubstrateHandlerKind, -} from '@subql/common-substrate'; -import { - StoreService, - PoiService, - PoiSyncService, - NodeConfig, - ConnectionPoolService, - StoreCacheService, - IProjectUpgradeService, - InMemoryCacheService, - SandboxService, -} from '@subql/node-core'; -import { Sequelize } from '@subql/x-sequelize'; -import { GraphQLSchema } from 'graphql'; -import { SubqueryProject } from '../configure/SubqueryProject'; -import { ApiService } from './api.service'; -import { ApiPromiseConnection } from './apiPromise.connection'; -import { DsProcessorService } from './ds-processor.service'; -import { DynamicDsService } from './dynamic-ds.service'; -import { IndexerManager } from './indexer.manager'; -import { ProjectService } from './project.service'; -import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; - -jest.mock('@subql/x-sequelize', () => { - const mSequelize = { - authenticate: jest.fn(), - define: () => ({ - findOne: jest.fn(), - create: (input: any) => input, - }), - query: () => [{ nextval: 1 }], - showAllSchemas: () => ['subquery_1'], - model: () => ({ upsert: jest.fn() }), - sync: jest.fn(), - transaction: () => ({ - commit: jest.fn(), - rollback: jest.fn(), - afterCommit: jest.fn(), - }), - // createSchema: jest.fn(), - }; - const actualSequelize = jest.requireActual('@subql/x-sequelize'); - return { - ...actualSequelize, - Sequelize: jest.fn(() => mSequelize), - }; -}); - -jest.setTimeout(200000); - -const nodeConfig = new NodeConfig({ - subquery: 'asdf', - subqueryName: 'asdf', - networkEndpoint: { 'wss://polkadot.api.onfinality.io/public-ws': {} }, -}); - -function testSubqueryProject_1(): SubqueryProject { - return { - id: 'test', - root: './', - network: { - chainId: '0x', - endpoint: ['wss://polkadot.api.onfinality.io/public-ws'], - }, - dataSources: [ - { - kind: SubstrateDatasourceKind.Runtime, - startBlock: 1, - mapping: { - file: '', - handlers: [ - { handler: 'testSandbox', kind: SubstrateHandlerKind.Event }, - ], - }, - }, - { - kind: SubstrateDatasourceKind.Runtime, - startBlock: 1, - mapping: { - file: '', - handlers: [ - { handler: 'testSandbox', kind: SubstrateHandlerKind.Event }, - ], - }, - }, - ], - schema: new GraphQLSchema({}), - templates: [], - } as unknown as SubqueryProject; -} - -function testSubqueryProject_2(): SubqueryProject { - return { - id: 'test', - root: './', - network: { - endpoint: ['wss://polkadot.api.onfinality.io/public-ws'], - dictionary: `https://api.subquery.network/sq/subquery/dictionary-polkadot`, - chainId: - '0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3', - }, - dataSources: [ - { - kind: SubstrateDatasourceKind.Runtime, - startBlock: 1, - mapping: { - file: '', - handlers: [ - { handler: 'testSandbox', kind: SubstrateHandlerKind.Event }, - ], - }, - }, - ], - schema: new GraphQLSchema({}), - templates: [], - } as unknown as SubqueryProject; -} - -// eslint-disable-next-line jest/no-export -export function mockProjectUpgradeService( - project: SubqueryProject, -): IProjectUpgradeService { - const startBlock = Math.min( - ...project.dataSources.map((ds) => ds.startBlock || 1), - ); - - let currentHeight = startBlock; - return { - isRewindable: true, - init: jest.fn(), - initWorker: jest.fn(), - updateIndexedDeployments: jest.fn(), - currentHeight: currentHeight, - // eslint-disable-next-line @typescript-eslint/require-await - setCurrentHeight: async (height: number) => { - currentHeight = height; - }, - currentProject: project, - projects: new Map([[startBlock, project]]), - getProject: () => project, - rewind: () => Promise.resolve(), - }; -} - -async function createIndexerManager( - project: SubqueryProject, - connectionPoolService: ConnectionPoolService, - nodeConfig: NodeConfig, -): Promise { - const sequelize = new Sequelize(); - const eventEmitter = new EventEmitter2(); - const apiService = await ApiService.init( - project, - connectionPoolService, - eventEmitter, - nodeConfig, - ); - const dsProcessorService = new DsProcessorService(project, nodeConfig); - const dynamicDsService = new DynamicDsService(dsProcessorService, project); - - const storeCache = new StoreCacheService( - sequelize, - nodeConfig, - eventEmitter, - new SchedulerRegistry(), - ); - const storeService = new StoreService( - sequelize, - nodeConfig, - storeCache, - project, - ); - const cacheService = new InMemoryCacheService(); - const poiService = new PoiService(storeCache); - - const poiSyncService = new PoiSyncService(nodeConfig, eventEmitter, project); - const unfinalizedBlocksService = new UnfinalizedBlocksService( - apiService, - nodeConfig, - storeCache, - ); - const sandboxService = new SandboxService( - storeService, - cacheService, - nodeConfig, - project, - ); - - const projectUpgradeService = mockProjectUpgradeService(project); - const projectService = new ProjectService( - dsProcessorService, - apiService, - poiService, - poiSyncService, - sequelize, - project, - projectUpgradeService, - storeService, - nodeConfig, - dynamicDsService, - eventEmitter, - unfinalizedBlocksService, - ); - - return new IndexerManager( - apiService, - nodeConfig, - sandboxService, - dsProcessorService, - dynamicDsService, - unfinalizedBlocksService, - ); -} - -/* - * These tests aren't run because of setup requirements with such a large number of dependencies - */ -describe('IndexerManager', () => { - let indexerManager: IndexerManager; - - afterEach(() => { - (indexerManager as any)?.fetchService.onApplicationShutdown(); - }); - - it.skip('should be able to start the manager (v0.0.1)', async () => { - // indexerManager = await createIndexerManager( - // testSubqueryProject_1(), - // new ConnectionPoolService( - // nodeConfig, - // new ConnectionPoolStateManager(), - // ), - // nodeConfig, - // ); - // await expect(indexerManager.start()).resolves.toBe(undefined); - // expect(Object.keys((indexerManager as any).vms).length).toBe(1); - }); - - it.skip('should be able to start the manager (v0.2.0)', async () => { - // indexerManager = await createIndexerManager( - // testSubqueryProject_2(), - // new ConnectionPoolService( - // nodeConfig, - // new ConnectionPoolStateManager(), - // ), - // nodeConfig, - // ); - // await expect(indexerManager.start()).resolves.toBe(undefined); - // expect(Object.keys((indexerManager as any).vms).length).toBe(1); - }); -}); diff --git a/packages/node/src/indexer/indexer.manager.ts b/packages/node/src/indexer/indexer.manager.ts index bc19f5ffed..3a51ccf1b9 100644 --- a/packages/node/src/indexer/indexer.manager.ts +++ b/packages/node/src/indexer/indexer.manager.ts @@ -1,15 +1,12 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { ApiPromise } from '@polkadot/api'; -import { RuntimeVersion } from '@polkadot/types/interfaces'; import { isBlockHandlerProcessor, isCallHandlerProcessor, isEventHandlerProcessor, - isCustomDs, - isRuntimeDs, SubstrateCustomDataSource, SubstrateHandlerKind, SubstrateRuntimeHandlerInputMap, @@ -21,24 +18,29 @@ import { IndexerSandbox, ProcessBlockResponse, BaseIndexerManager, + DsProcessorService, IBlock, getLogger, + UnfinalizedBlocksService, + IBlockchainService, + DynamicDsService, } from '@subql/node-core'; import { LightSubstrateEvent, SubstrateBlock, SubstrateBlockFilter, + SubstrateCustomDatasource, SubstrateDatasource, SubstrateEvent, SubstrateExtrinsic, } from '@subql/types'; -import { SubstrateProjectDs } from '../configure/SubqueryProject'; +import { + SubqueryProject, + SubstrateProjectDs, +} from '../configure/SubqueryProject'; import * as SubstrateUtil from '../utils/substrate'; import { ApiService as SubstrateApiService } from './api.service'; -import { DsProcessorService } from './ds-processor.service'; -import { DynamicDsService } from './dynamic-ds.service'; import { ApiAt, BlockContent, isFullBlock, LightBlockContent } from './types'; -import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; const logger = getLogger('indexer'); @@ -54,16 +56,28 @@ export class IndexerManager extends BaseIndexerManager< typeof ProcessorTypeMap, SubstrateRuntimeHandlerInputMap > { - protected isRuntimeDs = isRuntimeDs; - protected isCustomDs = isCustomDs; - constructor( - apiService: SubstrateApiService, + @Inject('APIService') apiService: SubstrateApiService, nodeConfig: NodeConfig, sandboxService: SandboxService, - dsProcessorService: DsProcessorService, - dynamicDsService: DynamicDsService, - unfinalizedBlocksService: UnfinalizedBlocksService, + dsProcessorService: DsProcessorService< + SubstrateDatasource, + SubstrateCustomDatasource + >, + dynamicDsService: DynamicDsService, + @Inject('IUnfinalizedBlocksService') + unfinalizedBlocksService: UnfinalizedBlocksService< + BlockContent | LightBlockContent + >, + @Inject('IBlockchainService') + blockchainService: IBlockchainService< + SubstrateDatasource, + SubstrateCustomDatasource, + SubqueryProject, + ApiAt, + LightBlockContent, + BlockContent + >, ) { super( apiService, @@ -74,6 +88,7 @@ export class IndexerManager extends BaseIndexerManager< unfinalizedBlocksService, FilterTypeMap, ProcessorTypeMap, + blockchainService, ); } @@ -81,21 +96,9 @@ export class IndexerManager extends BaseIndexerManager< async indexBlock( block: IBlock, dataSources: SubstrateDatasource[], - runtimeVersion?: RuntimeVersion, ): Promise { return super.internalIndexBlock(block, dataSources, () => - this.getApi(block.block, runtimeVersion), - ); - } - - // eslint-disable-next-line @typescript-eslint/require-await - private async getApi( - block: LightBlockContent | BlockContent, - runtimeVersion?: RuntimeVersion, - ): Promise { - return this.apiService.getPatchedApi( - block.block.block.header, - runtimeVersion, + this.blockchainService.getSafeApi(block.block), ); } @@ -106,7 +109,11 @@ export class IndexerManager extends BaseIndexerManager< ): Promise { if (isFullBlock(blockContent)) { const { block, events, extrinsics } = blockContent; - await this.indexBlockContent(block, dataSources, getVM); + await this.indexContent(SubstrateHandlerKind.Block)( + block, + dataSources, + getVM, + ); // Group the events so they only need to be iterated over a single time const groupedEvents = events.reduce( @@ -134,11 +141,15 @@ export class IndexerManager extends BaseIndexerManager< // Run initialization events for (const event of groupedEvents.init) { - await this.indexEvent(event, dataSources, getVM); + await this.indexContent(SubstrateHandlerKind.Event)(event, dataSources, getVM); } for (const extrinsic of extrinsics) { - await this.indexExtrinsic(extrinsic, dataSources, getVM); + await this.indexContent(SubstrateHandlerKind.Call)( + extrinsic, + dataSources, + getVM, + ); // Process extrinsic events const extrinsicEvents = (groupedEvents[extrinsic.idx] ?? []).sort( @@ -146,49 +157,45 @@ export class IndexerManager extends BaseIndexerManager< ); for (const event of extrinsicEvents) { - await this.indexEvent(event, dataSources, getVM); + await this.indexContent(SubstrateHandlerKind.Event)( + event, + dataSources, + getVM, + ); } } // Run finalization events for (const event of groupedEvents.finalize) { - await this.indexEvent(event, dataSources, getVM); + await this.indexContent(SubstrateHandlerKind.Event)(event, dataSources, getVM); } } else { for (const event of blockContent.events) { - await this.indexEvent(event, dataSources, getVM); + await this.indexContent(SubstrateHandlerKind.Event)( + event, + dataSources, + getVM, + ); } } } - private async indexBlockContent( - block: SubstrateBlock, - dataSources: SubstrateProjectDs[], - getVM: (d: SubstrateProjectDs) => Promise, - ): Promise { - for (const ds of dataSources) { - await this.indexData(SubstrateHandlerKind.Block, block, ds, getVM); - } - } - - private async indexExtrinsic( - extrinsic: SubstrateExtrinsic, - dataSources: SubstrateProjectDs[], - getVM: (d: SubstrateProjectDs) => Promise, - ): Promise { - for (const ds of dataSources) { - await this.indexData(SubstrateHandlerKind.Call, extrinsic, ds, getVM); - } - } - - private async indexEvent( - event: SubstrateEvent | LightSubstrateEvent, + private indexContent( + kind: SubstrateHandlerKind, + ): ( + content: + | SubstrateBlock + | SubstrateExtrinsic + | SubstrateEvent + | LightSubstrateEvent, dataSources: SubstrateProjectDs[], getVM: (d: SubstrateProjectDs) => Promise, - ): Promise { - for (const ds of dataSources) { - await this.indexData(SubstrateHandlerKind.Event, event, ds, getVM); - } + ) => Promise { + return async (content, dataSources, getVM) => { + for (const ds of dataSources) { + await this.indexData(kind, content, ds, getVM); + } + }; } protected async prepareFilteredData( diff --git a/packages/node/src/indexer/project.service.spec.ts b/packages/node/src/indexer/project.service.spec.ts index d148a804f1..b71ca8cd0b 100644 --- a/packages/node/src/indexer/project.service.spec.ts +++ b/packages/node/src/indexer/project.service.spec.ts @@ -4,20 +4,20 @@ import { EventEmitter2, EventEmitterModule } from '@nestjs/event-emitter'; import { Test } from '@nestjs/testing'; import { - BaseProjectService, + ProjectService, ConnectionPoolService, ConnectionPoolStateManager, NodeConfig, ProjectUpgradeService, upgradableSubqueryProject, + DsProcessorService, + DynamicDsService } from '@subql/node-core'; import { SubstrateDatasourceKind, SubstrateHandlerKind } from '@subql/types'; import { GraphQLSchema } from 'graphql'; +import { BlockchainService } from '../blockchain.service'; import { SubqueryProject } from '../configure/SubqueryProject'; import { ApiService } from './api.service'; -import { DsProcessorService } from './ds-processor.service'; -import { DynamicDsService } from './dynamic-ds.service'; -import { ProjectService } from './project.service'; function testSubqueryProject(): SubqueryProject { return { @@ -52,7 +52,7 @@ function testSubqueryProject(): SubqueryProject { } // @ts-ignore -class TestProjectService extends BaseProjectService { +class TestProjectService extends ProjectService { packageVersion = '1.0.0'; async init(startHeight?: number): Promise { @@ -140,7 +140,7 @@ describe('ProjectService', () => { }, { provide: ProjectService, - useFactory: (apiService: ApiService, project: SubqueryProject) => + useFactory: (apiService: ApiService, project: SubqueryProject, blockchainService: BlockchainService) => new TestProjectService( { validateProjectCustomDatasources: jest.fn(), @@ -166,31 +166,31 @@ describe('ProjectService', () => { } as unknown as DynamicDsService, null as unknown as any, null as unknown as any, + blockchainService ), - inject: [ApiService, 'ISubqueryProject'], + inject: ['APIService', 'ISubqueryProject', BlockchainService,], }, EventEmitter2, { - provide: ApiService, + provide: 'APIService', useFactory: ApiService.init, - inject: [ - 'ISubqueryProject', - ConnectionPoolService, - EventEmitter2, - NodeConfig, - ], + inject: ['ISubqueryProject', ConnectionPoolService, EventEmitter2, NodeConfig] }, { provide: ProjectUpgradeService, useValue: projectUpgrade, }, + { + provide: 'IBlockchainService', + useClass: BlockchainService, + } ], imports: [EventEmitterModule.forRoot()], }).compile(); const app = module.createNestApplication(); await app.init(); - apiService = app.get(ApiService); + apiService = app.get('APIService'); projectUpgradeService = app.get( ProjectUpgradeService, ) as ProjectUpgradeService; diff --git a/packages/node/src/indexer/project.service.ts b/packages/node/src/indexer/project.service.ts deleted file mode 100644 index eb9c4ebe03..0000000000 --- a/packages/node/src/indexer/project.service.ts +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { isMainThread } from 'worker_threads'; -import { Inject, Injectable } from '@nestjs/common'; -import { EventEmitter2 } from '@nestjs/event-emitter'; -import { - PoiService, - PoiSyncService, - BaseProjectService, - StoreService, - NodeConfig, - IProjectUpgradeService, - profiler, -} from '@subql/node-core'; -import { SubstrateDatasource } from '@subql/types'; -import { Sequelize } from '@subql/x-sequelize'; -import { SubqueryProject } from '../configure/SubqueryProject'; -import { getBlockByHeight, getTimestamp } from '../utils/substrate'; -import { ApiService } from './api.service'; -import { DsProcessorService } from './ds-processor.service'; -import { DynamicDsService } from './dynamic-ds.service'; -import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; - -// eslint-disable-next-line @typescript-eslint/no-var-requires -const { version: packageVersion } = require('../../package.json'); - -@Injectable() -export class ProjectService extends BaseProjectService< - ApiService, - SubstrateDatasource -> { - protected packageVersion = packageVersion; - - constructor( - dsProcessorService: DsProcessorService, - apiService: ApiService, - @Inject(isMainThread ? PoiService : 'Null') poiService: PoiService, - @Inject(isMainThread ? PoiSyncService : 'Null') - poiSyncService: PoiSyncService, - @Inject(isMainThread ? Sequelize : 'Null') sequelize: Sequelize, - @Inject('ISubqueryProject') project: SubqueryProject, - @Inject('IProjectUpgradeService') - projectUpgradeService: IProjectUpgradeService, - @Inject(isMainThread ? StoreService : 'Null') storeService: StoreService, - nodeConfig: NodeConfig, - dynamicDsService: DynamicDsService, - eventEmitter: EventEmitter2, - unfinalizedBlockService: UnfinalizedBlocksService, - ) { - super( - dsProcessorService, - apiService, - poiService, - poiSyncService, - sequelize, - project, - projectUpgradeService, - storeService, - nodeConfig, - dynamicDsService, - eventEmitter, - unfinalizedBlockService, - ); - } - - @profiler() - async init(startHeight?: number): Promise { - return super.init(startHeight); - } - - protected async getBlockTimestamp(height: number): Promise { - const block = await getBlockByHeight(this.apiService.api, height); - return getTimestamp(block); - } - - protected async onProjectChange(project: SubqueryProject): Promise { - // Only network with chainTypes require to reload - await this.apiService.updateChainTypes(); - this.apiService.updateBlockFetching(); - } -} diff --git a/packages/node/src/indexer/runtime/base-runtime.service.ts b/packages/node/src/indexer/runtime/base-runtime.service.ts index 49dbdf7d04..03764400b9 100644 --- a/packages/node/src/indexer/runtime/base-runtime.service.ts +++ b/packages/node/src/indexer/runtime/base-runtime.service.ts @@ -1,7 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { ApiPromise } from '@polkadot/api'; import { RuntimeVersion } from '@polkadot/types/interfaces'; import { profiler } from '@subql/node-core'; @@ -10,7 +10,6 @@ import * as SubstrateUtil from '../../utils/substrate'; import { ApiService } from '../api.service'; import { SpecVersion } from '../dictionary'; export const SPEC_VERSION_BLOCK_GAP = 100; -type GetLatestFinalizedHeight = () => number; @Injectable() export abstract class BaseRuntimeService { @@ -19,7 +18,7 @@ export abstract class BaseRuntimeService { private currentRuntimeVersion?: RuntimeVersion; latestFinalizedHeight?: number; - constructor(protected apiService: ApiService) {} + constructor(@Inject('APIService') protected apiService: ApiService) {} async specChanged(height: number, specVersion: number): Promise { if (this.parentSpecVersion !== specVersion) { @@ -37,10 +36,6 @@ export abstract class BaseRuntimeService { blockHeight: number, ): Promise<{ blockSpecVersion: number; syncedDictionary: boolean }>; - init(getLatestFinalizedHeight: GetLatestFinalizedHeight): void { - this.latestFinalizedHeight = getLatestFinalizedHeight(); - } - get api(): ApiPromise { return this.apiService.api; } diff --git a/packages/node/src/indexer/runtime/runtimeService.ts b/packages/node/src/indexer/runtime/runtimeService.ts index 231153d811..28e32e0d17 100644 --- a/packages/node/src/indexer/runtime/runtimeService.ts +++ b/packages/node/src/indexer/runtime/runtimeService.ts @@ -1,7 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { getLogger, profiler } from '@subql/node-core'; import { ApiService } from '../api.service'; import { SubstrateDictionaryService } from '../dictionary'; @@ -15,12 +15,24 @@ const logger = getLogger('RuntimeService'); @Injectable() export class RuntimeService extends BaseRuntimeService { constructor( - protected apiService: ApiService, + @Inject('APIService') protected apiService: ApiService, protected dictionaryService?: SubstrateDictionaryService, ) { super(apiService); } + async init( + startHeight: number, + latestFinalizedHeight: number, + ): Promise { + this.latestFinalizedHeight = latestFinalizedHeight; + + await this.syncDictionarySpecVersions(startHeight); + + await this.specChanged(startHeight); + await this.prefetchMeta(startHeight); + } + // get latest specVersions from dictionary async syncDictionarySpecVersions(height: number): Promise { try { diff --git a/packages/node/src/indexer/store.service.test.ts b/packages/node/src/indexer/store.service.test.ts index 3278789dda..b201a9cf5d 100644 --- a/packages/node/src/indexer/store.service.test.ts +++ b/packages/node/src/indexer/store.service.test.ts @@ -8,11 +8,11 @@ import { DbOption, getFunctions, getTriggers, + ProjectService, } from '@subql/node-core'; import { QueryTypes, Sequelize } from '@subql/x-sequelize'; import { rimraf } from 'rimraf'; import { prepareApp } from '../utils/test.utils'; -import { ProjectService } from './project.service'; const option: DbOption = { host: process.env.DB_HOST ?? '127.0.0.1', diff --git a/packages/node/src/indexer/unfinalizedBlocks.service.ts b/packages/node/src/indexer/unfinalizedBlocks.service.ts deleted file mode 100644 index 67c7455058..0000000000 --- a/packages/node/src/indexer/unfinalizedBlocks.service.ts +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { Inject, Injectable } from '@nestjs/common'; -import { - BaseUnfinalizedBlocksService, - Header, - IStoreModelProvider, - mainThreadOnly, - NodeConfig, -} from '@subql/node-core'; -import { - getBlockByHeight, - substrateBlockToHeader, - substrateHeaderToHeader, -} from '../utils/substrate'; -import { ApiService } from './api.service'; -import { BlockContent, LightBlockContent } from './types'; - -@Injectable() -export class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService< - BlockContent | LightBlockContent -> { - constructor( - private readonly apiService: ApiService, - nodeConfig: NodeConfig, - @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider, - ) { - super(nodeConfig, storeModelProvider); - } - - @mainThreadOnly() - protected async getFinalizedHead(): Promise
{ - return substrateHeaderToHeader( - await this.apiService.api.rpc.chain.getHeader( - await this.apiService.api.rpc.chain.getFinalizedHead(), - ), - ); - } - - // TODO: add cache here - @mainThreadOnly() - protected async getHeaderForHash(hash: string): Promise
{ - return substrateBlockToHeader( - await this.apiService.api.rpc.chain.getBlock(hash), - ); - } - - @mainThreadOnly() - async getHeaderForHeight(height: number): Promise
{ - return substrateBlockToHeader( - await getBlockByHeight(this.apiService.api, height), - ); - } -} diff --git a/packages/node/src/indexer/worker/worker-fetch.module.ts b/packages/node/src/indexer/worker/worker-fetch.module.ts index 8455afae1f..5fadd6ab0d 100644 --- a/packages/node/src/indexer/worker/worker-fetch.module.ts +++ b/packages/node/src/indexer/worker/worker-fetch.module.ts @@ -5,18 +5,15 @@ import { Module } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { ConnectionPoolService, - WorkerDynamicDsService, NodeConfig, - WorkerUnfinalizedBlocksService, WorkerCoreModule, + ProjectService, + DsProcessorService, } from '@subql/node-core'; +import { BlockchainService } from '../../blockchain.service'; import { ApiService } from '../api.service'; -import { DsProcessorService } from '../ds-processor.service'; -import { DynamicDsService } from '../dynamic-ds.service'; import { IndexerManager } from '../indexer.manager'; -import { ProjectService } from '../project.service'; import { WorkerRuntimeService } from '../runtime/workerRuntimeService'; -import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service'; import { WorkerService } from './worker.service'; /** @@ -26,9 +23,10 @@ import { WorkerService } from './worker.service'; @Module({ imports: [WorkerCoreModule], providers: [ + DsProcessorService, IndexerManager, { - provide: ApiService, + provide: 'APIService', useFactory: ApiService.init, inject: [ 'ISubqueryProject', @@ -37,22 +35,16 @@ import { WorkerService } from './worker.service'; NodeConfig, ], }, - DsProcessorService, - { - provide: DynamicDsService, - useFactory: () => new WorkerDynamicDsService((global as any).host), - }, { provide: 'IProjectService', useClass: ProjectService, }, + WorkerRuntimeService, { - provide: UnfinalizedBlocksService, - useFactory: () => - new WorkerUnfinalizedBlocksService((global as any).host), + provide: 'IBlockchainService', + useClass: BlockchainService, }, WorkerService, - WorkerRuntimeService, ], exports: [], }) diff --git a/packages/node/src/indexer/worker/worker.service.ts b/packages/node/src/indexer/worker/worker.service.ts index 6ea497f36e..d607b97b87 100644 --- a/packages/node/src/indexer/worker/worker.service.ts +++ b/packages/node/src/indexer/worker/worker.service.ts @@ -20,7 +20,6 @@ import { WorkerRuntimeService } from '../runtime/workerRuntimeService'; import { BlockContent, getBlockSize, - isFullBlock, LightBlockContent, } from '../types'; @@ -34,7 +33,7 @@ export class WorkerService extends BaseWorkerService< { specVersion: number } > { constructor( - private apiService: ApiService, + @Inject('APIService') private apiService: ApiService, private indexerManager: IndexerManager, private workerRuntimeService: WorkerRuntimeService, @Inject('IProjectService') @@ -83,11 +82,7 @@ export class WorkerService extends BaseWorkerService< block: IBlock, dataSources: SubstrateDatasource[], ): Promise { - const runtimeVersion = !isFullBlock(block.block) - ? undefined - : await this.workerRuntimeService.getRuntimeVersion(block.block.block); - - return this.indexerManager.indexBlock(block, dataSources, runtimeVersion); + return this.indexerManager.indexBlock(block, dataSources); } getSpecFromMap(height: number): number | undefined { diff --git a/packages/node/src/indexer/worker/worker.ts b/packages/node/src/indexer/worker/worker.ts index 07cb8534e2..3627fdb956 100644 --- a/packages/node/src/indexer/worker/worker.ts +++ b/packages/node/src/indexer/worker/worker.ts @@ -25,9 +25,9 @@ import { initWorkerServices, getWorkerService, IBaseIndexerWorker, + ProjectService, } from '@subql/node-core'; import { SpecVersion } from '../dictionary'; -import { ProjectService } from '../project.service'; import { WorkerModule } from './worker.module'; import { WorkerService } from './worker.service'; diff --git a/packages/node/src/init.ts b/packages/node/src/init.ts index 469f3056a0..a9289cc9ea 100644 --- a/packages/node/src/init.ts +++ b/packages/node/src/init.ts @@ -8,10 +8,10 @@ import { getLogger, getValidPort, NestLogger, + ProjectService, } from '@subql/node-core'; import { AppModule } from './app.module'; import { FetchService } from './indexer/fetch.service'; -import { ProjectService } from './indexer/project.service'; import { yargsOptions } from './yargs'; const pjson = require('../package.json'); diff --git a/packages/node/src/subcommands/reindex.module.ts b/packages/node/src/subcommands/reindex.module.ts index 6f45c5ad19..a41f02d342 100644 --- a/packages/node/src/subcommands/reindex.module.ts +++ b/packages/node/src/subcommands/reindex.module.ts @@ -14,13 +14,13 @@ import { NodeConfig, ConnectionPoolStateManager, storeModelFactory, + DsProcessorService, + UnfinalizedBlocksService, + DynamicDsService, } from '@subql/node-core'; import { Sequelize } from '@subql/x-sequelize'; import { ConfigureModule } from '../configure/configure.module'; import { ApiService } from '../indexer/api.service'; -import { DsProcessorService } from '../indexer/ds-processor.service'; -import { DynamicDsService } from '../indexer/dynamic-ds.service'; -import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service'; @Module({ providers: [ @@ -46,7 +46,7 @@ import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service'; ConnectionPoolService, { // Used to work with DI for unfinalizedBlocksService but not used with reindex - provide: ApiService, + provide: 'APIService', useFactory: ApiService.init, inject: [ 'ISubqueryProject', diff --git a/packages/node/src/subcommands/testing.module.ts b/packages/node/src/subcommands/testing.module.ts index bbaa0c079f..28fe51ee57 100644 --- a/packages/node/src/subcommands/testing.module.ts +++ b/packages/node/src/subcommands/testing.module.ts @@ -3,53 +3,28 @@ import { Module } from '@nestjs/common'; import { EventEmitter2, EventEmitterModule } from '@nestjs/event-emitter'; -import { ScheduleModule, SchedulerRegistry } from '@nestjs/schedule'; +import { ScheduleModule } from '@nestjs/schedule'; import { ConnectionPoolService, - ConnectionPoolStateManager, DbModule, - InMemoryCacheService, - PoiService, - PoiSyncService, - StoreService, TestRunner, - SandboxService, NodeConfig, - storeModelFactory, + ProjectService, + TestingCoreModule, } from '@subql/node-core'; -import { Sequelize } from '@subql/x-sequelize'; +import { BlockchainService } from '../blockchain.service'; import { ConfigureModule } from '../configure/configure.module'; import { ApiService } from '../indexer/api.service'; -import { DsProcessorService } from '../indexer/ds-processor.service'; -import { DynamicDsService } from '../indexer/dynamic-ds.service'; import { IndexerManager } from '../indexer/indexer.manager'; -import { ProjectService } from '../indexer/project.service'; -import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service'; @Module({ providers: [ - InMemoryCacheService, - StoreService, - { - provide: 'IStoreModelProvider', - useFactory: storeModelFactory, - inject: [NodeConfig, EventEmitter2, SchedulerRegistry, Sequelize], - }, - EventEmitter2, - PoiService, - PoiSyncService, - SandboxService, - DsProcessorService, - DynamicDsService, - UnfinalizedBlocksService, - ConnectionPoolStateManager, - ConnectionPoolService, { provide: 'IProjectService', useClass: ProjectService, }, { - provide: ApiService, + provide: 'APIService', useFactory: ApiService.init, inject: [ 'ISubqueryProject', @@ -58,12 +33,11 @@ import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service'; NodeConfig, ], }, - SchedulerRegistry, - TestRunner, { - provide: 'IApi', - useExisting: ApiService, + provide: 'IBlockchainService', + useClass: BlockchainService, }, + TestRunner, { provide: 'IIndexerManager', useClass: IndexerManager, @@ -80,6 +54,7 @@ export class TestingFeatureModule {} ConfigureModule.register(), EventEmitterModule.forRoot(), ScheduleModule.forRoot(), + TestingCoreModule, TestingFeatureModule, ], controllers: [], diff --git a/packages/node/src/subcommands/testing.service.ts b/packages/node/src/subcommands/testing.service.ts index 1ae9933a48..f112d2f5f6 100644 --- a/packages/node/src/subcommands/testing.service.ts +++ b/packages/node/src/subcommands/testing.service.ts @@ -9,13 +9,10 @@ import { TestingService as BaseTestingService, NestLogger, TestRunner, - IBlock, + ProjectService, } from '@subql/node-core'; import { SubstrateDatasource } from '@subql/types'; import { SubqueryProject } from '../configure/SubqueryProject'; -import { ApiService } from '../indexer/api.service'; -import { IndexerManager } from '../indexer/indexer.manager'; -import { ProjectService } from '../indexer/project.service'; import { ApiAt, BlockContent, LightBlockContent } from '../indexer/types'; import { TestingModule } from './testing.module'; @@ -50,27 +47,8 @@ export class TestingService extends BaseTestingService< const projectService: ProjectService = testContext.get('IProjectService'); - // Initialise async services, we do this here rather than in factories, so we can capture one off events await projectService.init(); return [testContext.close.bind(testContext), testContext.get(TestRunner)]; } - - async indexBlock( - block: IBlock, - handler: string, - indexerManager: IndexerManager, - apiService: ApiService, - ): Promise { - const runtimeVersion = - await apiService.unsafeApi.rpc.state.getRuntimeVersion( - block.getHeader().blockHash, - ); - - await indexerManager.indexBlock( - block, - this.getDsWithHandler(handler), - runtimeVersion, - ); - } } From 17dbc294e01b8603776ad96f2cb0fb5454b2d179 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Mon, 5 Aug 2024 16:46:39 +1200 Subject: [PATCH 2/8] Simplify worker block dispatcher --- packages/node-core/src/blockchain.service.ts | 29 +++- .../blockDispatcher/base-block-dispatcher.ts | 1 + .../worker-block-dispatcher.ts | 109 ++++++++----- .../src/indexer/project.service.spec.ts | 6 + .../node-core/src/indexer/project.service.ts | 6 +- .../node-core/src/indexer/worker/worker.ts | 4 +- packages/node/src/blockchain.service.ts | 37 ++++- .../node/src/indexer/blockDispatcher/index.ts | 7 - .../substrate-block-dispatcher.ts | 14 -- .../worker-block-dispatcher.service.ts | 146 ------------------ packages/node/src/indexer/fetch.module.ts | 21 ++- packages/node/src/indexer/fetch.service.ts | 13 +- .../src/indexer/worker/worker-fetch.module.ts | 7 +- .../node/src/indexer/worker/worker.service.ts | 1 + 14 files changed, 165 insertions(+), 236 deletions(-) delete mode 100644 packages/node/src/indexer/blockDispatcher/index.ts delete mode 100644 packages/node/src/indexer/blockDispatcher/substrate-block-dispatcher.ts delete mode 100644 packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts diff --git a/packages/node-core/src/blockchain.service.ts b/packages/node-core/src/blockchain.service.ts index 0a2f4e9f48..ef12444183 100644 --- a/packages/node-core/src/blockchain.service.ts +++ b/packages/node-core/src/blockchain.service.ts @@ -2,10 +2,23 @@ // SPDX-License-Identifier: GPL-3.0 import { BaseCustomDataSource, BaseDataSource, IProjectNetworkConfig } from '@subql/types-core'; -import { DatasourceParams, Header, IBlock, ISubqueryProject } from './indexer'; +import { DatasourceParams, Header, IBaseIndexerWorker, IBlock, ISubqueryProject } from './indexer'; // TODO probably need to split this in 2 to have a worker specific subset +export interface ICoreBlockchainService< + DS extends BaseDataSource = BaseDataSource, + SubQueryProject extends ISubqueryProject = ISubqueryProject +> { + /* The semver of the node */ + packageVersion: string; + + // Project service + onProjectChange(project: SubQueryProject): Promise | void; + /* Not all networks have a block timestamp, e.g. Shiden */ + getBlockTimestamp(height: number): Promise; +} + export interface IBlockchainService< DS extends BaseDataSource = BaseDataSource, CDS extends DS & BaseCustomDataSource = BaseCustomDataSource & DS, @@ -13,20 +26,20 @@ export interface IBlockchainService< SafeAPI = any, LightBlock = any, FullBlock = any, -> { - /* The semver of the node */ - packageVersion: string; - + Worker extends IBaseIndexerWorker = IBaseIndexerWorker +> extends ICoreBlockchainService { blockHandlerKind: string; // TODO SubqueryProject methods // Block dispatcher service fetchBlocks(blockNums: number[]): Promise[] | IBlock[]>; // TODO this probably needs to change to get light block type correct + /* This is the worker equivalent of fetchBlocks, it provides a context to allow syncing anything between workers */ + fetchBlockWorker(worker: Worker, blockNum: number, context: { workers: Worker[] }): Promise
; // Project service - onProjectChange(project: SubQueryProject): Promise | void; - /* Not all networks have a block timestamp, e.g. Shiden */ - getBlockTimestamp(height: number): Promise; + // onProjectChange(project: SubQueryProject): Promise | void; + // /* Not all networks have a block timestamp, e.g. Shiden */ + // getBlockTimestamp(height: number): Promise; // Block dispatcher /* Gets the size of the block, used to calculate a median */ diff --git a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts index f2b4eb862d..7425cd9610 100644 --- a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts @@ -26,6 +26,7 @@ export type ProcessBlockResponse = { }; export interface IBlockDispatcher { + init(onDynamicDsCreated: (height: number) => void): Promise; // now within enqueueBlock should handle getLatestBufferHeight enqueueBlocks(heights: (IBlock | number)[], latestBufferHeight: number): void | Promise; queueSize: number; diff --git a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts index 9e90e47ba8..872efabfb4 100644 --- a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts @@ -2,33 +2,39 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import {OnApplicationShutdown} from '@nestjs/common'; -import {EventEmitter2} from '@nestjs/event-emitter'; -import {Interval} from '@nestjs/schedule'; -import {last} from 'lodash'; -import {NodeConfig} from '../../configure'; -import {IProjectUpgradeService} from '../../configure/ProjectUpgrade.service'; -import {IndexerEvent} from '../../events'; -import {IBlock, PoiSyncService, WorkerStatusResponse} from '../../indexer'; -import {getLogger} from '../../logger'; -import {monitorWrite} from '../../process'; -import {AutoQueue, isTaskFlushedError} from '../../utils'; -import {MonitorServiceInterface} from '../monitor.service'; -import {StoreService} from '../store.service'; -import {IStoreModelProvider} from '../storeModelProvider'; -import {ISubqueryProject, IProjectService, Header} from '../types'; -import {isBlockUnavailableError} from '../worker/utils'; -import {BaseBlockDispatcher, ProcessBlockResponse} from './base-block-dispatcher'; +import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { Interval } from '@nestjs/schedule'; +import { BaseDataSource } from '@subql/types-core'; +import { last } from 'lodash'; +import { IApiConnectionSpecific } from '../../api.service'; +import { IBlockchainService } from '../../blockchain.service'; +import { NodeConfig } from '../../configure'; +import { IProjectUpgradeService } from '../../configure/ProjectUpgrade.service'; +import { IndexerEvent } from '../../events'; +import { + ConnectionPoolStateManager, + createIndexerWorker, + DynamicDsService, + IBaseIndexerWorker, + IBlock, + InMemoryCacheService, + PoiSyncService, + TerminateableWorker, + UnfinalizedBlocksService, +} from '../../indexer'; +import { getLogger } from '../../logger'; +import { monitorWrite } from '../../process'; +import { AutoQueue, isTaskFlushedError } from '../../utils'; +import { MonitorServiceInterface } from '../monitor.service'; +import { StoreService } from '../store.service'; +import { IStoreModelProvider } from '../storeModelProvider'; +import { ISubqueryProject, IProjectService } from '../types'; +import { isBlockUnavailableError } from '../worker/utils'; +import { BaseBlockDispatcher } from './base-block-dispatcher'; const logger = getLogger('WorkerBlockDispatcherService'); -type Worker = { - processBlock: (height: number) => Promise; - getStatus: () => Promise; - getMemoryLeft: () => Promise; - terminate: () => Promise; -}; - function initAutoQueue( workers: number | undefined, batchSize: number, @@ -39,26 +45,37 @@ function initAutoQueue( return new AutoQueue(workers * batchSize * 2, 1, timeout, name); } -export abstract class WorkerBlockDispatcher - extends BaseBlockDispatcher, DS, B> - implements OnApplicationShutdown -{ - protected workers: W[] = []; +@Injectable() +export class WorkerBlockDispatcher< + DS extends BaseDataSource = BaseDataSource, + Worker extends IBaseIndexerWorker = IBaseIndexerWorker, + Block = any, + ApiConn extends IApiConnectionSpecific = IApiConnectionSpecific +> + extends BaseBlockDispatcher, DS, Block> + implements OnApplicationShutdown { + protected workers: TerminateableWorker[] = []; private numWorkers: number; private isShutdown = false; - protected abstract fetchBlock(worker: W, height: number): Promise
; + private createWorker: () => Promise>; constructor( nodeConfig: NodeConfig, eventEmitter: EventEmitter2, - projectService: IProjectService, - projectUpgradeService: IProjectUpgradeService, + @Inject('IProjectService') projectService: IProjectService, + @Inject('IProjectUpgradeService') projectUpgradeService: IProjectUpgradeService, storeService: StoreService, storeModelProvider: IStoreModelProvider, + cacheService: InMemoryCacheService, poiSyncService: PoiSyncService, - project: ISubqueryProject, - private createIndexerWorker: () => Promise, + dynamicDsService: DynamicDsService, + unfinalizedBlocksService: UnfinalizedBlocksService, + connectionPoolState: ConnectionPoolStateManager, + @Inject('ISubqueryProject') project: ISubqueryProject, + @Inject('IBlockchainService') private blockchainService: IBlockchainService, + workerPath: string, + workerFns: Parameters>[1], monitorService?: MonitorServiceInterface ) { super( @@ -73,13 +90,27 @@ export abstract class WorkerBlockDispatcher poiSyncService, monitorService ); + + this.createWorker = () => + createIndexerWorker( + workerPath, + workerFns, + storeService.getStore(), + cacheService.getCache(), + dynamicDsService, + unfinalizedBlocksService, + connectionPoolState, + project.root, + projectService.startHeight, + monitorService + ); // initAutoQueue will assert that workers is set. unfortunately we cant do anything before the super call // eslint-disable-next-line @typescript-eslint/no-non-null-assertion this.numWorkers = nodeConfig.workers!; } async init(onDynamicDsCreated: (height: number) => void): Promise { - this.workers = await Promise.all(new Array(this.numWorkers).fill(0).map(() => this.createIndexerWorker())); + this.workers = await Promise.all(new Array(this.numWorkers).fill(0).map(() => this.createWorker())); return super.init(onDynamicDsCreated); } @@ -93,7 +124,8 @@ export abstract class WorkerBlockDispatcher await Promise.all(this.workers.map((w) => w.terminate())); } } - async enqueueBlocks(heights: (IBlock | number)[], latestBufferHeight?: number): Promise { + + async enqueueBlocks(heights: (IBlock | number)[], latestBufferHeight?: number): Promise { assert( heights.every((h) => typeof h === 'number'), 'Worker block dispatcher only supports enqueuing numbers, not blocks.' @@ -137,7 +169,8 @@ export abstract class WorkerBlockDispatcher // Used to compare before and after as a way to check if queue was flushed const bufferedHeight = this.latestBufferedHeight; - const pendingBlock = this.fetchBlock(worker, height); + + const pendingBlock = this.blockchainService.fetchBlockWorker(worker, height, { workers: this.workers }); const processBlock = async () => { try { @@ -150,7 +183,7 @@ export abstract class WorkerBlockDispatcher await this.preProcessBlock(header); monitorWrite(`Processing from worker #${workerIdx}`); - const {dynamicDsCreated, reindexBlockHeader} = await worker.processBlock(height); + const { dynamicDsCreated, reindexBlockHeader } = await worker.processBlock(height); await this.postProcessBlock(header, { dynamicDsCreated, diff --git a/packages/node-core/src/indexer/project.service.spec.ts b/packages/node-core/src/indexer/project.service.spec.ts index 9890542044..c5f8ffca0b 100644 --- a/packages/node-core/src/indexer/project.service.spec.ts +++ b/packages/node-core/src/indexer/project.service.spec.ts @@ -10,6 +10,7 @@ import { DatasourceParams, DynamicDsService } from './dynamic-ds.service'; import { ProjectService } from './project.service'; import { Header, IBlock, ISubqueryProject } from './types'; import { METADATA_LAST_FINALIZED_PROCESSED_KEY, METADATA_UNFINALIZED_BLOCKS_KEY, UnfinalizedBlocksService } from './unfinalizedBlocks.service'; +import { IBaseIndexerWorker } from './worker'; class TestProjectService extends ProjectService { packageVersion = '1.0.0'; @@ -36,6 +37,11 @@ class TestBlockchainService implements IBlockchainService { throw new Error('Method not implemented.'); } + // eslint-disable-next-line @typescript-eslint/promise-function-async + fetchBlockWorker(worker: IBaseIndexerWorker, blockNum: number, context: { workers: IBaseIndexerWorker[]; }): Promise
{ + throw new Error('Method not implemented.'); + } + onProjectChange(project: ISubqueryProject): Promise | void { throw new Error('Method not implemented.'); } diff --git a/packages/node-core/src/indexer/project.service.ts b/packages/node-core/src/indexer/project.service.ts index fe2c4ac752..a3cb665c8f 100644 --- a/packages/node-core/src/indexer/project.service.ts +++ b/packages/node-core/src/indexer/project.service.ts @@ -8,7 +8,7 @@ import { EventEmitter2 } from '@nestjs/event-emitter'; import { BaseDataSource, IProjectNetworkConfig } from '@subql/types-core'; import { Sequelize } from '@subql/x-sequelize'; import { IApi } from '../api.service'; -import { IBlockchainService } from '../blockchain.service'; +import { ICoreBlockchainService } from '../blockchain.service'; import { IProjectUpgradeService, NodeConfig } from '../configure'; import { IndexerEvent } from '../events'; import { getLogger } from '../logger'; @@ -36,7 +36,7 @@ class NotInitError extends Error { export class ProjectService< DS extends BaseDataSource = BaseDataSource, API extends IApi = IApi, - UnfinalizedBlocksService extends IUnfinalizedBlocksService = IUnfinalizedBlocksService, + UnfinalizedBlocksService extends IUnfinalizedBlocksService = IUnfinalizedBlocksService > implements IProjectService { private _schema?: string; private _startHeight?: number; @@ -55,7 +55,7 @@ export class ProjectService< private readonly dynamicDsService: DynamicDsService, private eventEmitter: EventEmitter2, @Inject('IUnfinalizedBlocksService') private readonly unfinalizedBlockService: UnfinalizedBlocksService, - @Inject('IBlockchainService') private blockchainService: IBlockchainService + @Inject('IBlockchainService') private blockchainService: ICoreBlockchainService ) { if (this.nodeConfig.unfinalizedBlocks && this.nodeConfig.allowSchemaMigration) { throw new Error('Unfinalized Blocks and Schema Migration cannot be enabled at the same time'); diff --git a/packages/node-core/src/indexer/worker/worker.ts b/packages/node-core/src/indexer/worker/worker.ts index d68036b210..66306b5623 100644 --- a/packages/node-core/src/indexer/worker/worker.ts +++ b/packages/node-core/src/indexer/worker/worker.ts @@ -156,6 +156,8 @@ export function createWorkerHost< ); } +export type TerminateableWorker = T & {terminate: () => Promise}; + export async function createIndexerWorker< T extends IBaseIndexerWorker, ApiConnection extends IApiConnectionSpecific /*ApiPromiseConnection*/ /*ApiPromiseConnection*/, @@ -173,7 +175,7 @@ export async function createIndexerWorker< startHeight: number, monitorService?: MonitorServiceInterface, workerData?: any -): Promise Promise}> { +): Promise> { const indexerWorker = Worker.create< T & {initWorker: (startHeight: number) => Promise}, DefaultWorkerFunctions diff --git a/packages/node/src/blockchain.service.ts b/packages/node/src/blockchain.service.ts index 9e22e17e64..aaa76d853f 100644 --- a/packages/node/src/blockchain.service.ts +++ b/packages/node/src/blockchain.service.ts @@ -7,6 +7,7 @@ import { isCustomDs, isRuntimeDs } from '@subql/common-substrate'; import { DatasourceParams, Header, + IBaseIndexerWorker, IBlock, IBlockchainService, mainThreadOnly, @@ -31,6 +32,7 @@ import { isFullBlock, LightBlockContent, } from './indexer/types'; +import { IIndexerWorker } from './indexer/worker/worker'; import { calcInterval, getBlockByHeight, @@ -56,12 +58,12 @@ export class BlockchainService SubqueryProject, ApiAt, LightBlockContent, - BlockContent + BlockContent, + IIndexerWorker > { constructor( @Inject('APIService') private apiService: ApiService, - @Inject(isMainThread ? RuntimeService : 'Null') - private runtimeService: RuntimeService, + @Inject('RuntimeService') private runtimeService: RuntimeService, ) {} isCustomDs = isCustomDs; @@ -69,6 +71,7 @@ export class BlockchainService blockHandlerKind = SubstrateHandlerKind.Block; packageVersion = packageVersion; + @mainThreadOnly() async fetchBlocks( blockNums: number[], ): Promise[] | IBlock[]> { @@ -84,6 +87,29 @@ export class BlockchainService ); } + async fetchBlockWorker( + worker: IIndexerWorker, + height: number, + context: { workers: IIndexerWorker[] }, + ): Promise
{ + // get SpecVersion from main runtime service + const { blockSpecVersion, syncedDictionary } = + await this.runtimeService.getSpecVersion(height); + + // if main runtime specVersion has been updated, then sync with all workers specVersion map, and lastFinalizedBlock + if (syncedDictionary) { + context.workers.map((w) => + w.syncRuntimeService( + this.runtimeService.specVersionMap, + this.runtimeService.latestFinalizedHeight, + ), + ); + } + + // const start = new Date(); + return worker.fetchBlock(height, blockSpecVersion); + } + async onProjectChange(project: SubqueryProject): Promise { // Only network with chainTypes require to reload await this.apiService.updateChainTypes(); @@ -102,8 +128,9 @@ export class BlockchainService async getFinalizedHeader(): Promise
{ const finalizedHash = await this.apiService.unsafeApi.rpc.chain.getFinalizedHead(); - const finalizedHeader = - await this.apiService.unsafeApi.rpc.chain.getHeader(finalizedHash); + const finalizedHeader = await this.apiService.unsafeApi.rpc.chain.getHeader( + finalizedHash, + ); return substrateHeaderToHeader(finalizedHeader); } diff --git a/packages/node/src/indexer/blockDispatcher/index.ts b/packages/node/src/indexer/blockDispatcher/index.ts deleted file mode 100644 index 0081a692eb..0000000000 --- a/packages/node/src/indexer/blockDispatcher/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -// import { BlockDispatcherService } from './block-dispatcher.service'; -import { WorkerBlockDispatcherService } from './worker-block-dispatcher.service'; - -export { WorkerBlockDispatcherService }; diff --git a/packages/node/src/indexer/blockDispatcher/substrate-block-dispatcher.ts b/packages/node/src/indexer/blockDispatcher/substrate-block-dispatcher.ts deleted file mode 100644 index a81b4e6b03..0000000000 --- a/packages/node/src/indexer/blockDispatcher/substrate-block-dispatcher.ts +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { IBlockDispatcher } from '@subql/node-core'; -import { SubstrateBlock } from '@subql/types'; -import { RuntimeService } from '../runtime/runtimeService'; - -export interface ISubstrateBlockDispatcher - extends IBlockDispatcher { - init( - onDynamicDsCreated: (height: number) => void, - runtimeService?: RuntimeService, - ): Promise; -} diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts deleted file mode 100644 index 5ac0d589cf..0000000000 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import assert from 'assert'; -import path from 'path'; -import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; -import { EventEmitter2 } from '@nestjs/event-emitter'; -import { - NodeConfig, - StoreService, - IProjectService, - WorkerBlockDispatcher, - ConnectionPoolStateManager, - IProjectUpgradeService, - PoiSyncService, - InMemoryCacheService, - createIndexerWorker as createIndexerWorkerCore, - MonitorServiceInterface, - IStoreModelProvider, - UnfinalizedBlocksService, - DynamicDsService, -} from '@subql/node-core'; -import { SubstrateDatasource } from '@subql/types'; -import { SubqueryProject } from '../../configure/SubqueryProject'; -import { ApiPromiseConnection } from '../apiPromise.connection'; -import { RuntimeService } from '../runtime/runtimeService'; -import { BlockContent, LightBlockContent } from '../types'; -import { IIndexerWorker } from '../worker/worker'; -import { FetchBlockResponse } from '../worker/worker.service'; - -type IndexerWorker = IIndexerWorker & { - terminate: () => Promise; -}; - -@Injectable() -export class WorkerBlockDispatcherService - extends WorkerBlockDispatcher< - SubstrateDatasource, - IndexerWorker, - BlockContent | LightBlockContent - > - implements OnApplicationShutdown { - private _runtimeService?: RuntimeService; - - constructor( - nodeConfig: NodeConfig, - eventEmitter: EventEmitter2, - @Inject('IProjectService') - projectService: IProjectService, - @Inject('IProjectUpgradeService') - projectUpgadeService: IProjectUpgradeService, - cacheService: InMemoryCacheService, - storeService: StoreService, - @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider, - poiSyncService: PoiSyncService, - @Inject('ISubqueryProject') project: SubqueryProject, - dynamicDsService: DynamicDsService, - unfinalizedBlocksService: UnfinalizedBlocksService< - BlockContent | LightBlockContent - >, - connectionPoolState: ConnectionPoolStateManager, - monitorService?: MonitorServiceInterface, - ) { - super( - nodeConfig, - eventEmitter, - projectService, - projectUpgadeService, - storeService, - storeModelProvider, - poiSyncService, - project, - () => - createIndexerWorkerCore< - IIndexerWorker, - ApiPromiseConnection, - BlockContent, - SubstrateDatasource - >( - path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'), - ['syncRuntimeService', 'getSpecFromMap'], - storeService.getStore(), - cacheService.getCache(), - dynamicDsService, - unfinalizedBlocksService, - connectionPoolState, - project.root, - projectService.startHeight, - monitorService, - ), - monitorService, - ); - } - - private get runtimeService(): RuntimeService { - assert(this._runtimeService, 'RuntimeService not initialized'); - return this._runtimeService; - } - - async init( - onDynamicDsCreated: (height: number) => void, - runtimeService?: RuntimeService, - ): Promise { - await super.init(onDynamicDsCreated); - // Sync workers runtime from main - if (runtimeService) this._runtimeService = runtimeService; - this.syncWorkerRuntimes(); - } - - syncWorkerRuntimes(): void { - this.workers.map((w) => - w.syncRuntimeService( - this.runtimeService.specVersionMap, - this.runtimeService.latestFinalizedHeight, - ), - ); - } - - protected async fetchBlock( - worker: IndexerWorker, - height: number, - ): Promise { - // get SpecVersion from main runtime service - const { blockSpecVersion, syncedDictionary } = - await this.runtimeService.getSpecVersion(height); - // if main runtime specVersion has been updated, then sync with all workers specVersion map, and lastFinalizedBlock - if (syncedDictionary) { - this.syncWorkerRuntimes(); - } - - // const start = new Date(); - return worker.fetchBlock(height, blockSpecVersion); - // const end = new Date(); - - // const waitTime = end.getTime() - start.getTime(); - // if (waitTime > 1000) { - // logger.info( - // `Waiting to fetch block ${height}: ${chalk.red(`${waitTime}ms`)}`, - // ); - // } else if (waitTime > 200) { - // logger.info( - // `Waiting to fetch block ${height}: ${chalk.yellow(`${waitTime}ms`)}`, - // ); - // } - } -} diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts index 1232c4e978..a7504b8dec 100644 --- a/packages/node/src/indexer/fetch.module.ts +++ b/packages/node/src/indexer/fetch.module.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import path from 'path'; import { Module } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { @@ -19,17 +20,19 @@ import { DsProcessorService, ProjectService, DynamicDsService, + WorkerBlockDispatcher, } from '@subql/node-core'; import { SubstrateDatasource } from '@subql/types'; import { BlockchainService } from '../blockchain.service'; import { SubqueryProject } from '../configure/SubqueryProject'; import { ApiService } from './api.service'; import { ApiPromiseConnection } from './apiPromise.connection'; -import { WorkerBlockDispatcherService } from './blockDispatcher'; import { SubstrateDictionaryService } from './dictionary/substrateDictionary.service'; import { FetchService } from './fetch.service'; import { IndexerManager } from './indexer.manager'; import { RuntimeService } from './runtime/runtimeService'; +import { BlockContent, LightBlockContent } from './types'; +import { IIndexerWorker } from './worker/worker'; @Module({ imports: [CoreModule], @@ -45,7 +48,7 @@ import { RuntimeService } from './runtime/runtimeService'; ], }, { - provide: RuntimeService, // TODO DOING this because of circular reference with dictionary service + provide: 'RuntimeService', // TODO DOING this because of circular reference with dictionary service useFactory: (apiService: ApiService) => new RuntimeService(apiService), inject: ['APIService'], }, @@ -86,19 +89,27 @@ import { RuntimeService } from './runtime/runtimeService'; monitorService?: MonitorService, ) => { return nodeConfig.workers - ? new WorkerBlockDispatcherService( + ? new WorkerBlockDispatcher< + SubstrateDatasource, + IIndexerWorker, + BlockContent | LightBlockContent, + ApiPromiseConnection + >( nodeConfig, eventEmitter, projectService, projectUpgradeService, - cacheService, storeService, storeModelProvider, + cacheService, poiSyncService, - project, dynamicDsService, unfinalizedBlocks, connectionPoolState, + project, + blockchainService, + path.resolve(__dirname, '../../dist/indexer/worker/worker.js'), + ['syncRuntimeService', 'getSpecFromMap'], monitorService, ) : new BlockDispatcher( diff --git a/packages/node/src/indexer/fetch.service.ts b/packages/node/src/indexer/fetch.service.ts index 784d6a07af..865bca4d34 100644 --- a/packages/node/src/indexer/fetch.service.ts +++ b/packages/node/src/indexer/fetch.service.ts @@ -10,17 +10,17 @@ import { IStoreModelProvider, UnfinalizedBlocksService, ProjectService, + IBlockDispatcher, } from '@subql/node-core'; import { SubstrateDatasource, SubstrateBlock } from '@subql/types'; import { BlockchainService } from '../blockchain.service'; -import { ISubstrateBlockDispatcher } from './blockDispatcher/substrate-block-dispatcher'; import { SubstrateDictionaryService } from './dictionary/substrateDictionary.service'; import { RuntimeService } from './runtime/runtimeService'; @Injectable() export class FetchService extends BaseFetchService< SubstrateDatasource, - ISubstrateBlockDispatcher, + IBlockDispatcher, SubstrateBlock > { constructor( @@ -28,13 +28,13 @@ export class FetchService extends BaseFetchService< @Inject('IProjectService') projectService: ProjectService, @Inject('IBlockDispatcher') - blockDispatcher: ISubstrateBlockDispatcher, + blockDispatcher: IBlockDispatcher, dictionaryService: SubstrateDictionaryService, @Inject('IUnfinalizedBlocksService') unfinalizedBlocksService: UnfinalizedBlocksService, eventEmitter: EventEmitter2, schedulerRegistry: SchedulerRegistry, - private runtimeService: RuntimeService, + @Inject('RuntimeService') private runtimeService: RuntimeService, @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider, @Inject('IBlockchainService') blockchainService: BlockchainService, ) { @@ -52,10 +52,7 @@ export class FetchService extends BaseFetchService< } protected async initBlockDispatcher(): Promise { - await this.blockDispatcher.init( - this.resetForNewDs.bind(this), - this.runtimeService, - ); + await this.blockDispatcher.init(this.resetForNewDs.bind(this)); } protected async preLoopHook({ diff --git a/packages/node/src/indexer/worker/worker-fetch.module.ts b/packages/node/src/indexer/worker/worker-fetch.module.ts index 5fadd6ab0d..64a3ed0b23 100644 --- a/packages/node/src/indexer/worker/worker-fetch.module.ts +++ b/packages/node/src/indexer/worker/worker-fetch.module.ts @@ -39,7 +39,12 @@ import { WorkerService } from './worker.service'; provide: 'IProjectService', useClass: ProjectService, }, - WorkerRuntimeService, + // This is alised so it satisfies the BlockchainService, other services are updated to reflect this + // TODO find a way to remove the alias, currently theres no common interface between worker and non-worker + { + provide: 'RuntimeService', + useClass: WorkerRuntimeService, + }, { provide: 'IBlockchainService', useClass: BlockchainService, diff --git a/packages/node/src/indexer/worker/worker.service.ts b/packages/node/src/indexer/worker/worker.service.ts index d607b97b87..aeb72de63a 100644 --- a/packages/node/src/indexer/worker/worker.service.ts +++ b/packages/node/src/indexer/worker/worker.service.ts @@ -35,6 +35,7 @@ export class WorkerService extends BaseWorkerService< constructor( @Inject('APIService') private apiService: ApiService, private indexerManager: IndexerManager, + @Inject('RuntimeService') private workerRuntimeService: WorkerRuntimeService, @Inject('IProjectService') projectService: IProjectService, From 4e5039ac51ec7edd5080e403d18b9f318c193a74 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Tue, 6 Aug 2024 10:37:03 +1200 Subject: [PATCH 3/8] Remove node fetch service --- .../src/indexer/fetch.service.spec.ts | 129 ++++++++++++------ .../node-core/src/indexer/fetch.service.ts | 53 ++----- packages/node/src/blockchain.service.ts | 2 - packages/node/src/indexer/fetch.module.ts | 8 +- .../node/src/indexer/fetch.service.spec.ts | 81 ----------- .../node/src/indexer/fetch.service.test.ts | 58 -------- packages/node/src/indexer/fetch.service.ts | 68 --------- packages/node/src/init.ts | 13 +- 8 files changed, 117 insertions(+), 295 deletions(-) delete mode 100644 packages/node/src/indexer/fetch.service.spec.ts delete mode 100644 packages/node/src/indexer/fetch.service.test.ts delete mode 100644 packages/node/src/indexer/fetch.service.ts diff --git a/packages/node-core/src/indexer/fetch.service.spec.ts b/packages/node-core/src/indexer/fetch.service.spec.ts index 12aadaf519..9de5b03bcd 100644 --- a/packages/node-core/src/indexer/fetch.service.spec.ts +++ b/packages/node-core/src/indexer/fetch.service.spec.ts @@ -3,7 +3,7 @@ import { EventEmitter2 } from '@nestjs/event-emitter'; import { SchedulerRegistry } from '@nestjs/schedule'; -import { BaseDataSource, BaseHandler, BaseMapping, DictionaryQueryEntry } from '@subql/types-core'; +import { BaseCustomDataSource, BaseDataSource, BaseHandler, BaseMapping, DictionaryQueryEntry } from '@subql/types-core'; import { UnfinalizedBlocksService, BlockDispatcher, @@ -14,39 +14,27 @@ import { IProjectService, NodeConfig, IBlockchainService, + ISubqueryProject, + DatasourceParams, + IBaseIndexerWorker, + BypassBlocks, } from '../'; import { BlockHeightMap } from '../utils/blockHeightMap'; import { DictionaryService } from './dictionary/dictionary.service'; -import { BaseFetchService } from './fetch.service'; +import { FetchService } from './fetch.service'; const CHAIN_INTERVAL = 100; // 100ms -const genesisHash = '0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3'; -class TestFetchService extends BaseFetchService, any> { - finalizedHeight = 1000; - bestHeight = 20; +class TestFetchService extends FetchService, any> { + setBypassBlocks(blocks: BypassBlocks) { + this.projectService.bypassBlocks = blocks; + } protected buildDictionaryQueryEntries( dataSources: BaseDataSource, BaseMapping>>[] ): DictionaryQueryEntry[] { return []; } - getGenesisHash(): string { - return genesisHash; - } - async getBestHeight(): Promise { - return Promise.resolve(this.bestHeight); - } - protected async getChainInterval(): Promise { - return Promise.resolve(CHAIN_INTERVAL); - } - - protected async initBlockDispatcher(): Promise { - return Promise.resolve(); - } - async preLoopHook(data: { startHeight: number }): Promise { - return Promise.resolve(); - } protected getModulos(dataSources: BaseDataSource[]): number[] { // This is mocks get modulos, checkes every handler @@ -70,15 +58,75 @@ class TestFetchService extends BaseFetchService): void { this.projectService.getDataSourcesMap = jest.fn(() => blockHeightMap); } +} +class TestBlockchainService implements IBlockchainService { + finalizedHeight = 1000; + bestHeight = 20; + blockHandlerKind = ''; + packageVersion = '1.0.0'; + // eslint-disable-next-line @typescript-eslint/promise-function-async + fetchBlocks(blockNums: number[]): Promise[]> { + throw new Error('Method not implemented.'); + } + // eslint-disable-next-line @typescript-eslint/promise-function-async + fetchBlockWorker( + worker: IBaseIndexerWorker, + blockNum: number, + context: { workers: IBaseIndexerWorker[] } + ): Promise
{ + throw new Error('Method not implemented.'); + } async getFinalizedHeader(): Promise
{ return Promise.resolve({ blockHeight: this.finalizedHeight, blockHash: '0xxx', parentHash: '0xxx', - timestamp: new Date(), + timestamp: new Date() }); } + async getBestHeight(): Promise { + return Promise.resolve(this.bestHeight); + } + async getChainInterval(): Promise { + return Promise.resolve(CHAIN_INTERVAL); + } + getBlockSize(block: IBlock): number { + throw new Error('Not implemented'); + } + // eslint-disable-next-line @typescript-eslint/promise-function-async + getHeaderForHash(hash: string): Promise
{ + throw new Error('Method not implemented.'); + } + // eslint-disable-next-line @typescript-eslint/promise-function-async + getHeaderForHeight(height: number): Promise
{ + throw new Error('Method not implemented.'); + } + // eslint-disable-next-line @typescript-eslint/promise-function-async + updateDynamicDs( + params: DatasourceParams, + template: BaseDataSource | (BaseCustomDataSource & BaseDataSource) + ): Promise { + throw new Error('Method not implemented.'); + } + isCustomDs(x: BaseDataSource | (BaseCustomDataSource & BaseDataSource)): x is BaseCustomDataSource { + throw new Error('Method not implemented.'); + } + isRuntimeDs(x: BaseDataSource | (BaseCustomDataSource & BaseDataSource)): x is BaseDataSource { + throw new Error('Method not implemented.'); + } + // eslint-disable-next-line @typescript-eslint/promise-function-async + getSafeApi(block: any): Promise { + throw new Error('Method not implemented.'); + } + // eslint-disable-next-line @typescript-eslint/promise-function-async + onProjectChange(project: ISubqueryProject): Promise | void { + throw new Error('Method not implemented.'); + } + // eslint-disable-next-line @typescript-eslint/promise-function-async + getBlockTimestamp(height: number): Promise { + throw new Error('Method not implemented.'); + } } const nodeConfig = new NodeConfig({ @@ -144,6 +192,7 @@ const getDictionaryService = () => const getBlockDispatcher = () => { const inst = { + init: (fn: any) => Promise.resolve(), latestBufferedHeight: 0, batchSize: 10, freeSize: 10, @@ -165,7 +214,7 @@ describe('Fetch Service', () => { let dictionaryService: DictionaryService; let dataSources: BaseDataSource[]; let unfinalizedBlocksService: UnfinalizedBlocksService; - let blockchainService: IBlockchainService; + let blockchainService: TestBlockchainService; let spyOnEnqueueSequential: jest.SpyInstance< void | Promise, @@ -201,6 +250,10 @@ describe('Fetch Service', () => { blockDispatcher = getBlockDispatcher(); dictionaryService = getDictionaryService(); + blockchainService = new TestBlockchainService(); + unfinalizedBlocksService = { + registerFinalizedBlock: jest.fn(), + } as unknown as UnfinalizedBlocksService; fetchService = new TestFetchService( nodeConfig, @@ -273,14 +326,6 @@ describe('Fetch Service', () => { jest.clearAllMocks(); }); - it('calls the preHookLoop when init is called', async () => { - const preHookLoopSpy = jest.spyOn(fetchService, 'preLoopHook'); - - await fetchService.init(1); - - expect(preHookLoopSpy).toHaveBeenCalled(); - }); - it('adds bypassBlocks for empty datasources', async () => { fetchService.mockDsMap( new BlockHeightMap( @@ -335,8 +380,8 @@ describe('Fetch Service', () => { }); it('checks chain heads at an interval', async () => { - const finalizedSpy = jest.spyOn(fetchService, 'getFinalizedHeader'); - const bestSpy = jest.spyOn(fetchService, 'getBestHeight'); + const finalizedSpy = jest.spyOn(blockchainService, 'getFinalizedHeader'); + const bestSpy = jest.spyOn(blockchainService, 'getBestHeight'); await fetchService.init(1); @@ -349,8 +394,8 @@ describe('Fetch Service', () => { expect(finalizedSpy).toHaveBeenCalledTimes(2); expect(bestSpy).toHaveBeenCalledTimes(2); - await expect(fetchService.getFinalizedHeader()).resolves.toMatchObject({ - blockHeight: fetchService.finalizedHeight, + await expect(blockchainService.getFinalizedHeader()).resolves.toMatchObject({ + blockHeight: blockchainService.finalizedHeight, blockHash: '0xxx', parentHash: '0xxx', }); @@ -513,7 +558,7 @@ describe('Fetch Service', () => { it('update the LatestBufferHeight when modulo blocks full synced', async () => { fetchService.mockGetModulos([20]); - fetchService.finalizedHeight = 55; + blockchainService.finalizedHeight = 55; // simulate we have synced to block 50, and modulo is 20, next block to handle suppose be 60,80,100... // we will still enqueue 55 to update LatestBufferHeight @@ -614,7 +659,7 @@ describe('Fetch Service', () => { }); it('skips bypassBlocks', async () => { - (fetchService as any).networkConfig.bypassBlocks = [3]; + fetchService.setBypassBlocks([3]); await fetchService.init(1); @@ -625,7 +670,7 @@ describe('Fetch Service', () => { it('transforms bypassBlocks', async () => { // Set a range so on init its transformed - (fetchService as any).networkConfig.bypassBlocks = ['2-5']; + fetchService.setBypassBlocks(['2-5']); await fetchService.init(1); @@ -661,7 +706,7 @@ describe('Fetch Service', () => { const FINALIZED_HEIGHT = 10; - fetchService.finalizedHeight = FINALIZED_HEIGHT; + blockchainService.finalizedHeight = FINALIZED_HEIGHT; // change query end (dictionaryService as any).getDictionary(1).getQueryEndBlock = () => 10; @@ -694,7 +739,7 @@ describe('Fetch Service', () => { (fetchService as any).dictionaryService.scopedDictionaryEntries = () => { return undefined; }; - fetchService.bestHeight = 500; + blockchainService.bestHeight = 500; const dictionarySpy = jest.spyOn((fetchService as any).dictionaryService, 'scopedDictionaryEntries'); await fetchService.init(10); expect(dictionarySpy).toHaveBeenCalledTimes(1); @@ -708,7 +753,7 @@ describe('Fetch Service', () => { (fetchService as any).dictionaryService.scopedDictionaryEntries = () => { return undefined; }; - fetchService.bestHeight = 500; + blockchainService.bestHeight = 500; const dictionarySpy = jest.spyOn((fetchService as any).dictionaryService, 'scopedDictionaryEntries'); await fetchService.init(490); expect(dictionarySpy).toHaveBeenCalledTimes(0); diff --git a/packages/node-core/src/indexer/fetch.service.ts b/packages/node-core/src/indexer/fetch.service.ts index 4aa45aa16f..2990df3b8b 100644 --- a/packages/node-core/src/indexer/fetch.service.ts +++ b/packages/node-core/src/indexer/fetch.service.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import { OnApplicationShutdown } from '@nestjs/common'; +import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { SchedulerRegistry } from '@nestjs/schedule'; import { BaseDataSource } from '@subql/types-core'; @@ -22,31 +22,23 @@ import { IUnfinalizedBlocksServiceUtil } from './unfinalizedBlocks.service'; const logger = getLogger('FetchService'); -export abstract class BaseFetchService, FB> +@Injectable() +export class FetchService, FB> implements OnApplicationShutdown { private _latestBestHeight?: number; private _latestFinalizedHeight?: number; private isShutdown = false; - #pendingFinalizedBlockHead?: Promise; - #pendingBestBlockHead?: Promise; - - protected abstract initBlockDispatcher(): Promise; - - // Gets called just before the loop is started - // Used by substrate to init runtime service and get runtime version data from the dictionary - protected abstract preLoopHook(data: { startHeight: number }): Promise; - constructor( private nodeConfig: NodeConfig, - protected projectService: IProjectService, - protected blockDispatcher: B, + @Inject('IProjectService') protected projectService: IProjectService, + @Inject('IBlockDispatcher') protected blockDispatcher: B, protected dictionaryService: DictionaryService, private eventEmitter: EventEmitter2, private schedulerRegistry: SchedulerRegistry, - private unfinalizedBlocksService: IUnfinalizedBlocksServiceUtil, + @Inject('IUnfinalizedBlocksService') private unfinalizedBlocksService: IUnfinalizedBlocksServiceUtil, private storeModelProvider: IStoreModelProvider, - private blockchainSevice: IBlockchainService + @Inject('IBlockchainService') private blockchainSevice: IBlockchainService ) {} private get latestBestHeight(): number { @@ -55,11 +47,8 @@ export abstract class BaseFetchService { - return (this.#pendingFinalizedBlockHead ??= this.getFinalizedBlockHead().finally( - () => (this.#pendingFinalizedBlockHead = undefined) - )); - } - - // Memoizes the request by not making new ones if one is already in progress - private async memoGetBestBlockHead(): Promise { - return (this.#pendingBestBlockHead ??= this.getBestBlockHead().finally( - () => (this.#pendingBestBlockHead = undefined) - )); - } - async init(startHeight: number): Promise { const interval = await this.blockchainSevice.getChainInterval(); - await Promise.all([this.memoGetFinalizedBlockHead(), this.memoGetBestBlockHead()]); + await Promise.all([this.getFinalizedBlockHead(), this.getBestBlockHead()]); const chainLatestHeight = this.latestHeight(); if (startHeight > chainLatestHeight) { @@ -113,11 +88,11 @@ export abstract class BaseFetchService void this.memoGetFinalizedBlockHead(), interval) + setInterval(() => void this.getFinalizedBlockHead(), interval) ); this.schedulerRegistry.addInterval( 'getBestBlockHead', - setInterval(() => void this.memoGetBestBlockHead(), interval) + setInterval(() => void this.getBestBlockHead(), interval) ); await this.dictionaryService.initDictionaries(); @@ -125,8 +100,7 @@ export abstract class BaseFetchService = { - getAllDataSources: () => { - return [ - makeDs([{ ...blockHandler, filter: { modulo: 5 } }]), - makeDs([callHandler]), - makeDs([eventHandler, { ...blockHandler, filter: { modulo: 2 } }]), - ]; - }, -} as any; - -const blockHandler: SubstrateBlockHandler = { - kind: SubstrateHandlerKind.Block, - handler: 'handleBlock', -}; -const callHandler: SubstrateCallHandler = { - kind: SubstrateHandlerKind.Call, - handler: 'handleCall', - filter: { method: 'call', module: 'module' }, -}; -const eventHandler: SubstrateEventHandler = { - kind: SubstrateHandlerKind.Event, - handler: 'handleEvent', - filter: { method: 'event', module: 'module' }, -}; - -const makeDs = (handlers: SubstrateRuntimeHandler[]) => { - return { - name: '', - kind: SubstrateDatasourceKind.Runtime, - mapping: { - file: '', - handlers, - }, - }; -}; - -describe('FetchSevice', () => { - let fetchService: FetchService & { - buildDictionaryQueryEntries: ( - ds: SubstrateDatasource[], - ) => DictionaryQueryEntry[]; // This is protected so we expose it here - getModulos: () => number[]; - }; - - beforeEach(() => { - fetchService = new FetchService( - null as any, // NodeConfig - projectService, // ProjectService - null as any, // BlockDispatcher, - null as any, // DictionaryService - null as any, // UnfinalizedBlocks - null as any, // EventEmitter - null as any, // SchedulerRegistry - null as any, // RuntimeService - null as any, // StoreCacheService - null as any, // BlockchainService - ) as any; - }); - - it('can extract modulo numbers from all datasources', () => { - expect( - (fetchService as any).getModulos(projectService.getAllDataSources()), - ).toEqual([5, 2]); - }); -}); diff --git a/packages/node/src/indexer/fetch.service.test.ts b/packages/node/src/indexer/fetch.service.test.ts deleted file mode 100644 index 8054208743..0000000000 --- a/packages/node/src/indexer/fetch.service.test.ts +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { ApiPromise, HttpProvider } from '@polkadot/api'; -import { FetchService } from './fetch.service'; -import { createCachedProvider } from './x-provider/cachedProvider'; - -const POLKADOT_ENDPOINT = 'https://rpc.polkadot.io'; - -// TODO move these tests to Blockchain service - -describe('FetchService', () => { - let fetchService: FetchService; - let api: ApiPromise; - - beforeAll(async () => { - // Use HTTP where possible to avoid cleanup issues - api = await ApiPromise.create({ - provider: createCachedProvider(new HttpProvider(POLKADOT_ENDPOINT)), - }); - - fetchService = new FetchService( - null as any, // NodeConfig - null as any, // ProjectService - null as any, // BlockDispatcher, - null as any, // DictionaryService - { - registerFinalizedBlock: () => { - /* Nothing */ - }, - } as any, // UnfinalizedBlocks - null as any, // EventEmitter - null as any, // SchedulerRegistry - null as any, // RuntimeService - null as any, // StoreCacheService - null as any, // BlockchainService - ) as any; - }, 10000); - - afterAll(async () => { - await api.disconnect(); - }); - - it('can get the finalized height', async () => { - const header = await (fetchService as any).getFinalizedHeader(); - expect(header.blockHeight).toBeGreaterThan(0); - }); - - it('can get the best height', async () => { - const height = await (fetchService as any).getBestHeight(); - expect(height).toBeGreaterThan(0); - }); - - it('can get the chain interval', async () => { - const interval = await (fetchService as any).getChainInterval(); - expect(interval).toEqual(5000); - }); -}); diff --git a/packages/node/src/indexer/fetch.service.ts b/packages/node/src/indexer/fetch.service.ts deleted file mode 100644 index 865bca4d34..0000000000 --- a/packages/node/src/indexer/fetch.service.ts +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { Inject, Injectable } from '@nestjs/common'; -import { EventEmitter2 } from '@nestjs/event-emitter'; -import { SchedulerRegistry } from '@nestjs/schedule'; -import { - NodeConfig, - BaseFetchService, - IStoreModelProvider, - UnfinalizedBlocksService, - ProjectService, - IBlockDispatcher, -} from '@subql/node-core'; -import { SubstrateDatasource, SubstrateBlock } from '@subql/types'; -import { BlockchainService } from '../blockchain.service'; -import { SubstrateDictionaryService } from './dictionary/substrateDictionary.service'; -import { RuntimeService } from './runtime/runtimeService'; - -@Injectable() -export class FetchService extends BaseFetchService< - SubstrateDatasource, - IBlockDispatcher, - SubstrateBlock -> { - constructor( - nodeConfig: NodeConfig, - @Inject('IProjectService') - projectService: ProjectService, - @Inject('IBlockDispatcher') - blockDispatcher: IBlockDispatcher, - dictionaryService: SubstrateDictionaryService, - @Inject('IUnfinalizedBlocksService') - unfinalizedBlocksService: UnfinalizedBlocksService, - eventEmitter: EventEmitter2, - schedulerRegistry: SchedulerRegistry, - @Inject('RuntimeService') private runtimeService: RuntimeService, - @Inject('IStoreModelProvider') storeModelProvider: IStoreModelProvider, - @Inject('IBlockchainService') blockchainService: BlockchainService, - ) { - super( - nodeConfig, - projectService, - blockDispatcher, - dictionaryService, - eventEmitter, - schedulerRegistry, - unfinalizedBlocksService, - storeModelProvider, - blockchainService, - ); - } - - protected async initBlockDispatcher(): Promise { - await this.blockDispatcher.init(this.resetForNewDs.bind(this)); - } - - protected async preLoopHook({ - startHeight, - }: { - startHeight: number; - }): Promise { - await this.runtimeService.init( - startHeight, - this.getLatestFinalizedHeight(), - ); - } -} diff --git a/packages/node/src/init.ts b/packages/node/src/init.ts index a9289cc9ea..da6955a7db 100644 --- a/packages/node/src/init.ts +++ b/packages/node/src/init.ts @@ -7,11 +7,13 @@ import { exitWithError, getLogger, getValidPort, + IBlockchainService, NestLogger, ProjectService, + FetchService, } from '@subql/node-core'; import { AppModule } from './app.module'; -import { FetchService } from './indexer/fetch.service'; +import { RuntimeService } from './indexer/runtime/runtimeService'; import { yargsOptions } from './yargs'; const pjson = require('../package.json'); @@ -33,10 +35,17 @@ export async function bootstrap(): Promise { const projectService: ProjectService = app.get('IProjectService'); const fetchService = app.get(FetchService); + const runtimeService: RuntimeService = app.get('RuntimeService'); + const blockchainService: IBlockchainService = app.get('IBlockchainService'); // Initialise async services, we do this here rather than in factories, so we can capture one off events await projectService.init(); - await fetchService.init(projectService.startHeight); + + const startHeight = projectService.startHeight; + const { blockHeight: finalizedHeight } = + await blockchainService.getFinalizedHeader(); + await runtimeService.init(startHeight, finalizedHeight); + await fetchService.init(startHeight); app.enableShutdownHooks(); From 614953eb75508a6fa3a6fa9b72b6f915a0f23983 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Tue, 6 Aug 2024 14:24:51 +1200 Subject: [PATCH 4/8] Fix up tests --- .../src/indexer/ds-processor.service.spec.ts | 52 +++++++---------- .../src/indexer/fetch.service.spec.ts | 9 ++- packages/node-core/src/utils/project.spec.ts | 57 ++++++++++++++++++ packages/node-core/test/jsonfy.js | 38 ++++++++++++ packages/node/src/blockchain.service.spec.ts | 58 +++++++++++++++++++ .../src/indexer/x-provider/cachedProvider.ts | 50 +++++----------- 6 files changed, 197 insertions(+), 67 deletions(-) rename packages/{node => node-core}/src/indexer/ds-processor.service.spec.ts (50%) create mode 100644 packages/node-core/src/utils/project.spec.ts create mode 100644 packages/node-core/test/jsonfy.js create mode 100644 packages/node/src/blockchain.service.spec.ts diff --git a/packages/node/src/indexer/ds-processor.service.spec.ts b/packages/node-core/src/indexer/ds-processor.service.spec.ts similarity index 50% rename from packages/node/src/indexer/ds-processor.service.spec.ts rename to packages/node-core/src/indexer/ds-processor.service.spec.ts index e6c0a40fde..99e8036f6e 100644 --- a/packages/node/src/indexer/ds-processor.service.spec.ts +++ b/packages/node-core/src/indexer/ds-processor.service.spec.ts @@ -2,15 +2,13 @@ // SPDX-License-Identifier: GPL-3.0 import path from 'path'; -import { isCustomDs } from '@subql/common-substrate'; -import { NodeConfig, DsProcessorService } from '@subql/node-core'; -import { SubstrateCustomDatasource, SubstrateDatasource } from '@subql/types'; -import { GraphQLSchema } from 'graphql'; -import { SubqueryProject } from '../configure/SubqueryProject'; +import {BaseCustomDataSource, BaseDataSource} from '@subql/types-core'; +import {GraphQLSchema} from 'graphql'; +import {NodeConfig} from '../configure'; +import {DsProcessorService} from './ds-processor.service'; +import {ISubqueryProject} from './types'; -function getTestProject( - extraDataSources: SubstrateCustomDatasource[], -): SubqueryProject { +function getTestProject(extraDataSources: BaseCustomDataSource[]): ISubqueryProject { return { id: 'test', root: path.resolve(__dirname, '../../'), @@ -21,46 +19,44 @@ function getTestProject( dataSources: [ { kind: 'substrate/Jsonfy', - processor: { file: 'test/jsonfy.js' }, + processor: {file: 'test/jsonfy.js'}, startBlock: 1, mapping: { - handlers: [{ handler: 'testSandbox', kind: 'substrate/JsonfyEvent' }], + handlers: [{handler: 'testSandbox', kind: 'substrate/JsonfyEvent'}], }, }, ...extraDataSources, ] as any, schema: new GraphQLSchema({}), templates: [], - } as unknown as SubqueryProject; + } as unknown as ISubqueryProject; } const nodeConfig = new NodeConfig({ subquery: 'asdf', subqueryName: 'asdf', }); +function isCustomDs(ds: BaseDataSource): ds is BaseCustomDataSource { + return ds.kind.startsWith('substrate/'); +} + describe('DsProcessorService', () => { - let service: DsProcessorService; - let project: SubqueryProject; + let service: DsProcessorService; + let project: ISubqueryProject; beforeEach(() => { project = getTestProject([]); - service = new DsProcessorService( - project, - { isCustomDs } as any, - nodeConfig, - ); + service = new DsProcessorService(project, {isCustomDs} as any, nodeConfig); }); it('can validate custom ds', async () => { - await expect( - service.validateProjectCustomDatasources(project.dataSources), - ).resolves.not.toThrow(); + await expect(service.validateProjectCustomDatasources(project.dataSources)).resolves.not.toThrow(); }); it('can catch an invalid datasource kind', async () => { - const badDs: SubstrateCustomDatasource = { + const badDs: BaseCustomDataSource = { kind: 'substrate/invalid', - processor: { file: 'contract-processors/dist/jsonfy.js' }, + processor: {file: 'contract-processors/dist/jsonfy.js'}, assets: new Map([]), mapping: { file: '', @@ -69,15 +65,9 @@ describe('DsProcessorService', () => { }; project = getTestProject([badDs]); - service = new DsProcessorService( - project, - { isCustomDs } as any, - nodeConfig, - ); + service = new DsProcessorService(project, {isCustomDs} as any, nodeConfig); - await expect( - service.validateProjectCustomDatasources(project.dataSources), - ).rejects.toThrow(); + await expect(service.validateProjectCustomDatasources(project.dataSources)).rejects.toThrow(); }); it('can run a custom ds processor', () => { diff --git a/packages/node-core/src/indexer/fetch.service.spec.ts b/packages/node-core/src/indexer/fetch.service.spec.ts index 9de5b03bcd..59abab394b 100644 --- a/packages/node-core/src/indexer/fetch.service.spec.ts +++ b/packages/node-core/src/indexer/fetch.service.spec.ts @@ -3,7 +3,13 @@ import { EventEmitter2 } from '@nestjs/event-emitter'; import { SchedulerRegistry } from '@nestjs/schedule'; -import { BaseCustomDataSource, BaseDataSource, BaseHandler, BaseMapping, DictionaryQueryEntry } from '@subql/types-core'; +import { + BaseCustomDataSource, + BaseDataSource, + BaseHandler, + BaseMapping, + DictionaryQueryEntry, +} from '@subql/types-core'; import { UnfinalizedBlocksService, BlockDispatcher, @@ -29,7 +35,6 @@ class TestFetchService extends FetchService, BaseMapping>>[] ): DictionaryQueryEntry[] { diff --git a/packages/node-core/src/utils/project.spec.ts b/packages/node-core/src/utils/project.spec.ts new file mode 100644 index 0000000000..2ff61120cc --- /dev/null +++ b/packages/node-core/src/utils/project.spec.ts @@ -0,0 +1,57 @@ +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import {isCustomDs} from '@subql/common-substrate'; +import { + SubstrateBlockHandler, + SubstrateCallHandler, + SubstrateDatasource, + SubstrateDatasourceKind, + SubstrateEventHandler, + SubstrateHandlerKind, + SubstrateRuntimeHandler, +} from '@subql/types'; +import {BaseCustomDataSource} from '@subql/types-core'; +import {getModulos} from './project'; + +const blockHandler: SubstrateBlockHandler = { + kind: SubstrateHandlerKind.Block, + handler: 'handleBlock', +}; +const callHandler: SubstrateCallHandler = { + kind: SubstrateHandlerKind.Call, + handler: 'handleCall', + filter: {method: 'call', module: 'module'}, +}; +const eventHandler: SubstrateEventHandler = { + kind: SubstrateHandlerKind.Event, + handler: 'handleEvent', + filter: {method: 'event', module: 'module'}, +}; + +const makeDs = (handlers: SubstrateRuntimeHandler[]) => { + return { + name: '', + kind: SubstrateDatasourceKind.Runtime, + mapping: { + file: '', + handlers, + }, + }; +}; + +describe('Project Utils', () => { + it('gets the correct modulos', () => { + const modulos = getModulos( + [ + makeDs([{...blockHandler, filter: {modulo: 5}}]), + makeDs([callHandler]), + makeDs([eventHandler, {...blockHandler, filter: {modulo: 2}}]), + ], + isCustomDs, + SubstrateHandlerKind.Block + ); + + expect(modulos).toEqual([5, 2]); + }); +}); diff --git a/packages/node-core/test/jsonfy.js b/packages/node-core/test/jsonfy.js new file mode 100644 index 0000000000..d17003e130 --- /dev/null +++ b/packages/node-core/test/jsonfy.js @@ -0,0 +1,38 @@ +'use strict'; +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 +Object.defineProperty(exports, '__esModule', {value: true}); +exports.JsonfyDatasourcePlugin = void 0; +exports.JsonfyDatasourcePlugin = { + kind: 'substrate/Jsonfy', + validate(ds) { + return; + }, + dsFilterProcessor(ds, api) { + return true; + }, + handlerProcessors: { + 'substrate/JsonfyEvent': { + specVersion: '1.0.0', + baseFilter: [], + baseHandlerKind: 'substrate/EventHandler', + // eslint-disable-next-line @typescript-eslint/require-await + async transformer({original, ds}) { + return JSON.parse(JSON.stringify(original.toJSON())); + }, + filterProcessor({filter, input, ds}) { + return ( + filter.module && + input.event.section === filter.module && + filter.method && + input.event.method === filter.method + ); + }, + filterValidator(filter) { + return; + }, + }, + }, +}; +exports.default = exports.JsonfyDatasourcePlugin; +//# sourceMappingURL=jsonfy.js.map diff --git a/packages/node/src/blockchain.service.spec.ts b/packages/node/src/blockchain.service.spec.ts new file mode 100644 index 0000000000..0ece505d29 --- /dev/null +++ b/packages/node/src/blockchain.service.spec.ts @@ -0,0 +1,58 @@ +// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { + ConnectionPoolService, + ConnectionPoolStateManager, + NodeConfig, +} from '@subql/node-core'; +import { BlockchainService } from './blockchain.service'; +import { ApiService } from './indexer/api.service'; + +const POLKADOT_ENDPOINT = 'https://rpc.polkadot.io'; + +describe('BlockchainService', () => { + let blockchainService: BlockchainService; + let apiService: ApiService; + + beforeAll(async () => { + const nodeConfig = new NodeConfig({} as any); + + apiService = await ApiService.init( + { + network: { + endpoint: { [POLKADOT_ENDPOINT]: {} }, + chainId: + '0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3', + }, + dataSources: [], + templates: [], + } as any, + new ConnectionPoolService(nodeConfig, new ConnectionPoolStateManager()), + new EventEmitter2(), + nodeConfig, + ); + + blockchainService = new BlockchainService(apiService, null as any) as any; + }, 10000); + + afterAll(async () => { + await apiService.onApplicationShutdown(); + }); + + it('can get the finalized height', async () => { + const header = await blockchainService.getFinalizedHeader(); + expect(header.blockHeight).toBeGreaterThan(0); + }); + + it('can get the best height', async () => { + const height = await blockchainService.getBestHeight(); + expect(height).toBeGreaterThan(0); + }); + + it('can get the chain interval', async () => { + const interval = await blockchainService.getChainInterval(); + expect(interval).toEqual(5000); + }); +}); diff --git a/packages/node/src/indexer/x-provider/cachedProvider.ts b/packages/node/src/indexer/x-provider/cachedProvider.ts index 8593b9158a..c747238aab 100644 --- a/packages/node/src/indexer/x-provider/cachedProvider.ts +++ b/packages/node/src/indexer/x-provider/cachedProvider.ts @@ -7,7 +7,6 @@ import { LRUCache } from 'lru-cache'; const MAX_CACHE_SIZE = 200; const CACHE_TTL = 60 * 1000; -/* eslint-disable prefer-rest-params */ export function createCachedProvider( provider: ProviderInterface, ): ProviderInterface { @@ -17,14 +16,13 @@ export function createCachedProvider( }); const cachedMethodHandler = ( - method: string, - params: unknown[], - target: any, - args: any[], + fn: ProviderInterface['send'], + args: Parameters, ) => { + const [method, params] = args; // If there are no parameters then we don't cache as we want the latest results if (!params.length) { - return Reflect.apply(target, provider, args); + return fn(...args); } const cacheKey = `${method}-${params[0]}`; @@ -32,38 +30,22 @@ export function createCachedProvider( return cacheMap.get(cacheKey); } - const resultPromise: Promise = Reflect.apply(target, provider, args); + const resultPromise = fn(...args); cacheMap.set(cacheKey, resultPromise); return resultPromise; }; - return new Proxy(provider, { - get: function (target, prop, receiver) { - if (prop === 'send') { - return function ( - method: string, - params: unknown[], - isCacheable: boolean, - subscription: boolean, - ) { - //caching state_getRuntimeVersion and chain_getHeader - //because they are fetched twice per block - if (['state_getRuntimeVersion', 'chain_getHeader'].includes(method)) { - return cachedMethodHandler( - method, - params, - target.send.bind(target), - arguments as unknown as any[], - ); - } + const originalSend = provider.send.bind(provider); + (provider as any).send = (...args: Parameters) => { + const [method] = args; + //caching state_getRuntimeVersion and chain_getHeader + //because they are fetched twice per block + if (['state_getRuntimeVersion', 'chain_getHeader'].includes(method)) { + return cachedMethodHandler(originalSend, args); + } - // For other methods, just forward the call to the original method. - return Reflect.apply(target.send.bind(target), provider, arguments); - }; - } + return originalSend(...args); + }; - // For other properties, just return the original value. - return Reflect.get(target, prop, receiver); - }, - }); + return provider; } From 3ffa2579131cdf1f55640730109ca18c0970693b Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Tue, 6 Aug 2024 17:42:08 +1200 Subject: [PATCH 5/8] Fix more tests --- .../indexer/unfinalizedBlocks.service.spec.ts | 2 +- packages/node/src/indexer/api.service.spec.ts | 14 +++--- .../node/src/indexer/project.service.spec.ts | 20 ++++++--- .../src/indexer/x-provider/cachedProvider.ts | 6 +-- .../src/indexer/x-provider/x-provider.spec.ts | 44 ++++++++----------- 5 files changed, 43 insertions(+), 43 deletions(-) diff --git a/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts b/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts index 913448665e..6a7e239876 100644 --- a/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts +++ b/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts @@ -89,7 +89,7 @@ describe('UnfinalizedBlocksService', () => { }); afterEach(() => { - (unfinalizedBlocksService as unknown as any).unfinalizedBlocks = {}; + (unfinalizedBlocksService as unknown as any)._unfinalizedBlocks = {}; }); it('can set finalized block', () => { diff --git a/packages/node/src/indexer/api.service.spec.ts b/packages/node/src/indexer/api.service.spec.ts index 564c36f1ee..a51549032f 100644 --- a/packages/node/src/indexer/api.service.spec.ts +++ b/packages/node/src/indexer/api.service.spec.ts @@ -25,11 +25,11 @@ jest.mock('@polkadot/api', () => { consts: jest.fn(), disconnect: jest.fn(), })); - return { ApiPromise, WsProvider: jest.fn() }; + return { ApiPromise, WsProvider: jest.fn(() => ({ send: jest.fn() })) }; }); const testNetwork = { - endpoint: { 'ws://kusama.api.onfinality.io/public-ws': {} }, + endpoint: ['ws://kusama.api.onfinality.io/public-ws'], types: { TestType: 'u32', }, @@ -108,13 +108,9 @@ describe('ApiService', () => { nodeConfig, ); const { version } = require('../../package.json'); - expect(WsProvider).toHaveBeenCalledWith( - Object.keys(testNetwork.endpoint)[0], - 2500, - { - 'User-Agent': `SubQuery-Node ${version}`, - }, - ); + expect(WsProvider).toHaveBeenCalledWith(testNetwork.endpoint[0], 2500, { + 'User-Agent': `SubQuery-Node ${version}`, + }); expect(createSpy).toHaveBeenCalledWith({ provider: expect.anything(), throwOnConnect: expect.anything(), diff --git a/packages/node/src/indexer/project.service.spec.ts b/packages/node/src/indexer/project.service.spec.ts index b71ca8cd0b..ad84ce979f 100644 --- a/packages/node/src/indexer/project.service.spec.ts +++ b/packages/node/src/indexer/project.service.spec.ts @@ -11,13 +11,14 @@ import { ProjectUpgradeService, upgradableSubqueryProject, DsProcessorService, - DynamicDsService + DynamicDsService, } from '@subql/node-core'; import { SubstrateDatasourceKind, SubstrateHandlerKind } from '@subql/types'; import { GraphQLSchema } from 'graphql'; import { BlockchainService } from '../blockchain.service'; import { SubqueryProject } from '../configure/SubqueryProject'; import { ApiService } from './api.service'; +import { RuntimeService } from './runtime/runtimeService'; function testSubqueryProject(): SubqueryProject { return { @@ -140,7 +141,11 @@ describe('ProjectService', () => { }, { provide: ProjectService, - useFactory: (apiService: ApiService, project: SubqueryProject, blockchainService: BlockchainService) => + useFactory: ( + apiService: ApiService, + project: SubqueryProject, + blockchainService: BlockchainService, + ) => new TestProjectService( { validateProjectCustomDatasources: jest.fn(), @@ -166,9 +171,9 @@ describe('ProjectService', () => { } as unknown as DynamicDsService, null as unknown as any, null as unknown as any, - blockchainService + blockchainService, ), - inject: ['APIService', 'ISubqueryProject', BlockchainService,], + inject: ['APIService', 'ISubqueryProject', 'IBlockchainService'], }, EventEmitter2, { @@ -180,10 +185,15 @@ describe('ProjectService', () => { provide: ProjectUpgradeService, useValue: projectUpgrade, }, + { + provide: 'RuntimeService', + useFactory: (apiService) => new RuntimeService(apiService), + inject: ['APIService'], + }, { provide: 'IBlockchainService', useClass: BlockchainService, - } + }, ], imports: [EventEmitterModule.forRoot()], }).compile(); diff --git a/packages/node/src/indexer/x-provider/cachedProvider.ts b/packages/node/src/indexer/x-provider/cachedProvider.ts index c747238aab..9f4e94de5f 100644 --- a/packages/node/src/indexer/x-provider/cachedProvider.ts +++ b/packages/node/src/indexer/x-provider/cachedProvider.ts @@ -7,9 +7,9 @@ import { LRUCache } from 'lru-cache'; const MAX_CACHE_SIZE = 200; const CACHE_TTL = 60 * 1000; -export function createCachedProvider( - provider: ProviderInterface, -): ProviderInterface { +export function createCachedProvider< + P extends ProviderInterface = ProviderInterface, +>(provider: P): P { const cacheMap = new LRUCache>({ max: MAX_CACHE_SIZE, ttl: CACHE_TTL, diff --git a/packages/node/src/indexer/x-provider/x-provider.spec.ts b/packages/node/src/indexer/x-provider/x-provider.spec.ts index 24333cdea0..7d7e84c983 100644 --- a/packages/node/src/indexer/x-provider/x-provider.spec.ts +++ b/packages/node/src/indexer/x-provider/x-provider.spec.ts @@ -14,13 +14,16 @@ describe('ApiPromiseConnection', () => { let httpProvider: HttpProvider; beforeEach(async () => { - wsProvider = await new WsProvider( - 'wss://kusama.api.onfinality.io/public-ws', - ).isReady; - httpProvider = new HttpProvider('https://kusama.api.onfinality.io/public'); + const ws = await new WsProvider('wss://kusama.api.onfinality.io/public-ws') + .isReady; + const http = new HttpProvider('https://kusama.api.onfinality.io/public'); - jest.spyOn(wsProvider, 'send'); - jest.spyOn(httpProvider, 'send'); + wsProvider = createCachedProvider(ws); + httpProvider = createCachedProvider(http); + + // TODO these don't work any more because the send method gets replaced + jest.spyOn(ws, 'send'); + jest.spyOn(http, 'send'); }); afterEach(async () => { @@ -28,52 +31,43 @@ describe('ApiPromiseConnection', () => { }); it('should not make duplicate requests for state_getRuntimeVersion on wsProvider', async () => { - const cachedProvider = createCachedProvider(wsProvider); - await Promise.all([ - cachedProvider.send('state_getRuntimeVersion', [TEST_BLOCKHASH]), - cachedProvider.send('state_getRuntimeVersion', [TEST_BLOCKHASH]), + wsProvider.send('state_getRuntimeVersion', [TEST_BLOCKHASH]), + wsProvider.send('state_getRuntimeVersion', [TEST_BLOCKHASH]), ]); expect(wsProvider.send).toHaveBeenCalledTimes(1); }); it('should not make duplicate requests for chain_getHeader on wsProvider', async () => { - const cachedProvider = createCachedProvider(wsProvider); await Promise.all([ - cachedProvider.send('chain_getHeader', [TEST_BLOCKHASH]), - cachedProvider.send('chain_getHeader', [TEST_BLOCKHASH]), + wsProvider.send('chain_getHeader', [TEST_BLOCKHASH]), + wsProvider.send('chain_getHeader', [TEST_BLOCKHASH]), ]); expect(wsProvider.send).toHaveBeenCalledTimes(1); }); it('should not make duplicate requests for state_getRuntimeVersion on httpProvider', async () => { - const cachedProvider = createCachedProvider(httpProvider); - await Promise.all([ - cachedProvider.send('state_getRuntimeVersion', [TEST_BLOCKHASH]), - cachedProvider.send('state_getRuntimeVersion', [TEST_BLOCKHASH]), + httpProvider.send('state_getRuntimeVersion', [TEST_BLOCKHASH]), + httpProvider.send('state_getRuntimeVersion', [TEST_BLOCKHASH]), ]); expect(httpProvider.send).toHaveBeenCalledTimes(1); }); it('should not make duplicate requests for chain_getHeader on httpProvider', async () => { - const cachedProvider = createCachedProvider(httpProvider); - await Promise.all([ - cachedProvider.send('chain_getHeader', [TEST_BLOCKHASH]), - cachedProvider.send('chain_getHeader', [TEST_BLOCKHASH]), + httpProvider.send('chain_getHeader', [TEST_BLOCKHASH]), + httpProvider.send('chain_getHeader', [TEST_BLOCKHASH]), ]); expect(httpProvider.send).toHaveBeenCalledTimes(1); }); it('should not cache requests if there are no args', async () => { - const cachedProvider = createCachedProvider(httpProvider); - - const result1 = await cachedProvider.send('chain_getHeader', []); + const result1 = await httpProvider.send('chain_getHeader', []); // Enough time for a new block await delay(7); - const result2 = await cachedProvider.send('chain_getHeader', []); + const result2 = await httpProvider.send('chain_getHeader', []); expect(httpProvider.send).toHaveBeenCalledTimes(2); expect(result1).not.toEqual(result2); From 5dd0efa907881d05e5a1647872b9a2a71efb2c71 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Wed, 13 Nov 2024 14:05:36 +1300 Subject: [PATCH 6/8] Rebase with main --- .../node-core/src/indexer/project.service.spec.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/node-core/src/indexer/project.service.spec.ts b/packages/node-core/src/indexer/project.service.spec.ts index c5f8ffca0b..42a5e446ea 100644 --- a/packages/node-core/src/indexer/project.service.spec.ts +++ b/packages/node-core/src/indexer/project.service.spec.ts @@ -9,7 +9,11 @@ import { DsProcessorService } from './ds-processor.service'; import { DatasourceParams, DynamicDsService } from './dynamic-ds.service'; import { ProjectService } from './project.service'; import { Header, IBlock, ISubqueryProject } from './types'; -import { METADATA_LAST_FINALIZED_PROCESSED_KEY, METADATA_UNFINALIZED_BLOCKS_KEY, UnfinalizedBlocksService } from './unfinalizedBlocks.service'; +import { + METADATA_LAST_FINALIZED_PROCESSED_KEY, + METADATA_UNFINALIZED_BLOCKS_KEY, + UnfinalizedBlocksService, +} from './unfinalizedBlocks.service'; import { IBaseIndexerWorker } from './worker'; class TestProjectService extends ProjectService { @@ -65,7 +69,7 @@ class TestBlockchainService implements IBlockchainService { throw new Error('Method not implemented.'); } isCustomDs(x: any): x is any { - return false + return false; } isRuntimeDs(x: any): x is any { return false; @@ -123,7 +127,7 @@ describe('BaseProjectService', () => { { getDynamicDatasources: jest.fn() } as unknown as DynamicDsService, null as unknown as any, null as unknown as any, - new TestBlockchainService(), + new TestBlockchainService() ); }); @@ -416,7 +420,7 @@ describe('BaseProjectService', () => { } as unknown as DynamicDsService, // dynamicDsService new EventEmitter2(), // eventEmitter new UnfinalizedBlocksService(nodeConfig, storeService.storeCache, blockchainService), // unfinalizedBlocksService - blockchainService, + blockchainService ); }; From 9060e9f1f10982c0618416914f62e5ecfb06c304 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Wed, 13 Nov 2024 15:06:45 +1300 Subject: [PATCH 7/8] Fix some tests --- .../src/indexer/dynamic-ds.service.spec.ts | 43 +++++++++++++------ .../src/indexer/fetch.service.spec.ts | 9 +--- .../src/indexer/project.service.spec.ts | 12 +++--- 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/packages/node-core/src/indexer/dynamic-ds.service.spec.ts b/packages/node-core/src/indexer/dynamic-ds.service.spec.ts index 50aa48a340..4ee80d09e2 100644 --- a/packages/node-core/src/indexer/dynamic-ds.service.spec.ts +++ b/packages/node-core/src/indexer/dynamic-ds.service.spec.ts @@ -14,6 +14,7 @@ class TestDynamicDsService extends DynamicDsService[number], 'name'> & {startBlock?: number}>( templateName: string, startBlock?: number | undefined @@ -42,7 +43,7 @@ const mockMetadata = (initData: DatasourceParams[] = []) => { describe('DynamicDsService', () => { let service: TestDynamicDsService; const project = { - templates: [{name: 'TestTemplate'}], + templates: [{name: 'Test'}], } as any as ISubqueryProject; beforeEach(() => { @@ -52,7 +53,9 @@ describe('DynamicDsService', () => { it('loads all datasources and params when init', async () => { await service.init(mockMetadata([testParam1])); - await expect(service.getDynamicDatasources()).resolves.toEqual([testParam1]); + await expect(service.getDynamicDatasources()).resolves.toEqual([ + {/*name: 'Test',*/ startBlock: testParam1.startBlock}, + ]); expect((service as any)._datasourceParams).toEqual([testParam1]); }); @@ -66,7 +69,10 @@ describe('DynamicDsService', () => { expect((service as any)._datasourceParams).toEqual([testParam1, testParam2]); await expect(meta.find('dynamicDatasources')).resolves.toEqual([testParam1, testParam2]); - await expect(service.getDynamicDatasources()).resolves.toEqual([testParam1, testParam2]); + await expect(service.getDynamicDatasources()).resolves.toEqual([ + {startBlock: testParam1.startBlock}, + {startBlock: testParam2.startBlock}, + ]); }); it('resets dynamic datasources', async () => { @@ -76,7 +82,10 @@ describe('DynamicDsService', () => { await service.resetDynamicDatasource(2, null as any); await expect(meta.find('dynamicDatasources')).resolves.toEqual([testParam1, testParam2]); - await expect(service.getDynamicDatasources()).resolves.toEqual([testParam1, testParam2]); + await expect(service.getDynamicDatasources()).resolves.toEqual([ + {startBlock: testParam1.startBlock}, + {startBlock: testParam2.startBlock}, + ]); }); it('getDynamicDatasources with force reloads from metadata', async () => { @@ -85,19 +94,27 @@ describe('DynamicDsService', () => { await meta.set('dynamicDatasources', [testParam1, testParam2, testParam3, testParam4]); - await expect(service.getDynamicDatasources()).resolves.toEqual([testParam1, testParam2]); + await expect(service.getDynamicDatasources()).resolves.toEqual([ + {startBlock: testParam1.startBlock}, + {startBlock: testParam2.startBlock}, + ]); await expect(service.getDynamicDatasources(true)).resolves.toEqual([ - testParam1, - testParam2, - testParam3, - testParam4, + {startBlock: testParam1.startBlock}, + {startBlock: testParam2.startBlock}, + {startBlock: testParam3.startBlock}, + {startBlock: testParam4.startBlock}, + ]); + await expect(service.getDynamicDatasources()).resolves.toEqual([ + {startBlock: testParam1.startBlock}, + {startBlock: testParam2.startBlock}, + {startBlock: testParam3.startBlock}, + {startBlock: testParam4.startBlock}, ]); - await expect(service.getDynamicDatasources()).resolves.toEqual([testParam1, testParam2, testParam3, testParam4]); }); it('can find a template and cannot mutate the template', () => { - const template1 = service.getTemplate('TestTemplate', 1); - const template2 = service.getTemplate('TestTemplate', 2); + const template1 = service.getTemplate('Test', 1); + const template2 = service.getTemplate('Test', 2); expect(template1.startBlock).toEqual(1); expect((template1 as any).name).toBeUndefined(); @@ -106,6 +123,6 @@ describe('DynamicDsService', () => { expect((template2 as any).name).toBeUndefined(); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - expect(project.templates![0]).toEqual({name: 'TestTemplate'}); + expect(project.templates![0]).toEqual({name: 'Test'}); }); }); diff --git a/packages/node-core/src/indexer/fetch.service.spec.ts b/packages/node-core/src/indexer/fetch.service.spec.ts index 59abab394b..9de5b03bcd 100644 --- a/packages/node-core/src/indexer/fetch.service.spec.ts +++ b/packages/node-core/src/indexer/fetch.service.spec.ts @@ -3,13 +3,7 @@ import { EventEmitter2 } from '@nestjs/event-emitter'; import { SchedulerRegistry } from '@nestjs/schedule'; -import { - BaseCustomDataSource, - BaseDataSource, - BaseHandler, - BaseMapping, - DictionaryQueryEntry, -} from '@subql/types-core'; +import { BaseCustomDataSource, BaseDataSource, BaseHandler, BaseMapping, DictionaryQueryEntry } from '@subql/types-core'; import { UnfinalizedBlocksService, BlockDispatcher, @@ -35,6 +29,7 @@ class TestFetchService extends FetchService, BaseMapping>>[] ): DictionaryQueryEntry[] { diff --git a/packages/node-core/src/indexer/project.service.spec.ts b/packages/node-core/src/indexer/project.service.spec.ts index 42a5e446ea..ae14e9ad0b 100644 --- a/packages/node-core/src/indexer/project.service.spec.ts +++ b/packages/node-core/src/indexer/project.service.spec.ts @@ -38,7 +38,7 @@ class TestBlockchainService implements IBlockchainService { // eslint-disable-next-line @typescript-eslint/promise-function-async fetchBlocks(blockNums: number[]): Promise[]> { - throw new Error('Method not implemented.'); + throw new Error('Method fetchBlocks not implemented.'); } // eslint-disable-next-line @typescript-eslint/promise-function-async @@ -47,26 +47,26 @@ class TestBlockchainService implements IBlockchainService { } onProjectChange(project: ISubqueryProject): Promise | void { - throw new Error('Method not implemented.'); + // throw new Error('Method onProjectChange not implemented.'); } // eslint-disable-next-line @typescript-eslint/promise-function-async getBlockTimestamp(height: number): Promise { - throw new Error('Method not implemented.'); + throw new Error('Method getBlockTimestamp not implemented.'); } getBlockSize(block: IBlock): number { return 0; } // eslint-disable-next-line @typescript-eslint/promise-function-async getBestHeight(): Promise { - throw new Error('Method not implemented.'); + throw new Error('Method getBestHeight not implemented.'); } // eslint-disable-next-line @typescript-eslint/promise-function-async getChainInterval(): Promise { - throw new Error('Method not implemented.'); + throw new Error('Method getChainInterval not implemented.'); } // eslint-disable-next-line @typescript-eslint/promise-function-async updateDynamicDs(params: DatasourceParams, template: any): Promise { - throw new Error('Method not implemented.'); + throw new Error('Method updateDynamicDs not implemented.'); } isCustomDs(x: any): x is any { return false; From a0e730a8ae83cbdb7b5c0591f8e6a609ab358138 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Wed, 13 Nov 2024 15:50:00 +1300 Subject: [PATCH 8/8] Fix up more tests --- .../src/indexer/x-provider/x-provider.spec.ts | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/packages/node/src/indexer/x-provider/x-provider.spec.ts b/packages/node/src/indexer/x-provider/x-provider.spec.ts index 7d7e84c983..20b89b3049 100644 --- a/packages/node/src/indexer/x-provider/x-provider.spec.ts +++ b/packages/node/src/indexer/x-provider/x-provider.spec.ts @@ -12,21 +12,28 @@ const TEST_BLOCKHASH = describe('ApiPromiseConnection', () => { let wsProvider: WsProvider; let httpProvider: HttpProvider; + let wsSpy: jest.SpyInstance; + let httpSpy: jest.SpyInstance; - beforeEach(async () => { + beforeAll(async () => { const ws = await new WsProvider('wss://kusama.api.onfinality.io/public-ws') .isReady; const http = new HttpProvider('https://kusama.api.onfinality.io/public'); + // Spys created before the send function is modified + wsSpy = jest.spyOn(ws, 'send'); + httpSpy = jest.spyOn(http, 'send'); + wsProvider = createCachedProvider(ws); httpProvider = createCachedProvider(http); + }); - // TODO these don't work any more because the send method gets replaced - jest.spyOn(ws, 'send'); - jest.spyOn(http, 'send'); + afterEach(() => { + wsSpy.mockClear(); + httpSpy.mockClear(); }); - afterEach(async () => { + afterAll(async () => { await Promise.all([wsProvider?.disconnect(), httpProvider?.disconnect()]); }); @@ -35,8 +42,7 @@ describe('ApiPromiseConnection', () => { wsProvider.send('state_getRuntimeVersion', [TEST_BLOCKHASH]), wsProvider.send('state_getRuntimeVersion', [TEST_BLOCKHASH]), ]); - - expect(wsProvider.send).toHaveBeenCalledTimes(1); + expect(wsSpy).toHaveBeenCalledTimes(1); }); it('should not make duplicate requests for chain_getHeader on wsProvider', async () => { @@ -44,7 +50,7 @@ describe('ApiPromiseConnection', () => { wsProvider.send('chain_getHeader', [TEST_BLOCKHASH]), wsProvider.send('chain_getHeader', [TEST_BLOCKHASH]), ]); - expect(wsProvider.send).toHaveBeenCalledTimes(1); + expect(wsSpy).toHaveBeenCalledTimes(1); }); it('should not make duplicate requests for state_getRuntimeVersion on httpProvider', async () => { @@ -52,7 +58,7 @@ describe('ApiPromiseConnection', () => { httpProvider.send('state_getRuntimeVersion', [TEST_BLOCKHASH]), httpProvider.send('state_getRuntimeVersion', [TEST_BLOCKHASH]), ]); - expect(httpProvider.send).toHaveBeenCalledTimes(1); + expect(httpSpy).toHaveBeenCalledTimes(1); }); it('should not make duplicate requests for chain_getHeader on httpProvider', async () => { @@ -60,7 +66,7 @@ describe('ApiPromiseConnection', () => { httpProvider.send('chain_getHeader', [TEST_BLOCKHASH]), httpProvider.send('chain_getHeader', [TEST_BLOCKHASH]), ]); - expect(httpProvider.send).toHaveBeenCalledTimes(1); + expect(httpSpy).toHaveBeenCalledTimes(1); }); it('should not cache requests if there are no args', async () => { @@ -69,7 +75,7 @@ describe('ApiPromiseConnection', () => { await delay(7); const result2 = await httpProvider.send('chain_getHeader', []); - expect(httpProvider.send).toHaveBeenCalledTimes(2); + expect(httpSpy).toHaveBeenCalledTimes(2); expect(result1).not.toEqual(result2); }, 10000); });