Fix up tests

packages/node/src/indexer/ds-processor.service.spec.ts → packages/node-core/src/indexer/ds-processor.service.spec.ts

@@ -2,15 +2,13 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import path from 'path';
-import { isCustomDs } from '@subql/common-substrate';
-import { NodeConfig, DsProcessorService } from '@subql/node-core';
-import { SubstrateCustomDatasource, SubstrateDatasource } from '@subql/types';
-import { GraphQLSchema } from 'graphql';
-import { SubqueryProject } from '../configure/SubqueryProject';
+import {BaseCustomDataSource, BaseDataSource} from '@subql/types-core';
+import {GraphQLSchema} from 'graphql';
+import {NodeConfig} from '../configure';
+import {DsProcessorService} from './ds-processor.service';
+import {ISubqueryProject} from './types';
 
-function getTestProject(
-  extraDataSources: SubstrateCustomDatasource[],
-): SubqueryProject {
+function getTestProject(extraDataSources: BaseCustomDataSource[]): ISubqueryProject {
   return {
     id: 'test',
     root: path.resolve(__dirname, '../../'),
@@ -21,46 +19,44 @@ function getTestProject(
     dataSources: [
       {
         kind: 'substrate/Jsonfy',
-        processor: { file: 'test/jsonfy.js' },
+        processor: {file: 'test/jsonfy.js'},
         startBlock: 1,
         mapping: {
-          handlers: [{ handler: 'testSandbox', kind: 'substrate/JsonfyEvent' }],
+          handlers: [{handler: 'testSandbox', kind: 'substrate/JsonfyEvent'}],
         },
       },
       ...extraDataSources,
     ] as any,
     schema: new GraphQLSchema({}),
     templates: [],
-  } as unknown as SubqueryProject;
+  } as unknown as ISubqueryProject;
 }
 const nodeConfig = new NodeConfig({
   subquery: 'asdf',
   subqueryName: 'asdf',
 });
 
+function isCustomDs(ds: BaseDataSource): ds is BaseCustomDataSource {
+  return ds.kind.startsWith('substrate/');
+}
+
 describe('DsProcessorService', () => {
-  let service: DsProcessorService<SubstrateDatasource>;
-  let project: SubqueryProject;
+  let service: DsProcessorService<BaseDataSource>;
+  let project: ISubqueryProject;
 
   beforeEach(() => {
     project = getTestProject([]);
-    service = new DsProcessorService(
-      project,
-      { isCustomDs } as any,
-      nodeConfig,
-    );
+    service = new DsProcessorService(project, {isCustomDs} as any, nodeConfig);
   });
 
   it('can validate custom ds', async () => {
-    await expect(
-      service.validateProjectCustomDatasources(project.dataSources),
-    ).resolves.not.toThrow();
+    await expect(service.validateProjectCustomDatasources(project.dataSources)).resolves.not.toThrow();
   });
 
   it('can catch an invalid datasource kind', async () => {
-    const badDs: SubstrateCustomDatasource<string, any> = {
+    const badDs: BaseCustomDataSource = {
       kind: 'substrate/invalid',
-      processor: { file: 'contract-processors/dist/jsonfy.js' },
+      processor: {file: 'contract-processors/dist/jsonfy.js'},
       assets: new Map([]),
       mapping: {
         file: '',
@@ -69,15 +65,9 @@ describe('DsProcessorService', () => {
     };
 
     project = getTestProject([badDs]);
-    service = new DsProcessorService(
-      project,
-      { isCustomDs } as any,
-      nodeConfig,
-    );
+    service = new DsProcessorService(project, {isCustomDs} as any, nodeConfig);
 
-    await expect(
-      service.validateProjectCustomDatasources(project.dataSources),
-    ).rejects.toThrow();
+    await expect(service.validateProjectCustomDatasources(project.dataSources)).rejects.toThrow();
   });
 
   it('can run a custom ds processor', () => {

packages/node-core/src/indexer/fetch.service.spec.ts

@@ -3,7 +3,14 @@
 
 import {EventEmitter2} from '@nestjs/event-emitter';
 import {SchedulerRegistry} from '@nestjs/schedule';
-import {BaseDataSource, BaseHandler, BaseMapping, DictionaryQueryEntry, IProjectNetworkConfig} from '@subql/types-core';
+import {
+  BaseCustomDataSource,
+  BaseDataSource,
+  BaseHandler,
+  BaseMapping,
+  DictionaryQueryEntry,
+  IProjectNetworkConfig,
+} from '@subql/types-core';
 import {range} from 'lodash';
 import {
   UnfinalizedBlocksService,
@@ -16,39 +23,21 @@ import {
   NodeConfig,
   IBlockchainService,
   ISubqueryProject,
+  DatasourceParams,
+  IBaseIndexerWorker,
 } from '../';
 import {BlockHeightMap} from '../utils/blockHeightMap';
 import {DictionaryService} from './dictionary/dictionary.service';
 import {FetchService} from './fetch.service';
 
 const CHAIN_INTERVAL = 100; // 100ms
-const genesisHash = '0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3';
 
 class TestFetchService extends FetchService<BaseDataSource, IBlockDispatcher<any>, any> {
-  finalizedHeight = 1000;
-  bestHeight = 20;
-
   protected buildDictionaryQueryEntries(
     dataSources: BaseDataSource<BaseHandler<any>, BaseMapping<BaseHandler<any>>>[]
   ): DictionaryQueryEntry[] {
     return [];
   }
-  getGenesisHash(): string {
-    return genesisHash;
-  }
-  async getBestHeight(): Promise<number> {
-    return Promise.resolve(this.bestHeight);
-  }
-  protected async getChainInterval(): Promise<number> {
-    return Promise.resolve(CHAIN_INTERVAL);
-  }
-
-  protected async initBlockDispatcher(): Promise<void> {
-    return Promise.resolve();
-  }
-  async preLoopHook(data: {startHeight: number}): Promise<void> {
-    return Promise.resolve();
-  }
 
   protected getModulos(dataSources: BaseDataSource[]): number[] {
     // This is mocks get modulos, checkes every handler
@@ -72,10 +61,67 @@ class TestFetchService extends FetchService<BaseDataSource, IBlockDispatcher<any
   mockDsMap(blockHeightMap: BlockHeightMap<any>): void {
     this.projectService.getDataSourcesMap = jest.fn(() => blockHeightMap);
   }
+}
 
+class TestBlockchainService implements IBlockchainService {
+  finalizedHeight = 1000;
+  bestHeight = 20;
+  blockHandlerKind = '';
+  packageVersion = '1.0.0';
+  // eslint-disable-next-line @typescript-eslint/promise-function-async
+  fetchBlocks(blockNums: number[]): Promise<IBlock<any>[]> {
+    throw new Error('Method not implemented.');
+  }
+  // eslint-disable-next-line @typescript-eslint/promise-function-async
+  fetchBlockWorker(
+    worker: IBaseIndexerWorker,
+    blockNum: number,
+    context: {workers: IBaseIndexerWorker[]}
+  ): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
   async getFinalizedHeader(): Promise<Header> {
     return Promise.resolve({blockHeight: this.finalizedHeight, blockHash: '0xxx', parentHash: '0xxx'});
   }
+  async getBestHeight(): Promise<number> {
+    return Promise.resolve(this.bestHeight);
+  }
+  async getChainInterval(): Promise<number> {
+    return Promise.resolve(CHAIN_INTERVAL);
+  }
+  // eslint-disable-next-line @typescript-eslint/promise-function-async
+  getHeaderForHash(hash: string): Promise<Header> {
+    throw new Error('Method not implemented.');
+  }
+  // eslint-disable-next-line @typescript-eslint/promise-function-async
+  getHeaderForHeight(height: number): Promise<Header> {
+    throw new Error('Method not implemented.');
+  }
+  // eslint-disable-next-line @typescript-eslint/promise-function-async
+  updateDynamicDs(
+    params: DatasourceParams,
+    template: BaseDataSource | (BaseCustomDataSource & BaseDataSource)
+  ): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
+  isCustomDs(x: BaseDataSource | (BaseCustomDataSource & BaseDataSource)): x is BaseCustomDataSource {
+    throw new Error('Method not implemented.');
+  }
+  isRuntimeDs(x: BaseDataSource | (BaseCustomDataSource & BaseDataSource)): x is BaseDataSource {
+    throw new Error('Method not implemented.');
+  }
+  // eslint-disable-next-line @typescript-eslint/promise-function-async
+  getSafeApi(block: any): Promise<any> {
+    throw new Error('Method not implemented.');
+  }
+  // eslint-disable-next-line @typescript-eslint/promise-function-async
+  onProjectChange(project: ISubqueryProject): Promise<void> | void {
+    throw new Error('Method not implemented.');
+  }
+  // eslint-disable-next-line @typescript-eslint/promise-function-async
+  getBlockTimestamp(height: number): Promise<Date | undefined> {
+    throw new Error('Method not implemented.');
+  }
 }
 
 const nodeConfig = new NodeConfig({
@@ -146,6 +192,7 @@ const getDictionaryService = () =>
 
 const getBlockDispatcher = () => {
   const inst = {
+    init: (fn: any) => Promise.resolve(),
     latestBufferedHeight: 0,
     smartBatchSize: 10,
     minimumHeapLimit: 1000,
@@ -169,7 +216,7 @@ describe('Fetch Service', () => {
   let networkConfig: IProjectNetworkConfig;
   let dataSources: BaseDataSource[];
   let unfinalizedBlocksService: UnfinalizedBlocksService<any>;
-  let blockchainService: IBlockchainService;
+  let blockchainService: TestBlockchainService;
 
   let spyOnEnqueueSequential: jest.SpyInstance<
     void | Promise<void>,
@@ -205,6 +252,10 @@ describe('Fetch Service', () => {
     blockDispatcher = getBlockDispatcher();
     dictionaryService = getDictionaryService();
     networkConfig = getNetworkConfig();
+    blockchainService = new TestBlockchainService();
+    unfinalizedBlocksService = {
+      registerFinalizedBlock: jest.fn(),
+    } as unknown as UnfinalizedBlocksService;
 
     fetchService = new TestFetchService(
       nodeConfig,
@@ -278,14 +329,6 @@ describe('Fetch Service', () => {
     jest.clearAllMocks();
   });
 
-  it('calls the preHookLoop when init is called', async () => {
-    const preHookLoopSpy = jest.spyOn(fetchService, 'preLoopHook');
-
-    await fetchService.init(1);
-
-    expect(preHookLoopSpy).toHaveBeenCalled();
-  });
-
   it('adds bypassBlocks for empty datasources', async () => {
     fetchService.mockDsMap(
       new BlockHeightMap(
@@ -339,8 +382,8 @@ describe('Fetch Service', () => {
   });
 
   it('checks chain heads at an interval', async () => {
-    const finalizedSpy = jest.spyOn(fetchService, 'getFinalizedHeader');
-    const bestSpy = jest.spyOn(fetchService, 'getBestHeight');
+    const finalizedSpy = jest.spyOn(blockchainService, 'getFinalizedHeader');
+    const bestSpy = jest.spyOn(blockchainService, 'getBestHeight');
 
     await fetchService.init(1);
 
@@ -353,8 +396,8 @@ describe('Fetch Service', () => {
     expect(finalizedSpy).toHaveBeenCalledTimes(2);
     expect(bestSpy).toHaveBeenCalledTimes(2);
 
-    await expect(fetchService.getFinalizedHeader()).resolves.toEqual({
-      blockHeight: fetchService.finalizedHeight,
+    await expect(blockchainService.getFinalizedHeader()).resolves.toEqual({
+      blockHeight: blockchainService.finalizedHeight,
       blockHash: '0xxx',
       parentHash: '0xxx',
     });
@@ -517,7 +560,7 @@ describe('Fetch Service', () => {
 
   it('update the LatestBufferHeight when modulo blocks full synced', async () => {
     fetchService.mockGetModulos([20]);
-    fetchService.finalizedHeight = 55;
+    blockchainService.finalizedHeight = 55;
 
     // simulate we have synced to block 50, and modulo is 20, next block to handle suppose be 60,80,100...
     // we will still enqueue 55 to update LatestBufferHeight
@@ -668,7 +711,7 @@ describe('Fetch Service', () => {
 
     const FINALIZED_HEIGHT = 10;
 
-    fetchService.finalizedHeight = FINALIZED_HEIGHT;
+    blockchainService.finalizedHeight = FINALIZED_HEIGHT;
     // change query end
     (dictionaryService as any).getDictionary(1).getQueryEndBlock = () => 10;
 
@@ -701,7 +744,7 @@ describe('Fetch Service', () => {
     (fetchService as any).dictionaryService.scopedDictionaryEntries = () => {
       return undefined;
     };
-    fetchService.bestHeight = 500;
+    blockchainService.bestHeight = 500;
     const dictionarySpy = jest.spyOn((fetchService as any).dictionaryService, 'scopedDictionaryEntries');
     await fetchService.init(10);
     expect(dictionarySpy).toHaveBeenCalledTimes(1);
@@ -715,7 +758,7 @@ describe('Fetch Service', () => {
     (fetchService as any).dictionaryService.scopedDictionaryEntries = () => {
       return undefined;
     };
-    fetchService.bestHeight = 500;
+    blockchainService.bestHeight = 500;
     const dictionarySpy = jest.spyOn((fetchService as any).dictionaryService, 'scopedDictionaryEntries');
     await fetchService.init(490);
     expect(dictionarySpy).toHaveBeenCalledTimes(0);

packages/node-core/src/indexer/fetch.service.ts

@@ -234,7 +234,6 @@ export class FetchService<DS extends BaseDataSource, B extends IBlockDispatcher<
       // This could be latestBestHeight, dictionary should never include finalized blocks
       // TODO add buffer so dictionary not used when project synced
       if (startBlockHeight < this.latestBestHeight - scaledBatchSize) {
-        // if (startBlockHeight < this.latestFinalizedHeight) {
         try {
           const dictionary = await this.dictionaryService.scopedDictionaryEntries(
             startBlockHeight,

packages/node-core/src/utils/project.spec.ts

@@ -0,0 +1,57 @@
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+
+import {isCustomDs} from '@subql/common-substrate';
+import {
+  SubstrateBlockHandler,
+  SubstrateCallHandler,
+  SubstrateDatasource,
+  SubstrateDatasourceKind,
+  SubstrateEventHandler,
+  SubstrateHandlerKind,
+  SubstrateRuntimeHandler,
+} from '@subql/types';
+import {BaseCustomDataSource} from '@subql/types-core';
+import {getModulos} from './project';
+
+const blockHandler: SubstrateBlockHandler = {
+  kind: SubstrateHandlerKind.Block,
+  handler: 'handleBlock',
+};
+const callHandler: SubstrateCallHandler = {
+  kind: SubstrateHandlerKind.Call,
+  handler: 'handleCall',
+  filter: {method: 'call', module: 'module'},
+};
+const eventHandler: SubstrateEventHandler = {
+  kind: SubstrateHandlerKind.Event,
+  handler: 'handleEvent',
+  filter: {method: 'event', module: 'module'},
+};
+
+const makeDs = (handlers: SubstrateRuntimeHandler[]) => {
+  return {
+    name: '',
+    kind: SubstrateDatasourceKind.Runtime,
+    mapping: {
+      file: '',
+      handlers,
+    },
+  };
+};
+
+describe('Project Utils', () => {
+  it('gets the correct modulos', () => {
+    const modulos = getModulos<SubstrateDatasource, BaseCustomDataSource>(
+      [
+        makeDs([{...blockHandler, filter: {modulo: 5}}]),
+        makeDs([callHandler]),
+        makeDs([eventHandler, {...blockHandler, filter: {modulo: 2}}]),
+      ],
+      isCustomDs,
+      SubstrateHandlerKind.Block
+    );
+
+    expect(modulos).toEqual([5, 2]);
+  });
+});