From 86aa14975f74a4d9fa07a0e24675c9632249b505 Mon Sep 17 00:00:00 2001 From: Matthew Whitehead Date: Wed, 15 Mar 2023 12:20:19 +0000 Subject: [PATCH 1/4] Retry logic for blockchain calls Signed-off-by: Matthew Whitehead --- README.md | 27 ++++++- src/main.ts | 20 ++++- src/tokens/blockchain.service.ts | 117 ++++++++++++++++++++++++------ src/tokens/tokens.service.spec.ts | 2 +- test/app.e2e-context.ts | 4 +- 5 files changed, 142 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 5a5cca4..7e4b8e8 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,9 @@ are additional methods used by the token connector to guess at the contract ABI but is the preferred method for most use cases. To leverage this capability in a running FireFly environment, you must: + 1. [Upload the token contract ABI to FireFly](https://hyperledger.github.io/firefly/tutorials/custom_contracts/ethereum.html) -as a contract interface. + as a contract interface. 2. Include the `interface` parameter when [creating the pool on FireFly](https://hyperledger.github.io/firefly/tutorials/tokens). This will cause FireFly to parse the interface and provide ABI details @@ -119,7 +120,29 @@ that specific token. If omitted, the approval covers all tokens. The following APIs are not part of the fftokens standard, but are exposed under `/api/v1`: -* `GET /receipt/:id` - Get receipt for a previous request +- `GET /receipt/:id` - Get receipt for a previous request + +## Retry behaviour + +Most short-term outages should be handled by the blockchain connector. For example if the blockchain node returns `HTTP 429` due to rate limiting +it is the blockchain connector's responsibility to use appropriate back-off retries to attempt to make the required blockchain call successfully. + +There are cases where the token connector may need to perform it's own back-off retry for a blockchain action. For example if the blockchain connector +microservice has crashed and is in the process of restarting just as the token connector is trying to query an NFT token URI to enrich a token event, if +the token connector doesn't perform a retry then the event will be returned without the token URI populated. + +The token connector has configurable retry behaviour for all blockchain related calls. By default the connector will perform up to 15 retries with a back-off +interval between each one. The default first retry interval is 100ms and doubles up to a maximum of 10s per retry interval. Retries are only performed where +the error returned from the REST call matches a configurable regular expression retry condition. The default retry condition is `._ECONN._` which ensures +retries take place for common TCP errors such as `ECONNRESET` and `ECONNREFUSED`. + +Setting the retry condition to "" disables retries. The configurable retry settings are: + +- `RETRY_BACKOFF_FACTOR` (default `2`) +- `RETRY_BACKOFF_LIMIT_MS` (default `10000`) +- `RETRY_BACKOFF_INITIAL_MS` (default `100`) +- `RETRY_CONDITION` (default `.*ECONN.*`) +- `RETRY_MAX_ATTEMPTS` (default `15`) ## Running the service diff --git a/src/main.ts b/src/main.ts index 89d29f2..2b7db0a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -84,6 +84,13 @@ async function bootstrap() { const legacyERC20 = config.get('USE_LEGACY_ERC20_SAMPLE', '').toLowerCase() === 'true'; const legacyERC721 = config.get('USE_LEGACY_ERC721_SAMPLE', '').toLowerCase() === 'true'; + // Configuration for retries + const retryBackOffFactor = config.get('RETRY_BACKOFF_FACTOR', 2); + const retryBackOffLimit = config.get('RETRY_BACKOFF_LIMIT_MS', 10000); + const retryBackOffInitial = config.get('RETRY_BACKOFF_INITIAL_MS', 100); + const retryCondition = config.get('RETRY_CONDITION', '.*ECONN.*'); + const retriesMax = config.get('RETRY_MAX_ATTEMPTS', 15); + const passthroughHeaders: string[] = []; for (const h of passthroughHeaderString.split(',')) { passthroughHeaders.push(h.toLowerCase()); @@ -93,7 +100,18 @@ async function bootstrap() { app.get(TokensService).configure(ethConnectUrl, topic, factoryAddress); app .get(BlockchainConnectorService) - .configure(ethConnectUrl, fftmUrl, username, password, passthroughHeaders); + .configure( + ethConnectUrl, + fftmUrl, + username, + password, + passthroughHeaders, + retryBackOffFactor, + retryBackOffLimit, + retryBackOffInitial, + retryCondition, + retriesMax, + ); app.get(AbiMapperService).configure(legacyERC20, legacyERC721); if (autoInit.toLowerCase() !== 'false') { diff --git a/src/tokens/blockchain.service.ts b/src/tokens/blockchain.service.ts index 3ff67ad..fd811b8 100644 --- a/src/tokens/blockchain.service.ts +++ b/src/tokens/blockchain.service.ts @@ -43,6 +43,12 @@ export class BlockchainConnectorService { password: string; passthroughHeaders: string[]; + retryBackOffFactor: number; + retryBackOffLimit: number; + retryBackOffInitial: number; + retryCondition: string; + retriesMax: number; + constructor(public http: HttpService) {} configure( @@ -51,12 +57,22 @@ export class BlockchainConnectorService { username: string, password: string, passthroughHeaders: string[], + retryBackOffFactor: number, + retryBackOffLimit: number, + retryBackOffInitial: number, + retryCondition: string, + retriesMax: number, ) { this.baseUrl = baseUrl; this.fftmUrl = fftmUrl; this.username = username; this.password = password; this.passthroughHeaders = passthroughHeaders; + this.retryBackOffFactor = retryBackOffFactor; + this.retryBackOffLimit = retryBackOffLimit; + this.retryBackOffInitial = retryBackOffInitial; + this.retryCondition = retryCondition; + this.retriesMax = retriesMax; } private requestOptions(ctx: Context): AxiosRequestConfig { @@ -88,15 +104,60 @@ export class BlockchainConnectorService { }); } + // Check if retry condition matches the err that's been hit + private matchesRetryCondition(err: any): boolean { + return this.retryCondition != '' && err?.toString().match(this.retryCondition) !== null; + } + + // Delay by the appropriate amount of time given the iteration the caller is in + private async backoffDelay(iteration: number) { + const delay = Math.min( + this.retryBackOffInitial * Math.pow(this.retryBackOffFactor, iteration), + this.retryBackOffLimit, + ); + await new Promise(resolve => setTimeout(resolve, delay)); + } + + // Generic helper function that makes an given blockchain function retryable + // by using synchronous, back-off delays for cases where the function returns + // an error which matches the configured retry condition + private async retryableCall( + blockchainFunction: () => Promise>, + ): Promise> { + let response: any; + for (let retries = 0; retries <= this.retriesMax; retries++) { + try { + response = await blockchainFunction(); + break; + } catch (e) { + if (this.matchesRetryCondition(e)) { + this.logger.debug(`Retry condition matched for error ${e}`); + // Wait for a backed-off delay before trying again + await this.backoffDelay(retries); + } else { + // Whatever the error was it's not one we will retry for + break; + } + } + } + + return response; + } + async query(ctx: Context, to: string, method?: IAbiMethod, params?: any[]) { - const response = await this.wrapError( - lastValueFrom( - this.http.post( - this.baseUrl, - { headers: { type: queryHeader }, to, method, params }, - this.requestOptions(ctx), - ), - ), + const url = this.baseUrl; + const response = await this.retryableCall( + async (): Promise> => { + return await this.wrapError( + lastValueFrom( + this.http.post( + url, + { headers: { type: queryHeader }, to, method, params }, + this.requestOptions(ctx), + ), + ), + ); + }, ); return response.data; } @@ -110,26 +171,36 @@ export class BlockchainConnectorService { params?: any[], ) { const url = this.fftmUrl !== undefined && this.fftmUrl !== '' ? this.fftmUrl : this.baseUrl; - const response = await this.wrapError( - lastValueFrom( - this.http.post( - url, - { headers: { id, type: sendTransactionHeader }, from, to, method, params }, - this.requestOptions(ctx), - ), - ), + + const response = await this.retryableCall( + async (): Promise> => { + return await this.wrapError( + lastValueFrom( + this.http.post( + url, + { headers: { id, type: sendTransactionHeader }, from, to, method, params }, + this.requestOptions(ctx), + ), + ), + ); + }, ); return response.data; } async getReceipt(ctx: Context, id: string): Promise { - const response = await this.wrapError( - lastValueFrom( - this.http.get(new URL(`/reply/${id}`, this.baseUrl).href, { - validateStatus: status => status < 300 || status === 404, - ...this.requestOptions(ctx), - }), - ), + const url = this.baseUrl; + const response = await this.retryableCall( + async (): Promise> => { + return await this.wrapError( + lastValueFrom( + this.http.get(new URL(`/reply/${id}`, url).href, { + validateStatus: status => status < 300 || status === 404, + ...this.requestOptions(ctx), + }), + ), + ); + }, ); if (response.status === 404) { throw new NotFoundException(); diff --git a/src/tokens/tokens.service.spec.ts b/src/tokens/tokens.service.spec.ts index f83c22f..4de5ed8 100644 --- a/src/tokens/tokens.service.spec.ts +++ b/src/tokens/tokens.service.spec.ts @@ -235,7 +235,7 @@ describe('TokensService', () => { service = module.get(TokensService); service.configure(BASE_URL, TOPIC, ''); blockchain = module.get(BlockchainConnectorService); - blockchain.configure(BASE_URL, '', '', '', []); + blockchain.configure(BASE_URL, '', '', '', [], 2, 1000, 250, '.*ECONN.*', 15); }); it('should be defined', () => { diff --git a/test/app.e2e-context.ts b/test/app.e2e-context.ts index bdb20da..7ab24f7 100644 --- a/test/app.e2e-context.ts +++ b/test/app.e2e-context.ts @@ -71,7 +71,9 @@ export class TestContext { this.app.get(EventStreamProxyGateway).configure('url', TOPIC); this.app.get(TokensService).configure(BASE_URL, TOPIC, ''); - this.app.get(BlockchainConnectorService).configure(BASE_URL, '', '', '', []); + this.app + .get(BlockchainConnectorService) + .configure(BASE_URL, '', '', '', [], 2, 1000, 250, '.*ECONN.*', 15); (this.app.getHttpServer() as Server).listen(); this.server = request(this.app.getHttpServer()); From a1ddb32fa282ef6dee1ed9e2bc56ac0fe022f992 Mon Sep 17 00:00:00 2001 From: Matthew Whitehead Date: Tue, 28 Mar 2023 08:46:12 +0100 Subject: [PATCH 2/4] Addressing PR comments Signed-off-by: Matthew Whitehead --- README.md | 46 +++++++------ src/main.ts | 29 +++----- src/tokens/blockchain.service.ts | 107 +++++++++++++++--------------- src/tokens/tokens.service.spec.ts | 12 +++- 4 files changed, 98 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index 7e4b8e8..8f53aa9 100644 --- a/README.md +++ b/README.md @@ -122,28 +122,6 @@ The following APIs are not part of the fftokens standard, but are exposed under - `GET /receipt/:id` - Get receipt for a previous request -## Retry behaviour - -Most short-term outages should be handled by the blockchain connector. For example if the blockchain node returns `HTTP 429` due to rate limiting -it is the blockchain connector's responsibility to use appropriate back-off retries to attempt to make the required blockchain call successfully. - -There are cases where the token connector may need to perform it's own back-off retry for a blockchain action. For example if the blockchain connector -microservice has crashed and is in the process of restarting just as the token connector is trying to query an NFT token URI to enrich a token event, if -the token connector doesn't perform a retry then the event will be returned without the token URI populated. - -The token connector has configurable retry behaviour for all blockchain related calls. By default the connector will perform up to 15 retries with a back-off -interval between each one. The default first retry interval is 100ms and doubles up to a maximum of 10s per retry interval. Retries are only performed where -the error returned from the REST call matches a configurable regular expression retry condition. The default retry condition is `._ECONN._` which ensures -retries take place for common TCP errors such as `ECONNRESET` and `ECONNREFUSED`. - -Setting the retry condition to "" disables retries. The configurable retry settings are: - -- `RETRY_BACKOFF_FACTOR` (default `2`) -- `RETRY_BACKOFF_LIMIT_MS` (default `10000`) -- `RETRY_BACKOFF_INITIAL_MS` (default `100`) -- `RETRY_CONDITION` (default `.*ECONN.*`) -- `RETRY_MAX_ATTEMPTS` (default `15`) - ## Running the service The easiest way to run this service is as part of a stack created via @@ -202,3 +180,27 @@ $ npm run lint # formatting $ npm run format ``` + +## Blockchain retry behaviour + +Most short-term outages should be handled by the blockchain connector. For example if the blockchain node returns `HTTP 429` due to rate limiting +it is the blockchain connector's responsibility to use appropriate back-off retries to attempt to make the required blockchain call successfully. + +There are cases where the token connector may need to perform its own back-off retry for a blockchain action. For example if the blockchain connector +microservice has crashed and is in the process of restarting just as the token connector is trying to query an NFT token URI to enrich a token event, if +the token connector doesn't perform a retry then the event will be returned without the token URI populated. + +The token connector has configurable retry behaviour for all blockchain related calls. By default the connector will perform up to 15 retries with a back-off +interval between each one. The default first retry interval is 100ms and doubles up to a maximum of 10s per retry interval. Retries are only performed where +the error returned from the REST call matches a configurable regular expression retry condition. The default retry condition is `.*ECONN.*` which ensures +retries take place for common TCP errors such as `ECONNRESET` and `ECONNREFUSED`. + +The configurable retry settings are: + +- `RETRY_BACKOFF_FACTOR` (default `2`) +- `RETRY_BACKOFF_LIMIT_MS` (default `10000`) +- `RETRY_BACKOFF_INITIAL_MS` (default `100`) +- `RETRY_CONDITION` (default `.*ECONN.*`) +- `RETRY_MAX_ATTEMPTS` (default `15`) + +Setting `RETRY_CONDITION` to `""` disables retries. Setting `RETRY_MAX_ATTEMPTS` to `-1` causes it to retry indefinitely. diff --git a/src/main.ts b/src/main.ts index 2b7db0a..f5e9844 100644 --- a/src/main.ts +++ b/src/main.ts @@ -25,7 +25,7 @@ import { EventStreamReply } from './event-stream/event-stream.interfaces'; import { EventStreamService } from './event-stream/event-stream.service'; import { requestIDMiddleware } from './request-context/request-id.middleware'; import { RequestLoggingInterceptor } from './request-logging.interceptor'; -import { BlockchainConnectorService } from './tokens/blockchain.service'; +import { BlockchainConnectorService, RetryConfiguration } from './tokens/blockchain.service'; import { TokenApprovalEvent, TokenBurnEvent, @@ -84,12 +84,14 @@ async function bootstrap() { const legacyERC20 = config.get('USE_LEGACY_ERC20_SAMPLE', '').toLowerCase() === 'true'; const legacyERC721 = config.get('USE_LEGACY_ERC721_SAMPLE', '').toLowerCase() === 'true'; - // Configuration for retries - const retryBackOffFactor = config.get('RETRY_BACKOFF_FACTOR', 2); - const retryBackOffLimit = config.get('RETRY_BACKOFF_LIMIT_MS', 10000); - const retryBackOffInitial = config.get('RETRY_BACKOFF_INITIAL_MS', 100); - const retryCondition = config.get('RETRY_CONDITION', '.*ECONN.*'); - const retriesMax = config.get('RETRY_MAX_ATTEMPTS', 15); + // Configuration for blockchain call retries + const blockchainRetryCfg: RetryConfiguration = { + retryBackOffFactor: config.get('RETRY_BACKOFF_FACTOR', 2), + retryBackOffLimit: config.get('RETRY_BACKOFF_LIMIT_MS', 10000), + retryBackOffInitial: config.get('RETRY_BACKOFF_INITIAL_MS', 100), + retryCondition: config.get('RETRY_CONDITION', '.*ECONN.*'), + retriesMax: config.get('RETRY_MAX_ATTEMPTS', 15), + }; const passthroughHeaders: string[] = []; for (const h of passthroughHeaderString.split(',')) { @@ -100,18 +102,7 @@ async function bootstrap() { app.get(TokensService).configure(ethConnectUrl, topic, factoryAddress); app .get(BlockchainConnectorService) - .configure( - ethConnectUrl, - fftmUrl, - username, - password, - passthroughHeaders, - retryBackOffFactor, - retryBackOffLimit, - retryBackOffInitial, - retryCondition, - retriesMax, - ); + .configure(ethConnectUrl, fftmUrl, username, password, passthroughHeaders, blockchainRetryCfg); app.get(AbiMapperService).configure(legacyERC20, legacyERC721); if (autoInit.toLowerCase() !== 'false') { diff --git a/src/tokens/blockchain.service.ts b/src/tokens/blockchain.service.ts index fd811b8..7a70da5 100644 --- a/src/tokens/blockchain.service.ts +++ b/src/tokens/blockchain.service.ts @@ -30,6 +30,14 @@ import { Context } from '../request-context/request-context.decorator'; import { FFRequestIDHeader } from '../request-context/constants'; import { EthConnectAsyncResponse, EthConnectReturn, IAbiMethod } from './tokens.interfaces'; +export interface RetryConfiguration { + retryBackOffFactor: number; + retryBackOffLimit: number; + retryBackOffInitial: number; + retryCondition: string; + retriesMax: number; +} + const sendTransactionHeader = 'SendTransaction'; const queryHeader = 'Query'; @@ -43,11 +51,7 @@ export class BlockchainConnectorService { password: string; passthroughHeaders: string[]; - retryBackOffFactor: number; - retryBackOffLimit: number; - retryBackOffInitial: number; - retryCondition: string; - retriesMax: number; + retryConfiguration: RetryConfiguration; constructor(public http: HttpService) {} @@ -57,22 +61,14 @@ export class BlockchainConnectorService { username: string, password: string, passthroughHeaders: string[], - retryBackOffFactor: number, - retryBackOffLimit: number, - retryBackOffInitial: number, - retryCondition: string, - retriesMax: number, + retryConfiguration: RetryConfiguration, ) { this.baseUrl = baseUrl; this.fftmUrl = fftmUrl; this.username = username; this.password = password; this.passthroughHeaders = passthroughHeaders; - this.retryBackOffFactor = retryBackOffFactor; - this.retryBackOffLimit = retryBackOffLimit; - this.retryBackOffInitial = retryBackOffInitial; - this.retryCondition = retryCondition; - this.retriesMax = retriesMax; + this.retryConfiguration = retryConfiguration; } private requestOptions(ctx: Context): AxiosRequestConfig { @@ -106,29 +102,36 @@ export class BlockchainConnectorService { // Check if retry condition matches the err that's been hit private matchesRetryCondition(err: any): boolean { - return this.retryCondition != '' && err?.toString().match(this.retryCondition) !== null; + return ( + this.retryConfiguration.retryCondition != '' && + err?.toString().match(this.retryConfiguration.retryCondition) !== null + ); } // Delay by the appropriate amount of time given the iteration the caller is in private async backoffDelay(iteration: number) { const delay = Math.min( - this.retryBackOffInitial * Math.pow(this.retryBackOffFactor, iteration), - this.retryBackOffLimit, + this.retryConfiguration.retryBackOffInitial * + Math.pow(this.retryConfiguration.retryBackOffFactor, iteration), + this.retryConfiguration.retryBackOffLimit, ); await new Promise(resolve => setTimeout(resolve, delay)); } - // Generic helper function that makes an given blockchain function retryable - // by using synchronous, back-off delays for cases where the function returns + // Generic helper function that makes a given blockchain function retryable + // by using synchronous back-off delays for cases where the function returns // an error which matches the configured retry condition private async retryableCall( blockchainFunction: () => Promise>, ): Promise> { - let response: any; - for (let retries = 0; retries <= this.retriesMax; retries++) { + let retries = 0; + for ( + ; + this.retryConfiguration.retriesMax == -1 || retries <= this.retryConfiguration.retriesMax; + this.retryConfiguration.retriesMax == -1 || retries++ // Don't inc 'retries' if 'retriesMax' if set to -1 (infinite retries) + ) { try { - response = await blockchainFunction(); - break; + return await blockchainFunction(); } catch (e) { if (this.matchesRetryCondition(e)) { this.logger.debug(`Retry condition matched for error ${e}`); @@ -136,28 +139,28 @@ export class BlockchainConnectorService { await this.backoffDelay(retries); } else { // Whatever the error was it's not one we will retry for - break; + throw e; } } } - return response; + throw new InternalServerErrorException( + `Call to blockchain connector failed after ${retries} attempts`, + ); } async query(ctx: Context, to: string, method?: IAbiMethod, params?: any[]) { const url = this.baseUrl; - const response = await this.retryableCall( - async (): Promise> => { - return await this.wrapError( - lastValueFrom( - this.http.post( - url, - { headers: { type: queryHeader }, to, method, params }, - this.requestOptions(ctx), - ), + const response = await this.wrapError( + this.retryableCall(async (): Promise> => { + return await lastValueFrom( + this.http.post( + url, + { headers: { type: queryHeader }, to, method, params }, + this.requestOptions(ctx), ), ); - }, + }), ); return response.data; } @@ -172,35 +175,33 @@ export class BlockchainConnectorService { ) { const url = this.fftmUrl !== undefined && this.fftmUrl !== '' ? this.fftmUrl : this.baseUrl; - const response = await this.retryableCall( - async (): Promise> => { - return await this.wrapError( - lastValueFrom( + const response = await this.wrapError( + this.retryableCall( + async (): Promise> => { + return await lastValueFrom( this.http.post( url, { headers: { id, type: sendTransactionHeader }, from, to, method, params }, this.requestOptions(ctx), ), - ), - ); - }, + ); + }, + ), ); return response.data; } async getReceipt(ctx: Context, id: string): Promise { const url = this.baseUrl; - const response = await this.retryableCall( - async (): Promise> => { - return await this.wrapError( - lastValueFrom( - this.http.get(new URL(`/reply/${id}`, url).href, { - validateStatus: status => status < 300 || status === 404, - ...this.requestOptions(ctx), - }), - ), + const response = await this.wrapError( + this.retryableCall(async (): Promise> => { + return await lastValueFrom( + this.http.get(new URL(`/reply/${id}`, url).href, { + validateStatus: status => status < 300 || status === 404, + ...this.requestOptions(ctx), + }), ); - }, + }), ); if (response.status === 404) { throw new NotFoundException(); diff --git a/src/tokens/tokens.service.spec.ts b/src/tokens/tokens.service.spec.ts index 4de5ed8..bc090dc 100644 --- a/src/tokens/tokens.service.spec.ts +++ b/src/tokens/tokens.service.spec.ts @@ -33,7 +33,7 @@ import { import { EventStreamService } from '../event-stream/event-stream.service'; import { EventStreamProxyGateway } from '../eventstream-proxy/eventstream-proxy.gateway'; import { AbiMapperService } from './abimapper.service'; -import { BlockchainConnectorService } from './blockchain.service'; +import { BlockchainConnectorService, RetryConfiguration } from './blockchain.service'; import { AsyncResponse, EthConnectAsyncResponse, @@ -232,10 +232,18 @@ describe('TokensService', () => { .useValue(eventstream) .compile(); + let blockchainRetryCfg: RetryConfiguration = { + retryBackOffFactor: 2, + retryBackOffLimit: 10000, + retryBackOffInitial: 100, + retryCondition: '.*ECONN.*', + retriesMax: 15, + }; + service = module.get(TokensService); service.configure(BASE_URL, TOPIC, ''); blockchain = module.get(BlockchainConnectorService); - blockchain.configure(BASE_URL, '', '', '', [], 2, 1000, 250, '.*ECONN.*', 15); + blockchain.configure(BASE_URL, '', '', '', [], blockchainRetryCfg); }); it('should be defined', () => { From 79d012f8ad62dca9ad01b714d5d926b760dd1899 Mon Sep 17 00:00:00 2001 From: Matthew Whitehead Date: Tue, 28 Mar 2023 17:16:34 +0100 Subject: [PATCH 3/4] Add unit test for retries Signed-off-by: Matthew Whitehead --- src/tokens/tokens.service.spec.ts | 55 +++++++++++++++++++++++++++++-- test/app.e2e-context.ts | 12 +++++-- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/src/tokens/tokens.service.spec.ts b/src/tokens/tokens.service.spec.ts index bc090dc..7f11621 100644 --- a/src/tokens/tokens.service.spec.ts +++ b/src/tokens/tokens.service.spec.ts @@ -196,6 +196,14 @@ describe('TokensService', () => { ); }; + const mockECONNErrors = (count: number) => { + for (let i = 0; i < count; i++) { + http.post.mockImplementationOnce(() => { + throw new Error('connect ECONNREFUSED 10.1.2.3'); + }); + } + }; + beforeEach(async () => { http = { get: jest.fn(), @@ -234,8 +242,8 @@ describe('TokensService', () => { let blockchainRetryCfg: RetryConfiguration = { retryBackOffFactor: 2, - retryBackOffLimit: 10000, - retryBackOffInitial: 100, + retryBackOffLimit: 500, + retryBackOffInitial: 50, retryCondition: '.*ECONN.*', retriesMax: 15, }; @@ -1050,6 +1058,49 @@ describe('TokensService', () => { expect(http.post).toHaveBeenCalledWith(BASE_URL, mockEthConnectRequest, { headers }); }); + it('should mint ERC721WithData token with correct abi, custom uri, and inputs after 6 ECONNREFUSED retries', async () => { + const ctx = newContext(); + const headers = { + 'x-firefly-request-id': ctx.requestId, + }; + + const request: TokenMint = { + tokenIndex: '721', + signer: IDENTITY, + poolLocator: ERC721_WITH_DATA_V1_POOL_ID, + to: '0x123', + uri: 'ipfs://CID', + }; + + const response: EthConnectAsyncResponse = { + id: 'responseId', + sent: true, + }; + + const mockEthConnectRequest: EthConnectMsgRequest = { + headers: { + type: 'SendTransaction', + }, + from: IDENTITY, + to: CONTRACT_ADDRESS, + method: ERC721WithDataV1ABI.abi.find(abi => abi.name === MINT_WITH_URI) as IAbiMethod, + params: ['0x123', '721', '0x00', 'ipfs://CID'], + }; + + http.post.mockReturnValueOnce( + new FakeObservable({ + output: true, + }), + ); + mockECONNErrors(6); + http.post.mockReturnValueOnce(new FakeObservable(response)); + + await service.mint(ctx, request); + + expect(http.post).toHaveBeenCalledWith(BASE_URL, mockEthConnectRequest, { headers }); + expect(http.post).toHaveBeenCalledTimes(8); // Expect initial submit OK, 6 ECONN errors, final call OK = 8 POSTs + }); + it('should mint ERC721WithData token with correct abi, custom uri, auto-indexing, and inputs', async () => { const ctx = newContext(); const headers = { diff --git a/test/app.e2e-context.ts b/test/app.e2e-context.ts index 7ab24f7..cd59514 100644 --- a/test/app.e2e-context.ts +++ b/test/app.e2e-context.ts @@ -12,7 +12,7 @@ import { EventStreamService } from '../src/event-stream/event-stream.service'; import { EventStreamProxyGateway } from '../src/eventstream-proxy/eventstream-proxy.gateway'; import { TokensService } from '../src/tokens/tokens.service'; import { requestIDMiddleware } from '../src/request-context/request-id.middleware'; -import { BlockchainConnectorService } from '../src/tokens/blockchain.service'; +import { BlockchainConnectorService, RetryConfiguration } from '../src/tokens/blockchain.service'; export const BASE_URL = 'http://eth'; export const INSTANCE_PATH = '/tokens'; @@ -69,11 +69,19 @@ export class TestContext { this.app.use(requestIDMiddleware); await this.app.init(); + let blockchainRetryCfg: RetryConfiguration = { + retryBackOffFactor: 2, + retryBackOffLimit: 500, + retryBackOffInitial: 50, + retryCondition: '.*ECONN.*', + retriesMax: 15, + }; + this.app.get(EventStreamProxyGateway).configure('url', TOPIC); this.app.get(TokensService).configure(BASE_URL, TOPIC, ''); this.app .get(BlockchainConnectorService) - .configure(BASE_URL, '', '', '', [], 2, 1000, 250, '.*ECONN.*', 15); + .configure(BASE_URL, '', '', '', [], blockchainRetryCfg); (this.app.getHttpServer() as Server).listen(); this.server = request(this.app.getHttpServer()); From 0982c004cf72bbd31b74af39cff622f80342fb94 Mon Sep 17 00:00:00 2001 From: Matthew Whitehead Date: Wed, 29 Mar 2023 08:38:40 +0100 Subject: [PATCH 4/4] Final PR comment actions Signed-off-by: Matthew Whitehead --- README.md | 2 ++ src/tokens/blockchain.service.ts | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8f53aa9..c475302 100644 --- a/README.md +++ b/README.md @@ -204,3 +204,5 @@ The configurable retry settings are: - `RETRY_MAX_ATTEMPTS` (default `15`) Setting `RETRY_CONDITION` to `""` disables retries. Setting `RETRY_MAX_ATTEMPTS` to `-1` causes it to retry indefinitely. + +Note, the token connector will make a total of `RETRY_MAX_ATTEMPTS` + 1 calls for a given retryable call (1 original attempt and `RETRY_MAX_ATTEMPTS` retries) diff --git a/src/tokens/blockchain.service.ts b/src/tokens/blockchain.service.ts index 7a70da5..df174bf 100644 --- a/src/tokens/blockchain.service.ts +++ b/src/tokens/blockchain.service.ts @@ -104,7 +104,7 @@ export class BlockchainConnectorService { private matchesRetryCondition(err: any): boolean { return ( this.retryConfiguration.retryCondition != '' && - err?.toString().match(this.retryConfiguration.retryCondition) !== null + `${err}`.match(this.retryConfiguration.retryCondition) !== null ); } @@ -153,7 +153,7 @@ export class BlockchainConnectorService { const url = this.baseUrl; const response = await this.wrapError( this.retryableCall(async (): Promise> => { - return await lastValueFrom( + return lastValueFrom( this.http.post( url, { headers: { type: queryHeader }, to, method, params }, @@ -178,7 +178,7 @@ export class BlockchainConnectorService { const response = await this.wrapError( this.retryableCall( async (): Promise> => { - return await lastValueFrom( + return lastValueFrom( this.http.post( url, { headers: { id, type: sendTransactionHeader }, from, to, method, params }, @@ -195,7 +195,7 @@ export class BlockchainConnectorService { const url = this.baseUrl; const response = await this.wrapError( this.retryableCall(async (): Promise> => { - return await lastValueFrom( + return lastValueFrom( this.http.get(new URL(`/reply/${id}`, url).href, { validateStatus: status => status < 300 || status === 404, ...this.requestOptions(ctx),