From e41db8805120953be31894bcdcd04dcd700677fb Mon Sep 17 00:00:00 2001 From: Jong-hoon Oh Date: Mon, 7 Dec 2020 18:28:46 +0900 Subject: [PATCH] fix: handle RequestLimitExceeded error --- src/lib/core.spec.ts | 44 ++++ src/lib/core.ts | 76 ++++-- test/__snapshots__/lib/core.spec.ts.snap | 284 +++++++++++++++++++++++ test/mock-ec2-endpoints.ts | 30 ++- 4 files changed, 414 insertions(+), 20 deletions(-) diff --git a/src/lib/core.spec.ts b/src/lib/core.spec.ts index 06a30f34..a2ca595c 100644 --- a/src/lib/core.spec.ts +++ b/src/lib/core.spec.ts @@ -1,6 +1,7 @@ import mockConsole, { RestoreConsole } from 'jest-mock-console'; import { filter } from 'lodash'; import nock from 'nock'; +import { stdout } from 'process'; import { mockAwsCredentials, mockAwsCredentialsClear } from '../../test/mock-credential-endpoints'; import { @@ -260,6 +261,27 @@ describe('lib', () => { }); }); + describe('should handle RequestLimitExceeded error', () => { + const region = 'us-east-1'; + let results: SpotPriceExtended[]; + let restoreConsole: RestoreConsole; + + beforeAll(async () => { + restoreConsole = mockConsole(); + mockDefaultRegionEndpoints({ returnRequestLimitExceededErrorCount: 10 }); + results = await getGlobalSpotPrices({ regions: [region] }); + }); + + afterAll(() => { + restoreConsole(); + mockDefaultRegionEndpointsClear(); + }); + + it('should return expected values', async () => { + expect(results).toMatchSnapshot(); + }); + }); + describe('should fetch ec2 instance type info dynamically if not found from constants', () => { let results: SpotPriceExtended[]; let restoreConsole: RestoreConsole; @@ -346,5 +368,27 @@ describe('lib', () => { expect(results).toMatchSnapshot(); }); }); + + describe('should handle RequestLimitExceeded error', () => { + let results: GetEc2InfoResults; + + beforeAll(async () => { + mockDefaultRegionEndpoints({ + returnRequestLimitExceededErrorCount: 10, + maxLength: 5, + returnPartialBlankValues: true, + }); + results = await getEc2Info({ InstanceTypes: ['dummy.large'] }); + }); + + afterAll(() => { + mockDefaultRegionEndpointsClear(); + }); + + it('should return expected values', () => { + expect(Object.keys(results).length).toEqual(1); + expect(results).toMatchSnapshot(); + }); + }); }); }); diff --git a/src/lib/core.ts b/src/lib/core.ts index b34e9c32..72c1b5e7 100644 --- a/src/lib/core.ts +++ b/src/lib/core.ts @@ -1,4 +1,6 @@ import EC2 from 'aws-sdk/clients/ec2'; +import { AWSError } from 'aws-sdk/lib/error'; +import { PromiseResult } from 'aws-sdk/lib/request'; import { ec2Info, Ec2InstanceInfo } from '../constants/ec2-info'; import { InstanceFamilyType, InstanceSize, InstanceType } from '../constants/ec2-types'; @@ -82,18 +84,36 @@ const getEc2SpotPrice = async (options: { secretAccessKey, }); + const startTime = new Date(); + startTime.setHours(startTime.getHours() - 3); + const fetch = async (nextToken?: string): Promise => { - const startTime = new Date(); - startTime.setHours(startTime.getHours() - 3); - - const result = await ec2 - .describeSpotPriceHistory({ - NextToken: nextToken, - StartTime: startTime, - ProductDescriptions: platforms, - InstanceTypes: instanceTypes, - }) - .promise(); + let retryMS = 0; + + const describeSpotPriceHistory = async (): Promise< + PromiseResult + > => { + try { + return await ec2 + .describeSpotPriceHistory({ + NextToken: nextToken, + StartTime: startTime, + ProductDescriptions: platforms, + InstanceTypes: instanceTypes, + }) + .promise(); + } catch (error) { + if (error.code === 'RequestLimitExceeded') { + retryMS = retryMS ? retryMS * 2 : 200; + await new Promise(res => setTimeout(res, retryMS)); + return describeSpotPriceHistory(); + } else { + throw error; + } + } + }; + + const result = await describeSpotPriceHistory(); const nextList = result.NextToken ? await fetch(result.NextToken) : []; @@ -144,14 +164,34 @@ export const getEc2Info = async ({ const ec2 = new EC2({ region }); const fetchInfo = async (NextToken?: string): Promise => { + let retryMS = 0; + const rtn: Ec2InstanceInfos = {}; - const res = await ec2 - .describeInstanceTypes({ - NextToken, - MaxResults: InstanceTypes ? undefined : 100, - InstanceTypes, - }) - .promise(); + + const describeInstanceTypes = async (): Promise< + PromiseResult + > => { + try { + return await ec2 + .describeInstanceTypes({ + NextToken, + MaxResults: InstanceTypes ? undefined : 100, + InstanceTypes, + }) + .promise(); + } catch (error) { + if (error.code === 'RequestLimitExceeded') { + retryMS = retryMS ? retryMS * 2 : 200; + await new Promise(res => setTimeout(res, retryMS)); + return describeInstanceTypes(); + } else { + throw error; + } + } + }; + + const res = await describeInstanceTypes(); + res.InstanceTypes?.forEach(i => { if (i.InstanceType) { rtn[i.InstanceType] = { diff --git a/test/__snapshots__/lib/core.spec.ts.snap b/test/__snapshots__/lib/core.spec.ts.snap index 71151850..901b2382 100644 --- a/test/__snapshots__/lib/core.spec.ts.snap +++ b/test/__snapshots__/lib/core.spec.ts.snap @@ -1378,6 +1378,15 @@ Object { } `; +exports[`lib getEc2Info should handle RequestLimitExceeded error should return expected values 1`] = ` +Object { + "dummy.large": Object { + "memoryGiB": 16, + "vCpu": 2, + }, +} +`; + exports[`lib getGlobalSpotPrices run with default options should return expected values 1`] = ` Array [ Object { @@ -1887,3 +1896,278 @@ Array [ }, ] `; + +exports[`lib getGlobalSpotPrices should handle RequestLimitExceeded error should return expected values 1`] = ` +Array [ + Object { + "availabilityZone": "us-east-1a", + "instanceType": "t3.nano", + "memoryGiB": 0.5, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.0016, + "timestamp": 2019-10-15T17:09:43.000Z, + "vCpu": 2, + }, + Object { + "availabilityZone": "us-east-1b", + "instanceType": "t3.nano", + "memoryGiB": 0.5, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.0016, + "timestamp": 2019-10-15T17:09:43.000Z, + "vCpu": 2, + }, + Object { + "availabilityZone": "us-east-1c", + "instanceType": "t3.nano", + "memoryGiB": 0.5, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.0016, + "timestamp": 2019-10-15T17:09:43.000Z, + "vCpu": 2, + }, + Object { + "availabilityZone": "us-east-1d", + "instanceType": "t3.nano", + "memoryGiB": 0.5, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.0016, + "timestamp": 2019-10-15T17:09:43.000Z, + "vCpu": 2, + }, + Object { + "availabilityZone": "us-east-1f", + "instanceType": "t3.nano", + "memoryGiB": 0.5, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.0016, + "timestamp": 2019-10-15T17:09:43.000Z, + "vCpu": 2, + }, + Object { + "availabilityZone": "us-east-1a", + "instanceType": "t3a.nano", + "memoryGiB": 0.5, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.0017, + "timestamp": 2019-10-15T22:11:29.000Z, + "vCpu": 2, + }, + Object { + "availabilityZone": "us-east-1c", + "instanceType": "t3a.nano", + "memoryGiB": 0.5, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.0017, + "timestamp": 2019-10-15T22:11:29.000Z, + "vCpu": 2, + }, + Object { + "availabilityZone": "us-east-1b", + "instanceType": "t3a.nano", + "memoryGiB": 0.5, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.0018, + "timestamp": 2019-10-15T22:11:29.000Z, + "vCpu": 2, + }, + Object { + "availabilityZone": "us-east-1d", + "instanceType": "t3a.nano", + "memoryGiB": 0.5, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.0018, + "timestamp": 2019-10-15T20:21:10.000Z, + "vCpu": 2, + }, + Object { + "availabilityZone": "us-east-1f", + "instanceType": "t3a.nano", + "memoryGiB": 0.5, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.0018, + "timestamp": 2019-10-15T22:11:29.000Z, + "vCpu": 2, + }, + Object { + "availabilityZone": "us-east-1a", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Linux/UNIX", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:11:12.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1a", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:11:25.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1a", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Windows", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:12:13.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1a", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Windows (Amazon VPC)", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:12:16.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1b", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Linux/UNIX", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:11:12.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1b", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:11:25.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1b", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Windows", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:12:13.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1b", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Windows (Amazon VPC)", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:12:16.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1c", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Linux/UNIX", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:11:12.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1c", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:11:25.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1c", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Windows", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:12:13.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1c", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Windows (Amazon VPC)", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:12:16.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1d", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Linux/UNIX", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:11:12.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1d", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:11:25.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1d", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Windows", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:12:13.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1d", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Windows (Amazon VPC)", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:12:16.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1f", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:11:25.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1f", + "instanceType": "t1.micro", + "memoryGiB": 0.613, + "platform": "Windows (Amazon VPC)", + "spotPrice": 0.002, + "timestamp": 2019-10-15T13:12:16.000Z, + "vCpu": 1, + }, + Object { + "availabilityZone": "us-east-1a", + "instanceType": "t3a.micro", + "memoryGiB": 1, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.0028, + "timestamp": 2019-10-16T00:11:49.000Z, + "vCpu": 2, + }, + Object { + "availabilityZone": "us-east-1c", + "instanceType": "t3a.micro", + "memoryGiB": 1, + "platform": "Linux/UNIX (Amazon VPC)", + "spotPrice": 0.0028, + "timestamp": 2019-10-16T00:11:49.000Z, + "vCpu": 2, + }, +] +`; diff --git a/test/mock-ec2-endpoints.ts b/test/mock-ec2-endpoints.ts index 5d34f577..73d268b4 100644 --- a/test/mock-ec2-endpoints.ts +++ b/test/mock-ec2-endpoints.ts @@ -30,13 +30,31 @@ const nockEndpoint = (options: { region: Region; maxLength?: number; returnPartialBlankValues?: boolean; + returnRequestLimitExceededErrorCount?: number; }): void => { const { region, returnPartialBlankValues, maxLength } = options; + let { returnRequestLimitExceededErrorCount } = options; nock(`https://ec2.${region}.amazonaws.com`) .persist() .post('/') .reply((uri, body) => { + if (returnRequestLimitExceededErrorCount) { + returnRequestLimitExceededErrorCount -= 1; + return [ + 503, + ` + + + RequestLimitExceeded + Request limit exceeded. + + + RequestLimitExceededRequestID + `, + ]; + } + const params = parse(body as string); if (params.Action === 'DescribeSpotPriceHistory') { @@ -144,11 +162,19 @@ export const mockDefaultRegionEndpoints = ( options: { maxLength?: number; returnPartialBlankValues?: boolean; + returnRequestLimitExceededErrorCount?: number; } = {}, ): void => { - const { maxLength, returnPartialBlankValues } = options; + const { maxLength, returnPartialBlankValues, returnRequestLimitExceededErrorCount } = options; mockAwsCredentials(); - defaultRegions.forEach(region => nockEndpoint({ region, maxLength, returnPartialBlankValues })); + defaultRegions.forEach(region => + nockEndpoint({ + region, + maxLength, + returnPartialBlankValues, + returnRequestLimitExceededErrorCount, + }), + ); }; export const mockDefaultRegionEndpointsClear = (): void => {