Initial commit: Cloud Instances API
Multi-cloud VM instance database with Cloudflare Workers - Linode, Vultr, AWS connector integration - D1 database with regions, instances, pricing - Query API with filtering, caching, pagination - Cron-based auto-sync (daily + 6-hourly) - Health monitoring endpoint Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
kappa
481
src/connectors/aws.ts
Normal file
481
src/connectors/aws.ts
Normal file
@@ -0,0 +1,481 @@
|
||||
import type { RegionInput, InstanceTypeInput, InstanceFamily } from '../types';
|
||||
import { VaultClient, VaultError } from './vault';
|
||||
import { RateLimiter } from './base';
|
||||
|
||||
/**
|
||||
* AWS connector error class
|
||||
*/
|
||||
export class AWSError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public statusCode?: number,
|
||||
public details?: unknown
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'AWSError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* AWS region data structure
|
||||
*/
|
||||
interface AWSRegion {
|
||||
code: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* AWS instance type data from ec2.shop API
|
||||
*/
|
||||
interface AWSInstanceType {
|
||||
instance_type: string;
|
||||
memory: number; // GiB
|
||||
vcpus: number;
|
||||
storage: string;
|
||||
network: string;
|
||||
price?: number;
|
||||
region?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* AWS EC2 Connector
|
||||
*
|
||||
* Features:
|
||||
* - Uses public ec2.shop API for instance type data
|
||||
* - No authentication required for basic data
|
||||
* - Rate limiting: 20 requests/second
|
||||
* - Hardcoded region list (relatively static)
|
||||
* - Comprehensive error handling
|
||||
*
|
||||
* @example
|
||||
* const vault = new VaultClient(vaultUrl, vaultToken);
|
||||
* const connector = new AWSConnector(vault);
|
||||
* await connector.initialize();
|
||||
* const regions = await connector.fetchRegions();
|
||||
*/
|
||||
export class AWSConnector {
|
||||
readonly provider = 'aws';
|
||||
private readonly instanceDataUrl = 'https://ec2.shop/instances.json';
|
||||
private readonly rateLimiter: RateLimiter;
|
||||
private readonly requestTimeout = 15000; // 15 seconds
|
||||
|
||||
/**
|
||||
* AWS regions list (relatively static data)
|
||||
* Based on AWS public region information
|
||||
*/
|
||||
private readonly awsRegions: AWSRegion[] = [
|
||||
{ code: 'us-east-1', name: 'US East (N. Virginia)' },
|
||||
{ code: 'us-east-2', name: 'US East (Ohio)' },
|
||||
{ code: 'us-west-1', name: 'US West (N. California)' },
|
||||
{ code: 'us-west-2', name: 'US West (Oregon)' },
|
||||
{ code: 'eu-west-1', name: 'EU (Ireland)' },
|
||||
{ code: 'eu-west-2', name: 'EU (London)' },
|
||||
{ code: 'eu-west-3', name: 'EU (Paris)' },
|
||||
{ code: 'eu-central-1', name: 'EU (Frankfurt)' },
|
||||
{ code: 'eu-central-2', name: 'EU (Zurich)' },
|
||||
{ code: 'eu-north-1', name: 'EU (Stockholm)' },
|
||||
{ code: 'eu-south-1', name: 'EU (Milan)' },
|
||||
{ code: 'eu-south-2', name: 'EU (Spain)' },
|
||||
{ code: 'ap-northeast-1', name: 'Asia Pacific (Tokyo)' },
|
||||
{ code: 'ap-northeast-2', name: 'Asia Pacific (Seoul)' },
|
||||
{ code: 'ap-northeast-3', name: 'Asia Pacific (Osaka)' },
|
||||
{ code: 'ap-southeast-1', name: 'Asia Pacific (Singapore)' },
|
||||
{ code: 'ap-southeast-2', name: 'Asia Pacific (Sydney)' },
|
||||
{ code: 'ap-southeast-3', name: 'Asia Pacific (Jakarta)' },
|
||||
{ code: 'ap-southeast-4', name: 'Asia Pacific (Melbourne)' },
|
||||
{ code: 'ap-south-1', name: 'Asia Pacific (Mumbai)' },
|
||||
{ code: 'ap-south-2', name: 'Asia Pacific (Hyderabad)' },
|
||||
{ code: 'ap-east-1', name: 'Asia Pacific (Hong Kong)' },
|
||||
{ code: 'ca-central-1', name: 'Canada (Central)' },
|
||||
{ code: 'ca-west-1', name: 'Canada (Calgary)' },
|
||||
{ code: 'sa-east-1', name: 'South America (São Paulo)' },
|
||||
{ code: 'af-south-1', name: 'Africa (Cape Town)' },
|
||||
{ code: 'me-south-1', name: 'Middle East (Bahrain)' },
|
||||
{ code: 'me-central-1', name: 'Middle East (UAE)' },
|
||||
{ code: 'il-central-1', name: 'Israel (Tel Aviv)' },
|
||||
];
|
||||
|
||||
constructor(private vaultClient: VaultClient) {
|
||||
// Rate limit: 20 requests/second per region
|
||||
// Use 10 tokens with 10/second refill to be conservative
|
||||
this.rateLimiter = new RateLimiter(20, 10);
|
||||
console.log('[AWSConnector] Initialized');
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize connector by fetching credentials from Vault
|
||||
* Note: Currently not required for public API access,
|
||||
* but included for future AWS API integration
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
console.log('[AWSConnector] Fetching credentials from Vault');
|
||||
|
||||
try {
|
||||
const credentials = await this.vaultClient.getCredentials(this.provider);
|
||||
|
||||
// AWS uses different credential keys
|
||||
const awsCreds = credentials as unknown as {
|
||||
aws_access_key_id?: string;
|
||||
aws_secret_access_key?: string;
|
||||
};
|
||||
|
||||
// Credentials loaded for future AWS API direct access
|
||||
console.log('[AWSConnector] Credentials loaded successfully', {
|
||||
hasAccessKey: !!awsCreds.aws_access_key_id,
|
||||
hasSecretKey: !!awsCreds.aws_secret_access_key,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof VaultError) {
|
||||
console.warn('[AWSConnector] Vault credentials not available, using public API only');
|
||||
// Not critical for public API access
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all regions
|
||||
* Returns hardcoded region list as AWS regions are relatively static
|
||||
*
|
||||
* @returns Array of AWS regions
|
||||
*/
|
||||
async fetchRegions(): Promise<AWSRegion[]> {
|
||||
console.log('[AWSConnector] Fetching regions', { count: this.awsRegions.length });
|
||||
return this.awsRegions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all instance types from ec2.shop API
|
||||
*
|
||||
* @returns Array of AWS instance types
|
||||
* @throws AWSError on API failures
|
||||
*/
|
||||
async fetchInstanceTypes(): Promise<AWSInstanceType[]> {
|
||||
console.log('[AWSConnector] Fetching instance types from ec2.shop');
|
||||
|
||||
await this.rateLimiter.waitForToken();
|
||||
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), this.requestTimeout);
|
||||
|
||||
const response = await fetch(this.instanceDataUrl, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new AWSError(
|
||||
`Failed to fetch instance types: ${response.statusText}`,
|
||||
response.status
|
||||
);
|
||||
}
|
||||
|
||||
const data = await response.json() as AWSInstanceType[];
|
||||
|
||||
console.log('[AWSConnector] Instance types fetched', { count: data.length });
|
||||
return data;
|
||||
|
||||
} catch (error) {
|
||||
// Handle timeout
|
||||
if (error instanceof Error && error.name === 'AbortError') {
|
||||
console.error('[AWSConnector] Request timeout', { timeout: this.requestTimeout });
|
||||
throw new AWSError(
|
||||
`Request to ec2.shop API timed out after ${this.requestTimeout}ms`,
|
||||
504
|
||||
);
|
||||
}
|
||||
|
||||
// Re-throw AWSError
|
||||
if (error instanceof AWSError) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Handle unexpected errors
|
||||
console.error('[AWSConnector] Unexpected error', { error });
|
||||
throw new AWSError(
|
||||
`Failed to fetch instance types: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
500,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize AWS region data for database storage
|
||||
*
|
||||
* @param raw - Raw AWS region data
|
||||
* @param providerId - Database provider ID
|
||||
* @returns Normalized region data ready for insertion
|
||||
*/
|
||||
normalizeRegion(raw: AWSRegion, providerId: number): RegionInput {
|
||||
return {
|
||||
provider_id: providerId,
|
||||
region_code: raw.code,
|
||||
region_name: raw.name,
|
||||
country_code: this.extractCountryCode(raw.code),
|
||||
latitude: null, // AWS doesn't provide coordinates in basic data
|
||||
longitude: null,
|
||||
available: 1, // All listed regions are available
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize AWS instance type data for database storage
|
||||
*
|
||||
* @param raw - Raw AWS instance type data
|
||||
* @param providerId - Database provider ID
|
||||
* @returns Normalized instance type data ready for insertion
|
||||
*/
|
||||
normalizeInstance(raw: AWSInstanceType, providerId: number): InstanceTypeInput {
|
||||
// Convert memory from GiB to MB
|
||||
const memoryMb = Math.round(raw.memory * 1024);
|
||||
|
||||
// Parse storage information
|
||||
const storageGb = this.parseStorage(raw.storage);
|
||||
|
||||
// Parse GPU information from instance type name
|
||||
const { gpuCount, gpuType } = this.parseGpuInfo(raw.instance_type);
|
||||
|
||||
return {
|
||||
provider_id: providerId,
|
||||
instance_id: raw.instance_type,
|
||||
instance_name: raw.instance_type,
|
||||
vcpu: raw.vcpus,
|
||||
memory_mb: memoryMb,
|
||||
storage_gb: storageGb,
|
||||
transfer_tb: null, // ec2.shop doesn't provide transfer limits
|
||||
network_speed_gbps: this.parseNetworkSpeed(raw.network),
|
||||
gpu_count: gpuCount,
|
||||
gpu_type: gpuType,
|
||||
instance_family: this.mapInstanceFamily(raw.instance_type),
|
||||
metadata: JSON.stringify({
|
||||
storage_type: raw.storage,
|
||||
network: raw.network,
|
||||
price: raw.price,
|
||||
region: raw.region,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract country code from AWS region code
|
||||
*
|
||||
* @param regionCode - AWS region code (e.g., 'us-east-1')
|
||||
* @returns Lowercase ISO alpha-2 country code or null
|
||||
*/
|
||||
private extractCountryCode(regionCode: string): string | null {
|
||||
const countryMap: Record<string, string> = {
|
||||
'us': 'us',
|
||||
'eu': 'eu',
|
||||
'ap': 'ap',
|
||||
'ca': 'ca',
|
||||
'sa': 'br',
|
||||
'af': 'za',
|
||||
'me': 'ae',
|
||||
'il': 'il',
|
||||
};
|
||||
|
||||
const prefix = regionCode.split('-')[0];
|
||||
return countryMap[prefix] || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse storage information from AWS storage string
|
||||
*
|
||||
* @param storage - AWS storage string (e.g., "EBS only", "1 x 900 NVMe SSD")
|
||||
* @returns Storage size in GB or 0 if EBS only
|
||||
*/
|
||||
private parseStorage(storage: string): number {
|
||||
if (!storage || storage.toLowerCase().includes('ebs only')) {
|
||||
return 0; // EBS only instances have no instance storage
|
||||
}
|
||||
|
||||
// Parse format like "1 x 900 NVMe SSD" or "2 x 1900 NVMe SSD"
|
||||
const match = storage.match(/(\d+)\s*x\s*(\d+)/);
|
||||
if (match) {
|
||||
const count = parseInt(match[1], 10);
|
||||
const sizePerDisk = parseInt(match[2], 10);
|
||||
return count * sizePerDisk;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse network speed from AWS network string
|
||||
*
|
||||
* @param network - AWS network string (e.g., "Up to 5 Gigabit", "25 Gigabit")
|
||||
* @returns Network speed in Gbps or null
|
||||
*/
|
||||
private parseNetworkSpeed(network: string): number | null {
|
||||
if (!network) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const match = network.match(/(\d+)\s*Gigabit/i);
|
||||
if (match) {
|
||||
return parseInt(match[1], 10);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse GPU information from instance type name
|
||||
*
|
||||
* @param instanceType - AWS instance type name
|
||||
* @returns GPU count and type
|
||||
*/
|
||||
private parseGpuInfo(instanceType: string): { gpuCount: number; gpuType: string | null } {
|
||||
const typeLower = instanceType.toLowerCase();
|
||||
|
||||
// GPU instance families
|
||||
if (typeLower.startsWith('p2.')) {
|
||||
return { gpuCount: this.getGpuCount(instanceType, 'p2'), gpuType: 'NVIDIA K80' };
|
||||
}
|
||||
if (typeLower.startsWith('p3.')) {
|
||||
return { gpuCount: this.getGpuCount(instanceType, 'p3'), gpuType: 'NVIDIA V100' };
|
||||
}
|
||||
if (typeLower.startsWith('p4.')) {
|
||||
return { gpuCount: this.getGpuCount(instanceType, 'p4'), gpuType: 'NVIDIA A100' };
|
||||
}
|
||||
if (typeLower.startsWith('p5.')) {
|
||||
return { gpuCount: this.getGpuCount(instanceType, 'p5'), gpuType: 'NVIDIA H100' };
|
||||
}
|
||||
if (typeLower.startsWith('g3.')) {
|
||||
return { gpuCount: this.getGpuCount(instanceType, 'g3'), gpuType: 'NVIDIA M60' };
|
||||
}
|
||||
if (typeLower.startsWith('g4.')) {
|
||||
return { gpuCount: this.getGpuCount(instanceType, 'g4'), gpuType: 'NVIDIA T4' };
|
||||
}
|
||||
if (typeLower.startsWith('g5.')) {
|
||||
return { gpuCount: this.getGpuCount(instanceType, 'g5'), gpuType: 'NVIDIA A10G' };
|
||||
}
|
||||
if (typeLower.startsWith('inf')) {
|
||||
return { gpuCount: this.getInferentiaCount(instanceType), gpuType: 'AWS Inferentia' };
|
||||
}
|
||||
if (typeLower.startsWith('trn')) {
|
||||
return { gpuCount: this.getTrainiumCount(instanceType), gpuType: 'AWS Trainium' };
|
||||
}
|
||||
|
||||
return { gpuCount: 0, gpuType: null };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get GPU count based on instance size
|
||||
*
|
||||
* @param instanceType - Full instance type name
|
||||
* @param family - Instance family prefix
|
||||
* @returns Number of GPUs
|
||||
*/
|
||||
private getGpuCount(instanceType: string, _family: string): number {
|
||||
const size = instanceType.split('.')[1];
|
||||
|
||||
// Common GPU counts by size
|
||||
const gpuMap: Record<string, number> = {
|
||||
'xlarge': 1,
|
||||
'2xlarge': 1,
|
||||
'4xlarge': 2,
|
||||
'8xlarge': 4,
|
||||
'16xlarge': 8,
|
||||
'24xlarge': 8,
|
||||
'48xlarge': 8,
|
||||
};
|
||||
|
||||
return gpuMap[size] || 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Inferentia accelerator count
|
||||
*
|
||||
* @param instanceType - Full instance type name
|
||||
* @returns Number of Inferentia chips
|
||||
*/
|
||||
private getInferentiaCount(instanceType: string): number {
|
||||
const size = instanceType.split('.')[1];
|
||||
|
||||
const infMap: Record<string, number> = {
|
||||
'xlarge': 1,
|
||||
'2xlarge': 1,
|
||||
'6xlarge': 4,
|
||||
'24xlarge': 16,
|
||||
};
|
||||
|
||||
return infMap[size] || 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Trainium accelerator count
|
||||
*
|
||||
* @param instanceType - Full instance type name
|
||||
* @returns Number of Trainium chips
|
||||
*/
|
||||
private getTrainiumCount(instanceType: string): number {
|
||||
const size = instanceType.split('.')[1];
|
||||
|
||||
const trnMap: Record<string, number> = {
|
||||
'2xlarge': 1,
|
||||
'32xlarge': 16,
|
||||
};
|
||||
|
||||
return trnMap[size] || 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map AWS instance type to standard instance family
|
||||
*
|
||||
* @param instanceType - AWS instance type name
|
||||
* @returns Standard instance family type
|
||||
*/
|
||||
private mapInstanceFamily(instanceType: string): InstanceFamily {
|
||||
const family = instanceType.split('.')[0].toLowerCase();
|
||||
|
||||
// General purpose
|
||||
if (family.match(/^[tm]\d+[a-z]?$/)) {
|
||||
return 'general';
|
||||
}
|
||||
if (family.match(/^a\d+$/)) {
|
||||
return 'general';
|
||||
}
|
||||
|
||||
// Compute optimized
|
||||
if (family.match(/^c\d+[a-z]?$/)) {
|
||||
return 'compute';
|
||||
}
|
||||
|
||||
// Memory optimized
|
||||
if (family.match(/^[rx]\d+[a-z]?$/)) {
|
||||
return 'memory';
|
||||
}
|
||||
if (family.match(/^u-\d+/)) {
|
||||
return 'memory';
|
||||
}
|
||||
if (family.match(/^z\d+[a-z]?$/)) {
|
||||
return 'memory';
|
||||
}
|
||||
|
||||
// Storage optimized
|
||||
if (family.match(/^[dhi]\d+[a-z]?$/)) {
|
||||
return 'storage';
|
||||
}
|
||||
|
||||
// GPU/accelerated computing
|
||||
if (family.match(/^[pg]\d+[a-z]?$/)) {
|
||||
return 'gpu';
|
||||
}
|
||||
if (family.match(/^(inf|trn|dl)\d*/)) {
|
||||
return 'gpu';
|
||||
}
|
||||
|
||||
// Default to general for unknown types
|
||||
console.warn('[AWSConnector] Unknown instance family, defaulting to general', { type: instanceType });
|
||||
return 'general';
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user