feat: KRW 가격 지원 및 GPU/G8/VPU 인스턴스 추가
## KRW 가격 기능 - pricing 테이블에 hourly_price_krw, monthly_price_krw 컬럼 추가 - 부가세 10% + 영업이익 10% + 환율 적용 (기본 1450원) - 시간당: 1원 단위 반올림 (최소 1원) - 월간: 100원 단위 반올림 (최소 100원) - 환율/부가세/영업이익률 환경변수로 분리 (배포 없이 변경 가능) ## GPU/G8/VPU 인스턴스 지원 - gpu_instances, gpu_pricing 테이블 추가 - g8_instances, g8_pricing 테이블 추가 - vpu_instances, vpu_pricing 테이블 추가 - Linode/Vultr 커넥터에 GPU 동기화 로직 추가 ## 환경변수 추가 - KRW_EXCHANGE_RATE: 환율 (기본 1450) - KRW_VAT_RATE: 부가세율 (기본 1.1) - KRW_MARKUP_RATE: 영업이익률 (기본 1.1) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
kappa
@@ -25,6 +25,8 @@ import type {
|
||||
RegionInput,
|
||||
InstanceTypeInput,
|
||||
PricingInput,
|
||||
GpuInstanceInput,
|
||||
GpuPricingInput,
|
||||
} from '../types';
|
||||
import { SyncStage } from '../types';
|
||||
|
||||
@@ -46,17 +48,32 @@ export interface SyncConnectorAdapter {
|
||||
/** Fetch all instance types (normalized) */
|
||||
getInstanceTypes(): Promise<InstanceTypeInput[]>;
|
||||
|
||||
/** Fetch GPU instances (optional, only for providers with GPU support) */
|
||||
getGpuInstances?(): Promise<GpuInstanceInput[]>;
|
||||
|
||||
/** Fetch G8 instances (optional, only for Linode) */
|
||||
getG8Instances?(): Promise<any[]>;
|
||||
|
||||
/** Fetch VPU instances (optional, only for Linode) */
|
||||
getVpuInstances?(): Promise<any[]>;
|
||||
|
||||
/**
|
||||
* Fetch pricing data for instances and regions
|
||||
* @param instanceTypeIds - Array of database instance type IDs
|
||||
* @param regionIds - Array of database region IDs
|
||||
* @param dbInstanceMap - Map of DB instance type ID to instance_id (API ID) for avoiding redundant queries
|
||||
* @param dbGpuMap - Map of GPU instance IDs (optional)
|
||||
* @param dbG8Map - Map of G8 instance IDs (optional)
|
||||
* @param dbVpuMap - Map of VPU instance IDs (optional)
|
||||
* @returns Array of pricing records OR number of records if batched internally
|
||||
*/
|
||||
getPricing(
|
||||
instanceTypeIds: number[],
|
||||
regionIds: number[],
|
||||
dbInstanceMap: Map<number, { instance_id: string }>
|
||||
dbInstanceMap: Map<number, { instance_id: string }>,
|
||||
dbGpuMap?: Map<number, { instance_id: string }>,
|
||||
dbG8Map?: Map<number, { instance_id: string }>,
|
||||
dbVpuMap?: Map<number, { instance_id: string }>
|
||||
): Promise<PricingInput[] | number>;
|
||||
}
|
||||
|
||||
@@ -73,7 +90,7 @@ export class SyncOrchestrator {
|
||||
private vault: VaultClient,
|
||||
env?: Env
|
||||
) {
|
||||
this.repos = new RepositoryFactory(db);
|
||||
this.repos = new RepositoryFactory(db, env);
|
||||
this.env = env;
|
||||
this.logger = createLogger('[SyncOrchestrator]', env);
|
||||
this.logger.info('Initialized');
|
||||
@@ -138,17 +155,79 @@ export class SyncOrchestrator {
|
||||
providerRecord.id,
|
||||
normalizedRegions
|
||||
);
|
||||
const instancesCount = await this.repos.instances.upsertMany(
|
||||
|
||||
// Persist regular instances (already filtered in getInstanceTypes)
|
||||
const regularInstancesCount = await this.repos.instances.upsertMany(
|
||||
providerRecord.id,
|
||||
normalizedInstances
|
||||
);
|
||||
|
||||
// Handle specialized instances separately for Linode and Vultr
|
||||
let gpuInstancesCount = 0;
|
||||
let g8InstancesCount = 0;
|
||||
let vpuInstancesCount = 0;
|
||||
|
||||
if (provider.toLowerCase() === 'linode') {
|
||||
// GPU instances
|
||||
if ('getGpuInstances' in connector) {
|
||||
const gpuInstances = await (connector as any).getGpuInstances();
|
||||
if (gpuInstances && gpuInstances.length > 0) {
|
||||
gpuInstancesCount = await this.repos.gpuInstances.upsertMany(
|
||||
providerRecord.id,
|
||||
gpuInstances
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// G8 instances
|
||||
if ('getG8Instances' in connector) {
|
||||
const g8Instances = await (connector as any).getG8Instances();
|
||||
if (g8Instances && g8Instances.length > 0) {
|
||||
g8InstancesCount = await this.repos.g8Instances.upsertMany(
|
||||
providerRecord.id,
|
||||
g8Instances
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// VPU instances
|
||||
if ('getVpuInstances' in connector) {
|
||||
const vpuInstances = await (connector as any).getVpuInstances();
|
||||
if (vpuInstances && vpuInstances.length > 0) {
|
||||
vpuInstancesCount = await this.repos.vpuInstances.upsertMany(
|
||||
providerRecord.id,
|
||||
vpuInstances
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle Vultr GPU instances
|
||||
if (provider.toLowerCase() === 'vultr') {
|
||||
if ('getGpuInstances' in connector) {
|
||||
const gpuInstances = await (connector as any).getGpuInstances();
|
||||
if (gpuInstances && gpuInstances.length > 0) {
|
||||
gpuInstancesCount = await this.repos.gpuInstances.upsertMany(
|
||||
providerRecord.id,
|
||||
gpuInstances
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const instancesCount = regularInstancesCount + gpuInstancesCount + g8InstancesCount + vpuInstancesCount;
|
||||
|
||||
// Fetch pricing data - need instance and region IDs from DB
|
||||
// Use D1 batch to reduce query count from 2 to 1 (50% reduction in queries)
|
||||
const [dbRegionsResult, dbInstancesResult] = await this.repos.db.batch([
|
||||
// Use D1 batch to reduce query count (fetch all instance types in one batch)
|
||||
const batchQueries = [
|
||||
this.repos.db.prepare('SELECT id, region_code FROM regions WHERE provider_id = ?').bind(providerRecord.id),
|
||||
this.repos.db.prepare('SELECT id, instance_id FROM instance_types WHERE provider_id = ?').bind(providerRecord.id)
|
||||
]);
|
||||
this.repos.db.prepare('SELECT id, instance_id FROM instance_types WHERE provider_id = ?').bind(providerRecord.id),
|
||||
this.repos.db.prepare('SELECT id, instance_id FROM gpu_instances WHERE provider_id = ?').bind(providerRecord.id),
|
||||
this.repos.db.prepare('SELECT id, instance_id FROM g8_instances WHERE provider_id = ?').bind(providerRecord.id),
|
||||
this.repos.db.prepare('SELECT id, instance_id FROM vpu_instances WHERE provider_id = ?').bind(providerRecord.id)
|
||||
];
|
||||
|
||||
const [dbRegionsResult, dbInstancesResult, dbGpuResult, dbG8Result, dbVpuResult] = await this.repos.db.batch(batchQueries);
|
||||
|
||||
if (!dbRegionsResult.success || !dbInstancesResult.success) {
|
||||
throw new Error('Failed to fetch regions/instances for pricing');
|
||||
@@ -164,8 +243,27 @@ export class SyncOrchestrator {
|
||||
dbInstancesData.map(i => [i.id, { instance_id: i.instance_id }])
|
||||
);
|
||||
|
||||
// Create specialized instance mappings
|
||||
const dbGpuMap = new Map(
|
||||
(dbGpuResult.results as Array<{ id: number; instance_id: string }>).map(i => [i.id, { instance_id: i.instance_id }])
|
||||
);
|
||||
const dbG8Map = new Map(
|
||||
(dbG8Result.results as Array<{ id: number; instance_id: string }>).map(i => [i.id, { instance_id: i.instance_id }])
|
||||
);
|
||||
const dbVpuMap = new Map(
|
||||
(dbVpuResult.results as Array<{ id: number; instance_id: string }>).map(i => [i.id, { instance_id: i.instance_id }])
|
||||
);
|
||||
|
||||
// Get pricing data - may return array or count depending on provider
|
||||
const pricingResult = await connector.getPricing(instanceTypeIds, regionIds, dbInstanceMap);
|
||||
// Pass all instance maps for specialized pricing
|
||||
const pricingResult = await connector.getPricing(
|
||||
instanceTypeIds,
|
||||
regionIds,
|
||||
dbInstanceMap,
|
||||
dbGpuMap,
|
||||
dbG8Map,
|
||||
dbVpuMap
|
||||
);
|
||||
|
||||
// Handle both return types: array (Linode, Vultr) or number (AWS with generator)
|
||||
let pricingCount = 0;
|
||||
@@ -177,7 +275,15 @@ export class SyncOrchestrator {
|
||||
pricingCount = await this.repos.pricing.upsertMany(pricingResult);
|
||||
}
|
||||
|
||||
this.logger.info(`${provider} → ${stage}`, { regions: regionsCount, instances: instancesCount, pricing: pricingCount });
|
||||
this.logger.info(`${provider} → ${stage}`, {
|
||||
regions: regionsCount,
|
||||
regular_instances: regularInstancesCount,
|
||||
gpu_instances: gpuInstancesCount,
|
||||
g8_instances: g8InstancesCount,
|
||||
vpu_instances: vpuInstancesCount,
|
||||
total_instances: instancesCount,
|
||||
pricing: pricingCount
|
||||
});
|
||||
|
||||
// Stage 7: Validate
|
||||
stage = SyncStage.VALIDATE;
|
||||
@@ -497,6 +603,235 @@ export class SyncOrchestrator {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate Linode GPU pricing records in batches using Generator pattern
|
||||
* Minimizes memory usage by yielding batches at a time (default: 100)
|
||||
*
|
||||
* @param gpuInstanceTypeIds - Array of database GPU instance type IDs
|
||||
* @param regionIds - Array of database region IDs
|
||||
* @param dbGpuInstanceMap - Map of GPU instance type ID to DB instance data
|
||||
* @param rawInstanceMap - Map of instance_id (API ID) to raw Linode data
|
||||
* @param env - Environment configuration for SYNC_BATCH_SIZE
|
||||
* @yields Batches of GpuPricingInput records (configurable batch size)
|
||||
*
|
||||
* Manual Test:
|
||||
* For typical Linode GPU instances (~10 GPU types × 20 regions = 200 records):
|
||||
* - Default batch size (100): ~2 batches
|
||||
* - Memory savings: ~50% (200 records → 100 records in memory)
|
||||
* - Verify: Check logs for "Generated and upserted GPU pricing records for Linode"
|
||||
*/
|
||||
private *generateLinodeGpuPricingBatches(
|
||||
gpuInstanceTypeIds: number[],
|
||||
regionIds: number[],
|
||||
dbGpuInstanceMap: Map<number, { instance_id: string }>,
|
||||
rawInstanceMap: Map<string, { id: string; price: { hourly: number; monthly: number } }>,
|
||||
env?: Env
|
||||
): Generator<GpuPricingInput[], void, void> {
|
||||
const BATCH_SIZE = parseInt(env?.SYNC_BATCH_SIZE || '100', 10);
|
||||
let batch: GpuPricingInput[] = [];
|
||||
|
||||
for (const regionId of regionIds) {
|
||||
for (const gpuInstanceId of gpuInstanceTypeIds) {
|
||||
const dbInstance = dbGpuInstanceMap.get(gpuInstanceId);
|
||||
if (!dbInstance) {
|
||||
this.logger.warn('GPU instance type not found', { gpuInstanceId });
|
||||
continue;
|
||||
}
|
||||
|
||||
const rawInstance = rawInstanceMap.get(dbInstance.instance_id);
|
||||
if (!rawInstance) {
|
||||
this.logger.warn('Raw GPU instance data not found', { instance_id: dbInstance.instance_id });
|
||||
continue;
|
||||
}
|
||||
|
||||
batch.push({
|
||||
gpu_instance_id: gpuInstanceId,
|
||||
region_id: regionId,
|
||||
hourly_price: rawInstance.price.hourly,
|
||||
monthly_price: rawInstance.price.monthly,
|
||||
currency: 'USD',
|
||||
available: 1,
|
||||
});
|
||||
|
||||
if (batch.length >= BATCH_SIZE) {
|
||||
yield batch;
|
||||
batch = [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Yield remaining records
|
||||
if (batch.length > 0) {
|
||||
yield batch;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate Vultr GPU pricing records in batches using Generator pattern
|
||||
* Minimizes memory usage by yielding batches at a time (default: 100)
|
||||
*
|
||||
* @param gpuInstanceTypeIds - Array of database GPU instance type IDs
|
||||
* @param regionIds - Array of database region IDs
|
||||
* @param dbGpuInstanceMap - Map of GPU instance type ID to DB instance data
|
||||
* @param rawPlanMap - Map of plan_id (API ID) to raw Vultr plan data
|
||||
* @param env - Environment configuration for SYNC_BATCH_SIZE
|
||||
* @yields Batches of GpuPricingInput records (configurable batch size)
|
||||
*
|
||||
* Manual Test:
|
||||
* For typical Vultr GPU instances (~35 vcg types × 20 regions = 700 records):
|
||||
* - Default batch size (100): ~7 batches
|
||||
* - Memory savings: ~85% (700 records → 100 records in memory)
|
||||
* - Verify: Check logs for "Generated and upserted GPU pricing records for Vultr"
|
||||
*/
|
||||
private *generateVultrGpuPricingBatches(
|
||||
gpuInstanceTypeIds: number[],
|
||||
regionIds: number[],
|
||||
dbGpuInstanceMap: Map<number, { instance_id: string }>,
|
||||
rawPlanMap: Map<string, { id: string; monthly_cost: number }>,
|
||||
env?: Env
|
||||
): Generator<GpuPricingInput[], void, void> {
|
||||
const BATCH_SIZE = parseInt(env?.SYNC_BATCH_SIZE || '100', 10);
|
||||
let batch: GpuPricingInput[] = [];
|
||||
|
||||
for (const regionId of regionIds) {
|
||||
for (const gpuInstanceId of gpuInstanceTypeIds) {
|
||||
const dbInstance = dbGpuInstanceMap.get(gpuInstanceId);
|
||||
if (!dbInstance) {
|
||||
this.logger.warn('GPU instance type not found', { gpuInstanceId });
|
||||
continue;
|
||||
}
|
||||
|
||||
const rawPlan = rawPlanMap.get(dbInstance.instance_id);
|
||||
if (!rawPlan) {
|
||||
this.logger.warn('Raw GPU plan data not found', { instance_id: dbInstance.instance_id });
|
||||
continue;
|
||||
}
|
||||
|
||||
// Calculate hourly price: monthly_cost / 730 hours
|
||||
const hourlyPrice = rawPlan.monthly_cost / 730;
|
||||
|
||||
batch.push({
|
||||
gpu_instance_id: gpuInstanceId,
|
||||
region_id: regionId,
|
||||
hourly_price: hourlyPrice,
|
||||
monthly_price: rawPlan.monthly_cost,
|
||||
currency: 'USD',
|
||||
available: 1,
|
||||
});
|
||||
|
||||
if (batch.length >= BATCH_SIZE) {
|
||||
yield batch;
|
||||
batch = [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Yield remaining records
|
||||
if (batch.length > 0) {
|
||||
yield batch;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate G8 pricing records in batches for Linode
|
||||
* Similar to GPU pricing generator but for G8 instances
|
||||
*/
|
||||
private *generateLinodeG8PricingBatches(
|
||||
g8InstanceTypeIds: number[],
|
||||
regionIds: number[],
|
||||
dbG8InstanceMap: Map<number, { instance_id: string }>,
|
||||
rawInstanceMap: Map<string, { id: string; price: { hourly: number; monthly: number } }>,
|
||||
env?: Env
|
||||
): Generator<any[], void, void> {
|
||||
const BATCH_SIZE = parseInt(env?.SYNC_BATCH_SIZE || '100', 10);
|
||||
let batch: any[] = [];
|
||||
|
||||
for (const regionId of regionIds) {
|
||||
for (const g8InstanceId of g8InstanceTypeIds) {
|
||||
const dbInstance = dbG8InstanceMap.get(g8InstanceId);
|
||||
if (!dbInstance) {
|
||||
this.logger.warn('G8 instance type not found', { g8InstanceId });
|
||||
continue;
|
||||
}
|
||||
|
||||
const rawInstance = rawInstanceMap.get(dbInstance.instance_id);
|
||||
if (!rawInstance) {
|
||||
this.logger.warn('Raw G8 instance data not found', { instance_id: dbInstance.instance_id });
|
||||
continue;
|
||||
}
|
||||
|
||||
batch.push({
|
||||
g8_instance_id: g8InstanceId,
|
||||
region_id: regionId,
|
||||
hourly_price: rawInstance.price.hourly,
|
||||
monthly_price: rawInstance.price.monthly,
|
||||
currency: 'USD',
|
||||
available: 1,
|
||||
});
|
||||
|
||||
if (batch.length >= BATCH_SIZE) {
|
||||
yield batch;
|
||||
batch = [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Yield remaining records
|
||||
if (batch.length > 0) {
|
||||
yield batch;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate VPU pricing records in batches for Linode
|
||||
* Similar to GPU pricing generator but for VPU instances
|
||||
*/
|
||||
private *generateLinodeVpuPricingBatches(
|
||||
vpuInstanceTypeIds: number[],
|
||||
regionIds: number[],
|
||||
dbVpuInstanceMap: Map<number, { instance_id: string }>,
|
||||
rawInstanceMap: Map<string, { id: string; price: { hourly: number; monthly: number } }>,
|
||||
env?: Env
|
||||
): Generator<any[], void, void> {
|
||||
const BATCH_SIZE = parseInt(env?.SYNC_BATCH_SIZE || '100', 10);
|
||||
let batch: any[] = [];
|
||||
|
||||
for (const regionId of regionIds) {
|
||||
for (const vpuInstanceId of vpuInstanceTypeIds) {
|
||||
const dbInstance = dbVpuInstanceMap.get(vpuInstanceId);
|
||||
if (!dbInstance) {
|
||||
this.logger.warn('VPU instance type not found', { vpuInstanceId });
|
||||
continue;
|
||||
}
|
||||
|
||||
const rawInstance = rawInstanceMap.get(dbInstance.instance_id);
|
||||
if (!rawInstance) {
|
||||
this.logger.warn('Raw VPU instance data not found', { instance_id: dbInstance.instance_id });
|
||||
continue;
|
||||
}
|
||||
|
||||
batch.push({
|
||||
vpu_instance_id: vpuInstanceId,
|
||||
region_id: regionId,
|
||||
hourly_price: rawInstance.price.hourly,
|
||||
monthly_price: rawInstance.price.monthly,
|
||||
currency: 'USD',
|
||||
available: 1,
|
||||
});
|
||||
|
||||
if (batch.length >= BATCH_SIZE) {
|
||||
yield batch;
|
||||
batch = [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Yield remaining records
|
||||
if (batch.length > 0) {
|
||||
yield batch;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create connector for a specific provider
|
||||
*
|
||||
@@ -521,20 +856,73 @@ export class SyncOrchestrator {
|
||||
getInstanceTypes: async () => {
|
||||
const instances = await connector.fetchInstanceTypes();
|
||||
cachedInstanceTypes = instances; // Cache for pricing
|
||||
return instances.map(i => connector.normalizeInstance(i, providerId));
|
||||
|
||||
// Classification priority:
|
||||
// 1. GPU (gpus > 0) → handled in getGpuInstances
|
||||
// 2. VPU (id contains 'netint' or 'accelerated') → handled in getVpuInstances
|
||||
// 3. G8 (id starts with 'g8-') → handled in getG8Instances
|
||||
// 4. Default → regular instance_types
|
||||
const regularInstances = instances.filter(i => {
|
||||
if (i.gpus > 0) return false;
|
||||
if (i.id.includes('netint') || i.id.includes('accelerated')) return false;
|
||||
if (i.id.startsWith('g8-')) return false;
|
||||
return true;
|
||||
});
|
||||
return regularInstances.map(i => connector.normalizeInstance(i, providerId));
|
||||
},
|
||||
getGpuInstances: async (): Promise<GpuInstanceInput[]> => {
|
||||
// Use cached instances if available to avoid redundant API calls
|
||||
if (!cachedInstanceTypes) {
|
||||
this.logger.info('Fetching instance types for GPU extraction');
|
||||
cachedInstanceTypes = await connector.fetchInstanceTypes();
|
||||
}
|
||||
|
||||
// Filter and normalize GPU instances
|
||||
const gpuInstances = cachedInstanceTypes.filter(i => i.gpus > 0);
|
||||
return gpuInstances.map(i => connector.normalizeGpuInstance(i, providerId));
|
||||
},
|
||||
getG8Instances: async (): Promise<any[]> => {
|
||||
// Use cached instances if available to avoid redundant API calls
|
||||
if (!cachedInstanceTypes) {
|
||||
this.logger.info('Fetching instance types for G8 extraction');
|
||||
cachedInstanceTypes = await connector.fetchInstanceTypes();
|
||||
}
|
||||
|
||||
// Filter and normalize G8 instances (g8- prefix)
|
||||
const g8Instances = cachedInstanceTypes.filter(i =>
|
||||
i.id.startsWith('g8-') && (!i.gpus || i.gpus === 0)
|
||||
);
|
||||
return g8Instances.map(i => connector.normalizeG8Instance(i, providerId));
|
||||
},
|
||||
getVpuInstances: async (): Promise<any[]> => {
|
||||
// Use cached instances if available to avoid redundant API calls
|
||||
if (!cachedInstanceTypes) {
|
||||
this.logger.info('Fetching instance types for VPU extraction');
|
||||
cachedInstanceTypes = await connector.fetchInstanceTypes();
|
||||
}
|
||||
|
||||
// Filter and normalize VPU instances (netint or accelerated)
|
||||
const vpuInstances = cachedInstanceTypes.filter(i =>
|
||||
(i.id.includes('netint') || i.id.includes('accelerated')) && (!i.gpus || i.gpus === 0)
|
||||
);
|
||||
return vpuInstances.map(i => connector.normalizeVpuInstance(i, providerId));
|
||||
},
|
||||
getPricing: async (
|
||||
instanceTypeIds: number[],
|
||||
_instanceTypeIds: number[],
|
||||
regionIds: number[],
|
||||
dbInstanceMap: Map<number, { instance_id: string }>
|
||||
dbInstanceMap: Map<number, { instance_id: string }>,
|
||||
dbGpuMap?: Map<number, { instance_id: string }>,
|
||||
dbG8Map?: Map<number, { instance_id: string }>,
|
||||
dbVpuMap?: Map<number, { instance_id: string }>
|
||||
): Promise<number> => {
|
||||
/**
|
||||
* Linode Pricing Extraction Strategy (Generator Pattern):
|
||||
*
|
||||
* Linode pricing is embedded in instance type data (price.hourly, price.monthly).
|
||||
* Generate all region × instance combinations using generator pattern.
|
||||
* GPU instances are separated and stored in gpu_pricing table.
|
||||
*
|
||||
* Expected volume: ~200 instances × 20 regions = ~4,000 pricing records
|
||||
* Expected volume: ~190 regular + ~10 GPU instances × 20 regions = ~4,000 pricing records
|
||||
* Generator pattern with 100 records/batch minimizes memory usage
|
||||
* Each batch is immediately persisted to database to avoid memory buildup
|
||||
*
|
||||
@@ -542,9 +930,9 @@ export class SyncOrchestrator {
|
||||
*
|
||||
* Manual Test:
|
||||
* 1. Run sync: curl -X POST http://localhost:8787/api/sync/linode
|
||||
* 2. Verify pricing count: wrangler d1 execute cloud-instances-db --local --command "SELECT COUNT(*) FROM pricing WHERE instance_type_id IN (SELECT id FROM instance_types WHERE provider_id = (SELECT id FROM providers WHERE name = 'linode'))"
|
||||
* 3. Sample pricing: wrangler d1 execute cloud-instances-db --local --command "SELECT p.*, i.instance_name, r.region_code FROM pricing p JOIN instance_types i ON p.instance_type_id = i.id JOIN regions r ON p.region_id = r.id WHERE i.provider_id = (SELECT id FROM providers WHERE name = 'linode') LIMIT 10"
|
||||
* 4. Verify data integrity: wrangler d1 execute cloud-instances-db --local --command "SELECT COUNT(*) FROM pricing WHERE hourly_price = 0 OR monthly_price = 0"
|
||||
* 2. Verify regular pricing: wrangler d1 execute cloud-instances-db --local --command "SELECT COUNT(*) FROM pricing WHERE instance_type_id IN (SELECT id FROM instance_types WHERE provider_id = (SELECT id FROM providers WHERE name = 'linode'))"
|
||||
* 3. Verify GPU pricing: wrangler d1 execute cloud-instances-db --local --command "SELECT COUNT(*) FROM gpu_pricing WHERE gpu_instance_id IN (SELECT id FROM gpu_instances WHERE provider_id = (SELECT id FROM providers WHERE name = 'linode'))"
|
||||
* 4. Sample pricing: wrangler d1 execute cloud-instances-db --local --command "SELECT p.*, i.instance_name, r.region_code FROM pricing p JOIN instance_types i ON p.instance_type_id = i.id JOIN regions r ON p.region_id = r.id WHERE i.provider_id = (SELECT id FROM providers WHERE name = 'linode') LIMIT 10"
|
||||
*/
|
||||
|
||||
// Re-fetch instance types if not cached
|
||||
@@ -558,23 +946,113 @@ export class SyncOrchestrator {
|
||||
cachedInstanceTypes.map(i => [i.id, i])
|
||||
);
|
||||
|
||||
// Use generator pattern for memory-efficient processing
|
||||
const pricingGenerator = this.generateLinodePricingBatches(
|
||||
instanceTypeIds,
|
||||
regionIds,
|
||||
dbInstanceMap,
|
||||
rawInstanceMap,
|
||||
this.env
|
||||
);
|
||||
// Use provided maps or create empty ones
|
||||
const gpuMap = dbGpuMap || new Map();
|
||||
const g8Map = dbG8Map || new Map();
|
||||
const vpuMap = dbVpuMap || new Map();
|
||||
|
||||
// Process batches incrementally
|
||||
let totalCount = 0;
|
||||
for (const batch of pricingGenerator) {
|
||||
const batchCount = await this.repos.pricing.upsertMany(batch);
|
||||
totalCount += batchCount;
|
||||
// Separate instances by type: GPU, VPU, G8, and regular
|
||||
const gpuInstanceTypeIds: number[] = [];
|
||||
const g8InstanceTypeIds: number[] = [];
|
||||
const vpuInstanceTypeIds: number[] = [];
|
||||
const regularInstanceTypeIds: number[] = [];
|
||||
|
||||
// Extract GPU instance IDs from gpuMap
|
||||
for (const dbId of gpuMap.keys()) {
|
||||
gpuInstanceTypeIds.push(dbId);
|
||||
}
|
||||
|
||||
this.logger.info('Generated and upserted pricing records for Linode', { count: totalCount });
|
||||
// Extract G8 instance IDs from g8Map
|
||||
for (const dbId of g8Map.keys()) {
|
||||
g8InstanceTypeIds.push(dbId);
|
||||
}
|
||||
|
||||
// Extract VPU instance IDs from vpuMap
|
||||
for (const dbId of vpuMap.keys()) {
|
||||
vpuInstanceTypeIds.push(dbId);
|
||||
}
|
||||
|
||||
// Regular instances from dbInstanceMap
|
||||
for (const dbId of dbInstanceMap.keys()) {
|
||||
regularInstanceTypeIds.push(dbId);
|
||||
}
|
||||
|
||||
// Process regular instance pricing
|
||||
let regularPricingCount = 0;
|
||||
if (regularInstanceTypeIds.length > 0) {
|
||||
const regularGenerator = this.generateLinodePricingBatches(
|
||||
regularInstanceTypeIds,
|
||||
regionIds,
|
||||
dbInstanceMap,
|
||||
rawInstanceMap,
|
||||
this.env
|
||||
);
|
||||
|
||||
for (const batch of regularGenerator) {
|
||||
const batchCount = await this.repos.pricing.upsertMany(batch);
|
||||
regularPricingCount += batchCount;
|
||||
}
|
||||
}
|
||||
|
||||
// Process GPU instance pricing
|
||||
let gpuPricingCount = 0;
|
||||
if (gpuInstanceTypeIds.length > 0) {
|
||||
const gpuGenerator = this.generateLinodeGpuPricingBatches(
|
||||
gpuInstanceTypeIds,
|
||||
regionIds,
|
||||
gpuMap,
|
||||
rawInstanceMap,
|
||||
this.env
|
||||
);
|
||||
|
||||
for (const batch of gpuGenerator) {
|
||||
const batchCount = await this.repos.gpuPricing.upsertMany(batch);
|
||||
gpuPricingCount += batchCount;
|
||||
}
|
||||
}
|
||||
|
||||
// Process G8 instance pricing
|
||||
let g8PricingCount = 0;
|
||||
if (g8InstanceTypeIds.length > 0) {
|
||||
const g8Generator = this.generateLinodeG8PricingBatches(
|
||||
g8InstanceTypeIds,
|
||||
regionIds,
|
||||
g8Map,
|
||||
rawInstanceMap,
|
||||
this.env
|
||||
);
|
||||
|
||||
for (const batch of g8Generator) {
|
||||
const batchCount = await this.repos.g8Pricing.upsertMany(batch);
|
||||
g8PricingCount += batchCount;
|
||||
}
|
||||
}
|
||||
|
||||
// Process VPU instance pricing
|
||||
let vpuPricingCount = 0;
|
||||
if (vpuInstanceTypeIds.length > 0) {
|
||||
const vpuGenerator = this.generateLinodeVpuPricingBatches(
|
||||
vpuInstanceTypeIds,
|
||||
regionIds,
|
||||
vpuMap,
|
||||
rawInstanceMap,
|
||||
this.env
|
||||
);
|
||||
|
||||
for (const batch of vpuGenerator) {
|
||||
const batchCount = await this.repos.vpuPricing.upsertMany(batch);
|
||||
vpuPricingCount += batchCount;
|
||||
}
|
||||
}
|
||||
|
||||
const totalCount = regularPricingCount + gpuPricingCount + g8PricingCount + vpuPricingCount;
|
||||
this.logger.info('Generated and upserted pricing records for Linode', {
|
||||
regular_pricing: regularPricingCount,
|
||||
gpu_pricing: gpuPricingCount,
|
||||
g8_pricing: g8PricingCount,
|
||||
vpu_pricing: vpuPricingCount,
|
||||
total: totalCount
|
||||
});
|
||||
|
||||
// Return total count of processed records
|
||||
return totalCount;
|
||||
@@ -596,12 +1074,27 @@ export class SyncOrchestrator {
|
||||
getInstanceTypes: async () => {
|
||||
const plans = await connector.fetchPlans();
|
||||
cachedPlans = plans; // Cache for pricing
|
||||
return plans.map(p => connector.normalizeInstance(p, providerId));
|
||||
|
||||
// Filter out GPU instances (vcg type)
|
||||
const regularPlans = plans.filter(p => !p.id.startsWith('vcg'));
|
||||
return regularPlans.map(p => connector.normalizeInstance(p, providerId));
|
||||
},
|
||||
getGpuInstances: async (): Promise<GpuInstanceInput[]> => {
|
||||
// Use cached plans if available to avoid redundant API calls
|
||||
if (!cachedPlans) {
|
||||
this.logger.info('Fetching plans for GPU extraction');
|
||||
cachedPlans = await connector.fetchPlans();
|
||||
}
|
||||
|
||||
// Filter and normalize GPU instances (vcg type)
|
||||
const gpuPlans = cachedPlans.filter(p => p.id.startsWith('vcg'));
|
||||
return gpuPlans.map(p => connector.normalizeGpuInstance(p, providerId));
|
||||
},
|
||||
getPricing: async (
|
||||
instanceTypeIds: number[],
|
||||
regionIds: number[],
|
||||
dbInstanceMap: Map<number, { instance_id: string }>
|
||||
dbInstanceMap: Map<number, { instance_id: string }>,
|
||||
dbGpuMap?: Map<number, { instance_id: string }>
|
||||
): Promise<number> => {
|
||||
/**
|
||||
* Vultr Pricing Extraction Strategy (Generator Pattern):
|
||||
@@ -609,17 +1102,19 @@ export class SyncOrchestrator {
|
||||
* Vultr pricing is embedded in plan data (monthly_cost).
|
||||
* Generate all region × plan combinations using generator pattern.
|
||||
*
|
||||
* Expected volume: ~100 plans × 20 regions = ~2,000 pricing records
|
||||
* Expected volume: ~100 regular plans × 20 regions = ~2,000 pricing records
|
||||
* ~35 GPU plans × 20 regions = ~700 GPU pricing records
|
||||
* Generator pattern with 100 records/batch minimizes memory usage
|
||||
* Each batch is immediately persisted to database to avoid memory buildup
|
||||
*
|
||||
* Memory savings: ~95% (2,000 records → 100 records in memory at a time)
|
||||
* Memory savings: ~95% (2,700 records → 100 records in memory at a time)
|
||||
*
|
||||
* Manual Test:
|
||||
* 1. Run sync: curl -X POST http://localhost:8787/api/sync/vultr
|
||||
* 2. Verify pricing count: wrangler d1 execute cloud-instances-db --local --command "SELECT COUNT(*) FROM pricing WHERE instance_type_id IN (SELECT id FROM instance_types WHERE provider_id = (SELECT id FROM providers WHERE name = 'vultr'))"
|
||||
* 3. Sample pricing: wrangler d1 execute cloud-instances-db --local --command "SELECT p.*, i.instance_name, r.region_code FROM pricing p JOIN instance_types i ON p.instance_type_id = i.id JOIN regions r ON p.region_id = r.id WHERE i.provider_id = (SELECT id FROM providers WHERE name = 'vultr') LIMIT 10"
|
||||
* 4. Verify data integrity: wrangler d1 execute cloud-instances-db --local --command "SELECT COUNT(*) FROM pricing WHERE hourly_price = 0 OR monthly_price = 0"
|
||||
* 3. Verify GPU pricing: wrangler d1 execute cloud-instances-db --local --command "SELECT COUNT(*) FROM gpu_pricing WHERE gpu_instance_id IN (SELECT id FROM gpu_instances WHERE provider_id = (SELECT id FROM providers WHERE name = 'vultr'))"
|
||||
* 4. Sample pricing: wrangler d1 execute cloud-instances-db --local --command "SELECT p.*, i.instance_name, r.region_code FROM pricing p JOIN instance_types i ON p.instance_type_id = i.id JOIN regions r ON p.region_id = r.id WHERE i.provider_id = (SELECT id FROM providers WHERE name = 'vultr') LIMIT 10"
|
||||
* 5. Verify data integrity: wrangler d1 execute cloud-instances-db --local --command "SELECT COUNT(*) FROM pricing WHERE hourly_price = 0 OR monthly_price = 0"
|
||||
*/
|
||||
|
||||
// Re-fetch plans if not cached
|
||||
@@ -633,23 +1128,48 @@ export class SyncOrchestrator {
|
||||
cachedPlans.map(p => [p.id, p])
|
||||
);
|
||||
|
||||
// Use generator pattern for memory-efficient processing
|
||||
const pricingGenerator = this.generateVultrPricingBatches(
|
||||
instanceTypeIds,
|
||||
regionIds,
|
||||
dbInstanceMap,
|
||||
rawPlanMap,
|
||||
this.env
|
||||
);
|
||||
// Process regular instance pricing
|
||||
let regularPricingCount = 0;
|
||||
if (instanceTypeIds.length > 0) {
|
||||
const regularGenerator = this.generateVultrPricingBatches(
|
||||
instanceTypeIds,
|
||||
regionIds,
|
||||
dbInstanceMap,
|
||||
rawPlanMap,
|
||||
this.env
|
||||
);
|
||||
|
||||
// Process batches incrementally
|
||||
let totalCount = 0;
|
||||
for (const batch of pricingGenerator) {
|
||||
const batchCount = await this.repos.pricing.upsertMany(batch);
|
||||
totalCount += batchCount;
|
||||
for (const batch of regularGenerator) {
|
||||
const batchCount = await this.repos.pricing.upsertMany(batch);
|
||||
regularPricingCount += batchCount;
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info('Generated and upserted pricing records for Vultr', { count: totalCount });
|
||||
// Process GPU instance pricing
|
||||
let gpuPricingCount = 0;
|
||||
const gpuMap = dbGpuMap || new Map();
|
||||
if (gpuMap.size > 0) {
|
||||
const gpuInstanceTypeIds = Array.from(gpuMap.keys());
|
||||
const gpuGenerator = this.generateVultrGpuPricingBatches(
|
||||
gpuInstanceTypeIds,
|
||||
regionIds,
|
||||
gpuMap,
|
||||
rawPlanMap,
|
||||
this.env
|
||||
);
|
||||
|
||||
for (const batch of gpuGenerator) {
|
||||
const batchCount = await this.repos.gpuPricing.upsertMany(batch);
|
||||
gpuPricingCount += batchCount;
|
||||
}
|
||||
}
|
||||
|
||||
const totalCount = regularPricingCount + gpuPricingCount;
|
||||
this.logger.info('Generated and upserted pricing records for Vultr', {
|
||||
regular_pricing: regularPricingCount,
|
||||
gpu_pricing: gpuPricingCount,
|
||||
total: totalCount
|
||||
});
|
||||
|
||||
// Return total count of processed records
|
||||
return totalCount;
|
||||
|
||||
Reference in New Issue
Block a user