You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudTrafficSettlementRecov...

466 lines
22 KiB
PHP

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Command\CloudTrafficSettlementRecoveryCommand;
use App\Services\Cloud\CloudTrafficSettlementRecovery;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
require __DIR__ . '/Support/TrafficTestDatabase.php';
$capsule = trafficTestDatabase();
$schema = $capsule->schema('default');
$schema->create('cloud_traffic_pool_groups', static function (Blueprint $t): void {
$t->increments('id');
$t->string('mode');
});
$schema->create('cloud_traffic_pools', static function (Blueprint $t): void {
$t->increments('id');
$t->integer('user_id');
$t->integer('traffic_pool_group_id');
$t->string('state');
$t->integer('available_bytes')->default(0);
$t->integer('overage_bytes')->default(0);
$t->integer('version')->default(1);
$t->integer('last_sequence')->default(0);
$t->unique(['user_id', 'traffic_pool_group_id'], 'test_pool_identity');
$t->integer('last_event_id')->nullable();
$t->integer('created_at');
$t->integer('updated_at');
});
$schema->create('cloud_traffic_ledger_events', static function (Blueprint $t): void {
$t->increments('id');
$t->integer('pool_id');
$t->integer('user_id');
$t->integer('pool_sequence');
$t->string('event_key')->unique();
$t->string('event_type');
$t->string('source_type');
$t->string('source_id');
$t->integer('source_revision');
$t->string('payload_hash');
$t->integer('instance_id')->nullable();
$t->integer('service_period_id')->nullable();
$t->integer('entitlement_id')->nullable();
$t->integer('amount_bytes');
$t->integer('effective_at');
$t->integer('observed_at')->nullable();
$t->text('payload_json');
$t->integer('created_at');
$t->unique(['pool_id', 'pool_sequence']);
});
$schema->create('cloud_traffic_usage_settlement_buckets',
static function (Blueprint $t): void {
$t->increments('id');
$t->integer('pool_id');
$t->integer('user_id');
$t->integer('instance_id');
$t->integer('service_period_id');
$t->integer('entitlement_id')->nullable();
$t->string('allocation_kind');
$t->integer('allocation_target_id');
$t->integer('bucket_start');
$t->integer('seal_at');
$t->integer('bucket_revision');
$t->binary('event_identity', 16)->unique('recovery_bucket_identity');
$t->integer('amount_bytes');
$t->integer('download_bytes');
$t->integer('upload_bytes');
$t->integer('sample_count');
$t->integer('first_observed_at');
$t->integer('last_observed_at');
$t->string('state');
$t->integer('sealed_event_id')->nullable();
$t->integer('created_at');
$t->integer('updated_at');
});
$schema->create('cloud_traffic_settlement_receipts', static function (Blueprint $t): void {
$t->integer('pool_id');
$t->integer('instance_id');
$t->integer('service_period_id');
$t->string('allocation_kind');
$t->integer('allocation_target_id');
$t->integer('bucket_start');
$t->integer('max_revision');
});
$schema->create('cloud_traffic_instance_cursors',
static function (Blueprint $t): void {
$t->integer('instance_id')->primary();
$t->integer('user_id');
$t->integer('pool_id');
$t->integer('accounted_download_bytes');
$t->integer('accounted_upload_bytes');
$t->integer('last_observed_at');
$t->integer('last_event_id')->nullable();
$t->string('state');
$t->string('attention_code');
$t->integer('version');
$t->integer('updated_at');
});
$schema->create('cloud_instances', static function (Blueprint $t): void {
$t->increments('id');
$t->integer('user_id');
$t->integer('current_period_id')->nullable();
$t->integer('lifetime_download_bytes')->default(0);
$t->integer('lifetime_upload_bytes')->default(0);
$t->integer('traffic_synced_at')->default(0);
$t->string('status')->default('active');
$t->integer('created_at')->default(0);
$t->integer('updated_at')->default(0);
});
$schema->create('cloud_service_periods', static function (Blueprint $t): void {
$t->increments('id');
$t->integer('instance_id');
$t->integer('user_id');
$t->integer('traffic_pool_group_id')->nullable();
$t->string('traffic_quota_mode')->default('finite');
$t->integer('traffic_quota_bytes')->default(0);
$t->integer('starts_at');
$t->integer('ends_at');
$t->string('status');
$t->integer('updated_at')->default(0);
});
$schema->create('cloud_traffic_entitlements', static function (Blueprint $t): void {
$t->increments('id');
$t->integer('pool_id');
$t->integer('user_id');
$t->integer('instance_id');
$t->integer('service_period_id');
$t->integer('starts_at');
$t->integer('ends_at');
$t->string('quota_mode');
$t->integer('granted_bytes');
$t->integer('consumed_bytes')->default(0);
$t->integer('forfeited_bytes')->default(0);
$t->string('state');
$t->integer('activated_at')->nullable();
$t->integer('closed_at')->nullable();
$t->integer('version')->default(1);
$t->integer('created_at')->default(0);
$t->integer('updated_at')->default(0);
});
$schema->create('cloud_traffic_scan_failures', static function (Blueprint $t): void {
$t->increments('id');
$t->string('scan_kind');
$t->integer('subject_id');
$t->integer('attempt_count');
$t->integer('last_failed_at');
$t->integer('first_failed_at');
$t->integer('retry_at');
$t->string('last_error_code');
});
$now = time();
Capsule::table('cloud_traffic_pool_groups')->insert([
'id' => 1, 'mode' => 'enforced',
]);
Capsule::table('cloud_traffic_pools')->insert([
'id' => 1, 'user_id' => 7, 'traffic_pool_group_id' => 1,
'state' => 'available', 'available_bytes' => 0, 'overage_bytes' => 10,
'version' => 1, 'last_sequence' => 0, 'last_event_id' => null,
'created_at' => $now - 100, 'updated_at' => $now - 100,
]);
$identity = random_bytes(16);
$bucket = [
'id' => 1, 'pool_id' => 1, 'user_id' => 7, 'instance_id' => 11,
'service_period_id' => 101, 'entitlement_id' => null,
'allocation_kind' => 'overage', 'allocation_target_id' => 0,
'bucket_start' => $now - 86_500, 'seal_at' => $now - 100,
'bucket_revision' => 1, 'event_identity' => $identity,
'amount_bytes' => 10, 'download_bytes' => 7, 'upload_bytes' => 3,
'sample_count' => 1, 'first_observed_at' => $now - 86_490,
'last_observed_at' => $now - 86_490, 'state' => 'open',
'sealed_event_id' => null, 'created_at' => $now - 100,
'updated_at' => $now - 100,
];
Capsule::table('cloud_traffic_usage_settlement_buckets')->insert($bucket);
$plan = CloudTrafficSettlementRecovery::inspect();
$assert($plan['actionable'] === true
&& $plan['counts']['due_buckets'] === 1
&& preg_match('/^[a-f0-9]{64}$/D', $plan['plan_hash']) === 1,
'Inspect did not produce a bounded actionable settlement plan: '
. json_encode($plan, JSON_THROW_ON_ERROR));
Capsule::table('cloud_traffic_usage_settlement_buckets')->where('id', 1)
->update(['amount_bytes' => 11, 'download_bytes' => 8]);
try {
CloudTrafficSettlementRecovery::apply($plan['plan_hash']);
throw new RuntimeException('A changed recovery plan was applied.');
} catch (RuntimeException $error) {
$assert($error->getMessage()
=== 'traffic_settlement_recovery_plan_changed',
'Changed plan failed with the wrong safety code.');
}
Capsule::table('cloud_traffic_usage_settlement_buckets')->where('id', 1)
->update(['amount_bytes' => 10, 'download_bytes' => 7]);
$fresh = CloudTrafficSettlementRecovery::inspect();
$applied = CloudTrafficSettlementRecovery::apply($fresh['plan_hash']);
$assert($applied['sealed_buckets'] === 1
&& Capsule::table('cloud_traffic_usage_settlement_buckets')
->where('id', 1)->value('state') === 'sealed'
&& Capsule::table('cloud_traffic_ledger_events')
->where('event_key', 'traffic-settlement:v2:' . bin2hex($identity))
->count() === 1,
'The exact approved plan did not use the normal immutable seal path.');
$empty = CloudTrafficSettlementRecovery::inspect();
$repeat = CloudTrafficSettlementRecovery::apply($empty['plan_hash']);
$assert($repeat['sealed_buckets'] === 0
&& $repeat['reconciled_periods'] === 0
&& $repeat['settled_instances'] === 0,
'A repeated empty recovery plan was not idempotent.');
// Health reads a bounded, coherent bucket/event view even as another connection seals.
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'log');
$capsule->getConnection('log')->getSchemaBuilder()->create('cloud_vm_traffic_checkpoints', static function (Blueprint $t): void {
$t->integer('instance_id');
$t->integer('first_observed_at');
});
$probe = static fn (): array => App\Services\Cloud\CloudTrafficStorageHealthService::snapshot($now)['metrics'];
$bucketRow = Capsule::table('cloud_traffic_usage_settlement_buckets')->where('id', 1);
$eventRow = Capsule::table('cloud_traffic_ledger_events')->where('id', 1);
$event = (array) $eventRow->first();
$bucketRow->update(['state' => 'open', 'sealed_event_id' => null]);
$assert($probe()['settlement_identity_conflicts'] === 0, 'Health rejected an exact immutable event.');
$eventRow->update(['amount_bytes' => 99]);
$assert($probe()['settlement_identity_conflicts'] === 1, 'Health accepted a mismatched immutable event.');
$eventRow->delete();
$raceSeal = true;
$capsule->getConnection()->beforeExecuting(static function ($sql) use ($event, $capsule, &$raceSeal): void {
if ($raceSeal && str_starts_with(strtolower($sql), 'select * from')
&& str_contains($sql, 'cloud_traffic_ledger_events') && str_contains($sql, 'event_key')) {
$raceSeal = false;
$peer = $capsule->getConnection('peer');
$peer->transaction(static function () use ($peer, $event): void {
$peer->table('cloud_traffic_ledger_events')->insert($event);
$peer->table('cloud_traffic_usage_settlement_buckets')->where('id', 1)
->update(['state' => 'sealed', 'sealed_event_id' => 1]);
});
}
});
$assert($probe()['settlement_identity_conflicts'] === 0 && !$raceSeal && $probe()['open_buckets_due'] === 0,
'A concurrent valid seal produced an identity alarm.');
// Commit a peer update after plan validation but before the second approved action.
foreach ([3, 4] as $id) {
Capsule::table('cloud_traffic_usage_settlement_buckets')->insert(array_replace($bucket, [
'id' => $id, 'event_identity' => random_bytes(16),
]));
}
$racePlan = CloudTrafficSettlementRecovery::inspect();
$countdown = 2;
$capsule->getConnection()->beforeStartingTransaction(static function () use (&$countdown, $capsule): void {
if (--$countdown === 0) {
$capsule->getConnection('peer')->table('cloud_traffic_usage_settlement_buckets')
->where('id', 4)->update(['amount_bytes' => 90, 'download_bytes' => 87]);
}
});
$partial = CloudTrafficSettlementRecovery::apply($racePlan['plan_hash']);
$assert($partial['status'] === 'stopped' && $partial['completed']['due_buckets'] === [3]
&& $partial['remaining']['due_buckets'] === [4]
&& $partial['failed']['reason'] === 'traffic_settlement_recovery_plan_changed'
&& Capsule::table('cloud_traffic_ledger_events')->count() === 2,
'A 10-byte approval consumed a concurrent 90-byte bucket or lost partial progress.');
$fresh = CloudTrafficSettlementRecovery::inspect();
$assert(CloudTrafficSettlementRecovery::apply($fresh['plan_hash'])['sealed_buckets'] === 1,
'A freshly reviewed plan did not resume from the remaining bucket.');
Capsule::table('cloud_instances')->insert(['id' => 11, 'user_id' => 7,
'traffic_synced_at' => $now, 'lifetime_download_bytes' => 10]);
Capsule::table('cloud_service_periods')->insert(['id' => 101, 'instance_id' => 11,
'user_id' => 7, 'traffic_pool_group_id' => 1, 'status' => 'active',
'traffic_quota_bytes' => 1000, 'starts_at' => $now - 100, 'ends_at' => $now + 10000]);
Capsule::table('cloud_traffic_entitlements')->insert(['id' => 41, 'pool_id' => 1,
'instance_id' => 11, 'user_id' => 7, 'service_period_id' => 101, 'quota_mode' => 'finite',
'granted_bytes' => 1000, 'state' => 'active', 'starts_at' => $now - 100, 'ends_at' => $now + 10000]);
Capsule::table('cloud_traffic_instance_cursors')->insert(['instance_id' => 11, 'user_id' => 7,
'pool_id' => 1, 'accounted_download_bytes' => 0, 'accounted_upload_bytes' => 0,
'last_observed_at' => $now - 60, 'state' => 'ready', 'attention_code' => '', 'version' => 1, 'updated_at' => $now]);
$plan = CloudTrafficSettlementRecovery::inspect();
$changeSample = true;
$capsule->getConnection()->beforeStartingTransaction(static function () use (&$changeSample, $capsule): void {
if ($changeSample) {
$changeSample = false;
$capsule->getConnection('peer')->table('cloud_instances')->where('id', 11)->update(['lifetime_download_bytes' => 90]);
}
});
$stopped = CloudTrafficSettlementRecovery::apply($plan['plan_hash']);
$assert($stopped['failed']['kind'] === 'unsettled_instances'
&& $stopped['failed']['reason'] === 'traffic_settlement_recovery_plan_changed'
&& Capsule::table('cloud_traffic_instance_cursors')->value('accounted_download_bytes') == 0,
'The recovery writer advanced to an unapproved sample.');
Capsule::table('cloud_traffic_instance_cursors')->update(['accounted_download_bytes' => 90, 'last_observed_at' => $now]);
Capsule::table('cloud_service_periods')->where('id', 101)->update(['traffic_quota_bytes' => 1100]);
$plan = CloudTrafficSettlementRecovery::inspect();
$changeQuota = true;
$capsule->getConnection()->beforeStartingTransaction(static function () use (&$changeQuota, $capsule): void {
if ($changeQuota) {
$changeQuota = false;
$capsule->getConnection('peer')->table('cloud_service_periods')->where('id', 101)->update(['traffic_quota_bytes' => 2000]);
}
});
$stopped = CloudTrafficSettlementRecovery::apply($plan['plan_hash']);
$assert($stopped['failed']['kind'] === 'unsettled_periods'
&& $stopped['failed']['reason'] === 'traffic_settlement_recovery_plan_changed'
&& Capsule::table('cloud_traffic_entitlements')->value('granted_bytes') == 1000,
'The recovery writer granted an unapproved quota.');
Capsule::table('cloud_service_periods')->where('id', 101)->update(['traffic_quota_bytes' => 1000]);
$failure = ['id' => 1, 'scan_kind' => 'instance', 'subject_id' => 11, 'attempt_count' => 1,
'last_failed_at' => $now, 'first_failed_at' => $now, 'retry_at' => $now + 30, 'last_error_code' => 'Test'];
Capsule::table('cloud_traffic_scan_failures')->insert($failure);
$cleanup = CloudTrafficSettlementRecovery::inspect();
$assert($cleanup['counts']['scan_failures'] === 1 && !$cleanup['counts']['unsettled_instances'],
'Already settled failure has no manual recovery path.');
$race = true;
$capsule->getConnection()->beforeExecuting(static function ($sql) use (&$race, $capsule): void {
if ($race && str_starts_with(strtolower($sql), 'delete from')) {
$race = false;
$capsule->getConnection('peer')->table('cloud_traffic_scan_failures')->where('id', 1)
->update(['attempt_count' => 2]);
}
});
$stopped = CloudTrafficSettlementRecovery::apply($cleanup['plan_hash']);
$assert($stopped['failed']['kind'] === 'scan_failures' && Capsule::table('cloud_traffic_scan_failures')->count() === 1,
'Cleanup erased a newer failure generation.');
$cleanup = CloudTrafficSettlementRecovery::inspect();
$assert(CloudTrafficSettlementRecovery::apply($cleanup['plan_hash'])['cleared_scan_failures'] === 1,
'Reviewed settled failure was left behind.');
Capsule::table('cloud_traffic_scan_failures')->insert($failure);
Capsule::table('cloud_traffic_instance_cursors')->update(['state' => 'attention']);
$assert(CloudTrafficSettlementRecovery::inspect()['counts']['scan_failures'] === 0
&& !App\Services\Cloud\CloudTrafficScanRecovery::clearIfResolved($failure, $now),
'An attention cursor was treated as successfully recovered.');
Capsule::table('cloud_traffic_instance_cursors')->update(['state' => 'ready']);
$cleanup = CloudTrafficSettlementRecovery::inspect();
$changeOnCleanup = true;
$capsule->getConnection()->beforeExecuting(static function ($sql) use (&$changeOnCleanup, $capsule): void {
if ($changeOnCleanup && str_starts_with(strtolower($sql), 'delete from')) {
$changeOnCleanup = false;
$capsule->getConnection('peer')->table('cloud_instances')->where('id', 11)
->update(['lifetime_download_bytes' => 100]);
}
});
$result = CloudTrafficSettlementRecovery::apply($cleanup['plan_hash']);
$assert(!$changeOnCleanup && $result['status'] === 'stopped'
&& Capsule::table('cloud_traffic_scan_failures')->count() === 1,
'Cleanup erased a failure while a concurrently committed sample was unsettled.');
Capsule::table('cloud_traffic_pools')->where('id', 1)->update(['overage_bytes' => 0]);
$plan = CloudTrafficSettlementRecovery::inspect();
$expected = 100;
if ($capsule->getConnection()->getDriverName() === 'mysql') {
$changeDuringWrite = true;
$capsule->getConnection()->beforeExecuting(static function ($sql) use (&$changeDuringWrite, $capsule): void {
if ($changeDuringWrite && str_starts_with(strtolower($sql), 'update `cloud_traffic_instance_cursors`')) {
$changeDuringWrite = false;
$capsule->getConnection('peer')->table('cloud_instances')->where('id', 11)
->update(['lifetime_download_bytes' => 105]);
}
});
$stopped = CloudTrafficSettlementRecovery::apply($plan['plan_hash']);
$assert(!$changeDuringWrite && $stopped['failed']['reason'] === 'traffic_settlement_recovery_plan_changed'
&& (int) Capsule::table('cloud_traffic_entitlements')->value('consumed_bytes') === 0
&& (int) Capsule::table('cloud_traffic_instance_cursors')->value('accounted_download_bytes') === 90,
'A changed sample did not roll back the entire manual accounting write.');
$plan = CloudTrafficSettlementRecovery::inspect();
$expected = 105;
}
$result = CloudTrafficSettlementRecovery::apply($plan['plan_hash']);
$assert($result['status'] === 'applied' && $result['settled_instances'] === 1 && $result['cleared_scan_failures'] === 1
&& (int) Capsule::table('cloud_traffic_entitlements')->value('consumed_bytes') === $expected - 90
&& (int) Capsule::table('cloud_traffic_instance_cursors')->value('accounted_download_bytes') === $expected
&& (int) Capsule::table('cloud_traffic_usage_settlement_buckets')->where('entitlement_id', 41)->sum('amount_bytes') === $expected - 90,
'The reviewed sample was not booked exactly once with its failure retired: ' . json_encode($result));
$empty = CloudTrafficSettlementRecovery::inspect();
$assert(CloudTrafficSettlementRecovery::apply($empty['plan_hash'])['settled_instances'] === 0,
'A successfully booked sample was selected again.');
Capsule::table('cloud_instances')->where('id', 11)->update(['lifetime_download_bytes' => $expected - 1]);
$regression = CloudTrafficSettlementRecovery::inspect();
$result = CloudTrafficSettlementRecovery::apply($regression['plan_hash']);
$assert($result['status'] === 'stopped' && $result['failed']['reason'] === 'traffic_counter_regression'
&& $result['completed']['unsettled_instances'] === [] && $result['remaining']['unsettled_instances'] === [11]
&& Capsule::table('cloud_traffic_instance_cursors')->value('state') === 'attention'
&& (int) Capsule::table('cloud_traffic_entitlements')->value('consumed_bytes') === $expected - 90,
'A regressed counter was reported as settled or lost its durable attention marker.');
Capsule::table('cloud_traffic_usage_settlement_buckets')->insert(array_merge(
$bucket,
[
'id' => 2,
'event_identity' => '',
'state' => 'open',
'sealed_event_id' => null,
]
));
$blocked = CloudTrafficSettlementRecovery::inspect();
$assert($blocked['actionable'] === false
&& $blocked['blockers'][0]['code']
=== 'settlement_event_identity_missing',
'A malformed persistent event identity did not block recovery.');
try {
CloudTrafficSettlementRecovery::apply($blocked['plan_hash']);
throw new RuntimeException('A blocked recovery plan was applied.');
} catch (RuntimeException $error) {
$assert($error->getMessage() === 'traffic_settlement_recovery_blocked',
'Blocked plan failed with the wrong safety code.');
}
$parse = new ReflectionMethod(
CloudTrafficSettlementRecoveryCommand::class,
'parse'
);
$assert($parse->invoke(null, [
'xcat', 'cloudTrafficSettlementRecovery', 'inspect',
]) === ['action' => 'inspect'], 'CLI inspect arguments drifted.');
$hash = str_repeat('a', 64);
$assert($parse->invoke(null, [
'xcat', 'cloudTrafficSettlementRecovery', 'apply',
'--plan-hash=' . $hash, '--apply',
]) === ['action' => 'apply', 'plan_hash' => $hash],
'CLI apply did not bind the exact reviewed plan hash.');
foreach ([
['apply', '--plan-hash=' . $hash],
['apply', '--plan-hash=bad', '--apply'],
['apply', '--plan-hash=' . $hash, '--apply', '--force'],
['inspect', '--apply'],
] as $arguments) {
try {
$parse->invoke(null, array_merge([
'xcat', 'cloudTrafficSettlementRecovery',
], $arguments));
throw new RuntimeException('Unsafe CLI arguments were accepted.');
} catch (ReflectionException $error) {
throw $error;
} catch (Throwable $error) {
$actual = $error->getPrevious() ?? $error;
$assert($actual instanceof InvalidArgumentException,
'Unsafe CLI arguments failed with the wrong exception.');
}
}
$service = file_get_contents(dirname(__DIR__)
. '/app/Services/Cloud/CloudTrafficSettlementRecovery.php');
$xcat = file_get_contents(dirname(__DIR__) . '/app/Command/XCat.php');
$assert(is_string($service)
&& is_string($xcat)
&& !str_contains($service, '->delete(')
&& !str_contains($service, 'CloudAgentClient')
&& !str_contains($service, 'CloudJob')
&& str_contains($service,
'CloudTrafficPoolAccounting::sealPlannedUsageBuckets(')
&& str_contains($service,
'CloudTrafficPoolAccounting::reconcileServicePeriod(')
&& str_contains($service,
'CloudTrafficPoolAccounting::settleInstanceUsage(')
&& str_contains($xcat, "'cloudTrafficSettlementRecovery'")
&& str_contains($xcat,
'cloudTrafficSettlementRecovery apply --plan-hash=HASH --apply'),
'Recovery acquired a destructive or remote bypass path.');
echo "Traffic settlement recovery plan/apply checks passed.\n";