You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
466 lines
22 KiB
PHP
466 lines
22 KiB
PHP
<?php
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Command\CloudTrafficSettlementRecoveryCommand;
|
|
use App\Services\Cloud\CloudTrafficSettlementRecovery;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
require __DIR__ . '/Support/TrafficTestDatabase.php';
|
|
$capsule = trafficTestDatabase();
|
|
$schema = $capsule->schema('default');
|
|
|
|
$schema->create('cloud_traffic_pool_groups', static function (Blueprint $t): void {
|
|
$t->increments('id');
|
|
$t->string('mode');
|
|
});
|
|
$schema->create('cloud_traffic_pools', static function (Blueprint $t): void {
|
|
$t->increments('id');
|
|
$t->integer('user_id');
|
|
$t->integer('traffic_pool_group_id');
|
|
$t->string('state');
|
|
$t->integer('available_bytes')->default(0);
|
|
$t->integer('overage_bytes')->default(0);
|
|
$t->integer('version')->default(1);
|
|
$t->integer('last_sequence')->default(0);
|
|
$t->unique(['user_id', 'traffic_pool_group_id'], 'test_pool_identity');
|
|
$t->integer('last_event_id')->nullable();
|
|
$t->integer('created_at');
|
|
$t->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_traffic_ledger_events', static function (Blueprint $t): void {
|
|
$t->increments('id');
|
|
$t->integer('pool_id');
|
|
$t->integer('user_id');
|
|
$t->integer('pool_sequence');
|
|
$t->string('event_key')->unique();
|
|
$t->string('event_type');
|
|
$t->string('source_type');
|
|
$t->string('source_id');
|
|
$t->integer('source_revision');
|
|
$t->string('payload_hash');
|
|
$t->integer('instance_id')->nullable();
|
|
$t->integer('service_period_id')->nullable();
|
|
$t->integer('entitlement_id')->nullable();
|
|
$t->integer('amount_bytes');
|
|
$t->integer('effective_at');
|
|
$t->integer('observed_at')->nullable();
|
|
$t->text('payload_json');
|
|
$t->integer('created_at');
|
|
$t->unique(['pool_id', 'pool_sequence']);
|
|
});
|
|
$schema->create('cloud_traffic_usage_settlement_buckets',
|
|
static function (Blueprint $t): void {
|
|
$t->increments('id');
|
|
$t->integer('pool_id');
|
|
$t->integer('user_id');
|
|
$t->integer('instance_id');
|
|
$t->integer('service_period_id');
|
|
$t->integer('entitlement_id')->nullable();
|
|
$t->string('allocation_kind');
|
|
$t->integer('allocation_target_id');
|
|
$t->integer('bucket_start');
|
|
$t->integer('seal_at');
|
|
$t->integer('bucket_revision');
|
|
$t->binary('event_identity', 16)->unique('recovery_bucket_identity');
|
|
$t->integer('amount_bytes');
|
|
$t->integer('download_bytes');
|
|
$t->integer('upload_bytes');
|
|
$t->integer('sample_count');
|
|
$t->integer('first_observed_at');
|
|
$t->integer('last_observed_at');
|
|
$t->string('state');
|
|
$t->integer('sealed_event_id')->nullable();
|
|
$t->integer('created_at');
|
|
$t->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_traffic_settlement_receipts', static function (Blueprint $t): void {
|
|
$t->integer('pool_id');
|
|
$t->integer('instance_id');
|
|
$t->integer('service_period_id');
|
|
$t->string('allocation_kind');
|
|
$t->integer('allocation_target_id');
|
|
$t->integer('bucket_start');
|
|
$t->integer('max_revision');
|
|
});
|
|
$schema->create('cloud_traffic_instance_cursors',
|
|
static function (Blueprint $t): void {
|
|
$t->integer('instance_id')->primary();
|
|
$t->integer('user_id');
|
|
$t->integer('pool_id');
|
|
$t->integer('accounted_download_bytes');
|
|
$t->integer('accounted_upload_bytes');
|
|
$t->integer('last_observed_at');
|
|
$t->integer('last_event_id')->nullable();
|
|
$t->string('state');
|
|
$t->string('attention_code');
|
|
$t->integer('version');
|
|
$t->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_instances', static function (Blueprint $t): void {
|
|
$t->increments('id');
|
|
$t->integer('user_id');
|
|
$t->integer('current_period_id')->nullable();
|
|
$t->integer('lifetime_download_bytes')->default(0);
|
|
$t->integer('lifetime_upload_bytes')->default(0);
|
|
$t->integer('traffic_synced_at')->default(0);
|
|
$t->string('status')->default('active');
|
|
$t->integer('created_at')->default(0);
|
|
$t->integer('updated_at')->default(0);
|
|
});
|
|
$schema->create('cloud_service_periods', static function (Blueprint $t): void {
|
|
$t->increments('id');
|
|
$t->integer('instance_id');
|
|
$t->integer('user_id');
|
|
$t->integer('traffic_pool_group_id')->nullable();
|
|
$t->string('traffic_quota_mode')->default('finite');
|
|
$t->integer('traffic_quota_bytes')->default(0);
|
|
$t->integer('starts_at');
|
|
$t->integer('ends_at');
|
|
$t->string('status');
|
|
$t->integer('updated_at')->default(0);
|
|
});
|
|
$schema->create('cloud_traffic_entitlements', static function (Blueprint $t): void {
|
|
$t->increments('id');
|
|
$t->integer('pool_id');
|
|
$t->integer('user_id');
|
|
$t->integer('instance_id');
|
|
$t->integer('service_period_id');
|
|
$t->integer('starts_at');
|
|
$t->integer('ends_at');
|
|
$t->string('quota_mode');
|
|
$t->integer('granted_bytes');
|
|
$t->integer('consumed_bytes')->default(0);
|
|
$t->integer('forfeited_bytes')->default(0);
|
|
$t->string('state');
|
|
$t->integer('activated_at')->nullable();
|
|
$t->integer('closed_at')->nullable();
|
|
$t->integer('version')->default(1);
|
|
$t->integer('created_at')->default(0);
|
|
$t->integer('updated_at')->default(0);
|
|
});
|
|
|
|
$schema->create('cloud_traffic_scan_failures', static function (Blueprint $t): void {
|
|
$t->increments('id');
|
|
$t->string('scan_kind');
|
|
$t->integer('subject_id');
|
|
$t->integer('attempt_count');
|
|
$t->integer('last_failed_at');
|
|
$t->integer('first_failed_at');
|
|
$t->integer('retry_at');
|
|
$t->string('last_error_code');
|
|
});
|
|
$now = time();
|
|
Capsule::table('cloud_traffic_pool_groups')->insert([
|
|
'id' => 1, 'mode' => 'enforced',
|
|
]);
|
|
Capsule::table('cloud_traffic_pools')->insert([
|
|
'id' => 1, 'user_id' => 7, 'traffic_pool_group_id' => 1,
|
|
'state' => 'available', 'available_bytes' => 0, 'overage_bytes' => 10,
|
|
'version' => 1, 'last_sequence' => 0, 'last_event_id' => null,
|
|
'created_at' => $now - 100, 'updated_at' => $now - 100,
|
|
]);
|
|
$identity = random_bytes(16);
|
|
$bucket = [
|
|
'id' => 1, 'pool_id' => 1, 'user_id' => 7, 'instance_id' => 11,
|
|
'service_period_id' => 101, 'entitlement_id' => null,
|
|
'allocation_kind' => 'overage', 'allocation_target_id' => 0,
|
|
'bucket_start' => $now - 86_500, 'seal_at' => $now - 100,
|
|
'bucket_revision' => 1, 'event_identity' => $identity,
|
|
'amount_bytes' => 10, 'download_bytes' => 7, 'upload_bytes' => 3,
|
|
'sample_count' => 1, 'first_observed_at' => $now - 86_490,
|
|
'last_observed_at' => $now - 86_490, 'state' => 'open',
|
|
'sealed_event_id' => null, 'created_at' => $now - 100,
|
|
'updated_at' => $now - 100,
|
|
];
|
|
Capsule::table('cloud_traffic_usage_settlement_buckets')->insert($bucket);
|
|
|
|
$plan = CloudTrafficSettlementRecovery::inspect();
|
|
$assert($plan['actionable'] === true
|
|
&& $plan['counts']['due_buckets'] === 1
|
|
&& preg_match('/^[a-f0-9]{64}$/D', $plan['plan_hash']) === 1,
|
|
'Inspect did not produce a bounded actionable settlement plan: '
|
|
. json_encode($plan, JSON_THROW_ON_ERROR));
|
|
Capsule::table('cloud_traffic_usage_settlement_buckets')->where('id', 1)
|
|
->update(['amount_bytes' => 11, 'download_bytes' => 8]);
|
|
try {
|
|
CloudTrafficSettlementRecovery::apply($plan['plan_hash']);
|
|
throw new RuntimeException('A changed recovery plan was applied.');
|
|
} catch (RuntimeException $error) {
|
|
$assert($error->getMessage()
|
|
=== 'traffic_settlement_recovery_plan_changed',
|
|
'Changed plan failed with the wrong safety code.');
|
|
}
|
|
Capsule::table('cloud_traffic_usage_settlement_buckets')->where('id', 1)
|
|
->update(['amount_bytes' => 10, 'download_bytes' => 7]);
|
|
$fresh = CloudTrafficSettlementRecovery::inspect();
|
|
$applied = CloudTrafficSettlementRecovery::apply($fresh['plan_hash']);
|
|
$assert($applied['sealed_buckets'] === 1
|
|
&& Capsule::table('cloud_traffic_usage_settlement_buckets')
|
|
->where('id', 1)->value('state') === 'sealed'
|
|
&& Capsule::table('cloud_traffic_ledger_events')
|
|
->where('event_key', 'traffic-settlement:v2:' . bin2hex($identity))
|
|
->count() === 1,
|
|
'The exact approved plan did not use the normal immutable seal path.');
|
|
$empty = CloudTrafficSettlementRecovery::inspect();
|
|
$repeat = CloudTrafficSettlementRecovery::apply($empty['plan_hash']);
|
|
$assert($repeat['sealed_buckets'] === 0
|
|
&& $repeat['reconciled_periods'] === 0
|
|
&& $repeat['settled_instances'] === 0,
|
|
'A repeated empty recovery plan was not idempotent.');
|
|
|
|
// Health reads a bounded, coherent bucket/event view even as another connection seals.
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'log');
|
|
$capsule->getConnection('log')->getSchemaBuilder()->create('cloud_vm_traffic_checkpoints', static function (Blueprint $t): void {
|
|
$t->integer('instance_id');
|
|
$t->integer('first_observed_at');
|
|
});
|
|
$probe = static fn (): array => App\Services\Cloud\CloudTrafficStorageHealthService::snapshot($now)['metrics'];
|
|
$bucketRow = Capsule::table('cloud_traffic_usage_settlement_buckets')->where('id', 1);
|
|
$eventRow = Capsule::table('cloud_traffic_ledger_events')->where('id', 1);
|
|
$event = (array) $eventRow->first();
|
|
$bucketRow->update(['state' => 'open', 'sealed_event_id' => null]);
|
|
$assert($probe()['settlement_identity_conflicts'] === 0, 'Health rejected an exact immutable event.');
|
|
$eventRow->update(['amount_bytes' => 99]);
|
|
$assert($probe()['settlement_identity_conflicts'] === 1, 'Health accepted a mismatched immutable event.');
|
|
$eventRow->delete();
|
|
$raceSeal = true;
|
|
$capsule->getConnection()->beforeExecuting(static function ($sql) use ($event, $capsule, &$raceSeal): void {
|
|
if ($raceSeal && str_starts_with(strtolower($sql), 'select * from')
|
|
&& str_contains($sql, 'cloud_traffic_ledger_events') && str_contains($sql, 'event_key')) {
|
|
$raceSeal = false;
|
|
$peer = $capsule->getConnection('peer');
|
|
$peer->transaction(static function () use ($peer, $event): void {
|
|
$peer->table('cloud_traffic_ledger_events')->insert($event);
|
|
$peer->table('cloud_traffic_usage_settlement_buckets')->where('id', 1)
|
|
->update(['state' => 'sealed', 'sealed_event_id' => 1]);
|
|
});
|
|
}
|
|
});
|
|
$assert($probe()['settlement_identity_conflicts'] === 0 && !$raceSeal && $probe()['open_buckets_due'] === 0,
|
|
'A concurrent valid seal produced an identity alarm.');
|
|
|
|
// Commit a peer update after plan validation but before the second approved action.
|
|
foreach ([3, 4] as $id) {
|
|
Capsule::table('cloud_traffic_usage_settlement_buckets')->insert(array_replace($bucket, [
|
|
'id' => $id, 'event_identity' => random_bytes(16),
|
|
]));
|
|
}
|
|
$racePlan = CloudTrafficSettlementRecovery::inspect();
|
|
$countdown = 2;
|
|
$capsule->getConnection()->beforeStartingTransaction(static function () use (&$countdown, $capsule): void {
|
|
if (--$countdown === 0) {
|
|
$capsule->getConnection('peer')->table('cloud_traffic_usage_settlement_buckets')
|
|
->where('id', 4)->update(['amount_bytes' => 90, 'download_bytes' => 87]);
|
|
}
|
|
});
|
|
$partial = CloudTrafficSettlementRecovery::apply($racePlan['plan_hash']);
|
|
$assert($partial['status'] === 'stopped' && $partial['completed']['due_buckets'] === [3]
|
|
&& $partial['remaining']['due_buckets'] === [4]
|
|
&& $partial['failed']['reason'] === 'traffic_settlement_recovery_plan_changed'
|
|
&& Capsule::table('cloud_traffic_ledger_events')->count() === 2,
|
|
'A 10-byte approval consumed a concurrent 90-byte bucket or lost partial progress.');
|
|
$fresh = CloudTrafficSettlementRecovery::inspect();
|
|
$assert(CloudTrafficSettlementRecovery::apply($fresh['plan_hash'])['sealed_buckets'] === 1,
|
|
'A freshly reviewed plan did not resume from the remaining bucket.');
|
|
|
|
Capsule::table('cloud_instances')->insert(['id' => 11, 'user_id' => 7,
|
|
'traffic_synced_at' => $now, 'lifetime_download_bytes' => 10]);
|
|
Capsule::table('cloud_service_periods')->insert(['id' => 101, 'instance_id' => 11,
|
|
'user_id' => 7, 'traffic_pool_group_id' => 1, 'status' => 'active',
|
|
'traffic_quota_bytes' => 1000, 'starts_at' => $now - 100, 'ends_at' => $now + 10000]);
|
|
Capsule::table('cloud_traffic_entitlements')->insert(['id' => 41, 'pool_id' => 1,
|
|
'instance_id' => 11, 'user_id' => 7, 'service_period_id' => 101, 'quota_mode' => 'finite',
|
|
'granted_bytes' => 1000, 'state' => 'active', 'starts_at' => $now - 100, 'ends_at' => $now + 10000]);
|
|
Capsule::table('cloud_traffic_instance_cursors')->insert(['instance_id' => 11, 'user_id' => 7,
|
|
'pool_id' => 1, 'accounted_download_bytes' => 0, 'accounted_upload_bytes' => 0,
|
|
'last_observed_at' => $now - 60, 'state' => 'ready', 'attention_code' => '', 'version' => 1, 'updated_at' => $now]);
|
|
$plan = CloudTrafficSettlementRecovery::inspect();
|
|
$changeSample = true;
|
|
$capsule->getConnection()->beforeStartingTransaction(static function () use (&$changeSample, $capsule): void {
|
|
if ($changeSample) {
|
|
$changeSample = false;
|
|
$capsule->getConnection('peer')->table('cloud_instances')->where('id', 11)->update(['lifetime_download_bytes' => 90]);
|
|
}
|
|
});
|
|
$stopped = CloudTrafficSettlementRecovery::apply($plan['plan_hash']);
|
|
$assert($stopped['failed']['kind'] === 'unsettled_instances'
|
|
&& $stopped['failed']['reason'] === 'traffic_settlement_recovery_plan_changed'
|
|
&& Capsule::table('cloud_traffic_instance_cursors')->value('accounted_download_bytes') == 0,
|
|
'The recovery writer advanced to an unapproved sample.');
|
|
Capsule::table('cloud_traffic_instance_cursors')->update(['accounted_download_bytes' => 90, 'last_observed_at' => $now]);
|
|
Capsule::table('cloud_service_periods')->where('id', 101)->update(['traffic_quota_bytes' => 1100]);
|
|
$plan = CloudTrafficSettlementRecovery::inspect();
|
|
$changeQuota = true;
|
|
$capsule->getConnection()->beforeStartingTransaction(static function () use (&$changeQuota, $capsule): void {
|
|
if ($changeQuota) {
|
|
$changeQuota = false;
|
|
$capsule->getConnection('peer')->table('cloud_service_periods')->where('id', 101)->update(['traffic_quota_bytes' => 2000]);
|
|
}
|
|
});
|
|
$stopped = CloudTrafficSettlementRecovery::apply($plan['plan_hash']);
|
|
$assert($stopped['failed']['kind'] === 'unsettled_periods'
|
|
&& $stopped['failed']['reason'] === 'traffic_settlement_recovery_plan_changed'
|
|
&& Capsule::table('cloud_traffic_entitlements')->value('granted_bytes') == 1000,
|
|
'The recovery writer granted an unapproved quota.');
|
|
|
|
Capsule::table('cloud_service_periods')->where('id', 101)->update(['traffic_quota_bytes' => 1000]);
|
|
$failure = ['id' => 1, 'scan_kind' => 'instance', 'subject_id' => 11, 'attempt_count' => 1,
|
|
'last_failed_at' => $now, 'first_failed_at' => $now, 'retry_at' => $now + 30, 'last_error_code' => 'Test'];
|
|
Capsule::table('cloud_traffic_scan_failures')->insert($failure);
|
|
$cleanup = CloudTrafficSettlementRecovery::inspect();
|
|
$assert($cleanup['counts']['scan_failures'] === 1 && !$cleanup['counts']['unsettled_instances'],
|
|
'Already settled failure has no manual recovery path.');
|
|
$race = true;
|
|
$capsule->getConnection()->beforeExecuting(static function ($sql) use (&$race, $capsule): void {
|
|
if ($race && str_starts_with(strtolower($sql), 'delete from')) {
|
|
$race = false;
|
|
$capsule->getConnection('peer')->table('cloud_traffic_scan_failures')->where('id', 1)
|
|
->update(['attempt_count' => 2]);
|
|
}
|
|
});
|
|
$stopped = CloudTrafficSettlementRecovery::apply($cleanup['plan_hash']);
|
|
$assert($stopped['failed']['kind'] === 'scan_failures' && Capsule::table('cloud_traffic_scan_failures')->count() === 1,
|
|
'Cleanup erased a newer failure generation.');
|
|
$cleanup = CloudTrafficSettlementRecovery::inspect();
|
|
$assert(CloudTrafficSettlementRecovery::apply($cleanup['plan_hash'])['cleared_scan_failures'] === 1,
|
|
'Reviewed settled failure was left behind.');
|
|
Capsule::table('cloud_traffic_scan_failures')->insert($failure);
|
|
Capsule::table('cloud_traffic_instance_cursors')->update(['state' => 'attention']);
|
|
$assert(CloudTrafficSettlementRecovery::inspect()['counts']['scan_failures'] === 0
|
|
&& !App\Services\Cloud\CloudTrafficScanRecovery::clearIfResolved($failure, $now),
|
|
'An attention cursor was treated as successfully recovered.');
|
|
Capsule::table('cloud_traffic_instance_cursors')->update(['state' => 'ready']);
|
|
$cleanup = CloudTrafficSettlementRecovery::inspect();
|
|
$changeOnCleanup = true;
|
|
$capsule->getConnection()->beforeExecuting(static function ($sql) use (&$changeOnCleanup, $capsule): void {
|
|
if ($changeOnCleanup && str_starts_with(strtolower($sql), 'delete from')) {
|
|
$changeOnCleanup = false;
|
|
$capsule->getConnection('peer')->table('cloud_instances')->where('id', 11)
|
|
->update(['lifetime_download_bytes' => 100]);
|
|
}
|
|
});
|
|
$result = CloudTrafficSettlementRecovery::apply($cleanup['plan_hash']);
|
|
$assert(!$changeOnCleanup && $result['status'] === 'stopped'
|
|
&& Capsule::table('cloud_traffic_scan_failures')->count() === 1,
|
|
'Cleanup erased a failure while a concurrently committed sample was unsettled.');
|
|
|
|
Capsule::table('cloud_traffic_pools')->where('id', 1)->update(['overage_bytes' => 0]);
|
|
$plan = CloudTrafficSettlementRecovery::inspect();
|
|
$expected = 100;
|
|
if ($capsule->getConnection()->getDriverName() === 'mysql') {
|
|
$changeDuringWrite = true;
|
|
$capsule->getConnection()->beforeExecuting(static function ($sql) use (&$changeDuringWrite, $capsule): void {
|
|
if ($changeDuringWrite && str_starts_with(strtolower($sql), 'update `cloud_traffic_instance_cursors`')) {
|
|
$changeDuringWrite = false;
|
|
$capsule->getConnection('peer')->table('cloud_instances')->where('id', 11)
|
|
->update(['lifetime_download_bytes' => 105]);
|
|
}
|
|
});
|
|
$stopped = CloudTrafficSettlementRecovery::apply($plan['plan_hash']);
|
|
$assert(!$changeDuringWrite && $stopped['failed']['reason'] === 'traffic_settlement_recovery_plan_changed'
|
|
&& (int) Capsule::table('cloud_traffic_entitlements')->value('consumed_bytes') === 0
|
|
&& (int) Capsule::table('cloud_traffic_instance_cursors')->value('accounted_download_bytes') === 90,
|
|
'A changed sample did not roll back the entire manual accounting write.');
|
|
$plan = CloudTrafficSettlementRecovery::inspect();
|
|
$expected = 105;
|
|
}
|
|
$result = CloudTrafficSettlementRecovery::apply($plan['plan_hash']);
|
|
$assert($result['status'] === 'applied' && $result['settled_instances'] === 1 && $result['cleared_scan_failures'] === 1
|
|
&& (int) Capsule::table('cloud_traffic_entitlements')->value('consumed_bytes') === $expected - 90
|
|
&& (int) Capsule::table('cloud_traffic_instance_cursors')->value('accounted_download_bytes') === $expected
|
|
&& (int) Capsule::table('cloud_traffic_usage_settlement_buckets')->where('entitlement_id', 41)->sum('amount_bytes') === $expected - 90,
|
|
'The reviewed sample was not booked exactly once with its failure retired: ' . json_encode($result));
|
|
$empty = CloudTrafficSettlementRecovery::inspect();
|
|
$assert(CloudTrafficSettlementRecovery::apply($empty['plan_hash'])['settled_instances'] === 0,
|
|
'A successfully booked sample was selected again.');
|
|
Capsule::table('cloud_instances')->where('id', 11)->update(['lifetime_download_bytes' => $expected - 1]);
|
|
$regression = CloudTrafficSettlementRecovery::inspect();
|
|
$result = CloudTrafficSettlementRecovery::apply($regression['plan_hash']);
|
|
$assert($result['status'] === 'stopped' && $result['failed']['reason'] === 'traffic_counter_regression'
|
|
&& $result['completed']['unsettled_instances'] === [] && $result['remaining']['unsettled_instances'] === [11]
|
|
&& Capsule::table('cloud_traffic_instance_cursors')->value('state') === 'attention'
|
|
&& (int) Capsule::table('cloud_traffic_entitlements')->value('consumed_bytes') === $expected - 90,
|
|
'A regressed counter was reported as settled or lost its durable attention marker.');
|
|
Capsule::table('cloud_traffic_usage_settlement_buckets')->insert(array_merge(
|
|
$bucket,
|
|
[
|
|
'id' => 2,
|
|
'event_identity' => '',
|
|
'state' => 'open',
|
|
'sealed_event_id' => null,
|
|
]
|
|
));
|
|
$blocked = CloudTrafficSettlementRecovery::inspect();
|
|
$assert($blocked['actionable'] === false
|
|
&& $blocked['blockers'][0]['code']
|
|
=== 'settlement_event_identity_missing',
|
|
'A malformed persistent event identity did not block recovery.');
|
|
try {
|
|
CloudTrafficSettlementRecovery::apply($blocked['plan_hash']);
|
|
throw new RuntimeException('A blocked recovery plan was applied.');
|
|
} catch (RuntimeException $error) {
|
|
$assert($error->getMessage() === 'traffic_settlement_recovery_blocked',
|
|
'Blocked plan failed with the wrong safety code.');
|
|
}
|
|
|
|
$parse = new ReflectionMethod(
|
|
CloudTrafficSettlementRecoveryCommand::class,
|
|
'parse'
|
|
);
|
|
$assert($parse->invoke(null, [
|
|
'xcat', 'cloudTrafficSettlementRecovery', 'inspect',
|
|
]) === ['action' => 'inspect'], 'CLI inspect arguments drifted.');
|
|
$hash = str_repeat('a', 64);
|
|
$assert($parse->invoke(null, [
|
|
'xcat', 'cloudTrafficSettlementRecovery', 'apply',
|
|
'--plan-hash=' . $hash, '--apply',
|
|
]) === ['action' => 'apply', 'plan_hash' => $hash],
|
|
'CLI apply did not bind the exact reviewed plan hash.');
|
|
foreach ([
|
|
['apply', '--plan-hash=' . $hash],
|
|
['apply', '--plan-hash=bad', '--apply'],
|
|
['apply', '--plan-hash=' . $hash, '--apply', '--force'],
|
|
['inspect', '--apply'],
|
|
] as $arguments) {
|
|
try {
|
|
$parse->invoke(null, array_merge([
|
|
'xcat', 'cloudTrafficSettlementRecovery',
|
|
], $arguments));
|
|
throw new RuntimeException('Unsafe CLI arguments were accepted.');
|
|
} catch (ReflectionException $error) {
|
|
throw $error;
|
|
} catch (Throwable $error) {
|
|
$actual = $error->getPrevious() ?? $error;
|
|
$assert($actual instanceof InvalidArgumentException,
|
|
'Unsafe CLI arguments failed with the wrong exception.');
|
|
}
|
|
}
|
|
|
|
$service = file_get_contents(dirname(__DIR__)
|
|
. '/app/Services/Cloud/CloudTrafficSettlementRecovery.php');
|
|
$xcat = file_get_contents(dirname(__DIR__) . '/app/Command/XCat.php');
|
|
$assert(is_string($service)
|
|
&& is_string($xcat)
|
|
&& !str_contains($service, '->delete(')
|
|
&& !str_contains($service, 'CloudAgentClient')
|
|
&& !str_contains($service, 'CloudJob')
|
|
&& str_contains($service,
|
|
'CloudTrafficPoolAccounting::sealPlannedUsageBuckets(')
|
|
&& str_contains($service,
|
|
'CloudTrafficPoolAccounting::reconcileServicePeriod(')
|
|
&& str_contains($service,
|
|
'CloudTrafficPoolAccounting::settleInstanceUsage(')
|
|
&& str_contains($xcat, "'cloudTrafficSettlementRecovery'")
|
|
&& str_contains($xcat,
|
|
'cloudTrafficSettlementRecovery apply --plan-hash=HASH --apply'),
|
|
'Recovery acquired a destructive or remote bypass path.');
|
|
|
|
echo "Traffic settlement recovery plan/apply checks passed.\n";
|