You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
163 lines
7.8 KiB
PHTML
163 lines
7.8 KiB
PHTML
|
1 month ago
|
<?php
|
||
|
|
|
||
|
|
namespace App\Services\Cloud {
|
||
|
|
// Reconciliation in this suite must settle from durable local evidence;
|
||
|
|
// fail immediately if it unexpectedly reaches the private HTTP boundary.
|
||
|
|
final class CloudEngineClient
|
||
|
|
{
|
||
|
|
public static int $requests = 0;
|
||
|
|
|
||
|
|
public function __construct(...$arguments)
|
||
|
|
{
|
||
|
|
}
|
||
|
|
|
||
|
|
public function vpc(...$arguments): array
|
||
|
|
{
|
||
|
|
self::$requests++;
|
||
|
|
throw new \RuntimeException('Candidate recovery must not contact the engine');
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
namespace {
|
||
|
|
use App\Services\Cloud\CloudEngineClient;
|
||
|
|
use App\Services\Cloud\CloudVpcPageService;
|
||
|
|
use App\Services\Cloud\CloudVpcService;
|
||
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
||
|
|
use Illuminate\Database\Schema\Blueprint;
|
||
|
|
|
||
|
|
require __DIR__ . '/CloudVpcMemberReuseTest.php';
|
||
|
|
if (!isset($capsule)) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
$schema->table('cloud_instances', static function (Blueprint $table): void {
|
||
|
|
$table->text('spec_json')->default('{}');
|
||
|
|
});
|
||
|
|
$seedRecovery = static function (int $id, string $memberState = 'deleted') use (
|
||
|
|
$seedVpc, $seedInstance, $seedPair
|
||
|
|
): array {
|
||
|
|
$octet = $id - 900;
|
||
|
|
$vpc = $seedVpc($id, '10.98.' . $octet . '.0/24',
|
||
|
|
'10.98.' . $octet . '.0/24', 10, 'deleted', 'active');
|
||
|
|
$instanceId = $id * 100;
|
||
|
|
$memberId = $id * 1000;
|
||
|
|
$privateId = $memberId + 1;
|
||
|
|
$seedInstance($instanceId, 10);
|
||
|
|
Capsule::table('cloud_instances')->where('id', $instanceId)->update([
|
||
|
|
'spec_json' => json_encode(['region_id' => 1, 'region' => 'Test Region']),
|
||
|
|
]);
|
||
|
|
$seedPair($memberId, $privateId, $vpc, $instanceId, 10,
|
||
|
|
'10.98.' . $octet . '.1', $memberState);
|
||
|
|
return $vpc + [
|
||
|
|
'instance_id' => $instanceId, 'member_id' => $memberId,
|
||
|
|
'private_id' => $privateId, 'private_ip' => '10.98.' . $octet . '.1',
|
||
|
|
];
|
||
|
|
};
|
||
|
|
$attachment = static function (array $vpc) use ($actor, $assert): array {
|
||
|
|
$rows = array_column(CloudVpcPageService::bootstrap($actor)['instances'], null, 'id');
|
||
|
|
$assert(isset($rows[$vpc['instance_id']]), 'Recovery instance is missing from the page bootstrap.');
|
||
|
|
return $rows[$vpc['instance_id']]['vpc_attachment'];
|
||
|
|
};
|
||
|
|
$assertRetained = static function (array $vpc) use ($assert, $attachment): void {
|
||
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', $vpc['member_id'])->count() === 1
|
||
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', $vpc['private_id'])->count() === 1
|
||
|
|
&& $attachment($vpc)['state'] === 'cleanup_pending',
|
||
|
|
'An unproven or in-flight identity was released or became selectable.');
|
||
|
|
};
|
||
|
|
|
||
|
|
$recoverable = $seedRecovery(980);
|
||
|
|
$beforeRoot = (array) Capsule::table('cloud_vpcs')->where('id', 980)->first();
|
||
|
|
$beforeOperation = (array) Capsule::table('cloud_vpc_operations')
|
||
|
|
->where('operation_id', $recoverable['operation_id'])->first();
|
||
|
|
CloudVpcService::show($recoverable['vpc_key'], $actor);
|
||
|
|
$assertRetained($recoverable);
|
||
|
|
foreach ([false, true] as $allowDeferredRetry) {
|
||
|
|
$expectCloudError(static fn () => CloudVpcService::reconcile(
|
||
|
|
$recoverable['vpc_key'], $actor, $allowDeferredRetry
|
||
|
|
), 409);
|
||
|
|
$assertRetained($recoverable);
|
||
|
|
}
|
||
|
|
CloudVpcService::reconcile($recoverable['vpc_key'], $actor, true, true);
|
||
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', $recoverable['member_id'])->count() === 0
|
||
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', $recoverable['private_id'])->count() === 0
|
||
|
|
&& $attachment($recoverable) === ['state' => 'available', 'vpc_key' => null, 'vpc_name' => null]
|
||
|
|
&& (array) Capsule::table('cloud_vpcs')->where('id', 980)->first() === $beforeRoot
|
||
|
|
&& (array) Capsule::table('cloud_vpc_operations')
|
||
|
|
->where('operation_id', $recoverable['operation_id'])->first() === $beforeOperation
|
||
|
|
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 980)->count() === 1,
|
||
|
|
'Explicit proof-based recovery must release only the old pair and leave root/revision/operation audit unchanged.');
|
||
|
|
$expectCloudError(static fn () => CloudVpcService::reconcile(
|
||
|
|
$recoverable['vpc_key'], $actor, true, true
|
||
|
|
), 409);
|
||
|
|
|
||
|
|
// A later root marker does not erase exact historical absence evidence when
|
||
|
|
// there is no operation to claim; the existing proof verifier remains owner.
|
||
|
|
$advanced = $seedRecovery(981);
|
||
|
|
Capsule::table('cloud_vpcs')->where('id', 981)->update([
|
||
|
|
'desired_revision' => 2, 'desired_digest' => hash('sha256', 'recovery-newer-root'),
|
||
|
|
'state' => 'degraded', 'sync_state' => 'uncertain',
|
||
|
|
'last_operation_id' => $keyFor('recovery-newer-root', 981),
|
||
|
|
]);
|
||
|
|
CloudVpcService::reconcile($advanced['vpc_key'], $actor, true, true);
|
||
|
|
$assert($attachment($advanced)['state'] === 'available'
|
||
|
|
&& (int) Capsule::table('cloud_vpcs')->where('id', 981)->value('desired_revision') === 2
|
||
|
|
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 981)->count() === 1,
|
||
|
|
'Recovery either lost valid historical proof or created a replacement operation.');
|
||
|
|
|
||
|
|
$missingProof = $seedRecovery(982);
|
||
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $missingProof['operation_id'])
|
||
|
|
->update(['observed_json' => null]);
|
||
|
|
$deleting = $seedRecovery(983, 'deleting');
|
||
|
|
$referenced = $seedRecovery(984);
|
||
|
|
$seedRoute(984000, $referenced, $referenced['member_id'], '10.99.0.1/32', $referenced['private_ip']);
|
||
|
|
$rootDeleting = $seedRecovery(985);
|
||
|
|
Capsule::table('cloud_vpcs')->where('id', 985)->update(['state' => 'deleting']);
|
||
|
|
foreach ([$missingProof, $deleting, $referenced, $rootDeleting] as $blocked) {
|
||
|
|
$expectCloudError(static fn () => CloudVpcService::reconcile(
|
||
|
|
$blocked['vpc_key'], $actor, true, true
|
||
|
|
), 409);
|
||
|
|
$assertRetained($blocked);
|
||
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('vpc_id', $blocked['id'])->count() === 1,
|
||
|
|
'A refused cleanup must not create an operation.');
|
||
|
|
}
|
||
|
|
|
||
|
|
// A real pending target takes precedence even with a valid older removal
|
||
|
|
// proof. Its live lease returns the ordinary document without GC or replay.
|
||
|
|
$inFlight = $seedRecovery(986);
|
||
|
|
$historical = (array) Capsule::table('cloud_vpc_operations')
|
||
|
|
->where('operation_id', $inFlight['operation_id'])->first();
|
||
|
|
$pendingDocument = json_decode($historical['payload_json'], true);
|
||
|
|
unset($pendingDocument['operation_id'], $pendingDocument['desired_digest']);
|
||
|
|
$pendingDocument['desired_revision'] = 2;
|
||
|
|
$pendingDigest = hash('sha256', $canonicalJson($pendingDocument));
|
||
|
|
$pendingId = $keyFor('candidate-recovery-pending', 986);
|
||
|
|
$pendingDocument['operation_id'] = $pendingId;
|
||
|
|
$pendingDocument['desired_digest'] = $pendingDigest;
|
||
|
|
$pending = $historical;
|
||
|
|
unset($pending['id']);
|
||
|
|
$pending = array_replace($pending, [
|
||
|
|
'operation_id' => $pendingId, 'desired_revision' => 2,
|
||
|
|
'idempotency_key' => 'candidate-recovery-pending-986',
|
||
|
|
'request_hash' => hash('sha256', 'candidate-recovery-pending-986'),
|
||
|
|
'status' => 'running', 'payload_json' => json_encode($pendingDocument),
|
||
|
|
'observed_json' => null, 'finished_at' => null,
|
||
|
|
'lease_token' => str_repeat('a', 32), 'lease_until' => time() + 300,
|
||
|
|
]);
|
||
|
|
Capsule::table('cloud_vpc_operations')->insert($pending);
|
||
|
|
Capsule::table('cloud_vpcs')->where('id', 986)->update([
|
||
|
|
'desired_revision' => 2, 'desired_digest' => $pendingDigest,
|
||
|
|
'state' => 'degraded', 'sync_state' => 'uncertain', 'last_operation_id' => $pendingId,
|
||
|
|
]);
|
||
|
|
foreach (['queued', 'running', 'uncertain', 'waiting_observed'] as $status) {
|
||
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $pendingId)->update(['status' => $status]);
|
||
|
|
CloudVpcService::reconcile($inFlight['vpc_key'], $actor, true, true);
|
||
|
|
$assertRetained($inFlight);
|
||
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $pendingId)->value('status') === $status
|
||
|
|
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', 986)->count() === 2,
|
||
|
|
'A live operation was bypassed, replayed or replaced by candidate cleanup.');
|
||
|
|
}
|
||
|
|
$assert(CloudEngineClient::$requests === 0, 'Candidate cleanup unexpectedly contacted the engine.');
|
||
|
|
|
||
|
|
echo "VPC member candidate recovery runtime passed.\n";
|
||
|
|
}
|