You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudVpcPortWorkflowTest.php

736 lines
33 KiB
PHP

<?php
namespace App\Services\Cloud {
// Isolated test cipher; production CloudSecrets is not loaded here.
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'vpc-test:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
return self::$values[substr($value, 9)];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstance;
use App\Models\CloudInstanceMigration;
use App\Models\CloudJob;
use App\Models\CloudVpc;
use App\Models\CloudVpcHostState;
use App\Models\CloudVpcPort;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudVpcPortService;
use App\Services\Cloud\CloudVpcProjection;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$terminalize = static function (CloudJob $job): void {
$now = time();
$job->phase = 'terminal';
$job->outcome = $job->status = 'succeeded';
$job->attempts = $job->dispatch_attempts = 1;
$job->dispatch_started_at = $job->dispatched_at = $now - 1;
$job->agent_accepted_at = $now - 1;
$job->receipt_deadline_at = $now + 900;
$job->terminal_at = $now;
$job->effect_may_have_applied = 0;
$job->safe_to_resubmit = 0;
$job->effect_resolution = 'applied';
$job->updated_at = $now;
$job->save();
};
$terminalizeFailed = static function (CloudJob $job): void {
$finishedAt = time() - 600;
$job->phase = 'terminal';
$job->outcome = $job->status = 'failed';
$job->attempts = $job->dispatch_attempts = 1;
$job->dispatch_started_at = $job->dispatched_at = $finishedAt - 1;
$job->agent_accepted_at = $finishedAt - 1;
$job->receipt_deadline_at = $finishedAt + 900;
$job->terminal_at = $finishedAt;
$job->effect_may_have_applied = 0;
$job->safe_to_resubmit = 1;
$job->effect_resolution = 'not_applied';
$job->error_code = 'vpc_host_remove_failed';
$job->updated_at = $finishedAt;
$job->save();
};
$convergeRoutingJobs = static function (
array $jobs
) use ($terminalize): void {
foreach ($jobs as $job) {
$state = CloudVpcHostState::where(
'vpc_id', (int) $job->vpc_id
)->where('host_id', (int) $job->host_id)->first();
if ((string) $job->kind === 'vpc_host_remove') {
if ($state === null) {
throw new RuntimeException(
'Route-remove test job lost its host state.'
);
}
$state->delete();
} else {
if ($state === null) {
throw new RuntimeException(
'Route-apply test job lost its host state.'
);
}
$state->applied_revision = (int) $state->desired_revision;
$state->applied_digest = (string) $state->desired_digest;
$state->sync_state = 'ready';
$state->last_error_code = '';
$state->save();
}
$terminalize($job);
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_network_lines', static function (
Blueprint $table
): void {
$table->increments('id'); $table->integer('region_id');
});
$schema->create('cloud_resource_pools', static function (
Blueprint $table
): void {
$table->increments('id'); $table->integer('line_id');
});
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->increments('id'); $table->boolean('enabled');
});
$schema->create('cloud_host_vpc_transports', static function (
Blueprint $table
): void {
$table->integer('host_id')->primary();
$table->string('transport_address');
$table->integer('transport_prefix_length');
$table->string('transport_interface'); $table->integer('mtu');
$table->integer('desired_revision');
$table->integer('observed_revision'); $table->string('sync_state');
});
$schema->create('cloud_instances', static function (
Blueprint $table
): void {
$table->increments('id'); $table->string('uuid');
$table->integer('host_id'); $table->integer('user_id');
$table->integer('resource_pool_id')->nullable();
$table->string('name'); $table->string('status');
$table->string('service_status'); $table->string('network_status');
$table->integer('expires_at')->nullable(); $table->text('spec_json');
$table->integer('created_at'); $table->integer('updated_at');
});
$schema->create('cloud_vpcs', static function (Blueprint $table): void {
$table->increments('id'); $table->string('vpc_key');
$table->integer('user_id'); $table->integer('region_id');
$table->integer('cidr_pool_id'); $table->string('name');
$table->string('cidr'); $table->integer('prefix_length');
$table->string('gateway'); $table->integer('active_claim')->nullable();
$table->string('state'); $table->integer('desired_revision');
$table->integer('deleted_at')->nullable();
$table->integer('created_at'); $table->integer('updated_at');
});
$schema->create('cloud_vpc_ports', static function (
Blueprint $table
): void {
$table->increments('id'); $table->string('port_key')->unique();
$table->integer('vpc_id'); $table->integer('instance_id');
$table->integer('migration_host_id')->nullable();
$table->integer('user_id'); $table->string('private_ip');
$table->string('mac_address')->unique();
$table->integer('active_claim')->nullable(); $table->string('state');
$table->integer('desired_revision');
$table->integer('applied_revision');
$table->string('last_operation_id')->nullable();
$table->string('last_error_code');
$table->integer('attached_at')->nullable();
$table->integer('released_at')->nullable();
$table->integer('created_at'); $table->integer('updated_at');
$table->unique(['vpc_id', 'private_ip', 'active_claim']);
$table->unique(['instance_id', 'active_claim']);
});
$schema->create('cloud_vpc_host_states', static function (
Blueprint $table
): void {
$table->increments('id'); $table->integer('vpc_id');
$table->integer('host_id'); $table->integer('desired_revision');
$table->integer('applied_revision'); $table->string('desired_digest');
$table->string('applied_digest')->nullable();
$table->string('sync_state');
$table->string('last_operation_id')->nullable();
$table->string('last_error_code');
$table->integer('last_success_at');
$table->integer('last_failure_at');
$table->integer('created_at'); $table->integer('updated_at');
$table->unique(['vpc_id', 'host_id']);
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id'); $table->string('operation_id')->unique();
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('backup_id')->nullable();
$table->integer('backup_restore_id')->nullable();
$table->integer('backup_recovery_id')->nullable();
$table->integer('vpc_id')->nullable();
$table->integer('vpc_port_id')->nullable();
$table->integer('user_id'); $table->integer('host_id')->nullable();
$table->string('kind'); $table->string('execution_mode');
$table->string('phase'); $table->string('outcome')->nullable();
$table->string('status'); $table->text('sealed_payload');
$table->integer('attempts'); $table->integer('dispatch_attempts');
$table->integer('dispatch_deadline_at');
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->text('effect_resolution_reason')->default('');
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at'); $table->string('error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at'); $table->integer('updated_at');
$table->unique(['user_id', 'request_idempotency_key']);
});
Capsule::table('cloud_network_lines')->insert([
'id' => 1, 'region_id' => 1,
]);
Capsule::table('cloud_resource_pools')->insert([
'id' => 1, 'line_id' => 1,
]);
foreach ([1, 2] as $hostId) {
Capsule::table('cloud_hosts')->insert([
'id' => $hostId, 'enabled' => 1,
]);
Capsule::table('cloud_host_vpc_transports')->insert([
'host_id' => $hostId,
'transport_address' => '172.16.10.' . (10 + $hostId),
'transport_prefix_length' => 24,
'transport_interface' => 'br-private', 'mtu' => 1500,
'desired_revision' => 1, 'observed_revision' => 1,
'sync_state' => 'ready',
]);
}
$newInstance = static function (int $id, int $hostId): CloudInstance {
$instance = new CloudInstance();
$instance->id = $id;
$instance->uuid = sprintf('11111111-1111-4111-8111-%012d', $id);
$instance->host_id = $hostId; $instance->user_id = 7;
$instance->resource_pool_id = 1; $instance->name = 'vm-' . $id;
$instance->status = 'active'; $instance->service_status = 'exempt';
$instance->network_status = 'active'; $instance->expires_at = null;
$networkSpec = $id === 2
? ['public_ip' => '203.0.113.12']
: ['public_ip_cidr' => '203.0.113.11/24'];
$instance->spec_json = json_encode(array_merge([
'os_source' => 'cloud_template',
], $networkSpec), JSON_THROW_ON_ERROR);
$instance->created_at = $instance->updated_at = time();
$instance->save();
return $instance;
};
$target = $newInstance(1, 1);
$peer = $newInstance(2, 2);
$vpc = new CloudVpc();
$vpc->id = 1; $vpc->vpc_key = str_repeat('a', 32);
$vpc->user_id = 7; $vpc->region_id = 1; $vpc->cidr_pool_id = 1;
$vpc->name = 'private'; $vpc->cidr = '10.60.0.0/24';
$vpc->prefix_length = 24; $vpc->gateway = '10.60.0.1';
$vpc->active_claim = 1; $vpc->state = 'active';
$vpc->desired_revision = 1; $vpc->deleted_at = null;
$vpc->created_at = $vpc->updated_at = time(); $vpc->save();
$peerPort = new CloudVpcPort();
$peerPort->id = 1; $peerPort->port_key = str_repeat('b', 32);
$peerPort->vpc_id = 1; $peerPort->instance_id = 2;
$peerPort->user_id = 7; $peerPort->private_ip = '10.60.0.3';
$peerPort->mac_address = '02:00:00:00:00:03';
$peerPort->active_claim = 1; $peerPort->state = 'attached';
$peerPort->desired_revision = $peerPort->applied_revision = 1;
$peerPort->last_operation_id = null; $peerPort->last_error_code = '';
$peerPort->attached_at = time(); $peerPort->released_at = null;
$peerPort->created_at = $peerPort->updated_at = time(); $peerPort->save();
$initial = CloudVpcProjection::build([
'vpc_key' => (string) $vpc->vpc_key, 'cidr' => (string) $vpc->cidr,
'gateway' => (string) $vpc->gateway,
], [['private_ip' => '10.60.0.3',
'mac_address' => '02:00:00:00:00:03', 'host_id' => 2]], [
2 => ['transport_address' => '172.16.10.12',
'transport_prefix_length' => 24,
'transport_interface' => 'br-private', 'mtu' => 1500],
], 1);
$state = new CloudVpcHostState();
$state->vpc_id = 1; $state->host_id = 2;
$state->desired_revision = $state->applied_revision = 1;
$state->desired_digest = $state->applied_digest = $initial[2]['digest'];
$state->sync_state = 'ready'; $state->last_operation_id = null;
$state->last_error_code = ''; $state->last_success_at = time();
$state->last_failure_at = 0;
$state->created_at = $state->updated_at = time(); $state->save();
$management = CloudVpcPortService::managementListing(['user_id' => 7]);
$targetManagement = collect($management)->firstWhere('id', '1');
$peerManagement = collect($management)->firstWhere('id', '2');
$assert(($targetManagement['region_id'] ?? null) === 1
&& ($targetManagement['public_ipv4'] ?? null) === '203.0.113.11'
&& array_key_exists('port', $targetManagement)
&& $targetManagement['port'] === null
&& ($targetManagement['can_attach'] ?? false) === true,
'The VPC page projection did not expose the eligible same-region instance.');
$assert(($peerManagement['port']['vpc_key'] ?? null)
=== (string) $vpc->vpc_key
&& ($peerManagement['public_ipv4'] ?? null) === '203.0.113.12'
&& ($peerManagement['can_detach'] ?? false) === true,
'The VPC page projection did not expose its attached instance action.');
$actor = ['user_id' => 7,
'request_idempotency_key' => 'vpc-attach-request-0001'];
$queued = CloudVpcPortService::attach($target, $actor, [
'vpc_key' => (string) $vpc->vpc_key, 'confirm_restart' => true,
]);
$assert($queued['job'] !== null
&& CloudJob::where('kind', 'vpc_host_apply')->count() === 2,
'Attach did not queue a complete two-host projection.');
$replayedAttach = CloudVpcPortService::attach($target, $actor, [
'vpc_key' => (string) $vpc->vpc_key, 'confirm_restart' => true,
]);
$activeHostJobIds = CloudJob::where('kind', 'vpc_host_apply')
->pluck('id')->map(static fn ($id): int => (int) $id)->all();
$assert($replayedAttach['job'] !== null
&& in_array((int) $replayedAttach['job']->id, $activeHostJobIds, true)
&& CloudJob::where('kind', 'vpc_host_apply')->count() === 2,
'An accepted attach retry did not return its existing workflow.');
$reconnectedAttach = CloudVpcPortService::attach($target, [
'user_id' => 7,
'request_idempotency_key' => 'vpc-attach-new-browser-0002',
], [
'vpc_key' => (string) $vpc->vpc_key, 'confirm_restart' => true,
]);
$assert($reconnectedAttach['job'] !== null
&& in_array((int) $reconnectedAttach['job']->id,
$activeHostJobIds, true)
&& CloudJob::where('kind', 'vpc_host_apply')->count() === 2,
'A reconnect with a new browser request key duplicated the accepted workflow.');
$port = CloudVpcPort::where('instance_id', 1)
->where('active_claim', 1)->first();
$assert($port !== null && (string) $port->state === 'preparing'
&& (string) $port->private_ip === '10.60.0.2',
'Attach did not reserve the first usable free address.');
$listing = CloudVpcPortService::listing($target, ['user_id' => 7]);
$assert(($listing['active_task']['task_id'] ?? null)
=== (string) $replayedAttach['job']->id
&& ($listing['can_attach'] ?? true) === false,
'The read model did not expose the tenant VPC task after reconnect.');
$management = CloudVpcPortService::managementListing(['user_id' => 7]);
$targetManagement = collect($management)->firstWhere('id', '1');
$assert(($targetManagement['port']['state'] ?? null) === 'preparing'
&& ($targetManagement['active_task']['task_id'] ?? null)
=== (string) $replayedAttach['job']->id
&& ($targetManagement['can_retry_attach'] ?? true) === false,
'The VPC page projection did not retain the active attach task fence.');
foreach (CloudJob::where('kind', 'vpc_host_apply')->orderBy('id')->get()
as $hostJob) {
$payload = CloudSecrets::decrypt(
(string) $hostJob->sealed_payload, 'job-payload'
);
$hostState = CloudVpcHostState::where('vpc_id', 1)
->where('host_id', (int) $hostJob->host_id)->first();
CloudVpcPortService::finishJobLocked($hostJob, 'succeeded', [
'vpc_key' => (string) $vpc->vpc_key,
'desired_revision' => (int) $hostState->desired_revision,
'desired_digest' => (string) $hostState->desired_digest,
'bridge' => CloudVpcProjection::bridgeName((string) $vpc->vpc_key),
'applied' => true,
], $payload, ['operation_id' => (string) $hostJob->operation_id]);
$terminalize($hostJob);
}
$attachJob = CloudJob::where('kind', 'vpc_port_attach')->first();
$assert($attachJob !== null,
'Converged host projections did not queue the VM port attach.');
$attachPayload = CloudSecrets::decrypt(
(string) $attachJob->sealed_payload, 'job-payload'
);
$body = $attachPayload['body'];
CloudVpcPortService::finishJobLocked($attachJob, 'succeeded', [
'uuid' => (string) $target->uuid,
'port_key' => (string) $port->port_key,
'vpc_key' => (string) $vpc->vpc_key,
'vpc_port' => [
'port_key' => (string) $port->port_key,
'vpc_key' => (string) $vpc->vpc_key,
'private_ip' => (string) $port->private_ip,
'cidr' => (string) $vpc->cidr,
'gateway' => (string) $vpc->gateway,
'mac' => (string) $port->mac_address,
'bridge' => CloudVpcProjection::bridgeName((string) $vpc->vpc_key),
'host_revision' => (int) $body['host_revision'],
'host_digest' => (string) $body['host_digest'],
],
'status' => 'running', 'attached' => true,
], $attachPayload, ['operation_id' => (string) $attachJob->operation_id]);
$terminalize($attachJob);
$assert((string) $port->fresh()->state === 'attached',
'Successful VM port result did not attach the port ledger.');
$migration = new CloudInstanceMigration();
$migration->id = 99;
$migration->instance_id = 1;
$migration->vpc_port_id = CloudVpcPortService::migrationPortIdLocked(
$target,
2
);
$migration->source_host_id = 1;
$migration->target_host_id = 2;
$migration->source_uuid = (string) $target->uuid;
$targetJobs = $target->getConnection()->transaction(
static fn (): array =>
CloudVpcPortService::beginMigrationToTargetLocked(
$migration,
$target
)
);
$migrationPayload = CloudSecrets::decrypt(
(string) $targetJobs[0]->sealed_payload,
'job-payload'
);
$assert(count($targetJobs) === 2
&& ($migrationPayload['maintenance_migration_id'] ?? null) === 99
&& ($migrationPayload['vpc_migration_direction'] ?? null)
=== 'target'
&& (string) $port->fresh()->state === 'migrating'
&& (int) $port->fresh()->migration_host_id === 2,
'Maintenance migration did not durably project the port to target.');
$convergeRoutingJobs($targetJobs);
$assert(CloudVpcPortService::finishMigrationProjectionLocked(
$migration,
$target,
'target'
) === true,
'Converged target VPC projection did not complete.');
$targetMetadata = CloudVpcPortService::migrationTargetMetadataLocked(
$migration,
$target
);
$assert(($targetMetadata['private_ip'] ?? null) === '10.60.0.2'
&& ($targetMetadata['mac'] ?? null)
=== (string) $port->mac_address
&& ($targetMetadata['host_revision'] ?? 0) > 1,
'Migration target metadata did not preserve the logical port identity.');
CloudVpcPortService::finalizeMigrationCutoverLocked($migration, $target);
$cutoverPort = $port->fresh();
$assert((string) $cutoverPort->state === 'attached'
&& $cutoverPort->migration_host_id === null
&& (string) $cutoverPort->private_ip === '10.60.0.2'
&& (string) $cutoverPort->mac_address
=== (string) $port->mac_address,
'VPC cutover changed or retained migration state on the logical port.');
$target->host_id = 2;
$target->save();
$reverse = new CloudInstanceMigration();
$reverse->id = 100;
$reverse->instance_id = 1;
$reverse->vpc_port_id = CloudVpcPortService::migrationPortIdLocked(
$target,
1
);
$reverse->source_host_id = 2;
$reverse->target_host_id = 1;
$reverse->source_uuid = (string) $target->uuid;
$reverseJobs = $target->getConnection()->transaction(
static fn (): array =>
CloudVpcPortService::beginMigrationToTargetLocked(
$reverse,
$target
)
);
$convergeRoutingJobs($reverseJobs);
$assert(CloudVpcPortService::finishMigrationProjectionLocked(
$reverse,
$target,
'target'
),
'Reverse target projection did not converge.');
CloudVpcPortService::finalizeMigrationCutoverLocked($reverse, $target);
$target->host_id = 1;
$target->save();
$abort = new CloudInstanceMigration();
$abort->id = 101;
$abort->instance_id = 1;
$abort->vpc_port_id = CloudVpcPortService::migrationPortIdLocked(
$target,
2
);
$abort->source_host_id = 1;
$abort->target_host_id = 2;
$abort->source_uuid = (string) $target->uuid;
$abortTargetJobs = $target->getConnection()->transaction(
static fn (): array =>
CloudVpcPortService::beginMigrationToTargetLocked(
$abort,
$target
)
);
$unknownJob = $abortTargetJobs[0];
foreach ($abortTargetJobs as $routeJob) {
$state = CloudVpcHostState::where('vpc_id', 1)
->where('host_id', (int) $routeJob->host_id)->first();
if ((int) $routeJob->id === (int) $unknownJob->id) {
$state->sync_state = 'unknown';
$state->last_operation_id = (string) $routeJob->operation_id;
$state->last_error_code = 'vpc_result_uncertain';
$state->save();
$routeJob->phase = 'terminal';
$routeJob->outcome = $routeJob->status = 'failed';
$routeJob->effect_may_have_applied = 1;
$routeJob->safe_to_resubmit = 0;
$routeJob->effect_resolution = 'unknown';
$routeJob->terminal_at = $routeJob->updated_at = time();
$routeJob->save();
continue;
}
if ((string) $routeJob->kind === 'vpc_host_remove') {
$state->delete();
} else {
$state->applied_revision = (int) $state->desired_revision;
$state->applied_digest = (string) $state->desired_digest;
$state->sync_state = 'ready';
$state->save();
}
$terminalize($routeJob);
}
$jobsBeforeUnknownAbort = CloudJob::count();
$unknownReplayBlocked = false;
try {
$target->getConnection()->transaction(
static fn (): array =>
CloudVpcPortService::beginMigrationToSourceLocked(
$abort,
$target
)
);
} catch (Throwable $error) {
$unknownReplayBlocked = true;
}
$assert($unknownReplayBlocked
&& CloudJob::count() === $jobsBeforeUnknownAbort,
'Unknown VPC routing effect was overwritten by source rollback.');
$unknownState = CloudVpcHostState::where('vpc_id', 1)
->where('host_id', (int) $unknownJob->host_id)->first();
$unknownState->sync_state = 'failed';
$unknownState->save();
$terminalizeFailed($unknownJob);
$sourceJobs = $target->getConnection()->transaction(
static fn (): array =>
CloudVpcPortService::beginMigrationToSourceLocked(
$abort,
$target
)
);
$convergeRoutingJobs($sourceJobs);
$assert(CloudVpcPortService::finishMigrationProjectionLocked(
$abort,
$target,
'source'
)
&& (string) $port->fresh()->state === 'attached'
&& $port->fresh()->migration_host_id === null,
'Confirmed-not-applied abort did not restore the source VPC route.');
$detachActor = ['user_id' => 7,
'request_idempotency_key' => 'vpc-detach-request-0001'];
$detaching = CloudVpcPortService::detach($target, $detachActor, [
'confirm_restart' => true,
]);
$detachJob = $detaching['job'];
$replayedDetach = CloudVpcPortService::detach($target, $detachActor, [
'confirm_restart' => true,
]);
$assert((int) ($replayedDetach['job']->id ?? 0) === (int) $detachJob->id
&& CloudJob::where('kind', 'vpc_port_detach')->count() === 1,
'An accepted detach retry did not return its existing workflow.');
$detachPayload = CloudSecrets::decrypt(
(string) $detachJob->sealed_payload, 'job-payload'
);
CloudVpcPortService::finishJobLocked($detachJob, 'succeeded', [
'uuid' => (string) $target->uuid,
'port_key' => (string) $port->port_key,
'vpc_key' => (string) $vpc->vpc_key,
'status' => 'running', 'removed' => true,
], $detachPayload, ['operation_id' => (string) $detachJob->operation_id]);
$terminalize($detachJob);
$assert(CloudJob::whereIn('kind',
['vpc_host_apply', 'vpc_host_remove'])->where('phase', 'accepted')
->count() === 2,
'Detach did not queue route cleanup after VM port removal.');
$cleanupJobs = CloudJob::whereIn('kind',
['vpc_host_apply', 'vpc_host_remove'])->where('phase', 'accepted')
->orderBy('id')->get();
foreach ($cleanupJobs as $cleanupJob) {
$payload = CloudSecrets::decrypt(
(string) $cleanupJob->sealed_payload, 'job-payload'
);
$hostState = CloudVpcHostState::where('vpc_id', 1)
->where('host_id', (int) $cleanupJob->host_id)->first();
$result = [
'vpc_key' => (string) $vpc->vpc_key,
'desired_revision' => (int) ($cleanupJob->kind === 'vpc_host_apply'
? $hostState->desired_revision : $hostState->applied_revision),
'desired_digest' => (string) ($cleanupJob->kind === 'vpc_host_apply'
? $hostState->desired_digest : $hostState->applied_digest),
];
if ((string) $cleanupJob->kind === 'vpc_host_apply') {
$result['bridge'] = CloudVpcProjection::bridgeName(
(string) $vpc->vpc_key
);
$result['applied'] = true;
} else {
$result['removed'] = true;
}
CloudVpcPortService::finishJobLocked($cleanupJob, 'succeeded',
$result, $payload,
['operation_id' => (string) $cleanupJob->operation_id]);
$terminalize($cleanupJob);
}
$released = $port->fresh();
$assert((string) $released->state === 'released'
&& $released->active_claim === null
&& CloudVpcHostState::where('vpc_id', 1)->count() === 1
&& (int) CloudVpcHostState::where('vpc_id', 1)->value('host_id') === 2,
'Detach did not release the IP after every host cleanup converged.');
$knownFailedState = CloudVpcHostState::where('vpc_id', 1)
->where('host_id', 2)->first();
$knownFailedState->desired_revision = (int) $knownFailedState->desired_revision + 1;
$knownFailedState->desired_digest = str_repeat('f', 64);
$knownFailedState->sync_state = 'failed';
$knownFailedState->last_error_code = 'vpc_host_apply_failed';
$knownFailedState->save();
$knownFailedJob = CloudJob::where(
'operation_id', (string) $knownFailedState->last_operation_id
)->first();
$assert($knownFailedJob !== null,
'Failed host-state fixture lost its exact operation generation.');
$terminalizeFailed($knownFailedJob);
$vpc->desired_revision = (int) $knownFailedState->desired_revision;
$vpc->state = 'degraded';
$vpc->save();
$deleteCleanupJobs = $peer->getConnection()->transaction(
static function (): array {
$locked = CloudInstance::where('id', 2)
->lockForUpdate()->first();
return CloudVpcPortService::releaseAfterInstanceDeletedLocked(
$locked
);
}
);
$peerPort = $peerPort->fresh();
$assert(count($deleteCleanupJobs) === 1
&& (string) $deleteCleanupJobs[0]->kind === 'vpc_host_remove'
&& (string) $peerPort->state === 'release_pending'
&& $peerPort->active_claim !== null,
'Confirmed instance deletion released its private IP before route cleanup.');
$deleteCleanupJob = $deleteCleanupJobs[0];
$deleteCleanupPayload = CloudSecrets::decrypt(
(string) $deleteCleanupJob->sealed_payload, 'job-payload'
);
$deleteHostState = CloudVpcHostState::where('vpc_id', 1)
->where('host_id', 2)->first();
CloudVpcPortService::finishJobLocked($deleteCleanupJob, 'failed', null,
$deleteCleanupPayload, [
'operation_id' => (string) $deleteCleanupJob->operation_id,
'error_code' => 'vpc_host_remove_failed',
]);
$terminalizeFailed($deleteCleanupJob);
$peer->status = 'deleted';
$peer->save();
$deleteCleanupJob->effect_resolution = 'unknown';
$deleteCleanupJob->effect_may_have_applied = 1;
$deleteCleanupJob->safe_to_resubmit = 0;
$deleteCleanupJob->save();
$jobCountBeforeUnknownRetry = CloudJob::count();
$assert(CloudVpcPortService::retryDeletedCleanupDue(10) === 0
&& CloudJob::count() === $jobCountBeforeUnknownRetry,
'Unknown-effect deleted-instance cleanup was replayed.');
$terminalizeFailed($deleteCleanupJob);
$assert(CloudVpcPortService::retryDeletedCleanupDue(10) === 1,
'Known-not-applied deleted-instance cleanup was not retried.');
$peerPort = $peerPort->fresh();
$assert((string) $peerPort->state === 'release_pending'
&& $peerPort->active_claim !== null,
'A failed deleted-instance cleanup released the address early.');
$deleteCleanupJob = CloudJob::where('kind', 'vpc_host_remove')
->where('phase', 'accepted')->orderByDesc('id')->first();
$deleteCleanupPayload = CloudSecrets::decrypt(
(string) $deleteCleanupJob->sealed_payload, 'job-payload'
);
$deleteHostState = CloudVpcHostState::where('vpc_id', 1)
->where('host_id', 2)->first();
CloudVpcPortService::finishJobLocked($deleteCleanupJob, 'succeeded', [
'vpc_key' => (string) $vpc->vpc_key,
'desired_revision' => (int) $deleteHostState->applied_revision,
'desired_digest' => (string) $deleteHostState->applied_digest,
'removed' => true,
], $deleteCleanupPayload, [
'operation_id' => (string) $deleteCleanupJob->operation_id,
]);
$terminalize($deleteCleanupJob);
$peerPort = $peerPort->fresh();
$assert((string) $peerPort->state === 'released'
&& $peerPort->active_claim === null
&& CloudVpcHostState::where('vpc_id', 1)->count() === 0,
'Deleted instance VPC cleanup did not release the address after convergence.');
$worker = file_get_contents(
dirname(__DIR__) . '/app/Services/Cloud/CloudWorkerService.php'
);
$cleanupPosition = strpos($worker,
'CloudVpcPortService::releaseAfterInstanceDeletedLocked');
$deletedPosition = strpos($worker, "\$instance->status = 'deleted';");
$placementPosition = strpos($worker,
'CloudPlacementService::releaseDeletedInstance');
$assert($cleanupPosition !== false && $deletedPosition !== false
&& $placementPosition !== false
&& $cleanupPosition < $deletedPosition
&& $deletedPosition < $placementPosition,
'Delete finalization does not fence VPC cleanup before resource release.');
echo "VPC host projection and VM port workflow passed.\n";
}