You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1373 lines
63 KiB
PHP
1373 lines
63 KiB
PHP
<?php
|
|
namespace App\Services\Cloud {
|
|
// Isolated test cipher; production CloudSecrets is not loaded here.
|
|
final class CloudSecrets
|
|
{
|
|
private static array $values = [];
|
|
public static function encrypt(array $value, string $purpose): string
|
|
{
|
|
$key = bin2hex(random_bytes(16));
|
|
self::$values[$key] = $value;
|
|
return 'vpc-test:' . $key;
|
|
}
|
|
public static function decrypt(string $value, string $purpose): array
|
|
{
|
|
return self::$values[substr($value, 9)];
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInstanceMigration;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudVpc;
|
|
use App\Models\CloudVpcHostState;
|
|
use App\Models\CloudVpcMembershipBatch;
|
|
use App\Models\CloudVpcMembershipBatchItem;
|
|
use App\Models\CloudVpcPort;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use App\Services\Cloud\CloudJobService;
|
|
use App\Services\Cloud\CloudOperationRecoveryIdentity;
|
|
use App\Services\Cloud\CloudOperationRecoveryService;
|
|
use App\Services\Cloud\CloudVpcMembershipBatchService;
|
|
use App\Services\Cloud\CloudVpcPortService;
|
|
use App\Services\Cloud\CloudVpcProjection;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$terminalize = static function (CloudJob $job): void {
|
|
$now = time();
|
|
$job->phase = 'terminal';
|
|
$job->outcome = $job->status = 'succeeded';
|
|
$job->attempts = $job->dispatch_attempts = 1;
|
|
$job->dispatch_started_at = $job->dispatched_at = $now - 1;
|
|
$job->agent_accepted_at = $now - 1;
|
|
$job->receipt_deadline_at = $now + 900;
|
|
$job->terminal_at = $now;
|
|
$job->effect_may_have_applied = 0;
|
|
$job->safe_to_resubmit = 0;
|
|
$job->effect_resolution = 'applied';
|
|
$job->updated_at = $now;
|
|
$job->save();
|
|
};
|
|
$terminalizeFailed = static function (CloudJob $job): void {
|
|
$finishedAt = time() - 600;
|
|
$job->phase = 'terminal';
|
|
$job->outcome = $job->status = 'failed';
|
|
$job->attempts = $job->dispatch_attempts = 1;
|
|
$job->dispatch_started_at = $job->dispatched_at = $finishedAt - 1;
|
|
$job->agent_accepted_at = $finishedAt - 1;
|
|
$job->receipt_deadline_at = $finishedAt + 900;
|
|
$job->terminal_at = $finishedAt;
|
|
$job->effect_may_have_applied = 0;
|
|
$job->safe_to_resubmit = 1;
|
|
$job->effect_resolution = 'not_applied';
|
|
$job->error_code = 'vpc_host_remove_failed';
|
|
$job->updated_at = $finishedAt;
|
|
$job->save();
|
|
};
|
|
$convergeRoutingJobs = static function (
|
|
array $jobs
|
|
) use ($terminalize): void {
|
|
foreach ($jobs as $job) {
|
|
$state = CloudVpcHostState::where(
|
|
'vpc_id', (int) $job->vpc_id
|
|
)->where('host_id', (int) $job->host_id)->first();
|
|
if ((string) $job->kind === 'vpc_host_remove') {
|
|
if ($state === null) {
|
|
throw new RuntimeException(
|
|
'Route-remove test job lost its host state.'
|
|
);
|
|
}
|
|
$state->delete();
|
|
} else {
|
|
if ($state === null) {
|
|
throw new RuntimeException(
|
|
'Route-apply test job lost its host state.'
|
|
);
|
|
}
|
|
$state->applied_revision = (int) $state->desired_revision;
|
|
$state->applied_digest = (string) $state->desired_digest;
|
|
$state->sync_state = 'ready';
|
|
$state->last_error_code = '';
|
|
$state->save();
|
|
}
|
|
$terminalize($job);
|
|
}
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('user', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
});
|
|
Capsule::table('user')->insert(['id' => 7]);
|
|
$schema->create('cloud_network_lines', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id'); $table->integer('region_id');
|
|
});
|
|
$schema->create('cloud_resource_pools', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id'); $table->integer('line_id');
|
|
});
|
|
$schema->create('cloud_hosts', static function (Blueprint $table): void {
|
|
$table->increments('id'); $table->boolean('enabled');
|
|
});
|
|
$schema->create('cloud_host_vpc_transports', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->integer('host_id')->primary();
|
|
$table->string('transport_address');
|
|
$table->integer('transport_prefix_length');
|
|
$table->string('transport_interface'); $table->integer('mtu');
|
|
$table->integer('desired_revision');
|
|
$table->integer('observed_revision'); $table->string('sync_state');
|
|
});
|
|
$schema->create('cloud_instances', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id'); $table->string('uuid');
|
|
$table->integer('host_id'); $table->integer('user_id');
|
|
$table->integer('resource_pool_id')->nullable();
|
|
$table->string('name'); $table->string('status');
|
|
$table->string('service_status'); $table->string('network_status');
|
|
$table->integer('expires_at')->nullable(); $table->text('spec_json');
|
|
$table->integer('created_at'); $table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_vpcs', static function (Blueprint $table): void {
|
|
$table->increments('id'); $table->string('vpc_key');
|
|
$table->integer('user_id'); $table->integer('region_id');
|
|
$table->integer('cidr_pool_id'); $table->string('name');
|
|
$table->string('cidr'); $table->integer('prefix_length');
|
|
$table->string('gateway'); $table->integer('active_claim')->nullable();
|
|
$table->string('state'); $table->integer('desired_revision');
|
|
$table->integer('deleted_at')->nullable();
|
|
$table->integer('created_at'); $table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_vpc_ports', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id'); $table->string('port_key')->unique();
|
|
$table->integer('vpc_id'); $table->integer('instance_id');
|
|
$table->integer('migration_host_id')->nullable();
|
|
$table->integer('user_id'); $table->string('private_ip');
|
|
$table->string('mac_address')->unique();
|
|
$table->integer('active_claim')->nullable(); $table->string('state');
|
|
$table->integer('desired_revision');
|
|
$table->integer('applied_revision');
|
|
$table->string('last_operation_id')->nullable();
|
|
$table->string('last_error_code');
|
|
$table->integer('attached_at')->nullable();
|
|
$table->integer('released_at')->nullable();
|
|
$table->integer('created_at'); $table->integer('updated_at');
|
|
$table->unique(['vpc_id', 'private_ip', 'active_claim']);
|
|
$table->unique(['instance_id', 'active_claim']);
|
|
});
|
|
$schema->create('cloud_vpc_host_states', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id'); $table->integer('vpc_id');
|
|
$table->integer('host_id'); $table->integer('desired_revision');
|
|
$table->integer('applied_revision'); $table->string('desired_digest');
|
|
$table->string('applied_digest')->nullable();
|
|
$table->string('sync_state');
|
|
$table->string('last_operation_id')->nullable();
|
|
$table->string('last_error_code');
|
|
$table->integer('last_success_at');
|
|
$table->integer('last_failure_at');
|
|
$table->integer('created_at'); $table->integer('updated_at');
|
|
$table->unique(['vpc_id', 'host_id']);
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id'); $table->string('operation_id')->unique();
|
|
$table->string('request_idempotency_key');
|
|
$table->string('request_fingerprint');
|
|
$table->integer('parent_job_id')->nullable();
|
|
$table->integer('supersedes_job_id')->nullable();
|
|
$table->integer('vpc_repair_root_job_id')->nullable();
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('invoice_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('snapshot_id')->nullable();
|
|
$table->integer('backup_id')->nullable();
|
|
$table->integer('backup_restore_id')->nullable();
|
|
$table->integer('backup_recovery_id')->nullable();
|
|
$table->integer('vpc_id')->nullable();
|
|
$table->integer('vpc_port_id')->nullable();
|
|
$table->integer('vpc_batch_id')->nullable();
|
|
$table->integer('user_id'); $table->integer('host_id')->nullable();
|
|
$table->string('kind'); $table->string('execution_mode');
|
|
$table->string('phase'); $table->string('outcome')->nullable();
|
|
$table->string('status'); $table->text('sealed_payload');
|
|
$table->integer('attempts'); $table->integer('dispatch_attempts');
|
|
$table->integer('dispatch_deadline_at');
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->string('effect_resolution');
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
$table->text('effect_resolution_reason')->default('');
|
|
$table->boolean('finalization_incident')->default(0);
|
|
$table->string('finalization_error_code')->nullable();
|
|
$table->string('completion_event_id')->nullable();
|
|
$table->integer('read_at')->nullable();
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('available_at'); $table->string('error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
$table->integer('created_at'); $table->integer('updated_at');
|
|
$table->unique(['user_id', 'request_idempotency_key']);
|
|
});
|
|
$schema->create('cloud_vpc_membership_batches', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id'); $table->string('batch_key')->unique();
|
|
$table->integer('vpc_id'); $table->integer('user_id');
|
|
$table->integer('parent_job_id')->unique(); $table->string('kind');
|
|
$table->string('state'); $table->integer('desired_revision');
|
|
$table->integer('active_claim')->nullable();
|
|
$table->string('last_error_code')->default('');
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->integer('created_at'); $table->integer('updated_at');
|
|
$table->unique(['vpc_id', 'active_claim']);
|
|
});
|
|
$schema->create('cloud_vpc_membership_batch_items', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id'); $table->integer('batch_id');
|
|
$table->integer('user_id'); $table->integer('instance_id');
|
|
$table->integer('vpc_port_id'); $table->integer('host_id');
|
|
$table->integer('child_job_id')->nullable(); $table->string('state');
|
|
$table->string('last_error_code')->default('');
|
|
$table->integer('created_at'); $table->integer('updated_at');
|
|
$table->unique(['batch_id', 'instance_id']);
|
|
$table->unique(['batch_id', 'vpc_port_id']);
|
|
$table->unique('child_job_id');
|
|
});
|
|
|
|
Capsule::table('cloud_network_lines')->insert([
|
|
'id' => 1, 'region_id' => 1,
|
|
]);
|
|
Capsule::table('cloud_resource_pools')->insert([
|
|
'id' => 1, 'line_id' => 1,
|
|
]);
|
|
foreach ([1, 2] as $hostId) {
|
|
Capsule::table('cloud_hosts')->insert([
|
|
'id' => $hostId, 'enabled' => 1,
|
|
]);
|
|
Capsule::table('cloud_host_vpc_transports')->insert([
|
|
'host_id' => $hostId,
|
|
'transport_address' => '172.16.10.' . (10 + $hostId),
|
|
'transport_prefix_length' => 24,
|
|
'transport_interface' => 'br-private', 'mtu' => 1500,
|
|
'desired_revision' => 1, 'observed_revision' => 1,
|
|
'sync_state' => 'ready',
|
|
]);
|
|
}
|
|
$newInstance = static function (int $id, int $hostId): CloudInstance {
|
|
$instance = new CloudInstance();
|
|
$instance->id = $id;
|
|
$instance->uuid = sprintf('11111111-1111-4111-8111-%012d', $id);
|
|
$instance->host_id = $hostId; $instance->user_id = 7;
|
|
$instance->resource_pool_id = 1; $instance->name = 'vm-' . $id;
|
|
$instance->status = 'active'; $instance->service_status = 'exempt';
|
|
$instance->network_status = 'active'; $instance->expires_at = null;
|
|
$networkSpec = $id === 2
|
|
? ['public_ip' => '203.0.113.12']
|
|
: ['public_ip_cidr' => '203.0.113.11/24'];
|
|
$instance->spec_json = json_encode(array_merge([
|
|
'os_source' => 'cloud_template',
|
|
], $networkSpec), JSON_THROW_ON_ERROR);
|
|
$instance->created_at = $instance->updated_at = time();
|
|
$instance->save();
|
|
return $instance;
|
|
};
|
|
$target = $newInstance(1, 1);
|
|
$peer = $newInstance(2, 2);
|
|
$vpc = new CloudVpc();
|
|
$vpc->id = 1; $vpc->vpc_key = str_repeat('a', 32);
|
|
$vpc->user_id = 7; $vpc->region_id = 1; $vpc->cidr_pool_id = 1;
|
|
$vpc->name = 'private'; $vpc->cidr = '10.60.0.0/24';
|
|
$vpc->prefix_length = 24; $vpc->gateway = '10.60.0.1';
|
|
$vpc->active_claim = 1; $vpc->state = 'active';
|
|
$vpc->desired_revision = 1; $vpc->deleted_at = null;
|
|
$vpc->created_at = $vpc->updated_at = time(); $vpc->save();
|
|
$peerPort = new CloudVpcPort();
|
|
$peerPort->id = 1; $peerPort->port_key = str_repeat('b', 32);
|
|
$peerPort->vpc_id = 1; $peerPort->instance_id = 2;
|
|
$peerPort->user_id = 7; $peerPort->private_ip = '10.60.0.3';
|
|
$peerPort->mac_address = '02:00:00:00:00:03';
|
|
$peerPort->active_claim = 1; $peerPort->state = 'attached';
|
|
$peerPort->desired_revision = $peerPort->applied_revision = 1;
|
|
$peerPort->last_operation_id = null; $peerPort->last_error_code = '';
|
|
$peerPort->attached_at = time(); $peerPort->released_at = null;
|
|
$peerPort->created_at = $peerPort->updated_at = time(); $peerPort->save();
|
|
$initial = CloudVpcProjection::build([
|
|
'vpc_key' => (string) $vpc->vpc_key, 'cidr' => (string) $vpc->cidr,
|
|
'gateway' => (string) $vpc->gateway,
|
|
], [['private_ip' => '10.60.0.3',
|
|
'mac_address' => '02:00:00:00:00:03', 'host_id' => 2]], [
|
|
2 => ['transport_address' => '172.16.10.12',
|
|
'transport_prefix_length' => 24,
|
|
'transport_interface' => 'br-private', 'mtu' => 1500],
|
|
], 1);
|
|
$state = new CloudVpcHostState();
|
|
$state->vpc_id = 1; $state->host_id = 2;
|
|
$state->desired_revision = $state->applied_revision = 1;
|
|
$state->desired_digest = $state->applied_digest = $initial[2]['digest'];
|
|
$state->sync_state = 'ready'; $state->last_operation_id = null;
|
|
$state->last_error_code = ''; $state->last_success_at = time();
|
|
$state->last_failure_at = 0;
|
|
$state->created_at = $state->updated_at = time(); $state->save();
|
|
|
|
$management = CloudVpcPortService::managementListing(['user_id' => 7]);
|
|
$targetManagement = collect($management)->firstWhere('id', '1');
|
|
$peerManagement = collect($management)->firstWhere('id', '2');
|
|
$assert(($targetManagement['region_id'] ?? null) === 1
|
|
&& ($targetManagement['public_ipv4'] ?? null) === '203.0.113.11'
|
|
&& array_key_exists('port', $targetManagement)
|
|
&& $targetManagement['port'] === null
|
|
&& ($targetManagement['can_attach'] ?? false) === true,
|
|
'The VPC page projection did not expose the eligible same-region instance.');
|
|
$assert(($peerManagement['port']['vpc_key'] ?? null)
|
|
=== (string) $vpc->vpc_key
|
|
&& ($peerManagement['public_ipv4'] ?? null) === '203.0.113.12'
|
|
&& ($peerManagement['can_detach'] ?? false) === true,
|
|
'The VPC page projection did not expose its attached instance action.');
|
|
|
|
$actor = ['user_id' => 7,
|
|
'request_idempotency_key' => 'vpc-attach-request-0001'];
|
|
$queued = CloudVpcPortService::attach($target, $actor, [
|
|
'vpc_key' => (string) $vpc->vpc_key, 'confirm_restart' => true,
|
|
]);
|
|
$assert($queued['job'] !== null
|
|
&& CloudJob::where('kind', 'vpc_host_apply')->count() === 2,
|
|
'Attach did not queue a complete two-host projection.');
|
|
$replayedAttach = CloudVpcPortService::attach($target, $actor, [
|
|
'vpc_key' => (string) $vpc->vpc_key, 'confirm_restart' => true,
|
|
]);
|
|
$activeHostJobIds = CloudJob::where('kind', 'vpc_host_apply')
|
|
->pluck('id')->map(static fn ($id): int => (int) $id)->all();
|
|
$assert($replayedAttach['job'] !== null
|
|
&& in_array((int) $replayedAttach['job']->id, $activeHostJobIds, true)
|
|
&& CloudJob::where('kind', 'vpc_host_apply')->count() === 2,
|
|
'An accepted attach retry did not return its existing workflow.');
|
|
$reconnectedAttach = CloudVpcPortService::attach($target, [
|
|
'user_id' => 7,
|
|
'request_idempotency_key' => 'vpc-attach-new-browser-0002',
|
|
], [
|
|
'vpc_key' => (string) $vpc->vpc_key, 'confirm_restart' => true,
|
|
]);
|
|
$assert($reconnectedAttach['job'] !== null
|
|
&& in_array((int) $reconnectedAttach['job']->id,
|
|
$activeHostJobIds, true)
|
|
&& CloudJob::where('kind', 'vpc_host_apply')->count() === 2,
|
|
'A reconnect with a new browser request key duplicated the accepted workflow.');
|
|
$port = CloudVpcPort::where('instance_id', 1)
|
|
->where('active_claim', 1)->first();
|
|
$assert($port !== null && (string) $port->state === 'preparing'
|
|
&& (string) $port->private_ip === '10.60.0.2',
|
|
'Attach did not reserve the first usable free address.');
|
|
$listing = CloudVpcPortService::listing($target, ['user_id' => 7]);
|
|
$assert(($listing['active_task']['task_id'] ?? null)
|
|
=== (string) $replayedAttach['job']->id
|
|
&& ($listing['can_attach'] ?? true) === false,
|
|
'The read model did not expose the tenant VPC task after reconnect.');
|
|
$management = CloudVpcPortService::managementListing(['user_id' => 7]);
|
|
$targetManagement = collect($management)->firstWhere('id', '1');
|
|
$assert(($targetManagement['port']['state'] ?? null) === 'preparing'
|
|
&& ($targetManagement['active_task']['task_id'] ?? null)
|
|
=== (string) $replayedAttach['job']->id
|
|
&& ($targetManagement['can_retry_attach'] ?? true) === false,
|
|
'The VPC page projection did not retain the active attach task fence.');
|
|
|
|
foreach (CloudJob::where('kind', 'vpc_host_apply')->orderBy('id')->get()
|
|
as $hostJob) {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $hostJob->sealed_payload, 'job-payload'
|
|
);
|
|
$hostState = CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', (int) $hostJob->host_id)->first();
|
|
CloudVpcPortService::finishJobLocked($hostJob, 'succeeded', [
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'desired_revision' => (int) $hostState->desired_revision,
|
|
'desired_digest' => (string) $hostState->desired_digest,
|
|
'bridge' => CloudVpcProjection::bridgeName((string) $vpc->vpc_key),
|
|
'applied' => true,
|
|
], $payload, ['operation_id' => (string) $hostJob->operation_id]);
|
|
$terminalize($hostJob);
|
|
}
|
|
$attachJob = CloudJob::where('kind', 'vpc_port_attach')->first();
|
|
$assert($attachJob !== null,
|
|
'Converged host projections did not queue the VM port attach.');
|
|
$attachPayload = CloudSecrets::decrypt(
|
|
(string) $attachJob->sealed_payload, 'job-payload'
|
|
);
|
|
$body = $attachPayload['body'];
|
|
CloudVpcPortService::finishJobLocked($attachJob, 'succeeded', [
|
|
'uuid' => (string) $target->uuid,
|
|
'port_key' => (string) $port->port_key,
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'vpc_port' => [
|
|
'port_key' => (string) $port->port_key,
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'private_ip' => (string) $port->private_ip,
|
|
'cidr' => (string) $vpc->cidr,
|
|
'gateway' => (string) $vpc->gateway,
|
|
'mac' => (string) $port->mac_address,
|
|
'bridge' => CloudVpcProjection::bridgeName((string) $vpc->vpc_key),
|
|
'host_revision' => (int) $body['host_revision'],
|
|
'host_digest' => (string) $body['host_digest'],
|
|
],
|
|
'status' => 'running', 'attached' => true,
|
|
], $attachPayload, ['operation_id' => (string) $attachJob->operation_id]);
|
|
$terminalize($attachJob);
|
|
$assert((string) $port->fresh()->state === 'attached',
|
|
'Successful VM port result did not attach the port ledger.');
|
|
|
|
$migration = new CloudInstanceMigration();
|
|
$migration->id = 99;
|
|
$migration->instance_id = 1;
|
|
$migration->vpc_port_id = CloudVpcPortService::migrationPortIdLocked(
|
|
$target,
|
|
2
|
|
);
|
|
$migration->source_host_id = 1;
|
|
$migration->target_host_id = 2;
|
|
$migration->source_uuid = (string) $target->uuid;
|
|
$targetJobs = $target->getConnection()->transaction(
|
|
static fn (): array =>
|
|
CloudVpcPortService::beginMigrationToTargetLocked(
|
|
$migration,
|
|
$target
|
|
)
|
|
);
|
|
$migrationPayload = CloudSecrets::decrypt(
|
|
(string) $targetJobs[0]->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$assert(count($targetJobs) === 2
|
|
&& ($migrationPayload['maintenance_migration_id'] ?? null) === 99
|
|
&& ($migrationPayload['vpc_migration_direction'] ?? null)
|
|
=== 'target'
|
|
&& (string) $port->fresh()->state === 'migrating'
|
|
&& (int) $port->fresh()->migration_host_id === 2,
|
|
'Maintenance migration did not durably project the port to target.');
|
|
$convergeRoutingJobs($targetJobs);
|
|
$assert(CloudVpcPortService::finishMigrationProjectionLocked(
|
|
$migration,
|
|
$target,
|
|
'target'
|
|
) === true,
|
|
'Converged target VPC projection did not complete.');
|
|
$targetMetadata = CloudVpcPortService::migrationTargetMetadataLocked(
|
|
$migration,
|
|
$target
|
|
);
|
|
$assert(($targetMetadata['private_ip'] ?? null) === '10.60.0.2'
|
|
&& ($targetMetadata['mac'] ?? null)
|
|
=== (string) $port->mac_address
|
|
&& ($targetMetadata['host_revision'] ?? 0) > 1,
|
|
'Migration target metadata did not preserve the logical port identity.');
|
|
CloudVpcPortService::finalizeMigrationCutoverLocked($migration, $target);
|
|
$cutoverPort = $port->fresh();
|
|
$assert((string) $cutoverPort->state === 'attached'
|
|
&& $cutoverPort->migration_host_id === null
|
|
&& (string) $cutoverPort->private_ip === '10.60.0.2'
|
|
&& (string) $cutoverPort->mac_address
|
|
=== (string) $port->mac_address,
|
|
'VPC cutover changed or retained migration state on the logical port.');
|
|
|
|
$target->host_id = 2;
|
|
$target->save();
|
|
$reverse = new CloudInstanceMigration();
|
|
$reverse->id = 100;
|
|
$reverse->instance_id = 1;
|
|
$reverse->vpc_port_id = CloudVpcPortService::migrationPortIdLocked(
|
|
$target,
|
|
1
|
|
);
|
|
$reverse->source_host_id = 2;
|
|
$reverse->target_host_id = 1;
|
|
$reverse->source_uuid = (string) $target->uuid;
|
|
$reverseJobs = $target->getConnection()->transaction(
|
|
static fn (): array =>
|
|
CloudVpcPortService::beginMigrationToTargetLocked(
|
|
$reverse,
|
|
$target
|
|
)
|
|
);
|
|
$convergeRoutingJobs($reverseJobs);
|
|
$assert(CloudVpcPortService::finishMigrationProjectionLocked(
|
|
$reverse,
|
|
$target,
|
|
'target'
|
|
),
|
|
'Reverse target projection did not converge.');
|
|
CloudVpcPortService::finalizeMigrationCutoverLocked($reverse, $target);
|
|
$target->host_id = 1;
|
|
$target->save();
|
|
|
|
$abort = new CloudInstanceMigration();
|
|
$abort->id = 101;
|
|
$abort->instance_id = 1;
|
|
$abort->vpc_port_id = CloudVpcPortService::migrationPortIdLocked(
|
|
$target,
|
|
2
|
|
);
|
|
$abort->source_host_id = 1;
|
|
$abort->target_host_id = 2;
|
|
$abort->source_uuid = (string) $target->uuid;
|
|
$abortTargetJobs = $target->getConnection()->transaction(
|
|
static fn (): array =>
|
|
CloudVpcPortService::beginMigrationToTargetLocked(
|
|
$abort,
|
|
$target
|
|
)
|
|
);
|
|
$unknownJob = $abortTargetJobs[0];
|
|
foreach ($abortTargetJobs as $routeJob) {
|
|
$state = CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', (int) $routeJob->host_id)->first();
|
|
if ((int) $routeJob->id === (int) $unknownJob->id) {
|
|
$state->sync_state = 'unknown';
|
|
$state->last_operation_id = (string) $routeJob->operation_id;
|
|
$state->last_error_code = 'vpc_result_uncertain';
|
|
$state->save();
|
|
$routeJob->phase = 'terminal';
|
|
$routeJob->outcome = $routeJob->status = 'failed';
|
|
$routeJob->effect_may_have_applied = 1;
|
|
$routeJob->safe_to_resubmit = 0;
|
|
$routeJob->effect_resolution = 'unknown';
|
|
$routeJob->terminal_at = $routeJob->updated_at = time();
|
|
$routeJob->save();
|
|
continue;
|
|
}
|
|
if ((string) $routeJob->kind === 'vpc_host_remove') {
|
|
$state->delete();
|
|
} else {
|
|
$state->applied_revision = (int) $state->desired_revision;
|
|
$state->applied_digest = (string) $state->desired_digest;
|
|
$state->sync_state = 'ready';
|
|
$state->save();
|
|
}
|
|
$terminalize($routeJob);
|
|
}
|
|
$jobsBeforeUnknownAbort = CloudJob::count();
|
|
$unknownReplayBlocked = false;
|
|
try {
|
|
$target->getConnection()->transaction(
|
|
static fn (): array =>
|
|
CloudVpcPortService::beginMigrationToSourceLocked(
|
|
$abort,
|
|
$target
|
|
)
|
|
);
|
|
} catch (Throwable $error) {
|
|
$unknownReplayBlocked = true;
|
|
}
|
|
$assert($unknownReplayBlocked
|
|
&& CloudJob::count() === $jobsBeforeUnknownAbort,
|
|
'Unknown VPC routing effect was overwritten by source rollback.');
|
|
$unknownState = CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', (int) $unknownJob->host_id)->first();
|
|
$unknownState->sync_state = 'failed';
|
|
$unknownState->save();
|
|
$terminalizeFailed($unknownJob);
|
|
$sourceJobs = $target->getConnection()->transaction(
|
|
static fn (): array =>
|
|
CloudVpcPortService::beginMigrationToSourceLocked(
|
|
$abort,
|
|
$target
|
|
)
|
|
);
|
|
$convergeRoutingJobs($sourceJobs);
|
|
$assert(CloudVpcPortService::finishMigrationProjectionLocked(
|
|
$abort,
|
|
$target,
|
|
'source'
|
|
)
|
|
&& (string) $port->fresh()->state === 'attached'
|
|
&& $port->fresh()->migration_host_id === null,
|
|
'Confirmed-not-applied abort did not restore the source VPC route.');
|
|
|
|
$detachActor = ['user_id' => 7,
|
|
'request_idempotency_key' => 'vpc-detach-request-0001'];
|
|
$detaching = CloudVpcPortService::detach($target, $detachActor, [
|
|
'confirm_restart' => true,
|
|
]);
|
|
$detachJob = $detaching['job'];
|
|
$replayedDetach = CloudVpcPortService::detach($target, $detachActor, [
|
|
'confirm_restart' => true,
|
|
]);
|
|
$assert((int) ($replayedDetach['job']->id ?? 0) === (int) $detachJob->id
|
|
&& CloudJob::where('kind', 'vpc_port_detach')->count() === 1,
|
|
'An accepted detach retry did not return its existing workflow.');
|
|
$detachPayload = CloudSecrets::decrypt(
|
|
(string) $detachJob->sealed_payload, 'job-payload'
|
|
);
|
|
CloudVpcPortService::finishJobLocked($detachJob, 'succeeded', [
|
|
'uuid' => (string) $target->uuid,
|
|
'port_key' => (string) $port->port_key,
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'status' => 'running', 'removed' => true,
|
|
], $detachPayload, ['operation_id' => (string) $detachJob->operation_id]);
|
|
$terminalize($detachJob);
|
|
$assert(CloudJob::whereIn('kind',
|
|
['vpc_host_apply', 'vpc_host_remove'])->where('phase', 'accepted')
|
|
->count() === 2,
|
|
'Detach did not queue route cleanup after VM port removal.');
|
|
|
|
$cleanupJobs = CloudJob::whereIn('kind',
|
|
['vpc_host_apply', 'vpc_host_remove'])->where('phase', 'accepted')
|
|
->orderBy('id')->get();
|
|
foreach ($cleanupJobs as $cleanupJob) {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $cleanupJob->sealed_payload, 'job-payload'
|
|
);
|
|
$hostState = CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', (int) $cleanupJob->host_id)->first();
|
|
$result = [
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'desired_revision' => (int) ($cleanupJob->kind === 'vpc_host_apply'
|
|
? $hostState->desired_revision : $hostState->applied_revision),
|
|
'desired_digest' => (string) ($cleanupJob->kind === 'vpc_host_apply'
|
|
? $hostState->desired_digest : $hostState->applied_digest),
|
|
];
|
|
if ((string) $cleanupJob->kind === 'vpc_host_apply') {
|
|
$result['bridge'] = CloudVpcProjection::bridgeName(
|
|
(string) $vpc->vpc_key
|
|
);
|
|
$result['applied'] = true;
|
|
} else {
|
|
$result['removed'] = true;
|
|
}
|
|
CloudVpcPortService::finishJobLocked($cleanupJob, 'succeeded',
|
|
$result, $payload,
|
|
['operation_id' => (string) $cleanupJob->operation_id]);
|
|
$terminalize($cleanupJob);
|
|
}
|
|
$released = $port->fresh();
|
|
$assert((string) $released->state === 'released'
|
|
&& $released->active_claim === null
|
|
&& CloudVpcHostState::where('vpc_id', 1)->count() === 1
|
|
&& (int) CloudVpcHostState::where('vpc_id', 1)->value('host_id') === 2,
|
|
'Detach did not release the IP after every host cleanup converged.');
|
|
|
|
$knownFailedState = CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', 2)->first();
|
|
$knownFailedState->desired_revision = (int) $knownFailedState->desired_revision + 1;
|
|
$knownFailedState->desired_digest = str_repeat('f', 64);
|
|
$knownFailedState->sync_state = 'failed';
|
|
$knownFailedState->last_error_code = 'vpc_host_apply_failed';
|
|
$knownFailedState->save();
|
|
$knownFailedJob = CloudJob::where(
|
|
'operation_id', (string) $knownFailedState->last_operation_id
|
|
)->first();
|
|
$assert($knownFailedJob !== null,
|
|
'Failed host-state fixture lost its exact operation generation.');
|
|
$terminalizeFailed($knownFailedJob);
|
|
$vpc->desired_revision = (int) $knownFailedState->desired_revision;
|
|
$vpc->state = 'degraded';
|
|
$vpc->save();
|
|
$deleteCleanupJobs = $peer->getConnection()->transaction(
|
|
static function (): array {
|
|
$locked = CloudInstance::where('id', 2)
|
|
->lockForUpdate()->first();
|
|
return CloudVpcPortService::releaseAfterInstanceDeletedLocked(
|
|
$locked
|
|
);
|
|
}
|
|
);
|
|
$peerPort = $peerPort->fresh();
|
|
$assert(count($deleteCleanupJobs) === 1
|
|
&& (string) $deleteCleanupJobs[0]->kind === 'vpc_host_remove'
|
|
&& (string) $peerPort->state === 'release_pending'
|
|
&& $peerPort->active_claim !== null,
|
|
'Confirmed instance deletion released its private IP before route cleanup.');
|
|
$deleteCleanupJob = $deleteCleanupJobs[0];
|
|
$deleteCleanupPayload = CloudSecrets::decrypt(
|
|
(string) $deleteCleanupJob->sealed_payload, 'job-payload'
|
|
);
|
|
$deleteHostState = CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', 2)->first();
|
|
CloudVpcPortService::finishJobLocked($deleteCleanupJob, 'failed', null,
|
|
$deleteCleanupPayload, [
|
|
'operation_id' => (string) $deleteCleanupJob->operation_id,
|
|
'error_code' => 'vpc_host_remove_failed',
|
|
]);
|
|
$terminalizeFailed($deleteCleanupJob);
|
|
$peer->status = 'deleted';
|
|
$peer->save();
|
|
$deleteCleanupJob->effect_resolution = 'unknown';
|
|
$deleteCleanupJob->effect_may_have_applied = 1;
|
|
$deleteCleanupJob->safe_to_resubmit = 0;
|
|
$deleteCleanupJob->save();
|
|
$jobCountBeforeUnknownRetry = CloudJob::count();
|
|
$assert(CloudVpcPortService::retryDeletedCleanupDue(10) === 0
|
|
&& CloudJob::count() === $jobCountBeforeUnknownRetry,
|
|
'Unknown-effect deleted-instance cleanup was replayed.');
|
|
$terminalizeFailed($deleteCleanupJob);
|
|
$assert(CloudVpcPortService::retryDeletedCleanupDue(10) === 1,
|
|
'Known-not-applied deleted-instance cleanup was not retried.');
|
|
$peerPort = $peerPort->fresh();
|
|
$assert((string) $peerPort->state === 'release_pending'
|
|
&& $peerPort->active_claim !== null,
|
|
'A failed deleted-instance cleanup released the address early.');
|
|
$deleteCleanupJob = CloudJob::where('kind', 'vpc_host_remove')
|
|
->where('phase', 'accepted')->orderByDesc('id')->first();
|
|
$deleteCleanupPayload = CloudSecrets::decrypt(
|
|
(string) $deleteCleanupJob->sealed_payload, 'job-payload'
|
|
);
|
|
$deleteHostState = CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', 2)->first();
|
|
CloudVpcPortService::finishJobLocked($deleteCleanupJob, 'succeeded', [
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'desired_revision' => (int) $deleteHostState->applied_revision,
|
|
'desired_digest' => (string) $deleteHostState->applied_digest,
|
|
'removed' => true,
|
|
], $deleteCleanupPayload, [
|
|
'operation_id' => (string) $deleteCleanupJob->operation_id,
|
|
]);
|
|
$terminalize($deleteCleanupJob);
|
|
$peerPort = $peerPort->fresh();
|
|
$assert((string) $peerPort->state === 'released'
|
|
&& $peerPort->active_claim === null
|
|
&& CloudVpcHostState::where('vpc_id', 1)->count() === 0,
|
|
'Deleted instance VPC cleanup did not release the address after convergence.');
|
|
|
|
// A membership batch has no four-instance ceiling. All attach mutations
|
|
// remain fenced behind one route generation, then become claimable in the
|
|
// same finalizer transaction. Detach mutations are claimable immediately
|
|
// and route cleanup is deferred until every VM result is known.
|
|
$batchInstances = [];
|
|
foreach (range(10, 15) as $id) {
|
|
$batchInstances[] = $newInstance($id, $id % 2 === 0 ? 1 : 2);
|
|
}
|
|
$attachBatch = CloudVpcMembershipBatchService::start([
|
|
'user_id' => 7,
|
|
'request_idempotency_key' => 'vpc-batch-attach-more-than-four-0001',
|
|
], (string) $vpc->vpc_key, [
|
|
'action' => 'attach',
|
|
'instance_ids' => array_map(
|
|
static fn (CloudInstance $instance): int => (int) $instance->id,
|
|
$batchInstances
|
|
),
|
|
'confirm_restart' => true,
|
|
]);
|
|
$attachReplay = CloudVpcMembershipBatchService::start([
|
|
'user_id' => 7,
|
|
'request_idempotency_key' => 'vpc-batch-attach-more-than-four-0001',
|
|
], (string) $vpc->vpc_key, [
|
|
'action' => 'attach',
|
|
'instance_ids' => [15, 14, 13, 12, 11, 10],
|
|
'confirm_restart' => true,
|
|
]);
|
|
$assert((int) $attachReplay['job']->id === (int) $attachBatch['job']->id
|
|
&& ($attachReplay['unchanged'] ?? false) === true,
|
|
'Batch retry did not return the same durable parent task.');
|
|
$attachBatchId = (int) CloudVpcMembershipBatch::where(
|
|
'parent_job_id', (int) $attachBatch['job']->id
|
|
)->value('id');
|
|
$attachRouteJobs = CloudJob::where('vpc_batch_id', $attachBatchId)
|
|
->where('kind', 'vpc_host_apply')->orderBy('id')->get();
|
|
$assert($attachBatchId > 0 && $attachRouteJobs->count() === 2
|
|
&& CloudJob::where('vpc_batch_id', $attachBatchId)
|
|
->where('kind', 'vpc_port_attach')->count() === 0,
|
|
'Batch attach did not fence more than four instances behind one route generation.');
|
|
$attachFences = CloudJob::where('vpc_batch_id', $attachBatchId)
|
|
->where('kind', 'vpc_batch_instance_fence')->orderBy('instance_id')
|
|
->get();
|
|
$assert($attachFences->count() === count($batchInstances)
|
|
&& $attachFences->every(static fn (CloudJob $job): bool =>
|
|
(string) $job->phase === 'accepted'
|
|
&& $job->host_id === null
|
|
&& (int) $job->parent_job_id === (int) $attachBatch['job']->id)
|
|
&& collect($batchInstances)->every(static fn (
|
|
CloudInstance $instance
|
|
): bool => CloudJobService::blockingQueryForInstance(
|
|
(int) $instance->id
|
|
)->where('kind', 'vpc_batch_instance_fence')->exists()),
|
|
'Batch route stage did not create one durable instance fence per VM.');
|
|
foreach ($attachRouteJobs as $index => $routeJob) {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $routeJob->sealed_payload, 'job-payload'
|
|
);
|
|
$hostState = CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', (int) $routeJob->host_id)->first();
|
|
CloudVpcPortService::finishJobLocked($routeJob, 'succeeded', [
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'desired_revision' => (int) $hostState->desired_revision,
|
|
'desired_digest' => (string) $hostState->desired_digest,
|
|
'bridge' => CloudVpcProjection::bridgeName((string) $vpc->vpc_key),
|
|
'applied' => true,
|
|
], $payload, ['operation_id' => (string) $routeJob->operation_id]);
|
|
$terminalize($routeJob);
|
|
$releasedCount = CloudJob::where('vpc_batch_id', $attachBatchId)
|
|
->where('kind', 'vpc_port_attach')->count();
|
|
$assert($releasedCount === ($index + 1 === $attachRouteJobs->count()
|
|
? count($batchInstances) : 0),
|
|
'Batch attach released VM jobs before or after the exact route barrier.');
|
|
}
|
|
$attachPortJobs = CloudJob::where('vpc_batch_id', $attachBatchId)
|
|
->where('kind', 'vpc_port_attach')->get();
|
|
$assert($attachPortJobs->count() === 6
|
|
&& $attachPortJobs->every(static fn (CloudJob $job): bool =>
|
|
(string) $job->phase === 'accepted'
|
|
&& (int) $job->dispatch_attempts === 0),
|
|
'Batch attach did not release every selected instance together.');
|
|
$assert($attachFences->every(static fn (CloudJob $job): bool =>
|
|
(string) $job->fresh()->phase === 'terminal')
|
|
&& collect($batchInstances)->every(static fn (
|
|
CloudInstance $instance
|
|
): bool => CloudJobService::blockingQueryForInstance(
|
|
(int) $instance->id
|
|
)->where('kind', 'vpc_port_attach')->exists()),
|
|
'Batch route fence was not atomically handed to each Agent port job.');
|
|
|
|
foreach ($attachPortJobs as $index => $attachPortJob) {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $attachPortJob->sealed_payload, 'job-payload'
|
|
);
|
|
$body = $payload['body'];
|
|
$instance = CloudInstance::find((int) $attachPortJob->instance_id);
|
|
$batchPort = CloudVpcPort::find((int) $attachPortJob->vpc_port_id);
|
|
$result = [
|
|
'uuid' => (string) $instance->uuid,
|
|
'port_key' => (string) $batchPort->port_key,
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'vpc_port' => [
|
|
'port_key' => (string) $batchPort->port_key,
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'private_ip' => (string) $batchPort->private_ip,
|
|
'cidr' => (string) $vpc->cidr,
|
|
'gateway' => (string) $vpc->gateway,
|
|
'mac' => (string) $batchPort->mac_address,
|
|
'bridge' => CloudVpcProjection::bridgeName(
|
|
(string) $vpc->vpc_key
|
|
),
|
|
'host_revision' => (int) $body['host_revision'],
|
|
'host_digest' => (string) $body['host_digest'],
|
|
],
|
|
'status' => 'running', 'attached' => true,
|
|
];
|
|
if ($index === 0) {
|
|
// Recovery of one uncertain child must use the same VPC finalizer
|
|
// as an ordinary completion. In particular, it must advance the
|
|
// membership batch without completing the parent early or
|
|
// leaving the selected instance behind a permanent fence.
|
|
$identity = CloudOperationRecoveryIdentity::expected(
|
|
$attachPortJob,
|
|
true
|
|
);
|
|
$operation = [
|
|
'operation_id' => (string) $attachPortJob->operation_id,
|
|
'status' => 'succeeded',
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => false,
|
|
'retryable' => false,
|
|
'error_code' => '',
|
|
'action' => $identity['action'],
|
|
'resource_id' => $identity['resource_id'],
|
|
'request_fingerprint' => $identity['request_fingerprint'],
|
|
'result' => $result,
|
|
];
|
|
$finishedAt = time();
|
|
$attachPortJob->phase = 'terminal';
|
|
$attachPortJob->outcome = $attachPortJob->status = 'failed';
|
|
$attachPortJob->attempts =
|
|
$attachPortJob->dispatch_attempts = 1;
|
|
$attachPortJob->dispatch_started_at =
|
|
$attachPortJob->dispatched_at = $finishedAt - 2;
|
|
$attachPortJob->agent_accepted_at = $finishedAt - 1;
|
|
$attachPortJob->receipt_deadline_at = $finishedAt + 900;
|
|
$attachPortJob->terminal_at = $finishedAt;
|
|
$attachPortJob->effect_may_have_applied = 1;
|
|
$attachPortJob->safe_to_resubmit = 0;
|
|
$attachPortJob->effect_resolution = 'unknown';
|
|
$attachPortJob->error_code = 'operation_effect_unknown';
|
|
$attachPortJob->updated_at = $finishedAt;
|
|
$attachPortJob->save();
|
|
|
|
CloudOperationRecoveryService::prepareApplied(
|
|
(int) $attachPortJob->id,
|
|
$operation,
|
|
'test_probe:vpc_batch_child'
|
|
);
|
|
$preparedChild = $attachPortJob->fresh();
|
|
$assert((string) $preparedChild->effect_resolution === 'unknown'
|
|
&& str_starts_with(
|
|
(string) $preparedChild->effect_resolution_reason,
|
|
'prepared:test_probe:vpc_batch_child:'
|
|
),
|
|
'Recovered VPC child released its fence before commit.');
|
|
CloudOperationRecoveryService::commitApplied(
|
|
(int) $attachPortJob->id,
|
|
$operation,
|
|
'test_probe:vpc_batch_child'
|
|
);
|
|
$assert((string) $attachPortJob->fresh()->effect_resolution
|
|
=== 'applied',
|
|
'Recovered VPC child did not publish its exact resolution.');
|
|
} else {
|
|
CloudVpcPortService::finishJobLocked(
|
|
$attachPortJob,
|
|
'succeeded',
|
|
$result,
|
|
$payload,
|
|
['operation_id' => (string) $attachPortJob->operation_id]
|
|
);
|
|
$terminalize($attachPortJob);
|
|
}
|
|
if ($index + 1 < $attachPortJobs->count()) {
|
|
$assert((string) $attachBatch['job']->fresh()->phase
|
|
!== 'terminal',
|
|
'Batch attach parent completed before every VM result.');
|
|
}
|
|
}
|
|
$assert((string) $attachBatch['job']->fresh()->outcome === 'succeeded'
|
|
&& CloudVpcPort::where('vpc_id', 1)
|
|
->where('active_claim', 1)->where('state', 'attached')->count() === 6,
|
|
'Batch attach did not converge its parent and all selected ports.');
|
|
$assert(collect($batchInstances)->every(static fn (
|
|
CloudInstance $instance
|
|
): bool => !CloudJobService::blockingQueryForInstance(
|
|
(int) $instance->id
|
|
)->exists()), 'Completed batch left an instance operation fence behind.');
|
|
|
|
$detachBatch = CloudVpcMembershipBatchService::start([
|
|
'user_id' => 7,
|
|
'request_idempotency_key' => 'vpc-batch-detach-more-than-four-0001',
|
|
], (string) $vpc->vpc_key, [
|
|
'action' => 'detach',
|
|
'instance_ids' => array_map(
|
|
static fn (CloudInstance $instance): int => (int) $instance->id,
|
|
$batchInstances
|
|
),
|
|
'confirm_restart' => true,
|
|
]);
|
|
$detachBatchId = (int) CloudVpcMembershipBatch::where(
|
|
'parent_job_id', (int) $detachBatch['job']->id
|
|
)->value('id');
|
|
$detachPortJobs = CloudJob::where('vpc_batch_id', $detachBatchId)
|
|
->where('kind', 'vpc_port_detach')->get();
|
|
$assert($detachBatchId > 0 && $detachPortJobs->count() === 6
|
|
&& $detachPortJobs->every(static fn (CloudJob $job): bool =>
|
|
(string) $job->phase === 'accepted'
|
|
&& (int) $job->dispatch_attempts === 0)
|
|
&& CloudJob::where('vpc_batch_id', $detachBatchId)
|
|
->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove'])
|
|
->count() === 0,
|
|
'Batch detach did not release every selected instance before route cleanup.');
|
|
foreach ($detachPortJobs as $index => $detachPortJob) {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $detachPortJob->sealed_payload, 'job-payload'
|
|
);
|
|
$instance = CloudInstance::find((int) $detachPortJob->instance_id);
|
|
$batchPort = CloudVpcPort::find((int) $detachPortJob->vpc_port_id);
|
|
CloudVpcPortService::finishJobLocked($detachPortJob, 'succeeded', [
|
|
'uuid' => (string) $instance->uuid,
|
|
'port_key' => (string) $batchPort->port_key,
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'status' => 'running', 'removed' => true,
|
|
], $payload, [
|
|
'operation_id' => (string) $detachPortJob->operation_id,
|
|
]);
|
|
$terminalize($detachPortJob);
|
|
$cleanupCount = CloudJob::where('vpc_batch_id', $detachBatchId)
|
|
->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove'])->count();
|
|
$assert($cleanupCount === ($index + 1 === $detachPortJobs->count()
|
|
? 2 : 0),
|
|
'Batch detach started route cleanup before or after the exact VM barrier.');
|
|
}
|
|
$detachRouteJobs = CloudJob::where('vpc_batch_id', $detachBatchId)
|
|
->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove'])
|
|
->orderBy('id')->get();
|
|
foreach ($detachRouteJobs as $routeJob) {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $routeJob->sealed_payload, 'job-payload'
|
|
);
|
|
$hostState = CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', (int) $routeJob->host_id)->first();
|
|
$result = [
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'desired_revision' => (int) $hostState->applied_revision,
|
|
'desired_digest' => (string) $hostState->applied_digest,
|
|
'removed' => true,
|
|
];
|
|
CloudVpcPortService::finishJobLocked($routeJob, 'succeeded',
|
|
$result, $payload,
|
|
['operation_id' => (string) $routeJob->operation_id]);
|
|
$terminalize($routeJob);
|
|
}
|
|
$assert((string) $detachBatch['job']->fresh()->outcome === 'succeeded'
|
|
&& CloudVpcPort::where('vpc_id', 1)
|
|
->where('active_claim', 1)->count() === 0
|
|
&& CloudVpcHostState::where('vpc_id', 1)->count() === 0,
|
|
'Batch detach did not converge route cleanup before releasing ports.');
|
|
|
|
// A known route-stage failure automatically creates a new projection that
|
|
// excludes every reserved port. Instance fences remain active until that
|
|
// rollback converges, then the IPs and fences are released together.
|
|
$rollbackInstances = [$newInstance(20, 1), $newInstance(21, 2)];
|
|
$rollbackBatch = CloudVpcMembershipBatchService::start([
|
|
'user_id' => 7,
|
|
'request_idempotency_key' => 'vpc-batch-attach-rollback-safe-0001',
|
|
], (string) $vpc->vpc_key, [
|
|
'action' => 'attach',
|
|
'instance_ids' => [20, 21],
|
|
'confirm_restart' => true,
|
|
]);
|
|
$rollbackBatchId = (int) CloudVpcMembershipBatch::where(
|
|
'parent_job_id', (int) $rollbackBatch['job']->id
|
|
)->value('id');
|
|
$rollbackRouteJobs = CloudJob::where('vpc_batch_id', $rollbackBatchId)
|
|
->where('kind', 'vpc_host_apply')->orderBy('id')->get();
|
|
foreach ($rollbackRouteJobs as $index => $routeJob) {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $routeJob->sealed_payload, 'job-payload'
|
|
);
|
|
if ($index === 0) {
|
|
$hostState = CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', (int) $routeJob->host_id)->first();
|
|
CloudVpcPortService::finishJobLocked($routeJob, 'succeeded', [
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'desired_revision' => (int) $hostState->desired_revision,
|
|
'desired_digest' => (string) $hostState->desired_digest,
|
|
'bridge' => CloudVpcProjection::bridgeName(
|
|
(string) $vpc->vpc_key
|
|
),
|
|
'applied' => true,
|
|
], $payload, [
|
|
'operation_id' => (string) $routeJob->operation_id,
|
|
]);
|
|
$terminalize($routeJob);
|
|
} else {
|
|
CloudVpcPortService::finishJobLocked(
|
|
$routeJob,
|
|
'failed',
|
|
null,
|
|
$payload,
|
|
[
|
|
'operation_id' => (string) $routeJob->operation_id,
|
|
'error_code' => 'vpc_host_apply_failed',
|
|
]
|
|
);
|
|
$terminalizeFailed($routeJob);
|
|
}
|
|
}
|
|
$rollbackBatchRow = CloudVpcMembershipBatch::find($rollbackBatchId);
|
|
$rollbackCleanupJobs = CloudJob::where(
|
|
'vpc_batch_id', $rollbackBatchId
|
|
)->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove'])
|
|
->where('phase', 'accepted')->orderBy('id')->get();
|
|
$assert((string) $rollbackBatchRow->state === 'cleanup'
|
|
&& $rollbackCleanupJobs->count() === 1
|
|
&& CloudVpcPort::whereIn('instance_id', [20, 21])
|
|
->where('state', 'release_pending')->count() === 2
|
|
&& CloudJob::where('vpc_batch_id', $rollbackBatchId)
|
|
->where('kind', 'vpc_batch_instance_fence')
|
|
->where('phase', '<>', 'terminal')->count() === 2,
|
|
'Known route failure did not enter a fenced rollback generation.');
|
|
foreach ($rollbackCleanupJobs as $cleanupJob) {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $cleanupJob->sealed_payload, 'job-payload'
|
|
);
|
|
$hostState = CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', (int) $cleanupJob->host_id)->first();
|
|
CloudVpcPortService::finishJobLocked($cleanupJob, 'succeeded', [
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'desired_revision' => (int) $hostState->applied_revision,
|
|
'desired_digest' => (string) $hostState->applied_digest,
|
|
'removed' => true,
|
|
], $payload, [
|
|
'operation_id' => (string) $cleanupJob->operation_id,
|
|
]);
|
|
$terminalize($cleanupJob);
|
|
}
|
|
$assert((string) $rollbackBatch['job']->fresh()->outcome === 'failed'
|
|
&& (int) $rollbackBatch['job']->fresh()->safe_to_resubmit === 1
|
|
&& CloudVpcPort::whereIn('instance_id', [20, 21])
|
|
->whereNotNull('active_claim')->count() === 0
|
|
&& CloudJob::where('vpc_batch_id', $rollbackBatchId)
|
|
->where('kind', 'vpc_batch_instance_fence')
|
|
->where('phase', '<>', 'terminal')->count() === 0,
|
|
'Successful attach rollback did not release ports and instance fences.');
|
|
|
|
// A cleanup failure remains fenced. Once its exact Agent job is terminal
|
|
// and proven not applied, the user can continue the same rollback parent;
|
|
// an unknown result would keep can_cancel false and be rejected.
|
|
$newInstance(30, 1); $newInstance(31, 2);
|
|
$retryBatch = CloudVpcMembershipBatchService::start([
|
|
'user_id' => 7,
|
|
'request_idempotency_key' => 'vpc-batch-attach-rollback-retry-0001',
|
|
], (string) $vpc->vpc_key, [
|
|
'action' => 'attach', 'instance_ids' => [30, 31],
|
|
'confirm_restart' => true,
|
|
]);
|
|
$retryBatchId = (int) CloudVpcMembershipBatch::where(
|
|
'parent_job_id', (int) $retryBatch['job']->id
|
|
)->value('id');
|
|
$initialJobs = CloudJob::where('vpc_batch_id', $retryBatchId)
|
|
->where('kind', 'vpc_host_apply')->orderBy('id')->get();
|
|
foreach ($initialJobs as $index => $routeJob) {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $routeJob->sealed_payload, 'job-payload'
|
|
);
|
|
if ($index === 0) {
|
|
$hostState = CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', (int) $routeJob->host_id)->first();
|
|
CloudVpcPortService::finishJobLocked($routeJob, 'succeeded', [
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'desired_revision' => (int) $hostState->desired_revision,
|
|
'desired_digest' => (string) $hostState->desired_digest,
|
|
'bridge' => CloudVpcProjection::bridgeName(
|
|
(string) $vpc->vpc_key
|
|
),
|
|
'applied' => true,
|
|
], $payload, [
|
|
'operation_id' => (string) $routeJob->operation_id,
|
|
]);
|
|
$terminalize($routeJob);
|
|
} else {
|
|
CloudVpcPortService::finishJobLocked($routeJob, 'failed', null,
|
|
$payload, [
|
|
'operation_id' => (string) $routeJob->operation_id,
|
|
'error_code' => 'vpc_host_apply_failed',
|
|
]);
|
|
$terminalizeFailed($routeJob);
|
|
}
|
|
}
|
|
$failedCleanup = CloudJob::where('vpc_batch_id', $retryBatchId)
|
|
->where('kind', 'vpc_host_remove')->where('phase', 'accepted')
|
|
->first();
|
|
$failedCleanupPayload = CloudSecrets::decrypt(
|
|
(string) $failedCleanup->sealed_payload, 'job-payload'
|
|
);
|
|
CloudVpcPortService::finishJobLocked($failedCleanup, 'failed', null,
|
|
$failedCleanupPayload, [
|
|
'operation_id' => (string) $failedCleanup->operation_id,
|
|
'error_code' => 'vpc_host_remove_failed',
|
|
]);
|
|
$terminalizeFailed($failedCleanup);
|
|
$retryBatchRow = CloudVpcMembershipBatch::find($retryBatchId);
|
|
$retryDto = collect(
|
|
CloudVpcMembershipBatchService::activeForOwner(7)
|
|
)->firstWhere('batch_key', (string) $retryBatchRow->batch_key);
|
|
$expectedItemStates = CloudVpcMembershipBatchItem::query()
|
|
->where('batch_id', $retryBatchId)->orderBy('instance_id')->get()
|
|
->mapWithKeys(static fn ($item): array => [
|
|
(string) $item->instance_id => (string) $item->state,
|
|
])->all();
|
|
$projectedItemStates = collect($retryDto['items'] ?? [])
|
|
->mapWithKeys(static fn (array $item): array => [
|
|
(string) ($item['instance_id'] ?? '') =>
|
|
(string) ($item['state'] ?? ''),
|
|
])->all();
|
|
$assert((string) $retryBatchRow->state === 'blocked'
|
|
&& ($retryDto['can_cancel'] ?? false) === true
|
|
&& $projectedItemStates === $expectedItemStates
|
|
&& collect([30, 31])->every(static fn (int $id): bool =>
|
|
CloudJobService::blockingQueryForInstance($id)->exists()),
|
|
'Known-not-applied rollback failure did not expose item progress and a safe cancel exit.');
|
|
$retryResult = CloudVpcMembershipBatchService::cancelAttach(
|
|
['user_id' => 7],
|
|
(string) $vpc->vpc_key,
|
|
(string) $retryBatchRow->batch_key
|
|
);
|
|
$assert((int) $retryResult['job']->id === (int) $retryBatch['job']->id
|
|
&& (string) CloudVpcMembershipBatch::find($retryBatchId)->state
|
|
=== 'cleanup',
|
|
'Safe rollback retry did not continue the original parent task.');
|
|
$retriedCleanup = CloudJob::where('vpc_batch_id', $retryBatchId)
|
|
->where('kind', 'vpc_host_remove')->where('phase', 'accepted')
|
|
->first();
|
|
$retriedPayload = CloudSecrets::decrypt(
|
|
(string) $retriedCleanup->sealed_payload, 'job-payload'
|
|
);
|
|
$retryHostState = CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', (int) $retriedCleanup->host_id)->first();
|
|
CloudVpcPortService::finishJobLocked($retriedCleanup, 'succeeded', [
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'desired_revision' => (int) $retryHostState->applied_revision,
|
|
'desired_digest' => (string) $retryHostState->applied_digest,
|
|
'removed' => true,
|
|
], $retriedPayload, [
|
|
'operation_id' => (string) $retriedCleanup->operation_id,
|
|
]);
|
|
$terminalize($retriedCleanup);
|
|
$assert((string) $retryBatch['job']->fresh()->outcome === 'failed',
|
|
'Retried rollback did not terminalize its original parent.');
|
|
$assert(CloudVpcPort::whereIn('instance_id', [30, 31])
|
|
->whereNotNull('active_claim')->count() === 0,
|
|
'Retried rollback did not release every reserved private IP.');
|
|
$assert(collect([30, 31])->every(static fn (int $id): bool =>
|
|
!CloudJobService::blockingQueryForInstance($id)->exists()),
|
|
'Retried rollback left an instance operation fence behind.');
|
|
|
|
// A reachable but inconclusive VPC generation is repaired by a new
|
|
// operation. If the first repair also loses its result, a second exact
|
|
// repair may supersede the complete chain without rewriting history.
|
|
$repairInstance = $newInstance(40, 1);
|
|
$repairAttach = CloudVpcPortService::attach($repairInstance, [
|
|
'user_id' => 7,
|
|
'request_idempotency_key' => 'vpc-repair-fixture-attach-0001',
|
|
], [
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'confirm_restart' => true,
|
|
]);
|
|
$repairOriginal = CloudJob::where('kind', 'vpc_host_apply')
|
|
->where('instance_id', 40)->where('phase', 'accepted')->first();
|
|
$assert($repairAttach['job'] !== null && $repairOriginal !== null,
|
|
'VPC repair fixture did not create its host generation.');
|
|
$repairFinishedAt = time();
|
|
$repairOriginal->phase = 'terminal';
|
|
$repairOriginal->outcome = $repairOriginal->status = 'failed';
|
|
$repairOriginal->attempts = $repairOriginal->dispatch_attempts = 1;
|
|
$repairOriginal->dispatch_started_at =
|
|
$repairOriginal->dispatched_at = $repairFinishedAt - 2;
|
|
$repairOriginal->agent_accepted_at = $repairFinishedAt - 1;
|
|
$repairOriginal->receipt_deadline_at = $repairFinishedAt + 900;
|
|
$repairOriginal->terminal_at = $repairFinishedAt;
|
|
$repairOriginal->effect_may_have_applied = 1;
|
|
$repairOriginal->safe_to_resubmit = 0;
|
|
$repairOriginal->effect_resolution = 'unknown';
|
|
$repairOriginal->error_code = 'operation_effect_unknown';
|
|
$repairOriginal->updated_at = $repairFinishedAt;
|
|
$repairOriginal->save();
|
|
CloudVpcPortService::markJobUncertain($repairOriginal);
|
|
$repairInspection = static function (CloudJob $job): array {
|
|
return [
|
|
'operation_id' => (string) $job->operation_id,
|
|
'journal_status' => 'failed',
|
|
'checked_at' => time(),
|
|
'outcome' => 'inconclusive',
|
|
'effect_resolution' => 'inconclusive',
|
|
'fence_state' => 'active',
|
|
'fence_resolution_required' => true,
|
|
];
|
|
};
|
|
$repairActor = [
|
|
'user_id' => 1, 'is_admin' => true,
|
|
'is_super_admin' => true, 'can_manage' => true,
|
|
];
|
|
$repairOne = CloudVpcPortService::repairUnknownEffect(
|
|
$repairActor,
|
|
(int) $repairOriginal->id,
|
|
$repairInspection($repairOriginal)
|
|
);
|
|
$repairOnePayload = CloudSecrets::decrypt(
|
|
(string) $repairOne->sealed_payload, 'job-payload'
|
|
);
|
|
$repairState = CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', 1)->first();
|
|
$assert((int) $repairOne->supersedes_job_id
|
|
=== (int) $repairOriginal->id
|
|
&& (int) $repairOne->vpc_repair_root_job_id
|
|
=== (int) $repairOriginal->id
|
|
&& ($repairOnePayload['body']['vpc_repair'] ?? false) === true
|
|
&& ($repairOnePayload['body']['supersedes_operation_id'] ?? null)
|
|
=== (string) $repairOriginal->operation_id
|
|
&& str_ends_with(
|
|
(string) $repairOnePayload['request_target'], '/repair'
|
|
)
|
|
&& (string) $repairState->last_operation_id
|
|
=== (string) $repairOne->operation_id,
|
|
'First VPC repair was not bound to the exact predecessor generation.');
|
|
CloudVpcPortService::finishJobLocked(
|
|
$repairOne,
|
|
'uncertain',
|
|
null,
|
|
$repairOnePayload,
|
|
[
|
|
'operation_id' => (string) $repairOne->operation_id,
|
|
'error_code' => 'vpc_reconcile_unconfirmed',
|
|
]
|
|
);
|
|
$repairOne->phase = 'terminal';
|
|
$repairOne->outcome = $repairOne->status = 'failed';
|
|
$repairOne->attempts = $repairOne->dispatch_attempts = 1;
|
|
$repairOne->dispatch_started_at =
|
|
$repairOne->dispatched_at = time() - 2;
|
|
$repairOne->agent_accepted_at = time() - 1;
|
|
$repairOne->receipt_deadline_at = time() + 900;
|
|
$repairOne->terminal_at = time();
|
|
$repairOne->effect_may_have_applied = 1;
|
|
$repairOne->safe_to_resubmit = 0;
|
|
$repairOne->effect_resolution = 'unknown';
|
|
$repairOne->error_code = 'operation_effect_unknown';
|
|
$repairOne->updated_at = time();
|
|
$repairOne->save();
|
|
|
|
$repairTwo = CloudVpcPortService::repairUnknownEffect(
|
|
$repairActor,
|
|
(int) $repairOne->id,
|
|
$repairInspection($repairOne)
|
|
);
|
|
$repairTwoPayload = CloudSecrets::decrypt(
|
|
(string) $repairTwo->sealed_payload, 'job-payload'
|
|
);
|
|
$repairState = CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', 1)->first();
|
|
CloudVpcPortService::finishJobLocked($repairTwo, 'succeeded', [
|
|
'vpc_key' => (string) $vpc->vpc_key,
|
|
'desired_revision' => (int) $repairState->desired_revision,
|
|
'desired_digest' => (string) $repairState->desired_digest,
|
|
'bridge' => CloudVpcProjection::bridgeName((string) $vpc->vpc_key),
|
|
'applied' => true,
|
|
], $repairTwoPayload, [
|
|
'operation_id' => (string) $repairTwo->operation_id,
|
|
]);
|
|
$terminalize($repairTwo);
|
|
$assert((int) $repairTwo->supersedes_job_id === (int) $repairOne->id
|
|
&& (int) $repairTwo->vpc_repair_root_job_id
|
|
=== (int) $repairOriginal->id
|
|
&& (string) $repairOriginal->fresh()->effect_resolution === 'applied'
|
|
&& str_starts_with(
|
|
(string) $repairOriginal->fresh()->effect_resolution_reason,
|
|
'superseded_by_vpc_repair:'
|
|
)
|
|
&& (string) $repairOne->fresh()->effect_resolution === 'applied'
|
|
&& (string) CloudVpcHostState::where('vpc_id', 1)
|
|
->where('host_id', 1)->value('sync_state') === 'ready'
|
|
&& CloudJob::where('kind', 'vpc_port_attach')
|
|
->where('instance_id', 40)->where('phase', 'accepted')->exists(),
|
|
'A successful chained VPC repair did not converge every fence and ledger.');
|
|
|
|
$worker = file_get_contents(
|
|
dirname(__DIR__) . '/app/Services/Cloud/CloudWorkerService.php'
|
|
);
|
|
$cleanupPosition = strpos($worker,
|
|
'CloudVpcPortService::releaseAfterInstanceDeletedLocked');
|
|
$deletedPosition = strpos($worker, "\$instance->status = 'deleted';");
|
|
$placementPosition = strpos($worker,
|
|
'CloudPlacementService::releaseDeletedInstance');
|
|
$assert($cleanupPosition !== false && $deletedPosition !== false
|
|
&& $placementPosition !== false
|
|
&& $cleanupPosition < $deletedPosition
|
|
&& $deletedPosition < $placementPosition,
|
|
'Delete finalization does not fence VPC cleanup before resource release.');
|
|
|
|
echo "VPC host projection and VM port workflow passed.\n";
|
|
}
|