You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
758 lines
30 KiB
PHP
758 lines
30 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud {
|
|
final class CloudSecrets
|
|
{
|
|
public static function encrypt(array $value, string $purpose): string
|
|
{
|
|
return json_encode(
|
|
$value,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
);
|
|
}
|
|
|
|
public static function decrypt(string $value, string $purpose): array
|
|
{
|
|
$decoded = json_decode($value, true, 64, JSON_THROW_ON_ERROR);
|
|
return is_array($decoded) ? $decoded : [];
|
|
}
|
|
}
|
|
|
|
final class CloudInstanceMigrationService
|
|
{
|
|
public static function assertInstanceMutationAvailable(
|
|
\App\Models\CloudInstance $instance,
|
|
?string $operationKind = null
|
|
): void {
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudPublicIpPoolReplacementBatch;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudJobService;
|
|
use App\Services\Cloud\CloudPlacementService;
|
|
use App\Services\Cloud\CloudPublicIpPoolReplacementBatchService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$expectCloudError = static function (
|
|
callable $callback,
|
|
int $status,
|
|
string $message
|
|
) use ($assert): void {
|
|
try {
|
|
$callback();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === $status, $message
|
|
. ' (unexpected HTTP status)');
|
|
return;
|
|
}
|
|
throw new RuntimeException($message . ' (no CloudError)');
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite',
|
|
'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
|
|
$schema->create('cloud_hosts', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('name');
|
|
$table->boolean('enabled')->default(true);
|
|
$table->integer('created_at')->default(0);
|
|
$table->integer('updated_at')->default(0);
|
|
});
|
|
$schema->create('cloud_host_agents', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->integer('host_id')->primary();
|
|
$table->text('sealed_api_secret')->default('sealed');
|
|
$table->boolean('enabled');
|
|
$table->integer('revision');
|
|
$table->integer('tested_revision');
|
|
$table->string('last_test_status');
|
|
$table->string('last_error_code')->default('');
|
|
$table->text('network_selectors_json');
|
|
$table->integer('last_tested_at')->default(0);
|
|
$table->integer('created_at')->default(0);
|
|
$table->integer('updated_at')->default(0);
|
|
});
|
|
$schema->create('cloud_host_network_attachments', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->integer('host_id');
|
|
$table->integer('line_id');
|
|
$table->string('network_selector');
|
|
$table->string('state');
|
|
$table->integer('priority')->default(0);
|
|
$table->string('verification_state');
|
|
$table->integer('verified_agent_revision');
|
|
$table->integer('last_verified_at');
|
|
$table->integer('created_at')->default(0);
|
|
$table->integer('updated_at')->default(0);
|
|
});
|
|
$schema->create('cloud_public_ip_pools', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->string('pool_key', 32)->unique();
|
|
$table->integer('host_id');
|
|
$table->integer('network_attachment_id');
|
|
$table->string('name');
|
|
$table->string('cidr');
|
|
$table->string('gateway');
|
|
$table->integer('guest_prefix_length');
|
|
$table->text('dns_servers_json');
|
|
$table->text('reserved_ips_json');
|
|
$table->string('state');
|
|
$table->integer('priority')->default(0);
|
|
$table->integer('desired_revision');
|
|
$table->integer('observed_revision');
|
|
$table->string('sync_state');
|
|
$table->string('last_error_code')->default('');
|
|
$table->integer('created_at')->default(0);
|
|
$table->integer('updated_at')->default(0);
|
|
});
|
|
$schema->create('cloud_resource_pools', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->integer('line_id');
|
|
});
|
|
$schema->create('cloud_resource_reservations', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id', 32)->unique();
|
|
$table->integer('resource_pool_id');
|
|
$table->integer('host_id');
|
|
$table->integer('user_id');
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('network_attachment_id');
|
|
$table->integer('public_ip_pool_id');
|
|
$table->integer('public_ip_id');
|
|
$table->string('state');
|
|
$table->integer('created_at')->default(0);
|
|
$table->integer('updated_at')->default(0);
|
|
});
|
|
$schema->create('cloud_instances', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->string('uuid');
|
|
$table->string('name');
|
|
$table->integer('user_id');
|
|
$table->integer('host_id');
|
|
$table->integer('resource_pool_id');
|
|
$table->integer('reservation_id');
|
|
$table->string('status');
|
|
$table->text('spec_json');
|
|
$table->integer('created_at')->default(0);
|
|
$table->integer('updated_at')->default(0);
|
|
});
|
|
$schema->create('cloud_public_ips', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->integer('host_id');
|
|
$table->integer('public_ip_pool_id');
|
|
$table->string('address');
|
|
$table->integer('prefix_length');
|
|
$table->string('state');
|
|
$table->integer('reservation_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->string('operation_id', 32)->nullable();
|
|
$table->integer('created_at')->default(0);
|
|
$table->integer('updated_at')->default(0);
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id', 32)->unique();
|
|
$table->string('request_idempotency_key', 128);
|
|
$table->string('request_fingerprint', 64);
|
|
$table->integer('parent_job_id')->nullable();
|
|
$table->integer('supersedes_job_id')->nullable();
|
|
$table->integer('vpc_repair_root_job_id')->nullable();
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('invoice_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('snapshot_id')->nullable();
|
|
$table->integer('backup_id')->nullable();
|
|
$table->integer('backup_restore_id')->nullable();
|
|
$table->integer('backup_recovery_id')->nullable();
|
|
$table->integer('vpc_id')->nullable();
|
|
$table->integer('vpc_port_id')->nullable();
|
|
$table->integer('vpc_batch_id')->nullable();
|
|
$table->integer('user_id');
|
|
$table->integer('host_id')->nullable();
|
|
$table->string('kind');
|
|
$table->string('execution_mode');
|
|
$table->string('phase');
|
|
$table->string('outcome')->nullable();
|
|
$table->string('status');
|
|
$table->text('sealed_payload');
|
|
$table->integer('attempts');
|
|
$table->integer('dispatch_attempts');
|
|
$table->integer('dispatch_deadline_at');
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->string('effect_resolution');
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
$table->text('effect_resolution_reason')->nullable();
|
|
$table->boolean('finalization_incident')->default(false);
|
|
$table->string('finalization_error_code')->nullable();
|
|
$table->string('completion_event_id')->nullable();
|
|
$table->integer('read_at')->nullable();
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('available_at');
|
|
$table->string('error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->unique(['user_id', 'request_idempotency_key']);
|
|
});
|
|
$schema->create('cloud_public_ip_pool_replacement_batches',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('batch_key', 32)->unique();
|
|
$table->integer('host_id');
|
|
$table->integer('source_public_ip_pool_id');
|
|
$table->integer('target_public_ip_pool_id');
|
|
$table->string('source_initial_state');
|
|
$table->integer('parent_job_id')->unique();
|
|
$table->integer('requested_by_user_id');
|
|
$table->string('state');
|
|
$table->integer('active_claim')->nullable();
|
|
$table->string('last_error_code')->default('');
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_public_ip_pool_replacement_batch_items',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('batch_id');
|
|
$table->integer('instance_id');
|
|
$table->integer('source_reservation_id');
|
|
$table->integer('source_public_ip_id');
|
|
$table->integer('target_public_ip_id');
|
|
$table->integer('source_network_attachment_id');
|
|
$table->integer('target_network_attachment_id');
|
|
$table->integer('child_job_id')->unique();
|
|
$table->string('state');
|
|
$table->string('last_error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->unique(['batch_id', 'target_public_ip_id']);
|
|
});
|
|
|
|
$now = time();
|
|
Capsule::table('cloud_hosts')->insert([
|
|
'id' => 1,
|
|
'name' => 'HKG-A',
|
|
'enabled' => 1,
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_host_agents')->insert([
|
|
'host_id' => 1,
|
|
'enabled' => 1,
|
|
'revision' => 7,
|
|
'tested_revision' => 7,
|
|
'last_test_status' => 'healthy',
|
|
'network_selectors_json' => json_encode([
|
|
'bridge:cross-line', 'bridge:other-source', 'bridge:source',
|
|
'bridge:target',
|
|
], JSON_THROW_ON_ERROR),
|
|
'last_tested_at' => $now,
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_host_network_attachments')->insert([
|
|
['id' => 11, 'host_id' => 1, 'line_id' => 10,
|
|
'network_selector' => 'bridge:source', 'state' => 'active',
|
|
'priority' => 10, 'verification_state' => 'verified',
|
|
'verified_agent_revision' => 7, 'last_verified_at' => $now],
|
|
['id' => 12, 'host_id' => 1, 'line_id' => 10,
|
|
'network_selector' => 'bridge:target', 'state' => 'active',
|
|
'priority' => 20, 'verification_state' => 'verified',
|
|
'verified_agent_revision' => 7, 'last_verified_at' => $now],
|
|
['id' => 13, 'host_id' => 1, 'line_id' => 10,
|
|
'network_selector' => 'bridge:other-source', 'state' => 'active',
|
|
'priority' => 30, 'verification_state' => 'verified',
|
|
'verified_agent_revision' => 7, 'last_verified_at' => $now],
|
|
['id' => 14, 'host_id' => 1, 'line_id' => 20,
|
|
'network_selector' => 'bridge:cross-line', 'state' => 'active',
|
|
'priority' => 40, 'verification_state' => 'verified',
|
|
'verified_agent_revision' => 7, 'last_verified_at' => $now],
|
|
]);
|
|
Capsule::table('cloud_resource_pools')->insert([
|
|
['id' => 21, 'line_id' => 10],
|
|
]);
|
|
$pool = static function (
|
|
int $id,
|
|
int $attachmentId,
|
|
string $keyDigit,
|
|
string $name,
|
|
string $cidr,
|
|
string $gateway,
|
|
int $revision
|
|
): array {
|
|
return [
|
|
'id' => $id,
|
|
'pool_key' => str_repeat($keyDigit, 32),
|
|
'host_id' => 1,
|
|
'network_attachment_id' => $attachmentId,
|
|
'name' => $name,
|
|
'cidr' => $cidr,
|
|
'gateway' => $gateway,
|
|
'guest_prefix_length' => 29,
|
|
'dns_servers_json' => json_encode(['1.1.1.1']),
|
|
'reserved_ips_json' => '[]',
|
|
'state' => 'active',
|
|
'priority' => $id,
|
|
'desired_revision' => $revision,
|
|
'observed_revision' => $revision,
|
|
'sync_state' => 'applied',
|
|
'last_error_code' => '',
|
|
];
|
|
};
|
|
Capsule::table('cloud_public_ip_pools')->insert([
|
|
$pool(31, 11, 'a', 'Source A', '198.51.100.0/29',
|
|
'198.51.100.1', 7),
|
|
$pool(32, 12, 'b', 'Target B', '198.51.100.8/29',
|
|
'198.51.100.9', 9),
|
|
$pool(33, 13, 'c', 'Source D', '198.51.100.16/29',
|
|
'198.51.100.17', 4),
|
|
$pool(34, 14, 'd', 'Cross Line', '198.51.100.24/29',
|
|
'198.51.100.25', 3),
|
|
]);
|
|
$reservationOperation = static fn (int $id): string => str_pad(
|
|
dechex($id),
|
|
32,
|
|
'0',
|
|
STR_PAD_LEFT
|
|
);
|
|
Capsule::table('cloud_resource_reservations')->insert([
|
|
['id' => 41, 'operation_id' => $reservationOperation(41),
|
|
'resource_pool_id' => 21, 'host_id' => 1, 'user_id' => 501,
|
|
'instance_id' => 51, 'network_attachment_id' => 11,
|
|
'public_ip_pool_id' => 31, 'public_ip_id' => 61,
|
|
'state' => 'committed'],
|
|
['id' => 42, 'operation_id' => $reservationOperation(42),
|
|
'resource_pool_id' => 21, 'host_id' => 1, 'user_id' => 502,
|
|
'instance_id' => 52, 'network_attachment_id' => 11,
|
|
'public_ip_pool_id' => 31, 'public_ip_id' => 62,
|
|
'state' => 'committed'],
|
|
['id' => 43, 'operation_id' => $reservationOperation(43),
|
|
'resource_pool_id' => 21, 'host_id' => 1, 'user_id' => 503,
|
|
'instance_id' => 53, 'network_attachment_id' => 13,
|
|
'public_ip_pool_id' => 33, 'public_ip_id' => 63,
|
|
'state' => 'committed'],
|
|
]);
|
|
$instanceSpec = static function (string $publicIp): string {
|
|
return json_encode([
|
|
'managed' => true,
|
|
'os_source' => 'cloud_template',
|
|
'public_ip_cidr' => $publicIp . '/29',
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES);
|
|
};
|
|
Capsule::table('cloud_instances')->insert([
|
|
['id' => 51, 'uuid' => '11111111-1111-4111-8111-111111111111',
|
|
'name' => 'vm-a1', 'user_id' => 501, 'host_id' => 1,
|
|
'resource_pool_id' => 21, 'reservation_id' => 41,
|
|
'status' => 'active',
|
|
'spec_json' => $instanceSpec('198.51.100.2')],
|
|
['id' => 52, 'uuid' => '22222222-2222-4222-8222-222222222222',
|
|
'name' => 'vm-a2', 'user_id' => 502, 'host_id' => 1,
|
|
'resource_pool_id' => 21, 'reservation_id' => 42,
|
|
'status' => 'active',
|
|
'spec_json' => $instanceSpec('198.51.100.3')],
|
|
['id' => 53, 'uuid' => '33333333-3333-4333-8333-333333333333',
|
|
'name' => 'vm-d1', 'user_id' => 503, 'host_id' => 1,
|
|
'resource_pool_id' => 21, 'reservation_id' => 43,
|
|
'status' => 'active',
|
|
'spec_json' => $instanceSpec('198.51.100.18')],
|
|
]);
|
|
Capsule::table('cloud_public_ips')->insert([
|
|
['id' => 61, 'host_id' => 1, 'public_ip_pool_id' => 31,
|
|
'address' => '198.51.100.2', 'prefix_length' => 29,
|
|
'state' => 'assigned', 'reservation_id' => 41,
|
|
'instance_id' => 51,
|
|
'operation_id' => $reservationOperation(41)],
|
|
['id' => 62, 'host_id' => 1, 'public_ip_pool_id' => 31,
|
|
'address' => '198.51.100.3', 'prefix_length' => 29,
|
|
'state' => 'assigned', 'reservation_id' => 42,
|
|
'instance_id' => 52,
|
|
'operation_id' => $reservationOperation(42)],
|
|
['id' => 63, 'host_id' => 1, 'public_ip_pool_id' => 33,
|
|
'address' => '198.51.100.18', 'prefix_length' => 29,
|
|
'state' => 'assigned', 'reservation_id' => 43,
|
|
'instance_id' => 53,
|
|
'operation_id' => $reservationOperation(43)],
|
|
['id' => 71, 'host_id' => 1, 'public_ip_pool_id' => 32,
|
|
'address' => '198.51.100.10', 'prefix_length' => 29,
|
|
'state' => 'available', 'reservation_id' => null,
|
|
'instance_id' => null, 'operation_id' => null],
|
|
['id' => 72, 'host_id' => 1, 'public_ip_pool_id' => 32,
|
|
'address' => '198.51.100.11', 'prefix_length' => 29,
|
|
'state' => 'available', 'reservation_id' => null,
|
|
'instance_id' => null, 'operation_id' => null],
|
|
['id' => 73, 'host_id' => 1, 'public_ip_pool_id' => 32,
|
|
'address' => '198.51.100.12', 'prefix_length' => 29,
|
|
'state' => 'available', 'reservation_id' => null,
|
|
'instance_id' => null, 'operation_id' => null],
|
|
['id' => 74, 'host_id' => 1, 'public_ip_pool_id' => 32,
|
|
'address' => '198.51.100.13', 'prefix_length' => 29,
|
|
'state' => 'available', 'reservation_id' => null,
|
|
'instance_id' => null, 'operation_id' => null],
|
|
['id' => 75, 'host_id' => 1, 'public_ip_pool_id' => 34,
|
|
'address' => '198.51.100.26', 'prefix_length' => 29,
|
|
'state' => 'available', 'reservation_id' => null,
|
|
'instance_id' => null, 'operation_id' => null],
|
|
]);
|
|
|
|
$admin = [
|
|
'user_id' => 99,
|
|
'is_admin' => true,
|
|
'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
];
|
|
$readAdmin = [
|
|
'user_id' => 98,
|
|
'is_admin' => true,
|
|
'is_super_admin' => false,
|
|
'can_manage' => false,
|
|
];
|
|
$sourceKey = str_repeat('a', 32);
|
|
$targetKey = str_repeat('b', 32);
|
|
$otherSourceKey = str_repeat('c', 32);
|
|
|
|
$overview = CloudPublicIpPoolReplacementBatchService::overview(
|
|
1,
|
|
$sourceKey,
|
|
$readAdmin
|
|
);
|
|
$assert($overview['source_pool']['expected_revision'] === 7
|
|
&& $overview['source_pool']['assigned_count'] === 2
|
|
&& $overview['source_pool']['replacement_locked'] === false
|
|
&& $overview['can_start'] === true,
|
|
'Read-only administrator overview lost source revision or occupancy.');
|
|
$targetOptions = array_column(
|
|
$overview['target_options'],
|
|
null,
|
|
'pool_key'
|
|
);
|
|
$assert(isset($targetOptions[$targetKey])
|
|
&& $targetOptions[$targetKey]['expected_revision'] === 9
|
|
&& $targetOptions[$targetKey]['available_count'] === 4
|
|
&& $targetOptions[$targetKey]['eligible'] === true
|
|
&& !isset($targetOptions[str_repeat('d', 32)]),
|
|
'Overview did not enforce same-line verified target selection.');
|
|
$expectCloudError(
|
|
static fn () => CloudPublicIpPoolReplacementBatchService::overview(
|
|
1,
|
|
$sourceKey,
|
|
['user_id' => 501, 'is_admin' => false]
|
|
),
|
|
403,
|
|
'Tenant must not read administrator replacement batches.'
|
|
);
|
|
|
|
$staleInput = [
|
|
'target_pool_key' => $targetKey,
|
|
'expected_source_revision' => 6,
|
|
'expected_target_revision' => 9,
|
|
'confirm_replacement' => true,
|
|
'request_key' => 'replacement-stale-0001',
|
|
];
|
|
$expectCloudError(
|
|
static fn () => CloudPublicIpPoolReplacementBatchService::start(
|
|
1,
|
|
$sourceKey,
|
|
$admin,
|
|
$staleInput
|
|
),
|
|
409,
|
|
'A stale source revision must fail after the locked recheck.'
|
|
);
|
|
$assert(CloudJob::count() === 0,
|
|
'Stale revision rejection created a durable operation.');
|
|
|
|
$input = [
|
|
'target_pool_key' => $targetKey,
|
|
'expected_source_revision' => 7,
|
|
'expected_target_revision' => 9,
|
|
'confirm_replacement' => true,
|
|
'request_key' => 'replacement-valid-0001',
|
|
];
|
|
$started = CloudPublicIpPoolReplacementBatchService::start(
|
|
1,
|
|
$sourceKey,
|
|
$admin,
|
|
$input
|
|
);
|
|
$assert($started['unchanged'] === false
|
|
&& $started['batch']['state'] === 'running'
|
|
&& $started['batch']['source_initial_state'] === 'active'
|
|
&& $started['batch']['counts']['total'] === 2,
|
|
'Valid replacement did not create one running durable batch.');
|
|
$parent = CloudJob::find((int) $started['job']->id);
|
|
$children = CloudJob::where('parent_job_id', (int) $parent->id)
|
|
->orderBy('instance_id')->get();
|
|
$assert($children->count() === 2
|
|
&& $children->every(static fn ($job): bool =>
|
|
(string) $job->kind === 'network_change'
|
|
&& (string) $job->execution_mode === 'agent')
|
|
&& Capsule::table('cloud_public_ips')
|
|
->whereIn('id', [71, 72])->where('state', 'changing')
|
|
->count() === 2
|
|
&& Capsule::table('cloud_public_ip_pools')->where('id', 31)
|
|
->value('state') === 'draining',
|
|
'Batch did not atomically fence every target IP and drain the source.');
|
|
$runningChild = $children->first();
|
|
$runningChild->phase = 'dispatching';
|
|
$runningChild->status = 'running';
|
|
$runningChild->save();
|
|
$runningDetail = CloudPublicIpPoolReplacementBatchService::detail(
|
|
1,
|
|
$sourceKey,
|
|
$started['batch']['batch_key'],
|
|
$readAdmin
|
|
);
|
|
$assert($runningDetail['counts']['running'] === 1
|
|
&& $runningDetail['counts']['queued'] === 1
|
|
&& $runningDetail['items'][0]['state'] === 'running',
|
|
'Batch DTO did not project an executing child as running.');
|
|
|
|
$idempotent = CloudPublicIpPoolReplacementBatchService::start(
|
|
1,
|
|
$sourceKey,
|
|
$admin,
|
|
$input
|
|
);
|
|
$assert($idempotent['unchanged'] === true
|
|
&& (int) $idempotent['job']->id === (int) $parent->id
|
|
&& CloudJob::where('parent_job_id', (int) $parent->id)->count() === 2,
|
|
'Idempotent retry duplicated child mutations.');
|
|
|
|
$activeOverview = CloudPublicIpPoolReplacementBatchService::overview(
|
|
1,
|
|
$sourceKey,
|
|
$readAdmin
|
|
);
|
|
$activeTargets = array_column(
|
|
$activeOverview['target_options'],
|
|
null,
|
|
'pool_key'
|
|
);
|
|
$assert($activeOverview['source_pool']['replacement_locked'] === true
|
|
&& $activeOverview['can_start'] === false
|
|
&& $activeOverview['active_batch']['batch_key']
|
|
=== $started['batch']['batch_key']
|
|
&& $activeTargets[$targetKey]['replacement_locked'] === true
|
|
&& $activeTargets[$targetKey]['eligible'] === false,
|
|
'Overview did not publish source/target participation locks.');
|
|
|
|
$overlapInput = [
|
|
'target_pool_key' => $targetKey,
|
|
'expected_source_revision' => 4,
|
|
'expected_target_revision' => 9,
|
|
'confirm_replacement' => true,
|
|
'request_key' => 'replacement-overlap-001',
|
|
];
|
|
$expectCloudError(
|
|
static fn () => CloudPublicIpPoolReplacementBatchService::start(
|
|
1,
|
|
$otherSourceKey,
|
|
$admin,
|
|
$overlapInput
|
|
),
|
|
409,
|
|
'An active target pool must not join a second concurrent batch.'
|
|
);
|
|
$expectCloudError(
|
|
static fn () => CloudPublicIpPoolReplacementBatchService::detail(
|
|
1,
|
|
$otherSourceKey,
|
|
$started['batch']['batch_key'],
|
|
$readAdmin
|
|
),
|
|
404,
|
|
'Batch detail must bind the batch to its exact source pool.'
|
|
);
|
|
|
|
$settleChild = static function (
|
|
CloudJob $child,
|
|
string $outcome,
|
|
bool $publish = true
|
|
): void {
|
|
$item = Capsule::table(
|
|
'cloud_public_ip_pool_replacement_batch_items'
|
|
)->where('child_job_id', (int) $child->id)->first();
|
|
$instance = CloudInstance::find((int) $child->instance_id);
|
|
CloudPlacementService::finishNetworkChange(
|
|
$instance,
|
|
(string) $child->operation_id,
|
|
(int) $item->source_public_ip_id,
|
|
(int) $item->target_public_ip_id,
|
|
(int) $item->source_network_attachment_id,
|
|
(int) $item->target_network_attachment_id,
|
|
(int) Capsule::table('cloud_public_ips')
|
|
->where('id', (int) $item->source_public_ip_id)
|
|
->value('public_ip_pool_id'),
|
|
32,
|
|
$outcome
|
|
);
|
|
if ($outcome === 'succeeded') {
|
|
$target = Capsule::table('cloud_public_ips')
|
|
->where('id', (int) $item->target_public_ip_id)->first();
|
|
$spec = json_decode((string) $instance->spec_json, true);
|
|
$spec['public_ip_cidr'] = (string) $target->address . '/29';
|
|
$instance->spec_json = json_encode(
|
|
$spec,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
);
|
|
$instance->save();
|
|
}
|
|
if ($publish) {
|
|
CloudPublicIpPoolReplacementBatchService::finishChildLocked(
|
|
$child,
|
|
$outcome
|
|
);
|
|
}
|
|
};
|
|
|
|
$first = $children->first();
|
|
$settleChild($first, 'uncertain');
|
|
$blocked = CloudPublicIpPoolReplacementBatch::where(
|
|
'batch_key', $started['batch']['batch_key']
|
|
)->first();
|
|
$assert((string) $blocked->state === 'blocked'
|
|
&& (int) $blocked->active_claim === 1
|
|
&& (string) $parent->fresh()->phase !== 'terminal',
|
|
'Unknown child effect did not retain the active claim and parent fence.');
|
|
|
|
$settleChild($first->fresh(), 'succeeded', false);
|
|
$resolvedFirst = $first->fresh();
|
|
$resolvedFirst->phase = 'terminal';
|
|
$resolvedFirst->outcome = 'failed';
|
|
$resolvedFirst->status = 'failed';
|
|
$resolvedFirst->effect_may_have_applied = 1;
|
|
$resolvedFirst->safe_to_resubmit = 0;
|
|
$resolvedFirst->effect_resolution = 'applied';
|
|
$resolvedFirst->save();
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
$resolvedFirst,
|
|
99
|
|
);
|
|
$settleChild($children->last()->fresh(), 'succeeded');
|
|
$completed = $blocked->fresh();
|
|
$parent = $parent->fresh();
|
|
$assert((string) $completed->state === 'succeeded'
|
|
&& $completed->active_claim === null
|
|
&& (string) $parent->phase === 'terminal'
|
|
&& (string) $parent->outcome === 'succeeded'
|
|
&& Capsule::table('cloud_public_ip_pools')->where('id', 31)
|
|
->value('state') === 'disabled'
|
|
&& Capsule::table('cloud_public_ips')->where('public_ip_pool_id', 31)
|
|
->where('state', '<>', 'available')->count() === 0,
|
|
'Evidence-backed recovery did not converge the batch and disable the empty source.');
|
|
|
|
$completedOverview = CloudPublicIpPoolReplacementBatchService::overview(
|
|
1,
|
|
$sourceKey,
|
|
$readAdmin
|
|
);
|
|
$assert($completedOverview['replacement_history_count'] === 1
|
|
&& $completedOverview['has_replacement_history'] === true
|
|
&& $completedOverview['source_pool']['replacement_history_count'] === 1,
|
|
'Read-only overview did not expose terminal replacement history.');
|
|
|
|
Capsule::table('cloud_public_ip_pools')->where('id', 33)->update([
|
|
'state' => 'draining',
|
|
'desired_revision' => 5,
|
|
'observed_revision' => 5,
|
|
]);
|
|
|
|
$failureInput = [
|
|
'target_pool_key' => $targetKey,
|
|
'expected_source_revision' => 5,
|
|
'expected_target_revision' => 9,
|
|
'confirm_replacement' => true,
|
|
'request_key' => 'replacement-failure-001',
|
|
];
|
|
$failure = CloudPublicIpPoolReplacementBatchService::start(
|
|
1,
|
|
$otherSourceKey,
|
|
$admin,
|
|
$failureInput
|
|
);
|
|
$failureParent = CloudJob::find((int) $failure['job']->id);
|
|
$failureChild = CloudJob::where(
|
|
'parent_job_id', (int) $failureParent->id
|
|
)->first();
|
|
$settleChild($failureChild, 'failed');
|
|
$resolvedFailureChild = $failureChild->fresh();
|
|
$resolvedFailureChild->phase = 'terminal';
|
|
$resolvedFailureChild->outcome = 'failed';
|
|
$resolvedFailureChild->status = 'failed';
|
|
$resolvedFailureChild->effect_may_have_applied = 0;
|
|
$resolvedFailureChild->safe_to_resubmit = 1;
|
|
$resolvedFailureChild->effect_resolution = 'not_applied';
|
|
$resolvedFailureChild->error_code = 'agent_network_not_applied';
|
|
$resolvedFailureChild->save();
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
$resolvedFailureChild,
|
|
99
|
|
);
|
|
$failedBatch = CloudPublicIpPoolReplacementBatch::where(
|
|
'batch_key', $failure['batch']['batch_key']
|
|
)->first();
|
|
$failureParent = $failureParent->fresh();
|
|
$assert((string) $failedBatch->state === 'failed'
|
|
&& (string) $failedBatch->source_initial_state === 'draining'
|
|
&& $failedBatch->active_claim === null
|
|
&& (string) $failureParent->outcome === 'failed'
|
|
&& (int) $failureParent->effect_may_have_applied === 0
|
|
&& (int) $failureParent->safe_to_resubmit === 1
|
|
&& Capsule::table('cloud_public_ip_pools')->where('id', 33)
|
|
->value('state') === 'draining'
|
|
&& Capsule::table('cloud_public_ips')
|
|
->where('id', (int) Capsule::table(
|
|
'cloud_public_ip_pool_replacement_batch_items'
|
|
)->where('batch_id', (int) $failedBatch->id)
|
|
->value('target_public_ip_id'))
|
|
->value('state') === 'available'
|
|
&& Capsule::table(
|
|
'cloud_public_ip_pool_replacement_batch_items'
|
|
)->where('batch_id', (int) $failedBatch->id)
|
|
->value('last_error_code') === 'agent_network_not_applied',
|
|
'All-not-applied failure did not restore the exact initial source state.');
|
|
|
|
echo "Cloud public IP pool replacement batch service tests passed\n";
|
|
}
|