You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudPublicIpPoolReplacemen...

758 lines
30 KiB
PHP

<?php
namespace App\Services\Cloud {
final class CloudSecrets
{
public static function encrypt(array $value, string $purpose): string
{
return json_encode(
$value,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
}
public static function decrypt(string $value, string $purpose): array
{
$decoded = json_decode($value, true, 64, JSON_THROW_ON_ERROR);
return is_array($decoded) ? $decoded : [];
}
}
final class CloudInstanceMigrationService
{
public static function assertInstanceMutationAvailable(
\App\Models\CloudInstance $instance,
?string $operationKind = null
): void {
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstance;
use App\Models\CloudJob;
use App\Models\CloudPublicIpPoolReplacementBatch;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudPlacementService;
use App\Services\Cloud\CloudPublicIpPoolReplacementBatchService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expectCloudError = static function (
callable $callback,
int $status,
string $message
) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === $status, $message
. ' (unexpected HTTP status)');
return;
}
throw new RuntimeException($message . ' (no CloudError)');
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite',
'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->increments('id');
$table->string('name');
$table->boolean('enabled')->default(true);
$table->integer('created_at')->default(0);
$table->integer('updated_at')->default(0);
});
$schema->create('cloud_host_agents', static function (
Blueprint $table
): void {
$table->integer('host_id')->primary();
$table->text('sealed_api_secret')->default('sealed');
$table->boolean('enabled');
$table->integer('revision');
$table->integer('tested_revision');
$table->string('last_test_status');
$table->string('last_error_code')->default('');
$table->text('network_selectors_json');
$table->integer('last_tested_at')->default(0);
$table->integer('created_at')->default(0);
$table->integer('updated_at')->default(0);
});
$schema->create('cloud_host_network_attachments', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('line_id');
$table->string('network_selector');
$table->string('state');
$table->integer('priority')->default(0);
$table->string('verification_state');
$table->integer('verified_agent_revision');
$table->integer('last_verified_at');
$table->integer('created_at')->default(0);
$table->integer('updated_at')->default(0);
});
$schema->create('cloud_public_ip_pools', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('pool_key', 32)->unique();
$table->integer('host_id');
$table->integer('network_attachment_id');
$table->string('name');
$table->string('cidr');
$table->string('gateway');
$table->integer('guest_prefix_length');
$table->text('dns_servers_json');
$table->text('reserved_ips_json');
$table->string('state');
$table->integer('priority')->default(0);
$table->integer('desired_revision');
$table->integer('observed_revision');
$table->string('sync_state');
$table->string('last_error_code')->default('');
$table->integer('created_at')->default(0);
$table->integer('updated_at')->default(0);
});
$schema->create('cloud_resource_pools', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('line_id');
});
$schema->create('cloud_resource_reservations', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('operation_id', 32)->unique();
$table->integer('resource_pool_id');
$table->integer('host_id');
$table->integer('user_id');
$table->integer('instance_id')->nullable();
$table->integer('network_attachment_id');
$table->integer('public_ip_pool_id');
$table->integer('public_ip_id');
$table->string('state');
$table->integer('created_at')->default(0);
$table->integer('updated_at')->default(0);
});
$schema->create('cloud_instances', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('uuid');
$table->string('name');
$table->integer('user_id');
$table->integer('host_id');
$table->integer('resource_pool_id');
$table->integer('reservation_id');
$table->string('status');
$table->text('spec_json');
$table->integer('created_at')->default(0);
$table->integer('updated_at')->default(0);
});
$schema->create('cloud_public_ips', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('public_ip_pool_id');
$table->string('address');
$table->integer('prefix_length');
$table->string('state');
$table->integer('reservation_id')->nullable();
$table->integer('instance_id')->nullable();
$table->string('operation_id', 32)->nullable();
$table->integer('created_at')->default(0);
$table->integer('updated_at')->default(0);
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id', 32)->unique();
$table->string('request_idempotency_key', 128);
$table->string('request_fingerprint', 64);
$table->integer('parent_job_id')->nullable();
$table->integer('supersedes_job_id')->nullable();
$table->integer('vpc_repair_root_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('backup_id')->nullable();
$table->integer('backup_restore_id')->nullable();
$table->integer('backup_recovery_id')->nullable();
$table->integer('vpc_id')->nullable();
$table->integer('vpc_port_id')->nullable();
$table->integer('vpc_batch_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts');
$table->integer('dispatch_attempts');
$table->integer('dispatch_deadline_at');
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->text('effect_resolution_reason')->nullable();
$table->boolean('finalization_incident')->default(false);
$table->string('finalization_error_code')->nullable();
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at');
$table->string('error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['user_id', 'request_idempotency_key']);
});
$schema->create('cloud_public_ip_pool_replacement_batches',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('batch_key', 32)->unique();
$table->integer('host_id');
$table->integer('source_public_ip_pool_id');
$table->integer('target_public_ip_pool_id');
$table->string('source_initial_state');
$table->integer('parent_job_id')->unique();
$table->integer('requested_by_user_id');
$table->string('state');
$table->integer('active_claim')->nullable();
$table->string('last_error_code')->default('');
$table->integer('terminal_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_public_ip_pool_replacement_batch_items',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('batch_id');
$table->integer('instance_id');
$table->integer('source_reservation_id');
$table->integer('source_public_ip_id');
$table->integer('target_public_ip_id');
$table->integer('source_network_attachment_id');
$table->integer('target_network_attachment_id');
$table->integer('child_job_id')->unique();
$table->string('state');
$table->string('last_error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['batch_id', 'target_public_ip_id']);
});
$now = time();
Capsule::table('cloud_hosts')->insert([
'id' => 1,
'name' => 'HKG-A',
'enabled' => 1,
'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_host_agents')->insert([
'host_id' => 1,
'enabled' => 1,
'revision' => 7,
'tested_revision' => 7,
'last_test_status' => 'healthy',
'network_selectors_json' => json_encode([
'bridge:cross-line', 'bridge:other-source', 'bridge:source',
'bridge:target',
], JSON_THROW_ON_ERROR),
'last_tested_at' => $now,
'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_host_network_attachments')->insert([
['id' => 11, 'host_id' => 1, 'line_id' => 10,
'network_selector' => 'bridge:source', 'state' => 'active',
'priority' => 10, 'verification_state' => 'verified',
'verified_agent_revision' => 7, 'last_verified_at' => $now],
['id' => 12, 'host_id' => 1, 'line_id' => 10,
'network_selector' => 'bridge:target', 'state' => 'active',
'priority' => 20, 'verification_state' => 'verified',
'verified_agent_revision' => 7, 'last_verified_at' => $now],
['id' => 13, 'host_id' => 1, 'line_id' => 10,
'network_selector' => 'bridge:other-source', 'state' => 'active',
'priority' => 30, 'verification_state' => 'verified',
'verified_agent_revision' => 7, 'last_verified_at' => $now],
['id' => 14, 'host_id' => 1, 'line_id' => 20,
'network_selector' => 'bridge:cross-line', 'state' => 'active',
'priority' => 40, 'verification_state' => 'verified',
'verified_agent_revision' => 7, 'last_verified_at' => $now],
]);
Capsule::table('cloud_resource_pools')->insert([
['id' => 21, 'line_id' => 10],
]);
$pool = static function (
int $id,
int $attachmentId,
string $keyDigit,
string $name,
string $cidr,
string $gateway,
int $revision
): array {
return [
'id' => $id,
'pool_key' => str_repeat($keyDigit, 32),
'host_id' => 1,
'network_attachment_id' => $attachmentId,
'name' => $name,
'cidr' => $cidr,
'gateway' => $gateway,
'guest_prefix_length' => 29,
'dns_servers_json' => json_encode(['1.1.1.1']),
'reserved_ips_json' => '[]',
'state' => 'active',
'priority' => $id,
'desired_revision' => $revision,
'observed_revision' => $revision,
'sync_state' => 'applied',
'last_error_code' => '',
];
};
Capsule::table('cloud_public_ip_pools')->insert([
$pool(31, 11, 'a', 'Source A', '198.51.100.0/29',
'198.51.100.1', 7),
$pool(32, 12, 'b', 'Target B', '198.51.100.8/29',
'198.51.100.9', 9),
$pool(33, 13, 'c', 'Source D', '198.51.100.16/29',
'198.51.100.17', 4),
$pool(34, 14, 'd', 'Cross Line', '198.51.100.24/29',
'198.51.100.25', 3),
]);
$reservationOperation = static fn (int $id): string => str_pad(
dechex($id),
32,
'0',
STR_PAD_LEFT
);
Capsule::table('cloud_resource_reservations')->insert([
['id' => 41, 'operation_id' => $reservationOperation(41),
'resource_pool_id' => 21, 'host_id' => 1, 'user_id' => 501,
'instance_id' => 51, 'network_attachment_id' => 11,
'public_ip_pool_id' => 31, 'public_ip_id' => 61,
'state' => 'committed'],
['id' => 42, 'operation_id' => $reservationOperation(42),
'resource_pool_id' => 21, 'host_id' => 1, 'user_id' => 502,
'instance_id' => 52, 'network_attachment_id' => 11,
'public_ip_pool_id' => 31, 'public_ip_id' => 62,
'state' => 'committed'],
['id' => 43, 'operation_id' => $reservationOperation(43),
'resource_pool_id' => 21, 'host_id' => 1, 'user_id' => 503,
'instance_id' => 53, 'network_attachment_id' => 13,
'public_ip_pool_id' => 33, 'public_ip_id' => 63,
'state' => 'committed'],
]);
$instanceSpec = static function (string $publicIp): string {
return json_encode([
'managed' => true,
'os_source' => 'cloud_template',
'public_ip_cidr' => $publicIp . '/29',
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES);
};
Capsule::table('cloud_instances')->insert([
['id' => 51, 'uuid' => '11111111-1111-4111-8111-111111111111',
'name' => 'vm-a1', 'user_id' => 501, 'host_id' => 1,
'resource_pool_id' => 21, 'reservation_id' => 41,
'status' => 'active',
'spec_json' => $instanceSpec('198.51.100.2')],
['id' => 52, 'uuid' => '22222222-2222-4222-8222-222222222222',
'name' => 'vm-a2', 'user_id' => 502, 'host_id' => 1,
'resource_pool_id' => 21, 'reservation_id' => 42,
'status' => 'active',
'spec_json' => $instanceSpec('198.51.100.3')],
['id' => 53, 'uuid' => '33333333-3333-4333-8333-333333333333',
'name' => 'vm-d1', 'user_id' => 503, 'host_id' => 1,
'resource_pool_id' => 21, 'reservation_id' => 43,
'status' => 'active',
'spec_json' => $instanceSpec('198.51.100.18')],
]);
Capsule::table('cloud_public_ips')->insert([
['id' => 61, 'host_id' => 1, 'public_ip_pool_id' => 31,
'address' => '198.51.100.2', 'prefix_length' => 29,
'state' => 'assigned', 'reservation_id' => 41,
'instance_id' => 51,
'operation_id' => $reservationOperation(41)],
['id' => 62, 'host_id' => 1, 'public_ip_pool_id' => 31,
'address' => '198.51.100.3', 'prefix_length' => 29,
'state' => 'assigned', 'reservation_id' => 42,
'instance_id' => 52,
'operation_id' => $reservationOperation(42)],
['id' => 63, 'host_id' => 1, 'public_ip_pool_id' => 33,
'address' => '198.51.100.18', 'prefix_length' => 29,
'state' => 'assigned', 'reservation_id' => 43,
'instance_id' => 53,
'operation_id' => $reservationOperation(43)],
['id' => 71, 'host_id' => 1, 'public_ip_pool_id' => 32,
'address' => '198.51.100.10', 'prefix_length' => 29,
'state' => 'available', 'reservation_id' => null,
'instance_id' => null, 'operation_id' => null],
['id' => 72, 'host_id' => 1, 'public_ip_pool_id' => 32,
'address' => '198.51.100.11', 'prefix_length' => 29,
'state' => 'available', 'reservation_id' => null,
'instance_id' => null, 'operation_id' => null],
['id' => 73, 'host_id' => 1, 'public_ip_pool_id' => 32,
'address' => '198.51.100.12', 'prefix_length' => 29,
'state' => 'available', 'reservation_id' => null,
'instance_id' => null, 'operation_id' => null],
['id' => 74, 'host_id' => 1, 'public_ip_pool_id' => 32,
'address' => '198.51.100.13', 'prefix_length' => 29,
'state' => 'available', 'reservation_id' => null,
'instance_id' => null, 'operation_id' => null],
['id' => 75, 'host_id' => 1, 'public_ip_pool_id' => 34,
'address' => '198.51.100.26', 'prefix_length' => 29,
'state' => 'available', 'reservation_id' => null,
'instance_id' => null, 'operation_id' => null],
]);
$admin = [
'user_id' => 99,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
];
$readAdmin = [
'user_id' => 98,
'is_admin' => true,
'is_super_admin' => false,
'can_manage' => false,
];
$sourceKey = str_repeat('a', 32);
$targetKey = str_repeat('b', 32);
$otherSourceKey = str_repeat('c', 32);
$overview = CloudPublicIpPoolReplacementBatchService::overview(
1,
$sourceKey,
$readAdmin
);
$assert($overview['source_pool']['expected_revision'] === 7
&& $overview['source_pool']['assigned_count'] === 2
&& $overview['source_pool']['replacement_locked'] === false
&& $overview['can_start'] === true,
'Read-only administrator overview lost source revision or occupancy.');
$targetOptions = array_column(
$overview['target_options'],
null,
'pool_key'
);
$assert(isset($targetOptions[$targetKey])
&& $targetOptions[$targetKey]['expected_revision'] === 9
&& $targetOptions[$targetKey]['available_count'] === 4
&& $targetOptions[$targetKey]['eligible'] === true
&& !isset($targetOptions[str_repeat('d', 32)]),
'Overview did not enforce same-line verified target selection.');
$expectCloudError(
static fn () => CloudPublicIpPoolReplacementBatchService::overview(
1,
$sourceKey,
['user_id' => 501, 'is_admin' => false]
),
403,
'Tenant must not read administrator replacement batches.'
);
$staleInput = [
'target_pool_key' => $targetKey,
'expected_source_revision' => 6,
'expected_target_revision' => 9,
'confirm_replacement' => true,
'request_key' => 'replacement-stale-0001',
];
$expectCloudError(
static fn () => CloudPublicIpPoolReplacementBatchService::start(
1,
$sourceKey,
$admin,
$staleInput
),
409,
'A stale source revision must fail after the locked recheck.'
);
$assert(CloudJob::count() === 0,
'Stale revision rejection created a durable operation.');
$input = [
'target_pool_key' => $targetKey,
'expected_source_revision' => 7,
'expected_target_revision' => 9,
'confirm_replacement' => true,
'request_key' => 'replacement-valid-0001',
];
$started = CloudPublicIpPoolReplacementBatchService::start(
1,
$sourceKey,
$admin,
$input
);
$assert($started['unchanged'] === false
&& $started['batch']['state'] === 'running'
&& $started['batch']['source_initial_state'] === 'active'
&& $started['batch']['counts']['total'] === 2,
'Valid replacement did not create one running durable batch.');
$parent = CloudJob::find((int) $started['job']->id);
$children = CloudJob::where('parent_job_id', (int) $parent->id)
->orderBy('instance_id')->get();
$assert($children->count() === 2
&& $children->every(static fn ($job): bool =>
(string) $job->kind === 'network_change'
&& (string) $job->execution_mode === 'agent')
&& Capsule::table('cloud_public_ips')
->whereIn('id', [71, 72])->where('state', 'changing')
->count() === 2
&& Capsule::table('cloud_public_ip_pools')->where('id', 31)
->value('state') === 'draining',
'Batch did not atomically fence every target IP and drain the source.');
$runningChild = $children->first();
$runningChild->phase = 'dispatching';
$runningChild->status = 'running';
$runningChild->save();
$runningDetail = CloudPublicIpPoolReplacementBatchService::detail(
1,
$sourceKey,
$started['batch']['batch_key'],
$readAdmin
);
$assert($runningDetail['counts']['running'] === 1
&& $runningDetail['counts']['queued'] === 1
&& $runningDetail['items'][0]['state'] === 'running',
'Batch DTO did not project an executing child as running.');
$idempotent = CloudPublicIpPoolReplacementBatchService::start(
1,
$sourceKey,
$admin,
$input
);
$assert($idempotent['unchanged'] === true
&& (int) $idempotent['job']->id === (int) $parent->id
&& CloudJob::where('parent_job_id', (int) $parent->id)->count() === 2,
'Idempotent retry duplicated child mutations.');
$activeOverview = CloudPublicIpPoolReplacementBatchService::overview(
1,
$sourceKey,
$readAdmin
);
$activeTargets = array_column(
$activeOverview['target_options'],
null,
'pool_key'
);
$assert($activeOverview['source_pool']['replacement_locked'] === true
&& $activeOverview['can_start'] === false
&& $activeOverview['active_batch']['batch_key']
=== $started['batch']['batch_key']
&& $activeTargets[$targetKey]['replacement_locked'] === true
&& $activeTargets[$targetKey]['eligible'] === false,
'Overview did not publish source/target participation locks.');
$overlapInput = [
'target_pool_key' => $targetKey,
'expected_source_revision' => 4,
'expected_target_revision' => 9,
'confirm_replacement' => true,
'request_key' => 'replacement-overlap-001',
];
$expectCloudError(
static fn () => CloudPublicIpPoolReplacementBatchService::start(
1,
$otherSourceKey,
$admin,
$overlapInput
),
409,
'An active target pool must not join a second concurrent batch.'
);
$expectCloudError(
static fn () => CloudPublicIpPoolReplacementBatchService::detail(
1,
$otherSourceKey,
$started['batch']['batch_key'],
$readAdmin
),
404,
'Batch detail must bind the batch to its exact source pool.'
);
$settleChild = static function (
CloudJob $child,
string $outcome,
bool $publish = true
): void {
$item = Capsule::table(
'cloud_public_ip_pool_replacement_batch_items'
)->where('child_job_id', (int) $child->id)->first();
$instance = CloudInstance::find((int) $child->instance_id);
CloudPlacementService::finishNetworkChange(
$instance,
(string) $child->operation_id,
(int) $item->source_public_ip_id,
(int) $item->target_public_ip_id,
(int) $item->source_network_attachment_id,
(int) $item->target_network_attachment_id,
(int) Capsule::table('cloud_public_ips')
->where('id', (int) $item->source_public_ip_id)
->value('public_ip_pool_id'),
32,
$outcome
);
if ($outcome === 'succeeded') {
$target = Capsule::table('cloud_public_ips')
->where('id', (int) $item->target_public_ip_id)->first();
$spec = json_decode((string) $instance->spec_json, true);
$spec['public_ip_cidr'] = (string) $target->address . '/29';
$instance->spec_json = json_encode(
$spec,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
$instance->save();
}
if ($publish) {
CloudPublicIpPoolReplacementBatchService::finishChildLocked(
$child,
$outcome
);
}
};
$first = $children->first();
$settleChild($first, 'uncertain');
$blocked = CloudPublicIpPoolReplacementBatch::where(
'batch_key', $started['batch']['batch_key']
)->first();
$assert((string) $blocked->state === 'blocked'
&& (int) $blocked->active_claim === 1
&& (string) $parent->fresh()->phase !== 'terminal',
'Unknown child effect did not retain the active claim and parent fence.');
$settleChild($first->fresh(), 'succeeded', false);
$resolvedFirst = $first->fresh();
$resolvedFirst->phase = 'terminal';
$resolvedFirst->outcome = 'failed';
$resolvedFirst->status = 'failed';
$resolvedFirst->effect_may_have_applied = 1;
$resolvedFirst->safe_to_resubmit = 0;
$resolvedFirst->effect_resolution = 'applied';
$resolvedFirst->save();
CloudJobService::reconcileAggregateParentEffectLocked(
$resolvedFirst,
99
);
$settleChild($children->last()->fresh(), 'succeeded');
$completed = $blocked->fresh();
$parent = $parent->fresh();
$assert((string) $completed->state === 'succeeded'
&& $completed->active_claim === null
&& (string) $parent->phase === 'terminal'
&& (string) $parent->outcome === 'succeeded'
&& Capsule::table('cloud_public_ip_pools')->where('id', 31)
->value('state') === 'disabled'
&& Capsule::table('cloud_public_ips')->where('public_ip_pool_id', 31)
->where('state', '<>', 'available')->count() === 0,
'Evidence-backed recovery did not converge the batch and disable the empty source.');
$completedOverview = CloudPublicIpPoolReplacementBatchService::overview(
1,
$sourceKey,
$readAdmin
);
$assert($completedOverview['replacement_history_count'] === 1
&& $completedOverview['has_replacement_history'] === true
&& $completedOverview['source_pool']['replacement_history_count'] === 1,
'Read-only overview did not expose terminal replacement history.');
Capsule::table('cloud_public_ip_pools')->where('id', 33)->update([
'state' => 'draining',
'desired_revision' => 5,
'observed_revision' => 5,
]);
$failureInput = [
'target_pool_key' => $targetKey,
'expected_source_revision' => 5,
'expected_target_revision' => 9,
'confirm_replacement' => true,
'request_key' => 'replacement-failure-001',
];
$failure = CloudPublicIpPoolReplacementBatchService::start(
1,
$otherSourceKey,
$admin,
$failureInput
);
$failureParent = CloudJob::find((int) $failure['job']->id);
$failureChild = CloudJob::where(
'parent_job_id', (int) $failureParent->id
)->first();
$settleChild($failureChild, 'failed');
$resolvedFailureChild = $failureChild->fresh();
$resolvedFailureChild->phase = 'terminal';
$resolvedFailureChild->outcome = 'failed';
$resolvedFailureChild->status = 'failed';
$resolvedFailureChild->effect_may_have_applied = 0;
$resolvedFailureChild->safe_to_resubmit = 1;
$resolvedFailureChild->effect_resolution = 'not_applied';
$resolvedFailureChild->error_code = 'agent_network_not_applied';
$resolvedFailureChild->save();
CloudJobService::reconcileAggregateParentEffectLocked(
$resolvedFailureChild,
99
);
$failedBatch = CloudPublicIpPoolReplacementBatch::where(
'batch_key', $failure['batch']['batch_key']
)->first();
$failureParent = $failureParent->fresh();
$assert((string) $failedBatch->state === 'failed'
&& (string) $failedBatch->source_initial_state === 'draining'
&& $failedBatch->active_claim === null
&& (string) $failureParent->outcome === 'failed'
&& (int) $failureParent->effect_may_have_applied === 0
&& (int) $failureParent->safe_to_resubmit === 1
&& Capsule::table('cloud_public_ip_pools')->where('id', 33)
->value('state') === 'draining'
&& Capsule::table('cloud_public_ips')
->where('id', (int) Capsule::table(
'cloud_public_ip_pool_replacement_batch_items'
)->where('batch_id', (int) $failedBatch->id)
->value('target_public_ip_id'))
->value('state') === 'available'
&& Capsule::table(
'cloud_public_ip_pool_replacement_batch_items'
)->where('batch_id', (int) $failedBatch->id)
->value('last_error_code') === 'agent_network_not_applied',
'All-not-applied failure did not restore the exact initial source state.');
echo "Cloud public IP pool replacement batch service tests passed\n";
}