You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudAggregateRecoveryConve...

4182 lines
172 KiB
PHP

<?php
declare(strict_types=1);
namespace App\Services\Cloud {
/** In-memory envelope for this focused completion/convergence contract. */
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'aggregate-convergence-test:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
$key = substr($value, strlen('aggregate-convergence-test:'));
if (!array_key_exists($key, self::$values)) {
throw new \RuntimeException('Missing aggregate test payload.');
}
return self::$values[$key];
}
}
final class CloudAgentSettingsService
{
public static int $requests = 0;
public static function enabledHostIds(): array
{
return [7, 8];
}
public static function testedNetworkSelectors(
\App\Models\CloudHostAgent $setting
): array {
return $setting->testedNetworkSelectors();
}
public static function clientForConfiguration(
int $hostId,
float $timeoutSeconds = 120.0
): never {
self::$requests++;
throw new \RuntimeException(
'Aggregate recovery must not contact an Agent.'
);
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudOperationCompletionService;
use App\Services\Cloud\CloudOperationRecoveryIdentity;
use App\Services\Cloud\CloudRecoveryAggregateStateConflict;
use App\Services\Cloud\CloudRecoveryInstanceStateConflict;
use App\Services\Cloud\CloudRecoveryOperationIdentityConflict;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudTemplateCatalogService;
use App\Services\Cloud\CloudUserIsoService;
use App\Services\Cloud\CloudWorkerService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$automaticSource = file_get_contents(
dirname(__DIR__)
. '/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php'
);
$lockMethodStart = is_string($automaticSource)
? strpos($automaticSource, 'private static function resolveFenceAndCommit(')
: false;
$lockMethodEnd = $lockMethodStart === false ? false : strpos(
$automaticSource,
'private static function assertFreshInspection(',
$lockMethodStart
);
$lockMethod = $lockMethodStart === false || $lockMethodEnd === false
? '' : substr(
$automaticSource,
$lockMethodStart,
$lockMethodEnd - $lockMethodStart
);
$parentLock = strpos(
$lockMethod,
'$aggregateParent = CloudJob::where('
);
$catalogLock = strpos(
$lockMethod,
'CloudTemplateCatalogService::lockAggregateCatalogLocked('
);
$leafLock = strpos(
$lockMethod,
'$aggregateChildren = CloudJob::where('
);
$assert(str_contains($lockMethod, "->value('parent_job_id')")
&& $parentLock !== false
&& $catalogLock !== false
&& $leafLock !== false
&& $parentLock < $catalogLock
&& $catalogLock < $leafLock
&& str_contains(
$lockMethod,
"(int) (\$job->parent_job_id ?? 0)\n"
. ' !== $aggregateParentId'
),
'Fence commit does not preserve aggregate parent-before-child lock order.');
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('user', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('enable')->default(1);
$table->integer('is_admin')->default(0);
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key')->unique();
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('public_network_transition_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at');
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->string('effect_resolution_reason')->default('');
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at')->default(0);
$table->string('error_code')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_public_network_transitions',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('state');
$table->string('effect_resolution');
$table->string('last_error_code')->default('');
});
$schema->create('cloud_agent_completion_events',
static function (Blueprint $table): void {
$table->string('event_id')->primary();
$table->integer('host_id');
$table->string('operation_id');
$table->string('payload_sha256');
$table->string('outcome');
$table->boolean('effect_may_have_applied');
$table->boolean('safe_to_resubmit');
$table->string('error_code')->default('');
$table->text('result_json')->nullable();
$table->string('disposition')->default('received');
$table->integer('received_at');
$table->integer('applied_at')->nullable();
$table->unique(['host_id', 'operation_id']);
});
$schema->create('cloud_orders', static function (Blueprint $table): void {
$table->increments('id');
$table->string('status');
$table->text('spec_json');
});
$schema->create('cloud_templates',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('catalog_key')->default('test-catalog');
$table->string('version_label')->default('test-version');
$table->boolean('is_default')->default(true);
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->integer('image_bytes');
$table->string('sha256');
$table->text('startup_script');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('active_job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_import_intents',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('user_id');
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->string('source_kind');
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->string('source_filename')->default('');
$table->integer('source_size_bytes')->default(0);
$table->integer('source_last_modified')->default(0);
$table->string('source_sha256')->default('');
$table->text('sealed_source');
$table->string('template_id')->unique();
$table->string('catalog_key');
$table->string('version_label');
$table->string('generation_token')->unique();
$table->integer('storage_revision');
$table->text('sealed_storage_runtime');
$table->string('object_key')->unique();
$table->string('multipart_upload_id')->default('');
$table->integer('multipart_part_size_bytes')->default(0);
$table->integer('multipart_part_count')->default(0);
$table->string('object_version_id')->default('');
$table->string('object_etag')->default('');
$table->string('state')->default('prepared');
$table->integer('active_job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('expires_at');
$table->integer('completed_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['user_id', 'request_idempotency_key']);
});
$schema->create('cloud_template_import_parts',
static function (Blueprint $table): void {
$table->string('intent_id');
$table->integer('part_number');
$table->integer('size_bytes');
$table->string('etag');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['intent_id', 'part_number']);
});
$schema->create('cloud_template_artifacts',
static function (Blueprint $table): void {
$table->string('template_id')->primary();
$table->integer('storage_revision')->default(1);
$table->text('sealed_storage_runtime');
$table->string('object_key');
$table->string('object_version_id')->default('');
$table->string('object_etag')->default('"test-etag"');
$table->integer('size_bytes');
$table->string('sha256');
$table->string('state')->default('ready');
$table->string('error_code')->default('');
$table->string('delete_operation_id')->default('');
$table->integer('delete_generation')->default(0);
$table->integer('delete_cycle_attempts')->default(0);
$table->integer('delete_presence_confirmations')->default(0);
$table->integer('delete_last_checked_at')->default(0);
$table->integer('delete_next_check_at')->default(0);
$table->integer('delete_attention_at')->nullable();
$table->integer('verified_at')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
});
Capsule::connection('default')->statement(<<<'SQL'
CREATE TRIGGER cloud_template_artifact_fixture
AFTER INSERT ON cloud_templates
BEGIN
INSERT INTO cloud_template_artifacts (
template_id, storage_revision, sealed_storage_runtime, object_key,
object_version_id, object_etag, size_bytes, sha256, state, error_code,
delete_operation_id, delete_generation, delete_cycle_attempts,
delete_presence_confirmations, delete_last_checked_at,
delete_next_check_at, delete_attention_at, verified_at, created_at,
updated_at
) VALUES (
NEW.id, 1, 'test-only', 'test/' || NEW.id, '', '"test-etag"',
NEW.image_bytes, NEW.sha256, 'ready', '', '', 0, 0, 0, 0, 0, NULL,
NEW.created_at, NEW.created_at, NEW.updated_at
);
END
SQL);
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('image_bytes');
$table->string('sha256');
$table->string('operation_id');
$table->integer('job_id')->nullable();
$table->string('error_code');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
$schema->create('configurations', static function (Blueprint $table): void {
$table->increments('id');
$table->string('name')->unique();
$table->text('value');
});
$schema->create('shop', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('status');
});
$schema->create('cloud_regions', static function (Blueprint $table): void {
$table->increments('id');
$table->string('status');
});
$schema->create('cloud_network_lines', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('region_id');
$table->integer('is_default');
$table->string('status');
});
$schema->create('cloud_resource_pools', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('line_id');
$table->string('status');
});
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('enabled');
$table->string('default_network');
});
$schema->create('cloud_host_agents', static function (
Blueprint $table
): void {
$table->integer('host_id')->primary();
$table->integer('enabled');
$table->integer('revision');
$table->integer('tested_revision');
$table->string('last_test_status');
$table->text('network_selectors_json')->default('[]');
});
$schema->create('cloud_host_storage_configurations', static function (
Blueprint $table
): void {
$table->integer('host_id')->primary();
$table->string('state');
});
$schema->create('cloud_host_resource_pool_memberships', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('resource_pool_id');
$table->string('scheduler_state');
$table->integer('revision')->default(1);
});
$schema->create('cloud_host_capacity_policies', static function (
Blueprint $table
): void {
$table->integer('host_id')->primary();
$table->integer('configured_cpu_cores');
$table->integer('configured_memory_mib');
$table->integer('configured_disk_gib');
});
$schema->create('cloud_host_network_attachments', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('line_id');
$table->string('network_selector');
$table->string('state');
$table->string('verification_state');
$table->integer('verified_agent_revision');
$table->integer('last_verified_at');
});
$schema->create('cloud_public_ip_pools', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('pool_key');
$table->integer('host_id');
$table->integer('network_attachment_id');
$table->string('cidr');
$table->string('gateway');
$table->integer('guest_prefix_length');
$table->text('dns_servers_json');
$table->text('reserved_ips_json');
$table->string('state');
$table->string('sync_state');
$table->integer('desired_revision');
$table->integer('observed_revision');
});
$schema->create('cloud_public_ips', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('public_ip_pool_id');
$table->string('address');
$table->integer('prefix_length');
$table->string('state');
$table->integer('reservation_id')->nullable();
$table->integer('instance_id')->nullable();
$table->string('operation_id')->nullable();
});
$schema->create('cloud_offers', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('plan_id');
$table->integer('resource_pool_id');
$table->integer('status');
$table->string('currency');
});
$schema->create('cloud_user_isos',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('user_id');
$table->string('name');
$table->text('sealed_source_url');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('state')->default('importing');
$table->string('import_operation_id');
$table->integer('active_job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_user_iso_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('user_iso_id');
$table->integer('user_id');
$table->string('state')->default('pending');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id');
$table->integer('job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'user_iso_id']);
});
Capsule::table('user')->insert([
'id' => 1, 'enable' => 1, 'is_admin' => 0,
]);
Capsule::table('configurations')->insert([
'name' => 'cloud_network_lines_enabled', 'value' => 'false',
]);
Capsule::table('shop')->insert(['id' => 1, 'status' => 1]);
Capsule::table('cloud_regions')->insert([
'id' => 1, 'status' => 'active',
]);
Capsule::table('cloud_network_lines')->insert([
'id' => 1, 'region_id' => 1, 'is_default' => 1,
'status' => 'active',
]);
Capsule::table('cloud_resource_pools')->insert([
'id' => 1, 'line_id' => 1, 'status' => 'active',
]);
Capsule::table('cloud_offers')->insert([
'id' => 1, 'plan_id' => 1, 'resource_pool_id' => 1,
'status' => 1, 'currency' => 'USD',
]);
foreach ([7, 8] as $hostId) {
Capsule::table('cloud_hosts')->insert([
'id' => $hostId, 'enabled' => 1,
'default_network' => 'bridge:br0',
]);
Capsule::table('cloud_host_agents')->insert([
'host_id' => $hostId, 'enabled' => 1, 'revision' => 1,
'tested_revision' => 1, 'last_test_status' => 'healthy',
'network_selectors_json' => json_encode(
['bridge:public-' . $hostId],
JSON_THROW_ON_ERROR
),
]);
Capsule::table('cloud_host_storage_configurations')->insert([
'host_id' => $hostId, 'state' => 'applied',
]);
Capsule::table('cloud_host_resource_pool_memberships')->insert([
'host_id' => $hostId, 'resource_pool_id' => 1,
'scheduler_state' => 'active',
]);
Capsule::table('cloud_host_capacity_policies')->insert([
'host_id' => $hostId, 'configured_cpu_cores' => 16,
'configured_memory_mib' => 32768,
'configured_disk_gib' => 512,
]);
$attachmentId = 100 + $hostId;
Capsule::table('cloud_host_network_attachments')->insert([
'id' => $attachmentId,
'host_id' => $hostId,
'line_id' => 1,
'network_selector' => 'bridge:public-' . $hostId,
'state' => 'active',
'verification_state' => 'verified',
'verified_agent_revision' => 1,
'last_verified_at' => time(),
]);
$publicPoolId = 200 + $hostId;
Capsule::table('cloud_public_ip_pools')->insert([
'id' => $publicPoolId,
'pool_key' => str_repeat((string) $hostId, 32),
'host_id' => $hostId,
'network_attachment_id' => $attachmentId,
'cidr' => '203.0.11' . $hostId . '.0/29',
'gateway' => '203.0.11' . $hostId . '.1',
'guest_prefix_length' => 29,
'dns_servers_json' => '[]',
'reserved_ips_json' => '[]',
'state' => 'active',
'sync_state' => 'applied',
'desired_revision' => 1,
'observed_revision' => 1,
]);
Capsule::table('cloud_public_ips')->insert([
'host_id' => $hostId,
'public_ip_pool_id' => $publicPoolId,
'address' => '203.0.11' . $hostId . '.2',
'prefix_length' => 29,
'state' => 'available',
'reservation_id' => null,
'instance_id' => null,
'operation_id' => null,
]);
}
$now = time();
$nextId = 1;
$insertParent = static function (
string $kind = 'template_import',
int $userId = 1
) use (&$nextId, $now): int {
$id = $nextId++;
$operationId = md5('aggregate-parent:' . $id);
Capsule::table('cloud_jobs')->insert([
'id' => $id,
'operation_id' => $operationId,
'request_idempotency_key' => $operationId,
'request_fingerprint' => hash('sha256', $operationId),
'parent_job_id' => null,
'order_id' => null,
'invoice_id' => null,
'instance_id' => null,
'snapshot_id' => null,
'user_id' => $userId,
'host_id' => null,
'kind' => $kind,
'execution_mode' => 'aggregate',
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now - 600,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'receipt_deadline_at' => $now - 1,
'terminal_at' => $now - 1,
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown',
'effect_resolved_at' => null,
'effect_resolved_by' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null,
'read_at' => $now - 1,
'dispatched_at' => null,
'lease_token' => str_repeat('a', 32),
'lease_until' => $now + 300,
'available_at' => $now + 300,
'error_code' => 'aggregate_deadline_exceeded',
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_child_convergence',
'result_json' => null,
'created_at' => $now - 900,
'updated_at' => $now - 1,
]);
return $id;
};
$insertResolvedChild = static function (
int $parentId,
string $resolution,
string $kind = 'action',
int $userId = 1,
string $executionMode = 'agent'
) use (&$nextId, $now): int {
$id = $nextId++;
$operationId = md5('resolved-child:' . $id);
Capsule::table('cloud_jobs')->insert([
'id' => $id,
'operation_id' => $operationId,
'request_idempotency_key' => $operationId,
'request_fingerprint' => hash('sha256', $operationId),
'parent_job_id' => $parentId,
'order_id' => null,
'invoice_id' => null,
'instance_id' => null,
'snapshot_id' => null,
'user_id' => $userId,
'host_id' => 7,
'kind' => $kind,
'execution_mode' => $executionMode,
'phase' => 'terminal',
'outcome' => $resolution === 'applied' ? 'succeeded' : 'failed',
'status' => $resolution === 'applied' ? 'succeeded' : 'failed',
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
'attempts' => 1,
'dispatch_attempts' => 1,
'dispatch_deadline_at' => $now - 600,
'dispatch_started_at' => $now - 500,
'agent_accepted_at' => $now - 490,
'receipt_deadline_at' => $now - 1,
'terminal_at' => $now - 1,
'effect_may_have_applied' => 0,
'safe_to_resubmit' => $resolution === 'not_applied' ? 1 : 0,
'effect_resolution' => $resolution,
'effect_resolved_at' => $now - 1,
'effect_resolved_by' => null,
'effect_resolution_reason' => 'normal_completion',
'completion_event_id' => null,
'read_at' => null,
'dispatched_at' => $now - 490,
'lease_token' => null,
'lease_until' => 0,
'available_at' => 0,
'error_code' => $resolution === 'not_applied'
? 'operation_rejected' : null,
'finalization_incident' => 0,
'finalization_error_code' => null,
'result_json' => null,
'created_at' => $now - 600,
'updated_at' => $now - 1,
]);
return $id;
};
$insertProjectionForParent = static function (int $parentId) use ($now): string {
$parent = CloudJob::findOrFail($parentId);
$children = CloudJob::where('parent_job_id', $parentId)
->orderBy('id')->get();
if ($children->isEmpty()) {
throw new RuntimeException('Projection fixture requires children.');
}
foreach ($children as $index => $child) {
$hostId = 1000 + (int) $child->id;
$networkSelector = 'bridge:b' . $hostId;
Capsule::table('cloud_jobs')->where('id', (int) $child->id)
->update(['host_id' => $hostId]);
Capsule::table('cloud_hosts')->insertOrIgnore([
'id' => $hostId, 'enabled' => 1,
'default_network' => 'bridge:br0',
]);
Capsule::table('cloud_host_agents')->insertOrIgnore([
'host_id' => $hostId, 'enabled' => 1, 'revision' => 1,
'tested_revision' => 1, 'last_test_status' => 'healthy',
'network_selectors_json' => json_encode(
[$networkSelector],
JSON_THROW_ON_ERROR
),
]);
Capsule::table('cloud_host_storage_configurations')
->insertOrIgnore([
'host_id' => $hostId, 'state' => 'applied',
]);
Capsule::table('cloud_host_resource_pool_memberships')
->insertOrIgnore([
'host_id' => $hostId, 'resource_pool_id' => 1,
'scheduler_state' => 'active',
]);
Capsule::table('cloud_host_capacity_policies')->insertOrIgnore([
'host_id' => $hostId, 'configured_cpu_cores' => 16,
'configured_memory_mib' => 32768,
'configured_disk_gib' => 512,
]);
$attachmentId = 100000 + $hostId;
Capsule::table('cloud_host_network_attachments')
->insertOrIgnore([
'id' => $attachmentId,
'host_id' => $hostId,
'line_id' => 1,
'network_selector' => $networkSelector,
'state' => 'active',
'verification_state' => 'verified',
'verified_agent_revision' => 1,
'last_verified_at' => $now,
]);
$publicPoolId = 200000 + $hostId;
$thirdOctet = intdiv($hostId, 30) % 250;
$networkOctet = ($hostId % 30) * 8;
$networkPrefix = '198.18.' . $thirdOctet . '.';
Capsule::table('cloud_public_ip_pools')->insertOrIgnore([
'id' => $publicPoolId,
'pool_key' => md5('aggregate-network-pool:' . $hostId),
'host_id' => $hostId,
'network_attachment_id' => $attachmentId,
'cidr' => $networkPrefix . $networkOctet . '/29',
'gateway' => $networkPrefix . ($networkOctet + 1),
'guest_prefix_length' => 29,
'dns_servers_json' => '[]',
'reserved_ips_json' => '[]',
'state' => 'active',
'sync_state' => 'applied',
'desired_revision' => 1,
'observed_revision' => 1,
]);
Capsule::table('cloud_public_ips')->insertOrIgnore([
'host_id' => $hostId,
'public_ip_pool_id' => $publicPoolId,
'address' => $networkPrefix . ($networkOctet + 2),
'prefix_length' => 29,
'state' => 'available',
'reservation_id' => null,
'instance_id' => null,
'operation_id' => null,
]);
$child->host_id = $hostId;
}
if (str_starts_with((string) $parent->kind, 'template_')) {
$resourceId = 'aggregate-projection-' . $parentId;
$generation = md5('projection-generation:' . $parentId);
$sha256 = hash('sha256', 'projection-sha:' . $parentId);
$state = match ((string) $parent->kind) {
'template_import' => 'degraded',
'template_metadata_sync' => 'active',
'template_delete' => 'delete_failed',
};
Capsule::table('cloud_templates')->insert([
'id' => $resourceId,
'name' => 'Aggregate projection ' . $parentId,
'os_family' => 'Debian',
'os_version' => '13',
'image_bytes' => 4096,
'sha256' => $sha256,
'startup_script' => '',
'revision' => (string) $parent->kind
=== 'template_metadata_sync' ? 2 : 1,
'generation_token' => $generation,
'state' => $state,
'active_job_id' => $parentId,
'created_at' => $now - 60,
'updated_at' => $now,
]);
foreach ($children as $child) {
$applied = (string) $child->effect_resolution === 'applied';
$replicaState = match ((string) $parent->kind) {
'template_import' => $applied ? 'active' : 'failed',
'template_metadata_sync' => $applied ? 'active' : 'stale',
'template_delete' => $applied
? 'delete_succeeded' : 'delete_failed',
};
Capsule::table('cloud_template_replicas')->insert([
'host_id' => (int) $child->host_id,
'template_id' => $resourceId,
'revision' => (string) $parent->kind
=== 'template_metadata_sync'
? ($applied ? 2 : 1) : 1,
'generation_token' => $generation,
'state' => $replicaState,
'image_bytes' => (string) $parent->kind
=== 'template_import' && !$applied
? 0 : 4096,
'sha256' => $sha256,
'operation_id' => (string) $child->operation_id,
'job_id' => (int) $child->id,
'error_code' => $applied ? '' : 'operation_not_applied',
'created_at' => $now - 60,
'updated_at' => $now,
]);
}
return $resourceId;
}
$resourceId = md5('aggregate-iso-projection:' . $parentId);
$sha256 = hash('sha256', 'projection-iso-sha:' . $parentId);
Capsule::table('cloud_user_isos')->insert([
'id' => $resourceId,
'user_id' => (int) $parent->user_id,
'name' => 'Aggregate ISO ' . $parentId,
'sealed_source_url' => 'test-only',
'size_bytes' => 4096,
'sha256' => $sha256,
'state' => (string) $parent->kind === 'user_iso_import'
? 'import_failed' : 'delete_failed',
'import_operation_id' => (string) $parent->operation_id,
'active_job_id' => $parentId,
'created_at' => $now - 60,
'updated_at' => $now,
]);
foreach ($children as $child) {
$applied = (string) $child->effect_resolution === 'applied';
$replicaState = (string) $parent->kind === 'user_iso_import'
? ($applied ? 'active' : 'failed')
: ($applied ? 'delete_succeeded' : 'delete_failed');
Capsule::table('cloud_user_iso_replicas')->insert([
'host_id' => (int) $child->host_id,
'user_iso_id' => $resourceId,
'user_id' => (int) $parent->user_id,
'state' => $replicaState,
'size_bytes' => (string) $parent->kind === 'user_iso_delete'
|| $applied ? 4096 : 0,
'sha256' => $sha256,
'operation_id' => (string) $child->operation_id,
'job_id' => (int) $child->id,
'error_code' => $applied ? '' : 'operation_not_applied',
'created_at' => $now - 60,
'updated_at' => $now,
]);
}
return $resourceId;
};
$settleLastChild = static function (
string $firstResolution
) use (
$insertParent,
$insertResolvedChild,
$insertProjectionForParent,
$assert
): CloudJob {
$parentId = $insertParent();
$insertResolvedChild(
$parentId,
$firstResolution,
'template_import_replica'
);
$lastChildId = $insertResolvedChild(
$parentId,
'not_applied',
'template_import_replica'
);
$insertProjectionForParent($parentId);
Capsule::table('cloud_jobs')->where('id', $lastChildId)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
'terminal_at' => null,
]);
$before = CloudJob::find($parentId);
$assert($before !== null
&& CloudJobService::recoveryPresentation($before, true) !== null,
'The aggregate fixture was not fenced before its last child.');
Capsule::table('cloud_jobs')->where('id', $lastChildId)->update([
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 1,
'effect_resolution' => 'not_applied',
'effect_resolved_at' => time(),
'effect_resolution_reason' => 'normal_completion',
'terminal_at' => time(),
]);
$completed = CloudJob::find($lastChildId);
CloudJobService::reconcileAggregateParentEffectLocked(
$completed,
null
);
$assert((string) $completed->phase === 'terminal'
&& (string) $completed->outcome === 'failed'
&& (string) $completed->effect_resolution === 'not_applied',
'The last leaf did not enter its normal terminal state.');
return CloudJob::find($parentId);
};
$allAbsent = $settleLastChild('not_applied');
$assert($allAbsent !== null
&& (string) $allAbsent->phase === 'terminal'
&& (string) $allAbsent->outcome === 'failed'
&& (string) $allAbsent->status === 'failed'
&& (string) $allAbsent->effect_resolution === 'not_applied'
&& (int) $allAbsent->effect_resolved_at > 0
&& $allAbsent->effect_resolved_by === null
&& str_starts_with(
(string) $allAbsent->effect_resolution_reason,
'child_effects_resolved:'
)
&& !(bool) $allAbsent->finalization_incident
&& $allAbsent->finalization_error_code === null
&& $allAbsent->lease_token === null
&& (int) $allAbsent->lease_until === 0
&& (int) $allAbsent->available_at === 0
&& $allAbsent->read_at === null
&& (string) Capsule::table('cloud_templates')
->where('active_job_id', (int) $allAbsent->id)
->value('state') === 'retired'
&& !Capsule::table('cloud_template_replicas')
->whereIn('template_id', static function ($query) use ($allAbsent) {
$query->select('id')->from('cloud_templates')
->where('active_job_id', (int) $allAbsent->id);
})->exists()
&& CloudJobService::recoveryPresentation($allAbsent, true) === null,
'All-not-applied import children did not retire their placeholder.');
$partial = $settleLastChild('applied');
$partialTemplateId = 'aggregate-projection-' . (int) $partial->id;
$partialTemplate = Capsule::table('cloud_templates')
->where('id', $partialTemplateId)->first();
$partialReplicaStates = Capsule::table('cloud_template_replicas')
->where('template_id', $partialTemplateId)->orderBy('state')
->pluck('state')->all();
$partialTemplateDto = collect(
CloudTemplateCatalogService::adminCatalog()['templates']
)->firstWhere('id', $partialTemplateId);
$partialDisplay = CloudJobService::display($partial);
$assert($partial !== null
&& (string) $partial->phase === 'terminal'
&& (string) $partial->outcome === 'failed'
&& (string) $partial->status === 'failed'
&& (string) $partial->effect_resolution === 'applied'
&& str_ends_with(
(string) $partial->effect_resolution_reason,
':mixed'
)
&& !(bool) $partial->finalization_incident
&& $partial->finalization_error_code === null
&& $partial->lease_token === null
&& (int) $partial->available_at === 0
&& $partialTemplate !== null
&& (string) $partialTemplate->state === 'degraded'
&& $partialTemplate->active_job_id === null
&& $partialReplicaStates === ['active', 'failed']
&& is_array($partialTemplateDto)
&& ($partialTemplateDto['cleanup_reason'] ?? null)
=== 'partial_import'
&& ($partialDisplay['effect_scope'] ?? null) === 'partial'
&& CloudJobService::recoveryPresentation($partial, true) === null,
'A proven partial template import did not settle as failed-but-applied.');
$partialAppliedHost = (int) Capsule::table('cloud_template_replicas')
->where('template_id', $partialTemplateId)
->where('state', 'active')->value('host_id');
$partialOriginalOperations = CloudJob::where(
'parent_job_id',
(int) $partial->id
)->pluck('operation_id')->all();
$partialCleanup = CloudTemplateCatalogService::delete(
$partialTemplateId,
(int) $partialTemplate->revision,
(string) $partialTemplate->generation_token,
[
'user_id' => 1,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
]
);
$partialCleanupId = (int) ($partialCleanup['task_id'] ?? 0);
$partialCleanupParent = CloudJob::findOrFail($partialCleanupId);
$partialCleanupChildren = CloudJob::where(
'parent_job_id',
$partialCleanupId
)->get();
$partialCleanupTemplate = Capsule::table('cloud_templates')
->where('id', $partialTemplateId)->first();
$partialCleanupReplicas = Capsule::table('cloud_template_replicas')
->where('template_id', $partialTemplateId)->get();
$assert($partialCleanupId > 0
&& $partialCleanupId !== (int) $partial->id
&& (string) $partialCleanupParent->kind === 'template_delete'
&& (string) $partialCleanupParent->phase === 'awaiting_receipt'
&& $partialCleanupTemplate !== null
&& (string) $partialCleanupTemplate->state === 'deleting'
&& (int) $partialCleanupTemplate->active_job_id === $partialCleanupId
&& $partialCleanupChildren->count() === 1
&& (int) $partialCleanupChildren->first()->host_id
=== $partialAppliedHost
&& !in_array(
(string) $partialCleanupChildren->first()->operation_id,
$partialOriginalOperations,
true
)
&& $partialCleanupReplicas->count() === 1
&& (string) $partialCleanupReplicas->first()->state === 'deleting',
'Partial template import could not start an independent cleanup.');
$nextId = max(
$nextId,
(int) Capsule::table('cloud_jobs')->max('id') + 1
);
$startMixedCleanup = static function (
string $parentKind,
string $childKind
) use (
$insertParent,
$insertResolvedChild,
$insertProjectionForParent,
$assert,
&$nextId
): array {
$parentId = $insertParent($parentKind);
$appliedChildId = $insertResolvedChild(
$parentId,
'applied',
$childKind
);
$insertResolvedChild($parentId, 'not_applied', $childKind);
$resourceId = $insertProjectionForParent($parentId);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($appliedChildId),
null
);
$parent = CloudJob::findOrFail($parentId);
$validReason = (string) $parent->effect_resolution_reason;
$jobsBeforeMalformed = CloudJob::count();
$isTemplate = str_starts_with($parentKind, 'template_');
$replicaTable = $isTemplate
? 'cloud_template_replicas' : 'cloud_user_iso_replicas';
$replicaKey = $isTemplate ? 'template_id' : 'user_iso_id';
$replicasBeforeMalformed = Capsule::table($replicaTable)
->where($replicaKey, $resourceId)->orderBy('host_id')->get()
->map(static fn ($row): array => (array) $row)->all();
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
'effect_resolution_reason' => 'garbage:mixed',
]);
$malformedRejected = false;
try {
if ($isTemplate) {
$malformedTemplate = Capsule::table('cloud_templates')
->where('id', $resourceId)->first();
CloudTemplateCatalogService::delete(
$resourceId,
(int) $malformedTemplate->revision,
(string) $malformedTemplate->generation_token,
[
'user_id' => 1,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
]
);
} else {
$malformedDto = collect(
CloudUserIsoService::listing(1)['isos']
)->firstWhere('id', $resourceId);
$assert(is_array($malformedDto)
&& ($malformedDto['can_delete'] ?? true) === false
&& !array_key_exists('cleanup_reason', $malformedDto),
'Malformed mixed ISO marker exposed cleanup in its DTO.');
CloudUserIsoService::delete(
$resourceId,
['user_id' => 1]
);
}
} catch (CloudError $error) {
$malformedRejected = $error->httpStatus === 409;
}
$replicasAfterMalformed = Capsule::table($replicaTable)
->where($replicaKey, $resourceId)->orderBy('host_id')->get()
->map(static fn ($row): array => (array) $row)->all();
$malformedPointer = $isTemplate
? Capsule::table('cloud_templates')->where('id', $resourceId)
->value('active_job_id')
: Capsule::table('cloud_user_isos')->where('id', $resourceId)
->value('active_job_id');
$assert($malformedRejected
&& CloudJob::count() === $jobsBeforeMalformed
&& $malformedPointer === null
&& $replicasAfterMalformed === $replicasBeforeMalformed,
'Malformed mixed marker crossed the cleanup mutation boundary: '
. $parentKind);
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
'effect_resolution_reason' => $validReason,
]);
$parent = CloudJob::findOrFail($parentId);
$oldOperations = CloudJob::where('parent_job_id', $parentId)
->pluck('operation_id')->all();
if ($isTemplate) {
$resource = Capsule::table('cloud_templates')
->where('id', $resourceId)->first();
$preCleanupDto = null;
$cleanup = CloudTemplateCatalogService::delete(
$resourceId,
(int) $resource->revision,
(string) $resource->generation_token,
[
'user_id' => 1,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
]
);
} else {
$resource = Capsule::table('cloud_user_isos')
->where('id', $resourceId)->first();
$preCleanupDto = collect(
CloudUserIsoService::listing(1)['isos']
)->firstWhere('id', $resourceId);
$cleanup = CloudUserIsoService::delete(
$resourceId,
['user_id' => 1]
);
}
$cleanupId = (int) ($cleanup['task_id'] ?? 0);
$children = CloudJob::where('parent_job_id', $cleanupId)->get();
$nextId = max(
$nextId,
(int) Capsule::table('cloud_jobs')->max('id') + 1
);
return [
$parent,
$resource,
$resourceId,
$cleanupId,
$children,
$oldOperations,
$preCleanupDto,
];
};
[$partialDelete, $partialDeleteTemplate, $partialDeleteTemplateId,
$partialDeleteCleanupId, $partialDeleteCleanupChildren,
$partialDeleteOldOperations] = $startMixedCleanup(
'template_delete',
'template_delete_replica'
);
$partialDeleteReplicas = Capsule::table('cloud_template_replicas')
->where('template_id', $partialDeleteTemplateId)->get();
$assert((string) $partialDelete->effect_resolution === 'applied'
&& str_ends_with(
(string) $partialDelete->effect_resolution_reason,
':mixed'
)
&& (string) $partialDeleteTemplate->state === 'delete_failed'
&& $partialDeleteTemplate->active_job_id === null
&& $partialDeleteCleanupId > 0
&& $partialDeleteCleanupId !== (int) $partialDelete->id
&& $partialDeleteCleanupChildren->count() === 1
&& !in_array(
(string) $partialDeleteCleanupChildren->first()->operation_id,
$partialDeleteOldOperations,
true
)
&& $partialDeleteReplicas->count() === 1
&& (string) $partialDeleteReplicas->first()->state === 'deleting',
'Partial template delete did not preserve failure and retry survivors.');
[$partialIsoImport, $partialIsoImportResource, $partialIsoImportId,
$partialIsoImportCleanupId, $partialIsoImportCleanupChildren,
$partialIsoImportOldOperations, $partialIsoImportDto] =
$startMixedCleanup(
'user_iso_import',
'user_iso_import_replica'
);
$partialIsoImportReplicas = Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $partialIsoImportId)->get();
$assert((string) $partialIsoImport->effect_resolution === 'applied'
&& str_ends_with(
(string) $partialIsoImport->effect_resolution_reason,
':mixed'
)
&& (string) $partialIsoImportResource->state === 'import_failed'
&& $partialIsoImportResource->active_job_id === null
&& is_array($partialIsoImportDto)
&& ($partialIsoImportDto['can_delete'] ?? false) === true
&& ($partialIsoImportDto['cleanup_reason'] ?? null)
=== 'partial_import'
&& $partialIsoImportCleanupId > 0
&& $partialIsoImportCleanupChildren->count() === 1
&& !in_array(
(string) $partialIsoImportCleanupChildren->first()->operation_id,
$partialIsoImportOldOperations,
true
)
&& $partialIsoImportReplicas->count() === 1
&& (string) $partialIsoImportReplicas->first()->state === 'deleting',
'Partial ISO import did not expose an independent safe cleanup.');
[$partialIsoDelete, $partialIsoDeleteResource, $partialIsoDeleteId,
$partialIsoDeleteCleanupId, $partialIsoDeleteCleanupChildren,
$partialIsoDeleteOldOperations, $partialIsoDeleteDto] =
$startMixedCleanup(
'user_iso_delete',
'user_iso_delete_replica'
);
$partialIsoDeleteReplicas = Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $partialIsoDeleteId)->get();
$assert((string) $partialIsoDelete->effect_resolution === 'applied'
&& str_ends_with(
(string) $partialIsoDelete->effect_resolution_reason,
':mixed'
)
&& (string) $partialIsoDeleteResource->state === 'delete_failed'
&& $partialIsoDeleteResource->active_job_id === null
&& is_array($partialIsoDeleteDto)
&& ($partialIsoDeleteDto['can_delete'] ?? false) === true
&& ($partialIsoDeleteDto['cleanup_reason'] ?? null)
=== 'partial_delete'
&& $partialIsoDeleteCleanupId > 0
&& $partialIsoDeleteCleanupChildren->count() === 1
&& !in_array(
(string) $partialIsoDeleteCleanupChildren->first()->operation_id,
$partialIsoDeleteOldOperations,
true
)
&& $partialIsoDeleteReplicas->count() === 1
&& (string) $partialIsoDeleteReplicas->first()->state === 'deleting',
'Partial ISO delete did not expose an independent safe cleanup.');
// A zero-dispatch child can know its host effect is not-applied while its
// PHP replica/catalog finalizer is still rolled back. Neither the realtime
// sibling hook nor the legacy sweep may publish the parent until that
// independent local incident has committed and cleared.
$localIncidentParentId = $insertParent('template_import');
$localIncidentChildId = $insertResolvedChild(
$localIncidentParentId,
'not_applied',
'template_import_replica'
);
$insertProjectionForParent($localIncidentParentId);
Capsule::table('cloud_jobs')->where(
'id',
$localIncidentChildId
)->update([
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'dispatched_at' => null,
'receipt_deadline_at' => null,
'completion_event_id' => null,
'safe_to_resubmit' => 0,
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_ledger_conflict',
]);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($localIncidentChildId),
null
);
$localIncidentParent = CloudJob::findOrFail($localIncidentParentId);
$assert((string) $localIncidentParent->effect_resolution === 'unknown'
&& (bool) $localIncidentParent->finalization_incident,
'A local-finalizer child prematurely released its aggregate parent.');
Capsule::table('cloud_jobs')->where(
'id',
$localIncidentChildId
)->update([
'safe_to_resubmit' => 1,
'finalization_incident' => 0,
'finalization_error_code' => null,
'effect_resolved_at' => time(),
'effect_resolution_reason' => 'automatic_local_not_dispatched',
]);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($localIncidentChildId),
null
);
$localIncidentParent->refresh();
$assert((string) $localIncidentParent->effect_resolution
=== 'not_applied'
&& !(bool) $localIncidentParent->finalization_incident,
'The aggregate parent stayed fenced after its local child converged.');
$makeLegacyEligible = static function (int $parentId): void {
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
'lease_token' => null,
'lease_until' => 0,
'available_at' => 0,
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_child_convergence',
]);
};
$legacyLocalIncidentParentId = $insertParent('template_import');
$makeLegacyEligible($legacyLocalIncidentParentId);
$legacyLocalIncidentChildId = $insertResolvedChild(
$legacyLocalIncidentParentId,
'not_applied',
'template_import_replica'
);
$insertProjectionForParent($legacyLocalIncidentParentId);
Capsule::table('cloud_jobs')->where(
'id',
$legacyLocalIncidentChildId
)->update([
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'dispatched_at' => null,
'receipt_deadline_at' => null,
'completion_event_id' => null,
'safe_to_resubmit' => 0,
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_ledger_conflict',
]);
$assert(CloudJobService::reconcileOneLegacyAggregateEffect(),
'The legacy sweep did not inspect a local-finalizer child incident.');
$legacyLocalIncidentParent = CloudJob::findOrFail(
$legacyLocalIncidentParentId
);
$assert((string) $legacyLocalIncidentParent->effect_resolution
=== 'unknown'
&& (string) $legacyLocalIncidentParent->finalization_error_code
=== 'recovery_child_convergence',
'The legacy sweep prematurely released a local-finalizer child.');
Capsule::table('cloud_jobs')->where(
'id',
$legacyLocalIncidentChildId
)->update([
'safe_to_resubmit' => 1,
'finalization_incident' => 0,
'finalization_error_code' => null,
'effect_resolved_at' => time(),
'effect_resolution_reason' => 'automatic_local_not_dispatched',
]);
$makeLegacyEligible($legacyLocalIncidentParentId);
$assert(CloudJobService::reconcileOneLegacyAggregateEffect(),
'The legacy sweep did not resume after local finalizer convergence.');
$legacyLocalIncidentParent->refresh();
$assert((string) $legacyLocalIncidentParent->effect_resolution
=== 'not_applied'
&& !(bool) $legacyLocalIncidentParent->finalization_incident,
'The legacy parent stayed fenced after local finalizer convergence.');
// The realtime hook applies the same topology rules to both the child that
// triggered it and every sibling. A wrong link is an incident, never
// evidence that the aggregate effect can be published.
$liveWrongCurrentParent = $insertParent('template_delete');
$makeLegacyEligible($liveWrongCurrentParent);
$liveWrongCurrent = $insertResolvedChild(
$liveWrongCurrentParent,
'applied',
'template_import_replica'
);
$insertResolvedChild(
$liveWrongCurrentParent,
'not_applied',
'template_delete_replica'
);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::find($liveWrongCurrent),
null
);
$liveWrongSiblingParent = $insertParent('user_iso_delete');
$makeLegacyEligible($liveWrongSiblingParent);
$liveValidCurrent = $insertResolvedChild(
$liveWrongSiblingParent,
'applied',
'user_iso_delete_replica'
);
$insertResolvedChild(
$liveWrongSiblingParent,
'not_applied',
'user_iso_import_replica'
);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::find($liveValidCurrent),
null
);
foreach ([
$liveWrongCurrentParent => 'current child',
$liveWrongSiblingParent => 'sibling child',
] as $parentId => $description) {
$conflict = CloudJob::find($parentId);
$display = CloudJobService::recoveryPresentation($conflict, true);
$assert((string) $conflict->effect_resolution === 'unknown'
&& (bool) $conflict->finalization_incident
&& (string) $conflict->finalization_error_code
=== 'recovery_aggregate_state_conflict'
&& (int) $conflict->available_at >= time() + 290
&& $conflict->read_at === null
&& ($display['code'] ?? null)
=== 'recovery_aggregate_state_conflict',
'The live hook released a parent with a wrong '
. $description . ' relationship.');
}
foreach ([
'missing parent' => static function () use (
$insertResolvedChild
): CloudJob {
return CloudJob::findOrFail($insertResolvedChild(
999999,
'applied',
'template_import_replica'
));
},
'non-aggregate parent' => static function () use (
$insertParent,
$insertResolvedChild
): CloudJob {
$parentId = $insertParent('template_import');
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
'execution_mode' => 'local',
]);
return CloudJob::findOrFail($insertResolvedChild(
$parentId,
'applied',
'template_import_replica'
));
},
] as $description => $fixture) {
$thrown = false;
try {
CloudJobService::reconcileAggregateParentEffectLocked(
$fixture(),
null
);
} catch (CloudRecoveryAggregateStateConflict $error) {
$thrown = true;
}
$assert($thrown,
'A child with a ' . $description
. ' could be silently acknowledged.');
}
// A damaged aggregate relationship must be rejected before the recovery
// coordinator writes the Agent journal fence. Cover both parent-envelope
// and child-kind corruption at the private transaction boundary.
$resolveFence = new ReflectionMethod(
CloudAutomaticOperationRecoveryService::class,
'resolveFenceAndCommit'
);
$preparedFence = [
'resolution' => 'applied',
'journal_status' => 'failed',
'fence_resolution_required' => true,
'source' => 'host_inspection',
'operation' => [],
];
foreach ([
'bad parent mode' => static function () use (
$insertParent,
$insertResolvedChild
): CloudJob {
$parentId = $insertParent('template_import');
$childId = $insertResolvedChild(
$parentId,
'applied',
'template_import_replica'
);
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
'execution_mode' => 'local',
]);
Capsule::table('cloud_jobs')->where('id', $childId)->update([
'effect_resolution' => 'unknown',
'lease_token' => str_repeat('f', 32),
'lease_until' => time() + 300,
]);
return CloudJob::findOrFail($childId);
},
'wrong child kind' => static function () use (
$insertParent,
$insertResolvedChild,
$insertProjectionForParent
): CloudJob {
$parentId = $insertParent('template_delete');
$childId = $insertResolvedChild(
$parentId,
'applied',
'template_import_replica'
);
$insertProjectionForParent($parentId);
Capsule::table('cloud_jobs')->where('id', $childId)->update([
'effect_resolution' => 'unknown',
'lease_token' => str_repeat('e', 32),
'lease_until' => time() + 300,
]);
return CloudJob::findOrFail($childId);
},
] as $description => $fixture) {
$beforeFenceRequests = CloudAgentSettingsService::$requests;
$blocked = false;
try {
$resolveFence->invoke(
null,
$fixture(),
$preparedFence,
'test-only',
[]
);
} catch (CloudRecoveryAggregateStateConflict $error) {
$blocked = true;
}
$assert(
$blocked
&& CloudAgentSettingsService::$requests === $beforeFenceRequests,
'A ' . $description . ' reached an Agent fence write.'
);
}
// Upgrade sweep: these parents were already terminal failed/unknown before
// the realtime leaf hook existed. Each tick may converge exactly one local
// parent from already-known children and must never need an Agent client.
$legacyMappings = [
'template_import' => 'template_import_replica',
'template_metadata_sync' => 'template_metadata_replica',
'template_delete' => 'template_delete_replica',
'user_iso_import' => 'user_iso_import_replica',
'user_iso_delete' => 'user_iso_delete_replica',
];
$legacyParents = [];
foreach ($legacyMappings as $parentKind => $childKind) {
$parentId = $insertParent($parentKind);
$makeLegacyEligible($parentId);
$firstResolution = $parentKind === 'template_import'
? 'not_applied' : 'applied';
$insertResolvedChild(
$parentId,
$firstResolution,
$childKind
);
$insertResolvedChild($parentId, $firstResolution, $childKind);
$resourceId = $insertProjectionForParent($parentId);
$legacyParents[$parentId] = [
'kind' => $parentKind,
'resolution' => $firstResolution === 'applied'
? 'applied' : 'not_applied',
'resource_id' => $resourceId,
];
}
$automatic = new CloudAutomaticOperationRecoveryService();
$resolvedLegacyCount = 0;
foreach ($legacyParents as $parentId => $expectation) {
$assert($automatic->tick(),
'The local legacy aggregate sweep did not claim a parent.');
$resolved = CloudJob::whereIn('id', array_keys($legacyParents))
->whereIn('effect_resolution', ['applied', 'not_applied'])
->count();
$current = CloudJob::find($parentId);
if ($expectation['kind'] === 'template_delete') {
$resourceId = $expectation['resource_id'];
$assert($resolved === $resolvedLegacyCount
&& $current !== null
&& (string) $current->effect_resolution === 'unknown'
&& (bool) $current->finalization_incident
&& (string) $current->finalization_error_code
=== 'recovery_child_convergence'
&& (string) Capsule::table('cloud_templates')
->where('id', $resourceId)->value('state')
=== 'delete_failed'
&& (string) Capsule::table('cloud_template_artifacts')
->where('template_id', $resourceId)->value('state')
=== 'delete_pending'
&& Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->exists(),
'Template delete recovery published applied before the exact '
. 'master object was confirmed absent.');
continue;
}
$resolvedLegacyCount++;
$assert($resolved === $resolvedLegacyCount
&& $current !== null
&& (string) $current->effect_resolution
=== $expectation['resolution']
&& (string) $current->outcome === 'failed'
&& !(bool) $current->finalization_incident
&& $current->finalization_error_code === null
&& CloudJobService::recoveryPresentation(
$current,
true
) === null,
'A tick handled multiple legacy parents or mapped children incorrectly.');
$resourceId = $expectation['resource_id'];
$projectionOkay = match ($expectation['kind']) {
'template_import' =>
(string) Capsule::table('cloud_templates')
->where('id', $resourceId)->value('state') === 'retired'
&& !Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->exists(),
'template_metadata_sync' =>
(string) Capsule::table('cloud_templates')
->where('id', $resourceId)->value('state') === 'active'
&& !Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)
->where('state', '<>', 'active')->exists(),
'user_iso_import' =>
(string) Capsule::table('cloud_user_isos')
->where('id', $resourceId)->value('state') === 'active',
'user_iso_delete' =>
(string) Capsule::table('cloud_user_isos')
->where('id', $resourceId)->value('state') === 'retired'
&& !Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $resourceId)->exists(),
};
$assert($projectionOkay,
'A legacy terminal parent did not recompute its resource projection: '
. $expectation['kind']);
}
$waitingParent = $insertParent('template_import');
$makeLegacyEligible($waitingParent);
$waitingChild = $insertResolvedChild(
$waitingParent,
'not_applied',
'template_import_replica'
);
$insertProjectionForParent($waitingParent);
Capsule::table('cloud_jobs')->where('id', $waitingChild)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
]);
// Receipt handling must run before a legacy aggregate that can remain in
// child-convergence for an arbitrarily long time. Use an invalid durable
// event here so the scheduler must quarantine it without touching Agent
// or catalog state; the aggregate remains immediately eligible for the
// following tick instead of starving the receipt forever.
$priorityChild = $insertResolvedChild(
0,
'not_applied',
'template_import_replica'
);
Capsule::table('cloud_jobs')->where('id', $priorityChild)->update([
'outcome' => 'failed',
'status' => 'failed',
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
'available_at' => 0,
'dispatched_at' => $now - 20,
]);
$priorityOperation = (string) CloudJob::findOrFail(
$priorityChild
)->operation_id;
Capsule::table('cloud_agent_completion_events')->insert([
'event_id' => 'aggregate-priority-receipt',
'host_id' => 7,
'operation_id' => $priorityOperation,
'payload_sha256' => str_repeat('0', 64),
'outcome' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'error_code' => '',
'result_json' => null,
'disposition' => 'received',
'received_at' => $now,
'applied_at' => null,
]);
$makeLegacyEligible($waitingParent);
$assert($automatic->tick(),
'A durable receipt was not prioritised over a waiting aggregate.');
$priorityEvent = Capsule::table('cloud_agent_completion_events')
->where('event_id', 'aggregate-priority-receipt')->first();
$stillEligibleParent = CloudJob::findOrFail($waitingParent);
$assert((string) $priorityEvent->disposition === 'rejected'
&& (int) $stillEligibleParent->available_at === 0,
'A waiting legacy aggregate starved receipt validation.');
$assert($automatic->tick(),
'A legacy aggregate with an active child was not inspected locally.');
$waiting = CloudJob::find($waitingParent);
$waitingDisplay = CloudJobService::recoveryPresentation($waiting, true);
$assert((string) $waiting->effect_resolution === 'unknown'
&& (string) $waiting->finalization_error_code
=== 'recovery_child_convergence'
&& (int) $waiting->available_at >= time() + 290
&& ($waitingDisplay['code'] ?? null)
=== 'recovery_child_convergence',
'An unresolved but valid child released or corrupted its parent fence.');
// Corrupt aggregate relationships remain fenced and retryable. Cover every
// structural invariant independently so a future relaxed query cannot turn
// a wrong child into evidence for releasing its parent.
$conflictFixtures = [];
$missingChildParent = $insertParent('template_import');
$makeLegacyEligible($missingChildParent);
$conflictFixtures[$missingChildParent] = 'missing child';
$wrongParentMode = $insertParent('template_import');
$makeLegacyEligible($wrongParentMode);
$insertResolvedChild(
$wrongParentMode,
'applied',
'template_import_replica'
);
Capsule::table('cloud_jobs')->where('id', $wrongParentMode)->update([
'execution_mode' => 'local',
]);
$conflictFixtures[$wrongParentMode] = 'wrong parent execution mode';
$wrongParentKind = $insertParent('template_import');
$makeLegacyEligible($wrongParentKind);
$insertResolvedChild(
$wrongParentKind,
'applied',
'template_import_replica'
);
Capsule::table('cloud_jobs')->where('id', $wrongParentKind)->update([
'kind' => 'corrupt_aggregate_kind',
]);
$conflictFixtures[$wrongParentKind] = 'wrong parent kind';
$wrongKindParent = $insertParent('template_delete');
$makeLegacyEligible($wrongKindParent);
$insertResolvedChild(
$wrongKindParent,
'applied',
'template_import_replica'
);
$conflictFixtures[$wrongKindParent] = 'wrong child kind';
$wrongUserParent = $insertParent('user_iso_import');
$makeLegacyEligible($wrongUserParent);
$insertResolvedChild(
$wrongUserParent,
'applied',
'user_iso_import_replica',
2
);
$conflictFixtures[$wrongUserParent] = 'wrong child owner';
$wrongModeParent = $insertParent('user_iso_delete');
$makeLegacyEligible($wrongModeParent);
$insertResolvedChild(
$wrongModeParent,
'not_applied',
'user_iso_delete_replica',
1,
'local'
);
$conflictFixtures[$wrongModeParent] = 'wrong child execution mode';
$nestedParent = $insertParent('template_metadata_sync');
$makeLegacyEligible($nestedParent);
Capsule::table('cloud_jobs')->where('id', $nestedParent)->update([
// The parent only needs to be non-null to exercise the invalid
// aggregate nesting invariant. Do not attach it to an earlier valid
// aggregate fixture: doing so also (correctly) corrupts that parent's
// already-published child topology and changes the next claimed row.
'parent_job_id' => 999999,
]);
$insertResolvedChild(
$nestedParent,
'applied',
'template_metadata_replica'
);
$conflictFixtures[$nestedParent] = 'non-top-level aggregate parent';
// A malformed Agent child is itself a higher-priority PHP-only leaf
// repair, so it may be claimed before an earlier aggregate parent. Drain
// the bounded fixture set, then verify every parent retained its fence.
$beforeConflictRequests = CloudAgentSettingsService::$requests;
for ($attempt = 0; $attempt < count($conflictFixtures) * 3; $attempt++) {
$remaining = CloudJob::whereIn('id', array_keys($conflictFixtures))
->where(static function ($pending): void {
$pending->where('finalization_incident', 0)
->orWhereNull('finalization_error_code')
->orWhere(
'finalization_error_code',
'<>',
'recovery_aggregate_state_conflict'
);
})->exists();
if (!$remaining) {
break;
}
$assert($automatic->tick(),
'The aggregate conflict fixture queue stopped before convergence.');
}
foreach ($conflictFixtures as $parentId => $description) {
$conflict = CloudJob::find($parentId);
$display = $conflict === null ? null
: CloudJobService::recoveryPresentation($conflict, true);
$assert($conflict !== null
&& (string) $conflict->effect_resolution === 'unknown'
&& (string) $conflict->outcome === 'failed'
&& (bool) $conflict->finalization_incident
&& (string) $conflict->finalization_error_code
=== 'recovery_aggregate_state_conflict'
&& (int) $conflict->available_at >= time() + 290
&& $conflict->lease_token === null
&& (int) $conflict->lease_until === 0
&& $conflict->read_at === null
&& ($display['code'] ?? null)
=== 'recovery_aggregate_state_conflict'
&& str_contains(
(string) ($display['message'] ?? ''),
'关系异常'
)
&& CloudAgentSettingsService::$requests
=== $beforeConflictRequests,
'A ' . $description
. ' relationship released or concealed its aggregate fence.');
}
// prepareRecoveredEffect updates a replica before the external Agent fence
// write. Simulate that exact local prepare seam: the public parent must
// remain visibly recovering until the child effect commit runs, then the
// same transaction may publish the aggregate success.
$preparedParentId = $nextId++;
$preparedParentOperation = md5(
'aggregate-prepared-parent:' . $preparedParentId
);
Capsule::table('cloud_jobs')->insert([
'id' => $preparedParentId,
'operation_id' => $preparedParentOperation,
'request_idempotency_key' => $preparedParentOperation,
'request_fingerprint' => hash('sha256', $preparedParentOperation),
'parent_job_id' => null,
'user_id' => 1,
'host_id' => null,
'kind' => 'template_import',
'execution_mode' => 'aggregate',
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now + 300,
'receipt_deadline_at' => $now + 900,
'effect_resolution' => 'not_applicable',
'effect_resolution_reason' => '',
'lease_until' => 0,
'available_at' => $now,
'finalization_incident' => 0,
'created_at' => $now - 60,
'updated_at' => $now,
]);
$preparedChildId = $insertResolvedChild(
$preparedParentId,
'not_applied',
'template_import_replica'
);
Capsule::table('cloud_jobs')->where('id', $preparedChildId)->update([
'outcome' => 'failed',
'status' => 'failed',
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown',
'effect_resolved_at' => null,
'effect_resolution_reason' =>
'automatic_recovery:v1:1:1:prepared:test',
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_payload_unavailable',
]);
$preparedTemplateId = 'prepared-import-template';
$preparedGeneration = str_repeat('c', 32);
$preparedSha = str_repeat('d', 64);
Capsule::table('cloud_templates')->insert([
'id' => $preparedTemplateId,
'name' => 'Prepared import',
'os_family' => 'Debian',
'os_version' => '13',
'image_bytes' => 4096,
'sha256' => $preparedSha,
'startup_script' => '',
'revision' => 1,
'generation_token' => $preparedGeneration,
'state' => 'importing',
'active_job_id' => $preparedParentId,
'created_at' => $now - 60,
'updated_at' => $now,
]);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 7,
'template_id' => $preparedTemplateId,
'revision' => 1,
'generation_token' => $preparedGeneration,
'state' => 'failed',
'image_bytes' => 0,
'sha256' => $preparedSha,
'operation_id' => (string) CloudJob::findOrFail(
$preparedChildId
)->operation_id,
'job_id' => $preparedChildId,
'error_code' => 'aggregate_deadline_exceeded',
'created_at' => $now - 60,
'updated_at' => $now,
]);
$preparedOperation = [
'operation_id' => (string) CloudJob::findOrFail(
$preparedChildId
)->operation_id,
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
];
CloudTemplateCatalogService::finishJobLocked(
CloudJob::findOrFail($preparedChildId),
'succeeded',
[
'id' => $preparedTemplateId,
'revision' => 1,
'generation_token' => $preparedGeneration,
'size_bytes' => 4096,
'sha256' => $preparedSha,
],
['template_id' => $preparedTemplateId],
$preparedOperation,
true
);
$preparedParent = CloudJob::findOrFail($preparedParentId);
$preparedParentDisplay = CloudJobService::display(
$preparedParent,
true
);
$assert((string) $preparedParent->phase === 'awaiting_receipt'
&& (string) Capsule::table('cloud_templates')
->where('id', $preparedTemplateId)->value('state') === 'importing'
&& (string) Capsule::table('cloud_template_replicas')
->where('template_id', $preparedTemplateId)
->value('state') === 'active'
&& ($preparedParentDisplay['recovery']['pending'] ?? null) === true
&& ($preparedParentDisplay['recovery']['code'] ?? null)
=== 'recovery_payload_unavailable',
'A recovered child prepare prematurely completed or hid its parent.');
Capsule::table('cloud_jobs')->where('id', $preparedChildId)->update([
'effect_resolution' => 'applied',
'effect_resolved_at' => time(),
'effect_resolution_reason' => 'agent_inspection:test',
'finalization_incident' => 0,
'finalization_error_code' => null,
'available_at' => 0,
]);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($preparedChildId),
null
);
$preparedParent->refresh();
$assert((string) $preparedParent->phase === 'terminal'
&& (string) $preparedParent->outcome === 'succeeded'
&& (string) $preparedParent->effect_resolution === 'applied'
&& (string) Capsule::table('cloud_templates')
->where('id', $preparedTemplateId)->value('state') === 'active'
&& CloudJobService::recoveryPresentation(
$preparedParent,
true
) === null,
'The aggregate parent did not settle after the child effect commit.');
$immutableParentAudit = static fn (CloudJob $job): array => [
'phase' => (string) $job->phase,
'outcome' => $job->outcome === null ? null : (string) $job->outcome,
'status' => (string) $job->status,
'terminal_at' => $job->terminal_at === null
? null : (int) $job->terminal_at,
'error_code' => $job->error_code === null
? null : (string) $job->error_code,
'result_json' => $job->result_json === null
? null : (string) $job->result_json,
];
$markDeadlineAudit = static function (int $parentId): CloudJob {
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
'result_json' => json_encode(
['deadline_parent' => $parentId],
JSON_THROW_ON_ERROR
),
]);
return CloudJob::findOrFail($parentId);
};
// A parent deadline is an immutable failed audit, not the final catalog
// projection. Once every late child resolves, imports become usable and
// deletes become retired without rewriting that original failed task.
$deadlineImportParentId = $insertParent('template_import');
$deadlineImportParent = $markDeadlineAudit($deadlineImportParentId);
$deadlineImportAudit = $immutableParentAudit($deadlineImportParent);
$deadlineImportChildId = $insertResolvedChild(
$deadlineImportParentId,
'applied',
'template_import_replica'
);
$deadlineImportTemplateId = 'deadline-import-template';
$deadlineImportGeneration = str_repeat('5', 32);
$deadlineImportSha = str_repeat('6', 64);
Capsule::table('cloud_templates')->insert([
'id' => $deadlineImportTemplateId,
'name' => 'Deadline import',
'os_family' => 'Debian',
'os_version' => '13',
'image_bytes' => 4096,
'sha256' => $deadlineImportSha,
'startup_script' => '',
'revision' => 1,
'generation_token' => $deadlineImportGeneration,
'state' => 'error',
'active_job_id' => $deadlineImportParentId,
'created_at' => $now - 60,
'updated_at' => $now,
]);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 7,
'template_id' => $deadlineImportTemplateId,
'revision' => 1,
'generation_token' => $deadlineImportGeneration,
'state' => 'active',
'image_bytes' => 4096,
'sha256' => $deadlineImportSha,
'operation_id' => (string) CloudJob::findOrFail(
$deadlineImportChildId
)->operation_id,
'job_id' => $deadlineImportChildId,
'error_code' => '',
'created_at' => $now - 60,
'updated_at' => $now,
]);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($deadlineImportChildId),
null
);
$deadlineImportParent->refresh();
$deadlinePublicIds = array_column(
CloudTemplateCatalogService::publicCatalog()['templates'],
'id'
);
$assert($immutableParentAudit($deadlineImportParent)
=== $deadlineImportAudit
&& (string) $deadlineImportParent->effect_resolution === 'applied'
&& (string) Capsule::table('cloud_templates')
->where('id', $deadlineImportTemplateId)->value('state')
=== 'active'
&& in_array($deadlineImportTemplateId, $deadlinePublicIds, true),
'A recovered deadline import did not publish its catalog projection '
. 'without rewriting the failed task.');
$deadlineDeleteParentId = $insertParent('template_delete');
$deadlineDeleteParent = $markDeadlineAudit($deadlineDeleteParentId);
$deadlineDeleteAudit = $immutableParentAudit($deadlineDeleteParent);
$deadlineDeleteChildId = $insertResolvedChild(
$deadlineDeleteParentId,
'applied',
'template_delete_replica'
);
$deadlineDeleteTemplateId = 'deadline-delete-template';
$deadlineDeleteGeneration = str_repeat('7', 32);
$deadlineDeleteSha = str_repeat('8', 64);
Capsule::table('cloud_templates')->insert([
'id' => $deadlineDeleteTemplateId,
'name' => 'Deadline delete',
'os_family' => 'Debian',
'os_version' => '13',
'image_bytes' => 4096,
'sha256' => $deadlineDeleteSha,
'startup_script' => '',
'revision' => 1,
'generation_token' => $deadlineDeleteGeneration,
'state' => 'delete_failed',
'active_job_id' => $deadlineDeleteParentId,
'created_at' => $now - 60,
'updated_at' => $now,
]);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 7,
'template_id' => $deadlineDeleteTemplateId,
'revision' => 1,
'generation_token' => $deadlineDeleteGeneration,
'state' => 'delete_succeeded',
'image_bytes' => 4096,
'sha256' => $deadlineDeleteSha,
'operation_id' => (string) CloudJob::findOrFail(
$deadlineDeleteChildId
)->operation_id,
'job_id' => $deadlineDeleteChildId,
'error_code' => '',
'created_at' => $now - 60,
'updated_at' => $now,
]);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($deadlineDeleteChildId),
null
);
$deadlineDeleteParent->refresh();
$assert($immutableParentAudit($deadlineDeleteParent)
=== $deadlineDeleteAudit
&& (string) $deadlineDeleteParent->effect_resolution === 'unknown'
&& (bool) $deadlineDeleteParent->finalization_incident
&& (string) $deadlineDeleteParent->finalization_error_code
=== 'recovery_child_convergence'
&& (string) Capsule::table('cloud_templates')
->where('id', $deadlineDeleteTemplateId)->value('state')
=== 'delete_failed'
&& (int) Capsule::table('cloud_templates')
->where('id', $deadlineDeleteTemplateId)->value('revision') === 1
&& Capsule::table('cloud_template_replicas')
->where('template_id', $deadlineDeleteTemplateId)->exists()
&& (string) Capsule::table('cloud_template_artifacts')
->where('template_id', $deadlineDeleteTemplateId)->value('state')
=== 'delete_pending',
'A recovered deadline delete published applied before the master '
. 'object cleanup was confirmed.');
$deadlineIsoParentId = $insertParent('user_iso_import');
$deadlineIsoParent = $markDeadlineAudit($deadlineIsoParentId);
$deadlineIsoAudit = $immutableParentAudit($deadlineIsoParent);
$deadlineIsoChildId = $insertResolvedChild(
$deadlineIsoParentId,
'applied',
'user_iso_import_replica'
);
$deadlineIsoId = str_repeat('9', 32);
$deadlineIsoSha = str_repeat('a', 64);
Capsule::table('cloud_user_isos')->insert([
'id' => $deadlineIsoId,
'user_id' => 1,
'name' => 'Deadline ISO',
'sealed_source_url' => 'test-only',
'size_bytes' => 4096,
'sha256' => $deadlineIsoSha,
'state' => 'import_failed',
'import_operation_id' => (string) $deadlineIsoParent->operation_id,
'active_job_id' => $deadlineIsoParentId,
'created_at' => $now - 60,
'updated_at' => $now,
]);
Capsule::table('cloud_user_iso_replicas')->insert([
'host_id' => 7,
'user_iso_id' => $deadlineIsoId,
'user_id' => 1,
'state' => 'active',
'size_bytes' => 4096,
'sha256' => $deadlineIsoSha,
'operation_id' => (string) CloudJob::findOrFail(
$deadlineIsoChildId
)->operation_id,
'job_id' => $deadlineIsoChildId,
'error_code' => '',
'created_at' => $now - 60,
'updated_at' => $now,
]);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($deadlineIsoChildId),
null
);
$deadlineIsoParent->refresh();
$assert($immutableParentAudit($deadlineIsoParent) === $deadlineIsoAudit
&& (string) $deadlineIsoParent->effect_resolution === 'applied'
&& (string) Capsule::table('cloud_user_isos')
->where('id', $deadlineIsoId)->value('state') === 'active',
'A recovered deadline ISO import did not publish its catalog '
. 'projection without rewriting the failed task.');
$settledPartialBefore = CloudJob::findOrFail((int) $partial->id);
$settledPartialUpdatedAt = (int) $settledPartialBefore->updated_at;
$automatic->tick();
$settledPartialAfter = CloudJob::findOrFail((int) $partial->id);
$assert((int) $settledPartialAfter->updated_at
=== $settledPartialUpdatedAt
&& (int) $settledPartialAfter->available_at === 0
&& (string) $settledPartialAfter->effect_resolution === 'applied'
&& !(bool) $settledPartialAfter->finalization_incident,
'A settled partial aggregate re-entered the recovery queue.');
$assert(CloudAgentSettingsService::$requests === 0,
'Aggregate recovery contacted an Agent instead of local ledgers.');
// A corrupt replica dispatch ledger must not be interpreted as an
// undispatched/no-effect child. Doing so would fail the replica and
// release the public template aggregate even though the Agent boundary is
// unknown. Keep the catalog, child escrow and parent fence intact instead.
$invalidDispatchParentId = $insertParent('template_import');
Capsule::table('cloud_jobs')->where(
'id',
$invalidDispatchParentId
)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
'finalization_incident' => 0,
'finalization_error_code' => null,
'lease_token' => null,
'lease_until' => 0,
'available_at' => 0,
'receipt_deadline_at' => $now + 600,
]);
$invalidDispatchChildId = $insertResolvedChild(
$invalidDispatchParentId,
'not_applied',
'template_import_replica'
);
$invalidDispatchTemplateId = 'invalid-dispatch-template';
$invalidDispatchGeneration = str_repeat('e', 32);
$invalidDispatchSha = str_repeat('f', 64);
$invalidDispatchPayload = CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/templates/import',
'body' => [],
'template_id' => $invalidDispatchTemplateId,
'owner_id' => 1,
'can_manage' => true,
], 'job-payload');
Capsule::table('cloud_jobs')->where(
'id',
$invalidDispatchChildId
)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'sealed_payload' => $invalidDispatchPayload,
'attempts' => 2,
'dispatch_attempts' => 2,
'dispatch_started_at' => $now - 30,
'agent_accepted_at' => $now - 29,
'receipt_deadline_at' => $now - 1,
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null,
'dispatched_at' => $now - 29,
'lease_token' => null,
'lease_until' => 0,
'available_at' => 0,
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
Capsule::table('cloud_templates')->insert([
'id' => $invalidDispatchTemplateId,
'name' => 'Invalid dispatch template',
'os_family' => 'Debian',
'os_version' => '13',
'image_bytes' => 4096,
'sha256' => $invalidDispatchSha,
'startup_script' => '',
'revision' => 1,
'generation_token' => $invalidDispatchGeneration,
'state' => 'importing',
'active_job_id' => $invalidDispatchParentId,
'created_at' => $now - 60,
'updated_at' => $now,
]);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 7,
'template_id' => $invalidDispatchTemplateId,
'revision' => 1,
'generation_token' => $invalidDispatchGeneration,
'state' => 'importing',
'image_bytes' => 0,
'sha256' => $invalidDispatchSha,
'operation_id' => (string) CloudJob::findOrFail(
$invalidDispatchChildId
)->operation_id,
'job_id' => $invalidDispatchChildId,
'error_code' => '',
'created_at' => $now - 60,
'updated_at' => $now,
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$invalidDispatchChildId,
1
) === 1, 'The invalid template child deadline was not claimed.');
$invalidDispatchChild = CloudJob::findOrFail(
$invalidDispatchChildId
);
$invalidDispatchParent = CloudJob::findOrFail(
$invalidDispatchParentId
);
$invalidDispatchParentRecovery = CloudJobService::recoveryPresentation(
$invalidDispatchParent,
true
);
$assert((string) $invalidDispatchChild->effect_resolution === 'unknown'
&& (bool) $invalidDispatchChild->effect_may_have_applied
&& !(bool) $invalidDispatchChild->safe_to_resubmit
&& (bool) $invalidDispatchChild->finalization_incident
&& (string) $invalidDispatchChild->finalization_error_code
=== 'recovery_dispatch_ledger_conflict'
&& (string) $invalidDispatchChild->sealed_payload
=== $invalidDispatchPayload
&& (string) $invalidDispatchParent->effect_resolution
!== 'not_applied'
&& (string) Capsule::table('cloud_templates')
->where('id', $invalidDispatchTemplateId)->value('state')
=== 'importing'
&& (string) Capsule::table('cloud_template_replicas')
->where('template_id', $invalidDispatchTemplateId)
->value('state') === 'importing'
&& ($invalidDispatchParentRecovery['code'] ?? null)
=== 'recovery_dispatch_ledger_conflict',
'An invalid template child released its catalog or aggregate as no-effect.');
// Ordinary receipt finalization runs before the currently completing child
// is made terminal. That current transition is allowed, but a different
// sibling with an invalid recovery ledger must keep the public catalog and
// aggregate parent fenced. Once the sibling is repaired, the normal child
// recovery hook publishes the already-prepared replicas.
$normalGateParentId = $insertParent('template_import');
Capsule::table('cloud_jobs')->where('id', $normalGateParentId)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
'finalization_incident' => 0,
'finalization_error_code' => null,
'lease_token' => null,
'lease_until' => 0,
'available_at' => 0,
'receipt_deadline_at' => $now + 600,
]);
$normalGateSiblingId = $insertResolvedChild(
$normalGateParentId,
'applied',
'template_import_replica'
);
Capsule::table('cloud_jobs')->where(
'id',
$normalGateSiblingId
)->update(['dispatch_attempts' => 2]);
$normalGateCurrentId = $insertResolvedChild(
$normalGateParentId,
'applied',
'template_import_replica'
);
Capsule::table('cloud_jobs')->where('id', $normalGateCurrentId)->update([
'host_id' => 8,
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
]);
$normalGateTemplateId = 'normal-settle-gated-template';
$normalGateGeneration = str_repeat('1', 32);
$normalGateSha = str_repeat('2', 64);
Capsule::table('cloud_templates')->insert([
'id' => $normalGateTemplateId,
'name' => 'Normal settle gated',
'os_family' => 'Debian',
'os_version' => '13',
'image_bytes' => 8192,
'sha256' => $normalGateSha,
'startup_script' => '',
'revision' => 1,
'generation_token' => $normalGateGeneration,
'state' => 'importing',
'active_job_id' => $normalGateParentId,
'created_at' => $now - 60,
'updated_at' => $now,
]);
foreach ([
[7, $normalGateSiblingId, 'active', 8192],
[8, $normalGateCurrentId, 'importing', 0],
] as [$hostId, $jobId, $state, $imageBytes]) {
Capsule::table('cloud_template_replicas')->insert([
'host_id' => $hostId,
'template_id' => $normalGateTemplateId,
'revision' => 1,
'generation_token' => $normalGateGeneration,
'state' => $state,
'image_bytes' => $imageBytes,
'sha256' => $normalGateSha,
'operation_id' => (string) CloudJob::findOrFail($jobId)
->operation_id,
'job_id' => $jobId,
'error_code' => '',
'created_at' => $now - 60,
'updated_at' => $now,
]);
}
CloudTemplateCatalogService::finishJobLocked(
CloudJob::findOrFail($normalGateCurrentId),
'succeeded',
[
'id' => $normalGateTemplateId,
'revision' => 1,
'generation_token' => $normalGateGeneration,
'size_bytes' => 8192,
'sha256' => $normalGateSha,
],
['template_id' => $normalGateTemplateId],
[
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
]
);
$normalGateParent = CloudJob::findOrFail($normalGateParentId);
$assert((string) $normalGateParent->phase === 'awaiting_receipt'
&& (string) Capsule::table('cloud_templates')
->where('id', $normalGateTemplateId)->value('state')
=== 'importing'
&& Capsule::table('cloud_template_replicas')
->where('template_id', $normalGateTemplateId)
->where('state', 'active')->count() === 2,
'Normal completion published a parent with an unsafe sibling.');
Capsule::table('cloud_jobs')->where('id', $normalGateSiblingId)->update([
'dispatch_attempts' => 1,
]);
Capsule::table('cloud_jobs')->where('id', $normalGateCurrentId)->update([
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'terminal_at' => time(),
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'effect_resolved_at' => time(),
'effect_resolution_reason' => 'normal_completion',
]);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($normalGateSiblingId),
null
);
$normalGateParent->refresh();
$assert((string) $normalGateParent->phase === 'terminal'
&& (string) $normalGateParent->outcome === 'succeeded'
&& (string) $normalGateParent->effect_resolution === 'applied'
&& (string) Capsule::table('cloud_templates')
->where('id', $normalGateTemplateId)->value('state') === 'active',
'The recovery hook did not publish a safe repaired aggregate.');
// The normal last-child receipt must use the same strict projection gate
// as late recovery. Missing, extra or cross-wired sibling replicas may be
// projected locally, but can never publish the aggregate parent.
foreach ([
'missing sibling replica' => [
'mutation' => 'missing',
'exception' => CloudRecoveryAggregateStateConflict::class,
],
'extra sibling replica' => [
'mutation' => 'extra',
'exception' => CloudRecoveryAggregateStateConflict::class,
],
'cross-wired sibling operation' => [
'mutation' => 'mapping',
'exception' => CloudRecoveryOperationIdentityConflict::class,
],
] as $description => $fixture) {
$parentId = $insertParent('template_import');
$siblingId = $insertResolvedChild(
$parentId,
'applied',
'template_import_replica'
);
$currentId = $insertResolvedChild(
$parentId,
'applied',
'template_import_replica'
);
$resourceId = $insertProjectionForParent($parentId);
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
'lease_token' => null,
'lease_until' => 0,
'available_at' => 0,
]);
Capsule::table('cloud_jobs')->where('id', $currentId)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
]);
Capsule::table('cloud_templates')->where('id', $resourceId)->update([
'state' => 'importing',
]);
Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)
->where('job_id', $currentId)->update([
'state' => 'importing',
'image_bytes' => 0,
]);
$template = Capsule::table('cloud_templates')
->where('id', $resourceId)->first();
CloudTemplateCatalogService::finishJobLocked(
CloudJob::findOrFail($currentId),
'succeeded',
[
'id' => $resourceId,
'revision' => (int) $template->revision,
'generation_token' => (string) $template->generation_token,
'size_bytes' => (int) $template->image_bytes,
'sha256' => (string) $template->sha256,
],
['template_id' => $resourceId],
[]
);
if ($fixture['mutation'] === 'missing') {
Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)
->where('job_id', $siblingId)->delete();
} elseif ($fixture['mutation'] === 'extra') {
$extra = (array) Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)
->where('job_id', $siblingId)->first();
$extra['host_id'] = 900000 + $parentId;
$extra['job_id'] = null;
$extra['operation_id'] = md5('normal-extra:' . $parentId);
Capsule::table('cloud_template_replicas')->insert($extra);
} else {
Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)
->where('job_id', $siblingId)->update([
'operation_id' => md5('normal-cross-wire:' . $parentId),
]);
}
Capsule::table('cloud_jobs')->where('id', $currentId)->update([
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'terminal_at' => time(),
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'effect_resolved_at' => time(),
'effect_resolution_reason' => 'normal_completion',
]);
try {
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($currentId),
null
);
throw new RuntimeException(
'Normal aggregate accepted ' . $description . '.'
);
} catch (CloudError $error) {
$assert($error instanceof $fixture['exception'],
'Normal aggregate reported the wrong conflict for '
. $description . '.');
}
$assert((string) CloudJob::findOrFail($parentId)->phase
=== 'awaiting_receipt'
&& (string) Capsule::table('cloud_templates')
->where('id', $resourceId)->value('state') === 'importing',
'Normal aggregate published after ' . $description . '.');
}
// Exercise the actual ordinary completion order for all five catalog
// aggregates: project the final failed replica, persist its terminal
// no-effect conclusion, then let the shared strict reconciler restore the
// safe resource boundary.
foreach ([
'template_import' => 'template_import_replica',
'template_metadata_sync' => 'template_metadata_replica',
'template_delete' => 'template_delete_replica',
'user_iso_import' => 'user_iso_import_replica',
'user_iso_delete' => 'user_iso_delete_replica',
] as $parentKind => $childKind) {
$parentId = $insertParent($parentKind);
$childId = $insertResolvedChild(
$parentId,
'not_applied',
$childKind
);
$resourceId = $insertProjectionForParent($parentId);
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
'lease_token' => null,
'lease_until' => 0,
'available_at' => 0,
]);
Capsule::table('cloud_jobs')->where('id', $childId)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
]);
if (str_starts_with($parentKind, 'template_')) {
$resourceState = $parentKind === 'template_delete'
? 'deleting' : ($parentKind === 'template_import'
? 'importing' : 'active');
$replicaState = match ($parentKind) {
'template_import' => 'importing',
'template_metadata_sync' => 'syncing',
'template_delete' => 'deleting',
};
Capsule::table('cloud_templates')->where('id', $resourceId)
->update(['state' => $resourceState]);
Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->update([
'state' => $replicaState,
]);
$template = Capsule::table('cloud_templates')
->where('id', $resourceId)->first();
CloudTemplateCatalogService::finishJobLocked(
CloudJob::findOrFail($childId),
'failed',
null,
['template_id' => $resourceId],
[
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
'error_code' => 'operation_not_applied',
]
);
} else {
$resourceState = $parentKind === 'user_iso_delete'
? 'deleting' : 'importing';
$replicaState = $parentKind === 'user_iso_delete'
? 'deleting' : 'importing';
Capsule::table('cloud_user_isos')->where('id', $resourceId)
->update(['state' => $resourceState]);
Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $resourceId)->update([
'state' => $replicaState,
]);
CloudUserIsoService::finishJobLocked(
CloudJob::findOrFail($childId),
'failed',
null,
['user_iso_id' => $resourceId],
[
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
'error_code' => 'operation_not_applied',
]
);
}
Capsule::table('cloud_jobs')->where('id', $childId)->update([
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'terminal_at' => time(),
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 1,
'effect_resolution' => 'not_applied',
'effect_resolved_at' => time(),
'effect_resolution_reason' => 'normal_completion',
'error_code' => 'operation_rejected',
]);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($childId),
null
);
$parent = CloudJob::findOrFail($parentId);
$projectionOkay = match ($parentKind) {
'template_import' =>
(string) Capsule::table('cloud_templates')
->where('id', $resourceId)->value('state') === 'retired'
&& !Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->exists(),
'template_metadata_sync' =>
(string) Capsule::table('cloud_templates')
->where('id', $resourceId)->value('state') === 'active'
&& (string) Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->value('state')
=== 'stale',
'template_delete' =>
(string) Capsule::table('cloud_templates')
->where('id', $resourceId)->value('state') === 'active'
&& (string) Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->value('state')
=== 'active',
'user_iso_import' =>
(string) Capsule::table('cloud_user_isos')
->where('id', $resourceId)->value('state') === 'retired'
&& !Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $resourceId)->exists(),
'user_iso_delete' =>
(string) Capsule::table('cloud_user_isos')
->where('id', $resourceId)->value('state') === 'active'
&& (string) Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $resourceId)->value('state')
=== 'active',
};
if ($parentKind === 'template_metadata_sync') {
$publicIds = array_column(
CloudTemplateCatalogService::publicCatalog()['templates'],
'id'
);
$projectionOkay = $projectionOkay
&& !in_array($resourceId, $publicIds, true);
}
$assert((string) $parent->phase === 'terminal'
&& (string) $parent->outcome === 'failed'
&& (string) $parent->effect_resolution === 'not_applied'
&& !(bool) $parent->finalization_incident
&& $projectionOkay,
'Normal all-not-applied completion did not restore '
. $parentKind . '.');
}
// The ordinary last-child path must lock the one-and-only main catalog
// record before publishing its aggregate parent. A duplicate binding is
// data corruption, not permission to complete one arbitrary record.
foreach ([
'template_import' => 'template_import_replica',
'user_iso_import' => 'user_iso_import_replica',
] as $parentKind => $childKind) {
$parentId = $insertParent($parentKind);
$childId = $insertResolvedChild($parentId, 'applied', $childKind);
$resourceId = $insertProjectionForParent($parentId);
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
Capsule::table('cloud_jobs')->where('id', $childId)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
]);
if ($parentKind === 'template_import') {
Capsule::table('cloud_templates')->where('id', $resourceId)
->update(['state' => 'importing']);
Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->update([
'state' => 'importing',
'image_bytes' => 0,
]);
$duplicate = (array) Capsule::table('cloud_templates')
->where('id', $resourceId)->first();
$duplicate['id'] = $resourceId . '-normal-duplicate';
Capsule::table('cloud_templates')->insert($duplicate);
$template = Capsule::table('cloud_templates')
->where('id', $resourceId)->first();
CloudTemplateCatalogService::finishJobLocked(
CloudJob::findOrFail($childId),
'succeeded',
[
'id' => $resourceId,
'revision' => (int) $template->revision,
'generation_token' =>
(string) $template->generation_token,
'size_bytes' => (int) $template->image_bytes,
'sha256' => (string) $template->sha256,
],
['template_id' => $resourceId],
[]
);
} else {
Capsule::table('cloud_user_isos')->where('id', $resourceId)
->update(['state' => 'importing']);
Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $resourceId)->update([
'state' => 'importing',
'size_bytes' => 0,
]);
$duplicate = (array) Capsule::table('cloud_user_isos')
->where('id', $resourceId)->first();
$duplicate['id'] = md5('normal-duplicate:' . $parentId);
Capsule::table('cloud_user_isos')->insert($duplicate);
$iso = Capsule::table('cloud_user_isos')
->where('id', $resourceId)->first();
CloudUserIsoService::finishJobLocked(
CloudJob::findOrFail($childId),
'succeeded',
[
'id' => $resourceId,
'owner_id' => (int) $iso->user_id,
'name' => (string) $iso->name,
'size_bytes' => (int) $iso->size_bytes,
'sha256' => (string) $iso->sha256,
],
['user_iso_id' => $resourceId],
[]
);
}
Capsule::table('cloud_jobs')->where('id', $childId)->update([
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'terminal_at' => time(),
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'effect_resolved_at' => time(),
'effect_resolution_reason' => 'normal_completion',
]);
try {
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($childId),
null
);
throw new RuntimeException(
'Normal catalog settle accepted duplicate main records.'
);
} catch (CloudRecoveryAggregateStateConflict $expected) {
// Expected: the child projection exists, but the parent remains
// fenced until its one-and-only main record is repairable.
}
$assert((string) CloudJob::findOrFail($parentId)->phase
=== 'awaiting_receipt',
'A duplicate main record completed its normal aggregate parent.');
}
foreach ([
'template_delete' => [
CloudTemplateCatalogService::class,
'template_delete_replica',
'cloud_templates',
],
'user_iso_delete' => [
CloudUserIsoService::class,
'user_iso_delete_replica',
'cloud_user_isos',
],
] as $parentKind => [$service, $childKind, $table]) {
$parentId = $insertParent($parentKind);
$childId = $insertResolvedChild($parentId, 'applied', $childKind);
$resourceId = $insertProjectionForParent($parentId);
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
Capsule::table('cloud_jobs')->where('id', $childId)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
]);
$duplicate = (array) Capsule::table($table)
->where('id', $resourceId)->first();
$duplicate['id'] = $parentKind === 'template_delete'
? $resourceId . '-normal-duplicate'
: md5('normal-delete-duplicate:' . $parentId);
Capsule::table($table)->insert($duplicate);
$resource = $parentKind === 'template_delete'
? \App\Models\CloudTemplate::findOrFail($resourceId)
: \App\Models\CloudUserIso::findOrFail($resourceId);
try {
if ($parentKind === 'template_delete') {
CloudTemplateCatalogService::settleRecoveredParentLocked(
CloudJob::findOrFail($parentId),
'applied'
);
} else {
$settleDelete = new ReflectionMethod(
$service,
'settleDeleteParent'
);
$settleDelete->invoke(
null,
$resource,
$parentId,
['effect_may_have_applied' => true],
$childId
);
}
throw new RuntimeException(
'Normal delete settle accepted duplicate main records.'
);
} catch (CloudRecoveryAggregateStateConflict $expected) {
// Expected.
}
$assert((string) CloudJob::findOrFail($parentId)->phase
=== 'awaiting_receipt',
'A duplicate main record completed its normal delete parent.');
}
// ISO identity reconstruction must bind both import and delete children to
// the expected aggregate root, owner and active main record before any
// normal receipt can reach catalog finalization.
$expectIsoIdentityConflict = static function (
CloudJob $child,
array $payload
): void {
try {
CloudOperationRecoveryIdentity::expectedFromPayload(
$child,
$payload
);
throw new RuntimeException(
'Cross-wired ISO aggregate identity was accepted.'
);
} catch (CloudRecoveryInstanceStateConflict $expected) {
// Expected.
}
};
$identityImportParent = $insertParent('user_iso_import');
$identityImportChild = $insertResolvedChild(
$identityImportParent,
'applied',
'user_iso_import_replica'
);
$identityImportIso = $insertProjectionForParent($identityImportParent);
Capsule::table('cloud_user_isos')->where('id', $identityImportIso)
->update(['active_job_id' => $identityImportParent + 900000]);
$identityImportRow = Capsule::table('cloud_user_isos')
->where('id', $identityImportIso)->first();
$expectIsoIdentityConflict(
CloudJob::findOrFail($identityImportChild),
[
'method' => 'POST',
'request_target' => '/v1/user-isos',
'body' => [
'id' => $identityImportIso,
'owner_id' => 1,
'name' => (string) $identityImportRow->name,
],
'user_iso_id' => $identityImportIso,
'owner_id' => 1,
]
);
$identityDeleteParent = $insertParent('user_iso_delete');
$identityDeleteChild = $insertResolvedChild(
$identityDeleteParent,
'applied',
'user_iso_delete_replica'
);
$identityDeleteIso = $insertProjectionForParent($identityDeleteParent);
Capsule::table('cloud_jobs')->where('id', $identityDeleteParent)
->update(['user_id' => 2]);
$expectIsoIdentityConflict(
CloudJob::findOrFail($identityDeleteChild),
[
'method' => 'DELETE',
'request_target' => '/v1/user-isos/'
. rawurlencode($identityDeleteIso) . '?owner_id=1',
'body' => [],
'user_iso_id' => $identityDeleteIso,
'owner_id' => 1,
]
);
// A historical fail-open parent can already say succeeded/applied while a
// child remains unsafe. Such a template must disappear from the public
// catalog and be rejected by every provisioning metadata boundary until
// the child ledger is repaired.
$historicalParentId = $insertParent('template_import');
Capsule::table('cloud_jobs')->where('id', $historicalParentId)->update([
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'effect_resolved_at' => $now - 1,
'effect_resolution_reason' => 'historical_fail_open',
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
$historicalChildId = $insertResolvedChild(
$historicalParentId,
'applied',
'template_import_replica'
);
Capsule::table('cloud_jobs')->where('id', $historicalChildId)->update([
'dispatch_attempts' => 2,
]);
$historicalTemplateId = 'historical-fail-open-template';
$historicalGeneration = str_repeat('3', 32);
$historicalSha = str_repeat('4', 64);
Capsule::table('cloud_templates')->insert([
'id' => $historicalTemplateId,
'name' => 'Historical fail open',
'os_family' => 'Debian',
'os_version' => '13',
'image_bytes' => 4096,
'sha256' => $historicalSha,
'startup_script' => '',
'revision' => 1,
'generation_token' => $historicalGeneration,
'state' => 'active',
'active_job_id' => $historicalParentId,
'created_at' => $now - 60,
'updated_at' => $now,
]);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 7,
'template_id' => $historicalTemplateId,
'revision' => 1,
'generation_token' => $historicalGeneration,
'state' => 'active',
'image_bytes' => 4096,
'sha256' => $historicalSha,
'operation_id' => (string) CloudJob::findOrFail($historicalChildId)
->operation_id,
'job_id' => $historicalChildId,
'error_code' => '',
'created_at' => $now - 60,
'updated_at' => $now,
]);
$publicIds = array_column(
CloudTemplateCatalogService::publicCatalog()['templates'],
'id'
);
$assert(!in_array($historicalTemplateId, $publicIds, true),
'A fail-open template remained visible in the public catalog.');
$expectUnsafeTemplate = static function (
callable $callable,
string $message
) use ($assert): void {
try {
$callable();
} catch (CloudError $error) {
$assert($error->httpStatus === 409, $message);
return;
}
throw new RuntimeException($message);
};
$expectUnsafeTemplate(
static fn (): array =>
CloudTemplateCatalogService::applyProvisioningSnapshot([
'template_id' => $historicalTemplateId,
]),
'Provisioning consumed a fail-open template.'
);
$expectUnsafeTemplate(
static fn (): array =>
CloudTemplateCatalogService::templateMetadataSnapshot([
'template_id' => $historicalTemplateId,
'template_revision' => 1,
'template_generation_token' => $historicalGeneration,
]),
'Metadata validation consumed a fail-open template.'
);
$expectUnsafeTemplate(
static function () use ($historicalTemplateId): void {
CloudTemplateCatalogService::assertProvisionable(
$historicalTemplateId
);
},
'Provisionability accepted a fail-open template.'
);
Capsule::table('cloud_jobs')->where('id', $historicalChildId)->update([
'dispatch_attempts' => 1,
]);
$publicIds = array_column(
CloudTemplateCatalogService::publicCatalog()['templates'],
'id'
);
$snapshot = CloudTemplateCatalogService::applyProvisioningSnapshot([
'template_id' => $historicalTemplateId,
]);
$metadata = CloudTemplateCatalogService::templateMetadataSnapshot([
'template_id' => $historicalTemplateId,
'template_revision' => 1,
'template_generation_token' => $historicalGeneration,
]);
CloudTemplateCatalogService::assertProvisionable($historicalTemplateId);
$assert(in_array($historicalTemplateId, $publicIds, true)
&& ($snapshot['template_id'] ?? null) === $historicalTemplateId
&& ($metadata['template_id'] ?? null) === $historicalTemplateId,
'A safely repaired template remained unavailable.');
// Uniform not-applied evidence is a safe rollback boundary. Imports retire
// their unused placeholders; deletes restore their prior usable records;
// metadata keeps the panel revision while old host revisions remain stale.
foreach ([
'template_import' => 'template_import_replica',
'template_metadata_sync' => 'template_metadata_replica',
'template_delete' => 'template_delete_replica',
'user_iso_import' => 'user_iso_import_replica',
'user_iso_delete' => 'user_iso_delete_replica',
] as $parentKind => $childKind) {
$parentId = $insertParent($parentKind);
$childId = $insertResolvedChild(
$parentId,
'not_applied',
$childKind
);
$resourceId = $insertProjectionForParent($parentId);
$parent = $markDeadlineAudit($parentId);
$audit = $immutableParentAudit($parent);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($childId),
null
);
$parent->refresh();
$projectionOkay = match ($parentKind) {
'template_import' =>
(string) Capsule::table('cloud_templates')
->where('id', $resourceId)->value('state') === 'retired'
&& !Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->exists(),
'template_metadata_sync' =>
(string) Capsule::table('cloud_templates')
->where('id', $resourceId)->value('state') === 'active'
&& (string) Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->value('state')
=== 'stale'
&& (int) Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->value('revision')
< (int) Capsule::table('cloud_templates')
->where('id', $resourceId)->value('revision'),
'template_delete' =>
(string) Capsule::table('cloud_templates')
->where('id', $resourceId)->value('state') === 'active'
&& (string) Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->value('state')
=== 'active',
'user_iso_import' =>
(string) Capsule::table('cloud_user_isos')
->where('id', $resourceId)->value('state') === 'retired'
&& (int) Capsule::table('cloud_user_isos')
->where('id', $resourceId)->value('size_bytes') === 0
&& (string) Capsule::table('cloud_user_isos')
->where('id', $resourceId)->value('name') === ''
&& !Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $resourceId)->exists(),
'user_iso_delete' =>
(string) Capsule::table('cloud_user_isos')
->where('id', $resourceId)->value('state') === 'active'
&& (string) Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $resourceId)->value('state')
=== 'active',
};
if ($parentKind === 'template_metadata_sync') {
$publicIds = array_column(
CloudTemplateCatalogService::publicCatalog()['templates'],
'id'
);
$snapshotBlocked = false;
try {
CloudTemplateCatalogService::applyProvisioningSnapshot([
'template_id' => $resourceId,
]);
} catch (CloudError $error) {
$snapshotBlocked = $error->httpStatus === 409;
}
$assertBlocked = false;
try {
CloudTemplateCatalogService::assertProvisionable($resourceId);
} catch (CloudError $error) {
$assertBlocked = $error->httpStatus === 409;
}
$projectionOkay = $projectionOkay
&& !in_array($resourceId, $publicIds, true)
&& $snapshotBlocked
&& $assertBlocked;
}
$assert($immutableParentAudit($parent) === $audit
&& (string) $parent->effect_resolution === 'not_applied'
&& !(bool) $parent->finalization_incident
&& (int) $parent->available_at === 0
&& $projectionOkay,
'Uniform not-applied recovery did not unlock ' . $parentKind . '.');
}
$insertMixedMetadata = static function (
bool $terminal,
bool $legacy
) use (
$insertParent,
$insertResolvedChild,
$insertProjectionForParent,
$makeLegacyEligible,
$now
): array {
$parentId = $insertParent('template_metadata_sync');
if (!$terminal) {
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
'lease_token' => null,
'lease_until' => 0,
'available_at' => $now,
]);
} elseif ($legacy) {
$makeLegacyEligible($parentId);
}
$appliedId = $insertResolvedChild(
$parentId,
'applied',
'template_metadata_replica'
);
$notAppliedId = $insertResolvedChild(
$parentId,
'not_applied',
'template_metadata_replica'
);
$resourceId = $insertProjectionForParent($parentId);
return [$parentId, $appliedId, $notAppliedId, $resourceId];
};
$assertMixedMetadata = static function (
int $parentId,
string $resourceId,
string $context
) use ($assert): void {
$parent = CloudJob::findOrFail($parentId);
$replicas = Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->orderBy('host_id')->get();
$publicIds = array_column(
CloudTemplateCatalogService::publicCatalog()['templates'],
'id'
);
$snapshot = CloudTemplateCatalogService::applyProvisioningSnapshot([
'template_id' => $resourceId,
]);
CloudTemplateCatalogService::assertProvisionable($resourceId);
$assert((string) $parent->phase === 'terminal'
&& (string) $parent->outcome === 'failed'
&& (string) $parent->effect_resolution === 'applied'
&& !(bool) $parent->finalization_incident
&& (string) Capsule::table('cloud_templates')
->where('id', $resourceId)->value('state') === 'active'
&& $replicas->count() === 2
&& $replicas->contains(static fn ($replica): bool =>
(string) $replica->state === 'active'
&& (int) $replica->revision === 2)
&& $replicas->contains(static fn ($replica): bool =>
(string) $replica->state === 'stale'
&& (int) $replica->revision === 1)
&& in_array($resourceId, $publicIds, true)
&& ($snapshot['template_id'] ?? null) === $resourceId,
'Mixed metadata did not converge at the ' . $context . ' seam.');
};
[$realtimeMetadataParent, $realtimeMetadataApplied,
$_realtimeMetadataAbsent, $realtimeMetadataResource] =
$insertMixedMetadata(false, false);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($realtimeMetadataApplied),
null
);
$assertMixedMetadata(
$realtimeMetadataParent,
$realtimeMetadataResource,
'realtime'
);
[$terminalMetadataParent, $terminalMetadataApplied,
$_terminalMetadataAbsent, $terminalMetadataResource] =
$insertMixedMetadata(true, false);
$terminalMetadataAudit = $immutableParentAudit(
$markDeadlineAudit($terminalMetadataParent)
);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($terminalMetadataApplied),
null
);
$assertMixedMetadata(
$terminalMetadataParent,
$terminalMetadataResource,
'terminal deadline'
);
$assert($immutableParentAudit(
CloudJob::findOrFail($terminalMetadataParent)
) === $terminalMetadataAudit,
'Mixed metadata recovery rewrote the deadline parent audit.');
[$legacyMetadataParent, $_legacyMetadataApplied,
$_legacyMetadataAbsent, $legacyMetadataResource] =
$insertMixedMetadata(true, true);
$assert(CloudJobService::reconcileOneLegacyAggregateEffect(),
'Legacy mixed metadata was not inspected.');
$assertMixedMetadata(
$legacyMetadataParent,
$legacyMetadataResource,
'legacy sweep'
);
// Historical partial imports/deletes use the same evidence-only exit. The
// legacy sweep must publish the mixed result and release the catalog
// pointer; it must not leave upgraded installations permanently fenced.
foreach ([
'template_import' => 'template_import_replica',
'template_delete' => 'template_delete_replica',
'user_iso_import' => 'user_iso_import_replica',
'user_iso_delete' => 'user_iso_delete_replica',
] as $parentKind => $childKind) {
$legacyPartialId = $insertParent($parentKind);
$makeLegacyEligible($legacyPartialId);
$insertResolvedChild($legacyPartialId, 'applied', $childKind);
$insertResolvedChild($legacyPartialId, 'not_applied', $childKind);
$legacyPartialResourceId = $insertProjectionForParent(
$legacyPartialId
);
$assert(CloudJobService::reconcileOneLegacyAggregateEffect(),
'Legacy partial aggregate was not inspected: ' . $parentKind);
$legacyPartial = CloudJob::findOrFail($legacyPartialId);
$resource = str_starts_with($parentKind, 'template_')
? Capsule::table('cloud_templates')
->where('id', $legacyPartialResourceId)->first()
: Capsule::table('cloud_user_isos')
->where('id', $legacyPartialResourceId)->first();
$expectedState = str_ends_with($parentKind, '_import')
? (str_starts_with($parentKind, 'template_')
? 'degraded' : 'import_failed')
: 'delete_failed';
$assert((string) $legacyPartial->effect_resolution === 'applied'
&& str_ends_with(
(string) $legacyPartial->effect_resolution_reason,
':mixed'
)
&& !(bool) $legacyPartial->finalization_incident
&& (int) $legacyPartial->available_at === 0
&& $resource !== null
&& (string) $resource->state === $expectedState
&& $resource->active_job_id === null,
'Legacy partial aggregate stayed fenced: ' . $parentKind);
}
// An all-applied import can exceed quota only after host work finished.
// Keep it unavailable, expose an explicit cleanup capability, and reserve
// a brand-new delete aggregate instead of replaying an import child.
$quotaExistingIsoId = md5('quota-existing-active-iso');
Capsule::table('cloud_user_isos')->insert([
'id' => $quotaExistingIsoId,
'user_id' => 1,
'name' => 'Existing quota usage',
'sealed_source_url' => 'test-only',
'size_bytes' => 6 * 1024 * 1024 * 1024,
'sha256' => hash('sha256', 'existing-quota-iso'),
'state' => 'active',
'import_operation_id' => md5('existing-quota-operation'),
'active_job_id' => null,
'created_at' => $now - 120,
'updated_at' => $now,
]);
$quotaParentId = $insertParent('user_iso_import');
Capsule::table('cloud_jobs')->where('id', $quotaParentId)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
'lease_token' => null,
'lease_until' => 0,
'available_at' => $now,
'receipt_deadline_at' => $now + 600,
]);
$quotaChildId = $insertResolvedChild(
$quotaParentId,
'applied',
'user_iso_import_replica'
);
$quotaIsoId = $insertProjectionForParent($quotaParentId);
$quotaRecoveredBytes = 5 * 1024 * 1024 * 1024;
Capsule::table('cloud_user_isos')->where('id', $quotaIsoId)->update([
'size_bytes' => $quotaRecoveredBytes,
]);
Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $quotaIsoId)->update([
'size_bytes' => $quotaRecoveredBytes,
]);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($quotaChildId),
null
);
$quotaParent = CloudJob::findOrFail($quotaParentId);
$quotaIso = Capsule::table('cloud_user_isos')
->where('id', $quotaIsoId)->first();
$quotaDto = collect(CloudUserIsoService::listing(1)['isos'])
->firstWhere('id', $quotaIsoId);
$assert((string) $quotaParent->phase === 'terminal'
&& (string) $quotaParent->outcome === 'failed'
&& (string) $quotaParent->error_code
=== 'user_iso_quota_exceeded'
&& (string) $quotaParent->effect_resolution === 'applied'
&& !(bool) $quotaParent->finalization_incident
&& $quotaIso !== null
&& (string) $quotaIso->state === 'import_failed'
&& is_array($quotaDto)
&& ($quotaDto['can_delete'] ?? false) === true
&& ($quotaDto['cleanup_reason'] ?? null)
=== 'quota_exceeded_after_import',
'Quota-rejected applied ISO import was not exposed for safe cleanup.');
$cleanup = CloudUserIsoService::delete($quotaIsoId, ['user_id' => 1]);
$cleanupParentId = (int) ($cleanup['task_id'] ?? 0);
$cleanupParent = CloudJob::findOrFail($cleanupParentId);
$quotaIso = Capsule::table('cloud_user_isos')
->where('id', $quotaIsoId)->first();
$cleanupChild = CloudJob::where('parent_job_id', $cleanupParentId)->first();
$assert($cleanupParentId > 0 && $cleanupParentId !== $quotaParentId
&& (string) $cleanupParent->kind === 'user_iso_delete'
&& (string) $cleanupParent->phase === 'awaiting_receipt'
&& $quotaIso !== null
&& (string) $quotaIso->state === 'deleting'
&& (int) $quotaIso->active_job_id === $cleanupParentId
&& $cleanupChild !== null
&& (string) $cleanupChild->kind === 'user_iso_delete_replica'
&& (string) Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $quotaIsoId)->value('state') === 'deleting',
'Quota cleanup did not reserve an independent delete operation.');
$nextId = max(
$nextId,
(int) Capsule::table('cloud_jobs')->max('id') + 1
);
// A deadline keeps its immutable aggregate_deadline_exceeded audit even
// when every late host later proves the import applied. If quota prevented
// publication, that exact all-active projection must still be removable
// by a new delete operation without relying on the parent's old error.
$deadlineQuotaParentId = $insertParent('user_iso_import');
$deadlineQuotaAudit = $immutableParentAudit(
$markDeadlineAudit($deadlineQuotaParentId)
);
$deadlineQuotaChildId = $insertResolvedChild(
$deadlineQuotaParentId,
'applied',
'user_iso_import_replica'
);
$deadlineQuotaIsoId = $insertProjectionForParent(
$deadlineQuotaParentId
);
Capsule::table('cloud_user_isos')->where(
'id',
$deadlineQuotaIsoId
)->update(['size_bytes' => $quotaRecoveredBytes]);
Capsule::table('cloud_user_iso_replicas')->where(
'user_iso_id',
$deadlineQuotaIsoId
)->update(['size_bytes' => $quotaRecoveredBytes]);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($deadlineQuotaChildId),
null
);
$deadlineQuotaParent = CloudJob::findOrFail($deadlineQuotaParentId);
$deadlineQuotaDto = collect(CloudUserIsoService::listing(1)['isos'])
->firstWhere('id', $deadlineQuotaIsoId);
$assert($immutableParentAudit($deadlineQuotaParent)
=== $deadlineQuotaAudit
&& (string) $deadlineQuotaParent->error_code
=== 'aggregate_deadline_exceeded'
&& (string) $deadlineQuotaParent->effect_resolution === 'applied'
&& !(bool) $deadlineQuotaParent->finalization_incident
&& (string) Capsule::table('cloud_user_isos')
->where('id', $deadlineQuotaIsoId)->value('state')
=== 'import_failed'
&& is_array($deadlineQuotaDto)
&& ($deadlineQuotaDto['can_delete'] ?? false) === true
&& ($deadlineQuotaDto['cleanup_reason'] ?? null)
=== 'quota_exceeded_after_import',
'Deadline quota recovery lost its audit or safe cleanup capability.');
// Projection failure rolls back atomically and delays only that parent. A
// later valid parent is claimed by the next tick instead of being starved.
$missingCatalogParentId = $insertParent('template_import');
$missingCatalogChildId = $insertResolvedChild(
$missingCatalogParentId,
'applied',
'template_import_replica'
);
$makeLegacyEligible($missingCatalogParentId);
$missingCatalogAudit = $immutableParentAudit(
CloudJob::findOrFail($missingCatalogParentId)
);
$validAfterDamageParentId = $insertParent('template_import');
$insertResolvedChild(
$validAfterDamageParentId,
'applied',
'template_import_replica'
);
$validAfterDamageResource = $insertProjectionForParent(
$validAfterDamageParentId
);
$makeLegacyEligible($validAfterDamageParentId);
$assert($automatic->tick(),
'The damaged projection was not isolated by the legacy sweep.');
$missingCatalogParent = CloudJob::findOrFail($missingCatalogParentId);
$assert($immutableParentAudit($missingCatalogParent)
=== $missingCatalogAudit
&& (string) $missingCatalogParent->effect_resolution === 'unknown'
&& (bool) $missingCatalogParent->finalization_incident
&& (string) $missingCatalogParent->finalization_error_code
=== 'recovery_instance_state_conflict'
&& (int) $missingCatalogParent->available_at >= time() + 290
&& CloudJob::findOrFail($missingCatalogChildId)->effect_resolution
=== 'applied',
'A missing catalog escaped or rewrote the terminal parent audit.');
$assert($automatic->tick(),
'A damaged projection starved the following valid aggregate.');
$validAfterDamageParent = CloudJob::findOrFail(
$validAfterDamageParentId
);
$assert((string) $validAfterDamageParent->effect_resolution === 'applied'
&& !(bool) $validAfterDamageParent->finalization_incident
&& (string) Capsule::table('cloud_templates')
->where('id', $validAfterDamageResource)->value('state')
=== 'active',
'The valid aggregate after a damaged parent did not converge.');
$projectionConflictFixtures = [];
foreach ([
'missing replica' => 'missing',
'extra replica' => 'extra',
'residual importing replica' => 'pending',
'cross-wired operation' => 'mapping',
] as $description => $mutation) {
$parentId = $insertParent('template_import');
$insertResolvedChild(
$parentId,
'applied',
'template_import_replica'
);
$resourceId = $insertProjectionForParent($parentId);
if ($mutation === 'missing') {
Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->delete();
} elseif ($mutation === 'extra') {
$extra = (array) Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->first();
$extra['host_id'] = 999000 + $parentId;
$extra['job_id'] = null;
$extra['operation_id'] = md5('extra-replica:' . $parentId);
Capsule::table('cloud_template_replicas')->insert($extra);
} elseif ($mutation === 'pending') {
Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)
->update(['state' => 'importing']);
} else {
Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->update([
'operation_id' => md5('wrong-operation:' . $parentId),
]);
}
$makeLegacyEligible($parentId);
$projectionConflictFixtures[$parentId] = [
'description' => $description,
'code' => in_array($mutation, ['missing', 'extra'], true)
? 'recovery_aggregate_state_conflict'
: ($mutation === 'mapping'
? 'recovery_operation_identity_conflict'
: 'recovery_instance_state_conflict'),
];
}
foreach ([
'template generation mismatch' => 'generation',
'template import content mismatch' => 'content',
] as $description => $mutation) {
$parentId = $insertParent('template_import');
$insertResolvedChild(
$parentId,
'applied',
'template_import_replica'
);
$resourceId = $insertProjectionForParent($parentId);
Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->update(
$mutation === 'generation'
? ['generation_token' => str_repeat('f', 32)]
: [
'image_bytes' => 1,
'sha256' => str_repeat('0', 64),
]
);
$makeLegacyEligible($parentId);
$projectionConflictFixtures[$parentId] = [
'description' => $description,
'code' => 'recovery_instance_state_conflict',
];
}
$metadataContentParent = $insertParent('template_metadata_sync');
$insertResolvedChild(
$metadataContentParent,
'applied',
'template_metadata_replica'
);
$insertResolvedChild(
$metadataContentParent,
'not_applied',
'template_metadata_replica'
);
$metadataContentResource = $insertProjectionForParent(
$metadataContentParent
);
Capsule::table('cloud_template_replicas')
->where('template_id', $metadataContentResource)
->where('state', 'active')->update([
'image_bytes' => 1,
'sha256' => str_repeat('0', 64),
]);
$makeLegacyEligible($metadataContentParent);
$projectionConflictFixtures[$metadataContentParent] = [
'description' => 'template metadata mixed content mismatch',
'code' => 'recovery_instance_state_conflict',
];
$templateDeleteContentParent = $insertParent('template_delete');
$insertResolvedChild(
$templateDeleteContentParent,
'not_applied',
'template_delete_replica'
);
$templateDeleteContentResource = $insertProjectionForParent(
$templateDeleteContentParent
);
Capsule::table('cloud_template_replicas')
->where('template_id', $templateDeleteContentResource)->update([
'image_bytes' => 1,
'sha256' => str_repeat('0', 64),
]);
$makeLegacyEligible($templateDeleteContentParent);
$projectionConflictFixtures[$templateDeleteContentParent] = [
'description' => 'template delete not-applied content mismatch',
'code' => 'recovery_instance_state_conflict',
];
foreach ([
'ISO import parent operation mismatch' => 'parent_operation',
'ISO import content mismatch' => 'content',
'ISO active job cross-user binding' => 'user',
] as $description => $mutation) {
$parentId = $insertParent('user_iso_import');
$insertResolvedChild(
$parentId,
'applied',
'user_iso_import_replica'
);
$resourceId = $insertProjectionForParent($parentId);
if ($mutation === 'parent_operation') {
Capsule::table('cloud_user_isos')->where('id', $resourceId)
->update([
'import_operation_id' => md5(
'wrong-import-parent:' . $parentId
),
]);
} elseif ($mutation === 'content') {
Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $resourceId)->update([
'size_bytes' => 1,
'sha256' => str_repeat('0', 64),
]);
} else {
Capsule::table('cloud_user_isos')->where('id', $resourceId)
->update(['user_id' => 2]);
}
$makeLegacyEligible($parentId);
$projectionConflictFixtures[$parentId] = [
'description' => $description,
'code' => in_array(
$mutation,
['parent_operation', 'user'],
true
)
? 'recovery_operation_identity_conflict'
: 'recovery_instance_state_conflict',
];
}
$isoDeleteContentParent = $insertParent('user_iso_delete');
$insertResolvedChild(
$isoDeleteContentParent,
'not_applied',
'user_iso_delete_replica'
);
$isoDeleteContentResource = $insertProjectionForParent(
$isoDeleteContentParent
);
Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $isoDeleteContentResource)->update([
'size_bytes' => 1,
'sha256' => str_repeat('0', 64),
]);
$makeLegacyEligible($isoDeleteContentParent);
$projectionConflictFixtures[$isoDeleteContentParent] = [
'description' => 'ISO delete not-applied content mismatch',
'code' => 'recovery_instance_state_conflict',
];
foreach ([
'template duplicate main binding' => 'template_import',
'ISO duplicate main binding' => 'user_iso_import',
] as $description => $parentKind) {
$childKind = $parentKind === 'template_import'
? 'template_import_replica' : 'user_iso_import_replica';
$parentId = $insertParent($parentKind);
$insertResolvedChild($parentId, 'applied', $childKind);
$resourceId = $insertProjectionForParent($parentId);
if ($parentKind === 'template_import') {
$duplicate = (array) Capsule::table('cloud_templates')
->where('id', $resourceId)->first();
$duplicate['id'] = $resourceId . '-duplicate';
$duplicate['state'] = 'error';
Capsule::table('cloud_templates')->insert($duplicate);
} else {
$duplicate = (array) Capsule::table('cloud_user_isos')
->where('id', $resourceId)->first();
$duplicate['id'] = md5('duplicate-main:' . $parentId);
$duplicate['state'] = 'retired';
Capsule::table('cloud_user_isos')->insert($duplicate);
}
$makeLegacyEligible($parentId);
$projectionConflictFixtures[$parentId] = [
'description' => $description,
'code' => 'recovery_aggregate_state_conflict',
];
}
foreach ([
'template mixed effects forged all-success' => [
'parent' => 'template_import',
'child' => 'template_import_replica',
],
'ISO delete mixed effects forged all-success' => [
'parent' => 'user_iso_delete',
'child' => 'user_iso_delete_replica',
],
] as $description => $kinds) {
$parentId = $insertParent($kinds['parent']);
$insertResolvedChild($parentId, 'applied', $kinds['child']);
$insertResolvedChild($parentId, 'not_applied', $kinds['child']);
$resourceId = $insertProjectionForParent($parentId);
if ($kinds['parent'] === 'template_import') {
Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->update([
'state' => 'active',
'image_bytes' => 4096,
]);
} else {
Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $resourceId)->update([
'state' => 'delete_succeeded',
]);
}
$makeLegacyEligible($parentId);
$projectionConflictFixtures[$parentId] = [
'description' => $description,
'code' => 'recovery_instance_state_conflict',
];
}
foreach ($projectionConflictFixtures as $parentId => $expectation) {
$assert($automatic->tick(),
'The projection conflict was not inspected: '
. $expectation['description']);
$parent = CloudJob::findOrFail($parentId);
$assert((string) $parent->effect_resolution === 'unknown'
&& (bool) $parent->finalization_incident
&& (string) $parent->finalization_error_code
=== $expectation['code']
&& (int) $parent->available_at >= time() + 290,
'The projection conflict escaped its recovery fence: '
. $expectation['description']);
}
$assert(CloudAgentSettingsService::$requests === 0,
'Aggregate projection recovery replayed an Agent mutation.');
// Published parents with no pre-existing incident are still audited while
// they remain the catalog's current active_job_id. Damage introduced
// after the original settlement must become a sticky local incident; the
// immutable parent/child audit is not rewritten and Agent is never called.
foreach ([
'mapping' => 'recovery_operation_identity_conflict',
'content' => 'recovery_instance_state_conflict',
'state' => 'recovery_instance_state_conflict',
] as $mutation => $expectedCode) {
$parentId = $insertParent('template_import');
$childId = $insertResolvedChild(
$parentId,
'applied',
'template_import_replica'
);
$resourceId = $insertProjectionForParent($parentId);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($childId),
null
);
$publishedAudit = $immutableParentAudit(
CloudJob::findOrFail($parentId)
);
$damage = match ($mutation) {
'mapping' => [
'operation_id' => md5('published-wrong-op:' . $parentId),
],
'content' => [
'image_bytes' => 1,
'sha256' => str_repeat('0', 64),
],
'state' => ['state' => 'stale'],
};
Capsule::table('cloud_template_replicas')
->where('template_id', $resourceId)->update($damage);
Capsule::table('cloud_jobs')->where('id', '<>', $parentId)->update([
'available_at' => time() + 3600,
'lease_until' => time() + 3600,
]);
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
'available_at' => 0,
'lease_until' => 0,
]);
$beforePublishedAuditRequests = CloudAgentSettingsService::$requests;
$assert($automatic->tick(),
'A current published projection was not audited: ' . $mutation);
$damaged = CloudJob::findOrFail($parentId);
$assert($immutableParentAudit($damaged) === $publishedAudit
&& (string) $damaged->effect_resolution === 'applied'
&& (bool) $damaged->finalization_incident
&& (string) $damaged->finalization_error_code === $expectedCode
&& (int) $damaged->available_at >= time() + 290
&& CloudAgentSettingsService::$requests
=== $beforePublishedAuditRequests,
'Published projection damage rewrote audit, disappeared, or contacted Agent: '
. $mutation);
}
// A resolved child in a nonterminal phase is a PHP-ledger contradiction,
// not ordinary child convergence. It stays in the same 300-second local
// loop with the concrete aggregate-state code.
$phaseConflictParentId = $insertParent('template_import');
$phaseConflictChildId = $insertResolvedChild(
$phaseConflictParentId,
'applied',
'template_import_replica'
);
$insertProjectionForParent($phaseConflictParentId);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($phaseConflictChildId),
null
);
Capsule::table('cloud_jobs')->where('id', $phaseConflictChildId)->update([
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'terminal_at' => null,
]);
Capsule::table('cloud_jobs')->where(
'id',
'<>',
$phaseConflictParentId
)->update([
'available_at' => time() + 3600,
'lease_until' => time() + 3600,
]);
Capsule::table('cloud_jobs')->where('id', $phaseConflictParentId)->update([
'available_at' => 0,
'lease_until' => 0,
]);
$phaseBefore = CloudJobService::recoveryPresentation(
CloudJob::findOrFail($phaseConflictParentId),
true
);
$beforePhaseRequests = CloudAgentSettingsService::$requests;
$assert(($phaseBefore['code'] ?? null)
=== 'recovery_aggregate_state_conflict'
&& $automatic->tick(),
'A nonterminal child with a final effect was not retained locally.');
$phaseConflict = CloudJob::findOrFail($phaseConflictParentId);
$assert((bool) $phaseConflict->finalization_incident
&& (string) $phaseConflict->finalization_error_code
=== 'recovery_aggregate_state_conflict'
&& (int) $phaseConflict->available_at >= time() + 290
&& CloudAgentSettingsService::$requests === $beforePhaseRequests,
'A nonterminal/final child lost its concrete conflict or contacted Agent.');
// Ordinary leaf recovery has priority over healthy periodic catalog audit,
// and a successful audit advances available_at so it cannot hot-loop.
$healthyParentId = $insertParent('template_import');
$healthyChildId = $insertResolvedChild(
$healthyParentId,
'applied',
'template_import_replica'
);
$insertProjectionForParent($healthyParentId);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($healthyChildId),
null
);
$priorityLeafId = $insertResolvedChild(
0,
'applied',
'template_import_replica'
);
Capsule::table('cloud_jobs')->where('id', $priorityLeafId)->update([
'dispatch_attempts' => 0,
'dispatched_at' => null,
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown',
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
Capsule::table('cloud_jobs')->update([
'available_at' => time() + 3600,
'lease_until' => time() + 3600,
]);
Capsule::table('cloud_jobs')->whereIn('id', [
$healthyParentId,
$priorityLeafId,
])->update(['available_at' => 0, 'lease_until' => 0]);
$assert($automatic->tick(),
'The ordinary invalid leaf was not preferred over healthy catalog audit.');
$priorityLeaf = CloudJob::findOrFail($priorityLeafId);
$healthyBeforeAudit = CloudJob::findOrFail($healthyParentId);
$assert((string) $priorityLeaf->finalization_error_code
=== 'recovery_instance_state_conflict'
&& !(bool) $healthyBeforeAudit->finalization_incident
&& (int) $healthyBeforeAudit->available_at === 0,
'Healthy aggregate audit starved an ordinary leaf recovery claim.');
$assert($automatic->tick(),
'The healthy current aggregate was not periodically audited.');
$healthyAfterAudit = CloudJob::findOrFail($healthyParentId);
$assert(!(bool) $healthyAfterAudit->finalization_incident
&& (int) $healthyAfterAudit->available_at >= time() + 290,
'A healthy published aggregate audit hot-looped or created an incident.');
// A success receipt arriving after the immutable deadline failure must
// converge the original delete generation in place. It may update the
// effect/catalog projection, but it must not flip either failed audit to
// succeeded or contact the destructive Agent endpoint again.
$lateDeleteParentId = $insertParent('template_delete');
$lateDeleteChildId = $insertResolvedChild(
$lateDeleteParentId,
'not_applied',
'template_delete_replica'
);
$lateDeleteTemplateId = $insertProjectionForParent(
$lateDeleteParentId
);
$lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId);
Capsule::table('cloud_jobs')->where('id', $lateDeleteChildId)->update([
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'DELETE',
'request_target' => '/v1/templates/'
. rawurlencode($lateDeleteTemplateId),
'body' => [
'operation_id' => (string) $lateDeleteChild->operation_id,
],
'owner_id' => 1,
'can_manage' => true,
'template_id' => $lateDeleteTemplateId,
], 'job-payload'),
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null,
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_receipt_invalid',
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId);
$lateDeleteIdentity = CloudOperationRecoveryIdentity::expected(
$lateDeleteChild,
true
);
$beforeLateDeleteRequests = CloudAgentSettingsService::$requests;
$lateDeleteAccepted = CloudOperationCompletionService::accept([
'type' => 'completion',
'id' => 'aggregate-late-template-delete',
'agent_id' => (string) $lateDeleteChild->host_id,
'operation_id' => (string) $lateDeleteChild->operation_id,
'action' => $lateDeleteIdentity['action'],
'resource_id' => $lateDeleteIdentity['resource_id'],
'request_fingerprint' =>
$lateDeleteIdentity['request_fingerprint'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'resource_type' => 'template',
'id' => $lateDeleteTemplateId,
'deleted' => true,
'managed_image_absent' => true,
'managed_metadata_absent' => true,
'managed_artifacts_absent' => true,
],
]);
$lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId);
$lateDeleteParent = CloudJob::findOrFail($lateDeleteParentId);
$assert($lateDeleteAccepted === [
'accepted' => true,
'disposition' => 'applied',
]
&& (string) $lateDeleteChild->outcome === 'failed'
&& (string) $lateDeleteChild->effect_resolution === 'applied'
&& !(bool) $lateDeleteChild->finalization_incident
&& (string) $lateDeleteParent->outcome === 'failed'
&& (string) $lateDeleteParent->error_code
=== 'aggregate_deadline_exceeded'
&& (string) $lateDeleteParent->effect_resolution === 'unknown'
&& (bool) $lateDeleteParent->finalization_incident
&& (string) $lateDeleteParent->finalization_error_code
=== 'recovery_child_convergence'
&& (string) Capsule::table('cloud_templates')
->where('id', $lateDeleteTemplateId)->value('state')
=== 'delete_failed'
&& Capsule::table('cloud_template_replicas')
->where('template_id', $lateDeleteTemplateId)->exists()
&& (string) Capsule::table('cloud_template_artifacts')
->where('template_id', $lateDeleteTemplateId)->value('state')
=== 'delete_pending'
&& CloudAgentSettingsService::$requests === $beforeLateDeleteRequests,
'A late template deletion receipt did not converge the original '
. 'host operation into the two-phase master cleanup fence.');
$lateIsoParentId = $insertParent('user_iso_delete');
$lateIsoChildId = $insertResolvedChild(
$lateIsoParentId,
'not_applied',
'user_iso_delete_replica'
);
$lateIsoId = $insertProjectionForParent($lateIsoParentId);
$lateIsoChild = CloudJob::findOrFail($lateIsoChildId);
Capsule::table('cloud_jobs')->where('id', $lateIsoChildId)->update([
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'DELETE',
'request_target' => '/v1/user-isos/'
. rawurlencode($lateIsoId) . '?owner_id=1',
'body' => [
'operation_id' => (string) $lateIsoChild->operation_id,
],
'owner_id' => 1,
'can_manage' => false,
'user_iso_id' => $lateIsoId,
], 'job-payload'),
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null,
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_receipt_invalid',
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$lateIsoChild = CloudJob::findOrFail($lateIsoChildId);
$lateIsoIdentity = CloudOperationRecoveryIdentity::expected(
$lateIsoChild,
true
);
$lateIsoAccepted = CloudOperationCompletionService::accept([
'type' => 'completion',
'id' => 'aggregate-late-user-iso-delete',
'agent_id' => (string) $lateIsoChild->host_id,
'operation_id' => (string) $lateIsoChild->operation_id,
'action' => $lateIsoIdentity['action'],
'resource_id' => $lateIsoIdentity['resource_id'],
'request_fingerprint' => $lateIsoIdentity['request_fingerprint'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'resource_type' => 'user_iso',
'id' => $lateIsoId,
'owner_id' => 1,
'deleted' => true,
'managed_image_absent' => true,
'managed_metadata_absent' => true,
'managed_artifacts_absent' => true,
],
]);
$lateIsoChild = CloudJob::findOrFail($lateIsoChildId);
$lateIsoParent = CloudJob::findOrFail($lateIsoParentId);
$lateIso = Capsule::table('cloud_user_isos')
->where('id', $lateIsoId)->first();
$assert($lateIsoAccepted === [
'accepted' => true,
'disposition' => 'applied',
]
&& (string) $lateIsoChild->outcome === 'failed'
&& (string) $lateIsoChild->effect_resolution === 'applied'
&& !(bool) $lateIsoChild->finalization_incident
&& (string) $lateIsoParent->outcome === 'failed'
&& (string) $lateIsoParent->effect_resolution === 'applied'
&& !(bool) $lateIsoParent->finalization_incident
&& $lateIso !== null
&& (string) $lateIso->state === 'retired'
&& (string) $lateIso->name === ''
&& (int) $lateIso->size_bytes === 0
&& !Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $lateIsoId)->exists()
&& CloudAgentSettingsService::$requests === $beforeLateDeleteRequests,
'A late user-ISO deletion receipt did not converge its owner-bound '
. 'catalog ledgers without replay.');
echo "Cloud aggregate recovery convergence tests passed.\n";
}