You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
3788 lines
155 KiB
PHP
3788 lines
155 KiB
PHP
<?php
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Services\Cloud {
|
|
/** In-memory envelope for this focused completion/convergence contract. */
|
|
final class CloudSecrets
|
|
{
|
|
private static array $values = [];
|
|
|
|
public static function encrypt(array $value, string $purpose): string
|
|
{
|
|
$key = bin2hex(random_bytes(16));
|
|
self::$values[$key] = $value;
|
|
return 'aggregate-convergence-test:' . $key;
|
|
}
|
|
|
|
public static function decrypt(string $value, string $purpose): array
|
|
{
|
|
$key = substr($value, strlen('aggregate-convergence-test:'));
|
|
if (!array_key_exists($key, self::$values)) {
|
|
throw new \RuntimeException('Missing aggregate test payload.');
|
|
}
|
|
return self::$values[$key];
|
|
}
|
|
}
|
|
|
|
final class CloudAgentSettingsService
|
|
{
|
|
public static int $requests = 0;
|
|
|
|
public static function clientForConfiguration(
|
|
int $hostId,
|
|
float $timeoutSeconds = 120.0
|
|
): never {
|
|
self::$requests++;
|
|
throw new \RuntimeException(
|
|
'Aggregate recovery must not contact an Agent.'
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudJob;
|
|
use App\Services\Cloud\CloudAgentSettingsService;
|
|
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudJobService;
|
|
use App\Services\Cloud\CloudOperationCompletionService;
|
|
use App\Services\Cloud\CloudOperationRecoveryIdentity;
|
|
use App\Services\Cloud\CloudRecoveryAggregateStateConflict;
|
|
use App\Services\Cloud\CloudRecoveryInstanceStateConflict;
|
|
use App\Services\Cloud\CloudRecoveryOperationIdentityConflict;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use App\Services\Cloud\CloudTemplateCatalogService;
|
|
use App\Services\Cloud\CloudUserIsoService;
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
|
|
$automaticSource = file_get_contents(
|
|
dirname(__DIR__)
|
|
. '/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php'
|
|
);
|
|
$lockMethodStart = is_string($automaticSource)
|
|
? strpos($automaticSource, 'private static function resolveFenceAndCommit(')
|
|
: false;
|
|
$lockMethodEnd = $lockMethodStart === false ? false : strpos(
|
|
$automaticSource,
|
|
'private static function assertFreshInspection(',
|
|
$lockMethodStart
|
|
);
|
|
$lockMethod = $lockMethodStart === false || $lockMethodEnd === false
|
|
? '' : substr(
|
|
$automaticSource,
|
|
$lockMethodStart,
|
|
$lockMethodEnd - $lockMethodStart
|
|
);
|
|
$parentLock = strpos(
|
|
$lockMethod,
|
|
'$aggregateParent = CloudJob::where('
|
|
);
|
|
$catalogLock = strpos(
|
|
$lockMethod,
|
|
'CloudTemplateCatalogService::lockAggregateCatalogLocked('
|
|
);
|
|
$leafLock = strpos(
|
|
$lockMethod,
|
|
'$aggregateChildren = CloudJob::where('
|
|
);
|
|
$assert(str_contains($lockMethod, "->value('parent_job_id')")
|
|
&& $parentLock !== false
|
|
&& $catalogLock !== false
|
|
&& $leafLock !== false
|
|
&& $parentLock < $catalogLock
|
|
&& $catalogLock < $leafLock
|
|
&& str_contains(
|
|
$lockMethod,
|
|
"(int) (\$job->parent_job_id ?? 0)\n"
|
|
. ' !== $aggregateParentId'
|
|
),
|
|
'Fence commit does not preserve aggregate parent-before-child lock order.');
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('user', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('enable')->default(1);
|
|
$table->integer('is_admin')->default(0);
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id')->unique();
|
|
$table->string('request_idempotency_key')->unique();
|
|
$table->string('request_fingerprint');
|
|
$table->integer('parent_job_id')->nullable();
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('invoice_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('snapshot_id')->nullable();
|
|
$table->integer('user_id');
|
|
$table->integer('host_id')->nullable();
|
|
$table->string('kind');
|
|
$table->string('execution_mode');
|
|
$table->string('phase');
|
|
$table->string('outcome')->nullable();
|
|
$table->string('status');
|
|
$table->text('sealed_payload');
|
|
$table->integer('attempts')->default(0);
|
|
$table->integer('dispatch_attempts')->default(0);
|
|
$table->integer('dispatch_deadline_at');
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->string('effect_resolution');
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
$table->string('effect_resolution_reason')->default('');
|
|
$table->string('completion_event_id')->nullable();
|
|
$table->integer('read_at')->nullable();
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('available_at')->default(0);
|
|
$table->string('error_code')->nullable();
|
|
$table->boolean('finalization_incident')->default(0);
|
|
$table->string('finalization_error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_agent_completion_events',
|
|
static function (Blueprint $table): void {
|
|
$table->string('event_id')->primary();
|
|
$table->integer('host_id');
|
|
$table->string('operation_id');
|
|
$table->string('payload_sha256');
|
|
$table->string('outcome');
|
|
$table->boolean('effect_may_have_applied');
|
|
$table->boolean('safe_to_resubmit');
|
|
$table->string('error_code')->default('');
|
|
$table->text('result_json')->nullable();
|
|
$table->string('disposition')->default('received');
|
|
$table->integer('received_at');
|
|
$table->integer('applied_at')->nullable();
|
|
$table->unique(['host_id', 'operation_id']);
|
|
});
|
|
$schema->create('cloud_orders', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('status');
|
|
$table->text('spec_json');
|
|
});
|
|
$schema->create('cloud_templates',
|
|
static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->string('name');
|
|
$table->string('os_family');
|
|
$table->string('os_version');
|
|
$table->string('source_type');
|
|
$table->text('sealed_source_url');
|
|
$table->integer('image_bytes');
|
|
$table->string('sha256');
|
|
$table->text('startup_script');
|
|
$table->integer('revision');
|
|
$table->string('generation_token');
|
|
$table->string('state');
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_template_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->integer('revision');
|
|
$table->string('generation_token');
|
|
$table->string('state');
|
|
$table->integer('image_bytes');
|
|
$table->string('sha256');
|
|
$table->string('operation_id');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('error_code');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->primary(['host_id', 'template_id']);
|
|
});
|
|
$schema->create('cloud_user_isos',
|
|
static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->integer('user_id');
|
|
$table->string('name');
|
|
$table->text('sealed_source_url');
|
|
$table->integer('size_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('state')->default('importing');
|
|
$table->string('import_operation_id');
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_user_iso_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('user_iso_id');
|
|
$table->integer('user_id');
|
|
$table->string('state')->default('pending');
|
|
$table->integer('size_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('operation_id');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->primary(['host_id', 'user_iso_id']);
|
|
});
|
|
Capsule::table('user')->insert([
|
|
'id' => 1, 'enable' => 1, 'is_admin' => 0,
|
|
]);
|
|
|
|
$now = time();
|
|
$nextId = 1;
|
|
$insertParent = static function (
|
|
string $kind = 'template_import',
|
|
int $userId = 1
|
|
) use (&$nextId, $now): int {
|
|
$id = $nextId++;
|
|
$operationId = md5('aggregate-parent:' . $id);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => $id,
|
|
'operation_id' => $operationId,
|
|
'request_idempotency_key' => $operationId,
|
|
'request_fingerprint' => hash('sha256', $operationId),
|
|
'parent_job_id' => null,
|
|
'order_id' => null,
|
|
'invoice_id' => null,
|
|
'instance_id' => null,
|
|
'snapshot_id' => null,
|
|
'user_id' => $userId,
|
|
'host_id' => null,
|
|
'kind' => $kind,
|
|
'execution_mode' => 'aggregate',
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'status' => 'failed',
|
|
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
|
|
'attempts' => 0,
|
|
'dispatch_attempts' => 0,
|
|
'dispatch_deadline_at' => $now - 600,
|
|
'dispatch_started_at' => null,
|
|
'agent_accepted_at' => null,
|
|
'receipt_deadline_at' => $now - 1,
|
|
'terminal_at' => $now - 1,
|
|
'effect_may_have_applied' => 1,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'unknown',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolved_by' => null,
|
|
'effect_resolution_reason' => '',
|
|
'completion_event_id' => null,
|
|
'read_at' => $now - 1,
|
|
'dispatched_at' => null,
|
|
'lease_token' => str_repeat('a', 32),
|
|
'lease_until' => $now + 300,
|
|
'available_at' => $now + 300,
|
|
'error_code' => 'aggregate_deadline_exceeded',
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'recovery_child_convergence',
|
|
'result_json' => null,
|
|
'created_at' => $now - 900,
|
|
'updated_at' => $now - 1,
|
|
]);
|
|
return $id;
|
|
};
|
|
$insertResolvedChild = static function (
|
|
int $parentId,
|
|
string $resolution,
|
|
string $kind = 'action',
|
|
int $userId = 1,
|
|
string $executionMode = 'agent'
|
|
) use (&$nextId, $now): int {
|
|
$id = $nextId++;
|
|
$operationId = md5('resolved-child:' . $id);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => $id,
|
|
'operation_id' => $operationId,
|
|
'request_idempotency_key' => $operationId,
|
|
'request_fingerprint' => hash('sha256', $operationId),
|
|
'parent_job_id' => $parentId,
|
|
'order_id' => null,
|
|
'invoice_id' => null,
|
|
'instance_id' => null,
|
|
'snapshot_id' => null,
|
|
'user_id' => $userId,
|
|
'host_id' => 7,
|
|
'kind' => $kind,
|
|
'execution_mode' => $executionMode,
|
|
'phase' => 'terminal',
|
|
'outcome' => $resolution === 'applied' ? 'succeeded' : 'failed',
|
|
'status' => $resolution === 'applied' ? 'succeeded' : 'failed',
|
|
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
|
|
'attempts' => 1,
|
|
'dispatch_attempts' => 1,
|
|
'dispatch_deadline_at' => $now - 600,
|
|
'dispatch_started_at' => $now - 500,
|
|
'agent_accepted_at' => $now - 490,
|
|
'receipt_deadline_at' => $now - 1,
|
|
'terminal_at' => $now - 1,
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => $resolution === 'not_applied' ? 1 : 0,
|
|
'effect_resolution' => $resolution,
|
|
'effect_resolved_at' => $now - 1,
|
|
'effect_resolved_by' => null,
|
|
'effect_resolution_reason' => 'normal_completion',
|
|
'completion_event_id' => null,
|
|
'read_at' => null,
|
|
'dispatched_at' => $now - 490,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'available_at' => 0,
|
|
'error_code' => $resolution === 'not_applied'
|
|
? 'operation_rejected' : null,
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
'result_json' => null,
|
|
'created_at' => $now - 600,
|
|
'updated_at' => $now - 1,
|
|
]);
|
|
return $id;
|
|
};
|
|
$insertProjectionForParent = static function (int $parentId) use ($now): string {
|
|
$parent = CloudJob::findOrFail($parentId);
|
|
$children = CloudJob::where('parent_job_id', $parentId)
|
|
->orderBy('id')->get();
|
|
if ($children->isEmpty()) {
|
|
throw new RuntimeException('Projection fixture requires children.');
|
|
}
|
|
foreach ($children as $index => $child) {
|
|
$hostId = 1000 + (int) $child->id;
|
|
Capsule::table('cloud_jobs')->where('id', (int) $child->id)
|
|
->update(['host_id' => $hostId]);
|
|
$child->host_id = $hostId;
|
|
}
|
|
if (str_starts_with((string) $parent->kind, 'template_')) {
|
|
$resourceId = 'aggregate-projection-' . $parentId;
|
|
$generation = md5('projection-generation:' . $parentId);
|
|
$sha256 = hash('sha256', 'projection-sha:' . $parentId);
|
|
$state = match ((string) $parent->kind) {
|
|
'template_import' => 'error',
|
|
'template_metadata_sync' => 'active',
|
|
'template_delete' => 'delete_failed',
|
|
};
|
|
Capsule::table('cloud_templates')->insert([
|
|
'id' => $resourceId,
|
|
'name' => 'Aggregate projection ' . $parentId,
|
|
'os_family' => 'Debian',
|
|
'os_version' => '13',
|
|
'source_type' => 'qcow2',
|
|
'sealed_source_url' => 'test-only',
|
|
'image_bytes' => 4096,
|
|
'sha256' => $sha256,
|
|
'startup_script' => '',
|
|
'revision' => (string) $parent->kind
|
|
=== 'template_metadata_sync' ? 2 : 1,
|
|
'generation_token' => $generation,
|
|
'state' => $state,
|
|
'active_job_id' => $parentId,
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
foreach ($children as $child) {
|
|
$applied = (string) $child->effect_resolution === 'applied';
|
|
$replicaState = match ((string) $parent->kind) {
|
|
'template_import' => $applied ? 'active' : 'failed',
|
|
'template_metadata_sync' => $applied ? 'active' : 'stale',
|
|
'template_delete' => $applied
|
|
? 'delete_succeeded' : 'delete_failed',
|
|
};
|
|
Capsule::table('cloud_template_replicas')->insert([
|
|
'host_id' => (int) $child->host_id,
|
|
'template_id' => $resourceId,
|
|
'revision' => (string) $parent->kind
|
|
=== 'template_metadata_sync'
|
|
? ($applied ? 2 : 1) : 1,
|
|
'generation_token' => $generation,
|
|
'state' => $replicaState,
|
|
'image_bytes' => (string) $parent->kind
|
|
=== 'template_import' && !$applied
|
|
? 0 : 4096,
|
|
'sha256' => $sha256,
|
|
'operation_id' => (string) $child->operation_id,
|
|
'job_id' => (int) $child->id,
|
|
'error_code' => $applied ? '' : 'operation_not_applied',
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
}
|
|
return $resourceId;
|
|
}
|
|
|
|
$resourceId = md5('aggregate-iso-projection:' . $parentId);
|
|
$sha256 = hash('sha256', 'projection-iso-sha:' . $parentId);
|
|
Capsule::table('cloud_user_isos')->insert([
|
|
'id' => $resourceId,
|
|
'user_id' => (int) $parent->user_id,
|
|
'name' => 'Aggregate ISO ' . $parentId,
|
|
'sealed_source_url' => 'test-only',
|
|
'size_bytes' => 4096,
|
|
'sha256' => $sha256,
|
|
'state' => (string) $parent->kind === 'user_iso_import'
|
|
? 'import_failed' : 'delete_failed',
|
|
'import_operation_id' => (string) $parent->operation_id,
|
|
'active_job_id' => $parentId,
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
foreach ($children as $child) {
|
|
$applied = (string) $child->effect_resolution === 'applied';
|
|
$replicaState = (string) $parent->kind === 'user_iso_import'
|
|
? ($applied ? 'active' : 'failed')
|
|
: ($applied ? 'delete_succeeded' : 'delete_failed');
|
|
Capsule::table('cloud_user_iso_replicas')->insert([
|
|
'host_id' => (int) $child->host_id,
|
|
'user_iso_id' => $resourceId,
|
|
'user_id' => (int) $parent->user_id,
|
|
'state' => $replicaState,
|
|
'size_bytes' => (string) $parent->kind === 'user_iso_delete'
|
|
|| $applied ? 4096 : 0,
|
|
'sha256' => $sha256,
|
|
'operation_id' => (string) $child->operation_id,
|
|
'job_id' => (int) $child->id,
|
|
'error_code' => $applied ? '' : 'operation_not_applied',
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
}
|
|
return $resourceId;
|
|
};
|
|
$settleLastChild = static function (
|
|
string $firstResolution
|
|
) use (
|
|
$insertParent,
|
|
$insertResolvedChild,
|
|
$insertProjectionForParent,
|
|
$assert
|
|
): CloudJob {
|
|
$parentId = $insertParent();
|
|
$insertResolvedChild(
|
|
$parentId,
|
|
$firstResolution,
|
|
'template_import_replica'
|
|
);
|
|
$lastChildId = $insertResolvedChild(
|
|
$parentId,
|
|
'not_applied',
|
|
'template_import_replica'
|
|
);
|
|
$insertProjectionForParent($parentId);
|
|
Capsule::table('cloud_jobs')->where('id', $lastChildId)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
'terminal_at' => null,
|
|
]);
|
|
|
|
$before = CloudJob::find($parentId);
|
|
$assert($before !== null
|
|
&& CloudJobService::recoveryPresentation($before, true) !== null,
|
|
'The aggregate fixture was not fenced before its last child.');
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $lastChildId)->update([
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'status' => 'failed',
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 1,
|
|
'effect_resolution' => 'not_applied',
|
|
'effect_resolved_at' => time(),
|
|
'effect_resolution_reason' => 'normal_completion',
|
|
'terminal_at' => time(),
|
|
]);
|
|
$completed = CloudJob::find($lastChildId);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
$completed,
|
|
null
|
|
);
|
|
$assert((string) $completed->phase === 'terminal'
|
|
&& (string) $completed->outcome === 'failed'
|
|
&& (string) $completed->effect_resolution === 'not_applied',
|
|
'The last leaf did not enter its normal terminal state.');
|
|
|
|
return CloudJob::find($parentId);
|
|
};
|
|
|
|
$allAbsent = $settleLastChild('not_applied');
|
|
$assert($allAbsent !== null
|
|
&& (string) $allAbsent->phase === 'terminal'
|
|
&& (string) $allAbsent->outcome === 'failed'
|
|
&& (string) $allAbsent->status === 'failed'
|
|
&& (string) $allAbsent->effect_resolution === 'not_applied'
|
|
&& (int) $allAbsent->effect_resolved_at > 0
|
|
&& $allAbsent->effect_resolved_by === null
|
|
&& str_starts_with(
|
|
(string) $allAbsent->effect_resolution_reason,
|
|
'child_effects_resolved:'
|
|
)
|
|
&& !(bool) $allAbsent->finalization_incident
|
|
&& $allAbsent->finalization_error_code === null
|
|
&& $allAbsent->lease_token === null
|
|
&& (int) $allAbsent->lease_until === 0
|
|
&& (int) $allAbsent->available_at === 0
|
|
&& $allAbsent->read_at === null
|
|
&& (string) Capsule::table('cloud_templates')
|
|
->where('active_job_id', (int) $allAbsent->id)
|
|
->value('state') === 'retired'
|
|
&& !Capsule::table('cloud_template_replicas')
|
|
->whereIn('template_id', static function ($query) use ($allAbsent) {
|
|
$query->select('id')->from('cloud_templates')
|
|
->where('active_job_id', (int) $allAbsent->id);
|
|
})->exists()
|
|
&& CloudJobService::recoveryPresentation($allAbsent, true) === null,
|
|
'All-not-applied import children did not retire their placeholder.');
|
|
|
|
$partial = $settleLastChild('applied');
|
|
$partialTemplateId = 'aggregate-projection-' . (int) $partial->id;
|
|
$partialTemplate = Capsule::table('cloud_templates')
|
|
->where('id', $partialTemplateId)->first();
|
|
$partialReplicaStates = Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $partialTemplateId)->orderBy('state')
|
|
->pluck('state')->all();
|
|
$partialTemplateDto = collect(
|
|
CloudTemplateCatalogService::adminCatalog()['templates']
|
|
)->firstWhere('id', $partialTemplateId);
|
|
$partialDisplay = CloudJobService::display($partial);
|
|
$assert($partial !== null
|
|
&& (string) $partial->phase === 'terminal'
|
|
&& (string) $partial->outcome === 'failed'
|
|
&& (string) $partial->status === 'failed'
|
|
&& (string) $partial->effect_resolution === 'applied'
|
|
&& str_ends_with(
|
|
(string) $partial->effect_resolution_reason,
|
|
':mixed'
|
|
)
|
|
&& !(bool) $partial->finalization_incident
|
|
&& $partial->finalization_error_code === null
|
|
&& $partial->lease_token === null
|
|
&& (int) $partial->available_at === 0
|
|
&& $partialTemplate !== null
|
|
&& (string) $partialTemplate->state === 'error'
|
|
&& $partialTemplate->active_job_id === null
|
|
&& $partialReplicaStates === ['active', 'failed']
|
|
&& is_array($partialTemplateDto)
|
|
&& ($partialTemplateDto['cleanup_reason'] ?? null)
|
|
=== 'partial_import'
|
|
&& ($partialDisplay['effect_scope'] ?? null) === 'partial'
|
|
&& CloudJobService::recoveryPresentation($partial, true) === null,
|
|
'A proven partial template import did not settle as failed-but-applied.');
|
|
|
|
$partialAppliedHost = (int) Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $partialTemplateId)
|
|
->where('state', 'active')->value('host_id');
|
|
$partialOriginalOperations = CloudJob::where(
|
|
'parent_job_id',
|
|
(int) $partial->id
|
|
)->pluck('operation_id')->all();
|
|
$partialCleanup = CloudTemplateCatalogService::delete(
|
|
$partialTemplateId,
|
|
(int) $partialTemplate->revision,
|
|
(string) $partialTemplate->generation_token,
|
|
[
|
|
'user_id' => 1,
|
|
'is_admin' => true,
|
|
'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
]
|
|
);
|
|
$partialCleanupId = (int) ($partialCleanup['task_id'] ?? 0);
|
|
$partialCleanupParent = CloudJob::findOrFail($partialCleanupId);
|
|
$partialCleanupChildren = CloudJob::where(
|
|
'parent_job_id',
|
|
$partialCleanupId
|
|
)->get();
|
|
$partialCleanupTemplate = Capsule::table('cloud_templates')
|
|
->where('id', $partialTemplateId)->first();
|
|
$partialCleanupReplicas = Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $partialTemplateId)->get();
|
|
$assert($partialCleanupId > 0
|
|
&& $partialCleanupId !== (int) $partial->id
|
|
&& (string) $partialCleanupParent->kind === 'template_delete'
|
|
&& (string) $partialCleanupParent->phase === 'awaiting_receipt'
|
|
&& $partialCleanupTemplate !== null
|
|
&& (string) $partialCleanupTemplate->state === 'deleting'
|
|
&& (int) $partialCleanupTemplate->active_job_id === $partialCleanupId
|
|
&& $partialCleanupChildren->count() === 1
|
|
&& (int) $partialCleanupChildren->first()->host_id
|
|
=== $partialAppliedHost
|
|
&& !in_array(
|
|
(string) $partialCleanupChildren->first()->operation_id,
|
|
$partialOriginalOperations,
|
|
true
|
|
)
|
|
&& $partialCleanupReplicas->count() === 1
|
|
&& (string) $partialCleanupReplicas->first()->state === 'deleting',
|
|
'Partial template import could not start an independent cleanup.');
|
|
$nextId = max(
|
|
$nextId,
|
|
(int) Capsule::table('cloud_jobs')->max('id') + 1
|
|
);
|
|
|
|
$startMixedCleanup = static function (
|
|
string $parentKind,
|
|
string $childKind
|
|
) use (
|
|
$insertParent,
|
|
$insertResolvedChild,
|
|
$insertProjectionForParent,
|
|
$assert,
|
|
&$nextId
|
|
): array {
|
|
$parentId = $insertParent($parentKind);
|
|
$appliedChildId = $insertResolvedChild(
|
|
$parentId,
|
|
'applied',
|
|
$childKind
|
|
);
|
|
$insertResolvedChild($parentId, 'not_applied', $childKind);
|
|
$resourceId = $insertProjectionForParent($parentId);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($appliedChildId),
|
|
null
|
|
);
|
|
$parent = CloudJob::findOrFail($parentId);
|
|
$validReason = (string) $parent->effect_resolution_reason;
|
|
$jobsBeforeMalformed = CloudJob::count();
|
|
$isTemplate = str_starts_with($parentKind, 'template_');
|
|
$replicaTable = $isTemplate
|
|
? 'cloud_template_replicas' : 'cloud_user_iso_replicas';
|
|
$replicaKey = $isTemplate ? 'template_id' : 'user_iso_id';
|
|
$replicasBeforeMalformed = Capsule::table($replicaTable)
|
|
->where($replicaKey, $resourceId)->orderBy('host_id')->get()
|
|
->map(static fn ($row): array => (array) $row)->all();
|
|
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
|
|
'effect_resolution_reason' => 'garbage:mixed',
|
|
]);
|
|
$malformedRejected = false;
|
|
try {
|
|
if ($isTemplate) {
|
|
$malformedTemplate = Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->first();
|
|
CloudTemplateCatalogService::delete(
|
|
$resourceId,
|
|
(int) $malformedTemplate->revision,
|
|
(string) $malformedTemplate->generation_token,
|
|
[
|
|
'user_id' => 1,
|
|
'is_admin' => true,
|
|
'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
]
|
|
);
|
|
} else {
|
|
$malformedDto = collect(
|
|
CloudUserIsoService::listing(1)['isos']
|
|
)->firstWhere('id', $resourceId);
|
|
$assert(is_array($malformedDto)
|
|
&& ($malformedDto['can_delete'] ?? true) === false
|
|
&& !array_key_exists('cleanup_reason', $malformedDto),
|
|
'Malformed mixed ISO marker exposed cleanup in its DTO.');
|
|
CloudUserIsoService::delete(
|
|
$resourceId,
|
|
['user_id' => 1]
|
|
);
|
|
}
|
|
} catch (CloudError $error) {
|
|
$malformedRejected = $error->httpStatus === 409;
|
|
}
|
|
$replicasAfterMalformed = Capsule::table($replicaTable)
|
|
->where($replicaKey, $resourceId)->orderBy('host_id')->get()
|
|
->map(static fn ($row): array => (array) $row)->all();
|
|
$malformedPointer = $isTemplate
|
|
? Capsule::table('cloud_templates')->where('id', $resourceId)
|
|
->value('active_job_id')
|
|
: Capsule::table('cloud_user_isos')->where('id', $resourceId)
|
|
->value('active_job_id');
|
|
$assert($malformedRejected
|
|
&& CloudJob::count() === $jobsBeforeMalformed
|
|
&& $malformedPointer === null
|
|
&& $replicasAfterMalformed === $replicasBeforeMalformed,
|
|
'Malformed mixed marker crossed the cleanup mutation boundary: '
|
|
. $parentKind);
|
|
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
|
|
'effect_resolution_reason' => $validReason,
|
|
]);
|
|
$parent = CloudJob::findOrFail($parentId);
|
|
$oldOperations = CloudJob::where('parent_job_id', $parentId)
|
|
->pluck('operation_id')->all();
|
|
if ($isTemplate) {
|
|
$resource = Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->first();
|
|
$preCleanupDto = null;
|
|
$cleanup = CloudTemplateCatalogService::delete(
|
|
$resourceId,
|
|
(int) $resource->revision,
|
|
(string) $resource->generation_token,
|
|
[
|
|
'user_id' => 1,
|
|
'is_admin' => true,
|
|
'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
]
|
|
);
|
|
} else {
|
|
$resource = Capsule::table('cloud_user_isos')
|
|
->where('id', $resourceId)->first();
|
|
$preCleanupDto = collect(
|
|
CloudUserIsoService::listing(1)['isos']
|
|
)->firstWhere('id', $resourceId);
|
|
$cleanup = CloudUserIsoService::delete(
|
|
$resourceId,
|
|
['user_id' => 1]
|
|
);
|
|
}
|
|
$cleanupId = (int) ($cleanup['task_id'] ?? 0);
|
|
$children = CloudJob::where('parent_job_id', $cleanupId)->get();
|
|
$nextId = max(
|
|
$nextId,
|
|
(int) Capsule::table('cloud_jobs')->max('id') + 1
|
|
);
|
|
return [
|
|
$parent,
|
|
$resource,
|
|
$resourceId,
|
|
$cleanupId,
|
|
$children,
|
|
$oldOperations,
|
|
$preCleanupDto,
|
|
];
|
|
};
|
|
|
|
[$partialDelete, $partialDeleteTemplate, $partialDeleteTemplateId,
|
|
$partialDeleteCleanupId, $partialDeleteCleanupChildren,
|
|
$partialDeleteOldOperations] = $startMixedCleanup(
|
|
'template_delete',
|
|
'template_delete_replica'
|
|
);
|
|
$partialDeleteReplicas = Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $partialDeleteTemplateId)->get();
|
|
$assert((string) $partialDelete->effect_resolution === 'applied'
|
|
&& str_ends_with(
|
|
(string) $partialDelete->effect_resolution_reason,
|
|
':mixed'
|
|
)
|
|
&& (string) $partialDeleteTemplate->state === 'delete_failed'
|
|
&& $partialDeleteTemplate->active_job_id === null
|
|
&& $partialDeleteCleanupId > 0
|
|
&& $partialDeleteCleanupId !== (int) $partialDelete->id
|
|
&& $partialDeleteCleanupChildren->count() === 1
|
|
&& !in_array(
|
|
(string) $partialDeleteCleanupChildren->first()->operation_id,
|
|
$partialDeleteOldOperations,
|
|
true
|
|
)
|
|
&& $partialDeleteReplicas->count() === 1
|
|
&& (string) $partialDeleteReplicas->first()->state === 'deleting',
|
|
'Partial template delete did not preserve failure and retry survivors.');
|
|
|
|
[$partialIsoImport, $partialIsoImportResource, $partialIsoImportId,
|
|
$partialIsoImportCleanupId, $partialIsoImportCleanupChildren,
|
|
$partialIsoImportOldOperations, $partialIsoImportDto] =
|
|
$startMixedCleanup(
|
|
'user_iso_import',
|
|
'user_iso_import_replica'
|
|
);
|
|
$partialIsoImportReplicas = Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $partialIsoImportId)->get();
|
|
$assert((string) $partialIsoImport->effect_resolution === 'applied'
|
|
&& str_ends_with(
|
|
(string) $partialIsoImport->effect_resolution_reason,
|
|
':mixed'
|
|
)
|
|
&& (string) $partialIsoImportResource->state === 'import_failed'
|
|
&& $partialIsoImportResource->active_job_id === null
|
|
&& is_array($partialIsoImportDto)
|
|
&& ($partialIsoImportDto['can_delete'] ?? false) === true
|
|
&& ($partialIsoImportDto['cleanup_reason'] ?? null)
|
|
=== 'partial_import'
|
|
&& $partialIsoImportCleanupId > 0
|
|
&& $partialIsoImportCleanupChildren->count() === 1
|
|
&& !in_array(
|
|
(string) $partialIsoImportCleanupChildren->first()->operation_id,
|
|
$partialIsoImportOldOperations,
|
|
true
|
|
)
|
|
&& $partialIsoImportReplicas->count() === 1
|
|
&& (string) $partialIsoImportReplicas->first()->state === 'deleting',
|
|
'Partial ISO import did not expose an independent safe cleanup.');
|
|
|
|
[$partialIsoDelete, $partialIsoDeleteResource, $partialIsoDeleteId,
|
|
$partialIsoDeleteCleanupId, $partialIsoDeleteCleanupChildren,
|
|
$partialIsoDeleteOldOperations, $partialIsoDeleteDto] =
|
|
$startMixedCleanup(
|
|
'user_iso_delete',
|
|
'user_iso_delete_replica'
|
|
);
|
|
$partialIsoDeleteReplicas = Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $partialIsoDeleteId)->get();
|
|
$assert((string) $partialIsoDelete->effect_resolution === 'applied'
|
|
&& str_ends_with(
|
|
(string) $partialIsoDelete->effect_resolution_reason,
|
|
':mixed'
|
|
)
|
|
&& (string) $partialIsoDeleteResource->state === 'delete_failed'
|
|
&& $partialIsoDeleteResource->active_job_id === null
|
|
&& is_array($partialIsoDeleteDto)
|
|
&& ($partialIsoDeleteDto['can_delete'] ?? false) === true
|
|
&& ($partialIsoDeleteDto['cleanup_reason'] ?? null)
|
|
=== 'partial_delete'
|
|
&& $partialIsoDeleteCleanupId > 0
|
|
&& $partialIsoDeleteCleanupChildren->count() === 1
|
|
&& !in_array(
|
|
(string) $partialIsoDeleteCleanupChildren->first()->operation_id,
|
|
$partialIsoDeleteOldOperations,
|
|
true
|
|
)
|
|
&& $partialIsoDeleteReplicas->count() === 1
|
|
&& (string) $partialIsoDeleteReplicas->first()->state === 'deleting',
|
|
'Partial ISO delete did not expose an independent safe cleanup.');
|
|
|
|
// A zero-dispatch child can know its host effect is not-applied while its
|
|
// PHP replica/catalog finalizer is still rolled back. Neither the realtime
|
|
// sibling hook nor the legacy sweep may publish the parent until that
|
|
// independent local incident has committed and cleared.
|
|
$localIncidentParentId = $insertParent('template_import');
|
|
$localIncidentChildId = $insertResolvedChild(
|
|
$localIncidentParentId,
|
|
'not_applied',
|
|
'template_import_replica'
|
|
);
|
|
$insertProjectionForParent($localIncidentParentId);
|
|
Capsule::table('cloud_jobs')->where(
|
|
'id',
|
|
$localIncidentChildId
|
|
)->update([
|
|
'attempts' => 0,
|
|
'dispatch_attempts' => 0,
|
|
'dispatch_started_at' => null,
|
|
'agent_accepted_at' => null,
|
|
'dispatched_at' => null,
|
|
'receipt_deadline_at' => null,
|
|
'completion_event_id' => null,
|
|
'safe_to_resubmit' => 0,
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'recovery_ledger_conflict',
|
|
]);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($localIncidentChildId),
|
|
null
|
|
);
|
|
$localIncidentParent = CloudJob::findOrFail($localIncidentParentId);
|
|
$assert((string) $localIncidentParent->effect_resolution === 'unknown'
|
|
&& (bool) $localIncidentParent->finalization_incident,
|
|
'A local-finalizer child prematurely released its aggregate parent.');
|
|
Capsule::table('cloud_jobs')->where(
|
|
'id',
|
|
$localIncidentChildId
|
|
)->update([
|
|
'safe_to_resubmit' => 1,
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
'effect_resolved_at' => time(),
|
|
'effect_resolution_reason' => 'automatic_local_not_dispatched',
|
|
]);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($localIncidentChildId),
|
|
null
|
|
);
|
|
$localIncidentParent->refresh();
|
|
$assert((string) $localIncidentParent->effect_resolution
|
|
=== 'not_applied'
|
|
&& !(bool) $localIncidentParent->finalization_incident,
|
|
'The aggregate parent stayed fenced after its local child converged.');
|
|
|
|
$makeLegacyEligible = static function (int $parentId): void {
|
|
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'available_at' => 0,
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'recovery_child_convergence',
|
|
]);
|
|
};
|
|
|
|
$legacyLocalIncidentParentId = $insertParent('template_import');
|
|
$makeLegacyEligible($legacyLocalIncidentParentId);
|
|
$legacyLocalIncidentChildId = $insertResolvedChild(
|
|
$legacyLocalIncidentParentId,
|
|
'not_applied',
|
|
'template_import_replica'
|
|
);
|
|
$insertProjectionForParent($legacyLocalIncidentParentId);
|
|
Capsule::table('cloud_jobs')->where(
|
|
'id',
|
|
$legacyLocalIncidentChildId
|
|
)->update([
|
|
'attempts' => 0,
|
|
'dispatch_attempts' => 0,
|
|
'dispatch_started_at' => null,
|
|
'agent_accepted_at' => null,
|
|
'dispatched_at' => null,
|
|
'receipt_deadline_at' => null,
|
|
'completion_event_id' => null,
|
|
'safe_to_resubmit' => 0,
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'recovery_ledger_conflict',
|
|
]);
|
|
$assert(CloudJobService::reconcileOneLegacyAggregateEffect(),
|
|
'The legacy sweep did not inspect a local-finalizer child incident.');
|
|
$legacyLocalIncidentParent = CloudJob::findOrFail(
|
|
$legacyLocalIncidentParentId
|
|
);
|
|
$assert((string) $legacyLocalIncidentParent->effect_resolution
|
|
=== 'unknown'
|
|
&& (string) $legacyLocalIncidentParent->finalization_error_code
|
|
=== 'recovery_child_convergence',
|
|
'The legacy sweep prematurely released a local-finalizer child.');
|
|
Capsule::table('cloud_jobs')->where(
|
|
'id',
|
|
$legacyLocalIncidentChildId
|
|
)->update([
|
|
'safe_to_resubmit' => 1,
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
'effect_resolved_at' => time(),
|
|
'effect_resolution_reason' => 'automatic_local_not_dispatched',
|
|
]);
|
|
$makeLegacyEligible($legacyLocalIncidentParentId);
|
|
$assert(CloudJobService::reconcileOneLegacyAggregateEffect(),
|
|
'The legacy sweep did not resume after local finalizer convergence.');
|
|
$legacyLocalIncidentParent->refresh();
|
|
$assert((string) $legacyLocalIncidentParent->effect_resolution
|
|
=== 'not_applied'
|
|
&& !(bool) $legacyLocalIncidentParent->finalization_incident,
|
|
'The legacy parent stayed fenced after local finalizer convergence.');
|
|
|
|
// The realtime hook applies the same topology rules to both the child that
|
|
// triggered it and every sibling. A wrong link is an incident, never
|
|
// evidence that the aggregate effect can be published.
|
|
$liveWrongCurrentParent = $insertParent('template_delete');
|
|
$makeLegacyEligible($liveWrongCurrentParent);
|
|
$liveWrongCurrent = $insertResolvedChild(
|
|
$liveWrongCurrentParent,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
$insertResolvedChild(
|
|
$liveWrongCurrentParent,
|
|
'not_applied',
|
|
'template_delete_replica'
|
|
);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::find($liveWrongCurrent),
|
|
null
|
|
);
|
|
|
|
$liveWrongSiblingParent = $insertParent('user_iso_delete');
|
|
$makeLegacyEligible($liveWrongSiblingParent);
|
|
$liveValidCurrent = $insertResolvedChild(
|
|
$liveWrongSiblingParent,
|
|
'applied',
|
|
'user_iso_delete_replica'
|
|
);
|
|
$insertResolvedChild(
|
|
$liveWrongSiblingParent,
|
|
'not_applied',
|
|
'user_iso_import_replica'
|
|
);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::find($liveValidCurrent),
|
|
null
|
|
);
|
|
|
|
foreach ([
|
|
$liveWrongCurrentParent => 'current child',
|
|
$liveWrongSiblingParent => 'sibling child',
|
|
] as $parentId => $description) {
|
|
$conflict = CloudJob::find($parentId);
|
|
$display = CloudJobService::recoveryPresentation($conflict, true);
|
|
$assert((string) $conflict->effect_resolution === 'unknown'
|
|
&& (bool) $conflict->finalization_incident
|
|
&& (string) $conflict->finalization_error_code
|
|
=== 'recovery_aggregate_state_conflict'
|
|
&& (int) $conflict->available_at >= time() + 290
|
|
&& $conflict->read_at === null
|
|
&& ($display['code'] ?? null)
|
|
=== 'recovery_aggregate_state_conflict',
|
|
'The live hook released a parent with a wrong '
|
|
. $description . ' relationship.');
|
|
}
|
|
|
|
foreach ([
|
|
'missing parent' => static function () use (
|
|
$insertResolvedChild
|
|
): CloudJob {
|
|
return CloudJob::findOrFail($insertResolvedChild(
|
|
999999,
|
|
'applied',
|
|
'template_import_replica'
|
|
));
|
|
},
|
|
'non-aggregate parent' => static function () use (
|
|
$insertParent,
|
|
$insertResolvedChild
|
|
): CloudJob {
|
|
$parentId = $insertParent('template_import');
|
|
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
|
|
'execution_mode' => 'local',
|
|
]);
|
|
return CloudJob::findOrFail($insertResolvedChild(
|
|
$parentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
));
|
|
},
|
|
] as $description => $fixture) {
|
|
$thrown = false;
|
|
try {
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
$fixture(),
|
|
null
|
|
);
|
|
} catch (CloudRecoveryAggregateStateConflict $error) {
|
|
$thrown = true;
|
|
}
|
|
$assert($thrown,
|
|
'A child with a ' . $description
|
|
. ' could be silently acknowledged.');
|
|
}
|
|
|
|
// A damaged aggregate relationship must be rejected before the recovery
|
|
// coordinator writes the Agent journal fence. Cover both parent-envelope
|
|
// and child-kind corruption at the private transaction boundary.
|
|
$resolveFence = new ReflectionMethod(
|
|
CloudAutomaticOperationRecoveryService::class,
|
|
'resolveFenceAndCommit'
|
|
);
|
|
$preparedFence = [
|
|
'resolution' => 'applied',
|
|
'journal_status' => 'failed',
|
|
'fence_resolution_required' => true,
|
|
'source' => 'host_inspection',
|
|
'operation' => [],
|
|
];
|
|
foreach ([
|
|
'bad parent mode' => static function () use (
|
|
$insertParent,
|
|
$insertResolvedChild
|
|
): CloudJob {
|
|
$parentId = $insertParent('template_import');
|
|
$childId = $insertResolvedChild(
|
|
$parentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
|
|
'execution_mode' => 'local',
|
|
]);
|
|
Capsule::table('cloud_jobs')->where('id', $childId)->update([
|
|
'effect_resolution' => 'unknown',
|
|
'lease_token' => str_repeat('f', 32),
|
|
'lease_until' => time() + 300,
|
|
]);
|
|
return CloudJob::findOrFail($childId);
|
|
},
|
|
'wrong child kind' => static function () use (
|
|
$insertParent,
|
|
$insertResolvedChild,
|
|
$insertProjectionForParent
|
|
): CloudJob {
|
|
$parentId = $insertParent('template_delete');
|
|
$childId = $insertResolvedChild(
|
|
$parentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
$insertProjectionForParent($parentId);
|
|
Capsule::table('cloud_jobs')->where('id', $childId)->update([
|
|
'effect_resolution' => 'unknown',
|
|
'lease_token' => str_repeat('e', 32),
|
|
'lease_until' => time() + 300,
|
|
]);
|
|
return CloudJob::findOrFail($childId);
|
|
},
|
|
] as $description => $fixture) {
|
|
$beforeFenceRequests = CloudAgentSettingsService::$requests;
|
|
$blocked = false;
|
|
try {
|
|
$resolveFence->invoke(
|
|
null,
|
|
$fixture(),
|
|
$preparedFence,
|
|
'test-only',
|
|
[]
|
|
);
|
|
} catch (CloudRecoveryAggregateStateConflict $error) {
|
|
$blocked = true;
|
|
}
|
|
$assert(
|
|
$blocked
|
|
&& CloudAgentSettingsService::$requests === $beforeFenceRequests,
|
|
'A ' . $description . ' reached an Agent fence write.'
|
|
);
|
|
}
|
|
|
|
// Upgrade sweep: these parents were already terminal failed/unknown before
|
|
// the realtime leaf hook existed. Each tick may converge exactly one local
|
|
// parent from already-known children and must never need an Agent client.
|
|
$legacyMappings = [
|
|
'template_import' => 'template_import_replica',
|
|
'template_metadata_sync' => 'template_metadata_replica',
|
|
'template_delete' => 'template_delete_replica',
|
|
'user_iso_import' => 'user_iso_import_replica',
|
|
'user_iso_delete' => 'user_iso_delete_replica',
|
|
];
|
|
$legacyParents = [];
|
|
foreach ($legacyMappings as $parentKind => $childKind) {
|
|
$parentId = $insertParent($parentKind);
|
|
$makeLegacyEligible($parentId);
|
|
$firstResolution = $parentKind === 'template_import'
|
|
? 'not_applied' : 'applied';
|
|
$insertResolvedChild(
|
|
$parentId,
|
|
$firstResolution,
|
|
$childKind
|
|
);
|
|
$insertResolvedChild($parentId, $firstResolution, $childKind);
|
|
$resourceId = $insertProjectionForParent($parentId);
|
|
$legacyParents[$parentId] = [
|
|
'kind' => $parentKind,
|
|
'resolution' => $firstResolution === 'applied'
|
|
? 'applied' : 'not_applied',
|
|
'resource_id' => $resourceId,
|
|
];
|
|
}
|
|
$automatic = new CloudAutomaticOperationRecoveryService();
|
|
$resolvedLegacyCount = 0;
|
|
foreach ($legacyParents as $parentId => $expectation) {
|
|
$assert($automatic->tick(),
|
|
'The local legacy aggregate sweep did not claim a parent.');
|
|
$resolved = CloudJob::whereIn('id', array_keys($legacyParents))
|
|
->whereIn('effect_resolution', ['applied', 'not_applied'])
|
|
->count();
|
|
$resolvedLegacyCount++;
|
|
$current = CloudJob::find($parentId);
|
|
$assert($resolved === $resolvedLegacyCount
|
|
&& $current !== null
|
|
&& (string) $current->effect_resolution
|
|
=== $expectation['resolution']
|
|
&& (string) $current->outcome === 'failed'
|
|
&& !(bool) $current->finalization_incident
|
|
&& $current->finalization_error_code === null
|
|
&& CloudJobService::recoveryPresentation(
|
|
$current,
|
|
true
|
|
) === null,
|
|
'A tick handled multiple legacy parents or mapped children incorrectly.');
|
|
$resourceId = $expectation['resource_id'];
|
|
$projectionOkay = match ($expectation['kind']) {
|
|
'template_import' =>
|
|
(string) Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->value('state') === 'retired'
|
|
&& !Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->exists(),
|
|
'template_metadata_sync' =>
|
|
(string) Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->value('state') === 'active'
|
|
&& !Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)
|
|
->where('state', '<>', 'active')->exists(),
|
|
'template_delete' =>
|
|
(string) Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->value('state') === 'retired'
|
|
&& !Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->exists(),
|
|
'user_iso_import' =>
|
|
(string) Capsule::table('cloud_user_isos')
|
|
->where('id', $resourceId)->value('state') === 'active',
|
|
'user_iso_delete' =>
|
|
(string) Capsule::table('cloud_user_isos')
|
|
->where('id', $resourceId)->value('state') === 'retired'
|
|
&& !Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $resourceId)->exists(),
|
|
};
|
|
$assert($projectionOkay,
|
|
'A legacy terminal parent did not recompute its resource projection: '
|
|
. $expectation['kind']);
|
|
}
|
|
|
|
$waitingParent = $insertParent('template_import');
|
|
$makeLegacyEligible($waitingParent);
|
|
$waitingChild = $insertResolvedChild(
|
|
$waitingParent,
|
|
'not_applied',
|
|
'template_import_replica'
|
|
);
|
|
$insertProjectionForParent($waitingParent);
|
|
Capsule::table('cloud_jobs')->where('id', $waitingChild)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
]);
|
|
|
|
// Receipt handling must run before a legacy aggregate that can remain in
|
|
// child-convergence for an arbitrarily long time. Use an invalid durable
|
|
// event here so the scheduler must quarantine it without touching Agent
|
|
// or catalog state; the aggregate remains immediately eligible for the
|
|
// following tick instead of starving the receipt forever.
|
|
$priorityChild = $insertResolvedChild(
|
|
0,
|
|
'not_applied',
|
|
'template_import_replica'
|
|
);
|
|
Capsule::table('cloud_jobs')->where('id', $priorityChild)->update([
|
|
'outcome' => 'failed',
|
|
'status' => 'failed',
|
|
'effect_may_have_applied' => 1,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'unknown',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
'available_at' => 0,
|
|
'dispatched_at' => $now - 20,
|
|
]);
|
|
$priorityOperation = (string) CloudJob::findOrFail(
|
|
$priorityChild
|
|
)->operation_id;
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
|
'event_id' => 'aggregate-priority-receipt',
|
|
'host_id' => 7,
|
|
'operation_id' => $priorityOperation,
|
|
'payload_sha256' => str_repeat('0', 64),
|
|
'outcome' => 'succeeded',
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0,
|
|
'error_code' => '',
|
|
'result_json' => null,
|
|
'disposition' => 'received',
|
|
'received_at' => $now,
|
|
'applied_at' => null,
|
|
]);
|
|
$makeLegacyEligible($waitingParent);
|
|
$assert($automatic->tick(),
|
|
'A durable receipt was not prioritised over a waiting aggregate.');
|
|
$priorityEvent = Capsule::table('cloud_agent_completion_events')
|
|
->where('event_id', 'aggregate-priority-receipt')->first();
|
|
$stillEligibleParent = CloudJob::findOrFail($waitingParent);
|
|
$assert((string) $priorityEvent->disposition === 'rejected'
|
|
&& (int) $stillEligibleParent->available_at === 0,
|
|
'A waiting legacy aggregate starved receipt validation.');
|
|
|
|
$assert($automatic->tick(),
|
|
'A legacy aggregate with an active child was not inspected locally.');
|
|
$waiting = CloudJob::find($waitingParent);
|
|
$waitingDisplay = CloudJobService::recoveryPresentation($waiting, true);
|
|
$assert((string) $waiting->effect_resolution === 'unknown'
|
|
&& (string) $waiting->finalization_error_code
|
|
=== 'recovery_child_convergence'
|
|
&& (int) $waiting->available_at >= time() + 290
|
|
&& ($waitingDisplay['code'] ?? null)
|
|
=== 'recovery_child_convergence',
|
|
'An unresolved but valid child released or corrupted its parent fence.');
|
|
|
|
// Corrupt aggregate relationships remain fenced and retryable. Cover every
|
|
// structural invariant independently so a future relaxed query cannot turn
|
|
// a wrong child into evidence for releasing its parent.
|
|
$conflictFixtures = [];
|
|
|
|
$missingChildParent = $insertParent('template_import');
|
|
$makeLegacyEligible($missingChildParent);
|
|
$conflictFixtures[$missingChildParent] = 'missing child';
|
|
|
|
$wrongParentMode = $insertParent('template_import');
|
|
$makeLegacyEligible($wrongParentMode);
|
|
$insertResolvedChild(
|
|
$wrongParentMode,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
Capsule::table('cloud_jobs')->where('id', $wrongParentMode)->update([
|
|
'execution_mode' => 'local',
|
|
]);
|
|
$conflictFixtures[$wrongParentMode] = 'wrong parent execution mode';
|
|
|
|
$wrongParentKind = $insertParent('template_import');
|
|
$makeLegacyEligible($wrongParentKind);
|
|
$insertResolvedChild(
|
|
$wrongParentKind,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
Capsule::table('cloud_jobs')->where('id', $wrongParentKind)->update([
|
|
'kind' => 'corrupt_aggregate_kind',
|
|
]);
|
|
$conflictFixtures[$wrongParentKind] = 'wrong parent kind';
|
|
|
|
$wrongKindParent = $insertParent('template_delete');
|
|
$makeLegacyEligible($wrongKindParent);
|
|
$insertResolvedChild(
|
|
$wrongKindParent,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
$conflictFixtures[$wrongKindParent] = 'wrong child kind';
|
|
|
|
$wrongUserParent = $insertParent('user_iso_import');
|
|
$makeLegacyEligible($wrongUserParent);
|
|
$insertResolvedChild(
|
|
$wrongUserParent,
|
|
'applied',
|
|
'user_iso_import_replica',
|
|
2
|
|
);
|
|
$conflictFixtures[$wrongUserParent] = 'wrong child owner';
|
|
|
|
$wrongModeParent = $insertParent('user_iso_delete');
|
|
$makeLegacyEligible($wrongModeParent);
|
|
$insertResolvedChild(
|
|
$wrongModeParent,
|
|
'not_applied',
|
|
'user_iso_delete_replica',
|
|
1,
|
|
'local'
|
|
);
|
|
$conflictFixtures[$wrongModeParent] = 'wrong child execution mode';
|
|
|
|
$nestedParent = $insertParent('template_metadata_sync');
|
|
$makeLegacyEligible($nestedParent);
|
|
Capsule::table('cloud_jobs')->where('id', $nestedParent)->update([
|
|
// The parent only needs to be non-null to exercise the invalid
|
|
// aggregate nesting invariant. Do not attach it to an earlier valid
|
|
// aggregate fixture: doing so also (correctly) corrupts that parent's
|
|
// already-published child topology and changes the next claimed row.
|
|
'parent_job_id' => 999999,
|
|
]);
|
|
$insertResolvedChild(
|
|
$nestedParent,
|
|
'applied',
|
|
'template_metadata_replica'
|
|
);
|
|
$conflictFixtures[$nestedParent] = 'non-top-level aggregate parent';
|
|
|
|
// A malformed Agent child is itself a higher-priority PHP-only leaf
|
|
// repair, so it may be claimed before an earlier aggregate parent. Drain
|
|
// the bounded fixture set, then verify every parent retained its fence.
|
|
$beforeConflictRequests = CloudAgentSettingsService::$requests;
|
|
for ($attempt = 0; $attempt < count($conflictFixtures) * 3; $attempt++) {
|
|
$remaining = CloudJob::whereIn('id', array_keys($conflictFixtures))
|
|
->where(static function ($pending): void {
|
|
$pending->where('finalization_incident', 0)
|
|
->orWhereNull('finalization_error_code')
|
|
->orWhere(
|
|
'finalization_error_code',
|
|
'<>',
|
|
'recovery_aggregate_state_conflict'
|
|
);
|
|
})->exists();
|
|
if (!$remaining) {
|
|
break;
|
|
}
|
|
$assert($automatic->tick(),
|
|
'The aggregate conflict fixture queue stopped before convergence.');
|
|
}
|
|
foreach ($conflictFixtures as $parentId => $description) {
|
|
$conflict = CloudJob::find($parentId);
|
|
$display = $conflict === null ? null
|
|
: CloudJobService::recoveryPresentation($conflict, true);
|
|
$assert($conflict !== null
|
|
&& (string) $conflict->effect_resolution === 'unknown'
|
|
&& (string) $conflict->outcome === 'failed'
|
|
&& (bool) $conflict->finalization_incident
|
|
&& (string) $conflict->finalization_error_code
|
|
=== 'recovery_aggregate_state_conflict'
|
|
&& (int) $conflict->available_at >= time() + 290
|
|
&& $conflict->lease_token === null
|
|
&& (int) $conflict->lease_until === 0
|
|
&& $conflict->read_at === null
|
|
&& ($display['code'] ?? null)
|
|
=== 'recovery_aggregate_state_conflict'
|
|
&& str_contains(
|
|
(string) ($display['message'] ?? ''),
|
|
'关系异常'
|
|
)
|
|
&& CloudAgentSettingsService::$requests
|
|
=== $beforeConflictRequests,
|
|
'A ' . $description
|
|
. ' relationship released or concealed its aggregate fence.');
|
|
}
|
|
|
|
// prepareRecoveredEffect updates a replica before the external Agent fence
|
|
// write. Simulate that exact local prepare seam: the public parent must
|
|
// remain visibly recovering until the child effect commit runs, then the
|
|
// same transaction may publish the aggregate success.
|
|
$preparedParentId = $nextId++;
|
|
$preparedParentOperation = md5(
|
|
'aggregate-prepared-parent:' . $preparedParentId
|
|
);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => $preparedParentId,
|
|
'operation_id' => $preparedParentOperation,
|
|
'request_idempotency_key' => $preparedParentOperation,
|
|
'request_fingerprint' => hash('sha256', $preparedParentOperation),
|
|
'parent_job_id' => null,
|
|
'user_id' => 1,
|
|
'host_id' => null,
|
|
'kind' => 'template_import',
|
|
'execution_mode' => 'aggregate',
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
|
|
'attempts' => 0,
|
|
'dispatch_attempts' => 0,
|
|
'dispatch_deadline_at' => $now + 300,
|
|
'receipt_deadline_at' => $now + 900,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolution_reason' => '',
|
|
'lease_until' => 0,
|
|
'available_at' => $now,
|
|
'finalization_incident' => 0,
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
$preparedChildId = $insertResolvedChild(
|
|
$preparedParentId,
|
|
'not_applied',
|
|
'template_import_replica'
|
|
);
|
|
Capsule::table('cloud_jobs')->where('id', $preparedChildId)->update([
|
|
'outcome' => 'failed',
|
|
'status' => 'failed',
|
|
'effect_may_have_applied' => 1,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'unknown',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' =>
|
|
'automatic_recovery:v1:1:1:prepared:test',
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'recovery_payload_unavailable',
|
|
]);
|
|
$preparedTemplateId = 'prepared-import-template';
|
|
$preparedGeneration = str_repeat('c', 32);
|
|
$preparedSha = str_repeat('d', 64);
|
|
Capsule::table('cloud_templates')->insert([
|
|
'id' => $preparedTemplateId,
|
|
'name' => 'Prepared import',
|
|
'os_family' => 'Debian',
|
|
'os_version' => '13',
|
|
'source_type' => 'qcow2',
|
|
'sealed_source_url' => 'test-only',
|
|
'image_bytes' => 4096,
|
|
'sha256' => $preparedSha,
|
|
'startup_script' => '',
|
|
'revision' => 1,
|
|
'generation_token' => $preparedGeneration,
|
|
'state' => 'importing',
|
|
'active_job_id' => $preparedParentId,
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_template_replicas')->insert([
|
|
'host_id' => 7,
|
|
'template_id' => $preparedTemplateId,
|
|
'revision' => 1,
|
|
'generation_token' => $preparedGeneration,
|
|
'state' => 'failed',
|
|
'image_bytes' => 0,
|
|
'sha256' => $preparedSha,
|
|
'operation_id' => (string) CloudJob::findOrFail(
|
|
$preparedChildId
|
|
)->operation_id,
|
|
'job_id' => $preparedChildId,
|
|
'error_code' => 'aggregate_deadline_exceeded',
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
$preparedOperation = [
|
|
'operation_id' => (string) CloudJob::findOrFail(
|
|
$preparedChildId
|
|
)->operation_id,
|
|
'status' => 'succeeded',
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => false,
|
|
'error_code' => '',
|
|
];
|
|
CloudTemplateCatalogService::finishJobLocked(
|
|
CloudJob::findOrFail($preparedChildId),
|
|
'succeeded',
|
|
[
|
|
'id' => $preparedTemplateId,
|
|
'revision' => 1,
|
|
'generation_token' => $preparedGeneration,
|
|
'size_bytes' => 4096,
|
|
'sha256' => $preparedSha,
|
|
],
|
|
['template_id' => $preparedTemplateId],
|
|
$preparedOperation,
|
|
true
|
|
);
|
|
$preparedParent = CloudJob::findOrFail($preparedParentId);
|
|
$preparedParentDisplay = CloudJobService::display(
|
|
$preparedParent,
|
|
true
|
|
);
|
|
$assert((string) $preparedParent->phase === 'awaiting_receipt'
|
|
&& (string) Capsule::table('cloud_templates')
|
|
->where('id', $preparedTemplateId)->value('state') === 'importing'
|
|
&& (string) Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $preparedTemplateId)
|
|
->value('state') === 'active'
|
|
&& ($preparedParentDisplay['recovery']['pending'] ?? null) === true
|
|
&& ($preparedParentDisplay['recovery']['code'] ?? null)
|
|
=== 'recovery_payload_unavailable',
|
|
'A recovered child prepare prematurely completed or hid its parent.');
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $preparedChildId)->update([
|
|
'effect_resolution' => 'applied',
|
|
'effect_resolved_at' => time(),
|
|
'effect_resolution_reason' => 'agent_inspection:test',
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
'available_at' => 0,
|
|
]);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($preparedChildId),
|
|
null
|
|
);
|
|
$preparedParent->refresh();
|
|
$assert((string) $preparedParent->phase === 'terminal'
|
|
&& (string) $preparedParent->outcome === 'succeeded'
|
|
&& (string) $preparedParent->effect_resolution === 'applied'
|
|
&& (string) Capsule::table('cloud_templates')
|
|
->where('id', $preparedTemplateId)->value('state') === 'active'
|
|
&& CloudJobService::recoveryPresentation(
|
|
$preparedParent,
|
|
true
|
|
) === null,
|
|
'The aggregate parent did not settle after the child effect commit.');
|
|
|
|
$immutableParentAudit = static fn (CloudJob $job): array => [
|
|
'phase' => (string) $job->phase,
|
|
'outcome' => $job->outcome === null ? null : (string) $job->outcome,
|
|
'status' => (string) $job->status,
|
|
'terminal_at' => $job->terminal_at === null
|
|
? null : (int) $job->terminal_at,
|
|
'error_code' => $job->error_code === null
|
|
? null : (string) $job->error_code,
|
|
'result_json' => $job->result_json === null
|
|
? null : (string) $job->result_json,
|
|
];
|
|
$markDeadlineAudit = static function (int $parentId): CloudJob {
|
|
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
|
|
'result_json' => json_encode(
|
|
['deadline_parent' => $parentId],
|
|
JSON_THROW_ON_ERROR
|
|
),
|
|
]);
|
|
return CloudJob::findOrFail($parentId);
|
|
};
|
|
|
|
// A parent deadline is an immutable failed audit, not the final catalog
|
|
// projection. Once every late child resolves, imports become usable and
|
|
// deletes become retired without rewriting that original failed task.
|
|
$deadlineImportParentId = $insertParent('template_import');
|
|
$deadlineImportParent = $markDeadlineAudit($deadlineImportParentId);
|
|
$deadlineImportAudit = $immutableParentAudit($deadlineImportParent);
|
|
$deadlineImportChildId = $insertResolvedChild(
|
|
$deadlineImportParentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
$deadlineImportTemplateId = 'deadline-import-template';
|
|
$deadlineImportGeneration = str_repeat('5', 32);
|
|
$deadlineImportSha = str_repeat('6', 64);
|
|
Capsule::table('cloud_templates')->insert([
|
|
'id' => $deadlineImportTemplateId,
|
|
'name' => 'Deadline import',
|
|
'os_family' => 'Debian',
|
|
'os_version' => '13',
|
|
'source_type' => 'qcow2',
|
|
'sealed_source_url' => 'test-only',
|
|
'image_bytes' => 4096,
|
|
'sha256' => $deadlineImportSha,
|
|
'startup_script' => '',
|
|
'revision' => 1,
|
|
'generation_token' => $deadlineImportGeneration,
|
|
'state' => 'error',
|
|
'active_job_id' => $deadlineImportParentId,
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_template_replicas')->insert([
|
|
'host_id' => 7,
|
|
'template_id' => $deadlineImportTemplateId,
|
|
'revision' => 1,
|
|
'generation_token' => $deadlineImportGeneration,
|
|
'state' => 'active',
|
|
'image_bytes' => 4096,
|
|
'sha256' => $deadlineImportSha,
|
|
'operation_id' => (string) CloudJob::findOrFail(
|
|
$deadlineImportChildId
|
|
)->operation_id,
|
|
'job_id' => $deadlineImportChildId,
|
|
'error_code' => '',
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($deadlineImportChildId),
|
|
null
|
|
);
|
|
$deadlineImportParent->refresh();
|
|
$deadlinePublicIds = array_column(
|
|
CloudTemplateCatalogService::publicCatalog()['templates'],
|
|
'id'
|
|
);
|
|
$assert($immutableParentAudit($deadlineImportParent)
|
|
=== $deadlineImportAudit
|
|
&& (string) $deadlineImportParent->effect_resolution === 'applied'
|
|
&& (string) Capsule::table('cloud_templates')
|
|
->where('id', $deadlineImportTemplateId)->value('state')
|
|
=== 'active'
|
|
&& in_array($deadlineImportTemplateId, $deadlinePublicIds, true),
|
|
'A recovered deadline import did not publish its catalog projection '
|
|
. 'without rewriting the failed task.');
|
|
|
|
$deadlineDeleteParentId = $insertParent('template_delete');
|
|
$deadlineDeleteParent = $markDeadlineAudit($deadlineDeleteParentId);
|
|
$deadlineDeleteAudit = $immutableParentAudit($deadlineDeleteParent);
|
|
$deadlineDeleteChildId = $insertResolvedChild(
|
|
$deadlineDeleteParentId,
|
|
'applied',
|
|
'template_delete_replica'
|
|
);
|
|
$deadlineDeleteTemplateId = 'deadline-delete-template';
|
|
$deadlineDeleteGeneration = str_repeat('7', 32);
|
|
$deadlineDeleteSha = str_repeat('8', 64);
|
|
Capsule::table('cloud_templates')->insert([
|
|
'id' => $deadlineDeleteTemplateId,
|
|
'name' => 'Deadline delete',
|
|
'os_family' => 'Debian',
|
|
'os_version' => '13',
|
|
'source_type' => 'qcow2',
|
|
'sealed_source_url' => 'test-only',
|
|
'image_bytes' => 4096,
|
|
'sha256' => $deadlineDeleteSha,
|
|
'startup_script' => '',
|
|
'revision' => 1,
|
|
'generation_token' => $deadlineDeleteGeneration,
|
|
'state' => 'delete_failed',
|
|
'active_job_id' => $deadlineDeleteParentId,
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_template_replicas')->insert([
|
|
'host_id' => 7,
|
|
'template_id' => $deadlineDeleteTemplateId,
|
|
'revision' => 1,
|
|
'generation_token' => $deadlineDeleteGeneration,
|
|
'state' => 'delete_succeeded',
|
|
'image_bytes' => 4096,
|
|
'sha256' => $deadlineDeleteSha,
|
|
'operation_id' => (string) CloudJob::findOrFail(
|
|
$deadlineDeleteChildId
|
|
)->operation_id,
|
|
'job_id' => $deadlineDeleteChildId,
|
|
'error_code' => '',
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($deadlineDeleteChildId),
|
|
null
|
|
);
|
|
$deadlineDeleteParent->refresh();
|
|
$assert($immutableParentAudit($deadlineDeleteParent)
|
|
=== $deadlineDeleteAudit
|
|
&& (string) $deadlineDeleteParent->effect_resolution === 'applied'
|
|
&& (string) Capsule::table('cloud_templates')
|
|
->where('id', $deadlineDeleteTemplateId)->value('state')
|
|
=== 'retired'
|
|
&& (int) Capsule::table('cloud_templates')
|
|
->where('id', $deadlineDeleteTemplateId)->value('revision') === 2
|
|
&& !Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $deadlineDeleteTemplateId)->exists(),
|
|
'A recovered deadline delete did not retire its catalog projection '
|
|
. 'without rewriting the failed task.');
|
|
|
|
$deadlineIsoParentId = $insertParent('user_iso_import');
|
|
$deadlineIsoParent = $markDeadlineAudit($deadlineIsoParentId);
|
|
$deadlineIsoAudit = $immutableParentAudit($deadlineIsoParent);
|
|
$deadlineIsoChildId = $insertResolvedChild(
|
|
$deadlineIsoParentId,
|
|
'applied',
|
|
'user_iso_import_replica'
|
|
);
|
|
$deadlineIsoId = str_repeat('9', 32);
|
|
$deadlineIsoSha = str_repeat('a', 64);
|
|
Capsule::table('cloud_user_isos')->insert([
|
|
'id' => $deadlineIsoId,
|
|
'user_id' => 1,
|
|
'name' => 'Deadline ISO',
|
|
'sealed_source_url' => 'test-only',
|
|
'size_bytes' => 4096,
|
|
'sha256' => $deadlineIsoSha,
|
|
'state' => 'import_failed',
|
|
'import_operation_id' => (string) $deadlineIsoParent->operation_id,
|
|
'active_job_id' => $deadlineIsoParentId,
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_user_iso_replicas')->insert([
|
|
'host_id' => 7,
|
|
'user_iso_id' => $deadlineIsoId,
|
|
'user_id' => 1,
|
|
'state' => 'active',
|
|
'size_bytes' => 4096,
|
|
'sha256' => $deadlineIsoSha,
|
|
'operation_id' => (string) CloudJob::findOrFail(
|
|
$deadlineIsoChildId
|
|
)->operation_id,
|
|
'job_id' => $deadlineIsoChildId,
|
|
'error_code' => '',
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($deadlineIsoChildId),
|
|
null
|
|
);
|
|
$deadlineIsoParent->refresh();
|
|
$assert($immutableParentAudit($deadlineIsoParent) === $deadlineIsoAudit
|
|
&& (string) $deadlineIsoParent->effect_resolution === 'applied'
|
|
&& (string) Capsule::table('cloud_user_isos')
|
|
->where('id', $deadlineIsoId)->value('state') === 'active',
|
|
'A recovered deadline ISO import did not publish its catalog '
|
|
. 'projection without rewriting the failed task.');
|
|
|
|
$settledPartialBefore = CloudJob::findOrFail((int) $partial->id);
|
|
$settledPartialUpdatedAt = (int) $settledPartialBefore->updated_at;
|
|
$automatic->tick();
|
|
$settledPartialAfter = CloudJob::findOrFail((int) $partial->id);
|
|
$assert((int) $settledPartialAfter->updated_at
|
|
=== $settledPartialUpdatedAt
|
|
&& (int) $settledPartialAfter->available_at === 0
|
|
&& (string) $settledPartialAfter->effect_resolution === 'applied'
|
|
&& !(bool) $settledPartialAfter->finalization_incident,
|
|
'A settled partial aggregate re-entered the recovery queue.');
|
|
$assert(CloudAgentSettingsService::$requests === 0,
|
|
'Aggregate recovery contacted an Agent instead of local ledgers.');
|
|
|
|
// A corrupt replica dispatch ledger must not be interpreted as an
|
|
// undispatched/no-effect child. Doing so would fail the replica and
|
|
// release the public template aggregate even though the Agent boundary is
|
|
// unknown. Keep the catalog, child escrow and parent fence intact instead.
|
|
$invalidDispatchParentId = $insertParent('template_import');
|
|
Capsule::table('cloud_jobs')->where(
|
|
'id',
|
|
$invalidDispatchParentId
|
|
)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'available_at' => 0,
|
|
'receipt_deadline_at' => $now + 600,
|
|
]);
|
|
$invalidDispatchChildId = $insertResolvedChild(
|
|
$invalidDispatchParentId,
|
|
'not_applied',
|
|
'template_import_replica'
|
|
);
|
|
$invalidDispatchTemplateId = 'invalid-dispatch-template';
|
|
$invalidDispatchGeneration = str_repeat('e', 32);
|
|
$invalidDispatchSha = str_repeat('f', 64);
|
|
$invalidDispatchPayload = CloudSecrets::encrypt([
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/templates/import',
|
|
'body' => [],
|
|
'template_id' => $invalidDispatchTemplateId,
|
|
'owner_id' => 1,
|
|
'can_manage' => true,
|
|
], 'job-payload');
|
|
Capsule::table('cloud_jobs')->where(
|
|
'id',
|
|
$invalidDispatchChildId
|
|
)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'sealed_payload' => $invalidDispatchPayload,
|
|
'attempts' => 2,
|
|
'dispatch_attempts' => 2,
|
|
'dispatch_started_at' => $now - 30,
|
|
'agent_accepted_at' => $now - 29,
|
|
'receipt_deadline_at' => $now - 1,
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
'completion_event_id' => null,
|
|
'dispatched_at' => $now - 29,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'available_at' => 0,
|
|
'error_code' => null,
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
]);
|
|
Capsule::table('cloud_templates')->insert([
|
|
'id' => $invalidDispatchTemplateId,
|
|
'name' => 'Invalid dispatch template',
|
|
'os_family' => 'Debian',
|
|
'os_version' => '13',
|
|
'source_type' => 'qcow2',
|
|
'sealed_source_url' => 'test-only',
|
|
'image_bytes' => 4096,
|
|
'sha256' => $invalidDispatchSha,
|
|
'startup_script' => '',
|
|
'revision' => 1,
|
|
'generation_token' => $invalidDispatchGeneration,
|
|
'state' => 'importing',
|
|
'active_job_id' => $invalidDispatchParentId,
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_template_replicas')->insert([
|
|
'host_id' => 7,
|
|
'template_id' => $invalidDispatchTemplateId,
|
|
'revision' => 1,
|
|
'generation_token' => $invalidDispatchGeneration,
|
|
'state' => 'importing',
|
|
'image_bytes' => 0,
|
|
'sha256' => $invalidDispatchSha,
|
|
'operation_id' => (string) CloudJob::findOrFail(
|
|
$invalidDispatchChildId
|
|
)->operation_id,
|
|
'job_id' => $invalidDispatchChildId,
|
|
'error_code' => '',
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
1,
|
|
$invalidDispatchChildId,
|
|
1
|
|
) === 1, 'The invalid template child deadline was not claimed.');
|
|
$invalidDispatchChild = CloudJob::findOrFail(
|
|
$invalidDispatchChildId
|
|
);
|
|
$invalidDispatchParent = CloudJob::findOrFail(
|
|
$invalidDispatchParentId
|
|
);
|
|
$invalidDispatchParentRecovery = CloudJobService::recoveryPresentation(
|
|
$invalidDispatchParent,
|
|
true
|
|
);
|
|
$assert((string) $invalidDispatchChild->effect_resolution === 'unknown'
|
|
&& (bool) $invalidDispatchChild->effect_may_have_applied
|
|
&& !(bool) $invalidDispatchChild->safe_to_resubmit
|
|
&& (bool) $invalidDispatchChild->finalization_incident
|
|
&& (string) $invalidDispatchChild->finalization_error_code
|
|
=== 'recovery_dispatch_ledger_conflict'
|
|
&& (string) $invalidDispatchChild->sealed_payload
|
|
=== $invalidDispatchPayload
|
|
&& (string) $invalidDispatchParent->effect_resolution
|
|
!== 'not_applied'
|
|
&& (string) Capsule::table('cloud_templates')
|
|
->where('id', $invalidDispatchTemplateId)->value('state')
|
|
=== 'importing'
|
|
&& (string) Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $invalidDispatchTemplateId)
|
|
->value('state') === 'importing'
|
|
&& ($invalidDispatchParentRecovery['code'] ?? null)
|
|
=== 'recovery_dispatch_ledger_conflict',
|
|
'An invalid template child released its catalog or aggregate as no-effect.');
|
|
|
|
// Ordinary receipt finalization runs before the currently completing child
|
|
// is made terminal. That current transition is allowed, but a different
|
|
// sibling with an invalid recovery ledger must keep the public catalog and
|
|
// aggregate parent fenced. Once the sibling is repaired, the normal child
|
|
// recovery hook publishes the already-prepared replicas.
|
|
$normalGateParentId = $insertParent('template_import');
|
|
Capsule::table('cloud_jobs')->where('id', $normalGateParentId)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'available_at' => 0,
|
|
'receipt_deadline_at' => $now + 600,
|
|
]);
|
|
$normalGateSiblingId = $insertResolvedChild(
|
|
$normalGateParentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
Capsule::table('cloud_jobs')->where(
|
|
'id',
|
|
$normalGateSiblingId
|
|
)->update(['dispatch_attempts' => 2]);
|
|
$normalGateCurrentId = $insertResolvedChild(
|
|
$normalGateParentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
Capsule::table('cloud_jobs')->where('id', $normalGateCurrentId)->update([
|
|
'host_id' => 8,
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
]);
|
|
$normalGateTemplateId = 'normal-settle-gated-template';
|
|
$normalGateGeneration = str_repeat('1', 32);
|
|
$normalGateSha = str_repeat('2', 64);
|
|
Capsule::table('cloud_templates')->insert([
|
|
'id' => $normalGateTemplateId,
|
|
'name' => 'Normal settle gated',
|
|
'os_family' => 'Debian',
|
|
'os_version' => '13',
|
|
'source_type' => 'qcow2',
|
|
'sealed_source_url' => 'test-only',
|
|
'image_bytes' => 8192,
|
|
'sha256' => $normalGateSha,
|
|
'startup_script' => '',
|
|
'revision' => 1,
|
|
'generation_token' => $normalGateGeneration,
|
|
'state' => 'importing',
|
|
'active_job_id' => $normalGateParentId,
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
foreach ([
|
|
[7, $normalGateSiblingId, 'active', 8192],
|
|
[8, $normalGateCurrentId, 'importing', 0],
|
|
] as [$hostId, $jobId, $state, $imageBytes]) {
|
|
Capsule::table('cloud_template_replicas')->insert([
|
|
'host_id' => $hostId,
|
|
'template_id' => $normalGateTemplateId,
|
|
'revision' => 1,
|
|
'generation_token' => $normalGateGeneration,
|
|
'state' => $state,
|
|
'image_bytes' => $imageBytes,
|
|
'sha256' => $normalGateSha,
|
|
'operation_id' => (string) CloudJob::findOrFail($jobId)
|
|
->operation_id,
|
|
'job_id' => $jobId,
|
|
'error_code' => '',
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
}
|
|
CloudTemplateCatalogService::finishJobLocked(
|
|
CloudJob::findOrFail($normalGateCurrentId),
|
|
'succeeded',
|
|
[
|
|
'id' => $normalGateTemplateId,
|
|
'revision' => 1,
|
|
'generation_token' => $normalGateGeneration,
|
|
'size_bytes' => 8192,
|
|
'sha256' => $normalGateSha,
|
|
],
|
|
['template_id' => $normalGateTemplateId],
|
|
[
|
|
'status' => 'succeeded',
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => false,
|
|
'error_code' => '',
|
|
]
|
|
);
|
|
$normalGateParent = CloudJob::findOrFail($normalGateParentId);
|
|
$assert((string) $normalGateParent->phase === 'awaiting_receipt'
|
|
&& (string) Capsule::table('cloud_templates')
|
|
->where('id', $normalGateTemplateId)->value('state')
|
|
=== 'importing'
|
|
&& Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $normalGateTemplateId)
|
|
->where('state', 'active')->count() === 2,
|
|
'Normal completion published a parent with an unsafe sibling.');
|
|
Capsule::table('cloud_jobs')->where('id', $normalGateSiblingId)->update([
|
|
'dispatch_attempts' => 1,
|
|
]);
|
|
Capsule::table('cloud_jobs')->where('id', $normalGateCurrentId)->update([
|
|
'phase' => 'terminal',
|
|
'outcome' => 'succeeded',
|
|
'status' => 'succeeded',
|
|
'terminal_at' => time(),
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'applied',
|
|
'effect_resolved_at' => time(),
|
|
'effect_resolution_reason' => 'normal_completion',
|
|
]);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($normalGateSiblingId),
|
|
null
|
|
);
|
|
$normalGateParent->refresh();
|
|
$assert((string) $normalGateParent->phase === 'terminal'
|
|
&& (string) $normalGateParent->outcome === 'succeeded'
|
|
&& (string) $normalGateParent->effect_resolution === 'applied'
|
|
&& (string) Capsule::table('cloud_templates')
|
|
->where('id', $normalGateTemplateId)->value('state') === 'active',
|
|
'The recovery hook did not publish a safe repaired aggregate.');
|
|
|
|
// The normal last-child receipt must use the same strict projection gate
|
|
// as late recovery. Missing, extra or cross-wired sibling replicas may be
|
|
// projected locally, but can never publish the aggregate parent.
|
|
foreach ([
|
|
'missing sibling replica' => [
|
|
'mutation' => 'missing',
|
|
'exception' => CloudRecoveryAggregateStateConflict::class,
|
|
],
|
|
'extra sibling replica' => [
|
|
'mutation' => 'extra',
|
|
'exception' => CloudRecoveryAggregateStateConflict::class,
|
|
],
|
|
'cross-wired sibling operation' => [
|
|
'mutation' => 'mapping',
|
|
'exception' => CloudRecoveryOperationIdentityConflict::class,
|
|
],
|
|
] as $description => $fixture) {
|
|
$parentId = $insertParent('template_import');
|
|
$siblingId = $insertResolvedChild(
|
|
$parentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
$currentId = $insertResolvedChild(
|
|
$parentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
$resourceId = $insertProjectionForParent($parentId);
|
|
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
'error_code' => null,
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'available_at' => 0,
|
|
]);
|
|
Capsule::table('cloud_jobs')->where('id', $currentId)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
]);
|
|
Capsule::table('cloud_templates')->where('id', $resourceId)->update([
|
|
'state' => 'importing',
|
|
]);
|
|
Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)
|
|
->where('job_id', $currentId)->update([
|
|
'state' => 'importing',
|
|
'image_bytes' => 0,
|
|
]);
|
|
$template = Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->first();
|
|
CloudTemplateCatalogService::finishJobLocked(
|
|
CloudJob::findOrFail($currentId),
|
|
'succeeded',
|
|
[
|
|
'id' => $resourceId,
|
|
'revision' => (int) $template->revision,
|
|
'generation_token' => (string) $template->generation_token,
|
|
'size_bytes' => (int) $template->image_bytes,
|
|
'sha256' => (string) $template->sha256,
|
|
],
|
|
['template_id' => $resourceId],
|
|
[]
|
|
);
|
|
if ($fixture['mutation'] === 'missing') {
|
|
Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)
|
|
->where('job_id', $siblingId)->delete();
|
|
} elseif ($fixture['mutation'] === 'extra') {
|
|
$extra = (array) Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)
|
|
->where('job_id', $siblingId)->first();
|
|
$extra['host_id'] = 900000 + $parentId;
|
|
$extra['job_id'] = null;
|
|
$extra['operation_id'] = md5('normal-extra:' . $parentId);
|
|
Capsule::table('cloud_template_replicas')->insert($extra);
|
|
} else {
|
|
Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)
|
|
->where('job_id', $siblingId)->update([
|
|
'operation_id' => md5('normal-cross-wire:' . $parentId),
|
|
]);
|
|
}
|
|
Capsule::table('cloud_jobs')->where('id', $currentId)->update([
|
|
'phase' => 'terminal',
|
|
'outcome' => 'succeeded',
|
|
'status' => 'succeeded',
|
|
'terminal_at' => time(),
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'applied',
|
|
'effect_resolved_at' => time(),
|
|
'effect_resolution_reason' => 'normal_completion',
|
|
]);
|
|
try {
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($currentId),
|
|
null
|
|
);
|
|
throw new RuntimeException(
|
|
'Normal aggregate accepted ' . $description . '.'
|
|
);
|
|
} catch (CloudError $error) {
|
|
$assert($error instanceof $fixture['exception'],
|
|
'Normal aggregate reported the wrong conflict for '
|
|
. $description . '.');
|
|
}
|
|
$assert((string) CloudJob::findOrFail($parentId)->phase
|
|
=== 'awaiting_receipt'
|
|
&& (string) Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->value('state') === 'importing',
|
|
'Normal aggregate published after ' . $description . '.');
|
|
}
|
|
|
|
// Exercise the actual ordinary completion order for all five catalog
|
|
// aggregates: project the final failed replica, persist its terminal
|
|
// no-effect conclusion, then let the shared strict reconciler restore the
|
|
// safe resource boundary.
|
|
foreach ([
|
|
'template_import' => 'template_import_replica',
|
|
'template_metadata_sync' => 'template_metadata_replica',
|
|
'template_delete' => 'template_delete_replica',
|
|
'user_iso_import' => 'user_iso_import_replica',
|
|
'user_iso_delete' => 'user_iso_delete_replica',
|
|
] as $parentKind => $childKind) {
|
|
$parentId = $insertParent($parentKind);
|
|
$childId = $insertResolvedChild(
|
|
$parentId,
|
|
'not_applied',
|
|
$childKind
|
|
);
|
|
$resourceId = $insertProjectionForParent($parentId);
|
|
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
'error_code' => null,
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'available_at' => 0,
|
|
]);
|
|
Capsule::table('cloud_jobs')->where('id', $childId)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
]);
|
|
if (str_starts_with($parentKind, 'template_')) {
|
|
$resourceState = $parentKind === 'template_delete'
|
|
? 'deleting' : ($parentKind === 'template_import'
|
|
? 'importing' : 'active');
|
|
$replicaState = match ($parentKind) {
|
|
'template_import' => 'importing',
|
|
'template_metadata_sync' => 'syncing',
|
|
'template_delete' => 'deleting',
|
|
};
|
|
Capsule::table('cloud_templates')->where('id', $resourceId)
|
|
->update(['state' => $resourceState]);
|
|
Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->update([
|
|
'state' => $replicaState,
|
|
]);
|
|
$template = Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->first();
|
|
CloudTemplateCatalogService::finishJobLocked(
|
|
CloudJob::findOrFail($childId),
|
|
'failed',
|
|
null,
|
|
['template_id' => $resourceId],
|
|
[
|
|
'status' => 'failed',
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => true,
|
|
'error_code' => 'operation_not_applied',
|
|
]
|
|
);
|
|
} else {
|
|
$resourceState = $parentKind === 'user_iso_delete'
|
|
? 'deleting' : 'importing';
|
|
$replicaState = $parentKind === 'user_iso_delete'
|
|
? 'deleting' : 'importing';
|
|
Capsule::table('cloud_user_isos')->where('id', $resourceId)
|
|
->update(['state' => $resourceState]);
|
|
Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $resourceId)->update([
|
|
'state' => $replicaState,
|
|
]);
|
|
CloudUserIsoService::finishJobLocked(
|
|
CloudJob::findOrFail($childId),
|
|
'failed',
|
|
null,
|
|
['user_iso_id' => $resourceId],
|
|
[
|
|
'status' => 'failed',
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => true,
|
|
'error_code' => 'operation_not_applied',
|
|
]
|
|
);
|
|
}
|
|
Capsule::table('cloud_jobs')->where('id', $childId)->update([
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'status' => 'failed',
|
|
'terminal_at' => time(),
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 1,
|
|
'effect_resolution' => 'not_applied',
|
|
'effect_resolved_at' => time(),
|
|
'effect_resolution_reason' => 'normal_completion',
|
|
'error_code' => 'operation_rejected',
|
|
]);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($childId),
|
|
null
|
|
);
|
|
$parent = CloudJob::findOrFail($parentId);
|
|
$projectionOkay = match ($parentKind) {
|
|
'template_import' =>
|
|
(string) Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->value('state') === 'retired'
|
|
&& !Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->exists(),
|
|
'template_metadata_sync' =>
|
|
(string) Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->value('state') === 'active'
|
|
&& (string) Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->value('state')
|
|
=== 'stale',
|
|
'template_delete' =>
|
|
(string) Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->value('state') === 'active'
|
|
&& (string) Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->value('state')
|
|
=== 'active',
|
|
'user_iso_import' =>
|
|
(string) Capsule::table('cloud_user_isos')
|
|
->where('id', $resourceId)->value('state') === 'retired'
|
|
&& !Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $resourceId)->exists(),
|
|
'user_iso_delete' =>
|
|
(string) Capsule::table('cloud_user_isos')
|
|
->where('id', $resourceId)->value('state') === 'active'
|
|
&& (string) Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $resourceId)->value('state')
|
|
=== 'active',
|
|
};
|
|
if ($parentKind === 'template_metadata_sync') {
|
|
$publicIds = array_column(
|
|
CloudTemplateCatalogService::publicCatalog()['templates'],
|
|
'id'
|
|
);
|
|
$projectionOkay = $projectionOkay
|
|
&& !in_array($resourceId, $publicIds, true);
|
|
}
|
|
$assert((string) $parent->phase === 'terminal'
|
|
&& (string) $parent->outcome === 'failed'
|
|
&& (string) $parent->effect_resolution === 'not_applied'
|
|
&& !(bool) $parent->finalization_incident
|
|
&& $projectionOkay,
|
|
'Normal all-not-applied completion did not restore '
|
|
. $parentKind . '.');
|
|
}
|
|
|
|
// The ordinary last-child path must lock the one-and-only main catalog
|
|
// record before publishing its aggregate parent. A duplicate binding is
|
|
// data corruption, not permission to complete one arbitrary record.
|
|
foreach ([
|
|
'template_import' => 'template_import_replica',
|
|
'user_iso_import' => 'user_iso_import_replica',
|
|
] as $parentKind => $childKind) {
|
|
$parentId = $insertParent($parentKind);
|
|
$childId = $insertResolvedChild($parentId, 'applied', $childKind);
|
|
$resourceId = $insertProjectionForParent($parentId);
|
|
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
'error_code' => null,
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
]);
|
|
Capsule::table('cloud_jobs')->where('id', $childId)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
]);
|
|
if ($parentKind === 'template_import') {
|
|
Capsule::table('cloud_templates')->where('id', $resourceId)
|
|
->update(['state' => 'importing']);
|
|
Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->update([
|
|
'state' => 'importing',
|
|
'image_bytes' => 0,
|
|
]);
|
|
$duplicate = (array) Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->first();
|
|
$duplicate['id'] = $resourceId . '-normal-duplicate';
|
|
Capsule::table('cloud_templates')->insert($duplicate);
|
|
$template = Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->first();
|
|
CloudTemplateCatalogService::finishJobLocked(
|
|
CloudJob::findOrFail($childId),
|
|
'succeeded',
|
|
[
|
|
'id' => $resourceId,
|
|
'revision' => (int) $template->revision,
|
|
'generation_token' =>
|
|
(string) $template->generation_token,
|
|
'size_bytes' => (int) $template->image_bytes,
|
|
'sha256' => (string) $template->sha256,
|
|
],
|
|
['template_id' => $resourceId],
|
|
[]
|
|
);
|
|
} else {
|
|
Capsule::table('cloud_user_isos')->where('id', $resourceId)
|
|
->update(['state' => 'importing']);
|
|
Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $resourceId)->update([
|
|
'state' => 'importing',
|
|
'size_bytes' => 0,
|
|
]);
|
|
$duplicate = (array) Capsule::table('cloud_user_isos')
|
|
->where('id', $resourceId)->first();
|
|
$duplicate['id'] = md5('normal-duplicate:' . $parentId);
|
|
Capsule::table('cloud_user_isos')->insert($duplicate);
|
|
$iso = Capsule::table('cloud_user_isos')
|
|
->where('id', $resourceId)->first();
|
|
CloudUserIsoService::finishJobLocked(
|
|
CloudJob::findOrFail($childId),
|
|
'succeeded',
|
|
[
|
|
'id' => $resourceId,
|
|
'owner_id' => (int) $iso->user_id,
|
|
'name' => (string) $iso->name,
|
|
'size_bytes' => (int) $iso->size_bytes,
|
|
'sha256' => (string) $iso->sha256,
|
|
],
|
|
['user_iso_id' => $resourceId],
|
|
[]
|
|
);
|
|
}
|
|
Capsule::table('cloud_jobs')->where('id', $childId)->update([
|
|
'phase' => 'terminal',
|
|
'outcome' => 'succeeded',
|
|
'status' => 'succeeded',
|
|
'terminal_at' => time(),
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'applied',
|
|
'effect_resolved_at' => time(),
|
|
'effect_resolution_reason' => 'normal_completion',
|
|
]);
|
|
try {
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($childId),
|
|
null
|
|
);
|
|
throw new RuntimeException(
|
|
'Normal catalog settle accepted duplicate main records.'
|
|
);
|
|
} catch (CloudRecoveryAggregateStateConflict $expected) {
|
|
// Expected: the child projection exists, but the parent remains
|
|
// fenced until its one-and-only main record is repairable.
|
|
}
|
|
$assert((string) CloudJob::findOrFail($parentId)->phase
|
|
=== 'awaiting_receipt',
|
|
'A duplicate main record completed its normal aggregate parent.');
|
|
}
|
|
foreach ([
|
|
'template_delete' => [
|
|
CloudTemplateCatalogService::class,
|
|
'template_delete_replica',
|
|
'cloud_templates',
|
|
],
|
|
'user_iso_delete' => [
|
|
CloudUserIsoService::class,
|
|
'user_iso_delete_replica',
|
|
'cloud_user_isos',
|
|
],
|
|
] as $parentKind => [$service, $childKind, $table]) {
|
|
$parentId = $insertParent($parentKind);
|
|
$childId = $insertResolvedChild($parentId, 'applied', $childKind);
|
|
$resourceId = $insertProjectionForParent($parentId);
|
|
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
'error_code' => null,
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
]);
|
|
Capsule::table('cloud_jobs')->where('id', $childId)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
]);
|
|
$duplicate = (array) Capsule::table($table)
|
|
->where('id', $resourceId)->first();
|
|
$duplicate['id'] = $parentKind === 'template_delete'
|
|
? $resourceId . '-normal-duplicate'
|
|
: md5('normal-delete-duplicate:' . $parentId);
|
|
Capsule::table($table)->insert($duplicate);
|
|
$resource = $parentKind === 'template_delete'
|
|
? \App\Models\CloudTemplate::findOrFail($resourceId)
|
|
: \App\Models\CloudUserIso::findOrFail($resourceId);
|
|
$settleDelete = new ReflectionMethod($service, 'settleDeleteParent');
|
|
try {
|
|
$settleDelete->invoke(
|
|
null,
|
|
$resource,
|
|
$parentId,
|
|
['effect_may_have_applied' => true],
|
|
$childId
|
|
);
|
|
throw new RuntimeException(
|
|
'Normal delete settle accepted duplicate main records.'
|
|
);
|
|
} catch (CloudRecoveryAggregateStateConflict $expected) {
|
|
// Expected.
|
|
}
|
|
$assert((string) CloudJob::findOrFail($parentId)->phase
|
|
=== 'awaiting_receipt',
|
|
'A duplicate main record completed its normal delete parent.');
|
|
}
|
|
|
|
// ISO identity reconstruction must bind both import and delete children to
|
|
// the expected aggregate root, owner and active main record before any
|
|
// normal receipt can reach catalog finalization.
|
|
$expectIsoIdentityConflict = static function (
|
|
CloudJob $child,
|
|
array $payload
|
|
): void {
|
|
try {
|
|
CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
$child,
|
|
$payload
|
|
);
|
|
throw new RuntimeException(
|
|
'Cross-wired ISO aggregate identity was accepted.'
|
|
);
|
|
} catch (CloudRecoveryInstanceStateConflict $expected) {
|
|
// Expected.
|
|
}
|
|
};
|
|
$identityImportParent = $insertParent('user_iso_import');
|
|
$identityImportChild = $insertResolvedChild(
|
|
$identityImportParent,
|
|
'applied',
|
|
'user_iso_import_replica'
|
|
);
|
|
$identityImportIso = $insertProjectionForParent($identityImportParent);
|
|
Capsule::table('cloud_user_isos')->where('id', $identityImportIso)
|
|
->update(['active_job_id' => $identityImportParent + 900000]);
|
|
$identityImportRow = Capsule::table('cloud_user_isos')
|
|
->where('id', $identityImportIso)->first();
|
|
$expectIsoIdentityConflict(
|
|
CloudJob::findOrFail($identityImportChild),
|
|
[
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/user-isos',
|
|
'body' => [
|
|
'id' => $identityImportIso,
|
|
'owner_id' => 1,
|
|
'name' => (string) $identityImportRow->name,
|
|
],
|
|
'user_iso_id' => $identityImportIso,
|
|
'owner_id' => 1,
|
|
]
|
|
);
|
|
|
|
$identityDeleteParent = $insertParent('user_iso_delete');
|
|
$identityDeleteChild = $insertResolvedChild(
|
|
$identityDeleteParent,
|
|
'applied',
|
|
'user_iso_delete_replica'
|
|
);
|
|
$identityDeleteIso = $insertProjectionForParent($identityDeleteParent);
|
|
Capsule::table('cloud_jobs')->where('id', $identityDeleteParent)
|
|
->update(['user_id' => 2]);
|
|
$expectIsoIdentityConflict(
|
|
CloudJob::findOrFail($identityDeleteChild),
|
|
[
|
|
'method' => 'DELETE',
|
|
'request_target' => '/v1/user-isos/'
|
|
. rawurlencode($identityDeleteIso) . '?owner_id=1',
|
|
'body' => [],
|
|
'user_iso_id' => $identityDeleteIso,
|
|
'owner_id' => 1,
|
|
]
|
|
);
|
|
|
|
// A historical fail-open parent can already say succeeded/applied while a
|
|
// child remains unsafe. Such a template must disappear from the public
|
|
// catalog and be rejected by every provisioning metadata boundary until
|
|
// the child ledger is repaired.
|
|
$historicalParentId = $insertParent('template_import');
|
|
Capsule::table('cloud_jobs')->where('id', $historicalParentId)->update([
|
|
'outcome' => 'succeeded',
|
|
'status' => 'succeeded',
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'applied',
|
|
'effect_resolved_at' => $now - 1,
|
|
'effect_resolution_reason' => 'historical_fail_open',
|
|
'error_code' => null,
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
]);
|
|
$historicalChildId = $insertResolvedChild(
|
|
$historicalParentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
Capsule::table('cloud_jobs')->where('id', $historicalChildId)->update([
|
|
'dispatch_attempts' => 2,
|
|
]);
|
|
$historicalTemplateId = 'historical-fail-open-template';
|
|
$historicalGeneration = str_repeat('3', 32);
|
|
$historicalSha = str_repeat('4', 64);
|
|
Capsule::table('cloud_templates')->insert([
|
|
'id' => $historicalTemplateId,
|
|
'name' => 'Historical fail open',
|
|
'os_family' => 'Debian',
|
|
'os_version' => '13',
|
|
'source_type' => 'qcow2',
|
|
'sealed_source_url' => 'test-only',
|
|
'image_bytes' => 4096,
|
|
'sha256' => $historicalSha,
|
|
'startup_script' => '',
|
|
'revision' => 1,
|
|
'generation_token' => $historicalGeneration,
|
|
'state' => 'active',
|
|
'active_job_id' => $historicalParentId,
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_template_replicas')->insert([
|
|
'host_id' => 7,
|
|
'template_id' => $historicalTemplateId,
|
|
'revision' => 1,
|
|
'generation_token' => $historicalGeneration,
|
|
'state' => 'active',
|
|
'image_bytes' => 4096,
|
|
'sha256' => $historicalSha,
|
|
'operation_id' => (string) CloudJob::findOrFail($historicalChildId)
|
|
->operation_id,
|
|
'job_id' => $historicalChildId,
|
|
'error_code' => '',
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
$publicIds = array_column(
|
|
CloudTemplateCatalogService::publicCatalog()['templates'],
|
|
'id'
|
|
);
|
|
$assert(!in_array($historicalTemplateId, $publicIds, true),
|
|
'A fail-open template remained visible in the public catalog.');
|
|
$expectUnsafeTemplate = static function (
|
|
callable $callable,
|
|
string $message
|
|
) use ($assert): void {
|
|
try {
|
|
$callable();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === 409, $message);
|
|
return;
|
|
}
|
|
throw new RuntimeException($message);
|
|
};
|
|
$expectUnsafeTemplate(
|
|
static fn (): array =>
|
|
CloudTemplateCatalogService::applyProvisioningSnapshot([
|
|
'template_id' => $historicalTemplateId,
|
|
]),
|
|
'Provisioning consumed a fail-open template.'
|
|
);
|
|
$expectUnsafeTemplate(
|
|
static fn (): array =>
|
|
CloudTemplateCatalogService::templateMetadataSnapshot([
|
|
'template_id' => $historicalTemplateId,
|
|
'template_revision' => 1,
|
|
'template_generation_token' => $historicalGeneration,
|
|
]),
|
|
'Metadata validation consumed a fail-open template.'
|
|
);
|
|
$expectUnsafeTemplate(
|
|
static function () use ($historicalTemplateId): void {
|
|
CloudTemplateCatalogService::assertProvisionable(
|
|
$historicalTemplateId
|
|
);
|
|
},
|
|
'Provisionability accepted a fail-open template.'
|
|
);
|
|
Capsule::table('cloud_jobs')->where('id', $historicalChildId)->update([
|
|
'dispatch_attempts' => 1,
|
|
]);
|
|
$publicIds = array_column(
|
|
CloudTemplateCatalogService::publicCatalog()['templates'],
|
|
'id'
|
|
);
|
|
$snapshot = CloudTemplateCatalogService::applyProvisioningSnapshot([
|
|
'template_id' => $historicalTemplateId,
|
|
]);
|
|
$metadata = CloudTemplateCatalogService::templateMetadataSnapshot([
|
|
'template_id' => $historicalTemplateId,
|
|
'template_revision' => 1,
|
|
'template_generation_token' => $historicalGeneration,
|
|
]);
|
|
CloudTemplateCatalogService::assertProvisionable($historicalTemplateId);
|
|
$assert(in_array($historicalTemplateId, $publicIds, true)
|
|
&& ($snapshot['template_id'] ?? null) === $historicalTemplateId
|
|
&& ($metadata['template_id'] ?? null) === $historicalTemplateId,
|
|
'A safely repaired template remained unavailable.');
|
|
|
|
// Uniform not-applied evidence is a safe rollback boundary. Imports retire
|
|
// their unused placeholders; deletes restore their prior usable records;
|
|
// metadata keeps the panel revision while old host revisions remain stale.
|
|
foreach ([
|
|
'template_import' => 'template_import_replica',
|
|
'template_metadata_sync' => 'template_metadata_replica',
|
|
'template_delete' => 'template_delete_replica',
|
|
'user_iso_import' => 'user_iso_import_replica',
|
|
'user_iso_delete' => 'user_iso_delete_replica',
|
|
] as $parentKind => $childKind) {
|
|
$parentId = $insertParent($parentKind);
|
|
$childId = $insertResolvedChild(
|
|
$parentId,
|
|
'not_applied',
|
|
$childKind
|
|
);
|
|
$resourceId = $insertProjectionForParent($parentId);
|
|
$parent = $markDeadlineAudit($parentId);
|
|
$audit = $immutableParentAudit($parent);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($childId),
|
|
null
|
|
);
|
|
$parent->refresh();
|
|
$projectionOkay = match ($parentKind) {
|
|
'template_import' =>
|
|
(string) Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->value('state') === 'retired'
|
|
&& !Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->exists(),
|
|
'template_metadata_sync' =>
|
|
(string) Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->value('state') === 'active'
|
|
&& (string) Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->value('state')
|
|
=== 'stale'
|
|
&& (int) Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->value('revision')
|
|
< (int) Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->value('revision'),
|
|
'template_delete' =>
|
|
(string) Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->value('state') === 'active'
|
|
&& (string) Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->value('state')
|
|
=== 'active',
|
|
'user_iso_import' =>
|
|
(string) Capsule::table('cloud_user_isos')
|
|
->where('id', $resourceId)->value('state') === 'retired'
|
|
&& (int) Capsule::table('cloud_user_isos')
|
|
->where('id', $resourceId)->value('size_bytes') === 0
|
|
&& (string) Capsule::table('cloud_user_isos')
|
|
->where('id', $resourceId)->value('name') === ''
|
|
&& !Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $resourceId)->exists(),
|
|
'user_iso_delete' =>
|
|
(string) Capsule::table('cloud_user_isos')
|
|
->where('id', $resourceId)->value('state') === 'active'
|
|
&& (string) Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $resourceId)->value('state')
|
|
=== 'active',
|
|
};
|
|
if ($parentKind === 'template_metadata_sync') {
|
|
$publicIds = array_column(
|
|
CloudTemplateCatalogService::publicCatalog()['templates'],
|
|
'id'
|
|
);
|
|
$snapshotBlocked = false;
|
|
try {
|
|
CloudTemplateCatalogService::applyProvisioningSnapshot([
|
|
'template_id' => $resourceId,
|
|
]);
|
|
} catch (CloudError $error) {
|
|
$snapshotBlocked = $error->httpStatus === 409;
|
|
}
|
|
$assertBlocked = false;
|
|
try {
|
|
CloudTemplateCatalogService::assertProvisionable($resourceId);
|
|
} catch (CloudError $error) {
|
|
$assertBlocked = $error->httpStatus === 409;
|
|
}
|
|
$projectionOkay = $projectionOkay
|
|
&& !in_array($resourceId, $publicIds, true)
|
|
&& $snapshotBlocked
|
|
&& $assertBlocked;
|
|
}
|
|
$assert($immutableParentAudit($parent) === $audit
|
|
&& (string) $parent->effect_resolution === 'not_applied'
|
|
&& !(bool) $parent->finalization_incident
|
|
&& (int) $parent->available_at === 0
|
|
&& $projectionOkay,
|
|
'Uniform not-applied recovery did not unlock ' . $parentKind . '.');
|
|
}
|
|
|
|
$insertMixedMetadata = static function (
|
|
bool $terminal,
|
|
bool $legacy
|
|
) use (
|
|
$insertParent,
|
|
$insertResolvedChild,
|
|
$insertProjectionForParent,
|
|
$makeLegacyEligible,
|
|
$now
|
|
): array {
|
|
$parentId = $insertParent('template_metadata_sync');
|
|
if (!$terminal) {
|
|
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
'error_code' => null,
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'available_at' => $now,
|
|
]);
|
|
} elseif ($legacy) {
|
|
$makeLegacyEligible($parentId);
|
|
}
|
|
$appliedId = $insertResolvedChild(
|
|
$parentId,
|
|
'applied',
|
|
'template_metadata_replica'
|
|
);
|
|
$notAppliedId = $insertResolvedChild(
|
|
$parentId,
|
|
'not_applied',
|
|
'template_metadata_replica'
|
|
);
|
|
$resourceId = $insertProjectionForParent($parentId);
|
|
return [$parentId, $appliedId, $notAppliedId, $resourceId];
|
|
};
|
|
$assertMixedMetadata = static function (
|
|
int $parentId,
|
|
string $resourceId,
|
|
string $context
|
|
) use ($assert): void {
|
|
$parent = CloudJob::findOrFail($parentId);
|
|
$replicas = Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->orderBy('host_id')->get();
|
|
$publicIds = array_column(
|
|
CloudTemplateCatalogService::publicCatalog()['templates'],
|
|
'id'
|
|
);
|
|
$snapshot = CloudTemplateCatalogService::applyProvisioningSnapshot([
|
|
'template_id' => $resourceId,
|
|
]);
|
|
CloudTemplateCatalogService::assertProvisionable($resourceId);
|
|
$assert((string) $parent->phase === 'terminal'
|
|
&& (string) $parent->outcome === 'failed'
|
|
&& (string) $parent->effect_resolution === 'applied'
|
|
&& !(bool) $parent->finalization_incident
|
|
&& (string) Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->value('state') === 'active'
|
|
&& $replicas->count() === 2
|
|
&& $replicas->contains(static fn ($replica): bool =>
|
|
(string) $replica->state === 'active'
|
|
&& (int) $replica->revision === 2)
|
|
&& $replicas->contains(static fn ($replica): bool =>
|
|
(string) $replica->state === 'stale'
|
|
&& (int) $replica->revision === 1)
|
|
&& in_array($resourceId, $publicIds, true)
|
|
&& ($snapshot['template_id'] ?? null) === $resourceId,
|
|
'Mixed metadata did not converge at the ' . $context . ' seam.');
|
|
};
|
|
|
|
[$realtimeMetadataParent, $realtimeMetadataApplied,
|
|
$_realtimeMetadataAbsent, $realtimeMetadataResource] =
|
|
$insertMixedMetadata(false, false);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($realtimeMetadataApplied),
|
|
null
|
|
);
|
|
$assertMixedMetadata(
|
|
$realtimeMetadataParent,
|
|
$realtimeMetadataResource,
|
|
'realtime'
|
|
);
|
|
|
|
[$terminalMetadataParent, $terminalMetadataApplied,
|
|
$_terminalMetadataAbsent, $terminalMetadataResource] =
|
|
$insertMixedMetadata(true, false);
|
|
$terminalMetadataAudit = $immutableParentAudit(
|
|
$markDeadlineAudit($terminalMetadataParent)
|
|
);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($terminalMetadataApplied),
|
|
null
|
|
);
|
|
$assertMixedMetadata(
|
|
$terminalMetadataParent,
|
|
$terminalMetadataResource,
|
|
'terminal deadline'
|
|
);
|
|
$assert($immutableParentAudit(
|
|
CloudJob::findOrFail($terminalMetadataParent)
|
|
) === $terminalMetadataAudit,
|
|
'Mixed metadata recovery rewrote the deadline parent audit.');
|
|
|
|
[$legacyMetadataParent, $_legacyMetadataApplied,
|
|
$_legacyMetadataAbsent, $legacyMetadataResource] =
|
|
$insertMixedMetadata(true, true);
|
|
$assert(CloudJobService::reconcileOneLegacyAggregateEffect(),
|
|
'Legacy mixed metadata was not inspected.');
|
|
$assertMixedMetadata(
|
|
$legacyMetadataParent,
|
|
$legacyMetadataResource,
|
|
'legacy sweep'
|
|
);
|
|
|
|
// Historical partial imports/deletes use the same evidence-only exit. The
|
|
// legacy sweep must publish the mixed result and release the catalog
|
|
// pointer; it must not leave upgraded installations permanently fenced.
|
|
foreach ([
|
|
'template_import' => 'template_import_replica',
|
|
'template_delete' => 'template_delete_replica',
|
|
'user_iso_import' => 'user_iso_import_replica',
|
|
'user_iso_delete' => 'user_iso_delete_replica',
|
|
] as $parentKind => $childKind) {
|
|
$legacyPartialId = $insertParent($parentKind);
|
|
$makeLegacyEligible($legacyPartialId);
|
|
$insertResolvedChild($legacyPartialId, 'applied', $childKind);
|
|
$insertResolvedChild($legacyPartialId, 'not_applied', $childKind);
|
|
$legacyPartialResourceId = $insertProjectionForParent(
|
|
$legacyPartialId
|
|
);
|
|
$assert(CloudJobService::reconcileOneLegacyAggregateEffect(),
|
|
'Legacy partial aggregate was not inspected: ' . $parentKind);
|
|
$legacyPartial = CloudJob::findOrFail($legacyPartialId);
|
|
$resource = str_starts_with($parentKind, 'template_')
|
|
? Capsule::table('cloud_templates')
|
|
->where('id', $legacyPartialResourceId)->first()
|
|
: Capsule::table('cloud_user_isos')
|
|
->where('id', $legacyPartialResourceId)->first();
|
|
$expectedState = str_ends_with($parentKind, '_import')
|
|
? (str_starts_with($parentKind, 'template_')
|
|
? 'error' : 'import_failed')
|
|
: 'delete_failed';
|
|
$assert((string) $legacyPartial->effect_resolution === 'applied'
|
|
&& str_ends_with(
|
|
(string) $legacyPartial->effect_resolution_reason,
|
|
':mixed'
|
|
)
|
|
&& !(bool) $legacyPartial->finalization_incident
|
|
&& (int) $legacyPartial->available_at === 0
|
|
&& $resource !== null
|
|
&& (string) $resource->state === $expectedState
|
|
&& $resource->active_job_id === null,
|
|
'Legacy partial aggregate stayed fenced: ' . $parentKind);
|
|
}
|
|
|
|
// An all-applied import can exceed quota only after host work finished.
|
|
// Keep it unavailable, expose an explicit cleanup capability, and reserve
|
|
// a brand-new delete aggregate instead of replaying an import child.
|
|
$quotaExistingIsoId = md5('quota-existing-active-iso');
|
|
Capsule::table('cloud_user_isos')->insert([
|
|
'id' => $quotaExistingIsoId,
|
|
'user_id' => 1,
|
|
'name' => 'Existing quota usage',
|
|
'sealed_source_url' => 'test-only',
|
|
'size_bytes' => 6 * 1024 * 1024 * 1024,
|
|
'sha256' => hash('sha256', 'existing-quota-iso'),
|
|
'state' => 'active',
|
|
'import_operation_id' => md5('existing-quota-operation'),
|
|
'active_job_id' => null,
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now,
|
|
]);
|
|
$quotaParentId = $insertParent('user_iso_import');
|
|
Capsule::table('cloud_jobs')->where('id', $quotaParentId)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
'error_code' => null,
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'available_at' => $now,
|
|
'receipt_deadline_at' => $now + 600,
|
|
]);
|
|
$quotaChildId = $insertResolvedChild(
|
|
$quotaParentId,
|
|
'applied',
|
|
'user_iso_import_replica'
|
|
);
|
|
$quotaIsoId = $insertProjectionForParent($quotaParentId);
|
|
$quotaRecoveredBytes = 5 * 1024 * 1024 * 1024;
|
|
Capsule::table('cloud_user_isos')->where('id', $quotaIsoId)->update([
|
|
'size_bytes' => $quotaRecoveredBytes,
|
|
]);
|
|
Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $quotaIsoId)->update([
|
|
'size_bytes' => $quotaRecoveredBytes,
|
|
]);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($quotaChildId),
|
|
null
|
|
);
|
|
$quotaParent = CloudJob::findOrFail($quotaParentId);
|
|
$quotaIso = Capsule::table('cloud_user_isos')
|
|
->where('id', $quotaIsoId)->first();
|
|
$quotaDto = collect(CloudUserIsoService::listing(1)['isos'])
|
|
->firstWhere('id', $quotaIsoId);
|
|
$assert((string) $quotaParent->phase === 'terminal'
|
|
&& (string) $quotaParent->outcome === 'failed'
|
|
&& (string) $quotaParent->error_code
|
|
=== 'user_iso_quota_exceeded'
|
|
&& (string) $quotaParent->effect_resolution === 'applied'
|
|
&& !(bool) $quotaParent->finalization_incident
|
|
&& $quotaIso !== null
|
|
&& (string) $quotaIso->state === 'import_failed'
|
|
&& is_array($quotaDto)
|
|
&& ($quotaDto['can_delete'] ?? false) === true
|
|
&& ($quotaDto['cleanup_reason'] ?? null)
|
|
=== 'quota_exceeded_after_import',
|
|
'Quota-rejected applied ISO import was not exposed for safe cleanup.');
|
|
$cleanup = CloudUserIsoService::delete($quotaIsoId, ['user_id' => 1]);
|
|
$cleanupParentId = (int) ($cleanup['task_id'] ?? 0);
|
|
$cleanupParent = CloudJob::findOrFail($cleanupParentId);
|
|
$quotaIso = Capsule::table('cloud_user_isos')
|
|
->where('id', $quotaIsoId)->first();
|
|
$cleanupChild = CloudJob::where('parent_job_id', $cleanupParentId)->first();
|
|
$assert($cleanupParentId > 0 && $cleanupParentId !== $quotaParentId
|
|
&& (string) $cleanupParent->kind === 'user_iso_delete'
|
|
&& (string) $cleanupParent->phase === 'awaiting_receipt'
|
|
&& $quotaIso !== null
|
|
&& (string) $quotaIso->state === 'deleting'
|
|
&& (int) $quotaIso->active_job_id === $cleanupParentId
|
|
&& $cleanupChild !== null
|
|
&& (string) $cleanupChild->kind === 'user_iso_delete_replica'
|
|
&& (string) Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $quotaIsoId)->value('state') === 'deleting',
|
|
'Quota cleanup did not reserve an independent delete operation.');
|
|
$nextId = max(
|
|
$nextId,
|
|
(int) Capsule::table('cloud_jobs')->max('id') + 1
|
|
);
|
|
|
|
// A deadline keeps its immutable aggregate_deadline_exceeded audit even
|
|
// when every late host later proves the import applied. If quota prevented
|
|
// publication, that exact all-active projection must still be removable
|
|
// by a new delete operation without relying on the parent's old error.
|
|
$deadlineQuotaParentId = $insertParent('user_iso_import');
|
|
$deadlineQuotaAudit = $immutableParentAudit(
|
|
$markDeadlineAudit($deadlineQuotaParentId)
|
|
);
|
|
$deadlineQuotaChildId = $insertResolvedChild(
|
|
$deadlineQuotaParentId,
|
|
'applied',
|
|
'user_iso_import_replica'
|
|
);
|
|
$deadlineQuotaIsoId = $insertProjectionForParent(
|
|
$deadlineQuotaParentId
|
|
);
|
|
Capsule::table('cloud_user_isos')->where(
|
|
'id',
|
|
$deadlineQuotaIsoId
|
|
)->update(['size_bytes' => $quotaRecoveredBytes]);
|
|
Capsule::table('cloud_user_iso_replicas')->where(
|
|
'user_iso_id',
|
|
$deadlineQuotaIsoId
|
|
)->update(['size_bytes' => $quotaRecoveredBytes]);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($deadlineQuotaChildId),
|
|
null
|
|
);
|
|
$deadlineQuotaParent = CloudJob::findOrFail($deadlineQuotaParentId);
|
|
$deadlineQuotaDto = collect(CloudUserIsoService::listing(1)['isos'])
|
|
->firstWhere('id', $deadlineQuotaIsoId);
|
|
$assert($immutableParentAudit($deadlineQuotaParent)
|
|
=== $deadlineQuotaAudit
|
|
&& (string) $deadlineQuotaParent->error_code
|
|
=== 'aggregate_deadline_exceeded'
|
|
&& (string) $deadlineQuotaParent->effect_resolution === 'applied'
|
|
&& !(bool) $deadlineQuotaParent->finalization_incident
|
|
&& (string) Capsule::table('cloud_user_isos')
|
|
->where('id', $deadlineQuotaIsoId)->value('state')
|
|
=== 'import_failed'
|
|
&& is_array($deadlineQuotaDto)
|
|
&& ($deadlineQuotaDto['can_delete'] ?? false) === true
|
|
&& ($deadlineQuotaDto['cleanup_reason'] ?? null)
|
|
=== 'quota_exceeded_after_import',
|
|
'Deadline quota recovery lost its audit or safe cleanup capability.');
|
|
|
|
// Projection failure rolls back atomically and delays only that parent. A
|
|
// later valid parent is claimed by the next tick instead of being starved.
|
|
$missingCatalogParentId = $insertParent('template_import');
|
|
$missingCatalogChildId = $insertResolvedChild(
|
|
$missingCatalogParentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
$makeLegacyEligible($missingCatalogParentId);
|
|
$missingCatalogAudit = $immutableParentAudit(
|
|
CloudJob::findOrFail($missingCatalogParentId)
|
|
);
|
|
$validAfterDamageParentId = $insertParent('template_import');
|
|
$insertResolvedChild(
|
|
$validAfterDamageParentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
$validAfterDamageResource = $insertProjectionForParent(
|
|
$validAfterDamageParentId
|
|
);
|
|
$makeLegacyEligible($validAfterDamageParentId);
|
|
$assert($automatic->tick(),
|
|
'The damaged projection was not isolated by the legacy sweep.');
|
|
$missingCatalogParent = CloudJob::findOrFail($missingCatalogParentId);
|
|
$assert($immutableParentAudit($missingCatalogParent)
|
|
=== $missingCatalogAudit
|
|
&& (string) $missingCatalogParent->effect_resolution === 'unknown'
|
|
&& (bool) $missingCatalogParent->finalization_incident
|
|
&& (string) $missingCatalogParent->finalization_error_code
|
|
=== 'recovery_instance_state_conflict'
|
|
&& (int) $missingCatalogParent->available_at >= time() + 290
|
|
&& CloudJob::findOrFail($missingCatalogChildId)->effect_resolution
|
|
=== 'applied',
|
|
'A missing catalog escaped or rewrote the terminal parent audit.');
|
|
$assert($automatic->tick(),
|
|
'A damaged projection starved the following valid aggregate.');
|
|
$validAfterDamageParent = CloudJob::findOrFail(
|
|
$validAfterDamageParentId
|
|
);
|
|
$assert((string) $validAfterDamageParent->effect_resolution === 'applied'
|
|
&& !(bool) $validAfterDamageParent->finalization_incident
|
|
&& (string) Capsule::table('cloud_templates')
|
|
->where('id', $validAfterDamageResource)->value('state')
|
|
=== 'active',
|
|
'The valid aggregate after a damaged parent did not converge.');
|
|
|
|
$projectionConflictFixtures = [];
|
|
foreach ([
|
|
'missing replica' => 'missing',
|
|
'extra replica' => 'extra',
|
|
'residual importing replica' => 'pending',
|
|
'cross-wired operation' => 'mapping',
|
|
] as $description => $mutation) {
|
|
$parentId = $insertParent('template_import');
|
|
$insertResolvedChild(
|
|
$parentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
$resourceId = $insertProjectionForParent($parentId);
|
|
if ($mutation === 'missing') {
|
|
Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->delete();
|
|
} elseif ($mutation === 'extra') {
|
|
$extra = (array) Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->first();
|
|
$extra['host_id'] = 999000 + $parentId;
|
|
$extra['job_id'] = null;
|
|
$extra['operation_id'] = md5('extra-replica:' . $parentId);
|
|
Capsule::table('cloud_template_replicas')->insert($extra);
|
|
} elseif ($mutation === 'pending') {
|
|
Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)
|
|
->update(['state' => 'importing']);
|
|
} else {
|
|
Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->update([
|
|
'operation_id' => md5('wrong-operation:' . $parentId),
|
|
]);
|
|
}
|
|
$makeLegacyEligible($parentId);
|
|
$projectionConflictFixtures[$parentId] = [
|
|
'description' => $description,
|
|
'code' => in_array($mutation, ['missing', 'extra'], true)
|
|
? 'recovery_aggregate_state_conflict'
|
|
: ($mutation === 'mapping'
|
|
? 'recovery_operation_identity_conflict'
|
|
: 'recovery_instance_state_conflict'),
|
|
];
|
|
}
|
|
|
|
foreach ([
|
|
'template generation mismatch' => 'generation',
|
|
'template import content mismatch' => 'content',
|
|
] as $description => $mutation) {
|
|
$parentId = $insertParent('template_import');
|
|
$insertResolvedChild(
|
|
$parentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
$resourceId = $insertProjectionForParent($parentId);
|
|
Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->update(
|
|
$mutation === 'generation'
|
|
? ['generation_token' => str_repeat('f', 32)]
|
|
: [
|
|
'image_bytes' => 1,
|
|
'sha256' => str_repeat('0', 64),
|
|
]
|
|
);
|
|
$makeLegacyEligible($parentId);
|
|
$projectionConflictFixtures[$parentId] = [
|
|
'description' => $description,
|
|
'code' => 'recovery_instance_state_conflict',
|
|
];
|
|
}
|
|
|
|
$metadataContentParent = $insertParent('template_metadata_sync');
|
|
$insertResolvedChild(
|
|
$metadataContentParent,
|
|
'applied',
|
|
'template_metadata_replica'
|
|
);
|
|
$insertResolvedChild(
|
|
$metadataContentParent,
|
|
'not_applied',
|
|
'template_metadata_replica'
|
|
);
|
|
$metadataContentResource = $insertProjectionForParent(
|
|
$metadataContentParent
|
|
);
|
|
Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $metadataContentResource)
|
|
->where('state', 'active')->update([
|
|
'image_bytes' => 1,
|
|
'sha256' => str_repeat('0', 64),
|
|
]);
|
|
$makeLegacyEligible($metadataContentParent);
|
|
$projectionConflictFixtures[$metadataContentParent] = [
|
|
'description' => 'template metadata mixed content mismatch',
|
|
'code' => 'recovery_instance_state_conflict',
|
|
];
|
|
|
|
$templateDeleteContentParent = $insertParent('template_delete');
|
|
$insertResolvedChild(
|
|
$templateDeleteContentParent,
|
|
'not_applied',
|
|
'template_delete_replica'
|
|
);
|
|
$templateDeleteContentResource = $insertProjectionForParent(
|
|
$templateDeleteContentParent
|
|
);
|
|
Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $templateDeleteContentResource)->update([
|
|
'image_bytes' => 1,
|
|
'sha256' => str_repeat('0', 64),
|
|
]);
|
|
$makeLegacyEligible($templateDeleteContentParent);
|
|
$projectionConflictFixtures[$templateDeleteContentParent] = [
|
|
'description' => 'template delete not-applied content mismatch',
|
|
'code' => 'recovery_instance_state_conflict',
|
|
];
|
|
|
|
foreach ([
|
|
'ISO import parent operation mismatch' => 'parent_operation',
|
|
'ISO import content mismatch' => 'content',
|
|
'ISO active job cross-user binding' => 'user',
|
|
] as $description => $mutation) {
|
|
$parentId = $insertParent('user_iso_import');
|
|
$insertResolvedChild(
|
|
$parentId,
|
|
'applied',
|
|
'user_iso_import_replica'
|
|
);
|
|
$resourceId = $insertProjectionForParent($parentId);
|
|
if ($mutation === 'parent_operation') {
|
|
Capsule::table('cloud_user_isos')->where('id', $resourceId)
|
|
->update([
|
|
'import_operation_id' => md5(
|
|
'wrong-import-parent:' . $parentId
|
|
),
|
|
]);
|
|
} elseif ($mutation === 'content') {
|
|
Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $resourceId)->update([
|
|
'size_bytes' => 1,
|
|
'sha256' => str_repeat('0', 64),
|
|
]);
|
|
} else {
|
|
Capsule::table('cloud_user_isos')->where('id', $resourceId)
|
|
->update(['user_id' => 2]);
|
|
}
|
|
$makeLegacyEligible($parentId);
|
|
$projectionConflictFixtures[$parentId] = [
|
|
'description' => $description,
|
|
'code' => in_array(
|
|
$mutation,
|
|
['parent_operation', 'user'],
|
|
true
|
|
)
|
|
? 'recovery_operation_identity_conflict'
|
|
: 'recovery_instance_state_conflict',
|
|
];
|
|
}
|
|
|
|
$isoDeleteContentParent = $insertParent('user_iso_delete');
|
|
$insertResolvedChild(
|
|
$isoDeleteContentParent,
|
|
'not_applied',
|
|
'user_iso_delete_replica'
|
|
);
|
|
$isoDeleteContentResource = $insertProjectionForParent(
|
|
$isoDeleteContentParent
|
|
);
|
|
Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $isoDeleteContentResource)->update([
|
|
'size_bytes' => 1,
|
|
'sha256' => str_repeat('0', 64),
|
|
]);
|
|
$makeLegacyEligible($isoDeleteContentParent);
|
|
$projectionConflictFixtures[$isoDeleteContentParent] = [
|
|
'description' => 'ISO delete not-applied content mismatch',
|
|
'code' => 'recovery_instance_state_conflict',
|
|
];
|
|
|
|
foreach ([
|
|
'template duplicate main binding' => 'template_import',
|
|
'ISO duplicate main binding' => 'user_iso_import',
|
|
] as $description => $parentKind) {
|
|
$childKind = $parentKind === 'template_import'
|
|
? 'template_import_replica' : 'user_iso_import_replica';
|
|
$parentId = $insertParent($parentKind);
|
|
$insertResolvedChild($parentId, 'applied', $childKind);
|
|
$resourceId = $insertProjectionForParent($parentId);
|
|
if ($parentKind === 'template_import') {
|
|
$duplicate = (array) Capsule::table('cloud_templates')
|
|
->where('id', $resourceId)->first();
|
|
$duplicate['id'] = $resourceId . '-duplicate';
|
|
$duplicate['state'] = 'error';
|
|
Capsule::table('cloud_templates')->insert($duplicate);
|
|
} else {
|
|
$duplicate = (array) Capsule::table('cloud_user_isos')
|
|
->where('id', $resourceId)->first();
|
|
$duplicate['id'] = md5('duplicate-main:' . $parentId);
|
|
$duplicate['state'] = 'retired';
|
|
Capsule::table('cloud_user_isos')->insert($duplicate);
|
|
}
|
|
$makeLegacyEligible($parentId);
|
|
$projectionConflictFixtures[$parentId] = [
|
|
'description' => $description,
|
|
'code' => 'recovery_aggregate_state_conflict',
|
|
];
|
|
}
|
|
|
|
foreach ([
|
|
'template mixed effects forged all-success' => [
|
|
'parent' => 'template_import',
|
|
'child' => 'template_import_replica',
|
|
],
|
|
'ISO delete mixed effects forged all-success' => [
|
|
'parent' => 'user_iso_delete',
|
|
'child' => 'user_iso_delete_replica',
|
|
],
|
|
] as $description => $kinds) {
|
|
$parentId = $insertParent($kinds['parent']);
|
|
$insertResolvedChild($parentId, 'applied', $kinds['child']);
|
|
$insertResolvedChild($parentId, 'not_applied', $kinds['child']);
|
|
$resourceId = $insertProjectionForParent($parentId);
|
|
if ($kinds['parent'] === 'template_import') {
|
|
Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->update([
|
|
'state' => 'active',
|
|
'image_bytes' => 4096,
|
|
]);
|
|
} else {
|
|
Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $resourceId)->update([
|
|
'state' => 'delete_succeeded',
|
|
]);
|
|
}
|
|
$makeLegacyEligible($parentId);
|
|
$projectionConflictFixtures[$parentId] = [
|
|
'description' => $description,
|
|
'code' => 'recovery_instance_state_conflict',
|
|
];
|
|
}
|
|
foreach ($projectionConflictFixtures as $parentId => $expectation) {
|
|
$assert($automatic->tick(),
|
|
'The projection conflict was not inspected: '
|
|
. $expectation['description']);
|
|
$parent = CloudJob::findOrFail($parentId);
|
|
$assert((string) $parent->effect_resolution === 'unknown'
|
|
&& (bool) $parent->finalization_incident
|
|
&& (string) $parent->finalization_error_code
|
|
=== $expectation['code']
|
|
&& (int) $parent->available_at >= time() + 290,
|
|
'The projection conflict escaped its recovery fence: '
|
|
. $expectation['description']);
|
|
}
|
|
$assert(CloudAgentSettingsService::$requests === 0,
|
|
'Aggregate projection recovery replayed an Agent mutation.');
|
|
|
|
// Published parents with no pre-existing incident are still audited while
|
|
// they remain the catalog's current active_job_id. Damage introduced
|
|
// after the original settlement must become a sticky local incident; the
|
|
// immutable parent/child audit is not rewritten and Agent is never called.
|
|
foreach ([
|
|
'mapping' => 'recovery_operation_identity_conflict',
|
|
'content' => 'recovery_instance_state_conflict',
|
|
'state' => 'recovery_instance_state_conflict',
|
|
] as $mutation => $expectedCode) {
|
|
$parentId = $insertParent('template_import');
|
|
$childId = $insertResolvedChild(
|
|
$parentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
$resourceId = $insertProjectionForParent($parentId);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($childId),
|
|
null
|
|
);
|
|
$publishedAudit = $immutableParentAudit(
|
|
CloudJob::findOrFail($parentId)
|
|
);
|
|
$damage = match ($mutation) {
|
|
'mapping' => [
|
|
'operation_id' => md5('published-wrong-op:' . $parentId),
|
|
],
|
|
'content' => [
|
|
'image_bytes' => 1,
|
|
'sha256' => str_repeat('0', 64),
|
|
],
|
|
'state' => ['state' => 'stale'],
|
|
};
|
|
Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $resourceId)->update($damage);
|
|
Capsule::table('cloud_jobs')->where('id', '<>', $parentId)->update([
|
|
'available_at' => time() + 3600,
|
|
'lease_until' => time() + 3600,
|
|
]);
|
|
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
|
|
'available_at' => 0,
|
|
'lease_until' => 0,
|
|
]);
|
|
$beforePublishedAuditRequests = CloudAgentSettingsService::$requests;
|
|
$assert($automatic->tick(),
|
|
'A current published projection was not audited: ' . $mutation);
|
|
$damaged = CloudJob::findOrFail($parentId);
|
|
$assert($immutableParentAudit($damaged) === $publishedAudit
|
|
&& (string) $damaged->effect_resolution === 'applied'
|
|
&& (bool) $damaged->finalization_incident
|
|
&& (string) $damaged->finalization_error_code === $expectedCode
|
|
&& (int) $damaged->available_at >= time() + 290
|
|
&& CloudAgentSettingsService::$requests
|
|
=== $beforePublishedAuditRequests,
|
|
'Published projection damage rewrote audit, disappeared, or contacted Agent: '
|
|
. $mutation);
|
|
}
|
|
|
|
// A resolved child in a nonterminal phase is a PHP-ledger contradiction,
|
|
// not ordinary child convergence. It stays in the same 300-second local
|
|
// loop with the concrete aggregate-state code.
|
|
$phaseConflictParentId = $insertParent('template_import');
|
|
$phaseConflictChildId = $insertResolvedChild(
|
|
$phaseConflictParentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
$insertProjectionForParent($phaseConflictParentId);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($phaseConflictChildId),
|
|
null
|
|
);
|
|
Capsule::table('cloud_jobs')->where('id', $phaseConflictChildId)->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'terminal_at' => null,
|
|
]);
|
|
Capsule::table('cloud_jobs')->where(
|
|
'id',
|
|
'<>',
|
|
$phaseConflictParentId
|
|
)->update([
|
|
'available_at' => time() + 3600,
|
|
'lease_until' => time() + 3600,
|
|
]);
|
|
Capsule::table('cloud_jobs')->where('id', $phaseConflictParentId)->update([
|
|
'available_at' => 0,
|
|
'lease_until' => 0,
|
|
]);
|
|
$phaseBefore = CloudJobService::recoveryPresentation(
|
|
CloudJob::findOrFail($phaseConflictParentId),
|
|
true
|
|
);
|
|
$beforePhaseRequests = CloudAgentSettingsService::$requests;
|
|
$assert(($phaseBefore['code'] ?? null)
|
|
=== 'recovery_aggregate_state_conflict'
|
|
&& $automatic->tick(),
|
|
'A nonterminal child with a final effect was not retained locally.');
|
|
$phaseConflict = CloudJob::findOrFail($phaseConflictParentId);
|
|
$assert((bool) $phaseConflict->finalization_incident
|
|
&& (string) $phaseConflict->finalization_error_code
|
|
=== 'recovery_aggregate_state_conflict'
|
|
&& (int) $phaseConflict->available_at >= time() + 290
|
|
&& CloudAgentSettingsService::$requests === $beforePhaseRequests,
|
|
'A nonterminal/final child lost its concrete conflict or contacted Agent.');
|
|
|
|
// Ordinary leaf recovery has priority over healthy periodic catalog audit,
|
|
// and a successful audit advances available_at so it cannot hot-loop.
|
|
$healthyParentId = $insertParent('template_import');
|
|
$healthyChildId = $insertResolvedChild(
|
|
$healthyParentId,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
$insertProjectionForParent($healthyParentId);
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
CloudJob::findOrFail($healthyChildId),
|
|
null
|
|
);
|
|
$priorityLeafId = $insertResolvedChild(
|
|
0,
|
|
'applied',
|
|
'template_import_replica'
|
|
);
|
|
Capsule::table('cloud_jobs')->where('id', $priorityLeafId)->update([
|
|
'dispatch_attempts' => 0,
|
|
'dispatched_at' => null,
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'status' => 'failed',
|
|
'effect_may_have_applied' => 1,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'unknown',
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
]);
|
|
Capsule::table('cloud_jobs')->update([
|
|
'available_at' => time() + 3600,
|
|
'lease_until' => time() + 3600,
|
|
]);
|
|
Capsule::table('cloud_jobs')->whereIn('id', [
|
|
$healthyParentId,
|
|
$priorityLeafId,
|
|
])->update(['available_at' => 0, 'lease_until' => 0]);
|
|
$assert($automatic->tick(),
|
|
'The ordinary invalid leaf was not preferred over healthy catalog audit.');
|
|
$priorityLeaf = CloudJob::findOrFail($priorityLeafId);
|
|
$healthyBeforeAudit = CloudJob::findOrFail($healthyParentId);
|
|
$assert((string) $priorityLeaf->finalization_error_code
|
|
=== 'recovery_instance_state_conflict'
|
|
&& !(bool) $healthyBeforeAudit->finalization_incident
|
|
&& (int) $healthyBeforeAudit->available_at === 0,
|
|
'Healthy aggregate audit starved an ordinary leaf recovery claim.');
|
|
$assert($automatic->tick(),
|
|
'The healthy current aggregate was not periodically audited.');
|
|
$healthyAfterAudit = CloudJob::findOrFail($healthyParentId);
|
|
$assert(!(bool) $healthyAfterAudit->finalization_incident
|
|
&& (int) $healthyAfterAudit->available_at >= time() + 290,
|
|
'A healthy published aggregate audit hot-looped or created an incident.');
|
|
|
|
// A success receipt arriving after the immutable deadline failure must
|
|
// converge the original delete generation in place. It may update the
|
|
// effect/catalog projection, but it must not flip either failed audit to
|
|
// succeeded or contact the destructive Agent endpoint again.
|
|
$lateDeleteParentId = $insertParent('template_delete');
|
|
$lateDeleteChildId = $insertResolvedChild(
|
|
$lateDeleteParentId,
|
|
'not_applied',
|
|
'template_delete_replica'
|
|
);
|
|
$lateDeleteTemplateId = $insertProjectionForParent(
|
|
$lateDeleteParentId
|
|
);
|
|
$lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId);
|
|
Capsule::table('cloud_jobs')->where('id', $lateDeleteChildId)->update([
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
'method' => 'DELETE',
|
|
'request_target' => '/v1/templates/'
|
|
. rawurlencode($lateDeleteTemplateId),
|
|
'body' => [
|
|
'operation_id' => (string) $lateDeleteChild->operation_id,
|
|
],
|
|
'owner_id' => 1,
|
|
'can_manage' => true,
|
|
'template_id' => $lateDeleteTemplateId,
|
|
], 'job-payload'),
|
|
'effect_may_have_applied' => 1,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'unknown',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
'completion_event_id' => null,
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'recovery_receipt_invalid',
|
|
'available_at' => 0,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
]);
|
|
$lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId);
|
|
$lateDeleteIdentity = CloudOperationRecoveryIdentity::expected(
|
|
$lateDeleteChild,
|
|
true
|
|
);
|
|
$beforeLateDeleteRequests = CloudAgentSettingsService::$requests;
|
|
$lateDeleteAccepted = CloudOperationCompletionService::accept([
|
|
'type' => 'completion',
|
|
'id' => 'aggregate-late-template-delete',
|
|
'agent_id' => (string) $lateDeleteChild->host_id,
|
|
'operation_id' => (string) $lateDeleteChild->operation_id,
|
|
'action' => $lateDeleteIdentity['action'],
|
|
'resource_id' => $lateDeleteIdentity['resource_id'],
|
|
'request_fingerprint' =>
|
|
$lateDeleteIdentity['request_fingerprint'],
|
|
'status' => 'succeeded',
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => false,
|
|
'error_code' => '',
|
|
'result' => [
|
|
'resource_type' => 'template',
|
|
'id' => $lateDeleteTemplateId,
|
|
'deleted' => true,
|
|
'managed_image_absent' => true,
|
|
'managed_metadata_absent' => true,
|
|
'managed_artifacts_absent' => true,
|
|
],
|
|
]);
|
|
$lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId);
|
|
$lateDeleteParent = CloudJob::findOrFail($lateDeleteParentId);
|
|
$assert($lateDeleteAccepted === [
|
|
'accepted' => true,
|
|
'disposition' => 'applied',
|
|
]
|
|
&& (string) $lateDeleteChild->outcome === 'failed'
|
|
&& (string) $lateDeleteChild->effect_resolution === 'applied'
|
|
&& !(bool) $lateDeleteChild->finalization_incident
|
|
&& (string) $lateDeleteParent->outcome === 'failed'
|
|
&& (string) $lateDeleteParent->error_code
|
|
=== 'aggregate_deadline_exceeded'
|
|
&& (string) $lateDeleteParent->effect_resolution === 'applied'
|
|
&& !(bool) $lateDeleteParent->finalization_incident
|
|
&& (string) Capsule::table('cloud_templates')
|
|
->where('id', $lateDeleteTemplateId)->value('state') === 'retired'
|
|
&& !Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $lateDeleteTemplateId)->exists()
|
|
&& CloudAgentSettingsService::$requests === $beforeLateDeleteRequests,
|
|
'A late template deletion receipt did not converge the original '
|
|
. 'operation without replaying or rewriting its failure audit.');
|
|
|
|
$lateIsoParentId = $insertParent('user_iso_delete');
|
|
$lateIsoChildId = $insertResolvedChild(
|
|
$lateIsoParentId,
|
|
'not_applied',
|
|
'user_iso_delete_replica'
|
|
);
|
|
$lateIsoId = $insertProjectionForParent($lateIsoParentId);
|
|
$lateIsoChild = CloudJob::findOrFail($lateIsoChildId);
|
|
Capsule::table('cloud_jobs')->where('id', $lateIsoChildId)->update([
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
'method' => 'DELETE',
|
|
'request_target' => '/v1/user-isos/'
|
|
. rawurlencode($lateIsoId) . '?owner_id=1',
|
|
'body' => [
|
|
'operation_id' => (string) $lateIsoChild->operation_id,
|
|
],
|
|
'owner_id' => 1,
|
|
'can_manage' => false,
|
|
'user_iso_id' => $lateIsoId,
|
|
], 'job-payload'),
|
|
'effect_may_have_applied' => 1,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'unknown',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolution_reason' => '',
|
|
'completion_event_id' => null,
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'recovery_receipt_invalid',
|
|
'available_at' => 0,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
]);
|
|
$lateIsoChild = CloudJob::findOrFail($lateIsoChildId);
|
|
$lateIsoIdentity = CloudOperationRecoveryIdentity::expected(
|
|
$lateIsoChild,
|
|
true
|
|
);
|
|
$lateIsoAccepted = CloudOperationCompletionService::accept([
|
|
'type' => 'completion',
|
|
'id' => 'aggregate-late-user-iso-delete',
|
|
'agent_id' => (string) $lateIsoChild->host_id,
|
|
'operation_id' => (string) $lateIsoChild->operation_id,
|
|
'action' => $lateIsoIdentity['action'],
|
|
'resource_id' => $lateIsoIdentity['resource_id'],
|
|
'request_fingerprint' => $lateIsoIdentity['request_fingerprint'],
|
|
'status' => 'succeeded',
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => false,
|
|
'error_code' => '',
|
|
'result' => [
|
|
'resource_type' => 'user_iso',
|
|
'id' => $lateIsoId,
|
|
'owner_id' => 1,
|
|
'deleted' => true,
|
|
'managed_image_absent' => true,
|
|
'managed_metadata_absent' => true,
|
|
'managed_artifacts_absent' => true,
|
|
],
|
|
]);
|
|
$lateIsoChild = CloudJob::findOrFail($lateIsoChildId);
|
|
$lateIsoParent = CloudJob::findOrFail($lateIsoParentId);
|
|
$lateIso = Capsule::table('cloud_user_isos')
|
|
->where('id', $lateIsoId)->first();
|
|
$assert($lateIsoAccepted === [
|
|
'accepted' => true,
|
|
'disposition' => 'applied',
|
|
]
|
|
&& (string) $lateIsoChild->outcome === 'failed'
|
|
&& (string) $lateIsoChild->effect_resolution === 'applied'
|
|
&& !(bool) $lateIsoChild->finalization_incident
|
|
&& (string) $lateIsoParent->outcome === 'failed'
|
|
&& (string) $lateIsoParent->effect_resolution === 'applied'
|
|
&& !(bool) $lateIsoParent->finalization_incident
|
|
&& $lateIso !== null
|
|
&& (string) $lateIso->state === 'retired'
|
|
&& (string) $lateIso->name === ''
|
|
&& (int) $lateIso->size_bytes === 0
|
|
&& !Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $lateIsoId)->exists()
|
|
&& CloudAgentSettingsService::$requests === $beforeLateDeleteRequests,
|
|
'A late user-ISO deletion receipt did not converge its owner-bound '
|
|
. 'catalog ledgers without replay.');
|
|
|
|
echo "Cloud aggregate recovery convergence tests passed.\n";
|
|
}
|