You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudOperationRecoveryMatri...

1607 lines
56 KiB
PHP

<?php
declare(strict_types=1);
namespace App\Services\Cloud {
/** Isolated encrypted envelope for the recovery-matrix contract test. */
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'operation-recovery-matrix-test:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
$prefix = 'operation-recovery-matrix-test:';
if (!str_starts_with($value, $prefix)) {
throw new \RuntimeException('Invalid matrix test payload.');
}
$key = substr($value, strlen($prefix));
if (!array_key_exists($key, self::$values)) {
throw new \RuntimeException('Missing matrix test payload.');
}
return self::$values[$key];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Services\Cloud\CloudAdminOperationRecoveryService;
use App\Services\Cloud\CloudAgentError;
use App\Services\Cloud\CloudOperationRecoveryContract;
use App\Services\Cloud\CloudOperationRecoveryIdentity;
use App\Services\Cloud\CloudOperationRecoveryPolicy;
use App\Services\Cloud\CloudRecoveryInstanceStateConflict;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudWorkerService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expectStateConflict = static function (callable $callback) use (
$assert
): void {
try {
$callback();
} catch (CloudRecoveryInstanceStateConflict $error) {
$assert($error->httpStatus === 409,
'Unexpected recovery state-conflict status.');
return;
}
throw new RuntimeException(
'A dedicated recovery kind entered generic worker recovery.'
);
};
$manifestPath = dirname(__DIR__, 2)
. '/agent/operation/recovery_contract.json';
$manifest = json_decode(
file_get_contents($manifestPath),
true,
64,
JSON_THROW_ON_ERROR
);
$assert(($manifest['schema_version'] ?? null) === 1,
'Unsupported recovery contract schema version.');
$manifestOperations = [];
foreach ($manifest['operations'] ?? [] as $entry) {
$kind = $entry['kind'] ?? null;
$assert(is_string($kind) && $kind !== ''
&& !array_key_exists($kind, $manifestOperations),
'Recovery contract contains a missing or duplicate kind.');
unset($entry['kind']);
$manifestOperations[$kind] = $entry;
}
$assert(CloudOperationRecoveryContract::OPERATIONS
=== $manifestOperations,
'Generated PHP recovery contract is stale.');
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite',
'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key')->unique();
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(1);
$table->integer('dispatch_attempts')->default(1);
$table->integer('dispatch_deadline_at')->nullable();
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution')->default('not_applicable');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->string('effect_resolution_reason')->default('');
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at')->default(0);
$table->string('error_code')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->string('uuid');
$table->integer('host_id');
$table->integer('user_id');
$table->string('name');
$table->string('status');
$table->string('service_status')->default('exempt');
$table->string('network_status')->default('active');
$table->text('spec_json')->default('{}');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_instance_snapshots',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('snapshot_key')->unique();
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('host_id');
$table->string('name');
$table->string('state');
$table->string('consistency');
$table->integer('size_bytes')->nullable();
$table->string('last_operation_id')->nullable();
$table->integer('ready_at')->nullable();
$table->integer('last_restored_at')->nullable();
$table->integer('deleted_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_instance_firewall_versions',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('instance_id');
$table->integer('revision');
$table->boolean('enabled')->default(1);
$table->string('policy_digest');
$table->string('request_hash');
$table->string('idempotency_key');
$table->string('state');
$table->integer('job_id')->nullable();
$table->string('operation_id')->nullable();
$table->string('error_code')->nullable();
$table->integer('applied_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_templates',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('name');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('active_job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id');
$table->integer('job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
$schema->create('cloud_rescue_template_imports',
static function (Blueprint $table): void {
$table->string('operation_id')->primary();
$table->integer('host_id');
$table->string('template_id');
$table->integer('job_id');
$table->string('import_type');
$table->string('name');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('state');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_rescue_template_deletions',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->string('operation_id');
$table->integer('job_id');
$table->string('state');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
$schema->create('cloud_user_isos',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('user_id');
$table->string('name');
$table->string('state');
$table->integer('active_job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_user_iso_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('user_iso_id');
$table->integer('user_id');
$table->string('state');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id');
$table->integer('job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'user_iso_id']);
});
$schema->create('cloud_host_storage_configurations',
static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->integer('revision');
foreach (['managed', 'template', 'seed', 'user_iso', 'rescue']
as $role) {
$table->text($role . '_directory');
}
$table->string('state');
$table->string('operation_id')->nullable();
$table->integer('job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$now = time();
$hostId = 7;
$ownerId = 41;
$instanceId = 1;
$instanceUuid = '123e4567-e89b-42d3-a456-426614174000';
Capsule::table('cloud_instances')->insert([
'id' => $instanceId,
'uuid' => $instanceUuid,
'host_id' => $hostId,
'user_id' => $ownerId,
'name' => 'matrix-vm',
'status' => 'active',
'service_status' => 'exempt',
'network_status' => 'active',
'spec_json' => '{}',
'created_at' => $now - 3600,
'updated_at' => $now - 3600,
]);
$nextJobId = 1;
/**
* @return array{id:int,operation_id:string,job:CloudJob,payload:array}
*/
$insertJob = static function (
string $kind,
callable $payloadFactory,
?int $boundInstanceId = null,
?int $snapshotId = null,
?int $parentJobId = null,
?int $orderId = null,
string $executionMode = 'agent',
int $userId = 41
) use (&$nextJobId, $hostId, $now): array {
$id = $nextJobId++;
$operationId = hash('md5', 'recovery-matrix-job:' . $id);
$payload = $payloadFactory($id, $operationId);
Capsule::table('cloud_jobs')->insert([
'id' => $id,
'operation_id' => $operationId,
'request_idempotency_key' => 'recovery-matrix-' . $operationId,
'request_fingerprint' => hash('sha256', $operationId),
'parent_job_id' => $parentJobId,
'order_id' => $orderId,
'invoice_id' => null,
'instance_id' => $boundInstanceId,
'snapshot_id' => $snapshotId,
'user_id' => $userId,
'host_id' => $hostId,
'kind' => $kind,
'execution_mode' => $executionMode,
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'sealed_payload' => CloudSecrets::encrypt(
$payload,
'job-payload'
),
'attempts' => 1,
'dispatch_attempts' => 1,
'dispatch_deadline_at' => $now - 120,
'dispatch_started_at' => $now - 110,
'agent_accepted_at' => $now - 100,
'receipt_deadline_at' => $now - 30,
'dispatched_at' => $now - 110,
'terminal_at' => $now - 20,
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown',
'effect_resolution_reason' => '',
'lease_until' => 0,
'available_at' => 0,
'finalization_incident' => 0,
'created_at' => $now - 180,
'updated_at' => $now - 20,
]);
return [
'id' => $id,
'operation_id' => $operationId,
'job' => CloudJob::findOrFail($id),
'payload' => $payload,
];
};
$fixtures = [];
$addFixture = static function (
string $kind,
string $variant,
array $record,
string $expectedAction,
string $expectedResource,
array $projection
) use (&$fixtures): void {
$fixtures[$kind][] = $record + [
'variant' => $variant,
'expected_action' => $expectedAction,
'expected_resource' => $expectedResource,
'projection' => $projection,
];
};
$instanceFixture = static function (
string $kind,
string $variant,
string $method,
string $suffix,
array $body,
string $expectedAction,
array $projection,
array $metadata = []
) use (
$insertJob,
$addFixture,
$instanceId,
$instanceUuid,
$ownerId
): array {
$record = $insertJob(
$kind,
static function (int $id, string $operationId) use (
$method,
$suffix,
$body,
$metadata,
$instanceUuid,
$ownerId
): array {
$resolvedBody = $body;
foreach ($resolvedBody as $field => $value) {
if ($value === '__OPERATION_ID__') {
$resolvedBody[$field] = $operationId;
}
}
return $metadata + [
'method' => $method,
'request_target' => '/v1/vms/'
. rawurlencode($instanceUuid) . $suffix,
'body' => $resolvedBody,
'owner_id' => $ownerId,
'can_manage' => false,
];
},
$instanceId
);
$addFixture(
$kind,
$variant,
$record,
$expectedAction,
$instanceUuid,
$projection
);
return $record;
};
$powerProjection = ['uuid' => $instanceUuid, 'state' => 'running'];
foreach ([
'start' => 'power-start',
'shutdown' => 'power-shutdown',
'reboot' => 'power-reboot',
'reset' => 'power-reset',
'poweroff' => 'power-destroy',
] as $panelAction => $wireAction) {
$instanceFixture(
'action',
$panelAction,
'POST',
'/actions',
['action' => $panelAction],
$wireAction,
$powerProjection
);
}
$instanceFixture(
'reinstall',
'default',
'POST',
'/reinstall',
[
'template_id' => 'debian-13',
'template_revision' => 4,
'template_generation_token' => str_repeat('a', 32),
'preserve_data' => false,
'root_password' => 'MatrixPassword!123',
],
'reinstall-vm',
['uuid' => $instanceUuid, 'reinstall_committed' => true]
);
$instanceFixture(
'rescue',
'enter',
'POST',
'/rescue/enter',
['template_id' => 'rescue-debian-13'],
'enter-rescue',
['active' => true, 'staged' => true, 'network_suspended' => false]
);
$instanceFixture(
'rescue',
'force-start',
'POST',
'/rescue/force-start',
['ack_data_loss' => true],
'force-start-rescue',
['active' => true, 'staged' => true, 'network_suspended' => false]
);
foreach ([
['network_suspended' => false, 'state' => 'connected'],
['network_suspended' => true, 'state' => 'suspended'],
] as $rescueExit) {
$networkSuspended = $rescueExit['network_suspended'];
$state = $rescueExit['state'];
$instanceFixture(
'rescue',
'exit-' . $state,
'POST',
'/rescue/exit',
[
'network_suspended' => $networkSuspended,
'ack_data_loss' => true,
],
'exit-rescue:' . $state,
[
'active' => false,
'staged' => false,
'network_suspended' => $networkSuspended,
]
);
}
$deleteProjection = [
'deleted' => true,
'managed_artifacts_absent' => true,
'forced' => false,
];
$instanceFixture(
'delete',
'user-delete',
'DELETE',
'',
['remove_managed_disks' => true],
'delete-vm',
$deleteProjection
);
$instanceFixture(
'expiry_delete',
'expiry-delete',
'DELETE',
'',
['remove_managed_disks' => true],
'delete-vm',
$deleteProjection
);
$priorOperationId = str_repeat('b', 32);
$instanceFixture(
'delete_force',
'force-delete',
'POST',
'/force-delete',
[
'operation_id' => '__OPERATION_ID__',
'prior_operation_id' => $priorOperationId,
'ack_data_loss' => true,
'remove_managed_disks' => true,
],
'force-delete-vm',
array_replace($deleteProjection, ['forced' => true]),
['prior_operation_id' => $priorOperationId]
);
$instanceFixture(
'network_change',
'public-ip-change',
'POST',
'/network',
[
'network' => 'bridge:br0',
'public_pool_key' => str_repeat('c', 32),
'public_pool_cidr' => '203.0.113.0/24',
'public_ip_cidr' => '203.0.113.20/24',
'gateway' => '203.0.113.1',
'dns_servers' => ['1.1.1.1'],
'confirm_restart' => true,
],
'change-network',
['uuid' => $instanceUuid, 'network_changed' => true]
);
$instanceFixture(
'instance_settings',
'hostname',
'PATCH',
'/settings',
[
'hostname' => 'matrix.example',
'confirm_restart' => true,
],
'update-settings',
['uuid' => $instanceUuid, 'settings_changed' => true],
['settings_fields' => ['hostname'], 'credential' => null]
);
$upgradeProjection = [
'uuid' => $instanceUuid,
'from_vcpu' => 2,
'target_vcpu' => 4,
'applied_vcpu' => 4,
'from_memory_mib' => 2048,
'target_memory_mib' => 4096,
'applied_memory_mib' => 4096,
'from_disk_gib' => 40,
'target_disk_gib' => 80,
'applied_disk_gib' => 80,
'from_bandwidth_mbps' => 100,
'target_bandwidth_mbps' => 200,
'applied_bandwidth_mbps' => 200,
'status' => 'shutoff',
'resources_resized' => true,
];
$upgradeRecord = $instanceFixture(
'instance_upgrade',
'grow-only',
'POST',
'/resources',
[
'from_vcpu' => 2,
'target_vcpu' => 4,
'from_memory_mib' => 2048,
'target_memory_mib' => 4096,
'from_disk_gib' => 40,
'target_disk_gib' => 80,
'from_bandwidth_mbps' => 100,
'target_bandwidth_mbps' => 200,
'confirm_shutdown' => true,
],
'resize-vm',
$upgradeProjection
);
$snapshotFixture = static function (
string $kind,
string $variant,
string $method,
string $suffix,
array $body,
string $expectedAction,
array $projection,
string $consistency = 'crash'
) use (
$insertJob,
$addFixture,
$instanceId,
$instanceUuid,
$ownerId,
$hostId,
$now
): void {
static $snapshotId = 100;
$id = $snapshotId++;
$key = hash('md5', 'recovery-matrix-snapshot:' . $id);
$record = $insertJob(
$kind,
static function (int $jobId, string $operationId) use (
$kind,
$method,
$suffix,
$body,
$key,
$instanceUuid,
$ownerId
): array {
return [
'method' => $method,
'request_target' => '/v1/vms/'
. rawurlencode($instanceUuid) . '/snapshots'
. ($kind === 'snapshot_create' ? ''
: '/' . rawurlencode($key) . $suffix),
'body' => $body + ($body === [] ? [] : [
'snapshot_key' => $key,
]),
'owner_id' => $ownerId,
'can_manage' => false,
'snapshot_key' => $key,
'snapshot_previous_state' => 'ready',
];
},
$instanceId,
$id
);
Capsule::table('cloud_instance_snapshots')->insert([
'id' => $id,
'snapshot_key' => $key,
'instance_id' => $instanceId,
'user_id' => $ownerId,
'host_id' => $hostId,
'name' => 'matrix-snapshot-' . $id,
'state' => 'uncertain',
'consistency' => $consistency,
'size_bytes' => null,
'last_operation_id' => $record['operation_id'],
'ready_at' => $now - 300,
'last_restored_at' => null,
'deleted_at' => null,
'created_at' => $now - 600,
'updated_at' => $now - 20,
]);
$resolvedAction = str_replace('__SNAPSHOT_KEY__', $key,
$expectedAction);
$resolvedProjection = array_map(
static fn (mixed $value): mixed =>
$value === '__SNAPSHOT_KEY__' ? $key : $value,
$projection
);
$addFixture(
$kind,
$variant,
$record,
$resolvedAction,
$instanceUuid,
$resolvedProjection
);
};
foreach (['filesystem', 'crash'] as $consistency) {
$snapshotFixture(
'snapshot_create',
$consistency,
'POST',
'',
[
'name' => 'matrix-' . $consistency,
'description' => '',
'quiesce' => $consistency === 'filesystem',
'consistency_mode' => $consistency,
],
'create-snapshot:__SNAPSHOT_KEY__:' . $consistency,
[
'id' => 'matrix-id',
'snapshot_key' => '__SNAPSHOT_KEY__',
'created_at' => '2026-10-21T00:00:00Z',
'state' => 'ready',
'consistency' => $consistency,
],
$consistency
);
}
$snapshotFixture(
'snapshot_restore',
'restore',
'POST',
'/restore',
[],
'restore-snapshot:__SNAPSHOT_KEY__',
['snapshot_key' => '__SNAPSHOT_KEY__', 'restored' => true]
);
$snapshotFixture(
'snapshot_delete',
'delete',
'DELETE',
'',
[],
'delete-snapshot:__SNAPSHOT_KEY__',
['snapshot_key' => '__SNAPSHOT_KEY__', 'deleted' => true]
);
$firewallVersionId = 1;
$firewallRevision = 9;
$firewallDigest = str_repeat('d', 64);
$firewallRecord = $instanceFixture(
'firewall_apply',
'apply',
'PUT',
'/firewall',
[
'revision' => $firewallRevision,
'default_in' => 'drop',
'default_out' => 'accept',
'policy_digest' => $firewallDigest,
'rules' => [],
],
'apply-firewall',
[
'revision' => $firewallRevision,
'default_in' => 'drop',
'default_out' => 'accept',
'rules' => [],
],
[
'firewall_version_id' => $firewallVersionId,
'firewall_revision' => $firewallRevision,
'agent_firewall_policy_digest' => $firewallDigest,
]
);
Capsule::table('cloud_instance_firewall_versions')->insert([
'id' => $firewallVersionId,
'instance_id' => $instanceId,
'revision' => $firewallRevision,
'enabled' => 1,
'policy_digest' => $firewallDigest,
'request_hash' => str_repeat('e', 64),
'idempotency_key' => 'matrix-firewall-version',
'state' => 'uncertain',
'job_id' => $firewallRecord['id'],
'operation_id' => $firewallRecord['operation_id'],
'error_code' => '',
'created_at' => $now - 120,
'updated_at' => $now - 20,
]);
$instanceFixture(
'service_suspend',
'purge',
'POST',
'/service/suspend',
['reason' => 'purge'],
'service-suspend',
['uuid' => $instanceUuid, 'state' => 'shutoff']
);
$instanceFixture(
'network_suspend',
'traffic',
'POST',
'/network/suspend',
['reason' => 'traffic'],
'network-suspend',
['network_enabled' => false]
);
$instanceFixture(
'network_resume',
'resume',
'POST',
'/network/resume',
[],
'network-resume',
['network_enabled' => true]
);
$insertAggregateParent = static function (
string $kind,
int $userId = 41
) use ($insertJob): array {
return $insertJob(
$kind,
static fn (int $id, string $operationId): array => [
'aggregate_kind' => $kind,
],
null,
null,
null,
null,
'aggregate',
$userId
);
};
$templateFixture = static function (
string $kind,
string $parentKind,
string $method,
string $expectedAction,
array $projection
) use (
$insertAggregateParent,
$insertJob,
$addFixture,
$hostId,
$ownerId,
$now
): void {
$parent = $insertAggregateParent($parentKind);
$templateId = 'matrix-' . str_replace('_', '-', $kind);
$revision = 5;
$generationToken = hash('md5', 'generation:' . $kind);
$record = $insertJob(
$kind,
static function (int $jobId, string $operationId) use (
$kind,
$method,
$templateId,
$revision,
$generationToken,
$ownerId
): array {
$target = $kind === 'template_import_replica'
? '/v1/templates'
: '/v1/templates/' . rawurlencode($templateId);
$body = ['operation_id' => $operationId];
if ($kind === 'template_import_replica') {
$body += [
'id' => $templateId,
'name' => 'Matrix Template',
'source_url' => 'https://example.invalid/template.qcow2',
'source_token' => 'matrix-token',
'source_size_bytes' => 2097152,
'os_family' => 'linux',
'version' => '13',
'architecture' => 'x86_64',
'sha256' => str_repeat('1', 64),
'revision' => $revision,
'generation_token' => $generationToken,
];
} elseif ($kind === 'template_metadata_replica') {
$body += [
'name' => 'Matrix Template Updated',
'revision' => $revision,
'generation_token' => $generationToken,
];
}
return [
'method' => $method,
'request_target' => $target,
'body' => $body,
'owner_id' => $ownerId,
'can_manage' => true,
'template_id' => $templateId,
];
},
null,
null,
$parent['id']
);
Capsule::table('cloud_templates')->insert([
'id' => $templateId,
'name' => 'Matrix Template',
'revision' => $revision,
'generation_token' => $generationToken,
'state' => 'active',
'active_job_id' => $parent['id'],
'created_at' => $now - 300,
'updated_at' => $now - 20,
]);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => $hostId,
'template_id' => $templateId,
'revision' => $revision,
'generation_token' => $generationToken,
'state' => 'uncertain',
'image_bytes' => 0,
'sha256' => '',
'operation_id' => $record['operation_id'],
'job_id' => $record['id'],
'error_code' => '',
'created_at' => $now - 120,
'updated_at' => $now - 20,
]);
$addFixture(
$kind,
'catalog',
$record,
$expectedAction,
'template:' . $templateId,
$projection
);
};
$templateFixture(
'template_import_replica',
'template_import',
'POST',
'import-template',
[
'id' => 'matrix-template-import-replica',
'generation_token' => hash('md5',
'generation:template_import_replica'),
'created_at' => '2026-10-21',
]
);
$templateFixture(
'template_metadata_replica',
'template_metadata_sync',
'PATCH',
'update-template',
[
'id' => 'matrix-template-metadata-replica',
'name' => 'Matrix Template Updated',
'revision' => 5,
'generation_token' => hash('md5',
'generation:template_metadata_replica'),
]
);
$templateFixture(
'template_delete_replica',
'template_delete',
'DELETE',
'delete-template',
[
'resource_type' => 'template',
'id' => 'matrix-template-delete-replica',
'deleted' => true,
'managed_image_absent' => true,
'managed_metadata_absent' => true,
'managed_artifacts_absent' => true,
]
);
$rescueImportId = hash('md5', 'matrix-rescue-import');
$rescueImport = $insertJob(
'rescue_template_import',
static fn (int $jobId, string $operationId): array => [
'method' => 'POST',
'request_target' => '/v1/rescue-templates',
'body' => [
'operation_id' => $operationId,
'id' => $rescueImportId,
'name' => 'Matrix Rescue',
'source_url' => 'https://example.invalid/rescue.iso',
'source_token' => 'matrix-token',
'source_size_bytes' => 2097152,
'sha256' => str_repeat('2', 64),
],
'owner_id' => $ownerId,
'can_manage' => true,
'rescue_template_id' => $rescueImportId,
]
);
Capsule::table('cloud_rescue_template_imports')->insert([
'operation_id' => $rescueImport['operation_id'],
'host_id' => $hostId,
'template_id' => $rescueImportId,
'job_id' => $rescueImport['id'],
'import_type' => 'url',
'name' => 'Matrix Rescue',
'size_bytes' => 0,
'sha256' => '',
'state' => 'uncertain',
'error_code' => '',
'created_at' => $now - 120,
'updated_at' => $now - 20,
]);
$addFixture(
'rescue_template_import',
'catalog',
$rescueImport,
'import-rescue-template',
'rescue-template:' . $rescueImportId,
['id' => $rescueImportId, 'created_at' => '2026-10-21']
);
$rescueDeleteId = hash('md5', 'matrix-rescue-delete');
$rescueDelete = $insertJob(
'rescue_template_delete',
static fn (int $jobId, string $operationId): array => [
'method' => 'DELETE',
'request_target' => '/v1/rescue-templates/'
. rawurlencode($rescueDeleteId),
'body' => ['operation_id' => $operationId],
'owner_id' => $ownerId,
'can_manage' => true,
'rescue_template_id' => $rescueDeleteId,
]
);
Capsule::table('cloud_rescue_template_deletions')->insert([
'host_id' => $hostId,
'template_id' => $rescueDeleteId,
'operation_id' => $rescueDelete['operation_id'],
'job_id' => $rescueDelete['id'],
'state' => 'uncertain',
'error_code' => '',
'created_at' => $now - 120,
'updated_at' => $now - 20,
]);
$addFixture(
'rescue_template_delete',
'catalog',
$rescueDelete,
'delete-rescue-template',
'rescue-template:' . $rescueDeleteId,
[
'resource_type' => 'rescue_template',
'id' => $rescueDeleteId,
'deleted' => true,
'managed_image_absent' => true,
'managed_metadata_absent' => true,
'managed_artifacts_absent' => true,
]
);
$isoFixture = static function (
string $kind,
string $parentKind,
string $method,
string $expectedAction,
array $projection
) use (
$insertAggregateParent,
$insertJob,
$addFixture,
$hostId,
$ownerId,
$now
): void {
$parent = $insertAggregateParent($parentKind, $ownerId);
$isoId = hash('md5', 'matrix-iso:' . $kind);
$record = $insertJob(
$kind,
static function (int $jobId, string $operationId) use (
$kind,
$method,
$isoId,
$ownerId
): array {
$target = $kind === 'user_iso_import_replica'
? '/v1/user-isos'
: '/v1/user-isos/' . rawurlencode($isoId)
. '?owner_id=' . $ownerId;
$body = ['operation_id' => $operationId];
if ($kind === 'user_iso_import_replica') {
$body += [
'id' => $isoId,
'owner_id' => $ownerId,
'name' => 'Matrix ISO',
'source_url' => 'https://example.invalid/user.iso',
'source_token' => 'matrix-token',
'sha256' => str_repeat('3', 64),
];
}
return [
'method' => $method,
'request_target' => $target,
'body' => $body,
'owner_id' => $ownerId,
'can_manage' => false,
'user_iso_id' => $isoId,
];
},
null,
null,
$parent['id'],
null,
'agent',
$ownerId
);
Capsule::table('cloud_user_isos')->insert([
'id' => $isoId,
'user_id' => $ownerId,
'name' => 'Matrix ISO',
'state' => 'active',
'active_job_id' => $parent['id'],
'created_at' => $now - 300,
'updated_at' => $now - 20,
]);
Capsule::table('cloud_user_iso_replicas')->insert([
'host_id' => $hostId,
'user_iso_id' => $isoId,
'user_id' => $ownerId,
'state' => 'uncertain',
'size_bytes' => 0,
'sha256' => '',
'operation_id' => $record['operation_id'],
'job_id' => $record['id'],
'error_code' => '',
'created_at' => $now - 120,
'updated_at' => $now - 20,
]);
$addFixture(
$kind,
'catalog',
$record,
$expectedAction,
'user-iso:' . $ownerId . ':' . $isoId,
$projection
);
};
$isoFixture(
'user_iso_import_replica',
'user_iso_import',
'POST',
'import-user-iso',
['id' => hash('md5', 'matrix-iso:user_iso_import_replica'),
'created_at' => '2026-10-21']
);
$isoDeleteId = hash('md5', 'matrix-iso:user_iso_delete_replica');
$isoFixture(
'user_iso_delete_replica',
'user_iso_delete',
'DELETE',
'delete-user-iso',
[
'resource_type' => 'user_iso',
'id' => $isoDeleteId,
'deleted' => true,
'managed_image_absent' => true,
'managed_metadata_absent' => true,
'managed_artifacts_absent' => true,
]
);
// Dedicated create/storage identities are covered without admitting them
// to the generic worker/finalizer matrix.
$dedicatedFixtures = [];
$createRecord = $insertJob(
'create',
static fn (int $jobId, string $operationId): array => [
'method' => 'POST',
'request_target' => '/v1/vms',
'body' => [
'operation_id' => $operationId,
'source' => 'cloud',
'name' => 'matrix-create',
'vcpu' => 2,
'memory_mib' => 2048,
'disk_gib' => 40,
'bandwidth_mbps' => 100,
'network' => 'bridge:br0',
'template_id' => 'debian-13',
'template_revision' => 1,
'template_generation_token' => str_repeat('4', 32),
'root_password' => 'MatrixPassword!123',
'public_pool_key' => str_repeat('5', 32),
'public_pool_cidr' => '203.0.113.0/24',
'public_ip_cidr' => '203.0.113.30/24',
'gateway' => '203.0.113.1',
'dns_servers' => ['1.1.1.1'],
'platform_network_policy_snapshot' => [
'schema_version' => 1,
],
],
'credential' => [
'password' => 'MatrixPassword!123',
'login_name' => 'root',
'auth_mode' => 'password',
],
'template_metadata' => [
'template_id' => 'debian-13',
'os_source' => 'cloud_template',
'os_name' => 'Debian 13',
'login_name' => 'root',
],
],
null,
null,
null,
9001
);
$dedicatedFixtures['create'] = $createRecord + [
'expected_action' => 'create-vm',
];
$storageBody = [
'managed_directory' => '/srv/virt/managed',
'template_directory' => '/srv/virt/templates',
'seed_directory' => '/srv/virt/seeds',
'user_iso_directory' => '/srv/virt/user-isos',
'rescue_directory' => '/srv/virt/rescue',
];
$storageRecord = $insertJob(
'host_storage_configure',
static fn (int $jobId, string $operationId): array => [
'method' => 'POST',
'request_target' => '/v1/storage/configuration',
'body' => $storageBody + ['operation_id' => $operationId],
'owner_id' => $ownerId,
'can_manage' => true,
'storage_revision' => 3,
]
);
Capsule::table('cloud_host_storage_configurations')->insert([
'host_id' => $hostId,
'revision' => 3,
'managed_directory' => $storageBody['managed_directory'],
'template_directory' => $storageBody['template_directory'],
'seed_directory' => $storageBody['seed_directory'],
'user_iso_directory' => $storageBody['user_iso_directory'],
'rescue_directory' => $storageBody['rescue_directory'],
'state' => 'uncertain',
'operation_id' => $storageRecord['operation_id'],
'job_id' => $storageRecord['id'],
'created_at' => $now - 120,
'updated_at' => $now - 20,
]);
$dedicatedFixtures['host_storage_configure'] = $storageRecord + [
'expected_action' => 'configure-storage',
'expected_resource' => 'host-storage',
];
$classificationOnlyKinds = [
'payment_create' => 'payment',
'template_import' => 'aggregate',
'template_metadata_sync' => 'aggregate',
'template_delete' => 'aggregate',
'user_iso_import' => 'aggregate',
'user_iso_delete' => 'aggregate',
];
$genericKinds = array_keys(array_filter(
$manifestOperations,
static fn (array $operation): bool =>
$operation['recovery_mode'] === 'generic'
));
$fixtureKinds = array_keys($fixtures);
sort($genericKinds, SORT_STRING);
sort($fixtureKinds, SORT_STRING);
$assert($fixtureKinds === $genericKinds,
'Every generic recovery kind must have an executable fixture.');
$coveredKinds = array_merge(
$fixtureKinds,
array_keys($dedicatedFixtures),
array_keys($classificationOnlyKinds)
);
$contractKinds = array_keys($manifestOperations);
sort($coveredKinds, SORT_STRING);
sort($contractKinds, SORT_STRING);
$assert($coveredKinds === $contractKinds,
'A recovery contract kind was added without a matrix fixture.');
$expectedAutomaticKinds = array_keys(array_filter(
$manifestOperations,
static fn (array $operation): bool => in_array(
$operation['recovery_mode'],
['generic', 'create'],
true
)
));
$assert(CloudOperationRecoveryPolicy::kinds()
=== array_keys(array_filter(
$manifestOperations,
static fn (array $operation): bool =>
$operation['recovery_mode'] === 'generic'
))
&& CloudOperationRecoveryPolicy::automaticKinds()
=== $expectedAutomaticKinds,
'Policy kind lists diverged from the recovery modes.');
$projectionMethod = new ReflectionMethod(
CloudAdminOperationRecoveryService::class,
'effectProjection'
);
$expectInvalidProjection = static function (
array $projection,
string $kind,
string $message
) use ($projectionMethod, $assert): void {
try {
$projectionMethod->invoke(null, $projection, $kind, 'applied');
} catch (CloudAgentError $error) {
$assert($error->agentCode === 'agent_invalid_inspection'
&& $error->agentHttpStatus === 200,
'Unexpected invalid-projection error for ' . $kind . '.');
return;
}
throw new RuntimeException($message);
};
$upgradeIdentity = CloudOperationRecoveryIdentity::expectedFromPayload(
$upgradeRecord['job'],
$upgradeRecord['payload']
);
$inspectionMethod = new ReflectionMethod(
CloudAdminOperationRecoveryService::class,
'inspection'
);
$notAppliedUpgrade = $inspectionMethod->invoke(null, [
'operation_id' => $upgradeRecord['operation_id'],
'action' => $upgradeIdentity['action'],
'resource_id' => $upgradeIdentity['resource_id'],
'request_fingerprint' => $upgradeIdentity['request_fingerprint'],
'operation_status' => 'failed',
'checked_at' => time(),
'outcome' => 'failed',
'summary_code' => 'operation_failed',
'error_code' => 'resources_not_committed',
'effect_resolution' => 'not_applied',
'fence_state' => 'active',
'fence_resolution_required' => true,
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
], $upgradeRecord['operation_id'], 'instance_upgrade', true);
$assert(($notAppliedUpgrade['effect_resolution'] ?? null)
=== 'not_applied'
&& !array_key_exists('effect_projection', $notAppliedUpgrade),
'Instance upgrade not-applied evidence was not kept projection-free.');
$malformedUpgradeProjection = $upgradeProjection;
unset($malformedUpgradeProjection['applied_disk_gib']);
$expectInvalidProjection(
$malformedUpgradeProjection,
'instance_upgrade',
'Incomplete instance-upgrade evidence crossed the projection boundary.'
);
foreach ($fixtures as $kind => $kindFixtures) {
$entry = $manifestOperations[$kind];
$assert(CloudOperationRecoveryPolicy::supports($kind)
&& CloudOperationRecoveryPolicy::identityScope($kind)
=== $entry['identity_scope']
&& CloudOperationRecoveryIdentity::supports($kind),
'Generic policy/identity support diverged for ' . $kind . '.');
$actualActions = [];
foreach ($kindFixtures as $fixture) {
$job = $fixture['job'];
$payload = $fixture['payload'];
$fromPayload = CloudOperationRecoveryIdentity::expectedFromPayload(
$job,
$payload
);
$fromEncryptedJob = CloudOperationRecoveryIdentity::expected($job);
$label = $kind . ':' . $fixture['variant'];
$assert($fromPayload === $fromEncryptedJob,
'Encrypted identity reconstruction diverged for ' . $label);
$assert($fromPayload['action']
=== $fixture['expected_action']
&& $fromPayload['resource_id']
=== $fixture['expected_resource'],
'Identity action/resource mismatch for ' . $label);
$assert(CloudOperationRecoveryPolicy::allowsAction(
$kind,
$fromPayload['action']
),
'Contract rejected the rebuilt action for ' . $label);
$wireBody = $payload['body'];
if ($kind === 'action'
&& ($wireBody['action'] ?? null) === 'poweroff') {
$wireBody['action'] = 'destroy';
}
$encodedBody = json_encode(
$wireBody === [] ? (object) [] : $wireBody,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
);
$expectedFingerprint = hash(
'sha256',
strtoupper($payload['method']) . "\n"
. $payload['request_target'] . "\n" . $encodedBody
);
$assert(hash_equals(
$expectedFingerprint,
$fromPayload['request_fingerprint']
),
'Wire fingerprint mismatch for ' . $label);
$context = $kind === 'instance_settings'
? CloudWorkerService::settingsRecoveryContext($job)
: CloudWorkerService::recoveryContext($job);
$assert($context['payload'] === $payload,
'Worker lost the encrypted recovery payload for ' . $label);
if ($entry['identity_scope'] === 'instance') {
$assert(($context['instance']->id ?? null) === $instanceId,
'Worker lost the instance binding for ' . $label);
} else {
$assert($context['instance'] === null,
'Catalog recovery unexpectedly acquired an instance for '
. $label);
}
$projection = $projectionMethod->invoke(
null,
$fixture['projection'],
$kind,
'applied'
);
$assert($projection === $fixture['projection'],
'Evidence projection branch is missing for ' . $label);
$actualActions[] = $fromPayload['action'];
}
foreach ($entry['actions'] as $action) {
$assert(in_array($action, $actualActions, true),
'Literal action lacks a real identity fixture: '
. $kind . ':' . $action);
}
foreach ($entry['action_patterns'] as $pattern) {
$matched = array_filter(
$actualActions,
static fn (string $action): bool =>
preg_match('~' . $pattern . '~D', $action) === 1
);
$assert($matched !== [],
'Pattern action lacks a real identity fixture: '
. $kind . ':' . $pattern);
}
foreach ($actualActions as $action) {
$literal = in_array($action, $entry['actions'], true);
$pattern = false;
foreach ($entry['action_patterns'] as $candidate) {
$pattern = $pattern
|| preg_match('~' . $candidate . '~D', $action) === 1;
}
$assert($literal || $pattern,
'Fixture produced an uncontracted action: '
. $kind . ':' . $action);
}
}
$snapshotCreateActions = array_column(
$fixtures['snapshot_create'],
'expected_action'
);
$snapshotKeyPattern = '[a-f0-9]{32}';
$assert(count($snapshotCreateActions) === 2
&& count(array_filter(
$snapshotCreateActions,
static fn (string $action): bool => preg_match(
'~^create-snapshot:' . $snapshotKeyPattern
. ':filesystem$~D',
$action
) === 1
)) === 1
&& count(array_filter(
$snapshotCreateActions,
static fn (string $action): bool => preg_match(
'~^create-snapshot:' . $snapshotKeyPattern . ':crash$~D',
$action
) === 1
)) === 1,
'Snapshot create must cover both filesystem and crash consistency.');
$validTemplateMetadataProjection =
$fixtures['template_metadata_replica'][0]['projection'];
$invalidGeneration = $validTemplateMetadataProjection;
$invalidGeneration['generation_token'] = 'NOT-32-LOWER-HEX';
$expectInvalidProjection(
$invalidGeneration,
'template_metadata_replica',
'An invalid template generation token crossed the projection boundary.'
);
$expectInvalidProjection(
[
'id' => 'matrix-rescue',
'created_at' => '2026-10-21',
'generation_token' => str_repeat('a', 32),
],
'rescue_template_import',
'A generation token escaped its two template projection kinds.'
);
$nestedToken = $validTemplateMetadataProjection;
$nestedToken['generation_token'] = [
'generation_token' => str_repeat('a', 32),
];
$expectInvalidProjection(
$nestedToken,
'template_metadata_replica',
'A nested generation token bypassed sensitive-key filtering.'
);
$accessToken = $validTemplateMetadataProjection;
$accessToken['access_token'] = str_repeat('a', 32);
$expectInvalidProjection(
$accessToken,
'template_metadata_replica',
'An access token bypassed sensitive-key filtering.'
);
foreach ($dedicatedFixtures as $kind => $fixture) {
$identity = CloudOperationRecoveryIdentity::expectedFromPayload(
$fixture['job'],
$fixture['payload']
);
$assert($identity === CloudOperationRecoveryIdentity::expected(
$fixture['job']
)
&& $identity['action'] === $fixture['expected_action']
&& CloudOperationRecoveryPolicy::allowsAction(
$kind,
$identity['action']
)
&& !CloudOperationRecoveryPolicy::supports($kind),
'Dedicated identity contract diverged for ' . $kind . '.');
if ($kind === 'create') {
$assert(CloudOperationRecoveryPolicy::identityScope($kind)
=== 'create'
&& preg_match(
'/^[a-f0-9]{8}-[a-f0-9]{4}-5[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/D',
$identity['resource_id']
) === 1,
'Create identity lost its deterministic UUID scope.');
$projection = $projectionMethod->invoke(null, [
'creation_absent' => true,
'managed_artifacts_absent' => true,
], 'create', 'not_applied');
$assert(($projection['creation_absent'] ?? null) === true
&& ($projection['managed_artifacts_absent'] ?? null)
=== true,
'Create absence projection is no longer complete.');
} else {
$assert(CloudOperationRecoveryPolicy::identityScope($kind)
=== 'storage'
&& $identity['resource_id']
=== $fixture['expected_resource'],
'Storage identity lost its dedicated scope.');
$assert($projectionMethod->invoke(
null,
['configured' => true],
'host_storage_configure',
'applied'
) === ['configured' => true],
'Storage projection branch is missing.');
}
$expectStateConflict(static fn () =>
CloudWorkerService::recoveryContext($fixture['job']));
}
foreach ($classificationOnlyKinds as $kind => $mode) {
$entry = $manifestOperations[$kind];
$assert($entry['recovery_mode'] === $mode
&& $entry['identity_scope'] === 'none'
&& $entry['actions'] === []
&& $entry['action_patterns'] === []
&& !CloudOperationRecoveryPolicy::supports($kind)
&& CloudOperationRecoveryPolicy::identityScope($kind) === null
&& !CloudOperationRecoveryIdentity::supports($kind),
'Aggregate/payment classification diverged for ' . $kind . '.');
}
foreach ($manifestOperations as $kind => $entry) {
foreach (['applied', 'not_applied', 'partial'] as $resolution) {
$assert(CloudOperationRecoveryPolicy::allowsSystemResolution(
$kind,
$resolution
) === in_array(
$resolution,
$entry['system_resolutions'],
true
),
'System resolution policy diverged for '
. $kind . ':' . $resolution);
}
$assert(!CloudOperationRecoveryPolicy::allowsAction(
$kind,
'unlisted-operation'
),
'Recovery contract accepted an unlisted action for ' . $kind);
}
echo "Cloud operation recovery matrix tests passed\n";
}