You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2092 lines
76 KiB
PHP
2092 lines
76 KiB
PHP
<?php
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Services\Cloud {
|
|
/** Isolated encrypted envelope for the recovery-matrix contract test. */
|
|
final class CloudSecrets
|
|
{
|
|
private static array $values = [];
|
|
|
|
public static function encrypt(array $value, string $purpose): string
|
|
{
|
|
$key = bin2hex(random_bytes(16));
|
|
self::$values[$key] = $value;
|
|
return 'operation-recovery-matrix-test:' . $key;
|
|
}
|
|
|
|
public static function decrypt(string $value, string $purpose): array
|
|
{
|
|
$prefix = 'operation-recovery-matrix-test:';
|
|
if (!str_starts_with($value, $prefix)) {
|
|
throw new \RuntimeException('Invalid matrix test payload.');
|
|
}
|
|
$key = substr($value, strlen($prefix));
|
|
if (!array_key_exists($key, self::$values)) {
|
|
throw new \RuntimeException('Missing matrix test payload.');
|
|
}
|
|
return self::$values[$key];
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudJob;
|
|
use App\Services\Cloud\CloudAdminOperationRecoveryService;
|
|
use App\Services\Cloud\CloudAgentError;
|
|
use App\Services\Cloud\CloudOperationRecoveryContract;
|
|
use App\Services\Cloud\CloudOperationRecoveryIdentity;
|
|
use App\Services\Cloud\CloudOperationRecoveryPolicy;
|
|
use App\Services\Cloud\CloudRecoveryInstanceStateConflict;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$expectStateConflict = static function (callable $callback) use (
|
|
$assert
|
|
): void {
|
|
try {
|
|
$callback();
|
|
} catch (CloudRecoveryInstanceStateConflict $error) {
|
|
$assert($error->httpStatus === 409,
|
|
'Unexpected recovery state-conflict status.');
|
|
return;
|
|
}
|
|
throw new RuntimeException(
|
|
'A dedicated recovery kind entered generic worker recovery.'
|
|
);
|
|
};
|
|
|
|
$manifestPath = dirname(__DIR__, 2)
|
|
. '/agent/operation/recovery_contract.json';
|
|
$manifest = json_decode(
|
|
file_get_contents($manifestPath),
|
|
true,
|
|
64,
|
|
JSON_THROW_ON_ERROR
|
|
);
|
|
$assert(($manifest['schema_version'] ?? null) === 1,
|
|
'Unsupported recovery contract schema version.');
|
|
$manifestOperations = [];
|
|
foreach ($manifest['operations'] ?? [] as $entry) {
|
|
$kind = $entry['kind'] ?? null;
|
|
$assert(is_string($kind) && $kind !== ''
|
|
&& !array_key_exists($kind, $manifestOperations),
|
|
'Recovery contract contains a missing or duplicate kind.');
|
|
unset($entry['kind']);
|
|
$manifestOperations[$kind] = $entry;
|
|
}
|
|
$assert(CloudOperationRecoveryContract::OPERATIONS
|
|
=== $manifestOperations,
|
|
'Generated PHP recovery contract is stale.');
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite',
|
|
'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id')->unique();
|
|
$table->string('request_idempotency_key')->unique();
|
|
$table->string('request_fingerprint');
|
|
$table->integer('parent_job_id')->nullable();
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('invoice_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('snapshot_id')->nullable();
|
|
$table->integer('backup_id')->nullable();
|
|
$table->integer('backup_restore_id')->nullable();
|
|
$table->integer('user_id');
|
|
$table->integer('host_id')->nullable();
|
|
$table->string('kind');
|
|
$table->string('execution_mode');
|
|
$table->string('phase');
|
|
$table->string('outcome')->nullable();
|
|
$table->string('status');
|
|
$table->text('sealed_payload');
|
|
$table->integer('attempts')->default(1);
|
|
$table->integer('dispatch_attempts')->default(1);
|
|
$table->integer('dispatch_deadline_at')->nullable();
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->string('effect_resolution')->default('not_applicable');
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
$table->string('effect_resolution_reason')->default('');
|
|
$table->string('completion_event_id')->nullable();
|
|
$table->integer('read_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('available_at')->default(0);
|
|
$table->string('error_code')->nullable();
|
|
$table->boolean('finalization_incident')->default(0);
|
|
$table->string('finalization_error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('uuid');
|
|
$table->integer('host_id');
|
|
$table->integer('user_id');
|
|
$table->integer('reservation_id')->nullable();
|
|
$table->string('name');
|
|
$table->string('status');
|
|
$table->string('service_status')->default('exempt');
|
|
$table->string('network_status')->default('active');
|
|
$table->text('spec_json')->default('{}');
|
|
$table->text('sealed_credential')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_instance_snapshots',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('snapshot_key')->unique();
|
|
$table->integer('instance_id');
|
|
$table->integer('user_id');
|
|
$table->integer('host_id');
|
|
$table->string('name');
|
|
$table->string('state');
|
|
$table->string('consistency');
|
|
$table->integer('size_bytes')->nullable();
|
|
$table->string('last_operation_id')->nullable();
|
|
$table->integer('ready_at')->nullable();
|
|
$table->integer('last_restored_at')->nullable();
|
|
$table->integer('deleted_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_instance_backups',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('backup_key')->unique();
|
|
$table->integer('source_instance_id');
|
|
$table->integer('source_host_id');
|
|
$table->integer('user_id');
|
|
$table->string('source_uuid');
|
|
$table->string('name');
|
|
$table->string('state');
|
|
$table->string('consistency');
|
|
$table->string('object_key');
|
|
$table->integer('object_size_bytes')->default(0);
|
|
$table->integer('virtual_size_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->text('spec_json');
|
|
$table->text('sealed_storage_runtime');
|
|
$table->text('sealed_credential')->nullable();
|
|
$table->integer('storage_revision')->default(1);
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->string('last_operation_id')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->integer('ready_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_backup_restores',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('restore_key')->unique();
|
|
$table->integer('backup_id');
|
|
$table->integer('target_host_id');
|
|
$table->integer('target_instance_id')->nullable();
|
|
$table->integer('reservation_id')->nullable();
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('operation_id')->unique();
|
|
$table->string('new_uuid');
|
|
$table->string('new_name');
|
|
$table->string('state');
|
|
$table->string('error_code')->default('');
|
|
$table->integer('ready_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_instance_migrations',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('migration_key')->unique();
|
|
$table->integer('instance_id');
|
|
$table->integer('backup_id');
|
|
$table->integer('restore_id')->nullable();
|
|
$table->integer('source_host_id');
|
|
$table->string('source_uuid');
|
|
$table->integer('source_reservation_id');
|
|
$table->integer('target_host_id');
|
|
$table->string('target_uuid');
|
|
$table->integer('target_reservation_id')->nullable();
|
|
$table->integer('backup_job_id');
|
|
$table->integer('restore_job_id')->nullable();
|
|
$table->integer('firewall_job_id')->nullable();
|
|
$table->integer('rollback_job_id')->nullable();
|
|
$table->integer('cleanup_job_id')->nullable();
|
|
$table->string('cleanup_mode');
|
|
$table->string('state');
|
|
$table->boolean('source_was_running')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->integer('cutover_at')->nullable();
|
|
$table->integer('completed_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_resource_reservations',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('backup_restore_id');
|
|
$table->integer('host_id');
|
|
$table->integer('user_id');
|
|
$table->string('operation_id');
|
|
$table->integer('public_ip_id');
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('cpu_cores');
|
|
$table->integer('memory_mib');
|
|
$table->integer('disk_gib');
|
|
$table->string('state');
|
|
$table->integer('expires_at')->nullable();
|
|
$table->integer('released_at')->nullable();
|
|
$table->string('release_reason')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_public_ips',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('host_id');
|
|
$table->string('address');
|
|
$table->integer('prefix_length');
|
|
$table->string('state');
|
|
$table->integer('reservation_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->string('operation_id')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_instance_firewall_versions',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('instance_id');
|
|
$table->integer('revision');
|
|
$table->boolean('enabled')->default(1);
|
|
$table->string('policy_digest');
|
|
$table->string('request_hash');
|
|
$table->string('idempotency_key');
|
|
$table->string('state');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('operation_id')->nullable();
|
|
$table->string('error_code')->nullable();
|
|
$table->integer('applied_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_templates',
|
|
static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->string('name');
|
|
$table->integer('revision');
|
|
$table->string('generation_token');
|
|
$table->string('state');
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_template_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->integer('revision');
|
|
$table->string('generation_token');
|
|
$table->string('state');
|
|
$table->integer('image_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('operation_id');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->primary(['host_id', 'template_id']);
|
|
});
|
|
$schema->create('cloud_rescue_template_imports',
|
|
static function (Blueprint $table): void {
|
|
$table->string('operation_id')->primary();
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->integer('job_id');
|
|
$table->string('import_type');
|
|
$table->string('name');
|
|
$table->integer('size_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('state');
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_rescue_template_deletions',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->string('operation_id');
|
|
$table->integer('job_id');
|
|
$table->string('state');
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->primary(['host_id', 'template_id']);
|
|
});
|
|
$schema->create('cloud_user_isos',
|
|
static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->integer('user_id');
|
|
$table->string('name');
|
|
$table->string('state');
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_user_iso_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('user_iso_id');
|
|
$table->integer('user_id');
|
|
$table->string('state');
|
|
$table->integer('size_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('operation_id');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->primary(['host_id', 'user_iso_id']);
|
|
});
|
|
$schema->create('cloud_host_storage_configurations',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id')->primary();
|
|
$table->integer('revision');
|
|
foreach (['managed', 'template', 'seed', 'user_iso', 'rescue']
|
|
as $role) {
|
|
$table->text($role . '_directory');
|
|
}
|
|
$table->string('state');
|
|
$table->string('operation_id')->nullable();
|
|
$table->integer('job_id')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
|
|
$now = time();
|
|
$hostId = 7;
|
|
$ownerId = 41;
|
|
$instanceId = 1;
|
|
$instanceUuid = '123e4567-e89b-42d3-a456-426614174000';
|
|
Capsule::table('cloud_instances')->insert([
|
|
'id' => $instanceId,
|
|
'uuid' => $instanceUuid,
|
|
'host_id' => $hostId,
|
|
'user_id' => $ownerId,
|
|
'name' => 'matrix-vm',
|
|
'status' => 'active',
|
|
'service_status' => 'exempt',
|
|
'network_status' => 'active',
|
|
'spec_json' => '{}',
|
|
'created_at' => $now - 3600,
|
|
'updated_at' => $now - 3600,
|
|
]);
|
|
|
|
$nextJobId = 1;
|
|
/**
|
|
* @return array{id:int,operation_id:string,job:CloudJob,payload:array}
|
|
*/
|
|
$insertJob = static function (
|
|
string $kind,
|
|
callable $payloadFactory,
|
|
?int $boundInstanceId = null,
|
|
?int $snapshotId = null,
|
|
?int $parentJobId = null,
|
|
?int $orderId = null,
|
|
string $executionMode = 'agent',
|
|
int $userId = 41,
|
|
?int $backupId = null,
|
|
?int $backupRestoreId = null
|
|
) use (&$nextJobId, $hostId, $now): array {
|
|
$id = $nextJobId++;
|
|
$operationId = hash('md5', 'recovery-matrix-job:' . $id);
|
|
$payload = $payloadFactory($id, $operationId);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => $id,
|
|
'operation_id' => $operationId,
|
|
'request_idempotency_key' => 'recovery-matrix-' . $operationId,
|
|
'request_fingerprint' => hash('sha256', $operationId),
|
|
'parent_job_id' => $parentJobId,
|
|
'order_id' => $orderId,
|
|
'invoice_id' => null,
|
|
'instance_id' => $boundInstanceId,
|
|
'snapshot_id' => $snapshotId,
|
|
'backup_id' => $backupId,
|
|
'backup_restore_id' => $backupRestoreId,
|
|
'user_id' => $userId,
|
|
'host_id' => $hostId,
|
|
'kind' => $kind,
|
|
'execution_mode' => $executionMode,
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'status' => 'failed',
|
|
'sealed_payload' => CloudSecrets::encrypt(
|
|
$payload,
|
|
'job-payload'
|
|
),
|
|
'attempts' => 1,
|
|
'dispatch_attempts' => 1,
|
|
'dispatch_deadline_at' => $now - 120,
|
|
'dispatch_started_at' => $now - 110,
|
|
'agent_accepted_at' => $now - 100,
|
|
'receipt_deadline_at' => $now - 30,
|
|
'dispatched_at' => $now - 110,
|
|
'terminal_at' => $now - 20,
|
|
'effect_may_have_applied' => 1,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'unknown',
|
|
'effect_resolution_reason' => '',
|
|
'lease_until' => 0,
|
|
'available_at' => 0,
|
|
'finalization_incident' => 0,
|
|
'created_at' => $now - 180,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
return [
|
|
'id' => $id,
|
|
'operation_id' => $operationId,
|
|
'job' => CloudJob::findOrFail($id),
|
|
'payload' => $payload,
|
|
];
|
|
};
|
|
|
|
$fixtures = [];
|
|
$addFixture = static function (
|
|
string $kind,
|
|
string $variant,
|
|
array $record,
|
|
string $expectedAction,
|
|
string $expectedResource,
|
|
array $projection
|
|
) use (&$fixtures): void {
|
|
$fixtures[$kind][] = $record + [
|
|
'variant' => $variant,
|
|
'expected_action' => $expectedAction,
|
|
'expected_resource' => $expectedResource,
|
|
'projection' => $projection,
|
|
];
|
|
};
|
|
|
|
$instanceFixture = static function (
|
|
string $kind,
|
|
string $variant,
|
|
string $method,
|
|
string $suffix,
|
|
array $body,
|
|
string $expectedAction,
|
|
array $projection,
|
|
array $metadata = []
|
|
) use (
|
|
$insertJob,
|
|
$addFixture,
|
|
$instanceId,
|
|
$instanceUuid,
|
|
$ownerId
|
|
): array {
|
|
$record = $insertJob(
|
|
$kind,
|
|
static function (int $id, string $operationId) use (
|
|
$method,
|
|
$suffix,
|
|
$body,
|
|
$metadata,
|
|
$instanceUuid,
|
|
$ownerId
|
|
): array {
|
|
$resolvedBody = $body;
|
|
foreach ($resolvedBody as $field => $value) {
|
|
if ($value === '__OPERATION_ID__') {
|
|
$resolvedBody[$field] = $operationId;
|
|
}
|
|
}
|
|
return $metadata + [
|
|
'method' => $method,
|
|
'request_target' => '/v1/vms/'
|
|
. rawurlencode($instanceUuid) . $suffix,
|
|
'body' => $resolvedBody,
|
|
'owner_id' => $ownerId,
|
|
'can_manage' => false,
|
|
];
|
|
},
|
|
$instanceId
|
|
);
|
|
$addFixture(
|
|
$kind,
|
|
$variant,
|
|
$record,
|
|
$expectedAction,
|
|
$instanceUuid,
|
|
$projection
|
|
);
|
|
return $record;
|
|
};
|
|
|
|
$powerProjection = ['uuid' => $instanceUuid, 'state' => 'running'];
|
|
foreach ([
|
|
'start' => 'power-start',
|
|
'shutdown' => 'power-shutdown',
|
|
'reboot' => 'power-reboot',
|
|
'reset' => 'power-reset',
|
|
'poweroff' => 'power-destroy',
|
|
] as $panelAction => $wireAction) {
|
|
$instanceFixture(
|
|
'action',
|
|
$panelAction,
|
|
'POST',
|
|
'/actions',
|
|
['action' => $panelAction],
|
|
$wireAction,
|
|
$powerProjection
|
|
);
|
|
}
|
|
|
|
$instanceFixture(
|
|
'reinstall',
|
|
'default',
|
|
'POST',
|
|
'/reinstall',
|
|
[
|
|
'template_id' => 'debian-13',
|
|
'template_revision' => 4,
|
|
'template_generation_token' => str_repeat('a', 32),
|
|
'preserve_data' => false,
|
|
'root_password' => 'MatrixPassword!123',
|
|
],
|
|
'reinstall-vm',
|
|
['uuid' => $instanceUuid, 'reinstall_committed' => true]
|
|
);
|
|
|
|
$instanceFixture(
|
|
'rescue',
|
|
'enter',
|
|
'POST',
|
|
'/rescue/enter',
|
|
['template_id' => 'rescue-debian-13'],
|
|
'enter-rescue',
|
|
['active' => true, 'staged' => true, 'network_suspended' => false]
|
|
);
|
|
$instanceFixture(
|
|
'rescue',
|
|
'force-start',
|
|
'POST',
|
|
'/rescue/force-start',
|
|
['ack_data_loss' => true],
|
|
'force-start-rescue',
|
|
['active' => true, 'staged' => true, 'network_suspended' => false]
|
|
);
|
|
foreach ([
|
|
['network_suspended' => false, 'state' => 'connected'],
|
|
['network_suspended' => true, 'state' => 'suspended'],
|
|
] as $rescueExit) {
|
|
$networkSuspended = $rescueExit['network_suspended'];
|
|
$state = $rescueExit['state'];
|
|
$instanceFixture(
|
|
'rescue',
|
|
'exit-' . $state,
|
|
'POST',
|
|
'/rescue/exit',
|
|
[
|
|
'network_suspended' => $networkSuspended,
|
|
'ack_data_loss' => true,
|
|
],
|
|
'exit-rescue:' . $state,
|
|
[
|
|
'active' => false,
|
|
'staged' => false,
|
|
'network_suspended' => $networkSuspended,
|
|
]
|
|
);
|
|
}
|
|
|
|
$deleteProjection = [
|
|
'deleted' => true,
|
|
'managed_artifacts_absent' => true,
|
|
'forced' => false,
|
|
];
|
|
$instanceFixture(
|
|
'delete',
|
|
'user-delete',
|
|
'DELETE',
|
|
'',
|
|
['remove_managed_disks' => true],
|
|
'delete-vm',
|
|
$deleteProjection
|
|
);
|
|
$instanceFixture(
|
|
'expiry_delete',
|
|
'expiry-delete',
|
|
'DELETE',
|
|
'',
|
|
['remove_managed_disks' => true],
|
|
'delete-vm',
|
|
$deleteProjection
|
|
);
|
|
$priorOperationId = str_repeat('b', 32);
|
|
$instanceFixture(
|
|
'delete_force',
|
|
'force-delete',
|
|
'POST',
|
|
'/force-delete',
|
|
[
|
|
'operation_id' => '__OPERATION_ID__',
|
|
'prior_operation_id' => $priorOperationId,
|
|
'ack_data_loss' => true,
|
|
'remove_managed_disks' => true,
|
|
],
|
|
'force-delete-vm',
|
|
array_replace($deleteProjection, ['forced' => true]),
|
|
['prior_operation_id' => $priorOperationId]
|
|
);
|
|
$migrationCleanup = $insertJob(
|
|
'instance_migration_cleanup',
|
|
static fn (int $id, string $operationId): array => [
|
|
'method' => 'DELETE',
|
|
'request_target' => '/v1/vms/' . rawurlencode($instanceUuid),
|
|
'body' => ['remove_managed_disks' => true],
|
|
'owner_id' => $ownerId,
|
|
'can_manage' => true,
|
|
'maintenance_migration_id' => 1,
|
|
],
|
|
$instanceId
|
|
);
|
|
$migrationRollback = $insertJob(
|
|
'instance_migration_rollback',
|
|
static fn (int $id, string $operationId): array => [
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/vms/' . rawurlencode($instanceUuid)
|
|
. '/service/resume',
|
|
'body' => [
|
|
'operation_id' => $operationId,
|
|
'reason' => 'maintenance_migration_rollback',
|
|
'start_if_suspended' => true,
|
|
],
|
|
'owner_id' => $ownerId,
|
|
'can_manage' => true,
|
|
'maintenance_migration_id' => 1,
|
|
],
|
|
$instanceId
|
|
);
|
|
Capsule::table('cloud_instance_migrations')->insert([
|
|
'id' => 1,
|
|
'migration_key' => str_repeat('e', 32),
|
|
'instance_id' => $instanceId,
|
|
'backup_id' => 9001,
|
|
'restore_id' => 9002,
|
|
'source_host_id' => $hostId,
|
|
'source_uuid' => $instanceUuid,
|
|
'source_reservation_id' => 9003,
|
|
'target_host_id' => $hostId + 1,
|
|
'target_uuid' => '123e4567-e89b-42d3-a456-426614174099',
|
|
'target_reservation_id' => 9004,
|
|
'backup_job_id' => 9005,
|
|
'restore_job_id' => 9006,
|
|
'firewall_job_id' => null,
|
|
'rollback_job_id' => $migrationRollback['id'],
|
|
'cleanup_job_id' => $migrationCleanup['id'],
|
|
'cleanup_mode' => 'delete',
|
|
'state' => 'cleanup_uncertain',
|
|
'source_was_running' => true,
|
|
'error_code' => 'operation_effect_unknown',
|
|
'cutover_at' => $now - 60,
|
|
'completed_at' => null,
|
|
'created_at' => $now - 300,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
$addFixture(
|
|
'instance_migration_cleanup',
|
|
'source-cleanup',
|
|
$migrationCleanup,
|
|
'delete-vm',
|
|
$instanceUuid,
|
|
$deleteProjection
|
|
);
|
|
$addFixture(
|
|
'instance_migration_rollback',
|
|
'source-rollback',
|
|
$migrationRollback,
|
|
'service-resume',
|
|
$instanceUuid,
|
|
['uuid' => $instanceUuid, 'state' => 'running']
|
|
);
|
|
|
|
$instanceFixture(
|
|
'network_change',
|
|
'public-ip-change',
|
|
'POST',
|
|
'/network',
|
|
[
|
|
'network' => 'bridge:br0',
|
|
'public_pool_key' => str_repeat('c', 32),
|
|
'public_pool_cidr' => '203.0.113.0/24',
|
|
'public_ip_cidr' => '203.0.113.20/24',
|
|
'gateway' => '203.0.113.1',
|
|
'dns_servers' => ['1.1.1.1'],
|
|
'confirm_restart' => true,
|
|
],
|
|
'change-network',
|
|
['uuid' => $instanceUuid, 'network_changed' => true]
|
|
);
|
|
$instanceFixture(
|
|
'instance_settings',
|
|
'hostname',
|
|
'PATCH',
|
|
'/settings',
|
|
[
|
|
'hostname' => 'matrix.example',
|
|
'confirm_restart' => true,
|
|
],
|
|
'update-settings',
|
|
['uuid' => $instanceUuid, 'settings_changed' => true],
|
|
['settings_fields' => ['hostname'], 'credential' => null]
|
|
);
|
|
$upgradeProjection = [
|
|
'uuid' => $instanceUuid,
|
|
'from_vcpu' => 2,
|
|
'target_vcpu' => 4,
|
|
'applied_vcpu' => 4,
|
|
'from_memory_mib' => 2048,
|
|
'target_memory_mib' => 4096,
|
|
'applied_memory_mib' => 4096,
|
|
'from_disk_gib' => 40,
|
|
'target_disk_gib' => 80,
|
|
'applied_disk_gib' => 80,
|
|
'from_bandwidth_mbps' => 100,
|
|
'target_bandwidth_mbps' => 200,
|
|
'applied_bandwidth_mbps' => 200,
|
|
'status' => 'running',
|
|
'originally_running' => true,
|
|
'power_state_restored' => true,
|
|
'resources_resized' => true,
|
|
];
|
|
$upgradeRecord = $instanceFixture(
|
|
'instance_upgrade',
|
|
'grow-only',
|
|
'POST',
|
|
'/resources',
|
|
[
|
|
'from_vcpu' => 2,
|
|
'target_vcpu' => 4,
|
|
'from_memory_mib' => 2048,
|
|
'target_memory_mib' => 4096,
|
|
'from_disk_gib' => 40,
|
|
'target_disk_gib' => 80,
|
|
'from_bandwidth_mbps' => 100,
|
|
'target_bandwidth_mbps' => 200,
|
|
'confirm_shutdown' => true,
|
|
],
|
|
'resize-vm',
|
|
$upgradeProjection
|
|
);
|
|
|
|
$snapshotFixture = static function (
|
|
string $kind,
|
|
string $variant,
|
|
string $method,
|
|
string $suffix,
|
|
array $body,
|
|
string $expectedAction,
|
|
array $projection,
|
|
string $consistency = 'crash'
|
|
) use (
|
|
$insertJob,
|
|
$addFixture,
|
|
$instanceId,
|
|
$instanceUuid,
|
|
$ownerId,
|
|
$hostId,
|
|
$now
|
|
): void {
|
|
static $snapshotId = 100;
|
|
$id = $snapshotId++;
|
|
$key = hash('md5', 'recovery-matrix-snapshot:' . $id);
|
|
$record = $insertJob(
|
|
$kind,
|
|
static function (int $jobId, string $operationId) use (
|
|
$kind,
|
|
$method,
|
|
$suffix,
|
|
$body,
|
|
$key,
|
|
$instanceUuid,
|
|
$ownerId
|
|
): array {
|
|
return [
|
|
'method' => $method,
|
|
'request_target' => '/v1/vms/'
|
|
. rawurlencode($instanceUuid) . '/snapshots'
|
|
. ($kind === 'snapshot_create' ? ''
|
|
: '/' . rawurlencode($key) . $suffix),
|
|
'body' => $body + ($body === [] ? [] : [
|
|
'snapshot_key' => $key,
|
|
]),
|
|
'owner_id' => $ownerId,
|
|
'can_manage' => false,
|
|
'snapshot_key' => $key,
|
|
'snapshot_previous_state' => 'ready',
|
|
];
|
|
},
|
|
$instanceId,
|
|
$id
|
|
);
|
|
Capsule::table('cloud_instance_snapshots')->insert([
|
|
'id' => $id,
|
|
'snapshot_key' => $key,
|
|
'instance_id' => $instanceId,
|
|
'user_id' => $ownerId,
|
|
'host_id' => $hostId,
|
|
'name' => 'matrix-snapshot-' . $id,
|
|
'state' => 'uncertain',
|
|
'consistency' => $consistency,
|
|
'size_bytes' => null,
|
|
'last_operation_id' => $record['operation_id'],
|
|
'ready_at' => $now - 300,
|
|
'last_restored_at' => null,
|
|
'deleted_at' => null,
|
|
'created_at' => $now - 600,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
$resolvedAction = str_replace('__SNAPSHOT_KEY__', $key,
|
|
$expectedAction);
|
|
$resolvedProjection = array_map(
|
|
static fn (mixed $value): mixed =>
|
|
$value === '__SNAPSHOT_KEY__' ? $key : $value,
|
|
$projection
|
|
);
|
|
$addFixture(
|
|
$kind,
|
|
$variant,
|
|
$record,
|
|
$resolvedAction,
|
|
$instanceUuid,
|
|
$resolvedProjection
|
|
);
|
|
};
|
|
foreach (['filesystem', 'crash'] as $consistency) {
|
|
$snapshotFixture(
|
|
'snapshot_create',
|
|
$consistency,
|
|
'POST',
|
|
'',
|
|
[
|
|
'name' => 'matrix-' . $consistency,
|
|
'description' => '',
|
|
'quiesce' => $consistency === 'filesystem',
|
|
'consistency_mode' => $consistency,
|
|
],
|
|
'create-snapshot:__SNAPSHOT_KEY__:' . $consistency,
|
|
[
|
|
'id' => 'matrix-id',
|
|
'snapshot_key' => '__SNAPSHOT_KEY__',
|
|
'created_at' => '2026-10-21T00:00:00Z',
|
|
'state' => 'ready',
|
|
'consistency' => $consistency,
|
|
],
|
|
$consistency
|
|
);
|
|
}
|
|
$snapshotFixture(
|
|
'snapshot_restore',
|
|
'restore',
|
|
'POST',
|
|
'/restore',
|
|
[],
|
|
'restore-snapshot:__SNAPSHOT_KEY__',
|
|
['snapshot_key' => '__SNAPSHOT_KEY__', 'restored' => true]
|
|
);
|
|
$snapshotFixture(
|
|
'snapshot_delete',
|
|
'delete',
|
|
'DELETE',
|
|
'',
|
|
[],
|
|
'delete-snapshot:__SNAPSHOT_KEY__',
|
|
['snapshot_key' => '__SNAPSHOT_KEY__', 'deleted' => true]
|
|
);
|
|
|
|
foreach (['filesystem', 'crash'] as $consistency) {
|
|
$backupId = $consistency === 'filesystem' ? 201 : 202;
|
|
$backupKey = hash('md5', 'recovery-matrix-backup:' . $backupId);
|
|
$record = $insertJob(
|
|
'backup_create',
|
|
static function (int $jobId, string $operationId) use (
|
|
$instanceUuid,
|
|
$ownerId,
|
|
$backupKey,
|
|
$consistency
|
|
): array {
|
|
return [
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/vms/'
|
|
. rawurlencode($instanceUuid) . '/backups',
|
|
'body' => [
|
|
'operation_id' => $operationId,
|
|
'backup_key' => $backupKey,
|
|
'name' => 'matrix-' . $consistency,
|
|
'consistency_mode' => $consistency,
|
|
'object_upload' => [
|
|
'part_size_bytes' => 8388608,
|
|
'maximum_size_bytes' => 42949672960,
|
|
'part_urls' => [
|
|
'https://objects.invalid/part-1',
|
|
],
|
|
'complete_url' =>
|
|
'https://objects.invalid/complete',
|
|
'abort_url' => 'https://objects.invalid/abort',
|
|
'verify_url' => 'https://objects.invalid/verify',
|
|
],
|
|
],
|
|
'owner_id' => $ownerId,
|
|
'can_manage' => true,
|
|
'backup_key' => $backupKey,
|
|
'maximum_size_bytes' => 42949672960,
|
|
];
|
|
},
|
|
$instanceId,
|
|
null,
|
|
null,
|
|
null,
|
|
'agent',
|
|
$ownerId,
|
|
$backupId
|
|
);
|
|
Capsule::table('cloud_instance_backups')->insert([
|
|
'id' => $backupId,
|
|
'backup_key' => $backupKey,
|
|
'source_instance_id' => $instanceId,
|
|
'source_host_id' => $hostId,
|
|
'user_id' => $ownerId,
|
|
'source_uuid' => $instanceUuid,
|
|
'name' => 'matrix-' . $consistency,
|
|
'state' => 'uncertain',
|
|
'consistency' => $consistency,
|
|
'object_key' => 'vm-backups/' . $backupKey . '/system.qcow2',
|
|
'spec_json' => '{}',
|
|
'sealed_storage_runtime' => 'sealed-runtime',
|
|
'active_job_id' => $record['id'],
|
|
'last_operation_id' => $record['operation_id'],
|
|
'created_at' => $now - 180,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
$addFixture(
|
|
'backup_create',
|
|
$consistency,
|
|
$record,
|
|
'create-backup:' . $backupKey . ':' . $consistency,
|
|
$instanceUuid,
|
|
[
|
|
'backup_key' => $backupKey,
|
|
'size_bytes' => 1073741824,
|
|
'virtual_size_bytes' => 42949672960,
|
|
'sha256' => str_repeat('6', 64),
|
|
'consistency' => $consistency,
|
|
]
|
|
);
|
|
}
|
|
|
|
$firewallVersionId = 1;
|
|
$firewallRevision = 9;
|
|
$firewallDigest = str_repeat('d', 64);
|
|
$firewallRecord = $instanceFixture(
|
|
'firewall_apply',
|
|
'apply',
|
|
'PUT',
|
|
'/firewall',
|
|
[
|
|
'revision' => $firewallRevision,
|
|
'default_in' => 'drop',
|
|
'default_out' => 'accept',
|
|
'policy_digest' => $firewallDigest,
|
|
'rules' => [],
|
|
],
|
|
'apply-firewall',
|
|
[
|
|
'revision' => $firewallRevision,
|
|
'default_in' => 'drop',
|
|
'default_out' => 'accept',
|
|
'rules' => [],
|
|
],
|
|
[
|
|
'firewall_version_id' => $firewallVersionId,
|
|
'firewall_revision' => $firewallRevision,
|
|
'agent_firewall_policy_digest' => $firewallDigest,
|
|
]
|
|
);
|
|
Capsule::table('cloud_instance_firewall_versions')->insert([
|
|
'id' => $firewallVersionId,
|
|
'instance_id' => $instanceId,
|
|
'revision' => $firewallRevision,
|
|
'enabled' => 1,
|
|
'policy_digest' => $firewallDigest,
|
|
'request_hash' => str_repeat('e', 64),
|
|
'idempotency_key' => 'matrix-firewall-version',
|
|
'state' => 'uncertain',
|
|
'job_id' => $firewallRecord['id'],
|
|
'operation_id' => $firewallRecord['operation_id'],
|
|
'error_code' => '',
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
|
|
$instanceFixture(
|
|
'service_suspend',
|
|
'purge',
|
|
'POST',
|
|
'/service/suspend',
|
|
['reason' => 'purge'],
|
|
'service-suspend',
|
|
['uuid' => $instanceUuid, 'state' => 'shutoff']
|
|
);
|
|
$instanceFixture(
|
|
'network_suspend',
|
|
'traffic',
|
|
'POST',
|
|
'/network/suspend',
|
|
['reason' => 'traffic'],
|
|
'network-suspend',
|
|
['network_enabled' => false]
|
|
);
|
|
$instanceFixture(
|
|
'network_resume',
|
|
'resume',
|
|
'POST',
|
|
'/network/resume',
|
|
[],
|
|
'network-resume',
|
|
['network_enabled' => true]
|
|
);
|
|
|
|
$insertAggregateParent = static function (
|
|
string $kind,
|
|
int $userId = 41
|
|
) use ($insertJob): array {
|
|
return $insertJob(
|
|
$kind,
|
|
static fn (int $id, string $operationId): array => [
|
|
'aggregate_kind' => $kind,
|
|
],
|
|
null,
|
|
null,
|
|
null,
|
|
null,
|
|
'aggregate',
|
|
$userId
|
|
);
|
|
};
|
|
|
|
$templateFixture = static function (
|
|
string $kind,
|
|
string $parentKind,
|
|
string $method,
|
|
string $expectedAction,
|
|
array $projection
|
|
) use (
|
|
$insertAggregateParent,
|
|
$insertJob,
|
|
$addFixture,
|
|
$hostId,
|
|
$ownerId,
|
|
$now
|
|
): void {
|
|
$parent = $insertAggregateParent($parentKind);
|
|
$templateId = 'matrix-' . str_replace('_', '-', $kind);
|
|
$revision = 5;
|
|
$generationToken = hash('md5', 'generation:' . $kind);
|
|
$record = $insertJob(
|
|
$kind,
|
|
static function (int $jobId, string $operationId) use (
|
|
$kind,
|
|
$method,
|
|
$templateId,
|
|
$revision,
|
|
$generationToken,
|
|
$ownerId
|
|
): array {
|
|
$target = $kind === 'template_import_replica'
|
|
? '/v1/templates'
|
|
: '/v1/templates/' . rawurlencode($templateId);
|
|
$body = ['operation_id' => $operationId];
|
|
if ($kind === 'template_import_replica') {
|
|
$body += [
|
|
'id' => $templateId,
|
|
'name' => 'Matrix Template',
|
|
'source_url' => 'https://example.invalid/template.qcow2',
|
|
'source_token' => 'matrix-token',
|
|
'source_size_bytes' => 2097152,
|
|
'os_family' => 'linux',
|
|
'version' => '13',
|
|
'architecture' => 'x86_64',
|
|
'sha256' => str_repeat('1', 64),
|
|
'revision' => $revision,
|
|
'generation_token' => $generationToken,
|
|
];
|
|
} elseif ($kind === 'template_metadata_replica') {
|
|
$body += [
|
|
'name' => 'Matrix Template Updated',
|
|
'revision' => $revision,
|
|
'generation_token' => $generationToken,
|
|
];
|
|
}
|
|
return [
|
|
'method' => $method,
|
|
'request_target' => $target,
|
|
'body' => $body,
|
|
'owner_id' => $ownerId,
|
|
'can_manage' => true,
|
|
'template_id' => $templateId,
|
|
];
|
|
},
|
|
null,
|
|
null,
|
|
$parent['id']
|
|
);
|
|
Capsule::table('cloud_templates')->insert([
|
|
'id' => $templateId,
|
|
'name' => 'Matrix Template',
|
|
'revision' => $revision,
|
|
'generation_token' => $generationToken,
|
|
'state' => 'active',
|
|
'active_job_id' => $parent['id'],
|
|
'created_at' => $now - 300,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
Capsule::table('cloud_template_replicas')->insert([
|
|
'host_id' => $hostId,
|
|
'template_id' => $templateId,
|
|
'revision' => $revision,
|
|
'generation_token' => $generationToken,
|
|
'state' => 'uncertain',
|
|
'image_bytes' => 0,
|
|
'sha256' => '',
|
|
'operation_id' => $record['operation_id'],
|
|
'job_id' => $record['id'],
|
|
'error_code' => '',
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
$addFixture(
|
|
$kind,
|
|
'catalog',
|
|
$record,
|
|
$expectedAction,
|
|
'template:' . $templateId,
|
|
$projection
|
|
);
|
|
};
|
|
$templateFixture(
|
|
'template_import_replica',
|
|
'template_import',
|
|
'POST',
|
|
'import-template',
|
|
[
|
|
'id' => 'matrix-template-import-replica',
|
|
'generation_token' => hash('md5',
|
|
'generation:template_import_replica'),
|
|
'created_at' => '2026-10-21',
|
|
]
|
|
);
|
|
$templateFixture(
|
|
'template_metadata_replica',
|
|
'template_metadata_sync',
|
|
'PATCH',
|
|
'update-template',
|
|
[
|
|
'id' => 'matrix-template-metadata-replica',
|
|
'name' => 'Matrix Template Updated',
|
|
'revision' => 5,
|
|
'generation_token' => hash('md5',
|
|
'generation:template_metadata_replica'),
|
|
]
|
|
);
|
|
$templateFixture(
|
|
'template_delete_replica',
|
|
'template_delete',
|
|
'DELETE',
|
|
'delete-template',
|
|
[
|
|
'resource_type' => 'template',
|
|
'id' => 'matrix-template-delete-replica',
|
|
'deleted' => true,
|
|
'managed_image_absent' => true,
|
|
'managed_metadata_absent' => true,
|
|
'managed_artifacts_absent' => true,
|
|
]
|
|
);
|
|
|
|
$rescueImportId = hash('md5', 'matrix-rescue-import');
|
|
$rescueImport = $insertJob(
|
|
'rescue_template_import',
|
|
static fn (int $jobId, string $operationId): array => [
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/rescue-templates',
|
|
'body' => [
|
|
'operation_id' => $operationId,
|
|
'id' => $rescueImportId,
|
|
'name' => 'Matrix Rescue',
|
|
'source_url' => 'https://example.invalid/rescue.iso',
|
|
'source_token' => 'matrix-token',
|
|
'source_size_bytes' => 2097152,
|
|
'sha256' => str_repeat('2', 64),
|
|
],
|
|
'owner_id' => $ownerId,
|
|
'can_manage' => true,
|
|
'rescue_template_id' => $rescueImportId,
|
|
]
|
|
);
|
|
Capsule::table('cloud_rescue_template_imports')->insert([
|
|
'operation_id' => $rescueImport['operation_id'],
|
|
'host_id' => $hostId,
|
|
'template_id' => $rescueImportId,
|
|
'job_id' => $rescueImport['id'],
|
|
'import_type' => 'url',
|
|
'name' => 'Matrix Rescue',
|
|
'size_bytes' => 0,
|
|
'sha256' => '',
|
|
'state' => 'uncertain',
|
|
'error_code' => '',
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
$addFixture(
|
|
'rescue_template_import',
|
|
'catalog',
|
|
$rescueImport,
|
|
'import-rescue-template',
|
|
'rescue-template:' . $rescueImportId,
|
|
['id' => $rescueImportId, 'created_at' => '2026-10-21']
|
|
);
|
|
|
|
$rescueDeleteId = hash('md5', 'matrix-rescue-delete');
|
|
$rescueDelete = $insertJob(
|
|
'rescue_template_delete',
|
|
static fn (int $jobId, string $operationId): array => [
|
|
'method' => 'DELETE',
|
|
'request_target' => '/v1/rescue-templates/'
|
|
. rawurlencode($rescueDeleteId),
|
|
'body' => ['operation_id' => $operationId],
|
|
'owner_id' => $ownerId,
|
|
'can_manage' => true,
|
|
'rescue_template_id' => $rescueDeleteId,
|
|
]
|
|
);
|
|
Capsule::table('cloud_rescue_template_deletions')->insert([
|
|
'host_id' => $hostId,
|
|
'template_id' => $rescueDeleteId,
|
|
'operation_id' => $rescueDelete['operation_id'],
|
|
'job_id' => $rescueDelete['id'],
|
|
'state' => 'uncertain',
|
|
'error_code' => '',
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
$addFixture(
|
|
'rescue_template_delete',
|
|
'catalog',
|
|
$rescueDelete,
|
|
'delete-rescue-template',
|
|
'rescue-template:' . $rescueDeleteId,
|
|
[
|
|
'resource_type' => 'rescue_template',
|
|
'id' => $rescueDeleteId,
|
|
'deleted' => true,
|
|
'managed_image_absent' => true,
|
|
'managed_metadata_absent' => true,
|
|
'managed_artifacts_absent' => true,
|
|
]
|
|
);
|
|
|
|
$isoFixture = static function (
|
|
string $kind,
|
|
string $parentKind,
|
|
string $method,
|
|
string $expectedAction,
|
|
array $projection
|
|
) use (
|
|
$insertAggregateParent,
|
|
$insertJob,
|
|
$addFixture,
|
|
$hostId,
|
|
$ownerId,
|
|
$now
|
|
): void {
|
|
$parent = $insertAggregateParent($parentKind, $ownerId);
|
|
$isoId = hash('md5', 'matrix-iso:' . $kind);
|
|
$record = $insertJob(
|
|
$kind,
|
|
static function (int $jobId, string $operationId) use (
|
|
$kind,
|
|
$method,
|
|
$isoId,
|
|
$ownerId
|
|
): array {
|
|
$target = $kind === 'user_iso_import_replica'
|
|
? '/v1/user-isos'
|
|
: '/v1/user-isos/' . rawurlencode($isoId)
|
|
. '?owner_id=' . $ownerId;
|
|
$body = ['operation_id' => $operationId];
|
|
if ($kind === 'user_iso_import_replica') {
|
|
$body += [
|
|
'id' => $isoId,
|
|
'owner_id' => $ownerId,
|
|
'name' => 'Matrix ISO',
|
|
'source_url' => 'https://example.invalid/user.iso',
|
|
'source_token' => 'matrix-token',
|
|
'sha256' => str_repeat('3', 64),
|
|
];
|
|
}
|
|
return [
|
|
'method' => $method,
|
|
'request_target' => $target,
|
|
'body' => $body,
|
|
'owner_id' => $ownerId,
|
|
'can_manage' => false,
|
|
'user_iso_id' => $isoId,
|
|
];
|
|
},
|
|
null,
|
|
null,
|
|
$parent['id'],
|
|
null,
|
|
'agent',
|
|
$ownerId
|
|
);
|
|
Capsule::table('cloud_user_isos')->insert([
|
|
'id' => $isoId,
|
|
'user_id' => $ownerId,
|
|
'name' => 'Matrix ISO',
|
|
'state' => 'active',
|
|
'active_job_id' => $parent['id'],
|
|
'created_at' => $now - 300,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
Capsule::table('cloud_user_iso_replicas')->insert([
|
|
'host_id' => $hostId,
|
|
'user_iso_id' => $isoId,
|
|
'user_id' => $ownerId,
|
|
'state' => 'uncertain',
|
|
'size_bytes' => 0,
|
|
'sha256' => '',
|
|
'operation_id' => $record['operation_id'],
|
|
'job_id' => $record['id'],
|
|
'error_code' => '',
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
$addFixture(
|
|
$kind,
|
|
'catalog',
|
|
$record,
|
|
$expectedAction,
|
|
'user-iso:' . $ownerId . ':' . $isoId,
|
|
$projection
|
|
);
|
|
};
|
|
$isoFixture(
|
|
'user_iso_import_replica',
|
|
'user_iso_import',
|
|
'POST',
|
|
'import-user-iso',
|
|
['id' => hash('md5', 'matrix-iso:user_iso_import_replica'),
|
|
'created_at' => '2026-10-21']
|
|
);
|
|
$isoDeleteId = hash('md5', 'matrix-iso:user_iso_delete_replica');
|
|
$isoFixture(
|
|
'user_iso_delete_replica',
|
|
'user_iso_delete',
|
|
'DELETE',
|
|
'delete-user-iso',
|
|
[
|
|
'resource_type' => 'user_iso',
|
|
'id' => $isoDeleteId,
|
|
'deleted' => true,
|
|
'managed_image_absent' => true,
|
|
'managed_metadata_absent' => true,
|
|
'managed_artifacts_absent' => true,
|
|
]
|
|
);
|
|
|
|
// Dedicated create/storage identities are covered without admitting them
|
|
// to the generic worker/finalizer matrix.
|
|
$dedicatedFixtures = [];
|
|
$restoreBackupId = 203;
|
|
$restoreId = 301;
|
|
$restoreInstanceId = 2;
|
|
$restoreReservationId = 401;
|
|
$restoreIpId = 501;
|
|
$restoreBackupKey = hash('md5', 'recovery-matrix-restore-backup');
|
|
$restoreKey = hash('md5', 'recovery-matrix-restore');
|
|
$restoreUuid = '923e4567-e89b-42d3-a456-426614174001';
|
|
$restoreName = 'matrix-restored-vm';
|
|
$restoreIp = '203.0.113.31';
|
|
$restorePoolKey = str_repeat('7', 32);
|
|
$restoreSha256 = str_repeat('8', 64);
|
|
$restoreSpec = [
|
|
'managed' => true,
|
|
'os_source' => 'cloud_template',
|
|
'os_name' => 'Debian 12',
|
|
'login_name' => 'root',
|
|
'vcpu' => 2,
|
|
'memory_mib' => 2048,
|
|
'disk_gib' => 40,
|
|
'bandwidth_mbps' => 100,
|
|
'network' => 'bridge:br0',
|
|
'public_pool_key' => $restorePoolKey,
|
|
'public_ip_cidr' => $restoreIp . '/24',
|
|
'debian_12_network_fix' => true,
|
|
];
|
|
$restoreRecord = $insertJob(
|
|
'backup_restore',
|
|
static function (int $jobId, string $operationId) use (
|
|
$restoreUuid,
|
|
$restoreName,
|
|
$restoreIp,
|
|
$restorePoolKey,
|
|
$restoreBackupKey,
|
|
$restoreKey,
|
|
$restoreReservationId,
|
|
$restoreSha256,
|
|
$ownerId
|
|
): array {
|
|
return [
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/vms/'
|
|
. rawurlencode($restoreUuid) . '/restore-backup',
|
|
'body' => [
|
|
'operation_id' => $operationId,
|
|
'uuid' => $restoreUuid,
|
|
'source' => '',
|
|
'name' => $restoreName,
|
|
'description' => 'Recovery matrix restore',
|
|
'vcpu' => 2,
|
|
'memory_mib' => 2048,
|
|
'disk_gib' => 40,
|
|
'bandwidth_mbps' => 100,
|
|
'network' => 'bridge:br0',
|
|
'template_id' => 'debian-12',
|
|
'template_revision' => 1,
|
|
'template_generation_token' => str_repeat('9', 32),
|
|
'startup_script' => '',
|
|
'hostname' => $restoreName,
|
|
'guest_ssh_public_key' => '',
|
|
'root_password' => '',
|
|
'public_pool_key' => $restorePoolKey,
|
|
'public_pool_cidr' => '203.0.113.0/24',
|
|
'public_ip_cidr' => $restoreIp . '/24',
|
|
'gateway' => '203.0.113.1',
|
|
'dns_servers' => ['1.1.1.1'],
|
|
'platform_network_policy_snapshot' => [
|
|
'schema_version' => 1,
|
|
],
|
|
'debian_12_network_fix' => true,
|
|
'backup_key' => $restoreBackupKey,
|
|
'source_url' =>
|
|
'https://objects.invalid/matrix-restore.qcow2',
|
|
'source_size_bytes' => 1073741824,
|
|
'source_sha256' => $restoreSha256,
|
|
],
|
|
'owner_id' => $ownerId,
|
|
'can_manage' => true,
|
|
'backup_key' => $restoreBackupKey,
|
|
'restore_key' => $restoreKey,
|
|
'reservation_id' => $restoreReservationId,
|
|
];
|
|
},
|
|
$restoreInstanceId,
|
|
null,
|
|
null,
|
|
null,
|
|
'agent',
|
|
$ownerId,
|
|
$restoreBackupId,
|
|
$restoreId
|
|
);
|
|
Capsule::table('cloud_instance_backups')->insert([
|
|
'id' => $restoreBackupId,
|
|
'backup_key' => $restoreBackupKey,
|
|
'source_instance_id' => 999,
|
|
'source_host_id' => $hostId + 1,
|
|
'user_id' => $ownerId,
|
|
'source_uuid' => '123e4567-e89b-42d3-a456-426614174099',
|
|
'name' => 'matrix-restore-source',
|
|
'state' => 'ready',
|
|
'consistency' => 'crash',
|
|
'object_key' => 'vm-backups/' . $restoreBackupKey . '/system.qcow2',
|
|
'object_size_bytes' => 1073741824,
|
|
'virtual_size_bytes' => 42949672960,
|
|
'sha256' => $restoreSha256,
|
|
'spec_json' => json_encode($restoreSpec, JSON_THROW_ON_ERROR),
|
|
'sealed_storage_runtime' => 'sealed-runtime',
|
|
'sealed_credential' => 'sealed-credential',
|
|
'storage_revision' => 1,
|
|
'active_job_id' => null,
|
|
'last_operation_id' => null,
|
|
'error_code' => '',
|
|
'ready_at' => $now - 300,
|
|
'created_at' => $now - 600,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
Capsule::table('cloud_instances')->insert([
|
|
'id' => $restoreInstanceId,
|
|
'uuid' => $restoreUuid,
|
|
'host_id' => $hostId,
|
|
'user_id' => $ownerId,
|
|
'reservation_id' => $restoreReservationId,
|
|
'name' => $restoreName,
|
|
'status' => 'provisioning',
|
|
'service_status' => 'exempt',
|
|
'network_status' => 'pending',
|
|
'spec_json' => json_encode($restoreSpec, JSON_THROW_ON_ERROR),
|
|
'sealed_credential' => 'sealed-credential',
|
|
'created_at' => $now - 180,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
Capsule::table('cloud_public_ips')->insert([
|
|
'id' => $restoreIpId,
|
|
'host_id' => $hostId,
|
|
'address' => $restoreIp,
|
|
'prefix_length' => 24,
|
|
'state' => 'reserved',
|
|
'reservation_id' => $restoreReservationId,
|
|
'instance_id' => null,
|
|
'operation_id' => $restoreRecord['operation_id'],
|
|
'created_at' => $now - 180,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
Capsule::table('cloud_resource_reservations')->insert([
|
|
'id' => $restoreReservationId,
|
|
'backup_restore_id' => $restoreId,
|
|
'host_id' => $hostId,
|
|
'user_id' => $ownerId,
|
|
'operation_id' => $restoreRecord['operation_id'],
|
|
'public_ip_id' => $restoreIpId,
|
|
'instance_id' => $restoreInstanceId,
|
|
'cpu_cores' => 2,
|
|
'memory_mib' => 2048,
|
|
'disk_gib' => 40,
|
|
'state' => 'held',
|
|
'created_at' => $now - 180,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
Capsule::table('cloud_backup_restores')->insert([
|
|
'id' => $restoreId,
|
|
'restore_key' => $restoreKey,
|
|
'backup_id' => $restoreBackupId,
|
|
'target_host_id' => $hostId,
|
|
'target_instance_id' => $restoreInstanceId,
|
|
'reservation_id' => $restoreReservationId,
|
|
'job_id' => $restoreRecord['id'],
|
|
'operation_id' => $restoreRecord['operation_id'],
|
|
'new_uuid' => $restoreUuid,
|
|
'new_name' => $restoreName,
|
|
'state' => 'uncertain',
|
|
'created_at' => $now - 180,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
$dedicatedFixtures['backup_restore'] = $restoreRecord + [
|
|
'expected_action' => 'restore-backup:' . $restoreBackupKey,
|
|
'expected_resource' => $restoreUuid,
|
|
];
|
|
$createRecord = $insertJob(
|
|
'create',
|
|
static fn (int $jobId, string $operationId): array => [
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/vms',
|
|
'body' => [
|
|
'operation_id' => $operationId,
|
|
'source' => 'cloud',
|
|
'name' => 'matrix-create',
|
|
'vcpu' => 2,
|
|
'memory_mib' => 2048,
|
|
'disk_gib' => 40,
|
|
'bandwidth_mbps' => 100,
|
|
'network' => 'bridge:br0',
|
|
'template_id' => 'debian-13',
|
|
'template_revision' => 1,
|
|
'template_generation_token' => str_repeat('4', 32),
|
|
'root_password' => 'MatrixPassword!123',
|
|
'public_pool_key' => str_repeat('5', 32),
|
|
'public_pool_cidr' => '203.0.113.0/24',
|
|
'public_ip_cidr' => '203.0.113.30/24',
|
|
'gateway' => '203.0.113.1',
|
|
'dns_servers' => ['1.1.1.1'],
|
|
'platform_network_policy_snapshot' => [
|
|
'schema_version' => 1,
|
|
],
|
|
],
|
|
'credential' => [
|
|
'password' => 'MatrixPassword!123',
|
|
'login_name' => 'root',
|
|
'auth_mode' => 'password',
|
|
],
|
|
'template_metadata' => [
|
|
'template_id' => 'debian-13',
|
|
'os_source' => 'cloud_template',
|
|
'os_name' => 'Debian 13',
|
|
'login_name' => 'root',
|
|
'debian_12_network_fix' => false,
|
|
],
|
|
],
|
|
null,
|
|
null,
|
|
null,
|
|
9001
|
|
);
|
|
$dedicatedFixtures['create'] = $createRecord + [
|
|
'expected_action' => 'create-vm',
|
|
];
|
|
|
|
$storageBody = [
|
|
'managed_directory' => '/srv/virt/managed',
|
|
'template_directory' => '/srv/virt/templates',
|
|
'seed_directory' => '/srv/virt/seeds',
|
|
'user_iso_directory' => '/srv/virt/user-isos',
|
|
'rescue_directory' => '/srv/virt/rescue',
|
|
];
|
|
$storageRecord = $insertJob(
|
|
'host_storage_configure',
|
|
static fn (int $jobId, string $operationId): array => [
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/storage/configuration',
|
|
'body' => $storageBody + ['operation_id' => $operationId],
|
|
'owner_id' => $ownerId,
|
|
'can_manage' => true,
|
|
'storage_revision' => 3,
|
|
]
|
|
);
|
|
Capsule::table('cloud_host_storage_configurations')->insert([
|
|
'host_id' => $hostId,
|
|
'revision' => 3,
|
|
'managed_directory' => $storageBody['managed_directory'],
|
|
'template_directory' => $storageBody['template_directory'],
|
|
'seed_directory' => $storageBody['seed_directory'],
|
|
'user_iso_directory' => $storageBody['user_iso_directory'],
|
|
'rescue_directory' => $storageBody['rescue_directory'],
|
|
'state' => 'uncertain',
|
|
'operation_id' => $storageRecord['operation_id'],
|
|
'job_id' => $storageRecord['id'],
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
$dedicatedFixtures['host_storage_configure'] = $storageRecord + [
|
|
'expected_action' => 'configure-storage',
|
|
'expected_resource' => 'host-storage',
|
|
];
|
|
|
|
$classificationOnlyKinds = [
|
|
'payment_create' => 'payment',
|
|
'template_import' => 'aggregate',
|
|
'template_metadata_sync' => 'aggregate',
|
|
'template_delete' => 'aggregate',
|
|
'user_iso_import' => 'aggregate',
|
|
'user_iso_delete' => 'aggregate',
|
|
];
|
|
$genericKinds = array_keys(array_filter(
|
|
$manifestOperations,
|
|
static fn (array $operation): bool =>
|
|
$operation['recovery_mode'] === 'generic'
|
|
));
|
|
$fixtureKinds = array_keys($fixtures);
|
|
sort($genericKinds, SORT_STRING);
|
|
sort($fixtureKinds, SORT_STRING);
|
|
$assert($fixtureKinds === $genericKinds,
|
|
'Every generic recovery kind must have an executable fixture.');
|
|
|
|
$coveredKinds = array_merge(
|
|
$fixtureKinds,
|
|
array_keys($dedicatedFixtures),
|
|
array_keys($classificationOnlyKinds)
|
|
);
|
|
$contractKinds = array_keys($manifestOperations);
|
|
sort($coveredKinds, SORT_STRING);
|
|
sort($contractKinds, SORT_STRING);
|
|
$assert($coveredKinds === $contractKinds,
|
|
'A recovery contract kind was added without a matrix fixture.');
|
|
|
|
$expectedAutomaticKinds = array_keys(array_filter(
|
|
$manifestOperations,
|
|
static fn (array $operation): bool => in_array(
|
|
$operation['recovery_mode'],
|
|
['generic', 'create'],
|
|
true
|
|
)
|
|
));
|
|
$assert(CloudOperationRecoveryPolicy::kinds()
|
|
=== array_keys(array_filter(
|
|
$manifestOperations,
|
|
static fn (array $operation): bool =>
|
|
$operation['recovery_mode'] === 'generic'
|
|
))
|
|
&& CloudOperationRecoveryPolicy::automaticKinds()
|
|
=== $expectedAutomaticKinds,
|
|
'Policy kind lists diverged from the recovery modes.');
|
|
|
|
$projectionMethod = new ReflectionMethod(
|
|
CloudAdminOperationRecoveryService::class,
|
|
'effectProjection'
|
|
);
|
|
$expectInvalidProjection = static function (
|
|
array $projection,
|
|
string $kind,
|
|
string $message
|
|
) use ($projectionMethod, $assert): void {
|
|
try {
|
|
$projectionMethod->invoke(null, $projection, $kind, 'applied');
|
|
} catch (CloudAgentError $error) {
|
|
$assert($error->agentCode === 'agent_invalid_inspection'
|
|
&& $error->agentHttpStatus === 200,
|
|
'Unexpected invalid-projection error for ' . $kind . '.');
|
|
return;
|
|
}
|
|
throw new RuntimeException($message);
|
|
};
|
|
|
|
$upgradeIdentity = CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
$upgradeRecord['job'],
|
|
$upgradeRecord['payload']
|
|
);
|
|
$inspectionMethod = new ReflectionMethod(
|
|
CloudAdminOperationRecoveryService::class,
|
|
'inspection'
|
|
);
|
|
$upgradeNotAppliedRejected = false;
|
|
try {
|
|
$inspectionMethod->invoke(null, [
|
|
'operation_id' => $upgradeRecord['operation_id'],
|
|
'action' => $upgradeIdentity['action'],
|
|
'resource_id' => $upgradeIdentity['resource_id'],
|
|
'request_fingerprint' => $upgradeIdentity['request_fingerprint'],
|
|
'operation_status' => 'failed',
|
|
'checked_at' => time(),
|
|
'outcome' => 'failed',
|
|
'summary_code' => 'operation_failed',
|
|
'error_code' => 'resources_not_committed',
|
|
'effect_resolution' => 'not_applied',
|
|
'fence_state' => 'active',
|
|
'fence_resolution_required' => true,
|
|
'abort_job' => ['available' => false],
|
|
'cleanup' => ['available' => false],
|
|
], $upgradeRecord['operation_id'], 'instance_upgrade', true);
|
|
} catch (CloudAgentError $error) {
|
|
$assert($error->agentCode === 'agent_invalid_inspection'
|
|
&& $error->agentHttpStatus === 200,
|
|
'Unexpected fail-closed upgrade inspection error.');
|
|
$upgradeNotAppliedRejected = true;
|
|
}
|
|
$assert($upgradeNotAppliedRejected,
|
|
'Automatic recovery accepted unsafe not-applied upgrade evidence.');
|
|
$malformedUpgradeProjection = $upgradeProjection;
|
|
unset($malformedUpgradeProjection['applied_disk_gib']);
|
|
$expectInvalidProjection(
|
|
$malformedUpgradeProjection,
|
|
'instance_upgrade',
|
|
'Incomplete instance-upgrade evidence crossed the projection boundary.'
|
|
);
|
|
$unrestoredUpgradeProjection = $upgradeProjection;
|
|
$unrestoredUpgradeProjection['power_state_restored'] = false;
|
|
$expectInvalidProjection(
|
|
$unrestoredUpgradeProjection,
|
|
'instance_upgrade',
|
|
'An upgrade without restored power state crossed the projection boundary.'
|
|
);
|
|
$wrongPowerUpgradeProjection = $upgradeProjection;
|
|
$wrongPowerUpgradeProjection['originally_running'] = false;
|
|
$wrongPowerUpgradeProjection['status'] = 'running';
|
|
$expectInvalidProjection(
|
|
$wrongPowerUpgradeProjection,
|
|
'instance_upgrade',
|
|
'An upgrade with the wrong final power state crossed the projection boundary.'
|
|
);
|
|
|
|
foreach ($fixtures as $kind => $kindFixtures) {
|
|
$entry = $manifestOperations[$kind];
|
|
$assert(CloudOperationRecoveryPolicy::supports($kind)
|
|
&& CloudOperationRecoveryPolicy::identityScope($kind)
|
|
=== $entry['identity_scope']
|
|
&& CloudOperationRecoveryIdentity::supports($kind),
|
|
'Generic policy/identity support diverged for ' . $kind . '.');
|
|
|
|
$actualActions = [];
|
|
foreach ($kindFixtures as $fixture) {
|
|
$job = $fixture['job'];
|
|
$payload = $fixture['payload'];
|
|
$fromPayload = CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
$job,
|
|
$payload
|
|
);
|
|
$fromEncryptedJob = CloudOperationRecoveryIdentity::expected($job);
|
|
$label = $kind . ':' . $fixture['variant'];
|
|
$assert($fromPayload === $fromEncryptedJob,
|
|
'Encrypted identity reconstruction diverged for ' . $label);
|
|
$assert($fromPayload['action']
|
|
=== $fixture['expected_action']
|
|
&& $fromPayload['resource_id']
|
|
=== $fixture['expected_resource'],
|
|
'Identity action/resource mismatch for ' . $label);
|
|
$assert(CloudOperationRecoveryPolicy::allowsAction(
|
|
$kind,
|
|
$fromPayload['action']
|
|
),
|
|
'Contract rejected the rebuilt action for ' . $label);
|
|
|
|
$wireBody = $payload['body'];
|
|
if ($kind === 'action'
|
|
&& ($wireBody['action'] ?? null) === 'poweroff') {
|
|
$wireBody['action'] = 'destroy';
|
|
}
|
|
$encodedBody = json_encode(
|
|
$wireBody === [] ? (object) [] : $wireBody,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
);
|
|
$expectedFingerprint = hash(
|
|
'sha256',
|
|
strtoupper($payload['method']) . "\n"
|
|
. $payload['request_target'] . "\n" . $encodedBody
|
|
);
|
|
$assert(hash_equals(
|
|
$expectedFingerprint,
|
|
$fromPayload['request_fingerprint']
|
|
),
|
|
'Wire fingerprint mismatch for ' . $label);
|
|
|
|
$context = $kind === 'instance_settings'
|
|
? CloudWorkerService::settingsRecoveryContext($job)
|
|
: CloudWorkerService::recoveryContext($job);
|
|
$assert($context['payload'] === $payload,
|
|
'Worker lost the encrypted recovery payload for ' . $label);
|
|
if ($entry['identity_scope'] === 'instance') {
|
|
$assert(($context['instance']->id ?? null) === $instanceId,
|
|
'Worker lost the instance binding for ' . $label);
|
|
} else {
|
|
$assert($context['instance'] === null,
|
|
'Catalog recovery unexpectedly acquired an instance for '
|
|
. $label);
|
|
}
|
|
|
|
$projection = $projectionMethod->invoke(
|
|
null,
|
|
$fixture['projection'],
|
|
$kind,
|
|
'applied'
|
|
);
|
|
$assert($projection === $fixture['projection'],
|
|
'Evidence projection branch is missing for ' . $label);
|
|
$actualActions[] = $fromPayload['action'];
|
|
}
|
|
|
|
foreach ($entry['actions'] as $action) {
|
|
$assert(in_array($action, $actualActions, true),
|
|
'Literal action lacks a real identity fixture: '
|
|
. $kind . ':' . $action);
|
|
}
|
|
foreach ($entry['action_patterns'] as $pattern) {
|
|
$matched = array_filter(
|
|
$actualActions,
|
|
static fn (string $action): bool =>
|
|
preg_match('~' . $pattern . '~D', $action) === 1
|
|
);
|
|
$assert($matched !== [],
|
|
'Pattern action lacks a real identity fixture: '
|
|
. $kind . ':' . $pattern);
|
|
}
|
|
foreach ($actualActions as $action) {
|
|
$literal = in_array($action, $entry['actions'], true);
|
|
$pattern = false;
|
|
foreach ($entry['action_patterns'] as $candidate) {
|
|
$pattern = $pattern
|
|
|| preg_match('~' . $candidate . '~D', $action) === 1;
|
|
}
|
|
$assert($literal || $pattern,
|
|
'Fixture produced an uncontracted action: '
|
|
. $kind . ':' . $action);
|
|
}
|
|
}
|
|
|
|
$snapshotCreateActions = array_column(
|
|
$fixtures['snapshot_create'],
|
|
'expected_action'
|
|
);
|
|
$snapshotKeyPattern = '[a-f0-9]{32}';
|
|
$assert(count($snapshotCreateActions) === 2
|
|
&& count(array_filter(
|
|
$snapshotCreateActions,
|
|
static fn (string $action): bool => preg_match(
|
|
'~^create-snapshot:' . $snapshotKeyPattern
|
|
. ':filesystem$~D',
|
|
$action
|
|
) === 1
|
|
)) === 1
|
|
&& count(array_filter(
|
|
$snapshotCreateActions,
|
|
static fn (string $action): bool => preg_match(
|
|
'~^create-snapshot:' . $snapshotKeyPattern . ':crash$~D',
|
|
$action
|
|
) === 1
|
|
)) === 1,
|
|
'Snapshot create must cover both filesystem and crash consistency.');
|
|
|
|
$validTemplateMetadataProjection =
|
|
$fixtures['template_metadata_replica'][0]['projection'];
|
|
$invalidGeneration = $validTemplateMetadataProjection;
|
|
$invalidGeneration['generation_token'] = 'NOT-32-LOWER-HEX';
|
|
$expectInvalidProjection(
|
|
$invalidGeneration,
|
|
'template_metadata_replica',
|
|
'An invalid template generation token crossed the projection boundary.'
|
|
);
|
|
$expectInvalidProjection(
|
|
[
|
|
'id' => 'matrix-rescue',
|
|
'created_at' => '2026-10-21',
|
|
'generation_token' => str_repeat('a', 32),
|
|
],
|
|
'rescue_template_import',
|
|
'A generation token escaped its two template projection kinds.'
|
|
);
|
|
$nestedToken = $validTemplateMetadataProjection;
|
|
$nestedToken['generation_token'] = [
|
|
'generation_token' => str_repeat('a', 32),
|
|
];
|
|
$expectInvalidProjection(
|
|
$nestedToken,
|
|
'template_metadata_replica',
|
|
'A nested generation token bypassed sensitive-key filtering.'
|
|
);
|
|
$accessToken = $validTemplateMetadataProjection;
|
|
$accessToken['access_token'] = str_repeat('a', 32);
|
|
$expectInvalidProjection(
|
|
$accessToken,
|
|
'template_metadata_replica',
|
|
'An access token bypassed sensitive-key filtering.'
|
|
);
|
|
|
|
foreach ($dedicatedFixtures as $kind => $fixture) {
|
|
$identity = CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
$fixture['job'],
|
|
$fixture['payload']
|
|
);
|
|
$assert($identity === CloudOperationRecoveryIdentity::expected(
|
|
$fixture['job']
|
|
)
|
|
&& $identity['action'] === $fixture['expected_action']
|
|
&& CloudOperationRecoveryPolicy::allowsAction(
|
|
$kind,
|
|
$identity['action']
|
|
)
|
|
&& !CloudOperationRecoveryPolicy::supports($kind),
|
|
'Dedicated identity contract diverged for ' . $kind . '.');
|
|
if ($kind === 'create') {
|
|
$assert(CloudOperationRecoveryPolicy::identityScope($kind)
|
|
=== 'create'
|
|
&& preg_match(
|
|
'/^[a-f0-9]{8}-[a-f0-9]{4}-5[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/D',
|
|
$identity['resource_id']
|
|
) === 1,
|
|
'Create identity lost its deterministic UUID scope.');
|
|
$projection = $projectionMethod->invoke(null, [
|
|
'creation_absent' => true,
|
|
'managed_artifacts_absent' => true,
|
|
], 'create', 'not_applied');
|
|
$assert(($projection['creation_absent'] ?? null) === true
|
|
&& ($projection['managed_artifacts_absent'] ?? null)
|
|
=== true,
|
|
'Create absence projection is no longer complete.');
|
|
$expectStateConflict(static fn () =>
|
|
CloudWorkerService::recoveryContext($fixture['job']));
|
|
} elseif ($kind === 'backup_restore') {
|
|
$assert(CloudOperationRecoveryPolicy::identityScope($kind)
|
|
=== 'instance'
|
|
&& $identity['resource_id']
|
|
=== $fixture['expected_resource'],
|
|
'Backup restore identity lost its target VM scope.');
|
|
$context = CloudWorkerService::recoveryContext($fixture['job']);
|
|
$assert(($context['instance']->id ?? null)
|
|
=== $restoreInstanceId
|
|
&& $context['payload'] === $fixture['payload'],
|
|
'Backup restore lost its fenced recovery context.');
|
|
$projection = $projectionMethod->invoke(null, [
|
|
'creation_absent' => true,
|
|
'managed_artifacts_absent' => true,
|
|
], 'backup_restore', 'not_applied');
|
|
$assert(($projection['creation_absent'] ?? null) === true
|
|
&& ($projection['managed_artifacts_absent'] ?? null)
|
|
=== true,
|
|
'Backup restore absence projection is no longer complete.');
|
|
} else {
|
|
$assert(CloudOperationRecoveryPolicy::identityScope($kind)
|
|
=== 'storage'
|
|
&& $identity['resource_id']
|
|
=== $fixture['expected_resource'],
|
|
'Storage identity lost its dedicated scope.');
|
|
$assert($projectionMethod->invoke(
|
|
null,
|
|
['configured' => true],
|
|
'host_storage_configure',
|
|
'applied'
|
|
) === ['configured' => true],
|
|
'Storage projection branch is missing.');
|
|
$expectStateConflict(static fn () =>
|
|
CloudWorkerService::recoveryContext($fixture['job']));
|
|
}
|
|
}
|
|
|
|
foreach ($classificationOnlyKinds as $kind => $mode) {
|
|
$entry = $manifestOperations[$kind];
|
|
$assert($entry['recovery_mode'] === $mode
|
|
&& $entry['identity_scope'] === 'none'
|
|
&& $entry['actions'] === []
|
|
&& $entry['action_patterns'] === []
|
|
&& !CloudOperationRecoveryPolicy::supports($kind)
|
|
&& CloudOperationRecoveryPolicy::identityScope($kind) === null
|
|
&& !CloudOperationRecoveryIdentity::supports($kind),
|
|
'Aggregate/payment classification diverged for ' . $kind . '.');
|
|
}
|
|
|
|
foreach ($manifestOperations as $kind => $entry) {
|
|
foreach (['applied', 'not_applied', 'partial'] as $resolution) {
|
|
$assert(CloudOperationRecoveryPolicy::allowsSystemResolution(
|
|
$kind,
|
|
$resolution
|
|
) === in_array(
|
|
$resolution,
|
|
$entry['system_resolutions'],
|
|
true
|
|
),
|
|
'System resolution policy diverged for '
|
|
. $kind . ':' . $resolution);
|
|
}
|
|
$assert(!CloudOperationRecoveryPolicy::allowsAction(
|
|
$kind,
|
|
'unlisted-operation'
|
|
),
|
|
'Recovery contract accepted an unlisted action for ' . $kind);
|
|
}
|
|
|
|
echo "Cloud operation recovery matrix tests passed\n";
|
|
}
|