|
|
|
|
<?php
|
|
|
|
|
declare(strict_types=1);
|
|
|
|
|
|
|
|
|
|
namespace App\Services\Cloud {
|
|
|
|
|
/** Isolated encrypted envelope for the recovery-matrix contract test. */
|
|
|
|
|
final class CloudSecrets
|
|
|
|
|
{
|
|
|
|
|
private static array $values = [];
|
|
|
|
|
|
|
|
|
|
public static function encrypt(array $value, string $purpose): string
|
|
|
|
|
{
|
|
|
|
|
$key = bin2hex(random_bytes(16));
|
|
|
|
|
self::$values[$key] = $value;
|
|
|
|
|
return 'operation-recovery-matrix-test:' . $key;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static function decrypt(string $value, string $purpose): array
|
|
|
|
|
{
|
|
|
|
|
$prefix = 'operation-recovery-matrix-test:';
|
|
|
|
|
if (!str_starts_with($value, $prefix)) {
|
|
|
|
|
throw new \RuntimeException('Invalid matrix test payload.');
|
|
|
|
|
}
|
|
|
|
|
$key = substr($value, strlen($prefix));
|
|
|
|
|
if (!array_key_exists($key, self::$values)) {
|
|
|
|
|
throw new \RuntimeException('Missing matrix test payload.');
|
|
|
|
|
}
|
|
|
|
|
return self::$values[$key];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudAgentCompletionEvent;
|
|
|
|
|
use App\Models\CloudInstance;
|
|
|
|
|
use App\Models\CloudInstanceMigration;
|
|
|
|
|
use App\Models\CloudJob;
|
|
|
|
|
use App\Services\Cloud\CloudAdminOperationRecoveryService;
|
|
|
|
|
use App\Services\Cloud\CloudAgentError;
|
|
|
|
|
use App\Services\Cloud\CloudError;
|
|
|
|
|
use App\Services\Cloud\CloudFrozenPublicNetworkIdentity;
|
|
|
|
|
use App\Services\Cloud\CloudInstanceMigrationService;
|
|
|
|
|
use App\Services\Cloud\CloudOperationCompletionService;
|
|
|
|
|
use App\Services\Cloud\CloudOperationRecoveryContract;
|
|
|
|
|
use App\Services\Cloud\CloudOperationRecoveryIdentity;
|
|
|
|
|
use App\Services\Cloud\CloudOperationRecoveryPolicy;
|
|
|
|
|
use App\Services\Cloud\CloudPortableRestoreCompatibility;
|
|
|
|
|
use App\Services\Cloud\CloudPublicNetworkAllocationService;
|
|
|
|
|
use App\Services\Cloud\CloudRecoveryInstanceStateConflict;
|
|
|
|
|
use App\Services\Cloud\CloudSecrets;
|
|
|
|
|
use App\Services\Cloud\CloudVpcProjection;
|
|
|
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
|
|
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
|
|
|
if (!$condition) {
|
|
|
|
|
throw new RuntimeException($message);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
$expectStateConflict = static function (callable $callback) use (
|
|
|
|
|
$assert
|
|
|
|
|
): void {
|
|
|
|
|
try {
|
|
|
|
|
$callback();
|
|
|
|
|
} catch (CloudRecoveryInstanceStateConflict $error) {
|
|
|
|
|
$assert($error->httpStatus === 409,
|
|
|
|
|
'Unexpected recovery state-conflict status.');
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
'A dedicated recovery kind entered generic worker recovery.'
|
|
|
|
|
);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$manifestPath = dirname(__DIR__, 2)
|
|
|
|
|
. '/agent/operation/recovery_contract.json';
|
|
|
|
|
$manifest = json_decode(
|
|
|
|
|
file_get_contents($manifestPath),
|
|
|
|
|
true,
|
|
|
|
|
64,
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
);
|
|
|
|
|
$assert(($manifest['schema_version'] ?? null) === 1,
|
|
|
|
|
'Unsupported recovery contract schema version.');
|
|
|
|
|
$manifestOperations = [];
|
|
|
|
|
foreach ($manifest['operations'] ?? [] as $entry) {
|
|
|
|
|
$kind = $entry['kind'] ?? null;
|
|
|
|
|
$assert(is_string($kind) && $kind !== ''
|
|
|
|
|
&& !array_key_exists($kind, $manifestOperations),
|
|
|
|
|
'Recovery contract contains a missing or duplicate kind.');
|
|
|
|
|
unset($entry['kind']);
|
|
|
|
|
$manifestOperations[$kind] = $entry;
|
|
|
|
|
}
|
|
|
|
|
$assert(CloudOperationRecoveryContract::OPERATIONS
|
|
|
|
|
=== $manifestOperations,
|
|
|
|
|
'Generated PHP recovery contract is stale.');
|
|
|
|
|
|
|
|
|
|
$capsule = new Capsule();
|
|
|
|
|
$capsule->addConnection([
|
|
|
|
|
'driver' => 'sqlite',
|
|
|
|
|
'database' => ':memory:',
|
|
|
|
|
], 'default');
|
|
|
|
|
$capsule->setAsGlobal();
|
|
|
|
|
$capsule->bootEloquent();
|
|
|
|
|
$schema = $capsule->schema('default');
|
|
|
|
|
|
|
|
|
|
$schema->create('cloud_hosts', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('operation_id')->unique();
|
|
|
|
|
$table->string('request_idempotency_key')->unique();
|
|
|
|
|
$table->string('request_fingerprint');
|
|
|
|
|
$table->integer('parent_job_id')->nullable();
|
|
|
|
|
$table->integer('order_id')->nullable();
|
|
|
|
|
$table->integer('invoice_id')->nullable();
|
|
|
|
|
$table->integer('instance_id')->nullable();
|
|
|
|
|
$table->integer('snapshot_id')->nullable();
|
|
|
|
|
$table->integer('backup_id')->nullable();
|
|
|
|
|
$table->integer('backup_restore_id')->nullable();
|
|
|
|
|
$table->integer('backup_recovery_id')->nullable();
|
|
|
|
|
$table->integer('vpc_id')->nullable();
|
|
|
|
|
$table->integer('vpc_port_id')->nullable();
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->integer('host_id')->nullable();
|
|
|
|
|
$table->string('kind');
|
|
|
|
|
$table->string('execution_mode');
|
|
|
|
|
$table->string('phase');
|
|
|
|
|
$table->string('outcome')->nullable();
|
|
|
|
|
$table->string('status');
|
|
|
|
|
$table->text('sealed_payload');
|
|
|
|
|
$table->integer('attempts')->default(1);
|
|
|
|
|
$table->integer('dispatch_attempts')->default(1);
|
|
|
|
|
$table->integer('dispatch_deadline_at')->nullable();
|
|
|
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
|
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
|
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
|
|
|
$table->integer('dispatched_at')->nullable();
|
|
|
|
|
$table->integer('terminal_at')->nullable();
|
|
|
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
|
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
|
|
|
$table->string('effect_resolution')->default('not_applicable');
|
|
|
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
|
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
|
|
|
$table->string('effect_resolution_reason')->default('');
|
|
|
|
|
$table->string('completion_event_id')->nullable();
|
|
|
|
|
$table->integer('read_at')->nullable();
|
|
|
|
|
$table->string('lease_token')->nullable();
|
|
|
|
|
$table->integer('lease_until')->default(0);
|
|
|
|
|
$table->integer('available_at')->default(0);
|
|
|
|
|
$table->string('error_code')->nullable();
|
|
|
|
|
$table->boolean('finalization_incident')->default(0);
|
|
|
|
|
$table->string('finalization_error_code')->nullable();
|
|
|
|
|
$table->text('result_json')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_agent_completion_events',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->string('event_id')->primary();
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('operation_id');
|
|
|
|
|
$table->string('payload_sha256');
|
|
|
|
|
$table->string('outcome');
|
|
|
|
|
$table->boolean('effect_may_have_applied');
|
|
|
|
|
$table->boolean('safe_to_resubmit');
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->text('result_json')->nullable();
|
|
|
|
|
$table->string('disposition')->default('received');
|
|
|
|
|
$table->integer('received_at');
|
|
|
|
|
$table->integer('applied_at')->nullable();
|
|
|
|
|
$table->unique(['host_id', 'operation_id']);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('user', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('uuid');
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->integer('resource_pool_id')->default(7001);
|
|
|
|
|
$table->integer('reservation_id')->nullable();
|
|
|
|
|
$table->string('name');
|
|
|
|
|
$table->string('status');
|
|
|
|
|
$table->string('service_status')->default('exempt');
|
|
|
|
|
$table->string('network_status')->default('active');
|
|
|
|
|
$table->text('spec_json')->default('{}');
|
|
|
|
|
$table->text('sealed_credential')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_instance_snapshots',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('snapshot_key')->unique();
|
|
|
|
|
$table->integer('instance_id');
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('name');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->string('consistency');
|
|
|
|
|
$table->integer('size_bytes')->nullable();
|
|
|
|
|
$table->string('last_operation_id')->nullable();
|
|
|
|
|
$table->integer('ready_at')->nullable();
|
|
|
|
|
$table->integer('last_restored_at')->nullable();
|
|
|
|
|
$table->integer('deleted_at')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_vpcs', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('vpc_key')->unique();
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->integer('region_id');
|
|
|
|
|
$table->integer('cidr_pool_id');
|
|
|
|
|
$table->string('name');
|
|
|
|
|
$table->string('cidr');
|
|
|
|
|
$table->integer('prefix_length');
|
|
|
|
|
$table->string('gateway');
|
|
|
|
|
$table->integer('active_claim')->nullable();
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('desired_revision');
|
|
|
|
|
$table->integer('deleted_at')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_vpc_ports',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('port_key')->unique();
|
|
|
|
|
$table->integer('vpc_id');
|
|
|
|
|
$table->integer('instance_id');
|
|
|
|
|
$table->integer('migration_host_id')->nullable();
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->string('private_ip');
|
|
|
|
|
$table->string('mac_address');
|
|
|
|
|
$table->integer('active_claim')->nullable();
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('desired_revision');
|
|
|
|
|
$table->integer('applied_revision');
|
|
|
|
|
$table->string('last_operation_id')->nullable();
|
|
|
|
|
$table->string('last_error_code')->default('');
|
|
|
|
|
$table->integer('attached_at')->nullable();
|
|
|
|
|
$table->integer('released_at')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_vpc_host_states',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->integer('vpc_id');
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->integer('desired_revision');
|
|
|
|
|
$table->integer('applied_revision');
|
|
|
|
|
$table->string('desired_digest');
|
|
|
|
|
$table->string('applied_digest')->nullable();
|
|
|
|
|
$table->string('sync_state');
|
|
|
|
|
$table->string('last_operation_id')->nullable();
|
|
|
|
|
$table->string('last_error_code')->default('');
|
|
|
|
|
$table->integer('last_success_at')->default(0);
|
|
|
|
|
$table->integer('last_failure_at')->default(0);
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_instance_backups',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('backup_key')->unique();
|
|
|
|
|
$table->integer('source_instance_id');
|
|
|
|
|
$table->integer('source_host_id');
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->string('source_uuid');
|
|
|
|
|
$table->string('name');
|
|
|
|
|
$table->string('purpose')->default('portable');
|
|
|
|
|
$table->string('source')->default('manual');
|
|
|
|
|
$table->integer('subscription_id')->nullable();
|
|
|
|
|
$table->integer('scheduled_for')->nullable();
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->string('consistency');
|
|
|
|
|
$table->string('object_key');
|
|
|
|
|
$table->integer('object_size_bytes')->default(0);
|
|
|
|
|
$table->integer('virtual_size_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->text('spec_json');
|
|
|
|
|
$table->text('sealed_storage_runtime');
|
|
|
|
|
$table->text('sealed_credential')->nullable();
|
|
|
|
|
$table->integer('storage_revision')->default(1);
|
|
|
|
|
$table->integer('active_job_id')->nullable();
|
|
|
|
|
$table->string('last_operation_id')->nullable();
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('ready_at')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_backup_recoveries',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('recovery_key')->unique();
|
|
|
|
|
$table->integer('backup_id')->nullable();
|
|
|
|
|
$table->integer('instance_id');
|
|
|
|
|
$table->integer('job_id')->nullable();
|
|
|
|
|
$table->string('operation_id')->unique();
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('completed_at')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_backup_restores',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('restore_key')->unique();
|
|
|
|
|
$table->integer('backup_id')->nullable();
|
|
|
|
|
$table->integer('target_host_id');
|
|
|
|
|
$table->integer('target_instance_id')->nullable();
|
|
|
|
|
$table->integer('reservation_id')->nullable();
|
|
|
|
|
$table->integer('job_id')->nullable();
|
|
|
|
|
$table->string('operation_id')->unique();
|
|
|
|
|
$table->string('new_uuid');
|
|
|
|
|
$table->string('new_name');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('ready_at')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_backup_restore_attempts',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->integer('job_id')->nullable()->unique();
|
|
|
|
|
$table->string('operation_id')->unique();
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_instance_migrations',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('migration_key')->unique();
|
|
|
|
|
$table->integer('instance_id');
|
|
|
|
|
$table->integer('vpc_port_id')->nullable();
|
|
|
|
|
$table->integer('restore_id');
|
|
|
|
|
$table->integer('source_host_id');
|
|
|
|
|
$table->string('source_uuid');
|
|
|
|
|
$table->integer('source_reservation_id');
|
|
|
|
|
$table->integer('target_host_id');
|
|
|
|
|
$table->string('target_uuid');
|
|
|
|
|
$table->integer('target_reservation_id')->nullable();
|
|
|
|
|
$table->integer('source_prepare_job_id')->nullable();
|
|
|
|
|
$table->integer('target_receive_job_id')->nullable();
|
|
|
|
|
$table->integer('firewall_job_id')->nullable();
|
|
|
|
|
$table->integer('activate_job_id')->nullable();
|
|
|
|
|
$table->integer('rollback_job_id')->nullable();
|
|
|
|
|
$table->integer('cleanup_job_id')->nullable();
|
|
|
|
|
$table->string('cleanup_mode');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->string('recovery_action')->default('');
|
|
|
|
|
$table->integer('retry_generation')->default(0);
|
|
|
|
|
$table->string('source_endpoint');
|
|
|
|
|
$table->string('peer_trust_id');
|
|
|
|
|
$table->string('manifest_digest')->default('');
|
|
|
|
|
$table->boolean('source_was_running')->nullable();
|
|
|
|
|
$table->boolean('source_autostart_enabled')->nullable();
|
|
|
|
|
$table->string('source_service_status')->default('active');
|
|
|
|
|
$table->string('source_network_status')->default('active');
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('cutover_at')->nullable();
|
|
|
|
|
$table->integer('completed_at')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_peer_migration_artifacts',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->integer('migration_id');
|
|
|
|
|
$table->string('artifact_id');
|
|
|
|
|
$table->string('role');
|
|
|
|
|
$table->integer('source_snapshot_id')->nullable();
|
|
|
|
|
$table->string('snapshot_key')->nullable();
|
|
|
|
|
$table->string('name')->default('');
|
|
|
|
|
$table->string('consistency');
|
|
|
|
|
$table->integer('object_size_bytes')->default(0);
|
|
|
|
|
$table->integer('virtual_size_bytes');
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('snapshot_created_at')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_resource_reservations',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('reservation_key')->nullable();
|
|
|
|
|
$table->integer('order_id')->nullable();
|
|
|
|
|
$table->integer('backup_restore_id')->nullable();
|
|
|
|
|
$table->string('backup_restore_attempt_key')->nullable();
|
|
|
|
|
$table->integer('migration_source_debt_id')->nullable();
|
|
|
|
|
$table->integer('offer_id')->nullable();
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->integer('resource_pool_id')->nullable();
|
|
|
|
|
$table->integer('network_attachment_id')->nullable();
|
|
|
|
|
$table->integer('public_ip_pool_id')->nullable();
|
|
|
|
|
$table->string('operation_id');
|
|
|
|
|
$table->integer('public_ip_id')->nullable();
|
|
|
|
|
$table->integer('instance_id')->nullable();
|
|
|
|
|
$table->integer('cpu_cores');
|
|
|
|
|
$table->integer('memory_mib');
|
|
|
|
|
$table->integer('disk_gib');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('expires_at')->nullable();
|
|
|
|
|
$table->integer('released_at')->nullable();
|
|
|
|
|
$table->string('release_reason')->default('');
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_orders',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->integer('offer_id');
|
|
|
|
|
$table->integer('resource_pool_id');
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->integer('reservation_id');
|
|
|
|
|
$table->text('spec_json');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_public_ips',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->integer('public_ip_pool_id')->nullable();
|
|
|
|
|
$table->integer('address_family')->nullable();
|
|
|
|
|
$table->string('address');
|
|
|
|
|
$table->binary('address_binary')->nullable();
|
|
|
|
|
$table->integer('prefix_length');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('reservation_id')->nullable();
|
|
|
|
|
$table->integer('reservation_ip_claim_id')->nullable();
|
|
|
|
|
$table->integer('instance_id')->nullable();
|
|
|
|
|
$table->string('operation_id')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_resource_reservation_ips', static function (
|
|
|
|
|
Blueprint $table
|
|
|
|
|
): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->integer('reservation_id');
|
|
|
|
|
$table->integer('address_family');
|
|
|
|
|
$table->integer('claim_generation')->default(1);
|
|
|
|
|
$table->integer('active_claim')->nullable()->default(1);
|
|
|
|
|
$table->integer('network_attachment_id');
|
|
|
|
|
$table->integer('public_ip_pool_id');
|
|
|
|
|
$table->integer('public_ip_id');
|
|
|
|
|
$table->string('operation_id');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('released_at')->nullable();
|
|
|
|
|
$table->string('release_reason')->default('');
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_resource_pools',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->integer('line_id');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_host_network_attachments',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->integer('line_id');
|
|
|
|
|
$table->string('network_selector');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_public_ip_pools',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->integer('network_attachment_id');
|
|
|
|
|
$table->string('pool_key');
|
|
|
|
|
$table->integer('address_family')->nullable();
|
|
|
|
|
$table->string('cidr');
|
|
|
|
|
$table->string('gateway');
|
|
|
|
|
$table->boolean('gateway_on_link')->default(0);
|
|
|
|
|
$table->integer('guest_prefix_length')->nullable();
|
|
|
|
|
$table->text('dns_servers_json');
|
|
|
|
|
$table->text('reserved_ips_json')->default('[]');
|
|
|
|
|
$table->string('allocation_start')->nullable();
|
|
|
|
|
$table->string('allocation_end')->nullable();
|
|
|
|
|
$table->string('allocation_cursor')->nullable();
|
|
|
|
|
$table->integer('max_allocations')->default(0);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_instance_migration_source_debts', static function (
|
|
|
|
|
Blueprint $table
|
|
|
|
|
): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('debt_key');
|
|
|
|
|
$table->integer('migration_id');
|
|
|
|
|
$table->integer('source_reservation_id');
|
|
|
|
|
$table->integer('source_host_id');
|
|
|
|
|
$table->string('source_uuid');
|
|
|
|
|
$table->integer('source_network_attachment_id');
|
|
|
|
|
$table->text('source_identity_json');
|
|
|
|
|
$table->string('source_identity_sha256');
|
|
|
|
|
$table->string('source_claim_set_sha256');
|
|
|
|
|
$table->text('source_public_network_json');
|
|
|
|
|
$table->string('source_public_network_sha256');
|
|
|
|
|
$table->string('cleanup_operation_id')->nullable();
|
|
|
|
|
$table->integer('cleanup_job_id')->nullable();
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('active_debt')->nullable();
|
|
|
|
|
$table->string('last_error_code')->default('');
|
|
|
|
|
$table->integer('resolved_at')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_instance_firewall_versions',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->integer('instance_id');
|
|
|
|
|
$table->integer('revision');
|
|
|
|
|
$table->boolean('enabled')->default(1);
|
|
|
|
|
$table->string('policy_digest');
|
|
|
|
|
$table->string('request_hash');
|
|
|
|
|
$table->string('idempotency_key');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('job_id')->nullable();
|
|
|
|
|
$table->string('operation_id')->nullable();
|
|
|
|
|
$table->string('error_code')->nullable();
|
|
|
|
|
$table->integer('applied_at')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_templates',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->string('id')->primary();
|
|
|
|
|
$table->string('catalog_key')->default('test-catalog');
|
|
|
|
|
$table->string('version_label')->default('test-version');
|
|
|
|
|
$table->boolean('is_default')->default(true);
|
|
|
|
|
$table->string('name');
|
|
|
|
|
$table->integer('revision');
|
|
|
|
|
$table->string('generation_token');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('active_job_id')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_template_replicas',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('template_id');
|
|
|
|
|
$table->integer('revision');
|
|
|
|
|
$table->string('generation_token');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('image_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->string('operation_id');
|
|
|
|
|
$table->integer('job_id')->nullable();
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
$table->primary(['host_id', 'template_id']);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_rescue_templates',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->string('id')->primary();
|
|
|
|
|
$table->string('name');
|
|
|
|
|
$table->integer('size_bytes');
|
|
|
|
|
$table->string('sha256');
|
|
|
|
|
$table->integer('revision');
|
|
|
|
|
$table->string('generation_token');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('active_job_id')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_rescue_template_replicas',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('template_id');
|
|
|
|
|
$table->integer('revision');
|
|
|
|
|
$table->string('generation_token');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('size_bytes');
|
|
|
|
|
$table->string('sha256');
|
|
|
|
|
$table->string('operation_id');
|
|
|
|
|
$table->integer('job_id')->nullable();
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
$table->primary(['host_id', 'template_id']);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_user_isos',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->string('id')->primary();
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->string('name');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('active_job_id')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_user_iso_replicas',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('user_iso_id');
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('size_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->string('operation_id');
|
|
|
|
|
$table->integer('job_id')->nullable();
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
$table->primary(['host_id', 'user_iso_id']);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_host_storage_configurations',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id')->primary();
|
|
|
|
|
$table->integer('revision');
|
|
|
|
|
foreach (['managed', 'template', 'seed', 'user_iso', 'rescue']
|
|
|
|
|
as $role) {
|
|
|
|
|
$table->text($role . '_directory');
|
|
|
|
|
}
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->string('operation_id')->nullable();
|
|
|
|
|
$table->integer('job_id')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
$now = time();
|
|
|
|
|
$hostId = 7;
|
|
|
|
|
$ownerId = 41;
|
|
|
|
|
$instanceId = 1;
|
|
|
|
|
$instanceUuid = '123e4567-e89b-42d3-a456-426614174000';
|
|
|
|
|
Capsule::table('user')->insert(['id' => $ownerId]);
|
|
|
|
|
Capsule::table('cloud_hosts')->insert(['id' => $hostId]);
|
|
|
|
|
Capsule::table('cloud_instances')->insert([
|
|
|
|
|
'id' => $instanceId,
|
|
|
|
|
'uuid' => $instanceUuid,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'name' => 'matrix-vm',
|
|
|
|
|
'status' => 'active',
|
|
|
|
|
'service_status' => 'exempt',
|
|
|
|
|
'network_status' => 'active',
|
|
|
|
|
'spec_json' => json_encode([
|
|
|
|
|
'managed' => true,
|
|
|
|
|
'disk_gib' => 20,
|
|
|
|
|
'bandwidth_mbps' => 100,
|
|
|
|
|
'debian_12_network_fix' => false,
|
|
|
|
|
], JSON_THROW_ON_ERROR),
|
|
|
|
|
'created_at' => $now - 3600,
|
|
|
|
|
'updated_at' => $now - 3600,
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
$nextJobId = 1;
|
|
|
|
|
/**
|
|
|
|
|
* @return array{id:int,operation_id:string,job:CloudJob,payload:array}
|
|
|
|
|
*/
|
|
|
|
|
$insertJob = static function (
|
|
|
|
|
string $kind,
|
|
|
|
|
callable $payloadFactory,
|
|
|
|
|
?int $boundInstanceId = null,
|
|
|
|
|
?int $snapshotId = null,
|
|
|
|
|
?int $parentJobId = null,
|
|
|
|
|
?int $orderId = null,
|
|
|
|
|
string $executionMode = 'agent',
|
|
|
|
|
int $userId = 41,
|
|
|
|
|
?int $backupId = null,
|
|
|
|
|
?int $backupRestoreId = null,
|
|
|
|
|
?int $backupRecoveryId = null,
|
|
|
|
|
?int $vpcId = null,
|
|
|
|
|
?int $vpcPortId = null
|
|
|
|
|
) use (&$nextJobId, $hostId, $now): array {
|
|
|
|
|
$id = $nextJobId++;
|
|
|
|
|
$operationId = hash('md5', 'recovery-matrix-job:' . $id);
|
|
|
|
|
$payload = $payloadFactory($id, $operationId);
|
|
|
|
|
Capsule::table('cloud_jobs')->insert([
|
|
|
|
|
'id' => $id,
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'request_idempotency_key' => 'recovery-matrix-' . $operationId,
|
|
|
|
|
'request_fingerprint' => hash('sha256', $operationId),
|
|
|
|
|
'parent_job_id' => $parentJobId,
|
|
|
|
|
'order_id' => $orderId,
|
|
|
|
|
'invoice_id' => null,
|
|
|
|
|
'instance_id' => $boundInstanceId,
|
|
|
|
|
'snapshot_id' => $snapshotId,
|
|
|
|
|
'backup_id' => $backupId,
|
|
|
|
|
'backup_restore_id' => $backupRestoreId,
|
|
|
|
|
'backup_recovery_id' => $backupRecoveryId,
|
|
|
|
|
'vpc_id' => $vpcId,
|
|
|
|
|
'vpc_port_id' => $vpcPortId,
|
|
|
|
|
'user_id' => $userId,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'kind' => $kind,
|
|
|
|
|
'execution_mode' => $executionMode,
|
|
|
|
|
'phase' => 'terminal',
|
|
|
|
|
'outcome' => 'failed',
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt(
|
|
|
|
|
$payload,
|
|
|
|
|
'job-payload'
|
|
|
|
|
),
|
|
|
|
|
'attempts' => 1,
|
|
|
|
|
'dispatch_attempts' => 1,
|
|
|
|
|
'dispatch_deadline_at' => $now - 120,
|
|
|
|
|
'dispatch_started_at' => $now - 110,
|
|
|
|
|
'agent_accepted_at' => $now - 100,
|
|
|
|
|
'receipt_deadline_at' => $now - 30,
|
|
|
|
|
'dispatched_at' => $now - 110,
|
|
|
|
|
'terminal_at' => $now - 20,
|
|
|
|
|
'effect_may_have_applied' => 1,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'unknown',
|
|
|
|
|
'effect_resolution_reason' => '',
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'created_at' => $now - 180,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
return [
|
|
|
|
|
'id' => $id,
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'job' => CloudJob::findOrFail($id),
|
|
|
|
|
'payload' => $payload,
|
|
|
|
|
];
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$fixtures = [];
|
|
|
|
|
$addFixture = static function (
|
|
|
|
|
string $kind,
|
|
|
|
|
string $variant,
|
|
|
|
|
array $record,
|
|
|
|
|
string $expectedAction,
|
|
|
|
|
string $expectedResource,
|
|
|
|
|
array $projection
|
|
|
|
|
) use (&$fixtures): void {
|
|
|
|
|
$fixtures[$kind][] = $record + [
|
|
|
|
|
'variant' => $variant,
|
|
|
|
|
'expected_action' => $expectedAction,
|
|
|
|
|
'expected_resource' => $expectedResource,
|
|
|
|
|
'projection' => $projection,
|
|
|
|
|
];
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$instanceFixture = static function (
|
|
|
|
|
string $kind,
|
|
|
|
|
string $variant,
|
|
|
|
|
string $method,
|
|
|
|
|
string $suffix,
|
|
|
|
|
array $body,
|
|
|
|
|
string $expectedAction,
|
|
|
|
|
array $projection,
|
|
|
|
|
array $metadata = []
|
|
|
|
|
) use (
|
|
|
|
|
$insertJob,
|
|
|
|
|
$addFixture,
|
|
|
|
|
$instanceId,
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
$ownerId
|
|
|
|
|
): array {
|
|
|
|
|
$record = $insertJob(
|
|
|
|
|
$kind,
|
|
|
|
|
static function (int $id, string $operationId) use (
|
|
|
|
|
$method,
|
|
|
|
|
$suffix,
|
|
|
|
|
$body,
|
|
|
|
|
$metadata,
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
$ownerId
|
|
|
|
|
): array {
|
|
|
|
|
$resolvedBody = $body;
|
|
|
|
|
foreach ($resolvedBody as $field => $value) {
|
|
|
|
|
if ($value === '__OPERATION_ID__') {
|
|
|
|
|
$resolvedBody[$field] = $operationId;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return $metadata + [
|
|
|
|
|
'method' => $method,
|
|
|
|
|
'request_target' => '/v1/vms/'
|
|
|
|
|
. rawurlencode($instanceUuid) . $suffix,
|
|
|
|
|
'body' => $resolvedBody,
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => false,
|
|
|
|
|
];
|
|
|
|
|
},
|
|
|
|
|
$instanceId
|
|
|
|
|
);
|
|
|
|
|
$addFixture(
|
|
|
|
|
$kind,
|
|
|
|
|
$variant,
|
|
|
|
|
$record,
|
|
|
|
|
$expectedAction,
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
$projection
|
|
|
|
|
);
|
|
|
|
|
return $record;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$powerProjection = ['uuid' => $instanceUuid, 'state' => 'running'];
|
|
|
|
|
foreach ([
|
|
|
|
|
'start' => 'power-start',
|
|
|
|
|
'shutdown' => 'power-shutdown',
|
|
|
|
|
'reboot' => 'power-reboot',
|
|
|
|
|
'reset' => 'power-reset',
|
|
|
|
|
'poweroff' => 'power-destroy',
|
|
|
|
|
] as $panelAction => $wireAction) {
|
|
|
|
|
$instanceFixture(
|
|
|
|
|
'action',
|
|
|
|
|
$panelAction,
|
|
|
|
|
'POST',
|
|
|
|
|
'/actions',
|
|
|
|
|
['action' => $panelAction],
|
|
|
|
|
$wireAction,
|
|
|
|
|
$powerProjection
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$instanceFixture(
|
|
|
|
|
'reinstall',
|
|
|
|
|
'default',
|
|
|
|
|
'POST',
|
|
|
|
|
'/reinstall',
|
|
|
|
|
[
|
|
|
|
|
'template_id' => 'debian-13',
|
|
|
|
|
'template_revision' => 4,
|
|
|
|
|
'template_generation_token' => str_repeat('a', 32),
|
|
|
|
|
'preserve_data' => false,
|
|
|
|
|
'root_password' => 'MatrixPassword!123',
|
|
|
|
|
],
|
|
|
|
|
'reinstall-vm',
|
|
|
|
|
['uuid' => $instanceUuid, 'reinstall_committed' => true]
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$instanceFixture(
|
|
|
|
|
'rescue',
|
|
|
|
|
'enter',
|
|
|
|
|
'POST',
|
|
|
|
|
'/rescue/enter',
|
|
|
|
|
[
|
|
|
|
|
'template_id' => str_repeat('e', 32),
|
|
|
|
|
'revision' => 3,
|
|
|
|
|
'generation_token' => str_repeat('f', 32),
|
|
|
|
|
],
|
|
|
|
|
'enter-rescue',
|
|
|
|
|
[
|
|
|
|
|
'active' => true,
|
|
|
|
|
'staged' => true,
|
|
|
|
|
'template_id' => str_repeat('e', 32),
|
|
|
|
|
'revision' => 3,
|
|
|
|
|
'generation_token' => str_repeat('f', 32),
|
|
|
|
|
'network_suspended' => false,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$instanceFixture(
|
|
|
|
|
'rescue',
|
|
|
|
|
'force-start',
|
|
|
|
|
'POST',
|
|
|
|
|
'/rescue/force-start',
|
|
|
|
|
['ack_data_loss' => true],
|
|
|
|
|
'force-start-rescue',
|
|
|
|
|
[
|
|
|
|
|
'active' => true,
|
|
|
|
|
'staged' => true,
|
|
|
|
|
'template_id' => str_repeat('e', 32),
|
|
|
|
|
'revision' => 3,
|
|
|
|
|
'generation_token' => str_repeat('f', 32),
|
|
|
|
|
'network_suspended' => false,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
foreach ([
|
|
|
|
|
['network_suspended' => false, 'state' => 'connected'],
|
|
|
|
|
['network_suspended' => true, 'state' => 'suspended'],
|
|
|
|
|
] as $rescueExit) {
|
|
|
|
|
$networkSuspended = $rescueExit['network_suspended'];
|
|
|
|
|
$state = $rescueExit['state'];
|
|
|
|
|
$instanceFixture(
|
|
|
|
|
'rescue',
|
|
|
|
|
'exit-' . $state,
|
|
|
|
|
'POST',
|
|
|
|
|
'/rescue/exit',
|
|
|
|
|
[
|
|
|
|
|
'network_suspended' => $networkSuspended,
|
|
|
|
|
'ack_data_loss' => true,
|
|
|
|
|
],
|
|
|
|
|
'exit-rescue:' . $state,
|
|
|
|
|
[
|
|
|
|
|
'active' => false,
|
|
|
|
|
'staged' => false,
|
|
|
|
|
'network_suspended' => $networkSuspended,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$deleteProjection = [
|
|
|
|
|
'deleted' => true,
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
'forced' => false,
|
|
|
|
|
];
|
|
|
|
|
$instanceFixture(
|
|
|
|
|
'delete',
|
|
|
|
|
'user-delete',
|
|
|
|
|
'DELETE',
|
|
|
|
|
'',
|
|
|
|
|
['remove_managed_disks' => true],
|
|
|
|
|
'delete-vm',
|
|
|
|
|
$deleteProjection
|
|
|
|
|
);
|
|
|
|
|
$instanceFixture(
|
|
|
|
|
'expiry_delete',
|
|
|
|
|
'expiry-delete',
|
|
|
|
|
'DELETE',
|
|
|
|
|
'',
|
|
|
|
|
['remove_managed_disks' => true],
|
|
|
|
|
'delete-vm',
|
|
|
|
|
$deleteProjection
|
|
|
|
|
);
|
|
|
|
|
$priorOperationId = str_repeat('b', 32);
|
|
|
|
|
$instanceFixture(
|
|
|
|
|
'delete_force',
|
|
|
|
|
'force-delete',
|
|
|
|
|
'POST',
|
|
|
|
|
'/force-delete',
|
|
|
|
|
[
|
|
|
|
|
'operation_id' => '__OPERATION_ID__',
|
|
|
|
|
'prior_operation_id' => $priorOperationId,
|
|
|
|
|
'ack_data_loss' => true,
|
|
|
|
|
'remove_managed_disks' => true,
|
|
|
|
|
],
|
|
|
|
|
'force-delete-vm',
|
|
|
|
|
array_replace($deleteProjection, ['forced' => true]),
|
|
|
|
|
['prior_operation_id' => $priorOperationId]
|
|
|
|
|
);
|
|
|
|
|
$migrationKey = str_repeat('e', 32);
|
|
|
|
|
$migrationTicket = str_repeat('a', 64);
|
|
|
|
|
$migrationDigest = str_repeat('d', 64);
|
|
|
|
|
$migrationArtifactId = str_repeat('c', 32);
|
|
|
|
|
$migrationArtifactDigest = str_repeat('f', 64);
|
|
|
|
|
$migrationTargetUuid = '123e4567-e89b-42d3-a456-426614174099';
|
|
|
|
|
$migrationResourcePoolId = 7001;
|
|
|
|
|
$migrationLineId = 7101;
|
|
|
|
|
$migrationAttachmentId = 7201;
|
|
|
|
|
$migrationPublicPoolId = 7301;
|
|
|
|
|
$migrationIpv6PublicPoolId = 7306;
|
|
|
|
|
$migrationNetworkSelector = 'bridge:migration-wan';
|
|
|
|
|
$migrationSourceOperationId = str_repeat('7', 32);
|
|
|
|
|
Capsule::table('cloud_resource_pools')->insert([
|
|
|
|
|
'id' => $migrationResourcePoolId,
|
|
|
|
|
'line_id' => $migrationLineId,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_host_network_attachments')->insert([
|
|
|
|
|
'id' => $migrationAttachmentId,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'line_id' => $migrationLineId,
|
|
|
|
|
'network_selector' => $migrationNetworkSelector,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_public_ip_pools')->insert([
|
|
|
|
|
[
|
|
|
|
|
'id' => $migrationPublicPoolId,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'network_attachment_id' => $migrationAttachmentId,
|
|
|
|
|
'pool_key' => str_repeat('b', 32),
|
|
|
|
|
'address_family' => 4,
|
|
|
|
|
'cidr' => '8.8.30.0/24',
|
|
|
|
|
'gateway' => '8.8.30.1',
|
|
|
|
|
'gateway_on_link' => 0,
|
|
|
|
|
'guest_prefix_length' => 24,
|
|
|
|
|
'dns_servers_json' => '["1.1.1.1"]',
|
|
|
|
|
'reserved_ips_json' => '[]',
|
|
|
|
|
'allocation_start' => null,
|
|
|
|
|
'allocation_end' => null,
|
|
|
|
|
'allocation_cursor' => null,
|
|
|
|
|
'max_allocations' => 0,
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
'id' => $migrationIpv6PublicPoolId,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'network_attachment_id' => $migrationAttachmentId,
|
|
|
|
|
'pool_key' => str_repeat('c', 32),
|
|
|
|
|
'address_family' => 6,
|
|
|
|
|
'cidr' => '2404:c140:156:30::/64',
|
|
|
|
|
'gateway' => '2404:c140:156:30::1',
|
|
|
|
|
'gateway_on_link' => 0,
|
|
|
|
|
'guest_prefix_length' => 64,
|
|
|
|
|
'dns_servers_json' => '["2606:4700:4700::1111"]',
|
|
|
|
|
'reserved_ips_json' => '[]',
|
|
|
|
|
'allocation_start' => '2404:c140:156:30::100',
|
|
|
|
|
'allocation_end' => '2404:c140:156:30::1ff',
|
|
|
|
|
'allocation_cursor' => null,
|
|
|
|
|
'max_allocations' => 100,
|
|
|
|
|
],
|
|
|
|
|
]);
|
|
|
|
|
$migrationArtifact = [
|
|
|
|
|
'id' => $migrationArtifactId,
|
|
|
|
|
'role' => 'root',
|
|
|
|
|
'consistency' => 'crash',
|
|
|
|
|
'size_bytes' => 1048576,
|
|
|
|
|
'virtual_size_bytes' => 20 * 1073741824,
|
|
|
|
|
'sha256' => $migrationArtifactDigest,
|
|
|
|
|
'created_at' => $now - 90,
|
|
|
|
|
];
|
|
|
|
|
$migrationPrepare = $insertJob(
|
|
|
|
|
'instance_migration_prepare',
|
|
|
|
|
static fn (int $id, string $operationId): array => [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . rawurlencode($instanceUuid)
|
|
|
|
|
. '/peer-migration/prepare',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'migration_id' => $migrationKey,
|
|
|
|
|
'ticket' => $migrationTicket,
|
|
|
|
|
'expires_at' => $now + 3600,
|
|
|
|
|
'artifacts' => [[
|
|
|
|
|
'id' => $migrationArtifactId,
|
|
|
|
|
'role' => 'root',
|
|
|
|
|
]],
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'maintenance_migration_id' => 1,
|
|
|
|
|
],
|
|
|
|
|
$instanceId
|
|
|
|
|
);
|
|
|
|
|
$migrationTransfer = $insertJob(
|
|
|
|
|
'instance_migration_transfer',
|
|
|
|
|
static fn (int $id, string $operationId): array => [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/'
|
|
|
|
|
. rawurlencode($migrationTargetUuid)
|
|
|
|
|
. '/peer-migration/receive',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'migration_id' => $migrationKey,
|
|
|
|
|
'ticket' => $migrationTicket,
|
|
|
|
|
'source_endpoints' => ['https://192.0.2.10:9443'],
|
|
|
|
|
'source_identity_sha256' => str_repeat('1', 64),
|
|
|
|
|
'source_uuid' => $instanceUuid,
|
|
|
|
|
'manifest_digest' => $migrationDigest,
|
|
|
|
|
'network_suspended' => false,
|
|
|
|
|
'debian_12_network_fix' => false,
|
|
|
|
|
'uuid' => $migrationTargetUuid,
|
|
|
|
|
'source' => '',
|
|
|
|
|
'name' => 'matrix-vm',
|
|
|
|
|
'description' => '',
|
|
|
|
|
'vcpu' => 2,
|
|
|
|
|
'memory_mib' => 2048,
|
|
|
|
|
'disk_gib' => 20,
|
|
|
|
|
'bandwidth_mbps' => 100,
|
|
|
|
|
'public_network' => [],
|
|
|
|
|
'template_id' => '',
|
|
|
|
|
'template_revision' => 0,
|
|
|
|
|
'template_generation_token' => '',
|
|
|
|
|
'startup_script' => '',
|
|
|
|
|
'hostname' => '',
|
|
|
|
|
'guest_ssh_public_key' => '',
|
|
|
|
|
'root_password' => '',
|
|
|
|
|
'platform_network_policy_snapshot' => [],
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'reservation_id' => 9004,
|
|
|
|
|
'maintenance_migration_id' => 1,
|
|
|
|
|
'migration_target' => [],
|
|
|
|
|
],
|
|
|
|
|
$instanceId,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
'agent',
|
|
|
|
|
$ownerId,
|
|
|
|
|
null,
|
|
|
|
|
9002
|
|
|
|
|
);
|
|
|
|
|
$migrationTargetClean = $insertJob(
|
|
|
|
|
'instance_migration_target_clean',
|
|
|
|
|
static fn (int $id, string $operationId): array => [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/'
|
|
|
|
|
. rawurlencode($migrationTargetUuid)
|
|
|
|
|
. '/peer-migration/cleanup-target',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'migration_id' => $migrationKey,
|
|
|
|
|
'prior_operation_id' =>
|
|
|
|
|
$migrationTransfer['operation_id'],
|
|
|
|
|
'cleanup_plan_id' => str_repeat('c', 64),
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'maintenance_migration_id' => 1,
|
|
|
|
|
'prior_job_id' => $migrationTransfer['id'],
|
|
|
|
|
'cleanup_plan_id' => str_repeat('c', 64),
|
|
|
|
|
'cleanup_steps' => ['delete_managed_volumes'],
|
|
|
|
|
],
|
|
|
|
|
$instanceId,
|
|
|
|
|
null,
|
|
|
|
|
$migrationTransfer['id'],
|
|
|
|
|
null,
|
|
|
|
|
'agent',
|
|
|
|
|
$ownerId,
|
|
|
|
|
null,
|
|
|
|
|
9002
|
|
|
|
|
);
|
|
|
|
|
$migrationCleanup = $insertJob(
|
|
|
|
|
'instance_migration_cleanup',
|
|
|
|
|
static fn (int $id, string $operationId): array => [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . rawurlencode($instanceUuid)
|
|
|
|
|
. '/peer-migration/cleanup',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'migration_id' => $migrationKey,
|
|
|
|
|
'remove_managed_disks' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'maintenance_migration_id' => 1,
|
|
|
|
|
],
|
|
|
|
|
$instanceId
|
|
|
|
|
);
|
|
|
|
|
$migrationRollback = $insertJob(
|
|
|
|
|
'instance_migration_rollback',
|
|
|
|
|
static fn (int $id, string $operationId): array => [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . rawurlencode($instanceUuid)
|
|
|
|
|
. '/peer-migration/rollback',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'migration_id' => $migrationKey,
|
|
|
|
|
'source_was_running' => true,
|
|
|
|
|
'source_autostart_enabled' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'maintenance_migration_id' => 1,
|
|
|
|
|
],
|
|
|
|
|
$instanceId
|
|
|
|
|
);
|
|
|
|
|
$migrationActivate = $insertJob(
|
|
|
|
|
'instance_migration_activate',
|
|
|
|
|
static fn (int $id, string $operationId): array => [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/'
|
|
|
|
|
. rawurlencode($migrationTargetUuid)
|
|
|
|
|
. '/peer-migration/activate',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'migration_id' => $migrationKey,
|
|
|
|
|
'target_was_running' => true,
|
|
|
|
|
'target_autostart_enabled' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'maintenance_migration_id' => 1,
|
|
|
|
|
],
|
|
|
|
|
$instanceId
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_instance_migrations')->insert([
|
|
|
|
|
'id' => 1,
|
|
|
|
|
'migration_key' => $migrationKey,
|
|
|
|
|
'instance_id' => $instanceId,
|
|
|
|
|
'restore_id' => 9002,
|
|
|
|
|
'source_host_id' => $hostId,
|
|
|
|
|
'source_uuid' => $instanceUuid,
|
|
|
|
|
'source_reservation_id' => 9003,
|
|
|
|
|
'target_host_id' => $hostId,
|
|
|
|
|
'target_uuid' => $migrationTargetUuid,
|
|
|
|
|
'target_reservation_id' => 9004,
|
|
|
|
|
'source_prepare_job_id' => $migrationPrepare['id'],
|
|
|
|
|
'target_receive_job_id' => $migrationTransfer['id'],
|
|
|
|
|
'firewall_job_id' => null,
|
|
|
|
|
'activate_job_id' => $migrationActivate['id'],
|
|
|
|
|
'rollback_job_id' => $migrationRollback['id'],
|
|
|
|
|
'cleanup_job_id' => $migrationCleanup['id'],
|
|
|
|
|
'cleanup_mode' => 'delete',
|
|
|
|
|
'state' => 'source_retained',
|
|
|
|
|
'recovery_action' => 'reconcile_cleanup',
|
|
|
|
|
'retry_generation' => 0,
|
|
|
|
|
'source_endpoint' => '["https://192.0.2.10:9443"]',
|
|
|
|
|
'peer_trust_id' => str_repeat('1', 64),
|
|
|
|
|
'manifest_digest' => $migrationDigest,
|
|
|
|
|
'source_was_running' => true,
|
|
|
|
|
'source_autostart_enabled' => true,
|
|
|
|
|
'source_service_status' => 'active',
|
|
|
|
|
'source_network_status' => 'active',
|
|
|
|
|
'error_code' => 'operation_effect_unknown',
|
|
|
|
|
'cutover_at' => $now - 60,
|
|
|
|
|
'completed_at' => null,
|
|
|
|
|
'created_at' => $now - 300,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_peer_migration_artifacts')->insert([
|
|
|
|
|
'id' => 1,
|
|
|
|
|
'migration_id' => 1,
|
|
|
|
|
'artifact_id' => $migrationArtifactId,
|
|
|
|
|
'role' => 'root',
|
|
|
|
|
'source_snapshot_id' => null,
|
|
|
|
|
'snapshot_key' => null,
|
|
|
|
|
'name' => '',
|
|
|
|
|
'consistency' => 'crash',
|
|
|
|
|
'object_size_bytes' => 1048576,
|
|
|
|
|
'virtual_size_bytes' => 20 * 1073741824,
|
|
|
|
|
'sha256' => $migrationArtifactDigest,
|
|
|
|
|
'state' => 'ready',
|
|
|
|
|
'snapshot_created_at' => null,
|
|
|
|
|
'created_at' => $now - 300,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_backup_restores')->insert([
|
|
|
|
|
'id' => 9002,
|
|
|
|
|
'restore_key' => str_repeat('2', 32),
|
|
|
|
|
'backup_id' => null,
|
|
|
|
|
'target_host_id' => $hostId,
|
|
|
|
|
'target_instance_id' => null,
|
|
|
|
|
'reservation_id' => 9004,
|
|
|
|
|
'job_id' => $migrationTransfer['id'],
|
|
|
|
|
'operation_id' => $migrationTransfer['operation_id'],
|
|
|
|
|
'new_uuid' => $migrationTargetUuid,
|
|
|
|
|
'new_name' => 'matrix-vm',
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'error_code' => 'operation_effect_unknown',
|
|
|
|
|
'ready_at' => null,
|
|
|
|
|
'created_at' => $now - 300,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_resource_reservations')->insert([
|
|
|
|
|
[
|
|
|
|
|
'id' => 9003,
|
|
|
|
|
'reservation_key' => str_repeat('3', 32),
|
|
|
|
|
'backup_restore_id' => null,
|
|
|
|
|
'backup_restore_attempt_key' => null,
|
|
|
|
|
'migration_source_debt_id' => 1,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'resource_pool_id' => $migrationResourcePoolId,
|
|
|
|
|
'network_attachment_id' => null,
|
|
|
|
|
'public_ip_pool_id' => null,
|
|
|
|
|
'operation_id' => $migrationSourceOperationId,
|
|
|
|
|
'public_ip_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'cpu_cores' => 2,
|
|
|
|
|
'memory_mib' => 2048,
|
|
|
|
|
'disk_gib' => 20,
|
|
|
|
|
'state' => 'held',
|
|
|
|
|
'expires_at' => null,
|
|
|
|
|
'released_at' => null,
|
|
|
|
|
'release_reason' => '',
|
|
|
|
|
'created_at' => $now - 300,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
'id' => 9004,
|
|
|
|
|
'reservation_key' => str_repeat('4', 32),
|
|
|
|
|
'backup_restore_id' => 9002,
|
|
|
|
|
'backup_restore_attempt_key' => null,
|
|
|
|
|
'migration_source_debt_id' => null,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'resource_pool_id' => $migrationResourcePoolId,
|
|
|
|
|
'network_attachment_id' => null,
|
|
|
|
|
'public_ip_pool_id' => null,
|
|
|
|
|
'operation_id' => $migrationTransfer['operation_id'],
|
|
|
|
|
'public_ip_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'cpu_cores' => 2,
|
|
|
|
|
'memory_mib' => 2048,
|
|
|
|
|
'disk_gib' => 20,
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'expires_at' => null,
|
|
|
|
|
'released_at' => null,
|
|
|
|
|
'release_reason' => '',
|
|
|
|
|
'created_at' => $now - 300,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
],
|
|
|
|
|
]);
|
|
|
|
|
$insertMigrationClaim = static function (
|
|
|
|
|
int $claimId,
|
|
|
|
|
int $ipId,
|
|
|
|
|
int $reservationId,
|
|
|
|
|
int $family,
|
|
|
|
|
int $poolId,
|
|
|
|
|
string $address,
|
|
|
|
|
string $operationId,
|
|
|
|
|
string $state
|
|
|
|
|
) use ($hostId, $migrationAttachmentId, $now): void {
|
|
|
|
|
Capsule::table('cloud_public_ips')->insert([
|
|
|
|
|
'id' => $ipId,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'public_ip_pool_id' => $poolId,
|
|
|
|
|
'address_family' => $family,
|
|
|
|
|
'address' => $address,
|
|
|
|
|
'address_binary' => inet_pton($address),
|
|
|
|
|
'prefix_length' => $family === 4 ? 24 : 64,
|
|
|
|
|
'state' => $state === 'reserved' ? 'reserved' : 'uncertain',
|
|
|
|
|
'reservation_id' => $reservationId,
|
|
|
|
|
'reservation_ip_claim_id' => $claimId,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'created_at' => $now - 300,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_resource_reservation_ips')->insert([
|
|
|
|
|
'id' => $claimId,
|
|
|
|
|
'reservation_id' => $reservationId,
|
|
|
|
|
'address_family' => $family,
|
|
|
|
|
'claim_generation' => 1,
|
|
|
|
|
'active_claim' => 1,
|
|
|
|
|
'network_attachment_id' => $migrationAttachmentId,
|
|
|
|
|
'public_ip_pool_id' => $poolId,
|
|
|
|
|
'public_ip_id' => $ipId,
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'state' => $state,
|
|
|
|
|
'released_at' => null,
|
|
|
|
|
'release_reason' => '',
|
|
|
|
|
'created_at' => $now - 300,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
};
|
|
|
|
|
$insertMigrationClaim(
|
|
|
|
|
9201,
|
|
|
|
|
9007,
|
|
|
|
|
9004,
|
|
|
|
|
4,
|
|
|
|
|
$migrationPublicPoolId,
|
|
|
|
|
'8.8.30.30',
|
|
|
|
|
$migrationTransfer['operation_id'],
|
|
|
|
|
'uncertain'
|
|
|
|
|
);
|
|
|
|
|
$insertMigrationClaim(
|
|
|
|
|
9202,
|
|
|
|
|
9008,
|
|
|
|
|
9004,
|
|
|
|
|
6,
|
|
|
|
|
$migrationIpv6PublicPoolId,
|
|
|
|
|
'2404:c140:156:30::130',
|
|
|
|
|
$migrationTransfer['operation_id'],
|
|
|
|
|
'uncertain'
|
|
|
|
|
);
|
|
|
|
|
$insertMigrationClaim(
|
|
|
|
|
9203,
|
|
|
|
|
9009,
|
|
|
|
|
9003,
|
|
|
|
|
4,
|
|
|
|
|
$migrationPublicPoolId,
|
|
|
|
|
'8.8.30.29',
|
|
|
|
|
$migrationSourceOperationId,
|
|
|
|
|
'reserved'
|
|
|
|
|
);
|
|
|
|
|
$insertMigrationClaim(
|
|
|
|
|
9204,
|
|
|
|
|
9010,
|
|
|
|
|
9003,
|
|
|
|
|
6,
|
|
|
|
|
$migrationIpv6PublicPoolId,
|
|
|
|
|
'2404:c140:156:30::129',
|
|
|
|
|
$migrationSourceOperationId,
|
|
|
|
|
'reserved'
|
|
|
|
|
);
|
|
|
|
|
$migrationTargetClaims =
|
|
|
|
|
CloudPublicNetworkAllocationService::currentClaimSet(
|
|
|
|
|
9004,
|
|
|
|
|
'uncertain'
|
|
|
|
|
);
|
|
|
|
|
$migrationTargetIdentity = CloudFrozenPublicNetworkIdentity::fromClaimSet(
|
|
|
|
|
$migrationTargetUuid,
|
|
|
|
|
$migrationTargetClaims
|
|
|
|
|
);
|
|
|
|
|
$migrationSourceClaims =
|
|
|
|
|
CloudPublicNetworkAllocationService::currentClaimSet(9003, 'held');
|
|
|
|
|
$migrationSourceIdentity = CloudFrozenPublicNetworkIdentity::fromClaimSet(
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
$migrationSourceClaims
|
|
|
|
|
);
|
|
|
|
|
$migrationTransfer['payload']['body']['public_network'] =
|
|
|
|
|
$migrationTargetIdentity->publicNetwork()->toArray();
|
|
|
|
|
$migrationTransfer['payload']['migration_target'] = [
|
|
|
|
|
'contract' => 1,
|
|
|
|
|
'backup_restore_id' => 9002,
|
|
|
|
|
'reservation_id' => 9004,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'resource_pool_id' => $migrationResourcePoolId,
|
|
|
|
|
'operation_id' => $migrationTransfer['operation_id'],
|
|
|
|
|
'target_uuid' => strtolower($migrationTargetUuid),
|
|
|
|
|
'target_agent_revision' => 3,
|
|
|
|
|
'target_agent_contract_sha256' =>
|
|
|
|
|
CloudPortableRestoreCompatibility::supported()->sha256(),
|
|
|
|
|
'target_identity_json' => $migrationTargetIdentity->identityJson(),
|
|
|
|
|
'target_identity_sha256' =>
|
|
|
|
|
$migrationTargetIdentity->identitySha256(),
|
|
|
|
|
'target_claim_set_sha256' =>
|
|
|
|
|
$migrationTargetIdentity->claimSetSha256(),
|
|
|
|
|
'target_public_network_json' =>
|
|
|
|
|
$migrationTargetIdentity->publicNetworkJson(),
|
|
|
|
|
'target_public_network_sha256' =>
|
|
|
|
|
$migrationTargetIdentity->publicNetworkSha256(),
|
|
|
|
|
'vpc_port' => null,
|
|
|
|
|
];
|
|
|
|
|
$migrationTransferPayload = CloudSecrets::encrypt(
|
|
|
|
|
$migrationTransfer['payload'],
|
|
|
|
|
'job-payload'
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $migrationTransfer['id'])
|
|
|
|
|
->update(['sealed_payload' => $migrationTransferPayload]);
|
|
|
|
|
$migrationTransfer['job']->sealed_payload = $migrationTransferPayload;
|
|
|
|
|
Capsule::table('cloud_instance_migration_source_debts')->insert([
|
|
|
|
|
'id' => 1,
|
|
|
|
|
'debt_key' => str_repeat('5', 32),
|
|
|
|
|
'migration_id' => 1,
|
|
|
|
|
'source_reservation_id' => 9003,
|
|
|
|
|
'source_host_id' => $hostId,
|
|
|
|
|
'source_uuid' => strtolower($instanceUuid),
|
|
|
|
|
'source_network_attachment_id' => $migrationAttachmentId,
|
|
|
|
|
'source_identity_json' => $migrationSourceIdentity->identityJson(),
|
|
|
|
|
'source_identity_sha256' =>
|
|
|
|
|
$migrationSourceIdentity->identitySha256(),
|
|
|
|
|
'source_claim_set_sha256' =>
|
|
|
|
|
$migrationSourceIdentity->claimSetSha256(),
|
|
|
|
|
'source_public_network_json' =>
|
|
|
|
|
$migrationSourceIdentity->publicNetworkJson(),
|
|
|
|
|
'source_public_network_sha256' =>
|
|
|
|
|
$migrationSourceIdentity->publicNetworkSha256(),
|
|
|
|
|
'cleanup_operation_id' => $migrationCleanup['operation_id'],
|
|
|
|
|
'cleanup_job_id' => $migrationCleanup['id'],
|
|
|
|
|
'state' => 'attention',
|
|
|
|
|
'active_debt' => 1,
|
|
|
|
|
'last_error_code' => 'operation_effect_unknown',
|
|
|
|
|
'resolved_at' => null,
|
|
|
|
|
'created_at' => $now - 300,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
$addFixture(
|
|
|
|
|
'instance_migration_prepare',
|
|
|
|
|
'source-prepare',
|
|
|
|
|
$migrationPrepare,
|
|
|
|
|
'prepare-peer-migration:' . $migrationKey,
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
[
|
|
|
|
|
'migration_id' => $migrationKey,
|
|
|
|
|
'manifest_digest' => $migrationDigest,
|
|
|
|
|
'artifacts' => [$migrationArtifact],
|
|
|
|
|
'source_stopped' => true,
|
|
|
|
|
'source_was_running' => true,
|
|
|
|
|
'source_autostart_enabled' => true,
|
|
|
|
|
'source_autostart_disabled' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$addFixture(
|
|
|
|
|
'instance_migration_transfer',
|
|
|
|
|
'target-receive',
|
|
|
|
|
$migrationTransfer,
|
|
|
|
|
'receive-peer-migration:' . $migrationKey,
|
|
|
|
|
$migrationTargetUuid,
|
|
|
|
|
[
|
|
|
|
|
'uuid' => $migrationTargetUuid,
|
|
|
|
|
'public_ip_cidr' => '203.0.113.30/24',
|
|
|
|
|
'public_pool_key' => str_repeat('b', 32),
|
|
|
|
|
'migration_id' => $migrationKey,
|
|
|
|
|
'manifest_digest' => $migrationDigest,
|
|
|
|
|
'artifacts' => [$migrationArtifact],
|
|
|
|
|
'staged_powered_off' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$addFixture(
|
|
|
|
|
'instance_migration_target_clean',
|
|
|
|
|
'target-residue-cleanup',
|
|
|
|
|
$migrationTargetClean,
|
|
|
|
|
'cleanup-peer-migration-target:' . $migrationKey,
|
|
|
|
|
$migrationTargetUuid,
|
|
|
|
|
$deleteProjection
|
|
|
|
|
);
|
|
|
|
|
$addFixture(
|
|
|
|
|
'instance_migration_cleanup',
|
|
|
|
|
'source-cleanup',
|
|
|
|
|
$migrationCleanup,
|
|
|
|
|
'delete-vm',
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
$deleteProjection
|
|
|
|
|
);
|
|
|
|
|
$addFixture(
|
|
|
|
|
'instance_migration_rollback',
|
|
|
|
|
'source-rollback',
|
|
|
|
|
$migrationRollback,
|
|
|
|
|
'rollback-peer-migration:' . $migrationKey,
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
[
|
|
|
|
|
'uuid' => $instanceUuid,
|
|
|
|
|
'state' => 'running',
|
|
|
|
|
'running' => true,
|
|
|
|
|
'autostart_enabled' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$addFixture(
|
|
|
|
|
'instance_migration_activate',
|
|
|
|
|
'target-activate',
|
|
|
|
|
$migrationActivate,
|
|
|
|
|
'activate-peer-migration:' . $migrationKey,
|
|
|
|
|
$migrationTargetUuid,
|
|
|
|
|
[
|
|
|
|
|
'uuid' => $migrationTargetUuid,
|
|
|
|
|
'state' => 'running',
|
|
|
|
|
'running' => true,
|
|
|
|
|
'autostart_enabled' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$instanceFixture(
|
|
|
|
|
'network_change',
|
|
|
|
|
'public-ip-change',
|
|
|
|
|
'POST',
|
|
|
|
|
'/network',
|
|
|
|
|
[
|
|
|
|
|
'network' => 'bridge:br0',
|
|
|
|
|
'public_pool_key' => str_repeat('c', 32),
|
|
|
|
|
'public_pool_cidr' => '203.0.113.0/24',
|
|
|
|
|
'public_ip_cidr' => '203.0.113.20/24',
|
|
|
|
|
'gateway' => '203.0.113.1',
|
|
|
|
|
'dns_servers' => ['1.1.1.1'],
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'change-network',
|
|
|
|
|
['uuid' => $instanceUuid, 'network_changed' => true]
|
|
|
|
|
);
|
|
|
|
|
$instanceFixture(
|
|
|
|
|
'instance_settings',
|
|
|
|
|
'hostname',
|
|
|
|
|
'PATCH',
|
|
|
|
|
'/settings',
|
|
|
|
|
[
|
|
|
|
|
'hostname' => 'matrix.example',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'update-settings',
|
|
|
|
|
['uuid' => $instanceUuid, 'settings_changed' => true],
|
|
|
|
|
['settings_fields' => ['hostname'], 'credential' => null]
|
|
|
|
|
);
|
|
|
|
|
$upgradeProjection = [
|
|
|
|
|
'uuid' => $instanceUuid,
|
|
|
|
|
'from_vcpu' => 2,
|
|
|
|
|
'target_vcpu' => 4,
|
|
|
|
|
'applied_vcpu' => 4,
|
|
|
|
|
'from_memory_mib' => 2048,
|
|
|
|
|
'target_memory_mib' => 4096,
|
|
|
|
|
'applied_memory_mib' => 4096,
|
|
|
|
|
'from_disk_gib' => 40,
|
|
|
|
|
'target_disk_gib' => 80,
|
|
|
|
|
'applied_disk_gib' => 80,
|
|
|
|
|
'from_bandwidth_mbps' => 100,
|
|
|
|
|
'target_bandwidth_mbps' => 200,
|
|
|
|
|
'applied_bandwidth_mbps' => 200,
|
|
|
|
|
'status' => 'running',
|
|
|
|
|
'originally_running' => true,
|
|
|
|
|
'power_state_restored' => true,
|
|
|
|
|
'resources_resized' => true,
|
|
|
|
|
];
|
|
|
|
|
$upgradeRecord = $instanceFixture(
|
|
|
|
|
'instance_upgrade',
|
|
|
|
|
'grow-only',
|
|
|
|
|
'POST',
|
|
|
|
|
'/resources',
|
|
|
|
|
[
|
|
|
|
|
'from_vcpu' => 2,
|
|
|
|
|
'target_vcpu' => 4,
|
|
|
|
|
'from_memory_mib' => 2048,
|
|
|
|
|
'target_memory_mib' => 4096,
|
|
|
|
|
'from_disk_gib' => 40,
|
|
|
|
|
'target_disk_gib' => 80,
|
|
|
|
|
'from_bandwidth_mbps' => 100,
|
|
|
|
|
'target_bandwidth_mbps' => 200,
|
|
|
|
|
'confirm_shutdown' => true,
|
|
|
|
|
],
|
|
|
|
|
'resize-vm',
|
|
|
|
|
$upgradeProjection
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$snapshotFixture = static function (
|
|
|
|
|
string $kind,
|
|
|
|
|
string $variant,
|
|
|
|
|
string $method,
|
|
|
|
|
string $suffix,
|
|
|
|
|
array $body,
|
|
|
|
|
string $expectedAction,
|
|
|
|
|
array $projection,
|
|
|
|
|
string $consistency = 'crash'
|
|
|
|
|
) use (
|
|
|
|
|
$insertJob,
|
|
|
|
|
$addFixture,
|
|
|
|
|
$instanceId,
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
$ownerId,
|
|
|
|
|
$hostId,
|
|
|
|
|
$now
|
|
|
|
|
): void {
|
|
|
|
|
static $snapshotId = 100;
|
|
|
|
|
$id = $snapshotId++;
|
|
|
|
|
$key = hash('md5', 'recovery-matrix-snapshot:' . $id);
|
|
|
|
|
$record = $insertJob(
|
|
|
|
|
$kind,
|
|
|
|
|
static function (int $jobId, string $operationId) use (
|
|
|
|
|
$kind,
|
|
|
|
|
$method,
|
|
|
|
|
$suffix,
|
|
|
|
|
$body,
|
|
|
|
|
$key,
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
$ownerId
|
|
|
|
|
): array {
|
|
|
|
|
return [
|
|
|
|
|
'method' => $method,
|
|
|
|
|
'request_target' => '/v1/vms/'
|
|
|
|
|
. rawurlencode($instanceUuid) . '/snapshots'
|
|
|
|
|
. ($kind === 'snapshot_create' ? ''
|
|
|
|
|
: '/' . rawurlencode($key) . $suffix),
|
|
|
|
|
'body' => $body + ($body === [] ? [] : [
|
|
|
|
|
'snapshot_key' => $key,
|
|
|
|
|
]),
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => false,
|
|
|
|
|
'snapshot_key' => $key,
|
|
|
|
|
'snapshot_previous_state' => 'ready',
|
|
|
|
|
];
|
|
|
|
|
},
|
|
|
|
|
$instanceId,
|
|
|
|
|
$id
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_instance_snapshots')->insert([
|
|
|
|
|
'id' => $id,
|
|
|
|
|
'snapshot_key' => $key,
|
|
|
|
|
'instance_id' => $instanceId,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'name' => 'matrix-snapshot-' . $id,
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'consistency' => $consistency,
|
|
|
|
|
'size_bytes' => null,
|
|
|
|
|
'last_operation_id' => $record['operation_id'],
|
|
|
|
|
'ready_at' => $now - 300,
|
|
|
|
|
'last_restored_at' => null,
|
|
|
|
|
'deleted_at' => null,
|
|
|
|
|
'created_at' => $now - 600,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
$resolvedAction = str_replace('__SNAPSHOT_KEY__', $key,
|
|
|
|
|
$expectedAction);
|
|
|
|
|
$resolvedProjection = array_map(
|
|
|
|
|
static fn (mixed $value): mixed =>
|
|
|
|
|
$value === '__SNAPSHOT_KEY__' ? $key : $value,
|
|
|
|
|
$projection
|
|
|
|
|
);
|
|
|
|
|
$addFixture(
|
|
|
|
|
$kind,
|
|
|
|
|
$variant,
|
|
|
|
|
$record,
|
|
|
|
|
$resolvedAction,
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
$resolvedProjection
|
|
|
|
|
);
|
|
|
|
|
};
|
|
|
|
|
foreach (['filesystem', 'crash'] as $consistency) {
|
|
|
|
|
$snapshotFixture(
|
|
|
|
|
'snapshot_create',
|
|
|
|
|
$consistency,
|
|
|
|
|
'POST',
|
|
|
|
|
'',
|
|
|
|
|
[
|
|
|
|
|
'name' => 'matrix-' . $consistency,
|
|
|
|
|
'description' => '',
|
|
|
|
|
'quiesce' => $consistency === 'filesystem',
|
|
|
|
|
'consistency_mode' => $consistency,
|
|
|
|
|
],
|
|
|
|
|
'create-snapshot:__SNAPSHOT_KEY__:' . $consistency,
|
|
|
|
|
[
|
|
|
|
|
'id' => 'matrix-id',
|
|
|
|
|
'snapshot_key' => '__SNAPSHOT_KEY__',
|
|
|
|
|
'created_at' => '2026-10-21T00:00:00Z',
|
|
|
|
|
'state' => 'ready',
|
|
|
|
|
'consistency' => $consistency,
|
|
|
|
|
],
|
|
|
|
|
$consistency
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
$snapshotFixture(
|
|
|
|
|
'snapshot_restore',
|
|
|
|
|
'restore',
|
|
|
|
|
'POST',
|
|
|
|
|
'/restore',
|
|
|
|
|
[],
|
|
|
|
|
'restore-snapshot:__SNAPSHOT_KEY__',
|
|
|
|
|
['snapshot_key' => '__SNAPSHOT_KEY__', 'restored' => true]
|
|
|
|
|
);
|
|
|
|
|
$snapshotFixture(
|
|
|
|
|
'snapshot_delete',
|
|
|
|
|
'delete',
|
|
|
|
|
'DELETE',
|
|
|
|
|
'',
|
|
|
|
|
[],
|
|
|
|
|
'delete-snapshot:__SNAPSHOT_KEY__',
|
|
|
|
|
['snapshot_key' => '__SNAPSHOT_KEY__', 'deleted' => true]
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
foreach (['filesystem', 'crash'] as $consistency) {
|
|
|
|
|
$backupId = $consistency === 'filesystem' ? 201 : 202;
|
|
|
|
|
$backupKey = hash('md5', 'recovery-matrix-backup:' . $backupId);
|
|
|
|
|
$record = $insertJob(
|
|
|
|
|
'backup_create',
|
|
|
|
|
static function (int $jobId, string $operationId) use (
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
$ownerId,
|
|
|
|
|
$backupKey,
|
|
|
|
|
$consistency
|
|
|
|
|
): array {
|
|
|
|
|
return [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/'
|
|
|
|
|
. rawurlencode($instanceUuid) . '/backups',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'backup_key' => $backupKey,
|
|
|
|
|
'name' => 'matrix-' . $consistency,
|
|
|
|
|
'consistency_mode' => $consistency,
|
|
|
|
|
'object_upload' => [
|
|
|
|
|
'part_size_bytes' => 8388608,
|
|
|
|
|
'maximum_size_bytes' => 42949672960,
|
|
|
|
|
'part_urls' => [
|
|
|
|
|
'https://objects.invalid/part-1',
|
|
|
|
|
],
|
|
|
|
|
'complete_url' =>
|
|
|
|
|
'https://objects.invalid/complete',
|
|
|
|
|
'abort_url' => 'https://objects.invalid/abort',
|
|
|
|
|
'verify_url' => 'https://objects.invalid/verify',
|
|
|
|
|
],
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'backup_key' => $backupKey,
|
|
|
|
|
'maximum_size_bytes' => 42949672960,
|
|
|
|
|
];
|
|
|
|
|
},
|
|
|
|
|
$instanceId,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
'agent',
|
|
|
|
|
$ownerId,
|
|
|
|
|
$backupId
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_instance_backups')->insert([
|
|
|
|
|
'id' => $backupId,
|
|
|
|
|
'backup_key' => $backupKey,
|
|
|
|
|
'source_instance_id' => $instanceId,
|
|
|
|
|
'source_host_id' => $hostId,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'source_uuid' => $instanceUuid,
|
|
|
|
|
'name' => 'matrix-' . $consistency,
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'consistency' => $consistency,
|
|
|
|
|
'object_key' => 'vm-backups/' . $backupKey . '/system.qcow2',
|
|
|
|
|
'spec_json' => '{}',
|
|
|
|
|
'sealed_storage_runtime' => 'sealed-runtime',
|
|
|
|
|
'active_job_id' => $record['id'],
|
|
|
|
|
'last_operation_id' => $record['operation_id'],
|
|
|
|
|
'created_at' => $now - 180,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
$addFixture(
|
|
|
|
|
'backup_create',
|
|
|
|
|
$consistency,
|
|
|
|
|
$record,
|
|
|
|
|
'create-backup:' . $backupKey . ':' . $consistency,
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
[
|
|
|
|
|
'backup_key' => $backupKey,
|
|
|
|
|
'size_bytes' => 1073741824,
|
|
|
|
|
'virtual_size_bytes' => 42949672960,
|
|
|
|
|
'sha256' => str_repeat('6', 64),
|
|
|
|
|
'consistency' => $consistency,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$cleanupBackupId = 205;
|
|
|
|
|
$cleanupBackupKey = hash('md5',
|
|
|
|
|
'recovery-matrix-backup:' . $cleanupBackupId);
|
|
|
|
|
$cleanupPlanId = str_repeat('8', 64);
|
|
|
|
|
$cleanupPrior = $insertJob(
|
|
|
|
|
'backup_create',
|
|
|
|
|
static function (int $jobId, string $operationId) use (
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
$ownerId,
|
|
|
|
|
$cleanupBackupKey
|
|
|
|
|
): array {
|
|
|
|
|
return [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/'
|
|
|
|
|
. rawurlencode($instanceUuid) . '/backups',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'backup_key' => $cleanupBackupKey,
|
|
|
|
|
'name' => 'matrix-cleanup-source',
|
|
|
|
|
'consistency_mode' => 'filesystem',
|
|
|
|
|
'object_upload' => [
|
|
|
|
|
'part_size_bytes' => 8388608,
|
|
|
|
|
'maximum_size_bytes' => 42949672960,
|
|
|
|
|
'part_urls' => [
|
|
|
|
|
'https://objects.invalid/part-1',
|
|
|
|
|
],
|
|
|
|
|
'complete_url' =>
|
|
|
|
|
'https://objects.invalid/complete',
|
|
|
|
|
'abort_url' => 'https://objects.invalid/abort',
|
|
|
|
|
'verify_url' => 'https://objects.invalid/verify',
|
|
|
|
|
],
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'backup_key' => $cleanupBackupKey,
|
|
|
|
|
'maximum_size_bytes' => 42949672960,
|
|
|
|
|
];
|
|
|
|
|
},
|
|
|
|
|
$instanceId,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
'agent',
|
|
|
|
|
$ownerId,
|
|
|
|
|
$cleanupBackupId
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $cleanupPrior['id'])->update([
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'result_json' => json_encode([
|
|
|
|
|
'backup_key' => $cleanupBackupKey,
|
|
|
|
|
'size_bytes' => 1073741824,
|
|
|
|
|
'virtual_size_bytes' => 42949672960,
|
|
|
|
|
'sha256' => str_repeat('6', 64),
|
|
|
|
|
'consistency' => 'filesystem',
|
|
|
|
|
'cleanup_pending' => true,
|
|
|
|
|
], JSON_THROW_ON_ERROR),
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_instance_backups')->insert([
|
|
|
|
|
'id' => $cleanupBackupId,
|
|
|
|
|
'backup_key' => $cleanupBackupKey,
|
|
|
|
|
'source_instance_id' => $instanceId,
|
|
|
|
|
'source_host_id' => $hostId,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'source_uuid' => $instanceUuid,
|
|
|
|
|
'name' => 'matrix-cleanup-source',
|
|
|
|
|
'state' => 'ready',
|
|
|
|
|
'consistency' => 'filesystem',
|
|
|
|
|
'object_key' => 'vm-backups/' . $cleanupBackupKey . '/system.qcow2',
|
|
|
|
|
'object_size_bytes' => 1073741824,
|
|
|
|
|
'virtual_size_bytes' => 42949672960,
|
|
|
|
|
'sha256' => str_repeat('6', 64),
|
|
|
|
|
'spec_json' => '{}',
|
|
|
|
|
'sealed_storage_runtime' => 'sealed-runtime',
|
|
|
|
|
'last_operation_id' => $cleanupPrior['operation_id'],
|
|
|
|
|
'error_code' => 'backup_local_cleanup_pending',
|
|
|
|
|
'created_at' => $now - 180,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
$cleanupRecord = $insertJob(
|
|
|
|
|
'backup_workspace_cleanup',
|
|
|
|
|
static function (int $jobId, string $operationId) use (
|
|
|
|
|
$ownerId,
|
|
|
|
|
$cleanupBackupKey,
|
|
|
|
|
$cleanupPlanId,
|
|
|
|
|
$cleanupPrior
|
|
|
|
|
): array {
|
|
|
|
|
return [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/operations/'
|
|
|
|
|
. rawurlencode($operationId)
|
|
|
|
|
. '/cleanup-backup-workspace',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'prior_operation_id' => $cleanupPrior['operation_id'],
|
|
|
|
|
'cleanup_plan_id' => $cleanupPlanId,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'backup_key' => $cleanupBackupKey,
|
|
|
|
|
'prior_job_id' => $cleanupPrior['id'],
|
|
|
|
|
'prior_operation_id' => $cleanupPrior['operation_id'],
|
|
|
|
|
'cleanup_plan_id' => $cleanupPlanId,
|
|
|
|
|
];
|
|
|
|
|
},
|
|
|
|
|
$instanceId,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
'agent',
|
|
|
|
|
$ownerId,
|
|
|
|
|
$cleanupBackupId
|
|
|
|
|
);
|
|
|
|
|
$addFixture(
|
|
|
|
|
'backup_workspace_cleanup',
|
|
|
|
|
'cleanup',
|
|
|
|
|
$cleanupRecord,
|
|
|
|
|
'cleanup-backup-workspace:' . $cleanupBackupKey,
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
[
|
|
|
|
|
'backup_key' => $cleanupBackupKey,
|
|
|
|
|
'cleaned' => true,
|
|
|
|
|
'workspace_absent' => true,
|
|
|
|
|
'metadata_absent' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$recoverBackupId = 204;
|
|
|
|
|
$recoverBackupKey = hash('md5',
|
|
|
|
|
'recovery-matrix-backup:' . $recoverBackupId);
|
|
|
|
|
$recoverBackupSha = str_repeat('7', 64);
|
|
|
|
|
$recoverLedgerId = 301;
|
|
|
|
|
$recoverLedgerKey = hash('md5',
|
|
|
|
|
'recovery-matrix-ledger:' . $recoverLedgerId);
|
|
|
|
|
$recoverRecord = $insertJob(
|
|
|
|
|
'backup_recover',
|
|
|
|
|
static function (int $jobId, string $operationId) use (
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
$ownerId,
|
|
|
|
|
$recoverBackupKey,
|
|
|
|
|
$recoverBackupSha,
|
|
|
|
|
$recoverLedgerKey
|
|
|
|
|
): array {
|
|
|
|
|
return [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/'
|
|
|
|
|
. rawurlencode($instanceUuid) . '/recover-backup',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'backup_key' => $recoverBackupKey,
|
|
|
|
|
'source_url' => 'https://objects.invalid/recover.qcow2',
|
|
|
|
|
'source_size_bytes' => 1073741824,
|
|
|
|
|
'source_sha256' => $recoverBackupSha,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => false,
|
|
|
|
|
'backup_key' => $recoverBackupKey,
|
|
|
|
|
'recovery_key' => $recoverLedgerKey,
|
|
|
|
|
];
|
|
|
|
|
},
|
|
|
|
|
$instanceId,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
'agent',
|
|
|
|
|
$ownerId,
|
|
|
|
|
$recoverBackupId,
|
|
|
|
|
null,
|
|
|
|
|
$recoverLedgerId
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_instance_backups')->insert([
|
|
|
|
|
'id' => $recoverBackupId,
|
|
|
|
|
'backup_key' => $recoverBackupKey,
|
|
|
|
|
'source_instance_id' => $instanceId,
|
|
|
|
|
'source_host_id' => $hostId,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'source_uuid' => $instanceUuid,
|
|
|
|
|
'name' => 'matrix-automatic-backup',
|
|
|
|
|
'purpose' => 'portable',
|
|
|
|
|
'source' => 'automatic',
|
|
|
|
|
'state' => 'ready',
|
|
|
|
|
'consistency' => 'filesystem',
|
|
|
|
|
'object_key' => 'vm-backups/' . $recoverBackupKey . '/system.qcow2',
|
|
|
|
|
'object_size_bytes' => 1073741824,
|
|
|
|
|
'virtual_size_bytes' => 42949672960,
|
|
|
|
|
'sha256' => $recoverBackupSha,
|
|
|
|
|
'spec_json' => '{}',
|
|
|
|
|
'sealed_storage_runtime' => 'sealed-runtime',
|
|
|
|
|
'created_at' => $now - 600,
|
|
|
|
|
'updated_at' => $now - 300,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_backup_recoveries')->insert([
|
|
|
|
|
'id' => $recoverLedgerId,
|
|
|
|
|
'recovery_key' => $recoverLedgerKey,
|
|
|
|
|
'backup_id' => $recoverBackupId,
|
|
|
|
|
'instance_id' => $instanceId,
|
|
|
|
|
'job_id' => $recoverRecord['id'],
|
|
|
|
|
'operation_id' => $recoverRecord['operation_id'],
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'error_code' => 'operation_effect_unknown',
|
|
|
|
|
'completed_at' => null,
|
|
|
|
|
'created_at' => $now - 180,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
$addFixture(
|
|
|
|
|
'backup_recover',
|
|
|
|
|
'recover',
|
|
|
|
|
$recoverRecord,
|
|
|
|
|
'recover-backup:' . $recoverBackupKey . ':' . $recoverBackupSha,
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
[
|
|
|
|
|
'backup_key' => $recoverBackupKey,
|
|
|
|
|
'sha256' => $recoverBackupSha,
|
|
|
|
|
'restored' => true,
|
|
|
|
|
'power_state' => 'running',
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$firewallVersionId = 1;
|
|
|
|
|
$firewallRevision = 9;
|
|
|
|
|
$firewallDigest = str_repeat('d', 64);
|
|
|
|
|
$firewallRecord = $instanceFixture(
|
|
|
|
|
'firewall_apply',
|
|
|
|
|
'apply',
|
|
|
|
|
'PUT',
|
|
|
|
|
'/firewall',
|
|
|
|
|
[
|
|
|
|
|
'revision' => $firewallRevision,
|
|
|
|
|
'default_in' => 'drop',
|
|
|
|
|
'default_out' => 'accept',
|
|
|
|
|
'policy_digest' => $firewallDigest,
|
|
|
|
|
'rules' => [],
|
|
|
|
|
],
|
|
|
|
|
'apply-firewall',
|
|
|
|
|
[
|
|
|
|
|
'revision' => $firewallRevision,
|
|
|
|
|
'default_in' => 'drop',
|
|
|
|
|
'default_out' => 'accept',
|
|
|
|
|
'rules' => [],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
'firewall_version_id' => $firewallVersionId,
|
|
|
|
|
'firewall_revision' => $firewallRevision,
|
|
|
|
|
'agent_firewall_policy_digest' => $firewallDigest,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_instance_firewall_versions')->insert([
|
|
|
|
|
'id' => $firewallVersionId,
|
|
|
|
|
'instance_id' => $instanceId,
|
|
|
|
|
'revision' => $firewallRevision,
|
|
|
|
|
'enabled' => 1,
|
|
|
|
|
'policy_digest' => $firewallDigest,
|
|
|
|
|
'request_hash' => str_repeat('e', 64),
|
|
|
|
|
'idempotency_key' => 'matrix-firewall-version',
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'job_id' => $firewallRecord['id'],
|
|
|
|
|
'operation_id' => $firewallRecord['operation_id'],
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'created_at' => $now - 120,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
$vpcFixture = static function (
|
|
|
|
|
string $kind,
|
|
|
|
|
int $vpcId
|
|
|
|
|
) use (
|
|
|
|
|
$insertJob,
|
|
|
|
|
$addFixture,
|
|
|
|
|
$instanceId,
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
$hostId,
|
|
|
|
|
$ownerId,
|
|
|
|
|
$now
|
|
|
|
|
): void {
|
|
|
|
|
$vpcKey = hash('md5', 'recovery-matrix-vpc:' . $vpcId);
|
|
|
|
|
$portKey = hash('md5', 'recovery-matrix-port:' . $vpcId);
|
|
|
|
|
$cidr = '10.6' . $vpcId . '.0.0/24';
|
|
|
|
|
$gateway = '10.6' . $vpcId . '.0.1';
|
|
|
|
|
$privateIp = '10.6' . $vpcId . '.0.2';
|
|
|
|
|
$mac = sprintf('02:00:00:00:00:%02x', $vpcId);
|
|
|
|
|
$revision = $kind === 'vpc_host_remove' ? 6 : 7;
|
|
|
|
|
$network = [
|
|
|
|
|
'vpc_key' => $vpcKey,
|
|
|
|
|
'desired_revision' => $revision,
|
|
|
|
|
'cidr' => $cidr,
|
|
|
|
|
'gateway' => $gateway,
|
|
|
|
|
'transport' => [
|
|
|
|
|
'address_cidr' => '172.16.10.11/24',
|
|
|
|
|
'interface' => 'br-private',
|
|
|
|
|
'mtu' => 1500,
|
|
|
|
|
],
|
|
|
|
|
'local_ports' => [[
|
|
|
|
|
'ip' => $privateIp,
|
|
|
|
|
'mac' => $mac,
|
|
|
|
|
]],
|
|
|
|
|
'peer_routes' => [],
|
|
|
|
|
];
|
|
|
|
|
$digest = CloudVpcProjection::digest($network);
|
|
|
|
|
$portState = match ($kind) {
|
|
|
|
|
'vpc_port_attach' => 'preparing',
|
|
|
|
|
'vpc_port_detach' => 'detaching',
|
|
|
|
|
default => 'attached',
|
|
|
|
|
};
|
|
|
|
|
Capsule::table('cloud_vpcs')->insert([
|
|
|
|
|
'id' => $vpcId,
|
|
|
|
|
'vpc_key' => $vpcKey,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'region_id' => 1,
|
|
|
|
|
'cidr_pool_id' => 1,
|
|
|
|
|
'name' => 'matrix-vpc-' . $vpcId,
|
|
|
|
|
'cidr' => $cidr,
|
|
|
|
|
'prefix_length' => 24,
|
|
|
|
|
'gateway' => $gateway,
|
|
|
|
|
'active_claim' => 1,
|
|
|
|
|
'state' => 'degraded',
|
|
|
|
|
'desired_revision' => $revision,
|
|
|
|
|
'deleted_at' => null,
|
|
|
|
|
'created_at' => $now - 180,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_vpc_ports')->insert([
|
|
|
|
|
'id' => $vpcId,
|
|
|
|
|
'port_key' => $portKey,
|
|
|
|
|
'vpc_id' => $vpcId,
|
|
|
|
|
'instance_id' => $instanceId,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'private_ip' => $privateIp,
|
|
|
|
|
'mac_address' => $mac,
|
|
|
|
|
'active_claim' => 1,
|
|
|
|
|
'state' => $portState,
|
|
|
|
|
'desired_revision' => $revision,
|
|
|
|
|
'applied_revision' => $kind === 'vpc_port_attach'
|
|
|
|
|
? 0 : $revision,
|
|
|
|
|
'last_operation_id' => null,
|
|
|
|
|
'last_error_code' => 'vpc_result_uncertain',
|
|
|
|
|
'attached_at' => $kind === 'vpc_port_attach'
|
|
|
|
|
? null : $now - 300,
|
|
|
|
|
'released_at' => null,
|
|
|
|
|
'created_at' => $now - 180,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
$record = $insertJob(
|
|
|
|
|
$kind,
|
|
|
|
|
static function (int $id, string $operationId) use (
|
|
|
|
|
$kind,
|
|
|
|
|
$vpcId,
|
|
|
|
|
$vpcKey,
|
|
|
|
|
$portKey,
|
|
|
|
|
$cidr,
|
|
|
|
|
$gateway,
|
|
|
|
|
$privateIp,
|
|
|
|
|
$mac,
|
|
|
|
|
$revision,
|
|
|
|
|
$digest,
|
|
|
|
|
$network,
|
|
|
|
|
$instanceUuid,
|
|
|
|
|
$ownerId
|
|
|
|
|
): array {
|
|
|
|
|
$body = match ($kind) {
|
|
|
|
|
'vpc_host_apply' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'desired_digest' => $digest,
|
|
|
|
|
'network' => $network,
|
|
|
|
|
],
|
|
|
|
|
'vpc_host_remove' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'vpc_key' => $vpcKey,
|
|
|
|
|
'expected_revision' => $revision,
|
|
|
|
|
'expected_digest' => $digest,
|
|
|
|
|
],
|
|
|
|
|
default => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'port_key' => $portKey,
|
|
|
|
|
'vpc_key' => $vpcKey,
|
|
|
|
|
'private_ip' => $privateIp,
|
|
|
|
|
'cidr' => $cidr,
|
|
|
|
|
'gateway' => $gateway,
|
|
|
|
|
'mac' => $mac,
|
|
|
|
|
'host_revision' => $revision,
|
|
|
|
|
'host_digest' => $digest,
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
};
|
|
|
|
|
$hostKind = in_array($kind,
|
|
|
|
|
['vpc_host_apply', 'vpc_host_remove'], true);
|
|
|
|
|
return [
|
|
|
|
|
'method' => $kind === 'vpc_port_detach'
|
|
|
|
|
? 'DELETE' : 'POST',
|
|
|
|
|
'request_target' => $hostKind
|
|
|
|
|
? '/v1/vpc/networks/' . $vpcKey
|
|
|
|
|
. ($kind === 'vpc_host_apply'
|
|
|
|
|
? '/apply' : '/remove')
|
|
|
|
|
: '/v1/vms/' . rawurlencode($instanceUuid)
|
|
|
|
|
. '/vpc-port',
|
|
|
|
|
'body' => $body,
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'system_vpc_orchestration' => true,
|
|
|
|
|
'vpc_id' => $vpcId,
|
|
|
|
|
'vpc_port_id' => $vpcId,
|
|
|
|
|
] + ($kind === 'vpc_host_apply'
|
|
|
|
|
? ['host_projection' => $network]
|
|
|
|
|
: ($kind === 'vpc_host_remove'
|
|
|
|
|
? ['host_projection' => null] : []));
|
|
|
|
|
},
|
|
|
|
|
$instanceId,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
'agent',
|
|
|
|
|
$ownerId,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
$vpcId,
|
|
|
|
|
$vpcId
|
|
|
|
|
);
|
|
|
|
|
$hostKind = in_array($kind,
|
|
|
|
|
['vpc_host_apply', 'vpc_host_remove'], true);
|
|
|
|
|
if (!$hostKind) {
|
|
|
|
|
Capsule::table('cloud_vpc_ports')->where('id', $vpcId)->update([
|
|
|
|
|
'last_operation_id' => $record['operation_id'],
|
|
|
|
|
]);
|
|
|
|
|
}
|
|
|
|
|
Capsule::table('cloud_vpc_host_states')->insert([
|
|
|
|
|
'id' => $vpcId,
|
|
|
|
|
'vpc_id' => $vpcId,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'desired_revision' => $revision,
|
|
|
|
|
'applied_revision' => $kind === 'vpc_host_apply'
|
|
|
|
|
? $revision - 1 : $revision,
|
|
|
|
|
'desired_digest' => $digest,
|
|
|
|
|
'applied_digest' => $kind === 'vpc_host_apply'
|
|
|
|
|
? hash('sha256', 'previous-vpc:' . $vpcId) : $digest,
|
|
|
|
|
'sync_state' => $hostKind ? 'unknown' : 'ready',
|
|
|
|
|
'last_operation_id' => $hostKind
|
|
|
|
|
? $record['operation_id'] : null,
|
|
|
|
|
'last_error_code' => $hostKind
|
|
|
|
|
? 'vpc_result_uncertain' : '',
|
|
|
|
|
'last_success_at' => $hostKind ? 0 : $now - 60,
|
|
|
|
|
'last_failure_at' => $hostKind ? $now - 20 : 0,
|
|
|
|
|
'created_at' => $now - 180,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
$projection = match ($kind) {
|
|
|
|
|
'vpc_host_apply' => [
|
|
|
|
|
'vpc_key' => $vpcKey,
|
|
|
|
|
'desired_revision' => $revision,
|
|
|
|
|
'desired_digest' => $digest,
|
|
|
|
|
'bridge' => CloudVpcProjection::bridgeName($vpcKey),
|
|
|
|
|
'applied' => true,
|
|
|
|
|
],
|
|
|
|
|
'vpc_host_remove' => [
|
|
|
|
|
'vpc_key' => $vpcKey,
|
|
|
|
|
'desired_revision' => $revision,
|
|
|
|
|
'desired_digest' => $digest,
|
|
|
|
|
'removed' => true,
|
|
|
|
|
],
|
|
|
|
|
'vpc_port_attach' => [
|
|
|
|
|
'uuid' => $instanceUuid,
|
|
|
|
|
'port_key' => $portKey,
|
|
|
|
|
'vpc_key' => $vpcKey,
|
|
|
|
|
'vpc_port' => [
|
|
|
|
|
'port_key' => $portKey,
|
|
|
|
|
'vpc_key' => $vpcKey,
|
|
|
|
|
'private_ip' => $privateIp,
|
|
|
|
|
'cidr' => $cidr,
|
|
|
|
|
'gateway' => $gateway,
|
|
|
|
|
'mac' => $mac,
|
|
|
|
|
'bridge' => CloudVpcProjection::bridgeName($vpcKey),
|
|
|
|
|
'host_revision' => $revision,
|
|
|
|
|
'host_digest' => $digest,
|
|
|
|
|
],
|
|
|
|
|
'status' => 'running',
|
|
|
|
|
'attached' => true,
|
|
|
|
|
],
|
|
|
|
|
'vpc_port_detach' => [
|
|
|
|
|
'uuid' => $instanceUuid,
|
|
|
|
|
'port_key' => $portKey,
|
|
|
|
|
'vpc_key' => $vpcKey,
|
|
|
|
|
'status' => 'running',
|
|
|
|
|
'removed' => true,
|
|
|
|
|
],
|
|
|
|
|
};
|
|
|
|
|
$action = match ($kind) {
|
|
|
|
|
'vpc_host_apply' => 'apply-vpc-network:' . $vpcKey,
|
|
|
|
|
'vpc_host_remove' => 'remove-vpc-network:' . $vpcKey,
|
|
|
|
|
'vpc_port_attach' => 'apply-vpc-port',
|
|
|
|
|
'vpc_port_detach' => 'remove-vpc-port',
|
|
|
|
|
};
|
|
|
|
|
$addFixture(
|
|
|
|
|
$kind,
|
|
|
|
|
str_replace('vpc_', '', $kind),
|
|
|
|
|
$record,
|
|
|
|
|
$action,
|
|
|
|
|
$hostKind ? 'vpc:' . $vpcKey : $instanceUuid,
|
|
|
|
|
$projection
|
|
|
|
|
);
|
|
|
|
|
};
|
|
|
|
|
$vpcFixture('vpc_host_apply', 1);
|
|
|
|
|
$vpcFixture('vpc_host_remove', 2);
|
|
|
|
|
$vpcFixture('vpc_port_attach', 3);
|
|
|
|
|
$vpcFixture('vpc_port_detach', 4);
|
|
|
|
|
|
|
|
|
|
$instanceFixture(
|
|
|
|
|
'service_suspend',
|
|
|
|
|
'purge',
|
|
|
|
|
'POST',
|
|
|
|
|
'/service/suspend',
|
|
|
|
|
['reason' => 'purge'],
|
|
|
|
|
'service-suspend',
|
|
|
|
|
['uuid' => $instanceUuid, 'state' => 'shutoff']
|
|
|
|
|
);
|
|
|
|
|
$instanceFixture(
|
|
|
|
|
'network_suspend',
|
|
|
|
|
'traffic',
|
|
|
|
|
'POST',
|
|
|
|
|
'/network/suspend',
|
|
|
|
|
['reason' => 'traffic'],
|
|
|
|
|
'network-suspend',
|
|
|
|
|
['network_enabled' => false]
|
|
|
|
|
);
|
|
|
|
|
$instanceFixture(
|
|
|
|
|
'network_resume',
|
|
|
|
|
'resume',
|
|
|
|
|
'POST',
|
|
|
|
|
'/network/resume',
|
|
|
|
|
[],
|
|
|
|
|
'network-resume',
|
|
|
|
|
['network_enabled' => true]
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$insertAggregateParent = static function (
|
|
|
|
|
string $kind,
|
|
|
|
|
int $userId = 41
|
|
|
|
|
) use ($insertJob): array {
|
|
|
|
|
return $insertJob(
|
|
|
|
|
$kind,
|
|
|
|
|
static fn (int $id, string $operationId): array => [
|
|
|
|
|
'aggregate_kind' => $kind,
|
|
|
|
|
],
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
'aggregate',
|
|
|
|
|
$userId
|
|
|
|
|
);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$templateFixture = static function (
|
|
|
|
|
string $kind,
|
|
|
|
|
string $parentKind,
|
|
|
|
|
string $method,
|
|
|
|
|
string $expectedAction,
|
|
|
|
|
array $projection
|
|
|
|
|
) use (
|
|
|
|
|
$insertAggregateParent,
|
|
|
|
|
$insertJob,
|
|
|
|
|
$addFixture,
|
|
|
|
|
$hostId,
|
|
|
|
|
$ownerId,
|
|
|
|
|
$now
|
|
|
|
|
): void {
|
|
|
|
|
$parent = $insertAggregateParent($parentKind);
|
|
|
|
|
$templateId = 'matrix-' . str_replace('_', '-', $kind);
|
|
|
|
|
$revision = 5;
|
|
|
|
|
$generationToken = hash('md5', 'generation:' . $kind);
|
|
|
|
|
$record = $insertJob(
|
|
|
|
|
$kind,
|
|
|
|
|
static function (int $jobId, string $operationId) use (
|
|
|
|
|
$kind,
|
|
|
|
|
$method,
|
|
|
|
|
$templateId,
|
|
|
|
|
$revision,
|
|
|
|
|
$generationToken,
|
|
|
|
|
$ownerId
|
|
|
|
|
): array {
|
|
|
|
|
$target = $kind === 'template_import_replica'
|
|
|
|
|
? '/v1/templates'
|
|
|
|
|
: '/v1/templates/' . rawurlencode($templateId);
|
|
|
|
|
$body = ['operation_id' => $operationId];
|
|
|
|
|
if ($kind === 'template_import_replica') {
|
|
|
|
|
$body += [
|
|
|
|
|
'id' => $templateId,
|
|
|
|
|
'name' => 'Matrix Template',
|
|
|
|
|
'source_url' => 'https://example.invalid/template.qcow2',
|
|
|
|
|
'source_token' => 'matrix-token',
|
|
|
|
|
'source_size_bytes' => 2097152,
|
|
|
|
|
'os_family' => 'linux',
|
|
|
|
|
'version' => '13',
|
|
|
|
|
'architecture' => 'x86_64',
|
|
|
|
|
'sha256' => str_repeat('1', 64),
|
|
|
|
|
'revision' => $revision,
|
|
|
|
|
'generation_token' => $generationToken,
|
|
|
|
|
];
|
|
|
|
|
} elseif ($kind === 'template_metadata_replica') {
|
|
|
|
|
$body += [
|
|
|
|
|
'name' => 'Matrix Template Updated',
|
|
|
|
|
'revision' => $revision,
|
|
|
|
|
'generation_token' => $generationToken,
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
return [
|
|
|
|
|
'method' => $method,
|
|
|
|
|
'request_target' => $target,
|
|
|
|
|
'body' => $body,
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'template_id' => $templateId,
|
|
|
|
|
];
|
|
|
|
|
},
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
$parent['id']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_templates')->insert([
|
|
|
|
|
'id' => $templateId,
|
|
|
|
|
'name' => 'Matrix Template',
|
|
|
|
|
'revision' => $revision,
|
|
|
|
|
'generation_token' => $generationToken,
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
'active_job_id' => $parent['id'],
|
|
|
|
|
'created_at' => $now - 300,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_template_replicas')->insert([
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'template_id' => $templateId,
|
|
|
|
|
'revision' => $revision,
|
|
|
|
|
'generation_token' => $generationToken,
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'image_bytes' => 0,
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
'operation_id' => $record['operation_id'],
|
|
|
|
|
'job_id' => $record['id'],
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'created_at' => $now - 120,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
$addFixture(
|
|
|
|
|
$kind,
|
|
|
|
|
'catalog',
|
|
|
|
|
$record,
|
|
|
|
|
$expectedAction,
|
|
|
|
|
'template:' . $templateId,
|
|
|
|
|
$projection
|
|
|
|
|
);
|
|
|
|
|
};
|
|
|
|
|
$templateFixture(
|
|
|
|
|
'template_import_replica',
|
|
|
|
|
'template_import',
|
|
|
|
|
'POST',
|
|
|
|
|
'import-template',
|
|
|
|
|
[
|
|
|
|
|
'id' => 'matrix-template-import-replica',
|
|
|
|
|
'generation_token' => hash('md5',
|
|
|
|
|
'generation:template_import_replica'),
|
|
|
|
|
'created_at' => '2026-10-21',
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$templateFixture(
|
|
|
|
|
'template_metadata_replica',
|
|
|
|
|
'template_metadata_sync',
|
|
|
|
|
'PATCH',
|
|
|
|
|
'update-template',
|
|
|
|
|
[
|
|
|
|
|
'id' => 'matrix-template-metadata-replica',
|
|
|
|
|
'name' => 'Matrix Template Updated',
|
|
|
|
|
'revision' => 5,
|
|
|
|
|
'generation_token' => hash('md5',
|
|
|
|
|
'generation:template_metadata_replica'),
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$templateFixture(
|
|
|
|
|
'template_delete_replica',
|
|
|
|
|
'template_delete',
|
|
|
|
|
'DELETE',
|
|
|
|
|
'delete-template',
|
|
|
|
|
[
|
|
|
|
|
'resource_type' => 'template',
|
|
|
|
|
'id' => 'matrix-template-delete-replica',
|
|
|
|
|
'deleted' => true,
|
|
|
|
|
'managed_image_absent' => true,
|
|
|
|
|
'managed_metadata_absent' => true,
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$rescueCatalogFixture = static function (string $kind) use (
|
|
|
|
|
$insertJob,
|
|
|
|
|
$addFixture,
|
|
|
|
|
$hostId,
|
|
|
|
|
$ownerId,
|
|
|
|
|
$now
|
|
|
|
|
): void {
|
|
|
|
|
$id = hash('md5', 'matrix-' . $kind);
|
|
|
|
|
$generation = hash('md5', 'generation:' . $kind);
|
|
|
|
|
$sha256 = hash('sha256', 'content:' . $kind);
|
|
|
|
|
$bytes = 2097152;
|
|
|
|
|
$record = $insertJob(
|
|
|
|
|
$kind,
|
|
|
|
|
static fn (int $jobId, string $operationId): array => [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/rescue-templates',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'id' => $id,
|
|
|
|
|
'name' => 'Matrix Global Rescue',
|
|
|
|
|
'source_url' => 'https://example.invalid/rescue.iso',
|
|
|
|
|
'source_size_bytes' => $bytes,
|
|
|
|
|
'sha256' => $sha256,
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
'generation_token' => $generation,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'rescue_template_id' => $id,
|
|
|
|
|
'replica_sync' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_rescue_templates')->insert([
|
|
|
|
|
'id' => $id,
|
|
|
|
|
'name' => 'Matrix Global Rescue',
|
|
|
|
|
'size_bytes' => $bytes,
|
|
|
|
|
'sha256' => $sha256,
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
'generation_token' => $generation,
|
|
|
|
|
'state' => $kind === 'rescue_catalog_validate_replica'
|
|
|
|
|
? 'validating' : 'degraded',
|
|
|
|
|
'active_job_id' => $kind
|
|
|
|
|
=== 'rescue_catalog_validate_replica' ? $record['id'] : null,
|
|
|
|
|
'created_at' => $now - 300,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_rescue_template_replicas')->insert([
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'template_id' => $id,
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
'generation_token' => $generation,
|
|
|
|
|
'state' => 'import_unknown',
|
|
|
|
|
'size_bytes' => 0,
|
|
|
|
|
'sha256' => $sha256,
|
|
|
|
|
'operation_id' => $record['operation_id'],
|
|
|
|
|
'job_id' => $record['id'],
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'created_at' => $now - 120,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
$addFixture(
|
|
|
|
|
$kind,
|
|
|
|
|
'catalog',
|
|
|
|
|
$record,
|
|
|
|
|
'import-rescue-template',
|
|
|
|
|
'rescue-template:' . $id,
|
|
|
|
|
[
|
|
|
|
|
'id' => $id,
|
|
|
|
|
'name' => 'Matrix Global Rescue',
|
|
|
|
|
'size_bytes' => $bytes,
|
|
|
|
|
'sha256' => $sha256,
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
'generation_token' => $generation,
|
|
|
|
|
'created_at' => '2026-10-21',
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
};
|
|
|
|
|
$rescueCatalogFixture('rescue_catalog_validate_replica');
|
|
|
|
|
$rescueCatalogFixture('rescue_catalog_import_replica');
|
|
|
|
|
|
|
|
|
|
$rescueCatalogDeleteId = hash(
|
|
|
|
|
'md5',
|
|
|
|
|
'matrix-rescue-catalog-delete'
|
|
|
|
|
);
|
|
|
|
|
$rescueCatalogDeleteGeneration = hash(
|
|
|
|
|
'md5',
|
|
|
|
|
'generation:rescue-catalog-delete'
|
|
|
|
|
);
|
|
|
|
|
$rescueCatalogDelete = $insertJob(
|
|
|
|
|
'rescue_catalog_delete_replica',
|
|
|
|
|
static fn (int $jobId, string $operationId): array => [
|
|
|
|
|
'method' => 'DELETE',
|
|
|
|
|
'request_target' => '/v1/rescue-templates/'
|
|
|
|
|
. rawurlencode($rescueCatalogDeleteId),
|
|
|
|
|
'body' => ['operation_id' => $operationId],
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'rescue_template_id' => $rescueCatalogDeleteId,
|
|
|
|
|
'revision' => 2,
|
|
|
|
|
'generation_token' => $rescueCatalogDeleteGeneration,
|
|
|
|
|
'retirement_cleanup' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_rescue_templates')->insert([
|
|
|
|
|
'id' => $rescueCatalogDeleteId,
|
|
|
|
|
'name' => 'Matrix Retiring Rescue',
|
|
|
|
|
'size_bytes' => 2097152,
|
|
|
|
|
'sha256' => hash('sha256', 'retiring-rescue'),
|
|
|
|
|
'revision' => 2,
|
|
|
|
|
'generation_token' => $rescueCatalogDeleteGeneration,
|
|
|
|
|
'state' => 'retiring',
|
|
|
|
|
'active_job_id' => null,
|
|
|
|
|
'created_at' => $now - 300,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_rescue_template_replicas')->insert([
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'template_id' => $rescueCatalogDeleteId,
|
|
|
|
|
'revision' => 2,
|
|
|
|
|
'generation_token' => $rescueCatalogDeleteGeneration,
|
|
|
|
|
'state' => 'delete_unknown',
|
|
|
|
|
'size_bytes' => 2097152,
|
|
|
|
|
'sha256' => hash('sha256', 'retiring-rescue'),
|
|
|
|
|
'operation_id' => $rescueCatalogDelete['operation_id'],
|
|
|
|
|
'job_id' => $rescueCatalogDelete['id'],
|
|
|
|
|
'error_code' => 'rescue_replica_delete_unknown',
|
|
|
|
|
'created_at' => $now - 120,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
$addFixture(
|
|
|
|
|
'rescue_catalog_delete_replica',
|
|
|
|
|
'catalog',
|
|
|
|
|
$rescueCatalogDelete,
|
|
|
|
|
'delete-rescue-template',
|
|
|
|
|
'rescue-template:' . $rescueCatalogDeleteId,
|
|
|
|
|
[
|
|
|
|
|
'resource_type' => 'rescue_template',
|
|
|
|
|
'id' => $rescueCatalogDeleteId,
|
|
|
|
|
'deleted' => true,
|
|
|
|
|
'managed_image_absent' => true,
|
|
|
|
|
'managed_metadata_absent' => true,
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$isoFixture = static function (
|
|
|
|
|
string $kind,
|
|
|
|
|
string $parentKind,
|
|
|
|
|
string $method,
|
|
|
|
|
string $expectedAction,
|
|
|
|
|
array $projection
|
|
|
|
|
) use (
|
|
|
|
|
$insertAggregateParent,
|
|
|
|
|
$insertJob,
|
|
|
|
|
$addFixture,
|
|
|
|
|
$hostId,
|
|
|
|
|
$ownerId,
|
|
|
|
|
$now
|
|
|
|
|
): void {
|
|
|
|
|
$parent = $insertAggregateParent($parentKind, $ownerId);
|
|
|
|
|
$isoId = hash('md5', 'matrix-iso:' . $kind);
|
|
|
|
|
$record = $insertJob(
|
|
|
|
|
$kind,
|
|
|
|
|
static function (int $jobId, string $operationId) use (
|
|
|
|
|
$kind,
|
|
|
|
|
$method,
|
|
|
|
|
$isoId,
|
|
|
|
|
$ownerId
|
|
|
|
|
): array {
|
|
|
|
|
$target = $kind === 'user_iso_import_replica'
|
|
|
|
|
? '/v1/user-isos'
|
|
|
|
|
: '/v1/user-isos/' . rawurlencode($isoId)
|
|
|
|
|
. '?owner_id=' . $ownerId;
|
|
|
|
|
$body = ['operation_id' => $operationId];
|
|
|
|
|
if ($kind === 'user_iso_import_replica') {
|
|
|
|
|
$body += [
|
|
|
|
|
'id' => $isoId,
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'name' => 'Matrix ISO',
|
|
|
|
|
'source_url' => 'https://example.invalid/user.iso',
|
|
|
|
|
'source_token' => 'matrix-token',
|
|
|
|
|
'sha256' => str_repeat('3', 64),
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
return [
|
|
|
|
|
'method' => $method,
|
|
|
|
|
'request_target' => $target,
|
|
|
|
|
'body' => $body,
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => false,
|
|
|
|
|
'user_iso_id' => $isoId,
|
|
|
|
|
];
|
|
|
|
|
},
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
$parent['id'],
|
|
|
|
|
null,
|
|
|
|
|
'agent',
|
|
|
|
|
$ownerId
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_user_isos')->insert([
|
|
|
|
|
'id' => $isoId,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'name' => 'Matrix ISO',
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
'active_job_id' => $parent['id'],
|
|
|
|
|
'created_at' => $now - 300,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_user_iso_replicas')->insert([
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'user_iso_id' => $isoId,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'size_bytes' => 0,
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
'operation_id' => $record['operation_id'],
|
|
|
|
|
'job_id' => $record['id'],
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'created_at' => $now - 120,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
$addFixture(
|
|
|
|
|
$kind,
|
|
|
|
|
'catalog',
|
|
|
|
|
$record,
|
|
|
|
|
$expectedAction,
|
|
|
|
|
'user-iso:' . $ownerId . ':' . $isoId,
|
|
|
|
|
$projection
|
|
|
|
|
);
|
|
|
|
|
};
|
|
|
|
|
$isoFixture(
|
|
|
|
|
'user_iso_import_replica',
|
|
|
|
|
'user_iso_import',
|
|
|
|
|
'POST',
|
|
|
|
|
'import-user-iso',
|
|
|
|
|
['id' => hash('md5', 'matrix-iso:user_iso_import_replica'),
|
|
|
|
|
'created_at' => '2026-10-21']
|
|
|
|
|
);
|
|
|
|
|
$isoDeleteId = hash('md5', 'matrix-iso:user_iso_delete_replica');
|
|
|
|
|
$isoFixture(
|
|
|
|
|
'user_iso_delete_replica',
|
|
|
|
|
'user_iso_delete',
|
|
|
|
|
'DELETE',
|
|
|
|
|
'delete-user-iso',
|
|
|
|
|
[
|
|
|
|
|
'resource_type' => 'user_iso',
|
|
|
|
|
'id' => $isoDeleteId,
|
|
|
|
|
'deleted' => true,
|
|
|
|
|
'managed_image_absent' => true,
|
|
|
|
|
'managed_metadata_absent' => true,
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// Dedicated create/storage identities are covered without admitting them
|
|
|
|
|
// to the generic worker/finalizer matrix.
|
|
|
|
|
$dedicatedFixtures = [];
|
|
|
|
|
$restoreBackupId = 203;
|
|
|
|
|
$restoreId = 301;
|
|
|
|
|
$restoreInstanceId = 2;
|
|
|
|
|
$restoreReservationId = 401;
|
|
|
|
|
$restoreIpId = 501;
|
|
|
|
|
$restoreBackupKey = hash('md5', 'recovery-matrix-restore-backup');
|
|
|
|
|
$restoreKey = hash('md5', 'recovery-matrix-restore');
|
|
|
|
|
$restoreUuid = '923e4567-e89b-42d3-a456-426614174001';
|
|
|
|
|
$restoreName = 'matrix-restored-vm';
|
|
|
|
|
$restoreIp = '203.0.113.31';
|
|
|
|
|
$restorePoolKey = str_repeat('7', 32);
|
|
|
|
|
$restoreAttachmentId = 7202;
|
|
|
|
|
$restorePublicPoolId = 7302;
|
|
|
|
|
$restoreNetworkSelector = 'bridge:restore-wan';
|
|
|
|
|
$restoreSha256 = str_repeat('8', 64);
|
|
|
|
|
$restoreSpec = [
|
|
|
|
|
'managed' => true,
|
|
|
|
|
'os_source' => 'cloud_template',
|
|
|
|
|
'os_name' => 'Debian 12',
|
|
|
|
|
'login_name' => 'root',
|
|
|
|
|
'vcpu' => 2,
|
|
|
|
|
'memory_mib' => 2048,
|
|
|
|
|
'disk_gib' => 40,
|
|
|
|
|
'bandwidth_mbps' => 100,
|
|
|
|
|
'network' => $restoreNetworkSelector,
|
|
|
|
|
'public_pool_key' => $restorePoolKey,
|
|
|
|
|
'public_ip_cidr' => $restoreIp . '/24',
|
|
|
|
|
'debian_12_network_fix' => true,
|
|
|
|
|
];
|
|
|
|
|
$restoreRecord = $insertJob(
|
|
|
|
|
'backup_restore',
|
|
|
|
|
static function (int $jobId, string $operationId) use (
|
|
|
|
|
$restoreUuid,
|
|
|
|
|
$restoreName,
|
|
|
|
|
$restoreIp,
|
|
|
|
|
$restorePoolKey,
|
|
|
|
|
$restoreBackupKey,
|
|
|
|
|
$restoreKey,
|
|
|
|
|
$restoreReservationId,
|
|
|
|
|
$restoreSha256,
|
|
|
|
|
$restoreNetworkSelector,
|
|
|
|
|
$ownerId
|
|
|
|
|
): array {
|
|
|
|
|
return [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/'
|
|
|
|
|
. rawurlencode($restoreUuid) . '/restore-backup',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'uuid' => $restoreUuid,
|
|
|
|
|
'source' => '',
|
|
|
|
|
'name' => $restoreName,
|
|
|
|
|
'description' => 'Recovery matrix restore',
|
|
|
|
|
'vcpu' => 2,
|
|
|
|
|
'memory_mib' => 2048,
|
|
|
|
|
'disk_gib' => 40,
|
|
|
|
|
'bandwidth_mbps' => 100,
|
|
|
|
|
'network' => $restoreNetworkSelector,
|
|
|
|
|
'template_id' => 'debian-12',
|
|
|
|
|
'template_revision' => 1,
|
|
|
|
|
'template_generation_token' => str_repeat('9', 32),
|
|
|
|
|
'startup_script' => '',
|
|
|
|
|
'hostname' => $restoreName,
|
|
|
|
|
'guest_ssh_public_key' => '',
|
|
|
|
|
'root_password' => '',
|
|
|
|
|
'public_pool_key' => $restorePoolKey,
|
|
|
|
|
'public_pool_cidr' => '203.0.113.0/24',
|
|
|
|
|
'public_ip_cidr' => $restoreIp . '/24',
|
|
|
|
|
'gateway' => '203.0.113.1',
|
|
|
|
|
'dns_servers' => ['1.1.1.1'],
|
|
|
|
|
'platform_network_policy_snapshot' => [
|
|
|
|
|
'schema_version' => 1,
|
|
|
|
|
],
|
|
|
|
|
'debian_12_network_fix' => true,
|
|
|
|
|
'backup_key' => $restoreBackupKey,
|
|
|
|
|
'source_url' =>
|
|
|
|
|
'https://objects.invalid/matrix-restore.qcow2',
|
|
|
|
|
'source_size_bytes' => 1073741824,
|
|
|
|
|
'source_sha256' => $restoreSha256,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'backup_key' => $restoreBackupKey,
|
|
|
|
|
'restore_key' => $restoreKey,
|
|
|
|
|
'reservation_id' => $restoreReservationId,
|
|
|
|
|
];
|
|
|
|
|
},
|
|
|
|
|
$restoreInstanceId,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
'agent',
|
|
|
|
|
$ownerId,
|
|
|
|
|
$restoreBackupId,
|
|
|
|
|
$restoreId
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_instance_backups')->insert([
|
|
|
|
|
'id' => $restoreBackupId,
|
|
|
|
|
'backup_key' => $restoreBackupKey,
|
|
|
|
|
'source_instance_id' => 999,
|
|
|
|
|
'source_host_id' => $hostId + 1,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'source_uuid' => '123e4567-e89b-42d3-a456-426614174099',
|
|
|
|
|
'name' => 'matrix-restore-source',
|
|
|
|
|
'state' => 'ready',
|
|
|
|
|
'consistency' => 'crash',
|
|
|
|
|
'object_key' => 'vm-backups/' . $restoreBackupKey . '/system.qcow2',
|
|
|
|
|
'object_size_bytes' => 1073741824,
|
|
|
|
|
'virtual_size_bytes' => 42949672960,
|
|
|
|
|
'sha256' => $restoreSha256,
|
|
|
|
|
'spec_json' => json_encode($restoreSpec, JSON_THROW_ON_ERROR),
|
|
|
|
|
'sealed_storage_runtime' => 'sealed-runtime',
|
|
|
|
|
'sealed_credential' => 'sealed-credential',
|
|
|
|
|
'storage_revision' => 1,
|
|
|
|
|
'active_job_id' => null,
|
|
|
|
|
'last_operation_id' => null,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'ready_at' => $now - 300,
|
|
|
|
|
'created_at' => $now - 600,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_instances')->insert([
|
|
|
|
|
'id' => $restoreInstanceId,
|
|
|
|
|
'uuid' => $restoreUuid,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'reservation_id' => $restoreReservationId,
|
|
|
|
|
'name' => $restoreName,
|
|
|
|
|
'status' => 'provisioning',
|
|
|
|
|
'service_status' => 'exempt',
|
|
|
|
|
'network_status' => 'pending',
|
|
|
|
|
'spec_json' => json_encode($restoreSpec, JSON_THROW_ON_ERROR),
|
|
|
|
|
'sealed_credential' => 'sealed-credential',
|
|
|
|
|
'created_at' => $now - 180,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_host_network_attachments')->insert([
|
|
|
|
|
'id' => $restoreAttachmentId,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'line_id' => $migrationLineId,
|
|
|
|
|
'network_selector' => $restoreNetworkSelector,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_public_ip_pools')->insert([
|
|
|
|
|
'id' => $restorePublicPoolId,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'network_attachment_id' => $restoreAttachmentId,
|
|
|
|
|
'pool_key' => $restorePoolKey,
|
|
|
|
|
'cidr' => '203.0.113.0/24',
|
|
|
|
|
'gateway' => '203.0.113.1',
|
|
|
|
|
'dns_servers_json' => '["1.1.1.1"]',
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_public_ips')->insert([
|
|
|
|
|
'id' => $restoreIpId,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'public_ip_pool_id' => $restorePublicPoolId,
|
|
|
|
|
'address' => $restoreIp,
|
|
|
|
|
'prefix_length' => 24,
|
|
|
|
|
'state' => 'reserved',
|
|
|
|
|
'reservation_id' => $restoreReservationId,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'operation_id' => $restoreRecord['operation_id'],
|
|
|
|
|
'created_at' => $now - 180,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_resource_reservations')->insert([
|
|
|
|
|
'id' => $restoreReservationId,
|
|
|
|
|
'backup_restore_id' => $restoreId,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'resource_pool_id' => $migrationResourcePoolId,
|
|
|
|
|
'network_attachment_id' => $restoreAttachmentId,
|
|
|
|
|
'public_ip_pool_id' => $restorePublicPoolId,
|
|
|
|
|
'operation_id' => $restoreRecord['operation_id'],
|
|
|
|
|
'public_ip_id' => $restoreIpId,
|
|
|
|
|
'instance_id' => $restoreInstanceId,
|
|
|
|
|
'cpu_cores' => 2,
|
|
|
|
|
'memory_mib' => 2048,
|
|
|
|
|
'disk_gib' => 40,
|
|
|
|
|
'state' => 'held',
|
|
|
|
|
'created_at' => $now - 180,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_backup_restores')->insert([
|
|
|
|
|
'id' => $restoreId,
|
|
|
|
|
'restore_key' => $restoreKey,
|
|
|
|
|
'backup_id' => $restoreBackupId,
|
|
|
|
|
'target_host_id' => $hostId,
|
|
|
|
|
'target_instance_id' => $restoreInstanceId,
|
|
|
|
|
'reservation_id' => $restoreReservationId,
|
|
|
|
|
'job_id' => $restoreRecord['id'],
|
|
|
|
|
'operation_id' => $restoreRecord['operation_id'],
|
|
|
|
|
'new_uuid' => $restoreUuid,
|
|
|
|
|
'new_name' => $restoreName,
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'created_at' => $now - 180,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
$dedicatedFixtures['backup_restore'] = $restoreRecord + [
|
|
|
|
|
'expected_action' => 'restore-backup:' . $restoreBackupKey,
|
|
|
|
|
'expected_resource' => $restoreUuid,
|
|
|
|
|
];
|
|
|
|
|
$createPublicNetwork = [
|
|
|
|
|
'contract' => 1,
|
|
|
|
|
'network_selector' => 'bridge:br0',
|
|
|
|
|
'ipv4' => [
|
|
|
|
|
'pool_key' => str_repeat('5', 32),
|
|
|
|
|
'pool_cidr' => '8.8.8.0/24',
|
|
|
|
|
'address_cidr' => '8.8.8.30/24',
|
|
|
|
|
'gateway' => '8.8.8.1',
|
|
|
|
|
'gateway_on_link' => false,
|
|
|
|
|
],
|
|
|
|
|
'ipv6' => [
|
|
|
|
|
'pool_key' => str_repeat('6', 32),
|
|
|
|
|
'pool_cidr' => '2404:c140:156:3a::/64',
|
|
|
|
|
'address_cidr' => '2404:c140:156:3a::30/64',
|
|
|
|
|
'gateway' => '2404:c140:156::1',
|
|
|
|
|
'gateway_on_link' => true,
|
|
|
|
|
],
|
|
|
|
|
'dns_servers' => ['1.1.1.1', '2606:4700:4700::1111'],
|
|
|
|
|
];
|
|
|
|
|
$createRecord = $insertJob(
|
|
|
|
|
'create',
|
|
|
|
|
static fn (int $jobId, string $operationId): array => [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'source' => 'cloud',
|
|
|
|
|
'name' => 'matrix-create',
|
|
|
|
|
'vcpu' => 2,
|
|
|
|
|
'memory_mib' => 2048,
|
|
|
|
|
'disk_gib' => 40,
|
|
|
|
|
'bandwidth_mbps' => 100,
|
|
|
|
|
'template_id' => 'debian-13',
|
|
|
|
|
'template_revision' => 1,
|
|
|
|
|
'template_generation_token' => str_repeat('4', 32),
|
|
|
|
|
'root_password' => 'MatrixPassword!123',
|
|
|
|
|
'public_network' => $createPublicNetwork,
|
|
|
|
|
'platform_network_policy_snapshot' => [
|
|
|
|
|
'schema_version' => 1,
|
|
|
|
|
],
|
|
|
|
|
],
|
|
|
|
|
'credential' => [
|
|
|
|
|
'password' => 'MatrixPassword!123',
|
|
|
|
|
'login_name' => 'root',
|
|
|
|
|
'auth_mode' => 'password',
|
|
|
|
|
],
|
|
|
|
|
'template_metadata' => [
|
|
|
|
|
'template_id' => 'debian-13',
|
|
|
|
|
'os_source' => 'cloud_template',
|
|
|
|
|
'os_name' => 'Debian 13',
|
|
|
|
|
'login_name' => 'root',
|
|
|
|
|
'debian_12_network_fix' => false,
|
|
|
|
|
],
|
|
|
|
|
],
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
9001
|
|
|
|
|
);
|
|
|
|
|
$createReservationId = 9101;
|
|
|
|
|
Capsule::table('cloud_orders')->insert([
|
|
|
|
|
'id' => 9001,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'offer_id' => 1,
|
|
|
|
|
'resource_pool_id' => 1,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'reservation_id' => $createReservationId,
|
|
|
|
|
'spec_json' => json_encode([
|
|
|
|
|
'source' => 'cloud',
|
|
|
|
|
'public_network' => $createPublicNetwork,
|
|
|
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES),
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_resource_reservations')->insert([
|
|
|
|
|
'id' => $createReservationId,
|
|
|
|
|
'order_id' => 9001,
|
|
|
|
|
'backup_restore_id' => null,
|
|
|
|
|
'offer_id' => 1,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'resource_pool_id' => 1,
|
|
|
|
|
'network_attachment_id' => null,
|
|
|
|
|
'public_ip_pool_id' => null,
|
|
|
|
|
'operation_id' => $createRecord['operation_id'],
|
|
|
|
|
'public_ip_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'cpu_cores' => 2,
|
|
|
|
|
'memory_mib' => 2048,
|
|
|
|
|
'disk_gib' => 40,
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'created_at' => $now - 180,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
$dedicatedFixtures['create'] = $createRecord + [
|
|
|
|
|
'expected_action' => 'create-vm',
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
$storageBody = [
|
|
|
|
|
'managed_directory' => '/srv/virt/managed',
|
|
|
|
|
'template_directory' => '/srv/virt/templates',
|
|
|
|
|
'seed_directory' => '/srv/virt/seeds',
|
|
|
|
|
'user_iso_directory' => '/srv/virt/user-isos',
|
|
|
|
|
'rescue_directory' => '/srv/virt/rescue',
|
|
|
|
|
];
|
|
|
|
|
$storageRecord = $insertJob(
|
|
|
|
|
'host_storage_configure',
|
|
|
|
|
static fn (int $jobId, string $operationId): array => [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/storage/configuration',
|
|
|
|
|
'body' => $storageBody + ['operation_id' => $operationId],
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'storage_revision' => 3,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_host_storage_configurations')->insert([
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'revision' => 3,
|
|
|
|
|
'managed_directory' => $storageBody['managed_directory'],
|
|
|
|
|
'template_directory' => $storageBody['template_directory'],
|
|
|
|
|
'seed_directory' => $storageBody['seed_directory'],
|
|
|
|
|
'user_iso_directory' => $storageBody['user_iso_directory'],
|
|
|
|
|
'rescue_directory' => $storageBody['rescue_directory'],
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'operation_id' => $storageRecord['operation_id'],
|
|
|
|
|
'job_id' => $storageRecord['id'],
|
|
|
|
|
'created_at' => $now - 120,
|
|
|
|
|
'updated_at' => $now - 20,
|
|
|
|
|
]);
|
|
|
|
|
$dedicatedFixtures['host_storage_configure'] = $storageRecord + [
|
|
|
|
|
'expected_action' => 'configure-storage',
|
|
|
|
|
'expected_resource' => 'host-storage',
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
$classificationOnlyKinds = [
|
|
|
|
|
'payment_create' => 'payment',
|
|
|
|
|
'template_import' => 'aggregate',
|
|
|
|
|
'template_metadata_sync' => 'aggregate',
|
|
|
|
|
'template_delete' => 'aggregate',
|
|
|
|
|
'user_iso_import' => 'aggregate',
|
|
|
|
|
'user_iso_delete' => 'aggregate',
|
|
|
|
|
];
|
|
|
|
|
$genericKinds = array_keys(array_filter(
|
|
|
|
|
$manifestOperations,
|
|
|
|
|
static fn (array $operation): bool =>
|
|
|
|
|
$operation['recovery_mode'] === 'generic'
|
|
|
|
|
));
|
|
|
|
|
$fixtureKinds = array_keys($fixtures);
|
|
|
|
|
sort($genericKinds, SORT_STRING);
|
|
|
|
|
sort($fixtureKinds, SORT_STRING);
|
|
|
|
|
$assert($fixtureKinds === $genericKinds,
|
|
|
|
|
'Every generic recovery kind must have an executable fixture.');
|
|
|
|
|
|
|
|
|
|
$coveredKinds = array_merge(
|
|
|
|
|
$fixtureKinds,
|
|
|
|
|
array_keys($dedicatedFixtures),
|
|
|
|
|
array_keys($classificationOnlyKinds)
|
|
|
|
|
);
|
|
|
|
|
$contractKinds = array_keys($manifestOperations);
|
|
|
|
|
sort($coveredKinds, SORT_STRING);
|
|
|
|
|
sort($contractKinds, SORT_STRING);
|
|
|
|
|
$assert($coveredKinds === $contractKinds,
|
|
|
|
|
'A recovery contract kind was added without a matrix fixture.');
|
|
|
|
|
|
|
|
|
|
$expectedAutomaticKinds = array_keys(array_filter(
|
|
|
|
|
$manifestOperations,
|
|
|
|
|
static fn (array $operation): bool => in_array(
|
|
|
|
|
$operation['recovery_mode'],
|
|
|
|
|
['generic', 'create'],
|
|
|
|
|
true
|
|
|
|
|
)
|
|
|
|
|
));
|
|
|
|
|
$assert(CloudOperationRecoveryPolicy::kinds()
|
|
|
|
|
=== array_keys(array_filter(
|
|
|
|
|
$manifestOperations,
|
|
|
|
|
static fn (array $operation): bool =>
|
|
|
|
|
$operation['recovery_mode'] === 'generic'
|
|
|
|
|
))
|
|
|
|
|
&& CloudOperationRecoveryPolicy::automaticKinds()
|
|
|
|
|
=== $expectedAutomaticKinds,
|
|
|
|
|
'Policy kind lists diverged from the recovery modes.');
|
|
|
|
|
|
|
|
|
|
$projectionMethod = new ReflectionMethod(
|
|
|
|
|
CloudAdminOperationRecoveryService::class,
|
|
|
|
|
'effectProjection'
|
|
|
|
|
);
|
|
|
|
|
$expectInvalidProjection = static function (
|
|
|
|
|
array $projection,
|
|
|
|
|
string $kind,
|
|
|
|
|
string $message
|
|
|
|
|
) use ($projectionMethod, $assert): void {
|
|
|
|
|
try {
|
|
|
|
|
$projectionMethod->invoke(null, $projection, $kind, 'applied');
|
|
|
|
|
} catch (CloudAgentError $error) {
|
|
|
|
|
$assert($error->agentCode === 'agent_invalid_inspection'
|
|
|
|
|
&& $error->agentHttpStatus === 200,
|
|
|
|
|
'Unexpected invalid-projection error for ' . $kind . '.');
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
throw new RuntimeException($message);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$rescueResultMethod = new ReflectionMethod(
|
|
|
|
|
CloudWorkerService::class,
|
|
|
|
|
'assertRescueResultMatchesRequest'
|
|
|
|
|
);
|
|
|
|
|
foreach ($fixtures['rescue'] as $fixture) {
|
|
|
|
|
$rescueResultMethod->invoke(
|
|
|
|
|
null,
|
|
|
|
|
$fixture['job'],
|
|
|
|
|
'succeeded',
|
|
|
|
|
$fixture['projection'],
|
|
|
|
|
$fixture['payload']
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
$wrongRescueGeneration = $fixtures['rescue'][0]['projection'];
|
|
|
|
|
$wrongRescueGeneration['generation_token'] = str_repeat('0', 32);
|
|
|
|
|
try {
|
|
|
|
|
$rescueResultMethod->invoke(
|
|
|
|
|
null,
|
|
|
|
|
$fixtures['rescue'][0]['job'],
|
|
|
|
|
'succeeded',
|
|
|
|
|
$wrongRescueGeneration,
|
|
|
|
|
$fixtures['rescue'][0]['payload']
|
|
|
|
|
);
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
'A rescue result from the wrong catalog generation was accepted.'
|
|
|
|
|
);
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$assert($error->httpStatus === 502,
|
|
|
|
|
'A mismatched rescue result returned the wrong status.');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$upgradeIdentity = CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
|
|
|
$upgradeRecord['job'],
|
|
|
|
|
$upgradeRecord['payload']
|
|
|
|
|
);
|
|
|
|
|
$inspectionMethod = new ReflectionMethod(
|
|
|
|
|
CloudAdminOperationRecoveryService::class,
|
|
|
|
|
'inspection'
|
|
|
|
|
);
|
|
|
|
|
$transferIdentity = CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
|
|
|
$migrationTransfer['job'],
|
|
|
|
|
$migrationTransfer['payload']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_host_network_attachments')
|
|
|
|
|
->where('id', $migrationAttachmentId)->update([
|
|
|
|
|
'network_selector' => 'bridge:tampered',
|
|
|
|
|
]);
|
|
|
|
|
$selectorTamperRejected = false;
|
|
|
|
|
try {
|
|
|
|
|
CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
|
|
|
$migrationTransfer['job'],
|
|
|
|
|
$migrationTransfer['payload']
|
|
|
|
|
);
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$selectorTamperRejected = $error->httpStatus === 409;
|
|
|
|
|
}
|
|
|
|
|
$assert($selectorTamperRejected,
|
|
|
|
|
'PeerMigration recovery identity accepted a changed target attachment selector.');
|
|
|
|
|
Capsule::table('cloud_host_network_attachments')
|
|
|
|
|
->where('id', $migrationAttachmentId)->update([
|
|
|
|
|
'network_selector' => $migrationNetworkSelector,
|
|
|
|
|
'line_id' => $migrationLineId + 1,
|
|
|
|
|
]);
|
|
|
|
|
$lineTamperRejected = false;
|
|
|
|
|
try {
|
|
|
|
|
CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
|
|
|
$migrationTransfer['job'],
|
|
|
|
|
$migrationTransfer['payload']
|
|
|
|
|
);
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$lineTamperRejected = $error->httpStatus === 409;
|
|
|
|
|
}
|
|
|
|
|
$assert($lineTamperRejected,
|
|
|
|
|
'PeerMigration recovery identity accepted an attachment from the wrong line.');
|
|
|
|
|
Capsule::table('cloud_host_network_attachments')
|
|
|
|
|
->where('id', $migrationAttachmentId)->update([
|
|
|
|
|
'line_id' => $migrationLineId,
|
|
|
|
|
]);
|
|
|
|
|
$cleanupInspection = $inspectionMethod->invoke(null, [
|
|
|
|
|
'operation_id' => $migrationTransfer['operation_id'],
|
|
|
|
|
'action' => $transferIdentity['action'],
|
|
|
|
|
'resource_id' => $transferIdentity['resource_id'],
|
|
|
|
|
'request_fingerprint' =>
|
|
|
|
|
$transferIdentity['request_fingerprint'],
|
|
|
|
|
'operation_status' => 'failed',
|
|
|
|
|
'checked_at' => time(),
|
|
|
|
|
'outcome' => 'uncertain',
|
|
|
|
|
'summary_code' => 'cleanup_required',
|
|
|
|
|
'effect_resolution' => 'partial',
|
|
|
|
|
'fence_state' => 'active',
|
|
|
|
|
'fence_resolution_required' => true,
|
|
|
|
|
'abort_job' => ['available' => false],
|
|
|
|
|
'cleanup' => [
|
|
|
|
|
'available' => true,
|
|
|
|
|
'plan_id' => str_repeat('c', 64),
|
|
|
|
|
'steps' => ['delete_managed_volumes'],
|
|
|
|
|
],
|
|
|
|
|
], $migrationTransfer['operation_id'],
|
|
|
|
|
'instance_migration_transfer', true);
|
|
|
|
|
$assert(($cleanupInspection['cleanup']['plan_id'] ?? null)
|
|
|
|
|
=== str_repeat('c', 64),
|
|
|
|
|
'Validated target cleanup evidence was dropped before recovery.');
|
|
|
|
|
$upgradeNotAppliedRejected = false;
|
|
|
|
|
try {
|
|
|
|
|
$inspectionMethod->invoke(null, [
|
|
|
|
|
'operation_id' => $upgradeRecord['operation_id'],
|
|
|
|
|
'action' => $upgradeIdentity['action'],
|
|
|
|
|
'resource_id' => $upgradeIdentity['resource_id'],
|
|
|
|
|
'request_fingerprint' => $upgradeIdentity['request_fingerprint'],
|
|
|
|
|
'operation_status' => 'failed',
|
|
|
|
|
'checked_at' => time(),
|
|
|
|
|
'outcome' => 'failed',
|
|
|
|
|
'summary_code' => 'operation_failed',
|
|
|
|
|
'error_code' => 'resources_not_committed',
|
|
|
|
|
'effect_resolution' => 'not_applied',
|
|
|
|
|
'fence_state' => 'active',
|
|
|
|
|
'fence_resolution_required' => true,
|
|
|
|
|
'abort_job' => ['available' => false],
|
|
|
|
|
'cleanup' => ['available' => false],
|
|
|
|
|
], $upgradeRecord['operation_id'], 'instance_upgrade', true);
|
|
|
|
|
} catch (CloudAgentError $error) {
|
|
|
|
|
$assert($error->agentCode === 'agent_invalid_inspection'
|
|
|
|
|
&& $error->agentHttpStatus === 200,
|
|
|
|
|
'Unexpected fail-closed upgrade inspection error.');
|
|
|
|
|
$upgradeNotAppliedRejected = true;
|
|
|
|
|
}
|
|
|
|
|
$assert($upgradeNotAppliedRejected,
|
|
|
|
|
'Automatic recovery accepted unsafe not-applied upgrade evidence.');
|
|
|
|
|
$malformedUpgradeProjection = $upgradeProjection;
|
|
|
|
|
unset($malformedUpgradeProjection['applied_disk_gib']);
|
|
|
|
|
$expectInvalidProjection(
|
|
|
|
|
$malformedUpgradeProjection,
|
|
|
|
|
'instance_upgrade',
|
|
|
|
|
'Incomplete instance-upgrade evidence crossed the projection boundary.'
|
|
|
|
|
);
|
|
|
|
|
$unrestoredUpgradeProjection = $upgradeProjection;
|
|
|
|
|
$unrestoredUpgradeProjection['power_state_restored'] = false;
|
|
|
|
|
$expectInvalidProjection(
|
|
|
|
|
$unrestoredUpgradeProjection,
|
|
|
|
|
'instance_upgrade',
|
|
|
|
|
'An upgrade without restored power state crossed the projection boundary.'
|
|
|
|
|
);
|
|
|
|
|
$wrongPowerUpgradeProjection = $upgradeProjection;
|
|
|
|
|
$wrongPowerUpgradeProjection['originally_running'] = false;
|
|
|
|
|
$wrongPowerUpgradeProjection['status'] = 'running';
|
|
|
|
|
$expectInvalidProjection(
|
|
|
|
|
$wrongPowerUpgradeProjection,
|
|
|
|
|
'instance_upgrade',
|
|
|
|
|
'An upgrade with the wrong final power state crossed the projection boundary.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$workspaceCleanupFixture = $fixtures['backup_workspace_cleanup'][0];
|
|
|
|
|
Capsule::table('cloud_instances')->where('id', $instanceId)->update([
|
|
|
|
|
'host_id' => $hostId + 1,
|
|
|
|
|
'status' => 'deleted',
|
|
|
|
|
]);
|
|
|
|
|
$migratedCleanupIdentity = CloudOperationRecoveryIdentity::
|
|
|
|
|
expectedFromPayload(
|
|
|
|
|
$workspaceCleanupFixture['job'],
|
|
|
|
|
$workspaceCleanupFixture['payload']
|
|
|
|
|
);
|
|
|
|
|
$assert($migratedCleanupIdentity['resource_id'] === $instanceUuid,
|
|
|
|
|
'Backup workspace cleanup lost its frozen source-host identity after migration.');
|
|
|
|
|
Capsule::table('cloud_instances')->where('id', $instanceId)->update([
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'status' => 'active',
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
foreach ($fixtures as $kind => $kindFixtures) {
|
|
|
|
|
$entry = $manifestOperations[$kind];
|
|
|
|
|
$assert(CloudOperationRecoveryPolicy::supports($kind)
|
|
|
|
|
&& CloudOperationRecoveryPolicy::identityScope($kind)
|
|
|
|
|
=== $entry['identity_scope']
|
|
|
|
|
&& CloudOperationRecoveryIdentity::supports($kind),
|
|
|
|
|
'Generic policy/identity support diverged for ' . $kind . '.');
|
|
|
|
|
|
|
|
|
|
$actualActions = [];
|
|
|
|
|
foreach ($kindFixtures as $fixture) {
|
|
|
|
|
$job = $fixture['job'];
|
|
|
|
|
$payload = $fixture['payload'];
|
|
|
|
|
$fromPayload = CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
|
|
|
$job,
|
|
|
|
|
$payload
|
|
|
|
|
);
|
|
|
|
|
$fromEncryptedJob = CloudOperationRecoveryIdentity::expected($job);
|
|
|
|
|
$label = $kind . ':' . $fixture['variant'];
|
|
|
|
|
$assert($fromPayload === $fromEncryptedJob,
|
|
|
|
|
'Encrypted identity reconstruction diverged for ' . $label);
|
|
|
|
|
$assert($fromPayload['action']
|
|
|
|
|
=== $fixture['expected_action']
|
|
|
|
|
&& $fromPayload['resource_id']
|
|
|
|
|
=== $fixture['expected_resource'],
|
|
|
|
|
'Identity action/resource mismatch for ' . $label);
|
|
|
|
|
$assert(CloudOperationRecoveryPolicy::allowsAction(
|
|
|
|
|
$kind,
|
|
|
|
|
$fromPayload['action']
|
|
|
|
|
),
|
|
|
|
|
'Contract rejected the rebuilt action for ' . $label);
|
|
|
|
|
|
|
|
|
|
$wireBody = $payload['body'];
|
|
|
|
|
if ($kind === 'action'
|
|
|
|
|
&& ($wireBody['action'] ?? null) === 'poweroff') {
|
|
|
|
|
$wireBody['action'] = 'destroy';
|
|
|
|
|
}
|
|
|
|
|
$encodedBody = json_encode(
|
|
|
|
|
$wireBody === [] ? (object) [] : $wireBody,
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE
|
|
|
|
|
);
|
|
|
|
|
$expectedFingerprint = hash(
|
|
|
|
|
'sha256',
|
|
|
|
|
strtoupper($payload['method']) . "\n"
|
|
|
|
|
. $payload['request_target'] . "\n" . $encodedBody
|
|
|
|
|
);
|
|
|
|
|
$assert(hash_equals(
|
|
|
|
|
$expectedFingerprint,
|
|
|
|
|
$fromPayload['request_fingerprint']
|
|
|
|
|
),
|
|
|
|
|
'Wire fingerprint mismatch for ' . $label);
|
|
|
|
|
|
|
|
|
|
$context = $kind === 'instance_settings'
|
|
|
|
|
? CloudWorkerService::settingsRecoveryContext($job)
|
|
|
|
|
: CloudWorkerService::recoveryContext($job);
|
|
|
|
|
$assert($context['payload'] === $payload,
|
|
|
|
|
'Worker lost the encrypted recovery payload for ' . $label);
|
|
|
|
|
if ($entry['identity_scope'] === 'instance') {
|
|
|
|
|
$assert(($context['instance']->id ?? null) === $instanceId,
|
|
|
|
|
'Worker lost the instance binding for ' . $label);
|
|
|
|
|
} else {
|
|
|
|
|
$assert($context['instance'] === null,
|
|
|
|
|
'Catalog recovery unexpectedly acquired an instance for '
|
|
|
|
|
. $label);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$projection = $projectionMethod->invoke(
|
|
|
|
|
null,
|
|
|
|
|
$fixture['projection'],
|
|
|
|
|
$kind,
|
|
|
|
|
'applied'
|
|
|
|
|
);
|
|
|
|
|
$assert($projection === $fixture['projection'],
|
|
|
|
|
'Evidence projection branch is missing for ' . $label);
|
|
|
|
|
$actualActions[] = $fromPayload['action'];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
foreach ($entry['actions'] as $action) {
|
|
|
|
|
$assert(in_array($action, $actualActions, true),
|
|
|
|
|
'Literal action lacks a real identity fixture: '
|
|
|
|
|
. $kind . ':' . $action);
|
|
|
|
|
}
|
|
|
|
|
foreach ($entry['action_patterns'] as $pattern) {
|
|
|
|
|
$matched = array_filter(
|
|
|
|
|
$actualActions,
|
|
|
|
|
static fn (string $action): bool =>
|
|
|
|
|
preg_match('~' . $pattern . '~D', $action) === 1
|
|
|
|
|
);
|
|
|
|
|
$assert($matched !== [],
|
|
|
|
|
'Pattern action lacks a real identity fixture: '
|
|
|
|
|
. $kind . ':' . $pattern);
|
|
|
|
|
}
|
|
|
|
|
foreach ($actualActions as $action) {
|
|
|
|
|
$literal = in_array($action, $entry['actions'], true);
|
|
|
|
|
$pattern = false;
|
|
|
|
|
foreach ($entry['action_patterns'] as $candidate) {
|
|
|
|
|
$pattern = $pattern
|
|
|
|
|
|| preg_match('~' . $candidate . '~D', $action) === 1;
|
|
|
|
|
}
|
|
|
|
|
$assert($literal || $pattern,
|
|
|
|
|
'Fixture produced an uncontracted action: '
|
|
|
|
|
. $kind . ':' . $action);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$snapshotCreateActions = array_column(
|
|
|
|
|
$fixtures['snapshot_create'],
|
|
|
|
|
'expected_action'
|
|
|
|
|
);
|
|
|
|
|
$snapshotKeyPattern = '[a-f0-9]{32}';
|
|
|
|
|
$assert(count($snapshotCreateActions) === 2
|
|
|
|
|
&& count(array_filter(
|
|
|
|
|
$snapshotCreateActions,
|
|
|
|
|
static fn (string $action): bool => preg_match(
|
|
|
|
|
'~^create-snapshot:' . $snapshotKeyPattern
|
|
|
|
|
. ':filesystem$~D',
|
|
|
|
|
$action
|
|
|
|
|
) === 1
|
|
|
|
|
)) === 1
|
|
|
|
|
&& count(array_filter(
|
|
|
|
|
$snapshotCreateActions,
|
|
|
|
|
static fn (string $action): bool => preg_match(
|
|
|
|
|
'~^create-snapshot:' . $snapshotKeyPattern . ':crash$~D',
|
|
|
|
|
$action
|
|
|
|
|
) === 1
|
|
|
|
|
)) === 1,
|
|
|
|
|
'Snapshot create must cover both filesystem and crash consistency.');
|
|
|
|
|
|
|
|
|
|
$validTemplateMetadataProjection =
|
|
|
|
|
$fixtures['template_metadata_replica'][0]['projection'];
|
|
|
|
|
$invalidGeneration = $validTemplateMetadataProjection;
|
|
|
|
|
$invalidGeneration['generation_token'] = 'NOT-32-LOWER-HEX';
|
|
|
|
|
$expectInvalidProjection(
|
|
|
|
|
$invalidGeneration,
|
|
|
|
|
'template_metadata_replica',
|
|
|
|
|
'An invalid template generation token crossed the projection boundary.'
|
|
|
|
|
);
|
|
|
|
|
$nestedToken = $validTemplateMetadataProjection;
|
|
|
|
|
$nestedToken['generation_token'] = [
|
|
|
|
|
'generation_token' => str_repeat('a', 32),
|
|
|
|
|
];
|
|
|
|
|
$expectInvalidProjection(
|
|
|
|
|
$nestedToken,
|
|
|
|
|
'template_metadata_replica',
|
|
|
|
|
'A nested generation token bypassed sensitive-key filtering.'
|
|
|
|
|
);
|
|
|
|
|
$accessToken = $validTemplateMetadataProjection;
|
|
|
|
|
$accessToken['access_token'] = str_repeat('a', 32);
|
|
|
|
|
$expectInvalidProjection(
|
|
|
|
|
$accessToken,
|
|
|
|
|
'template_metadata_replica',
|
|
|
|
|
'An access token bypassed sensitive-key filtering.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
foreach ($dedicatedFixtures as $kind => $fixture) {
|
|
|
|
|
$identity = CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
|
|
|
$fixture['job'],
|
|
|
|
|
$fixture['payload']
|
|
|
|
|
);
|
|
|
|
|
$assert($identity === CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
$fixture['job']
|
|
|
|
|
)
|
|
|
|
|
&& $identity['action'] === $fixture['expected_action']
|
|
|
|
|
&& CloudOperationRecoveryPolicy::allowsAction(
|
|
|
|
|
$kind,
|
|
|
|
|
$identity['action']
|
|
|
|
|
)
|
|
|
|
|
&& !CloudOperationRecoveryPolicy::supports($kind),
|
|
|
|
|
'Dedicated identity contract diverged for ' . $kind . '.');
|
|
|
|
|
if ($kind === 'create') {
|
|
|
|
|
$assert(CloudOperationRecoveryPolicy::identityScope($kind)
|
|
|
|
|
=== 'create'
|
|
|
|
|
&& preg_match(
|
|
|
|
|
'/^[a-f0-9]{8}-[a-f0-9]{4}-5[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/D',
|
|
|
|
|
$identity['resource_id']
|
|
|
|
|
) === 1,
|
|
|
|
|
'Create identity lost its deterministic UUID scope.');
|
|
|
|
|
$tamperedCreate = $fixture['payload'];
|
|
|
|
|
$tamperedCreate['body']['public_network']['ipv6'][
|
|
|
|
|
'address_cidr'
|
|
|
|
|
] = '2404:c140:156:3a::31/64';
|
|
|
|
|
$tamperRejected = false;
|
|
|
|
|
try {
|
|
|
|
|
CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
|
|
|
$fixture['job'],
|
|
|
|
|
$tamperedCreate
|
|
|
|
|
);
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$tamperRejected = $error->httpStatus === 409;
|
|
|
|
|
}
|
|
|
|
|
$assert($tamperRejected,
|
|
|
|
|
'Create recovery accepted a different dual-stack snapshot.');
|
|
|
|
|
$projection = $projectionMethod->invoke(null, [
|
|
|
|
|
'creation_absent' => true,
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
], 'create', 'not_applied');
|
|
|
|
|
$assert(($projection['creation_absent'] ?? null) === true
|
|
|
|
|
&& ($projection['managed_artifacts_absent'] ?? null)
|
|
|
|
|
=== true,
|
|
|
|
|
'Create absence projection is no longer complete.');
|
|
|
|
|
$expectStateConflict(static fn () =>
|
|
|
|
|
CloudWorkerService::recoveryContext($fixture['job']));
|
|
|
|
|
} elseif ($kind === 'backup_restore') {
|
|
|
|
|
$assert(CloudOperationRecoveryPolicy::identityScope($kind)
|
|
|
|
|
=== 'create'
|
|
|
|
|
&& $identity['resource_id']
|
|
|
|
|
=== $fixture['expected_resource'],
|
|
|
|
|
'Backup restore identity lost its target VM scope.');
|
|
|
|
|
$context = CloudWorkerService::recoveryContext($fixture['job']);
|
|
|
|
|
$assert(($context['instance']->id ?? null)
|
|
|
|
|
=== $restoreInstanceId
|
|
|
|
|
&& $context['payload'] === $fixture['payload'],
|
|
|
|
|
'Backup restore lost its fenced recovery context.');
|
|
|
|
|
$projection = $projectionMethod->invoke(null, [
|
|
|
|
|
'creation_absent' => true,
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
], 'backup_restore', 'not_applied');
|
|
|
|
|
$assert(($projection['creation_absent'] ?? null) === true
|
|
|
|
|
&& ($projection['managed_artifacts_absent'] ?? null)
|
|
|
|
|
=== true,
|
|
|
|
|
'Backup restore absence projection is no longer complete.');
|
|
|
|
|
} else {
|
|
|
|
|
$assert(CloudOperationRecoveryPolicy::identityScope($kind)
|
|
|
|
|
=== 'storage'
|
|
|
|
|
&& $identity['resource_id']
|
|
|
|
|
=== $fixture['expected_resource'],
|
|
|
|
|
'Storage identity lost its dedicated scope.');
|
|
|
|
|
$assert($projectionMethod->invoke(
|
|
|
|
|
null,
|
|
|
|
|
['configured' => true],
|
|
|
|
|
'host_storage_configure',
|
|
|
|
|
'applied'
|
|
|
|
|
) === ['configured' => true],
|
|
|
|
|
'Storage projection branch is missing.');
|
|
|
|
|
$expectStateConflict(static fn () =>
|
|
|
|
|
CloudWorkerService::recoveryContext($fixture['job']));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
foreach ($classificationOnlyKinds as $kind => $mode) {
|
|
|
|
|
$entry = $manifestOperations[$kind];
|
|
|
|
|
$assert($entry['recovery_mode'] === $mode
|
|
|
|
|
&& $entry['identity_scope'] === 'none'
|
|
|
|
|
&& $entry['actions'] === []
|
|
|
|
|
&& $entry['action_patterns'] === []
|
|
|
|
|
&& !CloudOperationRecoveryPolicy::supports($kind)
|
|
|
|
|
&& CloudOperationRecoveryPolicy::identityScope($kind) === null
|
|
|
|
|
&& !CloudOperationRecoveryIdentity::supports($kind),
|
|
|
|
|
'Aggregate/payment classification diverged for ' . $kind . '.');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
foreach ($manifestOperations as $kind => $entry) {
|
|
|
|
|
foreach (['applied', 'not_applied', 'partial'] as $resolution) {
|
|
|
|
|
$assert(CloudOperationRecoveryPolicy::allowsSystemResolution(
|
|
|
|
|
$kind,
|
|
|
|
|
$resolution
|
|
|
|
|
) === in_array(
|
|
|
|
|
$resolution,
|
|
|
|
|
$entry['system_resolutions'],
|
|
|
|
|
true
|
|
|
|
|
),
|
|
|
|
|
'System resolution policy diverged for '
|
|
|
|
|
. $kind . ':' . $resolution);
|
|
|
|
|
}
|
|
|
|
|
$assert(!CloudOperationRecoveryPolicy::allowsAction(
|
|
|
|
|
$kind,
|
|
|
|
|
'unlisted-operation'
|
|
|
|
|
),
|
|
|
|
|
'Recovery contract accepted an unlisted action for ' . $kind);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// A source prepare can succeed after its PHP finalizer fails. The exact
|
|
|
|
|
// immutable receipt must support a compensating rollback without replaying
|
|
|
|
|
// prepare or accepting the artifact catalog that caused the incident.
|
|
|
|
|
$abortTargetOperation = str_repeat('9', 32);
|
|
|
|
|
$abortEventId = 'peer-migration-abort-prepare-0001';
|
|
|
|
|
Capsule::table('cloud_instances')->where('id', $instanceId)->update([
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'uuid' => $instanceUuid,
|
|
|
|
|
'reservation_id' => 9003,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_instance_migration_source_debts')
|
|
|
|
|
->where('migration_id', 1)->delete();
|
|
|
|
|
Capsule::table('cloud_resource_reservations')->where('id', 9003)->update([
|
|
|
|
|
'migration_source_debt_id' => null,
|
|
|
|
|
'instance_id' => $instanceId,
|
|
|
|
|
'state' => 'committed',
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
|
|
|
->where('reservation_id', 9003)->update([
|
|
|
|
|
'state' => 'committed',
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_public_ips')->where('reservation_id', 9003)->update([
|
|
|
|
|
'state' => 'assigned',
|
|
|
|
|
'instance_id' => $instanceId,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_instance_migrations')->where('id', 1)->update([
|
|
|
|
|
'target_receive_job_id' => null,
|
|
|
|
|
'firewall_job_id' => null,
|
|
|
|
|
'activate_job_id' => null,
|
|
|
|
|
'rollback_job_id' => null,
|
|
|
|
|
'cleanup_job_id' => null,
|
|
|
|
|
'state' => 'attention_required',
|
|
|
|
|
'recovery_action' => 'reconcile_prepare',
|
|
|
|
|
'retry_generation' => 0,
|
|
|
|
|
'manifest_digest' => '',
|
|
|
|
|
'source_was_running' => null,
|
|
|
|
|
'source_autostart_enabled' => null,
|
|
|
|
|
'error_code' => 'operation_effect_unknown',
|
|
|
|
|
'cutover_at' => null,
|
|
|
|
|
'completed_at' => null,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_backup_restores')->where('id', 9002)->update([
|
|
|
|
|
'reservation_id' => 9004,
|
|
|
|
|
'job_id' => null,
|
|
|
|
|
'operation_id' => $abortTargetOperation,
|
|
|
|
|
'state' => 'reserved',
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_resource_reservations')->where('id', 9004)->update([
|
|
|
|
|
'operation_id' => $abortTargetOperation,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'state' => 'held',
|
|
|
|
|
'released_at' => null,
|
|
|
|
|
'release_reason' => '',
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
|
|
|
->where('reservation_id', 9004)->update([
|
|
|
|
|
'state' => 'reserved',
|
|
|
|
|
'operation_id' => $abortTargetOperation,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_public_ips')->where('reservation_id', 9004)->update([
|
|
|
|
|
'state' => 'reserved',
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'operation_id' => $abortTargetOperation,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_peer_migration_artifacts')->where('id', 1)->update([
|
|
|
|
|
'object_size_bytes' => 0,
|
|
|
|
|
'virtual_size_bytes' => 20 * 1073741824,
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
'state' => 'creating',
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $migrationPrepare['id'])->update([
|
|
|
|
|
'completion_event_id' => $abortEventId,
|
|
|
|
|
'effect_resolution' => 'unknown',
|
|
|
|
|
'effect_resolution_reason' => '',
|
|
|
|
|
'effect_may_have_applied' => 1,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'local_finalization_failed',
|
|
|
|
|
'available_at' => $now + 300,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
$abortPrepareJob = CloudJob::findOrFail($migrationPrepare['id']);
|
|
|
|
|
$abortIdentity = CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
$abortPrepareJob,
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$abortResult = [
|
|
|
|
|
'migration_id' => $migrationKey,
|
|
|
|
|
'manifest_digest' => $migrationDigest,
|
|
|
|
|
'artifacts' => [array_replace($migrationArtifact, [
|
|
|
|
|
// Prove abort does not depend on the ordinary capacity validator.
|
|
|
|
|
'virtual_size_bytes' => 3 * 1073741824,
|
|
|
|
|
])],
|
|
|
|
|
'source_stopped' => true,
|
|
|
|
|
'source_was_running' => true,
|
|
|
|
|
'source_autostart_enabled' => true,
|
|
|
|
|
'source_autostart_disabled' => true,
|
|
|
|
|
];
|
|
|
|
|
$invalidAbortResult = $abortResult;
|
|
|
|
|
$invalidAbortResult['source_stopped'] = false;
|
|
|
|
|
$validateAbortResult = new ReflectionMethod(
|
|
|
|
|
CloudInstanceMigrationService::class,
|
|
|
|
|
'assertAbortPrepareResult'
|
|
|
|
|
);
|
|
|
|
|
try {
|
|
|
|
|
$validateAbortResult->invoke(
|
|
|
|
|
null,
|
|
|
|
|
CloudInstanceMigration::findOrFail(1),
|
|
|
|
|
$invalidAbortResult
|
|
|
|
|
);
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
'Abort prepare accepted a receipt that did not prove source stop.'
|
|
|
|
|
);
|
|
|
|
|
} catch (\App\Services\Cloud\CloudRecoveryReceiptInvalid $error) {
|
|
|
|
|
$assert($error->httpStatus === 409,
|
|
|
|
|
'Invalid abort-prepare evidence returned an unexpected status.');
|
|
|
|
|
}
|
|
|
|
|
$abortWire = [
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => $abortEventId,
|
|
|
|
|
'agent_id' => (string) $hostId,
|
|
|
|
|
'operation_id' => $migrationPrepare['operation_id'],
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'action' => $abortIdentity['action'],
|
|
|
|
|
'resource_id' => $abortIdentity['resource_id'],
|
|
|
|
|
'request_fingerprint' => $abortIdentity['request_fingerprint'],
|
|
|
|
|
'result' => $abortResult,
|
|
|
|
|
];
|
|
|
|
|
$validateCompletion = new ReflectionMethod(
|
|
|
|
|
CloudOperationCompletionService::class,
|
|
|
|
|
'validate'
|
|
|
|
|
);
|
|
|
|
|
$normalizedAbort = $validateCompletion->invoke(null, $abortWire);
|
|
|
|
|
$abortPayloadHash = hash('sha256', json_encode(
|
|
|
|
|
$normalizedAbort,
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE
|
|
|
|
|
));
|
|
|
|
|
$assert(CloudInstanceMigrationService::display(
|
|
|
|
|
CloudInstanceMigration::findOrFail(1)
|
|
|
|
|
)['recovery_actions'] === [],
|
|
|
|
|
'Abort prepare was exposed without an exact success receipt.');
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
|
|
|
|
'event_id' => $abortEventId,
|
|
|
|
|
'host_id' => $hostId,
|
|
|
|
|
'operation_id' => $migrationPrepare['operation_id'],
|
|
|
|
|
'payload_sha256' => $abortPayloadHash,
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result_json' => json_encode([
|
|
|
|
|
'schema' => 'virt_cloud_completion_event',
|
|
|
|
|
'version' => 2,
|
|
|
|
|
'identity' => [
|
|
|
|
|
'action' => $abortIdentity['action'],
|
|
|
|
|
'resource_id' => $abortIdentity['resource_id'],
|
|
|
|
|
'request_fingerprint' =>
|
|
|
|
|
$abortIdentity['request_fingerprint'],
|
|
|
|
|
],
|
|
|
|
|
'cause_code' => null,
|
|
|
|
|
'result' => $abortResult,
|
|
|
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE),
|
|
|
|
|
'disposition' => 'received',
|
|
|
|
|
'received_at' => $now,
|
|
|
|
|
'applied_at' => null,
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
Capsule::table('cloud_instance_migrations')->where('id', 1)->update([
|
|
|
|
|
'target_receive_job_id' => $migrationTransfer['id'],
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudInstanceMigrationService::display(
|
|
|
|
|
CloudInstanceMigration::findOrFail(1)
|
|
|
|
|
)['recovery_actions'] === [],
|
|
|
|
|
'Abort prepare was exposed after a target receive task existed.');
|
|
|
|
|
Capsule::table('cloud_instance_migrations')->where('id', 1)->update([
|
|
|
|
|
'target_receive_job_id' => null,
|
|
|
|
|
]);
|
|
|
|
|
$abortDisplay = CloudInstanceMigrationService::display(
|
|
|
|
|
CloudInstanceMigration::findOrFail(1)
|
|
|
|
|
);
|
|
|
|
|
$assert(($abortDisplay['recovery_actions'] ?? null) === ['abort_prepare'],
|
|
|
|
|
'An exact prepare success receipt did not expose the safe abort exit.');
|
|
|
|
|
$rollback = CloudInstanceMigrationService::recover(
|
|
|
|
|
CloudInstance::findOrFail($instanceId),
|
|
|
|
|
[
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'is_admin' => true,
|
|
|
|
|
'is_super_admin' => true,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
],
|
|
|
|
|
['action' => 'abort_prepare', 'confirm_recovery' => true]
|
|
|
|
|
);
|
|
|
|
|
$abortedMigration = CloudInstanceMigration::findOrFail(1);
|
|
|
|
|
$settledPrepare = CloudJob::findOrFail($migrationPrepare['id']);
|
|
|
|
|
$settledEvent = CloudAgentCompletionEvent::findOrFail($abortEventId);
|
|
|
|
|
$heldReservation = Capsule::table('cloud_resource_reservations')
|
|
|
|
|
->where('id', 9004)->first();
|
|
|
|
|
$heldIp = Capsule::table('cloud_public_ips')->where('id', 9007)->first();
|
|
|
|
|
$rollbackPayload = CloudSecrets::decrypt(
|
|
|
|
|
(string) $rollback->sealed_payload,
|
|
|
|
|
'job-payload'
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $rollback->kind === 'instance_migration_rollback'
|
|
|
|
|
&& !hash_equals(
|
|
|
|
|
(string) $rollback->operation_id,
|
|
|
|
|
(string) $migrationPrepare['operation_id']
|
|
|
|
|
)
|
|
|
|
|
&& (int) $abortedMigration->rollback_job_id === (int) $rollback->id
|
|
|
|
|
&& (string) $abortedMigration->state === 'rollback_running'
|
|
|
|
|
&& (bool) $abortedMigration->source_was_running
|
|
|
|
|
&& (bool) $abortedMigration->source_autostart_enabled
|
|
|
|
|
&& (string) $settledPrepare->outcome === 'failed'
|
|
|
|
|
&& (string) $settledPrepare->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $settledPrepare->finalization_incident
|
|
|
|
|
&& (string) $settledEvent->disposition === 'applied'
|
|
|
|
|
&& ($rollbackPayload['body']['migration_id'] ?? null)
|
|
|
|
|
=== $migrationKey
|
|
|
|
|
&& ($rollbackPayload['body']['source_was_running'] ?? null) === true
|
|
|
|
|
&& ($rollbackPayload['body']['source_autostart_enabled'] ?? null)
|
|
|
|
|
=== true
|
|
|
|
|
&& (string) $heldReservation->state === 'held'
|
|
|
|
|
&& (string) $heldIp->state === 'reserved',
|
|
|
|
|
'Abort prepare did not settle the receipt, queue one rollback, and retain the target fence.');
|
|
|
|
|
$sameRollback = CloudInstanceMigrationService::recover(
|
|
|
|
|
CloudInstance::findOrFail($instanceId),
|
|
|
|
|
[
|
|
|
|
|
'user_id' => $ownerId,
|
|
|
|
|
'is_admin' => true,
|
|
|
|
|
'is_super_admin' => true,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
],
|
|
|
|
|
['action' => 'abort_prepare', 'confirm_recovery' => true]
|
|
|
|
|
);
|
|
|
|
|
$assert((int) $sameRollback->id === (int) $rollback->id,
|
|
|
|
|
'Repeated abort_prepare did not return its existing rollback task.');
|
|
|
|
|
$assert(Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'kind', 'instance_migration_rollback'
|
|
|
|
|
)->where('id', '<>', $migrationRollback['id'])->count() === 1,
|
|
|
|
|
'Abort prepare created more than one compensating rollback task.');
|
|
|
|
|
|
|
|
|
|
// A partial target-receive inspection queues one independent, plan-bound
|
|
|
|
|
// cleanup generation. Re-entering automatic recovery must reuse that
|
|
|
|
|
// durable child rather than replaying the receive or creating duplicates.
|
|
|
|
|
$nextJobId = max(
|
|
|
|
|
$nextJobId,
|
|
|
|
|
(int) Capsule::table('cloud_jobs')->max('id') + 1
|
|
|
|
|
);
|
|
|
|
|
$residueMigrationKey = str_repeat('9', 32);
|
|
|
|
|
$residueTargetUuid = '88888888-8888-4888-8888-888888888888';
|
|
|
|
|
$residueTransfer = $insertJob(
|
|
|
|
|
'instance_migration_transfer',
|
|
|
|
|
static fn (int $id, string $operationId): array => [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/'
|
|
|
|
|
. rawurlencode($residueTargetUuid)
|
|
|
|
|
. '/peer-migration/receive',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'public_network' =>
|
|
|
|
|
$migrationTargetIdentity->publicNetwork()->toArray(),
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => $ownerId,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'maintenance_migration_id' => 2,
|
|
|
|
|
],
|
|
|
|
|
$instanceId,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
'agent',
|
|
|
|
|
$ownerId,
|
|
|
|
|
null,
|
|
|
|
|
9012
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_backup_restores')->insert([
|
|
|
|
|
'id' => 9012,
|
|
|
|
|
'restore_key' => str_repeat('f', 32),
|
|
|
|
|
'backup_id' => null,
|
|
|
|
|
'target_host_id' => $hostId,
|
|
|
|
|
'target_instance_id' => null,
|
|
|
|
|
'reservation_id' => 9004,
|
|
|
|
|
'job_id' => $residueTransfer['id'],
|
|
|
|
|
'operation_id' => $residueTransfer['operation_id'],
|
|
|
|
|
'new_uuid' => $residueTargetUuid,
|
|
|
|
|
'new_name' => 'residue-target',
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'error_code' => 'operation_effect_unknown',
|
|
|
|
|
'ready_at' => null,
|
|
|
|
|
'created_at' => $now - 120,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_instance_migrations')->insert([
|
|
|
|
|
'id' => 2,
|
|
|
|
|
'migration_key' => $residueMigrationKey,
|
|
|
|
|
'instance_id' => $instanceId,
|
|
|
|
|
'restore_id' => 9012,
|
|
|
|
|
'source_host_id' => $hostId,
|
|
|
|
|
'source_uuid' => $instanceUuid,
|
|
|
|
|
'source_reservation_id' => 9003,
|
|
|
|
|
'target_host_id' => $hostId,
|
|
|
|
|
'target_uuid' => $residueTargetUuid,
|
|
|
|
|
'target_reservation_id' => 9004,
|
|
|
|
|
'source_prepare_job_id' => null,
|
|
|
|
|
'target_receive_job_id' => $residueTransfer['id'],
|
|
|
|
|
'firewall_job_id' => null,
|
|
|
|
|
'activate_job_id' => null,
|
|
|
|
|
'rollback_job_id' => null,
|
|
|
|
|
'cleanup_job_id' => null,
|
|
|
|
|
'cleanup_mode' => 'delete',
|
|
|
|
|
'state' => 'attention_required',
|
|
|
|
|
'recovery_action' => 'reconcile_transfer',
|
|
|
|
|
'retry_generation' => 0,
|
|
|
|
|
'source_endpoint' => '["https://192.0.2.10:9443"]',
|
|
|
|
|
'peer_trust_id' => str_repeat('1', 64),
|
|
|
|
|
'manifest_digest' => str_repeat('2', 64),
|
|
|
|
|
'source_was_running' => true,
|
|
|
|
|
'source_autostart_enabled' => true,
|
|
|
|
|
'source_service_status' => 'active',
|
|
|
|
|
'source_network_status' => 'active',
|
|
|
|
|
'error_code' => 'operation_effect_unknown',
|
|
|
|
|
'cutover_at' => null,
|
|
|
|
|
'completed_at' => null,
|
|
|
|
|
'created_at' => $now - 120,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_resource_reservations')->where('id', 9004)
|
|
|
|
|
->update([
|
|
|
|
|
'backup_restore_id' => 9012,
|
|
|
|
|
'operation_id' => $residueTransfer['operation_id'],
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
|
|
|
->where('reservation_id', 9004)->update([
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'operation_id' => $residueTransfer['operation_id'],
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_public_ips')->where('reservation_id', 9004)->update([
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'operation_id' => $residueTransfer['operation_id'],
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
$residueInspection = [
|
|
|
|
|
'cleanup' => [
|
|
|
|
|
'available' => true,
|
|
|
|
|
'plan_id' => str_repeat('3', 64),
|
|
|
|
|
'steps' => ['delete_managed_volumes'],
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
$assert(CloudInstanceMigrationService::ensureTargetResidueCleanup(
|
|
|
|
|
CloudJob::findOrFail($residueTransfer['id']),
|
|
|
|
|
$residueInspection
|
|
|
|
|
), 'A safe target-residue plan did not queue cleanup.');
|
|
|
|
|
$targetCleanup = CloudJob::where(
|
|
|
|
|
'parent_job_id', $residueTransfer['id']
|
|
|
|
|
)->where('kind', 'instance_migration_target_clean')->first();
|
|
|
|
|
$targetCleanupPayload = $targetCleanup === null ? null
|
|
|
|
|
: CloudSecrets::decrypt(
|
|
|
|
|
(string) $targetCleanup->sealed_payload,
|
|
|
|
|
'job-payload'
|
|
|
|
|
);
|
|
|
|
|
$assert($targetCleanup !== null
|
|
|
|
|
&& (int) $targetCleanup->host_id === $hostId
|
|
|
|
|
&& (int) $targetCleanup->instance_id === $instanceId
|
|
|
|
|
&& ($targetCleanupPayload['body']['prior_operation_id'] ?? null)
|
|
|
|
|
=== $residueTransfer['operation_id']
|
|
|
|
|
&& ($targetCleanupPayload['body']['cleanup_plan_id'] ?? null)
|
|
|
|
|
=== str_repeat('3', 64)
|
|
|
|
|
&& str_ends_with(
|
|
|
|
|
(string) ($targetCleanupPayload['request_target'] ?? ''),
|
|
|
|
|
'/peer-migration/cleanup-target'
|
|
|
|
|
), 'Target-residue cleanup did not retain its exact plan and parent.');
|
|
|
|
|
$assert(CloudInstanceMigrationService::ensureTargetResidueCleanup(
|
|
|
|
|
CloudJob::findOrFail($residueTransfer['id']),
|
|
|
|
|
$residueInspection
|
|
|
|
|
) && CloudJob::where(
|
|
|
|
|
'parent_job_id', $residueTransfer['id']
|
|
|
|
|
)->where('kind', 'instance_migration_target_clean')->count() === 1,
|
|
|
|
|
'Repeated recovery created duplicate target-residue cleanups.');
|
|
|
|
|
|
|
|
|
|
// Model the next recovery pass after the independent cleanup succeeds and
|
|
|
|
|
// the original receive inspection proves the target is absent. The normal
|
|
|
|
|
// transfer finalizer must restore every target ledger to one retryable
|
|
|
|
|
// operation identity instead of leaving uncertain capacity or IP claims.
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $targetCleanup->id)->update([
|
|
|
|
|
'phase' => 'terminal',
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'terminal_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $residueTransfer['id'])
|
|
|
|
|
->update([
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'not_applied',
|
|
|
|
|
'effect_resolution_reason' => 'system_inspection',
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
CloudInstanceMigrationService::finishJobLocked(
|
|
|
|
|
CloudJob::findOrFail($residueTransfer['id']),
|
|
|
|
|
'failed',
|
|
|
|
|
null,
|
|
|
|
|
$residueTransfer['payload']
|
|
|
|
|
);
|
|
|
|
|
$repairedMigration = CloudInstanceMigration::findOrFail(2);
|
|
|
|
|
$repairedRestore = Capsule::table('cloud_backup_restores')
|
|
|
|
|
->where('id', 9012)->first();
|
|
|
|
|
$repairedReservation = Capsule::table('cloud_resource_reservations')
|
|
|
|
|
->where('id', 9004)->first();
|
|
|
|
|
$repairedIp = Capsule::table('cloud_public_ips')
|
|
|
|
|
->where('id', 9007)->first();
|
|
|
|
|
$assert((string) $repairedMigration->state === 'attention_required'
|
|
|
|
|
&& (string) $repairedMigration->recovery_action === 'retry_transfer'
|
|
|
|
|
&& (string) $repairedMigration->error_code
|
|
|
|
|
=== 'peer_transfer_failed'
|
|
|
|
|
&& (string) $repairedRestore->state === 'reserved'
|
|
|
|
|
&& (string) $repairedRestore->error_code === 'peer_transfer_failed'
|
|
|
|
|
&& (string) $repairedReservation->state === 'held'
|
|
|
|
|
&& (string) $repairedIp->state === 'reserved'
|
|
|
|
|
&& $repairedIp->instance_id === null,
|
|
|
|
|
'Target cleanup did not repair migration, restore, capacity, and IP ledgers for a safe retry.');
|
|
|
|
|
|
|
|
|
|
echo "Cloud operation recovery matrix tests passed\n";
|
|
|
|
|
}
|