From c8fc5000d3194568791c5d987e988a20184ced95 Mon Sep 17 00:00:00 2001 From: codex Date: Tue, 3 Nov 2026 22:17:10 +0800 Subject: [PATCH] feat: reserve portable restore attempts --- web/app/Command/CloudSetup.php | 22 +- web/app/Models/CloudBackupRestore.php | 1 + web/app/Services/Cloud/CloudBackupService.php | 65 ++ .../Cloud/CloudHostCapacityAdmission.php | 148 +++ ...CloudPortableRestoreAttemptCoordinator.php | 966 ++++++++++++++++++ ...CloudPortableRestoreAttemptReservation.php | 159 +++ .../CloudPortableRestoreBackupEvidence.php | 162 +++ .../Cloud/CloudPortableRestoreTargetPlan.php | 33 + .../CloudPublicNetworkAllocationService.php | 37 +- .../CloudPublicNetworkClaimSetIdentity.php | 59 ++ ...oudReleasedPublicNetworkBundleVerifier.php | 189 ++++ web/sql/cloud.sql | 35 +- web/tests/CloudFreshSchemaContractTest.php | 2 +- ...dPortableRestoreAttemptCoordinatorTest.php | 801 +++++++++++++++ ...licIpPoolReplacementSchemaContractTest.php | 2 +- ...licNetworkTransitionSchemaContractTest.php | 2 +- .../CloudRescueCatalogSchemaContractTest.php | 2 +- ...loudRestoreMigrationLedgerContractTest.php | 53 +- .../CloudTemplateImportIntentContractTest.php | 2 +- .../CloudTrafficPoolSchemaContractTest.php | 2 +- web/tests/CloudVpcSchemaContractTest.php | 2 +- 21 files changed, 2727 insertions(+), 17 deletions(-) create mode 100644 web/app/Services/Cloud/CloudHostCapacityAdmission.php create mode 100644 web/app/Services/Cloud/CloudPortableRestoreAttemptCoordinator.php create mode 100644 web/app/Services/Cloud/CloudPortableRestoreAttemptReservation.php create mode 100644 web/app/Services/Cloud/CloudPortableRestoreBackupEvidence.php create mode 100644 web/app/Services/Cloud/CloudReleasedPublicNetworkBundleVerifier.php create mode 100644 web/tests/CloudPortableRestoreAttemptCoordinatorTest.php diff --git a/web/app/Command/CloudSetup.php b/web/app/Command/CloudSetup.php index 19486238..c37082a0 100644 --- a/web/app/Command/CloudSetup.php +++ b/web/app/Command/CloudSetup.php @@ -8,7 +8,7 @@ use App\Services\Config; /** Initializes only the clean-break PHP business and Agent log schemas. */ final class CloudSetup { - private const SCHEMA_VERSION = 'go-agent-wss-v32-portable-restore-contract'; + private const SCHEMA_VERSION = 'go-agent-wss-v33-portable-restore-attempts'; private const MAIN_TABLES = [ 'cloud_regions', 'cloud_traffic_pool_groups', 'cloud_network_lines', 'cloud_resource_pools', @@ -287,7 +287,9 @@ final class CloudSetup 'deployment_snapshot_id', ], 'cloud_backup_restores' => [ - 'id', 'restore_key', 'backup_id', 'target_host_id', + 'id', 'restore_key', 'backup_id', 'requested_by_user_id', + 'request_idempotency_key', 'request_fingerprint', + 'selected_plan_json', 'selected_plan_sha256', 'target_host_id', 'target_instance_id', 'reservation_id', 'job_id', 'operation_id', 'new_uuid', 'new_name', 'state', 'error_code', 'ready_at', 'created_at', 'updated_at', @@ -295,7 +297,8 @@ final class CloudSetup 'cloud_backup_restore_attempts' => [ 'id', 'attempt_key', 'backup_restore_id', 'generation', 'operation_id', 'job_id', 'reservation_id', 'target_host_id', - 'target_uuid', 'target_name', 'target_network_attachment_id', + 'target_uuid', 'target_name', 'target_plan_json', + 'target_plan_sha256', 'target_network_attachment_id', 'target_identity_json', 'target_identity_sha256', 'target_claim_set_sha256', 'target_public_network_json', 'target_public_network_sha256', 'state', 'effect_resolution', @@ -567,9 +570,13 @@ final class CloudSetup 'cloud_backup_restore_attempts' => [ 'chk_cloud_backup_restore_attempt_generation', 'chk_cloud_backup_restore_attempt_active', + 'chk_cloud_backup_restore_attempt_plan', 'chk_cloud_backup_restore_attempt_state', 'chk_cloud_backup_restore_attempt_shape', ], + 'cloud_backup_restores' => [ + 'chk_cloud_backup_restore_request', + ], 'cloud_instance_migration_source_debts' => [ 'chk_cloud_migration_source_debt_active', 'chk_cloud_migration_source_debt_state', @@ -827,6 +834,7 @@ final class CloudSetup ], 'cloud_backup_restores' => [ 'PRIMARY', 'cloud_backup_restore_key', + 'cloud_backup_restore_request', 'cloud_backup_restore_operation', 'cloud_backup_restore_reservation', 'cloud_backup_restore_instance', 'cloud_backup_restore_backup', @@ -901,6 +909,14 @@ final class CloudSetup * non-unique. Column order is part of the ownership/fencing semantics. */ private const REQUIRED_INDEX_SHAPES = [ + 'cloud_backup_restores' => [ + 'cloud_backup_restore_request' => [ + 'non_unique' => 0, + 'columns' => [ + 'requested_by_user_id', 'request_idempotency_key', + ], + ], + ], 'cloud_backup_restore_attempts' => [ 'cloud_backup_restore_attempt_generation' => [ 'non_unique' => 0, diff --git a/web/app/Models/CloudBackupRestore.php b/web/app/Models/CloudBackupRestore.php index 37e0fca4..e9771c11 100644 --- a/web/app/Models/CloudBackupRestore.php +++ b/web/app/Models/CloudBackupRestore.php @@ -10,6 +10,7 @@ final class CloudBackupRestore extends Model protected $casts = [ 'id' => 'int', 'backup_id' => 'int', + 'requested_by_user_id' => 'int', 'target_host_id' => 'int', 'target_instance_id' => 'int', 'reservation_id' => 'int', diff --git a/web/app/Services/Cloud/CloudBackupService.php b/web/app/Services/Cloud/CloudBackupService.php index f572043c..dce6f059 100644 --- a/web/app/Services/Cloud/CloudBackupService.php +++ b/web/app/Services/Cloud/CloudBackupService.php @@ -3,6 +3,7 @@ namespace App\Services\Cloud; use App\Models\CloudBackupRestore; +use App\Models\CloudBackupRestoreAttempt; use App\Models\CloudBackupRecovery; use App\Models\CloudBackupPeriodEntitlement; use App\Models\CloudBackupSubscription; @@ -13,6 +14,7 @@ use App\Models\CloudInstance; use App\Models\CloudInstanceBackup; use App\Models\CloudInstanceMigration; use App\Models\CloudJob; +use App\Models\CloudResourceReservation; /** PHP-owned catalog and resource ledger for portable system-disk backups. */ final class CloudBackupService @@ -1921,6 +1923,7 @@ final class CloudBackupService ->whereIn('state', ['restoring', 'uncertain'])->exists()) { throw new CloudError('备份仍有恢复任务或未知效果,不能删除', 409); } + self::assertPortableRestoreAttemptsDeletable($backup); if (CloudBackupRecovery::where('backup_id', (int) $backup->id) ->whereIn('state', ['recovering', 'uncertain'])->exists()) { throw new CloudError('备份仍有原实例恢复任务或未知效果,不能删除', 409); @@ -1939,6 +1942,68 @@ final class CloudBackupService } } + private static function assertPortableRestoreAttemptsDeletable( + CloudInstanceBackup $backup + ): void { + $restoreIds = CloudBackupRestore::where( + 'backup_id', + (int) $backup->id + ) + ->pluck('id')->map(static fn ($id): int => (int) $id)->all(); + if ($restoreIds === []) { + return; + } + $attempts = CloudBackupRestoreAttempt::whereIn( + 'backup_restore_id', + $restoreIds + )->orderBy('id')->get(); + foreach ($attempts as $attempt) { + $state = (string) $attempt->state; + $resolution = (string) $attempt->effect_resolution; + if ($attempt->active_attempt !== null + || in_array($state, [ + 'reserved', 'queued', 'awaiting_receipt', + 'effect_unknown', 'finalization_pending', + ], true)) { + throw new CloudError( + '备份仍有恢复尝试或未知效果,不能删除', + 409 + ); + } + if ($state === 'succeeded' && $resolution === 'applied') { + continue; + } + if ($state !== 'failed' || $resolution !== 'not_applied' + || $attempt->effect_resolved_at === null + || $attempt->terminal_at === null) { + throw new CloudError('备份恢复尝试终态不明确,不能删除', 409); + } + $reservation = CloudResourceReservation::where( + 'id', + (int) $attempt->reservation_id + )->where( + 'backup_restore_attempt_key', + (string) $attempt->attempt_key + )->first(); + if (!$reservation instanceof CloudResourceReservation + || (string) $reservation->state !== 'released' + || (int) $reservation->host_id + !== (int) $attempt->target_host_id + || !hash_equals( + (string) $reservation->operation_id, + (string) $attempt->operation_id + ) + || $reservation->released_at === null) { + throw new CloudError('备份恢复尝试仍占用资源,不能删除', 409); + } + CloudReleasedPublicNetworkBundleVerifier::verify( + $attempt, + $reservation, + (int) $backup->user_id + ); + } + } + /** @param array{size_bytes:int,etag:string,version_id:string} $identity */ private static function claimDeleteGeneration(int $backupId, string $previousState, array $identity, int $now, diff --git a/web/app/Services/Cloud/CloudHostCapacityAdmission.php b/web/app/Services/Cloud/CloudHostCapacityAdmission.php new file mode 100644 index 00000000..1b189181 --- /dev/null +++ b/web/app/Services/Cloud/CloudHostCapacityAdmission.php @@ -0,0 +1,148 @@ + $evidence['limits'][$field] + - $evidence['requested'][$field]) { + return false; + } + } + return $evidence['limits']['instance_count'] < 1 + || $evidence['used']['instance_count'] + 1 + <= $evidence['limits']['instance_count']; + } + + public static function evidence(object $policy, array $usage, + array $resources): array + { + if (!self::validUsage($usage)) { + throw new \InvalidArgumentException( + 'Capacity usage evidence is invalid.' + ); + } + foreach (['cpu_cores', 'memory_mib', 'disk_gib'] as $field) { + if (!isset($resources[$field]) || !is_int($resources[$field]) + || $resources[$field] < 1) { + throw new \InvalidArgumentException( + 'Capacity request is invalid.' + ); + } + } + $updatedAt = self::policyInteger( + $policy, + 'capacity_updated_at', + self::policyInteger($policy, 'updated_at', null) + ); + $configuredCpu = self::policyInteger( + $policy, 'configured_cpu_cores' + ); + $configuredMemory = self::policyInteger( + $policy, 'configured_memory_mib' + ); + $configuredDisk = self::policyInteger( + $policy, 'configured_disk_gib' + ); + $safetyCpu = self::policyInteger($policy, 'safety_cpu_cores'); + $safetyMemory = self::policyInteger($policy, 'safety_memory_mib'); + $safetyDisk = self::policyInteger($policy, 'safety_disk_gib'); + $maxInstances = self::policyInteger($policy, 'max_instances'); + if ($updatedAt === null || $updatedAt < 1 + || $configuredCpu === null || $configuredCpu < 1 + || $configuredMemory === null || $configuredMemory < 1 + || $configuredDisk === null || $configuredDisk < 1 + || $safetyCpu === null || $safetyCpu < 0 + || $safetyMemory === null || $safetyMemory < 0 + || $safetyDisk === null || $safetyDisk < 0 + || $maxInstances === null || $maxInstances < 0) { + throw new \InvalidArgumentException( + 'Capacity policy evidence is invalid.' + ); + } + return [ + 'policy_updated_at' => $updatedAt, + 'requested' => [ + 'cpu_cores' => $resources['cpu_cores'], + 'memory_mib' => $resources['memory_mib'], + 'disk_gib' => $resources['disk_gib'], + ], + 'used' => [ + 'instance_count' => (int) ($usage['instance_count'] ?? -1), + 'cpu_cores' => (int) ($usage['cpu_sum'] ?? -1), + 'memory_mib' => (int) ($usage['memory_sum'] ?? -1), + 'disk_gib' => (int) ($usage['disk_sum'] ?? -1), + ], + 'limits' => [ + 'instance_count' => $maxInstances, + 'cpu_cores' => max(0, $configuredCpu - $safetyCpu), + 'memory_mib' => max(0, + $configuredMemory - $safetyMemory), + 'disk_gib' => max(0, $configuredDisk - $safetyDisk), + ], + ]; + } + + private static function validUsage(array $usage): bool + { + if (!array_key_exists('unknown', $usage) + || !is_int($usage['unknown']) || $usage['unknown'] !== 0) { + return false; + } + foreach (['instance_count', 'cpu_sum', 'memory_sum', 'disk_sum'] + as $field) { + if (!array_key_exists($field, $usage) + || !is_int($usage[$field]) || $usage[$field] < 0) { + return false; + } + } + return true; + } + + private static function policyInteger(object $policy, string $field, + ?int $fallback = null): ?int + { + $present = false; + $value = null; + if (method_exists($policy, 'getAttributes')) { + $attributes = $policy->getAttributes(); + if (array_key_exists($field, $attributes)) { + $present = true; + $value = $policy->{$field}; + } + } elseif (property_exists($policy, $field)) { + $present = true; + $value = $policy->{$field}; + } + if (!$present) { + return $fallback; + } + if (is_int($value)) { + return $value; + } + if (is_string($value) && preg_match('/^-?[0-9]+$/D', $value) === 1) { + $normalized = filter_var($value, FILTER_VALIDATE_INT); + return $normalized === false ? null : (int) $normalized; + } + return null; + } +} diff --git a/web/app/Services/Cloud/CloudPortableRestoreAttemptCoordinator.php b/web/app/Services/Cloud/CloudPortableRestoreAttemptCoordinator.php new file mode 100644 index 00000000..badf72d4 --- /dev/null +++ b/web/app/Services/Cloud/CloudPortableRestoreAttemptCoordinator.php @@ -0,0 +1,966 @@ + ['min_range' => 1]] + ); + if ($actorUserId === false) { + throw new CloudError('恢复请求用户身份无效', 403); + } + if (preg_match('/^[A-Za-z0-9_.-]{16,128}$/D', $requestKey) !== 1) { + throw new CloudError('恢复请求幂等身份无效', 409); + } + $targetName = CloudInput::name($targetName); + if (preg_match('/^[a-f0-9]{64}$/D', $planSha256) !== 1 + || !hash_equals($plan->sha256(), $planSha256)) { + throw new CloudError('恢复目标计划身份无效', 409); + } + $planValue = $plan->toArray(); + $constraint = self::bundleConstraint($planValue); + $backupId = (int) $backup->id; + if ($backupId < 1) { + throw new CloudError('备份不存在', 404); + } + $ownerUserId = (int) $backup->user_id; + if ($ownerUserId < 1) { + throw new CloudError('备份缺少资源所有者', 409); + } + $fingerprint = self::requestFingerprint( + (int) $actorUserId, + $ownerUserId, + $planValue['backup'], + $planSha256, + $targetName + ); + + return $backup->getConnection()->transaction(static function () use ( + $actorUserId, + $ownerUserId, + $backupId, + $plan, + $planValue, + $planSha256, + $constraint, + $fingerprint, + $targetName, + $requestKey, + $retryAttemptKey + ): CloudPortableRestoreAttemptReservation { + $userIds = array_values(array_unique([ + (int) $actorUserId, + $ownerUserId, + ])); + sort($userIds, SORT_NUMERIC); + $users = User::whereIn('id', $userIds)->orderBy('id') + ->lockForUpdate()->get()->keyBy('id'); + $requester = $users->get((int) $actorUserId); + $owner = $users->get($ownerUserId); + $lockedBackup = CloudInstanceBackup::where('id', $backupId) + ->lockForUpdate()->first(); + if (!$requester instanceof User + || (int) $requester->enable !== 1) { + throw new CloudError('恢复请求用户不可用', 403); + } + if (!$owner instanceof User + || !$lockedBackup instanceof CloudInstanceBackup + || (int) $lockedBackup->user_id !== $ownerUserId) { + throw new CloudError('备份资源所有者不一致', 409); + } + // Hints only identify the request's own reservation for capacity + // subtraction. They are verified under locks before any result. + $restoreHint = CloudBackupRestore::where( + 'requested_by_user_id', + (int) $actorUserId + )->where('request_idempotency_key', $requestKey)->first(); + $attemptHint = $restoreHint instanceof CloudBackupRestore + ? CloudBackupRestoreAttempt::where( + 'backup_restore_id', + (int) $restoreHint->id + )->orderByDesc('generation')->first() + : null; + if ($restoreHint instanceof CloudBackupRestore) { + self::assertRequestReplay( + $restoreHint, + $lockedBackup, + $plan, + $fingerprint, + (int) $actorUserId, + $targetName + ); + } + $fastReplay = $restoreHint instanceof CloudBackupRestore + && ($retryAttemptKey === null + || !($attemptHint instanceof CloudBackupRestoreAttempt) + || !hash_equals( + (string) $attemptHint->attempt_key, + $retryAttemptKey + )); + if ($fastReplay) { + $host = CloudHost::where( + 'id', + (int) $planValue['target']['host_id'] + )->lockForUpdate()->first(); + if (!$host instanceof CloudHost) { + throw new CloudError('恢复目标宿主机账本缺失', 409); + } + $restore = CloudBackupRestore::where( + 'id', + (int) $restoreHint->id + )->where( + 'requested_by_user_id', + (int) $actorUserId + )->where('request_idempotency_key', $requestKey) + ->lockForUpdate()->first(); + if (!$restore instanceof CloudBackupRestore) { + throw new CloudError('恢复请求幂等账本已经变化', 409); + } + self::assertRequestReplay( + $restore, + $lockedBackup, + $plan, + $fingerprint, + (int) $actorUserId, + $targetName + ); + $attempt = CloudBackupRestoreAttempt::where( + 'backup_restore_id', + (int) $restore->id + )->orderByDesc('generation')->lockForUpdate()->first(); + if (!$attempt instanceof CloudBackupRestoreAttempt) { + throw new CloudError('恢复请求缺少尝试账本', 409); + } + self::assertAttemptPlan($restore, $attempt, $plan); + if ($retryAttemptKey !== null) { + self::assertRetrySuccessor( + $restore, + $attempt, + $retryAttemptKey, + $plan, + $ownerUserId + ); + } + return self::replay( + $restore, + $attempt, + $plan, + $ownerUserId + ); + } + + $backupEvidence = CloudPortableRestoreBackupEvidence::fromBackup( + $lockedBackup + ); + if (self::canonical($backupEvidence->backup()) + !== self::canonical($planValue['backup']) + || self::canonical($backupEvidence->resources()) + !== self::canonical($planValue['resources'])) { + throw new CloudError('备份身份或冻结规格已经变化', 409); + } + $excludedReservationId = $attemptHint + instanceof CloudBackupRestoreAttempt + ? (int) $attemptHint->reservation_id : 0; + $requiresAdmission = true; + + $target = $planValue['target']; + // Match the established placement order exactly. + $region = CloudRegion::where('id', (int) $target['region_id']) + ->lockForUpdate()->first(); + $line = CloudNetworkLine::where('id', (int) $target['line_id']) + ->lockForUpdate()->first(); + $pool = CloudResourcePool::where( + 'id', + (int) $target['resource_pool_id'] + )->lockForUpdate()->first(); + $host = CloudHost::where('id', (int) $target['host_id']) + ->lockForUpdate()->first(); + if (!$region instanceof CloudRegion + || !$line instanceof CloudNetworkLine + || !$pool instanceof CloudResourcePool + || !$host instanceof CloudHost) { + throw new CloudError('恢复目标拓扑账本缺失', 409); + } + if ($requiresAdmission) { + $membership = CloudHostResourcePoolMembership::where( + 'host_id', + (int) $target['host_id'] + )->where( + 'resource_pool_id', + (int) $target['resource_pool_id'] + )->lockForUpdate()->first(); + $policy = CloudHostCapacityPolicy::where( + 'host_id', + (int) $target['host_id'] + )->lockForUpdate()->first(); + $storage = CloudHostStorageConfiguration::where( + 'host_id', + (int) $target['host_id'] + )->lockForUpdate()->first(); + $agent = CloudHostAgent::where( + 'host_id', + (int) $target['host_id'] + )->lockForUpdate()->first(); + self::assertTargetEvidence( + $planValue, + $region, + $line, + $pool, + $host, + $membership, + $storage, + $agent, + $backupEvidence + ); + $reservationRows = CloudResourceReservation::where( + 'host_id', + (int) $target['host_id'] + )->whereIn('state', ['held', 'uncertain', 'committed']) + ->orderBy('id')->lockForUpdate()->get([ + 'id', 'instance_id', 'state', 'cpu_cores', + 'memory_mib', 'disk_gib', 'updated_at', + ]); + $capacityRows = $excludedReservationId > 0 + ? $reservationRows->reject( + static fn ($row): bool => + (int) $row->id === $excludedReservationId + )->values() + : $reservationRows; + $usage = CloudPanelUsageService::forHost( + (int) $target['host_id'], + $capacityRows + ); + if (!CloudHostCapacityAdmission::allows( + $policy, + $usage, + $backupEvidence->capacityResources() + )) { + throw new CloudError('恢复目标容量已经不足', 409); + } + $capacityEvidence = CloudHostCapacityAdmission::evidence( + $policy, + $usage, + $backupEvidence->capacityResources() + ); + if (!($restoreHint instanceof CloudBackupRestore) + && self::canonical($capacityEvidence) + !== self::canonical($planValue['capacity'])) { + throw new CloudError('恢复目标容量计划已经过期', 409); + } + if ($restoreHint instanceof CloudBackupRestore + && !self::sameCapacityContract( + $capacityEvidence, + $planValue['capacity'] + )) { + throw new CloudError('恢复目标容量合同已经变化', 409); + } + self::lockPlannedNetworkCatalog($planValue, $constraint); + } + + $restore = CloudBackupRestore::where( + 'requested_by_user_id', + (int) $actorUserId + )->where('request_idempotency_key', $requestKey) + ->lockForUpdate()->first(); + if (!$restore instanceof CloudBackupRestore) { + if ($retryAttemptKey !== null) { + throw new CloudError('恢复重试的原请求不存在', 409); + } + self::assertTargetNameAvailable( + (int) $target['host_id'], + $targetName, + 0, + 0 + ); + $operationId = self::key(); + $restore = new CloudBackupRestore(); + $restore->restore_key = self::key(); + $restore->backup_id = (int) $lockedBackup->id; + $restore->requested_by_user_id = (int) $actorUserId; + $restore->request_idempotency_key = $requestKey; + $restore->request_fingerprint = $fingerprint; + $restore->selected_plan_json = $plan->json(); + $restore->selected_plan_sha256 = $planSha256; + $restore->target_host_id = (int) $target['host_id']; + $restore->target_instance_id = null; + $restore->reservation_id = null; + $restore->job_id = null; + $restore->operation_id = $operationId; + $restore->new_uuid = self::uuidV4(); + $restore->new_name = $targetName; + $restore->state = 'restoring'; + $restore->error_code = ''; + $restore->ready_at = null; + $restore->created_at = $restore->updated_at = time(); + $restore->save(); + return self::createGeneration( + $restore, + $plan, + $constraint, + $backupEvidence, + $ownerUserId, + 1, + $operationId + ); + } + self::assertRequestReplay( + $restore, + $lockedBackup, + $plan, + $fingerprint, + (int) $actorUserId, + $targetName + ); + $attempt = CloudBackupRestoreAttempt::where( + 'backup_restore_id', + (int) $restore->id + )->orderByDesc('generation')->lockForUpdate()->first(); + if (!$attempt instanceof CloudBackupRestoreAttempt) { + throw new CloudError('恢复请求缺少尝试账本', 409); + } + self::assertAttemptPlan($restore, $attempt, $plan); + if ($retryAttemptKey === null) { + return self::replay( + $restore, + $attempt, + $plan, + $ownerUserId + ); + } + if (!hash_equals( + (string) $attempt->attempt_key, + $retryAttemptKey + )) { + self::assertRetrySuccessor( + $restore, + $attempt, + $retryAttemptKey, + $plan, + $ownerUserId + ); + return self::replay( + $restore, + $attempt, + $plan, + $ownerUserId + ); + } + self::assertTargetNameAvailable( + (int) $target['host_id'], + $targetName, + (int) $attempt->id, + $restore->target_instance_id !== null + ? (int) $restore->target_instance_id : 0 + ); + self::assertRetryable( + $restore, + $attempt, + $retryAttemptKey, + $plan, + $ownerUserId + ); + return self::createGeneration( + $restore, + $plan, + $constraint, + $backupEvidence, + $ownerUserId, + (int) $attempt->generation + 1, + self::key() + ); + }, 3); + } + + private static function createGeneration(CloudBackupRestore $restore, + CloudPortableRestoreTargetPlan $plan, + CloudPublicNetworkBundleConstraint $constraint, + CloudPortableRestoreBackupEvidence $backupEvidence, int $ownerUserId, + int $generation, string $operationId + ): CloudPortableRestoreAttemptReservation + { + $value = $plan->toArray(); + $attemptKey = self::key(); + $now = time(); + $reservation = new CloudResourceReservation(); + $reservation->reservation_key = self::key(); + $reservation->operation_id = $operationId; + $reservation->order_id = null; + $reservation->backup_restore_id = null; + $reservation->backup_restore_attempt_key = $attemptKey; + $reservation->migration_source_debt_id = null; + $reservation->offer_id = null; + $reservation->resource_pool_id = + (int) $value['target']['resource_pool_id']; + $reservation->host_id = (int) $value['target']['host_id']; + $reservation->user_id = $ownerUserId; + $reservation->instance_id = null; + $reservation->network_attachment_id = null; + $reservation->public_ip_pool_id = null; + $reservation->public_ip_id = null; + $resources = $backupEvidence->capacityResources(); + $reservation->cpu_cores = $resources['cpu_cores']; + $reservation->memory_mib = $resources['memory_mib']; + $reservation->disk_gib = $resources['disk_gib']; + $reservation->state = 'held'; + $reservation->expires_at = null; + $reservation->released_at = null; + $reservation->release_reason = ''; + $reservation->created_at = $reservation->updated_at = $now; + $reservation->save(); + + CloudPublicNetworkAllocationService::reservePlannedBundle( + (int) $reservation->host_id, + (int) $value['target']['line_id'], + (int) $reservation->id, + $operationId, + $constraint + ); + $claimSet = CloudPublicNetworkAllocationService::currentClaimSet( + (int) $reservation->id, + 'held' + ); + $frozen = CloudFrozenPublicNetworkIdentity::fromClaimSet( + (string) $restore->new_uuid, + $claimSet + ); + $reservation->network_attachment_id = + $claimSet->networkAttachmentId(); + $reservation->updated_at = time(); + $reservation->save(); + + $attempt = new CloudBackupRestoreAttempt(); + $attempt->attempt_key = $attemptKey; + $attempt->backup_restore_id = (int) $restore->id; + $attempt->generation = $generation; + $attempt->operation_id = $operationId; + $attempt->job_id = null; + $attempt->reservation_id = (int) $reservation->id; + $attempt->target_host_id = (int) $reservation->host_id; + $attempt->target_uuid = (string) $restore->new_uuid; + $attempt->target_name = (string) $restore->new_name; + $attempt->target_plan_json = $plan->json(); + $attempt->target_plan_sha256 = $plan->sha256(); + $attempt->target_network_attachment_id = + $claimSet->networkAttachmentId(); + $attempt->target_identity_json = $frozen->identityJson(); + $attempt->target_identity_sha256 = $frozen->identitySha256(); + $attempt->target_claim_set_sha256 = $frozen->claimSetSha256(); + $attempt->target_public_network_json = + $frozen->publicNetworkJson(); + $attempt->target_public_network_sha256 = + $frozen->publicNetworkSha256(); + $attempt->state = 'reserved'; + $attempt->effect_resolution = 'not_applicable'; + $attempt->active_attempt = 1; + $attempt->last_error_code = ''; + $attempt->effect_resolved_at = null; + $attempt->terminal_at = null; + $attempt->created_at = $attempt->updated_at = $now; + $attempt->save(); + + // These legacy parent columns are an initial projection only. Later + // generations never rewrite them; the attempt row is authoritative. + if ($generation === 1) { + $restore->reservation_id = (int) $reservation->id; + } + $restore->state = 'restoring'; + $restore->error_code = ''; + $restore->updated_at = time(); + $restore->save(); + return CloudPortableRestoreAttemptReservation::active( + $restore, + $attempt, + $reservation, + $claimSet, + $frozen, + $plan + ); + } + + private static function replay(CloudBackupRestore $restore, + CloudBackupRestoreAttempt $attempt, + CloudPortableRestoreTargetPlan $plan, int $ownerUserId + ): CloudPortableRestoreAttemptReservation { + $reservation = $attempt->boundReservationQuery() + ->lockForUpdate()->first(); + if (!$reservation instanceof CloudResourceReservation + || (int) $reservation->host_id !== (int) $attempt->target_host_id + || (int) $reservation->resource_pool_id + !== (int) $plan->toArray()['target']['resource_pool_id'] + || (int) $reservation->user_id + !== $ownerUserId + || !hash_equals( + (string) $reservation->operation_id, + (string) $attempt->operation_id + )) { + throw new CloudError('恢复尝试资源预留身份不一致', 409); + } + if ((string) $attempt->state === 'failed' + && (string) $attempt->effect_resolution === 'not_applied') { + return self::terminalResult( + $restore, + $attempt, + $reservation, + $plan, + $ownerUserId + ); + } + if (!in_array((string) $reservation->state, + ['held', 'uncertain', 'committed'], true)) { + throw new CloudError('恢复尝试资源预留状态不明确', 409); + } + CloudPublicNetworkAllocationService::verifyExistingBundle( + (int) $reservation->host_id, + (int) $plan->toArray()['target']['line_id'], + (int) $reservation->id, + (string) $attempt->operation_id + ); + $claimSet = CloudPublicNetworkAllocationService::currentClaimSet( + (int) $reservation->id, + (string) $reservation->state + ); + try { + $frozen = CloudFrozenPublicNetworkIdentity::fromStored( + (int) $reservation->id, + (int) $attempt->target_host_id, + (string) $attempt->target_uuid, + (int) $attempt->target_network_attachment_id, + (string) $attempt->target_identity_json, + (string) $attempt->target_identity_sha256, + (string) $attempt->target_public_network_json, + (string) $attempt->target_public_network_sha256, + (string) $attempt->target_claim_set_sha256, + $claimSet + ); + } catch (\InvalidArgumentException) { + throw new CloudError('恢复尝试冻结公网身份损坏', 409); + } + return CloudPortableRestoreAttemptReservation::active( + $restore, + $attempt, + $reservation, + $claimSet, + $frozen, + $plan + ); + } + + private static function terminalResult(CloudBackupRestore $restore, + CloudBackupRestoreAttempt $attempt, + CloudResourceReservation $reservation, + CloudPortableRestoreTargetPlan $plan, int $ownerUserId + ): CloudPortableRestoreAttemptReservation { + $ids = CloudReleasedPublicNetworkBundleVerifier::verify( + $attempt, + $reservation, + $ownerUserId + ); + return CloudPortableRestoreAttemptReservation::terminal( + $restore, + $attempt, + $reservation, + $plan, + $ids + ); + } + + private static function assertRetryable(CloudBackupRestore $restore, + CloudBackupRestoreAttempt $attempt, string $priorAttemptKey, + CloudPortableRestoreTargetPlan $plan, int $ownerUserId): void + { + $job = $attempt->job_id === null ? null + : CloudJob::where('id', (int) $attempt->job_id) + ->lockForUpdate()->first(); + if (!hash_equals((string) $attempt->attempt_key, $priorAttemptKey) + || (string) $attempt->state !== 'failed' + || (string) $attempt->effect_resolution !== 'not_applied' + || $attempt->active_attempt !== null + || $attempt->effect_resolved_at === null + || $attempt->terminal_at === null + || !$job instanceof CloudJob + || (int) $job->id !== (int) $attempt->job_id + || (string) $job->kind !== 'backup_restore' + || (int) $job->host_id !== (int) $attempt->target_host_id + || (int) $job->user_id !== $ownerUserId + || (int) $job->backup_restore_id + !== (int) $restore->id + || !hash_equals( + (string) $job->operation_id, + (string) $attempt->operation_id + ) + || (string) $job->phase !== 'terminal' + || (string) $job->outcome !== 'failed' + || (string) $job->effect_resolution !== 'not_applied' + || (int) $job->terminal_at < 1 + || (int) $job->finalization_incident !== 0) { + throw new CloudError('恢复尝试尚无可重试的未生效终态证据', 409); + } + $reservation = $attempt->boundReservationQuery() + ->lockForUpdate()->first(); + if (!$reservation instanceof CloudResourceReservation) { + throw new CloudError('恢复重试缺少原资源预留', 409); + } + self::terminalResult( + $restore, + $attempt, + $reservation, + $plan, + $ownerUserId + ); + } + + private static function assertRetrySuccessor(CloudBackupRestore $restore, + CloudBackupRestoreAttempt $latest, string $priorAttemptKey, + CloudPortableRestoreTargetPlan $plan, int $ownerUserId): void + { + $prior = CloudBackupRestoreAttempt::where( + 'backup_restore_id', + (int) $restore->id + )->where('attempt_key', $priorAttemptKey)->lockForUpdate()->first(); + if (!$prior instanceof CloudBackupRestoreAttempt + || (int) $latest->generation !== (int) $prior->generation + 1) { + throw new CloudError('恢复重试身份与最新尝试不一致', 409); + } + self::assertAttemptPlan($restore, $prior, $plan); + self::assertRetryable( + $restore, + $prior, + $priorAttemptKey, + $plan, + $ownerUserId + ); + } + + private static function assertRequestReplay(CloudBackupRestore $restore, + CloudInstanceBackup $backup, CloudPortableRestoreTargetPlan $plan, + string $fingerprint, int $actorUserId, string $targetName): void + { + try { + $storedPlan = CloudPortableRestoreTargetPlan::fromStored( + (string) $restore->selected_plan_json, + (string) $restore->selected_plan_sha256 + ); + } catch (\InvalidArgumentException) { + throw new CloudError('恢复请求保存的目标计划损坏', 409); + } + if ((int) $restore->backup_id !== (int) $backup->id + || (int) $restore->requested_by_user_id !== $actorUserId + || !hash_equals( + (string) $restore->request_fingerprint, + $fingerprint + ) + || !hash_equals($storedPlan->sha256(), $plan->sha256()) + || !hash_equals((string) $restore->new_name, $targetName) + || (int) $restore->target_host_id + !== (int) $plan->toArray()['target']['host_id']) { + throw new CloudError('恢复请求幂等身份冲突', 409); + } + } + + private static function assertAttemptPlan(CloudBackupRestore $restore, + CloudBackupRestoreAttempt $attempt, + CloudPortableRestoreTargetPlan $plan): void + { + try { + $stored = CloudPortableRestoreTargetPlan::fromStored( + (string) $attempt->target_plan_json, + (string) $attempt->target_plan_sha256 + ); + } catch (\InvalidArgumentException) { + throw new CloudError('恢复尝试保存的目标计划损坏', 409); + } + if (!hash_equals($stored->sha256(), $plan->sha256()) + || !hash_equals( + (string) $restore->selected_plan_sha256, + (string) $attempt->target_plan_sha256 + ) + || !hash_equals( + (string) $restore->selected_plan_json, + (string) $attempt->target_plan_json + ) + || (int) $attempt->target_host_id + !== (int) $restore->target_host_id + || !hash_equals( + strtolower((string) $attempt->target_uuid), + strtolower((string) $restore->new_uuid) + ) + || !hash_equals( + (string) $attempt->target_name, + (string) $restore->new_name + )) { + throw new CloudError('恢复尝试与父请求身份冲突', 409); + } + } + + private static function assertTargetEvidence(array $plan, + ?CloudRegion $region, ?CloudNetworkLine $line, + ?CloudResourcePool $pool, ?CloudHost $host, + ?CloudHostResourcePoolMembership $membership, + ?CloudHostStorageConfiguration $storage, ?CloudHostAgent $agent, + CloudPortableRestoreBackupEvidence $backup): void + { + $target = $plan['target']; + $targetContract = $agent?->testedPortableRestoreCompatibility(); + if (!$region instanceof CloudRegion + || !$line instanceof CloudNetworkLine + || !$pool instanceof CloudResourcePool + || !$host instanceof CloudHost + || !$membership instanceof CloudHostResourcePoolMembership + || !$storage instanceof CloudHostStorageConfiguration + || !$agent instanceof CloudHostAgent + || !(bool) $host->enabled + || (string) $region->status !== 'active' + || (string) $line->status !== 'active' + || (string) $pool->status !== 'active' + || (string) $membership->scheduler_state !== 'active' + || !$storage->isApplied() || !$agent->isReady() + || (int) $line->region_id !== (int) $region->id + || (int) $pool->line_id !== (int) $line->id + || (int) $membership->revision + !== (int) $target['membership_revision'] + || (int) $storage->revision + !== (int) $target['storage_revision'] + || (int) $agent->revision !== (int) $target['agent_revision'] + || $targetContract === null + || !$backup->compatibility()->matches($targetContract) + || !hash_equals( + $targetContract->sha256(), + (string) $target['agent_contract_sha256'] + ) + || !hash_equals((string) $host->name, + (string) $target['host_name']) + || !hash_equals((string) $pool->name, + (string) $target['resource_pool_name']) + || !hash_equals((string) $line->name, + (string) $target['line_name']) + || !hash_equals((string) $region->name, + (string) $target['region_name']) + || !hash_equals(strtoupper((string) $region->country_code), + (string) $target['country_code']) + || (bool) $target['is_source_host'] + !== ((int) $host->id + === (int) $plan['backup']['source_host_id'])) { + throw new CloudError('恢复目标宿主机计划已经过期', 409); + } + } + + private static function assertTargetNameAvailable(int $hostId, + string $targetName, int $ownAttemptId, + int $ownTargetInstanceId): void + { + $instances = CloudInstance::where('host_id', $hostId) + ->where('name', $targetName)->where('status', '<>', 'deleted'); + if ($ownTargetInstanceId > 0) { + $instances->where('id', '<>', $ownTargetInstanceId); + } + if ($instances->orderBy('id')->lockForUpdate()->get(['id']) + ->isNotEmpty()) { + throw new CloudError('目标宿主机已有同名实例', 409); + } + $query = CloudBackupRestoreAttempt::where('target_host_id', $hostId) + ->where('target_name', $targetName) + ->where('active_attempt', 1); + if ($ownAttemptId > 0) { + $query->where('id', '<>', $ownAttemptId); + } + if ($query->orderBy('id')->lockForUpdate()->get(['id']) + ->isNotEmpty()) { + throw new CloudError('目标宿主机已有同名恢复尝试', 409); + } + } + + private static function lockPlannedNetworkCatalog(array $plan, + CloudPublicNetworkBundleConstraint $constraint): void + { + $network = $plan['public_network']; + $attachments = DB::table('cloud_host_network_attachments') + ->where('id', (int) $network['network_attachment_id']) + ->where('host_id', (int) $network['host_id']) + ->where('line_id', (int) $network['line_id']) + ->where('state', 'active')->orderBy('id') + ->lockForUpdate()->get()->all(); + $attachmentIds = array_map( + static fn (object $row): int => (int) $row->id, + $attachments + ); + $pools = $attachmentIds === [] ? [] + : DB::table('cloud_public_ip_pools') + ->whereIn('id', [ + (int) $network['ipv4_pool']['public_ip_pool_id'], + (int) $network['ipv6_pool']['public_ip_pool_id'], + ]) + ->where('host_id', (int) $network['host_id']) + ->whereIn('network_attachment_id', $attachmentIds) + ->where('state', 'active')->whereIn('address_family', [4, 6]) + ->orderBy('id')->lockForUpdate()->get()->all(); + $attachment = null; + foreach ($attachments as $row) { + if ((int) $row->id === (int) $network['network_attachment_id']) { + $attachment = $row; + break; + } + } + $selectedPools = []; + foreach ($pools as $row) { + $family = (int) $row->address_family; + $poolKey = $family === 4 ? 'ipv4_pool' + : ($family === 6 ? 'ipv6_pool' : null); + if ($poolKey !== null + && (int) $row->network_attachment_id + === (int) $network['network_attachment_id'] + && (int) $row->id + === (int) $network[$poolKey]['public_ip_pool_id']) { + $selectedPools[$family] = $row; + } + } + if (!is_object($attachment) + || !isset($selectedPools[4], $selectedPools[6])) { + throw new CloudError('恢复目标双栈网络计划已经过期', 409); + } + $constraint->assertMatches($attachment, $selectedPools); + } + + private static function bundleConstraint(array $plan): + CloudPublicNetworkBundleConstraint + { + $network = $plan['public_network']; + unset($network['mode']); + foreach (['ipv4_pool', 'ipv6_pool'] as $family) { + unset($network[$family]['available_units']); + } + try { + return CloudPublicNetworkBundleConstraint::fromArray($network); + } catch (\InvalidArgumentException) { + throw new CloudError('恢复目标双栈网络计划无效', 409); + } + } + + private static function sameCapacityContract(array $current, + array $planned): bool + { + foreach (['policy_updated_at', 'requested', 'limits'] as $field) { + if (!array_key_exists($field, $current) + || !array_key_exists($field, $planned) + || self::canonical((array) $current[$field]) + !== self::canonical((array) $planned[$field])) { + return false; + } + } + return true; + } + + private static function requestFingerprint(int $actorUserId, + int $ownerUserId, array $backup, string $planSha256, + string $targetName): string + { + return hash('sha256', json_encode(self::canonical([ + 'contract' => 1, + 'requested_by_user_id' => $actorUserId, + 'owner_user_id' => $ownerUserId, + 'backup' => $backup, + 'selected_plan_sha256' => $planSha256, + 'target_name' => $targetName, + ]), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)); + } + + private static function canonical(array $value): array + { + if (array_is_list($value)) { + return array_map( + static fn ($item) => is_array($item) + ? self::canonical($item) : $item, + $value + ); + } + ksort($value, SORT_STRING); + foreach ($value as $key => $item) { + if (is_array($item)) { + $value[$key] = self::canonical($item); + } + } + return $value; + } + + private static function key(): string + { + return bin2hex(random_bytes(16)); + } + + private static function uuidV4(): string + { + $bytes = random_bytes(16); + $bytes[6] = chr((ord($bytes[6]) & 0x0f) | 0x40); + $bytes[8] = chr((ord($bytes[8]) & 0x3f) | 0x80); + $hex = bin2hex($bytes); + return substr($hex, 0, 8) . '-' . substr($hex, 8, 4) . '-' + . substr($hex, 12, 4) . '-' . substr($hex, 16, 4) . '-' + . substr($hex, 20); + } +} diff --git a/web/app/Services/Cloud/CloudPortableRestoreAttemptReservation.php b/web/app/Services/Cloud/CloudPortableRestoreAttemptReservation.php new file mode 100644 index 00000000..0532c7cd --- /dev/null +++ b/web/app/Services/Cloud/CloudPortableRestoreAttemptReservation.php @@ -0,0 +1,159 @@ +toArray()['resources']; + $dispatchable = (string) $attempt->state === 'reserved' + && (string) $reservation->state === 'held' + && $attempt->job_id === null + && (string) $attempt->effect_resolution === 'not_applicable'; + return new self([ + 'restore_id' => (int) $restore->id, + 'restore_key' => (string) $restore->restore_key, + 'attempt_id' => (int) $attempt->id, + 'attempt_key' => (string) $attempt->attempt_key, + 'generation' => (int) $attempt->generation, + 'state' => (string) $attempt->state, + 'effect_resolution' => (string) $attempt->effect_resolution, + 'operation_id' => (string) $attempt->operation_id, + 'reservation_id' => (int) $reservation->id, + 'claim_ids' => [ + 'ipv4' => $claimSet->claimId(4), + 'ipv6' => $claimSet->claimId(6), + ], + 'claim_set_sha256' => $claimSet->digest(), + 'target_plan_sha256' => $plan->sha256(), + 'vpc_inherited' => false, + 'dispatchable' => $dispatchable, + 'dispatch_body' => $dispatchable ? [ + 'uuid' => strtolower((string) $attempt->target_uuid), + 'name' => (string) $attempt->target_name, + 'vcpu' => $resources['cpu_cores'], + 'memory_mib' => $resources['memory_mib'], + 'disk_gib' => $resources['disk_gib'], + 'bandwidth_mbps' => $resources['bandwidth_mbps'], + 'public_network' => $network->publicNetwork()->toArray(), + ] : null, + ]); + } + + public static function terminal(CloudBackupRestore $restore, + CloudBackupRestoreAttempt $attempt, + CloudResourceReservation $reservation, + CloudPortableRestoreTargetPlan $plan, + array $claimIds): self + { + self::assertRows($restore, $attempt, $reservation, $plan); + foreach (['ipv4', 'ipv6'] as $family) { + if (!isset($claimIds[$family]) || !is_int($claimIds[$family]) + || $claimIds[$family] < 1) { + throw new CloudError('恢复尝试历史双栈声明不完整', 409); + } + } + return new self([ + 'restore_id' => (int) $restore->id, + 'restore_key' => (string) $restore->restore_key, + 'attempt_id' => (int) $attempt->id, + 'attempt_key' => (string) $attempt->attempt_key, + 'generation' => (int) $attempt->generation, + 'state' => (string) $attempt->state, + 'effect_resolution' => (string) $attempt->effect_resolution, + 'operation_id' => (string) $attempt->operation_id, + 'reservation_id' => (int) $reservation->id, + 'claim_ids' => $claimIds, + 'claim_set_sha256' => + (string) $attempt->target_claim_set_sha256, + 'target_plan_sha256' => $plan->sha256(), + 'vpc_inherited' => false, + 'dispatchable' => false, + 'dispatch_body' => null, + ]); + } + + public function toArray(): array + { + return $this->value; + } + + public function dispatchBody(): ?array + { + return $this->value['dispatch_body']; + } + + private static function assertRows(CloudBackupRestore $restore, + CloudBackupRestoreAttempt $attempt, + CloudResourceReservation $reservation, + CloudPortableRestoreTargetPlan $plan): void + { + if ((int) $restore->id < 1 || (int) $attempt->id < 1 + || (int) $reservation->id < 1 + || (int) $attempt->backup_restore_id !== (int) $restore->id + || (int) $attempt->reservation_id !== (int) $reservation->id + || !$attempt->matchesReservation($reservation) + || (int) $attempt->target_host_id + !== (int) $restore->target_host_id + || !hash_equals( + strtolower((string) $attempt->target_uuid), + strtolower((string) $restore->new_uuid) + ) + || !hash_equals( + (string) $attempt->target_name, + (string) $restore->new_name + ) + || !hash_equals( + (string) $attempt->target_plan_sha256, + $plan->sha256() + ) + || !hash_equals( + (string) $restore->selected_plan_sha256, + $plan->sha256() + )) { + throw new CloudError('恢复尝试资源身份不一致', 409); + } + } + + private static function assertActiveLifecycle( + CloudBackupRestoreAttempt $attempt, + CloudResourceReservation $reservation + ): void { + $state = (string) $attempt->state; + $resolution = (string) $attempt->effect_resolution; + $expectedReservation = match ($state) { + 'reserved', 'queued', 'awaiting_receipt' => + $resolution === 'not_applicable' ? 'held' : '', + 'effect_unknown' => + $resolution === 'unknown' ? 'uncertain' : '', + 'finalization_pending' => match ($resolution) { + 'applied' => 'committed', + 'not_applied' => 'held', + default => '', + }, + 'succeeded' => $resolution === 'applied' ? 'committed' : '', + default => '', + }; + if ($expectedReservation === '' + || (string) $reservation->state !== $expectedReservation) { + throw new CloudError('恢复尝试与资源预留生命周期不一致', 409); + } + } +} diff --git a/web/app/Services/Cloud/CloudPortableRestoreBackupEvidence.php b/web/app/Services/Cloud/CloudPortableRestoreBackupEvidence.php new file mode 100644 index 00000000..2743dc81 --- /dev/null +++ b/web/app/Services/Cloud/CloudPortableRestoreBackupEvidence.php @@ -0,0 +1,162 @@ +sourcePortableRestoreCompatibility(); + $objectKey = (string) $backup->object_key; + $sealedRuntime = (string) $backup->sealed_storage_runtime; + $objectEtag = (string) $backup->object_etag; + $objectVersionId = (string) $backup->object_version_id; + if ((int) $backup->id < 1 + || preg_match('/^[a-f0-9]{32}$/D', + (string) $backup->backup_key) !== 1 + || (int) $backup->source_host_id < 1 + || !self::validObjectKey($objectKey) + || trim($sealedRuntime) === '' + || strlen($objectEtag) > 256 + || preg_match('/[\x00-\x1f\x7f]/D', $objectEtag) === 1 + || strlen($objectVersionId) > 1024 + || preg_match('/[\x00-\x1f\x7f]/D', $objectVersionId) === 1 + || ($objectVersionId !== '' && $objectEtag === '') + || (string) $backup->purpose !== 'portable' + || (string) $backup->state !== 'ready' + || $contract === null + || (int) $backup->object_size_bytes < 1 + || (int) $backup->virtual_size_bytes < 1 + || (int) $backup->storage_revision < 1 + || preg_match('/^[a-f0-9]{64}$/D', + (string) $backup->sha256) !== 1) { + throw new CloudError('备份缺少可信的可移植恢复证据', 409); + } + try { + $spec = json_decode( + (string) $backup->spec_json, + true, + 64, + JSON_THROW_ON_ERROR + ); + if (!is_array($spec) || array_is_list($spec)) { + throw new \InvalidArgumentException(); + } + $sourceNetwork = CloudPublicNetworkSnapshot::fromArray( + $spec['public_network'] ?? null + ); + } catch (\Throwable) { + throw new CloudError('备份缺少可信的冻结双栈网络快照', 409); + } + $lineId = self::positiveInt($spec['line_id'] ?? null); + $resources = [ + 'cpu_cores' => self::positiveInt( + $spec['vcpu'] ?? $spec['cpu_cores'] ?? null + ), + 'memory_mib' => self::positiveInt( + $spec['memory_mib'] ?? $spec['ram_mib'] ?? null + ), + 'disk_gib' => self::positiveInt($spec['disk_gib'] ?? null), + 'bandwidth_mbps' => self::positiveInt( + $spec['bandwidth_mbps'] ?? null + ), + ]; + if ($lineId === null || in_array(null, $resources, true) + || $resources['disk_gib'] > intdiv(PHP_INT_MAX, self::GIB)) { + throw new CloudError('备份缺少可信的资源与商业线路快照', 409); + } + $virtualSize = (int) $backup->virtual_size_bytes; + if ($virtualSize > $resources['disk_gib'] * self::GIB) { + throw new CloudError('备份系统盘容量超过冻结规格', 409); + } + $objectIdentitySha256 = hash('sha256', json_encode([ + 'object_key' => $objectKey, + 'object_size_bytes' => (int) $backup->object_size_bytes, + 'object_sha256' => (string) $backup->sha256, + 'object_etag' => $objectEtag, + 'object_version_id' => $objectVersionId, + 'storage_revision' => (int) $backup->storage_revision, + 'sealed_storage_runtime_sha256' => hash( + 'sha256', + $sealedRuntime + ), + ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)); + return new self($contract, [ + 'id' => (int) $backup->id, + 'backup_key' => (string) $backup->backup_key, + 'object_size_bytes' => (int) $backup->object_size_bytes, + 'object_sha256' => (string) $backup->sha256, + 'object_identity_sha256' => $objectIdentitySha256, + 'source_contract_sha256' => $contract->sha256(), + 'source_public_network_sha256' => + $sourceNetwork->fingerprint(), + 'source_host_id' => (int) $backup->source_host_id, + 'source_line_id' => $lineId, + 'source_storage_revision' => (int) $backup->storage_revision, + 'virtual_size_bytes' => $virtualSize, + ], [ + 'cpu_cores' => $resources['cpu_cores'], + 'memory_mib' => $resources['memory_mib'], + 'disk_gib' => $resources['disk_gib'], + 'bandwidth_mbps' => $resources['bandwidth_mbps'], + 'system_disk_virtual_size_bytes' => $virtualSize, + ]); + } + + public function compatibility(): CloudPortableRestoreCompatibility + { + return $this->compatibility; + } + + public function backup(): array + { + return $this->backup; + } + + public function resources(): array + { + return $this->resources; + } + + public function capacityResources(): array + { + return [ + 'cpu_cores' => $this->resources['cpu_cores'], + 'memory_mib' => $this->resources['memory_mib'], + 'disk_gib' => $this->resources['disk_gib'], + ]; + } + + private static function positiveInt(mixed $value): ?int + { + $normalized = filter_var( + $value, + FILTER_VALIDATE_INT, + ['options' => ['min_range' => 1, 'max_range' => PHP_INT_MAX]] + ); + return $normalized === false ? null : (int) $normalized; + } + + private static function validObjectKey(string $objectKey): bool + { + return $objectKey !== '' + && strlen($objectKey) <= 512 + && !str_starts_with($objectKey, '/') + && !str_ends_with($objectKey, '/') + && !str_contains($objectKey, '\\') + && preg_match('/[\x00-\x1f\x7f]/D', $objectKey) !== 1 + && !in_array('', explode('/', $objectKey), true); + } +} diff --git a/web/app/Services/Cloud/CloudPortableRestoreTargetPlan.php b/web/app/Services/Cloud/CloudPortableRestoreTargetPlan.php index 8746db48..478ad805 100644 --- a/web/app/Services/Cloud/CloudPortableRestoreTargetPlan.php +++ b/web/app/Services/Cloud/CloudPortableRestoreTargetPlan.php @@ -196,6 +196,39 @@ final class CloudPortableRestoreTargetPlan return new self($plan); } + /** Rehydrate only exact canonical scheduler output. */ + public static function fromStored(string $json, string $sha256): self + { + if (preg_match('/^[a-f0-9]{64}$/D', $sha256) !== 1 + || !hash_equals(hash('sha256', $json), $sha256)) { + throw new \InvalidArgumentException( + 'Portable restore plan digest is invalid' + ); + } + try { + $decoded = json_decode($json, true, 64, JSON_THROW_ON_ERROR); + } catch (\JsonException $exception) { + throw new \InvalidArgumentException( + 'Portable restore plan JSON is invalid', + 0, + $exception + ); + } + if (!is_array($decoded) || array_is_list($decoded)) { + throw new \InvalidArgumentException( + 'Portable restore plan JSON is invalid' + ); + } + $plan = self::fromEvidence($decoded); + if (!hash_equals($plan->sha256(), $sha256) + || !hash_equals($plan->json(), $json)) { + throw new \InvalidArgumentException( + 'Portable restore plan is not canonical' + ); + } + return $plan; + } + public function toArray(): array { return $this->plan; diff --git a/web/app/Services/Cloud/CloudPublicNetworkAllocationService.php b/web/app/Services/Cloud/CloudPublicNetworkAllocationService.php index b3e59102..482aafa1 100644 --- a/web/app/Services/Cloud/CloudPublicNetworkAllocationService.php +++ b/web/app/Services/Cloud/CloudPublicNetworkAllocationService.php @@ -135,7 +135,27 @@ final class CloudPublicNetworkAllocationService $lineId, $reservationId, $operationId, - null + null, + false + ); + } + + /** + * Lock and verify one already-reserved bundle without ever allocating. + * + * Restore replay must fail closed when its durable claims disappeared; + * silently selecting replacement addresses would change guest identity. + */ + public static function verifyExistingBundle(int $hostId, int $lineId, + int $reservationId, string $operationId): CloudPublicNetworkSnapshot + { + return self::reserveBundleUsingConstraint( + $hostId, + $lineId, + $reservationId, + $operationId, + null, + true ); } @@ -156,13 +176,15 @@ final class CloudPublicNetworkAllocationService $lineId, $reservationId, $operationId, - $constraint + $constraint, + false ); } private static function reserveBundleUsingConstraint(int $hostId, int $lineId, int $reservationId, string $operationId, - ?CloudPublicNetworkBundleConstraint $constraint + ?CloudPublicNetworkBundleConstraint $constraint, + bool $requireExisting ): CloudPublicNetworkSnapshot { self::assertIdentity($reservationId, $operationId); if ($hostId < 1 || $lineId < 1) { @@ -173,13 +195,20 @@ final class CloudPublicNetworkAllocationService $lineId, $reservationId, $operationId, - $constraint + $constraint, + $requireExisting ): CloudPublicNetworkSnapshot { self::lockHost($hostId); $claimHints = self::unlockedClaims($reservationId); $catalog = null; $selection = null; if ($claimHints === []) { + if ($requireExisting) { + throw new CloudError( + '公网网络资源预留缺少既有双栈声明', + 409 + ); + } $catalog = self::lockPoolCatalog( $hostId, $lineId, diff --git a/web/app/Services/Cloud/CloudPublicNetworkClaimSetIdentity.php b/web/app/Services/Cloud/CloudPublicNetworkClaimSetIdentity.php index 66a18d53..1a79774f 100644 --- a/web/app/Services/Cloud/CloudPublicNetworkClaimSetIdentity.php +++ b/web/app/Services/Cloud/CloudPublicNetworkClaimSetIdentity.php @@ -83,6 +83,65 @@ final class CloudPublicNetworkClaimSetIdentity ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES))); } + /** + * Rebuild immutable identity after addresses have been safely released. + * + * Current address rows may already belong to another reservation, so the + * append-only claim history and the attempt's frozen network are the only + * valid terminal evidence. + * + * @param array $claims keyed by address family + */ + public static function fromReleasedLedger(int $reservationId, + array $claims, CloudPublicNetworkSnapshot $network): self + { + if ($reservationId < 1 || count($claims) !== 2) { + throw new CloudError('已释放公网网络声明集合不完整', 409); + } + $canonical = []; + $attachmentId = null; + foreach ([4, 6] as $family) { + $claim = $claims[$family] ?? null; + $currentAttachment = is_object($claim) + ? (int) ($claim->network_attachment_id ?? 0) : 0; + $operationId = is_object($claim) + ? (string) ($claim->operation_id ?? '') : ''; + if (!is_object($claim) + || (int) ($claim->id ?? 0) < 1 + || (int) ($claim->reservation_id ?? 0) !== $reservationId + || (int) ($claim->address_family ?? 0) !== $family + || (int) ($claim->claim_generation ?? 0) < 1 + || $claim->active_claim !== null + || (string) ($claim->state ?? '') !== 'released' + || ($claim->released_at ?? null) === null + || preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1 + || $currentAttachment < 1 + || (int) ($claim->public_ip_pool_id ?? 0) < 1 + || (int) ($claim->public_ip_id ?? 0) < 1) { + throw new CloudError('已释放公网网络声明集合不一致', 409); + } + if ($attachmentId === null) { + $attachmentId = $currentAttachment; + } elseif ($attachmentId !== $currentAttachment) { + throw new CloudError('已释放公网网络声明不属于同一接入', 409); + } + $canonical[] = [ + 'family' => $family, + 'claim_id' => (int) $claim->id, + 'claim_generation' => (int) $claim->claim_generation, + 'claim_operation_id' => $operationId, + 'public_ip_id' => (int) $claim->public_ip_id, + 'network_attachment_id' => $currentAttachment, + 'public_ip_pool_id' => (int) $claim->public_ip_pool_id, + ]; + } + return new self(hash('sha256', json_encode([ + 'reservation_id' => $reservationId, + 'claims' => $canonical, + 'public_network_sha256' => $network->fingerprint(), + ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES))); + } + public function digest(): string { return $this->digest; diff --git a/web/app/Services/Cloud/CloudReleasedPublicNetworkBundleVerifier.php b/web/app/Services/Cloud/CloudReleasedPublicNetworkBundleVerifier.php new file mode 100644 index 00000000..141bc491 --- /dev/null +++ b/web/app/Services/Cloud/CloudReleasedPublicNetworkBundleVerifier.php @@ -0,0 +1,189 @@ +target_plan_json, + (string) $attempt->target_plan_sha256 + ); + } catch (\InvalidArgumentException) { + throw new CloudError('失败恢复尝试的目标计划损坏', 409); + } + $target = $plan->toArray()['target']; + if ((int) $attempt->reservation_id !== (int) $reservation->id + || !hash_equals( + (string) $attempt->attempt_key, + (string) $reservation->backup_restore_attempt_key + ) + || (int) $reservation->host_id !== (int) $attempt->target_host_id + || (int) $reservation->host_id !== (int) $target['host_id'] + || (int) $reservation->resource_pool_id + !== (int) $target['resource_pool_id'] + || (int) $reservation->user_id !== $ownerUserId + || !hash_equals( + (string) $reservation->operation_id, + (string) $attempt->operation_id + ) + || (string) $reservation->state !== 'released' + || $reservation->released_at === null) { + throw new CloudError('失败恢复尝试仍占用资源或身份不一致', 409); + } + + $claimsQuery = CloudResourceReservationIp::where( + 'reservation_id', + (int) $reservation->id + )->orderBy('address_family')->orderBy('id'); + if ($lock) { + $claimsQuery->lockForUpdate(); + } + $claims = $claimsQuery->get(); + $ids = []; + $claimsByFamily = []; + foreach ($claims as $claim) { + $family = (int) $claim->address_family; + if (!in_array($family, [4, 6], true) + || isset($ids[$family]) + || (string) $claim->state !== 'released' + || !hash_equals( + (string) $claim->operation_id, + (string) $attempt->operation_id + ) + || (int) $claim->network_attachment_id + !== (int) $attempt->target_network_attachment_id + || $claim->active_claim !== null + || $claim->released_at === null) { + throw new CloudError('失败恢复尝试的公网声明未完全释放', 409); + } + $ids[$family] = (int) $claim->id; + $claimsByFamily[$family] = $claim; + } + if (!isset($ids[4], $ids[6])) { + throw new CloudError('失败恢复尝试的公网声明不完整', 409); + } + + try { + $networkValue = json_decode( + (string) $attempt->target_public_network_json, + true, + 64, + JSON_THROW_ON_ERROR + ); + $network = CloudPublicNetworkSnapshot::fromArray($networkValue); + $claimIdentity = + CloudPublicNetworkClaimSetIdentity::fromReleasedLedger( + (int) $reservation->id, + $claimsByFamily, + $network + ); + } catch (\Throwable) { + throw new CloudError('失败恢复尝试的冻结公网身份损坏', 409); + } + if (!hash_equals( + $network->fingerprint(), + (string) $attempt->target_public_network_sha256 + ) + || !hash_equals( + $claimIdentity->digest(), + (string) $attempt->target_claim_set_sha256 + )) { + throw new CloudError('失败恢复尝试的冻结公网身份不一致', 409); + } + + $plannedNetwork = $plan->toArray()['public_network']; + if ((int) $attempt->target_network_attachment_id + !== (int) $plannedNetwork['network_attachment_id'] + || (int) $claimsByFamily[4]->public_ip_pool_id + !== (int) $plannedNetwork['ipv4_pool']['public_ip_pool_id'] + || (int) $claimsByFamily[6]->public_ip_pool_id + !== (int) $plannedNetwork['ipv6_pool']['public_ip_pool_id']) { + throw new CloudError('失败恢复尝试的公网池身份不一致', 409); + } + + $addressQuery = DB::table('cloud_public_ips')->whereIn( + 'id', + [ + (int) $claimsByFamily[4]->public_ip_id, + (int) $claimsByFamily[6]->public_ip_id, + ] + )->orderBy('id'); + if ($lock) { + $addressQuery->lockForUpdate(); + } + $addresses = []; + foreach ($addressQuery->get() as $address) { + $addresses[(int) $address->id] = $address; + } + foreach ([4, 6] as $family) { + $claim = $claimsByFamily[$family]; + $address = $addresses[(int) $claim->public_ip_id] ?? null; + if (!is_object($address) + || (int) $address->host_id !== (int) $target['host_id'] + || (int) $address->address_family !== $family + || (int) $address->public_ip_pool_id + !== (int) $claim->public_ip_pool_id + || !hash_equals( + (string) $address->address . '/' + . (int) $address->prefix_length, + $network->address($family) + ) + || self::retainsOldOwnership( + $address, + (int) $reservation->id, + (int) $claim->id, + (string) $attempt->operation_id + ) + || !self::hasValidCurrentOwnership($address)) { + throw new CloudError('失败恢复尝试的公网地址未完整解除归属', 409); + } + } + return ['ipv4' => $ids[4], 'ipv6' => $ids[6]]; + } + + private static function retainsOldOwnership(object $address, + int $reservationId, int $claimId, string $operationId): bool + { + return (int) ($address->reservation_id ?? 0) === $reservationId + || (int) ($address->reservation_ip_claim_id ?? 0) === $claimId + || hash_equals( + $operationId, + (string) ($address->operation_id ?? '') + ); + } + + private static function hasValidCurrentOwnership(object $address): bool + { + $state = (string) ($address->state ?? ''); + $reservation = $address->reservation_id; + $claim = $address->reservation_ip_claim_id; + $instance = $address->instance_id; + $operation = $address->operation_id; + if (in_array($state, ['available', 'disabled'], true)) { + return $reservation === null && $claim === null + && $instance === null && $operation === null; + } + if ($reservation === null || $claim === null || $operation === null + || preg_match('/^[a-f0-9]{32}$/D', (string) $operation) !== 1) { + return false; + } + return match ($state) { + 'reserved' => $instance === null, + 'assigned' => $instance !== null, + 'changing', 'uncertain' => true, + default => false, + }; + } +} diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index 9960838e..076adfb4 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -1252,6 +1252,11 @@ CREATE TABLE cloud_backup_restores ( id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, restore_key CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, backup_id BIGINT UNSIGNED DEFAULT NULL, + requested_by_user_id BIGINT UNSIGNED DEFAULT NULL, + request_idempotency_key VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, + request_fingerprint CHAR(64) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, + selected_plan_json LONGTEXT DEFAULT NULL, + selected_plan_sha256 CHAR(64) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, target_host_id BIGINT UNSIGNED NOT NULL, target_instance_id BIGINT UNSIGNED DEFAULT NULL, reservation_id BIGINT UNSIGNED DEFAULT NULL, @@ -1265,11 +1270,33 @@ CREATE TABLE cloud_backup_restores ( created_at BIGINT UNSIGNED NOT NULL, updated_at BIGINT UNSIGNED NOT NULL, UNIQUE KEY cloud_backup_restore_key (restore_key), + UNIQUE KEY cloud_backup_restore_request ( + requested_by_user_id, request_idempotency_key + ), UNIQUE KEY cloud_backup_restore_operation (operation_id), UNIQUE KEY cloud_backup_restore_reservation (reservation_id), UNIQUE KEY cloud_backup_restore_instance (target_instance_id), KEY cloud_backup_restore_backup (backup_id, state, id), - KEY cloud_backup_restore_host (target_host_id, state, id) + KEY cloud_backup_restore_host (target_host_id, state, id), + CONSTRAINT chk_cloud_backup_restore_request CHECK ( + (backup_id IS NULL + AND requested_by_user_id IS NULL + AND request_idempotency_key IS NULL + AND request_fingerprint IS NULL + AND selected_plan_json IS NULL + AND selected_plan_sha256 IS NULL) + OR (backup_id IS NOT NULL + AND requested_by_user_id IS NOT NULL + AND requested_by_user_id > 0 + AND request_idempotency_key IS NOT NULL + AND CHAR_LENGTH(request_idempotency_key) BETWEEN 16 AND 128 + AND request_fingerprint IS NOT NULL + AND CHAR_LENGTH(request_fingerprint) = 64 + AND selected_plan_json IS NOT NULL + AND CHAR_LENGTH(selected_plan_json) > 1 + AND selected_plan_sha256 IS NOT NULL + AND CHAR_LENGTH(selected_plan_sha256) = 64) + ) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- Every independent restore dispatch owns one immutable attempt. A confirmed -- not-applied retry creates a new generation and operation instead of @@ -1286,6 +1313,8 @@ CREATE TABLE cloud_backup_restore_attempts ( target_host_id BIGINT UNSIGNED NOT NULL, target_uuid CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, target_name VARCHAR(63) NOT NULL, + target_plan_json LONGTEXT NOT NULL, + target_plan_sha256 CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, target_network_attachment_id BIGINT UNSIGNED NOT NULL, target_identity_json LONGTEXT NOT NULL, target_identity_sha256 CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, @@ -1317,6 +1346,10 @@ CREATE TABLE cloud_backup_restore_attempts ( CONSTRAINT chk_cloud_backup_restore_attempt_active CHECK ( active_attempt IS NULL OR active_attempt = 1 ), + CONSTRAINT chk_cloud_backup_restore_attempt_plan CHECK ( + CHAR_LENGTH(target_plan_json) > 1 + AND CHAR_LENGTH(target_plan_sha256) = 64 + ), CONSTRAINT chk_cloud_backup_restore_attempt_state CHECK ( state IN ('reserved','queued','awaiting_receipt','effect_unknown', 'finalization_pending','succeeded','failed') diff --git a/web/tests/CloudFreshSchemaContractTest.php b/web/tests/CloudFreshSchemaContractTest.php index 6ab91f3d..618e8b5f 100644 --- a/web/tests/CloudFreshSchemaContractTest.php +++ b/web/tests/CloudFreshSchemaContractTest.php @@ -22,7 +22,7 @@ $assert(stripos($schema, 'compatibility import') === false $assert(!preg_match('/\bengine_[A-Za-z0-9_]*\b/', $schema), 'cloud.sql retained an engine-prefixed identifier.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v32-portable-restore-contract'") + "SCHEMA_VERSION = 'go-agent-wss-v33-portable-restore-attempts'") && !str_contains($setup, "SCHEMA_VERSION = 'go-v1'"), 'Outbound Agent schema must reject the incompatible earlier Go database.'); $assert(str_contains($schema, diff --git a/web/tests/CloudPortableRestoreAttemptCoordinatorTest.php b/web/tests/CloudPortableRestoreAttemptCoordinatorTest.php new file mode 100644 index 00000000..178ebfd2 --- /dev/null +++ b/web/tests/CloudPortableRestoreAttemptCoordinatorTest.php @@ -0,0 +1,801 @@ +httpStatus === 409, $message . ' (wrong status)'); + return; + } + throw new RuntimeException($message); +}; + +// Restore the valid source row after the target-scheduler negative cases. +Capsule::table('cloud_instance_backups')->where('id', 1)->update([ + 'virtual_size_bytes' => 20 * 1073741824, +]); + +// Extend the target-scheduler fixture with the exact mutable ledgers owned by +// the attempt coordinator and atomic public-network allocator. +$schema->create('user', static function (Blueprint $t): void { + $t->increments('id'); + $t->integer('enable'); + $t->boolean('is_admin')->default(false); +}); +Capsule::table('user')->insert([ + ['id' => 9, 'enable' => 1, 'is_admin' => 0], + ['id' => 99, 'enable' => 1, 'is_admin' => 1], +]); + +$schema->table('cloud_instances', static function (Blueprint $t): void { + $t->string('name')->default('source-vm'); + $t->integer('user_id')->nullable(); + $t->integer('resource_pool_id')->nullable(); + $t->integer('reservation_id')->nullable(); +}); +Capsule::table('cloud_instances')->where('id', 1)->update(['user_id' => 9]); + +$schema->table('cloud_public_ip_pools', static function (Blueprint $t): void { + $t->string('allocation_cursor')->nullable(); +}); +$schema->table('cloud_public_ips', static function (Blueprint $t): void { + $t->binary('address_binary')->nullable(); + $t->integer('reservation_ip_claim_id')->nullable(); + $t->integer('created_at')->default(0); + $t->integer('updated_at')->default(0); +}); +foreach (Capsule::table('cloud_public_ips')->orderBy('id')->get() as $row) { + Capsule::table('cloud_public_ips')->where('id', (int) $row->id)->update([ + 'address_binary' => CloudIpAddress::parse( + (string) $row->address + )->binary(), + 'created_at' => $now, + 'updated_at' => $now, + ]); +} + +$schema->drop('cloud_resource_reservation_ips'); +$schema->drop('cloud_resource_reservations'); +$schema->drop('cloud_backup_restores'); +$schema->drop('cloud_jobs'); + +$schema->create('cloud_resource_reservations', static function ( + Blueprint $t +): void { + $t->increments('id'); + $t->string('reservation_key'); + $t->string('operation_id'); + $t->integer('order_id')->nullable(); + $t->integer('backup_restore_id')->nullable(); + $t->string('backup_restore_attempt_key')->nullable(); + $t->integer('migration_source_debt_id')->nullable(); + $t->integer('offer_id')->nullable(); + $t->integer('resource_pool_id'); + $t->integer('host_id'); + $t->integer('user_id'); + $t->integer('instance_id')->nullable(); + $t->integer('network_attachment_id')->nullable(); + $t->integer('public_ip_pool_id')->nullable(); + $t->integer('public_ip_id')->nullable(); + $t->integer('cpu_cores'); + $t->integer('memory_mib'); + $t->integer('disk_gib'); + $t->string('state'); + $t->integer('expires_at')->nullable(); + $t->integer('released_at')->nullable(); + $t->string('release_reason')->default(''); + $t->integer('created_at'); + $t->integer('updated_at'); + $t->unique('operation_id'); + $t->unique('backup_restore_attempt_key'); +}); +$schema->create('cloud_resource_reservation_ips', static function ( + Blueprint $t +): void { + $t->increments('id'); + $t->integer('reservation_id'); + $t->integer('address_family'); + $t->integer('claim_generation'); + $t->integer('active_claim')->nullable(); + $t->integer('network_attachment_id'); + $t->integer('public_ip_pool_id'); + $t->integer('public_ip_id'); + $t->string('operation_id'); + $t->string('state'); + $t->integer('created_at'); + $t->integer('updated_at'); + $t->integer('released_at')->nullable(); + $t->string('release_reason')->default(''); + $t->unique(['reservation_id', 'address_family', 'active_claim']); +}); +$schema->create('cloud_backup_restores', static function (Blueprint $t): void { + $t->increments('id'); + $t->string('restore_key')->unique(); + $t->integer('backup_id'); + $t->integer('requested_by_user_id'); + $t->string('request_idempotency_key'); + $t->string('request_fingerprint'); + $t->text('selected_plan_json'); + $t->string('selected_plan_sha256'); + $t->integer('target_host_id'); + $t->integer('target_instance_id')->nullable(); + $t->integer('reservation_id')->nullable(); + $t->integer('job_id')->nullable(); + $t->string('operation_id'); + $t->string('new_uuid'); + $t->string('new_name'); + $t->string('state'); + $t->string('error_code')->default(''); + $t->integer('ready_at')->nullable(); + $t->integer('created_at'); + $t->integer('updated_at'); + $t->unique(['requested_by_user_id', 'request_idempotency_key']); +}); +$schema->create('cloud_backup_restore_attempts', static function ( + Blueprint $t +): void { + $t->increments('id'); + $t->string('attempt_key')->unique(); + $t->integer('backup_restore_id'); + $t->integer('generation'); + $t->string('operation_id')->unique(); + $t->integer('job_id')->nullable(); + $t->integer('reservation_id')->unique(); + $t->integer('target_host_id'); + $t->string('target_uuid'); + $t->string('target_name'); + $t->text('target_plan_json'); + $t->string('target_plan_sha256'); + $t->integer('target_network_attachment_id'); + $t->text('target_identity_json'); + $t->string('target_identity_sha256'); + $t->string('target_claim_set_sha256'); + $t->text('target_public_network_json'); + $t->string('target_public_network_sha256'); + $t->string('state'); + $t->string('effect_resolution'); + $t->integer('active_attempt')->nullable(); + $t->string('last_error_code')->default(''); + $t->integer('effect_resolved_at')->nullable(); + $t->integer('terminal_at')->nullable(); + $t->integer('created_at'); + $t->integer('updated_at'); + $t->unique(['backup_restore_id', 'generation']); + $t->unique(['backup_restore_id', 'active_attempt']); +}); +$schema->create('cloud_jobs', static function (Blueprint $t): void { + $t->increments('id'); + $t->integer('user_id'); + $t->integer('backup_id')->nullable(); + $t->integer('backup_restore_id')->nullable(); + $t->integer('host_id')->nullable(); + $t->string('kind'); + $t->string('operation_id'); + $t->string('phase'); + $t->string('outcome')->nullable(); + $t->string('effect_resolution'); + $t->integer('terminal_at')->nullable(); + $t->integer('finalization_incident')->default(0); +}); +$schema->create('cloud_backup_recoveries', static function (Blueprint $t): void { + $t->increments('id'); + $t->integer('backup_id'); + $t->string('state'); +}); +$schema->create('cloud_deployment_snapshots', static function ( + Blueprint $t +): void { + $t->increments('id'); + $t->integer('backup_id'); + $t->string('state'); +}); +$schema->create('cloud_instance_migrations', static function ( + Blueprint $t +): void { + $t->increments('id'); + $t->integer('source_host_id'); + $t->integer('source_reservation_id'); + $t->integer('target_host_id'); + $t->integer('target_reservation_id'); + $t->integer('instance_id'); + $t->integer('restore_id'); + $t->string('state'); +}); + +$backup = CloudInstanceBackup::findOrFail(1); +$plans = CloudPortableRestoreTargetService::candidates($backup); +$plan = array_values(array_filter( + $plans, + static fn (CloudPortableRestoreTargetPlan $candidate): bool => + $candidate->toArray()['target']['host_id'] === 2 + && $candidate->toArray()['target']['resource_pool_id'] === 200 +))[0] ?? null; +$assert($plan instanceof CloudPortableRestoreTargetPlan, + 'Host-two portable restore plan is missing.'); + +$actor = [ + 'user_id' => 99, + 'is_admin' => true, + 'is_super_admin' => true, + 'can_manage' => true, +]; +$requestKey = 'restore-request-0001'; +$first = CloudPortableRestoreAttemptCoordinator::reserve( + $actor, + $backup, + $plan, + $plan->sha256(), + 'restored-vm', + $requestKey +)->toArray(); +$assert($first['generation'] === 1 && $first['dispatchable'] === true + && is_array($first['dispatch_body']) + && Capsule::table('cloud_resource_reservations') + ->where('id', $first['reservation_id'])->value('user_id') === 9 + && Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', $first['reservation_id'])->count() === 2, + 'Initial restore did not atomically reserve owner capacity and dual stack.'); + +$same = CloudPortableRestoreAttemptCoordinator::reserve( + $actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey +)->toArray(); +$assert($same['attempt_key'] === $first['attempt_key'] + && $same['reservation_id'] === $first['reservation_id'], + 'Exact reserve replay created another attempt.'); +$rejects409( + static fn () => CloudPortableRestoreAttemptCoordinator::reserve( + $actor, $backup, $plan, $plan->sha256(), 'different-vm', $requestKey + ), + 'Same idempotency key accepted a different restore request.' +); + +// Exact replay is a lookup of already-frozen evidence. Current eligibility +// drift must neither reject it nor generate a replacement attempt/bundle. +Capsule::table('cloud_instance_backups')->where('id', 1)->update([ + 'state' => 'failed', +]); +Capsule::table('cloud_host_agents')->where('host_id', 2)->update([ + 'enabled' => 0, + 'revision' => 99, + 'tested_revision' => 98, + 'last_test_status' => 'failed', +]); +Capsule::table('cloud_host_storage_configurations')->where('host_id', 2) + ->update(['revision' => 33, 'state' => 'pending']); +Capsule::table('cloud_host_resource_pool_memberships') + ->where('host_id', 2)->where('resource_pool_id', 200)->update([ + 'scheduler_state' => 'disabled', + 'revision' => 13, + ]); +$eligibilityReplay = CloudPortableRestoreAttemptCoordinator::reserve( + $actor, + CloudInstanceBackup::findOrFail(1), + $plan, + $plan->sha256(), + 'restored-vm', + $requestKey +)->toArray(); +$assert($eligibilityReplay['attempt_key'] === $first['attempt_key'] + && $eligibilityReplay['reservation_id'] === $first['reservation_id'], + 'Current backup or target eligibility drift broke exact replay.'); +Capsule::table('cloud_instance_backups')->where('id', 1)->update([ + 'state' => 'ready', +]); +Capsule::table('cloud_host_agents')->where('host_id', 2)->update([ + 'enabled' => 1, + 'revision' => 42, + 'tested_revision' => 42, + 'last_test_status' => 'healthy', +]); +Capsule::table('cloud_host_storage_configurations')->where('host_id', 2) + ->update(['revision' => 32, 'state' => 'applied']); +Capsule::table('cloud_host_resource_pool_memberships') + ->where('host_id', 2)->where('resource_pool_id', 200)->update([ + 'scheduler_state' => 'active', + 'revision' => 12, + ]); + +// A torn durable claim set must never be silently replaced during replay. +$claimRowsBefore = Capsule::table('cloud_resource_reservation_ips')->count(); +$ipIdsBefore = Capsule::table('cloud_public_ips')->orderBy('id') + ->pluck('id')->all(); +Capsule::connection()->beginTransaction(); +try { + Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', $first['reservation_id'])->delete(); + $rejects409( + static fn () => CloudPortableRestoreAttemptCoordinator::reserve( + $actor, + CloudInstanceBackup::findOrFail(1), + $plan, + $plan->sha256(), + 'restored-vm', + $requestKey + ), + 'Replay allocated a replacement bundle for missing durable claims.' + ); + $assert(Capsule::table('cloud_resource_reservation_ips')->count() + === $claimRowsBefore - 2 + && Capsule::table('cloud_public_ips')->orderBy('id') + ->pluck('id')->all() === $ipIdsBefore, + 'Failed replay inserted replacement claims or public addresses.'); +} finally { + Capsule::connection()->rollBack(); +} + +// Volatile capacity changes must not invalidate an already-reserved replay. +Capsule::table('cloud_resource_reservations')->insert([ + 'reservation_key' => str_repeat('1', 32), + 'operation_id' => str_repeat('2', 32), + 'order_id' => 77, + 'backup_restore_id' => null, + 'backup_restore_attempt_key' => null, + 'migration_source_debt_id' => null, + 'offer_id' => 88, + 'resource_pool_id' => 200, + 'host_id' => 2, + 'user_id' => 9, + 'instance_id' => null, + 'network_attachment_id' => null, + 'public_ip_pool_id' => null, + 'public_ip_id' => null, + 'cpu_cores' => 1, + 'memory_mib' => 256, + 'disk_gib' => 1, + 'state' => 'held', + 'expires_at' => null, + 'released_at' => null, + 'release_reason' => '', + 'created_at' => $now, + 'updated_at' => $now, +]); +$capacityReplay = CloudPortableRestoreAttemptCoordinator::reserve( + $actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey +)->toArray(); +$assert($capacityReplay['attempt_key'] === $first['attempt_key'], + 'Volatile capacity drift broke exact replay.'); + +$attempt = CloudBackupRestoreAttempt::where( + 'attempt_key', + $first['attempt_key'] +)->firstOrFail(); +$reservation = CloudResourceReservation::findOrFail( + $first['reservation_id'] +); + +// Reservation/claim fencing is not a dispatchable state. A stale reserved +// attempt must fail closed instead of returning a body that could be sent. +$reservation->state = 'uncertain'; +$reservation->save(); +Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', (int) $reservation->id)->update([ + 'state' => 'uncertain', 'updated_at' => $now, + ]); +Capsule::table('cloud_public_ips') + ->where('reservation_id', (int) $reservation->id)->update([ + 'state' => 'uncertain', 'updated_at' => $now, + ]); +$rejects409( + static fn () => CloudPortableRestoreAttemptCoordinator::reserve( + $actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey + ), + 'Reserved attempt accepted an uncertain fenced resource bundle.' +); +$reservation->state = 'held'; +$reservation->save(); +Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', (int) $reservation->id)->update([ + 'state' => 'reserved', 'updated_at' => $now, + ]); +Capsule::table('cloud_public_ips') + ->where('reservation_id', (int) $reservation->id)->update([ + 'state' => 'reserved', 'updated_at' => $now, + ]); + +$attempt->state = 'effect_unknown'; +$attempt->effect_resolution = 'unknown'; +$attempt->job_id = 500; +$attempt->save(); +$reservation->state = 'uncertain'; +$reservation->save(); +Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', (int) $reservation->id)->update([ + 'state' => 'uncertain', 'updated_at' => $now, + ]); +Capsule::table('cloud_public_ips') + ->where('reservation_id', (int) $reservation->id)->update([ + 'state' => 'uncertain', 'updated_at' => $now, + ]); +$unknown = CloudPortableRestoreAttemptCoordinator::reserve( + $actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey +)->toArray(); +$assert($unknown['dispatchable'] === false + && $unknown['dispatch_body'] === null, + 'Unknown-effect replay exposed a dispatch instruction.'); + +Capsule::table('cloud_instance_backups')->where('id', 1)->update([ + 'state' => 'failed', +]); +$unknownAfterBackupDrift = CloudPortableRestoreAttemptCoordinator::reserve( + $actor, + CloudInstanceBackup::findOrFail(1), + $plan, + $plan->sha256(), + 'restored-vm', + $requestKey +)->toArray(); +$assert($unknownAfterBackupDrift['attempt_key'] === $first['attempt_key'] + && $unknownAfterBackupDrift['dispatchable'] === false + && $unknownAfterBackupDrift['dispatch_body'] === null, + 'Backup-state drift broke stable unknown-effect replay.'); +Capsule::table('cloud_instance_backups')->where('id', 1)->update([ + 'state' => 'ready', +]); + +// The attempt row, not the mutable parent projection, protects the backup. +Capsule::table('cloud_backup_restores')->where('id', $first['restore_id']) + ->update(['state' => 'failed']); +$guard = new ReflectionMethod( + CloudBackupService::class, + 'assertBackupDeletionAvailable' +); +$rejects409( + static fn () => $guard->invoke(null, CloudInstanceBackup::findOrFail(1)), + 'An active attempt did not protect its source backup.' +); + +CloudPublicNetworkAllocationService::releaseBundle( + (int) $reservation->id, + (string) $attempt->operation_id, + 'restore_not_applied', + true +); +$releasedAt = $now + 10; +$reservation->state = 'released'; +$reservation->released_at = $releasedAt; +$reservation->release_reason = 'restore_not_applied'; +$reservation->save(); +$attempt->state = 'failed'; +$attempt->effect_resolution = 'not_applied'; +$attempt->active_attempt = null; +$attempt->effect_resolved_at = $releasedAt; +$attempt->terminal_at = $releasedAt; +$attempt->save(); +Capsule::table('cloud_jobs')->insert([ + 'id' => 500, + 'user_id' => 9, + 'backup_id' => 1, + 'backup_restore_id' => $first['restore_id'], + 'host_id' => 2, + 'kind' => 'backup_restore', + 'operation_id' => (string) $attempt->operation_id, + 'phase' => 'terminal', + 'outcome' => 'failed', + 'effect_resolution' => 'not_applied', + 'terminal_at' => $releasedAt, + 'finalization_incident' => 0, +]); + +// A fully released terminal not-applied attempt no longer protects the object. +$guard->invoke(null, CloudInstanceBackup::findOrFail(1)); + +// Retry authority is the exact terminal job evidence, including ownership and +// finalization status. Each mutation must fail without creating a generation. +$jobEvidenceMutations = [ + ['kind', 'wrong_kind', 'backup_restore'], + ['host_id', 1, 2], + ['user_id', 99, 9], + ['backup_restore_id', 999, $first['restore_id']], + ['operation_id', str_repeat('9', 32), (string) $attempt->operation_id], + ['finalization_incident', 1, 0], +]; +foreach ($jobEvidenceMutations as [$field, $invalid, $valid]) { + Capsule::table('cloud_jobs')->where('id', 500)->update([$field => $invalid]); + $rejects409( + static fn () => CloudPortableRestoreAttemptCoordinator::retry( + $actor, + $backup, + $plan, + $plan->sha256(), + 'restored-vm', + $requestKey, + (string) $attempt->attempt_key + ), + 'Retry accepted mismatched terminal job evidence: ' . $field . '.' + ); + $assert(CloudBackupRestoreAttempt::where( + 'backup_restore_id', + $first['restore_id'] + )->count() === 1, 'Rejected retry created another attempt row.'); + Capsule::table('cloud_jobs')->where('id', 500)->update([$field => $valid]); +} + +// Released reservation and address claims remain part of the predecessor's +// exact operation identity. A torn terminal ledger must never authorize a new +// dispatch generation even when the job itself says not_applied. +$ledgerOperationMutations = [ + ['cloud_resource_reservations', null], + ['cloud_resource_reservation_ips', 4], + ['cloud_resource_reservation_ips', 6], +]; +foreach ($ledgerOperationMutations as [$table, $family]) { + $query = Capsule::table($table)->where( + 'reservation_id', + (int) $reservation->id + ); + if ($table === 'cloud_resource_reservations') { + $query = Capsule::table($table)->where('id', (int) $reservation->id); + } else { + $query->where('address_family', $family); + } + $query->update(['operation_id' => str_repeat('8', 32)]); + $rejects409( + static fn () => CloudPortableRestoreAttemptCoordinator::retry( + $actor, + $backup, + $plan, + $plan->sha256(), + 'restored-vm', + $requestKey, + (string) $attempt->attempt_key + ), + 'Retry accepted torn terminal ledger operation identity: ' + . $table . ':' . (string) $family . '.' + ); + $assert(CloudBackupRestoreAttempt::where( + 'backup_restore_id', + $first['restore_id'] + )->count() === 1, 'Rejected ledger retry created another attempt row.'); + $query->update(['operation_id' => (string) $attempt->operation_id]); +} + +$ipv4Claim = Capsule::table('cloud_resource_reservation_ips') + ->where('reservation_id', (int) $reservation->id) + ->where('address_family', 4)->first(); +$assert(is_object($ipv4Claim), 'Released IPv4 claim fixture is missing.'); +Capsule::table('cloud_resource_reservation_ips') + ->where('id', (int) $ipv4Claim->id) + ->update(['public_ip_pool_id' => (int) $ipv4Claim->public_ip_pool_id + 1]); +$rejects409( + static fn () => CloudPortableRestoreAttemptCoordinator::retry( + $actor, + $backup, + $plan, + $plan->sha256(), + 'restored-vm', + $requestKey, + (string) $attempt->attempt_key + ), + 'Retry accepted a released claim that changed frozen claim-set identity.' +); +$assert(CloudBackupRestoreAttempt::where( + 'backup_restore_id', + $first['restore_id'] +)->count() === 1, 'Rejected claim identity created another attempt row.'); +Capsule::table('cloud_resource_reservation_ips') + ->where('id', (int) $ipv4Claim->id) + ->update(['public_ip_pool_id' => (int) $ipv4Claim->public_ip_pool_id]); + +foreach ([ + ['claim_generation', (int) $ipv4Claim->claim_generation + 1], + ['public_ip_id', (int) $ipv4Claim->public_ip_id + 999999], +] as [$field, $invalid]) { + Capsule::table('cloud_resource_reservation_ips') + ->where('id', (int) $ipv4Claim->id)->update([$field => $invalid]); + $rejects409( + static fn () => CloudPortableRestoreAttemptCoordinator::retry( + $actor, + $backup, + $plan, + $plan->sha256(), + 'restored-vm', + $requestKey, + (string) $attempt->attempt_key + ), + 'Retry accepted changed released claim identity: ' . $field . '.' + ); + $assert(CloudBackupRestoreAttempt::where( + 'backup_restore_id', + $first['restore_id'] + )->count() === 1, 'Rejected claim mutation created another attempt row.'); + Capsule::table('cloud_resource_reservation_ips') + ->where('id', (int) $ipv4Claim->id) + ->update([$field => $ipv4Claim->{$field}]); +} + +// A released claim is not sufficient when its concrete address still points +// at the predecessor. The address may later be legitimately reused by another +// operation, but all ownership references to this operation must be gone. +Capsule::table('cloud_public_ips') + ->where('id', (int) $ipv4Claim->public_ip_id)->update([ + 'state' => 'reserved', + 'reservation_id' => (int) $reservation->id, + 'reservation_ip_claim_id' => (int) $ipv4Claim->id, + 'instance_id' => null, + 'operation_id' => (string) $attempt->operation_id, + ]); +$rejects409( + static fn () => CloudPortableRestoreAttemptCoordinator::retry( + $actor, + $backup, + $plan, + $plan->sha256(), + 'restored-vm', + $requestKey, + (string) $attempt->attempt_key + ), + 'Retry accepted a public address that retained predecessor ownership.' +); +$assert(CloudBackupRestoreAttempt::where( + 'backup_restore_id', + $first['restore_id'] +)->count() === 1, 'Residual address ownership created another attempt row.'); +Capsule::table('cloud_public_ips') + ->where('id', (int) $ipv4Claim->public_ip_id)->update([ + 'state' => 'available', + 'reservation_id' => null, + 'reservation_ip_claim_id' => null, + 'instance_id' => null, + 'operation_id' => null, + ]); + +$second = CloudPortableRestoreAttemptCoordinator::retry( + $actor, + $backup, + $plan, + $plan->sha256(), + 'restored-vm', + $requestKey, + (string) $attempt->attempt_key +)->toArray(); +$assert($second['generation'] === 2 + && $second['attempt_key'] !== $first['attempt_key'] + && $second['reservation_id'] !== $first['reservation_id'] + && $second['dispatchable'] === true, + 'Explicit not-applied retry did not create one fresh generation.'); +$retryReplay = CloudPortableRestoreAttemptCoordinator::retry( + $actor, + $backup, + $plan, + $plan->sha256(), + 'restored-vm', + $requestKey, + (string) $attempt->attempt_key +)->toArray(); +$assert($retryReplay['attempt_key'] === $second['attempt_key'] + && $retryReplay['generation'] === 2, + 'Retry replay skipped to another generation or failed idempotency.'); + +// A queued generation is already owned by a future job and cannot be emitted +// as another dispatch instruction. +$secondAttempt = CloudBackupRestoreAttempt::where( + 'attempt_key', + $second['attempt_key'] +)->firstOrFail(); +$secondAttempt->state = 'queued'; +$secondAttempt->job_id = 501; +$secondAttempt->save(); +$queued = CloudPortableRestoreAttemptCoordinator::reserve( + $actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey +)->toArray(); +$assert($queued['dispatchable'] === false && $queued['dispatch_body'] === null, + 'Queued replay exposed a second dispatch instruction.'); + +// A completed attempt replays only its durable result and never another Agent +// instruction. Model the already-finalized instance/network ownership chain. +$secondReservation = CloudResourceReservation::findOrFail( + $second['reservation_id'] +); +Capsule::table('cloud_instances')->insert([ + 'id' => 2, + 'host_id' => 2, + 'status' => 'active', + 'spec_json' => json_encode([ + 'public_network' => $second['dispatch_body']['public_network'], + ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES), + 'name' => 'restored-vm', + 'user_id' => 9, + 'resource_pool_id' => 200, + 'reservation_id' => (int) $secondReservation->id, +]); +CloudPublicNetworkAllocationService::commitBundle( + (int) $secondReservation->id, + 2, + (string) $secondAttempt->operation_id +); +$secondReservation->instance_id = 2; +$secondReservation->state = 'committed'; +$secondReservation->updated_at = $now + 20; +$secondReservation->save(); +$secondAttempt->state = 'succeeded'; +$secondAttempt->effect_resolution = 'applied'; +$secondAttempt->active_attempt = null; +$secondAttempt->effect_resolved_at = $now + 20; +$secondAttempt->terminal_at = $now + 20; +$secondAttempt->save(); +Capsule::table('cloud_backup_restores')->where('id', $first['restore_id']) + ->update([ + 'target_instance_id' => 2, + 'state' => 'ready', + 'ready_at' => $now + 20, + ]); +$succeeded = CloudPortableRestoreAttemptCoordinator::reserve( + $actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey +)->toArray(); +$assert($succeeded['attempt_key'] === $second['attempt_key'] + && $succeeded['state'] === 'succeeded' + && $succeeded['dispatchable'] === false + && $succeeded['dispatch_body'] === null, + 'Succeeded replay exposed another dispatch instruction.'); + +$policy = (object) [ + 'updated_at' => 1, + 'configured_cpu_cores' => 8, + 'configured_memory_mib' => 8192, + 'configured_disk_gib' => 100, + 'safety_cpu_cores' => 0, + 'safety_memory_mib' => 0, + 'safety_disk_gib' => 0, + 'max_instances' => 10, +]; +$resources = ['cpu_cores' => 1, 'memory_mib' => 512, 'disk_gib' => 10]; +$validUsage = [ + 'unknown' => 0, + 'instance_count' => 0, + 'cpu_sum' => 0, + 'memory_sum' => 0, + 'disk_sum' => 0, +]; +$missingMaximum = clone $policy; +unset($missingMaximum->max_instances); +$missingTimestamp = clone $policy; +unset($missingTimestamp->updated_at); +$assert(!CloudHostCapacityAdmission::allows($policy, [], $resources) + && !CloudHostCapacityAdmission::allows($policy, [ + 'unknown' => 0, + 'instance_count' => -1, + 'cpu_sum' => 0, + 'memory_sum' => 0, + 'disk_sum' => 0, + ], $resources) + && !CloudHostCapacityAdmission::allows($policy, [ + 'unknown' => 1, + 'instance_count' => 0, + 'cpu_sum' => 0, + 'memory_sum' => 0, + 'disk_sum' => 0, + ], $resources) + && !CloudHostCapacityAdmission::allows( + $missingMaximum, + $validUsage, + $resources + ) + && !CloudHostCapacityAdmission::allows( + $missingTimestamp, + $validUsage, + $resources + ), + 'Capacity admission accepted incomplete policy or invalid usage.'); + +echo "Portable restore attempt coordinator behavior tests passed.\n"; diff --git a/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php b/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php index dd50f446..876d2b1c 100644 --- a/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php +++ b/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php @@ -65,7 +65,7 @@ $assert(is_string($schema) 'Public-pool replacement lacks durable batch, child, or active-claim schema.'); $assert(is_string($setup) && str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v32-portable-restore-contract'") + "SCHEMA_VERSION = 'go-agent-wss-v33-portable-restore-attempts'") && str_contains($setup, "'cloud_public_ip_pool_replacement_batches'") && str_contains($setup, diff --git a/web/tests/CloudPublicNetworkTransitionSchemaContractTest.php b/web/tests/CloudPublicNetworkTransitionSchemaContractTest.php index 9a50a876..cd0bfb32 100644 --- a/web/tests/CloudPublicNetworkTransitionSchemaContractTest.php +++ b/web/tests/CloudPublicNetworkTransitionSchemaContractTest.php @@ -170,7 +170,7 @@ $assert(str_contains($setup, 'CloudSetup does not verify index uniqueness and ordered columns.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v32-portable-restore-contract'") + "SCHEMA_VERSION = 'go-agent-wss-v33-portable-restore-attempts'") && str_contains($setup, "'cloud_public_network_transitions'") && str_contains($setup, "'cloud_public_network_transition_ips'") && str_contains($setup, "'reservation_ip_claim_id'") diff --git a/web/tests/CloudRescueCatalogSchemaContractTest.php b/web/tests/CloudRescueCatalogSchemaContractTest.php index a04aa38b..244c20cd 100644 --- a/web/tests/CloudRescueCatalogSchemaContractTest.php +++ b/web/tests/CloudRescueCatalogSchemaContractTest.php @@ -56,7 +56,7 @@ $assert($replica !== '' && str_contains($replica, 'cleanup_resolved_at BIGINT UNSIGNED'), 'Rescue replica cannot prove exact identity or retain cleanup debt.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v32-portable-restore-contract'") + "SCHEMA_VERSION = 'go-agent-wss-v33-portable-restore-attempts'") && str_contains($setup, "'cloud_rescue_artifact_delete_due'") && str_contains($setup, "'cloud_rescue_replica_cleanup'"), 'CloudSetup version or rescue catalog indexes are incomplete.'); diff --git a/web/tests/CloudRestoreMigrationLedgerContractTest.php b/web/tests/CloudRestoreMigrationLedgerContractTest.php index ae9cf404..7705b183 100644 --- a/web/tests/CloudRestoreMigrationLedgerContractTest.php +++ b/web/tests/CloudRestoreMigrationLedgerContractTest.php @@ -19,9 +19,16 @@ $attemptModel = file_get_contents( $debtModel = file_get_contents( $root . '/app/Models/CloudInstanceMigrationSourceDebt.php' ); +$coordinator = file_get_contents( + $root . '/app/Services/Cloud/CloudPortableRestoreAttemptCoordinator.php' +); +$allocator = file_get_contents( + $root . '/app/Services/Cloud/CloudPublicNetworkAllocationService.php' +); if (!is_string($schema) || !is_string($setup) || !is_string($reservationModel) || !is_string($attemptModel) - || !is_string($debtModel)) { + || !is_string($debtModel) || !is_string($coordinator) + || !is_string($allocator)) { throw new RuntimeException('Restore/migration ledger sources are unreadable.'); } @@ -76,6 +83,7 @@ $schemaBuilder->create('cloud_resource_reservations', static function ( }); $attempts = $table('cloud_backup_restore_attempts'); +$restores = $table('cloud_backup_restores'); $debts = $table('cloud_instance_migration_source_debts'); $reservations = $table('cloud_resource_reservations'); $assert($attempts !== '' @@ -93,6 +101,12 @@ $assert($attempts !== '' 'target_claim_set_sha256 CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL') && str_contains($attempts, 'target_public_network_sha256 CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL') + && str_contains($attempts, + 'target_plan_json LONGTEXT NOT NULL') + && str_contains($attempts, + 'target_plan_sha256 CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL') + && str_contains($attempts, + 'CONSTRAINT chk_cloud_backup_restore_attempt_plan CHECK') && str_contains($attempts, 'CONSTRAINT chk_cloud_backup_restore_attempt_shape CHECK'), 'Backup restore attempts do not fence immutable generations and network identity.'); @@ -111,6 +125,22 @@ $assert(str_contains($attemptShape, "state = 'failed' AND effect_resolution = 'not_applied' AND job_id IS NOT NULL AND active_attempt IS NULL"), 'Restore-attempt terminal and unknown-effect fencing can diverge.'); +$assert($restores !== '' + && str_contains($restores, + 'requested_by_user_id BIGINT UNSIGNED DEFAULT NULL') + && str_contains($restores, + 'request_idempotency_key VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL') + && str_contains($restores, + 'request_fingerprint CHAR(64) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL') + && str_contains($restores, 'selected_plan_json LONGTEXT DEFAULT NULL') + && str_contains($restores, + 'selected_plan_sha256 CHAR(64) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL') + && str_contains($restores, + 'UNIQUE KEY cloud_backup_restore_request') + && str_contains($restores, + 'CONSTRAINT chk_cloud_backup_restore_request CHECK'), + 'Portable restore request identity and selected plan are not durable.'); + $assert($debts !== '' && str_contains($debts, 'UNIQUE KEY cloud_migration_source_debt_migration (migration_id)') @@ -171,9 +201,12 @@ foreach ([ "'backup_restore_attempt_key'", "'migration_source_debt_id'", "'chk_cloud_backup_restore_attempt_shape'", + "'chk_cloud_backup_restore_attempt_plan'", + "'chk_cloud_backup_restore_request'", "'chk_cloud_migration_source_debt_shape'", "'chk_cloud_migration_source_debt_cleanup_pair'", "'cloud_backup_restore_attempt_active'", + "'cloud_backup_restore_request'", "'cloud_migration_source_debt_active_reservation'", "'columns' => ['backup_restore_attempt_key']", "'columns' => ['migration_source_debt_id']", @@ -182,7 +215,7 @@ foreach ([ 'CloudSetup omits restore/migration ledger contract: ' . $required); } $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v32-portable-restore-contract'") + "SCHEMA_VERSION = 'go-agent-wss-v33-portable-restore-attempts'") && str_contains($attemptModel, "protected \$table = 'cloud_backup_restore_attempts';") && str_contains($attemptModel, "'active_attempt' => 'int'") @@ -199,6 +232,22 @@ $assert(str_contains($setup, "'migration_source_debt_id' => 'int'"), 'Models or setup version do not expose the new ownership boundary.'); +$assert(str_contains($coordinator, 'CloudAccess::manage($actor)') + && str_contains($coordinator, + 'CloudPublicNetworkAllocationService::reservePlannedBundle(') + && str_contains($coordinator, + 'CloudPublicNetworkAllocationService::verifyExistingBundle(') + && !str_contains($coordinator, 'reserveBackupRestore(') + && !str_contains($coordinator, 'preparedBackupRestore(') + && !str_contains($coordinator, 'finishBackupRestore(') + && !str_contains($coordinator, 'new CloudJob()') + && !str_contains($coordinator, 'new CloudInstance()') + && str_contains($allocator, + 'public static function verifyExistingBundle(') + && str_contains($allocator, + '公网网络资源预留缺少既有双栈声明'), + 'Portable restore admission can dispatch, create instances, or reallocate a missing replay bundle.'); + $frozenIdentity = file_get_contents( $root . '/app/Services/Cloud/CloudFrozenPublicNetworkIdentity.php' ); diff --git a/web/tests/CloudTemplateImportIntentContractTest.php b/web/tests/CloudTemplateImportIntentContractTest.php index 620bda31..5a54571c 100644 --- a/web/tests/CloudTemplateImportIntentContractTest.php +++ b/web/tests/CloudTemplateImportIntentContractTest.php @@ -63,7 +63,7 @@ $assert(str_contains($schema, && str_contains($schema, 'CONSTRAINT chk_cloud_template_import_completion CHECK (') && str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v32-portable-restore-contract'"), + "SCHEMA_VERSION = 'go-agent-wss-v33-portable-restore-attempts'"), 'Template imports are missing durable identity or terminal-state constraints.'); $assert(str_contains($routes, "'/admin/cloud-template-imports'") && str_contains($routes, "'/admin/cloud-template-imports/{id}'") diff --git a/web/tests/CloudTrafficPoolSchemaContractTest.php b/web/tests/CloudTrafficPoolSchemaContractTest.php index 498d93d6..9c84241a 100644 --- a/web/tests/CloudTrafficPoolSchemaContractTest.php +++ b/web/tests/CloudTrafficPoolSchemaContractTest.php @@ -71,7 +71,7 @@ $assert(str_contains($schema, 'Service-period traffic-pool identity is not frozen or indexed.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v32-portable-restore-contract'") + "SCHEMA_VERSION = 'go-agent-wss-v33-portable-restore-attempts'") && str_contains($setup, "'chk_cloud_traffic_entitlement_balance'") && str_contains($setup, "'cloud_traffic_ledger_sequence'"), 'CloudSetup schema version or named-object validation is incomplete.'); diff --git a/web/tests/CloudVpcSchemaContractTest.php b/web/tests/CloudVpcSchemaContractTest.php index ae0713ed..4a690a5e 100644 --- a/web/tests/CloudVpcSchemaContractTest.php +++ b/web/tests/CloudVpcSchemaContractTest.php @@ -101,7 +101,7 @@ $assert(str_contains($jobs, 'vpc_id BIGINT UNSIGNED DEFAULT NULL') 'Cloud jobs cannot be correlated to VPC or VPC-port work.'); $assert(str_contains($setup, - "private const SCHEMA_VERSION = 'go-agent-wss-v32-portable-restore-contract'") + "private const SCHEMA_VERSION = 'go-agent-wss-v33-portable-restore-attempts'") && str_contains($setup, "'name' => 'cloud_vpc_enabled'") && str_contains($setup, "'value' => 'false'") && str_contains($setup,