feat: reserve portable restore attempts
parent
98a3b46e88
commit
c8fc5000d3
@ -0,0 +1,148 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
/** Shared capacity arithmetic for advisory and locked restore admission. */
|
||||
final class CloudHostCapacityAdmission
|
||||
{
|
||||
public static function allows(object $policy, array $usage,
|
||||
array $resources): bool
|
||||
{
|
||||
if (!self::validUsage($usage)) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
$evidence = self::evidence($policy, $usage, $resources);
|
||||
} catch (\InvalidArgumentException) {
|
||||
return false;
|
||||
}
|
||||
if ($evidence['policy_updated_at'] < 1
|
||||
|| $evidence['limits']['instance_count'] < 0) {
|
||||
return false;
|
||||
}
|
||||
foreach (['cpu_cores', 'memory_mib', 'disk_gib'] as $field) {
|
||||
if ($evidence['limits'][$field] < 1
|
||||
|| $evidence['used'][$field]
|
||||
> $evidence['limits'][$field]
|
||||
- $evidence['requested'][$field]) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return $evidence['limits']['instance_count'] < 1
|
||||
|| $evidence['used']['instance_count'] + 1
|
||||
<= $evidence['limits']['instance_count'];
|
||||
}
|
||||
|
||||
public static function evidence(object $policy, array $usage,
|
||||
array $resources): array
|
||||
{
|
||||
if (!self::validUsage($usage)) {
|
||||
throw new \InvalidArgumentException(
|
||||
'Capacity usage evidence is invalid.'
|
||||
);
|
||||
}
|
||||
foreach (['cpu_cores', 'memory_mib', 'disk_gib'] as $field) {
|
||||
if (!isset($resources[$field]) || !is_int($resources[$field])
|
||||
|| $resources[$field] < 1) {
|
||||
throw new \InvalidArgumentException(
|
||||
'Capacity request is invalid.'
|
||||
);
|
||||
}
|
||||
}
|
||||
$updatedAt = self::policyInteger(
|
||||
$policy,
|
||||
'capacity_updated_at',
|
||||
self::policyInteger($policy, 'updated_at', null)
|
||||
);
|
||||
$configuredCpu = self::policyInteger(
|
||||
$policy, 'configured_cpu_cores'
|
||||
);
|
||||
$configuredMemory = self::policyInteger(
|
||||
$policy, 'configured_memory_mib'
|
||||
);
|
||||
$configuredDisk = self::policyInteger(
|
||||
$policy, 'configured_disk_gib'
|
||||
);
|
||||
$safetyCpu = self::policyInteger($policy, 'safety_cpu_cores');
|
||||
$safetyMemory = self::policyInteger($policy, 'safety_memory_mib');
|
||||
$safetyDisk = self::policyInteger($policy, 'safety_disk_gib');
|
||||
$maxInstances = self::policyInteger($policy, 'max_instances');
|
||||
if ($updatedAt === null || $updatedAt < 1
|
||||
|| $configuredCpu === null || $configuredCpu < 1
|
||||
|| $configuredMemory === null || $configuredMemory < 1
|
||||
|| $configuredDisk === null || $configuredDisk < 1
|
||||
|| $safetyCpu === null || $safetyCpu < 0
|
||||
|| $safetyMemory === null || $safetyMemory < 0
|
||||
|| $safetyDisk === null || $safetyDisk < 0
|
||||
|| $maxInstances === null || $maxInstances < 0) {
|
||||
throw new \InvalidArgumentException(
|
||||
'Capacity policy evidence is invalid.'
|
||||
);
|
||||
}
|
||||
return [
|
||||
'policy_updated_at' => $updatedAt,
|
||||
'requested' => [
|
||||
'cpu_cores' => $resources['cpu_cores'],
|
||||
'memory_mib' => $resources['memory_mib'],
|
||||
'disk_gib' => $resources['disk_gib'],
|
||||
],
|
||||
'used' => [
|
||||
'instance_count' => (int) ($usage['instance_count'] ?? -1),
|
||||
'cpu_cores' => (int) ($usage['cpu_sum'] ?? -1),
|
||||
'memory_mib' => (int) ($usage['memory_sum'] ?? -1),
|
||||
'disk_gib' => (int) ($usage['disk_sum'] ?? -1),
|
||||
],
|
||||
'limits' => [
|
||||
'instance_count' => $maxInstances,
|
||||
'cpu_cores' => max(0, $configuredCpu - $safetyCpu),
|
||||
'memory_mib' => max(0,
|
||||
$configuredMemory - $safetyMemory),
|
||||
'disk_gib' => max(0, $configuredDisk - $safetyDisk),
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
private static function validUsage(array $usage): bool
|
||||
{
|
||||
if (!array_key_exists('unknown', $usage)
|
||||
|| !is_int($usage['unknown']) || $usage['unknown'] !== 0) {
|
||||
return false;
|
||||
}
|
||||
foreach (['instance_count', 'cpu_sum', 'memory_sum', 'disk_sum']
|
||||
as $field) {
|
||||
if (!array_key_exists($field, $usage)
|
||||
|| !is_int($usage[$field]) || $usage[$field] < 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static function policyInteger(object $policy, string $field,
|
||||
?int $fallback = null): ?int
|
||||
{
|
||||
$present = false;
|
||||
$value = null;
|
||||
if (method_exists($policy, 'getAttributes')) {
|
||||
$attributes = $policy->getAttributes();
|
||||
if (array_key_exists($field, $attributes)) {
|
||||
$present = true;
|
||||
$value = $policy->{$field};
|
||||
}
|
||||
} elseif (property_exists($policy, $field)) {
|
||||
$present = true;
|
||||
$value = $policy->{$field};
|
||||
}
|
||||
if (!$present) {
|
||||
return $fallback;
|
||||
}
|
||||
if (is_int($value)) {
|
||||
return $value;
|
||||
}
|
||||
if (is_string($value) && preg_match('/^-?[0-9]+$/D', $value) === 1) {
|
||||
$normalized = filter_var($value, FILTER_VALIDATE_INT);
|
||||
return $normalized === false ? null : (int) $normalized;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,966 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudBackupRestore;
|
||||
use App\Models\CloudBackupRestoreAttempt;
|
||||
use App\Models\CloudHost;
|
||||
use App\Models\CloudHostAgent;
|
||||
use App\Models\CloudHostCapacityPolicy;
|
||||
use App\Models\CloudHostResourcePoolMembership;
|
||||
use App\Models\CloudHostStorageConfiguration;
|
||||
use App\Models\CloudInstance;
|
||||
use App\Models\CloudInstanceBackup;
|
||||
use App\Models\CloudJob;
|
||||
use App\Models\CloudNetworkLine;
|
||||
use App\Models\CloudRegion;
|
||||
use App\Models\CloudResourcePool;
|
||||
use App\Models\CloudResourceReservation;
|
||||
use App\Models\CloudResourceReservationIp;
|
||||
use App\Models\User;
|
||||
use Illuminate\Database\Capsule\Manager as DB;
|
||||
|
||||
/**
|
||||
* Atomically reserves a future portable-restore attempt without dispatching it.
|
||||
*
|
||||
* No instance or job is created here. The externally reachable restore gate
|
||||
* remains closed; this is only the durable local admission boundary.
|
||||
*/
|
||||
final class CloudPortableRestoreAttemptCoordinator
|
||||
{
|
||||
public static function reserve(array $actor, CloudInstanceBackup $backup,
|
||||
CloudPortableRestoreTargetPlan $plan, string $planSha256,
|
||||
string $targetName, string $requestKey
|
||||
): CloudPortableRestoreAttemptReservation {
|
||||
return self::coordinate(
|
||||
$actor,
|
||||
$backup,
|
||||
$plan,
|
||||
$planSha256,
|
||||
$targetName,
|
||||
$requestKey,
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
/** A new generation requires an explicit retry of one exact predecessor. */
|
||||
public static function retry(array $actor, CloudInstanceBackup $backup,
|
||||
CloudPortableRestoreTargetPlan $plan, string $planSha256,
|
||||
string $targetName, string $requestKey, string $priorAttemptKey
|
||||
): CloudPortableRestoreAttemptReservation {
|
||||
if (preg_match('/^[a-f0-9]{32}$/D', $priorAttemptKey) !== 1) {
|
||||
throw new CloudError('恢复重试身份无效', 409);
|
||||
}
|
||||
return self::coordinate(
|
||||
$actor,
|
||||
$backup,
|
||||
$plan,
|
||||
$planSha256,
|
||||
$targetName,
|
||||
$requestKey,
|
||||
$priorAttemptKey
|
||||
);
|
||||
}
|
||||
|
||||
private static function coordinate(array $actor,
|
||||
CloudInstanceBackup $backup, CloudPortableRestoreTargetPlan $plan,
|
||||
string $planSha256, string $targetName, string $requestKey,
|
||||
?string $retryAttemptKey
|
||||
): CloudPortableRestoreAttemptReservation {
|
||||
CloudAccess::manage($actor);
|
||||
$actorUserId = filter_var(
|
||||
$actor['user_id'] ?? null,
|
||||
FILTER_VALIDATE_INT,
|
||||
['options' => ['min_range' => 1]]
|
||||
);
|
||||
if ($actorUserId === false) {
|
||||
throw new CloudError('恢复请求用户身份无效', 403);
|
||||
}
|
||||
if (preg_match('/^[A-Za-z0-9_.-]{16,128}$/D', $requestKey) !== 1) {
|
||||
throw new CloudError('恢复请求幂等身份无效', 409);
|
||||
}
|
||||
$targetName = CloudInput::name($targetName);
|
||||
if (preg_match('/^[a-f0-9]{64}$/D', $planSha256) !== 1
|
||||
|| !hash_equals($plan->sha256(), $planSha256)) {
|
||||
throw new CloudError('恢复目标计划身份无效', 409);
|
||||
}
|
||||
$planValue = $plan->toArray();
|
||||
$constraint = self::bundleConstraint($planValue);
|
||||
$backupId = (int) $backup->id;
|
||||
if ($backupId < 1) {
|
||||
throw new CloudError('备份不存在', 404);
|
||||
}
|
||||
$ownerUserId = (int) $backup->user_id;
|
||||
if ($ownerUserId < 1) {
|
||||
throw new CloudError('备份缺少资源所有者', 409);
|
||||
}
|
||||
$fingerprint = self::requestFingerprint(
|
||||
(int) $actorUserId,
|
||||
$ownerUserId,
|
||||
$planValue['backup'],
|
||||
$planSha256,
|
||||
$targetName
|
||||
);
|
||||
|
||||
return $backup->getConnection()->transaction(static function () use (
|
||||
$actorUserId,
|
||||
$ownerUserId,
|
||||
$backupId,
|
||||
$plan,
|
||||
$planValue,
|
||||
$planSha256,
|
||||
$constraint,
|
||||
$fingerprint,
|
||||
$targetName,
|
||||
$requestKey,
|
||||
$retryAttemptKey
|
||||
): CloudPortableRestoreAttemptReservation {
|
||||
$userIds = array_values(array_unique([
|
||||
(int) $actorUserId,
|
||||
$ownerUserId,
|
||||
]));
|
||||
sort($userIds, SORT_NUMERIC);
|
||||
$users = User::whereIn('id', $userIds)->orderBy('id')
|
||||
->lockForUpdate()->get()->keyBy('id');
|
||||
$requester = $users->get((int) $actorUserId);
|
||||
$owner = $users->get($ownerUserId);
|
||||
$lockedBackup = CloudInstanceBackup::where('id', $backupId)
|
||||
->lockForUpdate()->first();
|
||||
if (!$requester instanceof User
|
||||
|| (int) $requester->enable !== 1) {
|
||||
throw new CloudError('恢复请求用户不可用', 403);
|
||||
}
|
||||
if (!$owner instanceof User
|
||||
|| !$lockedBackup instanceof CloudInstanceBackup
|
||||
|| (int) $lockedBackup->user_id !== $ownerUserId) {
|
||||
throw new CloudError('备份资源所有者不一致', 409);
|
||||
}
|
||||
// Hints only identify the request's own reservation for capacity
|
||||
// subtraction. They are verified under locks before any result.
|
||||
$restoreHint = CloudBackupRestore::where(
|
||||
'requested_by_user_id',
|
||||
(int) $actorUserId
|
||||
)->where('request_idempotency_key', $requestKey)->first();
|
||||
$attemptHint = $restoreHint instanceof CloudBackupRestore
|
||||
? CloudBackupRestoreAttempt::where(
|
||||
'backup_restore_id',
|
||||
(int) $restoreHint->id
|
||||
)->orderByDesc('generation')->first()
|
||||
: null;
|
||||
if ($restoreHint instanceof CloudBackupRestore) {
|
||||
self::assertRequestReplay(
|
||||
$restoreHint,
|
||||
$lockedBackup,
|
||||
$plan,
|
||||
$fingerprint,
|
||||
(int) $actorUserId,
|
||||
$targetName
|
||||
);
|
||||
}
|
||||
$fastReplay = $restoreHint instanceof CloudBackupRestore
|
||||
&& ($retryAttemptKey === null
|
||||
|| !($attemptHint instanceof CloudBackupRestoreAttempt)
|
||||
|| !hash_equals(
|
||||
(string) $attemptHint->attempt_key,
|
||||
$retryAttemptKey
|
||||
));
|
||||
if ($fastReplay) {
|
||||
$host = CloudHost::where(
|
||||
'id',
|
||||
(int) $planValue['target']['host_id']
|
||||
)->lockForUpdate()->first();
|
||||
if (!$host instanceof CloudHost) {
|
||||
throw new CloudError('恢复目标宿主机账本缺失', 409);
|
||||
}
|
||||
$restore = CloudBackupRestore::where(
|
||||
'id',
|
||||
(int) $restoreHint->id
|
||||
)->where(
|
||||
'requested_by_user_id',
|
||||
(int) $actorUserId
|
||||
)->where('request_idempotency_key', $requestKey)
|
||||
->lockForUpdate()->first();
|
||||
if (!$restore instanceof CloudBackupRestore) {
|
||||
throw new CloudError('恢复请求幂等账本已经变化', 409);
|
||||
}
|
||||
self::assertRequestReplay(
|
||||
$restore,
|
||||
$lockedBackup,
|
||||
$plan,
|
||||
$fingerprint,
|
||||
(int) $actorUserId,
|
||||
$targetName
|
||||
);
|
||||
$attempt = CloudBackupRestoreAttempt::where(
|
||||
'backup_restore_id',
|
||||
(int) $restore->id
|
||||
)->orderByDesc('generation')->lockForUpdate()->first();
|
||||
if (!$attempt instanceof CloudBackupRestoreAttempt) {
|
||||
throw new CloudError('恢复请求缺少尝试账本', 409);
|
||||
}
|
||||
self::assertAttemptPlan($restore, $attempt, $plan);
|
||||
if ($retryAttemptKey !== null) {
|
||||
self::assertRetrySuccessor(
|
||||
$restore,
|
||||
$attempt,
|
||||
$retryAttemptKey,
|
||||
$plan,
|
||||
$ownerUserId
|
||||
);
|
||||
}
|
||||
return self::replay(
|
||||
$restore,
|
||||
$attempt,
|
||||
$plan,
|
||||
$ownerUserId
|
||||
);
|
||||
}
|
||||
|
||||
$backupEvidence = CloudPortableRestoreBackupEvidence::fromBackup(
|
||||
$lockedBackup
|
||||
);
|
||||
if (self::canonical($backupEvidence->backup())
|
||||
!== self::canonical($planValue['backup'])
|
||||
|| self::canonical($backupEvidence->resources())
|
||||
!== self::canonical($planValue['resources'])) {
|
||||
throw new CloudError('备份身份或冻结规格已经变化', 409);
|
||||
}
|
||||
$excludedReservationId = $attemptHint
|
||||
instanceof CloudBackupRestoreAttempt
|
||||
? (int) $attemptHint->reservation_id : 0;
|
||||
$requiresAdmission = true;
|
||||
|
||||
$target = $planValue['target'];
|
||||
// Match the established placement order exactly.
|
||||
$region = CloudRegion::where('id', (int) $target['region_id'])
|
||||
->lockForUpdate()->first();
|
||||
$line = CloudNetworkLine::where('id', (int) $target['line_id'])
|
||||
->lockForUpdate()->first();
|
||||
$pool = CloudResourcePool::where(
|
||||
'id',
|
||||
(int) $target['resource_pool_id']
|
||||
)->lockForUpdate()->first();
|
||||
$host = CloudHost::where('id', (int) $target['host_id'])
|
||||
->lockForUpdate()->first();
|
||||
if (!$region instanceof CloudRegion
|
||||
|| !$line instanceof CloudNetworkLine
|
||||
|| !$pool instanceof CloudResourcePool
|
||||
|| !$host instanceof CloudHost) {
|
||||
throw new CloudError('恢复目标拓扑账本缺失', 409);
|
||||
}
|
||||
if ($requiresAdmission) {
|
||||
$membership = CloudHostResourcePoolMembership::where(
|
||||
'host_id',
|
||||
(int) $target['host_id']
|
||||
)->where(
|
||||
'resource_pool_id',
|
||||
(int) $target['resource_pool_id']
|
||||
)->lockForUpdate()->first();
|
||||
$policy = CloudHostCapacityPolicy::where(
|
||||
'host_id',
|
||||
(int) $target['host_id']
|
||||
)->lockForUpdate()->first();
|
||||
$storage = CloudHostStorageConfiguration::where(
|
||||
'host_id',
|
||||
(int) $target['host_id']
|
||||
)->lockForUpdate()->first();
|
||||
$agent = CloudHostAgent::where(
|
||||
'host_id',
|
||||
(int) $target['host_id']
|
||||
)->lockForUpdate()->first();
|
||||
self::assertTargetEvidence(
|
||||
$planValue,
|
||||
$region,
|
||||
$line,
|
||||
$pool,
|
||||
$host,
|
||||
$membership,
|
||||
$storage,
|
||||
$agent,
|
||||
$backupEvidence
|
||||
);
|
||||
$reservationRows = CloudResourceReservation::where(
|
||||
'host_id',
|
||||
(int) $target['host_id']
|
||||
)->whereIn('state', ['held', 'uncertain', 'committed'])
|
||||
->orderBy('id')->lockForUpdate()->get([
|
||||
'id', 'instance_id', 'state', 'cpu_cores',
|
||||
'memory_mib', 'disk_gib', 'updated_at',
|
||||
]);
|
||||
$capacityRows = $excludedReservationId > 0
|
||||
? $reservationRows->reject(
|
||||
static fn ($row): bool =>
|
||||
(int) $row->id === $excludedReservationId
|
||||
)->values()
|
||||
: $reservationRows;
|
||||
$usage = CloudPanelUsageService::forHost(
|
||||
(int) $target['host_id'],
|
||||
$capacityRows
|
||||
);
|
||||
if (!CloudHostCapacityAdmission::allows(
|
||||
$policy,
|
||||
$usage,
|
||||
$backupEvidence->capacityResources()
|
||||
)) {
|
||||
throw new CloudError('恢复目标容量已经不足', 409);
|
||||
}
|
||||
$capacityEvidence = CloudHostCapacityAdmission::evidence(
|
||||
$policy,
|
||||
$usage,
|
||||
$backupEvidence->capacityResources()
|
||||
);
|
||||
if (!($restoreHint instanceof CloudBackupRestore)
|
||||
&& self::canonical($capacityEvidence)
|
||||
!== self::canonical($planValue['capacity'])) {
|
||||
throw new CloudError('恢复目标容量计划已经过期', 409);
|
||||
}
|
||||
if ($restoreHint instanceof CloudBackupRestore
|
||||
&& !self::sameCapacityContract(
|
||||
$capacityEvidence,
|
||||
$planValue['capacity']
|
||||
)) {
|
||||
throw new CloudError('恢复目标容量合同已经变化', 409);
|
||||
}
|
||||
self::lockPlannedNetworkCatalog($planValue, $constraint);
|
||||
}
|
||||
|
||||
$restore = CloudBackupRestore::where(
|
||||
'requested_by_user_id',
|
||||
(int) $actorUserId
|
||||
)->where('request_idempotency_key', $requestKey)
|
||||
->lockForUpdate()->first();
|
||||
if (!$restore instanceof CloudBackupRestore) {
|
||||
if ($retryAttemptKey !== null) {
|
||||
throw new CloudError('恢复重试的原请求不存在', 409);
|
||||
}
|
||||
self::assertTargetNameAvailable(
|
||||
(int) $target['host_id'],
|
||||
$targetName,
|
||||
0,
|
||||
0
|
||||
);
|
||||
$operationId = self::key();
|
||||
$restore = new CloudBackupRestore();
|
||||
$restore->restore_key = self::key();
|
||||
$restore->backup_id = (int) $lockedBackup->id;
|
||||
$restore->requested_by_user_id = (int) $actorUserId;
|
||||
$restore->request_idempotency_key = $requestKey;
|
||||
$restore->request_fingerprint = $fingerprint;
|
||||
$restore->selected_plan_json = $plan->json();
|
||||
$restore->selected_plan_sha256 = $planSha256;
|
||||
$restore->target_host_id = (int) $target['host_id'];
|
||||
$restore->target_instance_id = null;
|
||||
$restore->reservation_id = null;
|
||||
$restore->job_id = null;
|
||||
$restore->operation_id = $operationId;
|
||||
$restore->new_uuid = self::uuidV4();
|
||||
$restore->new_name = $targetName;
|
||||
$restore->state = 'restoring';
|
||||
$restore->error_code = '';
|
||||
$restore->ready_at = null;
|
||||
$restore->created_at = $restore->updated_at = time();
|
||||
$restore->save();
|
||||
return self::createGeneration(
|
||||
$restore,
|
||||
$plan,
|
||||
$constraint,
|
||||
$backupEvidence,
|
||||
$ownerUserId,
|
||||
1,
|
||||
$operationId
|
||||
);
|
||||
}
|
||||
self::assertRequestReplay(
|
||||
$restore,
|
||||
$lockedBackup,
|
||||
$plan,
|
||||
$fingerprint,
|
||||
(int) $actorUserId,
|
||||
$targetName
|
||||
);
|
||||
$attempt = CloudBackupRestoreAttempt::where(
|
||||
'backup_restore_id',
|
||||
(int) $restore->id
|
||||
)->orderByDesc('generation')->lockForUpdate()->first();
|
||||
if (!$attempt instanceof CloudBackupRestoreAttempt) {
|
||||
throw new CloudError('恢复请求缺少尝试账本', 409);
|
||||
}
|
||||
self::assertAttemptPlan($restore, $attempt, $plan);
|
||||
if ($retryAttemptKey === null) {
|
||||
return self::replay(
|
||||
$restore,
|
||||
$attempt,
|
||||
$plan,
|
||||
$ownerUserId
|
||||
);
|
||||
}
|
||||
if (!hash_equals(
|
||||
(string) $attempt->attempt_key,
|
||||
$retryAttemptKey
|
||||
)) {
|
||||
self::assertRetrySuccessor(
|
||||
$restore,
|
||||
$attempt,
|
||||
$retryAttemptKey,
|
||||
$plan,
|
||||
$ownerUserId
|
||||
);
|
||||
return self::replay(
|
||||
$restore,
|
||||
$attempt,
|
||||
$plan,
|
||||
$ownerUserId
|
||||
);
|
||||
}
|
||||
self::assertTargetNameAvailable(
|
||||
(int) $target['host_id'],
|
||||
$targetName,
|
||||
(int) $attempt->id,
|
||||
$restore->target_instance_id !== null
|
||||
? (int) $restore->target_instance_id : 0
|
||||
);
|
||||
self::assertRetryable(
|
||||
$restore,
|
||||
$attempt,
|
||||
$retryAttemptKey,
|
||||
$plan,
|
||||
$ownerUserId
|
||||
);
|
||||
return self::createGeneration(
|
||||
$restore,
|
||||
$plan,
|
||||
$constraint,
|
||||
$backupEvidence,
|
||||
$ownerUserId,
|
||||
(int) $attempt->generation + 1,
|
||||
self::key()
|
||||
);
|
||||
}, 3);
|
||||
}
|
||||
|
||||
private static function createGeneration(CloudBackupRestore $restore,
|
||||
CloudPortableRestoreTargetPlan $plan,
|
||||
CloudPublicNetworkBundleConstraint $constraint,
|
||||
CloudPortableRestoreBackupEvidence $backupEvidence, int $ownerUserId,
|
||||
int $generation, string $operationId
|
||||
): CloudPortableRestoreAttemptReservation
|
||||
{
|
||||
$value = $plan->toArray();
|
||||
$attemptKey = self::key();
|
||||
$now = time();
|
||||
$reservation = new CloudResourceReservation();
|
||||
$reservation->reservation_key = self::key();
|
||||
$reservation->operation_id = $operationId;
|
||||
$reservation->order_id = null;
|
||||
$reservation->backup_restore_id = null;
|
||||
$reservation->backup_restore_attempt_key = $attemptKey;
|
||||
$reservation->migration_source_debt_id = null;
|
||||
$reservation->offer_id = null;
|
||||
$reservation->resource_pool_id =
|
||||
(int) $value['target']['resource_pool_id'];
|
||||
$reservation->host_id = (int) $value['target']['host_id'];
|
||||
$reservation->user_id = $ownerUserId;
|
||||
$reservation->instance_id = null;
|
||||
$reservation->network_attachment_id = null;
|
||||
$reservation->public_ip_pool_id = null;
|
||||
$reservation->public_ip_id = null;
|
||||
$resources = $backupEvidence->capacityResources();
|
||||
$reservation->cpu_cores = $resources['cpu_cores'];
|
||||
$reservation->memory_mib = $resources['memory_mib'];
|
||||
$reservation->disk_gib = $resources['disk_gib'];
|
||||
$reservation->state = 'held';
|
||||
$reservation->expires_at = null;
|
||||
$reservation->released_at = null;
|
||||
$reservation->release_reason = '';
|
||||
$reservation->created_at = $reservation->updated_at = $now;
|
||||
$reservation->save();
|
||||
|
||||
CloudPublicNetworkAllocationService::reservePlannedBundle(
|
||||
(int) $reservation->host_id,
|
||||
(int) $value['target']['line_id'],
|
||||
(int) $reservation->id,
|
||||
$operationId,
|
||||
$constraint
|
||||
);
|
||||
$claimSet = CloudPublicNetworkAllocationService::currentClaimSet(
|
||||
(int) $reservation->id,
|
||||
'held'
|
||||
);
|
||||
$frozen = CloudFrozenPublicNetworkIdentity::fromClaimSet(
|
||||
(string) $restore->new_uuid,
|
||||
$claimSet
|
||||
);
|
||||
$reservation->network_attachment_id =
|
||||
$claimSet->networkAttachmentId();
|
||||
$reservation->updated_at = time();
|
||||
$reservation->save();
|
||||
|
||||
$attempt = new CloudBackupRestoreAttempt();
|
||||
$attempt->attempt_key = $attemptKey;
|
||||
$attempt->backup_restore_id = (int) $restore->id;
|
||||
$attempt->generation = $generation;
|
||||
$attempt->operation_id = $operationId;
|
||||
$attempt->job_id = null;
|
||||
$attempt->reservation_id = (int) $reservation->id;
|
||||
$attempt->target_host_id = (int) $reservation->host_id;
|
||||
$attempt->target_uuid = (string) $restore->new_uuid;
|
||||
$attempt->target_name = (string) $restore->new_name;
|
||||
$attempt->target_plan_json = $plan->json();
|
||||
$attempt->target_plan_sha256 = $plan->sha256();
|
||||
$attempt->target_network_attachment_id =
|
||||
$claimSet->networkAttachmentId();
|
||||
$attempt->target_identity_json = $frozen->identityJson();
|
||||
$attempt->target_identity_sha256 = $frozen->identitySha256();
|
||||
$attempt->target_claim_set_sha256 = $frozen->claimSetSha256();
|
||||
$attempt->target_public_network_json =
|
||||
$frozen->publicNetworkJson();
|
||||
$attempt->target_public_network_sha256 =
|
||||
$frozen->publicNetworkSha256();
|
||||
$attempt->state = 'reserved';
|
||||
$attempt->effect_resolution = 'not_applicable';
|
||||
$attempt->active_attempt = 1;
|
||||
$attempt->last_error_code = '';
|
||||
$attempt->effect_resolved_at = null;
|
||||
$attempt->terminal_at = null;
|
||||
$attempt->created_at = $attempt->updated_at = $now;
|
||||
$attempt->save();
|
||||
|
||||
// These legacy parent columns are an initial projection only. Later
|
||||
// generations never rewrite them; the attempt row is authoritative.
|
||||
if ($generation === 1) {
|
||||
$restore->reservation_id = (int) $reservation->id;
|
||||
}
|
||||
$restore->state = 'restoring';
|
||||
$restore->error_code = '';
|
||||
$restore->updated_at = time();
|
||||
$restore->save();
|
||||
return CloudPortableRestoreAttemptReservation::active(
|
||||
$restore,
|
||||
$attempt,
|
||||
$reservation,
|
||||
$claimSet,
|
||||
$frozen,
|
||||
$plan
|
||||
);
|
||||
}
|
||||
|
||||
private static function replay(CloudBackupRestore $restore,
|
||||
CloudBackupRestoreAttempt $attempt,
|
||||
CloudPortableRestoreTargetPlan $plan, int $ownerUserId
|
||||
): CloudPortableRestoreAttemptReservation {
|
||||
$reservation = $attempt->boundReservationQuery()
|
||||
->lockForUpdate()->first();
|
||||
if (!$reservation instanceof CloudResourceReservation
|
||||
|| (int) $reservation->host_id !== (int) $attempt->target_host_id
|
||||
|| (int) $reservation->resource_pool_id
|
||||
!== (int) $plan->toArray()['target']['resource_pool_id']
|
||||
|| (int) $reservation->user_id
|
||||
!== $ownerUserId
|
||||
|| !hash_equals(
|
||||
(string) $reservation->operation_id,
|
||||
(string) $attempt->operation_id
|
||||
)) {
|
||||
throw new CloudError('恢复尝试资源预留身份不一致', 409);
|
||||
}
|
||||
if ((string) $attempt->state === 'failed'
|
||||
&& (string) $attempt->effect_resolution === 'not_applied') {
|
||||
return self::terminalResult(
|
||||
$restore,
|
||||
$attempt,
|
||||
$reservation,
|
||||
$plan,
|
||||
$ownerUserId
|
||||
);
|
||||
}
|
||||
if (!in_array((string) $reservation->state,
|
||||
['held', 'uncertain', 'committed'], true)) {
|
||||
throw new CloudError('恢复尝试资源预留状态不明确', 409);
|
||||
}
|
||||
CloudPublicNetworkAllocationService::verifyExistingBundle(
|
||||
(int) $reservation->host_id,
|
||||
(int) $plan->toArray()['target']['line_id'],
|
||||
(int) $reservation->id,
|
||||
(string) $attempt->operation_id
|
||||
);
|
||||
$claimSet = CloudPublicNetworkAllocationService::currentClaimSet(
|
||||
(int) $reservation->id,
|
||||
(string) $reservation->state
|
||||
);
|
||||
try {
|
||||
$frozen = CloudFrozenPublicNetworkIdentity::fromStored(
|
||||
(int) $reservation->id,
|
||||
(int) $attempt->target_host_id,
|
||||
(string) $attempt->target_uuid,
|
||||
(int) $attempt->target_network_attachment_id,
|
||||
(string) $attempt->target_identity_json,
|
||||
(string) $attempt->target_identity_sha256,
|
||||
(string) $attempt->target_public_network_json,
|
||||
(string) $attempt->target_public_network_sha256,
|
||||
(string) $attempt->target_claim_set_sha256,
|
||||
$claimSet
|
||||
);
|
||||
} catch (\InvalidArgumentException) {
|
||||
throw new CloudError('恢复尝试冻结公网身份损坏', 409);
|
||||
}
|
||||
return CloudPortableRestoreAttemptReservation::active(
|
||||
$restore,
|
||||
$attempt,
|
||||
$reservation,
|
||||
$claimSet,
|
||||
$frozen,
|
||||
$plan
|
||||
);
|
||||
}
|
||||
|
||||
private static function terminalResult(CloudBackupRestore $restore,
|
||||
CloudBackupRestoreAttempt $attempt,
|
||||
CloudResourceReservation $reservation,
|
||||
CloudPortableRestoreTargetPlan $plan, int $ownerUserId
|
||||
): CloudPortableRestoreAttemptReservation {
|
||||
$ids = CloudReleasedPublicNetworkBundleVerifier::verify(
|
||||
$attempt,
|
||||
$reservation,
|
||||
$ownerUserId
|
||||
);
|
||||
return CloudPortableRestoreAttemptReservation::terminal(
|
||||
$restore,
|
||||
$attempt,
|
||||
$reservation,
|
||||
$plan,
|
||||
$ids
|
||||
);
|
||||
}
|
||||
|
||||
private static function assertRetryable(CloudBackupRestore $restore,
|
||||
CloudBackupRestoreAttempt $attempt, string $priorAttemptKey,
|
||||
CloudPortableRestoreTargetPlan $plan, int $ownerUserId): void
|
||||
{
|
||||
$job = $attempt->job_id === null ? null
|
||||
: CloudJob::where('id', (int) $attempt->job_id)
|
||||
->lockForUpdate()->first();
|
||||
if (!hash_equals((string) $attempt->attempt_key, $priorAttemptKey)
|
||||
|| (string) $attempt->state !== 'failed'
|
||||
|| (string) $attempt->effect_resolution !== 'not_applied'
|
||||
|| $attempt->active_attempt !== null
|
||||
|| $attempt->effect_resolved_at === null
|
||||
|| $attempt->terminal_at === null
|
||||
|| !$job instanceof CloudJob
|
||||
|| (int) $job->id !== (int) $attempt->job_id
|
||||
|| (string) $job->kind !== 'backup_restore'
|
||||
|| (int) $job->host_id !== (int) $attempt->target_host_id
|
||||
|| (int) $job->user_id !== $ownerUserId
|
||||
|| (int) $job->backup_restore_id
|
||||
!== (int) $restore->id
|
||||
|| !hash_equals(
|
||||
(string) $job->operation_id,
|
||||
(string) $attempt->operation_id
|
||||
)
|
||||
|| (string) $job->phase !== 'terminal'
|
||||
|| (string) $job->outcome !== 'failed'
|
||||
|| (string) $job->effect_resolution !== 'not_applied'
|
||||
|| (int) $job->terminal_at < 1
|
||||
|| (int) $job->finalization_incident !== 0) {
|
||||
throw new CloudError('恢复尝试尚无可重试的未生效终态证据', 409);
|
||||
}
|
||||
$reservation = $attempt->boundReservationQuery()
|
||||
->lockForUpdate()->first();
|
||||
if (!$reservation instanceof CloudResourceReservation) {
|
||||
throw new CloudError('恢复重试缺少原资源预留', 409);
|
||||
}
|
||||
self::terminalResult(
|
||||
$restore,
|
||||
$attempt,
|
||||
$reservation,
|
||||
$plan,
|
||||
$ownerUserId
|
||||
);
|
||||
}
|
||||
|
||||
private static function assertRetrySuccessor(CloudBackupRestore $restore,
|
||||
CloudBackupRestoreAttempt $latest, string $priorAttemptKey,
|
||||
CloudPortableRestoreTargetPlan $plan, int $ownerUserId): void
|
||||
{
|
||||
$prior = CloudBackupRestoreAttempt::where(
|
||||
'backup_restore_id',
|
||||
(int) $restore->id
|
||||
)->where('attempt_key', $priorAttemptKey)->lockForUpdate()->first();
|
||||
if (!$prior instanceof CloudBackupRestoreAttempt
|
||||
|| (int) $latest->generation !== (int) $prior->generation + 1) {
|
||||
throw new CloudError('恢复重试身份与最新尝试不一致', 409);
|
||||
}
|
||||
self::assertAttemptPlan($restore, $prior, $plan);
|
||||
self::assertRetryable(
|
||||
$restore,
|
||||
$prior,
|
||||
$priorAttemptKey,
|
||||
$plan,
|
||||
$ownerUserId
|
||||
);
|
||||
}
|
||||
|
||||
private static function assertRequestReplay(CloudBackupRestore $restore,
|
||||
CloudInstanceBackup $backup, CloudPortableRestoreTargetPlan $plan,
|
||||
string $fingerprint, int $actorUserId, string $targetName): void
|
||||
{
|
||||
try {
|
||||
$storedPlan = CloudPortableRestoreTargetPlan::fromStored(
|
||||
(string) $restore->selected_plan_json,
|
||||
(string) $restore->selected_plan_sha256
|
||||
);
|
||||
} catch (\InvalidArgumentException) {
|
||||
throw new CloudError('恢复请求保存的目标计划损坏', 409);
|
||||
}
|
||||
if ((int) $restore->backup_id !== (int) $backup->id
|
||||
|| (int) $restore->requested_by_user_id !== $actorUserId
|
||||
|| !hash_equals(
|
||||
(string) $restore->request_fingerprint,
|
||||
$fingerprint
|
||||
)
|
||||
|| !hash_equals($storedPlan->sha256(), $plan->sha256())
|
||||
|| !hash_equals((string) $restore->new_name, $targetName)
|
||||
|| (int) $restore->target_host_id
|
||||
!== (int) $plan->toArray()['target']['host_id']) {
|
||||
throw new CloudError('恢复请求幂等身份冲突', 409);
|
||||
}
|
||||
}
|
||||
|
||||
private static function assertAttemptPlan(CloudBackupRestore $restore,
|
||||
CloudBackupRestoreAttempt $attempt,
|
||||
CloudPortableRestoreTargetPlan $plan): void
|
||||
{
|
||||
try {
|
||||
$stored = CloudPortableRestoreTargetPlan::fromStored(
|
||||
(string) $attempt->target_plan_json,
|
||||
(string) $attempt->target_plan_sha256
|
||||
);
|
||||
} catch (\InvalidArgumentException) {
|
||||
throw new CloudError('恢复尝试保存的目标计划损坏', 409);
|
||||
}
|
||||
if (!hash_equals($stored->sha256(), $plan->sha256())
|
||||
|| !hash_equals(
|
||||
(string) $restore->selected_plan_sha256,
|
||||
(string) $attempt->target_plan_sha256
|
||||
)
|
||||
|| !hash_equals(
|
||||
(string) $restore->selected_plan_json,
|
||||
(string) $attempt->target_plan_json
|
||||
)
|
||||
|| (int) $attempt->target_host_id
|
||||
!== (int) $restore->target_host_id
|
||||
|| !hash_equals(
|
||||
strtolower((string) $attempt->target_uuid),
|
||||
strtolower((string) $restore->new_uuid)
|
||||
)
|
||||
|| !hash_equals(
|
||||
(string) $attempt->target_name,
|
||||
(string) $restore->new_name
|
||||
)) {
|
||||
throw new CloudError('恢复尝试与父请求身份冲突', 409);
|
||||
}
|
||||
}
|
||||
|
||||
private static function assertTargetEvidence(array $plan,
|
||||
?CloudRegion $region, ?CloudNetworkLine $line,
|
||||
?CloudResourcePool $pool, ?CloudHost $host,
|
||||
?CloudHostResourcePoolMembership $membership,
|
||||
?CloudHostStorageConfiguration $storage, ?CloudHostAgent $agent,
|
||||
CloudPortableRestoreBackupEvidence $backup): void
|
||||
{
|
||||
$target = $plan['target'];
|
||||
$targetContract = $agent?->testedPortableRestoreCompatibility();
|
||||
if (!$region instanceof CloudRegion
|
||||
|| !$line instanceof CloudNetworkLine
|
||||
|| !$pool instanceof CloudResourcePool
|
||||
|| !$host instanceof CloudHost
|
||||
|| !$membership instanceof CloudHostResourcePoolMembership
|
||||
|| !$storage instanceof CloudHostStorageConfiguration
|
||||
|| !$agent instanceof CloudHostAgent
|
||||
|| !(bool) $host->enabled
|
||||
|| (string) $region->status !== 'active'
|
||||
|| (string) $line->status !== 'active'
|
||||
|| (string) $pool->status !== 'active'
|
||||
|| (string) $membership->scheduler_state !== 'active'
|
||||
|| !$storage->isApplied() || !$agent->isReady()
|
||||
|| (int) $line->region_id !== (int) $region->id
|
||||
|| (int) $pool->line_id !== (int) $line->id
|
||||
|| (int) $membership->revision
|
||||
!== (int) $target['membership_revision']
|
||||
|| (int) $storage->revision
|
||||
!== (int) $target['storage_revision']
|
||||
|| (int) $agent->revision !== (int) $target['agent_revision']
|
||||
|| $targetContract === null
|
||||
|| !$backup->compatibility()->matches($targetContract)
|
||||
|| !hash_equals(
|
||||
$targetContract->sha256(),
|
||||
(string) $target['agent_contract_sha256']
|
||||
)
|
||||
|| !hash_equals((string) $host->name,
|
||||
(string) $target['host_name'])
|
||||
|| !hash_equals((string) $pool->name,
|
||||
(string) $target['resource_pool_name'])
|
||||
|| !hash_equals((string) $line->name,
|
||||
(string) $target['line_name'])
|
||||
|| !hash_equals((string) $region->name,
|
||||
(string) $target['region_name'])
|
||||
|| !hash_equals(strtoupper((string) $region->country_code),
|
||||
(string) $target['country_code'])
|
||||
|| (bool) $target['is_source_host']
|
||||
!== ((int) $host->id
|
||||
=== (int) $plan['backup']['source_host_id'])) {
|
||||
throw new CloudError('恢复目标宿主机计划已经过期', 409);
|
||||
}
|
||||
}
|
||||
|
||||
private static function assertTargetNameAvailable(int $hostId,
|
||||
string $targetName, int $ownAttemptId,
|
||||
int $ownTargetInstanceId): void
|
||||
{
|
||||
$instances = CloudInstance::where('host_id', $hostId)
|
||||
->where('name', $targetName)->where('status', '<>', 'deleted');
|
||||
if ($ownTargetInstanceId > 0) {
|
||||
$instances->where('id', '<>', $ownTargetInstanceId);
|
||||
}
|
||||
if ($instances->orderBy('id')->lockForUpdate()->get(['id'])
|
||||
->isNotEmpty()) {
|
||||
throw new CloudError('目标宿主机已有同名实例', 409);
|
||||
}
|
||||
$query = CloudBackupRestoreAttempt::where('target_host_id', $hostId)
|
||||
->where('target_name', $targetName)
|
||||
->where('active_attempt', 1);
|
||||
if ($ownAttemptId > 0) {
|
||||
$query->where('id', '<>', $ownAttemptId);
|
||||
}
|
||||
if ($query->orderBy('id')->lockForUpdate()->get(['id'])
|
||||
->isNotEmpty()) {
|
||||
throw new CloudError('目标宿主机已有同名恢复尝试', 409);
|
||||
}
|
||||
}
|
||||
|
||||
private static function lockPlannedNetworkCatalog(array $plan,
|
||||
CloudPublicNetworkBundleConstraint $constraint): void
|
||||
{
|
||||
$network = $plan['public_network'];
|
||||
$attachments = DB::table('cloud_host_network_attachments')
|
||||
->where('id', (int) $network['network_attachment_id'])
|
||||
->where('host_id', (int) $network['host_id'])
|
||||
->where('line_id', (int) $network['line_id'])
|
||||
->where('state', 'active')->orderBy('id')
|
||||
->lockForUpdate()->get()->all();
|
||||
$attachmentIds = array_map(
|
||||
static fn (object $row): int => (int) $row->id,
|
||||
$attachments
|
||||
);
|
||||
$pools = $attachmentIds === [] ? []
|
||||
: DB::table('cloud_public_ip_pools')
|
||||
->whereIn('id', [
|
||||
(int) $network['ipv4_pool']['public_ip_pool_id'],
|
||||
(int) $network['ipv6_pool']['public_ip_pool_id'],
|
||||
])
|
||||
->where('host_id', (int) $network['host_id'])
|
||||
->whereIn('network_attachment_id', $attachmentIds)
|
||||
->where('state', 'active')->whereIn('address_family', [4, 6])
|
||||
->orderBy('id')->lockForUpdate()->get()->all();
|
||||
$attachment = null;
|
||||
foreach ($attachments as $row) {
|
||||
if ((int) $row->id === (int) $network['network_attachment_id']) {
|
||||
$attachment = $row;
|
||||
break;
|
||||
}
|
||||
}
|
||||
$selectedPools = [];
|
||||
foreach ($pools as $row) {
|
||||
$family = (int) $row->address_family;
|
||||
$poolKey = $family === 4 ? 'ipv4_pool'
|
||||
: ($family === 6 ? 'ipv6_pool' : null);
|
||||
if ($poolKey !== null
|
||||
&& (int) $row->network_attachment_id
|
||||
=== (int) $network['network_attachment_id']
|
||||
&& (int) $row->id
|
||||
=== (int) $network[$poolKey]['public_ip_pool_id']) {
|
||||
$selectedPools[$family] = $row;
|
||||
}
|
||||
}
|
||||
if (!is_object($attachment)
|
||||
|| !isset($selectedPools[4], $selectedPools[6])) {
|
||||
throw new CloudError('恢复目标双栈网络计划已经过期', 409);
|
||||
}
|
||||
$constraint->assertMatches($attachment, $selectedPools);
|
||||
}
|
||||
|
||||
private static function bundleConstraint(array $plan):
|
||||
CloudPublicNetworkBundleConstraint
|
||||
{
|
||||
$network = $plan['public_network'];
|
||||
unset($network['mode']);
|
||||
foreach (['ipv4_pool', 'ipv6_pool'] as $family) {
|
||||
unset($network[$family]['available_units']);
|
||||
}
|
||||
try {
|
||||
return CloudPublicNetworkBundleConstraint::fromArray($network);
|
||||
} catch (\InvalidArgumentException) {
|
||||
throw new CloudError('恢复目标双栈网络计划无效', 409);
|
||||
}
|
||||
}
|
||||
|
||||
private static function sameCapacityContract(array $current,
|
||||
array $planned): bool
|
||||
{
|
||||
foreach (['policy_updated_at', 'requested', 'limits'] as $field) {
|
||||
if (!array_key_exists($field, $current)
|
||||
|| !array_key_exists($field, $planned)
|
||||
|| self::canonical((array) $current[$field])
|
||||
!== self::canonical((array) $planned[$field])) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static function requestFingerprint(int $actorUserId,
|
||||
int $ownerUserId, array $backup, string $planSha256,
|
||||
string $targetName): string
|
||||
{
|
||||
return hash('sha256', json_encode(self::canonical([
|
||||
'contract' => 1,
|
||||
'requested_by_user_id' => $actorUserId,
|
||||
'owner_user_id' => $ownerUserId,
|
||||
'backup' => $backup,
|
||||
'selected_plan_sha256' => $planSha256,
|
||||
'target_name' => $targetName,
|
||||
]), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES));
|
||||
}
|
||||
|
||||
private static function canonical(array $value): array
|
||||
{
|
||||
if (array_is_list($value)) {
|
||||
return array_map(
|
||||
static fn ($item) => is_array($item)
|
||||
? self::canonical($item) : $item,
|
||||
$value
|
||||
);
|
||||
}
|
||||
ksort($value, SORT_STRING);
|
||||
foreach ($value as $key => $item) {
|
||||
if (is_array($item)) {
|
||||
$value[$key] = self::canonical($item);
|
||||
}
|
||||
}
|
||||
return $value;
|
||||
}
|
||||
|
||||
private static function key(): string
|
||||
{
|
||||
return bin2hex(random_bytes(16));
|
||||
}
|
||||
|
||||
private static function uuidV4(): string
|
||||
{
|
||||
$bytes = random_bytes(16);
|
||||
$bytes[6] = chr((ord($bytes[6]) & 0x0f) | 0x40);
|
||||
$bytes[8] = chr((ord($bytes[8]) & 0x3f) | 0x80);
|
||||
$hex = bin2hex($bytes);
|
||||
return substr($hex, 0, 8) . '-' . substr($hex, 8, 4) . '-'
|
||||
. substr($hex, 12, 4) . '-' . substr($hex, 16, 4) . '-'
|
||||
. substr($hex, 20);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,159 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudBackupRestore;
|
||||
use App\Models\CloudBackupRestoreAttempt;
|
||||
use App\Models\CloudResourceReservation;
|
||||
|
||||
/** Immutable coordinator result; it is not an Agent dispatch instruction. */
|
||||
final class CloudPortableRestoreAttemptReservation
|
||||
{
|
||||
private function __construct(private readonly array $value)
|
||||
{
|
||||
}
|
||||
|
||||
public static function active(CloudBackupRestore $restore,
|
||||
CloudBackupRestoreAttempt $attempt,
|
||||
CloudResourceReservation $reservation,
|
||||
CloudActivePublicNetworkClaimSet $claimSet,
|
||||
CloudFrozenPublicNetworkIdentity $network,
|
||||
CloudPortableRestoreTargetPlan $plan): self
|
||||
{
|
||||
self::assertRows($restore, $attempt, $reservation, $plan);
|
||||
self::assertActiveLifecycle($attempt, $reservation);
|
||||
$resources = $plan->toArray()['resources'];
|
||||
$dispatchable = (string) $attempt->state === 'reserved'
|
||||
&& (string) $reservation->state === 'held'
|
||||
&& $attempt->job_id === null
|
||||
&& (string) $attempt->effect_resolution === 'not_applicable';
|
||||
return new self([
|
||||
'restore_id' => (int) $restore->id,
|
||||
'restore_key' => (string) $restore->restore_key,
|
||||
'attempt_id' => (int) $attempt->id,
|
||||
'attempt_key' => (string) $attempt->attempt_key,
|
||||
'generation' => (int) $attempt->generation,
|
||||
'state' => (string) $attempt->state,
|
||||
'effect_resolution' => (string) $attempt->effect_resolution,
|
||||
'operation_id' => (string) $attempt->operation_id,
|
||||
'reservation_id' => (int) $reservation->id,
|
||||
'claim_ids' => [
|
||||
'ipv4' => $claimSet->claimId(4),
|
||||
'ipv6' => $claimSet->claimId(6),
|
||||
],
|
||||
'claim_set_sha256' => $claimSet->digest(),
|
||||
'target_plan_sha256' => $plan->sha256(),
|
||||
'vpc_inherited' => false,
|
||||
'dispatchable' => $dispatchable,
|
||||
'dispatch_body' => $dispatchable ? [
|
||||
'uuid' => strtolower((string) $attempt->target_uuid),
|
||||
'name' => (string) $attempt->target_name,
|
||||
'vcpu' => $resources['cpu_cores'],
|
||||
'memory_mib' => $resources['memory_mib'],
|
||||
'disk_gib' => $resources['disk_gib'],
|
||||
'bandwidth_mbps' => $resources['bandwidth_mbps'],
|
||||
'public_network' => $network->publicNetwork()->toArray(),
|
||||
] : null,
|
||||
]);
|
||||
}
|
||||
|
||||
public static function terminal(CloudBackupRestore $restore,
|
||||
CloudBackupRestoreAttempt $attempt,
|
||||
CloudResourceReservation $reservation,
|
||||
CloudPortableRestoreTargetPlan $plan,
|
||||
array $claimIds): self
|
||||
{
|
||||
self::assertRows($restore, $attempt, $reservation, $plan);
|
||||
foreach (['ipv4', 'ipv6'] as $family) {
|
||||
if (!isset($claimIds[$family]) || !is_int($claimIds[$family])
|
||||
|| $claimIds[$family] < 1) {
|
||||
throw new CloudError('恢复尝试历史双栈声明不完整', 409);
|
||||
}
|
||||
}
|
||||
return new self([
|
||||
'restore_id' => (int) $restore->id,
|
||||
'restore_key' => (string) $restore->restore_key,
|
||||
'attempt_id' => (int) $attempt->id,
|
||||
'attempt_key' => (string) $attempt->attempt_key,
|
||||
'generation' => (int) $attempt->generation,
|
||||
'state' => (string) $attempt->state,
|
||||
'effect_resolution' => (string) $attempt->effect_resolution,
|
||||
'operation_id' => (string) $attempt->operation_id,
|
||||
'reservation_id' => (int) $reservation->id,
|
||||
'claim_ids' => $claimIds,
|
||||
'claim_set_sha256' =>
|
||||
(string) $attempt->target_claim_set_sha256,
|
||||
'target_plan_sha256' => $plan->sha256(),
|
||||
'vpc_inherited' => false,
|
||||
'dispatchable' => false,
|
||||
'dispatch_body' => null,
|
||||
]);
|
||||
}
|
||||
|
||||
public function toArray(): array
|
||||
{
|
||||
return $this->value;
|
||||
}
|
||||
|
||||
public function dispatchBody(): ?array
|
||||
{
|
||||
return $this->value['dispatch_body'];
|
||||
}
|
||||
|
||||
private static function assertRows(CloudBackupRestore $restore,
|
||||
CloudBackupRestoreAttempt $attempt,
|
||||
CloudResourceReservation $reservation,
|
||||
CloudPortableRestoreTargetPlan $plan): void
|
||||
{
|
||||
if ((int) $restore->id < 1 || (int) $attempt->id < 1
|
||||
|| (int) $reservation->id < 1
|
||||
|| (int) $attempt->backup_restore_id !== (int) $restore->id
|
||||
|| (int) $attempt->reservation_id !== (int) $reservation->id
|
||||
|| !$attempt->matchesReservation($reservation)
|
||||
|| (int) $attempt->target_host_id
|
||||
!== (int) $restore->target_host_id
|
||||
|| !hash_equals(
|
||||
strtolower((string) $attempt->target_uuid),
|
||||
strtolower((string) $restore->new_uuid)
|
||||
)
|
||||
|| !hash_equals(
|
||||
(string) $attempt->target_name,
|
||||
(string) $restore->new_name
|
||||
)
|
||||
|| !hash_equals(
|
||||
(string) $attempt->target_plan_sha256,
|
||||
$plan->sha256()
|
||||
)
|
||||
|| !hash_equals(
|
||||
(string) $restore->selected_plan_sha256,
|
||||
$plan->sha256()
|
||||
)) {
|
||||
throw new CloudError('恢复尝试资源身份不一致', 409);
|
||||
}
|
||||
}
|
||||
|
||||
private static function assertActiveLifecycle(
|
||||
CloudBackupRestoreAttempt $attempt,
|
||||
CloudResourceReservation $reservation
|
||||
): void {
|
||||
$state = (string) $attempt->state;
|
||||
$resolution = (string) $attempt->effect_resolution;
|
||||
$expectedReservation = match ($state) {
|
||||
'reserved', 'queued', 'awaiting_receipt' =>
|
||||
$resolution === 'not_applicable' ? 'held' : '',
|
||||
'effect_unknown' =>
|
||||
$resolution === 'unknown' ? 'uncertain' : '',
|
||||
'finalization_pending' => match ($resolution) {
|
||||
'applied' => 'committed',
|
||||
'not_applied' => 'held',
|
||||
default => '',
|
||||
},
|
||||
'succeeded' => $resolution === 'applied' ? 'committed' : '',
|
||||
default => '',
|
||||
};
|
||||
if ($expectedReservation === ''
|
||||
|| (string) $reservation->state !== $expectedReservation) {
|
||||
throw new CloudError('恢复尝试与资源预留生命周期不一致', 409);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,162 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudInstanceBackup;
|
||||
|
||||
/** Secret-free, canonical identity of one ready portable backup object. */
|
||||
final class CloudPortableRestoreBackupEvidence
|
||||
{
|
||||
private const GIB = 1073741824;
|
||||
|
||||
private function __construct(
|
||||
private readonly CloudPortableRestoreCompatibility $compatibility,
|
||||
private readonly array $backup,
|
||||
private readonly array $resources
|
||||
) {
|
||||
}
|
||||
|
||||
public static function fromBackup(CloudInstanceBackup $backup): self
|
||||
{
|
||||
$contract = $backup->sourcePortableRestoreCompatibility();
|
||||
$objectKey = (string) $backup->object_key;
|
||||
$sealedRuntime = (string) $backup->sealed_storage_runtime;
|
||||
$objectEtag = (string) $backup->object_etag;
|
||||
$objectVersionId = (string) $backup->object_version_id;
|
||||
if ((int) $backup->id < 1
|
||||
|| preg_match('/^[a-f0-9]{32}$/D',
|
||||
(string) $backup->backup_key) !== 1
|
||||
|| (int) $backup->source_host_id < 1
|
||||
|| !self::validObjectKey($objectKey)
|
||||
|| trim($sealedRuntime) === ''
|
||||
|| strlen($objectEtag) > 256
|
||||
|| preg_match('/[\x00-\x1f\x7f]/D', $objectEtag) === 1
|
||||
|| strlen($objectVersionId) > 1024
|
||||
|| preg_match('/[\x00-\x1f\x7f]/D', $objectVersionId) === 1
|
||||
|| ($objectVersionId !== '' && $objectEtag === '')
|
||||
|| (string) $backup->purpose !== 'portable'
|
||||
|| (string) $backup->state !== 'ready'
|
||||
|| $contract === null
|
||||
|| (int) $backup->object_size_bytes < 1
|
||||
|| (int) $backup->virtual_size_bytes < 1
|
||||
|| (int) $backup->storage_revision < 1
|
||||
|| preg_match('/^[a-f0-9]{64}$/D',
|
||||
(string) $backup->sha256) !== 1) {
|
||||
throw new CloudError('备份缺少可信的可移植恢复证据', 409);
|
||||
}
|
||||
try {
|
||||
$spec = json_decode(
|
||||
(string) $backup->spec_json,
|
||||
true,
|
||||
64,
|
||||
JSON_THROW_ON_ERROR
|
||||
);
|
||||
if (!is_array($spec) || array_is_list($spec)) {
|
||||
throw new \InvalidArgumentException();
|
||||
}
|
||||
$sourceNetwork = CloudPublicNetworkSnapshot::fromArray(
|
||||
$spec['public_network'] ?? null
|
||||
);
|
||||
} catch (\Throwable) {
|
||||
throw new CloudError('备份缺少可信的冻结双栈网络快照', 409);
|
||||
}
|
||||
$lineId = self::positiveInt($spec['line_id'] ?? null);
|
||||
$resources = [
|
||||
'cpu_cores' => self::positiveInt(
|
||||
$spec['vcpu'] ?? $spec['cpu_cores'] ?? null
|
||||
),
|
||||
'memory_mib' => self::positiveInt(
|
||||
$spec['memory_mib'] ?? $spec['ram_mib'] ?? null
|
||||
),
|
||||
'disk_gib' => self::positiveInt($spec['disk_gib'] ?? null),
|
||||
'bandwidth_mbps' => self::positiveInt(
|
||||
$spec['bandwidth_mbps'] ?? null
|
||||
),
|
||||
];
|
||||
if ($lineId === null || in_array(null, $resources, true)
|
||||
|| $resources['disk_gib'] > intdiv(PHP_INT_MAX, self::GIB)) {
|
||||
throw new CloudError('备份缺少可信的资源与商业线路快照', 409);
|
||||
}
|
||||
$virtualSize = (int) $backup->virtual_size_bytes;
|
||||
if ($virtualSize > $resources['disk_gib'] * self::GIB) {
|
||||
throw new CloudError('备份系统盘容量超过冻结规格', 409);
|
||||
}
|
||||
$objectIdentitySha256 = hash('sha256', json_encode([
|
||||
'object_key' => $objectKey,
|
||||
'object_size_bytes' => (int) $backup->object_size_bytes,
|
||||
'object_sha256' => (string) $backup->sha256,
|
||||
'object_etag' => $objectEtag,
|
||||
'object_version_id' => $objectVersionId,
|
||||
'storage_revision' => (int) $backup->storage_revision,
|
||||
'sealed_storage_runtime_sha256' => hash(
|
||||
'sha256',
|
||||
$sealedRuntime
|
||||
),
|
||||
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES));
|
||||
return new self($contract, [
|
||||
'id' => (int) $backup->id,
|
||||
'backup_key' => (string) $backup->backup_key,
|
||||
'object_size_bytes' => (int) $backup->object_size_bytes,
|
||||
'object_sha256' => (string) $backup->sha256,
|
||||
'object_identity_sha256' => $objectIdentitySha256,
|
||||
'source_contract_sha256' => $contract->sha256(),
|
||||
'source_public_network_sha256' =>
|
||||
$sourceNetwork->fingerprint(),
|
||||
'source_host_id' => (int) $backup->source_host_id,
|
||||
'source_line_id' => $lineId,
|
||||
'source_storage_revision' => (int) $backup->storage_revision,
|
||||
'virtual_size_bytes' => $virtualSize,
|
||||
], [
|
||||
'cpu_cores' => $resources['cpu_cores'],
|
||||
'memory_mib' => $resources['memory_mib'],
|
||||
'disk_gib' => $resources['disk_gib'],
|
||||
'bandwidth_mbps' => $resources['bandwidth_mbps'],
|
||||
'system_disk_virtual_size_bytes' => $virtualSize,
|
||||
]);
|
||||
}
|
||||
|
||||
public function compatibility(): CloudPortableRestoreCompatibility
|
||||
{
|
||||
return $this->compatibility;
|
||||
}
|
||||
|
||||
public function backup(): array
|
||||
{
|
||||
return $this->backup;
|
||||
}
|
||||
|
||||
public function resources(): array
|
||||
{
|
||||
return $this->resources;
|
||||
}
|
||||
|
||||
public function capacityResources(): array
|
||||
{
|
||||
return [
|
||||
'cpu_cores' => $this->resources['cpu_cores'],
|
||||
'memory_mib' => $this->resources['memory_mib'],
|
||||
'disk_gib' => $this->resources['disk_gib'],
|
||||
];
|
||||
}
|
||||
|
||||
private static function positiveInt(mixed $value): ?int
|
||||
{
|
||||
$normalized = filter_var(
|
||||
$value,
|
||||
FILTER_VALIDATE_INT,
|
||||
['options' => ['min_range' => 1, 'max_range' => PHP_INT_MAX]]
|
||||
);
|
||||
return $normalized === false ? null : (int) $normalized;
|
||||
}
|
||||
|
||||
private static function validObjectKey(string $objectKey): bool
|
||||
{
|
||||
return $objectKey !== ''
|
||||
&& strlen($objectKey) <= 512
|
||||
&& !str_starts_with($objectKey, '/')
|
||||
&& !str_ends_with($objectKey, '/')
|
||||
&& !str_contains($objectKey, '\\')
|
||||
&& preg_match('/[\x00-\x1f\x7f]/D', $objectKey) !== 1
|
||||
&& !in_array('', explode('/', $objectKey), true);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,189 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudBackupRestoreAttempt;
|
||||
use App\Models\CloudResourceReservation;
|
||||
use App\Models\CloudResourceReservationIp;
|
||||
use Illuminate\Database\Capsule\Manager as DB;
|
||||
|
||||
/** Verifies one fully released restore bundle without requiring it to be free. */
|
||||
final class CloudReleasedPublicNetworkBundleVerifier
|
||||
{
|
||||
/** @return array{ipv4:int,ipv6:int} */
|
||||
public static function verify(CloudBackupRestoreAttempt $attempt,
|
||||
CloudResourceReservation $reservation, int $ownerUserId,
|
||||
bool $lock = true): array
|
||||
{
|
||||
try {
|
||||
$plan = CloudPortableRestoreTargetPlan::fromStored(
|
||||
(string) $attempt->target_plan_json,
|
||||
(string) $attempt->target_plan_sha256
|
||||
);
|
||||
} catch (\InvalidArgumentException) {
|
||||
throw new CloudError('失败恢复尝试的目标计划损坏', 409);
|
||||
}
|
||||
$target = $plan->toArray()['target'];
|
||||
if ((int) $attempt->reservation_id !== (int) $reservation->id
|
||||
|| !hash_equals(
|
||||
(string) $attempt->attempt_key,
|
||||
(string) $reservation->backup_restore_attempt_key
|
||||
)
|
||||
|| (int) $reservation->host_id !== (int) $attempt->target_host_id
|
||||
|| (int) $reservation->host_id !== (int) $target['host_id']
|
||||
|| (int) $reservation->resource_pool_id
|
||||
!== (int) $target['resource_pool_id']
|
||||
|| (int) $reservation->user_id !== $ownerUserId
|
||||
|| !hash_equals(
|
||||
(string) $reservation->operation_id,
|
||||
(string) $attempt->operation_id
|
||||
)
|
||||
|| (string) $reservation->state !== 'released'
|
||||
|| $reservation->released_at === null) {
|
||||
throw new CloudError('失败恢复尝试仍占用资源或身份不一致', 409);
|
||||
}
|
||||
|
||||
$claimsQuery = CloudResourceReservationIp::where(
|
||||
'reservation_id',
|
||||
(int) $reservation->id
|
||||
)->orderBy('address_family')->orderBy('id');
|
||||
if ($lock) {
|
||||
$claimsQuery->lockForUpdate();
|
||||
}
|
||||
$claims = $claimsQuery->get();
|
||||
$ids = [];
|
||||
$claimsByFamily = [];
|
||||
foreach ($claims as $claim) {
|
||||
$family = (int) $claim->address_family;
|
||||
if (!in_array($family, [4, 6], true)
|
||||
|| isset($ids[$family])
|
||||
|| (string) $claim->state !== 'released'
|
||||
|| !hash_equals(
|
||||
(string) $claim->operation_id,
|
||||
(string) $attempt->operation_id
|
||||
)
|
||||
|| (int) $claim->network_attachment_id
|
||||
!== (int) $attempt->target_network_attachment_id
|
||||
|| $claim->active_claim !== null
|
||||
|| $claim->released_at === null) {
|
||||
throw new CloudError('失败恢复尝试的公网声明未完全释放', 409);
|
||||
}
|
||||
$ids[$family] = (int) $claim->id;
|
||||
$claimsByFamily[$family] = $claim;
|
||||
}
|
||||
if (!isset($ids[4], $ids[6])) {
|
||||
throw new CloudError('失败恢复尝试的公网声明不完整', 409);
|
||||
}
|
||||
|
||||
try {
|
||||
$networkValue = json_decode(
|
||||
(string) $attempt->target_public_network_json,
|
||||
true,
|
||||
64,
|
||||
JSON_THROW_ON_ERROR
|
||||
);
|
||||
$network = CloudPublicNetworkSnapshot::fromArray($networkValue);
|
||||
$claimIdentity =
|
||||
CloudPublicNetworkClaimSetIdentity::fromReleasedLedger(
|
||||
(int) $reservation->id,
|
||||
$claimsByFamily,
|
||||
$network
|
||||
);
|
||||
} catch (\Throwable) {
|
||||
throw new CloudError('失败恢复尝试的冻结公网身份损坏', 409);
|
||||
}
|
||||
if (!hash_equals(
|
||||
$network->fingerprint(),
|
||||
(string) $attempt->target_public_network_sha256
|
||||
)
|
||||
|| !hash_equals(
|
||||
$claimIdentity->digest(),
|
||||
(string) $attempt->target_claim_set_sha256
|
||||
)) {
|
||||
throw new CloudError('失败恢复尝试的冻结公网身份不一致', 409);
|
||||
}
|
||||
|
||||
$plannedNetwork = $plan->toArray()['public_network'];
|
||||
if ((int) $attempt->target_network_attachment_id
|
||||
!== (int) $plannedNetwork['network_attachment_id']
|
||||
|| (int) $claimsByFamily[4]->public_ip_pool_id
|
||||
!== (int) $plannedNetwork['ipv4_pool']['public_ip_pool_id']
|
||||
|| (int) $claimsByFamily[6]->public_ip_pool_id
|
||||
!== (int) $plannedNetwork['ipv6_pool']['public_ip_pool_id']) {
|
||||
throw new CloudError('失败恢复尝试的公网池身份不一致', 409);
|
||||
}
|
||||
|
||||
$addressQuery = DB::table('cloud_public_ips')->whereIn(
|
||||
'id',
|
||||
[
|
||||
(int) $claimsByFamily[4]->public_ip_id,
|
||||
(int) $claimsByFamily[6]->public_ip_id,
|
||||
]
|
||||
)->orderBy('id');
|
||||
if ($lock) {
|
||||
$addressQuery->lockForUpdate();
|
||||
}
|
||||
$addresses = [];
|
||||
foreach ($addressQuery->get() as $address) {
|
||||
$addresses[(int) $address->id] = $address;
|
||||
}
|
||||
foreach ([4, 6] as $family) {
|
||||
$claim = $claimsByFamily[$family];
|
||||
$address = $addresses[(int) $claim->public_ip_id] ?? null;
|
||||
if (!is_object($address)
|
||||
|| (int) $address->host_id !== (int) $target['host_id']
|
||||
|| (int) $address->address_family !== $family
|
||||
|| (int) $address->public_ip_pool_id
|
||||
!== (int) $claim->public_ip_pool_id
|
||||
|| !hash_equals(
|
||||
(string) $address->address . '/'
|
||||
. (int) $address->prefix_length,
|
||||
$network->address($family)
|
||||
)
|
||||
|| self::retainsOldOwnership(
|
||||
$address,
|
||||
(int) $reservation->id,
|
||||
(int) $claim->id,
|
||||
(string) $attempt->operation_id
|
||||
)
|
||||
|| !self::hasValidCurrentOwnership($address)) {
|
||||
throw new CloudError('失败恢复尝试的公网地址未完整解除归属', 409);
|
||||
}
|
||||
}
|
||||
return ['ipv4' => $ids[4], 'ipv6' => $ids[6]];
|
||||
}
|
||||
|
||||
private static function retainsOldOwnership(object $address,
|
||||
int $reservationId, int $claimId, string $operationId): bool
|
||||
{
|
||||
return (int) ($address->reservation_id ?? 0) === $reservationId
|
||||
|| (int) ($address->reservation_ip_claim_id ?? 0) === $claimId
|
||||
|| hash_equals(
|
||||
$operationId,
|
||||
(string) ($address->operation_id ?? '')
|
||||
);
|
||||
}
|
||||
|
||||
private static function hasValidCurrentOwnership(object $address): bool
|
||||
{
|
||||
$state = (string) ($address->state ?? '');
|
||||
$reservation = $address->reservation_id;
|
||||
$claim = $address->reservation_ip_claim_id;
|
||||
$instance = $address->instance_id;
|
||||
$operation = $address->operation_id;
|
||||
if (in_array($state, ['available', 'disabled'], true)) {
|
||||
return $reservation === null && $claim === null
|
||||
&& $instance === null && $operation === null;
|
||||
}
|
||||
if ($reservation === null || $claim === null || $operation === null
|
||||
|| preg_match('/^[a-f0-9]{32}$/D', (string) $operation) !== 1) {
|
||||
return false;
|
||||
}
|
||||
return match ($state) {
|
||||
'reserved' => $instance === null,
|
||||
'assigned' => $instance !== null,
|
||||
'changing', 'uncertain' => true,
|
||||
default => false,
|
||||
};
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,801 @@
|
||||
<?php
|
||||
|
||||
ob_start();
|
||||
require __DIR__ . '/CloudPortableRestoreTargetServiceTest.php';
|
||||
ob_end_clean();
|
||||
|
||||
use App\Models\CloudBackupRestore;
|
||||
use App\Models\CloudBackupRestoreAttempt;
|
||||
use App\Models\CloudInstanceBackup;
|
||||
use App\Models\CloudResourceReservation;
|
||||
use App\Services\Cloud\CloudBackupService;
|
||||
use App\Services\Cloud\CloudError;
|
||||
use App\Services\Cloud\CloudHostCapacityAdmission;
|
||||
use App\Services\Cloud\CloudIpAddress;
|
||||
use App\Services\Cloud\CloudPortableRestoreAttemptCoordinator;
|
||||
use App\Services\Cloud\CloudPortableRestoreTargetPlan;
|
||||
use App\Services\Cloud\CloudPortableRestoreTargetService;
|
||||
use App\Services\Cloud\CloudPublicNetworkAllocationService;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
$rejects409 = static function (callable $callback, string $message) use (
|
||||
$assert
|
||||
): void {
|
||||
try {
|
||||
$callback();
|
||||
} catch (CloudError $error) {
|
||||
$assert($error->httpStatus === 409, $message . ' (wrong status)');
|
||||
return;
|
||||
}
|
||||
throw new RuntimeException($message);
|
||||
};
|
||||
|
||||
// Restore the valid source row after the target-scheduler negative cases.
|
||||
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
|
||||
'virtual_size_bytes' => 20 * 1073741824,
|
||||
]);
|
||||
|
||||
// Extend the target-scheduler fixture with the exact mutable ledgers owned by
|
||||
// the attempt coordinator and atomic public-network allocator.
|
||||
$schema->create('user', static function (Blueprint $t): void {
|
||||
$t->increments('id');
|
||||
$t->integer('enable');
|
||||
$t->boolean('is_admin')->default(false);
|
||||
});
|
||||
Capsule::table('user')->insert([
|
||||
['id' => 9, 'enable' => 1, 'is_admin' => 0],
|
||||
['id' => 99, 'enable' => 1, 'is_admin' => 1],
|
||||
]);
|
||||
|
||||
$schema->table('cloud_instances', static function (Blueprint $t): void {
|
||||
$t->string('name')->default('source-vm');
|
||||
$t->integer('user_id')->nullable();
|
||||
$t->integer('resource_pool_id')->nullable();
|
||||
$t->integer('reservation_id')->nullable();
|
||||
});
|
||||
Capsule::table('cloud_instances')->where('id', 1)->update(['user_id' => 9]);
|
||||
|
||||
$schema->table('cloud_public_ip_pools', static function (Blueprint $t): void {
|
||||
$t->string('allocation_cursor')->nullable();
|
||||
});
|
||||
$schema->table('cloud_public_ips', static function (Blueprint $t): void {
|
||||
$t->binary('address_binary')->nullable();
|
||||
$t->integer('reservation_ip_claim_id')->nullable();
|
||||
$t->integer('created_at')->default(0);
|
||||
$t->integer('updated_at')->default(0);
|
||||
});
|
||||
foreach (Capsule::table('cloud_public_ips')->orderBy('id')->get() as $row) {
|
||||
Capsule::table('cloud_public_ips')->where('id', (int) $row->id)->update([
|
||||
'address_binary' => CloudIpAddress::parse(
|
||||
(string) $row->address
|
||||
)->binary(),
|
||||
'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
]);
|
||||
}
|
||||
|
||||
$schema->drop('cloud_resource_reservation_ips');
|
||||
$schema->drop('cloud_resource_reservations');
|
||||
$schema->drop('cloud_backup_restores');
|
||||
$schema->drop('cloud_jobs');
|
||||
|
||||
$schema->create('cloud_resource_reservations', static function (
|
||||
Blueprint $t
|
||||
): void {
|
||||
$t->increments('id');
|
||||
$t->string('reservation_key');
|
||||
$t->string('operation_id');
|
||||
$t->integer('order_id')->nullable();
|
||||
$t->integer('backup_restore_id')->nullable();
|
||||
$t->string('backup_restore_attempt_key')->nullable();
|
||||
$t->integer('migration_source_debt_id')->nullable();
|
||||
$t->integer('offer_id')->nullable();
|
||||
$t->integer('resource_pool_id');
|
||||
$t->integer('host_id');
|
||||
$t->integer('user_id');
|
||||
$t->integer('instance_id')->nullable();
|
||||
$t->integer('network_attachment_id')->nullable();
|
||||
$t->integer('public_ip_pool_id')->nullable();
|
||||
$t->integer('public_ip_id')->nullable();
|
||||
$t->integer('cpu_cores');
|
||||
$t->integer('memory_mib');
|
||||
$t->integer('disk_gib');
|
||||
$t->string('state');
|
||||
$t->integer('expires_at')->nullable();
|
||||
$t->integer('released_at')->nullable();
|
||||
$t->string('release_reason')->default('');
|
||||
$t->integer('created_at');
|
||||
$t->integer('updated_at');
|
||||
$t->unique('operation_id');
|
||||
$t->unique('backup_restore_attempt_key');
|
||||
});
|
||||
$schema->create('cloud_resource_reservation_ips', static function (
|
||||
Blueprint $t
|
||||
): void {
|
||||
$t->increments('id');
|
||||
$t->integer('reservation_id');
|
||||
$t->integer('address_family');
|
||||
$t->integer('claim_generation');
|
||||
$t->integer('active_claim')->nullable();
|
||||
$t->integer('network_attachment_id');
|
||||
$t->integer('public_ip_pool_id');
|
||||
$t->integer('public_ip_id');
|
||||
$t->string('operation_id');
|
||||
$t->string('state');
|
||||
$t->integer('created_at');
|
||||
$t->integer('updated_at');
|
||||
$t->integer('released_at')->nullable();
|
||||
$t->string('release_reason')->default('');
|
||||
$t->unique(['reservation_id', 'address_family', 'active_claim']);
|
||||
});
|
||||
$schema->create('cloud_backup_restores', static function (Blueprint $t): void {
|
||||
$t->increments('id');
|
||||
$t->string('restore_key')->unique();
|
||||
$t->integer('backup_id');
|
||||
$t->integer('requested_by_user_id');
|
||||
$t->string('request_idempotency_key');
|
||||
$t->string('request_fingerprint');
|
||||
$t->text('selected_plan_json');
|
||||
$t->string('selected_plan_sha256');
|
||||
$t->integer('target_host_id');
|
||||
$t->integer('target_instance_id')->nullable();
|
||||
$t->integer('reservation_id')->nullable();
|
||||
$t->integer('job_id')->nullable();
|
||||
$t->string('operation_id');
|
||||
$t->string('new_uuid');
|
||||
$t->string('new_name');
|
||||
$t->string('state');
|
||||
$t->string('error_code')->default('');
|
||||
$t->integer('ready_at')->nullable();
|
||||
$t->integer('created_at');
|
||||
$t->integer('updated_at');
|
||||
$t->unique(['requested_by_user_id', 'request_idempotency_key']);
|
||||
});
|
||||
$schema->create('cloud_backup_restore_attempts', static function (
|
||||
Blueprint $t
|
||||
): void {
|
||||
$t->increments('id');
|
||||
$t->string('attempt_key')->unique();
|
||||
$t->integer('backup_restore_id');
|
||||
$t->integer('generation');
|
||||
$t->string('operation_id')->unique();
|
||||
$t->integer('job_id')->nullable();
|
||||
$t->integer('reservation_id')->unique();
|
||||
$t->integer('target_host_id');
|
||||
$t->string('target_uuid');
|
||||
$t->string('target_name');
|
||||
$t->text('target_plan_json');
|
||||
$t->string('target_plan_sha256');
|
||||
$t->integer('target_network_attachment_id');
|
||||
$t->text('target_identity_json');
|
||||
$t->string('target_identity_sha256');
|
||||
$t->string('target_claim_set_sha256');
|
||||
$t->text('target_public_network_json');
|
||||
$t->string('target_public_network_sha256');
|
||||
$t->string('state');
|
||||
$t->string('effect_resolution');
|
||||
$t->integer('active_attempt')->nullable();
|
||||
$t->string('last_error_code')->default('');
|
||||
$t->integer('effect_resolved_at')->nullable();
|
||||
$t->integer('terminal_at')->nullable();
|
||||
$t->integer('created_at');
|
||||
$t->integer('updated_at');
|
||||
$t->unique(['backup_restore_id', 'generation']);
|
||||
$t->unique(['backup_restore_id', 'active_attempt']);
|
||||
});
|
||||
$schema->create('cloud_jobs', static function (Blueprint $t): void {
|
||||
$t->increments('id');
|
||||
$t->integer('user_id');
|
||||
$t->integer('backup_id')->nullable();
|
||||
$t->integer('backup_restore_id')->nullable();
|
||||
$t->integer('host_id')->nullable();
|
||||
$t->string('kind');
|
||||
$t->string('operation_id');
|
||||
$t->string('phase');
|
||||
$t->string('outcome')->nullable();
|
||||
$t->string('effect_resolution');
|
||||
$t->integer('terminal_at')->nullable();
|
||||
$t->integer('finalization_incident')->default(0);
|
||||
});
|
||||
$schema->create('cloud_backup_recoveries', static function (Blueprint $t): void {
|
||||
$t->increments('id');
|
||||
$t->integer('backup_id');
|
||||
$t->string('state');
|
||||
});
|
||||
$schema->create('cloud_deployment_snapshots', static function (
|
||||
Blueprint $t
|
||||
): void {
|
||||
$t->increments('id');
|
||||
$t->integer('backup_id');
|
||||
$t->string('state');
|
||||
});
|
||||
$schema->create('cloud_instance_migrations', static function (
|
||||
Blueprint $t
|
||||
): void {
|
||||
$t->increments('id');
|
||||
$t->integer('source_host_id');
|
||||
$t->integer('source_reservation_id');
|
||||
$t->integer('target_host_id');
|
||||
$t->integer('target_reservation_id');
|
||||
$t->integer('instance_id');
|
||||
$t->integer('restore_id');
|
||||
$t->string('state');
|
||||
});
|
||||
|
||||
$backup = CloudInstanceBackup::findOrFail(1);
|
||||
$plans = CloudPortableRestoreTargetService::candidates($backup);
|
||||
$plan = array_values(array_filter(
|
||||
$plans,
|
||||
static fn (CloudPortableRestoreTargetPlan $candidate): bool =>
|
||||
$candidate->toArray()['target']['host_id'] === 2
|
||||
&& $candidate->toArray()['target']['resource_pool_id'] === 200
|
||||
))[0] ?? null;
|
||||
$assert($plan instanceof CloudPortableRestoreTargetPlan,
|
||||
'Host-two portable restore plan is missing.');
|
||||
|
||||
$actor = [
|
||||
'user_id' => 99,
|
||||
'is_admin' => true,
|
||||
'is_super_admin' => true,
|
||||
'can_manage' => true,
|
||||
];
|
||||
$requestKey = 'restore-request-0001';
|
||||
$first = CloudPortableRestoreAttemptCoordinator::reserve(
|
||||
$actor,
|
||||
$backup,
|
||||
$plan,
|
||||
$plan->sha256(),
|
||||
'restored-vm',
|
||||
$requestKey
|
||||
)->toArray();
|
||||
$assert($first['generation'] === 1 && $first['dispatchable'] === true
|
||||
&& is_array($first['dispatch_body'])
|
||||
&& Capsule::table('cloud_resource_reservations')
|
||||
->where('id', $first['reservation_id'])->value('user_id') === 9
|
||||
&& Capsule::table('cloud_resource_reservation_ips')
|
||||
->where('reservation_id', $first['reservation_id'])->count() === 2,
|
||||
'Initial restore did not atomically reserve owner capacity and dual stack.');
|
||||
|
||||
$same = CloudPortableRestoreAttemptCoordinator::reserve(
|
||||
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
|
||||
)->toArray();
|
||||
$assert($same['attempt_key'] === $first['attempt_key']
|
||||
&& $same['reservation_id'] === $first['reservation_id'],
|
||||
'Exact reserve replay created another attempt.');
|
||||
$rejects409(
|
||||
static fn () => CloudPortableRestoreAttemptCoordinator::reserve(
|
||||
$actor, $backup, $plan, $plan->sha256(), 'different-vm', $requestKey
|
||||
),
|
||||
'Same idempotency key accepted a different restore request.'
|
||||
);
|
||||
|
||||
// Exact replay is a lookup of already-frozen evidence. Current eligibility
|
||||
// drift must neither reject it nor generate a replacement attempt/bundle.
|
||||
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
|
||||
'state' => 'failed',
|
||||
]);
|
||||
Capsule::table('cloud_host_agents')->where('host_id', 2)->update([
|
||||
'enabled' => 0,
|
||||
'revision' => 99,
|
||||
'tested_revision' => 98,
|
||||
'last_test_status' => 'failed',
|
||||
]);
|
||||
Capsule::table('cloud_host_storage_configurations')->where('host_id', 2)
|
||||
->update(['revision' => 33, 'state' => 'pending']);
|
||||
Capsule::table('cloud_host_resource_pool_memberships')
|
||||
->where('host_id', 2)->where('resource_pool_id', 200)->update([
|
||||
'scheduler_state' => 'disabled',
|
||||
'revision' => 13,
|
||||
]);
|
||||
$eligibilityReplay = CloudPortableRestoreAttemptCoordinator::reserve(
|
||||
$actor,
|
||||
CloudInstanceBackup::findOrFail(1),
|
||||
$plan,
|
||||
$plan->sha256(),
|
||||
'restored-vm',
|
||||
$requestKey
|
||||
)->toArray();
|
||||
$assert($eligibilityReplay['attempt_key'] === $first['attempt_key']
|
||||
&& $eligibilityReplay['reservation_id'] === $first['reservation_id'],
|
||||
'Current backup or target eligibility drift broke exact replay.');
|
||||
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
|
||||
'state' => 'ready',
|
||||
]);
|
||||
Capsule::table('cloud_host_agents')->where('host_id', 2)->update([
|
||||
'enabled' => 1,
|
||||
'revision' => 42,
|
||||
'tested_revision' => 42,
|
||||
'last_test_status' => 'healthy',
|
||||
]);
|
||||
Capsule::table('cloud_host_storage_configurations')->where('host_id', 2)
|
||||
->update(['revision' => 32, 'state' => 'applied']);
|
||||
Capsule::table('cloud_host_resource_pool_memberships')
|
||||
->where('host_id', 2)->where('resource_pool_id', 200)->update([
|
||||
'scheduler_state' => 'active',
|
||||
'revision' => 12,
|
||||
]);
|
||||
|
||||
// A torn durable claim set must never be silently replaced during replay.
|
||||
$claimRowsBefore = Capsule::table('cloud_resource_reservation_ips')->count();
|
||||
$ipIdsBefore = Capsule::table('cloud_public_ips')->orderBy('id')
|
||||
->pluck('id')->all();
|
||||
Capsule::connection()->beginTransaction();
|
||||
try {
|
||||
Capsule::table('cloud_resource_reservation_ips')
|
||||
->where('reservation_id', $first['reservation_id'])->delete();
|
||||
$rejects409(
|
||||
static fn () => CloudPortableRestoreAttemptCoordinator::reserve(
|
||||
$actor,
|
||||
CloudInstanceBackup::findOrFail(1),
|
||||
$plan,
|
||||
$plan->sha256(),
|
||||
'restored-vm',
|
||||
$requestKey
|
||||
),
|
||||
'Replay allocated a replacement bundle for missing durable claims.'
|
||||
);
|
||||
$assert(Capsule::table('cloud_resource_reservation_ips')->count()
|
||||
=== $claimRowsBefore - 2
|
||||
&& Capsule::table('cloud_public_ips')->orderBy('id')
|
||||
->pluck('id')->all() === $ipIdsBefore,
|
||||
'Failed replay inserted replacement claims or public addresses.');
|
||||
} finally {
|
||||
Capsule::connection()->rollBack();
|
||||
}
|
||||
|
||||
// Volatile capacity changes must not invalidate an already-reserved replay.
|
||||
Capsule::table('cloud_resource_reservations')->insert([
|
||||
'reservation_key' => str_repeat('1', 32),
|
||||
'operation_id' => str_repeat('2', 32),
|
||||
'order_id' => 77,
|
||||
'backup_restore_id' => null,
|
||||
'backup_restore_attempt_key' => null,
|
||||
'migration_source_debt_id' => null,
|
||||
'offer_id' => 88,
|
||||
'resource_pool_id' => 200,
|
||||
'host_id' => 2,
|
||||
'user_id' => 9,
|
||||
'instance_id' => null,
|
||||
'network_attachment_id' => null,
|
||||
'public_ip_pool_id' => null,
|
||||
'public_ip_id' => null,
|
||||
'cpu_cores' => 1,
|
||||
'memory_mib' => 256,
|
||||
'disk_gib' => 1,
|
||||
'state' => 'held',
|
||||
'expires_at' => null,
|
||||
'released_at' => null,
|
||||
'release_reason' => '',
|
||||
'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
]);
|
||||
$capacityReplay = CloudPortableRestoreAttemptCoordinator::reserve(
|
||||
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
|
||||
)->toArray();
|
||||
$assert($capacityReplay['attempt_key'] === $first['attempt_key'],
|
||||
'Volatile capacity drift broke exact replay.');
|
||||
|
||||
$attempt = CloudBackupRestoreAttempt::where(
|
||||
'attempt_key',
|
||||
$first['attempt_key']
|
||||
)->firstOrFail();
|
||||
$reservation = CloudResourceReservation::findOrFail(
|
||||
$first['reservation_id']
|
||||
);
|
||||
|
||||
// Reservation/claim fencing is not a dispatchable state. A stale reserved
|
||||
// attempt must fail closed instead of returning a body that could be sent.
|
||||
$reservation->state = 'uncertain';
|
||||
$reservation->save();
|
||||
Capsule::table('cloud_resource_reservation_ips')
|
||||
->where('reservation_id', (int) $reservation->id)->update([
|
||||
'state' => 'uncertain', 'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_public_ips')
|
||||
->where('reservation_id', (int) $reservation->id)->update([
|
||||
'state' => 'uncertain', 'updated_at' => $now,
|
||||
]);
|
||||
$rejects409(
|
||||
static fn () => CloudPortableRestoreAttemptCoordinator::reserve(
|
||||
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
|
||||
),
|
||||
'Reserved attempt accepted an uncertain fenced resource bundle.'
|
||||
);
|
||||
$reservation->state = 'held';
|
||||
$reservation->save();
|
||||
Capsule::table('cloud_resource_reservation_ips')
|
||||
->where('reservation_id', (int) $reservation->id)->update([
|
||||
'state' => 'reserved', 'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_public_ips')
|
||||
->where('reservation_id', (int) $reservation->id)->update([
|
||||
'state' => 'reserved', 'updated_at' => $now,
|
||||
]);
|
||||
|
||||
$attempt->state = 'effect_unknown';
|
||||
$attempt->effect_resolution = 'unknown';
|
||||
$attempt->job_id = 500;
|
||||
$attempt->save();
|
||||
$reservation->state = 'uncertain';
|
||||
$reservation->save();
|
||||
Capsule::table('cloud_resource_reservation_ips')
|
||||
->where('reservation_id', (int) $reservation->id)->update([
|
||||
'state' => 'uncertain', 'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_public_ips')
|
||||
->where('reservation_id', (int) $reservation->id)->update([
|
||||
'state' => 'uncertain', 'updated_at' => $now,
|
||||
]);
|
||||
$unknown = CloudPortableRestoreAttemptCoordinator::reserve(
|
||||
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
|
||||
)->toArray();
|
||||
$assert($unknown['dispatchable'] === false
|
||||
&& $unknown['dispatch_body'] === null,
|
||||
'Unknown-effect replay exposed a dispatch instruction.');
|
||||
|
||||
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
|
||||
'state' => 'failed',
|
||||
]);
|
||||
$unknownAfterBackupDrift = CloudPortableRestoreAttemptCoordinator::reserve(
|
||||
$actor,
|
||||
CloudInstanceBackup::findOrFail(1),
|
||||
$plan,
|
||||
$plan->sha256(),
|
||||
'restored-vm',
|
||||
$requestKey
|
||||
)->toArray();
|
||||
$assert($unknownAfterBackupDrift['attempt_key'] === $first['attempt_key']
|
||||
&& $unknownAfterBackupDrift['dispatchable'] === false
|
||||
&& $unknownAfterBackupDrift['dispatch_body'] === null,
|
||||
'Backup-state drift broke stable unknown-effect replay.');
|
||||
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
|
||||
'state' => 'ready',
|
||||
]);
|
||||
|
||||
// The attempt row, not the mutable parent projection, protects the backup.
|
||||
Capsule::table('cloud_backup_restores')->where('id', $first['restore_id'])
|
||||
->update(['state' => 'failed']);
|
||||
$guard = new ReflectionMethod(
|
||||
CloudBackupService::class,
|
||||
'assertBackupDeletionAvailable'
|
||||
);
|
||||
$rejects409(
|
||||
static fn () => $guard->invoke(null, CloudInstanceBackup::findOrFail(1)),
|
||||
'An active attempt did not protect its source backup.'
|
||||
);
|
||||
|
||||
CloudPublicNetworkAllocationService::releaseBundle(
|
||||
(int) $reservation->id,
|
||||
(string) $attempt->operation_id,
|
||||
'restore_not_applied',
|
||||
true
|
||||
);
|
||||
$releasedAt = $now + 10;
|
||||
$reservation->state = 'released';
|
||||
$reservation->released_at = $releasedAt;
|
||||
$reservation->release_reason = 'restore_not_applied';
|
||||
$reservation->save();
|
||||
$attempt->state = 'failed';
|
||||
$attempt->effect_resolution = 'not_applied';
|
||||
$attempt->active_attempt = null;
|
||||
$attempt->effect_resolved_at = $releasedAt;
|
||||
$attempt->terminal_at = $releasedAt;
|
||||
$attempt->save();
|
||||
Capsule::table('cloud_jobs')->insert([
|
||||
'id' => 500,
|
||||
'user_id' => 9,
|
||||
'backup_id' => 1,
|
||||
'backup_restore_id' => $first['restore_id'],
|
||||
'host_id' => 2,
|
||||
'kind' => 'backup_restore',
|
||||
'operation_id' => (string) $attempt->operation_id,
|
||||
'phase' => 'terminal',
|
||||
'outcome' => 'failed',
|
||||
'effect_resolution' => 'not_applied',
|
||||
'terminal_at' => $releasedAt,
|
||||
'finalization_incident' => 0,
|
||||
]);
|
||||
|
||||
// A fully released terminal not-applied attempt no longer protects the object.
|
||||
$guard->invoke(null, CloudInstanceBackup::findOrFail(1));
|
||||
|
||||
// Retry authority is the exact terminal job evidence, including ownership and
|
||||
// finalization status. Each mutation must fail without creating a generation.
|
||||
$jobEvidenceMutations = [
|
||||
['kind', 'wrong_kind', 'backup_restore'],
|
||||
['host_id', 1, 2],
|
||||
['user_id', 99, 9],
|
||||
['backup_restore_id', 999, $first['restore_id']],
|
||||
['operation_id', str_repeat('9', 32), (string) $attempt->operation_id],
|
||||
['finalization_incident', 1, 0],
|
||||
];
|
||||
foreach ($jobEvidenceMutations as [$field, $invalid, $valid]) {
|
||||
Capsule::table('cloud_jobs')->where('id', 500)->update([$field => $invalid]);
|
||||
$rejects409(
|
||||
static fn () => CloudPortableRestoreAttemptCoordinator::retry(
|
||||
$actor,
|
||||
$backup,
|
||||
$plan,
|
||||
$plan->sha256(),
|
||||
'restored-vm',
|
||||
$requestKey,
|
||||
(string) $attempt->attempt_key
|
||||
),
|
||||
'Retry accepted mismatched terminal job evidence: ' . $field . '.'
|
||||
);
|
||||
$assert(CloudBackupRestoreAttempt::where(
|
||||
'backup_restore_id',
|
||||
$first['restore_id']
|
||||
)->count() === 1, 'Rejected retry created another attempt row.');
|
||||
Capsule::table('cloud_jobs')->where('id', 500)->update([$field => $valid]);
|
||||
}
|
||||
|
||||
// Released reservation and address claims remain part of the predecessor's
|
||||
// exact operation identity. A torn terminal ledger must never authorize a new
|
||||
// dispatch generation even when the job itself says not_applied.
|
||||
$ledgerOperationMutations = [
|
||||
['cloud_resource_reservations', null],
|
||||
['cloud_resource_reservation_ips', 4],
|
||||
['cloud_resource_reservation_ips', 6],
|
||||
];
|
||||
foreach ($ledgerOperationMutations as [$table, $family]) {
|
||||
$query = Capsule::table($table)->where(
|
||||
'reservation_id',
|
||||
(int) $reservation->id
|
||||
);
|
||||
if ($table === 'cloud_resource_reservations') {
|
||||
$query = Capsule::table($table)->where('id', (int) $reservation->id);
|
||||
} else {
|
||||
$query->where('address_family', $family);
|
||||
}
|
||||
$query->update(['operation_id' => str_repeat('8', 32)]);
|
||||
$rejects409(
|
||||
static fn () => CloudPortableRestoreAttemptCoordinator::retry(
|
||||
$actor,
|
||||
$backup,
|
||||
$plan,
|
||||
$plan->sha256(),
|
||||
'restored-vm',
|
||||
$requestKey,
|
||||
(string) $attempt->attempt_key
|
||||
),
|
||||
'Retry accepted torn terminal ledger operation identity: '
|
||||
. $table . ':' . (string) $family . '.'
|
||||
);
|
||||
$assert(CloudBackupRestoreAttempt::where(
|
||||
'backup_restore_id',
|
||||
$first['restore_id']
|
||||
)->count() === 1, 'Rejected ledger retry created another attempt row.');
|
||||
$query->update(['operation_id' => (string) $attempt->operation_id]);
|
||||
}
|
||||
|
||||
$ipv4Claim = Capsule::table('cloud_resource_reservation_ips')
|
||||
->where('reservation_id', (int) $reservation->id)
|
||||
->where('address_family', 4)->first();
|
||||
$assert(is_object($ipv4Claim), 'Released IPv4 claim fixture is missing.');
|
||||
Capsule::table('cloud_resource_reservation_ips')
|
||||
->where('id', (int) $ipv4Claim->id)
|
||||
->update(['public_ip_pool_id' => (int) $ipv4Claim->public_ip_pool_id + 1]);
|
||||
$rejects409(
|
||||
static fn () => CloudPortableRestoreAttemptCoordinator::retry(
|
||||
$actor,
|
||||
$backup,
|
||||
$plan,
|
||||
$plan->sha256(),
|
||||
'restored-vm',
|
||||
$requestKey,
|
||||
(string) $attempt->attempt_key
|
||||
),
|
||||
'Retry accepted a released claim that changed frozen claim-set identity.'
|
||||
);
|
||||
$assert(CloudBackupRestoreAttempt::where(
|
||||
'backup_restore_id',
|
||||
$first['restore_id']
|
||||
)->count() === 1, 'Rejected claim identity created another attempt row.');
|
||||
Capsule::table('cloud_resource_reservation_ips')
|
||||
->where('id', (int) $ipv4Claim->id)
|
||||
->update(['public_ip_pool_id' => (int) $ipv4Claim->public_ip_pool_id]);
|
||||
|
||||
foreach ([
|
||||
['claim_generation', (int) $ipv4Claim->claim_generation + 1],
|
||||
['public_ip_id', (int) $ipv4Claim->public_ip_id + 999999],
|
||||
] as [$field, $invalid]) {
|
||||
Capsule::table('cloud_resource_reservation_ips')
|
||||
->where('id', (int) $ipv4Claim->id)->update([$field => $invalid]);
|
||||
$rejects409(
|
||||
static fn () => CloudPortableRestoreAttemptCoordinator::retry(
|
||||
$actor,
|
||||
$backup,
|
||||
$plan,
|
||||
$plan->sha256(),
|
||||
'restored-vm',
|
||||
$requestKey,
|
||||
(string) $attempt->attempt_key
|
||||
),
|
||||
'Retry accepted changed released claim identity: ' . $field . '.'
|
||||
);
|
||||
$assert(CloudBackupRestoreAttempt::where(
|
||||
'backup_restore_id',
|
||||
$first['restore_id']
|
||||
)->count() === 1, 'Rejected claim mutation created another attempt row.');
|
||||
Capsule::table('cloud_resource_reservation_ips')
|
||||
->where('id', (int) $ipv4Claim->id)
|
||||
->update([$field => $ipv4Claim->{$field}]);
|
||||
}
|
||||
|
||||
// A released claim is not sufficient when its concrete address still points
|
||||
// at the predecessor. The address may later be legitimately reused by another
|
||||
// operation, but all ownership references to this operation must be gone.
|
||||
Capsule::table('cloud_public_ips')
|
||||
->where('id', (int) $ipv4Claim->public_ip_id)->update([
|
||||
'state' => 'reserved',
|
||||
'reservation_id' => (int) $reservation->id,
|
||||
'reservation_ip_claim_id' => (int) $ipv4Claim->id,
|
||||
'instance_id' => null,
|
||||
'operation_id' => (string) $attempt->operation_id,
|
||||
]);
|
||||
$rejects409(
|
||||
static fn () => CloudPortableRestoreAttemptCoordinator::retry(
|
||||
$actor,
|
||||
$backup,
|
||||
$plan,
|
||||
$plan->sha256(),
|
||||
'restored-vm',
|
||||
$requestKey,
|
||||
(string) $attempt->attempt_key
|
||||
),
|
||||
'Retry accepted a public address that retained predecessor ownership.'
|
||||
);
|
||||
$assert(CloudBackupRestoreAttempt::where(
|
||||
'backup_restore_id',
|
||||
$first['restore_id']
|
||||
)->count() === 1, 'Residual address ownership created another attempt row.');
|
||||
Capsule::table('cloud_public_ips')
|
||||
->where('id', (int) $ipv4Claim->public_ip_id)->update([
|
||||
'state' => 'available',
|
||||
'reservation_id' => null,
|
||||
'reservation_ip_claim_id' => null,
|
||||
'instance_id' => null,
|
||||
'operation_id' => null,
|
||||
]);
|
||||
|
||||
$second = CloudPortableRestoreAttemptCoordinator::retry(
|
||||
$actor,
|
||||
$backup,
|
||||
$plan,
|
||||
$plan->sha256(),
|
||||
'restored-vm',
|
||||
$requestKey,
|
||||
(string) $attempt->attempt_key
|
||||
)->toArray();
|
||||
$assert($second['generation'] === 2
|
||||
&& $second['attempt_key'] !== $first['attempt_key']
|
||||
&& $second['reservation_id'] !== $first['reservation_id']
|
||||
&& $second['dispatchable'] === true,
|
||||
'Explicit not-applied retry did not create one fresh generation.');
|
||||
$retryReplay = CloudPortableRestoreAttemptCoordinator::retry(
|
||||
$actor,
|
||||
$backup,
|
||||
$plan,
|
||||
$plan->sha256(),
|
||||
'restored-vm',
|
||||
$requestKey,
|
||||
(string) $attempt->attempt_key
|
||||
)->toArray();
|
||||
$assert($retryReplay['attempt_key'] === $second['attempt_key']
|
||||
&& $retryReplay['generation'] === 2,
|
||||
'Retry replay skipped to another generation or failed idempotency.');
|
||||
|
||||
// A queued generation is already owned by a future job and cannot be emitted
|
||||
// as another dispatch instruction.
|
||||
$secondAttempt = CloudBackupRestoreAttempt::where(
|
||||
'attempt_key',
|
||||
$second['attempt_key']
|
||||
)->firstOrFail();
|
||||
$secondAttempt->state = 'queued';
|
||||
$secondAttempt->job_id = 501;
|
||||
$secondAttempt->save();
|
||||
$queued = CloudPortableRestoreAttemptCoordinator::reserve(
|
||||
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
|
||||
)->toArray();
|
||||
$assert($queued['dispatchable'] === false && $queued['dispatch_body'] === null,
|
||||
'Queued replay exposed a second dispatch instruction.');
|
||||
|
||||
// A completed attempt replays only its durable result and never another Agent
|
||||
// instruction. Model the already-finalized instance/network ownership chain.
|
||||
$secondReservation = CloudResourceReservation::findOrFail(
|
||||
$second['reservation_id']
|
||||
);
|
||||
Capsule::table('cloud_instances')->insert([
|
||||
'id' => 2,
|
||||
'host_id' => 2,
|
||||
'status' => 'active',
|
||||
'spec_json' => json_encode([
|
||||
'public_network' => $second['dispatch_body']['public_network'],
|
||||
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES),
|
||||
'name' => 'restored-vm',
|
||||
'user_id' => 9,
|
||||
'resource_pool_id' => 200,
|
||||
'reservation_id' => (int) $secondReservation->id,
|
||||
]);
|
||||
CloudPublicNetworkAllocationService::commitBundle(
|
||||
(int) $secondReservation->id,
|
||||
2,
|
||||
(string) $secondAttempt->operation_id
|
||||
);
|
||||
$secondReservation->instance_id = 2;
|
||||
$secondReservation->state = 'committed';
|
||||
$secondReservation->updated_at = $now + 20;
|
||||
$secondReservation->save();
|
||||
$secondAttempt->state = 'succeeded';
|
||||
$secondAttempt->effect_resolution = 'applied';
|
||||
$secondAttempt->active_attempt = null;
|
||||
$secondAttempt->effect_resolved_at = $now + 20;
|
||||
$secondAttempt->terminal_at = $now + 20;
|
||||
$secondAttempt->save();
|
||||
Capsule::table('cloud_backup_restores')->where('id', $first['restore_id'])
|
||||
->update([
|
||||
'target_instance_id' => 2,
|
||||
'state' => 'ready',
|
||||
'ready_at' => $now + 20,
|
||||
]);
|
||||
$succeeded = CloudPortableRestoreAttemptCoordinator::reserve(
|
||||
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
|
||||
)->toArray();
|
||||
$assert($succeeded['attempt_key'] === $second['attempt_key']
|
||||
&& $succeeded['state'] === 'succeeded'
|
||||
&& $succeeded['dispatchable'] === false
|
||||
&& $succeeded['dispatch_body'] === null,
|
||||
'Succeeded replay exposed another dispatch instruction.');
|
||||
|
||||
$policy = (object) [
|
||||
'updated_at' => 1,
|
||||
'configured_cpu_cores' => 8,
|
||||
'configured_memory_mib' => 8192,
|
||||
'configured_disk_gib' => 100,
|
||||
'safety_cpu_cores' => 0,
|
||||
'safety_memory_mib' => 0,
|
||||
'safety_disk_gib' => 0,
|
||||
'max_instances' => 10,
|
||||
];
|
||||
$resources = ['cpu_cores' => 1, 'memory_mib' => 512, 'disk_gib' => 10];
|
||||
$validUsage = [
|
||||
'unknown' => 0,
|
||||
'instance_count' => 0,
|
||||
'cpu_sum' => 0,
|
||||
'memory_sum' => 0,
|
||||
'disk_sum' => 0,
|
||||
];
|
||||
$missingMaximum = clone $policy;
|
||||
unset($missingMaximum->max_instances);
|
||||
$missingTimestamp = clone $policy;
|
||||
unset($missingTimestamp->updated_at);
|
||||
$assert(!CloudHostCapacityAdmission::allows($policy, [], $resources)
|
||||
&& !CloudHostCapacityAdmission::allows($policy, [
|
||||
'unknown' => 0,
|
||||
'instance_count' => -1,
|
||||
'cpu_sum' => 0,
|
||||
'memory_sum' => 0,
|
||||
'disk_sum' => 0,
|
||||
], $resources)
|
||||
&& !CloudHostCapacityAdmission::allows($policy, [
|
||||
'unknown' => 1,
|
||||
'instance_count' => 0,
|
||||
'cpu_sum' => 0,
|
||||
'memory_sum' => 0,
|
||||
'disk_sum' => 0,
|
||||
], $resources)
|
||||
&& !CloudHostCapacityAdmission::allows(
|
||||
$missingMaximum,
|
||||
$validUsage,
|
||||
$resources
|
||||
)
|
||||
&& !CloudHostCapacityAdmission::allows(
|
||||
$missingTimestamp,
|
||||
$validUsage,
|
||||
$resources
|
||||
),
|
||||
'Capacity admission accepted incomplete policy or invalid usage.');
|
||||
|
||||
echo "Portable restore attempt coordinator behavior tests passed.\n";
|
||||
Loading…
Reference in New Issue