diff --git a/agent/hypervisor/libvirt_backup.go b/agent/hypervisor/libvirt_backup.go index d532f326..92898639 100644 --- a/agent/hypervisor/libvirt_backup.go +++ b/agent/hypervisor/libvirt_backup.go @@ -283,10 +283,16 @@ func abortRejectedBackupPlan(ctx context.Context, abort func(context.Context, objecttransfer.MultipartPlan) error, plan objecttransfer.MultipartPlan, rejection error) error { if cleanupErr := abort(ctx, plan); cleanupErr != nil { - // The VM and backup object were never mutated. Retain retry-safe - // semantics and the original preflight cause while keeping the bounded - // multipart cleanup failure in the private Agent error chain. - return operation.MarkNotApplied(errors.Join(rejection, cleanupErr)) + // The VM and backup object were never mutated, but PHP must not treat + // the original transient preflight cause as proof that multipart + // cleanup completed. Keep retry-safe semantics while publishing a + // distinct bounded cause that requires operator reconciliation. + return operation.MarkNotApplied(NewError( + ErrUnavailable, + "backup_upload_abort_failed", + "backup upload cleanup could not be confirmed", + errors.Join(rejection, cleanupErr), + )) } return operation.MarkNotApplied(rejection) } diff --git a/agent/hypervisor/libvirt_backup_test.go b/agent/hypervisor/libvirt_backup_test.go index 48803a81..5d1c3d6e 100644 --- a/agent/hypervisor/libvirt_backup_test.go +++ b/agent/hypervisor/libvirt_backup_test.go @@ -9,6 +9,7 @@ import ( "testing" "git.luoli.live/root/virt/agent/objecttransfer" + "git.luoli.live/root/virt/agent/operation" "libvirt.org/go/libvirt" "libvirt.org/go/libvirtxml" ) @@ -294,3 +295,29 @@ func TestAbortRejectedBackupPlanPreservesPreflightCause(t *testing.T) { t.Fatalf("called=%v err=%v", called, err) } } + +func TestAbortRejectedBackupPlanSeparatesAbortFailureCause(t *testing.T) { + rejected := NewError(ErrConflict, "backup_disk_transition_active", + "VM backup is unavailable during a disk transition", nil) + abortFailure := errors.New("multipart abort failed") + err := abortRejectedBackupPlan( + context.Background(), + func(context.Context, objecttransfer.MultipartPlan) error { + return abortFailure + }, + objecttransfer.MultipartPlan{}, + rejected, + ) + var public interface{ PublicCode() string } + var semantics interface{ FailureSemantics() (bool, bool) } + if !errors.As(err, &semantics) { + t.Fatalf("abort failure has no explicit semantics: %v", err) + } + mayHaveApplied, safeToResubmit := semantics.FailureSemantics() + if !operation.IsNotApplied(err) || !errors.As(err, &public) || + public.PublicCode() != "backup_upload_abort_failed" || + operation.IsUncertain(err) || mayHaveApplied || !safeToResubmit || + !errors.Is(err, rejected) || !errors.Is(err, abortFailure) { + t.Fatalf("unexpected abort failure classification: %v", err) + } +} diff --git a/web/app/Command/CloudSetup.php b/web/app/Command/CloudSetup.php index 0c7a638d..f92ddce5 100644 --- a/web/app/Command/CloudSetup.php +++ b/web/app/Command/CloudSetup.php @@ -8,7 +8,7 @@ use App\Services\Config; /** Initializes only the clean-break PHP business and Agent log schemas. */ final class CloudSetup { - private const SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'; + private const SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'; /** Tables installed by ss.sql before cloudInit and owned by this schema. */ private const PREINSTALLED_MAIN_TABLES = [ 'ui_copy_locales', 'ui_copy_entries', 'ui_copy_drafts', @@ -356,7 +356,8 @@ final class CloudSetup 'create_request_fingerprint', 'storage_artifact_id', 'source_instance_id', 'source_host_id', 'user_id', 'source_uuid', 'name', 'purpose', 'source', - 'subscription_id', 'scheduled_for', 'state', 'consistency', + 'subscription_id', 'scheduled_for', 'schedule_attempt', + 'state', 'consistency', 'object_key', 'object_size_bytes', 'virtual_size_bytes', 'sha256', 'object_etag', 'object_version_id', 'spec_json', 'source_portable_restore_contract_json', @@ -663,6 +664,7 @@ final class CloudSetup ], 'cloud_instance_backups' => [ 'chk_cloud_instance_backup_create_request', + 'chk_cloud_instance_backup_schedule_shape', ], 'cloud_public_ip_pool_catalog_mutex' => [ 'chk_cloud_public_ip_pool_catalog_mutex', diff --git a/web/app/Models/CloudInstanceBackup.php b/web/app/Models/CloudInstanceBackup.php index 2daa6379..f7070a33 100644 --- a/web/app/Models/CloudInstanceBackup.php +++ b/web/app/Models/CloudInstanceBackup.php @@ -24,6 +24,7 @@ final class CloudInstanceBackup extends Model 'user_id' => 'int', 'subscription_id' => 'int', 'scheduled_for' => 'int', + 'schedule_attempt' => 'int', 'object_size_bytes' => 'int', 'virtual_size_bytes' => 'int', 'storage_revision' => 'int', diff --git a/web/app/Services/Cloud/CloudAutomaticBackupService.php b/web/app/Services/Cloud/CloudAutomaticBackupService.php index b6e09d79..5ed8e38f 100644 --- a/web/app/Services/Cloud/CloudAutomaticBackupService.php +++ b/web/app/Services/Cloud/CloudAutomaticBackupService.php @@ -20,6 +20,7 @@ final class CloudAutomaticBackupService private const RETRY_SECONDS = 300; private const MAX_DEFER_COUNT = 4294967295; + private const MAX_SCHEDULE_ATTEMPT = 4294967295; private const DEFER_REASONS = [ CloudBackupEligibilityDecision::MIGRATION_ACTIVE, CloudBackupEligibilityDecision::INSTANCE_MUTATION_ACTIVE, @@ -337,7 +338,8 @@ final class CloudAutomaticBackupService */ private static function hasPendingCreateIntent( int $subscriptionId, - int $now + int $now, + ?int $scheduleAttempt = null ): bool { $probe = CloudBackupSubscription::find($subscriptionId); if (!$probe instanceof CloudBackupSubscription) { @@ -345,7 +347,8 @@ final class CloudAutomaticBackupService } return $probe->getConnection()->transaction(function () use ( $subscriptionId, - $now + $now, + $scheduleAttempt ): bool { $subscription = CloudBackupSubscription::where( 'id', $subscriptionId @@ -357,11 +360,29 @@ final class CloudAutomaticBackupService return false; } $scheduledFor = (int) $subscription->next_run_at; + if ($scheduleAttempt === null) { + $latest = CloudInstanceBackup::where( + 'subscription_id', + $subscriptionId + )->where('scheduled_for', $scheduledFor) + ->where('source', 'automatic') + ->orderByDesc('schedule_attempt') + ->orderByDesc('id')->first(); + if (!$latest instanceof CloudInstanceBackup + || (string) $latest->state === 'deleted') { + return false; + } + $scheduleAttempt = max( + 1, + (int) $latest->schedule_attempt + ); + } $requestKey = 'automatic-backup-' . $subscriptionId - . '-' . $scheduledFor; + . '-' . $scheduledFor . '-' . $scheduleAttempt; return CloudInstanceBackup::where( 'subscription_id', $subscriptionId )->where('scheduled_for', $scheduledFor) + ->where('schedule_attempt', $scheduleAttempt) ->where('source', 'automatic') // `uncertain` with no Agent job is the operator-attention // form of this same pre-dispatch intent. It must keep the @@ -376,6 +397,58 @@ final class CloudAutomaticBackupService }, 3); } + /** + * Return the durable attempt generation for the current logical slot. + * A crash after intent creation reuses that row; only a safely retired + * transient attempt authorizes the next generation. + */ + private static function scheduleAttempt( + int $subscriptionId, + int $scheduledFor + ): int { + $latest = CloudInstanceBackup::where( + 'subscription_id', + $subscriptionId + )->where('scheduled_for', $scheduledFor) + ->where('source', 'automatic') + ->orderByDesc('schedule_attempt')->orderByDesc('id')->first(); + if (!$latest instanceof CloudInstanceBackup) { + return 1; + } + $attempt = max(1, (int) $latest->schedule_attempt); + if ((string) $latest->state !== 'deleted') { + return $attempt; + } + if ((string) $latest->error_code + !== 'automatic_transient_retried') { + throw new CloudError( + '自动备份计划存在不可重用的已删除代次', + 409 + ); + } + if ($attempt >= self::MAX_SCHEDULE_ATTEMPT) { + throw new CloudError('自动备份计划重试代次已耗尽', 409); + } + return $attempt + 1; + } + + /** A bound Agent generation owns the slot until its finalizer commits. */ + private static function hasBoundCreateIntent( + int $subscriptionId, + int $scheduledFor, + int $scheduleAttempt + ): bool { + return CloudInstanceBackup::where( + 'subscription_id', + $subscriptionId + )->where('scheduled_for', $scheduledFor) + ->where('schedule_attempt', $scheduleAttempt) + ->where('source', 'automatic') + ->where('state', 'creating') + ->whereNotNull('active_job_id') + ->exists(); + } + /** * Prune at most $limit fresh retention candidates plus a separate bounded * cleanup-debt batch, so stuck remote objects cannot starve new rotation. @@ -552,8 +625,20 @@ final class CloudAutomaticBackupService public static function advanceAfterScheduled( CloudBackupSubscription $subscription, int $scheduledFor, - int $backupId + int $backupId, + int $scheduleAttempt ): void { + $latest = CloudInstanceBackup::where( + 'subscription_id', + (int) $subscription->id + )->where('scheduled_for', $scheduledFor) + ->where('source', 'automatic') + ->orderByDesc('schedule_attempt')->orderByDesc('id')->first(); + if (!$latest instanceof CloudInstanceBackup + || (int) $latest->id !== $backupId + || (int) $latest->schedule_attempt !== $scheduleAttempt) { + throw new CloudError('自动备份计划终态代次不一致', 409); + } $lastScheduled = max( 0, (int) ($subscription->last_scheduled_at ?? 0) @@ -616,6 +701,10 @@ final class CloudAutomaticBackupService self::pauseWithoutEntitlement($subscriptionId, $scheduledFor, $now); return false; } + $scheduleAttempt = self::scheduleAttempt( + (int) $subscription->id, + $scheduledFor + ); $actor = [ 'user_id' => (int) $instance->user_id, 'is_admin' => false, @@ -623,9 +712,27 @@ final class CloudAutomaticBackupService 'can_manage' => true, 'session_id' => 'automatic-backup-scheduler', 'request_idempotency_key' => 'automatic-backup-' - . (int) $subscription->id . '-' . $scheduledFor, + . (int) $subscription->id . '-' . $scheduledFor + . '-' . $scheduleAttempt, ]; - if (!self::hasPendingCreateIntent($subscriptionId, $now)) { + if (self::hasBoundCreateIntent( + $subscriptionId, + $scheduledFor, + $scheduleAttempt + )) { + self::deferCurrentSlot( + $subscriptionId, + $now, + CloudBackupEligibilityDecision::INSTANCE_MUTATION_ACTIVE, + $scheduledFor + ); + return false; + } + if (!self::hasPendingCreateIntent( + $subscriptionId, + $now, + $scheduleAttempt + )) { $eligibility = (new CloudBackupEligibility())->assess( $instance, $actor @@ -641,7 +748,11 @@ final class CloudAutomaticBackupService } } $queued = CloudBackupService::createAutomatic( - $instance, $subscription, $actor, $scheduledFor + $instance, + $subscription, + $actor, + $scheduledFor, + $scheduleAttempt ); return $queued['job'] instanceof CloudJob; } @@ -889,6 +1000,67 @@ final class CloudAutomaticBackupService $subscription->defer_count = 0; } + /** + * Re-arm the same logical slot after a strictly not-applied Agent race. + * The retired backup row is the durable generation ledger; no historical + * job or object-storage operation is ever reused. + */ + public static function rearmAfterTransientFailure( + CloudInstanceBackup $backup, + string $reason, + int $now + ): bool { + if (!in_array($reason, [ + CloudBackupEligibilityDecision::BACKUP_RESCUE_ACTIVE, + CloudBackupEligibilityDecision::BACKUP_DISK_TRANSITION_ACTIVE, + ], true) || $backup->subscription_id === null + || $backup->scheduled_for === null + || (int) $backup->schedule_attempt < 1) { + throw new \InvalidArgumentException( + 'Invalid automatic backup transient retry' + ); + } + $subscription = CloudBackupSubscription::where( + 'id', + (int) $backup->subscription_id + )->lockForUpdate()->first(); + if (!$subscription instanceof CloudBackupSubscription) { + throw new CloudError('自动备份订阅账本缺失', 409); + } + if (!(bool) $subscription->enabled + || (int) $subscription->next_run_at + !== (int) $backup->scheduled_for) { + return false; + } + $latest = CloudInstanceBackup::where( + 'subscription_id', + (int) $subscription->id + )->where('scheduled_for', (int) $backup->scheduled_for) + ->where('source', 'automatic') + ->orderByDesc('schedule_attempt')->orderByDesc('id') + ->lockForUpdate()->first(); + if (!$latest instanceof CloudInstanceBackup + || (int) $latest->id !== (int) $backup->id + || (int) $latest->schedule_attempt + !== (int) $backup->schedule_attempt + || (string) $latest->state !== 'deleted' + || (string) $latest->error_code + !== 'automatic_transient_retried') { + throw new CloudError('自动备份重试代次账本不一致', 409); + } + $count = max(0, (int) $subscription->defer_count); + $subscription->deferred_until = $now + self::RETRY_SECONDS; + $subscription->defer_reason = $reason; + $subscription->deferred_at = $now; + $subscription->defer_count = min( + self::MAX_DEFER_COUNT, + $count + 1 + ); + $subscription->updated_at = $now; + $subscription->save(); + return true; + } + private static function backupsForInstance(CloudInstance $instance): array { $items = []; diff --git a/web/app/Services/Cloud/CloudBackupPublicationPreparation.php b/web/app/Services/Cloud/CloudBackupPublicationPreparation.php index 8e5110d8..3fc7ead3 100644 --- a/web/app/Services/Cloud/CloudBackupPublicationPreparation.php +++ b/web/app/Services/Cloud/CloudBackupPublicationPreparation.php @@ -581,6 +581,8 @@ final class CloudBackupPublicationPreparation (int) $backup->subscription_id; $canonical['scheduled_for'] = (int) $backup->scheduled_for; + $canonical['schedule_attempt'] = + (int) $backup->schedule_attempt; } $operationId = substr(hash( 'sha256', @@ -635,17 +637,10 @@ final class CloudBackupPublicationPreparation $backup->updated_at = time(); $backup->save(); - if ($subscription instanceof CloudBackupSubscription) { - if ((int) $subscription->last_scheduled_at - !== (int) $backup->scheduled_for - || (int) $subscription->last_backup_id !== $backupId) { - CloudAutomaticBackupService::advanceAfterScheduled( - $subscription, - (int) $backup->scheduled_for, - $backupId - ); - } - } + // Keep next_run_at on the original logical slot until this + // exact Agent generation reaches a durable terminal result. + // The finalizer advances ordinary outcomes or re-arms only a + // strictly proven not-applied transient race. return $job; }, 3 diff --git a/web/app/Services/Cloud/CloudBackupService.php b/web/app/Services/Cloud/CloudBackupService.php index 68896716..fc0b03a6 100644 --- a/web/app/Services/Cloud/CloudBackupService.php +++ b/web/app/Services/Cloud/CloudBackupService.php @@ -568,9 +568,9 @@ final class CloudBackupService /** Queue one deterministic paid automatic backup for a scheduled slot. */ public static function createAutomatic(CloudInstance $instance, CloudBackupSubscription $subscription, array $actor, - int $scheduledFor): array + int $scheduledFor, int $scheduleAttempt): array { - if ($scheduledFor < 1) { + if ($scheduledFor < 1 || $scheduleAttempt < 1) { throw new CloudError('自动备份计划时间无效', 409); } $zone = new \DateTimeZone(CloudAutomaticBackupService::TIMEZONE); @@ -583,6 +583,7 @@ final class CloudBackupService ], [ 'subscription_id' => (int) $subscription->id, 'scheduled_for' => $scheduledFor, + 'schedule_attempt' => $scheduleAttempt, ]); } @@ -613,6 +614,8 @@ final class CloudBackupService (int) $automatic['subscription_id']; $canonicalRequest['scheduled_for'] = (int) $automatic['scheduled_for']; + $canonicalRequest['schedule_attempt'] = + (int) $automatic['schedule_attempt']; } $fingerprint = CloudJobService::standaloneRequestFingerprint( (int) $instance->host_id, @@ -701,6 +704,7 @@ final class CloudBackupService 'id', (int) $automatic['subscription_id'] )->lockForUpdate()->first(); $scheduledFor = (int) $automatic['scheduled_for']; + $scheduleAttempt = (int) $automatic['schedule_attempt']; $entitled = $automaticSubscription !== null && CloudBackupPeriodEntitlement::where( 'subscription_id', @@ -719,10 +723,18 @@ final class CloudBackupService || !$entitled) { throw new CloudError('自动备份订阅与实例账本不一致', 409); } + self::assertAutomaticScheduleAttempt( + $automaticSubscription, + $scheduledFor, + $scheduleAttempt + ); $existing = CloudInstanceBackup::where( 'subscription_id', (int) $automaticSubscription->id )->where('scheduled_for', - $scheduledFor)->first(); + $scheduledFor)->where( + 'schedule_attempt', + $scheduleAttempt + )->first(); if ($existing !== null) { if ((int) ($existing->create_request_actor_id ?? 0) !== $actorId @@ -834,6 +846,10 @@ final class CloudBackupService : (int) $automaticSubscription->id; $backup->scheduled_for = $automatic === null ? null : (int) $automatic['scheduled_for']; + if ($automatic !== null) { + $backup->schedule_attempt = + (int) $automatic['schedule_attempt']; + } $backup->state = 'creating'; $backup->consistency = $mode; $backup->object_key = $objectKey; @@ -1081,10 +1097,18 @@ final class CloudBackupService /** Apply a terminal/running Agent result inside CloudWorkerService's lock. */ public static function finishJobLocked(CloudJob $job, string $status, - $result, array $payload, bool $deferPublish = false): void + $result, array $payload, bool $deferPublish = false, + array $operation = []): void { if ($job->kind === 'backup_create') { - self::finishCreateLocked($job, $status, $result, $payload); + self::finishCreateLocked( + $job, + $status, + $result, + $payload, + $deferPublish, + $operation + ); } elseif ($job->kind === 'backup_restore') { CloudPortableRestoreFinalizer::finishJobLocked( $job, $status, $result, $payload, $deferPublish @@ -1252,7 +1276,8 @@ final class CloudBackupService } private static function finishCreateLocked(CloudJob $job, string $status, - $result, array $payload): void + $result, array $payload, bool $deferPublish, + array $operation): void { $backup = $job->backup_id === null ? null : CloudInstanceBackup::where('id', (int) $job->backup_id) @@ -1295,6 +1320,32 @@ final class CloudBackupService ); } $now = time(); + $transientRetryReason = self::automaticTransientRetryReason( + $backup, + $job, + $status, + $deferPublish, + $operation + ); + if ((string) $backup->state === 'deleted' + && (string) $backup->error_code + === 'automatic_transient_retried') { + if ($transientRetryReason === null + || $backup->sealed_credential !== null + || $backup->active_job_id !== null) { + throw new CloudRecoveryInstanceStateConflict( + '自动备份安全重试终态与重复回执不一致', + 409 + ); + } + CloudStorageArtifactService::retireBlankNotAppliedPublication( + $storageOperationId, + (int) $artifactId, + (int) $backup->id, + $now + ); + return; + } if ($status === 'uncertain') { CloudStorageArtifactService::failPrimary( $storageOperationId, @@ -1311,6 +1362,27 @@ final class CloudBackupService 'not_applied', 'backup_agent_failed' ); + if ($transientRetryReason !== null) { + CloudStorageArtifactService:: + retireBlankNotAppliedPublication( + $storageOperationId, + (int) $artifactId, + (int) $backup->id, + $now + ); + $backup->state = 'deleted'; + $backup->sealed_credential = null; + $backup->active_job_id = null; + $backup->error_code = 'automatic_transient_retried'; + $backup->updated_at = $now; + $backup->save(); + CloudAutomaticBackupService::rearmAfterTransientFailure( + $backup, + $transientRetryReason, + $now + ); + return; + } $backup->state = 'failed'; $backup->error_code = 'backup_failed'; $backup->active_job_id = null; @@ -1414,6 +1486,50 @@ final class CloudBackupService ); } } + if ((string) $backup->source === 'automatic') { + $subscription = CloudBackupSubscription::where( + 'id', + (int) $backup->subscription_id + )->lockForUpdate()->first(); + if (!$subscription instanceof CloudBackupSubscription + || (int) $subscription->instance_id + !== (int) $backup->source_instance_id + || (int) $subscription->user_id !== (int) $backup->user_id) { + throw new CloudRecoveryInstanceStateConflict( + '自动备份终态与订阅账本不一致', + 409 + ); + } + CloudAutomaticBackupService::advanceAfterScheduled( + $subscription, + (int) $backup->scheduled_for, + (int) $backup->id, + (int) $backup->schedule_attempt + ); + } + } + + private static function automaticTransientRetryReason( + CloudInstanceBackup $backup, + CloudJob $job, + string $status, + bool $deferPublish, + array $operation + ): ?string { + if ((string) $backup->source !== 'automatic' + || $status !== 'failed' + || $deferPublish + || (bool) ($job->finalization_incident ?? false) + || ($operation['status'] ?? null) !== 'failed' + || ($operation['effect_may_have_applied'] ?? null) !== false + || ($operation['safe_to_resubmit'] ?? null) !== true) { + return null; + } + $reason = $operation['cause_code'] ?? null; + return in_array($reason, [ + CloudBackupEligibilityDecision::BACKUP_RESCUE_ACTIVE, + CloudBackupEligibilityDecision::BACKUP_DISK_TRANSITION_ACTIVE, + ], true) ? $reason : null; } private static function finishRecoveryLocked(CloudJob $job, @@ -1815,6 +1931,45 @@ final class CloudBackupService return $key; } + /** + * Reject a stale scheduler generation while the subscription lock is + * held. A safely retired row is the sole authority for incrementing the + * generation; every other state must be resumed idempotently. + */ + private static function assertAutomaticScheduleAttempt( + CloudBackupSubscription $subscription, + int $scheduledFor, + int $scheduleAttempt + ): void { + if ($scheduleAttempt < 1) { + throw new CloudError('自动备份计划重试代次无效', 409); + } + $latest = CloudInstanceBackup::where( + 'subscription_id', + (int) $subscription->id + )->where('scheduled_for', $scheduledFor) + ->where('source', 'automatic') + ->orderByDesc('schedule_attempt')->orderByDesc('id')->first(); + $expected = 1; + if ($latest instanceof CloudInstanceBackup) { + $expected = max(1, (int) $latest->schedule_attempt); + if ((string) $latest->state === 'deleted') { + if ((string) $latest->error_code + !== 'automatic_transient_retried' + || $expected >= 4294967295) { + throw new CloudError( + '自动备份计划存在不可重用的已删除代次', + 409 + ); + } + $expected++; + } + } + if ($scheduleAttempt !== $expected) { + throw new CloudError('自动备份计划重试代次已变化', 409); + } + } + private static function findCreateIntent( int $actorId, string $requestKey, diff --git a/web/app/Services/Cloud/CloudStorageArtifactService.php b/web/app/Services/Cloud/CloudStorageArtifactService.php index ea922b3c..5997f3ad 100644 --- a/web/app/Services/Cloud/CloudStorageArtifactService.php +++ b/web/app/Services/Cloud/CloudStorageArtifactService.php @@ -543,6 +543,127 @@ final class CloudStorageArtifactService return self::publicationView($operationId); } + /** + * Retire a publication whose Agent proved that neither VM backup data nor + * its multipart upload survived a rejected preflight. Operation rows are + * retained as immutable audit evidence; no provider request is replayed. + */ + public static function retireBlankNotAppliedPublication( + string $operationId, + int $expectedArtifactId, + int $expectedBackupId, + int $now + ): void { + self::assertOperationId($operationId); + if ($expectedArtifactId < 1 || $expectedBackupId < 1 || $now < 1) { + throw new CloudError('空白对象存储制品退休参数无效', 422); + } + $connection = (new CloudStorageOperation())->getConnection(); + $connection->transaction(static function () use ( + $operationId, + $expectedArtifactId, + $expectedBackupId, + $now + ): void { + $operation = self::primaryOperationLocked($operationId); + if ((int) $operation->artifact_id !== $expectedArtifactId) { + throw new CloudError('空白发布操作与制品身份不一致', 409); + } + $copies = self::copyOperationsLocked($expectedArtifactId); + [$artifact, $replicas] = self::artifactContextLocked($operation); + $publication = self::publicationLocked($expectedArtifactId); + if ((string) $artifact->state === 'deleted' + && (string) $publication->state === 'retired') { + if ((int) ($publication->backup_id ?? 0) + !== $expectedBackupId + || (int) $artifact->size_bytes !== 0 + || (string) $artifact->sha256 !== '' + || $publication->finalized_at === null) { + throw new CloudError('空白发布退休终态与归属不一致', 409); + } + foreach ($replicas as $replica) { + if ((string) $replica->state !== 'deleted') { + throw new CloudError('空白发布副本退休终态不完整', 409); + } + } + return; + } + self::assertTerminalFailure( + $operation, + false, + true, + 'not_applied', + 'backup_agent_failed', + '主副本发布尚未证明为未应用' + ); + if ((string) $artifact->state !== 'publishing' + || (int) $artifact->size_bytes !== 0 + || (string) $artifact->sha256 !== '' + || (string) $publication->state !== 'failed' + || (int) ($publication->backup_id ?? 0) + !== $expectedBackupId + || $publication->coverage_ready_at !== null + || $publication->finalized_at !== null) { + throw new CloudError('对象发布不属于可安全退休的空白终态', 409); + } + $operationsByReplica = [ + (int) $operation->replica_id => $operation, + ]; + foreach ($copies as $copy) { + self::assertTerminalFailure( + $copy, + false, + false, + 'not_applied', + 'primary_publish_not_applied', + '副本复制尚未证明为未应用' + ); + $replicaId = (int) $copy->replica_id; + if ($replicaId < 1 || isset($operationsByReplica[$replicaId])) { + throw new CloudError('空白发布副本操作身份冲突', 409); + } + $operationsByReplica[$replicaId] = $copy; + } + if (count($operationsByReplica) !== count($replicas)) { + throw new CloudError('空白发布副本操作账本不完整', 409); + } + foreach ($replicas as $replica) { + $bound = $operationsByReplica[(int) $replica->id] ?? null; + if (!$bound instanceof CloudStorageOperation + || !is_string($replica->publish_operation_id) + || !hash_equals( + (string) $bound->operation_id, + (string) $replica->publish_operation_id + ) + || (int) $replica->size_bytes !== 0 + || (string) $replica->sha256 !== '' + || (string) $replica->object_etag !== '' + || (string) $replica->object_version_id !== '' + || (int) $replica->verified_at !== 0 + || $replica->delete_operation_id !== null) { + throw new CloudError('空白发布副本已有对象或删除身份', 409); + } + $replica->state = 'deleted'; + $replica->lease_token = null; + $replica->lease_until = 0; + $replica->error_code = 'blank_publication_retired'; + $replica->updated_at = $now; + $replica->save(); + } + $artifact->state = 'deleted'; + $artifact->updated_at = $now; + $artifact->save(); + $publication->state = 'retired'; + $publication->available_at = 0; + $publication->lease_token = null; + $publication->lease_until = 0; + $publication->last_error_code = 'blank_publication_retired'; + $publication->finalized_at = $now; + $publication->updated_at = $now; + $publication->save(); + }); + } + /** * Mint a read-only URL for one verified replica selected by health. * diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index 5fe0a899..d3bea080 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -2620,7 +2620,8 @@ final class CloudWorkerService $catalogStatus, $result, $payload, - $preparedRecoveryMarker !== null + $preparedRecoveryMarker !== null, + $operation ); CloudSnapshotService::finishInstanceDeleteLocked( $job, $catalogStatus diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index caedb983..a06a7749 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -593,7 +593,7 @@ CREATE TABLE cloud_storage_artifacts ( CONSTRAINT chk_cloud_storage_artifact_identity CHECK ( maximum_size_bytes > 0 AND ( - (state = 'publishing' AND size_bytes = 0 AND sha256 = '') + (state IN ('publishing','deleted') AND size_bytes = 0 AND sha256 = '') OR ( size_bytes BETWEEN 1 AND maximum_size_bytes AND sha256 REGEXP '^[a-f0-9]{64}$' @@ -1415,6 +1415,7 @@ CREATE TABLE cloud_instance_backups ( source VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'manual', subscription_id BIGINT UNSIGNED DEFAULT NULL, scheduled_for BIGINT UNSIGNED DEFAULT NULL, + schedule_attempt INT UNSIGNED NOT NULL DEFAULT 0, state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'creating', consistency VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, object_key VARCHAR(512) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, @@ -1446,7 +1447,9 @@ CREATE TABLE cloud_instance_backups ( UNIQUE KEY cloud_instance_backup_create_request (create_request_actor_id, create_request_idempotency_key), UNIQUE KEY cloud_instance_backup_storage (storage_artifact_id), UNIQUE KEY cloud_instance_backup_object (object_key), - UNIQUE KEY cloud_instance_backup_schedule (subscription_id, scheduled_for), + UNIQUE KEY cloud_instance_backup_schedule ( + subscription_id, scheduled_for, schedule_attempt + ), KEY cloud_instance_backup_source (source_instance_id, state, id), KEY cloud_instance_backup_owner (user_id, id), KEY cloud_instance_backup_operation (last_operation_id), @@ -1460,6 +1463,12 @@ CREATE TABLE cloud_instance_backups ( AND CHAR_LENGTH(create_request_idempotency_key) BETWEEN 16 AND 128 AND create_request_fingerprint REGEXP '^[a-f0-9]{64}$') ), + CONSTRAINT chk_cloud_instance_backup_schedule_shape CHECK ( + (source = 'manual' AND subscription_id IS NULL + AND scheduled_for IS NULL AND schedule_attempt = 0) + OR (source = 'automatic' AND subscription_id IS NOT NULL + AND scheduled_for IS NOT NULL AND schedule_attempt >= 1) + ), CONSTRAINT fk_cloud_instance_backup_storage FOREIGN KEY (storage_artifact_id) REFERENCES cloud_storage_artifacts (id) ON DELETE RESTRICT ON UPDATE RESTRICT @@ -1512,8 +1521,9 @@ CREATE TABLE cloud_storage_publications ( CONSTRAINT chk_cloud_storage_publication_coverage CHECK ( (state IN ('waiting_primary','waiting_protection') AND coverage_ready_at IS NULL) - OR (state IN ('ready_to_finalize','finalizing','completed','retired') + OR (state IN ('ready_to_finalize','finalizing','completed') AND coverage_ready_at IS NOT NULL) + OR state = 'retired' OR state = 'failed' ), CONSTRAINT fk_cloud_storage_publication_artifact FOREIGN KEY (artifact_id) diff --git a/web/tests/CloudAutomaticBackupDurableIntentTest.php b/web/tests/CloudAutomaticBackupDurableIntentTest.php index a1bc6cb7..6e98efee 100644 --- a/web/tests/CloudAutomaticBackupDurableIntentTest.php +++ b/web/tests/CloudAutomaticBackupDurableIntentTest.php @@ -103,6 +103,7 @@ $schema->create('cloud_instance_backups', $t->string('source'); $t->unsignedBigInteger('subscription_id')->nullable(); $t->unsignedBigInteger('scheduled_for')->nullable(); + $t->unsignedInteger('schedule_attempt')->default(0); $t->string('state'); $t->string('consistency'); $t->unsignedBigInteger('active_job_id')->nullable(); @@ -165,6 +166,7 @@ $canonical = [ 'source' => 'automatic', 'subscription_id' => (int) $subscription->id, 'scheduled_for' => $scheduledFor, + 'schedule_attempt' => 1, ]; $fingerprint = CloudJobService::standaloneRequestFingerprint( (int) $instance->host_id, @@ -174,7 +176,7 @@ $fingerprint = CloudJobService::standaloneRequestFingerprint( $canonical ); $requestKey = 'automatic-backup-' . (int) $subscription->id - . '-' . $scheduledFor; + . '-' . $scheduledFor . '-1'; $intent = new CloudInstanceBackup(); foreach ([ 'backup_key' => str_repeat('a', 32), @@ -191,6 +193,7 @@ foreach ([ 'source' => 'automatic', 'subscription_id' => (int) $subscription->id, 'scheduled_for' => $scheduledFor, + 'schedule_attempt' => 1, 'state' => 'creating', 'consistency' => 'filesystem', 'active_job_id' => null, @@ -213,6 +216,9 @@ $assert($queued === 0 && (int) $subscription->next_run_at === $scheduledFor && $subscription->last_scheduled_at === null && $subscription->last_backup_id === null + && (int) $subscription->deferred_until === 0 + && (string) $subscription->defer_reason === '' + && (int) $subscription->defer_count === 0 && CloudInstanceBackup::where( 'subscription_id', (int) $subscription->id )->count() === 1, diff --git a/web/tests/CloudBackupMigrationFenceTest.php b/web/tests/CloudBackupMigrationFenceTest.php index 507f426e..010e34be 100644 --- a/web/tests/CloudBackupMigrationFenceTest.php +++ b/web/tests/CloudBackupMigrationFenceTest.php @@ -110,6 +110,7 @@ $schema->create('cloud_instance_backups', $table->increments('id'); $table->integer('subscription_id')->nullable(); $table->integer('scheduled_for')->nullable(); + $table->integer('schedule_attempt')->default(0); $table->string('source'); $table->string('state'); $table->integer('storage_artifact_id')->nullable(); diff --git a/web/tests/CloudBackupPublicationPreparationSchemaContractTest.php b/web/tests/CloudBackupPublicationPreparationSchemaContractTest.php index 7013ada4..28b46083 100644 --- a/web/tests/CloudBackupPublicationPreparationSchemaContractTest.php +++ b/web/tests/CloudBackupPublicationPreparationSchemaContractTest.php @@ -27,6 +27,7 @@ foreach ([ 'create_request_actor_id BIGINT UNSIGNED DEFAULT NULL', 'create_request_idempotency_key VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL', 'create_request_fingerprint CHAR(64) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL', + 'schedule_attempt INT UNSIGNED NOT NULL DEFAULT 0', ] as $column) { $assert(str_contains($backup, $column), 'Missing durable backup request field: ' . $column); @@ -40,6 +41,15 @@ $assert(str_contains($backup, && str_contains($backup, "create_request_fingerprint REGEXP '^[a-f0-9]{64}$'"), 'Backup request identity is not atomically shaped or uniquely fenced.'); +$assert(str_contains($backup, + 'UNIQUE KEY cloud_instance_backup_schedule (') + && str_contains($backup, + 'subscription_id, scheduled_for, schedule_attempt') + && str_contains($backup, + 'CONSTRAINT chk_cloud_instance_backup_schedule_shape CHECK (') + && str_contains($backup, 'schedule_attempt = 0') + && str_contains($backup, 'schedule_attempt >= 1'), + 'Automatic backup schedule attempts are not durably generation-fenced.'); foreach ([ 'create_request_actor_id', @@ -47,18 +57,21 @@ foreach ([ 'create_request_fingerprint', 'cloud_instance_backup_create_request', 'chk_cloud_instance_backup_create_request', + 'schedule_attempt', + 'chk_cloud_instance_backup_schedule_shape', ] as $required) { $assert(str_contains($setup, "'" . $required . "'"), 'CloudSetup omits backup preparation contract: ' . $required); } $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'") + "SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'") && str_contains($setup, "'columns' => [\n 'create_request_actor_id',\n 'create_request_idempotency_key',") && str_contains($setup, "'non_unique' => 0"), 'CloudSetup does not verify the exact unique request fence.'); $assert(str_contains($model, "'create_request_actor_id' => 'int'") && str_contains($model, "'storage_artifact_id' => 'int'") + && str_contains($model, "'schedule_attempt' => 'int'") && str_contains($model, "'create_request_idempotency_key'") && str_contains($model, "'create_request_fingerprint'"), 'Backup model exposes or miscasts durable preparation identity.'); diff --git a/web/tests/CloudBackupPublicationPreparationTest.php b/web/tests/CloudBackupPublicationPreparationTest.php index ee8bb64c..2b3fd642 100644 --- a/web/tests/CloudBackupPublicationPreparationTest.php +++ b/web/tests/CloudBackupPublicationPreparationTest.php @@ -168,6 +168,7 @@ namespace { $t->string('source_uuid'); $t->string('name'); $t->string('purpose'); $t->string('source'); $t->unsignedBigInteger('subscription_id')->nullable(); $t->unsignedBigInteger('scheduled_for')->nullable(); + $t->unsignedInteger('schedule_attempt')->default(0); $t->string('state'); $t->string('consistency'); $t->string('object_key'); $t->text('source_portable_restore_contract_json'); $t->string('source_portable_restore_contract_sha256'); @@ -302,6 +303,7 @@ namespace { $canonical['source'] = 'automatic'; $canonical['subscription_id'] = $subscriptionId; $canonical['scheduled_for'] = $scheduledFor; + $canonical['schedule_attempt'] = 1; } $target = '/v1/vms/' . rawurlencode($uuid) . '/backups'; $fingerprint = CloudJobService::standaloneRequestFingerprint( @@ -337,6 +339,7 @@ namespace { 'user_id' => 33, 'source_uuid' => $uuid, 'name' => 'backup-' . $id, 'purpose' => 'portable', 'source' => $source, 'subscription_id' => $subscriptionId, 'scheduled_for' => $scheduledFor, + 'schedule_attempt' => $source === 'automatic' ? 1 : 0, 'state' => 'creating', 'consistency' => 'filesystem', 'object_key' => 'pending-' . $id, 'source_portable_restore_contract_json' => '{}', @@ -476,7 +479,7 @@ namespace { 'A second worker/call created a duplicate recovered job.'); // Automatic intent without a pre-existing job uses the same preparation - // path and advances its subscription exactly when the job binding commits. + // path while preserving the logical slot until terminal finalization. $scheduledFor = $now - 60; $subscription = new CloudBackupSubscription(); foreach ([ @@ -514,14 +517,13 @@ namespace { $automaticJob = $automaticService->ensureAgentJob((int) $automatic->id); $subscription->refresh(); $assert($automaticJob instanceof CloudJob - && (int) $subscription->last_scheduled_at === $scheduledFor - && (int) $subscription->last_backup_id === (int) $automatic->id - && (int) $subscription->next_run_at > $scheduledFor, - 'Automatic backup intent did not atomically bind its job and advance schedule.'); - $advancedNextRun = (int) $subscription->next_run_at; + && $subscription->last_scheduled_at === null + && $subscription->last_backup_id === null + && (int) $subscription->next_run_at === $scheduledFor, + 'Automatic backup job binding advanced the logical slot before terminal finalization.'); $automaticService->ensureAgentJob((int) $automatic->id); $subscription->refresh(); - $assert((int) $subscription->next_run_at === $advancedNextRun + $assert((int) $subscription->next_run_at === $scheduledFor && CloudJob::where('backup_id', (int) $automatic->id)->count() === 1, 'Repeated automatic preparation advanced the schedule or duplicated the job.'); @@ -654,7 +656,8 @@ namespace { // The user changes the schedule while CreateMultipartUpload is in // progress. The old slot may still bind, but it must not overwrite the - // newly selected frequency or next execution time. + // newly selected frequency or next execution time, nor mark the slot + // complete before terminal finalization. $staleSlot = $clock - 60; $newNextRun = $clock + 7 * 86400; $changedSubscription = new CloudBackupSubscription(); @@ -704,9 +707,8 @@ namespace { && (string) $changedSubscription->frequency === 'weekly' && (int) $changedSubscription->weekday === 3 && (int) $changedSubscription->next_run_at === $newNextRun - && (int) $changedSubscription->last_scheduled_at === $staleSlot - && (int) $changedSubscription->last_backup_id - === (int) $changedBackup->id, + && $changedSubscription->last_scheduled_at === null + && $changedSubscription->last_backup_id === null, 'An old backup intent overwrote the user\'s newer schedule.'); // Disabling the subscription during provider preparation is also a real diff --git a/web/tests/CloudBackupStorageFinalizerTest.php b/web/tests/CloudBackupStorageFinalizerTest.php index bfe3921f..3b202d48 100644 --- a/web/tests/CloudBackupStorageFinalizerTest.php +++ b/web/tests/CloudBackupStorageFinalizerTest.php @@ -3,6 +3,7 @@ require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudInstanceBackup; +use App\Models\CloudBackupSubscription; use App\Models\CloudJob; use App\Models\CloudStorageArtifact; use App\Models\CloudStorageOperation; @@ -128,6 +129,7 @@ $schema->create('cloud_instance_backups', static function (Blueprint $t): void { $t->string('purpose')->default('portable'); $t->string('source')->default('manual'); $t->unsignedBigInteger('subscription_id')->nullable(); $t->unsignedBigInteger('scheduled_for')->nullable(); + $t->unsignedInteger('schedule_attempt')->default(0); $t->string('state'); $t->string('consistency'); $t->string('object_key'); $t->unsignedBigInteger('object_size_bytes')->default(0); $t->unsignedBigInteger('virtual_size_bytes')->default(0); @@ -150,6 +152,33 @@ $schema->create('cloud_jobs', static function (Blueprint $t): void { $t->unsignedBigInteger('user_id')->nullable(); $t->string('operation_id')->unique(); $t->string('effect_resolution')->default('not_applicable'); + $t->boolean('finalization_incident')->default(false); +}); +$schema->create('cloud_backup_subscriptions', + static function (Blueprint $t): void { + $t->increments('id'); + $t->unsignedBigInteger('instance_id')->unique(); + $t->unsignedBigInteger('user_id'); + $t->unsignedBigInteger('offer_id'); + $t->boolean('enabled'); + $t->boolean('renew_next_period'); + $t->string('frequency'); + $t->unsignedTinyInteger('weekday'); + $t->unsignedTinyInteger('retention_count'); + $t->string('timezone'); + $t->string('price'); + $t->string('currency'); + $t->unsignedInteger('price_version'); + $t->unsignedInteger('state_revision')->default(1); + $t->unsignedBigInteger('next_run_at'); + $t->unsignedBigInteger('deferred_until')->default(0); + $t->string('defer_reason')->default(''); + $t->unsignedBigInteger('deferred_at')->nullable(); + $t->unsignedInteger('defer_count')->default(0); + $t->unsignedBigInteger('last_scheduled_at')->nullable(); + $t->unsignedBigInteger('last_backup_id')->nullable(); + $t->unsignedBigInteger('created_at'); + $t->unsignedBigInteger('updated_at'); }); $now = time(); @@ -186,13 +215,42 @@ $sourceSpec = json_encode([ 'disk_gib' => 20, 'bandwidth_mbps' => 1000, ], JSON_THROW_ON_ERROR); -$fixture = static function (int $id) use ( +$fixture = static function (int $id, bool $automatic = false) use ( $now, $sourceSpec, $compatibility ): array { $backupKey = substr(hash('sha256', 'storage-finalizer-' . $id), 0, 32); $agentOperationId = substr(hash('sha256', 'agent-' . $id), 0, 32); + $scheduledFor = $automatic ? $now - 60 : null; + $subscriptionId = $automatic ? 100 + $id : null; + if ($automatic) { + Capsule::table('cloud_backup_subscriptions')->insert([ + 'id' => $subscriptionId, + 'instance_id' => 100 + $id, + 'user_id' => 9, + 'offer_id' => 1, + 'enabled' => 1, + 'renew_next_period' => 1, + 'frequency' => 'daily', + 'weekday' => 0, + 'retention_count' => 2, + 'timezone' => 'UTC', + 'price' => '2.00', + 'currency' => 'USD', + 'price_version' => 1, + 'state_revision' => 1, + 'next_run_at' => $scheduledFor, + 'deferred_until' => 0, + 'defer_reason' => '', + 'deferred_at' => null, + 'defer_count' => 0, + 'last_scheduled_at' => null, + 'last_backup_id' => null, + 'created_at' => $now, + 'updated_at' => $now, + ]); + } Capsule::table('cloud_instance_backups')->insert([ 'id' => $id, 'backup_key' => $backupKey, 'storage_artifact_id' => null, @@ -200,6 +258,10 @@ $fixture = static function (int $id) use ( 'user_id' => 9, 'source_uuid' => sprintf('00000000-0000-4000-8000-%012d', $id), 'name' => 'storage-finalizer-' . $id, 'state' => 'creating', + 'source' => $automatic ? 'automatic' : 'manual', + 'subscription_id' => $subscriptionId, + 'scheduled_for' => $scheduledFor, + 'schedule_attempt' => $automatic ? 1 : 0, 'consistency' => 'filesystem', 'object_key' => 'vm-backups/' . $backupKey . '/system.qcow2', 'source_portable_restore_contract_json' => $compatibility->json(), @@ -244,6 +306,8 @@ $fixture = static function (int $id) use ( 'job' => CloudJob::findOrFail($id), 'storage_operation_id' => (string) $publication['operation_id'], 'artifact_id' => (int) $publication['artifact_id'], + 'subscription_id' => $subscriptionId, + 'scheduled_for' => $scheduledFor, ]; }; $payload = static fn (array $item): array => [ @@ -382,4 +446,232 @@ $assert((string) CloudInstanceBackup::findOrFail(5)->state === 'creating' )->value('phase') === 'awaiting_receipt', 'A foreign storage operation mutated either artifact ledger.'); +$safeTransient = [ + 'status' => 'failed', + 'effect_may_have_applied' => false, + 'safe_to_resubmit' => true, + 'cause_code' => 'backup_disk_transition_active', +]; +$automaticRetry = $fixture(7, true); +$retryStartedAt = time(); +CloudBackupService::finishJobLocked( + $automaticRetry['job'], + 'failed', + null, + $payload($automaticRetry), + false, + $safeTransient +); +$retiredBackup = CloudInstanceBackup::findOrFail(7); +$retiredArtifact = CloudStorageArtifact::findOrFail( + $automaticRetry['artifact_id'] +); +$retiredPublication = CloudStoragePublication::where( + 'artifact_id', + $automaticRetry['artifact_id'] +)->firstOrFail(); +$retiredSubscription = CloudBackupSubscription::findOrFail( + $automaticRetry['subscription_id'] +); +$retiredOperations = CloudStorageOperation::where( + 'artifact_id', + $automaticRetry['artifact_id'] +)->get(); +$assert((string) $retiredBackup->state === 'deleted' + && (string) $retiredBackup->error_code + === 'automatic_transient_retried' + && $retiredBackup->sealed_credential === null + && $retiredBackup->active_job_id === null + && (string) $retiredArtifact->state === 'deleted' + && (string) $retiredPublication->state === 'retired' + && $retiredPublication->coverage_ready_at === null + && (int) $retiredPublication->finalized_at >= $retryStartedAt + && CloudStorageReplica::where( + 'artifact_id', + $automaticRetry['artifact_id'] + )->where('state', '<>', 'deleted')->count() === 0 + && $retiredOperations->count() === 2 + && $retiredOperations->every( + static fn (CloudStorageOperation $operation): bool => + (string) $operation->phase === 'terminal' + && (string) $operation->outcome === 'failed' + && (string) $operation->effect_resolution === 'not_applied' + ) + && (int) $retiredSubscription->next_run_at + === (int) $automaticRetry['scheduled_for'] + && (string) $retiredSubscription->defer_reason + === 'backup_disk_transition_active' + && (int) $retiredSubscription->defer_count === 1 + && (int) $retiredSubscription->deferred_until + >= $retryStartedAt + 300 + && $retiredSubscription->last_scheduled_at === null + && $retiredSubscription->last_backup_id === null, + 'A proven not-applied automatic race was not retired and re-armed on the same slot.'); +$retryDeferredUntil = (int) $retiredSubscription->deferred_until; +CloudBackupService::finishJobLocked( + $automaticRetry['job'], + 'failed', + null, + $payload($automaticRetry), + false, + $safeTransient +); +$retiredSubscription->refresh(); +$assert((int) $retiredSubscription->defer_count === 1 + && (int) $retiredSubscription->deferred_until === $retryDeferredUntil + && CloudStorageOperation::where( + 'artifact_id', + $automaticRetry['artifact_id'] + )->count() === 2, + 'A repeated transient failure receipt re-armed or replaced durable evidence.'); + +$abortFailure = $fixture(8, true); +CloudBackupService::finishJobLocked( + $abortFailure['job'], + 'failed', + null, + $payload($abortFailure), + false, + [ + 'status' => 'failed', + 'effect_may_have_applied' => false, + 'safe_to_resubmit' => true, + 'cause_code' => 'backup_upload_abort_failed', + ] +); +$abortSubscription = CloudBackupSubscription::findOrFail( + $abortFailure['subscription_id'] +); +$assert((string) CloudInstanceBackup::findOrFail(8)->state === 'failed' + && (string) CloudStorageArtifact::findOrFail( + $abortFailure['artifact_id'] + )->state === 'publishing' + && (string) CloudStoragePublication::where( + 'artifact_id', + $abortFailure['artifact_id'] + )->value('state') === 'failed' + && (int) $abortSubscription->last_scheduled_at + === (int) $abortFailure['scheduled_for'] + && (int) $abortSubscription->last_backup_id === 8 + && (int) $abortSubscription->next_run_at + > (int) $abortFailure['scheduled_for'] + && (int) $abortSubscription->deferred_until === 0, + 'Multipart abort failure was incorrectly treated as a safe transient retry.'); + +$automaticSuccess = $fixture(9, true); +CloudBackupService::finishJobLocked( + $automaticSuccess['job'], + 'succeeded', + $receipt($automaticSuccess), + $payload($automaticSuccess) +); +$successSubscription = CloudBackupSubscription::findOrFail( + $automaticSuccess['subscription_id'] +); +$successNextRun = (int) $successSubscription->next_run_at; +CloudBackupService::finishJobLocked( + $automaticSuccess['job'], + 'succeeded', + $receipt($automaticSuccess), + $payload($automaticSuccess) +); +$successSubscription->refresh(); +$assert((string) CloudInstanceBackup::findOrFail(9)->state === 'ready' + && (int) $successSubscription->last_scheduled_at + === (int) $automaticSuccess['scheduled_for'] + && (int) $successSubscription->last_backup_id === 9 + && (int) $successSubscription->next_run_at === $successNextRun, + 'Ordinary automatic terminal success did not advance exactly once.'); + +$unknownAutomatic = $fixture(10, true); +CloudBackupService::finishJobLocked( + $unknownAutomatic['job'], + 'uncertain', + null, + $payload($unknownAutomatic), + false, + [ + 'status' => 'failed', + 'effect_may_have_applied' => true, + 'safe_to_resubmit' => false, + 'cause_code' => 'backup_disk_transition_active', + ] +); +$unknownSubscription = CloudBackupSubscription::findOrFail( + $unknownAutomatic['subscription_id'] +); +$assert((string) CloudInstanceBackup::findOrFail(10)->state === 'uncertain' + && (string) CloudStorageArtifact::findOrFail( + $unknownAutomatic['artifact_id'] + )->state !== 'deleted' + && (string) CloudStoragePublication::where( + 'artifact_id', + $unknownAutomatic['artifact_id'] + )->value('state') !== 'retired' + && (int) $unknownSubscription->next_run_at + > (int) $unknownAutomatic['scheduled_for'] + && (int) $unknownSubscription->deferred_until === 0, + 'Unknown automatic backup effect was replayed as a safe transient failure.'); + +$manualTransient = $fixture(11); +CloudBackupService::finishJobLocked( + $manualTransient['job'], + 'failed', + null, + $payload($manualTransient), + false, + $safeTransient +); +$assert((string) CloudInstanceBackup::findOrFail(11)->state === 'failed' + && (string) CloudStorageArtifact::findOrFail( + $manualTransient['artifact_id'] + )->state === 'publishing' + && (string) CloudStoragePublication::where( + 'artifact_id', + $manualTransient['artifact_id'] + )->value('state') === 'failed', + 'A manual backup failure was incorrectly armed for automatic retry.'); + +$rollback = $fixture(12, true); +$connection = Capsule::connection('default'); +$connection->beginTransaction(); +try { + $poisonedReplicaId = CloudStorageReplica::where( + 'artifact_id', + $rollback['artifact_id'] + )->orderBy('id')->value('id'); + Capsule::table('cloud_storage_replicas')->where( + 'id', + $poisonedReplicaId + )->update([ + 'object_etag' => 'unexpected-provider-identity', + ]); + CloudBackupService::finishJobLocked( + $rollback['job'], + 'failed', + null, + $payload($rollback), + false, + $safeTransient + ); + throw new RuntimeException('Expected blank retirement conflict.'); +} catch (CloudError $error) { + $assert($error->httpStatus === 409, + 'Unexpected blank retirement conflict status.'); + $connection->rollBack(); +} +$assert((string) CloudInstanceBackup::findOrFail(12)->state === 'creating' + && (string) CloudStorageArtifact::findOrFail( + $rollback['artifact_id'] + )->state === 'publishing' + && (string) CloudStorageOperation::where( + 'operation_id', + $rollback['storage_operation_id'] + )->value('phase') === 'awaiting_receipt' + && (string) CloudStoragePublication::where( + 'artifact_id', + $rollback['artifact_id'] + )->value('state') === 'waiting_primary', + 'A failed transient finalization did not roll back the entire ledger.'); + echo "Cloud backup storage finalizer tests passed.\n"; diff --git a/web/tests/CloudFreshSchemaContractTest.php b/web/tests/CloudFreshSchemaContractTest.php index 75f59e94..6ef8d924 100644 --- a/web/tests/CloudFreshSchemaContractTest.php +++ b/web/tests/CloudFreshSchemaContractTest.php @@ -22,7 +22,7 @@ $assert(stripos($schema, 'compatibility import') === false $assert(!preg_match('/\bengine_[A-Za-z0-9_]*\b/', $schema), 'cloud.sql retained an engine-prefixed identifier.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'") + "SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'") && !str_contains($setup, "SCHEMA_VERSION = 'go-v1'"), 'Outbound Agent schema must reject the incompatible earlier Go database.'); $assert(str_contains($schema, @@ -37,6 +37,10 @@ $assert(str_contains($schema, 'KEY cloud_backup_subscription_due (enabled, next_run_at, deferred_until, id)') && str_contains($schema, 'CONSTRAINT chk_cloud_backup_subscription_deferral CHECK (') + && str_contains($schema, + 'schedule_attempt INT UNSIGNED NOT NULL DEFAULT 0') + && str_contains($schema, + 'CONSTRAINT chk_cloud_instance_backup_schedule_shape CHECK (') && str_contains($setup, "'state_revision', 'next_run_at', 'deferred_until'") && !str_contains($schema, 'America/Los_Angeles'), 'Fresh automatic-backup subscriptions must default to UTC with a state revision.'); diff --git a/web/tests/CloudMaintenanceNoticeSchemaContractTest.php b/web/tests/CloudMaintenanceNoticeSchemaContractTest.php index 669cfcc6..a0d792a8 100644 --- a/web/tests/CloudMaintenanceNoticeSchemaContractTest.php +++ b/web/tests/CloudMaintenanceNoticeSchemaContractTest.php @@ -56,7 +56,7 @@ foreach ([ } $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'"), + "SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'"), 'CloudSetup schema version was not advanced for maintenance notices.'); echo "Maintenance notice schema contracts passed.\n"; diff --git a/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php b/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php index c411afe5..0f7c4571 100644 --- a/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php +++ b/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php @@ -53,7 +53,7 @@ $assert(is_string($schema) 'Public-pool replacement lacks durable batch, child, or claim schema.'); $assert(is_string($setup) && str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'") + "SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'") && str_contains($setup, "'cloud_public_ip_pool_replacement_batches'") && str_contains($setup, diff --git a/web/tests/CloudPublicNetworkTransitionSchemaContractTest.php b/web/tests/CloudPublicNetworkTransitionSchemaContractTest.php index bf229644..2fa513ef 100644 --- a/web/tests/CloudPublicNetworkTransitionSchemaContractTest.php +++ b/web/tests/CloudPublicNetworkTransitionSchemaContractTest.php @@ -170,7 +170,7 @@ $assert(str_contains($setup, 'CloudSetup does not verify index uniqueness and ordered columns.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'") + "SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'") && str_contains($setup, "'cloud_public_network_transitions'") && str_contains($setup, "'cloud_public_network_transition_ips'") && str_contains($setup, "'reservation_ip_claim_id'") diff --git a/web/tests/CloudRescueCatalogSchemaContractTest.php b/web/tests/CloudRescueCatalogSchemaContractTest.php index 40aa5491..9c68c31a 100644 --- a/web/tests/CloudRescueCatalogSchemaContractTest.php +++ b/web/tests/CloudRescueCatalogSchemaContractTest.php @@ -57,7 +57,7 @@ $assert($replica !== '' && str_contains($replica, 'cleanup_resolved_at BIGINT UNSIGNED'), 'Rescue replica cannot prove exact identity or retain cleanup debt.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'") + "SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'") && str_contains($setup, "'cloud_rescue_artifact_delete_due'") && str_contains($setup, "'cloud_rescue_replica_cleanup'"), 'CloudSetup version or rescue catalog indexes are incomplete.'); diff --git a/web/tests/CloudRestoreMigrationLedgerContractTest.php b/web/tests/CloudRestoreMigrationLedgerContractTest.php index 3da5e205..7aec9841 100644 --- a/web/tests/CloudRestoreMigrationLedgerContractTest.php +++ b/web/tests/CloudRestoreMigrationLedgerContractTest.php @@ -218,7 +218,7 @@ foreach ([ 'CloudSetup omits restore/migration ledger contract: ' . $required); } $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'") + "SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'") && str_contains($attemptModel, "protected \$table = 'cloud_backup_restore_attempts';") && str_contains($attemptModel, "'active_attempt' => 'int'") diff --git a/web/tests/CloudServiceCompensationSchemaContractTest.php b/web/tests/CloudServiceCompensationSchemaContractTest.php index c602b760..cb7a0cbd 100644 --- a/web/tests/CloudServiceCompensationSchemaContractTest.php +++ b/web/tests/CloudServiceCompensationSchemaContractTest.php @@ -105,7 +105,7 @@ $assert(str_contains($setup, 'CloudSetup lost the open-compensation fence index contract.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'"), + "SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'"), 'CloudSetup schema version was not advanced for service compensation.'); $appliedShape = preg_match( diff --git a/web/tests/CloudStoragePublicationSchemaContractTest.php b/web/tests/CloudStoragePublicationSchemaContractTest.php index 8382d362..5a8329b6 100644 --- a/web/tests/CloudStoragePublicationSchemaContractTest.php +++ b/web/tests/CloudStoragePublicationSchemaContractTest.php @@ -33,7 +33,7 @@ $assert(str_contains($artifact, && str_contains($artifact, 'chk_cloud_storage_artifact_identity CHECK') && str_contains($artifact, - "(state = 'publishing' AND size_bytes = 0 AND sha256 = '')") + "(state IN ('publishing','deleted') AND size_bytes = 0 AND sha256 = '')") && str_contains($artifact, 'size_bytes BETWEEN 1 AND maximum_size_bytes') && str_contains($artifact, @@ -98,8 +98,9 @@ $assert(str_contains($publication, "state IN ('waiting_primary','waiting_protection')") && str_contains($publication, 'AND coverage_ready_at IS NULL') && str_contains($publication, - "state IN ('ready_to_finalize','finalizing','completed','retired')") + "state IN ('ready_to_finalize','finalizing','completed')") && str_contains($publication, 'AND coverage_ready_at IS NOT NULL') + && str_contains($publication, "OR state = 'retired'") && str_contains($publication, "OR state = 'failed'"), 'Publication finalization states lack durable required-copy evidence.'); @@ -137,7 +138,7 @@ $assert(is_string($publicationModel) && str_contains($artifactModel, "'maximum_size_bytes' => 'int'"), 'Storage publication/artifact model contracts are incomplete.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'"), + "SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'"), 'CloudSetup schema version was not advanced for durable publications.'); echo "Storage publication schema contracts passed.\n"; diff --git a/web/tests/CloudTemplateImportIntentContractTest.php b/web/tests/CloudTemplateImportIntentContractTest.php index e6043385..40ce8462 100644 --- a/web/tests/CloudTemplateImportIntentContractTest.php +++ b/web/tests/CloudTemplateImportIntentContractTest.php @@ -63,7 +63,7 @@ $assert(str_contains($schema, && str_contains($schema, 'CONSTRAINT chk_cloud_template_import_completion CHECK (') && str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'"), + "SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'"), 'Template imports are missing durable identity or terminal-state constraints.'); $assert(str_contains($routes, "'/admin/cloud-template-imports'") && str_contains($routes, "'/admin/cloud-template-imports/{id}'") diff --git a/web/tests/CloudTrafficPoolSchemaContractTest.php b/web/tests/CloudTrafficPoolSchemaContractTest.php index d35ed647..8b604245 100644 --- a/web/tests/CloudTrafficPoolSchemaContractTest.php +++ b/web/tests/CloudTrafficPoolSchemaContractTest.php @@ -71,7 +71,7 @@ $assert(str_contains($schema, 'Service-period traffic-pool identity is not frozen or indexed.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'") + "SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'") && str_contains($setup, "'chk_cloud_traffic_entitlement_balance'") && str_contains($setup, "'cloud_traffic_ledger_sequence'"), 'CloudSetup schema version or named-object validation is incomplete.'); diff --git a/web/tests/CloudVpcSchemaContractTest.php b/web/tests/CloudVpcSchemaContractTest.php index be9aa200..facf72a6 100644 --- a/web/tests/CloudVpcSchemaContractTest.php +++ b/web/tests/CloudVpcSchemaContractTest.php @@ -101,7 +101,7 @@ $assert(str_contains($jobs, 'vpc_id BIGINT UNSIGNED DEFAULT NULL') 'Cloud jobs cannot be correlated to VPC or VPC-port work.'); $assert(str_contains($setup, - "private const SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'") + "private const SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'") && str_contains($setup, "'name' => 'cloud_vpc_enabled'") && str_contains($setup, "'value' => 'false'") && str_contains($setup, diff --git a/web/tests/UiCopyCatalogSchemaContractTest.php b/web/tests/UiCopyCatalogSchemaContractTest.php index 34554401..80621827 100644 --- a/web/tests/UiCopyCatalogSchemaContractTest.php +++ b/web/tests/UiCopyCatalogSchemaContractTest.php @@ -104,7 +104,7 @@ foreach ($models as $class => $modelTable) { } $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'") + "SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'") && str_contains($setup, "self::schema('ss.sql') . \"\\0\" . self::schema('cloud.sql')") && str_contains($setup,