feat(backup): safely retry transient races

go
codex 2 weeks ago
parent dbc5566679
commit 4ca037f65e

@ -283,10 +283,16 @@ func abortRejectedBackupPlan(ctx context.Context,
abort func(context.Context, objecttransfer.MultipartPlan) error,
plan objecttransfer.MultipartPlan, rejection error) error {
if cleanupErr := abort(ctx, plan); cleanupErr != nil {
// The VM and backup object were never mutated. Retain retry-safe
// semantics and the original preflight cause while keeping the bounded
// multipart cleanup failure in the private Agent error chain.
return operation.MarkNotApplied(errors.Join(rejection, cleanupErr))
// The VM and backup object were never mutated, but PHP must not treat
// the original transient preflight cause as proof that multipart
// cleanup completed. Keep retry-safe semantics while publishing a
// distinct bounded cause that requires operator reconciliation.
return operation.MarkNotApplied(NewError(
ErrUnavailable,
"backup_upload_abort_failed",
"backup upload cleanup could not be confirmed",
errors.Join(rejection, cleanupErr),
))
}
return operation.MarkNotApplied(rejection)
}

@ -9,6 +9,7 @@ import (
"testing"
"git.luoli.live/root/virt/agent/objecttransfer"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
@ -294,3 +295,29 @@ func TestAbortRejectedBackupPlanPreservesPreflightCause(t *testing.T) {
t.Fatalf("called=%v err=%v", called, err)
}
}
func TestAbortRejectedBackupPlanSeparatesAbortFailureCause(t *testing.T) {
rejected := NewError(ErrConflict, "backup_disk_transition_active",
"VM backup is unavailable during a disk transition", nil)
abortFailure := errors.New("multipart abort failed")
err := abortRejectedBackupPlan(
context.Background(),
func(context.Context, objecttransfer.MultipartPlan) error {
return abortFailure
},
objecttransfer.MultipartPlan{},
rejected,
)
var public interface{ PublicCode() string }
var semantics interface{ FailureSemantics() (bool, bool) }
if !errors.As(err, &semantics) {
t.Fatalf("abort failure has no explicit semantics: %v", err)
}
mayHaveApplied, safeToResubmit := semantics.FailureSemantics()
if !operation.IsNotApplied(err) || !errors.As(err, &public) ||
public.PublicCode() != "backup_upload_abort_failed" ||
operation.IsUncertain(err) || mayHaveApplied || !safeToResubmit ||
!errors.Is(err, rejected) || !errors.Is(err, abortFailure) {
t.Fatalf("unexpected abort failure classification: %v", err)
}
}

@ -8,7 +8,7 @@ use App\Services\Config;
/** Initializes only the clean-break PHP business and Agent log schemas. */
final class CloudSetup
{
private const SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral';
private const SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts';
/** Tables installed by ss.sql before cloudInit and owned by this schema. */
private const PREINSTALLED_MAIN_TABLES = [
'ui_copy_locales', 'ui_copy_entries', 'ui_copy_drafts',
@ -356,7 +356,8 @@ final class CloudSetup
'create_request_fingerprint', 'storage_artifact_id',
'source_instance_id', 'source_host_id',
'user_id', 'source_uuid', 'name', 'purpose', 'source',
'subscription_id', 'scheduled_for', 'state', 'consistency',
'subscription_id', 'scheduled_for', 'schedule_attempt',
'state', 'consistency',
'object_key', 'object_size_bytes', 'virtual_size_bytes',
'sha256', 'object_etag', 'object_version_id', 'spec_json',
'source_portable_restore_contract_json',
@ -663,6 +664,7 @@ final class CloudSetup
],
'cloud_instance_backups' => [
'chk_cloud_instance_backup_create_request',
'chk_cloud_instance_backup_schedule_shape',
],
'cloud_public_ip_pool_catalog_mutex' => [
'chk_cloud_public_ip_pool_catalog_mutex',

@ -24,6 +24,7 @@ final class CloudInstanceBackup extends Model
'user_id' => 'int',
'subscription_id' => 'int',
'scheduled_for' => 'int',
'schedule_attempt' => 'int',
'object_size_bytes' => 'int',
'virtual_size_bytes' => 'int',
'storage_revision' => 'int',

@ -20,6 +20,7 @@ final class CloudAutomaticBackupService
private const RETRY_SECONDS = 300;
private const MAX_DEFER_COUNT = 4294967295;
private const MAX_SCHEDULE_ATTEMPT = 4294967295;
private const DEFER_REASONS = [
CloudBackupEligibilityDecision::MIGRATION_ACTIVE,
CloudBackupEligibilityDecision::INSTANCE_MUTATION_ACTIVE,
@ -337,7 +338,8 @@ final class CloudAutomaticBackupService
*/
private static function hasPendingCreateIntent(
int $subscriptionId,
int $now
int $now,
?int $scheduleAttempt = null
): bool {
$probe = CloudBackupSubscription::find($subscriptionId);
if (!$probe instanceof CloudBackupSubscription) {
@ -345,7 +347,8 @@ final class CloudAutomaticBackupService
}
return $probe->getConnection()->transaction(function () use (
$subscriptionId,
$now
$now,
$scheduleAttempt
): bool {
$subscription = CloudBackupSubscription::where(
'id', $subscriptionId
@ -357,11 +360,29 @@ final class CloudAutomaticBackupService
return false;
}
$scheduledFor = (int) $subscription->next_run_at;
if ($scheduleAttempt === null) {
$latest = CloudInstanceBackup::where(
'subscription_id',
$subscriptionId
)->where('scheduled_for', $scheduledFor)
->where('source', 'automatic')
->orderByDesc('schedule_attempt')
->orderByDesc('id')->first();
if (!$latest instanceof CloudInstanceBackup
|| (string) $latest->state === 'deleted') {
return false;
}
$scheduleAttempt = max(
1,
(int) $latest->schedule_attempt
);
}
$requestKey = 'automatic-backup-' . $subscriptionId
. '-' . $scheduledFor;
. '-' . $scheduledFor . '-' . $scheduleAttempt;
return CloudInstanceBackup::where(
'subscription_id', $subscriptionId
)->where('scheduled_for', $scheduledFor)
->where('schedule_attempt', $scheduleAttempt)
->where('source', 'automatic')
// `uncertain` with no Agent job is the operator-attention
// form of this same pre-dispatch intent. It must keep the
@ -376,6 +397,58 @@ final class CloudAutomaticBackupService
}, 3);
}
/**
* Return the durable attempt generation for the current logical slot.
* A crash after intent creation reuses that row; only a safely retired
* transient attempt authorizes the next generation.
*/
private static function scheduleAttempt(
int $subscriptionId,
int $scheduledFor
): int {
$latest = CloudInstanceBackup::where(
'subscription_id',
$subscriptionId
)->where('scheduled_for', $scheduledFor)
->where('source', 'automatic')
->orderByDesc('schedule_attempt')->orderByDesc('id')->first();
if (!$latest instanceof CloudInstanceBackup) {
return 1;
}
$attempt = max(1, (int) $latest->schedule_attempt);
if ((string) $latest->state !== 'deleted') {
return $attempt;
}
if ((string) $latest->error_code
!== 'automatic_transient_retried') {
throw new CloudError(
'自动备份计划存在不可重用的已删除代次',
409
);
}
if ($attempt >= self::MAX_SCHEDULE_ATTEMPT) {
throw new CloudError('自动备份计划重试代次已耗尽', 409);
}
return $attempt + 1;
}
/** A bound Agent generation owns the slot until its finalizer commits. */
private static function hasBoundCreateIntent(
int $subscriptionId,
int $scheduledFor,
int $scheduleAttempt
): bool {
return CloudInstanceBackup::where(
'subscription_id',
$subscriptionId
)->where('scheduled_for', $scheduledFor)
->where('schedule_attempt', $scheduleAttempt)
->where('source', 'automatic')
->where('state', 'creating')
->whereNotNull('active_job_id')
->exists();
}
/**
* Prune at most $limit fresh retention candidates plus a separate bounded
* cleanup-debt batch, so stuck remote objects cannot starve new rotation.
@ -552,8 +625,20 @@ final class CloudAutomaticBackupService
public static function advanceAfterScheduled(
CloudBackupSubscription $subscription,
int $scheduledFor,
int $backupId
int $backupId,
int $scheduleAttempt
): void {
$latest = CloudInstanceBackup::where(
'subscription_id',
(int) $subscription->id
)->where('scheduled_for', $scheduledFor)
->where('source', 'automatic')
->orderByDesc('schedule_attempt')->orderByDesc('id')->first();
if (!$latest instanceof CloudInstanceBackup
|| (int) $latest->id !== $backupId
|| (int) $latest->schedule_attempt !== $scheduleAttempt) {
throw new CloudError('自动备份计划终态代次不一致', 409);
}
$lastScheduled = max(
0,
(int) ($subscription->last_scheduled_at ?? 0)
@ -616,6 +701,10 @@ final class CloudAutomaticBackupService
self::pauseWithoutEntitlement($subscriptionId, $scheduledFor, $now);
return false;
}
$scheduleAttempt = self::scheduleAttempt(
(int) $subscription->id,
$scheduledFor
);
$actor = [
'user_id' => (int) $instance->user_id,
'is_admin' => false,
@ -623,9 +712,27 @@ final class CloudAutomaticBackupService
'can_manage' => true,
'session_id' => 'automatic-backup-scheduler',
'request_idempotency_key' => 'automatic-backup-'
. (int) $subscription->id . '-' . $scheduledFor,
. (int) $subscription->id . '-' . $scheduledFor
. '-' . $scheduleAttempt,
];
if (!self::hasPendingCreateIntent($subscriptionId, $now)) {
if (self::hasBoundCreateIntent(
$subscriptionId,
$scheduledFor,
$scheduleAttempt
)) {
self::deferCurrentSlot(
$subscriptionId,
$now,
CloudBackupEligibilityDecision::INSTANCE_MUTATION_ACTIVE,
$scheduledFor
);
return false;
}
if (!self::hasPendingCreateIntent(
$subscriptionId,
$now,
$scheduleAttempt
)) {
$eligibility = (new CloudBackupEligibility())->assess(
$instance,
$actor
@ -641,7 +748,11 @@ final class CloudAutomaticBackupService
}
}
$queued = CloudBackupService::createAutomatic(
$instance, $subscription, $actor, $scheduledFor
$instance,
$subscription,
$actor,
$scheduledFor,
$scheduleAttempt
);
return $queued['job'] instanceof CloudJob;
}
@ -889,6 +1000,67 @@ final class CloudAutomaticBackupService
$subscription->defer_count = 0;
}
/**
* Re-arm the same logical slot after a strictly not-applied Agent race.
* The retired backup row is the durable generation ledger; no historical
* job or object-storage operation is ever reused.
*/
public static function rearmAfterTransientFailure(
CloudInstanceBackup $backup,
string $reason,
int $now
): bool {
if (!in_array($reason, [
CloudBackupEligibilityDecision::BACKUP_RESCUE_ACTIVE,
CloudBackupEligibilityDecision::BACKUP_DISK_TRANSITION_ACTIVE,
], true) || $backup->subscription_id === null
|| $backup->scheduled_for === null
|| (int) $backup->schedule_attempt < 1) {
throw new \InvalidArgumentException(
'Invalid automatic backup transient retry'
);
}
$subscription = CloudBackupSubscription::where(
'id',
(int) $backup->subscription_id
)->lockForUpdate()->first();
if (!$subscription instanceof CloudBackupSubscription) {
throw new CloudError('自动备份订阅账本缺失', 409);
}
if (!(bool) $subscription->enabled
|| (int) $subscription->next_run_at
!== (int) $backup->scheduled_for) {
return false;
}
$latest = CloudInstanceBackup::where(
'subscription_id',
(int) $subscription->id
)->where('scheduled_for', (int) $backup->scheduled_for)
->where('source', 'automatic')
->orderByDesc('schedule_attempt')->orderByDesc('id')
->lockForUpdate()->first();
if (!$latest instanceof CloudInstanceBackup
|| (int) $latest->id !== (int) $backup->id
|| (int) $latest->schedule_attempt
!== (int) $backup->schedule_attempt
|| (string) $latest->state !== 'deleted'
|| (string) $latest->error_code
!== 'automatic_transient_retried') {
throw new CloudError('自动备份重试代次账本不一致', 409);
}
$count = max(0, (int) $subscription->defer_count);
$subscription->deferred_until = $now + self::RETRY_SECONDS;
$subscription->defer_reason = $reason;
$subscription->deferred_at = $now;
$subscription->defer_count = min(
self::MAX_DEFER_COUNT,
$count + 1
);
$subscription->updated_at = $now;
$subscription->save();
return true;
}
private static function backupsForInstance(CloudInstance $instance): array
{
$items = [];

@ -581,6 +581,8 @@ final class CloudBackupPublicationPreparation
(int) $backup->subscription_id;
$canonical['scheduled_for'] =
(int) $backup->scheduled_for;
$canonical['schedule_attempt'] =
(int) $backup->schedule_attempt;
}
$operationId = substr(hash(
'sha256',
@ -635,17 +637,10 @@ final class CloudBackupPublicationPreparation
$backup->updated_at = time();
$backup->save();
if ($subscription instanceof CloudBackupSubscription) {
if ((int) $subscription->last_scheduled_at
!== (int) $backup->scheduled_for
|| (int) $subscription->last_backup_id !== $backupId) {
CloudAutomaticBackupService::advanceAfterScheduled(
$subscription,
(int) $backup->scheduled_for,
$backupId
);
}
}
// Keep next_run_at on the original logical slot until this
// exact Agent generation reaches a durable terminal result.
// The finalizer advances ordinary outcomes or re-arms only a
// strictly proven not-applied transient race.
return $job;
},
3

@ -568,9 +568,9 @@ final class CloudBackupService
/** Queue one deterministic paid automatic backup for a scheduled slot. */
public static function createAutomatic(CloudInstance $instance,
CloudBackupSubscription $subscription, array $actor,
int $scheduledFor): array
int $scheduledFor, int $scheduleAttempt): array
{
if ($scheduledFor < 1) {
if ($scheduledFor < 1 || $scheduleAttempt < 1) {
throw new CloudError('自动备份计划时间无效', 409);
}
$zone = new \DateTimeZone(CloudAutomaticBackupService::TIMEZONE);
@ -583,6 +583,7 @@ final class CloudBackupService
], [
'subscription_id' => (int) $subscription->id,
'scheduled_for' => $scheduledFor,
'schedule_attempt' => $scheduleAttempt,
]);
}
@ -613,6 +614,8 @@ final class CloudBackupService
(int) $automatic['subscription_id'];
$canonicalRequest['scheduled_for'] =
(int) $automatic['scheduled_for'];
$canonicalRequest['schedule_attempt'] =
(int) $automatic['schedule_attempt'];
}
$fingerprint = CloudJobService::standaloneRequestFingerprint(
(int) $instance->host_id,
@ -701,6 +704,7 @@ final class CloudBackupService
'id', (int) $automatic['subscription_id']
)->lockForUpdate()->first();
$scheduledFor = (int) $automatic['scheduled_for'];
$scheduleAttempt = (int) $automatic['schedule_attempt'];
$entitled = $automaticSubscription !== null
&& CloudBackupPeriodEntitlement::where(
'subscription_id',
@ -719,10 +723,18 @@ final class CloudBackupService
|| !$entitled) {
throw new CloudError('自动备份订阅与实例账本不一致', 409);
}
self::assertAutomaticScheduleAttempt(
$automaticSubscription,
$scheduledFor,
$scheduleAttempt
);
$existing = CloudInstanceBackup::where(
'subscription_id', (int) $automaticSubscription->id
)->where('scheduled_for',
$scheduledFor)->first();
$scheduledFor)->where(
'schedule_attempt',
$scheduleAttempt
)->first();
if ($existing !== null) {
if ((int) ($existing->create_request_actor_id ?? 0)
!== $actorId
@ -834,6 +846,10 @@ final class CloudBackupService
: (int) $automaticSubscription->id;
$backup->scheduled_for = $automatic === null ? null
: (int) $automatic['scheduled_for'];
if ($automatic !== null) {
$backup->schedule_attempt =
(int) $automatic['schedule_attempt'];
}
$backup->state = 'creating';
$backup->consistency = $mode;
$backup->object_key = $objectKey;
@ -1081,10 +1097,18 @@ final class CloudBackupService
/** Apply a terminal/running Agent result inside CloudWorkerService's lock. */
public static function finishJobLocked(CloudJob $job, string $status,
$result, array $payload, bool $deferPublish = false): void
$result, array $payload, bool $deferPublish = false,
array $operation = []): void
{
if ($job->kind === 'backup_create') {
self::finishCreateLocked($job, $status, $result, $payload);
self::finishCreateLocked(
$job,
$status,
$result,
$payload,
$deferPublish,
$operation
);
} elseif ($job->kind === 'backup_restore') {
CloudPortableRestoreFinalizer::finishJobLocked(
$job, $status, $result, $payload, $deferPublish
@ -1252,7 +1276,8 @@ final class CloudBackupService
}
private static function finishCreateLocked(CloudJob $job, string $status,
$result, array $payload): void
$result, array $payload, bool $deferPublish,
array $operation): void
{
$backup = $job->backup_id === null ? null
: CloudInstanceBackup::where('id', (int) $job->backup_id)
@ -1295,6 +1320,32 @@ final class CloudBackupService
);
}
$now = time();
$transientRetryReason = self::automaticTransientRetryReason(
$backup,
$job,
$status,
$deferPublish,
$operation
);
if ((string) $backup->state === 'deleted'
&& (string) $backup->error_code
=== 'automatic_transient_retried') {
if ($transientRetryReason === null
|| $backup->sealed_credential !== null
|| $backup->active_job_id !== null) {
throw new CloudRecoveryInstanceStateConflict(
'自动备份安全重试终态与重复回执不一致',
409
);
}
CloudStorageArtifactService::retireBlankNotAppliedPublication(
$storageOperationId,
(int) $artifactId,
(int) $backup->id,
$now
);
return;
}
if ($status === 'uncertain') {
CloudStorageArtifactService::failPrimary(
$storageOperationId,
@ -1311,6 +1362,27 @@ final class CloudBackupService
'not_applied',
'backup_agent_failed'
);
if ($transientRetryReason !== null) {
CloudStorageArtifactService::
retireBlankNotAppliedPublication(
$storageOperationId,
(int) $artifactId,
(int) $backup->id,
$now
);
$backup->state = 'deleted';
$backup->sealed_credential = null;
$backup->active_job_id = null;
$backup->error_code = 'automatic_transient_retried';
$backup->updated_at = $now;
$backup->save();
CloudAutomaticBackupService::rearmAfterTransientFailure(
$backup,
$transientRetryReason,
$now
);
return;
}
$backup->state = 'failed';
$backup->error_code = 'backup_failed';
$backup->active_job_id = null;
@ -1414,6 +1486,50 @@ final class CloudBackupService
);
}
}
if ((string) $backup->source === 'automatic') {
$subscription = CloudBackupSubscription::where(
'id',
(int) $backup->subscription_id
)->lockForUpdate()->first();
if (!$subscription instanceof CloudBackupSubscription
|| (int) $subscription->instance_id
!== (int) $backup->source_instance_id
|| (int) $subscription->user_id !== (int) $backup->user_id) {
throw new CloudRecoveryInstanceStateConflict(
'自动备份终态与订阅账本不一致',
409
);
}
CloudAutomaticBackupService::advanceAfterScheduled(
$subscription,
(int) $backup->scheduled_for,
(int) $backup->id,
(int) $backup->schedule_attempt
);
}
}
private static function automaticTransientRetryReason(
CloudInstanceBackup $backup,
CloudJob $job,
string $status,
bool $deferPublish,
array $operation
): ?string {
if ((string) $backup->source !== 'automatic'
|| $status !== 'failed'
|| $deferPublish
|| (bool) ($job->finalization_incident ?? false)
|| ($operation['status'] ?? null) !== 'failed'
|| ($operation['effect_may_have_applied'] ?? null) !== false
|| ($operation['safe_to_resubmit'] ?? null) !== true) {
return null;
}
$reason = $operation['cause_code'] ?? null;
return in_array($reason, [
CloudBackupEligibilityDecision::BACKUP_RESCUE_ACTIVE,
CloudBackupEligibilityDecision::BACKUP_DISK_TRANSITION_ACTIVE,
], true) ? $reason : null;
}
private static function finishRecoveryLocked(CloudJob $job,
@ -1815,6 +1931,45 @@ final class CloudBackupService
return $key;
}
/**
* Reject a stale scheduler generation while the subscription lock is
* held. A safely retired row is the sole authority for incrementing the
* generation; every other state must be resumed idempotently.
*/
private static function assertAutomaticScheduleAttempt(
CloudBackupSubscription $subscription,
int $scheduledFor,
int $scheduleAttempt
): void {
if ($scheduleAttempt < 1) {
throw new CloudError('自动备份计划重试代次无效', 409);
}
$latest = CloudInstanceBackup::where(
'subscription_id',
(int) $subscription->id
)->where('scheduled_for', $scheduledFor)
->where('source', 'automatic')
->orderByDesc('schedule_attempt')->orderByDesc('id')->first();
$expected = 1;
if ($latest instanceof CloudInstanceBackup) {
$expected = max(1, (int) $latest->schedule_attempt);
if ((string) $latest->state === 'deleted') {
if ((string) $latest->error_code
!== 'automatic_transient_retried'
|| $expected >= 4294967295) {
throw new CloudError(
'自动备份计划存在不可重用的已删除代次',
409
);
}
$expected++;
}
}
if ($scheduleAttempt !== $expected) {
throw new CloudError('自动备份计划重试代次已变化', 409);
}
}
private static function findCreateIntent(
int $actorId,
string $requestKey,

@ -543,6 +543,127 @@ final class CloudStorageArtifactService
return self::publicationView($operationId);
}
/**
* Retire a publication whose Agent proved that neither VM backup data nor
* its multipart upload survived a rejected preflight. Operation rows are
* retained as immutable audit evidence; no provider request is replayed.
*/
public static function retireBlankNotAppliedPublication(
string $operationId,
int $expectedArtifactId,
int $expectedBackupId,
int $now
): void {
self::assertOperationId($operationId);
if ($expectedArtifactId < 1 || $expectedBackupId < 1 || $now < 1) {
throw new CloudError('空白对象存储制品退休参数无效', 422);
}
$connection = (new CloudStorageOperation())->getConnection();
$connection->transaction(static function () use (
$operationId,
$expectedArtifactId,
$expectedBackupId,
$now
): void {
$operation = self::primaryOperationLocked($operationId);
if ((int) $operation->artifact_id !== $expectedArtifactId) {
throw new CloudError('空白发布操作与制品身份不一致', 409);
}
$copies = self::copyOperationsLocked($expectedArtifactId);
[$artifact, $replicas] = self::artifactContextLocked($operation);
$publication = self::publicationLocked($expectedArtifactId);
if ((string) $artifact->state === 'deleted'
&& (string) $publication->state === 'retired') {
if ((int) ($publication->backup_id ?? 0)
!== $expectedBackupId
|| (int) $artifact->size_bytes !== 0
|| (string) $artifact->sha256 !== ''
|| $publication->finalized_at === null) {
throw new CloudError('空白发布退休终态与归属不一致', 409);
}
foreach ($replicas as $replica) {
if ((string) $replica->state !== 'deleted') {
throw new CloudError('空白发布副本退休终态不完整', 409);
}
}
return;
}
self::assertTerminalFailure(
$operation,
false,
true,
'not_applied',
'backup_agent_failed',
'主副本发布尚未证明为未应用'
);
if ((string) $artifact->state !== 'publishing'
|| (int) $artifact->size_bytes !== 0
|| (string) $artifact->sha256 !== ''
|| (string) $publication->state !== 'failed'
|| (int) ($publication->backup_id ?? 0)
!== $expectedBackupId
|| $publication->coverage_ready_at !== null
|| $publication->finalized_at !== null) {
throw new CloudError('对象发布不属于可安全退休的空白终态', 409);
}
$operationsByReplica = [
(int) $operation->replica_id => $operation,
];
foreach ($copies as $copy) {
self::assertTerminalFailure(
$copy,
false,
false,
'not_applied',
'primary_publish_not_applied',
'副本复制尚未证明为未应用'
);
$replicaId = (int) $copy->replica_id;
if ($replicaId < 1 || isset($operationsByReplica[$replicaId])) {
throw new CloudError('空白发布副本操作身份冲突', 409);
}
$operationsByReplica[$replicaId] = $copy;
}
if (count($operationsByReplica) !== count($replicas)) {
throw new CloudError('空白发布副本操作账本不完整', 409);
}
foreach ($replicas as $replica) {
$bound = $operationsByReplica[(int) $replica->id] ?? null;
if (!$bound instanceof CloudStorageOperation
|| !is_string($replica->publish_operation_id)
|| !hash_equals(
(string) $bound->operation_id,
(string) $replica->publish_operation_id
)
|| (int) $replica->size_bytes !== 0
|| (string) $replica->sha256 !== ''
|| (string) $replica->object_etag !== ''
|| (string) $replica->object_version_id !== ''
|| (int) $replica->verified_at !== 0
|| $replica->delete_operation_id !== null) {
throw new CloudError('空白发布副本已有对象或删除身份', 409);
}
$replica->state = 'deleted';
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = 'blank_publication_retired';
$replica->updated_at = $now;
$replica->save();
}
$artifact->state = 'deleted';
$artifact->updated_at = $now;
$artifact->save();
$publication->state = 'retired';
$publication->available_at = 0;
$publication->lease_token = null;
$publication->lease_until = 0;
$publication->last_error_code = 'blank_publication_retired';
$publication->finalized_at = $now;
$publication->updated_at = $now;
$publication->save();
});
}
/**
* Mint a read-only URL for one verified replica selected by health.
*

@ -2620,7 +2620,8 @@ final class CloudWorkerService
$catalogStatus,
$result,
$payload,
$preparedRecoveryMarker !== null
$preparedRecoveryMarker !== null,
$operation
);
CloudSnapshotService::finishInstanceDeleteLocked(
$job, $catalogStatus

@ -593,7 +593,7 @@ CREATE TABLE cloud_storage_artifacts (
CONSTRAINT chk_cloud_storage_artifact_identity CHECK (
maximum_size_bytes > 0
AND (
(state = 'publishing' AND size_bytes = 0 AND sha256 = '')
(state IN ('publishing','deleted') AND size_bytes = 0 AND sha256 = '')
OR (
size_bytes BETWEEN 1 AND maximum_size_bytes
AND sha256 REGEXP '^[a-f0-9]{64}$'
@ -1415,6 +1415,7 @@ CREATE TABLE cloud_instance_backups (
source VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'manual',
subscription_id BIGINT UNSIGNED DEFAULT NULL,
scheduled_for BIGINT UNSIGNED DEFAULT NULL,
schedule_attempt INT UNSIGNED NOT NULL DEFAULT 0,
state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'creating',
consistency VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
object_key VARCHAR(512) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
@ -1446,7 +1447,9 @@ CREATE TABLE cloud_instance_backups (
UNIQUE KEY cloud_instance_backup_create_request (create_request_actor_id, create_request_idempotency_key),
UNIQUE KEY cloud_instance_backup_storage (storage_artifact_id),
UNIQUE KEY cloud_instance_backup_object (object_key),
UNIQUE KEY cloud_instance_backup_schedule (subscription_id, scheduled_for),
UNIQUE KEY cloud_instance_backup_schedule (
subscription_id, scheduled_for, schedule_attempt
),
KEY cloud_instance_backup_source (source_instance_id, state, id),
KEY cloud_instance_backup_owner (user_id, id),
KEY cloud_instance_backup_operation (last_operation_id),
@ -1460,6 +1463,12 @@ CREATE TABLE cloud_instance_backups (
AND CHAR_LENGTH(create_request_idempotency_key) BETWEEN 16 AND 128
AND create_request_fingerprint REGEXP '^[a-f0-9]{64}$')
),
CONSTRAINT chk_cloud_instance_backup_schedule_shape CHECK (
(source = 'manual' AND subscription_id IS NULL
AND scheduled_for IS NULL AND schedule_attempt = 0)
OR (source = 'automatic' AND subscription_id IS NOT NULL
AND scheduled_for IS NOT NULL AND schedule_attempt >= 1)
),
CONSTRAINT fk_cloud_instance_backup_storage FOREIGN KEY (storage_artifact_id)
REFERENCES cloud_storage_artifacts (id)
ON DELETE RESTRICT ON UPDATE RESTRICT
@ -1512,8 +1521,9 @@ CREATE TABLE cloud_storage_publications (
CONSTRAINT chk_cloud_storage_publication_coverage CHECK (
(state IN ('waiting_primary','waiting_protection')
AND coverage_ready_at IS NULL)
OR (state IN ('ready_to_finalize','finalizing','completed','retired')
OR (state IN ('ready_to_finalize','finalizing','completed')
AND coverage_ready_at IS NOT NULL)
OR state = 'retired'
OR state = 'failed'
),
CONSTRAINT fk_cloud_storage_publication_artifact FOREIGN KEY (artifact_id)

@ -103,6 +103,7 @@ $schema->create('cloud_instance_backups',
$t->string('source');
$t->unsignedBigInteger('subscription_id')->nullable();
$t->unsignedBigInteger('scheduled_for')->nullable();
$t->unsignedInteger('schedule_attempt')->default(0);
$t->string('state');
$t->string('consistency');
$t->unsignedBigInteger('active_job_id')->nullable();
@ -165,6 +166,7 @@ $canonical = [
'source' => 'automatic',
'subscription_id' => (int) $subscription->id,
'scheduled_for' => $scheduledFor,
'schedule_attempt' => 1,
];
$fingerprint = CloudJobService::standaloneRequestFingerprint(
(int) $instance->host_id,
@ -174,7 +176,7 @@ $fingerprint = CloudJobService::standaloneRequestFingerprint(
$canonical
);
$requestKey = 'automatic-backup-' . (int) $subscription->id
. '-' . $scheduledFor;
. '-' . $scheduledFor . '-1';
$intent = new CloudInstanceBackup();
foreach ([
'backup_key' => str_repeat('a', 32),
@ -191,6 +193,7 @@ foreach ([
'source' => 'automatic',
'subscription_id' => (int) $subscription->id,
'scheduled_for' => $scheduledFor,
'schedule_attempt' => 1,
'state' => 'creating',
'consistency' => 'filesystem',
'active_job_id' => null,
@ -213,6 +216,9 @@ $assert($queued === 0
&& (int) $subscription->next_run_at === $scheduledFor
&& $subscription->last_scheduled_at === null
&& $subscription->last_backup_id === null
&& (int) $subscription->deferred_until === 0
&& (string) $subscription->defer_reason === ''
&& (int) $subscription->defer_count === 0
&& CloudInstanceBackup::where(
'subscription_id', (int) $subscription->id
)->count() === 1,

@ -110,6 +110,7 @@ $schema->create('cloud_instance_backups',
$table->increments('id');
$table->integer('subscription_id')->nullable();
$table->integer('scheduled_for')->nullable();
$table->integer('schedule_attempt')->default(0);
$table->string('source');
$table->string('state');
$table->integer('storage_artifact_id')->nullable();

@ -27,6 +27,7 @@ foreach ([
'create_request_actor_id BIGINT UNSIGNED DEFAULT NULL',
'create_request_idempotency_key VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL',
'create_request_fingerprint CHAR(64) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL',
'schedule_attempt INT UNSIGNED NOT NULL DEFAULT 0',
] as $column) {
$assert(str_contains($backup, $column),
'Missing durable backup request field: ' . $column);
@ -40,6 +41,15 @@ $assert(str_contains($backup,
&& str_contains($backup,
"create_request_fingerprint REGEXP '^[a-f0-9]{64}$'"),
'Backup request identity is not atomically shaped or uniquely fenced.');
$assert(str_contains($backup,
'UNIQUE KEY cloud_instance_backup_schedule (')
&& str_contains($backup,
'subscription_id, scheduled_for, schedule_attempt')
&& str_contains($backup,
'CONSTRAINT chk_cloud_instance_backup_schedule_shape CHECK (')
&& str_contains($backup, 'schedule_attempt = 0')
&& str_contains($backup, 'schedule_attempt >= 1'),
'Automatic backup schedule attempts are not durably generation-fenced.');
foreach ([
'create_request_actor_id',
@ -47,18 +57,21 @@ foreach ([
'create_request_fingerprint',
'cloud_instance_backup_create_request',
'chk_cloud_instance_backup_create_request',
'schedule_attempt',
'chk_cloud_instance_backup_schedule_shape',
] as $required) {
$assert(str_contains($setup, "'" . $required . "'"),
'CloudSetup omits backup preparation contract: ' . $required);
}
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'")
"SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'")
&& str_contains($setup,
"'columns' => [\n 'create_request_actor_id',\n 'create_request_idempotency_key',")
&& str_contains($setup, "'non_unique' => 0"),
'CloudSetup does not verify the exact unique request fence.');
$assert(str_contains($model, "'create_request_actor_id' => 'int'")
&& str_contains($model, "'storage_artifact_id' => 'int'")
&& str_contains($model, "'schedule_attempt' => 'int'")
&& str_contains($model, "'create_request_idempotency_key'")
&& str_contains($model, "'create_request_fingerprint'"),
'Backup model exposes or miscasts durable preparation identity.');

@ -168,6 +168,7 @@ namespace {
$t->string('source_uuid'); $t->string('name'); $t->string('purpose');
$t->string('source'); $t->unsignedBigInteger('subscription_id')->nullable();
$t->unsignedBigInteger('scheduled_for')->nullable();
$t->unsignedInteger('schedule_attempt')->default(0);
$t->string('state'); $t->string('consistency'); $t->string('object_key');
$t->text('source_portable_restore_contract_json');
$t->string('source_portable_restore_contract_sha256');
@ -302,6 +303,7 @@ namespace {
$canonical['source'] = 'automatic';
$canonical['subscription_id'] = $subscriptionId;
$canonical['scheduled_for'] = $scheduledFor;
$canonical['schedule_attempt'] = 1;
}
$target = '/v1/vms/' . rawurlencode($uuid) . '/backups';
$fingerprint = CloudJobService::standaloneRequestFingerprint(
@ -337,6 +339,7 @@ namespace {
'user_id' => 33, 'source_uuid' => $uuid, 'name' => 'backup-' . $id,
'purpose' => 'portable', 'source' => $source,
'subscription_id' => $subscriptionId, 'scheduled_for' => $scheduledFor,
'schedule_attempt' => $source === 'automatic' ? 1 : 0,
'state' => 'creating', 'consistency' => 'filesystem',
'object_key' => 'pending-' . $id,
'source_portable_restore_contract_json' => '{}',
@ -476,7 +479,7 @@ namespace {
'A second worker/call created a duplicate recovered job.');
// Automatic intent without a pre-existing job uses the same preparation
// path and advances its subscription exactly when the job binding commits.
// path while preserving the logical slot until terminal finalization.
$scheduledFor = $now - 60;
$subscription = new CloudBackupSubscription();
foreach ([
@ -514,14 +517,13 @@ namespace {
$automaticJob = $automaticService->ensureAgentJob((int) $automatic->id);
$subscription->refresh();
$assert($automaticJob instanceof CloudJob
&& (int) $subscription->last_scheduled_at === $scheduledFor
&& (int) $subscription->last_backup_id === (int) $automatic->id
&& (int) $subscription->next_run_at > $scheduledFor,
'Automatic backup intent did not atomically bind its job and advance schedule.');
$advancedNextRun = (int) $subscription->next_run_at;
&& $subscription->last_scheduled_at === null
&& $subscription->last_backup_id === null
&& (int) $subscription->next_run_at === $scheduledFor,
'Automatic backup job binding advanced the logical slot before terminal finalization.');
$automaticService->ensureAgentJob((int) $automatic->id);
$subscription->refresh();
$assert((int) $subscription->next_run_at === $advancedNextRun
$assert((int) $subscription->next_run_at === $scheduledFor
&& CloudJob::where('backup_id', (int) $automatic->id)->count() === 1,
'Repeated automatic preparation advanced the schedule or duplicated the job.');
@ -654,7 +656,8 @@ namespace {
// The user changes the schedule while CreateMultipartUpload is in
// progress. The old slot may still bind, but it must not overwrite the
// newly selected frequency or next execution time.
// newly selected frequency or next execution time, nor mark the slot
// complete before terminal finalization.
$staleSlot = $clock - 60;
$newNextRun = $clock + 7 * 86400;
$changedSubscription = new CloudBackupSubscription();
@ -704,9 +707,8 @@ namespace {
&& (string) $changedSubscription->frequency === 'weekly'
&& (int) $changedSubscription->weekday === 3
&& (int) $changedSubscription->next_run_at === $newNextRun
&& (int) $changedSubscription->last_scheduled_at === $staleSlot
&& (int) $changedSubscription->last_backup_id
=== (int) $changedBackup->id,
&& $changedSubscription->last_scheduled_at === null
&& $changedSubscription->last_backup_id === null,
'An old backup intent overwrote the user\'s newer schedule.');
// Disabling the subscription during provider preparation is also a real

@ -3,6 +3,7 @@
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstanceBackup;
use App\Models\CloudBackupSubscription;
use App\Models\CloudJob;
use App\Models\CloudStorageArtifact;
use App\Models\CloudStorageOperation;
@ -128,6 +129,7 @@ $schema->create('cloud_instance_backups', static function (Blueprint $t): void {
$t->string('purpose')->default('portable'); $t->string('source')->default('manual');
$t->unsignedBigInteger('subscription_id')->nullable();
$t->unsignedBigInteger('scheduled_for')->nullable();
$t->unsignedInteger('schedule_attempt')->default(0);
$t->string('state'); $t->string('consistency'); $t->string('object_key');
$t->unsignedBigInteger('object_size_bytes')->default(0);
$t->unsignedBigInteger('virtual_size_bytes')->default(0);
@ -150,6 +152,33 @@ $schema->create('cloud_jobs', static function (Blueprint $t): void {
$t->unsignedBigInteger('user_id')->nullable();
$t->string('operation_id')->unique();
$t->string('effect_resolution')->default('not_applicable');
$t->boolean('finalization_incident')->default(false);
});
$schema->create('cloud_backup_subscriptions',
static function (Blueprint $t): void {
$t->increments('id');
$t->unsignedBigInteger('instance_id')->unique();
$t->unsignedBigInteger('user_id');
$t->unsignedBigInteger('offer_id');
$t->boolean('enabled');
$t->boolean('renew_next_period');
$t->string('frequency');
$t->unsignedTinyInteger('weekday');
$t->unsignedTinyInteger('retention_count');
$t->string('timezone');
$t->string('price');
$t->string('currency');
$t->unsignedInteger('price_version');
$t->unsignedInteger('state_revision')->default(1);
$t->unsignedBigInteger('next_run_at');
$t->unsignedBigInteger('deferred_until')->default(0);
$t->string('defer_reason')->default('');
$t->unsignedBigInteger('deferred_at')->nullable();
$t->unsignedInteger('defer_count')->default(0);
$t->unsignedBigInteger('last_scheduled_at')->nullable();
$t->unsignedBigInteger('last_backup_id')->nullable();
$t->unsignedBigInteger('created_at');
$t->unsignedBigInteger('updated_at');
});
$now = time();
@ -186,13 +215,42 @@ $sourceSpec = json_encode([
'disk_gib' => 20, 'bandwidth_mbps' => 1000,
], JSON_THROW_ON_ERROR);
$fixture = static function (int $id) use (
$fixture = static function (int $id, bool $automatic = false) use (
$now,
$sourceSpec,
$compatibility
): array {
$backupKey = substr(hash('sha256', 'storage-finalizer-' . $id), 0, 32);
$agentOperationId = substr(hash('sha256', 'agent-' . $id), 0, 32);
$scheduledFor = $automatic ? $now - 60 : null;
$subscriptionId = $automatic ? 100 + $id : null;
if ($automatic) {
Capsule::table('cloud_backup_subscriptions')->insert([
'id' => $subscriptionId,
'instance_id' => 100 + $id,
'user_id' => 9,
'offer_id' => 1,
'enabled' => 1,
'renew_next_period' => 1,
'frequency' => 'daily',
'weekday' => 0,
'retention_count' => 2,
'timezone' => 'UTC',
'price' => '2.00',
'currency' => 'USD',
'price_version' => 1,
'state_revision' => 1,
'next_run_at' => $scheduledFor,
'deferred_until' => 0,
'defer_reason' => '',
'deferred_at' => null,
'defer_count' => 0,
'last_scheduled_at' => null,
'last_backup_id' => null,
'created_at' => $now,
'updated_at' => $now,
]);
}
Capsule::table('cloud_instance_backups')->insert([
'id' => $id, 'backup_key' => $backupKey,
'storage_artifact_id' => null,
@ -200,6 +258,10 @@ $fixture = static function (int $id) use (
'user_id' => 9,
'source_uuid' => sprintf('00000000-0000-4000-8000-%012d', $id),
'name' => 'storage-finalizer-' . $id, 'state' => 'creating',
'source' => $automatic ? 'automatic' : 'manual',
'subscription_id' => $subscriptionId,
'scheduled_for' => $scheduledFor,
'schedule_attempt' => $automatic ? 1 : 0,
'consistency' => 'filesystem',
'object_key' => 'vm-backups/' . $backupKey . '/system.qcow2',
'source_portable_restore_contract_json' => $compatibility->json(),
@ -244,6 +306,8 @@ $fixture = static function (int $id) use (
'job' => CloudJob::findOrFail($id),
'storage_operation_id' => (string) $publication['operation_id'],
'artifact_id' => (int) $publication['artifact_id'],
'subscription_id' => $subscriptionId,
'scheduled_for' => $scheduledFor,
];
};
$payload = static fn (array $item): array => [
@ -382,4 +446,232 @@ $assert((string) CloudInstanceBackup::findOrFail(5)->state === 'creating'
)->value('phase') === 'awaiting_receipt',
'A foreign storage operation mutated either artifact ledger.');
$safeTransient = [
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
'cause_code' => 'backup_disk_transition_active',
];
$automaticRetry = $fixture(7, true);
$retryStartedAt = time();
CloudBackupService::finishJobLocked(
$automaticRetry['job'],
'failed',
null,
$payload($automaticRetry),
false,
$safeTransient
);
$retiredBackup = CloudInstanceBackup::findOrFail(7);
$retiredArtifact = CloudStorageArtifact::findOrFail(
$automaticRetry['artifact_id']
);
$retiredPublication = CloudStoragePublication::where(
'artifact_id',
$automaticRetry['artifact_id']
)->firstOrFail();
$retiredSubscription = CloudBackupSubscription::findOrFail(
$automaticRetry['subscription_id']
);
$retiredOperations = CloudStorageOperation::where(
'artifact_id',
$automaticRetry['artifact_id']
)->get();
$assert((string) $retiredBackup->state === 'deleted'
&& (string) $retiredBackup->error_code
=== 'automatic_transient_retried'
&& $retiredBackup->sealed_credential === null
&& $retiredBackup->active_job_id === null
&& (string) $retiredArtifact->state === 'deleted'
&& (string) $retiredPublication->state === 'retired'
&& $retiredPublication->coverage_ready_at === null
&& (int) $retiredPublication->finalized_at >= $retryStartedAt
&& CloudStorageReplica::where(
'artifact_id',
$automaticRetry['artifact_id']
)->where('state', '<>', 'deleted')->count() === 0
&& $retiredOperations->count() === 2
&& $retiredOperations->every(
static fn (CloudStorageOperation $operation): bool =>
(string) $operation->phase === 'terminal'
&& (string) $operation->outcome === 'failed'
&& (string) $operation->effect_resolution === 'not_applied'
)
&& (int) $retiredSubscription->next_run_at
=== (int) $automaticRetry['scheduled_for']
&& (string) $retiredSubscription->defer_reason
=== 'backup_disk_transition_active'
&& (int) $retiredSubscription->defer_count === 1
&& (int) $retiredSubscription->deferred_until
>= $retryStartedAt + 300
&& $retiredSubscription->last_scheduled_at === null
&& $retiredSubscription->last_backup_id === null,
'A proven not-applied automatic race was not retired and re-armed on the same slot.');
$retryDeferredUntil = (int) $retiredSubscription->deferred_until;
CloudBackupService::finishJobLocked(
$automaticRetry['job'],
'failed',
null,
$payload($automaticRetry),
false,
$safeTransient
);
$retiredSubscription->refresh();
$assert((int) $retiredSubscription->defer_count === 1
&& (int) $retiredSubscription->deferred_until === $retryDeferredUntil
&& CloudStorageOperation::where(
'artifact_id',
$automaticRetry['artifact_id']
)->count() === 2,
'A repeated transient failure receipt re-armed or replaced durable evidence.');
$abortFailure = $fixture(8, true);
CloudBackupService::finishJobLocked(
$abortFailure['job'],
'failed',
null,
$payload($abortFailure),
false,
[
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
'cause_code' => 'backup_upload_abort_failed',
]
);
$abortSubscription = CloudBackupSubscription::findOrFail(
$abortFailure['subscription_id']
);
$assert((string) CloudInstanceBackup::findOrFail(8)->state === 'failed'
&& (string) CloudStorageArtifact::findOrFail(
$abortFailure['artifact_id']
)->state === 'publishing'
&& (string) CloudStoragePublication::where(
'artifact_id',
$abortFailure['artifact_id']
)->value('state') === 'failed'
&& (int) $abortSubscription->last_scheduled_at
=== (int) $abortFailure['scheduled_for']
&& (int) $abortSubscription->last_backup_id === 8
&& (int) $abortSubscription->next_run_at
> (int) $abortFailure['scheduled_for']
&& (int) $abortSubscription->deferred_until === 0,
'Multipart abort failure was incorrectly treated as a safe transient retry.');
$automaticSuccess = $fixture(9, true);
CloudBackupService::finishJobLocked(
$automaticSuccess['job'],
'succeeded',
$receipt($automaticSuccess),
$payload($automaticSuccess)
);
$successSubscription = CloudBackupSubscription::findOrFail(
$automaticSuccess['subscription_id']
);
$successNextRun = (int) $successSubscription->next_run_at;
CloudBackupService::finishJobLocked(
$automaticSuccess['job'],
'succeeded',
$receipt($automaticSuccess),
$payload($automaticSuccess)
);
$successSubscription->refresh();
$assert((string) CloudInstanceBackup::findOrFail(9)->state === 'ready'
&& (int) $successSubscription->last_scheduled_at
=== (int) $automaticSuccess['scheduled_for']
&& (int) $successSubscription->last_backup_id === 9
&& (int) $successSubscription->next_run_at === $successNextRun,
'Ordinary automatic terminal success did not advance exactly once.');
$unknownAutomatic = $fixture(10, true);
CloudBackupService::finishJobLocked(
$unknownAutomatic['job'],
'uncertain',
null,
$payload($unknownAutomatic),
false,
[
'status' => 'failed',
'effect_may_have_applied' => true,
'safe_to_resubmit' => false,
'cause_code' => 'backup_disk_transition_active',
]
);
$unknownSubscription = CloudBackupSubscription::findOrFail(
$unknownAutomatic['subscription_id']
);
$assert((string) CloudInstanceBackup::findOrFail(10)->state === 'uncertain'
&& (string) CloudStorageArtifact::findOrFail(
$unknownAutomatic['artifact_id']
)->state !== 'deleted'
&& (string) CloudStoragePublication::where(
'artifact_id',
$unknownAutomatic['artifact_id']
)->value('state') !== 'retired'
&& (int) $unknownSubscription->next_run_at
> (int) $unknownAutomatic['scheduled_for']
&& (int) $unknownSubscription->deferred_until === 0,
'Unknown automatic backup effect was replayed as a safe transient failure.');
$manualTransient = $fixture(11);
CloudBackupService::finishJobLocked(
$manualTransient['job'],
'failed',
null,
$payload($manualTransient),
false,
$safeTransient
);
$assert((string) CloudInstanceBackup::findOrFail(11)->state === 'failed'
&& (string) CloudStorageArtifact::findOrFail(
$manualTransient['artifact_id']
)->state === 'publishing'
&& (string) CloudStoragePublication::where(
'artifact_id',
$manualTransient['artifact_id']
)->value('state') === 'failed',
'A manual backup failure was incorrectly armed for automatic retry.');
$rollback = $fixture(12, true);
$connection = Capsule::connection('default');
$connection->beginTransaction();
try {
$poisonedReplicaId = CloudStorageReplica::where(
'artifact_id',
$rollback['artifact_id']
)->orderBy('id')->value('id');
Capsule::table('cloud_storage_replicas')->where(
'id',
$poisonedReplicaId
)->update([
'object_etag' => 'unexpected-provider-identity',
]);
CloudBackupService::finishJobLocked(
$rollback['job'],
'failed',
null,
$payload($rollback),
false,
$safeTransient
);
throw new RuntimeException('Expected blank retirement conflict.');
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'Unexpected blank retirement conflict status.');
$connection->rollBack();
}
$assert((string) CloudInstanceBackup::findOrFail(12)->state === 'creating'
&& (string) CloudStorageArtifact::findOrFail(
$rollback['artifact_id']
)->state === 'publishing'
&& (string) CloudStorageOperation::where(
'operation_id',
$rollback['storage_operation_id']
)->value('phase') === 'awaiting_receipt'
&& (string) CloudStoragePublication::where(
'artifact_id',
$rollback['artifact_id']
)->value('state') === 'waiting_primary',
'A failed transient finalization did not roll back the entire ledger.');
echo "Cloud backup storage finalizer tests passed.\n";

@ -22,7 +22,7 @@ $assert(stripos($schema, 'compatibility import') === false
$assert(!preg_match('/\bengine_[A-Za-z0-9_]*\b/', $schema),
'cloud.sql retained an engine-prefixed identifier.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'")
"SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'")
&& !str_contains($setup, "SCHEMA_VERSION = 'go-v1'"),
'Outbound Agent schema must reject the incompatible earlier Go database.');
$assert(str_contains($schema,
@ -37,6 +37,10 @@ $assert(str_contains($schema,
'KEY cloud_backup_subscription_due (enabled, next_run_at, deferred_until, id)')
&& str_contains($schema,
'CONSTRAINT chk_cloud_backup_subscription_deferral CHECK (')
&& str_contains($schema,
'schedule_attempt INT UNSIGNED NOT NULL DEFAULT 0')
&& str_contains($schema,
'CONSTRAINT chk_cloud_instance_backup_schedule_shape CHECK (')
&& str_contains($setup, "'state_revision', 'next_run_at', 'deferred_until'")
&& !str_contains($schema, 'America/Los_Angeles'),
'Fresh automatic-backup subscriptions must default to UTC with a state revision.');

@ -56,7 +56,7 @@ foreach ([
}
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'"),
"SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'"),
'CloudSetup schema version was not advanced for maintenance notices.');
echo "Maintenance notice schema contracts passed.\n";

@ -53,7 +53,7 @@ $assert(is_string($schema)
'Public-pool replacement lacks durable batch, child, or claim schema.');
$assert(is_string($setup)
&& str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'")
"SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'")
&& str_contains($setup,
"'cloud_public_ip_pool_replacement_batches'")
&& str_contains($setup,

@ -170,7 +170,7 @@ $assert(str_contains($setup,
'CloudSetup does not verify index uniqueness and ordered columns.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'")
"SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'")
&& str_contains($setup, "'cloud_public_network_transitions'")
&& str_contains($setup, "'cloud_public_network_transition_ips'")
&& str_contains($setup, "'reservation_ip_claim_id'")

@ -57,7 +57,7 @@ $assert($replica !== ''
&& str_contains($replica, 'cleanup_resolved_at BIGINT UNSIGNED'),
'Rescue replica cannot prove exact identity or retain cleanup debt.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'")
"SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'")
&& str_contains($setup, "'cloud_rescue_artifact_delete_due'")
&& str_contains($setup, "'cloud_rescue_replica_cleanup'"),
'CloudSetup version or rescue catalog indexes are incomplete.');

@ -218,7 +218,7 @@ foreach ([
'CloudSetup omits restore/migration ledger contract: ' . $required);
}
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'")
"SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'")
&& str_contains($attemptModel,
"protected \$table = 'cloud_backup_restore_attempts';")
&& str_contains($attemptModel, "'active_attempt' => 'int'")

@ -105,7 +105,7 @@ $assert(str_contains($setup,
'CloudSetup lost the open-compensation fence index contract.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'"),
"SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'"),
'CloudSetup schema version was not advanced for service compensation.');
$appliedShape = preg_match(

@ -33,7 +33,7 @@ $assert(str_contains($artifact,
&& str_contains($artifact,
'chk_cloud_storage_artifact_identity CHECK')
&& str_contains($artifact,
"(state = 'publishing' AND size_bytes = 0 AND sha256 = '')")
"(state IN ('publishing','deleted') AND size_bytes = 0 AND sha256 = '')")
&& str_contains($artifact,
'size_bytes BETWEEN 1 AND maximum_size_bytes')
&& str_contains($artifact,
@ -98,8 +98,9 @@ $assert(str_contains($publication,
"state IN ('waiting_primary','waiting_protection')")
&& str_contains($publication, 'AND coverage_ready_at IS NULL')
&& str_contains($publication,
"state IN ('ready_to_finalize','finalizing','completed','retired')")
"state IN ('ready_to_finalize','finalizing','completed')")
&& str_contains($publication, 'AND coverage_ready_at IS NOT NULL')
&& str_contains($publication, "OR state = 'retired'")
&& str_contains($publication, "OR state = 'failed'"),
'Publication finalization states lack durable required-copy evidence.');
@ -137,7 +138,7 @@ $assert(is_string($publicationModel)
&& str_contains($artifactModel, "'maximum_size_bytes' => 'int'"),
'Storage publication/artifact model contracts are incomplete.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'"),
"SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'"),
'CloudSetup schema version was not advanced for durable publications.');
echo "Storage publication schema contracts passed.\n";

@ -63,7 +63,7 @@ $assert(str_contains($schema,
&& str_contains($schema,
'CONSTRAINT chk_cloud_template_import_completion CHECK (')
&& str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'"),
"SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'"),
'Template imports are missing durable identity or terminal-state constraints.');
$assert(str_contains($routes, "'/admin/cloud-template-imports'")
&& str_contains($routes, "'/admin/cloud-template-imports/{id}'")

@ -71,7 +71,7 @@ $assert(str_contains($schema,
'Service-period traffic-pool identity is not frozen or indexed.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'")
"SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'")
&& str_contains($setup, "'chk_cloud_traffic_entitlement_balance'")
&& str_contains($setup, "'cloud_traffic_ledger_sequence'"),
'CloudSetup schema version or named-object validation is incomplete.');

@ -101,7 +101,7 @@ $assert(str_contains($jobs, 'vpc_id BIGINT UNSIGNED DEFAULT NULL')
'Cloud jobs cannot be correlated to VPC or VPC-port work.');
$assert(str_contains($setup,
"private const SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'")
"private const SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'")
&& str_contains($setup, "'name' => 'cloud_vpc_enabled'")
&& str_contains($setup, "'value' => 'false'")
&& str_contains($setup,

@ -104,7 +104,7 @@ foreach ($models as $class => $modelTable) {
}
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v40-backup-deferral'")
"SCHEMA_VERSION = 'go-agent-wss-v41-backup-attempts'")
&& str_contains($setup,
"self::schema('ss.sql') . \"\\0\" . self::schema('cloud.sql')")
&& str_contains($setup,

Loading…
Cancel
Save