|
|
|
|
@ -19,6 +19,16 @@ final class CloudAutomaticBackupService
|
|
|
|
|
CloudInvoiceService::PAYMENT_GRACE_SECONDS;
|
|
|
|
|
|
|
|
|
|
private const RETRY_SECONDS = 300;
|
|
|
|
|
private const MAX_DEFER_COUNT = 4294967295;
|
|
|
|
|
private const DEFER_REASONS = [
|
|
|
|
|
CloudBackupEligibilityDecision::MIGRATION_ACTIVE,
|
|
|
|
|
CloudBackupEligibilityDecision::INSTANCE_MUTATION_ACTIVE,
|
|
|
|
|
CloudBackupEligibilityDecision::BACKUP_RESCUE_ACTIVE,
|
|
|
|
|
CloudBackupEligibilityDecision::BACKUP_DISK_TRANSITION_ACTIVE,
|
|
|
|
|
CloudBackupEligibilityDecision::DOMAIN_JOB_ACTIVE,
|
|
|
|
|
CloudBackupEligibilityDecision::READINESS_UNAVAILABLE,
|
|
|
|
|
'scheduler_unavailable',
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
public static function activatePurchased(CloudInstance $instance): ?CloudBackupSubscription
|
|
|
|
|
{
|
|
|
|
|
@ -80,6 +90,7 @@ final class CloudAutomaticBackupService
|
|
|
|
|
$subscription->next_run_at = self::nextRunAt(
|
|
|
|
|
$now, 'daily', 0
|
|
|
|
|
);
|
|
|
|
|
self::clearDeferredState($subscription);
|
|
|
|
|
$subscription->last_scheduled_at = null;
|
|
|
|
|
$subscription->last_backup_id = null;
|
|
|
|
|
$subscription->created_at = $subscription->updated_at = $now;
|
|
|
|
|
@ -202,6 +213,7 @@ final class CloudAutomaticBackupService
|
|
|
|
|
$locked->next_run_at = self::nextRunAt(
|
|
|
|
|
$now, $frequency, (int) $weekday
|
|
|
|
|
);
|
|
|
|
|
self::clearDeferredState($locked);
|
|
|
|
|
}
|
|
|
|
|
$locked->updated_at = $now;
|
|
|
|
|
$locked->save();
|
|
|
|
|
@ -283,7 +295,8 @@ final class CloudAutomaticBackupService
|
|
|
|
|
$now ??= time();
|
|
|
|
|
$ids = CloudBackupSubscription::where('enabled', 1)
|
|
|
|
|
->where('next_run_at', '>', 0)
|
|
|
|
|
->where('next_run_at', '<=', $now)->orderBy('next_run_at')
|
|
|
|
|
->where('next_run_at', '<=', $now)
|
|
|
|
|
->where('deferred_until', '<=', $now)->orderBy('next_run_at')
|
|
|
|
|
->orderBy('id')->limit($limit)->pluck('id')->all();
|
|
|
|
|
$queued = 0;
|
|
|
|
|
foreach ($ids as $id) {
|
|
|
|
|
@ -295,9 +308,6 @@ final class CloudAutomaticBackupService
|
|
|
|
|
if (self::pauseIfInstanceExpired((int) $id, $now)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (self::deferIfMigrationOpen((int) $id, $now)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
// Creation owns a durable intent before object-storage
|
|
|
|
|
// preparation begins. Keep that exact scheduled slot stable
|
|
|
|
|
// while the preparation worker reconciles it; moving
|
|
|
|
|
@ -306,13 +316,11 @@ final class CloudAutomaticBackupService
|
|
|
|
|
if (self::hasPendingCreateIntent((int) $id, $now)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
CloudBackupSubscription::where('id', (int) $id)
|
|
|
|
|
->where('enabled', 1)
|
|
|
|
|
->where('next_run_at', '<=', $now)
|
|
|
|
|
->update([
|
|
|
|
|
'next_run_at' => $now + self::RETRY_SECONDS,
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
]);
|
|
|
|
|
self::deferCurrentSlot(
|
|
|
|
|
(int) $id,
|
|
|
|
|
$now,
|
|
|
|
|
'scheduler_unavailable'
|
|
|
|
|
);
|
|
|
|
|
error_log('cloud automatic backup schedule unavailable: '
|
|
|
|
|
. get_class($error));
|
|
|
|
|
}
|
|
|
|
|
@ -573,6 +581,7 @@ final class CloudAutomaticBackupService
|
|
|
|
|
(string) $subscription->frequency,
|
|
|
|
|
(int) $subscription->weekday
|
|
|
|
|
);
|
|
|
|
|
self::clearDeferredState($subscription);
|
|
|
|
|
}
|
|
|
|
|
$subscription->updated_at = time();
|
|
|
|
|
$subscription->save();
|
|
|
|
|
@ -582,7 +591,8 @@ final class CloudAutomaticBackupService
|
|
|
|
|
{
|
|
|
|
|
$subscription = CloudBackupSubscription::find($subscriptionId);
|
|
|
|
|
if ($subscription === null || !(bool) $subscription->enabled
|
|
|
|
|
|| (int) $subscription->next_run_at > $now) {
|
|
|
|
|
|| (int) $subscription->next_run_at > $now
|
|
|
|
|
|| (int) $subscription->deferred_until > $now) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
$instance = CloudInstance::find((int) $subscription->instance_id);
|
|
|
|
|
@ -596,9 +606,6 @@ final class CloudAutomaticBackupService
|
|
|
|
|
|| (int) $instance->user_id !== (int) $subscription->user_id) {
|
|
|
|
|
throw new CloudError('实例当前不可创建自动备份', 409);
|
|
|
|
|
}
|
|
|
|
|
if (self::deferIfMigrationOpen($subscriptionId, $now)) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
$scheduledFor = (int) $subscription->next_run_at;
|
|
|
|
|
if (!CloudBackupPeriodEntitlement::where(
|
|
|
|
|
'subscription_id', (int) $subscription->id
|
|
|
|
|
@ -606,7 +613,8 @@ final class CloudAutomaticBackupService
|
|
|
|
|
->where('user_id', (int) $instance->user_id)
|
|
|
|
|
->where('starts_at', '<=', $scheduledFor)
|
|
|
|
|
->where('ends_at', '>', $scheduledFor)->exists()) {
|
|
|
|
|
throw new CloudError('自动备份执行时间不在已付权益周期内', 409);
|
|
|
|
|
self::pauseWithoutEntitlement($subscriptionId, $scheduledFor, $now);
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
$actor = [
|
|
|
|
|
'user_id' => (int) $instance->user_id,
|
|
|
|
|
@ -617,6 +625,21 @@ final class CloudAutomaticBackupService
|
|
|
|
|
'request_idempotency_key' => 'automatic-backup-'
|
|
|
|
|
. (int) $subscription->id . '-' . $scheduledFor,
|
|
|
|
|
];
|
|
|
|
|
if (!self::hasPendingCreateIntent($subscriptionId, $now)) {
|
|
|
|
|
$eligibility = (new CloudBackupEligibility())->assess(
|
|
|
|
|
$instance,
|
|
|
|
|
$actor
|
|
|
|
|
);
|
|
|
|
|
if (!$eligibility->allowed()) {
|
|
|
|
|
self::deferCurrentSlot(
|
|
|
|
|
$subscriptionId,
|
|
|
|
|
$now,
|
|
|
|
|
$eligibility->reasonCode,
|
|
|
|
|
$scheduledFor
|
|
|
|
|
);
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
$queued = CloudBackupService::createAutomatic(
|
|
|
|
|
$instance, $subscription, $actor, $scheduledFor
|
|
|
|
|
);
|
|
|
|
|
@ -673,6 +696,7 @@ final class CloudAutomaticBackupService
|
|
|
|
|
|| (int) $subscription->next_run_at !== 0) {
|
|
|
|
|
$subscription->enabled = false;
|
|
|
|
|
$subscription->next_run_at = 0;
|
|
|
|
|
self::clearDeferredState($subscription);
|
|
|
|
|
self::advanceStateRevision($subscription);
|
|
|
|
|
$subscription->updated_at = $now;
|
|
|
|
|
$subscription->save();
|
|
|
|
|
@ -682,14 +706,21 @@ final class CloudAutomaticBackupService
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* An open maintenance migration is a temporary instance mutation fence,
|
|
|
|
|
* not an automatic-backup failure. Serialize with migration start through
|
|
|
|
|
* the instance row, then move only the due slot by the bounded retry delay.
|
|
|
|
|
* Record one bounded retry without changing the logical schedule slot.
|
|
|
|
|
* Locking the instance before the subscription keeps mutation admission
|
|
|
|
|
* ordering consistent while preserving the idempotency identity.
|
|
|
|
|
*/
|
|
|
|
|
private static function deferIfMigrationOpen(
|
|
|
|
|
private static function deferCurrentSlot(
|
|
|
|
|
int $subscriptionId,
|
|
|
|
|
int $now
|
|
|
|
|
int $now,
|
|
|
|
|
string $reason,
|
|
|
|
|
?int $expectedSlot = null
|
|
|
|
|
): bool {
|
|
|
|
|
if (!in_array($reason, self::DEFER_REASONS, true)) {
|
|
|
|
|
throw new \InvalidArgumentException(
|
|
|
|
|
'Invalid automatic backup defer reason'
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
$probe = CloudBackupSubscription::find($subscriptionId);
|
|
|
|
|
if ($probe === null) {
|
|
|
|
|
return false;
|
|
|
|
|
@ -697,7 +728,9 @@ final class CloudAutomaticBackupService
|
|
|
|
|
return $probe->getConnection()->transaction(function () use (
|
|
|
|
|
$subscriptionId,
|
|
|
|
|
$probe,
|
|
|
|
|
$now
|
|
|
|
|
$now,
|
|
|
|
|
$reason,
|
|
|
|
|
$expectedSlot
|
|
|
|
|
): bool {
|
|
|
|
|
$instance = CloudInstance::where(
|
|
|
|
|
'id', (int) $probe->instance_id
|
|
|
|
|
@ -710,18 +743,66 @@ final class CloudAutomaticBackupService
|
|
|
|
|
|| (int) $subscription->instance_id !== (int) $instance->id
|
|
|
|
|
|| (int) $subscription->next_run_at < 1
|
|
|
|
|
|| (int) $subscription->next_run_at > $now
|
|
|
|
|
|| !CloudInstanceMigrationService::hasOpenMigration(
|
|
|
|
|
(int) $instance->id
|
|
|
|
|
)) {
|
|
|
|
|
|| (int) $subscription->deferred_until > $now
|
|
|
|
|
|| ($expectedSlot !== null
|
|
|
|
|
&& (int) $subscription->next_run_at !== $expectedSlot)) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
$subscription->next_run_at = $now + self::RETRY_SECONDS;
|
|
|
|
|
$count = max(0, (int) $subscription->defer_count);
|
|
|
|
|
$subscription->deferred_until = $now + self::RETRY_SECONDS;
|
|
|
|
|
$subscription->defer_reason = $reason;
|
|
|
|
|
$subscription->deferred_at = $now;
|
|
|
|
|
$subscription->defer_count = min(
|
|
|
|
|
self::MAX_DEFER_COUNT,
|
|
|
|
|
$count + 1
|
|
|
|
|
);
|
|
|
|
|
$subscription->updated_at = $now;
|
|
|
|
|
$subscription->save();
|
|
|
|
|
return true;
|
|
|
|
|
}, 3);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** A missing paid slot is a parked subscription, never an infinite retry. */
|
|
|
|
|
private static function pauseWithoutEntitlement(
|
|
|
|
|
int $subscriptionId,
|
|
|
|
|
int $scheduledFor,
|
|
|
|
|
int $now
|
|
|
|
|
): void {
|
|
|
|
|
$subscription = CloudBackupSubscription::find($subscriptionId);
|
|
|
|
|
if (!$subscription instanceof CloudBackupSubscription) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
$subscription->getConnection()->transaction(function () use (
|
|
|
|
|
$subscriptionId,
|
|
|
|
|
$scheduledFor,
|
|
|
|
|
$now
|
|
|
|
|
): void {
|
|
|
|
|
$locked = CloudBackupSubscription::where(
|
|
|
|
|
'id', $subscriptionId
|
|
|
|
|
)->lockForUpdate()->first();
|
|
|
|
|
if (!$locked instanceof CloudBackupSubscription
|
|
|
|
|
|| !(bool) $locked->enabled
|
|
|
|
|
|| (int) $locked->next_run_at !== $scheduledFor) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
$locked->enabled = false;
|
|
|
|
|
$locked->next_run_at = 0;
|
|
|
|
|
self::clearDeferredState($locked);
|
|
|
|
|
self::advanceStateRevision($locked);
|
|
|
|
|
$locked->updated_at = $now;
|
|
|
|
|
$locked->save();
|
|
|
|
|
}, 3);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static function clearDeferredState(
|
|
|
|
|
CloudBackupSubscription $subscription
|
|
|
|
|
): void {
|
|
|
|
|
$subscription->deferred_until = 0;
|
|
|
|
|
$subscription->defer_reason = '';
|
|
|
|
|
$subscription->deferred_at = null;
|
|
|
|
|
$subscription->defer_count = 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function backupsForInstance(CloudInstance $instance): array
|
|
|
|
|
{
|
|
|
|
|
$items = [];
|
|
|
|
|
@ -815,6 +896,19 @@ final class CloudAutomaticBackupService
|
|
|
|
|
'timezone' => self::TIMEZONE,
|
|
|
|
|
'run_at' => self::RUN_AT,
|
|
|
|
|
'next_run_at' => max(0, (int) $subscription->next_run_at),
|
|
|
|
|
'deferred' => (int) $subscription->deferred_until > 0,
|
|
|
|
|
'deferred_until' => max(
|
|
|
|
|
0,
|
|
|
|
|
(int) $subscription->deferred_until
|
|
|
|
|
),
|
|
|
|
|
'defer_reason' => in_array(
|
|
|
|
|
(string) $subscription->defer_reason,
|
|
|
|
|
self::DEFER_REASONS,
|
|
|
|
|
true
|
|
|
|
|
) ? (string) $subscription->defer_reason : '',
|
|
|
|
|
'deferred_at' => $subscription->deferred_at === null
|
|
|
|
|
? null : max(0, (int) $subscription->deferred_at),
|
|
|
|
|
'defer_count' => max(0, (int) $subscription->defer_count),
|
|
|
|
|
'last_scheduled_at' => $subscription->last_scheduled_at === null
|
|
|
|
|
? null : max(0, (int) $subscription->last_scheduled_at),
|
|
|
|
|
'price' => (string) $subscription->price,
|
|
|
|
|
|