|
|
|
|
@ -20,8 +20,15 @@ final class CloudBackupService
|
|
|
|
|
|
|
|
|
|
private const STORAGE_PURPOSE = 'object-storage-backup-runtime';
|
|
|
|
|
private const ACTIVE_BACKUP_STATES = [
|
|
|
|
|
'creating', 'ready', 'uncertain', 'deleting', 'delete_uncertain',
|
|
|
|
|
'creating', 'ready', 'uncertain',
|
|
|
|
|
];
|
|
|
|
|
private const CLEANUP_DEBT_STATES = [
|
|
|
|
|
'deleting', 'delete_uncertain', 'delete_attention',
|
|
|
|
|
];
|
|
|
|
|
private const DELETE_STABLE_PROBES = 2;
|
|
|
|
|
private const DELETE_STABLE_SECONDS = 60;
|
|
|
|
|
private const DELETE_RETRY_SECONDS = 300;
|
|
|
|
|
private const DELETE_MAX_AUTOMATIC_ATTEMPTS = 3;
|
|
|
|
|
private const TRANSFER_URL_SECONDS = 604800;
|
|
|
|
|
private const MAX_AGENT_BODY_BYTES = 1048576;
|
|
|
|
|
// The create transaction opens a remote S3 multipart upload. Database
|
|
|
|
|
@ -330,8 +337,9 @@ final class CloudBackupService
|
|
|
|
|
/**
|
|
|
|
|
* Delete or safely reconcile one frozen S3 backup object.
|
|
|
|
|
*
|
|
|
|
|
* A second call after an ambiguous DELETE never issues another DELETE. It
|
|
|
|
|
* only performs HEAD and converges to deleted after an explicit 404.
|
|
|
|
|
* An ambiguous DELETE is never replayed. Two spaced HEAD observations of
|
|
|
|
|
* the same immutable identity may authorize a fresh deletion generation;
|
|
|
|
|
* at most three generations run automatically in one supervised cycle.
|
|
|
|
|
*/
|
|
|
|
|
public static function delete(CloudInstanceBackup $backup, array $actor,
|
|
|
|
|
?CloudObjectStorageS3Client $client = null,
|
|
|
|
|
@ -339,8 +347,9 @@ final class CloudBackupService
|
|
|
|
|
{
|
|
|
|
|
CloudAccess::manage($actor);
|
|
|
|
|
$client ??= new CloudObjectStorageS3Client();
|
|
|
|
|
$now = time();
|
|
|
|
|
$prepared = $backup->getConnection()->transaction(function () use (
|
|
|
|
|
$backup, $beforeDeleteLocked
|
|
|
|
|
$backup, $beforeDeleteLocked, $now
|
|
|
|
|
): array {
|
|
|
|
|
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
@ -352,86 +361,252 @@ final class CloudBackupService
|
|
|
|
|
return ['mode' => 'deleted', 'backup' => $locked];
|
|
|
|
|
}
|
|
|
|
|
self::assertBackupDeletionAvailable($locked);
|
|
|
|
|
if ($state === 'delete_attention') {
|
|
|
|
|
return ['mode' => 'attention', 'backup' => $locked];
|
|
|
|
|
}
|
|
|
|
|
if (in_array($state, ['deleting', 'delete_uncertain'], true)) {
|
|
|
|
|
if ((int) $locked->delete_next_check_at > $now) {
|
|
|
|
|
return ['mode' => 'waiting', 'backup' => $locked];
|
|
|
|
|
}
|
|
|
|
|
return [
|
|
|
|
|
'mode' => 'reconcile',
|
|
|
|
|
'backup' => $locked,
|
|
|
|
|
'runtime' => self::storageRuntime($locked),
|
|
|
|
|
'object_key' => (string) $locked->object_key,
|
|
|
|
|
'object_etag' => (string) $locked->object_etag,
|
|
|
|
|
'object_version_id' => (string) $locked->object_version_id,
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
if (!in_array($state, ['ready', 'failed'], true)) {
|
|
|
|
|
throw new CloudError('当前备份状态不允许删除', 409);
|
|
|
|
|
}
|
|
|
|
|
if ($beforeDeleteLocked !== null) {
|
|
|
|
|
$beforeDeleteLocked($locked);
|
|
|
|
|
}
|
|
|
|
|
// Decrypt before crossing the durable deleting boundary. A crash
|
|
|
|
|
// after the state change is treated as unknown and only HEAD may
|
|
|
|
|
// follow; DELETE is never replayed.
|
|
|
|
|
// Decrypt before any remote request. The exact immutable identity
|
|
|
|
|
// is captured by HEAD before crossing the deleting boundary.
|
|
|
|
|
$runtime = self::storageRuntime($locked);
|
|
|
|
|
$locked->state = 'deleting';
|
|
|
|
|
$locked->error_code = '';
|
|
|
|
|
$locked->updated_at = time();
|
|
|
|
|
$locked->save();
|
|
|
|
|
return [
|
|
|
|
|
'mode' => 'delete',
|
|
|
|
|
'mode' => 'identify',
|
|
|
|
|
'previous_state' => $state,
|
|
|
|
|
'backup' => $locked,
|
|
|
|
|
'runtime' => $runtime,
|
|
|
|
|
'object_key' => (string) $locked->object_key,
|
|
|
|
|
'object_etag' => (string) $locked->object_etag,
|
|
|
|
|
'object_version_id' => (string) $locked->object_version_id,
|
|
|
|
|
];
|
|
|
|
|
}, 3);
|
|
|
|
|
|
|
|
|
|
if ($prepared['mode'] === 'deleted') {
|
|
|
|
|
if (in_array($prepared['mode'], ['deleted', 'waiting', 'attention'], true)) {
|
|
|
|
|
return self::deletionResult($prepared['backup']);
|
|
|
|
|
}
|
|
|
|
|
$runtime = $prepared['runtime'];
|
|
|
|
|
$objectKey = $prepared['object_key'];
|
|
|
|
|
if ($prepared['mode'] === 'identify') {
|
|
|
|
|
try {
|
|
|
|
|
$identity = $client->inspectObject(
|
|
|
|
|
$runtime['config'], $runtime['credentials'], $objectKey,
|
|
|
|
|
(string) $prepared['object_version_id']
|
|
|
|
|
);
|
|
|
|
|
} catch (CloudObjectStorageTransferError $error) {
|
|
|
|
|
throw new CloudError($error->getMessage(), 502);
|
|
|
|
|
}
|
|
|
|
|
if ($identity === null) {
|
|
|
|
|
return self::finishDelete((int) $backup->id);
|
|
|
|
|
}
|
|
|
|
|
self::assertObjectIdentity(
|
|
|
|
|
$prepared['backup'], $identity,
|
|
|
|
|
(string) $prepared['object_etag'] !== ''
|
|
|
|
|
);
|
|
|
|
|
$prepared = self::claimDeleteGeneration(
|
|
|
|
|
(int) $backup->id,
|
|
|
|
|
(string) $prepared['previous_state'],
|
|
|
|
|
$identity,
|
|
|
|
|
time(),
|
|
|
|
|
$beforeDeleteLocked
|
|
|
|
|
);
|
|
|
|
|
if ($prepared['mode'] !== 'delete') {
|
|
|
|
|
return self::deletionResult($prepared['backup']);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if ($prepared['mode'] === 'reconcile') {
|
|
|
|
|
try {
|
|
|
|
|
$presence = $client->objectPresence(
|
|
|
|
|
$runtime['config'], $runtime['credentials'], $objectKey
|
|
|
|
|
$identity = $client->inspectObject(
|
|
|
|
|
$runtime['config'], $runtime['credentials'], $objectKey,
|
|
|
|
|
(string) $prepared['object_version_id']
|
|
|
|
|
);
|
|
|
|
|
} catch (CloudObjectStorageTransferError $error) {
|
|
|
|
|
self::markDeleteUncertain((int) $backup->id,
|
|
|
|
|
$error->publicCode);
|
|
|
|
|
self::markDeleteUncertain(
|
|
|
|
|
(int) $backup->id, $error->publicCode, time(), true,
|
|
|
|
|
self::DELETE_RETRY_SECONDS
|
|
|
|
|
);
|
|
|
|
|
return self::deletionResult(
|
|
|
|
|
CloudInstanceBackup::findOrFail((int) $backup->id)
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
if ($presence === 'absent') {
|
|
|
|
|
if ($identity === null) {
|
|
|
|
|
return self::finishDelete((int) $backup->id);
|
|
|
|
|
}
|
|
|
|
|
self::markDeleteUncertain((int) $backup->id,
|
|
|
|
|
'object_delete_still_present');
|
|
|
|
|
return self::deletionResult(
|
|
|
|
|
CloudInstanceBackup::findOrFail((int) $backup->id)
|
|
|
|
|
try {
|
|
|
|
|
self::assertObjectIdentity($prepared['backup'], $identity, true);
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
self::markDeleteAttention(
|
|
|
|
|
(int) $backup->id, 'object_identity_changed', time()
|
|
|
|
|
);
|
|
|
|
|
return self::deletionResult(
|
|
|
|
|
CloudInstanceBackup::findOrFail((int) $backup->id)
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
$prepared = self::recordStableDeleteProbe(
|
|
|
|
|
(int) $backup->id, time()
|
|
|
|
|
);
|
|
|
|
|
if ($prepared['mode'] !== 'delete') {
|
|
|
|
|
return self::deletionResult($prepared['backup']);
|
|
|
|
|
}
|
|
|
|
|
$runtime = $prepared['runtime'];
|
|
|
|
|
$objectKey = $prepared['object_key'];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
$client->deleteObject(
|
|
|
|
|
$runtime['config'], $runtime['credentials'], $objectKey
|
|
|
|
|
$runtime['config'], $runtime['credentials'], $objectKey,
|
|
|
|
|
(string) $prepared['object_version_id']
|
|
|
|
|
);
|
|
|
|
|
} catch (CloudObjectStorageTransferError $error) {
|
|
|
|
|
if ($error->effectMayHaveApplied) {
|
|
|
|
|
self::markDeleteUncertain((int) $backup->id,
|
|
|
|
|
$error->publicCode);
|
|
|
|
|
self::markDeleteUncertain(
|
|
|
|
|
(int) $backup->id, $error->publicCode, time(), true
|
|
|
|
|
);
|
|
|
|
|
return self::deletionResult(
|
|
|
|
|
CloudInstanceBackup::findOrFail((int) $backup->id)
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
self::restoreAfterRejectedDelete(
|
|
|
|
|
(int) $backup->id,
|
|
|
|
|
(string) $prepared['previous_state'],
|
|
|
|
|
$error->publicCode
|
|
|
|
|
);
|
|
|
|
|
if (isset($prepared['previous_state'])) {
|
|
|
|
|
self::restoreAfterRejectedDelete(
|
|
|
|
|
(int) $backup->id,
|
|
|
|
|
(string) $prepared['previous_state'],
|
|
|
|
|
$error->publicCode
|
|
|
|
|
);
|
|
|
|
|
} else {
|
|
|
|
|
self::markDeleteAttention(
|
|
|
|
|
(int) $backup->id, $error->publicCode, time()
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
throw new CloudError($error->getMessage(), 502);
|
|
|
|
|
}
|
|
|
|
|
return self::finishDelete((int) $backup->id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Sanitized administrator view of remote objects awaiting cleanup. */
|
|
|
|
|
public static function cleanupDebt(array $actor, array $input = []): array
|
|
|
|
|
{
|
|
|
|
|
if (($actor['is_admin'] ?? false) !== true) {
|
|
|
|
|
throw new CloudError('只有管理员可以查看对象清理异常', 403);
|
|
|
|
|
}
|
|
|
|
|
CloudInput::only($input, ['page', 'per_page']);
|
|
|
|
|
$page = self::libraryInteger($input['page'] ?? 1, 1, 1000000, 1);
|
|
|
|
|
$perPage = self::libraryInteger($input['per_page'] ?? 20, 10, 50, 20);
|
|
|
|
|
$query = CloudInstanceBackup::whereIn('state', self::CLEANUP_DEBT_STATES);
|
|
|
|
|
$total = (int) (clone $query)->count('id');
|
|
|
|
|
$bytes = (int) (clone $query)->sum('object_size_bytes');
|
|
|
|
|
$attention = (int) (clone $query)
|
|
|
|
|
->where('state', 'delete_attention')->count('id');
|
|
|
|
|
$totalPages = max(1, (int) ceil($total / $perPage));
|
|
|
|
|
$page = min($page, $totalPages);
|
|
|
|
|
$rows = $query->orderByRaw(
|
|
|
|
|
"CASE WHEN state = 'delete_attention' THEN 0 ELSE 1 END"
|
|
|
|
|
)->orderBy('delete_next_check_at')->orderBy('id')
|
|
|
|
|
->offset(($page - 1) * $perPage)->limit($perPage)->get();
|
|
|
|
|
$items = [];
|
|
|
|
|
foreach ($rows as $row) {
|
|
|
|
|
$items[] = [
|
|
|
|
|
'id' => (string) $row->id,
|
|
|
|
|
'name' => (string) $row->name,
|
|
|
|
|
'source_instance_id' => (string) $row->source_instance_id,
|
|
|
|
|
'user_id' => (string) $row->user_id,
|
|
|
|
|
'state' => (string) $row->state,
|
|
|
|
|
'object_size_bytes' => max(0, (int) $row->object_size_bytes),
|
|
|
|
|
'delete_generation' => max(0, (int) $row->delete_generation),
|
|
|
|
|
'cycle_attempts' => max(0, (int) $row->delete_cycle_attempts),
|
|
|
|
|
'next_check_at' => max(0, (int) $row->delete_next_check_at),
|
|
|
|
|
'attention_at' => $row->delete_attention_at === null ? null
|
|
|
|
|
: max(0, (int) $row->delete_attention_at),
|
|
|
|
|
'error_code' => (string) $row->error_code,
|
|
|
|
|
'can_retry' => (string) $row->state === 'delete_attention'
|
|
|
|
|
&& ($actor['is_super_admin'] ?? false) === true
|
|
|
|
|
&& ($actor['can_manage'] ?? false) === true,
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
return [
|
|
|
|
|
'summary' => [
|
|
|
|
|
'count' => $total,
|
|
|
|
|
'bytes' => $bytes,
|
|
|
|
|
'attention_count' => $attention,
|
|
|
|
|
],
|
|
|
|
|
'items' => $items,
|
|
|
|
|
'pagination' => [
|
|
|
|
|
'page' => $page,
|
|
|
|
|
'per_page' => $perPage,
|
|
|
|
|
'total' => $total,
|
|
|
|
|
'total_pages' => $totalPages,
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Begin one new bounded cycle after an administrator has inspected debt. */
|
|
|
|
|
public static function retryCleanup(CloudInstanceBackup $backup, array $actor,
|
|
|
|
|
?CloudObjectStorageS3Client $client = null): array
|
|
|
|
|
{
|
|
|
|
|
CloudAccess::manage($actor);
|
|
|
|
|
$client ??= new CloudObjectStorageS3Client();
|
|
|
|
|
$prepared = $backup->getConnection()->transaction(function () use ($backup): array {
|
|
|
|
|
$locked = CloudInstanceBackup::where('id', (int) $backup->id)
|
|
|
|
|
->lockForUpdate()->firstOrFail();
|
|
|
|
|
if ((string) $locked->state !== 'delete_attention') {
|
|
|
|
|
throw new CloudError('只有待人工处理的对象可以重新进入清理流程', 409);
|
|
|
|
|
}
|
|
|
|
|
self::assertBackupDeletionAvailable($locked);
|
|
|
|
|
return [
|
|
|
|
|
'runtime' => self::storageRuntime($locked),
|
|
|
|
|
'object_key' => (string) $locked->object_key,
|
|
|
|
|
'object_version_id' => (string) $locked->object_version_id,
|
|
|
|
|
'backup' => $locked,
|
|
|
|
|
];
|
|
|
|
|
}, 3);
|
|
|
|
|
try {
|
|
|
|
|
$identity = $client->inspectObject(
|
|
|
|
|
$prepared['runtime']['config'],
|
|
|
|
|
$prepared['runtime']['credentials'],
|
|
|
|
|
$prepared['object_key'],
|
|
|
|
|
$prepared['object_version_id']
|
|
|
|
|
);
|
|
|
|
|
} catch (CloudObjectStorageTransferError $error) {
|
|
|
|
|
throw new CloudError($error->getMessage(), 502);
|
|
|
|
|
}
|
|
|
|
|
if ($identity === null) {
|
|
|
|
|
return self::finishDelete((int) $backup->id);
|
|
|
|
|
}
|
|
|
|
|
self::assertObjectIdentity($prepared['backup'], $identity, true);
|
|
|
|
|
$now = time();
|
|
|
|
|
$locked = $backup->getConnection()->transaction(function () use (
|
|
|
|
|
$backup, $now
|
|
|
|
|
): CloudInstanceBackup {
|
|
|
|
|
$row = CloudInstanceBackup::where('id', (int) $backup->id)
|
|
|
|
|
->lockForUpdate()->firstOrFail();
|
|
|
|
|
if ((string) $row->state !== 'delete_attention') {
|
|
|
|
|
throw new CloudError('对象清理状态已发生变化', 409);
|
|
|
|
|
}
|
|
|
|
|
$row->state = 'delete_uncertain';
|
|
|
|
|
$row->delete_cycle_attempts = 0;
|
|
|
|
|
$row->delete_presence_confirmations = 1;
|
|
|
|
|
$row->delete_last_checked_at = $now;
|
|
|
|
|
$row->delete_next_check_at = $now + self::DELETE_STABLE_SECONDS;
|
|
|
|
|
$row->delete_attention_at = null;
|
|
|
|
|
$row->error_code = 'manual_retry_awaiting_stable_probe';
|
|
|
|
|
$row->updated_at = $now;
|
|
|
|
|
$row->save();
|
|
|
|
|
return $row;
|
|
|
|
|
}, 3);
|
|
|
|
|
return self::deletionResult($locked);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static function create(CloudInstance $instance, array $actor,
|
|
|
|
|
array $input): array
|
|
|
|
|
{
|
|
|
|
|
@ -1213,6 +1388,8 @@ final class CloudBackupService
|
|
|
|
|
$virtual = filter_var($result['virtual_size_bytes'] ?? null,
|
|
|
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
|
|
|
$sha = $result['sha256'] ?? null;
|
|
|
|
|
$objectEtag = $result['object_etag'] ?? '';
|
|
|
|
|
$objectVersionId = $result['object_version_id'] ?? '';
|
|
|
|
|
$consistency = $result['consistency'] ?? null;
|
|
|
|
|
$cleanupPending = $result['cleanup_pending'] ?? false;
|
|
|
|
|
if (!is_array($result)
|
|
|
|
|
@ -1221,6 +1398,12 @@ final class CloudBackupService
|
|
|
|
|
|| $size === false || $virtual === false
|
|
|
|
|
|| !is_string($sha)
|
|
|
|
|
|| preg_match('/^[a-f0-9]{64}$/D', $sha) !== 1
|
|
|
|
|
|| !is_string($objectEtag) || strlen($objectEtag) > 256
|
|
|
|
|
|| preg_match('/[\x00-\x1f\x7f]/D', $objectEtag) === 1
|
|
|
|
|
|| !is_string($objectVersionId)
|
|
|
|
|
|| strlen($objectVersionId) > 1024
|
|
|
|
|
|| preg_match('/[\x00-\x1f\x7f]/D', $objectVersionId) === 1
|
|
|
|
|
|| ($objectVersionId !== '' && $objectEtag === '')
|
|
|
|
|
|| !in_array($consistency, ['filesystem', 'crash'], true)
|
|
|
|
|
|| !is_bool($cleanupPending)
|
|
|
|
|
|| (int) $size > (int) ($payload['maximum_size_bytes'] ?? 0)) {
|
|
|
|
|
@ -1231,6 +1414,8 @@ final class CloudBackupService
|
|
|
|
|
$backup->object_size_bytes = (int) $size;
|
|
|
|
|
$backup->virtual_size_bytes = (int) $virtual;
|
|
|
|
|
$backup->sha256 = $sha;
|
|
|
|
|
$backup->object_etag = $objectEtag;
|
|
|
|
|
$backup->object_version_id = $objectVersionId;
|
|
|
|
|
$backup->ready_at = $now;
|
|
|
|
|
$backup->active_job_id = null;
|
|
|
|
|
// The remote object is complete and restorable. A host-side
|
|
|
|
|
@ -1577,15 +1762,178 @@ final class CloudBackupService
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** @param array{size_bytes:int,etag:string,version_id:string} $identity */
|
|
|
|
|
private static function claimDeleteGeneration(int $backupId,
|
|
|
|
|
string $previousState, array $identity, int $now,
|
|
|
|
|
?callable $beforeDeleteLocked): array
|
|
|
|
|
{
|
|
|
|
|
$backup = CloudInstanceBackup::findOrFail($backupId);
|
|
|
|
|
return $backup->getConnection()->transaction(function () use (
|
|
|
|
|
$backupId, $previousState, $identity, $now, $beforeDeleteLocked
|
|
|
|
|
): array {
|
|
|
|
|
$locked = CloudInstanceBackup::where('id', $backupId)
|
|
|
|
|
->lockForUpdate()->firstOrFail();
|
|
|
|
|
if ((string) $locked->state === 'deleted') {
|
|
|
|
|
return ['mode' => 'deleted', 'backup' => $locked];
|
|
|
|
|
}
|
|
|
|
|
if (in_array((string) $locked->state,
|
|
|
|
|
self::CLEANUP_DEBT_STATES, true)) {
|
|
|
|
|
return ['mode' => 'waiting', 'backup' => $locked];
|
|
|
|
|
}
|
|
|
|
|
if ((string) $locked->state !== $previousState) {
|
|
|
|
|
throw new CloudError('备份删除状态已发生变化', 409);
|
|
|
|
|
}
|
|
|
|
|
self::assertBackupDeletionAvailable($locked);
|
|
|
|
|
if ($beforeDeleteLocked !== null) {
|
|
|
|
|
$beforeDeleteLocked($locked);
|
|
|
|
|
}
|
|
|
|
|
$locked->object_etag = (string) $identity['etag'];
|
|
|
|
|
$locked->object_version_id = (string) $identity['version_id'];
|
|
|
|
|
$locked->state = 'deleting';
|
|
|
|
|
$locked->delete_operation_id = bin2hex(random_bytes(16));
|
|
|
|
|
$locked->delete_generation = max(0,
|
|
|
|
|
(int) $locked->delete_generation) + 1;
|
|
|
|
|
$locked->delete_cycle_attempts = 1;
|
|
|
|
|
$locked->delete_presence_confirmations = 0;
|
|
|
|
|
$locked->delete_requested_at = $now;
|
|
|
|
|
$locked->delete_last_checked_at = null;
|
|
|
|
|
$locked->delete_next_check_at = 0;
|
|
|
|
|
$locked->delete_attention_at = null;
|
|
|
|
|
$locked->error_code = '';
|
|
|
|
|
$locked->updated_at = $now;
|
|
|
|
|
$locked->save();
|
|
|
|
|
return [
|
|
|
|
|
'mode' => 'delete',
|
|
|
|
|
'previous_state' => $previousState,
|
|
|
|
|
'backup' => $locked,
|
|
|
|
|
'runtime' => self::storageRuntime($locked),
|
|
|
|
|
'object_key' => (string) $locked->object_key,
|
|
|
|
|
'object_version_id' => (string) $locked->object_version_id,
|
|
|
|
|
];
|
|
|
|
|
}, 3);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function recordStableDeleteProbe(int $backupId,
|
|
|
|
|
int $now): array
|
|
|
|
|
{
|
|
|
|
|
$backup = CloudInstanceBackup::findOrFail($backupId);
|
|
|
|
|
return $backup->getConnection()->transaction(function () use (
|
|
|
|
|
$backupId, $now
|
|
|
|
|
): array {
|
|
|
|
|
$locked = CloudInstanceBackup::where('id', $backupId)
|
|
|
|
|
->lockForUpdate()->firstOrFail();
|
|
|
|
|
if ((string) $locked->state === 'deleted') {
|
|
|
|
|
return ['mode' => 'deleted', 'backup' => $locked];
|
|
|
|
|
}
|
|
|
|
|
if ((string) $locked->state === 'delete_attention') {
|
|
|
|
|
return ['mode' => 'attention', 'backup' => $locked];
|
|
|
|
|
}
|
|
|
|
|
if (!in_array((string) $locked->state,
|
|
|
|
|
['deleting', 'delete_uncertain'], true)) {
|
|
|
|
|
throw new CloudError('备份删除状态已发生变化', 409);
|
|
|
|
|
}
|
|
|
|
|
$last = max(0, (int) $locked->delete_last_checked_at);
|
|
|
|
|
if ($last > 0 && $now - $last < self::DELETE_STABLE_SECONDS) {
|
|
|
|
|
$locked->state = 'delete_uncertain';
|
|
|
|
|
$locked->delete_next_check_at = $last
|
|
|
|
|
+ self::DELETE_STABLE_SECONDS;
|
|
|
|
|
$locked->updated_at = $now;
|
|
|
|
|
$locked->save();
|
|
|
|
|
return ['mode' => 'waiting', 'backup' => $locked];
|
|
|
|
|
}
|
|
|
|
|
$confirmations = min(self::DELETE_STABLE_PROBES,
|
|
|
|
|
max(0, (int) $locked->delete_presence_confirmations) + 1);
|
|
|
|
|
$locked->state = 'delete_uncertain';
|
|
|
|
|
$locked->delete_presence_confirmations = $confirmations;
|
|
|
|
|
$locked->delete_last_checked_at = $now;
|
|
|
|
|
$locked->delete_next_check_at = $now
|
|
|
|
|
+ self::DELETE_STABLE_SECONDS;
|
|
|
|
|
$locked->error_code = 'object_delete_still_present';
|
|
|
|
|
$locked->updated_at = $now;
|
|
|
|
|
if ($confirmations < self::DELETE_STABLE_PROBES) {
|
|
|
|
|
$locked->save();
|
|
|
|
|
return ['mode' => 'waiting', 'backup' => $locked];
|
|
|
|
|
}
|
|
|
|
|
if ((int) $locked->delete_cycle_attempts
|
|
|
|
|
>= self::DELETE_MAX_AUTOMATIC_ATTEMPTS) {
|
|
|
|
|
$locked->state = 'delete_attention';
|
|
|
|
|
$locked->delete_attention_at = $now;
|
|
|
|
|
$locked->delete_next_check_at = 0;
|
|
|
|
|
$locked->error_code = 'object_delete_retry_exhausted';
|
|
|
|
|
$locked->save();
|
|
|
|
|
return ['mode' => 'attention', 'backup' => $locked];
|
|
|
|
|
}
|
|
|
|
|
$locked->state = 'deleting';
|
|
|
|
|
$locked->delete_operation_id = bin2hex(random_bytes(16));
|
|
|
|
|
$locked->delete_generation = max(0,
|
|
|
|
|
(int) $locked->delete_generation) + 1;
|
|
|
|
|
$locked->delete_cycle_attempts = max(0,
|
|
|
|
|
(int) $locked->delete_cycle_attempts) + 1;
|
|
|
|
|
$locked->delete_presence_confirmations = 0;
|
|
|
|
|
$locked->delete_next_check_at = 0;
|
|
|
|
|
$locked->error_code = '';
|
|
|
|
|
$locked->save();
|
|
|
|
|
return [
|
|
|
|
|
'mode' => 'delete',
|
|
|
|
|
'backup' => $locked,
|
|
|
|
|
'runtime' => self::storageRuntime($locked),
|
|
|
|
|
'object_key' => (string) $locked->object_key,
|
|
|
|
|
'object_version_id' => (string) $locked->object_version_id,
|
|
|
|
|
];
|
|
|
|
|
}, 3);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** @param array{size_bytes:int,etag:string,version_id:string} $identity */
|
|
|
|
|
private static function assertObjectIdentity(CloudInstanceBackup $backup,
|
|
|
|
|
array $identity, bool $strict): void
|
|
|
|
|
{
|
|
|
|
|
if ((int) $identity['size_bytes'] !== (int) $backup->object_size_bytes) {
|
|
|
|
|
throw new CloudError('对象存储中的备份大小与目录账本不一致', 409);
|
|
|
|
|
}
|
|
|
|
|
if (!$strict) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
$etag = (string) $backup->object_etag;
|
|
|
|
|
if ($etag === '' || !hash_equals($etag, (string) $identity['etag'])) {
|
|
|
|
|
throw new CloudError('对象存储中的备份 ETag 已发生变化', 409);
|
|
|
|
|
}
|
|
|
|
|
$versionId = (string) $backup->object_version_id;
|
|
|
|
|
if ($versionId !== ''
|
|
|
|
|
&& !hash_equals($versionId, (string) $identity['version_id'])) {
|
|
|
|
|
throw new CloudError('对象存储中的备份版本已发生变化', 409);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function markDeleteUncertain(int $backupId,
|
|
|
|
|
string $errorCode): void
|
|
|
|
|
string $errorCode, int $now, bool $resetEvidence,
|
|
|
|
|
int $retryAfter = self::DELETE_STABLE_SECONDS): void
|
|
|
|
|
{
|
|
|
|
|
$values = [
|
|
|
|
|
'state' => 'delete_uncertain',
|
|
|
|
|
'delete_next_check_at' => $now + max(1, $retryAfter),
|
|
|
|
|
'error_code' => substr($errorCode, 0, 64),
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
];
|
|
|
|
|
if ($resetEvidence) {
|
|
|
|
|
$values['delete_presence_confirmations'] = 0;
|
|
|
|
|
$values['delete_last_checked_at'] = null;
|
|
|
|
|
}
|
|
|
|
|
CloudInstanceBackup::where('id', $backupId)
|
|
|
|
|
->whereIn('state', ['deleting', 'delete_uncertain'])
|
|
|
|
|
->update($values);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function markDeleteAttention(int $backupId,
|
|
|
|
|
string $errorCode, int $now): void
|
|
|
|
|
{
|
|
|
|
|
CloudInstanceBackup::where('id', $backupId)
|
|
|
|
|
->whereIn('state', ['deleting', 'delete_uncertain'])
|
|
|
|
|
->update([
|
|
|
|
|
'state' => 'delete_uncertain',
|
|
|
|
|
'state' => 'delete_attention',
|
|
|
|
|
'delete_next_check_at' => 0,
|
|
|
|
|
'delete_attention_at' => $now,
|
|
|
|
|
'error_code' => substr($errorCode, 0, 64),
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -1596,6 +1944,13 @@ final class CloudBackupService
|
|
|
|
|
->where('state', 'deleting')
|
|
|
|
|
->update([
|
|
|
|
|
'state' => $previousState,
|
|
|
|
|
'delete_operation_id' => null,
|
|
|
|
|
'delete_cycle_attempts' => 0,
|
|
|
|
|
'delete_presence_confirmations' => 0,
|
|
|
|
|
'delete_requested_at' => null,
|
|
|
|
|
'delete_last_checked_at' => null,
|
|
|
|
|
'delete_next_check_at' => 0,
|
|
|
|
|
'delete_attention_at' => null,
|
|
|
|
|
'error_code' => substr($errorCode, 0, 64),
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
]);
|
|
|
|
|
@ -1612,16 +1967,28 @@ final class CloudBackupService
|
|
|
|
|
if ((string) $locked->state !== 'deleted') {
|
|
|
|
|
self::assertBackupDeletionAvailable($locked);
|
|
|
|
|
if (!in_array((string) $locked->state,
|
|
|
|
|
['deleting', 'delete_uncertain'], true)) {
|
|
|
|
|
['ready', 'failed', 'deleting', 'delete_uncertain',
|
|
|
|
|
'delete_attention'], true)) {
|
|
|
|
|
throw new CloudError('备份删除状态已发生变化', 409);
|
|
|
|
|
}
|
|
|
|
|
$locked->state = 'deleted';
|
|
|
|
|
$locked->sealed_storage_runtime = '';
|
|
|
|
|
$locked->sealed_credential = null;
|
|
|
|
|
$locked->active_job_id = null;
|
|
|
|
|
$locked->delete_presence_confirmations = 0;
|
|
|
|
|
$locked->delete_next_check_at = 0;
|
|
|
|
|
$locked->delete_attention_at = null;
|
|
|
|
|
$locked->error_code = '';
|
|
|
|
|
$locked->updated_at = time();
|
|
|
|
|
$locked->save();
|
|
|
|
|
CloudDeploymentSnapshot::where('backup_id', $backupId)
|
|
|
|
|
->whereIn('state', ['deleting', 'delete_uncertain'])
|
|
|
|
|
->update([
|
|
|
|
|
'state' => 'deleted',
|
|
|
|
|
'deleted_at' => time(),
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
]);
|
|
|
|
|
}
|
|
|
|
|
return self::deletionResult($locked);
|
|
|
|
|
}, 3);
|
|
|
|
|
@ -1634,7 +2001,10 @@ final class CloudBackupService
|
|
|
|
|
'backup_id' => (string) $backup->id,
|
|
|
|
|
'state' => $state,
|
|
|
|
|
'deleted' => $state === 'deleted',
|
|
|
|
|
'requires_reconciliation' => $state === 'delete_uncertain',
|
|
|
|
|
'requires_reconciliation' => in_array($state,
|
|
|
|
|
['deleting', 'delete_uncertain'], true),
|
|
|
|
|
'requires_attention' => $state === 'delete_attention',
|
|
|
|
|
'next_check_at' => max(0, (int) $backup->delete_next_check_at),
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -1782,7 +2152,7 @@ final class CloudBackupService
|
|
|
|
|
{
|
|
|
|
|
$state = in_array((string) $backup->state,
|
|
|
|
|
['creating', 'ready', 'failed', 'uncertain', 'deleting',
|
|
|
|
|
'delete_uncertain', 'deleted'], true)
|
|
|
|
|
'delete_uncertain', 'delete_attention', 'deleted'], true)
|
|
|
|
|
? (string) $backup->state : 'uncertain';
|
|
|
|
|
$canDelete = $canManage && in_array($state,
|
|
|
|
|
['ready', 'failed', 'deleting', 'delete_uncertain'], true);
|
|
|
|
|
@ -1822,9 +2192,11 @@ final class CloudBackupService
|
|
|
|
|
'delete_action' => in_array($state,
|
|
|
|
|
['deleting', 'delete_uncertain'], true)
|
|
|
|
|
? 'reconcile' : 'delete',
|
|
|
|
|
'error' => $state === 'delete_uncertain'
|
|
|
|
|
? '删除结果尚未确认;再次核对只会读取对象状态,不会重发删除请求。'
|
|
|
|
|
: null,
|
|
|
|
|
'error' => $state === 'delete_attention'
|
|
|
|
|
? '对象自动清理已达到三次上限,需要管理员检查对象存储后手动恢复清理。'
|
|
|
|
|
: ($state === 'delete_uncertain'
|
|
|
|
|
? '删除结果尚未确认;系统会在两次稳定核对后创建新的删除代次。'
|
|
|
|
|
: null),
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|