You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
4107 lines
183 KiB
PHP
4107 lines
183 KiB
PHP
<?php
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudHost;
|
|
use App\Models\CloudHostAgent;
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudResourcePool;
|
|
use App\Models\CloudTemplate;
|
|
use App\Models\CloudUserIso;
|
|
use App\Models\User;
|
|
|
|
final class CloudJobService
|
|
{
|
|
private const RECEIPT_DEADLINE_SECONDS = 960;
|
|
private const STORAGE_DISPATCH_DEADLINE_SECONDS = 60;
|
|
private const STORAGE_RECEIPT_DEADLINE_SECONDS = 180;
|
|
private const IMPORT_RECEIPT_DEADLINE_SECONDS = 8100;
|
|
private const BACKUP_RECEIPT_DEADLINE_SECONDS = 90000;
|
|
private const AGGREGATE_FINALIZATION_BUFFER_SECONDS = 300;
|
|
private const AGGREGATE_DEADLINE_SECONDS =
|
|
CloudJob::DISPATCH_DEADLINE_SECONDS
|
|
+ self::RECEIPT_DEADLINE_SECONDS
|
|
+ self::AGGREGATE_FINALIZATION_BUFFER_SECONDS;
|
|
private const IMPORT_AGGREGATE_DEADLINE_SECONDS =
|
|
CloudJob::DISPATCH_DEADLINE_SECONDS
|
|
+ self::IMPORT_RECEIPT_DEADLINE_SECONDS
|
|
+ self::AGGREGATE_FINALIZATION_BUFFER_SECONDS;
|
|
private const LONG_IMPORT_KINDS = [
|
|
'template_import_replica',
|
|
'rescue_catalog_validate_replica',
|
|
'rescue_catalog_import_replica',
|
|
'user_iso_import_replica',
|
|
];
|
|
private const LONG_IMPORT_AGGREGATE_KINDS = [
|
|
'template_import',
|
|
'user_iso_import',
|
|
];
|
|
private const PORTABLE_BACKUP_KINDS = [
|
|
'backup_create',
|
|
'backup_restore',
|
|
'backup_recover',
|
|
'instance_migration_prepare',
|
|
'instance_migration_transfer',
|
|
];
|
|
private const AGGREGATE_KINDS = [
|
|
'template_import',
|
|
'template_metadata_sync',
|
|
'template_delete',
|
|
'user_iso_import',
|
|
'user_iso_delete',
|
|
];
|
|
private const AGGREGATE_CHILD_KIND_MAP = [
|
|
'template_import' => 'template_import_replica',
|
|
'template_metadata_sync' => 'template_metadata_replica',
|
|
'template_delete' => 'template_delete_replica',
|
|
'user_iso_import' => 'user_iso_import_replica',
|
|
'user_iso_delete' => 'user_iso_delete_replica',
|
|
];
|
|
private const RECOVERY_RETRY_SECONDS = 300;
|
|
private const RECOVERY_MESSAGES = [
|
|
'recovery_pending' =>
|
|
'系统正在核对原任务结果;不会重复执行原操作,仍会继续自动核对。',
|
|
'recovery_limit_reached' =>
|
|
'旧任务已重新进入持续自动核对;不会重复执行原操作,系统仍会继续自动核对。',
|
|
'recovery_receipt_limit_reached' =>
|
|
'旧任务回执已重新进入持续自动核对;不会重复执行原操作,系统仍会继续自动核对。',
|
|
'recovery_state_invalid' =>
|
|
'任务状态与恢复记录暂不一致;不会重复执行原操作,修复记录后系统仍会继续自动核对。',
|
|
'recovery_evidence_conflict' =>
|
|
'Agent 回执与宿主机证据暂不一致;资源保持安全锁定,系统仍会继续自动核对。',
|
|
'recovery_evidence_partial' =>
|
|
'宿主机目前只能确认部分操作结果;资源保持安全锁定,系统仍会继续自动核对。',
|
|
'recovery_target_cleanup_pending' =>
|
|
'系统正在自动清理维护迁移目标机残留;原迁移不会重放,清理后会继续核对并修复迁移账本。',
|
|
'recovery_evidence_inconclusive' =>
|
|
'宿主机证据暂不足以确认操作结果;资源保持安全锁定,系统仍会继续自动核对。',
|
|
'recovery_evidence_invalid' =>
|
|
'Agent 返回的核对或栅栏证据格式不匹配;不会采信异常证据,系统仍会继续自动核对。',
|
|
'recovery_receipt_invalid' =>
|
|
'Agent 回执未通过完整性核验;不会采信异常回执,系统仍会继续自动核对。',
|
|
'recovery_completion_outbox_invalid' =>
|
|
'宿主机的完成回执队列状态损坏;不会采信当前结果,修复后系统仍会继续自动核对。',
|
|
'recovery_completion_receipt_missing' =>
|
|
'宿主机的完成回执内容缺失;不会采信当前结果,修复后系统仍会继续自动核对。',
|
|
'recovery_completion_receipt_invalid' =>
|
|
'宿主机的完成回执内容无效;不会采信当前结果,修复后系统仍会继续自动核对。',
|
|
'recovery_completion_operation_missing' =>
|
|
'宿主机的完成回执缺少对应原操作记录;不会采信当前结果,修复后系统仍会继续自动核对。',
|
|
'recovery_completion_operation_invalid' =>
|
|
'宿主机的完成回执对应原操作记录无效;不会采信当前结果,修复后系统仍会继续自动核对。',
|
|
'recovery_completion_identity_mismatch' =>
|
|
'宿主机的完成回执身份与原操作记录不匹配;不会采信当前结果,修复后系统仍会继续自动核对。',
|
|
'recovery_agent_unavailable' =>
|
|
'宿主机 Agent 或网络暂不可用;通信恢复后系统仍会继续自动核对。',
|
|
'recovery_authentication_failed' =>
|
|
'面板与宿主机 Agent 的认证暂未通过;修复认证后系统仍会继续自动核对。',
|
|
'recovery_waiting_connection' =>
|
|
'宿主机 Agent 或网络暂不可用;通信恢复后系统仍会继续自动核对。',
|
|
'recovery_operation_missing' =>
|
|
'宿主机暂未找到原任务记录;不会重复执行原操作,系统仍会继续自动核对。',
|
|
'recovery_operation_conflict' =>
|
|
'任务编号与宿主机记录暂不一致;不会重复执行原操作,系统仍会继续自动核对。',
|
|
'recovery_operation_identity_conflict' =>
|
|
'Agent 证据中的操作、资源或请求指纹与原任务不匹配;不会重复执行或解锁,系统仍会继续自动核对。',
|
|
'recovery_payload_invalid' =>
|
|
'任务恢复数据暂时无法安全读取;修复记录后系统仍会继续自动核对。',
|
|
'recovery_payload_unavailable' =>
|
|
'任务恢复数据暂时无法安全读取;修复记录后系统仍会继续自动核对。',
|
|
'recovery_instance_state_conflict' =>
|
|
'实例状态与任务证据暂不一致;资源保持安全锁定,系统仍会继续自动核对。',
|
|
'recovery_aggregate_state_conflict' =>
|
|
'汇总任务与子任务账本关系异常;资源保持安全锁定,修复账本后系统仍会继续自动核对。',
|
|
'recovery_dispatch_ledger_conflict' =>
|
|
'任务单次派发计数账本异常;资源保持安全锁定,修复账本后系统仍会继续自动核对。',
|
|
'recovery_dispatch_repair_completion_receipt_missing' =>
|
|
'派发计数已修复,但原任务绑定的完成回执缺失;终态保持不变,资源保持锁定并持续恢复。',
|
|
'recovery_dispatch_repair_receipt_invalid' =>
|
|
'派发计数已修复,但原任务回执未通过完整性校验;不会采信异常回执或解锁。',
|
|
'recovery_dispatch_repair_aggregate_state_conflict' =>
|
|
'派发计数已修复,但汇总任务与子任务账本关系仍不一致;资源保持锁定并持续恢复。',
|
|
'recovery_dispatch_repair_payload_unavailable' =>
|
|
'派发计数已修复,但原任务恢复数据暂时无法安全读取;不会重复执行或解锁。',
|
|
'recovery_dispatch_repair_operation_identity_conflict' =>
|
|
'派发计数已修复,但回执中的操作、资源或请求指纹与原任务不匹配;不会重复执行或解锁。',
|
|
'recovery_dispatch_repair_evidence_conflict' =>
|
|
'派发计数已修复,但原任务回执与宿主机证据仍不一致;资源保持锁定并持续恢复。',
|
|
'recovery_dispatch_repair_instance_state_conflict' =>
|
|
'派发计数已修复,但实例状态与原任务证据仍不一致;资源保持锁定并持续恢复。',
|
|
'recovery_dispatch_repair_ledger_conflict' =>
|
|
'派发计数已修复,但面板业务账本仍未安全收敛;终态保持不变,资源保持锁定。',
|
|
'recovery_dispatch_repair_internal_error' =>
|
|
'派发计数修复后的安全校验遇到内部异常;不会重复执行或解锁,系统会持续恢复。',
|
|
'recovery_child_convergence' =>
|
|
'子任务仍在持续自动核对;汇总任务会在子任务收敛后更新,原操作不会重复执行。',
|
|
'recovery_ledger_conflict' =>
|
|
'任务结果已确定,但面板账本暂未安全收敛;资源保持安全锁定,系统仍会继续自动核对并恢复。',
|
|
'recovery_internal_error' =>
|
|
'自动核对暂时遇到内部异常;不会重复执行原操作,系统仍会继续自动核对。',
|
|
];
|
|
|
|
/** Absolute dispatch windows are selected once and are never extended. */
|
|
public static function dispatchDeadlineSeconds(string $kind): int
|
|
{
|
|
return $kind === 'host_storage_configure'
|
|
? self::STORAGE_DISPATCH_DEADLINE_SECONDS
|
|
: CloudJob::DISPATCH_DEADLINE_SECONDS;
|
|
}
|
|
|
|
public static function effectiveDispatchDeadline(CloudJob $job): int
|
|
{
|
|
$stored = (int) ($job->dispatch_deadline_at ?? 0);
|
|
if ($stored < 1 || (string) $job->kind !== 'host_storage_configure') {
|
|
return $stored;
|
|
}
|
|
$createdAt = (int) ($job->created_at ?? 0);
|
|
return $createdAt < 1 ? $stored : min(
|
|
$stored,
|
|
$createdAt + self::STORAGE_DISPATCH_DEADLINE_SECONDS
|
|
);
|
|
}
|
|
|
|
/** Absolute receipt windows are selected once and are never extended. */
|
|
public static function receiptDeadlineSeconds(string $kind): int
|
|
{
|
|
if (in_array($kind, self::PORTABLE_BACKUP_KINDS, true)) {
|
|
return self::BACKUP_RECEIPT_DEADLINE_SECONDS;
|
|
}
|
|
if (in_array($kind, self::LONG_IMPORT_KINDS, true)) {
|
|
return self::IMPORT_RECEIPT_DEADLINE_SECONDS;
|
|
}
|
|
return $kind === 'host_storage_configure'
|
|
? self::STORAGE_RECEIPT_DEADLINE_SECONDS
|
|
: self::RECEIPT_DEADLINE_SECONDS;
|
|
}
|
|
|
|
/** Cloud-snapshot creates download a backup and need the restore window. */
|
|
public static function receiptDeadlineSecondsForJob(CloudJob $job): int
|
|
{
|
|
if ((string) $job->kind === 'create' && $job->backup_id !== null) {
|
|
return self::BACKUP_RECEIPT_DEADLINE_SECONDS;
|
|
}
|
|
return self::receiptDeadlineSeconds((string) $job->kind);
|
|
}
|
|
|
|
/**
|
|
* Apply the current bounded window to persisted work as well as new work.
|
|
* This lets a deployment shorten an old storage task's 960-second value
|
|
* without rewriting history or extending any previously stored deadline.
|
|
*/
|
|
public static function effectiveReceiptDeadline(CloudJob $job): int
|
|
{
|
|
$stored = (int) ($job->receipt_deadline_at ?? 0);
|
|
if ($stored < 1 || (string) $job->kind !== 'host_storage_configure') {
|
|
return $stored;
|
|
}
|
|
$dispatchedAt = (int) ($job->dispatched_at ?? 0);
|
|
return $dispatchedAt < 1 ? $stored : min(
|
|
$stored,
|
|
$dispatchedAt + self::STORAGE_RECEIPT_DEADLINE_SECONDS
|
|
);
|
|
}
|
|
|
|
public static function receiptDeadlineReached(CloudJob $job,
|
|
?int $now = null): bool
|
|
{
|
|
$deadline = self::effectiveReceiptDeadline($job);
|
|
return $deadline > 0 && $deadline <= ($now ?? time());
|
|
}
|
|
|
|
public static function activeDeadlineReached(CloudJob $job,
|
|
?int $now = null): bool
|
|
{
|
|
$now ??= time();
|
|
if ((string) $job->phase === 'accepted') {
|
|
$deadline = self::effectiveDispatchDeadline($job);
|
|
return $deadline > 0 && $deadline <= $now;
|
|
}
|
|
return in_array((string) $job->phase,
|
|
['dispatching', 'awaiting_receipt'], true)
|
|
&& self::receiptDeadlineReached($job, $now);
|
|
}
|
|
|
|
public static function aggregateDeadlineSeconds(string $kind): int
|
|
{
|
|
return in_array($kind, self::LONG_IMPORT_AGGREGATE_KINDS, true)
|
|
? self::IMPORT_AGGREGATE_DEADLINE_SECONDS
|
|
: self::AGGREGATE_DEADLINE_SECONDS;
|
|
}
|
|
|
|
/** Aggregate parents are local coordination rows and are never dispatched. */
|
|
public static function isAggregateKind(string $kind): bool
|
|
{
|
|
return in_array($kind, self::AGGREGATE_KINDS, true);
|
|
}
|
|
|
|
/** @return list<string> */
|
|
public static function aggregateKinds(): array
|
|
{
|
|
return self::AGGREGATE_KINDS;
|
|
}
|
|
|
|
/** True only for a damaged terminal recovery or aggregate-local tuple. */
|
|
public static function hasTerminalRecoveryStateConflict(object $job): bool
|
|
{
|
|
if ((string) ($job->phase ?? '') !== 'terminal') {
|
|
return false;
|
|
}
|
|
$mode = (string) ($job->execution_mode ?? '');
|
|
$kind = (string) ($job->kind ?? '');
|
|
if (CloudOperationRecoveryPolicy::supportsLedgerFinalizer($kind)) {
|
|
if ($mode !== 'agent'
|
|
|| self::hasAgentRecoveryEnvelopeConflict($job)) {
|
|
return true;
|
|
}
|
|
if (self::hasResolvedTerminalStateConflict($job)) {
|
|
return true;
|
|
}
|
|
}
|
|
if (self::isAggregateKind($kind)) {
|
|
if (self::hasAggregateLocalStateConflict($job)) {
|
|
return true;
|
|
}
|
|
if ((string) ($job->effect_resolution ?? '') !== 'unknown') {
|
|
return false;
|
|
}
|
|
} elseif ((string) ($job->effect_resolution ?? '') !== 'unknown') {
|
|
return false;
|
|
}
|
|
$supported = ($mode === 'agent'
|
|
&& !CloudDispatchLedger::isInvalid($job)
|
|
&& CloudOperationRecoveryPolicy::supportsLedgerFinalizer($kind))
|
|
|| ($mode === 'aggregate'
|
|
&& ($job->parent_job_id ?? null) === null
|
|
&& self::isAggregateKind($kind));
|
|
if (!$supported) {
|
|
return false;
|
|
}
|
|
// Generation zero proves the mutation never crossed the Agent
|
|
// boundary. A terminal/unknown row may remain there only while a
|
|
// durable PHP-only finalizer incident is retrying; without that
|
|
// incident the otherwise-canonical dispatched safety tuple is itself
|
|
// contradictory and must stay in the local invalid-state queue.
|
|
if ($mode === 'agent'
|
|
&& (int) ($job->dispatch_attempts ?? -1) === 0
|
|
&& !(bool) ($job->finalization_incident ?? false)) {
|
|
return true;
|
|
}
|
|
return (string) ($job->outcome ?? '') !== 'failed'
|
|
|| (string) ($job->status ?? '') !== 'failed'
|
|
|| !(bool) ($job->effect_may_have_applied ?? false)
|
|
|| (bool) ($job->safe_to_resubmit ?? false);
|
|
}
|
|
|
|
/** Aggregate parents are PHP-local ledgers and must never resemble Agent work. */
|
|
public static function hasAggregateLocalStateConflict(object $job): bool
|
|
{
|
|
if ((string) ($job->execution_mode ?? '') !== 'aggregate'
|
|
|| !self::isAggregateKind((string) ($job->kind ?? ''))
|
|
|| ($job->parent_job_id ?? null) !== null
|
|
|| ($job->host_id ?? null) !== null
|
|
|| ($job->instance_id ?? null) !== null
|
|
|| ($job->order_id ?? null) !== null
|
|
|| ($job->invoice_id ?? null) !== null
|
|
|| ($job->snapshot_id ?? null) !== null
|
|
|| (int) ($job->user_id ?? 0) < 1
|
|
|| (string) ($job->attempts ?? '') !== '0'
|
|
|| CloudDispatchLedger::attempts($job) !== 0
|
|
|| (int) ($job->dispatch_deadline_at ?? 0) < 1
|
|
|| (int) ($job->receipt_deadline_at ?? 0) < 1
|
|
|| ($job->dispatch_started_at ?? null) !== null
|
|
|| ($job->agent_accepted_at ?? null) !== null
|
|
|| ($job->dispatched_at ?? null) !== null
|
|
|| ($job->completion_event_id ?? null) !== null
|
|
|| preg_match(
|
|
'/^[a-f0-9]{32}$/D',
|
|
(string) ($job->operation_id ?? '')
|
|
) !== 1
|
|
|| preg_match(
|
|
'/^[a-f0-9]{64}$/D',
|
|
(string) ($job->request_fingerprint ?? '')
|
|
) !== 1
|
|
|| preg_match(
|
|
'/^[A-Za-z0-9_.-]{16,128}$/D',
|
|
(string) ($job->request_idempotency_key ?? '')
|
|
) !== 1) {
|
|
return true;
|
|
}
|
|
$phase = (string) ($job->phase ?? '');
|
|
if ($phase === 'awaiting_receipt') {
|
|
return ($job->outcome ?? null) !== null
|
|
|| (string) ($job->status ?? '') !== 'running'
|
|
|| ($job->terminal_at ?? null) !== null
|
|
|| ($job->effect_may_have_applied ?? null) !== null
|
|
|| ($job->safe_to_resubmit ?? null) !== null
|
|
|| (string) ($job->effect_resolution ?? '')
|
|
!== 'not_applicable'
|
|
|| (bool) ($job->finalization_incident ?? false);
|
|
}
|
|
if ($phase !== 'terminal'
|
|
|| (int) ($job->terminal_at ?? 0) < 1
|
|
|| !in_array(
|
|
(string) ($job->outcome ?? ''),
|
|
['succeeded', 'failed'],
|
|
true
|
|
)
|
|
|| (string) ($job->status ?? '')
|
|
!== (string) ($job->outcome ?? '')) {
|
|
return true;
|
|
}
|
|
$outcome = (string) $job->outcome;
|
|
$resolution = (string) ($job->effect_resolution ?? '');
|
|
$effectMayHaveApplied = $job->effect_may_have_applied ?? null;
|
|
$safeToResubmit = $job->safe_to_resubmit ?? null;
|
|
if ($outcome === 'succeeded') {
|
|
return !in_array(
|
|
$effectMayHaveApplied,
|
|
[0, '0', false],
|
|
true
|
|
) || !in_array($safeToResubmit, [0, '0', false], true)
|
|
|| $resolution !== 'applied'
|
|
|| ($job->error_code ?? null) !== null;
|
|
}
|
|
if (!in_array($resolution, [
|
|
'unknown', 'applied', 'not_applied',
|
|
], true)) {
|
|
return true;
|
|
}
|
|
if (in_array($effectMayHaveApplied, [1, '1', true], true)) {
|
|
return !in_array($safeToResubmit, [0, '0', false], true);
|
|
}
|
|
return !in_array($effectMayHaveApplied, [0, '0', false], true)
|
|
|| !in_array(
|
|
$safeToResubmit,
|
|
[0, '0', false, 1, '1', true],
|
|
true
|
|
)
|
|
|| $resolution !== 'not_applied';
|
|
}
|
|
|
|
/** A published terminal effect must retain one canonical immutable tuple. */
|
|
public static function hasResolvedTerminalStateConflict(object $job): bool
|
|
{
|
|
if ((string) ($job->phase ?? '') !== 'terminal'
|
|
|| !in_array(
|
|
(string) ($job->effect_resolution ?? ''),
|
|
['applied', 'not_applied'],
|
|
true
|
|
)) {
|
|
return false;
|
|
}
|
|
if ((int) ($job->terminal_at ?? 0) < 1) {
|
|
return true;
|
|
}
|
|
$outcome = (string) ($job->outcome ?? '');
|
|
$status = (string) ($job->status ?? '');
|
|
$resolution = (string) $job->effect_resolution;
|
|
$effect = $job->effect_may_have_applied ?? null;
|
|
$safe = $job->safe_to_resubmit ?? null;
|
|
if ($outcome === 'succeeded') {
|
|
$tupleConflict = $status !== 'succeeded'
|
|
|| $resolution !== 'applied'
|
|
|| !in_array($effect, [0, '0', false], true)
|
|
|| !in_array($safe, [0, '0', false], true)
|
|
|| ($job->error_code ?? null) !== null;
|
|
if ($tupleConflict) {
|
|
return true;
|
|
}
|
|
} elseif ($outcome !== 'failed' || $status !== 'failed') {
|
|
return true;
|
|
} elseif (in_array($effect, [1, '1', true], true)) {
|
|
if (!in_array($safe, [0, '0', false], true)) {
|
|
return true;
|
|
}
|
|
} elseif (!in_array($effect, [0, '0', false], true)
|
|
|| !in_array($safe, [0, '0', false, 1, '1', true], true)
|
|
|| $resolution !== 'not_applied') {
|
|
return true;
|
|
}
|
|
|
|
$kind = (string) ($job->kind ?? '');
|
|
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer($kind)) {
|
|
return false;
|
|
}
|
|
return self::hasAgentRecoveryEnvelopeConflict($job);
|
|
}
|
|
|
|
/** True when one supported Agent generation's immutable envelope is bad. */
|
|
public static function hasAgentRecoveryEnvelopeConflict(object $job): bool
|
|
{
|
|
$kind = (string) ($job->kind ?? '');
|
|
$outcome = (string) ($job->outcome ?? '');
|
|
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer($kind)) {
|
|
return false;
|
|
}
|
|
if ((string) ($job->execution_mode ?? '') !== 'agent'
|
|
|| self::hasAgentRecoveryIdentityConflict($job)
|
|
|| (int) ($job->host_id ?? 0) < 1
|
|
|| (int) ($job->dispatch_deadline_at ?? 0) < 1) {
|
|
return true;
|
|
}
|
|
$dispatchAttempts = CloudDispatchLedger::attempts($job);
|
|
if ($dispatchAttempts === 1) {
|
|
return (string) ($job->attempts ?? '') !== '1'
|
|
|| (int) ($job->dispatch_started_at ?? 0) < 1
|
|
|| (($job->agent_accepted_at ?? null) !== null
|
|
&& (int) $job->agent_accepted_at < 1)
|
|
|| (int) ($job->dispatched_at ?? 0) < 1
|
|
|| (int) ($job->receipt_deadline_at ?? 0) < 1;
|
|
}
|
|
if ($dispatchAttempts !== 0
|
|
|| (string) ($job->attempts ?? '') !== '0'
|
|
|| ($job->dispatch_started_at ?? null) !== null
|
|
|| ($job->agent_accepted_at ?? null) !== null
|
|
|| ($job->dispatched_at ?? null) !== null
|
|
|| ($job->receipt_deadline_at ?? null) !== null
|
|
|| ($job->completion_event_id ?? null) !== null) {
|
|
return true;
|
|
}
|
|
if ($outcome !== 'succeeded') {
|
|
return false;
|
|
}
|
|
return !self::isValidUndispatchedFirewallNoop($job);
|
|
}
|
|
|
|
/** True when one supported Agent generation lost immutable request identity. */
|
|
public static function hasAgentRecoveryIdentityConflict(object $job): bool
|
|
{
|
|
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
|
|
(string) ($job->kind ?? '')
|
|
)) {
|
|
return false;
|
|
}
|
|
return preg_match(
|
|
'/^[a-f0-9]{32}$/D',
|
|
(string) ($job->operation_id ?? '')
|
|
) !== 1 || preg_match(
|
|
'/^[a-f0-9]{64}$/D',
|
|
(string) ($job->request_fingerprint ?? '')
|
|
) !== 1 || preg_match(
|
|
'/^[A-Za-z0-9_.-]{16,128}$/D',
|
|
(string) ($job->request_idempotency_key ?? '')
|
|
) !== 1;
|
|
}
|
|
|
|
/** Generation zero success exists only for an unchanged firewall policy. */
|
|
private static function isValidUndispatchedFirewallNoop(object $job): bool
|
|
{
|
|
if ((string) ($job->kind ?? '') !== 'firewall_apply'
|
|
|| (int) ($job->instance_id ?? 0) < 1
|
|
|| !is_string($job->result_json ?? null)) {
|
|
return false;
|
|
}
|
|
$result = json_decode((string) $job->result_json, true);
|
|
return is_array($result)
|
|
&& !array_is_list($result)
|
|
&& ($result['unchanged'] ?? null) === true
|
|
&& is_string($result['policy_digest'] ?? null)
|
|
&& preg_match(
|
|
'/^[a-f0-9]{64}$/D',
|
|
$result['policy_digest']
|
|
) === 1
|
|
&& is_int($result['revision'] ?? null)
|
|
&& $result['revision'] > 0;
|
|
}
|
|
/**
|
|
* Operations that still own an instance resource. Active work blocks new
|
|
* mutations; a terminal unknown-effect failure remains fenced until a
|
|
* kind-specific, audited ledger finalizer has safely converged it.
|
|
*/
|
|
public static function blockingQueryForInstance(int $instanceId)
|
|
{
|
|
return self::blockingQueryForInstances([$instanceId]);
|
|
}
|
|
|
|
/** @param list<int> $instanceIds */
|
|
public static function blockingQueryForInstances(array $instanceIds)
|
|
{
|
|
$query = CloudJob::whereIn('instance_id', $instanceIds);
|
|
self::applyBlockingInstanceState($query);
|
|
return $query;
|
|
}
|
|
|
|
/** @param list<int> $orderIds */
|
|
public static function blockingCreateQueryForOrders(array $orderIds)
|
|
{
|
|
$query = CloudJob::whereIn('order_id', $orderIds)
|
|
->where('kind', 'create');
|
|
self::applyBlockingInstanceState($query);
|
|
return $query;
|
|
}
|
|
|
|
/**
|
|
* Serialize host-global VPC route mutations before Agent admission.
|
|
* Independent VM-port jobs remain concurrent. A terminal route with an
|
|
* unresolved effect or finalization incident keeps the host lane fenced.
|
|
*/
|
|
public static function applyVpcHostDispatchFence($query)
|
|
{
|
|
$routeKinds = ['vpc_host_apply', 'vpc_host_remove'];
|
|
return $query->where(static function ($eligible) use (
|
|
$routeKinds
|
|
): void {
|
|
$eligible->whereNotIn('cloud_jobs.kind', $routeKinds)
|
|
->orWhereNotExists(static function ($predecessors) use (
|
|
$routeKinds
|
|
): void {
|
|
$predecessors->selectRaw('1')
|
|
->from('cloud_jobs as vpc_route_predecessors')
|
|
->whereColumn(
|
|
'vpc_route_predecessors.host_id',
|
|
'cloud_jobs.host_id'
|
|
)
|
|
->whereColumn(
|
|
'vpc_route_predecessors.id',
|
|
'<',
|
|
'cloud_jobs.id'
|
|
)
|
|
->whereIn(
|
|
'vpc_route_predecessors.kind',
|
|
$routeKinds
|
|
)
|
|
->where(static function ($outsideRepair): void {
|
|
$outsideRepair->whereNull(
|
|
'cloud_jobs.vpc_repair_root_job_id'
|
|
)->orWhereRaw(
|
|
'COALESCE('
|
|
. 'vpc_route_predecessors.vpc_repair_root_job_id, '
|
|
. 'vpc_route_predecessors.id) <> '
|
|
. 'cloud_jobs.vpc_repair_root_job_id'
|
|
);
|
|
})
|
|
->where(static function ($blocking): void {
|
|
$blocking->where(
|
|
'vpc_route_predecessors.phase',
|
|
'<>',
|
|
'terminal'
|
|
)->orWhere(static function ($terminal): void {
|
|
$terminal->where(
|
|
'vpc_route_predecessors.phase',
|
|
'terminal'
|
|
)->where(static function ($unresolved): void {
|
|
$unresolved->where(
|
|
'vpc_route_predecessors.effect_resolution',
|
|
'unknown'
|
|
)->orWhere(
|
|
'vpc_route_predecessors.finalization_incident',
|
|
1
|
|
);
|
|
});
|
|
});
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* A stock-mode switch cannot cross an upgrade that still owns either the
|
|
* source offer or the target offer. The offer row is locked by the caller,
|
|
* so enqueue either commits its job first and is observed here, or reads
|
|
* the administrator's new mode as its immutable source snapshot.
|
|
*/
|
|
public static function blockingUpgradeQueryForOffer(int $offerId)
|
|
{
|
|
$query = CloudJob::where('kind', 'instance_upgrade')
|
|
->where(static function ($linked) use ($offerId): void {
|
|
$linked->whereExists(static function ($instances) use ($offerId): void {
|
|
$instances->selectRaw('1')->from('cloud_instances')
|
|
->whereColumn('cloud_instances.id', 'cloud_jobs.instance_id')
|
|
->where('cloud_instances.offer_id', $offerId);
|
|
})->orWhereExists(static function ($invoices) use ($offerId): void {
|
|
$invoices->selectRaw('1')->from('cloud_invoices')
|
|
->whereColumn('cloud_invoices.id', 'cloud_jobs.invoice_id')
|
|
->where('cloud_invoices.purpose', 'upgrade')
|
|
->where('cloud_invoices.offer_id', $offerId);
|
|
});
|
|
});
|
|
self::applyBlockingInstanceState($query);
|
|
return $query;
|
|
}
|
|
|
|
private static function applyBlockingInstanceState($query): void
|
|
{
|
|
$query->where(static function ($query): void {
|
|
$query->where('phase', '<>', 'terminal')
|
|
->orWhere(static function ($terminal): void {
|
|
$terminal->where('phase', 'terminal')
|
|
->where('outcome', 'failed')
|
|
->where(static function ($blocked): void {
|
|
$blocked->where('error_code', 'force_required')
|
|
->orWhere(static function ($unknown): void {
|
|
$unknown->where('effect_resolution', 'unknown')
|
|
->where(static function ($effect): void {
|
|
$effect->where('effect_may_have_applied', 1)
|
|
->orWhere('safe_to_resubmit', 0);
|
|
});
|
|
});
|
|
});
|
|
})->orWhere(static function ($mode): void {
|
|
$mode->whereIn(
|
|
'kind',
|
|
CloudOperationRecoveryPolicy::automaticKinds()
|
|
)->where('phase', 'terminal')
|
|
->where(static function ($invalid): void {
|
|
$invalid->whereNull('execution_mode')
|
|
->orWhere(
|
|
'execution_mode',
|
|
'<>',
|
|
'agent'
|
|
);
|
|
});
|
|
})->orWhere(static function ($invalid): void {
|
|
// Historical rows written by a fail-open release must
|
|
// become blocking again immediately after upgrade,
|
|
// even before a recovery worker has touched them.
|
|
$invalid->where('execution_mode', 'agent')
|
|
->where(static function ($counter): void {
|
|
$counter->whereNull('dispatch_attempts')
|
|
->orWhereNotIn(
|
|
'dispatch_attempts',
|
|
[0, 1]
|
|
);
|
|
});
|
|
})->orWhere(static function ($incident): void {
|
|
$incident->where('execution_mode', 'agent')
|
|
->where('phase', 'terminal')
|
|
->where('finalization_incident', 1)
|
|
->where(static function ($recoverable): void {
|
|
$recoverable->whereIn(
|
|
'finalization_error_code',
|
|
CloudDispatchLedger::repairIncidentCodes()
|
|
)->orWhere(static function ($local): void {
|
|
$local->where('outcome', 'failed')
|
|
->where('status', 'failed')
|
|
->where('dispatch_attempts', 0)
|
|
->whereIn(
|
|
'kind',
|
|
CloudOperationRecoveryPolicy::automaticKinds()
|
|
);
|
|
})->orWhere(static function ($resolved): void {
|
|
$resolved->whereIn(
|
|
'dispatch_attempts',
|
|
[0, 1]
|
|
)
|
|
->whereIn(
|
|
'effect_resolution',
|
|
['applied', 'not_applied']
|
|
)->whereIn(
|
|
'kind',
|
|
CloudOperationRecoveryPolicy::automaticKinds()
|
|
);
|
|
});
|
|
});
|
|
})->orWhere(static function ($state): void {
|
|
$state->where('execution_mode', 'agent')
|
|
->whereIn(
|
|
'kind',
|
|
CloudOperationRecoveryPolicy::automaticKinds()
|
|
)->where('phase', 'terminal')
|
|
->whereIn('dispatch_attempts', [0, 1])
|
|
->where('effect_resolution', 'unknown')
|
|
->where(static function ($invalid): void {
|
|
self::applyInvalidTerminalTuple($invalid);
|
|
});
|
|
})->orWhere(static function ($resolvedState): void {
|
|
$resolvedState->whereIn(
|
|
'kind',
|
|
CloudOperationRecoveryPolicy::automaticKinds()
|
|
)->where('phase', 'terminal')
|
|
->where(static function ($invalid): void {
|
|
self::applyInvalidResolvedTerminalTuple(
|
|
$invalid
|
|
);
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
public static function blockingQueryForInvoice(int $invoiceId)
|
|
{
|
|
return CloudJob::where('invoice_id', $invoiceId)
|
|
->where('kind', 'payment_create')
|
|
->where(static function ($query): void {
|
|
$query->where('phase', '<>', 'terminal')
|
|
->orWhere(static function ($terminal): void {
|
|
$terminal->where('phase', 'terminal')
|
|
->where('outcome', 'failed')
|
|
->where('effect_resolution', 'unknown')
|
|
->where('safe_to_resubmit', 0);
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Reserve the one public task that represents a multi-host mutation.
|
|
*
|
|
* Callers must invoke this while holding their catalog authority row (or
|
|
* before creating that row) inside the same database transaction. The
|
|
* aggregate is never claimed by a worker and can only be completed by the
|
|
* terminal transition of its persisted child jobs.
|
|
*
|
|
* @return array{0: CloudJob, 1: bool} job and whether it was newly created
|
|
*/
|
|
public static function reserveAggregate(
|
|
array $actor,
|
|
string $kind,
|
|
array $canonicalRequest
|
|
): array {
|
|
$userId = (int) ($actor['user_id'] ?? 0);
|
|
if ($userId < 1 || !self::isAggregateKind($kind)) {
|
|
throw new CloudError('任务请求无效', 400);
|
|
}
|
|
$fingerprint = hash('sha256', json_encode(
|
|
self::canonicalRequestValue([
|
|
'kind' => $kind,
|
|
'request' => $canonicalRequest,
|
|
]), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
));
|
|
$requestKey = self::requestKey($actor);
|
|
if ($requestKey !== null) {
|
|
$prior = CloudJob::where('user_id', $userId)
|
|
->where('request_idempotency_key', $requestKey)
|
|
->lockForUpdate()->first();
|
|
if ($prior !== null) {
|
|
if (!hash_equals((string) $prior->request_fingerprint,
|
|
$fingerprint)) {
|
|
throw new CloudError(
|
|
'同一请求标识不能用于不同的任务', 409
|
|
);
|
|
}
|
|
return [$prior, false];
|
|
}
|
|
}
|
|
|
|
$job = new CloudJob();
|
|
$job->operation_id = bin2hex(random_bytes(16));
|
|
$job->request_idempotency_key = $requestKey
|
|
?? (string) $job->operation_id;
|
|
$job->request_fingerprint = $fingerprint;
|
|
$job->user_id = $userId;
|
|
$job->host_id = null;
|
|
$job->kind = $kind;
|
|
$job->execution_mode = 'aggregate';
|
|
$job->sealed_payload = CloudSecrets::encrypt([
|
|
'aggregate' => true,
|
|
], 'job-payload');
|
|
$now = time();
|
|
$job->receipt_deadline_at = $now
|
|
+ self::aggregateDeadlineSeconds($kind);
|
|
$job->available_at = $job->created_at = $job->updated_at = $now;
|
|
$job->save();
|
|
return [$job, true];
|
|
}
|
|
|
|
/**
|
|
* Reserve one public, host-targeted mutation with a plaintext canonical
|
|
* fingerprint. Replaying the same HTTP idempotency key returns the first
|
|
* CloudJob; it never creates another Agent operation.
|
|
*
|
|
* @return array{0: CloudJob, 1: bool}
|
|
*/
|
|
public static function reserveStandaloneAgentMutation(
|
|
array $actor,
|
|
int $hostId,
|
|
string $kind,
|
|
string $method,
|
|
string $target,
|
|
array $body,
|
|
array $metadata,
|
|
array $canonicalRequest,
|
|
?string $operationId = null,
|
|
?string $idempotencyTarget = null
|
|
): array {
|
|
$userId = (int) ($actor['user_id'] ?? 0);
|
|
$method = strtoupper($method);
|
|
if ($userId < 1 || $hostId < 1
|
|
|| !array_key_exists($kind, CloudOperationCatalog::kinds())
|
|
|| !in_array($method, ['POST', 'PUT', 'PATCH', 'DELETE'], true)
|
|
|| !str_starts_with($target, '/v1/')) {
|
|
throw new CloudError('Agent 任务请求无效', 400);
|
|
}
|
|
$fingerprint = self::standaloneRequestFingerprint(
|
|
$hostId,
|
|
$kind,
|
|
$method,
|
|
$idempotencyTarget ?? $target,
|
|
$canonicalRequest
|
|
);
|
|
$requestKey = self::requestKey($actor);
|
|
if ($requestKey !== null) {
|
|
$prior = CloudJob::where('user_id', $userId)
|
|
->where('request_idempotency_key', $requestKey)
|
|
->lockForUpdate()->first();
|
|
if ($prior !== null) {
|
|
if (!hash_equals((string) $prior->request_fingerprint,
|
|
$fingerprint)) {
|
|
throw new CloudError(
|
|
'同一请求标识不能用于不同的任务', 409
|
|
);
|
|
}
|
|
return [$prior, false];
|
|
}
|
|
}
|
|
$operationId ??= bin2hex(random_bytes(16));
|
|
if (preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1) {
|
|
throw new CloudError('Agent 任务操作标识无效', 400);
|
|
}
|
|
$body['operation_id'] = $operationId;
|
|
$job = new CloudJob();
|
|
$job->operation_id = $operationId;
|
|
$job->request_idempotency_key = $requestKey ?? $operationId;
|
|
$job->request_fingerprint = $fingerprint;
|
|
$job->user_id = $userId;
|
|
$job->host_id = $hostId;
|
|
$job->kind = $kind;
|
|
$job->execution_mode = 'agent';
|
|
$job->sealed_payload = CloudSecrets::encrypt(array_merge($metadata, [
|
|
'method' => $method,
|
|
'request_target' => $target,
|
|
'body' => $body,
|
|
'owner_id' => (int) ($metadata['owner_id'] ?? $userId),
|
|
'can_manage' => (bool) ($metadata['can_manage'] ?? false),
|
|
]), 'job-payload');
|
|
$now = time();
|
|
$job->dispatch_deadline_at = $now
|
|
+ self::dispatchDeadlineSeconds($kind);
|
|
$job->available_at = $job->created_at = $job->updated_at = $now;
|
|
$job->save();
|
|
return [$job, true];
|
|
}
|
|
|
|
/** Exact public idempotency fingerprint used before external preparation. */
|
|
public static function standaloneRequestFingerprint(
|
|
int $hostId,
|
|
string $kind,
|
|
string $method,
|
|
string $target,
|
|
array $canonicalRequest
|
|
): string {
|
|
$method = strtoupper($method);
|
|
if ($hostId < 1
|
|
|| !array_key_exists($kind, CloudOperationCatalog::kinds())
|
|
|| !in_array($method, ['POST', 'PUT', 'PATCH', 'DELETE'], true)
|
|
|| !str_starts_with($target, '/v1/')) {
|
|
throw new CloudError('Agent 任务请求无效', 400);
|
|
}
|
|
return hash('sha256', json_encode(
|
|
self::canonicalRequestValue([
|
|
'host_id' => $hostId,
|
|
'kind' => $kind,
|
|
'method' => $method,
|
|
'target' => $target,
|
|
'request' => $canonicalRequest,
|
|
]), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
));
|
|
}
|
|
|
|
/** Persist one exact Agent mutation. It can be claimed only once. */
|
|
public static function reserveAgentMutation(
|
|
array $actor,
|
|
int $hostId,
|
|
string $kind,
|
|
string $method,
|
|
string $target,
|
|
array $body,
|
|
array $metadata = [],
|
|
?int $parentJobId = null,
|
|
?string $operationId = null
|
|
): CloudJob {
|
|
$userId = (int) ($actor['user_id'] ?? 0);
|
|
$method = strtoupper($method);
|
|
if ($userId < 1 || $hostId < 1
|
|
|| !array_key_exists($kind, CloudOperationCatalog::kinds())
|
|
|| !in_array($method, ['POST', 'PUT', 'PATCH', 'DELETE'], true)
|
|
|| !str_starts_with($target, '/v1/')) {
|
|
throw new CloudError('Agent 任务请求无效', 400);
|
|
}
|
|
$operationId ??= bin2hex(random_bytes(16));
|
|
if (preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1) {
|
|
throw new CloudError('Agent 任务操作标识无效', 400);
|
|
}
|
|
$body['operation_id'] = $operationId;
|
|
$fingerprint = hash('sha256', json_encode(
|
|
self::canonicalRequestValue([
|
|
'host_id' => $hostId,
|
|
'kind' => $kind,
|
|
'method' => $method,
|
|
'target' => $target,
|
|
'body' => $body,
|
|
'metadata' => $metadata,
|
|
]), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
));
|
|
$job = new CloudJob();
|
|
$job->operation_id = $operationId;
|
|
$job->request_idempotency_key = $operationId;
|
|
$job->request_fingerprint = $fingerprint;
|
|
$job->parent_job_id = $parentJobId;
|
|
$job->user_id = $userId;
|
|
$job->host_id = $hostId;
|
|
$job->kind = $kind;
|
|
$job->execution_mode = 'agent';
|
|
$job->sealed_payload = CloudSecrets::encrypt(array_merge($metadata, [
|
|
'method' => $method,
|
|
'request_target' => $target,
|
|
'body' => $body,
|
|
'owner_id' => (int) ($metadata['owner_id'] ?? $userId),
|
|
'can_manage' => (bool) ($metadata['can_manage'] ?? false),
|
|
]), 'job-payload');
|
|
$now = time();
|
|
$job->dispatch_deadline_at = $now
|
|
+ self::dispatchDeadlineSeconds($kind);
|
|
$job->available_at = $job->created_at = $job->updated_at = $now;
|
|
$job->save();
|
|
return $job;
|
|
}
|
|
|
|
/** Strict HTTP acknowledgement used by every asynchronous mutation. */
|
|
public static function acknowledgement(CloudJob $job): array
|
|
{
|
|
return [
|
|
'task_id' => (string) $job->id,
|
|
'status_url' => '/api/v1/tasks/' . (int) $job->id,
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Record a proven no-op as a successful public task without ever making it
|
|
* claimable by a worker. The row is first persisted with the normal task
|
|
* invariants, then terminalized in the caller's database transaction.
|
|
*/
|
|
public static function completeWithoutDispatchLocked(
|
|
CloudJob $job,
|
|
array $result
|
|
): void {
|
|
if (!$job->exists
|
|
|| (string) $job->phase !== 'accepted'
|
|
|| (int) $job->dispatch_attempts !== 0
|
|
|| (int) $job->attempts !== 0
|
|
|| $job->dispatch_started_at !== null
|
|
|| $job->dispatched_at !== null
|
|
|| $job->agent_accepted_at !== null
|
|
|| $job->receipt_deadline_at !== null) {
|
|
throw new \LogicException(
|
|
'Only a fresh, undispatched task can complete as a no-op'
|
|
);
|
|
}
|
|
self::completeAggregateLocked(
|
|
$job,
|
|
'succeeded',
|
|
false,
|
|
false,
|
|
'',
|
|
$result
|
|
);
|
|
}
|
|
|
|
/** Complete an aggregate without dispatching another Agent mutation. */
|
|
public static function completeAggregateLocked(
|
|
CloudJob $job,
|
|
string $outcome,
|
|
bool $effectMayHaveApplied,
|
|
bool $safeToResubmit,
|
|
string $errorCode = '',
|
|
?array $result = null
|
|
): void {
|
|
if ((string) $job->phase === 'terminal') {
|
|
return;
|
|
}
|
|
if (!in_array($outcome, ['succeeded', 'failed'], true)) {
|
|
throw new \InvalidArgumentException('Invalid aggregate outcome');
|
|
}
|
|
if ($outcome !== 'failed') {
|
|
$effectMayHaveApplied = false;
|
|
$safeToResubmit = false;
|
|
$errorCode = '';
|
|
} elseif ($effectMayHaveApplied) {
|
|
$safeToResubmit = false;
|
|
}
|
|
$now = time();
|
|
$job->phase = 'terminal';
|
|
$job->outcome = $outcome;
|
|
$job->status = $outcome;
|
|
$job->terminal_at = $now;
|
|
$job->effect_may_have_applied = $effectMayHaveApplied ? 1 : 0;
|
|
$job->safe_to_resubmit = $safeToResubmit ? 1 : 0;
|
|
$job->effect_resolution = $outcome === 'succeeded' ? 'applied'
|
|
: ($effectMayHaveApplied ? 'unknown' : 'not_applied');
|
|
$job->error_code = $errorCode === '' ? null
|
|
: substr($errorCode, 0, 128);
|
|
$job->result_json = $result === null ? null : json_encode(
|
|
$result,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
);
|
|
$job->sealed_payload = CloudSecrets::encrypt([], 'job-payload');
|
|
$job->lease_token = null;
|
|
$job->lease_until = 0;
|
|
$job->available_at = 0;
|
|
$job->updated_at = $now;
|
|
$job->save();
|
|
}
|
|
|
|
/**
|
|
* Release a terminal aggregate fence after every Agent child has reached
|
|
* an evidence-backed resolution. The parent's failed audit outcome remains
|
|
* immutable; only its effect resolution is published.
|
|
*/
|
|
public static function reconcileAggregateParentEffectLocked(
|
|
CloudJob $child,
|
|
?int $resolvedBy
|
|
): void {
|
|
if (!in_array(
|
|
(string) $child->kind,
|
|
array_values(self::AGGREGATE_CHILD_KIND_MAP),
|
|
true
|
|
)) {
|
|
return;
|
|
}
|
|
$parentId = (int) ($child->parent_job_id ?? 0);
|
|
if ($parentId < 1) {
|
|
throw new CloudRecoveryAggregateStateConflict(
|
|
'汇总子任务缺少父任务账本',
|
|
409
|
|
);
|
|
}
|
|
$parent = CloudJob::where('id', $parentId)
|
|
->lockForUpdate()->first();
|
|
if ($parent === null
|
|
|| (string) $parent->execution_mode !== 'aggregate') {
|
|
throw new CloudRecoveryAggregateStateConflict(
|
|
'汇总子任务父任务账本不存在或模式不匹配',
|
|
409
|
|
);
|
|
}
|
|
// Reject a malformed parent/child topology before catalog discovery.
|
|
// This is a read-only preflight while the parent is held; valid paths
|
|
// still acquire every write lock in parent -> catalog -> child order
|
|
// and revalidate the relationship after the child locks are taken.
|
|
$childSnapshot = CloudJob::where('parent_job_id', $parentId)
|
|
->orderBy('id')->get();
|
|
if (self::aggregateRelationshipConflict(
|
|
$parent,
|
|
$childSnapshot,
|
|
(int) $child->id
|
|
)) {
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
'recovery_aggregate_state_conflict',
|
|
time()
|
|
);
|
|
return;
|
|
}
|
|
CloudTemplateCatalogService::lockAggregateCatalogLocked($parent);
|
|
CloudUserIsoService::lockAggregateCatalogLocked($parent);
|
|
$children = CloudJob::where('parent_job_id', $parentId)
|
|
->orderBy('id')->lockForUpdate()->get();
|
|
if (self::aggregateRelationshipConflict(
|
|
$parent,
|
|
$children,
|
|
(int) $child->id
|
|
)) {
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
'recovery_aggregate_state_conflict',
|
|
time()
|
|
);
|
|
return;
|
|
}
|
|
$publishedParent = self::aggregateParentHasPublishedEffect($parent);
|
|
if ($children->contains(static fn (CloudJob $row): bool =>
|
|
self::aggregateChildRecoveryPending($row))) {
|
|
if ($publishedParent) {
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
'recovery_child_convergence',
|
|
time()
|
|
);
|
|
}
|
|
return;
|
|
}
|
|
$resolution = self::aggregateEffectResolution($children);
|
|
$metadataMixed = $resolution === 'mixed'
|
|
&& (string) $parent->kind === 'template_metadata_sync';
|
|
if ($resolution === null) {
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
'recovery_aggregate_state_conflict',
|
|
time()
|
|
);
|
|
return;
|
|
}
|
|
if ($publishedParent) {
|
|
self::recheckPublishedAggregateProjectionLocked(
|
|
$parent,
|
|
$resolution,
|
|
$metadataMixed
|
|
);
|
|
return;
|
|
}
|
|
$partialMixed = $resolution === 'mixed' && !$metadataMixed;
|
|
if ($partialMixed) {
|
|
// Validate every child-to-replica identity before publishing a
|
|
// concrete failed-but-applied partial result. The aggregate stays
|
|
// failed; "applied" means at least one proven host effect exists,
|
|
// never that every replica succeeded.
|
|
CloudTemplateCatalogService::settleRecoveredParentLocked(
|
|
$parent,
|
|
'mixed'
|
|
);
|
|
CloudUserIsoService::settleRecoveredParentLocked(
|
|
$parent,
|
|
'mixed'
|
|
);
|
|
if ((string) $parent->phase !== 'terminal') {
|
|
$parent->refresh();
|
|
}
|
|
}
|
|
$projectionSettled = $partialMixed;
|
|
if (!$partialMixed && (string) $parent->phase !== 'terminal') {
|
|
// Recovered leaf projections are prepared before the matching
|
|
// Agent fence is released. Only this post-commit hook may now
|
|
// publish the user-visible aggregate result.
|
|
CloudTemplateCatalogService::settleRecoveredParentLocked(
|
|
$parent,
|
|
$resolution
|
|
);
|
|
CloudUserIsoService::settleRecoveredParentLocked(
|
|
$parent,
|
|
$resolution
|
|
);
|
|
$parent->refresh();
|
|
if ((string) $parent->outcome !== 'failed'
|
|
|| (string) $parent->effect_resolution !== 'unknown') {
|
|
return;
|
|
}
|
|
$projectionSettled = true;
|
|
}
|
|
if ((string) $parent->outcome !== 'failed'
|
|
|| (string) $parent->effect_resolution !== 'unknown') {
|
|
return;
|
|
}
|
|
// A deadline may have failed the immutable public task and marked the
|
|
// catalog record failed before late Agent evidence converged every
|
|
// replica. Recompute only that resource projection now. The service
|
|
// finalizers call completeAggregateLocked(), whose terminal guard keeps
|
|
// the original parent phase/outcome/status/timestamps/error/result
|
|
// immutable.
|
|
if (!$projectionSettled) {
|
|
CloudTemplateCatalogService::settleRecoveredParentLocked(
|
|
$parent,
|
|
$resolution
|
|
);
|
|
CloudUserIsoService::settleRecoveredParentLocked(
|
|
$parent,
|
|
$resolution
|
|
);
|
|
}
|
|
if (!CloudTemplateCatalogService::aggregateEffectProjectionReadyLocked(
|
|
$parent,
|
|
$resolution
|
|
)) {
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
'recovery_child_convergence',
|
|
time()
|
|
);
|
|
return;
|
|
}
|
|
$publishedResolution = $metadataMixed || $partialMixed
|
|
? 'applied' : $resolution;
|
|
$parent->effect_resolution = $publishedResolution;
|
|
$parent->effect_resolved_at = time();
|
|
$parent->effect_resolved_by = $resolvedBy;
|
|
$parent->effect_resolution_reason = sprintf(
|
|
'child_effects_resolved:%d:%s',
|
|
(int) $child->id,
|
|
$partialMixed ? 'mixed' : $publishedResolution
|
|
);
|
|
$parent->finalization_incident = 0;
|
|
$parent->finalization_error_code = null;
|
|
$parent->lease_token = null;
|
|
$parent->lease_until = 0;
|
|
$parent->available_at = 0;
|
|
$parent->read_at = null;
|
|
$parent->updated_at = time();
|
|
$parent->save();
|
|
}
|
|
|
|
/**
|
|
* Converge one aggregate fence left behind before the realtime child hook
|
|
* existed. This is a PHP-ledger-only sweep: it never dispatches or inspects
|
|
* an Agent operation. Parent then children is the only explicit lock order,
|
|
* and children are locked by id to avoid sibling-order inversions. It is
|
|
* also the durable fallback if a realtime sibling hook loses a deadlock
|
|
* retry or PHP exits after committing the final child.
|
|
*/
|
|
private static function wherePublishedAggregateNeedsRecheck(
|
|
$published
|
|
): void {
|
|
$published->whereIn(
|
|
'effect_resolution',
|
|
['applied', 'not_applied']
|
|
)->where(static function ($needs): void {
|
|
$needs->whereNull('dispatch_attempts')
|
|
->orWhere('dispatch_attempts', '<>', 0)
|
|
->orWhere(static function ($incident): void {
|
|
$incident->where('finalization_incident', 1);
|
|
})->orWhereExists(static function ($templates): void {
|
|
$templates->selectRaw('1')
|
|
->from('cloud_templates as current_templates')
|
|
->whereColumn(
|
|
'current_templates.active_job_id',
|
|
'cloud_jobs.id'
|
|
);
|
|
})->orWhereExists(static function ($isos): void {
|
|
$isos->selectRaw('1')
|
|
->from('cloud_user_isos as current_isos')
|
|
->whereColumn(
|
|
'current_isos.active_job_id',
|
|
'cloud_jobs.id'
|
|
);
|
|
});
|
|
});
|
|
}
|
|
|
|
public static function reconcileOneLegacyAggregateEffect(): bool
|
|
{
|
|
$connection = (new CloudJob())->getConnection();
|
|
$claimedParentId = 0;
|
|
try {
|
|
return $connection->transaction(
|
|
static function () use (&$claimedParentId): bool {
|
|
$now = time();
|
|
$parent = CloudJob::where('phase', 'terminal')
|
|
->where(static function ($candidate): void {
|
|
$candidate->where(
|
|
static function ($aggregate): void {
|
|
$aggregate->whereIn(
|
|
'kind',
|
|
array_keys(
|
|
self::AGGREGATE_CHILD_KIND_MAP
|
|
)
|
|
)->where(
|
|
static function ($recoverable): void {
|
|
$recoverable->where(
|
|
static function ($unknown): void {
|
|
$unknown->where(
|
|
'outcome',
|
|
'failed'
|
|
)->where(
|
|
'status',
|
|
'failed'
|
|
)->where(
|
|
'effect_may_have_applied',
|
|
1
|
|
)->where(
|
|
'safe_to_resubmit',
|
|
0
|
|
)->where(
|
|
'effect_resolution',
|
|
'unknown'
|
|
);
|
|
}
|
|
)->orWhere(
|
|
static function ($ledger): void {
|
|
$ledger->whereNull(
|
|
'dispatch_attempts'
|
|
)->orWhere(
|
|
'dispatch_attempts',
|
|
'<>',
|
|
0
|
|
);
|
|
}
|
|
)->orWhere(
|
|
static function ($published): void {
|
|
self::wherePublishedAggregateNeedsRecheck(
|
|
$published
|
|
);
|
|
}
|
|
)->orWhere(
|
|
static function ($damaged): void {
|
|
self::whereAggregateLocalStateConflict(
|
|
$damaged
|
|
);
|
|
}
|
|
);
|
|
}
|
|
);
|
|
}
|
|
)->orWhereRaw(
|
|
self::aggregateLocalStateConflictSql(
|
|
'cloud_jobs'
|
|
)
|
|
)->orWhere(
|
|
static function ($bound): void {
|
|
self::whereCurrentAggregateCatalogBinding(
|
|
$bound
|
|
);
|
|
}
|
|
)->orWhere(static function ($incident): void {
|
|
$incident->whereNull('parent_job_id')
|
|
->where('finalization_incident', 1)
|
|
->where(
|
|
'finalization_error_code',
|
|
'recovery_aggregate_state_conflict'
|
|
);
|
|
});
|
|
})
|
|
->where('available_at', '<=', $now)
|
|
->where('lease_until', '<=', $now)
|
|
->orderByRaw(
|
|
"CASE WHEN effect_resolution = 'unknown' "
|
|
. 'THEN 0 ELSE 1 END'
|
|
)
|
|
->orderBy('available_at')
|
|
->orderBy('id')
|
|
->lockForUpdate()
|
|
->first();
|
|
if ($parent === null) {
|
|
return false;
|
|
}
|
|
$claimedParentId = (int) $parent->id;
|
|
|
|
// Classify a corrupt aggregate locally before catalog
|
|
// discovery. The parent lock serializes legitimate child
|
|
// writers; valid paths still re-read every child under
|
|
// lock after the catalog row is locked.
|
|
$childSnapshot = CloudJob::where(
|
|
'parent_job_id',
|
|
(int) $parent->id
|
|
)->orderBy('id')->get();
|
|
if (self::aggregateRelationshipConflict(
|
|
$parent,
|
|
$childSnapshot
|
|
)) {
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
'recovery_aggregate_state_conflict',
|
|
$now
|
|
);
|
|
return true;
|
|
}
|
|
|
|
CloudTemplateCatalogService::lockAggregateCatalogLocked(
|
|
$parent
|
|
);
|
|
CloudUserIsoService::lockAggregateCatalogLocked($parent);
|
|
|
|
$children = CloudJob::where(
|
|
'parent_job_id',
|
|
(int) $parent->id
|
|
)->orderBy('id')->lockForUpdate()->get();
|
|
if (self::aggregateRelationshipConflict(
|
|
$parent,
|
|
$children
|
|
)) {
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
'recovery_aggregate_state_conflict',
|
|
$now
|
|
);
|
|
return true;
|
|
}
|
|
|
|
$waiting = $children->contains(
|
|
static fn (CloudJob $child): bool =>
|
|
self::aggregateChildRecoveryPending($child)
|
|
);
|
|
if ($waiting) {
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
'recovery_child_convergence',
|
|
$now
|
|
);
|
|
return true;
|
|
}
|
|
|
|
$resolution = self::aggregateEffectResolution($children);
|
|
$metadataMixed = $resolution === 'mixed'
|
|
&& (string) $parent->kind
|
|
=== 'template_metadata_sync';
|
|
if ($resolution === null) {
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
'recovery_aggregate_state_conflict',
|
|
$now
|
|
);
|
|
return true;
|
|
}
|
|
if (self::aggregateParentHasPublishedEffect($parent)) {
|
|
self::recheckPublishedAggregateProjectionLocked(
|
|
$parent,
|
|
$resolution,
|
|
$metadataMixed
|
|
);
|
|
return true;
|
|
}
|
|
$partialMixed = $resolution === 'mixed'
|
|
&& !$metadataMixed;
|
|
if ($partialMixed) {
|
|
CloudTemplateCatalogService::settleRecoveredParentLocked(
|
|
$parent,
|
|
'mixed'
|
|
);
|
|
CloudUserIsoService::settleRecoveredParentLocked(
|
|
$parent,
|
|
'mixed'
|
|
);
|
|
}
|
|
|
|
// See reconcileAggregateParentEffectLocked(): only repair
|
|
// the catalog projection; never rewrite the failed audit.
|
|
if (!$partialMixed) {
|
|
CloudTemplateCatalogService::settleRecoveredParentLocked(
|
|
$parent,
|
|
$resolution
|
|
);
|
|
CloudUserIsoService::settleRecoveredParentLocked(
|
|
$parent,
|
|
$resolution
|
|
);
|
|
}
|
|
if (!CloudTemplateCatalogService::aggregateEffectProjectionReadyLocked(
|
|
$parent,
|
|
$resolution
|
|
)) {
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
'recovery_child_convergence',
|
|
$now
|
|
);
|
|
return true;
|
|
}
|
|
$publishedResolution = $metadataMixed || $partialMixed
|
|
? 'applied' : $resolution;
|
|
$lastChild = $children->last();
|
|
$parent->effect_resolution = $publishedResolution;
|
|
$parent->effect_resolved_at = $now;
|
|
$parent->effect_resolved_by = null;
|
|
$parent->effect_resolution_reason = sprintf(
|
|
'legacy_child_effects_resolved:%d:%s',
|
|
(int) $lastChild->id,
|
|
$partialMixed ? 'mixed' : $publishedResolution
|
|
);
|
|
$parent->finalization_incident = 0;
|
|
$parent->finalization_error_code = null;
|
|
$parent->lease_token = null;
|
|
$parent->lease_until = 0;
|
|
$parent->available_at = 0;
|
|
$parent->read_at = null;
|
|
$parent->updated_at = $now;
|
|
$parent->save();
|
|
return true;
|
|
},
|
|
3
|
|
);
|
|
} catch (\Throwable $error) {
|
|
if ($claimedParentId < 1) {
|
|
throw $error;
|
|
}
|
|
// The first transaction rolled back both the resource projection
|
|
// and parent effect resolution. Persist only a delayed incident in
|
|
// a fresh transaction so one damaged legacy parent cannot abort
|
|
// every recovery tick or starve later jobs.
|
|
$connection->transaction(
|
|
static function () use ($claimedParentId, $error): void {
|
|
$parent = CloudJob::where('id', $claimedParentId)
|
|
->lockForUpdate()->first();
|
|
if ($parent === null
|
|
|| (string) $parent->execution_mode !== 'aggregate'
|
|
|| (string) $parent->phase !== 'terminal') {
|
|
return;
|
|
}
|
|
$unknownFence = (string) $parent->outcome === 'failed'
|
|
&& (string) $parent->status === 'failed'
|
|
&& (string) $parent->effect_resolution === 'unknown';
|
|
$publishedParent = self::aggregateParentHasPublishedEffect(
|
|
$parent
|
|
);
|
|
if (!$unknownFence && !$publishedParent) {
|
|
return;
|
|
}
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
self::aggregateProjectionErrorCode($error),
|
|
time()
|
|
);
|
|
},
|
|
3
|
|
);
|
|
error_log(sprintf(
|
|
'cloud aggregate projection recovery pending job_id=%d error=%s',
|
|
$claimedParentId,
|
|
get_class($error)
|
|
));
|
|
return true;
|
|
}
|
|
}
|
|
|
|
/** Reject aggregate topology damage before any external fence mutation. */
|
|
public static function assertAggregateChildRelationshipLocked(
|
|
CloudJob $parent,
|
|
CloudJob $child,
|
|
$children
|
|
): void {
|
|
if (self::aggregateRelationshipConflict(
|
|
$parent,
|
|
$children,
|
|
(int) $child->id
|
|
)) {
|
|
throw new CloudRecoveryAggregateStateConflict(
|
|
'汇总任务与子任务账本关系异常',
|
|
409
|
|
);
|
|
}
|
|
}
|
|
|
|
private static function aggregateRelationshipConflict(
|
|
CloudJob $parent,
|
|
$children,
|
|
?int $currentChildId = null
|
|
): bool {
|
|
$parentId = (int) $parent->id;
|
|
$expectedKind = self::AGGREGATE_CHILD_KIND_MAP[
|
|
(string) $parent->kind
|
|
] ?? null;
|
|
return self::hasAggregateLocalStateConflict($parent)
|
|
|| $expectedKind === null
|
|
|| $children->isEmpty()
|
|
|| ($currentChildId !== null && !$children->contains(
|
|
static fn (CloudJob $child): bool =>
|
|
(int) $child->id === $currentChildId
|
|
))
|
|
|| $children->contains(
|
|
static fn (CloudJob $child): bool =>
|
|
(int) ($child->parent_job_id ?? 0) !== $parentId
|
|
|| (int) $child->user_id !== (int) $parent->user_id
|
|
|| (string) $child->execution_mode !== 'agent'
|
|
|| (string) $child->kind !== $expectedKind
|
|
|| self::aggregateChildHasPhaseResolutionConflict($child)
|
|
|| self::hasResolvedTerminalStateConflict($child)
|
|
);
|
|
}
|
|
|
|
/**
|
|
* SQL counterpart of hasAggregateLocalStateConflict() for terminal rows.
|
|
* This keeps an unbound, already-published parent reachable by the local
|
|
* ledger sweep without polling every healthy historical aggregate.
|
|
*/
|
|
private static function whereAggregateLocalStateConflict($damaged): void
|
|
{
|
|
$operationInvalid = self::invalidIdentifierCharactersSql(
|
|
'operation_id', '0-9a-f'
|
|
);
|
|
$fingerprintInvalid = self::invalidIdentifierCharactersSql(
|
|
'request_fingerprint', '0-9a-f'
|
|
);
|
|
$idempotencyInvalid = self::invalidIdentifierCharactersSql(
|
|
'request_idempotency_key', 'A-Za-z0-9_.-'
|
|
);
|
|
$damaged->where(static function ($invalid) use (
|
|
$operationInvalid,
|
|
$fingerprintInvalid,
|
|
$idempotencyInvalid
|
|
): void {
|
|
$invalid->whereNotNull('parent_job_id')
|
|
->orWhereNotNull('host_id')
|
|
->orWhereNotNull('instance_id')
|
|
->orWhereNotNull('order_id')
|
|
->orWhereNotNull('invoice_id')
|
|
->orWhereNotNull('snapshot_id')
|
|
->orWhereNull('user_id')
|
|
->orWhere('user_id', '<', 1)
|
|
->orWhereNull('attempts')
|
|
->orWhere('attempts', '<>', 0)
|
|
->orWhereNull('dispatch_attempts')
|
|
->orWhere('dispatch_attempts', '<>', 0)
|
|
->orWhereNull('dispatch_deadline_at')
|
|
->orWhere('dispatch_deadline_at', '<', 1)
|
|
->orWhereNull('receipt_deadline_at')
|
|
->orWhere('receipt_deadline_at', '<', 1)
|
|
->orWhereNotNull('dispatch_started_at')
|
|
->orWhereNotNull('agent_accepted_at')
|
|
->orWhereNotNull('dispatched_at')
|
|
->orWhereNotNull('completion_event_id')
|
|
->orWhereNull('operation_id')
|
|
->orWhereRaw('LENGTH(operation_id) <> 32')
|
|
->orWhereRaw('LOWER(operation_id) <> operation_id')
|
|
->orWhereRaw($operationInvalid)
|
|
->orWhereNull('request_fingerprint')
|
|
->orWhereRaw('LENGTH(request_fingerprint) <> 64')
|
|
->orWhereRaw(
|
|
'LOWER(request_fingerprint) <> request_fingerprint'
|
|
)->orWhereRaw($fingerprintInvalid)
|
|
->orWhereNull('request_idempotency_key')
|
|
->orWhereRaw(
|
|
'LENGTH(request_idempotency_key) NOT BETWEEN 16 AND 128'
|
|
)->orWhereRaw($idempotencyInvalid)
|
|
->orWhereNull('terminal_at')
|
|
->orWhere('terminal_at', '<', 1)
|
|
->orWhereNull('outcome')
|
|
->orWhereNotIn('outcome', ['succeeded', 'failed'])
|
|
->orWhereNull('status')
|
|
->orWhereColumn('status', '<>', 'outcome')
|
|
->orWhere(static function ($succeeded): void {
|
|
$succeeded->where('outcome', 'succeeded')
|
|
->where(static function ($mismatch): void {
|
|
$mismatch->whereNull('effect_resolution')
|
|
->orWhere(
|
|
'effect_resolution',
|
|
'<>',
|
|
'applied'
|
|
)->orWhereNull('effect_may_have_applied')
|
|
->orWhere('effect_may_have_applied', '<>', 0)
|
|
->orWhereNull('safe_to_resubmit')
|
|
->orWhere('safe_to_resubmit', '<>', 0)
|
|
->orWhereNotNull('error_code');
|
|
});
|
|
})->orWhere(static function ($failed): void {
|
|
$failed->where('outcome', 'failed')
|
|
->where(static function ($mismatch): void {
|
|
$mismatch->whereNull('effect_resolution')
|
|
->orWhereNotIn(
|
|
'effect_resolution',
|
|
['unknown', 'applied', 'not_applied']
|
|
)->orWhereNull('effect_may_have_applied')
|
|
->orWhereNotIn(
|
|
'effect_may_have_applied',
|
|
[0, 1]
|
|
)->orWhereNull('safe_to_resubmit')
|
|
->orWhereNotIn('safe_to_resubmit', [0, 1])
|
|
->orWhere(static function ($effectful): void {
|
|
$effectful->where(
|
|
'effect_may_have_applied',
|
|
1
|
|
)->where('safe_to_resubmit', '<>', 0);
|
|
})->orWhere(static function ($notEffectful): void {
|
|
$notEffectful->where(
|
|
'effect_may_have_applied',
|
|
0
|
|
)->where(
|
|
'effect_resolution',
|
|
'<>',
|
|
'not_applied'
|
|
);
|
|
});
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
/** Add resolved Agent rows whose immutable dispatch envelope is damaged. */
|
|
private static function applyInvalidResolvedAgentEnvelope(
|
|
$invalid,
|
|
string $prefix
|
|
): void {
|
|
$operation = $prefix . 'operation_id';
|
|
$fingerprint = $prefix . 'request_fingerprint';
|
|
$idempotency = $prefix . 'request_idempotency_key';
|
|
$invalid->whereNull($prefix . 'execution_mode')
|
|
->orWhere($prefix . 'execution_mode', '<>', 'agent')
|
|
->orWhereNull($prefix . 'dispatch_attempts')
|
|
->orWhereNotIn($prefix . 'dispatch_attempts', [0, 1])
|
|
->orWhereNull($prefix . 'host_id')
|
|
->orWhere($prefix . 'host_id', '<', 1)
|
|
->orWhereNull($prefix . 'dispatch_deadline_at')
|
|
->orWhere($prefix . 'dispatch_deadline_at', '<', 1)
|
|
->orWhereNull($operation)
|
|
->orWhereRaw('LENGTH(' . $operation . ') <> 32')
|
|
->orWhereRaw('LOWER(' . $operation . ') <> ' . $operation)
|
|
->orWhereRaw(self::invalidIdentifierCharactersSql(
|
|
$operation,
|
|
'0-9a-f'
|
|
))->orWhereNull($fingerprint)
|
|
->orWhereRaw('LENGTH(' . $fingerprint . ') <> 64')
|
|
->orWhereRaw('LOWER(' . $fingerprint . ') <> ' . $fingerprint)
|
|
->orWhereRaw(self::invalidIdentifierCharactersSql(
|
|
$fingerprint,
|
|
'0-9a-f'
|
|
))->orWhereNull($idempotency)
|
|
->orWhereRaw(
|
|
'LENGTH(' . $idempotency . ') NOT BETWEEN 16 AND 128'
|
|
)->orWhereRaw(self::invalidIdentifierCharactersSql(
|
|
$idempotency,
|
|
'A-Za-z0-9_.-'
|
|
))->orWhere(static function ($dispatched) use ($prefix): void {
|
|
$dispatched->where($prefix . 'dispatch_attempts', 1)
|
|
->where(static function ($broken) use ($prefix): void {
|
|
$broken->whereNull($prefix . 'attempts')
|
|
->orWhere($prefix . 'attempts', '<>', 1)
|
|
->orWhereNull($prefix . 'dispatch_started_at')
|
|
->orWhere($prefix . 'dispatch_started_at', '<', 1)
|
|
->orWhere(static function ($accepted) use (
|
|
$prefix
|
|
): void {
|
|
$accepted->whereNotNull(
|
|
$prefix . 'agent_accepted_at'
|
|
)->where(
|
|
$prefix . 'agent_accepted_at',
|
|
'<',
|
|
1
|
|
);
|
|
})
|
|
->orWhereNull($prefix . 'dispatched_at')
|
|
->orWhere($prefix . 'dispatched_at', '<', 1)
|
|
->orWhereNull($prefix . 'receipt_deadline_at')
|
|
->orWhere($prefix . 'receipt_deadline_at', '<', 1);
|
|
});
|
|
})->orWhere(static function ($undispatched) use ($prefix): void {
|
|
$undispatched->where($prefix . 'dispatch_attempts', 0)
|
|
->where(static function ($broken) use ($prefix): void {
|
|
$broken->whereNull($prefix . 'attempts')
|
|
->orWhere($prefix . 'attempts', '<>', 0)
|
|
->orWhereNotNull($prefix . 'dispatch_started_at')
|
|
->orWhereNotNull($prefix . 'agent_accepted_at')
|
|
->orWhereNotNull($prefix . 'dispatched_at')
|
|
->orWhereNotNull($prefix . 'receipt_deadline_at')
|
|
->orWhereNotNull($prefix . 'completion_event_id')
|
|
->orWhere(static function ($success) use (
|
|
$prefix
|
|
): void {
|
|
$success->where(
|
|
$prefix . 'outcome',
|
|
'succeeded'
|
|
)->where(static function ($notNoop) use (
|
|
$prefix
|
|
): void {
|
|
$notNoop->where(
|
|
$prefix . 'kind',
|
|
'<>',
|
|
'firewall_apply'
|
|
)->orWhereNull($prefix . 'instance_id')
|
|
->orWhere(
|
|
$prefix . 'instance_id',
|
|
'<',
|
|
1
|
|
)->orWhereRaw(
|
|
self::invalidFirewallNoopResultSql(
|
|
$prefix . 'result_json'
|
|
)
|
|
);
|
|
});
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
/** Driver-specific JSON validation for the sole generation-zero success. */
|
|
private static function invalidFirewallNoopResultSql(string $column): string
|
|
{
|
|
$driver = (new CloudJob())->getConnection()->getDriverName();
|
|
if ($driver === 'sqlite') {
|
|
$digest = "json_extract(" . $column . ", '$.policy_digest')";
|
|
return '(CASE WHEN COALESCE(json_valid(' . $column . '), 0) = 1 '
|
|
. 'THEN CASE WHEN '
|
|
. "json_type(" . $column . ", '$.unchanged') = 'true' AND "
|
|
. "json_type(" . $column . ", '$.policy_digest') = 'text' AND "
|
|
. 'LENGTH(' . $digest . ') = 64 AND NOT ('
|
|
. self::invalidIdentifierCharactersSql($digest, '0-9a-f')
|
|
. ') AND '
|
|
. "json_type(" . $column . ", '$.revision') = 'integer' AND "
|
|
. "json_extract(" . $column . ", '$.revision') > 0 "
|
|
. 'THEN 0 ELSE 1 END ELSE 1 END) = 1';
|
|
}
|
|
$digest = "JSON_UNQUOTE(JSON_EXTRACT(" . $column
|
|
. ", '$.policy_digest'))";
|
|
$revision = "JSON_UNQUOTE(JSON_EXTRACT(" . $column
|
|
. ", '$.revision'))";
|
|
return '(CASE WHEN COALESCE(JSON_VALID(' . $column . '), 0) = 1 '
|
|
. 'THEN CASE WHEN JSON_UNQUOTE(JSON_EXTRACT(' . $column
|
|
. ", '$.unchanged')) = 'true' AND LENGTH(" . $digest
|
|
. ') = 64 AND NOT ('
|
|
. self::invalidIdentifierCharactersSql($digest, '0-9a-f')
|
|
. ') AND ' . $revision . " REGEXP '^[1-9][0-9]*$' "
|
|
. 'THEN 0 ELSE 1 END ELSE 1 END) = 1';
|
|
}
|
|
|
|
/** Build a portable invalid-character predicate for MySQL and SQLite. */
|
|
private static function invalidIdentifierCharactersSql(
|
|
string $column,
|
|
string $allowedClass
|
|
): string {
|
|
$driver = (new CloudJob())->getConnection()->getDriverName();
|
|
if ($driver === 'sqlite') {
|
|
return $column . " GLOB '*[^" . $allowedClass . "]*'";
|
|
}
|
|
return $column . " REGEXP '[^" . $allowedClass . "]'";
|
|
}
|
|
|
|
/**
|
|
* Raw-SQL counterpart of hasTerminalRecoveryStateConflict(). Inventory
|
|
* filters and counters cannot hydrate every row before filtering, so keep
|
|
* their fail-closed envelope test in the same service as the DTO check.
|
|
*/
|
|
public static function terminalRecoveryStateConflictSql(
|
|
string $alias
|
|
): string {
|
|
$supported = self::quotedSqlValues(
|
|
CloudOperationRecoveryPolicy::ledgerFinalizerKinds()
|
|
);
|
|
$operation = $alias . '.operation_id';
|
|
$fingerprint = $alias . '.request_fingerprint';
|
|
$idempotency = $alias . '.request_idempotency_key';
|
|
$invalidEnvelope = '(' . $alias . '.execution_mode IS NULL OR '
|
|
. $alias . ".execution_mode <> 'agent' OR "
|
|
. $alias . '.dispatch_attempts IS NULL OR '
|
|
. $alias . '.dispatch_attempts NOT IN (0, 1) OR '
|
|
. $alias . '.host_id IS NULL OR ' . $alias . '.host_id < 1 OR '
|
|
. $alias . '.dispatch_deadline_at IS NULL OR '
|
|
. $alias . '.dispatch_deadline_at < 1 OR '
|
|
. $operation . ' IS NULL OR LENGTH(' . $operation . ') <> 32 OR '
|
|
. 'LOWER(' . $operation . ') <> ' . $operation . ' OR '
|
|
. self::invalidIdentifierCharactersSql($operation, '0-9a-f')
|
|
. ' OR ' . $fingerprint . ' IS NULL OR LENGTH('
|
|
. $fingerprint . ') <> 64 OR LOWER(' . $fingerprint . ') <> '
|
|
. $fingerprint . ' OR '
|
|
. self::invalidIdentifierCharactersSql($fingerprint, '0-9a-f')
|
|
. ' OR ' . $idempotency . ' IS NULL OR LENGTH('
|
|
. $idempotency . ') NOT BETWEEN 16 AND 128 OR '
|
|
. self::invalidIdentifierCharactersSql(
|
|
$idempotency,
|
|
'A-Za-z0-9_.-'
|
|
) . ' OR (' . $alias . '.dispatch_attempts = 1 AND ('
|
|
. $alias . '.attempts IS NULL OR ' . $alias . '.attempts <> 1 OR '
|
|
. $alias . '.dispatch_started_at IS NULL OR '
|
|
. $alias . '.dispatch_started_at < 1 OR ('
|
|
. $alias . '.agent_accepted_at IS NOT NULL AND '
|
|
. $alias . '.agent_accepted_at < 1) OR '
|
|
. $alias . '.dispatched_at IS NULL OR '
|
|
. $alias . '.dispatched_at < 1 OR '
|
|
. $alias . '.receipt_deadline_at IS NULL OR '
|
|
. $alias . '.receipt_deadline_at < 1)) OR ('
|
|
. $alias . '.dispatch_attempts = 0 AND ('
|
|
. $alias . '.attempts IS NULL OR ' . $alias . '.attempts <> 0 OR '
|
|
. $alias . '.dispatch_started_at IS NOT NULL OR '
|
|
. $alias . '.agent_accepted_at IS NOT NULL OR '
|
|
. $alias . '.dispatched_at IS NOT NULL OR '
|
|
. $alias . '.receipt_deadline_at IS NOT NULL OR '
|
|
. $alias . '.completion_event_id IS NOT NULL OR ('
|
|
. $alias . ".outcome = 'succeeded' AND ("
|
|
. $alias . ".kind <> 'firewall_apply' OR "
|
|
. $alias . '.instance_id IS NULL OR '
|
|
. $alias . '.instance_id < 1 OR '
|
|
. self::invalidFirewallNoopResultSql(
|
|
$alias . '.result_json'
|
|
) . ')))))';
|
|
$invalidResolved = '(' . $alias
|
|
. ".effect_resolution IN ('applied', 'not_applied') AND ("
|
|
. $alias . '.terminal_at IS NULL OR '
|
|
. $alias . '.terminal_at < 1 OR '
|
|
. $alias . '.outcome IS NULL OR '
|
|
. $alias . ".outcome NOT IN ('succeeded', 'failed') OR ("
|
|
. $alias . ".outcome = 'succeeded' AND ("
|
|
. $alias . ".status IS NULL OR "
|
|
. $alias . ".status <> 'succeeded' OR "
|
|
. $alias . ".effect_resolution <> 'applied' OR "
|
|
. $alias . '.effect_may_have_applied IS NULL OR '
|
|
. $alias . '.effect_may_have_applied <> 0 OR '
|
|
. $alias . '.safe_to_resubmit IS NULL OR '
|
|
. $alias . '.safe_to_resubmit <> 0 OR '
|
|
. $alias . '.error_code IS NOT NULL)) OR ('
|
|
. $alias . ".outcome = 'failed' AND ("
|
|
. $alias . ".status IS NULL OR "
|
|
. $alias . ".status <> 'failed' OR "
|
|
. $alias . '.effect_may_have_applied IS NULL OR '
|
|
. $alias . '.effect_may_have_applied NOT IN (0, 1) OR ('
|
|
. $alias . '.effect_may_have_applied = 1 AND ('
|
|
. $alias . '.safe_to_resubmit IS NULL OR '
|
|
. $alias . '.safe_to_resubmit <> 0)) OR ('
|
|
. $alias . '.effect_may_have_applied = 0 AND ('
|
|
. $alias . ".effect_resolution <> 'not_applied' OR "
|
|
. $alias . '.safe_to_resubmit IS NULL OR '
|
|
. $alias . '.safe_to_resubmit NOT IN (0, 1))))) OR '
|
|
. $invalidEnvelope . '))';
|
|
$invalidUnknown = '(' . $alias . ".effect_resolution = 'unknown' AND (("
|
|
. $alias . '.dispatch_attempts = 0 AND COALESCE('
|
|
. $alias . '.finalization_incident, 0) = 0) OR NOT ('
|
|
. "COALESCE(" . $alias . ".outcome, '') = 'failed' AND "
|
|
. "COALESCE(" . $alias . ".status, '') = 'failed' AND "
|
|
. 'COALESCE(' . $alias . '.effect_may_have_applied, 0) = 1 AND '
|
|
. 'COALESCE(' . $alias . '.safe_to_resubmit, 1) = 0)))';
|
|
return '(' . $alias . ".phase = 'terminal' AND "
|
|
. $alias . '.kind IN (' . $supported . ') AND ('
|
|
. $alias . '.execution_mode IS NULL OR '
|
|
. $alias . ".execution_mode <> 'agent' OR "
|
|
. $alias . '.dispatch_attempts IS NULL OR '
|
|
. $alias . '.dispatch_attempts NOT IN (0, 1) OR '
|
|
. $invalidEnvelope . ' OR ' . $invalidUnknown . ' OR '
|
|
. $invalidResolved . '))';
|
|
}
|
|
|
|
/** SQL counterpart of hasAggregateLocalStateConflict() for terminal rows. */
|
|
public static function aggregateLocalStateConflictSql(
|
|
string $alias
|
|
): string {
|
|
$kinds = self::quotedSqlValues(self::AGGREGATE_KINDS);
|
|
$operation = $alias . '.operation_id';
|
|
$fingerprint = $alias . '.request_fingerprint';
|
|
$idempotency = $alias . '.request_idempotency_key';
|
|
return '(' . $alias . ".phase = 'terminal' AND ("
|
|
. $alias . ".execution_mode = 'aggregate' OR "
|
|
. $alias . '.kind IN (' . $kinds . ')) AND ('
|
|
. $alias . '.execution_mode IS NULL OR '
|
|
. $alias . ".execution_mode <> 'aggregate' OR "
|
|
. $alias . '.kind IS NULL OR '
|
|
. $alias . '.kind NOT IN (' . $kinds . ') OR '
|
|
. $alias . '.parent_job_id IS NOT NULL OR '
|
|
. $alias . '.host_id IS NOT NULL OR '
|
|
. $alias . '.instance_id IS NOT NULL OR '
|
|
. $alias . '.order_id IS NOT NULL OR '
|
|
. $alias . '.invoice_id IS NOT NULL OR '
|
|
. $alias . '.snapshot_id IS NOT NULL OR '
|
|
. $alias . '.user_id IS NULL OR ' . $alias . '.user_id < 1 OR '
|
|
. $alias . '.attempts IS NULL OR ' . $alias . '.attempts <> 0 OR '
|
|
. $alias . '.dispatch_attempts IS NULL OR '
|
|
. $alias . '.dispatch_attempts <> 0 OR '
|
|
. $alias . '.dispatch_deadline_at IS NULL OR '
|
|
. $alias . '.dispatch_deadline_at < 1 OR '
|
|
. $alias . '.receipt_deadline_at IS NULL OR '
|
|
. $alias . '.receipt_deadline_at < 1 OR '
|
|
. $alias . '.dispatch_started_at IS NOT NULL OR '
|
|
. $alias . '.agent_accepted_at IS NOT NULL OR '
|
|
. $alias . '.dispatched_at IS NOT NULL OR '
|
|
. $alias . '.completion_event_id IS NOT NULL OR '
|
|
. $operation . ' IS NULL OR LENGTH(' . $operation . ') <> 32 OR '
|
|
. 'LOWER(' . $operation . ') <> ' . $operation . ' OR '
|
|
. self::invalidIdentifierCharactersSql($operation, '0-9a-f')
|
|
. ' OR ' . $fingerprint . ' IS NULL OR LENGTH('
|
|
. $fingerprint . ') <> 64 OR LOWER(' . $fingerprint . ') <> '
|
|
. $fingerprint . ' OR '
|
|
. self::invalidIdentifierCharactersSql($fingerprint, '0-9a-f')
|
|
. ' OR ' . $idempotency . ' IS NULL OR LENGTH('
|
|
. $idempotency . ') NOT BETWEEN 16 AND 128 OR '
|
|
. self::invalidIdentifierCharactersSql(
|
|
$idempotency,
|
|
'A-Za-z0-9_.-'
|
|
) . ' OR ' . $alias . '.terminal_at IS NULL OR '
|
|
. $alias . '.terminal_at < 1 OR '
|
|
. $alias . '.outcome IS NULL OR '
|
|
. $alias . ".outcome NOT IN ('succeeded', 'failed') OR "
|
|
. $alias . '.status IS NULL OR '
|
|
. $alias . '.status <> ' . $alias . '.outcome OR ('
|
|
. $alias . ".outcome = 'succeeded' AND ("
|
|
. $alias . ".effect_resolution IS NULL OR "
|
|
. $alias . ".effect_resolution <> 'applied' OR "
|
|
. $alias . '.effect_may_have_applied IS NULL OR '
|
|
. $alias . '.effect_may_have_applied <> 0 OR '
|
|
. $alias . '.safe_to_resubmit IS NULL OR '
|
|
. $alias . '.safe_to_resubmit <> 0 OR '
|
|
. $alias . '.error_code IS NOT NULL)) OR ('
|
|
. $alias . ".outcome = 'failed' AND ("
|
|
. $alias . '.effect_resolution IS NULL OR '
|
|
. $alias . ".effect_resolution NOT IN ('unknown', 'applied', 'not_applied') OR "
|
|
. $alias . '.effect_may_have_applied IS NULL OR '
|
|
. $alias . '.effect_may_have_applied NOT IN (0, 1) OR '
|
|
. $alias . '.safe_to_resubmit IS NULL OR '
|
|
. $alias . '.safe_to_resubmit NOT IN (0, 1) OR ('
|
|
. $alias . '.effect_may_have_applied = 1 AND '
|
|
. $alias . '.safe_to_resubmit <> 0) OR ('
|
|
. $alias . '.effect_may_have_applied = 0 AND '
|
|
. $alias . ".effect_resolution <> 'not_applied')))))";
|
|
}
|
|
|
|
/** @param list<string> $values */
|
|
private static function quotedSqlValues(array $values): string
|
|
{
|
|
return implode(', ', array_map(
|
|
static fn (string $value): string => "'"
|
|
. str_replace("'", "''", $value) . "'",
|
|
$values
|
|
));
|
|
}
|
|
|
|
/** Add the catalog-driven candidate predicate without assuming job shape. */
|
|
private static function whereCurrentAggregateCatalogBinding($bound): void
|
|
{
|
|
$bound->whereExists(static function ($templates): void {
|
|
$templates->selectRaw('1')
|
|
->from('cloud_templates as current_templates')
|
|
->whereColumn(
|
|
'current_templates.active_job_id',
|
|
'cloud_jobs.id'
|
|
);
|
|
})->orWhereExists(static function ($isos): void {
|
|
$isos->selectRaw('1')
|
|
->from('cloud_user_isos as current_isos')
|
|
->whereColumn(
|
|
'current_isos.active_job_id',
|
|
'cloud_jobs.id'
|
|
);
|
|
})->orWhere(static function ($repairable): void {
|
|
self::whereRepairableMissingAggregateCatalogPointer(
|
|
$repairable
|
|
);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Keep a terminal parent discoverable when only active_job_id was cleared.
|
|
* This predicate is intentionally broad enough to surface ambiguity; the
|
|
* service-level locked repair performs the complete one-to-one proof.
|
|
*/
|
|
private static function whereRepairableMissingAggregateCatalogPointer(
|
|
$candidate
|
|
): void {
|
|
$settledPartial = self::settledReleasedPartialAggregateSql(
|
|
'cloud_jobs'
|
|
);
|
|
$candidate->whereExists(static function ($templates) use (
|
|
$settledPartial
|
|
): void {
|
|
$templates->selectRaw('1')
|
|
->from('cloud_template_replicas as pointer_template_replicas')
|
|
->join(
|
|
'cloud_templates as pointer_templates',
|
|
'pointer_templates.id',
|
|
'=',
|
|
'pointer_template_replicas.template_id'
|
|
)->join(
|
|
'cloud_jobs as pointer_template_children',
|
|
'pointer_template_children.id',
|
|
'=',
|
|
'pointer_template_replicas.job_id'
|
|
)->whereColumn(
|
|
'pointer_template_children.parent_job_id',
|
|
'cloud_jobs.id'
|
|
)->where(static function ($binding) use (
|
|
$settledPartial
|
|
): void {
|
|
$binding->where(static function ($missing) use (
|
|
$settledPartial
|
|
): void {
|
|
$missing->where(static function ($released): void {
|
|
$released->whereNull(
|
|
'pointer_templates.active_job_id'
|
|
)->orWhere(
|
|
'pointer_templates.active_job_id',
|
|
'<=',
|
|
0
|
|
);
|
|
})->whereRaw('NOT ' . $settledPartial);
|
|
})->orWhere(static function ($conflict): void {
|
|
$conflict->where(
|
|
'pointer_templates.active_job_id',
|
|
'>',
|
|
0
|
|
)->whereColumn(
|
|
'pointer_templates.active_job_id',
|
|
'<>',
|
|
'cloud_jobs.id'
|
|
);
|
|
});
|
|
})->where(static function ($identity): void {
|
|
$identity->where(static function ($import): void {
|
|
$import->where('cloud_jobs.kind', 'template_import')
|
|
->where(
|
|
'pointer_template_children.kind',
|
|
'template_import_replica'
|
|
)->whereIn(
|
|
'pointer_templates.state',
|
|
['importing', 'error', 'active']
|
|
);
|
|
})->orWhere(static function ($metadata): void {
|
|
$metadata->where(
|
|
'cloud_jobs.kind',
|
|
'template_metadata_sync'
|
|
)->where(
|
|
'pointer_template_children.kind',
|
|
'template_metadata_replica'
|
|
)->where(
|
|
'pointer_templates.state',
|
|
'active'
|
|
)->where(static function ($evidence): void {
|
|
$evidence->whereIn(
|
|
'pointer_template_replicas.state',
|
|
['syncing', 'stale']
|
|
)->orWhere(static function ($unknown): void {
|
|
$unknown->where(
|
|
'cloud_jobs.outcome',
|
|
'failed'
|
|
)->where(
|
|
'cloud_jobs.status',
|
|
'failed'
|
|
)->where(
|
|
'cloud_jobs.effect_resolution',
|
|
'unknown'
|
|
);
|
|
})->orWhere(
|
|
'cloud_jobs.finalization_incident',
|
|
1
|
|
)->orWhere(static function ($damaged): void {
|
|
$damaged->where(
|
|
'pointer_templates.active_job_id',
|
|
'>',
|
|
0
|
|
)->whereColumn(
|
|
'pointer_templates.active_job_id',
|
|
'<>',
|
|
'cloud_jobs.id'
|
|
);
|
|
});
|
|
});
|
|
})->orWhere(static function ($delete): void {
|
|
$delete->where('cloud_jobs.kind', 'template_delete')
|
|
->where(
|
|
'pointer_template_children.kind',
|
|
'template_delete_replica'
|
|
)->whereIn(
|
|
'pointer_templates.state',
|
|
['deleting', 'delete_failed', 'active']
|
|
);
|
|
});
|
|
});
|
|
})->orWhereExists(static function ($isos) use (
|
|
$settledPartial
|
|
): void {
|
|
$isos->selectRaw('1')
|
|
->from('cloud_user_iso_replicas as pointer_iso_replicas')
|
|
->join(
|
|
'cloud_user_isos as pointer_isos',
|
|
'pointer_isos.id',
|
|
'=',
|
|
'pointer_iso_replicas.user_iso_id'
|
|
)->join(
|
|
'cloud_jobs as pointer_iso_children',
|
|
'pointer_iso_children.id',
|
|
'=',
|
|
'pointer_iso_replicas.job_id'
|
|
)->whereColumn(
|
|
'pointer_iso_children.parent_job_id',
|
|
'cloud_jobs.id'
|
|
)->where(static function ($binding) use (
|
|
$settledPartial
|
|
): void {
|
|
$binding->where(static function ($missing) use (
|
|
$settledPartial
|
|
): void {
|
|
$missing->where(static function ($released): void {
|
|
$released->whereNull(
|
|
'pointer_isos.active_job_id'
|
|
)->orWhere(
|
|
'pointer_isos.active_job_id',
|
|
'<=',
|
|
0
|
|
);
|
|
})->whereRaw('NOT ' . $settledPartial);
|
|
})->orWhere(static function ($conflict): void {
|
|
$conflict->where(
|
|
'pointer_isos.active_job_id',
|
|
'>',
|
|
0
|
|
)->whereColumn(
|
|
'pointer_isos.active_job_id',
|
|
'<>',
|
|
'cloud_jobs.id'
|
|
);
|
|
});
|
|
})->where(static function ($identity): void {
|
|
$identity->where(static function ($import): void {
|
|
$import->where('cloud_jobs.kind', 'user_iso_import')
|
|
->where(
|
|
'pointer_iso_children.kind',
|
|
'user_iso_import_replica'
|
|
)->whereIn(
|
|
'pointer_isos.state',
|
|
['importing', 'import_failed', 'active']
|
|
);
|
|
})->orWhere(static function ($delete): void {
|
|
$delete->where('cloud_jobs.kind', 'user_iso_delete')
|
|
->where(
|
|
'pointer_iso_children.kind',
|
|
'user_iso_delete_replica'
|
|
)->whereIn(
|
|
'pointer_isos.state',
|
|
['deleting', 'delete_failed', 'active']
|
|
);
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
/** Exact durable marker for a safely released non-metadata partial. */
|
|
private static function settledReleasedPartialAggregateSql(
|
|
string $alias
|
|
): string {
|
|
$kinds = self::quotedSqlValues([
|
|
'template_import',
|
|
'template_delete',
|
|
'user_iso_import',
|
|
'user_iso_delete',
|
|
]);
|
|
return '(' . $alias . ".execution_mode = 'aggregate' AND "
|
|
. $alias . '.parent_job_id IS NULL AND '
|
|
. $alias . '.kind IN (' . $kinds . ') AND '
|
|
. $alias . ".phase = 'terminal' AND "
|
|
. $alias . ".outcome = 'failed' AND "
|
|
. $alias . ".status = 'failed' AND "
|
|
. $alias . '.effect_may_have_applied = 1 AND '
|
|
. $alias . '.safe_to_resubmit = 0 AND '
|
|
. $alias . ".effect_resolution = 'applied' AND "
|
|
. $alias . '.effect_resolved_at > 0 AND '
|
|
. 'COALESCE(' . $alias . '.finalization_incident, 0) = 0 AND '
|
|
. 'NOT ' . self::aggregateLocalStateConflictSql($alias) . ' AND '
|
|
. self::settledPartialReasonSql(
|
|
$alias . '.effect_resolution_reason'
|
|
) . ')';
|
|
}
|
|
|
|
/** Match only our immutable numeric-child mixed publication reasons. */
|
|
private static function settledPartialReasonSql(string $column): string
|
|
{
|
|
$driver = (new CloudJob())->getConnection()->getDriverName();
|
|
if ($driver !== 'sqlite') {
|
|
return $column
|
|
. " REGEXP '^(legacy_)?child_effects_resolved:"
|
|
. "[1-9][0-9]*:mixed$'";
|
|
}
|
|
$suffix = ':mixed';
|
|
$parts = [];
|
|
foreach ([
|
|
'child_effects_resolved:',
|
|
'legacy_child_effects_resolved:',
|
|
] as $prefix) {
|
|
$offset = strlen($prefix) + 1;
|
|
$middleLength = 'LENGTH(' . $column . ') - '
|
|
. (strlen($prefix) + strlen($suffix));
|
|
$middle = 'SUBSTR(' . $column . ', ' . $offset . ', '
|
|
. $middleLength . ')';
|
|
$parts[] = '(SUBSTR(' . $column . ', 1, ' . strlen($prefix)
|
|
. ") = '" . $prefix . "' AND SUBSTR(" . $column
|
|
. ", -" . strlen($suffix) . ") = '" . $suffix . "' AND "
|
|
. $middleLength . ' > 0 AND SUBSTR(' . $middle
|
|
. ", 1, 1) BETWEEN '1' AND '9' AND NOT (" . $middle
|
|
. " GLOB '*[^0-9]*'))";
|
|
}
|
|
return '(' . implode(' OR ', $parts) . ')';
|
|
}
|
|
|
|
/** A final effect cannot be published while its leaf is still nonterminal. */
|
|
private static function aggregateChildHasPhaseResolutionConflict(
|
|
CloudJob $child
|
|
): bool {
|
|
return (string) $child->phase !== 'terminal'
|
|
&& in_array(
|
|
(string) $child->effect_resolution,
|
|
['applied', 'not_applied'],
|
|
true
|
|
);
|
|
}
|
|
|
|
/** Return the uniform effect, mixed, or null for an invalid child state. */
|
|
private static function aggregateEffectResolution($children): ?string
|
|
{
|
|
$resolutions = $children->map(
|
|
static fn (CloudJob $child): string =>
|
|
(string) $child->effect_resolution
|
|
)->unique()->values();
|
|
if ($resolutions->contains(
|
|
static fn ($resolution): bool => !in_array(
|
|
(string) $resolution,
|
|
['applied', 'not_applied'],
|
|
true
|
|
)
|
|
)) {
|
|
return null;
|
|
}
|
|
return $resolutions->count() === 1
|
|
? (string) $resolutions->first() : 'mixed';
|
|
}
|
|
|
|
private static function aggregateParentHasPublishedEffect(
|
|
CloudJob $parent
|
|
): bool {
|
|
return (string) $parent->phase === 'terminal'
|
|
&& in_array(
|
|
(string) $parent->effect_resolution,
|
|
['applied', 'not_applied'],
|
|
true
|
|
);
|
|
}
|
|
|
|
/** Revalidate a published aggregate without rewriting its terminal audit. */
|
|
private static function recheckPublishedAggregateProjectionLocked(
|
|
CloudJob $parent,
|
|
string $resolution,
|
|
bool $metadataMixed
|
|
): void {
|
|
try {
|
|
$bindingState = self::currentAggregateBindingStateLocked($parent);
|
|
} catch (\Throwable $error) {
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
self::aggregateProjectionErrorCode($error),
|
|
time()
|
|
);
|
|
return;
|
|
}
|
|
if ($bindingState === 'conflict') {
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
'recovery_aggregate_state_conflict',
|
|
time()
|
|
);
|
|
return;
|
|
}
|
|
if ($bindingState === 'absent') {
|
|
if ((bool) $parent->finalization_incident) {
|
|
$errorCode = strtolower(trim((string) (
|
|
$parent->finalization_error_code ?? ''
|
|
)));
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
preg_match(
|
|
'/^recovery_[a-z0-9_]{1,55}$/D',
|
|
$errorCode
|
|
) === 1 ? $errorCode : 'recovery_ledger_conflict',
|
|
time()
|
|
);
|
|
return;
|
|
}
|
|
$parent->lease_token = null;
|
|
$parent->lease_until = 0;
|
|
$parent->available_at = time() + self::RECOVERY_RETRY_SECONDS;
|
|
$parent->updated_at = time();
|
|
$parent->save();
|
|
return;
|
|
}
|
|
$partialMixed = $resolution === 'mixed' && !$metadataMixed;
|
|
if ($partialMixed
|
|
&& !self::hasPublishedPartialAggregateMarker($parent)) {
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
'recovery_aggregate_state_conflict',
|
|
time()
|
|
);
|
|
return;
|
|
}
|
|
$publishedResolution = $metadataMixed || $partialMixed
|
|
? 'applied' : $resolution;
|
|
if (!hash_equals(
|
|
(string) $parent->effect_resolution,
|
|
$publishedResolution
|
|
)) {
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
'recovery_aggregate_state_conflict',
|
|
time()
|
|
);
|
|
return;
|
|
}
|
|
try {
|
|
if (str_starts_with((string) $parent->kind, 'template_')) {
|
|
CloudTemplateCatalogService::
|
|
assertSettledParentProjectionLocked(
|
|
$parent,
|
|
$resolution
|
|
);
|
|
} else {
|
|
CloudUserIsoService::assertSettledParentProjectionLocked(
|
|
$parent,
|
|
$resolution
|
|
);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
self::deferAggregateRecoveryLocked(
|
|
$parent,
|
|
self::aggregateProjectionErrorCode($error),
|
|
time()
|
|
);
|
|
return;
|
|
}
|
|
$now = time();
|
|
$clearedIncident = (bool) $parent->finalization_incident;
|
|
if ($clearedIncident) {
|
|
$parent->finalization_incident = 0;
|
|
$parent->finalization_error_code = null;
|
|
$parent->read_at = null;
|
|
}
|
|
$parent->lease_token = null;
|
|
$parent->lease_until = 0;
|
|
$parent->available_at = $now + self::RECOVERY_RETRY_SECONDS;
|
|
$parent->updated_at = $now;
|
|
$parent->save();
|
|
}
|
|
|
|
/**
|
|
* Lock both catalog authorities in one order and classify their binding.
|
|
*
|
|
* @return 'valid'|'absent'|'conflict'
|
|
*/
|
|
private static function currentAggregateBindingStateLocked(
|
|
CloudJob $parent
|
|
): string {
|
|
$templates = CloudTemplate::query()
|
|
->where('active_job_id', (int) $parent->id)
|
|
->orderBy('id')->lockForUpdate()->get(['id']);
|
|
$isos = CloudUserIso::query()
|
|
->where('active_job_id', (int) $parent->id)
|
|
->orderBy('id')->lockForUpdate()->get(['id']);
|
|
if ($templates->isEmpty() && $isos->isEmpty()) {
|
|
$kind = (string) $parent->kind;
|
|
if (in_array($kind, [
|
|
'template_import', 'template_metadata_sync',
|
|
'template_delete',
|
|
], true)) {
|
|
CloudTemplateCatalogService::repairMissingParentBindingLocked(
|
|
$parent
|
|
);
|
|
} elseif (in_array($kind, [
|
|
'user_iso_import', 'user_iso_delete',
|
|
], true)) {
|
|
CloudUserIsoService::repairMissingParentBindingLocked($parent);
|
|
} else {
|
|
return 'absent';
|
|
}
|
|
$templates = CloudTemplate::query()
|
|
->where('active_job_id', (int) $parent->id)
|
|
->orderBy('id')->lockForUpdate()->get(['id']);
|
|
$isos = CloudUserIso::query()
|
|
->where('active_job_id', (int) $parent->id)
|
|
->orderBy('id')->lockForUpdate()->get(['id']);
|
|
}
|
|
if (str_starts_with((string) $parent->kind, 'template_')) {
|
|
return $templates->count() === 1 && $isos->isEmpty()
|
|
? 'valid' : 'conflict';
|
|
}
|
|
if (str_starts_with((string) $parent->kind, 'user_iso_')) {
|
|
return $isos->count() === 1 && $templates->isEmpty()
|
|
? 'valid' : 'conflict';
|
|
}
|
|
return 'conflict';
|
|
}
|
|
|
|
/** @return list<string> */
|
|
public static function aggregateProjectionIncidentCodes(): array
|
|
{
|
|
return [
|
|
CloudDispatchLedger::CONFLICT_CODE,
|
|
'recovery_child_convergence',
|
|
'recovery_aggregate_state_conflict',
|
|
'recovery_payload_unavailable',
|
|
'recovery_operation_identity_conflict',
|
|
'recovery_instance_state_conflict',
|
|
'recovery_ledger_conflict',
|
|
'recovery_internal_error',
|
|
];
|
|
}
|
|
|
|
private static function deferAggregateRecoveryLocked(
|
|
CloudJob $parent,
|
|
string $errorCode,
|
|
int $now
|
|
): void {
|
|
$parent->read_at = null;
|
|
$parent->finalization_incident = 1;
|
|
$parent->finalization_error_code = $errorCode;
|
|
$parent->lease_token = null;
|
|
$parent->lease_until = 0;
|
|
$parent->available_at = $now + self::RECOVERY_RETRY_SECONDS;
|
|
$parent->updated_at = $now;
|
|
$parent->save();
|
|
}
|
|
|
|
private static function aggregateProjectionErrorCode(
|
|
\Throwable $error
|
|
): string {
|
|
return match (true) {
|
|
$error instanceof CloudRecoveryDispatchLedgerConflict =>
|
|
CloudDispatchLedger::CONFLICT_CODE,
|
|
$error instanceof CloudRecoveryAggregateStateConflict =>
|
|
'recovery_aggregate_state_conflict',
|
|
$error instanceof CloudRecoveryPayloadUnavailable =>
|
|
'recovery_payload_unavailable',
|
|
$error instanceof CloudRecoveryOperationIdentityConflict =>
|
|
'recovery_operation_identity_conflict',
|
|
$error instanceof CloudRecoveryInstanceStateConflict =>
|
|
'recovery_instance_state_conflict',
|
|
$error instanceof CloudError => 'recovery_ledger_conflict',
|
|
default => 'recovery_internal_error',
|
|
};
|
|
}
|
|
|
|
public static function enqueue(CloudInstance $instance, array $actor,
|
|
string $kind, string $method, string $suffix, array $body = [],
|
|
?array $credential = null): CloudJob
|
|
{
|
|
CloudAccess::vmActor($instance, $actor);
|
|
if ($kind === 'network_change') {
|
|
throw new CloudError('旧公网网络写入口已停用', 409);
|
|
}
|
|
if (in_array($kind, ['delete', 'expiry_delete'], true)) {
|
|
CloudInstanceResourceRelease::assertCanDispatchDelete(
|
|
$instance
|
|
);
|
|
}
|
|
$requestKey = self::requestKey($actor);
|
|
return $instance->getConnection()->transaction(function () use (
|
|
$instance, $actor, $kind, $method, $suffix, $body, $credential,
|
|
$requestKey
|
|
) {
|
|
$templateMetadata = null;
|
|
$settingsFields = [];
|
|
if ($kind === 'reinstall') {
|
|
$body['template_id'] = CloudInput::template(
|
|
$body['template_id'] ?? ''
|
|
);
|
|
// Defense in depth for internal callers: this UI confirmation
|
|
// is not part of the strict Agent ReinstallRequest schema.
|
|
unset($body['confirmation_name']);
|
|
}
|
|
if ($kind === 'instance_settings') {
|
|
CloudInstanceSettingsGuard::assertSupported($instance);
|
|
CloudInput::only($body, ['hostname', 'root_password', 'ssh_public_key', 'confirm_restart']);
|
|
if (($body['confirm_restart'] ?? false) !== true) {
|
|
throw new CloudError('修改实例设置前必须确认虚拟机会短暂重启');
|
|
}
|
|
$changed = false;
|
|
if (array_key_exists('hostname', $body)) {
|
|
$body['hostname'] = CloudInput::hostname($body['hostname']);
|
|
$settingsFields[] = 'hostname';
|
|
$changed = true;
|
|
}
|
|
if (array_key_exists('root_password', $body)) {
|
|
if (!is_string($body['root_password']) || $body['root_password'] === '') {
|
|
throw new CloudError('实例密码不能为空');
|
|
}
|
|
$body['root_password'] = CloudInput::password($body['root_password']);
|
|
$settingsFields[] = 'password';
|
|
$changed = true;
|
|
}
|
|
if (array_key_exists('ssh_public_key', $body)) {
|
|
$parsed = CloudSSHKeys::parse((string) $body['ssh_public_key']);
|
|
$body['ssh_public_key'] = $parsed['public_key'];
|
|
$settingsFields[] = 'ssh_key';
|
|
$changed = true;
|
|
}
|
|
if (!$changed) {
|
|
throw new CloudError('请至少填写一项需要修改的实例设置');
|
|
}
|
|
}
|
|
$requestValue = [
|
|
'instance_id' => (int) $instance->id,
|
|
'kind' => $kind,
|
|
'method' => strtoupper($method),
|
|
'suffix' => $suffix,
|
|
'body' => $body,
|
|
];
|
|
$requestFingerprint = hash('sha256', json_encode(
|
|
self::canonicalRequestValue($requestValue),
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
));
|
|
$locked = CloudInstance::where('id', $instance->id)->lockForUpdate()->first();
|
|
if ($locked === null || (string) $locked->status === 'deleted') {
|
|
throw new CloudError('虚拟机不存在', 404);
|
|
}
|
|
if (in_array($kind, ['delete', 'expiry_delete'], true)) {
|
|
CloudInstanceResourceRelease::assertCanDispatchDelete(
|
|
$locked
|
|
);
|
|
}
|
|
if ($requestKey !== null) {
|
|
$prior = CloudJob::where('user_id', (int) $actor['user_id'])
|
|
->where('request_idempotency_key', $requestKey)
|
|
->lockForUpdate()->first();
|
|
if ($prior !== null) {
|
|
if (!hash_equals((string) $prior->request_fingerprint,
|
|
$requestFingerprint)) {
|
|
throw new CloudError(
|
|
'同一请求标识不能用于不同的虚拟机操作', 409
|
|
);
|
|
}
|
|
return $prior;
|
|
}
|
|
}
|
|
if ((string) $locked->status !== 'active') {
|
|
throw new CloudError('实例正在执行到期销毁流程,不能提交其他操作', 409);
|
|
}
|
|
CloudInstanceMigrationService::assertInstanceMutationAvailable(
|
|
$locked,
|
|
$kind
|
|
);
|
|
if ($kind === 'reinstall') {
|
|
CloudSnapshotService::assertReinstallAllowedLocked($locked);
|
|
$body = CloudTemplateCatalogService::
|
|
applyProvisioningSnapshotForHost(
|
|
$body,
|
|
(int) $locked->host_id,
|
|
false
|
|
);
|
|
$templateMetadata =
|
|
CloudTemplateCatalogService::templateMetadataSnapshot(
|
|
$body
|
|
);
|
|
}
|
|
$policyService = ['suspending_expired', 'resume_pending', 'resuming',
|
|
'suspend_failed', 'resume_failed', 'expired_grace',
|
|
'purge_stopping', 'purge_stop_failed', 'purge_force_required',
|
|
'purge_deleting',
|
|
'purge_failed'];
|
|
$policyNetwork = ['blocking', 'blocking_expired', 'blocked_expired',
|
|
'restore_pending', 'restoring', 'block_failed', 'restore_failed'];
|
|
$superAdmin = ($actor['is_super_admin'] ?? false)
|
|
&& ($actor['can_manage'] ?? false);
|
|
if ($kind !== 'delete' && !$superAdmin
|
|
&& (in_array((string) $locked->service_status, $policyService, true)
|
|
|| in_array((string) $locked->network_status, $policyNetwork, true)
|
|
|| ((string) $locked->service_status !== 'exempt'
|
|
&& ($locked->expires_at === null || (int) $locked->expires_at <= time())))) {
|
|
throw new CloudError('实例正在执行到期或流量策略,请稍后重试', 409);
|
|
}
|
|
$activeJobs = self::blockingQueryForInstance((int) $locked->id);
|
|
if ($activeJobs->exists()) {
|
|
throw new CloudError('虚拟机已有任务,请等待完成', 409);
|
|
}
|
|
$job = new CloudJob();
|
|
$job->operation_id = bin2hex(random_bytes(16));
|
|
$job->instance_id = $locked->id;
|
|
$job->user_id = $actor['user_id'];
|
|
$job->host_id = $locked->host_id;
|
|
$job->kind = $kind;
|
|
$job->execution_mode = 'agent';
|
|
$job->status = 'queued';
|
|
if ($requestKey !== null) {
|
|
$job->request_idempotency_key = $requestKey;
|
|
}
|
|
$job->request_fingerprint = $requestFingerprint;
|
|
$jobPayload = ['method' => $method,
|
|
'request_target' => '/v1/vms/' . rawurlencode($locked->uuid) . $suffix,
|
|
'body' => $body,
|
|
'credential' => $credential, 'owner_id' => (int) $locked->user_id,
|
|
'can_manage' => (bool) ($actor['can_manage'] ?? false)];
|
|
if ($settingsFields !== []) {
|
|
// This allow-listed metadata remains outside the strict Agent
|
|
// body and survives terminal secret redaction. Never persist a
|
|
// hostname, password or public-key value as presentation data.
|
|
$jobPayload['settings_fields'] = $settingsFields;
|
|
}
|
|
if ($templateMetadata !== null) {
|
|
// Encrypted panel metadata is deliberately outside body:
|
|
// body is the exact strict Go ReinstallRequest DTO.
|
|
$jobPayload['template_metadata'] = $templateMetadata;
|
|
}
|
|
$job->sealed_payload = CloudSecrets::encrypt(
|
|
$jobPayload, 'job-payload'
|
|
);
|
|
$job->available_at = $job->created_at = $job->updated_at = time();
|
|
$job->save();
|
|
return $job;
|
|
}, 3);
|
|
}
|
|
|
|
/**
|
|
* Queue the irreversible second phase of expiry cleanup.
|
|
*
|
|
* Only the lifecycle state machine may reach this method. Unlike the
|
|
* four target-state policy jobs, expiry deletion is intentionally never
|
|
* replayed after dispatch because an interrupted volume deletion may be
|
|
* only partly complete.
|
|
*/
|
|
public static function enqueueExpiryDelete(CloudInstance $instance): CloudJob
|
|
{
|
|
return $instance->getConnection()->transaction(function () use ($instance) {
|
|
$locked = CloudInstance::where('id', $instance->id)->lockForUpdate()->first();
|
|
if ($locked === null || (string) $locked->status === 'deleted') {
|
|
throw new CloudError('虚拟机不存在', 404);
|
|
}
|
|
if ((string) $locked->status !== 'deleting'
|
|
|| (string) $locked->service_status !== 'purge_deleting') {
|
|
throw new CloudError('实例尚未完成销毁前停机,不能删除磁盘', 409);
|
|
}
|
|
CloudInstanceResourceRelease::assertCanDispatchDelete(
|
|
$locked
|
|
);
|
|
CloudInstanceMigrationService::assertInstanceMutationAvailable(
|
|
$locked,
|
|
'expiry_delete'
|
|
);
|
|
$existing = self::blockingQueryForInstance((int) $locked->id)
|
|
->orderByDesc('id')->first();
|
|
if ($existing !== null) {
|
|
if ((string) $existing->kind === 'expiry_delete') {
|
|
return $existing;
|
|
}
|
|
throw new CloudError('虚拟机已有任务,请等待完成', 409);
|
|
}
|
|
|
|
$job = new CloudJob();
|
|
$job->operation_id = bin2hex(random_bytes(16));
|
|
$job->instance_id = $locked->id;
|
|
$job->user_id = $locked->user_id;
|
|
$job->host_id = $locked->host_id;
|
|
$job->kind = 'expiry_delete';
|
|
$job->execution_mode = 'agent';
|
|
$job->status = 'queued';
|
|
$job->sealed_payload = CloudSecrets::encrypt([
|
|
'method' => 'DELETE',
|
|
'request_target' => '/v1/vms/' . rawurlencode($locked->uuid),
|
|
'body' => ['remove_managed_disks' => true],
|
|
'owner_id' => (int) $locked->user_id,
|
|
'system_cleanup' => true,
|
|
], 'job-payload');
|
|
$job->available_at = $job->created_at = $job->updated_at = time();
|
|
$job->save();
|
|
return $job;
|
|
}, 3);
|
|
}
|
|
|
|
/**
|
|
* Queue the separately audited force phase of a delete operation.
|
|
*
|
|
* The original graceful operation remains the authorization anchor. It is
|
|
* never replayed or changed into a force request; the Agent validates that
|
|
* exact prior operation before accepting this new operation ID.
|
|
*/
|
|
public static function enqueueForceDelete(
|
|
CloudJob $prior,
|
|
array $actor
|
|
): CloudJob {
|
|
CloudAccess::manage($actor);
|
|
$priorId = filter_var($prior->id, FILTER_VALIDATE_INT,
|
|
['options' => ['min_range' => 1]]);
|
|
$instanceId = filter_var($prior->instance_id, FILTER_VALIDATE_INT,
|
|
['options' => ['min_range' => 1]]);
|
|
if ($priorId === false || $instanceId === false) {
|
|
throw new CloudError('当前任务没有关联实例', 409);
|
|
}
|
|
return $prior->getConnection()->transaction(function () use (
|
|
$priorId, $instanceId, $actor
|
|
) {
|
|
// Keep the global instance-mutation lock order: instance first,
|
|
// then jobs. The prior row is fully revalidated after both locks.
|
|
$instance = CloudInstance::where('id', (int) $instanceId)
|
|
->lockForUpdate()->first();
|
|
$lockedPrior = CloudJob::where('id', (int) $priorId)
|
|
->lockForUpdate()->first();
|
|
if ($lockedPrior === null) {
|
|
throw new CloudError('任务不存在', 404);
|
|
}
|
|
if ((bool) ($lockedPrior->finalization_incident ?? false)
|
|
|| self::hasTerminalRecoveryStateConflict($lockedPrior)) {
|
|
throw new CloudError(
|
|
'原删除任务的恢复账本异常尚未收敛,不能创建新的强制删除任务',
|
|
409
|
|
);
|
|
}
|
|
if (!in_array((string) $lockedPrior->kind,
|
|
['delete', 'expiry_delete', 'service_suspend'], true)
|
|
|| (string) $lockedPrior->phase !== 'terminal'
|
|
|| (string) $lockedPrior->outcome !== 'failed'
|
|
|| (string) $lockedPrior->error_code !== 'force_required'
|
|
|| !CloudDispatchLedger::isDispatched($lockedPrior)
|
|
|| $lockedPrior->dispatched_at === null
|
|
|| preg_match('/^[a-f0-9]{32}$/D',
|
|
(string) $lockedPrior->operation_id) !== 1
|
|
|| (int) $lockedPrior->instance_id !== (int) $instanceId) {
|
|
throw new CloudError('当前任务没有经过确认的强制删除前置状态', 409);
|
|
}
|
|
|
|
if ($instance === null || (string) $instance->status === 'deleted'
|
|
|| (int) $instance->host_id !== (int) $lockedPrior->host_id
|
|
|| preg_match('/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD',
|
|
(string) $instance->uuid) !== 1) {
|
|
throw new CloudError('关联实例已不存在或与原删除任务不一致', 409);
|
|
}
|
|
CloudInstanceResourceRelease::assertCanDispatchDelete(
|
|
$instance
|
|
);
|
|
|
|
$priorPayload = CloudSecrets::decrypt(
|
|
(string) $lockedPrior->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$target = '/v1/vms/' . rawurlencode((string) $instance->uuid);
|
|
$gracefulDelete = ($priorPayload['method'] ?? null) === 'DELETE'
|
|
&& ($priorPayload['request_target'] ?? null) === $target
|
|
&& ($priorPayload['body'] ?? null)
|
|
=== ['remove_managed_disks' => true];
|
|
$expiryGracefulStop = (string) $lockedPrior->kind
|
|
=== 'service_suspend'
|
|
&& ($priorPayload['method'] ?? null) === 'POST'
|
|
&& ($priorPayload['request_target'] ?? null)
|
|
=== $target . '/service/suspend'
|
|
&& ($priorPayload['body'] ?? null) === ['reason' => 'purge']
|
|
&& ($priorPayload['system_policy'] ?? false) === true
|
|
&& (string) $instance->status === 'deleting'
|
|
&& (string) $instance->service_status
|
|
=== 'purge_force_required';
|
|
if (!$gracefulDelete && !$expiryGracefulStop) {
|
|
throw new CloudError('原删除任务不允许强制删除面板管理磁盘', 409);
|
|
}
|
|
|
|
$otherBlocker = self::blockingQueryForInstance((int) $instance->id)
|
|
->where('id', '<>', (int) $lockedPrior->id)
|
|
->lockForUpdate()->first();
|
|
if ($otherBlocker !== null) {
|
|
throw new CloudError('虚拟机还有其他未完成任务,不能发起强制删除', 409);
|
|
}
|
|
|
|
do {
|
|
$operationId = bin2hex(random_bytes(16));
|
|
} while (hash_equals((string) $lockedPrior->operation_id, $operationId));
|
|
|
|
$job = new CloudJob();
|
|
$job->operation_id = $operationId;
|
|
$job->instance_id = $instance->id;
|
|
$job->user_id = (int) ($actor['user_id'] ?? 0);
|
|
$job->host_id = $instance->host_id;
|
|
$job->kind = 'delete_force';
|
|
$job->execution_mode = 'agent';
|
|
$job->status = 'queued';
|
|
$job->sealed_payload = CloudSecrets::encrypt([
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/vms/'
|
|
. rawurlencode((string) $instance->uuid) . '/force-delete',
|
|
'body' => [
|
|
'operation_id' => $operationId,
|
|
'prior_operation_id' => (string) $lockedPrior->operation_id,
|
|
'ack_data_loss' => true,
|
|
'remove_managed_disks' => true,
|
|
],
|
|
'owner_id' => (int) $instance->user_id,
|
|
'can_manage' => true,
|
|
'prior_job_id' => (int) $lockedPrior->id,
|
|
'prior_operation_id' => (string) $lockedPrior->operation_id,
|
|
], 'job-payload');
|
|
$now = time();
|
|
$job->available_at = $job->created_at = $job->updated_at = $now;
|
|
$job->save();
|
|
|
|
error_log(sprintf(
|
|
'cloud admin queued force delete prior_job_id=%d force_job_id=%d prior_operation_id=%s force_operation_id=%s actor_id=%d',
|
|
(int) $lockedPrior->id, (int) $job->id,
|
|
(string) $lockedPrior->operation_id, $operationId,
|
|
(int) ($actor['user_id'] ?? 0)
|
|
));
|
|
return $job;
|
|
}, 3);
|
|
}
|
|
|
|
public static function display(CloudJob $job, bool $administrator = false): array
|
|
{
|
|
$completed = (string) $job->phase === 'terminal'
|
|
&& in_array((string) $job->outcome,
|
|
['succeeded', 'failed'], true);
|
|
$action = self::action($job);
|
|
$labels = CloudOperationCatalog::labels((string) $job->kind, $action,
|
|
self::settingsFields($job));
|
|
$base = [
|
|
'id' => (string) $job->id,
|
|
'operation_id' => (string) $job->operation_id,
|
|
'kind' => (string) $job->kind,
|
|
'completed' => $completed,
|
|
'created_at' => max(0, (int) $job->created_at),
|
|
'updated_at' => max(0, (int) $job->updated_at),
|
|
'status_url' => '/api/v1/tasks/' . (int) $job->id,
|
|
'action' => $action,
|
|
'kind_label' => $labels['kind_label'],
|
|
'action_label' => $labels['action_label'],
|
|
];
|
|
if (!$completed) {
|
|
$recovery = self::recoveryPresentation($job, $administrator);
|
|
if ($recovery !== null) {
|
|
$base['recovery'] = $recovery;
|
|
}
|
|
return $base;
|
|
}
|
|
$result = $job->result_json === null ? null : json_decode($job->result_json, true);
|
|
$status = (string) $job->outcome;
|
|
$safeResult = (string) $job->kind === 'payment_create'
|
|
&& $status === 'succeeded' && is_array($result)
|
|
? self::displayPaymentResult($job, $result)
|
|
: self::displayResult(is_array($result) ? $result : null,
|
|
$administrator, (string) $job->kind, $status,
|
|
$job->order_id !== null, $action);
|
|
$error = $administrator
|
|
? CloudOperationErrorCatalog::adminSummary((string) $job->kind,
|
|
$status, $job->error_code, $job->order_id !== null, $action)
|
|
: CloudOperationErrorCatalog::userMessage((string) $job->kind,
|
|
$status, $job->error_code, $job->order_id !== null, $action);
|
|
$phase = $safeResult['phase'] ?? null;
|
|
$phase = $administrator
|
|
? CloudOperationErrorCatalog::adminSummary((string) $job->kind,
|
|
$status, $phase, $job->order_id !== null, $action)
|
|
: CloudOperationErrorCatalog::userPhase($phase, $status, (string) $job->kind);
|
|
$terminal = [
|
|
'status' => $status, 'error' => $error,
|
|
'error_code' => CloudOperationErrorCatalog::publicCode($job->error_code),
|
|
'phase' => is_string($phase) ? mb_substr($phase, 0, 120) : null,
|
|
'result' => $safeResult,
|
|
'effect_may_have_applied' => $job->effect_may_have_applied === null
|
|
? null : (bool) $job->effect_may_have_applied,
|
|
'safe_to_resubmit' => $job->safe_to_resubmit === null
|
|
? null : (bool) $job->safe_to_resubmit,
|
|
'unread' => $job->read_at === null,
|
|
'terminal_at' => max(0, (int) $job->terminal_at),
|
|
];
|
|
$effectResolution = (string) $job->effect_resolution;
|
|
if ($status === 'failed' && (bool) $job->effect_may_have_applied
|
|
&& in_array($effectResolution, ['applied', 'not_applied'], true)) {
|
|
// This is deliberately not a generic recovery/audit field. The
|
|
// public contract exposes only the two conclusions needed to stop
|
|
// presenting an already-released resource fence as active.
|
|
$terminal['effect_resolution'] = $effectResolution;
|
|
}
|
|
$recovery = self::recoveryPresentation($job, $administrator);
|
|
if ($recovery !== null) {
|
|
$terminal['recovery'] = $recovery;
|
|
} else {
|
|
$effectScope = self::effectScope($job);
|
|
if ($effectScope !== null) {
|
|
$terminal['effect_scope'] = $effectScope;
|
|
}
|
|
}
|
|
return array_merge($base, $terminal);
|
|
}
|
|
|
|
/**
|
|
* Distinguish a fully proven mixed aggregate from an all-host applied
|
|
* result without exposing the internal convergence reason to clients.
|
|
*/
|
|
public static function effectScope(object $job): ?string
|
|
{
|
|
return self::hasPublishedPartialAggregateMarker($job)
|
|
&& !(bool) ($job->finalization_incident ?? false)
|
|
? 'partial' : null;
|
|
}
|
|
|
|
/** Canonical mixed publication tuple, including rows under recheck. */
|
|
public static function hasPublishedPartialAggregateMarker(
|
|
object $job
|
|
): bool {
|
|
$reason = (string) ($job->effect_resolution_reason ?? '');
|
|
return (string) ($job->execution_mode ?? '') === 'aggregate'
|
|
&& ($job->parent_job_id ?? null) === null
|
|
&& in_array((string) ($job->kind ?? ''), [
|
|
'template_import',
|
|
'template_delete',
|
|
'user_iso_import',
|
|
'user_iso_delete',
|
|
], true)
|
|
&& !self::hasAggregateLocalStateConflict($job)
|
|
&& (string) ($job->phase ?? '') === 'terminal'
|
|
&& (string) ($job->outcome ?? '') === 'failed'
|
|
&& (string) ($job->status ?? '') === 'failed'
|
|
&& (bool) ($job->effect_may_have_applied ?? false)
|
|
&& !(bool) ($job->safe_to_resubmit ?? true)
|
|
&& (string) ($job->effect_resolution ?? '') === 'applied'
|
|
&& (int) ($job->effect_resolved_at ?? 0) > 0
|
|
&& preg_match(
|
|
'/^(?:legacy_)?child_effects_resolved:[1-9][0-9]*:mixed$/D',
|
|
$reason
|
|
) === 1;
|
|
}
|
|
|
|
/**
|
|
* Return a bounded recovery status for panels. Arbitrary server/Agent
|
|
* exception text never crosses this DTO; unknown recovery codes collapse
|
|
* to one reviewed message.
|
|
*/
|
|
public static function recoveryPresentation(
|
|
CloudJob $job,
|
|
bool $administrator = false
|
|
): ?array {
|
|
$mode = self::recoveryMode($job);
|
|
if ($mode === null) {
|
|
return null;
|
|
}
|
|
if ($mode === 'aggregate') {
|
|
$details = self::aggregateRecoveryDetails($job);
|
|
$code = $details['code'];
|
|
$message = $details['message'];
|
|
} else {
|
|
$candidate = self::agentRecoveryPresentationCandidate($job);
|
|
if ($mode === 'local_finalizer'
|
|
&& in_array(
|
|
$candidate,
|
|
CloudDispatchLedger::repairIncidentCodes(),
|
|
true
|
|
)) {
|
|
$candidate = 'recovery_ledger_conflict';
|
|
}
|
|
$code = array_key_exists($candidate, self::RECOVERY_MESSAGES)
|
|
? $candidate : ($mode === 'local_finalizer'
|
|
? 'recovery_ledger_conflict' : 'recovery_pending');
|
|
$message = self::RECOVERY_MESSAGES[$code];
|
|
}
|
|
$nextCheckAt = max(0, (int) ($job->available_at ?? 0));
|
|
$lastCheckedAt = max(0, (int) ($job->updated_at ?? 0));
|
|
if ($mode === 'aggregate') {
|
|
$nextCheckAt = max($nextCheckAt, $details['next_check_at']);
|
|
$lastCheckedAt = max(
|
|
$lastCheckedAt,
|
|
$details['last_checked_at']
|
|
);
|
|
}
|
|
$result = [
|
|
'pending' => true,
|
|
'message' => $message,
|
|
'last_checked_at' => $lastCheckedAt > 0
|
|
? $lastCheckedAt : null,
|
|
'next_check_at' => $nextCheckAt > time()
|
|
? $nextCheckAt : null,
|
|
];
|
|
if ($administrator) {
|
|
$result['code'] = $code;
|
|
if ($mode === 'aggregate') {
|
|
$result['affected_child_count'] =
|
|
$details['affected_child_count'];
|
|
$result['additional_error_count'] =
|
|
$details['additional_error_count'];
|
|
}
|
|
}
|
|
return $result;
|
|
}
|
|
|
|
/** Select the most actionable reviewed cause without exposing raw text. */
|
|
private static function agentRecoveryPresentationCandidate(
|
|
CloudJob $job
|
|
): string {
|
|
if (CloudDispatchLedger::isInvalid($job)) {
|
|
return CloudDispatchLedger::CONFLICT_CODE;
|
|
}
|
|
$stored = strtolower(trim(
|
|
(string) ($job->finalization_error_code ?? '')
|
|
));
|
|
if (CloudDispatchLedger::hasRepairIncident($job)
|
|
&& array_key_exists($stored, self::RECOVERY_MESSAGES)) {
|
|
return $stored;
|
|
}
|
|
if (self::hasAgentRecoveryIdentityConflict($job)) {
|
|
return 'recovery_operation_identity_conflict';
|
|
}
|
|
if (in_array($stored, [
|
|
'recovery_operation_identity_conflict',
|
|
'recovery_receipt_invalid',
|
|
'recovery_completion_outbox_invalid',
|
|
'recovery_completion_receipt_missing',
|
|
'recovery_completion_receipt_invalid',
|
|
'recovery_completion_operation_missing',
|
|
'recovery_completion_operation_invalid',
|
|
'recovery_completion_identity_mismatch',
|
|
'recovery_payload_invalid',
|
|
'recovery_payload_unavailable',
|
|
], true)) {
|
|
return $stored;
|
|
}
|
|
if (self::hasTerminalRecoveryStateConflict($job)) {
|
|
return 'recovery_instance_state_conflict';
|
|
}
|
|
return $stored;
|
|
}
|
|
|
|
private static function recoveryMode(CloudJob $job): ?string
|
|
{
|
|
$kind = (string) $job->kind;
|
|
$aggregateIdentity = (string) $job->execution_mode === 'aggregate'
|
|
|| in_array($kind, self::AGGREGATE_KINDS, true);
|
|
if ($aggregateIdentity) {
|
|
$ownFence = (string) $job->phase === 'terminal'
|
|
&& (string) $job->outcome === 'failed'
|
|
&& (string) $job->status === 'failed'
|
|
&& (bool) $job->effect_may_have_applied
|
|
&& !(bool) $job->safe_to_resubmit
|
|
&& (string) $job->effect_resolution === 'unknown';
|
|
$stickyIncident = (string) $job->phase === 'terminal'
|
|
&& (bool) $job->finalization_incident;
|
|
if ($ownFence || $stickyIncident
|
|
|| self::hasTerminalRecoveryStateConflict($job)
|
|
|| self::hasUnresolvedRecoverableChildren($job)) {
|
|
return 'aggregate';
|
|
}
|
|
return null;
|
|
}
|
|
if ((string) $job->execution_mode === 'agent'
|
|
&& CloudDispatchLedger::isInvalid($job)) {
|
|
return 'agent';
|
|
}
|
|
if (CloudDispatchLedger::hasRepairIncident($job)) {
|
|
return 'agent';
|
|
}
|
|
if (self::isLocalFinalizerRecovery($job)) {
|
|
return 'local_finalizer';
|
|
}
|
|
if (self::isResolvedStickyAgentIncident($job)) {
|
|
return 'agent';
|
|
}
|
|
if (self::hasTerminalRecoveryStateConflict($job)) {
|
|
return 'agent';
|
|
}
|
|
if ((string) $job->phase !== 'terminal'
|
|
|| (string) $job->outcome !== 'failed'
|
|
|| (string) $job->status !== 'failed'
|
|
|| !(bool) $job->effect_may_have_applied
|
|
|| (bool) $job->safe_to_resubmit
|
|
|| (string) $job->effect_resolution !== 'unknown') {
|
|
return null;
|
|
}
|
|
if ((string) $job->execution_mode === 'agent'
|
|
&& (int) $job->dispatch_attempts === 1
|
|
&& CloudOperationRecoveryPolicy::supportsLedgerFinalizer($kind)) {
|
|
return 'agent';
|
|
}
|
|
return null;
|
|
}
|
|
|
|
private static function hasUnresolvedRecoverableChildren(
|
|
CloudJob $parent
|
|
): bool {
|
|
if ((int) $parent->id < 1) {
|
|
return false;
|
|
}
|
|
return CloudJob::where('parent_job_id', (int) $parent->id)
|
|
->whereIn('kind', CloudOperationRecoveryPolicy::kinds())
|
|
->where(static function ($pending): void {
|
|
$pending->where(static function ($mode): void {
|
|
$mode->whereNull('execution_mode')
|
|
->orWhere('execution_mode', '<>', 'agent');
|
|
})->orWhere(static function ($invalid): void {
|
|
$invalid->whereNull('dispatch_attempts')
|
|
->orWhereNotIn('dispatch_attempts', [0, 1]);
|
|
})->orWhere(static function ($repair): void {
|
|
$repair->where('phase', 'terminal')
|
|
->where('finalization_incident', 1)
|
|
->whereIn(
|
|
'finalization_error_code',
|
|
CloudDispatchLedger::repairIncidentCodes()
|
|
);
|
|
})->orWhere(static function ($local): void {
|
|
$local->where('phase', 'terminal')
|
|
->where('outcome', 'failed')
|
|
->where('status', 'failed')
|
|
->where('dispatch_attempts', 0)
|
|
->where('finalization_incident', 1);
|
|
})->orWhere(static function ($sticky): void {
|
|
$sticky->where('phase', 'terminal')
|
|
->where('finalization_incident', 1);
|
|
})
|
|
->orWhere(static function ($normal): void {
|
|
$normal->where('phase', 'terminal')
|
|
->where('outcome', 'failed')
|
|
->where('status', 'failed')
|
|
->where('dispatch_attempts', 1)
|
|
->where('effect_may_have_applied', 1)
|
|
->where('safe_to_resubmit', 0)
|
|
->where('effect_resolution', 'unknown');
|
|
})->orWhere(static function ($state): void {
|
|
$state->where('phase', 'terminal')
|
|
->whereIn('dispatch_attempts', [0, 1])
|
|
->where('effect_resolution', 'unknown')
|
|
->where(static function ($invalid): void {
|
|
self::applyInvalidTerminalTuple($invalid);
|
|
});
|
|
})->orWhere(static function ($phase): void {
|
|
$phase->where('phase', '<>', 'terminal')
|
|
->whereIn(
|
|
'effect_resolution',
|
|
['applied', 'not_applied']
|
|
);
|
|
})->orWhere(static function ($resolved): void {
|
|
$resolved->where('phase', 'terminal')
|
|
->whereIn('dispatch_attempts', [0, 1])
|
|
->where(static function ($invalid): void {
|
|
self::applyInvalidResolvedTerminalTuple(
|
|
$invalid
|
|
);
|
|
});
|
|
});
|
|
})
|
|
->exists();
|
|
}
|
|
|
|
private static function unresolvedRecoverableChildExists($children): void
|
|
{
|
|
$children->selectRaw('1')
|
|
->from('cloud_jobs as recovery_children')
|
|
->whereColumn(
|
|
'recovery_children.parent_job_id',
|
|
'cloud_jobs.id'
|
|
)
|
|
->whereIn(
|
|
'recovery_children.kind',
|
|
CloudOperationRecoveryPolicy::kinds()
|
|
)->where(static function ($pending): void {
|
|
$pending->where(static function ($mode): void {
|
|
$mode->whereNull('recovery_children.execution_mode')
|
|
->orWhere(
|
|
'recovery_children.execution_mode',
|
|
'<>',
|
|
'agent'
|
|
);
|
|
})->orWhere(static function ($invalid): void {
|
|
$invalid->whereNull(
|
|
'recovery_children.dispatch_attempts'
|
|
)->orWhereNotIn(
|
|
'recovery_children.dispatch_attempts',
|
|
[0, 1]
|
|
);
|
|
})->orWhere(static function ($repair): void {
|
|
$repair->where(
|
|
'recovery_children.phase',
|
|
'terminal'
|
|
)->where(
|
|
'recovery_children.finalization_incident',
|
|
1
|
|
)->whereIn(
|
|
'recovery_children.finalization_error_code',
|
|
CloudDispatchLedger::repairIncidentCodes()
|
|
);
|
|
})->orWhere(static function ($local): void {
|
|
$local->where(
|
|
'recovery_children.phase',
|
|
'terminal'
|
|
)->where(
|
|
'recovery_children.outcome',
|
|
'failed'
|
|
)->where(
|
|
'recovery_children.status',
|
|
'failed'
|
|
)->where(
|
|
'recovery_children.dispatch_attempts',
|
|
0
|
|
)->where(
|
|
'recovery_children.finalization_incident',
|
|
1
|
|
);
|
|
})->orWhere(static function ($sticky): void {
|
|
$sticky->where(
|
|
'recovery_children.phase',
|
|
'terminal'
|
|
)->where(
|
|
'recovery_children.finalization_incident',
|
|
1
|
|
);
|
|
})->orWhere(static function ($normal): void {
|
|
$normal->where(
|
|
'recovery_children.phase',
|
|
'terminal'
|
|
)->where(
|
|
'recovery_children.outcome',
|
|
'failed'
|
|
)->where(
|
|
'recovery_children.status',
|
|
'failed'
|
|
)->where(
|
|
'recovery_children.dispatch_attempts',
|
|
1
|
|
)->where(
|
|
'recovery_children.effect_may_have_applied',
|
|
1
|
|
)->where(
|
|
'recovery_children.safe_to_resubmit',
|
|
0
|
|
)->where(
|
|
'recovery_children.effect_resolution',
|
|
'unknown'
|
|
);
|
|
})->orWhere(static function ($state): void {
|
|
$state->where(
|
|
'recovery_children.phase',
|
|
'terminal'
|
|
)->whereIn(
|
|
'recovery_children.dispatch_attempts',
|
|
[0, 1]
|
|
)->where(
|
|
'recovery_children.effect_resolution',
|
|
'unknown'
|
|
)->where(static function ($invalid): void {
|
|
self::applyInvalidTerminalTuple(
|
|
$invalid,
|
|
'recovery_children.'
|
|
);
|
|
});
|
|
})->orWhere(static function ($phase): void {
|
|
$phase->where(
|
|
'recovery_children.phase',
|
|
'<>',
|
|
'terminal'
|
|
)->whereIn(
|
|
'recovery_children.effect_resolution',
|
|
['applied', 'not_applied']
|
|
);
|
|
})->orWhere(static function ($resolved): void {
|
|
$resolved->where(
|
|
'recovery_children.phase',
|
|
'terminal'
|
|
)->whereIn(
|
|
'recovery_children.dispatch_attempts',
|
|
[0, 1]
|
|
)->where(static function ($invalid): void {
|
|
self::applyInvalidResolvedTerminalTuple(
|
|
$invalid,
|
|
'recovery_children.'
|
|
);
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Select one deterministic, reviewed child incident for the public batch.
|
|
* Raw Agent/exception text and child result payloads never enter this DTO.
|
|
*
|
|
* @return array{code:string,message:string,affected_child_count:int,additional_error_count:int,last_checked_at:int,next_check_at:int}
|
|
*/
|
|
private static function aggregateRecoveryDetails(CloudJob $parent): array
|
|
{
|
|
if (!$parent->exists) {
|
|
// display() is also used with detached, read-only task snapshots
|
|
// (for example notification/rendering contracts). Only persisted
|
|
// aggregate jobs have child rows that can be safely summarized.
|
|
return [
|
|
'code' => 'recovery_child_convergence',
|
|
'message' => self::RECOVERY_MESSAGES[
|
|
'recovery_child_convergence'
|
|
],
|
|
'affected_child_count' => 0,
|
|
'additional_error_count' => 0,
|
|
'last_checked_at' => max(0, (int) $parent->updated_at),
|
|
'next_check_at' => max(0, (int) $parent->available_at),
|
|
];
|
|
}
|
|
$children = CloudJob::where('parent_job_id', (int) $parent->id)
|
|
->orderBy('id')
|
|
->get([
|
|
'id', 'operation_id', 'request_idempotency_key',
|
|
'request_fingerprint', 'parent_job_id', 'instance_id',
|
|
'user_id', 'host_id', 'kind', 'execution_mode',
|
|
'phase', 'outcome', 'status', 'attempts',
|
|
'dispatch_attempts', 'dispatch_deadline_at',
|
|
'dispatch_started_at', 'agent_accepted_at', 'dispatched_at',
|
|
'receipt_deadline_at', 'terminal_at', 'completion_event_id',
|
|
'error_code', 'result_json',
|
|
'effect_may_have_applied', 'safe_to_resubmit',
|
|
'effect_resolution', 'finalization_incident',
|
|
'finalization_error_code',
|
|
'available_at', 'updated_at',
|
|
]);
|
|
$expectedKind = self::AGGREGATE_CHILD_KIND_MAP[
|
|
(string) $parent->kind
|
|
] ?? null;
|
|
$parentStateConflict = self::hasAggregateLocalStateConflict($parent)
|
|
|| (string) ($parent->finalization_error_code ?? '')
|
|
=== 'recovery_aggregate_state_conflict';
|
|
$parentCandidate = $parentStateConflict
|
|
? 'recovery_aggregate_state_conflict'
|
|
: strtolower(trim(
|
|
(string) ($parent->finalization_error_code ?? '')
|
|
));
|
|
$parentCode = array_key_exists(
|
|
$parentCandidate,
|
|
self::RECOVERY_MESSAGES
|
|
) ? $parentCandidate : null;
|
|
if ($parentCode === null && (bool) $parent->finalization_incident) {
|
|
$parentCode = 'recovery_ledger_conflict';
|
|
}
|
|
$relationshipConflict = $parentCandidate
|
|
=== 'recovery_aggregate_state_conflict'
|
|
|| $expectedKind === null
|
|
|| $children->isEmpty()
|
|
|| $children->contains(
|
|
static fn (CloudJob $child): bool =>
|
|
(int) $child->user_id !== (int) $parent->user_id
|
|
|| (string) $child->execution_mode !== 'agent'
|
|
|| (string) $child->kind !== $expectedKind
|
|
|| self::aggregateChildHasPhaseResolutionConflict($child)
|
|
);
|
|
$unresolved = $children->filter(
|
|
static fn (CloudJob $child): bool =>
|
|
(string) $child->execution_mode === 'agent'
|
|
&& CloudOperationRecoveryPolicy::supports(
|
|
(string) $child->kind
|
|
)
|
|
&& (CloudDispatchLedger::isInvalid($child)
|
|
|| (bool) $child->finalization_incident
|
|
|| CloudDispatchLedger::hasRepairIncident($child)
|
|
|| self::isLocalFinalizerRecovery($child)
|
|
|| self::hasTerminalRecoveryStateConflict($child)
|
|
|| self::aggregateChildHasPhaseResolutionConflict($child)
|
|
|| self::hasResolvedTerminalStateConflict($child)
|
|
|| ((string) $child->phase === 'terminal'
|
|
&& (string) $child->outcome === 'failed'
|
|
&& (string) $child->status === 'failed'
|
|
&& CloudDispatchLedger::isDispatched($child)
|
|
&& (bool) $child->effect_may_have_applied
|
|
&& !(bool) $child->safe_to_resubmit
|
|
&& (string) $child->effect_resolution === 'unknown'))
|
|
);
|
|
$lastCheckedAt = (int) ($unresolved->max('updated_at') ?? 0);
|
|
$nextCheckAt = (int) ($unresolved->max('available_at') ?? 0);
|
|
$priority = [
|
|
'recovery_dispatch_ledger_conflict',
|
|
'recovery_dispatch_repair_completion_receipt_missing',
|
|
'recovery_dispatch_repair_receipt_invalid',
|
|
'recovery_dispatch_repair_aggregate_state_conflict',
|
|
'recovery_dispatch_repair_payload_unavailable',
|
|
'recovery_dispatch_repair_operation_identity_conflict',
|
|
'recovery_dispatch_repair_evidence_conflict',
|
|
'recovery_dispatch_repair_instance_state_conflict',
|
|
'recovery_dispatch_repair_ledger_conflict',
|
|
'recovery_dispatch_repair_internal_error',
|
|
'recovery_operation_identity_conflict',
|
|
'recovery_receipt_invalid',
|
|
'recovery_completion_identity_mismatch',
|
|
'recovery_completion_receipt_invalid',
|
|
'recovery_completion_receipt_missing',
|
|
'recovery_completion_operation_invalid',
|
|
'recovery_completion_operation_missing',
|
|
'recovery_completion_outbox_invalid',
|
|
'recovery_payload_unavailable',
|
|
'recovery_payload_invalid',
|
|
'recovery_instance_state_conflict',
|
|
'recovery_evidence_conflict',
|
|
'recovery_evidence_invalid',
|
|
'recovery_state_invalid',
|
|
'recovery_authentication_failed',
|
|
'recovery_agent_unavailable',
|
|
'recovery_waiting_connection',
|
|
'recovery_operation_conflict',
|
|
'recovery_operation_missing',
|
|
'recovery_evidence_partial',
|
|
'recovery_evidence_inconclusive',
|
|
'recovery_ledger_conflict',
|
|
'recovery_internal_error',
|
|
'recovery_pending',
|
|
];
|
|
$codes = $parentCode === null
|
|
|| $parentCode === 'recovery_child_convergence'
|
|
? [] : [$parentCode];
|
|
if ($relationshipConflict) {
|
|
$codes[] = 'recovery_aggregate_state_conflict';
|
|
}
|
|
foreach ($unresolved as $child) {
|
|
$candidate = self::aggregateChildHasPhaseResolutionConflict($child)
|
|
? 'recovery_aggregate_state_conflict'
|
|
: self::agentRecoveryPresentationCandidate($child);
|
|
if (self::isLocalFinalizerRecovery($child)
|
|
&& in_array(
|
|
$candidate,
|
|
CloudDispatchLedger::repairIncidentCodes(),
|
|
true
|
|
)) {
|
|
$candidate = 'recovery_ledger_conflict';
|
|
}
|
|
$codes[] = array_key_exists($candidate, self::RECOVERY_MESSAGES)
|
|
? $candidate : 'recovery_pending';
|
|
}
|
|
$codes = array_values(array_unique($codes));
|
|
usort($codes, static function (string $left, string $right) use (
|
|
$priority
|
|
): int {
|
|
$leftPriority = array_search($left, $priority, true);
|
|
$rightPriority = array_search($right, $priority, true);
|
|
$leftPriority = $leftPriority === false
|
|
? PHP_INT_MAX : $leftPriority;
|
|
$rightPriority = $rightPriority === false
|
|
? PHP_INT_MAX : $rightPriority;
|
|
return $leftPriority <=> $rightPriority
|
|
?: strcmp($left, $right);
|
|
});
|
|
$concreteParentCode = $parentCode !== null
|
|
&& $parentCode !== 'recovery_child_convergence'
|
|
? $parentCode : null;
|
|
$code = $concreteParentCode
|
|
?? ($codes[0] ?? ($parentCode ?? 'recovery_child_convergence'));
|
|
$parentLevelCode = $concreteParentCode !== null
|
|
|| ($relationshipConflict
|
|
&& $code === 'recovery_aggregate_state_conflict');
|
|
$message = $parentLevelCode || $code === 'recovery_child_convergence'
|
|
? self::RECOVERY_MESSAGES[$code]
|
|
: '某个宿主机子任务:' . self::RECOVERY_MESSAGES[$code];
|
|
$additional = max(0, count($codes) - 1);
|
|
if ($additional > 0) {
|
|
$message .= ';另有 ' . $additional
|
|
. ' 类子任务异常仍在持续自动核对。';
|
|
}
|
|
return [
|
|
'code' => $code,
|
|
'message' => $message,
|
|
'affected_child_count' => $relationshipConflict
|
|
? max(1, $unresolved->count()) : $unresolved->count(),
|
|
'additional_error_count' => $additional,
|
|
'last_checked_at' => $lastCheckedAt,
|
|
'next_check_at' => $nextCheckAt,
|
|
];
|
|
}
|
|
|
|
private static function applyRecoveringFilter($query): void
|
|
{
|
|
$query->where(static function ($modes): void {
|
|
$modes->where(static function ($leaf): void {
|
|
$leaf->whereIn('kind',
|
|
CloudOperationRecoveryPolicy::ledgerFinalizerKinds())
|
|
->where(static function ($pending): void {
|
|
$pending->where(static function ($mode): void {
|
|
$mode->whereNull('execution_mode')
|
|
->orWhere('execution_mode', '<>', 'agent');
|
|
})->orWhere(static function ($invalid): void {
|
|
$invalid->whereNull('dispatch_attempts')
|
|
->orWhereNotIn(
|
|
'dispatch_attempts',
|
|
[0, 1]
|
|
);
|
|
})->orWhere(static function ($repair): void {
|
|
$repair->where('phase', 'terminal')
|
|
->where('finalization_incident', 1)
|
|
->whereIn(
|
|
'finalization_error_code',
|
|
CloudDispatchLedger::repairIncidentCodes()
|
|
);
|
|
})->orWhere(static function ($local): void {
|
|
$local->where('phase', 'terminal')
|
|
->where('outcome', 'failed')
|
|
->where('status', 'failed')
|
|
->where('dispatch_attempts', 0)
|
|
->where('finalization_incident', 1);
|
|
})->orWhere(static function ($resolved): void {
|
|
$resolved->where('phase', 'terminal')
|
|
->whereIn('dispatch_attempts', [0, 1])
|
|
->whereIn(
|
|
'effect_resolution',
|
|
['applied', 'not_applied']
|
|
)->where('finalization_incident', 1);
|
|
})->orWhere(static function ($normal): void {
|
|
$normal->where('phase', 'terminal')
|
|
->where('outcome', 'failed')
|
|
->where('status', 'failed')
|
|
->where('effect_may_have_applied', 1)
|
|
->where('safe_to_resubmit', 0)
|
|
->where('effect_resolution', 'unknown')
|
|
->where('dispatch_attempts', 1);
|
|
})->orWhere(static function ($state): void {
|
|
$state->where('phase', 'terminal')
|
|
->whereIn('dispatch_attempts', [0, 1])
|
|
->where('effect_resolution', 'unknown')
|
|
->where(static function ($invalid): void {
|
|
self::applyInvalidTerminalTuple($invalid);
|
|
});
|
|
})->orWhere(static function ($resolvedState): void {
|
|
$resolvedState->where('phase', 'terminal')
|
|
->whereIn('dispatch_attempts', [0, 1])
|
|
->where(static function ($invalid): void {
|
|
self::applyInvalidResolvedTerminalTuple(
|
|
$invalid
|
|
);
|
|
});
|
|
});
|
|
});
|
|
})->orWhere(static function ($aggregate): void {
|
|
$aggregate->where('execution_mode', 'aggregate')
|
|
->whereIn('kind', self::AGGREGATE_KINDS)
|
|
->where(static function ($pending): void {
|
|
$pending->where(static function ($ledger): void {
|
|
$ledger->whereNull('dispatch_attempts')
|
|
->orWhere('dispatch_attempts', '<>', 0);
|
|
})->orWhere(static function ($own): void {
|
|
$own->where('phase', 'terminal')
|
|
->where('outcome', 'failed')
|
|
->where('status', 'failed')
|
|
->where('effect_may_have_applied', 1)
|
|
->where('safe_to_resubmit', 0)
|
|
->where('effect_resolution', 'unknown');
|
|
})->orWhere(static function ($state): void {
|
|
$state->where('phase', 'terminal')
|
|
->where('effect_resolution', 'unknown')
|
|
->where(static function ($invalid): void {
|
|
self::applyInvalidTerminalTuple($invalid);
|
|
});
|
|
})->orWhere(static function ($published): void {
|
|
$published->where('phase', 'terminal')
|
|
->where('finalization_incident', 1);
|
|
})->orWhereExists(
|
|
static function ($children): void {
|
|
self::unresolvedRecoverableChildExists(
|
|
$children
|
|
);
|
|
}
|
|
);
|
|
});
|
|
})->orWhereRaw(
|
|
self::aggregateLocalStateConflictSql('cloud_jobs')
|
|
)->orWhere(static function ($damagedCatalogParent): void {
|
|
$damagedCatalogParent->where('phase', 'terminal')
|
|
->where('finalization_incident', 1)
|
|
->where(static function ($identity): void {
|
|
$identity->where('execution_mode', 'aggregate')
|
|
->orWhereIn('kind', self::AGGREGATE_KINDS)
|
|
->orWhere(
|
|
'finalization_error_code',
|
|
'recovery_aggregate_state_conflict'
|
|
);
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
private static function isLocalFinalizerRecovery(object $job): bool
|
|
{
|
|
return (string) ($job->execution_mode ?? '') === 'agent'
|
|
&& (string) ($job->phase ?? '') === 'terminal'
|
|
&& (string) ($job->outcome ?? '') === 'failed'
|
|
&& (string) ($job->status ?? '') === 'failed'
|
|
&& CloudDispatchLedger::
|
|
isLocallyFinalizableNotDispatched($job)
|
|
&& (bool) ($job->finalization_incident ?? false)
|
|
&& !CloudDispatchLedger::hasRepairIncident($job)
|
|
&& CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
|
|
(string) ($job->kind ?? '')
|
|
);
|
|
}
|
|
|
|
/** A resolved Agent row with a sticky PHP incident remains locally queued. */
|
|
private static function isResolvedStickyAgentIncident(object $job): bool
|
|
{
|
|
return (string) ($job->execution_mode ?? '') === 'agent'
|
|
&& (string) ($job->phase ?? '') === 'terminal'
|
|
&& !CloudDispatchLedger::isInvalid($job)
|
|
&& in_array(
|
|
(string) ($job->effect_resolution ?? ''),
|
|
['applied', 'not_applied'],
|
|
true
|
|
)
|
|
&& (bool) ($job->finalization_incident ?? false)
|
|
&& !CloudDispatchLedger::hasRepairIncident($job)
|
|
&& CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
|
|
(string) ($job->kind ?? '')
|
|
);
|
|
}
|
|
|
|
/** Add the negation of the canonical failed/effectful/non-retryable tuple. */
|
|
private static function applyInvalidTerminalTuple(
|
|
$invalid,
|
|
string $prefix = ''
|
|
): void {
|
|
$invalid->where(static function ($tuple) use ($prefix): void {
|
|
$tuple->whereNull($prefix . 'outcome')
|
|
->orWhere($prefix . 'outcome', '<>', 'failed')
|
|
->orWhereNull($prefix . 'status')
|
|
->orWhere($prefix . 'status', '<>', 'failed')
|
|
->orWhereNull($prefix . 'effect_may_have_applied')
|
|
->orWhere($prefix . 'effect_may_have_applied', '<>', 1)
|
|
->orWhereNull($prefix . 'safe_to_resubmit')
|
|
->orWhere($prefix . 'safe_to_resubmit', '<>', 0);
|
|
})->orWhere(static function ($neverDispatched) use ($prefix): void {
|
|
$neverDispatched->where($prefix . 'dispatch_attempts', 0)
|
|
->where($prefix . 'finalization_incident', 0);
|
|
});
|
|
}
|
|
|
|
/** Add terminal applied/not-applied rows that contradict their outcome. */
|
|
public static function applyInvalidResolvedTerminalTuple(
|
|
$invalid,
|
|
string $prefix = ''
|
|
): void {
|
|
$invalid->whereIn(
|
|
$prefix . 'effect_resolution',
|
|
['applied', 'not_applied']
|
|
)->where(static function ($shape) use ($prefix): void {
|
|
$shape->whereNull($prefix . 'terminal_at')
|
|
->orWhere($prefix . 'terminal_at', '<=', 0)
|
|
->orWhereNull($prefix . 'outcome')
|
|
->orWhereNotIn(
|
|
$prefix . 'outcome',
|
|
['succeeded', 'failed']
|
|
)->orWhere(static function ($succeeded) use ($prefix): void {
|
|
$succeeded->where($prefix . 'outcome', 'succeeded')
|
|
->where(static function ($mismatch) use ($prefix): void {
|
|
$mismatch->whereNull($prefix . 'status')
|
|
->orWhere(
|
|
$prefix . 'status',
|
|
'<>',
|
|
'succeeded'
|
|
)->orWhere(
|
|
$prefix . 'effect_resolution',
|
|
'<>',
|
|
'applied'
|
|
)->orWhereNull(
|
|
$prefix . 'effect_may_have_applied'
|
|
)->orWhere(
|
|
$prefix . 'effect_may_have_applied',
|
|
'<>',
|
|
0
|
|
)->orWhereNull(
|
|
$prefix . 'safe_to_resubmit'
|
|
)->orWhere(
|
|
$prefix . 'safe_to_resubmit',
|
|
'<>',
|
|
0
|
|
)->orWhereNotNull($prefix . 'error_code');
|
|
});
|
|
})->orWhere(static function ($failed) use ($prefix): void {
|
|
$failed->where($prefix . 'outcome', 'failed')
|
|
->where(static function ($mismatch) use ($prefix): void {
|
|
$mismatch->whereNull($prefix . 'status')
|
|
->orWhere($prefix . 'status', '<>', 'failed')
|
|
->orWhereNull(
|
|
$prefix . 'effect_may_have_applied'
|
|
)->orWhereNotIn(
|
|
$prefix . 'effect_may_have_applied',
|
|
[0, 1]
|
|
)->orWhere(static function ($effectful) use (
|
|
$prefix
|
|
): void {
|
|
$effectful->where(
|
|
$prefix . 'effect_may_have_applied',
|
|
1
|
|
)->where(static function ($safe) use (
|
|
$prefix
|
|
): void {
|
|
$safe->whereNull(
|
|
$prefix . 'safe_to_resubmit'
|
|
)->orWhere(
|
|
$prefix . 'safe_to_resubmit',
|
|
'<>',
|
|
0
|
|
);
|
|
});
|
|
})->orWhere(static function ($noEffect) use (
|
|
$prefix
|
|
): void {
|
|
$noEffect->where(
|
|
$prefix . 'effect_may_have_applied',
|
|
0
|
|
)->where(static function ($conclusion) use (
|
|
$prefix
|
|
): void {
|
|
$conclusion->where(
|
|
$prefix . 'effect_resolution',
|
|
'<>',
|
|
'not_applied'
|
|
)->orWhereNull(
|
|
$prefix . 'safe_to_resubmit'
|
|
)->orWhereNotIn(
|
|
$prefix . 'safe_to_resubmit',
|
|
[0, 1]
|
|
);
|
|
});
|
|
});
|
|
});
|
|
})->orWhere(static function ($envelope) use ($prefix): void {
|
|
self::applyInvalidResolvedAgentEnvelope(
|
|
$envelope,
|
|
$prefix
|
|
);
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Keep an aggregate/catalog boundary fenced until every Agent child has a
|
|
* trustworthy terminal effect. During ordinary receipt finalization the
|
|
* current child is intentionally still nonterminal, so only that expected
|
|
* phase/effect transition is ignored; a corrupt dispatch ledger or sticky
|
|
* recovery incident on the current child still blocks publication.
|
|
*/
|
|
public static function aggregateParentRecoveryPending(
|
|
CloudJob $parent,
|
|
?int $completingChildId = null,
|
|
bool $lockForUpdate = false
|
|
): bool {
|
|
$query = CloudJob::where('parent_job_id', (int) $parent->id)
|
|
->orderBy('id');
|
|
if ($lockForUpdate) {
|
|
$query->lockForUpdate();
|
|
}
|
|
$children = $query->get();
|
|
if (self::aggregateRelationshipConflict(
|
|
$parent,
|
|
$children,
|
|
$completingChildId
|
|
)) {
|
|
return true;
|
|
}
|
|
return $children->contains(
|
|
static fn (CloudJob $child): bool =>
|
|
self::aggregateChildRecoveryPending(
|
|
$child,
|
|
$completingChildId !== null
|
|
&& (int) $child->id === $completingChildId
|
|
)
|
|
);
|
|
}
|
|
|
|
/** A child cannot publish its parent while any safety incident is sticky. */
|
|
private static function aggregateChildRecoveryPending(
|
|
CloudJob $child,
|
|
bool $isCompletingChild = false
|
|
): bool {
|
|
return (!$isCompletingChild
|
|
&& ((string) $child->phase !== 'terminal'
|
|
|| !in_array(
|
|
(string) $child->effect_resolution,
|
|
['applied', 'not_applied'],
|
|
true
|
|
)))
|
|
|| CloudDispatchLedger::isInvalid($child)
|
|
|| (bool) $child->finalization_incident
|
|
|| CloudDispatchLedger::hasRepairIncident($child)
|
|
|| self::isLocalFinalizerRecovery($child)
|
|
|| self::hasResolvedTerminalStateConflict($child);
|
|
}
|
|
|
|
public static function tasks(array $actor, string $state,
|
|
?int $cursor = null, int $limit = 20): array
|
|
{
|
|
if (!in_array($state, ['active', 'completed'], true)) {
|
|
throw new CloudError('任务状态筛选无效', 400);
|
|
}
|
|
// The task center is also the owner-scoped daemon-loss fail-safe. It
|
|
// persists only already-due local outcomes and never contacts Agents.
|
|
CloudWorkerService::expireDueTasksForOwner(
|
|
(int) ($actor['user_id'] ?? 0), null, 100
|
|
);
|
|
$limit = max(1, min(50, $limit));
|
|
$owner = static function ($query) use ($actor): void {
|
|
$query->where('user_id', (int) ($actor['user_id'] ?? 0));
|
|
};
|
|
// Per-host child jobs are implementation details of one public batch.
|
|
// Only their parent appears in the task center.
|
|
$base = CloudJob::query()->whereNull('vpc_batch_id')->where(
|
|
static function ($public): void {
|
|
$public->whereNull('parent_job_id')
|
|
->orWhere(static function ($currentCatalog): void {
|
|
self::whereCurrentAggregateCatalogBinding(
|
|
$currentCatalog
|
|
);
|
|
});
|
|
}
|
|
);
|
|
$owner($base);
|
|
$activeCount = (clone $base)->where('phase', '<>', 'terminal')->count();
|
|
$recovering = clone $base;
|
|
self::applyRecoveringFilter($recovering);
|
|
$recoveringCount = $recovering->count();
|
|
$unreadCount = (clone $base)->where('phase', 'terminal')
|
|
->whereNull('read_at')->count();
|
|
$query = clone $base;
|
|
$query->where('phase', $state === 'active' ? '<>' : '=', 'terminal');
|
|
if ($cursor !== null) {
|
|
$query->where('id', '<', $cursor);
|
|
}
|
|
$rows = $query->orderByDesc('id')->limit($limit + 1)->get();
|
|
$hasMore = $rows->count() > $limit;
|
|
if ($hasMore) {
|
|
$rows = $rows->take($limit);
|
|
}
|
|
$items = $rows->map(static fn (CloudJob $job): array =>
|
|
self::display($job, (bool) ($actor['is_admin'] ?? false)))->all();
|
|
$last = $rows->last();
|
|
return [
|
|
'items' => $items,
|
|
'pagination' => [
|
|
'next_cursor' => $hasMore && $last !== null
|
|
? (string) $last->id : null,
|
|
'has_more' => $hasMore,
|
|
],
|
|
'active_count' => (int) $activeCount,
|
|
'recovering_count' => (int) $recoveringCount,
|
|
'unread_count' => (int) $unreadCount,
|
|
];
|
|
}
|
|
|
|
public static function markRead(CloudJob $job): void
|
|
{
|
|
if ((string) $job->phase !== 'terminal' || $job->read_at !== null) {
|
|
return;
|
|
}
|
|
CloudJob::where('id', (int) $job->id)->where('phase', 'terminal')
|
|
->whereNull('read_at')->update(['read_at' => time()]);
|
|
}
|
|
|
|
public static function unreadCount(array $actor): int
|
|
{
|
|
$query = CloudJob::where('phase', 'terminal')->whereNull('read_at')
|
|
->whereNull('vpc_batch_id')
|
|
->where('user_id', (int) ($actor['user_id'] ?? 0))
|
|
->where(static function ($public): void {
|
|
$public->whereNull('parent_job_id')
|
|
->orWhere(static function ($currentCatalog): void {
|
|
self::whereCurrentAggregateCatalogBinding(
|
|
$currentCatalog
|
|
);
|
|
});
|
|
});
|
|
return (int) $query->count();
|
|
}
|
|
|
|
private static function requestKey(array $actor): ?string
|
|
{
|
|
$key = $actor['request_idempotency_key'] ?? null;
|
|
if ($key === null || $key === '') {
|
|
return null;
|
|
}
|
|
if (!is_string($key)
|
|
|| preg_match('/^[A-Za-z0-9_.-]{16,128}$/D', $key) !== 1) {
|
|
throw new CloudError('请求幂等标识无效', 400);
|
|
}
|
|
return $key;
|
|
}
|
|
|
|
private static function canonicalRequestValue($value)
|
|
{
|
|
if (!is_array($value)) {
|
|
return $value;
|
|
}
|
|
if (!array_is_list($value)) {
|
|
ksort($value, SORT_STRING);
|
|
}
|
|
foreach ($value as $key => $item) {
|
|
$value[$key] = self::canonicalRequestValue($item);
|
|
}
|
|
return $value;
|
|
}
|
|
|
|
/**
|
|
* Keep only the catalogued result DTO. Historical result JSON is not a
|
|
* trusted presentation format and must not bypass the error boundary.
|
|
*/
|
|
private static function displayResult(?array $result, bool $administrator,
|
|
?string $kind, ?string $status, bool $hasOrder, ?string $action): ?array
|
|
{
|
|
if ($result === null) {
|
|
return null;
|
|
}
|
|
return CloudOperationErrorCatalog::result($result, $administrator, $kind,
|
|
$status, $hasOrder, $action);
|
|
}
|
|
|
|
/** Resolve a trusted payment DTO from its owner-scoped durable record. */
|
|
private static function displayPaymentResult(CloudJob $job,
|
|
array $result): ?array
|
|
{
|
|
$id = $result['payment_id'] ?? null;
|
|
if (!is_string($id)
|
|
|| preg_match('/^[1-9][0-9]{0,18}$/D', $id) !== 1
|
|
|| $job->invoice_id === null) {
|
|
return null;
|
|
}
|
|
$payment = \App\Models\CloudInvoicePayment::where('id', (int) $id)
|
|
->where('invoice_id', (int) $job->invoice_id)
|
|
->where('user_id', (int) $job->user_id)->first();
|
|
if ($payment === null) {
|
|
return null;
|
|
}
|
|
try {
|
|
return ['payment' => CloudInvoicePaymentService::display($payment)];
|
|
} catch (\Throwable $error) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Return the non-sensitive power action from the encrypted task payload.
|
|
*
|
|
* The durable job payload is encrypted at rest and is never returned to
|
|
* the browser. Terminal jobs retain only CloudInput::safeResult($payload),
|
|
* so this lookup continues to work for historical rows while removing
|
|
* credentials and all other payload fields from the public response.
|
|
*/
|
|
public static function action(CloudJob $job): ?string
|
|
{
|
|
if ((string) $job->kind !== 'action' || trim((string) $job->sealed_payload) === '') {
|
|
return null;
|
|
}
|
|
try {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $job->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
} catch (\Throwable $error) {
|
|
return null;
|
|
}
|
|
$body = $payload['body'] ?? null;
|
|
$action = is_array($body) && is_string($body['action'] ?? null)
|
|
? $body['action'] : '';
|
|
return CloudOperationCatalog::isPowerAction($action) ? $action : null;
|
|
}
|
|
|
|
/** Return the same safe labels used by every administrator DTO. */
|
|
public static function labels(CloudJob $job): array
|
|
{
|
|
return CloudOperationCatalog::labels((string) $job->kind,
|
|
self::action($job), self::settingsFields($job));
|
|
}
|
|
|
|
/**
|
|
* Read only safe setting-category codes from an encrypted job payload.
|
|
*
|
|
* New jobs persist the codes outside the strict Agent body so terminal
|
|
* redaction retains them. Jobs already active during deployment can still
|
|
* be described from body key presence; completed legacy password/key jobs
|
|
* have intentionally lost those keys and safely fall back to the generic
|
|
* instance-settings label.
|
|
*/
|
|
private static function settingsFields(CloudJob $job): array
|
|
{
|
|
if ((string) $job->kind !== 'instance_settings'
|
|
|| trim((string) $job->sealed_payload) === '') {
|
|
return [];
|
|
}
|
|
try {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $job->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
} catch (\Throwable $error) {
|
|
return [];
|
|
}
|
|
if (array_key_exists('settings_fields', $payload)) {
|
|
$fields = $payload['settings_fields'];
|
|
return is_array($fields) && array_is_list($fields)
|
|
? $fields : [];
|
|
}
|
|
$body = $payload['body'] ?? null;
|
|
if (!is_array($body) || array_is_list($body)) {
|
|
return [];
|
|
}
|
|
$fields = [];
|
|
foreach ([
|
|
'hostname' => 'hostname',
|
|
'root_password' => 'password',
|
|
'ssh_public_key' => 'ssh_key',
|
|
] as $bodyField => $detailCode) {
|
|
if (array_key_exists($bodyField, $body)) {
|
|
$fields[] = $detailCode;
|
|
}
|
|
}
|
|
return $fields;
|
|
}
|
|
|
|
/** Read the user's complete instance operation history from cloud_jobs. */
|
|
public static function operationLogs(CloudInstance $instance, int $page = 1, int $perPage = 10): array
|
|
{
|
|
$perPage = max(1, min(50, $perPage));
|
|
$query = CloudJob::where('instance_id', $instance->id)
|
|
->whereIn('kind', CloudOperationCatalog::instanceLogKinds())
|
|
->orderByDesc('id');
|
|
$total = max(0, (int) (clone $query)->count('id'));
|
|
$pages = max(1, (int) ceil($total / $perPage));
|
|
$page = max(1, min($pages, $page));
|
|
$jobs = $query->offset(($page - 1) * $perPage)->limit($perPage)
|
|
->get(['id', 'operation_id', 'kind', 'order_id', 'status',
|
|
'parent_job_id', 'execution_mode', 'dispatch_attempts',
|
|
'phase', 'outcome', 'attempts', 'error_code', 'result_json',
|
|
'effect_may_have_applied', 'safe_to_resubmit',
|
|
'effect_resolution', 'finalization_incident',
|
|
'finalization_error_code', 'available_at',
|
|
'terminal_at', 'read_at',
|
|
'sealed_payload', 'created_at', 'updated_at']);
|
|
$items = [];
|
|
foreach ($jobs as $job) {
|
|
$entry = self::display($job);
|
|
unset($entry['operation_id'], $entry['result']);
|
|
$items[] = $entry;
|
|
}
|
|
$from = $total === 0 ? 0 : ($page - 1) * $perPage + 1;
|
|
return ['items' => $items, 'pagination' => [
|
|
'page' => $page, 'per_page' => $perPage, 'total' => $total,
|
|
'total_pages' => $pages, 'from' => $from,
|
|
'to' => $from === 0 ? 0 : $from + count($items) - 1,
|
|
]];
|
|
}
|
|
|
|
}
|