You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudJobService.php

4109 lines
183 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudHost;
use App\Models\CloudHostAgent;
use App\Models\CloudInstance;
use App\Models\CloudJob;
use App\Models\CloudResourcePool;
use App\Models\CloudTemplate;
use App\Models\CloudUserIso;
use App\Models\User;
final class CloudJobService
{
private const RECEIPT_DEADLINE_SECONDS = 960;
private const STORAGE_DISPATCH_DEADLINE_SECONDS = 60;
private const STORAGE_RECEIPT_DEADLINE_SECONDS = 180;
private const IMPORT_RECEIPT_DEADLINE_SECONDS = 8100;
private const BACKUP_RECEIPT_DEADLINE_SECONDS = 90000;
private const AGGREGATE_FINALIZATION_BUFFER_SECONDS = 300;
private const AGGREGATE_DEADLINE_SECONDS =
CloudJob::DISPATCH_DEADLINE_SECONDS
+ self::RECEIPT_DEADLINE_SECONDS
+ self::AGGREGATE_FINALIZATION_BUFFER_SECONDS;
private const IMPORT_AGGREGATE_DEADLINE_SECONDS =
CloudJob::DISPATCH_DEADLINE_SECONDS
+ self::IMPORT_RECEIPT_DEADLINE_SECONDS
+ self::AGGREGATE_FINALIZATION_BUFFER_SECONDS;
private const LONG_IMPORT_KINDS = [
'template_import_replica',
'rescue_catalog_validate_replica',
'rescue_catalog_import_replica',
'user_iso_import_replica',
];
private const LONG_IMPORT_AGGREGATE_KINDS = [
'template_import',
'user_iso_import',
];
private const PORTABLE_BACKUP_KINDS = [
'backup_create',
'backup_restore',
'backup_recover',
'instance_migration_prepare',
'instance_migration_transfer',
];
private const AGGREGATE_KINDS = [
'template_import',
'template_metadata_sync',
'template_delete',
'user_iso_import',
'user_iso_delete',
];
private const AGGREGATE_CHILD_KIND_MAP = [
'template_import' => 'template_import_replica',
'template_metadata_sync' => 'template_metadata_replica',
'template_delete' => 'template_delete_replica',
'user_iso_import' => 'user_iso_import_replica',
'user_iso_delete' => 'user_iso_delete_replica',
];
private const RECOVERY_RETRY_SECONDS = 300;
private const RECOVERY_MESSAGES = [
'recovery_pending' =>
'系统正在核对原任务结果;不会重复执行原操作,仍会继续自动核对。',
'recovery_limit_reached' =>
'旧任务已重新进入持续自动核对;不会重复执行原操作,系统仍会继续自动核对。',
'recovery_receipt_limit_reached' =>
'旧任务回执已重新进入持续自动核对;不会重复执行原操作,系统仍会继续自动核对。',
'recovery_state_invalid' =>
'任务状态与恢复记录暂不一致;不会重复执行原操作,修复记录后系统仍会继续自动核对。',
'recovery_evidence_conflict' =>
'Agent 回执与宿主机证据暂不一致;资源保持安全锁定,系统仍会继续自动核对。',
'recovery_evidence_partial' =>
'宿主机目前只能确认部分操作结果;资源保持安全锁定,系统仍会继续自动核对。',
'recovery_target_cleanup_pending' =>
'系统正在自动清理维护迁移目标机残留;原迁移不会重放,清理后会继续核对并修复迁移账本。',
'recovery_evidence_inconclusive' =>
'宿主机证据暂不足以确认操作结果;资源保持安全锁定,系统仍会继续自动核对。',
'recovery_evidence_invalid' =>
'Agent 返回的核对或栅栏证据格式不匹配;不会采信异常证据,系统仍会继续自动核对。',
'recovery_receipt_invalid' =>
'Agent 回执未通过完整性核验;不会采信异常回执,系统仍会继续自动核对。',
'recovery_completion_outbox_invalid' =>
'宿主机的完成回执队列状态损坏;不会采信当前结果,修复后系统仍会继续自动核对。',
'recovery_completion_receipt_missing' =>
'宿主机的完成回执内容缺失;不会采信当前结果,修复后系统仍会继续自动核对。',
'recovery_completion_receipt_invalid' =>
'宿主机的完成回执内容无效;不会采信当前结果,修复后系统仍会继续自动核对。',
'recovery_completion_operation_missing' =>
'宿主机的完成回执缺少对应原操作记录;不会采信当前结果,修复后系统仍会继续自动核对。',
'recovery_completion_operation_invalid' =>
'宿主机的完成回执对应原操作记录无效;不会采信当前结果,修复后系统仍会继续自动核对。',
'recovery_completion_identity_mismatch' =>
'宿主机的完成回执身份与原操作记录不匹配;不会采信当前结果,修复后系统仍会继续自动核对。',
'recovery_agent_unavailable' =>
'宿主机 Agent 或网络暂不可用;通信恢复后系统仍会继续自动核对。',
'recovery_authentication_failed' =>
'面板与宿主机 Agent 的认证暂未通过;修复认证后系统仍会继续自动核对。',
'recovery_waiting_connection' =>
'宿主机 Agent 或网络暂不可用;通信恢复后系统仍会继续自动核对。',
'recovery_operation_missing' =>
'宿主机暂未找到原任务记录;不会重复执行原操作,系统仍会继续自动核对。',
'recovery_operation_conflict' =>
'任务编号与宿主机记录暂不一致;不会重复执行原操作,系统仍会继续自动核对。',
'recovery_operation_identity_conflict' =>
'Agent 证据中的操作、资源或请求指纹与原任务不匹配;不会重复执行或解锁,系统仍会继续自动核对。',
'recovery_payload_invalid' =>
'任务恢复数据暂时无法安全读取;修复记录后系统仍会继续自动核对。',
'recovery_payload_unavailable' =>
'任务恢复数据暂时无法安全读取;修复记录后系统仍会继续自动核对。',
'recovery_instance_state_conflict' =>
'实例状态与任务证据暂不一致;资源保持安全锁定,系统仍会继续自动核对。',
'recovery_aggregate_state_conflict' =>
'汇总任务与子任务账本关系异常;资源保持安全锁定,修复账本后系统仍会继续自动核对。',
'recovery_dispatch_ledger_conflict' =>
'任务单次派发计数账本异常;资源保持安全锁定,修复账本后系统仍会继续自动核对。',
'recovery_dispatch_repair_completion_receipt_missing' =>
'派发计数已修复,但原任务绑定的完成回执缺失;终态保持不变,资源保持锁定并持续恢复。',
'recovery_dispatch_repair_receipt_invalid' =>
'派发计数已修复,但原任务回执未通过完整性校验;不会采信异常回执或解锁。',
'recovery_dispatch_repair_aggregate_state_conflict' =>
'派发计数已修复,但汇总任务与子任务账本关系仍不一致;资源保持锁定并持续恢复。',
'recovery_dispatch_repair_payload_unavailable' =>
'派发计数已修复,但原任务恢复数据暂时无法安全读取;不会重复执行或解锁。',
'recovery_dispatch_repair_operation_identity_conflict' =>
'派发计数已修复,但回执中的操作、资源或请求指纹与原任务不匹配;不会重复执行或解锁。',
'recovery_dispatch_repair_evidence_conflict' =>
'派发计数已修复,但原任务回执与宿主机证据仍不一致;资源保持锁定并持续恢复。',
'recovery_dispatch_repair_instance_state_conflict' =>
'派发计数已修复,但实例状态与原任务证据仍不一致;资源保持锁定并持续恢复。',
'recovery_dispatch_repair_ledger_conflict' =>
'派发计数已修复,但面板业务账本仍未安全收敛;终态保持不变,资源保持锁定。',
'recovery_dispatch_repair_internal_error' =>
'派发计数修复后的安全校验遇到内部异常;不会重复执行或解锁,系统会持续恢复。',
'recovery_child_convergence' =>
'子任务仍在持续自动核对;汇总任务会在子任务收敛后更新,原操作不会重复执行。',
'recovery_ledger_conflict' =>
'任务结果已确定,但面板账本暂未安全收敛;资源保持安全锁定,系统仍会继续自动核对并恢复。',
'recovery_internal_error' =>
'自动核对暂时遇到内部异常;不会重复执行原操作,系统仍会继续自动核对。',
];
/** Absolute dispatch windows are selected once and are never extended. */
public static function dispatchDeadlineSeconds(string $kind): int
{
return $kind === 'host_storage_configure'
? self::STORAGE_DISPATCH_DEADLINE_SECONDS
: CloudJob::DISPATCH_DEADLINE_SECONDS;
}
public static function effectiveDispatchDeadline(CloudJob $job): int
{
$stored = (int) ($job->dispatch_deadline_at ?? 0);
if ($stored < 1 || (string) $job->kind !== 'host_storage_configure') {
return $stored;
}
$createdAt = (int) ($job->created_at ?? 0);
return $createdAt < 1 ? $stored : min(
$stored,
$createdAt + self::STORAGE_DISPATCH_DEADLINE_SECONDS
);
}
/** Absolute receipt windows are selected once and are never extended. */
public static function receiptDeadlineSeconds(string $kind): int
{
if (in_array($kind, self::PORTABLE_BACKUP_KINDS, true)) {
return self::BACKUP_RECEIPT_DEADLINE_SECONDS;
}
if (in_array($kind, self::LONG_IMPORT_KINDS, true)) {
return self::IMPORT_RECEIPT_DEADLINE_SECONDS;
}
return $kind === 'host_storage_configure'
? self::STORAGE_RECEIPT_DEADLINE_SECONDS
: self::RECEIPT_DEADLINE_SECONDS;
}
/** Cloud-snapshot creates download a backup and need the restore window. */
public static function receiptDeadlineSecondsForJob(CloudJob $job): int
{
if ((string) $job->kind === 'create' && $job->backup_id !== null) {
return self::BACKUP_RECEIPT_DEADLINE_SECONDS;
}
return self::receiptDeadlineSeconds((string) $job->kind);
}
/**
* Apply the current bounded window to persisted work as well as new work.
* This lets a deployment shorten an old storage task's 960-second value
* without rewriting history or extending any previously stored deadline.
*/
public static function effectiveReceiptDeadline(CloudJob $job): int
{
$stored = (int) ($job->receipt_deadline_at ?? 0);
if ($stored < 1 || (string) $job->kind !== 'host_storage_configure') {
return $stored;
}
$dispatchedAt = (int) ($job->dispatched_at ?? 0);
return $dispatchedAt < 1 ? $stored : min(
$stored,
$dispatchedAt + self::STORAGE_RECEIPT_DEADLINE_SECONDS
);
}
public static function receiptDeadlineReached(CloudJob $job,
?int $now = null): bool
{
$deadline = self::effectiveReceiptDeadline($job);
return $deadline > 0 && $deadline <= ($now ?? time());
}
public static function activeDeadlineReached(CloudJob $job,
?int $now = null): bool
{
$now ??= time();
if ((string) $job->phase === 'accepted') {
$deadline = self::effectiveDispatchDeadline($job);
return $deadline > 0 && $deadline <= $now;
}
return in_array((string) $job->phase,
['dispatching', 'awaiting_receipt'], true)
&& self::receiptDeadlineReached($job, $now);
}
public static function aggregateDeadlineSeconds(string $kind): int
{
return in_array($kind, self::LONG_IMPORT_AGGREGATE_KINDS, true)
? self::IMPORT_AGGREGATE_DEADLINE_SECONDS
: self::AGGREGATE_DEADLINE_SECONDS;
}
/** Aggregate parents are local coordination rows and are never dispatched. */
public static function isAggregateKind(string $kind): bool
{
return in_array($kind, self::AGGREGATE_KINDS, true);
}
/** @return list<string> */
public static function aggregateKinds(): array
{
return self::AGGREGATE_KINDS;
}
/** True only for a damaged terminal recovery or aggregate-local tuple. */
public static function hasTerminalRecoveryStateConflict(object $job): bool
{
if ((string) ($job->phase ?? '') !== 'terminal') {
return false;
}
$mode = (string) ($job->execution_mode ?? '');
$kind = (string) ($job->kind ?? '');
if (CloudOperationRecoveryPolicy::supportsLedgerFinalizer($kind)) {
if ($mode !== 'agent'
|| self::hasAgentRecoveryEnvelopeConflict($job)) {
return true;
}
if (self::hasResolvedTerminalStateConflict($job)) {
return true;
}
}
if (self::isAggregateKind($kind)) {
if (self::hasAggregateLocalStateConflict($job)) {
return true;
}
if ((string) ($job->effect_resolution ?? '') !== 'unknown') {
return false;
}
} elseif ((string) ($job->effect_resolution ?? '') !== 'unknown') {
return false;
}
$supported = ($mode === 'agent'
&& !CloudDispatchLedger::isInvalid($job)
&& CloudOperationRecoveryPolicy::supportsLedgerFinalizer($kind))
|| ($mode === 'aggregate'
&& ($job->parent_job_id ?? null) === null
&& self::isAggregateKind($kind));
if (!$supported) {
return false;
}
// Generation zero proves the mutation never crossed the Agent
// boundary. A terminal/unknown row may remain there only while a
// durable PHP-only finalizer incident is retrying; without that
// incident the otherwise-canonical dispatched safety tuple is itself
// contradictory and must stay in the local invalid-state queue.
if ($mode === 'agent'
&& (int) ($job->dispatch_attempts ?? -1) === 0
&& !(bool) ($job->finalization_incident ?? false)) {
return true;
}
return (string) ($job->outcome ?? '') !== 'failed'
|| (string) ($job->status ?? '') !== 'failed'
|| !(bool) ($job->effect_may_have_applied ?? false)
|| (bool) ($job->safe_to_resubmit ?? false);
}
/** Aggregate parents are PHP-local ledgers and must never resemble Agent work. */
public static function hasAggregateLocalStateConflict(object $job): bool
{
if ((string) ($job->execution_mode ?? '') !== 'aggregate'
|| !self::isAggregateKind((string) ($job->kind ?? ''))
|| ($job->parent_job_id ?? null) !== null
|| ($job->host_id ?? null) !== null
|| ($job->instance_id ?? null) !== null
|| ($job->order_id ?? null) !== null
|| ($job->invoice_id ?? null) !== null
|| ($job->snapshot_id ?? null) !== null
|| (int) ($job->user_id ?? 0) < 1
|| (string) ($job->attempts ?? '') !== '0'
|| CloudDispatchLedger::attempts($job) !== 0
|| (int) ($job->dispatch_deadline_at ?? 0) < 1
|| (int) ($job->receipt_deadline_at ?? 0) < 1
|| ($job->dispatch_started_at ?? null) !== null
|| ($job->agent_accepted_at ?? null) !== null
|| ($job->dispatched_at ?? null) !== null
|| ($job->completion_event_id ?? null) !== null
|| preg_match(
'/^[a-f0-9]{32}$/D',
(string) ($job->operation_id ?? '')
) !== 1
|| preg_match(
'/^[a-f0-9]{64}$/D',
(string) ($job->request_fingerprint ?? '')
) !== 1
|| preg_match(
'/^[A-Za-z0-9_.-]{16,128}$/D',
(string) ($job->request_idempotency_key ?? '')
) !== 1) {
return true;
}
$phase = (string) ($job->phase ?? '');
if ($phase === 'awaiting_receipt') {
return ($job->outcome ?? null) !== null
|| (string) ($job->status ?? '') !== 'running'
|| ($job->terminal_at ?? null) !== null
|| ($job->effect_may_have_applied ?? null) !== null
|| ($job->safe_to_resubmit ?? null) !== null
|| (string) ($job->effect_resolution ?? '')
!== 'not_applicable'
|| (bool) ($job->finalization_incident ?? false);
}
if ($phase !== 'terminal'
|| (int) ($job->terminal_at ?? 0) < 1
|| !in_array(
(string) ($job->outcome ?? ''),
['succeeded', 'failed'],
true
)
|| (string) ($job->status ?? '')
!== (string) ($job->outcome ?? '')) {
return true;
}
$outcome = (string) $job->outcome;
$resolution = (string) ($job->effect_resolution ?? '');
$effectMayHaveApplied = $job->effect_may_have_applied ?? null;
$safeToResubmit = $job->safe_to_resubmit ?? null;
if ($outcome === 'succeeded') {
return !in_array(
$effectMayHaveApplied,
[0, '0', false],
true
) || !in_array($safeToResubmit, [0, '0', false], true)
|| $resolution !== 'applied'
|| ($job->error_code ?? null) !== null;
}
if (!in_array($resolution, [
'unknown', 'applied', 'not_applied',
], true)) {
return true;
}
if (in_array($effectMayHaveApplied, [1, '1', true], true)) {
return !in_array($safeToResubmit, [0, '0', false], true);
}
return !in_array($effectMayHaveApplied, [0, '0', false], true)
|| !in_array(
$safeToResubmit,
[0, '0', false, 1, '1', true],
true
)
|| $resolution !== 'not_applied';
}
/** A published terminal effect must retain one canonical immutable tuple. */
public static function hasResolvedTerminalStateConflict(object $job): bool
{
if ((string) ($job->phase ?? '') !== 'terminal'
|| !in_array(
(string) ($job->effect_resolution ?? ''),
['applied', 'not_applied'],
true
)) {
return false;
}
if ((int) ($job->terminal_at ?? 0) < 1) {
return true;
}
$outcome = (string) ($job->outcome ?? '');
$status = (string) ($job->status ?? '');
$resolution = (string) $job->effect_resolution;
$effect = $job->effect_may_have_applied ?? null;
$safe = $job->safe_to_resubmit ?? null;
if ($outcome === 'succeeded') {
$tupleConflict = $status !== 'succeeded'
|| $resolution !== 'applied'
|| !in_array($effect, [0, '0', false], true)
|| !in_array($safe, [0, '0', false], true)
|| ($job->error_code ?? null) !== null;
if ($tupleConflict) {
return true;
}
} elseif ($outcome !== 'failed' || $status !== 'failed') {
return true;
} elseif (in_array($effect, [1, '1', true], true)) {
if (!in_array($safe, [0, '0', false], true)) {
return true;
}
} elseif (!in_array($effect, [0, '0', false], true)
|| !in_array($safe, [0, '0', false, 1, '1', true], true)
|| $resolution !== 'not_applied') {
return true;
}
$kind = (string) ($job->kind ?? '');
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer($kind)) {
return false;
}
return self::hasAgentRecoveryEnvelopeConflict($job);
}
/** True when one supported Agent generation's immutable envelope is bad. */
public static function hasAgentRecoveryEnvelopeConflict(object $job): bool
{
$kind = (string) ($job->kind ?? '');
$outcome = (string) ($job->outcome ?? '');
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer($kind)) {
return false;
}
if ((string) ($job->execution_mode ?? '') !== 'agent'
|| self::hasAgentRecoveryIdentityConflict($job)
|| (int) ($job->host_id ?? 0) < 1
|| (int) ($job->dispatch_deadline_at ?? 0) < 1) {
return true;
}
$dispatchAttempts = CloudDispatchLedger::attempts($job);
if ($dispatchAttempts === 1) {
return (string) ($job->attempts ?? '') !== '1'
|| (int) ($job->dispatch_started_at ?? 0) < 1
|| (($job->agent_accepted_at ?? null) !== null
&& (int) $job->agent_accepted_at < 1)
|| (int) ($job->dispatched_at ?? 0) < 1
|| (int) ($job->receipt_deadline_at ?? 0) < 1;
}
if ($dispatchAttempts !== 0
|| (string) ($job->attempts ?? '') !== '0'
|| ($job->dispatch_started_at ?? null) !== null
|| ($job->agent_accepted_at ?? null) !== null
|| ($job->dispatched_at ?? null) !== null
|| ($job->receipt_deadline_at ?? null) !== null
|| ($job->completion_event_id ?? null) !== null) {
return true;
}
if ($outcome !== 'succeeded') {
return false;
}
return !self::isValidUndispatchedFirewallNoop($job);
}
/** True when one supported Agent generation lost immutable request identity. */
public static function hasAgentRecoveryIdentityConflict(object $job): bool
{
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
(string) ($job->kind ?? '')
)) {
return false;
}
return preg_match(
'/^[a-f0-9]{32}$/D',
(string) ($job->operation_id ?? '')
) !== 1 || preg_match(
'/^[a-f0-9]{64}$/D',
(string) ($job->request_fingerprint ?? '')
) !== 1 || preg_match(
'/^[A-Za-z0-9_.-]{16,128}$/D',
(string) ($job->request_idempotency_key ?? '')
) !== 1;
}
/** Generation zero success exists only for an unchanged firewall policy. */
private static function isValidUndispatchedFirewallNoop(object $job): bool
{
if ((string) ($job->kind ?? '') !== 'firewall_apply'
|| (int) ($job->instance_id ?? 0) < 1
|| !is_string($job->result_json ?? null)) {
return false;
}
$result = json_decode((string) $job->result_json, true);
return is_array($result)
&& !array_is_list($result)
&& ($result['unchanged'] ?? null) === true
&& is_string($result['policy_digest'] ?? null)
&& preg_match(
'/^[a-f0-9]{64}$/D',
$result['policy_digest']
) === 1
&& is_int($result['revision'] ?? null)
&& $result['revision'] > 0;
}
/**
* Operations that still own an instance resource. Active work blocks new
* mutations; a terminal unknown-effect failure remains fenced until a
* kind-specific, audited ledger finalizer has safely converged it.
*/
public static function blockingQueryForInstance(int $instanceId)
{
return self::blockingQueryForInstances([$instanceId]);
}
/** @param list<int> $instanceIds */
public static function blockingQueryForInstances(array $instanceIds)
{
$query = CloudJob::whereIn('instance_id', $instanceIds);
self::applyBlockingInstanceState($query);
return $query;
}
/** @param list<int> $orderIds */
public static function blockingCreateQueryForOrders(array $orderIds)
{
$query = CloudJob::whereIn('order_id', $orderIds)
->where('kind', 'create');
self::applyBlockingInstanceState($query);
return $query;
}
/**
* Serialize host-global VPC route mutations before Agent admission.
* Independent VM-port jobs remain concurrent. A terminal route with an
* unresolved effect or finalization incident keeps the host lane fenced.
*/
public static function applyVpcHostDispatchFence($query)
{
$routeKinds = ['vpc_host_apply', 'vpc_host_remove'];
return $query->where(static function ($eligible) use (
$routeKinds
): void {
$eligible->whereNotIn('cloud_jobs.kind', $routeKinds)
->orWhereNotExists(static function ($predecessors) use (
$routeKinds
): void {
$predecessors->selectRaw('1')
->from('cloud_jobs as vpc_route_predecessors')
->whereColumn(
'vpc_route_predecessors.host_id',
'cloud_jobs.host_id'
)
->whereColumn(
'vpc_route_predecessors.id',
'<',
'cloud_jobs.id'
)
->whereIn(
'vpc_route_predecessors.kind',
$routeKinds
)
->where(static function ($outsideRepair): void {
$outsideRepair->whereNull(
'cloud_jobs.vpc_repair_root_job_id'
)->orWhereRaw(
'COALESCE('
. 'vpc_route_predecessors.vpc_repair_root_job_id, '
. 'vpc_route_predecessors.id) <> '
. 'cloud_jobs.vpc_repair_root_job_id'
);
})
->where(static function ($blocking): void {
$blocking->where(
'vpc_route_predecessors.phase',
'<>',
'terminal'
)->orWhere(static function ($terminal): void {
$terminal->where(
'vpc_route_predecessors.phase',
'terminal'
)->where(static function ($unresolved): void {
$unresolved->where(
'vpc_route_predecessors.effect_resolution',
'unknown'
)->orWhere(
'vpc_route_predecessors.finalization_incident',
1
);
});
});
});
});
});
}
/**
* A stock-mode switch cannot cross an upgrade that still owns either the
* source offer or the target offer. The offer row is locked by the caller,
* so enqueue either commits its job first and is observed here, or reads
* the administrator's new mode as its immutable source snapshot.
*/
public static function blockingUpgradeQueryForOffer(int $offerId)
{
$query = CloudJob::where('kind', 'instance_upgrade')
->where(static function ($linked) use ($offerId): void {
$linked->whereExists(static function ($instances) use ($offerId): void {
$instances->selectRaw('1')->from('cloud_instances')
->whereColumn('cloud_instances.id', 'cloud_jobs.instance_id')
->where('cloud_instances.offer_id', $offerId);
})->orWhereExists(static function ($invoices) use ($offerId): void {
$invoices->selectRaw('1')->from('cloud_invoices')
->whereColumn('cloud_invoices.id', 'cloud_jobs.invoice_id')
->where('cloud_invoices.purpose', 'upgrade')
->where('cloud_invoices.offer_id', $offerId);
});
});
self::applyBlockingInstanceState($query);
return $query;
}
private static function applyBlockingInstanceState($query): void
{
$query->where(static function ($query): void {
$query->where('phase', '<>', 'terminal')
->orWhere(static function ($terminal): void {
$terminal->where('phase', 'terminal')
->where('outcome', 'failed')
->where(static function ($blocked): void {
$blocked->where('error_code', 'force_required')
->orWhere(static function ($unknown): void {
$unknown->where('effect_resolution', 'unknown')
->where(static function ($effect): void {
$effect->where('effect_may_have_applied', 1)
->orWhere('safe_to_resubmit', 0);
});
});
});
})->orWhere(static function ($mode): void {
$mode->whereIn(
'kind',
CloudOperationRecoveryPolicy::automaticKinds()
)->where('phase', 'terminal')
->where(static function ($invalid): void {
$invalid->whereNull('execution_mode')
->orWhere(
'execution_mode',
'<>',
'agent'
);
});
})->orWhere(static function ($invalid): void {
// Historical rows written by a fail-open release must
// become blocking again immediately after upgrade,
// even before a recovery worker has touched them.
$invalid->where('execution_mode', 'agent')
->where(static function ($counter): void {
$counter->whereNull('dispatch_attempts')
->orWhereNotIn(
'dispatch_attempts',
[0, 1]
);
});
})->orWhere(static function ($incident): void {
$incident->where('execution_mode', 'agent')
->where('phase', 'terminal')
->where('finalization_incident', 1)
->where(static function ($recoverable): void {
$recoverable->whereIn(
'finalization_error_code',
CloudDispatchLedger::repairIncidentCodes()
)->orWhere(static function ($local): void {
$local->where('outcome', 'failed')
->where('status', 'failed')
->where('dispatch_attempts', 0)
->whereIn(
'kind',
CloudOperationRecoveryPolicy::automaticKinds()
);
})->orWhere(static function ($resolved): void {
$resolved->whereIn(
'dispatch_attempts',
[0, 1]
)
->whereIn(
'effect_resolution',
['applied', 'not_applied']
)->whereIn(
'kind',
CloudOperationRecoveryPolicy::automaticKinds()
);
});
});
})->orWhere(static function ($state): void {
$state->where('execution_mode', 'agent')
->whereIn(
'kind',
CloudOperationRecoveryPolicy::automaticKinds()
)->where('phase', 'terminal')
->whereIn('dispatch_attempts', [0, 1])
->where('effect_resolution', 'unknown')
->where(static function ($invalid): void {
self::applyInvalidTerminalTuple($invalid);
});
})->orWhere(static function ($resolvedState): void {
$resolvedState->whereIn(
'kind',
CloudOperationRecoveryPolicy::automaticKinds()
)->where('phase', 'terminal')
->where(static function ($invalid): void {
self::applyInvalidResolvedTerminalTuple(
$invalid
);
});
});
});
}
public static function blockingQueryForInvoice(int $invoiceId)
{
return CloudJob::where('invoice_id', $invoiceId)
->where('kind', 'payment_create')
->where(static function ($query): void {
$query->where('phase', '<>', 'terminal')
->orWhere(static function ($terminal): void {
$terminal->where('phase', 'terminal')
->where('outcome', 'failed')
->where('effect_resolution', 'unknown')
->where('safe_to_resubmit', 0);
});
});
}
/**
* Reserve the one public task that represents a multi-host mutation.
*
* Callers must invoke this while holding their catalog authority row (or
* before creating that row) inside the same database transaction. The
* aggregate is never claimed by a worker and can only be completed by the
* terminal transition of its persisted child jobs.
*
* @return array{0: CloudJob, 1: bool} job and whether it was newly created
*/
public static function reserveAggregate(
array $actor,
string $kind,
array $canonicalRequest
): array {
$userId = (int) ($actor['user_id'] ?? 0);
if ($userId < 1 || !self::isAggregateKind($kind)) {
throw new CloudError('任务请求无效', 400);
}
$fingerprint = hash('sha256', json_encode(
self::canonicalRequestValue([
'kind' => $kind,
'request' => $canonicalRequest,
]), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
));
$requestKey = self::requestKey($actor);
if ($requestKey !== null) {
$prior = CloudJob::where('user_id', $userId)
->where('request_idempotency_key', $requestKey)
->lockForUpdate()->first();
if ($prior !== null) {
if (!hash_equals((string) $prior->request_fingerprint,
$fingerprint)) {
throw new CloudError(
'同一请求标识不能用于不同的任务', 409
);
}
return [$prior, false];
}
}
$job = new CloudJob();
$job->operation_id = bin2hex(random_bytes(16));
$job->request_idempotency_key = $requestKey
?? (string) $job->operation_id;
$job->request_fingerprint = $fingerprint;
$job->user_id = $userId;
$job->host_id = null;
$job->kind = $kind;
$job->execution_mode = 'aggregate';
$job->sealed_payload = CloudSecrets::encrypt([
'aggregate' => true,
], 'job-payload');
$now = time();
$job->receipt_deadline_at = $now
+ self::aggregateDeadlineSeconds($kind);
$job->available_at = $job->created_at = $job->updated_at = $now;
$job->save();
return [$job, true];
}
/**
* Reserve one public, host-targeted mutation with a plaintext canonical
* fingerprint. Replaying the same HTTP idempotency key returns the first
* CloudJob; it never creates another Agent operation.
*
* @return array{0: CloudJob, 1: bool}
*/
public static function reserveStandaloneAgentMutation(
array $actor,
int $hostId,
string $kind,
string $method,
string $target,
array $body,
array $metadata,
array $canonicalRequest,
?string $operationId = null,
?string $idempotencyTarget = null
): array {
$userId = (int) ($actor['user_id'] ?? 0);
$method = strtoupper($method);
if ($userId < 1 || $hostId < 1
|| !array_key_exists($kind, CloudOperationCatalog::kinds())
|| !in_array($method, ['POST', 'PUT', 'PATCH', 'DELETE'], true)
|| !str_starts_with($target, '/v1/')) {
throw new CloudError('Agent 任务请求无效', 400);
}
$fingerprint = self::standaloneRequestFingerprint(
$hostId,
$kind,
$method,
$idempotencyTarget ?? $target,
$canonicalRequest
);
$requestKey = self::requestKey($actor);
if ($requestKey !== null) {
$prior = CloudJob::where('user_id', $userId)
->where('request_idempotency_key', $requestKey)
->lockForUpdate()->first();
if ($prior !== null) {
if (!hash_equals((string) $prior->request_fingerprint,
$fingerprint)) {
throw new CloudError(
'同一请求标识不能用于不同的任务', 409
);
}
return [$prior, false];
}
}
$operationId ??= bin2hex(random_bytes(16));
if (preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1) {
throw new CloudError('Agent 任务操作标识无效', 400);
}
$body['operation_id'] = $operationId;
$job = new CloudJob();
$job->operation_id = $operationId;
$job->request_idempotency_key = $requestKey ?? $operationId;
$job->request_fingerprint = $fingerprint;
$job->user_id = $userId;
$job->host_id = $hostId;
$job->kind = $kind;
$job->execution_mode = 'agent';
$job->sealed_payload = CloudSecrets::encrypt(array_merge($metadata, [
'method' => $method,
'request_target' => $target,
'body' => $body,
'owner_id' => (int) ($metadata['owner_id'] ?? $userId),
'can_manage' => (bool) ($metadata['can_manage'] ?? false),
]), 'job-payload');
$now = time();
$job->dispatch_deadline_at = $now
+ self::dispatchDeadlineSeconds($kind);
$job->available_at = $job->created_at = $job->updated_at = $now;
$job->save();
return [$job, true];
}
/** Exact public idempotency fingerprint used before external preparation. */
public static function standaloneRequestFingerprint(
int $hostId,
string $kind,
string $method,
string $target,
array $canonicalRequest
): string {
$method = strtoupper($method);
if ($hostId < 1
|| !array_key_exists($kind, CloudOperationCatalog::kinds())
|| !in_array($method, ['POST', 'PUT', 'PATCH', 'DELETE'], true)
|| !str_starts_with($target, '/v1/')) {
throw new CloudError('Agent 任务请求无效', 400);
}
return hash('sha256', json_encode(
self::canonicalRequestValue([
'host_id' => $hostId,
'kind' => $kind,
'method' => $method,
'target' => $target,
'request' => $canonicalRequest,
]), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
));
}
/** Persist one exact Agent mutation. It can be claimed only once. */
public static function reserveAgentMutation(
array $actor,
int $hostId,
string $kind,
string $method,
string $target,
array $body,
array $metadata = [],
?int $parentJobId = null,
?string $operationId = null
): CloudJob {
$userId = (int) ($actor['user_id'] ?? 0);
$method = strtoupper($method);
if ($userId < 1 || $hostId < 1
|| !array_key_exists($kind, CloudOperationCatalog::kinds())
|| !in_array($method, ['POST', 'PUT', 'PATCH', 'DELETE'], true)
|| !str_starts_with($target, '/v1/')) {
throw new CloudError('Agent 任务请求无效', 400);
}
$operationId ??= bin2hex(random_bytes(16));
if (preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1) {
throw new CloudError('Agent 任务操作标识无效', 400);
}
$body['operation_id'] = $operationId;
$fingerprint = hash('sha256', json_encode(
self::canonicalRequestValue([
'host_id' => $hostId,
'kind' => $kind,
'method' => $method,
'target' => $target,
'body' => $body,
'metadata' => $metadata,
]), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
));
$job = new CloudJob();
$job->operation_id = $operationId;
$job->request_idempotency_key = $operationId;
$job->request_fingerprint = $fingerprint;
$job->parent_job_id = $parentJobId;
$job->user_id = $userId;
$job->host_id = $hostId;
$job->kind = $kind;
$job->execution_mode = 'agent';
$job->sealed_payload = CloudSecrets::encrypt(array_merge($metadata, [
'method' => $method,
'request_target' => $target,
'body' => $body,
'owner_id' => (int) ($metadata['owner_id'] ?? $userId),
'can_manage' => (bool) ($metadata['can_manage'] ?? false),
]), 'job-payload');
$now = time();
$job->dispatch_deadline_at = $now
+ self::dispatchDeadlineSeconds($kind);
$job->available_at = $job->created_at = $job->updated_at = $now;
$job->save();
return $job;
}
/** Strict HTTP acknowledgement used by every asynchronous mutation. */
public static function acknowledgement(CloudJob $job): array
{
return [
'task_id' => (string) $job->id,
'status_url' => '/api/v1/tasks/' . (int) $job->id,
];
}
/**
* Record a proven no-op as a successful public task without ever making it
* claimable by a worker. The row is first persisted with the normal task
* invariants, then terminalized in the caller's database transaction.
*/
public static function completeWithoutDispatchLocked(
CloudJob $job,
array $result
): void {
if (!$job->exists
|| (string) $job->phase !== 'accepted'
|| (int) $job->dispatch_attempts !== 0
|| (int) $job->attempts !== 0
|| $job->dispatch_started_at !== null
|| $job->dispatched_at !== null
|| $job->agent_accepted_at !== null
|| $job->receipt_deadline_at !== null) {
throw new \LogicException(
'Only a fresh, undispatched task can complete as a no-op'
);
}
self::completeAggregateLocked(
$job,
'succeeded',
false,
false,
'',
$result
);
}
/** Complete an aggregate without dispatching another Agent mutation. */
public static function completeAggregateLocked(
CloudJob $job,
string $outcome,
bool $effectMayHaveApplied,
bool $safeToResubmit,
string $errorCode = '',
?array $result = null
): void {
if ((string) $job->phase === 'terminal') {
return;
}
if (!in_array($outcome, ['succeeded', 'failed'], true)) {
throw new \InvalidArgumentException('Invalid aggregate outcome');
}
if ($outcome !== 'failed') {
$effectMayHaveApplied = false;
$safeToResubmit = false;
$errorCode = '';
} elseif ($effectMayHaveApplied) {
$safeToResubmit = false;
}
$now = time();
$job->phase = 'terminal';
$job->outcome = $outcome;
$job->status = $outcome;
$job->terminal_at = $now;
$job->effect_may_have_applied = $effectMayHaveApplied ? 1 : 0;
$job->safe_to_resubmit = $safeToResubmit ? 1 : 0;
$job->effect_resolution = $outcome === 'succeeded' ? 'applied'
: ($effectMayHaveApplied ? 'unknown' : 'not_applied');
$job->error_code = $errorCode === '' ? null
: substr($errorCode, 0, 128);
$job->result_json = $result === null ? null : json_encode(
$result,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
);
$job->sealed_payload = CloudSecrets::encrypt([], 'job-payload');
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->updated_at = $now;
$job->save();
}
/**
* Release a terminal aggregate fence after every Agent child has reached
* an evidence-backed resolution. The parent's failed audit outcome remains
* immutable; only its effect resolution is published.
*/
public static function reconcileAggregateParentEffectLocked(
CloudJob $child,
?int $resolvedBy
): void {
if (!in_array(
(string) $child->kind,
array_values(self::AGGREGATE_CHILD_KIND_MAP),
true
)) {
return;
}
$parentId = (int) ($child->parent_job_id ?? 0);
if ($parentId < 1) {
throw new CloudRecoveryAggregateStateConflict(
'汇总子任务缺少父任务账本',
409
);
}
$parent = CloudJob::where('id', $parentId)
->lockForUpdate()->first();
if ($parent === null
|| (string) $parent->execution_mode !== 'aggregate') {
throw new CloudRecoveryAggregateStateConflict(
'汇总子任务父任务账本不存在或模式不匹配',
409
);
}
// Reject a malformed parent/child topology before catalog discovery.
// This is a read-only preflight while the parent is held; valid paths
// still acquire every write lock in parent -> catalog -> child order
// and revalidate the relationship after the child locks are taken.
$childSnapshot = CloudJob::where('parent_job_id', $parentId)
->orderBy('id')->get();
if (self::aggregateRelationshipConflict(
$parent,
$childSnapshot,
(int) $child->id
)) {
self::deferAggregateRecoveryLocked(
$parent,
'recovery_aggregate_state_conflict',
time()
);
return;
}
CloudTemplateCatalogService::lockAggregateCatalogLocked($parent);
CloudUserIsoService::lockAggregateCatalogLocked($parent);
$children = CloudJob::where('parent_job_id', $parentId)
->orderBy('id')->lockForUpdate()->get();
if (self::aggregateRelationshipConflict(
$parent,
$children,
(int) $child->id
)) {
self::deferAggregateRecoveryLocked(
$parent,
'recovery_aggregate_state_conflict',
time()
);
return;
}
$publishedParent = self::aggregateParentHasPublishedEffect($parent);
if ($children->contains(static fn (CloudJob $row): bool =>
self::aggregateChildRecoveryPending($row))) {
if ($publishedParent) {
self::deferAggregateRecoveryLocked(
$parent,
'recovery_child_convergence',
time()
);
}
return;
}
$resolution = self::aggregateEffectResolution($children);
$metadataMixed = $resolution === 'mixed'
&& (string) $parent->kind === 'template_metadata_sync';
if ($resolution === null) {
self::deferAggregateRecoveryLocked(
$parent,
'recovery_aggregate_state_conflict',
time()
);
return;
}
if ($publishedParent) {
self::recheckPublishedAggregateProjectionLocked(
$parent,
$resolution,
$metadataMixed
);
return;
}
$partialMixed = $resolution === 'mixed' && !$metadataMixed;
if ($partialMixed) {
// Validate every child-to-replica identity before publishing a
// concrete failed-but-applied partial result. The aggregate stays
// failed; "applied" means at least one proven host effect exists,
// never that every replica succeeded.
CloudTemplateCatalogService::settleRecoveredParentLocked(
$parent,
'mixed'
);
CloudUserIsoService::settleRecoveredParentLocked(
$parent,
'mixed'
);
if ((string) $parent->phase !== 'terminal') {
$parent->refresh();
}
}
$projectionSettled = $partialMixed;
if (!$partialMixed && (string) $parent->phase !== 'terminal') {
// Recovered leaf projections are prepared before the matching
// Agent fence is released. Only this post-commit hook may now
// publish the user-visible aggregate result.
CloudTemplateCatalogService::settleRecoveredParentLocked(
$parent,
$resolution
);
CloudUserIsoService::settleRecoveredParentLocked(
$parent,
$resolution
);
$parent->refresh();
if ((string) $parent->outcome !== 'failed'
|| (string) $parent->effect_resolution !== 'unknown') {
return;
}
$projectionSettled = true;
}
if ((string) $parent->outcome !== 'failed'
|| (string) $parent->effect_resolution !== 'unknown') {
return;
}
// A deadline may have failed the immutable public task and marked the
// catalog record failed before late Agent evidence converged every
// replica. Recompute only that resource projection now. The service
// finalizers call completeAggregateLocked(), whose terminal guard keeps
// the original parent phase/outcome/status/timestamps/error/result
// immutable.
if (!$projectionSettled) {
CloudTemplateCatalogService::settleRecoveredParentLocked(
$parent,
$resolution
);
CloudUserIsoService::settleRecoveredParentLocked(
$parent,
$resolution
);
}
if (!CloudTemplateCatalogService::aggregateEffectProjectionReadyLocked(
$parent,
$resolution
)) {
self::deferAggregateRecoveryLocked(
$parent,
'recovery_child_convergence',
time()
);
return;
}
$publishedResolution = $metadataMixed || $partialMixed
? 'applied' : $resolution;
$parent->effect_resolution = $publishedResolution;
$parent->effect_resolved_at = time();
$parent->effect_resolved_by = $resolvedBy;
$parent->effect_resolution_reason = sprintf(
'child_effects_resolved:%d:%s',
(int) $child->id,
$partialMixed ? 'mixed' : $publishedResolution
);
$parent->finalization_incident = 0;
$parent->finalization_error_code = null;
$parent->lease_token = null;
$parent->lease_until = 0;
$parent->available_at = 0;
$parent->read_at = null;
$parent->updated_at = time();
$parent->save();
}
/**
* Converge one aggregate fence left behind before the realtime child hook
* existed. This is a PHP-ledger-only sweep: it never dispatches or inspects
* an Agent operation. Parent then children is the only explicit lock order,
* and children are locked by id to avoid sibling-order inversions. It is
* also the durable fallback if a realtime sibling hook loses a deadlock
* retry or PHP exits after committing the final child.
*/
private static function wherePublishedAggregateNeedsRecheck(
$published
): void {
$published->whereIn(
'effect_resolution',
['applied', 'not_applied']
)->where(static function ($needs): void {
$needs->whereNull('dispatch_attempts')
->orWhere('dispatch_attempts', '<>', 0)
->orWhere(static function ($incident): void {
$incident->where('finalization_incident', 1);
})->orWhereExists(static function ($templates): void {
$templates->selectRaw('1')
->from('cloud_templates as current_templates')
->whereColumn(
'current_templates.active_job_id',
'cloud_jobs.id'
);
})->orWhereExists(static function ($isos): void {
$isos->selectRaw('1')
->from('cloud_user_isos as current_isos')
->whereColumn(
'current_isos.active_job_id',
'cloud_jobs.id'
);
});
});
}
public static function reconcileOneLegacyAggregateEffect(): bool
{
$connection = (new CloudJob())->getConnection();
$claimedParentId = 0;
try {
return $connection->transaction(
static function () use (&$claimedParentId): bool {
$now = time();
$parent = CloudJob::where('phase', 'terminal')
->where(static function ($candidate): void {
$candidate->where(
static function ($aggregate): void {
$aggregate->whereIn(
'kind',
array_keys(
self::AGGREGATE_CHILD_KIND_MAP
)
)->where(
static function ($recoverable): void {
$recoverable->where(
static function ($unknown): void {
$unknown->where(
'outcome',
'failed'
)->where(
'status',
'failed'
)->where(
'effect_may_have_applied',
1
)->where(
'safe_to_resubmit',
0
)->where(
'effect_resolution',
'unknown'
);
}
)->orWhere(
static function ($ledger): void {
$ledger->whereNull(
'dispatch_attempts'
)->orWhere(
'dispatch_attempts',
'<>',
0
);
}
)->orWhere(
static function ($published): void {
self::wherePublishedAggregateNeedsRecheck(
$published
);
}
)->orWhere(
static function ($damaged): void {
self::whereAggregateLocalStateConflict(
$damaged
);
}
);
}
);
}
)->orWhereRaw(
self::aggregateLocalStateConflictSql(
'cloud_jobs'
)
)->orWhere(
static function ($bound): void {
self::whereCurrentAggregateCatalogBinding(
$bound
);
}
)->orWhere(static function ($incident): void {
$incident->whereNull('parent_job_id')
->where('finalization_incident', 1)
->where(
'finalization_error_code',
'recovery_aggregate_state_conflict'
);
});
})
->where('available_at', '<=', $now)
->where('lease_until', '<=', $now)
->orderByRaw(
"CASE WHEN effect_resolution = 'unknown' "
. 'THEN 0 ELSE 1 END'
)
->orderBy('available_at')
->orderBy('id')
->lockForUpdate()
->first();
if ($parent === null) {
return false;
}
$claimedParentId = (int) $parent->id;
// Classify a corrupt aggregate locally before catalog
// discovery. The parent lock serializes legitimate child
// writers; valid paths still re-read every child under
// lock after the catalog row is locked.
$childSnapshot = CloudJob::where(
'parent_job_id',
(int) $parent->id
)->orderBy('id')->get();
if (self::aggregateRelationshipConflict(
$parent,
$childSnapshot
)) {
self::deferAggregateRecoveryLocked(
$parent,
'recovery_aggregate_state_conflict',
$now
);
return true;
}
CloudTemplateCatalogService::lockAggregateCatalogLocked(
$parent
);
CloudUserIsoService::lockAggregateCatalogLocked($parent);
$children = CloudJob::where(
'parent_job_id',
(int) $parent->id
)->orderBy('id')->lockForUpdate()->get();
if (self::aggregateRelationshipConflict(
$parent,
$children
)) {
self::deferAggregateRecoveryLocked(
$parent,
'recovery_aggregate_state_conflict',
$now
);
return true;
}
$waiting = $children->contains(
static fn (CloudJob $child): bool =>
self::aggregateChildRecoveryPending($child)
);
if ($waiting) {
self::deferAggregateRecoveryLocked(
$parent,
'recovery_child_convergence',
$now
);
return true;
}
$resolution = self::aggregateEffectResolution($children);
$metadataMixed = $resolution === 'mixed'
&& (string) $parent->kind
=== 'template_metadata_sync';
if ($resolution === null) {
self::deferAggregateRecoveryLocked(
$parent,
'recovery_aggregate_state_conflict',
$now
);
return true;
}
if (self::aggregateParentHasPublishedEffect($parent)) {
self::recheckPublishedAggregateProjectionLocked(
$parent,
$resolution,
$metadataMixed
);
return true;
}
$partialMixed = $resolution === 'mixed'
&& !$metadataMixed;
if ($partialMixed) {
CloudTemplateCatalogService::settleRecoveredParentLocked(
$parent,
'mixed'
);
CloudUserIsoService::settleRecoveredParentLocked(
$parent,
'mixed'
);
}
// See reconcileAggregateParentEffectLocked(): only repair
// the catalog projection; never rewrite the failed audit.
if (!$partialMixed) {
CloudTemplateCatalogService::settleRecoveredParentLocked(
$parent,
$resolution
);
CloudUserIsoService::settleRecoveredParentLocked(
$parent,
$resolution
);
}
if (!CloudTemplateCatalogService::aggregateEffectProjectionReadyLocked(
$parent,
$resolution
)) {
self::deferAggregateRecoveryLocked(
$parent,
'recovery_child_convergence',
$now
);
return true;
}
$publishedResolution = $metadataMixed || $partialMixed
? 'applied' : $resolution;
$lastChild = $children->last();
$parent->effect_resolution = $publishedResolution;
$parent->effect_resolved_at = $now;
$parent->effect_resolved_by = null;
$parent->effect_resolution_reason = sprintf(
'legacy_child_effects_resolved:%d:%s',
(int) $lastChild->id,
$partialMixed ? 'mixed' : $publishedResolution
);
$parent->finalization_incident = 0;
$parent->finalization_error_code = null;
$parent->lease_token = null;
$parent->lease_until = 0;
$parent->available_at = 0;
$parent->read_at = null;
$parent->updated_at = $now;
$parent->save();
return true;
},
3
);
} catch (\Throwable $error) {
if ($claimedParentId < 1) {
throw $error;
}
// The first transaction rolled back both the resource projection
// and parent effect resolution. Persist only a delayed incident in
// a fresh transaction so one damaged legacy parent cannot abort
// every recovery tick or starve later jobs.
$connection->transaction(
static function () use ($claimedParentId, $error): void {
$parent = CloudJob::where('id', $claimedParentId)
->lockForUpdate()->first();
if ($parent === null
|| (string) $parent->execution_mode !== 'aggregate'
|| (string) $parent->phase !== 'terminal') {
return;
}
$unknownFence = (string) $parent->outcome === 'failed'
&& (string) $parent->status === 'failed'
&& (string) $parent->effect_resolution === 'unknown';
$publishedParent = self::aggregateParentHasPublishedEffect(
$parent
);
if (!$unknownFence && !$publishedParent) {
return;
}
self::deferAggregateRecoveryLocked(
$parent,
self::aggregateProjectionErrorCode($error),
time()
);
},
3
);
error_log(sprintf(
'cloud aggregate projection recovery pending job_id=%d error=%s',
$claimedParentId,
get_class($error)
));
return true;
}
}
/** Reject aggregate topology damage before any external fence mutation. */
public static function assertAggregateChildRelationshipLocked(
CloudJob $parent,
CloudJob $child,
$children
): void {
if (self::aggregateRelationshipConflict(
$parent,
$children,
(int) $child->id
)) {
throw new CloudRecoveryAggregateStateConflict(
'汇总任务与子任务账本关系异常',
409
);
}
}
private static function aggregateRelationshipConflict(
CloudJob $parent,
$children,
?int $currentChildId = null
): bool {
$parentId = (int) $parent->id;
$expectedKind = self::AGGREGATE_CHILD_KIND_MAP[
(string) $parent->kind
] ?? null;
return self::hasAggregateLocalStateConflict($parent)
|| $expectedKind === null
|| $children->isEmpty()
|| ($currentChildId !== null && !$children->contains(
static fn (CloudJob $child): bool =>
(int) $child->id === $currentChildId
))
|| $children->contains(
static fn (CloudJob $child): bool =>
(int) ($child->parent_job_id ?? 0) !== $parentId
|| (int) $child->user_id !== (int) $parent->user_id
|| (string) $child->execution_mode !== 'agent'
|| (string) $child->kind !== $expectedKind
|| self::aggregateChildHasPhaseResolutionConflict($child)
|| self::hasResolvedTerminalStateConflict($child)
);
}
/**
* SQL counterpart of hasAggregateLocalStateConflict() for terminal rows.
* This keeps an unbound, already-published parent reachable by the local
* ledger sweep without polling every healthy historical aggregate.
*/
private static function whereAggregateLocalStateConflict($damaged): void
{
$operationInvalid = self::invalidIdentifierCharactersSql(
'operation_id', '0-9a-f'
);
$fingerprintInvalid = self::invalidIdentifierCharactersSql(
'request_fingerprint', '0-9a-f'
);
$idempotencyInvalid = self::invalidIdentifierCharactersSql(
'request_idempotency_key', 'A-Za-z0-9_.-'
);
$damaged->where(static function ($invalid) use (
$operationInvalid,
$fingerprintInvalid,
$idempotencyInvalid
): void {
$invalid->whereNotNull('parent_job_id')
->orWhereNotNull('host_id')
->orWhereNotNull('instance_id')
->orWhereNotNull('order_id')
->orWhereNotNull('invoice_id')
->orWhereNotNull('snapshot_id')
->orWhereNull('user_id')
->orWhere('user_id', '<', 1)
->orWhereNull('attempts')
->orWhere('attempts', '<>', 0)
->orWhereNull('dispatch_attempts')
->orWhere('dispatch_attempts', '<>', 0)
->orWhereNull('dispatch_deadline_at')
->orWhere('dispatch_deadline_at', '<', 1)
->orWhereNull('receipt_deadline_at')
->orWhere('receipt_deadline_at', '<', 1)
->orWhereNotNull('dispatch_started_at')
->orWhereNotNull('agent_accepted_at')
->orWhereNotNull('dispatched_at')
->orWhereNotNull('completion_event_id')
->orWhereNull('operation_id')
->orWhereRaw('LENGTH(operation_id) <> 32')
->orWhereRaw('LOWER(operation_id) <> operation_id')
->orWhereRaw($operationInvalid)
->orWhereNull('request_fingerprint')
->orWhereRaw('LENGTH(request_fingerprint) <> 64')
->orWhereRaw(
'LOWER(request_fingerprint) <> request_fingerprint'
)->orWhereRaw($fingerprintInvalid)
->orWhereNull('request_idempotency_key')
->orWhereRaw(
'LENGTH(request_idempotency_key) NOT BETWEEN 16 AND 128'
)->orWhereRaw($idempotencyInvalid)
->orWhereNull('terminal_at')
->orWhere('terminal_at', '<', 1)
->orWhereNull('outcome')
->orWhereNotIn('outcome', ['succeeded', 'failed'])
->orWhereNull('status')
->orWhereColumn('status', '<>', 'outcome')
->orWhere(static function ($succeeded): void {
$succeeded->where('outcome', 'succeeded')
->where(static function ($mismatch): void {
$mismatch->whereNull('effect_resolution')
->orWhere(
'effect_resolution',
'<>',
'applied'
)->orWhereNull('effect_may_have_applied')
->orWhere('effect_may_have_applied', '<>', 0)
->orWhereNull('safe_to_resubmit')
->orWhere('safe_to_resubmit', '<>', 0)
->orWhereNotNull('error_code');
});
})->orWhere(static function ($failed): void {
$failed->where('outcome', 'failed')
->where(static function ($mismatch): void {
$mismatch->whereNull('effect_resolution')
->orWhereNotIn(
'effect_resolution',
['unknown', 'applied', 'not_applied']
)->orWhereNull('effect_may_have_applied')
->orWhereNotIn(
'effect_may_have_applied',
[0, 1]
)->orWhereNull('safe_to_resubmit')
->orWhereNotIn('safe_to_resubmit', [0, 1])
->orWhere(static function ($effectful): void {
$effectful->where(
'effect_may_have_applied',
1
)->where('safe_to_resubmit', '<>', 0);
})->orWhere(static function ($notEffectful): void {
$notEffectful->where(
'effect_may_have_applied',
0
)->where(
'effect_resolution',
'<>',
'not_applied'
);
});
});
});
});
}
/** Add resolved Agent rows whose immutable dispatch envelope is damaged. */
private static function applyInvalidResolvedAgentEnvelope(
$invalid,
string $prefix
): void {
$operation = $prefix . 'operation_id';
$fingerprint = $prefix . 'request_fingerprint';
$idempotency = $prefix . 'request_idempotency_key';
$invalid->whereNull($prefix . 'execution_mode')
->orWhere($prefix . 'execution_mode', '<>', 'agent')
->orWhereNull($prefix . 'dispatch_attempts')
->orWhereNotIn($prefix . 'dispatch_attempts', [0, 1])
->orWhereNull($prefix . 'host_id')
->orWhere($prefix . 'host_id', '<', 1)
->orWhereNull($prefix . 'dispatch_deadline_at')
->orWhere($prefix . 'dispatch_deadline_at', '<', 1)
->orWhereNull($operation)
->orWhereRaw('LENGTH(' . $operation . ') <> 32')
->orWhereRaw('LOWER(' . $operation . ') <> ' . $operation)
->orWhereRaw(self::invalidIdentifierCharactersSql(
$operation,
'0-9a-f'
))->orWhereNull($fingerprint)
->orWhereRaw('LENGTH(' . $fingerprint . ') <> 64')
->orWhereRaw('LOWER(' . $fingerprint . ') <> ' . $fingerprint)
->orWhereRaw(self::invalidIdentifierCharactersSql(
$fingerprint,
'0-9a-f'
))->orWhereNull($idempotency)
->orWhereRaw(
'LENGTH(' . $idempotency . ') NOT BETWEEN 16 AND 128'
)->orWhereRaw(self::invalidIdentifierCharactersSql(
$idempotency,
'A-Za-z0-9_.-'
))->orWhere(static function ($dispatched) use ($prefix): void {
$dispatched->where($prefix . 'dispatch_attempts', 1)
->where(static function ($broken) use ($prefix): void {
$broken->whereNull($prefix . 'attempts')
->orWhere($prefix . 'attempts', '<>', 1)
->orWhereNull($prefix . 'dispatch_started_at')
->orWhere($prefix . 'dispatch_started_at', '<', 1)
->orWhere(static function ($accepted) use (
$prefix
): void {
$accepted->whereNotNull(
$prefix . 'agent_accepted_at'
)->where(
$prefix . 'agent_accepted_at',
'<',
1
);
})
->orWhereNull($prefix . 'dispatched_at')
->orWhere($prefix . 'dispatched_at', '<', 1)
->orWhereNull($prefix . 'receipt_deadline_at')
->orWhere($prefix . 'receipt_deadline_at', '<', 1);
});
})->orWhere(static function ($undispatched) use ($prefix): void {
$undispatched->where($prefix . 'dispatch_attempts', 0)
->where(static function ($broken) use ($prefix): void {
$broken->whereNull($prefix . 'attempts')
->orWhere($prefix . 'attempts', '<>', 0)
->orWhereNotNull($prefix . 'dispatch_started_at')
->orWhereNotNull($prefix . 'agent_accepted_at')
->orWhereNotNull($prefix . 'dispatched_at')
->orWhereNotNull($prefix . 'receipt_deadline_at')
->orWhereNotNull($prefix . 'completion_event_id')
->orWhere(static function ($success) use (
$prefix
): void {
$success->where(
$prefix . 'outcome',
'succeeded'
)->where(static function ($notNoop) use (
$prefix
): void {
$notNoop->where(
$prefix . 'kind',
'<>',
'firewall_apply'
)->orWhereNull($prefix . 'instance_id')
->orWhere(
$prefix . 'instance_id',
'<',
1
)->orWhereRaw(
self::invalidFirewallNoopResultSql(
$prefix . 'result_json'
)
);
});
});
});
});
}
/** Driver-specific JSON validation for the sole generation-zero success. */
private static function invalidFirewallNoopResultSql(string $column): string
{
$driver = (new CloudJob())->getConnection()->getDriverName();
if ($driver === 'sqlite') {
$digest = "json_extract(" . $column . ", '$.policy_digest')";
return '(CASE WHEN COALESCE(json_valid(' . $column . '), 0) = 1 '
. 'THEN CASE WHEN '
. "json_type(" . $column . ", '$.unchanged') = 'true' AND "
. "json_type(" . $column . ", '$.policy_digest') = 'text' AND "
. 'LENGTH(' . $digest . ') = 64 AND NOT ('
. self::invalidIdentifierCharactersSql($digest, '0-9a-f')
. ') AND '
. "json_type(" . $column . ", '$.revision') = 'integer' AND "
. "json_extract(" . $column . ", '$.revision') > 0 "
. 'THEN 0 ELSE 1 END ELSE 1 END) = 1';
}
$digest = "JSON_UNQUOTE(JSON_EXTRACT(" . $column
. ", '$.policy_digest'))";
$revision = "JSON_UNQUOTE(JSON_EXTRACT(" . $column
. ", '$.revision'))";
return '(CASE WHEN COALESCE(JSON_VALID(' . $column . '), 0) = 1 '
. 'THEN CASE WHEN JSON_UNQUOTE(JSON_EXTRACT(' . $column
. ", '$.unchanged')) = 'true' AND LENGTH(" . $digest
. ') = 64 AND NOT ('
. self::invalidIdentifierCharactersSql($digest, '0-9a-f')
. ') AND ' . $revision . " REGEXP '^[1-9][0-9]*$' "
. 'THEN 0 ELSE 1 END ELSE 1 END) = 1';
}
/** Build a portable invalid-character predicate for MySQL and SQLite. */
private static function invalidIdentifierCharactersSql(
string $column,
string $allowedClass
): string {
$driver = (new CloudJob())->getConnection()->getDriverName();
if ($driver === 'sqlite') {
return $column . " GLOB '*[^" . $allowedClass . "]*'";
}
return $column . " REGEXP '[^" . $allowedClass . "]'";
}
/**
* Raw-SQL counterpart of hasTerminalRecoveryStateConflict(). Inventory
* filters and counters cannot hydrate every row before filtering, so keep
* their fail-closed envelope test in the same service as the DTO check.
*/
public static function terminalRecoveryStateConflictSql(
string $alias
): string {
$supported = self::quotedSqlValues(
CloudOperationRecoveryPolicy::ledgerFinalizerKinds()
);
$operation = $alias . '.operation_id';
$fingerprint = $alias . '.request_fingerprint';
$idempotency = $alias . '.request_idempotency_key';
$invalidEnvelope = '(' . $alias . '.execution_mode IS NULL OR '
. $alias . ".execution_mode <> 'agent' OR "
. $alias . '.dispatch_attempts IS NULL OR '
. $alias . '.dispatch_attempts NOT IN (0, 1) OR '
. $alias . '.host_id IS NULL OR ' . $alias . '.host_id < 1 OR '
. $alias . '.dispatch_deadline_at IS NULL OR '
. $alias . '.dispatch_deadline_at < 1 OR '
. $operation . ' IS NULL OR LENGTH(' . $operation . ') <> 32 OR '
. 'LOWER(' . $operation . ') <> ' . $operation . ' OR '
. self::invalidIdentifierCharactersSql($operation, '0-9a-f')
. ' OR ' . $fingerprint . ' IS NULL OR LENGTH('
. $fingerprint . ') <> 64 OR LOWER(' . $fingerprint . ') <> '
. $fingerprint . ' OR '
. self::invalidIdentifierCharactersSql($fingerprint, '0-9a-f')
. ' OR ' . $idempotency . ' IS NULL OR LENGTH('
. $idempotency . ') NOT BETWEEN 16 AND 128 OR '
. self::invalidIdentifierCharactersSql(
$idempotency,
'A-Za-z0-9_.-'
) . ' OR (' . $alias . '.dispatch_attempts = 1 AND ('
. $alias . '.attempts IS NULL OR ' . $alias . '.attempts <> 1 OR '
. $alias . '.dispatch_started_at IS NULL OR '
. $alias . '.dispatch_started_at < 1 OR ('
. $alias . '.agent_accepted_at IS NOT NULL AND '
. $alias . '.agent_accepted_at < 1) OR '
. $alias . '.dispatched_at IS NULL OR '
. $alias . '.dispatched_at < 1 OR '
. $alias . '.receipt_deadline_at IS NULL OR '
. $alias . '.receipt_deadline_at < 1)) OR ('
. $alias . '.dispatch_attempts = 0 AND ('
. $alias . '.attempts IS NULL OR ' . $alias . '.attempts <> 0 OR '
. $alias . '.dispatch_started_at IS NOT NULL OR '
. $alias . '.agent_accepted_at IS NOT NULL OR '
. $alias . '.dispatched_at IS NOT NULL OR '
. $alias . '.receipt_deadline_at IS NOT NULL OR '
. $alias . '.completion_event_id IS NOT NULL OR ('
. $alias . ".outcome = 'succeeded' AND ("
. $alias . ".kind <> 'firewall_apply' OR "
. $alias . '.instance_id IS NULL OR '
. $alias . '.instance_id < 1 OR '
. self::invalidFirewallNoopResultSql(
$alias . '.result_json'
) . ')))))';
$invalidResolved = '(' . $alias
. ".effect_resolution IN ('applied', 'not_applied') AND ("
. $alias . '.terminal_at IS NULL OR '
. $alias . '.terminal_at < 1 OR '
. $alias . '.outcome IS NULL OR '
. $alias . ".outcome NOT IN ('succeeded', 'failed') OR ("
. $alias . ".outcome = 'succeeded' AND ("
. $alias . ".status IS NULL OR "
. $alias . ".status <> 'succeeded' OR "
. $alias . ".effect_resolution <> 'applied' OR "
. $alias . '.effect_may_have_applied IS NULL OR '
. $alias . '.effect_may_have_applied <> 0 OR '
. $alias . '.safe_to_resubmit IS NULL OR '
. $alias . '.safe_to_resubmit <> 0 OR '
. $alias . '.error_code IS NOT NULL)) OR ('
. $alias . ".outcome = 'failed' AND ("
. $alias . ".status IS NULL OR "
. $alias . ".status <> 'failed' OR "
. $alias . '.effect_may_have_applied IS NULL OR '
. $alias . '.effect_may_have_applied NOT IN (0, 1) OR ('
. $alias . '.effect_may_have_applied = 1 AND ('
. $alias . '.safe_to_resubmit IS NULL OR '
. $alias . '.safe_to_resubmit <> 0)) OR ('
. $alias . '.effect_may_have_applied = 0 AND ('
. $alias . ".effect_resolution <> 'not_applied' OR "
. $alias . '.safe_to_resubmit IS NULL OR '
. $alias . '.safe_to_resubmit NOT IN (0, 1))))) OR '
. $invalidEnvelope . '))';
$invalidUnknown = '(' . $alias . ".effect_resolution = 'unknown' AND (("
. $alias . '.dispatch_attempts = 0 AND COALESCE('
. $alias . '.finalization_incident, 0) = 0) OR NOT ('
. "COALESCE(" . $alias . ".outcome, '') = 'failed' AND "
. "COALESCE(" . $alias . ".status, '') = 'failed' AND "
. 'COALESCE(' . $alias . '.effect_may_have_applied, 0) = 1 AND '
. 'COALESCE(' . $alias . '.safe_to_resubmit, 1) = 0)))';
return '(' . $alias . ".phase = 'terminal' AND "
. $alias . '.kind IN (' . $supported . ') AND ('
. $alias . '.execution_mode IS NULL OR '
. $alias . ".execution_mode <> 'agent' OR "
. $alias . '.dispatch_attempts IS NULL OR '
. $alias . '.dispatch_attempts NOT IN (0, 1) OR '
. $invalidEnvelope . ' OR ' . $invalidUnknown . ' OR '
. $invalidResolved . '))';
}
/** SQL counterpart of hasAggregateLocalStateConflict() for terminal rows. */
public static function aggregateLocalStateConflictSql(
string $alias
): string {
$kinds = self::quotedSqlValues(self::AGGREGATE_KINDS);
$operation = $alias . '.operation_id';
$fingerprint = $alias . '.request_fingerprint';
$idempotency = $alias . '.request_idempotency_key';
return '(' . $alias . ".phase = 'terminal' AND ("
. $alias . ".execution_mode = 'aggregate' OR "
. $alias . '.kind IN (' . $kinds . ')) AND ('
. $alias . '.execution_mode IS NULL OR '
. $alias . ".execution_mode <> 'aggregate' OR "
. $alias . '.kind IS NULL OR '
. $alias . '.kind NOT IN (' . $kinds . ') OR '
. $alias . '.parent_job_id IS NOT NULL OR '
. $alias . '.host_id IS NOT NULL OR '
. $alias . '.instance_id IS NOT NULL OR '
. $alias . '.order_id IS NOT NULL OR '
. $alias . '.invoice_id IS NOT NULL OR '
. $alias . '.snapshot_id IS NOT NULL OR '
. $alias . '.user_id IS NULL OR ' . $alias . '.user_id < 1 OR '
. $alias . '.attempts IS NULL OR ' . $alias . '.attempts <> 0 OR '
. $alias . '.dispatch_attempts IS NULL OR '
. $alias . '.dispatch_attempts <> 0 OR '
. $alias . '.dispatch_deadline_at IS NULL OR '
. $alias . '.dispatch_deadline_at < 1 OR '
. $alias . '.receipt_deadline_at IS NULL OR '
. $alias . '.receipt_deadline_at < 1 OR '
. $alias . '.dispatch_started_at IS NOT NULL OR '
. $alias . '.agent_accepted_at IS NOT NULL OR '
. $alias . '.dispatched_at IS NOT NULL OR '
. $alias . '.completion_event_id IS NOT NULL OR '
. $operation . ' IS NULL OR LENGTH(' . $operation . ') <> 32 OR '
. 'LOWER(' . $operation . ') <> ' . $operation . ' OR '
. self::invalidIdentifierCharactersSql($operation, '0-9a-f')
. ' OR ' . $fingerprint . ' IS NULL OR LENGTH('
. $fingerprint . ') <> 64 OR LOWER(' . $fingerprint . ') <> '
. $fingerprint . ' OR '
. self::invalidIdentifierCharactersSql($fingerprint, '0-9a-f')
. ' OR ' . $idempotency . ' IS NULL OR LENGTH('
. $idempotency . ') NOT BETWEEN 16 AND 128 OR '
. self::invalidIdentifierCharactersSql(
$idempotency,
'A-Za-z0-9_.-'
) . ' OR ' . $alias . '.terminal_at IS NULL OR '
. $alias . '.terminal_at < 1 OR '
. $alias . '.outcome IS NULL OR '
. $alias . ".outcome NOT IN ('succeeded', 'failed') OR "
. $alias . '.status IS NULL OR '
. $alias . '.status <> ' . $alias . '.outcome OR ('
. $alias . ".outcome = 'succeeded' AND ("
. $alias . ".effect_resolution IS NULL OR "
. $alias . ".effect_resolution <> 'applied' OR "
. $alias . '.effect_may_have_applied IS NULL OR '
. $alias . '.effect_may_have_applied <> 0 OR '
. $alias . '.safe_to_resubmit IS NULL OR '
. $alias . '.safe_to_resubmit <> 0 OR '
. $alias . '.error_code IS NOT NULL)) OR ('
. $alias . ".outcome = 'failed' AND ("
. $alias . '.effect_resolution IS NULL OR '
. $alias . ".effect_resolution NOT IN ('unknown', 'applied', 'not_applied') OR "
. $alias . '.effect_may_have_applied IS NULL OR '
. $alias . '.effect_may_have_applied NOT IN (0, 1) OR '
. $alias . '.safe_to_resubmit IS NULL OR '
. $alias . '.safe_to_resubmit NOT IN (0, 1) OR ('
. $alias . '.effect_may_have_applied = 1 AND '
. $alias . '.safe_to_resubmit <> 0) OR ('
. $alias . '.effect_may_have_applied = 0 AND '
. $alias . ".effect_resolution <> 'not_applied')))))";
}
/** @param list<string> $values */
private static function quotedSqlValues(array $values): string
{
return implode(', ', array_map(
static fn (string $value): string => "'"
. str_replace("'", "''", $value) . "'",
$values
));
}
/** Add the catalog-driven candidate predicate without assuming job shape. */
private static function whereCurrentAggregateCatalogBinding($bound): void
{
$bound->whereExists(static function ($templates): void {
$templates->selectRaw('1')
->from('cloud_templates as current_templates')
->whereColumn(
'current_templates.active_job_id',
'cloud_jobs.id'
);
})->orWhereExists(static function ($isos): void {
$isos->selectRaw('1')
->from('cloud_user_isos as current_isos')
->whereColumn(
'current_isos.active_job_id',
'cloud_jobs.id'
);
})->orWhere(static function ($repairable): void {
self::whereRepairableMissingAggregateCatalogPointer(
$repairable
);
});
}
/**
* Keep a terminal parent discoverable when only active_job_id was cleared.
* This predicate is intentionally broad enough to surface ambiguity; the
* service-level locked repair performs the complete one-to-one proof.
*/
private static function whereRepairableMissingAggregateCatalogPointer(
$candidate
): void {
$settledPartial = self::settledReleasedPartialAggregateSql(
'cloud_jobs'
);
$candidate->whereExists(static function ($templates) use (
$settledPartial
): void {
$templates->selectRaw('1')
->from('cloud_template_replicas as pointer_template_replicas')
->join(
'cloud_templates as pointer_templates',
'pointer_templates.id',
'=',
'pointer_template_replicas.template_id'
)->join(
'cloud_jobs as pointer_template_children',
'pointer_template_children.id',
'=',
'pointer_template_replicas.job_id'
)->whereColumn(
'pointer_template_children.parent_job_id',
'cloud_jobs.id'
)->where(static function ($binding) use (
$settledPartial
): void {
$binding->where(static function ($missing) use (
$settledPartial
): void {
$missing->where(static function ($released): void {
$released->whereNull(
'pointer_templates.active_job_id'
)->orWhere(
'pointer_templates.active_job_id',
'<=',
0
);
})->whereRaw('NOT ' . $settledPartial);
})->orWhere(static function ($conflict): void {
$conflict->where(
'pointer_templates.active_job_id',
'>',
0
)->whereColumn(
'pointer_templates.active_job_id',
'<>',
'cloud_jobs.id'
);
});
})->where(static function ($identity): void {
$identity->where(static function ($import): void {
$import->where('cloud_jobs.kind', 'template_import')
->where(
'pointer_template_children.kind',
'template_import_replica'
)->whereIn(
'pointer_templates.state',
['importing', 'error', 'active']
);
})->orWhere(static function ($metadata): void {
$metadata->where(
'cloud_jobs.kind',
'template_metadata_sync'
)->where(
'pointer_template_children.kind',
'template_metadata_replica'
)->where(
'pointer_templates.state',
'active'
)->where(static function ($evidence): void {
$evidence->whereIn(
'pointer_template_replicas.state',
['syncing', 'stale']
)->orWhere(static function ($unknown): void {
$unknown->where(
'cloud_jobs.outcome',
'failed'
)->where(
'cloud_jobs.status',
'failed'
)->where(
'cloud_jobs.effect_resolution',
'unknown'
);
})->orWhere(
'cloud_jobs.finalization_incident',
1
)->orWhere(static function ($damaged): void {
$damaged->where(
'pointer_templates.active_job_id',
'>',
0
)->whereColumn(
'pointer_templates.active_job_id',
'<>',
'cloud_jobs.id'
);
});
});
})->orWhere(static function ($delete): void {
$delete->where('cloud_jobs.kind', 'template_delete')
->where(
'pointer_template_children.kind',
'template_delete_replica'
)->whereIn(
'pointer_templates.state',
['deleting', 'delete_failed', 'active']
);
});
});
})->orWhereExists(static function ($isos) use (
$settledPartial
): void {
$isos->selectRaw('1')
->from('cloud_user_iso_replicas as pointer_iso_replicas')
->join(
'cloud_user_isos as pointer_isos',
'pointer_isos.id',
'=',
'pointer_iso_replicas.user_iso_id'
)->join(
'cloud_jobs as pointer_iso_children',
'pointer_iso_children.id',
'=',
'pointer_iso_replicas.job_id'
)->whereColumn(
'pointer_iso_children.parent_job_id',
'cloud_jobs.id'
)->where(static function ($binding) use (
$settledPartial
): void {
$binding->where(static function ($missing) use (
$settledPartial
): void {
$missing->where(static function ($released): void {
$released->whereNull(
'pointer_isos.active_job_id'
)->orWhere(
'pointer_isos.active_job_id',
'<=',
0
);
})->whereRaw('NOT ' . $settledPartial);
})->orWhere(static function ($conflict): void {
$conflict->where(
'pointer_isos.active_job_id',
'>',
0
)->whereColumn(
'pointer_isos.active_job_id',
'<>',
'cloud_jobs.id'
);
});
})->where(static function ($identity): void {
$identity->where(static function ($import): void {
$import->where('cloud_jobs.kind', 'user_iso_import')
->where(
'pointer_iso_children.kind',
'user_iso_import_replica'
)->whereIn(
'pointer_isos.state',
['importing', 'import_failed', 'active']
);
})->orWhere(static function ($delete): void {
$delete->where('cloud_jobs.kind', 'user_iso_delete')
->where(
'pointer_iso_children.kind',
'user_iso_delete_replica'
)->whereIn(
'pointer_isos.state',
['deleting', 'delete_failed', 'active']
);
});
});
});
}
/** Exact durable marker for a safely released non-metadata partial. */
private static function settledReleasedPartialAggregateSql(
string $alias
): string {
$kinds = self::quotedSqlValues([
'template_import',
'template_delete',
'user_iso_import',
'user_iso_delete',
]);
return '(' . $alias . ".execution_mode = 'aggregate' AND "
. $alias . '.parent_job_id IS NULL AND '
. $alias . '.kind IN (' . $kinds . ') AND '
. $alias . ".phase = 'terminal' AND "
. $alias . ".outcome = 'failed' AND "
. $alias . ".status = 'failed' AND "
. $alias . '.effect_may_have_applied = 1 AND '
. $alias . '.safe_to_resubmit = 0 AND '
. $alias . ".effect_resolution = 'applied' AND "
. $alias . '.effect_resolved_at > 0 AND '
. 'COALESCE(' . $alias . '.finalization_incident, 0) = 0 AND '
. 'NOT ' . self::aggregateLocalStateConflictSql($alias) . ' AND '
. self::settledPartialReasonSql(
$alias . '.effect_resolution_reason'
) . ')';
}
/** Match only our immutable numeric-child mixed publication reasons. */
private static function settledPartialReasonSql(string $column): string
{
$driver = (new CloudJob())->getConnection()->getDriverName();
if ($driver !== 'sqlite') {
return $column
. " REGEXP '^(legacy_)?child_effects_resolved:"
. "[1-9][0-9]*:mixed$'";
}
$suffix = ':mixed';
$parts = [];
foreach ([
'child_effects_resolved:',
'legacy_child_effects_resolved:',
] as $prefix) {
$offset = strlen($prefix) + 1;
$middleLength = 'LENGTH(' . $column . ') - '
. (strlen($prefix) + strlen($suffix));
$middle = 'SUBSTR(' . $column . ', ' . $offset . ', '
. $middleLength . ')';
$parts[] = '(SUBSTR(' . $column . ', 1, ' . strlen($prefix)
. ") = '" . $prefix . "' AND SUBSTR(" . $column
. ", -" . strlen($suffix) . ") = '" . $suffix . "' AND "
. $middleLength . ' > 0 AND SUBSTR(' . $middle
. ", 1, 1) BETWEEN '1' AND '9' AND NOT (" . $middle
. " GLOB '*[^0-9]*'))";
}
return '(' . implode(' OR ', $parts) . ')';
}
/** A final effect cannot be published while its leaf is still nonterminal. */
private static function aggregateChildHasPhaseResolutionConflict(
CloudJob $child
): bool {
return (string) $child->phase !== 'terminal'
&& in_array(
(string) $child->effect_resolution,
['applied', 'not_applied'],
true
);
}
/** Return the uniform effect, mixed, or null for an invalid child state. */
private static function aggregateEffectResolution($children): ?string
{
$resolutions = $children->map(
static fn (CloudJob $child): string =>
(string) $child->effect_resolution
)->unique()->values();
if ($resolutions->contains(
static fn ($resolution): bool => !in_array(
(string) $resolution,
['applied', 'not_applied'],
true
)
)) {
return null;
}
return $resolutions->count() === 1
? (string) $resolutions->first() : 'mixed';
}
private static function aggregateParentHasPublishedEffect(
CloudJob $parent
): bool {
return (string) $parent->phase === 'terminal'
&& in_array(
(string) $parent->effect_resolution,
['applied', 'not_applied'],
true
);
}
/** Revalidate a published aggregate without rewriting its terminal audit. */
private static function recheckPublishedAggregateProjectionLocked(
CloudJob $parent,
string $resolution,
bool $metadataMixed
): void {
try {
$bindingState = self::currentAggregateBindingStateLocked($parent);
} catch (\Throwable $error) {
self::deferAggregateRecoveryLocked(
$parent,
self::aggregateProjectionErrorCode($error),
time()
);
return;
}
if ($bindingState === 'conflict') {
self::deferAggregateRecoveryLocked(
$parent,
'recovery_aggregate_state_conflict',
time()
);
return;
}
if ($bindingState === 'absent') {
if ((bool) $parent->finalization_incident) {
$errorCode = strtolower(trim((string) (
$parent->finalization_error_code ?? ''
)));
self::deferAggregateRecoveryLocked(
$parent,
preg_match(
'/^recovery_[a-z0-9_]{1,55}$/D',
$errorCode
) === 1 ? $errorCode : 'recovery_ledger_conflict',
time()
);
return;
}
$parent->lease_token = null;
$parent->lease_until = 0;
$parent->available_at = time() + self::RECOVERY_RETRY_SECONDS;
$parent->updated_at = time();
$parent->save();
return;
}
$partialMixed = $resolution === 'mixed' && !$metadataMixed;
if ($partialMixed
&& !self::hasPublishedPartialAggregateMarker($parent)) {
self::deferAggregateRecoveryLocked(
$parent,
'recovery_aggregate_state_conflict',
time()
);
return;
}
$publishedResolution = $metadataMixed || $partialMixed
? 'applied' : $resolution;
if (!hash_equals(
(string) $parent->effect_resolution,
$publishedResolution
)) {
self::deferAggregateRecoveryLocked(
$parent,
'recovery_aggregate_state_conflict',
time()
);
return;
}
try {
if (str_starts_with((string) $parent->kind, 'template_')) {
CloudTemplateCatalogService::
assertSettledParentProjectionLocked(
$parent,
$resolution
);
} else {
CloudUserIsoService::assertSettledParentProjectionLocked(
$parent,
$resolution
);
}
} catch (\Throwable $error) {
self::deferAggregateRecoveryLocked(
$parent,
self::aggregateProjectionErrorCode($error),
time()
);
return;
}
$now = time();
$clearedIncident = (bool) $parent->finalization_incident;
if ($clearedIncident) {
$parent->finalization_incident = 0;
$parent->finalization_error_code = null;
$parent->read_at = null;
}
$parent->lease_token = null;
$parent->lease_until = 0;
$parent->available_at = $now + self::RECOVERY_RETRY_SECONDS;
$parent->updated_at = $now;
$parent->save();
}
/**
* Lock both catalog authorities in one order and classify their binding.
*
* @return 'valid'|'absent'|'conflict'
*/
private static function currentAggregateBindingStateLocked(
CloudJob $parent
): string {
$templates = CloudTemplate::query()
->where('active_job_id', (int) $parent->id)
->orderBy('id')->lockForUpdate()->get(['id']);
$isos = CloudUserIso::query()
->where('active_job_id', (int) $parent->id)
->orderBy('id')->lockForUpdate()->get(['id']);
if ($templates->isEmpty() && $isos->isEmpty()) {
$kind = (string) $parent->kind;
if (in_array($kind, [
'template_import', 'template_metadata_sync',
'template_delete',
], true)) {
CloudTemplateCatalogService::repairMissingParentBindingLocked(
$parent
);
} elseif (in_array($kind, [
'user_iso_import', 'user_iso_delete',
], true)) {
CloudUserIsoService::repairMissingParentBindingLocked($parent);
} else {
return 'absent';
}
$templates = CloudTemplate::query()
->where('active_job_id', (int) $parent->id)
->orderBy('id')->lockForUpdate()->get(['id']);
$isos = CloudUserIso::query()
->where('active_job_id', (int) $parent->id)
->orderBy('id')->lockForUpdate()->get(['id']);
}
if (str_starts_with((string) $parent->kind, 'template_')) {
return $templates->count() === 1 && $isos->isEmpty()
? 'valid' : 'conflict';
}
if (str_starts_with((string) $parent->kind, 'user_iso_')) {
return $isos->count() === 1 && $templates->isEmpty()
? 'valid' : 'conflict';
}
return 'conflict';
}
/** @return list<string> */
public static function aggregateProjectionIncidentCodes(): array
{
return [
CloudDispatchLedger::CONFLICT_CODE,
'recovery_child_convergence',
'recovery_aggregate_state_conflict',
'recovery_payload_unavailable',
'recovery_operation_identity_conflict',
'recovery_instance_state_conflict',
'recovery_ledger_conflict',
'recovery_internal_error',
];
}
private static function deferAggregateRecoveryLocked(
CloudJob $parent,
string $errorCode,
int $now
): void {
$parent->read_at = null;
$parent->finalization_incident = 1;
$parent->finalization_error_code = $errorCode;
$parent->lease_token = null;
$parent->lease_until = 0;
$parent->available_at = $now + self::RECOVERY_RETRY_SECONDS;
$parent->updated_at = $now;
$parent->save();
}
private static function aggregateProjectionErrorCode(
\Throwable $error
): string {
return match (true) {
$error instanceof CloudRecoveryDispatchLedgerConflict =>
CloudDispatchLedger::CONFLICT_CODE,
$error instanceof CloudRecoveryAggregateStateConflict =>
'recovery_aggregate_state_conflict',
$error instanceof CloudRecoveryPayloadUnavailable =>
'recovery_payload_unavailable',
$error instanceof CloudRecoveryOperationIdentityConflict =>
'recovery_operation_identity_conflict',
$error instanceof CloudRecoveryInstanceStateConflict =>
'recovery_instance_state_conflict',
$error instanceof CloudError => 'recovery_ledger_conflict',
default => 'recovery_internal_error',
};
}
public static function enqueue(CloudInstance $instance, array $actor,
string $kind, string $method, string $suffix, array $body = [],
?array $credential = null): CloudJob
{
CloudAccess::vmActor($instance, $actor);
if ($kind === 'network_change') {
throw new CloudError('旧公网网络写入口已停用', 409);
}
if (in_array($kind, ['delete', 'expiry_delete'], true)) {
CloudInstanceResourceRelease::assertCanDispatchDelete(
$instance
);
}
$requestKey = self::requestKey($actor);
return $instance->getConnection()->transaction(function () use (
$instance, $actor, $kind, $method, $suffix, $body, $credential,
$requestKey
) {
$templateMetadata = null;
$settingsFields = [];
if ($kind === 'reinstall') {
$body['template_id'] = CloudInput::template(
$body['template_id'] ?? ''
);
// Defense in depth for internal callers: this UI confirmation
// is not part of the strict Agent ReinstallRequest schema.
unset($body['confirmation_name']);
}
if ($kind === 'instance_settings') {
CloudInstanceSettingsGuard::assertSupported($instance);
CloudInput::only($body, ['hostname', 'root_password', 'ssh_public_key', 'confirm_restart']);
if (($body['confirm_restart'] ?? false) !== true) {
throw new CloudError('修改实例设置前必须确认虚拟机会短暂重启');
}
$changed = false;
if (array_key_exists('hostname', $body)) {
$body['hostname'] = CloudInput::hostname($body['hostname']);
$settingsFields[] = 'hostname';
$changed = true;
}
if (array_key_exists('root_password', $body)) {
if (!is_string($body['root_password']) || $body['root_password'] === '') {
throw new CloudError('实例密码不能为空');
}
$body['root_password'] = CloudInput::password($body['root_password']);
$settingsFields[] = 'password';
$changed = true;
}
if (array_key_exists('ssh_public_key', $body)) {
$parsed = CloudSSHKeys::parse((string) $body['ssh_public_key']);
$body['ssh_public_key'] = $parsed['public_key'];
$settingsFields[] = 'ssh_key';
$changed = true;
}
if (!$changed) {
throw new CloudError('请至少填写一项需要修改的实例设置');
}
}
$requestValue = [
'instance_id' => (int) $instance->id,
'kind' => $kind,
'method' => strtoupper($method),
'suffix' => $suffix,
'body' => $body,
];
$requestFingerprint = hash('sha256', json_encode(
self::canonicalRequestValue($requestValue),
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
));
$locked = CloudInstance::where('id', $instance->id)->lockForUpdate()->first();
if ($locked === null || (string) $locked->status === 'deleted') {
throw new CloudError('虚拟机不存在', 404);
}
if (in_array($kind, ['delete', 'expiry_delete'], true)) {
CloudInstanceResourceRelease::assertCanDispatchDelete(
$locked
);
}
if ($requestKey !== null) {
$prior = CloudJob::where('user_id', (int) $actor['user_id'])
->where('request_idempotency_key', $requestKey)
->lockForUpdate()->first();
if ($prior !== null) {
if (!hash_equals((string) $prior->request_fingerprint,
$requestFingerprint)) {
throw new CloudError(
'同一请求标识不能用于不同的虚拟机操作', 409
);
}
return $prior;
}
}
if ((string) $locked->status !== 'active') {
throw new CloudError('实例正在执行到期销毁流程,不能提交其他操作', 409);
}
CloudInstanceMigrationService::assertInstanceMutationAvailable(
$locked,
$kind
);
if ($kind === 'reinstall') {
CloudSnapshotService::assertReinstallAllowedLocked($locked);
$body = CloudTemplateCatalogService::
applyProvisioningSnapshotForHost(
$body,
(int) $locked->host_id,
false
);
$templateMetadata =
CloudTemplateCatalogService::templateMetadataSnapshot(
$body
);
}
$policyService = ['suspending_expired', 'resume_pending', 'resuming',
'suspend_failed', 'resume_failed', 'expired_grace',
'purge_stopping', 'purge_stop_failed', 'purge_force_required',
'purge_deleting',
'purge_failed'];
$policyNetwork = ['blocking', 'blocking_expired', 'blocked_expired',
'restore_pending', 'restoring', 'block_failed', 'restore_failed'];
$superAdmin = ($actor['is_super_admin'] ?? false)
&& ($actor['can_manage'] ?? false);
if ($kind !== 'delete' && !$superAdmin
&& (in_array((string) $locked->service_status, $policyService, true)
|| in_array((string) $locked->network_status, $policyNetwork, true)
|| ((string) $locked->service_status !== 'exempt'
&& ($locked->expires_at === null || (int) $locked->expires_at <= time())))) {
throw new CloudError('实例正在执行到期或流量策略,请稍后重试', 409);
}
$activeJobs = self::blockingQueryForInstance((int) $locked->id);
if ($activeJobs->exists()) {
throw new CloudError('虚拟机已有任务,请等待完成', 409);
}
$job = new CloudJob();
$job->operation_id = bin2hex(random_bytes(16));
$job->instance_id = $locked->id;
$job->user_id = $actor['user_id'];
$job->host_id = $locked->host_id;
$job->kind = $kind;
$job->execution_mode = 'agent';
$job->status = 'queued';
if ($requestKey !== null) {
$job->request_idempotency_key = $requestKey;
}
$job->request_fingerprint = $requestFingerprint;
$jobPayload = ['method' => $method,
'request_target' => '/v1/vms/' . rawurlencode($locked->uuid) . $suffix,
'body' => $body,
'credential' => $credential, 'owner_id' => (int) $locked->user_id,
'can_manage' => (bool) ($actor['can_manage'] ?? false)];
if ($settingsFields !== []) {
// This allow-listed metadata remains outside the strict Agent
// body and survives terminal secret redaction. Never persist a
// hostname, password or public-key value as presentation data.
$jobPayload['settings_fields'] = $settingsFields;
}
if ($templateMetadata !== null) {
// Encrypted panel metadata is deliberately outside body:
// body is the exact strict Go ReinstallRequest DTO.
$jobPayload['template_metadata'] = $templateMetadata;
}
$job->sealed_payload = CloudSecrets::encrypt(
$jobPayload, 'job-payload'
);
$job->available_at = $job->created_at = $job->updated_at = time();
$job->save();
return $job;
}, 3);
}
/**
* Queue the irreversible second phase of expiry cleanup.
*
* Only the lifecycle state machine may reach this method. Unlike the
* four target-state policy jobs, expiry deletion is intentionally never
* replayed after dispatch because an interrupted volume deletion may be
* only partly complete.
*/
public static function enqueueExpiryDelete(CloudInstance $instance): CloudJob
{
return $instance->getConnection()->transaction(function () use ($instance) {
$locked = CloudInstance::where('id', $instance->id)->lockForUpdate()->first();
if ($locked === null || (string) $locked->status === 'deleted') {
throw new CloudError('虚拟机不存在', 404);
}
if ((string) $locked->status !== 'deleting'
|| (string) $locked->service_status !== 'purge_deleting') {
throw new CloudError('实例尚未完成销毁前停机,不能删除磁盘', 409);
}
CloudInstanceResourceRelease::assertCanDispatchDelete(
$locked
);
CloudInstanceMigrationService::assertInstanceMutationAvailable(
$locked,
'expiry_delete'
);
$existing = self::blockingQueryForInstance((int) $locked->id)
->orderByDesc('id')->first();
if ($existing !== null) {
if ((string) $existing->kind === 'expiry_delete') {
return $existing;
}
throw new CloudError('虚拟机已有任务,请等待完成', 409);
}
$job = new CloudJob();
$job->operation_id = bin2hex(random_bytes(16));
$job->instance_id = $locked->id;
$job->user_id = $locked->user_id;
$job->host_id = $locked->host_id;
$job->kind = 'expiry_delete';
$job->execution_mode = 'agent';
$job->status = 'queued';
$job->sealed_payload = CloudSecrets::encrypt([
'method' => 'DELETE',
'request_target' => '/v1/vms/' . rawurlencode($locked->uuid),
'body' => ['remove_managed_disks' => true],
'owner_id' => (int) $locked->user_id,
'system_cleanup' => true,
], 'job-payload');
$job->available_at = $job->created_at = $job->updated_at = time();
$job->save();
return $job;
}, 3);
}
/**
* Queue the separately audited force phase of a delete operation.
*
* The original graceful operation remains the authorization anchor. It is
* never replayed or changed into a force request; the Agent validates that
* exact prior operation before accepting this new operation ID.
*/
public static function enqueueForceDelete(
CloudJob $prior,
array $actor
): CloudJob {
CloudAccess::manage($actor);
$priorId = filter_var($prior->id, FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]);
$instanceId = filter_var($prior->instance_id, FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]);
if ($priorId === false || $instanceId === false) {
throw new CloudError('当前任务没有关联实例', 409);
}
return $prior->getConnection()->transaction(function () use (
$priorId, $instanceId, $actor
) {
// Keep the global instance-mutation lock order: instance first,
// then jobs. The prior row is fully revalidated after both locks.
$instance = CloudInstance::where('id', (int) $instanceId)
->lockForUpdate()->first();
$lockedPrior = CloudJob::where('id', (int) $priorId)
->lockForUpdate()->first();
if ($lockedPrior === null) {
throw new CloudError('任务不存在', 404);
}
if ((bool) ($lockedPrior->finalization_incident ?? false)
|| self::hasTerminalRecoveryStateConflict($lockedPrior)) {
throw new CloudError(
'原删除任务的恢复账本异常尚未收敛,不能创建新的强制删除任务',
409
);
}
if (!in_array((string) $lockedPrior->kind,
['delete', 'expiry_delete', 'service_suspend'], true)
|| (string) $lockedPrior->phase !== 'terminal'
|| (string) $lockedPrior->outcome !== 'failed'
|| (string) $lockedPrior->error_code !== 'force_required'
|| !CloudDispatchLedger::isDispatched($lockedPrior)
|| $lockedPrior->dispatched_at === null
|| preg_match('/^[a-f0-9]{32}$/D',
(string) $lockedPrior->operation_id) !== 1
|| (int) $lockedPrior->instance_id !== (int) $instanceId) {
throw new CloudError('当前任务没有经过确认的强制删除前置状态', 409);
}
if ($instance === null || (string) $instance->status === 'deleted'
|| (int) $instance->host_id !== (int) $lockedPrior->host_id
|| preg_match('/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD',
(string) $instance->uuid) !== 1) {
throw new CloudError('关联实例已不存在或与原删除任务不一致', 409);
}
CloudInstanceResourceRelease::assertCanDispatchDelete(
$instance
);
$priorPayload = CloudSecrets::decrypt(
(string) $lockedPrior->sealed_payload,
'job-payload'
);
$target = '/v1/vms/' . rawurlencode((string) $instance->uuid);
$gracefulDelete = ($priorPayload['method'] ?? null) === 'DELETE'
&& ($priorPayload['request_target'] ?? null) === $target
&& ($priorPayload['body'] ?? null)
=== ['remove_managed_disks' => true];
$expiryGracefulStop = (string) $lockedPrior->kind
=== 'service_suspend'
&& ($priorPayload['method'] ?? null) === 'POST'
&& ($priorPayload['request_target'] ?? null)
=== $target . '/service/suspend'
&& ($priorPayload['body'] ?? null) === ['reason' => 'purge']
&& ($priorPayload['system_policy'] ?? false) === true
&& (string) $instance->status === 'deleting'
&& (string) $instance->service_status
=== 'purge_force_required';
if (!$gracefulDelete && !$expiryGracefulStop) {
throw new CloudError('原删除任务不允许强制删除面板管理磁盘', 409);
}
$otherBlocker = self::blockingQueryForInstance((int) $instance->id)
->where('id', '<>', (int) $lockedPrior->id)
->lockForUpdate()->first();
if ($otherBlocker !== null) {
throw new CloudError('虚拟机还有其他未完成任务,不能发起强制删除', 409);
}
do {
$operationId = bin2hex(random_bytes(16));
} while (hash_equals((string) $lockedPrior->operation_id, $operationId));
$job = new CloudJob();
$job->operation_id = $operationId;
$job->instance_id = $instance->id;
$job->user_id = (int) ($actor['user_id'] ?? 0);
$job->host_id = $instance->host_id;
$job->kind = 'delete_force';
$job->execution_mode = 'agent';
$job->status = 'queued';
$job->sealed_payload = CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/'
. rawurlencode((string) $instance->uuid) . '/force-delete',
'body' => [
'operation_id' => $operationId,
'prior_operation_id' => (string) $lockedPrior->operation_id,
'ack_data_loss' => true,
'remove_managed_disks' => true,
],
'owner_id' => (int) $instance->user_id,
'can_manage' => true,
'prior_job_id' => (int) $lockedPrior->id,
'prior_operation_id' => (string) $lockedPrior->operation_id,
], 'job-payload');
$now = time();
$job->available_at = $job->created_at = $job->updated_at = $now;
$job->save();
error_log(sprintf(
'cloud admin queued force delete prior_job_id=%d force_job_id=%d prior_operation_id=%s force_operation_id=%s actor_id=%d',
(int) $lockedPrior->id, (int) $job->id,
(string) $lockedPrior->operation_id, $operationId,
(int) ($actor['user_id'] ?? 0)
));
return $job;
}, 3);
}
public static function display(CloudJob $job, bool $administrator = false): array
{
$completed = (string) $job->phase === 'terminal'
&& in_array((string) $job->outcome,
['succeeded', 'failed'], true);
$action = self::action($job);
$settingsFields = self::settingsFields($job);
$labels = CloudOperationCatalog::labels((string) $job->kind, $action,
$settingsFields);
$base = [
'id' => (string) $job->id,
'operation_id' => (string) $job->operation_id,
'kind' => (string) $job->kind,
'completed' => $completed,
'created_at' => max(0, (int) $job->created_at),
'updated_at' => max(0, (int) $job->updated_at),
'status_url' => '/api/v1/tasks/' . (int) $job->id,
'action' => $action,
'kind_label' => $labels['kind_label'],
'action_label' => $labels['action_label'],
'operation_copy_keys' => $labels['copy_keys'],
];
if (!$completed) {
$recovery = self::recoveryPresentation($job, $administrator);
if ($recovery !== null) {
$base['recovery'] = $recovery;
}
return $base;
}
$result = $job->result_json === null ? null : json_decode($job->result_json, true);
$status = (string) $job->outcome;
$safeResult = (string) $job->kind === 'payment_create'
&& $status === 'succeeded' && is_array($result)
? self::displayPaymentResult($job, $result)
: self::displayResult(is_array($result) ? $result : null,
$administrator, (string) $job->kind, $status,
$job->order_id !== null, $action);
$error = $administrator
? CloudOperationErrorCatalog::adminSummary((string) $job->kind,
$status, $job->error_code, $job->order_id !== null, $action)
: CloudOperationErrorCatalog::userMessage((string) $job->kind,
$status, $job->error_code, $job->order_id !== null, $action);
$phase = $safeResult['phase'] ?? null;
$phase = $administrator
? CloudOperationErrorCatalog::adminSummary((string) $job->kind,
$status, $phase, $job->order_id !== null, $action)
: CloudOperationErrorCatalog::userPhase($phase, $status, (string) $job->kind);
$terminal = [
'status' => $status, 'error' => $error,
'error_code' => CloudOperationErrorCatalog::publicCode($job->error_code),
'phase' => is_string($phase) ? mb_substr($phase, 0, 120) : null,
'result' => $safeResult,
'effect_may_have_applied' => $job->effect_may_have_applied === null
? null : (bool) $job->effect_may_have_applied,
'safe_to_resubmit' => $job->safe_to_resubmit === null
? null : (bool) $job->safe_to_resubmit,
'unread' => $job->read_at === null,
'terminal_at' => max(0, (int) $job->terminal_at),
];
$effectResolution = (string) $job->effect_resolution;
if ($status === 'failed' && (bool) $job->effect_may_have_applied
&& in_array($effectResolution, ['applied', 'not_applied'], true)) {
// This is deliberately not a generic recovery/audit field. The
// public contract exposes only the two conclusions needed to stop
// presenting an already-released resource fence as active.
$terminal['effect_resolution'] = $effectResolution;
}
$recovery = self::recoveryPresentation($job, $administrator);
if ($recovery !== null) {
$terminal['recovery'] = $recovery;
} else {
$effectScope = self::effectScope($job);
if ($effectScope !== null) {
$terminal['effect_scope'] = $effectScope;
}
}
return array_merge($base, $terminal);
}
/**
* Distinguish a fully proven mixed aggregate from an all-host applied
* result without exposing the internal convergence reason to clients.
*/
public static function effectScope(object $job): ?string
{
return self::hasPublishedPartialAggregateMarker($job)
&& !(bool) ($job->finalization_incident ?? false)
? 'partial' : null;
}
/** Canonical mixed publication tuple, including rows under recheck. */
public static function hasPublishedPartialAggregateMarker(
object $job
): bool {
$reason = (string) ($job->effect_resolution_reason ?? '');
return (string) ($job->execution_mode ?? '') === 'aggregate'
&& ($job->parent_job_id ?? null) === null
&& in_array((string) ($job->kind ?? ''), [
'template_import',
'template_delete',
'user_iso_import',
'user_iso_delete',
], true)
&& !self::hasAggregateLocalStateConflict($job)
&& (string) ($job->phase ?? '') === 'terminal'
&& (string) ($job->outcome ?? '') === 'failed'
&& (string) ($job->status ?? '') === 'failed'
&& (bool) ($job->effect_may_have_applied ?? false)
&& !(bool) ($job->safe_to_resubmit ?? true)
&& (string) ($job->effect_resolution ?? '') === 'applied'
&& (int) ($job->effect_resolved_at ?? 0) > 0
&& preg_match(
'/^(?:legacy_)?child_effects_resolved:[1-9][0-9]*:mixed$/D',
$reason
) === 1;
}
/**
* Return a bounded recovery status for panels. Arbitrary server/Agent
* exception text never crosses this DTO; unknown recovery codes collapse
* to one reviewed message.
*/
public static function recoveryPresentation(
CloudJob $job,
bool $administrator = false
): ?array {
$mode = self::recoveryMode($job);
if ($mode === null) {
return null;
}
if ($mode === 'aggregate') {
$details = self::aggregateRecoveryDetails($job);
$code = $details['code'];
$message = $details['message'];
} else {
$candidate = self::agentRecoveryPresentationCandidate($job);
if ($mode === 'local_finalizer'
&& in_array(
$candidate,
CloudDispatchLedger::repairIncidentCodes(),
true
)) {
$candidate = 'recovery_ledger_conflict';
}
$code = array_key_exists($candidate, self::RECOVERY_MESSAGES)
? $candidate : ($mode === 'local_finalizer'
? 'recovery_ledger_conflict' : 'recovery_pending');
$message = self::RECOVERY_MESSAGES[$code];
}
$nextCheckAt = max(0, (int) ($job->available_at ?? 0));
$lastCheckedAt = max(0, (int) ($job->updated_at ?? 0));
if ($mode === 'aggregate') {
$nextCheckAt = max($nextCheckAt, $details['next_check_at']);
$lastCheckedAt = max(
$lastCheckedAt,
$details['last_checked_at']
);
}
$result = [
'pending' => true,
'message' => $message,
'last_checked_at' => $lastCheckedAt > 0
? $lastCheckedAt : null,
'next_check_at' => $nextCheckAt > time()
? $nextCheckAt : null,
];
if ($administrator) {
$result['code'] = $code;
if ($mode === 'aggregate') {
$result['affected_child_count'] =
$details['affected_child_count'];
$result['additional_error_count'] =
$details['additional_error_count'];
}
}
return $result;
}
/** Select the most actionable reviewed cause without exposing raw text. */
private static function agentRecoveryPresentationCandidate(
CloudJob $job
): string {
if (CloudDispatchLedger::isInvalid($job)) {
return CloudDispatchLedger::CONFLICT_CODE;
}
$stored = strtolower(trim(
(string) ($job->finalization_error_code ?? '')
));
if (CloudDispatchLedger::hasRepairIncident($job)
&& array_key_exists($stored, self::RECOVERY_MESSAGES)) {
return $stored;
}
if (self::hasAgentRecoveryIdentityConflict($job)) {
return 'recovery_operation_identity_conflict';
}
if (in_array($stored, [
'recovery_operation_identity_conflict',
'recovery_receipt_invalid',
'recovery_completion_outbox_invalid',
'recovery_completion_receipt_missing',
'recovery_completion_receipt_invalid',
'recovery_completion_operation_missing',
'recovery_completion_operation_invalid',
'recovery_completion_identity_mismatch',
'recovery_payload_invalid',
'recovery_payload_unavailable',
], true)) {
return $stored;
}
if (self::hasTerminalRecoveryStateConflict($job)) {
return 'recovery_instance_state_conflict';
}
return $stored;
}
private static function recoveryMode(CloudJob $job): ?string
{
$kind = (string) $job->kind;
$aggregateIdentity = (string) $job->execution_mode === 'aggregate'
|| in_array($kind, self::AGGREGATE_KINDS, true);
if ($aggregateIdentity) {
$ownFence = (string) $job->phase === 'terminal'
&& (string) $job->outcome === 'failed'
&& (string) $job->status === 'failed'
&& (bool) $job->effect_may_have_applied
&& !(bool) $job->safe_to_resubmit
&& (string) $job->effect_resolution === 'unknown';
$stickyIncident = (string) $job->phase === 'terminal'
&& (bool) $job->finalization_incident;
if ($ownFence || $stickyIncident
|| self::hasTerminalRecoveryStateConflict($job)
|| self::hasUnresolvedRecoverableChildren($job)) {
return 'aggregate';
}
return null;
}
if ((string) $job->execution_mode === 'agent'
&& CloudDispatchLedger::isInvalid($job)) {
return 'agent';
}
if (CloudDispatchLedger::hasRepairIncident($job)) {
return 'agent';
}
if (self::isLocalFinalizerRecovery($job)) {
return 'local_finalizer';
}
if (self::isResolvedStickyAgentIncident($job)) {
return 'agent';
}
if (self::hasTerminalRecoveryStateConflict($job)) {
return 'agent';
}
if ((string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| !(bool) $job->effect_may_have_applied
|| (bool) $job->safe_to_resubmit
|| (string) $job->effect_resolution !== 'unknown') {
return null;
}
if ((string) $job->execution_mode === 'agent'
&& (int) $job->dispatch_attempts === 1
&& CloudOperationRecoveryPolicy::supportsLedgerFinalizer($kind)) {
return 'agent';
}
return null;
}
private static function hasUnresolvedRecoverableChildren(
CloudJob $parent
): bool {
if ((int) $parent->id < 1) {
return false;
}
return CloudJob::where('parent_job_id', (int) $parent->id)
->whereIn('kind', CloudOperationRecoveryPolicy::kinds())
->where(static function ($pending): void {
$pending->where(static function ($mode): void {
$mode->whereNull('execution_mode')
->orWhere('execution_mode', '<>', 'agent');
})->orWhere(static function ($invalid): void {
$invalid->whereNull('dispatch_attempts')
->orWhereNotIn('dispatch_attempts', [0, 1]);
})->orWhere(static function ($repair): void {
$repair->where('phase', 'terminal')
->where('finalization_incident', 1)
->whereIn(
'finalization_error_code',
CloudDispatchLedger::repairIncidentCodes()
);
})->orWhere(static function ($local): void {
$local->where('phase', 'terminal')
->where('outcome', 'failed')
->where('status', 'failed')
->where('dispatch_attempts', 0)
->where('finalization_incident', 1);
})->orWhere(static function ($sticky): void {
$sticky->where('phase', 'terminal')
->where('finalization_incident', 1);
})
->orWhere(static function ($normal): void {
$normal->where('phase', 'terminal')
->where('outcome', 'failed')
->where('status', 'failed')
->where('dispatch_attempts', 1)
->where('effect_may_have_applied', 1)
->where('safe_to_resubmit', 0)
->where('effect_resolution', 'unknown');
})->orWhere(static function ($state): void {
$state->where('phase', 'terminal')
->whereIn('dispatch_attempts', [0, 1])
->where('effect_resolution', 'unknown')
->where(static function ($invalid): void {
self::applyInvalidTerminalTuple($invalid);
});
})->orWhere(static function ($phase): void {
$phase->where('phase', '<>', 'terminal')
->whereIn(
'effect_resolution',
['applied', 'not_applied']
);
})->orWhere(static function ($resolved): void {
$resolved->where('phase', 'terminal')
->whereIn('dispatch_attempts', [0, 1])
->where(static function ($invalid): void {
self::applyInvalidResolvedTerminalTuple(
$invalid
);
});
});
})
->exists();
}
private static function unresolvedRecoverableChildExists($children): void
{
$children->selectRaw('1')
->from('cloud_jobs as recovery_children')
->whereColumn(
'recovery_children.parent_job_id',
'cloud_jobs.id'
)
->whereIn(
'recovery_children.kind',
CloudOperationRecoveryPolicy::kinds()
)->where(static function ($pending): void {
$pending->where(static function ($mode): void {
$mode->whereNull('recovery_children.execution_mode')
->orWhere(
'recovery_children.execution_mode',
'<>',
'agent'
);
})->orWhere(static function ($invalid): void {
$invalid->whereNull(
'recovery_children.dispatch_attempts'
)->orWhereNotIn(
'recovery_children.dispatch_attempts',
[0, 1]
);
})->orWhere(static function ($repair): void {
$repair->where(
'recovery_children.phase',
'terminal'
)->where(
'recovery_children.finalization_incident',
1
)->whereIn(
'recovery_children.finalization_error_code',
CloudDispatchLedger::repairIncidentCodes()
);
})->orWhere(static function ($local): void {
$local->where(
'recovery_children.phase',
'terminal'
)->where(
'recovery_children.outcome',
'failed'
)->where(
'recovery_children.status',
'failed'
)->where(
'recovery_children.dispatch_attempts',
0
)->where(
'recovery_children.finalization_incident',
1
);
})->orWhere(static function ($sticky): void {
$sticky->where(
'recovery_children.phase',
'terminal'
)->where(
'recovery_children.finalization_incident',
1
);
})->orWhere(static function ($normal): void {
$normal->where(
'recovery_children.phase',
'terminal'
)->where(
'recovery_children.outcome',
'failed'
)->where(
'recovery_children.status',
'failed'
)->where(
'recovery_children.dispatch_attempts',
1
)->where(
'recovery_children.effect_may_have_applied',
1
)->where(
'recovery_children.safe_to_resubmit',
0
)->where(
'recovery_children.effect_resolution',
'unknown'
);
})->orWhere(static function ($state): void {
$state->where(
'recovery_children.phase',
'terminal'
)->whereIn(
'recovery_children.dispatch_attempts',
[0, 1]
)->where(
'recovery_children.effect_resolution',
'unknown'
)->where(static function ($invalid): void {
self::applyInvalidTerminalTuple(
$invalid,
'recovery_children.'
);
});
})->orWhere(static function ($phase): void {
$phase->where(
'recovery_children.phase',
'<>',
'terminal'
)->whereIn(
'recovery_children.effect_resolution',
['applied', 'not_applied']
);
})->orWhere(static function ($resolved): void {
$resolved->where(
'recovery_children.phase',
'terminal'
)->whereIn(
'recovery_children.dispatch_attempts',
[0, 1]
)->where(static function ($invalid): void {
self::applyInvalidResolvedTerminalTuple(
$invalid,
'recovery_children.'
);
});
});
});
}
/**
* Select one deterministic, reviewed child incident for the public batch.
* Raw Agent/exception text and child result payloads never enter this DTO.
*
* @return array{code:string,message:string,affected_child_count:int,additional_error_count:int,last_checked_at:int,next_check_at:int}
*/
private static function aggregateRecoveryDetails(CloudJob $parent): array
{
if (!$parent->exists) {
// display() is also used with detached, read-only task snapshots
// (for example notification/rendering contracts). Only persisted
// aggregate jobs have child rows that can be safely summarized.
return [
'code' => 'recovery_child_convergence',
'message' => self::RECOVERY_MESSAGES[
'recovery_child_convergence'
],
'affected_child_count' => 0,
'additional_error_count' => 0,
'last_checked_at' => max(0, (int) $parent->updated_at),
'next_check_at' => max(0, (int) $parent->available_at),
];
}
$children = CloudJob::where('parent_job_id', (int) $parent->id)
->orderBy('id')
->get([
'id', 'operation_id', 'request_idempotency_key',
'request_fingerprint', 'parent_job_id', 'instance_id',
'user_id', 'host_id', 'kind', 'execution_mode',
'phase', 'outcome', 'status', 'attempts',
'dispatch_attempts', 'dispatch_deadline_at',
'dispatch_started_at', 'agent_accepted_at', 'dispatched_at',
'receipt_deadline_at', 'terminal_at', 'completion_event_id',
'error_code', 'result_json',
'effect_may_have_applied', 'safe_to_resubmit',
'effect_resolution', 'finalization_incident',
'finalization_error_code',
'available_at', 'updated_at',
]);
$expectedKind = self::AGGREGATE_CHILD_KIND_MAP[
(string) $parent->kind
] ?? null;
$parentStateConflict = self::hasAggregateLocalStateConflict($parent)
|| (string) ($parent->finalization_error_code ?? '')
=== 'recovery_aggregate_state_conflict';
$parentCandidate = $parentStateConflict
? 'recovery_aggregate_state_conflict'
: strtolower(trim(
(string) ($parent->finalization_error_code ?? '')
));
$parentCode = array_key_exists(
$parentCandidate,
self::RECOVERY_MESSAGES
) ? $parentCandidate : null;
if ($parentCode === null && (bool) $parent->finalization_incident) {
$parentCode = 'recovery_ledger_conflict';
}
$relationshipConflict = $parentCandidate
=== 'recovery_aggregate_state_conflict'
|| $expectedKind === null
|| $children->isEmpty()
|| $children->contains(
static fn (CloudJob $child): bool =>
(int) $child->user_id !== (int) $parent->user_id
|| (string) $child->execution_mode !== 'agent'
|| (string) $child->kind !== $expectedKind
|| self::aggregateChildHasPhaseResolutionConflict($child)
);
$unresolved = $children->filter(
static fn (CloudJob $child): bool =>
(string) $child->execution_mode === 'agent'
&& CloudOperationRecoveryPolicy::supports(
(string) $child->kind
)
&& (CloudDispatchLedger::isInvalid($child)
|| (bool) $child->finalization_incident
|| CloudDispatchLedger::hasRepairIncident($child)
|| self::isLocalFinalizerRecovery($child)
|| self::hasTerminalRecoveryStateConflict($child)
|| self::aggregateChildHasPhaseResolutionConflict($child)
|| self::hasResolvedTerminalStateConflict($child)
|| ((string) $child->phase === 'terminal'
&& (string) $child->outcome === 'failed'
&& (string) $child->status === 'failed'
&& CloudDispatchLedger::isDispatched($child)
&& (bool) $child->effect_may_have_applied
&& !(bool) $child->safe_to_resubmit
&& (string) $child->effect_resolution === 'unknown'))
);
$lastCheckedAt = (int) ($unresolved->max('updated_at') ?? 0);
$nextCheckAt = (int) ($unresolved->max('available_at') ?? 0);
$priority = [
'recovery_dispatch_ledger_conflict',
'recovery_dispatch_repair_completion_receipt_missing',
'recovery_dispatch_repair_receipt_invalid',
'recovery_dispatch_repair_aggregate_state_conflict',
'recovery_dispatch_repair_payload_unavailable',
'recovery_dispatch_repair_operation_identity_conflict',
'recovery_dispatch_repair_evidence_conflict',
'recovery_dispatch_repair_instance_state_conflict',
'recovery_dispatch_repair_ledger_conflict',
'recovery_dispatch_repair_internal_error',
'recovery_operation_identity_conflict',
'recovery_receipt_invalid',
'recovery_completion_identity_mismatch',
'recovery_completion_receipt_invalid',
'recovery_completion_receipt_missing',
'recovery_completion_operation_invalid',
'recovery_completion_operation_missing',
'recovery_completion_outbox_invalid',
'recovery_payload_unavailable',
'recovery_payload_invalid',
'recovery_instance_state_conflict',
'recovery_evidence_conflict',
'recovery_evidence_invalid',
'recovery_state_invalid',
'recovery_authentication_failed',
'recovery_agent_unavailable',
'recovery_waiting_connection',
'recovery_operation_conflict',
'recovery_operation_missing',
'recovery_evidence_partial',
'recovery_evidence_inconclusive',
'recovery_ledger_conflict',
'recovery_internal_error',
'recovery_pending',
];
$codes = $parentCode === null
|| $parentCode === 'recovery_child_convergence'
? [] : [$parentCode];
if ($relationshipConflict) {
$codes[] = 'recovery_aggregate_state_conflict';
}
foreach ($unresolved as $child) {
$candidate = self::aggregateChildHasPhaseResolutionConflict($child)
? 'recovery_aggregate_state_conflict'
: self::agentRecoveryPresentationCandidate($child);
if (self::isLocalFinalizerRecovery($child)
&& in_array(
$candidate,
CloudDispatchLedger::repairIncidentCodes(),
true
)) {
$candidate = 'recovery_ledger_conflict';
}
$codes[] = array_key_exists($candidate, self::RECOVERY_MESSAGES)
? $candidate : 'recovery_pending';
}
$codes = array_values(array_unique($codes));
usort($codes, static function (string $left, string $right) use (
$priority
): int {
$leftPriority = array_search($left, $priority, true);
$rightPriority = array_search($right, $priority, true);
$leftPriority = $leftPriority === false
? PHP_INT_MAX : $leftPriority;
$rightPriority = $rightPriority === false
? PHP_INT_MAX : $rightPriority;
return $leftPriority <=> $rightPriority
?: strcmp($left, $right);
});
$concreteParentCode = $parentCode !== null
&& $parentCode !== 'recovery_child_convergence'
? $parentCode : null;
$code = $concreteParentCode
?? ($codes[0] ?? ($parentCode ?? 'recovery_child_convergence'));
$parentLevelCode = $concreteParentCode !== null
|| ($relationshipConflict
&& $code === 'recovery_aggregate_state_conflict');
$message = $parentLevelCode || $code === 'recovery_child_convergence'
? self::RECOVERY_MESSAGES[$code]
: '某个宿主机子任务:' . self::RECOVERY_MESSAGES[$code];
$additional = max(0, count($codes) - 1);
if ($additional > 0) {
$message .= ';另有 ' . $additional
. ' 类子任务异常仍在持续自动核对。';
}
return [
'code' => $code,
'message' => $message,
'affected_child_count' => $relationshipConflict
? max(1, $unresolved->count()) : $unresolved->count(),
'additional_error_count' => $additional,
'last_checked_at' => $lastCheckedAt,
'next_check_at' => $nextCheckAt,
];
}
private static function applyRecoveringFilter($query): void
{
$query->where(static function ($modes): void {
$modes->where(static function ($leaf): void {
$leaf->whereIn('kind',
CloudOperationRecoveryPolicy::ledgerFinalizerKinds())
->where(static function ($pending): void {
$pending->where(static function ($mode): void {
$mode->whereNull('execution_mode')
->orWhere('execution_mode', '<>', 'agent');
})->orWhere(static function ($invalid): void {
$invalid->whereNull('dispatch_attempts')
->orWhereNotIn(
'dispatch_attempts',
[0, 1]
);
})->orWhere(static function ($repair): void {
$repair->where('phase', 'terminal')
->where('finalization_incident', 1)
->whereIn(
'finalization_error_code',
CloudDispatchLedger::repairIncidentCodes()
);
})->orWhere(static function ($local): void {
$local->where('phase', 'terminal')
->where('outcome', 'failed')
->where('status', 'failed')
->where('dispatch_attempts', 0)
->where('finalization_incident', 1);
})->orWhere(static function ($resolved): void {
$resolved->where('phase', 'terminal')
->whereIn('dispatch_attempts', [0, 1])
->whereIn(
'effect_resolution',
['applied', 'not_applied']
)->where('finalization_incident', 1);
})->orWhere(static function ($normal): void {
$normal->where('phase', 'terminal')
->where('outcome', 'failed')
->where('status', 'failed')
->where('effect_may_have_applied', 1)
->where('safe_to_resubmit', 0)
->where('effect_resolution', 'unknown')
->where('dispatch_attempts', 1);
})->orWhere(static function ($state): void {
$state->where('phase', 'terminal')
->whereIn('dispatch_attempts', [0, 1])
->where('effect_resolution', 'unknown')
->where(static function ($invalid): void {
self::applyInvalidTerminalTuple($invalid);
});
})->orWhere(static function ($resolvedState): void {
$resolvedState->where('phase', 'terminal')
->whereIn('dispatch_attempts', [0, 1])
->where(static function ($invalid): void {
self::applyInvalidResolvedTerminalTuple(
$invalid
);
});
});
});
})->orWhere(static function ($aggregate): void {
$aggregate->where('execution_mode', 'aggregate')
->whereIn('kind', self::AGGREGATE_KINDS)
->where(static function ($pending): void {
$pending->where(static function ($ledger): void {
$ledger->whereNull('dispatch_attempts')
->orWhere('dispatch_attempts', '<>', 0);
})->orWhere(static function ($own): void {
$own->where('phase', 'terminal')
->where('outcome', 'failed')
->where('status', 'failed')
->where('effect_may_have_applied', 1)
->where('safe_to_resubmit', 0)
->where('effect_resolution', 'unknown');
})->orWhere(static function ($state): void {
$state->where('phase', 'terminal')
->where('effect_resolution', 'unknown')
->where(static function ($invalid): void {
self::applyInvalidTerminalTuple($invalid);
});
})->orWhere(static function ($published): void {
$published->where('phase', 'terminal')
->where('finalization_incident', 1);
})->orWhereExists(
static function ($children): void {
self::unresolvedRecoverableChildExists(
$children
);
}
);
});
})->orWhereRaw(
self::aggregateLocalStateConflictSql('cloud_jobs')
)->orWhere(static function ($damagedCatalogParent): void {
$damagedCatalogParent->where('phase', 'terminal')
->where('finalization_incident', 1)
->where(static function ($identity): void {
$identity->where('execution_mode', 'aggregate')
->orWhereIn('kind', self::AGGREGATE_KINDS)
->orWhere(
'finalization_error_code',
'recovery_aggregate_state_conflict'
);
});
});
});
}
private static function isLocalFinalizerRecovery(object $job): bool
{
return (string) ($job->execution_mode ?? '') === 'agent'
&& (string) ($job->phase ?? '') === 'terminal'
&& (string) ($job->outcome ?? '') === 'failed'
&& (string) ($job->status ?? '') === 'failed'
&& CloudDispatchLedger::
isLocallyFinalizableNotDispatched($job)
&& (bool) ($job->finalization_incident ?? false)
&& !CloudDispatchLedger::hasRepairIncident($job)
&& CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
(string) ($job->kind ?? '')
);
}
/** A resolved Agent row with a sticky PHP incident remains locally queued. */
private static function isResolvedStickyAgentIncident(object $job): bool
{
return (string) ($job->execution_mode ?? '') === 'agent'
&& (string) ($job->phase ?? '') === 'terminal'
&& !CloudDispatchLedger::isInvalid($job)
&& in_array(
(string) ($job->effect_resolution ?? ''),
['applied', 'not_applied'],
true
)
&& (bool) ($job->finalization_incident ?? false)
&& !CloudDispatchLedger::hasRepairIncident($job)
&& CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
(string) ($job->kind ?? '')
);
}
/** Add the negation of the canonical failed/effectful/non-retryable tuple. */
private static function applyInvalidTerminalTuple(
$invalid,
string $prefix = ''
): void {
$invalid->where(static function ($tuple) use ($prefix): void {
$tuple->whereNull($prefix . 'outcome')
->orWhere($prefix . 'outcome', '<>', 'failed')
->orWhereNull($prefix . 'status')
->orWhere($prefix . 'status', '<>', 'failed')
->orWhereNull($prefix . 'effect_may_have_applied')
->orWhere($prefix . 'effect_may_have_applied', '<>', 1)
->orWhereNull($prefix . 'safe_to_resubmit')
->orWhere($prefix . 'safe_to_resubmit', '<>', 0);
})->orWhere(static function ($neverDispatched) use ($prefix): void {
$neverDispatched->where($prefix . 'dispatch_attempts', 0)
->where($prefix . 'finalization_incident', 0);
});
}
/** Add terminal applied/not-applied rows that contradict their outcome. */
public static function applyInvalidResolvedTerminalTuple(
$invalid,
string $prefix = ''
): void {
$invalid->whereIn(
$prefix . 'effect_resolution',
['applied', 'not_applied']
)->where(static function ($shape) use ($prefix): void {
$shape->whereNull($prefix . 'terminal_at')
->orWhere($prefix . 'terminal_at', '<=', 0)
->orWhereNull($prefix . 'outcome')
->orWhereNotIn(
$prefix . 'outcome',
['succeeded', 'failed']
)->orWhere(static function ($succeeded) use ($prefix): void {
$succeeded->where($prefix . 'outcome', 'succeeded')
->where(static function ($mismatch) use ($prefix): void {
$mismatch->whereNull($prefix . 'status')
->orWhere(
$prefix . 'status',
'<>',
'succeeded'
)->orWhere(
$prefix . 'effect_resolution',
'<>',
'applied'
)->orWhereNull(
$prefix . 'effect_may_have_applied'
)->orWhere(
$prefix . 'effect_may_have_applied',
'<>',
0
)->orWhereNull(
$prefix . 'safe_to_resubmit'
)->orWhere(
$prefix . 'safe_to_resubmit',
'<>',
0
)->orWhereNotNull($prefix . 'error_code');
});
})->orWhere(static function ($failed) use ($prefix): void {
$failed->where($prefix . 'outcome', 'failed')
->where(static function ($mismatch) use ($prefix): void {
$mismatch->whereNull($prefix . 'status')
->orWhere($prefix . 'status', '<>', 'failed')
->orWhereNull(
$prefix . 'effect_may_have_applied'
)->orWhereNotIn(
$prefix . 'effect_may_have_applied',
[0, 1]
)->orWhere(static function ($effectful) use (
$prefix
): void {
$effectful->where(
$prefix . 'effect_may_have_applied',
1
)->where(static function ($safe) use (
$prefix
): void {
$safe->whereNull(
$prefix . 'safe_to_resubmit'
)->orWhere(
$prefix . 'safe_to_resubmit',
'<>',
0
);
});
})->orWhere(static function ($noEffect) use (
$prefix
): void {
$noEffect->where(
$prefix . 'effect_may_have_applied',
0
)->where(static function ($conclusion) use (
$prefix
): void {
$conclusion->where(
$prefix . 'effect_resolution',
'<>',
'not_applied'
)->orWhereNull(
$prefix . 'safe_to_resubmit'
)->orWhereNotIn(
$prefix . 'safe_to_resubmit',
[0, 1]
);
});
});
});
})->orWhere(static function ($envelope) use ($prefix): void {
self::applyInvalidResolvedAgentEnvelope(
$envelope,
$prefix
);
});
});
}
/**
* Keep an aggregate/catalog boundary fenced until every Agent child has a
* trustworthy terminal effect. During ordinary receipt finalization the
* current child is intentionally still nonterminal, so only that expected
* phase/effect transition is ignored; a corrupt dispatch ledger or sticky
* recovery incident on the current child still blocks publication.
*/
public static function aggregateParentRecoveryPending(
CloudJob $parent,
?int $completingChildId = null,
bool $lockForUpdate = false
): bool {
$query = CloudJob::where('parent_job_id', (int) $parent->id)
->orderBy('id');
if ($lockForUpdate) {
$query->lockForUpdate();
}
$children = $query->get();
if (self::aggregateRelationshipConflict(
$parent,
$children,
$completingChildId
)) {
return true;
}
return $children->contains(
static fn (CloudJob $child): bool =>
self::aggregateChildRecoveryPending(
$child,
$completingChildId !== null
&& (int) $child->id === $completingChildId
)
);
}
/** A child cannot publish its parent while any safety incident is sticky. */
private static function aggregateChildRecoveryPending(
CloudJob $child,
bool $isCompletingChild = false
): bool {
return (!$isCompletingChild
&& ((string) $child->phase !== 'terminal'
|| !in_array(
(string) $child->effect_resolution,
['applied', 'not_applied'],
true
)))
|| CloudDispatchLedger::isInvalid($child)
|| (bool) $child->finalization_incident
|| CloudDispatchLedger::hasRepairIncident($child)
|| self::isLocalFinalizerRecovery($child)
|| self::hasResolvedTerminalStateConflict($child);
}
public static function tasks(array $actor, string $state,
?int $cursor = null, int $limit = 20): array
{
if (!in_array($state, ['active', 'completed'], true)) {
throw new CloudError('任务状态筛选无效', 400);
}
// The task center is also the owner-scoped daemon-loss fail-safe. It
// persists only already-due local outcomes and never contacts Agents.
CloudWorkerService::expireDueTasksForOwner(
(int) ($actor['user_id'] ?? 0), null, 100
);
$limit = max(1, min(50, $limit));
$owner = static function ($query) use ($actor): void {
$query->where('user_id', (int) ($actor['user_id'] ?? 0));
};
// Per-host child jobs are implementation details of one public batch.
// Only their parent appears in the task center.
$base = CloudJob::query()->whereNull('vpc_batch_id')->where(
static function ($public): void {
$public->whereNull('parent_job_id')
->orWhere(static function ($currentCatalog): void {
self::whereCurrentAggregateCatalogBinding(
$currentCatalog
);
});
}
);
$owner($base);
$activeCount = (clone $base)->where('phase', '<>', 'terminal')->count();
$recovering = clone $base;
self::applyRecoveringFilter($recovering);
$recoveringCount = $recovering->count();
$unreadCount = (clone $base)->where('phase', 'terminal')
->whereNull('read_at')->count();
$query = clone $base;
$query->where('phase', $state === 'active' ? '<>' : '=', 'terminal');
if ($cursor !== null) {
$query->where('id', '<', $cursor);
}
$rows = $query->orderByDesc('id')->limit($limit + 1)->get();
$hasMore = $rows->count() > $limit;
if ($hasMore) {
$rows = $rows->take($limit);
}
$items = $rows->map(static fn (CloudJob $job): array =>
self::display($job, (bool) ($actor['is_admin'] ?? false)))->all();
$last = $rows->last();
return [
'items' => $items,
'pagination' => [
'next_cursor' => $hasMore && $last !== null
? (string) $last->id : null,
'has_more' => $hasMore,
],
'active_count' => (int) $activeCount,
'recovering_count' => (int) $recoveringCount,
'unread_count' => (int) $unreadCount,
];
}
public static function markRead(CloudJob $job): void
{
if ((string) $job->phase !== 'terminal' || $job->read_at !== null) {
return;
}
CloudJob::where('id', (int) $job->id)->where('phase', 'terminal')
->whereNull('read_at')->update(['read_at' => time()]);
}
public static function unreadCount(array $actor): int
{
$query = CloudJob::where('phase', 'terminal')->whereNull('read_at')
->whereNull('vpc_batch_id')
->where('user_id', (int) ($actor['user_id'] ?? 0))
->where(static function ($public): void {
$public->whereNull('parent_job_id')
->orWhere(static function ($currentCatalog): void {
self::whereCurrentAggregateCatalogBinding(
$currentCatalog
);
});
});
return (int) $query->count();
}
private static function requestKey(array $actor): ?string
{
$key = $actor['request_idempotency_key'] ?? null;
if ($key === null || $key === '') {
return null;
}
if (!is_string($key)
|| preg_match('/^[A-Za-z0-9_.-]{16,128}$/D', $key) !== 1) {
throw new CloudError('请求幂等标识无效', 400);
}
return $key;
}
private static function canonicalRequestValue($value)
{
if (!is_array($value)) {
return $value;
}
if (!array_is_list($value)) {
ksort($value, SORT_STRING);
}
foreach ($value as $key => $item) {
$value[$key] = self::canonicalRequestValue($item);
}
return $value;
}
/**
* Keep only the catalogued result DTO. Historical result JSON is not a
* trusted presentation format and must not bypass the error boundary.
*/
private static function displayResult(?array $result, bool $administrator,
?string $kind, ?string $status, bool $hasOrder, ?string $action): ?array
{
if ($result === null) {
return null;
}
return CloudOperationErrorCatalog::result($result, $administrator, $kind,
$status, $hasOrder, $action);
}
/** Resolve a trusted payment DTO from its owner-scoped durable record. */
private static function displayPaymentResult(CloudJob $job,
array $result): ?array
{
$id = $result['payment_id'] ?? null;
if (!is_string($id)
|| preg_match('/^[1-9][0-9]{0,18}$/D', $id) !== 1
|| $job->invoice_id === null) {
return null;
}
$payment = \App\Models\CloudInvoicePayment::where('id', (int) $id)
->where('invoice_id', (int) $job->invoice_id)
->where('user_id', (int) $job->user_id)->first();
if ($payment === null) {
return null;
}
try {
return ['payment' => CloudInvoicePaymentService::display($payment)];
} catch (\Throwable $error) {
return null;
}
}
/**
* Return the non-sensitive power action from the encrypted task payload.
*
* The durable job payload is encrypted at rest and is never returned to
* the browser. Terminal jobs retain only CloudInput::safeResult($payload),
* so this lookup continues to work for historical rows while removing
* credentials and all other payload fields from the public response.
*/
public static function action(CloudJob $job): ?string
{
if ((string) $job->kind !== 'action' || trim((string) $job->sealed_payload) === '') {
return null;
}
try {
$payload = CloudSecrets::decrypt(
(string) $job->sealed_payload,
'job-payload'
);
} catch (\Throwable $error) {
return null;
}
$body = $payload['body'] ?? null;
$action = is_array($body) && is_string($body['action'] ?? null)
? $body['action'] : '';
return CloudOperationCatalog::isPowerAction($action) ? $action : null;
}
/** Return the same safe labels used by every administrator DTO. */
public static function labels(CloudJob $job): array
{
return CloudOperationCatalog::labels((string) $job->kind,
self::action($job), self::settingsFields($job));
}
/**
* Read only safe setting-category codes from an encrypted job payload.
*
* New jobs persist the codes outside the strict Agent body so terminal
* redaction retains them. Jobs already active during deployment can still
* be described from body key presence; completed legacy password/key jobs
* have intentionally lost those keys and safely fall back to the generic
* instance-settings label.
*/
private static function settingsFields(CloudJob $job): array
{
if ((string) $job->kind !== 'instance_settings'
|| trim((string) $job->sealed_payload) === '') {
return [];
}
try {
$payload = CloudSecrets::decrypt(
(string) $job->sealed_payload,
'job-payload'
);
} catch (\Throwable $error) {
return [];
}
if (array_key_exists('settings_fields', $payload)) {
$fields = $payload['settings_fields'];
return is_array($fields) && array_is_list($fields)
? $fields : [];
}
$body = $payload['body'] ?? null;
if (!is_array($body) || array_is_list($body)) {
return [];
}
$fields = [];
foreach ([
'hostname' => 'hostname',
'root_password' => 'password',
'ssh_public_key' => 'ssh_key',
] as $bodyField => $detailCode) {
if (array_key_exists($bodyField, $body)) {
$fields[] = $detailCode;
}
}
return $fields;
}
/** Read the user's complete instance operation history from cloud_jobs. */
public static function operationLogs(CloudInstance $instance, int $page = 1, int $perPage = 10): array
{
$perPage = max(1, min(50, $perPage));
$query = CloudJob::where('instance_id', $instance->id)
->whereIn('kind', CloudOperationCatalog::instanceLogKinds())
->orderByDesc('id');
$total = max(0, (int) (clone $query)->count('id'));
$pages = max(1, (int) ceil($total / $perPage));
$page = max(1, min($pages, $page));
$jobs = $query->offset(($page - 1) * $perPage)->limit($perPage)
->get(['id', 'operation_id', 'kind', 'order_id', 'status',
'parent_job_id', 'execution_mode', 'dispatch_attempts',
'phase', 'outcome', 'attempts', 'error_code', 'result_json',
'effect_may_have_applied', 'safe_to_resubmit',
'effect_resolution', 'finalization_incident',
'finalization_error_code', 'available_at',
'terminal_at', 'read_at',
'sealed_payload', 'created_at', 'updated_at']);
$items = [];
foreach ($jobs as $job) {
$entry = self::display($job);
unset($entry['operation_id'], $entry['result']);
$items[] = $entry;
}
$from = $total === 0 ? 0 : ($page - 1) * $perPage + 1;
return ['items' => $items, 'pagination' => [
'page' => $page, 'per_page' => $perPage, 'total' => $total,
'total_pages' => $pages, 'from' => $from,
'to' => $from === 0 ? 0 : $from + count($items) - 1,
]];
}
}