You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudLifecycleService.php

969 lines
42 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudInstance;
use App\Models\CloudInvoice;
use App\Models\CloudJob;
use App\Models\CloudServicePeriod;
final class CloudLifecycleService
{
private const SAFE_NEW_GENERATION_DELAY = 300;
private array $observedErrors = [];
private int $templateReconcileAfter = 0;
private int $templateReconcileCursor = 0;
private int $rescueReconcileCursor = 0;
private int $trafficUsageScanCursor = 0;
private int $trafficPeriodScanCursor = 0;
private int $trafficCheckpointRetentionCursor = 0;
public function __construct(private ?CloudRuntimeHeartbeat $heartbeat = null)
{
}
public function observationErrors(): array
{
return array_keys($this->observedErrors);
}
private function observeFailure(string $code): void
{
$this->observedErrors[$code] = true;
$this->heartbeat?->error($code);
}
public function tick(int $limit = 200): int
{
$this->observedErrors = [];
$batchSize = max(1, min(1000, $limit));
$this->heartbeat?->busy('provision_expiry');
try {
CloudInvoiceService::expireProvision($batchSize);
} catch (\Throwable $error) {
$this->observeFailure('provision_expiry_failed');
// Releasing an abandoned provisioning reservation is database-only
// and must not stop instance expiry or traffic enforcement.
error_log('cloud provisioning invoice expiry unavailable: '
. get_class($error));
}
$this->heartbeat?->busy('invoice_generation');
try {
CloudInvoiceService::issueDue($batchSize);
foreach (CloudInvoiceService::observationErrors('issue_due') as $code) {
$this->observeFailure($code);
}
} catch (\Throwable $error) {
$this->observeFailure('invoice_generation_failed');
// Invoice creation is database-only and must not stop policy.
error_log('cloud invoice generation unavailable: ' . get_class($error));
}
$queued = 0;
$automaticLimit = max(1, min(10, $batchSize));
$this->heartbeat?->busy('automatic_backup_schedule');
try {
$queued += CloudAutomaticBackupService::scheduleDue(
$automaticLimit
);
} catch (\Throwable $error) {
$this->observeFailure('automatic_backup_schedule_failed');
error_log('cloud automatic backup schedule unavailable: '
. get_class($error));
}
$this->heartbeat?->busy('automatic_backup_retention');
try {
CloudAutomaticBackupService::pruneRetention($automaticLimit);
} catch (\Throwable $error) {
$this->observeFailure('automatic_backup_retention_failed');
error_log('cloud automatic backup retention unavailable: '
. get_class($error));
}
$this->heartbeat?->busy('template_artifact_cleanup');
try {
$cleanup = CloudTemplateDeletionService::processDue(
min(3, $automaticLimit)
);
$queued += (int) $cleanup['completed'];
if ((int) $cleanup['errors'] > 0
|| (int) $cleanup['attention'] > 0) {
$this->observeFailure('template_artifact_cleanup_failed');
}
} catch (\Throwable $error) {
$this->observeFailure('template_artifact_cleanup_failed');
error_log('cloud template artifact cleanup unavailable: '
. get_class($error));
}
$now = time();
if ($now >= $this->templateReconcileAfter) {
$this->templateReconcileAfter = $now + 300;
try {
$actor = CloudDaemonActor::resolve(
'cloud-template-reconciliation'
);
} catch (\Throwable $error) {
// An unbootstrapped configuration cannot authenticate a
// read-only host request. Skip this bounded pass; the normal
// lifecycle stages still retain their own observations.
$actor = null;
}
if ($actor !== null) {
$this->heartbeat?->busy('template_reconcile');
try {
$report = (new CloudTemplateReconciliationService())
->reconcile(
$actor,
min(4, $automaticLimit),
$automaticLimit,
$this->templateReconcileCursor
);
$this->templateReconcileCursor =
(int) $report['last_host_id'];
$queued += (int) $report['scheduled'];
if ((int) $report['errors'] > 0) {
$this->observeFailure(
'template_reconcile_failed'
);
}
} catch (\Throwable $error) {
$this->observeFailure('template_reconcile_failed');
error_log('cloud template reconciliation unavailable: '
. get_class($error));
}
$this->heartbeat?->busy('rescue_template_reconcile');
try {
$report = (new
CloudRescueTemplateReconciliationService())
->reconcile(
$actor,
min(4, $automaticLimit),
$automaticLimit,
$this->rescueReconcileCursor
);
$this->rescueReconcileCursor =
(int) $report['last_host_id'];
$queued += (int) $report['scheduled'];
if ((int) $report['errors'] > 0) {
$this->observeFailure(
'rescue_template_reconcile_failed'
);
}
} catch (\Throwable $error) {
$this->observeFailure(
'rescue_template_reconcile_failed'
);
error_log(
'cloud rescue template reconciliation unavailable: '
. get_class($error)
);
}
$this->heartbeat?->busy('rescue_template_cleanup');
try {
$cleanup = CloudRescueTemplateRetirementService::
processDue($actor, $automaticLimit);
$queued += (int) $cleanup['scheduled']
+ (int) $cleanup['completed'];
if ((int) $cleanup['errors'] > 0
|| (int) $cleanup['attention'] > 0) {
$this->observeFailure(
'rescue_template_cleanup_failed'
);
}
} catch (\Throwable $error) {
$this->observeFailure(
'rescue_template_cleanup_failed'
);
error_log(
'cloud rescue template cleanup unavailable: '
. get_class($error)
);
}
}
}
$this->heartbeat?->busy('vpc_deleted_instance_cleanup');
try {
$queued += CloudVpcPortService::retryDeletedCleanupDue(
$automaticLimit
);
} catch (\Throwable $error) {
$this->observeFailure('vpc_deleted_instance_cleanup_failed');
error_log('cloud VPC deleted-instance cleanup unavailable: '
. get_class($error));
}
$this->heartbeat?->busy('traffic_sync');
try {
$collection = CloudTrafficService::collect($batchSize);
foreach (array_unique($collection['errors'] ?? []) as $code) {
$this->observeFailure((string) $code);
}
CloudTrafficService::sync($batchSize);
} catch (\Throwable $error) {
$this->observeFailure('traffic_sync_failed');
// Unknown traffic is fail-open. Time expiry remains enforceable.
error_log('cloud traffic sync unavailable: ' . get_class($error));
}
// Shared-pool accounting consumes only counters already persisted by
// the traffic sync above. It deliberately runs outside every
// per-instance lifecycle transaction: pool accounting owns the
// pool -> cursor -> entitlement lock order, while network policy
// remains on the existing instance reconciliation path until the
// restriction/convergence layer is enabled.
$this->heartbeat?->busy('traffic_pool_accounting');
try {
$settled = CloudTrafficPoolAccounting::settlePendingUsage(
$batchSize,
$this->trafficUsageScanCursor
);
$this->trafficUsageScanCursor = max(
0,
(int) ($settled['next_cursor'] ?? 0)
);
$due = CloudTrafficPoolAccounting::reconcileDue(
$batchSize,
null,
$this->trafficPeriodScanCursor
);
$this->trafficPeriodScanCursor = max(
0,
(int) ($due['next_cursor'] ?? 0)
);
if (($settled['errors'] ?? []) !== []
|| ($due['errors'] ?? []) !== []) {
$this->observeFailure('traffic_pool_accounting_failed');
error_log('cloud traffic-pool accounting had bounded errors');
}
} catch (\Throwable $error) {
$this->observeFailure('traffic_pool_accounting_failed');
// A projection failure must never become a guessed network
// decision. Existing instance policy remains authoritative.
error_log('cloud traffic-pool accounting unavailable: '
. get_class($error));
}
$this->heartbeat?->busy('traffic_settlement_compaction');
try {
$compaction = CloudTrafficSettlementRetention::compact(
$batchSize,
time()
);
if ($compaction['blocked'] !== []) {
$this->observeFailure('traffic_retention_failed');
}
} catch (\Throwable $error) {
$this->observeFailure('traffic_retention_failed');
// Retention is never allowed to block current accounting or
// instance policy. Full buckets remain intact for a later retry.
error_log('cloud traffic settlement retention unavailable: '
. get_class($error));
}
$this->heartbeat?->busy('traffic_checkpoint_compaction');
try {
$checkpointRetention = CloudTrafficCheckpointRetention::prune(
$batchSize,
time(),
$this->trafficCheckpointRetentionCursor
);
$this->trafficCheckpointRetentionCursor = max(
0,
(int) ($checkpointRetention['next_cursor'] ?? 0)
);
if ($checkpointRetention['blocked'] !== []) {
$this->observeFailure('traffic_retention_failed');
}
} catch (\Throwable $error) {
$this->observeFailure('traffic_retention_failed');
// Hourly history is retained on every ambiguity; compaction must
// never gate live accounting, billing or instance policy.
error_log('cloud traffic checkpoint retention unavailable: '
. get_class($error));
}
// Power state is an observation cache, not a billing decision. Each
// host is contacted through its own Agent settings, so one unavailable
// host must not gate database-only billing or other hosts.
$this->heartbeat?->busy('runtime_status_sync');
try {
$runtime = (new CloudInstanceRuntimeSyncService())->sync($batchSize);
foreach (array_unique($runtime['errors'] ?? []) as $code) {
$this->observeFailure((string) $code);
}
} catch (\Throwable $error) {
$this->observeFailure('runtime_status_sync_failed');
error_log('cloud runtime status sync unavailable: ' . get_class($error));
}
$this->heartbeat?->busy('instance_reconcile');
CloudInstance::where('status', '<>', 'deleted')
->where('service_status', '<>', 'exempt')
->select('id')->chunkById($batchSize, function ($instances) use (&$queued) {
foreach ($instances as $instance) {
$this->heartbeat?->busy('instance_reconcile');
try {
$queued += $this->reconcile((int) $instance->id) ? 1 : 0;
} catch (\Throwable $error) {
$this->observeFailure('instance_reconcile_failed');
// One corrupt or temporarily contended record must not
// starve every higher instance ID on every daemon pass.
error_log('cloud lifecycle instance ' . (int) $instance->id
. ' unavailable: ' . get_class($error));
}
}
});
return $queued;
}
private function reconcile(int $instanceId): bool
{
return (new CloudInstance())->getConnection()->transaction(function () use ($instanceId) {
$now = time();
$instance = CloudInstance::where('id', $instanceId)->lockForUpdate()->first();
if ($instance === null || (string) $instance->status === 'deleted'
|| $instance->service_status === 'exempt') {
return false;
}
// A paid grow-only resize owns the instance, current service
// period and renewal snapshot as one recovery unit. In
// particular, an unknown-effect resize may cross the expiry
// boundary while its original operation is being inspected.
// Freeze lifecycle projection before finishing that period or
// clearing current_period_id; the upgrade finalizer will then
// either apply the immutable target or restore the source ledger.
if (CloudJobService::blockingQueryForInstance(
(int) $instance->id
)->where('kind', 'instance_upgrade')->exists()) {
return false;
}
if ((string) $instance->status === 'delete_failed'
|| (string) $instance->service_status === 'purge_failed') {
if ((string) $instance->status !== 'delete_failed'
|| (string) $instance->service_status !== 'purge_failed'
|| $this->retryableNotAppliedJobLocked(
$instance,
'expiry_delete',
$now
) === null) {
return false;
}
$instance->status = 'deleting';
$instance->service_status = 'purge_deleting';
$instance->updated_at = $now;
$instance->save();
CloudJobService::enqueueExpiryDelete($instance);
return true;
}
if ((string) $instance->status === 'deleting'
&& (string) $instance->service_status === 'purge_stop_failed') {
if ($this->retryableNotAppliedJobLocked(
$instance,
'service_suspend',
$now
) === null) {
return false;
}
$instance->service_status = 'purge_stopping';
$instance->updated_at = $now;
$instance->save();
return $this->enqueue(
$instance,
'service_suspend',
'service/suspend',
'purge',
['reason' => 'purge']
);
}
if ((string) $instance->service_status === 'purge_stop_failed') {
return false;
}
// A delete with a lost response is already a terminal failed task,
// but its resource fence remains: the domain and disks may already
// be gone. Freeze later mutations until a super administrator uses
// a fresh read-only inspection to resolve that exact operation's
// fence. Resolution never reopens or replays the original task.
if (CloudJobService::blockingQueryForInstance((int) $instance->id)
->whereIn('kind',
['delete', 'expiry_delete', 'delete_force'])->exists()) {
return false;
}
$period = $instance->current_period_id === null ? null
: CloudServicePeriod::where('id', $instance->current_period_id)
->where('instance_id', $instance->id)->lockForUpdate()->first();
$policyPeriod = $period;
if ($period !== null && in_array((string) $period->status, ['active', 'exhausted'], true)) {
$this->updateUsage($instance, $period, $now);
if ((int) $period->ends_at <= $now) {
$period->status = 'finished';
$period->updated_at = $now;
$period->save();
$period = null;
$instance->current_period_id = null;
}
} else {
$period = null;
$instance->current_period_id = null;
}
if ($period === null) {
// The lifecycle daemon may be stopped for longer than one
// prepaid period. Never briefly activate an already expired
// historical period (which would also restore its network).
CloudServicePeriod::where('instance_id', $instance->id)
->where('status', 'scheduled')->where('ends_at', '<=', $now)
->update(['status' => 'finished', 'updated_at' => $now]);
$period = CloudServicePeriod::where('instance_id', $instance->id)
->where('status', 'scheduled')->where('starts_at', '<=', $now)
->where('ends_at', '>', $now)
->orderBy('starts_at')->orderBy('id')->lockForUpdate()->first();
if ($period !== null) {
$period->status = 'active';
$period->activated_at = $now;
$baseline = CloudTrafficService::periodBaseline(
$instance,
(int) $period->starts_at,
(int) $period->id
);
if ($baseline === null
&& (int) $instance->traffic_synced_at >= (int) $period->starts_at) {
$baseline = [
'download' => max(0, (int) $instance->lifetime_download_bytes),
'upload' => max(0, (int) $instance->lifetime_upload_bytes),
];
}
$period->download_base_bytes = $baseline['download'] ?? null;
$period->upload_base_bytes = $baseline['upload'] ?? null;
$period->download_used_bytes = $baseline === null ? 0
: max(0, (int) $instance->lifetime_download_bytes
- (int) $baseline['download']);
$period->upload_used_bytes = $baseline === null ? 0
: max(0, (int) $instance->lifetime_upload_bytes
- (int) $baseline['upload']);
$period->updated_at = $now;
$period->save();
$instance->current_period_id = $period->id;
$instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes);
$instance->traffic_download_bytes = (int) $period->download_used_bytes;
$instance->traffic_upload_bytes = (int) $period->upload_used_bytes;
$policyPeriod = $period;
$aggregateNetworkPolicy =
CloudInstanceNetworkPolicy::isEnforcedPeriod($period)
|| CloudInstanceNetworkPolicy::hasState(
(int) $instance->id
);
$this->convergePaidPeriodProjectionLocked(
$instance,
$now,
!$aggregateNetworkPolicy
);
}
}
$aggregateNetworkPolicy =
CloudInstanceNetworkPolicy::isEnforcedPeriod(
$period ?? $policyPeriod
) || CloudInstanceNetworkPolicy::hasState((int) $instance->id);
if ($period !== null) {
$usageKnown = $this->updateUsage($instance, $period, $now);
$used = (int) $period->download_used_bytes + (int) $period->upload_used_bytes;
$quota = max(0, (int) $period->traffic_quota_bytes);
if (!$aggregateNetworkPolicy
&& $usageKnown && $quota > 0 && $used >= $quota) {
if ((string) $period->status !== 'exhausted') {
$period->status = 'exhausted';
$period->updated_at = $now;
$period->save();
}
if (!in_array((string) $instance->network_status,
['blocking', 'blocked_traffic', 'block_failed'], true)) {
$instance->network_status = 'blocking';
}
} elseif ($usageKnown) {
$this->convergePaidPeriodProjectionLocked(
$instance,
$now,
!$aggregateNetworkPolicy
);
}
} elseif ($instance->expires_at !== null && (int) $instance->expires_at <= $now) {
$invoice = CloudInvoice::where('instance_id', $instance->id)
->where('renews_expires_at', $instance->expires_at)
->where('status', '<>', 'void')->orderByDesc('id')
->lockForUpdate()->first();
if ($invoice === null) {
// A missing invoice is an observable billing anomaly, not
// an unlimited service extension. The immutable instance
// expiry remains a safe fallback grace anchor.
$this->observeFailure('renewal_invoice_missing');
$pastGrace = (int) $instance->expires_at
<= $now - CloudInvoiceService::PAYMENT_GRACE_SECONDS;
} else {
$pastGrace = in_array((string) $invoice->status,
['open', 'overdue', 'expired'], true)
&& CloudInvoiceService::paymentDeadlinePassed(
$invoice,
$now
);
}
// A maintenance migration can temporarily retain two host
// identities and two locked address reservations. Never let
// the ordinary expiry deleter remove only the current target
// while leaving the fenced source behind. Network expiry still
// converges below; permanent purge resumes after migration
// cleanup reaches a terminal state.
if ($pastGrace
&& CloudInstanceMigrationService::hasOpenMigration(
(int) $instance->id
)) {
$pastGrace = false;
}
if ($pastGrace) {
if ($invoice !== null && in_array(
(string) $invoice->status,
['open', 'overdue'],
true
)) {
$invoice->status = 'expired';
$invoice->updated_at = $now;
$invoice->save();
}
$instance->status = 'deleting';
if (!in_array((string) $instance->service_status,
['purge_stopping', 'purge_force_required',
'purge_deleting', 'purge_failed',
'purge_stop_failed'], true)) {
$instance->service_status = 'purge_stopping';
}
} elseif (!in_array((string) $instance->service_status,
['purge_stopping', 'purge_stop_failed',
'purge_force_required', 'purge_deleting',
'purge_failed'], true)) {
// Expiry keeps QEMU running throughout the three-day
// payment grace period. Only its NIC links are persisted
// down; a paid renewal restores those links. During an
$instance->service_status = 'expired_grace';
$instance->suspended_at = null;
if (!$aggregateNetworkPolicy) {
$network = (string) $instance->network_status;
if ($network === 'blocked_traffic') {
$instance->network_status = 'blocked_expired';
} elseif (!in_array($network,
['blocking', 'blocking_expired', 'blocked_expired',
'block_failed'], true)) {
$instance->network_status = 'blocking_expired';
}
}
}
}
$networkPolicy = null;
if ($aggregateNetworkPolicy) {
$networkPolicy = CloudInstanceNetworkPolicy::synchronizeLocked(
$instance,
$period ?? $policyPeriod,
$now
);
}
$policyDeferred = ($networkPolicy['deferred'] ?? false) === true;
$network = (string) $instance->network_status;
if ($aggregateNetworkPolicy && !$policyDeferred
&& ($networkPolicy['state'] ?? null) === 'failed') {
$retryKind = ($networkPolicy['desired_suspended'] ?? false)
=== true && ($networkPolicy['applied_suspended'] ?? true)
=== false ? 'network_suspend'
: ((($networkPolicy['desired_suspended'] ?? true) === false
&& ($networkPolicy['applied_suspended'] ?? false) === true)
? 'network_resume' : null);
if ($retryKind !== null
&& $this->retryableNotAppliedJobLocked(
$instance,
$retryKind,
$now
) !== null
&& CloudInstanceNetworkPolicy::allowRetryLocked(
$instance,
$retryKind,
$now
)) {
$networkPolicy =
CloudInstanceNetworkPolicy::snapshotLocked(
$instance,
$now
);
}
} elseif ((!$aggregateNetworkPolicy || $policyDeferred)
&& $network === 'block_failed') {
$reason = null;
$nextState = null;
if ($period !== null
&& (string) $period->status === 'exhausted') {
$reason = 'traffic';
$nextState = 'blocking';
} elseif ($period === null
&& (string) $instance->service_status === 'expired_grace'
&& $instance->expires_at !== null
&& (int) $instance->expires_at <= $now) {
$reason = 'expired';
$nextState = 'blocking_expired';
}
if ($reason !== null
&& $this->retryableNotAppliedJobLocked(
$instance,
'network_suspend',
$now
) !== null) {
$instance->network_status = $nextState;
}
} elseif ((!$aggregateNetworkPolicy || $policyDeferred)
&& $network === 'restore_failed'
&& $period !== null
&& (string) $period->status === 'active'
&& $this->retryableNotAppliedJobLocked(
$instance,
'network_resume',
$now
) !== null) {
$instance->network_status = 'restore_pending';
$instance->network_suspended_at = null;
}
$instance->updated_at = $now;
$instance->save();
if ($this->hasConflictingJob($instance)) {
return false;
}
$service = (string) $instance->service_status;
$network = (string) $instance->network_status;
if ($service === 'purge_stopping') {
return $this->enqueue($instance, 'service_suspend',
'service/suspend', 'purge', ['reason' => 'purge']);
}
if ($service === 'purge_deleting') {
$instance->status = 'deleting';
$instance->updated_at = $now;
$instance->save();
CloudJobService::enqueueExpiryDelete($instance);
return true;
}
if ($aggregateNetworkPolicy && !$policyDeferred) {
$action = (string) ($networkPolicy['action'] ?? 'none');
if ($action === 'suspend') {
return $this->enqueue(
$instance,
'network_suspend',
'network/suspend',
(string) ($networkPolicy['suspend_reason'] ?? 'traffic'),
[],
true
);
}
if ($action === 'resume') {
$instance->network_status = 'restoring';
$instance->updated_at = $now;
$instance->save();
return $this->enqueue(
$instance,
'network_resume',
'network/resume',
null,
[],
true
);
}
return false;
}
if ($network === 'blocking_expired') {
return $this->enqueue($instance, 'network_suspend',
'network/suspend', 'expired', ['reason' => 'expired']);
}
if ($network === 'blocking') {
return $this->enqueue($instance, 'network_suspend',
'network/suspend', 'traffic', ['reason' => 'traffic']);
}
if (in_array($network, ['restore_pending', 'restoring'], true)) {
$instance->network_status = 'restoring';
$instance->save();
return $this->enqueue($instance, 'network_resume',
'network/resume');
}
return false;
}, 3);
}
private function hasConflictingJob(CloudInstance $instance): bool
{
return CloudJobService::blockingQueryForInstance(
(int) $instance->id
)->exists();
}
/**
* Bring a paid period's local projection forward without overwriting an
* in-flight or uncertain policy mutation. Definitively applied network
* suspension can start a resume; definitively not-applied failures may be
* cleared or moved to a new generation by the strict ledger helper.
*/
private function convergePaidPeriodProjectionLocked(
CloudInstance $instance,
int $now,
bool $convergeNetwork = true
): void {
$service = (string) $instance->service_status;
if ($service === 'suspend_failed') {
if ($this->retryableNotAppliedJobLocked(
$instance,
'service_suspend',
$now
) !== null) {
$instance->service_status = 'active';
$instance->suspended_at = null;
}
} elseif (!in_array($service, [
'active',
'exempt',
'suspending_expired',
'purge_stopping',
'purge_stop_failed',
'purge_force_required',
'purge_deleting',
'purge_failed',
], true)) {
$instance->service_status = 'active';
$instance->suspended_at = null;
}
if (!$convergeNetwork) {
return;
}
$network = (string) $instance->network_status;
if ($network === 'block_failed') {
if ($this->retryableNotAppliedJobLocked(
$instance,
'network_suspend',
$now
) !== null) {
// A proven no-effect suspension means the links never left
// their paid-period target state; no compensating mutation is
// required.
$instance->network_status = 'active';
$instance->network_suspended_at = null;
}
} elseif ($network === 'restore_failed') {
if ($this->retryableNotAppliedJobLocked(
$instance,
'network_resume',
$now
) !== null) {
$instance->network_status = 'restore_pending';
$instance->network_suspended_at = null;
}
} elseif (!in_array($network, [
'active',
'blocking',
'blocking_expired',
'restore_pending',
'restoring',
], true)) {
$instance->network_status = 'restore_pending';
$instance->network_suspended_at = null;
}
}
/**
* A policy retry is a new audited generation, never a replay. It is
* allowed only after the newest generation proved that no effect happened
* and all immutable Agent and resource bindings still match.
*/
private function retryableNotAppliedJobLocked(
CloudInstance $instance,
string $kind,
int $now
): ?CloudJob {
$job = CloudJob::where('instance_id', (int) $instance->id)
->where('kind', $kind)
->orderByDesc('id')
->lockForUpdate()
->first();
if ($job === null) {
return null;
}
if ((bool) $job->finalization_incident) {
return null;
}
if ((string) $job->execution_mode !== 'agent'
|| $job->parent_job_id !== null
|| (int) $job->user_id !== (int) $instance->user_id
|| (int) $job->host_id !== (int) $instance->host_id) {
$this->markLifecycleRecoveryIncident(
$job,
'recovery_instance_state_conflict',
$now
);
return null;
}
if ((string) $job->phase !== 'terminal') {
return null;
}
if (CloudDispatchLedger::isInvalid($job)
|| CloudJobService::hasTerminalRecoveryStateConflict($job)) {
$this->markLifecycleRecoveryIncident(
$job,
CloudDispatchLedger::isInvalid($job)
? 'recovery_dispatch_ledger_conflict'
: 'recovery_instance_state_conflict',
$now
);
return null;
}
if ((string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| (string) $job->effect_resolution !== 'not_applied'
|| (bool) $job->effect_may_have_applied
|| (string) ($job->error_code ?? '') === 'force_required') {
if ((string) $job->effect_resolution === 'applied'
|| (string) ($job->error_code ?? '') === 'force_required') {
// The resource is still in a failed policy projection even
// though the immutable job says the mutation happened (or
// requires a different authorized action). Keep that
// contradiction visible to the PHP-only recovery queue.
$this->markLifecycleRecoveryIncident(
$job,
'recovery_instance_state_conflict',
$now
);
}
return null;
}
if (max(
(int) ($job->terminal_at ?? 0),
(int) ($job->updated_at ?? 0)
) > $now - self::SAFE_NEW_GENERATION_DELAY) {
return null;
}
try {
$identity = CloudOperationRecoveryIdentity::expected($job);
} catch (\Throwable $error) {
$code = match (true) {
$error instanceof CloudRecoveryPayloadUnavailable =>
'recovery_payload_unavailable',
$error instanceof CloudRecoveryOperationIdentityConflict =>
'recovery_operation_identity_conflict',
$error instanceof CloudRecoveryInstanceStateConflict =>
'recovery_instance_state_conflict',
default => 'recovery_internal_error',
};
$this->markLifecycleRecoveryIncident($job, $code, $now);
return null;
}
$expectedAction = [
'expiry_delete' => 'delete-vm',
'service_suspend' => 'service-suspend',
'network_suspend' => 'network-suspend',
'network_resume' => 'network-resume',
][$kind] ?? null;
if ($expectedAction === null
|| ($identity['action'] ?? null) !== $expectedAction
|| ($identity['resource_id'] ?? null)
!== strtolower((string) $instance->uuid)
|| !is_string($identity['request_fingerprint'] ?? null)
|| preg_match(
'/^[a-f0-9]{64}$/D',
$identity['request_fingerprint']
) !== 1) {
$this->markLifecycleRecoveryIncident(
$job,
'recovery_operation_identity_conflict',
$now
);
return null;
}
return $job;
}
private function markLifecycleRecoveryIncident(
CloudJob $job,
string $code,
int $now
): void {
if ((bool) $job->finalization_incident) {
return;
}
$job->finalization_incident = 1;
$job->finalization_error_code = $code;
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = $now + self::SAFE_NEW_GENERATION_DELAY;
$job->updated_at = $now;
$job->save();
}
private function updateUsage(CloudInstance $instance, CloudServicePeriod $period, int $now): bool
{
if ($period->download_base_bytes === null || $period->upload_base_bytes === null) {
// At a period boundary, an old cached total would charge bytes
// from the previous period to the new one. Wait for the first
// successful sample at/after the boundary and fail open until it
// establishes both baselines.
if ((int) $instance->traffic_synced_at < (int) $period->starts_at) {
$period->download_base_bytes = null;
$period->upload_base_bytes = null;
$period->download_used_bytes = 0;
$period->upload_used_bytes = 0;
$period->updated_at = $now;
$period->save();
$instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes);
$instance->traffic_download_bytes = 0;
$instance->traffic_upload_bytes = 0;
return false;
}
$period->download_base_bytes = max(0, (int) $instance->lifetime_download_bytes);
$period->upload_base_bytes = max(0, (int) $instance->lifetime_upload_bytes);
}
$download = max(0, (int) $instance->lifetime_download_bytes - (int) $period->download_base_bytes);
$upload = max(0, (int) $instance->lifetime_upload_bytes - (int) $period->upload_base_bytes);
$period->download_used_bytes = $download;
$period->upload_used_bytes = $upload;
$period->updated_at = $now;
$period->save();
$instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes);
$instance->traffic_download_bytes = $download;
$instance->traffic_upload_bytes = $upload;
return true;
}
private function enqueue(CloudInstance $instance, string $kind, string $suffix,
?string $reason = null, array $body = [],
bool $aggregateNetworkPolicy = false): bool
{
$job = new CloudJob();
$job->operation_id = bin2hex(random_bytes(16));
$networkPolicyMarker = null;
if ($aggregateNetworkPolicy) {
$networkPolicyMarker =
CloudInstanceNetworkPolicy::prepareJobLocked(
$instance,
$kind,
(string) $job->operation_id
);
if ($kind === 'network_suspend') {
$reason = (string) $networkPolicyMarker['suspend_reason'];
}
}
$job->instance_id = $instance->id;
$job->user_id = $instance->user_id;
$job->host_id = $instance->host_id;
$job->kind = $kind;
$job->execution_mode = 'agent';
$job->status = 'queued';
$payload = [
'method' => 'POST',
'request_target' => '/v1/vms/' . rawurlencode($instance->uuid)
. '/' . $suffix,
'body' => $body, 'owner_id' => (int) $instance->user_id,
'system_policy' => true, 'suspend_reason' => $reason,
];
if ($networkPolicyMarker !== null) {
$payload['_network_policy'] = $networkPolicyMarker;
}
$job->sealed_payload = CloudSecrets::encrypt($payload, 'job-payload');
$job->available_at = $job->created_at = $job->updated_at = time();
$job->save();
return true;
}
}