observedErrors); } private function observeFailure(string $code): void { $this->observedErrors[$code] = true; $this->heartbeat?->error($code); } public function tick(int $limit = 200, bool $skipEngineObservations = false): int { $this->observedErrors = []; if ($skipEngineObservations) { $this->observeFailure('engine_readiness_timeout'); } $batchSize = max(1, min(1000, $limit)); $this->heartbeat?->busy('provision_expiry'); try { CloudInvoiceService::expireProvision($batchSize); } catch (\Throwable $error) { $this->observeFailure('provision_expiry_failed'); // Releasing an abandoned provisioning reservation is database-only // and must not stop instance expiry or traffic enforcement. error_log('cloud provisioning invoice expiry unavailable: ' . get_class($error)); } $this->heartbeat?->busy('invoice_generation'); try { CloudInvoiceService::issueDue($batchSize); foreach (CloudInvoiceService::observationErrors('issue_due') as $code) { $this->observeFailure($code); } } catch (\Throwable $error) { $this->observeFailure('invoice_generation_failed'); // Invoice creation is database-only and must not stop policy. error_log('cloud invoice generation unavailable: ' . get_class($error)); } $this->heartbeat?->busy('traffic_sync'); try { CloudTrafficService::sync($batchSize); } catch (\Throwable $error) { $this->observeFailure('traffic_sync_failed'); // Unknown traffic is fail-open. Time expiry remains enforceable. error_log('cloud traffic sync unavailable: ' . get_class($error)); } // Power state is an observation cache, not a billing decision. Keep // it on the lifecycle cadence so list pages can render immediately // from the last successful host batch without waiting on libvirt. if (!$skipEngineObservations) { $this->heartbeat?->busy('runtime_status_sync'); try { $runtime = (new CloudInstanceRuntimeSyncService())->sync($batchSize); foreach (array_unique($runtime['errors'] ?? []) as $code) { $this->observeFailure((string) $code); } } catch (\Throwable $error) { $this->observeFailure('runtime_status_sync_failed'); // A status poll must never prevent invoice or expiry policy work. error_log('cloud runtime status sync unavailable: ' . get_class($error)); } } $queued = 0; $this->heartbeat?->busy('instance_reconcile'); CloudInstance::where('status', '<>', 'deleted') ->where('service_status', '<>', 'exempt') ->select('id')->chunkById($batchSize, function ($instances) use (&$queued) { foreach ($instances as $instance) { $this->heartbeat?->busy('instance_reconcile'); try { $queued += $this->reconcile((int) $instance->id) ? 1 : 0; } catch (\Throwable $error) { $this->observeFailure('instance_reconcile_failed'); // One corrupt or temporarily contended record must not // starve every higher instance ID on every daemon pass. error_log('cloud lifecycle instance ' . (int) $instance->id . ' unavailable: ' . get_class($error)); } } }); return $queued; } private function reconcile(int $instanceId): bool { return (new CloudInstance())->getConnection()->transaction(function () use ($instanceId) { $now = time(); $instance = CloudInstance::where('id', $instanceId)->lockForUpdate()->first(); if ($instance === null || in_array((string) $instance->status, ['deleted', 'delete_failed'], true) || $instance->service_status === 'exempt') { return false; } if ((string) $instance->service_status === 'purge_failed') { return false; } $period = $instance->current_period_id === null ? null : CloudServicePeriod::where('id', $instance->current_period_id) ->where('instance_id', $instance->id)->lockForUpdate()->first(); if ($period !== null && in_array((string) $period->status, ['active', 'exhausted'], true)) { $this->updateUsage($instance, $period, $now); if ((int) $period->ends_at <= $now) { $period->status = 'finished'; $period->updated_at = $now; $period->save(); $period = null; $instance->current_period_id = null; } } else { $period = null; $instance->current_period_id = null; } if ($period === null) { // The lifecycle daemon may be stopped for longer than one // prepaid period. Never briefly activate an already expired // historical period (which would also restore its network). CloudServicePeriod::where('instance_id', $instance->id) ->where('status', 'scheduled')->where('ends_at', '<=', $now) ->update(['status' => 'finished', 'updated_at' => $now]); $period = CloudServicePeriod::where('instance_id', $instance->id) ->where('status', 'scheduled')->where('starts_at', '<=', $now) ->where('ends_at', '>', $now) ->orderBy('starts_at')->orderBy('id')->lockForUpdate()->first(); if ($period !== null) { $period->status = 'active'; $period->activated_at = $now; $baseline = CloudTrafficService::periodBaseline( $instance, (int) $period->starts_at); if ($baseline === null && (int) $instance->traffic_synced_at >= (int) $period->starts_at) { $baseline = [ 'download' => max(0, (int) $instance->lifetime_download_bytes), 'upload' => max(0, (int) $instance->lifetime_upload_bytes), ]; } $period->download_base_bytes = $baseline['download'] ?? null; $period->upload_base_bytes = $baseline['upload'] ?? null; $period->download_used_bytes = $baseline === null ? 0 : max(0, (int) $instance->lifetime_download_bytes - (int) $baseline['download']); $period->upload_used_bytes = $baseline === null ? 0 : max(0, (int) $instance->lifetime_upload_bytes - (int) $baseline['upload']); $period->updated_at = $now; $period->save(); $instance->current_period_id = $period->id; $instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes); $instance->traffic_download_bytes = (int) $period->download_used_bytes; $instance->traffic_upload_bytes = (int) $period->upload_used_bytes; if (!in_array((string) $instance->service_status, ['active', 'exempt'], true)) { $legacySuspend = $instance->suspended_at !== null || CloudJob::where('instance_id', $instance->id) ->where('kind', 'service_suspend') ->whereIn('status', ['queued', 'running', 'uncertain'])->exists(); $instance->service_status = $legacySuspend ? 'resume_pending' : 'active'; } if ((string) $instance->network_status !== 'active') { $instance->network_status = 'restore_pending'; $instance->network_suspended_at = null; } } } if ($period !== null) { $usageKnown = $this->updateUsage($instance, $period, $now); $used = (int) $period->download_used_bytes + (int) $period->upload_used_bytes; $quota = max(0, (int) $period->traffic_quota_bytes); if ($usageKnown && $quota > 0 && $used >= $quota) { if ((string) $period->status !== 'exhausted') { $period->status = 'exhausted'; $period->updated_at = $now; $period->save(); } if ((string) $instance->network_status === 'block_failed' && $this->policyRetryReady($instance, 'network_suspend', $now)) { $instance->network_status = 'blocking'; } elseif (!in_array((string) $instance->network_status, ['blocking', 'blocked_traffic', 'block_failed'], true)) { $instance->network_status = 'blocking'; } } elseif ($usageKnown) { if ((string) $instance->network_status === 'restore_failed' && $this->policyRetryReady($instance, 'network_resume', $now)) { $instance->network_status = 'restore_pending'; $instance->network_suspended_at = null; } elseif (!in_array((string) $instance->network_status, ['active', 'restore_failed'], true)) { $instance->network_status = 'restore_pending'; $instance->network_suspended_at = null; } } if ((string) $instance->service_status === 'resume_failed' && $this->policyRetryReady($instance, 'service_resume', $now)) { $instance->service_status = 'resume_pending'; } } elseif ($instance->expires_at !== null && (int) $instance->expires_at <= $now) { $invoice = CloudInvoice::where('instance_id', $instance->id) ->where('renews_expires_at', $instance->expires_at) ->where('status', '<>', 'void')->orderByDesc('id') ->lockForUpdate()->first(); $pastGrace = $invoice !== null && in_array((string) $invoice->status, ['open', 'overdue', 'expired'], true) && CloudInvoiceService::paymentDeadlinePassed($invoice, $now); if ($pastGrace) { if (in_array((string) $invoice->status, ['open', 'overdue'], true)) { $invoice->status = 'expired'; $invoice->updated_at = $now; $invoice->save(); } $instance->status = 'deleting'; if ((string) $instance->service_status === 'purge_stop_failed') { if ($this->policyRetryReady($instance, 'service_suspend', $now)) { $instance->service_status = 'purge_stopping'; } } elseif (!in_array((string) $instance->service_status, ['purge_stopping', 'purge_deleting', 'purge_failed'], true)) { $instance->service_status = 'purge_stopping'; } } elseif (!in_array((string) $instance->service_status, ['purge_stopping', 'purge_stop_failed', 'purge_deleting', 'purge_failed'], true)) { // Expiry keeps QEMU running throughout the three-day // payment grace period. Only its NIC links are persisted // down; a paid renewal restores those links. During an // online upgrade, a durable suspended_at marker proves // that the retired policy stopped a previously running // guest, which must be restarted only after link-down. $instance->service_status = $instance->suspended_at === null ? 'expired_grace' : 'resume_pending'; $network = (string) $instance->network_status; if ($network === 'blocked_traffic') { $instance->network_status = 'blocked_expired'; } elseif ($network === 'block_failed') { if ($this->policyRetryReady($instance, 'network_suspend', $now)) { $instance->network_status = 'blocking_expired'; } } elseif (!in_array($network, ['blocking', 'blocking_expired', 'blocked_expired'], true)) { $instance->network_status = 'blocking_expired'; } } } $instance->updated_at = $now; $instance->save(); if ($this->hasConflictingJob($instance)) { return false; } $service = (string) $instance->service_status; $network = (string) $instance->network_status; if ($service === 'purge_stopping') { return $this->enqueue($instance, 'service_suspend', 'service-suspend', 'purge'); } if ($service === 'purge_deleting') { $instance->status = 'deleting'; $instance->updated_at = $now; $instance->save(); CloudJobService::enqueueExpiryDelete($instance); return true; } if ($network === 'blocking_expired') { return $this->enqueue($instance, 'network_suspend', 'network-suspend', 'expired'); } if ($network === 'blocking') { return $this->enqueue($instance, 'network_suspend', 'network-suspend', 'traffic'); } if (in_array($service, ['resume_pending', 'resuming'], true)) { $legacyExpiryResume = $instance->expires_at !== null && (int) $instance->expires_at <= $now; if ($legacyExpiryResume && $network !== 'blocked_expired') { return false; } $instance->service_status = 'resuming'; $instance->save(); return $this->enqueue($instance, 'service_resume', 'service-resume', null, ['start_if_suspended' => $instance->suspended_at !== null]); } if (in_array($network, ['restore_pending', 'restoring'], true)) { $instance->network_status = 'restoring'; $instance->save(); return $this->enqueue($instance, 'network_resume', 'network-resume'); } return false; }, 3); } private function hasConflictingJob(CloudInstance $instance): bool { // Match reconcile()'s target ordering: purge stop/delete wins before // network policy. Only a tick that will actually enqueue // network_suspend may defer snapshot serialization to the engine's // durable queued-versus-running decision. $failSafeSuspend = !in_array((string) $instance->service_status, ['purge_stopping', 'purge_deleting'], true) && in_array((string) $instance->network_status, ['blocking', 'blocking_expired'], true); $snapshotKinds = [ 'snapshot_create', 'snapshot_restore', 'snapshot_delete', ]; // Firewall changes never alter power, link state or resource // ownership. Let lifecycle policy enter its own durable queue even if // a firewall operation is queued, running or awaiting exact evidence. // The engine's per-VM target lock still serializes the actual libvirt // mutations, while the worker's pre-dispatch policy check cancels a // stale queued tenant change once expiry enforcement has started. $nonBlockingActiveKinds = ['firewall_apply']; $nonBlockingUncertainKinds = ['action', 'firewall_apply']; if ($failSafeSuspend) { // A lost snapshot response must not permanently defeat quota or // expiry enforcement. The engine still rejects these target-state // suspends while a snapshot executor or libvirt BACKUP is active; // once neither is active, an old durable snapshot uncertainty is // not allowed to keep networking up. Purge shutdown remains // serialized because an uncertain restore reconciler may still // need to recover the VM's original power state. // Let the target-state request reach the engine even while PHP is // still polling a snapshot job. The engine may cancel only a // durably queued snapshot that has not touched libvirt; a running // backup remains protected and returns 409 for lifecycle retry. $nonBlockingActiveKinds = array_merge( $nonBlockingActiveKinds, $snapshotKinds ); $nonBlockingUncertainKinds = array_merge( $nonBlockingUncertainKinds, $snapshotKinds ); } return CloudJob::where('instance_id', $instance->id) ->where(function ($query) use ($nonBlockingActiveKinds, $nonBlockingUncertainKinds) { $query->where(function ($active) use ($nonBlockingActiveKinds) { $active->whereIn('status', ['queued', 'running']); if ($nonBlockingActiveKinds !== []) { $active->whereNotIn('kind', $nonBlockingActiveKinds); } }) ->orWhere(function ($uncertain) use ($nonBlockingUncertainKinds) { // A reset/shutdown action does not change resource // ownership, so expiry policy may still converge around // an old uncertain action. Delete is different: until // the original destructive operation is reconciled it // must serialize every later lifecycle mutation. $uncertain->where('status', 'uncertain') ->whereNotIn('kind', $nonBlockingUncertainKinds); }); })->exists(); } private function policyRetryReady(CloudInstance $instance, string $kind, int $now): bool { $failed = CloudJob::where('instance_id', $instance->id) ->where('kind', $kind)->where('status', 'failed') ->orderByDesc('updated_at')->orderByDesc('id')->first(); if ($failed === null) { return true; } $recentFailures = CloudJob::where('instance_id', $instance->id) ->where('kind', $kind)->where('status', 'failed') ->where('updated_at', '>=', $now - 3600)->count(); // Retry temporary host/SSH failures without creating a 15-second job // storm. Enforcement always converges again after an administrator // repairs the host; the delay is capped at five minutes. $exponent = min(5, max(0, $recentFailures - 1)); $delay = min(300, 15 * (1 << $exponent)); return (int) $failed->updated_at + $delay <= $now; } private function updateUsage(CloudInstance $instance, CloudServicePeriod $period, int $now): bool { if ($period->download_base_bytes === null || $period->upload_base_bytes === null) { // At a period boundary, an old cached total would charge bytes // from the previous period to the new one. Wait for the first // successful sample at/after the boundary and fail open until it // establishes both baselines. if ((int) $instance->traffic_synced_at < (int) $period->starts_at) { $period->download_base_bytes = null; $period->upload_base_bytes = null; $period->download_used_bytes = 0; $period->upload_used_bytes = 0; $period->updated_at = $now; $period->save(); $instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes); $instance->traffic_download_bytes = 0; $instance->traffic_upload_bytes = 0; return false; } $period->download_base_bytes = max(0, (int) $instance->lifetime_download_bytes); $period->upload_base_bytes = max(0, (int) $instance->lifetime_upload_bytes); } $download = max(0, (int) $instance->lifetime_download_bytes - (int) $period->download_base_bytes); $upload = max(0, (int) $instance->lifetime_upload_bytes - (int) $period->upload_base_bytes); $period->download_used_bytes = $download; $period->upload_used_bytes = $upload; $period->updated_at = $now; $period->save(); $instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes); $instance->traffic_download_bytes = $download; $instance->traffic_upload_bytes = $upload; return true; } private function enqueue(CloudInstance $instance, string $kind, string $suffix, ?string $reason = null, array $body = []): bool { $job = new CloudJob(); $job->operation_id = bin2hex(random_bytes(16)); $job->instance_id = $instance->id; $job->user_id = $instance->user_id; $job->host_id = $instance->host_id; $job->kind = $kind; $job->status = 'queued'; $job->payload_cipher = CloudSecrets::encrypt([ 'method' => 'POST', 'resource' => 'vms/' . rawurlencode($instance->uuid) . '/' . $suffix, 'body' => $body, 'owner_id' => (int) $instance->user_id, 'system_policy' => true, 'suspend_reason' => $reason, ]); $job->available_at = $job->created_at = $job->updated_at = time(); $job->save(); return true; } }