You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
969 lines
42 KiB
PHP
969 lines
42 KiB
PHP
<?php
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInvoice;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudServicePeriod;
|
|
|
|
final class CloudLifecycleService
|
|
{
|
|
private const SAFE_NEW_GENERATION_DELAY = 300;
|
|
|
|
private array $observedErrors = [];
|
|
private int $templateReconcileAfter = 0;
|
|
private int $templateReconcileCursor = 0;
|
|
private int $rescueReconcileCursor = 0;
|
|
private int $trafficUsageScanCursor = 0;
|
|
private int $trafficPeriodScanCursor = 0;
|
|
private int $trafficCheckpointRetentionCursor = 0;
|
|
|
|
public function __construct(private ?CloudRuntimeHeartbeat $heartbeat = null)
|
|
{
|
|
}
|
|
|
|
public function observationErrors(): array
|
|
{
|
|
return array_keys($this->observedErrors);
|
|
}
|
|
|
|
private function observeFailure(string $code): void
|
|
{
|
|
$this->observedErrors[$code] = true;
|
|
$this->heartbeat?->error($code);
|
|
}
|
|
|
|
public function tick(int $limit = 200): int
|
|
{
|
|
$this->observedErrors = [];
|
|
$batchSize = max(1, min(1000, $limit));
|
|
$this->heartbeat?->busy('provision_expiry');
|
|
try {
|
|
CloudInvoiceService::expireProvision($batchSize);
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('provision_expiry_failed');
|
|
// Releasing an abandoned provisioning reservation is database-only
|
|
// and must not stop instance expiry or traffic enforcement.
|
|
error_log('cloud provisioning invoice expiry unavailable: '
|
|
. get_class($error));
|
|
}
|
|
$this->heartbeat?->busy('invoice_generation');
|
|
try {
|
|
CloudInvoiceService::issueDue($batchSize);
|
|
foreach (CloudInvoiceService::observationErrors('issue_due') as $code) {
|
|
$this->observeFailure($code);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('invoice_generation_failed');
|
|
// Invoice creation is database-only and must not stop policy.
|
|
error_log('cloud invoice generation unavailable: ' . get_class($error));
|
|
}
|
|
$queued = 0;
|
|
$automaticLimit = max(1, min(10, $batchSize));
|
|
$this->heartbeat?->busy('automatic_backup_schedule');
|
|
try {
|
|
$queued += CloudAutomaticBackupService::scheduleDue(
|
|
$automaticLimit
|
|
);
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('automatic_backup_schedule_failed');
|
|
error_log('cloud automatic backup schedule unavailable: '
|
|
. get_class($error));
|
|
}
|
|
$this->heartbeat?->busy('automatic_backup_retention');
|
|
try {
|
|
CloudAutomaticBackupService::pruneRetention($automaticLimit);
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('automatic_backup_retention_failed');
|
|
error_log('cloud automatic backup retention unavailable: '
|
|
. get_class($error));
|
|
}
|
|
$this->heartbeat?->busy('template_artifact_cleanup');
|
|
try {
|
|
$cleanup = CloudTemplateDeletionService::processDue(
|
|
min(3, $automaticLimit)
|
|
);
|
|
$queued += (int) $cleanup['completed'];
|
|
if ((int) $cleanup['errors'] > 0
|
|
|| (int) $cleanup['attention'] > 0) {
|
|
$this->observeFailure('template_artifact_cleanup_failed');
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('template_artifact_cleanup_failed');
|
|
error_log('cloud template artifact cleanup unavailable: '
|
|
. get_class($error));
|
|
}
|
|
$now = time();
|
|
if ($now >= $this->templateReconcileAfter) {
|
|
$this->templateReconcileAfter = $now + 300;
|
|
try {
|
|
$actor = CloudDaemonActor::resolve(
|
|
'cloud-template-reconciliation'
|
|
);
|
|
} catch (\Throwable $error) {
|
|
// An unbootstrapped configuration cannot authenticate a
|
|
// read-only host request. Skip this bounded pass; the normal
|
|
// lifecycle stages still retain their own observations.
|
|
$actor = null;
|
|
}
|
|
if ($actor !== null) {
|
|
$this->heartbeat?->busy('template_reconcile');
|
|
try {
|
|
$report = (new CloudTemplateReconciliationService())
|
|
->reconcile(
|
|
$actor,
|
|
min(4, $automaticLimit),
|
|
$automaticLimit,
|
|
$this->templateReconcileCursor
|
|
);
|
|
$this->templateReconcileCursor =
|
|
(int) $report['last_host_id'];
|
|
$queued += (int) $report['scheduled'];
|
|
if ((int) $report['errors'] > 0) {
|
|
$this->observeFailure(
|
|
'template_reconcile_failed'
|
|
);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('template_reconcile_failed');
|
|
error_log('cloud template reconciliation unavailable: '
|
|
. get_class($error));
|
|
}
|
|
$this->heartbeat?->busy('rescue_template_reconcile');
|
|
try {
|
|
$report = (new
|
|
CloudRescueTemplateReconciliationService())
|
|
->reconcile(
|
|
$actor,
|
|
min(4, $automaticLimit),
|
|
$automaticLimit,
|
|
$this->rescueReconcileCursor
|
|
);
|
|
$this->rescueReconcileCursor =
|
|
(int) $report['last_host_id'];
|
|
$queued += (int) $report['scheduled'];
|
|
if ((int) $report['errors'] > 0) {
|
|
$this->observeFailure(
|
|
'rescue_template_reconcile_failed'
|
|
);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure(
|
|
'rescue_template_reconcile_failed'
|
|
);
|
|
error_log(
|
|
'cloud rescue template reconciliation unavailable: '
|
|
. get_class($error)
|
|
);
|
|
}
|
|
$this->heartbeat?->busy('rescue_template_cleanup');
|
|
try {
|
|
$cleanup = CloudRescueTemplateRetirementService::
|
|
processDue($actor, $automaticLimit);
|
|
$queued += (int) $cleanup['scheduled']
|
|
+ (int) $cleanup['completed'];
|
|
if ((int) $cleanup['errors'] > 0
|
|
|| (int) $cleanup['attention'] > 0) {
|
|
$this->observeFailure(
|
|
'rescue_template_cleanup_failed'
|
|
);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure(
|
|
'rescue_template_cleanup_failed'
|
|
);
|
|
error_log(
|
|
'cloud rescue template cleanup unavailable: '
|
|
. get_class($error)
|
|
);
|
|
}
|
|
}
|
|
}
|
|
$this->heartbeat?->busy('vpc_deleted_instance_cleanup');
|
|
try {
|
|
$queued += CloudVpcPortService::retryDeletedCleanupDue(
|
|
$automaticLimit
|
|
);
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('vpc_deleted_instance_cleanup_failed');
|
|
error_log('cloud VPC deleted-instance cleanup unavailable: '
|
|
. get_class($error));
|
|
}
|
|
$this->heartbeat?->busy('traffic_sync');
|
|
try {
|
|
$collection = CloudTrafficService::collect($batchSize);
|
|
foreach (array_unique($collection['errors'] ?? []) as $code) {
|
|
$this->observeFailure((string) $code);
|
|
}
|
|
CloudTrafficService::sync($batchSize);
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('traffic_sync_failed');
|
|
// Unknown traffic is fail-open. Time expiry remains enforceable.
|
|
error_log('cloud traffic sync unavailable: ' . get_class($error));
|
|
}
|
|
// Shared-pool accounting consumes only counters already persisted by
|
|
// the traffic sync above. It deliberately runs outside every
|
|
// per-instance lifecycle transaction: pool accounting owns the
|
|
// pool -> cursor -> entitlement lock order, while network policy
|
|
// remains on the existing instance reconciliation path until the
|
|
// restriction/convergence layer is enabled.
|
|
$this->heartbeat?->busy('traffic_pool_accounting');
|
|
try {
|
|
$settled = CloudTrafficPoolAccounting::settlePendingUsage(
|
|
$batchSize,
|
|
$this->trafficUsageScanCursor
|
|
);
|
|
$this->trafficUsageScanCursor = max(
|
|
0,
|
|
(int) ($settled['next_cursor'] ?? 0)
|
|
);
|
|
$due = CloudTrafficPoolAccounting::reconcileDue(
|
|
$batchSize,
|
|
null,
|
|
$this->trafficPeriodScanCursor
|
|
);
|
|
$this->trafficPeriodScanCursor = max(
|
|
0,
|
|
(int) ($due['next_cursor'] ?? 0)
|
|
);
|
|
if (($settled['errors'] ?? []) !== []
|
|
|| ($due['errors'] ?? []) !== []) {
|
|
$this->observeFailure('traffic_pool_accounting_failed');
|
|
error_log('cloud traffic-pool accounting had bounded errors');
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('traffic_pool_accounting_failed');
|
|
// A projection failure must never become a guessed network
|
|
// decision. Existing instance policy remains authoritative.
|
|
error_log('cloud traffic-pool accounting unavailable: '
|
|
. get_class($error));
|
|
}
|
|
$this->heartbeat?->busy('traffic_settlement_compaction');
|
|
try {
|
|
$compaction = CloudTrafficSettlementRetention::compact(
|
|
$batchSize,
|
|
time()
|
|
);
|
|
if ($compaction['blocked'] !== []) {
|
|
$this->observeFailure('traffic_retention_failed');
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('traffic_retention_failed');
|
|
// Retention is never allowed to block current accounting or
|
|
// instance policy. Full buckets remain intact for a later retry.
|
|
error_log('cloud traffic settlement retention unavailable: '
|
|
. get_class($error));
|
|
}
|
|
$this->heartbeat?->busy('traffic_checkpoint_compaction');
|
|
try {
|
|
$checkpointRetention = CloudTrafficCheckpointRetention::prune(
|
|
$batchSize,
|
|
time(),
|
|
$this->trafficCheckpointRetentionCursor
|
|
);
|
|
$this->trafficCheckpointRetentionCursor = max(
|
|
0,
|
|
(int) ($checkpointRetention['next_cursor'] ?? 0)
|
|
);
|
|
if ($checkpointRetention['blocked'] !== []) {
|
|
$this->observeFailure('traffic_retention_failed');
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('traffic_retention_failed');
|
|
// Hourly history is retained on every ambiguity; compaction must
|
|
// never gate live accounting, billing or instance policy.
|
|
error_log('cloud traffic checkpoint retention unavailable: '
|
|
. get_class($error));
|
|
}
|
|
// Power state is an observation cache, not a billing decision. Each
|
|
// host is contacted through its own Agent settings, so one unavailable
|
|
// host must not gate database-only billing or other hosts.
|
|
$this->heartbeat?->busy('runtime_status_sync');
|
|
try {
|
|
$runtime = (new CloudInstanceRuntimeSyncService())->sync($batchSize);
|
|
foreach (array_unique($runtime['errors'] ?? []) as $code) {
|
|
$this->observeFailure((string) $code);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('runtime_status_sync_failed');
|
|
error_log('cloud runtime status sync unavailable: ' . get_class($error));
|
|
}
|
|
$this->heartbeat?->busy('instance_reconcile');
|
|
CloudInstance::where('status', '<>', 'deleted')
|
|
->where('service_status', '<>', 'exempt')
|
|
->select('id')->chunkById($batchSize, function ($instances) use (&$queued) {
|
|
foreach ($instances as $instance) {
|
|
$this->heartbeat?->busy('instance_reconcile');
|
|
try {
|
|
$queued += $this->reconcile((int) $instance->id) ? 1 : 0;
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('instance_reconcile_failed');
|
|
// One corrupt or temporarily contended record must not
|
|
// starve every higher instance ID on every daemon pass.
|
|
error_log('cloud lifecycle instance ' . (int) $instance->id
|
|
. ' unavailable: ' . get_class($error));
|
|
}
|
|
}
|
|
});
|
|
return $queued;
|
|
}
|
|
|
|
private function reconcile(int $instanceId): bool
|
|
{
|
|
return (new CloudInstance())->getConnection()->transaction(function () use ($instanceId) {
|
|
$now = time();
|
|
$instance = CloudInstance::where('id', $instanceId)->lockForUpdate()->first();
|
|
if ($instance === null || (string) $instance->status === 'deleted'
|
|
|| $instance->service_status === 'exempt') {
|
|
return false;
|
|
}
|
|
// A paid grow-only resize owns the instance, current service
|
|
// period and renewal snapshot as one recovery unit. In
|
|
// particular, an unknown-effect resize may cross the expiry
|
|
// boundary while its original operation is being inspected.
|
|
// Freeze lifecycle projection before finishing that period or
|
|
// clearing current_period_id; the upgrade finalizer will then
|
|
// either apply the immutable target or restore the source ledger.
|
|
if (CloudJobService::blockingQueryForInstance(
|
|
(int) $instance->id
|
|
)->where('kind', 'instance_upgrade')->exists()) {
|
|
return false;
|
|
}
|
|
if ((string) $instance->status === 'delete_failed'
|
|
|| (string) $instance->service_status === 'purge_failed') {
|
|
if ((string) $instance->status !== 'delete_failed'
|
|
|| (string) $instance->service_status !== 'purge_failed'
|
|
|| $this->retryableNotAppliedJobLocked(
|
|
$instance,
|
|
'expiry_delete',
|
|
$now
|
|
) === null) {
|
|
return false;
|
|
}
|
|
$instance->status = 'deleting';
|
|
$instance->service_status = 'purge_deleting';
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
CloudJobService::enqueueExpiryDelete($instance);
|
|
return true;
|
|
}
|
|
if ((string) $instance->status === 'deleting'
|
|
&& (string) $instance->service_status === 'purge_stop_failed') {
|
|
if ($this->retryableNotAppliedJobLocked(
|
|
$instance,
|
|
'service_suspend',
|
|
$now
|
|
) === null) {
|
|
return false;
|
|
}
|
|
$instance->service_status = 'purge_stopping';
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
return $this->enqueue(
|
|
$instance,
|
|
'service_suspend',
|
|
'service/suspend',
|
|
'purge',
|
|
['reason' => 'purge']
|
|
);
|
|
}
|
|
if ((string) $instance->service_status === 'purge_stop_failed') {
|
|
return false;
|
|
}
|
|
// A delete with a lost response is already a terminal failed task,
|
|
// but its resource fence remains: the domain and disks may already
|
|
// be gone. Freeze later mutations until a super administrator uses
|
|
// a fresh read-only inspection to resolve that exact operation's
|
|
// fence. Resolution never reopens or replays the original task.
|
|
if (CloudJobService::blockingQueryForInstance((int) $instance->id)
|
|
->whereIn('kind',
|
|
['delete', 'expiry_delete', 'delete_force'])->exists()) {
|
|
return false;
|
|
}
|
|
$period = $instance->current_period_id === null ? null
|
|
: CloudServicePeriod::where('id', $instance->current_period_id)
|
|
->where('instance_id', $instance->id)->lockForUpdate()->first();
|
|
$policyPeriod = $period;
|
|
|
|
if ($period !== null && in_array((string) $period->status, ['active', 'exhausted'], true)) {
|
|
$this->updateUsage($instance, $period, $now);
|
|
if ((int) $period->ends_at <= $now) {
|
|
$period->status = 'finished';
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
$period = null;
|
|
$instance->current_period_id = null;
|
|
}
|
|
} else {
|
|
$period = null;
|
|
$instance->current_period_id = null;
|
|
}
|
|
|
|
if ($period === null) {
|
|
// The lifecycle daemon may be stopped for longer than one
|
|
// prepaid period. Never briefly activate an already expired
|
|
// historical period (which would also restore its network).
|
|
CloudServicePeriod::where('instance_id', $instance->id)
|
|
->where('status', 'scheduled')->where('ends_at', '<=', $now)
|
|
->update(['status' => 'finished', 'updated_at' => $now]);
|
|
$period = CloudServicePeriod::where('instance_id', $instance->id)
|
|
->where('status', 'scheduled')->where('starts_at', '<=', $now)
|
|
->where('ends_at', '>', $now)
|
|
->orderBy('starts_at')->orderBy('id')->lockForUpdate()->first();
|
|
if ($period !== null) {
|
|
$period->status = 'active';
|
|
$period->activated_at = $now;
|
|
$baseline = CloudTrafficService::periodBaseline(
|
|
$instance,
|
|
(int) $period->starts_at,
|
|
(int) $period->id
|
|
);
|
|
if ($baseline === null
|
|
&& (int) $instance->traffic_synced_at >= (int) $period->starts_at) {
|
|
$baseline = [
|
|
'download' => max(0, (int) $instance->lifetime_download_bytes),
|
|
'upload' => max(0, (int) $instance->lifetime_upload_bytes),
|
|
];
|
|
}
|
|
$period->download_base_bytes = $baseline['download'] ?? null;
|
|
$period->upload_base_bytes = $baseline['upload'] ?? null;
|
|
$period->download_used_bytes = $baseline === null ? 0
|
|
: max(0, (int) $instance->lifetime_download_bytes
|
|
- (int) $baseline['download']);
|
|
$period->upload_used_bytes = $baseline === null ? 0
|
|
: max(0, (int) $instance->lifetime_upload_bytes
|
|
- (int) $baseline['upload']);
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
$instance->current_period_id = $period->id;
|
|
$instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes);
|
|
$instance->traffic_download_bytes = (int) $period->download_used_bytes;
|
|
$instance->traffic_upload_bytes = (int) $period->upload_used_bytes;
|
|
$policyPeriod = $period;
|
|
$aggregateNetworkPolicy =
|
|
CloudInstanceNetworkPolicy::isEnforcedPeriod($period)
|
|
|| CloudInstanceNetworkPolicy::hasState(
|
|
(int) $instance->id
|
|
);
|
|
$this->convergePaidPeriodProjectionLocked(
|
|
$instance,
|
|
$now,
|
|
!$aggregateNetworkPolicy
|
|
);
|
|
}
|
|
}
|
|
|
|
$aggregateNetworkPolicy =
|
|
CloudInstanceNetworkPolicy::isEnforcedPeriod(
|
|
$period ?? $policyPeriod
|
|
) || CloudInstanceNetworkPolicy::hasState((int) $instance->id);
|
|
if ($period !== null) {
|
|
$usageKnown = $this->updateUsage($instance, $period, $now);
|
|
$used = (int) $period->download_used_bytes + (int) $period->upload_used_bytes;
|
|
$quota = max(0, (int) $period->traffic_quota_bytes);
|
|
if (!$aggregateNetworkPolicy
|
|
&& $usageKnown && $quota > 0 && $used >= $quota) {
|
|
if ((string) $period->status !== 'exhausted') {
|
|
$period->status = 'exhausted';
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
}
|
|
if (!in_array((string) $instance->network_status,
|
|
['blocking', 'blocked_traffic', 'block_failed'], true)) {
|
|
$instance->network_status = 'blocking';
|
|
}
|
|
} elseif ($usageKnown) {
|
|
$this->convergePaidPeriodProjectionLocked(
|
|
$instance,
|
|
$now,
|
|
!$aggregateNetworkPolicy
|
|
);
|
|
}
|
|
} elseif ($instance->expires_at !== null && (int) $instance->expires_at <= $now) {
|
|
$invoice = CloudInvoice::where('instance_id', $instance->id)
|
|
->where('renews_expires_at', $instance->expires_at)
|
|
->where('status', '<>', 'void')->orderByDesc('id')
|
|
->lockForUpdate()->first();
|
|
if ($invoice === null) {
|
|
// A missing invoice is an observable billing anomaly, not
|
|
// an unlimited service extension. The immutable instance
|
|
// expiry remains a safe fallback grace anchor.
|
|
$this->observeFailure('renewal_invoice_missing');
|
|
$pastGrace = (int) $instance->expires_at
|
|
<= $now - CloudInvoiceService::PAYMENT_GRACE_SECONDS;
|
|
} else {
|
|
$pastGrace = in_array((string) $invoice->status,
|
|
['open', 'overdue', 'expired'], true)
|
|
&& CloudInvoiceService::paymentDeadlinePassed(
|
|
$invoice,
|
|
$now
|
|
);
|
|
}
|
|
// A maintenance migration can temporarily retain two host
|
|
// identities and two locked address reservations. Never let
|
|
// the ordinary expiry deleter remove only the current target
|
|
// while leaving the fenced source behind. Network expiry still
|
|
// converges below; permanent purge resumes after migration
|
|
// cleanup reaches a terminal state.
|
|
if ($pastGrace
|
|
&& CloudInstanceMigrationService::hasOpenMigration(
|
|
(int) $instance->id
|
|
)) {
|
|
$pastGrace = false;
|
|
}
|
|
if ($pastGrace) {
|
|
if ($invoice !== null && in_array(
|
|
(string) $invoice->status,
|
|
['open', 'overdue'],
|
|
true
|
|
)) {
|
|
$invoice->status = 'expired';
|
|
$invoice->updated_at = $now;
|
|
$invoice->save();
|
|
}
|
|
$instance->status = 'deleting';
|
|
if (!in_array((string) $instance->service_status,
|
|
['purge_stopping', 'purge_force_required',
|
|
'purge_deleting', 'purge_failed',
|
|
'purge_stop_failed'], true)) {
|
|
$instance->service_status = 'purge_stopping';
|
|
}
|
|
} elseif (!in_array((string) $instance->service_status,
|
|
['purge_stopping', 'purge_stop_failed',
|
|
'purge_force_required', 'purge_deleting',
|
|
'purge_failed'], true)) {
|
|
// Expiry keeps QEMU running throughout the three-day
|
|
// payment grace period. Only its NIC links are persisted
|
|
// down; a paid renewal restores those links. During an
|
|
$instance->service_status = 'expired_grace';
|
|
$instance->suspended_at = null;
|
|
if (!$aggregateNetworkPolicy) {
|
|
$network = (string) $instance->network_status;
|
|
if ($network === 'blocked_traffic') {
|
|
$instance->network_status = 'blocked_expired';
|
|
} elseif (!in_array($network,
|
|
['blocking', 'blocking_expired', 'blocked_expired',
|
|
'block_failed'], true)) {
|
|
$instance->network_status = 'blocking_expired';
|
|
}
|
|
}
|
|
}
|
|
}
|
|
$networkPolicy = null;
|
|
if ($aggregateNetworkPolicy) {
|
|
$networkPolicy = CloudInstanceNetworkPolicy::synchronizeLocked(
|
|
$instance,
|
|
$period ?? $policyPeriod,
|
|
$now
|
|
);
|
|
}
|
|
$policyDeferred = ($networkPolicy['deferred'] ?? false) === true;
|
|
$network = (string) $instance->network_status;
|
|
if ($aggregateNetworkPolicy && !$policyDeferred
|
|
&& ($networkPolicy['state'] ?? null) === 'failed') {
|
|
$retryKind = ($networkPolicy['desired_suspended'] ?? false)
|
|
=== true && ($networkPolicy['applied_suspended'] ?? true)
|
|
=== false ? 'network_suspend'
|
|
: ((($networkPolicy['desired_suspended'] ?? true) === false
|
|
&& ($networkPolicy['applied_suspended'] ?? false) === true)
|
|
? 'network_resume' : null);
|
|
if ($retryKind !== null
|
|
&& $this->retryableNotAppliedJobLocked(
|
|
$instance,
|
|
$retryKind,
|
|
$now
|
|
) !== null
|
|
&& CloudInstanceNetworkPolicy::allowRetryLocked(
|
|
$instance,
|
|
$retryKind,
|
|
$now
|
|
)) {
|
|
$networkPolicy =
|
|
CloudInstanceNetworkPolicy::snapshotLocked(
|
|
$instance,
|
|
$now
|
|
);
|
|
}
|
|
} elseif ((!$aggregateNetworkPolicy || $policyDeferred)
|
|
&& $network === 'block_failed') {
|
|
$reason = null;
|
|
$nextState = null;
|
|
if ($period !== null
|
|
&& (string) $period->status === 'exhausted') {
|
|
$reason = 'traffic';
|
|
$nextState = 'blocking';
|
|
} elseif ($period === null
|
|
&& (string) $instance->service_status === 'expired_grace'
|
|
&& $instance->expires_at !== null
|
|
&& (int) $instance->expires_at <= $now) {
|
|
$reason = 'expired';
|
|
$nextState = 'blocking_expired';
|
|
}
|
|
if ($reason !== null
|
|
&& $this->retryableNotAppliedJobLocked(
|
|
$instance,
|
|
'network_suspend',
|
|
$now
|
|
) !== null) {
|
|
$instance->network_status = $nextState;
|
|
}
|
|
} elseif ((!$aggregateNetworkPolicy || $policyDeferred)
|
|
&& $network === 'restore_failed'
|
|
&& $period !== null
|
|
&& (string) $period->status === 'active'
|
|
&& $this->retryableNotAppliedJobLocked(
|
|
$instance,
|
|
'network_resume',
|
|
$now
|
|
) !== null) {
|
|
$instance->network_status = 'restore_pending';
|
|
$instance->network_suspended_at = null;
|
|
}
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
|
|
if ($this->hasConflictingJob($instance)) {
|
|
return false;
|
|
}
|
|
|
|
$service = (string) $instance->service_status;
|
|
$network = (string) $instance->network_status;
|
|
if ($service === 'purge_stopping') {
|
|
return $this->enqueue($instance, 'service_suspend',
|
|
'service/suspend', 'purge', ['reason' => 'purge']);
|
|
}
|
|
if ($service === 'purge_deleting') {
|
|
$instance->status = 'deleting';
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
CloudJobService::enqueueExpiryDelete($instance);
|
|
return true;
|
|
}
|
|
if ($aggregateNetworkPolicy && !$policyDeferred) {
|
|
$action = (string) ($networkPolicy['action'] ?? 'none');
|
|
if ($action === 'suspend') {
|
|
return $this->enqueue(
|
|
$instance,
|
|
'network_suspend',
|
|
'network/suspend',
|
|
(string) ($networkPolicy['suspend_reason'] ?? 'traffic'),
|
|
[],
|
|
true
|
|
);
|
|
}
|
|
if ($action === 'resume') {
|
|
$instance->network_status = 'restoring';
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
return $this->enqueue(
|
|
$instance,
|
|
'network_resume',
|
|
'network/resume',
|
|
null,
|
|
[],
|
|
true
|
|
);
|
|
}
|
|
return false;
|
|
}
|
|
if ($network === 'blocking_expired') {
|
|
return $this->enqueue($instance, 'network_suspend',
|
|
'network/suspend', 'expired', ['reason' => 'expired']);
|
|
}
|
|
if ($network === 'blocking') {
|
|
return $this->enqueue($instance, 'network_suspend',
|
|
'network/suspend', 'traffic', ['reason' => 'traffic']);
|
|
}
|
|
if (in_array($network, ['restore_pending', 'restoring'], true)) {
|
|
$instance->network_status = 'restoring';
|
|
$instance->save();
|
|
return $this->enqueue($instance, 'network_resume',
|
|
'network/resume');
|
|
}
|
|
return false;
|
|
}, 3);
|
|
}
|
|
|
|
private function hasConflictingJob(CloudInstance $instance): bool
|
|
{
|
|
return CloudJobService::blockingQueryForInstance(
|
|
(int) $instance->id
|
|
)->exists();
|
|
}
|
|
|
|
/**
|
|
* Bring a paid period's local projection forward without overwriting an
|
|
* in-flight or uncertain policy mutation. Definitively applied network
|
|
* suspension can start a resume; definitively not-applied failures may be
|
|
* cleared or moved to a new generation by the strict ledger helper.
|
|
*/
|
|
private function convergePaidPeriodProjectionLocked(
|
|
CloudInstance $instance,
|
|
int $now,
|
|
bool $convergeNetwork = true
|
|
): void {
|
|
$service = (string) $instance->service_status;
|
|
if ($service === 'suspend_failed') {
|
|
if ($this->retryableNotAppliedJobLocked(
|
|
$instance,
|
|
'service_suspend',
|
|
$now
|
|
) !== null) {
|
|
$instance->service_status = 'active';
|
|
$instance->suspended_at = null;
|
|
}
|
|
} elseif (!in_array($service, [
|
|
'active',
|
|
'exempt',
|
|
'suspending_expired',
|
|
'purge_stopping',
|
|
'purge_stop_failed',
|
|
'purge_force_required',
|
|
'purge_deleting',
|
|
'purge_failed',
|
|
], true)) {
|
|
$instance->service_status = 'active';
|
|
$instance->suspended_at = null;
|
|
}
|
|
|
|
if (!$convergeNetwork) {
|
|
return;
|
|
}
|
|
$network = (string) $instance->network_status;
|
|
if ($network === 'block_failed') {
|
|
if ($this->retryableNotAppliedJobLocked(
|
|
$instance,
|
|
'network_suspend',
|
|
$now
|
|
) !== null) {
|
|
// A proven no-effect suspension means the links never left
|
|
// their paid-period target state; no compensating mutation is
|
|
// required.
|
|
$instance->network_status = 'active';
|
|
$instance->network_suspended_at = null;
|
|
}
|
|
} elseif ($network === 'restore_failed') {
|
|
if ($this->retryableNotAppliedJobLocked(
|
|
$instance,
|
|
'network_resume',
|
|
$now
|
|
) !== null) {
|
|
$instance->network_status = 'restore_pending';
|
|
$instance->network_suspended_at = null;
|
|
}
|
|
} elseif (!in_array($network, [
|
|
'active',
|
|
'blocking',
|
|
'blocking_expired',
|
|
'restore_pending',
|
|
'restoring',
|
|
], true)) {
|
|
$instance->network_status = 'restore_pending';
|
|
$instance->network_suspended_at = null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* A policy retry is a new audited generation, never a replay. It is
|
|
* allowed only after the newest generation proved that no effect happened
|
|
* and all immutable Agent and resource bindings still match.
|
|
*/
|
|
private function retryableNotAppliedJobLocked(
|
|
CloudInstance $instance,
|
|
string $kind,
|
|
int $now
|
|
): ?CloudJob {
|
|
$job = CloudJob::where('instance_id', (int) $instance->id)
|
|
->where('kind', $kind)
|
|
->orderByDesc('id')
|
|
->lockForUpdate()
|
|
->first();
|
|
if ($job === null) {
|
|
return null;
|
|
}
|
|
if ((bool) $job->finalization_incident) {
|
|
return null;
|
|
}
|
|
if ((string) $job->execution_mode !== 'agent'
|
|
|| $job->parent_job_id !== null
|
|
|| (int) $job->user_id !== (int) $instance->user_id
|
|
|| (int) $job->host_id !== (int) $instance->host_id) {
|
|
$this->markLifecycleRecoveryIncident(
|
|
$job,
|
|
'recovery_instance_state_conflict',
|
|
$now
|
|
);
|
|
return null;
|
|
}
|
|
if ((string) $job->phase !== 'terminal') {
|
|
return null;
|
|
}
|
|
if (CloudDispatchLedger::isInvalid($job)
|
|
|| CloudJobService::hasTerminalRecoveryStateConflict($job)) {
|
|
$this->markLifecycleRecoveryIncident(
|
|
$job,
|
|
CloudDispatchLedger::isInvalid($job)
|
|
? 'recovery_dispatch_ledger_conflict'
|
|
: 'recovery_instance_state_conflict',
|
|
$now
|
|
);
|
|
return null;
|
|
}
|
|
if ((string) $job->outcome !== 'failed'
|
|
|| (string) $job->status !== 'failed'
|
|
|| (string) $job->effect_resolution !== 'not_applied'
|
|
|| (bool) $job->effect_may_have_applied
|
|
|| (string) ($job->error_code ?? '') === 'force_required') {
|
|
if ((string) $job->effect_resolution === 'applied'
|
|
|| (string) ($job->error_code ?? '') === 'force_required') {
|
|
// The resource is still in a failed policy projection even
|
|
// though the immutable job says the mutation happened (or
|
|
// requires a different authorized action). Keep that
|
|
// contradiction visible to the PHP-only recovery queue.
|
|
$this->markLifecycleRecoveryIncident(
|
|
$job,
|
|
'recovery_instance_state_conflict',
|
|
$now
|
|
);
|
|
}
|
|
return null;
|
|
}
|
|
if (max(
|
|
(int) ($job->terminal_at ?? 0),
|
|
(int) ($job->updated_at ?? 0)
|
|
) > $now - self::SAFE_NEW_GENERATION_DELAY) {
|
|
return null;
|
|
}
|
|
try {
|
|
$identity = CloudOperationRecoveryIdentity::expected($job);
|
|
} catch (\Throwable $error) {
|
|
$code = match (true) {
|
|
$error instanceof CloudRecoveryPayloadUnavailable =>
|
|
'recovery_payload_unavailable',
|
|
$error instanceof CloudRecoveryOperationIdentityConflict =>
|
|
'recovery_operation_identity_conflict',
|
|
$error instanceof CloudRecoveryInstanceStateConflict =>
|
|
'recovery_instance_state_conflict',
|
|
default => 'recovery_internal_error',
|
|
};
|
|
$this->markLifecycleRecoveryIncident($job, $code, $now);
|
|
return null;
|
|
}
|
|
$expectedAction = [
|
|
'expiry_delete' => 'delete-vm',
|
|
'service_suspend' => 'service-suspend',
|
|
'network_suspend' => 'network-suspend',
|
|
'network_resume' => 'network-resume',
|
|
][$kind] ?? null;
|
|
if ($expectedAction === null
|
|
|| ($identity['action'] ?? null) !== $expectedAction
|
|
|| ($identity['resource_id'] ?? null)
|
|
!== strtolower((string) $instance->uuid)
|
|
|| !is_string($identity['request_fingerprint'] ?? null)
|
|
|| preg_match(
|
|
'/^[a-f0-9]{64}$/D',
|
|
$identity['request_fingerprint']
|
|
) !== 1) {
|
|
$this->markLifecycleRecoveryIncident(
|
|
$job,
|
|
'recovery_operation_identity_conflict',
|
|
$now
|
|
);
|
|
return null;
|
|
}
|
|
return $job;
|
|
}
|
|
|
|
private function markLifecycleRecoveryIncident(
|
|
CloudJob $job,
|
|
string $code,
|
|
int $now
|
|
): void {
|
|
if ((bool) $job->finalization_incident) {
|
|
return;
|
|
}
|
|
$job->finalization_incident = 1;
|
|
$job->finalization_error_code = $code;
|
|
$job->lease_token = null;
|
|
$job->lease_until = 0;
|
|
$job->available_at = $now + self::SAFE_NEW_GENERATION_DELAY;
|
|
$job->updated_at = $now;
|
|
$job->save();
|
|
}
|
|
|
|
private function updateUsage(CloudInstance $instance, CloudServicePeriod $period, int $now): bool
|
|
{
|
|
if ($period->download_base_bytes === null || $period->upload_base_bytes === null) {
|
|
// At a period boundary, an old cached total would charge bytes
|
|
// from the previous period to the new one. Wait for the first
|
|
// successful sample at/after the boundary and fail open until it
|
|
// establishes both baselines.
|
|
if ((int) $instance->traffic_synced_at < (int) $period->starts_at) {
|
|
$period->download_base_bytes = null;
|
|
$period->upload_base_bytes = null;
|
|
$period->download_used_bytes = 0;
|
|
$period->upload_used_bytes = 0;
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
$instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes);
|
|
$instance->traffic_download_bytes = 0;
|
|
$instance->traffic_upload_bytes = 0;
|
|
return false;
|
|
}
|
|
$period->download_base_bytes = max(0, (int) $instance->lifetime_download_bytes);
|
|
$period->upload_base_bytes = max(0, (int) $instance->lifetime_upload_bytes);
|
|
}
|
|
$download = max(0, (int) $instance->lifetime_download_bytes - (int) $period->download_base_bytes);
|
|
$upload = max(0, (int) $instance->lifetime_upload_bytes - (int) $period->upload_base_bytes);
|
|
$period->download_used_bytes = $download;
|
|
$period->upload_used_bytes = $upload;
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
$instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes);
|
|
$instance->traffic_download_bytes = $download;
|
|
$instance->traffic_upload_bytes = $upload;
|
|
return true;
|
|
}
|
|
|
|
private function enqueue(CloudInstance $instance, string $kind, string $suffix,
|
|
?string $reason = null, array $body = [],
|
|
bool $aggregateNetworkPolicy = false): bool
|
|
{
|
|
$job = new CloudJob();
|
|
$job->operation_id = bin2hex(random_bytes(16));
|
|
$networkPolicyMarker = null;
|
|
if ($aggregateNetworkPolicy) {
|
|
$networkPolicyMarker =
|
|
CloudInstanceNetworkPolicy::prepareJobLocked(
|
|
$instance,
|
|
$kind,
|
|
(string) $job->operation_id
|
|
);
|
|
if ($kind === 'network_suspend') {
|
|
$reason = (string) $networkPolicyMarker['suspend_reason'];
|
|
}
|
|
}
|
|
$job->instance_id = $instance->id;
|
|
$job->user_id = $instance->user_id;
|
|
$job->host_id = $instance->host_id;
|
|
$job->kind = $kind;
|
|
$job->execution_mode = 'agent';
|
|
$job->status = 'queued';
|
|
$payload = [
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/vms/' . rawurlencode($instance->uuid)
|
|
. '/' . $suffix,
|
|
'body' => $body, 'owner_id' => (int) $instance->user_id,
|
|
'system_policy' => true, 'suspend_reason' => $reason,
|
|
];
|
|
if ($networkPolicyMarker !== null) {
|
|
$payload['_network_policy'] = $networkPolicyMarker;
|
|
}
|
|
$job->sealed_payload = CloudSecrets::encrypt($payload, 'job-payload');
|
|
$job->available_at = $job->created_at = $job->updated_at = time();
|
|
$job->save();
|
|
return true;
|
|
}
|
|
|
|
}
|