From e35e93c8a6f3ffcb175ded240e162fb01fd7048f Mon Sep 17 00:00:00 2001 From: codex Date: Sat, 31 Oct 2026 23:48:32 +0800 Subject: [PATCH] feat(traffic): schedule starvation-safe shadow accounting --- web/app/Command/CloudSetup.php | 15 +- web/app/Models/CloudTrafficScanFailure.php | 18 ++ .../Services/Cloud/CloudLifecycleService.php | 39 ++++ .../Cloud/CloudTrafficPoolAccounting.php | 183 +++++++++++++++--- web/app/Services/UserDataDeletionService.php | 54 +++++- web/sql/cloud.sql | 19 ++ web/tests/CloudFreshSchemaContractTest.php | 2 +- .../CloudRescueCatalogSchemaContractTest.php | 2 +- web/tests/CloudRuntimeHeartbeatTest.php | 6 +- .../CloudTemplateImportIntentContractTest.php | 2 +- web/tests/CloudTrafficPoolAccountingTest.php | 165 ++++++++++++++++ ...fficPoolLifecycleSchedulerContractTest.php | 55 ++++++ .../CloudTrafficPoolSchemaContractTest.php | 20 +- web/tests/CloudVpcSchemaContractTest.php | 2 +- web/tests/ManualAccountDeletionSqliteTest.php | 11 ++ 15 files changed, 558 insertions(+), 35 deletions(-) create mode 100644 web/app/Models/CloudTrafficScanFailure.php create mode 100644 web/tests/CloudTrafficPoolLifecycleSchedulerContractTest.php diff --git a/web/app/Command/CloudSetup.php b/web/app/Command/CloudSetup.php index 53802d43..3ecb985a 100644 --- a/web/app/Command/CloudSetup.php +++ b/web/app/Command/CloudSetup.php @@ -8,7 +8,7 @@ use App\Services\Config; /** Initializes only the clean-break PHP business and Agent log schemas. */ final class CloudSetup { - private const SCHEMA_VERSION = 'go-agent-wss-v20-traffic-pool-ledger'; + private const SCHEMA_VERSION = 'go-agent-wss-v21-traffic-scan-backoff'; private const MAIN_TABLES = [ 'cloud_regions', 'cloud_traffic_pool_groups', 'cloud_network_lines', 'cloud_resource_pools', @@ -37,6 +37,7 @@ final class CloudSetup 'cloud_resource_reservations', 'cloud_service_periods', 'cloud_traffic_pools', 'cloud_traffic_entitlements', 'cloud_traffic_ledger_events', 'cloud_traffic_instance_cursors', + 'cloud_traffic_scan_failures', 'cloud_instance_network_restrictions', 'cloud_renewals', 'cloud_invoices', 'cloud_invoice_items', 'cloud_invoice_payments', @@ -297,6 +298,11 @@ final class CloudSetup 'last_observed_at', 'last_event_id', 'state', 'attention_code', 'version', 'updated_at', ], + 'cloud_traffic_scan_failures' => [ + 'id', 'scan_kind', 'subject_id', 'attempt_count', + 'last_error_code', 'retry_at', 'first_failed_at', + 'last_failed_at', + ], 'cloud_instance_network_restrictions' => [ 'instance_id', 'user_id', 'reason_code', 'desired_active', 'applied_active', 'revision', 'state', 'source_type', @@ -341,6 +347,9 @@ final class CloudSetup 'cloud_traffic_instance_cursors' => [ 'chk_cloud_traffic_cursor_state', ], + 'cloud_traffic_scan_failures' => [ + 'chk_cloud_traffic_scan_failure_kind', + ], 'cloud_instance_network_restrictions' => [ 'chk_cloud_network_restriction_reason', 'chk_cloud_network_restriction_state', @@ -448,6 +457,10 @@ final class CloudSetup 'PRIMARY', 'cloud_traffic_cursor_pool', 'cloud_traffic_cursor_owner', ], + 'cloud_traffic_scan_failures' => [ + 'PRIMARY', 'cloud_traffic_scan_failure_subject', + 'cloud_traffic_scan_failure_retry', + ], 'cloud_instance_network_restrictions' => [ 'PRIMARY', 'cloud_network_restriction_operation', 'cloud_network_restriction_state', diff --git a/web/app/Models/CloudTrafficScanFailure.php b/web/app/Models/CloudTrafficScanFailure.php new file mode 100644 index 00000000..c983a023 --- /dev/null +++ b/web/app/Models/CloudTrafficScanFailure.php @@ -0,0 +1,18 @@ + 'int', + 'subject_id' => 'int', + 'attempt_count' => 'int', + 'retry_at' => 'int', + 'first_failed_at' => 'int', + 'last_failed_at' => 'int', + ]; +} diff --git a/web/app/Services/Cloud/CloudLifecycleService.php b/web/app/Services/Cloud/CloudLifecycleService.php index 7a26a080..9c4bc418 100644 --- a/web/app/Services/Cloud/CloudLifecycleService.php +++ b/web/app/Services/Cloud/CloudLifecycleService.php @@ -14,6 +14,8 @@ final class CloudLifecycleService private int $templateReconcileAfter = 0; private int $templateReconcileCursor = 0; private int $rescueReconcileCursor = 0; + private int $trafficUsageScanCursor = 0; + private int $trafficPeriodScanCursor = 0; public function __construct(private ?CloudRuntimeHeartbeat $heartbeat = null) { @@ -198,6 +200,43 @@ final class CloudLifecycleService // Unknown traffic is fail-open. Time expiry remains enforceable. error_log('cloud traffic sync unavailable: ' . get_class($error)); } + // Shared-pool accounting consumes only counters already persisted by + // the traffic sync above. It deliberately runs outside every + // per-instance lifecycle transaction: pool accounting owns the + // pool -> cursor -> entitlement lock order, while network policy + // remains on the existing instance reconciliation path until the + // restriction/convergence layer is enabled. + $this->heartbeat?->busy('traffic_pool_accounting'); + try { + $settled = CloudTrafficPoolAccounting::settlePendingUsage( + $batchSize, + $this->trafficUsageScanCursor + ); + $this->trafficUsageScanCursor = max( + 0, + (int) ($settled['next_cursor'] ?? 0) + ); + $due = CloudTrafficPoolAccounting::reconcileDue( + $batchSize, + null, + $this->trafficPeriodScanCursor + ); + $this->trafficPeriodScanCursor = max( + 0, + (int) ($due['next_cursor'] ?? 0) + ); + if (($settled['errors'] ?? []) !== [] + || ($due['errors'] ?? []) !== []) { + $this->observeFailure('traffic_pool_accounting_failed'); + error_log('cloud traffic-pool accounting had bounded errors'); + } + } catch (\Throwable $error) { + $this->observeFailure('traffic_pool_accounting_failed'); + // A projection failure must never become a guessed network + // decision. Existing instance policy remains authoritative. + error_log('cloud traffic-pool accounting unavailable: ' + . get_class($error)); + } // Power state is an observation cache, not a billing decision. Each // host is contacted through its own Agent settings, so one unavailable // host must not gate database-only billing or other hosts. diff --git a/web/app/Services/Cloud/CloudTrafficPoolAccounting.php b/web/app/Services/Cloud/CloudTrafficPoolAccounting.php index b67d8c85..0f34f2f4 100644 --- a/web/app/Services/Cloud/CloudTrafficPoolAccounting.php +++ b/web/app/Services/Cloud/CloudTrafficPoolAccounting.php @@ -9,6 +9,7 @@ use App\Models\CloudTrafficInstanceCursor; use App\Models\CloudTrafficLedgerEvent; use App\Models\CloudTrafficPool; use App\Models\CloudTrafficPoolGroup; +use App\Models\CloudTrafficScanFailure; /** * Shared prepaid traffic accounting. @@ -645,14 +646,26 @@ final class CloudTrafficPoolAccounting } /** Discover missing and due period projections without locking instances. */ - public static function reconcileDue(int $limit = 200, ?int $now = null): array + public static function reconcileDue( + int $limit = 200, + ?int $now = null, + int $afterPeriodId = 0 + ): array { $now ??= time(); + $limit = max(1, $limit); + $afterPeriodId = max(0, $afterPeriodId); $periodIds = []; $missing = CloudServicePeriod::query() ->leftJoin('cloud_traffic_entitlements as traffic_entitlement', 'traffic_entitlement.service_period_id', '=', 'cloud_service_periods.id') + ->leftJoin('cloud_traffic_scan_failures as missing_failure', + static function ($join): void { + $join->on('missing_failure.subject_id', '=', + 'cloud_service_periods.id') + ->where('missing_failure.scan_kind', '=', 'period'); + }) ->join('cloud_traffic_pool_groups as traffic_group', 'traffic_group.id', '=', 'cloud_service_periods.traffic_pool_group_id') @@ -660,13 +673,20 @@ final class CloudTrafficPoolAccounting ->whereIn('cloud_service_periods.status', ['scheduled', 'active', 'exhausted', 'finished']) ->whereNull('traffic_entitlement.id') + ->where(function ($retry) use ($now): void { + $retry->whereNull('missing_failure.id') + ->orWhere('missing_failure.retry_at', '<=', $now); + }) + ->where('cloud_service_periods.id', '>', $afterPeriodId) ->orderBy('cloud_service_periods.id')->limit($limit) ->pluck('cloud_service_periods.id')->all(); - foreach ($missing as $id) { - $periodIds[(int) $id] = true; - } - if (count($periodIds) < $limit) { - $due = CloudTrafficEntitlement::query() + $due = CloudTrafficEntitlement::query() + ->leftJoin('cloud_traffic_scan_failures as due_failure', + static function ($join): void { + $join->on('due_failure.subject_id', '=', + 'cloud_traffic_entitlements.service_period_id') + ->where('due_failure.scan_kind', '=', 'period'); + }) ->join('cloud_traffic_pools as due_pool', 'due_pool.id', '=', 'cloud_traffic_entitlements.pool_id') @@ -674,6 +694,12 @@ final class CloudTrafficPoolAccounting 'due_group.id', '=', 'due_pool.traffic_pool_group_id') ->whereIn('due_group.mode', ['shadow', 'enforced']) + ->where(function ($retry) use ($now): void { + $retry->whereNull('due_failure.id') + ->orWhere('due_failure.retry_at', '<=', $now); + }) + ->where('cloud_traffic_entitlements.service_period_id', '>', + $afterPeriodId) ->where(function ($query) use ($now): void { $query->where(function ($scheduled) use ($now): void { $scheduled->where( @@ -692,15 +718,16 @@ final class CloudTrafficPoolAccounting )->where('cloud_traffic_entitlements.closed_at', '<=', $now); }); - })->orderBy('cloud_traffic_entitlements.id') - ->limit($limit - count($periodIds)) + })->orderBy('cloud_traffic_entitlements.service_period_id') + ->limit($limit) ->pluck('cloud_traffic_entitlements.service_period_id')->all(); - foreach ($due as $id) { - $periodIds[(int) $id] = true; - } - } - if (count($periodIds) < $limit) { - $changed = CloudTrafficEntitlement::query() + $changed = CloudTrafficEntitlement::query() + ->leftJoin('cloud_traffic_scan_failures as changed_failure', + static function ($join): void { + $join->on('changed_failure.subject_id', '=', + 'cloud_traffic_entitlements.service_period_id') + ->where('changed_failure.scan_kind', '=', 'period'); + }) ->join('cloud_service_periods as traffic_period', 'traffic_period.id', '=', 'cloud_traffic_entitlements.service_period_id') @@ -711,18 +738,25 @@ final class CloudTrafficPoolAccounting 'changed_group.id', '=', 'changed_pool.traffic_pool_group_id') ->whereIn('changed_group.mode', ['shadow', 'enforced']) + ->where(function ($retry) use ($now): void { + $retry->whereNull('changed_failure.id') + ->orWhere('changed_failure.retry_at', '<=', $now); + }) + ->where('cloud_traffic_entitlements.service_period_id', '>', + $afterPeriodId) ->whereColumn( 'traffic_period.traffic_quota_bytes', '>', 'cloud_traffic_entitlements.granted_bytes' - )->orderBy('cloud_traffic_entitlements.id') - ->limit($limit - count($periodIds)) + )->orderBy('cloud_traffic_entitlements.service_period_id') + ->limit($limit) ->pluck('cloud_traffic_entitlements.service_period_id') ->all(); - foreach ($changed as $id) { - $periodIds[(int) $id] = true; - } + foreach (array_merge($missing, $due, $changed) as $id) { + $periodIds[(int) $id] = true; } + ksort($periodIds, SORT_NUMERIC); + $periodIds = array_slice($periodIds, 0, $limit, true); $processed = 0; $errors = []; foreach (array_keys($periodIds) as $periodId) { @@ -732,23 +766,44 @@ final class CloudTrafficPoolAccounting 'period-scan:' . $periodId, $now ); + self::clearScanFailure('period', $periodId); $processed++; } catch (\Throwable $error) { + self::recordScanFailure('period', $periodId, $error, $now); $errors[] = [ 'period_id' => $periodId, 'error' => get_class($error), ]; } } - return ['processed' => $processed, 'errors' => $errors]; + $selectedIds = array_keys($periodIds); + $nextCursor = count($selectedIds) >= $limit + ? (int) max($selectedIds) : 0; + return [ + 'processed' => $processed, + 'errors' => $errors, + 'next_cursor' => $nextCursor, + ]; } /** Discover persisted counters that have not yet crossed the accounting seam. */ - public static function settlePendingUsage(int $limit = 200): array + public static function settlePendingUsage( + int $limit = 200, + int $afterInstanceId = 0 + ): array { + $now = time(); + $limit = max(1, $limit); + $afterInstanceId = max(0, $afterInstanceId); $rows = CloudInstance::query() ->leftJoin('cloud_traffic_instance_cursors as traffic_cursor', 'traffic_cursor.instance_id', '=', 'cloud_instances.id') + ->leftJoin('cloud_traffic_scan_failures as instance_failure', + static function ($join): void { + $join->on('instance_failure.subject_id', '=', + 'cloud_instances.id') + ->where('instance_failure.scan_kind', '=', 'instance'); + }) ->whereExists(function ($eligible): void { $eligible->selectRaw('1') ->from('cloud_service_periods as eligible_period') @@ -776,7 +831,11 @@ final class CloudTrafficPoolAccounting '<', 'traffic_cursor.accounted_download_bytes') ->orWhereColumn('cloud_instances.lifetime_upload_bytes', '<', 'traffic_cursor.accounted_upload_bytes'); - })->orderBy('cloud_instances.id')->limit($limit)->distinct() + })->where(function ($retry) use ($now): void { + $retry->whereNull('instance_failure.id') + ->orWhere('instance_failure.retry_at', '<=', $now); + })->where('cloud_instances.id', '>', $afterInstanceId) + ->orderBy('cloud_instances.id')->limit($limit)->distinct() ->get([ 'cloud_instances.id', 'cloud_instances.traffic_synced_at', @@ -789,15 +848,93 @@ final class CloudTrafficPoolAccounting (int) $row->id, (int) $row->traffic_synced_at ); + self::clearScanFailure('instance', (int) $row->id); $processed++; } catch (\Throwable $error) { + self::recordScanFailure( + 'instance', + (int) $row->id, + $error, + $now + ); $errors[] = [ 'instance_id' => (int) $row->id, 'error' => get_class($error), ]; } } - return ['processed' => $processed, 'errors' => $errors]; + $nextCursor = count($rows) >= $limit + ? (int) $rows->last()->id : 0; + return [ + 'processed' => $processed, + 'errors' => $errors, + 'next_cursor' => $nextCursor, + ]; + } + + private static function recordScanFailure( + string $scanKind, + int $subjectId, + \Throwable $error, + int $now + ): void { + if (!in_array($scanKind, ['period', 'instance'], true) + || $subjectId < 1) { + return; + } + $model = new CloudTrafficScanFailure(); + $connection = $model->getConnection(); + $connection->transaction(function () use ( + $connection, + $scanKind, + $subjectId, + $error, + $now + ): void { + $subjectExists = $scanKind === 'period' + ? CloudServicePeriod::where('id', $subjectId) + ->select('id')->lockForUpdate()->first() !== null + : CloudInstance::where('id', $subjectId) + ->select('id')->lockForUpdate()->first() !== null; + if (!$subjectExists) { + return; + } + $connection->table('cloud_traffic_scan_failures') + ->insertOrIgnore([ + 'scan_kind' => $scanKind, + 'subject_id' => $subjectId, + 'attempt_count' => 0, + 'last_error_code' => '', + 'retry_at' => $now, + 'first_failed_at' => $now, + 'last_failed_at' => $now, + ]); + $failure = CloudTrafficScanFailure::where( + 'scan_kind', $scanKind + )->where('subject_id', $subjectId)->lockForUpdate()->first(); + if ($failure === null) { + return; + } + $attempt = min(65535, (int) $failure->attempt_count + 1); + $class = get_class($error); + $separator = strrpos($class, '\\'); + $code = $separator === false + ? $class : substr($class, $separator + 1); + $backoff = min(3600, 30 * (2 ** min(7, $attempt - 1))); + $failure->attempt_count = $attempt; + $failure->last_error_code = substr($code, 0, 128); + $failure->retry_at = $now + $backoff; + $failure->last_failed_at = $now; + $failure->save(); + }, 3); + } + + private static function clearScanFailure( + string $scanKind, + int $subjectId + ): void { + CloudTrafficScanFailure::where('scan_kind', $scanKind) + ->where('subject_id', $subjectId)->delete(); } private static function assertAccountingAttentionResolvedLocked( diff --git a/web/app/Services/UserDataDeletionService.php b/web/app/Services/UserDataDeletionService.php index 236d1d19..547e1818 100644 --- a/web/app/Services/UserDataDeletionService.php +++ b/web/app/Services/UserDataDeletionService.php @@ -108,8 +108,32 @@ class UserDataDeletionService // jobs before their parent rows so encrypted owner/VM payloads do // not survive as orphaned administrator audit records. $resourceJobRows = $this->deleteCloudResourceJobs($connection, $ids); - $rows = $deletedMailRows + $resourceJobRows - + $this->deleteRelatedRecords($connection, $ids, $emails); + $trafficInstanceIds = $this->pluckByUsers( + $connection, + 'cloud_instances', + 'user_id', + $ids, + 'id' + ); + $trafficPeriodIds = $this->pluckByUsers( + $connection, + 'cloud_service_periods', + 'user_id', + $ids, + 'id' + ); + $relatedRows = $this->deleteRelatedRecords( + $connection, + $ids, + $emails + ); + $trafficScanRows = $this->deleteCloudTrafficScanFailures( + $connection, + $trafficInstanceIds, + $trafficPeriodIds + ); + $rows = $deletedMailRows + $resourceJobRows + $trafficScanRows + + $relatedRows; $deleted = (int) User::whereIn('id', $ids)->where('is_admin', 0)->delete(); if ($deleted !== count($ids)) { throw new \RuntimeException('用户状态发生变化,删除已取消'); @@ -263,6 +287,32 @@ class UserDataDeletionService })->delete(); } + private function deleteCloudTrafficScanFailures( + $connection, + array $instanceIds, + array $periodIds + ): int { + if (!$this->hasColumns($connection, 'cloud_traffic_scan_failures', [ + 'scan_kind', 'subject_id', + ])) { + return 0; + } + $deleted = 0; + if ($instanceIds !== []) { + $deleted += (int) $connection->table( + 'cloud_traffic_scan_failures' + )->where('scan_kind', 'instance') + ->whereIn('subject_id', $instanceIds)->delete(); + } + if ($periodIds !== []) { + $deleted += (int) $connection->table( + 'cloud_traffic_scan_failures' + )->where('scan_kind', 'period') + ->whereIn('subject_id', $periodIds)->delete(); + } + return $deleted; + } + private function deleteLogRecords(array $ids, array $emails) { if (Config::get('enable_log', 'false') !== 'true') { diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index 29455825..263a605b 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -1426,6 +1426,25 @@ CREATE TABLE cloud_traffic_instance_cursors ( ) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +-- A failed bounded scan is quarantined with backoff so a deterministic bad +-- row cannot permanently starve higher IDs, including one-shot workers that +-- do not retain an in-memory cursor between invocations. +CREATE TABLE cloud_traffic_scan_failures ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + scan_kind VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + subject_id BIGINT UNSIGNED NOT NULL, + attempt_count INT UNSIGNED NOT NULL DEFAULT 1, + last_error_code VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + retry_at BIGINT UNSIGNED NOT NULL, + first_failed_at BIGINT UNSIGNED NOT NULL, + last_failed_at BIGINT UNSIGNED NOT NULL, + UNIQUE KEY cloud_traffic_scan_failure_subject (scan_kind, subject_id), + KEY cloud_traffic_scan_failure_retry (scan_kind, retry_at, subject_id), + CONSTRAINT chk_cloud_traffic_scan_failure_kind CHECK ( + scan_kind IN ('period','instance') + ) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + CREATE TABLE cloud_instance_network_restrictions ( instance_id BIGINT UNSIGNED NOT NULL, user_id BIGINT UNSIGNED NOT NULL, diff --git a/web/tests/CloudFreshSchemaContractTest.php b/web/tests/CloudFreshSchemaContractTest.php index 588c271e..b13f135b 100644 --- a/web/tests/CloudFreshSchemaContractTest.php +++ b/web/tests/CloudFreshSchemaContractTest.php @@ -22,7 +22,7 @@ $assert(stripos($schema, 'compatibility import') === false $assert(!preg_match('/\bengine_[A-Za-z0-9_]*\b/', $schema), 'cloud.sql retained an engine-prefixed identifier.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v20-traffic-pool-ledger'") + "SCHEMA_VERSION = 'go-agent-wss-v21-traffic-scan-backoff'") && !str_contains($setup, "SCHEMA_VERSION = 'go-v1'"), 'Outbound Agent schema must reject the incompatible earlier Go database.'); $assert(str_contains($schema, diff --git a/web/tests/CloudRescueCatalogSchemaContractTest.php b/web/tests/CloudRescueCatalogSchemaContractTest.php index 01eee8b2..dc144ceb 100644 --- a/web/tests/CloudRescueCatalogSchemaContractTest.php +++ b/web/tests/CloudRescueCatalogSchemaContractTest.php @@ -56,7 +56,7 @@ $assert($replica !== '' && str_contains($replica, 'cleanup_resolved_at BIGINT UNSIGNED'), 'Rescue replica cannot prove exact identity or retain cleanup debt.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v20-traffic-pool-ledger'") + "SCHEMA_VERSION = 'go-agent-wss-v21-traffic-scan-backoff'") && str_contains($setup, "'cloud_rescue_artifact_delete_due'") && str_contains($setup, "'cloud_rescue_replica_cleanup'"), 'CloudSetup version or rescue catalog indexes are incomplete.'); diff --git a/web/tests/CloudRuntimeHeartbeatTest.php b/web/tests/CloudRuntimeHeartbeatTest.php index 8db0e7ff..b48f7ada 100644 --- a/web/tests/CloudRuntimeHeartbeatTest.php +++ b/web/tests/CloudRuntimeHeartbeatTest.php @@ -496,7 +496,8 @@ try { $lifecycle = new CloudLifecycleService($partial); $assert($lifecycle->tick() === 0 && $lifecycle->observationErrors() === ['provision_expiry_failed', - 'invoice_generation_failed', 'traffic_sync_failed'] + 'invoice_generation_failed', 'traffic_sync_failed', + 'traffic_pool_accounting_failed'] && $checks($partialDirectory)['lifecycle']['status'] === 'warning' && $checks($partialDirectory)['lifecycle']['last_success_at'] === null, 'Lifecycle swallowed sub-stage failures and falsely reported normal operation: ' @@ -510,7 +511,8 @@ try { Capsule::table('cloud_instances')->insert(['id' => 1, 'status' => 'active', 'service_status' => 'active']); $assert($lifecycle->tick() === 0 && $lifecycle->observationErrors() === ['provision_expiry_failed', 'invoice_generation_failed', - 'traffic_sync_failed', 'instance_reconcile_failed'] + 'traffic_sync_failed', 'traffic_pool_accounting_failed', + 'instance_reconcile_failed'] && $checks($partialDirectory)['lifecycle']['status'] === 'warning', 'All internally caught lifecycle stages failed without an unhealthy observation: ' . json_encode($lifecycle->observationErrors(), JSON_UNESCAPED_SLASHES)); diff --git a/web/tests/CloudTemplateImportIntentContractTest.php b/web/tests/CloudTemplateImportIntentContractTest.php index ad20c0e2..88a29b8e 100644 --- a/web/tests/CloudTemplateImportIntentContractTest.php +++ b/web/tests/CloudTemplateImportIntentContractTest.php @@ -63,7 +63,7 @@ $assert(str_contains($schema, && str_contains($schema, 'CONSTRAINT chk_cloud_template_import_completion CHECK (') && str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v20-traffic-pool-ledger'"), + "SCHEMA_VERSION = 'go-agent-wss-v21-traffic-scan-backoff'"), 'Template imports are missing durable identity or terminal-state constraints.'); $assert(str_contains($routes, "'/admin/cloud-template-imports'") && str_contains($routes, "'/admin/cloud-template-imports/{id}'") diff --git a/web/tests/CloudTrafficPoolAccountingTest.php b/web/tests/CloudTrafficPoolAccountingTest.php index 3ab55999..afd87321 100644 --- a/web/tests/CloudTrafficPoolAccountingTest.php +++ b/web/tests/CloudTrafficPoolAccountingTest.php @@ -137,6 +137,17 @@ $schema->create('cloud_traffic_instance_cursors', static function (Blueprint $ta $table->integer('version')->default(0); $table->integer('updated_at'); }); +$schema->create('cloud_traffic_scan_failures', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('scan_kind'); + $table->integer('subject_id'); + $table->integer('attempt_count')->default(1); + $table->string('last_error_code'); + $table->integer('retry_at'); + $table->integer('first_failed_at'); + $table->integer('last_failed_at'); + $table->unique(['scan_kind', 'subject_id']); +}); $base = time(); Capsule::table('cloud_traffic_pool_groups')->insert([ @@ -646,6 +657,160 @@ $assert($snapshotEvent !== null && (int) $snapshotEvent->observed_at === $base + 10, 'Counters were paired with a stale observation timestamp.'); +$scanInstanceA = $newInstance(57, 15); +$scanPeriodA = $newPeriod( + $scanInstanceA, 507, 1, 100, $base, $base + 500 +); +$scanInstanceB = $newInstance(58, 15); +$scanPeriodB = $newPeriod( + $scanInstanceB, 508, 1, 100, $base, $base + 500 +); +foreach ([507, 508] as $scanPeriodId) { + CloudTrafficPoolAccounting::reconcileServicePeriod( + $scanPeriodId, + 'test-scan:' . $scanPeriodId, + $base + 1 + ); +} +$scanInstanceA->lifetime_download_bytes = 10; +$scanInstanceA->traffic_synced_at = $base + 10; +$scanInstanceA->save(); +$scanInstanceB->lifetime_download_bytes = 20; +$scanInstanceB->traffic_synced_at = $base + 10; +$scanInstanceB->save(); +$firstUsageWindow = CloudTrafficPoolAccounting::settlePendingUsage(1, 0); +$secondUsageWindow = CloudTrafficPoolAccounting::settlePendingUsage( + 1, + (int) $firstUsageWindow['next_cursor'] +); +$usageWrap = CloudTrafficPoolAccounting::settlePendingUsage( + 1, + (int) $secondUsageWindow['next_cursor'] +); +$assert((int) $firstUsageWindow['next_cursor'] === 57 + && (int) $secondUsageWindow['next_cursor'] === 58 + && (int) $usageWrap['next_cursor'] === 0 + && (int) CloudTrafficInstanceCursor::find(58)->accounted_download_bytes + === 20, + 'The bounded usage scan did not advance past the first selected row.'); + +$scanFinishedA = $newInstance(59, 16); +$scanFinishedPeriodA = $newPeriod( + $scanFinishedA, 509, 1, 50, $base - 200, $base - 100 +); +$scanFinishedPeriodA->status = 'finished'; +$scanFinishedPeriodA->save(); +$scanFinishedB = $newInstance(60, 17); +$scanFinishedPeriodB = $newPeriod( + $scanFinishedB, 510, 1, 50, $base - 200, $base - 100 +); +$scanFinishedPeriodB->status = 'finished'; +$scanFinishedPeriodB->save(); +$firstPeriodWindow = CloudTrafficPoolAccounting::reconcileDue( + 1, + $base, + 508 +); +$secondPeriodWindow = CloudTrafficPoolAccounting::reconcileDue( + 1, + $base, + (int) $firstPeriodWindow['next_cursor'] +); +$periodWrap = CloudTrafficPoolAccounting::reconcileDue( + 1, + $base, + (int) $secondPeriodWindow['next_cursor'] +); +$assert((int) $firstPeriodWindow['next_cursor'] === 509 + && (int) $secondPeriodWindow['next_cursor'] === 510 + && (int) $periodWrap['next_cursor'] === 0 + && CloudTrafficEntitlement::where('service_period_id', 510)->exists(), + 'The bounded period scan did not advance past the first selected row.'); + +$dueBeforePoisonInstance = $newInstance(61, 18); +$dueBeforePoison = $newPeriod( + $dueBeforePoisonInstance, + 511, + 1, + 50, + $base - 200, + $base - 100 +); +CloudTrafficPoolAccounting::reconcileServicePeriod( + 511, 'test-cross-category:activate', $base - 150 +); +$poisonPeriodInstance = $newInstance(62, 19); +$poisonPeriod = $newPeriod( + $poisonPeriodInstance, + 600, + 1, + 50, + $base, + $base +); +$crossCategoryWindow = CloudTrafficPoolAccounting::reconcileDue( + 1, + $base, + 0 +); +$assert((int) $crossCategoryWindow['next_cursor'] === 511 + && (string) CloudTrafficEntitlement::where( + 'service_period_id', 511 + )->first()->state === 'expired', + 'A higher-ID missing period starved a lower-ID due entitlement.'); +$poisonWindow = CloudTrafficPoolAccounting::reconcileDue( + 1, + $base, + 511 +); +$poisonRetry = CloudTrafficPoolAccounting::reconcileDue(1, $base, 0); +$assert(count($poisonWindow['errors']) === 1 + && Capsule::table('cloud_traffic_scan_failures') + ->where('scan_kind', 'period')->where('subject_id', 600)->exists() + && $poisonRetry['processed'] === 0 + && $poisonRetry['errors'] === [], + 'A deterministic period error was not quarantined across scan restarts.'); + +$poisonUsageInstance = $newInstance(63, 20); +$poisonUsagePeriod = $newPeriod( + $poisonUsageInstance, + 601, + 1, + 50, + $base, + $base +); +$poisonUsageInstance->lifetime_download_bytes = 10; +$poisonUsageInstance->traffic_synced_at = $base + 1; +$poisonUsageInstance->save(); +$healthyUsageInstance = $newInstance(64, 20); +$healthyUsagePeriod = $newPeriod( + $healthyUsageInstance, + 602, + 1, + 50, + $base, + $base + 500 +); +CloudTrafficPoolAccounting::reconcileServicePeriod( + 602, 'test-usage-quarantine:healthy', $base + 1 +); +$healthyUsageInstance->lifetime_download_bytes = 10; +$healthyUsageInstance->traffic_synced_at = $base + 1; +$healthyUsageInstance->save(); +$badUsageWindow = CloudTrafficPoolAccounting::settlePendingUsage(1, 0); +$goodUsageWindow = CloudTrafficPoolAccounting::settlePendingUsage( + 1, + (int) $badUsageWindow['next_cursor'] +); +$usageRestart = CloudTrafficPoolAccounting::settlePendingUsage(1, 0); +$assert(count($badUsageWindow['errors']) === 1 + && (int) $goodUsageWindow['next_cursor'] === 64 + && Capsule::table('cloud_traffic_scan_failures') + ->where('scan_kind', 'instance')->where('subject_id', 63)->exists() + && $usageRestart['errors'] === [], + 'A deterministic instance error starved later usage after a restart.'); + $idleScan = CloudTrafficPoolAccounting::settlePendingUsage(1); $assert($idleScan['processed'] === 0, 'An instance without a trusted sample poisoned the bounded scan queue.'); diff --git a/web/tests/CloudTrafficPoolLifecycleSchedulerContractTest.php b/web/tests/CloudTrafficPoolLifecycleSchedulerContractTest.php new file mode 100644 index 00000000..60c2c741 --- /dev/null +++ b/web/tests/CloudTrafficPoolLifecycleSchedulerContractTest.php @@ -0,0 +1,55 @@ +sync($batchSize)'); +$instanceReconcile = strpos($lifecycle, + "\$this->heartbeat?->busy('instance_reconcile')"); +$assert($sync !== false && $settle !== false && $due !== false + && $runtime !== false && $instanceReconcile !== false + && $sync < $settle && $settle < $due && $due < $runtime + && $runtime < $instanceReconcile, + 'Lifecycle must sync counters, settle usage, reconcile due periods, then ' + . 'continue with runtime and per-instance reconciliation.'); +$assert(str_contains($lifecycle, 'trafficUsageScanCursor') + && str_contains($lifecycle, 'trafficPeriodScanCursor') + && substr_count($lifecycle, "['next_cursor']") >= 2, + 'Bounded accounting scans no longer advance past a persistent bad row.'); + +$privateReconcile = strpos($lifecycle, 'private function reconcile('); +$assert($privateReconcile !== false + && strpos(substr($lifecycle, $privateReconcile), + 'CloudTrafficPoolAccounting::') === false, + 'Pool accounting was nested inside the instance-first lifecycle transaction.'); + +$stage = substr($lifecycle, $settle, $runtime - $settle); +$assert(!str_contains($stage, 'network_status') + && !str_contains($stage, 'CloudJobService') + && !str_contains($stage, 'CloudAgentClient') + && str_contains($stage, + "observeFailure('traffic_pool_accounting_failed')"), + 'Shadow accounting changed network policy or lost fail-open observations.'); + +$assert(!str_contains($accounting, 'CloudAgentClient') + && !str_contains($accounting, 'network_status') + && !str_contains($accounting, "'kind' => 'network_suspend'") + && !str_contains($accounting, "'kind' => 'network_resume'"), + 'The shadow accounting module acquired an Agent/network side effect.'); + +echo "Cloud traffic-pool lifecycle scheduler contract passed.\n"; diff --git a/web/tests/CloudTrafficPoolSchemaContractTest.php b/web/tests/CloudTrafficPoolSchemaContractTest.php index 60869f3c..f1402c72 100644 --- a/web/tests/CloudTrafficPoolSchemaContractTest.php +++ b/web/tests/CloudTrafficPoolSchemaContractTest.php @@ -24,6 +24,7 @@ foreach ([ 'cloud_traffic_entitlements', 'cloud_traffic_ledger_events', 'cloud_traffic_instance_cursors', + 'cloud_traffic_scan_failures', 'cloud_instance_network_restrictions', ] as $table) { $assert(str_contains($schema, 'CREATE TABLE ' . $table . ' ('), @@ -41,7 +42,9 @@ $assert(str_contains($schema, && str_contains($schema, 'UNIQUE KEY cloud_traffic_ledger_event_key (event_key)') && str_contains($schema, - 'UNIQUE KEY cloud_traffic_ledger_sequence (pool_id, pool_sequence)'), + 'UNIQUE KEY cloud_traffic_ledger_sequence (pool_id, pool_sequence)') + && str_contains($schema, + 'UNIQUE KEY cloud_traffic_scan_failure_subject (scan_kind, subject_id)'), 'Traffic-pool identity, FEFO or immutable ledger indexes are incomplete.'); $assert(str_contains($schema, @@ -51,7 +54,9 @@ $assert(str_contains($schema, && str_contains($schema, 'consumed_bytes + forfeited_bytes <= granted_bytes') && str_contains($schema, - "reason_code IN ('traffic_pool_exhausted','service_expired','admin_hold')"), + "reason_code IN ('traffic_pool_exhausted','service_expired','admin_hold')") + && str_contains($schema, + "scan_kind IN ('period','instance')"), 'Traffic-pool state or conservation checks are incomplete.'); $assert(str_contains($schema, @@ -63,7 +68,7 @@ $assert(str_contains($schema, 'Service-period traffic-pool identity is not frozen or indexed.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v20-traffic-pool-ledger'") + "SCHEMA_VERSION = 'go-agent-wss-v21-traffic-scan-backoff'") && str_contains($setup, "'chk_cloud_traffic_entitlement_balance'") && str_contains($setup, "'cloud_traffic_ledger_sequence'"), 'CloudSetup schema version or named-object validation is incomplete.'); @@ -83,10 +88,19 @@ foreach ($expectedDeletionOrder as $entry) { 'Account deletion order is missing or unsafe at ' . $entry); $offset = $next; } +$assert(str_contains($deletion, 'deleteCloudTrafficScanFailures('), + 'Account deletion leaves traffic scan quarantine rows behind.'); +$deleteParents = strpos($deletion, '$relatedRows = $this->deleteRelatedRecords('); +$deleteFailures = strpos($deletion, + '$trafficScanRows = $this->deleteCloudTrafficScanFailures('); +$assert($deleteParents !== false && $deleteFailures !== false + && $deleteParents < $deleteFailures, + 'Traffic scan quarantine cleanup can race after account deletion.'); foreach ([ 'CloudTrafficPoolGroup', 'CloudTrafficPool', 'CloudTrafficEntitlement', 'CloudTrafficLedgerEvent', 'CloudTrafficInstanceCursor', + 'CloudTrafficScanFailure', 'CloudInstanceNetworkRestriction', ] as $model) { $assert(is_file($root . '/app/Models/' . $model . '.php'), diff --git a/web/tests/CloudVpcSchemaContractTest.php b/web/tests/CloudVpcSchemaContractTest.php index 7014b048..c65b5efe 100644 --- a/web/tests/CloudVpcSchemaContractTest.php +++ b/web/tests/CloudVpcSchemaContractTest.php @@ -101,7 +101,7 @@ $assert(str_contains($jobs, 'vpc_id BIGINT UNSIGNED DEFAULT NULL') 'Cloud jobs cannot be correlated to VPC or VPC-port work.'); $assert(str_contains($setup, - "private const SCHEMA_VERSION = 'go-agent-wss-v20-traffic-pool-ledger'") + "private const SCHEMA_VERSION = 'go-agent-wss-v21-traffic-scan-backoff'") && str_contains($setup, "'name' => 'cloud_vpc_enabled'") && str_contains($setup, "'value' => 'false'") && str_contains($setup, diff --git a/web/tests/ManualAccountDeletionSqliteTest.php b/web/tests/ManualAccountDeletionSqliteTest.php index fbc09ce8..f322a269 100644 --- a/web/tests/ManualAccountDeletionSqliteTest.php +++ b/web/tests/ManualAccountDeletionSqliteTest.php @@ -92,6 +92,10 @@ foreach (['cloud_instance_network_restrictions', $table->increments('id'); $table->integer('user_id'); }); } +$schema->create('cloud_traffic_scan_failures', static function (Blueprint $table) { + $table->increments('id'); $table->string('scan_kind'); + $table->integer('subject_id'); +}); $schema->create('cloud_invoices', static function (Blueprint $table) { $table->increments('id'); $table->integer('user_id'); }); @@ -178,6 +182,10 @@ foreach (['cloud_instance_network_restrictions', ['user_id' => 1], ['user_id' => 3], ]); } +Capsule::table('cloud_traffic_scan_failures')->insert([ + ['scan_kind' => 'instance', 'subject_id' => 101], + ['scan_kind' => 'instance', 'subject_id' => 301], +]); Capsule::table('cloud_invoices')->insert([ ['id' => 103, 'user_id' => 1], ['id' => 303, 'user_id' => 3], ]); @@ -213,6 +221,9 @@ foreach (['cloud_instances', 'cloud_orders', 'cloud_instance_snapshots', $assert(Capsule::table($tableName)->pluck('user_id')->all() === [3], 'Cloud account history deletion was not isolated: ' . $tableName); } +$assert(Capsule::table('cloud_traffic_scan_failures') + ->pluck('subject_id')->all() === [301], + 'Cloud traffic scan quarantine deletion was not isolated.'); $assert(Capsule::table('cloud_instance_firewall_versions')->pluck('id')->all() === [306] && Capsule::table('cloud_instance_firewall_rules')->pluck('id')->all()