feat(traffic): schedule starvation-safe shadow accounting

go
codex 2 weeks ago
parent 1f7a6d9a19
commit e35e93c8a6

@ -8,7 +8,7 @@ use App\Services\Config;
/** Initializes only the clean-break PHP business and Agent log schemas. */
final class CloudSetup
{
private const SCHEMA_VERSION = 'go-agent-wss-v20-traffic-pool-ledger';
private const SCHEMA_VERSION = 'go-agent-wss-v21-traffic-scan-backoff';
private const MAIN_TABLES = [
'cloud_regions', 'cloud_traffic_pool_groups',
'cloud_network_lines', 'cloud_resource_pools',
@ -37,6 +37,7 @@ final class CloudSetup
'cloud_resource_reservations', 'cloud_service_periods',
'cloud_traffic_pools', 'cloud_traffic_entitlements',
'cloud_traffic_ledger_events', 'cloud_traffic_instance_cursors',
'cloud_traffic_scan_failures',
'cloud_instance_network_restrictions',
'cloud_renewals', 'cloud_invoices', 'cloud_invoice_items',
'cloud_invoice_payments',
@ -297,6 +298,11 @@ final class CloudSetup
'last_observed_at', 'last_event_id', 'state',
'attention_code', 'version', 'updated_at',
],
'cloud_traffic_scan_failures' => [
'id', 'scan_kind', 'subject_id', 'attempt_count',
'last_error_code', 'retry_at', 'first_failed_at',
'last_failed_at',
],
'cloud_instance_network_restrictions' => [
'instance_id', 'user_id', 'reason_code', 'desired_active',
'applied_active', 'revision', 'state', 'source_type',
@ -341,6 +347,9 @@ final class CloudSetup
'cloud_traffic_instance_cursors' => [
'chk_cloud_traffic_cursor_state',
],
'cloud_traffic_scan_failures' => [
'chk_cloud_traffic_scan_failure_kind',
],
'cloud_instance_network_restrictions' => [
'chk_cloud_network_restriction_reason',
'chk_cloud_network_restriction_state',
@ -448,6 +457,10 @@ final class CloudSetup
'PRIMARY', 'cloud_traffic_cursor_pool',
'cloud_traffic_cursor_owner',
],
'cloud_traffic_scan_failures' => [
'PRIMARY', 'cloud_traffic_scan_failure_subject',
'cloud_traffic_scan_failure_retry',
],
'cloud_instance_network_restrictions' => [
'PRIMARY', 'cloud_network_restriction_operation',
'cloud_network_restriction_state',

@ -0,0 +1,18 @@
<?php
namespace App\Models;
final class CloudTrafficScanFailure extends Model
{
protected $connection = 'default';
protected $table = 'cloud_traffic_scan_failures';
protected $casts = [
'id' => 'int',
'subject_id' => 'int',
'attempt_count' => 'int',
'retry_at' => 'int',
'first_failed_at' => 'int',
'last_failed_at' => 'int',
];
}

@ -14,6 +14,8 @@ final class CloudLifecycleService
private int $templateReconcileAfter = 0;
private int $templateReconcileCursor = 0;
private int $rescueReconcileCursor = 0;
private int $trafficUsageScanCursor = 0;
private int $trafficPeriodScanCursor = 0;
public function __construct(private ?CloudRuntimeHeartbeat $heartbeat = null)
{
@ -198,6 +200,43 @@ final class CloudLifecycleService
// Unknown traffic is fail-open. Time expiry remains enforceable.
error_log('cloud traffic sync unavailable: ' . get_class($error));
}
// Shared-pool accounting consumes only counters already persisted by
// the traffic sync above. It deliberately runs outside every
// per-instance lifecycle transaction: pool accounting owns the
// pool -> cursor -> entitlement lock order, while network policy
// remains on the existing instance reconciliation path until the
// restriction/convergence layer is enabled.
$this->heartbeat?->busy('traffic_pool_accounting');
try {
$settled = CloudTrafficPoolAccounting::settlePendingUsage(
$batchSize,
$this->trafficUsageScanCursor
);
$this->trafficUsageScanCursor = max(
0,
(int) ($settled['next_cursor'] ?? 0)
);
$due = CloudTrafficPoolAccounting::reconcileDue(
$batchSize,
null,
$this->trafficPeriodScanCursor
);
$this->trafficPeriodScanCursor = max(
0,
(int) ($due['next_cursor'] ?? 0)
);
if (($settled['errors'] ?? []) !== []
|| ($due['errors'] ?? []) !== []) {
$this->observeFailure('traffic_pool_accounting_failed');
error_log('cloud traffic-pool accounting had bounded errors');
}
} catch (\Throwable $error) {
$this->observeFailure('traffic_pool_accounting_failed');
// A projection failure must never become a guessed network
// decision. Existing instance policy remains authoritative.
error_log('cloud traffic-pool accounting unavailable: '
. get_class($error));
}
// Power state is an observation cache, not a billing decision. Each
// host is contacted through its own Agent settings, so one unavailable
// host must not gate database-only billing or other hosts.

@ -9,6 +9,7 @@ use App\Models\CloudTrafficInstanceCursor;
use App\Models\CloudTrafficLedgerEvent;
use App\Models\CloudTrafficPool;
use App\Models\CloudTrafficPoolGroup;
use App\Models\CloudTrafficScanFailure;
/**
* Shared prepaid traffic accounting.
@ -645,14 +646,26 @@ final class CloudTrafficPoolAccounting
}
/** Discover missing and due period projections without locking instances. */
public static function reconcileDue(int $limit = 200, ?int $now = null): array
public static function reconcileDue(
int $limit = 200,
?int $now = null,
int $afterPeriodId = 0
): array
{
$now ??= time();
$limit = max(1, $limit);
$afterPeriodId = max(0, $afterPeriodId);
$periodIds = [];
$missing = CloudServicePeriod::query()
->leftJoin('cloud_traffic_entitlements as traffic_entitlement',
'traffic_entitlement.service_period_id', '=',
'cloud_service_periods.id')
->leftJoin('cloud_traffic_scan_failures as missing_failure',
static function ($join): void {
$join->on('missing_failure.subject_id', '=',
'cloud_service_periods.id')
->where('missing_failure.scan_kind', '=', 'period');
})
->join('cloud_traffic_pool_groups as traffic_group',
'traffic_group.id', '=',
'cloud_service_periods.traffic_pool_group_id')
@ -660,13 +673,20 @@ final class CloudTrafficPoolAccounting
->whereIn('cloud_service_periods.status',
['scheduled', 'active', 'exhausted', 'finished'])
->whereNull('traffic_entitlement.id')
->where(function ($retry) use ($now): void {
$retry->whereNull('missing_failure.id')
->orWhere('missing_failure.retry_at', '<=', $now);
})
->where('cloud_service_periods.id', '>', $afterPeriodId)
->orderBy('cloud_service_periods.id')->limit($limit)
->pluck('cloud_service_periods.id')->all();
foreach ($missing as $id) {
$periodIds[(int) $id] = true;
}
if (count($periodIds) < $limit) {
$due = CloudTrafficEntitlement::query()
$due = CloudTrafficEntitlement::query()
->leftJoin('cloud_traffic_scan_failures as due_failure',
static function ($join): void {
$join->on('due_failure.subject_id', '=',
'cloud_traffic_entitlements.service_period_id')
->where('due_failure.scan_kind', '=', 'period');
})
->join('cloud_traffic_pools as due_pool',
'due_pool.id', '=',
'cloud_traffic_entitlements.pool_id')
@ -674,6 +694,12 @@ final class CloudTrafficPoolAccounting
'due_group.id', '=',
'due_pool.traffic_pool_group_id')
->whereIn('due_group.mode', ['shadow', 'enforced'])
->where(function ($retry) use ($now): void {
$retry->whereNull('due_failure.id')
->orWhere('due_failure.retry_at', '<=', $now);
})
->where('cloud_traffic_entitlements.service_period_id', '>',
$afterPeriodId)
->where(function ($query) use ($now): void {
$query->where(function ($scheduled) use ($now): void {
$scheduled->where(
@ -692,15 +718,16 @@ final class CloudTrafficPoolAccounting
)->where('cloud_traffic_entitlements.closed_at',
'<=', $now);
});
})->orderBy('cloud_traffic_entitlements.id')
->limit($limit - count($periodIds))
})->orderBy('cloud_traffic_entitlements.service_period_id')
->limit($limit)
->pluck('cloud_traffic_entitlements.service_period_id')->all();
foreach ($due as $id) {
$periodIds[(int) $id] = true;
}
}
if (count($periodIds) < $limit) {
$changed = CloudTrafficEntitlement::query()
$changed = CloudTrafficEntitlement::query()
->leftJoin('cloud_traffic_scan_failures as changed_failure',
static function ($join): void {
$join->on('changed_failure.subject_id', '=',
'cloud_traffic_entitlements.service_period_id')
->where('changed_failure.scan_kind', '=', 'period');
})
->join('cloud_service_periods as traffic_period',
'traffic_period.id', '=',
'cloud_traffic_entitlements.service_period_id')
@ -711,18 +738,25 @@ final class CloudTrafficPoolAccounting
'changed_group.id', '=',
'changed_pool.traffic_pool_group_id')
->whereIn('changed_group.mode', ['shadow', 'enforced'])
->where(function ($retry) use ($now): void {
$retry->whereNull('changed_failure.id')
->orWhere('changed_failure.retry_at', '<=', $now);
})
->where('cloud_traffic_entitlements.service_period_id', '>',
$afterPeriodId)
->whereColumn(
'traffic_period.traffic_quota_bytes',
'>',
'cloud_traffic_entitlements.granted_bytes'
)->orderBy('cloud_traffic_entitlements.id')
->limit($limit - count($periodIds))
)->orderBy('cloud_traffic_entitlements.service_period_id')
->limit($limit)
->pluck('cloud_traffic_entitlements.service_period_id')
->all();
foreach ($changed as $id) {
$periodIds[(int) $id] = true;
}
foreach (array_merge($missing, $due, $changed) as $id) {
$periodIds[(int) $id] = true;
}
ksort($periodIds, SORT_NUMERIC);
$periodIds = array_slice($periodIds, 0, $limit, true);
$processed = 0;
$errors = [];
foreach (array_keys($periodIds) as $periodId) {
@ -732,23 +766,44 @@ final class CloudTrafficPoolAccounting
'period-scan:' . $periodId,
$now
);
self::clearScanFailure('period', $periodId);
$processed++;
} catch (\Throwable $error) {
self::recordScanFailure('period', $periodId, $error, $now);
$errors[] = [
'period_id' => $periodId,
'error' => get_class($error),
];
}
}
return ['processed' => $processed, 'errors' => $errors];
$selectedIds = array_keys($periodIds);
$nextCursor = count($selectedIds) >= $limit
? (int) max($selectedIds) : 0;
return [
'processed' => $processed,
'errors' => $errors,
'next_cursor' => $nextCursor,
];
}
/** Discover persisted counters that have not yet crossed the accounting seam. */
public static function settlePendingUsage(int $limit = 200): array
public static function settlePendingUsage(
int $limit = 200,
int $afterInstanceId = 0
): array
{
$now = time();
$limit = max(1, $limit);
$afterInstanceId = max(0, $afterInstanceId);
$rows = CloudInstance::query()
->leftJoin('cloud_traffic_instance_cursors as traffic_cursor',
'traffic_cursor.instance_id', '=', 'cloud_instances.id')
->leftJoin('cloud_traffic_scan_failures as instance_failure',
static function ($join): void {
$join->on('instance_failure.subject_id', '=',
'cloud_instances.id')
->where('instance_failure.scan_kind', '=', 'instance');
})
->whereExists(function ($eligible): void {
$eligible->selectRaw('1')
->from('cloud_service_periods as eligible_period')
@ -776,7 +831,11 @@ final class CloudTrafficPoolAccounting
'<', 'traffic_cursor.accounted_download_bytes')
->orWhereColumn('cloud_instances.lifetime_upload_bytes',
'<', 'traffic_cursor.accounted_upload_bytes');
})->orderBy('cloud_instances.id')->limit($limit)->distinct()
})->where(function ($retry) use ($now): void {
$retry->whereNull('instance_failure.id')
->orWhere('instance_failure.retry_at', '<=', $now);
})->where('cloud_instances.id', '>', $afterInstanceId)
->orderBy('cloud_instances.id')->limit($limit)->distinct()
->get([
'cloud_instances.id',
'cloud_instances.traffic_synced_at',
@ -789,15 +848,93 @@ final class CloudTrafficPoolAccounting
(int) $row->id,
(int) $row->traffic_synced_at
);
self::clearScanFailure('instance', (int) $row->id);
$processed++;
} catch (\Throwable $error) {
self::recordScanFailure(
'instance',
(int) $row->id,
$error,
$now
);
$errors[] = [
'instance_id' => (int) $row->id,
'error' => get_class($error),
];
}
}
return ['processed' => $processed, 'errors' => $errors];
$nextCursor = count($rows) >= $limit
? (int) $rows->last()->id : 0;
return [
'processed' => $processed,
'errors' => $errors,
'next_cursor' => $nextCursor,
];
}
private static function recordScanFailure(
string $scanKind,
int $subjectId,
\Throwable $error,
int $now
): void {
if (!in_array($scanKind, ['period', 'instance'], true)
|| $subjectId < 1) {
return;
}
$model = new CloudTrafficScanFailure();
$connection = $model->getConnection();
$connection->transaction(function () use (
$connection,
$scanKind,
$subjectId,
$error,
$now
): void {
$subjectExists = $scanKind === 'period'
? CloudServicePeriod::where('id', $subjectId)
->select('id')->lockForUpdate()->first() !== null
: CloudInstance::where('id', $subjectId)
->select('id')->lockForUpdate()->first() !== null;
if (!$subjectExists) {
return;
}
$connection->table('cloud_traffic_scan_failures')
->insertOrIgnore([
'scan_kind' => $scanKind,
'subject_id' => $subjectId,
'attempt_count' => 0,
'last_error_code' => '',
'retry_at' => $now,
'first_failed_at' => $now,
'last_failed_at' => $now,
]);
$failure = CloudTrafficScanFailure::where(
'scan_kind', $scanKind
)->where('subject_id', $subjectId)->lockForUpdate()->first();
if ($failure === null) {
return;
}
$attempt = min(65535, (int) $failure->attempt_count + 1);
$class = get_class($error);
$separator = strrpos($class, '\\');
$code = $separator === false
? $class : substr($class, $separator + 1);
$backoff = min(3600, 30 * (2 ** min(7, $attempt - 1)));
$failure->attempt_count = $attempt;
$failure->last_error_code = substr($code, 0, 128);
$failure->retry_at = $now + $backoff;
$failure->last_failed_at = $now;
$failure->save();
}, 3);
}
private static function clearScanFailure(
string $scanKind,
int $subjectId
): void {
CloudTrafficScanFailure::where('scan_kind', $scanKind)
->where('subject_id', $subjectId)->delete();
}
private static function assertAccountingAttentionResolvedLocked(

@ -108,8 +108,32 @@ class UserDataDeletionService
// jobs before their parent rows so encrypted owner/VM payloads do
// not survive as orphaned administrator audit records.
$resourceJobRows = $this->deleteCloudResourceJobs($connection, $ids);
$rows = $deletedMailRows + $resourceJobRows
+ $this->deleteRelatedRecords($connection, $ids, $emails);
$trafficInstanceIds = $this->pluckByUsers(
$connection,
'cloud_instances',
'user_id',
$ids,
'id'
);
$trafficPeriodIds = $this->pluckByUsers(
$connection,
'cloud_service_periods',
'user_id',
$ids,
'id'
);
$relatedRows = $this->deleteRelatedRecords(
$connection,
$ids,
$emails
);
$trafficScanRows = $this->deleteCloudTrafficScanFailures(
$connection,
$trafficInstanceIds,
$trafficPeriodIds
);
$rows = $deletedMailRows + $resourceJobRows + $trafficScanRows
+ $relatedRows;
$deleted = (int) User::whereIn('id', $ids)->where('is_admin', 0)->delete();
if ($deleted !== count($ids)) {
throw new \RuntimeException('用户状态发生变化,删除已取消');
@ -263,6 +287,32 @@ class UserDataDeletionService
})->delete();
}
private function deleteCloudTrafficScanFailures(
$connection,
array $instanceIds,
array $periodIds
): int {
if (!$this->hasColumns($connection, 'cloud_traffic_scan_failures', [
'scan_kind', 'subject_id',
])) {
return 0;
}
$deleted = 0;
if ($instanceIds !== []) {
$deleted += (int) $connection->table(
'cloud_traffic_scan_failures'
)->where('scan_kind', 'instance')
->whereIn('subject_id', $instanceIds)->delete();
}
if ($periodIds !== []) {
$deleted += (int) $connection->table(
'cloud_traffic_scan_failures'
)->where('scan_kind', 'period')
->whereIn('subject_id', $periodIds)->delete();
}
return $deleted;
}
private function deleteLogRecords(array $ids, array $emails)
{
if (Config::get('enable_log', 'false') !== 'true') {

@ -1426,6 +1426,25 @@ CREATE TABLE cloud_traffic_instance_cursors (
)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- A failed bounded scan is quarantined with backoff so a deterministic bad
-- row cannot permanently starve higher IDs, including one-shot workers that
-- do not retain an in-memory cursor between invocations.
CREATE TABLE cloud_traffic_scan_failures (
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
scan_kind VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
subject_id BIGINT UNSIGNED NOT NULL,
attempt_count INT UNSIGNED NOT NULL DEFAULT 1,
last_error_code VARCHAR(128) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
retry_at BIGINT UNSIGNED NOT NULL,
first_failed_at BIGINT UNSIGNED NOT NULL,
last_failed_at BIGINT UNSIGNED NOT NULL,
UNIQUE KEY cloud_traffic_scan_failure_subject (scan_kind, subject_id),
KEY cloud_traffic_scan_failure_retry (scan_kind, retry_at, subject_id),
CONSTRAINT chk_cloud_traffic_scan_failure_kind CHECK (
scan_kind IN ('period','instance')
)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE cloud_instance_network_restrictions (
instance_id BIGINT UNSIGNED NOT NULL,
user_id BIGINT UNSIGNED NOT NULL,

@ -22,7 +22,7 @@ $assert(stripos($schema, 'compatibility import') === false
$assert(!preg_match('/\bengine_[A-Za-z0-9_]*\b/', $schema),
'cloud.sql retained an engine-prefixed identifier.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v20-traffic-pool-ledger'")
"SCHEMA_VERSION = 'go-agent-wss-v21-traffic-scan-backoff'")
&& !str_contains($setup, "SCHEMA_VERSION = 'go-v1'"),
'Outbound Agent schema must reject the incompatible earlier Go database.');
$assert(str_contains($schema,

@ -56,7 +56,7 @@ $assert($replica !== ''
&& str_contains($replica, 'cleanup_resolved_at BIGINT UNSIGNED'),
'Rescue replica cannot prove exact identity or retain cleanup debt.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v20-traffic-pool-ledger'")
"SCHEMA_VERSION = 'go-agent-wss-v21-traffic-scan-backoff'")
&& str_contains($setup, "'cloud_rescue_artifact_delete_due'")
&& str_contains($setup, "'cloud_rescue_replica_cleanup'"),
'CloudSetup version or rescue catalog indexes are incomplete.');

@ -496,7 +496,8 @@ try {
$lifecycle = new CloudLifecycleService($partial);
$assert($lifecycle->tick() === 0
&& $lifecycle->observationErrors() === ['provision_expiry_failed',
'invoice_generation_failed', 'traffic_sync_failed']
'invoice_generation_failed', 'traffic_sync_failed',
'traffic_pool_accounting_failed']
&& $checks($partialDirectory)['lifecycle']['status'] === 'warning'
&& $checks($partialDirectory)['lifecycle']['last_success_at'] === null,
'Lifecycle swallowed sub-stage failures and falsely reported normal operation: '
@ -510,7 +511,8 @@ try {
Capsule::table('cloud_instances')->insert(['id' => 1, 'status' => 'active', 'service_status' => 'active']);
$assert($lifecycle->tick() === 0
&& $lifecycle->observationErrors() === ['provision_expiry_failed', 'invoice_generation_failed',
'traffic_sync_failed', 'instance_reconcile_failed']
'traffic_sync_failed', 'traffic_pool_accounting_failed',
'instance_reconcile_failed']
&& $checks($partialDirectory)['lifecycle']['status'] === 'warning',
'All internally caught lifecycle stages failed without an unhealthy observation: '
. json_encode($lifecycle->observationErrors(), JSON_UNESCAPED_SLASHES));

@ -63,7 +63,7 @@ $assert(str_contains($schema,
&& str_contains($schema,
'CONSTRAINT chk_cloud_template_import_completion CHECK (')
&& str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v20-traffic-pool-ledger'"),
"SCHEMA_VERSION = 'go-agent-wss-v21-traffic-scan-backoff'"),
'Template imports are missing durable identity or terminal-state constraints.');
$assert(str_contains($routes, "'/admin/cloud-template-imports'")
&& str_contains($routes, "'/admin/cloud-template-imports/{id}'")

@ -137,6 +137,17 @@ $schema->create('cloud_traffic_instance_cursors', static function (Blueprint $ta
$table->integer('version')->default(0);
$table->integer('updated_at');
});
$schema->create('cloud_traffic_scan_failures', static function (Blueprint $table): void {
$table->increments('id');
$table->string('scan_kind');
$table->integer('subject_id');
$table->integer('attempt_count')->default(1);
$table->string('last_error_code');
$table->integer('retry_at');
$table->integer('first_failed_at');
$table->integer('last_failed_at');
$table->unique(['scan_kind', 'subject_id']);
});
$base = time();
Capsule::table('cloud_traffic_pool_groups')->insert([
@ -646,6 +657,160 @@ $assert($snapshotEvent !== null
&& (int) $snapshotEvent->observed_at === $base + 10,
'Counters were paired with a stale observation timestamp.');
$scanInstanceA = $newInstance(57, 15);
$scanPeriodA = $newPeriod(
$scanInstanceA, 507, 1, 100, $base, $base + 500
);
$scanInstanceB = $newInstance(58, 15);
$scanPeriodB = $newPeriod(
$scanInstanceB, 508, 1, 100, $base, $base + 500
);
foreach ([507, 508] as $scanPeriodId) {
CloudTrafficPoolAccounting::reconcileServicePeriod(
$scanPeriodId,
'test-scan:' . $scanPeriodId,
$base + 1
);
}
$scanInstanceA->lifetime_download_bytes = 10;
$scanInstanceA->traffic_synced_at = $base + 10;
$scanInstanceA->save();
$scanInstanceB->lifetime_download_bytes = 20;
$scanInstanceB->traffic_synced_at = $base + 10;
$scanInstanceB->save();
$firstUsageWindow = CloudTrafficPoolAccounting::settlePendingUsage(1, 0);
$secondUsageWindow = CloudTrafficPoolAccounting::settlePendingUsage(
1,
(int) $firstUsageWindow['next_cursor']
);
$usageWrap = CloudTrafficPoolAccounting::settlePendingUsage(
1,
(int) $secondUsageWindow['next_cursor']
);
$assert((int) $firstUsageWindow['next_cursor'] === 57
&& (int) $secondUsageWindow['next_cursor'] === 58
&& (int) $usageWrap['next_cursor'] === 0
&& (int) CloudTrafficInstanceCursor::find(58)->accounted_download_bytes
=== 20,
'The bounded usage scan did not advance past the first selected row.');
$scanFinishedA = $newInstance(59, 16);
$scanFinishedPeriodA = $newPeriod(
$scanFinishedA, 509, 1, 50, $base - 200, $base - 100
);
$scanFinishedPeriodA->status = 'finished';
$scanFinishedPeriodA->save();
$scanFinishedB = $newInstance(60, 17);
$scanFinishedPeriodB = $newPeriod(
$scanFinishedB, 510, 1, 50, $base - 200, $base - 100
);
$scanFinishedPeriodB->status = 'finished';
$scanFinishedPeriodB->save();
$firstPeriodWindow = CloudTrafficPoolAccounting::reconcileDue(
1,
$base,
508
);
$secondPeriodWindow = CloudTrafficPoolAccounting::reconcileDue(
1,
$base,
(int) $firstPeriodWindow['next_cursor']
);
$periodWrap = CloudTrafficPoolAccounting::reconcileDue(
1,
$base,
(int) $secondPeriodWindow['next_cursor']
);
$assert((int) $firstPeriodWindow['next_cursor'] === 509
&& (int) $secondPeriodWindow['next_cursor'] === 510
&& (int) $periodWrap['next_cursor'] === 0
&& CloudTrafficEntitlement::where('service_period_id', 510)->exists(),
'The bounded period scan did not advance past the first selected row.');
$dueBeforePoisonInstance = $newInstance(61, 18);
$dueBeforePoison = $newPeriod(
$dueBeforePoisonInstance,
511,
1,
50,
$base - 200,
$base - 100
);
CloudTrafficPoolAccounting::reconcileServicePeriod(
511, 'test-cross-category:activate', $base - 150
);
$poisonPeriodInstance = $newInstance(62, 19);
$poisonPeriod = $newPeriod(
$poisonPeriodInstance,
600,
1,
50,
$base,
$base
);
$crossCategoryWindow = CloudTrafficPoolAccounting::reconcileDue(
1,
$base,
0
);
$assert((int) $crossCategoryWindow['next_cursor'] === 511
&& (string) CloudTrafficEntitlement::where(
'service_period_id', 511
)->first()->state === 'expired',
'A higher-ID missing period starved a lower-ID due entitlement.');
$poisonWindow = CloudTrafficPoolAccounting::reconcileDue(
1,
$base,
511
);
$poisonRetry = CloudTrafficPoolAccounting::reconcileDue(1, $base, 0);
$assert(count($poisonWindow['errors']) === 1
&& Capsule::table('cloud_traffic_scan_failures')
->where('scan_kind', 'period')->where('subject_id', 600)->exists()
&& $poisonRetry['processed'] === 0
&& $poisonRetry['errors'] === [],
'A deterministic period error was not quarantined across scan restarts.');
$poisonUsageInstance = $newInstance(63, 20);
$poisonUsagePeriod = $newPeriod(
$poisonUsageInstance,
601,
1,
50,
$base,
$base
);
$poisonUsageInstance->lifetime_download_bytes = 10;
$poisonUsageInstance->traffic_synced_at = $base + 1;
$poisonUsageInstance->save();
$healthyUsageInstance = $newInstance(64, 20);
$healthyUsagePeriod = $newPeriod(
$healthyUsageInstance,
602,
1,
50,
$base,
$base + 500
);
CloudTrafficPoolAccounting::reconcileServicePeriod(
602, 'test-usage-quarantine:healthy', $base + 1
);
$healthyUsageInstance->lifetime_download_bytes = 10;
$healthyUsageInstance->traffic_synced_at = $base + 1;
$healthyUsageInstance->save();
$badUsageWindow = CloudTrafficPoolAccounting::settlePendingUsage(1, 0);
$goodUsageWindow = CloudTrafficPoolAccounting::settlePendingUsage(
1,
(int) $badUsageWindow['next_cursor']
);
$usageRestart = CloudTrafficPoolAccounting::settlePendingUsage(1, 0);
$assert(count($badUsageWindow['errors']) === 1
&& (int) $goodUsageWindow['next_cursor'] === 64
&& Capsule::table('cloud_traffic_scan_failures')
->where('scan_kind', 'instance')->where('subject_id', 63)->exists()
&& $usageRestart['errors'] === [],
'A deterministic instance error starved later usage after a restart.');
$idleScan = CloudTrafficPoolAccounting::settlePendingUsage(1);
$assert($idleScan['processed'] === 0,
'An instance without a trusted sample poisoned the bounded scan queue.');

@ -0,0 +1,55 @@
<?php
$lifecycle = file_get_contents(dirname(__DIR__)
. '/app/Services/Cloud/CloudLifecycleService.php');
$accounting = file_get_contents(dirname(__DIR__)
. '/app/Services/Cloud/CloudTrafficPoolAccounting.php');
if (!is_string($lifecycle) || !is_string($accounting)) {
throw new RuntimeException('Unable to read traffic-pool lifecycle sources.');
}
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$sync = strpos($lifecycle, 'CloudTrafficService::sync($batchSize)');
$settle = strpos($lifecycle,
'CloudTrafficPoolAccounting::settlePendingUsage(');
$due = strpos($lifecycle, 'CloudTrafficPoolAccounting::reconcileDue(');
$runtime = strpos($lifecycle,
'(new CloudInstanceRuntimeSyncService())->sync($batchSize)');
$instanceReconcile = strpos($lifecycle,
"\$this->heartbeat?->busy('instance_reconcile')");
$assert($sync !== false && $settle !== false && $due !== false
&& $runtime !== false && $instanceReconcile !== false
&& $sync < $settle && $settle < $due && $due < $runtime
&& $runtime < $instanceReconcile,
'Lifecycle must sync counters, settle usage, reconcile due periods, then '
. 'continue with runtime and per-instance reconciliation.');
$assert(str_contains($lifecycle, 'trafficUsageScanCursor')
&& str_contains($lifecycle, 'trafficPeriodScanCursor')
&& substr_count($lifecycle, "['next_cursor']") >= 2,
'Bounded accounting scans no longer advance past a persistent bad row.');
$privateReconcile = strpos($lifecycle, 'private function reconcile(');
$assert($privateReconcile !== false
&& strpos(substr($lifecycle, $privateReconcile),
'CloudTrafficPoolAccounting::') === false,
'Pool accounting was nested inside the instance-first lifecycle transaction.');
$stage = substr($lifecycle, $settle, $runtime - $settle);
$assert(!str_contains($stage, 'network_status')
&& !str_contains($stage, 'CloudJobService')
&& !str_contains($stage, 'CloudAgentClient')
&& str_contains($stage,
"observeFailure('traffic_pool_accounting_failed')"),
'Shadow accounting changed network policy or lost fail-open observations.');
$assert(!str_contains($accounting, 'CloudAgentClient')
&& !str_contains($accounting, 'network_status')
&& !str_contains($accounting, "'kind' => 'network_suspend'")
&& !str_contains($accounting, "'kind' => 'network_resume'"),
'The shadow accounting module acquired an Agent/network side effect.');
echo "Cloud traffic-pool lifecycle scheduler contract passed.\n";

@ -24,6 +24,7 @@ foreach ([
'cloud_traffic_entitlements',
'cloud_traffic_ledger_events',
'cloud_traffic_instance_cursors',
'cloud_traffic_scan_failures',
'cloud_instance_network_restrictions',
] as $table) {
$assert(str_contains($schema, 'CREATE TABLE ' . $table . ' ('),
@ -41,7 +42,9 @@ $assert(str_contains($schema,
&& str_contains($schema,
'UNIQUE KEY cloud_traffic_ledger_event_key (event_key)')
&& str_contains($schema,
'UNIQUE KEY cloud_traffic_ledger_sequence (pool_id, pool_sequence)'),
'UNIQUE KEY cloud_traffic_ledger_sequence (pool_id, pool_sequence)')
&& str_contains($schema,
'UNIQUE KEY cloud_traffic_scan_failure_subject (scan_kind, subject_id)'),
'Traffic-pool identity, FEFO or immutable ledger indexes are incomplete.');
$assert(str_contains($schema,
@ -51,7 +54,9 @@ $assert(str_contains($schema,
&& str_contains($schema,
'consumed_bytes + forfeited_bytes <= granted_bytes')
&& str_contains($schema,
"reason_code IN ('traffic_pool_exhausted','service_expired','admin_hold')"),
"reason_code IN ('traffic_pool_exhausted','service_expired','admin_hold')")
&& str_contains($schema,
"scan_kind IN ('period','instance')"),
'Traffic-pool state or conservation checks are incomplete.');
$assert(str_contains($schema,
@ -63,7 +68,7 @@ $assert(str_contains($schema,
'Service-period traffic-pool identity is not frozen or indexed.');
$assert(str_contains($setup,
"SCHEMA_VERSION = 'go-agent-wss-v20-traffic-pool-ledger'")
"SCHEMA_VERSION = 'go-agent-wss-v21-traffic-scan-backoff'")
&& str_contains($setup, "'chk_cloud_traffic_entitlement_balance'")
&& str_contains($setup, "'cloud_traffic_ledger_sequence'"),
'CloudSetup schema version or named-object validation is incomplete.');
@ -83,10 +88,19 @@ foreach ($expectedDeletionOrder as $entry) {
'Account deletion order is missing or unsafe at ' . $entry);
$offset = $next;
}
$assert(str_contains($deletion, 'deleteCloudTrafficScanFailures('),
'Account deletion leaves traffic scan quarantine rows behind.');
$deleteParents = strpos($deletion, '$relatedRows = $this->deleteRelatedRecords(');
$deleteFailures = strpos($deletion,
'$trafficScanRows = $this->deleteCloudTrafficScanFailures(');
$assert($deleteParents !== false && $deleteFailures !== false
&& $deleteParents < $deleteFailures,
'Traffic scan quarantine cleanup can race after account deletion.');
foreach ([
'CloudTrafficPoolGroup', 'CloudTrafficPool', 'CloudTrafficEntitlement',
'CloudTrafficLedgerEvent', 'CloudTrafficInstanceCursor',
'CloudTrafficScanFailure',
'CloudInstanceNetworkRestriction',
] as $model) {
$assert(is_file($root . '/app/Models/' . $model . '.php'),

@ -101,7 +101,7 @@ $assert(str_contains($jobs, 'vpc_id BIGINT UNSIGNED DEFAULT NULL')
'Cloud jobs cannot be correlated to VPC or VPC-port work.');
$assert(str_contains($setup,
"private const SCHEMA_VERSION = 'go-agent-wss-v20-traffic-pool-ledger'")
"private const SCHEMA_VERSION = 'go-agent-wss-v21-traffic-scan-backoff'")
&& str_contains($setup, "'name' => 'cloud_vpc_enabled'")
&& str_contains($setup, "'value' => 'false'")
&& str_contains($setup,

@ -92,6 +92,10 @@ foreach (['cloud_instance_network_restrictions',
$table->increments('id'); $table->integer('user_id');
});
}
$schema->create('cloud_traffic_scan_failures', static function (Blueprint $table) {
$table->increments('id'); $table->string('scan_kind');
$table->integer('subject_id');
});
$schema->create('cloud_invoices', static function (Blueprint $table) {
$table->increments('id'); $table->integer('user_id');
});
@ -178,6 +182,10 @@ foreach (['cloud_instance_network_restrictions',
['user_id' => 1], ['user_id' => 3],
]);
}
Capsule::table('cloud_traffic_scan_failures')->insert([
['scan_kind' => 'instance', 'subject_id' => 101],
['scan_kind' => 'instance', 'subject_id' => 301],
]);
Capsule::table('cloud_invoices')->insert([
['id' => 103, 'user_id' => 1], ['id' => 303, 'user_id' => 3],
]);
@ -213,6 +221,9 @@ foreach (['cloud_instances', 'cloud_orders', 'cloud_instance_snapshots',
$assert(Capsule::table($tableName)->pluck('user_id')->all() === [3],
'Cloud account history deletion was not isolated: ' . $tableName);
}
$assert(Capsule::table('cloud_traffic_scan_failures')
->pluck('subject_id')->all() === [301],
'Cloud traffic scan quarantine deletion was not isolated.');
$assert(Capsule::table('cloud_instance_firewall_versions')->pluck('id')->all()
=== [306]
&& Capsule::table('cloud_instance_firewall_rules')->pluck('id')->all()

Loading…
Cancel
Save