diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index 8630fb33..9926d438 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -30,6 +30,7 @@ final class CloudWorkerService private bool $policyOnly; private bool $vpcBatchOnly; private array $observedErrors = []; + private CloudServiceCompensationExecutionService $compensations; public function observationErrors(): array { @@ -37,7 +38,8 @@ final class CloudWorkerService } public function __construct(private ?CloudAgentClient $agent = null, - bool $policyOnly = false, bool $vpcBatchOnly = false) + bool $policyOnly = false, bool $vpcBatchOnly = false, + ?CloudServiceCompensationExecutionService $compensations = null) { if ($policyOnly && $vpcBatchOnly) { throw new \InvalidArgumentException( @@ -46,6 +48,8 @@ final class CloudWorkerService } $this->policyOnly = $policyOnly; $this->vpcBatchOnly = $vpcBatchOnly; + $this->compensations = $compensations + ?? new CloudServiceCompensationExecutionService(); // A cloud-image clone can legitimately take minutes on remote storage. // The layered deadlines are Agent API 840s, Agent WSS control 855s, // Gateway 870s, PHP 900s and worker lease 930s. Each outer layer has @@ -140,9 +144,20 @@ final class CloudWorkerService // the dedicated --recovery-only worker, so this worker never waits on // an offline recovery target before dispatching ordinary work. $jobWorked = !$quarantined && $this->tickJob(); + $compensationWorked = false; $templateCleanup = false; $templateTemporaryCleanup = false; if (!$this->policyOnly && !$this->vpcBatchOnly) { + try { + // This is a PHP-only ledger mutation. The execution service + // owns its bounded lease and idempotency evidence; the worker + // advances at most one item and never replays an Agent call. + $compensationWorked = $this->compensations->tick(); + } catch (\Throwable $error) { + // Ordinary Agent work already advanced above and remains + // independent from a temporarily unavailable local ledger. + $this->observedErrors[] = 'service_compensation_failed'; + } try { $templateCleanup = (new CloudTemplateImportCancellation()) ->cleanupOneExpired(); @@ -161,7 +176,8 @@ final class CloudWorkerService 'template_import_temporary_cleanup_failed'; } } - return $quarantined || $jobWorked || $templateCleanup + return $quarantined || $jobWorked || $compensationWorked + || $templateCleanup || $templateTemporaryCleanup || $expired > 0; } diff --git a/web/tests/CloudRuntimeHeartbeatTest.php b/web/tests/CloudRuntimeHeartbeatTest.php index 85af4693..85d8419d 100644 --- a/web/tests/CloudRuntimeHeartbeatTest.php +++ b/web/tests/CloudRuntimeHeartbeatTest.php @@ -433,6 +433,43 @@ try { $table->integer('user_id')->default(0); $table->integer('created_at')->default(0); }); + $schema->create( + 'cloud_service_compensations', + static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('instance_count'); + $table->integer('applied_count')->default(0); + $table->integer('deferred_count')->default(0); + $table->integer('rejected_count')->default(0); + $table->integer('attention_count')->default(0); + $table->string('state'); + $table->string('last_error_code')->default(''); + $table->integer('terminal_at')->nullable(); + $table->integer('updated_at'); + } + ); + $schema->create( + 'cloud_service_compensation_items', + static function (Blueprint $table): void { + $table->increments('id'); + $table->string('item_key'); + $table->integer('batch_id'); + $table->integer('instance_id'); + $table->string('state'); + $table->integer('attempts')->default(0); + $table->integer('available_at'); + $table->string('lease_token')->nullable(); + $table->integer('lease_until')->default(0); + $table->integer('audit_job_id')->nullable(); + $table->integer('before_expires_at')->nullable(); + $table->integer('after_expires_at')->nullable(); + $table->integer('applied_at')->nullable(); + $table->integer('terminal_at')->nullable(); + $table->string('last_error_code')->default(''); + $table->string('last_error')->default(''); + $table->integer('updated_at'); + } + ); foreach (['normal', 'policy', 'vpc', 'recovery', 'lifecycle'] as $kind) { $commandDirectory = $testRoot . '/command-' . $kind; $heartbeat = CloudRuntimeHeartbeat::start($kind, true, $commandDirectory, $clock); diff --git a/web/tests/CloudServiceCompensationWorkerContractTest.php b/web/tests/CloudServiceCompensationWorkerContractTest.php new file mode 100644 index 00000000..ac1ecd5c --- /dev/null +++ b/web/tests/CloudServiceCompensationWorkerContractTest.php @@ -0,0 +1,66 @@ +tickJob();'); +$compensation = strpos( + $tick, + '$compensationWorked = $this->compensations->tick();' +); +$assert($job !== false && $compensation !== false && $job < $compensation, + 'A compensation failure may prevent ordinary Agent work in the same tick.'); +$assert(str_contains($tick, 'if (!$this->policyOnly && !$this->vpcBatchOnly)') + && $compensation > strpos( + $tick, + 'if (!$this->policyOnly && !$this->vpcBatchOnly)' + ), + 'Policy-only or VPC-only workers may execute service compensation.'); +$assert(str_contains( + $tick, + "\$this->observedErrors[] = 'service_compensation_failed';" + ) && str_contains($tick, 'catch (\\Throwable $error)') + && str_contains($tick, '|| $compensationWorked'), + 'Compensation work or its stable observation error is not projected.'); +$assert(substr_count( + $tick, + '$compensationWorked = $this->compensations->tick();' + ) === 1 + && !str_contains($tick, 'while ($this->compensations') + && !str_contains($tick, 'do { $this->compensations'), + 'One worker tick may claim more than one compensation item.'); + +echo "Service compensation worker contracts passed.\n";