feat(cloud): schedule service compensation work

go
codex 2 weeks ago
parent 310ed83989
commit 4e7533deb4

@ -30,6 +30,7 @@ final class CloudWorkerService
private bool $policyOnly;
private bool $vpcBatchOnly;
private array $observedErrors = [];
private CloudServiceCompensationExecutionService $compensations;
public function observationErrors(): array
{
@ -37,7 +38,8 @@ final class CloudWorkerService
}
public function __construct(private ?CloudAgentClient $agent = null,
bool $policyOnly = false, bool $vpcBatchOnly = false)
bool $policyOnly = false, bool $vpcBatchOnly = false,
?CloudServiceCompensationExecutionService $compensations = null)
{
if ($policyOnly && $vpcBatchOnly) {
throw new \InvalidArgumentException(
@ -46,6 +48,8 @@ final class CloudWorkerService
}
$this->policyOnly = $policyOnly;
$this->vpcBatchOnly = $vpcBatchOnly;
$this->compensations = $compensations
?? new CloudServiceCompensationExecutionService();
// A cloud-image clone can legitimately take minutes on remote storage.
// The layered deadlines are Agent API 840s, Agent WSS control 855s,
// Gateway 870s, PHP 900s and worker lease 930s. Each outer layer has
@ -140,9 +144,20 @@ final class CloudWorkerService
// the dedicated --recovery-only worker, so this worker never waits on
// an offline recovery target before dispatching ordinary work.
$jobWorked = !$quarantined && $this->tickJob();
$compensationWorked = false;
$templateCleanup = false;
$templateTemporaryCleanup = false;
if (!$this->policyOnly && !$this->vpcBatchOnly) {
try {
// This is a PHP-only ledger mutation. The execution service
// owns its bounded lease and idempotency evidence; the worker
// advances at most one item and never replays an Agent call.
$compensationWorked = $this->compensations->tick();
} catch (\Throwable $error) {
// Ordinary Agent work already advanced above and remains
// independent from a temporarily unavailable local ledger.
$this->observedErrors[] = 'service_compensation_failed';
}
try {
$templateCleanup = (new CloudTemplateImportCancellation())
->cleanupOneExpired();
@ -161,7 +176,8 @@ final class CloudWorkerService
'template_import_temporary_cleanup_failed';
}
}
return $quarantined || $jobWorked || $templateCleanup
return $quarantined || $jobWorked || $compensationWorked
|| $templateCleanup
|| $templateTemporaryCleanup || $expired > 0;
}

@ -433,6 +433,43 @@ try {
$table->integer('user_id')->default(0);
$table->integer('created_at')->default(0);
});
$schema->create(
'cloud_service_compensations',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('instance_count');
$table->integer('applied_count')->default(0);
$table->integer('deferred_count')->default(0);
$table->integer('rejected_count')->default(0);
$table->integer('attention_count')->default(0);
$table->string('state');
$table->string('last_error_code')->default('');
$table->integer('terminal_at')->nullable();
$table->integer('updated_at');
}
);
$schema->create(
'cloud_service_compensation_items',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('item_key');
$table->integer('batch_id');
$table->integer('instance_id');
$table->string('state');
$table->integer('attempts')->default(0);
$table->integer('available_at');
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('audit_job_id')->nullable();
$table->integer('before_expires_at')->nullable();
$table->integer('after_expires_at')->nullable();
$table->integer('applied_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->string('last_error_code')->default('');
$table->string('last_error')->default('');
$table->integer('updated_at');
}
);
foreach (['normal', 'policy', 'vpc', 'recovery', 'lifecycle'] as $kind) {
$commandDirectory = $testRoot . '/command-' . $kind;
$heartbeat = CloudRuntimeHeartbeat::start($kind, true, $commandDirectory, $clock);

@ -0,0 +1,66 @@
<?php
declare(strict_types=1);
$source = file_get_contents(
dirname(__DIR__) . '/app/Services/Cloud/CloudWorkerService.php'
);
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$assert(is_string($source), 'Cloud worker source could not be read.');
$constructorStart = strpos($source, 'public function __construct(');
$constructorEnd = $constructorStart === false ? false : strpos(
$source,
'private function agentForJob(',
$constructorStart
);
$constructor = $constructorStart === false || $constructorEnd === false
? '' : substr($source, $constructorStart, $constructorEnd - $constructorStart);
$assert(str_contains(
$constructor,
'?CloudServiceCompensationExecutionService $compensations = null'
) && str_contains(
$constructor,
'new CloudServiceCompensationExecutionService()'
),
'Normal cloud worker does not own an injectable compensation executor.');
$tickStart = strpos($source, 'public function tick(): bool');
$tickEnd = $tickStart === false ? false : strpos(
$source,
'private static function quarantineInvalidExecutionModeJob(',
$tickStart
);
$tick = $tickStart === false || $tickEnd === false
? '' : substr($source, $tickStart, $tickEnd - $tickStart);
$job = strpos($tick, '$jobWorked = !$quarantined && $this->tickJob();');
$compensation = strpos(
$tick,
'$compensationWorked = $this->compensations->tick();'
);
$assert($job !== false && $compensation !== false && $job < $compensation,
'A compensation failure may prevent ordinary Agent work in the same tick.');
$assert(str_contains($tick, 'if (!$this->policyOnly && !$this->vpcBatchOnly)')
&& $compensation > strpos(
$tick,
'if (!$this->policyOnly && !$this->vpcBatchOnly)'
),
'Policy-only or VPC-only workers may execute service compensation.');
$assert(str_contains(
$tick,
"\$this->observedErrors[] = 'service_compensation_failed';"
) && str_contains($tick, 'catch (\\Throwable $error)')
&& str_contains($tick, '|| $compensationWorked'),
'Compensation work or its stable observation error is not projected.');
$assert(substr_count(
$tick,
'$compensationWorked = $this->compensations->tick();'
) === 1
&& !str_contains($tick, 'while ($this->compensations')
&& !str_contains($tick, 'do { $this->compensations'),
'One worker tick may claim more than one compensation item.');
echo "Service compensation worker contracts passed.\n";
Loading…
Cancel
Save