|
|
|
|
<?php
|
|
|
|
|
declare(strict_types=1);
|
|
|
|
|
|
|
|
|
|
namespace App\Services\Cloud {
|
|
|
|
|
/** In-memory envelope for the isolated deadline/finalization test. */
|
|
|
|
|
final class CloudSecrets
|
|
|
|
|
{
|
|
|
|
|
private static array $values = [];
|
|
|
|
|
|
|
|
|
|
public static function encrypt(array $value, string $purpose): string
|
|
|
|
|
{
|
|
|
|
|
$key = bin2hex(random_bytes(16));
|
|
|
|
|
self::$values[$key] = $value;
|
|
|
|
|
return 'deadline-test:' . $key;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static function decrypt(string $value, string $purpose): array
|
|
|
|
|
{
|
|
|
|
|
$key = substr($value, strlen('deadline-test:'));
|
|
|
|
|
if (!array_key_exists($key, self::$values)) {
|
|
|
|
|
throw new \RuntimeException('Missing test payload');
|
|
|
|
|
}
|
|
|
|
|
return self::$values[$key];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudAgentCompletionEvent;
|
|
|
|
|
use App\Models\CloudJob;
|
|
|
|
|
use App\Services\Cloud\CloudAgentClient;
|
|
|
|
|
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
|
|
|
|
|
use App\Services\Cloud\CloudJobService;
|
|
|
|
|
use App\Services\Cloud\CloudOperationCompletionService;
|
|
|
|
|
use App\Services\Cloud\CloudOperationRecoveryIdentity;
|
|
|
|
|
use App\Services\Cloud\CloudOperationRecoveryPolicy;
|
|
|
|
|
use App\Services\Cloud\CloudOperationRecoveryService;
|
|
|
|
|
use App\Services\Cloud\CloudSecrets;
|
|
|
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
|
|
|
use GuzzleHttp\Client;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
|
|
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
|
|
|
if (!$condition) {
|
|
|
|
|
throw new RuntimeException($message);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$capsule = new Capsule();
|
|
|
|
|
$capsule->addConnection([
|
|
|
|
|
'driver' => 'sqlite',
|
|
|
|
|
'database' => ':memory:',
|
|
|
|
|
], 'default');
|
|
|
|
|
$capsule->setAsGlobal();
|
|
|
|
|
$capsule->bootEloquent();
|
|
|
|
|
$schema = $capsule->schema('default');
|
|
|
|
|
$schema->create('user', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->decimal('money', 12, 2)->default(0);
|
|
|
|
|
$table->integer('enable')->default(1);
|
|
|
|
|
$table->integer('is_admin')->default(0);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('operation_id')->unique();
|
|
|
|
|
$table->string('request_idempotency_key')->unique();
|
|
|
|
|
$table->string('request_fingerprint');
|
|
|
|
|
$table->integer('parent_job_id')->nullable();
|
|
|
|
|
$table->integer('order_id')->nullable();
|
|
|
|
|
$table->integer('invoice_id')->nullable();
|
|
|
|
|
$table->integer('instance_id')->nullable();
|
|
|
|
|
$table->integer('snapshot_id')->nullable();
|
|
|
|
|
$table->integer('vpc_repair_root_job_id')->nullable();
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->integer('host_id')->nullable();
|
|
|
|
|
$table->string('kind');
|
|
|
|
|
$table->string('execution_mode')->default('agent');
|
|
|
|
|
$table->string('phase');
|
|
|
|
|
$table->string('outcome')->nullable();
|
|
|
|
|
$table->string('status');
|
|
|
|
|
$table->text('sealed_payload');
|
|
|
|
|
$table->integer('attempts')->default(0);
|
|
|
|
|
$table->integer('dispatch_attempts')->default(0);
|
|
|
|
|
$table->integer('dispatch_deadline_at')->nullable();
|
|
|
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
|
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
|
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
|
|
|
$table->integer('terminal_at')->nullable();
|
|
|
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
|
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
|
|
|
$table->string('effect_resolution')->default('not_applicable');
|
|
|
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
|
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
|
|
|
$table->string('effect_resolution_reason')->default('');
|
|
|
|
|
$table->string('completion_event_id')->nullable();
|
|
|
|
|
$table->integer('read_at')->nullable();
|
|
|
|
|
$table->integer('dispatched_at')->nullable();
|
|
|
|
|
$table->string('lease_token')->nullable();
|
|
|
|
|
$table->integer('lease_until')->default(0);
|
|
|
|
|
$table->integer('available_at')->default(0);
|
|
|
|
|
$table->string('error_code')->nullable();
|
|
|
|
|
$table->boolean('finalization_incident')->default(0);
|
|
|
|
|
$table->string('finalization_error_code')->nullable();
|
|
|
|
|
$table->text('result_json')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('uuid');
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->integer('order_id')->nullable();
|
|
|
|
|
$table->integer('plan_id')->nullable();
|
|
|
|
|
$table->integer('offer_id')->nullable();
|
|
|
|
|
$table->integer('resource_pool_id')->nullable();
|
|
|
|
|
$table->integer('reservation_id')->nullable();
|
|
|
|
|
$table->integer('current_period_id')->nullable();
|
|
|
|
|
$table->string('name');
|
|
|
|
|
$table->string('status');
|
|
|
|
|
$table->string('service_status');
|
|
|
|
|
$table->string('network_status');
|
|
|
|
|
$table->integer('expires_at')->nullable();
|
|
|
|
|
$table->decimal('renewal_price', 12, 2)->nullable();
|
|
|
|
|
$table->integer('renewal_days')->default(0);
|
|
|
|
|
$table->integer('renewal_traffic_bytes')->default(0);
|
|
|
|
|
$table->integer('traffic_quota_bytes')->default(0);
|
|
|
|
|
$table->integer('traffic_download_bytes')->default(0);
|
|
|
|
|
$table->integer('traffic_upload_bytes')->default(0);
|
|
|
|
|
$table->integer('lifetime_download_bytes')->default(0);
|
|
|
|
|
$table->integer('lifetime_upload_bytes')->default(0);
|
|
|
|
|
$table->integer('traffic_synced_at')->default(0);
|
|
|
|
|
$table->integer('suspended_at')->nullable();
|
|
|
|
|
$table->integer('network_suspended_at')->nullable();
|
|
|
|
|
$table->text('spec_json');
|
|
|
|
|
$table->text('sealed_credential')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_host_storage_configurations',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id')->primary();
|
|
|
|
|
$table->integer('revision')->default(0);
|
|
|
|
|
foreach (['managed', 'template', 'seed', 'user_iso', 'rescue']
|
|
|
|
|
as $role) {
|
|
|
|
|
$table->text($role . '_directory');
|
|
|
|
|
}
|
|
|
|
|
$table->string('state')->default('applying');
|
|
|
|
|
$table->string('operation_id')->nullable()->unique();
|
|
|
|
|
$table->integer('job_id')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_agent_completion_events',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->string('event_id')->primary();
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('operation_id');
|
|
|
|
|
$table->string('payload_sha256');
|
|
|
|
|
$table->string('outcome');
|
|
|
|
|
$table->boolean('effect_may_have_applied');
|
|
|
|
|
$table->boolean('safe_to_resubmit');
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->text('result_json')->nullable();
|
|
|
|
|
$table->string('disposition')->default('received');
|
|
|
|
|
$table->integer('received_at');
|
|
|
|
|
$table->integer('applied_at')->nullable();
|
|
|
|
|
$table->unique(['host_id', 'operation_id']);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_templates',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->string('id')->primary();
|
|
|
|
|
$table->string('catalog_key')->default('test-catalog');
|
|
|
|
|
$table->string('version_label')->default('test-version');
|
|
|
|
|
$table->boolean('is_default')->default(true);
|
|
|
|
|
$table->integer('active_job_id')->nullable();
|
|
|
|
|
$table->string('state')->default('active');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_user_isos',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->string('id')->primary();
|
|
|
|
|
$table->integer('active_job_id')->nullable();
|
|
|
|
|
$table->string('state')->default('active');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_template_replicas',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('template_id');
|
|
|
|
|
$table->integer('revision')->default(0);
|
|
|
|
|
$table->string('generation_token')->default('');
|
|
|
|
|
$table->string('state')->default('pending');
|
|
|
|
|
$table->integer('image_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->string('operation_id')->default('');
|
|
|
|
|
$table->integer('job_id')->nullable();
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('created_at')->default(0);
|
|
|
|
|
$table->integer('updated_at')->default(0);
|
|
|
|
|
$table->primary(['host_id', 'template_id']);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_user_iso_replicas',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('user_iso_id');
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->string('state')->default('pending');
|
|
|
|
|
$table->integer('size_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->string('operation_id')->default('');
|
|
|
|
|
$table->integer('job_id')->nullable();
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('created_at')->default(0);
|
|
|
|
|
$table->integer('updated_at')->default(0);
|
|
|
|
|
$table->primary(['host_id', 'user_iso_id']);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_invoice_payments',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->integer('invoice_id');
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->string('provider');
|
|
|
|
|
$table->string('provider_order_no');
|
|
|
|
|
$table->string('idempotency_key');
|
|
|
|
|
$table->decimal('amount', 12, 2)->default(0);
|
|
|
|
|
$table->string('status');
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
$table->unique(['user_id', 'idempotency_key']);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
Capsule::table('user')->insert([
|
|
|
|
|
['id' => 1, 'money' => 0, 'enable' => 1, 'is_admin' => 0],
|
|
|
|
|
['id' => 2, 'money' => 0, 'enable' => 1, 'is_admin' => 0],
|
|
|
|
|
]);
|
|
|
|
|
$now = time();
|
|
|
|
|
$insertJob = static function (int $userId, string $operationId,
|
|
|
|
|
string $phase, int $dispatchAttempts) use ($now): int {
|
|
|
|
|
return (int) Capsule::table('cloud_jobs')->insertGetId([
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'request_idempotency_key' => $operationId,
|
|
|
|
|
'request_fingerprint' => hash('sha256', $operationId),
|
|
|
|
|
'parent_job_id' => null,
|
|
|
|
|
'order_id' => null,
|
|
|
|
|
'invoice_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'snapshot_id' => null,
|
|
|
|
|
'user_id' => $userId,
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
// An unrecognised kind exercises the generic terminal ledger path
|
|
|
|
|
// without requiring a business-resource fixture.
|
|
|
|
|
'kind' => 'test_mutation',
|
|
|
|
|
'execution_mode' => 'agent',
|
|
|
|
|
'phase' => $phase,
|
|
|
|
|
'outcome' => null,
|
|
|
|
|
'status' => $phase === 'accepted' ? 'queued' : 'running',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/test',
|
|
|
|
|
'body' => ['operation_id' => $operationId],
|
|
|
|
|
'owner_id' => $userId,
|
|
|
|
|
// Crossed-dispatch failures must retain one encrypted copy
|
|
|
|
|
// until their immutable Agent receipt resolves the effect.
|
|
|
|
|
'credential' => [
|
|
|
|
|
'username' => 'root',
|
|
|
|
|
'password' => 'deadline-recovery-secret',
|
|
|
|
|
],
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
'attempts' => $dispatchAttempts,
|
|
|
|
|
'dispatch_attempts' => $dispatchAttempts,
|
|
|
|
|
'dispatch_deadline_at' => $now - 20,
|
|
|
|
|
'dispatch_started_at' => $dispatchAttempts === 1
|
|
|
|
|
? $now - 30 : null,
|
|
|
|
|
'agent_accepted_at' => null,
|
|
|
|
|
'receipt_deadline_at' => $dispatchAttempts === 1
|
|
|
|
|
? $now - 10 : null,
|
|
|
|
|
'terminal_at' => null,
|
|
|
|
|
'effect_may_have_applied' => null,
|
|
|
|
|
'safe_to_resubmit' => null,
|
|
|
|
|
'effect_resolution' => 'not_applicable',
|
|
|
|
|
'effect_resolved_at' => null,
|
|
|
|
|
'effect_resolved_by' => null,
|
|
|
|
|
'effect_resolution_reason' => '',
|
|
|
|
|
'completion_event_id' => null,
|
|
|
|
|
'read_at' => null,
|
|
|
|
|
'dispatched_at' => $dispatchAttempts === 1 ? $now - 30 : null,
|
|
|
|
|
// Simulate a dead daemon whose stale lease extends past deadline.
|
|
|
|
|
'lease_token' => str_repeat('f', 32),
|
|
|
|
|
'lease_until' => $now + 600,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'error_code' => null,
|
|
|
|
|
'result_json' => null,
|
|
|
|
|
'created_at' => $now - 60,
|
|
|
|
|
'updated_at' => $now - 30,
|
|
|
|
|
]);
|
|
|
|
|
};
|
|
|
|
|
$insertInstance = static function (
|
|
|
|
|
int $id,
|
|
|
|
|
string $uuid,
|
|
|
|
|
string $name
|
|
|
|
|
) use ($now): void {
|
|
|
|
|
Capsule::table('cloud_instances')->insert([
|
|
|
|
|
'id' => $id,
|
|
|
|
|
'uuid' => $uuid,
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'user_id' => 1,
|
|
|
|
|
'order_id' => null,
|
|
|
|
|
'plan_id' => null,
|
|
|
|
|
'offer_id' => null,
|
|
|
|
|
'resource_pool_id' => null,
|
|
|
|
|
'reservation_id' => null,
|
|
|
|
|
'current_period_id' => null,
|
|
|
|
|
'name' => $name,
|
|
|
|
|
'status' => 'active',
|
|
|
|
|
'service_status' => 'exempt',
|
|
|
|
|
'network_status' => 'active',
|
|
|
|
|
'expires_at' => null,
|
|
|
|
|
'renewal_price' => null,
|
|
|
|
|
'renewal_days' => 0,
|
|
|
|
|
'renewal_traffic_bytes' => 0,
|
|
|
|
|
'traffic_quota_bytes' => 0,
|
|
|
|
|
'traffic_download_bytes' => 0,
|
|
|
|
|
'traffic_upload_bytes' => 0,
|
|
|
|
|
'lifetime_download_bytes' => 0,
|
|
|
|
|
'lifetime_upload_bytes' => 0,
|
|
|
|
|
'traffic_synced_at' => 0,
|
|
|
|
|
'suspended_at' => null,
|
|
|
|
|
'network_suspended_at' => null,
|
|
|
|
|
'spec_json' => json_encode(
|
|
|
|
|
['managed' => true],
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
),
|
|
|
|
|
'sealed_credential' => null,
|
|
|
|
|
'created_at' => $now - 3600,
|
|
|
|
|
'updated_at' => $now - 3600,
|
|
|
|
|
]);
|
|
|
|
|
};
|
|
|
|
|
$storagePayload = static function (string $operationId): string {
|
|
|
|
|
return CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/storage/configuration',
|
|
|
|
|
'body' => [
|
|
|
|
|
'managed_directory' => '/home/libvirt/kvm',
|
|
|
|
|
'template_directory' => '/home/libvirt/images',
|
|
|
|
|
'seed_directory' => '/home/libvirt/seed',
|
|
|
|
|
'user_iso_directory' => '/home/libvirt/iso',
|
|
|
|
|
'rescue_directory' => '/home/libvirt/rescue-iso',
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'storage_revision' => 1,
|
|
|
|
|
], 'job-payload');
|
|
|
|
|
};
|
|
|
|
|
$identityFor = static function (
|
|
|
|
|
string $operationId,
|
|
|
|
|
bool $applied
|
|
|
|
|
): array {
|
|
|
|
|
$job = CloudJob::where('operation_id', $operationId)->firstOrFail();
|
|
|
|
|
if (CloudOperationRecoveryIdentity::supports((string) $job->kind)) {
|
|
|
|
|
return CloudOperationRecoveryIdentity::expected($job, $applied);
|
|
|
|
|
}
|
|
|
|
|
return [
|
|
|
|
|
'action' => 'test-mutation',
|
|
|
|
|
'resource_id' => 'operation-' . substr($operationId, 0, 16),
|
|
|
|
|
'request_fingerprint' => hash('sha256', $operationId),
|
|
|
|
|
];
|
|
|
|
|
};
|
|
|
|
|
$withIdentity = static function (array $receipt) use (
|
|
|
|
|
$identityFor
|
|
|
|
|
): array {
|
|
|
|
|
return $receipt + $identityFor(
|
|
|
|
|
(string) $receipt['operation_id'],
|
|
|
|
|
($receipt['status'] ?? null) === 'succeeded'
|
|
|
|
|
);
|
|
|
|
|
};
|
|
|
|
|
$storedEvent = static function (
|
|
|
|
|
array $receipt,
|
|
|
|
|
int $receivedAt,
|
|
|
|
|
string $disposition = 'received'
|
|
|
|
|
): array {
|
|
|
|
|
$canonical = [
|
|
|
|
|
'event_id' => $receipt['id'],
|
|
|
|
|
'host_id' => (int) $receipt['agent_id'],
|
|
|
|
|
'operation_id' => $receipt['operation_id'],
|
|
|
|
|
'action' => $receipt['action'],
|
|
|
|
|
'resource_id' => $receipt['resource_id'],
|
|
|
|
|
'request_fingerprint' => $receipt['request_fingerprint'],
|
|
|
|
|
'status' => $receipt['status'],
|
|
|
|
|
'effect_may_have_applied' =>
|
|
|
|
|
$receipt['effect_may_have_applied'],
|
|
|
|
|
'safe_to_resubmit' => $receipt['safe_to_resubmit'],
|
|
|
|
|
'error_code' => $receipt['error_code'],
|
|
|
|
|
'result' => $receipt['result'],
|
|
|
|
|
];
|
|
|
|
|
return [
|
|
|
|
|
'event_id' => $canonical['event_id'],
|
|
|
|
|
'host_id' => $canonical['host_id'],
|
|
|
|
|
'operation_id' => $canonical['operation_id'],
|
|
|
|
|
'payload_sha256' => hash('sha256', json_encode(
|
|
|
|
|
$canonical,
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE
|
|
|
|
|
)),
|
|
|
|
|
'outcome' => $canonical['status'],
|
|
|
|
|
'effect_may_have_applied' =>
|
|
|
|
|
$canonical['effect_may_have_applied'] ? 1 : 0,
|
|
|
|
|
'safe_to_resubmit' => $canonical['safe_to_resubmit'] ? 1 : 0,
|
|
|
|
|
'error_code' => $canonical['error_code'],
|
|
|
|
|
'result_json' => json_encode([
|
|
|
|
|
'schema' => 'virt_cloud_completion_event',
|
|
|
|
|
'version' => 1,
|
|
|
|
|
'identity' => [
|
|
|
|
|
'action' => $canonical['action'],
|
|
|
|
|
'resource_id' => $canonical['resource_id'],
|
|
|
|
|
'request_fingerprint' =>
|
|
|
|
|
$canonical['request_fingerprint'],
|
|
|
|
|
],
|
|
|
|
|
'result' => $canonical['result'],
|
|
|
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE),
|
|
|
|
|
'disposition' => $disposition,
|
|
|
|
|
'received_at' => $receivedAt,
|
|
|
|
|
'applied_at' => null,
|
|
|
|
|
];
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$receiptOperation = str_repeat('a', 32);
|
|
|
|
|
$receiptJobId = $insertJob(1, $receiptOperation, 'dispatching', 1);
|
|
|
|
|
$receiptIdentity = [
|
|
|
|
|
'action' => 'test-mutation',
|
|
|
|
|
'resource_id' => 'test-resource',
|
|
|
|
|
'request_fingerprint' => str_repeat('1', 64),
|
|
|
|
|
];
|
|
|
|
|
$receipt = [
|
|
|
|
|
'event_id' => str_repeat('e', 32),
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'operation_id' => $receiptOperation,
|
|
|
|
|
] + $receiptIdentity + [
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => ['confirmed' => true],
|
|
|
|
|
];
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
|
|
|
|
'event_id' => $receipt['event_id'],
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'operation_id' => $receiptOperation,
|
|
|
|
|
'payload_sha256' => hash('sha256', json_encode(
|
|
|
|
|
$receipt,
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE
|
|
|
|
|
)),
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result_json' => json_encode([
|
|
|
|
|
'schema' => 'virt_cloud_completion_event',
|
|
|
|
|
'version' => 1,
|
|
|
|
|
'identity' => $receiptIdentity,
|
|
|
|
|
'result' => $receipt['result'],
|
|
|
|
|
], JSON_THROW_ON_ERROR),
|
|
|
|
|
'disposition' => 'received',
|
|
|
|
|
'received_at' => $now - 15,
|
|
|
|
|
'applied_at' => null,
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
$otherOperation = str_repeat('b', 32);
|
|
|
|
|
$otherJobId = $insertJob(2, $otherOperation, 'dispatching', 1);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1, $receiptJobId, 1
|
|
|
|
|
) === 1, 'The owner-scoped fallback did not claim the due task.');
|
|
|
|
|
|
|
|
|
|
$receiptJob = CloudJob::find($receiptJobId);
|
|
|
|
|
$otherJob = CloudJob::find($otherJobId);
|
|
|
|
|
$event = Capsule::table('cloud_agent_completion_events')
|
|
|
|
|
->where('event_id', str_repeat('e', 32))->first();
|
|
|
|
|
$assert($receiptJob !== null
|
|
|
|
|
&& (string) $receiptJob->phase === 'terminal'
|
|
|
|
|
&& (string) $receiptJob->outcome === 'succeeded'
|
|
|
|
|
&& (int) $receiptJob->dispatch_attempts === 1
|
|
|
|
|
&& (string) $event->disposition === 'applied',
|
|
|
|
|
'A receipt saved during dispatch was replaced by deadline failure.');
|
|
|
|
|
$assert($otherJob !== null && (string) $otherJob->phase === 'dispatching',
|
|
|
|
|
'Owner-scoped expiry mutated another user\'s task.');
|
|
|
|
|
|
|
|
|
|
$failureOperation = str_repeat('c', 32);
|
|
|
|
|
$failureJobId = $insertJob(1, $failureOperation, 'dispatching', 1);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1, $failureJobId, 1
|
|
|
|
|
) === 1, 'The receipt-less due task was not closed.');
|
|
|
|
|
$failed = CloudJob::find($failureJobId);
|
|
|
|
|
$failedRecoveryPayload = CloudSecrets::decrypt(
|
|
|
|
|
(string) $failed->sealed_payload,
|
|
|
|
|
'job-payload'
|
|
|
|
|
);
|
|
|
|
|
$assert($failed !== null
|
|
|
|
|
&& (string) $failed->phase === 'terminal'
|
|
|
|
|
&& (string) $failed->outcome === 'failed'
|
|
|
|
|
&& (bool) $failed->effect_may_have_applied
|
|
|
|
|
&& !(bool) $failed->safe_to_resubmit
|
|
|
|
|
&& !array_key_exists('credential', $failedRecoveryPayload),
|
|
|
|
|
'An unsupported unknown-effect kind retained secret recovery escrow.');
|
|
|
|
|
|
|
|
|
|
// dispatch_attempts is a durable 0/1 fence, not a retry counter. Values
|
|
|
|
|
// outside that state machine must fail closed: neither deadline handling
|
|
|
|
|
// nor its last-resort finalizer may reinterpret them as "not dispatched".
|
|
|
|
|
$invalidLedgerUuid =
|
|
|
|
|
'00000000-0000-4000-8000-000000000301';
|
|
|
|
|
$insertInstance(301, $invalidLedgerUuid, 'invalid-ledger-settings');
|
|
|
|
|
$invalidLedgerOperation = str_repeat('cd', 16);
|
|
|
|
|
$invalidLedgerJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$invalidLedgerOperation,
|
|
|
|
|
'dispatching',
|
|
|
|
|
-1
|
|
|
|
|
);
|
|
|
|
|
$invalidLedgerPayload = CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $invalidLedgerUuid . '/settings',
|
|
|
|
|
'body' => [
|
|
|
|
|
'hostname' => 'invalid-ledger.example',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'can_manage' => false,
|
|
|
|
|
'settings_fields' => ['hostname'],
|
|
|
|
|
], 'job-payload');
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $invalidLedgerJobId)->update([
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'instance_id' => 301,
|
|
|
|
|
'sealed_payload' => $invalidLedgerPayload,
|
|
|
|
|
'dispatch_started_at' => $now - 30,
|
|
|
|
|
'dispatched_at' => $now - 30,
|
|
|
|
|
'receipt_deadline_at' => $now - 10,
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$invalidLedgerJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The invalid dispatch-ledger deadline was not claimed.');
|
|
|
|
|
$invalidLedgerJob = CloudJob::findOrFail($invalidLedgerJobId);
|
|
|
|
|
$assert((string) $invalidLedgerJob->phase === 'terminal'
|
|
|
|
|
&& (string) $invalidLedgerJob->outcome === 'failed'
|
|
|
|
|
&& (int) $invalidLedgerJob->dispatch_attempts === -1
|
|
|
|
|
&& (bool) $invalidLedgerJob->effect_may_have_applied
|
|
|
|
|
&& !(bool) $invalidLedgerJob->safe_to_resubmit
|
|
|
|
|
&& (string) $invalidLedgerJob->effect_resolution === 'unknown'
|
|
|
|
|
&& (bool) $invalidLedgerJob->finalization_incident
|
|
|
|
|
&& (string) $invalidLedgerJob->finalization_error_code
|
|
|
|
|
=== 'recovery_dispatch_ledger_conflict'
|
|
|
|
|
&& (string) $invalidLedgerJob->sealed_payload
|
|
|
|
|
=== $invalidLedgerPayload,
|
|
|
|
|
'A negative dispatch ledger was treated as a safe undispatched failure.');
|
|
|
|
|
|
|
|
|
|
// Exercise the same invariant through forceTerminalFailure(): the linked
|
|
|
|
|
// instance is intentionally absent so the ordinary policy finalizer fails
|
|
|
|
|
// and the last-resort path owns the durable incident write.
|
|
|
|
|
$forcedLedgerOperation = str_repeat('ce', 16);
|
|
|
|
|
$forcedLedgerJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$forcedLedgerOperation,
|
|
|
|
|
'dispatching',
|
|
|
|
|
2
|
|
|
|
|
);
|
|
|
|
|
$forcedLedgerPayload = CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/'
|
|
|
|
|
. '00000000-0000-4000-8000-000000009999/suspend',
|
|
|
|
|
'body' => [],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'can_manage' => false,
|
|
|
|
|
], 'job-payload');
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $forcedLedgerJobId)->update([
|
|
|
|
|
'kind' => 'service_suspend',
|
|
|
|
|
'instance_id' => 9999,
|
|
|
|
|
'sealed_payload' => $forcedLedgerPayload,
|
|
|
|
|
'dispatch_started_at' => $now - 30,
|
|
|
|
|
'dispatched_at' => $now - 30,
|
|
|
|
|
'receipt_deadline_at' => $now - 10,
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$forcedLedgerJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The forced invalid dispatch-ledger deadline was not claimed.');
|
|
|
|
|
$forcedLedgerJob = CloudJob::findOrFail($forcedLedgerJobId);
|
|
|
|
|
$assert((string) $forcedLedgerJob->phase === 'terminal'
|
|
|
|
|
&& (string) $forcedLedgerJob->outcome === 'failed'
|
|
|
|
|
&& (int) $forcedLedgerJob->dispatch_attempts === 2
|
|
|
|
|
&& (bool) $forcedLedgerJob->effect_may_have_applied
|
|
|
|
|
&& !(bool) $forcedLedgerJob->safe_to_resubmit
|
|
|
|
|
&& (string) $forcedLedgerJob->effect_resolution === 'unknown'
|
|
|
|
|
&& (bool) $forcedLedgerJob->finalization_incident
|
|
|
|
|
&& (string) $forcedLedgerJob->finalization_error_code
|
|
|
|
|
=== 'recovery_dispatch_ledger_conflict'
|
|
|
|
|
&& (string) $forcedLedgerJob->sealed_payload
|
|
|
|
|
=== $forcedLedgerPayload,
|
|
|
|
|
'The forced finalizer concealed or unlocked an invalid dispatch ledger.');
|
|
|
|
|
|
|
|
|
|
// Deploying the shorter storage window must also bound tasks which already
|
|
|
|
|
// persisted the old 960-second receipt deadline. No data rewrite is needed:
|
|
|
|
|
// the effective deadline is the earlier of the two absolute boundaries.
|
|
|
|
|
$legacyStorageOperation = str_repeat('3', 32);
|
|
|
|
|
$legacyStorageJobId = $insertJob(
|
|
|
|
|
1, $legacyStorageOperation, 'awaiting_receipt', 1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $legacyStorageJobId)->update([
|
|
|
|
|
'kind' => 'host_storage_configure',
|
|
|
|
|
'sealed_payload' => $storagePayload($legacyStorageOperation),
|
|
|
|
|
'dispatched_at' => $now - 181,
|
|
|
|
|
'dispatch_started_at' => $now - 181,
|
|
|
|
|
'receipt_deadline_at' => $now + 779,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_host_storage_configurations')->insert([
|
|
|
|
|
'host_id' => 7, 'revision' => 1,
|
|
|
|
|
'managed_directory' => '/home/libvirt/kvm',
|
|
|
|
|
'template_directory' => '/home/libvirt/images',
|
|
|
|
|
'seed_directory' => '/home/libvirt/seed',
|
|
|
|
|
'user_iso_directory' => '/home/libvirt/iso',
|
|
|
|
|
'rescue_directory' => '/home/libvirt/rescue-iso',
|
|
|
|
|
'state' => 'applying', 'operation_id' => $legacyStorageOperation,
|
|
|
|
|
'job_id' => $legacyStorageJobId,
|
|
|
|
|
'created_at' => $now - 200, 'updated_at' => $now - 181,
|
|
|
|
|
]);
|
|
|
|
|
$legacyStorage = CloudJob::find($legacyStorageJobId);
|
|
|
|
|
$assert(CloudJobService::effectiveReceiptDeadline($legacyStorage)
|
|
|
|
|
=== $now - 1
|
|
|
|
|
&& CloudJobService::receiptDeadlineReached($legacyStorage, $now),
|
|
|
|
|
'The persisted storage deadline was not bounded by dispatch plus 180 seconds.');
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1, $legacyStorageJobId, 1
|
|
|
|
|
) === 1, 'An old 960-second storage task remained active after 180 seconds.');
|
|
|
|
|
$legacyStorage = CloudJob::find($legacyStorageJobId);
|
|
|
|
|
$legacyStorageRow = Capsule::table(
|
|
|
|
|
'cloud_host_storage_configurations'
|
|
|
|
|
)->where('host_id', 7)->first();
|
|
|
|
|
$assert((string) $legacyStorage->phase === 'terminal'
|
|
|
|
|
&& (string) $legacyStorage->outcome === 'failed'
|
|
|
|
|
&& (string) $legacyStorageRow->state === 'failed',
|
|
|
|
|
'Storage deadline fallback did not close the task and its projection.');
|
|
|
|
|
|
|
|
|
|
// Receipt classification must use exactly the same effective boundary as
|
|
|
|
|
// the expiry query, otherwise a late success could revive a timed-out task.
|
|
|
|
|
$lateStorageOperation = str_repeat('2', 32);
|
|
|
|
|
$lateStorageJobId = $insertJob(
|
|
|
|
|
1, $lateStorageOperation, 'awaiting_receipt', 1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $lateStorageJobId)->update([
|
|
|
|
|
'kind' => 'host_storage_configure',
|
|
|
|
|
'sealed_payload' => $storagePayload($lateStorageOperation),
|
|
|
|
|
'dispatched_at' => $now - 181,
|
|
|
|
|
'dispatch_started_at' => $now - 181,
|
|
|
|
|
'receipt_deadline_at' => $now + 779,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_host_storage_configurations')
|
|
|
|
|
->where('host_id', 7)->update([
|
|
|
|
|
'state' => 'applying', 'operation_id' => $lateStorageOperation,
|
|
|
|
|
'job_id' => $lateStorageJobId, 'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
$lateStorageEventId = str_repeat('4', 32);
|
|
|
|
|
$lateStorageReceipt = $withIdentity([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => $lateStorageEventId,
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => $lateStorageOperation,
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => ['configured' => true],
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->insert(
|
|
|
|
|
$storedEvent($lateStorageReceipt, $now - 1)
|
|
|
|
|
);
|
|
|
|
|
$lateStorage = (new CloudWorkerService())->settleCompletionReceipt(
|
|
|
|
|
$lateStorageJobId, $lateStorageEventId
|
|
|
|
|
);
|
|
|
|
|
$lateStorageEvent = Capsule::table('cloud_agent_completion_events')
|
|
|
|
|
->where('event_id', $lateStorageEventId)->first();
|
|
|
|
|
$assert((string) $lateStorage->phase === 'terminal'
|
|
|
|
|
&& (string) $lateStorage->outcome === 'failed'
|
|
|
|
|
&& (string) $lateStorageEvent->disposition === 'late',
|
|
|
|
|
'A storage receipt at the effective deadline was not classified as late.');
|
|
|
|
|
|
|
|
|
|
$acceptedStorageOperation = str_repeat('1', 32);
|
|
|
|
|
$acceptedStorageJobId = $insertJob(
|
|
|
|
|
1, $acceptedStorageOperation, 'accepted', 0
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $acceptedStorageJobId)->update([
|
|
|
|
|
'kind' => 'host_storage_configure',
|
|
|
|
|
'sealed_payload' => $storagePayload($acceptedStorageOperation),
|
|
|
|
|
'created_at' => $now - 61,
|
|
|
|
|
'dispatch_deadline_at' => $now + 239,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_host_storage_configurations')
|
|
|
|
|
->where('host_id', 7)->update([
|
|
|
|
|
'state' => 'applying', 'operation_id' => $acceptedStorageOperation,
|
|
|
|
|
'job_id' => $acceptedStorageJobId, 'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
$acceptedStorage = CloudJob::find($acceptedStorageJobId);
|
|
|
|
|
$assert(CloudJobService::effectiveDispatchDeadline($acceptedStorage)
|
|
|
|
|
=== $now - 1
|
|
|
|
|
&& CloudJobService::activeDeadlineReached($acceptedStorage, $now),
|
|
|
|
|
'The persisted storage dispatch deadline was not bounded to 60 seconds.');
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1, $acceptedStorageJobId, 1
|
|
|
|
|
) === 1, 'An old 300-second accepted storage task remained active.');
|
|
|
|
|
$acceptedStorage = CloudJob::find($acceptedStorageJobId);
|
|
|
|
|
$assert((string) $acceptedStorage->phase === 'terminal'
|
|
|
|
|
&& (string) $acceptedStorage->outcome === 'failed'
|
|
|
|
|
&& !(bool) $acceptedStorage->effect_may_have_applied
|
|
|
|
|
&& (bool) $acceptedStorage->safe_to_resubmit,
|
|
|
|
|
'An undispatched storage deadline did not become a safe failure.');
|
|
|
|
|
|
|
|
|
|
// The normal worker tick must run local deadline convergence before it can
|
|
|
|
|
// claim an old storage task. Its injected Agent transport is a tripwire:
|
|
|
|
|
// a request here would prove the former 300-second deadline was used.
|
|
|
|
|
$tickStorageOperation = '0123456789abcdef0123456789abcdef';
|
|
|
|
|
$tickStorageJobId = $insertJob(
|
|
|
|
|
1, $tickStorageOperation, 'accepted', 0
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $tickStorageJobId)->update([
|
|
|
|
|
'kind' => 'host_storage_configure',
|
|
|
|
|
'sealed_payload' => $storagePayload($tickStorageOperation),
|
|
|
|
|
'created_at' => time() - 61,
|
|
|
|
|
'dispatch_deadline_at' => time() + 239,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_host_storage_configurations')
|
|
|
|
|
->where('host_id', 7)->update([
|
|
|
|
|
'state' => 'applying', 'operation_id' => $tickStorageOperation,
|
|
|
|
|
'job_id' => $tickStorageJobId, 'updated_at' => time(),
|
|
|
|
|
]);
|
|
|
|
|
$agentCalls = 0;
|
|
|
|
|
$transport = new Client(['handler' => static function () use (
|
|
|
|
|
&$agentCalls
|
|
|
|
|
) {
|
|
|
|
|
$agentCalls++;
|
|
|
|
|
throw new RuntimeException('Expired storage task reached Agent');
|
|
|
|
|
}]);
|
|
|
|
|
$tripwireAgent = new CloudAgentClient(
|
|
|
|
|
'7', str_repeat('a', 32), $transport, 120.0,
|
|
|
|
|
'http://127.0.0.1:8001/internal/agent/request',
|
|
|
|
|
str_repeat('b', 32)
|
|
|
|
|
);
|
|
|
|
|
$assert((new CloudWorkerService($tripwireAgent))->tick() === true,
|
|
|
|
|
'Worker tick did not converge the expired storage task.');
|
|
|
|
|
$tickStorage = CloudJob::find($tickStorageJobId);
|
|
|
|
|
$assert($agentCalls === 0
|
|
|
|
|
&& (string) $tickStorage->phase === 'terminal'
|
|
|
|
|
&& (string) $tickStorage->outcome === 'failed'
|
|
|
|
|
&& (int) $tickStorage->dispatch_attempts === 0,
|
|
|
|
|
'Worker dispatched an old storage task after its effective 60-second boundary.');
|
|
|
|
|
|
|
|
|
|
$incidentOperation = str_repeat('5', 32);
|
|
|
|
|
$incidentJobId = $insertJob(
|
|
|
|
|
1, $incidentOperation, 'awaiting_receipt', 1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $incidentJobId)->update([
|
|
|
|
|
'sealed_payload' => 'deadline-test:missing-payload',
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1, $incidentJobId, 1
|
|
|
|
|
) === 1, 'The finalization-incident fixture was not claimed.');
|
|
|
|
|
$incident = CloudJob::find($incidentJobId);
|
|
|
|
|
$assert($incident !== null
|
|
|
|
|
&& (string) $incident->phase === 'terminal'
|
|
|
|
|
&& (string) $incident->outcome === 'failed'
|
|
|
|
|
&& (string) $incident->status === 'failed'
|
|
|
|
|
&& (bool) $incident->effect_may_have_applied
|
|
|
|
|
&& !(bool) $incident->safe_to_resubmit
|
|
|
|
|
&& (string) $incident->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $incident->error_code === 'local_finalization_failed'
|
|
|
|
|
&& (bool) $incident->finalization_incident
|
|
|
|
|
&& (string) $incident->finalization_error_code
|
|
|
|
|
=== 'recovery_payload_unavailable'
|
|
|
|
|
&& (string) $incident->sealed_payload === '',
|
|
|
|
|
'A local finalizer crash left the task active, leaked its payload, or concealed the incident.');
|
|
|
|
|
|
|
|
|
|
$redactedIncidentOperation = '8123456789abcdef8123456789abcdef';
|
|
|
|
|
$redactedIncidentJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$redactedIncidentOperation,
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$redactedIncidentJobId
|
|
|
|
|
)->update(['kind' => 'network_change']);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$redactedIncidentJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The redacted finalization-incident fixture was not claimed.');
|
|
|
|
|
$redactedIncident = CloudJob::find($redactedIncidentJobId);
|
|
|
|
|
$redactedIncidentPayload = CloudSecrets::decrypt(
|
|
|
|
|
(string) $redactedIncident->sealed_payload,
|
|
|
|
|
'job-payload'
|
|
|
|
|
);
|
|
|
|
|
$assert((bool) $redactedIncident->finalization_incident
|
|
|
|
|
&& (string) $redactedIncident->effect_resolution === 'unknown'
|
|
|
|
|
&& array_key_exists('credential', $redactedIncidentPayload),
|
|
|
|
|
'A recoverable finalization incident lost its encrypted recovery escrow.');
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$redactedIncidentJobId
|
|
|
|
|
)->update(['available_at' => time() + 3600]);
|
|
|
|
|
|
|
|
|
|
$unreadableEnvelope = 'deadline-test:missing-policy-payload';
|
|
|
|
|
$unreadableIncidentJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
'823456789abcdef0823456789abcdef0',
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$unreadableIncidentJobId
|
|
|
|
|
)->update([
|
|
|
|
|
'kind' => 'network_change',
|
|
|
|
|
'sealed_payload' => $unreadableEnvelope,
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$unreadableIncidentJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The unreadable recovery escrow fixture was not claimed.');
|
|
|
|
|
$unreadableIncident = CloudJob::findOrFail($unreadableIncidentJobId);
|
|
|
|
|
$assert((bool) $unreadableIncident->finalization_incident
|
|
|
|
|
&& (string) $unreadableIncident->finalization_error_code
|
|
|
|
|
=== 'recovery_payload_unavailable'
|
|
|
|
|
&& (string) $unreadableIncident->sealed_payload
|
|
|
|
|
=== $unreadableEnvelope
|
|
|
|
|
&& (int) $unreadableIncident->available_at >= time() + 290,
|
|
|
|
|
'An unreadable recovery escrow was erased or left the retry queue.');
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$unreadableIncidentJobId
|
|
|
|
|
)->update(['available_at' => time() + 3600]);
|
|
|
|
|
|
|
|
|
|
$lateOperation = str_repeat('7', 32);
|
|
|
|
|
$lateJobId = $insertJob(1, $lateOperation, 'awaiting_receipt', 1);
|
|
|
|
|
$lateEventId = str_repeat('6', 32);
|
|
|
|
|
$lateReceipt = $withIdentity([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => $lateEventId,
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => $lateOperation,
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => ['confirmed' => true],
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->insert($storedEvent(
|
|
|
|
|
$lateReceipt,
|
|
|
|
|
// Equal to or later than the absolute boundary is audit-only.
|
|
|
|
|
$now - 9
|
|
|
|
|
));
|
|
|
|
|
$lateJob = (new CloudWorkerService())->settleCompletionReceipt(
|
|
|
|
|
$lateJobId, $lateEventId
|
|
|
|
|
);
|
|
|
|
|
$lateEvent = Capsule::table('cloud_agent_completion_events')
|
|
|
|
|
->where('event_id', $lateEventId)->first();
|
|
|
|
|
$assert((string) $lateJob->phase === 'terminal'
|
|
|
|
|
&& (string) $lateJob->outcome === 'failed'
|
|
|
|
|
&& (string) $lateJob->error_code === 'operation_failed'
|
|
|
|
|
&& (bool) $lateJob->effect_may_have_applied
|
|
|
|
|
&& !(bool) $lateJob->safe_to_resubmit
|
|
|
|
|
&& !(bool) $lateJob->finalization_incident
|
|
|
|
|
&& (string) $lateEvent->disposition === 'late',
|
|
|
|
|
'A receipt persisted after the deadline revived or succeeded the task.');
|
|
|
|
|
|
|
|
|
|
// completion_event_id is reserved before the business finalizer commits.
|
|
|
|
|
// If that finalizer fails, the matching on-time receipt must stay received
|
|
|
|
|
// so the settings recovery worker can finish PHP without Agent replay.
|
|
|
|
|
$onTimeRecoveryInstanceId = 90;
|
|
|
|
|
$onTimeRecoveryUuid = '00000000-1111-4222-8333-444444444444';
|
|
|
|
|
$insertInstance(
|
|
|
|
|
$onTimeRecoveryInstanceId,
|
|
|
|
|
$onTimeRecoveryUuid,
|
|
|
|
|
'on-time-finalizer-recovery-vm'
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_instances')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$onTimeRecoveryInstanceId
|
|
|
|
|
)->update(['spec_json' => '{invalid-json']);
|
|
|
|
|
$onTimeRecoveryOperation = '0bcdef0123456789abcdef0123456789';
|
|
|
|
|
$onTimeRecoveryJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$onTimeRecoveryOperation,
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$onTimeRecoveryJobId
|
|
|
|
|
)->update([
|
|
|
|
|
'instance_id' => $onTimeRecoveryInstanceId,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'receipt_deadline_at' => time() + 60,
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $onTimeRecoveryUuid
|
|
|
|
|
. '/settings',
|
|
|
|
|
'body' => [
|
|
|
|
|
'hostname' => 'on-time-recovered.example',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'settings_fields' => ['hostname'],
|
|
|
|
|
'credential' => null,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$onTimeRecoveryReceipt = $withIdentity([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => 'settings-on-time-finalizer-0001',
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => $onTimeRecoveryOperation,
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => [
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
'hostname' => 'on-time-recovered.example',
|
|
|
|
|
],
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudOperationCompletionService::accept(
|
|
|
|
|
$onTimeRecoveryReceipt
|
|
|
|
|
) === ['accepted' => true, 'disposition' => 'received'],
|
|
|
|
|
'A failed on-time finalizer incorrectly acknowledged its receipt.');
|
|
|
|
|
$onTimeFailed = CloudJob::find($onTimeRecoveryJobId);
|
|
|
|
|
$onTimeEvent = Capsule::table('cloud_agent_completion_events')
|
|
|
|
|
->where('event_id', $onTimeRecoveryReceipt['id'])->first();
|
|
|
|
|
$assert((string) $onTimeFailed->phase === 'terminal'
|
|
|
|
|
&& (string) $onTimeFailed->outcome === 'failed'
|
|
|
|
|
&& (string) $onTimeFailed->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $onTimeFailed->completion_event_id
|
|
|
|
|
=== $onTimeRecoveryReceipt['id']
|
|
|
|
|
&& (string) $onTimeEvent->disposition === 'received'
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$onTimeRecoveryInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'A failed on-time finalizer lost its recovery receipt or blocker.');
|
|
|
|
|
Capsule::table('cloud_instances')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$onTimeRecoveryInstanceId
|
|
|
|
|
)->update([
|
|
|
|
|
'spec_json' => json_encode(
|
|
|
|
|
['managed' => true],
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
),
|
|
|
|
|
]);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The recovery worker did not resume the on-time receipt.');
|
|
|
|
|
$onTimeRecovered = CloudJob::find($onTimeRecoveryJobId);
|
|
|
|
|
$onTimeEvent = Capsule::table('cloud_agent_completion_events')
|
|
|
|
|
->where('event_id', $onTimeRecoveryReceipt['id'])->first();
|
|
|
|
|
$onTimeSpec = json_decode(
|
|
|
|
|
(string) Capsule::table('cloud_instances')
|
|
|
|
|
->where('id', $onTimeRecoveryInstanceId)
|
|
|
|
|
->value('spec_json'),
|
|
|
|
|
true,
|
|
|
|
|
flags: JSON_THROW_ON_ERROR
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $onTimeRecovered->outcome === 'failed'
|
|
|
|
|
&& (string) $onTimeRecovered->status === 'failed'
|
|
|
|
|
&& (string) $onTimeRecovered->effect_resolution === 'applied'
|
|
|
|
|
&& (string) $onTimeRecovered->completion_event_id
|
|
|
|
|
=== $onTimeRecoveryReceipt['id']
|
|
|
|
|
&& (string) $onTimeEvent->disposition === 'applied'
|
|
|
|
|
&& ($onTimeSpec['guest_hostname'] ?? null)
|
|
|
|
|
=== 'on-time-recovered.example'
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$onTimeRecoveryInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'The on-time receipt did not finish PHP-only recovery exactly once.');
|
|
|
|
|
|
|
|
|
|
// A password reset can finish on the Agent after PHP has conservatively
|
|
|
|
|
// failed its task. The immutable late receipt must converge the instance
|
|
|
|
|
// ledger and release the blocker without rewriting the failed task.
|
|
|
|
|
$settingsInstanceId = 91;
|
|
|
|
|
$settingsUuid = '11111111-2222-4333-8444-555555555555';
|
|
|
|
|
$insertInstance($settingsInstanceId, $settingsUuid, 'late-settings-vm');
|
|
|
|
|
$settingsOperation = 'abcdef0123456789abcdef0123456789';
|
|
|
|
|
$settingsJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$settingsOperation,
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $settingsJobId)->update([
|
|
|
|
|
'instance_id' => $settingsInstanceId,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $settingsUuid . '/settings',
|
|
|
|
|
'body' => [
|
|
|
|
|
'root_password' => 'New-late-password9!',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'settings_fields' => ['password'],
|
|
|
|
|
'credential' => [
|
|
|
|
|
'password' => 'New-late-password9!',
|
|
|
|
|
'login_name' => 'root',
|
|
|
|
|
'auth_mode' => 'password',
|
|
|
|
|
],
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$settingsJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The settings fixture did not cross its receipt deadline.');
|
|
|
|
|
$timedOutSettings = CloudJob::find($settingsJobId);
|
|
|
|
|
$originalTerminal = [
|
|
|
|
|
'operation_id' => (string) $timedOutSettings->operation_id,
|
|
|
|
|
'kind' => (string) $timedOutSettings->kind,
|
|
|
|
|
'phase' => (string) $timedOutSettings->phase,
|
|
|
|
|
'outcome' => (string) $timedOutSettings->outcome,
|
|
|
|
|
'status' => (string) $timedOutSettings->status,
|
|
|
|
|
'dispatch_attempts' => (int) $timedOutSettings->dispatch_attempts,
|
|
|
|
|
'effect_may_have_applied' =>
|
|
|
|
|
(bool) $timedOutSettings->effect_may_have_applied,
|
|
|
|
|
'safe_to_resubmit' => (bool) $timedOutSettings->safe_to_resubmit,
|
|
|
|
|
'error_code' => (string) $timedOutSettings->error_code,
|
|
|
|
|
'result_json' => $timedOutSettings->result_json,
|
|
|
|
|
'terminal_at' => (int) $timedOutSettings->terminal_at,
|
|
|
|
|
'completion_event_id' => $timedOutSettings->completion_event_id,
|
|
|
|
|
'read_at' => $timedOutSettings->read_at,
|
|
|
|
|
];
|
|
|
|
|
$settingsReceipt = $withIdentity([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => 'settings-recovery-event-000001',
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => $settingsOperation,
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => [
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
],
|
|
|
|
|
]);
|
|
|
|
|
$settingsAccepted = CloudOperationCompletionService::accept(
|
|
|
|
|
$settingsReceipt
|
|
|
|
|
);
|
|
|
|
|
$recoveredSettings = CloudJob::find($settingsJobId);
|
|
|
|
|
$recoveredInstance = Capsule::table('cloud_instances')
|
|
|
|
|
->where('id', $settingsInstanceId)->first();
|
|
|
|
|
$recoveredSpec = json_decode(
|
|
|
|
|
(string) $recoveredInstance->spec_json,
|
|
|
|
|
true,
|
|
|
|
|
flags: JSON_THROW_ON_ERROR
|
|
|
|
|
);
|
|
|
|
|
$recoveredCredential = CloudSecrets::decrypt(
|
|
|
|
|
(string) $recoveredInstance->sealed_credential,
|
|
|
|
|
'instance-credential'
|
|
|
|
|
);
|
|
|
|
|
$redactedRecoveryPayload = CloudSecrets::decrypt(
|
|
|
|
|
(string) $recoveredSettings->sealed_payload,
|
|
|
|
|
'job-payload'
|
|
|
|
|
);
|
|
|
|
|
$settingsEvent = Capsule::table('cloud_agent_completion_events')
|
|
|
|
|
->where('event_id', $settingsReceipt['id'])->first();
|
|
|
|
|
$assert($settingsAccepted === [
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
'disposition' => 'applied',
|
|
|
|
|
]
|
|
|
|
|
&& (string) $settingsEvent->disposition === 'applied'
|
|
|
|
|
&& (string) $recoveredSettings->effect_resolution === 'applied'
|
|
|
|
|
&& (int) $recoveredSettings->effect_resolved_at > 0
|
|
|
|
|
&& $originalTerminal === [
|
|
|
|
|
'operation_id' => (string) $recoveredSettings->operation_id,
|
|
|
|
|
'kind' => (string) $recoveredSettings->kind,
|
|
|
|
|
'phase' => (string) $recoveredSettings->phase,
|
|
|
|
|
'outcome' => (string) $recoveredSettings->outcome,
|
|
|
|
|
'status' => (string) $recoveredSettings->status,
|
|
|
|
|
'dispatch_attempts' =>
|
|
|
|
|
(int) $recoveredSettings->dispatch_attempts,
|
|
|
|
|
'effect_may_have_applied' =>
|
|
|
|
|
(bool) $recoveredSettings->effect_may_have_applied,
|
|
|
|
|
'safe_to_resubmit' =>
|
|
|
|
|
(bool) $recoveredSettings->safe_to_resubmit,
|
|
|
|
|
'error_code' => (string) $recoveredSettings->error_code,
|
|
|
|
|
'result_json' => $recoveredSettings->result_json,
|
|
|
|
|
'terminal_at' => (int) $recoveredSettings->terminal_at,
|
|
|
|
|
'completion_event_id' => $recoveredSettings->completion_event_id,
|
|
|
|
|
'read_at' => $recoveredSettings->read_at,
|
|
|
|
|
]
|
|
|
|
|
&& $recoveredSpec === ['managed' => true]
|
|
|
|
|
&& ($recoveredCredential['password'] ?? null)
|
|
|
|
|
=== 'New-late-password9!'
|
|
|
|
|
&& !array_key_exists('credential', $redactedRecoveryPayload)
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$settingsInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'A matching late settings receipt did not safely converge the ledger.');
|
|
|
|
|
$recoveredUpdatedAt = (int) $recoveredSettings->updated_at;
|
|
|
|
|
$recoveredCredentialEnvelope = (string) $recoveredInstance
|
|
|
|
|
->sealed_credential;
|
|
|
|
|
$assert(
|
|
|
|
|
CloudOperationCompletionService::accept($settingsReceipt) === [
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
'disposition' => 'applied',
|
|
|
|
|
]
|
|
|
|
|
&& (int) CloudJob::find($settingsJobId)->updated_at
|
|
|
|
|
=== $recoveredUpdatedAt
|
|
|
|
|
&& (string) Capsule::table('cloud_instances')
|
|
|
|
|
->where('id', $settingsInstanceId)
|
|
|
|
|
->value('sealed_credential') === $recoveredCredentialEnvelope,
|
|
|
|
|
'An identical late receipt was not idempotent after reconciliation.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$forgedSettingsReceipt = $settingsReceipt;
|
|
|
|
|
$forgedSettingsReceipt['id'] = 'settings-recovery-event-forged01';
|
|
|
|
|
$forgedConflict = false;
|
|
|
|
|
try {
|
|
|
|
|
CloudOperationCompletionService::accept($forgedSettingsReceipt);
|
|
|
|
|
} catch (\App\Services\Cloud\CloudError $error) {
|
|
|
|
|
$forgedConflict = $error->httpStatus === 409;
|
|
|
|
|
}
|
|
|
|
|
$unknownOperationReceipt = $settingsReceipt;
|
|
|
|
|
$unknownOperationReceipt['id'] = 'settings-recovery-event-unknown1';
|
|
|
|
|
$unknownOperationReceipt['operation_id'] =
|
|
|
|
|
'fedcba9876543210fedcba9876543210';
|
|
|
|
|
$assert($forgedConflict
|
|
|
|
|
&& CloudOperationCompletionService::accept(
|
|
|
|
|
$unknownOperationReceipt
|
|
|
|
|
) === ['accepted' => true, 'disposition' => 'rejected'],
|
|
|
|
|
'A different event or operation was accepted as recovery evidence.');
|
|
|
|
|
|
|
|
|
|
// An exact terminal Agent failure which explicitly says no effect was
|
|
|
|
|
// possible can also settle a timed-out settings task. Its transport error
|
|
|
|
|
// vocabulary is normalized to the settings no-effect finalizer; it never
|
|
|
|
|
// authorizes replay even when safe_to_resubmit=true.
|
|
|
|
|
$notAppliedInstanceId = 93;
|
|
|
|
|
$notAppliedUuid = 'bbbbbbbb-cccc-4ddd-8eee-ffffffffffff';
|
|
|
|
|
$insertInstance(
|
|
|
|
|
$notAppliedInstanceId,
|
|
|
|
|
$notAppliedUuid,
|
|
|
|
|
'not-applied-settings-vm'
|
|
|
|
|
);
|
|
|
|
|
$notAppliedOperationId = '2234567890abcdef1234567890abcdef';
|
|
|
|
|
$notAppliedJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$notAppliedOperationId,
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $notAppliedJobId)->update([
|
|
|
|
|
'instance_id' => $notAppliedInstanceId,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $notAppliedUuid
|
|
|
|
|
. '/settings',
|
|
|
|
|
'body' => [
|
|
|
|
|
'hostname' => 'not-applied.example',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'settings_fields' => ['hostname'],
|
|
|
|
|
'credential' => null,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$notAppliedJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The late no-effect fixture did not reach terminal failure.');
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $notAppliedJobId)->update([
|
|
|
|
|
'effect_resolution_reason' => 'automatic_recovery:v1:'
|
|
|
|
|
. (time() - 21600) . ':12:pending',
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_limit_reached',
|
|
|
|
|
]);
|
|
|
|
|
$notAppliedTerminal = CloudJob::find($notAppliedJobId);
|
|
|
|
|
$notAppliedAudit = [
|
|
|
|
|
'phase' => (string) $notAppliedTerminal->phase,
|
|
|
|
|
'outcome' => (string) $notAppliedTerminal->outcome,
|
|
|
|
|
'status' => (string) $notAppliedTerminal->status,
|
|
|
|
|
'error_code' => (string) $notAppliedTerminal->error_code,
|
|
|
|
|
'result_json' => $notAppliedTerminal->result_json,
|
|
|
|
|
'terminal_at' => (int) $notAppliedTerminal->terminal_at,
|
|
|
|
|
'completion_event_id' =>
|
|
|
|
|
$notAppliedTerminal->completion_event_id,
|
|
|
|
|
];
|
|
|
|
|
$notAppliedReceipt = $withIdentity([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => 'settings-no-effect-event-000001',
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => $notAppliedOperationId,
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => true,
|
|
|
|
|
'error_code' => 'guest_update_failed',
|
|
|
|
|
'result' => ['agent_detail' => 'bounded'],
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudOperationCompletionService::accept(
|
|
|
|
|
$notAppliedReceipt
|
|
|
|
|
) === ['accepted' => true, 'disposition' => 'applied'],
|
|
|
|
|
'An exact late no-effect settings receipt was not applied.');
|
|
|
|
|
$notAppliedRecovered = CloudJob::find($notAppliedJobId);
|
|
|
|
|
$notAppliedPayload = CloudSecrets::decrypt(
|
|
|
|
|
(string) $notAppliedRecovered->sealed_payload,
|
|
|
|
|
'job-payload'
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $notAppliedRecovered->effect_resolution
|
|
|
|
|
=== 'not_applied'
|
|
|
|
|
&& $notAppliedAudit === [
|
|
|
|
|
'phase' => (string) $notAppliedRecovered->phase,
|
|
|
|
|
'outcome' => (string) $notAppliedRecovered->outcome,
|
|
|
|
|
'status' => (string) $notAppliedRecovered->status,
|
|
|
|
|
'error_code' => (string) $notAppliedRecovered->error_code,
|
|
|
|
|
'result_json' => $notAppliedRecovered->result_json,
|
|
|
|
|
'terminal_at' => (int) $notAppliedRecovered->terminal_at,
|
|
|
|
|
'completion_event_id' =>
|
|
|
|
|
$notAppliedRecovered->completion_event_id,
|
|
|
|
|
]
|
|
|
|
|
&& !array_key_exists('credential', $notAppliedPayload)
|
|
|
|
|
&& !(bool) $notAppliedRecovered->finalization_incident
|
|
|
|
|
&& $notAppliedRecovered->finalization_error_code === null
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$notAppliedInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'Late no-effect recovery rewrote task audit fields or retained its lock.');
|
|
|
|
|
|
|
|
|
|
// Receipt persistence and late-effect settlement are deliberately
|
|
|
|
|
// separate transactions. If PHP dies between them, the recovery-only
|
|
|
|
|
// worker must discover the exact received event through the job's indexed
|
|
|
|
|
// host+operation key, even when inspection retries were already exhausted.
|
|
|
|
|
$persistedEventInstanceId = 96;
|
|
|
|
|
$persistedEventUuid = 'eeeeeeee-ffff-4000-8111-000000000003';
|
|
|
|
|
$insertInstance(
|
|
|
|
|
$persistedEventInstanceId,
|
|
|
|
|
$persistedEventUuid,
|
|
|
|
|
'persisted-event-settings-vm'
|
|
|
|
|
);
|
|
|
|
|
$persistedEventOperationId = '5234567890abcdef1234567890abcdef';
|
|
|
|
|
$persistedEventJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$persistedEventOperationId,
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $persistedEventJobId)->update([
|
|
|
|
|
'instance_id' => $persistedEventInstanceId,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $persistedEventUuid
|
|
|
|
|
. '/settings',
|
|
|
|
|
'body' => [
|
|
|
|
|
'hostname' => 'persisted-event.example',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'settings_fields' => ['hostname'],
|
|
|
|
|
'credential' => null,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$persistedEventJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The persisted-event crash fixture did not expire.');
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $persistedEventJobId)->update([
|
|
|
|
|
'effect_resolution_reason' => 'automatic_recovery:v1:'
|
|
|
|
|
. (time() - 21600) . ':12:pending',
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_limit_reached',
|
|
|
|
|
// Exact immutable receipt evidence is stronger than an earlier
|
|
|
|
|
// inconclusive/offline inspection backoff.
|
|
|
|
|
'available_at' => time() + 600,
|
|
|
|
|
]);
|
|
|
|
|
$persistedEventId = 'settings-persisted-event-00001';
|
|
|
|
|
$persistedEventResult = ['agent_detail' => 'persisted-before-prepare'];
|
|
|
|
|
$persistedEventReceipt = $withIdentity([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => $persistedEventId,
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => $persistedEventOperationId,
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => true,
|
|
|
|
|
'error_code' => 'guest_update_failed',
|
|
|
|
|
'result' => $persistedEventResult,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->insert(
|
|
|
|
|
$storedEvent($persistedEventReceipt, time())
|
|
|
|
|
);
|
|
|
|
|
$persistedEventAudit = CloudJob::find($persistedEventJobId);
|
|
|
|
|
$persistedEventAudit = [
|
|
|
|
|
'phase' => (string) $persistedEventAudit->phase,
|
|
|
|
|
'outcome' => (string) $persistedEventAudit->outcome,
|
|
|
|
|
'status' => (string) $persistedEventAudit->status,
|
|
|
|
|
'error_code' => (string) $persistedEventAudit->error_code,
|
|
|
|
|
'result_json' => $persistedEventAudit->result_json,
|
|
|
|
|
'terminal_at' => (int) $persistedEventAudit->terminal_at,
|
|
|
|
|
];
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The worker did not drain a persisted pre-prepare receipt.');
|
|
|
|
|
$persistedEventRecovered = CloudJob::find($persistedEventJobId);
|
|
|
|
|
$persistedEventRow = Capsule::table('cloud_agent_completion_events')
|
|
|
|
|
->where('event_id', $persistedEventId)->first();
|
|
|
|
|
$assert((string) $persistedEventRecovered->effect_resolution
|
|
|
|
|
=== 'not_applied'
|
|
|
|
|
&& (string) $persistedEventRecovered->effect_resolution_reason
|
|
|
|
|
=== 'late_completion_receipt:' . $persistedEventId
|
|
|
|
|
&& !(bool) $persistedEventRecovered->finalization_incident
|
|
|
|
|
&& $persistedEventRecovered->finalization_error_code === null
|
|
|
|
|
&& (string) $persistedEventRow->disposition === 'applied'
|
|
|
|
|
&& $persistedEventAudit === [
|
|
|
|
|
'phase' => (string) $persistedEventRecovered->phase,
|
|
|
|
|
'outcome' => (string) $persistedEventRecovered->outcome,
|
|
|
|
|
'status' => (string) $persistedEventRecovered->status,
|
|
|
|
|
'error_code' => (string) $persistedEventRecovered->error_code,
|
|
|
|
|
'result_json' => $persistedEventRecovered->result_json,
|
|
|
|
|
'terminal_at' => (int) $persistedEventRecovered->terminal_at,
|
|
|
|
|
]
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$persistedEventInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'Persisted pre-prepare receipt did not converge locally and exactly.');
|
|
|
|
|
|
|
|
|
|
// A row with the right indexed host+operation but a conflicting immutable
|
|
|
|
|
// payload hash is not receipt authority. Reject only that event, surface
|
|
|
|
|
// the concrete receipt error, and leave the job queued for fresh
|
|
|
|
|
// inspection without contacting the Agent in the same tick.
|
|
|
|
|
$conflictEventInstanceId = 97;
|
|
|
|
|
$conflictEventUuid = 'ffffffff-0000-4111-8222-000000000004';
|
|
|
|
|
$insertInstance(
|
|
|
|
|
$conflictEventInstanceId,
|
|
|
|
|
$conflictEventUuid,
|
|
|
|
|
'conflict-event-settings-vm'
|
|
|
|
|
);
|
|
|
|
|
$conflictEventOperationId = '6234567890abcdef1234567890abcdef';
|
|
|
|
|
$conflictEventJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$conflictEventOperationId,
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $conflictEventJobId)->update([
|
|
|
|
|
'instance_id' => $conflictEventInstanceId,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $conflictEventUuid
|
|
|
|
|
. '/settings',
|
|
|
|
|
'body' => [
|
|
|
|
|
'root_password' => 'Conflict-password9!',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'settings_fields' => ['password'],
|
|
|
|
|
'credential' => [
|
|
|
|
|
'password' => 'Conflict-password9!',
|
|
|
|
|
'login_name' => 'root',
|
|
|
|
|
'auth_mode' => 'password',
|
|
|
|
|
],
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$conflictEventJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The conflicting-event fixture did not expire.');
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $conflictEventJobId)->update([
|
|
|
|
|
'effect_resolution_reason' => 'automatic_recovery:v1:'
|
|
|
|
|
. (time() - 21600) . ':12:pending',
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_state_invalid',
|
|
|
|
|
]);
|
|
|
|
|
$conflictEventId = 'settings-conflict-event-000001';
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
|
|
|
|
'event_id' => $conflictEventId,
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'operation_id' => $conflictEventOperationId,
|
|
|
|
|
'payload_sha256' => str_repeat('0', 64),
|
|
|
|
|
'outcome' => 'failed',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 1,
|
|
|
|
|
'error_code' => 'guest_update_failed',
|
|
|
|
|
'result_json' => json_encode(
|
|
|
|
|
['agent_detail' => 'does-not-match-hash'],
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
),
|
|
|
|
|
'disposition' => 'received',
|
|
|
|
|
'received_at' => time() + 1,
|
|
|
|
|
'applied_at' => null,
|
|
|
|
|
]);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The worker did not examine a conflicting persisted event.');
|
|
|
|
|
$conflictEventJob = CloudJob::find($conflictEventJobId);
|
|
|
|
|
$conflictEventRow = Capsule::table('cloud_agent_completion_events')
|
|
|
|
|
->where('event_id', $conflictEventId)->first();
|
|
|
|
|
$assert((string) $conflictEventRow->disposition === 'rejected'
|
|
|
|
|
&& (string) $conflictEventJob->effect_resolution === 'unknown'
|
|
|
|
|
&& (bool) $conflictEventJob->finalization_incident
|
|
|
|
|
&& (string) $conflictEventJob->finalization_error_code
|
|
|
|
|
=== 'recovery_receipt_invalid'
|
|
|
|
|
&& (int) $conflictEventJob->available_at >= time() + 290
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$conflictEventInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'A conflicting persisted event altered the job or released its lock.');
|
|
|
|
|
|
|
|
|
|
// A self-consistent late receipt may still contradict an already prepared
|
|
|
|
|
// conclusion. Such conflicts are deterministic: atomically isolate the
|
|
|
|
|
// event once, retain the original marker/escrow/blocker, and schedule a
|
|
|
|
|
// later fresh inspection without polling Agent in the same tick.
|
|
|
|
|
$conflictEscrow = (string) $conflictEventJob->sealed_payload;
|
|
|
|
|
$conflictIdentity = $identityFor($conflictEventOperationId, false);
|
|
|
|
|
$noEffectOperation = $conflictIdentity + [
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'error_code' => 'settings_not_committed',
|
|
|
|
|
'operation_id' => $conflictEventOperationId,
|
|
|
|
|
'result' => [],
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
];
|
|
|
|
|
$canonicalNoEffectOperation = $noEffectOperation;
|
|
|
|
|
ksort($canonicalNoEffectOperation, SORT_STRING);
|
|
|
|
|
$noEffectOperationHash = hash(
|
|
|
|
|
'sha256',
|
|
|
|
|
json_encode(
|
|
|
|
|
$canonicalNoEffectOperation,
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
| JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE
|
|
|
|
|
)
|
|
|
|
|
);
|
|
|
|
|
$armConflictReceipt = static function (
|
|
|
|
|
string $status,
|
|
|
|
|
bool $safeToResubmit,
|
|
|
|
|
string $errorCode,
|
|
|
|
|
array $result,
|
|
|
|
|
string $marker
|
|
|
|
|
) use (
|
|
|
|
|
$conflictEventId,
|
|
|
|
|
$conflictEventJobId,
|
|
|
|
|
$conflictEventOperationId,
|
|
|
|
|
$storedEvent,
|
|
|
|
|
$withIdentity
|
|
|
|
|
): void {
|
|
|
|
|
$receipt = $withIdentity([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => $conflictEventId,
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => $conflictEventOperationId,
|
|
|
|
|
'status' => $status,
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => $safeToResubmit,
|
|
|
|
|
'error_code' => $errorCode,
|
|
|
|
|
'result' => $result,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$conflictEventJobId
|
|
|
|
|
)->update([
|
|
|
|
|
'effect_resolution_reason' => $marker,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$eventRow = $storedEvent($receipt, time());
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->where(
|
|
|
|
|
'event_id',
|
|
|
|
|
$conflictEventId
|
|
|
|
|
)->update($eventRow);
|
|
|
|
|
};
|
|
|
|
|
$assertPreparedConflict = static function (
|
|
|
|
|
string $marker,
|
|
|
|
|
string $message
|
|
|
|
|
) use (
|
|
|
|
|
$assert,
|
|
|
|
|
$conflictEscrow,
|
|
|
|
|
$conflictEventId,
|
|
|
|
|
$conflictEventInstanceId,
|
|
|
|
|
$conflictEventJobId
|
|
|
|
|
): void {
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
$message . ' was not examined.');
|
|
|
|
|
$job = CloudJob::find($conflictEventJobId);
|
|
|
|
|
$event = Capsule::table('cloud_agent_completion_events')
|
|
|
|
|
->where('event_id', $conflictEventId)->first();
|
|
|
|
|
$updatedAt = (int) $job->updated_at;
|
|
|
|
|
$assert((string) $event->disposition === 'late'
|
|
|
|
|
&& $event->applied_at === null
|
|
|
|
|
&& (string) $job->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $job->effect_resolution_reason === $marker
|
|
|
|
|
&& (bool) $job->finalization_incident
|
|
|
|
|
&& (string) $job->finalization_error_code
|
|
|
|
|
=== 'recovery_evidence_conflict'
|
|
|
|
|
&& (string) $job->sealed_payload === $conflictEscrow
|
|
|
|
|
&& $job->lease_token === null
|
|
|
|
|
&& (int) $job->lease_until === 0
|
|
|
|
|
&& (int) $job->available_at >= time() + 290
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$conflictEventInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
$message . ' did not retain the fail-closed evidence fence.');
|
|
|
|
|
// Other newly recoverable kinds may be claimed here; the isolated job
|
|
|
|
|
// itself must remain untouched until its own retry time arrives.
|
|
|
|
|
(new CloudAutomaticOperationRecoveryService())->tick();
|
|
|
|
|
$afterSecondTick = CloudJob::find($conflictEventJobId);
|
|
|
|
|
$afterSecondEvent = Capsule::table(
|
|
|
|
|
'cloud_agent_completion_events'
|
|
|
|
|
)->where('event_id', $conflictEventId)->first();
|
|
|
|
|
$assert((int) $afterSecondTick->updated_at === $updatedAt
|
|
|
|
|
&& (string) $afterSecondTick->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $afterSecondTick->effect_resolution_reason === $marker
|
|
|
|
|
&& (bool) $afterSecondTick->finalization_incident
|
|
|
|
|
&& (string) $afterSecondTick->finalization_error_code
|
|
|
|
|
=== 'recovery_evidence_conflict'
|
|
|
|
|
&& (string) $afterSecondTick->sealed_payload === $conflictEscrow
|
|
|
|
|
&& (string) $afterSecondEvent->disposition === 'late'
|
|
|
|
|
&& $afterSecondEvent->applied_at === null
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$conflictEventInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
$message . ' ignored its retry backoff.');
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$directionConflictMarker = 'automatic_recovery:v1:'
|
|
|
|
|
. (time() - 300)
|
|
|
|
|
. ':2:prepared:automatic_applied_succeeded_nofence:'
|
|
|
|
|
. str_repeat('1', 64) . ':' . $noEffectOperationHash;
|
|
|
|
|
$armConflictReceipt(
|
|
|
|
|
'failed',
|
|
|
|
|
true,
|
|
|
|
|
'guest_update_failed',
|
|
|
|
|
['agent_detail' => 'direction-conflict'],
|
|
|
|
|
$directionConflictMarker
|
|
|
|
|
);
|
|
|
|
|
$assertPreparedConflict(
|
|
|
|
|
$directionConflictMarker,
|
|
|
|
|
'Opposite-direction late evidence'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$hashConflictMarker = 'automatic_recovery:v1:'
|
|
|
|
|
. (time() - 300)
|
|
|
|
|
. ':2:prepared:automatic_not_applied_failed_nofence:'
|
|
|
|
|
. str_repeat('2', 64) . ':' . str_repeat('3', 64);
|
|
|
|
|
$armConflictReceipt(
|
|
|
|
|
'failed',
|
|
|
|
|
true,
|
|
|
|
|
'guest_update_failed',
|
|
|
|
|
['agent_detail' => 'hash-conflict'],
|
|
|
|
|
$hashConflictMarker
|
|
|
|
|
);
|
|
|
|
|
$assertPreparedConflict(
|
|
|
|
|
$hashConflictMarker,
|
|
|
|
|
'Different-operation-hash late evidence'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$invalidStateConflictMarker = 'invalid-recovery-state';
|
|
|
|
|
$armConflictReceipt(
|
|
|
|
|
'failed',
|
|
|
|
|
true,
|
|
|
|
|
'guest_update_failed',
|
|
|
|
|
['agent_detail' => 'invalid-state-conflict'],
|
|
|
|
|
$invalidStateConflictMarker
|
|
|
|
|
);
|
|
|
|
|
$assertPreparedConflict(
|
|
|
|
|
$invalidStateConflictMarker,
|
|
|
|
|
'Invalid-state late evidence'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$noEffectFenceConflictMarker = 'automatic_recovery:v1:'
|
|
|
|
|
. (time() - 300)
|
|
|
|
|
. ':2:prepared:automatic_not_applied_failed_fence:'
|
|
|
|
|
. str_repeat('4', 64) . ':' . $noEffectOperationHash;
|
|
|
|
|
$armConflictReceipt(
|
|
|
|
|
'failed',
|
|
|
|
|
true,
|
|
|
|
|
'guest_update_failed',
|
|
|
|
|
['agent_detail' => 'no-effect-fence-conflict'],
|
|
|
|
|
$noEffectFenceConflictMarker
|
|
|
|
|
);
|
|
|
|
|
$assertPreparedConflict(
|
|
|
|
|
$noEffectFenceConflictMarker,
|
|
|
|
|
'No-effect active-fence late evidence'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$appliedReceiptResult = ['settings_changed' => true];
|
|
|
|
|
$appliedOperation = $conflictIdentity + [
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'operation_id' => $conflictEventOperationId,
|
|
|
|
|
'result' => $appliedReceiptResult,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
];
|
|
|
|
|
$canonicalAppliedOperation = $appliedOperation;
|
|
|
|
|
ksort($canonicalAppliedOperation, SORT_STRING);
|
|
|
|
|
$appliedOperationHash = hash(
|
|
|
|
|
'sha256',
|
|
|
|
|
json_encode(
|
|
|
|
|
$canonicalAppliedOperation,
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
| JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE
|
|
|
|
|
)
|
|
|
|
|
);
|
|
|
|
|
$appliedFenceConflictMarker = 'automatic_recovery:v1:'
|
|
|
|
|
. (time() - 300)
|
|
|
|
|
. ':2:prepared:automatic_applied_failed_fence:'
|
|
|
|
|
. str_repeat('5', 64) . ':' . $appliedOperationHash;
|
|
|
|
|
$armConflictReceipt(
|
|
|
|
|
'succeeded',
|
|
|
|
|
false,
|
|
|
|
|
'',
|
|
|
|
|
$appliedReceiptResult,
|
|
|
|
|
$appliedFenceConflictMarker
|
|
|
|
|
);
|
|
|
|
|
$assertPreparedConflict(
|
|
|
|
|
$appliedFenceConflictMarker,
|
|
|
|
|
'Applied active-fence late evidence'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// A receipt can become durable while an automatic worker is outside the
|
|
|
|
|
// database resolving an Agent fence. Its stale commit must yield before
|
|
|
|
|
// publishing, so the next event-first turn checks compatibility.
|
|
|
|
|
$pendingReceiptSource = 'automatic_not_applied_failed_nofence:'
|
|
|
|
|
. str_repeat('6', 64);
|
|
|
|
|
$pendingReceiptMarker = 'automatic_recovery:v1:'
|
|
|
|
|
. (time() - 300) . ':2:prepared:' . $pendingReceiptSource
|
|
|
|
|
. ':' . $noEffectOperationHash;
|
|
|
|
|
$armConflictReceipt(
|
|
|
|
|
'failed',
|
|
|
|
|
false,
|
|
|
|
|
'settings_not_committed',
|
|
|
|
|
[],
|
|
|
|
|
$pendingReceiptMarker
|
|
|
|
|
);
|
|
|
|
|
$pendingReceiptCommitRejected = false;
|
|
|
|
|
try {
|
|
|
|
|
CloudOperationRecoveryService::commitNotApplied(
|
|
|
|
|
$conflictEventJobId,
|
|
|
|
|
$noEffectOperation,
|
|
|
|
|
$pendingReceiptSource,
|
|
|
|
|
0
|
|
|
|
|
);
|
|
|
|
|
} catch (\App\Services\Cloud\CloudError $error) {
|
|
|
|
|
$pendingReceiptCommitRejected = $error->httpStatus === 409;
|
|
|
|
|
}
|
|
|
|
|
$pendingReceiptJob = CloudJob::find($conflictEventJobId);
|
|
|
|
|
$pendingReceiptEvent = Capsule::table(
|
|
|
|
|
'cloud_agent_completion_events'
|
|
|
|
|
)->where('event_id', $conflictEventId)->first();
|
|
|
|
|
$assert($pendingReceiptCommitRejected
|
|
|
|
|
&& (string) $pendingReceiptJob->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $pendingReceiptJob->effect_resolution_reason
|
|
|
|
|
=== $pendingReceiptMarker
|
|
|
|
|
&& (string) $pendingReceiptEvent->disposition === 'received',
|
|
|
|
|
'An automatic commit ignored a newly durable completion receipt.');
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The compatible priority receipt was not claimed after commit yield.');
|
|
|
|
|
$pendingReceiptResolved = CloudJob::find($conflictEventJobId);
|
|
|
|
|
$pendingReceiptEvent = Capsule::table(
|
|
|
|
|
'cloud_agent_completion_events'
|
|
|
|
|
)->where('event_id', $conflictEventId)->first();
|
|
|
|
|
$assert((string) $pendingReceiptResolved->effect_resolution
|
|
|
|
|
=== 'not_applied'
|
|
|
|
|
&& (string) $pendingReceiptResolved->effect_resolution_reason
|
|
|
|
|
=== 'late_completion_receipt:' . $conflictEventId
|
|
|
|
|
&& (string) $pendingReceiptEvent->disposition === 'applied'
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$conflictEventInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'The priority receipt did not safely win the automatic commit race.');
|
|
|
|
|
|
|
|
|
|
// A receipt can crash after its PHP prepare transaction but before the
|
|
|
|
|
// effect commit/event acknowledgement. Even if it replaced an exhausted
|
|
|
|
|
// automatic prepared intent, the dedicated recovery worker must rebuild
|
|
|
|
|
// the exact immutable event and finish PHP locally without an Agent
|
|
|
|
|
// retransmission or host request.
|
|
|
|
|
$lateCrashInstanceId = 95;
|
|
|
|
|
$lateCrashUuid = 'dddddddd-eeee-4fff-8000-000000000002';
|
|
|
|
|
$insertInstance(
|
|
|
|
|
$lateCrashInstanceId,
|
|
|
|
|
$lateCrashUuid,
|
|
|
|
|
'late-crash-settings-vm'
|
|
|
|
|
);
|
|
|
|
|
$lateCrashOperationId = '4234567890abcdef1234567890abcdef';
|
|
|
|
|
$lateCrashJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$lateCrashOperationId,
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $lateCrashJobId)->update([
|
|
|
|
|
'instance_id' => $lateCrashInstanceId,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $lateCrashUuid
|
|
|
|
|
. '/settings',
|
|
|
|
|
'body' => [
|
|
|
|
|
'hostname' => 'late-crash.example',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'settings_fields' => ['hostname'],
|
|
|
|
|
'credential' => null,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$lateCrashJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The late-receipt crash fixture did not expire.');
|
|
|
|
|
$lateCrashOperation = $identityFor($lateCrashOperationId, false) + [
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'error_code' => 'guest_update_failed',
|
|
|
|
|
'operation_id' => $lateCrashOperationId,
|
|
|
|
|
'result' => ['agent_detail' => 'bounded'],
|
|
|
|
|
'safe_to_resubmit' => true,
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
];
|
|
|
|
|
$canonicalLateCrashOperation = $lateCrashOperation;
|
|
|
|
|
ksort($canonicalLateCrashOperation, SORT_STRING);
|
|
|
|
|
$lateCrashOperationHash = hash(
|
|
|
|
|
'sha256',
|
|
|
|
|
json_encode(
|
|
|
|
|
$canonicalLateCrashOperation,
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
| JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE
|
|
|
|
|
)
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $lateCrashJobId)->update([
|
|
|
|
|
'effect_resolution_reason' => 'automatic_recovery:v1:'
|
|
|
|
|
. (time() - 21600)
|
|
|
|
|
. ':12:prepared:automatic_not_applied_failed_nofence:'
|
|
|
|
|
. str_repeat('a', 64) . ':' . $lateCrashOperationHash,
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_limit_reached',
|
|
|
|
|
]);
|
|
|
|
|
$lateCrashEventId = 'settings-late-crash-event-0001';
|
|
|
|
|
$lateCrashReceipt = $withIdentity([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => $lateCrashEventId,
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => $lateCrashOperationId,
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => true,
|
|
|
|
|
'error_code' => 'guest_update_failed',
|
|
|
|
|
'result' => ['agent_detail' => 'bounded'],
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->insert(
|
|
|
|
|
$storedEvent($lateCrashReceipt, time())
|
|
|
|
|
);
|
|
|
|
|
CloudOperationRecoveryService::prepareNotApplied(
|
|
|
|
|
$lateCrashJobId,
|
|
|
|
|
$lateCrashOperation,
|
|
|
|
|
'late_completion_receipt:' . $lateCrashEventId
|
|
|
|
|
);
|
|
|
|
|
$lateCrashPrepared = CloudJob::find($lateCrashJobId);
|
|
|
|
|
$legacyLateCrashPreparedMarker = (string)
|
|
|
|
|
$lateCrashPrepared->effect_resolution_reason;
|
|
|
|
|
$assert(str_starts_with(
|
|
|
|
|
$legacyLateCrashPreparedMarker,
|
|
|
|
|
'prepared:late_completion_receipt:' . $lateCrashEventId . ':'
|
|
|
|
|
)
|
|
|
|
|
&& !(bool) $lateCrashPrepared->finalization_incident
|
|
|
|
|
&& $lateCrashPrepared->finalization_error_code === null
|
|
|
|
|
&& (string) $lateCrashPrepared->effect_resolution === 'unknown',
|
|
|
|
|
'Exact late evidence did not replace exhausted matching prepare state.');
|
|
|
|
|
// Older crash windows could retain the stable late-receipt marker inside
|
|
|
|
|
// the mutable automatic-attempt envelope. The exact same receipt must
|
|
|
|
|
// upgrade that legacy wrapper without weakening operation-hash matching.
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $lateCrashJobId)->update([
|
|
|
|
|
'effect_resolution_reason' => 'automatic_recovery:v1:'
|
|
|
|
|
. (time() - 21600) . ':12:'
|
|
|
|
|
. $legacyLateCrashPreparedMarker,
|
|
|
|
|
]);
|
|
|
|
|
$lateCrashPrepared = CloudJob::find($lateCrashJobId);
|
|
|
|
|
$lateCrashAudit = [
|
|
|
|
|
'phase' => (string) $lateCrashPrepared->phase,
|
|
|
|
|
'outcome' => (string) $lateCrashPrepared->outcome,
|
|
|
|
|
'status' => (string) $lateCrashPrepared->status,
|
|
|
|
|
'error_code' => (string) $lateCrashPrepared->error_code,
|
|
|
|
|
'result_json' => $lateCrashPrepared->result_json,
|
|
|
|
|
'terminal_at' => (int) $lateCrashPrepared->terminal_at,
|
|
|
|
|
'completion_event_id' => $lateCrashPrepared->completion_event_id,
|
|
|
|
|
];
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The recovery worker did not resume a prepared late receipt.');
|
|
|
|
|
$lateCrashRecovered = CloudJob::find($lateCrashJobId);
|
|
|
|
|
$lateCrashEvent = Capsule::table('cloud_agent_completion_events')
|
|
|
|
|
->where('event_id', $lateCrashEventId)->first();
|
|
|
|
|
$assert((string) $lateCrashRecovered->effect_resolution
|
|
|
|
|
=== 'not_applied'
|
|
|
|
|
&& (string) $lateCrashRecovered->effect_resolution_reason
|
|
|
|
|
=== 'late_completion_receipt:' . $lateCrashEventId
|
|
|
|
|
&& (string) $lateCrashEvent->disposition === 'applied'
|
|
|
|
|
&& $lateCrashAudit === [
|
|
|
|
|
'phase' => (string) $lateCrashRecovered->phase,
|
|
|
|
|
'outcome' => (string) $lateCrashRecovered->outcome,
|
|
|
|
|
'status' => (string) $lateCrashRecovered->status,
|
|
|
|
|
'error_code' => (string) $lateCrashRecovered->error_code,
|
|
|
|
|
'result_json' => $lateCrashRecovered->result_json,
|
|
|
|
|
'terminal_at' => (int) $lateCrashRecovered->terminal_at,
|
|
|
|
|
'completion_event_id' =>
|
|
|
|
|
$lateCrashRecovered->completion_event_id,
|
|
|
|
|
]
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$lateCrashInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'Prepared late receipt did not finish PHP-only convergence.');
|
|
|
|
|
|
|
|
|
|
// A hash-valid receipt can still violate the kind-specific finalizer
|
|
|
|
|
// contract. This deterministic local error must become one incident, not
|
|
|
|
|
// an unbounded received-event loop every five minutes.
|
|
|
|
|
$invalidReceiptInstanceId = 98;
|
|
|
|
|
$invalidReceiptUuid = '99999999-aaaa-4bbb-8ccc-dddddddddddd';
|
|
|
|
|
$insertInstance(
|
|
|
|
|
$invalidReceiptInstanceId,
|
|
|
|
|
$invalidReceiptUuid,
|
|
|
|
|
'invalid-receipt-settings-vm'
|
|
|
|
|
);
|
|
|
|
|
$invalidReceiptOperationId = '7234567890abcdef1234567890abcdef';
|
|
|
|
|
$invalidReceiptJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$invalidReceiptOperationId,
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$invalidReceiptJobId
|
|
|
|
|
)->update([
|
|
|
|
|
'instance_id' => $invalidReceiptInstanceId,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $invalidReceiptUuid
|
|
|
|
|
. '/settings',
|
|
|
|
|
'body' => [
|
|
|
|
|
'hostname' => 'invalid-receipt.example',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'settings_fields' => ['hostname'],
|
|
|
|
|
'credential' => null,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$invalidReceiptJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The invalid-receipt fixture did not expire.');
|
|
|
|
|
$invalidReceipt = $withIdentity([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => 'settings-invalid-result-event-01',
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => $invalidReceiptOperationId,
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
// The generic receipt is valid, but instance_settings requires the
|
|
|
|
|
// explicit settings_changed proof before touching its PHP ledger.
|
|
|
|
|
'result' => ['hostname' => 'invalid-receipt.example'],
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudOperationCompletionService::accept(
|
|
|
|
|
$invalidReceipt
|
|
|
|
|
) === ['accepted' => true, 'disposition' => 'late'],
|
|
|
|
|
'A kind-invalid exact receipt was not terminally parked.');
|
|
|
|
|
$invalidReceiptJob = CloudJob::find($invalidReceiptJobId);
|
|
|
|
|
$invalidReceiptEvent = Capsule::table(
|
|
|
|
|
'cloud_agent_completion_events'
|
|
|
|
|
)->where('event_id', $invalidReceipt['id'])->first();
|
|
|
|
|
$assert((string) $invalidReceiptJob->effect_resolution === 'unknown'
|
|
|
|
|
&& (bool) $invalidReceiptJob->finalization_incident
|
|
|
|
|
&& (string) $invalidReceiptJob->finalization_error_code
|
|
|
|
|
=== 'recovery_receipt_invalid'
|
|
|
|
|
&& (string) $invalidReceiptEvent->disposition === 'late'
|
|
|
|
|
&& (int) $invalidReceiptJob->available_at >= time() + 290
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$invalidReceiptInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'An invalid receipt altered the ledger or released its blocker.');
|
|
|
|
|
// The incident is diagnostic, not a permanent queue state. A later fresh
|
|
|
|
|
// operation-bound inspection may prepare and publish a valid conclusion.
|
|
|
|
|
$freshInspectionOperation = $identityFor(
|
|
|
|
|
$invalidReceiptOperationId,
|
|
|
|
|
true
|
|
|
|
|
) + [
|
|
|
|
|
'operation_id' => $invalidReceiptOperationId,
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => [
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
'hostname' => 'invalid-receipt.example',
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
$freshInspectionSource =
|
|
|
|
|
'agent_inspection:fresh-after-receipt-incident';
|
|
|
|
|
CloudOperationRecoveryService::prepareApplied(
|
|
|
|
|
$invalidReceiptJobId,
|
|
|
|
|
$freshInspectionOperation,
|
|
|
|
|
$freshInspectionSource
|
|
|
|
|
);
|
|
|
|
|
CloudOperationRecoveryService::commitApplied(
|
|
|
|
|
$invalidReceiptJobId,
|
|
|
|
|
$freshInspectionOperation,
|
|
|
|
|
$freshInspectionSource,
|
|
|
|
|
0
|
|
|
|
|
);
|
|
|
|
|
$invalidReceiptAfter = CloudJob::find($invalidReceiptJobId);
|
|
|
|
|
$invalidReceiptSpec = json_decode(
|
|
|
|
|
(string) Capsule::table('cloud_instances')
|
|
|
|
|
->where('id', $invalidReceiptInstanceId)->value('spec_json'),
|
|
|
|
|
true,
|
|
|
|
|
flags: JSON_THROW_ON_ERROR
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $invalidReceiptAfter->effect_resolution === 'applied'
|
|
|
|
|
&& (string) $invalidReceiptAfter->effect_resolution_reason
|
|
|
|
|
=== $freshInspectionSource
|
|
|
|
|
&& !(bool) $invalidReceiptAfter->finalization_incident
|
|
|
|
|
&& $invalidReceiptAfter->finalization_error_code === null
|
|
|
|
|
&& $invalidReceiptAfter->read_at === null
|
|
|
|
|
&& ($invalidReceiptSpec['guest_hostname'] ?? null)
|
|
|
|
|
=== 'invalid-receipt.example'
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$invalidReceiptInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'A fresh inspection could not converge after an invalid receipt.');
|
|
|
|
|
|
|
|
|
|
// A valid immutable receipt may be temporarily unable to update a damaged
|
|
|
|
|
// local projection. Age is audit metadata only: keep it retryable until an
|
|
|
|
|
// operator repairs PHP state, even beyond the former six-hour window.
|
|
|
|
|
$receiptLimitInstanceId = 99;
|
|
|
|
|
$receiptLimitUuid = 'aaaaaaaa-1111-4222-8333-bbbbbbbbbbbb';
|
|
|
|
|
$insertInstance(
|
|
|
|
|
$receiptLimitInstanceId,
|
|
|
|
|
$receiptLimitUuid,
|
|
|
|
|
'receipt-limit-settings-vm'
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_instances')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$receiptLimitInstanceId
|
|
|
|
|
)->update(['spec_json' => '{invalid-json']);
|
|
|
|
|
$receiptLimitOperationId = '8234567890abcdef1234567890abcdef';
|
|
|
|
|
$receiptLimitJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$receiptLimitOperationId,
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$receiptLimitJobId
|
|
|
|
|
)->update([
|
|
|
|
|
'instance_id' => $receiptLimitInstanceId,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $receiptLimitUuid
|
|
|
|
|
. '/settings',
|
|
|
|
|
'body' => [
|
|
|
|
|
'hostname' => 'receipt-limit.example',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'settings_fields' => ['hostname'],
|
|
|
|
|
'credential' => null,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$receiptLimitJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The receipt-limit fixture did not expire.');
|
|
|
|
|
$receiptLimitReceipt = $withIdentity([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => 'settings-receipt-limit-event-01',
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => $receiptLimitOperationId,
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => [
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
'hostname' => 'receipt-limit.example',
|
|
|
|
|
],
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudOperationCompletionService::accept(
|
|
|
|
|
$receiptLimitReceipt
|
|
|
|
|
) === ['accepted' => true, 'disposition' => 'received'],
|
|
|
|
|
'A repairable local finalizer error was parked immediately.');
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->where(
|
|
|
|
|
'event_id',
|
|
|
|
|
$receiptLimitReceipt['id']
|
|
|
|
|
)->update(['received_at' => time() - 21599]);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The recovery worker did not claim the near-limit receipt.');
|
|
|
|
|
$nearLimitJob = CloudJob::find($receiptLimitJobId);
|
|
|
|
|
$assert((string) Capsule::table('cloud_agent_completion_events')
|
|
|
|
|
->where('event_id', $receiptLimitReceipt['id'])
|
|
|
|
|
->value('disposition') === 'received'
|
|
|
|
|
&& (int) $nearLimitJob->available_at >= time() + 290,
|
|
|
|
|
'A repairable receipt did not retain its retry backoff.');
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->where(
|
|
|
|
|
'event_id',
|
|
|
|
|
$receiptLimitReceipt['id']
|
|
|
|
|
)->update(['received_at' => time() - 21601]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$receiptLimitJobId
|
|
|
|
|
)->update([
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
]);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The recovery worker did not claim the six-hour-old receipt.');
|
|
|
|
|
$receiptLimitJob = CloudJob::find($receiptLimitJobId);
|
|
|
|
|
$receiptLimitEvent = CloudAgentCompletionEvent::find(
|
|
|
|
|
$receiptLimitReceipt['id']
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $receiptLimitJob->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $receiptLimitEvent->disposition === 'received'
|
|
|
|
|
&& (int) $receiptLimitJob->available_at >= time() + 290
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$receiptLimitInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'A six-hour-old receipt left the recovery queue.');
|
|
|
|
|
Capsule::table('cloud_instances')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$receiptLimitInstanceId
|
|
|
|
|
)->update(['spec_json' => json_encode(
|
|
|
|
|
['managed' => true],
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
)]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$receiptLimitJobId
|
|
|
|
|
)->update([
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
]);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The repaired six-hour-old receipt was not claimed.');
|
|
|
|
|
$receiptLimitJob = CloudJob::find($receiptLimitJobId);
|
|
|
|
|
$receiptLimitEvent = CloudAgentCompletionEvent::find(
|
|
|
|
|
$receiptLimitReceipt['id']
|
|
|
|
|
);
|
|
|
|
|
$receiptLimitSpec = json_decode(
|
|
|
|
|
(string) Capsule::table('cloud_instances')
|
|
|
|
|
->where('id', $receiptLimitInstanceId)->value('spec_json'),
|
|
|
|
|
true,
|
|
|
|
|
flags: JSON_THROW_ON_ERROR
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $receiptLimitJob->effect_resolution === 'applied'
|
|
|
|
|
&& (string) $receiptLimitEvent->disposition === 'applied'
|
|
|
|
|
&& ($receiptLimitSpec['guest_hostname'] ?? null)
|
|
|
|
|
=== 'receipt-limit.example'
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$receiptLimitInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'A repaired six-hour-old receipt did not converge.');
|
|
|
|
|
|
|
|
|
|
// Decryptable escrow is not enough: a password operation whose retained
|
|
|
|
|
// credential DTO is missing must never be treated as a complete ledger
|
|
|
|
|
// finalization merely because the Agent receipt says settings_changed.
|
|
|
|
|
$damagedEscrowInstanceId = 100;
|
|
|
|
|
$damagedEscrowUuid = 'aaaaaaaa-2222-4333-8444-cccccccccccc';
|
|
|
|
|
$insertInstance(
|
|
|
|
|
$damagedEscrowInstanceId,
|
|
|
|
|
$damagedEscrowUuid,
|
|
|
|
|
'damaged-escrow-settings-vm'
|
|
|
|
|
);
|
|
|
|
|
$damagedEscrowOperationId = '9234567890abcdef1234567890abcdef';
|
|
|
|
|
$damagedEscrowJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$damagedEscrowOperationId,
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$damagedEscrowJobId
|
|
|
|
|
)->update([
|
|
|
|
|
'instance_id' => $damagedEscrowInstanceId,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $damagedEscrowUuid
|
|
|
|
|
. '/settings',
|
|
|
|
|
'body' => [
|
|
|
|
|
'root_password' => 'Escrow-damaged9!',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'settings_fields' => ['password'],
|
|
|
|
|
'credential' => null,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$damagedEscrowJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The damaged-escrow fixture did not expire.');
|
|
|
|
|
$damagedEscrowPayload = (string) CloudJob::find(
|
|
|
|
|
$damagedEscrowJobId
|
|
|
|
|
)->sealed_payload;
|
|
|
|
|
$damagedEscrowReceipt = $withIdentity([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => 'settings-damaged-escrow-event-01',
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => $damagedEscrowOperationId,
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => ['settings_changed' => true],
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudOperationCompletionService::accept(
|
|
|
|
|
$damagedEscrowReceipt
|
|
|
|
|
) === ['accepted' => true, 'disposition' => 'received'],
|
|
|
|
|
'Damaged recovery escrow falsely converged from a valid receipt.');
|
|
|
|
|
$damagedEscrowJob = CloudJob::find($damagedEscrowJobId);
|
|
|
|
|
$assert((string) $damagedEscrowJob->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $damagedEscrowJob->sealed_payload
|
|
|
|
|
=== $damagedEscrowPayload
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$damagedEscrowInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'Damaged escrow changed the ledger or released its blocker.');
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->where(
|
|
|
|
|
'event_id',
|
|
|
|
|
$damagedEscrowReceipt['id']
|
|
|
|
|
)->update(['received_at' => time() - 21601]);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'Damaged escrow was not retried beyond six hours.');
|
|
|
|
|
$damagedEscrowJob = CloudJob::find($damagedEscrowJobId);
|
|
|
|
|
$damagedEscrowEvent = CloudAgentCompletionEvent::find(
|
|
|
|
|
$damagedEscrowReceipt['id']
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $damagedEscrowJob->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $damagedEscrowEvent->disposition === 'received'
|
|
|
|
|
&& (int) $damagedEscrowJob->available_at >= time() + 290
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$damagedEscrowInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'Damaged escrow left the recovery queue or released its blocker.');
|
|
|
|
|
|
|
|
|
|
// A structurally valid credential DTO must still match the setting being
|
|
|
|
|
// recovered. Otherwise a late receipt could overwrite credential metadata
|
|
|
|
|
// for a hostname-only change or forget that a newly installed SSH key is
|
|
|
|
|
// available.
|
|
|
|
|
$sshBlob = pack('N', strlen('ssh-ed25519')) . 'ssh-ed25519'
|
|
|
|
|
. pack('N', 32) . str_repeat("\x02", 32);
|
|
|
|
|
$recoveryPublicKey = 'ssh-ed25519 ' . base64_encode($sshBlob);
|
|
|
|
|
foreach ([
|
|
|
|
|
[
|
|
|
|
|
'label' => 'hostname-only credential',
|
|
|
|
|
'instance_id' => 103,
|
|
|
|
|
'uuid' => 'aaaaaaaa-4444-4555-8666-eeeeeeeeeeee',
|
|
|
|
|
'operation_id' => 'd234567890abcdef1234567890abcdef',
|
|
|
|
|
'event_id' => 'settings-hostname-credential-damaged-01',
|
|
|
|
|
'body' => [
|
|
|
|
|
'hostname' => 'credential-damaged.example',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'settings_fields' => ['hostname'],
|
|
|
|
|
'credential' => [
|
|
|
|
|
'password' => 'Credential-damaged9!',
|
|
|
|
|
'login_name' => 'root',
|
|
|
|
|
'auth_mode' => 'password',
|
|
|
|
|
],
|
|
|
|
|
'result' => [
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
'hostname' => 'credential-damaged.example',
|
|
|
|
|
],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
'label' => 'SSH-key credential mode',
|
|
|
|
|
'instance_id' => 104,
|
|
|
|
|
'uuid' => 'aaaaaaaa-5555-4666-8777-ffffffffffff',
|
|
|
|
|
'operation_id' => 'e234567890abcdef1234567890abcdef',
|
|
|
|
|
'event_id' => 'settings-ssh-credential-damaged-0001',
|
|
|
|
|
'body' => [
|
|
|
|
|
'ssh_public_key' => $recoveryPublicKey,
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'settings_fields' => ['ssh_key'],
|
|
|
|
|
'credential' => [
|
|
|
|
|
'password' => 'Credential-damaged9!',
|
|
|
|
|
'login_name' => 'root',
|
|
|
|
|
'auth_mode' => 'password',
|
|
|
|
|
],
|
|
|
|
|
'result' => ['settings_changed' => true],
|
|
|
|
|
],
|
|
|
|
|
] as $credentialCase) {
|
|
|
|
|
$insertInstance(
|
|
|
|
|
$credentialCase['instance_id'],
|
|
|
|
|
$credentialCase['uuid'],
|
|
|
|
|
'credential-semantics-settings-vm-'
|
|
|
|
|
. $credentialCase['instance_id']
|
|
|
|
|
);
|
|
|
|
|
$credentialJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$credentialCase['operation_id'],
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $credentialJobId)->update([
|
|
|
|
|
'instance_id' => $credentialCase['instance_id'],
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $credentialCase['uuid']
|
|
|
|
|
. '/settings',
|
|
|
|
|
'body' => $credentialCase['body'],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'settings_fields' => $credentialCase['settings_fields'],
|
|
|
|
|
'credential' => $credentialCase['credential'],
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$credentialJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, $credentialCase['label'] . ' fixture did not expire.');
|
|
|
|
|
$credentialPayload = (string) CloudJob::find(
|
|
|
|
|
$credentialJobId
|
|
|
|
|
)->sealed_payload;
|
|
|
|
|
$credentialReceipt = $withIdentity([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => $credentialCase['event_id'],
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => $credentialCase['operation_id'],
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => $credentialCase['result'],
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudOperationCompletionService::accept(
|
|
|
|
|
$credentialReceipt
|
|
|
|
|
) === ['accepted' => true, 'disposition' => 'received'],
|
|
|
|
|
$credentialCase['label']
|
|
|
|
|
. ' corruption falsely converged from a valid receipt.');
|
|
|
|
|
$credentialJob = CloudJob::find($credentialJobId);
|
|
|
|
|
$credentialEvent = CloudAgentCompletionEvent::find(
|
|
|
|
|
$credentialCase['event_id']
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $credentialJob->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $credentialJob->sealed_payload === $credentialPayload
|
|
|
|
|
&& (string) $credentialEvent->disposition === 'received'
|
|
|
|
|
&& Capsule::table('cloud_instances')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$credentialCase['instance_id']
|
|
|
|
|
)->value('sealed_credential') === null
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$credentialCase['instance_id']
|
|
|
|
|
)->exists(),
|
|
|
|
|
$credentialCase['label']
|
|
|
|
|
. ' corruption changed the ledger or released its blocker.');
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->where(
|
|
|
|
|
'event_id',
|
|
|
|
|
$credentialCase['event_id']
|
|
|
|
|
)->update(['received_at' => time() - 21601]);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
$credentialCase['label']
|
|
|
|
|
. ' receipt was not retried beyond six hours.');
|
|
|
|
|
$credentialJob = CloudJob::find($credentialJobId);
|
|
|
|
|
$credentialEvent = CloudAgentCompletionEvent::find(
|
|
|
|
|
$credentialCase['event_id']
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $credentialJob->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $credentialEvent->disposition === 'received'
|
|
|
|
|
&& (int) $credentialJob->available_at >= time() + 290
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$credentialCase['instance_id']
|
|
|
|
|
)->exists(),
|
|
|
|
|
$credentialCase['label']
|
|
|
|
|
. ' corruption left the recovery queue or released its blocker.');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// A stored event whose immutable fields no longer match its accepted hash
|
|
|
|
|
// must be rejected before any settings projection or prepare marker moves.
|
|
|
|
|
$corruptEventInstanceId = 101;
|
|
|
|
|
$corruptEventUuid = 'aaaaaaaa-3333-4444-8555-dddddddddddd';
|
|
|
|
|
$insertInstance(
|
|
|
|
|
$corruptEventInstanceId,
|
|
|
|
|
$corruptEventUuid,
|
|
|
|
|
'corrupt-event-settings-vm'
|
|
|
|
|
);
|
|
|
|
|
$corruptEventOperationId = 'a234567890abcdef1234567890abcdef';
|
|
|
|
|
$corruptEventJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$corruptEventOperationId,
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$corruptEventJobId
|
|
|
|
|
)->update([
|
|
|
|
|
'instance_id' => $corruptEventInstanceId,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $corruptEventUuid
|
|
|
|
|
. '/settings',
|
|
|
|
|
'body' => [
|
|
|
|
|
'hostname' => 'corrupt-event.example',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'settings_fields' => ['hostname'],
|
|
|
|
|
'credential' => null,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$corruptEventJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The corrupt-event fixture did not expire.');
|
|
|
|
|
$corruptEventId = 'settings-corrupt-stored-event-01';
|
|
|
|
|
$corruptAccepted = [
|
|
|
|
|
'event_id' => $corruptEventId,
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'operation_id' => $corruptEventOperationId,
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => [
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
'hostname' => 'corrupt-event.example',
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
|
|
|
|
'event_id' => $corruptEventId,
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'operation_id' => $corruptEventOperationId,
|
|
|
|
|
'payload_sha256' => hash('sha256', json_encode(
|
|
|
|
|
$corruptAccepted,
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE
|
|
|
|
|
)),
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
// Simulate durable-row corruption after the original receipt hash was
|
|
|
|
|
// accepted. This must fail the exact-event reconstruction.
|
|
|
|
|
'result_json' => json_encode([
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
'hostname' => 'tampered.example',
|
|
|
|
|
], JSON_THROW_ON_ERROR),
|
|
|
|
|
'disposition' => 'received',
|
|
|
|
|
'received_at' => time(),
|
|
|
|
|
'applied_at' => null,
|
|
|
|
|
]);
|
|
|
|
|
$corruptEventJobBefore = CloudJob::find($corruptEventJobId);
|
|
|
|
|
$corruptEventSpecBefore = (string) Capsule::table('cloud_instances')
|
|
|
|
|
->where('id', $corruptEventInstanceId)->value('spec_json');
|
|
|
|
|
$assert(CloudOperationRecoveryService::settleLateReceipt(
|
|
|
|
|
$corruptEventJobBefore,
|
|
|
|
|
CloudAgentCompletionEvent::find($corruptEventId)
|
|
|
|
|
) === 'rejected',
|
|
|
|
|
'A hash-inconsistent stored event was not rejected.');
|
|
|
|
|
$corruptEventJobAfter = CloudJob::find($corruptEventJobId);
|
|
|
|
|
$assert((string) $corruptEventJobAfter->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $corruptEventJobAfter->effect_resolution_reason
|
|
|
|
|
=== (string) $corruptEventJobBefore->effect_resolution_reason
|
|
|
|
|
&& (string) $corruptEventJobAfter->sealed_payload
|
|
|
|
|
=== (string) $corruptEventJobBefore->sealed_payload
|
|
|
|
|
&& (string) Capsule::table('cloud_instances')
|
|
|
|
|
->where('id', $corruptEventInstanceId)->value('spec_json')
|
|
|
|
|
=== $corruptEventSpecBefore,
|
|
|
|
|
'A hash-inconsistent event changed PHP settings state.');
|
|
|
|
|
|
|
|
|
|
// Agent VM metadata may include a hostname even when the original setting
|
|
|
|
|
// changed only a password. Preserve that exact bounded observation in both
|
|
|
|
|
// recovery sources; the settings finalizer ignores it for password-only
|
|
|
|
|
// requests instead of rewriting the receipt into a synthetic result.
|
|
|
|
|
$extraHostnameInstanceId = 102;
|
|
|
|
|
$extraHostnameUuid = 'aaaaaaaa-4444-4555-8666-eeeeeeeeeeee';
|
|
|
|
|
$insertInstance(
|
|
|
|
|
$extraHostnameInstanceId,
|
|
|
|
|
$extraHostnameUuid,
|
|
|
|
|
'extra-hostname-settings-vm'
|
|
|
|
|
);
|
|
|
|
|
$extraHostnameOperationId = 'b234567890abcdef1234567890abcdef';
|
|
|
|
|
$extraHostnameJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$extraHostnameOperationId,
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$extraHostnameJobId
|
|
|
|
|
)->update([
|
|
|
|
|
'instance_id' => $extraHostnameInstanceId,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $extraHostnameUuid
|
|
|
|
|
. '/settings',
|
|
|
|
|
'body' => [
|
|
|
|
|
'root_password' => 'Extra-hostname9!Pass',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'settings_fields' => ['password'],
|
|
|
|
|
'credential' => [
|
|
|
|
|
'password' => 'Extra-hostname9!Pass',
|
|
|
|
|
'login_name' => 'root',
|
|
|
|
|
'auth_mode' => 'password',
|
|
|
|
|
],
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$extraHostnameJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The extra-hostname fixture did not expire.');
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$extraHostnameJobId
|
|
|
|
|
)->update([
|
|
|
|
|
'effect_resolution_reason' => 'automatic_recovery:v1:'
|
|
|
|
|
. (time() - 60) . ':1:pending',
|
|
|
|
|
]);
|
|
|
|
|
$extraHostnameOperation = $identityFor(
|
|
|
|
|
$extraHostnameOperationId,
|
|
|
|
|
true
|
|
|
|
|
) + [
|
|
|
|
|
'operation_id' => $extraHostnameOperationId,
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => [
|
|
|
|
|
'uuid' => $extraHostnameUuid,
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
'hostname' => 'metadata-only.example',
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
CloudOperationRecoveryService::prepareApplied(
|
|
|
|
|
$extraHostnameJobId,
|
|
|
|
|
$extraHostnameOperation,
|
|
|
|
|
'automatic_applied_succeeded_nofence:' . str_repeat('7', 64)
|
|
|
|
|
);
|
|
|
|
|
$extraHostnameReceipt = $withIdentity([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => 'settings-extra-hostname-event-01',
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => $extraHostnameOperationId,
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => [
|
|
|
|
|
'uuid' => $extraHostnameUuid,
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
'hostname' => 'metadata-only.example',
|
|
|
|
|
],
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudOperationCompletionService::accept(
|
|
|
|
|
$extraHostnameReceipt
|
|
|
|
|
) === ['accepted' => true, 'disposition' => 'applied'],
|
|
|
|
|
'An unrelated receipt hostname conflicted with password recovery.');
|
|
|
|
|
$extraHostnameJob = CloudJob::find($extraHostnameJobId);
|
|
|
|
|
$assert((string) $extraHostnameJob->effect_resolution === 'applied'
|
|
|
|
|
&& (string) $extraHostnameJob->effect_resolution_reason
|
|
|
|
|
=== 'late_completion_receipt:' . $extraHostnameReceipt['id']
|
|
|
|
|
&& !(bool) $extraHostnameJob->finalization_incident
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$extraHostnameInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'Compatible password recovery did not release its blocker.');
|
|
|
|
|
|
|
|
|
|
// A failed receipt which still reports possible effect is not no-effect
|
|
|
|
|
// evidence. It remains late and the original unknown-effect fence stays.
|
|
|
|
|
$ambiguousInstanceId = 94;
|
|
|
|
|
$ambiguousUuid = 'cccccccc-dddd-4eee-8fff-000000000001';
|
|
|
|
|
$insertInstance(
|
|
|
|
|
$ambiguousInstanceId,
|
|
|
|
|
$ambiguousUuid,
|
|
|
|
|
'ambiguous-settings-vm'
|
|
|
|
|
);
|
|
|
|
|
$ambiguousOperationId = '3234567890abcdef1234567890abcdef';
|
|
|
|
|
$ambiguousJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$ambiguousOperationId,
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $ambiguousJobId)->update([
|
|
|
|
|
'instance_id' => $ambiguousInstanceId,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $ambiguousUuid . '/settings',
|
|
|
|
|
'body' => [
|
|
|
|
|
'hostname' => 'ambiguous.example',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'settings_fields' => ['hostname'],
|
|
|
|
|
'credential' => null,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$ambiguousJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The ambiguous late receipt fixture did not expire.');
|
|
|
|
|
$ambiguousReceipt = $withIdentity([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => 'settings-ambiguous-event-0001',
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => $ambiguousOperationId,
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'effect_may_have_applied' => true,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => 'operation_failed',
|
|
|
|
|
'result' => null,
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudOperationCompletionService::accept(
|
|
|
|
|
$ambiguousReceipt
|
|
|
|
|
) === ['accepted' => true, 'disposition' => 'late']
|
|
|
|
|
&& (string) CloudJob::find($ambiguousJobId)->effect_resolution
|
|
|
|
|
=== 'unknown'
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$ambiguousInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'Ambiguous late failure incorrectly released the settings fence.');
|
|
|
|
|
|
|
|
|
|
// Live inspection uses the same PHP-only recovery seam in two phases:
|
|
|
|
|
// first converge the ledger while preserving the blocker, then publish
|
|
|
|
|
// the resolution only after the caller releases the Agent fence.
|
|
|
|
|
$preparedInstanceId = 92;
|
|
|
|
|
$preparedUuid = 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee';
|
|
|
|
|
$insertInstance(
|
|
|
|
|
$preparedInstanceId,
|
|
|
|
|
$preparedUuid,
|
|
|
|
|
'prepared-settings-vm'
|
|
|
|
|
);
|
|
|
|
|
$preparedOperationId = '1234567890abcdef1234567890abcdef';
|
|
|
|
|
$preparedJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$preparedOperationId,
|
|
|
|
|
'awaiting_receipt',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $preparedJobId)->update([
|
|
|
|
|
'instance_id' => $preparedInstanceId,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'PATCH',
|
|
|
|
|
'request_target' => '/v1/vms/' . $preparedUuid . '/settings',
|
|
|
|
|
'body' => [
|
|
|
|
|
'hostname' => 'prepared-recovered.example',
|
|
|
|
|
'root_password' => 'Prepared-password9!',
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'settings_fields' => ['hostname', 'password'],
|
|
|
|
|
'credential' => [
|
|
|
|
|
'password' => 'Prepared-password9!',
|
|
|
|
|
'login_name' => 'root',
|
|
|
|
|
'auth_mode' => 'password',
|
|
|
|
|
],
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$preparedJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'The two-phase recovery fixture did not reach terminal failure.');
|
|
|
|
|
$preparedTerminal = CloudJob::find($preparedJobId);
|
|
|
|
|
$preparedTerminalAt = (int) $preparedTerminal->terminal_at;
|
|
|
|
|
$preparedError = (string) $preparedTerminal->error_code;
|
|
|
|
|
$preparedResult = $preparedTerminal->result_json;
|
|
|
|
|
$preparedCompletionEvent = $preparedTerminal->completion_event_id;
|
|
|
|
|
$preparedReadAt = $preparedTerminal->read_at;
|
|
|
|
|
$preparedOperation = $identityFor($preparedOperationId, true) + [
|
|
|
|
|
'operation_id' => $preparedOperationId,
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => [
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
'hostname' => 'prepared-recovered.example',
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
$preparedEvidence =
|
|
|
|
|
'agent_inspection:settings-inspection-proof-0001';
|
|
|
|
|
$prepared = CloudOperationRecoveryService::prepareApplied(
|
|
|
|
|
$preparedJobId,
|
|
|
|
|
$preparedOperation,
|
|
|
|
|
$preparedEvidence
|
|
|
|
|
);
|
|
|
|
|
$preparedPayload = CloudSecrets::decrypt(
|
|
|
|
|
(string) $prepared->sealed_payload,
|
|
|
|
|
'job-payload'
|
|
|
|
|
);
|
|
|
|
|
$preparedInstance = Capsule::table('cloud_instances')
|
|
|
|
|
->where('id', $preparedInstanceId)->first();
|
|
|
|
|
$preparedSpec = json_decode(
|
|
|
|
|
(string) $preparedInstance->spec_json,
|
|
|
|
|
true,
|
|
|
|
|
flags: JSON_THROW_ON_ERROR
|
|
|
|
|
);
|
|
|
|
|
$preparedUpdatedAt = (int) $prepared->updated_at;
|
|
|
|
|
$preparedAgain = CloudOperationRecoveryService::prepareApplied(
|
|
|
|
|
$preparedJobId,
|
|
|
|
|
$preparedOperation,
|
|
|
|
|
$preparedEvidence
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $prepared->phase === 'terminal'
|
|
|
|
|
&& (string) $prepared->outcome === 'failed'
|
|
|
|
|
&& (string) $prepared->status === 'failed'
|
|
|
|
|
&& (bool) $prepared->effect_may_have_applied
|
|
|
|
|
&& !(bool) $prepared->safe_to_resubmit
|
|
|
|
|
&& (string) $prepared->effect_resolution === 'unknown'
|
|
|
|
|
&& str_starts_with(
|
|
|
|
|
(string) $prepared->effect_resolution_reason,
|
|
|
|
|
'prepared:' . $preparedEvidence . ':'
|
|
|
|
|
)
|
|
|
|
|
&& (int) $prepared->terminal_at === $preparedTerminalAt
|
|
|
|
|
&& (string) $prepared->error_code === $preparedError
|
|
|
|
|
&& $prepared->result_json === $preparedResult
|
|
|
|
|
&& $prepared->completion_event_id === $preparedCompletionEvent
|
|
|
|
|
&& $prepared->read_at === $preparedReadAt
|
|
|
|
|
&& ($preparedPayload['credential']['password'] ?? null)
|
|
|
|
|
=== 'Prepared-password9!'
|
|
|
|
|
&& ($preparedSpec['guest_hostname'] ?? null)
|
|
|
|
|
=== 'prepared-recovered.example'
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$preparedInstanceId
|
|
|
|
|
)->exists()
|
|
|
|
|
&& (int) $preparedAgain->updated_at === $preparedUpdatedAt,
|
|
|
|
|
'Recovery prepare did not atomically converge while retaining fences.');
|
|
|
|
|
foreach (['commitApplied', 'commitNotApplied'] as $commitMethod) {
|
|
|
|
|
$negativeResolverRejected = false;
|
|
|
|
|
try {
|
|
|
|
|
$commit = [CloudOperationRecoveryService::class, $commitMethod];
|
|
|
|
|
$commit(
|
|
|
|
|
$preparedJobId,
|
|
|
|
|
$preparedOperation,
|
|
|
|
|
$preparedEvidence,
|
|
|
|
|
-1
|
|
|
|
|
);
|
|
|
|
|
} catch (\App\Services\Cloud\CloudError $error) {
|
|
|
|
|
$negativeResolverRejected = $error->httpStatus === 409;
|
|
|
|
|
}
|
|
|
|
|
$assert($negativeResolverRejected
|
|
|
|
|
&& (string) CloudJob::find($preparedJobId)->effect_resolution
|
|
|
|
|
=== 'unknown',
|
|
|
|
|
$commitMethod . ' accepted a negative recovery audit actor.');
|
|
|
|
|
}
|
|
|
|
|
$committed = CloudOperationRecoveryService::commitApplied(
|
|
|
|
|
$preparedJobId,
|
|
|
|
|
$preparedOperation,
|
|
|
|
|
$preparedEvidence
|
|
|
|
|
);
|
|
|
|
|
$committedPayload = CloudSecrets::decrypt(
|
|
|
|
|
(string) $committed->sealed_payload,
|
|
|
|
|
'job-payload'
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $committed->phase === 'terminal'
|
|
|
|
|
&& (string) $committed->outcome === 'failed'
|
|
|
|
|
&& (string) $committed->status === 'failed'
|
|
|
|
|
&& (bool) $committed->effect_may_have_applied
|
|
|
|
|
&& !(bool) $committed->safe_to_resubmit
|
|
|
|
|
&& (string) $committed->effect_resolution === 'applied'
|
|
|
|
|
&& (string) $committed->effect_resolution_reason
|
|
|
|
|
=== $preparedEvidence
|
|
|
|
|
&& (int) $committed->effect_resolved_at > 0
|
|
|
|
|
&& (int) $committed->terminal_at === $preparedTerminalAt
|
|
|
|
|
&& (string) $committed->error_code === $preparedError
|
|
|
|
|
&& $committed->result_json === $preparedResult
|
|
|
|
|
&& $committed->completion_event_id === $preparedCompletionEvent
|
|
|
|
|
&& $committed->read_at === $preparedReadAt
|
|
|
|
|
&& !array_key_exists('credential', $committedPayload)
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$preparedInstanceId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'Recovery commit did not publish the effect and clear its escrow.');
|
|
|
|
|
$committedAgain = CloudOperationRecoveryService::commitApplied(
|
|
|
|
|
$preparedJobId,
|
|
|
|
|
$preparedOperation,
|
|
|
|
|
$preparedEvidence
|
|
|
|
|
);
|
|
|
|
|
$mismatchedPreparedOperation = $preparedOperation;
|
|
|
|
|
$mismatchedPreparedOperation['operation_id'] =
|
|
|
|
|
'00000000000000000000000000000001';
|
|
|
|
|
$mismatchedCommitRejected = false;
|
|
|
|
|
try {
|
|
|
|
|
CloudOperationRecoveryService::commitApplied(
|
|
|
|
|
$preparedJobId,
|
|
|
|
|
$mismatchedPreparedOperation,
|
|
|
|
|
$preparedEvidence
|
|
|
|
|
);
|
|
|
|
|
} catch (\App\Services\Cloud\CloudError $error) {
|
|
|
|
|
$mismatchedCommitRejected = $error->httpStatus === 409;
|
|
|
|
|
}
|
|
|
|
|
$assert((string) $committedAgain->effect_resolution === 'applied'
|
|
|
|
|
&& $mismatchedCommitRejected,
|
|
|
|
|
'Recovery commit was not idempotent or accepted another operation.');
|
|
|
|
|
|
|
|
|
|
$insertPaymentJob = static function (string $key, int $invoiceId)
|
|
|
|
|
use ($now): int {
|
|
|
|
|
$operationId = md5('payment-deadline:' . $key);
|
|
|
|
|
return (int) Capsule::table('cloud_jobs')->insertGetId([
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'request_idempotency_key' => $key,
|
|
|
|
|
'request_fingerprint' => hash('sha256', $key),
|
|
|
|
|
'parent_job_id' => null, 'order_id' => null,
|
|
|
|
|
'invoice_id' => $invoiceId, 'instance_id' => null,
|
|
|
|
|
'snapshot_id' => null, 'user_id' => 1, 'host_id' => null,
|
|
|
|
|
'kind' => 'payment_create', 'execution_mode' => 'local',
|
|
|
|
|
'phase' => 'dispatching', 'outcome' => null,
|
|
|
|
|
'status' => 'running',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'input' => ['idempotency_key' => $key,
|
|
|
|
|
'method' => 'paytaro'],
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
'attempts' => 1, 'dispatch_attempts' => 1,
|
|
|
|
|
'dispatch_deadline_at' => $now - 20,
|
|
|
|
|
'dispatch_started_at' => $now - 30,
|
|
|
|
|
'agent_accepted_at' => null,
|
|
|
|
|
'receipt_deadline_at' => $now - 10,
|
|
|
|
|
'terminal_at' => null,
|
|
|
|
|
'effect_may_have_applied' => null,
|
|
|
|
|
'safe_to_resubmit' => null,
|
|
|
|
|
'effect_resolution' => 'not_applicable',
|
|
|
|
|
'effect_resolved_at' => null, 'effect_resolved_by' => null,
|
|
|
|
|
'effect_resolution_reason' => '',
|
|
|
|
|
'completion_event_id' => null, 'read_at' => null,
|
|
|
|
|
'dispatched_at' => $now - 30,
|
|
|
|
|
'lease_token' => bin2hex(random_bytes(16)),
|
|
|
|
|
'lease_until' => $now + 600, 'available_at' => 0,
|
|
|
|
|
'error_code' => null, 'result_json' => null,
|
|
|
|
|
'created_at' => $now - 60, 'updated_at' => $now - 30,
|
|
|
|
|
]);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$missingPaymentKey = 'payment-missing-0001';
|
|
|
|
|
$pendingPaymentKey = 'payment-pending-0001';
|
|
|
|
|
$unknownPaymentKey = 'payment-unknown-0001';
|
|
|
|
|
Capsule::table('cloud_invoice_payments')->insert([
|
|
|
|
|
[
|
|
|
|
|
'invoice_id' => 502, 'user_id' => 1, 'provider' => 'paytaro',
|
|
|
|
|
'provider_order_no' => 'provider-pending-0001',
|
|
|
|
|
'idempotency_key' => $pendingPaymentKey,
|
|
|
|
|
'amount' => 10, 'status' => 'pending',
|
|
|
|
|
'created_at' => $now - 30, 'updated_at' => $now - 30,
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
'invoice_id' => 503, 'user_id' => 1, 'provider' => 'paytaro',
|
|
|
|
|
'provider_order_no' => 'provider-unknown-0001',
|
|
|
|
|
'idempotency_key' => $unknownPaymentKey,
|
|
|
|
|
'amount' => 10, 'status' => 'creation_unknown',
|
|
|
|
|
'created_at' => $now - 30, 'updated_at' => $now - 30,
|
|
|
|
|
],
|
|
|
|
|
]);
|
|
|
|
|
$missingPaymentJobId = $insertPaymentJob($missingPaymentKey, 501);
|
|
|
|
|
$pendingPaymentJobId = $insertPaymentJob($pendingPaymentKey, 502);
|
|
|
|
|
$unknownPaymentJobId = $insertPaymentJob($unknownPaymentKey, 503);
|
|
|
|
|
foreach ([$missingPaymentJobId, $pendingPaymentJobId,
|
|
|
|
|
$unknownPaymentJobId] as $paymentJobId) {
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1, $paymentJobId, 1
|
|
|
|
|
) === 1, 'A due local payment task was not terminalized.');
|
|
|
|
|
}
|
|
|
|
|
$missingPaymentJob = CloudJob::find($missingPaymentJobId);
|
|
|
|
|
$pendingPaymentJob = CloudJob::find($pendingPaymentJobId);
|
|
|
|
|
$unknownPaymentJob = CloudJob::find($unknownPaymentJobId);
|
|
|
|
|
$pendingPaymentResult = json_decode(
|
|
|
|
|
(string) $pendingPaymentJob->result_json, true,
|
|
|
|
|
flags: JSON_THROW_ON_ERROR
|
|
|
|
|
);
|
|
|
|
|
$assert($missingPaymentJob->phase === 'terminal'
|
|
|
|
|
&& $missingPaymentJob->outcome === 'failed'
|
|
|
|
|
&& $missingPaymentJob->effect_resolution === 'not_applied'
|
|
|
|
|
&& !(bool) $missingPaymentJob->effect_may_have_applied
|
|
|
|
|
&& (bool) $missingPaymentJob->safe_to_resubmit
|
|
|
|
|
&& $missingPaymentJob->error_code === 'payment_order_not_dispatched'
|
|
|
|
|
&& $pendingPaymentJob->phase === 'terminal'
|
|
|
|
|
&& $pendingPaymentJob->outcome === 'succeeded'
|
|
|
|
|
&& ($pendingPaymentResult['payment_id'] ?? null) === '1'
|
|
|
|
|
&& $unknownPaymentJob->phase === 'terminal'
|
|
|
|
|
&& $unknownPaymentJob->outcome === 'failed'
|
|
|
|
|
&& $unknownPaymentJob->effect_resolution === 'unknown'
|
|
|
|
|
&& (bool) $unknownPaymentJob->effect_may_have_applied
|
|
|
|
|
&& !(bool) $unknownPaymentJob->safe_to_resubmit
|
|
|
|
|
&& $unknownPaymentJob->error_code === 'payment_creation_unknown'
|
|
|
|
|
&& Capsule::table('cloud_invoice_payments')->count() === 2,
|
|
|
|
|
'Payment deadline fallback retried a provider POST or misclassified its durable attempt.');
|
|
|
|
|
|
|
|
|
|
$nullDispatchOperation = md5('null-dispatch-deadline-fixture');
|
|
|
|
|
$nullDispatchJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$nullDispatchOperation,
|
|
|
|
|
'accepted',
|
|
|
|
|
0
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $nullDispatchJobId)->update([
|
|
|
|
|
'dispatch_deadline_at' => null,
|
|
|
|
|
]);
|
|
|
|
|
$invalidReceiptOperation = md5('invalid-receipt-deadline-fixture');
|
|
|
|
|
$invalidReceiptJobId = $insertJob(
|
|
|
|
|
1,
|
|
|
|
|
$invalidReceiptOperation,
|
|
|
|
|
'dispatching',
|
|
|
|
|
1
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $invalidReceiptJobId)->update([
|
|
|
|
|
'receipt_deadline_at' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$nullDispatchJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'An accepted task with a NULL dispatch deadline stayed active.');
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1,
|
|
|
|
|
$invalidReceiptJobId,
|
|
|
|
|
1
|
|
|
|
|
) === 1, 'A dispatched task with an invalid receipt deadline stayed active.');
|
|
|
|
|
$nullDispatchJob = CloudJob::find($nullDispatchJobId);
|
|
|
|
|
$invalidReceiptJob = CloudJob::find($invalidReceiptJobId);
|
|
|
|
|
$assert($nullDispatchJob !== null
|
|
|
|
|
&& (string) $nullDispatchJob->phase === 'terminal'
|
|
|
|
|
&& (string) $nullDispatchJob->outcome === 'failed'
|
|
|
|
|
&& !(bool) $nullDispatchJob->effect_may_have_applied
|
|
|
|
|
&& (bool) $nullDispatchJob->safe_to_resubmit
|
|
|
|
|
&& $invalidReceiptJob !== null
|
|
|
|
|
&& (string) $invalidReceiptJob->phase === 'terminal'
|
|
|
|
|
&& (string) $invalidReceiptJob->outcome === 'failed'
|
|
|
|
|
&& (bool) $invalidReceiptJob->effect_may_have_applied
|
|
|
|
|
&& !(bool) $invalidReceiptJob->safe_to_resubmit,
|
|
|
|
|
'Malformed active deadlines did not fail closed by dispatch evidence.');
|
|
|
|
|
|
|
|
|
|
$assert(CloudJobService::dispatchDeadlineSeconds('action') === 300
|
|
|
|
|
&& CloudJobService::dispatchDeadlineSeconds(
|
|
|
|
|
'host_storage_configure') === 60
|
|
|
|
|
&& CloudJobService::receiptDeadlineSeconds('action') === 960
|
|
|
|
|
&& CloudJobService::receiptDeadlineSeconds(
|
|
|
|
|
'host_storage_configure') === 180
|
|
|
|
|
&& CloudJobService::receiptDeadlineSeconds(
|
|
|
|
|
'template_import_replica') === 8100
|
|
|
|
|
&& CloudJobService::receiptDeadlineSeconds(
|
|
|
|
|
'rescue_template_import') === 8100
|
|
|
|
|
&& CloudJobService::receiptDeadlineSeconds(
|
|
|
|
|
'rescue_catalog_validate_replica') === 8100
|
|
|
|
|
&& CloudJobService::receiptDeadlineSeconds(
|
|
|
|
|
'rescue_catalog_import_replica') === 8100
|
|
|
|
|
&& CloudJobService::receiptDeadlineSeconds(
|
|
|
|
|
'user_iso_import_replica') === 8100
|
|
|
|
|
&& CloudJobService::aggregateDeadlineSeconds(
|
|
|
|
|
'template_metadata_sync') === 1560
|
|
|
|
|
&& CloudJobService::aggregateDeadlineSeconds(
|
|
|
|
|
'template_import') === 8700
|
|
|
|
|
&& CloudJobService::aggregateDeadlineSeconds(
|
|
|
|
|
'user_iso_import') === 8700,
|
|
|
|
|
'Task deadlines lost dispatch, receipt, or aggregate finalization margin.');
|
|
|
|
|
|
|
|
|
|
$parentOperation = str_repeat('d', 32);
|
|
|
|
|
$parentId = (int) Capsule::table('cloud_jobs')->insertGetId([
|
|
|
|
|
'operation_id' => $parentOperation,
|
|
|
|
|
'request_idempotency_key' => $parentOperation,
|
|
|
|
|
'request_fingerprint' => hash('sha256', $parentOperation),
|
|
|
|
|
'parent_job_id' => null, 'order_id' => null, 'invoice_id' => null,
|
|
|
|
|
'instance_id' => null, 'snapshot_id' => null,
|
|
|
|
|
'user_id' => 1, 'host_id' => null, 'kind' => 'test_aggregate',
|
|
|
|
|
'execution_mode' => 'aggregate', 'phase' => 'awaiting_receipt',
|
|
|
|
|
'outcome' => null, 'status' => 'running',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
|
|
|
|
|
'attempts' => 0, 'dispatch_attempts' => 0,
|
|
|
|
|
'dispatch_deadline_at' => $now - 30,
|
|
|
|
|
'dispatch_started_at' => null, 'agent_accepted_at' => null,
|
|
|
|
|
'receipt_deadline_at' => $now - 1, 'terminal_at' => null,
|
|
|
|
|
'effect_may_have_applied' => null, 'safe_to_resubmit' => null,
|
|
|
|
|
'effect_resolution' => 'not_applicable',
|
|
|
|
|
'effect_resolved_at' => null, 'effect_resolved_by' => null,
|
|
|
|
|
'effect_resolution_reason' => '', 'completion_event_id' => null,
|
|
|
|
|
'read_at' => null, 'dispatched_at' => null,
|
|
|
|
|
'lease_token' => str_repeat('9', 32), 'lease_until' => $now + 600,
|
|
|
|
|
'available_at' => 0, 'error_code' => null, 'result_json' => null,
|
|
|
|
|
'created_at' => $now - 100, 'updated_at' => $now - 50,
|
|
|
|
|
]);
|
|
|
|
|
$childOperation = str_repeat('8', 32);
|
|
|
|
|
$childId = (int) Capsule::table('cloud_jobs')->insertGetId([
|
|
|
|
|
'operation_id' => $childOperation,
|
|
|
|
|
'request_idempotency_key' => $childOperation,
|
|
|
|
|
'request_fingerprint' => hash('sha256', $childOperation),
|
|
|
|
|
'parent_job_id' => $parentId, 'order_id' => null,
|
|
|
|
|
'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null,
|
|
|
|
|
'user_id' => 1, 'host_id' => 7, 'kind' => 'test_mutation',
|
|
|
|
|
'execution_mode' => 'agent', 'phase' => 'accepted',
|
|
|
|
|
'outcome' => null, 'status' => 'queued',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST', 'request_target' => '/v1/test',
|
|
|
|
|
'body' => ['operation_id' => $childOperation], 'owner_id' => 1,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
'attempts' => 0, 'dispatch_attempts' => 0,
|
|
|
|
|
'dispatch_deadline_at' => $now + 600,
|
|
|
|
|
'dispatch_started_at' => null, 'agent_accepted_at' => null,
|
|
|
|
|
'receipt_deadline_at' => null, 'terminal_at' => null,
|
|
|
|
|
'effect_may_have_applied' => null, 'safe_to_resubmit' => null,
|
|
|
|
|
'effect_resolution' => 'not_applicable',
|
|
|
|
|
'effect_resolved_at' => null, 'effect_resolved_by' => null,
|
|
|
|
|
'effect_resolution_reason' => '', 'completion_event_id' => null,
|
|
|
|
|
'read_at' => null, 'dispatched_at' => null,
|
|
|
|
|
'lease_token' => null, 'lease_until' => 0, 'available_at' => 0,
|
|
|
|
|
'error_code' => null, 'result_json' => null,
|
|
|
|
|
'created_at' => $now - 80, 'updated_at' => $now - 40,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::unprepared('CREATE TRIGGER simulate_child_claim_cas_miss '
|
|
|
|
|
. 'BEFORE UPDATE OF lease_token ON cloud_jobs '
|
|
|
|
|
. 'WHEN OLD.id = ' . $childId . " AND OLD.phase <> 'terminal' "
|
|
|
|
|
. 'BEGIN SELECT RAISE(IGNORE); END');
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1, $parentId, 10
|
|
|
|
|
) === 1, 'Owner fallback did not claim an expired aggregate parent.');
|
|
|
|
|
$parent = CloudJob::find($parentId);
|
|
|
|
|
$child = CloudJob::find($childId);
|
|
|
|
|
$assert($parent !== null && $child !== null
|
|
|
|
|
&& (string) $parent->phase === 'awaiting_receipt'
|
|
|
|
|
&& $parent->outcome === null
|
|
|
|
|
&& $parent->lease_token === null
|
|
|
|
|
&& (int) $parent->lease_until === 0
|
|
|
|
|
&& (string) $child->phase === 'accepted',
|
|
|
|
|
'A child claim CAS miss hid active work behind a terminal parent.');
|
|
|
|
|
|
|
|
|
|
Capsule::unprepared('DROP TRIGGER simulate_child_claim_cas_miss');
|
|
|
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
|
|
|
1, $parentId, 10
|
|
|
|
|
) === 1, 'A later fallback pass did not reclaim the aggregate parent.');
|
|
|
|
|
$parent = CloudJob::find($parentId);
|
|
|
|
|
$child = CloudJob::find($childId);
|
|
|
|
|
$assert($parent !== null && $child !== null
|
|
|
|
|
&& (string) $child->phase === 'terminal'
|
|
|
|
|
&& (string) $child->outcome === 'failed'
|
|
|
|
|
&& !(bool) $child->effect_may_have_applied
|
|
|
|
|
&& (bool) $child->safe_to_resubmit
|
|
|
|
|
&& (string) $parent->phase === 'terminal'
|
|
|
|
|
&& (string) $parent->outcome === 'failed',
|
|
|
|
|
'A later aggregate fallback did not terminalize children first.');
|
|
|
|
|
|
|
|
|
|
echo "Owner-scoped task deadline fallback tests passed.\n";
|
|
|
|
|
}
|