|
|
|
|
<?php
|
|
|
|
|
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Services\Cloud\CloudJobService;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
|
|
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
|
|
|
if (!$condition) {
|
|
|
|
|
throw new RuntimeException($message);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$capsule = new Capsule();
|
|
|
|
|
$capsule->addConnection([
|
|
|
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
|
|
|
], 'default');
|
|
|
|
|
$capsule->setAsGlobal();
|
|
|
|
|
$capsule->bootEloquent();
|
|
|
|
|
$schema = $capsule->schema('default');
|
|
|
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('operation_id');
|
|
|
|
|
$table->string('request_idempotency_key');
|
|
|
|
|
$table->string('request_fingerprint');
|
|
|
|
|
$table->integer('parent_job_id')->nullable();
|
|
|
|
|
$table->integer('order_id')->nullable();
|
|
|
|
|
$table->integer('invoice_id')->nullable();
|
|
|
|
|
$table->integer('instance_id')->nullable();
|
|
|
|
|
$table->integer('snapshot_id')->nullable();
|
|
|
|
|
$table->integer('vpc_batch_id')->nullable();
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->integer('host_id')->nullable();
|
|
|
|
|
$table->string('kind');
|
|
|
|
|
$table->string('execution_mode');
|
|
|
|
|
$table->string('phase');
|
|
|
|
|
$table->string('outcome')->nullable();
|
|
|
|
|
$table->string('status');
|
|
|
|
|
$table->integer('attempts')->default(0);
|
|
|
|
|
$table->integer('dispatch_attempts')->nullable()->default(0);
|
|
|
|
|
$table->integer('dispatch_deadline_at');
|
|
|
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
|
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
|
|
|
$table->integer('dispatched_at')->nullable();
|
|
|
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
|
|
|
$table->integer('terminal_at')->nullable();
|
|
|
|
|
$table->string('completion_event_id')->nullable();
|
|
|
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
|
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
|
|
|
$table->string('effect_resolution');
|
|
|
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
|
|
|
$table->string('effect_resolution_reason')->default('');
|
|
|
|
|
$table->boolean('finalization_incident')->default(0);
|
|
|
|
|
$table->string('finalization_error_code')->nullable();
|
|
|
|
|
$table->string('lease_token')->nullable();
|
|
|
|
|
$table->integer('lease_until')->default(0);
|
|
|
|
|
$table->integer('available_at')->default(0);
|
|
|
|
|
$table->string('error_code')->nullable();
|
|
|
|
|
$table->text('result_json')->nullable();
|
|
|
|
|
$table->text('sealed_payload')->default('');
|
|
|
|
|
$table->integer('read_at')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_templates', static function (Blueprint $table): void {
|
|
|
|
|
$table->string('id')->primary();
|
|
|
|
|
$table->string('catalog_key')->default('test-catalog');
|
|
|
|
|
$table->string('version_label')->default('test-version');
|
|
|
|
|
$table->boolean('is_default')->default(true);
|
|
|
|
|
$table->integer('active_job_id')->nullable();
|
|
|
|
|
$table->string('state')->default('active');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_user_isos', static function (Blueprint $table): void {
|
|
|
|
|
$table->string('id')->primary();
|
|
|
|
|
$table->integer('active_job_id')->nullable();
|
|
|
|
|
$table->string('state')->default('active');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_template_replicas',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('template_id');
|
|
|
|
|
$table->integer('revision')->default(0);
|
|
|
|
|
$table->string('generation_token')->default('');
|
|
|
|
|
$table->string('state')->default('pending');
|
|
|
|
|
$table->integer('image_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->string('operation_id')->default('');
|
|
|
|
|
$table->integer('job_id')->nullable();
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('created_at')->default(0);
|
|
|
|
|
$table->integer('updated_at')->default(0);
|
|
|
|
|
$table->primary(['host_id', 'template_id']);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_user_iso_replicas',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('user_iso_id');
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->string('state')->default('pending');
|
|
|
|
|
$table->integer('size_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->string('operation_id')->default('');
|
|
|
|
|
$table->integer('job_id')->nullable();
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('created_at')->default(0);
|
|
|
|
|
$table->integer('updated_at')->default(0);
|
|
|
|
|
$table->primary(['host_id', 'user_iso_id']);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
$now = time();
|
|
|
|
|
$insert = static function (int $id, array $overrides = []) use ($now): void {
|
|
|
|
|
$values = array_replace([
|
|
|
|
|
'id' => $id,
|
|
|
|
|
'operation_id' => sprintf('%032x', $id),
|
|
|
|
|
'request_idempotency_key' => sprintf('%032x', $id),
|
|
|
|
|
'request_fingerprint' => hash('sha256', (string) $id),
|
|
|
|
|
'parent_job_id' => null,
|
|
|
|
|
'order_id' => null,
|
|
|
|
|
'invoice_id' => null,
|
|
|
|
|
'instance_id' => 10,
|
|
|
|
|
'snapshot_id' => null,
|
|
|
|
|
'user_id' => 7,
|
|
|
|
|
'host_id' => 3,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'execution_mode' => 'agent',
|
|
|
|
|
'phase' => 'terminal',
|
|
|
|
|
'outcome' => 'failed',
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'attempts' => 1,
|
|
|
|
|
'dispatch_attempts' => 1,
|
|
|
|
|
'dispatch_deadline_at' => $now - 100,
|
|
|
|
|
'dispatch_started_at' => $now - 95,
|
|
|
|
|
'agent_accepted_at' => $now - 92,
|
|
|
|
|
'dispatched_at' => $now - 90,
|
|
|
|
|
'receipt_deadline_at' => $now - 60,
|
|
|
|
|
'terminal_at' => $now - 30,
|
|
|
|
|
'completion_event_id' => null,
|
|
|
|
|
'effect_may_have_applied' => 1,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'unknown',
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_evidence_conflict',
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
'available_at' => $now + 30,
|
|
|
|
|
'error_code' => 'operation_failed_effect_unknown',
|
|
|
|
|
'created_at' => $now - 120,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
], $overrides);
|
|
|
|
|
if (($values['execution_mode'] ?? '') === 'aggregate') {
|
|
|
|
|
foreach ([
|
|
|
|
|
'attempts' => 0,
|
|
|
|
|
'dispatch_started_at' => null,
|
|
|
|
|
'agent_accepted_at' => null,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'completion_event_id' => null,
|
|
|
|
|
] as $field => $value) {
|
|
|
|
|
if (!array_key_exists($field, $overrides)) {
|
|
|
|
|
$values[$field] = $value;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (($values['execution_mode'] ?? '') === 'agent'
|
|
|
|
|
&& (int) ($values['dispatch_attempts'] ?? -1) === 0) {
|
|
|
|
|
foreach (['attempts' => 0, 'dispatch_started_at' => null,
|
|
|
|
|
'agent_accepted_at' => null, 'dispatched_at' => null,
|
|
|
|
|
'receipt_deadline_at' => null, 'completion_event_id' => null]
|
|
|
|
|
as $field => $value) {
|
|
|
|
|
if (!array_key_exists($field, $overrides)) {
|
|
|
|
|
$values[$field] = $value;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (($values['outcome'] ?? null) === 'succeeded'
|
|
|
|
|
&& !array_key_exists('error_code', $overrides)) {
|
|
|
|
|
$values['error_code'] = null;
|
|
|
|
|
}
|
|
|
|
|
Capsule::table('cloud_jobs')->insert($values);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$insert(1);
|
|
|
|
|
$insert(2, [
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$insert(3, ['kind' => 'create']);
|
|
|
|
|
$insert(4, [
|
|
|
|
|
'phase' => 'accepted',
|
|
|
|
|
'outcome' => null,
|
|
|
|
|
'status' => 'queued',
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'receipt_deadline_at' => null,
|
|
|
|
|
'terminal_at' => null,
|
|
|
|
|
'effect_may_have_applied' => null,
|
|
|
|
|
'safe_to_resubmit' => null,
|
|
|
|
|
'effect_resolution' => 'not_applicable',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'dispatch_deadline_at' => $now + 300,
|
|
|
|
|
'available_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
$insert(5, [
|
|
|
|
|
'kind' => 'template_import',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$insert(6, [
|
|
|
|
|
'parent_job_id' => 5,
|
|
|
|
|
'kind' => 'template_import_replica',
|
|
|
|
|
'finalization_error_code' => 'recovery_operation_identity_conflict',
|
|
|
|
|
'error_code' => 'raw-agent-secret-must-not-appear',
|
|
|
|
|
'result_json' => json_encode([
|
|
|
|
|
'message' => 'raw-agent-text-must-not-appear',
|
|
|
|
|
], JSON_THROW_ON_ERROR),
|
|
|
|
|
]);
|
|
|
|
|
$insert(7, [
|
|
|
|
|
'parent_job_id' => 5,
|
|
|
|
|
'kind' => 'template_import_replica',
|
|
|
|
|
'finalization_error_code' => 'recovery_receipt_invalid',
|
|
|
|
|
]);
|
|
|
|
|
$insert(8000, [
|
|
|
|
|
'kind' => 'vpc_port_attach',
|
|
|
|
|
'vpc_batch_id' => 44,
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
$tasks = CloudJobService::tasks(['user_id' => 7, 'is_admin' => false],
|
|
|
|
|
'completed');
|
|
|
|
|
$assert(($tasks['active_count'] ?? null) === 1,
|
|
|
|
|
'Task center active count changed while adding recovery polling.');
|
|
|
|
|
$assert(($tasks['recovering_count'] ?? null) === 2,
|
|
|
|
|
'Task center did not count visible terminal tasks that still converge.');
|
|
|
|
|
$assert(array_map(static fn (array $item): string => $item['id'],
|
|
|
|
|
$tasks['items']) === ['5', '3', '2', '1'],
|
|
|
|
|
'Task center exposed hidden child work or changed terminal ordering.');
|
|
|
|
|
$assert(($tasks['items'][0]['recovery']['pending'] ?? null) === true
|
|
|
|
|
&& ($tasks['items'][3]['recovery']['pending'] ?? null) === true
|
|
|
|
|
&& !isset($tasks['items'][1]['recovery'])
|
|
|
|
|
&& !isset($tasks['items'][2]['recovery']),
|
|
|
|
|
'Task recovery status did not match the visible recovery count.');
|
|
|
|
|
$aggregateRecovery = $tasks['items'][0]['recovery'] ?? [];
|
|
|
|
|
$adminAggregate = CloudJobService::display(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(5),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(str_contains(
|
|
|
|
|
(string) ($aggregateRecovery['message'] ?? ''),
|
|
|
|
|
'操作、资源或请求指纹'
|
|
|
|
|
)
|
|
|
|
|
&& str_contains(
|
|
|
|
|
(string) ($aggregateRecovery['message'] ?? ''),
|
|
|
|
|
'另有 1 类子任务异常'
|
|
|
|
|
)
|
|
|
|
|
&& !array_key_exists('code', $aggregateRecovery)
|
|
|
|
|
&& !str_contains(
|
|
|
|
|
(string) ($aggregateRecovery['message'] ?? ''),
|
|
|
|
|
'raw-agent'
|
|
|
|
|
)
|
|
|
|
|
&& ($adminAggregate['recovery']['code'] ?? null)
|
|
|
|
|
=== 'recovery_operation_identity_conflict'
|
|
|
|
|
&& ($adminAggregate['recovery']['affected_child_count'] ?? null) === 2
|
|
|
|
|
&& ($adminAggregate['recovery']['additional_error_count'] ?? null) === 1,
|
|
|
|
|
'Aggregate recovery did not safely expose the deterministic child anomaly.');
|
|
|
|
|
|
|
|
|
|
// A dispatch counter outside the durable 0/1 state machine is corruption,
|
|
|
|
|
// not proof that the mutation stayed on the PHP side of the dispatch fence.
|
|
|
|
|
// Keep both the leaf and its public aggregate parent visibly recovering with
|
|
|
|
|
// one reviewed Chinese anomaly instead of hiding the child as undispatched.
|
|
|
|
|
$insert(8, [
|
|
|
|
|
'kind' => 'template_import',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$insert(9, [
|
|
|
|
|
'parent_job_id' => 8,
|
|
|
|
|
'kind' => 'template_import_replica',
|
|
|
|
|
'dispatch_attempts' => 2,
|
|
|
|
|
'finalization_error_code' => 'recovery_dispatch_ledger_conflict',
|
|
|
|
|
]);
|
|
|
|
|
$invalidDispatchLeaf = CloudJobService::recoveryPresentation(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(9),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$invalidDispatchParent = CloudJobService::recoveryPresentation(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(8),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(($invalidDispatchLeaf['code'] ?? null)
|
|
|
|
|
=== 'recovery_dispatch_ledger_conflict'
|
|
|
|
|
&& str_contains(
|
|
|
|
|
(string) ($invalidDispatchLeaf['message'] ?? ''),
|
|
|
|
|
'派发'
|
|
|
|
|
)
|
|
|
|
|
&& ($invalidDispatchParent['code'] ?? null)
|
|
|
|
|
=== 'recovery_dispatch_ledger_conflict'
|
|
|
|
|
&& ($invalidDispatchParent['affected_child_count'] ?? null) === 1
|
|
|
|
|
&& str_contains(
|
|
|
|
|
(string) ($invalidDispatchParent['message'] ?? ''),
|
|
|
|
|
'派发'
|
|
|
|
|
),
|
|
|
|
|
'An invalid dispatch ledger was hidden from the task center or aggregate parent.');
|
|
|
|
|
|
|
|
|
|
// A historical row may still carry an immutable succeeded receipt while its
|
|
|
|
|
// dispatch generation is corrupt. The public DTO must expose the concrete
|
|
|
|
|
// recovery incident without rewriting that terminal receipt.
|
|
|
|
|
$insert(10, [
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'dispatch_attempts' => 2,
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$invalidSucceeded = CloudJobService::display(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(10),
|
|
|
|
|
false
|
|
|
|
|
);
|
|
|
|
|
$assert(($invalidSucceeded['status'] ?? null) === 'succeeded'
|
|
|
|
|
&& ($invalidSucceeded['recovery']['pending'] ?? null) === true
|
|
|
|
|
&& str_contains(
|
|
|
|
|
(string) ($invalidSucceeded['recovery']['message'] ?? ''),
|
|
|
|
|
'派发'
|
|
|
|
|
),
|
|
|
|
|
'A succeeded task with an invalid dispatch ledger hid its recovery incident.');
|
|
|
|
|
|
|
|
|
|
// Repairing the counter to a legal value must not create a brief unlock
|
|
|
|
|
// window. The sticky repair incident remains visible until the coordinator
|
|
|
|
|
// validates the immutable terminal receipt and explicitly clears it.
|
|
|
|
|
$insert(11, [
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'dispatch_attempts' => 1,
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_dispatch_repair_receipt_invalid',
|
|
|
|
|
]);
|
|
|
|
|
$stickySucceeded = CloudJobService::display(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(11),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(($stickySucceeded['status'] ?? null) === 'succeeded'
|
|
|
|
|
&& ($stickySucceeded['recovery']['code'] ?? null)
|
|
|
|
|
=== 'recovery_dispatch_repair_receipt_invalid'
|
|
|
|
|
&& str_contains(
|
|
|
|
|
(string) ($stickySucceeded['recovery']['message'] ?? ''),
|
|
|
|
|
'完整性校验'
|
|
|
|
|
)
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(10)
|
|
|
|
|
->where('id', 11)->exists(),
|
|
|
|
|
'A repaired succeeded task unlocked before its sticky incident was validated.');
|
|
|
|
|
|
|
|
|
|
// A zero counter proves the Agent was never contacted, but a failed PHP
|
|
|
|
|
// business-ledger finalizer still owns the resource lock until local recovery
|
|
|
|
|
// completes.
|
|
|
|
|
$insert(12, [
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'not_applied',
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_ledger_conflict',
|
|
|
|
|
]);
|
|
|
|
|
$localFinalizer = CloudJobService::display(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(12),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(($localFinalizer['recovery']['code'] ?? null)
|
|
|
|
|
=== 'recovery_ledger_conflict'
|
|
|
|
|
&& str_contains(
|
|
|
|
|
(string) ($localFinalizer['recovery']['message'] ?? ''),
|
|
|
|
|
'面板账本'
|
|
|
|
|
)
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(10)
|
|
|
|
|
->where('id', 12)->exists(),
|
|
|
|
|
'An undispatched local finalizer failure was hidden or unlocked.');
|
|
|
|
|
|
|
|
|
|
$insert(13, [
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'dispatch_attempts' => null,
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$nullDispatch = CloudJobService::display(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(13),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(($nullDispatch['status'] ?? null) === 'succeeded'
|
|
|
|
|
&& ($nullDispatch['recovery']['code'] ?? null)
|
|
|
|
|
=== 'recovery_dispatch_ledger_conflict'
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(10)
|
|
|
|
|
->where('id', 13)->exists(),
|
|
|
|
|
'A NULL dispatch ledger was treated as undispatched and unlocked.');
|
|
|
|
|
|
|
|
|
|
// Aggregate parents must inherit a zero-dispatch child's unresolved local
|
|
|
|
|
// finalizer instead of appearing complete while the child remains fenced.
|
|
|
|
|
$insert(14, [
|
|
|
|
|
'kind' => 'template_import',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 1,
|
|
|
|
|
'effect_resolution' => 'not_applied',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$insert(15, [
|
|
|
|
|
'parent_job_id' => 14,
|
|
|
|
|
'kind' => 'template_import_replica',
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'not_applied',
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_ledger_conflict',
|
|
|
|
|
]);
|
|
|
|
|
$localAggregate = CloudJobService::display(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(14),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$afterRepairCases = CloudJobService::tasks(
|
|
|
|
|
['user_id' => 7, 'is_admin' => false],
|
|
|
|
|
'completed'
|
|
|
|
|
);
|
|
|
|
|
$assert(($localAggregate['recovery']['code'] ?? null)
|
|
|
|
|
=== 'recovery_ledger_conflict'
|
|
|
|
|
&& ($localAggregate['recovery']['affected_child_count'] ?? null) === 1
|
|
|
|
|
&& ($afterRepairCases['recovering_count'] ?? null) === 8,
|
|
|
|
|
'Aggregate/recovering queries omitted sticky, local-finalizer, or NULL-ledger work.');
|
|
|
|
|
|
|
|
|
|
// A resource-projection validator records its bounded diagnosis on the parent
|
|
|
|
|
// after every child is already resolved. The user must see that concrete
|
|
|
|
|
// Chinese reason, and administrators must receive the reviewed code instead
|
|
|
|
|
// of a misleading generic child-convergence status.
|
|
|
|
|
$insert(16, [
|
|
|
|
|
'kind' => 'template_import',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_instance_state_conflict',
|
|
|
|
|
]);
|
|
|
|
|
$insert(17, [
|
|
|
|
|
'parent_job_id' => 16,
|
|
|
|
|
'kind' => 'template_import_replica',
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$projectionUser = CloudJobService::display(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(16),
|
|
|
|
|
false
|
|
|
|
|
);
|
|
|
|
|
$projectionAdmin = CloudJobService::display(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(16),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(str_contains(
|
|
|
|
|
(string) ($projectionUser['recovery']['message'] ?? ''),
|
|
|
|
|
'实例状态与任务证据暂不一致'
|
|
|
|
|
)
|
|
|
|
|
&& !array_key_exists('code', $projectionUser['recovery'] ?? [])
|
|
|
|
|
&& ($projectionAdmin['recovery']['code'] ?? null)
|
|
|
|
|
=== 'recovery_instance_state_conflict'
|
|
|
|
|
&& ($projectionAdmin['recovery']['affected_child_count'] ?? null) === 0
|
|
|
|
|
&& ($projectionAdmin['recovery']['additional_error_count'] ?? null) === 0,
|
|
|
|
|
'Aggregate parent projection anomaly was hidden as child convergence.');
|
|
|
|
|
|
|
|
|
|
// Damaged terminal safety tuples are neither resolved jobs nor trustworthy
|
|
|
|
|
// Agent recovery candidates. They remain visibly fenced with one concrete
|
|
|
|
|
// reviewed code, including when the valid dispatch generation is zero.
|
|
|
|
|
$insert(18, [
|
|
|
|
|
'safe_to_resubmit' => 1,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$insert(19, [
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$insert(20, [
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$insert(21, [
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$insert(26, [
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'effect_may_have_applied' => 1,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
foreach ([18, 19, 20, 21, 26] as $stateConflictId) {
|
|
|
|
|
$stateConflict = CloudJobService::display(
|
|
|
|
|
\App\Models\CloudJob::findOrFail($stateConflictId),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(($stateConflict['recovery']['code'] ?? null)
|
|
|
|
|
=== 'recovery_instance_state_conflict'
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(10)
|
|
|
|
|
->where('id', $stateConflictId)->exists(),
|
|
|
|
|
'A damaged leaf safety tuple left recovery or unlocked its instance.');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$insert(22, [
|
|
|
|
|
'kind' => 'template_import',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'safe_to_resubmit' => 1,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$insert(23, [
|
|
|
|
|
'parent_job_id' => 22,
|
|
|
|
|
'kind' => 'template_import_replica',
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$aggregateStateConflict = CloudJobService::display(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(22),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(($aggregateStateConflict['recovery']['code'] ?? null)
|
|
|
|
|
=== 'recovery_aggregate_state_conflict',
|
|
|
|
|
'A damaged aggregate-parent safety tuple was hidden from administrators.');
|
|
|
|
|
|
|
|
|
|
$insert(24, [
|
|
|
|
|
'kind' => 'template_import',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$insert(25, [
|
|
|
|
|
'parent_job_id' => 24,
|
|
|
|
|
'kind' => 'template_import_replica',
|
|
|
|
|
'safe_to_resubmit' => 1,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$childStateConflict = CloudJobService::display(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(24),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(($childStateConflict['recovery']['code'] ?? null)
|
|
|
|
|
=== 'recovery_instance_state_conflict'
|
|
|
|
|
&& ($childStateConflict['recovery']['affected_child_count'] ?? null) === 1,
|
|
|
|
|
'An anomalous aggregate child did not keep its parent visibly pending.');
|
|
|
|
|
|
|
|
|
|
// Every code that a published aggregate projection validator can persist must
|
|
|
|
|
// remain selectable and visible. Missing codes here would leave the catalog
|
|
|
|
|
// locked by finalization_incident while both task center and administrators
|
|
|
|
|
// incorrectly report an ordinary completed task.
|
|
|
|
|
$beforePublishedIncidents = CloudJobService::tasks(
|
|
|
|
|
['user_id' => 7, 'is_admin' => false],
|
|
|
|
|
'completed'
|
|
|
|
|
)['recovering_count'];
|
|
|
|
|
foreach ([
|
|
|
|
|
27 => 'recovery_dispatch_ledger_conflict',
|
|
|
|
|
29 => 'recovery_payload_unavailable',
|
|
|
|
|
] as $parentId => $incidentCode) {
|
|
|
|
|
$insert($parentId, [
|
|
|
|
|
'kind' => 'template_import',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => $incidentCode,
|
|
|
|
|
]);
|
|
|
|
|
$insert($parentId + 1, [
|
|
|
|
|
'parent_job_id' => $parentId,
|
|
|
|
|
'kind' => 'template_import_replica',
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$publishedIncident = CloudJobService::display(
|
|
|
|
|
\App\Models\CloudJob::findOrFail($parentId),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(($publishedIncident['recovery']['code'] ?? null) === $incidentCode,
|
|
|
|
|
'A published aggregate projection incident disappeared: '
|
|
|
|
|
. $incidentCode);
|
|
|
|
|
}
|
|
|
|
|
$afterPublishedIncidents = CloudJobService::tasks(
|
|
|
|
|
['user_id' => 7, 'is_admin' => false],
|
|
|
|
|
'completed'
|
|
|
|
|
)['recovering_count'];
|
|
|
|
|
$assert($afterPublishedIncidents === $beforePublishedIncidents + 2,
|
|
|
|
|
'Published aggregate projection incidents left the recovering filter.');
|
|
|
|
|
|
|
|
|
|
// A child cannot have a final effect while its task is still nonterminal. The
|
|
|
|
|
// contradiction belongs to the PHP aggregate ledger and must be visible before
|
|
|
|
|
// a worker has had a chance to persist the parent incident.
|
|
|
|
|
$insert(31, [
|
|
|
|
|
'kind' => 'template_import',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$insert(32, [
|
|
|
|
|
'parent_job_id' => 31,
|
|
|
|
|
'kind' => 'template_import_replica',
|
|
|
|
|
'phase' => 'awaiting_receipt',
|
|
|
|
|
'outcome' => null,
|
|
|
|
|
'status' => 'running',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$nonterminalResolved = CloudJobService::display(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(31),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(($nonterminalResolved['recovery']['code'] ?? null)
|
|
|
|
|
=== 'recovery_aggregate_state_conflict'
|
|
|
|
|
&& ($nonterminalResolved['recovery']['affected_child_count'] ?? null) === 1,
|
|
|
|
|
'A nonterminal child with a published effect concealed its aggregate conflict.');
|
|
|
|
|
|
|
|
|
|
// Immutable request-identity damage is more specific than the generic terminal
|
|
|
|
|
// state conflict it also implies. Administrators need that concrete cause for
|
|
|
|
|
// both a direct leaf and a leaf summarized through its aggregate parent.
|
|
|
|
|
$insert(33, [
|
|
|
|
|
'request_idempotency_key' => 'bad-key',
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_operation_identity_conflict',
|
|
|
|
|
]);
|
|
|
|
|
$identityLeaf = CloudJobService::recoveryPresentation(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(33),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(($identityLeaf['code'] ?? null)
|
|
|
|
|
=== 'recovery_operation_identity_conflict'
|
|
|
|
|
&& str_contains(
|
|
|
|
|
(string) ($identityLeaf['message'] ?? ''),
|
|
|
|
|
'请求指纹'
|
|
|
|
|
),
|
|
|
|
|
'A damaged leaf request identity was generalized in the admin DTO.');
|
|
|
|
|
|
|
|
|
|
$insert(34, [
|
|
|
|
|
'kind' => 'template_import',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$insert(35, [
|
|
|
|
|
'parent_job_id' => 34,
|
|
|
|
|
'kind' => 'template_import_replica',
|
|
|
|
|
'request_fingerprint' => 'bad-fingerprint',
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_operation_identity_conflict',
|
|
|
|
|
]);
|
|
|
|
|
$identityAggregate = CloudJobService::recoveryPresentation(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(34),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(($identityAggregate['code'] ?? null)
|
|
|
|
|
=== 'recovery_operation_identity_conflict'
|
|
|
|
|
&& ($identityAggregate['affected_child_count'] ?? null) === 1,
|
|
|
|
|
'An aggregate child request-identity conflict was generalized.');
|
|
|
|
|
|
|
|
|
|
// Sticky repair and data-integrity incidents retain their origin even if a
|
|
|
|
|
// second malformed envelope field also satisfies the generic state predicate.
|
|
|
|
|
$insert(36, [
|
|
|
|
|
'receipt_deadline_at' => 0,
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' =>
|
|
|
|
|
'recovery_dispatch_repair_instance_state_conflict',
|
|
|
|
|
]);
|
|
|
|
|
$repairSpecific = CloudJobService::recoveryPresentation(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(36),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(($repairSpecific['code'] ?? null)
|
|
|
|
|
=== 'recovery_dispatch_repair_instance_state_conflict',
|
|
|
|
|
'A sticky dispatch-repair cause was generalized in the admin DTO.');
|
|
|
|
|
|
|
|
|
|
$insert(37, [
|
|
|
|
|
'receipt_deadline_at' => 0,
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_payload_unavailable',
|
|
|
|
|
]);
|
|
|
|
|
$payloadSpecific = CloudJobService::recoveryPresentation(
|
|
|
|
|
\App\Models\CloudJob::findOrFail(37),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(($payloadSpecific['code'] ?? null)
|
|
|
|
|
=== 'recovery_payload_unavailable',
|
|
|
|
|
'A concrete recovery-data incident was generalized in the admin DTO.');
|
|
|
|
|
|
|
|
|
|
echo "Cloud task recovery presentation checks passed.\n";
|