|
|
|
|
@ -66,9 +66,12 @@ namespace {
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudJob;
|
|
|
|
|
use App\Models\CloudRescueTemplateImport;
|
|
|
|
|
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
|
|
|
|
|
use App\Services\Cloud\CloudAgentSettingsService;
|
|
|
|
|
use App\Services\Cloud\CloudDispatchLedger;
|
|
|
|
|
use App\Services\Cloud\CloudJobService;
|
|
|
|
|
use App\Services\Cloud\CloudRescueTemplateImportService;
|
|
|
|
|
use App\Services\Cloud\CloudSecrets;
|
|
|
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
@ -180,6 +183,21 @@ namespace {
|
|
|
|
|
$table->integer('active_job_id')->nullable();
|
|
|
|
|
$table->string('state')->default('active');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_rescue_template_imports',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->string('operation_id')->primary();
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('template_id', 32);
|
|
|
|
|
$table->integer('job_id');
|
|
|
|
|
$table->string('import_type');
|
|
|
|
|
$table->string('name');
|
|
|
|
|
$table->integer('size_bytes')->default(0);
|
|
|
|
|
$table->string('sha256', 64)->default('');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_template_replicas',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
@ -777,5 +795,164 @@ namespace {
|
|
|
|
|
&& CloudAgentSettingsService::$requests === 0,
|
|
|
|
|
'Undispatched create recovery repeated refund/release or contacted Agent.');
|
|
|
|
|
|
|
|
|
|
// The dispatch CAS may already be generation one when an exact Agent
|
|
|
|
|
// validation response proves that its operation journal was never entered.
|
|
|
|
|
// If PHP's catalog finalizer then fails, preserve that proof without
|
|
|
|
|
// rewriting the immutable counter, recover only the local ledger, and
|
|
|
|
|
// allow a new same-name import generation afterward.
|
|
|
|
|
$transportOperationId = str_repeat('ba', 16);
|
|
|
|
|
$transportTemplateId = str_repeat('c', 32);
|
|
|
|
|
$transportName = 'transport-not-dispatched-rescue';
|
|
|
|
|
$transportBytes = 2 * 1024 * 1024;
|
|
|
|
|
$transportPayload = [
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'import_type' => 'upload',
|
|
|
|
|
'name' => $transportName,
|
|
|
|
|
'source' => '',
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
'source_size_bytes' => $transportBytes,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'rescue_template_id' => $transportTemplateId,
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/rescue-templates',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => $transportOperationId,
|
|
|
|
|
'id' => $transportTemplateId,
|
|
|
|
|
'name' => $transportName,
|
|
|
|
|
'source_url' => 'https://panel.example.test/staging/rescue.iso',
|
|
|
|
|
'source_size_bytes' => $transportBytes,
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
];
|
|
|
|
|
$transportEnvelope = CloudSecrets::encrypt(
|
|
|
|
|
$transportPayload,
|
|
|
|
|
'job-payload'
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->insert([
|
|
|
|
|
'id' => 701,
|
|
|
|
|
'operation_id' => $transportOperationId,
|
|
|
|
|
'request_idempotency_key' => $transportOperationId,
|
|
|
|
|
'request_fingerprint' => hash('sha256', $transportOperationId),
|
|
|
|
|
'parent_job_id' => null,
|
|
|
|
|
'order_id' => null,
|
|
|
|
|
'invoice_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'snapshot_id' => null,
|
|
|
|
|
'user_id' => 1,
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'kind' => 'rescue_template_import',
|
|
|
|
|
'execution_mode' => 'agent',
|
|
|
|
|
'phase' => 'dispatching',
|
|
|
|
|
'outcome' => null,
|
|
|
|
|
'status' => 'running',
|
|
|
|
|
'sealed_payload' => $transportEnvelope,
|
|
|
|
|
'attempts' => 1,
|
|
|
|
|
'dispatch_attempts' => 1,
|
|
|
|
|
'dispatch_deadline_at' => $now + 60,
|
|
|
|
|
'dispatch_started_at' => $now,
|
|
|
|
|
'agent_accepted_at' => null,
|
|
|
|
|
'receipt_deadline_at' => $now + 300,
|
|
|
|
|
'terminal_at' => null,
|
|
|
|
|
'effect_may_have_applied' => null,
|
|
|
|
|
'safe_to_resubmit' => null,
|
|
|
|
|
'effect_resolution' => 'unknown',
|
|
|
|
|
'effect_resolved_at' => null,
|
|
|
|
|
'effect_resolved_by' => null,
|
|
|
|
|
'effect_resolution_reason' => '',
|
|
|
|
|
'completion_event_id' => null,
|
|
|
|
|
'read_at' => null,
|
|
|
|
|
'dispatched_at' => $now,
|
|
|
|
|
'lease_token' => str_repeat('7', 32),
|
|
|
|
|
'lease_until' => $now + 180,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'error_code' => null,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'result_json' => null,
|
|
|
|
|
'created_at' => $now - 1,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
$forceTerminalFailure->invoke(
|
|
|
|
|
null,
|
|
|
|
|
701,
|
|
|
|
|
str_repeat('7', 32),
|
|
|
|
|
false,
|
|
|
|
|
true,
|
|
|
|
|
'local_finalization_failed',
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$transportIncident = CloudJob::findOrFail(701);
|
|
|
|
|
$assert((int) $transportIncident->dispatch_attempts === 1
|
|
|
|
|
&& CloudDispatchLedger::hasTransportNotDispatchedProof(
|
|
|
|
|
$transportIncident
|
|
|
|
|
)
|
|
|
|
|
&& (string) $transportIncident->sealed_payload === $transportEnvelope
|
|
|
|
|
&& (int) $transportIncident->available_at >= time() + 290
|
|
|
|
|
&& CloudAgentSettingsService::$requests === 0,
|
|
|
|
|
'A strict transport not-dispatched response lost its proof or immutable generation.');
|
|
|
|
|
|
|
|
|
|
Capsule::table('cloud_rescue_template_imports')->insert([
|
|
|
|
|
'operation_id' => $transportOperationId,
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'template_id' => $transportTemplateId,
|
|
|
|
|
'job_id' => 701,
|
|
|
|
|
'import_type' => 'upload',
|
|
|
|
|
'name' => $transportName,
|
|
|
|
|
'size_bytes' => 0,
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
'state' => 'importing',
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'created_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', 701)->update([
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$transportService = new CloudAutomaticOperationRecoveryService();
|
|
|
|
|
$assert($transportService->tick(),
|
|
|
|
|
'The strict transport not-dispatched finalizer was not claimed.');
|
|
|
|
|
$transportRecovered = CloudJob::findOrFail(701);
|
|
|
|
|
$assert((int) $transportRecovered->dispatch_attempts === 1
|
|
|
|
|
&& !(bool) $transportRecovered->effect_may_have_applied
|
|
|
|
|
&& (bool) $transportRecovered->safe_to_resubmit
|
|
|
|
|
&& (string) $transportRecovered->effect_resolution === 'not_applied'
|
|
|
|
|
&& (string) $transportRecovered->effect_resolution_reason
|
|
|
|
|
=== 'automatic_local_not_dispatched'
|
|
|
|
|
&& !(bool) $transportRecovered->finalization_incident
|
|
|
|
|
&& $transportRecovered->finalization_error_code === null
|
|
|
|
|
&& (string) CloudRescueTemplateImport::query()
|
|
|
|
|
->where('operation_id', $transportOperationId)
|
|
|
|
|
->value('state') === 'failed'
|
|
|
|
|
&& CloudAgentSettingsService::$requests === 0,
|
|
|
|
|
'Transport not-dispatched recovery did not converge the rescue ledger locally.');
|
|
|
|
|
|
|
|
|
|
$retryActor = [
|
|
|
|
|
'user_id' => 1,
|
|
|
|
|
'is_admin' => true,
|
|
|
|
|
'is_super_admin' => true,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
];
|
|
|
|
|
$transportRetry = CloudRescueTemplateImportService::start(
|
|
|
|
|
7,
|
|
|
|
|
'upload',
|
|
|
|
|
$transportName,
|
|
|
|
|
'https://panel.example.test/staging/rescue-retry.iso',
|
|
|
|
|
'',
|
|
|
|
|
'',
|
|
|
|
|
$transportBytes,
|
|
|
|
|
$retryActor
|
|
|
|
|
);
|
|
|
|
|
$assert((int) $transportRetry['task_id'] !== 701
|
|
|
|
|
&& CloudRescueTemplateImport::query()
|
|
|
|
|
->where('host_id', 7)
|
|
|
|
|
->where('name', $transportName)
|
|
|
|
|
->count() === 2
|
|
|
|
|
&& CloudAgentSettingsService::$requests === 0,
|
|
|
|
|
'A locally converged rescue import still blocked its same-name retry.');
|
|
|
|
|
|
|
|
|
|
echo "Cloud undispatched finalizer recovery tests passed.\n";
|
|
|
|
|
}
|
|
|
|
|
|