You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
674 lines
26 KiB
PHP
674 lines
26 KiB
PHP
<?php
|
|
|
|
ob_start();
|
|
require __DIR__ . '/CloudPortableRestoreJobDispatcherTest.php';
|
|
ob_end_clean();
|
|
|
|
use App\Models\CloudBackupRestore;
|
|
use App\Models\CloudBackupRestoreAttempt;
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudResourceReservation;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudOperationRecoveryIdentity;
|
|
use App\Services\Cloud\CloudPlacementService;
|
|
use App\Services\Cloud\CloudPortableRestoreFinalizer;
|
|
use App\Services\Cloud\CloudPublicNetworkTransitionGate;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
if (!class_exists(CloudPortableRestoreFinalizer::class)) {
|
|
throw new RuntimeException(
|
|
'CloudPortableRestoreFinalizer production seam is not implemented.'
|
|
);
|
|
}
|
|
|
|
// The target scheduler fixture intentionally has only the columns it needs.
|
|
// Extend it with the production instance projection written after a restore
|
|
// succeeds; the source row remains valid because every new field is nullable
|
|
// or has the same production default.
|
|
$schema->table('cloud_instances', static function (Blueprint $t): void {
|
|
$t->string('uuid')->nullable();
|
|
$t->integer('order_id')->nullable();
|
|
$t->integer('plan_id')->nullable();
|
|
$t->integer('offer_id')->nullable();
|
|
$t->integer('current_period_id')->nullable();
|
|
$t->string('service_status')->default('exempt');
|
|
$t->string('network_status')->default('active');
|
|
$t->text('sealed_credential')->nullable();
|
|
$t->integer('expires_at')->nullable();
|
|
$t->decimal('renewal_price', 12, 2)->nullable();
|
|
$t->integer('renewal_days')->default(0);
|
|
$t->integer('renewal_traffic_bytes')->default(0);
|
|
$t->integer('traffic_quota_bytes')->default(0);
|
|
$t->integer('traffic_download_bytes')->default(0);
|
|
$t->integer('traffic_upload_bytes')->default(0);
|
|
$t->integer('lifetime_download_bytes')->default(0);
|
|
$t->integer('lifetime_upload_bytes')->default(0);
|
|
$t->integer('traffic_synced_at')->default(0);
|
|
$t->integer('suspended_at')->nullable();
|
|
$t->integer('network_suspended_at')->nullable();
|
|
$t->integer('created_at')->default(0);
|
|
$t->integer('updated_at')->default(0);
|
|
});
|
|
|
|
$attempt = CloudBackupRestoreAttempt::whereNotNull('job_id')
|
|
->orderByDesc('id')->firstOrFail();
|
|
$job = CloudJob::findOrFail((int) $attempt->job_id);
|
|
$restore = CloudBackupRestore::findOrFail((int) $attempt->backup_restore_id);
|
|
$reservation = CloudResourceReservation::findOrFail(
|
|
(int) $attempt->reservation_id
|
|
);
|
|
$queuedPayload = CloudSecrets::decrypt(
|
|
(string) $job->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$prepareWire = new ReflectionMethod(
|
|
\App\Services\Cloud\CloudWorkerService::class,
|
|
'portableRestoreDispatchPayload'
|
|
);
|
|
[$stablePayload] = $prepareWire->invoke(null, $job, $queuedPayload);
|
|
$now = time();
|
|
Capsule::table('cloud_jobs')->where('id', (int) $job->id)->update([
|
|
'phase' => 'dispatching',
|
|
'status' => 'running',
|
|
'attempts' => 1,
|
|
'dispatch_attempts' => 1,
|
|
'dispatch_started_at' => $now,
|
|
'dispatched_at' => $now,
|
|
'receipt_deadline_at' => $now + 600,
|
|
'sealed_payload' => CloudSecrets::encrypt(
|
|
$stablePayload,
|
|
'job-payload'
|
|
),
|
|
'updated_at' => $now,
|
|
]);
|
|
$job = CloudJob::findOrFail((int) $job->id);
|
|
$body = $stablePayload['body'];
|
|
$result = [
|
|
'uuid' => $body['uuid'],
|
|
'name' => $body['name'],
|
|
'status' => 'running',
|
|
'vcpu' => $body['vcpu'],
|
|
'memory_mib' => $body['memory_mib'],
|
|
'disk_gib' => $body['disk_gib'],
|
|
'bandwidth_mbps' => $body['bandwidth_mbps'],
|
|
'public_network' => $body['public_network'],
|
|
'machine' => 'q35',
|
|
'firmware' => 'uefi',
|
|
'secure_boot' => false,
|
|
'disks' => [['device' => 'vda', 'format' => 'qcow2']],
|
|
'interfaces' => [],
|
|
'backup_key' => $body['backup_key'],
|
|
'sha256' => $body['source_sha256'],
|
|
];
|
|
$baselineInstances = CloudInstance::count();
|
|
$connection = $job->getConnection();
|
|
|
|
$assertHeld = static function () use ($assert, $reservation): void {
|
|
$locked = CloudResourceReservation::findOrFail((int) $reservation->id);
|
|
$claims = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $reservation->id)
|
|
->orderBy('address_family')->get();
|
|
$assert((string) $locked->state === 'held'
|
|
&& $locked->instance_id === null
|
|
&& $claims->count() === 2
|
|
&& $claims->every(static fn ($claim): bool =>
|
|
(string) $claim->state === 'reserved'
|
|
&& (int) $claim->active_claim === 1),
|
|
'Restore finalizer changed the held bundle before a valid conclusion.');
|
|
};
|
|
|
|
// Success creates the panel row only after the Agent effect is proven, then
|
|
// commits both address families and every authority row in the same outer
|
|
// transaction.
|
|
$connection->beginTransaction();
|
|
try {
|
|
CloudPortableRestoreFinalizer::finishJobLocked(
|
|
$job,
|
|
'succeeded',
|
|
$result,
|
|
$stablePayload
|
|
);
|
|
CloudPortableRestoreFinalizer::finishJobLocked(
|
|
$job,
|
|
'succeeded',
|
|
$result,
|
|
$stablePayload
|
|
);
|
|
$finishedAttempt = CloudBackupRestoreAttempt::findOrFail(
|
|
(int) $attempt->id
|
|
);
|
|
$finishedRestore = CloudBackupRestore::findOrFail((int) $restore->id);
|
|
$finishedReservation = CloudResourceReservation::findOrFail(
|
|
(int) $reservation->id
|
|
);
|
|
$instance = CloudInstance::where('uuid', (string) $body['uuid'])->first();
|
|
$claims = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $reservation->id)
|
|
->orderBy('address_family')->get();
|
|
$addresses = Capsule::table('cloud_public_ips')->whereIn(
|
|
'id',
|
|
$claims->pluck('public_ip_id')->map(
|
|
static fn ($value): int => (int) $value
|
|
)->all()
|
|
)->get();
|
|
$spec = $instance instanceof CloudInstance
|
|
? json_decode((string) $instance->spec_json, true) : null;
|
|
$assert($instance instanceof CloudInstance
|
|
&& CloudInstance::count() === $baselineInstances + 1
|
|
&& (int) $instance->host_id === (int) $attempt->target_host_id
|
|
&& (int) $instance->user_id === (int) $job->user_id
|
|
&& (int) $instance->reservation_id === (int) $reservation->id
|
|
&& (int) $instance->resource_pool_id
|
|
=== (int) $reservation->resource_pool_id
|
|
&& (string) $instance->status === 'active'
|
|
&& (string) $instance->service_status === 'exempt'
|
|
&& (string) $instance->network_status === 'active'
|
|
&& $instance->sealed_credential === null
|
|
&& is_array($spec)
|
|
&& ($spec['public_network'] ?? null) == $body['public_network']
|
|
&& ($spec['restored_from_backup'] ?? null) === $body['backup_key']
|
|
&& !array_key_exists('root_password', $spec)
|
|
&& !array_key_exists('guest_hostname', $spec),
|
|
'Successful restore did not create the canonical disk-owned instance projection.');
|
|
$assert((string) $finishedReservation->state === 'committed'
|
|
&& (int) $finishedReservation->instance_id === (int) $instance->id
|
|
&& $claims->count() === 2
|
|
&& $claims->every(static fn ($claim): bool =>
|
|
(string) $claim->state === 'committed'
|
|
&& (int) $claim->active_claim === 1)
|
|
&& $addresses->count() === 2
|
|
&& $addresses->every(static fn ($address): bool =>
|
|
(string) $address->state === 'assigned'
|
|
&& (int) $address->instance_id === (int) $instance->id),
|
|
'Successful restore did not atomically commit IPv4 and IPv6.');
|
|
$assert((string) $finishedAttempt->state === 'succeeded'
|
|
&& (string) $finishedAttempt->effect_resolution === 'applied'
|
|
&& $finishedAttempt->active_attempt === null
|
|
&& $finishedAttempt->effect_resolved_at !== null
|
|
&& $finishedAttempt->terminal_at !== null
|
|
&& (string) $finishedRestore->state === 'ready'
|
|
&& (int) $finishedRestore->target_instance_id === (int) $instance->id
|
|
&& (int) $finishedRestore->job_id === (int) $job->id
|
|
&& (int) $finishedRestore->reservation_id
|
|
=== (int) $reservation->id,
|
|
'Successful restore did not settle attempt and generation-one projections.');
|
|
|
|
// All three delete entry points share this gate and final settlement.
|
|
// Admission must recognize the attempt-owned source before any host
|
|
// mutation, then confirmed absence releases both families exactly once.
|
|
$instance->name = 'renamed-restored-vm';
|
|
$instance->save();
|
|
CloudPublicNetworkTransitionGate::assertInstanceDeleteReady($instance);
|
|
$instance->status = 'deleted';
|
|
$instance->save();
|
|
CloudPlacementService::releaseDeletedInstance(
|
|
$instance,
|
|
'instance_deleted'
|
|
);
|
|
$released = CloudResourceReservation::findOrFail(
|
|
(int) $reservation->id
|
|
);
|
|
$releasedClaims = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $reservation->id)
|
|
->orderBy('address_family')->get();
|
|
$releasedAddresses = Capsule::table('cloud_public_ips')->whereIn(
|
|
'id',
|
|
$releasedClaims->pluck('public_ip_id')->map(
|
|
static fn ($value): int => (int) $value
|
|
)->all()
|
|
)->get();
|
|
$releasedAt = (int) $released->released_at;
|
|
$assert((string) $released->state === 'released'
|
|
&& (int) $released->instance_id === (int) $instance->id
|
|
&& $releasedAt > 0
|
|
&& (string) $released->release_reason === 'instance_deleted'
|
|
&& $releasedClaims->count() === 2
|
|
&& $releasedClaims->every(static fn ($claim): bool =>
|
|
(string) $claim->state === 'released'
|
|
&& $claim->active_claim === null
|
|
&& $claim->released_at !== null)
|
|
&& $releasedAddresses->count() === 2
|
|
&& $releasedAddresses->every(static fn ($address): bool =>
|
|
(string) $address->state === 'available'
|
|
&& $address->reservation_id === null
|
|
&& $address->reservation_ip_claim_id === null
|
|
&& $address->instance_id === null
|
|
&& $address->operation_id === null),
|
|
'Confirmed restored-instance deletion did not release dual-stack ownership.');
|
|
$releasedAt = 123456789;
|
|
Capsule::table('cloud_resource_reservations')->where(
|
|
'id',
|
|
(int) $reservation->id
|
|
)->update([
|
|
'released_at' => $releasedAt,
|
|
]);
|
|
CloudPlacementService::releaseDeletedInstance(
|
|
$instance->fresh(),
|
|
'instance_deleted'
|
|
);
|
|
$assert((int) CloudResourceReservation::findOrFail(
|
|
(int) $reservation->id
|
|
)->released_at === $releasedAt
|
|
&& (string) CloudBackupRestoreAttempt::findOrFail(
|
|
(int) $attempt->id
|
|
)->state === 'succeeded'
|
|
&& (string) CloudBackupRestore::findOrFail(
|
|
(int) $restore->id
|
|
)->state === 'ready',
|
|
'Restored-instance deletion replay mutated terminal restore history.');
|
|
$assert((string) CloudResourceReservation::findOrFail(
|
|
(int) $reservation->id
|
|
)->release_reason === 'instance_deleted',
|
|
'Restored-instance deletion replay rewrote its release reason.');
|
|
} finally {
|
|
$connection->rollBack();
|
|
}
|
|
|
|
// An unknown or torn restore attempt must be rejected before a delete reaches
|
|
// the host, preserving the complete committed pair for reconciliation.
|
|
$connection->beginTransaction();
|
|
try {
|
|
CloudPortableRestoreFinalizer::finishJobLocked(
|
|
$job,
|
|
'succeeded',
|
|
$result,
|
|
$stablePayload
|
|
);
|
|
$guardedInstance = CloudInstance::where(
|
|
'uuid',
|
|
(string) $body['uuid']
|
|
)->firstOrFail();
|
|
Capsule::table('cloud_resource_reservations')->where(
|
|
'id',
|
|
(int) $reservation->id
|
|
)->update(['backup_restore_attempt_key' => null]);
|
|
$missingSourceRejected = false;
|
|
try {
|
|
CloudPublicNetworkTransitionGate::assertInstanceDeleteReady(
|
|
$guardedInstance
|
|
);
|
|
} catch (CloudError $error) {
|
|
$missingSourceRejected = $error->httpStatus === 409;
|
|
}
|
|
$assert($missingSourceRejected,
|
|
'A restore reservation without its attempt key reached deletion.');
|
|
Capsule::table('cloud_resource_reservations')->where(
|
|
'id',
|
|
(int) $reservation->id
|
|
)->update([
|
|
'backup_restore_attempt_key' => (string) $attempt->attempt_key,
|
|
]);
|
|
Capsule::table('cloud_backup_restore_attempts')->where(
|
|
'id',
|
|
(int) $attempt->id
|
|
)->update([
|
|
'state' => 'effect_unknown',
|
|
'effect_resolution' => 'unknown',
|
|
'active_attempt' => 1,
|
|
'effect_resolved_at' => null,
|
|
'terminal_at' => null,
|
|
]);
|
|
$deleteRejected = false;
|
|
try {
|
|
CloudPublicNetworkTransitionGate::assertInstanceDeleteReady(
|
|
$guardedInstance
|
|
);
|
|
} catch (CloudError $error) {
|
|
$deleteRejected = $error->httpStatus === 409;
|
|
}
|
|
$guardedReservation = CloudResourceReservation::findOrFail(
|
|
(int) $reservation->id
|
|
);
|
|
$guardedClaims = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $reservation->id)->get();
|
|
$assert($deleteRejected
|
|
&& (string) $guardedReservation->state === 'committed'
|
|
&& $guardedClaims->count() === 2
|
|
&& $guardedClaims->every(static fn ($claim): bool =>
|
|
(string) $claim->state === 'committed'
|
|
&& (int) $claim->active_claim === 1),
|
|
'Unknown restore attempt reached deletion or released its bundle.');
|
|
} finally {
|
|
$connection->rollBack();
|
|
}
|
|
|
|
// The restored domain must match the one fixed portable platform contract;
|
|
// UUID, disk and network equality cannot authorize a BIOS/i440fx variant.
|
|
$connection->beginTransaction();
|
|
try {
|
|
foreach ([
|
|
['machine', 'pc-i440fx'],
|
|
['firmware', 'bios'],
|
|
['secure_boot', true],
|
|
] as [$field, $wrong]) {
|
|
$incompatible = $result;
|
|
$incompatible[$field] = $wrong;
|
|
$rejected = false;
|
|
try {
|
|
CloudPortableRestoreFinalizer::finishJobLocked(
|
|
$job,
|
|
'succeeded',
|
|
$incompatible,
|
|
$stablePayload
|
|
);
|
|
} catch (CloudError $error) {
|
|
$rejected = $error->httpStatus === 502;
|
|
}
|
|
$assert($rejected,
|
|
'Restore accepted incompatible platform field: ' . $field);
|
|
$assertHeld();
|
|
}
|
|
} finally {
|
|
$connection->rollBack();
|
|
}
|
|
|
|
// A deterministic not-applied conclusion releases both claims and concrete
|
|
// addresses. Replaying the same conclusion is an exact no-op, not a second
|
|
// release or a new instance.
|
|
$connection->beginTransaction();
|
|
try {
|
|
CloudPortableRestoreFinalizer::finishJobLocked(
|
|
$job,
|
|
'failed',
|
|
[
|
|
'creation_absent' => true,
|
|
'managed_artifacts_absent' => true,
|
|
],
|
|
$stablePayload
|
|
);
|
|
CloudPortableRestoreFinalizer::finishJobLocked(
|
|
$job,
|
|
'failed',
|
|
[
|
|
'creation_absent' => true,
|
|
'managed_artifacts_absent' => true,
|
|
],
|
|
$stablePayload
|
|
);
|
|
$failedAttempt = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id);
|
|
$failedRestore = CloudBackupRestore::findOrFail((int) $restore->id);
|
|
$failedReservation = CloudResourceReservation::findOrFail(
|
|
(int) $reservation->id
|
|
);
|
|
$claims = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $reservation->id)
|
|
->orderBy('address_family')->get();
|
|
$addresses = Capsule::table('cloud_public_ips')->whereIn(
|
|
'id',
|
|
$claims->pluck('public_ip_id')->map(
|
|
static fn ($value): int => (int) $value
|
|
)->all()
|
|
)->get();
|
|
$assert(CloudInstance::count() === $baselineInstances
|
|
&& (string) $failedReservation->state === 'released'
|
|
&& $failedReservation->instance_id === null
|
|
&& $failedReservation->released_at !== null
|
|
&& $claims->count() === 2
|
|
&& $claims->every(static fn ($claim): bool =>
|
|
(string) $claim->state === 'released'
|
|
&& $claim->active_claim === null
|
|
&& $claim->released_at !== null)
|
|
&& $addresses->count() === 2
|
|
&& $addresses->every(static fn ($address): bool =>
|
|
(string) $address->state === 'available'
|
|
&& $address->reservation_id === null
|
|
&& $address->reservation_ip_claim_id === null
|
|
&& $address->instance_id === null
|
|
&& $address->operation_id === null),
|
|
'Confirmed-not-applied restore retained a resource or address claim.');
|
|
$assert((string) $failedAttempt->state === 'failed'
|
|
&& (string) $failedAttempt->effect_resolution === 'not_applied'
|
|
&& $failedAttempt->active_attempt === null
|
|
&& $failedAttempt->effect_resolved_at !== null
|
|
&& $failedAttempt->terminal_at !== null
|
|
&& (string) $failedRestore->state === 'failed'
|
|
&& $failedRestore->target_instance_id === null
|
|
&& (int) $failedRestore->job_id === (int) $job->id
|
|
&& (int) $failedRestore->reservation_id
|
|
=== (int) $reservation->id,
|
|
'Confirmed-not-applied restore did not expose an explicit retry boundary.');
|
|
} finally {
|
|
$connection->rollBack();
|
|
}
|
|
|
|
// A generic failure is not proof that the host and managed artifacts are
|
|
// absent. The deep finalizer itself must reject incomplete absence evidence.
|
|
$connection->beginTransaction();
|
|
try {
|
|
$absenceRejected = false;
|
|
try {
|
|
CloudPortableRestoreFinalizer::finishJobLocked(
|
|
$job,
|
|
'failed',
|
|
['creation_absent' => true],
|
|
$stablePayload
|
|
);
|
|
} catch (\App\Services\Cloud\CloudRecoveryEvidenceConflict $error) {
|
|
$absenceRejected = $error->httpStatus === 409;
|
|
}
|
|
$assert($absenceRejected, 'Incomplete absence evidence released a restore.');
|
|
$assertHeld();
|
|
} finally {
|
|
$connection->rollBack();
|
|
}
|
|
|
|
// Unknown effect never releases or reassigns anything. It fences both legs,
|
|
// keeps the generation active, and therefore cannot be retried automatically.
|
|
$connection->beginTransaction();
|
|
try {
|
|
CloudPortableRestoreFinalizer::finishJobLocked(
|
|
$job,
|
|
'uncertain',
|
|
null,
|
|
$stablePayload
|
|
);
|
|
CloudPortableRestoreFinalizer::finishJobLocked(
|
|
$job,
|
|
'uncertain',
|
|
null,
|
|
$stablePayload
|
|
);
|
|
$unknownAttempt = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id);
|
|
$unknownRestore = CloudBackupRestore::findOrFail((int) $restore->id);
|
|
$unknownReservation = CloudResourceReservation::findOrFail(
|
|
(int) $reservation->id
|
|
);
|
|
$claims = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $reservation->id)
|
|
->orderBy('address_family')->get();
|
|
$addresses = Capsule::table('cloud_public_ips')->whereIn(
|
|
'id',
|
|
$claims->pluck('public_ip_id')->map(
|
|
static fn ($value): int => (int) $value
|
|
)->all()
|
|
)->get();
|
|
$assert(CloudInstance::count() === $baselineInstances
|
|
&& (string) $unknownReservation->state === 'uncertain'
|
|
&& $unknownReservation->instance_id === null
|
|
&& $claims->count() === 2
|
|
&& $claims->every(static fn ($claim): bool =>
|
|
(string) $claim->state === 'uncertain'
|
|
&& (int) $claim->active_claim === 1)
|
|
&& $addresses->count() === 2
|
|
&& $addresses->every(static fn ($address): bool =>
|
|
(string) $address->state === 'uncertain'
|
|
&& (int) $address->reservation_id
|
|
=== (int) $reservation->id),
|
|
'Unknown restore effect released or reassigned a fenced resource.');
|
|
$assert((string) $unknownAttempt->state === 'effect_unknown'
|
|
&& (string) $unknownAttempt->effect_resolution === 'unknown'
|
|
&& (int) $unknownAttempt->active_attempt === 1
|
|
&& $unknownAttempt->effect_resolved_at === null
|
|
&& $unknownAttempt->terminal_at === null
|
|
&& (string) $unknownRestore->state === 'uncertain'
|
|
&& $unknownRestore->target_instance_id === null
|
|
&& (int) $unknownRestore->job_id === (int) $job->id
|
|
&& (int) $unknownRestore->reservation_id
|
|
=== (int) $reservation->id,
|
|
'Unknown restore effect did not retain its active recovery fence.');
|
|
} finally {
|
|
$connection->rollBack();
|
|
}
|
|
|
|
// Recovery prepare validates and persists exact evidence but must not expose
|
|
// an instance or commit either address family before the Agent fence commit.
|
|
$connection->beginTransaction();
|
|
try {
|
|
$preparedJob = CloudJob::findOrFail((int) $job->id);
|
|
CloudPortableRestoreFinalizer::finishJobLocked(
|
|
$preparedJob,
|
|
'succeeded',
|
|
$result,
|
|
$stablePayload,
|
|
true
|
|
);
|
|
$pendingAttempt = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id);
|
|
$assert((string) $pendingAttempt->state === 'finalization_pending'
|
|
&& (string) $pendingAttempt->effect_resolution === 'applied'
|
|
&& (string) $pendingAttempt->finalization_error_code
|
|
=== 'recovery_prepare_pending'
|
|
&& CloudInstance::count() === $baselineInstances,
|
|
'Prepared restore published the target before fence resolution.');
|
|
$assertHeld();
|
|
|
|
$compactMethod = new ReflectionMethod(
|
|
\App\Services\Cloud\CloudWorkerService::class,
|
|
'compactPreparedRecoveryPayload'
|
|
);
|
|
$compactPayload = $compactMethod->invoke(
|
|
null,
|
|
'backup_restore',
|
|
$stablePayload
|
|
);
|
|
$compactPayload['_operation_recovery'] = [
|
|
'version' => 1,
|
|
'kind' => 'backup_restore',
|
|
'prepared_marker' => 'prepared:test',
|
|
'operation' => ['status' => 'succeeded'],
|
|
];
|
|
Capsule::table('cloud_jobs')->where('id', (int) $job->id)->update([
|
|
'sealed_payload' => CloudSecrets::encrypt(
|
|
$compactPayload,
|
|
'job-payload'
|
|
),
|
|
]);
|
|
$preparedJob = CloudJob::findOrFail((int) $job->id);
|
|
$assert(CloudOperationRecoveryIdentity::same(
|
|
CloudOperationRecoveryIdentity::expected($preparedJob),
|
|
$compactPayload[CloudOperationRecoveryIdentity::payloadKey()]
|
|
),
|
|
'Compact prepared restore payload lost its recovery identity.');
|
|
$tamperedCompact = $compactPayload;
|
|
$tamperedCompact['generation'] = (int) $tamperedCompact['generation'] + 1;
|
|
$preparedJob->sealed_payload = CloudSecrets::encrypt(
|
|
$tamperedCompact,
|
|
'job-payload'
|
|
);
|
|
$preparedTamperRejected = false;
|
|
try {
|
|
CloudOperationRecoveryIdentity::expected($preparedJob);
|
|
} catch (CloudError $error) {
|
|
$preparedTamperRejected = $error->httpStatus === 409;
|
|
}
|
|
$assert($preparedTamperRejected,
|
|
'Prepared escrow bypassed portable restore recovery authority.');
|
|
$preparedJob->sealed_payload = CloudSecrets::encrypt(
|
|
$compactPayload,
|
|
'job-payload'
|
|
);
|
|
CloudPortableRestoreFinalizer::publishPreparedLocked(
|
|
$preparedJob,
|
|
'applied'
|
|
);
|
|
CloudPortableRestoreFinalizer::publishPreparedLocked(
|
|
$preparedJob,
|
|
'applied'
|
|
);
|
|
$published = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id);
|
|
$assert((string) $published->state === 'succeeded'
|
|
&& CloudInstance::count() === $baselineInstances + 1,
|
|
'Prepared restore did not publish exactly once after fence resolution.');
|
|
} finally {
|
|
$connection->rollBack();
|
|
}
|
|
|
|
// If exact evidence survived but the ordinary Worker transaction failed, the
|
|
// local retry consumes only that evidence and never dispatches the mutation.
|
|
$connection->beginTransaction();
|
|
try {
|
|
$retryJob = CloudJob::findOrFail((int) $job->id);
|
|
CloudPortableRestoreFinalizer::finishJobLocked(
|
|
$retryJob,
|
|
'succeeded',
|
|
$result,
|
|
$stablePayload,
|
|
true
|
|
);
|
|
Capsule::table('cloud_backup_restore_attempts')
|
|
->where('id', (int) $attempt->id)->update([
|
|
'last_error_code' => 'local_finalization_failed',
|
|
'finalization_error_code' => 'local_finalization_failed',
|
|
'updated_at' => time(),
|
|
]);
|
|
$leaseToken = str_repeat('a', 32);
|
|
Capsule::table('cloud_jobs')->where('id', (int) $job->id)->update([
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'status' => 'failed',
|
|
'terminal_at' => time(),
|
|
'effect_may_have_applied' => 1,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'unknown',
|
|
'error_code' => 'local_finalization_failed',
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'local_finalization_failed',
|
|
'lease_token' => $leaseToken,
|
|
'lease_until' => time() + 60,
|
|
'updated_at' => time(),
|
|
]);
|
|
CloudPortableRestoreFinalizer::retryPendingFinalization(
|
|
(int) $job->id,
|
|
$leaseToken
|
|
);
|
|
$retriedJob = CloudJob::findOrFail((int) $job->id);
|
|
$retriedAttempt = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id);
|
|
$assert((string) $retriedAttempt->state === 'succeeded'
|
|
&& (string) $retriedJob->effect_resolution === 'applied'
|
|
&& !(bool) $retriedJob->finalization_incident
|
|
&& $retriedJob->lease_token === null
|
|
&& CloudInstance::count() === $baselineInstances + 1,
|
|
'Portable restore PHP-only retry did not converge from durable evidence.');
|
|
} finally {
|
|
$connection->rollBack();
|
|
}
|
|
|
|
// Any frozen-identity tear must abort the whole outer transaction before an
|
|
// instance row or one-sided address mutation can escape.
|
|
$connection->beginTransaction();
|
|
try {
|
|
$torn = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id);
|
|
$torn->target_public_network_sha256 = str_repeat('0', 64);
|
|
$torn->save();
|
|
$rejected = false;
|
|
try {
|
|
CloudPortableRestoreFinalizer::finishJobLocked(
|
|
$job,
|
|
'succeeded',
|
|
$result,
|
|
$stablePayload
|
|
);
|
|
} catch (CloudError $error) {
|
|
$rejected = $error->httpStatus === 409;
|
|
}
|
|
$assert($rejected && CloudInstance::count() === $baselineInstances,
|
|
'Torn restore identity produced a partial instance.');
|
|
$assertHeld();
|
|
} finally {
|
|
$connection->rollBack();
|
|
}
|
|
|
|
echo "Portable restore finalizer behavior tests passed.\n";
|