You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudBackupPublicationPrepa...

776 lines
37 KiB
PHP

<?php
namespace App\Services\Cloud {
/** Test-only sealed envelope; capabilities may be inspected without keys. */
final class CloudSecrets
{
/** @var array<string,array<string,mixed>> */
private static array $values = [];
public static function encrypt(array $value, string $purpose): string
{
$id = bin2hex(random_bytes(16));
self::$values[$id] = $value;
return 'backup-preparation-test:' . $id;
}
public static function decrypt(string $value, string $purpose): array
{
$id = substr($value, strlen('backup-preparation-test:'));
if (!isset(self::$values[$id])) {
throw new \RuntimeException('Missing test envelope.');
}
return self::$values[$id];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudBackupSubscription;
use App\Models\CloudInstance;
use App\Models\CloudInstanceBackup;
use App\Models\CloudJob;
use App\Models\CloudStorageBackend;
use App\Models\CloudStorageOperation;
use App\Services\Cloud\CloudBackupPublicationPreparation;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudInput;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudObjectStorageS3Client;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudStorageArtifactService;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expect = static function (callable $callback, int $status) use (
$assert
): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
'Unexpected CloudError status: ' . $error->httpStatus);
return;
}
throw new RuntimeException('Expected CloudError was not raised.');
};
$db = new Capsule();
$db->addConnection([
'driver' => 'sqlite',
'database' => ':memory:',
], 'default');
$db->setAsGlobal();
$db->bootEloquent();
$schema = $db->schema('default');
$schema->create('cloud_storage_backends', static function (Blueprint $t): void {
$t->increments('id');
$t->string('backend_key'); $t->string('identity_sha256');
$t->string('name'); $t->string('driver'); $t->string('endpoint');
$t->string('region'); $t->string('bucket'); $t->string('prefix');
$t->boolean('path_style'); $t->string('failure_domain');
$t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision');
$t->unsignedBigInteger('tested_revision'); $t->string('admin_state');
$t->string('health_state'); $t->unsignedInteger('write_priority');
$t->unsignedInteger('consecutive_failures')->default(0);
$t->unsignedInteger('consecutive_successes')->default(0);
$t->string('last_error_code')->default('');
$t->unsignedBigInteger('last_tested_at')->default(0);
$t->unsignedBigInteger('last_healthy_at')->default(0);
$t->unsignedBigInteger('next_probe_at')->default(0);
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_artifacts', static function (Blueprint $t): void {
$t->increments('id'); $t->string('artifact_key')->unique();
$t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes');
$t->unsignedBigInteger('size_bytes'); $t->string('sha256');
$t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies');
$t->unsignedTinyInteger('required_copies'); $t->string('state');
$t->string('last_operation_id')->nullable();
$t->unsignedBigInteger('verified_at');
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_replicas', static function (Blueprint $t): void {
$t->increments('id'); $t->string('replica_key')->unique();
$t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id');
$t->string('role'); $t->string('object_key');
$t->string('object_version_id')->default('');
$t->string('object_etag')->default('');
$t->unsignedBigInteger('size_bytes')->default(0);
$t->string('sha256')->default(''); $t->string('state');
$t->string('publish_operation_id')->nullable();
$t->string('delete_operation_id')->nullable();
$t->unsignedInteger('delete_generation')->default(0);
$t->unsignedInteger('delete_cycle_attempts')->default(0);
$t->unsignedInteger('delete_presence_confirmations')->default(0);
$t->unsignedBigInteger('delete_last_checked_at')->default(0);
$t->unsignedBigInteger('delete_next_check_at')->default(0);
$t->unsignedBigInteger('delete_attention_at')->nullable();
$t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until')->default(0);
$t->string('error_code')->default('');
$t->unsignedBigInteger('verified_at')->default(0);
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_operations', static function (Blueprint $t): void {
$t->increments('id'); $t->string('operation_id')->unique();
$t->string('request_idempotency_key')->unique();
$t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id');
$t->unsignedBigInteger('replica_id')->nullable();
$t->unsignedBigInteger('source_backend_id')->nullable();
$t->unsignedBigInteger('target_backend_id')->nullable();
$t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable();
$t->string('effect_resolution');
$t->boolean('effect_may_have_applied')->nullable();
$t->boolean('safe_to_resubmit')->nullable();
$t->unsignedInteger('attempts'); $t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until'); $t->unsignedBigInteger('available_at');
$t->unsignedBigInteger('next_check_at'); $t->string('error_code');
$t->text('result_json')->nullable(); $t->unsignedBigInteger('terminal_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_publications', static function (Blueprint $t): void {
$t->increments('id'); $t->string('publication_key')->unique();
$t->unsignedBigInteger('artifact_id')->unique();
$t->unsignedBigInteger('backup_id')->nullable()->unique();
$t->string('template_import_intent_id')->nullable()->unique();
$t->string('rescue_template_id')->nullable()->unique();
$t->string('state'); $t->unsignedInteger('attempts')->default(0);
$t->unsignedBigInteger('available_at')->default(0);
$t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until')->default(0);
$t->string('last_error_code')->default('');
$t->unsignedBigInteger('coverage_ready_at')->nullable();
$t->unsignedBigInteger('finalized_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_instance_backups', static function (Blueprint $t): void {
$t->increments('id'); $t->string('backup_key')->unique();
$t->unsignedBigInteger('storage_artifact_id')->nullable()->unique();
$t->unsignedBigInteger('create_request_actor_id')->nullable();
$t->string('create_request_idempotency_key')->nullable();
$t->string('create_request_fingerprint')->nullable();
$t->unsignedBigInteger('source_instance_id');
$t->unsignedBigInteger('source_host_id'); $t->unsignedBigInteger('user_id');
$t->string('source_uuid'); $t->string('name'); $t->string('purpose');
$t->string('source'); $t->unsignedBigInteger('subscription_id')->nullable();
$t->unsignedBigInteger('scheduled_for')->nullable();
$t->string('state'); $t->string('consistency'); $t->string('object_key');
$t->text('source_portable_restore_contract_json');
$t->string('source_portable_restore_contract_sha256');
$t->text('spec_json'); $t->text('sealed_storage_runtime');
$t->text('sealed_credential')->nullable(); $t->unsignedBigInteger('storage_revision');
$t->unsignedBigInteger('active_job_id')->nullable();
$t->string('last_operation_id')->nullable();
$t->string('error_code')->default('');
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
$t->unique(['create_request_actor_id', 'create_request_idempotency_key']);
});
$schema->create('cloud_instances', static function (Blueprint $t): void {
$t->increments('id'); $t->unsignedBigInteger('user_id');
$t->unsignedBigInteger('host_id'); $t->string('uuid');
$t->string('status'); $t->string('service_status');
$t->unsignedBigInteger('expires_at')->nullable();
$t->text('spec_json');
});
$schema->create('cloud_instance_migrations', static function (Blueprint $t): void {
$t->increments('id'); $t->unsignedBigInteger('instance_id');
$t->string('state');
});
$schema->create('cloud_peer_migration_artifacts',
static function (Blueprint $t): void {
$t->increments('id');
});
$schema->create('cloud_jobs', static function (Blueprint $t): void {
$t->increments('id'); $t->string('operation_id')->unique();
$t->string('request_idempotency_key'); $t->string('request_fingerprint');
$t->unsignedBigInteger('instance_id')->nullable();
$t->unsignedBigInteger('backup_id')->nullable();
$t->unsignedBigInteger('user_id'); $t->unsignedBigInteger('host_id')->nullable();
$t->string('kind'); $t->string('execution_mode'); $t->string('phase');
$t->string('outcome')->nullable(); $t->string('status');
$t->string('error_code')->default('');
$t->text('sealed_payload'); $t->integer('attempts')->default(0);
$t->integer('dispatch_attempts')->default(0);
$t->unsignedBigInteger('dispatch_deadline_at');
$t->unsignedBigInteger('dispatch_started_at')->nullable();
$t->unsignedBigInteger('agent_accepted_at')->nullable();
$t->unsignedBigInteger('dispatched_at')->nullable();
$t->unsignedBigInteger('receipt_deadline_at')->nullable();
$t->unsignedBigInteger('terminal_at')->nullable();
$t->boolean('effect_may_have_applied')->nullable();
$t->boolean('safe_to_resubmit')->nullable();
$t->string('effect_resolution');
$t->boolean('finalization_incident')->default(false);
$t->string('finalization_error_code')->nullable();
$t->text('result_json')->nullable();
$t->string('completion_event_id')->nullable(); $t->unsignedBigInteger('read_at')->nullable();
$t->unsignedBigInteger('available_at');
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
$t->unique(['user_id', 'request_idempotency_key']);
});
$schema->create('cloud_backup_subscriptions', static function (Blueprint $t): void {
$t->increments('id'); $t->unsignedBigInteger('instance_id')->unique();
$t->unsignedBigInteger('user_id'); $t->unsignedBigInteger('offer_id');
$t->boolean('enabled'); $t->boolean('renew_next_period');
$t->string('frequency'); $t->unsignedTinyInteger('weekday');
$t->unsignedTinyInteger('retention_count'); $t->string('timezone');
$t->string('price'); $t->string('currency'); $t->unsignedInteger('price_version');
$t->unsignedInteger('state_revision')->default(1);
$t->unsignedBigInteger('next_run_at');
$t->unsignedBigInteger('last_scheduled_at')->nullable();
$t->unsignedBigInteger('last_backup_id')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_backup_period_entitlements',
static function (Blueprint $t): void {
$t->increments('id');
$t->unsignedBigInteger('subscription_id');
$t->unsignedBigInteger('instance_id');
$t->unsignedBigInteger('user_id');
$t->unsignedBigInteger('starts_at');
$t->unsignedBigInteger('ends_at');
});
$now = time();
$backend = new CloudStorageBackend();
foreach ([
'id' => 1, 'backend_key' => str_repeat('1', 32),
'identity_sha256' => str_repeat('1', 64), 'name' => 'primary',
'driver' => 's3', 'endpoint' => 'https://objects.example.test',
'region' => 'us-east-1', 'bucket' => 'virt-backups', 'prefix' => 'primary',
'path_style' => true, 'failure_domain' => 'a',
'sealed_credentials' => 'sealed', 'config_revision' => 1,
'tested_revision' => 1, 'admin_state' => 'active',
'health_state' => 'healthy', 'write_priority' => 10,
'consecutive_failures' => 0, 'consecutive_successes' => 1,
'last_error_code' => '', 'last_tested_at' => $now,
'last_healthy_at' => $now, 'next_probe_at' => $now + 300,
'created_at' => $now, 'updated_at' => $now,
] as $field => $value) {
$backend->{$field} = $value;
}
$backend->save();
$runtime = [
'config' => [
'endpoint' => 'https://objects.example.test',
'region' => 'us-east-1', 'bucket' => 'virt-backups',
'prefix' => 'primary', 'path_style' => true,
],
'credentials' => [
'access_key_id' => 'TESTACCESSKEY',
'secret_access_key' => 'test-secret-never-persist',
],
'revision' => 1,
];
$runtimeResolver = static fn (int $backendId, bool $healthy): array => $runtime;
$clock = $now;
$clockFn = static function () use (&$clock): int { return $clock; };
$newBackup = static function (
int $id,
string $source,
?int $subscriptionId = null,
?int $scheduledFor = null
) use ($now): array {
$requestKey = 'backup-prepare-request-' . str_pad((string) $id, 4, '0', STR_PAD_LEFT);
$uuid = sprintf('00000000-0000-4000-8000-%012d', $id);
$canonical = [
'instance_id' => 100 + $id,
'name' => 'backup-' . $id,
'consistency_mode' => 'filesystem',
];
if ($source === 'automatic') {
$canonical['source'] = 'automatic';
$canonical['subscription_id'] = $subscriptionId;
$canonical['scheduled_for'] = $scheduledFor;
}
$target = '/v1/vms/' . rawurlencode($uuid) . '/backups';
$fingerprint = CloudJobService::standaloneRequestFingerprint(
7, 'backup_create', 'POST', $target, $canonical
);
$instance = CloudInstance::find(100 + $id);
if (!$instance instanceof CloudInstance) {
$instance = new CloudInstance();
foreach ([
'id' => 100 + $id,
'user_id' => 33,
'host_id' => 7,
'uuid' => $uuid,
'status' => 'active',
'service_status' => 'active',
'expires_at' => $now + 86400,
'spec_json' => json_encode([
'managed' => true,
'os_source' => 'cloud_template',
], JSON_THROW_ON_ERROR),
] as $field => $value) {
$instance->{$field} = $value;
}
$instance->save();
}
$backup = new CloudInstanceBackup();
foreach ([
'id' => $id, 'backup_key' => str_pad(dechex($id), 32, 'a'),
'storage_artifact_id' => null, 'create_request_actor_id' => 91,
'create_request_idempotency_key' => $requestKey,
'create_request_fingerprint' => $fingerprint,
'source_instance_id' => 100 + $id, 'source_host_id' => 7,
'user_id' => 33, 'source_uuid' => $uuid, 'name' => 'backup-' . $id,
'purpose' => 'portable', 'source' => $source,
'subscription_id' => $subscriptionId, 'scheduled_for' => $scheduledFor,
'state' => 'creating', 'consistency' => 'filesystem',
'object_key' => 'pending-' . $id,
'source_portable_restore_contract_json' => '{}',
'source_portable_restore_contract_sha256' => str_repeat('c', 64),
'spec_json' => '{}', 'sealed_storage_runtime' => '',
'sealed_credential' => 'sealed-guest', 'storage_revision' => 0,
'active_job_id' => null, 'last_operation_id' => null,
'error_code' => '', 'created_at' => $now, 'updated_at' => $now,
] as $field => $value) {
$backup->{$field} = $value;
}
$backup->save();
$publication = CloudStorageArtifactService::beginAgentPublication(
'backup', 'vm-backups/' . $backup->backup_key . '/system.qcow2',
128 * 1024 * 1024,
'backup-publication-test-' . $id,
hash('sha256', 'backup-publication-test-' . $id),
['backup_id' => $id]
);
$backup->storage_artifact_id = (int) $publication['artifact_id'];
$backup->object_key = (string) $publication['object_key'];
$backup->save();
return [$backup, $publication];
};
// Known CreateMultipart result: one durable job is bound atomically and
// repeated ensure calls never contact S3 or create a replacement job.
[$manual, $manualPublication] = $newBackup(1, 'manual');
$manualHistory = [];
$manualStack = HandlerStack::create(new MockHandler([
static function () use ($assert): Response {
$assert(Capsule::connection('default')->transactionLevel() === 0,
'CreateMultipartUpload was called inside a database transaction.');
return new Response(200, ['Content-Type' => 'application/xml'],
'<InitiateMultipartUploadResult><UploadId>manual-upload</UploadId>'
. '</InitiateMultipartUploadResult>');
},
]));
$manualStack->push(Middleware::history($manualHistory));
$manualService = new CloudBackupPublicationPreparation(
new CloudObjectStorageS3Client(new Client(['handler' => $manualStack])),
$runtimeResolver,
$clockFn
);
$manualJob = $manualService->ensureAgentJob((int) $manual->id);
$assert($manualJob instanceof CloudJob,
'Known multipart result did not create an Agent job.');
$manual->refresh();
$assert((int) $manual->active_job_id === (int) $manualJob->id
&& (string) $manual->last_operation_id === (string) $manualJob->operation_id
&& (int) $manualJob->backup_id === (int) $manual->id
&& (int) $manualJob->instance_id === (int) $manual->source_instance_id,
'Agent job and backup were not bound in both directions.');
$sameJob = $manualService->ensureAgentJob((int) $manual->id);
$assert($sameJob instanceof CloudJob
&& (int) $sameJob->id === (int) $manualJob->id
&& CloudJob::count() === 1 && count($manualHistory) === 1,
'Repeated preparation created another job or multipart upload.');
$manualOperation = CloudStorageOperation::where(
'operation_id', (string) $manualPublication['operation_id']
)->firstOrFail();
$stored = (string) $manualOperation->result_json;
$assert(str_contains($stored, 'manual-upload')
&& !str_contains($stored, 'http')
&& !str_contains($stored, 'TESTACCESSKEY')
&& !str_contains($stored, 'test-secret-never-persist'),
'Storage operation persisted a capability URL or credential.');
$payload = CloudSecrets::decrypt(
(string) $manualJob->sealed_payload,
'job-payload'
);
$assert(isset($payload['body']['object_upload']['part_urls'][0]),
'Short-lived upload capabilities were not confined to the sealed job.');
$terminalPayload = json_encode(
CloudInput::safeResult($payload),
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
$assert(!str_contains($terminalPayload, 'https://')
&& !str_contains($terminalPayload, 'part_urls')
&& !str_contains($terminalPayload, 'complete_url')
&& !str_contains($terminalPayload, 'abort_url')
&& !str_contains($terminalPayload, 'verify_url'),
'Resolved terminal payload retained an upload capability URL.');
// A 5xx CreateMultipart response is unknown, not a safe retry. Exact-key
// reconciliation adopts the one original session and creates one job.
[$unknown, $unknownPublication] = $newBackup(2, 'manual');
$unknownService = new CloudBackupPublicationPreparation(
new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(new MockHandler([
new Response(503),
])),
])),
$runtimeResolver,
$clockFn
);
$assert($unknownService->ensureAgentJob((int) $unknown->id) === null,
'Unknown multipart result incorrectly created an Agent job.');
$unknownOperation = CloudStorageOperation::where(
'operation_id', (string) $unknownPublication['operation_id']
)->firstOrFail();
$assert((string) $unknownOperation->phase === 'reconciling'
&& (string) $unknownOperation->effect_resolution === 'unknown'
&& CloudJob::where('backup_id', (int) $unknown->id)->count() === 0,
'Unknown CreateMultipart effect was not retained on its original operation.');
$clock += 60;
$initiated = gmdate('Y-m-d\TH:i:s\Z', $now);
$key = (string) $unknownPublication['object_key'];
$lookupXml = '<?xml version="1.0"?><ListMultipartUploadsResult>'
. '<IsTruncated>false</IsTruncated><Upload><Key>'
. htmlspecialchars($key, ENT_XML1)
. '</Key><UploadId>recovered-upload</UploadId><Initiated>'
. $initiated . '</Initiated></Upload></ListMultipartUploadsResult>';
$recoveryHistory = [];
$recoveryStack = HandlerStack::create(new MockHandler([
new Response(200, ['Content-Type' => 'application/xml'], $lookupXml),
]));
$recoveryStack->push(Middleware::history($recoveryHistory));
$recoveryService = new CloudBackupPublicationPreparation(
new CloudObjectStorageS3Client(new Client(['handler' => $recoveryStack])),
$runtimeResolver,
$clockFn
);
$recoveredJob = $recoveryService->ensureAgentJob((int) $unknown->id);
$assert($recoveredJob instanceof CloudJob
&& CloudJob::where('backup_id', (int) $unknown->id)->count() === 1
&& count($recoveryHistory) === 1,
'Exact-key reconciliation did not adopt one original multipart session.');
$unknownOperation->refresh();
$assert(str_contains((string) $unknownOperation->result_json,
'recovered-upload')
&& (int) $unknownOperation->attempts === 1,
'Recovered upload identity was replaced or the provider effect was replayed.');
$assert((int) $recoveryService->ensureAgentJob((int) $unknown->id)->id
=== (int) $recoveredJob->id
&& CloudJob::where('backup_id', (int) $unknown->id)->count() === 1,
'A second worker/call created a duplicate recovered job.');
// Automatic intent without a pre-existing job uses the same preparation
// path and advances its subscription exactly when the job binding commits.
$scheduledFor = $now - 60;
$subscription = new CloudBackupSubscription();
foreach ([
'id' => 5, 'instance_id' => 103, 'user_id' => 33, 'offer_id' => 8,
'enabled' => true, 'renew_next_period' => true, 'frequency' => 'daily',
'weekday' => 0, 'retention_count' => 2, 'timezone' => 'UTC',
'price' => '2.00', 'currency' => 'USD', 'price_version' => 1,
'state_revision' => 1, 'next_run_at' => $scheduledFor,
'last_scheduled_at' => null, 'last_backup_id' => null,
'created_at' => $now, 'updated_at' => $now,
] as $field => $value) {
$subscription->{$field} = $value;
}
$subscription->save();
Capsule::table('cloud_backup_period_entitlements')->insert([
'subscription_id' => (int) $subscription->id,
'instance_id' => (int) $subscription->instance_id,
'user_id' => (int) $subscription->user_id,
'starts_at' => $now - 3600,
'ends_at' => $now + 3600,
]);
[$automatic] = $newBackup(3, 'automatic', 5, $scheduledFor);
$automaticService = new CloudBackupPublicationPreparation(
new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(new MockHandler([
new Response(200, ['Content-Type' => 'application/xml'],
'<InitiateMultipartUploadResult>'
. '<UploadId>automatic-upload</UploadId>'
. '</InitiateMultipartUploadResult>'),
])),
])),
$runtimeResolver,
$clockFn
);
$automaticJob = $automaticService->ensureAgentJob((int) $automatic->id);
$subscription->refresh();
$assert($automaticJob instanceof CloudJob
&& (int) $subscription->last_scheduled_at === $scheduledFor
&& (int) $subscription->last_backup_id === (int) $automatic->id
&& (int) $subscription->next_run_at > $scheduledFor,
'Automatic backup intent did not atomically bind its job and advance schedule.');
$advancedNextRun = (int) $subscription->next_run_at;
$automaticService->ensureAgentJob((int) $automatic->id);
$subscription->refresh();
$assert((int) $subscription->next_run_at === $advancedNextRun
&& CloudJob::where('backup_id', (int) $automatic->id)->count() === 1,
'Repeated automatic preparation advanced the schedule or duplicated the job.');
// A migration may start after the durable storage intent commits but
// before its Agent job is bound. The post-S3 instance fence must reject
// the bind without replacing the original multipart identity. Once the
// blocker is removed, the same upload is signed and bound without another
// CreateMultipartUpload request.
[$blocked, $blockedPublication] = $newBackup(4, 'manual');
$blockedHistory = [];
$blockedStack = HandlerStack::create(new MockHandler([
static function () use ($assert): Response {
$assert(Capsule::connection('default')->transactionLevel() === 0,
'Multipart preparation crossed the business transaction.');
Capsule::table('cloud_instance_migrations')->insert([
'instance_id' => 104,
'state' => 'preparing_source',
]);
return new Response(200, ['Content-Type' => 'application/xml'],
'<InitiateMultipartUploadResult>'
. '<UploadId>blocked-upload</UploadId>'
. '</InitiateMultipartUploadResult>');
},
]));
$blockedStack->push(Middleware::history($blockedHistory));
$blockedService = new CloudBackupPublicationPreparation(
new CloudObjectStorageS3Client(new Client([
'handler' => $blockedStack,
])),
$runtimeResolver,
$clockFn
);
$expect(static fn () => $blockedService->ensureAgentJob(
(int) $blocked->id
), 409);
$blockedOperation = CloudStorageOperation::where(
'operation_id',
(string) $blockedPublication['operation_id']
)->firstOrFail();
$assert((string) $blockedOperation->phase === 'awaiting_receipt'
&& str_contains(
(string) $blockedOperation->result_json,
'blocked-upload'
)
&& CloudJob::where('backup_id', (int) $blocked->id)->count() === 0,
'A concurrent migration did not fence the pre-Agent backup bind.');
Capsule::table('cloud_instance_migrations')
->where('instance_id', 104)->delete();
$clock += 301;
$resumedService = new CloudBackupPublicationPreparation(
new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(new MockHandler([])),
])),
$runtimeResolver,
$clockFn
);
$resumedJob = $resumedService->ensureAgentJob((int) $blocked->id);
$blockedOperation->refresh();
$assert($resumedJob instanceof CloudJob
&& count($blockedHistory) === 1
&& str_contains(
(string) $blockedOperation->result_json,
'blocked-upload'
),
'Clearing the blocker replayed or replaced the durable upload.');
// The same late fence applies to an ordinary in-flight instance job, not
// only to the migration ledger.
[$jobBlocked, $jobBlockedPublication] = $newBackup(7, 'manual');
$jobBlockedService = new CloudBackupPublicationPreparation(
new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(new MockHandler([
static function () use ($clock): Response {
Capsule::table('cloud_jobs')->insert([
'operation_id' => str_repeat('d', 32),
'request_idempotency_key' =>
'concurrent-instance-job-0007',
'request_fingerprint' => str_repeat('e', 64),
'instance_id' => 107, 'backup_id' => null,
'user_id' => 33, 'host_id' => 7,
'kind' => 'action', 'execution_mode' => 'agent',
'phase' => 'accepted', 'outcome' => null,
'status' => 'queued', 'error_code' => '',
'sealed_payload' => 'sealed', 'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_deadline_at' => $clock + 600,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'dispatched_at' => null,
'receipt_deadline_at' => null,
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'finalization_incident' => 0,
'finalization_error_code' => null,
'result_json' => null,
'completion_event_id' => null,
'read_at' => null, 'available_at' => $clock,
'created_at' => $clock, 'updated_at' => $clock,
]);
return new Response(200, [
'Content-Type' => 'application/xml',
], '<InitiateMultipartUploadResult>'
. '<UploadId>job-blocked-upload</UploadId>'
. '</InitiateMultipartUploadResult>');
},
])),
])),
$runtimeResolver,
$clockFn
);
$expect(static fn () => $jobBlockedService->ensureAgentJob(
(int) $jobBlocked->id
), 409);
$jobBlockedOperation = CloudStorageOperation::where(
'operation_id',
(string) $jobBlockedPublication['operation_id']
)->firstOrFail();
$assert(CloudJob::where(
'backup_id',
(int) $jobBlocked->id
)->count() === 0
&& str_contains(
(string) $jobBlockedOperation->result_json,
'job-blocked-upload'
),
'A concurrent instance job did not fence the backup bind.');
CloudJob::where('operation_id', str_repeat('d', 32))->delete();
// The user changes the schedule while CreateMultipartUpload is in
// progress. The old slot may still bind, but it must not overwrite the
// newly selected frequency or next execution time.
$staleSlot = $clock - 60;
$newNextRun = $clock + 7 * 86400;
$changedSubscription = new CloudBackupSubscription();
foreach ([
'id' => 6, 'instance_id' => 105, 'user_id' => 33, 'offer_id' => 8,
'enabled' => true, 'renew_next_period' => true,
'frequency' => 'daily', 'weekday' => 0, 'retention_count' => 2,
'timezone' => 'UTC', 'price' => '2.00', 'currency' => 'USD',
'price_version' => 1, 'state_revision' => 1,
'next_run_at' => $staleSlot, 'last_scheduled_at' => null,
'last_backup_id' => null, 'created_at' => $now,
'updated_at' => $now,
] as $field => $value) {
$changedSubscription->{$field} = $value;
}
$changedSubscription->save();
Capsule::table('cloud_backup_period_entitlements')->insert([
'subscription_id' => 6, 'instance_id' => 105, 'user_id' => 33,
'starts_at' => $now - 3600, 'ends_at' => $clock + 3600,
]);
[$changedBackup] = $newBackup(5, 'automatic', 6, $staleSlot);
$changedService = new CloudBackupPublicationPreparation(
new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(new MockHandler([
static function () use ($newNextRun): Response {
Capsule::table('cloud_backup_subscriptions')
->where('id', 6)->update([
'frequency' => 'weekly',
'weekday' => 3,
'state_revision' => 2,
'next_run_at' => $newNextRun,
]);
return new Response(200, [
'Content-Type' => 'application/xml',
], '<InitiateMultipartUploadResult>'
. '<UploadId>changed-plan-upload</UploadId>'
. '</InitiateMultipartUploadResult>');
},
])),
])),
$runtimeResolver,
$clockFn
);
$changedJob = $changedService->ensureAgentJob((int) $changedBackup->id);
$changedSubscription->refresh();
$assert($changedJob instanceof CloudJob
&& (string) $changedSubscription->frequency === 'weekly'
&& (int) $changedSubscription->weekday === 3
&& (int) $changedSubscription->next_run_at === $newNextRun
&& (int) $changedSubscription->last_scheduled_at === $staleSlot
&& (int) $changedSubscription->last_backup_id
=== (int) $changedBackup->id,
'An old backup intent overwrote the user\'s newer schedule.');
// Disabling the subscription during provider preparation is also a real
// fence. The remote multipart identity is retained for reconciliation,
// but no Agent job may escape after the paid entitlement is withdrawn.
$disabledSlot = $clock - 30;
$disabledSubscription = new CloudBackupSubscription();
foreach ([
'id' => 7, 'instance_id' => 106, 'user_id' => 33, 'offer_id' => 8,
'enabled' => true, 'renew_next_period' => true,
'frequency' => 'daily', 'weekday' => 0, 'retention_count' => 2,
'timezone' => 'UTC', 'price' => '2.00', 'currency' => 'USD',
'price_version' => 1, 'state_revision' => 1,
'next_run_at' => $disabledSlot, 'last_scheduled_at' => null,
'last_backup_id' => null, 'created_at' => $now,
'updated_at' => $now,
] as $field => $value) {
$disabledSubscription->{$field} = $value;
}
$disabledSubscription->save();
Capsule::table('cloud_backup_period_entitlements')->insert([
'subscription_id' => 7, 'instance_id' => 106, 'user_id' => 33,
'starts_at' => $now - 3600, 'ends_at' => $clock + 3600,
]);
[$disabledBackup, $disabledPublication] = $newBackup(
6,
'automatic',
7,
$disabledSlot
);
$disabledService = new CloudBackupPublicationPreparation(
new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(new MockHandler([
static function (): Response {
Capsule::table('cloud_backup_subscriptions')
->where('id', 7)->update([
'enabled' => 0,
'state_revision' => 2,
]);
return new Response(200, [
'Content-Type' => 'application/xml',
], '<InitiateMultipartUploadResult>'
. '<UploadId>disabled-upload</UploadId>'
. '</InitiateMultipartUploadResult>');
},
])),
])),
$runtimeResolver,
$clockFn
);
$expect(static fn () => $disabledService->ensureAgentJob(
(int) $disabledBackup->id
), 409);
$disabledOperation = CloudStorageOperation::where(
'operation_id',
(string) $disabledPublication['operation_id']
)->firstOrFail();
$assert(CloudJob::where(
'backup_id',
(int) $disabledBackup->id
)->count() === 0
&& (string) $disabledOperation->phase === 'awaiting_receipt'
&& str_contains(
(string) $disabledOperation->result_json,
'disabled-upload'
),
'A disabled subscription still escaped as an Agent backup job.');
echo "Cloud backup publication preparation behavior tests passed.\n";
}