You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
776 lines
37 KiB
PHP
776 lines
37 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud {
|
|
/** Test-only sealed envelope; capabilities may be inspected without keys. */
|
|
final class CloudSecrets
|
|
{
|
|
/** @var array<string,array<string,mixed>> */
|
|
private static array $values = [];
|
|
|
|
public static function encrypt(array $value, string $purpose): string
|
|
{
|
|
$id = bin2hex(random_bytes(16));
|
|
self::$values[$id] = $value;
|
|
return 'backup-preparation-test:' . $id;
|
|
}
|
|
|
|
public static function decrypt(string $value, string $purpose): array
|
|
{
|
|
$id = substr($value, strlen('backup-preparation-test:'));
|
|
if (!isset(self::$values[$id])) {
|
|
throw new \RuntimeException('Missing test envelope.');
|
|
}
|
|
return self::$values[$id];
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudBackupSubscription;
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInstanceBackup;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudStorageBackend;
|
|
use App\Models\CloudStorageOperation;
|
|
use App\Services\Cloud\CloudBackupPublicationPreparation;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudInput;
|
|
use App\Services\Cloud\CloudJobService;
|
|
use App\Services\Cloud\CloudObjectStorageS3Client;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use App\Services\Cloud\CloudStorageArtifactService;
|
|
use GuzzleHttp\Client;
|
|
use GuzzleHttp\Handler\MockHandler;
|
|
use GuzzleHttp\HandlerStack;
|
|
use GuzzleHttp\Middleware;
|
|
use GuzzleHttp\Psr7\Response;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$expect = static function (callable $callback, int $status) use (
|
|
$assert
|
|
): void {
|
|
try {
|
|
$callback();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === $status,
|
|
'Unexpected CloudError status: ' . $error->httpStatus);
|
|
return;
|
|
}
|
|
throw new RuntimeException('Expected CloudError was not raised.');
|
|
};
|
|
|
|
$db = new Capsule();
|
|
$db->addConnection([
|
|
'driver' => 'sqlite',
|
|
'database' => ':memory:',
|
|
], 'default');
|
|
$db->setAsGlobal();
|
|
$db->bootEloquent();
|
|
$schema = $db->schema('default');
|
|
|
|
$schema->create('cloud_storage_backends', static function (Blueprint $t): void {
|
|
$t->increments('id');
|
|
$t->string('backend_key'); $t->string('identity_sha256');
|
|
$t->string('name'); $t->string('driver'); $t->string('endpoint');
|
|
$t->string('region'); $t->string('bucket'); $t->string('prefix');
|
|
$t->boolean('path_style'); $t->string('failure_domain');
|
|
$t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision');
|
|
$t->unsignedBigInteger('tested_revision'); $t->string('admin_state');
|
|
$t->string('health_state'); $t->unsignedInteger('write_priority');
|
|
$t->unsignedInteger('consecutive_failures')->default(0);
|
|
$t->unsignedInteger('consecutive_successes')->default(0);
|
|
$t->string('last_error_code')->default('');
|
|
$t->unsignedBigInteger('last_tested_at')->default(0);
|
|
$t->unsignedBigInteger('last_healthy_at')->default(0);
|
|
$t->unsignedBigInteger('next_probe_at')->default(0);
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
});
|
|
$schema->create('cloud_storage_artifacts', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('artifact_key')->unique();
|
|
$t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes');
|
|
$t->unsignedBigInteger('size_bytes'); $t->string('sha256');
|
|
$t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies');
|
|
$t->unsignedTinyInteger('required_copies'); $t->string('state');
|
|
$t->string('last_operation_id')->nullable();
|
|
$t->unsignedBigInteger('verified_at');
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
});
|
|
$schema->create('cloud_storage_replicas', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('replica_key')->unique();
|
|
$t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id');
|
|
$t->string('role'); $t->string('object_key');
|
|
$t->string('object_version_id')->default('');
|
|
$t->string('object_etag')->default('');
|
|
$t->unsignedBigInteger('size_bytes')->default(0);
|
|
$t->string('sha256')->default(''); $t->string('state');
|
|
$t->string('publish_operation_id')->nullable();
|
|
$t->string('delete_operation_id')->nullable();
|
|
$t->unsignedInteger('delete_generation')->default(0);
|
|
$t->unsignedInteger('delete_cycle_attempts')->default(0);
|
|
$t->unsignedInteger('delete_presence_confirmations')->default(0);
|
|
$t->unsignedBigInteger('delete_last_checked_at')->default(0);
|
|
$t->unsignedBigInteger('delete_next_check_at')->default(0);
|
|
$t->unsignedBigInteger('delete_attention_at')->nullable();
|
|
$t->string('lease_token')->nullable();
|
|
$t->unsignedBigInteger('lease_until')->default(0);
|
|
$t->string('error_code')->default('');
|
|
$t->unsignedBigInteger('verified_at')->default(0);
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
});
|
|
$schema->create('cloud_storage_operations', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('operation_id')->unique();
|
|
$t->string('request_idempotency_key')->unique();
|
|
$t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id');
|
|
$t->unsignedBigInteger('replica_id')->nullable();
|
|
$t->unsignedBigInteger('source_backend_id')->nullable();
|
|
$t->unsignedBigInteger('target_backend_id')->nullable();
|
|
$t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable();
|
|
$t->string('effect_resolution');
|
|
$t->boolean('effect_may_have_applied')->nullable();
|
|
$t->boolean('safe_to_resubmit')->nullable();
|
|
$t->unsignedInteger('attempts'); $t->string('lease_token')->nullable();
|
|
$t->unsignedBigInteger('lease_until'); $t->unsignedBigInteger('available_at');
|
|
$t->unsignedBigInteger('next_check_at'); $t->string('error_code');
|
|
$t->text('result_json')->nullable(); $t->unsignedBigInteger('terminal_at')->nullable();
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
});
|
|
$schema->create('cloud_storage_publications', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('publication_key')->unique();
|
|
$t->unsignedBigInteger('artifact_id')->unique();
|
|
$t->unsignedBigInteger('backup_id')->nullable()->unique();
|
|
$t->string('template_import_intent_id')->nullable()->unique();
|
|
$t->string('rescue_template_id')->nullable()->unique();
|
|
$t->string('state'); $t->unsignedInteger('attempts')->default(0);
|
|
$t->unsignedBigInteger('available_at')->default(0);
|
|
$t->string('lease_token')->nullable();
|
|
$t->unsignedBigInteger('lease_until')->default(0);
|
|
$t->string('last_error_code')->default('');
|
|
$t->unsignedBigInteger('coverage_ready_at')->nullable();
|
|
$t->unsignedBigInteger('finalized_at')->nullable();
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
});
|
|
$schema->create('cloud_instance_backups', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('backup_key')->unique();
|
|
$t->unsignedBigInteger('storage_artifact_id')->nullable()->unique();
|
|
$t->unsignedBigInteger('create_request_actor_id')->nullable();
|
|
$t->string('create_request_idempotency_key')->nullable();
|
|
$t->string('create_request_fingerprint')->nullable();
|
|
$t->unsignedBigInteger('source_instance_id');
|
|
$t->unsignedBigInteger('source_host_id'); $t->unsignedBigInteger('user_id');
|
|
$t->string('source_uuid'); $t->string('name'); $t->string('purpose');
|
|
$t->string('source'); $t->unsignedBigInteger('subscription_id')->nullable();
|
|
$t->unsignedBigInteger('scheduled_for')->nullable();
|
|
$t->string('state'); $t->string('consistency'); $t->string('object_key');
|
|
$t->text('source_portable_restore_contract_json');
|
|
$t->string('source_portable_restore_contract_sha256');
|
|
$t->text('spec_json'); $t->text('sealed_storage_runtime');
|
|
$t->text('sealed_credential')->nullable(); $t->unsignedBigInteger('storage_revision');
|
|
$t->unsignedBigInteger('active_job_id')->nullable();
|
|
$t->string('last_operation_id')->nullable();
|
|
$t->string('error_code')->default('');
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
$t->unique(['create_request_actor_id', 'create_request_idempotency_key']);
|
|
});
|
|
$schema->create('cloud_instances', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->unsignedBigInteger('user_id');
|
|
$t->unsignedBigInteger('host_id'); $t->string('uuid');
|
|
$t->string('status'); $t->string('service_status');
|
|
$t->unsignedBigInteger('expires_at')->nullable();
|
|
$t->text('spec_json');
|
|
});
|
|
$schema->create('cloud_instance_migrations', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->unsignedBigInteger('instance_id');
|
|
$t->string('state');
|
|
});
|
|
$schema->create('cloud_peer_migration_artifacts',
|
|
static function (Blueprint $t): void {
|
|
$t->increments('id');
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('operation_id')->unique();
|
|
$t->string('request_idempotency_key'); $t->string('request_fingerprint');
|
|
$t->unsignedBigInteger('instance_id')->nullable();
|
|
$t->unsignedBigInteger('backup_id')->nullable();
|
|
$t->unsignedBigInteger('user_id'); $t->unsignedBigInteger('host_id')->nullable();
|
|
$t->string('kind'); $t->string('execution_mode'); $t->string('phase');
|
|
$t->string('outcome')->nullable(); $t->string('status');
|
|
$t->string('error_code')->default('');
|
|
$t->text('sealed_payload'); $t->integer('attempts')->default(0);
|
|
$t->integer('dispatch_attempts')->default(0);
|
|
$t->unsignedBigInteger('dispatch_deadline_at');
|
|
$t->unsignedBigInteger('dispatch_started_at')->nullable();
|
|
$t->unsignedBigInteger('agent_accepted_at')->nullable();
|
|
$t->unsignedBigInteger('dispatched_at')->nullable();
|
|
$t->unsignedBigInteger('receipt_deadline_at')->nullable();
|
|
$t->unsignedBigInteger('terminal_at')->nullable();
|
|
$t->boolean('effect_may_have_applied')->nullable();
|
|
$t->boolean('safe_to_resubmit')->nullable();
|
|
$t->string('effect_resolution');
|
|
$t->boolean('finalization_incident')->default(false);
|
|
$t->string('finalization_error_code')->nullable();
|
|
$t->text('result_json')->nullable();
|
|
$t->string('completion_event_id')->nullable(); $t->unsignedBigInteger('read_at')->nullable();
|
|
$t->unsignedBigInteger('available_at');
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
$t->unique(['user_id', 'request_idempotency_key']);
|
|
});
|
|
$schema->create('cloud_backup_subscriptions', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->unsignedBigInteger('instance_id')->unique();
|
|
$t->unsignedBigInteger('user_id'); $t->unsignedBigInteger('offer_id');
|
|
$t->boolean('enabled'); $t->boolean('renew_next_period');
|
|
$t->string('frequency'); $t->unsignedTinyInteger('weekday');
|
|
$t->unsignedTinyInteger('retention_count'); $t->string('timezone');
|
|
$t->string('price'); $t->string('currency'); $t->unsignedInteger('price_version');
|
|
$t->unsignedInteger('state_revision')->default(1);
|
|
$t->unsignedBigInteger('next_run_at');
|
|
$t->unsignedBigInteger('last_scheduled_at')->nullable();
|
|
$t->unsignedBigInteger('last_backup_id')->nullable();
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
});
|
|
$schema->create('cloud_backup_period_entitlements',
|
|
static function (Blueprint $t): void {
|
|
$t->increments('id');
|
|
$t->unsignedBigInteger('subscription_id');
|
|
$t->unsignedBigInteger('instance_id');
|
|
$t->unsignedBigInteger('user_id');
|
|
$t->unsignedBigInteger('starts_at');
|
|
$t->unsignedBigInteger('ends_at');
|
|
});
|
|
|
|
$now = time();
|
|
$backend = new CloudStorageBackend();
|
|
foreach ([
|
|
'id' => 1, 'backend_key' => str_repeat('1', 32),
|
|
'identity_sha256' => str_repeat('1', 64), 'name' => 'primary',
|
|
'driver' => 's3', 'endpoint' => 'https://objects.example.test',
|
|
'region' => 'us-east-1', 'bucket' => 'virt-backups', 'prefix' => 'primary',
|
|
'path_style' => true, 'failure_domain' => 'a',
|
|
'sealed_credentials' => 'sealed', 'config_revision' => 1,
|
|
'tested_revision' => 1, 'admin_state' => 'active',
|
|
'health_state' => 'healthy', 'write_priority' => 10,
|
|
'consecutive_failures' => 0, 'consecutive_successes' => 1,
|
|
'last_error_code' => '', 'last_tested_at' => $now,
|
|
'last_healthy_at' => $now, 'next_probe_at' => $now + 300,
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
] as $field => $value) {
|
|
$backend->{$field} = $value;
|
|
}
|
|
$backend->save();
|
|
|
|
$runtime = [
|
|
'config' => [
|
|
'endpoint' => 'https://objects.example.test',
|
|
'region' => 'us-east-1', 'bucket' => 'virt-backups',
|
|
'prefix' => 'primary', 'path_style' => true,
|
|
],
|
|
'credentials' => [
|
|
'access_key_id' => 'TESTACCESSKEY',
|
|
'secret_access_key' => 'test-secret-never-persist',
|
|
],
|
|
'revision' => 1,
|
|
];
|
|
$runtimeResolver = static fn (int $backendId, bool $healthy): array => $runtime;
|
|
$clock = $now;
|
|
$clockFn = static function () use (&$clock): int { return $clock; };
|
|
|
|
$newBackup = static function (
|
|
int $id,
|
|
string $source,
|
|
?int $subscriptionId = null,
|
|
?int $scheduledFor = null
|
|
) use ($now): array {
|
|
$requestKey = 'backup-prepare-request-' . str_pad((string) $id, 4, '0', STR_PAD_LEFT);
|
|
$uuid = sprintf('00000000-0000-4000-8000-%012d', $id);
|
|
$canonical = [
|
|
'instance_id' => 100 + $id,
|
|
'name' => 'backup-' . $id,
|
|
'consistency_mode' => 'filesystem',
|
|
];
|
|
if ($source === 'automatic') {
|
|
$canonical['source'] = 'automatic';
|
|
$canonical['subscription_id'] = $subscriptionId;
|
|
$canonical['scheduled_for'] = $scheduledFor;
|
|
}
|
|
$target = '/v1/vms/' . rawurlencode($uuid) . '/backups';
|
|
$fingerprint = CloudJobService::standaloneRequestFingerprint(
|
|
7, 'backup_create', 'POST', $target, $canonical
|
|
);
|
|
$instance = CloudInstance::find(100 + $id);
|
|
if (!$instance instanceof CloudInstance) {
|
|
$instance = new CloudInstance();
|
|
foreach ([
|
|
'id' => 100 + $id,
|
|
'user_id' => 33,
|
|
'host_id' => 7,
|
|
'uuid' => $uuid,
|
|
'status' => 'active',
|
|
'service_status' => 'active',
|
|
'expires_at' => $now + 86400,
|
|
'spec_json' => json_encode([
|
|
'managed' => true,
|
|
'os_source' => 'cloud_template',
|
|
], JSON_THROW_ON_ERROR),
|
|
] as $field => $value) {
|
|
$instance->{$field} = $value;
|
|
}
|
|
$instance->save();
|
|
}
|
|
$backup = new CloudInstanceBackup();
|
|
foreach ([
|
|
'id' => $id, 'backup_key' => str_pad(dechex($id), 32, 'a'),
|
|
'storage_artifact_id' => null, 'create_request_actor_id' => 91,
|
|
'create_request_idempotency_key' => $requestKey,
|
|
'create_request_fingerprint' => $fingerprint,
|
|
'source_instance_id' => 100 + $id, 'source_host_id' => 7,
|
|
'user_id' => 33, 'source_uuid' => $uuid, 'name' => 'backup-' . $id,
|
|
'purpose' => 'portable', 'source' => $source,
|
|
'subscription_id' => $subscriptionId, 'scheduled_for' => $scheduledFor,
|
|
'state' => 'creating', 'consistency' => 'filesystem',
|
|
'object_key' => 'pending-' . $id,
|
|
'source_portable_restore_contract_json' => '{}',
|
|
'source_portable_restore_contract_sha256' => str_repeat('c', 64),
|
|
'spec_json' => '{}', 'sealed_storage_runtime' => '',
|
|
'sealed_credential' => 'sealed-guest', 'storage_revision' => 0,
|
|
'active_job_id' => null, 'last_operation_id' => null,
|
|
'error_code' => '', 'created_at' => $now, 'updated_at' => $now,
|
|
] as $field => $value) {
|
|
$backup->{$field} = $value;
|
|
}
|
|
$backup->save();
|
|
$publication = CloudStorageArtifactService::beginAgentPublication(
|
|
'backup', 'vm-backups/' . $backup->backup_key . '/system.qcow2',
|
|
128 * 1024 * 1024,
|
|
'backup-publication-test-' . $id,
|
|
hash('sha256', 'backup-publication-test-' . $id),
|
|
['backup_id' => $id]
|
|
);
|
|
$backup->storage_artifact_id = (int) $publication['artifact_id'];
|
|
$backup->object_key = (string) $publication['object_key'];
|
|
$backup->save();
|
|
return [$backup, $publication];
|
|
};
|
|
|
|
// Known CreateMultipart result: one durable job is bound atomically and
|
|
// repeated ensure calls never contact S3 or create a replacement job.
|
|
[$manual, $manualPublication] = $newBackup(1, 'manual');
|
|
$manualHistory = [];
|
|
$manualStack = HandlerStack::create(new MockHandler([
|
|
static function () use ($assert): Response {
|
|
$assert(Capsule::connection('default')->transactionLevel() === 0,
|
|
'CreateMultipartUpload was called inside a database transaction.');
|
|
return new Response(200, ['Content-Type' => 'application/xml'],
|
|
'<InitiateMultipartUploadResult><UploadId>manual-upload</UploadId>'
|
|
. '</InitiateMultipartUploadResult>');
|
|
},
|
|
]));
|
|
$manualStack->push(Middleware::history($manualHistory));
|
|
$manualService = new CloudBackupPublicationPreparation(
|
|
new CloudObjectStorageS3Client(new Client(['handler' => $manualStack])),
|
|
$runtimeResolver,
|
|
$clockFn
|
|
);
|
|
$manualJob = $manualService->ensureAgentJob((int) $manual->id);
|
|
$assert($manualJob instanceof CloudJob,
|
|
'Known multipart result did not create an Agent job.');
|
|
$manual->refresh();
|
|
$assert((int) $manual->active_job_id === (int) $manualJob->id
|
|
&& (string) $manual->last_operation_id === (string) $manualJob->operation_id
|
|
&& (int) $manualJob->backup_id === (int) $manual->id
|
|
&& (int) $manualJob->instance_id === (int) $manual->source_instance_id,
|
|
'Agent job and backup were not bound in both directions.');
|
|
$sameJob = $manualService->ensureAgentJob((int) $manual->id);
|
|
$assert($sameJob instanceof CloudJob
|
|
&& (int) $sameJob->id === (int) $manualJob->id
|
|
&& CloudJob::count() === 1 && count($manualHistory) === 1,
|
|
'Repeated preparation created another job or multipart upload.');
|
|
$manualOperation = CloudStorageOperation::where(
|
|
'operation_id', (string) $manualPublication['operation_id']
|
|
)->firstOrFail();
|
|
$stored = (string) $manualOperation->result_json;
|
|
$assert(str_contains($stored, 'manual-upload')
|
|
&& !str_contains($stored, 'http')
|
|
&& !str_contains($stored, 'TESTACCESSKEY')
|
|
&& !str_contains($stored, 'test-secret-never-persist'),
|
|
'Storage operation persisted a capability URL or credential.');
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $manualJob->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$assert(isset($payload['body']['object_upload']['part_urls'][0]),
|
|
'Short-lived upload capabilities were not confined to the sealed job.');
|
|
$terminalPayload = json_encode(
|
|
CloudInput::safeResult($payload),
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
);
|
|
$assert(!str_contains($terminalPayload, 'https://')
|
|
&& !str_contains($terminalPayload, 'part_urls')
|
|
&& !str_contains($terminalPayload, 'complete_url')
|
|
&& !str_contains($terminalPayload, 'abort_url')
|
|
&& !str_contains($terminalPayload, 'verify_url'),
|
|
'Resolved terminal payload retained an upload capability URL.');
|
|
|
|
// A 5xx CreateMultipart response is unknown, not a safe retry. Exact-key
|
|
// reconciliation adopts the one original session and creates one job.
|
|
[$unknown, $unknownPublication] = $newBackup(2, 'manual');
|
|
$unknownService = new CloudBackupPublicationPreparation(
|
|
new CloudObjectStorageS3Client(new Client([
|
|
'handler' => HandlerStack::create(new MockHandler([
|
|
new Response(503),
|
|
])),
|
|
])),
|
|
$runtimeResolver,
|
|
$clockFn
|
|
);
|
|
$assert($unknownService->ensureAgentJob((int) $unknown->id) === null,
|
|
'Unknown multipart result incorrectly created an Agent job.');
|
|
$unknownOperation = CloudStorageOperation::where(
|
|
'operation_id', (string) $unknownPublication['operation_id']
|
|
)->firstOrFail();
|
|
$assert((string) $unknownOperation->phase === 'reconciling'
|
|
&& (string) $unknownOperation->effect_resolution === 'unknown'
|
|
&& CloudJob::where('backup_id', (int) $unknown->id)->count() === 0,
|
|
'Unknown CreateMultipart effect was not retained on its original operation.');
|
|
$clock += 60;
|
|
$initiated = gmdate('Y-m-d\TH:i:s\Z', $now);
|
|
$key = (string) $unknownPublication['object_key'];
|
|
$lookupXml = '<?xml version="1.0"?><ListMultipartUploadsResult>'
|
|
. '<IsTruncated>false</IsTruncated><Upload><Key>'
|
|
. htmlspecialchars($key, ENT_XML1)
|
|
. '</Key><UploadId>recovered-upload</UploadId><Initiated>'
|
|
. $initiated . '</Initiated></Upload></ListMultipartUploadsResult>';
|
|
$recoveryHistory = [];
|
|
$recoveryStack = HandlerStack::create(new MockHandler([
|
|
new Response(200, ['Content-Type' => 'application/xml'], $lookupXml),
|
|
]));
|
|
$recoveryStack->push(Middleware::history($recoveryHistory));
|
|
$recoveryService = new CloudBackupPublicationPreparation(
|
|
new CloudObjectStorageS3Client(new Client(['handler' => $recoveryStack])),
|
|
$runtimeResolver,
|
|
$clockFn
|
|
);
|
|
$recoveredJob = $recoveryService->ensureAgentJob((int) $unknown->id);
|
|
$assert($recoveredJob instanceof CloudJob
|
|
&& CloudJob::where('backup_id', (int) $unknown->id)->count() === 1
|
|
&& count($recoveryHistory) === 1,
|
|
'Exact-key reconciliation did not adopt one original multipart session.');
|
|
$unknownOperation->refresh();
|
|
$assert(str_contains((string) $unknownOperation->result_json,
|
|
'recovered-upload')
|
|
&& (int) $unknownOperation->attempts === 1,
|
|
'Recovered upload identity was replaced or the provider effect was replayed.');
|
|
$assert((int) $recoveryService->ensureAgentJob((int) $unknown->id)->id
|
|
=== (int) $recoveredJob->id
|
|
&& CloudJob::where('backup_id', (int) $unknown->id)->count() === 1,
|
|
'A second worker/call created a duplicate recovered job.');
|
|
|
|
// Automatic intent without a pre-existing job uses the same preparation
|
|
// path and advances its subscription exactly when the job binding commits.
|
|
$scheduledFor = $now - 60;
|
|
$subscription = new CloudBackupSubscription();
|
|
foreach ([
|
|
'id' => 5, 'instance_id' => 103, 'user_id' => 33, 'offer_id' => 8,
|
|
'enabled' => true, 'renew_next_period' => true, 'frequency' => 'daily',
|
|
'weekday' => 0, 'retention_count' => 2, 'timezone' => 'UTC',
|
|
'price' => '2.00', 'currency' => 'USD', 'price_version' => 1,
|
|
'state_revision' => 1, 'next_run_at' => $scheduledFor,
|
|
'last_scheduled_at' => null, 'last_backup_id' => null,
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
] as $field => $value) {
|
|
$subscription->{$field} = $value;
|
|
}
|
|
$subscription->save();
|
|
Capsule::table('cloud_backup_period_entitlements')->insert([
|
|
'subscription_id' => (int) $subscription->id,
|
|
'instance_id' => (int) $subscription->instance_id,
|
|
'user_id' => (int) $subscription->user_id,
|
|
'starts_at' => $now - 3600,
|
|
'ends_at' => $now + 3600,
|
|
]);
|
|
[$automatic] = $newBackup(3, 'automatic', 5, $scheduledFor);
|
|
$automaticService = new CloudBackupPublicationPreparation(
|
|
new CloudObjectStorageS3Client(new Client([
|
|
'handler' => HandlerStack::create(new MockHandler([
|
|
new Response(200, ['Content-Type' => 'application/xml'],
|
|
'<InitiateMultipartUploadResult>'
|
|
. '<UploadId>automatic-upload</UploadId>'
|
|
. '</InitiateMultipartUploadResult>'),
|
|
])),
|
|
])),
|
|
$runtimeResolver,
|
|
$clockFn
|
|
);
|
|
$automaticJob = $automaticService->ensureAgentJob((int) $automatic->id);
|
|
$subscription->refresh();
|
|
$assert($automaticJob instanceof CloudJob
|
|
&& (int) $subscription->last_scheduled_at === $scheduledFor
|
|
&& (int) $subscription->last_backup_id === (int) $automatic->id
|
|
&& (int) $subscription->next_run_at > $scheduledFor,
|
|
'Automatic backup intent did not atomically bind its job and advance schedule.');
|
|
$advancedNextRun = (int) $subscription->next_run_at;
|
|
$automaticService->ensureAgentJob((int) $automatic->id);
|
|
$subscription->refresh();
|
|
$assert((int) $subscription->next_run_at === $advancedNextRun
|
|
&& CloudJob::where('backup_id', (int) $automatic->id)->count() === 1,
|
|
'Repeated automatic preparation advanced the schedule or duplicated the job.');
|
|
|
|
// A migration may start after the durable storage intent commits but
|
|
// before its Agent job is bound. The post-S3 instance fence must reject
|
|
// the bind without replacing the original multipart identity. Once the
|
|
// blocker is removed, the same upload is signed and bound without another
|
|
// CreateMultipartUpload request.
|
|
[$blocked, $blockedPublication] = $newBackup(4, 'manual');
|
|
$blockedHistory = [];
|
|
$blockedStack = HandlerStack::create(new MockHandler([
|
|
static function () use ($assert): Response {
|
|
$assert(Capsule::connection('default')->transactionLevel() === 0,
|
|
'Multipart preparation crossed the business transaction.');
|
|
Capsule::table('cloud_instance_migrations')->insert([
|
|
'instance_id' => 104,
|
|
'state' => 'preparing_source',
|
|
]);
|
|
return new Response(200, ['Content-Type' => 'application/xml'],
|
|
'<InitiateMultipartUploadResult>'
|
|
. '<UploadId>blocked-upload</UploadId>'
|
|
. '</InitiateMultipartUploadResult>');
|
|
},
|
|
]));
|
|
$blockedStack->push(Middleware::history($blockedHistory));
|
|
$blockedService = new CloudBackupPublicationPreparation(
|
|
new CloudObjectStorageS3Client(new Client([
|
|
'handler' => $blockedStack,
|
|
])),
|
|
$runtimeResolver,
|
|
$clockFn
|
|
);
|
|
$expect(static fn () => $blockedService->ensureAgentJob(
|
|
(int) $blocked->id
|
|
), 409);
|
|
$blockedOperation = CloudStorageOperation::where(
|
|
'operation_id',
|
|
(string) $blockedPublication['operation_id']
|
|
)->firstOrFail();
|
|
$assert((string) $blockedOperation->phase === 'awaiting_receipt'
|
|
&& str_contains(
|
|
(string) $blockedOperation->result_json,
|
|
'blocked-upload'
|
|
)
|
|
&& CloudJob::where('backup_id', (int) $blocked->id)->count() === 0,
|
|
'A concurrent migration did not fence the pre-Agent backup bind.');
|
|
Capsule::table('cloud_instance_migrations')
|
|
->where('instance_id', 104)->delete();
|
|
$clock += 301;
|
|
$resumedService = new CloudBackupPublicationPreparation(
|
|
new CloudObjectStorageS3Client(new Client([
|
|
'handler' => HandlerStack::create(new MockHandler([])),
|
|
])),
|
|
$runtimeResolver,
|
|
$clockFn
|
|
);
|
|
$resumedJob = $resumedService->ensureAgentJob((int) $blocked->id);
|
|
$blockedOperation->refresh();
|
|
$assert($resumedJob instanceof CloudJob
|
|
&& count($blockedHistory) === 1
|
|
&& str_contains(
|
|
(string) $blockedOperation->result_json,
|
|
'blocked-upload'
|
|
),
|
|
'Clearing the blocker replayed or replaced the durable upload.');
|
|
|
|
// The same late fence applies to an ordinary in-flight instance job, not
|
|
// only to the migration ledger.
|
|
[$jobBlocked, $jobBlockedPublication] = $newBackup(7, 'manual');
|
|
$jobBlockedService = new CloudBackupPublicationPreparation(
|
|
new CloudObjectStorageS3Client(new Client([
|
|
'handler' => HandlerStack::create(new MockHandler([
|
|
static function () use ($clock): Response {
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'operation_id' => str_repeat('d', 32),
|
|
'request_idempotency_key' =>
|
|
'concurrent-instance-job-0007',
|
|
'request_fingerprint' => str_repeat('e', 64),
|
|
'instance_id' => 107, 'backup_id' => null,
|
|
'user_id' => 33, 'host_id' => 7,
|
|
'kind' => 'action', 'execution_mode' => 'agent',
|
|
'phase' => 'accepted', 'outcome' => null,
|
|
'status' => 'queued', 'error_code' => '',
|
|
'sealed_payload' => 'sealed', 'attempts' => 0,
|
|
'dispatch_attempts' => 0,
|
|
'dispatch_deadline_at' => $clock + 600,
|
|
'dispatch_started_at' => null,
|
|
'agent_accepted_at' => null,
|
|
'dispatched_at' => null,
|
|
'receipt_deadline_at' => null,
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
'result_json' => null,
|
|
'completion_event_id' => null,
|
|
'read_at' => null, 'available_at' => $clock,
|
|
'created_at' => $clock, 'updated_at' => $clock,
|
|
]);
|
|
return new Response(200, [
|
|
'Content-Type' => 'application/xml',
|
|
], '<InitiateMultipartUploadResult>'
|
|
. '<UploadId>job-blocked-upload</UploadId>'
|
|
. '</InitiateMultipartUploadResult>');
|
|
},
|
|
])),
|
|
])),
|
|
$runtimeResolver,
|
|
$clockFn
|
|
);
|
|
$expect(static fn () => $jobBlockedService->ensureAgentJob(
|
|
(int) $jobBlocked->id
|
|
), 409);
|
|
$jobBlockedOperation = CloudStorageOperation::where(
|
|
'operation_id',
|
|
(string) $jobBlockedPublication['operation_id']
|
|
)->firstOrFail();
|
|
$assert(CloudJob::where(
|
|
'backup_id',
|
|
(int) $jobBlocked->id
|
|
)->count() === 0
|
|
&& str_contains(
|
|
(string) $jobBlockedOperation->result_json,
|
|
'job-blocked-upload'
|
|
),
|
|
'A concurrent instance job did not fence the backup bind.');
|
|
CloudJob::where('operation_id', str_repeat('d', 32))->delete();
|
|
|
|
// The user changes the schedule while CreateMultipartUpload is in
|
|
// progress. The old slot may still bind, but it must not overwrite the
|
|
// newly selected frequency or next execution time.
|
|
$staleSlot = $clock - 60;
|
|
$newNextRun = $clock + 7 * 86400;
|
|
$changedSubscription = new CloudBackupSubscription();
|
|
foreach ([
|
|
'id' => 6, 'instance_id' => 105, 'user_id' => 33, 'offer_id' => 8,
|
|
'enabled' => true, 'renew_next_period' => true,
|
|
'frequency' => 'daily', 'weekday' => 0, 'retention_count' => 2,
|
|
'timezone' => 'UTC', 'price' => '2.00', 'currency' => 'USD',
|
|
'price_version' => 1, 'state_revision' => 1,
|
|
'next_run_at' => $staleSlot, 'last_scheduled_at' => null,
|
|
'last_backup_id' => null, 'created_at' => $now,
|
|
'updated_at' => $now,
|
|
] as $field => $value) {
|
|
$changedSubscription->{$field} = $value;
|
|
}
|
|
$changedSubscription->save();
|
|
Capsule::table('cloud_backup_period_entitlements')->insert([
|
|
'subscription_id' => 6, 'instance_id' => 105, 'user_id' => 33,
|
|
'starts_at' => $now - 3600, 'ends_at' => $clock + 3600,
|
|
]);
|
|
[$changedBackup] = $newBackup(5, 'automatic', 6, $staleSlot);
|
|
$changedService = new CloudBackupPublicationPreparation(
|
|
new CloudObjectStorageS3Client(new Client([
|
|
'handler' => HandlerStack::create(new MockHandler([
|
|
static function () use ($newNextRun): Response {
|
|
Capsule::table('cloud_backup_subscriptions')
|
|
->where('id', 6)->update([
|
|
'frequency' => 'weekly',
|
|
'weekday' => 3,
|
|
'state_revision' => 2,
|
|
'next_run_at' => $newNextRun,
|
|
]);
|
|
return new Response(200, [
|
|
'Content-Type' => 'application/xml',
|
|
], '<InitiateMultipartUploadResult>'
|
|
. '<UploadId>changed-plan-upload</UploadId>'
|
|
. '</InitiateMultipartUploadResult>');
|
|
},
|
|
])),
|
|
])),
|
|
$runtimeResolver,
|
|
$clockFn
|
|
);
|
|
$changedJob = $changedService->ensureAgentJob((int) $changedBackup->id);
|
|
$changedSubscription->refresh();
|
|
$assert($changedJob instanceof CloudJob
|
|
&& (string) $changedSubscription->frequency === 'weekly'
|
|
&& (int) $changedSubscription->weekday === 3
|
|
&& (int) $changedSubscription->next_run_at === $newNextRun
|
|
&& (int) $changedSubscription->last_scheduled_at === $staleSlot
|
|
&& (int) $changedSubscription->last_backup_id
|
|
=== (int) $changedBackup->id,
|
|
'An old backup intent overwrote the user\'s newer schedule.');
|
|
|
|
// Disabling the subscription during provider preparation is also a real
|
|
// fence. The remote multipart identity is retained for reconciliation,
|
|
// but no Agent job may escape after the paid entitlement is withdrawn.
|
|
$disabledSlot = $clock - 30;
|
|
$disabledSubscription = new CloudBackupSubscription();
|
|
foreach ([
|
|
'id' => 7, 'instance_id' => 106, 'user_id' => 33, 'offer_id' => 8,
|
|
'enabled' => true, 'renew_next_period' => true,
|
|
'frequency' => 'daily', 'weekday' => 0, 'retention_count' => 2,
|
|
'timezone' => 'UTC', 'price' => '2.00', 'currency' => 'USD',
|
|
'price_version' => 1, 'state_revision' => 1,
|
|
'next_run_at' => $disabledSlot, 'last_scheduled_at' => null,
|
|
'last_backup_id' => null, 'created_at' => $now,
|
|
'updated_at' => $now,
|
|
] as $field => $value) {
|
|
$disabledSubscription->{$field} = $value;
|
|
}
|
|
$disabledSubscription->save();
|
|
Capsule::table('cloud_backup_period_entitlements')->insert([
|
|
'subscription_id' => 7, 'instance_id' => 106, 'user_id' => 33,
|
|
'starts_at' => $now - 3600, 'ends_at' => $clock + 3600,
|
|
]);
|
|
[$disabledBackup, $disabledPublication] = $newBackup(
|
|
6,
|
|
'automatic',
|
|
7,
|
|
$disabledSlot
|
|
);
|
|
$disabledService = new CloudBackupPublicationPreparation(
|
|
new CloudObjectStorageS3Client(new Client([
|
|
'handler' => HandlerStack::create(new MockHandler([
|
|
static function (): Response {
|
|
Capsule::table('cloud_backup_subscriptions')
|
|
->where('id', 7)->update([
|
|
'enabled' => 0,
|
|
'state_revision' => 2,
|
|
]);
|
|
return new Response(200, [
|
|
'Content-Type' => 'application/xml',
|
|
], '<InitiateMultipartUploadResult>'
|
|
. '<UploadId>disabled-upload</UploadId>'
|
|
. '</InitiateMultipartUploadResult>');
|
|
},
|
|
])),
|
|
])),
|
|
$runtimeResolver,
|
|
$clockFn
|
|
);
|
|
$expect(static fn () => $disabledService->ensureAgentJob(
|
|
(int) $disabledBackup->id
|
|
), 409);
|
|
$disabledOperation = CloudStorageOperation::where(
|
|
'operation_id',
|
|
(string) $disabledPublication['operation_id']
|
|
)->firstOrFail();
|
|
$assert(CloudJob::where(
|
|
'backup_id',
|
|
(int) $disabledBackup->id
|
|
)->count() === 0
|
|
&& (string) $disabledOperation->phase === 'awaiting_receipt'
|
|
&& str_contains(
|
|
(string) $disabledOperation->result_json,
|
|
'disabled-upload'
|
|
),
|
|
'A disabled subscription still escaped as an Agent backup job.');
|
|
|
|
echo "Cloud backup publication preparation behavior tests passed.\n";
|
|
}
|