From 5c4f26944229fc4cc502b50a6714913340af9299 Mon Sep 17 00:00:00 2001 From: codex Date: Fri, 6 Nov 2026 20:48:06 +0800 Subject: [PATCH] feat(storage): split publication outbox worker --- ...t-cloud-storage-publication-worker.service | 25 +++ docs/DEPLOYMENT.md | 10 +- .../Command/CloudStoragePublicationWorker.php | 68 ++++++++ web/app/Command/XCat.php | 3 + .../Cloud/CloudRuntimeHealthService.php | 1 + .../Services/Cloud/CloudRuntimeHeartbeat.php | 9 +- .../CloudStorageReplicaWorkerService.php | 23 +-- web/tests/CloudRuntimeHealthTest.php | 7 +- web/tests/CloudRuntimeHeartbeatTest.php | 21 ++- ...oudStoragePublicationOutboxServiceTest.php | 156 ------------------ ...udStoragePublicationWorkerContractTest.php | 103 ++++++++++++ .../CloudStorageReplicaRepairServiceTest.php | 8 +- .../CloudStorageReplicaWorkerContractTest.php | 5 +- 13 files changed, 251 insertions(+), 188 deletions(-) create mode 100644 deploy/virt-cloud-storage-publication-worker.service create mode 100644 web/app/Command/CloudStoragePublicationWorker.php create mode 100644 web/tests/CloudStoragePublicationWorkerContractTest.php diff --git a/deploy/virt-cloud-storage-publication-worker.service b/deploy/virt-cloud-storage-publication-worker.service new file mode 100644 index 00000000..04751c4e --- /dev/null +++ b/deploy/virt-cloud-storage-publication-worker.service @@ -0,0 +1,25 @@ +[Unit] +Description=Virt Cloud object storage publication worker +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=apache +Group=apache +WorkingDirectory=/home/virt-cloud/web +ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudStoragePublicationWorker +Restart=on-failure +RestartSec=5 +LimitCORE=0 +UMask=0077 +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=false +ProtectSystem=strict +ReadWritePaths=-/home/virt-cloud/runtime/php +ReadWritePaths=-/home/virt-cloud/web/storage +TimeoutStopSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 5cbe8847..08fa062d 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -230,6 +230,7 @@ cp /home/virt-cloud/deploy/virt-cloud-vpc-worker@.service /etc/systemd/system/ cp /home/virt-cloud/deploy/virt-cloud-recovery-worker.service /etc/systemd/system/ cp /home/virt-cloud/deploy/virt-cloud-notification-worker.service /etc/systemd/system/ cp /home/virt-cloud/deploy/virt-cloud-storage-replica-worker.service /etc/systemd/system/ +cp /home/virt-cloud/deploy/virt-cloud-storage-publication-worker.service /etc/systemd/system/ cp /home/virt-cloud/deploy/virt-cloud-storage-health-worker.service /etc/systemd/system/ cd /home/virt-cloud/agent go build -trimpath -o /home/virt-cloud/bin/virt-cloud-storage-replicator \ @@ -239,7 +240,9 @@ chmod 0755 /home/virt-cloud/bin/virt-cloud-storage-replicator systemctl daemon-reload systemctl enable --now virt-cloud-worker.service \ virt-cloud-recovery-worker.service virt-cloud-notification-worker.service \ - virt-cloud-storage-replica-worker.service virt-cloud-storage-health-worker.service \ + virt-cloud-storage-replica-worker.service \ + virt-cloud-storage-publication-worker.service \ + virt-cloud-storage-health-worker.service \ virt-cloud-lifecycle.service \ virt-cloud-policy-worker@1.service virt-cloud-policy-worker@2.service \ virt-cloud-vpc-worker@{1..8}.service @@ -249,7 +252,9 @@ systemctl enable --now virt-cloud-worker.service \ `virt-cloud-notification-worker.service` 消费管理员维护通知和实例补偿结果通知;两个通道每轮各最多领取一封,任一通道异常不阻塞另一通道,也不占用虚拟机任务 worker。投递使用 300 秒租约;邮件传输一旦开始后返回失败、异常或租约过期都记为“效果未知”,不会自动重发。该服务是两类通知的必需组件;只更新 PHP 而不安装并启动该 unit 时,通知会安全保留在待发送状态。 -`virt-cloud-storage-replica-worker.service` 独立流式复制已冻结的对象存储制品,不占用虚拟机任务 worker。PHP 只通过标准输入把短期、限方法的 capability 交给 helper;命令行、环境、数据库和日志都不保存 capability 或对象存储凭据。复制租约覆盖 helper 的完整六小时硬截止;进程中断、回执损坏或完成结果不确定时只对固定目标对象执行 HEAD 与精确对象键的 Multipart 列表查询,不自动重放复制或 Abort。只有连续两次确认目标对象和对应 Multipart 会话都不存在后,才会重排同一复制意图;唯一且时间窗口匹配的未持久化会话可被原操作接管。同一对象键出现多个会话、记录会话与远端不匹配或会话无法归属原操作时,连续两次稳定证据会使原操作进入“副本人工核对”;管理员在提供商侧清理多余会话后只能重开该原操作的只读核对,本次点击不会由面板 Abort 或直接新建上传;只有两次确认无远端效果后才会沿用原操作安全重试。该 worker 独立写入 `storage_replica` 运行心跳;后台运行状态出现警告时应同时核对制品发布状态和该 unit 的 journal,不能仅凭 systemd 的 `active` 判断复制正常。 +`virt-cloud-storage-replica-worker.service` 独立流式复制已冻结的对象存储制品,不占用虚拟机任务 worker。PHP 只通过标准输入把短期、限方法的 capability 交给 helper;命令行、环境、数据库和日志都不保存 capability 或对象存储凭据。复制租约覆盖 helper 的完整六小时硬截止;进程中断、回执损坏或完成结果不确定时只对固定目标对象执行 HEAD 与精确对象键的 Multipart 列表查询,不自动重放复制或 Abort。只有连续两次确认目标对象和对应 Multipart 会话都不存在后,才会重排同一复制意图;唯一且时间窗口匹配的未持久化会话可被原操作接管。同一对象键出现多个会话、记录会话与远端不匹配或会话无法归属原操作时,连续两次稳定证据会使原操作进入“副本人工核对”;管理员在提供商侧清理多余会话后只能重开该原操作的只读核对,本次点击不会由面板 Abort 或直接新建上传;只有两次确认无远端效果后才会沿用原操作安全重试。该 worker 独立写入 `storage_replica` 运行心跳;后台运行状态出现警告时应同时核对副本修复、复制操作和该 unit 的 journal,不能仅凭 systemd 的 `active` 判断复制正常。 + +`virt-cloud-storage-publication-worker.service` 是单实例、本地数据库型业务发布 worker:每轮最多处理一个已经达到双副本保护的 `ready_to_finalize` 模板或救援 ISO 发布,不执行 PUT、COPY、HEAD、DELETE、Abort 或 Agent 请求。模板必须由它创建幂等的宿主机分发任务,救援 ISO 必须由它进入验证阶段;只更新 PHP 而未安装或启动该 unit 时,这两类制品会安全停在 `ready_to_finalize`,不会绕过保护直接发布。普通云备份仍由原同步流程最终化,不进入该 worker。它独立写入 `storage_publication` 运行心跳,并在后台运行状态中单独显示。由拆分前版本升级时,复制新 unit 后应同时重启 `virt-cloud-storage-publication-worker.service` 与 `virt-cloud-storage-replica-worker.service`,避免旧副本 worker 进程继续加载包含业务发布消费的旧代码。 `virt-cloud-storage-health-worker.service` 与副本复制 worker 彻底隔离:它每次只领取一个到期后端,在数据库事务外执行最长 15 秒的 `HeadBucket`,并且只能按领取时冻结的配置版本结算。成功只恢复健康证据,不会把人工设置的 `read_only`、`draining` 或 `disabled` 改回 `active`;失败使正在 `active` 的后端转为 `read_only`、按 60 秒到 1 小时退避,新写入随即停止选择它。这个 worker 不执行 PUT、COPY、DELETE 或 Abort,也不把凭据写入心跳、日志或命令行。 @@ -443,6 +448,7 @@ location = /agent/console { journalctl -u virt-cloud-agent.service -n 200 --no-pager journalctl -u virt-cloud-worker.service -u virt-cloud-recovery-worker.service \ -u virt-cloud-storage-replica-worker.service \ + -u virt-cloud-storage-publication-worker.service \ -u virt-cloud-storage-health-worker.service \ -u virt-cloud-lifecycle.service \ -u 'virt-cloud-policy-worker@*' -u 'virt-cloud-vpc-worker@*' \ diff --git a/web/app/Command/CloudStoragePublicationWorker.php b/web/app/Command/CloudStoragePublicationWorker.php new file mode 100644 index 00000000..124fef7f --- /dev/null +++ b/web/app/Command/CloudStoragePublicationWorker.php @@ -0,0 +1,68 @@ +busy('storage_publication_tick'); + try { + $worked = $worker->tick(); + $errors = $worker->observationErrors(); + } catch (\Throwable $error) { + $errors[] = 'storage_publication_failed'; + $heartbeat->error('storage_publication_failed'); + fwrite( + STDERR, + 'storage publication worker temporarily unavailable: ' + . get_class($error) . PHP_EOL + ); + if ($once) { + return 1; + } + $worked = false; + } + if ($errors === []) { + // An idle poll proves only that the process is alive. A + // real successful finalization is required to clear an + // earlier warning from this worker instance. + if ($worked) { + $unresolved = false; + $heartbeat->success(); + } elseif (!$unresolved) { + $heartbeat->success(); + } + } else { + $unresolved = true; + $heartbeat->error('storage_publication_failed'); + } + if (!$once && !$worked) { + usleep(1000000); + } + } while (!$once); + } catch (\Throwable $error) { + $heartbeat->error('storage_publication_failed'); + throw $error; + } finally { + $heartbeat->stopped(); + } + return 0; + } +} diff --git a/web/app/Command/XCat.php b/web/app/Command/XCat.php index 66aae93d..a087af8e 100644 --- a/web/app/Command/XCat.php +++ b/web/app/Command/XCat.php @@ -24,6 +24,8 @@ class XCat 'cloudWorker' => CloudWorker::run($this->argv) === 0, 'cloudStorageReplicaWorker' => CloudStorageReplicaWorker::run($this->argv) === 0, + 'cloudStoragePublicationWorker' => + CloudStoragePublicationWorker::run($this->argv) === 0, 'cloudStorageHealthWorker' => CloudStorageHealthWorker::run($this->argv) === 0, 'cloudLifecycle' => CloudLifecycle::run($this->argv) === 0, @@ -106,6 +108,7 @@ class XCat . " php xcat cloudInit\n" . " php xcat cloudWorker [--once] [--policy-only|--vpc-batch-only|--recovery-only|--notification-only]\n" . " php xcat cloudStorageReplicaWorker [--once]\n" + . " php xcat cloudStoragePublicationWorker [--once]\n" . " php xcat cloudStorageHealthWorker [--once]\n" . " php xcat cloudLifecycle [--once]\n" . " php xcat cloudTrafficPoolAttention inspect --pool-id=N\n" diff --git a/web/app/Services/Cloud/CloudRuntimeHealthService.php b/web/app/Services/Cloud/CloudRuntimeHealthService.php index 903f2655..1bc2d31d 100644 --- a/web/app/Services/Cloud/CloudRuntimeHealthService.php +++ b/web/app/Services/Cloud/CloudRuntimeHealthService.php @@ -93,6 +93,7 @@ final class CloudRuntimeHealthService 'recovery' => '任务安全收敛', 'notification' => '维护邮件投递', 'storage_replica' => '对象存储副本复制', + 'storage_publication' => '对象存储业务发布', 'storage_health' => '对象存储健康探测', 'lifecycle' => '到期与流量调度'] as $key => $label) { $found = null; diff --git a/web/app/Services/Cloud/CloudRuntimeHeartbeat.php b/web/app/Services/Cloud/CloudRuntimeHeartbeat.php index 2c3513cc..097e93c2 100644 --- a/web/app/Services/Cloud/CloudRuntimeHeartbeat.php +++ b/web/app/Services/Cloud/CloudRuntimeHeartbeat.php @@ -10,6 +10,9 @@ final class CloudRuntimeHeartbeat 'recovery' => ['recovery', '任务安全收敛 worker'], 'notification' => ['notification', '客户通知 worker'], 'storage_replica' => ['storage_replica', '对象存储副本 worker'], + 'storage_publication' => [ + 'storage_publication', '对象存储业务发布 worker', + ], 'storage_health' => ['storage_health', '对象存储健康探测 worker'], 'lifecycle' => ['lifecycle', '生命周期调度']]; private const PHASES = ['startup' => '启动', 'worker_tick' => '任务消费或核对', @@ -24,6 +27,7 @@ final class CloudRuntimeHeartbeat 'rescue_template_reconcile' => '救援 ISO 副本核对', 'rescue_template_cleanup' => '救援 ISO 退役清理', 'storage_replica_tick' => '对象存储副本复制或核对', + 'storage_publication_tick' => '对象存储业务目录最终化', 'storage_health_probe' => '对象存储健康探测', 'traffic_sync' => '流量同步', 'runtime_status_sync' => '实例状态同步', @@ -39,6 +43,7 @@ final class CloudRuntimeHeartbeat 'rescue_template_reconcile_failed', 'rescue_template_cleanup_failed', 'storage_replica_failed', + 'storage_publication_failed', 'storage_health_failed', 'storage_backend_unhealthy', 'traffic_sync_failed', 'runtime_status_sync_failed', 'runtime_status_sync_unconfigured', 'traffic_collection_unconfigured', @@ -229,7 +234,7 @@ final class CloudRuntimeHeartbeat if (++$scanned > self::SCAN_LIMIT || $removed >= 32) { break; } - if (preg_match('/^(normal|policy|vpc|recovery|notification|storage_replica|storage_health|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json(?:\.tmp)?$/D', + if (preg_match('/^(normal|policy|vpc|recovery|notification|storage_replica|storage_publication|storage_health|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json(?:\.tmp)?$/D', $entry->getFilename()) !== 1) { continue; } @@ -319,7 +324,7 @@ final class CloudRuntimeHeartbeat $incomplete = true; break; } - if (preg_match('/^(normal|policy|vpc|recovery|notification|storage_replica|storage_health|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json$/D', + if (preg_match('/^(normal|policy|vpc|recovery|notification|storage_replica|storage_publication|storage_health|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json$/D', $entry->getFilename(), $match) !== 1) { continue; } diff --git a/web/app/Services/Cloud/CloudStorageReplicaWorkerService.php b/web/app/Services/Cloud/CloudStorageReplicaWorkerService.php index a7738527..968de8e2 100644 --- a/web/app/Services/Cloud/CloudStorageReplicaWorkerService.php +++ b/web/app/Services/Cloud/CloudStorageReplicaWorkerService.php @@ -31,8 +31,6 @@ final class CloudStorageReplicaWorkerService private readonly ?CloudObjectStorageS3Client $client = null, ?callable $runtimeResolver = null, ?callable $clock = null, - private readonly ?CloudStoragePublicationOutboxService - $publicationOutbox = null, private readonly ?CloudStorageReplicaRepairService $replicaRepair = null ) { @@ -49,25 +47,6 @@ final class CloudStorageReplicaWorkerService { $this->errors = []; $now = ($this->clock)(); - // Publication finalization is a short, local DB operation. Run it - // before a remote copy that may occupy this worker for hours, while - // still bounding it to one item so replication always gets its turn. - $publicationWorked = false; - try { - $outbox = $this->publicationOutbox - ?? new CloudStoragePublicationOutboxService(); - $publicationWorked = $outbox->tick(); - $this->errors = array_merge( - $this->errors, - $outbox->observationErrors() - ); - } catch (\Throwable) { - // A local outbox/database failure must not prevent an unrelated - // replica operation from making progress. The unchanged durable - // publication remains eligible for a later worker cycle. - $this->errors[] = 'storage_publication_outbox_tick_failed'; - } - // Repair only reserves one local, durable copy intent. It performs no // provider I/O, so a backend that became writable can start healing // old single-copy artifacts without delaying existing remote work. @@ -106,7 +85,7 @@ final class CloudStorageReplicaWorkerService } } $this->errors = array_values(array_unique($this->errors)); - return $replicaWorked || $publicationWorked || $repairWorked; + return $replicaWorked || $repairWorked; } /** @return list */ diff --git a/web/tests/CloudRuntimeHealthTest.php b/web/tests/CloudRuntimeHealthTest.php index 6b3e63eb..622cb3cd 100644 --- a/web/tests/CloudRuntimeHealthTest.php +++ b/web/tests/CloudRuntimeHealthTest.php @@ -52,12 +52,15 @@ foreach ([['user_id' => 42, 'is_admin' => false], ['user_id' => 0, 'is_admin' => $assert($calls === [], 'Unauthorized requests performed probes.'); $payload = $service->snapshot($actor); $checks = array_column($payload['checks'], null, 'key'); -$assert(count($checks) === 12 && $calls === ['default', 'log', 'agents', 'redis'], 'Missing or duplicate probes.'); +$assert(count($checks) === 13 + && $calls === ['default', 'log', 'agents', 'redis'], + 'Missing or duplicate probes.'); $assert($checks['agents']['status'] === 'ok' && $checks['worker']['status'] === 'busy' && $checks['policy']['status'] === 'unknown' && $checks['vpc']['status'] === 'unknown' && $checks['notification']['status'] === 'unknown' && $checks['storage_replica']['status'] === 'unknown' + && $checks['storage_publication']['status'] === 'unknown' && $checks['storage_health']['status'] === 'unknown' && $checks['lifecycle']['status'] === 'warning', 'Health or heartbeat semantics changed.'); @@ -183,7 +186,7 @@ $response = (new CloudAdminRuntimeController())->index($request, new Response()) $assert($response->getStatusCode() === 200 && $response->getHeaderLine('Cache-Control') === 'private, no-store', 'Runtime response cached or unavailable.'); $decoded = json_decode((string) $response->getBody(), true); -$assert(count($decoded['checks']) === 12 +$assert(count($decoded['checks']) === 13 && array_column($decoded['checks'], 'status', 'key')['agents'] === 'error', 'Partial failure lost other checks or fabricated Agent configuration health.'); $routes = file_get_contents(dirname(__DIR__) . '/config/routes_cloud.php'); diff --git a/web/tests/CloudRuntimeHeartbeatTest.php b/web/tests/CloudRuntimeHeartbeatTest.php index ee294082..cf475c18 100644 --- a/web/tests/CloudRuntimeHeartbeatTest.php +++ b/web/tests/CloudRuntimeHeartbeatTest.php @@ -48,7 +48,8 @@ try { $empty = $checks($missing); $assert(array_keys($empty) === [ 'worker', 'policy', 'vpc', 'recovery', 'notification', - 'storage_replica', 'storage_health', 'lifecycle', + 'storage_replica', 'storage_publication', 'storage_health', + 'lifecycle', ] && array_unique(array_column($empty, 'status')) === ['unknown'] && !file_exists($missing), 'Missing snapshot must be unknown, cover all kinds, and never create runtime files.'); @@ -64,10 +65,14 @@ try { $storageReplica = CloudRuntimeHeartbeat::start( 'storage_replica', false, $directory, $clock ); + $storagePublication = CloudRuntimeHeartbeat::start( + 'storage_publication', false, $directory, $clock + ); $storageHealth = CloudRuntimeHeartbeat::start( 'storage_health', false, $directory, $clock ); - $assert(count($files($directory)) === 7, 'Concurrent worker identities overwrite one another.'); + $assert(count($files($directory)) === 8, + 'Concurrent worker identities overwrite one another.'); $assert(is_readable($directory) && is_writable($directory), 'Runtime directory is not readable and writable.'); foreach ($files($directory) as $path) { @@ -81,6 +86,9 @@ try { && $checks($directory)['notification']['label'] === '客户通知 worker' && $checks($directory)['storage_replica']['active_count'] === 1 && $checks($directory)['storage_replica']['label'] === '对象存储副本 worker' + && $checks($directory)['storage_publication']['active_count'] === 1 + && $checks($directory)['storage_publication']['label'] + === '对象存储业务发布 worker' && $checks($directory)['storage_health']['active_count'] === 1 && $checks($directory)['storage_health']['label'] === '对象存储健康探测 worker', 'Starting workers or multiple policy reporters were not represented.'); @@ -94,6 +102,15 @@ try { $checks($directory)['storage_replica']['message'], '对象存储副本复制或核对' ), 'Object replica work did not receive an isolated runtime phase.'); + $storagePublication->busy('storage_publication_tick'); + $assert(str_contains( + $checks($directory)['storage_publication']['message'], + '对象存储业务目录最终化' + ), 'Object publication did not receive an isolated runtime phase.'); + $now++; + $storagePublication->error('storage_publication_failed'); + $assert($checks($directory)['storage_publication']['status'] === 'warning', + 'Object publication failure was not preserved as a stable warning.'); $storageHealth->busy('storage_health_probe'); $assert(str_contains( $checks($directory)['storage_health']['message'], diff --git a/web/tests/CloudStoragePublicationOutboxServiceTest.php b/web/tests/CloudStoragePublicationOutboxServiceTest.php index 2a28fc6f..b12f79dc 100644 --- a/web/tests/CloudStoragePublicationOutboxServiceTest.php +++ b/web/tests/CloudStoragePublicationOutboxServiceTest.php @@ -3,14 +3,11 @@ require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudStorageArtifact; -use App\Models\CloudStorageOperation; use App\Models\CloudStoragePublication; -use App\Models\CloudStorageReplica; use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudRescueStoragePublicationService; use App\Services\Cloud\CloudStoragePublicationOutboxService; use App\Services\Cloud\CloudStorageReplicaAttentionService; -use App\Services\Cloud\CloudStorageReplicaWorkerService; use App\Services\Cloud\CloudTemplateStoragePublicationConsumer; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\QueryException; @@ -76,8 +73,6 @@ $db->schema()->create('cloud_storage_operations', static function ( ): void { $table->increments('id'); $table->string('operation_id')->unique(); - $table->string('request_idempotency_key')->unique(); - $table->string('request_fingerprint'); $table->unsignedBigInteger('artifact_id'); $table->unsignedBigInteger('replica_id')->nullable(); $table->unsignedBigInteger('source_backend_id')->nullable(); @@ -94,39 +89,18 @@ $db->schema()->create('cloud_storage_operations', static function ( $table->unsignedBigInteger('available_at')->default(0); $table->unsignedBigInteger('next_check_at')->default(0); $table->string('error_code')->default(''); - $table->text('result_json')->nullable(); $table->unsignedBigInteger('terminal_at')->nullable(); - $table->unsignedBigInteger('created_at'); - $table->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_replicas', static function ( Blueprint $table ): void { $table->increments('id'); - $table->string('replica_key')->unique(); $table->unsignedBigInteger('artifact_id'); $table->unsignedBigInteger('backend_id'); - $table->string('role'); - $table->string('object_key'); - $table->string('object_version_id')->default(''); - $table->string('object_etag')->default(''); $table->string('state'); - $table->string('publish_operation_id')->nullable(); - $table->string('delete_operation_id')->nullable(); - $table->unsignedInteger('delete_generation')->default(0); - $table->unsignedInteger('delete_cycle_attempts')->default(0); - $table->unsignedInteger('delete_presence_confirmations')->default(0); - $table->unsignedBigInteger('delete_last_checked_at')->default(0); - $table->unsignedBigInteger('delete_next_check_at')->default(0); - $table->unsignedBigInteger('delete_attention_at')->nullable(); - $table->string('lease_token')->nullable(); - $table->unsignedBigInteger('lease_until')->default(0); $table->unsignedBigInteger('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('error_code')->default(''); - $table->unsignedBigInteger('verified_at')->default(0); - $table->unsignedBigInteger('created_at'); - $table->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_backends', static function ( Blueprint $table @@ -424,134 +398,4 @@ $wrong->state = 'completed'; $wrong->finalized_at = $clock; $wrong->save(); -$workerOwned = $create('rescue', str_repeat('f', 32)); -$copyArtifact = new CloudStorageArtifact(); -$copyArtifact->artifact_key = bin2hex(random_bytes(16)); -$copyArtifact->kind = 'backup'; -$copyArtifact->maximum_size_bytes = 4096; -$copyArtifact->size_bytes = 4096; -$copyArtifact->sha256 = str_repeat('9', 64); -$copyArtifact->durability_policy = 'mirror_required_2'; -$copyArtifact->desired_copies = 2; -$copyArtifact->required_copies = 2; -$copyArtifact->state = 'available'; -$copyArtifact->last_operation_id = null; -$copyArtifact->verified_at = $clock; -$copyArtifact->created_at = $copyArtifact->updated_at = $clock; -$copyArtifact->save(); - -$sourceReplica = new CloudStorageReplica(); -$sourceReplica->replica_key = bin2hex(random_bytes(16)); -$sourceReplica->artifact_id = (int) $copyArtifact->id; -$sourceReplica->backend_id = 1; -$sourceReplica->role = 'primary'; -$sourceReplica->object_key = 'fixture/source.qcow2'; -$sourceReplica->object_version_id = ''; -$sourceReplica->object_etag = '"source"'; -$sourceReplica->size_bytes = 4096; -$sourceReplica->sha256 = str_repeat('9', 64); -$sourceReplica->state = 'ready'; -$sourceReplica->publish_operation_id = null; -$sourceReplica->delete_operation_id = null; -$sourceReplica->lease_token = null; -$sourceReplica->lease_until = 0; -$sourceReplica->error_code = ''; -$sourceReplica->verified_at = $clock; -$sourceReplica->created_at = $sourceReplica->updated_at = $clock; -$sourceReplica->save(); - -$targetReplica = new CloudStorageReplica(); -$targetReplica->replica_key = bin2hex(random_bytes(16)); -$targetReplica->artifact_id = (int) $copyArtifact->id; -$targetReplica->backend_id = 2; -$targetReplica->role = 'secondary'; -$targetReplica->object_key = 'fixture/target.qcow2'; -$targetReplica->object_version_id = ''; -$targetReplica->object_etag = ''; -$targetReplica->size_bytes = 0; -$targetReplica->sha256 = ''; -$targetReplica->state = 'reserved'; -$targetReplica->publish_operation_id = null; -$targetReplica->delete_operation_id = null; -$targetReplica->lease_token = null; -$targetReplica->lease_until = 0; -$targetReplica->error_code = ''; -$targetReplica->verified_at = 0; -$targetReplica->created_at = $targetReplica->updated_at = $clock; -$targetReplica->save(); - -$copyOperation = new CloudStorageOperation(); -$copyOperation->operation_id = bin2hex(random_bytes(16)); -$copyOperation->request_idempotency_key = - 'publication-worker-copy-fixture'; -$copyOperation->request_fingerprint = hash('sha256', 'copy-fixture'); -$copyOperation->artifact_id = (int) $copyArtifact->id; -$copyOperation->replica_id = (int) $targetReplica->id; -$copyOperation->source_backend_id = 1; -$copyOperation->target_backend_id = 2; -$copyOperation->kind = 'copy'; -$copyOperation->phase = 'accepted'; -$copyOperation->outcome = null; -$copyOperation->effect_resolution = 'not_applicable'; -$copyOperation->effect_may_have_applied = null; -$copyOperation->safe_to_resubmit = null; -$copyOperation->attempts = 0; -$copyOperation->lease_token = null; -$copyOperation->lease_until = 0; -$copyOperation->available_at = $clock; -$copyOperation->next_check_at = 0; -$copyOperation->error_code = ''; -$copyOperation->result_json = null; -$copyOperation->terminal_at = null; -$copyOperation->created_at = $copyOperation->updated_at = $clock; -$copyOperation->save(); -$integratedOutbox = new CloudStoragePublicationOutboxService( - static function ( - string $type, - int|string $owner, - int $publicationId, - int $artifactId - ) use (&$clock, $assert, $sourceReplica): void { - $assert($type === 'rescue' && $owner === str_repeat('f', 32), - 'Replica worker dispatched the wrong publication owner.'); - $publication = CloudStoragePublication::query() - ->whereKey($publicationId)->lockForUpdate()->firstOrFail(); - $publication->state = 'completed'; - $publication->finalized_at = $clock; - $publication->save(); - // Makes the replica source invalid only after publication work ran; - // the following replica_source_not_ready assertion therefore proves - // the short outbox step was consumed before the replica claim. - $sourceReplica->state = 'degraded'; - $sourceReplica->error_code = 'fixture_not_ready'; - $sourceReplica->verified_at = 0; - $sourceReplica->save(); - }, - static function () use (&$clock): int { - return $clock; - } -); -$replicaWorker = new CloudStorageReplicaWorkerService( - null, - null, - null, - static function () use (&$clock): int { - return $clock; - }, - $integratedOutbox -); -$assert($replicaWorker->tick() - && $replicaWorker->observationErrors() - === ['replica_source_not_ready'], - 'Replica worker did not process both bounded work queues in one tick.'); -$workerOwned->refresh(); -$copyOperation->refresh(); -$assert((string) $workerOwned->state === 'completed', - 'Replica worker did not settle the owner publication.'); -$assert((string) $copyOperation->phase === 'accepted' - && (int) $copyOperation->attempts === 1 - && (int) $copyOperation->available_at === $clock + 60 - && (string) $copyOperation->error_code === 'replica_source_not_ready', - 'Publication finalization starved the replica queue in the same tick.'); - echo "Storage publication outbox behavior passed.\n"; diff --git a/web/tests/CloudStoragePublicationWorkerContractTest.php b/web/tests/CloudStoragePublicationWorkerContractTest.php new file mode 100644 index 00000000..6b349381 --- /dev/null +++ b/web/tests/CloudStoragePublicationWorkerContractTest.php @@ -0,0 +1,103 @@ +whereNull('backup_id')") + && str_contains($outbox, 'CloudTemplateStoragePublicationConsumer') + && str_contains($outbox, 'finalizeReadyOnly'), + 'Publication worker can consume backups or lost its template/rescue ' + . 'owner boundary.'); +$assert(!str_contains($replicaWorker, 'CloudStoragePublicationOutboxService') + && !str_contains($replicaWorker, 'storage_publication') + && is_int($repairPosition) && is_int($claimPosition) + && $repairPosition < $claimPosition, + 'Replica worker retained publication work or lost repair-before-copy ' + . 'ordering.'); +$assert(str_contains($heartbeat, "'storage_publication' => [") + && str_contains($heartbeat, "'storage_publication_tick'") + && str_contains($heartbeat, "'storage_publication_failed'") + && str_contains( + $runtime, + "'storage_publication' => '对象存储业务发布'" + ), 'Publication runtime heartbeat is not independently observable.'); +$assert(str_contains( + $unit, + 'ExecStart=/usr/bin/php /home/virt-cloud/web/xcat ' + . 'cloudStoragePublicationWorker' + ) + && str_contains($unit, 'NoNewPrivileges=true') + && str_contains($unit, 'PrivateTmp=true') + && str_contains($unit, 'ProtectSystem=strict') + && str_contains($unit, 'LimitCORE=0') + && str_contains($unit, 'ReadWritePaths=-/home/virt-cloud/web/storage') + && !str_contains($unit, '%i'), + 'Publication systemd unit lost single-instance execution or hardening.'); +$assert(str_contains( + $deployment, + 'cp /home/virt-cloud/deploy/' + . 'virt-cloud-storage-publication-worker.service' + ) + && str_contains( + $deployment, + 'virt-cloud-storage-publication-worker.service' + ) + && str_contains( + $deployment, + '-u virt-cloud-storage-publication-worker.service' + ) + && str_contains($deployment, 'ready_to_finalize') + && str_contains($deployment, '普通云备份仍由原同步流程最终化'), + 'Publication worker deployment, journal, or missing-service impact is ' + . 'not documented.'); + +echo "Storage publication worker contract tests passed.\n"; diff --git a/web/tests/CloudStorageReplicaRepairServiceTest.php b/web/tests/CloudStorageReplicaRepairServiceTest.php index 24a6d7f9..06fbbb6c 100644 --- a/web/tests/CloudStorageReplicaRepairServiceTest.php +++ b/web/tests/CloudStorageReplicaRepairServiceTest.php @@ -274,9 +274,15 @@ $assert((int) $replacement->backend_id === 4 $workerSource = file_get_contents(dirname(__DIR__) . '/app/Services/Cloud/CloudStorageReplicaWorkerService.php'); +$repairPosition = is_string($workerSource) + ? strpos($workerSource, 'CloudStorageReplicaRepairService') : false; +$claimPosition = is_string($workerSource) + ? strpos($workerSource, 'claimReplicaWork') : false; $assert(is_string($workerSource) && str_contains($workerSource, 'CloudStorageReplicaRepairService') - && str_contains($workerSource, 'storage_replica_repair_tick_failed'), + && str_contains($workerSource, 'storage_replica_repair_tick_failed') + && is_int($repairPosition) && is_int($claimPosition) + && $repairPosition < $claimPosition, 'Replica worker does not consume the bounded repair queue.'); echo "Storage replica repair behavior passed.\n"; diff --git a/web/tests/CloudStorageReplicaWorkerContractTest.php b/web/tests/CloudStorageReplicaWorkerContractTest.php index ecf6907b..0e3a72a9 100644 --- a/web/tests/CloudStorageReplicaWorkerContractTest.php +++ b/web/tests/CloudStorageReplicaWorkerContractTest.php @@ -45,8 +45,11 @@ $assert(str_contains($unit, $assert(str_contains($worker, "'object-replica-verify-v1'") && str_contains($worker, 'presignedVersionedGet') && str_contains($worker, 'markReplicaUnknown') + && !str_contains($worker, 'CloudStoragePublicationOutboxService') + && !str_contains($worker, 'storage_publication') && !str_contains($worker, "result_json ="), - 'Worker lost exact-read reconciliation or bypasses sanitized persistence.'); + 'Worker lost exact-read reconciliation, retained publication work, or ' + . 'bypasses sanitized persistence.'); $assert(str_contains($worker, 'findMultipartUploadsByExactObjectKey') && str_contains($worker, 'resumeMultipartUpload') && str_contains($worker, 'operation_available_at')