feat(storage): split publication outbox worker

go
codex 2 weeks ago
parent 0757e98c9d
commit 5c4f269442

@ -0,0 +1,25 @@
[Unit]
Description=Virt Cloud object storage publication worker
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=apache
Group=apache
WorkingDirectory=/home/virt-cloud/web
ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudStoragePublicationWorker
Restart=on-failure
RestartSec=5
LimitCORE=0
UMask=0077
NoNewPrivileges=true
PrivateTmp=true
ProtectHome=false
ProtectSystem=strict
ReadWritePaths=-/home/virt-cloud/runtime/php
ReadWritePaths=-/home/virt-cloud/web/storage
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

@ -230,6 +230,7 @@ cp /home/virt-cloud/deploy/virt-cloud-vpc-worker@.service /etc/systemd/system/
cp /home/virt-cloud/deploy/virt-cloud-recovery-worker.service /etc/systemd/system/
cp /home/virt-cloud/deploy/virt-cloud-notification-worker.service /etc/systemd/system/
cp /home/virt-cloud/deploy/virt-cloud-storage-replica-worker.service /etc/systemd/system/
cp /home/virt-cloud/deploy/virt-cloud-storage-publication-worker.service /etc/systemd/system/
cp /home/virt-cloud/deploy/virt-cloud-storage-health-worker.service /etc/systemd/system/
cd /home/virt-cloud/agent
go build -trimpath -o /home/virt-cloud/bin/virt-cloud-storage-replicator \
@ -239,7 +240,9 @@ chmod 0755 /home/virt-cloud/bin/virt-cloud-storage-replicator
systemctl daemon-reload
systemctl enable --now virt-cloud-worker.service \
virt-cloud-recovery-worker.service virt-cloud-notification-worker.service \
virt-cloud-storage-replica-worker.service virt-cloud-storage-health-worker.service \
virt-cloud-storage-replica-worker.service \
virt-cloud-storage-publication-worker.service \
virt-cloud-storage-health-worker.service \
virt-cloud-lifecycle.service \
virt-cloud-policy-worker@1.service virt-cloud-policy-worker@2.service \
virt-cloud-vpc-worker@{1..8}.service
@ -249,7 +252,9 @@ systemctl enable --now virt-cloud-worker.service \
`virt-cloud-notification-worker.service` 消费管理员维护通知和实例补偿结果通知;两个通道每轮各最多领取一封,任一通道异常不阻塞另一通道,也不占用虚拟机任务 worker。投递使用 300 秒租约;邮件传输一旦开始后返回失败、异常或租约过期都记为“效果未知”,不会自动重发。该服务是两类通知的必需组件;只更新 PHP 而不安装并启动该 unit 时,通知会安全保留在待发送状态。
`virt-cloud-storage-replica-worker.service` 独立流式复制已冻结的对象存储制品,不占用虚拟机任务 worker。PHP 只通过标准输入把短期、限方法的 capability 交给 helper;命令行、环境、数据库和日志都不保存 capability 或对象存储凭据。复制租约覆盖 helper 的完整六小时硬截止;进程中断、回执损坏或完成结果不确定时只对固定目标对象执行 HEAD 与精确对象键的 Multipart 列表查询,不自动重放复制或 Abort。只有连续两次确认目标对象和对应 Multipart 会话都不存在后,才会重排同一复制意图;唯一且时间窗口匹配的未持久化会话可被原操作接管。同一对象键出现多个会话、记录会话与远端不匹配或会话无法归属原操作时,连续两次稳定证据会使原操作进入“副本人工核对”;管理员在提供商侧清理多余会话后只能重开该原操作的只读核对,本次点击不会由面板 Abort 或直接新建上传;只有两次确认无远端效果后才会沿用原操作安全重试。该 worker 独立写入 `storage_replica` 运行心跳;后台运行状态出现警告时应同时核对制品发布状态和该 unit 的 journal,不能仅凭 systemd 的 `active` 判断复制正常。
`virt-cloud-storage-replica-worker.service` 独立流式复制已冻结的对象存储制品,不占用虚拟机任务 worker。PHP 只通过标准输入把短期、限方法的 capability 交给 helper;命令行、环境、数据库和日志都不保存 capability 或对象存储凭据。复制租约覆盖 helper 的完整六小时硬截止;进程中断、回执损坏或完成结果不确定时只对固定目标对象执行 HEAD 与精确对象键的 Multipart 列表查询,不自动重放复制或 Abort。只有连续两次确认目标对象和对应 Multipart 会话都不存在后,才会重排同一复制意图;唯一且时间窗口匹配的未持久化会话可被原操作接管。同一对象键出现多个会话、记录会话与远端不匹配或会话无法归属原操作时,连续两次稳定证据会使原操作进入“副本人工核对”;管理员在提供商侧清理多余会话后只能重开该原操作的只读核对,本次点击不会由面板 Abort 或直接新建上传;只有两次确认无远端效果后才会沿用原操作安全重试。该 worker 独立写入 `storage_replica` 运行心跳;后台运行状态出现警告时应同时核对副本修复、复制操作和该 unit 的 journal,不能仅凭 systemd 的 `active` 判断复制正常。
`virt-cloud-storage-publication-worker.service` 是单实例、本地数据库型业务发布 worker:每轮最多处理一个已经达到双副本保护的 `ready_to_finalize` 模板或救援 ISO 发布,不执行 PUT、COPY、HEAD、DELETE、Abort 或 Agent 请求。模板必须由它创建幂等的宿主机分发任务,救援 ISO 必须由它进入验证阶段;只更新 PHP 而未安装或启动该 unit 时,这两类制品会安全停在 `ready_to_finalize`,不会绕过保护直接发布。普通云备份仍由原同步流程最终化,不进入该 worker。它独立写入 `storage_publication` 运行心跳,并在后台运行状态中单独显示。由拆分前版本升级时,复制新 unit 后应同时重启 `virt-cloud-storage-publication-worker.service` 与 `virt-cloud-storage-replica-worker.service`,避免旧副本 worker 进程继续加载包含业务发布消费的旧代码。
`virt-cloud-storage-health-worker.service` 与副本复制 worker 彻底隔离:它每次只领取一个到期后端,在数据库事务外执行最长 15 秒的 `HeadBucket`,并且只能按领取时冻结的配置版本结算。成功只恢复健康证据,不会把人工设置的 `read_only`、`draining` 或 `disabled` 改回 `active`;失败使正在 `active` 的后端转为 `read_only`、按 60 秒到 1 小时退避,新写入随即停止选择它。这个 worker 不执行 PUT、COPY、DELETE 或 Abort,也不把凭据写入心跳、日志或命令行。
@ -443,6 +448,7 @@ location = /agent/console {
journalctl -u virt-cloud-agent.service -n 200 --no-pager
journalctl -u virt-cloud-worker.service -u virt-cloud-recovery-worker.service \
-u virt-cloud-storage-replica-worker.service \
-u virt-cloud-storage-publication-worker.service \
-u virt-cloud-storage-health-worker.service \
-u virt-cloud-lifecycle.service \
-u 'virt-cloud-policy-worker@*' -u 'virt-cloud-vpc-worker@*' \

@ -0,0 +1,68 @@
<?php
namespace App\Command;
use App\Services\Cloud\CloudRuntimeHeartbeat;
use App\Services\Cloud\CloudStoragePublicationOutboxService;
/** Finalizes protected template and rescue publications outside copy workers. */
final class CloudStoragePublicationWorker
{
public static function run(
array $argv = [],
?CloudRuntimeHeartbeat $heartbeat = null
): int {
$once = in_array('--once', $argv, true);
$heartbeat ??= CloudRuntimeHeartbeat::start(
'storage_publication',
$once
);
$unresolved = false;
try {
$worker = new CloudStoragePublicationOutboxService();
do {
$errors = [];
$heartbeat->busy('storage_publication_tick');
try {
$worked = $worker->tick();
$errors = $worker->observationErrors();
} catch (\Throwable $error) {
$errors[] = 'storage_publication_failed';
$heartbeat->error('storage_publication_failed');
fwrite(
STDERR,
'storage publication worker temporarily unavailable: '
. get_class($error) . PHP_EOL
);
if ($once) {
return 1;
}
$worked = false;
}
if ($errors === []) {
// An idle poll proves only that the process is alive. A
// real successful finalization is required to clear an
// earlier warning from this worker instance.
if ($worked) {
$unresolved = false;
$heartbeat->success();
} elseif (!$unresolved) {
$heartbeat->success();
}
} else {
$unresolved = true;
$heartbeat->error('storage_publication_failed');
}
if (!$once && !$worked) {
usleep(1000000);
}
} while (!$once);
} catch (\Throwable $error) {
$heartbeat->error('storage_publication_failed');
throw $error;
} finally {
$heartbeat->stopped();
}
return 0;
}
}

@ -24,6 +24,8 @@ class XCat
'cloudWorker' => CloudWorker::run($this->argv) === 0,
'cloudStorageReplicaWorker' =>
CloudStorageReplicaWorker::run($this->argv) === 0,
'cloudStoragePublicationWorker' =>
CloudStoragePublicationWorker::run($this->argv) === 0,
'cloudStorageHealthWorker' =>
CloudStorageHealthWorker::run($this->argv) === 0,
'cloudLifecycle' => CloudLifecycle::run($this->argv) === 0,
@ -106,6 +108,7 @@ class XCat
. " php xcat cloudInit\n"
. " php xcat cloudWorker [--once] [--policy-only|--vpc-batch-only|--recovery-only|--notification-only]\n"
. " php xcat cloudStorageReplicaWorker [--once]\n"
. " php xcat cloudStoragePublicationWorker [--once]\n"
. " php xcat cloudStorageHealthWorker [--once]\n"
. " php xcat cloudLifecycle [--once]\n"
. " php xcat cloudTrafficPoolAttention inspect --pool-id=N\n"

@ -93,6 +93,7 @@ final class CloudRuntimeHealthService
'recovery' => '任务安全收敛',
'notification' => '维护邮件投递',
'storage_replica' => '对象存储副本复制',
'storage_publication' => '对象存储业务发布',
'storage_health' => '对象存储健康探测',
'lifecycle' => '到期与流量调度'] as $key => $label) {
$found = null;

@ -10,6 +10,9 @@ final class CloudRuntimeHeartbeat
'recovery' => ['recovery', '任务安全收敛 worker'],
'notification' => ['notification', '客户通知 worker'],
'storage_replica' => ['storage_replica', '对象存储副本 worker'],
'storage_publication' => [
'storage_publication', '对象存储业务发布 worker',
],
'storage_health' => ['storage_health', '对象存储健康探测 worker'],
'lifecycle' => ['lifecycle', '生命周期调度']];
private const PHASES = ['startup' => '启动', 'worker_tick' => '任务消费或核对',
@ -24,6 +27,7 @@ final class CloudRuntimeHeartbeat
'rescue_template_reconcile' => '救援 ISO 副本核对',
'rescue_template_cleanup' => '救援 ISO 退役清理',
'storage_replica_tick' => '对象存储副本复制或核对',
'storage_publication_tick' => '对象存储业务目录最终化',
'storage_health_probe' => '对象存储健康探测',
'traffic_sync' => '流量同步',
'runtime_status_sync' => '实例状态同步',
@ -39,6 +43,7 @@ final class CloudRuntimeHeartbeat
'rescue_template_reconcile_failed',
'rescue_template_cleanup_failed',
'storage_replica_failed',
'storage_publication_failed',
'storage_health_failed', 'storage_backend_unhealthy',
'traffic_sync_failed', 'runtime_status_sync_failed',
'runtime_status_sync_unconfigured', 'traffic_collection_unconfigured',
@ -229,7 +234,7 @@ final class CloudRuntimeHeartbeat
if (++$scanned > self::SCAN_LIMIT || $removed >= 32) {
break;
}
if (preg_match('/^(normal|policy|vpc|recovery|notification|storage_replica|storage_health|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json(?:\.tmp)?$/D',
if (preg_match('/^(normal|policy|vpc|recovery|notification|storage_replica|storage_publication|storage_health|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json(?:\.tmp)?$/D',
$entry->getFilename()) !== 1) {
continue;
}
@ -319,7 +324,7 @@ final class CloudRuntimeHeartbeat
$incomplete = true;
break;
}
if (preg_match('/^(normal|policy|vpc|recovery|notification|storage_replica|storage_health|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json$/D',
if (preg_match('/^(normal|policy|vpc|recovery|notification|storage_replica|storage_publication|storage_health|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json$/D',
$entry->getFilename(), $match) !== 1) {
continue;
}

@ -31,8 +31,6 @@ final class CloudStorageReplicaWorkerService
private readonly ?CloudObjectStorageS3Client $client = null,
?callable $runtimeResolver = null,
?callable $clock = null,
private readonly ?CloudStoragePublicationOutboxService
$publicationOutbox = null,
private readonly ?CloudStorageReplicaRepairService
$replicaRepair = null
) {
@ -49,25 +47,6 @@ final class CloudStorageReplicaWorkerService
{
$this->errors = [];
$now = ($this->clock)();
// Publication finalization is a short, local DB operation. Run it
// before a remote copy that may occupy this worker for hours, while
// still bounding it to one item so replication always gets its turn.
$publicationWorked = false;
try {
$outbox = $this->publicationOutbox
?? new CloudStoragePublicationOutboxService();
$publicationWorked = $outbox->tick();
$this->errors = array_merge(
$this->errors,
$outbox->observationErrors()
);
} catch (\Throwable) {
// A local outbox/database failure must not prevent an unrelated
// replica operation from making progress. The unchanged durable
// publication remains eligible for a later worker cycle.
$this->errors[] = 'storage_publication_outbox_tick_failed';
}
// Repair only reserves one local, durable copy intent. It performs no
// provider I/O, so a backend that became writable can start healing
// old single-copy artifacts without delaying existing remote work.
@ -106,7 +85,7 @@ final class CloudStorageReplicaWorkerService
}
}
$this->errors = array_values(array_unique($this->errors));
return $replicaWorked || $publicationWorked || $repairWorked;
return $replicaWorked || $repairWorked;
}
/** @return list<string> */

@ -52,12 +52,15 @@ foreach ([['user_id' => 42, 'is_admin' => false], ['user_id' => 0, 'is_admin' =>
$assert($calls === [], 'Unauthorized requests performed probes.');
$payload = $service->snapshot($actor);
$checks = array_column($payload['checks'], null, 'key');
$assert(count($checks) === 12 && $calls === ['default', 'log', 'agents', 'redis'], 'Missing or duplicate probes.');
$assert(count($checks) === 13
&& $calls === ['default', 'log', 'agents', 'redis'],
'Missing or duplicate probes.');
$assert($checks['agents']['status'] === 'ok' && $checks['worker']['status'] === 'busy'
&& $checks['policy']['status'] === 'unknown'
&& $checks['vpc']['status'] === 'unknown'
&& $checks['notification']['status'] === 'unknown'
&& $checks['storage_replica']['status'] === 'unknown'
&& $checks['storage_publication']['status'] === 'unknown'
&& $checks['storage_health']['status'] === 'unknown'
&& $checks['lifecycle']['status'] === 'warning',
'Health or heartbeat semantics changed.');
@ -183,7 +186,7 @@ $response = (new CloudAdminRuntimeController())->index($request, new Response())
$assert($response->getStatusCode() === 200
&& $response->getHeaderLine('Cache-Control') === 'private, no-store', 'Runtime response cached or unavailable.');
$decoded = json_decode((string) $response->getBody(), true);
$assert(count($decoded['checks']) === 12
$assert(count($decoded['checks']) === 13
&& array_column($decoded['checks'], 'status', 'key')['agents'] === 'error',
'Partial failure lost other checks or fabricated Agent configuration health.');
$routes = file_get_contents(dirname(__DIR__) . '/config/routes_cloud.php');

@ -48,7 +48,8 @@ try {
$empty = $checks($missing);
$assert(array_keys($empty) === [
'worker', 'policy', 'vpc', 'recovery', 'notification',
'storage_replica', 'storage_health', 'lifecycle',
'storage_replica', 'storage_publication', 'storage_health',
'lifecycle',
]
&& array_unique(array_column($empty, 'status')) === ['unknown'] && !file_exists($missing),
'Missing snapshot must be unknown, cover all kinds, and never create runtime files.');
@ -64,10 +65,14 @@ try {
$storageReplica = CloudRuntimeHeartbeat::start(
'storage_replica', false, $directory, $clock
);
$storagePublication = CloudRuntimeHeartbeat::start(
'storage_publication', false, $directory, $clock
);
$storageHealth = CloudRuntimeHeartbeat::start(
'storage_health', false, $directory, $clock
);
$assert(count($files($directory)) === 7, 'Concurrent worker identities overwrite one another.');
$assert(count($files($directory)) === 8,
'Concurrent worker identities overwrite one another.');
$assert(is_readable($directory) && is_writable($directory),
'Runtime directory is not readable and writable.');
foreach ($files($directory) as $path) {
@ -81,6 +86,9 @@ try {
&& $checks($directory)['notification']['label'] === '客户通知 worker'
&& $checks($directory)['storage_replica']['active_count'] === 1
&& $checks($directory)['storage_replica']['label'] === '对象存储副本 worker'
&& $checks($directory)['storage_publication']['active_count'] === 1
&& $checks($directory)['storage_publication']['label']
=== '对象存储业务发布 worker'
&& $checks($directory)['storage_health']['active_count'] === 1
&& $checks($directory)['storage_health']['label'] === '对象存储健康探测 worker',
'Starting workers or multiple policy reporters were not represented.');
@ -94,6 +102,15 @@ try {
$checks($directory)['storage_replica']['message'],
'对象存储副本复制或核对'
), 'Object replica work did not receive an isolated runtime phase.');
$storagePublication->busy('storage_publication_tick');
$assert(str_contains(
$checks($directory)['storage_publication']['message'],
'对象存储业务目录最终化'
), 'Object publication did not receive an isolated runtime phase.');
$now++;
$storagePublication->error('storage_publication_failed');
$assert($checks($directory)['storage_publication']['status'] === 'warning',
'Object publication failure was not preserved as a stable warning.');
$storageHealth->busy('storage_health_probe');
$assert(str_contains(
$checks($directory)['storage_health']['message'],

@ -3,14 +3,11 @@
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudStorageArtifact;
use App\Models\CloudStorageOperation;
use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudRescueStoragePublicationService;
use App\Services\Cloud\CloudStoragePublicationOutboxService;
use App\Services\Cloud\CloudStorageReplicaAttentionService;
use App\Services\Cloud\CloudStorageReplicaWorkerService;
use App\Services\Cloud\CloudTemplateStoragePublicationConsumer;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\QueryException;
@ -76,8 +73,6 @@ $db->schema()->create('cloud_storage_operations', static function (
): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key')->unique();
$table->string('request_fingerprint');
$table->unsignedBigInteger('artifact_id');
$table->unsignedBigInteger('replica_id')->nullable();
$table->unsignedBigInteger('source_backend_id')->nullable();
@ -94,39 +89,18 @@ $db->schema()->create('cloud_storage_operations', static function (
$table->unsignedBigInteger('available_at')->default(0);
$table->unsignedBigInteger('next_check_at')->default(0);
$table->string('error_code')->default('');
$table->text('result_json')->nullable();
$table->unsignedBigInteger('terminal_at')->nullable();
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_replicas', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('replica_key')->unique();
$table->unsignedBigInteger('artifact_id');
$table->unsignedBigInteger('backend_id');
$table->string('role');
$table->string('object_key');
$table->string('object_version_id')->default('');
$table->string('object_etag')->default('');
$table->string('state');
$table->string('publish_operation_id')->nullable();
$table->string('delete_operation_id')->nullable();
$table->unsignedInteger('delete_generation')->default(0);
$table->unsignedInteger('delete_cycle_attempts')->default(0);
$table->unsignedInteger('delete_presence_confirmations')->default(0);
$table->unsignedBigInteger('delete_last_checked_at')->default(0);
$table->unsignedBigInteger('delete_next_check_at')->default(0);
$table->unsignedBigInteger('delete_attention_at')->nullable();
$table->string('lease_token')->nullable();
$table->unsignedBigInteger('lease_until')->default(0);
$table->unsignedBigInteger('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('error_code')->default('');
$table->unsignedBigInteger('verified_at')->default(0);
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_backends', static function (
Blueprint $table
@ -424,134 +398,4 @@ $wrong->state = 'completed';
$wrong->finalized_at = $clock;
$wrong->save();
$workerOwned = $create('rescue', str_repeat('f', 32));
$copyArtifact = new CloudStorageArtifact();
$copyArtifact->artifact_key = bin2hex(random_bytes(16));
$copyArtifact->kind = 'backup';
$copyArtifact->maximum_size_bytes = 4096;
$copyArtifact->size_bytes = 4096;
$copyArtifact->sha256 = str_repeat('9', 64);
$copyArtifact->durability_policy = 'mirror_required_2';
$copyArtifact->desired_copies = 2;
$copyArtifact->required_copies = 2;
$copyArtifact->state = 'available';
$copyArtifact->last_operation_id = null;
$copyArtifact->verified_at = $clock;
$copyArtifact->created_at = $copyArtifact->updated_at = $clock;
$copyArtifact->save();
$sourceReplica = new CloudStorageReplica();
$sourceReplica->replica_key = bin2hex(random_bytes(16));
$sourceReplica->artifact_id = (int) $copyArtifact->id;
$sourceReplica->backend_id = 1;
$sourceReplica->role = 'primary';
$sourceReplica->object_key = 'fixture/source.qcow2';
$sourceReplica->object_version_id = '';
$sourceReplica->object_etag = '"source"';
$sourceReplica->size_bytes = 4096;
$sourceReplica->sha256 = str_repeat('9', 64);
$sourceReplica->state = 'ready';
$sourceReplica->publish_operation_id = null;
$sourceReplica->delete_operation_id = null;
$sourceReplica->lease_token = null;
$sourceReplica->lease_until = 0;
$sourceReplica->error_code = '';
$sourceReplica->verified_at = $clock;
$sourceReplica->created_at = $sourceReplica->updated_at = $clock;
$sourceReplica->save();
$targetReplica = new CloudStorageReplica();
$targetReplica->replica_key = bin2hex(random_bytes(16));
$targetReplica->artifact_id = (int) $copyArtifact->id;
$targetReplica->backend_id = 2;
$targetReplica->role = 'secondary';
$targetReplica->object_key = 'fixture/target.qcow2';
$targetReplica->object_version_id = '';
$targetReplica->object_etag = '';
$targetReplica->size_bytes = 0;
$targetReplica->sha256 = '';
$targetReplica->state = 'reserved';
$targetReplica->publish_operation_id = null;
$targetReplica->delete_operation_id = null;
$targetReplica->lease_token = null;
$targetReplica->lease_until = 0;
$targetReplica->error_code = '';
$targetReplica->verified_at = 0;
$targetReplica->created_at = $targetReplica->updated_at = $clock;
$targetReplica->save();
$copyOperation = new CloudStorageOperation();
$copyOperation->operation_id = bin2hex(random_bytes(16));
$copyOperation->request_idempotency_key =
'publication-worker-copy-fixture';
$copyOperation->request_fingerprint = hash('sha256', 'copy-fixture');
$copyOperation->artifact_id = (int) $copyArtifact->id;
$copyOperation->replica_id = (int) $targetReplica->id;
$copyOperation->source_backend_id = 1;
$copyOperation->target_backend_id = 2;
$copyOperation->kind = 'copy';
$copyOperation->phase = 'accepted';
$copyOperation->outcome = null;
$copyOperation->effect_resolution = 'not_applicable';
$copyOperation->effect_may_have_applied = null;
$copyOperation->safe_to_resubmit = null;
$copyOperation->attempts = 0;
$copyOperation->lease_token = null;
$copyOperation->lease_until = 0;
$copyOperation->available_at = $clock;
$copyOperation->next_check_at = 0;
$copyOperation->error_code = '';
$copyOperation->result_json = null;
$copyOperation->terminal_at = null;
$copyOperation->created_at = $copyOperation->updated_at = $clock;
$copyOperation->save();
$integratedOutbox = new CloudStoragePublicationOutboxService(
static function (
string $type,
int|string $owner,
int $publicationId,
int $artifactId
) use (&$clock, $assert, $sourceReplica): void {
$assert($type === 'rescue' && $owner === str_repeat('f', 32),
'Replica worker dispatched the wrong publication owner.');
$publication = CloudStoragePublication::query()
->whereKey($publicationId)->lockForUpdate()->firstOrFail();
$publication->state = 'completed';
$publication->finalized_at = $clock;
$publication->save();
// Makes the replica source invalid only after publication work ran;
// the following replica_source_not_ready assertion therefore proves
// the short outbox step was consumed before the replica claim.
$sourceReplica->state = 'degraded';
$sourceReplica->error_code = 'fixture_not_ready';
$sourceReplica->verified_at = 0;
$sourceReplica->save();
},
static function () use (&$clock): int {
return $clock;
}
);
$replicaWorker = new CloudStorageReplicaWorkerService(
null,
null,
null,
static function () use (&$clock): int {
return $clock;
},
$integratedOutbox
);
$assert($replicaWorker->tick()
&& $replicaWorker->observationErrors()
=== ['replica_source_not_ready'],
'Replica worker did not process both bounded work queues in one tick.');
$workerOwned->refresh();
$copyOperation->refresh();
$assert((string) $workerOwned->state === 'completed',
'Replica worker did not settle the owner publication.');
$assert((string) $copyOperation->phase === 'accepted'
&& (int) $copyOperation->attempts === 1
&& (int) $copyOperation->available_at === $clock + 60
&& (string) $copyOperation->error_code === 'replica_source_not_ready',
'Publication finalization starved the replica queue in the same tick.');
echo "Storage publication outbox behavior passed.\n";

@ -0,0 +1,103 @@
<?php
$root = dirname(__DIR__, 2);
$read = static function (string $path) use ($root): string {
$contents = file_get_contents($root . '/' . $path);
if (!is_string($contents)) {
throw new RuntimeException('Unable to read ' . $path);
}
return $contents;
};
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$xcat = $read('web/app/Command/XCat.php');
$command = $read('web/app/Command/CloudStoragePublicationWorker.php');
$outbox = $read(
'web/app/Services/Cloud/CloudStoragePublicationOutboxService.php'
);
$replicaWorker = $read(
'web/app/Services/Cloud/CloudStorageReplicaWorkerService.php'
);
$heartbeat = $read('web/app/Services/Cloud/CloudRuntimeHeartbeat.php');
$runtime = $read('web/app/Services/Cloud/CloudRuntimeHealthService.php');
$unit = $read(
'deploy/virt-cloud-storage-publication-worker.service'
);
$deployment = $read('docs/DEPLOYMENT.md');
$repairPosition = strpos(
$replicaWorker,
'CloudStorageReplicaRepairService'
);
$claimPosition = strpos($replicaWorker, 'claimReplicaWork');
$assert(str_contains($xcat, "'cloudStoragePublicationWorker'")
&& str_contains($xcat, 'CloudStoragePublicationWorker::run')
&& str_contains(
$xcat,
'php xcat cloudStoragePublicationWorker [--once]'
), 'Publication worker is not reachable through xcat and its help.');
$assert(str_contains($command, 'CloudStoragePublicationOutboxService')
&& !str_contains($command, 'CloudStorageReplicaWorkerService')
&& str_contains($command, "in_array('--once'")
&& str_contains($command, "'storage_publication'")
&& str_contains($command, "busy('storage_publication_tick')")
&& str_contains($command, "error('storage_publication_failed')")
&& str_contains($command, '$unresolved = true;')
&& str_contains($command, 'elseif (!$unresolved)')
&& str_contains($command, 'usleep(1000000)'),
'Publication worker lost its isolated loop, heartbeat, warning memory, '
. 'or bounded idle polling.');
$assert(str_contains($outbox, "where('state', 'ready_to_finalize')")
&& str_contains($outbox, "->whereNull('backup_id')")
&& str_contains($outbox, 'CloudTemplateStoragePublicationConsumer')
&& str_contains($outbox, 'finalizeReadyOnly'),
'Publication worker can consume backups or lost its template/rescue '
. 'owner boundary.');
$assert(!str_contains($replicaWorker, 'CloudStoragePublicationOutboxService')
&& !str_contains($replicaWorker, 'storage_publication')
&& is_int($repairPosition) && is_int($claimPosition)
&& $repairPosition < $claimPosition,
'Replica worker retained publication work or lost repair-before-copy '
. 'ordering.');
$assert(str_contains($heartbeat, "'storage_publication' => [")
&& str_contains($heartbeat, "'storage_publication_tick'")
&& str_contains($heartbeat, "'storage_publication_failed'")
&& str_contains(
$runtime,
"'storage_publication' => '对象存储业务发布'"
), 'Publication runtime heartbeat is not independently observable.');
$assert(str_contains(
$unit,
'ExecStart=/usr/bin/php /home/virt-cloud/web/xcat '
. 'cloudStoragePublicationWorker'
)
&& str_contains($unit, 'NoNewPrivileges=true')
&& str_contains($unit, 'PrivateTmp=true')
&& str_contains($unit, 'ProtectSystem=strict')
&& str_contains($unit, 'LimitCORE=0')
&& str_contains($unit, 'ReadWritePaths=-/home/virt-cloud/web/storage')
&& !str_contains($unit, '%i'),
'Publication systemd unit lost single-instance execution or hardening.');
$assert(str_contains(
$deployment,
'cp /home/virt-cloud/deploy/'
. 'virt-cloud-storage-publication-worker.service'
)
&& str_contains(
$deployment,
'virt-cloud-storage-publication-worker.service'
)
&& str_contains(
$deployment,
'-u virt-cloud-storage-publication-worker.service'
)
&& str_contains($deployment, 'ready_to_finalize')
&& str_contains($deployment, '普通云备份仍由原同步流程最终化'),
'Publication worker deployment, journal, or missing-service impact is '
. 'not documented.');
echo "Storage publication worker contract tests passed.\n";

@ -274,9 +274,15 @@ $assert((int) $replacement->backend_id === 4
$workerSource = file_get_contents(dirname(__DIR__)
. '/app/Services/Cloud/CloudStorageReplicaWorkerService.php');
$repairPosition = is_string($workerSource)
? strpos($workerSource, 'CloudStorageReplicaRepairService') : false;
$claimPosition = is_string($workerSource)
? strpos($workerSource, 'claimReplicaWork') : false;
$assert(is_string($workerSource)
&& str_contains($workerSource, 'CloudStorageReplicaRepairService')
&& str_contains($workerSource, 'storage_replica_repair_tick_failed'),
&& str_contains($workerSource, 'storage_replica_repair_tick_failed')
&& is_int($repairPosition) && is_int($claimPosition)
&& $repairPosition < $claimPosition,
'Replica worker does not consume the bounded repair queue.');
echo "Storage replica repair behavior passed.\n";

@ -45,8 +45,11 @@ $assert(str_contains($unit,
$assert(str_contains($worker, "'object-replica-verify-v1'")
&& str_contains($worker, 'presignedVersionedGet')
&& str_contains($worker, 'markReplicaUnknown')
&& !str_contains($worker, 'CloudStoragePublicationOutboxService')
&& !str_contains($worker, 'storage_publication')
&& !str_contains($worker, "result_json ="),
'Worker lost exact-read reconciliation or bypasses sanitized persistence.');
'Worker lost exact-read reconciliation, retained publication work, or '
. 'bypasses sanitized persistence.');
$assert(str_contains($worker, 'findMultipartUploadsByExactObjectKey')
&& str_contains($worker, 'resumeMultipartUpload')
&& str_contains($worker, 'operation_available_at')

Loading…
Cancel
Save