feat(storage): split publication outbox worker
parent
0757e98c9d
commit
5c4f269442
@ -0,0 +1,25 @@
|
||||
[Unit]
|
||||
Description=Virt Cloud object storage publication worker
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=apache
|
||||
Group=apache
|
||||
WorkingDirectory=/home/virt-cloud/web
|
||||
ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudStoragePublicationWorker
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
LimitCORE=0
|
||||
UMask=0077
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=false
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=-/home/virt-cloud/runtime/php
|
||||
ReadWritePaths=-/home/virt-cloud/web/storage
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@ -0,0 +1,68 @@
|
||||
<?php
|
||||
|
||||
namespace App\Command;
|
||||
|
||||
use App\Services\Cloud\CloudRuntimeHeartbeat;
|
||||
use App\Services\Cloud\CloudStoragePublicationOutboxService;
|
||||
|
||||
/** Finalizes protected template and rescue publications outside copy workers. */
|
||||
final class CloudStoragePublicationWorker
|
||||
{
|
||||
public static function run(
|
||||
array $argv = [],
|
||||
?CloudRuntimeHeartbeat $heartbeat = null
|
||||
): int {
|
||||
$once = in_array('--once', $argv, true);
|
||||
$heartbeat ??= CloudRuntimeHeartbeat::start(
|
||||
'storage_publication',
|
||||
$once
|
||||
);
|
||||
$unresolved = false;
|
||||
try {
|
||||
$worker = new CloudStoragePublicationOutboxService();
|
||||
do {
|
||||
$errors = [];
|
||||
$heartbeat->busy('storage_publication_tick');
|
||||
try {
|
||||
$worked = $worker->tick();
|
||||
$errors = $worker->observationErrors();
|
||||
} catch (\Throwable $error) {
|
||||
$errors[] = 'storage_publication_failed';
|
||||
$heartbeat->error('storage_publication_failed');
|
||||
fwrite(
|
||||
STDERR,
|
||||
'storage publication worker temporarily unavailable: '
|
||||
. get_class($error) . PHP_EOL
|
||||
);
|
||||
if ($once) {
|
||||
return 1;
|
||||
}
|
||||
$worked = false;
|
||||
}
|
||||
if ($errors === []) {
|
||||
// An idle poll proves only that the process is alive. A
|
||||
// real successful finalization is required to clear an
|
||||
// earlier warning from this worker instance.
|
||||
if ($worked) {
|
||||
$unresolved = false;
|
||||
$heartbeat->success();
|
||||
} elseif (!$unresolved) {
|
||||
$heartbeat->success();
|
||||
}
|
||||
} else {
|
||||
$unresolved = true;
|
||||
$heartbeat->error('storage_publication_failed');
|
||||
}
|
||||
if (!$once && !$worked) {
|
||||
usleep(1000000);
|
||||
}
|
||||
} while (!$once);
|
||||
} catch (\Throwable $error) {
|
||||
$heartbeat->error('storage_publication_failed');
|
||||
throw $error;
|
||||
} finally {
|
||||
$heartbeat->stopped();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,103 @@
|
||||
<?php
|
||||
|
||||
$root = dirname(__DIR__, 2);
|
||||
$read = static function (string $path) use ($root): string {
|
||||
$contents = file_get_contents($root . '/' . $path);
|
||||
if (!is_string($contents)) {
|
||||
throw new RuntimeException('Unable to read ' . $path);
|
||||
}
|
||||
return $contents;
|
||||
};
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
|
||||
$xcat = $read('web/app/Command/XCat.php');
|
||||
$command = $read('web/app/Command/CloudStoragePublicationWorker.php');
|
||||
$outbox = $read(
|
||||
'web/app/Services/Cloud/CloudStoragePublicationOutboxService.php'
|
||||
);
|
||||
$replicaWorker = $read(
|
||||
'web/app/Services/Cloud/CloudStorageReplicaWorkerService.php'
|
||||
);
|
||||
$heartbeat = $read('web/app/Services/Cloud/CloudRuntimeHeartbeat.php');
|
||||
$runtime = $read('web/app/Services/Cloud/CloudRuntimeHealthService.php');
|
||||
$unit = $read(
|
||||
'deploy/virt-cloud-storage-publication-worker.service'
|
||||
);
|
||||
$deployment = $read('docs/DEPLOYMENT.md');
|
||||
$repairPosition = strpos(
|
||||
$replicaWorker,
|
||||
'CloudStorageReplicaRepairService'
|
||||
);
|
||||
$claimPosition = strpos($replicaWorker, 'claimReplicaWork');
|
||||
|
||||
$assert(str_contains($xcat, "'cloudStoragePublicationWorker'")
|
||||
&& str_contains($xcat, 'CloudStoragePublicationWorker::run')
|
||||
&& str_contains(
|
||||
$xcat,
|
||||
'php xcat cloudStoragePublicationWorker [--once]'
|
||||
), 'Publication worker is not reachable through xcat and its help.');
|
||||
$assert(str_contains($command, 'CloudStoragePublicationOutboxService')
|
||||
&& !str_contains($command, 'CloudStorageReplicaWorkerService')
|
||||
&& str_contains($command, "in_array('--once'")
|
||||
&& str_contains($command, "'storage_publication'")
|
||||
&& str_contains($command, "busy('storage_publication_tick')")
|
||||
&& str_contains($command, "error('storage_publication_failed')")
|
||||
&& str_contains($command, '$unresolved = true;')
|
||||
&& str_contains($command, 'elseif (!$unresolved)')
|
||||
&& str_contains($command, 'usleep(1000000)'),
|
||||
'Publication worker lost its isolated loop, heartbeat, warning memory, '
|
||||
. 'or bounded idle polling.');
|
||||
$assert(str_contains($outbox, "where('state', 'ready_to_finalize')")
|
||||
&& str_contains($outbox, "->whereNull('backup_id')")
|
||||
&& str_contains($outbox, 'CloudTemplateStoragePublicationConsumer')
|
||||
&& str_contains($outbox, 'finalizeReadyOnly'),
|
||||
'Publication worker can consume backups or lost its template/rescue '
|
||||
. 'owner boundary.');
|
||||
$assert(!str_contains($replicaWorker, 'CloudStoragePublicationOutboxService')
|
||||
&& !str_contains($replicaWorker, 'storage_publication')
|
||||
&& is_int($repairPosition) && is_int($claimPosition)
|
||||
&& $repairPosition < $claimPosition,
|
||||
'Replica worker retained publication work or lost repair-before-copy '
|
||||
. 'ordering.');
|
||||
$assert(str_contains($heartbeat, "'storage_publication' => [")
|
||||
&& str_contains($heartbeat, "'storage_publication_tick'")
|
||||
&& str_contains($heartbeat, "'storage_publication_failed'")
|
||||
&& str_contains(
|
||||
$runtime,
|
||||
"'storage_publication' => '对象存储业务发布'"
|
||||
), 'Publication runtime heartbeat is not independently observable.');
|
||||
$assert(str_contains(
|
||||
$unit,
|
||||
'ExecStart=/usr/bin/php /home/virt-cloud/web/xcat '
|
||||
. 'cloudStoragePublicationWorker'
|
||||
)
|
||||
&& str_contains($unit, 'NoNewPrivileges=true')
|
||||
&& str_contains($unit, 'PrivateTmp=true')
|
||||
&& str_contains($unit, 'ProtectSystem=strict')
|
||||
&& str_contains($unit, 'LimitCORE=0')
|
||||
&& str_contains($unit, 'ReadWritePaths=-/home/virt-cloud/web/storage')
|
||||
&& !str_contains($unit, '%i'),
|
||||
'Publication systemd unit lost single-instance execution or hardening.');
|
||||
$assert(str_contains(
|
||||
$deployment,
|
||||
'cp /home/virt-cloud/deploy/'
|
||||
. 'virt-cloud-storage-publication-worker.service'
|
||||
)
|
||||
&& str_contains(
|
||||
$deployment,
|
||||
'virt-cloud-storage-publication-worker.service'
|
||||
)
|
||||
&& str_contains(
|
||||
$deployment,
|
||||
'-u virt-cloud-storage-publication-worker.service'
|
||||
)
|
||||
&& str_contains($deployment, 'ready_to_finalize')
|
||||
&& str_contains($deployment, '普通云备份仍由原同步流程最终化'),
|
||||
'Publication worker deployment, journal, or missing-service impact is '
|
||||
. 'not documented.');
|
||||
|
||||
echo "Storage publication worker contract tests passed.\n";
|
||||
Loading…
Reference in New Issue