You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudVpcFailedReconcileTest...

307 lines
15 KiB
PHP

<?php
namespace App\Services\Cloud {
// The database and service are real; only the private HTTP boundary is fake.
final class CloudEngineClient
{
public static array $requests = [];
public static $respond;
public function __construct(...$arguments)
{
}
public function vpc(string $method, string $path, array $actor,
?array $body, ?string $operationId): array
{
self::$requests[] = compact('method', 'path', 'actor', 'body', 'operationId');
return (self::$respond)($body);
}
}
}
namespace {
use App\Services\Cloud\CloudEngineClient;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudVpcService;
use Illuminate\Database\Capsule\Manager as Capsule;
require __DIR__ . '/CloudVpcMemberReuseTest.php';
if (!isset($capsule)) {
return;
}
$failOperation = static function (array $vpc, string $kind): object {
Capsule::table('cloud_vpc_operations')->where('operation_id', $vpc['operation_id'])
->update(['kind' => $kind, 'status' => 'failed', 'lease_until' => 0,
'error_code' => 'vpc_fabric_not_ready']);
Capsule::table('cloud_vpcs')->where('id', $vpc['id'])->update([
'state' => 'degraded', 'sync_state' => 'failed',
'last_error_code' => 'vpc_fabric_not_ready',
]);
return Capsule::table('cloud_vpc_operations')
->where('operation_id', $vpc['operation_id'])->first();
};
CloudEngineClient::$respond = static function (): array {
throw new CloudError('引擎结果暂不可核对', 502);
};
foreach (['vpc_create', 'member_attach', 'member_detach'] as $index => $kind) {
$id = 920 + $index;
$vpc = $seedVpc($id, '10.92.' . $index . '.0/24',
'10.92.' . $index . '.0/24', 10);
$before = $failOperation($vpc, $kind);
$expectCloudError(static fn () => CloudVpcService::reconcile(
$vpc['vpc_key'], $actor, false
), 409);
$expectCloudError(static fn () => CloudVpcService::reconcile(
$vpc['vpc_key'], $actor, true
), 409);
$expectCloudError(static fn () => CloudVpcService::reconcile(
$vpc['vpc_key'], $actor, true, true
), 502);
$request = end(CloudEngineClient::$requests);
$after = Capsule::table('cloud_vpc_operations')
->where('operation_id', $vpc['operation_id'])->first();
$assert($request['operationId'] === $vpc['operation_id']
&& ($request['actor']['vpc_reconcile_explicit'] ?? null) === true
&& $request['body'] === json_decode($before->payload_json, true)
&& $after->payload_json === $before->payload_json
&& $after->status === 'uncertain'
&& (int) $after->attempts === (int) $before->attempts + 1
&& (int) Capsule::table('cloud_vpcs')->where('id', $id)
->value('desired_revision') === 1
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', $id)->count() === 1,
'Explicit failed ' . $kind . ' retry changed identity or treated transport ambiguity as failed.');
}
// Even a definite refusal of the next request cannot settle the previous
// in-flight/partially observed attempt, and must not authorize IP release.
CloudEngineClient::$respond = static function (): array {
throw new CloudError('请先关机后核对', 409, 'vpc_private_remove_requires_stopped');
};
foreach (['uncertain', 'running', 'waiting_observed'] as $pendingStatus) {
Capsule::table('cloud_vpc_operations')->where('operation_id', $vpc['operation_id'])
->update(['status' => $pendingStatus, 'lease_until' => 0]);
$expectCloudError(static fn () => CloudVpcService::reconcile(
$vpc['vpc_key'], $actor, true, true
), 409);
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $vpc['operation_id'])
->value('status') === 'uncertain'
&& Capsule::table('cloud_vpcs')->where('id', $vpc['id'])->value('sync_state') === 'uncertain',
'A retry rejection incorrectly made the previous remote outcome definitive.');
}
// A successful retry still requires the full exact host/member observation.
$attached = $seedVpc(930, '10.93.0.0/24', '10.93.0.0/24', 10,
'active', 'active', [
'member_row_id' => 93001, 'instance_id' => 93001, 'private_ip' => '10.93.0.1',
]);
$seedInstance(93001, 10);
$seedPair(93001, 193001, $attached, 93001, 10, '10.93.0.1', 'applying');
$operation = $failOperation($attached, 'member_attach');
CloudEngineClient::$respond = static fn (array $body): array => [
'operation_id' => $body['operation_id'], 'vpc_id' => $body['vpc_id'],
'desired_revision' => $body['desired_revision'],
'desired_digest' => $body['desired_digest'],
'observed' => json_decode($operation->observed_json, true),
];
CloudVpcService::reconcile($attached['vpc_key'], $actor, true, true);
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id',
$attached['operation_id'])->value('status') === 'succeeded'
&& Capsule::table('cloud_vpc_members')->where('id', 93001)->value('state') === 'active',
'Exact readback could not complete the original failed attachment.');
CloudVpcService::detachMember($attached['vpc_key'], $keyFor('member', 93001),
$actor, ['request_key' => 'retry-detach-93001']);
$detachId = Capsule::table('cloud_vpcs')->where('id', 930)->value('last_operation_id');
$detaching = $attached;
$detaching['operation_id'] = $detachId;
foreach (['wrong_operation', 'wrong_digest', 'missing_cleanup'] as $invalid) {
$operation = $failOperation($detaching, 'member_detach');
CloudEngineClient::$respond = static function (array $body) use ($invalid): array {
return [
'operation_id' => $invalid === 'wrong_operation'
? str_repeat('f', 32) : $body['operation_id'],
'vpc_id' => $body['vpc_id'], 'desired_revision' => $body['desired_revision'],
'desired_digest' => $invalid === 'wrong_digest'
? str_repeat('f', 64) : $body['desired_digest'],
'observed' => ['state' => 'active', 'revision' => $body['desired_revision'],
'digest' => $body['desired_digest'], 'hosts' => [],
'fabric_gate' => ['active' => true],
'private_dataplane_verified' => true],
];
};
try {
CloudVpcService::reconcile($attached['vpc_key'], $actor, true, true);
} catch (CloudError $error) {
$assert($error->httpStatus >= 500, 'Invalid engine proof was reported as a definite failure.');
}
$assert(Capsule::table('cloud_vpc_members')->where('id', 93001)->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 193001)->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $detachId)
->value('status') !== 'succeeded',
'A failed detach retry released membership/IP without exact cleanup evidence.');
}
// Real stopped prerequisites carry an exact target but deliberately lack
// configuration proof. Keep the controlled reason through host/root/page
// projection without turning it into removal or readiness evidence.
foreach (['applying', 'degraded'] as $hostState) {
$failOperation($detaching, 'member_detach');
CloudEngineClient::$respond = static function (array $body) use ($hostState): array {
return [
'operation_id' => $body['operation_id'], 'vpc_id' => $body['vpc_id'],
'desired_revision' => $body['desired_revision'],
'desired_digest' => $body['desired_digest'],
'observed' => [
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
'state' => 'degraded', 'next_stage_allowed' => false,
'private_dataplane_verified' => false,
'fabric_gate' => ['active' => true],
'hosts' => ['10' => [
'vpc_id' => $body['vpc_id'], 'host_id' => '10',
'desired_revision' => $body['desired_revision'],
'desired_digest' => $body['desired_digest'],
'observed_revision' => 0, 'observed_digest' => null,
'state' => $hostState, 'transport_verified' => true,
'private_dataplane_verified' => false, 'config_verified' => false,
'last_error_code' => 'vpc_private_remove_requires_stopped',
]],
],
];
};
$view = CloudVpcService::reconcile($attached['vpc_key'], $actor, true, true);
$assert($view['vpc']['last_error_code'] === 'vpc_private_remove_requires_stopped'
&& $view['vpc']['usable'] === false
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 930)
->value('last_error_code') === 'vpc_private_remove_requires_stopped'
&& Capsule::table('cloud_vpc_members')->where('id', 93001)->value('state') === 'deleting'
&& Capsule::table('cloud_vpc_private_ips')->where('id', 193001)->value('state') === 'deleting',
'Stopped cleanup guidance was lost or incorrectly released the tombstone.');
}
$joining = $seedVpc(950, '10.95.0.0/24', '10.95.0.0/24', 10);
$seedInstance(95001, 10);
CloudVpcService::attachMember($joining['vpc_key'], $actor,
['instance_id' => 95001, 'request_key' => 'retry-joining-95001']);
$pendingResponse = CloudEngineClient::$respond;
foreach (['exact', 'wrong_host', 'wrong_digest', 'unreviewed'] as $diagnostic) {
CloudEngineClient::$respond = static function (array $body) use (
$pendingResponse, $diagnostic
): array {
$response = $pendingResponse($body);
$response['observed']['hosts']['10']['last_error_code'] =
$diagnostic === 'unreviewed' ? 'unreviewed_engine_message'
: 'vpc_private_guest_restart_required';
if ($diagnostic === 'wrong_host') {
$response['observed']['hosts']['10']['host_id'] = '999';
} elseif ($diagnostic === 'wrong_digest') {
$response['observed']['hosts']['10']['desired_digest'] = str_repeat('f', 64);
}
return $response;
};
$view = CloudVpcService::reconcile($joining['vpc_key'], $actor, true, true);
$assert(($view['vpc']['last_error_code'] === 'vpc_private_guest_restart_required')
=== ($diagnostic === 'exact')
&& $view['vpc']['usable'] === false
&& (int) Capsule::table('cloud_vpcs')->where('id', 950)
->value('observed_revision') < 2,
'Unapplied attach guidance was lost, unbound, or incorrectly became readiness proof.');
}
$corrupt = $seedVpc(940, '10.94.0.0/24', '10.94.0.0/24', 10);
$original = $failOperation($corrupt, 'member_attach');
foreach (['operation_id', 'vpc_id', 'desired_revision', 'desired_digest', 'cidr'] as $field) {
$payload = json_decode($original->payload_json, true);
$payload[$field] = match ($field) {
'desired_revision' => 2,
'desired_digest' => str_repeat('f', 64),
'cidr' => '10.95.0.0/24',
default => str_repeat('f', 32),
};
$failOperation($corrupt, 'member_attach');
Capsule::table('cloud_vpc_operations')->where('operation_id', $corrupt['operation_id'])
->update(['payload_json' => json_encode($payload)]);
$requestCount = count(CloudEngineClient::$requests);
CloudVpcService::reconcile($corrupt['vpc_key'], $actor, true, true);
$assert(count(CloudEngineClient::$requests) === $requestCount
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $corrupt['operation_id'])
->value('error_code') === 'vpc_operation_corrupt',
'A mismatched immutable ' . $field . ' was sent back to the engine.');
}
// A worker can select a waiting candidate just before a different request
// installs a newer, definitively failed target. Its deferred-observation flag
// must not inherit the user's permission to retry that new failed operation.
Capsule::table('cloud_vpc_operations')->update(['status' => 'superseded']);
$candidate = $seedVpc(960, '10.96.0.0/24', '10.96.0.0/24', 10);
Capsule::table('cloud_vpc_operations')->where('operation_id', $candidate['operation_id'])
->update(['status' => 'waiting_observed', 'updated_at' => time() - 60]);
$replacement = (array) Capsule::table('cloud_vpc_operations')
->where('operation_id', $candidate['operation_id'])->first();
unset($replacement['id']);
$replacementPayload = json_decode($replacement['payload_json'], true);
unset($replacementPayload['operation_id'], $replacementPayload['desired_digest']);
$replacementPayload['desired_revision'] = 2;
$replacementDigest = hash('sha256', $canonicalJson($replacementPayload));
$replacementId = $keyFor('replacement-operation', 960);
$replacementPayload['operation_id'] = $replacementId;
$replacementPayload['desired_digest'] = $replacementDigest;
$replacement = array_replace($replacement, [
'operation_id' => $replacementId, 'desired_revision' => 2,
'idempotency_key' => 'replacement-operation-960', 'status' => 'failed',
'payload_json' => json_encode($replacementPayload), 'attempts' => 0,
'lease_token' => null, 'lease_until' => 0, 'error_code' => 'vpc_fabric_not_ready',
]);
$transactionCount = 0;
$replaceTarget = true;
$capsule->getConnection('default')->beforeStartingTransaction(static function () use (
&$transactionCount, &$replaceTarget, $candidate, $replacement, $replacementId,
$replacementDigest
): void {
if (!$replaceTarget || ++$transactionCount !== 2) {
return;
}
$replaceTarget = false;
Capsule::table('cloud_vpc_operations')->where('operation_id', $candidate['operation_id'])
->update(['status' => 'superseded']);
Capsule::table('cloud_vpc_operations')->insert($replacement);
Capsule::table('cloud_vpcs')->where('id', 960)->update([
'last_operation_id' => $replacementId, 'desired_revision' => 2,
'desired_digest' => $replacementDigest, 'state' => 'degraded', 'sync_state' => 'failed',
]);
});
$requestCount = count(CloudEngineClient::$requests);
$assert((new \App\Services\Cloud\CloudVpcWorkerService())->tick()
&& $transactionCount === 2
&& count(CloudEngineClient::$requests) === $requestCount
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $replacementId)
->value('status') === 'failed'
&& (int) Capsule::table('cloud_vpc_operations')->where('operation_id', $replacementId)
->value('attempts') === 0,
'A deferred worker candidate automatically retried a newer failed target.');
// Only the explicit user HTTP controller authorizes retrying the failed row.
$httpRequest = new class($actor) {
public function __construct(private array $actor) {}
public function getAttribute(string $name): array { return $this->actor; }
};
$httpResponse = new class {
public function withJson(array $body, int $status): self { return $this; }
public function withHeader(string $name, string $value): self { return $this; }
};
CloudEngineClient::$respond = static function (): array {
throw new CloudError('引擎结果暂不可核对', 502);
};
$expectCloudError(static fn () => (new \App\Controllers\CloudVpcController())->reconcile(
$httpRequest, $httpResponse, ['vpc_key' => $candidate['vpc_key']]
), 502);
$assert(count(CloudEngineClient::$requests) === $requestCount + 1
&& end(CloudEngineClient::$requests)['operationId'] === $replacementId
&& (end(CloudEngineClient::$requests)['actor']['vpc_reconcile_explicit'] ?? null) === true
&& end(CloudEngineClient::$requests)['body'] === $replacementPayload,
'The explicit user reconcile did not retry the exact failed target.');
echo "VPC failed-operation reconcile database regressions passed.\n";
}