You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
307 lines
15 KiB
PHP
307 lines
15 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud {
|
|
// The database and service are real; only the private HTTP boundary is fake.
|
|
final class CloudEngineClient
|
|
{
|
|
public static array $requests = [];
|
|
public static $respond;
|
|
|
|
public function __construct(...$arguments)
|
|
{
|
|
}
|
|
|
|
public function vpc(string $method, string $path, array $actor,
|
|
?array $body, ?string $operationId): array
|
|
{
|
|
self::$requests[] = compact('method', 'path', 'actor', 'body', 'operationId');
|
|
return (self::$respond)($body);
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
use App\Services\Cloud\CloudEngineClient;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudVpcService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
require __DIR__ . '/CloudVpcMemberReuseTest.php';
|
|
if (!isset($capsule)) {
|
|
return;
|
|
}
|
|
|
|
$failOperation = static function (array $vpc, string $kind): object {
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $vpc['operation_id'])
|
|
->update(['kind' => $kind, 'status' => 'failed', 'lease_until' => 0,
|
|
'error_code' => 'vpc_fabric_not_ready']);
|
|
Capsule::table('cloud_vpcs')->where('id', $vpc['id'])->update([
|
|
'state' => 'degraded', 'sync_state' => 'failed',
|
|
'last_error_code' => 'vpc_fabric_not_ready',
|
|
]);
|
|
return Capsule::table('cloud_vpc_operations')
|
|
->where('operation_id', $vpc['operation_id'])->first();
|
|
};
|
|
CloudEngineClient::$respond = static function (): array {
|
|
throw new CloudError('引擎结果暂不可核对', 502);
|
|
};
|
|
foreach (['vpc_create', 'member_attach', 'member_detach'] as $index => $kind) {
|
|
$id = 920 + $index;
|
|
$vpc = $seedVpc($id, '10.92.' . $index . '.0/24',
|
|
'10.92.' . $index . '.0/24', 10);
|
|
$before = $failOperation($vpc, $kind);
|
|
$expectCloudError(static fn () => CloudVpcService::reconcile(
|
|
$vpc['vpc_key'], $actor, false
|
|
), 409);
|
|
$expectCloudError(static fn () => CloudVpcService::reconcile(
|
|
$vpc['vpc_key'], $actor, true
|
|
), 409);
|
|
$expectCloudError(static fn () => CloudVpcService::reconcile(
|
|
$vpc['vpc_key'], $actor, true, true
|
|
), 502);
|
|
$request = end(CloudEngineClient::$requests);
|
|
$after = Capsule::table('cloud_vpc_operations')
|
|
->where('operation_id', $vpc['operation_id'])->first();
|
|
$assert($request['operationId'] === $vpc['operation_id']
|
|
&& ($request['actor']['vpc_reconcile_explicit'] ?? null) === true
|
|
&& $request['body'] === json_decode($before->payload_json, true)
|
|
&& $after->payload_json === $before->payload_json
|
|
&& $after->status === 'uncertain'
|
|
&& (int) $after->attempts === (int) $before->attempts + 1
|
|
&& (int) Capsule::table('cloud_vpcs')->where('id', $id)
|
|
->value('desired_revision') === 1
|
|
&& Capsule::table('cloud_vpc_operations')->where('vpc_id', $id)->count() === 1,
|
|
'Explicit failed ' . $kind . ' retry changed identity or treated transport ambiguity as failed.');
|
|
}
|
|
|
|
// Even a definite refusal of the next request cannot settle the previous
|
|
// in-flight/partially observed attempt, and must not authorize IP release.
|
|
CloudEngineClient::$respond = static function (): array {
|
|
throw new CloudError('请先关机后核对', 409, 'vpc_private_remove_requires_stopped');
|
|
};
|
|
foreach (['uncertain', 'running', 'waiting_observed'] as $pendingStatus) {
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $vpc['operation_id'])
|
|
->update(['status' => $pendingStatus, 'lease_until' => 0]);
|
|
$expectCloudError(static fn () => CloudVpcService::reconcile(
|
|
$vpc['vpc_key'], $actor, true, true
|
|
), 409);
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $vpc['operation_id'])
|
|
->value('status') === 'uncertain'
|
|
&& Capsule::table('cloud_vpcs')->where('id', $vpc['id'])->value('sync_state') === 'uncertain',
|
|
'A retry rejection incorrectly made the previous remote outcome definitive.');
|
|
}
|
|
|
|
// A successful retry still requires the full exact host/member observation.
|
|
$attached = $seedVpc(930, '10.93.0.0/24', '10.93.0.0/24', 10,
|
|
'active', 'active', [
|
|
'member_row_id' => 93001, 'instance_id' => 93001, 'private_ip' => '10.93.0.1',
|
|
]);
|
|
$seedInstance(93001, 10);
|
|
$seedPair(93001, 193001, $attached, 93001, 10, '10.93.0.1', 'applying');
|
|
$operation = $failOperation($attached, 'member_attach');
|
|
CloudEngineClient::$respond = static fn (array $body): array => [
|
|
'operation_id' => $body['operation_id'], 'vpc_id' => $body['vpc_id'],
|
|
'desired_revision' => $body['desired_revision'],
|
|
'desired_digest' => $body['desired_digest'],
|
|
'observed' => json_decode($operation->observed_json, true),
|
|
];
|
|
CloudVpcService::reconcile($attached['vpc_key'], $actor, true, true);
|
|
$assert(Capsule::table('cloud_vpc_operations')->where('operation_id',
|
|
$attached['operation_id'])->value('status') === 'succeeded'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 93001)->value('state') === 'active',
|
|
'Exact readback could not complete the original failed attachment.');
|
|
|
|
CloudVpcService::detachMember($attached['vpc_key'], $keyFor('member', 93001),
|
|
$actor, ['request_key' => 'retry-detach-93001']);
|
|
$detachId = Capsule::table('cloud_vpcs')->where('id', 930)->value('last_operation_id');
|
|
$detaching = $attached;
|
|
$detaching['operation_id'] = $detachId;
|
|
foreach (['wrong_operation', 'wrong_digest', 'missing_cleanup'] as $invalid) {
|
|
$operation = $failOperation($detaching, 'member_detach');
|
|
CloudEngineClient::$respond = static function (array $body) use ($invalid): array {
|
|
return [
|
|
'operation_id' => $invalid === 'wrong_operation'
|
|
? str_repeat('f', 32) : $body['operation_id'],
|
|
'vpc_id' => $body['vpc_id'], 'desired_revision' => $body['desired_revision'],
|
|
'desired_digest' => $invalid === 'wrong_digest'
|
|
? str_repeat('f', 64) : $body['desired_digest'],
|
|
'observed' => ['state' => 'active', 'revision' => $body['desired_revision'],
|
|
'digest' => $body['desired_digest'], 'hosts' => [],
|
|
'fabric_gate' => ['active' => true],
|
|
'private_dataplane_verified' => true],
|
|
];
|
|
};
|
|
try {
|
|
CloudVpcService::reconcile($attached['vpc_key'], $actor, true, true);
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus >= 500, 'Invalid engine proof was reported as a definite failure.');
|
|
}
|
|
$assert(Capsule::table('cloud_vpc_members')->where('id', 93001)->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 193001)->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $detachId)
|
|
->value('status') !== 'succeeded',
|
|
'A failed detach retry released membership/IP without exact cleanup evidence.');
|
|
}
|
|
|
|
// Real stopped prerequisites carry an exact target but deliberately lack
|
|
// configuration proof. Keep the controlled reason through host/root/page
|
|
// projection without turning it into removal or readiness evidence.
|
|
foreach (['applying', 'degraded'] as $hostState) {
|
|
$failOperation($detaching, 'member_detach');
|
|
CloudEngineClient::$respond = static function (array $body) use ($hostState): array {
|
|
return [
|
|
'operation_id' => $body['operation_id'], 'vpc_id' => $body['vpc_id'],
|
|
'desired_revision' => $body['desired_revision'],
|
|
'desired_digest' => $body['desired_digest'],
|
|
'observed' => [
|
|
'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'state' => 'degraded', 'next_stage_allowed' => false,
|
|
'private_dataplane_verified' => false,
|
|
'fabric_gate' => ['active' => true],
|
|
'hosts' => ['10' => [
|
|
'vpc_id' => $body['vpc_id'], 'host_id' => '10',
|
|
'desired_revision' => $body['desired_revision'],
|
|
'desired_digest' => $body['desired_digest'],
|
|
'observed_revision' => 0, 'observed_digest' => null,
|
|
'state' => $hostState, 'transport_verified' => true,
|
|
'private_dataplane_verified' => false, 'config_verified' => false,
|
|
'last_error_code' => 'vpc_private_remove_requires_stopped',
|
|
]],
|
|
],
|
|
];
|
|
};
|
|
$view = CloudVpcService::reconcile($attached['vpc_key'], $actor, true, true);
|
|
$assert($view['vpc']['last_error_code'] === 'vpc_private_remove_requires_stopped'
|
|
&& $view['vpc']['usable'] === false
|
|
&& Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 930)
|
|
->value('last_error_code') === 'vpc_private_remove_requires_stopped'
|
|
&& Capsule::table('cloud_vpc_members')->where('id', 93001)->value('state') === 'deleting'
|
|
&& Capsule::table('cloud_vpc_private_ips')->where('id', 193001)->value('state') === 'deleting',
|
|
'Stopped cleanup guidance was lost or incorrectly released the tombstone.');
|
|
}
|
|
|
|
$joining = $seedVpc(950, '10.95.0.0/24', '10.95.0.0/24', 10);
|
|
$seedInstance(95001, 10);
|
|
CloudVpcService::attachMember($joining['vpc_key'], $actor,
|
|
['instance_id' => 95001, 'request_key' => 'retry-joining-95001']);
|
|
$pendingResponse = CloudEngineClient::$respond;
|
|
foreach (['exact', 'wrong_host', 'wrong_digest', 'unreviewed'] as $diagnostic) {
|
|
CloudEngineClient::$respond = static function (array $body) use (
|
|
$pendingResponse, $diagnostic
|
|
): array {
|
|
$response = $pendingResponse($body);
|
|
$response['observed']['hosts']['10']['last_error_code'] =
|
|
$diagnostic === 'unreviewed' ? 'unreviewed_engine_message'
|
|
: 'vpc_private_guest_restart_required';
|
|
if ($diagnostic === 'wrong_host') {
|
|
$response['observed']['hosts']['10']['host_id'] = '999';
|
|
} elseif ($diagnostic === 'wrong_digest') {
|
|
$response['observed']['hosts']['10']['desired_digest'] = str_repeat('f', 64);
|
|
}
|
|
return $response;
|
|
};
|
|
$view = CloudVpcService::reconcile($joining['vpc_key'], $actor, true, true);
|
|
$assert(($view['vpc']['last_error_code'] === 'vpc_private_guest_restart_required')
|
|
=== ($diagnostic === 'exact')
|
|
&& $view['vpc']['usable'] === false
|
|
&& (int) Capsule::table('cloud_vpcs')->where('id', 950)
|
|
->value('observed_revision') < 2,
|
|
'Unapplied attach guidance was lost, unbound, or incorrectly became readiness proof.');
|
|
}
|
|
|
|
$corrupt = $seedVpc(940, '10.94.0.0/24', '10.94.0.0/24', 10);
|
|
$original = $failOperation($corrupt, 'member_attach');
|
|
foreach (['operation_id', 'vpc_id', 'desired_revision', 'desired_digest', 'cidr'] as $field) {
|
|
$payload = json_decode($original->payload_json, true);
|
|
$payload[$field] = match ($field) {
|
|
'desired_revision' => 2,
|
|
'desired_digest' => str_repeat('f', 64),
|
|
'cidr' => '10.95.0.0/24',
|
|
default => str_repeat('f', 32),
|
|
};
|
|
$failOperation($corrupt, 'member_attach');
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $corrupt['operation_id'])
|
|
->update(['payload_json' => json_encode($payload)]);
|
|
$requestCount = count(CloudEngineClient::$requests);
|
|
CloudVpcService::reconcile($corrupt['vpc_key'], $actor, true, true);
|
|
$assert(count(CloudEngineClient::$requests) === $requestCount
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $corrupt['operation_id'])
|
|
->value('error_code') === 'vpc_operation_corrupt',
|
|
'A mismatched immutable ' . $field . ' was sent back to the engine.');
|
|
}
|
|
|
|
// A worker can select a waiting candidate just before a different request
|
|
// installs a newer, definitively failed target. Its deferred-observation flag
|
|
// must not inherit the user's permission to retry that new failed operation.
|
|
Capsule::table('cloud_vpc_operations')->update(['status' => 'superseded']);
|
|
$candidate = $seedVpc(960, '10.96.0.0/24', '10.96.0.0/24', 10);
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $candidate['operation_id'])
|
|
->update(['status' => 'waiting_observed', 'updated_at' => time() - 60]);
|
|
$replacement = (array) Capsule::table('cloud_vpc_operations')
|
|
->where('operation_id', $candidate['operation_id'])->first();
|
|
unset($replacement['id']);
|
|
$replacementPayload = json_decode($replacement['payload_json'], true);
|
|
unset($replacementPayload['operation_id'], $replacementPayload['desired_digest']);
|
|
$replacementPayload['desired_revision'] = 2;
|
|
$replacementDigest = hash('sha256', $canonicalJson($replacementPayload));
|
|
$replacementId = $keyFor('replacement-operation', 960);
|
|
$replacementPayload['operation_id'] = $replacementId;
|
|
$replacementPayload['desired_digest'] = $replacementDigest;
|
|
$replacement = array_replace($replacement, [
|
|
'operation_id' => $replacementId, 'desired_revision' => 2,
|
|
'idempotency_key' => 'replacement-operation-960', 'status' => 'failed',
|
|
'payload_json' => json_encode($replacementPayload), 'attempts' => 0,
|
|
'lease_token' => null, 'lease_until' => 0, 'error_code' => 'vpc_fabric_not_ready',
|
|
]);
|
|
$transactionCount = 0;
|
|
$replaceTarget = true;
|
|
$capsule->getConnection('default')->beforeStartingTransaction(static function () use (
|
|
&$transactionCount, &$replaceTarget, $candidate, $replacement, $replacementId,
|
|
$replacementDigest
|
|
): void {
|
|
if (!$replaceTarget || ++$transactionCount !== 2) {
|
|
return;
|
|
}
|
|
$replaceTarget = false;
|
|
Capsule::table('cloud_vpc_operations')->where('operation_id', $candidate['operation_id'])
|
|
->update(['status' => 'superseded']);
|
|
Capsule::table('cloud_vpc_operations')->insert($replacement);
|
|
Capsule::table('cloud_vpcs')->where('id', 960)->update([
|
|
'last_operation_id' => $replacementId, 'desired_revision' => 2,
|
|
'desired_digest' => $replacementDigest, 'state' => 'degraded', 'sync_state' => 'failed',
|
|
]);
|
|
});
|
|
$requestCount = count(CloudEngineClient::$requests);
|
|
$assert((new \App\Services\Cloud\CloudVpcWorkerService())->tick()
|
|
&& $transactionCount === 2
|
|
&& count(CloudEngineClient::$requests) === $requestCount
|
|
&& Capsule::table('cloud_vpc_operations')->where('operation_id', $replacementId)
|
|
->value('status') === 'failed'
|
|
&& (int) Capsule::table('cloud_vpc_operations')->where('operation_id', $replacementId)
|
|
->value('attempts') === 0,
|
|
'A deferred worker candidate automatically retried a newer failed target.');
|
|
|
|
// Only the explicit user HTTP controller authorizes retrying the failed row.
|
|
$httpRequest = new class($actor) {
|
|
public function __construct(private array $actor) {}
|
|
public function getAttribute(string $name): array { return $this->actor; }
|
|
};
|
|
$httpResponse = new class {
|
|
public function withJson(array $body, int $status): self { return $this; }
|
|
public function withHeader(string $name, string $value): self { return $this; }
|
|
};
|
|
CloudEngineClient::$respond = static function (): array {
|
|
throw new CloudError('引擎结果暂不可核对', 502);
|
|
};
|
|
$expectCloudError(static fn () => (new \App\Controllers\CloudVpcController())->reconcile(
|
|
$httpRequest, $httpResponse, ['vpc_key' => $candidate['vpc_key']]
|
|
), 502);
|
|
$assert(count(CloudEngineClient::$requests) === $requestCount + 1
|
|
&& end(CloudEngineClient::$requests)['operationId'] === $replacementId
|
|
&& (end(CloudEngineClient::$requests)['actor']['vpc_reconcile_explicit'] ?? null) === true
|
|
&& end(CloudEngineClient::$requests)['body'] === $replacementPayload,
|
|
'The explicit user reconcile did not retry the exact failed target.');
|
|
|
|
echo "VPC failed-operation reconcile database regressions passed.\n";
|
|
}
|