diff --git a/web/app/Services/Cloud/CloudObjectStorageS3Client.php b/web/app/Services/Cloud/CloudObjectStorageS3Client.php index 2a7e0a56..a94d6d2d 100644 --- a/web/app/Services/Cloud/CloudObjectStorageS3Client.php +++ b/web/app/Services/Cloud/CloudObjectStorageS3Client.php @@ -5,6 +5,7 @@ namespace App\Services\Cloud; use GuzzleHttp\Client; use GuzzleHttp\ClientInterface; use GuzzleHttp\Exception\GuzzleException; +use GuzzleHttp\Psr7\Utils; /** Signed S3 control plane. VM image bytes never pass through this client. */ final class CloudObjectStorageS3Client @@ -296,12 +297,16 @@ final class CloudObjectStorageS3Client 'Invalid immutable object source file' ); } - $stream = @fopen($path, 'rb'); - if ($stream === false) { + $resource = @fopen($path, 'rb'); + if ($resource === false) { throw new CloudObjectStorageTransferError( '无法读取待保存的模板主副本。', 'object_source_unreadable' ); } + // Keep one StreamInterface owner alive until request() returns. Guzzle + // receives this same object, so close() remains idempotent even when a + // handler releases the request body before returning its response. + $body = Utils::streamFor($resource); $url = self::objectUrl($config, $objectKey); $headers = CloudS3Signer::sign( 'PUT', $url, (string) $config['region'], @@ -315,7 +320,7 @@ final class CloudObjectStorageS3Client try { $response = $this->http()->request('PUT', $url, [ 'headers' => $headers, - 'body' => $stream, + 'body' => $body, ] + self::requestOptions(0.0)); } catch (GuzzleException $exception) { throw new CloudObjectStorageTransferError( @@ -323,8 +328,16 @@ final class CloudObjectStorageS3Client 'object_put_unknown', true ); + } catch (\Throwable $exception) { + // request() was entered, so an unexpected handler/runtime error + // cannot prove that the remote write did not happen. + throw new CloudObjectStorageTransferError( + '无法确认模板主副本是否已保存。', + 'object_put_unknown', + true + ); } finally { - fclose($stream); + $body->close(); } $status = $response->getStatusCode(); if ($status === 409 || $status === 412) { @@ -349,11 +362,20 @@ final class CloudObjectStorageS3Client || preg_match('/[\x00-\x1f\x7f]/', $etag) === 1) { throw new CloudObjectStorageTransferError( '对象存储没有返回可信的模板对象标识。', - 'object_put_invalid' + 'object_put_invalid', + true ); } $versionId = trim($response->getHeaderLine('x-amz-version-id')); - self::assertVersionId($versionId); + try { + self::assertVersionId($versionId); + } catch (\InvalidArgumentException $exception) { + throw new CloudObjectStorageTransferError( + '对象存储没有返回可信的模板版本标识。', + 'object_put_invalid', + true + ); + } return [ 'size_bytes' => $sizeBytes, 'etag' => $etag, diff --git a/web/app/Services/Cloud/CloudTemplateArtifactManager.php b/web/app/Services/Cloud/CloudTemplateArtifactManager.php index b983588f..ef550073 100644 --- a/web/app/Services/Cloud/CloudTemplateArtifactManager.php +++ b/web/app/Services/Cloud/CloudTemplateArtifactManager.php @@ -78,30 +78,44 @@ final class CloudTemplateArtifactManager throw new CloudError('模板主副本不在可发布状态', 409); } $runtime = self::runtime($artifact); + $s3 = $client ?? new CloudObjectStorageS3Client(); try { - $identity = ($client ?? new CloudObjectStorageS3Client()) - ->putImmutableFile( - $runtime['config'], - $runtime['credentials'], - (string) $artifact->object_key, - (string) $file['path'], - (int) $file['size_bytes'], - (string) $file['sha256'] - ); + $identity = $s3->putImmutableFile( + $runtime['config'], + $runtime['credentials'], + (string) $artifact->object_key, + (string) $file['path'], + (int) $file['size_bytes'], + (string) $file['sha256'] + ); } catch (CloudObjectStorageTransferError $error) { - self::recordPublishFailure( - (string) $template->id, - $error->publicCode, - $error->effectMayHaveApplied + if (!$error->effectMayHaveApplied) { + self::recordPublishFailure( + (string) $template->id, + $error->publicCode, + false + ); + throw new CloudError($error->getMessage(), 502); + } + $identity = self::reconcileUnknownPublish( + $template, + $artifact, + $file, + $runtime, + $s3, + $error->publicCode ); - throw new CloudError($error->getMessage(), 502); } catch (\Throwable $error) { - self::recordPublishFailure( - (string) $template->id, - 'template_source_publish_failed', - false + // An unclassified failure may still have happened after PUT was + // dispatched. Reconcile the reserved exact key before deciding. + $identity = self::reconcileUnknownPublish( + $template, + $artifact, + $file, + $runtime, + $s3, + 'template_source_publish_failed' ); - throw $error; } return $artifact->getConnection()->transaction(static function () use ( @@ -132,6 +146,68 @@ final class CloudTemplateArtifactManager }, 3); } + /** + * Resolve one ambiguous immutable PUT using the already reserved exact + * object key. Absence is a proven failure; matching presence can finish + * publication; unavailable or conflicting evidence remains fenced. + * + * @param array{path:string,size_bytes:int,sha256:string} $file + * @param array{config:array, + * credentials:array,revision:int} $runtime + * @return array{size_bytes:int,etag:string,version_id:string} + */ + private static function reconcileUnknownPublish( + CloudTemplate $template, + CloudTemplateArtifact $artifact, + array $file, + array $runtime, + CloudObjectStorageS3Client $client, + string $failureCode + ): array { + try { + $identity = $client->inspectObject( + $runtime['config'], + $runtime['credentials'], + (string) $artifact->object_key + ); + } catch (\Throwable $error) { + self::recordPublishFailure( + (string) $template->id, + 'template_source_reconcile_unavailable', + true + ); + throw new CloudError( + '无法确认模板主副本是否已保存,记录已锁定,请勿重新导入', + 502 + ); + } + + if ($identity === null) { + self::recordPublishFailure( + (string) $template->id, + $failureCode, + false + ); + throw new CloudError('对象存储已确认模板主副本未保存', 502); + } + // The key contains the reserved random generation and the PUT payload + // is SHA-256-bound by SigV4. Exact-key presence plus the expected size + // is therefore sufficient to freeze the provider identity returned by + // HEAD; a size conflict must remain fenced for operator review. + if ((int) $identity['size_bytes'] !== (int) $file['size_bytes']) { + self::recordPublishFailure( + (string) $template->id, + 'template_source_identity_mismatch', + true + ); + throw new CloudError( + '模板主副本对象大小与导入文件不一致,记录已锁定', + 409 + ); + } + return $identity; + } + /** Return a bounded read capability after revalidating the frozen object. */ public static function downloadSource( CloudTemplate $template, diff --git a/web/public/cloud/admin.js b/web/public/cloud/admin.js index 3da5fe04..772daae1 100644 --- a/web/public/cloud/admin.js +++ b/web/public/cloud/admin.js @@ -973,6 +973,9 @@ function cloudTemplateVersion(entry) { card.append(header, form, replicaMatrix(entry)); return card; } +function cloudTemplateAvailable(entry) { + return ['active', 'degraded'].includes(String(entry?.state || '').toLowerCase()); +} function renderCloudTemplates(list, entries) { const groups = new Map(); for (const entry of entries) { @@ -984,7 +987,8 @@ function renderCloudTemplates(list, entries) { const section = element('section', 'cloud-template-family'); const head = element('div', 'cloud-template-family-head'); const title = element('div'); title.append(element('h6', '', `${group.family} ${group.version}`.trim()), element('small', 'admin-mono', group.key)); - head.append(title, element('span', 'cloud-template-version-count', `${group.entries.length} 个版本`)); + const available = group.entries.filter(cloudTemplateAvailable).length; + head.append(title, element('span', 'cloud-template-version-count', `${available} 可用 / ${group.entries.length} 条记录`)); const versions = element('div', 'cloud-template-version-list'); group.entries.sort((left, right) => Number(right.is_default === true) - Number(left.is_default === true) || Number(right.created_at || 0) - Number(left.created_at || 0)); for (const entry of group.entries) versions.append(cloudTemplateVersion(entry)); @@ -1025,7 +1029,11 @@ async function loadCatalog() { if (!entries.length) list.append(element('p', 'text-muted', '暂无模版。')); else if (rescue) renderRescueTemplates(list, entries, hostId); else renderCloudTemplates(list, entries); - notice(rescue ? `已加载 ${entries.length} 个救援模版。` : `已加载 ${entries.length} 个云镜像版本。`); + if (rescue) notice(`已加载 ${entries.length} 个救援模版。`); + else { + const available = entries.filter(cloudTemplateAvailable).length; + notice(`已加载 ${entries.length} 条云镜像记录:${available} 个可用,${entries.length - available} 个不可用。`); + } } catch (error) { if (epoch === state.epoch) notice(error.message, true); } } function taskFailureMessage(task, fallback) { diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index 0cd7f17b..932ee1b4 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026103013' scope='global'} +{assign var=venix_asset_version value='2026103014' scope='global'} diff --git a/web/tests/AdminTemplateStartupScriptBehaviorTest.mjs b/web/tests/AdminTemplateStartupScriptBehaviorTest.mjs index fc86d62e..5456251d 100644 --- a/web/tests/AdminTemplateStartupScriptBehaviorTest.mjs +++ b/web/tests/AdminTemplateStartupScriptBehaviorTest.mjs @@ -180,6 +180,28 @@ function fixture(canManage = true) { assert.ok(findText(row, '失败并已锁定')); } +{ + const test = fixture(); + const loading = test.loadCatalog(); + await test.respond(test.requests[0], { templates: [ + entry, + { + ...entry, + id: 'debian-12-source-failed', + version_label: '20261030-failed', + is_default: false, + state: 'source_failed', + artifact: { state: 'failed', error_code: 'template_source_publish_failed' }, + }, + ] }); + await loading; + assert.equal( + test.get('cloud-admin-status').textContent, + '已加载 2 条云镜像记录:1 个可用,1 个不可用。', + ); + assert.ok(findText(test.get('catalog-list'), '1 可用 / 2 条记录')); +} + { const test = fixture(false); const opening = test.openStartupScript(entry, new Element()); diff --git a/web/tests/CloudObjectStorageImmutablePutTest.php b/web/tests/CloudObjectStorageImmutablePutTest.php new file mode 100644 index 00000000..ffcbee10 --- /dev/null +++ b/web/tests/CloudObjectStorageImmutablePutTest.php @@ -0,0 +1,68 @@ +getBody()->close(); + return Create::promiseFor(new Response(200, [ + 'ETag' => '"closed-body-etag"', + 'x-amz-version-id' => 'closed-body-version', + ])); +}; +$client = new CloudObjectStorageS3Client(new Client([ + 'handler' => HandlerStack::create($handler), +])); + +try { + $identity = $client->putImmutableFile( + [ + 'endpoint' => 'https://objects.example.com', + 'region' => 'ewr', + 'bucket' => 'template-bucket', + 'prefix' => 'virt-cloud', + 'path_style' => true, + ], + [ + 'access_key_id' => 'TEMPLATEACCESSKEY', + 'secret_access_key' => 'template-secret', + ], + 'virt-cloud/cloud-templates/template/generation/system.qcow2', + $path, + $bytes, + $sha256 + ); +} finally { + @unlink($path); +} + +$assert($identity === [ + 'size_bytes' => $bytes, + 'etag' => '"closed-body-etag"', + 'version_id' => 'closed-body-version', +], 'Immutable PUT double-closed an HTTP-owned body stream.'); + +echo "Immutable object PUT stream ownership passed.\n"; diff --git a/web/tests/CloudTemplateArtifactManagerTest.php b/web/tests/CloudTemplateArtifactManagerTest.php index b0e42f3b..e8db3365 100644 --- a/web/tests/CloudTemplateArtifactManagerTest.php +++ b/web/tests/CloudTemplateArtifactManagerTest.php @@ -16,13 +16,17 @@ namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudTemplate; + use App\Models\CloudTemplateArtifact; + use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudObjectStorageS3Client; use App\Services\Cloud\CloudTemplateArtifactManager; use App\Services\Config; use GuzzleHttp\Client; + use GuzzleHttp\Exception\ConnectException; use GuzzleHttp\Handler\MockHandler; use GuzzleHttp\HandlerStack; use GuzzleHttp\Middleware; + use GuzzleHttp\Psr7\Request; use GuzzleHttp\Psr7\Response; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Schema\Blueprint; @@ -86,6 +90,9 @@ namespace { if ($path === false) { throw new RuntimeException('Cannot create temporary template fixture.'); } + register_shutdown_function(static function () use ($path): void { + @unlink($path); + }); file_put_contents($path, str_repeat('immutable-template-', 1024)); $bytes = filesize($path); $sha256 = hash_file('sha256', $path); @@ -144,15 +151,11 @@ namespace { } $template->save(); CloudTemplateArtifactManager::reserveLocked($template, $runtime); - try { - $artifact = CloudTemplateArtifactManager::publishReserved( - $template, - ['path' => $path, 'size_bytes' => $bytes, 'sha256' => $sha256], - $client - ); - } finally { - @unlink($path); - } + $artifact = CloudTemplateArtifactManager::publishReserved( + $template, + ['path' => $path, 'size_bytes' => $bytes, 'sha256' => $sha256], + $client + ); $request = $history[0]['request']; $assert($request->getMethod() === 'PUT' && str_contains((string) $request->getUri(), @@ -172,5 +175,140 @@ namespace { && $history[1]['request']->getMethod() === 'HEAD', 'Template download capability did not bind the frozen provider version.'); + $reserveTemplate = static function ( + string $id, + string $generation + ) use ($bytes, $sha256, $runtime): CloudTemplate { + $row = new CloudTemplate(); + foreach ([ + 'id' => $id, + 'catalog_key' => 'debian-13', + 'version_label' => '20261030-' . substr($id, -8), + 'is_default' => 0, + 'name' => 'Debian 13 ' . $id, + 'os_family' => 'Debian', + 'os_version' => '13', + 'image_bytes' => $bytes, + 'sha256' => $sha256, + 'startup_script' => '', + 'revision' => 1, + 'generation_token' => $generation, + 'state' => 'preparing', + 'active_job_id' => null, + 'created_at' => time(), + 'updated_at' => time(), + ] as $field => $value) { + $row->{$field} = $value; + } + $row->save(); + CloudTemplateArtifactManager::reserveLocked($row, $runtime); + return $row; + }; + + $uncertainTemplate = $reserveTemplate( + 'debian-13-reconciled', + str_repeat('b', 32) + ); + + $uncertainHistory = []; + $uncertainStack = HandlerStack::create(new MockHandler([ + new ConnectException( + 'simulated lost PUT response', + new Request('PUT', 'https://objects.example.com') + ), + new Response(200, [ + 'Content-Length' => (string) $bytes, + 'ETag' => '"reconciled-etag"', + 'x-amz-version-id' => 'reconciled-version-1', + ]), + ])); + $uncertainStack->push(Middleware::history($uncertainHistory)); + $reconciled = CloudTemplateArtifactManager::publishReserved( + $uncertainTemplate, + ['path' => $path, 'size_bytes' => $bytes, 'sha256' => $sha256], + new CloudObjectStorageS3Client(new Client([ + 'handler' => $uncertainStack, + ])) + ); + $assert((string) $reconciled->state === 'ready' + && (string) $reconciled->object_etag === '"reconciled-etag"' + && (string) $reconciled->object_version_id + === 'reconciled-version-1' + && count($uncertainHistory) === 2 + && $uncertainHistory[0]['request']->getMethod() === 'PUT' + && $uncertainHistory[1]['request']->getMethod() === 'HEAD', + 'Unknown immutable PUT effect was not reconciled from exact object identity.'); + + $absentTemplate = $reserveTemplate( + 'debian-13-absent', + str_repeat('c', 32) + ); + $absentStack = HandlerStack::create(new MockHandler([ + new ConnectException( + 'simulated lost PUT response', + new Request('PUT', 'https://objects.example.com') + ), + new Response(404), + ])); + try { + CloudTemplateArtifactManager::publishReserved( + $absentTemplate, + ['path' => $path, 'size_bytes' => $bytes, 'sha256' => $sha256], + new CloudObjectStorageS3Client(new Client([ + 'handler' => $absentStack, + ])) + ); + throw new RuntimeException('Absent unknown PUT unexpectedly succeeded.'); + } catch (CloudError $error) { + $assert($error->httpStatus === 502, + 'Proven absent PUT returned an unsafe status.'); + } + $absentTemplate->refresh(); + $absentArtifact = CloudTemplateArtifact::find( + (string) $absentTemplate->id + ); + $assert((string) $absentTemplate->state === 'source_failed' + && (string) $absentArtifact?->state === 'failed', + 'Proven absent PUT remained permanently fenced.'); + + $unavailableTemplate = $reserveTemplate( + 'debian-13-unavailable', + str_repeat('d', 32) + ); + $unavailableStack = HandlerStack::create(new MockHandler([ + new ConnectException( + 'simulated lost PUT response', + new Request('PUT', 'https://objects.example.com') + ), + new ConnectException( + 'simulated unavailable HEAD', + new Request('HEAD', 'https://objects.example.com') + ), + ])); + try { + CloudTemplateArtifactManager::publishReserved( + $unavailableTemplate, + ['path' => $path, 'size_bytes' => $bytes, 'sha256' => $sha256], + new CloudObjectStorageS3Client(new Client([ + 'handler' => $unavailableStack, + ])) + ); + throw new RuntimeException( + 'Unverifiable unknown PUT unexpectedly succeeded.' + ); + } catch (CloudError $error) { + $assert($error->httpStatus === 502, + 'Unverifiable PUT returned an unsafe status.'); + } + $unavailableTemplate->refresh(); + $unavailableArtifact = CloudTemplateArtifact::find( + (string) $unavailableTemplate->id + ); + $assert((string) $unavailableTemplate->state === 'source_unknown' + && (string) $unavailableArtifact?->state === 'unknown' + && (string) $unavailableArtifact?->error_code + === 'template_source_reconcile_unavailable', + 'Unavailable reconciliation did not retain the immutable object fence.'); + echo "Immutable cloud-template artifact contract passed.\n"; }