You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/objecttransfer/client.go

491 lines
19 KiB
Go

// Package objecttransfer moves immutable backup objects through short-lived,
// pre-signed HTTPS capabilities. It deliberately knows nothing about S3
// credentials or host storage paths.
package objecttransfer
import (
"context"
"crypto/sha256"
"crypto/tls"
"encoding/hex"
"encoding/xml"
"errors"
"io"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"time"
)
const (
MaxMultipartParts = 512
MinPartSizeBytes = int64(5 << 20)
MaxPartSizeBytes = int64(5 << 30)
maxResponseBytes = int64(1 << 20)
)
// Error is intentionally safe to log. It never embeds a pre-signed URL,
// response body, or underlying transport error.
type Error struct {
Code string
Message string
Ambiguous bool
}
func (e *Error) Error() string { return e.Message }
// IsAmbiguous reports whether a request may have committed remotely. Callers
// must retain their operation fence and must not replay or abort in that case.
func IsAmbiguous(err error) bool {
var transferErr *Error
return errors.As(err, &transferErr) && transferErr.Ambiguous
}
func transferError(code, message string, ambiguous bool) error {
return &Error{Code: code, Message: message, Ambiguous: ambiguous}
}
type MultipartPlan struct {
PartSizeBytes int64 `json:"part_size_bytes"`
PartURLs []string `json:"part_urls"`
CompleteURL string `json:"complete_url"`
AbortURL string `json:"abort_url"`
VerifyURL string `json:"verify_url"`
MaximumSizeBytes int64 `json:"maximum_size_bytes"`
}
type CompletedPart struct {
PartNumber int `json:"part_number"`
ETag string `json:"etag"`
}
type UploadResult struct {
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
ETag string `json:"etag,omitempty"`
VersionID string `json:"version_id,omitempty"`
Parts []CompletedPart `json:"parts"`
}
type DownloadResult struct {
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
ETag string `json:"etag,omitempty"`
VersionID string `json:"version_id,omitempty"`
}
type Client struct {
http *http.Client
}
// New returns a client with no environment proxy, no transparent content
// decoding, TLS 1.2 or newer, and redirects disabled.
func New() *Client {
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.Proxy = nil
transport.DisableCompression = true
transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12}
return newWithHTTPClient(&http.Client{
Transport: transport,
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
},
})
}
// newWithHTTPClient is a focused test seam. URL validation remains active, so
// even test transports cannot turn this package into a plain-HTTP downloader.
func newWithHTTPClient(client *http.Client) *Client {
if client == nil {
panic("objecttransfer: nil HTTP client")
}
copyClient := *client
copyClient.CheckRedirect = func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
}
return &Client{http: &copyClient}
}
func (c *Client) CloseIdleConnections() { c.http.CloseIdleConnections() }
// ValidateMultipartPlan checks every capability and bound without contacting
// storage. API handlers can therefore reject an invalid plan before any local
// snapshot or volume mutation starts.
func ValidateMultipartPlan(plan MultipartPlan) error {
if len(plan.PartURLs) < 1 || len(plan.PartURLs) > MaxMultipartParts ||
plan.PartSizeBytes < MinPartSizeBytes || plan.PartSizeBytes > MaxPartSizeBytes ||
plan.MaximumSizeBytes < 1 || plan.MaximumSizeBytes > plan.PartSizeBytes*int64(len(plan.PartURLs)) {
return transferError("invalid_multipart_plan", "multipart upload plan is invalid", false)
}
complete, err := validateCapabilityURL(plan.CompleteURL)
if err != nil {
return err
}
origin := capabilityOrigin(complete)
abort, err := validateCapabilityURL(plan.AbortURL)
if err != nil || capabilityOrigin(abort) != origin {
return transferError("invalid_multipart_plan", "multipart upload plan is invalid", false)
}
verify, err := validateCapabilityURL(plan.VerifyURL)
if err != nil || capabilityOrigin(verify) != origin {
return transferError("invalid_multipart_plan", "multipart upload plan is invalid", false)
}
for _, value := range plan.PartURLs {
part, partErr := validateCapabilityURL(value)
if partErr != nil || capabilityOrigin(part) != origin {
return transferError("invalid_multipart_plan", "multipart upload plan is invalid", false)
}
}
return nil
}
// ValidateDownload checks one immutable object identity before a destination
// volume is created. It never returns the capability URL in an error.
func ValidateDownload(capability string, expectedSize int64, expectedSHA256 string) error {
if _, err := validateCapabilityURL(capability); err != nil {
return err
}
if expectedSize < 1 || len(expectedSHA256) != sha256.Size*2 {
return transferError("invalid_object_identity", "backup object identity is invalid", false)
}
decodedDigest, err := hex.DecodeString(expectedSHA256)
if err != nil || len(decodedDigest) != sha256.Size || strings.ToLower(expectedSHA256) != expectedSHA256 {
return transferError("invalid_object_identity", "backup object identity is invalid", false)
}
return nil
}
func validateCapabilityURL(value string) (*url.URL, error) {
parsed, err := url.Parse(value)
if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" {
return nil, transferError("invalid_capability_url", "object transfer capability is invalid", false)
}
return parsed, nil
}
func capabilityOrigin(value *url.URL) string {
if value == nil {
return ""
}
return strings.ToLower(value.Scheme + "://" + value.Host)
}
func hashFile(path string) (int64, string, os.FileInfo, error) {
file, err := os.Open(path)
if err != nil {
return 0, "", nil, transferError("backup_source_unavailable", "backup source could not be opened", false)
}
defer file.Close()
info, err := file.Stat()
if err != nil || !info.Mode().IsRegular() || info.Size() < 1 {
return 0, "", nil, transferError("backup_source_invalid", "backup source is not a non-empty regular file", false)
}
hash := sha256.New()
written, err := io.CopyBuffer(hash, file, make([]byte, 1<<20))
if err != nil || written != info.Size() {
return 0, "", nil, transferError("backup_source_read_failed", "backup source could not be read completely", false)
}
return written, hex.EncodeToString(hash.Sum(nil)), info, nil
}
// UploadFile uploads one immutable file and completes its multipart upload.
// A completion transport failure is deliberately returned as ambiguous and is
// never followed by AbortMultipartUpload.
func (c *Client) UploadFile(ctx context.Context, path string, plan MultipartPlan) (UploadResult, error) {
if c == nil || c.http == nil {
return UploadResult{}, transferError("transfer_client_unavailable", "object transfer client is unavailable", false)
}
if err := ValidateMultipartPlan(plan); err != nil {
return UploadResult{}, err
}
size, digest, before, err := hashFile(path)
if err != nil {
return UploadResult{}, c.failAndAbortError(ctx, plan.AbortURL, err)
}
partsNeeded := int((size + plan.PartSizeBytes - 1) / plan.PartSizeBytes)
if size > plan.MaximumSizeBytes || partsNeeded > len(plan.PartURLs) || partsNeeded > MaxMultipartParts {
return UploadResult{}, c.failAndAbortError(ctx, plan.AbortURL,
transferError("backup_source_too_large", "backup source exceeds the multipart upload plan", false))
}
file, err := os.Open(path)
if err != nil {
return UploadResult{}, c.failAndAbortError(ctx, plan.AbortURL,
transferError("backup_source_unavailable", "backup source could not be opened", false))
}
defer file.Close()
parts := make([]CompletedPart, 0, partsNeeded)
for index := 0; index < partsNeeded; index++ {
if err := ctx.Err(); err != nil {
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"upload_cancelled", "backup upload was cancelled")
}
offset := int64(index) * plan.PartSizeBytes
length := min(plan.PartSizeBytes, size-offset)
etag, putErr := c.putPart(ctx, plan.PartURLs[index],
io.NewSectionReader(file, offset, length), length)
if putErr != nil {
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"upload_part_failed", "backup upload part failed")
}
parts = append(parts, CompletedPart{PartNumber: index + 1, ETag: etag})
}
afterSize, afterDigest, after, err := hashFile(path)
if err != nil || afterSize != size || afterDigest != digest ||
!before.ModTime().Equal(after.ModTime()) || before.Mode() != after.Mode() {
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"backup_source_changed", "backup source changed during upload")
}
etag, versionID, err := c.complete(ctx, plan.CompleteURL, parts)
if err != nil {
if IsAmbiguous(err) {
// Completion can commit remotely even if its response is lost. The
// caller supplies a read-only capability for the exact destination
// object so a byte-for-byte identity check can safely turn that one
// ambiguous reply into success without aborting or replaying.
verified, verifyErr := c.Download(ctx, plan.VerifyURL, size, digest, io.Discard)
if verifyErr == nil && verified.SizeBytes == size && verified.SHA256 == digest {
return UploadResult{
SizeBytes: size, SHA256: digest, ETag: verified.ETag,
VersionID: verified.VersionID, Parts: parts,
}, nil
}
return UploadResult{}, err
}
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"upload_complete_failed", "backup multipart completion was rejected")
}
return UploadResult{
SizeBytes: size, SHA256: digest, ETag: etag,
VersionID: versionID, Parts: parts,
}, nil
}
func (c *Client) putPart(ctx context.Context, capability string, body io.Reader, length int64) (string, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodPut, capability, body)
if err != nil {
return "", transferError("invalid_capability_url", "object transfer capability is invalid", false)
}
request.ContentLength = length
response, err := c.http.Do(request)
if err != nil {
return "", transferError("upload_part_failed", "backup upload part failed", false)
}
defer response.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
if response.StatusCode < 200 || response.StatusCode >= 300 {
return "", transferError("upload_part_failed", "backup upload part failed", false)
}
etag := strings.TrimSpace(response.Header.Get("ETag"))
if !validETag(etag) {
return "", transferError("upload_part_invalid", "backup upload part response is invalid", false)
}
return etag, nil
}
func validETag(value string) bool {
if value == "" || len(value) > 256 {
return false
}
for _, current := range value {
if current < 0x20 || current == 0x7f {
return false
}
}
return true
}
type completeMultipartUpload struct {
XMLName xml.Name `xml:"CompleteMultipartUpload"`
Parts []completeMultipartPart `xml:"Part"`
}
type completeMultipartPart struct {
PartNumber int `xml:"PartNumber"`
ETag string `xml:"ETag"`
}
type completeMultipartResult struct {
XMLName xml.Name
ETag string `xml:"ETag"`
Code string `xml:"Code"`
}
func (c *Client) complete(ctx context.Context, capability string,
parts []CompletedPart) (string, string, error) {
payload := completeMultipartUpload{Parts: make([]completeMultipartPart, len(parts))}
for index, part := range parts {
payload.Parts[index] = completeMultipartPart{PartNumber: part.PartNumber, ETag: part.ETag}
}
body, err := xml.Marshal(payload)
if err != nil {
return "", "", transferError("upload_complete_invalid", "backup multipart completion could not be encoded", false)
}
request, err := http.NewRequestWithContext(ctx, http.MethodPost, capability, strings.NewReader(string(body)))
if err != nil {
return "", "", transferError("invalid_capability_url", "object transfer capability is invalid", false)
}
request.ContentLength = int64(len(body))
request.Header.Set("Content-Type", "application/xml")
response, err := c.http.Do(request)
if err != nil {
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
defer response.Body.Close()
encoded, readErr := io.ReadAll(io.LimitReader(response.Body, maxResponseBytes+1))
if readErr != nil || int64(len(encoded)) > maxResponseBytes {
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
if response.StatusCode >= http.StatusInternalServerError {
// A storage service or reverse proxy can lose the success response after
// commit. A 5xx therefore cannot authorize AbortMultipartUpload.
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
if response.StatusCode < 200 || response.StatusCode >= 300 {
return "", "", transferError("upload_complete_failed", "backup multipart completion was rejected", false)
}
if len(strings.TrimSpace(string(encoded))) == 0 {
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
var result completeMultipartResult
if err := xml.Unmarshal(encoded, &result); err != nil {
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
if result.XMLName.Local == "Error" || result.Code != "" {
return "", "", transferError("upload_complete_failed", "backup multipart completion was rejected", false)
}
if result.XMLName.Local != "CompleteMultipartUploadResult" {
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
etag := strings.TrimSpace(result.ETag)
if !validETag(etag) {
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
versionID := strings.TrimSpace(response.Header.Get("x-amz-version-id"))
if !validVersionID(versionID) {
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
}
return etag, versionID, nil
}
func validVersionID(value string) bool {
if len(value) > 1024 {
return false
}
for _, current := range []byte(value) {
if current < 0x20 || current == 0x7f {
return false
}
}
return true
}
func (c *Client) failAndAbort(ctx context.Context, capability, code, message string) error {
return c.failAndAbortError(ctx, capability, transferError(code, message, false))
}
func (c *Client) failAndAbortError(ctx context.Context, capability string, failure error) error {
// The caller's cancellation often caused the upload failure itself. Give
// cleanup one independent, tightly bounded chance; never inherit a long or
// already-expired operation deadline.
abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
defer cancel()
if err := c.abort(abortCtx, capability); err != nil {
return transferError("upload_abort_uncertain", "backup upload failed and multipart cleanup is unconfirmed", true)
}
return failure
}
// AbortMultipart consumes only the bounded DELETE capability from a validated
// plan. It is used when higher-level VM preflight rejects the backup before a
// local snapshot or object upload can start.
func (c *Client) AbortMultipart(ctx context.Context, plan MultipartPlan) error {
if c == nil || c.http == nil {
return transferError("transfer_client_unavailable", "object transfer client is unavailable", false)
}
if err := ValidateMultipartPlan(plan); err != nil {
return err
}
abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
defer cancel()
return c.abort(abortCtx, plan.AbortURL)
}
func (c *Client) abort(ctx context.Context, capability string) error {
request, err := http.NewRequestWithContext(ctx, http.MethodDelete, capability, nil)
if err != nil {
return transferError("invalid_capability_url", "object transfer capability is invalid", false)
}
response, err := c.http.Do(request)
if err != nil {
return transferError("upload_abort_uncertain", "multipart cleanup is unconfirmed", true)
}
defer response.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
if response.StatusCode < 200 || response.StatusCode >= 300 {
return transferError("upload_abort_uncertain", "multipart cleanup is unconfirmed", true)
}
return nil
}
// Download streams one exact immutable object to destination. Neither HTTP
// Content-Length nor ETag is trusted as integrity evidence.
func (c *Client) Download(ctx context.Context, capability string, expectedSize int64,
expectedSHA256 string, destination io.Writer) (DownloadResult, error) {
if c == nil || c.http == nil || destination == nil {
return DownloadResult{}, transferError("transfer_client_unavailable", "object transfer client is unavailable", false)
}
if err := ValidateDownload(capability, expectedSize, expectedSHA256); err != nil {
return DownloadResult{}, err
}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, capability, nil)
if err != nil {
return DownloadResult{}, transferError("invalid_capability_url", "object transfer capability is invalid", false)
}
response, err := c.http.Do(request)
if err != nil {
return DownloadResult{}, transferError("object_download_failed", "backup object could not be downloaded", false)
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
return DownloadResult{}, transferError("object_download_failed", "backup object could not be downloaded", false)
}
if encoding := strings.TrimSpace(strings.ToLower(response.Header.Get("Content-Encoding"))); encoding != "" && encoding != "identity" {
return DownloadResult{}, transferError("object_encoding_invalid", "backup object content encoding is invalid", false)
}
hash := sha256.New()
limited := &io.LimitedReader{R: response.Body, N: expectedSize + 1}
written, copyErr := io.CopyBuffer(io.MultiWriter(destination, hash), limited, make([]byte, 1<<20))
if copyErr != nil {
return DownloadResult{}, transferError("object_download_failed", "backup object could not be downloaded completely", false)
}
if written != expectedSize || limited.N != 1 {
return DownloadResult{}, transferError("object_size_mismatch", "backup object size does not match", false)
}
digest := hex.EncodeToString(hash.Sum(nil))
if !strings.EqualFold(digest, expectedSHA256) {
return DownloadResult{}, transferError("object_sha256_mismatch", "backup object SHA-256 does not match", false)
}
etag := strings.TrimSpace(response.Header.Get("ETag"))
if etag != "" && !validETag(etag) {
etag = ""
}
versionID := strings.TrimSpace(response.Header.Get("x-amz-version-id"))
if !validVersionID(versionID) {
versionID = ""
}
return DownloadResult{
SizeBytes: written, SHA256: digest, ETag: etag, VersionID: versionID,
}, nil
}
// String is useful in bounded diagnostics without revealing URL capabilities.
func (p CompletedPart) String() string {
return strconv.Itoa(p.PartNumber) + ":" + p.ETag
}