// Package objecttransfer moves immutable backup objects through short-lived, // pre-signed HTTPS capabilities. It deliberately knows nothing about S3 // credentials or host storage paths. package objecttransfer import ( "context" "crypto/sha256" "crypto/tls" "encoding/hex" "encoding/xml" "errors" "io" "net/http" "net/url" "os" "strconv" "strings" "time" ) const ( MaxMultipartParts = 512 MinPartSizeBytes = int64(5 << 20) MaxPartSizeBytes = int64(5 << 30) maxResponseBytes = int64(1 << 20) ) // Error is intentionally safe to log. It never embeds a pre-signed URL, // response body, or underlying transport error. type Error struct { Code string Message string Ambiguous bool } func (e *Error) Error() string { return e.Message } // IsAmbiguous reports whether a request may have committed remotely. Callers // must retain their operation fence and must not replay or abort in that case. func IsAmbiguous(err error) bool { var transferErr *Error return errors.As(err, &transferErr) && transferErr.Ambiguous } func transferError(code, message string, ambiguous bool) error { return &Error{Code: code, Message: message, Ambiguous: ambiguous} } type MultipartPlan struct { PartSizeBytes int64 `json:"part_size_bytes"` PartURLs []string `json:"part_urls"` CompleteURL string `json:"complete_url"` AbortURL string `json:"abort_url"` VerifyURL string `json:"verify_url"` MaximumSizeBytes int64 `json:"maximum_size_bytes"` } type CompletedPart struct { PartNumber int `json:"part_number"` ETag string `json:"etag"` } type UploadResult struct { SizeBytes int64 `json:"size_bytes"` SHA256 string `json:"sha256"` ETag string `json:"etag,omitempty"` VersionID string `json:"version_id,omitempty"` Parts []CompletedPart `json:"parts"` } type DownloadResult struct { SizeBytes int64 `json:"size_bytes"` SHA256 string `json:"sha256"` ETag string `json:"etag,omitempty"` VersionID string `json:"version_id,omitempty"` } type Client struct { http *http.Client } // New returns a client with no environment proxy, no transparent content // decoding, TLS 1.2 or newer, and redirects disabled. func New() *Client { transport := http.DefaultTransport.(*http.Transport).Clone() transport.Proxy = nil transport.DisableCompression = true transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12} return newWithHTTPClient(&http.Client{ Transport: transport, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, }) } // newWithHTTPClient is a focused test seam. URL validation remains active, so // even test transports cannot turn this package into a plain-HTTP downloader. func newWithHTTPClient(client *http.Client) *Client { if client == nil { panic("objecttransfer: nil HTTP client") } copyClient := *client copyClient.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } return &Client{http: ©Client} } func (c *Client) CloseIdleConnections() { c.http.CloseIdleConnections() } // ValidateMultipartPlan checks every capability and bound without contacting // storage. API handlers can therefore reject an invalid plan before any local // snapshot or volume mutation starts. func ValidateMultipartPlan(plan MultipartPlan) error { if len(plan.PartURLs) < 1 || len(plan.PartURLs) > MaxMultipartParts || plan.PartSizeBytes < MinPartSizeBytes || plan.PartSizeBytes > MaxPartSizeBytes || plan.MaximumSizeBytes < 1 || plan.MaximumSizeBytes > plan.PartSizeBytes*int64(len(plan.PartURLs)) { return transferError("invalid_multipart_plan", "multipart upload plan is invalid", false) } complete, err := validateCapabilityURL(plan.CompleteURL) if err != nil { return err } origin := capabilityOrigin(complete) abort, err := validateCapabilityURL(plan.AbortURL) if err != nil || capabilityOrigin(abort) != origin { return transferError("invalid_multipart_plan", "multipart upload plan is invalid", false) } verify, err := validateCapabilityURL(plan.VerifyURL) if err != nil || capabilityOrigin(verify) != origin { return transferError("invalid_multipart_plan", "multipart upload plan is invalid", false) } for _, value := range plan.PartURLs { part, partErr := validateCapabilityURL(value) if partErr != nil || capabilityOrigin(part) != origin { return transferError("invalid_multipart_plan", "multipart upload plan is invalid", false) } } return nil } // ValidateDownload checks one immutable object identity before a destination // volume is created. It never returns the capability URL in an error. func ValidateDownload(capability string, expectedSize int64, expectedSHA256 string) error { if _, err := validateCapabilityURL(capability); err != nil { return err } if expectedSize < 1 || len(expectedSHA256) != sha256.Size*2 { return transferError("invalid_object_identity", "backup object identity is invalid", false) } decodedDigest, err := hex.DecodeString(expectedSHA256) if err != nil || len(decodedDigest) != sha256.Size || strings.ToLower(expectedSHA256) != expectedSHA256 { return transferError("invalid_object_identity", "backup object identity is invalid", false) } return nil } func validateCapabilityURL(value string) (*url.URL, error) { parsed, err := url.Parse(value) if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" { return nil, transferError("invalid_capability_url", "object transfer capability is invalid", false) } return parsed, nil } func capabilityOrigin(value *url.URL) string { if value == nil { return "" } return strings.ToLower(value.Scheme + "://" + value.Host) } func hashFile(path string) (int64, string, os.FileInfo, error) { file, err := os.Open(path) if err != nil { return 0, "", nil, transferError("backup_source_unavailable", "backup source could not be opened", false) } defer file.Close() info, err := file.Stat() if err != nil || !info.Mode().IsRegular() || info.Size() < 1 { return 0, "", nil, transferError("backup_source_invalid", "backup source is not a non-empty regular file", false) } hash := sha256.New() written, err := io.CopyBuffer(hash, file, make([]byte, 1<<20)) if err != nil || written != info.Size() { return 0, "", nil, transferError("backup_source_read_failed", "backup source could not be read completely", false) } return written, hex.EncodeToString(hash.Sum(nil)), info, nil } // UploadFile uploads one immutable file and completes its multipart upload. // A completion transport failure is deliberately returned as ambiguous and is // never followed by AbortMultipartUpload. func (c *Client) UploadFile(ctx context.Context, path string, plan MultipartPlan) (UploadResult, error) { if c == nil || c.http == nil { return UploadResult{}, transferError("transfer_client_unavailable", "object transfer client is unavailable", false) } if err := ValidateMultipartPlan(plan); err != nil { return UploadResult{}, err } size, digest, before, err := hashFile(path) if err != nil { return UploadResult{}, c.failAndAbortError(ctx, plan.AbortURL, err) } partsNeeded := int((size + plan.PartSizeBytes - 1) / plan.PartSizeBytes) if size > plan.MaximumSizeBytes || partsNeeded > len(plan.PartURLs) || partsNeeded > MaxMultipartParts { return UploadResult{}, c.failAndAbortError(ctx, plan.AbortURL, transferError("backup_source_too_large", "backup source exceeds the multipart upload plan", false)) } file, err := os.Open(path) if err != nil { return UploadResult{}, c.failAndAbortError(ctx, plan.AbortURL, transferError("backup_source_unavailable", "backup source could not be opened", false)) } defer file.Close() parts := make([]CompletedPart, 0, partsNeeded) for index := 0; index < partsNeeded; index++ { if err := ctx.Err(); err != nil { return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL, "upload_cancelled", "backup upload was cancelled") } offset := int64(index) * plan.PartSizeBytes length := min(plan.PartSizeBytes, size-offset) etag, putErr := c.putPart(ctx, plan.PartURLs[index], io.NewSectionReader(file, offset, length), length) if putErr != nil { return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL, "upload_part_failed", "backup upload part failed") } parts = append(parts, CompletedPart{PartNumber: index + 1, ETag: etag}) } afterSize, afterDigest, after, err := hashFile(path) if err != nil || afterSize != size || afterDigest != digest || !before.ModTime().Equal(after.ModTime()) || before.Mode() != after.Mode() { return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL, "backup_source_changed", "backup source changed during upload") } etag, versionID, err := c.complete(ctx, plan.CompleteURL, parts) if err != nil { if IsAmbiguous(err) { // Completion can commit remotely even if its response is lost. The // caller supplies a read-only capability for the exact destination // object so a byte-for-byte identity check can safely turn that one // ambiguous reply into success without aborting or replaying. verified, verifyErr := c.Download(ctx, plan.VerifyURL, size, digest, io.Discard) if verifyErr == nil && verified.SizeBytes == size && verified.SHA256 == digest { return UploadResult{ SizeBytes: size, SHA256: digest, ETag: verified.ETag, VersionID: verified.VersionID, Parts: parts, }, nil } return UploadResult{}, err } return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL, "upload_complete_failed", "backup multipart completion was rejected") } return UploadResult{ SizeBytes: size, SHA256: digest, ETag: etag, VersionID: versionID, Parts: parts, }, nil } func (c *Client) putPart(ctx context.Context, capability string, body io.Reader, length int64) (string, error) { request, err := http.NewRequestWithContext(ctx, http.MethodPut, capability, body) if err != nil { return "", transferError("invalid_capability_url", "object transfer capability is invalid", false) } request.ContentLength = length response, err := c.http.Do(request) if err != nil { return "", transferError("upload_part_failed", "backup upload part failed", false) } defer response.Body.Close() _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) if response.StatusCode < 200 || response.StatusCode >= 300 { return "", transferError("upload_part_failed", "backup upload part failed", false) } etag := strings.TrimSpace(response.Header.Get("ETag")) if !validETag(etag) { return "", transferError("upload_part_invalid", "backup upload part response is invalid", false) } return etag, nil } func validETag(value string) bool { if value == "" || len(value) > 256 { return false } for _, current := range value { if current < 0x20 || current == 0x7f { return false } } return true } type completeMultipartUpload struct { XMLName xml.Name `xml:"CompleteMultipartUpload"` Parts []completeMultipartPart `xml:"Part"` } type completeMultipartPart struct { PartNumber int `xml:"PartNumber"` ETag string `xml:"ETag"` } type completeMultipartResult struct { XMLName xml.Name ETag string `xml:"ETag"` Code string `xml:"Code"` } func (c *Client) complete(ctx context.Context, capability string, parts []CompletedPart) (string, string, error) { payload := completeMultipartUpload{Parts: make([]completeMultipartPart, len(parts))} for index, part := range parts { payload.Parts[index] = completeMultipartPart{PartNumber: part.PartNumber, ETag: part.ETag} } body, err := xml.Marshal(payload) if err != nil { return "", "", transferError("upload_complete_invalid", "backup multipart completion could not be encoded", false) } request, err := http.NewRequestWithContext(ctx, http.MethodPost, capability, strings.NewReader(string(body))) if err != nil { return "", "", transferError("invalid_capability_url", "object transfer capability is invalid", false) } request.ContentLength = int64(len(body)) request.Header.Set("Content-Type", "application/xml") response, err := c.http.Do(request) if err != nil { return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) } defer response.Body.Close() encoded, readErr := io.ReadAll(io.LimitReader(response.Body, maxResponseBytes+1)) if readErr != nil || int64(len(encoded)) > maxResponseBytes { return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) } if response.StatusCode >= http.StatusInternalServerError { // A storage service or reverse proxy can lose the success response after // commit. A 5xx therefore cannot authorize AbortMultipartUpload. return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) } if response.StatusCode < 200 || response.StatusCode >= 300 { return "", "", transferError("upload_complete_failed", "backup multipart completion was rejected", false) } if len(strings.TrimSpace(string(encoded))) == 0 { return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) } var result completeMultipartResult if err := xml.Unmarshal(encoded, &result); err != nil { return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) } if result.XMLName.Local == "Error" || result.Code != "" { return "", "", transferError("upload_complete_failed", "backup multipart completion was rejected", false) } if result.XMLName.Local != "CompleteMultipartUploadResult" { return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) } etag := strings.TrimSpace(result.ETag) if !validETag(etag) { return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) } versionID := strings.TrimSpace(response.Header.Get("x-amz-version-id")) if !validVersionID(versionID) { return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true) } return etag, versionID, nil } func validVersionID(value string) bool { if len(value) > 1024 { return false } for _, current := range []byte(value) { if current < 0x20 || current == 0x7f { return false } } return true } func (c *Client) failAndAbort(ctx context.Context, capability, code, message string) error { return c.failAndAbortError(ctx, capability, transferError(code, message, false)) } func (c *Client) failAndAbortError(ctx context.Context, capability string, failure error) error { // The caller's cancellation often caused the upload failure itself. Give // cleanup one independent, tightly bounded chance; never inherit a long or // already-expired operation deadline. abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) defer cancel() if err := c.abort(abortCtx, capability); err != nil { return transferError("upload_abort_uncertain", "backup upload failed and multipart cleanup is unconfirmed", true) } return failure } // AbortMultipart consumes only the bounded DELETE capability from a validated // plan. It is used when higher-level VM preflight rejects the backup before a // local snapshot or object upload can start. func (c *Client) AbortMultipart(ctx context.Context, plan MultipartPlan) error { if c == nil || c.http == nil { return transferError("transfer_client_unavailable", "object transfer client is unavailable", false) } if err := ValidateMultipartPlan(plan); err != nil { return err } abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) defer cancel() return c.abort(abortCtx, plan.AbortURL) } func (c *Client) abort(ctx context.Context, capability string) error { request, err := http.NewRequestWithContext(ctx, http.MethodDelete, capability, nil) if err != nil { return transferError("invalid_capability_url", "object transfer capability is invalid", false) } response, err := c.http.Do(request) if err != nil { return transferError("upload_abort_uncertain", "multipart cleanup is unconfirmed", true) } defer response.Body.Close() _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) if response.StatusCode < 200 || response.StatusCode >= 300 { return transferError("upload_abort_uncertain", "multipart cleanup is unconfirmed", true) } return nil } // Download streams one exact immutable object to destination. Neither HTTP // Content-Length nor ETag is trusted as integrity evidence. func (c *Client) Download(ctx context.Context, capability string, expectedSize int64, expectedSHA256 string, destination io.Writer) (DownloadResult, error) { if c == nil || c.http == nil || destination == nil { return DownloadResult{}, transferError("transfer_client_unavailable", "object transfer client is unavailable", false) } if err := ValidateDownload(capability, expectedSize, expectedSHA256); err != nil { return DownloadResult{}, err } request, err := http.NewRequestWithContext(ctx, http.MethodGet, capability, nil) if err != nil { return DownloadResult{}, transferError("invalid_capability_url", "object transfer capability is invalid", false) } response, err := c.http.Do(request) if err != nil { return DownloadResult{}, transferError("object_download_failed", "backup object could not be downloaded", false) } defer response.Body.Close() if response.StatusCode != http.StatusOK { _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) return DownloadResult{}, transferError("object_download_failed", "backup object could not be downloaded", false) } if encoding := strings.TrimSpace(strings.ToLower(response.Header.Get("Content-Encoding"))); encoding != "" && encoding != "identity" { return DownloadResult{}, transferError("object_encoding_invalid", "backup object content encoding is invalid", false) } hash := sha256.New() limited := &io.LimitedReader{R: response.Body, N: expectedSize + 1} written, copyErr := io.CopyBuffer(io.MultiWriter(destination, hash), limited, make([]byte, 1<<20)) if copyErr != nil { return DownloadResult{}, transferError("object_download_failed", "backup object could not be downloaded completely", false) } if written != expectedSize || limited.N != 1 { return DownloadResult{}, transferError("object_size_mismatch", "backup object size does not match", false) } digest := hex.EncodeToString(hash.Sum(nil)) if !strings.EqualFold(digest, expectedSHA256) { return DownloadResult{}, transferError("object_sha256_mismatch", "backup object SHA-256 does not match", false) } etag := strings.TrimSpace(response.Header.Get("ETag")) if etag != "" && !validETag(etag) { etag = "" } versionID := strings.TrimSpace(response.Header.Get("x-amz-version-id")) if !validVersionID(versionID) { versionID = "" } return DownloadResult{ SizeBytes: written, SHA256: digest, ETag: etag, VersionID: versionID, }, nil } // String is useful in bounded diagnostics without revealing URL capabilities. func (p CompletedPart) String() string { return strconv.Itoa(p.PartNumber) + ":" + p.ETag }