You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
491 lines
19 KiB
Go
491 lines
19 KiB
Go
// Package objecttransfer moves immutable backup objects through short-lived,
|
|
// pre-signed HTTPS capabilities. It deliberately knows nothing about S3
|
|
// credentials or host storage paths.
|
|
package objecttransfer
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"crypto/tls"
|
|
"encoding/hex"
|
|
"encoding/xml"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
MaxMultipartParts = 512
|
|
MinPartSizeBytes = int64(5 << 20)
|
|
MaxPartSizeBytes = int64(5 << 30)
|
|
maxResponseBytes = int64(1 << 20)
|
|
)
|
|
|
|
// Error is intentionally safe to log. It never embeds a pre-signed URL,
|
|
// response body, or underlying transport error.
|
|
type Error struct {
|
|
Code string
|
|
Message string
|
|
Ambiguous bool
|
|
}
|
|
|
|
func (e *Error) Error() string { return e.Message }
|
|
|
|
// IsAmbiguous reports whether a request may have committed remotely. Callers
|
|
// must retain their operation fence and must not replay or abort in that case.
|
|
func IsAmbiguous(err error) bool {
|
|
var transferErr *Error
|
|
return errors.As(err, &transferErr) && transferErr.Ambiguous
|
|
}
|
|
|
|
func transferError(code, message string, ambiguous bool) error {
|
|
return &Error{Code: code, Message: message, Ambiguous: ambiguous}
|
|
}
|
|
|
|
type MultipartPlan struct {
|
|
PartSizeBytes int64 `json:"part_size_bytes"`
|
|
PartURLs []string `json:"part_urls"`
|
|
CompleteURL string `json:"complete_url"`
|
|
AbortURL string `json:"abort_url"`
|
|
VerifyURL string `json:"verify_url"`
|
|
MaximumSizeBytes int64 `json:"maximum_size_bytes"`
|
|
}
|
|
|
|
type CompletedPart struct {
|
|
PartNumber int `json:"part_number"`
|
|
ETag string `json:"etag"`
|
|
}
|
|
|
|
type UploadResult struct {
|
|
SizeBytes int64 `json:"size_bytes"`
|
|
SHA256 string `json:"sha256"`
|
|
ETag string `json:"etag,omitempty"`
|
|
VersionID string `json:"version_id,omitempty"`
|
|
Parts []CompletedPart `json:"parts"`
|
|
}
|
|
|
|
type DownloadResult struct {
|
|
SizeBytes int64 `json:"size_bytes"`
|
|
SHA256 string `json:"sha256"`
|
|
ETag string `json:"etag,omitempty"`
|
|
VersionID string `json:"version_id,omitempty"`
|
|
}
|
|
|
|
type Client struct {
|
|
http *http.Client
|
|
}
|
|
|
|
// New returns a client with no environment proxy, no transparent content
|
|
// decoding, TLS 1.2 or newer, and redirects disabled.
|
|
func New() *Client {
|
|
transport := http.DefaultTransport.(*http.Transport).Clone()
|
|
transport.Proxy = nil
|
|
transport.DisableCompression = true
|
|
transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12}
|
|
return newWithHTTPClient(&http.Client{
|
|
Transport: transport,
|
|
CheckRedirect: func(*http.Request, []*http.Request) error {
|
|
return http.ErrUseLastResponse
|
|
},
|
|
})
|
|
}
|
|
|
|
// newWithHTTPClient is a focused test seam. URL validation remains active, so
|
|
// even test transports cannot turn this package into a plain-HTTP downloader.
|
|
func newWithHTTPClient(client *http.Client) *Client {
|
|
if client == nil {
|
|
panic("objecttransfer: nil HTTP client")
|
|
}
|
|
copyClient := *client
|
|
copyClient.CheckRedirect = func(*http.Request, []*http.Request) error {
|
|
return http.ErrUseLastResponse
|
|
}
|
|
return &Client{http: ©Client}
|
|
}
|
|
|
|
func (c *Client) CloseIdleConnections() { c.http.CloseIdleConnections() }
|
|
|
|
// ValidateMultipartPlan checks every capability and bound without contacting
|
|
// storage. API handlers can therefore reject an invalid plan before any local
|
|
// snapshot or volume mutation starts.
|
|
func ValidateMultipartPlan(plan MultipartPlan) error {
|
|
if len(plan.PartURLs) < 1 || len(plan.PartURLs) > MaxMultipartParts ||
|
|
plan.PartSizeBytes < MinPartSizeBytes || plan.PartSizeBytes > MaxPartSizeBytes ||
|
|
plan.MaximumSizeBytes < 1 || plan.MaximumSizeBytes > plan.PartSizeBytes*int64(len(plan.PartURLs)) {
|
|
return transferError("invalid_multipart_plan", "multipart upload plan is invalid", false)
|
|
}
|
|
complete, err := validateCapabilityURL(plan.CompleteURL)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
origin := capabilityOrigin(complete)
|
|
abort, err := validateCapabilityURL(plan.AbortURL)
|
|
if err != nil || capabilityOrigin(abort) != origin {
|
|
return transferError("invalid_multipart_plan", "multipart upload plan is invalid", false)
|
|
}
|
|
verify, err := validateCapabilityURL(plan.VerifyURL)
|
|
if err != nil || capabilityOrigin(verify) != origin {
|
|
return transferError("invalid_multipart_plan", "multipart upload plan is invalid", false)
|
|
}
|
|
for _, value := range plan.PartURLs {
|
|
part, partErr := validateCapabilityURL(value)
|
|
if partErr != nil || capabilityOrigin(part) != origin {
|
|
return transferError("invalid_multipart_plan", "multipart upload plan is invalid", false)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ValidateDownload checks one immutable object identity before a destination
|
|
// volume is created. It never returns the capability URL in an error.
|
|
func ValidateDownload(capability string, expectedSize int64, expectedSHA256 string) error {
|
|
if _, err := validateCapabilityURL(capability); err != nil {
|
|
return err
|
|
}
|
|
if expectedSize < 1 || len(expectedSHA256) != sha256.Size*2 {
|
|
return transferError("invalid_object_identity", "backup object identity is invalid", false)
|
|
}
|
|
decodedDigest, err := hex.DecodeString(expectedSHA256)
|
|
if err != nil || len(decodedDigest) != sha256.Size || strings.ToLower(expectedSHA256) != expectedSHA256 {
|
|
return transferError("invalid_object_identity", "backup object identity is invalid", false)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateCapabilityURL(value string) (*url.URL, error) {
|
|
parsed, err := url.Parse(value)
|
|
if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" {
|
|
return nil, transferError("invalid_capability_url", "object transfer capability is invalid", false)
|
|
}
|
|
return parsed, nil
|
|
}
|
|
|
|
func capabilityOrigin(value *url.URL) string {
|
|
if value == nil {
|
|
return ""
|
|
}
|
|
return strings.ToLower(value.Scheme + "://" + value.Host)
|
|
}
|
|
|
|
func hashFile(path string) (int64, string, os.FileInfo, error) {
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return 0, "", nil, transferError("backup_source_unavailable", "backup source could not be opened", false)
|
|
}
|
|
defer file.Close()
|
|
info, err := file.Stat()
|
|
if err != nil || !info.Mode().IsRegular() || info.Size() < 1 {
|
|
return 0, "", nil, transferError("backup_source_invalid", "backup source is not a non-empty regular file", false)
|
|
}
|
|
hash := sha256.New()
|
|
written, err := io.CopyBuffer(hash, file, make([]byte, 1<<20))
|
|
if err != nil || written != info.Size() {
|
|
return 0, "", nil, transferError("backup_source_read_failed", "backup source could not be read completely", false)
|
|
}
|
|
return written, hex.EncodeToString(hash.Sum(nil)), info, nil
|
|
}
|
|
|
|
// UploadFile uploads one immutable file and completes its multipart upload.
|
|
// A completion transport failure is deliberately returned as ambiguous and is
|
|
// never followed by AbortMultipartUpload.
|
|
func (c *Client) UploadFile(ctx context.Context, path string, plan MultipartPlan) (UploadResult, error) {
|
|
if c == nil || c.http == nil {
|
|
return UploadResult{}, transferError("transfer_client_unavailable", "object transfer client is unavailable", false)
|
|
}
|
|
if err := ValidateMultipartPlan(plan); err != nil {
|
|
return UploadResult{}, err
|
|
}
|
|
size, digest, before, err := hashFile(path)
|
|
if err != nil {
|
|
return UploadResult{}, c.failAndAbortError(ctx, plan.AbortURL, err)
|
|
}
|
|
partsNeeded := int((size + plan.PartSizeBytes - 1) / plan.PartSizeBytes)
|
|
if size > plan.MaximumSizeBytes || partsNeeded > len(plan.PartURLs) || partsNeeded > MaxMultipartParts {
|
|
return UploadResult{}, c.failAndAbortError(ctx, plan.AbortURL,
|
|
transferError("backup_source_too_large", "backup source exceeds the multipart upload plan", false))
|
|
}
|
|
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return UploadResult{}, c.failAndAbortError(ctx, plan.AbortURL,
|
|
transferError("backup_source_unavailable", "backup source could not be opened", false))
|
|
}
|
|
defer file.Close()
|
|
|
|
parts := make([]CompletedPart, 0, partsNeeded)
|
|
for index := 0; index < partsNeeded; index++ {
|
|
if err := ctx.Err(); err != nil {
|
|
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
|
|
"upload_cancelled", "backup upload was cancelled")
|
|
}
|
|
offset := int64(index) * plan.PartSizeBytes
|
|
length := min(plan.PartSizeBytes, size-offset)
|
|
etag, putErr := c.putPart(ctx, plan.PartURLs[index],
|
|
io.NewSectionReader(file, offset, length), length)
|
|
if putErr != nil {
|
|
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
|
|
"upload_part_failed", "backup upload part failed")
|
|
}
|
|
parts = append(parts, CompletedPart{PartNumber: index + 1, ETag: etag})
|
|
}
|
|
|
|
afterSize, afterDigest, after, err := hashFile(path)
|
|
if err != nil || afterSize != size || afterDigest != digest ||
|
|
!before.ModTime().Equal(after.ModTime()) || before.Mode() != after.Mode() {
|
|
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
|
|
"backup_source_changed", "backup source changed during upload")
|
|
}
|
|
etag, versionID, err := c.complete(ctx, plan.CompleteURL, parts)
|
|
if err != nil {
|
|
if IsAmbiguous(err) {
|
|
// Completion can commit remotely even if its response is lost. The
|
|
// caller supplies a read-only capability for the exact destination
|
|
// object so a byte-for-byte identity check can safely turn that one
|
|
// ambiguous reply into success without aborting or replaying.
|
|
verified, verifyErr := c.Download(ctx, plan.VerifyURL, size, digest, io.Discard)
|
|
if verifyErr == nil && verified.SizeBytes == size && verified.SHA256 == digest {
|
|
return UploadResult{
|
|
SizeBytes: size, SHA256: digest, ETag: verified.ETag,
|
|
VersionID: verified.VersionID, Parts: parts,
|
|
}, nil
|
|
}
|
|
return UploadResult{}, err
|
|
}
|
|
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
|
|
"upload_complete_failed", "backup multipart completion was rejected")
|
|
}
|
|
return UploadResult{
|
|
SizeBytes: size, SHA256: digest, ETag: etag,
|
|
VersionID: versionID, Parts: parts,
|
|
}, nil
|
|
}
|
|
|
|
func (c *Client) putPart(ctx context.Context, capability string, body io.Reader, length int64) (string, error) {
|
|
request, err := http.NewRequestWithContext(ctx, http.MethodPut, capability, body)
|
|
if err != nil {
|
|
return "", transferError("invalid_capability_url", "object transfer capability is invalid", false)
|
|
}
|
|
request.ContentLength = length
|
|
response, err := c.http.Do(request)
|
|
if err != nil {
|
|
return "", transferError("upload_part_failed", "backup upload part failed", false)
|
|
}
|
|
defer response.Body.Close()
|
|
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
|
|
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
|
return "", transferError("upload_part_failed", "backup upload part failed", false)
|
|
}
|
|
etag := strings.TrimSpace(response.Header.Get("ETag"))
|
|
if !validETag(etag) {
|
|
return "", transferError("upload_part_invalid", "backup upload part response is invalid", false)
|
|
}
|
|
return etag, nil
|
|
}
|
|
|
|
func validETag(value string) bool {
|
|
if value == "" || len(value) > 256 {
|
|
return false
|
|
}
|
|
for _, current := range value {
|
|
if current < 0x20 || current == 0x7f {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
type completeMultipartUpload struct {
|
|
XMLName xml.Name `xml:"CompleteMultipartUpload"`
|
|
Parts []completeMultipartPart `xml:"Part"`
|
|
}
|
|
|
|
type completeMultipartPart struct {
|
|
PartNumber int `xml:"PartNumber"`
|
|
ETag string `xml:"ETag"`
|
|
}
|
|
|
|
type completeMultipartResult struct {
|
|
XMLName xml.Name
|
|
ETag string `xml:"ETag"`
|
|
Code string `xml:"Code"`
|
|
}
|
|
|
|
func (c *Client) complete(ctx context.Context, capability string,
|
|
parts []CompletedPart) (string, string, error) {
|
|
payload := completeMultipartUpload{Parts: make([]completeMultipartPart, len(parts))}
|
|
for index, part := range parts {
|
|
payload.Parts[index] = completeMultipartPart{PartNumber: part.PartNumber, ETag: part.ETag}
|
|
}
|
|
body, err := xml.Marshal(payload)
|
|
if err != nil {
|
|
return "", "", transferError("upload_complete_invalid", "backup multipart completion could not be encoded", false)
|
|
}
|
|
request, err := http.NewRequestWithContext(ctx, http.MethodPost, capability, strings.NewReader(string(body)))
|
|
if err != nil {
|
|
return "", "", transferError("invalid_capability_url", "object transfer capability is invalid", false)
|
|
}
|
|
request.ContentLength = int64(len(body))
|
|
request.Header.Set("Content-Type", "application/xml")
|
|
response, err := c.http.Do(request)
|
|
if err != nil {
|
|
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
|
|
}
|
|
defer response.Body.Close()
|
|
encoded, readErr := io.ReadAll(io.LimitReader(response.Body, maxResponseBytes+1))
|
|
if readErr != nil || int64(len(encoded)) > maxResponseBytes {
|
|
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
|
|
}
|
|
if response.StatusCode >= http.StatusInternalServerError {
|
|
// A storage service or reverse proxy can lose the success response after
|
|
// commit. A 5xx therefore cannot authorize AbortMultipartUpload.
|
|
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
|
|
}
|
|
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
|
return "", "", transferError("upload_complete_failed", "backup multipart completion was rejected", false)
|
|
}
|
|
if len(strings.TrimSpace(string(encoded))) == 0 {
|
|
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
|
|
}
|
|
var result completeMultipartResult
|
|
if err := xml.Unmarshal(encoded, &result); err != nil {
|
|
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
|
|
}
|
|
if result.XMLName.Local == "Error" || result.Code != "" {
|
|
return "", "", transferError("upload_complete_failed", "backup multipart completion was rejected", false)
|
|
}
|
|
if result.XMLName.Local != "CompleteMultipartUploadResult" {
|
|
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
|
|
}
|
|
etag := strings.TrimSpace(result.ETag)
|
|
if !validETag(etag) {
|
|
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
|
|
}
|
|
versionID := strings.TrimSpace(response.Header.Get("x-amz-version-id"))
|
|
if !validVersionID(versionID) {
|
|
return "", "", transferError("upload_complete_uncertain", "backup multipart completion result is unknown", true)
|
|
}
|
|
return etag, versionID, nil
|
|
}
|
|
|
|
func validVersionID(value string) bool {
|
|
if len(value) > 1024 {
|
|
return false
|
|
}
|
|
for _, current := range []byte(value) {
|
|
if current < 0x20 || current == 0x7f {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (c *Client) failAndAbort(ctx context.Context, capability, code, message string) error {
|
|
return c.failAndAbortError(ctx, capability, transferError(code, message, false))
|
|
}
|
|
|
|
func (c *Client) failAndAbortError(ctx context.Context, capability string, failure error) error {
|
|
// The caller's cancellation often caused the upload failure itself. Give
|
|
// cleanup one independent, tightly bounded chance; never inherit a long or
|
|
// already-expired operation deadline.
|
|
abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
|
|
defer cancel()
|
|
if err := c.abort(abortCtx, capability); err != nil {
|
|
return transferError("upload_abort_uncertain", "backup upload failed and multipart cleanup is unconfirmed", true)
|
|
}
|
|
return failure
|
|
}
|
|
|
|
// AbortMultipart consumes only the bounded DELETE capability from a validated
|
|
// plan. It is used when higher-level VM preflight rejects the backup before a
|
|
// local snapshot or object upload can start.
|
|
func (c *Client) AbortMultipart(ctx context.Context, plan MultipartPlan) error {
|
|
if c == nil || c.http == nil {
|
|
return transferError("transfer_client_unavailable", "object transfer client is unavailable", false)
|
|
}
|
|
if err := ValidateMultipartPlan(plan); err != nil {
|
|
return err
|
|
}
|
|
abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
|
|
defer cancel()
|
|
return c.abort(abortCtx, plan.AbortURL)
|
|
}
|
|
|
|
func (c *Client) abort(ctx context.Context, capability string) error {
|
|
request, err := http.NewRequestWithContext(ctx, http.MethodDelete, capability, nil)
|
|
if err != nil {
|
|
return transferError("invalid_capability_url", "object transfer capability is invalid", false)
|
|
}
|
|
response, err := c.http.Do(request)
|
|
if err != nil {
|
|
return transferError("upload_abort_uncertain", "multipart cleanup is unconfirmed", true)
|
|
}
|
|
defer response.Body.Close()
|
|
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
|
|
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
|
return transferError("upload_abort_uncertain", "multipart cleanup is unconfirmed", true)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Download streams one exact immutable object to destination. Neither HTTP
|
|
// Content-Length nor ETag is trusted as integrity evidence.
|
|
func (c *Client) Download(ctx context.Context, capability string, expectedSize int64,
|
|
expectedSHA256 string, destination io.Writer) (DownloadResult, error) {
|
|
if c == nil || c.http == nil || destination == nil {
|
|
return DownloadResult{}, transferError("transfer_client_unavailable", "object transfer client is unavailable", false)
|
|
}
|
|
if err := ValidateDownload(capability, expectedSize, expectedSHA256); err != nil {
|
|
return DownloadResult{}, err
|
|
}
|
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, capability, nil)
|
|
if err != nil {
|
|
return DownloadResult{}, transferError("invalid_capability_url", "object transfer capability is invalid", false)
|
|
}
|
|
response, err := c.http.Do(request)
|
|
if err != nil {
|
|
return DownloadResult{}, transferError("object_download_failed", "backup object could not be downloaded", false)
|
|
}
|
|
defer response.Body.Close()
|
|
if response.StatusCode != http.StatusOK {
|
|
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
|
|
return DownloadResult{}, transferError("object_download_failed", "backup object could not be downloaded", false)
|
|
}
|
|
if encoding := strings.TrimSpace(strings.ToLower(response.Header.Get("Content-Encoding"))); encoding != "" && encoding != "identity" {
|
|
return DownloadResult{}, transferError("object_encoding_invalid", "backup object content encoding is invalid", false)
|
|
}
|
|
hash := sha256.New()
|
|
limited := &io.LimitedReader{R: response.Body, N: expectedSize + 1}
|
|
written, copyErr := io.CopyBuffer(io.MultiWriter(destination, hash), limited, make([]byte, 1<<20))
|
|
if copyErr != nil {
|
|
return DownloadResult{}, transferError("object_download_failed", "backup object could not be downloaded completely", false)
|
|
}
|
|
if written != expectedSize || limited.N != 1 {
|
|
return DownloadResult{}, transferError("object_size_mismatch", "backup object size does not match", false)
|
|
}
|
|
digest := hex.EncodeToString(hash.Sum(nil))
|
|
if !strings.EqualFold(digest, expectedSHA256) {
|
|
return DownloadResult{}, transferError("object_sha256_mismatch", "backup object SHA-256 does not match", false)
|
|
}
|
|
etag := strings.TrimSpace(response.Header.Get("ETag"))
|
|
if etag != "" && !validETag(etag) {
|
|
etag = ""
|
|
}
|
|
versionID := strings.TrimSpace(response.Header.Get("x-amz-version-id"))
|
|
if !validVersionID(versionID) {
|
|
versionID = ""
|
|
}
|
|
return DownloadResult{
|
|
SizeBytes: written, SHA256: digest, ETag: etag, VersionID: versionID,
|
|
}, nil
|
|
}
|
|
|
|
// String is useful in bounded diagnostics without revealing URL capabilities.
|
|
func (p CompletedPart) String() string {
|
|
return strconv.Itoa(p.PartNumber) + ":" + p.ETag
|
|
}
|