You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/objecttransfer/replica.go

135 lines
4.6 KiB
Go

package objecttransfer
import (
"context"
"crypto/sha256"
"encoding/hex"
"io"
"net/http"
"strings"
)
// CopySource is one immutable, exact-version source object. GetURL is a
// short-lived read capability; this package never accepts provider credentials.
type CopySource struct {
GetURL string `json:"get_url"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
}
// CopyObject streams one immutable object directly into a destination
// multipart upload. At most one multipart part is in flight, and the complete
// object is never written to local disk or retained in memory.
//
// A failed or rejected completion is aborted only when its effect is known.
// An ambiguous completion is reconciled through the exact destination read
// capability and is never followed by abort or replay.
func (c *Client) CopyObject(ctx context.Context, source CopySource,
plan MultipartPlan) (UploadResult, error) {
if c == nil || c.http == nil {
return UploadResult{}, transferError("transfer_client_unavailable",
"object transfer client is unavailable", false)
}
if err := ValidateDownload(source.GetURL, source.SizeBytes,
source.SHA256); err != nil {
return UploadResult{}, err
}
if err := ValidateMultipartPlan(plan); err != nil {
return UploadResult{}, err
}
partsNeeded := int((source.SizeBytes + plan.PartSizeBytes - 1) /
plan.PartSizeBytes)
if source.SizeBytes > plan.MaximumSizeBytes ||
partsNeeded > len(plan.PartURLs) || partsNeeded > MaxMultipartParts {
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"copy_source_too_large", "source object exceeds the multipart upload plan")
}
request, err := http.NewRequestWithContext(ctx, http.MethodGet,
source.GetURL, nil)
if err != nil {
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"invalid_capability_url", "object transfer capability is invalid")
}
response, err := c.http.Do(request)
if err != nil {
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"copy_source_unavailable", "source object could not be downloaded")
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"copy_source_unavailable", "source object could not be downloaded")
}
if encoding := strings.TrimSpace(strings.ToLower(
response.Header.Get("Content-Encoding"))); encoding != "" &&
encoding != "identity" {
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"copy_source_encoding_invalid", "source object content encoding is invalid")
}
hash := sha256.New()
reader := io.TeeReader(response.Body, hash)
parts := make([]CompletedPart, 0, partsNeeded)
var copied int64
for index := 0; index < partsNeeded; index++ {
if ctx.Err() != nil {
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"copy_cancelled", "object copy was cancelled")
}
length := min(plan.PartSizeBytes, source.SizeBytes-copied)
limited := &io.LimitedReader{R: reader, N: length}
etag, putErr := c.putPart(ctx, plan.PartURLs[index], limited, length)
if putErr != nil || limited.N != 0 {
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"copy_part_failed", "object copy part failed")
}
copied += length
parts = append(parts, CompletedPart{
PartNumber: index + 1,
ETag: etag,
})
}
extra := make([]byte, 1)
extraRead, extraErr := io.ReadFull(response.Body, extra)
if extraRead != 0 || (extraErr != nil && extraErr != io.EOF) ||
copied != source.SizeBytes {
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"copy_source_size_mismatch", "source object size does not match")
}
digest := hex.EncodeToString(hash.Sum(nil))
if !strings.EqualFold(digest, source.SHA256) {
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"copy_source_sha256_mismatch", "source object SHA-256 does not match")
}
etag, versionID, err := c.complete(ctx, plan.CompleteURL, parts)
if err != nil {
if IsAmbiguous(err) {
verified, verifyErr := c.Download(ctx, plan.VerifyURL,
source.SizeBytes, digest, io.Discard)
if verifyErr == nil && verified.SizeBytes == source.SizeBytes &&
verified.SHA256 == digest {
return UploadResult{
SizeBytes: source.SizeBytes,
SHA256: digest,
ETag: verified.ETag,
VersionID: verified.VersionID,
Parts: parts,
}, nil
}
return UploadResult{}, err
}
return UploadResult{}, c.failAndAbort(ctx, plan.AbortURL,
"copy_complete_failed", "object copy completion was rejected")
}
return UploadResult{
SizeBytes: source.SizeBytes,
SHA256: digest,
ETag: etag,
VersionID: versionID,
Parts: parts,
}, nil
}