You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
206 lines
6.3 KiB
Go
206 lines
6.3 KiB
Go
package objecttransfer
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strconv"
|
|
"sync"
|
|
"testing"
|
|
)
|
|
|
|
func copyFixture(size int) ([]byte, string) {
|
|
contents := make([]byte, size)
|
|
for index := range contents {
|
|
contents[index] = byte((index*17 + 23) % 251)
|
|
}
|
|
digest := sha256.Sum256(contents)
|
|
return contents, hex.EncodeToString(digest[:])
|
|
}
|
|
|
|
func TestCopyObjectStreamsMultipartWithoutLocalFile(t *testing.T) {
|
|
contents, digest := copyFixture(int(MinPartSizeBytes + 173))
|
|
var mu sync.Mutex
|
|
parts := map[int][]byte{}
|
|
aborted := false
|
|
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter,
|
|
r *http.Request) {
|
|
switch {
|
|
case r.Method == http.MethodGet && r.URL.Path == "/source":
|
|
w.Header().Set("Content-Encoding", "identity")
|
|
_, _ = w.Write(contents)
|
|
case r.Method == http.MethodPut:
|
|
part, _ := strconv.Atoi(r.URL.Query().Get("partNumber"))
|
|
body, _ := io.ReadAll(r.Body)
|
|
mu.Lock()
|
|
parts[part] = body
|
|
mu.Unlock()
|
|
w.Header().Set("ETag", fmt.Sprintf("\"part-%d\"", part))
|
|
w.WriteHeader(http.StatusOK)
|
|
case r.Method == http.MethodPost:
|
|
w.Header().Set("x-amz-version-id", "version-1")
|
|
_, _ = io.WriteString(w,
|
|
`<CompleteMultipartUploadResult><ETag>"final"</ETag></CompleteMultipartUploadResult>`)
|
|
case r.Method == http.MethodDelete:
|
|
aborted = true
|
|
w.WriteHeader(http.StatusNoContent)
|
|
default:
|
|
w.WriteHeader(http.StatusNotFound)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
result, err := testClient(server).CopyObject(context.Background(), CopySource{
|
|
GetURL: server.URL + "/source?versionId=exact&signature=secret",
|
|
SizeBytes: int64(len(contents)), SHA256: digest,
|
|
}, multipartPlan(server.URL, 2, int64(len(contents))))
|
|
if err != nil {
|
|
t.Fatalf("CopyObject: %v", err)
|
|
}
|
|
mu.Lock()
|
|
joined := append(append([]byte{}, parts[1]...), parts[2]...)
|
|
mu.Unlock()
|
|
if !bytes.Equal(joined, contents) {
|
|
t.Fatal("destination parts did not reconstruct source object")
|
|
}
|
|
if result.SizeBytes != int64(len(contents)) || result.SHA256 != digest ||
|
|
result.ETag != `"final"` || result.VersionID != "version-1" ||
|
|
len(result.Parts) != 2 {
|
|
t.Fatalf("unexpected copy result: %#v", result)
|
|
}
|
|
if aborted {
|
|
t.Fatal("successful copy aborted multipart upload")
|
|
}
|
|
}
|
|
|
|
func TestCopyObjectAbortsAfterSourceDigestMismatch(t *testing.T) {
|
|
contents, _ := copyFixture(int(MinPartSizeBytes))
|
|
_, wrongDigest := copyFixture(int(MinPartSizeBytes + 1))
|
|
putCount := 0
|
|
abortCount := 0
|
|
completeCount := 0
|
|
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter,
|
|
r *http.Request) {
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
_, _ = w.Write(contents)
|
|
case http.MethodPut:
|
|
putCount++
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
w.Header().Set("ETag", `"part-1"`)
|
|
case http.MethodDelete:
|
|
abortCount++
|
|
w.WriteHeader(http.StatusNoContent)
|
|
case http.MethodPost:
|
|
completeCount++
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
_, err := testClient(server).CopyObject(context.Background(), CopySource{
|
|
GetURL: server.URL + "/source", SizeBytes: int64(len(contents)),
|
|
SHA256: wrongDigest,
|
|
}, multipartPlan(server.URL, 1, int64(len(contents))))
|
|
var transferErr *Error
|
|
if err == nil || !asTransferError(err, &transferErr) ||
|
|
transferErr.Code != "copy_source_sha256_mismatch" || transferErr.Ambiguous {
|
|
t.Fatalf("unexpected copy error: %#v", err)
|
|
}
|
|
if putCount != 1 || abortCount != 1 || completeCount != 0 {
|
|
t.Fatalf("unexpected mutation counts put=%d abort=%d complete=%d",
|
|
putCount, abortCount, completeCount)
|
|
}
|
|
}
|
|
|
|
func TestCopyObjectReconcilesAmbiguousCompletionWithoutAbort(t *testing.T) {
|
|
contents, digest := copyFixture(int(MinPartSizeBytes))
|
|
putCount := 0
|
|
abortCount := 0
|
|
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter,
|
|
r *http.Request) {
|
|
switch {
|
|
case r.Method == http.MethodGet && r.URL.Path == "/source":
|
|
_, _ = w.Write(contents)
|
|
case r.Method == http.MethodGet && r.URL.Path == "/object":
|
|
w.Header().Set("ETag", `"verified"`)
|
|
w.Header().Set("x-amz-version-id", "version-verified")
|
|
_, _ = w.Write(contents)
|
|
case r.Method == http.MethodPut:
|
|
putCount++
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
w.Header().Set("ETag", `"part-1"`)
|
|
case r.Method == http.MethodPost:
|
|
w.WriteHeader(http.StatusBadGateway)
|
|
case r.Method == http.MethodDelete:
|
|
abortCount++
|
|
w.WriteHeader(http.StatusNoContent)
|
|
default:
|
|
w.WriteHeader(http.StatusNotFound)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
result, err := testClient(server).CopyObject(context.Background(), CopySource{
|
|
GetURL: server.URL + "/source", SizeBytes: int64(len(contents)),
|
|
SHA256: digest,
|
|
}, multipartPlan(server.URL, 1, int64(len(contents))))
|
|
if err != nil {
|
|
t.Fatalf("CopyObject: %v", err)
|
|
}
|
|
if result.ETag != `"verified"` || result.VersionID != "version-verified" {
|
|
t.Fatalf("copy did not use verification evidence: %#v", result)
|
|
}
|
|
if putCount != 1 || abortCount != 0 {
|
|
t.Fatalf("unexpected mutation counts put=%d abort=%d", putCount, abortCount)
|
|
}
|
|
}
|
|
|
|
func TestCopyObjectRetainsFenceWhenCompletionCannotBeVerified(t *testing.T) {
|
|
contents, digest := copyFixture(int(MinPartSizeBytes))
|
|
abortCount := 0
|
|
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter,
|
|
r *http.Request) {
|
|
switch {
|
|
case r.Method == http.MethodGet && r.URL.Path == "/source":
|
|
_, _ = w.Write(contents)
|
|
case r.Method == http.MethodGet && r.URL.Path == "/object":
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
case r.Method == http.MethodPut:
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
w.Header().Set("ETag", `"part-1"`)
|
|
case r.Method == http.MethodPost:
|
|
w.WriteHeader(http.StatusBadGateway)
|
|
case r.Method == http.MethodDelete:
|
|
abortCount++
|
|
w.WriteHeader(http.StatusNoContent)
|
|
default:
|
|
w.WriteHeader(http.StatusNotFound)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
_, err := testClient(server).CopyObject(context.Background(), CopySource{
|
|
GetURL: server.URL + "/source", SizeBytes: int64(len(contents)),
|
|
SHA256: digest,
|
|
}, multipartPlan(server.URL, 1, int64(len(contents))))
|
|
if err == nil || !IsAmbiguous(err) {
|
|
t.Fatalf("completion must remain ambiguous: %#v", err)
|
|
}
|
|
if abortCount != 0 {
|
|
t.Fatal("ambiguous completion must not abort or replay the upload")
|
|
}
|
|
}
|
|
|
|
func asTransferError(err error, target **Error) bool {
|
|
current, ok := err.(*Error)
|
|
if ok {
|
|
*target = current
|
|
}
|
|
return ok
|
|
}
|