You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
327 lines
11 KiB
Go
327 lines
11 KiB
Go
package objecttransfer
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"crypto/tls"
|
|
"encoding/hex"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
)
|
|
|
|
func testClient(server *httptest.Server) *Client {
|
|
return newWithHTTPClient(server.Client())
|
|
}
|
|
|
|
func writeSource(t *testing.T, size int) (string, []byte) {
|
|
t.Helper()
|
|
contents := make([]byte, size)
|
|
for index := range contents {
|
|
contents[index] = byte((index*31 + 17) % 251)
|
|
}
|
|
path := filepath.Join(t.TempDir(), "backup.qcow2")
|
|
if err := os.WriteFile(path, contents, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return path, contents
|
|
}
|
|
|
|
func multipartPlan(serverURL string, parts int, maximum int64) MultipartPlan {
|
|
urls := make([]string, parts)
|
|
for index := range urls {
|
|
urls[index] = fmt.Sprintf("%s/object?uploadId=opaque&partNumber=%d&signature=do-not-leak", serverURL, index+1)
|
|
}
|
|
return MultipartPlan{
|
|
PartSizeBytes: MinPartSizeBytes,
|
|
PartURLs: urls,
|
|
CompleteURL: serverURL + "/object?uploadId=opaque&complete=1&signature=do-not-leak",
|
|
AbortURL: serverURL + "/object?uploadId=opaque&abort=1&signature=do-not-leak",
|
|
VerifyURL: serverURL + "/object?signature=do-not-leak",
|
|
MaximumSizeBytes: maximum,
|
|
}
|
|
}
|
|
|
|
func TestUploadFileCompletesMultipartWithOrderedParts(t *testing.T) {
|
|
path, contents := writeSource(t, int(2*MinPartSizeBytes+123))
|
|
var mu sync.Mutex
|
|
uploaded := map[int][]byte{}
|
|
aborted := false
|
|
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.Method {
|
|
case http.MethodPut:
|
|
part, _ := strconv.Atoi(r.URL.Query().Get("partNumber"))
|
|
body, _ := io.ReadAll(r.Body)
|
|
mu.Lock()
|
|
uploaded[part] = body
|
|
mu.Unlock()
|
|
w.Header().Set("ETag", fmt.Sprintf("\"part-%d\"", part))
|
|
w.WriteHeader(http.StatusOK)
|
|
case http.MethodPost:
|
|
var request completeMultipartUpload
|
|
if err := xml.NewDecoder(r.Body).Decode(&request); err != nil {
|
|
t.Errorf("decode complete request: %v", err)
|
|
}
|
|
if len(request.Parts) != 3 || request.Parts[0].PartNumber != 1 ||
|
|
request.Parts[1].PartNumber != 2 || request.Parts[2].PartNumber != 3 {
|
|
t.Errorf("unexpected completion parts: %#v", request.Parts)
|
|
}
|
|
w.Header().Set("Content-Type", "application/xml")
|
|
w.Header().Set("x-amz-version-id", "version/7+opaque")
|
|
_, _ = io.WriteString(w, `<CompleteMultipartUploadResult><ETag>"final"</ETag></CompleteMultipartUploadResult>`)
|
|
case http.MethodDelete:
|
|
aborted = true
|
|
w.WriteHeader(http.StatusNoContent)
|
|
default:
|
|
w.WriteHeader(http.StatusMethodNotAllowed)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
result, err := testClient(server).UploadFile(
|
|
context.Background(), path, multipartPlan(server.URL, 3, int64(len(contents))),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("UploadFile: %v", err)
|
|
}
|
|
wantHash := sha256.Sum256(contents)
|
|
if result.SizeBytes != int64(len(contents)) || result.SHA256 != hex.EncodeToString(wantHash[:]) ||
|
|
result.ETag != `"final"` || result.VersionID != "version/7+opaque" ||
|
|
len(result.Parts) != 3 || aborted {
|
|
t.Fatalf("unexpected result: %#v aborted=%v", result, aborted)
|
|
}
|
|
var reconstructed []byte
|
|
for part := 1; part <= 3; part++ {
|
|
reconstructed = append(reconstructed, uploaded[part]...)
|
|
}
|
|
if !bytes.Equal(reconstructed, contents) {
|
|
t.Fatal("uploaded parts did not reconstruct the source")
|
|
}
|
|
}
|
|
|
|
func TestUploadFileAbortsAfterKnownPartFailure(t *testing.T) {
|
|
path, contents := writeSource(t, int(MinPartSizeBytes+29))
|
|
aborted := false
|
|
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.Method {
|
|
case http.MethodPut:
|
|
if r.URL.Query().Get("partNumber") == "2" {
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
w.Header().Set("ETag", `"part-1"`)
|
|
case http.MethodDelete:
|
|
aborted = true
|
|
w.WriteHeader(http.StatusNoContent)
|
|
default:
|
|
w.WriteHeader(http.StatusMethodNotAllowed)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
_, err := testClient(server).UploadFile(
|
|
context.Background(), path, multipartPlan(server.URL, 2, int64(len(contents))),
|
|
)
|
|
if err == nil || IsAmbiguous(err) || !aborted {
|
|
t.Fatalf("err=%v ambiguous=%v aborted=%v", err, IsAmbiguous(err), aborted)
|
|
}
|
|
if strings.Contains(err.Error(), "do-not-leak") || strings.Contains(err.Error(), server.URL) {
|
|
t.Fatalf("error leaked capability: %q", err)
|
|
}
|
|
}
|
|
|
|
func TestUploadFileAbortsBeforeFirstPartWhenSourceIsUnavailable(t *testing.T) {
|
|
aborted := false
|
|
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodDelete {
|
|
t.Errorf("unexpected method before source validation: %s", r.Method)
|
|
w.WriteHeader(http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
aborted = true
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}))
|
|
defer server.Close()
|
|
|
|
missing := filepath.Join(t.TempDir(), "missing.qcow2")
|
|
_, err := testClient(server).UploadFile(
|
|
context.Background(), missing, multipartPlan(server.URL, 1, MinPartSizeBytes),
|
|
)
|
|
if err == nil || IsAmbiguous(err) || !aborted {
|
|
t.Fatalf("err=%v ambiguous=%v aborted=%v", err, IsAmbiguous(err), aborted)
|
|
}
|
|
}
|
|
|
|
func TestAbortMultipartUsesDeleteCapability(t *testing.T) {
|
|
aborted := false
|
|
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodDelete || r.URL.Query().Get("abort") != "1" {
|
|
t.Errorf("unexpected abort request: %s %s", r.Method, r.URL.String())
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
return
|
|
}
|
|
aborted = true
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}))
|
|
defer server.Close()
|
|
|
|
client := testClient(server)
|
|
if err := client.AbortMultipart(
|
|
context.Background(), multipartPlan(server.URL, 1, MinPartSizeBytes),
|
|
); err != nil {
|
|
t.Fatalf("AbortMultipart: %v", err)
|
|
}
|
|
if !aborted {
|
|
t.Fatal("multipart abort capability was not used")
|
|
}
|
|
}
|
|
|
|
func TestUploadFileDoesNotAbortAmbiguousCompletion(t *testing.T) {
|
|
path, contents := writeSource(t, int(MinPartSizeBytes))
|
|
aborted := false
|
|
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.Method {
|
|
case http.MethodPut:
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
w.Header().Set("ETag", `"part-1"`)
|
|
case http.MethodPost:
|
|
hijacker, ok := w.(http.Hijacker)
|
|
if !ok {
|
|
t.Fatal("test server does not support hijacking")
|
|
}
|
|
connection, _, err := hijacker.Hijack()
|
|
if err != nil {
|
|
t.Fatalf("hijack: %v", err)
|
|
}
|
|
_ = connection.Close()
|
|
case http.MethodDelete:
|
|
aborted = true
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
_, err := testClient(server).UploadFile(
|
|
context.Background(), path, multipartPlan(server.URL, 1, int64(len(contents))),
|
|
)
|
|
if err == nil || !IsAmbiguous(err) || aborted {
|
|
t.Fatalf("err=%v ambiguous=%v aborted=%v", err, IsAmbiguous(err), aborted)
|
|
}
|
|
if strings.Contains(err.Error(), "do-not-leak") || strings.Contains(err.Error(), server.URL) {
|
|
t.Fatalf("error leaked capability: %q", err)
|
|
}
|
|
}
|
|
|
|
func TestUploadFileVerifiesCommittedObjectAfterAmbiguousCompletion(t *testing.T) {
|
|
path, contents := writeSource(t, int(MinPartSizeBytes))
|
|
aborted := false
|
|
verified := false
|
|
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.Method {
|
|
case http.MethodPut:
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
w.Header().Set("ETag", `"part-1"`)
|
|
case http.MethodPost:
|
|
hijacker, ok := w.(http.Hijacker)
|
|
if !ok {
|
|
t.Fatal("test server does not support hijacking")
|
|
}
|
|
connection, _, err := hijacker.Hijack()
|
|
if err != nil {
|
|
t.Fatalf("hijack: %v", err)
|
|
}
|
|
_ = connection.Close()
|
|
case http.MethodGet:
|
|
verified = true
|
|
w.Header().Set("ETag", `"verified-etag"`)
|
|
w.Header().Set("x-amz-version-id", "verified-version")
|
|
_, _ = w.Write(contents)
|
|
case http.MethodDelete:
|
|
aborted = true
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
result, err := testClient(server).UploadFile(
|
|
context.Background(), path, multipartPlan(server.URL, 1, int64(len(contents))),
|
|
)
|
|
if err != nil || !verified || aborted || result.SizeBytes != int64(len(contents)) {
|
|
t.Fatalf("result=%#v err=%v verified=%v aborted=%v", result, err, verified, aborted)
|
|
}
|
|
wantHash := sha256.Sum256(contents)
|
|
if result.SHA256 != hex.EncodeToString(wantHash[:]) ||
|
|
result.ETag != `"verified-etag"` || result.VersionID != "verified-version" ||
|
|
len(result.Parts) != 1 {
|
|
t.Fatalf("unexpected verified result: %#v", result)
|
|
}
|
|
}
|
|
|
|
func TestDownloadChecksExactSizeAndSHA256(t *testing.T) {
|
|
contents := []byte("portable-backup-object")
|
|
digest := sha256.Sum256(contents)
|
|
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodGet {
|
|
w.WriteHeader(http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
_, _ = w.Write(contents)
|
|
}))
|
|
defer server.Close()
|
|
capability := server.URL + "/object?signature=do-not-leak"
|
|
|
|
var destination bytes.Buffer
|
|
result, err := testClient(server).Download(context.Background(), capability,
|
|
int64(len(contents)), hex.EncodeToString(digest[:]), &destination)
|
|
if err != nil {
|
|
t.Fatalf("Download: %v", err)
|
|
}
|
|
if result.SizeBytes != int64(len(contents)) || result.SHA256 != hex.EncodeToString(digest[:]) ||
|
|
!bytes.Equal(destination.Bytes(), contents) {
|
|
t.Fatalf("unexpected download: %#v bytes=%q", result, destination.Bytes())
|
|
}
|
|
|
|
wrong := sha256.Sum256([]byte("different"))
|
|
destination.Reset()
|
|
_, err = testClient(server).Download(context.Background(), capability,
|
|
int64(len(contents)), hex.EncodeToString(wrong[:]), &destination)
|
|
if err == nil || IsAmbiguous(err) {
|
|
t.Fatalf("expected deterministic digest failure, got %v", err)
|
|
}
|
|
if strings.Contains(err.Error(), "do-not-leak") || strings.Contains(err.Error(), server.URL) {
|
|
t.Fatalf("error leaked capability: %q", err)
|
|
}
|
|
}
|
|
|
|
func TestNewClientUsesStrictTransport(t *testing.T) {
|
|
client := New()
|
|
transport, ok := client.http.Transport.(*http.Transport)
|
|
if !ok || transport.Proxy != nil || !transport.DisableCompression ||
|
|
transport.TLSClientConfig == nil || transport.TLSClientConfig.MinVersion != tls.VersionTLS12 {
|
|
t.Fatalf("unexpected strict transport: %#v", client.http.Transport)
|
|
}
|
|
if _, err := validateCapabilityURL("http://object.invalid/file"); err == nil {
|
|
t.Fatal("plain HTTP capability was accepted")
|
|
}
|
|
tooMany := multipartPlan("https://object.invalid", MaxMultipartParts+1,
|
|
MinPartSizeBytes*int64(MaxMultipartParts+1))
|
|
if err := ValidateMultipartPlan(tooMany); err == nil ||
|
|
strings.Contains(err.Error(), tooMany.PartURLs[0]) {
|
|
t.Fatalf("invalid multipart plan was accepted or leaked: %v", err)
|
|
}
|
|
if err := ValidateDownload("https://object.invalid/file?secret=value", 1,
|
|
strings.Repeat("A", sha256.Size*2)); err == nil || strings.Contains(err.Error(), "secret") {
|
|
t.Fatalf("invalid download identity was accepted or leaked: %v", err)
|
|
}
|
|
}
|