package objecttransfer import ( "bytes" "context" "crypto/sha256" "crypto/tls" "encoding/hex" "encoding/xml" "fmt" "io" "net/http" "net/http/httptest" "os" "path/filepath" "strconv" "strings" "sync" "testing" ) func testClient(server *httptest.Server) *Client { return newWithHTTPClient(server.Client()) } func writeSource(t *testing.T, size int) (string, []byte) { t.Helper() contents := make([]byte, size) for index := range contents { contents[index] = byte((index*31 + 17) % 251) } path := filepath.Join(t.TempDir(), "backup.qcow2") if err := os.WriteFile(path, contents, 0o600); err != nil { t.Fatal(err) } return path, contents } func multipartPlan(serverURL string, parts int, maximum int64) MultipartPlan { urls := make([]string, parts) for index := range urls { urls[index] = fmt.Sprintf("%s/object?uploadId=opaque&partNumber=%d&signature=do-not-leak", serverURL, index+1) } return MultipartPlan{ PartSizeBytes: MinPartSizeBytes, PartURLs: urls, CompleteURL: serverURL + "/object?uploadId=opaque&complete=1&signature=do-not-leak", AbortURL: serverURL + "/object?uploadId=opaque&abort=1&signature=do-not-leak", VerifyURL: serverURL + "/object?signature=do-not-leak", MaximumSizeBytes: maximum, } } func TestUploadFileCompletesMultipartWithOrderedParts(t *testing.T) { path, contents := writeSource(t, int(2*MinPartSizeBytes+123)) var mu sync.Mutex uploaded := map[int][]byte{} aborted := false server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodPut: part, _ := strconv.Atoi(r.URL.Query().Get("partNumber")) body, _ := io.ReadAll(r.Body) mu.Lock() uploaded[part] = body mu.Unlock() w.Header().Set("ETag", fmt.Sprintf("\"part-%d\"", part)) w.WriteHeader(http.StatusOK) case http.MethodPost: var request completeMultipartUpload if err := xml.NewDecoder(r.Body).Decode(&request); err != nil { t.Errorf("decode complete request: %v", err) } if len(request.Parts) != 3 || request.Parts[0].PartNumber != 1 || request.Parts[1].PartNumber != 2 || request.Parts[2].PartNumber != 3 { t.Errorf("unexpected completion parts: %#v", request.Parts) } w.Header().Set("Content-Type", "application/xml") w.Header().Set("x-amz-version-id", "version/7+opaque") _, _ = io.WriteString(w, `"final"`) case http.MethodDelete: aborted = true w.WriteHeader(http.StatusNoContent) default: w.WriteHeader(http.StatusMethodNotAllowed) } })) defer server.Close() result, err := testClient(server).UploadFile( context.Background(), path, multipartPlan(server.URL, 3, int64(len(contents))), ) if err != nil { t.Fatalf("UploadFile: %v", err) } wantHash := sha256.Sum256(contents) if result.SizeBytes != int64(len(contents)) || result.SHA256 != hex.EncodeToString(wantHash[:]) || result.ETag != `"final"` || result.VersionID != "version/7+opaque" || len(result.Parts) != 3 || aborted { t.Fatalf("unexpected result: %#v aborted=%v", result, aborted) } var reconstructed []byte for part := 1; part <= 3; part++ { reconstructed = append(reconstructed, uploaded[part]...) } if !bytes.Equal(reconstructed, contents) { t.Fatal("uploaded parts did not reconstruct the source") } } func TestUploadFileAbortsAfterKnownPartFailure(t *testing.T) { path, contents := writeSource(t, int(MinPartSizeBytes+29)) aborted := false server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodPut: if r.URL.Query().Get("partNumber") == "2" { w.WriteHeader(http.StatusServiceUnavailable) return } _, _ = io.Copy(io.Discard, r.Body) w.Header().Set("ETag", `"part-1"`) case http.MethodDelete: aborted = true w.WriteHeader(http.StatusNoContent) default: w.WriteHeader(http.StatusMethodNotAllowed) } })) defer server.Close() _, err := testClient(server).UploadFile( context.Background(), path, multipartPlan(server.URL, 2, int64(len(contents))), ) if err == nil || IsAmbiguous(err) || !aborted { t.Fatalf("err=%v ambiguous=%v aborted=%v", err, IsAmbiguous(err), aborted) } if strings.Contains(err.Error(), "do-not-leak") || strings.Contains(err.Error(), server.URL) { t.Fatalf("error leaked capability: %q", err) } } func TestUploadFileAbortsBeforeFirstPartWhenSourceIsUnavailable(t *testing.T) { aborted := false server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodDelete { t.Errorf("unexpected method before source validation: %s", r.Method) w.WriteHeader(http.StatusMethodNotAllowed) return } aborted = true w.WriteHeader(http.StatusNoContent) })) defer server.Close() missing := filepath.Join(t.TempDir(), "missing.qcow2") _, err := testClient(server).UploadFile( context.Background(), missing, multipartPlan(server.URL, 1, MinPartSizeBytes), ) if err == nil || IsAmbiguous(err) || !aborted { t.Fatalf("err=%v ambiguous=%v aborted=%v", err, IsAmbiguous(err), aborted) } } func TestAbortMultipartUsesDeleteCapability(t *testing.T) { aborted := false server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodDelete || r.URL.Query().Get("abort") != "1" { t.Errorf("unexpected abort request: %s %s", r.Method, r.URL.String()) w.WriteHeader(http.StatusBadRequest) return } aborted = true w.WriteHeader(http.StatusNoContent) })) defer server.Close() client := testClient(server) if err := client.AbortMultipart( context.Background(), multipartPlan(server.URL, 1, MinPartSizeBytes), ); err != nil { t.Fatalf("AbortMultipart: %v", err) } if !aborted { t.Fatal("multipart abort capability was not used") } } func TestUploadFileDoesNotAbortAmbiguousCompletion(t *testing.T) { path, contents := writeSource(t, int(MinPartSizeBytes)) aborted := false server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodPut: _, _ = io.Copy(io.Discard, r.Body) w.Header().Set("ETag", `"part-1"`) case http.MethodPost: hijacker, ok := w.(http.Hijacker) if !ok { t.Fatal("test server does not support hijacking") } connection, _, err := hijacker.Hijack() if err != nil { t.Fatalf("hijack: %v", err) } _ = connection.Close() case http.MethodDelete: aborted = true w.WriteHeader(http.StatusNoContent) } })) defer server.Close() _, err := testClient(server).UploadFile( context.Background(), path, multipartPlan(server.URL, 1, int64(len(contents))), ) if err == nil || !IsAmbiguous(err) || aborted { t.Fatalf("err=%v ambiguous=%v aborted=%v", err, IsAmbiguous(err), aborted) } if strings.Contains(err.Error(), "do-not-leak") || strings.Contains(err.Error(), server.URL) { t.Fatalf("error leaked capability: %q", err) } } func TestUploadFileVerifiesCommittedObjectAfterAmbiguousCompletion(t *testing.T) { path, contents := writeSource(t, int(MinPartSizeBytes)) aborted := false verified := false server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodPut: _, _ = io.Copy(io.Discard, r.Body) w.Header().Set("ETag", `"part-1"`) case http.MethodPost: hijacker, ok := w.(http.Hijacker) if !ok { t.Fatal("test server does not support hijacking") } connection, _, err := hijacker.Hijack() if err != nil { t.Fatalf("hijack: %v", err) } _ = connection.Close() case http.MethodGet: verified = true w.Header().Set("ETag", `"verified-etag"`) w.Header().Set("x-amz-version-id", "verified-version") _, _ = w.Write(contents) case http.MethodDelete: aborted = true w.WriteHeader(http.StatusNoContent) } })) defer server.Close() result, err := testClient(server).UploadFile( context.Background(), path, multipartPlan(server.URL, 1, int64(len(contents))), ) if err != nil || !verified || aborted || result.SizeBytes != int64(len(contents)) { t.Fatalf("result=%#v err=%v verified=%v aborted=%v", result, err, verified, aborted) } wantHash := sha256.Sum256(contents) if result.SHA256 != hex.EncodeToString(wantHash[:]) || result.ETag != `"verified-etag"` || result.VersionID != "verified-version" || len(result.Parts) != 1 { t.Fatalf("unexpected verified result: %#v", result) } } func TestDownloadChecksExactSizeAndSHA256(t *testing.T) { contents := []byte("portable-backup-object") digest := sha256.Sum256(contents) server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { w.WriteHeader(http.StatusMethodNotAllowed) return } _, _ = w.Write(contents) })) defer server.Close() capability := server.URL + "/object?signature=do-not-leak" var destination bytes.Buffer result, err := testClient(server).Download(context.Background(), capability, int64(len(contents)), hex.EncodeToString(digest[:]), &destination) if err != nil { t.Fatalf("Download: %v", err) } if result.SizeBytes != int64(len(contents)) || result.SHA256 != hex.EncodeToString(digest[:]) || !bytes.Equal(destination.Bytes(), contents) { t.Fatalf("unexpected download: %#v bytes=%q", result, destination.Bytes()) } wrong := sha256.Sum256([]byte("different")) destination.Reset() _, err = testClient(server).Download(context.Background(), capability, int64(len(contents)), hex.EncodeToString(wrong[:]), &destination) if err == nil || IsAmbiguous(err) { t.Fatalf("expected deterministic digest failure, got %v", err) } if strings.Contains(err.Error(), "do-not-leak") || strings.Contains(err.Error(), server.URL) { t.Fatalf("error leaked capability: %q", err) } } func TestNewClientUsesStrictTransport(t *testing.T) { client := New() transport, ok := client.http.Transport.(*http.Transport) if !ok || transport.Proxy != nil || !transport.DisableCompression || transport.TLSClientConfig == nil || transport.TLSClientConfig.MinVersion != tls.VersionTLS12 { t.Fatalf("unexpected strict transport: %#v", client.http.Transport) } if _, err := validateCapabilityURL("http://object.invalid/file"); err == nil { t.Fatal("plain HTTP capability was accepted") } tooMany := multipartPlan("https://object.invalid", MaxMultipartParts+1, MinPartSizeBytes*int64(MaxMultipartParts+1)) if err := ValidateMultipartPlan(tooMany); err == nil || strings.Contains(err.Error(), tooMany.PartURLs[0]) { t.Fatalf("invalid multipart plan was accepted or leaked: %v", err) } if err := ValidateDownload("https://object.invalid/file?secret=value", 1, strings.Repeat("A", sha256.Size*2)); err == nil || strings.Contains(err.Error(), "secret") { t.Fatalf("invalid download identity was accepted or leaked: %v", err) } }