You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

246 lines
7.9 KiB
Go

// virt-cloud-storage-replicator is a panel-side capability-only data mover.
// It accepts one bounded JSON request on stdin and emits one bounded receipt
// on stdout. It never accepts provider credentials, object URLs in argv, or
// configuration from the process environment.
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"os"
"regexp"
"time"
"git.luoli.live/root/virt/agent/objecttransfer"
)
const (
protocolName = "object-replica-v1"
verifyProtocolName = "object-replica-verify-v1"
maxRequestBytes = int64(2 << 20)
maxDeadline = 6 * time.Hour
)
var operationIDPattern = regexp.MustCompile(`^[a-f0-9]{32}$`)
type replicaRequest struct {
Protocol string `json:"protocol"`
OperationID string `json:"operation_id"`
DeadlineAt int64 `json:"deadline_at"`
Source objecttransfer.CopySource `json:"source"`
Destination objecttransfer.MultipartPlan `json:"destination"`
}
type verifyRequest struct {
Protocol string `json:"protocol"`
OperationID string `json:"operation_id"`
DeadlineAt int64 `json:"deadline_at"`
GetURL string `json:"get_url"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
}
type replicaReceipt struct {
Protocol string `json:"protocol"`
OperationID string `json:"operation_id"`
Outcome string `json:"outcome"`
EffectResolution string `json:"effect_resolution"`
SizeBytes int64 `json:"size_bytes,omitempty"`
SHA256 string `json:"sha256,omitempty"`
ETag string `json:"etag,omitempty"`
VersionID string `json:"version_id"`
ErrorCode string `json:"error_code,omitempty"`
}
type objectWorker interface {
CopyObject(context.Context, objecttransfer.CopySource,
objecttransfer.MultipartPlan) (objecttransfer.UploadResult, error)
Download(context.Context, string, int64, string,
io.Writer) (objecttransfer.DownloadResult, error)
}
func main() {
receipt := run(os.Stdin, time.Now, objecttransfer.New())
encoder := json.NewEncoder(os.Stdout)
encoder.SetEscapeHTML(true)
if err := encoder.Encode(receipt); err != nil {
os.Exit(2)
}
}
func run(input io.Reader, now func() time.Time, worker objectWorker) replicaReceipt {
if input == nil || now == nil || worker == nil {
return failureReceipt(protocolName, "", "replicator_unavailable",
"not_applied")
}
limited := &io.LimitedReader{R: input, N: maxRequestBytes + 1}
payload, err := io.ReadAll(limited)
if err != nil || limited.N <= 0 || len(payload) == 0 {
return failureReceipt(protocolName, "", "invalid_replica_request",
"not_applied")
}
var envelope struct {
Protocol string `json:"protocol"`
OperationID string `json:"operation_id"`
}
if err := json.Unmarshal(payload, &envelope); err != nil {
return failureReceipt(protocolName, "", "invalid_replica_request",
"not_applied")
}
switch envelope.Protocol {
case protocolName:
var request replicaRequest
if err := decodeStrict(payload, &request); err != nil {
return failureReceipt(protocolName,
safeOperationID(envelope.OperationID),
"invalid_replica_request", "not_applied")
}
return runCopy(request, now, worker)
case verifyProtocolName:
var request verifyRequest
if err := decodeStrict(payload, &request); err != nil {
return failureReceipt(verifyProtocolName,
safeOperationID(envelope.OperationID),
"invalid_replica_verify_request", "not_applied")
}
return runVerify(request, now, worker)
default:
return failureReceipt(protocolName,
safeOperationID(envelope.OperationID),
"invalid_replica_request", "not_applied")
}
}
func runCopy(request replicaRequest, now func() time.Time,
worker objectWorker) replicaReceipt {
operationID := safeOperationID(request.OperationID)
if request.Protocol != protocolName || operationID == "" {
return failureReceipt(protocolName, operationID,
"invalid_replica_request", "not_applied")
}
deadline, ok := validDeadline(now(), request.DeadlineAt)
if !ok {
return failureReceipt(protocolName, operationID,
"invalid_replica_deadline", "not_applied")
}
if err := objecttransfer.ValidateDownload(request.Source.GetURL,
request.Source.SizeBytes, request.Source.SHA256); err != nil {
return failureReceipt(protocolName, operationID,
transferCode(err, "invalid_replica_source"), "not_applied")
}
if err := objecttransfer.ValidateMultipartPlan(request.Destination); err != nil {
return failureReceipt(protocolName, operationID,
transferCode(err, "invalid_replica_destination"), "not_applied")
}
ctx, cancel := context.WithDeadline(context.Background(), deadline)
defer cancel()
result, err := worker.CopyObject(ctx, request.Source, request.Destination)
if err != nil {
resolution := "not_applied"
if objecttransfer.IsAmbiguous(err) {
resolution = "unknown"
}
return failureReceipt(protocolName, operationID,
transferCode(err, "replica_copy_unknown"), resolution)
}
return replicaReceipt{
Protocol: protocolName, OperationID: operationID,
Outcome: "succeeded", EffectResolution: "applied",
SizeBytes: result.SizeBytes, SHA256: result.SHA256,
ETag: result.ETag, VersionID: result.VersionID,
}
}
func runVerify(request verifyRequest, now func() time.Time,
worker objectWorker) replicaReceipt {
operationID := safeOperationID(request.OperationID)
if request.Protocol != verifyProtocolName || operationID == "" {
return failureReceipt(verifyProtocolName, operationID,
"invalid_replica_verify_request", "not_applied")
}
deadline, ok := validDeadline(now(), request.DeadlineAt)
if !ok {
return failureReceipt(verifyProtocolName, operationID,
"invalid_replica_deadline", "not_applied")
}
if err := objecttransfer.ValidateDownload(request.GetURL,
request.SizeBytes, request.SHA256); err != nil {
return failureReceipt(verifyProtocolName, operationID,
transferCode(err, "invalid_replica_verify_source"), "not_applied")
}
ctx, cancel := context.WithDeadline(context.Background(), deadline)
defer cancel()
result, err := worker.Download(ctx, request.GetURL, request.SizeBytes,
request.SHA256, io.Discard)
if err != nil {
resolution := "unknown"
if verificationProvesNotApplied(err) {
resolution = "not_applied"
}
return failureReceipt(verifyProtocolName, operationID,
transferCode(err, "replica_verify_unknown"), resolution)
}
return replicaReceipt{
Protocol: verifyProtocolName, OperationID: operationID,
Outcome: "succeeded", EffectResolution: "applied",
SizeBytes: result.SizeBytes, SHA256: result.SHA256,
ETag: result.ETag, VersionID: result.VersionID,
}
}
func failureReceipt(protocol, operationID, code,
resolution string) replicaReceipt {
return replicaReceipt{
Protocol: protocol, OperationID: operationID,
Outcome: "failed", EffectResolution: resolution, ErrorCode: code,
}
}
func validDeadline(now time.Time, value int64) (time.Time, bool) {
current := now.UTC()
deadline := time.Unix(value, 0).UTC()
return deadline, deadline.After(current) &&
!deadline.After(current.Add(maxDeadline))
}
func decodeStrict(payload []byte, destination any) error {
decoder := json.NewDecoder(bytes.NewReader(payload))
decoder.DisallowUnknownFields()
if err := decoder.Decode(destination); err != nil {
return err
}
var trailing json.RawMessage
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return errors.New("trailing JSON value")
}
return nil
}
func verificationProvesNotApplied(err error) bool {
var transferErr *objecttransfer.Error
if !errors.As(err, &transferErr) {
return false
}
return transferErr.Code == "object_size_mismatch" ||
transferErr.Code == "object_sha256_mismatch"
}
func safeOperationID(value string) string {
if !operationIDPattern.MatchString(value) {
return ""
}
return value
}
func transferCode(err error, fallback string) string {
var transferErr *objecttransfer.Error
if errors.As(err, &transferErr) && transferErr.Code != "" {
return transferErr.Code
}
return fallback
}