You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1010 lines
46 KiB
Python
1010 lines
46 KiB
Python
"""Bound libvirt C calls in disposable processes, without moving business logic.
|
|
|
|
The private socket transports values and session-local libvirt handles only.
|
|
An interrupted write leaves a durable host fence: killing this process is NOT
|
|
proof that libvirtd stopped the operation. Only an operator who has reconciled
|
|
the host may archive that fence while the engine is stopped.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import importlib
|
|
import json
|
|
import logging
|
|
import math
|
|
import multiprocessing
|
|
import os
|
|
import re
|
|
import signal
|
|
import socket
|
|
import struct
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import xml.etree.ElementTree as ET
|
|
from pathlib import Path
|
|
|
|
|
|
_LOGGER = logging.getLogger(__name__)
|
|
EXECUTION_ERROR_CODES = frozenset({
|
|
"libvirt_execution_timeout", "libvirt_execution_uncertain",
|
|
"libvirt_execution_stopped", "libvirt_execution_blocked",
|
|
})
|
|
MAX_FRAME = 32 * 1024 * 1024
|
|
SNAPSHOT_RECOVERY_ERROR = "snapshot_thaw_recovery_required"
|
|
_QGA_RUNTIME_TAG = "{urn:virsh-panel:qga-runtime:1}qga-runtime"
|
|
READ_METHODS = frozenset({
|
|
"ID", "UUID", "UUIDString", "XMLDesc", "name", "path", "info",
|
|
"getName", "getDomain",
|
|
"state", "isActive", "isPersistent", "isUpdated", "hasManagedSaveImage",
|
|
"autostart", "maxMemory", "getInfo", "getType", "getVersion",
|
|
"getLibVersion", "getHostname", "getURI", "getCapabilities",
|
|
"getAllDomainStats", "domainListGetStats", "listAllDomains",
|
|
"listAllStoragePools", "listVolumes", "listAllVolumes",
|
|
"lookupByUUIDString", "lookupByName", "storagePoolLookupByName",
|
|
"storageVolLookupByName", "storageVolLookupByPath", "networkLookupByName",
|
|
"nwfilterLookupByName", "snapshotNum", "blockInfo", "blockStats",
|
|
"interfaceStats", "interfaceAddresses", "jobStats", "jobInfo",
|
|
"getMetadata", "metadata", "backupGetXMLDesc", "refresh",
|
|
"snapshotLookupByName", "snapshotListNames", "listAllSnapshots",
|
|
"agentSetResponseTimeout",
|
|
"newStream", "download", "recv", "close",
|
|
})
|
|
|
|
|
|
class ExecutionFailure(Exception):
|
|
def __init__(self, error_code: str, *, outcome_unknown: bool = False):
|
|
super().__init__(error_code)
|
|
self.error_code = error_code
|
|
self.outcome_unknown = outcome_unknown
|
|
|
|
|
|
class SnapshotRecoveryRequired(Exception):
|
|
"""The durable evidence does not authorize an emergency thaw."""
|
|
|
|
|
|
def _snapshot_thaw_preflight(connection, evidence, backend):
|
|
"""The only fence exception: thaw one proven, previously frozen runtime.
|
|
|
|
No absence observation proves a timed-out backupBegin never arrived.
|
|
A pending begin therefore requires the exact active backup target.
|
|
"""
|
|
domain = connection.lookupByUUIDString(evidence["vm_id"])
|
|
if not domain.isActive():
|
|
return {"thawed": True, "count": 0, "power_state": "stopped"}
|
|
if domain.ID() != evidence["runtime_id"]:
|
|
raise SnapshotRecoveryRequired()
|
|
live = ET.fromstring(domain.XMLDesc(0))
|
|
inactive = ET.fromstring(domain.XMLDesc(backend.VIR_DOMAIN_XML_INACTIVE))
|
|
nodes = live.findall("./metadata/" + _QGA_RUNTIME_TAG)
|
|
if (len(nodes) != 1 or nodes[0].attrib != {
|
|
"domain-uuid": evidence["vm_id"], "token": evidence["runtime_token"]}
|
|
or inactive.find("./metadata/" + _QGA_RUNTIME_TAG) is not None):
|
|
raise SnapshotRecoveryRequired()
|
|
volume = connection.storagePoolLookupByName(
|
|
evidence["pool_name"]).storageVolLookupByName(evidence["volume_name"])
|
|
if volume.path() != evidence["target_path"]:
|
|
raise SnapshotRecoveryRequired()
|
|
stats = domain.jobStats(0)
|
|
if not isinstance(stats, dict) or not isinstance(stats.get("type"), int):
|
|
raise SnapshotRecoveryRequired()
|
|
active_backup = stats["type"] != backend.VIR_DOMAIN_JOB_NONE
|
|
if active_backup:
|
|
if (evidence["backup_state"] == "not_sent"
|
|
or stats.get("operation") != backend.VIR_DOMAIN_JOB_OPERATION_BACKUP):
|
|
raise SnapshotRecoveryRequired()
|
|
backup = ET.fromstring(domain.backupGetXMLDesc(0))
|
|
all_disks = backup.findall("./disks/disk")
|
|
disks = [disk for disk in all_disks if disk.get("backup", "yes") != "no"]
|
|
if (backup.tag != "domainbackup" or backup.get("mode", "push") != "push"
|
|
or len(disks) != 1 or disks[0].get("name") != "vda"
|
|
or disks[0].get("backup", "yes") != "yes"
|
|
or disks[0].get("type", "file") != "file"
|
|
or disks[0].find("target") is None
|
|
or disks[0].find("target").get("file") != evidence["target_path"]
|
|
or any(disk.find("target") is not None or disk.find("scratch") is not None
|
|
for disk in all_disks if disk.get("backup") == "no")):
|
|
raise SnapshotRecoveryRequired()
|
|
if evidence["backup_state"] == "pending" and not active_backup:
|
|
raise SnapshotRecoveryRequired()
|
|
# The last check is immediately before the sole permitted mutation.
|
|
if (not domain.isActive() or domain.ID() != evidence["runtime_id"]
|
|
or ET.fromstring(domain.XMLDesc(0)).find(
|
|
"./metadata/" + _QGA_RUNTIME_TAG).get("token")
|
|
!= evidence["runtime_token"]):
|
|
raise SnapshotRecoveryRequired()
|
|
return domain
|
|
|
|
|
|
def _recover_snapshot_thaw(connection, evidence, backend):
|
|
try:
|
|
domain = _snapshot_thaw_preflight(connection, evidence, backend)
|
|
except (ET.ParseError, AttributeError, KeyError, TypeError, backend.libvirtError):
|
|
raise SnapshotRecoveryRequired() from None
|
|
if isinstance(domain, dict):
|
|
return domain
|
|
# From here onward any exception or invalid result is outcome-unknown.
|
|
# It must not reset the durable recovery attempt to "not_started".
|
|
count = domain.fsThaw(None, 0)
|
|
if isinstance(count, bool) or not isinstance(count, int) or count < 0:
|
|
raise RuntimeError("unconfirmed emergency thaw result")
|
|
return {"thawed": True, "count": count, "power_state": "running"}
|
|
|
|
|
|
def _send(sock, value, deadline):
|
|
payload = json.dumps(value, ensure_ascii=True, allow_nan=False).encode()
|
|
if len(payload) > MAX_FRAME:
|
|
raise ValueError("libvirt RPC message too large")
|
|
sock.settimeout(max(0.001, deadline - time.monotonic()))
|
|
sock.sendall(struct.pack("!I", len(payload)) + payload)
|
|
|
|
|
|
def _receive(sock, deadline):
|
|
def exact(size):
|
|
chunks = bytearray()
|
|
while len(chunks) < size:
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise TimeoutError("libvirt RPC deadline")
|
|
sock.settimeout(remaining)
|
|
part = sock.recv(min(size - len(chunks), 65536))
|
|
if not part:
|
|
raise EOFError("libvirt RPC closed")
|
|
chunks.extend(part)
|
|
return bytes(chunks)
|
|
size = struct.unpack("!I", exact(4))[0]
|
|
if size > MAX_FRAME:
|
|
raise ValueError("libvirt RPC message too large")
|
|
return json.loads(exact(size))
|
|
|
|
|
|
def _encode(value, handle):
|
|
if value is None or isinstance(value, (bool, int, float, str)):
|
|
return value
|
|
if isinstance(value, bytes):
|
|
return {"bytes": base64.b64encode(value).decode("ascii")}
|
|
if isinstance(value, (list, tuple)):
|
|
return {"tuple" if isinstance(value, tuple) else "list": [
|
|
_encode(item, handle) for item in value]}
|
|
if isinstance(value, dict):
|
|
return {"dict": [[_encode(k, handle), _encode(v, handle)]
|
|
for k, v in value.items()]}
|
|
return handle(value)
|
|
|
|
|
|
def _decode(value, handle):
|
|
if not isinstance(value, dict):
|
|
return value
|
|
if "handle" in value:
|
|
return handle(value)
|
|
if "bytes" in value:
|
|
return base64.b64decode(value["bytes"], validate=True)
|
|
if "tuple" in value:
|
|
return tuple(_decode(v, handle) for v in value["tuple"])
|
|
if "list" in value:
|
|
return [_decode(v, handle) for v in value["list"]]
|
|
if "dict" in value:
|
|
return {_decode(k, handle): _decode(v, handle) for k, v in value["dict"]}
|
|
raise ValueError("invalid libvirt RPC value")
|
|
|
|
|
|
def _worker(sock, uri, backend_module):
|
|
# spawn avoids inheriting libvirt connections and parent thread locks.
|
|
# A separate session lets the supervisor reap a stuck SSH child as well.
|
|
os.setsid()
|
|
_send(sock, {"ready": os.getpid()}, time.monotonic() + 30)
|
|
backend = importlib.import_module(backend_module)
|
|
objects = {}
|
|
identities = {}
|
|
|
|
def export(value):
|
|
if not type(value).__name__.startswith("vir"):
|
|
raise TypeError("unsupported libvirt RPC result")
|
|
key = identities.get(id(value))
|
|
if key is None:
|
|
key = len(objects) + 1
|
|
objects[key] = value
|
|
identities[id(value)] = key
|
|
methods = [name for name in dir(value)
|
|
if not name.startswith("_") and callable(getattr(value, name, None))]
|
|
return {"handle": key, "methods": methods}
|
|
|
|
def resolve(value):
|
|
return objects[value["handle"]]
|
|
|
|
try:
|
|
while True:
|
|
request = _receive(sock, time.monotonic() + 86400)
|
|
try:
|
|
if request["op"] == "open":
|
|
result = backend.open(uri)
|
|
if result is None:
|
|
raise RuntimeError("libvirt returned no connection")
|
|
elif request["op"] == "call":
|
|
obj = objects[request["handle"]]
|
|
method = request["method"]
|
|
if not isinstance(method, str) or method.startswith("_"):
|
|
raise ValueError("invalid libvirt method")
|
|
result = getattr(obj, method)(
|
|
*_decode(request["args"], resolve),
|
|
**_decode(request["kwargs"], resolve))
|
|
elif request["op"] == "qga":
|
|
qemu = (importlib.import_module("libvirt_qemu")
|
|
if backend_module == "libvirt" else backend)
|
|
result = qemu.qemuAgentCommand(
|
|
*_decode(request["args"], resolve))
|
|
elif request["op"] == "snapshot_thaw":
|
|
result = _recover_snapshot_thaw(
|
|
objects[request["handle"]], request["evidence"], backend)
|
|
else:
|
|
raise ValueError("invalid libvirt RPC operation")
|
|
response = {"result": _encode(result, export)}
|
|
except backend.libvirtError as exc:
|
|
response = {"libvirt_error": str(exc), "err": getattr(exc, "err", None)}
|
|
except SnapshotRecoveryRequired:
|
|
response = {"snapshot_recovery_required": True}
|
|
except Exception:
|
|
# Never expose arbitrary exception text/arguments or credentials.
|
|
response = {"worker_error": True}
|
|
_send(sock, response, time.monotonic() + 30)
|
|
except (EOFError, OSError, ValueError):
|
|
pass
|
|
finally:
|
|
sock.close()
|
|
|
|
|
|
class RemoteObject:
|
|
def __init__(self, session, reference):
|
|
self._session = session
|
|
self._handle = reference["handle"]
|
|
self._methods = frozenset(reference["methods"])
|
|
|
|
def __getattr__(self, method):
|
|
if method.startswith("_") or method not in self._methods:
|
|
raise AttributeError(method)
|
|
def call(*args, **kwargs):
|
|
return self._session.call(self._handle, method, args, kwargs)
|
|
return call
|
|
|
|
|
|
class LibvirtRuntime:
|
|
def __init__(self, state_dir: Path, *, rpc_timeout=120.0,
|
|
connect_timeout=20.0, queue_timeout=5.0,
|
|
max_connections=32, max_per_host=4, backend_module="libvirt"):
|
|
if any(not math.isfinite(value) or value <= 0 for value in
|
|
(rpc_timeout, connect_timeout, queue_timeout,
|
|
max_connections, max_per_host)):
|
|
raise ValueError("libvirt execution limits must be positive")
|
|
if any(not isinstance(value, int) for value in (max_connections, max_per_host)):
|
|
raise ValueError("libvirt execution capacity must be an integer")
|
|
self.state_dir = Path(state_dir)
|
|
self._ensure_directory(self.state_dir)
|
|
self.rpc_timeout = rpc_timeout
|
|
self.connect_timeout = connect_timeout
|
|
self.queue_timeout = queue_timeout
|
|
self.max_connections = max_connections
|
|
self.max_per_host = max_per_host
|
|
self.backend_module = backend_module # constructor-only test dependency
|
|
self._condition = threading.Condition()
|
|
self._sessions = set()
|
|
self._reserved = {}
|
|
self._closing = False
|
|
self._snapshot_recoveries = set()
|
|
|
|
def _directory(self, scope):
|
|
directory = self.state_dir / hashlib.sha256(scope.encode()).hexdigest()
|
|
self._ensure_directory(directory)
|
|
return directory
|
|
|
|
@classmethod
|
|
def _ensure_directory(cls, directory):
|
|
try:
|
|
directory.mkdir(mode=0o700)
|
|
except FileNotFoundError:
|
|
cls._ensure_directory(directory.parent)
|
|
directory.mkdir(mode=0o700)
|
|
except FileExistsError:
|
|
if directory.is_symlink() or not directory.is_dir():
|
|
raise ValueError("unsafe libvirt execution directory")
|
|
return
|
|
# The intent's fsync cannot persist the containing directory's own
|
|
# entry in its parent. Persist newly created levels before any write.
|
|
cls._sync(directory.parent)
|
|
|
|
@staticmethod
|
|
def _sync(directory):
|
|
fd = os.open(directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
|
try:
|
|
os.fsync(fd)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
def _before_write(self, session, method):
|
|
from .coordination import mark_remote_write_started
|
|
|
|
with self._condition:
|
|
self._write_admission(session)
|
|
if session.pending_write is not None:
|
|
raise session._fail("libvirt_execution_uncertain", outcome_unknown=True)
|
|
session.write_sequence += 1
|
|
session.pending_write = {"sequence": session.write_sequence, "method": method}
|
|
try:
|
|
self._persist_write_record(session)
|
|
except Exception:
|
|
raise session._fail("libvirt_execution_uncertain", outcome_unknown=True) from None
|
|
mark_remote_write_started()
|
|
return session.write_sequence
|
|
|
|
def _write_admission(self, session):
|
|
from .coordination import mark_admission_rejected
|
|
|
|
if session.failure is not None:
|
|
raise session._fail(session.failure.error_code,
|
|
outcome_unknown=session.failure.outcome_unknown)
|
|
if self._closing or session._closed:
|
|
raise ExecutionFailure("libvirt_execution_stopped")
|
|
active = {s.token for s in self._sessions if s.failure is None}
|
|
if (session.scope in self._snapshot_recoveries
|
|
or any(path.stem not in active
|
|
for path in self._directory(session.scope).glob("*.json"))):
|
|
mark_admission_rejected()
|
|
raise ExecutionFailure("libvirt_execution_blocked", outcome_unknown=True)
|
|
|
|
def _persist_write_record(self, session):
|
|
record = {"version": 2, "scope": session.scope, "session": session.token,
|
|
"sequence": session.write_sequence,
|
|
"pending_write": session.pending_write,
|
|
"last_write_ack": session.last_write_ack,
|
|
"started_at": int(time.time())}
|
|
if session.write_error_receipt is not None:
|
|
record["write_error_receipt"] = session.write_error_receipt
|
|
if session.snapshot_evidence is not None:
|
|
record["snapshot_recovery"] = session.snapshot_evidence
|
|
if session.intent is not None:
|
|
self._replace_snapshot_record(session.intent, record)
|
|
return
|
|
session.intent = self._directory(session.scope) / (session.token + ".json")
|
|
fd = os.open(session.intent, os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
|
| getattr(os, "O_NOFOLLOW", 0), 0o600)
|
|
with os.fdopen(fd, "wb") as stream:
|
|
stream.write(json.dumps(record).encode())
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
self._sync(session.intent.parent)
|
|
|
|
def _clear_intent(self, session):
|
|
"""Durably acknowledge this receipt; restore the fence on fsync failure."""
|
|
# Caller holds _condition through deletion, sync and restoration.
|
|
path, original = session.intent, None
|
|
try:
|
|
original = path.read_bytes()
|
|
path.unlink()
|
|
self._sync(path.parent)
|
|
except Exception:
|
|
try:
|
|
if original is not None and not path.exists():
|
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
|
| getattr(os, "O_NOFOLLOW", 0), 0o600)
|
|
with os.fdopen(fd, "wb") as stream:
|
|
stream.write(original)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
self._sync(path.parent)
|
|
if not path.is_file():
|
|
raise OSError("libvirt execution fence unavailable")
|
|
except Exception:
|
|
self._closing = True
|
|
self._condition.notify_all()
|
|
raise session._fail("libvirt_execution_uncertain", outcome_unknown=True) from None
|
|
session.intent = None
|
|
|
|
def _ack_write(self, session, sequence):
|
|
with self._condition:
|
|
if (session.failure is not None or self._closing
|
|
or session.pending_write is None
|
|
or session.pending_write["sequence"] != sequence):
|
|
raise session._fail("libvirt_execution_uncertain", outcome_unknown=True)
|
|
if session.snapshot_evidence is None and session.write_error_receipt is None:
|
|
# Absence is persisted before returning the successful RPC.
|
|
# A later call creates a fresh pending record, even in this
|
|
# same connection. No old intent is inspected or removed.
|
|
self._clear_intent(session)
|
|
else:
|
|
# Freeze recovery outlives individual RPC acknowledgements.
|
|
# Keep its identity/stages independently of the pending call.
|
|
try:
|
|
record = json.loads(session.intent.read_text())
|
|
record.update(pending_write=None, last_write_ack=sequence)
|
|
self._replace_snapshot_record(session.intent, record)
|
|
except Exception:
|
|
raise session._fail("libvirt_execution_uncertain", outcome_unknown=True) from None
|
|
session.last_write_ack = sequence
|
|
session.pending_write = None
|
|
|
|
def _record_write_error(self, session):
|
|
"""Keep a returned native error distinct from a success or lost reply.
|
|
|
|
Existing callers may inspect/compensate a libvirt rejection in this
|
|
same connection. As before, only a clean end to that workflow removes
|
|
its obligation; a later success must not overwrite the earlier error.
|
|
"""
|
|
with self._condition:
|
|
try:
|
|
record = json.loads(session.intent.read_text())
|
|
error_receipt = session.write_error_receipt or dict(session.pending_write)
|
|
record.update(pending_write=None, write_error_receipt=error_receipt)
|
|
self._replace_snapshot_record(session.intent, record)
|
|
except Exception:
|
|
raise session._fail("libvirt_execution_uncertain", outcome_unknown=True) from None
|
|
session.write_error_receipt = error_receipt
|
|
session.pending_write = None
|
|
|
|
def _save_snapshot_evidence(self, session, evidence):
|
|
"""Persist a conservative phase before sending the corresponding RPC."""
|
|
with self._condition:
|
|
if session.intent is None or session.failure is not None or self._closing:
|
|
raise SnapshotRecoveryRequired()
|
|
try:
|
|
original = json.loads(session.intent.read_text())
|
|
original["snapshot_recovery"] = evidence
|
|
self._replace_snapshot_record(session.intent, original)
|
|
except Exception:
|
|
session._fail("libvirt_execution_uncertain", outcome_unknown=True)
|
|
raise
|
|
session.snapshot_evidence = evidence
|
|
|
|
def _replace_snapshot_record(self, path, record):
|
|
temporary = path.with_suffix(".snapshot-tmp")
|
|
try:
|
|
with temporary.open("x", encoding="utf-8") as stream:
|
|
os.chmod(temporary, 0o600)
|
|
json.dump(record, stream)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(temporary, path)
|
|
self._sync(path.parent)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
def _snapshot_recovery_evidence(self, session, job_id, vm_id, volume_name):
|
|
"""Authorize only a bound job whose old worker can send no more RPCs."""
|
|
with self._condition:
|
|
if self._closing or session.scope in self._snapshot_recoveries:
|
|
raise SnapshotRecoveryRequired()
|
|
files = list(self._directory(session.scope).glob("*.json"))
|
|
# Another unresolved write may include a freeze we cannot identify.
|
|
if len(files) != 1 or files[0].stat().st_size > 16384:
|
|
raise SnapshotRecoveryRequired()
|
|
try:
|
|
record = json.loads(files[0].read_text())
|
|
except (ValueError, OSError):
|
|
raise SnapshotRecoveryRequired() from None
|
|
if not isinstance(record, dict):
|
|
raise SnapshotRecoveryRequired()
|
|
evidence = record.get("snapshot_recovery")
|
|
if (record.get("scope") != session.scope
|
|
or not isinstance(evidence, dict)
|
|
or evidence.get("job_id") != job_id
|
|
or evidence.get("vm_id") != vm_id
|
|
or evidence.get("volume_name") != volume_name
|
|
or evidence.get("freeze_ack") is not True
|
|
or not isinstance(evidence.get("thaw_ack"), bool)
|
|
or evidence.get("thaw_pending") is not False
|
|
or evidence.get("recovery_state") not in {"not_started", "ack"}
|
|
or evidence.get("backup_state") not in {"not_sent", "pending", "ack"}
|
|
or not isinstance(evidence.get("worker_pid"), int)
|
|
or isinstance(evidence["worker_pid"], bool)
|
|
or evidence["worker_pid"] <= 0
|
|
or not isinstance(evidence.get("runtime_id"), int)
|
|
or isinstance(evidence["runtime_id"], bool)
|
|
or evidence["runtime_id"] < 0
|
|
or not isinstance(evidence.get("runtime_token"), str)
|
|
or re.fullmatch(r"[0-9a-f]{32}", evidence["runtime_token"]) is None
|
|
or not isinstance(evidence.get("pool_name"), str)
|
|
or not evidence["pool_name"]
|
|
or not isinstance(evidence.get("target_path"), str)
|
|
or not evidence["target_path"].startswith("/")):
|
|
raise SnapshotRecoveryRequired()
|
|
for other in self._sessions:
|
|
if other is session or other.scope != session.scope:
|
|
continue
|
|
if other.intent is not None and (other.process is None
|
|
or other.process.is_alive()):
|
|
raise SnapshotRecoveryRequired()
|
|
try:
|
|
os.kill(evidence["worker_pid"], 0)
|
|
except ProcessLookupError:
|
|
pass
|
|
except OSError:
|
|
raise SnapshotRecoveryRequired() from None
|
|
else:
|
|
# PID reuse errs towards manual recovery, never false admission.
|
|
raise SnapshotRecoveryRequired()
|
|
self._snapshot_recoveries.add(session.scope)
|
|
return files[0], record, evidence
|
|
|
|
def open(self, uri, scope, *, timeout=180.0):
|
|
if (not isinstance(scope, str) or not scope or len(scope) > 256
|
|
or not math.isfinite(timeout) or timeout <= 0):
|
|
raise ValueError("invalid libvirt execution scope/budget")
|
|
request_deadline = time.monotonic() + timeout
|
|
deadline = min(
|
|
request_deadline, time.monotonic() + self.queue_timeout)
|
|
with self._condition:
|
|
while (sum(self._reserved.values()) >= self.max_connections
|
|
or self._reserved.get(scope, 0) >= self.max_per_host):
|
|
if self._closing:
|
|
raise ExecutionFailure("libvirt_execution_stopped")
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise ExecutionFailure("libvirt_execution_timeout")
|
|
self._condition.wait(remaining)
|
|
if self._closing:
|
|
raise ExecutionFailure("libvirt_execution_stopped")
|
|
self._reserved[scope] = self._reserved.get(scope, 0) + 1
|
|
session = None
|
|
try:
|
|
remaining = request_deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise ExecutionFailure("libvirt_execution_timeout")
|
|
session = RPCSession(self, uri, scope, remaining)
|
|
with self._condition:
|
|
self._sessions.add(session)
|
|
if self._closing:
|
|
session._fail("libvirt_execution_stopped")
|
|
session.start()
|
|
return session
|
|
except BaseException:
|
|
if session is not None:
|
|
session.close()
|
|
else:
|
|
self._release(scope, None)
|
|
raise
|
|
|
|
def _release(self, scope, session):
|
|
with self._condition:
|
|
self._sessions.discard(session)
|
|
remaining = self._reserved.get(scope, 1) - 1
|
|
if remaining:
|
|
self._reserved[scope] = remaining
|
|
else:
|
|
self._reserved.pop(scope, None)
|
|
self._condition.notify_all()
|
|
|
|
def begin_shutdown(self):
|
|
with self._condition:
|
|
self._closing = True
|
|
sessions = tuple(self._sessions)
|
|
self._condition.notify_all()
|
|
for session in sessions:
|
|
session._fail("libvirt_execution_stopped")
|
|
try:
|
|
session.interrupt()
|
|
except Exception:
|
|
session._fail("libvirt_execution_stopped")
|
|
_LOGGER.warning("Unable to interrupt libvirt worker (pid=%s)", session.pid)
|
|
|
|
def close(self, timeout=3.0):
|
|
self.begin_shutdown()
|
|
deadline = time.monotonic() + max(0, timeout)
|
|
with self._condition:
|
|
sessions = tuple(self._sessions)
|
|
reaped = True
|
|
for session in sessions:
|
|
try:
|
|
session.reap(deadline)
|
|
except Exception:
|
|
session._fail("libvirt_execution_stopped")
|
|
_LOGGER.warning("Unable to reap libvirt worker (pid=%s)", session.pid)
|
|
reaped = False
|
|
return reaped and all(s.process is None or not s.process.is_alive() for s in sessions)
|
|
|
|
|
|
class RPCSession:
|
|
def __init__(self, runtime, uri, scope, timeout):
|
|
self.runtime, self.uri, self.scope = runtime, uri, scope
|
|
self.deadline = time.monotonic() + timeout
|
|
self.token = uuid.uuid4().hex
|
|
self.intent = None
|
|
self.pending_write = None
|
|
self.write_error_receipt = None
|
|
self.write_sequence = 0
|
|
self.last_write_ack = 0
|
|
self.failure = None
|
|
self.cleanup_failure = None
|
|
self._cleaning_up = False
|
|
self.process = None
|
|
self.sock = None
|
|
self.connection = None
|
|
self._closed = False
|
|
self._io_lock = threading.RLock()
|
|
self._pgid = None
|
|
self._signal_lock = threading.Lock()
|
|
self._sent_signals = set()
|
|
self.snapshot_evidence = None
|
|
self._snapshot_handle = None
|
|
# Stream completion commits uploads, but only closes downloads. Keep
|
|
# the direction in the supervisor so a lost download finish/abort
|
|
# reply cannot create a false host write fence.
|
|
self._stream_modes = {}
|
|
|
|
@property
|
|
def pid(self):
|
|
return self.process.pid if self.process is not None else None
|
|
|
|
def _fail(self, code, *, outcome_unknown=False):
|
|
from .coordination import execution_failure_before_write, mark_execution_unknown
|
|
|
|
# Intent persistence and shutdown share this condition. A later local
|
|
# stop/error must never downgrade evidence of an unconfirmed write.
|
|
with self.runtime._condition:
|
|
previous = self.failure
|
|
if (self._cleaning_up and self.pending_write is None
|
|
and self.write_error_receipt is None
|
|
and previous is None and not outcome_unknown):
|
|
self.cleanup_failure = ExecutionFailure(code)
|
|
return self.cleanup_failure
|
|
if self.pending_write is not None or self.write_error_receipt is not None:
|
|
code, outcome_unknown = "libvirt_execution_uncertain", True
|
|
elif previous is not None and (previous.outcome_unknown or previous.error_code in {
|
|
"libvirt_execution_uncertain", "libvirt_execution_blocked"}):
|
|
return previous
|
|
elif code in {"libvirt_execution_uncertain", "libvirt_execution_blocked"}:
|
|
outcome_unknown = True
|
|
elif outcome_unknown:
|
|
code = "libvirt_execution_uncertain"
|
|
elif previous is not None:
|
|
return previous
|
|
if (previous is None or previous.error_code != code
|
|
or previous.outcome_unknown != outcome_unknown):
|
|
self.failure = ExecutionFailure(code, outcome_unknown=outcome_unknown)
|
|
if outcome_unknown and not (code == "libvirt_execution_blocked"
|
|
and execution_failure_before_write(code)):
|
|
mark_execution_unknown()
|
|
return self.failure
|
|
|
|
def start(self):
|
|
with self.runtime._condition:
|
|
if self.failure is not None or self.runtime._closing:
|
|
raise self._fail("libvirt_execution_stopped")
|
|
self.sock, child = socket.socketpair()
|
|
try:
|
|
self.process = multiprocessing.get_context("spawn").Process(
|
|
target=_worker, args=(child, self.uri, self.runtime.backend_module),
|
|
daemon=True, name="libvirt-rpc")
|
|
self.process.start()
|
|
finally:
|
|
child.close()
|
|
try:
|
|
ready = _receive(self.sock, min(self.deadline,
|
|
time.monotonic() + self.runtime.connect_timeout))
|
|
if ready != {"ready": self.process.pid}:
|
|
raise ValueError("invalid libvirt worker")
|
|
self._pgid = self.process.pid
|
|
except (OSError, EOFError, ValueError):
|
|
raise self._fail("libvirt_execution_timeout") from None
|
|
self.connection = self._request({"op": "open"}, self.runtime.connect_timeout)
|
|
|
|
def _reference(self, value):
|
|
if not isinstance(value, RemoteObject) or value._session is not self:
|
|
raise ValueError("libvirt handles cannot cross sessions")
|
|
return {"handle": value._handle}
|
|
|
|
def _request(self, request, timeout=None):
|
|
with self._io_lock:
|
|
if self.failure is not None:
|
|
raise self._fail(self.failure.error_code,
|
|
outcome_unknown=self.failure.outcome_unknown)
|
|
if self._closed:
|
|
raise self._fail("libvirt_execution_stopped")
|
|
deadline = min(self.deadline, time.monotonic() + (
|
|
self.runtime.rpc_timeout if timeout is None else timeout))
|
|
try:
|
|
if time.monotonic() >= deadline:
|
|
raise TimeoutError("libvirt RPC deadline")
|
|
_send(self.sock, request, deadline)
|
|
response = _receive(self.sock, deadline)
|
|
except (OSError, EOFError, ValueError):
|
|
failure = self._fail("libvirt_execution_timeout")
|
|
self.interrupt()
|
|
raise failure from None
|
|
if self.failure is not None:
|
|
raise self._fail(self.failure.error_code,
|
|
outcome_unknown=self.failure.outcome_unknown)
|
|
if not isinstance(response, dict):
|
|
raise self._fail("libvirt_execution_timeout")
|
|
if "libvirt_error" in response:
|
|
if self.pending_write is not None:
|
|
self.runtime._record_write_error(self)
|
|
backend = importlib.import_module(self.runtime.backend_module)
|
|
error = backend.libvirtError(response["libvirt_error"])
|
|
if response.get("err") is not None:
|
|
error.err = tuple(response["err"])
|
|
raise error
|
|
if response.get("snapshot_recovery_required") is True:
|
|
raise SnapshotRecoveryRequired()
|
|
if "result" not in response:
|
|
failure = self._fail("libvirt_execution_timeout")
|
|
self.interrupt()
|
|
raise failure
|
|
try:
|
|
return _decode(response["result"], lambda value: RemoteObject(self, value))
|
|
except (ValueError, TypeError, KeyError):
|
|
raise self._fail("libvirt_execution_timeout") from None
|
|
|
|
def call(self, handle, method, args, kwargs):
|
|
# Serialize intent publication, dispatch, decoding and the matching
|
|
# acknowledgement. A concurrent caller must never clear our pending
|
|
# write or allow a read to consume its reply.
|
|
with self._io_lock:
|
|
return self._call(handle, method, args, kwargs)
|
|
|
|
def _call(self, handle, method, args, kwargs):
|
|
if self.failure is not None:
|
|
raise self._fail(self.failure.error_code,
|
|
outcome_unknown=self.failure.outcome_unknown)
|
|
request = {"op": "call", "handle": handle, "method": method,
|
|
"args": _encode(args, self._reference),
|
|
"kwargs": _encode(kwargs, self._reference)}
|
|
if method in {"upload", "download"} and args:
|
|
stream = args[0]
|
|
if isinstance(stream, RemoteObject) and stream._session is self:
|
|
self._stream_modes[stream._handle] = method
|
|
read_download_completion = (
|
|
method in {"finish", "abort"}
|
|
and self._stream_modes.get(handle) == "download")
|
|
sequence = None
|
|
if method not in READ_METHODS and not read_download_completion:
|
|
try:
|
|
sequence = self.runtime._before_write(self, method)
|
|
except ExecutionFailure as exc:
|
|
raise self._fail(
|
|
exc.error_code, outcome_unknown=exc.outcome_unknown) from None
|
|
snapshot_call = self.snapshot_evidence is not None and handle == self._snapshot_handle
|
|
if snapshot_call and method in {"fsFreeze", "backupBegin", "fsThaw"}:
|
|
evidence = dict(self.snapshot_evidence)
|
|
if method == "fsFreeze":
|
|
# Never permit a second freeze under the same recovery identity.
|
|
if evidence["freeze_ack"]:
|
|
raise SnapshotRecoveryRequired()
|
|
elif method == "backupBegin":
|
|
if not evidence["freeze_ack"] or evidence["backup_state"] != "not_sent":
|
|
raise SnapshotRecoveryRequired()
|
|
evidence["backup_state"] = "pending"
|
|
else:
|
|
evidence["thaw_pending"] = True
|
|
self.runtime._save_snapshot_evidence(self, evidence)
|
|
# Full disk cloning is one synchronous libvirt RPC. Its bound is the
|
|
# storage operation budget, not the short per-query budget.
|
|
result = self._request(request, timeout=(
|
|
max(0, self.deadline - time.monotonic())
|
|
if method == "createXMLFrom" else None))
|
|
if snapshot_call and method in {"fsFreeze", "backupBegin", "fsThaw"}:
|
|
evidence = dict(self.snapshot_evidence)
|
|
if method == "fsFreeze" and isinstance(result, int) and not isinstance(result, bool) and result > 0:
|
|
evidence["freeze_ack"] = True
|
|
elif method == "backupBegin" and result in (None, 0):
|
|
evidence["backup_state"] = "ack"
|
|
elif method == "fsThaw" and isinstance(result, int) and not isinstance(result, bool) and result >= 0:
|
|
evidence["thaw_ack"] = True
|
|
evidence["thaw_pending"] = False
|
|
self.runtime._save_snapshot_evidence(self, evidence)
|
|
if sequence is not None:
|
|
self.runtime._ack_write(self, sequence)
|
|
if method in {"finish", "abort"}:
|
|
self._stream_modes.pop(handle, None)
|
|
return result
|
|
|
|
def bind_snapshot_recovery(self, domain, *, job_id, volume_name, pool_name,
|
|
target_path, runtime_token):
|
|
if (not isinstance(job_id, str) or str(uuid.UUID(job_id)) != job_id
|
|
or not isinstance(runtime_token, str)
|
|
or re.fullmatch(r"[0-9a-f]{32}", runtime_token) is None
|
|
or self.snapshot_evidence is not None):
|
|
raise SnapshotRecoveryRequired()
|
|
self._reference(domain)
|
|
evidence = {
|
|
"job_id": job_id, "vm_id": domain.UUIDString(),
|
|
"volume_name": volume_name, "pool_name": pool_name,
|
|
"target_path": target_path, "runtime_id": domain.ID(),
|
|
"runtime_token": runtime_token, "worker_pid": self.pid,
|
|
"freeze_ack": False, "backup_state": "not_sent", "thaw_ack": False,
|
|
"thaw_pending": False, "recovery_state": "not_started",
|
|
}
|
|
with self._io_lock, self.runtime._condition:
|
|
self.runtime._write_admission(self)
|
|
if self.pending_write is not None:
|
|
raise SnapshotRecoveryRequired()
|
|
self.snapshot_evidence = evidence
|
|
try:
|
|
self.runtime._persist_write_record(self)
|
|
except Exception:
|
|
raise self._fail("libvirt_execution_uncertain", outcome_unknown=True) from None
|
|
self._snapshot_handle = domain._handle
|
|
|
|
def recover_snapshot_thaw(self, *, job_id, vm_id, volume_name):
|
|
path, record, evidence = self.runtime._snapshot_recovery_evidence(
|
|
self, job_id, vm_id, volume_name)
|
|
try:
|
|
if evidence["thaw_ack"]:
|
|
# A durable receipt never authorizes thawing a later freeze.
|
|
return {"thawed": True, "count": 0, "power_state": "unknown"}
|
|
evidence["recovery_state"] = "pending"
|
|
self.runtime._replace_snapshot_record(path, record)
|
|
# All other RPCs on this connection remain subject to the fence.
|
|
# The original intent survives even if this one operation succeeds.
|
|
try:
|
|
result = self._request({"op": "snapshot_thaw",
|
|
"handle": self.connection._handle,
|
|
"evidence": evidence})
|
|
except SnapshotRecoveryRequired:
|
|
# The worker only returns this response before fsThaw begins.
|
|
evidence["recovery_state"] = "not_started"
|
|
self.runtime._replace_snapshot_record(path, record)
|
|
raise
|
|
evidence.update(thaw_ack=True, recovery_state="ack")
|
|
self.runtime._replace_snapshot_record(path, record)
|
|
return result
|
|
except (OSError, ExecutionFailure):
|
|
# The dedicated recovery session has no ordinary write intent of
|
|
# its own. Its persisted pending receipt still means fsThaw may
|
|
# have reached the guest, so a transport failure is outcome-unknown.
|
|
raise self._fail("libvirt_execution_uncertain", outcome_unknown=True) from None
|
|
finally:
|
|
with self.runtime._condition:
|
|
self.runtime._snapshot_recoveries.discard(self.scope)
|
|
|
|
def qemu_agent_command(self, domain, command, timeout, flags):
|
|
with self._io_lock:
|
|
return self._qemu_agent_command(domain, command, timeout, flags)
|
|
|
|
def _qemu_agent_command(self, domain, command, timeout, flags):
|
|
if self.failure is not None:
|
|
raise self._fail(self.failure.error_code,
|
|
outcome_unknown=self.failure.outcome_unknown)
|
|
# guest-exec and other QGA commands can mutate the guest. Do not infer
|
|
# read-only behavior from an opaque JSON command supplied by a caller.
|
|
try:
|
|
command_name = json.loads(command).get("execute")
|
|
except (TypeError, ValueError, AttributeError):
|
|
command_name = None
|
|
request = {"op": "qga", "args": _encode(
|
|
(domain, command, timeout, flags), self._reference)}
|
|
sequence = None
|
|
if command_name not in {"guest-info", "guest-ping", "guest-network-get-interfaces",
|
|
"guest-exec-status", "guest-fsfreeze-status"}:
|
|
try:
|
|
sequence = self.runtime._before_write(self, "qemuAgentCommand")
|
|
except ExecutionFailure as exc:
|
|
raise self._fail(
|
|
exc.error_code, outcome_unknown=exc.outcome_unknown) from None
|
|
result = self._request(request, timeout=min(self.runtime.rpc_timeout, max(1, timeout) + 2))
|
|
if sequence is not None:
|
|
self.runtime._ack_write(self, sequence)
|
|
return result
|
|
|
|
def interrupt(self):
|
|
if self.sock is not None:
|
|
try:
|
|
self.sock.shutdown(socket.SHUT_RDWR)
|
|
except OSError:
|
|
pass
|
|
try:
|
|
self._signal(signal.SIGTERM)
|
|
except OSError:
|
|
self._fail("libvirt_execution_stopped")
|
|
_LOGGER.warning("Unable to signal libvirt worker (pid=%s)", self.pid)
|
|
|
|
def _signal(self, sig):
|
|
with self._signal_lock:
|
|
process = self.process
|
|
if process is None or process.pid is None or sig in self._sent_signals:
|
|
return
|
|
exited = process.exitcode # reap a zombie before signalling its group
|
|
try:
|
|
if self._pgid is not None:
|
|
# The handshake proves this is our private group. The leader
|
|
# may exit on TERM while an SSH descendant still needs KILL.
|
|
os.killpg(self._pgid, sig)
|
|
elif exited is None:
|
|
os.kill(process.pid, sig)
|
|
except ProcessLookupError:
|
|
pass
|
|
except PermissionError:
|
|
# Darwin can report EPERM while a signalled group leader is
|
|
# exiting, before waitpid observes it. Reap for a bounded time,
|
|
# then retry; a live group permission failure remains an error.
|
|
process.join(timeout=0.05)
|
|
if self._pgid is None or process.exitcode is None:
|
|
raise
|
|
try:
|
|
os.killpg(self._pgid, sig)
|
|
except ProcessLookupError:
|
|
pass
|
|
self._sent_signals.add(sig)
|
|
|
|
def reap(self, deadline):
|
|
process = self.process
|
|
if process is None or process.pid is None:
|
|
return
|
|
self._signal(signal.SIGTERM)
|
|
process.join(timeout=min(0.1, max(0, deadline - time.monotonic())))
|
|
self._signal(signal.SIGKILL)
|
|
process.join(timeout=max(0, deadline - time.monotonic()))
|
|
|
|
def close(self):
|
|
with self._io_lock:
|
|
self._close()
|
|
|
|
def _close(self):
|
|
if self._closed:
|
|
return
|
|
reaped = False
|
|
self._cleaning_up = True
|
|
try:
|
|
if self.pending_write is not None:
|
|
self._fail("libvirt_execution_uncertain", outcome_unknown=True)
|
|
if self.connection is not None and self.failure is None:
|
|
try:
|
|
self._request({"op": "call", "handle": self.connection._handle,
|
|
"method": "close", "args": {"tuple": []},
|
|
"kwargs": {"dict": []}}, timeout=2.0)
|
|
except Exception:
|
|
self._fail("libvirt_execution_timeout")
|
|
self._closed = True
|
|
if self.sock is not None:
|
|
try:
|
|
self.sock.close()
|
|
except Exception:
|
|
self._fail("libvirt_execution_timeout")
|
|
try:
|
|
self.reap(time.monotonic() + 1.0)
|
|
reaped = self.process is None or not self.process.is_alive()
|
|
except Exception:
|
|
self._fail("libvirt_execution_timeout")
|
|
if not reaped:
|
|
self._fail("libvirt_execution_timeout")
|
|
with self.runtime._condition:
|
|
if self.intent is not None and self.pending_write is None:
|
|
evidence = self.snapshot_evidence
|
|
if ((evidence is not None and (not evidence["freeze_ack"] or evidence["thaw_ack"])
|
|
and (self.failure is None or not self.failure.outcome_unknown))
|
|
or evidence is None and self.failure is None and self.cleanup_failure is None):
|
|
try:
|
|
self.runtime._clear_intent(self)
|
|
self.write_error_receipt = None
|
|
except ExecutionFailure:
|
|
pass # _clear_intent has restored/fenced the receipt.
|
|
elif evidence is not None:
|
|
# A successful freeze is an outstanding recovery duty,
|
|
# independent of whether its RPC was acknowledged.
|
|
self._fail("libvirt_execution_uncertain", outcome_unknown=True)
|
|
finally:
|
|
self._cleaning_up = False
|
|
if reaped:
|
|
self.runtime._release(self.scope, self)
|
|
|
|
|
|
_runtime = None
|
|
_runtime_lock = threading.Lock()
|
|
|
|
|
|
def configure_runtime(state_dir: Path) -> LibvirtRuntime:
|
|
global _runtime
|
|
with _runtime_lock:
|
|
if _runtime is None:
|
|
_runtime = LibvirtRuntime(
|
|
state_dir,
|
|
rpc_timeout=float(os.environ.get("CLOUD_ENGINE_LIBVIRT_RPC_TIMEOUT", "120")),
|
|
connect_timeout=float(os.environ.get("CLOUD_ENGINE_LIBVIRT_CONNECT_TIMEOUT", "20")),
|
|
queue_timeout=float(os.environ.get("CLOUD_ENGINE_LIBVIRT_QUEUE_TIMEOUT", "5")),
|
|
max_connections=int(os.environ.get("CLOUD_ENGINE_LIBVIRT_MAX_CONNECTIONS", "32")),
|
|
max_per_host=int(os.environ.get("CLOUD_ENGINE_LIBVIRT_MAX_PER_HOST", "4")),
|
|
)
|
|
return _runtime
|
|
|
|
|
|
def get_runtime() -> LibvirtRuntime | None:
|
|
return _runtime
|