You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/virsh_panel/libvirt_rpc.py

1010 lines
46 KiB
Python

"""Bound libvirt C calls in disposable processes, without moving business logic.
The private socket transports values and session-local libvirt handles only.
An interrupted write leaves a durable host fence: killing this process is NOT
proof that libvirtd stopped the operation. Only an operator who has reconciled
the host may archive that fence while the engine is stopped.
"""
from __future__ import annotations
import base64
import hashlib
import importlib
import json
import logging
import math
import multiprocessing
import os
import re
import signal
import socket
import struct
import threading
import time
import uuid
import xml.etree.ElementTree as ET
from pathlib import Path
_LOGGER = logging.getLogger(__name__)
EXECUTION_ERROR_CODES = frozenset({
"libvirt_execution_timeout", "libvirt_execution_uncertain",
"libvirt_execution_stopped", "libvirt_execution_blocked",
})
MAX_FRAME = 32 * 1024 * 1024
SNAPSHOT_RECOVERY_ERROR = "snapshot_thaw_recovery_required"
_QGA_RUNTIME_TAG = "{urn:virsh-panel:qga-runtime:1}qga-runtime"
READ_METHODS = frozenset({
"ID", "UUID", "UUIDString", "XMLDesc", "name", "path", "info",
"getName", "getDomain",
"state", "isActive", "isPersistent", "isUpdated", "hasManagedSaveImage",
"autostart", "maxMemory", "getInfo", "getType", "getVersion",
"getLibVersion", "getHostname", "getURI", "getCapabilities",
"getAllDomainStats", "domainListGetStats", "listAllDomains",
"listAllStoragePools", "listVolumes", "listAllVolumes",
"lookupByUUIDString", "lookupByName", "storagePoolLookupByName",
"storageVolLookupByName", "storageVolLookupByPath", "networkLookupByName",
"nwfilterLookupByName", "snapshotNum", "blockInfo", "blockStats",
"interfaceStats", "interfaceAddresses", "jobStats", "jobInfo",
"getMetadata", "metadata", "backupGetXMLDesc", "refresh",
"snapshotLookupByName", "snapshotListNames", "listAllSnapshots",
"agentSetResponseTimeout",
"newStream", "download", "recv", "close",
})
class ExecutionFailure(Exception):
def __init__(self, error_code: str, *, outcome_unknown: bool = False):
super().__init__(error_code)
self.error_code = error_code
self.outcome_unknown = outcome_unknown
class SnapshotRecoveryRequired(Exception):
"""The durable evidence does not authorize an emergency thaw."""
def _snapshot_thaw_preflight(connection, evidence, backend):
"""The only fence exception: thaw one proven, previously frozen runtime.
No absence observation proves a timed-out backupBegin never arrived.
A pending begin therefore requires the exact active backup target.
"""
domain = connection.lookupByUUIDString(evidence["vm_id"])
if not domain.isActive():
return {"thawed": True, "count": 0, "power_state": "stopped"}
if domain.ID() != evidence["runtime_id"]:
raise SnapshotRecoveryRequired()
live = ET.fromstring(domain.XMLDesc(0))
inactive = ET.fromstring(domain.XMLDesc(backend.VIR_DOMAIN_XML_INACTIVE))
nodes = live.findall("./metadata/" + _QGA_RUNTIME_TAG)
if (len(nodes) != 1 or nodes[0].attrib != {
"domain-uuid": evidence["vm_id"], "token": evidence["runtime_token"]}
or inactive.find("./metadata/" + _QGA_RUNTIME_TAG) is not None):
raise SnapshotRecoveryRequired()
volume = connection.storagePoolLookupByName(
evidence["pool_name"]).storageVolLookupByName(evidence["volume_name"])
if volume.path() != evidence["target_path"]:
raise SnapshotRecoveryRequired()
stats = domain.jobStats(0)
if not isinstance(stats, dict) or not isinstance(stats.get("type"), int):
raise SnapshotRecoveryRequired()
active_backup = stats["type"] != backend.VIR_DOMAIN_JOB_NONE
if active_backup:
if (evidence["backup_state"] == "not_sent"
or stats.get("operation") != backend.VIR_DOMAIN_JOB_OPERATION_BACKUP):
raise SnapshotRecoveryRequired()
backup = ET.fromstring(domain.backupGetXMLDesc(0))
all_disks = backup.findall("./disks/disk")
disks = [disk for disk in all_disks if disk.get("backup", "yes") != "no"]
if (backup.tag != "domainbackup" or backup.get("mode", "push") != "push"
or len(disks) != 1 or disks[0].get("name") != "vda"
or disks[0].get("backup", "yes") != "yes"
or disks[0].get("type", "file") != "file"
or disks[0].find("target") is None
or disks[0].find("target").get("file") != evidence["target_path"]
or any(disk.find("target") is not None or disk.find("scratch") is not None
for disk in all_disks if disk.get("backup") == "no")):
raise SnapshotRecoveryRequired()
if evidence["backup_state"] == "pending" and not active_backup:
raise SnapshotRecoveryRequired()
# The last check is immediately before the sole permitted mutation.
if (not domain.isActive() or domain.ID() != evidence["runtime_id"]
or ET.fromstring(domain.XMLDesc(0)).find(
"./metadata/" + _QGA_RUNTIME_TAG).get("token")
!= evidence["runtime_token"]):
raise SnapshotRecoveryRequired()
return domain
def _recover_snapshot_thaw(connection, evidence, backend):
try:
domain = _snapshot_thaw_preflight(connection, evidence, backend)
except (ET.ParseError, AttributeError, KeyError, TypeError, backend.libvirtError):
raise SnapshotRecoveryRequired() from None
if isinstance(domain, dict):
return domain
# From here onward any exception or invalid result is outcome-unknown.
# It must not reset the durable recovery attempt to "not_started".
count = domain.fsThaw(None, 0)
if isinstance(count, bool) or not isinstance(count, int) or count < 0:
raise RuntimeError("unconfirmed emergency thaw result")
return {"thawed": True, "count": count, "power_state": "running"}
def _send(sock, value, deadline):
payload = json.dumps(value, ensure_ascii=True, allow_nan=False).encode()
if len(payload) > MAX_FRAME:
raise ValueError("libvirt RPC message too large")
sock.settimeout(max(0.001, deadline - time.monotonic()))
sock.sendall(struct.pack("!I", len(payload)) + payload)
def _receive(sock, deadline):
def exact(size):
chunks = bytearray()
while len(chunks) < size:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise TimeoutError("libvirt RPC deadline")
sock.settimeout(remaining)
part = sock.recv(min(size - len(chunks), 65536))
if not part:
raise EOFError("libvirt RPC closed")
chunks.extend(part)
return bytes(chunks)
size = struct.unpack("!I", exact(4))[0]
if size > MAX_FRAME:
raise ValueError("libvirt RPC message too large")
return json.loads(exact(size))
def _encode(value, handle):
if value is None or isinstance(value, (bool, int, float, str)):
return value
if isinstance(value, bytes):
return {"bytes": base64.b64encode(value).decode("ascii")}
if isinstance(value, (list, tuple)):
return {"tuple" if isinstance(value, tuple) else "list": [
_encode(item, handle) for item in value]}
if isinstance(value, dict):
return {"dict": [[_encode(k, handle), _encode(v, handle)]
for k, v in value.items()]}
return handle(value)
def _decode(value, handle):
if not isinstance(value, dict):
return value
if "handle" in value:
return handle(value)
if "bytes" in value:
return base64.b64decode(value["bytes"], validate=True)
if "tuple" in value:
return tuple(_decode(v, handle) for v in value["tuple"])
if "list" in value:
return [_decode(v, handle) for v in value["list"]]
if "dict" in value:
return {_decode(k, handle): _decode(v, handle) for k, v in value["dict"]}
raise ValueError("invalid libvirt RPC value")
def _worker(sock, uri, backend_module):
# spawn avoids inheriting libvirt connections and parent thread locks.
# A separate session lets the supervisor reap a stuck SSH child as well.
os.setsid()
_send(sock, {"ready": os.getpid()}, time.monotonic() + 30)
backend = importlib.import_module(backend_module)
objects = {}
identities = {}
def export(value):
if not type(value).__name__.startswith("vir"):
raise TypeError("unsupported libvirt RPC result")
key = identities.get(id(value))
if key is None:
key = len(objects) + 1
objects[key] = value
identities[id(value)] = key
methods = [name for name in dir(value)
if not name.startswith("_") and callable(getattr(value, name, None))]
return {"handle": key, "methods": methods}
def resolve(value):
return objects[value["handle"]]
try:
while True:
request = _receive(sock, time.monotonic() + 86400)
try:
if request["op"] == "open":
result = backend.open(uri)
if result is None:
raise RuntimeError("libvirt returned no connection")
elif request["op"] == "call":
obj = objects[request["handle"]]
method = request["method"]
if not isinstance(method, str) or method.startswith("_"):
raise ValueError("invalid libvirt method")
result = getattr(obj, method)(
*_decode(request["args"], resolve),
**_decode(request["kwargs"], resolve))
elif request["op"] == "qga":
qemu = (importlib.import_module("libvirt_qemu")
if backend_module == "libvirt" else backend)
result = qemu.qemuAgentCommand(
*_decode(request["args"], resolve))
elif request["op"] == "snapshot_thaw":
result = _recover_snapshot_thaw(
objects[request["handle"]], request["evidence"], backend)
else:
raise ValueError("invalid libvirt RPC operation")
response = {"result": _encode(result, export)}
except backend.libvirtError as exc:
response = {"libvirt_error": str(exc), "err": getattr(exc, "err", None)}
except SnapshotRecoveryRequired:
response = {"snapshot_recovery_required": True}
except Exception:
# Never expose arbitrary exception text/arguments or credentials.
response = {"worker_error": True}
_send(sock, response, time.monotonic() + 30)
except (EOFError, OSError, ValueError):
pass
finally:
sock.close()
class RemoteObject:
def __init__(self, session, reference):
self._session = session
self._handle = reference["handle"]
self._methods = frozenset(reference["methods"])
def __getattr__(self, method):
if method.startswith("_") or method not in self._methods:
raise AttributeError(method)
def call(*args, **kwargs):
return self._session.call(self._handle, method, args, kwargs)
return call
class LibvirtRuntime:
def __init__(self, state_dir: Path, *, rpc_timeout=120.0,
connect_timeout=20.0, queue_timeout=5.0,
max_connections=32, max_per_host=4, backend_module="libvirt"):
if any(not math.isfinite(value) or value <= 0 for value in
(rpc_timeout, connect_timeout, queue_timeout,
max_connections, max_per_host)):
raise ValueError("libvirt execution limits must be positive")
if any(not isinstance(value, int) for value in (max_connections, max_per_host)):
raise ValueError("libvirt execution capacity must be an integer")
self.state_dir = Path(state_dir)
self._ensure_directory(self.state_dir)
self.rpc_timeout = rpc_timeout
self.connect_timeout = connect_timeout
self.queue_timeout = queue_timeout
self.max_connections = max_connections
self.max_per_host = max_per_host
self.backend_module = backend_module # constructor-only test dependency
self._condition = threading.Condition()
self._sessions = set()
self._reserved = {}
self._closing = False
self._snapshot_recoveries = set()
def _directory(self, scope):
directory = self.state_dir / hashlib.sha256(scope.encode()).hexdigest()
self._ensure_directory(directory)
return directory
@classmethod
def _ensure_directory(cls, directory):
try:
directory.mkdir(mode=0o700)
except FileNotFoundError:
cls._ensure_directory(directory.parent)
directory.mkdir(mode=0o700)
except FileExistsError:
if directory.is_symlink() or not directory.is_dir():
raise ValueError("unsafe libvirt execution directory")
return
# The intent's fsync cannot persist the containing directory's own
# entry in its parent. Persist newly created levels before any write.
cls._sync(directory.parent)
@staticmethod
def _sync(directory):
fd = os.open(directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
try:
os.fsync(fd)
finally:
os.close(fd)
def _before_write(self, session, method):
from .coordination import mark_remote_write_started
with self._condition:
self._write_admission(session)
if session.pending_write is not None:
raise session._fail("libvirt_execution_uncertain", outcome_unknown=True)
session.write_sequence += 1
session.pending_write = {"sequence": session.write_sequence, "method": method}
try:
self._persist_write_record(session)
except Exception:
raise session._fail("libvirt_execution_uncertain", outcome_unknown=True) from None
mark_remote_write_started()
return session.write_sequence
def _write_admission(self, session):
from .coordination import mark_admission_rejected
if session.failure is not None:
raise session._fail(session.failure.error_code,
outcome_unknown=session.failure.outcome_unknown)
if self._closing or session._closed:
raise ExecutionFailure("libvirt_execution_stopped")
active = {s.token for s in self._sessions if s.failure is None}
if (session.scope in self._snapshot_recoveries
or any(path.stem not in active
for path in self._directory(session.scope).glob("*.json"))):
mark_admission_rejected()
raise ExecutionFailure("libvirt_execution_blocked", outcome_unknown=True)
def _persist_write_record(self, session):
record = {"version": 2, "scope": session.scope, "session": session.token,
"sequence": session.write_sequence,
"pending_write": session.pending_write,
"last_write_ack": session.last_write_ack,
"started_at": int(time.time())}
if session.write_error_receipt is not None:
record["write_error_receipt"] = session.write_error_receipt
if session.snapshot_evidence is not None:
record["snapshot_recovery"] = session.snapshot_evidence
if session.intent is not None:
self._replace_snapshot_record(session.intent, record)
return
session.intent = self._directory(session.scope) / (session.token + ".json")
fd = os.open(session.intent, os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, "O_NOFOLLOW", 0), 0o600)
with os.fdopen(fd, "wb") as stream:
stream.write(json.dumps(record).encode())
stream.flush()
os.fsync(stream.fileno())
self._sync(session.intent.parent)
def _clear_intent(self, session):
"""Durably acknowledge this receipt; restore the fence on fsync failure."""
# Caller holds _condition through deletion, sync and restoration.
path, original = session.intent, None
try:
original = path.read_bytes()
path.unlink()
self._sync(path.parent)
except Exception:
try:
if original is not None and not path.exists():
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, "O_NOFOLLOW", 0), 0o600)
with os.fdopen(fd, "wb") as stream:
stream.write(original)
stream.flush()
os.fsync(stream.fileno())
self._sync(path.parent)
if not path.is_file():
raise OSError("libvirt execution fence unavailable")
except Exception:
self._closing = True
self._condition.notify_all()
raise session._fail("libvirt_execution_uncertain", outcome_unknown=True) from None
session.intent = None
def _ack_write(self, session, sequence):
with self._condition:
if (session.failure is not None or self._closing
or session.pending_write is None
or session.pending_write["sequence"] != sequence):
raise session._fail("libvirt_execution_uncertain", outcome_unknown=True)
if session.snapshot_evidence is None and session.write_error_receipt is None:
# Absence is persisted before returning the successful RPC.
# A later call creates a fresh pending record, even in this
# same connection. No old intent is inspected or removed.
self._clear_intent(session)
else:
# Freeze recovery outlives individual RPC acknowledgements.
# Keep its identity/stages independently of the pending call.
try:
record = json.loads(session.intent.read_text())
record.update(pending_write=None, last_write_ack=sequence)
self._replace_snapshot_record(session.intent, record)
except Exception:
raise session._fail("libvirt_execution_uncertain", outcome_unknown=True) from None
session.last_write_ack = sequence
session.pending_write = None
def _record_write_error(self, session):
"""Keep a returned native error distinct from a success or lost reply.
Existing callers may inspect/compensate a libvirt rejection in this
same connection. As before, only a clean end to that workflow removes
its obligation; a later success must not overwrite the earlier error.
"""
with self._condition:
try:
record = json.loads(session.intent.read_text())
error_receipt = session.write_error_receipt or dict(session.pending_write)
record.update(pending_write=None, write_error_receipt=error_receipt)
self._replace_snapshot_record(session.intent, record)
except Exception:
raise session._fail("libvirt_execution_uncertain", outcome_unknown=True) from None
session.write_error_receipt = error_receipt
session.pending_write = None
def _save_snapshot_evidence(self, session, evidence):
"""Persist a conservative phase before sending the corresponding RPC."""
with self._condition:
if session.intent is None or session.failure is not None or self._closing:
raise SnapshotRecoveryRequired()
try:
original = json.loads(session.intent.read_text())
original["snapshot_recovery"] = evidence
self._replace_snapshot_record(session.intent, original)
except Exception:
session._fail("libvirt_execution_uncertain", outcome_unknown=True)
raise
session.snapshot_evidence = evidence
def _replace_snapshot_record(self, path, record):
temporary = path.with_suffix(".snapshot-tmp")
try:
with temporary.open("x", encoding="utf-8") as stream:
os.chmod(temporary, 0o600)
json.dump(record, stream)
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary, path)
self._sync(path.parent)
finally:
temporary.unlink(missing_ok=True)
def _snapshot_recovery_evidence(self, session, job_id, vm_id, volume_name):
"""Authorize only a bound job whose old worker can send no more RPCs."""
with self._condition:
if self._closing or session.scope in self._snapshot_recoveries:
raise SnapshotRecoveryRequired()
files = list(self._directory(session.scope).glob("*.json"))
# Another unresolved write may include a freeze we cannot identify.
if len(files) != 1 or files[0].stat().st_size > 16384:
raise SnapshotRecoveryRequired()
try:
record = json.loads(files[0].read_text())
except (ValueError, OSError):
raise SnapshotRecoveryRequired() from None
if not isinstance(record, dict):
raise SnapshotRecoveryRequired()
evidence = record.get("snapshot_recovery")
if (record.get("scope") != session.scope
or not isinstance(evidence, dict)
or evidence.get("job_id") != job_id
or evidence.get("vm_id") != vm_id
or evidence.get("volume_name") != volume_name
or evidence.get("freeze_ack") is not True
or not isinstance(evidence.get("thaw_ack"), bool)
or evidence.get("thaw_pending") is not False
or evidence.get("recovery_state") not in {"not_started", "ack"}
or evidence.get("backup_state") not in {"not_sent", "pending", "ack"}
or not isinstance(evidence.get("worker_pid"), int)
or isinstance(evidence["worker_pid"], bool)
or evidence["worker_pid"] <= 0
or not isinstance(evidence.get("runtime_id"), int)
or isinstance(evidence["runtime_id"], bool)
or evidence["runtime_id"] < 0
or not isinstance(evidence.get("runtime_token"), str)
or re.fullmatch(r"[0-9a-f]{32}", evidence["runtime_token"]) is None
or not isinstance(evidence.get("pool_name"), str)
or not evidence["pool_name"]
or not isinstance(evidence.get("target_path"), str)
or not evidence["target_path"].startswith("/")):
raise SnapshotRecoveryRequired()
for other in self._sessions:
if other is session or other.scope != session.scope:
continue
if other.intent is not None and (other.process is None
or other.process.is_alive()):
raise SnapshotRecoveryRequired()
try:
os.kill(evidence["worker_pid"], 0)
except ProcessLookupError:
pass
except OSError:
raise SnapshotRecoveryRequired() from None
else:
# PID reuse errs towards manual recovery, never false admission.
raise SnapshotRecoveryRequired()
self._snapshot_recoveries.add(session.scope)
return files[0], record, evidence
def open(self, uri, scope, *, timeout=180.0):
if (not isinstance(scope, str) or not scope or len(scope) > 256
or not math.isfinite(timeout) or timeout <= 0):
raise ValueError("invalid libvirt execution scope/budget")
request_deadline = time.monotonic() + timeout
deadline = min(
request_deadline, time.monotonic() + self.queue_timeout)
with self._condition:
while (sum(self._reserved.values()) >= self.max_connections
or self._reserved.get(scope, 0) >= self.max_per_host):
if self._closing:
raise ExecutionFailure("libvirt_execution_stopped")
remaining = deadline - time.monotonic()
if remaining <= 0:
raise ExecutionFailure("libvirt_execution_timeout")
self._condition.wait(remaining)
if self._closing:
raise ExecutionFailure("libvirt_execution_stopped")
self._reserved[scope] = self._reserved.get(scope, 0) + 1
session = None
try:
remaining = request_deadline - time.monotonic()
if remaining <= 0:
raise ExecutionFailure("libvirt_execution_timeout")
session = RPCSession(self, uri, scope, remaining)
with self._condition:
self._sessions.add(session)
if self._closing:
session._fail("libvirt_execution_stopped")
session.start()
return session
except BaseException:
if session is not None:
session.close()
else:
self._release(scope, None)
raise
def _release(self, scope, session):
with self._condition:
self._sessions.discard(session)
remaining = self._reserved.get(scope, 1) - 1
if remaining:
self._reserved[scope] = remaining
else:
self._reserved.pop(scope, None)
self._condition.notify_all()
def begin_shutdown(self):
with self._condition:
self._closing = True
sessions = tuple(self._sessions)
self._condition.notify_all()
for session in sessions:
session._fail("libvirt_execution_stopped")
try:
session.interrupt()
except Exception:
session._fail("libvirt_execution_stopped")
_LOGGER.warning("Unable to interrupt libvirt worker (pid=%s)", session.pid)
def close(self, timeout=3.0):
self.begin_shutdown()
deadline = time.monotonic() + max(0, timeout)
with self._condition:
sessions = tuple(self._sessions)
reaped = True
for session in sessions:
try:
session.reap(deadline)
except Exception:
session._fail("libvirt_execution_stopped")
_LOGGER.warning("Unable to reap libvirt worker (pid=%s)", session.pid)
reaped = False
return reaped and all(s.process is None or not s.process.is_alive() for s in sessions)
class RPCSession:
def __init__(self, runtime, uri, scope, timeout):
self.runtime, self.uri, self.scope = runtime, uri, scope
self.deadline = time.monotonic() + timeout
self.token = uuid.uuid4().hex
self.intent = None
self.pending_write = None
self.write_error_receipt = None
self.write_sequence = 0
self.last_write_ack = 0
self.failure = None
self.cleanup_failure = None
self._cleaning_up = False
self.process = None
self.sock = None
self.connection = None
self._closed = False
self._io_lock = threading.RLock()
self._pgid = None
self._signal_lock = threading.Lock()
self._sent_signals = set()
self.snapshot_evidence = None
self._snapshot_handle = None
# Stream completion commits uploads, but only closes downloads. Keep
# the direction in the supervisor so a lost download finish/abort
# reply cannot create a false host write fence.
self._stream_modes = {}
@property
def pid(self):
return self.process.pid if self.process is not None else None
def _fail(self, code, *, outcome_unknown=False):
from .coordination import execution_failure_before_write, mark_execution_unknown
# Intent persistence and shutdown share this condition. A later local
# stop/error must never downgrade evidence of an unconfirmed write.
with self.runtime._condition:
previous = self.failure
if (self._cleaning_up and self.pending_write is None
and self.write_error_receipt is None
and previous is None and not outcome_unknown):
self.cleanup_failure = ExecutionFailure(code)
return self.cleanup_failure
if self.pending_write is not None or self.write_error_receipt is not None:
code, outcome_unknown = "libvirt_execution_uncertain", True
elif previous is not None and (previous.outcome_unknown or previous.error_code in {
"libvirt_execution_uncertain", "libvirt_execution_blocked"}):
return previous
elif code in {"libvirt_execution_uncertain", "libvirt_execution_blocked"}:
outcome_unknown = True
elif outcome_unknown:
code = "libvirt_execution_uncertain"
elif previous is not None:
return previous
if (previous is None or previous.error_code != code
or previous.outcome_unknown != outcome_unknown):
self.failure = ExecutionFailure(code, outcome_unknown=outcome_unknown)
if outcome_unknown and not (code == "libvirt_execution_blocked"
and execution_failure_before_write(code)):
mark_execution_unknown()
return self.failure
def start(self):
with self.runtime._condition:
if self.failure is not None or self.runtime._closing:
raise self._fail("libvirt_execution_stopped")
self.sock, child = socket.socketpair()
try:
self.process = multiprocessing.get_context("spawn").Process(
target=_worker, args=(child, self.uri, self.runtime.backend_module),
daemon=True, name="libvirt-rpc")
self.process.start()
finally:
child.close()
try:
ready = _receive(self.sock, min(self.deadline,
time.monotonic() + self.runtime.connect_timeout))
if ready != {"ready": self.process.pid}:
raise ValueError("invalid libvirt worker")
self._pgid = self.process.pid
except (OSError, EOFError, ValueError):
raise self._fail("libvirt_execution_timeout") from None
self.connection = self._request({"op": "open"}, self.runtime.connect_timeout)
def _reference(self, value):
if not isinstance(value, RemoteObject) or value._session is not self:
raise ValueError("libvirt handles cannot cross sessions")
return {"handle": value._handle}
def _request(self, request, timeout=None):
with self._io_lock:
if self.failure is not None:
raise self._fail(self.failure.error_code,
outcome_unknown=self.failure.outcome_unknown)
if self._closed:
raise self._fail("libvirt_execution_stopped")
deadline = min(self.deadline, time.monotonic() + (
self.runtime.rpc_timeout if timeout is None else timeout))
try:
if time.monotonic() >= deadline:
raise TimeoutError("libvirt RPC deadline")
_send(self.sock, request, deadline)
response = _receive(self.sock, deadline)
except (OSError, EOFError, ValueError):
failure = self._fail("libvirt_execution_timeout")
self.interrupt()
raise failure from None
if self.failure is not None:
raise self._fail(self.failure.error_code,
outcome_unknown=self.failure.outcome_unknown)
if not isinstance(response, dict):
raise self._fail("libvirt_execution_timeout")
if "libvirt_error" in response:
if self.pending_write is not None:
self.runtime._record_write_error(self)
backend = importlib.import_module(self.runtime.backend_module)
error = backend.libvirtError(response["libvirt_error"])
if response.get("err") is not None:
error.err = tuple(response["err"])
raise error
if response.get("snapshot_recovery_required") is True:
raise SnapshotRecoveryRequired()
if "result" not in response:
failure = self._fail("libvirt_execution_timeout")
self.interrupt()
raise failure
try:
return _decode(response["result"], lambda value: RemoteObject(self, value))
except (ValueError, TypeError, KeyError):
raise self._fail("libvirt_execution_timeout") from None
def call(self, handle, method, args, kwargs):
# Serialize intent publication, dispatch, decoding and the matching
# acknowledgement. A concurrent caller must never clear our pending
# write or allow a read to consume its reply.
with self._io_lock:
return self._call(handle, method, args, kwargs)
def _call(self, handle, method, args, kwargs):
if self.failure is not None:
raise self._fail(self.failure.error_code,
outcome_unknown=self.failure.outcome_unknown)
request = {"op": "call", "handle": handle, "method": method,
"args": _encode(args, self._reference),
"kwargs": _encode(kwargs, self._reference)}
if method in {"upload", "download"} and args:
stream = args[0]
if isinstance(stream, RemoteObject) and stream._session is self:
self._stream_modes[stream._handle] = method
read_download_completion = (
method in {"finish", "abort"}
and self._stream_modes.get(handle) == "download")
sequence = None
if method not in READ_METHODS and not read_download_completion:
try:
sequence = self.runtime._before_write(self, method)
except ExecutionFailure as exc:
raise self._fail(
exc.error_code, outcome_unknown=exc.outcome_unknown) from None
snapshot_call = self.snapshot_evidence is not None and handle == self._snapshot_handle
if snapshot_call and method in {"fsFreeze", "backupBegin", "fsThaw"}:
evidence = dict(self.snapshot_evidence)
if method == "fsFreeze":
# Never permit a second freeze under the same recovery identity.
if evidence["freeze_ack"]:
raise SnapshotRecoveryRequired()
elif method == "backupBegin":
if not evidence["freeze_ack"] or evidence["backup_state"] != "not_sent":
raise SnapshotRecoveryRequired()
evidence["backup_state"] = "pending"
else:
evidence["thaw_pending"] = True
self.runtime._save_snapshot_evidence(self, evidence)
# Full disk cloning is one synchronous libvirt RPC. Its bound is the
# storage operation budget, not the short per-query budget.
result = self._request(request, timeout=(
max(0, self.deadline - time.monotonic())
if method == "createXMLFrom" else None))
if snapshot_call and method in {"fsFreeze", "backupBegin", "fsThaw"}:
evidence = dict(self.snapshot_evidence)
if method == "fsFreeze" and isinstance(result, int) and not isinstance(result, bool) and result > 0:
evidence["freeze_ack"] = True
elif method == "backupBegin" and result in (None, 0):
evidence["backup_state"] = "ack"
elif method == "fsThaw" and isinstance(result, int) and not isinstance(result, bool) and result >= 0:
evidence["thaw_ack"] = True
evidence["thaw_pending"] = False
self.runtime._save_snapshot_evidence(self, evidence)
if sequence is not None:
self.runtime._ack_write(self, sequence)
if method in {"finish", "abort"}:
self._stream_modes.pop(handle, None)
return result
def bind_snapshot_recovery(self, domain, *, job_id, volume_name, pool_name,
target_path, runtime_token):
if (not isinstance(job_id, str) or str(uuid.UUID(job_id)) != job_id
or not isinstance(runtime_token, str)
or re.fullmatch(r"[0-9a-f]{32}", runtime_token) is None
or self.snapshot_evidence is not None):
raise SnapshotRecoveryRequired()
self._reference(domain)
evidence = {
"job_id": job_id, "vm_id": domain.UUIDString(),
"volume_name": volume_name, "pool_name": pool_name,
"target_path": target_path, "runtime_id": domain.ID(),
"runtime_token": runtime_token, "worker_pid": self.pid,
"freeze_ack": False, "backup_state": "not_sent", "thaw_ack": False,
"thaw_pending": False, "recovery_state": "not_started",
}
with self._io_lock, self.runtime._condition:
self.runtime._write_admission(self)
if self.pending_write is not None:
raise SnapshotRecoveryRequired()
self.snapshot_evidence = evidence
try:
self.runtime._persist_write_record(self)
except Exception:
raise self._fail("libvirt_execution_uncertain", outcome_unknown=True) from None
self._snapshot_handle = domain._handle
def recover_snapshot_thaw(self, *, job_id, vm_id, volume_name):
path, record, evidence = self.runtime._snapshot_recovery_evidence(
self, job_id, vm_id, volume_name)
try:
if evidence["thaw_ack"]:
# A durable receipt never authorizes thawing a later freeze.
return {"thawed": True, "count": 0, "power_state": "unknown"}
evidence["recovery_state"] = "pending"
self.runtime._replace_snapshot_record(path, record)
# All other RPCs on this connection remain subject to the fence.
# The original intent survives even if this one operation succeeds.
try:
result = self._request({"op": "snapshot_thaw",
"handle": self.connection._handle,
"evidence": evidence})
except SnapshotRecoveryRequired:
# The worker only returns this response before fsThaw begins.
evidence["recovery_state"] = "not_started"
self.runtime._replace_snapshot_record(path, record)
raise
evidence.update(thaw_ack=True, recovery_state="ack")
self.runtime._replace_snapshot_record(path, record)
return result
except (OSError, ExecutionFailure):
# The dedicated recovery session has no ordinary write intent of
# its own. Its persisted pending receipt still means fsThaw may
# have reached the guest, so a transport failure is outcome-unknown.
raise self._fail("libvirt_execution_uncertain", outcome_unknown=True) from None
finally:
with self.runtime._condition:
self.runtime._snapshot_recoveries.discard(self.scope)
def qemu_agent_command(self, domain, command, timeout, flags):
with self._io_lock:
return self._qemu_agent_command(domain, command, timeout, flags)
def _qemu_agent_command(self, domain, command, timeout, flags):
if self.failure is not None:
raise self._fail(self.failure.error_code,
outcome_unknown=self.failure.outcome_unknown)
# guest-exec and other QGA commands can mutate the guest. Do not infer
# read-only behavior from an opaque JSON command supplied by a caller.
try:
command_name = json.loads(command).get("execute")
except (TypeError, ValueError, AttributeError):
command_name = None
request = {"op": "qga", "args": _encode(
(domain, command, timeout, flags), self._reference)}
sequence = None
if command_name not in {"guest-info", "guest-ping", "guest-network-get-interfaces",
"guest-exec-status", "guest-fsfreeze-status"}:
try:
sequence = self.runtime._before_write(self, "qemuAgentCommand")
except ExecutionFailure as exc:
raise self._fail(
exc.error_code, outcome_unknown=exc.outcome_unknown) from None
result = self._request(request, timeout=min(self.runtime.rpc_timeout, max(1, timeout) + 2))
if sequence is not None:
self.runtime._ack_write(self, sequence)
return result
def interrupt(self):
if self.sock is not None:
try:
self.sock.shutdown(socket.SHUT_RDWR)
except OSError:
pass
try:
self._signal(signal.SIGTERM)
except OSError:
self._fail("libvirt_execution_stopped")
_LOGGER.warning("Unable to signal libvirt worker (pid=%s)", self.pid)
def _signal(self, sig):
with self._signal_lock:
process = self.process
if process is None or process.pid is None or sig in self._sent_signals:
return
exited = process.exitcode # reap a zombie before signalling its group
try:
if self._pgid is not None:
# The handshake proves this is our private group. The leader
# may exit on TERM while an SSH descendant still needs KILL.
os.killpg(self._pgid, sig)
elif exited is None:
os.kill(process.pid, sig)
except ProcessLookupError:
pass
except PermissionError:
# Darwin can report EPERM while a signalled group leader is
# exiting, before waitpid observes it. Reap for a bounded time,
# then retry; a live group permission failure remains an error.
process.join(timeout=0.05)
if self._pgid is None or process.exitcode is None:
raise
try:
os.killpg(self._pgid, sig)
except ProcessLookupError:
pass
self._sent_signals.add(sig)
def reap(self, deadline):
process = self.process
if process is None or process.pid is None:
return
self._signal(signal.SIGTERM)
process.join(timeout=min(0.1, max(0, deadline - time.monotonic())))
self._signal(signal.SIGKILL)
process.join(timeout=max(0, deadline - time.monotonic()))
def close(self):
with self._io_lock:
self._close()
def _close(self):
if self._closed:
return
reaped = False
self._cleaning_up = True
try:
if self.pending_write is not None:
self._fail("libvirt_execution_uncertain", outcome_unknown=True)
if self.connection is not None and self.failure is None:
try:
self._request({"op": "call", "handle": self.connection._handle,
"method": "close", "args": {"tuple": []},
"kwargs": {"dict": []}}, timeout=2.0)
except Exception:
self._fail("libvirt_execution_timeout")
self._closed = True
if self.sock is not None:
try:
self.sock.close()
except Exception:
self._fail("libvirt_execution_timeout")
try:
self.reap(time.monotonic() + 1.0)
reaped = self.process is None or not self.process.is_alive()
except Exception:
self._fail("libvirt_execution_timeout")
if not reaped:
self._fail("libvirt_execution_timeout")
with self.runtime._condition:
if self.intent is not None and self.pending_write is None:
evidence = self.snapshot_evidence
if ((evidence is not None and (not evidence["freeze_ack"] or evidence["thaw_ack"])
and (self.failure is None or not self.failure.outcome_unknown))
or evidence is None and self.failure is None and self.cleanup_failure is None):
try:
self.runtime._clear_intent(self)
self.write_error_receipt = None
except ExecutionFailure:
pass # _clear_intent has restored/fenced the receipt.
elif evidence is not None:
# A successful freeze is an outstanding recovery duty,
# independent of whether its RPC was acknowledged.
self._fail("libvirt_execution_uncertain", outcome_unknown=True)
finally:
self._cleaning_up = False
if reaped:
self.runtime._release(self.scope, self)
_runtime = None
_runtime_lock = threading.Lock()
def configure_runtime(state_dir: Path) -> LibvirtRuntime:
global _runtime
with _runtime_lock:
if _runtime is None:
_runtime = LibvirtRuntime(
state_dir,
rpc_timeout=float(os.environ.get("CLOUD_ENGINE_LIBVIRT_RPC_TIMEOUT", "120")),
connect_timeout=float(os.environ.get("CLOUD_ENGINE_LIBVIRT_CONNECT_TIMEOUT", "20")),
queue_timeout=float(os.environ.get("CLOUD_ENGINE_LIBVIRT_QUEUE_TIMEOUT", "5")),
max_connections=int(os.environ.get("CLOUD_ENGINE_LIBVIRT_MAX_CONNECTIONS", "32")),
max_per_host=int(os.environ.get("CLOUD_ENGINE_LIBVIRT_MAX_PER_HOST", "4")),
)
return _runtime
def get_runtime() -> LibvirtRuntime | None:
return _runtime