From 8c6b51be3eb2ea949861937eddcffed74a439873 Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Sun, 28 Dec 2025 09:39:14 +1100 Subject: [PATCH] Manage apt stuff in its own role, not in etc_custom --- enroll/diff.py | 6 ++ enroll/harvest.py | 120 ++++++++++++++++++++++++++++++---- enroll/manifest.py | 157 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 270 insertions(+), 13 deletions(-) diff --git a/enroll/diff.py b/enroll/diff.py index a2b7d91..0110d17 100644 --- a/enroll/diff.py +++ b/enroll/diff.py @@ -184,6 +184,12 @@ def _iter_managed_files(state: Dict[str, Any]) -> Iterable[Tuple[str, Dict[str, for mf in u.get("managed_files", []) or []: yield str(u_role), mf + # apt_config + ac = state.get("apt_config") or {} + ac_role = ac.get("role_name") or "apt_config" + for mf in ac.get("managed_files", []) or []: + yield str(ac_role), mf + # etc_custom ec = state.get("etc_custom") or {} ec_role = ec.get("role_name") or "etc_custom" diff --git a/enroll/harvest.py b/enroll/harvest.py index d4cfacd..c1a1986 100644 --- a/enroll/harvest.py +++ b/enroll/harvest.py @@ -77,6 +77,14 @@ class UsersSnapshot: notes: List[str] +@dataclass +class AptConfigSnapshot: + role_name: str + managed_files: List[ManagedFile] + excluded: List[ExcludedFile] + notes: List[str] + + @dataclass class EtcCustomSnapshot: role_name: str @@ -126,7 +134,6 @@ ALLOWED_UNOWNED_EXTS = { } MAX_FILES_CAP = 4000 - MAX_UNOWNED_FILES_PER_ROLE = 500 # Directories that are shared across many packages; never attribute unowned files in these trees to a single package. @@ -401,30 +408,61 @@ def _parse_apt_signed_by(source_files: List[str]) -> Set[str]: return out -def _iter_system_capture_paths() -> List[tuple[str, str]]: - """Return (path, reason) pairs for essential system config/state.""" - out: List[tuple[str, str]] = [] +def _iter_apt_capture_paths() -> List[tuple[str, str]]: + """Return (path, reason) pairs for APT configuration. - # APT: capture sources and related config + This captures the full /etc/apt tree (subject to IgnorePolicy at copy time), + plus any keyrings referenced via signed-by/Signed-By which may live outside + /etc (e.g. /usr/share/keyrings). + """ + reasons: Dict[str, str] = {} + + # Capture all regular files under /etc/apt (no symlinks). + if os.path.isdir("/etc/apt"): + for dirpath, _, filenames in os.walk("/etc/apt"): + for fn in filenames: + p = os.path.join(dirpath, fn) + if os.path.islink(p) or not os.path.isfile(p): + continue + reasons.setdefault(p, "apt_config") + + # Identify source files explicitly for nicer reasons and keyring discovery. apt_sources: List[str] = [] for g in _APT_SOURCE_GLOBS: apt_sources.extend(_iter_matching_files(g)) for p in sorted(set(apt_sources)): - out.append((p, "system_apt_sources")) + reasons[p] = "apt_source" - # APT: misc config files/dirs - for g in _APT_MISC_GLOBS: + # Keyrings in standard locations. + for g in ( + "/etc/apt/trusted.gpg", + "/etc/apt/trusted.gpg.d/*", + "/etc/apt/keyrings/*", + ): for p in _iter_matching_files(g): - out.append((p, "system_apt_config")) + reasons[p] = "apt_keyring" - # APT: referenced keyrings (may live outside /etc) + # Keyrings referenced by sources (may live outside /etc/apt). signed_by = _parse_apt_signed_by(sorted(set(apt_sources))) for p in sorted(signed_by): if os.path.islink(p) or not os.path.isfile(p): continue - out.append((p, "system_apt_keyring")) + if p.startswith("/etc/apt/"): + reasons[p] = "apt_keyring" + else: + reasons[p] = "apt_signed_by_keyring" + + # De-dup with stable ordering. + uniq: List[tuple[str, str]] = [] + for p in sorted(reasons.keys()): + uniq.append((p, reasons[p])) + return uniq + + +def _iter_system_capture_paths() -> List[tuple[str, str]]: + """Return (path, reason) pairs for essential system config/state (non-APT).""" + out: List[tuple[str, str]] = [] - # Other system config/state globs for spec, reason in _SYSTEM_CAPTURE_GLOBS: for p in _iter_matching_files(spec): out.append((p, reason)) @@ -544,6 +582,8 @@ def harvest( for path in pkg_to_etc_paths.get(pkg, []): if not os.path.isfile(path) or os.path.islink(path): continue + if path.startswith("/etc/apt/"): + continue if path in conff: # Only capture conffiles when they differ from the package default. try: @@ -784,6 +824,8 @@ def harvest( for path in pkg_to_etc_paths.get(pkg, []): if not os.path.isfile(path) or os.path.islink(path): continue + if path.startswith("/etc/apt/"): + continue if path in conff: try: current = file_md5(path) @@ -946,6 +988,55 @@ def harvest( notes=users_notes, ) + # ------------------------- + # apt_config role (APT configuration and keyrings) + # ------------------------- + apt_notes: List[str] = [] + apt_excluded: List[ExcludedFile] = [] + apt_managed: List[ManagedFile] = [] + apt_role_name = "apt_config" + + for path, reason in _iter_apt_capture_paths(): + if path_filter.is_excluded(path): + apt_excluded.append(ExcludedFile(path=path, reason="user_excluded")) + continue + + deny = policy.deny_reason(path) + if deny: + apt_excluded.append(ExcludedFile(path=path, reason=deny)) + continue + + try: + owner, group, mode = stat_triplet(path) + except OSError: + apt_excluded.append(ExcludedFile(path=path, reason="unreadable")) + continue + + src_rel = path.lstrip("/") + try: + _copy_into_bundle(bundle_dir, apt_role_name, path, src_rel) + except OSError: + apt_excluded.append(ExcludedFile(path=path, reason="unreadable")) + continue + + apt_managed.append( + ManagedFile( + path=path, + src_rel=src_rel, + owner=owner, + group=group, + mode=mode, + reason=reason, + ) + ) + + apt_config_snapshot = AptConfigSnapshot( + role_name=apt_role_name, + managed_files=apt_managed, + excluded=apt_excluded, + notes=apt_notes, + ) + # ------------------------- # etc_custom role (unowned /etc files not already attributed elsewhere) # ------------------------- @@ -964,6 +1055,8 @@ def harvest( already.add(mf.path) for mf in users_managed: already.add(mf.path) + for mf in apt_managed: + already.add(mf.path) # Maps for re-attributing shared snippets (cron.d/logrotate.d) to existing roles. svc_by_role: Dict[str, ServiceSnapshot] = {s.role_name: s for s in service_snaps} @@ -1107,6 +1200,8 @@ def harvest( for dirpath, _, filenames in os.walk("/etc"): for fn in filenames: path = os.path.join(dirpath, fn) + if path.startswith("/etc/apt/"): + continue if path in already: continue if path in owned_etc: @@ -1408,6 +1503,7 @@ def harvest( "manual_packages": manual_pkgs, "manual_packages_skipped": manual_pkgs_skipped, "package_roles": [asdict(p) for p in pkg_snaps], + "apt_config": asdict(apt_config_snapshot), "etc_custom": asdict(etc_custom_snapshot), "usr_local_custom": asdict(usr_local_custom_snapshot), "extra_paths": asdict(extra_paths_snapshot), diff --git a/enroll/manifest.py b/enroll/manifest.py index d5ebff7..dbc2353 100644 --- a/enroll/manifest.py +++ b/enroll/manifest.py @@ -2,6 +2,7 @@ from __future__ import annotations import json import os +import re import shutil import stat import tarfile @@ -627,6 +628,7 @@ def _manifest_from_bundle_dir( services: List[Dict[str, Any]] = state.get("services", []) package_roles: List[Dict[str, Any]] = state.get("package_roles", []) users_snapshot: Dict[str, Any] = state.get("users", {}) + apt_config_snapshot: Dict[str, Any] = state.get("apt_config", {}) etc_custom_snapshot: Dict[str, Any] = state.get("etc_custom", {}) usr_local_custom_snapshot: Dict[str, Any] = state.get("usr_local_custom", {}) extra_paths_snapshot: Dict[str, Any] = state.get("extra_paths", {}) @@ -661,6 +663,7 @@ def _manifest_from_bundle_dir( _ensure_ansible_cfg(os.path.join(out_dir, "ansible.cfg")) manifested_users_roles: List[str] = [] + manifested_apt_config_roles: List[str] = [] manifested_etc_custom_roles: List[str] = [] manifested_usr_local_custom_roles: List[str] = [] manifested_extra_paths_roles: List[str] = [] @@ -887,6 +890,157 @@ Generated non-system user accounts and SSH public material. manifested_users_roles.append(role) + # ------------------------- + # apt_config role (APT sources, pinning, and keyrings) + # ------------------------- + if apt_config_snapshot and apt_config_snapshot.get("managed_files"): + role = apt_config_snapshot.get("role_name", "apt_config") + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + + var_prefix = role + + managed_files = apt_config_snapshot.get("managed_files", []) + excluded = apt_config_snapshot.get("excluded", []) + notes = apt_config_snapshot.get("notes", []) + + templated, jt_vars = _jinjify_managed_files( + bundle_dir, + role, + role_dir, + managed_files, + jt_exe=jt_exe, + jt_enabled=jt_enabled, + overwrite_templates=not site_mode, + ) + + # Copy only the non-templated artifacts (templates live in the role). + if site_mode: + _copy_artifacts( + bundle_dir, + role, + _host_role_files_dir(out_dir, fqdn or "", role), + exclude_rels=templated, + ) + else: + _copy_artifacts( + bundle_dir, + role, + os.path.join(role_dir, "files"), + exclude_rels=templated, + ) + + files_var = _build_managed_files_var( + managed_files, + templated, + notify_other=None, + notify_systemd=None, + ) + + jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} + vars_map: Dict[str, Any] = {f"{var_prefix}_managed_files": files_var} + vars_map = _merge_mappings_overwrite(vars_map, jt_map) + + if site_mode: + _write_role_defaults(role_dir, {f"{var_prefix}_managed_files": []}) + _write_hostvars(out_dir, fqdn or "", role, vars_map) + else: + _write_role_defaults(role_dir, vars_map) + + tasks = """---\n""" + _render_generic_files_tasks( + var_prefix, include_restart_notify=False + ) + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(tasks.rstrip() + "\n") + + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write("---\ndependencies: []\n") + + # README: summarise repos and keyrings + source_paths: List[str] = [] + keyring_paths: List[str] = [] + repo_hosts: Set[str] = set() + + url_re = re.compile(r"(?:https?|ftp)://([^/\s]+)", re.IGNORECASE) + + for mf in managed_files: + p = str(mf.get("path") or "") + src_rel = str(mf.get("src_rel") or "") + if not p or not src_rel: + continue + + if p == "/etc/apt/sources.list" or p.startswith("/etc/apt/sources.list.d/"): + source_paths.append(p) + art_path = os.path.join(bundle_dir, "artifacts", role, src_rel) + try: + with open(art_path, "r", encoding="utf-8", errors="replace") as sf: + for line in sf: + line = line.strip() + if not line or line.startswith("#"): + continue + for m in url_re.finditer(line): + repo_hosts.add(m.group(1)) + except OSError: + pass # nosec + + if ( + p.startswith("/etc/apt/trusted.gpg") + or p.startswith("/etc/apt/keyrings/") + or p.startswith("/usr/share/keyrings/") + ): + keyring_paths.append(p) + + source_paths = sorted(set(source_paths)) + keyring_paths = sorted(set(keyring_paths)) + repos = sorted(repo_hosts) + + readme = ( + """# apt_config + +APT configuration harvested from the system (sources, pinning, and keyrings). + +## Repository hosts +""" + + ("\n".join([f"- {h}" for h in repos]) or "- (none)") + + """\n +## Source files +""" + + ("\n".join([f"- {p}" for p in source_paths]) or "- (none)") + + """\n +## Keyrings +""" + + ("\n".join([f"- {p}" for p in keyring_paths]) or "- (none)") + + """\n +## Managed files +""" + + ( + "\n".join( + [f"- {mf.get('path')} ({mf.get('reason')})" for mf in managed_files] + ) + or "- (none)" + ) + + """\n +## Excluded +""" + + ( + "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) + or "- (none)" + ) + + """\n +## Notes +""" + + ("\n".join([f"- {n}" for n in notes]) or "- (none)") + + """\n""" + ) + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifested_apt_config_roles.append(role) + # ------------------------- # etc_custom role (unowned /etc not already attributed) # ------------------------- @@ -1512,7 +1666,8 @@ Generated for package `{pkg}`. manifested_pkg_roles.append(role) all_roles = ( - manifested_pkg_roles + manifested_apt_config_roles + + manifested_pkg_roles + manifested_service_roles + manifested_etc_custom_roles + manifested_usr_local_custom_roles