Manage apt stuff in its own role, not in etc_custom

enroll/diff.py

@@ -184,6 +184,12 @@ def _iter_managed_files(state: Dict[str, Any]) -> Iterable[Tuple[str, Dict[str,
     for mf in u.get("managed_files", []) or []:
         yield str(u_role), mf
 
+    # apt_config
+    ac = state.get("apt_config") or {}
+    ac_role = ac.get("role_name") or "apt_config"
+    for mf in ac.get("managed_files", []) or []:
+        yield str(ac_role), mf
+
     # etc_custom
     ec = state.get("etc_custom") or {}
     ec_role = ec.get("role_name") or "etc_custom"

enroll/harvest.py

@@ -77,6 +77,14 @@ class UsersSnapshot:
     notes: List[str]
 
 
+@dataclass
+class AptConfigSnapshot:
+    role_name: str
+    managed_files: List[ManagedFile]
+    excluded: List[ExcludedFile]
+    notes: List[str]
+
+
 @dataclass
 class EtcCustomSnapshot:
     role_name: str
@@ -126,7 +134,6 @@ ALLOWED_UNOWNED_EXTS = {
 }
 
 MAX_FILES_CAP = 4000
-
 MAX_UNOWNED_FILES_PER_ROLE = 500
 
 # Directories that are shared across many packages; never attribute unowned files in these trees to a single package.
@@ -401,30 +408,61 @@ def _parse_apt_signed_by(source_files: List[str]) -> Set[str]:
     return out
 
 
-def _iter_system_capture_paths() -> List[tuple[str, str]]:
-    """Return (path, reason) pairs for essential system config/state."""
-    out: List[tuple[str, str]] = []
+def _iter_apt_capture_paths() -> List[tuple[str, str]]:
+    """Return (path, reason) pairs for APT configuration.
 
-    # APT: capture sources and related config
+    This captures the full /etc/apt tree (subject to IgnorePolicy at copy time),
+    plus any keyrings referenced via signed-by/Signed-By which may live outside
+    /etc (e.g. /usr/share/keyrings).
+    """
+    reasons: Dict[str, str] = {}
+
+    # Capture all regular files under /etc/apt (no symlinks).
+    if os.path.isdir("/etc/apt"):
+        for dirpath, _, filenames in os.walk("/etc/apt"):
+            for fn in filenames:
+                p = os.path.join(dirpath, fn)
+                if os.path.islink(p) or not os.path.isfile(p):
+                    continue
+                reasons.setdefault(p, "apt_config")
+
+    # Identify source files explicitly for nicer reasons and keyring discovery.
     apt_sources: List[str] = []
     for g in _APT_SOURCE_GLOBS:
         apt_sources.extend(_iter_matching_files(g))
     for p in sorted(set(apt_sources)):
-        out.append((p, "system_apt_sources"))
+        reasons[p] = "apt_source"
 
-    # APT: misc config files/dirs
-    for g in _APT_MISC_GLOBS:
+    # Keyrings in standard locations.
+    for g in (
+        "/etc/apt/trusted.gpg",
+        "/etc/apt/trusted.gpg.d/*",
+        "/etc/apt/keyrings/*",
+    ):
         for p in _iter_matching_files(g):
-            out.append((p, "system_apt_config"))
+            reasons[p] = "apt_keyring"
 
-    # APT: referenced keyrings (may live outside /etc)
+    # Keyrings referenced by sources (may live outside /etc/apt).
     signed_by = _parse_apt_signed_by(sorted(set(apt_sources)))
     for p in sorted(signed_by):
         if os.path.islink(p) or not os.path.isfile(p):
             continue
-        out.append((p, "system_apt_keyring"))
+        if p.startswith("/etc/apt/"):
+            reasons[p] = "apt_keyring"
+        else:
+            reasons[p] = "apt_signed_by_keyring"
+
+    # De-dup with stable ordering.
+    uniq: List[tuple[str, str]] = []
+    for p in sorted(reasons.keys()):
+        uniq.append((p, reasons[p]))
+    return uniq
+
+
+def _iter_system_capture_paths() -> List[tuple[str, str]]:
+    """Return (path, reason) pairs for essential system config/state (non-APT)."""
+    out: List[tuple[str, str]] = []
 
-    # Other system config/state globs
     for spec, reason in _SYSTEM_CAPTURE_GLOBS:
         for p in _iter_matching_files(spec):
             out.append((p, reason))
@@ -544,6 +582,8 @@ def harvest(
             for path in pkg_to_etc_paths.get(pkg, []):
                 if not os.path.isfile(path) or os.path.islink(path):
                     continue
+                if path.startswith("/etc/apt/"):
+                    continue
                 if path in conff:
                     # Only capture conffiles when they differ from the package default.
                     try:
@@ -784,6 +824,8 @@ def harvest(
         for path in pkg_to_etc_paths.get(pkg, []):
             if not os.path.isfile(path) or os.path.islink(path):
                 continue
+            if path.startswith("/etc/apt/"):
+                continue
             if path in conff:
                 try:
                     current = file_md5(path)
@@ -946,6 +988,55 @@ def harvest(
         notes=users_notes,
     )
 
+    # -------------------------
+    # apt_config role (APT configuration and keyrings)
+    # -------------------------
+    apt_notes: List[str] = []
+    apt_excluded: List[ExcludedFile] = []
+    apt_managed: List[ManagedFile] = []
+    apt_role_name = "apt_config"
+
+    for path, reason in _iter_apt_capture_paths():
+        if path_filter.is_excluded(path):
+            apt_excluded.append(ExcludedFile(path=path, reason="user_excluded"))
+            continue
+
+        deny = policy.deny_reason(path)
+        if deny:
+            apt_excluded.append(ExcludedFile(path=path, reason=deny))
+            continue
+
+        try:
+            owner, group, mode = stat_triplet(path)
+        except OSError:
+            apt_excluded.append(ExcludedFile(path=path, reason="unreadable"))
+            continue
+
+        src_rel = path.lstrip("/")
+        try:
+            _copy_into_bundle(bundle_dir, apt_role_name, path, src_rel)
+        except OSError:
+            apt_excluded.append(ExcludedFile(path=path, reason="unreadable"))
+            continue
+
+        apt_managed.append(
+            ManagedFile(
+                path=path,
+                src_rel=src_rel,
+                owner=owner,
+                group=group,
+                mode=mode,
+                reason=reason,
+            )
+        )
+
+    apt_config_snapshot = AptConfigSnapshot(
+        role_name=apt_role_name,
+        managed_files=apt_managed,
+        excluded=apt_excluded,
+        notes=apt_notes,
+    )
+
     # -------------------------
     # etc_custom role (unowned /etc files not already attributed elsewhere)
     # -------------------------
@@ -964,6 +1055,8 @@ def harvest(
             already.add(mf.path)
     for mf in users_managed:
         already.add(mf.path)
+    for mf in apt_managed:
+        already.add(mf.path)
 
     # Maps for re-attributing shared snippets (cron.d/logrotate.d) to existing roles.
     svc_by_role: Dict[str, ServiceSnapshot] = {s.role_name: s for s in service_snaps}
@@ -1107,6 +1200,8 @@ def harvest(
     for dirpath, _, filenames in os.walk("/etc"):
         for fn in filenames:
             path = os.path.join(dirpath, fn)
+            if path.startswith("/etc/apt/"):
+                continue
             if path in already:
                 continue
             if path in owned_etc:
@@ -1408,6 +1503,7 @@ def harvest(
         "manual_packages": manual_pkgs,
         "manual_packages_skipped": manual_pkgs_skipped,
         "package_roles": [asdict(p) for p in pkg_snaps],
+        "apt_config": asdict(apt_config_snapshot),
         "etc_custom": asdict(etc_custom_snapshot),
         "usr_local_custom": asdict(usr_local_custom_snapshot),
         "extra_paths": asdict(extra_paths_snapshot),

enroll/manifest.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import json
 import os
+import re
 import shutil
 import stat
 import tarfile
@@ -627,6 +628,7 @@ def _manifest_from_bundle_dir(
     services: List[Dict[str, Any]] = state.get("services", [])
     package_roles: List[Dict[str, Any]] = state.get("package_roles", [])
     users_snapshot: Dict[str, Any] = state.get("users", {})
+    apt_config_snapshot: Dict[str, Any] = state.get("apt_config", {})
     etc_custom_snapshot: Dict[str, Any] = state.get("etc_custom", {})
     usr_local_custom_snapshot: Dict[str, Any] = state.get("usr_local_custom", {})
     extra_paths_snapshot: Dict[str, Any] = state.get("extra_paths", {})
@@ -661,6 +663,7 @@ def _manifest_from_bundle_dir(
         _ensure_ansible_cfg(os.path.join(out_dir, "ansible.cfg"))
 
     manifested_users_roles: List[str] = []
+    manifested_apt_config_roles: List[str] = []
     manifested_etc_custom_roles: List[str] = []
     manifested_usr_local_custom_roles: List[str] = []
     manifested_extra_paths_roles: List[str] = []
@@ -887,6 +890,157 @@ Generated non-system user accounts and SSH public material.
 
         manifested_users_roles.append(role)
 
+    # -------------------------
+    # apt_config role (APT sources, pinning, and keyrings)
+    # -------------------------
+    if apt_config_snapshot and apt_config_snapshot.get("managed_files"):
+        role = apt_config_snapshot.get("role_name", "apt_config")
+        role_dir = os.path.join(roles_root, role)
+        _write_role_scaffold(role_dir)
+
+        var_prefix = role
+
+        managed_files = apt_config_snapshot.get("managed_files", [])
+        excluded = apt_config_snapshot.get("excluded", [])
+        notes = apt_config_snapshot.get("notes", [])
+
+        templated, jt_vars = _jinjify_managed_files(
+            bundle_dir,
+            role,
+            role_dir,
+            managed_files,
+            jt_exe=jt_exe,
+            jt_enabled=jt_enabled,
+            overwrite_templates=not site_mode,
+        )
+
+        # Copy only the non-templated artifacts (templates live in the role).
+        if site_mode:
+            _copy_artifacts(
+                bundle_dir,
+                role,
+                _host_role_files_dir(out_dir, fqdn or "", role),
+                exclude_rels=templated,
+            )
+        else:
+            _copy_artifacts(
+                bundle_dir,
+                role,
+                os.path.join(role_dir, "files"),
+                exclude_rels=templated,
+            )
+
+        files_var = _build_managed_files_var(
+            managed_files,
+            templated,
+            notify_other=None,
+            notify_systemd=None,
+        )
+
+        jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {}
+        vars_map: Dict[str, Any] = {f"{var_prefix}_managed_files": files_var}
+        vars_map = _merge_mappings_overwrite(vars_map, jt_map)
+
+        if site_mode:
+            _write_role_defaults(role_dir, {f"{var_prefix}_managed_files": []})
+            _write_hostvars(out_dir, fqdn or "", role, vars_map)
+        else:
+            _write_role_defaults(role_dir, vars_map)
+
+        tasks = """---\n""" + _render_generic_files_tasks(
+            var_prefix, include_restart_notify=False
+        )
+        with open(
+            os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8"
+        ) as f:
+            f.write(tasks.rstrip() + "\n")
+
+        with open(
+            os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8"
+        ) as f:
+            f.write("---\ndependencies: []\n")
+
+        # README: summarise repos and keyrings
+        source_paths: List[str] = []
+        keyring_paths: List[str] = []
+        repo_hosts: Set[str] = set()
+
+        url_re = re.compile(r"(?:https?|ftp)://([^/\s]+)", re.IGNORECASE)
+
+        for mf in managed_files:
+            p = str(mf.get("path") or "")
+            src_rel = str(mf.get("src_rel") or "")
+            if not p or not src_rel:
+                continue
+
+            if p == "/etc/apt/sources.list" or p.startswith("/etc/apt/sources.list.d/"):
+                source_paths.append(p)
+                art_path = os.path.join(bundle_dir, "artifacts", role, src_rel)
+                try:
+                    with open(art_path, "r", encoding="utf-8", errors="replace") as sf:
+                        for line in sf:
+                            line = line.strip()
+                            if not line or line.startswith("#"):
+                                continue
+                            for m in url_re.finditer(line):
+                                repo_hosts.add(m.group(1))
+                except OSError:
+                    pass  # nosec
+
+            if (
+                p.startswith("/etc/apt/trusted.gpg")
+                or p.startswith("/etc/apt/keyrings/")
+                or p.startswith("/usr/share/keyrings/")
+            ):
+                keyring_paths.append(p)
+
+        source_paths = sorted(set(source_paths))
+        keyring_paths = sorted(set(keyring_paths))
+        repos = sorted(repo_hosts)
+
+        readme = (
+            """# apt_config
+
+APT configuration harvested from the system (sources, pinning, and keyrings).
+
+## Repository hosts
+"""
+            + ("\n".join([f"- {h}" for h in repos]) or "- (none)")
+            + """\n
+## Source files
+"""
+            + ("\n".join([f"- {p}" for p in source_paths]) or "- (none)")
+            + """\n
+## Keyrings
+"""
+            + ("\n".join([f"- {p}" for p in keyring_paths]) or "- (none)")
+            + """\n
+## Managed files
+"""
+            + (
+                "\n".join(
+                    [f"- {mf.get('path')} ({mf.get('reason')})" for mf in managed_files]
+                )
+                or "- (none)"
+            )
+            + """\n
+## Excluded
+"""
+            + (
+                "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded])
+                or "- (none)"
+            )
+            + """\n
+## Notes
+"""
+            + ("\n".join([f"- {n}" for n in notes]) or "- (none)")
+            + """\n"""
+        )
+        with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f:
+            f.write(readme)
+
+        manifested_apt_config_roles.append(role)
+
     # -------------------------
     # etc_custom role (unowned /etc not already attributed)
     # -------------------------
@@ -1512,7 +1666,8 @@ Generated for package `{pkg}`.
 
         manifested_pkg_roles.append(role)
     all_roles = (
-        manifested_pkg_roles
+        manifested_apt_config_roles
+        + manifested_pkg_roles
         + manifested_service_roles
         + manifested_etc_custom_roles
         + manifested_usr_local_custom_roles