From de7531424dcd479bc8936e8fccb7f0af83a8d2ed Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Wed, 17 Jun 2026 09:37:32 +1000 Subject: [PATCH] Huge refactor to support extending a generic Config Manager class for different types (Ansible, Puppet... Salt soon?) --- CHANGELOG.md | 1 + enroll/ansible.py | 3131 ++++++++++++++++++ enroll/cli.py | 2 +- enroll/cm.py | 299 ++ enroll/diff.py | 22 +- enroll/explain.py | 5 +- enroll/harvest.py | 1144 +------ enroll/harvest_collectors/__init__.py | 27 + enroll/harvest_collectors/context.py | 32 + enroll/harvest_collectors/cron_logrotate.py | 159 + enroll/harvest_collectors/package_manager.py | 81 + enroll/harvest_collectors/paths.py | 247 ++ enroll/harvest_collectors/runtime.py | 64 + enroll/harvest_collectors/services.py | 525 +++ enroll/harvest_collectors/users.py | 167 + enroll/manifest.py | 3087 +---------------- enroll/puppet.py | 550 ++- enroll/state.py | 53 + enroll/validate.py | 3 +- tests/test_cm.py | 40 + tests/test_harvest_collectors.py | 44 + tests/test_jinjaturtle.py | 5 +- tests/test_manifest.py | 169 +- tests/test_manifest_ansible_model.py | 91 + 24 files changed, 5413 insertions(+), 4535 deletions(-) create mode 100644 enroll/ansible.py create mode 100644 enroll/cm.py create mode 100644 enroll/harvest_collectors/__init__.py create mode 100644 enroll/harvest_collectors/context.py create mode 100644 enroll/harvest_collectors/cron_logrotate.py create mode 100644 enroll/harvest_collectors/package_manager.py create mode 100644 enroll/harvest_collectors/paths.py create mode 100644 enroll/harvest_collectors/runtime.py create mode 100644 enroll/harvest_collectors/services.py create mode 100644 enroll/harvest_collectors/users.py create mode 100644 enroll/state.py create mode 100644 tests/test_cm.py create mode 100644 tests/test_harvest_collectors.py create mode 100644 tests/test_manifest_ansible_model.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 75665e5..2d4cb1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ * Detect active sysctl parameters and write them to a `/etc/sysctl.d/99-enroll.conf` file * Use `no_log` on systemd unit interrogations to suppress potential sensitive output when applying Ansible * Support manifesting Puppet code, as well as Ansible! + * A lot of under-the-bonnet refactoring to make it easier to extend to cover other config managers later e.g Salt # 0.6.0 diff --git a/enroll/ansible.py b/enroll/ansible.py new file mode 100644 index 0000000..29a5e55 --- /dev/null +++ b/enroll/ansible.py @@ -0,0 +1,3131 @@ +from __future__ import annotations + +import os +import re +import shutil +import stat +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Set, Tuple + +from .cm import CMModule, package_section_label, section_label_for_packages +from .jinjaturtle import ( + can_jinjify_path, + find_jinjaturtle_cmd, + infer_other_formats, + run_jinjaturtle, +) +from .role_names import avoid_reserved_role_name +from .state import inventory_packages_from_state, roles_from_state + + +@dataclass +class AnsibleManifestContext: + bundle_dir: str + out_dir: str + roles_root: str + fqdn: Optional[str] + site_mode: bool + jt_exe: Optional[str] + jt_enabled: bool + + +@dataclass +class AnsibleRoleCollection: + services: List[Dict[str, Any]] + packages: List[Dict[str, Any]] + common_role_groups: Dict[str, List[Dict[str, Any]]] + + +class AnsibleRole(CMModule): + """Ansible-specific view of a renderer-neutral CMModule.""" + + def __init__( + self, + role_name: str, + *, + var_prefix: Optional[str] = None, + section_label: Optional[str] = None, + grouped: bool = False, + ) -> None: + super().__init__(role_name=role_name, module_name=role_name) + self.var_prefix = var_prefix or role_name + self.section_label = section_label + self.grouped = grouped + self.entries: List[Dict[str, Any]] = [] + self.excluded: List[Dict[str, Any]] = [] + self.origin_lines: List[str] = [] + + def add_package_snapshot(self, snap: Dict[str, Any]) -> None: + pkg = str(snap.get("package") or "").strip() + source_role = str(snap.get("role_name") or pkg or self.role_name) + self.entries.append({"kind": "package", "snapshot": snap}) + if pkg: + self.packages.add(pkg) + self.origin_lines.append(f"package `{pkg}` from role `{source_role}`") + self.add_managed_content(snap) + + def add_service_snapshot(self, snap: Dict[str, Any]) -> None: + unit = str(snap.get("unit") or "").strip() + source_role = str(snap.get("role_name") or unit or self.role_name) + self.entries.append({"kind": "service", "snapshot": snap}) + for pkg in snap.get("packages", []) or []: + pkg_s = str(pkg or "").strip() + if pkg_s: + self.packages.add(pkg_s) + if unit: + unit_file_state = str(snap.get("unit_file_state") or "") + self.services.setdefault( + unit, + { + "name": unit, + "manage": True, + "enabled": unit_file_state in ("enabled", "enabled-runtime"), + "state": ( + "started" if snap.get("active_state") == "active" else "stopped" + ), + }, + ) + self.origin_lines.append(f"service `{unit}` from role `{source_role}`") + self.add_managed_content(snap) + + def add_managed_content(self, snap: Dict[str, Any]) -> None: + for d in self.managed_dirs_from_snapshot(snap): + path = str(d.get("path") or "").strip() + self.add_managed_dir( + path, + dest=path, + owner=d.get("owner") or "root", + group=d.get("group") or "root", + mode=d.get("mode") or "0755", + ) + + for mf in self.managed_files_from_snapshot(snap): + path = str(mf.get("path") or "").strip() + src_rel = str(mf.get("src_rel") or "").strip() + if not path or not src_rel: + continue + self.add_managed_file( + path, + dest=path, + src_rel=src_rel, + owner=mf.get("owner") or "root", + group=mf.get("group") or "root", + mode=mf.get("mode") or "0644", + reason=mf.get("reason") or "managed_file", + ) + + for ml in self.managed_links_from_snapshot(snap): + path = str(ml.get("path") or "").strip() + target = str(ml.get("target") or "").strip() + if not path or not target: + continue + self.add_managed_link(path, dest=path, src=target) + + self.excluded.extend(snap.get("excluded", []) or []) + self.add_snapshot_notes(snap) + + @property + def sorted_packages(self) -> List[str]: + return sorted(self.packages) + + @property + def systemd_units_var(self) -> List[Dict[str, Any]]: + return [self.services[k] for k in sorted(self.services)] + + +class AnsibleManifestPlan: + """Track generated Ansible roles without scattering category lists.""" + + _ORDER = ( + "apt_config", + "dnf_config", + "package", + "service", + "etc_custom", + "usr_local_custom", + "extra_paths", + "flatpak", + "snap", + "users", + "tail_package", + "sysctl", + "firewall_runtime", + ) + + def __init__(self) -> None: + self._roles: Dict[str, List[str]] = {category: [] for category in self._ORDER} + self._tail_packages: List[str] = [] + + def add(self, category: str, role: str) -> None: + if category not in self._roles: + raise ValueError(f"unknown Ansible role category: {category}") + if role and role not in self._roles[category]: + self._roles[category].append(role) + + def roles(self, category: str) -> List[str]: + return list(self._roles.get(category, [])) + + def has(self, category: str, role: str) -> bool: + return role in self._roles.get(category, []) + + def mark_tail_package(self, role: str) -> None: + if self.has("package", role) and role not in self._tail_packages: + self._tail_packages.append(role) + + def ordered_roles(self) -> List[str]: + tail = set(self._tail_packages) + package_roles = [r for r in self._roles["package"] if r not in tail] + out: List[str] = [] + for category in self._ORDER: + if category == "package": + out.extend(package_roles) + elif category == "tail_package": + out.extend(self._tail_packages) + else: + out.extend(self._roles[category]) + return out + + +def _try_yaml(): + try: + import yaml # type: ignore + except Exception: + return None + return yaml + + +def _yaml_load_mapping(text: str) -> Dict[str, Any]: + yaml = _try_yaml() + if yaml is None: + return {} + try: + obj = yaml.safe_load(text) + except Exception: + return {} + if obj is None: + return {} + if isinstance(obj, dict): + return obj + return {} + + +def _yaml_dump_mapping(obj: Dict[str, Any], *, sort_keys: bool = True) -> str: + yaml = _try_yaml() + if yaml is None: + # fall back to a naive key: value dump (best-effort) + lines: List[str] = [] + for k, v in sorted(obj.items()) if sort_keys else obj.items(): + lines.append(f"{k}: {v!r}") + return "\n".join(lines).rstrip() + "\n" + + # ansible-lint/yamllint's indentation rules are stricter than YAML itself. + # In particular, they expect sequences nested under a mapping key to be + # indented (e.g. `foo:\n - a`), whereas PyYAML's default is often + # `foo:\n- a`. + class _IndentDumper(yaml.SafeDumper): # type: ignore + def increase_indent(self, flow: bool = False, indentless: bool = False): + return super().increase_indent(flow, False) + + return ( + yaml.dump( + obj, + Dumper=_IndentDumper, + default_flow_style=False, + sort_keys=sort_keys, + indent=2, + allow_unicode=True, + ).rstrip() + + "\n" + ) + + +def _role_id(raw: str) -> str: + """Return an Ansible-safe role identifier from an arbitrary label.""" + + s = re.sub(r"[^A-Za-z0-9]+", "_", raw or "misc") + s = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s) + s = s.lower() + s = re.sub(r"_+", "_", s).strip("_") + if not s: + s = "misc" + if not re.match(r"^[a-z_]", s): + s = "r_" + s + return s + + +def _section_role_name(label: str, occupied_roles: Set[str]) -> str: + """Create a stable section role name, avoiding generated-role collisions.""" + + base = avoid_reserved_role_name(_role_id(label), prefix="section") + role = base if base not in occupied_roles else f"section_{base}" + n = 2 + while role in occupied_roles: + role = f"section_{base}_{n}" + n += 1 + occupied_roles.add(role) + return role + + +def _merge_mappings_overwrite( + existing: Dict[str, Any], incoming: Dict[str, Any] +) -> Dict[str, Any]: + """Merge incoming into existing with overwrite. + + NOTE: Unlike role defaults merging, host_vars should reflect the current + harvest for a host. Therefore lists are replaced rather than unioned. + """ + merged = dict(existing) + merged.update(incoming) + return merged + + +def _copy2_replace(src: str, dst: str) -> None: + dst_dir = os.path.dirname(dst) + os.makedirs(dst_dir, exist_ok=True) + + # Copy to a temp file in the same directory, then atomically replace. + fd, tmp = tempfile.mkstemp(prefix=".enroll-tmp-", dir=dst_dir) + os.close(fd) + try: + shutil.copy2(src, tmp) + + # Ensure the working tree stays mergeable: make the file user-writable. + st = os.stat(tmp, follow_symlinks=False) + mode = stat.S_IMODE(st.st_mode) + if not (mode & stat.S_IWUSR): + os.chmod(tmp, mode | stat.S_IWUSR) + + os.replace(tmp, dst) + finally: + try: + os.unlink(tmp) + except FileNotFoundError: + pass + + +def _copy_artifacts( + bundle_dir: str, + role: str, + dst_files_dir: str, + *, + preserve_existing: bool = False, + exclude_rels: Optional[Set[str]] = None, +) -> None: + """Copy harvested artifacts for a role into a destination *files* directory. + + In non --fqdn mode, this is usually /files. + In --fqdn site mode, this is usually: + inventory/host_vars///.files + """ + artifacts_dir = os.path.join(bundle_dir, "artifacts", role) + if not os.path.isdir(artifacts_dir): + return + for root, _, files in os.walk(artifacts_dir): + for fn in files: + src = os.path.join(root, fn) + rel = os.path.relpath(src, artifacts_dir) + dst = os.path.join(dst_files_dir, rel) + + # If a file was successfully templatised by JinjaTurtle, do NOT + # also materialise the raw copy in the destination files dir. + if exclude_rels and rel in exclude_rels: + try: + if os.path.isfile(dst): + os.remove(dst) + except Exception: + pass # nosec + continue + + if preserve_existing and os.path.exists(dst): + continue + os.makedirs(os.path.dirname(dst), exist_ok=True) + _copy2_replace(src, dst) + + +def _write_role_scaffold(role_dir: str) -> None: + os.makedirs(os.path.join(role_dir, "tasks"), exist_ok=True) + os.makedirs(os.path.join(role_dir, "handlers"), exist_ok=True) + os.makedirs(os.path.join(role_dir, "defaults"), exist_ok=True) + os.makedirs(os.path.join(role_dir, "meta"), exist_ok=True) + os.makedirs(os.path.join(role_dir, "files"), exist_ok=True) + os.makedirs(os.path.join(role_dir, "templates"), exist_ok=True) + + +def _role_tag(role: str) -> str: + """Return a stable Ansible tag name for a role. + + Used by `enroll diff --enforce` to run only the roles needed to repair drift. + """ + r = str(role or "").strip() + # Ansible tag charset is fairly permissive, but keep it portable and consistent. + safe = re.sub(r"[^A-Za-z0-9_-]+", "_", r).strip("_") + if not safe: + safe = "other" + return f"role_{safe}" + + +def _write_playbook_all(path: str, roles: List[str]) -> None: + pb_lines = [ + "---", + "- name: Apply all roles on all hosts", + " gather_facts: true", + " hosts: all", + " become: true", + " roles:", + ] + for r in roles: + pb_lines.append(f" - role: {r}") + pb_lines.append(f" tags: [{_role_tag(r)}]") + with open(path, "w", encoding="utf-8") as f: + f.write("\n".join(pb_lines) + "\n") + + +def _write_playbook_host(path: str, fqdn: str, roles: List[str]) -> None: + pb_lines = [ + "---", + f"- name: Apply all roles on {fqdn}", + f" hosts: {fqdn}", + " gather_facts: true", + " become: true", + " roles:", + ] + for r in roles: + pb_lines.append(f" - role: {r}") + pb_lines.append(f" tags: [{_role_tag(r)}]") + with open(path, "w", encoding="utf-8") as f: + f.write("\n".join(pb_lines) + "\n") + + +def _ensure_ansible_cfg(cfg_path: str) -> None: + if not os.path.exists(cfg_path): + with open(cfg_path, "w", encoding="utf-8") as f: + f.write("[defaults]\n") + f.write("roles_path = roles\n") + f.write("interpreter_python=/usr/bin/python3\n") + f.write("inventory = inventory\n") + f.write("stdout_callback = unixy\n") + f.write("force_color = 1\n") + f.write("vars_plugins_enabled = host_group_vars\n") + f.write("fact_caching = jsonfile\n") + f.write("fact_caching_connection = .enroll_cached_facts\n") + f.write("forks = 30\n") + f.write("remote_tmp = /tmp/ansible-${USER}\n") + f.write("timeout = 12\n") + f.write("[ssh_connection]\n") + f.write("pipelining = True\n") + f.write("scp_if_ssh = True\n") + return + + +def _ensure_requirements_yaml(req_path: str) -> None: + if not os.path.exists(req_path): + with open(req_path, "w", encoding="utf-8") as f: + f.write("---\n") + f.write("collections:\n") + f.write(" - name: community.general\n") + f.write(' version: ">=13.0.0"\n') + return + + +def _normalise_flatpak_item( + item: Any, + *, + method: str, + user: Optional[str] = None, + home: Optional[str] = None, +) -> Dict[str, Any]: + if isinstance(item, str): + out: Dict[str, Any] = {"name": item, "method": method} + elif isinstance(item, dict): + out = dict(item) + out.setdefault("method", method) + else: + out = {"name": str(item), "method": method} + if user: + out.setdefault("user", user) + if home: + out.setdefault("home", home) + return out + + +def _normalise_flatpak_remote(item: Any) -> Dict[str, Any]: + if isinstance(item, dict): + out = dict(item) + else: + out = {"name": str(item)} + out.setdefault("method", "system") + return out + + +def _normalise_snap_item(item: Any) -> Dict[str, Any]: + if isinstance(item, str): + out: Dict[str, Any] = {"name": item} + elif isinstance(item, dict): + out = dict(item) + else: + out = {"name": str(item)} + + notes = out.get("notes") or [] + if isinstance(notes, str): + notes = [notes] + notes_l = {str(n).lower() for n in notes} + out["classic"] = bool(out.get("classic") or "classic" in notes_l) + out["devmode"] = bool(out.get("devmode") or "devmode" in notes_l) + out["dangerous"] = bool(out.get("dangerous") or "dangerous" in notes_l) + + # The Ansible snap module's revision parameter pins/holds the snap. For + # ordinary store snaps that track a channel, preserve the channel instead + # of freezing every harvested host at today's revision. + if out.get("revision") is not None and not out.get("channel"): + out["install_revision"] = True + else: + out["install_revision"] = False + return out + + +def _ensure_inventory_host(inv_path: str, fqdn: str) -> None: + os.makedirs(os.path.dirname(inv_path), exist_ok=True) + if not os.path.exists(inv_path): + with open(inv_path, "w", encoding="utf-8") as f: + f.write("[all]\n") + f.write(fqdn + "\n") + return + + with open(inv_path, "r", encoding="utf-8") as f: + lines = [ln.rstrip("\n") for ln in f.readlines()] + + # ensure there is an [all] group; if not, create it at top + if not any(ln.strip() == "[all]" for ln in lines): + lines = ["[all]"] + lines + + # check if fqdn already present (exact match, ignoring whitespace) + if any(ln.strip() == fqdn for ln in lines): + return + + # append at end + lines.append(fqdn) + with open(inv_path, "w", encoding="utf-8") as f: + f.write("\n".join(lines) + "\n") + + +def _hostvars_path(site_root: str, fqdn: str, role: str) -> str: + return os.path.join(site_root, "inventory", "host_vars", fqdn, f"{role}.yml") + + +def _host_role_files_dir(site_root: str, fqdn: str, role: str) -> str: + """Host-specific files dir for a given role. + + Layout: + inventory/host_vars///.files/ + """ + return os.path.join(site_root, "inventory", "host_vars", fqdn, role, ".files") + + +def _write_hostvars(site_root: str, fqdn: str, role: str, data: Dict[str, Any]) -> None: + """Write host_vars YAML for a role for a specific host. + + This is host-specific state and should track the current harvest output. + Existing keys not mentioned in `data` are preserved, but keys in `data` + are overwritten (including list values). + """ + path = _hostvars_path(site_root, fqdn, role) + os.makedirs(os.path.dirname(path), exist_ok=True) + + existing_map: Dict[str, Any] = {} + if os.path.exists(path): + try: + existing_text = Path(path).read_text(encoding="utf-8") + existing_map = _yaml_load_mapping(existing_text) + except Exception: + existing_map = {} + + merged = _merge_mappings_overwrite(existing_map, data) + + out = "---\n" + _yaml_dump_mapping(merged, sort_keys=True) + with open(path, "w", encoding="utf-8") as f: + f.write(out) + + +def _jinjify_managed_files( + bundle_dir: str, + role: str, + role_dir: str, + managed_files: List[Dict[str, Any]], + *, + jt_exe: Optional[str], + jt_enabled: bool, + overwrite_templates: bool, +) -> Tuple[Set[str], str]: + """ + Return (templated_src_rels, combined_vars_text). + combined_vars_text is a YAML mapping fragment (no leading ---). + """ + templated: Set[str] = set() + vars_map: Dict[str, Any] = {} + + if not (jt_enabled and jt_exe): + return templated, "" + + for mf in managed_files: + dest_path = mf.get("path", "") + src_rel = mf.get("src_rel", "") + if not dest_path or not src_rel: + continue + if not can_jinjify_path(dest_path): + continue + + artifact_path = os.path.join(bundle_dir, "artifacts", role, src_rel) + if not os.path.isfile(artifact_path): + continue + + try: + force_fmt = infer_other_formats(dest_path) + res = run_jinjaturtle( + jt_exe, artifact_path, role_name=role, force_format=force_fmt + ) + except Exception: + # If jinjaturtle cannot process a file for any reason, skip silently. + # (Enroll's core promise is to be optimistic and non-interactive.) + continue # nosec + + tmpl_rel = src_rel + ".j2" + tmpl_dst = os.path.join(role_dir, "templates", tmpl_rel) + if overwrite_templates or not os.path.exists(tmpl_dst): + os.makedirs(os.path.dirname(tmpl_dst), exist_ok=True) + with open(tmpl_dst, "w", encoding="utf-8") as f: + f.write(res.template_text) + + templated.add(src_rel) + if res.vars_text.strip(): + # merge YAML mappings; last wins (avoids duplicate keys) + chunk = _yaml_load_mapping(res.vars_text) + if chunk: + vars_map = _merge_mappings_overwrite(vars_map, chunk) + + if vars_map: + combined = _yaml_dump_mapping(vars_map, sort_keys=True) + return templated, combined + return templated, "" + + +def _write_role_defaults(role_dir: str, mapping: Dict[str, Any]) -> None: + """Overwrite role defaults/main.yml with the provided mapping.""" + defaults_path = os.path.join(role_dir, "defaults", "main.yml") + os.makedirs(os.path.dirname(defaults_path), exist_ok=True) + out = "---\n" + _yaml_dump_mapping(mapping, sort_keys=True) + with open(defaults_path, "w", encoding="utf-8") as f: + f.write(out) + + +def _build_managed_dirs_var( + managed_dirs: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Convert enroll managed_dirs into an Ansible-friendly list of dicts. + + Each dict drives a role task loop and is safe across hosts. + """ + out: List[Dict[str, Any]] = [] + for d in managed_dirs: + dest = d.get("path") or "" + if not dest: + continue + out.append( + { + "dest": dest, + "owner": d.get("owner") or "root", + "group": d.get("group") or "root", + "mode": d.get("mode") or "0755", + } + ) + return out + + +def _build_managed_files_var( + managed_files: List[Dict[str, Any]], + templated_src_rels: Set[str], + *, + notify_other: Optional[str] = None, + notify_systemd: Optional[str] = None, +) -> List[Dict[str, Any]]: + """Convert enroll managed_files into an Ansible-friendly list of dicts. + + Each dict drives a role task loop and is safe across hosts. + """ + out: List[Dict[str, Any]] = [] + for mf in managed_files: + dest = mf.get("path") or "" + src_rel = mf.get("src_rel") or "" + if not dest or not src_rel: + continue + is_unit = str(dest).startswith("/etc/systemd/system/") + kind = "template" if src_rel in templated_src_rels else "copy" + notify: List[str] = [] + if is_unit and notify_systemd: + notify.append(notify_systemd) + if (not is_unit) and notify_other: + notify.append(notify_other) + out.append( + { + "dest": dest, + "src_rel": src_rel, + "owner": mf.get("owner") or "root", + "group": mf.get("group") or "root", + "mode": mf.get("mode") or "0644", + "kind": kind, + "is_systemd_unit": bool(is_unit), + "notify": notify, + } + ) + return out + + +def _build_managed_links_var( + managed_links: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Convert enroll managed_links into an Ansible-friendly list of dicts.""" + out: List[Dict[str, Any]] = [] + for ml in managed_links or []: + dest = ml.get("path") or "" + src = ml.get("target") or "" + if not dest or not src: + continue + out.append({"dest": dest, "src": src}) + return out + + +def _render_generic_files_tasks( + var_prefix: str, *, include_restart_notify: bool +) -> str: + """Render generic tasks to deploy _managed_files safely.""" + # Using first_found makes roles work in both modes: + # - site-mode: inventory/host_vars///.files/... + # - non-site: roles//files/... + return f"""- name: Ensure managed directories exist (preserve owner/group/mode) + ansible.builtin.file: + path: "{{{{ item.dest }}}}" + state: directory + owner: "{{{{ item.owner }}}}" + group: "{{{{ item.group }}}}" + mode: "{{{{ item.mode }}}}" + loop: "{{{{ {var_prefix}_managed_dirs | default([]) }}}}" + +- name: Deploy any systemd unit files (templates) + ansible.builtin.template: + src: "{{{{ item.src_rel }}}}.j2" + dest: "{{{{ item.dest }}}}" + owner: "{{{{ item.owner }}}}" + group: "{{{{ item.group }}}}" + mode: "{{{{ item.mode }}}}" + loop: >- + {{{{ {var_prefix}_managed_files | default([]) + | selectattr('is_systemd_unit', 'equalto', true) + | selectattr('kind', 'equalto', 'template') + | list }}}} + notify: "{{{{ item.notify | default([]) }}}}" + +- name: Deploy any systemd unit files (raw files) + vars: + _enroll_ff: + files: + - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ item.src_rel }}}}" + - "{{{{ role_path }}}}/files/{{{{ item.src_rel }}}}" + ansible.builtin.copy: + src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" + dest: "{{{{ item.dest }}}}" + owner: "{{{{ item.owner }}}}" + group: "{{{{ item.group }}}}" + mode: "{{{{ item.mode }}}}" + loop: >- + {{{{ {var_prefix}_managed_files | default([]) + | selectattr('is_systemd_unit', 'equalto', true) + | selectattr('kind', 'equalto', 'copy') + | list }}}} + notify: "{{{{ item.notify | default([]) }}}}" + +- name: Reload systemd to pick up unit changes + ansible.builtin.meta: flush_handlers + when: >- + ({var_prefix}_managed_files | default([]) + | selectattr('is_systemd_unit', 'equalto', true) + | list + | length) > 0 + +- name: Deploy any other managed files (templates) + ansible.builtin.template: + src: "{{{{ item.src_rel }}}}.j2" + dest: "{{{{ item.dest }}}}" + owner: "{{{{ item.owner }}}}" + group: "{{{{ item.group }}}}" + mode: "{{{{ item.mode }}}}" + loop: >- + {{{{ {var_prefix}_managed_files | default([]) + | selectattr('is_systemd_unit', 'equalto', false) + | selectattr('kind', 'equalto', 'template') + | list }}}} + notify: "{{{{ item.notify | default([]) }}}}" + +- name: Deploy any other managed files (raw files) + vars: + _enroll_ff: + files: + - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ item.src_rel }}}}" + - "{{{{ role_path }}}}/files/{{{{ item.src_rel }}}}" + ansible.builtin.copy: + src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" + dest: "{{{{ item.dest }}}}" + owner: "{{{{ item.owner }}}}" + group: "{{{{ item.group }}}}" + mode: "{{{{ item.mode }}}}" + loop: >- + {{{{ {var_prefix}_managed_files | default([]) + | selectattr('is_systemd_unit', 'equalto', false) + | selectattr('kind', 'equalto', 'copy') + | list }}}} + notify: "{{{{ item.notify | default([]) }}}}" + +- name: Ensure managed symlinks exist + ansible.builtin.file: + src: "{{{{ item.src }}}}" + dest: "{{{{ item.dest }}}}" + state: link + force: true + loop: "{{{{ {var_prefix}_managed_links | default([]) }}}}" +""" + + +def _render_install_packages_tasks(role: str, var_prefix: str) -> str: + """Render cross-distro package installation tasks. + + We generate conditional tasks for apt/dnf/yum, falling back to the + generic `package` module. This keeps generated roles usable on both + Debian-like and RPM-like systems. + """ + return f"""- name: Install packages for {role} (APT) + ansible.builtin.apt: + name: "{{{{ {var_prefix}_packages | default([]) }}}}" + state: present + update_cache: true + when: + - ({var_prefix}_packages | default([])) | length > 0 + - ansible_facts.pkg_mgr | default('') == 'apt' + +- name: Install packages for {role} (DNF5) + ansible.builtin.dnf5: + name: "{{{{ {var_prefix}_packages | default([]) }}}}" + state: present + when: + - ({var_prefix}_packages | default([])) | length > 0 + - ansible_facts.pkg_mgr | default('') == 'dnf5' + +- name: Install packages for {role} (DNF/YUM) + ansible.builtin.dnf: + name: "{{{{ {var_prefix}_packages | default([]) }}}}" + state: present + when: + - ({var_prefix}_packages | default([])) | length > 0 + - ansible_facts.pkg_mgr | default('') in ['dnf', 'yum'] + +- name: Install packages for {role} (generic fallback) + ansible.builtin.package: + name: "{{{{ {var_prefix}_packages | default([]) }}}}" + state: present + when: + - ({var_prefix}_packages | default([])) | length > 0 + - ansible_facts.pkg_mgr | default('') not in ['apt', 'dnf', 'dnf5', 'yum'] + +""" + + +def _render_grouped_systemd_tasks(var_prefix: str) -> str: + """Render tasks to manage multiple systemd units in a common role.""" + + return f"""- name: Probe whether grouped systemd units exist and are manageable + ansible.builtin.systemd: + name: "{{{{ item.name }}}}" + no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" + check_mode: true + loop: "{{{{ {var_prefix}_systemd_units | default([]) }}}}" + register: _enroll_unit_probes + failed_when: false + changed_when: false + when: item.manage | default(false) + +- name: Ensure grouped unit enablement matches harvest + ansible.builtin.systemd: + name: "{{{{ item.item.name }}}}" + enabled: "{{{{ item.item.enabled | bool }}}}" + no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" + loop: "{{{{ _enroll_unit_probes.results | default([]) }}}}" + when: + - item.item.manage | default(false) + - not (item.failed | default(false)) + +- name: Ensure grouped unit running state matches harvest + ansible.builtin.systemd: + name: "{{{{ item.item.name }}}}" + state: "{{{{ item.item.state }}}}" + no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" + loop: "{{{{ _enroll_unit_probes.results | default([]) }}}}" + when: + - item.item.manage | default(false) + - not (item.failed | default(false)) +""" + + +def _render_sysctl_tasks(var_prefix: str) -> str: + return f"""- name: Ensure sysctl.d exists + ansible.builtin.file: + path: /etc/sysctl.d + state: directory + owner: root + group: root + mode: "0755" + +- name: Deploy captured sysctl configuration + vars: + _enroll_ff: + files: + - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ {var_prefix}_conf_src_rel }}}}" + - "{{{{ role_path }}}}/files/{{{{ {var_prefix}_conf_src_rel }}}}" + ansible.builtin.copy: + src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" + dest: /etc/sysctl.d/99-enroll.conf + owner: root + group: root + mode: "0644" + when: ({var_prefix}_conf_src_rel | default('') | length) > 0 + notify: Apply captured sysctl configuration +""" + + +def _render_sysctl_handlers(var_prefix: str) -> str: + return f"""--- +- name: Apply captured sysctl configuration + ansible.builtin.command: + argv: + - sysctl + - -e + - -p + - /etc/sysctl.d/99-enroll.conf + register: _enroll_sysctl_apply + changed_when: false + failed_when: + - not ({var_prefix}_ignore_apply_errors | default(true) | bool) + - _enroll_sysctl_apply.rc != 0 + when: {var_prefix}_apply | default(true) | bool +""" + + +def _render_firewall_runtime_tasks(var_prefix: str) -> str: + """Render tasks for live ipset/iptables snapshots.""" + return f"""- name: Ensure firewall runtime snapshot directory exists + ansible.builtin.file: + path: /etc/enroll/firewall + state: directory + owner: root + group: root + mode: "0750" + +- name: Deploy captured ipset snapshot + vars: + _enroll_ff: + files: + - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ {var_prefix}_ipset_save }}}}" + - "{{{{ role_path }}}}/files/{{{{ {var_prefix}_ipset_save }}}}" + ansible.builtin.copy: + src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" + dest: /etc/enroll/firewall/ipset.save + owner: root + group: root + mode: "0600" + when: ({var_prefix}_ipset_save | default('') | length) > 0 + +- name: Flush captured ipsets before restoring members + ansible.builtin.command: + cmd: "ipset flush {{{{ item }}}}" + loop: "{{{{ {var_prefix}_ipset_sets | default([]) }}}}" + register: _enroll_ipset_flush + failed_when: false + changed_when: false + when: + - ({var_prefix}_ipset_save | default('') | length) > 0 + - {var_prefix}_sync_ipsets_exact | default(true) | bool + +- name: Restore captured ipsets + ansible.builtin.shell: "ipset restore -exist < /etc/enroll/firewall/ipset.save" + args: + executable: /bin/sh + register: _enroll_ipset_restore + changed_when: _enroll_ipset_restore.rc == 0 + when: ({var_prefix}_ipset_save | default('') | length) > 0 + +- name: Deploy captured IPv4 iptables snapshot + vars: + _enroll_ff: + files: + - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ {var_prefix}_iptables_v4_save }}}}" + - "{{{{ role_path }}}}/files/{{{{ {var_prefix}_iptables_v4_save }}}}" + ansible.builtin.copy: + src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" + dest: /etc/enroll/firewall/iptables.v4 + owner: root + group: root + mode: "0600" + when: ({var_prefix}_iptables_v4_save | default('') | length) > 0 + +- name: Restore captured IPv4 iptables rules + ansible.builtin.command: + cmd: iptables-restore /etc/enroll/firewall/iptables.v4 + register: _enroll_iptables_v4_restore + changed_when: _enroll_iptables_v4_restore.rc == 0 + when: + - ({var_prefix}_iptables_v4_save | default('') | length) > 0 + - {var_prefix}_restore_iptables | default(true) | bool + +- name: Deploy captured IPv6 iptables snapshot + vars: + _enroll_ff: + files: + - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ {var_prefix}_iptables_v6_save }}}}" + - "{{{{ role_path }}}}/files/{{{{ {var_prefix}_iptables_v6_save }}}}" + ansible.builtin.copy: + src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" + dest: /etc/enroll/firewall/iptables.v6 + owner: root + group: root + mode: "0600" + when: ({var_prefix}_iptables_v6_save | default('') | length) > 0 + +- name: Restore captured IPv6 iptables rules + ansible.builtin.command: + cmd: ip6tables-restore /etc/enroll/firewall/iptables.v6 + register: _enroll_iptables_v6_restore + changed_when: _enroll_iptables_v6_restore.rc == 0 + when: + - ({var_prefix}_iptables_v6_save | default('') | length) > 0 + - {var_prefix}_restore_iptables | default(true) | bool +""" + + +def _markdown_list(items: List[str]) -> str: + values = [str(item) for item in items if str(item)] + return "\n".join(f"- {item}" for item in values) or "- (none)" + + +def _managed_file_lines( + managed_files: List[Dict[str, Any]], *, include_reason: bool +) -> List[str]: + out: List[str] = [] + for mf in managed_files: + path = str(mf.get("path") or "") + if not path: + continue + if include_reason: + out.append(f"{path} ({mf.get('reason')})") + else: + out.append(path) + return out + + +def _excluded_lines(excluded: List[Dict[str, Any]]) -> List[str]: + return [f"{e.get('path')} ({e.get('reason')})" for e in excluded if e.get("path")] + + +def _read_artifact_lines(bundle_dir: str, role: str, src_rel: str) -> List[str]: + art_path = os.path.join(bundle_dir, "artifacts", role, src_rel) + try: + with open(art_path, "r", encoding="utf-8", errors="replace") as f: + return [line.rstrip("\n") for line in f] + except OSError: + return [] + + +def _apt_config_readme( + *, + bundle_dir: str, + role: str, + snapshot: Dict[str, Any], + managed_files: List[Dict[str, Any]], + managed_dirs: List[Dict[str, Any]], + excluded: List[Dict[str, Any]], + notes: List[Any], +) -> str: + source_paths: List[str] = [] + keyring_paths: List[str] = [] + repo_hosts: Set[str] = set() + url_re = re.compile(r"(?:https?|ftp)://([^/\s]+)", re.IGNORECASE) + + for mf in managed_files: + path = str(mf.get("path") or "") + src_rel = str(mf.get("src_rel") or "") + if not path or not src_rel: + continue + if path == "/etc/apt/sources.list" or path.startswith( + "/etc/apt/sources.list.d/" + ): + source_paths.append(path) + for line in _read_artifact_lines(bundle_dir, role, src_rel): + s = line.strip() + if not s or s.startswith("#"): + continue + for match in url_re.finditer(s): + repo_hosts.add(match.group(1)) + if ( + path.startswith("/etc/apt/trusted.gpg") + or path.startswith("/etc/apt/keyrings/") + or path.startswith("/usr/share/keyrings/") + ): + keyring_paths.append(path) + + return f"""# apt_config + +APT configuration harvested from the system (sources, pinning, and keyrings). + +## Repository hosts +{_markdown_list(sorted(repo_hosts))} + +## Source files +{_markdown_list(sorted(set(source_paths)))} + +## Keyrings +{_markdown_list(sorted(set(keyring_paths)))} + +## Managed files +{_markdown_list(_managed_file_lines(managed_files, include_reason=True))} + +## Excluded +{_markdown_list(_excluded_lines(excluded))} + +## Notes +{_markdown_list([str(n) for n in notes])} +""" + + +def _dnf_config_readme( + *, + bundle_dir: str, + role: str, + snapshot: Dict[str, Any], + managed_files: List[Dict[str, Any]], + managed_dirs: List[Dict[str, Any]], + excluded: List[Dict[str, Any]], + notes: List[Any], +) -> str: + repo_paths: List[str] = [] + key_paths: List[str] = [] + repo_hosts: Set[str] = set() + url_re = re.compile(r"(?:https?|ftp)://([^/\s]+)", re.IGNORECASE) + file_url_re = re.compile(r"file://(/[^\s]+)") + + for mf in managed_files: + path = str(mf.get("path") or "") + src_rel = str(mf.get("src_rel") or "") + if not path or not src_rel: + continue + if path.startswith("/etc/yum.repos.d/") and path.endswith(".repo"): + repo_paths.append(path) + for line in _read_artifact_lines(bundle_dir, role, src_rel): + s = line.strip() + if not s or s.startswith("#") or s.startswith(";"): + continue + for match in url_re.finditer(s): + repo_hosts.add(match.group(1)) + for match in file_url_re.finditer(s): + key_paths.append(match.group(1)) + if path.startswith("/etc/pki/rpm-gpg/"): + key_paths.append(path) + + return f"""# dnf_config + +DNF/YUM configuration harvested from the system (repos, config files, and RPM GPG keys). + +## Repository hosts +{_markdown_list(sorted(repo_hosts))} + +## Repo files +{_markdown_list(sorted(set(repo_paths)))} + +## GPG keys +{_markdown_list(sorted(set(key_paths)))} + +## Managed files +{_markdown_list(_managed_file_lines(managed_files, include_reason=True))} + +## Excluded +{_markdown_list(_excluded_lines(excluded))} + +## Notes +{_markdown_list([str(n) for n in notes])} +""" + + +def _simple_managed_files_readme( + title: str, + description: str, + *, + include_reason: bool, +) -> Callable[..., str]: + def _builder( + *, + bundle_dir: str, + role: str, + snapshot: Dict[str, Any], + managed_files: List[Dict[str, Any]], + managed_dirs: List[Dict[str, Any]], + excluded: List[Dict[str, Any]], + notes: List[Any], + ) -> str: + return f"""# {title} + +{description} + +## Managed files +{_markdown_list(_managed_file_lines(managed_files, include_reason=include_reason))} + +## Excluded +{_markdown_list(_excluded_lines(excluded))} + +## Notes +{_markdown_list([str(n) for n in notes])} +""" + + return _builder + + +def _extra_paths_readme( + *, + bundle_dir: str, + role: str, + snapshot: Dict[str, Any], + managed_files: List[Dict[str, Any]], + managed_dirs: List[Dict[str, Any]], + excluded: List[Dict[str, Any]], + notes: List[Any], +) -> str: + include_pats = snapshot.get("include_patterns", []) or [] + exclude_pats = snapshot.get("exclude_patterns", []) or [] + return f"""# {role} + +User-requested extra file harvesting. + +## Include patterns +{_markdown_list([str(p) for p in include_pats])} + +## Exclude patterns +{_markdown_list([str(p) for p in exclude_pats])} + +## Managed directories +{_markdown_list([str(d.get('path') or '') for d in managed_dirs])} + +## Managed files +{_markdown_list(_managed_file_lines(managed_files, include_reason=False))} + +## Excluded +{_markdown_list(_excluded_lines(excluded))} + +## Notes +{_markdown_list([str(n) for n in notes])} +""" + + +def _write_managed_files_role( + *, + snapshot: Dict[str, Any], + default_role: str, + bundle_dir: str, + roles_root: str, + out_dir: str, + fqdn: Optional[str], + site_mode: bool, + jt_exe: Optional[str], + jt_enabled: bool, + notify_systemd: Optional[str], + handlers: str, + readme_builder: Callable[..., str], +) -> str: + """Render an Ansible role whose main purpose is managed files/dirs. + + This covers apt_config, dnf_config, etc_custom, usr_local_custom, and + extra_paths. Their harvested state shape is the same; only their README + and optional handler differ. + """ + + role = snapshot.get("role_name", default_role) + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + + var_prefix = role + managed_files = snapshot.get("managed_files", []) or [] + managed_dirs = snapshot.get("managed_dirs", []) or [] + excluded = snapshot.get("excluded", []) or [] + notes = snapshot.get("notes", []) or [] + + templated, jt_vars = _jinjify_managed_files( + bundle_dir, + role, + role_dir, + managed_files, + jt_exe=jt_exe, + jt_enabled=jt_enabled, + overwrite_templates=not site_mode, + ) + + if site_mode: + _copy_artifacts( + bundle_dir, + role, + _host_role_files_dir(out_dir, fqdn or "", role), + exclude_rels=templated, + ) + else: + _copy_artifacts( + bundle_dir, + role, + os.path.join(role_dir, "files"), + exclude_rels=templated, + ) + + files_var = _build_managed_files_var( + managed_files, + templated, + notify_other=None, + notify_systemd=notify_systemd, + ) + dirs_var = _build_managed_dirs_var(managed_dirs) + + jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} + vars_map: Dict[str, Any] = { + f"{var_prefix}_managed_files": files_var, + f"{var_prefix}_managed_dirs": dirs_var, + } + vars_map = _merge_mappings_overwrite(vars_map, jt_map) + + if site_mode: + _write_role_defaults( + role_dir, + {f"{var_prefix}_managed_files": [], f"{var_prefix}_managed_dirs": []}, + ) + _write_hostvars(out_dir, fqdn or "", role, vars_map) + else: + _write_role_defaults(role_dir, vars_map) + + tasks = "---\n" + _render_generic_files_tasks( + var_prefix, include_restart_notify=False + ) + with open(os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8") as f: + f.write(tasks.rstrip() + "\n") + + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(handlers.rstrip() + "\n") + + with open(os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8") as f: + f.write("---\ndependencies: []\n") + + readme = readme_builder( + bundle_dir=bundle_dir, + role=role, + snapshot=snapshot, + managed_files=managed_files, + managed_dirs=managed_dirs, + excluded=excluded, + notes=notes, + ) + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + return role + + +def _resolve_jinjaturtle_mode(jinjaturtle: str) -> Tuple[Optional[str], bool]: + jt_exe = find_jinjaturtle_cmd() + if jinjaturtle not in ("auto", "on", "off"): + raise ValueError("jinjaturtle must be one of: auto, on, off") + if jinjaturtle == "on": + if not jt_exe: + raise RuntimeError("jinjaturtle requested but not found on PATH") + return jt_exe, True + if jinjaturtle == "auto": + return jt_exe, jt_exe is not None + return jt_exe, False + + +def _prepare_ansible_context( + bundle_dir: str, + out_dir: str, + *, + fqdn: Optional[str], + jinjaturtle: str, +) -> AnsibleManifestContext: + site_mode = fqdn is not None and fqdn != "" + jt_exe, jt_enabled = _resolve_jinjaturtle_mode(jinjaturtle) + + os.makedirs(out_dir, exist_ok=True) + roles_root = os.path.join(out_dir, "roles") + os.makedirs(roles_root, exist_ok=True) + + return AnsibleManifestContext( + bundle_dir=bundle_dir, + out_dir=out_dir, + roles_root=roles_root, + fqdn=fqdn, + site_mode=site_mode, + jt_exe=jt_exe, + jt_enabled=jt_enabled, + ) + + +def _write_site_scaffold(ctx: AnsibleManifestContext) -> None: + if not ctx.site_mode: + return + os.makedirs(os.path.join(ctx.out_dir, "inventory"), exist_ok=True) + os.makedirs(os.path.join(ctx.out_dir, "inventory", "host_vars"), exist_ok=True) + os.makedirs(os.path.join(ctx.out_dir, "playbooks"), exist_ok=True) + _ensure_inventory_host( + os.path.join(ctx.out_dir, "inventory", "hosts.ini"), ctx.fqdn or "" + ) + _ensure_ansible_cfg(os.path.join(ctx.out_dir, "ansible.cfg")) + _ensure_requirements_yaml(os.path.join(ctx.out_dir, "requirements.yml")) + + +def _collect_ansible_roles( + roles: Dict[str, Any], + inventory_packages: Dict[str, Any], + *, + use_common_roles: bool, +) -> AnsibleRoleCollection: + services = roles.get("services", []) or [] + packages = roles.get("packages", []) or [] + common_role_groups: Dict[str, List[Dict[str, Any]]] = {} + + if use_common_roles: + for svc in services: + label = section_label_for_packages( + svc.get("packages", []) or [], inventory_packages + ) + common_role_groups.setdefault(label, []).append( + {"kind": "service", "snapshot": svc} + ) + for pr in packages: + label = package_section_label(pr, inventory_packages) + common_role_groups.setdefault(label, []).append( + {"kind": "package", "snapshot": pr} + ) + return AnsibleRoleCollection( + services=[], packages=[], common_role_groups=common_role_groups + ) + + return AnsibleRoleCollection( + services=services, + packages=packages, + common_role_groups=common_role_groups, + ) + + +def _render_managed_file_roles( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + roles: Dict[str, Any], +) -> None: + apt_config_snapshot = roles.get("apt_config", {}) + if apt_config_snapshot and apt_config_snapshot.get("managed_files"): + role = _write_managed_files_role( + snapshot=apt_config_snapshot, + default_role="apt_config", + bundle_dir=ctx.bundle_dir, + roles_root=ctx.roles_root, + out_dir=ctx.out_dir, + fqdn=ctx.fqdn, + site_mode=ctx.site_mode, + jt_exe=ctx.jt_exe, + jt_enabled=ctx.jt_enabled, + notify_systemd=None, + handlers="---\n", + readme_builder=_apt_config_readme, + ) + manifest_plan.add("apt_config", role) + + dnf_config_snapshot = roles.get("dnf_config", {}) + if dnf_config_snapshot and dnf_config_snapshot.get("managed_files"): + role = _write_managed_files_role( + snapshot=dnf_config_snapshot, + default_role="dnf_config", + bundle_dir=ctx.bundle_dir, + roles_root=ctx.roles_root, + out_dir=ctx.out_dir, + fqdn=ctx.fqdn, + site_mode=ctx.site_mode, + jt_exe=ctx.jt_exe, + jt_enabled=ctx.jt_enabled, + notify_systemd=None, + handlers="---\n", + readme_builder=_dnf_config_readme, + ) + manifest_plan.add("dnf_config", role) + + etc_custom_snapshot = roles.get("etc_custom", {}) + if etc_custom_snapshot and etc_custom_snapshot.get("managed_files"): + role = _write_managed_files_role( + snapshot=etc_custom_snapshot, + default_role="etc_custom", + bundle_dir=ctx.bundle_dir, + roles_root=ctx.roles_root, + out_dir=ctx.out_dir, + fqdn=ctx.fqdn, + site_mode=ctx.site_mode, + jt_exe=ctx.jt_exe, + jt_enabled=ctx.jt_enabled, + notify_systemd="Run systemd daemon-reload", + handlers="""--- +- name: Run systemd daemon-reload + ansible.builtin.systemd: + daemon_reload: true + no_log: "{{ enroll_hide_systemd_status | default(true) | bool }}" +""", + readme_builder=_simple_managed_files_readme( + "etc_custom", + "Unowned /etc config files not attributed to packages or services.", + include_reason=False, + ), + ) + manifest_plan.add("etc_custom", role) + + usr_local_custom_snapshot = roles.get("usr_local_custom", {}) + if usr_local_custom_snapshot and usr_local_custom_snapshot.get("managed_files"): + role = _write_managed_files_role( + snapshot=usr_local_custom_snapshot, + default_role="usr_local_custom", + bundle_dir=ctx.bundle_dir, + roles_root=ctx.roles_root, + out_dir=ctx.out_dir, + fqdn=ctx.fqdn, + site_mode=ctx.site_mode, + jt_exe=ctx.jt_exe, + jt_enabled=ctx.jt_enabled, + notify_systemd=None, + handlers="---\n", + readme_builder=_simple_managed_files_readme( + "usr_local_custom", + "Unowned /usr/local files (scripts in /usr/local/bin and config under /usr/local/etc).", + include_reason=False, + ), + ) + manifest_plan.add("usr_local_custom", role) + + extra_paths_snapshot = roles.get("extra_paths", {}) + if extra_paths_snapshot and ( + extra_paths_snapshot.get("managed_files") + or extra_paths_snapshot.get("managed_dirs") + ): + role = _write_managed_files_role( + snapshot=extra_paths_snapshot, + default_role="extra_paths", + bundle_dir=ctx.bundle_dir, + roles_root=ctx.roles_root, + out_dir=ctx.out_dir, + fqdn=ctx.fqdn, + site_mode=ctx.site_mode, + jt_exe=ctx.jt_exe, + jt_enabled=ctx.jt_enabled, + notify_systemd=None, + handlers="---\n", + readme_builder=_extra_paths_readme, + ) + manifest_plan.add("extra_paths", role) + + +def _render_sysctl_role( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + sysctl_snapshot: Dict[str, Any], +) -> None: + if not (sysctl_snapshot and (sysctl_snapshot.get("managed_files") or [])): + return + + role = sysctl_snapshot.get("role_name", "sysctl") + role_dir = os.path.join(ctx.roles_root, role) + _write_role_scaffold(role_dir) + + var_prefix = role + managed_files = sysctl_snapshot.get("managed_files", []) or [] + conf_src_rel = "" + for mf in managed_files: + if mf.get("path") == "/etc/sysctl.d/99-enroll.conf": + conf_src_rel = mf.get("src_rel") or "" + break + if not conf_src_rel and managed_files: + conf_src_rel = managed_files[0].get("src_rel") or "" + + parameters = sysctl_snapshot.get("parameters", {}) or {} + notes = sysctl_snapshot.get("notes", []) or [] + + if ctx.site_mode: + _copy_artifacts( + ctx.bundle_dir, + role, + _host_role_files_dir(ctx.out_dir, ctx.fqdn or "", role), + ) + else: + _copy_artifacts(ctx.bundle_dir, role, os.path.join(role_dir, "files")) + + vars_map: Dict[str, Any] = { + f"{var_prefix}_conf_src_rel": conf_src_rel, + f"{var_prefix}_apply": True, + f"{var_prefix}_ignore_apply_errors": True, + } + + if ctx.site_mode: + _write_role_defaults( + role_dir, + { + f"{var_prefix}_conf_src_rel": "", + f"{var_prefix}_apply": True, + f"{var_prefix}_ignore_apply_errors": True, + }, + ) + _write_hostvars(ctx.out_dir, ctx.fqdn or "", role, vars_map) + else: + _write_role_defaults(role_dir, vars_map) + + tasks = "---\n" + _render_sysctl_tasks(var_prefix) + with open(os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8") as f: + f.write(tasks.rstrip() + "\n") + + handlers_dir = os.path.join(role_dir, "handlers") + os.makedirs(handlers_dir, exist_ok=True) + with open(os.path.join(handlers_dir, "main.yml"), "w", encoding="utf-8") as f: + f.write(_render_sysctl_handlers(var_prefix)) + + with open(os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8") as f: + f.write("---\ndependencies: []\n") + + param_count = len(parameters) if isinstance(parameters, dict) else 0 + sample_params = [] + if isinstance(parameters, dict): + sample_params = sorted(parameters.keys())[:25] + + readme = f"""# {role} + +Generated from live writable sysctl state captured during harvest. + +This role deploys the captured values to `/etc/sysctl.d/99-enroll.conf`. The generated file intentionally contains writable, single-line sysctl values only; read-only, multiline, action-like, and host-identity keys are skipped to avoid creating a noisy or brittle boot-time configuration. + +## Captured parameters + +Captured parameter count: {param_count} + +{os.linesep.join("- " + x for x in sample_params) or "- (none)"} + +{"- ..." if param_count > len(sample_params) else ""} + +## Notes +{os.linesep.join("- " + n for n in notes) or "- (none)"} + +## Safety notes +- `sysctl_apply` defaults to `true` and applies `/etc/sysctl.d/99-enroll.conf` when the file changes. +- `sysctl_ignore_apply_errors` defaults to `true`, because sysctl availability and writability can vary across kernels, containers, and hardware. +- Review this role before applying it broadly across unlike hosts. +""" + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("sysctl", role) + + +def _render_firewall_runtime_role( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + firewall_runtime_snapshot: Dict[str, Any], +) -> None: + if not ( + firewall_runtime_snapshot + and ( + firewall_runtime_snapshot.get("ipset_save") + or firewall_runtime_snapshot.get("iptables_v4_save") + or firewall_runtime_snapshot.get("iptables_v6_save") + ) + ): + return + + role = firewall_runtime_snapshot.get("role_name", "firewall_runtime") + role_dir = os.path.join(ctx.roles_root, role) + _write_role_scaffold(role_dir) + + var_prefix = role + packages = firewall_runtime_snapshot.get("packages", []) or [] + ipset_save = firewall_runtime_snapshot.get("ipset_save") or "" + ipset_sets = firewall_runtime_snapshot.get("ipset_sets", []) or [] + iptables_v4_save = firewall_runtime_snapshot.get("iptables_v4_save") or "" + iptables_v6_save = firewall_runtime_snapshot.get("iptables_v6_save") or "" + notes = firewall_runtime_snapshot.get("notes", []) or [] + + if ctx.site_mode: + _copy_artifacts( + ctx.bundle_dir, + role, + _host_role_files_dir(ctx.out_dir, ctx.fqdn or "", role), + ) + else: + _copy_artifacts(ctx.bundle_dir, role, os.path.join(role_dir, "files")) + + vars_map: Dict[str, Any] = { + f"{var_prefix}_packages": packages, + f"{var_prefix}_ipset_save": ipset_save, + f"{var_prefix}_ipset_sets": ipset_sets, + f"{var_prefix}_iptables_v4_save": iptables_v4_save, + f"{var_prefix}_iptables_v6_save": iptables_v6_save, + f"{var_prefix}_sync_ipsets_exact": True, + f"{var_prefix}_restore_iptables": True, + } + + if ctx.site_mode: + _write_role_defaults( + role_dir, + { + f"{var_prefix}_packages": [], + f"{var_prefix}_ipset_save": "", + f"{var_prefix}_ipset_sets": [], + f"{var_prefix}_iptables_v4_save": "", + f"{var_prefix}_iptables_v6_save": "", + f"{var_prefix}_sync_ipsets_exact": True, + f"{var_prefix}_restore_iptables": True, + }, + ) + _write_hostvars(ctx.out_dir, ctx.fqdn or "", role, vars_map) + else: + _write_role_defaults(role_dir, vars_map) + + tasks = ( + "---\n" + + _render_install_packages_tasks(role, var_prefix) + + _render_firewall_runtime_tasks(var_prefix) + ) + with open(os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8") as f: + f.write(tasks.rstrip() + "\n") + + with open(os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8") as f: + f.write("---\ndependencies: []\n") + + readme = f"""# {role} + +Generated from live firewall runtime state captured during harvest. + +This role restores live ipset and iptables state only for firewall families where Enroll did not find corresponding persistent configuration on the source host. Static firewall configuration files, such as `/etc/iptables/rules.v4`, `/etc/iptables/rules.v6`, UFW, nftables, firewalld, or `/etc/ipset*`, are harvested separately as managed files and treated as authoritative for their respective family. + +## Captured snapshots +- ipset: {ipset_save or "(none)"} +- iptables IPv4: {iptables_v4_save or "(none)"} +- iptables IPv6: {iptables_v6_save or "(none)"} + +## Captured ipsets +{os.linesep.join("- " + x for x in ipset_sets) or "- (none)"} + +## Notes +{os.linesep.join("- " + n for n in notes) or "- (none)"} + +## Safety notes +- `firewall_runtime_sync_ipsets_exact` defaults to `true`; it flushes captured set members before replaying the saved members so stale entries are removed. This applies only when no persistent ipset config was found. +- `firewall_runtime_restore_iptables` defaults to `true`; `iptables-restore`/`ip6tables-restore` replace only captured families. A family is captured only when no corresponding persistent iptables config was found. +""" + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("firewall_runtime", role) + + +def _write_manifest_playbook(ctx: AnsibleManifestContext, roles: List[str]) -> None: + if ctx.site_mode: + _write_playbook_host( + os.path.join(ctx.out_dir, "playbooks", f"{ctx.fqdn}.yml"), + ctx.fqdn or "", + roles, + ) + else: + _write_playbook_all(os.path.join(ctx.out_dir, "playbook.yml"), roles) + + +def _render_users_role( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + users_snapshot: Dict[str, Any], +) -> None: + bundle_dir = ctx.bundle_dir + out_dir = ctx.out_dir + roles_root = ctx.roles_root + fqdn = ctx.fqdn + site_mode = ctx.site_mode + + # ------------------------- + # Users role (non-system users) + # ------------------------- + if users_snapshot: + role = users_snapshot.get("role_name", "users") + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + + # Users role includes harvested SSH-related files; in site mode keep them + # host-specific to avoid cross-host clobber. + if site_mode: + _copy_artifacts( + bundle_dir, role, _host_role_files_dir(out_dir, fqdn or "", role) + ) + else: + _copy_artifacts(bundle_dir, role, os.path.join(role_dir, "files")) + + users = users_snapshot.get("users", []) + managed_files = users_snapshot.get("managed_files", []) + excluded = users_snapshot.get("excluded", []) + notes = users_snapshot.get("notes", []) + + # Build groups list and a simplified user dict list suitable for loops + group_names: List[str] = [] + group_set = set() + users_data: List[Dict[str, Any]] = [] + for u in users: + name = u.get("name") + if not name: + continue + pg = u.get("primary_group") or name + home = u.get("home") or f"/home/{name}" + sshdir = home.rstrip("/") + "/.ssh" + supp = u.get("supplementary_groups") or [] + if pg: + group_set.add(pg) + for g in supp: + if g: + group_set.add(g) + + users_data.append( + { + "name": name, + "uid": u.get("uid"), + "primary_group": pg, + "home": home, + "ssh_dir": sshdir, + "shell": u.get("shell"), + "gecos": u.get("gecos"), + "supplementary_groups": sorted(set(supp)), + } + ) + + group_names = sorted(group_set) + + # User-managed files (authorized_keys plus dangerous-mode shell dotfiles). + # Keep the variable name for compatibility with existing generated data. + ssh_files: List[Dict[str, Any]] = [] + for mf in managed_files: + dest = mf.get("path") or "" + src_rel = mf.get("src_rel") or "" + if not dest or not src_rel: + continue + + owner = "root" + group = "root" + for u in users_data: + home_prefix = (u.get("home") or "").rstrip("/") + "/" + if home_prefix and dest.startswith(home_prefix): + owner = str(u.get("name") or "root") + group = str(u.get("primary_group") or owner) + break + + # Prefer the harvested file mode so we preserve any deliberate + # permissions (e.g. 0600 for certain dotfiles). For authorized_keys, + # enforce 0600 regardless. + mode = mf.get("mode") or "0644" + if mf.get("reason") == "authorized_keys": + mode = "0600" + ssh_files.append( + { + "dest": dest, + "src_rel": src_rel, + "owner": owner, + "group": group, + "mode": mode, + } + ) + + # Only create .ssh directories for users that actually have harvested + # files under .ssh. This mirrors Puppet's behaviour and avoids creating + # empty SSH directories merely because a user account exists. + ssh_dirs_by_dest: Dict[str, Dict[str, Any]] = {} + for item in ssh_files: + dest = str(item.get("dest") or "") + if not dest: + continue + for user in users_data: + ssh_dir = str(user.get("ssh_dir") or "").rstrip("/") + if not ssh_dir or not dest.startswith(ssh_dir + "/"): + continue + ssh_dirs_by_dest.setdefault( + ssh_dir, + { + "dest": ssh_dir, + "owner": str(user.get("name") or item.get("owner") or "root"), + "group": str( + user.get("primary_group") or item.get("group") or "root" + ), + "mode": "0700", + }, + ) + break + ssh_dirs = sorted( + ssh_dirs_by_dest.values(), key=lambda item: str(item.get("dest") or "") + ) + + # Build Flatpak and Snap lists. Flatpak can be installed system-wide or + # per-user. Snap packages are system-wide; per-user ~/snap/* directories + # are runtime/user data and are not treated as install sources. + users_flatpaks: List[Dict[str, Any]] = [] + user_flatpak_map = users_snapshot.get("user_flatpaks", {}) or {} + home_by_user = { + str(u.get("name")): str(u.get("home") or "") for u in users_data + } + for uname, flatpaks in user_flatpak_map.items(): + for fp in flatpaks or []: + users_flatpaks.append( + _normalise_flatpak_item( + fp, + method="user", + user=str(uname), + home=home_by_user.get(str(uname)) or None, + ) + ) + + flatpak_remotes = [ + _normalise_flatpak_remote(r) + for r in (users_snapshot.get("user_flatpak_remotes", []) or []) + ] + users_needs_community = bool(flatpak_remotes or users_flatpaks) + if users_needs_community: + _ensure_requirements_yaml(os.path.join(out_dir, "requirements.yml")) + + # Variables are host-specific in site mode; in non-site mode they live in role defaults. + if site_mode: + _write_role_defaults( + role_dir, + { + "users_groups": [], + "users_users": [], + "users_ssh_dirs": [], + "users_ssh_files": [], + "users_flatpaks": [], + "users_flatpak_remotes": [], + }, + ) + _write_hostvars( + out_dir, + fqdn or "", + role, + { + "users_groups": group_names, + "users_users": users_data, + "users_ssh_dirs": ssh_dirs, + "users_ssh_files": ssh_files, + "users_flatpaks": users_flatpaks, + "users_flatpak_remotes": flatpak_remotes, + }, + ) + else: + _write_role_defaults( + role_dir, + { + "users_groups": group_names, + "users_users": users_data, + "users_ssh_dirs": ssh_dirs, + "users_ssh_files": ssh_files, + "users_flatpaks": users_flatpaks, + "users_flatpak_remotes": flatpak_remotes, + }, + ) + + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: + if users_needs_community: + f.write( + "---\n" + "dependencies: []\n" + "collections:\n" + " - community.general\n" + ) + else: + f.write("---\ndependencies: []\n") + + # tasks (data-driven) + users_tasks = """--- + +- name: Ensure groups exist + ansible.builtin.group: + name: "{{ item }}" + state: present + loop: "{{ users_groups | default([]) }}" + +- name: Ensure users exist + ansible.builtin.user: + name: "{{ item.name }}" + uid: "{{ item.uid | default(omit) }}" + group: "{{ item.primary_group }}" + home: "{{ item.home }}" + create_home: true + shell: "{{ item.shell | default(omit) }}" + comment: "{{ item.gecos | default(omit) }}" + state: present + loop: "{{ users_users | default([]) }}" + +- name: Ensure users supplementary groups + ansible.builtin.user: + name: "{{ item.name }}" + groups: "{{ item.supplementary_groups | default([]) | join(',') }}" + append: true + loop: "{{ users_users | default([]) }}" + when: (item.supplementary_groups | default([])) | length > 0 + +- name: Ensure .ssh directories exist for managed SSH files + ansible.builtin.file: + path: "{{ item.dest }}" + state: directory + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: "{{ item.mode }}" + loop: "{{ users_ssh_dirs | default([]) }}" + +- name: Deploy user-managed files + vars: + _enroll_ff: + files: + - "{{ inventory_dir }}/host_vars/{{ inventory_hostname }}/{{ role_name }}/.files/{{ item.src_rel }}" + - "{{ role_path }}/files/{{ item.src_rel }}" + ansible.builtin.copy: + src: "{{ lookup('ansible.builtin.first_found', _enroll_ff) }}" + dest: "{{ item.dest }}" + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: "{{ item.mode }}" + loop: "{{ users_ssh_files | default([]) }}" +""" + + if flatpak_remotes or users_flatpaks: + users_tasks += """ +- name: Ensure user Flatpak remotes exist + ansible.builtin.command: + argv: + - flatpak + - remote-add + - --user + - --if-not-exists + - "{{ item.name }}" + - "{{ item.url }}" + loop: "{{ users_flatpak_remotes | default([]) | selectattr('method', 'equalto', 'user') | list }}" + when: + - item.name is defined + - item.url is defined + - item.url | length > 0 + - item.user is defined + become: true + become_user: "{{ item.user }}" + environment: + HOME: "{{ item.home | default('/home/' ~ item.user, true) }}" + XDG_DATA_HOME: "{{ (item.home | default('/home/' ~ item.user, true)) ~ '/.local/share' }}" + changed_when: false + +- name: Install user Flatpaks + community.general.flatpak: + name: + - "{{ item.name }}" + state: present + method: user + remote: "{{ item.remote | default(omit) }}" + from_url: "{{ item.from_url | default(omit) }}" + loop: "{{ users_flatpaks | default([]) }}" + when: + - item.name is defined + - item.name | length > 0 + - item.user is defined + become: true + become_user: "{{ item.user }}" + environment: + HOME: "{{ item.home | default('/home/' ~ item.user, true) }}" + XDG_DATA_HOME: "{{ (item.home | default('/home/' ~ item.user, true)) ~ '/.local/share' }}" +""" + + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(users_tasks) + + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write("---\n") + + def _fmt_app_list(items: List[Dict[str, Any]]) -> str: + lines = [] + for item in items: + name = item.get("name") + if not name: + continue + detail_parts = [] + for key in ("remote", "channel", "revision", "branch", "arch"): + value = item.get(key) + if value not in (None, "", []): + detail_parts.append(f"{key}={value}") + for key in ("classic", "devmode", "dangerous"): + if item.get(key): + detail_parts.append(key) + details = f" ({', '.join(detail_parts)})" if detail_parts else "" + lines.append(f"- {name}{details}") + return "\n".join(lines) or "- (none)" + + def _fmt_user_flatpaks(items: List[Dict[str, Any]]) -> str: + lines = [] + for item in items: + name = item.get("name") + user = item.get("user") + if not name or not user: + continue + detail_parts = [] + for key in ("remote", "branch", "arch"): + value = item.get(key) + if value not in (None, "", []): + detail_parts.append(f"{key}={value}") + details = f" ({', '.join(detail_parts)})" if detail_parts else "" + lines.append(f"- {user}: {name}{details}") + return "\n".join(lines) or "- (none)" + + def _fmt_remotes(items: List[Dict[str, Any]]) -> str: + lines = [] + for item in items: + name = item.get("name") + url = item.get("url") + method = item.get("method") or "system" + user = item.get("user") + if not name or not url: + continue + owner = f"user={user}" if user else "system" + lines.append(f"- {name} ({method}, {owner}): {url}") + return "\n".join(lines) or "- (none)" + + readme = ( + """# users + +Generated non-system user accounts, SSH public material, and per-user Flatpak +applications/remotes. + +**Note:** User Flatpak tasks require the `community.general` Ansible collection. +Install it with: `ansible-galaxy collection install -r requirements.yml`. + +Flatpak `remote` is harvested from the installed deployment where detectable. +The original `.flatpakref` URL is generally not preserved by Flatpak after +installation, so `from_url` is only emitted if a future/hand-edited state file +contains it. + + +## Users +""" + + ( + "\n".join([f"- {u.get('name')} (uid {u.get('uid')})" for u in users]) + or "- (none)" + ) + + """\n +## Included SSH files +""" + + ( + "\n".join( + [f"- {mf.get('path')} ({mf.get('reason')})" for mf in managed_files] + ) + or "- (none)" + ) + + """\n +## Flatpak remotes +""" + + _fmt_remotes(flatpak_remotes) + + """\n +## User Flatpaks +""" + + _fmt_user_flatpaks(users_flatpaks) + + """\n +## Excluded +""" + + ( + "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) + or "- (none)" + ) + + """\n +## Notes +""" + + ("\n".join([f"- {n}" for n in notes]) or "- (none)") + + """\n""" + ) + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("users", role) + + +def _render_flatpak_role( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + flatpak_snapshot: Dict[str, Any], +) -> None: + out_dir = ctx.out_dir + roles_root = ctx.roles_root + fqdn = ctx.fqdn + site_mode = ctx.site_mode + + # ------------------------- + # Flatpak role (system-wide Flatpak remotes and applications) + # ------------------------- + raw_flatpak_apps = flatpak_snapshot.get("system_flatpaks", []) or [] + raw_flatpak_remotes = flatpak_snapshot.get("remotes", []) or [] + + if flatpak_snapshot: + role = flatpak_snapshot.get("role_name", "flatpak") + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + _ensure_requirements_yaml(os.path.join(out_dir, "requirements.yml")) + + flatpak_system_flatpaks = [ + _normalise_flatpak_item(fp, method="system") for fp in raw_flatpak_apps + ] + flatpak_remotes = [_normalise_flatpak_remote(r) for r in raw_flatpak_remotes] + + vars_map = { + "flatpak_system_flatpaks": flatpak_system_flatpaks, + "flatpak_remotes": flatpak_remotes, + } + if site_mode: + _write_role_defaults( + role_dir, + {"flatpak_system_flatpaks": [], "flatpak_remotes": []}, + ) + _write_hostvars(out_dir, fqdn or "", role, vars_map) + else: + _write_role_defaults(role_dir, vars_map) + + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write( + "---\n" "dependencies: []\n" "collections:\n" " - community.general\n" + ) + + tasks = """--- + +- name: Ensure system Flatpak remotes exist + ansible.builtin.command: + argv: + - flatpak + - remote-add + - --system + - --if-not-exists + - "{{ item.name }}" + - "{{ item.url }}" + loop: "{{ flatpak_remotes | default([]) }}" + when: + - item.name is defined + - item.url is defined + - item.url | length > 0 + become: true + changed_when: false + +- name: Install system-wide Flatpaks + community.general.flatpak: + name: + - "{{ item.name }}" + state: present + method: system + remote: "{{ item.remote | default(omit) }}" + from_url: "{{ item.from_url | default(omit) }}" + loop: "{{ flatpak_system_flatpaks | default([]) }}" + when: + - item.name is defined + - item.name | length > 0 + become: true +""" + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(tasks) + + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write("---\n") + + def _fmt_flatpak_apps(items: List[Dict[str, Any]]) -> str: + lines = [] + for item in items: + name = item.get("name") + if not name: + continue + detail_parts = [] + for key in ("remote", "branch", "arch"): + value = item.get(key) + if value not in (None, "", []): + detail_parts.append(f"{key}={value}") + details = f" ({', '.join(detail_parts)})" if detail_parts else "" + lines.append(f"- {name}{details}") + return "\n".join(lines) or "- (none)" + + def _fmt_flatpak_remotes(items: List[Dict[str, Any]]) -> str: + lines = [] + for item in items: + name = item.get("name") + url = item.get("url") + if not name or not url: + continue + lines.append(f"- {name}: {url}") + return "\n".join(lines) or "- (none)" + + notes = flatpak_snapshot.get("notes", []) or [] + readme = ( + """# flatpak + +Generated system-wide Flatpak remotes and applications. + +**Note:** This role requires the `community.general` Ansible collection. +Install it with: `ansible-galaxy collection install -r requirements.yml`. + +Flatpak `remote` is harvested from the installed deployment where detectable. +The original `.flatpakref` URL is generally not preserved by Flatpak after +installation, so `from_url` is only emitted if a future/hand-edited state file +contains it. + +## System Flatpak remotes +""" + + _fmt_flatpak_remotes(flatpak_remotes) + + """\n +## System-wide Flatpaks +""" + + _fmt_flatpak_apps(flatpak_system_flatpaks) + + """\n +## Notes +""" + + ("\n".join([f"- {n}" for n in notes]) or "- (none)") + + """\n""" + ) + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("flatpak", role) + + +def _render_snap_role( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + snap_snapshot: Dict[str, Any], +) -> None: + out_dir = ctx.out_dir + roles_root = ctx.roles_root + fqdn = ctx.fqdn + site_mode = ctx.site_mode + + # ------------------------- + # Snap role (system-wide snap packages) + # ------------------------- + raw_system_snaps = snap_snapshot.get("system_snaps", []) or [] + + if raw_system_snaps: + role = snap_snapshot.get("role_name", "snap") if snap_snapshot else "snap" + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + _ensure_requirements_yaml(os.path.join(out_dir, "requirements.yml")) + + snap_system_snaps = [_normalise_snap_item(s) for s in raw_system_snaps] + + vars_map = {"snap_system_snaps": snap_system_snaps} + if site_mode: + _write_role_defaults(role_dir, {"snap_system_snaps": []}) + _write_hostvars(out_dir, fqdn or "", role, vars_map) + else: + _write_role_defaults(role_dir, vars_map) + + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write( + "---\n" "dependencies: []\n" "collections:\n" " - community.general\n" + ) + + tasks = """--- + +- name: Install system-wide snaps with full detected attributes + community.general.snap: + name: + - "{{ item.name }}" + state: present + channel: "{{ item.channel | default(omit) if not (item.install_revision | default(false)) else omit }}" + revision: "{{ item.revision | default(omit) if (item.install_revision | default(false)) else omit }}" + classic: "{{ item.classic | default(false) }}" + devmode: "{{ item.devmode | default(false) }}" + dangerous: "{{ item.dangerous | default(false) }}" + loop: "{{ snap_system_snaps | default([]) }}" + when: + - item.name is defined + - item.name | length > 0 + become: true + register: _enroll_snap_full_results + ignore_errors: true + +- name: Install system-wide snaps with compatibility options + community.general.snap: + name: + - "{{ item.item.name }}" + state: present + channel: "{{ item.item.channel | default(omit) if not (item.item.install_revision | default(false)) else omit }}" + classic: "{{ item.item.classic | default(false) }}" + loop: "{{ (_enroll_snap_full_results | default({})).results | default([]) }}" + when: + - item.failed | default(false) + - item.item.name is defined + - item.item.name | length > 0 + become: true + register: _enroll_snap_compat_results + ignore_errors: true + +- name: Install system-wide snaps with minimal options + community.general.snap: + name: + - "{{ item.item.item.name }}" + state: present + loop: "{{ (_enroll_snap_compat_results | default({})).results | default([]) }}" + when: + - item.failed | default(false) + - item.item.item.name is defined + - item.item.item.name | length > 0 + become: true + ignore_errors: true +""" + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(tasks) + + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write("---\n") + + def _fmt_snap_apps(items: List[Dict[str, Any]]) -> str: + lines = [] + for item in items: + name = item.get("name") + if not name: + continue + detail_parts = [] + for key in ("channel", "revision"): + value = item.get(key) + if value not in (None, "", []): + detail_parts.append(f"{key}={value}") + for key in ("classic", "devmode", "dangerous"): + if item.get(key): + detail_parts.append(key) + details = f" ({', '.join(detail_parts)})" if detail_parts else "" + lines.append(f"- {name}{details}") + return "\n".join(lines) or "- (none)" + + notes = snap_snapshot.get("notes", []) or [] + readme = ( + """# snap + +Generated system-wide snap packages. + +**Note:** This role requires the `community.general` Ansible collection. +Install it with: `ansible-galaxy collection install -r requirements.yml`. + +The first install task uses all harvested attributes. If the installed +`community.general.snap` module is too old for some parameters, the generated +role falls back to reduced then minimal install tasks on a best-effort basis. + +## System-wide snaps +""" + + _fmt_snap_apps(snap_system_snaps) + + """\n +## Notes +""" + + ("\n".join([f"- {n}" for n in notes]) or "- (none)") + + """\n""" + ) + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("snap", role) + + +def _render_service_roles( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + services_to_manifest: List[Dict[str, Any]], +) -> None: + bundle_dir = ctx.bundle_dir + out_dir = ctx.out_dir + roles_root = ctx.roles_root + fqdn = ctx.fqdn + site_mode = ctx.site_mode + jt_exe = ctx.jt_exe + jt_enabled = ctx.jt_enabled + + # ------------------------- + # Service roles + # ------------------------- + for svc in services_to_manifest: + source_role = svc["role_name"] + role = avoid_reserved_role_name(source_role, prefix="service") + unit = svc["unit"] + pkgs = svc.get("packages", []) or [] + managed_files = svc.get("managed_files", []) or [] + managed_dirs = svc.get("managed_dirs", []) or [] + managed_links = svc.get("managed_links", []) or [] + + ansible_role = AnsibleRole(role) + ansible_role.add_service_snapshot(svc) + + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + + var_prefix = role + + unit_state = ansible_role.services.get(unit, {}) + enabled_at_harvest = bool(unit_state.get("enabled")) + desired_state = str(unit_state.get("state") or "stopped") + + templated, jt_vars = _jinjify_managed_files( + bundle_dir, + source_role, + role_dir, + managed_files, + jt_exe=jt_exe, + jt_enabled=jt_enabled, + overwrite_templates=not site_mode, + ) + + # Copy only the non-templated artifacts. + if site_mode: + _copy_artifacts( + bundle_dir, + source_role, + _host_role_files_dir(out_dir, fqdn or "", role), + exclude_rels=templated, + ) + else: + _copy_artifacts( + bundle_dir, + source_role, + os.path.join(role_dir, "files"), + exclude_rels=templated, + ) + + files_var = _build_managed_files_var( + managed_files, + templated, + notify_other="Restart service", + notify_systemd="Run systemd daemon-reload", + ) + + links_var = _build_managed_links_var(managed_links) + + dirs_var = _build_managed_dirs_var(managed_dirs) + + jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} + base_vars: Dict[str, Any] = { + f"{var_prefix}_unit_name": unit, + f"{var_prefix}_packages": pkgs, + f"{var_prefix}_managed_files": files_var, + f"{var_prefix}_managed_dirs": dirs_var, + f"{var_prefix}_managed_links": links_var, + f"{var_prefix}_manage_unit": True, + f"{var_prefix}_systemd_enabled": bool(enabled_at_harvest), + f"{var_prefix}_systemd_state": desired_state, + } + base_vars = _merge_mappings_overwrite(base_vars, jt_map) + + if site_mode: + # Role defaults are host-agnostic/safe; all harvested state is in host_vars. + _write_role_defaults( + role_dir, + { + f"{var_prefix}_unit_name": unit, + f"{var_prefix}_packages": [], + f"{var_prefix}_managed_files": [], + f"{var_prefix}_managed_dirs": [], + f"{var_prefix}_managed_links": [], + f"{var_prefix}_manage_unit": False, + f"{var_prefix}_systemd_enabled": False, + f"{var_prefix}_systemd_state": "stopped", + }, + ) + _write_hostvars(out_dir, fqdn or "", role, base_vars) + else: + _write_role_defaults(role_dir, base_vars) + + handlers = f"""--- +- name: Run systemd daemon-reload + ansible.builtin.systemd: + daemon_reload: true + no_log: "{{ enroll_hide_systemd_status | default(true) | bool }}" + +- name: Restart service + ansible.builtin.service: + name: "{{{{ {var_prefix}_unit_name }}}}" + state: restarted + when: + - {var_prefix}_manage_unit | default(false) + - ({var_prefix}_systemd_state | default('stopped')) == 'started' +""" + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(handlers) + + task_parts: List[str] = [] + task_parts.append("---\n" + _render_install_packages_tasks(role, var_prefix)) + + task_parts.append( + _render_generic_files_tasks(var_prefix, include_restart_notify=True) + ) + + task_parts.append( + f"""- name: Probe whether systemd unit exists and is manageable + ansible.builtin.systemd: + name: "{{{{ {var_prefix}_unit_name }}}}" + no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" + check_mode: true + register: _unit_probe + failed_when: false + changed_when: false + when: {var_prefix}_manage_unit | default(false) + +- name: Ensure unit enablement matches harvest + ansible.builtin.systemd: + name: "{{{{ {var_prefix}_unit_name }}}}" + enabled: "{{{{ {var_prefix}_systemd_enabled | bool }}}}" + no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" + when: + - {var_prefix}_manage_unit | default(false) + - _unit_probe is succeeded + +- name: Ensure unit running state matches harvest + ansible.builtin.systemd: + name: "{{{{ {var_prefix}_unit_name }}}}" + state: "{{{{ {var_prefix}_systemd_state }}}}" + no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" + when: + - {var_prefix}_manage_unit | default(false) + - _unit_probe is succeeded +""" + ) + + tasks = "\n".join(task_parts).rstrip() + "\n" + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(tasks) + + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write("---\ndependencies: []\n") + + excluded = svc.get("excluded", []) + notes = svc.get("notes", []) + readme = f"""# {role} + +Generated from `{unit}`. + +## Packages +{os.linesep.join("- " + p for p in pkgs) or "- (none detected)"} + +## Managed files +{os.linesep.join("- " + mf["path"] + " (" + mf["reason"] + ")" for mf in managed_files) or "- (none)"} + +## Managed symlinks +{os.linesep.join("- " + ml["path"] + " -> " + ml["target"] + " (" + ml.get("reason", "") + ")" for ml in managed_links) or "- (none)"} + +## Excluded (possible secrets / unsafe) +{os.linesep.join("- " + e["path"] + " (" + e["reason"] + ")" for e in excluded) or "- (none)"} + +## Notes +{os.linesep.join("- " + n for n in notes) or "- (none)"} +""" + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("service", role) + + +def _render_common_ansible_roles( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + common_role_groups: Dict[str, List[Dict[str, Any]]], + package_roles: List[Dict[str, Any]], +) -> List[str]: + bundle_dir = ctx.bundle_dir + roles_root = ctx.roles_root + jt_exe = ctx.jt_exe + jt_enabled = ctx.jt_enabled + + common_tail_roles: List[str] = [] + + # ------------------------- + # Common package section/group roles + # + # Outside --fqdn/site mode, package and systemd-unit roles are grouped by + # Debian Section or RPM Group by default. Managed config and unit state can + # live in those section roles too; --no-common-roles preserves the historic + # one-role-per-package/unit output, and --fqdn implies that mode because + # grouped role contents would be unsafe across multiple harvested hosts. + # ------------------------- + # ------------------------- + # Manually installed package roles + # ------------------------- + occupied_roles: Set[str] = set( + manifest_plan.roles("apt_config") + + manifest_plan.roles("dnf_config") + + manifest_plan.roles("users") + + manifest_plan.roles("flatpak") + + manifest_plan.roles("snap") + + manifest_plan.roles("service") + + manifest_plan.roles("firewall_runtime") + + manifest_plan.roles("sysctl") + + manifest_plan.roles("etc_custom") + + manifest_plan.roles("usr_local_custom") + + manifest_plan.roles("extra_paths") + ) + for pr in package_roles: + occupied_roles.add( + avoid_reserved_role_name(str(pr.get("role_name") or ""), prefix="package") + ) + + for section_label, entries in sorted(common_role_groups.items()): + role = _section_role_name(section_label, occupied_roles) + ansible_role = AnsibleRole( + role, + var_prefix=role, + section_label=section_label, + grouped=True, + ) + for entry in entries: + kind = entry.get("kind") or "package" + snap = entry.get("snapshot") or {} + if kind == "service": + ansible_role.add_service_snapshot(snap) + else: + ansible_role.add_package_snapshot(snap) + + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + + var_prefix = ansible_role.var_prefix + files_var: List[Dict[str, Any]] = [] + dirs_var: List[Dict[str, Any]] = [] + links_var: List[Dict[str, Any]] = [] + jt_combined: Dict[str, Any] = {} + + seen_files: Set[tuple] = set() + seen_dirs: Set[tuple] = set() + seen_links: Set[tuple] = set() + + for entry in ansible_role.entries: + kind = entry.get("kind") or "package" + snap = entry.get("snapshot") or {} + source_role = str(snap.get("role_name") or "") + managed_files = snap.get("managed_files", []) or [] + managed_dirs = snap.get("managed_dirs", []) or [] + managed_links = snap.get("managed_links", []) or [] + + templated: Set[str] = set() + jt_vars = "" + if managed_files and source_role: + templated, jt_vars = _jinjify_managed_files( + bundle_dir, + source_role, + role_dir, + managed_files, + jt_exe=jt_exe, + jt_enabled=jt_enabled, + overwrite_templates=True, + ) + + _copy_artifacts( + bundle_dir, + source_role, + os.path.join(role_dir, "files"), + exclude_rels=templated, + ) + + notify_other = "Restart managed services" if kind == "service" else None + for item in _build_managed_files_var( + managed_files, + templated, + notify_other=notify_other, + notify_systemd="Run systemd daemon-reload", + ): + key = (item.get("dest"), item.get("src_rel"), item.get("kind")) + if key not in seen_files: + seen_files.add(key) + files_var.append(item) + + for item in _build_managed_dirs_var(managed_dirs): + key = ( + item.get("dest"), + item.get("owner"), + item.get("group"), + item.get("mode"), + ) + if key not in seen_dirs: + seen_dirs.add(key) + dirs_var.append(item) + + for item in _build_managed_links_var(managed_links): + key = (item.get("dest"), item.get("src")) + if key not in seen_links: + seen_links.add(key) + links_var.append(item) + + jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} + jt_combined = _merge_mappings_overwrite(jt_combined, jt_map) + + packages = ansible_role.sorted_packages + files_var = sorted(files_var, key=lambda x: str(x.get("dest") or "")) + dirs_var = sorted(dirs_var, key=lambda x: str(x.get("dest") or "")) + links_var = sorted(links_var, key=lambda x: str(x.get("dest") or "")) + systemd_units = ansible_role.systemd_units_var + + base_vars: Dict[str, Any] = { + f"{var_prefix}_packages": packages, + f"{var_prefix}_managed_files": files_var, + f"{var_prefix}_managed_dirs": dirs_var, + f"{var_prefix}_managed_links": links_var, + f"{var_prefix}_systemd_units": systemd_units, + } + base_vars = _merge_mappings_overwrite(base_vars, jt_combined) + + _write_role_defaults(role_dir, base_vars) + + if {"cron", "logrotate"}.intersection(ansible_role.packages): + common_tail_roles.append(role) + + handlers = ( + """--- +- name: Run systemd daemon-reload + ansible.builtin.systemd: + daemon_reload: true + no_log: "{{ enroll_hide_systemd_status | default(true) | bool }}" + +- name: Restart managed services + ansible.builtin.service: + name: "{{ item.name }}" + state: restarted + loop: "{{ """ + + f"{var_prefix}_systemd_units" + + """ | default([]) }}" + when: + - item.manage | default(false) + - (item.state | default('stopped')) == 'started' +""" + ) + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(handlers) + + task_parts: List[str] = [] + task_parts.append("---\n" + _render_install_packages_tasks(role, var_prefix)) + task_parts.append( + _render_generic_files_tasks(var_prefix, include_restart_notify=True) + ) + task_parts.append(_render_grouped_systemd_tasks(var_prefix)) + + tasks = "\n".join(task_parts).rstrip() + "\n" + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(tasks) + + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write("---\ndependencies: []\n") + + readme = f"""# {role} + +Common role for package section/group `{section_label}`. + +## Origin roles +{os.linesep.join("- " + line for line in sorted(ansible_role.origin_lines)) or "- (none)"} + +## Packages +{os.linesep.join("- " + p for p in packages) or "- (none)"} + +## Managed files +{os.linesep.join("- " + mf["dest"] for mf in files_var) or "- (none)"} + +## Managed symlinks +{os.linesep.join("- " + ml["dest"] + " -> " + ml["src"] for ml in links_var) or "- (none)"} + +## Systemd units +{os.linesep.join("- " + u["name"] + " (enabled=" + str(u["enabled"]).lower() + ", state=" + u["state"] + ")" for u in systemd_units) or "- (none)"} + +## Excluded (possible secrets / unsafe) +{os.linesep.join("- " + e.get("path", "") + " (" + e.get("reason", "") + ")" for e in ansible_role.excluded) or "- (none)"} + +## Notes +{os.linesep.join("- " + n for n in ansible_role.notes) or "- (none)"} +""" + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("package", role) + + return common_tail_roles + + +def _render_package_roles( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + package_roles: List[Dict[str, Any]], +) -> None: + bundle_dir = ctx.bundle_dir + out_dir = ctx.out_dir + roles_root = ctx.roles_root + fqdn = ctx.fqdn + site_mode = ctx.site_mode + jt_exe = ctx.jt_exe + jt_enabled = ctx.jt_enabled + + # Process package roles (those with configuration files) + for pr in package_roles: + source_role = pr["role_name"] + role = avoid_reserved_role_name(source_role, prefix="package") + pkg = pr.get("package") or "" + managed_files = pr.get("managed_files", []) or [] + managed_dirs = pr.get("managed_dirs", []) or [] + managed_links = pr.get("managed_links", []) or [] + + ansible_role = AnsibleRole(role) + ansible_role.add_package_snapshot(pr) + + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + + var_prefix = role + + templated, jt_vars = _jinjify_managed_files( + bundle_dir, + source_role, + role_dir, + managed_files, + jt_exe=jt_exe, + jt_enabled=jt_enabled, + overwrite_templates=not site_mode, + ) + + # Copy only the non-templated artifacts. + if site_mode: + _copy_artifacts( + bundle_dir, + source_role, + _host_role_files_dir(out_dir, fqdn or "", role), + exclude_rels=templated, + ) + else: + _copy_artifacts( + bundle_dir, + source_role, + os.path.join(role_dir, "files"), + exclude_rels=templated, + ) + + pkgs = ansible_role.sorted_packages + + files_var = _build_managed_files_var( + managed_files, + templated, + notify_other=None, + notify_systemd="Run systemd daemon-reload", + ) + + links_var = _build_managed_links_var(managed_links) + + dirs_var = _build_managed_dirs_var(managed_dirs) + + jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} + base_vars: Dict[str, Any] = { + f"{var_prefix}_packages": pkgs, + f"{var_prefix}_managed_files": files_var, + f"{var_prefix}_managed_dirs": dirs_var, + f"{var_prefix}_managed_links": links_var, + } + base_vars = _merge_mappings_overwrite(base_vars, jt_map) + + if site_mode: + _write_role_defaults( + role_dir, + { + f"{var_prefix}_packages": [], + f"{var_prefix}_managed_files": [], + f"{var_prefix}_managed_dirs": [], + f"{var_prefix}_managed_links": [], + }, + ) + _write_hostvars(out_dir, fqdn or "", role, base_vars) + else: + _write_role_defaults(role_dir, base_vars) + + handlers = """--- +- name: Run systemd daemon-reload + ansible.builtin.systemd: + daemon_reload: true + no_log: "{{ enroll_hide_systemd_status | default(true) | bool }}" +""" + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(handlers) + + task_parts: List[str] = [] + task_parts.append("---\n" + _render_install_packages_tasks(role, var_prefix)) + task_parts.append( + _render_generic_files_tasks(var_prefix, include_restart_notify=False) + ) + + tasks = "\n".join(task_parts).rstrip() + "\n" + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(tasks) + + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write("---\ndependencies: []\n") + + excluded = pr.get("excluded", []) + notes = pr.get("notes", []) + readme = f"""# {role} + +Generated for package `{pkg}`. + +## Managed files +{os.linesep.join("- " + mf["path"] + " (" + mf["reason"] + ")" for mf in managed_files) or "- (none)"} + +## Managed symlinks +{os.linesep.join("- " + ml["path"] + " -> " + ml["target"] + " (" + ml.get("reason", "") + ")" for ml in managed_links) or "- (none)"} + +## Excluded (possible secrets / unsafe) +{os.linesep.join("- " + e["path"] + " (" + e["reason"] + ")" for e in excluded) or "- (none)"} + +## Notes +{os.linesep.join("- " + n for n in notes) or "- (none)"} + +> Note: package roles (those not discovered via a systemd service) do not attempt to restart or enable services automatically. +""" + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("package", role) + + +class AnsibleManifestRenderer: + """Render Ansible roles and playbook from a harvest bundle.""" + + def __init__( + self, + bundle_dir: str, + out_dir: str, + *, + fqdn: Optional[str] = None, + jinjaturtle: str = "auto", + no_common_roles: bool = False, + ) -> None: + self.bundle_dir = bundle_dir + self.out_dir = out_dir + self.fqdn = fqdn + self.jinjaturtle = jinjaturtle + self.no_common_roles = no_common_roles + + def render(self) -> None: + bundle_dir = self.bundle_dir + out_dir = self.out_dir + fqdn = self.fqdn + jinjaturtle = self.jinjaturtle + no_common_roles = self.no_common_roles + state = AnsibleRole.load_state(bundle_dir) + + roles = roles_from_state(state) + inventory_packages = inventory_packages_from_state(state) + + ctx = _prepare_ansible_context( + bundle_dir, + out_dir, + fqdn=fqdn, + jinjaturtle=jinjaturtle, + ) + _write_site_scaffold(ctx) + + use_common_roles = (not ctx.site_mode) and (not no_common_roles) + collection = _collect_ansible_roles( + roles, + inventory_packages, + use_common_roles=use_common_roles, + ) + + services_to_manifest = collection.services + package_roles = collection.packages + common_role_groups = collection.common_role_groups + + users_snapshot: Dict[str, Any] = roles.get("users", {}) + flatpak_snapshot: Dict[str, Any] = roles.get("flatpak", {}) + snap_snapshot: Dict[str, Any] = roles.get("snap", {}) + firewall_runtime_snapshot: Dict[str, Any] = roles.get("firewall_runtime", {}) + sysctl_snapshot: Dict[str, Any] = roles.get("sysctl", {}) + + manifest_plan = AnsibleManifestPlan() + + _render_users_role(ctx, manifest_plan, users_snapshot) + + _render_flatpak_role(ctx, manifest_plan, flatpak_snapshot) + + _render_snap_role(ctx, manifest_plan, snap_snapshot) + + _render_managed_file_roles(ctx, manifest_plan, roles) + _render_sysctl_role(ctx, manifest_plan, sysctl_snapshot) + _render_firewall_runtime_role(ctx, manifest_plan, firewall_runtime_snapshot) + + _render_service_roles(ctx, manifest_plan, services_to_manifest) + + common_tail_roles = _render_common_ansible_roles( + ctx, manifest_plan, common_role_groups, package_roles + ) + + _render_package_roles(ctx, manifest_plan, package_roles) + + # Place cron/logrotate at the end of the playbook so: + # - users exist before we restore per-user crontabs in /var/spool + # - most packages/services are installed/configured first + for r in ("cron", "logrotate"): + manifest_plan.mark_tail_package(r) + for r in common_tail_roles: + manifest_plan.mark_tail_package(r) + + _write_manifest_playbook(ctx, manifest_plan.ordered_roles()) + + +def manifest_from_bundle_dir( + bundle_dir: str, + out_dir: str, + *, + fqdn: Optional[str] = None, + jinjaturtle: str = "auto", # auto|on|off + no_common_roles: bool = False, +) -> None: + AnsibleManifestRenderer( + bundle_dir, + out_dir, + fqdn=fqdn, + jinjaturtle=jinjaturtle, + no_common_roles=no_common_roles, + ).render() diff --git a/enroll/cli.py b/enroll/cli.py index 7106d7a..d8ab5cc 100644 --- a/enroll/cli.py +++ b/enroll/cli.py @@ -312,7 +312,7 @@ def _add_common_manifest_args(p: argparse.ArgumentParser) -> None: "--target", choices=["ansible", "puppet"], default="ansible", - help="Manifest target to generate (default: ansible). Puppet output is an initial conservative target.", + help="Manifest target to generate (default: ansible).", ) p.add_argument( "--fqdn", diff --git a/enroll/cm.py b/enroll/cm.py new file mode 100644 index 0000000..856b7c7 --- /dev/null +++ b/enroll/cm.py @@ -0,0 +1,299 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, Iterable, Iterator, List, Mapping, Set + +from .state import load_state, state_path, write_state + + +@dataclass +class CMModule: + """Renderer-neutral configuration-management resource group. + + A CMModule is intentionally small: it captures the resources that a target + renderer can turn into Ansible tasks, Puppet resources, Salt states, etc. + The renderer may still decide how to name/include/order the group. + """ + + role_name: str + module_name: str + packages: Set[str] = field(default_factory=set) + groups: Set[str] = field(default_factory=set) + users: Dict[str, Dict[str, Any]] = field(default_factory=dict) + dirs: Dict[str, Dict[str, Any]] = field(default_factory=dict) + files: Dict[str, Dict[str, Any]] = field(default_factory=dict) + links: Dict[str, Dict[str, Any]] = field(default_factory=dict) + services: Dict[str, Dict[str, Any]] = field(default_factory=dict) + notes: List[str] = field(default_factory=list) + + def has_resources(self) -> bool: + return bool( + self.packages + or self.groups + or self.users + or self.dirs + or self.files + or self.links + or self.services + or self.notes + ) + + @staticmethod + def state_path(bundle_dir: str | Path) -> Path: + """Return the canonical state.json path for a harvest bundle.""" + + return state_path(bundle_dir) + + @classmethod + def load_state(cls, bundle_dir: str | Path) -> Dict[str, Any]: + """Load state.json for a renderer using the shared bundle state loader.""" + + return load_state(bundle_dir) + + @classmethod + def _load_state(cls, bundle_dir: str | Path) -> Dict[str, Any]: + """Backward-compatible alias for renderer subclasses.""" + + return cls.load_state(bundle_dir) + + @classmethod + def write_state( + cls, + bundle_dir: str | Path, + state: Mapping[str, Any], + *, + indent: int = 2, + sort_keys: bool = True, + ) -> Path: + """Write state.json using the shared bundle state writer.""" + + return write_state(bundle_dir, state, indent=indent, sort_keys=sort_keys) + + @staticmethod + def _snapshot_items(snap: Dict[str, Any], key: str) -> Iterator[Dict[str, Any]]: + values = snap.get(key) or [] + if not isinstance(values, list): + return + for item in values: + if isinstance(item, dict): + yield item + + @classmethod + def managed_dirs_from_snapshot( + cls, snap: Dict[str, Any] + ) -> Iterator[Dict[str, Any]]: + return cls._snapshot_items(snap, "managed_dirs") + + @classmethod + def managed_files_from_snapshot( + cls, snap: Dict[str, Any] + ) -> Iterator[Dict[str, Any]]: + return cls._snapshot_items(snap, "managed_files") + + @classmethod + def managed_links_from_snapshot( + cls, snap: Dict[str, Any] + ) -> Iterator[Dict[str, Any]]: + return cls._snapshot_items(snap, "managed_links") + + def add_managed_dir( + self, + path: str, + *, + owner: Any = "root", + group: Any = "root", + mode: Any = "0755", + **attrs: Any, + ) -> None: + if not path: + return + data: Dict[str, Any] = { + "owner": owner or "root", + "group": group or "root", + "mode": mode or "0755", + } + data.update(attrs) + self.dirs.setdefault(path, data) + + def add_managed_file( + self, + path: str, + *, + owner: Any = "root", + group: Any = "root", + mode: Any = "0644", + **attrs: Any, + ) -> None: + if not path: + return + data: Dict[str, Any] = { + "owner": owner or "root", + "group": group or "root", + "mode": mode or "0644", + } + data.update(attrs) + self.files.setdefault(path, data) + + def add_managed_link(self, path: str, **attrs: Any) -> None: + if path: + self.links.setdefault(path, attrs) + + def add_snapshot_notes(self, snap: Dict[str, Any]) -> None: + self.notes.extend(str(n) for n in (snap.get("notes", []) or [])) + + def remove_directory_resource_conflicts(self) -> None: + for path in set(self.files) | set(self.links): + self.dirs.pop(path, None) + + +def package_section_label( + package_role: Dict[str, Any], inventory_packages: Dict[str, Any] +) -> str: + """Return the Debian Section/RPM Group label for a package role.""" + + pkg = str(package_role.get("package") or "").strip() + inv = inventory_packages.get(pkg) or {} + candidates: List[str] = [] + + for value in (package_role.get("section"), inv.get("section"), inv.get("group")): + if isinstance(value, str) and value.strip(): + candidates.append(value.strip()) + + for inst in inv.get("installations", []) or []: + if not isinstance(inst, dict): + continue + for key in ("section", "group"): + value = inst.get(key) + if isinstance(value, str) and value.strip(): + candidates.append(value.strip()) + + for value in candidates: + if value.lower() not in {"(none)", "none", "unspecified"}: + return value + return "misc" + + +def section_label_for_packages( + packages: List[str], inventory_packages: Dict[str, Any] +) -> str: + """Return a stable section/group label for a set of packages.""" + + for pkg in packages or []: + label = package_section_label({"package": pkg}, inventory_packages) + if label and label.lower() != "misc": + return label + return "misc" + + +def role_order_key(role: str) -> tuple[int, str]: + # Keep broadly similar ordering to generated Ansible playbooks: package/config + # scaffolding first, then services/users, then host-specific runtime state. + priority = { + "apt_config": 10, + "dnf_config": 11, + "etc_custom": 80, + "usr_local_custom": 81, + "extra_paths": 82, + "users": 90, + "sysctl": 95, + "firewall_runtime": 99, + } + return (priority.get(role, 50), role) + + +def _drop_duplicate_set_items( + module: CMModule, + values: Set[str], + seen: Set[str], + resource_type: str, +) -> Set[str]: + kept: Set[str] = set() + for value in sorted(values): + if value in seen: + module.notes.append( + f"Skipped duplicate {resource_type}[{value}] already emitted earlier in this catalog." + ) + continue + kept.add(value) + seen.add(value) + return kept + + +def _drop_duplicate_mapping_items( + module: CMModule, + values: Dict[str, Dict[str, Any]], + seen: Set[str], + resource_type: str, + *, + excluded_titles: Set[str] | None = None, + excluded_reason: str = "conflicts with another resource", +) -> Dict[str, Dict[str, Any]]: + kept: Dict[str, Dict[str, Any]] = {} + excluded_titles = excluded_titles or set() + for title, attrs in values.items(): + if title in excluded_titles: + module.notes.append(f"Skipped {resource_type}[{title}]: {excluded_reason}.") + continue + if title in seen: + module.notes.append( + f"Skipped duplicate {resource_type}[{title}] already emitted earlier in this catalog." + ) + continue + kept[title] = attrs + seen.add(title) + return kept + + +def resolve_catalog_conflicts(modules: Iterable[CMModule]) -> None: + """Resolve global catalog conflicts before renderer output. + + Puppet and Salt compile a single resource catalog. Ansible can tolerate the + same package, service, or parent directory appearing in more than one role; + catalog targets cannot. Resolve those conflicts in the shared model rather + than deleting renderer output after the fact. + """ + + ordered = list(modules) + concrete_file_paths: Set[str] = set() + for module in ordered: + concrete_file_paths.update(module.files) + concrete_file_paths.update(module.links) + + seen_packages: Set[str] = set() + seen_groups: Set[str] = set() + seen_users: Set[str] = set() + seen_dirs: Set[str] = set() + seen_files: Set[str] = set() + seen_links: Set[str] = set() + seen_services: Set[str] = set() + + for module in ordered: + module.packages = _drop_duplicate_set_items( + module, module.packages, seen_packages, "Package" + ) + module.groups = _drop_duplicate_set_items( + module, module.groups, seen_groups, "Group" + ) + module.users = _drop_duplicate_mapping_items( + module, module.users, seen_users, "User" + ) + module.dirs = _drop_duplicate_mapping_items( + module, + module.dirs, + seen_dirs, + "File", + excluded_titles=concrete_file_paths, + excluded_reason="a file or link with the same path is emitted in this catalog", + ) + module.files = _drop_duplicate_mapping_items( + module, module.files, seen_files | seen_links, "File" + ) + seen_files.update(module.files) + module.links = _drop_duplicate_mapping_items( + module, module.links, seen_links | seen_files, "File" + ) + seen_links.update(module.links) + module.services = _drop_duplicate_mapping_items( + module, module.services, seen_services, "Service" + ) diff --git a/enroll/diff.py b/enroll/diff.py index eb496a5..4784119 100644 --- a/enroll/diff.py +++ b/enroll/diff.py @@ -21,6 +21,12 @@ from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Tuple from .remote import _safe_extract_tar +from .state import ( + inventory_packages_from_state as _packages_inventory, + load_state as _load_state, + roles_from_state as _roles, + state_path, +) from .pathfilter import PathFilter from .sopsutil import decrypt_file_binary_to, require_sops_cmd @@ -116,7 +122,7 @@ class BundleRef: @property def state_path(self) -> Path: - return self.dir / "state.json" + return state_path(self.dir) def _bundle_from_input(path: str, *, sops_mode: bool) -> BundleRef: @@ -189,24 +195,10 @@ def _bundle_from_input(path: str, *, sops_mode: bool) -> BundleRef: ) -def _load_state(bundle_dir: Path) -> Dict[str, Any]: - sp = bundle_dir / "state.json" - with open(sp, "r", encoding="utf-8") as f: - return json.load(f) - - -def _packages_inventory(state: Dict[str, Any]) -> Dict[str, Any]: - return (state.get("inventory") or {}).get("packages") or {} - - def _all_packages(state: Dict[str, Any]) -> List[str]: return sorted(_packages_inventory(state).keys()) -def _roles(state: Dict[str, Any]) -> Dict[str, Any]: - return state.get("roles") or {} - - def _pkg_version_key(entry: Dict[str, Any]) -> Optional[str]: """Return a stable string used for version comparison.""" installs = entry.get("installations") or [] diff --git a/enroll/explain.py b/enroll/explain.py index b4f4de8..84d5de0 100644 --- a/enroll/explain.py +++ b/enroll/explain.py @@ -5,7 +5,8 @@ from collections import Counter, defaultdict from dataclasses import dataclass from typing import Any, Dict, Iterable, List, Tuple -from .diff import _bundle_from_input, _load_state # reuse existing bundle handling +from .diff import _bundle_from_input # reuse existing bundle handling +from .state import load_state @dataclass(frozen=True) @@ -289,7 +290,7 @@ def explain_state( - a SOPS-encrypted bundle (.sops) """ bundle = _bundle_from_input(harvest, sops_mode=sops_mode) - state = _load_state(bundle.dir) + state = load_state(bundle.dir) host = state.get("host") or {} enroll = state.get("enroll") or {} diff --git a/enroll/harvest.py b/enroll/harvest.py index 664d1ae..25146a2 100644 --- a/enroll/harvest.py +++ b/enroll/harvest.py @@ -1,7 +1,6 @@ from __future__ import annotations import glob -import json import os import re import shutil @@ -12,20 +11,37 @@ import time from dataclasses import dataclass, asdict, field from typing import Any, Dict, List, Optional, Set, Tuple +from . import accounts as _accounts +from . import systemd as _systemd from .role_names import avoid_reserved_role_name -from .systemd import ( - list_enabled_services, - list_enabled_timers, - get_unit_info, - get_timer_info, - UnitQueryError, -) from .fsutil import stat_triplet from .platform import detect_platform, get_backend from .ignore import IgnorePolicy -from .pathfilter import PathFilter, expand_includes -from .accounts import collect_non_system_users +from .pathfilter import PathFilter from .version import get_enroll_version +from .state import write_state + +UnitQueryError = _systemd.UnitQueryError + + +def list_enabled_services() -> List[str]: + return _systemd.list_enabled_services() + + +def list_enabled_timers() -> List[str]: + return _systemd.list_enabled_timers() + + +def get_unit_info(unit: str) -> Any: + return _systemd.get_unit_info(unit) + + +def get_timer_info(timer: str) -> Any: + return _systemd.get_timer_info(timer) + + +def collect_non_system_users() -> List[Any]: + return _accounts.collect_non_system_users() @dataclass @@ -1441,6 +1457,18 @@ def _collect_firewall_runtime_snapshot( ) +from .harvest_collectors import ( + CronLogrotateCollector, + ExtraPathsCollector, + HarvestContext, + PackageManagerConfigCollector, + RuntimeStateCollector, + ServicePackageCollector, + UsersCollector, + UsrLocalCustomCollector, +) + + def harvest( bundle_dir: str, policy: Optional[IgnorePolicy] = None, @@ -1502,845 +1530,72 @@ def harvest( _PERSISTENT_IPTABLES_V6_GLOBS ) - running_as_root = not hasattr(os, "geteuid") or os.geteuid() == 0 - if not running_as_root: - firewall_runtime_snapshot = FirewallRuntimeSnapshot( - role_name="firewall_runtime", - notes=[ - "Live ipset/iptables runtime capture skipped because harvest is not running as root." - ], - ) - sysctl_snapshot = SysctlSnapshot( - role_name="sysctl", - notes=[ - "Live sysctl runtime capture skipped because harvest is not running as root." - ], - ) - else: - firewall_runtime_snapshot = _collect_firewall_runtime_snapshot( - bundle_dir, - persistent_ipset_files=persistent_ipset_files, - persistent_iptables_v4_files=persistent_iptables_v4_files, - persistent_iptables_v6_files=persistent_iptables_v6_files, - ) - sysctl_snapshot = _collect_sysctl_snapshot(bundle_dir) + context = HarvestContext( + bundle_dir=bundle_dir, + policy=policy, + path_filter=path_filter, + platform=platform, + backend=backend, + installed_pkgs=installed_pkgs, + installed_names=installed_names, + owned_etc=owned_etc, + etc_owner_map=etc_owner_map, + topdir_to_pkgs=topdir_to_pkgs, + pkg_to_etc_paths=pkg_to_etc_paths, + captured_global=captured_global, + ) + + runtime_collection = RuntimeStateCollector( + context, + persistent_ipset_files=persistent_ipset_files, + persistent_iptables_v4_files=persistent_iptables_v4_files, + persistent_iptables_v6_files=persistent_iptables_v6_files, + ).collect() + firewall_runtime_snapshot = runtime_collection.firewall_runtime_snapshot + sysctl_snapshot = runtime_collection.sysctl_snapshot # The generated sysctl role owns /etc/sysctl.d/99-enroll.conf; do not also # capture an existing file at that path into etc_custom/package roles. for mf in sysctl_snapshot.managed_files: captured_global.add(mf.path) - def _pick_installed(cands: List[str]) -> Optional[str]: - for c in cands: - if c in installed_names: - return c - return None + cron_logrotate_collection = CronLogrotateCollector(context).collect() + cron_pkg = cron_logrotate_collection.cron_pkg + logrotate_pkg = cron_logrotate_collection.logrotate_pkg + cron_snapshot = cron_logrotate_collection.cron_snapshot + logrotate_snapshot = cron_logrotate_collection.logrotate_snapshot - cron_pkg = _pick_installed( - ["cron", "cronie", "cronie-anacron", "vixie-cron", "fcron"] - ) - logrotate_pkg = _pick_installed(["logrotate"]) - - cron_role_name = "cron" - logrotate_role_name = "logrotate" - - def _is_cron_path(p: str) -> bool: - return ( - p == "/etc/crontab" - or p == "/etc/anacrontab" - or p in ("/etc/cron.allow", "/etc/cron.deny") - or p.startswith("/etc/cron.") - or p.startswith("/etc/cron.d/") - or p.startswith("/etc/anacron/") - or p.startswith("/var/spool/cron/") - or p.startswith("/var/spool/crontabs/") - or p.startswith("/var/spool/anacron/") - ) - - def _is_logrotate_path(p: str) -> bool: - return p == "/etc/logrotate.conf" or p.startswith("/etc/logrotate.d/") - - cron_snapshot: Optional[PackageSnapshot] = None - logrotate_snapshot: Optional[PackageSnapshot] = None - - if cron_pkg: - cron_managed: List[ManagedFile] = [] - cron_excluded: List[ExcludedFile] = [] - cron_notes: List[str] = [] - cron_seen: Set[str] = set() - - cron_globs = [ - "/etc/crontab", - "/etc/cron.d/*", - "/etc/cron.hourly/*", - "/etc/cron.daily/*", - "/etc/cron.weekly/*", - "/etc/cron.monthly/*", - "/etc/cron.allow", - "/etc/cron.deny", - "/etc/anacrontab", - "/etc/anacron/*", - # user crontabs / spool state - "/var/spool/cron/*", - "/var/spool/cron/crontabs/*", - "/var/spool/crontabs/*", - "/var/spool/anacron/*", - ] - for spec in cron_globs: - for path in _iter_matching_files(spec): - if not os.path.isfile(path) or os.path.islink(path): - continue - _capture_file( - bundle_dir=bundle_dir, - role_name=cron_role_name, - abs_path=path, - reason="system_cron", - policy=policy, - path_filter=path_filter, - managed_out=cron_managed, - excluded_out=cron_excluded, - seen_role=cron_seen, - seen_global=captured_global, - ) - - cron_snapshot = PackageSnapshot( - package=cron_pkg, - role_name=cron_role_name, - section=_package_section_from_installations( - installed_pkgs.get(cron_pkg, []) - ), - managed_files=cron_managed, - excluded=cron_excluded, - notes=cron_notes, - ) - - if logrotate_pkg: - lr_managed: List[ManagedFile] = [] - lr_excluded: List[ExcludedFile] = [] - lr_notes: List[str] = [] - lr_seen: Set[str] = set() - - lr_globs = [ - "/etc/logrotate.conf", - "/etc/logrotate.d/*", - ] - for spec in lr_globs: - for path in _iter_matching_files(spec): - if not os.path.isfile(path) or os.path.islink(path): - continue - _capture_file( - bundle_dir=bundle_dir, - role_name=logrotate_role_name, - abs_path=path, - reason="system_logrotate", - policy=policy, - path_filter=path_filter, - managed_out=lr_managed, - excluded_out=lr_excluded, - seen_role=lr_seen, - seen_global=captured_global, - ) - - logrotate_snapshot = PackageSnapshot( - package=logrotate_pkg, - role_name=logrotate_role_name, - section=_package_section_from_installations( - installed_pkgs.get(logrotate_pkg, []) - ), - managed_files=lr_managed, - excluded=lr_excluded, - notes=lr_notes, - ) - # ------------------------- - # Service roles - # ------------------------- - service_snaps: List[ServiceSnapshot] = [] - # Track alias strings (service names, package names, stems) that should map - # back to the service role for shared snippet attribution (cron.d/logrotate.d). - service_role_aliases: Dict[str, Set[str]] = {} - # De-dupe per-role captures (avoids duplicate tasks in manifest generation). - seen_by_role: Dict[str, Set[str]] = {} - # Managed/excluded lists keyed by role so helper services can attribute shared - # configuration to their parent service role. - managed_by_role: Dict[str, List[ManagedFile]] = {} - excluded_by_role: Dict[str, List[ExcludedFile]] = {} - - enabled_services = list_enabled_services() - - # Avoid role-name collisions with dedicated cron/logrotate package roles. - if cron_snapshot is not None or logrotate_snapshot is not None: - blocked_roles = set() - if cron_snapshot is not None: - blocked_roles.add(cron_role_name) - if logrotate_snapshot is not None: - blocked_roles.add(logrotate_role_name) - enabled_services = [ - u for u in enabled_services if _role_name_from_unit(u) not in blocked_roles - ] - enabled_set = set(enabled_services) - - def _service_sort_key(unit: str) -> tuple[int, str, str]: - # Prefer "parent" services over helpers (e.g. NetworkManager.service before - # NetworkManager-dispatcher.service) so shared config lands in the main role. - base = unit.removesuffix(".service") - base = base.split("@", 1)[0] - return (base.count("-"), base.lower(), unit.lower()) - - def _parent_service_unit(unit: str) -> Optional[str]: - # If unit name contains '-' segments, treat dashed prefixes as potential parents. - # Example: NetworkManager-dispatcher.service -> NetworkManager.service (if enabled). - if not unit.endswith(".service"): - return None - base = unit.removesuffix(".service") - base = base.split("@", 1)[0] - parts = base.split("-") - for i in range(len(parts) - 1, 0, -1): - cand = "-".join(parts[:i]) + ".service" - if cand in enabled_set: - return cand - return None - - parent_unit_for: Dict[str, str] = {} - for u in enabled_services: - pu = _parent_service_unit(u) - if pu: - parent_unit_for[u] = pu - - for unit in sorted(enabled_services, key=_service_sort_key): - role = _role_name_from_unit(unit) - parent_unit = parent_unit_for.get(unit) - parent_role = _role_name_from_unit(parent_unit) if parent_unit else None - - try: - ui = get_unit_info(unit) - except UnitQueryError as e: - # Even when we can't query the unit, keep a minimal alias mapping so - # shared snippets can still be attributed to this role by name. - service_role_aliases.setdefault(role, _hint_names(unit, set()) | {role}) - seen_by_role.setdefault(role, set()) - managed = managed_by_role.setdefault(role, []) - excluded = excluded_by_role.setdefault(role, []) - service_snaps.append( - ServiceSnapshot( - unit=unit, - role_name=role, - packages=[], - active_state=None, - sub_state=None, - unit_file_state=None, - condition_result=None, - managed_files=managed, - excluded=excluded, - notes=[str(e)], - ) - ) - continue - - pkgs: Set[str] = set() - notes: List[str] = [] - excluded = excluded_by_role.setdefault(role, []) - managed = managed_by_role.setdefault(role, []) - candidates: Dict[str, str] = {} - - if ui.fragment_path: - p = backend.owner_of_path(ui.fragment_path) - if p: - pkgs.add(p) - - for exe in ui.exec_paths: - p = backend.owner_of_path(exe) - if p: - pkgs.add(p) - - for pth in ui.dropin_paths: - if pth.startswith("/etc/"): - candidates[pth] = "systemd_dropin" - - for ef in ui.env_files: - ef = ef.lstrip("-") - if any(ch in ef for ch in "*?["): - for g in glob.glob(ef): - if g.startswith("/etc/") and os.path.isfile(g): - candidates[g] = "systemd_envfile" - else: - if ef.startswith("/etc/") and os.path.isfile(ef): - candidates[ef] = "systemd_envfile" - - hints = _hint_names(unit, pkgs) - _add_pkgs_from_etc_topdirs(hints, topdir_to_pkgs, pkgs) - # Keep a stable set of aliases for this service role. Include current - # packages as well, so that package-named snippets (e.g. cron.d or - # logrotate.d entries) can still be attributed back to this service. - service_role_aliases[role] = set(hints) | set(pkgs) | {role} - - for sp in _maybe_add_specific_paths(hints, backend): - if not os.path.exists(sp): - continue - if sp in etc_owner_map: - pkgs.add(etc_owner_map[sp]) - else: - candidates.setdefault(sp, "custom_specific_path") - - for pkg in sorted(pkgs): - etc_paths = pkg_to_etc_paths.get(pkg, []) - for path, reason in backend.modified_paths(pkg, etc_paths).items(): - if not os.path.isfile(path) or os.path.islink(path): - continue - if cron_snapshot is not None and _is_cron_path(path): - continue - if logrotate_snapshot is not None and _is_logrotate_path(path): - continue - if backend.is_pkg_config_path(path): - continue - candidates.setdefault(path, reason) - - # Capture custom/unowned files living under /etc/ for this service. - # - # Historically we only captured "config-ish" files (by extension). That - # misses important runtime-generated artifacts like certificates and - # key material under service directories (e.g. /etc/openvpn/*.crt). - # - # To avoid exploding output for shared trees (e.g. /etc/systemd), keep - # the older "config-ish only" behaviour for known shared topdirs. - any_roots: List[str] = [] - confish_roots: List[str] = [] - for h in hints: - roots_for_h = [f"/etc/{h}", f"/etc/{h}.d"] - if h in SHARED_ETC_TOPDIRS: - confish_roots.extend(roots_for_h) - else: - any_roots.extend(roots_for_h) - - found: List[str] = [] - found.extend( - _scan_unowned_under_roots( - any_roots, - owned_etc, - limit=MAX_UNOWNED_FILES_PER_ROLE, - confish_only=False, - ) - ) - if len(found) < MAX_UNOWNED_FILES_PER_ROLE: - found.extend( - _scan_unowned_under_roots( - confish_roots, - owned_etc, - limit=MAX_UNOWNED_FILES_PER_ROLE - len(found), - confish_only=True, - ) - ) - for pth in found: - candidates.setdefault(pth, "custom_unowned") - - if not pkgs and not candidates: - notes.append( - "No packages or /etc candidates detected (unexpected for enabled service)." - ) - - # De-dupe within this role while capturing. This also avoids emitting - # duplicate Ansible tasks for the same destination path. - # Attribute shared /etc config to the parent service role when this unit looks - # like a helper (e.g. NetworkManager-dispatcher.service -> NetworkManager.service). - for path, reason in sorted(candidates.items()): - dest_role = role - if ( - parent_role - and path.startswith("/etc/") - and reason not in ("systemd_dropin", "systemd_envfile") - ): - dest_role = parent_role - - dest_managed = managed_by_role.setdefault(dest_role, []) - dest_excluded = excluded_by_role.setdefault(dest_role, []) - dest_seen = seen_by_role.setdefault(dest_role, set()) - _capture_file( - bundle_dir=bundle_dir, - role_name=dest_role, - abs_path=path, - reason=reason, - policy=policy, - path_filter=path_filter, - managed_out=dest_managed, - excluded_out=dest_excluded, - seen_role=dest_seen, - seen_global=captured_global, - ) - - service_snaps.append( - ServiceSnapshot( - unit=unit, - role_name=role, - packages=sorted(pkgs), - active_state=ui.active_state, - sub_state=ui.sub_state, - unit_file_state=ui.unit_file_state, - condition_result=ui.condition_result, - managed_files=managed, - excluded=excluded, - notes=notes, - ) - ) + service_package_collection = ServicePackageCollector( + context, + cron_snapshot=cron_snapshot, + logrotate_snapshot=logrotate_snapshot, + cron_pkg=cron_pkg, + logrotate_pkg=logrotate_pkg, + ).collect() + service_snaps = service_package_collection.service_snaps + pkg_snaps = service_package_collection.pkg_snaps + manual_pkgs = service_package_collection.manual_pkgs + service_role_aliases = service_package_collection.service_role_aliases + seen_by_role = service_package_collection.seen_by_role # ------------------------- - # Enabled systemd timers - # - # Timers are typically related to a service/package, so we try to attribute - # timer unit overrides to their associated role rather than creating a - # standalone timer role. If we can't attribute a timer, it will fall back - # to etc_custom (if it's a custom /etc unit). + # Users role, Flatpak and Snap state # ------------------------- - timer_extra_by_pkg: Dict[str, List[str]] = {} - try: - enabled_timers = list_enabled_timers() - except Exception: - enabled_timers = [] - - service_snap_by_unit: Dict[str, ServiceSnapshot] = { - s.unit: s for s in service_snaps - } - - for t in sorted(enabled_timers): - try: - ti = get_timer_info(t) - except Exception: # nosec - continue - - timer_paths: List[str] = [] - for pth in [ti.fragment_path, *ti.dropin_paths, *ti.env_files]: - if not pth: - continue - if not pth.startswith("/etc/"): - # Prefer capturing only custom/overridden units. - continue - if os.path.islink(pth) or not os.path.isfile(pth): - continue - timer_paths.append(pth) - - if not timer_paths: - continue - - # Primary attribution: timer -> trigger service role - snap = None - if ti.trigger_unit: - snap = service_snap_by_unit.get(ti.trigger_unit) - - if snap is not None: - role_seen = seen_by_role.setdefault(snap.role_name, set()) - for path in timer_paths: - _capture_file( - bundle_dir=bundle_dir, - role_name=snap.role_name, - abs_path=path, - reason="related_timer", - policy=policy, - path_filter=path_filter, - managed_out=snap.managed_files, - excluded_out=snap.excluded, - seen_role=role_seen, - seen_global=captured_global, - ) - continue - - # Secondary attribution: associate timer overrides with a package role - # (useful when a timer triggers a service that isn't enabled). - pkgs: Set[str] = set() - if ti.fragment_path: - p = backend.owner_of_path(ti.fragment_path) - if p: - pkgs.add(p) - if ti.trigger_unit and ti.trigger_unit.endswith(".service"): - try: - ui = get_unit_info(ti.trigger_unit) - if ui.fragment_path: - p = backend.owner_of_path(ui.fragment_path) - if p: - pkgs.add(p) - for exe in ui.exec_paths: - p = backend.owner_of_path(exe) - if p: - pkgs.add(p) - except Exception: # nosec - pass - - for pkg in pkgs: - timer_extra_by_pkg.setdefault(pkg, []).extend(timer_paths) - - # ------------------------- - # Manually installed package roles - # ------------------------- - manual_pkgs = backend.list_manual_packages() - # Avoid duplicate roles: if a manual package is already managed by any service role, skip its pkg_ role. - covered_by_services: Set[str] = set() - for s in service_snaps: - for p in s.packages: - covered_by_services.add(p) - - manual_pkgs_skipped: List[str] = [] - pkg_snaps: List[PackageSnapshot] = [] - simple_packages: List[str] = [] # Packages with no config/systemd/cron files - - # Add dedicated cron/logrotate roles (if detected) as package roles. - # These roles centralise all cron/logrotate managed files so they aren't scattered - # across unrelated roles. - if cron_snapshot is not None: - pkg_snaps.append(cron_snapshot) - if logrotate_snapshot is not None: - pkg_snaps.append(logrotate_snapshot) - for pkg in sorted(manual_pkgs): - # Skip packages that are already managed by service roles - if pkg in covered_by_services: - manual_pkgs_skipped.append(pkg) - continue - # Skip cron/logrotate packages (they have dedicated roles) - if cron_snapshot is not None and pkg == cron_pkg: - manual_pkgs_skipped.append(pkg) - continue - if logrotate_snapshot is not None and pkg == logrotate_pkg: - manual_pkgs_skipped.append(pkg) - continue - - role = _role_name_from_pkg(pkg) - - notes: List[str] = [] - excluded: List[ExcludedFile] = [] - managed: List[ManagedFile] = [] - candidates: Dict[str, str] = {} - - for tpath in timer_extra_by_pkg.get(pkg, []): - candidates.setdefault(tpath, "related_timer") - - etc_paths = pkg_to_etc_paths.get(pkg, []) - for path, reason in backend.modified_paths(pkg, etc_paths).items(): - if not os.path.isfile(path) or os.path.islink(path): - continue - if cron_snapshot is not None and _is_cron_path(path): - continue - if logrotate_snapshot is not None and _is_logrotate_path(path): - continue - if backend.is_pkg_config_path(path): - continue - candidates.setdefault(path, reason) - - topdirs = _topdirs_for_package(pkg, pkg_to_etc_paths) - roots: List[str] = [] - # Collect candidate directories plus backend-specific common files. - for td in sorted(topdirs): - if td in SHARED_ETC_TOPDIRS: - continue - if backend.is_pkg_config_path(f"/etc/{td}/") or backend.is_pkg_config_path( - f"/etc/{td}" - ): - continue - roots.extend([f"/etc/{td}", f"/etc/{td}.d"]) - roots.extend(_maybe_add_specific_paths(set(topdirs), backend)) - - # Capture any custom/unowned files under /etc/ for this - # manually-installed package. This may include runtime-generated - # artifacts like certificates, key files, and helper scripts which are - # not owned by any .deb. - for pth in _scan_unowned_under_roots( - [r for r in roots if os.path.isdir(r)], - owned_etc, - confish_only=False, - ): - candidates.setdefault(pth, "custom_unowned") - - for r in roots: - if os.path.isfile(r) and not os.path.islink(r): - if r not in owned_etc and _is_confish(r): - candidates.setdefault(r, "custom_specific_path") - - role_seen = seen_by_role.setdefault(role, set()) - for path, reason in sorted(candidates.items()): - _capture_file( - bundle_dir=bundle_dir, - role_name=role, - abs_path=path, - reason=reason, - policy=policy, - path_filter=path_filter, - managed_out=managed, - excluded_out=excluded, - seen_role=role_seen, - seen_global=captured_global, - ) - - has_config = bool(managed or excluded) - - if not has_config: - notes.append( - "No changed or custom configuration detected for this package." - ) - simple_packages.append(pkg) - - pkg_snaps.append( - PackageSnapshot( - package=pkg, - role_name=role, - section=_package_section_from_installations( - installed_pkgs.get(pkg, []) - ), - managed_files=managed, - managed_links=[], - excluded=excluded, - notes=notes, - has_config=has_config, - ) - ) - - # ------------------------- - # Web server enablement symlinks (nginx/apache2) - # - # Debian-style nginx/apache2 configurations often use *-enabled directories - # populated with symlinks pointing back into *-available. The symlinks - # represent the enablement state and are important to reproduce. - # - # We only harvest these when the relevant service/package has already been - # detected in this run (i.e. we have a role that will manage nginx/apache2). - # ------------------------- - - def _find_role_snapshot(role_name: str): - for s in service_snaps: - if s.role_name == role_name: - return s - for p in pkg_snaps: - if p.role_name == role_name: - return p - return None - - def _capture_enabled_symlinks(role_name: str, dirs: List[str]) -> None: - snap = _find_role_snapshot(role_name) - if snap is None: - return - - role_seen = seen_by_role.setdefault(role_name, set()) - for d in dirs: - if not os.path.isdir(d): - continue - for pth in sorted(glob.glob(os.path.join(d, "*"))): - if not os.path.islink(pth): - continue - _capture_link( - role_name=role_name, - abs_path=pth, - reason="enabled_symlink", - policy=policy, - path_filter=path_filter, - managed_out=snap.managed_links, - excluded_out=snap.excluded, - seen_role=role_seen, - seen_global=captured_global, - ) - - _capture_enabled_symlinks( - "nginx", - [ - "/etc/nginx/modules-enabled", - "/etc/nginx/sites-enabled", - ], - ) - _capture_enabled_symlinks( - "apache2", - [ - "/etc/apache2/conf-enabled", - "/etc/apache2/mods-enabled", - "/etc/apache2/sites-enabled", - ], - ) - - # ------------------------- - # Users role (non-system users) - # ------------------------- - users_notes: List[str] = [] - users_excluded: List[ExcludedFile] = [] - users_managed: List[ManagedFile] = [] - users_list: List[dict] = [] - - try: - user_records = collect_non_system_users() - except Exception as e: - user_records = [] - users_notes.append(f"Failed to enumerate users: {e!r}") - - # Detect system-wide Flatpaks/Snaps and configured Flatpak remotes. - from .accounts import ( - find_system_flatpak_remotes, - find_system_flatpaks, - find_system_snaps, - find_user_flatpak_remotes, - ) - - system_flatpaks = [asdict(f) for f in find_system_flatpaks()] - system_snaps = [asdict(s) for s in find_system_snaps()] - system_flatpak_remotes = [asdict(r) for r in find_system_flatpak_remotes()] - flatpak_notes: List[str] = [] - snap_notes: List[str] = [] - if system_flatpaks: - flatpak_notes.append( - "System-wide flatpaks detected: " - + ", ".join(str(f.get("name")) for f in system_flatpaks) - ) - if system_snaps: - snap_notes.append( - "System-wide snaps detected: " - + ", ".join(str(s.get("name")) for s in system_snaps) - ) - - users_role_name = "users" - users_role_seen = seen_by_role.setdefault(users_role_name, set()) - - skel_dir = "/etc/skel" - auto_capture_user_dotfiles = bool(getattr(policy, "dangerous", False)) - if user_records and not auto_capture_user_dotfiles: - users_notes.append( - "User shell dotfiles were not auto-harvested because --dangerous was not set; " - "use --dangerous for automatic shell-dotfile capture, or targeted --include-path patterns for safe-mode review." - ) - - user_flatpaks_map: Dict[str, List[Dict[str, Any]]] = {} - user_flatpak_remotes: List[Dict[str, Any]] = [] - - for u in user_records: - users_list.append( - { - "name": u.name, - "uid": u.uid, - "gid": u.gid, - "gecos": u.gecos, - "home": u.home, - "shell": u.shell, - "primary_group": u.primary_group, - "supplementary_groups": u.supplementary_groups, - } - ) - - # Copy only safe SSH public material: authorized_keys + *.pub - for sf in u.ssh_files: - reason = ( - "authorized_keys" - if sf.endswith("/authorized_keys") - else "ssh_public_key" - ) - _capture_file( - bundle_dir=bundle_dir, - role_name=users_role_name, - abs_path=sf, - reason=reason, - policy=policy, - path_filter=path_filter, - managed_out=users_managed, - excluded_out=users_excluded, - seen_role=users_role_seen, - seen_global=captured_global, - ) - - # Capture common per-user shell dotfiles only in dangerous mode. They - # often contain exported tokens or aliases/functions with embedded secrets. - home = (u.home or "").rstrip("/") - if home and home.startswith("/"): - _capture_user_shell_dotfiles( - bundle_dir=bundle_dir, - role_name=users_role_name, - home=home, - skel_dir=skel_dir, - enabled=auto_capture_user_dotfiles, - policy=policy, - path_filter=path_filter, - managed_out=users_managed, - excluded_out=users_excluded, - seen_role=users_role_seen, - seen_global=captured_global, - ) - - # Collect per-user Flatpak applications and remotes. Snap packages are - # system-wide; ~/snap/* is user data, not an install source. - if u.flatpaks: - user_flatpaks_map[u.name] = [asdict(fp) for fp in u.flatpaks] - user_flatpak_remotes.extend( - asdict(r) for r in find_user_flatpak_remotes(home, user=u.name) - ) - - users_snapshot = UsersSnapshot( - role_name=users_role_name, - users=users_list, - managed_files=users_managed, - excluded=users_excluded, - notes=users_notes, - user_flatpaks=user_flatpaks_map, - user_flatpak_remotes=user_flatpak_remotes, - ) - - flatpak_snapshot = FlatpakSnapshot( - role_name="flatpak", - system_flatpaks=system_flatpaks, - remotes=system_flatpak_remotes, - notes=flatpak_notes, - ) - - snap_snapshot = SnapSnapshot( - role_name="snap", - system_snaps=system_snaps, - notes=snap_notes, - ) + users_collection = UsersCollector(context, seen_by_role).collect() + users_snapshot = users_collection.users_snapshot + flatpak_snapshot = users_collection.flatpak_snapshot + snap_snapshot = users_collection.snap_snapshot # ------------------------- # Package manager config role # - Debian: apt_config # - Fedora/RHEL-like: dnf_config # ------------------------- - apt_notes: List[str] = [] - apt_excluded: List[ExcludedFile] = [] - apt_managed: List[ManagedFile] = [] - dnf_notes: List[str] = [] - dnf_excluded: List[ExcludedFile] = [] - dnf_managed: List[ManagedFile] = [] - - apt_role_name = "apt_config" - dnf_role_name = "dnf_config" - - if backend.name == "dpkg": - apt_role_seen = seen_by_role.setdefault(apt_role_name, set()) - for path, reason in _iter_apt_capture_paths(): - _capture_file( - bundle_dir=bundle_dir, - role_name=apt_role_name, - abs_path=path, - reason=reason, - policy=policy, - path_filter=path_filter, - managed_out=apt_managed, - excluded_out=apt_excluded, - seen_role=apt_role_seen, - seen_global=captured_global, - ) - elif backend.name == "rpm": - dnf_role_seen = seen_by_role.setdefault(dnf_role_name, set()) - for path, reason in _iter_dnf_capture_paths(): - _capture_file( - bundle_dir=bundle_dir, - role_name=dnf_role_name, - abs_path=path, - reason=reason, - policy=policy, - path_filter=path_filter, - managed_out=dnf_managed, - excluded_out=dnf_excluded, - seen_role=dnf_role_seen, - seen_global=captured_global, - ) - - apt_config_snapshot = AptConfigSnapshot( - role_name=apt_role_name, - managed_files=apt_managed, - excluded=apt_excluded, - notes=apt_notes, - ) - dnf_config_snapshot = DnfConfigSnapshot( - role_name=dnf_role_name, - managed_files=dnf_managed, - excluded=dnf_excluded, - notes=dnf_notes, - ) + package_manager_config = PackageManagerConfigCollector( + context, seen_by_role + ).collect() + apt_config_snapshot = package_manager_config.apt_config_snapshot + dnf_config_snapshot = package_manager_config.dnf_config_snapshot # ------------------------- # etc_custom role (unowned /etc files not already attributed elsewhere) @@ -2550,217 +1805,25 @@ def harvest( ) # ------------------------- - # usr_local_custom role (/usr/local/etc + /usr/local/bin scripts) + # usr_local_custom and extra_paths roles # ------------------------- - ul_notes: List[str] = [] - ul_excluded: List[ExcludedFile] = [] - ul_managed: List[ManagedFile] = [] - ul_role_name = "usr_local_custom" - - # Extend the already-captured set with etc_custom. already_all: Set[str] = set(already) for mf in etc_managed: already_all.add(mf.path) - def _scan_usr_local_tree( - root: str, *, require_executable: bool, cap: int, reason: str - ) -> None: - scanned = 0 - if not os.path.isdir(root): - return - role_seen = seen_by_role.setdefault(ul_role_name, set()) - for dirpath, _, filenames in os.walk(root): - for fn in filenames: - path = os.path.join(dirpath, fn) - if path in already_all: - continue - if not os.path.isfile(path) or os.path.islink(path): - continue - try: - owner, group, mode = stat_triplet(path) - except OSError: - ul_excluded.append(ExcludedFile(path=path, reason="unreadable")) - continue + usr_local_custom_snapshot = UsrLocalCustomCollector( + context, + seen_by_role, + already_all, + ).collect() - if require_executable: - try: - if (int(mode, 8) & 0o111) == 0: - continue - except ValueError: - # If mode parsing fails, be conservative and skip. - continue - - if _capture_file( - bundle_dir=bundle_dir, - role_name=ul_role_name, - abs_path=path, - reason=reason, - policy=policy, - path_filter=path_filter, - managed_out=ul_managed, - excluded_out=ul_excluded, - seen_role=role_seen, - seen_global=captured_global, - metadata=(owner, group, mode), - ): - already_all.add(path) - scanned += 1 - if scanned >= cap: - ul_notes.append(f"Reached file cap ({cap}) while scanning {root}.") - return - - # /usr/local/etc: capture all non-binary regular files (filtered by IgnorePolicy) - _scan_usr_local_tree( - "/usr/local/etc", - require_executable=False, - cap=MAX_FILES_CAP, - reason="usr_local_etc_custom", - ) - - # /usr/local/bin: capture executable scripts only (skip non-executable text) - _scan_usr_local_tree( - "/usr/local/bin", - require_executable=True, - cap=MAX_FILES_CAP, - reason="usr_local_bin_script", - ) - - usr_local_custom_snapshot = UsrLocalCustomSnapshot( - role_name=ul_role_name, - managed_files=ul_managed, - excluded=ul_excluded, - notes=ul_notes, - ) - - # ------------------------- - # extra_paths role (user-requested includes) - # ------------------------- - extra_notes: List[str] = [] - extra_excluded: List[ExcludedFile] = [] - extra_managed: List[ManagedFile] = [] - extra_managed_dirs: List[ManagedDir] = [] - extra_dir_seen: Set[str] = set() - - def _walk_and_capture_dirs(root: str) -> None: - root = os.path.normpath(root) - if not root.startswith("/"): - root = "/" + root - if not os.path.isdir(root) or os.path.islink(root): - return - for dirpath, dirnames, _ in os.walk(root, followlinks=False): - if len(extra_managed_dirs) >= MAX_FILES_CAP: - extra_notes.append( - f"Reached directory cap ({MAX_FILES_CAP}) while scanning {root}." - ) - return - dirpath = os.path.normpath(dirpath) - if not dirpath.startswith("/"): - dirpath = "/" + dirpath - if path_filter.is_excluded(dirpath): - # Prune excluded subtrees. - dirnames[:] = [] - continue - if os.path.islink(dirpath) or not os.path.isdir(dirpath): - dirnames[:] = [] - continue - - if dirpath not in extra_dir_seen: - deny = None - deny_dir = getattr(policy, "deny_reason_dir", None) - if callable(deny_dir): - deny = deny_dir(dirpath) - else: - deny = policy.deny_reason(dirpath) - if deny in ("not_regular_file", "not_file", "not_regular"): - deny = None - if not deny: - try: - owner, group, mode = stat_triplet(dirpath) - extra_managed_dirs.append( - ManagedDir( - path=dirpath, - owner=owner, - group=group, - mode=mode, - reason="user_include_dir", - ) - ) - except OSError: - pass - extra_dir_seen.add(dirpath) - - # Prune excluded dirs and symlinks early. - pruned: List[str] = [] - for d in dirnames: - p = os.path.join(dirpath, d) - if os.path.islink(p) or path_filter.is_excluded(p): - continue - pruned.append(d) - dirnames[:] = pruned - - extra_role_name = "extra_paths" - extra_role_seen = seen_by_role.setdefault(extra_role_name, set()) - - include_specs = list(include_paths or []) - exclude_specs = list(exclude_paths or []) - - # If any include pattern points at a directory, capture that directory tree's - # ownership/mode so the manifest can recreate it accurately. - include_pats = path_filter.iter_include_patterns() - for pat in include_pats: - if pat.kind == "prefix": - p = pat.value - if os.path.isdir(p) and not os.path.islink(p): - _walk_and_capture_dirs(p) - elif pat.kind == "glob": - for h in glob.glob(pat.value, recursive=True): - if os.path.isdir(h) and not os.path.islink(h): - _walk_and_capture_dirs(h) - - if include_specs: - extra_notes.append("User include patterns:") - extra_notes.extend([f"- {p}" for p in include_specs]) - if exclude_specs: - extra_notes.append("User exclude patterns:") - extra_notes.extend([f"- {p}" for p in exclude_specs]) - - included_files: List[str] = [] - if include_specs: - files, inc_notes = expand_includes( - path_filter.iter_include_patterns(), - exclude=path_filter, - max_files=MAX_FILES_CAP, - ) - included_files = files - extra_notes.extend(inc_notes) - - for path in included_files: - if path in already_all: - continue - - if _capture_file( - bundle_dir=bundle_dir, - role_name=extra_role_name, - abs_path=path, - reason="user_include", - policy=policy, - path_filter=path_filter, - managed_out=extra_managed, - excluded_out=extra_excluded, - seen_role=extra_role_seen, - seen_global=captured_global, - ): - already_all.add(path) - - extra_paths_snapshot = ExtraPathsSnapshot( - role_name=extra_role_name, - include_patterns=include_specs, - exclude_patterns=exclude_specs, - managed_dirs=extra_managed_dirs, - managed_files=extra_managed, - excluded=extra_excluded, - notes=extra_notes, - ) + extra_paths_snapshot = ExtraPathsCollector( + context, + seen_by_role, + already_all, + include_paths=include_paths, + exclude_paths=exclude_paths, + ).collect() # ------------------------- # Inventory: packages (SBOM-ish) @@ -2904,7 +1967,4 @@ def harvest( }, } - state_path = os.path.join(bundle_dir, "state.json") - with open(state_path, "w", encoding="utf-8") as f: - json.dump(state, f, indent=2, sort_keys=True) - return state_path + return str(write_state(bundle_dir, state)) diff --git a/enroll/harvest_collectors/__init__.py b/enroll/harvest_collectors/__init__.py new file mode 100644 index 0000000..7512081 --- /dev/null +++ b/enroll/harvest_collectors/__init__.py @@ -0,0 +1,27 @@ +from .context import HarvestCollector, HarvestContext +from .cron_logrotate import CronLogrotateCollection, CronLogrotateCollector +from .package_manager import ( + PackageManagerConfigCollection, + PackageManagerConfigCollector, +) +from .paths import ExtraPathsCollector, UsrLocalCustomCollector +from .runtime import RuntimeStateCollection, RuntimeStateCollector +from .services import ServicePackageCollection, ServicePackageCollector +from .users import UsersCollection, UsersCollector + +__all__ = [ + "CronLogrotateCollection", + "CronLogrotateCollector", + "ExtraPathsCollector", + "HarvestCollector", + "HarvestContext", + "PackageManagerConfigCollection", + "PackageManagerConfigCollector", + "RuntimeStateCollection", + "RuntimeStateCollector", + "ServicePackageCollection", + "ServicePackageCollector", + "UsersCollection", + "UsersCollector", + "UsrLocalCustomCollector", +] diff --git a/enroll/harvest_collectors/context.py b/enroll/harvest_collectors/context.py new file mode 100644 index 0000000..7c5b5d9 --- /dev/null +++ b/enroll/harvest_collectors/context.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict, List, Set + +from ..ignore import IgnorePolicy +from ..pathfilter import PathFilter + + +@dataclass +class HarvestContext: + """Shared context passed to feature collectors.""" + + bundle_dir: str + policy: IgnorePolicy + path_filter: PathFilter + platform: Dict[str, Any] + backend: Any + installed_pkgs: Dict[str, Any] + installed_names: Set[str] + owned_etc: Set[str] + etc_owner_map: Dict[str, str] + topdir_to_pkgs: Dict[str, Set[str]] + pkg_to_etc_paths: Dict[str, List[str]] + captured_global: Set[str] + + +class HarvestCollector: + """Base class for harvest feature collectors.""" + + def __init__(self, context: HarvestContext) -> None: + self.context = context diff --git a/enroll/harvest_collectors/cron_logrotate.py b/enroll/harvest_collectors/cron_logrotate.py new file mode 100644 index 0000000..c66fd3d --- /dev/null +++ b/enroll/harvest_collectors/cron_logrotate.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import List, Optional, Set + +from .. import harvest as h +from ..harvest import ExcludedFile, ManagedFile, PackageSnapshot +from .context import HarvestCollector + + +def _pick_installed(installed_names: Set[str], candidates: List[str]) -> Optional[str]: + for candidate in candidates: + if candidate in installed_names: + return candidate + return None + + +def _is_cron_path(path: str) -> bool: + return ( + path == "/etc/crontab" + or path == "/etc/anacrontab" + or path in ("/etc/cron.allow", "/etc/cron.deny") + or path.startswith("/etc/cron.") + or path.startswith("/etc/cron.d/") + or path.startswith("/etc/anacron/") + or path.startswith("/var/spool/cron/") + or path.startswith("/var/spool/crontabs/") + or path.startswith("/var/spool/anacron/") + ) + + +def _is_logrotate_path(path: str) -> bool: + return path == "/etc/logrotate.conf" or path.startswith("/etc/logrotate.d/") + + +_CRON_CAPTURE_GLOBS = [ + "/etc/crontab", + "/etc/cron.d/*", + "/etc/cron.hourly/*", + "/etc/cron.daily/*", + "/etc/cron.weekly/*", + "/etc/cron.monthly/*", + "/etc/cron.allow", + "/etc/cron.deny", + "/etc/anacrontab", + "/etc/anacron/*", + # user crontabs / spool state + "/var/spool/cron/*", + "/var/spool/cron/crontabs/*", + "/var/spool/crontabs/*", + "/var/spool/anacron/*", +] + +_LOGROTATE_CAPTURE_GLOBS = [ + "/etc/logrotate.conf", + "/etc/logrotate.d/*", +] + + +@dataclass +class CronLogrotateCollection: + cron_pkg: Optional[str] + logrotate_pkg: Optional[str] + cron_snapshot: Optional[PackageSnapshot] + logrotate_snapshot: Optional[PackageSnapshot] + + +class CronLogrotateCollector(HarvestCollector): + """Collect dedicated cron/logrotate package roles before general packages.""" + + cron_role_name = "cron" + logrotate_role_name = "logrotate" + + def collect(self) -> CronLogrotateCollection: + cron_pkg = _pick_installed( + self.context.installed_names, + ["cron", "cronie", "cronie-anacron", "vixie-cron", "fcron"], + ) + logrotate_pkg = _pick_installed(self.context.installed_names, ["logrotate"]) + + cron_snapshot = self._collect_cron_snapshot(cron_pkg) if cron_pkg else None + logrotate_snapshot = ( + self._collect_logrotate_snapshot(logrotate_pkg) if logrotate_pkg else None + ) + return CronLogrotateCollection( + cron_pkg=cron_pkg, + logrotate_pkg=logrotate_pkg, + cron_snapshot=cron_snapshot, + logrotate_snapshot=logrotate_snapshot, + ) + + def _collect_cron_snapshot(self, cron_pkg: str) -> PackageSnapshot: + managed: List[ManagedFile] = [] + excluded: List[ExcludedFile] = [] + notes: List[str] = [] + seen: Set[str] = set() + + for spec in _CRON_CAPTURE_GLOBS: + for path in h._iter_matching_files(spec): + if not os.path.isfile(path) or os.path.islink(path): + continue + h._capture_file( + bundle_dir=self.context.bundle_dir, + role_name=self.cron_role_name, + abs_path=path, + reason="system_cron", + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=managed, + excluded_out=excluded, + seen_role=seen, + seen_global=self.context.captured_global, + ) + + return PackageSnapshot( + package=cron_pkg, + role_name=self.cron_role_name, + section=h._package_section_from_installations( + self.context.installed_pkgs.get(cron_pkg, []) + ), + managed_files=managed, + excluded=excluded, + notes=notes, + ) + + def _collect_logrotate_snapshot(self, logrotate_pkg: str) -> PackageSnapshot: + managed: List[ManagedFile] = [] + excluded: List[ExcludedFile] = [] + notes: List[str] = [] + seen: Set[str] = set() + + for spec in _LOGROTATE_CAPTURE_GLOBS: + for path in h._iter_matching_files(spec): + if not os.path.isfile(path) or os.path.islink(path): + continue + h._capture_file( + bundle_dir=self.context.bundle_dir, + role_name=self.logrotate_role_name, + abs_path=path, + reason="system_logrotate", + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=managed, + excluded_out=excluded, + seen_role=seen, + seen_global=self.context.captured_global, + ) + + return PackageSnapshot( + package=logrotate_pkg, + role_name=self.logrotate_role_name, + section=h._package_section_from_installations( + self.context.installed_pkgs.get(logrotate_pkg, []) + ), + managed_files=managed, + excluded=excluded, + notes=notes, + ) diff --git a/enroll/harvest_collectors/package_manager.py b/enroll/harvest_collectors/package_manager.py new file mode 100644 index 0000000..09c270b --- /dev/null +++ b/enroll/harvest_collectors/package_manager.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict, List, Set + +from .. import harvest as h +from ..harvest import AptConfigSnapshot, DnfConfigSnapshot, ExcludedFile, ManagedFile +from .context import HarvestCollector, HarvestContext + + +@dataclass +class PackageManagerConfigCollection: + apt_config_snapshot: AptConfigSnapshot + dnf_config_snapshot: DnfConfigSnapshot + + +class PackageManagerConfigCollector(HarvestCollector): + """Collect package-manager configuration into existing role snapshots.""" + + def __init__( + self, context: HarvestContext, seen_by_role: Dict[str, Set[str]] + ) -> None: + super().__init__(context) + self.seen_by_role = seen_by_role + + def collect(self) -> PackageManagerConfigCollection: + apt_notes: List[str] = [] + apt_excluded: List[ExcludedFile] = [] + apt_managed: List[ManagedFile] = [] + dnf_notes: List[str] = [] + dnf_excluded: List[ExcludedFile] = [] + dnf_managed: List[ManagedFile] = [] + + apt_role_name = "apt_config" + dnf_role_name = "dnf_config" + + if self.context.backend.name == "dpkg": + apt_role_seen = self.seen_by_role.setdefault(apt_role_name, set()) + for path, reason in h._iter_apt_capture_paths(): + h._capture_file( + bundle_dir=self.context.bundle_dir, + role_name=apt_role_name, + abs_path=path, + reason=reason, + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=apt_managed, + excluded_out=apt_excluded, + seen_role=apt_role_seen, + seen_global=self.context.captured_global, + ) + elif self.context.backend.name == "rpm": + dnf_role_seen = self.seen_by_role.setdefault(dnf_role_name, set()) + for path, reason in h._iter_dnf_capture_paths(): + h._capture_file( + bundle_dir=self.context.bundle_dir, + role_name=dnf_role_name, + abs_path=path, + reason=reason, + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=dnf_managed, + excluded_out=dnf_excluded, + seen_role=dnf_role_seen, + seen_global=self.context.captured_global, + ) + + return PackageManagerConfigCollection( + apt_config_snapshot=AptConfigSnapshot( + role_name=apt_role_name, + managed_files=apt_managed, + excluded=apt_excluded, + notes=apt_notes, + ), + dnf_config_snapshot=DnfConfigSnapshot( + role_name=dnf_role_name, + managed_files=dnf_managed, + excluded=dnf_excluded, + notes=dnf_notes, + ), + ) diff --git a/enroll/harvest_collectors/paths.py b/enroll/harvest_collectors/paths.py new file mode 100644 index 0000000..af9fdbe --- /dev/null +++ b/enroll/harvest_collectors/paths.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +import glob +import os +from typing import Dict, List, Optional, Set + +from .. import harvest as h +from ..harvest import ( + ExcludedFile, + ExtraPathsSnapshot, + ManagedDir, + ManagedFile, + UsrLocalCustomSnapshot, +) +from ..pathfilter import expand_includes +from .context import HarvestCollector, HarvestContext + + +class UsrLocalCustomCollector(HarvestCollector): + """Collect selected /usr/local state into the usr_local_custom role.""" + + role_name = "usr_local_custom" + + def __init__( + self, + context: HarvestContext, + seen_by_role: Dict[str, Set[str]], + already_all: Set[str], + ) -> None: + super().__init__(context) + self.seen_by_role = seen_by_role + self.already_all = already_all + self.notes: List[str] = [] + self.excluded: List[ExcludedFile] = [] + self.managed: List[ManagedFile] = [] + + def collect(self) -> UsrLocalCustomSnapshot: + self._scan_tree( + "/usr/local/etc", + require_executable=False, + cap=h.MAX_FILES_CAP, + reason="usr_local_etc_custom", + ) + self._scan_tree( + "/usr/local/bin", + require_executable=True, + cap=h.MAX_FILES_CAP, + reason="usr_local_bin_script", + ) + return UsrLocalCustomSnapshot( + role_name=self.role_name, + managed_files=self.managed, + excluded=self.excluded, + notes=self.notes, + ) + + def _scan_tree( + self, + root: str, + *, + require_executable: bool, + cap: int, + reason: str, + ) -> None: + scanned = 0 + if not os.path.isdir(root): + return + role_seen = self.seen_by_role.setdefault(self.role_name, set()) + for dirpath, _, filenames in os.walk(root): + for filename in filenames: + path = os.path.join(dirpath, filename) + if path in self.already_all: + continue + if not os.path.isfile(path) or os.path.islink(path): + continue + try: + owner, group, mode = h.stat_triplet(path) + except OSError: + self.excluded.append(ExcludedFile(path=path, reason="unreadable")) + continue + + if require_executable: + try: + if (int(mode, 8) & 0o111) == 0: + continue + except ValueError: + continue + + if h._capture_file( + bundle_dir=self.context.bundle_dir, + role_name=self.role_name, + abs_path=path, + reason=reason, + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=self.managed, + excluded_out=self.excluded, + seen_role=role_seen, + seen_global=self.context.captured_global, + metadata=(owner, group, mode), + ): + self.already_all.add(path) + scanned += 1 + if scanned >= cap: + self.notes.append( + f"Reached file cap ({cap}) while scanning {root}." + ) + return + + +class ExtraPathsCollector(HarvestCollector): + """Collect user-requested include/exclude paths into extra_paths.""" + + role_name = "extra_paths" + + def __init__( + self, + context: HarvestContext, + seen_by_role: Dict[str, Set[str]], + already_all: Set[str], + *, + include_paths: Optional[List[str]] = None, + exclude_paths: Optional[List[str]] = None, + ) -> None: + super().__init__(context) + self.seen_by_role = seen_by_role + self.already_all = already_all + self.include_specs = list(include_paths or []) + self.exclude_specs = list(exclude_paths or []) + self.notes: List[str] = [] + self.excluded: List[ExcludedFile] = [] + self.managed: List[ManagedFile] = [] + self.managed_dirs: List[ManagedDir] = [] + self.dir_seen: Set[str] = set() + + def collect(self) -> ExtraPathsSnapshot: + self._collect_included_dirs() + if self.include_specs: + self.notes.append("User include patterns:") + self.notes.extend([f"- {p}" for p in self.include_specs]) + if self.exclude_specs: + self.notes.append("User exclude patterns:") + self.notes.extend([f"- {p}" for p in self.exclude_specs]) + + included_files: List[str] = [] + if self.include_specs: + files, inc_notes = expand_includes( + self.context.path_filter.iter_include_patterns(), + exclude=self.context.path_filter, + max_files=h.MAX_FILES_CAP, + ) + included_files = files + self.notes.extend(inc_notes) + + role_seen = self.seen_by_role.setdefault(self.role_name, set()) + for path in included_files: + if path in self.already_all: + continue + if h._capture_file( + bundle_dir=self.context.bundle_dir, + role_name=self.role_name, + abs_path=path, + reason="user_include", + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=self.managed, + excluded_out=self.excluded, + seen_role=role_seen, + seen_global=self.context.captured_global, + ): + self.already_all.add(path) + + return ExtraPathsSnapshot( + role_name=self.role_name, + include_patterns=self.include_specs, + exclude_patterns=self.exclude_specs, + managed_dirs=self.managed_dirs, + managed_files=self.managed, + excluded=self.excluded, + notes=self.notes, + ) + + def _collect_included_dirs(self) -> None: + for pat in self.context.path_filter.iter_include_patterns(): + if pat.kind == "prefix": + path = pat.value + if os.path.isdir(path) and not os.path.islink(path): + self._walk_and_capture_dirs(path) + elif pat.kind == "glob": + for hit in glob.glob(pat.value, recursive=True): + if os.path.isdir(hit) and not os.path.islink(hit): + self._walk_and_capture_dirs(hit) + + def _walk_and_capture_dirs(self, root: str) -> None: + root = os.path.normpath(root) + if not root.startswith("/"): + root = "/" + root + if not os.path.isdir(root) or os.path.islink(root): + return + for dirpath, dirnames, _ in os.walk(root, followlinks=False): + if len(self.managed_dirs) >= h.MAX_FILES_CAP: + self.notes.append( + f"Reached directory cap ({h.MAX_FILES_CAP}) while scanning {root}." + ) + return + dirpath = os.path.normpath(dirpath) + if not dirpath.startswith("/"): + dirpath = "/" + dirpath + if self.context.path_filter.is_excluded(dirpath): + dirnames[:] = [] + continue + if os.path.islink(dirpath) or not os.path.isdir(dirpath): + dirnames[:] = [] + continue + + if dirpath not in self.dir_seen: + deny = None + deny_dir = getattr(self.context.policy, "deny_reason_dir", None) + if callable(deny_dir): + deny = deny_dir(dirpath) + else: + deny = self.context.policy.deny_reason(dirpath) + if deny in ("not_regular_file", "not_file", "not_regular"): + deny = None + if not deny: + try: + owner, group, mode = h.stat_triplet(dirpath) + self.managed_dirs.append( + ManagedDir( + path=dirpath, + owner=owner, + group=group, + mode=mode, + reason="user_include_dir", + ) + ) + except OSError: + pass + self.dir_seen.add(dirpath) + + pruned: List[str] = [] + for dirname in dirnames: + path = os.path.join(dirpath, dirname) + if os.path.islink(path) or self.context.path_filter.is_excluded(path): + continue + pruned.append(dirname) + dirnames[:] = pruned diff --git a/enroll/harvest_collectors/runtime.py b/enroll/harvest_collectors/runtime.py new file mode 100644 index 0000000..2d1eafa --- /dev/null +++ b/enroll/harvest_collectors/runtime.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import List, Optional + +from .. import harvest as h +from ..harvest import FirewallRuntimeSnapshot, SysctlSnapshot +from .context import HarvestCollector, HarvestContext + + +@dataclass +class RuntimeStateCollection: + firewall_runtime_snapshot: FirewallRuntimeSnapshot + sysctl_snapshot: SysctlSnapshot + + +class RuntimeStateCollector(HarvestCollector): + """Collect root-only live runtime state that has generated roles.""" + + def __init__( + self, + context: HarvestContext, + *, + persistent_ipset_files: Optional[List[str]] = None, + persistent_iptables_v4_files: Optional[List[str]] = None, + persistent_iptables_v6_files: Optional[List[str]] = None, + ) -> None: + super().__init__(context) + self.persistent_ipset_files = persistent_ipset_files or [] + self.persistent_iptables_v4_files = persistent_iptables_v4_files or [] + self.persistent_iptables_v6_files = persistent_iptables_v6_files or [] + + def collect(self) -> RuntimeStateCollection: + running_as_root = not hasattr(os, "geteuid") or os.geteuid() == 0 + if not running_as_root: + return RuntimeStateCollection( + firewall_runtime_snapshot=FirewallRuntimeSnapshot( + role_name="firewall_runtime", + notes=[ + "Live ipset/iptables runtime capture skipped because harvest " + "is not running as root." + ], + ), + sysctl_snapshot=SysctlSnapshot( + role_name="sysctl", + notes=[ + "Live sysctl runtime capture skipped because harvest is not " + "running as root." + ], + ), + ) + + firewall_runtime_snapshot = h._collect_firewall_runtime_snapshot( + self.context.bundle_dir, + persistent_ipset_files=self.persistent_ipset_files, + persistent_iptables_v4_files=self.persistent_iptables_v4_files, + persistent_iptables_v6_files=self.persistent_iptables_v6_files, + ) + sysctl_snapshot = h._collect_sysctl_snapshot(self.context.bundle_dir) + return RuntimeStateCollection( + firewall_runtime_snapshot=firewall_runtime_snapshot, + sysctl_snapshot=sysctl_snapshot, + ) diff --git a/enroll/harvest_collectors/services.py b/enroll/harvest_collectors/services.py new file mode 100644 index 0000000..a962fe3 --- /dev/null +++ b/enroll/harvest_collectors/services.py @@ -0,0 +1,525 @@ +from __future__ import annotations + +import glob +import os +from dataclasses import dataclass +from typing import Dict, List, Optional, Set + +from .. import harvest as h +from ..harvest import ExcludedFile, ManagedFile, PackageSnapshot, ServiceSnapshot +from ..systemd import UnitQueryError +from .context import HarvestCollector, HarvestContext +from .cron_logrotate import CronLogrotateCollector, _is_cron_path, _is_logrotate_path + + +@dataclass +class ServicePackageCollection: + service_snaps: List[ServiceSnapshot] + pkg_snaps: List[PackageSnapshot] + manual_pkgs: List[str] + simple_packages: List[str] + manual_pkgs_skipped: List[str] + service_role_aliases: Dict[str, Set[str]] + seen_by_role: Dict[str, Set[str]] + + +class ServicePackageCollector(HarvestCollector): + """Collect service-attributed and manually-installed package snapshots.""" + + def __init__( + self, + context: HarvestContext, + *, + cron_snapshot: Optional[PackageSnapshot] = None, + logrotate_snapshot: Optional[PackageSnapshot] = None, + cron_pkg: Optional[str] = None, + logrotate_pkg: Optional[str] = None, + ) -> None: + super().__init__(context) + self.cron_snapshot = cron_snapshot + self.logrotate_snapshot = logrotate_snapshot + self.cron_pkg = cron_pkg + self.logrotate_pkg = logrotate_pkg + self.service_role_aliases: Dict[str, Set[str]] = {} + self.seen_by_role: Dict[str, Set[str]] = {} + self.managed_by_role: Dict[str, List[ManagedFile]] = {} + self.excluded_by_role: Dict[str, List[ExcludedFile]] = {} + + def collect(self) -> ServicePackageCollection: + service_snaps, timer_extra_by_pkg = self._collect_service_snapshots() + pkg_snaps, manual_pkgs, simple_packages, manual_pkgs_skipped = ( + self._collect_package_snapshots( + service_snaps, + timer_extra_by_pkg, + ) + ) + self._capture_common_enabled_symlinks(service_snaps, pkg_snaps) + return ServicePackageCollection( + service_snaps=service_snaps, + pkg_snaps=pkg_snaps, + manual_pkgs=manual_pkgs, + simple_packages=simple_packages, + manual_pkgs_skipped=manual_pkgs_skipped, + service_role_aliases=self.service_role_aliases, + seen_by_role=self.seen_by_role, + ) + + def _collect_service_snapshots( + self, + ) -> tuple[List[ServiceSnapshot], Dict[str, List[str]]]: + backend = self.context.backend + service_snaps: List[ServiceSnapshot] = [] + + enabled_services = h.list_enabled_services() + if self.cron_snapshot is not None or self.logrotate_snapshot is not None: + blocked_roles = set() + if self.cron_snapshot is not None: + blocked_roles.add(CronLogrotateCollector.cron_role_name) + if self.logrotate_snapshot is not None: + blocked_roles.add(CronLogrotateCollector.logrotate_role_name) + enabled_services = [ + u + for u in enabled_services + if h._role_name_from_unit(u) not in blocked_roles + ] + enabled_set = set(enabled_services) + + def service_sort_key(unit: str) -> tuple[int, str, str]: + base = unit.removesuffix(".service") + base = base.split("@", 1)[0] + return (base.count("-"), base.lower(), unit.lower()) + + def parent_service_unit(unit: str) -> Optional[str]: + if not unit.endswith(".service"): + return None + base = unit.removesuffix(".service") + base = base.split("@", 1)[0] + parts = base.split("-") + for i in range(len(parts) - 1, 0, -1): + cand = "-".join(parts[:i]) + ".service" + if cand in enabled_set: + return cand + return None + + parent_unit_for = { + u: pu for u in enabled_services if (pu := parent_service_unit(u)) + } + + for unit in sorted(enabled_services, key=service_sort_key): + role = h._role_name_from_unit(unit) + parent_unit = parent_unit_for.get(unit) + parent_role = h._role_name_from_unit(parent_unit) if parent_unit else None + + try: + ui = h.get_unit_info(unit) + except UnitQueryError as e: + self.service_role_aliases.setdefault( + role, h._hint_names(unit, set()) | {role} + ) + self.seen_by_role.setdefault(role, set()) + managed = self.managed_by_role.setdefault(role, []) + excluded = self.excluded_by_role.setdefault(role, []) + service_snaps.append( + ServiceSnapshot( + unit=unit, + role_name=role, + packages=[], + active_state=None, + sub_state=None, + unit_file_state=None, + condition_result=None, + managed_files=managed, + excluded=excluded, + notes=[str(e)], + ) + ) + continue + + pkgs: Set[str] = set() + notes: List[str] = [] + excluded = self.excluded_by_role.setdefault(role, []) + managed = self.managed_by_role.setdefault(role, []) + candidates: Dict[str, str] = {} + + if ui.fragment_path: + p = backend.owner_of_path(ui.fragment_path) + if p: + pkgs.add(p) + + for exe in ui.exec_paths: + p = backend.owner_of_path(exe) + if p: + pkgs.add(p) + + for pth in ui.dropin_paths: + if pth.startswith("/etc/"): + candidates[pth] = "systemd_dropin" + + for env_file in ui.env_files: + env_file = env_file.lstrip("-") + if any(ch in env_file for ch in "*?["): + for g in glob.glob(env_file): + if g.startswith("/etc/") and os.path.isfile(g): + candidates[g] = "systemd_envfile" + elif env_file.startswith("/etc/") and os.path.isfile(env_file): + candidates[env_file] = "systemd_envfile" + + hints = h._hint_names(unit, pkgs) + h._add_pkgs_from_etc_topdirs(hints, self.context.topdir_to_pkgs, pkgs) + self.service_role_aliases[role] = set(hints) | set(pkgs) | {role} + + for sp in h._maybe_add_specific_paths(hints, backend): + if not os.path.exists(sp): + continue + if sp in self.context.etc_owner_map: + pkgs.add(self.context.etc_owner_map[sp]) + else: + candidates.setdefault(sp, "custom_specific_path") + + for pkg in sorted(pkgs): + etc_paths = self.context.pkg_to_etc_paths.get(pkg, []) + for path, reason in backend.modified_paths(pkg, etc_paths).items(): + if not os.path.isfile(path) or os.path.islink(path): + continue + if self.cron_snapshot is not None and _is_cron_path(path): + continue + if self.logrotate_snapshot is not None and _is_logrotate_path(path): + continue + if backend.is_pkg_config_path(path): + continue + candidates.setdefault(path, reason) + + any_roots: List[str] = [] + confish_roots: List[str] = [] + for hint in hints: + roots_for_hint = [f"/etc/{hint}", f"/etc/{hint}.d"] + if hint in h.SHARED_ETC_TOPDIRS: + confish_roots.extend(roots_for_hint) + else: + any_roots.extend(roots_for_hint) + + found: List[str] = [] + found.extend( + h._scan_unowned_under_roots( + any_roots, + self.context.owned_etc, + limit=h.MAX_UNOWNED_FILES_PER_ROLE, + confish_only=False, + ) + ) + if len(found) < h.MAX_UNOWNED_FILES_PER_ROLE: + found.extend( + h._scan_unowned_under_roots( + confish_roots, + self.context.owned_etc, + limit=h.MAX_UNOWNED_FILES_PER_ROLE - len(found), + confish_only=True, + ) + ) + for pth in found: + candidates.setdefault(pth, "custom_unowned") + + if not pkgs and not candidates: + notes.append( + "No packages or /etc candidates detected (unexpected for enabled service)." + ) + + for path, reason in sorted(candidates.items()): + dest_role = role + if ( + parent_role + and path.startswith("/etc/") + and reason not in ("systemd_dropin", "systemd_envfile") + ): + dest_role = parent_role + + dest_managed = self.managed_by_role.setdefault(dest_role, []) + dest_excluded = self.excluded_by_role.setdefault(dest_role, []) + dest_seen = self.seen_by_role.setdefault(dest_role, set()) + h._capture_file( + bundle_dir=self.context.bundle_dir, + role_name=dest_role, + abs_path=path, + reason=reason, + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=dest_managed, + excluded_out=dest_excluded, + seen_role=dest_seen, + seen_global=self.context.captured_global, + ) + + service_snaps.append( + ServiceSnapshot( + unit=unit, + role_name=role, + packages=sorted(pkgs), + active_state=ui.active_state, + sub_state=ui.sub_state, + unit_file_state=ui.unit_file_state, + condition_result=ui.condition_result, + managed_files=managed, + excluded=excluded, + notes=notes, + ) + ) + + timer_extra_by_pkg = self._collect_timer_overrides(service_snaps) + return service_snaps, timer_extra_by_pkg + + def _collect_timer_overrides( + self, + service_snaps: List[ServiceSnapshot], + ) -> Dict[str, List[str]]: + backend = self.context.backend + timer_extra_by_pkg: Dict[str, List[str]] = {} + try: + enabled_timers = h.list_enabled_timers() + except Exception: + enabled_timers = [] + + service_snap_by_unit = {s.unit: s for s in service_snaps} + + for timer in sorted(enabled_timers): + try: + ti = h.get_timer_info(timer) + except Exception: # nosec + continue + + timer_paths: List[str] = [] + for pth in [ti.fragment_path, *ti.dropin_paths, *ti.env_files]: + if not pth: + continue + if not pth.startswith("/etc/"): + continue + if os.path.islink(pth) or not os.path.isfile(pth): + continue + timer_paths.append(pth) + + if not timer_paths: + continue + + snap = ( + service_snap_by_unit.get(ti.trigger_unit) if ti.trigger_unit else None + ) + if snap is not None: + role_seen = self.seen_by_role.setdefault(snap.role_name, set()) + for path in timer_paths: + h._capture_file( + bundle_dir=self.context.bundle_dir, + role_name=snap.role_name, + abs_path=path, + reason="related_timer", + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=snap.managed_files, + excluded_out=snap.excluded, + seen_role=role_seen, + seen_global=self.context.captured_global, + ) + continue + + pkgs: Set[str] = set() + if ti.fragment_path: + p = backend.owner_of_path(ti.fragment_path) + if p: + pkgs.add(p) + if ti.trigger_unit and ti.trigger_unit.endswith(".service"): + try: + ui = h.get_unit_info(ti.trigger_unit) + if ui.fragment_path: + p = backend.owner_of_path(ui.fragment_path) + if p: + pkgs.add(p) + for exe in ui.exec_paths: + p = backend.owner_of_path(exe) + if p: + pkgs.add(p) + except Exception: # nosec + pass + + for pkg in pkgs: + timer_extra_by_pkg.setdefault(pkg, []).extend(timer_paths) + + return timer_extra_by_pkg + + def _collect_package_snapshots( + self, + service_snaps: List[ServiceSnapshot], + timer_extra_by_pkg: Dict[str, List[str]], + ) -> tuple[List[PackageSnapshot], List[str], List[str], List[str]]: + backend = self.context.backend + manual_pkgs = backend.list_manual_packages() + covered_by_services: Set[str] = set() + for snap in service_snaps: + covered_by_services.update(snap.packages) + + manual_pkgs_skipped: List[str] = [] + pkg_snaps: List[PackageSnapshot] = [] + simple_packages: List[str] = [] + + if self.cron_snapshot is not None: + pkg_snaps.append(self.cron_snapshot) + if self.logrotate_snapshot is not None: + pkg_snaps.append(self.logrotate_snapshot) + + for pkg in sorted(manual_pkgs): + if pkg in covered_by_services: + manual_pkgs_skipped.append(pkg) + continue + if self.cron_snapshot is not None and pkg == self.cron_pkg: + manual_pkgs_skipped.append(pkg) + continue + if self.logrotate_snapshot is not None and pkg == self.logrotate_pkg: + manual_pkgs_skipped.append(pkg) + continue + + role = h._role_name_from_pkg(pkg) + notes: List[str] = [] + excluded: List[ExcludedFile] = [] + managed: List[ManagedFile] = [] + candidates: Dict[str, str] = {} + + for tpath in timer_extra_by_pkg.get(pkg, []): + candidates.setdefault(tpath, "related_timer") + + etc_paths = self.context.pkg_to_etc_paths.get(pkg, []) + for path, reason in backend.modified_paths(pkg, etc_paths).items(): + if not os.path.isfile(path) or os.path.islink(path): + continue + if self.cron_snapshot is not None and _is_cron_path(path): + continue + if self.logrotate_snapshot is not None and _is_logrotate_path(path): + continue + if backend.is_pkg_config_path(path): + continue + candidates.setdefault(path, reason) + + topdirs = h._topdirs_for_package(pkg, self.context.pkg_to_etc_paths) + roots: List[str] = [] + for topdir in sorted(topdirs): + if topdir in h.SHARED_ETC_TOPDIRS: + continue + if backend.is_pkg_config_path( + f"/etc/{topdir}/" + ) or backend.is_pkg_config_path(f"/etc/{topdir}"): + continue + roots.extend([f"/etc/{topdir}", f"/etc/{topdir}.d"]) + roots.extend(h._maybe_add_specific_paths(set(topdirs), backend)) + + for pth in h._scan_unowned_under_roots( + [r for r in roots if os.path.isdir(r)], + self.context.owned_etc, + confish_only=False, + ): + candidates.setdefault(pth, "custom_unowned") + + for root in roots: + if os.path.isfile(root) and not os.path.islink(root): + if root not in self.context.owned_etc and h._is_confish(root): + candidates.setdefault(root, "custom_specific_path") + + role_seen = self.seen_by_role.setdefault(role, set()) + for path, reason in sorted(candidates.items()): + h._capture_file( + bundle_dir=self.context.bundle_dir, + role_name=role, + abs_path=path, + reason=reason, + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=managed, + excluded_out=excluded, + seen_role=role_seen, + seen_global=self.context.captured_global, + ) + + has_config = bool(managed or excluded) + if not has_config: + notes.append( + "No changed or custom configuration detected for this package." + ) + simple_packages.append(pkg) + + pkg_snaps.append( + PackageSnapshot( + package=pkg, + role_name=role, + section=h._package_section_from_installations( + self.context.installed_pkgs.get(pkg, []) + ), + managed_files=managed, + managed_links=[], + excluded=excluded, + notes=notes, + has_config=has_config, + ) + ) + + return pkg_snaps, manual_pkgs, simple_packages, manual_pkgs_skipped + + def _find_role_snapshot( + self, + role_name: str, + service_snaps: List[ServiceSnapshot], + pkg_snaps: List[PackageSnapshot], + ): + for snap in service_snaps: + if snap.role_name == role_name: + return snap + for snap in pkg_snaps: + if snap.role_name == role_name: + return snap + return None + + def _capture_enabled_symlinks_for_role( + self, + role_name: str, + dirs: List[str], + service_snaps: List[ServiceSnapshot], + pkg_snaps: List[PackageSnapshot], + ) -> None: + snap = self._find_role_snapshot(role_name, service_snaps, pkg_snaps) + if snap is None: + return + + role_seen = self.seen_by_role.setdefault(role_name, set()) + for directory in dirs: + if not os.path.isdir(directory): + continue + for pth in sorted(glob.glob(os.path.join(directory, "*"))): + if not os.path.islink(pth): + continue + h._capture_link( + role_name=role_name, + abs_path=pth, + reason="enabled_symlink", + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=snap.managed_links, + excluded_out=snap.excluded, + seen_role=role_seen, + seen_global=self.context.captured_global, + ) + + def _capture_common_enabled_symlinks( + self, + service_snaps: List[ServiceSnapshot], + pkg_snaps: List[PackageSnapshot], + ) -> None: + self._capture_enabled_symlinks_for_role( + "nginx", + ["/etc/nginx/modules-enabled", "/etc/nginx/sites-enabled"], + service_snaps, + pkg_snaps, + ) + self._capture_enabled_symlinks_for_role( + "apache2", + [ + "/etc/apache2/conf-enabled", + "/etc/apache2/mods-enabled", + "/etc/apache2/sites-enabled", + ], + service_snaps, + pkg_snaps, + ) diff --git a/enroll/harvest_collectors/users.py b/enroll/harvest_collectors/users.py new file mode 100644 index 0000000..7640d1f --- /dev/null +++ b/enroll/harvest_collectors/users.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any, Dict, List, Set + +from .. import harvest as h +from ..harvest import ( + ExcludedFile, + FlatpakSnapshot, + ManagedFile, + SnapSnapshot, + UsersSnapshot, +) +from .context import HarvestCollector, HarvestContext + + +@dataclass +class UsersCollection: + users_snapshot: UsersSnapshot + flatpak_snapshot: FlatpakSnapshot + snap_snapshot: SnapSnapshot + + +class UsersCollector(HarvestCollector): + """Collect non-system users plus system/user Flatpak and Snap facts.""" + + def __init__( + self, context: HarvestContext, seen_by_role: Dict[str, Set[str]] + ) -> None: + super().__init__(context) + self.seen_by_role = seen_by_role + + def collect(self) -> UsersCollection: + users_notes: List[str] = [] + users_excluded: List[ExcludedFile] = [] + users_managed: List[ManagedFile] = [] + users_list: List[dict] = [] + + try: + user_records = h.collect_non_system_users() + except Exception as e: + user_records = [] + users_notes.append(f"Failed to enumerate users: {e!r}") + + # Detect system-wide Flatpaks/Snaps and configured Flatpak remotes. + from ..accounts import ( + find_system_flatpak_remotes, + find_system_flatpaks, + find_system_snaps, + find_user_flatpak_remotes, + ) + + system_flatpaks = [asdict(f) for f in find_system_flatpaks()] + system_snaps = [asdict(s) for s in find_system_snaps()] + system_flatpak_remotes = [asdict(r) for r in find_system_flatpak_remotes()] + flatpak_notes: List[str] = [] + snap_notes: List[str] = [] + if system_flatpaks: + flatpak_notes.append( + "System-wide flatpaks detected: " + + ", ".join(str(f.get("name")) for f in system_flatpaks) + ) + if system_snaps: + snap_notes.append( + "System-wide snaps detected: " + + ", ".join(str(s.get("name")) for s in system_snaps) + ) + + users_role_name = "users" + users_role_seen = self.seen_by_role.setdefault(users_role_name, set()) + + skel_dir = "/etc/skel" + auto_capture_user_dotfiles = bool( + getattr(self.context.policy, "dangerous", False) + ) + if user_records and not auto_capture_user_dotfiles: + users_notes.append( + "User shell dotfiles were not auto-harvested because --dangerous was not set; " + "use --dangerous for automatic shell-dotfile capture, or targeted " + "--include-path patterns for safe-mode review." + ) + + user_flatpaks_map: Dict[str, List[Dict[str, Any]]] = {} + user_flatpak_remotes: List[Dict[str, Any]] = [] + + for user in user_records: + users_list.append( + { + "name": user.name, + "uid": user.uid, + "gid": user.gid, + "gecos": user.gecos, + "home": user.home, + "shell": user.shell, + "primary_group": user.primary_group, + "supplementary_groups": user.supplementary_groups, + } + ) + + # Copy only safe SSH public material: authorized_keys + *.pub + for ssh_file in user.ssh_files: + reason = ( + "authorized_keys" + if ssh_file.endswith("/authorized_keys") + else "ssh_public_key" + ) + h._capture_file( + bundle_dir=self.context.bundle_dir, + role_name=users_role_name, + abs_path=ssh_file, + reason=reason, + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=users_managed, + excluded_out=users_excluded, + seen_role=users_role_seen, + seen_global=self.context.captured_global, + ) + + # Capture common per-user shell dotfiles only in dangerous mode. They + # often contain exported tokens or aliases/functions with embedded secrets. + home = (user.home or "").rstrip("/") + if home and home.startswith("/"): + h._capture_user_shell_dotfiles( + bundle_dir=self.context.bundle_dir, + role_name=users_role_name, + home=home, + skel_dir=skel_dir, + enabled=auto_capture_user_dotfiles, + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=users_managed, + excluded_out=users_excluded, + seen_role=users_role_seen, + seen_global=self.context.captured_global, + ) + + # Collect per-user Flatpak applications and remotes. Snap packages are + # system-wide; ~/snap/* is user data, not an install source. + if user.flatpaks: + user_flatpaks_map[user.name] = [asdict(fp) for fp in user.flatpaks] + user_flatpak_remotes.extend( + asdict(r) for r in find_user_flatpak_remotes(home, user=user.name) + ) + + return UsersCollection( + users_snapshot=UsersSnapshot( + role_name="users", + users=users_list, + managed_files=users_managed, + excluded=users_excluded, + notes=users_notes, + user_flatpaks=user_flatpaks_map, + user_flatpak_remotes=user_flatpak_remotes, + ), + flatpak_snapshot=FlatpakSnapshot( + role_name="flatpak", + system_flatpaks=system_flatpaks, + remotes=system_flatpak_remotes, + notes=flatpak_notes, + ), + snap_snapshot=SnapSnapshot( + role_name="snap", + system_snaps=system_snaps, + notes=snap_notes, + ), + ) diff --git a/enroll/manifest.py b/enroll/manifest.py index 97e26e7..32ea271 100644 --- a/enroll/manifest.py +++ b/enroll/manifest.py @@ -1,25 +1,14 @@ from __future__ import annotations -import json import os -import re import shutil -import stat import tarfile import tempfile from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple - -from .role_names import avoid_reserved_role_name -from .puppet import manifest_puppet_from_bundle_dir - -from .jinjaturtle import ( - can_jinjify_path, - find_jinjaturtle_cmd, - infer_other_formats, - run_jinjaturtle, -) +from typing import List, Optional +from .ansible import manifest_from_bundle_dir as manifest_ansible_from_bundle_dir +from .puppet import manifest_from_bundle_dir as manifest_puppet_from_bundle_dir from .remote import _safe_extract_tar from .sopsutil import ( decrypt_file_binary_to, @@ -28,872 +17,6 @@ from .sopsutil import ( ) -def _try_yaml(): - try: - import yaml # type: ignore - except Exception: - return None - return yaml - - -def _yaml_load_mapping(text: str) -> Dict[str, Any]: - yaml = _try_yaml() - if yaml is None: - return {} - try: - obj = yaml.safe_load(text) - except Exception: - return {} - if obj is None: - return {} - if isinstance(obj, dict): - return obj - return {} - - -def _yaml_dump_mapping(obj: Dict[str, Any], *, sort_keys: bool = True) -> str: - yaml = _try_yaml() - if yaml is None: - # fall back to a naive key: value dump (best-effort) - lines: List[str] = [] - for k, v in sorted(obj.items()) if sort_keys else obj.items(): - lines.append(f"{k}: {v!r}") - return "\n".join(lines).rstrip() + "\n" - - # ansible-lint/yamllint's indentation rules are stricter than YAML itself. - # In particular, they expect sequences nested under a mapping key to be - # indented (e.g. `foo:\n - a`), whereas PyYAML's default is often - # `foo:\n- a`. - class _IndentDumper(yaml.SafeDumper): # type: ignore - def increase_indent(self, flow: bool = False, indentless: bool = False): - return super().increase_indent(flow, False) - - return ( - yaml.dump( - obj, - Dumper=_IndentDumper, - default_flow_style=False, - sort_keys=sort_keys, - indent=2, - allow_unicode=True, - ).rstrip() - + "\n" - ) - - -def _role_id(raw: str) -> str: - """Return an Ansible-safe role identifier from an arbitrary label.""" - - s = re.sub(r"[^A-Za-z0-9]+", "_", raw or "misc") - s = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s) - s = s.lower() - s = re.sub(r"_+", "_", s).strip("_") - if not s: - s = "misc" - if not re.match(r"^[a-z_]", s): - s = "r_" + s - return s - - -def _package_section_label( - package_role: Dict[str, Any], inventory_packages: Dict[str, Any] -) -> str: - """Return the Debian Section/RPM Group label for a package role.""" - - pkg = str(package_role.get("package") or "") - inv = inventory_packages.get(pkg) or {} - candidates: List[str] = [] - - for value in (package_role.get("section"), inv.get("section")): - if isinstance(value, str) and value.strip(): - candidates.append(value.strip()) - - for inst in inv.get("installations", []) or []: - if not isinstance(inst, dict): - continue - for key in ("section", "group"): - value = inst.get(key) - if isinstance(value, str) and value.strip(): - candidates.append(value.strip()) - - for value in candidates: - if value.lower() in {"(none)", "none", "unspecified"}: - continue - return value - return "misc" - - -def _section_label_for_packages( - packages: List[str], inventory_packages: Dict[str, Any] -) -> str: - """Return a stable section/group label for a set of packages. - - Service roles can involve more than one package. Prefer the first - package with a concrete Debian Section/RPM Group and fall back to misc - when no package metadata is available. - """ - - for pkg in packages or []: - label = _package_section_label({"package": pkg}, inventory_packages) - if label and label.lower() != "misc": - return label - return "misc" - - -def _section_role_name(label: str, occupied_roles: Set[str]) -> str: - """Create a stable section role name, avoiding generated-role collisions.""" - - base = avoid_reserved_role_name(_role_id(label), prefix="section") - role = base if base not in occupied_roles else f"section_{base}" - n = 2 - while role in occupied_roles: - role = f"section_{base}_{n}" - n += 1 - occupied_roles.add(role) - return role - - -def _merge_mappings_overwrite( - existing: Dict[str, Any], incoming: Dict[str, Any] -) -> Dict[str, Any]: - """Merge incoming into existing with overwrite. - - NOTE: Unlike role defaults merging, host_vars should reflect the current - harvest for a host. Therefore lists are replaced rather than unioned. - """ - merged = dict(existing) - merged.update(incoming) - return merged - - -def _copy2_replace(src: str, dst: str) -> None: - dst_dir = os.path.dirname(dst) - os.makedirs(dst_dir, exist_ok=True) - - # Copy to a temp file in the same directory, then atomically replace. - fd, tmp = tempfile.mkstemp(prefix=".enroll-tmp-", dir=dst_dir) - os.close(fd) - try: - shutil.copy2(src, tmp) - - # Ensure the working tree stays mergeable: make the file user-writable. - st = os.stat(tmp, follow_symlinks=False) - mode = stat.S_IMODE(st.st_mode) - if not (mode & stat.S_IWUSR): - os.chmod(tmp, mode | stat.S_IWUSR) - - os.replace(tmp, dst) - finally: - try: - os.unlink(tmp) - except FileNotFoundError: - pass - - -def _copy_artifacts( - bundle_dir: str, - role: str, - dst_files_dir: str, - *, - preserve_existing: bool = False, - exclude_rels: Optional[Set[str]] = None, -) -> None: - """Copy harvested artifacts for a role into a destination *files* directory. - - In non --fqdn mode, this is usually /files. - In --fqdn site mode, this is usually: - inventory/host_vars///.files - """ - artifacts_dir = os.path.join(bundle_dir, "artifacts", role) - if not os.path.isdir(artifacts_dir): - return - for root, _, files in os.walk(artifacts_dir): - for fn in files: - src = os.path.join(root, fn) - rel = os.path.relpath(src, artifacts_dir) - dst = os.path.join(dst_files_dir, rel) - - # If a file was successfully templatised by JinjaTurtle, do NOT - # also materialise the raw copy in the destination files dir. - if exclude_rels and rel in exclude_rels: - try: - if os.path.isfile(dst): - os.remove(dst) - except Exception: - pass # nosec - continue - - if preserve_existing and os.path.exists(dst): - continue - os.makedirs(os.path.dirname(dst), exist_ok=True) - _copy2_replace(src, dst) - - -def _write_role_scaffold(role_dir: str) -> None: - os.makedirs(os.path.join(role_dir, "tasks"), exist_ok=True) - os.makedirs(os.path.join(role_dir, "handlers"), exist_ok=True) - os.makedirs(os.path.join(role_dir, "defaults"), exist_ok=True) - os.makedirs(os.path.join(role_dir, "meta"), exist_ok=True) - os.makedirs(os.path.join(role_dir, "files"), exist_ok=True) - os.makedirs(os.path.join(role_dir, "templates"), exist_ok=True) - - -def _role_tag(role: str) -> str: - """Return a stable Ansible tag name for a role. - - Used by `enroll diff --enforce` to run only the roles needed to repair drift. - """ - r = str(role or "").strip() - # Ansible tag charset is fairly permissive, but keep it portable and consistent. - safe = re.sub(r"[^A-Za-z0-9_-]+", "_", r).strip("_") - if not safe: - safe = "other" - return f"role_{safe}" - - -def _write_playbook_all(path: str, roles: List[str]) -> None: - pb_lines = [ - "---", - "- name: Apply all roles on all hosts", - " gather_facts: true", - " hosts: all", - " become: true", - " roles:", - ] - for r in roles: - pb_lines.append(f" - role: {r}") - pb_lines.append(f" tags: [{_role_tag(r)}]") - with open(path, "w", encoding="utf-8") as f: - f.write("\n".join(pb_lines) + "\n") - - -def _write_playbook_host(path: str, fqdn: str, roles: List[str]) -> None: - pb_lines = [ - "---", - f"- name: Apply all roles on {fqdn}", - f" hosts: {fqdn}", - " gather_facts: true", - " become: true", - " roles:", - ] - for r in roles: - pb_lines.append(f" - role: {r}") - pb_lines.append(f" tags: [{_role_tag(r)}]") - with open(path, "w", encoding="utf-8") as f: - f.write("\n".join(pb_lines) + "\n") - - -def _ensure_ansible_cfg(cfg_path: str) -> None: - if not os.path.exists(cfg_path): - with open(cfg_path, "w", encoding="utf-8") as f: - f.write("[defaults]\n") - f.write("roles_path = roles\n") - f.write("interpreter_python=/usr/bin/python3\n") - f.write("inventory = inventory\n") - f.write("stdout_callback = unixy\n") - f.write("force_color = 1\n") - f.write("vars_plugins_enabled = host_group_vars\n") - f.write("fact_caching = jsonfile\n") - f.write("fact_caching_connection = .enroll_cached_facts\n") - f.write("forks = 30\n") - f.write("remote_tmp = /tmp/ansible-${USER}\n") - f.write("timeout = 12\n") - f.write("[ssh_connection]\n") - f.write("pipelining = True\n") - f.write("scp_if_ssh = True\n") - return - - -def _ensure_requirements_yaml(req_path: str) -> None: - if not os.path.exists(req_path): - with open(req_path, "w", encoding="utf-8") as f: - f.write("---\n") - f.write("collections:\n") - f.write(" - name: community.general\n") - f.write(' version: ">=13.0.0"\n') - return - - -def _normalise_flatpak_item( - item: Any, - *, - method: str, - user: Optional[str] = None, - home: Optional[str] = None, -) -> Dict[str, Any]: - if isinstance(item, str): - out: Dict[str, Any] = {"name": item, "method": method} - elif isinstance(item, dict): - out = dict(item) - out.setdefault("method", method) - else: - out = {"name": str(item), "method": method} - if user: - out.setdefault("user", user) - if home: - out.setdefault("home", home) - return out - - -def _normalise_flatpak_remote(item: Any) -> Dict[str, Any]: - if isinstance(item, dict): - out = dict(item) - else: - out = {"name": str(item)} - out.setdefault("method", "system") - return out - - -def _normalise_snap_item(item: Any) -> Dict[str, Any]: - if isinstance(item, str): - out: Dict[str, Any] = {"name": item} - elif isinstance(item, dict): - out = dict(item) - else: - out = {"name": str(item)} - - notes = out.get("notes") or [] - if isinstance(notes, str): - notes = [notes] - notes_l = {str(n).lower() for n in notes} - out["classic"] = bool(out.get("classic") or "classic" in notes_l) - out["devmode"] = bool(out.get("devmode") or "devmode" in notes_l) - out["dangerous"] = bool(out.get("dangerous") or "dangerous" in notes_l) - - # The Ansible snap module's revision parameter pins/holds the snap. For - # ordinary store snaps that track a channel, preserve the channel instead - # of freezing every harvested host at today's revision. - if out.get("revision") is not None and not out.get("channel"): - out["install_revision"] = True - else: - out["install_revision"] = False - return out - - -def _ensure_inventory_host(inv_path: str, fqdn: str) -> None: - os.makedirs(os.path.dirname(inv_path), exist_ok=True) - if not os.path.exists(inv_path): - with open(inv_path, "w", encoding="utf-8") as f: - f.write("[all]\n") - f.write(fqdn + "\n") - return - - with open(inv_path, "r", encoding="utf-8") as f: - lines = [ln.rstrip("\n") for ln in f.readlines()] - - # ensure there is an [all] group; if not, create it at top - if not any(ln.strip() == "[all]" for ln in lines): - lines = ["[all]"] + lines - - # check if fqdn already present (exact match, ignoring whitespace) - if any(ln.strip() == fqdn for ln in lines): - return - - # append at end - lines.append(fqdn) - with open(inv_path, "w", encoding="utf-8") as f: - f.write("\n".join(lines) + "\n") - - -def _hostvars_path(site_root: str, fqdn: str, role: str) -> str: - return os.path.join(site_root, "inventory", "host_vars", fqdn, f"{role}.yml") - - -def _host_role_files_dir(site_root: str, fqdn: str, role: str) -> str: - """Host-specific files dir for a given role. - - Layout: - inventory/host_vars///.files/ - """ - return os.path.join(site_root, "inventory", "host_vars", fqdn, role, ".files") - - -def _write_hostvars(site_root: str, fqdn: str, role: str, data: Dict[str, Any]) -> None: - """Write host_vars YAML for a role for a specific host. - - This is host-specific state and should track the current harvest output. - Existing keys not mentioned in `data` are preserved, but keys in `data` - are overwritten (including list values). - """ - path = _hostvars_path(site_root, fqdn, role) - os.makedirs(os.path.dirname(path), exist_ok=True) - - existing_map: Dict[str, Any] = {} - if os.path.exists(path): - try: - existing_text = Path(path).read_text(encoding="utf-8") - existing_map = _yaml_load_mapping(existing_text) - except Exception: - existing_map = {} - - merged = _merge_mappings_overwrite(existing_map, data) - - out = "---\n" + _yaml_dump_mapping(merged, sort_keys=True) - with open(path, "w", encoding="utf-8") as f: - f.write(out) - - -def _jinjify_managed_files( - bundle_dir: str, - role: str, - role_dir: str, - managed_files: List[Dict[str, Any]], - *, - jt_exe: Optional[str], - jt_enabled: bool, - overwrite_templates: bool, -) -> Tuple[Set[str], str]: - """ - Return (templated_src_rels, combined_vars_text). - combined_vars_text is a YAML mapping fragment (no leading ---). - """ - templated: Set[str] = set() - vars_map: Dict[str, Any] = {} - - if not (jt_enabled and jt_exe): - return templated, "" - - for mf in managed_files: - dest_path = mf.get("path", "") - src_rel = mf.get("src_rel", "") - if not dest_path or not src_rel: - continue - if not can_jinjify_path(dest_path): - continue - - artifact_path = os.path.join(bundle_dir, "artifacts", role, src_rel) - if not os.path.isfile(artifact_path): - continue - - try: - force_fmt = infer_other_formats(dest_path) - res = run_jinjaturtle( - jt_exe, artifact_path, role_name=role, force_format=force_fmt - ) - except Exception: - # If jinjaturtle cannot process a file for any reason, skip silently. - # (Enroll's core promise is to be optimistic and non-interactive.) - continue # nosec - - tmpl_rel = src_rel + ".j2" - tmpl_dst = os.path.join(role_dir, "templates", tmpl_rel) - if overwrite_templates or not os.path.exists(tmpl_dst): - os.makedirs(os.path.dirname(tmpl_dst), exist_ok=True) - with open(tmpl_dst, "w", encoding="utf-8") as f: - f.write(res.template_text) - - templated.add(src_rel) - if res.vars_text.strip(): - # merge YAML mappings; last wins (avoids duplicate keys) - chunk = _yaml_load_mapping(res.vars_text) - if chunk: - vars_map = _merge_mappings_overwrite(vars_map, chunk) - - if vars_map: - combined = _yaml_dump_mapping(vars_map, sort_keys=True) - return templated, combined - return templated, "" - - -def _write_role_defaults(role_dir: str, mapping: Dict[str, Any]) -> None: - """Overwrite role defaults/main.yml with the provided mapping.""" - defaults_path = os.path.join(role_dir, "defaults", "main.yml") - os.makedirs(os.path.dirname(defaults_path), exist_ok=True) - out = "---\n" + _yaml_dump_mapping(mapping, sort_keys=True) - with open(defaults_path, "w", encoding="utf-8") as f: - f.write(out) - - -def _build_managed_dirs_var( - managed_dirs: List[Dict[str, Any]], -) -> List[Dict[str, Any]]: - """Convert enroll managed_dirs into an Ansible-friendly list of dicts. - - Each dict drives a role task loop and is safe across hosts. - """ - out: List[Dict[str, Any]] = [] - for d in managed_dirs: - dest = d.get("path") or "" - if not dest: - continue - out.append( - { - "dest": dest, - "owner": d.get("owner") or "root", - "group": d.get("group") or "root", - "mode": d.get("mode") or "0755", - } - ) - return out - - -def _build_managed_files_var( - managed_files: List[Dict[str, Any]], - templated_src_rels: Set[str], - *, - notify_other: Optional[str] = None, - notify_systemd: Optional[str] = None, -) -> List[Dict[str, Any]]: - """Convert enroll managed_files into an Ansible-friendly list of dicts. - - Each dict drives a role task loop and is safe across hosts. - """ - out: List[Dict[str, Any]] = [] - for mf in managed_files: - dest = mf.get("path") or "" - src_rel = mf.get("src_rel") or "" - if not dest or not src_rel: - continue - is_unit = str(dest).startswith("/etc/systemd/system/") - kind = "template" if src_rel in templated_src_rels else "copy" - notify: List[str] = [] - if is_unit and notify_systemd: - notify.append(notify_systemd) - if (not is_unit) and notify_other: - notify.append(notify_other) - out.append( - { - "dest": dest, - "src_rel": src_rel, - "owner": mf.get("owner") or "root", - "group": mf.get("group") or "root", - "mode": mf.get("mode") or "0644", - "kind": kind, - "is_systemd_unit": bool(is_unit), - "notify": notify, - } - ) - return out - - -def _build_managed_links_var( - managed_links: List[Dict[str, Any]], -) -> List[Dict[str, Any]]: - """Convert enroll managed_links into an Ansible-friendly list of dicts.""" - out: List[Dict[str, Any]] = [] - for ml in managed_links or []: - dest = ml.get("path") or "" - src = ml.get("target") or "" - if not dest or not src: - continue - out.append({"dest": dest, "src": src}) - return out - - -def _render_generic_files_tasks( - var_prefix: str, *, include_restart_notify: bool -) -> str: - """Render generic tasks to deploy _managed_files safely.""" - # Using first_found makes roles work in both modes: - # - site-mode: inventory/host_vars///.files/... - # - non-site: roles//files/... - return f"""- name: Ensure managed directories exist (preserve owner/group/mode) - ansible.builtin.file: - path: "{{{{ item.dest }}}}" - state: directory - owner: "{{{{ item.owner }}}}" - group: "{{{{ item.group }}}}" - mode: "{{{{ item.mode }}}}" - loop: "{{{{ {var_prefix}_managed_dirs | default([]) }}}}" - -- name: Deploy any systemd unit files (templates) - ansible.builtin.template: - src: "{{{{ item.src_rel }}}}.j2" - dest: "{{{{ item.dest }}}}" - owner: "{{{{ item.owner }}}}" - group: "{{{{ item.group }}}}" - mode: "{{{{ item.mode }}}}" - loop: >- - {{{{ {var_prefix}_managed_files | default([]) - | selectattr('is_systemd_unit', 'equalto', true) - | selectattr('kind', 'equalto', 'template') - | list }}}} - notify: "{{{{ item.notify | default([]) }}}}" - -- name: Deploy any systemd unit files (raw files) - vars: - _enroll_ff: - files: - - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ item.src_rel }}}}" - - "{{{{ role_path }}}}/files/{{{{ item.src_rel }}}}" - ansible.builtin.copy: - src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" - dest: "{{{{ item.dest }}}}" - owner: "{{{{ item.owner }}}}" - group: "{{{{ item.group }}}}" - mode: "{{{{ item.mode }}}}" - loop: >- - {{{{ {var_prefix}_managed_files | default([]) - | selectattr('is_systemd_unit', 'equalto', true) - | selectattr('kind', 'equalto', 'copy') - | list }}}} - notify: "{{{{ item.notify | default([]) }}}}" - -- name: Reload systemd to pick up unit changes - ansible.builtin.meta: flush_handlers - when: >- - ({var_prefix}_managed_files | default([]) - | selectattr('is_systemd_unit', 'equalto', true) - | list - | length) > 0 - -- name: Deploy any other managed files (templates) - ansible.builtin.template: - src: "{{{{ item.src_rel }}}}.j2" - dest: "{{{{ item.dest }}}}" - owner: "{{{{ item.owner }}}}" - group: "{{{{ item.group }}}}" - mode: "{{{{ item.mode }}}}" - loop: >- - {{{{ {var_prefix}_managed_files | default([]) - | selectattr('is_systemd_unit', 'equalto', false) - | selectattr('kind', 'equalto', 'template') - | list }}}} - notify: "{{{{ item.notify | default([]) }}}}" - -- name: Deploy any other managed files (raw files) - vars: - _enroll_ff: - files: - - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ item.src_rel }}}}" - - "{{{{ role_path }}}}/files/{{{{ item.src_rel }}}}" - ansible.builtin.copy: - src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" - dest: "{{{{ item.dest }}}}" - owner: "{{{{ item.owner }}}}" - group: "{{{{ item.group }}}}" - mode: "{{{{ item.mode }}}}" - loop: >- - {{{{ {var_prefix}_managed_files | default([]) - | selectattr('is_systemd_unit', 'equalto', false) - | selectattr('kind', 'equalto', 'copy') - | list }}}} - notify: "{{{{ item.notify | default([]) }}}}" - -- name: Ensure managed symlinks exist - ansible.builtin.file: - src: "{{{{ item.src }}}}" - dest: "{{{{ item.dest }}}}" - state: link - force: true - loop: "{{{{ {var_prefix}_managed_links | default([]) }}}}" -""" - - -def _render_install_packages_tasks(role: str, var_prefix: str) -> str: - """Render cross-distro package installation tasks. - - We generate conditional tasks for apt/dnf/yum, falling back to the - generic `package` module. This keeps generated roles usable on both - Debian-like and RPM-like systems. - """ - return f"""- name: Install packages for {role} (APT) - ansible.builtin.apt: - name: "{{{{ {var_prefix}_packages | default([]) }}}}" - state: present - update_cache: true - when: - - ({var_prefix}_packages | default([])) | length > 0 - - ansible_facts.pkg_mgr | default('') == 'apt' - -- name: Install packages for {role} (DNF5) - ansible.builtin.dnf5: - name: "{{{{ {var_prefix}_packages | default([]) }}}}" - state: present - when: - - ({var_prefix}_packages | default([])) | length > 0 - - ansible_facts.pkg_mgr | default('') == 'dnf5' - -- name: Install packages for {role} (DNF/YUM) - ansible.builtin.dnf: - name: "{{{{ {var_prefix}_packages | default([]) }}}}" - state: present - when: - - ({var_prefix}_packages | default([])) | length > 0 - - ansible_facts.pkg_mgr | default('') in ['dnf', 'yum'] - -- name: Install packages for {role} (generic fallback) - ansible.builtin.package: - name: "{{{{ {var_prefix}_packages | default([]) }}}}" - state: present - when: - - ({var_prefix}_packages | default([])) | length > 0 - - ansible_facts.pkg_mgr | default('') not in ['apt', 'dnf', 'dnf5', 'yum'] - -""" - - -def _render_grouped_systemd_tasks(var_prefix: str) -> str: - """Render tasks to manage multiple systemd units in a common role.""" - - return f"""- name: Probe whether grouped systemd units exist and are manageable - ansible.builtin.systemd: - name: "{{{{ item.name }}}}" - no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" - check_mode: true - loop: "{{{{ {var_prefix}_systemd_units | default([]) }}}}" - register: _enroll_unit_probes - failed_when: false - changed_when: false - when: item.manage | default(false) - -- name: Ensure grouped unit enablement matches harvest - ansible.builtin.systemd: - name: "{{{{ item.item.name }}}}" - enabled: "{{{{ item.item.enabled | bool }}}}" - no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" - loop: "{{{{ _enroll_unit_probes.results | default([]) }}}}" - when: - - item.item.manage | default(false) - - not (item.failed | default(false)) - -- name: Ensure grouped unit running state matches harvest - ansible.builtin.systemd: - name: "{{{{ item.item.name }}}}" - state: "{{{{ item.item.state }}}}" - no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" - loop: "{{{{ _enroll_unit_probes.results | default([]) }}}}" - when: - - item.item.manage | default(false) - - not (item.failed | default(false)) -""" - - -def _render_sysctl_tasks(var_prefix: str) -> str: - return f"""- name: Ensure sysctl.d exists - ansible.builtin.file: - path: /etc/sysctl.d - state: directory - owner: root - group: root - mode: "0755" - -- name: Deploy captured sysctl configuration - vars: - _enroll_ff: - files: - - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ {var_prefix}_conf_src_rel }}}}" - - "{{{{ role_path }}}}/files/{{{{ {var_prefix}_conf_src_rel }}}}" - ansible.builtin.copy: - src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" - dest: /etc/sysctl.d/99-enroll.conf - owner: root - group: root - mode: "0644" - when: ({var_prefix}_conf_src_rel | default('') | length) > 0 - notify: Apply captured sysctl configuration -""" - - -def _render_sysctl_handlers(var_prefix: str) -> str: - return f"""--- -- name: Apply captured sysctl configuration - ansible.builtin.command: - argv: - - sysctl - - -e - - -p - - /etc/sysctl.d/99-enroll.conf - register: _enroll_sysctl_apply - changed_when: false - failed_when: - - not ({var_prefix}_ignore_apply_errors | default(true) | bool) - - _enroll_sysctl_apply.rc != 0 - when: {var_prefix}_apply | default(true) | bool -""" - - -def _render_firewall_runtime_tasks(var_prefix: str) -> str: - """Render tasks for live ipset/iptables snapshots.""" - return f"""- name: Ensure firewall runtime snapshot directory exists - ansible.builtin.file: - path: /etc/enroll/firewall - state: directory - owner: root - group: root - mode: "0750" - -- name: Deploy captured ipset snapshot - vars: - _enroll_ff: - files: - - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ {var_prefix}_ipset_save }}}}" - - "{{{{ role_path }}}}/files/{{{{ {var_prefix}_ipset_save }}}}" - ansible.builtin.copy: - src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" - dest: /etc/enroll/firewall/ipset.save - owner: root - group: root - mode: "0600" - when: ({var_prefix}_ipset_save | default('') | length) > 0 - -- name: Flush captured ipsets before restoring members - ansible.builtin.command: - cmd: "ipset flush {{{{ item }}}}" - loop: "{{{{ {var_prefix}_ipset_sets | default([]) }}}}" - register: _enroll_ipset_flush - failed_when: false - changed_when: false - when: - - ({var_prefix}_ipset_save | default('') | length) > 0 - - {var_prefix}_sync_ipsets_exact | default(true) | bool - -- name: Restore captured ipsets - ansible.builtin.shell: "ipset restore -exist < /etc/enroll/firewall/ipset.save" - args: - executable: /bin/sh - register: _enroll_ipset_restore - changed_when: _enroll_ipset_restore.rc == 0 - when: ({var_prefix}_ipset_save | default('') | length) > 0 - -- name: Deploy captured IPv4 iptables snapshot - vars: - _enroll_ff: - files: - - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ {var_prefix}_iptables_v4_save }}}}" - - "{{{{ role_path }}}}/files/{{{{ {var_prefix}_iptables_v4_save }}}}" - ansible.builtin.copy: - src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" - dest: /etc/enroll/firewall/iptables.v4 - owner: root - group: root - mode: "0600" - when: ({var_prefix}_iptables_v4_save | default('') | length) > 0 - -- name: Restore captured IPv4 iptables rules - ansible.builtin.command: - cmd: iptables-restore /etc/enroll/firewall/iptables.v4 - register: _enroll_iptables_v4_restore - changed_when: _enroll_iptables_v4_restore.rc == 0 - when: - - ({var_prefix}_iptables_v4_save | default('') | length) > 0 - - {var_prefix}_restore_iptables | default(true) | bool - -- name: Deploy captured IPv6 iptables snapshot - vars: - _enroll_ff: - files: - - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ {var_prefix}_iptables_v6_save }}}}" - - "{{{{ role_path }}}}/files/{{{{ {var_prefix}_iptables_v6_save }}}}" - ansible.builtin.copy: - src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" - dest: /etc/enroll/firewall/iptables.v6 - owner: root - group: root - mode: "0600" - when: ({var_prefix}_iptables_v6_save | default('') | length) > 0 - -- name: Restore captured IPv6 iptables rules - ansible.builtin.command: - cmd: ip6tables-restore /etc/enroll/firewall/iptables.v6 - register: _enroll_iptables_v6_restore - changed_when: _enroll_iptables_v6_restore.rc == 0 - when: - - ({var_prefix}_iptables_v6_save | default('') | length) > 0 - - {var_prefix}_restore_iptables | default(true) | bool -""" - - def _prepare_bundle_dir( bundle: str, *, @@ -1040,2206 +163,6 @@ def _encrypt_manifest_out_dir_to_sops( return out_file -def _manifest_ansible_from_bundle_dir( - bundle_dir: str, - out_dir: str, - *, - fqdn: Optional[str] = None, - jinjaturtle: str = "auto", # auto|on|off - no_common_roles: bool = False, -) -> None: - state_path = os.path.join(bundle_dir, "state.json") - with open(state_path, "r", encoding="utf-8") as f: - state = json.load(f) - - roles: Dict[str, Any] = state.get("roles") or {} - inventory_packages: Dict[str, Any] = (state.get("inventory") or {}).get( - "packages" - ) or {} - - services: List[Dict[str, Any]] = roles.get("services", []) - package_roles: List[Dict[str, Any]] = roles.get("packages", []) - users_snapshot: Dict[str, Any] = roles.get("users", {}) - flatpak_snapshot: Dict[str, Any] = roles.get("flatpak", {}) - snap_snapshot: Dict[str, Any] = roles.get("snap", {}) - apt_config_snapshot: Dict[str, Any] = roles.get("apt_config", {}) - dnf_config_snapshot: Dict[str, Any] = roles.get("dnf_config", {}) - firewall_runtime_snapshot: Dict[str, Any] = roles.get("firewall_runtime", {}) - sysctl_snapshot: Dict[str, Any] = roles.get("sysctl", {}) - etc_custom_snapshot: Dict[str, Any] = roles.get("etc_custom", {}) - usr_local_custom_snapshot: Dict[str, Any] = roles.get("usr_local_custom", {}) - extra_paths_snapshot: Dict[str, Any] = roles.get("extra_paths", {}) - - site_mode = fqdn is not None and fqdn != "" - use_common_roles = (not site_mode) and (not no_common_roles) - - jt_exe = find_jinjaturtle_cmd() - jt_enabled = False - if jinjaturtle not in ("auto", "on", "off"): - raise ValueError("jinjaturtle must be one of: auto, on, off") - if jinjaturtle == "on": - if not jt_exe: - raise RuntimeError("jinjaturtle requested but not found on PATH") - jt_enabled = True - elif jinjaturtle == "auto": - jt_enabled = jt_exe is not None - else: - jt_enabled = False - - os.makedirs(out_dir, exist_ok=True) - roles_root = os.path.join(out_dir, "roles") - os.makedirs(roles_root, exist_ok=True) - - # Site-mode scaffolding - if site_mode: - os.makedirs(os.path.join(out_dir, "inventory"), exist_ok=True) - os.makedirs(os.path.join(out_dir, "inventory", "host_vars"), exist_ok=True) - os.makedirs(os.path.join(out_dir, "playbooks"), exist_ok=True) - _ensure_inventory_host( - os.path.join(out_dir, "inventory", "hosts.ini"), fqdn or "" - ) - _ensure_ansible_cfg(os.path.join(out_dir, "ansible.cfg")) - _ensure_requirements_yaml(os.path.join(out_dir, "requirements.yml")) - - manifested_users_roles: List[str] = [] - manifested_flatpak_roles: List[str] = [] - manifested_snap_roles: List[str] = [] - manifested_apt_config_roles: List[str] = [] - manifested_dnf_config_roles: List[str] = [] - manifested_firewall_runtime_roles: List[str] = [] - manifested_sysctl_roles: List[str] = [] - manifested_etc_custom_roles: List[str] = [] - manifested_usr_local_custom_roles: List[str] = [] - manifested_extra_paths_roles: List[str] = [] - manifested_service_roles: List[str] = [] - manifested_pkg_roles: List[str] = [] - common_role_groups: Dict[str, List[Dict[str, Any]]] = {} - common_tail_roles: List[str] = [] - - if use_common_roles: - for svc in services: - label = _section_label_for_packages( - svc.get("packages", []) or [], inventory_packages - ) - common_role_groups.setdefault(label, []).append( - {"kind": "service", "snapshot": svc} - ) - services_to_manifest: List[Dict[str, Any]] = [] - else: - services_to_manifest = services - - # ------------------------- - # Users role (non-system users) - # ------------------------- - if users_snapshot: - role = users_snapshot.get("role_name", "users") - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - - # Users role includes harvested SSH-related files; in site mode keep them - # host-specific to avoid cross-host clobber. - if site_mode: - _copy_artifacts( - bundle_dir, role, _host_role_files_dir(out_dir, fqdn or "", role) - ) - else: - _copy_artifacts(bundle_dir, role, os.path.join(role_dir, "files")) - - users = users_snapshot.get("users", []) - managed_files = users_snapshot.get("managed_files", []) - excluded = users_snapshot.get("excluded", []) - notes = users_snapshot.get("notes", []) - - # Build groups list and a simplified user dict list suitable for loops - group_names: List[str] = [] - group_set = set() - users_data: List[Dict[str, Any]] = [] - for u in users: - name = u.get("name") - if not name: - continue - pg = u.get("primary_group") or name - home = u.get("home") or f"/home/{name}" - sshdir = home.rstrip("/") + "/.ssh" - supp = u.get("supplementary_groups") or [] - if pg: - group_set.add(pg) - for g in supp: - if g: - group_set.add(g) - - users_data.append( - { - "name": name, - "uid": u.get("uid"), - "primary_group": pg, - "home": home, - "ssh_dir": sshdir, - "shell": u.get("shell"), - "gecos": u.get("gecos"), - "supplementary_groups": sorted(set(supp)), - } - ) - - group_names = sorted(group_set) - - # User-managed files (authorized_keys plus dangerous-mode shell dotfiles). - # Keep the variable name for compatibility with existing generated data. - ssh_files: List[Dict[str, Any]] = [] - for mf in managed_files: - dest = mf.get("path") or "" - src_rel = mf.get("src_rel") or "" - if not dest or not src_rel: - continue - - owner = "root" - group = "root" - for u in users_data: - home_prefix = (u.get("home") or "").rstrip("/") + "/" - if home_prefix and dest.startswith(home_prefix): - owner = str(u.get("name") or "root") - group = str(u.get("primary_group") or owner) - break - - # Prefer the harvested file mode so we preserve any deliberate - # permissions (e.g. 0600 for certain dotfiles). For authorized_keys, - # enforce 0600 regardless. - mode = mf.get("mode") or "0644" - if mf.get("reason") == "authorized_keys": - mode = "0600" - ssh_files.append( - { - "dest": dest, - "src_rel": src_rel, - "owner": owner, - "group": group, - "mode": mode, - } - ) - - # Build Flatpak and Snap lists. Flatpak can be installed system-wide or - # per-user. Snap packages are system-wide; per-user ~/snap/* directories - # are runtime/user data and are not treated as install sources. - users_flatpaks: List[Dict[str, Any]] = [] - user_flatpak_map = users_snapshot.get("user_flatpaks", {}) or {} - home_by_user = { - str(u.get("name")): str(u.get("home") or "") for u in users_data - } - for uname, flatpaks in user_flatpak_map.items(): - for fp in flatpaks or []: - users_flatpaks.append( - _normalise_flatpak_item( - fp, - method="user", - user=str(uname), - home=home_by_user.get(str(uname)) or None, - ) - ) - - flatpak_remotes = [ - _normalise_flatpak_remote(r) - for r in (users_snapshot.get("user_flatpak_remotes", []) or []) - ] - users_needs_community = bool(flatpak_remotes or users_flatpaks) - if users_needs_community: - _ensure_requirements_yaml(os.path.join(out_dir, "requirements.yml")) - - # Variables are host-specific in site mode; in non-site mode they live in role defaults. - if site_mode: - _write_role_defaults( - role_dir, - { - "users_groups": [], - "users_users": [], - "users_ssh_files": [], - "users_flatpaks": [], - "users_flatpak_remotes": [], - }, - ) - _write_hostvars( - out_dir, - fqdn or "", - role, - { - "users_groups": group_names, - "users_users": users_data, - "users_ssh_files": ssh_files, - "users_flatpaks": users_flatpaks, - "users_flatpak_remotes": flatpak_remotes, - }, - ) - else: - _write_role_defaults( - role_dir, - { - "users_groups": group_names, - "users_users": users_data, - "users_ssh_files": ssh_files, - "users_flatpaks": users_flatpaks, - "users_flatpak_remotes": flatpak_remotes, - }, - ) - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - if users_needs_community: - f.write( - "---\n" - "dependencies: []\n" - "collections:\n" - " - community.general\n" - ) - else: - f.write("---\ndependencies: []\n") - - # tasks (data-driven) - users_tasks = """--- - -- name: Ensure groups exist - ansible.builtin.group: - name: "{{ item }}" - state: present - loop: "{{ users_groups | default([]) }}" - -- name: Ensure users exist - ansible.builtin.user: - name: "{{ item.name }}" - uid: "{{ item.uid | default(omit) }}" - group: "{{ item.primary_group }}" - home: "{{ item.home }}" - create_home: true - shell: "{{ item.shell | default(omit) }}" - comment: "{{ item.gecos | default(omit) }}" - state: present - loop: "{{ users_users | default([]) }}" - -- name: Ensure users supplementary groups - ansible.builtin.user: - name: "{{ item.name }}" - groups: "{{ item.supplementary_groups | default([]) | join(',') }}" - append: true - loop: "{{ users_users | default([]) }}" - when: (item.supplementary_groups | default([])) | length > 0 - -- name: Ensure .ssh directories exist - ansible.builtin.file: - path: "{{ item.ssh_dir }}" - state: directory - owner: "{{ item.name }}" - group: "{{ item.primary_group }}" - mode: "0700" - loop: "{{ users_users | default([]) }}" - -- name: Deploy user-managed files - vars: - _enroll_ff: - files: - - "{{ inventory_dir }}/host_vars/{{ inventory_hostname }}/{{ role_name }}/.files/{{ item.src_rel }}" - - "{{ role_path }}/files/{{ item.src_rel }}" - ansible.builtin.copy: - src: "{{ lookup('ansible.builtin.first_found', _enroll_ff) }}" - dest: "{{ item.dest }}" - owner: "{{ item.owner }}" - group: "{{ item.group }}" - mode: "{{ item.mode }}" - loop: "{{ users_ssh_files | default([]) }}" -""" - - if flatpak_remotes or users_flatpaks: - users_tasks += """ -- name: Ensure user Flatpak remotes exist - ansible.builtin.command: - argv: - - flatpak - - remote-add - - --user - - --if-not-exists - - "{{ item.name }}" - - "{{ item.url }}" - loop: "{{ users_flatpak_remotes | default([]) | selectattr('method', 'equalto', 'user') | list }}" - when: - - item.name is defined - - item.url is defined - - item.url | length > 0 - - item.user is defined - become: true - become_user: "{{ item.user }}" - environment: - HOME: "{{ item.home | default('/home/' ~ item.user, true) }}" - XDG_DATA_HOME: "{{ (item.home | default('/home/' ~ item.user, true)) ~ '/.local/share' }}" - changed_when: false - -- name: Install user Flatpaks - community.general.flatpak: - name: - - "{{ item.name }}" - state: present - method: user - remote: "{{ item.remote | default(omit) }}" - from_url: "{{ item.from_url | default(omit) }}" - loop: "{{ users_flatpaks | default([]) }}" - when: - - item.name is defined - - item.name | length > 0 - - item.user is defined - become: true - become_user: "{{ item.user }}" - environment: - HOME: "{{ item.home | default('/home/' ~ item.user, true) }}" - XDG_DATA_HOME: "{{ (item.home | default('/home/' ~ item.user, true)) ~ '/.local/share' }}" -""" - - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(users_tasks) - - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\n") - - def _fmt_app_list(items: List[Dict[str, Any]]) -> str: - lines = [] - for item in items: - name = item.get("name") - if not name: - continue - detail_parts = [] - for key in ("remote", "channel", "revision", "branch", "arch"): - value = item.get(key) - if value not in (None, "", []): - detail_parts.append(f"{key}={value}") - for key in ("classic", "devmode", "dangerous"): - if item.get(key): - detail_parts.append(key) - details = f" ({', '.join(detail_parts)})" if detail_parts else "" - lines.append(f"- {name}{details}") - return "\n".join(lines) or "- (none)" - - def _fmt_user_flatpaks(items: List[Dict[str, Any]]) -> str: - lines = [] - for item in items: - name = item.get("name") - user = item.get("user") - if not name or not user: - continue - detail_parts = [] - for key in ("remote", "branch", "arch"): - value = item.get(key) - if value not in (None, "", []): - detail_parts.append(f"{key}={value}") - details = f" ({', '.join(detail_parts)})" if detail_parts else "" - lines.append(f"- {user}: {name}{details}") - return "\n".join(lines) or "- (none)" - - def _fmt_remotes(items: List[Dict[str, Any]]) -> str: - lines = [] - for item in items: - name = item.get("name") - url = item.get("url") - method = item.get("method") or "system" - user = item.get("user") - if not name or not url: - continue - owner = f"user={user}" if user else "system" - lines.append(f"- {name} ({method}, {owner}): {url}") - return "\n".join(lines) or "- (none)" - - readme = ( - """# users - -Generated non-system user accounts, SSH public material, and per-user Flatpak -applications/remotes. - -**Note:** User Flatpak tasks require the `community.general` Ansible collection. -Install it with: `ansible-galaxy collection install -r requirements.yml`. - -Flatpak `remote` is harvested from the installed deployment where detectable. -The original `.flatpakref` URL is generally not preserved by Flatpak after -installation, so `from_url` is only emitted if a future/hand-edited state file -contains it. - - -## Users -""" - + ( - "\n".join([f"- {u.get('name')} (uid {u.get('uid')})" for u in users]) - or "- (none)" - ) - + """\n -## Included SSH files -""" - + ( - "\n".join( - [f"- {mf.get('path')} ({mf.get('reason')})" for mf in managed_files] - ) - or "- (none)" - ) - + """\n -## Flatpak remotes -""" - + _fmt_remotes(flatpak_remotes) - + """\n -## User Flatpaks -""" - + _fmt_user_flatpaks(users_flatpaks) - + """\n -## Excluded -""" - + ( - "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) - or "- (none)" - ) - + """\n -## Notes -""" - + ("\n".join([f"- {n}" for n in notes]) or "- (none)") - + """\n""" - ) - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_users_roles.append(role) - - # ------------------------- - # Flatpak role (system-wide Flatpak remotes and applications) - # ------------------------- - raw_flatpak_apps = flatpak_snapshot.get("system_flatpaks", []) or [] - raw_flatpak_remotes = flatpak_snapshot.get("remotes", []) or [] - - if flatpak_snapshot: - role = flatpak_snapshot.get("role_name", "flatpak") - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - _ensure_requirements_yaml(os.path.join(out_dir, "requirements.yml")) - - flatpak_system_flatpaks = [ - _normalise_flatpak_item(fp, method="system") for fp in raw_flatpak_apps - ] - flatpak_remotes = [_normalise_flatpak_remote(r) for r in raw_flatpak_remotes] - - vars_map = { - "flatpak_system_flatpaks": flatpak_system_flatpaks, - "flatpak_remotes": flatpak_remotes, - } - if site_mode: - _write_role_defaults( - role_dir, - {"flatpak_system_flatpaks": [], "flatpak_remotes": []}, - ) - _write_hostvars(out_dir, fqdn or "", role, vars_map) - else: - _write_role_defaults(role_dir, vars_map) - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write( - "---\n" "dependencies: []\n" "collections:\n" " - community.general\n" - ) - - tasks = """--- - -- name: Ensure system Flatpak remotes exist - ansible.builtin.command: - argv: - - flatpak - - remote-add - - --system - - --if-not-exists - - "{{ item.name }}" - - "{{ item.url }}" - loop: "{{ flatpak_remotes | default([]) }}" - when: - - item.name is defined - - item.url is defined - - item.url | length > 0 - become: true - changed_when: false - -- name: Install system-wide Flatpaks - community.general.flatpak: - name: - - "{{ item.name }}" - state: present - method: system - remote: "{{ item.remote | default(omit) }}" - from_url: "{{ item.from_url | default(omit) }}" - loop: "{{ flatpak_system_flatpaks | default([]) }}" - when: - - item.name is defined - - item.name | length > 0 - become: true -""" - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks) - - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\n") - - def _fmt_flatpak_apps(items: List[Dict[str, Any]]) -> str: - lines = [] - for item in items: - name = item.get("name") - if not name: - continue - detail_parts = [] - for key in ("remote", "branch", "arch"): - value = item.get(key) - if value not in (None, "", []): - detail_parts.append(f"{key}={value}") - details = f" ({', '.join(detail_parts)})" if detail_parts else "" - lines.append(f"- {name}{details}") - return "\n".join(lines) or "- (none)" - - def _fmt_flatpak_remotes(items: List[Dict[str, Any]]) -> str: - lines = [] - for item in items: - name = item.get("name") - url = item.get("url") - if not name or not url: - continue - lines.append(f"- {name}: {url}") - return "\n".join(lines) or "- (none)" - - notes = flatpak_snapshot.get("notes", []) or [] - readme = ( - """# flatpak - -Generated system-wide Flatpak remotes and applications. - -**Note:** This role requires the `community.general` Ansible collection. -Install it with: `ansible-galaxy collection install -r requirements.yml`. - -Flatpak `remote` is harvested from the installed deployment where detectable. -The original `.flatpakref` URL is generally not preserved by Flatpak after -installation, so `from_url` is only emitted if a future/hand-edited state file -contains it. - -## System Flatpak remotes -""" - + _fmt_flatpak_remotes(flatpak_remotes) - + """\n -## System-wide Flatpaks -""" - + _fmt_flatpak_apps(flatpak_system_flatpaks) - + """\n -## Notes -""" - + ("\n".join([f"- {n}" for n in notes]) or "- (none)") - + """\n""" - ) - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_flatpak_roles.append(role) - - # ------------------------- - # Snap role (system-wide snap packages) - # ------------------------- - raw_system_snaps = snap_snapshot.get("system_snaps", []) or [] - - if raw_system_snaps: - role = snap_snapshot.get("role_name", "snap") if snap_snapshot else "snap" - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - _ensure_requirements_yaml(os.path.join(out_dir, "requirements.yml")) - - snap_system_snaps = [_normalise_snap_item(s) for s in raw_system_snaps] - - vars_map = {"snap_system_snaps": snap_system_snaps} - if site_mode: - _write_role_defaults(role_dir, {"snap_system_snaps": []}) - _write_hostvars(out_dir, fqdn or "", role, vars_map) - else: - _write_role_defaults(role_dir, vars_map) - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write( - "---\n" "dependencies: []\n" "collections:\n" " - community.general\n" - ) - - tasks = """--- - -- name: Install system-wide snaps with full detected attributes - community.general.snap: - name: - - "{{ item.name }}" - state: present - channel: "{{ item.channel | default(omit) if not (item.install_revision | default(false)) else omit }}" - revision: "{{ item.revision | default(omit) if (item.install_revision | default(false)) else omit }}" - classic: "{{ item.classic | default(false) }}" - devmode: "{{ item.devmode | default(false) }}" - dangerous: "{{ item.dangerous | default(false) }}" - loop: "{{ snap_system_snaps | default([]) }}" - when: - - item.name is defined - - item.name | length > 0 - become: true - register: _enroll_snap_full_results - ignore_errors: true - -- name: Install system-wide snaps with compatibility options - community.general.snap: - name: - - "{{ item.item.name }}" - state: present - channel: "{{ item.item.channel | default(omit) if not (item.item.install_revision | default(false)) else omit }}" - classic: "{{ item.item.classic | default(false) }}" - loop: "{{ (_enroll_snap_full_results | default({})).results | default([]) }}" - when: - - item.failed | default(false) - - item.item.name is defined - - item.item.name | length > 0 - become: true - register: _enroll_snap_compat_results - ignore_errors: true - -- name: Install system-wide snaps with minimal options - community.general.snap: - name: - - "{{ item.item.item.name }}" - state: present - loop: "{{ (_enroll_snap_compat_results | default({})).results | default([]) }}" - when: - - item.failed | default(false) - - item.item.item.name is defined - - item.item.item.name | length > 0 - become: true - ignore_errors: true -""" - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks) - - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\n") - - def _fmt_snap_apps(items: List[Dict[str, Any]]) -> str: - lines = [] - for item in items: - name = item.get("name") - if not name: - continue - detail_parts = [] - for key in ("channel", "revision"): - value = item.get(key) - if value not in (None, "", []): - detail_parts.append(f"{key}={value}") - for key in ("classic", "devmode", "dangerous"): - if item.get(key): - detail_parts.append(key) - details = f" ({', '.join(detail_parts)})" if detail_parts else "" - lines.append(f"- {name}{details}") - return "\n".join(lines) or "- (none)" - - notes = snap_snapshot.get("notes", []) or [] - readme = ( - """# snap - -Generated system-wide snap packages. - -**Note:** This role requires the `community.general` Ansible collection. -Install it with: `ansible-galaxy collection install -r requirements.yml`. - -The first install task uses all harvested attributes. If the installed -`community.general.snap` module is too old for some parameters, the generated -role falls back to reduced then minimal install tasks on a best-effort basis. - -## System-wide snaps -""" - + _fmt_snap_apps(snap_system_snaps) - + """\n -## Notes -""" - + ("\n".join([f"- {n}" for n in notes]) or "- (none)") - + """\n""" - ) - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_snap_roles.append(role) - - # ------------------------- - # apt_config role (APT sources, pinning, and keyrings) - # ------------------------- - if apt_config_snapshot and apt_config_snapshot.get("managed_files"): - role = apt_config_snapshot.get("role_name", "apt_config") - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - - var_prefix = role - - managed_files = apt_config_snapshot.get("managed_files", []) - managed_dirs = apt_config_snapshot.get("managed_dirs", []) or [] - excluded = apt_config_snapshot.get("excluded", []) - notes = apt_config_snapshot.get("notes", []) - - templated, jt_vars = _jinjify_managed_files( - bundle_dir, - role, - role_dir, - managed_files, - jt_exe=jt_exe, - jt_enabled=jt_enabled, - overwrite_templates=not site_mode, - ) - - # Copy only the non-templated artifacts (templates live in the role). - if site_mode: - _copy_artifacts( - bundle_dir, - role, - _host_role_files_dir(out_dir, fqdn or "", role), - exclude_rels=templated, - ) - else: - _copy_artifacts( - bundle_dir, - role, - os.path.join(role_dir, "files"), - exclude_rels=templated, - ) - - files_var = _build_managed_files_var( - managed_files, - templated, - notify_other=None, - notify_systemd=None, - ) - - dirs_var = _build_managed_dirs_var(managed_dirs) - - jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} - vars_map: Dict[str, Any] = { - f"{var_prefix}_managed_files": files_var, - f"{var_prefix}_managed_dirs": dirs_var, - } - vars_map = _merge_mappings_overwrite(vars_map, jt_map) - - if site_mode: - _write_role_defaults( - role_dir, - {f"{var_prefix}_managed_files": [], f"{var_prefix}_managed_dirs": []}, - ) - _write_hostvars(out_dir, fqdn or "", role, vars_map) - else: - _write_role_defaults(role_dir, vars_map) - - tasks = "---\n" + _render_generic_files_tasks( - var_prefix, include_restart_notify=False - ) - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks.rstrip() + "\n") - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\ndependencies: []\n") - - # README: summarise repos and keyrings - source_paths: List[str] = [] - keyring_paths: List[str] = [] - repo_hosts: Set[str] = set() - - url_re = re.compile(r"(?:https?|ftp)://([^/\s]+)", re.IGNORECASE) - - for mf in managed_files: - p = str(mf.get("path") or "") - src_rel = str(mf.get("src_rel") or "") - if not p or not src_rel: - continue - - if p == "/etc/apt/sources.list" or p.startswith("/etc/apt/sources.list.d/"): - source_paths.append(p) - art_path = os.path.join(bundle_dir, "artifacts", role, src_rel) - try: - with open(art_path, "r", encoding="utf-8", errors="replace") as sf: - for line in sf: - line = line.strip() - if not line or line.startswith("#"): - continue - for m in url_re.finditer(line): - repo_hosts.add(m.group(1)) - except OSError: - pass # nosec - - if ( - p.startswith("/etc/apt/trusted.gpg") - or p.startswith("/etc/apt/keyrings/") - or p.startswith("/usr/share/keyrings/") - ): - keyring_paths.append(p) - - source_paths = sorted(set(source_paths)) - keyring_paths = sorted(set(keyring_paths)) - repos = sorted(repo_hosts) - - readme = ( - """# apt_config - -APT configuration harvested from the system (sources, pinning, and keyrings). - -## Repository hosts -""" - + ("\n".join([f"- {h}" for h in repos]) or "- (none)") - + """\n -## Source files -""" - + ("\n".join([f"- {p}" for p in source_paths]) or "- (none)") - + """\n -## Keyrings -""" - + ("\n".join([f"- {p}" for p in keyring_paths]) or "- (none)") - + """\n -## Managed files -""" - + ( - "\n".join( - [f"- {mf.get('path')} ({mf.get('reason')})" for mf in managed_files] - ) - or "- (none)" - ) - + """\n -## Excluded -""" - + ( - "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) - or "- (none)" - ) - + """\n -## Notes -""" - + ("\n".join([f"- {n}" for n in notes]) or "- (none)") - + """\n""" - ) - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_apt_config_roles.append(role) - - # ------------------------- - # dnf_config role (DNF/YUM repos, config, and RPM GPG keys) - # ------------------------- - if dnf_config_snapshot and dnf_config_snapshot.get("managed_files"): - role = dnf_config_snapshot.get("role_name", "dnf_config") - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - - var_prefix = role - - managed_files = dnf_config_snapshot.get("managed_files", []) - managed_dirs = dnf_config_snapshot.get("managed_dirs", []) or [] - excluded = dnf_config_snapshot.get("excluded", []) - notes = dnf_config_snapshot.get("notes", []) - - templated, jt_vars = _jinjify_managed_files( - bundle_dir, - role, - role_dir, - managed_files, - jt_exe=jt_exe, - jt_enabled=jt_enabled, - overwrite_templates=not site_mode, - ) - - if site_mode: - _copy_artifacts( - bundle_dir, - role, - _host_role_files_dir(out_dir, fqdn or "", role), - exclude_rels=templated, - ) - else: - _copy_artifacts( - bundle_dir, - role, - os.path.join(role_dir, "files"), - exclude_rels=templated, - ) - - files_var = _build_managed_files_var( - managed_files, - templated, - notify_other=None, - notify_systemd=None, - ) - - dirs_var = _build_managed_dirs_var(managed_dirs) - - jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} - vars_map: Dict[str, Any] = { - f"{var_prefix}_managed_files": files_var, - f"{var_prefix}_managed_dirs": dirs_var, - } - vars_map = _merge_mappings_overwrite(vars_map, jt_map) - - if site_mode: - _write_role_defaults( - role_dir, - {f"{var_prefix}_managed_files": [], f"{var_prefix}_managed_dirs": []}, - ) - _write_hostvars(out_dir, fqdn or "", role, vars_map) - else: - _write_role_defaults(role_dir, vars_map) - - tasks = "---\n" + _render_generic_files_tasks( - var_prefix, include_restart_notify=False - ) - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks.rstrip() + "\n") - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\ndependencies: []\n") - - # README: summarise repos and GPG key material - repo_paths: List[str] = [] - key_paths: List[str] = [] - repo_hosts: Set[str] = set() - - url_re = re.compile(r"(?:https?|ftp)://([^/\s]+)", re.IGNORECASE) - file_url_re = re.compile(r"file://(/[^\s]+)") - - for mf in managed_files: - p = str(mf.get("path") or "") - src_rel = str(mf.get("src_rel") or "") - if not p or not src_rel: - continue - - if p.startswith("/etc/yum.repos.d/") and p.endswith(".repo"): - repo_paths.append(p) - art_path = os.path.join(bundle_dir, "artifacts", role, src_rel) - try: - with open(art_path, "r", encoding="utf-8", errors="replace") as rf: - for line in rf: - s = line.strip() - if not s or s.startswith("#") or s.startswith(";"): - continue - # Collect hostnames from URLs (baseurl, mirrorlist, metalink, gpgkey...) - for m in url_re.finditer(s): - repo_hosts.add(m.group(1)) - # Collect local gpgkey file paths referenced as file:///... - for m in file_url_re.finditer(s): - key_paths.append(m.group(1)) - except OSError: - pass # nosec - - if p.startswith("/etc/pki/rpm-gpg/"): - key_paths.append(p) - - repo_paths = sorted(set(repo_paths)) - key_paths = sorted(set(key_paths)) - repos = sorted(repo_hosts) - - readme = ( - """# dnf_config - -DNF/YUM configuration harvested from the system (repos, config files, and RPM GPG keys). - -## Repository hosts -""" - + ("\n".join([f"- {h}" for h in repos]) or "- (none)") - + """\n -## Repo files -""" - + ("\n".join([f"- {p}" for p in repo_paths]) or "- (none)") - + """\n -## GPG keys -""" - + ("\n".join([f"- {p}" for p in key_paths]) or "- (none)") - + """\n -## Managed files -""" - + ( - "\n".join( - [f"- {mf.get('path')} ({mf.get('reason')})" for mf in managed_files] - ) - or "- (none)" - ) - + """\n -## Excluded -""" - + ( - "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) - or "- (none)" - ) - + """\n -## Notes -""" - + ("\n".join([f"- {n}" for n in notes]) or "- (none)") - + """\n""" - ) - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_dnf_config_roles.append(role) - - # ------------------------- - # sysctl role (live writable sysctl state) - # ------------------------- - if sysctl_snapshot and (sysctl_snapshot.get("managed_files") or []): - role = sysctl_snapshot.get("role_name", "sysctl") - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - - var_prefix = role - managed_files = sysctl_snapshot.get("managed_files", []) or [] - conf_src_rel = "" - for mf in managed_files: - if mf.get("path") == "/etc/sysctl.d/99-enroll.conf": - conf_src_rel = mf.get("src_rel") or "" - break - if not conf_src_rel and managed_files: - conf_src_rel = managed_files[0].get("src_rel") or "" - - parameters = sysctl_snapshot.get("parameters", {}) or {} - notes = sysctl_snapshot.get("notes", []) or [] - - # Generated sysctl snapshots are host-specific in site mode. - if site_mode: - _copy_artifacts( - bundle_dir, - role, - _host_role_files_dir(out_dir, fqdn or "", role), - ) - else: - _copy_artifacts(bundle_dir, role, os.path.join(role_dir, "files")) - - vars_map: Dict[str, Any] = { - f"{var_prefix}_conf_src_rel": conf_src_rel, - f"{var_prefix}_apply": True, - f"{var_prefix}_ignore_apply_errors": True, - } - - if site_mode: - _write_role_defaults( - role_dir, - { - f"{var_prefix}_conf_src_rel": "", - f"{var_prefix}_apply": True, - f"{var_prefix}_ignore_apply_errors": True, - }, - ) - _write_hostvars(out_dir, fqdn or "", role, vars_map) - else: - _write_role_defaults(role_dir, vars_map) - - tasks = "---\n" + _render_sysctl_tasks(var_prefix) - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks.rstrip() + "\n") - - handlers_dir = os.path.join(role_dir, "handlers") - os.makedirs(handlers_dir, exist_ok=True) - with open(os.path.join(handlers_dir, "main.yml"), "w", encoding="utf-8") as f: - f.write(_render_sysctl_handlers(var_prefix)) - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\ndependencies: []\n") - - param_count = len(parameters) if isinstance(parameters, dict) else 0 - sample_params = [] - if isinstance(parameters, dict): - sample_params = sorted(parameters.keys())[:25] - - readme = f"""# {role} - -Generated from live writable sysctl state captured during harvest. - -This role deploys the captured values to `/etc/sysctl.d/99-enroll.conf`. The generated file intentionally contains writable, single-line sysctl values only; read-only, multiline, action-like, and host-identity keys are skipped to avoid creating a noisy or brittle boot-time configuration. - -## Captured parameters - -Captured parameter count: {param_count} - -{os.linesep.join("- " + x for x in sample_params) or "- (none)"} - -{"- ..." if param_count > len(sample_params) else ""} - -## Notes -{os.linesep.join("- " + n for n in notes) or "- (none)"} - -## Safety notes -- `sysctl_apply` defaults to `true` and applies `/etc/sysctl.d/99-enroll.conf` when the file changes. -- `sysctl_ignore_apply_errors` defaults to `true`, because sysctl availability and writability can vary across kernels, containers, and hardware. -- Review this role before applying it broadly across unlike hosts. -""" - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_sysctl_roles.append(role) - - # ------------------------- - # firewall_runtime role (live ipset/iptables kernel state) - # ------------------------- - if firewall_runtime_snapshot and ( - firewall_runtime_snapshot.get("ipset_save") - or firewall_runtime_snapshot.get("iptables_v4_save") - or firewall_runtime_snapshot.get("iptables_v6_save") - ): - role = firewall_runtime_snapshot.get("role_name", "firewall_runtime") - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - - var_prefix = role - packages = firewall_runtime_snapshot.get("packages", []) or [] - ipset_save = firewall_runtime_snapshot.get("ipset_save") or "" - ipset_sets = firewall_runtime_snapshot.get("ipset_sets", []) or [] - iptables_v4_save = firewall_runtime_snapshot.get("iptables_v4_save") or "" - iptables_v6_save = firewall_runtime_snapshot.get("iptables_v6_save") or "" - notes = firewall_runtime_snapshot.get("notes", []) or [] - - # Generated firewall snapshots are host-specific in site mode. - if site_mode: - _copy_artifacts( - bundle_dir, - role, - _host_role_files_dir(out_dir, fqdn or "", role), - ) - else: - _copy_artifacts(bundle_dir, role, os.path.join(role_dir, "files")) - - vars_map: Dict[str, Any] = { - f"{var_prefix}_packages": packages, - f"{var_prefix}_ipset_save": ipset_save, - f"{var_prefix}_ipset_sets": ipset_sets, - f"{var_prefix}_iptables_v4_save": iptables_v4_save, - f"{var_prefix}_iptables_v6_save": iptables_v6_save, - f"{var_prefix}_sync_ipsets_exact": True, - f"{var_prefix}_restore_iptables": True, - } - - if site_mode: - _write_role_defaults( - role_dir, - { - f"{var_prefix}_packages": [], - f"{var_prefix}_ipset_save": "", - f"{var_prefix}_ipset_sets": [], - f"{var_prefix}_iptables_v4_save": "", - f"{var_prefix}_iptables_v6_save": "", - f"{var_prefix}_sync_ipsets_exact": True, - f"{var_prefix}_restore_iptables": True, - }, - ) - _write_hostvars(out_dir, fqdn or "", role, vars_map) - else: - _write_role_defaults(role_dir, vars_map) - - tasks = ( - "---\n" - + _render_install_packages_tasks(role, var_prefix) - + _render_firewall_runtime_tasks(var_prefix) - ) - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks.rstrip() + "\n") - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\ndependencies: []\n") - - readme = f"""# {role} - -Generated from live firewall runtime state captured during harvest. - -This role restores live ipset and iptables state only for firewall families where Enroll did not find corresponding persistent configuration on the source host. Static firewall configuration files, such as `/etc/iptables/rules.v4`, `/etc/iptables/rules.v6`, UFW, nftables, firewalld, or `/etc/ipset*`, are harvested separately as managed files and treated as authoritative for their respective family. - -## Captured snapshots -- ipset: {ipset_save or "(none)"} -- iptables IPv4: {iptables_v4_save or "(none)"} -- iptables IPv6: {iptables_v6_save or "(none)"} - -## Captured ipsets -{os.linesep.join("- " + x for x in ipset_sets) or "- (none)"} - -## Notes -{os.linesep.join("- " + n for n in notes) or "- (none)"} - -## Safety notes -- `firewall_runtime_sync_ipsets_exact` defaults to `true`; it flushes captured set members before replaying the saved members so stale entries are removed. This applies only when no persistent ipset config was found. -- `firewall_runtime_restore_iptables` defaults to `true`; `iptables-restore`/`ip6tables-restore` replace only captured families. A family is captured only when no corresponding persistent iptables config was found. -""" - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_firewall_runtime_roles.append(role) - - # ------------------------- - # etc_custom role (unowned /etc not already attributed) - # ------------------------- - if etc_custom_snapshot and etc_custom_snapshot.get("managed_files"): - role = etc_custom_snapshot.get("role_name", "etc_custom") - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - - var_prefix = role - - managed_files = etc_custom_snapshot.get("managed_files", []) - managed_dirs = etc_custom_snapshot.get("managed_dirs", []) or [] - excluded = etc_custom_snapshot.get("excluded", []) - notes = etc_custom_snapshot.get("notes", []) - - templated, jt_vars = _jinjify_managed_files( - bundle_dir, - role, - role_dir, - managed_files, - jt_exe=jt_exe, - jt_enabled=jt_enabled, - overwrite_templates=not site_mode, - ) - - # Copy only the non-templated artifacts (templates live in the role). - if site_mode: - _copy_artifacts( - bundle_dir, - role, - _host_role_files_dir(out_dir, fqdn or "", role), - exclude_rels=templated, - ) - else: - _copy_artifacts( - bundle_dir, - role, - os.path.join(role_dir, "files"), - exclude_rels=templated, - ) - - files_var = _build_managed_files_var( - managed_files, - templated, - notify_other=None, - notify_systemd="Run systemd daemon-reload", - ) - - dirs_var = _build_managed_dirs_var(managed_dirs) - - jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} - vars_map: Dict[str, Any] = { - f"{var_prefix}_managed_files": files_var, - f"{var_prefix}_managed_dirs": dirs_var, - } - vars_map = _merge_mappings_overwrite(vars_map, jt_map) - - if site_mode: - _write_role_defaults( - role_dir, - {f"{var_prefix}_managed_files": [], f"{var_prefix}_managed_dirs": []}, - ) - _write_hostvars(out_dir, fqdn or "", role, vars_map) - else: - _write_role_defaults(role_dir, vars_map) - - tasks = "---\n" + _render_generic_files_tasks( - var_prefix, include_restart_notify=False - ) - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks.rstrip() + "\n") - - handlers = """--- -- name: Run systemd daemon-reload - ansible.builtin.systemd: - daemon_reload: true - no_log: "{{ enroll_hide_systemd_status | default(true) | bool }}" -""" - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(handlers) - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\ndependencies: []\n") - - readme = ( - """# etc_custom - -Unowned /etc config files not attributed to packages or services. - -## Managed files -""" - + ("\n".join([f"- {mf.get('path')}" for mf in managed_files]) or "- (none)") - + """\n -## Excluded -""" - + ( - "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) - or "- (none)" - ) - + """\n -## Notes -""" - + ("\n".join([f"- {n}" for n in notes]) or "- (none)") - + """\n""" - ) - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_etc_custom_roles.append(role) - - # ------------------------- - - # ------------------------- - - # ------------------------- - # usr_local_custom role (/usr/local/etc + /usr/local/bin scripts) - # ------------------------- - if usr_local_custom_snapshot and usr_local_custom_snapshot.get("managed_files"): - role = usr_local_custom_snapshot.get("role_name", "usr_local_custom") - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - - var_prefix = role - - managed_files = usr_local_custom_snapshot.get("managed_files", []) - managed_dirs = usr_local_custom_snapshot.get("managed_dirs", []) or [] - excluded = usr_local_custom_snapshot.get("excluded", []) - notes = usr_local_custom_snapshot.get("notes", []) - - templated, jt_vars = _jinjify_managed_files( - bundle_dir, - role, - role_dir, - managed_files, - jt_exe=jt_exe, - jt_enabled=jt_enabled, - overwrite_templates=not site_mode, - ) - - # Copy only the non-templated artifacts (templates live in the role). - if site_mode: - _copy_artifacts( - bundle_dir, - role, - _host_role_files_dir(out_dir, fqdn or "", role), - exclude_rels=templated, - ) - else: - _copy_artifacts( - bundle_dir, - role, - os.path.join(role_dir, "files"), - exclude_rels=templated, - ) - - files_var = _build_managed_files_var( - managed_files, - templated, - notify_other=None, - notify_systemd=None, - ) - - dirs_var = _build_managed_dirs_var(managed_dirs) - - jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} - vars_map: Dict[str, Any] = { - f"{var_prefix}_managed_files": files_var, - f"{var_prefix}_managed_dirs": dirs_var, - } - vars_map = _merge_mappings_overwrite(vars_map, jt_map) - - if site_mode: - _write_role_defaults( - role_dir, - {f"{var_prefix}_managed_files": [], f"{var_prefix}_managed_dirs": []}, - ) - _write_hostvars(out_dir, fqdn or "", role, vars_map) - else: - _write_role_defaults(role_dir, vars_map) - - tasks = "---\n" + _render_generic_files_tasks( - var_prefix, include_restart_notify=False - ) - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks.rstrip() + "\n") - - # No handlers needed for this role, but keep a valid YAML document. - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\n") - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\ndependencies: []\n") - - readme = ( - """# usr_local_custom\n\n""" - "Unowned /usr/local files (scripts in /usr/local/bin and config under /usr/local/etc).\n\n" - "## Managed files\n" - + ("\n".join([f"- {mf.get('path')}" for mf in managed_files]) or "- (none)") - + "\n\n## Excluded\n" - + ( - "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) - or "- (none)" - ) - + "\n\n## Notes\n" - + ("\n".join([f"- {n}" for n in notes]) or "- (none)") - + "\n" - ) - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_usr_local_custom_roles.append(role) - - # ------------------------- - # extra_paths role (user-requested includes) - # ------------------------- - if extra_paths_snapshot and ( - extra_paths_snapshot.get("managed_files") - or extra_paths_snapshot.get("managed_dirs") - ): - role = extra_paths_snapshot.get("role_name", "extra_paths") - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - - var_prefix = role - - managed_dirs = extra_paths_snapshot.get("managed_dirs", []) or [] - managed_files = extra_paths_snapshot.get("managed_files", []) - excluded = extra_paths_snapshot.get("excluded", []) - notes = extra_paths_snapshot.get("notes", []) - include_pats = extra_paths_snapshot.get("include_patterns", []) or [] - exclude_pats = extra_paths_snapshot.get("exclude_patterns", []) or [] - - templated, jt_vars = _jinjify_managed_files( - bundle_dir, - role, - role_dir, - managed_files, - jt_exe=jt_exe, - jt_enabled=jt_enabled, - overwrite_templates=not site_mode, - ) - - if site_mode: - _copy_artifacts( - bundle_dir, - role, - _host_role_files_dir(out_dir, fqdn or "", role), - exclude_rels=templated, - ) - else: - _copy_artifacts( - bundle_dir, - role, - os.path.join(role_dir, "files"), - exclude_rels=templated, - ) - - files_var = _build_managed_files_var( - managed_files, - templated, - notify_other=None, - notify_systemd=None, - ) - - dirs_var = _build_managed_dirs_var(managed_dirs) - - jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} - vars_map: Dict[str, Any] = { - f"{var_prefix}_managed_dirs": dirs_var, - f"{var_prefix}_managed_files": files_var, - } - vars_map = _merge_mappings_overwrite(vars_map, jt_map) - - if site_mode: - _write_role_defaults( - role_dir, - { - f"{var_prefix}_managed_dirs": [], - f"{var_prefix}_managed_files": [], - }, - ) - _write_hostvars(out_dir, fqdn or "", role, vars_map) - else: - _write_role_defaults(role_dir, vars_map) - - tasks = "---\n" + _render_generic_files_tasks( - var_prefix, include_restart_notify=False - ) - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks.rstrip() + "\n") - - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\n") - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\ndependencies: []\n") - - readme = ( - f"""# {role} - -User-requested extra file harvesting. - -## Include patterns -""" - + ("\n".join([f"- {p}" for p in include_pats]) or "- (none)") - + """\n -## Exclude patterns -""" - + ("\n".join([f"- {p}" for p in exclude_pats]) or "- (none)") - + """\n -## Managed directories -""" - + ("\n".join([f"- {d.get('path')}" for d in managed_dirs]) or "- (none)") - + """\n -## Managed files -""" - + ("\n".join([f"- {mf.get('path')}" for mf in managed_files]) or "- (none)") - + """\n -## Excluded -""" - + ( - "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) - or "- (none)" - ) - + """\n -## Notes -""" - + ("\n".join([f"- {n}" for n in notes]) or "- (none)") - + """\n""" - ) - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_extra_paths_roles.append(role) - - # ------------------------- - # Service roles - # ------------------------- - for svc in services_to_manifest: - source_role = svc["role_name"] - role = avoid_reserved_role_name(source_role, prefix="service") - unit = svc["unit"] - pkgs = svc.get("packages", []) or [] - managed_files = svc.get("managed_files", []) or [] - managed_dirs = svc.get("managed_dirs", []) or [] - managed_links = svc.get("managed_links", []) or [] - - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - - var_prefix = role - - was_active = svc.get("active_state") == "active" - unit_file_state = str(svc.get("unit_file_state") or "") - enabled_at_harvest = unit_file_state in ("enabled", "enabled-runtime") - desired_state = "started" if was_active else "stopped" - - templated, jt_vars = _jinjify_managed_files( - bundle_dir, - source_role, - role_dir, - managed_files, - jt_exe=jt_exe, - jt_enabled=jt_enabled, - overwrite_templates=not site_mode, - ) - - # Copy only the non-templated artifacts. - if site_mode: - _copy_artifacts( - bundle_dir, - source_role, - _host_role_files_dir(out_dir, fqdn or "", role), - exclude_rels=templated, - ) - else: - _copy_artifacts( - bundle_dir, - source_role, - os.path.join(role_dir, "files"), - exclude_rels=templated, - ) - - files_var = _build_managed_files_var( - managed_files, - templated, - notify_other="Restart service", - notify_systemd="Run systemd daemon-reload", - ) - - links_var = _build_managed_links_var(managed_links) - - dirs_var = _build_managed_dirs_var(managed_dirs) - - jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} - base_vars: Dict[str, Any] = { - f"{var_prefix}_unit_name": unit, - f"{var_prefix}_packages": pkgs, - f"{var_prefix}_managed_files": files_var, - f"{var_prefix}_managed_dirs": dirs_var, - f"{var_prefix}_managed_links": links_var, - f"{var_prefix}_manage_unit": True, - f"{var_prefix}_systemd_enabled": bool(enabled_at_harvest), - f"{var_prefix}_systemd_state": desired_state, - } - base_vars = _merge_mappings_overwrite(base_vars, jt_map) - - if site_mode: - # Role defaults are host-agnostic/safe; all harvested state is in host_vars. - _write_role_defaults( - role_dir, - { - f"{var_prefix}_unit_name": unit, - f"{var_prefix}_packages": [], - f"{var_prefix}_managed_files": [], - f"{var_prefix}_managed_dirs": [], - f"{var_prefix}_managed_links": [], - f"{var_prefix}_manage_unit": False, - f"{var_prefix}_systemd_enabled": False, - f"{var_prefix}_systemd_state": "stopped", - }, - ) - _write_hostvars(out_dir, fqdn or "", role, base_vars) - else: - _write_role_defaults(role_dir, base_vars) - - handlers = f"""--- -- name: Run systemd daemon-reload - ansible.builtin.systemd: - daemon_reload: true - no_log: "{{ enroll_hide_systemd_status | default(true) | bool }}" - -- name: Restart service - ansible.builtin.service: - name: "{{{{ {var_prefix}_unit_name }}}}" - state: restarted - when: - - {var_prefix}_manage_unit | default(false) - - ({var_prefix}_systemd_state | default('stopped')) == 'started' -""" - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(handlers) - - task_parts: List[str] = [] - task_parts.append("---\n" + _render_install_packages_tasks(role, var_prefix)) - - task_parts.append( - _render_generic_files_tasks(var_prefix, include_restart_notify=True) - ) - - task_parts.append( - f"""- name: Probe whether systemd unit exists and is manageable - ansible.builtin.systemd: - name: "{{{{ {var_prefix}_unit_name }}}}" - no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" - check_mode: true - register: _unit_probe - failed_when: false - changed_when: false - when: {var_prefix}_manage_unit | default(false) - -- name: Ensure unit enablement matches harvest - ansible.builtin.systemd: - name: "{{{{ {var_prefix}_unit_name }}}}" - enabled: "{{{{ {var_prefix}_systemd_enabled | bool }}}}" - no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" - when: - - {var_prefix}_manage_unit | default(false) - - _unit_probe is succeeded - -- name: Ensure unit running state matches harvest - ansible.builtin.systemd: - name: "{{{{ {var_prefix}_unit_name }}}}" - state: "{{{{ {var_prefix}_systemd_state }}}}" - no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" - when: - - {var_prefix}_manage_unit | default(false) - - _unit_probe is succeeded -""" - ) - - tasks = "\n".join(task_parts).rstrip() + "\n" - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks) - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\ndependencies: []\n") - - excluded = svc.get("excluded", []) - notes = svc.get("notes", []) - readme = f"""# {role} - -Generated from `{unit}`. - -## Packages -{os.linesep.join("- " + p for p in pkgs) or "- (none detected)"} - -## Managed files -{os.linesep.join("- " + mf["path"] + " (" + mf["reason"] + ")" for mf in managed_files) or "- (none)"} - -## Managed symlinks -{os.linesep.join("- " + ml["path"] + " -> " + ml["target"] + " (" + ml.get("reason", "") + ")" for ml in managed_links) or "- (none)"} - -## Excluded (possible secrets / unsafe) -{os.linesep.join("- " + e["path"] + " (" + e["reason"] + ")" for e in excluded) or "- (none)"} - -## Notes -{os.linesep.join("- " + n for n in notes) or "- (none)"} -""" - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_service_roles.append(role) - - # ------------------------- - # Common package section/group roles - # - # Outside --fqdn/site mode, package and systemd-unit roles are grouped by - # Debian Section or RPM Group by default. Managed config and unit state can - # live in those section roles too; --no-common-roles preserves the historic - # one-role-per-package/unit output, and --fqdn implies that mode because - # grouped role contents would be unsafe across multiple harvested hosts. - # ------------------------- - if use_common_roles: - for pr in package_roles: - label = _package_section_label(pr, inventory_packages) - common_role_groups.setdefault(label, []).append( - {"kind": "package", "snapshot": pr} - ) - package_roles = [] - - # ------------------------- - # Manually installed package roles - # ------------------------- - occupied_roles: Set[str] = set( - manifested_apt_config_roles - + manifested_dnf_config_roles - + manifested_users_roles - + manifested_flatpak_roles - + manifested_snap_roles - + manifested_service_roles - + manifested_firewall_runtime_roles - + manifested_sysctl_roles - + manifested_etc_custom_roles - + manifested_usr_local_custom_roles - + manifested_extra_paths_roles - ) - for pr in package_roles: - occupied_roles.add( - avoid_reserved_role_name(str(pr.get("role_name") or ""), prefix="package") - ) - - for section_label, entries in sorted(common_role_groups.items()): - role = _section_role_name(section_label, occupied_roles) - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - - var_prefix = role - packages_set: Set[str] = set() - files_var: List[Dict[str, Any]] = [] - dirs_var: List[Dict[str, Any]] = [] - links_var: List[Dict[str, Any]] = [] - systemd_units: List[Dict[str, Any]] = [] - excluded_all: List[Dict[str, Any]] = [] - notes_all: List[str] = [] - origin_lines: List[str] = [] - jt_combined: Dict[str, Any] = {} - - seen_files: Set[tuple] = set() - seen_dirs: Set[tuple] = set() - seen_links: Set[tuple] = set() - seen_units: Set[str] = set() - - for entry in entries: - kind = entry.get("kind") or "package" - snap = entry.get("snapshot") or {} - source_role = str(snap.get("role_name") or "") - managed_files = snap.get("managed_files", []) or [] - managed_dirs = snap.get("managed_dirs", []) or [] - managed_links = snap.get("managed_links", []) or [] - excluded = snap.get("excluded", []) or [] - notes = snap.get("notes", []) or [] - - if kind == "service": - pkgs = snap.get("packages", []) or [] - unit = str(snap.get("unit") or "") - origin_lines.append(f"service `{unit}` from role `{source_role}`") - else: - pkg = str(snap.get("package") or "") - pkgs = [pkg] if pkg else [] - origin_lines.append(f"package `{pkg}` from role `{source_role}`") - - for pkg in pkgs: - if pkg: - packages_set.add(str(pkg)) - - templated: Set[str] = set() - jt_vars = "" - if managed_files and source_role: - templated, jt_vars = _jinjify_managed_files( - bundle_dir, - source_role, - role_dir, - managed_files, - jt_exe=jt_exe, - jt_enabled=jt_enabled, - overwrite_templates=True, - ) - - _copy_artifacts( - bundle_dir, - source_role, - os.path.join(role_dir, "files"), - exclude_rels=templated, - ) - - notify_other = "Restart managed services" if kind == "service" else None - for item in _build_managed_files_var( - managed_files, - templated, - notify_other=notify_other, - notify_systemd="Run systemd daemon-reload", - ): - key = (item.get("dest"), item.get("src_rel"), item.get("kind")) - if key not in seen_files: - seen_files.add(key) - files_var.append(item) - - for item in _build_managed_dirs_var(managed_dirs): - key = ( - item.get("dest"), - item.get("owner"), - item.get("group"), - item.get("mode"), - ) - if key not in seen_dirs: - seen_dirs.add(key) - dirs_var.append(item) - - for item in _build_managed_links_var(managed_links): - key = (item.get("dest"), item.get("src")) - if key not in seen_links: - seen_links.add(key) - links_var.append(item) - - if kind == "service": - unit = str(snap.get("unit") or "") - if unit and unit not in seen_units: - seen_units.add(unit) - unit_file_state = str(snap.get("unit_file_state") or "") - enabled_at_harvest = unit_file_state in ( - "enabled", - "enabled-runtime", - ) - desired_state = ( - "started" if snap.get("active_state") == "active" else "stopped" - ) - systemd_units.append( - { - "name": unit, - "manage": True, - "enabled": bool(enabled_at_harvest), - "state": desired_state, - } - ) - - excluded_all.extend(excluded) - notes_all.extend(str(n) for n in notes) - - jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} - jt_combined = _merge_mappings_overwrite(jt_combined, jt_map) - - packages = sorted(packages_set) - files_var = sorted(files_var, key=lambda x: str(x.get("dest") or "")) - dirs_var = sorted(dirs_var, key=lambda x: str(x.get("dest") or "")) - links_var = sorted(links_var, key=lambda x: str(x.get("dest") or "")) - systemd_units = sorted(systemd_units, key=lambda x: str(x.get("name") or "")) - - base_vars: Dict[str, Any] = { - f"{var_prefix}_packages": packages, - f"{var_prefix}_managed_files": files_var, - f"{var_prefix}_managed_dirs": dirs_var, - f"{var_prefix}_managed_links": links_var, - f"{var_prefix}_systemd_units": systemd_units, - } - base_vars = _merge_mappings_overwrite(base_vars, jt_combined) - - _write_role_defaults(role_dir, base_vars) - - if {"cron", "logrotate"}.intersection(packages_set): - common_tail_roles.append(role) - - handlers = ( - """--- -- name: Run systemd daemon-reload - ansible.builtin.systemd: - daemon_reload: true - no_log: "{{ enroll_hide_systemd_status | default(true) | bool }}" - -- name: Restart managed services - ansible.builtin.service: - name: "{{ item.name }}" - state: restarted - loop: "{{ """ - + f"{var_prefix}_systemd_units" - + """ | default([]) }}" - when: - - item.manage | default(false) - - (item.state | default('stopped')) == 'started' -""" - ) - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(handlers) - - task_parts: List[str] = [] - task_parts.append("---\n" + _render_install_packages_tasks(role, var_prefix)) - task_parts.append( - _render_generic_files_tasks(var_prefix, include_restart_notify=True) - ) - task_parts.append(_render_grouped_systemd_tasks(var_prefix)) - - tasks = "\n".join(task_parts).rstrip() + "\n" - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks) - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\ndependencies: []\n") - - readme = f"""# {role} - -Common role for package section/group `{section_label}`. - -## Origin roles -{os.linesep.join("- " + line for line in sorted(origin_lines)) or "- (none)"} - -## Packages -{os.linesep.join("- " + p for p in packages) or "- (none)"} - -## Managed files -{os.linesep.join("- " + mf["dest"] for mf in files_var) or "- (none)"} - -## Managed symlinks -{os.linesep.join("- " + ml["dest"] + " -> " + ml["src"] for ml in links_var) or "- (none)"} - -## Systemd units -{os.linesep.join("- " + u["name"] + " (enabled=" + str(u["enabled"]).lower() + ", state=" + u["state"] + ")" for u in systemd_units) or "- (none)"} - -## Excluded (possible secrets / unsafe) -{os.linesep.join("- " + e.get("path", "") + " (" + e.get("reason", "") + ")" for e in excluded_all) or "- (none)"} - -## Notes -{os.linesep.join("- " + n for n in notes_all) or "- (none)"} -""" - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_pkg_roles.append(role) - - # Process package roles (those with configuration files) - for pr in package_roles: - source_role = pr["role_name"] - role = avoid_reserved_role_name(source_role, prefix="package") - pkg = pr.get("package") or "" - managed_files = pr.get("managed_files", []) or [] - managed_dirs = pr.get("managed_dirs", []) or [] - managed_links = pr.get("managed_links", []) or [] - - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - - var_prefix = role - - templated, jt_vars = _jinjify_managed_files( - bundle_dir, - source_role, - role_dir, - managed_files, - jt_exe=jt_exe, - jt_enabled=jt_enabled, - overwrite_templates=not site_mode, - ) - - # Copy only the non-templated artifacts. - if site_mode: - _copy_artifacts( - bundle_dir, - source_role, - _host_role_files_dir(out_dir, fqdn or "", role), - exclude_rels=templated, - ) - else: - _copy_artifacts( - bundle_dir, - source_role, - os.path.join(role_dir, "files"), - exclude_rels=templated, - ) - - pkgs = [pkg] if pkg else [] - - files_var = _build_managed_files_var( - managed_files, - templated, - notify_other=None, - notify_systemd="Run systemd daemon-reload", - ) - - links_var = _build_managed_links_var(managed_links) - - dirs_var = _build_managed_dirs_var(managed_dirs) - - jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} - base_vars: Dict[str, Any] = { - f"{var_prefix}_packages": pkgs, - f"{var_prefix}_managed_files": files_var, - f"{var_prefix}_managed_dirs": dirs_var, - f"{var_prefix}_managed_links": links_var, - } - base_vars = _merge_mappings_overwrite(base_vars, jt_map) - - if site_mode: - _write_role_defaults( - role_dir, - { - f"{var_prefix}_packages": [], - f"{var_prefix}_managed_files": [], - f"{var_prefix}_managed_dirs": [], - f"{var_prefix}_managed_links": [], - }, - ) - _write_hostvars(out_dir, fqdn or "", role, base_vars) - else: - _write_role_defaults(role_dir, base_vars) - - handlers = """--- -- name: Run systemd daemon-reload - ansible.builtin.systemd: - daemon_reload: true - no_log: "{{ enroll_hide_systemd_status | default(true) | bool }}" -""" - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(handlers) - - task_parts: List[str] = [] - task_parts.append("---\n" + _render_install_packages_tasks(role, var_prefix)) - task_parts.append( - _render_generic_files_tasks(var_prefix, include_restart_notify=False) - ) - - tasks = "\n".join(task_parts).rstrip() + "\n" - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks) - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\ndependencies: []\n") - - excluded = pr.get("excluded", []) - notes = pr.get("notes", []) - readme = f"""# {role} - -Generated for package `{pkg}`. - -## Managed files -{os.linesep.join("- " + mf["path"] + " (" + mf["reason"] + ")" for mf in managed_files) or "- (none)"} - -## Managed symlinks -{os.linesep.join("- " + ml["path"] + " -> " + ml["target"] + " (" + ml.get("reason", "") + ")" for ml in managed_links) or "- (none)"} - -## Excluded (possible secrets / unsafe) -{os.linesep.join("- " + e["path"] + " (" + e["reason"] + ")" for e in excluded) or "- (none)"} - -## Notes -{os.linesep.join("- " + n for n in notes) or "- (none)"} - -> Note: package roles (those not discovered via a systemd service) do not attempt to restart or enable services automatically. -""" - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_pkg_roles.append(role) - # Place cron/logrotate at the end of the playbook so: - # - users exist before we restore per-user crontabs in /var/spool - # - most packages/services are installed/configured first - tail_roles: List[str] = [] - for r in ("cron", "logrotate"): - if r in manifested_pkg_roles: - tail_roles.append(r) - for r in common_tail_roles: - if r in manifested_pkg_roles and r not in tail_roles: - tail_roles.append(r) - - main_pkg_roles = [r for r in manifested_pkg_roles if r not in set(tail_roles)] - - all_roles = ( - manifested_apt_config_roles - + manifested_dnf_config_roles - + main_pkg_roles - + manifested_service_roles - + manifested_etc_custom_roles - + manifested_usr_local_custom_roles - + manifested_extra_paths_roles - + manifested_flatpak_roles - + manifested_snap_roles - + manifested_users_roles - + tail_roles - + manifested_sysctl_roles - + manifested_firewall_runtime_roles - ) - - if site_mode: - _write_playbook_host( - os.path.join(out_dir, "playbooks", f"{fqdn}.yml"), fqdn or "", all_roles - ) - else: - _write_playbook_all(os.path.join(out_dir, "playbook.yml"), all_roles) - - def manifest( bundle_dir: str, out: str, @@ -3288,7 +211,7 @@ def manifest( no_common_roles=no_common_roles, ) else: - _manifest_ansible_from_bundle_dir( + manifest_ansible_from_bundle_dir( resolved_bundle_dir, out, fqdn=fqdn, @@ -3316,7 +239,7 @@ def manifest( no_common_roles=no_common_roles, ) else: - _manifest_ansible_from_bundle_dir( + manifest_ansible_from_bundle_dir( resolved_bundle_dir, str(tmp_out), fqdn=fqdn, diff --git a/enroll/puppet.py b/enroll/puppet.py index e62cb6e..2993e02 100644 --- a/enroll/puppet.py +++ b/enroll/puppet.py @@ -1,16 +1,137 @@ from __future__ import annotations import json -import os import re import shutil from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple +from typing import Any, Dict, Iterable, List, Optional, Tuple + +from .cm import ( + CMModule, + package_section_label, + resolve_catalog_conflicts, + role_order_key, + section_label_for_packages, +) +from .state import inventory_packages_from_state, roles_from_state -def _load_state(bundle_dir: str) -> Dict[str, Any]: - with open(os.path.join(bundle_dir, "state.json"), "r", encoding="utf-8") as f: - return json.load(f) +class PuppetRole(CMModule): + """Puppet-specific view of a renderer-neutral CMModule.""" + + def __init__(self, role_name: str) -> None: + super().__init__( + role_name=role_name, + module_name=_puppet_name(role_name, fallback="enroll_role"), + ) + + def add_package_snapshot(self, snap: Dict[str, Any]) -> None: + pkg = str(snap.get("package") or "").strip() + if pkg: + self.packages.add(pkg) + + def add_service_snapshot(self, snap: Dict[str, Any]) -> None: + for pkg in snap.get("packages", []) or []: + pkg_s = str(pkg or "").strip() + if pkg_s: + self.packages.add(pkg_s) + unit = str(snap.get("unit") or "").strip() + if unit: + unit_file_state = str(snap.get("unit_file_state") or "") + self.services[unit] = { + "name": unit, + "ensure": ( + "running" if snap.get("active_state") == "active" else "stopped" + ), + "enable": unit_file_state in ("enabled", "enabled-runtime"), + } + + def add_users_snapshot(self, snap: Dict[str, Any]) -> None: + for u in snap.get("users", []) or []: + if not isinstance(u, dict): + continue + name = str(u.get("name") or "").strip() + if not name: + continue + primary_group = str(u.get("primary_group") or name).strip() + if primary_group: + self.groups.add(primary_group) + supplementary = sorted( + { + str(g).strip() + for g in (u.get("supplementary_groups") or []) + if str(g).strip() + } + ) + self.groups.update(supplementary) + self.users[name] = { + "name": name, + "uid": u.get("uid"), + "gid": u.get("gid"), + "primary_group": primary_group or None, + "home": u.get("home") or f"/home/{name}", + "shell": u.get("shell"), + "gecos": u.get("gecos"), + "supplementary_groups": supplementary, + } + + if snap.get("user_flatpaks") or snap.get("user_flatpak_remotes"): + self.notes.append( + "Per-user Flatpak resources were detected but are not yet rendered as native Puppet resources." + ) + + def add_managed_content( + self, + snap: Dict[str, Any], + *, + bundle_dir: str, + artifact_role: str, + module_files_dir: Path, + ) -> None: + for d in self.managed_dirs_from_snapshot(snap): + path = str(d.get("path") or "").strip() + self.add_managed_dir( + path, + owner=d.get("owner") or "root", + group=d.get("group") or "root", + mode=d.get("mode") or "0755", + reason=d.get("reason") or "managed_dir", + ) + + for mf in self.managed_files_from_snapshot(snap): + path = str(mf.get("path") or "").strip() + src_rel = str(mf.get("src_rel") or "").strip() + if not path or not src_rel: + continue + module_rel = _copy_artifact( + bundle_dir, artifact_role, src_rel, module_files_dir + ) + if not module_rel: + self.notes.append( + f"Skipped {path}: harvested artifact {artifact_role}/{src_rel} was not present." + ) + continue + self.add_managed_file( + path, + owner=mf.get("owner") or "root", + group=mf.get("group") or "root", + mode=mf.get("mode") or "0644", + source=_source_uri(self.module_name, module_rel), + reason=mf.get("reason") or "managed_file", + ) + + for ml in self.managed_links_from_snapshot(snap): + path = str(ml.get("path") or "").strip() + target = str(ml.get("target") or "").strip() + if not path or not target: + continue + self.add_managed_link( + path, + target=target, + reason=ml.get("reason") or "managed_link", + ) + + self.remove_directory_resource_conflicts() # https://help.puppet.com/core/current/Content/PuppetCore/lang_reserved_words.htm @@ -99,230 +220,18 @@ def _source_uri(module_name: str, module_rel: str) -> str: return f"puppet:///modules/{module_name}/{module_rel}" -def _roles(state: Dict[str, Any]) -> Dict[str, Any]: - roles = state.get("roles") - return roles if isinstance(roles, dict) else {} - - -def _inventory_packages(state: Dict[str, Any]) -> Dict[str, Any]: - inventory = state.get("inventory") - if not isinstance(inventory, dict): - return {} - packages = inventory.get("packages") - return packages if isinstance(packages, dict) else {} - - -def _package_section_label( - package_role: Dict[str, Any], inventory_packages: Dict[str, Any] -) -> str: - pkg = str(package_role.get("package") or "").strip() - inv = inventory_packages.get(pkg) or {} - candidates: List[str] = [] - for value in (package_role.get("section"), inv.get("section"), inv.get("group")): - if isinstance(value, str) and value.strip(): - candidates.append(value.strip()) - for inst in inv.get("installations", []) or []: - if not isinstance(inst, dict): - continue - for key in ("section", "group"): - value = inst.get(key) - if isinstance(value, str) and value.strip(): - candidates.append(value.strip()) - for value in candidates: - if value.lower() not in {"(none)", "none", "unspecified"}: - return value - return "misc" - - -def _section_label_for_packages( - packages: List[str], inventory_packages: Dict[str, Any] -) -> str: - for pkg in packages or []: - label = _package_section_label({"package": pkg}, inventory_packages) - if label and label.lower() != "misc": - return label - return "misc" - - -class _PuppetRole: - def __init__(self, role_name: str) -> None: - self.role_name = role_name - self.module_name = _puppet_name(role_name, fallback="enroll_role") - self.packages: Set[str] = set() - self.groups: Set[str] = set() - self.users: Dict[str, Dict[str, Any]] = {} - self.dirs: Dict[str, Dict[str, Any]] = {} - self.files: Dict[str, Dict[str, Any]] = {} - self.links: Dict[str, Dict[str, Any]] = {} - self.services: Dict[str, Dict[str, Any]] = {} - self.notes: List[str] = [] - - def has_resources(self) -> bool: - return bool( - self.packages - or self.groups - or self.users - or self.dirs - or self.files - or self.links - or self.services - or self.notes - ) - - -def _role_order_key(role: str) -> tuple[int, str]: - # Keep broadly similar ordering to generated Ansible playbooks: package/config - # scaffolding first, then services/users, then host-specific runtime state. - priority = { - "apt_config": 10, - "dnf_config": 11, - "etc_custom": 80, - "usr_local_custom": 81, - "extra_paths": 82, - "users": 90, - "sysctl": 95, - "firewall_runtime": 99, - } - return (priority.get(role, 50), role) - - -def _add_managed_content( - prole: _PuppetRole, - snap: Dict[str, Any], - *, - bundle_dir: str, - artifact_role: str, - module_files_dir: Path, -) -> None: - for d in snap.get("managed_dirs", []) or []: - if not isinstance(d, dict): - continue - path = str(d.get("path") or "").strip() - if not path: - continue - prole.dirs.setdefault( - path, - { - "owner": d.get("owner") or "root", - "group": d.get("group") or "root", - "mode": d.get("mode") or "0755", - "reason": d.get("reason") or "managed_dir", - }, - ) - - for mf in snap.get("managed_files", []) or []: - if not isinstance(mf, dict): - continue - path = str(mf.get("path") or "").strip() - src_rel = str(mf.get("src_rel") or "").strip() - if not path or not src_rel: - continue - module_rel = _copy_artifact( - bundle_dir, artifact_role, src_rel, module_files_dir - ) - if not module_rel: - prole.notes.append( - f"Skipped {path}: harvested artifact {artifact_role}/{src_rel} was not present." - ) - continue - prole.files.setdefault( - path, - { - "owner": mf.get("owner") or "root", - "group": mf.get("group") or "root", - "mode": mf.get("mode") or "0644", - "source": _source_uri(prole.module_name, module_rel), - "reason": mf.get("reason") or "managed_file", - }, - ) - - for ml in snap.get("managed_links", []) or []: - if not isinstance(ml, dict): - continue - path = str(ml.get("path") or "").strip() - target = str(ml.get("target") or "").strip() - if not path or not target: - continue - prole.links.setdefault( - path, - { - "target": target, - "reason": ml.get("reason") or "managed_link", - }, - ) - - for path in set(prole.files) | set(prole.links): - prole.dirs.pop(path, None) - - -def _build_users_role(prole: _PuppetRole, snap: Dict[str, Any]) -> None: - for u in snap.get("users", []) or []: - if not isinstance(u, dict): - continue - name = str(u.get("name") or "").strip() - if not name: - continue - primary_group = str(u.get("primary_group") or name).strip() - if primary_group: - prole.groups.add(primary_group) - supplementary = sorted( - { - str(g).strip() - for g in (u.get("supplementary_groups") or []) - if str(g).strip() - } - ) - prole.groups.update(supplementary) - prole.users[name] = { - "name": name, - "uid": u.get("uid"), - "gid": u.get("gid"), - "primary_group": primary_group or None, - "home": u.get("home") or f"/home/{name}", - "shell": u.get("shell"), - "gecos": u.get("gecos"), - "supplementary_groups": supplementary, - } - - if snap.get("user_flatpaks") or snap.get("user_flatpak_remotes"): - prole.notes.append( - "Per-user Flatpak resources were detected but are not yet rendered as native Puppet resources." - ) - - -def _build_service_role(prole: _PuppetRole, snap: Dict[str, Any]) -> None: - for pkg in snap.get("packages", []) or []: - pkg_s = str(pkg or "").strip() - if pkg_s: - prole.packages.add(pkg_s) - unit = str(snap.get("unit") or "").strip() - if unit: - unit_file_state = str(snap.get("unit_file_state") or "") - prole.services[unit] = { - "name": unit, - "ensure": "running" if snap.get("active_state") == "active" else "stopped", - "enable": unit_file_state in ("enabled", "enabled-runtime"), - } - - -def _build_package_role(prole: _PuppetRole, snap: Dict[str, Any]) -> None: - pkg = str(snap.get("package") or "").strip() - if pkg: - prole.packages.add(pkg) - - -def _add_flatpak_snap_notes(roles: Dict[str, Any], out: Dict[str, _PuppetRole]) -> None: +def _add_flatpak_snap_notes(roles: Dict[str, Any], out: Dict[str, PuppetRole]) -> None: flatpak = roles.get("flatpak") or {} if isinstance(flatpak, dict) and ( flatpak.get("system_flatpaks") or flatpak.get("remotes") ): - prole = out.setdefault("flatpak", _PuppetRole("flatpak")) + prole = out.setdefault("flatpak", PuppetRole("flatpak")) prole.notes.append( "Flatpak resources were detected but are not yet rendered as native Puppet resources." ) snap = roles.get("snap") or {} if isinstance(snap, dict) and snap.get("system_snaps"): - prole = out.setdefault("snap", _PuppetRole("snap")) + prole = out.setdefault("snap", PuppetRole("snap")) prole.notes.append( "Snap resources were detected but are not yet rendered as native Puppet resources." ) @@ -335,15 +244,15 @@ def _collect_puppet_roles( *, fqdn: Optional[str] = None, no_common_roles: bool = False, -) -> List[_PuppetRole]: - roles = _roles(state) - inventory_packages = _inventory_packages(state) +) -> List[PuppetRole]: + roles = roles_from_state(state) + inventory_packages = inventory_packages_from_state(state) use_common_modules = not fqdn and not no_common_roles - out: Dict[str, _PuppetRole] = {} + out: Dict[str, PuppetRole] = {} - def ensure_role(role_name: str) -> _PuppetRole: + def ensure_role(role_name: str) -> PuppetRole: role_name = _puppet_name(role_name, fallback="enroll_role") - return out.setdefault(role_name, _PuppetRole(role_name)) + return out.setdefault(role_name, PuppetRole(role_name)) for key in ( "apt_config", @@ -361,8 +270,7 @@ def _collect_puppet_roles( ) prole = ensure_role(role_name) module_files_dir = modules_dir / prole.module_name / "files" - _add_managed_content( - prole, + prole.add_managed_content( snap, bundle_dir=bundle_dir, artifact_role=str(snap.get("role_name") or key), @@ -375,9 +283,8 @@ def _collect_puppet_roles( str(users_snap.get("role_name") or "users"), fallback="enroll_role" ) prole = ensure_role(role_name) - _build_users_role(prole, users_snap) - _add_managed_content( - prole, + prole.add_users_snapshot(users_snap) + prole.add_managed_content( users_snap, bundle_dir=bundle_dir, artifact_role=str(users_snap.get("role_name") or "users"), @@ -393,7 +300,7 @@ def _collect_puppet_roles( ) if use_common_modules: role_name = _puppet_name( - _section_label_for_packages( + section_label_for_packages( [ str(p).strip() for p in (svc.get("packages") or []) @@ -406,9 +313,8 @@ def _collect_puppet_roles( else: role_name = original_role_name prole = ensure_role(role_name) - _build_service_role(prole, svc) - _add_managed_content( - prole, + prole.add_service_snapshot(svc) + prole.add_managed_content( svc, bundle_dir=bundle_dir, artifact_role=str(svc.get("role_name") or original_role_name), @@ -424,15 +330,14 @@ def _collect_puppet_roles( ) if use_common_modules: role_name = _puppet_name( - _package_section_label(pkg, inventory_packages), + package_section_label(pkg, inventory_packages), fallback="package_group", ) else: role_name = original_role_name prole = ensure_role(role_name) - _build_package_role(prole, pkg) - _add_managed_content( - prole, + prole.add_package_snapshot(pkg) + prole.add_managed_content( pkg, bundle_dir=bundle_dir, artifact_role=str(pkg.get("role_name") or original_role_name), @@ -459,71 +364,12 @@ def _collect_puppet_roles( _add_flatpak_snap_notes(roles, out) - puppet_roles = sorted(out.values(), key=lambda r: _role_order_key(r.role_name)) - _dedupe_puppet_roles(puppet_roles) + puppet_roles = sorted(out.values(), key=lambda r: role_order_key(r.role_name)) + resolve_catalog_conflicts(puppet_roles) return [r for r in puppet_roles if r.has_resources()] -def _dedupe_puppet_roles(puppet_roles: List[_PuppetRole]) -> None: - """Remove duplicate catalog resources across generated Puppet classes. - - Ansible can repeat the same directory task in multiple roles. Puppet cannot: - a resource title such as File['/etc/default'] may appear only once in the - compiled catalog. Keep the first declaration in manifest order and drop - later duplicates. - """ - - concrete_file_paths: Set[str] = set() - for prole in puppet_roles: - concrete_file_paths.update(prole.files) - concrete_file_paths.update(prole.links) - - seen_packages: Set[str] = set() - seen_groups: Set[str] = set() - seen_users: Set[str] = set() - seen_dirs: Set[str] = set() - seen_files: Set[str] = set() - seen_links: Set[str] = set() - seen_services: Set[str] = set() - - for prole in puppet_roles: - prole.packages = {p for p in prole.packages if p not in seen_packages} - seen_packages.update(prole.packages) - - prole.groups = {g for g in prole.groups if g not in seen_groups} - seen_groups.update(prole.groups) - - prole.users = {k: v for k, v in prole.users.items() if k not in seen_users} - seen_users.update(prole.users) - - prole.dirs = { - k: v - for k, v in prole.dirs.items() - if k not in seen_dirs and k not in concrete_file_paths - } - seen_dirs.update(prole.dirs) - - prole.files = { - k: v - for k, v in prole.files.items() - if k not in seen_files and k not in seen_links - } - seen_files.update(prole.files) - - prole.links = { - k: v - for k, v in prole.links.items() - if k not in seen_links and k not in seen_files - } - seen_links.update(prole.links) - - prole.services = { - k: v for k, v in prole.services.items() if k not in seen_services - } - seen_services.update(prole.services) - - -def _render_role_class(prole: _PuppetRole) -> str: +def _render_role_class(prole: PuppetRole) -> str: has_sysctl_conf = "/etc/sysctl.d/99-enroll.conf" in prole.files if has_sysctl_conf: lines: List[str] = [ @@ -643,7 +489,7 @@ def _render_role_class(prole: _PuppetRole) -> str: return "\n".join(lines) -def _render_site_pp(puppet_roles: List[_PuppetRole], fqdn: Optional[str]) -> str: +def _render_site_pp(puppet_roles: List[PuppetRole], fqdn: Optional[str]) -> str: node_name = _pp_quote(fqdn) if fqdn else "default" if not puppet_roles: return f"node {node_name} {{\n # No Puppet classes were generated from this harvest.\n}}\n" @@ -671,7 +517,7 @@ def _write_metadata(module_dir: Path, module_name: str) -> None: ) -def _render_readme(state: Dict[str, Any], puppet_roles: List[_PuppetRole]) -> str: +def _render_readme(state: Dict[str, Any], puppet_roles: List[PuppetRole]) -> str: host = state.get("host", {}) if isinstance(state.get("host"), dict) else {} hostname = host.get("hostname") or "unknown" role_lines = ( @@ -726,7 +572,7 @@ sudo puppet apply --modulepath /path/to/generated/modules /path/to/generated/man ## Current limitations - Flatpak, Snap, and live firewall runtime snapshots are listed as notes when present rather than rendered as Puppet resources. -- JinjaTurtle templating is Ansible-oriented and is not applied to Puppet output. +- JinjaTurtle templating is currently Ansible-oriented and is not applied to Puppet output. - Review generated resources before applying them broadly across unlike hosts. ## Notes @@ -735,45 +581,75 @@ sudo puppet apply --modulepath /path/to/generated/modules /path/to/generated/man """ -def manifest_puppet_from_bundle_dir( +class PuppetManifestRenderer: + """Render Puppet modules and site manifest from a harvest bundle.""" + + def __init__( + self, + bundle_dir: str, + out_dir: str, + *, + fqdn: Optional[str] = None, + no_common_roles: bool = False, + ) -> None: + self.bundle_dir = bundle_dir + self.out_dir = out_dir + self.fqdn = fqdn + self.no_common_roles = no_common_roles + + def render(self) -> None: + """Render Puppet modules/site.pp from a harvest bundle.""" + + bundle_dir = self.bundle_dir + out_dir = self.out_dir + fqdn = self.fqdn + no_common_roles = self.no_common_roles + + state = PuppetRole.load_state(bundle_dir) + out = Path(out_dir) + if out.exists(): + shutil.rmtree(out) + manifests_dir = out / "manifests" + modules_dir = out / "modules" + manifests_dir.mkdir(parents=True, exist_ok=True) + modules_dir.mkdir(parents=True, exist_ok=True) + + puppet_roles = _collect_puppet_roles( + state, + bundle_dir, + modules_dir, + fqdn=fqdn, + no_common_roles=no_common_roles, + ) + for prole in puppet_roles: + module_dir = modules_dir / prole.module_name + module_manifests = module_dir / "manifests" + module_files = module_dir / "files" + module_manifests.mkdir(parents=True, exist_ok=True) + module_files.mkdir(parents=True, exist_ok=True) + (module_manifests / "init.pp").write_text( + _render_role_class(prole), encoding="utf-8" + ) + _write_metadata(module_dir, prole.module_name) + + (manifests_dir / "site.pp").write_text( + _render_site_pp(puppet_roles, fqdn), encoding="utf-8" + ) + (out / "README.md").write_text( + _render_readme(state, puppet_roles), encoding="utf-8" + ) + + +def manifest_from_bundle_dir( bundle_dir: str, out_dir: str, *, fqdn: Optional[str] = None, no_common_roles: bool = False, ) -> None: - """Render Puppet modules/site.pp from a harvest bundle.""" - - state = _load_state(bundle_dir) - out = Path(out_dir) - if out.exists(): - shutil.rmtree(out) - manifests_dir = out / "manifests" - modules_dir = out / "modules" - manifests_dir.mkdir(parents=True, exist_ok=True) - modules_dir.mkdir(parents=True, exist_ok=True) - - puppet_roles = _collect_puppet_roles( - state, + PuppetManifestRenderer( bundle_dir, - modules_dir, + out_dir, fqdn=fqdn, no_common_roles=no_common_roles, - ) - for prole in puppet_roles: - module_dir = modules_dir / prole.module_name - module_manifests = module_dir / "manifests" - module_files = module_dir / "files" - module_manifests.mkdir(parents=True, exist_ok=True) - module_files.mkdir(parents=True, exist_ok=True) - (module_manifests / "init.pp").write_text( - _render_role_class(prole), encoding="utf-8" - ) - _write_metadata(module_dir, prole.module_name) - - (manifests_dir / "site.pp").write_text( - _render_site_pp(puppet_roles, fqdn), encoding="utf-8" - ) - (out / "README.md").write_text( - _render_readme(state, puppet_roles), encoding="utf-8" - ) + ).render() diff --git a/enroll/state.py b/enroll/state.py new file mode 100644 index 0000000..ed5a264 --- /dev/null +++ b/enroll/state.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, Mapping, Union + +BundlePath = Union[str, Path] +State = Dict[str, Any] + + +def state_path(bundle_dir: BundlePath) -> Path: + """Return the canonical state.json path for a harvest bundle.""" + + return Path(bundle_dir) / "state.json" + + +def load_state(bundle_dir: BundlePath) -> State: + """Load state.json from a harvest bundle directory.""" + + with open(state_path(bundle_dir), "r", encoding="utf-8") as f: + return json.load(f) + + +def write_state( + bundle_dir: BundlePath, + state: Mapping[str, Any], + *, + indent: int = 2, + sort_keys: bool = True, +) -> Path: + """Write state.json to a harvest bundle directory and return its path.""" + + path = state_path(bundle_dir) + with open(path, "w", encoding="utf-8") as f: + json.dump(state, f, indent=indent, sort_keys=sort_keys) + return path + + +def roles_from_state(state: Mapping[str, Any]) -> Dict[str, Any]: + """Return the roles mapping from a harvest state, or an empty mapping.""" + + roles = state.get("roles") + return dict(roles) if isinstance(roles, dict) else {} + + +def inventory_packages_from_state(state: Mapping[str, Any]) -> Dict[str, Any]: + """Return inventory.packages from a harvest state, or an empty mapping.""" + + inventory = state.get("inventory") + if not isinstance(inventory, dict): + return {} + packages = inventory.get("packages") + return dict(packages) if isinstance(packages, dict) else {} diff --git a/enroll/validate.py b/enroll/validate.py index 48d7250..0a3e8cb 100644 --- a/enroll/validate.py +++ b/enroll/validate.py @@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional, Set, Tuple import jsonschema from .diff import BundleRef, _bundle_from_input +from .state import load_state @dataclass @@ -153,7 +154,7 @@ def validate_harvest( ) try: - state = json.loads(state_path.read_text(encoding="utf-8")) + state = load_state(bundle.dir) except Exception as e: # noqa: BLE001 return ValidationResult( errors=[f"failed to parse state.json: {e!r}"], warnings=[] diff --git a/tests/test_cm.py b/tests/test_cm.py new file mode 100644 index 0000000..89addbf --- /dev/null +++ b/tests/test_cm.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from enroll.cm import CMModule, resolve_catalog_conflicts + + +def test_resolve_catalog_conflicts_dedupes_before_rendering(): + first = CMModule(role_name="admin", module_name="admin") + first.packages.add("curl") + first.dirs["/etc/default"] = {"owner": "root"} + first.files["/etc/foo.conf"] = {"owner": "root"} + + second = CMModule(role_name="misc", module_name="misc") + second.packages.add("curl") + second.dirs["/etc/default"] = {"owner": "root"} + second.dirs["/etc/foo.conf"] = {"owner": "root"} + second.files["/etc/foo.conf"] = {"owner": "root"} + + resolve_catalog_conflicts([first, second]) + + assert first.packages == {"curl"} + assert "/etc/default" in first.dirs + assert "/etc/foo.conf" in first.files + + assert second.packages == set() + assert second.dirs == {} + assert second.files == {} + assert any("duplicate Package[curl]" in note for note in second.notes) + assert any("duplicate File[/etc/default]" in note for note in second.notes) + assert any("a file or link with the same path" in note for note in second.notes) + + +def test_cm_module_uses_shared_state_io(tmp_path): + state = {"roles": {"packages": []}} + + written = CMModule.write_state(tmp_path, state) + + assert written == tmp_path / "state.json" + assert CMModule.state_path(tmp_path) == written + assert CMModule.load_state(tmp_path) == state + assert CMModule._load_state(tmp_path) == state diff --git a/tests/test_harvest_collectors.py b/tests/test_harvest_collectors.py new file mode 100644 index 0000000..f72a0f9 --- /dev/null +++ b/tests/test_harvest_collectors.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from enroll.harvest import ( + FirewallRuntimeSnapshot, + HarvestContext, + IgnorePolicy, + PathFilter, + RuntimeStateCollector, + SysctlSnapshot, +) + + +class _Backend: + name = "dpkg" + + +def _context(tmp_path): + return HarvestContext( + bundle_dir=str(tmp_path), + policy=IgnorePolicy(), + path_filter=PathFilter(include=(), exclude=()), + platform={}, + backend=_Backend(), + installed_pkgs={}, + installed_names=set(), + owned_etc=set(), + etc_owner_map={}, + topdir_to_pkgs={}, + pkg_to_etc_paths={}, + captured_global=set(), + ) + + +def test_runtime_state_collector_preserves_non_root_skip_schema(monkeypatch, tmp_path): + monkeypatch.setattr("enroll.harvest.os.geteuid", lambda: 1000) + + result = RuntimeStateCollector(_context(tmp_path)).collect() + + assert isinstance(result.firewall_runtime_snapshot, FirewallRuntimeSnapshot) + assert isinstance(result.sysctl_snapshot, SysctlSnapshot) + assert result.firewall_runtime_snapshot.role_name == "firewall_runtime" + assert result.sysctl_snapshot.role_name == "sysctl" + assert "not running as root" in result.firewall_runtime_snapshot.notes[0] + assert "not running as root" in result.sysctl_snapshot.notes[0] diff --git a/tests/test_jinjaturtle.py b/tests/test_jinjaturtle.py index 8490052..757a398 100644 --- a/tests/test_jinjaturtle.py +++ b/tests/test_jinjaturtle.py @@ -2,6 +2,7 @@ import json from pathlib import Path import enroll.manifest as manifest_mod +from enroll import ansible as ansible_mod from enroll.jinjaturtle import JinjifyResult @@ -106,7 +107,7 @@ def test_manifest_uses_jinjaturtle_templates_and_does_not_copy_raw( # Pretend jinjaturtle exists. monkeypatch.setattr( - manifest_mod, "find_jinjaturtle_cmd", lambda: "/usr/bin/jinjaturtle" + ansible_mod, "find_jinjaturtle_cmd", lambda: "/usr/bin/jinjaturtle" ) # Stub jinjaturtle output. @@ -119,7 +120,7 @@ def test_manifest_uses_jinjaturtle_templates_and_does_not_copy_raw( vars_text="foo_key: 1\n", ) - monkeypatch.setattr(manifest_mod, "run_jinjaturtle", fake_run_jinjaturtle) + monkeypatch.setattr(ansible_mod, "run_jinjaturtle", fake_run_jinjaturtle) manifest_mod.manifest(str(bundle), str(out), jinjaturtle="on") diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 8c74064..47fe0aa 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -7,6 +7,7 @@ import tarfile import pytest import enroll.manifest as manifest +from enroll import ansible as ansible_mod def _minimal_package_state(packages): @@ -824,7 +825,7 @@ def test_copy2_replace_overwrites_readonly_destination(tmp_path: Path): import os import stat - from enroll.manifest import _copy2_replace + from enroll.ansible import _copy2_replace src = tmp_path / "src" dst = tmp_path / "dst" @@ -935,7 +936,7 @@ def test_manifest_includes_dnf_config_role_when_present(tmp_path: Path): def test_render_install_packages_tasks_contains_dnf_branch(): - from enroll.manifest import _render_install_packages_tasks + from enroll.ansible import _render_install_packages_tasks txt = _render_install_packages_tasks("role", "role") assert "ansible.builtin.apt" in txt @@ -1073,9 +1074,9 @@ def test_manifest_orders_cron_and_logrotate_at_playbook_tail(tmp_path: Path): def test_yaml_helpers_fallback_when_yaml_unavailable(monkeypatch): - monkeypatch.setattr(manifest, "_try_yaml", lambda: None) - assert manifest._yaml_load_mapping("foo: 1\n") == {} - out = manifest._yaml_dump_mapping({"b": 2, "a": 1}) + monkeypatch.setattr(ansible_mod, "_try_yaml", lambda: None) + assert ansible_mod._yaml_load_mapping("foo: 1\n") == {} + out = ansible_mod._yaml_dump_mapping({"b": 2, "a": 1}) # Best-effort fallback is key: repr(value) assert out.splitlines()[0].startswith("a: ") assert out.endswith("\n") @@ -1090,7 +1091,7 @@ def test_copy2_replace_makes_readonly_sources_user_writable( # Make source read-only; copy2 preserves mode, so tmp will be read-only too. os.chmod(src, 0o444) - manifest._copy2_replace(str(src), str(dst)) + ansible_mod._copy2_replace(str(src), str(dst)) st = os.stat(dst, follow_symlinks=False) assert stat.S_IMODE(st.st_mode) & stat.S_IWUSR @@ -1208,13 +1209,13 @@ def test_manifest_applies_jinjaturtle_to_jinjifyable_managed_file( __import__("json").dumps(state), encoding="utf-8" ) - monkeypatch.setattr(manifest, "find_jinjaturtle_cmd", lambda: "jinjaturtle") + monkeypatch.setattr(ansible_mod, "find_jinjaturtle_cmd", lambda: "jinjaturtle") class _Res: template_text = "key={{ foo }}\n" vars_text = "foo: 123\n" - monkeypatch.setattr(manifest, "run_jinjaturtle", lambda *a, **k: _Res()) + monkeypatch.setattr(ansible_mod, "run_jinjaturtle", lambda *a, **k: _Res()) out_dir = tmp_path / "out" manifest.manifest(str(bundle), str(out_dir), jinjaturtle="on") @@ -1330,7 +1331,7 @@ def test_manifest_writes_firewall_runtime_role(tmp_path: Path): def test_try_yaml_with_yaml_installed(): - result = manifest._try_yaml() + result = ansible_mod._try_yaml() # PyYAML should be installed for tests if result is None: pytest.skip("PyYAML not installed") @@ -1347,55 +1348,55 @@ list: - item1 - item2 """ - result = manifest._yaml_load_mapping(text) + result = ansible_mod._yaml_load_mapping(text) assert result["key1"] == "value1" assert result["key2"]["nested"] == "value" assert result["list"] == ["item1", "item2"] def test_yaml_load_mapping_empty(): - result = manifest._yaml_load_mapping("") + result = ansible_mod._yaml_load_mapping("") assert result == {} def test_yaml_load_mapping_invalid(): - result = manifest._yaml_load_mapping("invalid: yaml: :") + result = ansible_mod._yaml_load_mapping("invalid: yaml: :") assert result == {} def test_yaml_load_mapping_not_dict(): - result = manifest._yaml_load_mapping("- item1\n- item2") + result = ansible_mod._yaml_load_mapping("- item1\n- item2") assert result == {} def test_yaml_load_mapping_none(): - result = manifest._yaml_load_mapping("~") + result = ansible_mod._yaml_load_mapping("~") assert result == {} def test_yaml_dump_mapping_with_yaml(tmp_path: Path): obj = {"key1": "value1", "key2": 123} - result = manifest._yaml_dump_mapping(obj) + result = ansible_mod._yaml_dump_mapping(obj) assert "key1: value1" in result assert "key2:" in result def test_yaml_dump_mapping_empty(): - result = manifest._yaml_dump_mapping({}) + result = ansible_mod._yaml_dump_mapping({}) # Empty dict produces '{}' assert result.strip() == "{}" def test_yaml_dump_mapping_with_nested(tmp_path: Path): obj = {"key1": {"nested": "value"}} - result = manifest._yaml_dump_mapping(obj) + result = ansible_mod._yaml_dump_mapping(obj) assert "nested:" in result def test_merge_mappings_overwrite_simple(): existing = {"key1": "old", "key2": "keep"} incoming = {"key1": "new", "key3": "added"} - result = manifest._merge_mappings_overwrite(existing, incoming) + result = ansible_mod._merge_mappings_overwrite(existing, incoming) assert result["key1"] == "new" assert result["key2"] == "keep" assert result["key3"] == "added" @@ -1404,16 +1405,16 @@ def test_merge_mappings_overwrite_simple(): def test_merge_mappings_overwrite_nested(): existing = {"key1": {"a": 1}} incoming = {"key1": {"b": 2}} - result = manifest._merge_mappings_overwrite(existing, incoming) + result = ansible_mod._merge_mappings_overwrite(existing, incoming) # Nested dicts are replaced, not merged assert result["key1"] == {"b": 2} def test_merge_mappings_overwrite_empty(): - result = manifest._merge_mappings_overwrite({}, {"key": "value"}) + result = ansible_mod._merge_mappings_overwrite({}, {"key": "value"}) assert result == {"key": "value"} - result = manifest._merge_mappings_overwrite({"key": "value"}, {}) + result = ansible_mod._merge_mappings_overwrite({"key": "value"}, {}) assert result == {"key": "value"} @@ -1422,7 +1423,7 @@ def test_copy2_replace(tmp_path: Path): src.write_text("content", encoding="utf-8") dst = tmp_path / "dst" / "subdir" / "dst.txt" - manifest._copy2_replace(str(src), str(dst)) + ansible_mod._copy2_replace(str(src), str(dst)) assert dst.exists() assert dst.read_text(encoding="utf-8") == "content" @@ -1434,7 +1435,7 @@ def test_copy2_replace_preserves_metadata(tmp_path: Path): os.chmod(str(src), 0o644) dst = tmp_path / "dst.txt" - manifest._copy2_replace(str(src), str(dst)) + ansible_mod._copy2_replace(str(src), str(dst)) assert dst.exists() st = dst.stat() @@ -1449,55 +1450,30 @@ def test_copy2_replace_atomic(tmp_path: Path): # Write initial content dst.write_text("old", encoding="utf-8") - manifest._copy2_replace(str(src), str(dst)) + ansible_mod._copy2_replace(str(src), str(dst)) assert dst.read_text(encoding="utf-8") == "content" def test_render_firewall_runtime_tasks_empty(): - state = {"roles": {}} - result = manifest._render_firewall_runtime_tasks(state) + result = ansible_mod._render_firewall_runtime_tasks("firewall_runtime") # Function always returns at least a basic playbook structure assert isinstance(result, str) assert len(result) > 0 def test_render_firewall_runtime_tasks_with_iptables(): - state = { - "roles": { - "firewall_runtime": { - "role_name": "firewall_runtime", - "iptables_v4_save": "artifacts/firewall_runtime/iptables.save", - } - } - } - result = manifest._render_firewall_runtime_tasks(state) + result = ansible_mod._render_firewall_runtime_tasks("firewall_runtime") assert len(result) >= 1 def test_render_firewall_runtime_tasks_with_ipset(): - state = { - "roles": { - "firewall_runtime": { - "role_name": "firewall_runtime", - "ipset_save": "artifacts/firewall_runtime/ipset.save", - } - } - } - result = manifest._render_firewall_runtime_tasks(state) + result = ansible_mod._render_firewall_runtime_tasks("firewall_runtime") assert len(result) >= 1 def test_render_firewall_runtime_tasks_with_ipv6(): - state = { - "roles": { - "firewall_runtime": { - "role_name": "firewall_runtime", - "iptables_v6_save": "artifacts/firewall_runtime/ip6tables.save", - } - } - } - result = manifest._render_firewall_runtime_tasks(state) + result = ansible_mod._render_firewall_runtime_tasks("firewall_runtime") assert len(result) >= 1 @@ -1708,6 +1684,93 @@ def test_users_role_without_portable_apps_omits_community_general_tasks(tmp_path assert "collections:" not in users_meta +def test_users_role_only_creates_ssh_dir_when_managed_ssh_files_exist(tmp_path): + bundle = tmp_path / "bundle" + out = tmp_path / "out" + (bundle / "artifacts" / "users" / "alice" / ".ssh").mkdir( + parents=True, exist_ok=True + ) + (bundle / "artifacts" / "users" / "bob").mkdir(parents=True, exist_ok=True) + (bundle / "artifacts" / "users" / "alice" / ".ssh" / "authorized_keys").write_text( + "ssh-ed25519 example alice\n", encoding="utf-8" + ) + (bundle / "artifacts" / "users" / "bob" / ".bashrc").write_text( + "alias ll='ls -l'\n", encoding="utf-8" + ) + state = { + "roles": { + "users": { + "role_name": "users", + "users": [ + { + "name": "alice", + "uid": 1000, + "home": "/home/alice", + "primary_group": "alice", + "supplementary_groups": [], + }, + { + "name": "bob", + "uid": 1001, + "home": "/home/bob", + "primary_group": "bob", + "supplementary_groups": [], + }, + { + "name": "carol", + "uid": 1002, + "home": "/home/carol", + "primary_group": "carol", + "supplementary_groups": [], + }, + ], + "managed_files": [ + { + "path": "/home/alice/.ssh/authorized_keys", + "src_rel": "alice/.ssh/authorized_keys", + "mode": "0644", + "reason": "authorized_keys", + }, + { + "path": "/home/bob/.bashrc", + "src_rel": "bob/.bashrc", + "mode": "0644", + "reason": "dangerous_user_dotfile", + }, + ], + "excluded": [], + "notes": [], + }, + "services": [], + "packages": [], + }, + } + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + manifest.manifest(str(bundle), str(out)) + + users_defaults_text = (out / "roles" / "users" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + users_defaults = ansible_mod._yaml_load_mapping(users_defaults_text) + users_tasks = (out / "roles" / "users" / "tasks" / "main.yml").read_text( + encoding="utf-8" + ) + + assert users_defaults["users_ssh_dirs"] == [ + { + "dest": "/home/alice/.ssh", + "group": "alice", + "mode": "0700", + "owner": "alice", + } + ] + assert 'loop: "{{ users_ssh_dirs | default([]) }}"' in users_tasks + assert 'path: "{{ item.ssh_dir }}"' not in users_tasks + assert "users_ssh_files" in users_defaults + + def test_manifest_emits_flatpak_role_even_when_no_flatpaks(tmp_path): bundle = tmp_path / "bundle" out = tmp_path / "out" diff --git a/tests/test_manifest_ansible_model.py b/tests/test_manifest_ansible_model.py new file mode 100644 index 0000000..88e4995 --- /dev/null +++ b/tests/test_manifest_ansible_model.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from enroll.cm import CMModule +from enroll.ansible import AnsibleRole + + +def test_ansible_role_extends_cm_module_and_normalises_service_snapshot(): + role = AnsibleRole("network") + + role.add_service_snapshot( + { + "role_name": "networking", + "unit": "networking.service", + "packages": ["ifupdown"], + "active_state": "active", + "unit_file_state": "enabled", + "managed_dirs": [ + { + "path": "/etc/network", + "owner": "root", + "group": "root", + "mode": "0755", + } + ], + "managed_files": [ + { + "path": "/etc/network/interfaces", + "src_rel": "etc/network/interfaces", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "service_config", + } + ], + "managed_links": [ + { + "path": "/etc/systemd/system/multi-user.target.wants/networking.service", + "target": "/usr/lib/systemd/system/networking.service", + } + ], + "excluded": [{"path": "/etc/network/secrets", "reason": "secret"}], + "notes": ["captured for test"], + } + ) + + assert isinstance(role, CMModule) + assert role.sorted_packages == ["ifupdown"] + assert role.dirs["/etc/network"]["mode"] == "0755" + assert role.files["/etc/network/interfaces"]["src_rel"] == "etc/network/interfaces" + assert ( + role.links["/etc/systemd/system/multi-user.target.wants/networking.service"][ + "src" + ] + == "/usr/lib/systemd/system/networking.service" + ) + assert role.systemd_units_var == [ + { + "name": "networking.service", + "manage": True, + "enabled": True, + "state": "started", + } + ] + assert role.excluded == [{"path": "/etc/network/secrets", "reason": "secret"}] + assert role.notes == ["captured for test"] + assert "service `networking.service` from role `networking`" in role.origin_lines + + +def test_ansible_role_normalises_package_snapshot(): + role = AnsibleRole("admin") + role.add_package_snapshot( + { + "role_name": "curl", + "package": "curl", + "managed_files": [ + { + "path": "/etc/curlrc", + "src_rel": "etc/curlrc", + "owner": "root", + "group": "root", + "mode": "0644", + } + ], + } + ) + + assert isinstance(role, CMModule) + assert role.sorted_packages == ["curl"] + assert role.files["/etc/curlrc"]["dest"] == "/etc/curlrc" + assert role.services == {} + assert role.origin_lines == ["package `curl` from role `curl`"]