From f255ba566c59067e909e883088abef6e8187442f Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Tue, 16 Dec 2025 20:14:20 +1100 Subject: [PATCH 1/3] biiiiig refactor to support jinjaturtle and multi site mode --- CHANGELOG.md | 13 + Dockerfile.debbuild | 1 + README.md | 180 ++++++- debian/changelog | 9 + debian/control | 3 +- enroll/cli.py | 37 +- enroll/ignore.py | 33 +- enroll/jinjaturtle.py | 105 ++++ enroll/manifest.py | 1140 +++++++++++++++++++++++++++++++---------- poetry.lock | 105 +++- pyproject.toml | 3 +- 11 files changed, 1331 insertions(+), 298 deletions(-) create mode 100644 enroll/jinjaturtle.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d30a65a..76d737d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +# 0.0.5 + + * Use JinjaTurtle to generate dynamic template/inventory if it's on the PATH + * Support --fqdn flag for site-specific inventory and an inventory hosts file. + This radically re-architects the roles to loop through abstract inventory + because otherwise different servers can collide with each other through use + of the same role. Use 'single site' mode (no `--fqdn`) if you want more readable, + self-contained roles (in which case, store each manifested output in its own + repo per server) + * Generate an ansible.cfg if not present, to support host_vars plugin and other params, + when using `--fqdn` mode + * Be more permissive with files that we previously thought contained secrets (ignore commented lines) + # 0.0.4 * Fix dash package detection issue diff --git a/Dockerfile.debbuild b/Dockerfile.debbuild index 873018e..9009b41 100644 --- a/Dockerfile.debbuild +++ b/Dockerfile.debbuild @@ -24,6 +24,7 @@ RUN set -eux; \ pybuild-plugin-pyproject \ python3-all \ python3-poetry-core \ + python3-yaml \ rsync \ ca-certificates \ ; \ diff --git a/README.md b/README.md index 1839ad9..b9375dd 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Enroll +# Enroll
Enroll logo @@ -16,9 +16,127 @@ It aims to be **optimistic and noninteractive**: - Captures miscellaneous `/etc` files that it can't attribute to a package, and installs it in an `etc_custom` role - Avoids trying to start systemd services that were detected as being Inactive during harvest -## Install +--- -### Ubuntu/Debian apt repository +# Two modes: single-site vs multi-site (`--fqdn`) + +**enroll** has two distinct ways to generate Ansible: + +## 1) Single-site mode (default: *no* `--fqdn`) +Use this when you’re enrolling **one server** (or you’re generating a “golden” role set you intend to reuse). + +**What you get** +- Config, templates, and defaults are primarily **contained inside each role**. +- Raw config files (when not templated) live in the role’s `files/`. +- Template variables (when templated) live in the role’s `defaults/main.yml`. + +**Pros** +- Roles are more **self-contained** and easier to understand. +- Better starting point for **provisioning new servers**, because the role contains most of what it needs. +- Less inventory abstraction/duplication. + +**Cons** +- Less convenient for quickly enrolling multiple hosts with divergent configs (you’ll do more manual work to make roles flexible across hosts). + +## 2) Multi-site mode (`--fqdn`) +Use this when you want to enroll **several existing servers** quickly, especially if they differ. + +**What you get** +- Roles are **shared** across hosts, but host-specific data lives in inventory. +- Host inventory drives what’s managed: + - which files to deploy for that host + - which packages are relevant for that host + - which services should be enabled/started for that host +- For non-templated config, raw files live in host-specific inventory under `.files/` (per role). + +**Pros** +- Fastest way to retrofit **multiple servers** into config management. +- Avoids shared-role “host A breaks host B” problems by keeping host-specific state in inventory. +- Better fit when you already have a fleet and want to capture/reflect reality first. + +**Cons** +- More abstraction: roles become more “data-driven”. +- Potential duplication: raw files may exist per-host in inventory (even if identical). +- Harder to use the roles to **provision a brand-new server** without also building an inventory for that new host, because multi-site output assumes the server already exists and is being retrofitted. + +**Rule of thumb** +- If your goal is *“make this one server reproducible / provisionable”* → start with **single-site**. +- If your goal is *“get several already-running servers under management quickly”* → use **multi-site**. + +--- + +# Key concepts + +## Harvest + +**enroll** begins by 'harvesting' known state about your host. This includes detecting what running services exist, what packages have been installed 'manually' (that is, stuff that doesn't come out of the box with the OS), and anything 'custom' in `/etc` that it can't attribute to a specific package. + +It also detects if any config files have been *changed* from their packaged defaults. If they have, it will attempt to 'harvest' them. If the config file is identical to how it comes with the package, then it doesn't bother harvesting it, because there's little value in config-managing it if it's identical to what you get by simply installing the package! + +The harvest writes a state.json file explaining all the data it harvested and, if it chose not to harvest something, explanations as to why that is the case (see below: sensitive data). + +## Sensitive data + +**enroll** doesn't make any assumptions about how you might handle sensitive data from your config files, in Ansible. Some people might use SOPS, others might use Vault, others might do something else entirely. + +For this reason, **enroll** will attempt to read config files, and if it detects data that looks like a sensitive SSH/SSL private key, or password, or API key, etc, then it won't harvest it for config management. + +This inevitably means that it will deliberately miss some important config files that you probably *want* to manage in Ansible. + +Nonetheless, in the Harvest 'state' file, there should be an explanation of 'excluded files'. You can parse or inspect this file to find what it chose to ignore, and then you know what you might want to augment the results with later, once you 'manifest' the harvest into Ansible configuration. + +## Manifest + +The 'manifest' subcommand expects to be given a path to the 'harvest' obtained in the first step. It will then attempt to generate Ansible roles and playbooks (and potentially 'inventory') from that harvest. + +Manifesting is the most complex step because a lot of people will have opinions on how Ansible roles and inventory should work. No solution is perfect for everyone. However, **enroll** tries to strike a reasonable balance. + +Remember, the purpose of this tool is to save **time** getting your systems into a decently-managed state. It's still up to you to wrangle it into a form that works for you on an ongoing basis. + +--- + +# Single-shot mode for the impatient sysadmin + +**enroll** has a 'single-shot' subcommand which combines the two other phases (harvest and manifest) into one. Use it to generate both the harvest and then manifest ansible from that harvest all in one go. Perfect if you're in a hurry! + +--- + +# JinjaTurtle integration (both modes) + +If you also have my other tool [JinjaTurtle](https://git.mig5.net/mig5/jinjaturtle) installed, **enroll** will attempt to create Jinja2 templates for any ini/json/xml/toml style configuration that it finds. + +- Templates live in the **role** (`roles//templates/...`) +- Variables live in: + - **single-site**: `roles//defaults/main.yml` + - **multi-site** (`--fqdn`): `inventory/host_vars//.yml` + +JinjaTurtle will be used automatically if it is detected on the `$PATH`. You can also be explicit and pass `--jinjaturtle`, but this will throw an error if JinjaTurtle is not on the `$PATH`. + +If you *do* have JinjaTurtle installed, but *don't* wish to make use of it, you can use `--no-jinjaturtle`, in which case all config files will be kept as 'raw' files. + +--- + +# How multi-site avoids “shared role breaks a host” + +In multi-site mode, **roles are data-driven**. The role contains generic tasks like: + +- “deploy all files listed for this host” +- “install packages listed for this host” +- “apply systemd enable/start state listed for this host” + +The host inventory is what decides which files/packages/services apply to that host. This prevents the classic failure mode where host2 adds a config file to a shared role and host1 then fails trying to deploy a file it never had. + +Raw non-templated files are stored under: + +- `inventory/host_vars///.files/...` + +…and the host’s role variables describe which of those files should be deployed. + +--- + +# Install + +## Ubuntu/Debian apt repository ```bash sudo mkdir -p /usr/share/keyrings @@ -28,7 +146,7 @@ sudo apt update sudo apt install enroll ``` -### AppImage +## AppImage Download the AppImage file from the Releases page (verify with GPG if you wish, my fingerprint is [here](https://mig5.net/static/mig5.asc)), then make it executable and run it: @@ -53,49 +171,69 @@ poetry install poetry run enroll --help ``` -## Usage +--- + +# Usage + +## 1. Harvest state/information about the host On the host (root recommended to harvest as much data as possible): -### 1. Harvest state/information about the host - ```bash enroll harvest --out /tmp/enroll-harvest ``` -### 2. Generate Ansible manifests (roles/playbook) from that harvest +## 2. Generate Ansible manifests (roles/playbook) from that harvest + +### Single-site (default: no --fqdn) + +Good for one server, or for producing roles you want to reuse to provision new machines: ```bash enroll manifest --harvest /tmp/enroll-harvest --out /tmp/enroll-ansible ``` -### Alternatively, do both steps in one shot: +### Multi-site (--fqdn) + +Best when enrolling multiple already-running servers into one repo: ```bash -enroll enroll --harvest /tmp/enroll-harvest --out /tmp/enroll-ansible +enroll manifest --harvest /tmp/enroll-harvest --out /tmp/enroll-ansible --fqdn "$(hostname -f)" ``` -Then run Ansible however way you wish, for example (local execution): +## Single-shot + +Alternatively, do both steps in one shot: + +```bash +enroll single-shot --harvest /tmp/enroll-harvest --out /tmp/enroll-ansible --fqdn "$(hostname -f)" +``` + +## 3. Run Ansible + +### Single-site + +You can run it however you prefer (local connection or your own inventory). Example: ```bash ansible-playbook -i "localhost," -c local /tmp/enroll-ansible/playbook.yml ``` +### Multi-site (--fqdn) -## Notes / Safety +In multi-site mode, enroll generates an ansible.cfg, `host_vars` inventory, and a host-specific playbook: -- enroll **skips** common sensitive locations like `/etc/ssl/private/*`, `/etc/ssh/ssh_host_*`, and files that look like private keys/tokens. -- It also skips symlinks, binary-ish files, and large files by default. -- Review each generated role’s README before committing it anywhere. -- It only stores the raw config files. If you want to turn these into Jinja2 templates with dynamic inventory, see my other tool https://git.mig5.net/mig5/jinjaturtle . +```bash +ansible-playbook /tmp/enroll-ansible/playbooks/"$(hostname -f)".yml +``` +--- -## Troubleshooting +# Found a bug, have a suggestion? -- Run as root for the most complete harvest (`sudo ...`). +My Forgejo doesn't yet support proper federation, and for that reason I've not opened up registration/login to use the issue queue. -## Found a bug, have a suggestion? - -You can e-mail me (see the pyproject.toml for details) or contact me on the Fediverse: +Instead, you can e-mail me (see the pyproject.toml for details) or contact me on the Fediverse: https://goto.mig5.net/@mig5 + diff --git a/debian/changelog b/debian/changelog index 86ae088..b889be7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +enroll (0.0.5) unstable; urgency=medium + + * Use JinjaTurtle to generate dynamic template/inventory if it's on the PATH + * Support --fqdn flag for site-specific inventory and an inventory hosts file + * Generate an ansible.cfg if not present, to support host_vars plugin and other params + * Be more permissive with files that we previously thought contained secrets (ignore commented lines) + + -- Miguel Jacq Tue, 16 Dec 2025 12:00:00 +1100 + enroll (0.0.4) unstable; urgency=medium * Fix dash package detection issue diff --git a/debian/control b/debian/control index 372f83f..71b5661 100644 --- a/debian/control +++ b/debian/control @@ -8,12 +8,13 @@ Build-Depends: dh-python, pybuild-plugin-pyproject, python3-all, + python3-yaml, python3-poetry-core Standards-Version: 4.6.2 Homepage: https://git.mig5.net/mig5/enroll Package: enroll Architecture: all -Depends: ${misc:Depends}, ${python3:Depends} +Depends: ${misc:Depends}, ${python3:Depends}, python3-yaml Description: Harvest a host into Ansible roles A tool that inspects a system and emits Ansible roles/playbooks to reproduce it. diff --git a/enroll/cli.py b/enroll/cli.py index 530a388..0511b54 100644 --- a/enroll/cli.py +++ b/enroll/cli.py @@ -1,10 +1,37 @@ from __future__ import annotations import argparse + from .harvest import harvest from .manifest import manifest +def _add_common_manifest_args(p: argparse.ArgumentParser) -> None: + p.add_argument( + "--fqdn", + help="Host FQDN/name for site-mode output (creates inventory/, inventory/host_vars/, playbooks/).", + ) + g = p.add_mutually_exclusive_group() + g.add_argument( + "--jinjaturtle", + action="store_true", + help="Attempt jinjaturtle template integration (it will error if jinjaturtle is not found on PATH).", + ) + g.add_argument( + "--no-jinjaturtle", + action="store_true", + help="Do not use jinjaturtle integration, even if it is installed.", + ) + + +def _jt_mode(args: argparse.Namespace) -> str: + if getattr(args, "jinjaturtle", False): + return "on" + if getattr(args, "no_jinjaturtle", False): + return "off" + return "auto" + + def main() -> None: ap = argparse.ArgumentParser(prog="enroll") sub = ap.add_subparsers(dest="cmd", required=True) @@ -23,9 +50,10 @@ def main() -> None: required=True, help="Output directory for generated roles/playbook Ansible manifest", ) + _add_common_manifest_args(r) e = sub.add_parser( - "enroll", help="Harvest state, then manifest Ansible code, in one shot" + "single-shot", help="Harvest state, then manifest Ansible code, in one shot" ) e.add_argument( "--harvest", required=True, help="Path to the directory to place the harvest in" @@ -35,6 +63,7 @@ def main() -> None: required=True, help="Output directory for generated roles/playbook Ansible manifest", ) + _add_common_manifest_args(e) args = ap.parse_args() @@ -42,7 +71,7 @@ def main() -> None: path = harvest(args.out) print(path) elif args.cmd == "manifest": - manifest(args.harvest, args.out) - elif args.cmd == "enroll": + manifest(args.harvest, args.out, fqdn=args.fqdn, jinjaturtle=_jt_mode(args)) + elif args.cmd == "single-shot": harvest(args.harvest) - manifest(args.harvest, args.out) + manifest(args.harvest, args.out, fqdn=args.fqdn, jinjaturtle=_jt_mode(args)) diff --git a/enroll/ignore.py b/enroll/ignore.py index 217497f..9a9ecf2 100644 --- a/enroll/ignore.py +++ b/enroll/ignore.py @@ -31,6 +31,10 @@ SENSITIVE_CONTENT_PATTERNS = [ re.compile(rb"(?i)\b(pass|passwd|token|secret|api[_-]?key)\b"), ] +COMMENT_PREFIXES = (b"#", b";", b"//") +BLOCK_START = b"/*" +BLOCK_END = b"*/" + @dataclass class IgnorePolicy: @@ -42,6 +46,28 @@ class IgnorePolicy: if self.deny_globs is None: self.deny_globs = list(DEFAULT_DENY_GLOBS) + def iter_effective_lines(self, content: bytes): + in_block = False + for raw in content.splitlines(): + line = raw.lstrip() + + if in_block: + if BLOCK_END in line: + in_block = False + continue + + if not line: + continue + + if line.startswith(BLOCK_START): + in_block = True + continue + + if line.startswith(COMMENT_PREFIXES) or line.startswith(b"*"): + continue + + yield raw + def deny_reason(self, path: str) -> Optional[str]: for g in self.deny_globs: if fnmatch.fnmatch(path, g): @@ -67,8 +93,9 @@ class IgnorePolicy: if b"\x00" in data: return "binary_like" - for pat in SENSITIVE_CONTENT_PATTERNS: - if pat.search(data): - return "sensitive_content" + for line in self.iter_effective_lines(data): + for pat in SENSITIVE_CONTENT_PATTERNS: + if pat.search(line): + return "sensitive_content" return None diff --git a/enroll/jinjaturtle.py b/enroll/jinjaturtle.py new file mode 100644 index 0000000..f894f04 --- /dev/null +++ b/enroll/jinjaturtle.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import re +import shutil +import subprocess # nosec +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + + +SUPPORTED_EXTS = {".ini", ".json", ".toml", ".yaml", ".yml", ".xml"} + + +@dataclass(frozen=True) +class JinjifyResult: + template_text: str + vars_text: str # YAML mapping text (no leading --- expected) + + +def find_jinjaturtle_cmd() -> Optional[str]: + """Return the executable path for jinjaturtle if found on PATH.""" + return shutil.which("jinjaturtle") + + +def can_jinjify_path(path: str) -> bool: + p = Path(path) + return p.suffix.lower() in SUPPORTED_EXTS + + +def run_jinjaturtle( + jt_exe: str, + src_path: str, + *, + role_name: str, + force_format: Optional[str] = None, +) -> JinjifyResult: + """ + Run jinjaturtle against src_path and return (template, defaults-yaml). + Uses tempfiles and captures outputs. + + jinjaturtle CLI: + jinjaturtle -r [-f ] [-d ] [-t ] + """ + src = Path(src_path) + if not src.is_file(): + raise FileNotFoundError(src_path) + + with tempfile.TemporaryDirectory(prefix="enroll-jt-") as td: + td_path = Path(td) + defaults_out = td_path / "defaults.yml" + template_out = td_path / "template.j2" + + cmd = [ + jt_exe, + str(src), + "-r", + role_name, + "-d", + str(defaults_out), + "-t", + str(template_out), + ] + if force_format: + cmd.extend(["-f", force_format]) + + p = subprocess.run(cmd, text=True, capture_output=True) # nosec + if p.returncode != 0: + raise RuntimeError( + "jinjaturtle failed for %s (role=%s)\ncmd=%r\nstdout=%s\nstderr=%s" + % (src_path, role_name, cmd, p.stdout, p.stderr) + ) + + vars_text = defaults_out.read_text(encoding="utf-8").strip() + template_text = template_out.read_text(encoding="utf-8") + + # jinjaturtle outputs a YAML mapping; strip leading document marker if present + if vars_text.startswith("---"): + vars_text = "\n".join(vars_text.splitlines()[1:]).lstrip() + + return JinjifyResult( + template_text=template_text, vars_text=vars_text.rstrip() + "\n" + ) + + +def replace_or_append_block( + base_text: str, + *, + begin: str, + end: str, + block_body: str, +) -> str: + """Replace a marked block if present; else append it.""" + pattern = re.compile( + re.escape(begin) + r".*?" + re.escape(end), + flags=re.DOTALL, + ) + new_block = f"{begin}\n{block_body.rstrip()}\n{end}" + if pattern.search(base_text): + return pattern.sub(new_block, base_text).rstrip() + "\n" + # ensure base ends with newline + bt = base_text.rstrip() + "\n" + if not bt.endswith("\n"): + bt += "\n" + return bt + "\n" + new_block + "\n" diff --git a/enroll/manifest.py b/enroll/manifest.py index b92f59c..b772bcd 100644 --- a/enroll/manifest.py +++ b/enroll/manifest.py @@ -3,7 +3,136 @@ from __future__ import annotations import json import os import shutil -from typing import Any, Dict, List +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple + +from .jinjaturtle import ( + find_jinjaturtle_cmd, + can_jinjify_path, + run_jinjaturtle, +) + + +JINJATURTLE_BEGIN = "# BEGIN JINJATURTLE (generated by enroll)" +JINJATURTLE_END = "# END JINJATURTLE" + + +def _try_yaml(): + try: + import yaml # type: ignore + except Exception: + return None + return yaml + + +def _yaml_load_mapping(text: str) -> Dict[str, Any]: + yaml = _try_yaml() + if yaml is None: + return {} + try: + obj = yaml.safe_load(text) + except Exception: + return {} + if obj is None: + return {} + if isinstance(obj, dict): + return obj + return {} + + +def _yaml_dump_mapping(obj: Dict[str, Any], *, sort_keys: bool = True) -> str: + yaml = _try_yaml() + if yaml is None: + # fall back to a naive key: value dump (best-effort) + lines: List[str] = [] + for k, v in sorted(obj.items()) if sort_keys else obj.items(): + lines.append(f"{k}: {v!r}") + return "\n".join(lines).rstrip() + "\n" + return ( + yaml.safe_dump(obj, default_flow_style=False, sort_keys=sort_keys).rstrip() + + "\n" + ) + + +def _merge_list_keep_order(existing: List[Any], new: List[Any]) -> List[Any]: + out = list(existing) + seen = set(existing) + for item in new: + if item not in seen: + out.append(item) + seen.add(item) + return out + + +def _merge_mappings_preserve( + existing: Dict[str, Any], incoming: Dict[str, Any] +) -> Dict[str, Any]: + """Merge incoming into existing: + - lists: union (preserve existing order) + - scalars/dicts: only set if missing (do not overwrite) + """ + merged = dict(existing) + for k, v in incoming.items(): + if k in merged: + if isinstance(merged[k], list) and isinstance(v, list): + merged[k] = _merge_list_keep_order(merged[k], v) + else: + # keep existing value (non-overwriting) + continue + else: + merged[k] = v + return merged + + +def _merge_mappings_overwrite( + existing: Dict[str, Any], incoming: Dict[str, Any] +) -> Dict[str, Any]: + """Merge incoming into existing with overwrite. + + NOTE: Unlike role defaults merging, host_vars should reflect the current + harvest for a host. Therefore lists are replaced rather than unioned. + """ + merged = dict(existing) + merged.update(incoming) + return merged + + +def _write_role_defaults_merge(role_dir: str, incoming: Dict[str, Any]) -> None: + """Write/merge role defaults without clobbering existing values. + Used in site mode to keep roles reusable across hosts. + """ + defaults_path = os.path.join(role_dir, "defaults", "main.yml") + existing: Dict[str, Any] = {} + if os.path.exists(defaults_path): + try: + existing_text = Path(defaults_path).read_text(encoding="utf-8") + existing = _yaml_load_mapping(existing_text) + except Exception: + existing = {} + merged = _merge_mappings_preserve(existing, incoming) + body = "---\n" + _yaml_dump_mapping(merged, sort_keys=True) + with open(defaults_path, "w", encoding="utf-8") as f: + f.write(body) + + +def _extract_jinjaturtle_block(text: str) -> str: + """Return YAML text inside JINJATURTLE_BEGIN/END markers, or the whole text if no markers.""" + if JINJATURTLE_BEGIN in text and JINJATURTLE_END in text: + start = text.split(JINJATURTLE_BEGIN, 1)[1] + inner = start.split(JINJATURTLE_END, 1)[0] + return inner.strip() + "\n" + return text.strip() + "\n" + + +def _normalize_jinjaturtle_vars_text(vars_text: str) -> str: + """Deduplicate keys in a vars fragment by parsing as YAML and dumping it back.""" + m = _yaml_load_mapping(vars_text) + if not m: + # if YAML isn't available or parsing failed, return raw text (best-effort) + return vars_text.rstrip() + ( + "\n" if vars_text and not vars_text.endswith("\n") else "" + ) + return _yaml_dump_mapping(m, sort_keys=True) def _yaml_list(items: List[str], indent: int = 2) -> str: @@ -13,7 +142,20 @@ def _yaml_list(items: List[str], indent: int = 2) -> str: return "\n".join(f"{pad}- {x}" for x in items) -def _copy_artifacts(bundle_dir: str, role: str, role_dir: str) -> None: +def _copy_artifacts( + bundle_dir: str, + role: str, + dst_files_dir: str, + *, + preserve_existing: bool = False, + exclude_rels: Optional[Set[str]] = None, +) -> None: + """Copy harvested artifacts for a role into a destination *files* directory. + + In non --fqdn mode, this is usually /files. + In --fqdn site mode, this is usually: + inventory/host_vars///.files + """ artifacts_dir = os.path.join(bundle_dir, "artifacts", role) if not os.path.isdir(artifacts_dir): return @@ -21,7 +163,21 @@ def _copy_artifacts(bundle_dir: str, role: str, role_dir: str) -> None: for fn in files: src = os.path.join(root, fn) rel = os.path.relpath(src, artifacts_dir) - dst = os.path.join(role_dir, "files", rel) + dst = os.path.join(dst_files_dir, rel) + + # If a file was successfully templatised by JinjaTurtle, do NOT + # also materialize the raw copy in the destination files dir. + # (This keeps the output minimal and avoids redundant "raw" files.) + if exclude_rels and rel in exclude_rels: + try: + if os.path.isfile(dst): + os.remove(dst) + except Exception: + pass + continue + + if preserve_existing and os.path.exists(dst): + continue os.makedirs(os.path.dirname(dst), exist_ok=True) shutil.copy2(src, dst) @@ -32,9 +188,10 @@ def _write_role_scaffold(role_dir: str) -> None: os.makedirs(os.path.join(role_dir, "defaults"), exist_ok=True) os.makedirs(os.path.join(role_dir, "meta"), exist_ok=True) os.makedirs(os.path.join(role_dir, "files"), exist_ok=True) + os.makedirs(os.path.join(role_dir, "templates"), exist_ok=True) -def _write_playbook(path: str, roles: List[str]) -> None: +def _write_playbook_all(path: str, roles: List[str]) -> None: pb_lines = [ "---", "- name: Apply all roles on host", @@ -48,7 +205,306 @@ def _write_playbook(path: str, roles: List[str]) -> None: f.write("\n".join(pb_lines) + "\n") -def manifest(bundle_dir: str, out_dir: str) -> None: +def _write_playbook_host(path: str, fqdn: str, roles: List[str]) -> None: + pb_lines = [ + "---", + f"- name: Apply enroll roles on {fqdn}", + f" hosts: {fqdn}", + " become: true", + " roles:", + ] + for r in roles: + pb_lines.append(f" - {r}") + with open(path, "w", encoding="utf-8") as f: + f.write("\n".join(pb_lines) + "\n") + + +def _ensure_ansible_cfg(cfg_path: str) -> None: + if not os.path.exists(cfg_path): + with open(cfg_path, "w", encoding="utf-8") as f: + f.write("[defaults]\n") + f.write("roles_path = roles\n") + f.write("interpreter_python=/usr/bin/python3\n") + f.write("inventory = inventory\n") + f.write("stdout_callback = unixy\n") + f.write("force_color = 1\n") + f.write("vars_plugins_enabled = host_group_vars\n") + f.write("fact_caching = jsonfile\n") + f.write("fact_caching_connection = .enroll_cached_facts\n") + f.write("forks = 30\n") + f.write("remote_tmp = /tmp/ansible-${USER}\n") + f.write("timeout = 12\n") + f.write("[ssh_connection]\n") + f.write("pipelining = True\n") + f.write("scp_if_ssh = True\n") + return + + +def _ensure_inventory_host(inv_path: str, fqdn: str) -> None: + os.makedirs(os.path.dirname(inv_path), exist_ok=True) + if not os.path.exists(inv_path): + with open(inv_path, "w", encoding="utf-8") as f: + f.write("[all]\n") + f.write(fqdn + "\n") + return + + with open(inv_path, "r", encoding="utf-8") as f: + lines = [ln.rstrip("\n") for ln in f.readlines()] + + # ensure there is an [all] group; if not, create it at top + if not any(ln.strip() == "[all]" for ln in lines): + lines = ["[all]"] + lines + + # check if fqdn already present (exact match, ignoring whitespace) + if any(ln.strip() == fqdn for ln in lines): + return + + # append at end + lines.append(fqdn) + with open(inv_path, "w", encoding="utf-8") as f: + f.write("\n".join(lines) + "\n") + + +def _hostvars_path(site_root: str, fqdn: str, role: str) -> str: + return os.path.join(site_root, "inventory", "host_vars", fqdn, f"{role}.yml") + + +def _host_role_files_dir(site_root: str, fqdn: str, role: str) -> str: + """Host-specific files dir for a given role. + + Layout: + inventory/host_vars///.files/ + """ + return os.path.join(site_root, "inventory", "host_vars", fqdn, role, ".files") + + +def _write_hostvars(site_root: str, fqdn: str, role: str, data: Dict[str, Any]) -> None: + """Write host_vars YAML for a role for a specific host. + + This is host-specific state and should track the current harvest output. + Existing keys not mentioned in `data` are preserved, but keys in `data` + are overwritten (including list values). + """ + path = _hostvars_path(site_root, fqdn, role) + os.makedirs(os.path.dirname(path), exist_ok=True) + + existing_map: Dict[str, Any] = {} + if os.path.exists(path): + try: + existing_text = Path(path).read_text(encoding="utf-8") + existing_map = _yaml_load_mapping(existing_text) + except Exception: + existing_map = {} + + merged = _merge_mappings_overwrite(existing_map, data) + + out = "# Generated by enroll (host-specific vars)\n---\n" + _yaml_dump_mapping( + merged, sort_keys=True + ) + with open(path, "w", encoding="utf-8") as f: + f.write(out) + + +def _jinjify_managed_files( + bundle_dir: str, + role: str, + role_dir: str, + managed_files: List[Dict[str, Any]], + *, + jt_exe: Optional[str], + jt_enabled: bool, + overwrite_templates: bool, +) -> Tuple[Set[str], str]: + """ + Return (templated_src_rels, combined_vars_text). + combined_vars_text is a YAML mapping fragment (no leading ---). + """ + templated: Set[str] = set() + vars_map: Dict[str, Any] = {} + + if not (jt_enabled and jt_exe): + return templated, "" + + for mf in managed_files: + dest_path = mf.get("path", "") + src_rel = mf.get("src_rel", "") + if not dest_path or not src_rel: + continue + if not can_jinjify_path(dest_path): + continue + + artifact_path = os.path.join(bundle_dir, "artifacts", role, src_rel) + if not os.path.isfile(artifact_path): + continue + + try: + res = run_jinjaturtle(jt_exe, artifact_path, role_name=role) + except Exception: + # If jinjaturtle cannot process a file for any reason, skip silently. + # (Enroll's core promise is to be optimistic and non-interactive.) + continue + + tmpl_rel = src_rel + ".j2" + tmpl_dst = os.path.join(role_dir, "templates", tmpl_rel) + if overwrite_templates or not os.path.exists(tmpl_dst): + os.makedirs(os.path.dirname(tmpl_dst), exist_ok=True) + with open(tmpl_dst, "w", encoding="utf-8") as f: + f.write(res.template_text) + + templated.add(src_rel) + if res.vars_text.strip(): + # merge YAML mappings; last wins (avoids duplicate keys) + chunk = _yaml_load_mapping(res.vars_text) + if chunk: + vars_map = _merge_mappings_overwrite(vars_map, chunk) + + if vars_map: + combined = _yaml_dump_mapping(vars_map, sort_keys=True) + return templated, combined + return templated, "" + + +def _hostvars_only_jinjaturtle(vars_text: str) -> str: + # keep as valid YAML file + return _defaults_with_jinjaturtle("---\n", vars_text) + + +def _defaults_with_jinjaturtle(base_defaults: str, vars_text: str) -> str: + if not vars_text.strip(): + return base_defaults.rstrip() + "\n" + vars_text = _normalize_jinjaturtle_vars_text(vars_text) + # Always regenerate the block (we regenerate whole defaults files anyway) + return ( + base_defaults.rstrip() + + "\n\n" + + JINJATURTLE_BEGIN + + "\n" + + vars_text.rstrip() + + "\n" + + JINJATURTLE_END + + "\n" + ) + + + +def _write_role_defaults(role_dir: str, mapping: Dict[str, Any]) -> None: + """Overwrite role defaults/main.yml with the provided mapping.""" + defaults_path = os.path.join(role_dir, "defaults", "main.yml") + os.makedirs(os.path.dirname(defaults_path), exist_ok=True) + out = "---\n" + _yaml_dump_mapping(mapping, sort_keys=True) + with open(defaults_path, "w", encoding="utf-8") as f: + f.write(out) + + +def _build_managed_files_var( + managed_files: List[Dict[str, Any]], + templated_src_rels: Set[str], + *, + notify_other: Optional[str] = None, + notify_systemd: Optional[str] = None, +) -> List[Dict[str, Any]]: + """Convert enroll managed_files into an Ansible-friendly list of dicts. + + Each dict drives a role task loop and is safe across hosts. + """ + out: List[Dict[str, Any]] = [] + for mf in managed_files: + dest = mf.get("path") or "" + src_rel = mf.get("src_rel") or "" + if not dest or not src_rel: + continue + is_unit = str(dest).startswith("/etc/systemd/system/") + kind = "template" if src_rel in templated_src_rels else "copy" + notify: List[str] = [] + if is_unit and notify_systemd: + notify.append(notify_systemd) + if (not is_unit) and notify_other: + notify.append(notify_other) + out.append( + { + "dest": dest, + "src_rel": src_rel, + "owner": mf.get("owner") or "root", + "group": mf.get("group") or "root", + "mode": mf.get("mode") or "0644", + "kind": kind, + "is_systemd_unit": bool(is_unit), + "notify": notify, + } + ) + return out + + +def _render_generic_files_tasks(var_prefix: str, *, include_restart_notify: bool) -> str: + """Render generic tasks to deploy _managed_files safely.""" + # Using first_found makes roles work in both modes: + # - site-mode: inventory/host_vars///.files/... + # - non-site: roles//files/... + restart_notify = "Restart service" if include_restart_notify else "" + return f"""# Generated by enroll (data-driven tasks) + +- name: Deploy systemd unit files (templates) + ansible.builtin.template: + src: "{{{{ item.src_rel }}}}.j2" + dest: "{{{{ item.dest }}}}" + owner: "{{{{ item.owner }}}}" + group: "{{{{ item.group }}}}" + mode: "{{{{ item.mode }}}}" + loop: "{{{{ {var_prefix}_managed_files | default([]) | selectattr('is_systemd_unit','equalto', true) | selectattr('kind','equalto','template') | list }}}}" + notify: "{{{{ item.notify | default([]) }}}}" + +- name: Deploy systemd unit files (copies) + vars: + _enroll_ff: + files: + - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ item.src_rel }}}}" + - "{{{{ role_path }}}}/files/{{{{ item.src_rel }}}}" + ansible.builtin.copy: + src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" + dest: "{{{{ item.dest }}}}" + owner: "{{{{ item.owner }}}}" + group: "{{{{ item.group }}}}" + mode: "{{{{ item.mode }}}}" + loop: "{{{{ {var_prefix}_managed_files | default([]) | selectattr('is_systemd_unit','equalto', true) | selectattr('kind','equalto','copy') | list }}}}" + notify: "{{{{ item.notify | default([]) }}}}" + +- name: Reload systemd to pick up unit changes + ansible.builtin.meta: flush_handlers + when: "({var_prefix}_managed_files | default([]) | selectattr('is_systemd_unit','equalto', true) | list | length) > 0" + +- name: Deploy other managed files (templates) + ansible.builtin.template: + src: "{{{{ item.src_rel }}}}.j2" + dest: "{{{{ item.dest }}}}" + owner: "{{{{ item.owner }}}}" + group: "{{{{ item.group }}}}" + mode: "{{{{ item.mode }}}}" + loop: "{{{{ {var_prefix}_managed_files | default([]) | selectattr('is_systemd_unit','equalto', false) | selectattr('kind','equalto','template') | list }}}}" + notify: "{{{{ item.notify | default([]) }}}}" + +- name: Deploy other managed files (copies) + vars: + _enroll_ff: + files: + - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ item.src_rel }}}}" + - "{{{{ role_path }}}}/files/{{{{ item.src_rel }}}}" + ansible.builtin.copy: + src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" + dest: "{{{{ item.dest }}}}" + owner: "{{{{ item.owner }}}}" + group: "{{{{ item.group }}}}" + mode: "{{{{ item.mode }}}}" + loop: "{{{{ {var_prefix}_managed_files | default([]) | selectattr('is_systemd_unit','equalto', false) | selectattr('kind','equalto','copy') | list }}}}" + notify: "{{{{ item.notify | default([]) }}}}" +""" + +def manifest( + bundle_dir: str, + out_dir: str, + *, + fqdn: Optional[str] = None, + jinjaturtle: str = "auto", # auto|on|off +) -> None: state_path = os.path.join(bundle_dir, "state.json") with open(state_path, "r", encoding="utf-8") as f: state = json.load(f) @@ -58,15 +514,45 @@ def manifest(bundle_dir: str, out_dir: str) -> None: users_snapshot: Dict[str, Any] = state.get("users", {}) etc_custom_snapshot: Dict[str, Any] = state.get("etc_custom", {}) + site_mode = fqdn is not None and fqdn != "" + + jt_exe = find_jinjaturtle_cmd() + jt_enabled = False + if jinjaturtle not in ("auto", "on", "off"): + raise ValueError("jinjaturtle must be one of: auto, on, off") + if jinjaturtle == "on": + if not jt_exe: + raise RuntimeError("jinjaturtle requested but not found on PATH") + jt_enabled = True + elif jinjaturtle == "auto": + jt_enabled = jt_exe is not None + else: + jt_enabled = False + os.makedirs(out_dir, exist_ok=True) roles_root = os.path.join(out_dir, "roles") os.makedirs(roles_root, exist_ok=True) + # Site-mode scaffolding + if site_mode: + os.makedirs(os.path.join(out_dir, "inventory"), exist_ok=True) + os.makedirs(os.path.join(out_dir, "inventory", "host_vars"), exist_ok=True) + os.makedirs(os.path.join(out_dir, "playbooks"), exist_ok=True) + _ensure_inventory_host( + os.path.join(out_dir, "inventory", "hosts.ini"), fqdn or "" + ) + _ensure_ansible_cfg(os.path.join(out_dir, "ansible.cfg")) + manifested_users_roles: List[str] = [] manifested_etc_custom_roles: List[str] = [] manifested_service_roles: List[str] = [] manifested_pkg_roles: List[str] = [] + # In site_mode, raw harvested files are stored under host-specific inventory + # to avoid cross-host clobber while still sharing a role definition. + + # ------------------------- + # ------------------------- # Users role (non-system users) # ------------------------- @@ -74,146 +560,191 @@ def manifest(bundle_dir: str, out_dir: str) -> None: role = users_snapshot.get("role_name", "users") role_dir = os.path.join(roles_root, role) _write_role_scaffold(role_dir) - _copy_artifacts(bundle_dir, role, role_dir) + + # Users role includes harvested SSH-related files; in site mode keep them + # host-specific to avoid cross-host clobber. + if site_mode: + _copy_artifacts( + bundle_dir, role, _host_role_files_dir(out_dir, fqdn or "", role) + ) + else: + _copy_artifacts(bundle_dir, role, os.path.join(role_dir, "files")) users = users_snapshot.get("users", []) managed_files = users_snapshot.get("managed_files", []) excluded = users_snapshot.get("excluded", []) notes = users_snapshot.get("notes", []) - # Build group set from users - group_names = set() + # Build groups list and a simplified user dict list suitable for loops + group_names: List[str] = [] + group_set = set() + users_data: List[Dict[str, Any]] = [] for u in users: - pg = u.get("primary_group") - if pg: - group_names.add(pg) - for g in u.get("supplementary_groups", []) or []: - group_names.add(g) - group_names = sorted(group_names) - - # defaults: store users list (handy for later), but tasks are explicit for readability - defaults = """--- -users_accounts: -""" + ( - "\n".join([f" - name: {u.get('name')}" for u in users]) + "\n" - ) - with open( - os.path.join(role_dir, "defaults", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(defaults) - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\ndependencies: []\n") - - # tasks - lines: List[str] = ["---"] - # groups first (idempotent; safe even if already present) - for g in group_names: - lines.append(f"- name: Ensure group {g} exists") - lines.append(" ansible.builtin.group:") - lines.append(f" name: {g}") - lines.append(" state: present") - - # users - for u in users: - name = u["name"] - lines.append(f"- name: Ensure user {name} exists") - lines.append(" ansible.builtin.user:") - lines.append(f" name: {name}") - lines.append(f" uid: {u.get('uid')}") - lines.append(f" group: {u.get('primary_group')}") - supp = u.get("supplementary_groups") or [] - if supp: - lines.append(" groups: " + ",".join(sorted(supp))) - lines.append(" append: true") - lines.append(f" home: {u.get('home')}") - lines.append(" create_home: true") - if u.get("shell"): - lines.append(f" shell: {u.get('shell')}") - if u.get("gecos"): - # quote to avoid YAML surprises - gec = u.get("gecos").replace('"', '"') - lines.append(f' comment: "{gec}"') - lines.append(" state: present") - - # Ensure ~/.ssh + name = u.get("name") + if not name: + continue + pg = u.get("primary_group") or name home = u.get("home") or f"/home/{name}" sshdir = home.rstrip("/") + "/.ssh" - lines.append(f"- name: Ensure {name} .ssh directory exists") - lines.append(" ansible.builtin.file:") - lines.append(f" path: {sshdir}") - lines.append(" state: directory") - lines.append(f" owner: {name}") - lines.append(f" group: {u.get('primary_group')}") - lines.append(" mode: '0700'") + supp = u.get("supplementary_groups") or [] + if pg: + group_set.add(pg) + for g in supp: + if g: + group_set.add(g) - # Copy harvested SSH public material (authorized_keys) + users_data.append( + { + "name": name, + "uid": u.get("uid"), + "primary_group": pg, + "home": home, + "ssh_dir": sshdir, + "shell": u.get("shell"), + "gecos": u.get("gecos"), + "supplementary_groups": sorted(set(supp)), + } + ) + + group_names = sorted(group_set) + + # SSH-related files (authorized_keys, known_hosts, config, etc.) + ssh_files: List[Dict[str, Any]] = [] for mf in managed_files: - dest = mf["path"] - src = mf["src_rel"] - # Determine file owner from dest path: /home//... - owner = None - for u in users: - if dest.startswith((u.get("home") or "").rstrip("/") + "/"): - owner = u["name"] - group = u.get("primary_group") + dest = mf.get("path") or "" + src_rel = mf.get("src_rel") or "" + if not dest or not src_rel: + continue + + owner = "root" + group = "root" + for u in users_data: + home_prefix = (u.get("home") or "").rstrip("/") + "/" + if home_prefix and dest.startswith(home_prefix): + owner = str(u.get("name") or "root") + group = str(u.get("primary_group") or owner) break - if owner is None: - # fallback: try /home// - parts = dest.split("/") - owner = parts[2] if len(parts) > 2 and parts[1] == "home" else "root" - group = owner mode = "0600" if mf.get("reason") == "authorized_keys" else "0644" - lines.append(f"- name: Deploy {dest}") - lines.append(" ansible.builtin.copy:") - lines.append(f" src: {src}") - lines.append(f" dest: {dest}") - lines.append(f" owner: {owner}") - lines.append(f" group: {group}") - lines.append(f" mode: '{mode}'") + ssh_files.append( + { + "dest": dest, + "src_rel": src_rel, + "owner": owner, + "group": group, + "mode": mode, + } + ) - tasks = "\n".join(lines).rstrip() + "\n" - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks) + # Variables are host-specific in site mode; in non-site mode they live in role defaults. + if site_mode: + _write_role_defaults( + role_dir, + { + "users_groups": [], + "users_users": [], + "users_ssh_files": [], + }, + ) + _write_hostvars( + out_dir, + fqdn or "", + role, + { + "users_groups": group_names, + "users_users": users_data, + "users_ssh_files": ssh_files, + }, + ) + else: + _write_role_defaults( + role_dir, + { + "users_groups": group_names, + "users_users": users_data, + "users_ssh_files": ssh_files, + }, + ) - # handlers (none needed) - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: + with open(os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8") as f: + f.write("---\ndependencies: []\n") + + # tasks (data-driven) + users_tasks = """--- +# Generated by enroll (data-driven tasks) + +- name: Ensure groups exist + ansible.builtin.group: + name: "{{ item }}" + state: present + loop: "{{ users_groups | default([]) }}" + +- name: Ensure users exist + ansible.builtin.user: + name: "{{ item.name }}" + uid: "{{ item.uid | default(omit) }}" + group: "{{ item.primary_group }}" + home: "{{ item.home }}" + create_home: true + shell: "{{ item.shell | default(omit) }}" + comment: "{{ item.gecos | default(omit) }}" + state: present + loop: "{{ users_users | default([]) }}" + +- name: Ensure users supplementary groups + ansible.builtin.user: + name: "{{ item.name }}" + groups: "{{ item.supplementary_groups | default([]) | join(',') }}" + append: true + loop: "{{ users_users | default([]) }}" + when: (item.supplementary_groups | default([])) | length > 0 + +- name: Ensure .ssh directories exist + ansible.builtin.file: + path: "{{ item.ssh_dir }}" + state: directory + owner: "{{ item.name }}" + group: "{{ item.primary_group }}" + mode: "0700" + loop: "{{ users_users | default([]) }}" + +- name: Deploy SSH-related files + vars: + _enroll_ff: + files: + - "{{ inventory_dir }}/host_vars/{{ inventory_hostname }}/{{ role_name }}/.files/{{ item.src_rel }}" + - "{{ role_path }}/files/{{ item.src_rel }}" + ansible.builtin.copy: + src: "{{ lookup('ansible.builtin.first_found', _enroll_ff) }}" + dest: "{{ item.dest }}" + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: "{{ item.mode }}" + loop: "{{ users_ssh_files | default([]) }}" +""" + + with open(os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8") as f: + f.write(users_tasks) + + with open(os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8") as f: f.write("---\n") readme = ( - """# users + f"""# users Generated non-system user accounts and SSH public material. ## Users """ - + ( - "\n".join([f"- {u.get('name')} (uid {u.get('uid')})" for u in users]) - or "- (none)" - ) + + ("\n".join([f"- {u.get('name')} (uid {u.get('uid')})" for u in users]) or "- (none)") + """\n ## Included SSH files """ - + ( - "\n".join( - [f"- {mf.get('path')} ({mf.get('reason')})" for mf in managed_files] - ) - or "- (none)" - ) + + ("\n".join([f"- {mf.get('path')} ({mf.get('reason')})" for mf in managed_files]) or "- (none)") + """\n ## Excluded """ - + ( - "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) - or "- (none)" - ) + + ("\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) or "- (none)") + """\n ## Notes """ @@ -225,6 +756,8 @@ Generated non-system user accounts and SSH public material. manifested_users_roles.append(role) + # ------------------------- + # ------------------------- # etc_custom role (unowned /etc not already attributed) # ------------------------- @@ -232,38 +765,69 @@ Generated non-system user accounts and SSH public material. role = etc_custom_snapshot.get("role_name", "etc_custom") role_dir = os.path.join(roles_root, role) _write_role_scaffold(role_dir) - _copy_artifacts(bundle_dir, role, role_dir) + + var_prefix = role managed_files = etc_custom_snapshot.get("managed_files", []) excluded = etc_custom_snapshot.get("excluded", []) notes = etc_custom_snapshot.get("notes", []) - # tasks: just deploy files (no restarts) - lines: List[str] = ["---"] - for mf in managed_files: - dest = mf["path"] - src = mf["src_rel"] - lines.append(f"- name: Deploy {dest}") - lines.append(" ansible.builtin.copy:") - lines.append(f" src: {src}") - lines.append(f" dest: {dest}") - lines.append(f" owner: {mf.get('owner')}") - lines.append(f" group: {mf.get('group')}") - lines.append(f" mode: '{mf.get('mode')}'") + templated, jt_vars = _jinjify_managed_files( + bundle_dir, + role, + role_dir, + managed_files, + jt_exe=jt_exe, + jt_enabled=jt_enabled, + overwrite_templates=not site_mode, + ) - tasks = "\n".join(lines).rstrip() + "\n" - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks) + # Copy only the non-templated artifacts (templates live in the role). + if site_mode: + _copy_artifacts( + bundle_dir, + role, + _host_role_files_dir(out_dir, fqdn or "", role), + exclude_rels=templated, + ) + else: + _copy_artifacts( + bundle_dir, + role, + os.path.join(role_dir, "files"), + exclude_rels=templated, + ) - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\n") - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: + files_var = _build_managed_files_var( + managed_files, + templated, + notify_other=None, + notify_systemd="Run systemd daemon-reload", + ) + + jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} + vars_map: Dict[str, Any] = {f"{var_prefix}_managed_files": files_var} + vars_map = _merge_mappings_overwrite(vars_map, jt_map) + + if site_mode: + _write_role_defaults(role_dir, {f"{var_prefix}_managed_files": []}) + _write_hostvars(out_dir, fqdn or "", role, vars_map) + else: + _write_role_defaults(role_dir, vars_map) + + tasks = """---\n""" + _render_generic_files_tasks(var_prefix, include_restart_notify=False) + with open(os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8") as f: + f.write(tasks.rstrip() + "\n") + + handlers = """--- +- name: Run systemd daemon-reload + ansible.builtin.systemd: + daemon_reload: true +""" + with open(os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8") as f: + f.write(handlers) + + with open(os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8") as f: f.write("---\ndependencies: []\n") readme = ( @@ -277,10 +841,7 @@ Unowned /etc config files not attributed to packages or services. + """\n ## Excluded """ - + ( - "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) - or "- (none)" - ) + + ("\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) or "- (none)") + """\n ## Notes """ @@ -292,127 +853,155 @@ Unowned /etc config files not attributed to packages or services. manifested_etc_custom_roles.append(role) + # ------------------------- + # ------------------------- # Service roles # ------------------------- for svc in services: role = svc["role_name"] unit = svc["unit"] - pkgs = svc["packages"] - managed_files = svc["managed_files"] + pkgs = svc.get("packages", []) or [] + managed_files = svc.get("managed_files", []) or [] role_dir = os.path.join(roles_root, role) _write_role_scaffold(role_dir) - _copy_artifacts(bundle_dir, role, role_dir) var_prefix = role was_active = svc.get("active_state") == "active" - defaults = f"""--- -{var_prefix}_packages: -{_yaml_list(pkgs, indent=2)} -{var_prefix}_active_state_at_harvest: "{svc.get("active_state")}" -{var_prefix}_start: {"true" if was_active else "false"} -""" - with open( - os.path.join(role_dir, "defaults", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(defaults) + unit_file_state = str(svc.get("unit_file_state") or "") + enabled_at_harvest = unit_file_state in ("enabled", "enabled-runtime") + desired_state = "started" if was_active else "stopped" - handlers = """--- + templated, jt_vars = _jinjify_managed_files( + bundle_dir, + role, + role_dir, + managed_files, + jt_exe=jt_exe, + jt_enabled=jt_enabled, + overwrite_templates=not site_mode, + ) + + # Copy only the non-templated artifacts. + if site_mode: + _copy_artifacts( + bundle_dir, + role, + _host_role_files_dir(out_dir, fqdn or "", role), + exclude_rels=templated, + ) + else: + _copy_artifacts( + bundle_dir, + role, + os.path.join(role_dir, "files"), + exclude_rels=templated, + ) + + files_var = _build_managed_files_var( + managed_files, + templated, + notify_other="Restart service", + notify_systemd="Run systemd daemon-reload", + ) + + jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} + base_vars: Dict[str, Any] = { + f"{var_prefix}_unit_name": unit, + f"{var_prefix}_packages": pkgs, + f"{var_prefix}_managed_files": files_var, + f"{var_prefix}_manage_unit": True, + f"{var_prefix}_systemd_enabled": bool(enabled_at_harvest), + f"{var_prefix}_systemd_state": desired_state, + } + base_vars = _merge_mappings_overwrite(base_vars, jt_map) + + if site_mode: + # Role defaults are host-agnostic/safe; all harvested state is in host_vars. + _write_role_defaults( + role_dir, + { + f"{var_prefix}_unit_name": unit, + f"{var_prefix}_packages": [], + f"{var_prefix}_managed_files": [], + f"{var_prefix}_manage_unit": False, + f"{var_prefix}_systemd_enabled": False, + f"{var_prefix}_systemd_state": "stopped", + }, + ) + _write_hostvars(out_dir, fqdn or "", role, base_vars) + else: + _write_role_defaults(role_dir, base_vars) + + handlers = f"""--- - name: Run systemd daemon-reload ansible.builtin.systemd: daemon_reload: true - name: Restart service ansible.builtin.service: - name: "{{ unit_name }}" + name: "{{{{ {var_prefix}_unit_name }}}}" state: restarted + when: + - {var_prefix}_manage_unit | default(false) + - ({var_prefix}_systemd_state | default('stopped')) == 'started' """ - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: + with open(os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8") as f: f.write(handlers) - systemd_files = [ - mf for mf in managed_files if mf["path"].startswith("/etc/systemd/system/") - ] - other_files = [mf for mf in managed_files if mf not in systemd_files] - - def copy_task(mf: Dict[str, Any], notify: str | None) -> str: - notify_line = f" notify: {notify}\n" if notify else "" - return f"""- name: Deploy {mf["path"]} - ansible.builtin.copy: - src: "{mf["src_rel"]}" - dest: "{mf["path"]}" - owner: "{mf["owner"]}" - group: "{mf["group"]}" - mode: "{mf["mode"]}" -{notify_line}""" - task_parts: List[str] = [] task_parts.append( f"""--- -- name: Set unit name - ansible.builtin.set_fact: - unit_name: "{unit}" +# Generated by enroll (data-driven tasks) - name: Install packages for {role} ansible.builtin.apt: - name: "{{{{ {var_prefix}_packages }}}}" + name: "{{{{ {var_prefix}_packages | default([]) }}}}" state: present update_cache: true - when: {var_prefix}_packages | length > 0 + when: ({var_prefix}_packages | default([])) | length > 0 + """ ) - if systemd_files: - for mf in systemd_files: - task_parts.append(copy_task(mf, "[systemd daemon-reload]")) - task_parts.append( - """- name: Reload systemd to pick up unit changes - ansible.builtin.meta: flush_handlers -""" - ) - - for mf in other_files: - task_parts.append(copy_task(mf, "[Restart service]")) + task_parts.append(_render_generic_files_tasks(var_prefix, include_restart_notify=True)) task_parts.append( f""" - name: Probe whether systemd unit exists and is manageable ansible.builtin.systemd: - name: "{{{{ unit_name }}}}" + name: "{{{{ {var_prefix}_unit_name }}}}" check_mode: true register: _unit_probe failed_when: false changed_when: false + when: {var_prefix}_manage_unit | default(false) -- name: Ensure unit is enabled (preserve running state) +- name: Ensure unit enablement matches harvest ansible.builtin.systemd: - name: "{{{{ unit_name }}}}" - enabled: true - when: _unit_probe is succeeded - -- name: Start unit if it was active at harvest time - ansible.builtin.systemd: - name: "{{{{ unit_name }}}}" - state: started + name: "{{{{ {var_prefix}_unit_name }}}}" + enabled: "{{{{ {var_prefix}_systemd_enabled | bool }}}}" when: + - {var_prefix}_manage_unit | default(false) + - _unit_probe is succeeded + +- name: Ensure unit running state matches harvest + ansible.builtin.systemd: + name: "{{{{ {var_prefix}_unit_name }}}}" + state: "{{{{ {var_prefix}_systemd_state }}}}" + when: + - {var_prefix}_manage_unit | default(false) - _unit_probe is succeeded - - {var_prefix}_start | bool """ ) tasks = "\n".join(task_parts).rstrip() + "\n" - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: + with open(os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8") as f: f.write(tasks) - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: + with open(os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8") as f: f.write("---\ndependencies: []\n") excluded = svc.get("excluded", []) @@ -443,82 +1032,97 @@ Generated from `{unit}`. # ------------------------- for pr in package_roles: role = pr["role_name"] - pkg = pr["package"] - managed_files = pr["managed_files"] + pkg = pr.get("package") or "" + managed_files = pr.get("managed_files", []) or [] role_dir = os.path.join(roles_root, role) _write_role_scaffold(role_dir) - _copy_artifacts(bundle_dir, role, role_dir) var_prefix = role - defaults = f"""--- -{var_prefix}_packages: - - {pkg} -""" - with open( - os.path.join(role_dir, "defaults", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(defaults) + templated, jt_vars = _jinjify_managed_files( + bundle_dir, + role, + role_dir, + managed_files, + jt_exe=jt_exe, + jt_enabled=jt_enabled, + overwrite_templates=not site_mode, + ) + + # Copy only the non-templated artifacts. + if site_mode: + _copy_artifacts( + bundle_dir, + role, + _host_role_files_dir(out_dir, fqdn or "", role), + exclude_rels=templated, + ) + else: + _copy_artifacts( + bundle_dir, + role, + os.path.join(role_dir, "files"), + exclude_rels=templated, + ) + + pkgs = [pkg] if pkg else [] + + files_var = _build_managed_files_var( + managed_files, + templated, + notify_other=None, + notify_systemd="Run systemd daemon-reload", + ) + + jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} + base_vars: Dict[str, Any] = { + f"{var_prefix}_packages": pkgs, + f"{var_prefix}_managed_files": files_var, + } + base_vars = _merge_mappings_overwrite(base_vars, jt_map) + + if site_mode: + _write_role_defaults( + role_dir, + { + f"{var_prefix}_packages": [], + f"{var_prefix}_managed_files": [], + }, + ) + _write_hostvars(out_dir, fqdn or "", role, base_vars) + else: + _write_role_defaults(role_dir, base_vars) handlers = """--- - name: Run systemd daemon-reload ansible.builtin.systemd: daemon_reload: true """ - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: + with open(os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8") as f: f.write(handlers) - systemd_files = [ - mf for mf in managed_files if mf["path"].startswith("/etc/systemd/system/") - ] - other_files = [mf for mf in managed_files if mf not in systemd_files] - - def copy_task(mf: Dict[str, Any], notify: str | None) -> str: - notify_line = f" notify: {notify}\n" if notify else "" - return f"""- name: Deploy {mf["path"]} - ansible.builtin.copy: - src: "{mf["src_rel"]}" - dest: "{mf["path"]}" - owner: "{mf["owner"]}" - group: "{mf["group"]}" - mode: "{mf["mode"]}" -{notify_line}""" - task_parts: List[str] = [] task_parts.append( f"""--- -- name: Install package {pkg} +# Generated by enroll (data-driven tasks) + +- name: Install packages for {role} ansible.builtin.apt: - name: "{{{{ {var_prefix}_packages }}}}" + name: "{{{{ {var_prefix}_packages | default([]) }}}}" state: present update_cache: true + when: ({var_prefix}_packages | default([])) | length > 0 + """ ) - - if systemd_files: - for mf in systemd_files: - task_parts.append(copy_task(mf, "[systemd daemon-reload]")) - task_parts.append( - """- name: Reload systemd to pick up unit changes - ansible.builtin.meta: flush_handlers -""" - ) - - for mf in other_files: - task_parts.append(copy_task(mf, None)) + task_parts.append(_render_generic_files_tasks(var_prefix, include_restart_notify=False)) tasks = "\n".join(task_parts).rstrip() + "\n" - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: + with open(os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8") as f: f.write(tasks) - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: + with open(os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8") as f: f.write("---\ndependencies: []\n") excluded = pr.get("excluded", []) @@ -542,12 +1146,16 @@ Generated for package `{pkg}`. f.write(readme) manifested_pkg_roles.append(role) - - # Playbooks - _write_playbook( - os.path.join(out_dir, "playbook.yml"), + all_roles = ( manifested_pkg_roles + manifested_service_roles + manifested_etc_custom_roles - + manifested_users_roles, + + manifested_users_roles ) + + if site_mode: + _write_playbook_host( + os.path.join(out_dir, "playbooks", f"{fqdn}.yml"), fqdn or "", all_roles + ) + else: + _write_playbook_all(os.path.join(out_dir, "playbook.yml"), all_roles) diff --git a/poetry.lock b/poetry.lock index a056de4..d594a1a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "certifi" version = "2025.11.12" description = "Python package for providing Mozilla's CA Bundle." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -15,6 +16,7 @@ files = [ name = "charset-normalizer" version = "3.4.4" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -137,6 +139,7 @@ files = [ name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -148,6 +151,7 @@ files = [ name = "coverage" version = "7.13.0" description = "Code coverage measurement for Python" +category = "dev" optional = false python-versions = ">=3.10" files = [ @@ -255,6 +259,7 @@ toml = ["tomli"] name = "desktop-entry-lib" version = "5.0" description = "A library for working with .desktop files" +category = "dev" optional = false python-versions = ">=3.10" files = [ @@ -269,6 +274,7 @@ xdg-desktop-portal = ["jeepney"] name = "exceptiongroup" version = "1.3.1" description = "Backport of PEP 654 (exception groups)" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -286,6 +292,7 @@ test = ["pytest (>=6)"] name = "idna" version = "3.11" description = "Internationalized Domain Names in Applications (IDNA)" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -300,6 +307,7 @@ all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2 name = "iniconfig" version = "2.3.0" description = "brain-dead simple config-ini parsing" +category = "dev" optional = false python-versions = ">=3.10" files = [ @@ -311,6 +319,7 @@ files = [ name = "packaging" version = "25.0" description = "Core utilities for Python packages" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -322,6 +331,7 @@ files = [ name = "pluggy" version = "1.6.0" description = "plugin and hook calling mechanisms for python" +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -337,6 +347,7 @@ testing = ["coverage", "pytest", "pytest-benchmark"] name = "pygments" version = "2.19.2" description = "Pygments is a syntax highlighting package written in Python." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -351,6 +362,7 @@ windows-terminal = ["colorama (>=0.4.6)"] name = "pyproject-appimage" version = "4.2" description = "Generate AppImages from your Python projects" +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -367,6 +379,7 @@ tomli = {version = "*", markers = "python_version < \"3.11\""} name = "pytest" version = "8.4.2" description = "pytest: simple powerful testing with Python" +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -390,6 +403,7 @@ dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests name = "pytest-cov" version = "5.0.0" description = "Pytest plugin for measuring coverage." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -404,10 +418,94 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] +[[package]] +name = "pyyaml" +version = "6.0.3" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.8" +files = [ + {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, + {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, + {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, + {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, + {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, + {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, + {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, + {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, + {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, + {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, + {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, + {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, + {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, + {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, + {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, + {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, + {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, + {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, + {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, + {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, + {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, + {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, + {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, + {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, + {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, + {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, + {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, + {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, + {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, + {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, + {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, + {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, + {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, + {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, + {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, + {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, + {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, + {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, + {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, + {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, + {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, + {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, + {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, + {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, + {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, + {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, + {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, + {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, + {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, + {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, + {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, + {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, + {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, + {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, + {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, + {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, + {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, + {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, + {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, + {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, + {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, + {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, + {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, + {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, + {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, + {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, + {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, + {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, + {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, + {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, +] + [[package]] name = "requests" version = "2.32.5" description = "Python HTTP for Humans." +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -429,6 +527,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "tomli" version = "2.3.0" description = "A lil' TOML parser" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -480,6 +579,7 @@ files = [ name = "typing-extensions" version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -491,6 +591,7 @@ files = [ name = "urllib3" version = "2.6.2" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -507,4 +608,4 @@ zstd = ["backports-zstd (>=1.0.0)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "84c06974dfe822257ef324807672e51d71c0a6197e037fa56e92d8369c40d341" +content-hash = "10c279bd393cab27a94b4848c6f88f3a7a3d1af5062882c3e6fd2c2e15c945c8" diff --git a/pyproject.toml b/pyproject.toml index 6b4d1b1..444def3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "enroll" -version = "0.0.4" +version = "0.0.5" description = "Enroll a server's running state retrospectively into Ansible" authors = ["Miguel Jacq "] license = "GPL-3.0-or-later" @@ -10,6 +10,7 @@ repository = "https://git.mig5.net/mig5/enroll" [tool.poetry.dependencies] python = "^3.10" +pyyaml = "^6.0.3" [tool.poetry.scripts] enroll = "enroll.cli:main" From f40b9d834d008becd2bfab3e3845a75dfe86e991 Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Tue, 16 Dec 2025 20:15:21 +1100 Subject: [PATCH 2/3] black and pyflakes3 --- enroll/manifest.py | 91 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 68 insertions(+), 23 deletions(-) diff --git a/enroll/manifest.py b/enroll/manifest.py index b772bcd..e27cfd5 100644 --- a/enroll/manifest.py +++ b/enroll/manifest.py @@ -386,7 +386,6 @@ def _defaults_with_jinjaturtle(base_defaults: str, vars_text: str) -> str: ) - def _write_role_defaults(role_dir: str, mapping: Dict[str, Any]) -> None: """Overwrite role defaults/main.yml with the provided mapping.""" defaults_path = os.path.join(role_dir, "defaults", "main.yml") @@ -435,12 +434,13 @@ def _build_managed_files_var( return out -def _render_generic_files_tasks(var_prefix: str, *, include_restart_notify: bool) -> str: +def _render_generic_files_tasks( + var_prefix: str, *, include_restart_notify: bool +) -> str: """Render generic tasks to deploy _managed_files safely.""" # Using first_found makes roles work in both modes: # - site-mode: inventory/host_vars///.files/... # - non-site: roles//files/... - restart_notify = "Restart service" if include_restart_notify else "" return f"""# Generated by enroll (data-driven tasks) - name: Deploy systemd unit files (templates) @@ -498,6 +498,7 @@ def _render_generic_files_tasks(var_prefix: str, *, include_restart_notify: bool notify: "{{{{ item.notify | default([]) }}}}" """ + def manifest( bundle_dir: str, out_dir: str, @@ -666,7 +667,9 @@ def manifest( }, ) - with open(os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8") as f: + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: f.write("---\ndependencies: []\n") # tasks (data-driven) @@ -723,28 +726,43 @@ def manifest( loop: "{{ users_ssh_files | default([]) }}" """ - with open(os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8") as f: + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: f.write(users_tasks) - with open(os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8") as f: + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: f.write("---\n") readme = ( - f"""# users + """# users Generated non-system user accounts and SSH public material. ## Users """ - + ("\n".join([f"- {u.get('name')} (uid {u.get('uid')})" for u in users]) or "- (none)") + + ( + "\n".join([f"- {u.get('name')} (uid {u.get('uid')})" for u in users]) + or "- (none)" + ) + """\n ## Included SSH files """ - + ("\n".join([f"- {mf.get('path')} ({mf.get('reason')})" for mf in managed_files]) or "- (none)") + + ( + "\n".join( + [f"- {mf.get('path')} ({mf.get('reason')})" for mf in managed_files] + ) + or "- (none)" + ) + """\n ## Excluded """ - + ("\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) or "- (none)") + + ( + "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) + or "- (none)" + ) + """\n ## Notes """ @@ -815,8 +833,12 @@ Generated non-system user accounts and SSH public material. else: _write_role_defaults(role_dir, vars_map) - tasks = """---\n""" + _render_generic_files_tasks(var_prefix, include_restart_notify=False) - with open(os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8") as f: + tasks = """---\n""" + _render_generic_files_tasks( + var_prefix, include_restart_notify=False + ) + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: f.write(tasks.rstrip() + "\n") handlers = """--- @@ -824,10 +846,14 @@ Generated non-system user accounts and SSH public material. ansible.builtin.systemd: daemon_reload: true """ - with open(os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8") as f: + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: f.write(handlers) - with open(os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8") as f: + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: f.write("---\ndependencies: []\n") readme = ( @@ -841,7 +867,10 @@ Unowned /etc config files not attributed to packages or services. + """\n ## Excluded """ - + ("\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) or "- (none)") + + ( + "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) + or "- (none)" + ) + """\n ## Notes """ @@ -948,7 +977,9 @@ Unowned /etc config files not attributed to packages or services. - {var_prefix}_manage_unit | default(false) - ({var_prefix}_systemd_state | default('stopped')) == 'started' """ - with open(os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8") as f: + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: f.write(handlers) task_parts: List[str] = [] @@ -966,7 +997,9 @@ Unowned /etc config files not attributed to packages or services. """ ) - task_parts.append(_render_generic_files_tasks(var_prefix, include_restart_notify=True)) + task_parts.append( + _render_generic_files_tasks(var_prefix, include_restart_notify=True) + ) task_parts.append( f""" @@ -998,10 +1031,14 @@ Unowned /etc config files not attributed to packages or services. ) tasks = "\n".join(task_parts).rstrip() + "\n" - with open(os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8") as f: + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: f.write(tasks) - with open(os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8") as f: + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: f.write("---\ndependencies: []\n") excluded = svc.get("excluded", []) @@ -1099,7 +1136,9 @@ Generated from `{unit}`. ansible.builtin.systemd: daemon_reload: true """ - with open(os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8") as f: + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: f.write(handlers) task_parts: List[str] = [] @@ -1116,13 +1155,19 @@ Generated from `{unit}`. """ ) - task_parts.append(_render_generic_files_tasks(var_prefix, include_restart_notify=False)) + task_parts.append( + _render_generic_files_tasks(var_prefix, include_restart_notify=False) + ) tasks = "\n".join(task_parts).rstrip() + "\n" - with open(os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8") as f: + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: f.write(tasks) - with open(os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8") as f: + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: f.write("---\ndependencies: []\n") excluded = pr.get("excluded", []) From 026416d158baef451240c0d021e4b47b0e188e56 Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Tue, 16 Dec 2025 20:48:08 +1100 Subject: [PATCH 3/3] Fix tests --- enroll/harvest.py | 2 +- enroll/manifest.py | 58 +++++++++++++++---- tests.sh | 2 +- tests/test_cli.py | 56 ++++++++++++++++-- tests/test_jinjaturtle.py | 99 ++++++++++++++++++++++++++++++++ tests/test_manifest.py | 118 ++++++++++++++++++++++++++++++++++++-- 6 files changed, 313 insertions(+), 22 deletions(-) create mode 100644 tests/test_jinjaturtle.py diff --git a/enroll/harvest.py b/enroll/harvest.py index 78f7d1f..688a489 100644 --- a/enroll/harvest.py +++ b/enroll/harvest.py @@ -132,7 +132,7 @@ def _safe_name(s: str) -> str: def _role_id(raw: str) -> str: - # normalize separators first + # normalise separators first s = re.sub(r"[^A-Za-z0-9]+", "_", raw) # split CamelCase -> snake_case s = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s) diff --git a/enroll/manifest.py b/enroll/manifest.py index e27cfd5..7565160 100644 --- a/enroll/manifest.py +++ b/enroll/manifest.py @@ -48,8 +48,24 @@ def _yaml_dump_mapping(obj: Dict[str, Any], *, sort_keys: bool = True) -> str: for k, v in sorted(obj.items()) if sort_keys else obj.items(): lines.append(f"{k}: {v!r}") return "\n".join(lines).rstrip() + "\n" + + # ansible-lint/yamllint's indentation rules are stricter than YAML itself. + # In particular, they expect sequences nested under a mapping key to be + # indented (e.g. `foo:\n - a`), whereas PyYAML's default is often + # `foo:\n- a`. + class _IndentDumper(yaml.SafeDumper): # type: ignore + def increase_indent(self, flow: bool = False, indentless: bool = False): + return super().increase_indent(flow, False) + return ( - yaml.safe_dump(obj, default_flow_style=False, sort_keys=sort_keys).rstrip() + yaml.dump( + obj, + Dumper=_IndentDumper, + default_flow_style=False, + sort_keys=sort_keys, + indent=2, + allow_unicode=True, + ).rstrip() + "\n" ) @@ -124,7 +140,7 @@ def _extract_jinjaturtle_block(text: str) -> str: return text.strip() + "\n" -def _normalize_jinjaturtle_vars_text(vars_text: str) -> str: +def _normalise_jinjaturtle_vars_text(vars_text: str) -> str: """Deduplicate keys in a vars fragment by parsing as YAML and dumping it back.""" m = _yaml_load_mapping(vars_text) if not m: @@ -166,14 +182,14 @@ def _copy_artifacts( dst = os.path.join(dst_files_dir, rel) # If a file was successfully templatised by JinjaTurtle, do NOT - # also materialize the raw copy in the destination files dir. + # also materialise the raw copy in the destination files dir. # (This keeps the output minimal and avoids redundant "raw" files.) if exclude_rels and rel in exclude_rels: try: if os.path.isfile(dst): os.remove(dst) except Exception: - pass + pass # nosec continue if preserve_existing and os.path.exists(dst): @@ -342,7 +358,7 @@ def _jinjify_managed_files( except Exception: # If jinjaturtle cannot process a file for any reason, skip silently. # (Enroll's core promise is to be optimistic and non-interactive.) - continue + continue # nosec tmpl_rel = src_rel + ".j2" tmpl_dst = os.path.join(role_dir, "templates", tmpl_rel) @@ -372,7 +388,7 @@ def _hostvars_only_jinjaturtle(vars_text: str) -> str: def _defaults_with_jinjaturtle(base_defaults: str, vars_text: str) -> str: if not vars_text.strip(): return base_defaults.rstrip() + "\n" - vars_text = _normalize_jinjaturtle_vars_text(vars_text) + vars_text = _normalise_jinjaturtle_vars_text(vars_text) # Always regenerate the block (we regenerate whole defaults files anyway) return ( base_defaults.rstrip() @@ -450,7 +466,11 @@ def _render_generic_files_tasks( owner: "{{{{ item.owner }}}}" group: "{{{{ item.group }}}}" mode: "{{{{ item.mode }}}}" - loop: "{{{{ {var_prefix}_managed_files | default([]) | selectattr('is_systemd_unit','equalto', true) | selectattr('kind','equalto','template') | list }}}}" + loop: >- + {{{{ {var_prefix}_managed_files | default([]) + | selectattr('is_systemd_unit', 'equalto', true) + | selectattr('kind', 'equalto', 'template') + | list }}}} notify: "{{{{ item.notify | default([]) }}}}" - name: Deploy systemd unit files (copies) @@ -465,12 +485,20 @@ def _render_generic_files_tasks( owner: "{{{{ item.owner }}}}" group: "{{{{ item.group }}}}" mode: "{{{{ item.mode }}}}" - loop: "{{{{ {var_prefix}_managed_files | default([]) | selectattr('is_systemd_unit','equalto', true) | selectattr('kind','equalto','copy') | list }}}}" + loop: >- + {{{{ {var_prefix}_managed_files | default([]) + | selectattr('is_systemd_unit', 'equalto', true) + | selectattr('kind', 'equalto', 'copy') + | list }}}} notify: "{{{{ item.notify | default([]) }}}}" - name: Reload systemd to pick up unit changes ansible.builtin.meta: flush_handlers - when: "({var_prefix}_managed_files | default([]) | selectattr('is_systemd_unit','equalto', true) | list | length) > 0" + when: >- + ({var_prefix}_managed_files | default([]) + | selectattr('is_systemd_unit', 'equalto', true) + | list + | length) > 0 - name: Deploy other managed files (templates) ansible.builtin.template: @@ -479,7 +507,11 @@ def _render_generic_files_tasks( owner: "{{{{ item.owner }}}}" group: "{{{{ item.group }}}}" mode: "{{{{ item.mode }}}}" - loop: "{{{{ {var_prefix}_managed_files | default([]) | selectattr('is_systemd_unit','equalto', false) | selectattr('kind','equalto','template') | list }}}}" + loop: >- + {{{{ {var_prefix}_managed_files | default([]) + | selectattr('is_systemd_unit', 'equalto', false) + | selectattr('kind', 'equalto', 'template') + | list }}}} notify: "{{{{ item.notify | default([]) }}}}" - name: Deploy other managed files (copies) @@ -494,7 +526,11 @@ def _render_generic_files_tasks( owner: "{{{{ item.owner }}}}" group: "{{{{ item.group }}}}" mode: "{{{{ item.mode }}}}" - loop: "{{{{ {var_prefix}_managed_files | default([]) | selectattr('is_systemd_unit','equalto', false) | selectattr('kind','equalto','copy') | list }}}}" + loop: >- + {{{{ {var_prefix}_managed_files | default([]) + | selectattr('is_systemd_unit', 'equalto', false) + | selectattr('kind', 'equalto', 'copy') + | list }}}} notify: "{{{{ item.notify | default([]) }}}}" """ diff --git a/tests.sh b/tests.sh index f8d246c..ea7ad59 100755 --- a/tests.sh +++ b/tests.sh @@ -11,7 +11,7 @@ rm -rf "${BUNDLE_DIR}" "${ANSIBLE_DIR}" # Generate data poetry run \ - enroll enroll \ + enroll single-shot \ --harvest "${BUNDLE_DIR}" \ --out "${ANSIBLE_DIR}" diff --git a/tests/test_cli.py b/tests/test_cli.py index a93c509..9e3422c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -22,9 +22,12 @@ def test_cli_harvest_subcommand_calls_harvest(monkeypatch, capsys, tmp_path): def test_cli_manifest_subcommand_calls_manifest(monkeypatch, tmp_path): called = {} - def fake_manifest(harvest_dir: str, out_dir: str): + def fake_manifest(harvest_dir: str, out_dir: str, **kwargs): called["harvest"] = harvest_dir called["out"] = out_dir + # Common manifest args should be passed through by the CLI. + called["fqdn"] = kwargs.get("fqdn") + called["jinjaturtle"] = kwargs.get("jinjaturtle") monkeypatch.setattr(cli, "manifest", fake_manifest) monkeypatch.setattr( @@ -43,6 +46,8 @@ def test_cli_manifest_subcommand_calls_manifest(monkeypatch, tmp_path): cli.main() assert called["harvest"] == str(tmp_path / "bundle") assert called["out"] == str(tmp_path / "ansible") + assert called["fqdn"] is None + assert called["jinjaturtle"] == "auto" def test_cli_enroll_subcommand_runs_harvest_then_manifest(monkeypatch, tmp_path): @@ -52,8 +57,16 @@ def test_cli_enroll_subcommand_runs_harvest_then_manifest(monkeypatch, tmp_path) calls.append(("harvest", bundle_dir)) return str(tmp_path / "bundle" / "state.json") - def fake_manifest(bundle_dir: str, out_dir: str): - calls.append(("manifest", bundle_dir, out_dir)) + def fake_manifest(bundle_dir: str, out_dir: str, **kwargs): + calls.append( + ( + "manifest", + bundle_dir, + out_dir, + kwargs.get("fqdn"), + kwargs.get("jinjaturtle"), + ) + ) monkeypatch.setattr(cli, "harvest", fake_harvest) monkeypatch.setattr(cli, "manifest", fake_manifest) @@ -62,7 +75,7 @@ def test_cli_enroll_subcommand_runs_harvest_then_manifest(monkeypatch, tmp_path) "argv", [ "enroll", - "enroll", + "single-shot", "--harvest", str(tmp_path / "bundle"), "--out", @@ -73,5 +86,38 @@ def test_cli_enroll_subcommand_runs_harvest_then_manifest(monkeypatch, tmp_path) cli.main() assert calls == [ ("harvest", str(tmp_path / "bundle")), - ("manifest", str(tmp_path / "bundle"), str(tmp_path / "ansible")), + ("manifest", str(tmp_path / "bundle"), str(tmp_path / "ansible"), None, "auto"), ] + + +def test_cli_manifest_common_args(monkeypatch, tmp_path): + """Ensure --fqdn and jinjaturtle mode flags are forwarded correctly.""" + + called = {} + + def fake_manifest(harvest_dir: str, out_dir: str, **kwargs): + called["harvest"] = harvest_dir + called["out"] = out_dir + called["fqdn"] = kwargs.get("fqdn") + called["jinjaturtle"] = kwargs.get("jinjaturtle") + + monkeypatch.setattr(cli, "manifest", fake_manifest) + monkeypatch.setattr( + sys, + "argv", + [ + "enroll", + "manifest", + "--harvest", + str(tmp_path / "bundle"), + "--out", + str(tmp_path / "ansible"), + "--fqdn", + "example.test", + "--no-jinjaturtle", + ], + ) + + cli.main() + assert called["fqdn"] == "example.test" + assert called["jinjaturtle"] == "off" diff --git a/tests/test_jinjaturtle.py b/tests/test_jinjaturtle.py new file mode 100644 index 0000000..68bb04c --- /dev/null +++ b/tests/test_jinjaturtle.py @@ -0,0 +1,99 @@ +import json +from pathlib import Path + +import enroll.manifest as manifest_mod +from enroll.jinjaturtle import JinjifyResult + + +def test_manifest_uses_jinjaturtle_templates_and_does_not_copy_raw( + monkeypatch, tmp_path: Path +): + """If jinjaturtle can templatisize a file, we should store a template in the role + and avoid keeping the raw file copy in the destination files area. + + This test stubs out jinjaturtle execution so it doesn't depend on the external tool. + """ + + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + + # A jinjaturtle-compatible config file. + (bundle / "artifacts" / "foo" / "etc").mkdir(parents=True, exist_ok=True) + (bundle / "artifacts" / "foo" / "etc" / "foo.ini").write_text( + "[main]\nkey = 1\n", encoding="utf-8" + ) + + state = { + "host": {"hostname": "test", "os": "debian"}, + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [ + { + "unit": "foo.service", + "role_name": "foo", + "packages": ["foo"], + "active_state": "inactive", + "sub_state": "dead", + "unit_file_state": "disabled", + "condition_result": "no", + "managed_files": [ + { + "path": "/etc/foo.ini", + "src_rel": "etc/foo.ini", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "modified_conffile", + } + ], + "excluded": [], + "notes": [], + } + ], + "package_roles": [], + } + + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + # Pretend jinjaturtle exists. + monkeypatch.setattr( + manifest_mod, "find_jinjaturtle_cmd", lambda: "/usr/bin/jinjaturtle" + ) + + # Stub jinjaturtle output. + def fake_run_jinjaturtle( + jt_exe: str, src_path: str, *, role_name: str, force_format=None + ): + assert role_name == "foo" + return JinjifyResult( + template_text="[main]\nkey = {{ foo_key }}\n", + vars_text="foo_key: 1\n", + ) + + monkeypatch.setattr(manifest_mod, "run_jinjaturtle", fake_run_jinjaturtle) + + manifest_mod.manifest(str(bundle), str(out), jinjaturtle="on") + + # Template should exist in the role. + assert (out / "roles" / "foo" / "templates" / "etc" / "foo.ini.j2").exists() + + # Raw file should NOT be copied into role files/ because it was templatised. + assert not (out / "roles" / "foo" / "files" / "etc" / "foo.ini").exists() + + # Defaults should include jinjaturtle vars. + defaults = (out / "roles" / "foo" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + assert "foo_key: 1" in defaults diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 09c66e1..98f418f 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -94,10 +94,16 @@ def test_manifest_writes_roles_and_playbook_with_clean_when(tmp_path: Path): manifest(str(bundle), str(out)) - # Service role: conditional start must be a clean Ansible expression + # Service role: systemd management should be gated on foo_manage_unit and a probe. tasks = (out / "roles" / "foo" / "tasks" / "main.yml").read_text(encoding="utf-8") - assert "when:\n - _unit_probe is succeeded\n - foo_start | bool\n" in tasks - # Ensure we didn't emit deprecated/broken '{{ }}' delimiters in when: + assert "- name: Probe whether systemd unit exists and is manageable" in tasks + assert "when: foo_manage_unit | default(false)" in tasks + assert ( + "when:\n - foo_manage_unit | default(false)\n - _unit_probe is succeeded\n" + in tasks + ) + + # Ensure we didn't emit deprecated/broken '{{ }}' delimiters in when: lines. for line in tasks.splitlines(): if line.lstrip().startswith("when:"): assert "{{" not in line and "}}" not in line @@ -105,7 +111,9 @@ def test_manifest_writes_roles_and_playbook_with_clean_when(tmp_path: Path): defaults = (out / "roles" / "foo" / "defaults" / "main.yml").read_text( encoding="utf-8" ) - assert "foo_start: false" in defaults + assert "foo_manage_unit: true" in defaults + assert "foo_systemd_enabled: true" in defaults + assert "foo_systemd_state: stopped" in defaults # Playbook should include users, etc_custom, packages, and services pb = (out / "playbook.yml").read_text(encoding="utf-8") @@ -113,3 +121,105 @@ def test_manifest_writes_roles_and_playbook_with_clean_when(tmp_path: Path): assert "- etc_custom" in pb assert "- curl" in pb assert "- foo" in pb + + +def test_manifest_site_mode_creates_host_inventory_and_raw_files(tmp_path: Path): + """In --fqdn mode, host-specific state goes into inventory/host_vars.""" + + fqdn = "host1.example.test" + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + + # Artifacts for a service-managed file. + (bundle / "artifacts" / "foo" / "etc").mkdir(parents=True, exist_ok=True) + (bundle / "artifacts" / "foo" / "etc" / "foo.conf").write_text( + "x", encoding="utf-8" + ) + + # Artifacts for etc_custom file so copy works. + (bundle / "artifacts" / "etc_custom" / "etc" / "default").mkdir( + parents=True, exist_ok=True + ) + (bundle / "artifacts" / "etc_custom" / "etc" / "default" / "keyboard").write_text( + "kbd", encoding="utf-8" + ) + + state = { + "host": {"hostname": "test", "os": "debian"}, + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [ + { + "path": "/etc/default/keyboard", + "src_rel": "etc/default/keyboard", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "custom_unowned", + } + ], + "excluded": [], + "notes": [], + }, + "services": [ + { + "unit": "foo.service", + "role_name": "foo", + "packages": ["foo"], + "active_state": "active", + "sub_state": "running", + "unit_file_state": "enabled", + "condition_result": "yes", + "managed_files": [ + { + "path": "/etc/foo.conf", + "src_rel": "etc/foo.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "modified_conffile", + } + ], + "excluded": [], + "notes": [], + } + ], + "package_roles": [], + } + + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + manifest(str(bundle), str(out), fqdn=fqdn) + + # Host playbook exists. + assert (out / "playbooks" / f"{fqdn}.yml").exists() + + # Role defaults are safe/host-agnostic in site mode. + foo_defaults = (out / "roles" / "foo" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + assert "foo_packages: []" in foo_defaults + assert "foo_managed_files: []" in foo_defaults + assert "foo_manage_unit: false" in foo_defaults + + # Host vars contain host-specific state. + foo_hostvars = (out / "inventory" / "host_vars" / fqdn / "foo.yml").read_text( + encoding="utf-8" + ) + assert "foo_packages" in foo_hostvars + assert "foo_managed_files" in foo_hostvars + assert "foo_manage_unit: true" in foo_hostvars + assert "foo_systemd_state: started" in foo_hostvars + + # Non-templated raw config is stored per-host under .files. + assert ( + out / "inventory" / "host_vars" / fqdn / "foo" / ".files" / "etc" / "foo.conf" + ).exists()