diff --git a/.forgejo/workflows/build-deb.yml b/.forgejo/workflows/build-deb.yml new file mode 100644 index 0000000..047d2bc --- /dev/null +++ b/.forgejo/workflows/build-deb.yml @@ -0,0 +1,66 @@ +name: CI + +on: + push: + +jobs: + test: + runs-on: docker + + steps: + - name: Install system dependencies + run: | + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + devscripts \ + debhelper \ + dh-python \ + pybuild-plugin-pyproject \ + python3-all \ + python3-poetry-core \ + python3-yaml \ + python3-paramiko \ + python3-jsonschema \ + rsync \ + ca-certificates + + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Build deb + run: | + mkdir /out + + rsync -a --delete \ + --exclude '.git' \ + --exclude '.venv' \ + --exclude 'dist' \ + --exclude 'build' \ + --exclude '__pycache__' \ + --exclude '.pytest_cache' \ + --exclude '.mypy_cache' \ + ./ /out/ + + cd /out/ + export DEBEMAIL="mig@mig5.net" + export DEBFULLNAME="Miguel Jacq" + + dch --distribution "trixie" --local "~trixie" "CI build for trixie" + dpkg-buildpackage -us -uc -b + + # Notify if any previous step in this job failed + - name: Notify on failure + if: ${{ failure() }} + env: + WEBHOOK_URL: ${{ secrets.NODERED_WEBHOOK_URL }} + REPOSITORY: ${{ forgejo.repository }} + RUN_NUMBER: ${{ forgejo.run_number }} + SERVER_URL: ${{ forgejo.server_url }} + run: | + curl -X POST \ + -H "Content-Type: application/json" \ + -d "{\"repository\":\"$REPOSITORY\",\"run_number\":\"$RUN_NUMBER\",\"status\":\"failure\",\"url\":\"$SERVER_URL/$REPOSITORY/actions/runs/$RUN_NUMBER\"}" \ + "$WEBHOOK_URL" diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 41efa55..796d087 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -15,7 +15,8 @@ jobs: run: | apt-get update DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - ansible ansible-lint python3-venv pipx systemctl python3-apt + ansible ansible-lint python3-venv pipx systemctl python3-apt jq python3-jsonschema \ + puppet hiera - name: Install Poetry run: | @@ -27,6 +28,11 @@ jobs: run: | poetry install --with dev + - name: Install sops + run: | + curl -L -o /usr/local/bin/sops https://github.com/getsops/sops/releases/download/v3.13.1/sops-v3.13.1.linux.amd64 + chmod +x /usr/local/bin/sops + - name: Run test script run: | ./tests.sh diff --git a/.forgejo/workflows/trivy.yml b/.forgejo/workflows/trivy.yml deleted file mode 100644 index fad2f6f..0000000 --- a/.forgejo/workflows/trivy.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Trivy - -on: - schedule: - - cron: '0 1 * * *' - push: - -jobs: - test: - runs-on: docker - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install system dependencies - run: | - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends wget gnupg - wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | gpg --dearmor | tee /usr/share/keyrings/trivy.gpg > /dev/null - echo "deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb generic main" | tee -a /etc/apt/sources.list.d/trivy.list - apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends trivy - - - name: Run trivy - run: | - trivy fs --no-progress --ignore-unfixed --format table --disable-telemetry . - - # Notify if any previous step in this job failed - - name: Notify on failure - if: ${{ failure() }} - env: - WEBHOOK_URL: ${{ secrets.NODERED_WEBHOOK_URL }} - REPOSITORY: ${{ forgejo.repository }} - RUN_NUMBER: ${{ forgejo.run_number }} - SERVER_URL: ${{ forgejo.server_url }} - run: | - curl -X POST \ - -H "Content-Type: application/json" \ - -d "{\"repository\":\"$REPOSITORY\",\"run_number\":\"$RUN_NUMBER\",\"status\":\"failure\",\"url\":\"$SERVER_URL/$REPOSITORY/actions/runs/$RUN_NUMBER\"}" \ - "$WEBHOOK_URL" diff --git a/.gitignore b/.gitignore index 07c956d..4ef962d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ dist *.pdf *.csv *.html +coverage.xml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..09c6889 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +repos: + - repo: https://github.com/pycqa/flake8 + rev: 7.3.0 + hooks: + - id: flake8 + args: ["--select=F"] + types: [python] + + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 25.11.0 + hooks: + - id: black + language_version: python3 + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + + - repo: https://github.com/PyCQA/bandit + rev: 1.9.2 + hooks: + - id: bandit + files: ^enroll/ diff --git a/CHANGELOG.md b/CHANGELOG.md index d30a65a..3f0d98e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,145 @@ +# 0.7.0 + + * Add support for detecting flatpaks and snaps + * BREAKING CHANGE: Group all package and systemd-unit roles into Debian Section/RPM Group roles by default, including managed config files and unit state. This mode is not used if `--fqdn` or `--no-common-roles` is set, in which case, the traditional behaviour of preserving one role per package/unit is used instead. + * BREAKING CHANGE: Only capture user-specific .bashrc style files when using `--dangerous` mode, in case they contain sensitive env vars. + * Detect active sysctl parameters and write them to a `/etc/sysctl.d/99-enroll.conf` file + * Use `no_log` on systemd unit interrogations to suppress potential sensitive output when applying Ansible + * Support manifesting Puppet code, as well as Ansible! + * A lot of under-the-bonnet refactoring to make it easier to extend to cover other config managers (that don't suck) in future. + * Support for detecting Docker images + +# 0.6.0 + + * Add support for capturing ipset and iptables configuration files + * Add support for generating ipset and iptables configuration files from runtime, if the former weren't present (`firewall_runtime` role) + * Dependency updates + +# 0.5.0 + + * Add support for templating `sshd_config`, if a compatible version of JinjaTurtle is also present. + * Dependency updates + +# 0.4.4 + + * Update cryptography dependency + * Add capability to handle passphrases on encrypted SSH private keys. Prompting can be forced with `--ask-key-passphrase` or automated (e.g for CI) with `--ssh-key-passphrase env SOMEVAR` + +# 0.4.3 + + * Add support for AddressFamily and ConnectTimeout in the .ssh/config when using `--remote-ssh-config`. + * Update dependencies + +# 0.4.2 + + * Support `--remote-ssh-config [path-to-ssh-config]` as an argument in case extra params are required beyond `--remote-port` or `--remote-user`. Note: `--remote-host` must still be set, but it can be an 'alias' represented by the 'Host' value in the ssh config. + +# 0.4.1 + + * Add interactive output when 'enroll diff --enforce' is invoking Ansible. + +# 0.4.0 + + * Introduce `enroll validate` - a tool to validate a harvest against the state schema, or check for missing or orphaned obsolete artifacts in a harvest. + * Attempt to generate Jinja2 templates of systemd unit files and Postfix main.cf (now that JinjaTurtle supports it) + * Update pynacl dependency to resolve CVE-2025-69277 + * Add `--exclude-path` to `enroll diff` command, so that you can ignore certain churn from the diff (stuff you still wanted to harvest as a baseline but don't care if it changes day to day) + * Add `--ignore-package-versions` to `enroll diff` command, to optionally ignore package upgrades (e.g due to patching) from the diff. + * Add tags to the playbook for each role, to allow easier targeting of specific roles during play later. + * Add `--enforce` mode to `enroll diff`. If there is diff detected between the two harvests, and it can enforce restoring the state from the older harvest, it will manifest the state and apply it with ansible. Only the specific roles that had diffed will be applied (via the new tags capability) + +# 0.3.0 + + * Introduce `enroll explain` - a tool to analyze and explain what's in (or not in) a harvest and why. + * Centralise the cron and logrotate stuff into their respective roles, we had a bit of duplication between roles based on harvest discovery. + * Capture other files in the user's home directory such as `.bashrc`, `.bash_aliases`, `.profile`, if these files differ from the `/etc/skel` defaults + * Ignore files that end with a tilde or - (probably backup files generated by editors or shadow file changes) + * Manage certain symlinks e.g for apache2/nginx sites-enabled and so on + +# 0.2.3 + + * Introduce --ask-become-pass or -K to support password-required sudo on remote hosts, just like Ansible. It will also fall back to this prompt if a password is required but the arg wasn't passed in. + +# 0.2.2 + + * Fix stat() of parent directory so that we set directory perms correct on --include paths. + * Set pty for remote calls when sudo is required, to help systems with limits on sudo without pty + +# 0.2.1 + + * Don't accidentally add `extra_paths` role to `usr_local_custom` list, resulting in `extra_paths` appearing twice in manifested playbook + * Ensure directories in the tree of anything included with --include are defined in the state and manifest so we make dirs before we try to create files + +# 0.2.0 + + * Add version CLI arg + * Add ability to enroll RH-style systems (DNF5/DNF/RPM) + * Refactor harvest state to track package versions + +# 0.1.7 + + * Fix an attribution bug for certain files ending up in the wrong package/role. + +# 0.1.6 + + * DRY up some code logic + * More test coverage + +# 0.1.5 + + * Consolidate logrotate and cron files into their main service/package roles if they exist. + * Standardise on `MAX_FILES_CAP` in one place + * Manage apt stuff in its own role, not in `etc_custom` + +# 0.1.4 + + * Attempt to capture more stuff from /etc that might not be attributable to a specific package. This includes common singletons and systemd timers + * Avoid duplicate apt data in package-specific roles. + +# 0.1.3 + + * Allow the user to add extra paths to harvest, or paths to ignore, using `--exclude-path` and `--include-path` + arguments. + * Add support for an enroll.ini config file to store arguments per subcommand, to avoid having to remember + them all for repetitive executions. + +# 0.1.2 + + * Include files from `/usr/local/bin` and `/usr/local/etc` in harvest (assuming they aren't binaries or + symlinks) and store in `usr_local_custom` role, similar to `etc_custom`. + +# 0.1.1 + + * Add `diff` subcommand which can compare two harvests and send email or webhook notifications in different + formats. + +# 0.1.0 + + * Add remote mode for harvesting a remote machine via a local workstation (no need to install enroll remotely) + Optionally use `--no-sudo` if you don't want the remote user to have passwordless sudo when conducting the + harvest, albeit you'll end up with less useful data (same as if running `enroll harvest` on a machine without + sudo) + * Add `--dangerous` flag to capture even sensitive data (use at your own risk!) + * Add `--sops` flag which makes the harvest and the manifest 'out' data encrypted as a single SOPS data file. + This would make `--dangerous` a little bit safer, if your intention is just to store the Ansible manifest + in git or somewhere similar for disaster-recovery purposes (e.g encrypted at rest for safe-keeping). + * Do a better job at capturing other config files in `/etc//` even if that package doesn't normally + ship or manage those files. + * Don't collect files ending in `.log` + +# 0.0.5 + + * Use JinjaTurtle to generate dynamic template/inventory if it's on the PATH + * Support --fqdn flag for site-specific inventory and an inventory hosts file. + This radically re-architects the roles to loop through abstract inventory + because otherwise different servers can collide with each other through use + of the same role. Use 'single site' mode (no `--fqdn`) if you want more readable, + self-contained roles (in which case, store each manifested output in its own + repo per server) + * Generate an ansible.cfg if not present, to support `host_vars` plugin and other params, + when using `--fqdn` mode + * Be more permissive with files that we previously thought contained secrets (ignore commented lines) + # 0.0.4 * Fix dash package detection issue @@ -11,10 +153,10 @@ # 0.0.2 * Merge pkg_ and roles created based on file/service detection - * Avoid idempotency issue with users (password_lock) - * Rename subcommands/args ('export' is now 'enroll', '--bundle' is now '--harvest') + * Avoid idempotency issue with users (`password_lock`) + * Rename subcommands/args ('export' is now 'enroll', '--bundle' is now '--harvest') * Don't try and start systemd services that were Inactive at harvest time - * Capture miscellaneous files in /etc under their own etc_custom role, but not backup files + * Capture miscellaneous files in /etc under their own `etc_custom` role, but not backup files * Add tests * Various other bug fixes diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md new file mode 100644 index 0000000..a411acc --- /dev/null +++ b/CONTRIBUTORS.md @@ -0,0 +1,5 @@ +## Contributors + +mig5 would like to thank the following people for their contributions to Enroll. + + * [slhck](https://slhck.info/) diff --git a/Dockerfile.debbuild b/Dockerfile.debbuild index 873018e..c6ebedb 100644 --- a/Dockerfile.debbuild +++ b/Dockerfile.debbuild @@ -24,6 +24,9 @@ RUN set -eux; \ pybuild-plugin-pyproject \ python3-all \ python3-poetry-core \ + python3-yaml \ + python3-paramiko \ + python3-jsonschema \ rsync \ ca-certificates \ ; \ diff --git a/Dockerfile.rpmbuild b/Dockerfile.rpmbuild new file mode 100644 index 0000000..dd83546 --- /dev/null +++ b/Dockerfile.rpmbuild @@ -0,0 +1,88 @@ +# syntax=docker/dockerfile:1 +ARG BASE_IMAGE=fedora:42 +FROM ${BASE_IMAGE} + +RUN set -eux; \ + dnf -y update; \ + dnf -y install \ + rpm-build \ + rpmdevtools \ + redhat-rpm-config \ + gcc \ + make \ + findutils \ + tar \ + gzip \ + rsync \ + python3 \ + python3-devel \ + python3-setuptools \ + python3-wheel \ + pyproject-rpm-macros \ + python3-rpm-macros \ + python3-yaml \ + python3-paramiko \ + python3-jsonschema \ + openssl-devel \ + python3-poetry-core ; \ + dnf -y clean all + +# Build runner script (copies repo, tars, runs rpmbuild) +RUN set -eux; cat > /usr/local/bin/build-rpm <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +SRC="${SRC:-/src}" +WORKROOT="${WORKROOT:-/work}" +OUT="${OUT:-/out}" +VERSION_ID="$(grep VERSION_ID /etc/os-release | cut -d= -f2)" +echo "Version ID is ${VERSION_ID}" + +mkdir -p "${WORKROOT}" "${OUT}" +WORK="${WORKROOT}/src" +rm -rf "${WORK}" +mkdir -p "${WORK}" + +rsync -a --delete \ + --exclude '.git' \ + --exclude '.venv' \ + --exclude 'dist' \ + --exclude 'build' \ + --exclude '__pycache__' \ + --exclude '.pytest_cache' \ + --exclude '.mypy_cache' \ + "${SRC}/" "${WORK}/" + +cd "${WORK}" + +# Determine version from pyproject.toml unless provided +if [ -n "${VERSION:-}" ]; then + ver="${VERSION}" +else + ver="$(grep -m1 '^version = ' pyproject.toml | sed -E 's/version = "([^"]+)".*/\1/')" +fi + +TOPDIR="${WORKROOT}/rpmbuild" +mkdir -p "${TOPDIR}"/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} + +tarball="${TOPDIR}/SOURCES/enroll-${ver}.tar.gz" +tar -czf "${tarball}" --transform "s#^#enroll/#" . + +spec_src="rpm/enroll.spec" + +cp -v "${spec_src}" "${TOPDIR}/SPECS/enroll.spec" + +rpmbuild -ba "${TOPDIR}/SPECS/enroll.spec" \ + --define "_topdir ${TOPDIR}" \ + --define "upstream_version ${ver}" + +shopt -s nullglob +cp -v "${TOPDIR}"/RPMS/*/*.rpm "${OUT}/" || true +cp -v "${TOPDIR}"/SRPMS/*.src.rpm "${OUT}/" || true +echo "Artifacts copied to ${OUT}" +EOF + +RUN chmod +x /usr/local/bin/build-rpm + +WORKDIR /work +ENTRYPOINT ["/usr/local/bin/build-rpm"] diff --git a/README.md b/README.md index 33fe901..dfa693b 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,318 @@ -# Enroll +# Enroll
Enroll logo
-**enroll** inspects a Linux machine (currently Debian-only) and generates Ansible roles for things it finds running on the machine. +**enroll** inspects a Linux machine (Debian-like or RedHat-like) and generates configuration-management code: Ansible roles/playbooks by default, or Puppet control-repo style output for what it finds. -It aims to be **optimistic and noninteractive**: -- Detects packages that have been installed -- Detects Debian package ownership of `/etc` files using dpkg’s local database. -- Captures config that has **changed from packaged defaults** (dpkg conffile hashes + package md5sums when available). +- Detects packages that have been installed. +- Detects package ownership of `/etc` files where possible +- Captures config that has **changed from packaged defaults** where possible (e.g dpkg conffile hashes + package md5sums when available). - Also captures **service-relevant custom/unowned files** under `/etc//...` (e.g. drop-in config includes). - Defensively excludes likely secrets (path denylist + content sniff + size caps). -- Captures non-system users that exist on the system, and their SSH public keys -- Captures miscellaneous `/etc` files that it can't attribute to a package, and installs it in an `etc_custom` role -- Avoids trying to start systemd services that were detected as being Inactive during harvest +- Captures non-system users and their SSH public keys. In `--dangerous` mode, it also auto-harvests common shell dotfiles such as `.bashrc`, `.profile`, `.bash_logout`, and `.bash_aliases` when appropriate. +- Captures miscellaneous `/etc` files it can't attribute to a package and installs them in an `etc_custom` role. +- When running as root/sudo, captures live writable sysctl state into a `sysctl` role that manages `/etc/sysctl.d/99-enroll.conf`. +- Captures live ipset and iptables runtime state into a fallback `firewall_runtime` role, when active ipsets/iptables rules are present *and* no corresponding persistent ipset/iptables *files* were found. +- Captures symlinks in common applications that rely on them, e.g apache2/nginx 'sites-enabled' +- Ditto for /usr/local/bin (for non-binary files) and /usr/local/etc +- Avoids trying to start systemd services that were detected as inactive during harvest. -## Install +--- -### Ubuntu/Debian apt repository +## Mental model +`enroll` works in two phases: + +1) **Harvest**: collect host facts + relevant files into a harvest bundle (`state.json` + harvested artifacts) +2) **Manifest**: turn that harvest into configuration-management code such as Ansible roles/playbooks or Puppet manifests + +Additionally, some other functionalities exist: + +- **Diff**: compare two harvests and report what changed (packages/services/users/files) since the previous snapshot. +- **Single-shot mode**: run both harvest and manifest at once. + +--- + +## Output modes: single-site vs multi-site (`--fqdn`) + +`enroll manifest` (and `enroll single-shot`) support multiple output targets. Ansible is the default target and supports two distinct output styles. + +### Single-site mode (default: *no* `--fqdn`) +Use when enrolling **one server** (or generating a “golden” role set you intend to reuse). + +**Characteristics** +- Roles are more self-contained. +- Raw config files live in the role's `files/`. +- Template variables live in the role's `defaults/main.yml`. + +### Multi-site mode (`--fqdn`) +Use when enrolling **several existing servers** quickly, especially if they differ. + +**Characteristics** +- Roles are shared, host-specific state lives in inventory. +- Host inventory drives what gets managed (files/packages/services). +- Non-templated raw files live per-host under `inventory/host_vars///.files/...`. + +**Rule of thumb** +- “Make this one server reproducible/provisionable” → start with **single-site** +- “Get multiple already-running servers under management quickly” → use **multi-site** + +--- + +## Subcommands + +### `enroll harvest` +Harvest state about a host and write a harvest bundle. + +**What it captures (high level)** +- Detected services + service-relevant packages +- “Manual” packages +- Changed-from-default config (plus related custom/unowned files under service dirs) +- Non-system users + SSH public keys +- In `--dangerous` mode: common per-user shell dotfiles that are likely to represent deliberate account customisation +- Misc `/etc` that can't be attributed to a package (`etc_custom` role) +- Static firewall config files such as nftables, UFW, firewalld, `/etc/iptables/rules.v4`, `/etc/iptables/rules.v6`, and `/etc/ipset*` +- Live writable sysctl state via `sysctl -a`, emitted as `/etc/sysctl.d/99-enroll.conf` at manifest time when running as root/sudo (`sysctl` role) +- Live kernel ipset/iptables state via `ipset save`, `iptables-save`, and `ip6tables-save` as a fallback, but only when the corresponding persistent config was not found (`firewall_runtime` role at manifest time) +- Optional user-specified extra files/dirs via `--include-path` (emitted as an `extra_paths` role at manifest time) + +**Common flags** +- Remote harvesting: + - `--remote-host`, `--remote-user`, `--remote-port`, `--remote-ssh-config` + - `--no-sudo` (if you don't want/need sudo) +- Sensitive-data behaviour: + - default: tries to avoid likely secrets + - `--dangerous`: disables secret-safety checks (see “Sensitive data” below) +- Encrypt bundles at rest: + - `--sops `: writes a single encrypted `harvest.tar.gz.sops` instead of a plaintext directory +- Path selection (include/exclude): + - `--include-path ` (repeatable): add extra files/dirs to harvest (even from locations normally ignored, like `/home`). Still subject to secret-safety checks unless `--dangerous`. + - `--exclude-path ` (repeatable): skip files/dirs even if they would normally be harvested. + - Pattern syntax: + - plain path: matches that file; directories match the directory + everything under it + - glob (default): supports `*` and `**` (prefix with `glob:` to force) + - regex: prefix with `re:` or `regex:` + - Precedence: excludes win over includes. + * Using remote mode and auth requires secrets? + * sudo password: + * `--ask-become-pass` (or `-K`) prompts for the sudo password. + * If you forget, and remote sudo requires a password, Enroll will still fall back to prompting in interactive mode (slightly slower due to retry). + * SSH private-key passphrase: + * `--ask-key-passphrase` prompts for the SSH key passphrase. + * `--ssh-key-passphrase-env ENV_VAR` reads the SSH key passphrase from an environment variable (useful for CI/non-interactive runs). + * If neither is provided, and Enroll detects an encrypted key in an interactive session, it will still fall back to prompting on-demand. + * In non-interactive sessions, pass `--ask-key-passphrase` or `--ssh-key-passphrase-env ENV_VAR` when using encrypted private keys. + * Note: `--ask-key-passphrase` and `--ssh-key-passphrase-env` are mutually exclusive. + +Examples (encrypted SSH key) + +```bash +# Interactive +enroll harvest --remote-host myhost.example.com --remote-user myuser --ask-key-passphrase --out /tmp/enroll-harvest + +# Non-interactive / CI +export ENROLL_SSH_KEY_PASSPHRASE='correct horse battery staple' +enroll single-shot --remote-host myhost.example.com --remote-user myuser --ssh-key-passphrase-env ENROLL_SSH_KEY_PASSPHRASE --harvest /tmp/enroll-harvest --out /tmp/enroll-ansible --fqdn myhost.example.com +``` + +--- + +### `enroll manifest` +Generate configuration-management output from an existing harvest bundle. Ansible remains the default; use `--target puppet` for Puppet output. + +**Inputs** +- `--harvest /path/to/harvest` (directory) + or `--harvest /path/to/harvest.tar.gz.sops` (if using `--sops`) + +**Output** +- In plaintext Ansible mode: an Ansible repo-like directory structure (roles/playbooks, and inventory in multi-site mode). +- In plaintext Puppet mode: a Puppet control-repo style layout with `manifests/site.pp` and generated modules under `modules/`. By default, package and service resources are grouped by Debian Section/RPM Group where possible; `--fqdn` or `--no-common-roles` preserves one generated module per Enroll role/snapshot. +- In `--sops` mode: a single encrypted file `manifest.tar.gz.sops` containing the generated output. + +**Common flags** +- `--target ansible|puppet`: choose the manifest target (`ansible` is the default). +- `--fqdn `: enables **multi-site** output style for Ansible or emits Puppet Hiera/node output. Without `--fqdn`, Puppet emits `node default { ... }`. +- `--no-common-roles`: disables the default grouping of package and systemd-unit roles into Debian Section/RPM Group roles, preserving one generated role per package/unit. `--fqdn` implies this behaviour. + +**Role tags** +Generated playbooks tag each role so you can target just the parts you need: + +- Tag format: `role_` (e.g. `role_services`, `role_users`) +- Fallback/safe tag: `role_other` + +Example: +```bash +ansible-playbook -i "localhost," -c local /tmp/enroll-ansible/playbook.yml --tags role_services,role_users +``` + +--- + +### `enroll single-shot` +Convenience wrapper that runs **harvest → manifest** in one command. + +Use this when you want “get me something workable ASAP”. + +Supports the same general flags as harvest/manifest, including `--target`, `--fqdn`, `--no-common-roles`, remote harvest flags, and `--sops`. + +--- + +### `enroll diff` +Compare two harvest bundles and report what changed. + +**What it reports** +- Packages added/removed +- Services enabled added/removed, plus key state changes +- Users added/removed, plus field changes (uid/gid/home/shell/groups, etc.) +- Managed files added/removed/changed (metadata + content hash changes where available) + +**Inputs** +- `--old ` and `--new ` (directories or `state.json` paths) +- `--sops` when comparing SOPS-encrypted harvest bundles +- `--exclude-path ` (repeatable) to ignore file/dir drift under matching paths (same pattern syntax as harvest) +- `--ignore-package-versions` to ignore package version-only drift (upgrades/downgrades) +- `--enforce` to apply the **old** harvest state locally (requires `ansible-playbook` on `PATH`) + +**Noise suppression** +- `--exclude-path` is useful for things that change often but you still want in the harvest baseline (e.g. `/var/anacron`). +- `--ignore-package-versions` keeps routine upgrades from alerting; package add/remove drift is still reported. + +**Enforcement (`--enforce`)** +If a diff exists and `ansible-playbook` is available, Enroll will: +1) generate a manifest from the **old** harvest into a temporary directory +2) run `ansible-playbook -i localhost, -c local /playbook.yml` (often with `--tags role_<...>` to limit runtime) +3) record in the diff report that the old harvest was enforced + +Enforcement is intentionally “safe”: +- reinstalls packages that were removed (`state: present`), but does **not** attempt downgrades/pinning +- restores users, files (contents + permissions/ownership), and service enable/start state + +If `ansible-playbook` is not on `PATH`, Enroll returns an error and does not enforce. + + +**Output formats** +- `--format json` (default for webhooks) +- `--format markdown` / `--format text` (human-oriented) + +**Notifications** +- Webhook: + - `--webhook ` + - `--webhook-format json|markdown|text` + - `--webhook-header 'Header-Name: value'` (repeatable) +- Email (optional): + - `--email-to ` (plus optional SMTP/sendmail-related flags, depending on your install) + +--- + +### `enroll explain` +Analyze a harvest and provide user-friendly explanations for what's in it and why. + +This may also explain why something *wasn't* included (e.g a binary file, a file that was too large, unreadable due to permissions, or looked like a log file/secret. + +Provide either the path to the harvest or the path to its state.json. It can also handle SOPS-encrypted harvests. + +Output can be provided in plaintext or json. + +--- + +### `enroll validate` + +Validates a harvest by checking: + + * state.json exists and is valid JSON + * state.json validates against a JSON Schema (by default the vendored one) + * Every `managed_file` entry has a corresponding artifact at: `artifacts//` + * That there are no **unreferenced files** sitting in `artifacts/` that aren't in the state. + +#### Schema location + overrides + +The master schema lives at: `enroll/schema/state.schema.json`. + +You can override with a local file or URL: + +``` +enroll validate /path/to/harvest --schema ./state.schema.json +enroll validate /path/to/harvest --schema https://enroll.sh/schema/state.schema.json +``` + +Or skip schema checks (still does artifact consistency checks): + +``` +enroll validate /path/to/harvest --no-schema +``` + +#### CLI usage examples + +Validate a local harvest: + +``` +enroll validate ./harvest +``` + +Validate a harvest tarball or a sops bundle: + +``` +enroll validate ./harvest.tar.gz +enroll validate ./harvest.sops --sops +``` + +JSON output + write to file: + +``` +enroll validate ./harvest --format json --out validate.json +``` + +Return exit code 1 for any warnings, not just errors (useful for CI): + +``` +enroll validate ./harvest --fail-on-warnings +``` + +--- + +## Sensitive data + +By default, `enroll` does **not** assume how you handle secrets in Ansible. It will attempt to avoid harvesting likely sensitive data (private keys, passwords, tokens, etc.). This can mean it skips some config files you may ultimately want to manage. + +Automatic harvesting of per-user shell dotfiles is also disabled by default, even when those files differ from `/etc/skel`, because `.bashrc`, `.profile`, `.bash_aliases`, and similar files commonly contain exported tokens, credentials, or aliases/functions with embedded secrets. Use `--dangerous` for automatic shell-dotfile capture, or use targeted `--include-path` patterns for narrower safe-mode review. + +If you opt in to collecting everything: + +### `--dangerous` +**WARNING:** disables “likely secret” safety checks. This can copy private keys, TLS key material, API tokens, database passwords, and other credentials into the harvest output **in plaintext**. + +If you intend to keep harvests/manifests long-term (especially in git), strongly consider encrypting them at rest. + +### Encrypt bundles at rest with `--sops` +`--sops` encrypts the harvest and/or manifest outputs into a single `.tar.gz.sops` file (GPG). This is for **storage-at-rest**, not for direct “Ansible SOPS inventory” workflows. + +⚠️ Important: `manifest --sops` produces one encrypted file. You must decrypt + extract it before running `ansible-playbook`. + +--- + +## JinjaTurtle integration (both modes) + +If [JinjaTurtle](https://git.mig5.net/mig5/jinjaturtle) is installed, `enroll` can generate Jinja2 templates for ini/json/xml/toml-style config. + +- Templates live in `roles//templates/...` +- Variables live in: + - single-site: `roles//defaults/main.yml` + - multi-site: `inventory/host_vars//.yml` + +You can force it on with `--jinjaturtle` or disable with `--no-jinjaturtle`. + +--- + +## How multi-site avoids “shared role breaks a host” + +In multi-site mode, roles are **data-driven**. The role tasks are generic (“deploy the files listed for this host”, “install the packages listed for this host”, “apply systemd enable/start state listed for this host”). Host inventory decides what applies per-host, avoiding the classic “host2 adds config, host1 breaks” failure mode. + +--- + +# Install + +## Ubuntu/Debian apt repository ```bash sudo mkdir -p /usr/share/keyrings curl -fsSL https://mig5.net/static/mig5.asc | sudo gpg --dearmor -o /usr/share/keyrings/mig5.gpg @@ -28,74 +321,347 @@ sudo apt update sudo apt install enroll ``` -### AppImage +## Fedora -Download the AppImage file from the Releases page (verify with GPG if you wish, my fingerprint is [here](https://mig5.net/static/mig5.asc), -then make it executable and run it: +```bash +sudo rpm --import https://mig5.net/static/mig5.asc + +sudo tee /etc/yum.repos.d/mig5.repo > /dev/null << 'EOF' +[mig5] +name=mig5 Repository +baseurl=https://rpm.mig5.net/$releasever/rpm/$basearch +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey=https://mig5.net/static/mig5.asc +EOF + +sudo dnf upgrade --refresh +sudo dnf install enroll +``` + +## AppImage +Download it from my Releases page, then: ```bash chmod +x Enroll.AppImage ./Enroll.AppImage ``` -### Pip/PipX - +## Pip/PipX ```bash pip install enroll ``` -### Poetry - -Clone this repository with git, then: - +## Poetry (dev) ```bash poetry install poetry run enroll --help ``` -## Usage +--- -On the host (root recommended to harvest as much data as possible): +## Found a bug / have a suggestion? -### 1. Harvest state/information about the host +My Forgejo doesn't currently support federation, so I haven't opened registration/login for issues. +Instead, email me (see `pyproject.toml`) or contact me on the Fediverse: + +https://goto.mig5.net/@mig5 + +--- + +# Examples + +## Harvest + +### Local harvest ```bash enroll harvest --out /tmp/enroll-harvest ``` -### 2. Generate Ansible manifests (roles/playbook) from that harvest +### Remote harvest over SSH +```bash +enroll harvest --remote-host myhost.example.com --remote-user myuser --out /tmp/enroll-harvest +``` +### Remote harvest over SSH, where the SSH configuration is in ~/.ssh/config (e.g a different SSH key) + +Note: you must still pass `--remote-host`, but in this case, its value can be the 'Host' alias of an entry in your `~/.ssh/config`. + +```bash +enroll harvest --remote-host myhostalias --remote-ssh-config ~/.ssh/config --out /tmp/enroll-harvest +``` + +### Include paths (`--include-path`) +```bash +# Add a few dotfiles from /home (still secret-safe unless --dangerous) +enroll harvest --out /tmp/enroll-harvest --include-path '/home/*/.bashrc' --include-path '/home/*/.profile' +``` + +### Exclude paths (`--exclude-path`) +```bash +# Skip specific /usr/local/bin entries (or patterns) +enroll harvest --out /tmp/enroll-harvest --exclude-path '/usr/local/bin/docker-*' --exclude-path '/usr/local/bin/some-tool' +``` + +### Regex include +```bash +enroll harvest --out /tmp/enroll-harvest --include-path 're:^/home/[^/]+/\.config/myapp/.*$' +``` + +### `--dangerous` +```bash +enroll harvest --out /tmp/enroll-harvest --dangerous +``` + +### Remote + dangerous: +```bash +enroll harvest --remote-host myhost.example.com --remote-user myuser --dangerous +``` + +### `--sops` (encrypt at rest) +```bash +# Encrypted harvest bundle (writes /tmp/enroll-harvest/harvest.tar.gz.sops) +enroll harvest --out /tmp/enroll-harvest --dangerous --sops +``` + +--- + +## Manifest + +### Single-site (default: no --fqdn) ```bash enroll manifest --harvest /tmp/enroll-harvest --out /tmp/enroll-ansible ``` -### Alternatively, do both steps in one shot: - +### Multi-site (--fqdn) ```bash -enroll enroll --harvest /tmp/enroll-harvest --out /tmp/enroll-ansible +enroll manifest --harvest /tmp/enroll-harvest --out /tmp/enroll-ansible --fqdn "$(hostname -f)" ``` -Then run: +### Container image caches + +If Docker or Podman is available during harvest, Enroll records local image-cache metadata from `image ls` and `image inspect`. Images that expose registry `RepoDigest` values are reproducible by digest, for example `registry.example.net/app@sha256:...`; those are the references rendered into manifests. Local image IDs and tag-only images are preserved as evidence and notes, but are not treated as exact registry pull references. + +For Ansible, digest-pinned Docker images are pulled with `community.docker.docker_image_pull` and digest-pinned Podman images are pulled with `containers.podman.podman_image`; harvested tag aliases are re-applied where possible. The generated `requirements.yml` includes `community.docker` and `containers.podman` alongside any other required collections. In `--fqdn` mode the image list is host-specific inventory data. + +### Puppet target +```bash +enroll manifest --harvest /tmp/enroll-harvest --out /tmp/enroll-puppet --target puppet +``` + +The Puppet target renders native packages, users/groups, managed directories/files/symlinks, basic service state, and the generated sysctl file/apply exec when present. Without `--fqdn`, `site.pp` uses `node default { ... }`; with `--fqdn`, it uses `node '' { ... }`. Run from the generated output directory with the generated modules on Puppet's module path, for example: + +```bash +cd /tmp/enroll-puppet +sudo puppet apply --modulepath ./modules manifests/site.pp --noop +``` + +Or with absolute paths: + +```bash +sudo puppet apply --modulepath /tmp/enroll-puppet/modules /tmp/enroll-puppet/manifests/site.pp --noop +``` + +Docker images with registry digests are rendered as `docker::image` resources and require the Puppet environment to provide `puppetlabs-docker`; the generated module metadata records that dependency. Podman images with registry digests are rendered as guarded `podman pull` / `podman tag` exec resources. Images without `RepoDigest` are recorded in harvest state and notes, but are not converted into exact pull resources. Flatpak, Snap, and live firewall runtime snapshots are listed as notes in the generated Puppet README rather than converted into Puppet resources. + +### Manifest with `--sops` +```bash +# Generate encrypted manifest bundle (writes /tmp/enroll-ansible/manifest.tar.gz.sops) +enroll manifest --harvest /tmp/enroll-harvest/harvest.tar.gz.sops --out /tmp/enroll-ansible --sops + +# Decrypt/extract the manifest bundle, then run Ansible from inside ./manifest/ +cd /tmp/enroll-ansible +sops -d manifest.tar.gz.sops | tar -xzvf - +cd manifest +``` + +--- + +## Single-shot + +```bash +enroll single-shot --harvest /tmp/enroll-harvest --out /tmp/enroll-ansible --fqdn "$(hostname -f)" +``` + +Remote single-shot (run harvest over SSH, then manifest locally): +```bash +enroll single-shot --remote-host myhost.example.com --remote-user myuser --harvest /tmp/enroll-harvest --out /tmp/enroll-ansible --fqdn "myhost.example.com" +``` + +--- + +## Diff + +### Compare two harvest directories, output in json +```bash +enroll diff --old /path/to/harvestA --new /path/to/harvestB --format json +``` + +### Diff + webhook notify +```bash +enroll diff --old /path/to/golden/harvest --new /path/to/new/harvest --webhook https://nr.mig5.net/forms/webhooks/xxxx --webhook-format json --webhook-header 'X-Enroll-Secret: xxxx' +``` + +`diff` mode also supports email sending and text or markdown format, as well as `--exit-code` mode to trigger a return code of 2 (useful for crons or CI) + +### Ignore a specific directory or file from the diff +```bash +enroll diff --old /path/to/harvestA --new /path/to/harvestB --exclude-path /var/anacron +``` + +### Ignore package version drift (routine upgrades) but still alert on add/remove +```bash +enroll diff --old /path/to/harvestA --new /path/to/harvestB --ignore-package-versions +``` + +### Enforce the old harvest state when drift is detected (requires Ansible) +```bash +enroll diff --old /path/to/harvestA --new /path/to/harvestB --enforce --ignore-package-versions --exclude-path /var/anacron +``` + +--- + +## Explain + +### Explain a harvest + +All of these do the same thing: + +```bash +enroll explain /path/to/state.json +enroll explain /path/to/bundle_dir +enroll explain /path/to/harvest.tar.gz +``` + +### Explain a SOPS-encrypted harvest + +```bash +enroll explain /path/to/harvest.tar.gz.sops --sops +``` + +### Explain with JSON output and more examples + +```bash +enroll explain /path/to/state.json --format json --max-examples 25 +``` + +### Example output + +``` +❯ enroll explain /tmp/syrah.harvest +Enroll explain: /tmp/syrah.harvest +Host: syrah.mig5.net (os: debian, pkg: dpkg) +Enroll: 0.2.3 + +Inventory +- Packages: 254 +- Why packages were included (observed_via): + - user_installed: 248 – Package appears explicitly installed (as opposed to only pulled in as a dependency). + - package_role: 232 – Package was referenced by an enroll packages snapshot/role. (e.g. acl, acpid, adduser) + - systemd_unit: 22 – Package is associated with a systemd unit that was harvested. (e.g. postfix.service, tor.service, apparmor.service) + +Roles collected +- users: 1 user(s), 1 file(s), 0 excluded +- services: 19 unit(s), 111 file(s), 6 excluded +- packages: 232 package snapshot(s), 41 file(s), 0 excluded +- apt_config: 26 file(s), 7 dir(s), 10 excluded +- dnf_config: 0 file(s), 0 dir(s), 0 excluded +- firewall_runtime: 2 snapshot(s), 1 ipset(s) +- etc_custom: 70 file(s), 20 dir(s), 0 excluded +- usr_local_custom: 35 file(s), 1 dir(s), 0 excluded +- extra_paths: 0 file(s), 0 dir(s), 0 excluded + +Why files were included (managed_files.reason) +- custom_unowned (179): A file not owned by any package (often custom/operator-managed).. Examples: /etc/apparmor.d/local/lsb_release, /etc/apparmor.d/local/nvidia_modprobe, /etc/apparmor.d/local/sbin.dhclient +- usr_local_bin_script (35): Executable scripts under /usr/local/bin (often operator-installed).. Examples: /usr/local/bin/check_firewall, /usr/local/bin/awslogs +- apt_keyring (13): Repository signing key material used by APT.. Examples: /etc/apt/keyrings/openvpn-repo-public.asc, /etc/apt/trusted.gpg, /etc/apt/trusted.gpg.d/deb.torproject.org-keyring.gpg +- modified_conffile (10): A package-managed conffile differs from the packaged/default version.. Examples: /etc/dnsmasq.conf, /etc/ssh/moduli, /etc/tor/torrc +- logrotate_snippet (9): logrotate snippets/configs referenced in system configuration.. Examples: /etc/logrotate.d/rsyslog, /etc/logrotate.d/tor, /etc/logrotate.d/apt +- apt_config (7): APT configuration affecting package installation and repository behavior.. Examples: /etc/apt/apt.conf.d/01autoremove, /etc/apt/apt.conf.d/20listchanges, /etc/apt/apt.conf.d/70debconf +[...] +``` + +--- + +## Run Ansible + +### Single-site ```bash ansible-playbook -i "localhost," -c local /tmp/enroll-ansible/playbook.yml ``` +### Multi-site (--fqdn) +```bash +ansible-playbook /tmp/enroll-ansible/playbooks/"$(hostname -f)".yml +``` -## Notes / Safety +### Run only specific roles (tags) +Generated playbooks tag each role as `role_` (e.g. `role_users`, `role_services`), so you can speed up targeted runs: +```bash +ansible-playbook -i "localhost," -c local /tmp/enroll-ansible/playbook.yml --tags role_users +``` -- enroll **skips** common sensitive locations like `/etc/ssl/private/*`, `/etc/ssh/ssh_host_*`, and files that look like private keys/tokens. -- It also skips symlinks, binary-ish files, and large files by default. -- Review each generated role’s README before committing it anywhere. -- It only stores the raw config files. If you want to turn these into Jinja2 templates with dynamic inventory, see my other tool https://git.mig5.net/mig5/jinjaturtle . +## Configuration file +As can be seen above, there are a lot of powerful 'permutations' available to all four subcommands. -## Troubleshooting +Sometimes, it can be easier to store them in a config file so you don't have to remember them! -- Run as root for the most complete harvest (`sudo ...`). +Enroll supports reading an ini-style file of all the arguments for each subcommand. -## Found a bug, have a suggestion? +### Location of the config file -You can e-mail me (see the pyproject.toml for details) or contact me on the Fediverse: +The path the config file can be specified with `-c` or `--config` on the command-line. Otherwise, +Enroll will look for `./enroll.ini`, `./.enroll.ini` (in the current working directory), +`~/.config/enroll/enroll.ini` (or `$XDG_CONFIG_HOME/enroll/enroll.ini`). -https://goto.mig5.net/@mig5 +You may also pass `--no-config` if you deliberately want to ignore the config file even if it existed. + +### Precedence + +Highest wins: + + * Explicit CLI flags + * INI config ([cmd], [enroll]) + * argparse defaults + +### Example config file + +Here is an example. + +Whenever an argument on the command-line has a 'hyphen' in it, just be sure to change it to an underscore in the ini file. + +```ini +[enroll] +# (future global flags may live here) + +[harvest] +dangerous = false +include_path = + /home/*/.bashrc + /home/*/.profile +exclude_path = /usr/local/bin/docker-*, /usr/local/bin/some-tool +# remote_host = yourserver.example.com +# remote_user = you +# remote_port = 2222 + +[manifest] +# you can set defaults here too, e.g. +no_jinjaturtle = true +sops = 54A91143AE0AB4F7743B01FE888ED1B423A3BC99 + +[diff] +# ignore noisy drift +exclude_path = /var/anacron +ignore_package_versions = true +# enforce = true # requires ansible-playbook on PATH + +[single-shot] +# if you use single-shot, put its defaults here. +# It does not inherit those of the subsections above, so you +# may wish to repeat them here. +include_path = re:^/home/[^/]+/\.config/myapp/.*$ +``` diff --git a/debian/changelog b/debian/changelog index 86ae088..5292e0e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,164 @@ +enroll (0.6.0) unstable; urgency=medium + + * Add support for capturing ipset and iptables configuration files + * Add support for generating ipset and iptables configuration files from runtime, if the former weren't present ('firewall_runtime' role) + + -- Miguel Jacq Thu, 14 May 2026 15:00 +1000 + +enroll (0.5.0) unstable; urgency=medium + + * Add ssh config support where JinjaTurtle is used + + -- Miguel Jacq Tue, 12 May 2026 12:00 +1000 + +enroll (0.4.4) unstable; urgency=medium + + * Add capability to handle passphrases on encrypted SSH private keys. Prompting can be forced with `--ask-key-passphrase` or automated (e.g for CI) with `--ssh-key-passphrase env SOMEVAR` + + -- Miguel Jacq Tue, 17 Feb 2026 11:00 +1100 + +enroll (0.4.3) unstable; urgency=medium + + * Add support for AddressFamily and ConnectTimeout in the .ssh/config when using `--remote-ssh-config`. + + -- Miguel Jacq Fri, 16 Jan 2026 11:00 +1100 + +enroll (0.4.2) unstable; urgency=medium + + * Support `--remote-ssh-config [path-to-ssh-config]` as an argument in case extra params are required beyond `--remote-port` or `--remote-user`. Note: `--remote-host` must still be set, but it can be an 'alias' represented by the 'Host' value in the ssh config. + + -- Miguel Jacq Tue, 13 Jan 2026 21:55:00 +1100 + +enroll (0.4.1) unstable; urgency=medium + * Add interactive output when 'enroll diff --enforce' is invoking Ansible. + + -- Miguel Jacq Sun, 11 Jan 2026 10:00:00 +1100 + +enroll (0.4.0) unstable; urgency=medium + * Introduce `enroll validate` - a tool to validate a harvest against the state schema, or check for missing or orphaned obsolete artifacts in a harvest. + * Attempt to generate Jinja2 templates of systemd unit files and Postfix main.cf (now that JinjaTurtle supports it) + * Update pynacl dependency to resolve CVE-2025-69277 + * Add `--exclude-path` to `enroll diff` command, so that you can ignore certain churn from the diff (stuff you still wanted to harvest as a baseline but don't care if it changes day to day) + * Add `--ignore-package-versions` to `enroll diff` command, to optionally ignore package upgrades (e.g due to patching) from the diff. + * Add tags to the playbook for each role, to allow easier targeting of specific roles during play later. + * Add `--enforce` mode to `enroll diff`. If there is diff detected between the two harvests, and it can enforce restoring the state from the older harvest, it will manifest the state and apply it with ansible. +Only the specific roles that had diffed will be applied (via the new tags capability) + + -- Miguel Jacq Sat, 10 Jan 2026 10:30:00 +1100 + +enroll (0.3.0) unstable; urgency=medium + + * Introduce `enroll explain` - a tool to analyze and explain what's in (or not in) a harvest and why. + * Centralise the cron and logrotate stuff into their respective roles, we had a bit of duplication between roles based on harvest discovery. + * Capture other files in the user's home directory such as `.bashrc`, `.bash_aliases`, `.profile`, if these files differ from the `/etc/skel` defaults + * Ignore files that end with a tilde or - (probably backup files generated by editors or shadow file changes) + * Manage certain symlinks e.g for apache2/nginx sites-enabled and so on + + -- Miguel Jacq Mon, 05 Jan 2026 17:00:00 +1100 + +enroll (0.2.3) unstable; urgency=medium + + * Introduce --ask-become-pass or -K to support password-required sudo on remote hosts, just like Ansible. It will also fall back to this prompt if a password is required but the arg wasn't passed in. + + -- Miguel Jacq Sun, 04 Jan 2026 20:38:00 +1100 + +enroll (0.2.2) unstable; urgency=medium + + * Fix stat() of parent directory so that we set directory perms correct on --include paths. + * Set pty for remote calls when sudo is required, to help systems with limits on sudo without pty + + -- Miguel Jacq Sat, 03 Jan 2026 09:56:00 +1100 + +enroll (0.2.1) unstable; urgency=medium + + * Don't accidentally add extra_paths role to usr_local_custom list, resulting in extra_paths appearing twice in manifested playbook + * Ensure directories in the tree of anything included with --include are defined in the state and manifest so we make dirs before we try to create files + + -- Miguel Jacq Fri, 02 Jan 2026 21:30:00 +1100 + +enroll (0.2.0) unstable; urgency=medium + + * Add version CLI arg + * Add ability to enroll RH-style systems (DNF5/DNF/RPM) + * Refactor harvest state to track package versions + + -- Miguel Jacq Mon, 29 Dec 2025 17:30:00 +1100 + +enroll (0.1.7) unstable; urgency=medium + * Fix an attribution bug for certain files ending up in the wrong package/role. + + -- Miguel Jacq Sun, 28 Dec 2025 18:30:00 +1100 + +enroll (0.1.6) unstable; urgency=medium + + * DRY up some code logic + * More test coverage + + -- Miguel Jacq Sun, 28 Dec 2025 15:30:00 +1100 + +enroll (0.1.5) unstable; urgency=medium + + * Consolidate logrotate and cron files into their main service/package roles if they exist. + * Standardise on MAX_FILES_CAP in one place + * Manage apt stuff in its own role, not in etc_custom + + -- Miguel Jacq Sun, 28 Dec 2025 10:00:00 +1100 + +enroll (0.1.4) unstable; urgency=medium + + * Attempt to capture more stuff from /etc that might not be attributable to a specific package. This includes common singletons and systemd timers + * Avoid duplicate apt data in package-specific roles. + + -- Miguel Jacq Sat, 27 Dec 2025 19:00:00 +1100 + +enroll (0.1.3) unstable; urgency=medium + + * Allow the user to add extra paths to harvest, or paths to ignore, using `--exclude-path` and `--include-path` + arguments. + * Add support for an enroll.ini config file to store arguments per subcommand, to avoid having to remember + them all for repetitive executions. + + -- Miguel Jacq Sat, 20 Dec 2025 18:24:00 +1100 + +enroll (0.1.2) unstable; urgency=medium + + * Include files from `/usr/local/bin` and `/usr/local/etc` in harvest (assuming they aren't binaries or + symlinks) and store in `usr_local_custom` role, similar to `etc_custom`. + + -- Miguel Jacq Thu, 18 Dec 2025 17:07:00 +1100 + +enroll (0.1.1) unstable; urgency=medium + + * Add `diff` subcommand which can compare two harvests and send email or webhook notifications in different + formats. + + -- Miguel Jacq Thu, 18 Dec 2025 15:00:00 +1100 + +enroll (0.1.0) unstable; urgency=medium + + * Add remote mode for harvesting a remote machine via a local workstation (no need to install enroll remotely) + Optionally use `--no-sudo` if you don't want the remote user to have passwordless sudo when conducting the + harvest, albeit you'll end up with less useful data (same as if running `enroll harvest` on a machine without + sudo) + * Add `--dangerous` flag to capture even sensitive data (use at your own risk!) + * Add `--sops` flag which makes the harvest and the manifest 'out' data encrypted as a single SOPS data file. + This would make `--dangerous` a little bit safer, if your intention is just to store the Ansible manifest + in git or somewhere similar for disaster-recovery purposes (e.g encrypted at rest for safe-keeping). + * Do a better job at capturing other config files in `/etc//` even if that package doesn't normally + ship or manage those files. + * Don't collect files ending in `.log` + + -- Miguel Jacq Wed, 17 Dec 2025 18:00:00 +1100 + +enroll (0.0.5) unstable; urgency=medium + + * Use JinjaTurtle to generate dynamic template/inventory if it's on the PATH + * Support --fqdn flag for site-specific inventory and an inventory hosts file + * Generate an ansible.cfg if not present, to support host_vars plugin and other params + * Be more permissive with files that we previously thought contained secrets (ignore commented lines) + + -- Miguel Jacq Tue, 16 Dec 2025 12:00:00 +1100 + enroll (0.0.4) unstable; urgency=medium * Fix dash package detection issue diff --git a/debian/control b/debian/control index 372f83f..d5a21fe 100644 --- a/debian/control +++ b/debian/control @@ -8,12 +8,15 @@ Build-Depends: dh-python, pybuild-plugin-pyproject, python3-all, - python3-poetry-core + python3-yaml, + python3-poetry-core, + python3-paramiko, + python3-jsonschema Standards-Version: 4.6.2 Homepage: https://git.mig5.net/mig5/enroll Package: enroll Architecture: all -Depends: ${misc:Depends}, ${python3:Depends} +Depends: ${misc:Depends}, ${python3:Depends}, python3-yaml, python3-paramiko, python3-jsonschema Description: Harvest a host into Ansible roles A tool that inspects a system and emits Ansible roles/playbooks to reproduce it. diff --git a/enroll.svg b/enroll.svg index c986e1f..0ee1590 100644 --- a/enroll.svg +++ b/enroll.svg @@ -109,4 +109,3 @@ enroll - diff --git a/enroll/accounts.py b/enroll/accounts.py index cf2fcd3..b4f774b 100644 --- a/enroll/accounts.py +++ b/enroll/accounts.py @@ -1,8 +1,48 @@ from __future__ import annotations +import configparser import os -from dataclasses import dataclass -from typing import Dict, List, Set, Tuple +import re +import shutil +import subprocess # nosec +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Set, Tuple + + +@dataclass +class FlatpakInstall: + name: str + method: str + remote: Optional[str] = None + branch: Optional[str] = None + arch: Optional[str] = None + kind: Optional[str] = None + ref: Optional[str] = None + user: Optional[str] = None + home: Optional[str] = None + source: str = "filesystem" + + +@dataclass +class FlatpakRemote: + name: str + method: str + url: str + user: Optional[str] = None + home: Optional[str] = None + source: str = "filesystem" + + +@dataclass +class SnapInstall: + name: str + channel: Optional[str] = None + revision: Optional[int] = None + classic: bool = False + devmode: bool = False + dangerous: bool = False + notes: List[str] = field(default_factory=list) + source: str = "snap-list" @dataclass @@ -16,6 +56,7 @@ class UserRecord: primary_group: str supplementary_groups: List[str] ssh_files: List[str] + flatpaks: List[FlatpakInstall] = field(default_factory=list) def parse_login_defs(path: str = "/etc/login.defs") -> Dict[str, int]: @@ -115,6 +156,612 @@ def find_user_ssh_files(home: str) -> List[str]: return sorted(set(out)) +def _read_first_existing_text(paths: List[str]) -> Optional[str]: + for path in paths: + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + value = f.read().strip() + if value: + return value + except OSError: + continue + return None + + +def _parse_flatpak_ref( + ref: str, +) -> Tuple[Optional[str], str, Optional[str], Optional[str]]: + """Return (kind, name, arch, branch) for a Flatpak ref. + + refs look like app/org.example.App/x86_64/stable or + runtime/org.example.Platform/x86_64/23.08. If the value is already just an + application/runtime ID, keep it as the name and leave the other fields empty. + """ + parts = [p for p in (ref or "").strip().split("/") if p] + if len(parts) >= 4 and parts[0] in {"app", "runtime"}: + return parts[0], parts[1], parts[2], parts[3] + return None, (ref or "").strip(), None, None + + +def _parse_plain_flatpak_list_output( + output: str, + *, + method: str, + user: Optional[str] = None, + home: Optional[str] = None, +) -> List[FlatpakInstall]: + """Parse default `flatpak list` table output. + + Example: + Name Application ID Version Branch Installation + OnionShare org.onionshare.OnionShare 2.6.4 stable system + """ + out: List[FlatpakInstall] = [] + seen: Set[ + Tuple[str, Optional[str], Optional[str], Optional[str], Optional[str]] + ] = set() + id_re = re.compile(r"\b(?:[A-Za-z0-9_-]+\.)+[A-Za-z0-9_-]+\b") + for line in output.splitlines(): + line = line.rstrip() + if not line.strip(): + continue + if "Application ID" in line and "Installation" in line: + continue + match = id_re.search(line) + if not match: + continue + name = match.group(0) + tail = line[match.end() :].split() + installation = tail[-1] if tail else "" + if installation in {"system", "user"} and installation != method: + continue + branch = None + if len(tail) >= 2 and tail[-1] in {"system", "user"}: + branch = tail[-2] + elif tail: + branch = tail[-1] + + key = (name, None, branch, None, None) + if key in seen: + continue + seen.add(key) + out.append( + FlatpakInstall( + name=name, + method=method, + remote=None, + branch=branch, + arch=None, + kind=None, + ref=None, + user=user, + home=home, + source="flatpak-list", + ) + ) + return sorted(out, key=lambda f: (f.name, f.branch or "")) + + +def _parse_flatpak_list_output( + output: str, + *, + method: str, + columns: Optional[Tuple[str, ...]] = None, + user: Optional[str] = None, + home: Optional[str] = None, +) -> List[FlatpakInstall]: + """Parse Flatpak list output. + + If columns is None, parse the default table. Otherwise columns names must + match the order passed to `flatpak list --columns=...`. + """ + if columns is None: + return _parse_plain_flatpak_list_output( + output, method=method, user=user, home=home + ) + + out: List[FlatpakInstall] = [] + seen: Set[ + Tuple[str, Optional[str], Optional[str], Optional[str], Optional[str]] + ] = set() + for line in output.splitlines(): + line = line.strip() + if not line: + continue + lower = line.lower() + if lower.startswith("ref") or lower.startswith("application id"): + continue + + parts = line.split("\t") + if len(parts) < len(columns): + parts = line.split() + if not parts: + continue + + fields = { + name: parts[idx].strip() + for idx, name in enumerate(columns) + if idx < len(parts) + } + ref = fields.get("ref") or fields.get("application") or "" + kind, name, ref_arch, ref_branch = _parse_flatpak_ref(ref) + if not name: + continue + + remote = fields.get("origin") or None + branch = fields.get("branch") or ref_branch + arch = fields.get("arch") or ref_arch + if remote in {"", "-"}: + remote = None + if branch in {"", "-"}: + branch = None + if arch in {"", "-"}: + arch = None + + key = (name, remote, branch, arch, kind) + if key in seen: + continue + seen.add(key) + out.append( + FlatpakInstall( + name=name, + method=method, + remote=remote, + branch=branch, + arch=arch, + kind=kind, + ref=ref if "/" in ref else None, + user=user, + home=home, + source="flatpak-list", + ) + ) + return sorted( + out, + key=lambda f: ( + f.kind or "", + f.name, + f.remote or "", + f.branch or "", + f.arch or "", + ), + ) + + +_KNOWN_FLATPAK_LIST_COLUMNS = { + "name", + "description", + "application", + "version", + "branch", + "arch", + "origin", + "installation", + "ref", + "active", + "latest", + "size", + "options", +} + + +def _parse_flatpak_columns_help(output: str) -> Set[str]: + """Parse `flatpak list --columns=help` output into supported fields.""" + supported: Set[str] = set() + for line in output.splitlines(): + # Help output varies a bit between Flatpak versions. Treat any known + # token as a supported field, whether it appears alone or in a + # description table. + for token in re.findall(r"[A-Za-z_][A-Za-z0-9_-]*", line.lower()): + if token in _KNOWN_FLATPAK_LIST_COLUMNS: + supported.add(token) + return supported + + +def _run_flatpak_columns_help() -> Optional[Set[str]]: + if shutil.which("flatpak") is None: + return None + try: + proc = subprocess.run( # nosec + ["flatpak", "list", "--columns=help"], + shell=False, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=10, + ) + except Exception: + return None + if proc.returncode != 0: + return None + supported = _parse_flatpak_columns_help(proc.stdout or "") + return supported or None + + +def _flatpak_list_attempts( + scope: str, supported: Optional[Set[str]] +) -> List[Tuple[List[str], Optional[Tuple[str, ...]]]]: + def supported_columns(*wanted: str) -> Optional[Tuple[str, ...]]: + if supported is not None and not set(wanted).issubset(supported): + return None + return tuple(wanted) + + column_sets: List[Tuple[str, ...]] = [] + for wanted in ( + ("application", "origin", "branch", "arch"), + ("application", "branch", "arch"), + ("application", "branch"), + ("application",), + ("ref", "origin", "branch", "arch"), + ("ref", "branch", "arch"), + ("ref", "branch"), + ("ref",), + ): + cols = supported_columns(*wanted) + if cols is not None and cols not in column_sets: + column_sets.append(cols) + + attempts: List[Tuple[List[str], Optional[Tuple[str, ...]]]] = [ + ( + ["flatpak", "list", scope, "--columns=" + ",".join(cols)], + cols, + ) + for cols in column_sets + ] + attempts.append((["flatpak", "list", scope], None)) + return attempts + + +def _run_flatpak_list(method: str) -> Optional[Tuple[str, Optional[Tuple[str, ...]]]]: + if shutil.which("flatpak") is None: + return None + + scope = "--system" if method == "system" else "--user" + supported = _run_flatpak_columns_help() + for args, columns in _flatpak_list_attempts(scope, supported): + try: + proc = subprocess.run( # nosec + args, + shell=False, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=10, + ) + except Exception: # nosec B112 + continue + if proc.returncode == 0: + return proc.stdout or "", columns + return None + + +def _flatpak_remote_from_ref( + flatpak_root: str, app_id: str, arch: str, branch: str, remote_names: List[str] +) -> Optional[str]: + for remote_name in remote_names: + ref = os.path.join( + flatpak_root, + "repo", + "refs", + "remotes", + remote_name, + "app", + app_id, + arch, + branch, + ) + if os.path.exists(ref): + return remote_name + return None + + +def _parse_flatpak_deploy_origin(branch_dir: str) -> Optional[str]: + active_dir = os.path.join(branch_dir, "active") + candidates = [ + os.path.join(active_dir, "origin"), + os.path.join(active_dir, "metadata"), + ] + + origin = _read_first_existing_text([candidates[0]]) + if origin: + return origin + + metadata = candidates[1] + if os.path.isfile(metadata): + parser = configparser.ConfigParser(interpolation=None) + try: + parser.read(metadata, encoding="utf-8") + except Exception: + return None + for section in ("Application", "Runtime"): + if parser.has_option(section, "origin"): + value = parser.get(section, "origin", fallback="").strip() + if value: + return value + return None + + +def _find_flatpaks_in_root( + flatpak_root: str, + *, + method: str, + user: Optional[str] = None, + home: Optional[str] = None, +) -> List[FlatpakInstall]: + apps_dir = os.path.join(flatpak_root, "app") + if not os.path.isdir(apps_dir): + return [] + + remote_names = [ + r.name + for r in find_flatpak_remotes(flatpak_root, method=method, user=user, home=home) + ] + out: List[FlatpakInstall] = [] + + try: + app_ids = sorted(os.listdir(apps_dir)) + except OSError: + return [] + + seen: Set[Tuple[str, Optional[str], Optional[str], Optional[str]]] = set() + for app_id in app_ids: + app_path = os.path.join(apps_dir, app_id) + if not os.path.isdir(app_path): + continue + try: + arches = sorted(os.listdir(app_path)) + except OSError: + continue + for arch in arches: + arch_path = os.path.join(app_path, arch) + if not os.path.isdir(arch_path): + continue + try: + branches = sorted(os.listdir(arch_path)) + except OSError: + continue + for branch in branches: + branch_path = os.path.join(arch_path, branch) + if not os.path.isdir(branch_path): + continue + active_dir = os.path.join(branch_path, "active") + if not os.path.exists(active_dir): + continue + remote = _parse_flatpak_deploy_origin(branch_path) + if not remote: + remote = _flatpak_remote_from_ref( + flatpak_root, app_id, arch, branch, remote_names + ) + key = (app_id, remote, branch, arch) + if key in seen: + continue + seen.add(key) + out.append( + FlatpakInstall( + name=app_id, + method=method, + remote=remote, + branch=branch or None, + arch=arch or None, + kind="app", + ref=f"app/{app_id}/{arch}/{branch}", + user=user, + home=home, + ) + ) + + return sorted( + out, key=lambda f: (f.name, f.remote or "", f.branch or "", f.arch or "") + ) + + +def find_flatpak_remotes( + flatpak_root: str, + *, + method: str, + user: Optional[str] = None, + home: Optional[str] = None, +) -> List[FlatpakRemote]: + """Return configured Flatpak remotes for a Flatpak installation root. + + Flatpak stores remotes in the OSTree repo config. This gives us the remote + names and repository URLs. It does not reliably preserve the original + .flatpakref/.flatpakrepo URL that was used during installation. + """ + config_path = os.path.join(flatpak_root, "repo", "config") + if not os.path.isfile(config_path): + return [] + + parser = configparser.ConfigParser(interpolation=None, strict=False) + try: + parser.read(config_path, encoding="utf-8") + except Exception: + return [] + + out: List[FlatpakRemote] = [] + for section in parser.sections(): + match = re.fullmatch(r'remote\s+"(.+)"', section) + if not match: + continue + name = match.group(1).strip() + url = parser.get(section, "url", fallback="").strip() + if not name or not url: + continue + out.append( + FlatpakRemote( + name=name, + method=method, + url=url, + user=user, + home=home, + ) + ) + + return sorted(out, key=lambda r: (r.method, r.user or "", r.name)) + + +def find_user_flatpaks(home: str, user: Optional[str] = None) -> List[FlatpakInstall]: + """Return per-user Flatpak applications installed under a home directory.""" + flatpak_root = os.path.join(home, ".local", "share", "flatpak") + return _find_flatpaks_in_root(flatpak_root, method="user", user=user, home=home) + + +def find_user_flatpak_remotes( + home: str, user: Optional[str] = None +) -> List[FlatpakRemote]: + flatpak_root = os.path.join(home, ".local", "share", "flatpak") + return find_flatpak_remotes(flatpak_root, method="user", user=user, home=home) + + +def find_system_flatpaks() -> List[FlatpakInstall]: + """Return Flatpak refs installed system-wide. + + Prefer `flatpak list --system` because it is Flatpak's own view of + installed refs and includes layouts the filesystem scanner might miss. + Fall back to the on-disk app deployment tree when the command is + unavailable or produces unparsable output. + """ + listing = _run_flatpak_list("system") + if listing is not None: + output, columns = listing + parsed = _parse_flatpak_list_output(output, method="system", columns=columns) + if parsed or not output.strip(): + return parsed + return _find_flatpaks_in_root("/var/lib/flatpak", method="system") + + +def find_system_flatpak_remotes() -> List[FlatpakRemote]: + return find_flatpak_remotes("/var/lib/flatpak", method="system") + + +def _parse_snap_notes(notes: str) -> List[str]: + if not notes or notes == "-": + return [] + cleaned = notes.replace(",", " ").replace(";", " ") + return sorted( + {n.strip().lower() for n in cleaned.split() if n.strip() and n.strip() != "-"} + ) + + +def _parse_snap_list_output(output: str) -> List[SnapInstall]: + out: List[SnapInstall] = [] + for idx, line in enumerate(output.splitlines()): + line = line.strip() + if not line: + continue + if idx == 0 and line.lower().startswith("name"): + continue + parts = line.split(maxsplit=5) + if len(parts) < 5: + continue + name = parts[0] + revision: Optional[int] + try: + revision = int(parts[2]) + except ValueError: + revision = None + tracking = parts[3] + channel = None if tracking in {"-", ""} else tracking + notes = _parse_snap_notes(parts[5] if len(parts) > 5 else "") + out.append( + SnapInstall( + name=name, + channel=channel, + revision=revision, + classic="classic" in notes, + devmode="devmode" in notes, + dangerous="dangerous" in notes, + notes=notes, + source="snap-list", + ) + ) + return sorted(out, key=lambda s: s.name) + + +def _run_snap_list() -> Optional[str]: + if shutil.which("snap") is None: + return None + try: + proc = subprocess.run( # nosec + ["snap", "list"], + shell=False, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=10, + ) + except Exception: + return None + if proc.returncode != 0: + return None + return proc.stdout or "" + + +def _find_system_snaps_from_filesystem() -> List[SnapInstall]: + snapd_snaps = "/var/lib/snapd/snaps" + if not os.path.isdir(snapd_snaps): + return [] + + current_revisions: Dict[str, int] = {} + snap_mounts = "/snap" + if os.path.isdir(snap_mounts): + try: + mount_names = os.listdir(snap_mounts) + except OSError: + mount_names = [] + for name in mount_names: + current = os.path.join(snap_mounts, name, "current") + try: + target = os.readlink(current) + except OSError: + continue + try: + current_revisions[name] = int(os.path.basename(target.rstrip("/"))) + except ValueError: + continue + + candidates: Dict[str, List[int]] = {} + try: + entries = os.listdir(snapd_snaps) + except OSError: + return [] + + for entry in entries: + if not entry.endswith(".snap") or "_" not in entry: + continue + name, rev_text = entry[:-5].rsplit("_", 1) + try: + revision = int(rev_text) + except ValueError: + continue + candidates.setdefault(name, []).append(revision) + + out: List[SnapInstall] = [] + for name, revisions in candidates.items(): + revision = current_revisions.get(name) + if revision is None: + revision = max(revisions) + out.append(SnapInstall(name=name, revision=revision, source="filesystem")) + return sorted(out, key=lambda s: s.name) + + +def find_system_snaps() -> List[SnapInstall]: + """Return system-wide snap packages. + + Prefer `snap list` because it exposes channel tracking and confinement notes. + Fall back to snapd's on-disk snap filenames when the command is unavailable. + """ + output = _run_snap_list() + if output is not None: + parsed = _parse_snap_list_output(output) + if parsed: + return parsed + return _find_system_snaps_from_filesystem() + + def collect_non_system_users() -> List[UserRecord]: defs = parse_login_defs() uid_min = defs.get("UID_MIN", 1000) @@ -139,6 +786,10 @@ def collect_non_system_users() -> List[UserRecord]: ssh_files = find_user_ssh_files(home) if home and home.startswith("/") else [] + flatpaks: List[FlatpakInstall] = [] + if home and home.startswith("/"): + flatpaks = find_user_flatpaks(home, user=name) + users.append( UserRecord( name=name, @@ -150,6 +801,7 @@ def collect_non_system_users() -> List[UserRecord]: primary_group=primary_group, supplementary_groups=supp, ssh_files=ssh_files, + flatpaks=flatpaks, ) ) diff --git a/enroll/ansible.py b/enroll/ansible.py new file mode 100644 index 0000000..b041642 --- /dev/null +++ b/enroll/ansible.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +from typing import Optional + +from .ansible_renderer.context import _prepare_ansible_context +from .ansible_renderer.layout import _write_manifest_playbook, _write_site_scaffold +from .ansible_renderer.model import ( + AnsibleManifestPlan, + AnsibleRole, + _collect_ansible_roles, +) +from .ansible_renderer.roles.container_images import _render_container_images_role +from .ansible_renderer.roles.desktop import _render_flatpak_role, _render_snap_role +from .ansible_renderer.roles.managed_files import _render_managed_file_roles +from .ansible_renderer.roles.packages import ( + _render_common_ansible_roles, + _render_package_roles, + _render_service_roles, +) +from .ansible_renderer.roles.runtime import ( + _render_firewall_runtime_role, + _render_sysctl_role, +) +from .ansible_renderer.roles.users import _render_users_role +from .state import inventory_packages_from_state, roles_from_state + + +class AnsibleManifestRenderer: + """Render Ansible roles and playbook from a harvest bundle.""" + + def __init__( + self, + bundle_dir: str, + out_dir: str, + *, + fqdn: Optional[str] = None, + jinjaturtle: str = "auto", + no_common_roles: bool = False, + ) -> None: + self.bundle_dir = bundle_dir + self.out_dir = out_dir + self.fqdn = fqdn + self.jinjaturtle = jinjaturtle + self.no_common_roles = no_common_roles + + def render(self) -> None: + state = AnsibleRole.load_state(self.bundle_dir) + roles = roles_from_state(state) + inventory_packages = inventory_packages_from_state(state) + + ctx = _prepare_ansible_context( + self.bundle_dir, + self.out_dir, + fqdn=self.fqdn, + jinjaturtle=self.jinjaturtle, + ) + _write_site_scaffold(ctx) + + use_common_roles = (not ctx.site_mode) and (not self.no_common_roles) + collection = _collect_ansible_roles( + roles, + inventory_packages, + use_common_roles=use_common_roles, + ) + + manifest_plan = AnsibleManifestPlan() + + _render_users_role(ctx, manifest_plan, roles.get("users", {})) + _render_flatpak_role(ctx, manifest_plan, roles.get("flatpak", {})) + _render_snap_role(ctx, manifest_plan, roles.get("snap", {})) + _render_container_images_role( + ctx, manifest_plan, roles.get("container_images", {}) + ) + _render_managed_file_roles(ctx, manifest_plan, roles) + _render_sysctl_role(ctx, manifest_plan, roles.get("sysctl", {})) + _render_firewall_runtime_role( + ctx, manifest_plan, roles.get("firewall_runtime", {}) + ) + _render_service_roles(ctx, manifest_plan, collection.services) + + common_tail_roles = _render_common_ansible_roles( + ctx, manifest_plan, collection.common_role_groups, collection.packages + ) + _render_package_roles(ctx, manifest_plan, collection.packages) + + # Place cron/logrotate at the end of the playbook so users exist before + # per-user crontabs are restored and core packages/services are in place. + for role in ("cron", "logrotate"): + manifest_plan.mark_tail_package(role) + for role in common_tail_roles: + manifest_plan.mark_tail_package(role) + + _write_manifest_playbook(ctx, manifest_plan.ordered_roles()) + + +def manifest_from_bundle_dir( + bundle_dir: str, + out_dir: str, + *, + fqdn: Optional[str] = None, + jinjaturtle: str = "auto", + no_common_roles: bool = False, +) -> None: + AnsibleManifestRenderer( + bundle_dir, + out_dir, + fqdn=fqdn, + jinjaturtle=jinjaturtle, + no_common_roles=no_common_roles, + ).render() diff --git a/enroll/ansible_renderer/__init__.py b/enroll/ansible_renderer/__init__.py new file mode 100644 index 0000000..97eb797 --- /dev/null +++ b/enroll/ansible_renderer/__init__.py @@ -0,0 +1 @@ +"""Ansible manifest renderer implementation.""" diff --git a/enroll/ansible_renderer/context.py b/enroll/ansible_renderer/context.py new file mode 100644 index 0000000..be69711 --- /dev/null +++ b/enroll/ansible_renderer/context.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Optional, Tuple + +from ..jinjaturtle import find_jinjaturtle_cmd + + +@dataclass +class AnsibleManifestContext: + bundle_dir: str + out_dir: str + roles_root: str + fqdn: Optional[str] + site_mode: bool + jt_exe: Optional[str] + jt_enabled: bool + + +def _resolve_jinjaturtle_mode(jinjaturtle: str) -> Tuple[Optional[str], bool]: + jt_exe = find_jinjaturtle_cmd() + if jinjaturtle not in ("auto", "on", "off"): + raise ValueError("jinjaturtle must be one of: auto, on, off") + if jinjaturtle == "on": + if not jt_exe: + raise RuntimeError("jinjaturtle requested but not found on PATH") + return jt_exe, True + if jinjaturtle == "auto": + return jt_exe, jt_exe is not None + return jt_exe, False + + +def _prepare_ansible_context( + bundle_dir: str, + out_dir: str, + *, + fqdn: Optional[str], + jinjaturtle: str, +) -> AnsibleManifestContext: + site_mode = fqdn is not None and fqdn != "" + jt_exe, jt_enabled = _resolve_jinjaturtle_mode(jinjaturtle) + + os.makedirs(out_dir, exist_ok=True) + roles_root = os.path.join(out_dir, "roles") + os.makedirs(roles_root, exist_ok=True) + + return AnsibleManifestContext( + bundle_dir=bundle_dir, + out_dir=out_dir, + roles_root=roles_root, + fqdn=fqdn, + site_mode=site_mode, + jt_exe=jt_exe, + jt_enabled=jt_enabled, + ) diff --git a/enroll/ansible_renderer/jinjaturtle.py b/enroll/ansible_renderer/jinjaturtle.py new file mode 100644 index 0000000..a3f0382 --- /dev/null +++ b/enroll/ansible_renderer/jinjaturtle.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import os +from typing import Any, Dict, List, Optional, Set, Tuple + +from ..jinjaturtle import can_jinjify_path, infer_other_formats, run_jinjaturtle +from .yamlutil import _merge_mappings_overwrite, _yaml_dump_mapping, _yaml_load_mapping + + +def _jinjify_managed_files( + bundle_dir: str, + role: str, + role_dir: str, + managed_files: List[Dict[str, Any]], + *, + jt_exe: Optional[str], + jt_enabled: bool, + overwrite_templates: bool, +) -> Tuple[Set[str], str]: + """ + Return (templated_src_rels, combined_vars_text). + combined_vars_text is a YAML mapping fragment (no leading ---). + """ + templated: Set[str] = set() + vars_map: Dict[str, Any] = {} + + if not (jt_enabled and jt_exe): + return templated, "" + + for mf in managed_files: + dest_path = mf.get("path", "") + src_rel = mf.get("src_rel", "") + if not dest_path or not src_rel: + continue + if not can_jinjify_path(dest_path): + continue + + artifact_path = os.path.join(bundle_dir, "artifacts", role, src_rel) + if not os.path.isfile(artifact_path): + continue + + try: + force_fmt = infer_other_formats(dest_path) + res = run_jinjaturtle( + jt_exe, artifact_path, role_name=role, force_format=force_fmt + ) + except Exception: + # If jinjaturtle cannot process a file for any reason, skip silently. + # (Enroll's core promise is to be optimistic and non-interactive.) + continue # nosec + + tmpl_rel = src_rel + ".j2" + tmpl_dst = os.path.join(role_dir, "templates", tmpl_rel) + if overwrite_templates or not os.path.exists(tmpl_dst): + os.makedirs(os.path.dirname(tmpl_dst), exist_ok=True) + with open(tmpl_dst, "w", encoding="utf-8") as f: + f.write(res.template_text) + + templated.add(src_rel) + if res.vars_text.strip(): + # merge YAML mappings; last wins (avoids duplicate keys) + chunk = _yaml_load_mapping(res.vars_text) + if chunk: + vars_map = _merge_mappings_overwrite(vars_map, chunk) + + if vars_map: + combined = _yaml_dump_mapping(vars_map, sort_keys=True) + return templated, combined + return templated, "" diff --git a/enroll/ansible_renderer/layout.py b/enroll/ansible_renderer/layout.py new file mode 100644 index 0000000..ddadb1b --- /dev/null +++ b/enroll/ansible_renderer/layout.py @@ -0,0 +1,304 @@ +from __future__ import annotations + +import os +import re +import shutil +import stat +import tempfile +from pathlib import Path +from typing import Any, Dict, List, Optional, Set + +from .context import AnsibleManifestContext +from .yamlutil import _merge_mappings_overwrite, _yaml_dump_mapping, _yaml_load_mapping + + +def _copy2_replace(src: str, dst: str) -> None: + dst_dir = os.path.dirname(dst) + os.makedirs(dst_dir, exist_ok=True) + + # Copy to a temp file in the same directory, then atomically replace. + fd, tmp = tempfile.mkstemp(prefix=".enroll-tmp-", dir=dst_dir) + os.close(fd) + try: + shutil.copy2(src, tmp) + + # Ensure the working tree stays mergeable: make the file user-writable. + st = os.stat(tmp, follow_symlinks=False) + mode = stat.S_IMODE(st.st_mode) + if not (mode & stat.S_IWUSR): + os.chmod(tmp, mode | stat.S_IWUSR) + + os.replace(tmp, dst) + finally: + try: + os.unlink(tmp) + except FileNotFoundError: + pass + + +def _copy_artifacts( + bundle_dir: str, + role: str, + dst_files_dir: str, + *, + preserve_existing: bool = False, + exclude_rels: Optional[Set[str]] = None, +) -> None: + """Copy harvested artifacts for a role into a destination *files* directory. + + In non --fqdn mode, this is usually /files. + In --fqdn site mode, this is usually: + inventory/host_vars///.files + """ + artifacts_dir = os.path.join(bundle_dir, "artifacts", role) + if not os.path.isdir(artifacts_dir): + return + for root, _, files in os.walk(artifacts_dir): + for fn in files: + src = os.path.join(root, fn) + rel = os.path.relpath(src, artifacts_dir) + dst = os.path.join(dst_files_dir, rel) + + # If a file was successfully templatised by JinjaTurtle, do NOT + # also materialise the raw copy in the destination files dir. + if exclude_rels and rel in exclude_rels: + try: + if os.path.isfile(dst): + os.remove(dst) + except Exception: + pass # nosec + continue + + if preserve_existing and os.path.exists(dst): + continue + os.makedirs(os.path.dirname(dst), exist_ok=True) + _copy2_replace(src, dst) + + +def _write_role_scaffold(role_dir: str) -> None: + os.makedirs(os.path.join(role_dir, "tasks"), exist_ok=True) + os.makedirs(os.path.join(role_dir, "handlers"), exist_ok=True) + os.makedirs(os.path.join(role_dir, "defaults"), exist_ok=True) + os.makedirs(os.path.join(role_dir, "meta"), exist_ok=True) + os.makedirs(os.path.join(role_dir, "files"), exist_ok=True) + os.makedirs(os.path.join(role_dir, "templates"), exist_ok=True) + + +def _role_tag(role: str) -> str: + """Return a stable Ansible tag name for a role. + + Used by `enroll diff --enforce` to run only the roles needed to repair drift. + """ + r = str(role or "").strip() + # Ansible tag charset is fairly permissive, but keep it portable and consistent. + safe = re.sub(r"[^A-Za-z0-9_-]+", "_", r).strip("_") + if not safe: + safe = "other" + return f"role_{safe}" + + +def _write_playbook_all(path: str, roles: List[str]) -> None: + pb_lines = [ + "---", + "- name: Apply all roles on all hosts", + " gather_facts: true", + " hosts: all", + " become: true", + " roles:", + ] + for r in roles: + pb_lines.append(f" - role: {r}") + pb_lines.append(f" tags: [{_role_tag(r)}]") + with open(path, "w", encoding="utf-8") as f: + f.write("\n".join(pb_lines) + "\n") + + +def _write_playbook_host(path: str, fqdn: str, roles: List[str]) -> None: + pb_lines = [ + "---", + f"- name: Apply all roles on {fqdn}", + f" hosts: {fqdn}", + " gather_facts: true", + " become: true", + " roles:", + ] + for r in roles: + pb_lines.append(f" - role: {r}") + pb_lines.append(f" tags: [{_role_tag(r)}]") + with open(path, "w", encoding="utf-8") as f: + f.write("\n".join(pb_lines) + "\n") + + +def _ensure_ansible_cfg(cfg_path: str) -> None: + if not os.path.exists(cfg_path): + with open(cfg_path, "w", encoding="utf-8") as f: + f.write("[defaults]\n") + f.write("roles_path = roles\n") + f.write("interpreter_python=/usr/bin/python3\n") + f.write("inventory = inventory\n") + f.write("stdout_callback = unixy\n") + f.write("force_color = 1\n") + f.write("vars_plugins_enabled = host_group_vars\n") + f.write("fact_caching = jsonfile\n") + f.write("fact_caching_connection = .enroll_cached_facts\n") + f.write("forks = 30\n") + f.write("remote_tmp = /tmp/ansible-${USER}\n") + f.write("timeout = 12\n") + f.write("[ssh_connection]\n") + f.write("pipelining = True\n") + f.write("scp_if_ssh = True\n") + return + + +def _ensure_requirements_yaml( + req_path: str, + collections: Optional[List[Dict[str, str]]] = None, +) -> None: + requested = collections or [{"name": "community.general", "version": ">=13.0.0"}] + + existing: Dict[str, Any] = {} + if os.path.exists(req_path): + try: + existing = _yaml_load_mapping(Path(req_path).read_text(encoding="utf-8")) + except Exception: + existing = {} + + current_items = existing.get("collections") + if not isinstance(current_items, list): + current_items = [] + + by_name: Dict[str, Dict[str, str]] = {} + ordered_names: List[str] = [] + for item in current_items: + if isinstance(item, str): + name = item.strip() + if not name: + continue + entry: Dict[str, str] = {"name": name} + elif isinstance(item, dict): + name = str(item.get("name") or "").strip() + if not name: + continue + entry = {str(k): str(v) for k, v in item.items() if v is not None} + entry["name"] = name + else: + continue + if name not in by_name: + ordered_names.append(name) + by_name[name] = entry + + for item in requested: + name = str(item.get("name") or "").strip() + if not name: + continue + entry = dict(item) + entry["name"] = name + if name not in by_name: + ordered_names.append(name) + by_name[name] = entry + else: + by_name[name].update( + {k: v for k, v in entry.items() if v not in (None, "")} + ) + + out = {"collections": [by_name[name] for name in ordered_names]} + Path(req_path).parent.mkdir(parents=True, exist_ok=True) + Path(req_path).write_text( + "---\n" + _yaml_dump_mapping(out, sort_keys=False), encoding="utf-8" + ) + + +def _ensure_inventory_host(inv_path: str, fqdn: str) -> None: + os.makedirs(os.path.dirname(inv_path), exist_ok=True) + if not os.path.exists(inv_path): + with open(inv_path, "w", encoding="utf-8") as f: + f.write("[all]\n") + f.write(fqdn + "\n") + return + + with open(inv_path, "r", encoding="utf-8") as f: + lines = [ln.rstrip("\n") for ln in f.readlines()] + + # ensure there is an [all] group; if not, create it at top + if not any(ln.strip() == "[all]" for ln in lines): + lines = ["[all]"] + lines + + # check if fqdn already present (exact match, ignoring whitespace) + if any(ln.strip() == fqdn for ln in lines): + return + + # append at end + lines.append(fqdn) + with open(inv_path, "w", encoding="utf-8") as f: + f.write("\n".join(lines) + "\n") + + +def _hostvars_path(site_root: str, fqdn: str, role: str) -> str: + return os.path.join(site_root, "inventory", "host_vars", fqdn, f"{role}.yml") + + +def _host_role_files_dir(site_root: str, fqdn: str, role: str) -> str: + """Host-specific files dir for a given role. + + Layout: + inventory/host_vars///.files/ + """ + return os.path.join(site_root, "inventory", "host_vars", fqdn, role, ".files") + + +def _write_hostvars(site_root: str, fqdn: str, role: str, data: Dict[str, Any]) -> None: + """Write host_vars YAML for a role for a specific host. + + This is host-specific state and should track the current harvest output. + Existing keys not mentioned in `data` are preserved, but keys in `data` + are overwritten (including list values). + """ + path = _hostvars_path(site_root, fqdn, role) + os.makedirs(os.path.dirname(path), exist_ok=True) + + existing_map: Dict[str, Any] = {} + if os.path.exists(path): + try: + existing_text = Path(path).read_text(encoding="utf-8") + existing_map = _yaml_load_mapping(existing_text) + except Exception: + existing_map = {} + + merged = _merge_mappings_overwrite(existing_map, data) + + out = "---\n" + _yaml_dump_mapping(merged, sort_keys=True) + with open(path, "w", encoding="utf-8") as f: + f.write(out) + + +def _write_role_defaults(role_dir: str, mapping: Dict[str, Any]) -> None: + """Overwrite role defaults/main.yml with the provided mapping.""" + defaults_path = os.path.join(role_dir, "defaults", "main.yml") + os.makedirs(os.path.dirname(defaults_path), exist_ok=True) + out = "---\n" + _yaml_dump_mapping(mapping, sort_keys=True) + with open(defaults_path, "w", encoding="utf-8") as f: + f.write(out) + + +def _write_site_scaffold(ctx: AnsibleManifestContext) -> None: + if not ctx.site_mode: + return + os.makedirs(os.path.join(ctx.out_dir, "inventory"), exist_ok=True) + os.makedirs(os.path.join(ctx.out_dir, "inventory", "host_vars"), exist_ok=True) + os.makedirs(os.path.join(ctx.out_dir, "playbooks"), exist_ok=True) + _ensure_inventory_host( + os.path.join(ctx.out_dir, "inventory", "hosts.ini"), ctx.fqdn or "" + ) + _ensure_ansible_cfg(os.path.join(ctx.out_dir, "ansible.cfg")) + _ensure_requirements_yaml(os.path.join(ctx.out_dir, "requirements.yml")) + + +def _write_manifest_playbook(ctx: AnsibleManifestContext, roles: List[str]) -> None: + if ctx.site_mode: + _write_playbook_host( + os.path.join(ctx.out_dir, "playbooks", f"{ctx.fqdn}.yml"), + ctx.fqdn or "", + roles, + ) + else: + _write_playbook_all(os.path.join(ctx.out_dir, "playbook.yml"), roles) diff --git a/enroll/ansible_renderer/model.py b/enroll/ansible_renderer/model.py new file mode 100644 index 0000000..2a52cb3 --- /dev/null +++ b/enroll/ansible_renderer/model.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Set + +from ..cm import CMModule, package_section_label, section_label_for_packages +from ..role_names import avoid_reserved_role_name + + +@dataclass +class AnsibleRoleCollection: + services: List[Dict[str, Any]] + packages: List[Dict[str, Any]] + common_role_groups: Dict[str, List[Dict[str, Any]]] + + +class AnsibleRole(CMModule): + """Ansible-specific view of a renderer-neutral CMModule.""" + + def __init__( + self, + role_name: str, + *, + var_prefix: Optional[str] = None, + section_label: Optional[str] = None, + grouped: bool = False, + ) -> None: + super().__init__(role_name=role_name, module_name=role_name) + self.var_prefix = var_prefix or role_name + self.section_label = section_label + self.grouped = grouped + self.entries: List[Dict[str, Any]] = [] + self.excluded: List[Dict[str, Any]] = [] + self.origin_lines: List[str] = [] + + def add_package_snapshot(self, snap: Dict[str, Any]) -> None: + pkg = str(snap.get("package") or "").strip() + source_role = str(snap.get("role_name") or pkg or self.role_name) + self.entries.append({"kind": "package", "snapshot": snap}) + if pkg: + self.packages.add(pkg) + self.origin_lines.append(f"package `{pkg}` from role `{source_role}`") + self.add_managed_content(snap) + + def add_service_snapshot(self, snap: Dict[str, Any]) -> None: + unit = str(snap.get("unit") or "").strip() + source_role = str(snap.get("role_name") or unit or self.role_name) + self.entries.append({"kind": "service", "snapshot": snap}) + for pkg in snap.get("packages", []) or []: + pkg_s = str(pkg or "").strip() + if pkg_s: + self.packages.add(pkg_s) + if unit: + unit_file_state = str(snap.get("unit_file_state") or "") + self.services.setdefault( + unit, + { + "name": unit, + "manage": True, + "enabled": unit_file_state in ("enabled", "enabled-runtime"), + "state": ( + "started" if snap.get("active_state") == "active" else "stopped" + ), + }, + ) + self.origin_lines.append(f"service `{unit}` from role `{source_role}`") + self.add_managed_content(snap) + + def add_managed_content(self, snap: Dict[str, Any]) -> None: + for d in self.managed_dirs_from_snapshot(snap): + path = str(d.get("path") or "").strip() + self.add_managed_dir( + path, + dest=path, + owner=d.get("owner") or "root", + group=d.get("group") or "root", + mode=d.get("mode") or "0755", + ) + + for mf in self.managed_files_from_snapshot(snap): + path = str(mf.get("path") or "").strip() + src_rel = str(mf.get("src_rel") or "").strip() + if not path or not src_rel: + continue + self.add_managed_file( + path, + dest=path, + src_rel=src_rel, + owner=mf.get("owner") or "root", + group=mf.get("group") or "root", + mode=mf.get("mode") or "0644", + reason=mf.get("reason") or "managed_file", + ) + + for ml in self.managed_links_from_snapshot(snap): + path = str(ml.get("path") or "").strip() + target = str(ml.get("target") or "").strip() + if not path or not target: + continue + self.add_managed_link(path, dest=path, src=target) + + self.excluded.extend(snap.get("excluded", []) or []) + self.add_snapshot_notes(snap) + + @property + def sorted_packages(self) -> List[str]: + return sorted(self.packages) + + @property + def systemd_units_var(self) -> List[Dict[str, Any]]: + return [self.services[k] for k in sorted(self.services)] + + +class AnsibleManifestPlan: + """Track generated Ansible roles without scattering category lists.""" + + _ORDER = ( + "apt_config", + "dnf_config", + "package", + "service", + "etc_custom", + "usr_local_custom", + "extra_paths", + "flatpak", + "snap", + "container_images", + "users", + "tail_package", + "sysctl", + "firewall_runtime", + ) + + def __init__(self) -> None: + self._roles: Dict[str, List[str]] = {category: [] for category in self._ORDER} + self._tail_packages: List[str] = [] + + def add(self, category: str, role: str) -> None: + if category not in self._roles: + raise ValueError(f"unknown Ansible role category: {category}") + if role and role not in self._roles[category]: + self._roles[category].append(role) + + def roles(self, category: str) -> List[str]: + return list(self._roles.get(category, [])) + + def has(self, category: str, role: str) -> bool: + return role in self._roles.get(category, []) + + def mark_tail_package(self, role: str) -> None: + if self.has("package", role) and role not in self._tail_packages: + self._tail_packages.append(role) + + def ordered_roles(self) -> List[str]: + tail = set(self._tail_packages) + package_roles = [r for r in self._roles["package"] if r not in tail] + out: List[str] = [] + for category in self._ORDER: + if category == "package": + out.extend(package_roles) + elif category == "tail_package": + out.extend(self._tail_packages) + else: + out.extend(self._roles[category]) + return out + + +def _role_id(raw: str) -> str: + """Return an Ansible-safe role identifier from an arbitrary label.""" + + s = re.sub(r"[^A-Za-z0-9]+", "_", raw or "misc") + s = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s) + s = s.lower() + s = re.sub(r"_+", "_", s).strip("_") + if not s: + s = "misc" + if not re.match(r"^[a-z_]", s): + s = "r_" + s + return s + + +def _section_role_name(label: str, occupied_roles: Set[str]) -> str: + """Create a stable section role name, avoiding generated-role collisions.""" + + base = avoid_reserved_role_name(_role_id(label), prefix="section") + role = base if base not in occupied_roles else f"section_{base}" + n = 2 + while role in occupied_roles: + role = f"section_{base}_{n}" + n += 1 + occupied_roles.add(role) + return role + + +def _collect_ansible_roles( + roles: Dict[str, Any], + inventory_packages: Dict[str, Any], + *, + use_common_roles: bool, +) -> AnsibleRoleCollection: + services = roles.get("services", []) or [] + packages = roles.get("packages", []) or [] + common_role_groups: Dict[str, List[Dict[str, Any]]] = {} + + if use_common_roles: + for svc in services: + label = section_label_for_packages( + svc.get("packages", []) or [], inventory_packages + ) + common_role_groups.setdefault(label, []).append( + {"kind": "service", "snapshot": svc} + ) + for pr in packages: + label = package_section_label(pr, inventory_packages) + common_role_groups.setdefault(label, []).append( + {"kind": "package", "snapshot": pr} + ) + return AnsibleRoleCollection( + services=[], packages=[], common_role_groups=common_role_groups + ) + + return AnsibleRoleCollection( + services=services, + packages=packages, + common_role_groups=common_role_groups, + ) diff --git a/enroll/ansible_renderer/readme.py b/enroll/ansible_renderer/readme.py new file mode 100644 index 0000000..81b4741 --- /dev/null +++ b/enroll/ansible_renderer/readme.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +import os +import re +from typing import Any, Callable, Dict, List, Set + + +def _markdown_list(items: List[str]) -> str: + values = [str(item) for item in items if str(item)] + return "\n".join(f"- {item}" for item in values) or "- (none)" + + +def _managed_file_lines( + managed_files: List[Dict[str, Any]], *, include_reason: bool +) -> List[str]: + out: List[str] = [] + for mf in managed_files: + path = str(mf.get("path") or "") + if not path: + continue + if include_reason: + out.append(f"{path} ({mf.get('reason')})") + else: + out.append(path) + return out + + +def _excluded_lines(excluded: List[Dict[str, Any]]) -> List[str]: + return [f"{e.get('path')} ({e.get('reason')})" for e in excluded if e.get("path")] + + +def _read_artifact_lines(bundle_dir: str, role: str, src_rel: str) -> List[str]: + art_path = os.path.join(bundle_dir, "artifacts", role, src_rel) + try: + with open(art_path, "r", encoding="utf-8", errors="replace") as f: + return [line.rstrip("\n") for line in f] + except OSError: + return [] + + +def _apt_config_readme( + *, + bundle_dir: str, + role: str, + snapshot: Dict[str, Any], + managed_files: List[Dict[str, Any]], + managed_dirs: List[Dict[str, Any]], + excluded: List[Dict[str, Any]], + notes: List[Any], +) -> str: + source_paths: List[str] = [] + keyring_paths: List[str] = [] + repo_hosts: Set[str] = set() + url_re = re.compile(r"(?:https?|ftp)://([^/\s]+)", re.IGNORECASE) + + for mf in managed_files: + path = str(mf.get("path") or "") + src_rel = str(mf.get("src_rel") or "") + if not path or not src_rel: + continue + if path == "/etc/apt/sources.list" or path.startswith( + "/etc/apt/sources.list.d/" + ): + source_paths.append(path) + for line in _read_artifact_lines(bundle_dir, role, src_rel): + s = line.strip() + if not s or s.startswith("#"): + continue + for match in url_re.finditer(s): + repo_hosts.add(match.group(1)) + if ( + path.startswith("/etc/apt/trusted.gpg") + or path.startswith("/etc/apt/keyrings/") + or path.startswith("/usr/share/keyrings/") + ): + keyring_paths.append(path) + + return f"""# apt_config + +APT configuration harvested from the system (sources, pinning, and keyrings). + +## Repository hosts +{_markdown_list(sorted(repo_hosts))} + +## Source files +{_markdown_list(sorted(set(source_paths)))} + +## Keyrings +{_markdown_list(sorted(set(keyring_paths)))} + +## Managed files +{_markdown_list(_managed_file_lines(managed_files, include_reason=True))} + +## Excluded +{_markdown_list(_excluded_lines(excluded))} + +## Notes +{_markdown_list([str(n) for n in notes])} +""" + + +def _dnf_config_readme( + *, + bundle_dir: str, + role: str, + snapshot: Dict[str, Any], + managed_files: List[Dict[str, Any]], + managed_dirs: List[Dict[str, Any]], + excluded: List[Dict[str, Any]], + notes: List[Any], +) -> str: + repo_paths: List[str] = [] + key_paths: List[str] = [] + repo_hosts: Set[str] = set() + url_re = re.compile(r"(?:https?|ftp)://([^/\s]+)", re.IGNORECASE) + file_url_re = re.compile(r"file://(/[^\s]+)") + + for mf in managed_files: + path = str(mf.get("path") or "") + src_rel = str(mf.get("src_rel") or "") + if not path or not src_rel: + continue + if path.startswith("/etc/yum.repos.d/") and path.endswith(".repo"): + repo_paths.append(path) + for line in _read_artifact_lines(bundle_dir, role, src_rel): + s = line.strip() + if not s or s.startswith("#") or s.startswith(";"): + continue + for match in url_re.finditer(s): + repo_hosts.add(match.group(1)) + for match in file_url_re.finditer(s): + key_paths.append(match.group(1)) + if path.startswith("/etc/pki/rpm-gpg/"): + key_paths.append(path) + + return f"""# dnf_config + +DNF/YUM configuration harvested from the system (repos, config files, and RPM GPG keys). + +## Repository hosts +{_markdown_list(sorted(repo_hosts))} + +## Repo files +{_markdown_list(sorted(set(repo_paths)))} + +## GPG keys +{_markdown_list(sorted(set(key_paths)))} + +## Managed files +{_markdown_list(_managed_file_lines(managed_files, include_reason=True))} + +## Excluded +{_markdown_list(_excluded_lines(excluded))} + +## Notes +{_markdown_list([str(n) for n in notes])} +""" + + +def _simple_managed_files_readme( + title: str, + description: str, + *, + include_reason: bool, +) -> Callable[..., str]: + def _builder( + *, + bundle_dir: str, + role: str, + snapshot: Dict[str, Any], + managed_files: List[Dict[str, Any]], + managed_dirs: List[Dict[str, Any]], + excluded: List[Dict[str, Any]], + notes: List[Any], + ) -> str: + return f"""# {title} + +{description} + +## Managed files +{_markdown_list(_managed_file_lines(managed_files, include_reason=include_reason))} + +## Excluded +{_markdown_list(_excluded_lines(excluded))} + +## Notes +{_markdown_list([str(n) for n in notes])} +""" + + return _builder + + +def _extra_paths_readme( + *, + bundle_dir: str, + role: str, + snapshot: Dict[str, Any], + managed_files: List[Dict[str, Any]], + managed_dirs: List[Dict[str, Any]], + excluded: List[Dict[str, Any]], + notes: List[Any], +) -> str: + include_pats = snapshot.get("include_patterns", []) or [] + exclude_pats = snapshot.get("exclude_patterns", []) or [] + return f"""# {role} + +User-requested extra file harvesting. + +## Include patterns +{_markdown_list([str(p) for p in include_pats])} + +## Exclude patterns +{_markdown_list([str(p) for p in exclude_pats])} + +## Managed directories +{_markdown_list([str(d.get('path') or '') for d in managed_dirs])} + +## Managed files +{_markdown_list(_managed_file_lines(managed_files, include_reason=False))} + +## Excluded +{_markdown_list(_excluded_lines(excluded))} + +## Notes +{_markdown_list([str(n) for n in notes])} +""" diff --git a/enroll/ansible_renderer/roles/__init__.py b/enroll/ansible_renderer/roles/__init__.py new file mode 100644 index 0000000..93dc1a6 --- /dev/null +++ b/enroll/ansible_renderer/roles/__init__.py @@ -0,0 +1 @@ +"""Role writers for the Ansible renderer.""" diff --git a/enroll/ansible_renderer/roles/container_images.py b/enroll/ansible_renderer/roles/container_images.py new file mode 100644 index 0000000..099e85d --- /dev/null +++ b/enroll/ansible_renderer/roles/container_images.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import os +from typing import Any, Dict, List + +from ..context import AnsibleManifestContext +from ..layout import ( + _ensure_requirements_yaml, + _write_hostvars, + _write_role_defaults, + _write_role_scaffold, +) +from ..model import AnsibleManifestPlan +from ..vars import _normalise_container_image_item + +_CONTAINER_COLLECTIONS = [ + {"name": "community.docker", "version": ">=4.0.0"}, + {"name": "containers.podman", "version": ">=1.0.0"}, +] + + +def _render_container_images_role( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + container_images_snapshot: Dict[str, Any], +) -> None: + raw_images = container_images_snapshot.get("images", []) or [] + if not container_images_snapshot and not raw_images: + return + + images = [_normalise_container_image_item(img) for img in raw_images] + if not images and not (container_images_snapshot.get("notes") or []): + return + + role = container_images_snapshot.get("role_name", "container_images") + role_dir = os.path.join(ctx.roles_root, role) + _write_role_scaffold(role_dir) + _ensure_requirements_yaml( + os.path.join(ctx.out_dir, "requirements.yml"), _CONTAINER_COLLECTIONS + ) + + vars_map = {"container_images": images} + if ctx.site_mode: + _write_role_defaults(role_dir, {"container_images": []}) + _write_hostvars(ctx.out_dir, ctx.fqdn or "", role, vars_map) + else: + _write_role_defaults(role_dir, vars_map) + + with open(os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8") as f: + f.write( + "---\n" + "dependencies: []\n" + "collections:\n" + " - community.docker\n" + " - containers.podman\n" + ) + + tasks = """--- + +- name: Pull Docker images by immutable registry digest + community.docker.docker_image_pull: + name: "{{ item.pull_ref }}" + pull: not_present + platform: "{{ item.platform | default(omit, true) }}" + loop: "{{ container_images | default([]) | selectattr('engine', 'equalto', 'docker') | selectattr('pull_ref', 'defined') | list }}" + when: + - item.pull_ref | default('') | length > 0 + become: true + +- name: Tag Docker images with harvested tag aliases + community.docker.docker_image_tag: + name: "{{ item.0.pull_ref }}" + repository: + - "{{ item.1.ref }}" + loop: "{{ query('subelements', container_images | default([]) | selectattr('engine', 'equalto', 'docker') | selectattr('pull_ref', 'defined') | list, 'tag_aliases', {'skip_missing': True}) }}" + when: + - item.0.pull_ref | default('') | length > 0 + - item.1.repository | default('') | length > 0 + - item.1.tag | default('') | length > 0 + become: true + +- name: Pull system Podman images by immutable registry digest + containers.podman.podman_image: + name: "{{ item.pull_ref }}" + state: present + force: false + platform: "{{ item.platform | default(omit, true) }}" + loop: "{{ container_images | default([]) | selectattr('engine', 'equalto', 'podman') | rejectattr('scope', 'equalto', 'user') | selectattr('pull_ref', 'defined') | list }}" + when: + - item.pull_ref | default('') | length > 0 + become: true + +- name: Tag system Podman images with harvested tag aliases + containers.podman.podman_tag: + image: "{{ item.0.pull_ref }}" + target_names: + - "{{ item.1.ref }}" + loop: "{{ query('subelements', container_images | default([]) | selectattr('engine', 'equalto', 'podman') | rejectattr('scope', 'equalto', 'user') | selectattr('pull_ref', 'defined') | list, 'tag_aliases', {'skip_missing': True}) }}" + when: + - item.0.pull_ref | default('') | length > 0 + - item.1.ref | default('') | length > 0 + become: true + +- name: Pull user Podman images by immutable registry digest + containers.podman.podman_image: + name: "{{ item.pull_ref }}" + state: present + force: false + platform: "{{ item.platform | default(omit, true) }}" + loop: "{{ container_images | default([]) | selectattr('engine', 'equalto', 'podman') | selectattr('scope', 'equalto', 'user') | selectattr('pull_ref', 'defined') | list }}" + when: + - item.pull_ref | default('') | length > 0 + - item.user | default('') | length > 0 + become: true + become_user: "{{ item.user }}" + +- name: Tag user Podman images with harvested tag aliases + containers.podman.podman_tag: + image: "{{ item.0.pull_ref }}" + target_names: + - "{{ item.1.ref }}" + loop: "{{ query('subelements', container_images | default([]) | selectattr('engine', 'equalto', 'podman') | selectattr('scope', 'equalto', 'user') | selectattr('pull_ref', 'defined') | list, 'tag_aliases', {'skip_missing': True}) }}" + when: + - item.0.pull_ref | default('') | length > 0 + - item.0.user | default('') | length > 0 + - item.1.ref | default('') | length > 0 + become: true + become_user: "{{ item.0.user }}" +""" + with open(os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8") as f: + f.write(tasks) + + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write("---\n") + + def _fmt_image(img: Dict[str, Any]) -> str: + pull_ref = ( + img.get("pull_ref") or "(no registry digest; not rendered as an exact pull)" + ) + tags = img.get("repo_tags") or [] + tag_part = f" tags={', '.join(tags)}" if tags else "" + platform = img.get("platform") + platform_part = f" platform={platform}" if platform else "" + return f"- {img.get('engine', 'unknown')}: {pull_ref}{tag_part}{platform_part}" + + notes = list(container_images_snapshot.get("notes", []) or []) + unpinned_notes: List[str] = [] + for img in images: + if img.get("pull_ref"): + continue + label = ( + ", ".join(img.get("repo_tags") or []) + or img.get("image_id") + or "unknown image" + ) + unpinned_notes.append( + f"{label}: no RepoDigest was available, so no exact pull task is emitted." + ) + + readme = ( + """# container_images + +Generated Docker and Podman image-cache restoration role. + +Images are pulled by immutable registry digest, such as +`registry.example.net/app@sha256:...`, when the harvest found a usable +`RepoDigest`. Local image IDs are recorded in `state.json` for evidence but are +not registry pull references. + +**Note:** This role requires the `community.docker` and `containers.podman` +Ansible collections. Install them with: +`ansible-galaxy collection install -r requirements.yml`. + +Registry credentials are not harvested. Private-registry authentication must be +managed separately before this role runs. + +## Container images +""" + + "\n".join(_fmt_image(img) for img in images) + + """ + +## Notes +""" + + ("\n".join([f"- {n}" for n in notes + unpinned_notes]) or "- (none)") + + "\n" + ) + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("container_images", role) diff --git a/enroll/ansible_renderer/roles/desktop.py b/enroll/ansible_renderer/roles/desktop.py new file mode 100644 index 0000000..9a9c36f --- /dev/null +++ b/enroll/ansible_renderer/roles/desktop.py @@ -0,0 +1,308 @@ +from __future__ import annotations + +import os +from typing import Any, Dict, List + +from ..context import AnsibleManifestContext +from ..layout import ( + _ensure_requirements_yaml, + _write_hostvars, + _write_role_defaults, + _write_role_scaffold, +) +from ..model import AnsibleManifestPlan +from ..vars import ( + _normalise_flatpak_item, + _normalise_flatpak_remote, + _normalise_snap_item, +) + + +def _render_flatpak_role( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + flatpak_snapshot: Dict[str, Any], +) -> None: + out_dir = ctx.out_dir + roles_root = ctx.roles_root + fqdn = ctx.fqdn + site_mode = ctx.site_mode + + # ------------------------- + # Flatpak role (system-wide Flatpak remotes and applications) + # ------------------------- + raw_flatpak_apps = flatpak_snapshot.get("system_flatpaks", []) or [] + raw_flatpak_remotes = flatpak_snapshot.get("remotes", []) or [] + + if flatpak_snapshot: + role = flatpak_snapshot.get("role_name", "flatpak") + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + _ensure_requirements_yaml(os.path.join(out_dir, "requirements.yml")) + + flatpak_system_flatpaks = [ + _normalise_flatpak_item(fp, method="system") for fp in raw_flatpak_apps + ] + flatpak_remotes = [_normalise_flatpak_remote(r) for r in raw_flatpak_remotes] + + vars_map = { + "flatpak_system_flatpaks": flatpak_system_flatpaks, + "flatpak_remotes": flatpak_remotes, + } + if site_mode: + _write_role_defaults( + role_dir, + {"flatpak_system_flatpaks": [], "flatpak_remotes": []}, + ) + _write_hostvars(out_dir, fqdn or "", role, vars_map) + else: + _write_role_defaults(role_dir, vars_map) + + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write( + "---\n" "dependencies: []\n" "collections:\n" " - community.general\n" + ) + + tasks = """--- + +- name: Ensure system Flatpak remotes exist + ansible.builtin.command: + argv: + - flatpak + - remote-add + - --system + - --if-not-exists + - "{{ item.name }}" + - "{{ item.url }}" + loop: "{{ flatpak_remotes | default([]) }}" + when: + - item.name is defined + - item.url is defined + - item.url | length > 0 + become: true + changed_when: false + +- name: Install system-wide Flatpaks + community.general.flatpak: + name: + - "{{ item.name }}" + state: present + method: system + remote: "{{ item.remote | default(omit) }}" + from_url: "{{ item.from_url | default(omit) }}" + loop: "{{ flatpak_system_flatpaks | default([]) }}" + when: + - item.name is defined + - item.name | length > 0 + become: true +""" + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(tasks) + + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write("---\n") + + def _fmt_flatpak_apps(items: List[Dict[str, Any]]) -> str: + lines = [] + for item in items: + name = item.get("name") + if not name: + continue + detail_parts = [] + for key in ("remote", "branch", "arch"): + value = item.get(key) + if value not in (None, "", []): + detail_parts.append(f"{key}={value}") + details = f" ({', '.join(detail_parts)})" if detail_parts else "" + lines.append(f"- {name}{details}") + return "\n".join(lines) or "- (none)" + + def _fmt_flatpak_remotes(items: List[Dict[str, Any]]) -> str: + lines = [] + for item in items: + name = item.get("name") + url = item.get("url") + if not name or not url: + continue + lines.append(f"- {name}: {url}") + return "\n".join(lines) or "- (none)" + + notes = flatpak_snapshot.get("notes", []) or [] + readme = ( + """# flatpak + +Generated system-wide Flatpak remotes and applications. + +**Note:** This role requires the `community.general` Ansible collection. +Install it with: `ansible-galaxy collection install -r requirements.yml`. + +Flatpak `remote` is harvested from the installed deployment where detectable. +The original `.flatpakref` URL is generally not preserved by Flatpak after +installation, so `from_url` is only emitted if a future/hand-edited state file +contains it. + +## System Flatpak remotes +""" + + _fmt_flatpak_remotes(flatpak_remotes) + + """\n +## System-wide Flatpaks +""" + + _fmt_flatpak_apps(flatpak_system_flatpaks) + + """\n +## Notes +""" + + ("\n".join([f"- {n}" for n in notes]) or "- (none)") + + """\n""" + ) + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("flatpak", role) + + +def _render_snap_role( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + snap_snapshot: Dict[str, Any], +) -> None: + out_dir = ctx.out_dir + roles_root = ctx.roles_root + fqdn = ctx.fqdn + site_mode = ctx.site_mode + + # ------------------------- + # Snap role (system-wide snap packages) + # ------------------------- + raw_system_snaps = snap_snapshot.get("system_snaps", []) or [] + + if raw_system_snaps: + role = snap_snapshot.get("role_name", "snap") if snap_snapshot else "snap" + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + _ensure_requirements_yaml(os.path.join(out_dir, "requirements.yml")) + + snap_system_snaps = [_normalise_snap_item(s) for s in raw_system_snaps] + + vars_map = {"snap_system_snaps": snap_system_snaps} + if site_mode: + _write_role_defaults(role_dir, {"snap_system_snaps": []}) + _write_hostvars(out_dir, fqdn or "", role, vars_map) + else: + _write_role_defaults(role_dir, vars_map) + + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write( + "---\n" "dependencies: []\n" "collections:\n" " - community.general\n" + ) + + tasks = """--- + +- name: Install system-wide snaps with full detected attributes + community.general.snap: + name: + - "{{ item.name }}" + state: present + channel: "{{ item.channel | default(omit) if not (item.install_revision | default(false)) else omit }}" + revision: "{{ item.revision | default(omit) if (item.install_revision | default(false)) else omit }}" + classic: "{{ item.classic | default(false) }}" + devmode: "{{ item.devmode | default(false) }}" + dangerous: "{{ item.dangerous | default(false) }}" + loop: "{{ snap_system_snaps | default([]) }}" + when: + - item.name is defined + - item.name | length > 0 + become: true + register: _enroll_snap_full_results + ignore_errors: true + +- name: Install system-wide snaps with compatibility options + community.general.snap: + name: + - "{{ item.item.name }}" + state: present + channel: "{{ item.item.channel | default(omit) if not (item.item.install_revision | default(false)) else omit }}" + classic: "{{ item.item.classic | default(false) }}" + loop: "{{ (_enroll_snap_full_results | default({})).results | default([]) }}" + when: + - item.failed | default(false) + - item.item.name is defined + - item.item.name | length > 0 + become: true + register: _enroll_snap_compat_results + ignore_errors: true + +- name: Install system-wide snaps with minimal options + community.general.snap: + name: + - "{{ item.item.item.name }}" + state: present + loop: "{{ (_enroll_snap_compat_results | default({})).results | default([]) }}" + when: + - item.failed | default(false) + - item.item.item.name is defined + - item.item.item.name | length > 0 + become: true + ignore_errors: true +""" + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(tasks) + + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write("---\n") + + def _fmt_snap_apps(items: List[Dict[str, Any]]) -> str: + lines = [] + for item in items: + name = item.get("name") + if not name: + continue + detail_parts = [] + for key in ("channel", "revision"): + value = item.get(key) + if value not in (None, "", []): + detail_parts.append(f"{key}={value}") + for key in ("classic", "devmode", "dangerous"): + if item.get(key): + detail_parts.append(key) + details = f" ({', '.join(detail_parts)})" if detail_parts else "" + lines.append(f"- {name}{details}") + return "\n".join(lines) or "- (none)" + + notes = snap_snapshot.get("notes", []) or [] + readme = ( + """# snap + +Generated system-wide snap packages. + +**Note:** This role requires the `community.general` Ansible collection. +Install it with: `ansible-galaxy collection install -r requirements.yml`. + +The first install task uses all harvested attributes. If the installed +`community.general.snap` module is too old for some parameters, the generated +role falls back to reduced then minimal install tasks on a best-effort basis. + +## System-wide snaps +""" + + _fmt_snap_apps(snap_system_snaps) + + """\n +## Notes +""" + + ("\n".join([f"- {n}" for n in notes]) or "- (none)") + + """\n""" + ) + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("snap", role) diff --git a/enroll/ansible_renderer/roles/managed_files.py b/enroll/ansible_renderer/roles/managed_files.py new file mode 100644 index 0000000..f65297d --- /dev/null +++ b/enroll/ansible_renderer/roles/managed_files.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any, Callable, Dict, Optional, Tuple + +from ..context import AnsibleManifestContext +from ..jinjaturtle import _jinjify_managed_files +from ..layout import ( + _copy_artifacts, + _host_role_files_dir, + _write_hostvars, + _write_role_defaults, + _write_role_scaffold, +) +from ..model import AnsibleManifestPlan +from ..readme import ( + _apt_config_readme, + _dnf_config_readme, + _extra_paths_readme, + _simple_managed_files_readme, +) +from ..tasks import _render_generic_files_tasks +from ..vars import _build_managed_dirs_var, _build_managed_files_var +from ..yamlutil import _merge_mappings_overwrite, _yaml_load_mapping + + +@dataclass(frozen=True) +class AnsibleManagedFileRoleSpec: + """Declarative managed-file singleton role rendering spec. + + Puppet collects these singleton snapshots in a simple loop and feeds + each one through the same managed-content renderer. Ansible has more + layout concerns (defaults vs host_vars, optional JinjaTurtle templates, + handlers), but the resource intent is the same, so keep the per-role + differences in data rather than spelling out one branch per role. + """ + + key: str + default_role: str + category: str + readme_builder: Callable[..., str] + notify_systemd: Optional[str] = None + handlers: str = "---\n" + include_dirs_when_empty: bool = False + + +_SYSTEMD_DAEMON_RELOAD_HANDLER = """--- +- name: Run systemd daemon-reload + ansible.builtin.systemd: + daemon_reload: true + no_log: "{{ enroll_hide_systemd_status | default(true) | bool }}" +""" + + +MANAGED_FILE_ROLE_SPECS: Tuple[AnsibleManagedFileRoleSpec, ...] = ( + AnsibleManagedFileRoleSpec( + key="apt_config", + default_role="apt_config", + category="apt_config", + readme_builder=_apt_config_readme, + ), + AnsibleManagedFileRoleSpec( + key="dnf_config", + default_role="dnf_config", + category="dnf_config", + readme_builder=_dnf_config_readme, + ), + AnsibleManagedFileRoleSpec( + key="etc_custom", + default_role="etc_custom", + category="etc_custom", + notify_systemd="Run systemd daemon-reload", + handlers=_SYSTEMD_DAEMON_RELOAD_HANDLER, + readme_builder=_simple_managed_files_readme( + "etc_custom", + "Unowned /etc config files not attributed to packages or services.", + include_reason=False, + ), + ), + AnsibleManagedFileRoleSpec( + key="usr_local_custom", + default_role="usr_local_custom", + category="usr_local_custom", + readme_builder=_simple_managed_files_readme( + "usr_local_custom", + "Unowned /usr/local files (scripts in /usr/local/bin and config under /usr/local/etc).", + include_reason=False, + ), + ), + AnsibleManagedFileRoleSpec( + key="extra_paths", + default_role="extra_paths", + category="extra_paths", + readme_builder=_extra_paths_readme, + include_dirs_when_empty=True, + ), +) + + +def _managed_file_role_has_resources( + snapshot: Dict[str, Any], spec: AnsibleManagedFileRoleSpec +) -> bool: + if not snapshot: + return False + if snapshot.get("managed_files"): + return True + return bool(spec.include_dirs_when_empty and snapshot.get("managed_dirs")) + + +def _write_managed_files_role_from_spec( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + snapshot: Dict[str, Any], + spec: AnsibleManagedFileRoleSpec, +) -> None: + role = _write_managed_files_role( + snapshot=snapshot, + default_role=spec.default_role, + bundle_dir=ctx.bundle_dir, + roles_root=ctx.roles_root, + out_dir=ctx.out_dir, + fqdn=ctx.fqdn, + site_mode=ctx.site_mode, + jt_exe=ctx.jt_exe, + jt_enabled=ctx.jt_enabled, + notify_systemd=spec.notify_systemd, + handlers=spec.handlers, + readme_builder=spec.readme_builder, + ) + manifest_plan.add(spec.category, role) + + +def _write_managed_files_role( + *, + snapshot: Dict[str, Any], + default_role: str, + bundle_dir: str, + roles_root: str, + out_dir: str, + fqdn: Optional[str], + site_mode: bool, + jt_exe: Optional[str], + jt_enabled: bool, + notify_systemd: Optional[str], + handlers: str, + readme_builder: Callable[..., str], +) -> str: + """Render an Ansible role whose main purpose is managed files/dirs. + + This covers apt_config, dnf_config, etc_custom, usr_local_custom, and + extra_paths. Their harvested state shape is the same; only their README + and optional handler differ. + """ + + role = snapshot.get("role_name", default_role) + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + + var_prefix = role + managed_files = snapshot.get("managed_files", []) or [] + managed_dirs = snapshot.get("managed_dirs", []) or [] + excluded = snapshot.get("excluded", []) or [] + notes = snapshot.get("notes", []) or [] + + templated, jt_vars = _jinjify_managed_files( + bundle_dir, + role, + role_dir, + managed_files, + jt_exe=jt_exe, + jt_enabled=jt_enabled, + overwrite_templates=not site_mode, + ) + + if site_mode: + _copy_artifacts( + bundle_dir, + role, + _host_role_files_dir(out_dir, fqdn or "", role), + exclude_rels=templated, + ) + else: + _copy_artifacts( + bundle_dir, + role, + os.path.join(role_dir, "files"), + exclude_rels=templated, + ) + + files_var = _build_managed_files_var( + managed_files, + templated, + notify_other=None, + notify_systemd=notify_systemd, + ) + dirs_var = _build_managed_dirs_var(managed_dirs) + + jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} + vars_map: Dict[str, Any] = { + f"{var_prefix}_managed_files": files_var, + f"{var_prefix}_managed_dirs": dirs_var, + } + vars_map = _merge_mappings_overwrite(vars_map, jt_map) + + if site_mode: + _write_role_defaults( + role_dir, + {f"{var_prefix}_managed_files": [], f"{var_prefix}_managed_dirs": []}, + ) + _write_hostvars(out_dir, fqdn or "", role, vars_map) + else: + _write_role_defaults(role_dir, vars_map) + + tasks = "---\n" + _render_generic_files_tasks( + var_prefix, include_restart_notify=False + ) + with open(os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8") as f: + f.write(tasks.rstrip() + "\n") + + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(handlers.rstrip() + "\n") + + with open(os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8") as f: + f.write("---\ndependencies: []\n") + + readme = readme_builder( + bundle_dir=bundle_dir, + role=role, + snapshot=snapshot, + managed_files=managed_files, + managed_dirs=managed_dirs, + excluded=excluded, + notes=notes, + ) + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + return role + + +def _render_managed_file_roles( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + roles: Dict[str, Any], +) -> None: + """Render file-centric singleton roles in the same loop style as Puppet.""" + + for spec in MANAGED_FILE_ROLE_SPECS: + snapshot = roles.get(spec.key, {}) + if not isinstance(snapshot, dict): + continue + if not _managed_file_role_has_resources(snapshot, spec): + continue + _write_managed_files_role_from_spec(ctx, manifest_plan, snapshot, spec) diff --git a/enroll/ansible_renderer/roles/packages.py b/enroll/ansible_renderer/roles/packages.py new file mode 100644 index 0000000..739f8f1 --- /dev/null +++ b/enroll/ansible_renderer/roles/packages.py @@ -0,0 +1,601 @@ +from __future__ import annotations + +import os +from typing import Any, Dict, List, Set + +from ..context import AnsibleManifestContext +from ..jinjaturtle import _jinjify_managed_files +from ..layout import ( + _copy_artifacts, + _host_role_files_dir, + _write_hostvars, + _write_role_defaults, + _write_role_scaffold, +) +from ..model import AnsibleManifestPlan, AnsibleRole, _section_role_name +from ..tasks import ( + _render_generic_files_tasks, + _render_grouped_systemd_tasks, + _render_install_packages_tasks, +) +from ..vars import ( + _build_managed_dirs_var, + _build_managed_files_var, + _build_managed_links_var, +) +from ..yamlutil import _merge_mappings_overwrite, _yaml_load_mapping +from ...role_names import avoid_reserved_role_name + + +def _render_service_roles( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + services_to_manifest: List[Dict[str, Any]], +) -> None: + bundle_dir = ctx.bundle_dir + out_dir = ctx.out_dir + roles_root = ctx.roles_root + fqdn = ctx.fqdn + site_mode = ctx.site_mode + jt_exe = ctx.jt_exe + jt_enabled = ctx.jt_enabled + + # ------------------------- + # Service roles + # ------------------------- + for svc in services_to_manifest: + source_role = svc["role_name"] + role = avoid_reserved_role_name(source_role, prefix="service") + unit = svc["unit"] + pkgs = svc.get("packages", []) or [] + managed_files = svc.get("managed_files", []) or [] + managed_dirs = svc.get("managed_dirs", []) or [] + managed_links = svc.get("managed_links", []) or [] + + ansible_role = AnsibleRole(role) + ansible_role.add_service_snapshot(svc) + + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + + var_prefix = role + + unit_state = ansible_role.services.get(unit, {}) + enabled_at_harvest = bool(unit_state.get("enabled")) + desired_state = str(unit_state.get("state") or "stopped") + + templated, jt_vars = _jinjify_managed_files( + bundle_dir, + source_role, + role_dir, + managed_files, + jt_exe=jt_exe, + jt_enabled=jt_enabled, + overwrite_templates=not site_mode, + ) + + # Copy only the non-templated artifacts. + if site_mode: + _copy_artifacts( + bundle_dir, + source_role, + _host_role_files_dir(out_dir, fqdn or "", role), + exclude_rels=templated, + ) + else: + _copy_artifacts( + bundle_dir, + source_role, + os.path.join(role_dir, "files"), + exclude_rels=templated, + ) + + files_var = _build_managed_files_var( + managed_files, + templated, + notify_other="Restart service", + notify_systemd="Run systemd daemon-reload", + ) + + links_var = _build_managed_links_var(managed_links) + + dirs_var = _build_managed_dirs_var(managed_dirs) + + jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} + base_vars: Dict[str, Any] = { + f"{var_prefix}_unit_name": unit, + f"{var_prefix}_packages": pkgs, + f"{var_prefix}_managed_files": files_var, + f"{var_prefix}_managed_dirs": dirs_var, + f"{var_prefix}_managed_links": links_var, + f"{var_prefix}_manage_unit": True, + f"{var_prefix}_systemd_enabled": bool(enabled_at_harvest), + f"{var_prefix}_systemd_state": desired_state, + } + base_vars = _merge_mappings_overwrite(base_vars, jt_map) + + if site_mode: + # Role defaults are host-agnostic/safe; all harvested state is in host_vars. + _write_role_defaults( + role_dir, + { + f"{var_prefix}_unit_name": unit, + f"{var_prefix}_packages": [], + f"{var_prefix}_managed_files": [], + f"{var_prefix}_managed_dirs": [], + f"{var_prefix}_managed_links": [], + f"{var_prefix}_manage_unit": False, + f"{var_prefix}_systemd_enabled": False, + f"{var_prefix}_systemd_state": "stopped", + }, + ) + _write_hostvars(out_dir, fqdn or "", role, base_vars) + else: + _write_role_defaults(role_dir, base_vars) + + handlers = f"""--- +- name: Run systemd daemon-reload + ansible.builtin.systemd: + daemon_reload: true + no_log: "{{ enroll_hide_systemd_status | default(true) | bool }}" + +- name: Restart service + ansible.builtin.service: + name: "{{{{ {var_prefix}_unit_name }}}}" + state: restarted + when: + - {var_prefix}_manage_unit | default(false) + - ({var_prefix}_systemd_state | default('stopped')) == 'started' +""" + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(handlers) + + task_parts: List[str] = [] + task_parts.append("---\n" + _render_install_packages_tasks(role, var_prefix)) + + task_parts.append( + _render_generic_files_tasks(var_prefix, include_restart_notify=True) + ) + + task_parts.append( + f"""- name: Probe whether systemd unit exists and is manageable + ansible.builtin.systemd: + name: "{{{{ {var_prefix}_unit_name }}}}" + no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" + check_mode: true + register: _unit_probe + failed_when: false + changed_when: false + when: {var_prefix}_manage_unit | default(false) + +- name: Ensure unit enablement matches harvest + ansible.builtin.systemd: + name: "{{{{ {var_prefix}_unit_name }}}}" + enabled: "{{{{ {var_prefix}_systemd_enabled | bool }}}}" + no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" + when: + - {var_prefix}_manage_unit | default(false) + - _unit_probe is succeeded + +- name: Ensure unit running state matches harvest + ansible.builtin.systemd: + name: "{{{{ {var_prefix}_unit_name }}}}" + state: "{{{{ {var_prefix}_systemd_state }}}}" + no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" + when: + - {var_prefix}_manage_unit | default(false) + - _unit_probe is succeeded +""" + ) + + tasks = "\n".join(task_parts).rstrip() + "\n" + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(tasks) + + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write("---\ndependencies: []\n") + + excluded = svc.get("excluded", []) + notes = svc.get("notes", []) + readme = f"""# {role} + +Generated from `{unit}`. + +## Packages +{os.linesep.join("- " + p for p in pkgs) or "- (none detected)"} + +## Managed files +{os.linesep.join("- " + mf["path"] + " (" + mf["reason"] + ")" for mf in managed_files) or "- (none)"} + +## Managed symlinks +{os.linesep.join("- " + ml["path"] + " -> " + ml["target"] + " (" + ml.get("reason", "") + ")" for ml in managed_links) or "- (none)"} + +## Excluded (possible secrets / unsafe) +{os.linesep.join("- " + e["path"] + " (" + e["reason"] + ")" for e in excluded) or "- (none)"} + +## Notes +{os.linesep.join("- " + n for n in notes) or "- (none)"} +""" + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("service", role) + + +def _render_common_ansible_roles( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + common_role_groups: Dict[str, List[Dict[str, Any]]], + package_roles: List[Dict[str, Any]], +) -> List[str]: + bundle_dir = ctx.bundle_dir + roles_root = ctx.roles_root + jt_exe = ctx.jt_exe + jt_enabled = ctx.jt_enabled + + common_tail_roles: List[str] = [] + + # ------------------------- + # Common package section/group roles + # + # Outside --fqdn/site mode, package and systemd-unit roles are grouped by + # Debian Section or RPM Group by default. Managed config and unit state can + # live in those section roles too; --no-common-roles preserves the historic + # one-role-per-package/unit output, and --fqdn implies that mode because + # grouped role contents would be unsafe across multiple harvested hosts. + # ------------------------- + # ------------------------- + # Manually installed package roles + # ------------------------- + occupied_roles: Set[str] = set( + manifest_plan.roles("apt_config") + + manifest_plan.roles("dnf_config") + + manifest_plan.roles("users") + + manifest_plan.roles("flatpak") + + manifest_plan.roles("snap") + + manifest_plan.roles("service") + + manifest_plan.roles("firewall_runtime") + + manifest_plan.roles("sysctl") + + manifest_plan.roles("etc_custom") + + manifest_plan.roles("usr_local_custom") + + manifest_plan.roles("extra_paths") + ) + for pr in package_roles: + occupied_roles.add( + avoid_reserved_role_name(str(pr.get("role_name") or ""), prefix="package") + ) + + for section_label, entries in sorted(common_role_groups.items()): + role = _section_role_name(section_label, occupied_roles) + ansible_role = AnsibleRole( + role, + var_prefix=role, + section_label=section_label, + grouped=True, + ) + for entry in entries: + kind = entry.get("kind") or "package" + snap = entry.get("snapshot") or {} + if kind == "service": + ansible_role.add_service_snapshot(snap) + else: + ansible_role.add_package_snapshot(snap) + + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + + var_prefix = ansible_role.var_prefix + files_var: List[Dict[str, Any]] = [] + dirs_var: List[Dict[str, Any]] = [] + links_var: List[Dict[str, Any]] = [] + jt_combined: Dict[str, Any] = {} + + seen_files: Set[tuple] = set() + seen_dirs: Set[tuple] = set() + seen_links: Set[tuple] = set() + + for entry in ansible_role.entries: + kind = entry.get("kind") or "package" + snap = entry.get("snapshot") or {} + source_role = str(snap.get("role_name") or "") + managed_files = snap.get("managed_files", []) or [] + managed_dirs = snap.get("managed_dirs", []) or [] + managed_links = snap.get("managed_links", []) or [] + + templated: Set[str] = set() + jt_vars = "" + if managed_files and source_role: + templated, jt_vars = _jinjify_managed_files( + bundle_dir, + source_role, + role_dir, + managed_files, + jt_exe=jt_exe, + jt_enabled=jt_enabled, + overwrite_templates=True, + ) + + _copy_artifacts( + bundle_dir, + source_role, + os.path.join(role_dir, "files"), + exclude_rels=templated, + ) + + notify_other = "Restart managed services" if kind == "service" else None + for item in _build_managed_files_var( + managed_files, + templated, + notify_other=notify_other, + notify_systemd="Run systemd daemon-reload", + ): + key = (item.get("dest"), item.get("src_rel"), item.get("kind")) + if key not in seen_files: + seen_files.add(key) + files_var.append(item) + + for item in _build_managed_dirs_var(managed_dirs): + key = ( + item.get("dest"), + item.get("owner"), + item.get("group"), + item.get("mode"), + ) + if key not in seen_dirs: + seen_dirs.add(key) + dirs_var.append(item) + + for item in _build_managed_links_var(managed_links): + key = (item.get("dest"), item.get("src")) + if key not in seen_links: + seen_links.add(key) + links_var.append(item) + + jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} + jt_combined = _merge_mappings_overwrite(jt_combined, jt_map) + + packages = ansible_role.sorted_packages + files_var = sorted(files_var, key=lambda x: str(x.get("dest") or "")) + dirs_var = sorted(dirs_var, key=lambda x: str(x.get("dest") or "")) + links_var = sorted(links_var, key=lambda x: str(x.get("dest") or "")) + systemd_units = ansible_role.systemd_units_var + + base_vars: Dict[str, Any] = { + f"{var_prefix}_packages": packages, + f"{var_prefix}_managed_files": files_var, + f"{var_prefix}_managed_dirs": dirs_var, + f"{var_prefix}_managed_links": links_var, + f"{var_prefix}_systemd_units": systemd_units, + } + base_vars = _merge_mappings_overwrite(base_vars, jt_combined) + + _write_role_defaults(role_dir, base_vars) + + if {"cron", "logrotate"}.intersection(ansible_role.packages): + common_tail_roles.append(role) + + handlers = ( + """--- +- name: Run systemd daemon-reload + ansible.builtin.systemd: + daemon_reload: true + no_log: "{{ enroll_hide_systemd_status | default(true) | bool }}" + +- name: Restart managed services + ansible.builtin.service: + name: "{{ item.name }}" + state: restarted + loop: "{{ """ + + f"{var_prefix}_systemd_units" + + """ | default([]) }}" + when: + - item.manage | default(false) + - (item.state | default('stopped')) == 'started' +""" + ) + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(handlers) + + task_parts: List[str] = [] + task_parts.append("---\n" + _render_install_packages_tasks(role, var_prefix)) + task_parts.append( + _render_generic_files_tasks(var_prefix, include_restart_notify=True) + ) + task_parts.append(_render_grouped_systemd_tasks(var_prefix)) + + tasks = "\n".join(task_parts).rstrip() + "\n" + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(tasks) + + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write("---\ndependencies: []\n") + + readme = f"""# {role} + +Common role for package section/group `{section_label}`. + +## Origin roles +{os.linesep.join("- " + line for line in sorted(ansible_role.origin_lines)) or "- (none)"} + +## Packages +{os.linesep.join("- " + p for p in packages) or "- (none)"} + +## Managed files +{os.linesep.join("- " + mf["dest"] for mf in files_var) or "- (none)"} + +## Managed symlinks +{os.linesep.join("- " + ml["dest"] + " -> " + ml["src"] for ml in links_var) or "- (none)"} + +## Systemd units +{os.linesep.join("- " + u["name"] + " (enabled=" + str(u["enabled"]).lower() + ", state=" + u["state"] + ")" for u in systemd_units) or "- (none)"} + +## Excluded (possible secrets / unsafe) +{os.linesep.join("- " + e.get("path", "") + " (" + e.get("reason", "") + ")" for e in ansible_role.excluded) or "- (none)"} + +## Notes +{os.linesep.join("- " + n for n in ansible_role.notes) or "- (none)"} +""" + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("package", role) + + return common_tail_roles + + +def _render_package_roles( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + package_roles: List[Dict[str, Any]], +) -> None: + bundle_dir = ctx.bundle_dir + out_dir = ctx.out_dir + roles_root = ctx.roles_root + fqdn = ctx.fqdn + site_mode = ctx.site_mode + jt_exe = ctx.jt_exe + jt_enabled = ctx.jt_enabled + + # Process package roles (those with configuration files) + for pr in package_roles: + source_role = pr["role_name"] + role = avoid_reserved_role_name(source_role, prefix="package") + pkg = pr.get("package") or "" + managed_files = pr.get("managed_files", []) or [] + managed_dirs = pr.get("managed_dirs", []) or [] + managed_links = pr.get("managed_links", []) or [] + + ansible_role = AnsibleRole(role) + ansible_role.add_package_snapshot(pr) + + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + + var_prefix = role + + templated, jt_vars = _jinjify_managed_files( + bundle_dir, + source_role, + role_dir, + managed_files, + jt_exe=jt_exe, + jt_enabled=jt_enabled, + overwrite_templates=not site_mode, + ) + + # Copy only the non-templated artifacts. + if site_mode: + _copy_artifacts( + bundle_dir, + source_role, + _host_role_files_dir(out_dir, fqdn or "", role), + exclude_rels=templated, + ) + else: + _copy_artifacts( + bundle_dir, + source_role, + os.path.join(role_dir, "files"), + exclude_rels=templated, + ) + + pkgs = ansible_role.sorted_packages + + files_var = _build_managed_files_var( + managed_files, + templated, + notify_other=None, + notify_systemd="Run systemd daemon-reload", + ) + + links_var = _build_managed_links_var(managed_links) + + dirs_var = _build_managed_dirs_var(managed_dirs) + + jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {} + base_vars: Dict[str, Any] = { + f"{var_prefix}_packages": pkgs, + f"{var_prefix}_managed_files": files_var, + f"{var_prefix}_managed_dirs": dirs_var, + f"{var_prefix}_managed_links": links_var, + } + base_vars = _merge_mappings_overwrite(base_vars, jt_map) + + if site_mode: + _write_role_defaults( + role_dir, + { + f"{var_prefix}_packages": [], + f"{var_prefix}_managed_files": [], + f"{var_prefix}_managed_dirs": [], + f"{var_prefix}_managed_links": [], + }, + ) + _write_hostvars(out_dir, fqdn or "", role, base_vars) + else: + _write_role_defaults(role_dir, base_vars) + + handlers = """--- +- name: Run systemd daemon-reload + ansible.builtin.systemd: + daemon_reload: true + no_log: "{{ enroll_hide_systemd_status | default(true) | bool }}" +""" + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(handlers) + + task_parts: List[str] = [] + task_parts.append("---\n" + _render_install_packages_tasks(role, var_prefix)) + task_parts.append( + _render_generic_files_tasks(var_prefix, include_restart_notify=False) + ) + + tasks = "\n".join(task_parts).rstrip() + "\n" + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(tasks) + + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write("---\ndependencies: []\n") + + excluded = pr.get("excluded", []) + notes = pr.get("notes", []) + readme = f"""# {role} + +Generated for package `{pkg}`. + +## Managed files +{os.linesep.join("- " + mf["path"] + " (" + mf["reason"] + ")" for mf in managed_files) or "- (none)"} + +## Managed symlinks +{os.linesep.join("- " + ml["path"] + " -> " + ml["target"] + " (" + ml.get("reason", "") + ")" for ml in managed_links) or "- (none)"} + +## Excluded (possible secrets / unsafe) +{os.linesep.join("- " + e["path"] + " (" + e["reason"] + ")" for e in excluded) or "- (none)"} + +## Notes +{os.linesep.join("- " + n for n in notes) or "- (none)"} + +> Note: package roles (those not discovered via a systemd service) do not attempt to restart or enable services automatically. +""" + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("package", role) diff --git a/enroll/ansible_renderer/roles/runtime.py b/enroll/ansible_renderer/roles/runtime.py new file mode 100644 index 0000000..b71faa0 --- /dev/null +++ b/enroll/ansible_renderer/roles/runtime.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import os +from typing import Any, Dict + +from ..context import AnsibleManifestContext +from ..layout import ( + _copy_artifacts, + _host_role_files_dir, + _write_hostvars, + _write_role_defaults, + _write_role_scaffold, +) +from ..model import AnsibleManifestPlan +from ..tasks import ( + _render_firewall_runtime_tasks, + _render_install_packages_tasks, + _render_sysctl_handlers, + _render_sysctl_tasks, +) + + +def _render_sysctl_role( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + sysctl_snapshot: Dict[str, Any], +) -> None: + if not (sysctl_snapshot and (sysctl_snapshot.get("managed_files") or [])): + return + + role = sysctl_snapshot.get("role_name", "sysctl") + role_dir = os.path.join(ctx.roles_root, role) + _write_role_scaffold(role_dir) + + var_prefix = role + managed_files = sysctl_snapshot.get("managed_files", []) or [] + conf_src_rel = "" + for mf in managed_files: + if mf.get("path") == "/etc/sysctl.d/99-enroll.conf": + conf_src_rel = mf.get("src_rel") or "" + break + if not conf_src_rel and managed_files: + conf_src_rel = managed_files[0].get("src_rel") or "" + + parameters = sysctl_snapshot.get("parameters", {}) or {} + notes = sysctl_snapshot.get("notes", []) or [] + + if ctx.site_mode: + _copy_artifacts( + ctx.bundle_dir, + role, + _host_role_files_dir(ctx.out_dir, ctx.fqdn or "", role), + ) + else: + _copy_artifacts(ctx.bundle_dir, role, os.path.join(role_dir, "files")) + + vars_map: Dict[str, Any] = { + f"{var_prefix}_conf_src_rel": conf_src_rel, + f"{var_prefix}_apply": True, + f"{var_prefix}_ignore_apply_errors": True, + } + + if ctx.site_mode: + _write_role_defaults( + role_dir, + { + f"{var_prefix}_conf_src_rel": "", + f"{var_prefix}_apply": True, + f"{var_prefix}_ignore_apply_errors": True, + }, + ) + _write_hostvars(ctx.out_dir, ctx.fqdn or "", role, vars_map) + else: + _write_role_defaults(role_dir, vars_map) + + tasks = "---\n" + _render_sysctl_tasks(var_prefix) + with open(os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8") as f: + f.write(tasks.rstrip() + "\n") + + handlers_dir = os.path.join(role_dir, "handlers") + os.makedirs(handlers_dir, exist_ok=True) + with open(os.path.join(handlers_dir, "main.yml"), "w", encoding="utf-8") as f: + f.write(_render_sysctl_handlers(var_prefix)) + + with open(os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8") as f: + f.write("---\ndependencies: []\n") + + param_count = len(parameters) if isinstance(parameters, dict) else 0 + sample_params = [] + if isinstance(parameters, dict): + sample_params = sorted(parameters.keys())[:25] + + readme = f"""# {role} + +Generated from live writable sysctl state captured during harvest. + +This role deploys the captured values to `/etc/sysctl.d/99-enroll.conf`. The generated file intentionally contains writable, single-line sysctl values only; read-only, multiline, action-like, and host-identity keys are skipped to avoid creating a noisy or brittle boot-time configuration. + +## Captured parameters + +Captured parameter count: {param_count} + +{os.linesep.join("- " + x for x in sample_params) or "- (none)"} + +{"- ..." if param_count > len(sample_params) else ""} + +## Notes +{os.linesep.join("- " + n for n in notes) or "- (none)"} + +## Safety notes +- `sysctl_apply` defaults to `true` and applies `/etc/sysctl.d/99-enroll.conf` when the file changes. +- `sysctl_ignore_apply_errors` defaults to `true`, because sysctl availability and writability can vary across kernels, containers, and hardware. +- Review this role before applying it broadly across unlike hosts. +""" + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("sysctl", role) + + +def _render_firewall_runtime_role( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + firewall_runtime_snapshot: Dict[str, Any], +) -> None: + if not ( + firewall_runtime_snapshot + and ( + firewall_runtime_snapshot.get("ipset_save") + or firewall_runtime_snapshot.get("iptables_v4_save") + or firewall_runtime_snapshot.get("iptables_v6_save") + ) + ): + return + + role = firewall_runtime_snapshot.get("role_name", "firewall_runtime") + role_dir = os.path.join(ctx.roles_root, role) + _write_role_scaffold(role_dir) + + var_prefix = role + packages = firewall_runtime_snapshot.get("packages", []) or [] + ipset_save = firewall_runtime_snapshot.get("ipset_save") or "" + ipset_sets = firewall_runtime_snapshot.get("ipset_sets", []) or [] + iptables_v4_save = firewall_runtime_snapshot.get("iptables_v4_save") or "" + iptables_v6_save = firewall_runtime_snapshot.get("iptables_v6_save") or "" + notes = firewall_runtime_snapshot.get("notes", []) or [] + + if ctx.site_mode: + _copy_artifacts( + ctx.bundle_dir, + role, + _host_role_files_dir(ctx.out_dir, ctx.fqdn or "", role), + ) + else: + _copy_artifacts(ctx.bundle_dir, role, os.path.join(role_dir, "files")) + + vars_map: Dict[str, Any] = { + f"{var_prefix}_packages": packages, + f"{var_prefix}_ipset_save": ipset_save, + f"{var_prefix}_ipset_sets": ipset_sets, + f"{var_prefix}_iptables_v4_save": iptables_v4_save, + f"{var_prefix}_iptables_v6_save": iptables_v6_save, + f"{var_prefix}_sync_ipsets_exact": True, + f"{var_prefix}_restore_iptables": True, + } + + if ctx.site_mode: + _write_role_defaults( + role_dir, + { + f"{var_prefix}_packages": [], + f"{var_prefix}_ipset_save": "", + f"{var_prefix}_ipset_sets": [], + f"{var_prefix}_iptables_v4_save": "", + f"{var_prefix}_iptables_v6_save": "", + f"{var_prefix}_sync_ipsets_exact": True, + f"{var_prefix}_restore_iptables": True, + }, + ) + _write_hostvars(ctx.out_dir, ctx.fqdn or "", role, vars_map) + else: + _write_role_defaults(role_dir, vars_map) + + tasks = ( + "---\n" + + _render_install_packages_tasks(role, var_prefix) + + _render_firewall_runtime_tasks(var_prefix) + ) + with open(os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8") as f: + f.write(tasks.rstrip() + "\n") + + with open(os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8") as f: + f.write("---\ndependencies: []\n") + + readme = f"""# {role} + +Generated from live firewall runtime state captured during harvest. + +This role restores live ipset and iptables state only for firewall families where Enroll did not find corresponding persistent configuration on the source host. Static firewall configuration files, such as `/etc/iptables/rules.v4`, `/etc/iptables/rules.v6`, UFW, nftables, firewalld, or `/etc/ipset*`, are harvested separately as managed files and treated as authoritative for their respective family. + +## Captured snapshots +- ipset: {ipset_save or "(none)"} +- iptables IPv4: {iptables_v4_save or "(none)"} +- iptables IPv6: {iptables_v6_save or "(none)"} + +## Captured ipsets +{os.linesep.join("- " + x for x in ipset_sets) or "- (none)"} + +## Notes +{os.linesep.join("- " + n for n in notes) or "- (none)"} + +## Safety notes +- `firewall_runtime_sync_ipsets_exact` defaults to `true`; it flushes captured set members before replaying the saved members so stale entries are removed. This applies only when no persistent ipset config was found. +- `firewall_runtime_restore_iptables` defaults to `true`; `iptables-restore`/`ip6tables-restore` replace only captured families. A family is captured only when no corresponding persistent iptables config was found. +""" + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("firewall_runtime", role) diff --git a/enroll/ansible_renderer/roles/users.py b/enroll/ansible_renderer/roles/users.py new file mode 100644 index 0000000..99c1793 --- /dev/null +++ b/enroll/ansible_renderer/roles/users.py @@ -0,0 +1,434 @@ +from __future__ import annotations + +import os +from typing import Any, Dict, List + +from ..context import AnsibleManifestContext +from ..layout import ( + _copy_artifacts, + _ensure_requirements_yaml, + _host_role_files_dir, + _write_hostvars, + _write_role_defaults, + _write_role_scaffold, +) +from ..model import AnsibleManifestPlan +from ..vars import _normalise_flatpak_item, _normalise_flatpak_remote + + +def _render_users_role( + ctx: AnsibleManifestContext, + manifest_plan: AnsibleManifestPlan, + users_snapshot: Dict[str, Any], +) -> None: + bundle_dir = ctx.bundle_dir + out_dir = ctx.out_dir + roles_root = ctx.roles_root + fqdn = ctx.fqdn + site_mode = ctx.site_mode + + # ------------------------- + # Users role (non-system users) + # ------------------------- + if users_snapshot: + role = users_snapshot.get("role_name", "users") + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + + # Users role includes harvested SSH-related files; in site mode keep them + # host-specific to avoid cross-host clobber. + if site_mode: + _copy_artifacts( + bundle_dir, role, _host_role_files_dir(out_dir, fqdn or "", role) + ) + else: + _copy_artifacts(bundle_dir, role, os.path.join(role_dir, "files")) + + users = users_snapshot.get("users", []) + managed_files = users_snapshot.get("managed_files", []) + excluded = users_snapshot.get("excluded", []) + notes = users_snapshot.get("notes", []) + + # Build groups list and a simplified user dict list suitable for loops + group_names: List[str] = [] + group_set = set() + users_data: List[Dict[str, Any]] = [] + for u in users: + name = u.get("name") + if not name: + continue + pg = u.get("primary_group") or name + home = u.get("home") or f"/home/{name}" + sshdir = home.rstrip("/") + "/.ssh" + supp = u.get("supplementary_groups") or [] + if pg: + group_set.add(pg) + for g in supp: + if g: + group_set.add(g) + + users_data.append( + { + "name": name, + "uid": u.get("uid"), + "primary_group": pg, + "home": home, + "ssh_dir": sshdir, + "shell": u.get("shell"), + "gecos": u.get("gecos"), + "supplementary_groups": sorted(set(supp)), + } + ) + + group_names = sorted(group_set) + + # User-managed files (authorized_keys plus dangerous-mode shell dotfiles). + # Keep the variable name for compatibility with existing generated data. + ssh_files: List[Dict[str, Any]] = [] + for mf in managed_files: + dest = mf.get("path") or "" + src_rel = mf.get("src_rel") or "" + if not dest or not src_rel: + continue + + owner = "root" + group = "root" + for u in users_data: + home_prefix = (u.get("home") or "").rstrip("/") + "/" + if home_prefix and dest.startswith(home_prefix): + owner = str(u.get("name") or "root") + group = str(u.get("primary_group") or owner) + break + + # Prefer the harvested file mode so we preserve any deliberate + # permissions (e.g. 0600 for certain dotfiles). For authorized_keys, + # enforce 0600 regardless. + mode = mf.get("mode") or "0644" + if mf.get("reason") == "authorized_keys": + mode = "0600" + ssh_files.append( + { + "dest": dest, + "src_rel": src_rel, + "owner": owner, + "group": group, + "mode": mode, + } + ) + + # Only create .ssh directories for users that actually have harvested + # files under .ssh. This mirrors Puppet's behaviour and avoids creating + # empty SSH directories merely because a user account exists. + ssh_dirs_by_dest: Dict[str, Dict[str, Any]] = {} + for item in ssh_files: + dest = str(item.get("dest") or "") + if not dest: + continue + for user in users_data: + ssh_dir = str(user.get("ssh_dir") or "").rstrip("/") + if not ssh_dir or not dest.startswith(ssh_dir + "/"): + continue + ssh_dirs_by_dest.setdefault( + ssh_dir, + { + "dest": ssh_dir, + "owner": str(user.get("name") or item.get("owner") or "root"), + "group": str( + user.get("primary_group") or item.get("group") or "root" + ), + "mode": "0700", + }, + ) + break + ssh_dirs = sorted( + ssh_dirs_by_dest.values(), key=lambda item: str(item.get("dest") or "") + ) + + # Build Flatpak and Snap lists. Flatpak can be installed system-wide or + # per-user. Snap packages are system-wide; per-user ~/snap/* directories + # are runtime/user data and are not treated as install sources. + users_flatpaks: List[Dict[str, Any]] = [] + user_flatpak_map = users_snapshot.get("user_flatpaks", {}) or {} + home_by_user = { + str(u.get("name")): str(u.get("home") or "") for u in users_data + } + for uname, flatpaks in user_flatpak_map.items(): + for fp in flatpaks or []: + users_flatpaks.append( + _normalise_flatpak_item( + fp, + method="user", + user=str(uname), + home=home_by_user.get(str(uname)) or None, + ) + ) + + flatpak_remotes = [ + _normalise_flatpak_remote(r) + for r in (users_snapshot.get("user_flatpak_remotes", []) or []) + ] + users_needs_community = bool(flatpak_remotes or users_flatpaks) + if users_needs_community: + _ensure_requirements_yaml(os.path.join(out_dir, "requirements.yml")) + + # Variables are host-specific in site mode; in non-site mode they live in role defaults. + if site_mode: + _write_role_defaults( + role_dir, + { + "users_groups": [], + "users_users": [], + "users_ssh_dirs": [], + "users_ssh_files": [], + "users_flatpaks": [], + "users_flatpak_remotes": [], + }, + ) + _write_hostvars( + out_dir, + fqdn or "", + role, + { + "users_groups": group_names, + "users_users": users_data, + "users_ssh_dirs": ssh_dirs, + "users_ssh_files": ssh_files, + "users_flatpaks": users_flatpaks, + "users_flatpak_remotes": flatpak_remotes, + }, + ) + else: + _write_role_defaults( + role_dir, + { + "users_groups": group_names, + "users_users": users_data, + "users_ssh_dirs": ssh_dirs, + "users_ssh_files": ssh_files, + "users_flatpaks": users_flatpaks, + "users_flatpak_remotes": flatpak_remotes, + }, + ) + + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: + if users_needs_community: + f.write( + "---\n" + "dependencies: []\n" + "collections:\n" + " - community.general\n" + ) + else: + f.write("---\ndependencies: []\n") + + # tasks (data-driven) + users_tasks = """--- + +- name: Ensure groups exist + ansible.builtin.group: + name: "{{ item }}" + state: present + loop: "{{ users_groups | default([]) }}" + +- name: Ensure users exist + ansible.builtin.user: + name: "{{ item.name }}" + uid: "{{ item.uid | default(omit) }}" + group: "{{ item.primary_group }}" + home: "{{ item.home }}" + create_home: true + shell: "{{ item.shell | default(omit) }}" + comment: "{{ item.gecos | default(omit) }}" + state: present + loop: "{{ users_users | default([]) }}" + +- name: Ensure users supplementary groups + ansible.builtin.user: + name: "{{ item.name }}" + groups: "{{ item.supplementary_groups | default([]) | join(',') }}" + append: true + loop: "{{ users_users | default([]) }}" + when: (item.supplementary_groups | default([])) | length > 0 + +- name: Ensure .ssh directories exist for managed SSH files + ansible.builtin.file: + path: "{{ item.dest }}" + state: directory + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: "{{ item.mode }}" + loop: "{{ users_ssh_dirs | default([]) }}" + +- name: Deploy user-managed files + vars: + _enroll_ff: + files: + - "{{ inventory_dir }}/host_vars/{{ inventory_hostname }}/{{ role_name }}/.files/{{ item.src_rel }}" + - "{{ role_path }}/files/{{ item.src_rel }}" + ansible.builtin.copy: + src: "{{ lookup('ansible.builtin.first_found', _enroll_ff) }}" + dest: "{{ item.dest }}" + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: "{{ item.mode }}" + loop: "{{ users_ssh_files | default([]) }}" +""" + + if flatpak_remotes or users_flatpaks: + users_tasks += """ +- name: Ensure user Flatpak remotes exist + ansible.builtin.command: + argv: + - flatpak + - remote-add + - --user + - --if-not-exists + - "{{ item.name }}" + - "{{ item.url }}" + loop: "{{ users_flatpak_remotes | default([]) | selectattr('method', 'equalto', 'user') | list }}" + when: + - item.name is defined + - item.url is defined + - item.url | length > 0 + - item.user is defined + become: true + become_user: "{{ item.user }}" + environment: + HOME: "{{ item.home | default('/home/' ~ item.user, true) }}" + XDG_DATA_HOME: "{{ (item.home | default('/home/' ~ item.user, true)) ~ '/.local/share' }}" + changed_when: false + +- name: Install user Flatpaks + community.general.flatpak: + name: + - "{{ item.name }}" + state: present + method: user + remote: "{{ item.remote | default(omit) }}" + from_url: "{{ item.from_url | default(omit) }}" + loop: "{{ users_flatpaks | default([]) }}" + when: + - item.name is defined + - item.name | length > 0 + - item.user is defined + become: true + become_user: "{{ item.user }}" + environment: + HOME: "{{ item.home | default('/home/' ~ item.user, true) }}" + XDG_DATA_HOME: "{{ (item.home | default('/home/' ~ item.user, true)) ~ '/.local/share' }}" +""" + + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(users_tasks) + + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write("---\n") + + def _fmt_app_list(items: List[Dict[str, Any]]) -> str: + lines = [] + for item in items: + name = item.get("name") + if not name: + continue + detail_parts = [] + for key in ("remote", "channel", "revision", "branch", "arch"): + value = item.get(key) + if value not in (None, "", []): + detail_parts.append(f"{key}={value}") + for key in ("classic", "devmode", "dangerous"): + if item.get(key): + detail_parts.append(key) + details = f" ({', '.join(detail_parts)})" if detail_parts else "" + lines.append(f"- {name}{details}") + return "\n".join(lines) or "- (none)" + + def _fmt_user_flatpaks(items: List[Dict[str, Any]]) -> str: + lines = [] + for item in items: + name = item.get("name") + user = item.get("user") + if not name or not user: + continue + detail_parts = [] + for key in ("remote", "branch", "arch"): + value = item.get(key) + if value not in (None, "", []): + detail_parts.append(f"{key}={value}") + details = f" ({', '.join(detail_parts)})" if detail_parts else "" + lines.append(f"- {user}: {name}{details}") + return "\n".join(lines) or "- (none)" + + def _fmt_remotes(items: List[Dict[str, Any]]) -> str: + lines = [] + for item in items: + name = item.get("name") + url = item.get("url") + method = item.get("method") or "system" + user = item.get("user") + if not name or not url: + continue + owner = f"user={user}" if user else "system" + lines.append(f"- {name} ({method}, {owner}): {url}") + return "\n".join(lines) or "- (none)" + + readme = ( + """# users + +Generated non-system user accounts, SSH public material, and per-user Flatpak +applications/remotes. + +**Note:** User Flatpak tasks require the `community.general` Ansible collection. +Install it with: `ansible-galaxy collection install -r requirements.yml`. + +Flatpak `remote` is harvested from the installed deployment where detectable. +The original `.flatpakref` URL is generally not preserved by Flatpak after +installation, so `from_url` is only emitted if a future/hand-edited state file +contains it. + + +## Users +""" + + ( + "\n".join([f"- {u.get('name')} (uid {u.get('uid')})" for u in users]) + or "- (none)" + ) + + """\n +## Included SSH files +""" + + ( + "\n".join( + [f"- {mf.get('path')} ({mf.get('reason')})" for mf in managed_files] + ) + or "- (none)" + ) + + """\n +## Flatpak remotes +""" + + _fmt_remotes(flatpak_remotes) + + """\n +## User Flatpaks +""" + + _fmt_user_flatpaks(users_flatpaks) + + """\n +## Excluded +""" + + ( + "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) + or "- (none)" + ) + + """\n +## Notes +""" + + ("\n".join([f"- {n}" for n in notes]) or "- (none)") + + """\n""" + ) + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifest_plan.add("users", role) diff --git a/enroll/ansible_renderer/tasks.py b/enroll/ansible_renderer/tasks.py new file mode 100644 index 0000000..efdab0c --- /dev/null +++ b/enroll/ansible_renderer/tasks.py @@ -0,0 +1,290 @@ +from __future__ import annotations + + +def _render_generic_files_tasks( + var_prefix: str, *, include_restart_notify: bool +) -> str: + """Render generic tasks to deploy _managed_files safely.""" + # Using first_found makes roles work in both modes: + # - site-mode: inventory/host_vars///.files/... + # - non-site: roles//files/... + return f"""- name: Ensure managed directories exist (preserve owner/group/mode) + ansible.builtin.file: + path: "{{{{ item.dest }}}}" + state: directory + owner: "{{{{ item.owner }}}}" + group: "{{{{ item.group }}}}" + mode: "{{{{ item.mode }}}}" + loop: "{{{{ {var_prefix}_managed_dirs | default([]) }}}}" + +- name: Deploy any systemd unit files (templates) + ansible.builtin.template: + src: "{{{{ item.src_rel }}}}.j2" + dest: "{{{{ item.dest }}}}" + owner: "{{{{ item.owner }}}}" + group: "{{{{ item.group }}}}" + mode: "{{{{ item.mode }}}}" + loop: >- + {{{{ {var_prefix}_managed_files | default([]) + | selectattr('is_systemd_unit', 'equalto', true) + | selectattr('kind', 'equalto', 'template') + | list }}}} + notify: "{{{{ item.notify | default([]) }}}}" + +- name: Deploy any systemd unit files (raw files) + vars: + _enroll_ff: + files: + - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ item.src_rel }}}}" + - "{{{{ role_path }}}}/files/{{{{ item.src_rel }}}}" + ansible.builtin.copy: + src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" + dest: "{{{{ item.dest }}}}" + owner: "{{{{ item.owner }}}}" + group: "{{{{ item.group }}}}" + mode: "{{{{ item.mode }}}}" + loop: >- + {{{{ {var_prefix}_managed_files | default([]) + | selectattr('is_systemd_unit', 'equalto', true) + | selectattr('kind', 'equalto', 'copy') + | list }}}} + notify: "{{{{ item.notify | default([]) }}}}" + +- name: Reload systemd to pick up unit changes + ansible.builtin.meta: flush_handlers + when: >- + ({var_prefix}_managed_files | default([]) + | selectattr('is_systemd_unit', 'equalto', true) + | list + | length) > 0 + +- name: Deploy any other managed files (templates) + ansible.builtin.template: + src: "{{{{ item.src_rel }}}}.j2" + dest: "{{{{ item.dest }}}}" + owner: "{{{{ item.owner }}}}" + group: "{{{{ item.group }}}}" + mode: "{{{{ item.mode }}}}" + loop: >- + {{{{ {var_prefix}_managed_files | default([]) + | selectattr('is_systemd_unit', 'equalto', false) + | selectattr('kind', 'equalto', 'template') + | list }}}} + notify: "{{{{ item.notify | default([]) }}}}" + +- name: Deploy any other managed files (raw files) + vars: + _enroll_ff: + files: + - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ item.src_rel }}}}" + - "{{{{ role_path }}}}/files/{{{{ item.src_rel }}}}" + ansible.builtin.copy: + src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" + dest: "{{{{ item.dest }}}}" + owner: "{{{{ item.owner }}}}" + group: "{{{{ item.group }}}}" + mode: "{{{{ item.mode }}}}" + loop: >- + {{{{ {var_prefix}_managed_files | default([]) + | selectattr('is_systemd_unit', 'equalto', false) + | selectattr('kind', 'equalto', 'copy') + | list }}}} + notify: "{{{{ item.notify | default([]) }}}}" + +- name: Ensure managed symlinks exist + ansible.builtin.file: + src: "{{{{ item.src }}}}" + dest: "{{{{ item.dest }}}}" + state: link + force: true + loop: "{{{{ {var_prefix}_managed_links | default([]) }}}}" +""" + + +def _render_install_packages_tasks(role: str, var_prefix: str) -> str: + """Render package installation through Ansible's generic package provider. + + Puppet uses provider-backed package resources instead of selecting + apt/dnf/yum in the generated manifest. Ansible's package module is the + equivalent abstraction: it proxies to the target host's detected package + manager and keeps generated roles provider-neutral. + """ + + return f"""- name: Install packages for {role} + ansible.builtin.package: + name: "{{{{ {var_prefix}_packages | default([]) }}}}" + state: present + when: ({var_prefix}_packages | default([])) | length > 0 + +""" + + +def _render_grouped_systemd_tasks(var_prefix: str) -> str: + """Render tasks to manage multiple systemd units in a common role.""" + + return f"""- name: Probe whether grouped systemd units exist and are manageable + ansible.builtin.systemd: + name: "{{{{ item.name }}}}" + no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" + check_mode: true + loop: "{{{{ {var_prefix}_systemd_units | default([]) }}}}" + register: _enroll_unit_probes + failed_when: false + changed_when: false + when: item.manage | default(false) + +- name: Ensure grouped unit enablement matches harvest + ansible.builtin.systemd: + name: "{{{{ item.item.name }}}}" + enabled: "{{{{ item.item.enabled | bool }}}}" + no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" + loop: "{{{{ _enroll_unit_probes.results | default([]) }}}}" + when: + - item.item.manage | default(false) + - not (item.failed | default(false)) + +- name: Ensure grouped unit running state matches harvest + ansible.builtin.systemd: + name: "{{{{ item.item.name }}}}" + state: "{{{{ item.item.state }}}}" + no_log: "{{{{ enroll_hide_systemd_status | default(true) | bool }}}}" + loop: "{{{{ _enroll_unit_probes.results | default([]) }}}}" + when: + - item.item.manage | default(false) + - not (item.failed | default(false)) +""" + + +def _render_sysctl_tasks(var_prefix: str) -> str: + return f"""- name: Ensure sysctl.d exists + ansible.builtin.file: + path: /etc/sysctl.d + state: directory + owner: root + group: root + mode: "0755" + +- name: Deploy captured sysctl configuration + vars: + _enroll_ff: + files: + - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ {var_prefix}_conf_src_rel }}}}" + - "{{{{ role_path }}}}/files/{{{{ {var_prefix}_conf_src_rel }}}}" + ansible.builtin.copy: + src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" + dest: /etc/sysctl.d/99-enroll.conf + owner: root + group: root + mode: "0644" + when: ({var_prefix}_conf_src_rel | default('') | length) > 0 + notify: Apply captured sysctl configuration +""" + + +def _render_sysctl_handlers(var_prefix: str) -> str: + return f"""--- +- name: Apply captured sysctl configuration + ansible.builtin.command: + argv: + - sysctl + - -e + - -p + - /etc/sysctl.d/99-enroll.conf + register: _enroll_sysctl_apply + changed_when: false + failed_when: + - not ({var_prefix}_ignore_apply_errors | default(true) | bool) + - _enroll_sysctl_apply.rc != 0 + when: {var_prefix}_apply | default(true) | bool +""" + + +def _render_firewall_runtime_tasks(var_prefix: str) -> str: + """Render tasks for live ipset/iptables snapshots.""" + return f"""- name: Ensure firewall runtime snapshot directory exists + ansible.builtin.file: + path: /etc/enroll/firewall + state: directory + owner: root + group: root + mode: "0750" + +- name: Deploy captured ipset snapshot + vars: + _enroll_ff: + files: + - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ {var_prefix}_ipset_save }}}}" + - "{{{{ role_path }}}}/files/{{{{ {var_prefix}_ipset_save }}}}" + ansible.builtin.copy: + src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" + dest: /etc/enroll/firewall/ipset.save + owner: root + group: root + mode: "0600" + when: ({var_prefix}_ipset_save | default('') | length) > 0 + +- name: Flush captured ipsets before restoring members + ansible.builtin.command: + cmd: "ipset flush {{{{ item }}}}" + loop: "{{{{ {var_prefix}_ipset_sets | default([]) }}}}" + register: _enroll_ipset_flush + failed_when: false + changed_when: false + when: + - ({var_prefix}_ipset_save | default('') | length) > 0 + - {var_prefix}_sync_ipsets_exact | default(true) | bool + +- name: Restore captured ipsets + ansible.builtin.shell: "ipset restore -exist < /etc/enroll/firewall/ipset.save" + args: + executable: /bin/sh + register: _enroll_ipset_restore + changed_when: _enroll_ipset_restore.rc == 0 + when: ({var_prefix}_ipset_save | default('') | length) > 0 + +- name: Deploy captured IPv4 iptables snapshot + vars: + _enroll_ff: + files: + - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ {var_prefix}_iptables_v4_save }}}}" + - "{{{{ role_path }}}}/files/{{{{ {var_prefix}_iptables_v4_save }}}}" + ansible.builtin.copy: + src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" + dest: /etc/enroll/firewall/iptables.v4 + owner: root + group: root + mode: "0600" + when: ({var_prefix}_iptables_v4_save | default('') | length) > 0 + +- name: Restore captured IPv4 iptables rules + ansible.builtin.command: + cmd: iptables-restore /etc/enroll/firewall/iptables.v4 + register: _enroll_iptables_v4_restore + changed_when: _enroll_iptables_v4_restore.rc == 0 + when: + - ({var_prefix}_iptables_v4_save | default('') | length) > 0 + - {var_prefix}_restore_iptables | default(true) | bool + +- name: Deploy captured IPv6 iptables snapshot + vars: + _enroll_ff: + files: + - "{{{{ inventory_dir }}}}/host_vars/{{{{ inventory_hostname }}}}/{{{{ role_name }}}}/.files/{{{{ {var_prefix}_iptables_v6_save }}}}" + - "{{{{ role_path }}}}/files/{{{{ {var_prefix}_iptables_v6_save }}}}" + ansible.builtin.copy: + src: "{{{{ lookup('ansible.builtin.first_found', _enroll_ff) }}}}" + dest: /etc/enroll/firewall/iptables.v6 + owner: root + group: root + mode: "0600" + when: ({var_prefix}_iptables_v6_save | default('') | length) > 0 + +- name: Restore captured IPv6 iptables rules + ansible.builtin.command: + cmd: ip6tables-restore /etc/enroll/firewall/iptables.v6 + register: _enroll_iptables_v6_restore + changed_when: _enroll_iptables_v6_restore.rc == 0 + when: + - ({var_prefix}_iptables_v6_save | default('') | length) > 0 + - {var_prefix}_restore_iptables | default(true) | bool +""" diff --git a/enroll/ansible_renderer/vars.py b/enroll/ansible_renderer/vars.py new file mode 100644 index 0000000..1e1abaa --- /dev/null +++ b/enroll/ansible_renderer/vars.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Set + + +def _normalise_flatpak_item( + item: Any, + *, + method: str, + user: Optional[str] = None, + home: Optional[str] = None, +) -> Dict[str, Any]: + if isinstance(item, str): + out: Dict[str, Any] = {"name": item, "method": method} + elif isinstance(item, dict): + out = dict(item) + out.setdefault("method", method) + else: + out = {"name": str(item), "method": method} + if user: + out.setdefault("user", user) + if home: + out.setdefault("home", home) + return out + + +def _normalise_flatpak_remote(item: Any) -> Dict[str, Any]: + if isinstance(item, dict): + out = dict(item) + else: + out = {"name": str(item)} + out.setdefault("method", "system") + return out + + +def _normalise_snap_item(item: Any) -> Dict[str, Any]: + if isinstance(item, str): + out: Dict[str, Any] = {"name": item} + elif isinstance(item, dict): + out = dict(item) + else: + out = {"name": str(item)} + + notes = out.get("notes") or [] + if isinstance(notes, str): + notes = [notes] + notes_l = {str(n).lower() for n in notes} + out["classic"] = bool(out.get("classic") or "classic" in notes_l) + out["devmode"] = bool(out.get("devmode") or "devmode" in notes_l) + out["dangerous"] = bool(out.get("dangerous") or "dangerous" in notes_l) + + # The Ansible snap module's revision parameter pins/holds the snap. For + # ordinary store snaps that track a channel, preserve the channel instead + # of freezing every harvested host at today's revision. + if out.get("revision") is not None and not out.get("channel"): + out["install_revision"] = True + else: + out["install_revision"] = False + return out + + +def _build_managed_dirs_var( + managed_dirs: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Convert enroll managed_dirs into an Ansible-friendly list of dicts. + + Each dict drives a role task loop and is safe across hosts. + """ + out: List[Dict[str, Any]] = [] + for d in managed_dirs: + dest = d.get("path") or "" + if not dest: + continue + out.append( + { + "dest": dest, + "owner": d.get("owner") or "root", + "group": d.get("group") or "root", + "mode": d.get("mode") or "0755", + } + ) + return out + + +def _build_managed_files_var( + managed_files: List[Dict[str, Any]], + templated_src_rels: Set[str], + *, + notify_other: Optional[str] = None, + notify_systemd: Optional[str] = None, +) -> List[Dict[str, Any]]: + """Convert enroll managed_files into an Ansible-friendly list of dicts. + + Each dict drives a role task loop and is safe across hosts. + """ + out: List[Dict[str, Any]] = [] + for mf in managed_files: + dest = mf.get("path") or "" + src_rel = mf.get("src_rel") or "" + if not dest or not src_rel: + continue + is_unit = str(dest).startswith("/etc/systemd/system/") + kind = "template" if src_rel in templated_src_rels else "copy" + notify: List[str] = [] + if is_unit and notify_systemd: + notify.append(notify_systemd) + if (not is_unit) and notify_other: + notify.append(notify_other) + out.append( + { + "dest": dest, + "src_rel": src_rel, + "owner": mf.get("owner") or "root", + "group": mf.get("group") or "root", + "mode": mf.get("mode") or "0644", + "kind": kind, + "is_systemd_unit": bool(is_unit), + "notify": notify, + } + ) + return out + + +def _build_managed_links_var( + managed_links: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Convert enroll managed_links into an Ansible-friendly list of dicts.""" + out: List[Dict[str, Any]] = [] + for ml in managed_links or []: + dest = ml.get("path") or "" + src = ml.get("target") or "" + if not dest or not src: + continue + out.append({"dest": dest, "src": src}) + return out + + +def _normalise_container_image_item(item: Any) -> Dict[str, Any]: + if isinstance(item, dict): + out = dict(item) + else: + out = {"pull_ref": str(item)} + out.setdefault("engine", "docker") + out.setdefault("scope", "system") + out.setdefault("user", None) + out.setdefault("home", None) + out.setdefault("repo_tags", []) + out.setdefault("repo_digests", []) + out.setdefault("tag_aliases", []) + out.setdefault("notes", []) + return out diff --git a/enroll/ansible_renderer/yamlutil.py b/enroll/ansible_renderer/yamlutil.py new file mode 100644 index 0000000..c8ebbfb --- /dev/null +++ b/enroll/ansible_renderer/yamlutil.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from typing import Any, Dict, List + + +def _try_yaml(): + try: + import yaml # type: ignore + except Exception: + return None + return yaml + + +def _yaml_load_mapping(text: str) -> Dict[str, Any]: + yaml = _try_yaml() + if yaml is None: + return {} + try: + obj = yaml.safe_load(text) + except Exception: + return {} + if obj is None: + return {} + if isinstance(obj, dict): + return obj + return {} + + +def _yaml_dump_mapping(obj: Dict[str, Any], *, sort_keys: bool = True) -> str: + yaml = _try_yaml() + if yaml is None: + # fall back to a naive key: value dump (best-effort) + lines: List[str] = [] + for k, v in sorted(obj.items()) if sort_keys else obj.items(): + lines.append(f"{k}: {v!r}") + return "\n".join(lines).rstrip() + "\n" + + # ansible-lint/yamllint's indentation rules are stricter than YAML itself. + # In particular, they expect sequences nested under a mapping key to be + # indented (e.g. `foo:\n - a`), whereas PyYAML's default is often + # `foo:\n- a`. + class _IndentDumper(yaml.SafeDumper): # type: ignore + def increase_indent(self, flow: bool = False, indentless: bool = False): + return super().increase_indent(flow, False) + + return ( + yaml.dump( + obj, + Dumper=_IndentDumper, + default_flow_style=False, + sort_keys=sort_keys, + indent=2, + allow_unicode=True, + ).rstrip() + + "\n" + ) + + +def _merge_mappings_overwrite( + existing: Dict[str, Any], incoming: Dict[str, Any] +) -> Dict[str, Any]: + """Merge incoming into existing with overwrite. + + NOTE: Unlike role defaults merging, host_vars should reflect the current + harvest for a host. Therefore lists are replaced rather than unioned. + """ + merged = dict(existing) + merged.update(incoming) + return merged diff --git a/enroll/cache.py b/enroll/cache.py new file mode 100644 index 0000000..1dc656b --- /dev/null +++ b/enroll/cache.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import os +import re +import tempfile +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Optional + + +def _safe_component(s: str) -> str: + s = s.strip() + if not s: + return "unknown" + s = re.sub(r"[^A-Za-z0-9_.-]+", "_", s) + s = re.sub(r"_+", "_", s) + return s[:64] + + +def enroll_cache_dir() -> Path: + """Return the base cache directory for enroll. + + We default to ~/.local/cache to match common Linux conventions in personal + homedirs, but honour XDG_CACHE_HOME if set. + """ + base = os.environ.get("XDG_CACHE_HOME") + if base: + root = Path(base).expanduser() + else: + root = Path.home() / ".local" / "cache" + return root / "enroll" + + +@dataclass(frozen=True) +class HarvestCache: + """A locally-persistent directory that holds a harvested bundle.""" + + dir: Path + + @property + def state_json(self) -> Path: + return self.dir / "state.json" + + +def _ensure_dir_secure(path: Path) -> None: + """Create a directory with restrictive permissions; refuse symlinks.""" + # Refuse a symlink at the leaf. + if path.exists() and path.is_symlink(): + raise RuntimeError(f"Refusing to use symlink path: {path}") + path.mkdir(parents=True, exist_ok=True, mode=0o700) + try: + os.chmod(path, 0o700) + except OSError: + # Best-effort; on some FS types chmod may fail. + pass + + +def new_harvest_cache_dir(*, hint: Optional[str] = None) -> HarvestCache: + """Create a new, unpredictable harvest directory under the user's cache. + + This mitigates pre-guessing attacks (e.g. an attacker creating a directory + in advance in a shared temp location) by creating the bundle directory under + the user's home and using mkdtemp() randomness. + """ + base = enroll_cache_dir() / "harvest" + _ensure_dir_secure(base) + + ts = datetime.now().strftime("%Y%m%d-%H%M%S") + safe = _safe_component(hint or "harvest") + prefix = f"{ts}-{safe}-" + + # mkdtemp creates a new directory with a random suffix. + d = Path(tempfile.mkdtemp(prefix=prefix, dir=str(base))) + try: + os.chmod(d, 0o700) + except OSError: + pass + return HarvestCache(dir=d) diff --git a/enroll/capture.py b/enroll/capture.py new file mode 100644 index 0000000..24acc15 --- /dev/null +++ b/enroll/capture.py @@ -0,0 +1,275 @@ +from __future__ import annotations + +import os +import shutil +import stat +from typing import List, Optional, Set + +from .fsutil import stat_triplet +from .harvest_types import ExcludedFile, ManagedFile, ManagedLink +from .ignore import IgnorePolicy +from .pathfilter import PathFilter + + +def files_differ(a: str, b: str, *, max_bytes: int = 2_000_000) -> bool: + """Return True if file ``a`` differs from file ``b``. + + Best-effort and conservative: unreadable/missing baselines, non-regular + files, and unexpectedly large files are treated as different so callers err + on the side of preserving user state. + """ + + try: + st_a = os.stat(a, follow_symlinks=True) + except OSError: + return True + + if not stat.S_ISREG(st_a.st_mode): + return True + + try: + st_b = os.stat(b, follow_symlinks=True) + except OSError: + return True + + if not stat.S_ISREG(st_b.st_mode): + return True + + if st_a.st_size != st_b.st_size: + return True + + if st_a.st_size > max_bytes: + return True + + try: + with open(a, "rb") as fa, open(b, "rb") as fb: + while True: + ca = fa.read(1024 * 64) + cb = fb.read(1024 * 64) + if ca != cb: + return True + if not ca: + return False + except OSError: + return True + + +def copy_into_bundle( + bundle_dir: str, role_name: str, abs_path: str, src_rel: str +) -> None: + dst = os.path.join(bundle_dir, "artifacts", role_name, src_rel) + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy2(abs_path, dst) + + +def capture_file( + *, + bundle_dir: str, + role_name: str, + abs_path: str, + reason: str, + policy: IgnorePolicy, + path_filter: PathFilter, + managed_out: List[ManagedFile], + excluded_out: List[ExcludedFile], + seen_role: Optional[Set[str]] = None, + seen_global: Optional[Set[str]] = None, + metadata: Optional[tuple[str, str, str]] = None, +) -> bool: + """Try to capture a single file into the bundle. + + Returns True if the file was copied and appended to ``managed_out``. + ``seen_role`` de-duplicates within a role; ``seen_global`` de-duplicates + across harvest stages so multiple generated roles do not manage one path. + """ + + if seen_global is not None and abs_path in seen_global: + return False + if seen_role is not None and abs_path in seen_role: + return False + + def _mark_seen() -> None: + if seen_role is not None: + seen_role.add(abs_path) + if seen_global is not None: + seen_global.add(abs_path) + + if path_filter.is_excluded(abs_path): + excluded_out.append(ExcludedFile(path=abs_path, reason="user_excluded")) + _mark_seen() + return False + + deny = policy.deny_reason(abs_path) + if deny: + excluded_out.append(ExcludedFile(path=abs_path, reason=deny)) + _mark_seen() + return False + + try: + owner, group, mode = ( + metadata if metadata is not None else stat_triplet(abs_path) + ) + except OSError: + excluded_out.append(ExcludedFile(path=abs_path, reason="unreadable")) + _mark_seen() + return False + + src_rel = abs_path.lstrip("/") + try: + copy_into_bundle(bundle_dir, role_name, abs_path, src_rel) + except OSError: + excluded_out.append(ExcludedFile(path=abs_path, reason="unreadable")) + _mark_seen() + return False + + managed_out.append( + ManagedFile( + path=abs_path, + src_rel=src_rel, + owner=owner, + group=group, + mode=mode, + reason=reason, + ) + ) + _mark_seen() + return True + + +USER_SHELL_DOTFILES_WITH_SKEL_BASELINE = [ + (".bashrc", "user_shell_rc"), + (".profile", "user_profile"), + (".bash_logout", "user_shell_logout"), +] + +USER_SHELL_DOTFILES_WITHOUT_SKEL_BASELINE = [ + (".bash_aliases", "user_shell_aliases"), +] + + +def capture_user_shell_dotfiles( + *, + bundle_dir: str, + role_name: str, + home: str, + skel_dir: str, + enabled: bool, + policy: IgnorePolicy, + path_filter: PathFilter, + managed_out: List[ManagedFile], + excluded_out: List[ExcludedFile], + seen_role: Optional[Set[str]], + seen_global: Optional[Set[str]], +) -> int: + """Capture selected per-user shell dotfiles when explicitly enabled.""" + + if not enabled: + return 0 + + home = (home or "").rstrip("/") + if not home or not home.startswith("/"): + return 0 + + captured = 0 + max_compare_bytes = int(getattr(policy, "max_file_bytes", 256_000)) + + for rel, reason in USER_SHELL_DOTFILES_WITH_SKEL_BASELINE: + upath = os.path.join(home, rel) + if not os.path.isfile(upath) or os.path.islink(upath): + continue + skel_path = os.path.join(skel_dir, rel) + if not files_differ(upath, skel_path, max_bytes=max_compare_bytes): + continue + if capture_file( + bundle_dir=bundle_dir, + role_name=role_name, + abs_path=upath, + reason=reason, + policy=policy, + path_filter=path_filter, + managed_out=managed_out, + excluded_out=excluded_out, + seen_role=seen_role, + seen_global=seen_global, + ): + captured += 1 + + for rel, reason in USER_SHELL_DOTFILES_WITHOUT_SKEL_BASELINE: + upath = os.path.join(home, rel) + if not os.path.isfile(upath) or os.path.islink(upath): + continue + if capture_file( + bundle_dir=bundle_dir, + role_name=role_name, + abs_path=upath, + reason=reason, + policy=policy, + path_filter=path_filter, + managed_out=managed_out, + excluded_out=excluded_out, + seen_role=seen_role, + seen_global=seen_global, + ): + captured += 1 + + return captured + + +def capture_link( + *, + role_name: str, + abs_path: str, + reason: str, + policy: IgnorePolicy, + path_filter: PathFilter, + managed_out: List[ManagedLink], + excluded_out: List[ExcludedFile], + seen_role: Optional[Set[str]] = None, + seen_global: Optional[Set[str]] = None, +) -> bool: + """Record a symlink for later materialisation by the manifest renderer.""" + + if seen_global is not None and abs_path in seen_global: + return False + if seen_role is not None and abs_path in seen_role: + return False + + def _mark_seen() -> None: + if seen_role is not None: + seen_role.add(abs_path) + if seen_global is not None: + seen_global.add(abs_path) + + if path_filter.is_excluded(abs_path): + excluded_out.append(ExcludedFile(path=abs_path, reason="user_excluded")) + _mark_seen() + return False + + deny_link = getattr(policy, "deny_reason_link", None) + if callable(deny_link): + deny = deny_link(abs_path) + else: + deny = policy.deny_reason(abs_path) + if deny in ("not_regular_file", "not_file", "not_regular"): + deny = None + + if deny: + excluded_out.append(ExcludedFile(path=abs_path, reason=deny)) + _mark_seen() + return False + + if not os.path.islink(abs_path): + excluded_out.append(ExcludedFile(path=abs_path, reason="not_symlink")) + _mark_seen() + return False + + try: + target = os.readlink(abs_path) + except OSError: + excluded_out.append(ExcludedFile(path=abs_path, reason="unreadable")) + _mark_seen() + return False + + managed_out.append(ManagedLink(path=abs_path, target=target, reason=reason)) + _mark_seen() + return True diff --git a/enroll/cli.py b/enroll/cli.py index 530a388..3e70e2c 100644 --- a/enroll/cli.py +++ b/enroll/cli.py @@ -1,48 +1,1171 @@ from __future__ import annotations import argparse +import configparser +import json +import os +import sys +import tarfile +import tempfile +from pathlib import Path +from typing import Optional + +from .cache import new_harvest_cache_dir +from .diff import ( + compare_harvests, + enforce_old_harvest, + format_report, + has_enforceable_drift, + post_webhook, + send_email, +) +from .explain import explain_state from .harvest import harvest from .manifest import manifest +from .remote import ( + remote_harvest, + RemoteSudoPasswordRequired, + RemoteSSHKeyPassphraseRequired, +) +from .sopsutil import SopsError, encrypt_file_binary +from .validate import validate_harvest +from .version import get_enroll_version + + +def _discover_config_path(argv: list[str]) -> Optional[Path]: + """Return the config path to use, if any. + + Precedence: + 1) --no-config disables loading. + 2) --config PATH (or -c PATH) + 3) $ENROLL_CONFIG + 4) ./enroll.ini, ./.enroll.ini + 5) $XDG_CONFIG_HOME/enroll/enroll.ini (or ~/.config/enroll/enroll.ini) + + The config file is optional; if no file is found, returns None. + """ + + # Quick scan for explicit flags without needing to build the full parser. + if "--no-config" in argv: + return None + + def _value_after(flag: str) -> Optional[str]: + try: + i = argv.index(flag) + except ValueError: + return None + if i + 1 >= len(argv): + return None + return argv[i + 1] + + p = _value_after("--config") or _value_after("-c") + if p: + return Path(p).expanduser() + + envp = os.environ.get("ENROLL_CONFIG") + if envp: + return Path(envp).expanduser() + + cwd = Path.cwd() + for name in ("enroll.ini", ".enroll.ini"): + cp = cwd / name + if cp.exists() and cp.is_file(): + return cp + + xdg = os.environ.get("XDG_CONFIG_HOME") + if xdg: + base = Path(xdg).expanduser() + else: + base = Path.home() / ".config" + cp = base / "enroll" / "enroll.ini" + if cp.exists() and cp.is_file(): + return cp + + return None + + +def _parse_bool(s: str) -> Optional[bool]: + v = str(s).strip().lower() + if v in {"1", "true", "yes", "y", "on"}: + return True + if v in {"0", "false", "no", "n", "off"}: + return False + return None + + +def _action_lookup(p: argparse.ArgumentParser) -> dict[str, argparse.Action]: + """Map config keys -> argparse actions for a parser. + + Accepts both dest names and long option names without leading dashes, + normalized with '-' -> '_'. + """ + + m: dict[str, argparse.Action] = {} + for a in p._actions: # noqa: SLF001 (argparse internal) + if not getattr(a, "dest", None): + continue + dest = str(a.dest).strip().lower() + if dest: + m[dest] = a + for opt in getattr(a, "option_strings", []) or []: + k = opt.lstrip("-").strip().lower() + if k: + m[k.replace("-", "_")] = a + m[k] = a + return m + + +def _choose_flag(a: argparse.Action) -> Optional[str]: + # Prefer a long flag if available (e.g. --dangerous over -d) + for s in getattr(a, "option_strings", []) or []: + if s.startswith("--"): + return s + for s in getattr(a, "option_strings", []) or []: + return s + return None + + +def _split_list_value(v: str) -> list[str]: + # Support comma-separated and/or multi-line lists. + raw = str(v) + if "\n" in raw: + parts = [p.strip() for p in raw.splitlines()] + return [p for p in parts if p] + if "," in raw: + parts = [p.strip() for p in raw.split(",")] + return [p for p in parts if p] + raw = raw.strip() + return [raw] if raw else [] + + +def _section_to_argv( + p: argparse.ArgumentParser, cfg: configparser.ConfigParser, section: str +) -> list[str]: + """Translate an INI section into argv tokens for this parser.""" + if not cfg.has_section(section): + return [] + + lookup = _action_lookup(p) + out: list[str] = [] + + for k, v in cfg.items(section): + key = str(k).strip().lower().replace("-", "_") + # Avoid recursion / confusing self-configuration. + if key in {"config", "no_config"}: + continue + + a = lookup.get(key) + if not a: + # Unknown keys are ignored (but we try to be helpful). + print( + f"warning: config [{section}] contains unknown option '{k}' (ignored)", + file=sys.stderr, + ) + continue + + flag = _choose_flag(a) + if not flag: + continue + + # Boolean flags + if isinstance(a, argparse._StoreTrueAction): # noqa: SLF001 + b = _parse_bool(v) + if b is True: + out.append(flag) + continue + if isinstance(a, argparse._StoreFalseAction): # noqa: SLF001 + b = _parse_bool(v) + if b is False: + out.append(flag) + continue + + # Repeated options + if isinstance(a, argparse._AppendAction): # noqa: SLF001 + for item in _split_list_value(v): + out.extend([flag, item]) + continue + + # Count flags (rare, but easy to support) + if isinstance(a, argparse._CountAction): # noqa: SLF001 + b = _parse_bool(v) + if b is True: + out.append(flag) + else: + try: + n = int(str(v).strip()) + except ValueError: + n = 0 + out.extend([flag] * max(0, n)) + continue + + # Standard scalar options + sval = str(v).strip() + if sval: + out.extend([flag, sval]) + + return out + + +def _inject_config_argv( + argv: list[str], + *, + cfg_path: Optional[Path], + root_parser: argparse.ArgumentParser, + subparsers: dict[str, argparse.ArgumentParser], +) -> list[str]: + """Return argv with config-derived tokens inserted. + + We insert: + - [enroll] options before the subcommand + - [] options immediately after the subcommand token + + CLI flags always win because they come later in argv. + """ + + if not cfg_path: + return argv + cfg_path = Path(cfg_path).expanduser() + if not (cfg_path.exists() and cfg_path.is_file()): + return argv + + cfg = configparser.ConfigParser() + try: + cfg.read(cfg_path, encoding="utf-8") + except (OSError, configparser.Error) as e: + raise SystemExit(f"error: failed to read config file {cfg_path}: {e}") + + global_tokens = _section_to_argv(root_parser, cfg, "enroll") + + # Find the subcommand token position. + cmd_pos: Optional[int] = None + cmd_name: Optional[str] = None + for i, tok in enumerate(argv): + if tok in subparsers: + cmd_pos = i + cmd_name = tok + break + if cmd_pos is None or cmd_name is None: + # No subcommand found (argparse will handle the error); only apply global. + return global_tokens + argv + + cmd_tokens = _section_to_argv(subparsers[cmd_name], cfg, cmd_name) + # Also accept section names with '_' in place of '-' (e.g. [single_shot]) + if "-" in cmd_name: + alt = cmd_name.replace("-", "_") + if alt != cmd_name: + cmd_tokens += _section_to_argv(subparsers[cmd_name], cfg, alt) + + return global_tokens + argv[: cmd_pos + 1] + cmd_tokens + argv[cmd_pos + 1 :] + + +def _resolve_sops_out_file(out: Optional[str], *, hint: str) -> Path: + """Resolve an output *file* path for --sops mode. + + If `out` looks like a directory (or points to an existing directory), we + place the encrypted harvest inside it as harvest.tar.gz.sops. + """ + if out: + p = Path(out).expanduser() + if p.exists() and p.is_dir(): + return p / "harvest.tar.gz.sops" + # Heuristic: treat paths with a suffix as files; otherwise directories. + if p.suffix: + return p + return p / "harvest.tar.gz.sops" + + # Default: use a secure cache directory. + d = new_harvest_cache_dir(hint=hint).dir + return d / "harvest.tar.gz.sops" + + +def _tar_dir_to(path_dir: Path, tar_path: Path) -> None: + tar_path.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(tar_path, mode="w:gz") as tf: + # Keep a stable on-disk layout when extracted: state.json + artifacts/ + tf.add(str(path_dir), arcname=".") + + +def _encrypt_harvest_dir_to_sops( + bundle_dir: Path, out_file: Path, fps: list[str] +) -> Path: + out_file = Path(out_file) + out_file.parent.mkdir(parents=True, exist_ok=True) + + # Create the tarball alongside the output file (keeps filesystem permissions/locality sane). + fd, tmp_tgz = tempfile.mkstemp( + prefix=".enroll-harvest-", suffix=".tar.gz", dir=str(out_file.parent) + ) + os.close(fd) + try: + _tar_dir_to(bundle_dir, Path(tmp_tgz)) + encrypt_file_binary(Path(tmp_tgz), out_file, pgp_fingerprints=fps, mode=0o600) + finally: + try: + os.unlink(tmp_tgz) + except FileNotFoundError: + pass + return out_file + + +def _add_common_manifest_args(p: argparse.ArgumentParser) -> None: + p.add_argument( + "--target", + choices=["ansible", "puppet"], + default="ansible", + help="Manifest target to generate (default: ansible).", + ) + p.add_argument( + "--fqdn", + help="Host FQDN/name for site-mode output (creates target-specific host inventory/data such as Ansible host_vars or Puppet Hiera).", + ) + p.add_argument( + "--no-common-roles", + action="store_true", + help=( + "Do not group package and systemd-unit roles into common section/group roles. " + "This preserves one generated role per package/unit. --fqdn implies this." + ), + ) + g = p.add_mutually_exclusive_group() + g.add_argument( + "--jinjaturtle", + action="store_true", + help="Attempt jinjaturtle template integration (it will error if jinjaturtle is not found on PATH).", + ) + g.add_argument( + "--no-jinjaturtle", + action="store_true", + help="Do not use jinjaturtle integration, even if it is installed.", + ) + + +def _jt_mode(args: argparse.Namespace) -> str: + if getattr(args, "jinjaturtle", False): + return "on" + if getattr(args, "no_jinjaturtle", False): + return "off" + return "auto" + + +def _add_config_args(p: argparse.ArgumentParser) -> None: + p.add_argument( + "-c", + "--config", + help=( + "Path to an INI config file for default options. If omitted, enroll will look for " + "./enroll.ini, ./.enroll.ini, or ~/.config/enroll/enroll.ini (or $XDG_CONFIG_HOME/enroll/enroll.ini)." + ), + ) + p.add_argument( + "--no-config", + action="store_true", + help="Do not load any INI config file (even if one would be auto-discovered).", + ) + + +def _add_remote_args(p: argparse.ArgumentParser) -> None: + p.add_argument( + "--remote-host", + help="SSH host to run harvesting on (if set, harvest runs remotely and is pulled locally).", + ) + p.add_argument( + "--remote-ssh-config", + nargs="?", + const=str(Path.home() / ".ssh" / "config"), + default=None, + help=( + "Use OpenSSH-style ssh_config settings for --remote-host. " + "If provided without a value, defaults to ~/.ssh/config. " + "(Applies HostName/User/Port/IdentityFile/ProxyCommand/HostKeyAlias when supported.)" + ), + ) + p.add_argument( + "--remote-port", + type=int, + default=None, + help=( + "SSH port for --remote-host. If omitted, defaults to 22, or a value from ssh_config when " + "--remote-ssh-config is set." + ), + ) + p.add_argument( + "--remote-user", + default=None, + help=( + "SSH username for --remote-host. If omitted, defaults to local $USER, or a value from ssh_config when " + "--remote-ssh-config is set." + ), + ) + + # Align terminology with Ansible: "become" == sudo. + p.add_argument( + "--ask-become-pass", + "-K", + action="store_true", + help=( + "Prompt for the remote sudo (become) password when using --remote-host " + "(similar to ansible --ask-become-pass)." + ), + ) + + keyp = p.add_mutually_exclusive_group() + keyp.add_argument( + "--ask-key-passphrase", + action="store_true", + help=( + "Prompt for the SSH private key passphrase when using --remote-host. " + "If not set, enroll will still prompt on-demand if it detects an encrypted key in an interactive session." + ), + ) + keyp.add_argument( + "--ssh-key-passphrase-env", + metavar="ENV_VAR", + help=( + "Read the SSH private key passphrase from environment variable ENV_VAR " + "(useful for non-interactive runs/CI)." + ), + ) def main() -> None: ap = argparse.ArgumentParser(prog="enroll") + ap.add_argument( + "-v", + "--version", + action="version", + version=f"{get_enroll_version()}", + ) + _add_config_args(ap) sub = ap.add_subparsers(dest="cmd", required=True) h = sub.add_parser("harvest", help="Harvest service/package/config state") - h.add_argument("--out", required=True, help="Harvest output directory") + _add_config_args(h) + _add_remote_args(h) + h.add_argument( + "--out", + help=( + "Harvest output directory. If --sops is set, this may be either a directory " + "(an encrypted file named harvest.tar.gz.sops will be created inside) or a file path." + ), + ) + h.add_argument( + "--dangerous", + action="store_true", + help="Collect files more aggressively (may include secrets). Disables secret-avoidance checks.", + ) + h.add_argument( + "--include-path", + action="append", + default=[], + metavar="PATTERN", + help=( + "Include extra file paths to harvest (repeatable). Supports globs (including '**') and regex via 're:'. " + "Included files are still filtered by IgnorePolicy unless --dangerous is used." + ), + ) + h.add_argument( + "--exclude-path", + action="append", + default=[], + metavar="PATTERN", + help=( + "Exclude file paths from harvesting (repeatable). Supports globs (including '**') and regex via 're:'. " + "Excludes apply to all harvesting, including defaults." + ), + ) + h.add_argument( + "--sops", + nargs="+", + metavar="GPG_FINGERPRINT", + help=( + "Encrypt the harvest output as a SOPS-encrypted tarball using the given GPG fingerprint(s). " + "Requires `sops` on PATH." + ), + ) + h.add_argument( + "--no-sudo", + action="store_true", + help="Don't use sudo on the remote host (when using --remote options). This may result in a limited harvest due to permission restrictions.", + ) - r = sub.add_parser("manifest", help="Render Ansible roles from a harvest") - r.add_argument( + m = sub.add_parser( + "manifest", help="Render configuration-management code from a harvest" + ) + _add_config_args(m) + m.add_argument( "--harvest", required=True, - help="Path to the directory created by the harvest command", + help=( + "Path to the directory created by the harvest command, or (with --sops) " + "a SOPS-encrypted harvest tarball." + ), ) - r.add_argument( + m.add_argument( "--out", required=True, - help="Output directory for generated roles/playbook Ansible manifest", + help=( + "Output location for the generated manifest. In plain mode this is a directory. " + "In --sops mode this may be either a directory (an encrypted file named manifest.tar.gz.sops will be created inside) " + "or a file path." + ), ) + m.add_argument( + "--sops", + nargs="+", + metavar="GPG_FINGERPRINT", + help=( + "In --sops mode, decrypt the harvest using `sops -d` (if the harvest is an encrypted file) " + "and then bundle+encrypt the entire generated manifest output into a single SOPS-encrypted tarball " + "(binary) using the given GPG fingerprint(s). Requires `sops` on PATH." + ), + ) + _add_common_manifest_args(m) - e = sub.add_parser( - "enroll", help="Harvest state, then manifest Ansible code, in one shot" + s = sub.add_parser( + "single-shot", + help="Harvest state, then manifest configuration-management code, in one shot", ) - e.add_argument( - "--harvest", required=True, help="Path to the directory to place the harvest in" + _add_config_args(s) + _add_remote_args(s) + s.add_argument( + "--harvest", + help=( + "Where to place the harvest. In plain mode this is a directory; in --sops mode this may be " + "a directory or a file path (an encrypted file is produced)." + ), ) - e.add_argument( + s.add_argument( + "--dangerous", + action="store_true", + help="Collect files more aggressively (may include secrets). Disables secret-avoidance checks.", + ) + s.add_argument( + "--include-path", + action="append", + default=[], + metavar="PATTERN", + help=( + "Include extra file paths to harvest (repeatable). Supports globs (including '**') and regex via 're:'. " + "Included files are still filtered by IgnorePolicy unless --dangerous is used." + ), + ) + s.add_argument( + "--exclude-path", + action="append", + default=[], + metavar="PATTERN", + help=( + "Exclude file paths from harvesting (repeatable). Supports globs (including '**') and regex via 're:'. " + "Excludes apply to all harvesting, including defaults." + ), + ) + s.add_argument( + "--sops", + nargs="+", + metavar="GPG_FINGERPRINT", + help=( + "Encrypt the harvest as a SOPS-encrypted tarball, and bundle+encrypt the manifest output in --out " + "(same behaviour as `harvest --sops` and `manifest --sops`)." + ), + ) + s.add_argument( + "--no-sudo", + action="store_true", + help="Don't use sudo on the remote host (when using --remote options). This may result in a limited harvest due to permission restrictions.", + ) + s.add_argument( "--out", required=True, - help="Output directory for generated roles/playbook Ansible manifest", + help=( + "Output location for the generated manifest. In plain mode this is a directory. " + "In --sops mode this may be either a directory (an encrypted file named manifest.tar.gz.sops will be created inside) " + "or a file path." + ), + ) + _add_common_manifest_args(s) + + d = sub.add_parser("diff", help="Compare two harvests and report differences") + _add_config_args(d) + d.add_argument( + "--old", + required=True, + help=( + "Old/baseline harvest (directory, a path to state.json, a tarball, or a SOPS-encrypted bundle)." + ), + ) + d.add_argument( + "--new", + required=True, + help=( + "New/current harvest (directory, a path to state.json, a tarball, or a SOPS-encrypted bundle)." + ), + ) + d.add_argument( + "--sops", + action="store_true", + help="Allow SOPS-encrypted harvest bundle inputs (requires `sops` on PATH).", + ) + d.add_argument( + "--format", + choices=["text", "markdown", "json"], + default="text", + help="Report output format (default: text).", + ) + d.add_argument( + "--exclude-path", + action="append", + default=[], + metavar="PATTERN", + help=( + "Exclude file paths from the diff report (repeatable). Supports globs (including '**') and regex via 're:'. " + "This affects file drift reporting only (added/removed/changed files), not package/service/user diffs." + ), + ) + d.add_argument( + "--ignore-package-versions", + action="store_true", + help=( + "Ignore package version changes in the diff report and exit status. " + "Package additions/removals are still reported. Useful when routine upgrades would otherwise create noisy drift." + ), + ) + d.add_argument( + "--enforce", + action="store_true", + help=( + "If differences are detected, attempt to enforce the old harvest state locally by generating a manifest and " + "running ansible-playbook. Requires ansible-playbook on PATH. " + "Enroll does not attempt to downgrade packages; if the only drift is package version upgrades (or newly installed packages), enforcement is skipped." + ), + ) + d.add_argument( + "--out", + help="Write the report to this file instead of stdout.", + ) + d.add_argument( + "--exit-code", + action="store_true", + help="Exit with status 2 if differences are detected.", + ) + d.add_argument( + "--notify-always", + action="store_true", + help="Send webhook/email even when there are no differences.", + ) + d.add_argument( + "--webhook", + help="POST the report to this URL (only when differences are detected, unless --notify-always).", + ) + d.add_argument( + "--webhook-format", + choices=["json", "text", "markdown"], + default="json", + help="Payload format for --webhook (default: json).", + ) + d.add_argument( + "--webhook-header", + action="append", + default=[], + metavar="K:V", + help="Extra HTTP header for --webhook (repeatable), e.g. 'Authorization: Bearer ...'.", + ) + d.add_argument( + "--email-to", + action="append", + default=[], + help="Email the report to this address (repeatable; only when differences are detected unless --notify-always).", + ) + d.add_argument( + "--email-from", + help="From address for --email-to (default: enroll@).", + ) + d.add_argument( + "--email-subject", + help="Subject for --email-to (default: 'enroll diff report').", + ) + d.add_argument( + "--smtp", + help="SMTP server host[:port] for --email-to. If omitted, uses local sendmail.", + ) + d.add_argument( + "--smtp-user", + help="SMTP username (optional).", + ) + d.add_argument( + "--smtp-password-env", + help="Environment variable containing SMTP password (optional).", ) - args = ap.parse_args() + e = sub.add_parser("explain", help="Explain a harvest state.json") + _add_config_args(e) + e.add_argument( + "harvest", + help=( + "Harvest input (directory, a path to state.json, a tarball, or a SOPS-encrypted bundle)." + ), + ) + e.add_argument( + "--sops", + action="store_true", + help="Treat the input as a SOPS-encrypted bundle (auto-detected if the filename ends with .sops).", + ) + e.add_argument( + "--format", + choices=["text", "json"], + default="text", + help="Output format.", + ) + e.add_argument( + "--max-examples", + type=int, + default=3, + help="How many example paths/refs to show per reason.", + ) - if args.cmd == "harvest": - path = harvest(args.out) - print(path) - elif args.cmd == "manifest": - manifest(args.harvest, args.out) - elif args.cmd == "enroll": - harvest(args.harvest) - manifest(args.harvest, args.out) + v = sub.add_parser( + "validate", help="Validate a harvest bundle (state.json + artifacts)" + ) + _add_config_args(v) + v.add_argument( + "harvest", + help=( + "Harvest input (directory, a path to state.json, a tarball, or a SOPS-encrypted bundle)." + ), + ) + v.add_argument( + "--sops", + action="store_true", + help="Treat the input as a SOPS-encrypted bundle (auto-detected if the filename ends with .sops).", + ) + v.add_argument( + "--schema", + help=( + "Optional JSON schema source (file path or https:// URL). " + "If omitted, uses the schema vendored in the enroll codebase." + ), + ) + v.add_argument( + "--no-schema", + action="store_true", + help="Skip JSON schema validation and only perform bundle consistency checks.", + ) + v.add_argument( + "--fail-on-warnings", + action="store_true", + help="Exit non-zero if validation produces warnings.", + ) + v.add_argument( + "--format", + choices=["text", "json"], + default="text", + help="Output format.", + ) + v.add_argument( + "--out", + help="Write the report to this file instead of stdout.", + ) + + argv = sys.argv[1:] + cfg_path = _discover_config_path(argv) + argv = _inject_config_argv( + argv, + cfg_path=cfg_path, + root_parser=ap, + subparsers={ + "harvest": h, + "manifest": m, + "single-shot": s, + "diff": d, + "explain": e, + "validate": v, + }, + ) + args = ap.parse_args(argv) + + # Preserve historical defaults for remote harvesting unless ssh_config lookup is enabled. + # This lets ssh_config values take effect when the user did not explicitly set + # --remote-user / --remote-port. + if hasattr(args, "remote_host"): + rsc = getattr(args, "remote_ssh_config", None) + if not rsc: + if getattr(args, "remote_port", None) is None: + setattr(args, "remote_port", 22) + if getattr(args, "remote_user", None) is None: + setattr(args, "remote_user", os.environ.get("USER") or None) + + try: + if args.cmd == "harvest": + sops_fps = getattr(args, "sops", None) + if args.remote_host: + if sops_fps: + out_file = _resolve_sops_out_file(args.out, hint=args.remote_host) + with tempfile.TemporaryDirectory(prefix="enroll-harvest-") as td: + tmp_bundle = Path(td) / "bundle" + tmp_bundle.mkdir(parents=True, exist_ok=True) + try: + os.chmod(tmp_bundle, 0o700) + except OSError: + pass + remote_harvest( + ask_become_pass=args.ask_become_pass, + ask_key_passphrase=bool(args.ask_key_passphrase), + ssh_key_passphrase_env=getattr( + args, "ssh_key_passphrase_env", None + ), + local_out_dir=tmp_bundle, + remote_host=args.remote_host, + remote_port=args.remote_port, + remote_user=args.remote_user, + remote_ssh_config=args.remote_ssh_config, + dangerous=bool(args.dangerous), + no_sudo=bool(args.no_sudo), + include_paths=list(getattr(args, "include_path", []) or []), + exclude_paths=list(getattr(args, "exclude_path", []) or []), + ) + _encrypt_harvest_dir_to_sops( + tmp_bundle, out_file, list(sops_fps) + ) + print(str(out_file)) + else: + out_dir = ( + Path(args.out) + if args.out + else new_harvest_cache_dir(hint=args.remote_host).dir + ) + state = remote_harvest( + ask_become_pass=args.ask_become_pass, + ask_key_passphrase=bool(args.ask_key_passphrase), + ssh_key_passphrase_env=getattr( + args, "ssh_key_passphrase_env", None + ), + local_out_dir=out_dir, + remote_host=args.remote_host, + remote_port=args.remote_port, + remote_user=args.remote_user, + remote_ssh_config=args.remote_ssh_config, + dangerous=bool(args.dangerous), + no_sudo=bool(args.no_sudo), + include_paths=list(getattr(args, "include_path", []) or []), + exclude_paths=list(getattr(args, "exclude_path", []) or []), + ) + print(str(state)) + else: + if sops_fps: + out_file = _resolve_sops_out_file(args.out, hint="local") + with tempfile.TemporaryDirectory(prefix="enroll-harvest-") as td: + tmp_bundle = Path(td) / "bundle" + tmp_bundle.mkdir(parents=True, exist_ok=True) + try: + os.chmod(tmp_bundle, 0o700) + except OSError: + pass + harvest( + str(tmp_bundle), + dangerous=bool(args.dangerous), + include_paths=list(getattr(args, "include_path", []) or []), + exclude_paths=list(getattr(args, "exclude_path", []) or []), + ) + _encrypt_harvest_dir_to_sops( + tmp_bundle, out_file, list(sops_fps) + ) + print(str(out_file)) + else: + if args.out: + out_dir = args.out + else: + out_dir = ( + Path(args.out) + if args.out + else new_harvest_cache_dir(hint=args.remote_host).dir + ) + path = harvest( + out_dir, + dangerous=bool(args.dangerous), + include_paths=list(getattr(args, "include_path", []) or []), + exclude_paths=list(getattr(args, "exclude_path", []) or []), + ) + print(path) + elif args.cmd == "explain": + out = explain_state( + args.harvest, + sops_mode=bool(getattr(args, "sops", False)), + fmt=str(getattr(args, "format", "text")), + max_examples=int(getattr(args, "max_examples", 3)), + ) + sys.stdout.write(out) + + elif args.cmd == "validate": + res = validate_harvest( + args.harvest, + sops_mode=bool(getattr(args, "sops", False)), + schema=getattr(args, "schema", None), + no_schema=bool(getattr(args, "no_schema", False)), + ) + + fmt = str(getattr(args, "format", "text")) + if fmt == "json": + txt = json.dumps(res.to_dict(), indent=2, sort_keys=True) + "\n" + else: + txt = res.to_text() + + out_path = getattr(args, "out", None) + if out_path: + p = Path(out_path).expanduser() + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(txt, encoding="utf-8") + else: + sys.stdout.write(txt) + + if res.errors: + raise SystemExit(1) + if res.warnings and bool(getattr(args, "fail_on_warnings", False)): + raise SystemExit(1) + + elif args.cmd == "manifest": + out_enc = manifest( + args.harvest, + args.out, + fqdn=args.fqdn, + jinjaturtle=_jt_mode(args), + sops_fingerprints=getattr(args, "sops", None), + no_common_roles=bool(getattr(args, "no_common_roles", False)), + target=getattr(args, "target", "ansible"), + ) + if getattr(args, "sops", None) and out_enc: + print(str(out_enc)) + elif args.cmd == "diff": + report, has_changes = compare_harvests( + args.old, + args.new, + sops_mode=bool(getattr(args, "sops", False)), + exclude_paths=list(getattr(args, "exclude_path", []) or []), + ignore_package_versions=bool( + getattr(args, "ignore_package_versions", False) + ), + ) + + # Optional enforcement: if drift is detected, attempt to restore the + # system to the *old* (baseline) state using ansible-playbook. + if bool(getattr(args, "enforce", False)): + if has_changes: + if not has_enforceable_drift(report): + report["enforcement"] = { + "requested": True, + "status": "skipped", + "reason": ( + "no enforceable drift detected (only additions and/or package version changes); " + "enroll does not attempt to downgrade packages" + ), + } + else: + try: + info = enforce_old_harvest( + args.old, + sops_mode=bool(getattr(args, "sops", False)), + report=report, + ) + except Exception as e: + raise SystemExit( + f"error: could not enforce old harvest state: {e}" + ) from e + report["enforcement"] = { + "requested": True, + **(info or {}), + } + else: + report["enforcement"] = { + "requested": True, + "status": "skipped", + "reason": "no differences detected", + } + + txt = format_report(report, fmt=str(getattr(args, "format", "text"))) + out_path = getattr(args, "out", None) + if out_path: + p = Path(out_path).expanduser() + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(txt, encoding="utf-8") + else: + print(txt, end="" if txt.endswith("\n") else "\n") + + should_notify = has_changes or bool(getattr(args, "notify_always", False)) + + webhook = getattr(args, "webhook", None) + if webhook and should_notify: + wf = str(getattr(args, "webhook_format", "json")) + payload = format_report(report, fmt=wf) + body = payload.encode("utf-8") + headers = {} + if wf == "json": + headers["Content-Type"] = "application/json" + else: + headers["Content-Type"] = "text/plain; charset=utf-8" + for hv in getattr(args, "webhook_header", []) or []: + if ":" in hv: + k, v = hv.split(":", 1) + headers[k.strip()] = v.strip() + status, _resp = post_webhook(webhook, body, headers=headers) + if status and status >= 400: + raise SystemExit(f"error: webhook returned HTTP {status}") + + to_addrs = getattr(args, "email_to", []) or [] + if to_addrs and should_notify: + subject = getattr(args, "email_subject", None) or "enroll diff report" + smtp_pw = None + pw_env = getattr(args, "smtp_password_env", None) + if pw_env: + smtp_pw = os.environ.get(str(pw_env)) + send_email( + to_addrs=list(to_addrs), + subject=str(subject), + body=txt, + from_addr=getattr(args, "email_from", None), + smtp=getattr(args, "smtp", None), + smtp_user=getattr(args, "smtp_user", None), + smtp_password=smtp_pw, + ) + + if getattr(args, "exit_code", False) and has_changes: + raise SystemExit(2) + elif args.cmd == "single-shot": + sops_fps = getattr(args, "sops", None) + if args.remote_host: + if sops_fps: + out_file = _resolve_sops_out_file( + args.harvest, hint=args.remote_host + ) + with tempfile.TemporaryDirectory(prefix="enroll-harvest-") as td: + tmp_bundle = Path(td) / "bundle" + tmp_bundle.mkdir(parents=True, exist_ok=True) + try: + os.chmod(tmp_bundle, 0o700) + except OSError: + pass + remote_harvest( + ask_become_pass=args.ask_become_pass, + ask_key_passphrase=bool(args.ask_key_passphrase), + ssh_key_passphrase_env=getattr( + args, "ssh_key_passphrase_env", None + ), + local_out_dir=tmp_bundle, + remote_host=args.remote_host, + remote_port=args.remote_port, + remote_user=args.remote_user, + remote_ssh_config=args.remote_ssh_config, + dangerous=bool(args.dangerous), + no_sudo=bool(args.no_sudo), + include_paths=list(getattr(args, "include_path", []) or []), + exclude_paths=list(getattr(args, "exclude_path", []) or []), + ) + _encrypt_harvest_dir_to_sops( + tmp_bundle, out_file, list(sops_fps) + ) + + manifest( + str(out_file), + args.out, + fqdn=args.fqdn, + jinjaturtle=_jt_mode(args), + sops_fingerprints=list(sops_fps), + no_common_roles=bool(getattr(args, "no_common_roles", False)), + target=getattr(args, "target", "ansible"), + ) + if not args.harvest: + print(str(out_file)) + else: + harvest_dir = ( + Path(args.harvest) + if args.harvest + else new_harvest_cache_dir(hint=args.remote_host).dir + ) + remote_harvest( + ask_become_pass=args.ask_become_pass, + ask_key_passphrase=bool(args.ask_key_passphrase), + ssh_key_passphrase_env=getattr( + args, "ssh_key_passphrase_env", None + ), + local_out_dir=harvest_dir, + remote_host=args.remote_host, + remote_port=args.remote_port, + remote_user=args.remote_user, + remote_ssh_config=args.remote_ssh_config, + dangerous=bool(args.dangerous), + no_sudo=bool(args.no_sudo), + include_paths=list(getattr(args, "include_path", []) or []), + exclude_paths=list(getattr(args, "exclude_path", []) or []), + ) + manifest( + str(harvest_dir), + args.out, + fqdn=args.fqdn, + jinjaturtle=_jt_mode(args), + no_common_roles=bool(getattr(args, "no_common_roles", False)), + target=getattr(args, "target", "ansible"), + ) + # For usability (when --harvest wasn't provided), print the harvest path. + if not args.harvest: + print(str(harvest_dir / "state.json")) + else: + if sops_fps: + out_file = _resolve_sops_out_file(args.harvest, hint="local") + with tempfile.TemporaryDirectory(prefix="enroll-harvest-") as td: + tmp_bundle = Path(td) / "bundle" + tmp_bundle.mkdir(parents=True, exist_ok=True) + try: + os.chmod(tmp_bundle, 0o700) + except OSError: + pass + harvest( + str(tmp_bundle), + dangerous=bool(args.dangerous), + include_paths=list(getattr(args, "include_path", []) or []), + exclude_paths=list(getattr(args, "exclude_path", []) or []), + ) + _encrypt_harvest_dir_to_sops( + tmp_bundle, out_file, list(sops_fps) + ) + + manifest( + str(out_file), + args.out, + fqdn=args.fqdn, + jinjaturtle=_jt_mode(args), + sops_fingerprints=list(sops_fps), + no_common_roles=bool(getattr(args, "no_common_roles", False)), + target=getattr(args, "target", "ansible"), + ) + if not args.harvest: + print(str(out_file)) + else: + if not args.harvest: + raise SystemExit( + "error: --harvest is required unless --remote-host is set" + ) + harvest( + args.harvest, + dangerous=bool(args.dangerous), + include_paths=list(getattr(args, "include_path", []) or []), + exclude_paths=list(getattr(args, "exclude_path", []) or []), + ) + manifest( + args.harvest, + args.out, + fqdn=args.fqdn, + jinjaturtle=_jt_mode(args), + no_common_roles=bool(getattr(args, "no_common_roles", False)), + target=getattr(args, "target", "ansible"), + ) + except RemoteSudoPasswordRequired: + raise SystemExit( + "error: remote sudo requires a password. Re-run with --ask-become-pass." + ) from None + except RemoteSSHKeyPassphraseRequired as e: + msg = str(e).strip() or ( + "SSH private key passphrase is required. " + "Re-run with --ask-key-passphrase or --ssh-key-passphrase-env VAR." + ) + raise SystemExit(f"error: {msg}") from None + except RuntimeError as e: + raise SystemExit(f"error: {e}") from None + except SopsError as e: + raise SystemExit(f"error: {e}") from None diff --git a/enroll/cm.py b/enroll/cm.py new file mode 100644 index 0000000..b843ee1 --- /dev/null +++ b/enroll/cm.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, Iterable, Iterator, List, Mapping, Set + +from .state import load_state, state_path, write_state + + +@dataclass +class CMModule: + """Renderer-neutral configuration-management resource group. + + A CMModule is intentionally small: it captures the resources that a target + renderer can turn into Ansible tasks, Puppet resources, etc. + The renderer may still decide how to name/include/order the group. + """ + + role_name: str + module_name: str + packages: Set[str] = field(default_factory=set) + groups: Set[str] = field(default_factory=set) + users: Dict[str, Dict[str, Any]] = field(default_factory=dict) + dirs: Dict[str, Dict[str, Any]] = field(default_factory=dict) + files: Dict[str, Dict[str, Any]] = field(default_factory=dict) + links: Dict[str, Dict[str, Any]] = field(default_factory=dict) + services: Dict[str, Dict[str, Any]] = field(default_factory=dict) + notes: List[str] = field(default_factory=list) + + def has_resources(self) -> bool: + return bool( + self.packages + or self.groups + or self.users + or self.dirs + or self.files + or self.links + or self.services + or self.notes + ) + + @staticmethod + def state_path(bundle_dir: str | Path) -> Path: + """Return the canonical state.json path for a harvest bundle.""" + + return state_path(bundle_dir) + + @classmethod + def load_state(cls, bundle_dir: str | Path) -> Dict[str, Any]: + """Load state.json for a renderer using the shared bundle state loader.""" + + return load_state(bundle_dir) + + @classmethod + def _load_state(cls, bundle_dir: str | Path) -> Dict[str, Any]: + """Backward-compatible alias for renderer subclasses.""" + + return cls.load_state(bundle_dir) + + @classmethod + def write_state( + cls, + bundle_dir: str | Path, + state: Mapping[str, Any], + *, + indent: int = 2, + sort_keys: bool = True, + ) -> Path: + """Write state.json using the shared bundle state writer.""" + + return write_state(bundle_dir, state, indent=indent, sort_keys=sort_keys) + + @staticmethod + def _snapshot_items(snap: Dict[str, Any], key: str) -> Iterator[Dict[str, Any]]: + values = snap.get(key) or [] + if not isinstance(values, list): + return + for item in values: + if isinstance(item, dict): + yield item + + @classmethod + def managed_dirs_from_snapshot( + cls, snap: Dict[str, Any] + ) -> Iterator[Dict[str, Any]]: + return cls._snapshot_items(snap, "managed_dirs") + + @classmethod + def managed_files_from_snapshot( + cls, snap: Dict[str, Any] + ) -> Iterator[Dict[str, Any]]: + return cls._snapshot_items(snap, "managed_files") + + @classmethod + def managed_links_from_snapshot( + cls, snap: Dict[str, Any] + ) -> Iterator[Dict[str, Any]]: + return cls._snapshot_items(snap, "managed_links") + + def add_managed_dir( + self, + path: str, + *, + owner: Any = "root", + group: Any = "root", + mode: Any = "0755", + **attrs: Any, + ) -> None: + if not path: + return + data: Dict[str, Any] = { + "owner": owner or "root", + "group": group or "root", + "mode": mode or "0755", + } + data.update(attrs) + self.dirs.setdefault(path, data) + + def add_managed_file( + self, + path: str, + *, + owner: Any = "root", + group: Any = "root", + mode: Any = "0644", + **attrs: Any, + ) -> None: + if not path: + return + data: Dict[str, Any] = { + "owner": owner or "root", + "group": group or "root", + "mode": mode or "0644", + } + data.update(attrs) + self.files.setdefault(path, data) + + def add_managed_link(self, path: str, **attrs: Any) -> None: + if path: + self.links.setdefault(path, attrs) + + def add_snapshot_notes(self, snap: Dict[str, Any]) -> None: + self.notes.extend(str(n) for n in (snap.get("notes", []) or [])) + + def remove_directory_resource_conflicts(self) -> None: + for path in set(self.files) | set(self.links): + self.dirs.pop(path, None) + + +def package_section_label( + package_role: Dict[str, Any], inventory_packages: Dict[str, Any] +) -> str: + """Return the Debian Section/RPM Group label for a package role.""" + + pkg = str(package_role.get("package") or "").strip() + inv = inventory_packages.get(pkg) or {} + candidates: List[str] = [] + + for value in (package_role.get("section"), inv.get("section"), inv.get("group")): + if isinstance(value, str) and value.strip(): + candidates.append(value.strip()) + + for inst in inv.get("installations", []) or []: + if not isinstance(inst, dict): + continue + for key in ("section", "group"): + value = inst.get(key) + if isinstance(value, str) and value.strip(): + candidates.append(value.strip()) + + for value in candidates: + if value.lower() not in {"(none)", "none", "unspecified"}: + return value + return "misc" + + +def section_label_for_packages( + packages: List[str], inventory_packages: Dict[str, Any] +) -> str: + """Return a stable section/group label for a set of packages.""" + + for pkg in packages or []: + label = package_section_label({"package": pkg}, inventory_packages) + if label and label.lower() != "misc": + return label + return "misc" + + +def role_order_key(role: str) -> tuple[int, str]: + # Keep broadly similar ordering to generated Ansible playbooks: package/config + # scaffolding first, then services/users, then host-specific runtime state. + priority = { + "apt_config": 10, + "dnf_config": 11, + "etc_custom": 80, + "usr_local_custom": 81, + "extra_paths": 82, + "container_images": 88, + "users": 90, + "sysctl": 95, + "firewall_runtime": 99, + } + return (priority.get(role, 50), role) + + +def _drop_duplicate_set_items( + module: CMModule, + values: Set[str], + seen: Set[str], + resource_type: str, +) -> Set[str]: + kept: Set[str] = set() + for value in sorted(values): + if value in seen: + module.notes.append( + f"Skipped duplicate {resource_type}[{value}] already emitted earlier in this catalog." + ) + continue + kept.add(value) + seen.add(value) + return kept + + +def _drop_duplicate_mapping_items( + module: CMModule, + values: Dict[str, Dict[str, Any]], + seen: Set[str], + resource_type: str, + *, + excluded_titles: Set[str] | None = None, + excluded_reason: str = "conflicts with another resource", +) -> Dict[str, Dict[str, Any]]: + kept: Dict[str, Dict[str, Any]] = {} + excluded_titles = excluded_titles or set() + for title, attrs in values.items(): + if title in excluded_titles: + module.notes.append(f"Skipped {resource_type}[{title}]: {excluded_reason}.") + continue + if title in seen: + module.notes.append( + f"Skipped duplicate {resource_type}[{title}] already emitted earlier in this catalog." + ) + continue + kept[title] = attrs + seen.add(title) + return kept + + +def resolve_catalog_conflicts(modules: Iterable[CMModule]) -> None: + """Resolve global catalog conflicts before renderer output. + + Puppet compiles a single resource catalog. Ansible can tolerate the same + package, service, or parent directory appearing in more than one role; + catalog targets cannot. Resolve those conflicts in the shared model rather + than deleting renderer output after the fact. + """ + + ordered = list(modules) + concrete_file_paths: Set[str] = set() + for module in ordered: + concrete_file_paths.update(module.files) + concrete_file_paths.update(module.links) + + seen_packages: Set[str] = set() + seen_groups: Set[str] = set() + seen_users: Set[str] = set() + seen_dirs: Set[str] = set() + seen_files: Set[str] = set() + seen_links: Set[str] = set() + seen_services: Set[str] = set() + + for module in ordered: + module.packages = _drop_duplicate_set_items( + module, module.packages, seen_packages, "Package" + ) + module.groups = _drop_duplicate_set_items( + module, module.groups, seen_groups, "Group" + ) + module.users = _drop_duplicate_mapping_items( + module, module.users, seen_users, "User" + ) + module.dirs = _drop_duplicate_mapping_items( + module, + module.dirs, + seen_dirs, + "File", + excluded_titles=concrete_file_paths, + excluded_reason="a file or link with the same path is emitted in this catalog", + ) + module.files = _drop_duplicate_mapping_items( + module, module.files, seen_files | seen_links, "File" + ) + seen_files.update(module.files) + module.links = _drop_duplicate_mapping_items( + module, module.links, seen_links | seen_files, "File" + ) + seen_links.update(module.links) + module.services = _drop_duplicate_mapping_items( + module, module.services, seen_services, "Service" + ) diff --git a/enroll/debian.py b/enroll/debian.py index 58569e5..6d0ca06 100644 --- a/enroll/debian.py +++ b/enroll/debian.py @@ -1,7 +1,6 @@ from __future__ import annotations import glob -import hashlib import os import subprocess # nosec from typing import Dict, List, Optional, Set, Tuple @@ -64,6 +63,53 @@ def list_manual_packages() -> List[str]: return sorted(set(pkgs)) +def list_installed_packages() -> Dict[str, List[Dict[str, str]]]: + """Return mapping of installed package name -> installed instances. + + Uses dpkg-query and is expected to work on Debian/Ubuntu-like systems. + + Output format: + {"pkg": [{"version": "...", "arch": "...", "section": "..."}, ...], ...} + """ + + try: + p = subprocess.run( + [ + "dpkg-query", + "-W", + "-f=${Package}\t${Version}\t${Architecture}\t${Section}\n", + ], + text=True, + capture_output=True, + check=False, + ) # nosec + except Exception: + return {} + + out: Dict[str, List[Dict[str, str]]] = {} + for raw in (p.stdout or "").splitlines(): + line = raw.strip("\n") + if not line: + continue + parts = line.split("\t") + if len(parts) < 3: + continue + name, ver, arch = parts[0].strip(), parts[1].strip(), parts[2].strip() + if not name: + continue + instance = {"version": ver, "arch": arch} + if len(parts) >= 4 and parts[3].strip(): + instance["section"] = parts[3].strip() + out.setdefault(name, []).append(instance) + + # Stable ordering for deterministic JSON dumps. + for k in list(out.keys()): + out[k] = sorted( + out[k], key=lambda x: (x.get("arch") or "", x.get("version") or "") + ) + return out + + def build_dpkg_etc_index( info_dir: str = "/var/lib/dpkg/info", ) -> Tuple[Set[str], Dict[str, str], Dict[str, Set[str]], Dict[str, List[str]]]: @@ -154,7 +200,9 @@ def parse_status_conffiles( if ":" in line: k, v = line.split(":", 1) key = k - cur[key] = v.lstrip() + # Preserve leading spaces in continuation lines, but strip + # the trailing newline from the initial key line value. + cur[key] = v.lstrip().rstrip("\n") if cur: flush() @@ -178,28 +226,3 @@ def read_pkg_md5sums(pkg: str) -> Dict[str, str]: md5, rel = line.split(None, 1) m[rel.strip()] = md5.strip() return m - - -def file_md5(path: str) -> str: - h = hashlib.md5() # nosec - with open(path, "rb") as f: - for chunk in iter(lambda: f.read(1024 * 1024), b""): - h.update(chunk) - return h.hexdigest() - - -def stat_triplet(path: str) -> Tuple[str, str, str]: - st = os.stat(path, follow_symlinks=True) - mode = oct(st.st_mode & 0o777)[2:].zfill(4) - - import pwd, grp - - try: - owner = pwd.getpwuid(st.st_uid).pw_name - except KeyError: - owner = str(st.st_uid) - try: - group = grp.getgrgid(st.st_gid).gr_name - except KeyError: - group = str(st.st_gid) - return owner, group, mode diff --git a/enroll/diff.py b/enroll/diff.py new file mode 100644 index 0000000..4784119 --- /dev/null +++ b/enroll/diff.py @@ -0,0 +1,1351 @@ +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import subprocess # nosec +import tarfile +import tempfile +import sys +import threading +import time +import itertools +import urllib.request +from contextlib import ExitStack +from dataclasses import dataclass +from datetime import datetime, timezone +from email.message import EmailMessage +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Tuple + +from .remote import _safe_extract_tar +from .state import ( + inventory_packages_from_state as _packages_inventory, + load_state as _load_state, + roles_from_state as _roles, + state_path, +) +from .pathfilter import PathFilter +from .sopsutil import decrypt_file_binary_to, require_sops_cmd + + +def _progress_enabled() -> bool: + """Return True if we should display interactive progress UI on the CLI. + + We only emit progress when stderr is a TTY, so it won't pollute JSON/text reports + captured by systemd, CI, webhooks, etc. Users can also disable this explicitly via + ENROLL_NO_PROGRESS=1. + """ + if os.environ.get("ENROLL_NO_PROGRESS", "").strip() in {"1", "true", "yes"}: + return False + try: + return sys.stderr.isatty() + except Exception: + return False + + +class _Spinner: + """A tiny terminal spinner with an elapsed-time counter (stderr-only).""" + + def __init__(self, message: str, *, interval: float = 0.12) -> None: + self.message = message.rstrip() + self.interval = interval + self._stop = threading.Event() + self._thread: Optional[threading.Thread] = None + self._last_len = 0 + self._start = 0.0 + + def start(self) -> None: + if self._thread is not None: + return + self._start = time.monotonic() + self._thread = threading.Thread( + target=self._run, name="enroll-spinner", daemon=True + ) + self._thread.start() + + def stop(self, final_line: Optional[str] = None) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=1.0) + + # Clear spinner line. + try: + sys.stderr.write("\r" + (" " * max(self._last_len, 0)) + "\r") + if final_line: + sys.stderr.write(final_line.rstrip() + "\n") + sys.stderr.flush() + except Exception: + pass # nosec + + def _run(self) -> None: + frames = itertools.cycle("|/-\\") + while not self._stop.is_set(): + elapsed = time.monotonic() - self._start + line = f"{self.message} {next(frames)} {elapsed:0.1f}s" + try: + sys.stderr.write("\r" + line) + sys.stderr.flush() + self._last_len = max(self._last_len, len(line)) + except Exception: + return + self._stop.wait(self.interval) + + +def _utc_now_iso() -> str: + return datetime.now(tz=timezone.utc).isoformat() + + +def _sha256(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + while True: + chunk = f.read(1024 * 1024) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + +@dataclass +class BundleRef: + """A prepared harvest bundle. + + `dir` is a directory containing state.json + artifacts/. + `tempdir` is set when the bundle needed extraction into a temp directory. + """ + + dir: Path + tempdir: Optional[tempfile.TemporaryDirectory] = None + + @property + def state_path(self) -> Path: + return state_path(self.dir) + + +def _bundle_from_input(path: str, *, sops_mode: bool) -> BundleRef: + """Resolve a user-supplied path to a harvest bundle directory. + + Accepts: + - a bundle directory + - a path to state.json inside a bundle directory + - (sops mode or .sops) a SOPS-encrypted tar.gz bundle + - a plain tar.gz/tgz bundle + """ + + p = Path(path).expanduser() + + # Accept the state.json path directly (harvest often prints this). + if p.is_file() and p.name == "state.json": + p = p.parent + + if p.is_dir(): + return BundleRef(dir=p) + + if not p.exists(): + raise RuntimeError(f"Harvest path not found: {p}") + + # Auto-enable sops mode if it looks like an encrypted bundle. + is_sops = p.name.endswith(".sops") + if sops_mode or is_sops: + require_sops_cmd() + td = tempfile.TemporaryDirectory(prefix="enroll-harvest-") + td_path = Path(td.name) + try: + os.chmod(td_path, 0o700) + except OSError: + pass + + tar_path = td_path / "harvest.tar.gz" + out_dir = td_path / "bundle" + out_dir.mkdir(parents=True, exist_ok=True) + try: + os.chmod(out_dir, 0o700) + except OSError: + pass + + decrypt_file_binary_to(p, tar_path, mode=0o600) + with tarfile.open(tar_path, mode="r:gz") as tf: + _safe_extract_tar(tf, out_dir) + + return BundleRef(dir=out_dir, tempdir=td) + + # Plain tarballs (useful for operators who rsync/zip harvests around). + if p.suffixes[-2:] == [".tar", ".gz"] or p.suffix == ".tgz": + td = tempfile.TemporaryDirectory(prefix="enroll-harvest-") + td_path = Path(td.name) + try: + os.chmod(td_path, 0o700) + except OSError: + pass + out_dir = td_path / "bundle" + out_dir.mkdir(parents=True, exist_ok=True) + try: + os.chmod(out_dir, 0o700) + except OSError: + pass + with tarfile.open(p, mode="r:gz") as tf: + _safe_extract_tar(tf, out_dir) + return BundleRef(dir=out_dir, tempdir=td) + + raise RuntimeError( + f"Harvest path is not a directory, state.json, encrypted bundle, or tarball: {p}" + ) + + +def _all_packages(state: Dict[str, Any]) -> List[str]: + return sorted(_packages_inventory(state).keys()) + + +def _pkg_version_key(entry: Dict[str, Any]) -> Optional[str]: + """Return a stable string used for version comparison.""" + installs = entry.get("installations") or [] + if isinstance(installs, list) and installs: + parts: List[str] = [] + for inst in installs: + if not isinstance(inst, dict): + continue + arch = str(inst.get("arch") or "") + ver = str(inst.get("version") or "") + if not ver: + continue + parts.append(f"{arch}:{ver}" if arch else ver) + if parts: + return "|".join(sorted(parts)) + v = entry.get("version") + if v: + return str(v) + return None + + +def _pkg_version_display(entry: Dict[str, Any]) -> Optional[str]: + v = entry.get("version") + if v: + return str(v) + installs = entry.get("installations") or [] + if isinstance(installs, list) and installs: + parts: List[str] = [] + for inst in installs: + if not isinstance(inst, dict): + continue + arch = str(inst.get("arch") or "") + ver = str(inst.get("version") or "") + if not ver: + continue + parts.append(f"{ver} ({arch})" if arch else ver) + if parts: + return ", ".join(sorted(parts)) + return None + + +def _service_units(state: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: + out: Dict[str, Dict[str, Any]] = {} + for s in _roles(state).get("services") or []: + unit = s.get("unit") + if unit: + out[str(unit)] = s + return out + + +def _users_by_name(state: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: + users = (_roles(state).get("users") or {}).get("users") or [] + out: Dict[str, Dict[str, Any]] = {} + for u in users: + name = u.get("name") + if name: + out[str(name)] = u + return out + + +@dataclass(frozen=True) +class FileRec: + path: str + role: str + src_rel: str + owner: Optional[str] + group: Optional[str] + mode: Optional[str] + reason: Optional[str] + + +def _iter_managed_files(state: Dict[str, Any]) -> Iterable[Tuple[str, Dict[str, Any]]]: + # Services + for s in _roles(state).get("services") or []: + role = s.get("role_name") or "unknown" + for mf in s.get("managed_files", []) or []: + yield str(role), mf + + # Package roles + for p in _roles(state).get("packages") or []: + role = p.get("role_name") or "unknown" + for mf in p.get("managed_files", []) or []: + yield str(role), mf + + # Users + u = _roles(state).get("users") or {} + u_role = u.get("role_name") or "users" + for mf in u.get("managed_files", []) or []: + yield str(u_role), mf + + # apt_config + ac = _roles(state).get("apt_config") or {} + ac_role = ac.get("role_name") or "apt_config" + for mf in ac.get("managed_files", []) or []: + yield str(ac_role), mf + + # sysctl + sc = _roles(state).get("sysctl") or {} + sc_role = sc.get("role_name") or "sysctl" + for mf in sc.get("managed_files", []) or []: + yield str(sc_role), mf + + # etc_custom + ec = _roles(state).get("etc_custom") or {} + ec_role = ec.get("role_name") or "etc_custom" + for mf in ec.get("managed_files", []) or []: + yield str(ec_role), mf + + # usr_local_custom + ul = _roles(state).get("usr_local_custom") or {} + ul_role = ul.get("role_name") or "usr_local_custom" + for mf in ul.get("managed_files", []) or []: + yield str(ul_role), mf + + # extra_paths + xp = _roles(state).get("extra_paths") or {} + xp_role = xp.get("role_name") or "extra_paths" + for mf in xp.get("managed_files", []) or []: + yield str(xp_role), mf + + +def _file_index(bundle_dir: Path, state: Dict[str, Any]) -> Dict[str, FileRec]: + """Return mapping of absolute path -> FileRec. + + If duplicates occur, the first one wins (should be rare by design). + """ + + out: Dict[str, FileRec] = {} + for role, mf in _iter_managed_files(state): + p = mf.get("path") + src_rel = mf.get("src_rel") + if not p or not src_rel: + continue + p = str(p) + if p in out: + continue + out[p] = FileRec( + path=p, + role=str(role), + src_rel=str(src_rel), + owner=mf.get("owner"), + group=mf.get("group"), + mode=mf.get("mode"), + reason=mf.get("reason"), + ) + return out + + +def _artifact_path(bundle_dir: Path, rec: FileRec) -> Path: + return bundle_dir / "artifacts" / rec.role / rec.src_rel + + +def compare_harvests( + old_path: str, + new_path: str, + *, + sops_mode: bool = False, + exclude_paths: Optional[List[str]] = None, + ignore_package_versions: bool = False, +) -> Tuple[Dict[str, Any], bool]: + """Compare two harvests. + + Returns (report, has_changes). + """ + with ExitStack() as stack: + old_b = _bundle_from_input(old_path, sops_mode=sops_mode) + new_b = _bundle_from_input(new_path, sops_mode=sops_mode) + if old_b.tempdir: + stack.callback(old_b.tempdir.cleanup) + if new_b.tempdir: + stack.callback(new_b.tempdir.cleanup) + + old_state = _load_state(old_b.dir) + new_state = _load_state(new_b.dir) + + old_inv = _packages_inventory(old_state) + new_inv = _packages_inventory(new_state) + + old_pkgs = set(old_inv.keys()) + new_pkgs = set(new_inv.keys()) + + pkgs_added = sorted(new_pkgs - old_pkgs) + pkgs_removed = sorted(old_pkgs - new_pkgs) + + pkgs_version_changed: List[Dict[str, Any]] = [] + pkgs_version_changed_ignored_count = 0 + for pkg in sorted(old_pkgs & new_pkgs): + a = old_inv.get(pkg) or {} + b = new_inv.get(pkg) or {} + if _pkg_version_key(a) != _pkg_version_key(b): + if ignore_package_versions: + pkgs_version_changed_ignored_count += 1 + else: + pkgs_version_changed.append( + { + "package": pkg, + "old": _pkg_version_display(a), + "new": _pkg_version_display(b), + } + ) + + old_units = _service_units(old_state) + new_units = _service_units(new_state) + units_added = sorted(set(new_units) - set(old_units)) + units_removed = sorted(set(old_units) - set(new_units)) + + units_changed: List[Dict[str, Any]] = [] + for unit in sorted(set(old_units) & set(new_units)): + a = old_units[unit] + b = new_units[unit] + ch: Dict[str, Any] = {} + for k in [ + "active_state", + "sub_state", + "unit_file_state", + "condition_result", + ]: + if a.get(k) != b.get(k): + ch[k] = {"old": a.get(k), "new": b.get(k)} + a_pk = set(a.get("packages", []) or []) + b_pk = set(b.get("packages", []) or []) + if a_pk != b_pk: + ch["packages"] = { + "added": sorted(b_pk - a_pk), + "removed": sorted(a_pk - b_pk), + } + if ch: + units_changed.append({"unit": unit, "changes": ch}) + + old_users = _users_by_name(old_state) + new_users = _users_by_name(new_state) + users_added = sorted(set(new_users) - set(old_users)) + users_removed = sorted(set(old_users) - set(new_users)) + + users_changed: List[Dict[str, Any]] = [] + for name in sorted(set(old_users) & set(new_users)): + a = old_users[name] + b = new_users[name] + ch: Dict[str, Any] = {} + for k in [ + "uid", + "gid", + "gecos", + "home", + "shell", + "primary_group", + ]: + if a.get(k) != b.get(k): + ch[k] = {"old": a.get(k), "new": b.get(k)} + a_sg = set(a.get("supplementary_groups", []) or []) + b_sg = set(b.get("supplementary_groups", []) or []) + if a_sg != b_sg: + ch["supplementary_groups"] = { + "added": sorted(b_sg - a_sg), + "removed": sorted(a_sg - b_sg), + } + if ch: + users_changed.append({"name": name, "changes": ch}) + + old_files = _file_index(old_b.dir, old_state) + new_files = _file_index(new_b.dir, new_state) + + # Optional user-supplied path exclusions (same semantics as harvest --exclude-path), + # applied only to file drift reporting. + diff_filter = PathFilter(include=(), exclude=exclude_paths or ()) + if exclude_paths: + old_files = { + p: r for p, r in old_files.items() if not diff_filter.is_excluded(p) + } + new_files = { + p: r for p, r in new_files.items() if not diff_filter.is_excluded(p) + } + old_paths_set = set(old_files) + new_paths_set = set(new_files) + + files_added = sorted(new_paths_set - old_paths_set) + files_removed = sorted(old_paths_set - new_paths_set) + + # Hash cache to avoid reading the same file more than once. + hash_cache: Dict[str, str] = {} + + def _hash_for(bundle_dir: Path, rec: FileRec) -> Optional[str]: + ap = _artifact_path(bundle_dir, rec) + if not ap.exists() or not ap.is_file(): + return None + key = str(ap) + if key in hash_cache: + return hash_cache[key] + hash_cache[key] = _sha256(ap) + return hash_cache[key] + + files_changed: List[Dict[str, Any]] = [] + for p in sorted(old_paths_set & new_paths_set): + a = old_files[p] + b = new_files[p] + ch: Dict[str, Any] = {} + + # Role movement is itself interesting (e.g., file ownership attribution changed). + if a.role != b.role: + ch["role"] = {"old": a.role, "new": b.role} + for k in ["owner", "group", "mode", "reason"]: + av = getattr(a, k) + bv = getattr(b, k) + if av != bv: + ch[k] = {"old": av, "new": bv} + + ha = _hash_for(old_b.dir, a) + hb = _hash_for(new_b.dir, b) + if ha is None or hb is None: + if ha != hb: + ch["content"] = { + "old": "missing" if ha is None else "present", + "new": "missing" if hb is None else "present", + } + else: + if ha != hb: + ch["content"] = {"old_sha256": ha, "new_sha256": hb} + + if ch: + files_changed.append({"path": p, "changes": ch}) + + has_changes = any( + [ + pkgs_added, + pkgs_removed, + pkgs_version_changed, + units_added, + units_removed, + units_changed, + users_added, + users_removed, + users_changed, + files_added, + files_removed, + files_changed, + ] + ) + + def _mtime_iso(p: Path) -> Optional[str]: + try: + ts = p.stat().st_mtime + except OSError: + return None + return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat() + + report: Dict[str, Any] = { + "generated_at": _utc_now_iso(), + "filters": { + "exclude_paths": list(exclude_paths or []), + "ignore_package_versions": bool(ignore_package_versions), + }, + "old": { + "input": old_path, + "bundle_dir": str(old_b.dir), + "state_mtime": _mtime_iso(old_b.state_path), + "host": (old_state.get("host") or {}).get("hostname"), + }, + "new": { + "input": new_path, + "bundle_dir": str(new_b.dir), + "state_mtime": _mtime_iso(new_b.state_path), + "host": (new_state.get("host") or {}).get("hostname"), + }, + "packages": { + "added": pkgs_added, + "removed": pkgs_removed, + "version_changed": pkgs_version_changed, + "version_changed_ignored_count": int( + pkgs_version_changed_ignored_count + ), + }, + "services": { + "enabled_added": units_added, + "enabled_removed": units_removed, + "changed": units_changed, + }, + "users": { + "added": users_added, + "removed": users_removed, + "changed": users_changed, + }, + "files": { + "added": [ + { + "path": p, + "role": new_files[p].role, + "reason": new_files[p].reason, + } + for p in files_added + ], + "removed": [ + { + "path": p, + "role": old_files[p].role, + "reason": old_files[p].reason, + } + for p in files_removed + ], + "changed": files_changed, + }, + } + + return report, has_changes + + +def has_enforceable_drift(report: Dict[str, Any]) -> bool: + """Return True if the diff report contains drift that is safe/meaningful to enforce. + + Enforce mode is intended to restore *state* (files/users/services) and to + reinstall packages that were removed. + + It is deliberately conservative about package drift: + - Package *version* changes alone are not enforced (no downgrades). + - Newly installed packages are not removed. + + This helper lets the CLI decide whether `--enforce` should actually run. + """ + + pk = report.get("packages", {}) or {} + if pk.get("removed"): + return True + + sv = report.get("services", {}) or {} + # We do not try to disable newly-enabled services; we only restore units + # that were enabled in the baseline but are now missing. + if sv.get("enabled_removed") or []: + return True + + for ch in sv.get("changed", []) or []: + changes = ch.get("changes") or {} + # Ignore package set drift for enforceability decisions; package + # enforcement is handled via reinstalling removed packages, and we + # avoid trying to "undo" upgrades/renames. + for k in changes.keys(): + if k != "packages": + return True + + us = report.get("users", {}) or {} + # We restore baseline users (missing/changed). We do not remove newly-added users. + if (us.get("removed") or []) or (us.get("changed") or []): + return True + + fl = report.get("files", {}) or {} + # We restore baseline files (missing/changed). We do not delete newly-managed files. + if (fl.get("removed") or []) or (fl.get("changed") or []): + return True + + return False + + +def _role_tag(role: str) -> str: + """Return the Ansible tag name for a role (must match manifest generation).""" + r = str(role or "").strip() + safe = re.sub(r"[^A-Za-z0-9_-]+", "_", r).strip("_") + if not safe: + safe = "other" + return f"role_{safe}" + + +def _enforcement_plan( + report: Dict[str, Any], + old_state: Dict[str, Any], + old_bundle_dir: Path, +) -> Dict[str, Any]: + """Return a best-effort enforcement plan (roles/tags) for this diff report. + + We only plan for drift that the baseline manifest can safely restore: + - packages that were removed (reinstall, no downgrades) + - baseline users that were removed/changed + - baseline files that were removed/changed + - baseline systemd units that were disabled/changed + + We do NOT plan to remove newly-added packages/users/files/services. + """ + roles: set[str] = set() + + # --- Packages (only removals) + pk = report.get("packages", {}) or {} + removed_pkgs = set(pk.get("removed") or []) + if removed_pkgs: + pkg_to_roles: Dict[str, set[str]] = {} + + for svc in _roles(old_state).get("services") or []: + r = str(svc.get("role_name") or "").strip() + for p in svc.get("packages", []) or []: + if p: + pkg_to_roles.setdefault(str(p), set()).add(r) + + for pr in _roles(old_state).get("packages") or []: + r = str(pr.get("role_name") or "").strip() + p = pr.get("package") + if p: + pkg_to_roles.setdefault(str(p), set()).add(r) + + for p in removed_pkgs: + for r in pkg_to_roles.get(str(p), set()): + if r: + roles.add(r) + + # --- Users (removed/changed) + us = report.get("users", {}) or {} + if (us.get("removed") or []) or (us.get("changed") or []): + u = _roles(old_state).get("users") or {} + u_role = str(u.get("role_name") or "users") + if u_role: + roles.add(u_role) + + # --- Files (removed/changed) + fl = report.get("files", {}) or {} + file_paths: List[str] = [] + for e in fl.get("removed", []) or []: + if isinstance(e, dict): + p = e.get("path") + else: + p = e + if p: + file_paths.append(str(p)) + for e in fl.get("changed", []) or []: + if isinstance(e, dict): + p = e.get("path") + else: + p = e + if p: + file_paths.append(str(p)) + + if file_paths: + idx = _file_index(old_bundle_dir, old_state) + for p in file_paths: + rec = idx.get(p) + if rec and rec.role: + roles.add(str(rec.role)) + + # --- Services (enabled_removed + meaningful changes) + sv = report.get("services", {}) or {} + units: List[str] = [] + for u in sv.get("enabled_removed", []) or []: + if u: + units.append(str(u)) + for ch in sv.get("changed", []) or []: + if not isinstance(ch, dict): + continue + unit = ch.get("unit") + changes = ch.get("changes") or {} + if unit and any(k != "packages" for k in changes.keys()): + units.append(str(unit)) + + if units: + old_units = _service_units(old_state) + for u in units: + snap = old_units.get(u) + if snap and snap.get("role_name"): + roles.add(str(snap.get("role_name"))) + + # Drop empty/unknown roles. + roles = {r for r in roles if r and str(r).strip() and str(r).strip() != "unknown"} + + tags = sorted({_role_tag(r) for r in roles}) + return { + "roles": sorted(roles), + "tags": tags, + } + + +def enforce_old_harvest( + old_path: str, + *, + sops_mode: bool = False, + report: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Enforce the *old* (baseline) harvest state on the current machine. + + When Ansible is available, this: + 1) renders a temporary manifest from the old harvest, and + 2) runs ansible-playbook locally to apply it. + + Returns a dict suitable for attaching to the diff report under + report['enforcement']. + """ + + ansible_playbook = shutil.which("ansible-playbook") + if not ansible_playbook: + raise RuntimeError( + "ansible-playbook not found on PATH (cannot enforce; install Ansible)" + ) + + # Import lazily to avoid heavy import cost and potential CLI cycles. + from .manifest import manifest + + started_at = _utc_now_iso() + + with ExitStack() as stack: + old_b = _bundle_from_input(old_path, sops_mode=sops_mode) + if old_b.tempdir: + stack.callback(old_b.tempdir.cleanup) + + old_state = _load_state(old_b.dir) + + plan: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] = None + roles: List[str] = [] + if report is not None: + plan = _enforcement_plan(report, old_state, old_b.dir) + roles = list(plan.get("roles") or []) + t = list(plan.get("tags") or []) + tags = t if t else None + + with tempfile.TemporaryDirectory(prefix="enroll-enforce-") as td: + td_path = Path(td) + try: + os.chmod(td_path, 0o700) + except OSError: + pass + + # 1) Generate a manifest in a temp directory. + manifest(str(old_b.dir), str(td_path)) + + playbook = td_path / "playbook.yml" + if not playbook.exists(): + raise RuntimeError( + f"manifest did not produce expected playbook.yml at {playbook}" + ) + + # 2) Apply it locally. + env = dict(os.environ) + cfg = td_path / "ansible.cfg" + if cfg.exists(): + env["ANSIBLE_CONFIG"] = str(cfg) + + cmd = [ + ansible_playbook, + "-i", + "localhost,", + "-c", + "local", + str(playbook), + ] + if tags: + cmd.extend(["--tags", ",".join(tags)]) + + spinner: Optional[_Spinner] = None + p: Optional[subprocess.CompletedProcess[str]] = None + t0 = time.monotonic() + if _progress_enabled(): + if tags: + sys.stderr.write( + f"Enforce: running ansible-playbook (tags: {','.join(tags)})\n", + ) + else: + sys.stderr.write("Enforce: running ansible-playbook\n") + sys.stderr.flush() + spinner = _Spinner(" ansible-playbook") + spinner.start() + + try: + p = subprocess.run( + cmd, + cwd=str(td_path), + env=env, + capture_output=True, + text=True, + check=False, + ) # nosec + finally: + if spinner: + elapsed = time.monotonic() - t0 + rc = p.returncode if p is not None else None + spinner.stop( + final_line=( + f"Enforce: ansible-playbook finished in {elapsed:0.1f}s" + + (f" (rc={rc})" if rc is not None else ""), + ), + ) + + finished_at = _utc_now_iso() + + info: Dict[str, Any] = { + "status": "applied" if p.returncode == 0 else "failed", + "started_at": started_at, + "finished_at": finished_at, + "ansible_playbook": ansible_playbook, + "command": cmd, + "returncode": int(p.returncode), + } + + # Record tag selection (if we could attribute drift to specific roles). + info["roles"] = roles + info["tags"] = list(tags or []) + if not tags: + info["scope"] = "full_playbook" + + if p.returncode != 0: + err = (p.stderr or p.stdout or "").strip() + raise RuntimeError( + "ansible-playbook failed" + + (f" (rc={p.returncode})" if p.returncode is not None else "") + + (f": {err}" if err else "") + ) + + return info + + +def format_report(report: Dict[str, Any], *, fmt: str = "text") -> str: + fmt = (fmt or "text").lower() + if fmt == "json": + return json.dumps(report, indent=2, sort_keys=True) + if fmt == "markdown": + return _report_markdown(report) + return _report_text(report) + + +def _report_text(report: Dict[str, Any]) -> str: + lines: List[str] = [] + old = report.get("old", {}) + new = report.get("new", {}) + lines.append( + f"enroll diff report (generated {report.get('generated_at')})\n" + f"old: {old.get('input')} (host={old.get('host')}, state_mtime={old.get('state_mtime')})\n" + f"new: {new.get('input')} (host={new.get('host')}, state_mtime={new.get('state_mtime')})" + ) + + filt = report.get("filters", {}) or {} + ex_paths = filt.get("exclude_paths", []) or [] + if ex_paths: + lines.append(f"file exclude patterns: {', '.join(str(p) for p in ex_paths)}") + + if filt.get("ignore_package_versions"): + ignored = int( + (report.get("packages", {}) or {}).get("version_changed_ignored_count") or 0 + ) + msg = "package version drift: ignored (--ignore-package-versions)" + if ignored: + msg += f" (ignored {ignored} change{'s' if ignored != 1 else ''})" + lines.append(msg) + + enf = report.get("enforcement") or {} + if enf: + lines.append("\nEnforcement") + status = str(enf.get("status") or "").strip().lower() + if status == "applied": + extra = "" + tags = enf.get("tags") or [] + scope = enf.get("scope") + if tags: + extra = f" (tags={','.join(str(t) for t in tags)})" + elif scope: + extra = f" ({scope})" + lines.append( + f" applied old harvest via ansible-playbook (rc={enf.get('returncode')})" + + extra + + ( + f" (finished {enf.get('finished_at')})" + if enf.get("finished_at") + else "" + ) + ) + elif status == "failed": + lines.append( + f" attempted enforcement but ansible-playbook failed (rc={enf.get('returncode')})" + ) + elif status == "skipped": + r = enf.get("reason") + lines.append(" skipped" + (f": {r}" if r else "")) + else: + # Best-effort formatting for future fields. + lines.append(" " + json.dumps(enf, sort_keys=True)) + + pk = report.get("packages", {}) + lines.append("\nPackages") + lines.append(f" added: {len(pk.get('added', []) or [])}") + lines.append(f" removed: {len(pk.get('removed', []) or [])}") + ignored_v = int(pk.get("version_changed_ignored_count") or 0) + vc = len(pk.get("version_changed", []) or []) + suffix = f" (ignored {ignored_v})" if ignored_v else "" + lines.append(f" version_changed: {vc}{suffix}") + for p in pk.get("added", []) or []: + lines.append(f" + {p}") + for p in pk.get("removed", []) or []: + lines.append(f" - {p}") + for ch in pk.get("version_changed", []) or []: + lines.append(f" ~ {ch.get('package')}: {ch.get('old')} -> {ch.get('new')}") + + sv = report.get("services", {}) + lines.append("\nServices (enabled systemd units)") + for u in sv.get("enabled_added", []) or []: + lines.append(f" + {u}") + for u in sv.get("enabled_removed", []) or []: + lines.append(f" - {u}") + for ch in sv.get("changed", []) or []: + unit = ch.get("unit") + lines.append(f" * {unit} changed") + for k, v in (ch.get("changes") or {}).items(): + if k == "packages": + a = (v or {}).get("added", []) + r = (v or {}).get("removed", []) + if a: + lines.append(f" packages +: {', '.join(a)}") + if r: + lines.append(f" packages -: {', '.join(r)}") + else: + lines.append(f" {k}: {v.get('old')} -> {v.get('new')}") + + us = report.get("users", {}) + lines.append("\nUsers") + for u in us.get("added", []) or []: + lines.append(f" + {u}") + for u in us.get("removed", []) or []: + lines.append(f" - {u}") + for ch in us.get("changed", []) or []: + name = ch.get("name") + lines.append(f" * {name} changed") + for k, v in (ch.get("changes") or {}).items(): + if k == "supplementary_groups": + a = (v or {}).get("added", []) + r = (v or {}).get("removed", []) + if a: + lines.append(f" groups +: {', '.join(a)}") + if r: + lines.append(f" groups -: {', '.join(r)}") + else: + lines.append(f" {k}: {v.get('old')} -> {v.get('new')}") + + fl = report.get("files", {}) + lines.append("\nFiles") + for e in fl.get("added", []) or []: + lines.append( + f" + {e.get('path')} (role={e.get('role')}, reason={e.get('reason')})" + ) + for e in fl.get("removed", []) or []: + lines.append( + f" - {e.get('path')} (role={e.get('role')}, reason={e.get('reason')})" + ) + for ch in fl.get("changed", []) or []: + p = ch.get("path") + lines.append(f" * {p} changed") + for k, v in (ch.get("changes") or {}).items(): + if k == "content": + if "old_sha256" in (v or {}): + lines.append(" content: sha256 changed") + else: + lines.append(f" content: {v.get('old')} -> {v.get('new')}") + else: + lines.append(f" {k}: {v.get('old')} -> {v.get('new')}") + + if not any( + [ + (pk.get("added") or []), + (pk.get("removed") or []), + (pk.get("version_changed") or []), + (sv.get("enabled_added") or []), + (sv.get("enabled_removed") or []), + (sv.get("changed") or []), + (us.get("added") or []), + (us.get("removed") or []), + (us.get("changed") or []), + (fl.get("added") or []), + (fl.get("removed") or []), + (fl.get("changed") or []), + ] + ): + lines.append("\nNo differences detected.") + + return "\n".join(lines) + "\n" + + +def _report_markdown(report: Dict[str, Any]) -> str: + old = report.get("old", {}) + new = report.get("new", {}) + out: List[str] = [] + out.append("# enroll diff report\n") + out.append(f"Generated: `{report.get('generated_at')}`\n") + out.append( + f"- **Old**: `{old.get('input')}` (host={old.get('host')}, state_mtime={old.get('state_mtime')})\n" + f"- **New**: `{new.get('input')}` (host={new.get('host')}, state_mtime={new.get('state_mtime')})\n" + ) + + filt = report.get("filters", {}) or {} + ex_paths = filt.get("exclude_paths", []) or [] + if ex_paths: + out.append( + "- **File exclude patterns**: " + + ", ".join(f"`{p}`" for p in ex_paths) + + "\n" + ) + + if filt.get("ignore_package_versions"): + ignored = int( + (report.get("packages", {}) or {}).get("version_changed_ignored_count") or 0 + ) + msg = "- **Package version drift**: ignored (`--ignore-package-versions`)" + if ignored: + msg += f" (ignored {ignored} change{'s' if ignored != 1 else ''})" + out.append(msg + "\n") + + enf = report.get("enforcement") or {} + if enf: + out.append("\n## Enforcement\n") + status = str(enf.get("status") or "").strip().lower() + if status == "applied": + extra = "" + tags = enf.get("tags") or [] + scope = enf.get("scope") + if tags: + extra = " (tags=" + ",".join(str(t) for t in tags) + ")" + elif scope: + extra = f" ({scope})" + out.append( + "- ✅ Applied old harvest via ansible-playbook" + + extra + + ( + f" (rc={enf.get('returncode')})" + if enf.get("returncode") is not None + else "" + ) + + ( + f" (finished `{enf.get('finished_at')}`)" + if enf.get("finished_at") + else "" + ) + + "\n" + ) + elif status == "failed": + out.append( + "- ⚠️ Attempted enforcement but ansible-playbook failed" + + ( + f" (rc={enf.get('returncode')})" + if enf.get("returncode") is not None + else "" + ) + + "\n" + ) + elif status == "skipped": + r = enf.get("reason") + out.append("- Skipped" + (f": {r}" if r else "") + "\n") + else: + out.append(f"- {json.dumps(enf, sort_keys=True)}\n") + + pk = report.get("packages", {}) + out.append("## Packages\n") + out.append(f"- Added: {len(pk.get('added', []) or [])}\n") + for p in pk.get("added", []) or []: + out.append(f" - `+ {p}`\n") + out.append(f"- Removed: {len(pk.get('removed', []) or [])}\n") + for p in pk.get("removed", []) or []: + out.append(f" - `- {p}`\n") + + ignored_v = int(pk.get("version_changed_ignored_count") or 0) + vc = len(pk.get("version_changed", []) or []) + suffix = f" (ignored {ignored_v})" if ignored_v else "" + out.append(f"- Version changed: {vc}{suffix}\n") + for ch in pk.get("version_changed", []) or []: + out.append( + f" - `~ {ch.get('package')}`: `{ch.get('old')}` → `{ch.get('new')}`\n" + ) + + sv = report.get("services", {}) + out.append("## Services (enabled systemd units)\n") + if sv.get("enabled_added"): + out.append("- Enabled added\n") + for u in sv.get("enabled_added", []) or []: + out.append(f" - `+ {u}`\n") + if sv.get("enabled_removed"): + out.append("- Enabled removed\n") + for u in sv.get("enabled_removed", []) or []: + out.append(f" - `- {u}`\n") + if sv.get("changed"): + out.append("- Changed\n") + for ch in sv.get("changed", []) or []: + unit = ch.get("unit") + out.append(f" - `{unit}`\n") + for k, v in (ch.get("changes") or {}).items(): + if k == "packages": + a = (v or {}).get("added", []) + r = (v or {}).get("removed", []) + if a: + out.append( + f" - packages added: {', '.join('`'+x+'`' for x in a)}\n" + ) + if r: + out.append( + f" - packages removed: {', '.join('`'+x+'`' for x in r)}\n" + ) + else: + out.append(f" - {k}: `{v.get('old')}` → `{v.get('new')}`\n") + + us = report.get("users", {}) + out.append("## Users\n") + if us.get("added"): + out.append("- Added\n") + for u in us.get("added", []) or []: + out.append(f" - `+ {u}`\n") + if us.get("removed"): + out.append("- Removed\n") + for u in us.get("removed", []) or []: + out.append(f" - `- {u}`\n") + if us.get("changed"): + out.append("- Changed\n") + for ch in us.get("changed", []) or []: + name = ch.get("name") + out.append(f" - `{name}`\n") + for k, v in (ch.get("changes") or {}).items(): + if k == "supplementary_groups": + a = (v or {}).get("added", []) + r = (v or {}).get("removed", []) + if a: + out.append( + f" - groups added: {', '.join('`'+x+'`' for x in a)}\n" + ) + if r: + out.append( + f" - groups removed: {', '.join('`'+x+'`' for x in r)}\n" + ) + else: + out.append(f" - {k}: `{v.get('old')}` → `{v.get('new')}`\n") + + fl = report.get("files", {}) + out.append("## Files\n") + if fl.get("added"): + out.append("- Added\n") + for e in fl.get("added", []) or []: + out.append( + f" - `+ {e.get('path')}` (role={e.get('role')}, reason={e.get('reason')})\n" + ) + if fl.get("removed"): + out.append("- Removed\n") + for e in fl.get("removed", []) or []: + out.append( + f" - `- {e.get('path')}` (role={e.get('role')}, reason={e.get('reason')})\n" + ) + if fl.get("changed"): + out.append("- Changed\n") + for ch in fl.get("changed", []) or []: + p = ch.get("path") + out.append(f" - `{p}`\n") + for k, v in (ch.get("changes") or {}).items(): + if k == "content": + if "old_sha256" in (v or {}): + out.append(" - content: sha256 changed\n") + else: + out.append( + f" - content: `{v.get('old')}` → `{v.get('new')}`\n" + ) + else: + out.append(f" - {k}: `{v.get('old')}` → `{v.get('new')}`\n") + + if not any( + [ + (pk.get("added") or []), + (pk.get("removed") or []), + (pk.get("version_changed") or []), + (sv.get("enabled_added") or []), + (sv.get("enabled_removed") or []), + (sv.get("changed") or []), + (us.get("added") or []), + (us.get("removed") or []), + (us.get("changed") or []), + (fl.get("added") or []), + (fl.get("removed") or []), + (fl.get("changed") or []), + ] + ): + out.append("\n_No differences detected._\n") + + return "".join(out) + + +def post_webhook( + url: str, + body: bytes, + *, + headers: Optional[Dict[str, str]] = None, + timeout_s: int = 10, +) -> Tuple[int, str]: + req = urllib.request.Request(url=url, data=body, method="POST") + for k, v in (headers or {}).items(): + req.add_header(k, v) + try: + with urllib.request.urlopen(req, timeout=timeout_s) as resp: # nosec + status = int(getattr(resp, "status", 0) or 0) + text = resp.read().decode("utf-8", errors="replace") + return status, text + except Exception as e: + raise RuntimeError(f"webhook POST failed: {e}") from e + + +def send_email( + *, + to_addrs: List[str], + subject: str, + body: str, + from_addr: Optional[str] = None, + smtp: Optional[str] = None, + smtp_user: Optional[str] = None, + smtp_password: Optional[str] = None, +) -> None: + if not to_addrs: + raise RuntimeError("email: no recipients") + + msg = EmailMessage() + msg["To"] = ", ".join(to_addrs) + if from_addr: + msg["From"] = from_addr + else: + host = os.uname().nodename + msg["From"] = f"enroll@{host}" + msg["Subject"] = subject + msg.set_content(body) + + # Preferred: use local sendmail if smtp wasn't specified. + if not smtp: + sendmail = shutil.which("sendmail") + if not sendmail: + raise RuntimeError( + "email: no --smtp provided and sendmail not found on PATH" + ) + p = subprocess.run( + [sendmail, "-t", "-i"], + input=msg.as_bytes(), + capture_output=True, + check=False, + ) # nosec + if p.returncode != 0: + raise RuntimeError( + "email: sendmail failed:\n" + f" rc: {p.returncode}\n" + f" stderr: {p.stderr.decode('utf-8', errors='replace').strip()}" + ) + return + + import smtplib + + host = smtp + port = 25 + if ":" in smtp: + host, port_s = smtp.rsplit(":", 1) + try: + port = int(port_s) + except ValueError: + raise RuntimeError(f"email: invalid smtp port in {smtp!r}") + + with smtplib.SMTP(host, port, timeout=10) as s: + s.ehlo() + try: + s.starttls() + s.ehlo() + except Exception: + # STARTTLS is optional; ignore if unsupported. + pass # nosec + if smtp_user: + s.login(smtp_user, smtp_password or "") + s.send_message(msg) diff --git a/enroll/explain.py b/enroll/explain.py new file mode 100644 index 0000000..84d5de0 --- /dev/null +++ b/enroll/explain.py @@ -0,0 +1,601 @@ +from __future__ import annotations + +import json +from collections import Counter, defaultdict +from dataclasses import dataclass +from typing import Any, Dict, Iterable, List, Tuple + +from .diff import _bundle_from_input # reuse existing bundle handling +from .state import load_state + + +@dataclass(frozen=True) +class ReasonInfo: + title: str + why: str + + +_MANAGED_FILE_REASONS: Dict[str, ReasonInfo] = { + # Package manager / repo config + "apt_config": ReasonInfo( + "APT configuration", + "APT configuration affecting package installation and repository behavior.", + ), + "apt_source": ReasonInfo( + "APT repository source", + "APT source list entries (e.g. sources.list or sources.list.d).", + ), + "apt_keyring": ReasonInfo( + "APT keyring", + "Repository signing key material used by APT.", + ), + "apt_signed_by_keyring": ReasonInfo( + "APT Signed-By keyring", + "Keyring referenced via a Signed-By directive in an APT source.", + ), + "yum_conf": ReasonInfo( + "YUM/DNF main config", + "Primary YUM configuration (often /etc/yum.conf).", + ), + "yum_config": ReasonInfo( + "YUM/DNF config", + "YUM/DNF configuration files (including conf.d).", + ), + "yum_repo": ReasonInfo( + "YUM/DNF repository", + "YUM/DNF repository definitions (e.g. yum.repos.d).", + ), + "dnf_config": ReasonInfo( + "DNF configuration", + "DNF configuration affecting package installation and repositories.", + ), + "rpm_gpg_key": ReasonInfo( + "RPM GPG key", + "Repository signing keys used by RPM/YUM/DNF.", + ), + # SSH + "authorized_keys": ReasonInfo( + "SSH authorized keys", + "User authorized_keys files (controls who can log in with SSH keys).", + ), + "ssh_public_key": ReasonInfo( + "SSH public key", + "SSH host/user public keys relevant to authentication.", + ), + # System config / security + "system_security": ReasonInfo( + "Security configuration", + "Security-sensitive configuration (SSH, sudoers, PAM, auth, etc.).", + ), + "system_network": ReasonInfo( + "Network configuration", + "Network configuration (interfaces, resolv.conf, network managers, etc.).", + ), + "system_firewall": ReasonInfo( + "Firewall configuration", + "Firewall rules/configuration (ufw, nftables, iptables, ipset, etc.).", + ), + "system_sysctl": ReasonInfo( + "sysctl configuration", + "Kernel sysctl tuning (sysctl.conf / sysctl.d).", + ), + "system_modprobe": ReasonInfo( + "modprobe configuration", + "Kernel module configuration (modprobe.d).", + ), + "system_mounts": ReasonInfo( + "Mount configuration", + "Mount configuration (e.g. /etc/fstab and related).", + ), + "system_rc": ReasonInfo( + "Startup/rc configuration", + "Startup scripts / rc configuration that can affect boot behavior.", + ), + # systemd + timers + "systemd_dropin": ReasonInfo( + "systemd drop-in", + "systemd override/drop-in files that modify a unit's behavior.", + ), + "systemd_envfile": ReasonInfo( + "systemd EnvironmentFile", + "Files referenced by systemd units via EnvironmentFile.", + ), + "related_timer": ReasonInfo( + "Related systemd timer", + "A systemd timer captured because it is related to a unit/service.", + ), + # cron / logrotate + "system_cron": ReasonInfo( + "System cron", + "System cron configuration (crontab, cron.d, etc.).", + ), + "cron_snippet": ReasonInfo( + "Cron snippet", + "Cron snippets referenced/used by harvested services or configs.", + ), + "system_logrotate": ReasonInfo( + "System logrotate", + "System logrotate configuration.", + ), + "logrotate_snippet": ReasonInfo( + "logrotate snippet", + "logrotate snippets/configs referenced in system configuration.", + ), + # Custom paths / drift signals + "modified_conffile": ReasonInfo( + "Modified package conffile", + "A package-managed conffile differs from the packaged/default version.", + ), + "modified_packaged_file": ReasonInfo( + "Modified packaged file", + "A file owned by a package differs from the packaged version.", + ), + "custom_unowned": ReasonInfo( + "Unowned custom file", + "A file not owned by any package (often custom/operator-managed).", + ), + "custom_specific_path": ReasonInfo( + "Custom specific path", + "A specific path included by a custom rule or snapshot.", + ), + "usr_local_bin_script": ReasonInfo( + "/usr/local/bin script", + "Executable scripts under /usr/local/bin (often operator-installed).", + ), + "usr_local_etc_custom": ReasonInfo( + "/usr/local/etc custom", + "Custom configuration under /usr/local/etc.", + ), + # User includes + "user_include": ReasonInfo( + "User-included path", + "Included because you specified it via --include-path / include patterns.", + ), +} + +_MANAGED_DIR_REASONS: Dict[str, ReasonInfo] = { + "parent_of_managed_file": ReasonInfo( + "Parent directory", + "Included so permissions/ownership can be recreated for managed files.", + ), + "user_include_dir": ReasonInfo( + "User-included directory", + "Included because you specified it via --include-path / include patterns.", + ), +} + +_EXCLUDED_REASONS: Dict[str, ReasonInfo] = { + "user_excluded": ReasonInfo( + "User excluded", + "Excluded because you explicitly excluded it (e.g. --exclude-path / patterns).", + ), + "unreadable": ReasonInfo( + "Unreadable", + "Enroll could not read this path with the permissions it had.", + ), + "log_file": ReasonInfo( + "Log file", + "Excluded because it appears to be a log file (usually noisy/large).", + ), + "denied_path": ReasonInfo( + "Denied path", + "Excluded because the path is in a denylist for safety.", + ), + "too_large": ReasonInfo( + "Too large", + "Excluded because it exceeded the size limit for harvested files.", + ), + "not_regular_file": ReasonInfo( + "Not a regular file", + "Excluded because it was not a regular file (device, socket, etc.).", + ), + "binary_like": ReasonInfo( + "Binary-like", + "Excluded because it looked like binary content (not useful for config management).", + ), + "sensitive_content": ReasonInfo( + "Sensitive content", + "Excluded because it likely contains secrets (e.g. shadow, private keys).", + ), +} + +_OBSERVED_VIA: Dict[str, ReasonInfo] = { + "user_installed": ReasonInfo( + "User-installed", + "Package appears explicitly installed (as opposed to only pulled in as a dependency).", + ), + "systemd_unit": ReasonInfo( + "Referenced by systemd unit", + "Package is associated with a systemd unit that was harvested.", + ), + "package_role": ReasonInfo( + "Referenced by package role", + "Package was referenced by an enroll packages snapshot/role.", + ), + "firewall_runtime": ReasonInfo( + "Referenced by firewall runtime role", + "Package was referenced by captured live ipset/iptables runtime state.", + ), +} + + +def _ri(mapping: Dict[str, ReasonInfo], key: str) -> ReasonInfo: + return mapping.get(key) or ReasonInfo(key, f"Captured with reason '{key}'") + + +def _role_common_counts(role_obj: Dict[str, Any]) -> Tuple[int, int, int, int]: + """Return (managed_files, managed_dirs, excluded, notes) counts for a RoleCommon object.""" + mf = len(role_obj.get("managed_files") or []) + md = len(role_obj.get("managed_dirs") or []) + ex = len(role_obj.get("excluded") or []) + nt = len(role_obj.get("notes") or []) + return mf, md, ex, nt + + +def _summarize_reasons( + items: Iterable[Dict[str, Any]], + reason_key: str, + *, + mapping: Dict[str, ReasonInfo], + max_examples: int, +) -> List[Dict[str, Any]]: + by_reason: Dict[str, List[str]] = defaultdict(list) + counts: Counter[str] = Counter() + + for it in items: + if not isinstance(it, dict): + continue + r = it.get(reason_key) + if not r: + continue + r = str(r) + counts[r] += 1 + p = it.get("path") + if ( + max_examples > 0 + and isinstance(p, str) + and p + and len(by_reason[r]) < max_examples + ): + by_reason[r].append(p) + + out: List[Dict[str, Any]] = [] + for reason, count in counts.most_common(): + info = _ri(mapping, reason) + out.append( + { + "reason": reason, + "count": count, + "title": info.title, + "why": info.why, + "examples": by_reason.get(reason, []), + } + ) + return out + + +def explain_state( + harvest: str, + *, + sops_mode: bool = False, + fmt: str = "text", + max_examples: int = 3, +) -> str: + """Explain a harvest bundle's state.json. + + `harvest` may be: + - a bundle directory + - a path to state.json + - a tarball (.tar.gz/.tgz) + - a SOPS-encrypted bundle (.sops) + """ + bundle = _bundle_from_input(harvest, sops_mode=sops_mode) + state = load_state(bundle.dir) + + host = state.get("host") or {} + enroll = state.get("enroll") or {} + roles = state.get("roles") or {} + inv = state.get("inventory") or {} + inv_pkgs = (inv.get("packages") or {}) if isinstance(inv, dict) else {} + + role_summaries: List[Dict[str, Any]] = [] + + # Users + users_obj = roles.get("users") or {} + user_entries = users_obj.get("users") or [] + mf, md, ex, _nt = ( + _role_common_counts(users_obj) if isinstance(users_obj, dict) else (0, 0, 0, 0) + ) + role_summaries.append( + { + "role": "users", + "summary": f"{len(user_entries)} user(s), {mf} file(s), {ex} excluded", + "notes": users_obj.get("notes") or [], + } + ) + + # Services + services_list = roles.get("services") or [] + if isinstance(services_list, list): + total_mf = sum( + len((s.get("managed_files") or [])) + for s in services_list + if isinstance(s, dict) + ) + total_ex = sum( + len((s.get("excluded") or [])) for s in services_list if isinstance(s, dict) + ) + role_summaries.append( + { + "role": "services", + "summary": f"{len(services_list)} unit(s), {total_mf} file(s), {total_ex} excluded", + "units": [ + { + "unit": s.get("unit"), + "active_state": s.get("active_state"), + "sub_state": s.get("sub_state"), + "unit_file_state": s.get("unit_file_state"), + "condition_result": s.get("condition_result"), + } + for s in services_list + if isinstance(s, dict) + ], + } + ) + + # Package snapshots + pkgs_list = roles.get("packages") or [] + if isinstance(pkgs_list, list): + total_mf = sum( + len((p.get("managed_files") or [])) + for p in pkgs_list + if isinstance(p, dict) + ) + total_ex = sum( + len((p.get("excluded") or [])) for p in pkgs_list if isinstance(p, dict) + ) + role_summaries.append( + { + "role": "packages", + "summary": f"{len(pkgs_list)} package snapshot(s), {total_mf} file(s), {total_ex} excluded", + "packages": [ + p.get("package") for p in pkgs_list if isinstance(p, dict) + ], + } + ) + + # Runtime firewall snapshot + firewall_obj = roles.get("firewall_runtime") or {} + if isinstance(firewall_obj, dict) and firewall_obj: + captures = [ + key + for key in ("ipset_save", "iptables_v4_save", "iptables_v6_save") + if firewall_obj.get(key) + ] + role_summaries.append( + { + "role": "firewall_runtime", + "summary": f"{len(captures)} snapshot(s), {len(firewall_obj.get('ipset_sets') or [])} ipset(s)", + "notes": firewall_obj.get("notes") or [], + } + ) + + # Single snapshots + for rname in [ + "apt_config", + "dnf_config", + "sysctl", + "etc_custom", + "usr_local_custom", + "extra_paths", + ]: + robj = roles.get(rname) or {} + if not isinstance(robj, dict): + continue + mf, md, ex, _nt = _role_common_counts(robj) + extra: Dict[str, Any] = {} + if rname == "extra_paths": + extra = { + "include_patterns": robj.get("include_patterns") or [], + "exclude_patterns": robj.get("exclude_patterns") or [], + } + role_summaries.append( + { + "role": rname, + "summary": f"{mf} file(s), {md} dir(s), {ex} excluded", + "notes": robj.get("notes") or [], + **extra, + } + ) + + # Flatten managed/excluded across roles + all_managed_files: List[Dict[str, Any]] = [] + all_managed_dirs: List[Dict[str, Any]] = [] + all_excluded: List[Dict[str, Any]] = [] + + def _consume_role(role_obj: Dict[str, Any]) -> None: + for f in role_obj.get("managed_files") or []: + if isinstance(f, dict): + all_managed_files.append(f) + for d in role_obj.get("managed_dirs") or []: + if isinstance(d, dict): + all_managed_dirs.append(d) + for e in role_obj.get("excluded") or []: + if isinstance(e, dict): + all_excluded.append(e) + + if isinstance(users_obj, dict): + _consume_role(users_obj) + if isinstance(services_list, list): + for s in services_list: + if isinstance(s, dict): + _consume_role(s) + if isinstance(pkgs_list, list): + for p in pkgs_list: + if isinstance(p, dict): + _consume_role(p) + for rname in [ + "apt_config", + "dnf_config", + "sysctl", + "etc_custom", + "usr_local_custom", + "extra_paths", + ]: + robj = roles.get(rname) + if isinstance(robj, dict): + _consume_role(robj) + + managed_file_reasons = _summarize_reasons( + all_managed_files, + "reason", + mapping=_MANAGED_FILE_REASONS, + max_examples=max_examples, + ) + managed_dir_reasons = _summarize_reasons( + all_managed_dirs, + "reason", + mapping=_MANAGED_DIR_REASONS, + max_examples=max_examples, + ) + excluded_reasons = _summarize_reasons( + all_excluded, + "reason", + mapping=_EXCLUDED_REASONS, + max_examples=max_examples, + ) + + # Inventory observed_via breakdown (count packages that contain at least one entry for that kind) + observed_kinds: Counter[str] = Counter() + observed_refs: Dict[str, Counter[str]] = defaultdict(Counter) + for _pkg, entry in inv_pkgs.items(): + if not isinstance(entry, dict): + continue + seen_kinds = set() + for ov in entry.get("observed_via") or []: + if not isinstance(ov, dict): + continue + kind = ov.get("kind") + if not kind: + continue + kind = str(kind) + seen_kinds.add(kind) + ref = ov.get("ref") + if isinstance(ref, str) and ref: + observed_refs[kind][ref] += 1 + for k in seen_kinds: + observed_kinds[k] += 1 + + observed_via_summary: List[Dict[str, Any]] = [] + for kind, cnt in observed_kinds.most_common(): + info = _ri(_OBSERVED_VIA, kind) + top_refs = [ + r for r, _ in observed_refs.get(kind, Counter()).most_common(max_examples) + ] + observed_via_summary.append( + { + "kind": kind, + "count": cnt, + "title": info.title, + "why": info.why, + "top_refs": top_refs, + } + ) + + report: Dict[str, Any] = { + "bundle_dir": str(bundle.dir), + "host": host, + "enroll": enroll, + "inventory": { + "package_count": len(inv_pkgs), + "observed_via": observed_via_summary, + }, + "roles": role_summaries, + "reasons": { + "managed_files": managed_file_reasons, + "managed_dirs": managed_dir_reasons, + "excluded": excluded_reasons, + }, + } + + if fmt == "json": + return json.dumps(report, indent=2, sort_keys=True) + + # Text rendering + out: List[str] = [] + out.append(f"Enroll explained: {harvest}") + hn = host.get("hostname") or "(unknown host)" + os_family = host.get("os") or "unknown" + pkg_backend = host.get("pkg_backend") or "?" + ver = enroll.get("version") or "?" + out.append(f"Host: {hn} (os: {os_family}, pkg: {pkg_backend})") + out.append(f"Enroll: {ver}") + out.append("") + + out.append("Inventory") + out.append(f"- Packages: {len(inv_pkgs)}") + if observed_via_summary: + out.append("- Why packages were included (observed_via):") + for ov in observed_via_summary: + extra = "" + if ov.get("top_refs"): + extra = f" (e.g. {', '.join(ov['top_refs'])})" + out.append(f" - {ov['kind']}: {ov['count']} – {ov['why']}{extra}") + out.append("") + + out.append("Roles collected") + for rs in role_summaries: + out.append(f"- {rs['role']}: {rs['summary']}") + if rs["role"] == "extra_paths": + inc = rs.get("include_patterns") or [] + exc = rs.get("exclude_patterns") or [] + if inc: + suffix = "…" if len(inc) > max_examples else "" + out.append( + f" include_patterns: {', '.join(map(str, inc[:max_examples]))}{suffix}" + ) + if exc: + suffix = "…" if len(exc) > max_examples else "" + out.append( + f" exclude_patterns: {', '.join(map(str, exc[:max_examples]))}{suffix}" + ) + notes = rs.get("notes") or [] + if notes: + for n in notes[:max_examples]: + out.append(f" note: {n}") + if len(notes) > max_examples: + out.append( + f" note: (+{len(notes) - max_examples} more. Use --format json to see them all)" + ) + out.append("") + + out.append("Why files were included (managed_files.reason)") + if managed_file_reasons: + for r in managed_file_reasons[:15]: + exs = r.get("examples") or [] + ex_txt = f" Examples: {', '.join(exs)}" if exs else "" + out.append(f"- {r['reason']} ({r['count']}): {r['why']}.{ex_txt}") + if len(managed_file_reasons) > 15: + out.append( + f"- (+{len(managed_file_reasons) - 15} more reasons. Use --format json to see them all)" + ) + else: + out.append("- (no managed files)") + + if managed_dir_reasons: + out.append("") + out.append("Why directories were included (managed_dirs.reason)") + for r in managed_dir_reasons: + out.append(f"- {r['reason']} ({r['count']}): {r['why']}") + + out.append("") + out.append("Why paths were excluded") + if excluded_reasons: + for r in excluded_reasons: + exs = r.get("examples") or [] + ex_txt = f" Examples: {', '.join(exs)}" if exs else "" + out.append(f"- {r['reason']} ({r['count']}): {r['why']}.{ex_txt}") + else: + out.append("- (no excluded paths)") + + return "\n".join(out) + "\n" diff --git a/enroll/fsutil.py b/enroll/fsutil.py new file mode 100644 index 0000000..c852b9e --- /dev/null +++ b/enroll/fsutil.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import hashlib +import os +from typing import Tuple + + +def file_md5(path: str) -> str: + """Return hex MD5 of a file. + + Used for Debian dpkg baseline comparisons. + """ + h = hashlib.md5() # nosec + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def stat_triplet(path: str) -> Tuple[str, str, str]: + """Return (owner, group, mode) for a path. + + owner/group are usernames/group names when resolvable, otherwise numeric ids. + mode is a zero-padded octal string (e.g. "0644"). + """ + st = os.stat(path, follow_symlinks=True) + mode = oct(st.st_mode & 0o7777)[2:].zfill(4) + + import grp + import pwd + + try: + owner = pwd.getpwuid(st.st_uid).pw_name + except KeyError: + owner = str(st.st_uid) + try: + group = grp.getgrgid(st.st_gid).gr_name + except KeyError: + group = str(st.st_gid) + return owner, group, mode diff --git a/enroll/harvest.py b/enroll/harvest.py index 78f7d1f..3454825 100644 --- a/enroll/harvest.py +++ b/enroll/harvest.py @@ -1,575 +1,668 @@ from __future__ import annotations -import glob -import json import os import re import shutil -from dataclasses import dataclass, asdict -from typing import Dict, List, Optional, Set +import shlex +import stat +import subprocess # nosec +import time +from dataclasses import asdict +from typing import Any, Dict, List, Optional, Set, Tuple -from .systemd import list_enabled_services, get_unit_info, UnitQueryError -from .debian import ( - build_dpkg_etc_index, - dpkg_owner, - file_md5, - list_manual_packages, - parse_status_conffiles, - read_pkg_md5sums, - stat_triplet, -) +from . import accounts as _accounts +from . import systemd as _systemd +from .fsutil import stat_triplet +from .platform import detect_platform, get_backend from .ignore import IgnorePolicy -from .accounts import collect_non_system_users +from .pathfilter import PathFilter +from .version import get_enroll_version +from .state import write_state +from .harvest_collectors.context import HarvestContext +from .harvest_types import ( + EtcCustomSnapshot, + ExcludedFile, + FirewallRuntimeSnapshot, + ManagedDir, + ManagedFile, + PackageSnapshot, + ServiceSnapshot, + SysctlSnapshot, +) + +from .capture import capture_file +from . import system_paths +from .package_hints import package_section_from_installations, safe_name + +UnitQueryError = _systemd.UnitQueryError -@dataclass -class ManagedFile: - path: str - src_rel: str - owner: str - group: str - mode: str - reason: str +def list_enabled_services() -> List[str]: + return _systemd.list_enabled_services() -@dataclass -class ExcludedFile: - path: str - reason: str +def list_enabled_timers() -> List[str]: + return _systemd.list_enabled_timers() -@dataclass -class ServiceSnapshot: - unit: str - role_name: str - packages: List[str] - active_state: Optional[str] - sub_state: Optional[str] - unit_file_state: Optional[str] - condition_result: Optional[str] - managed_files: List[ManagedFile] - excluded: List[ExcludedFile] - notes: List[str] +def get_unit_info(unit: str) -> Any: + return _systemd.get_unit_info(unit) -@dataclass -class PackageSnapshot: - package: str - role_name: str - managed_files: List[ManagedFile] - excluded: List[ExcludedFile] - notes: List[str] +def get_timer_info(timer: str) -> Any: + return _systemd.get_timer_info(timer) -@dataclass -class UsersSnapshot: - role_name: str - users: List[dict] - managed_files: List[ManagedFile] - excluded: List[ExcludedFile] - notes: List[str] +def collect_non_system_users() -> List[Any]: + return _accounts.collect_non_system_users() -@dataclass -class EtcCustomSnapshot: - role_name: str - managed_files: List[ManagedFile] - excluded: List[ExcludedFile] - notes: List[str] +def _merge_parent_dirs( + existing_dirs: List[ManagedDir], + managed_files: List[ManagedFile], + *, + policy: IgnorePolicy, + extra_paths: Optional[List[str]] = None, +) -> List[ManagedDir]: + """Ensure parent directories for managed_files are present in managed_dirs. + + This is used so the Ansible manifest can create destination directories with + explicit owner/group/mode (ansible-lint friendly) without needing a separate + "mkdir without perms" task. + + We only add the immediate parent directory for each managed file. For + explicit directory includes (extra_paths), existing_dirs will already + contain the walked directory tree. + """ + by_path: Dict[str, ManagedDir] = { + d.path: d for d in (existing_dirs or []) if d.path + } + + def _iter_paths() -> List[str]: + paths: List[str] = [] + for mf in managed_files or []: + if mf and mf.path: + paths.append(str(mf.path)) + for p in extra_paths or []: + if p: + paths.append(str(p)) + return paths + + for p0 in _iter_paths(): + p = str(p0 or "").rstrip("/") + if not p: + continue + dpath = os.path.dirname(p) + if not dpath or dpath == "/": + continue + if dpath in by_path: + continue + + # Directory-deny logic: newer IgnorePolicy implementations provide + # deny_reason_dir(). Older/simple policies (including unit tests) may + # only implement deny_reason(), which is file-oriented and may return + # "not_regular_file" for directories. + deny = None + deny_dir = getattr(policy, "deny_reason_dir", None) + if callable(deny_dir): + deny = deny_dir(dpath) + else: + deny = policy.deny_reason(dpath) + if deny in ("not_regular_file", "not_file", "not_regular"): + deny = None + if deny: + # If the file itself was captured, its parent directory is likely safe, + # but still respect deny globs for directories to avoid managing + # sensitive/forbidden trees. + continue + + try: + owner, group, mode = stat_triplet(dpath) + except OSError: + continue + + by_path[dpath] = ManagedDir( + path=dpath, + owner=owner, + group=group, + mode=mode, + reason="parent_of_managed_file", + ) + + return [by_path[k] for k in sorted(by_path)] -ALLOWED_UNOWNED_EXTS = { - ".conf", - ".cfg", - ".ini", - ".cnf", - ".yaml", - ".yml", - ".json", - ".toml", - ".rules", - ".service", - ".socket", - ".timer", - ".target", - ".path", - ".mount", - ".network", - ".netdev", - ".link", - "", # allow extensionless (common in /etc/default and /etc/init.d) -} - -MAX_UNOWNED_FILES_PER_ROLE = 400 - -# Directories that are shared across many packages; never attribute unowned files in these trees to a single package. -SHARED_ETC_TOPDIRS = { - "default", - "apparmor.d", - "network", - "init.d", - "systemd", - "pam.d", - "ssh", - "ssl", - "sudoers.d", - "cron.d", - "cron.daily", - "cron.weekly", - "cron.monthly", - "cron.hourly", - "logrotate.d", - "sysctl.d", - "modprobe.d", +_FIREWALL_CAPTURE_COMMANDS: Dict[str, Tuple[str, ...]] = { + "ipset_save": ("ipset", "save"), + "iptables_v4_save": ("iptables-save",), + "iptables_v6_save": ("ip6tables-save",), + "sysctl_all": ("sysctl", "-a"), } -def _safe_name(s: str) -> str: - out: List[str] = [] - for ch in s: - out.append(ch if ch.isalnum() or ch in ("_", "-") else "_") - return "".join(out).replace("-", "_") +def _run_capture_command( + command_key: str, *, timeout: int = 10 +) -> tuple[Optional[str], Optional[str]]: + """Return (stdout, error_note) for an allowlisted local state command. + + The command key is resolved through ``_FIREWALL_CAPTURE_COMMANDS`` so this + helper never executes caller-supplied argv. Commands are run with + ``shell=False`` explicitly to avoid shell interpretation. + """ + argv = _FIREWALL_CAPTURE_COMMANDS.get(command_key) + if argv is None: + return None, f"Unknown capture command: {command_key}" + + exe = argv[0] + if shutil.which(exe) is None: + return None, f"{exe} not found on PATH." + + try: + proc = subprocess.run( # nosec + argv, + shell=False, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=timeout, + ) + except Exception as e: # noqa: BLE001 + return None, f"{' '.join(argv)} failed: {e!r}" + + if proc.returncode != 0: + stderr = (proc.stderr or "").strip() + if len(stderr) > 300: + stderr = stderr[:297] + "..." + return ( + None, + f"{' '.join(argv)} exited {proc.returncode}: {stderr or '(no stderr)'}", + ) + + return proc.stdout or "", None -def _role_id(raw: str) -> str: - # normalize separators first - s = re.sub(r"[^A-Za-z0-9]+", "_", raw) - # split CamelCase -> snake_case - s = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s) - s = s.lower() - s = re.sub(r"_+", "_", s).strip("_") - if not re.match(r"^[a-z_]", s): - s = "r_" + s - return s - - -def _role_name_from_unit(unit: str) -> str: - base = _role_id(unit.removesuffix(".service")) - return _safe_name(base) - - -def _role_name_from_pkg(pkg: str) -> str: - return _safe_name(pkg) - - -def _copy_into_bundle( - bundle_dir: str, role_name: str, abs_path: str, src_rel: str +def _write_generated_artifact( + bundle_dir: str, role_name: str, src_rel: str, content: str ) -> None: + """Write a generated harvest artifact that did not exist as a file on disk.""" dst = os.path.join(bundle_dir, "artifacts", role_name, src_rel) os.makedirs(os.path.dirname(dst), exist_ok=True) - shutil.copy2(abs_path, dst) + with open(dst, "w", encoding="utf-8") as f: + f.write(content) -def _is_confish(path: str) -> bool: - base = os.path.basename(path) - _, ext = os.path.splitext(base) - return ext in ALLOWED_UNOWNED_EXTS +_SYSCTL_KEY_RE = re.compile(r"^[A-Za-z0-9_.-]+$") +_SYSCTL_GENERATED_DEST = "/etc/sysctl.d/99-enroll.conf" +_SYSCTL_GENERATED_SRC_REL = "sysctl/99-enroll.conf" + +# Writable-looking action/identity keys that are poor candidates for persistent +# config. This avoids generating a file that tries to replay one-shot triggers or +# host identity that should be managed elsewhere (e.g. /etc/hostname). +_SYSCTL_VOLATILE_KEYS = { + "fs.binfmt_misc.status", + "kernel.domainname", + "kernel.hostname", + "kernel.kexec_load_disabled", + "kernel.kexec_load_limit_panic", + "kernel.kexec_load_limit_reboot", + "kernel.max_rcu_stall_to_panic", + "kernel.modules_disabled", + "kernel.ns_last_pid", + "net.ipv4.route.flush", + "net.ipv6.route.flush", + "vm.compact_memory", + "vm.drop_caches", + "vm.stat_refresh", +} + +_SYSCTL_VOLATILE_PREFIXES = ( + "fs.binfmt_misc.", + "kernel.sched_domain.", +) + +# These are paired with ratio/byte counterparts. The inactive side appears as 0 +# when read; replaying that 0 through sysctl -p is noisy and can be rejected by +# kernels that enforce minimum values. +_SYSCTL_SKIP_ZERO_VALUE_KEYS = { + "vm.dirty_background_bytes", + "vm.dirty_background_ratio", + "vm.dirty_bytes", + "vm.dirty_ratio", +} -def _hint_names(unit: str, pkgs: Set[str]) -> Set[str]: - base = unit.removesuffix(".service") - hints = {base} - if "@" in base: - hints.add(base.split("@", 1)[0]) - hints |= set(pkgs) - hints |= {h.split(".", 1)[0] for h in list(hints) if "." in h} - return {h for h in hints if h} +def _sysctl_proc_path(key: str) -> str: + return "/proc/sys/" + key.replace(".", "/") -def _add_pkgs_from_etc_topdirs( - hints: Set[str], topdir_to_pkgs: Dict[str, Set[str]], pkgs: Set[str] -) -> None: - for h in hints: - for p in topdir_to_pkgs.get(h, set()): - pkgs.add(p) +def _sysctl_key_is_persistable(key: str) -> tuple[bool, str]: + if not key or not _SYSCTL_KEY_RE.fullmatch(key): + return False, "invalid key" + if key in _SYSCTL_VOLATILE_KEYS or any( + key.startswith(prefix) for prefix in _SYSCTL_VOLATILE_PREFIXES + ): + return False, "volatile/action key" + + proc_path = _sysctl_proc_path(key) + try: + st = os.stat(proc_path) + except OSError: + return False, "no /proc/sys entry" + + if not stat.S_ISREG(st.st_mode): + return False, "not a regular /proc/sys entry" + if (stat.S_IMODE(st.st_mode) & 0o222) == 0: + return False, "read-only /proc/sys entry" + return True, "" -def _maybe_add_specific_paths(hints: Set[str]) -> List[str]: - paths: List[str] = [] - for h in hints: - paths.extend( - [ - f"/etc/default/{h}", - f"/etc/init.d/{h}", - f"/etc/sysctl.d/{h}.conf", - f"/etc/logrotate.d/{h}", - ] - ) - return paths +def _sysctl_entry_is_persistable(key: str, value: str) -> tuple[bool, str]: + ok, reason = _sysctl_key_is_persistable(key) + if not ok: + return ok, reason + + if key in _SYSCTL_SKIP_ZERO_VALUE_KEYS and str(value).strip() == "0": + return False, "inactive mutually-exclusive zero value" + + return True, "" -def _scan_unowned_under_roots( - roots: List[str], owned_etc: Set[str], limit: int = MAX_UNOWNED_FILES_PER_ROLE -) -> List[str]: - found: List[str] = [] - for root in roots: - if not os.path.isdir(root): +def _parse_sysctl_a_output( + text: str, + *, + require_persistable: bool = True, +) -> tuple[Dict[str, str], Dict[str, int]]: + """Parse `sysctl -a` output into persistable key/value pairs. + + `sysctl -a` includes read-only, write-only, multiline, action-like, and + host-identity values. Persisting those can create noisy or failing Ansible + runs, so the default parser keeps only single-line writable-looking keys. + """ + + out: Dict[str, str] = {} + skipped: Dict[str, int] = { + "malformed": 0, + "empty_value": 0, + "non_persistable": 0, + "duplicate": 0, + } + + for raw in (text or "").splitlines(): + line = raw.strip() + if not line: continue - for dirpath, _, filenames in os.walk(root): - if len(found) >= limit: - return found - for fn in filenames: - if len(found) >= limit: - return found - p = os.path.join(dirpath, fn) - if not p.startswith("/etc/"): - continue - if p in owned_etc: - continue - if not os.path.isfile(p) or os.path.islink(p): - continue - if not _is_confish(p): - continue - found.append(p) - return found + if " = " in line: + key, value = line.split(" = ", 1) + elif "=" in line: + key, value = line.split("=", 1) + else: + skipped["malformed"] += 1 + continue + + key = key.strip() + value = value.strip() + if not key: + skipped["malformed"] += 1 + continue + if value == "": + skipped["empty_value"] += 1 + continue + if key in out: + skipped["duplicate"] += 1 + continue + if require_persistable: + ok, _reason = _sysctl_entry_is_persistable(key, value) + if not ok: + skipped["non_persistable"] += 1 + continue + out[key] = value + + return dict(sorted(out.items())), skipped -def _topdirs_for_package(pkg: str, pkg_to_etc_paths: Dict[str, List[str]]) -> Set[str]: - topdirs: Set[str] = set() - for path in pkg_to_etc_paths.get(pkg, []): - parts = path.split("/", 3) - if len(parts) >= 3 and parts[1] == "etc" and parts[2]: - topdirs.add(parts[2]) - return topdirs +def _render_sysctl_conf(parameters: Dict[str, str], notes: List[str]) -> str: + lines = [ + "# Generated by Enroll from live sysctl state.", + "# Review before applying broadly; runtime sysctl state can be host/kernel-specific.", + ] + for note in notes: + lines.append(f"# {note}") + lines.append("") + for key, value in sorted((parameters or {}).items()): + safe_value = str(value).replace("\n", " ").strip() + lines.append(f"{key} = {safe_value}") + lines.append("") + return "\n".join(lines) -def harvest(bundle_dir: str, policy: Optional[IgnorePolicy] = None) -> str: - policy = policy or IgnorePolicy() +def _collect_sysctl_snapshot(bundle_dir: str) -> SysctlSnapshot: + role_name = "sysctl" + notes: List[str] = [] + managed_files: List[ManagedFile] = [] + + out, err = _run_capture_command("sysctl_all", timeout=20) + if err: + notes.append(err) + return SysctlSnapshot(role_name=role_name, notes=notes) + + parameters, skipped = _parse_sysctl_a_output(out or "") + if not parameters: + notes.append("No persistable live sysctl parameters were detected.") + return SysctlSnapshot(role_name=role_name, parameters=parameters, notes=notes) + + notes.append(f"Captured {len(parameters)} live writable sysctl parameter(s).") + skipped_total = sum(skipped.values()) + if skipped_total: + details = ", ".join(f"{k}={v}" for k, v in sorted(skipped.items()) if v) + notes.append( + "Skipped " + f"{skipped_total} sysctl entr{'y' if skipped_total == 1 else 'ies'} " + f"that were not suitable for persistence ({details})." + ) + + _write_generated_artifact( + bundle_dir, + role_name, + _SYSCTL_GENERATED_SRC_REL, + _render_sysctl_conf(parameters, notes), + ) + managed_files.append( + ManagedFile( + path=_SYSCTL_GENERATED_DEST, + src_rel=_SYSCTL_GENERATED_SRC_REL, + owner="root", + group="root", + mode="0644", + reason="system_sysctl", + ) + ) + return SysctlSnapshot( + role_name=role_name, + managed_files=managed_files, + parameters=parameters, + notes=notes, + ) + + +def _ipset_save_has_state(text: str) -> bool: + for raw in (text or "").splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + if line.startswith(("create ", "add ")): + return True + return False + + +def _parse_ipset_set_names(text: str) -> List[str]: + names: List[str] = [] + seen: Set[str] = set() + for raw in (text or "").splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + try: + toks = shlex.split(line) + except ValueError: + toks = line.split() + if len(toks) >= 2 and toks[0] == "create" and toks[1] not in seen: + seen.add(toks[1]) + names.append(toks[1]) + return names + + +def _iptables_save_has_state(text: str) -> bool: + """Return True when iptables-save output contains non-default state.""" + for raw in (text or "").splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + if line.startswith("*") or line == "COMMIT": + continue + if line.startswith(":"): + parts = line.split() + chain_name = parts[0][1:] if parts else "" + policy = parts[1] if len(parts) >= 2 else "" + # Built-in empty chains usually look like ':INPUT ACCEPT [0:0]'. + # A changed policy, or any custom chain, is meaningful state. + if policy not in ("ACCEPT", "-"): + return True + if policy == "-" and chain_name: + return True + continue + if line.startswith(("-A ", "-I ", "-N ", "-P ", "-R ")): + return True + return False + + +def _collect_firewall_runtime_snapshot( + bundle_dir: str, + *, + persistent_ipset_files: Optional[List[str]] = None, + persistent_iptables_v4_files: Optional[List[str]] = None, + persistent_iptables_v6_files: Optional[List[str]] = None, +) -> FirewallRuntimeSnapshot: + """Capture live kernel firewall state only when no persistent config exists. + + Enroll also harvests persistent firewall files such as + /etc/iptables/rules.v4, /etc/iptables/rules.v6, and /etc/ipset.conf as + managed files. The generated runtime restore role is therefore a fallback: + it captures each firewall family only when that family has no persistent + file to avoid generating two roles that try to manage the same state. + """ + role_name = "firewall_runtime" + packages: Set[str] = set() + notes: List[str] = [] + ipset_save_rel: Optional[str] = None + ipset_sets: List[str] = [] + iptables_v4_rel: Optional[str] = None + iptables_v6_rel: Optional[str] = None + + persistent_ipset_files = persistent_ipset_files or [] + persistent_iptables_v4_files = persistent_iptables_v4_files or [] + persistent_iptables_v6_files = persistent_iptables_v6_files or [] + + if persistent_ipset_files: + notes.append( + "Live ipset runtime capture skipped because persistent ipset " + f"configuration was found: {', '.join(persistent_ipset_files)}" + ) + else: + ipset_out, ipset_err = _run_capture_command("ipset_save") + if ipset_err: + notes.append(ipset_err) + elif ipset_out is not None and _ipset_save_has_state(ipset_out): + ipset_save_rel = "firewall/ipset.save" + _write_generated_artifact(bundle_dir, role_name, ipset_save_rel, ipset_out) + ipset_sets = _parse_ipset_set_names(ipset_out) + packages.add("ipset") + + if persistent_iptables_v4_files: + notes.append( + "Live IPv4 iptables runtime capture skipped because persistent " + f"IPv4 iptables configuration was found: {', '.join(persistent_iptables_v4_files)}" + ) + else: + ipt4_out, ipt4_err = _run_capture_command("iptables_v4_save") + if ipt4_err: + notes.append(ipt4_err) + elif ipt4_out is not None and _iptables_save_has_state(ipt4_out): + iptables_v4_rel = "firewall/iptables.v4" + _write_generated_artifact(bundle_dir, role_name, iptables_v4_rel, ipt4_out) + packages.add("iptables") + + if persistent_iptables_v6_files: + notes.append( + "Live IPv6 iptables runtime capture skipped because persistent " + f"IPv6 iptables configuration was found: {', '.join(persistent_iptables_v6_files)}" + ) + else: + ipt6_out, ipt6_err = _run_capture_command("iptables_v6_save") + if ipt6_err: + notes.append(ipt6_err) + elif ipt6_out is not None and _iptables_save_has_state(ipt6_out): + iptables_v6_rel = "firewall/iptables.v6" + _write_generated_artifact(bundle_dir, role_name, iptables_v6_rel, ipt6_out) + packages.add("iptables") + + # Package names are intentionally added only when matching live state was + # captured. Merely having iptables/ipset installed should not create a role. + + return FirewallRuntimeSnapshot( + role_name=role_name, + packages=sorted(packages), + ipset_save=ipset_save_rel, + ipset_sets=ipset_sets, + iptables_v4_save=iptables_v4_rel, + iptables_v6_save=iptables_v6_rel, + notes=notes, + ) + + +def harvest( + bundle_dir: str, + policy: Optional[IgnorePolicy] = None, + *, + dangerous: bool = False, + include_paths: Optional[List[str]] = None, + exclude_paths: Optional[List[str]] = None, +) -> str: + # If a policy is not supplied, build one. `--dangerous` relaxes secret + # detection and deny-glob skipping. + if policy is None: + policy = IgnorePolicy(dangerous=dangerous) + elif dangerous: + # If callers explicitly provided a policy but also requested + # dangerous behaviour, honour the CLI intent. + policy.dangerous = True os.makedirs(bundle_dir, exist_ok=True) + # User-provided includes/excludes. Excludes apply to all harvesting; + # includes are harvested into an extra role. + path_filter = PathFilter(include=include_paths or (), exclude=exclude_paths or ()) + + from .harvest_collectors.container_images import ContainerImagesCollector + from .harvest_collectors.cron_logrotate import CronLogrotateCollector + from .harvest_collectors.package_manager import PackageManagerConfigCollector + from .harvest_collectors.paths import ExtraPathsCollector, UsrLocalCustomCollector + from .harvest_collectors.runtime import RuntimeStateCollector + from .harvest_collectors.services import ServicePackageCollector + from .harvest_collectors.users import UsersCollector + if hasattr(os, "geteuid") and os.geteuid() != 0: print( "Warning: not running as root; harvest may miss files or metadata.", flush=True, ) - owned_etc, etc_owner_map, topdir_to_pkgs, pkg_to_etc_paths = build_dpkg_etc_index() - conffiles_by_pkg = parse_status_conffiles() + platform = detect_platform() + backend = get_backend(platform) - # ------------------------- - # Service roles - # ------------------------- - service_snaps: List[ServiceSnapshot] = [] - for unit in list_enabled_services(): - role = _role_name_from_unit(unit) - - try: - ui = get_unit_info(unit) - except UnitQueryError as e: - service_snaps.append( - ServiceSnapshot( - unit=unit, - role_name=role, - packages=[], - active_state=None, - sub_state=None, - unit_file_state=None, - condition_result=None, - managed_files=[], - excluded=[], - notes=[str(e)], - ) - ) - continue - - pkgs: Set[str] = set() - notes: List[str] = [] - excluded: List[ExcludedFile] = [] - managed: List[ManagedFile] = [] - candidates: Dict[str, str] = {} - - if ui.fragment_path: - p = dpkg_owner(ui.fragment_path) - if p: - pkgs.add(p) - - for exe in ui.exec_paths: - p = dpkg_owner(exe) - if p: - pkgs.add(p) - - for pth in ui.dropin_paths: - if pth.startswith("/etc/"): - candidates[pth] = "systemd_dropin" - - for ef in ui.env_files: - ef = ef.lstrip("-") - if any(ch in ef for ch in "*?["): - for g in glob.glob(ef): - if g.startswith("/etc/") and os.path.isfile(g): - candidates[g] = "systemd_envfile" - else: - if ef.startswith("/etc/") and os.path.isfile(ef): - candidates[ef] = "systemd_envfile" - - hints = _hint_names(unit, pkgs) - _add_pkgs_from_etc_topdirs(hints, topdir_to_pkgs, pkgs) - - for sp in _maybe_add_specific_paths(hints): - if not os.path.exists(sp): - continue - if sp in etc_owner_map: - pkgs.add(etc_owner_map[sp]) - else: - candidates.setdefault(sp, "custom_specific_path") - - for pkg in sorted(pkgs): - conff = conffiles_by_pkg.get(pkg, {}) - md5sums = read_pkg_md5sums(pkg) - for path in pkg_to_etc_paths.get(pkg, []): - if not os.path.isfile(path) or os.path.islink(path): - continue - if path in conff: - # Only capture conffiles when they differ from the package default. - try: - current = file_md5(path) - except OSError: - continue - if current != conff[path]: - candidates.setdefault(path, "modified_conffile") - continue - rel = path.lstrip("/") - baseline = md5sums.get(rel) - if baseline: - try: - current = file_md5(path) - except OSError: - continue - if current != baseline: - candidates.setdefault(path, "modified_packaged_file") - - roots: List[str] = [] - for h in hints: - roots.extend([f"/etc/{h}", f"/etc/{h}.d"]) - for pth in _scan_unowned_under_roots(roots, owned_etc): - candidates.setdefault(pth, "custom_unowned") - - if not pkgs and not candidates: - notes.append( - "No packages or /etc candidates detected (unexpected for enabled service)." - ) - - for path, reason in sorted(candidates.items()): - deny = policy.deny_reason(path) - if deny: - excluded.append(ExcludedFile(path=path, reason=deny)) - continue - try: - owner, group, mode = stat_triplet(path) - except OSError: - excluded.append(ExcludedFile(path=path, reason="unreadable")) - continue - src_rel = path.lstrip("/") - try: - _copy_into_bundle(bundle_dir, role, path, src_rel) - except OSError: - excluded.append(ExcludedFile(path=path, reason="unreadable")) - continue - managed.append( - ManagedFile( - path=path, - src_rel=src_rel, - owner=owner, - group=group, - mode=mode, - reason=reason, - ) - ) - - service_snaps.append( - ServiceSnapshot( - unit=unit, - role_name=role, - packages=sorted(pkgs), - active_state=ui.active_state, - sub_state=ui.sub_state, - unit_file_state=ui.unit_file_state, - condition_result=ui.condition_result, - managed_files=managed, - excluded=excluded, - notes=notes, - ) - ) - - # ------------------------- - # Manually installed package roles - # ------------------------- - manual_pkgs = list_manual_packages() - # Avoid duplicate roles: if a manual package is already managed by any service role, skip its pkg_ role. - covered_by_services: Set[str] = set() - for s in service_snaps: - for p in s.packages: - covered_by_services.add(p) - - manual_pkgs_skipped: List[str] = [] - pkg_snaps: List[PackageSnapshot] = [] - - for pkg in manual_pkgs: - if pkg in covered_by_services: - manual_pkgs_skipped.append(pkg) - continue - role = _role_name_from_pkg(pkg) - notes: List[str] = [] - excluded: List[ExcludedFile] = [] - managed: List[ManagedFile] = [] - candidates: Dict[str, str] = {} - - conff = conffiles_by_pkg.get(pkg, {}) - md5sums = read_pkg_md5sums(pkg) - - for path in pkg_to_etc_paths.get(pkg, []): - if not os.path.isfile(path) or os.path.islink(path): - continue - if path in conff: - try: - current = file_md5(path) - except OSError: - continue - if current != conff[path]: - candidates.setdefault(path, "modified_conffile") - continue - rel = path.lstrip("/") - baseline = md5sums.get(rel) - if baseline: - try: - current = file_md5(path) - except OSError: - continue - if current != baseline: - candidates.setdefault(path, "modified_packaged_file") - - topdirs = _topdirs_for_package(pkg, pkg_to_etc_paths) - roots: List[str] = [] - for td in sorted(topdirs): - if td in SHARED_ETC_TOPDIRS: - continue - roots.extend([f"/etc/{td}", f"/etc/{td}.d"]) - roots.extend([f"/etc/default/{td}"]) - roots.extend([f"/etc/init.d/{td}"]) - roots.extend([f"/etc/logrotate.d/{td}"]) - roots.extend([f"/etc/sysctl.d/{td}.conf"]) - - for pth in _scan_unowned_under_roots( - [r for r in roots if os.path.isdir(r)], owned_etc - ): - candidates.setdefault(pth, "custom_unowned") - - for r in roots: - if os.path.isfile(r) and not os.path.islink(r): - if r not in owned_etc and _is_confish(r): - candidates.setdefault(r, "custom_specific_path") - - for path, reason in sorted(candidates.items()): - deny = policy.deny_reason(path) - if deny: - excluded.append(ExcludedFile(path=path, reason=deny)) - continue - try: - owner, group, mode = stat_triplet(path) - except OSError: - excluded.append(ExcludedFile(path=path, reason="unreadable")) - continue - src_rel = path.lstrip("/") - try: - _copy_into_bundle(bundle_dir, role, path, src_rel) - except OSError: - excluded.append(ExcludedFile(path=path, reason="unreadable")) - continue - managed.append( - ManagedFile( - path=path, - src_rel=src_rel, - owner=owner, - group=group, - mode=mode, - reason=reason, - ) - ) - - if not pkg_to_etc_paths.get(pkg, []) and not managed: - notes.append("No /etc files detected for this package.") - - pkg_snaps.append( - PackageSnapshot( - package=pkg, - role_name=role, - managed_files=managed, - excluded=excluded, - notes=notes, - ) - ) - - # ------------------------- - # Users role (non-system users) - # ------------------------- - users_notes: List[str] = [] - users_excluded: List[ExcludedFile] = [] - users_managed: List[ManagedFile] = [] - users_list: List[dict] = [] - - try: - user_records = collect_non_system_users() - except Exception as e: - user_records = [] - users_notes.append(f"Failed to enumerate users: {e!r}") - - users_role_name = "users" - - for u in user_records: - users_list.append( - { - "name": u.name, - "uid": u.uid, - "gid": u.gid, - "gecos": u.gecos, - "home": u.home, - "shell": u.shell, - "primary_group": u.primary_group, - "supplementary_groups": u.supplementary_groups, - } - ) - - # Copy only safe SSH public material: authorized_keys + *.pub - for sf in u.ssh_files: - deny = policy.deny_reason(sf) - if deny: - users_excluded.append(ExcludedFile(path=sf, reason=deny)) - continue - try: - owner, group, mode = stat_triplet(sf) - except OSError: - users_excluded.append(ExcludedFile(path=sf, reason="unreadable")) - continue - src_rel = sf.lstrip("/") - try: - _copy_into_bundle(bundle_dir, users_role_name, sf, src_rel) - except OSError: - users_excluded.append(ExcludedFile(path=sf, reason="unreadable")) - continue - reason = ( - "authorized_keys" - if sf.endswith("/authorized_keys") - else "ssh_public_key" - ) - users_managed.append( - ManagedFile( - path=sf, - src_rel=src_rel, - owner=owner, - group=group, - mode=mode, - reason=reason, - ) - ) - - users_snapshot = UsersSnapshot( - role_name=users_role_name, - users=users_list, - managed_files=users_managed, - excluded=users_excluded, - notes=users_notes, + owned_etc, etc_owner_map, topdir_to_pkgs, pkg_to_etc_paths = ( + backend.build_etc_index() ) + # Global de-duplication across roles: each absolute path is captured at most once. + # This avoids multiple Ansible roles managing the same destination file. + captured_global: Set[str] = set() + + # ------------------------- + # Cron / logrotate unification + # + # If cron/logrotate are installed, capture all related configuration/state into + # dedicated package roles ("cron" and "logrotate") so the same destination path + # is never managed by unrelated roles. + # + # This includes user-specific crontabs under /var/spool, which means the cron role + # should be applied after users have been created (handled in manifest ordering). + # ------------------------- + + installed_pkgs = backend.installed_packages() or {} + installed_names: Set[str] = set(installed_pkgs.keys()) + + persistent_ipset_files = system_paths.persistent_firewall_files( + system_paths.persistent_ipset_globs() + ) + persistent_iptables_v4_files = system_paths.persistent_firewall_files( + system_paths.persistent_iptables_v4_globs() + ) + persistent_iptables_v6_files = system_paths.persistent_firewall_files( + system_paths.persistent_iptables_v6_globs() + ) + + context = HarvestContext( + bundle_dir=bundle_dir, + policy=policy, + path_filter=path_filter, + platform=platform, + backend=backend, + installed_pkgs=installed_pkgs, + installed_names=installed_names, + owned_etc=owned_etc, + etc_owner_map=etc_owner_map, + topdir_to_pkgs=topdir_to_pkgs, + pkg_to_etc_paths=pkg_to_etc_paths, + captured_global=captured_global, + ) + + runtime_collection = RuntimeStateCollector( + context, + persistent_ipset_files=persistent_ipset_files, + persistent_iptables_v4_files=persistent_iptables_v4_files, + persistent_iptables_v6_files=persistent_iptables_v6_files, + ).collect() + firewall_runtime_snapshot = runtime_collection.firewall_runtime_snapshot + sysctl_snapshot = runtime_collection.sysctl_snapshot + + # The generated sysctl role owns /etc/sysctl.d/99-enroll.conf; do not also + # capture an existing file at that path into etc_custom/package roles. + for mf in sysctl_snapshot.managed_files: + captured_global.add(mf.path) + + cron_logrotate_collection = CronLogrotateCollector(context).collect() + cron_pkg = cron_logrotate_collection.cron_pkg + logrotate_pkg = cron_logrotate_collection.logrotate_pkg + cron_snapshot = cron_logrotate_collection.cron_snapshot + logrotate_snapshot = cron_logrotate_collection.logrotate_snapshot + + service_package_collection = ServicePackageCollector( + context, + cron_snapshot=cron_snapshot, + logrotate_snapshot=logrotate_snapshot, + cron_pkg=cron_pkg, + logrotate_pkg=logrotate_pkg, + ).collect() + service_snaps = service_package_collection.service_snaps + pkg_snaps = service_package_collection.pkg_snaps + manual_pkgs = service_package_collection.manual_pkgs + service_role_aliases = service_package_collection.service_role_aliases + seen_by_role = service_package_collection.seen_by_role + + # ------------------------- + # Users role, Flatpak and Snap state + # ------------------------- + users_collection = UsersCollector(context, seen_by_role).collect() + users_snapshot = users_collection.users_snapshot + flatpak_snapshot = users_collection.flatpak_snapshot + snap_snapshot = users_collection.snap_snapshot + + # ------------------------- + # Container image inventory (Docker/Podman image caches) + # ------------------------- + container_images_snapshot = ContainerImagesCollector(context).collect() + + # ------------------------- + # Package manager config role + # - Debian: apt_config + # - Fedora/RHEL-like: dnf_config + # ------------------------- + package_manager_config = PackageManagerConfigCollector( + context, seen_by_role + ).collect() + apt_config_snapshot = package_manager_config.apt_config_snapshot + dnf_config_snapshot = package_manager_config.dnf_config_snapshot + # ------------------------- # etc_custom role (unowned /etc files not already attributed elsewhere) # ------------------------- @@ -578,66 +671,196 @@ def harvest(bundle_dir: str, policy: Optional[IgnorePolicy] = None) -> str: etc_managed: List[ManagedFile] = [] etc_role_name = "etc_custom" - # Build a set of files already captured by other roles. - already: Set[str] = set() - for s in service_snaps: - for mf in s.managed_files: - already.add(mf.path) - for p in pkg_snaps: - for mf in p.managed_files: - already.add(mf.path) - for mf in users_managed: - already.add(mf.path) + # Files already captured by earlier roles. Use the global set so we never + # end up with the same destination path managed by multiple roles. + already: Set[str] = captured_global - # Walk /etc for unowned config-ish files + # Maps for re-attributing shared snippets (cron.d/logrotate.d) to existing roles. + svc_by_role: Dict[str, ServiceSnapshot] = {s.role_name: s for s in service_snaps} + pkg_by_role: Dict[str, PackageSnapshot] = {p.role_name: p for p in pkg_snaps} + + # Package name -> role_name for manually-installed package roles. + pkg_name_to_role: Dict[str, str] = {p.package: p.role_name for p in pkg_snaps} + + # Package name -> list of service role names that reference it. + pkg_to_service_roles: Dict[str, List[str]] = {} + for s in service_snaps: + for pkg in s.packages: + pkg_to_service_roles.setdefault(pkg, []).append(s.role_name) + + # Alias -> role mapping used as a fallback when package ownership is missing. + # Prefer service roles over package roles when both would match. + alias_ranked: Dict[str, tuple[int, str]] = {} + + def _add_alias(alias: str, role_name: str, *, priority: int) -> None: + key = safe_name(alias) + if not key: + return + cur = alias_ranked.get(key) + if ( + cur is None + or priority < cur[0] + or (priority == cur[0] and role_name < cur[1]) + ): + alias_ranked[key] = (priority, role_name) + + for role_name, aliases in service_role_aliases.items(): + for a in aliases: + _add_alias(a, role_name, priority=0) + + for p in pkg_snaps: + _add_alias(p.package, p.role_name, priority=1) + + def _target_role_for_shared_snippet(path: str) -> Optional[tuple[str, str]]: + """If `path` is a shared snippet, return (role_name, reason) to attach to. + + This is used primarily for /etc/logrotate.d/* and /etc/cron.d/* where + files are "owned" by many packages but people tend to reason about them + per service. + + Resolution order: + 1) package owner -> service role (if any service references the package) + 2) package owner -> package role (manual package role exists) + 3) basename/stem alias match -> preferred role + """ + if path.startswith("/etc/logrotate.d/"): + tag = "logrotate_snippet" + elif path.startswith("/etc/cron.d/"): + tag = "cron_snippet" + else: + return None + + base = os.path.basename(path) + candidates: List[str] = [base] + if "." in base: + candidates.append(base.split(".", 1)[0]) + + seen: Set[str] = set() + uniq: List[str] = [] + for c in candidates: + if c and c not in seen: + seen.add(c) + uniq.append(c) + + pkg = backend.owner_of_path(path) + if pkg: + svc_roles = sorted(set(pkg_to_service_roles.get(pkg, []))) + if svc_roles: + # If multiple service roles reference the same package, prefer + # the role that most closely matches the snippet name (basename + # or stem). This avoids surprising attributions such as an + # AppArmor loader role "claiming" a cron/logrotate snippet + # that is clearly named after another package/service. + if len(svc_roles) > 1: + # Direct role-name matches first. + for c in [pkg, *uniq]: + rn = safe_name(c) + if rn in svc_roles: + return (rn, tag) + # Next, use the alias map if it points at one of the roles. + for c in [pkg, *uniq]: + hit = alias_ranked.get(safe_name(c)) + if hit is not None and hit[1] in svc_roles: + return (hit[1], tag) + + # Deterministic fallback: lowest role name. + return (svc_roles[0], tag) + pkg_role = pkg_name_to_role.get(pkg) + if pkg_role: + return (pkg_role, tag) + + for c in uniq: + key = safe_name(c) + hit = alias_ranked.get(key) + if hit is not None: + return (hit[1], tag) + + return None + + def _lists_for_role(role_name: str) -> tuple[List[ManagedFile], List[ExcludedFile]]: + if role_name in svc_by_role: + snap = svc_by_role[role_name] + return (snap.managed_files, snap.excluded) + if role_name in pkg_by_role: + snap = pkg_by_role[role_name] + return (snap.managed_files, snap.excluded) + # Fallback (shouldn't normally happen): attribute to etc_custom. + return (etc_managed, etc_excluded) + + # Capture essential system config/state (even if package-owned). + etc_role_seen = seen_by_role.setdefault(etc_role_name, set()) + for path, reason in system_paths.iter_system_capture_paths(): + if path in already: + continue + + target = _target_role_for_shared_snippet(path) + if target is not None: + role_for_copy, reason_for_role = target + managed_out, excluded_out = _lists_for_role(role_for_copy) + role_seen = seen_by_role.setdefault(role_for_copy, set()) + else: + role_for_copy, reason_for_role = (etc_role_name, reason) + managed_out, excluded_out = (etc_managed, etc_excluded) + role_seen = etc_role_seen + + capture_file( + bundle_dir=bundle_dir, + role_name=role_for_copy, + abs_path=path, + reason=reason_for_role, + policy=policy, + path_filter=path_filter, + managed_out=managed_out, + excluded_out=excluded_out, + seen_role=role_seen, + seen_global=captured_global, + ) + + # Walk /etc for remaining unowned config-ish files scanned = 0 for dirpath, _, filenames in os.walk("/etc"): for fn in filenames: path = os.path.join(dirpath, fn) + if backend.is_pkg_config_path(path): + continue if path in already: continue if path in owned_etc: continue if not os.path.isfile(path) or os.path.islink(path): continue - if not _is_confish(path): + if not system_paths.is_confish(path): continue - deny = policy.deny_reason(path) - if deny: - etc_excluded.append(ExcludedFile(path=path, reason=deny)) - continue + target = _target_role_for_shared_snippet(path) + if target is not None: + role_for_copy, reason_for_role = target + managed_out, excluded_out = _lists_for_role(role_for_copy) + role_seen = seen_by_role.setdefault(role_for_copy, set()) + else: + role_for_copy, reason_for_role = (etc_role_name, "custom_unowned") + managed_out, excluded_out = (etc_managed, etc_excluded) + role_seen = etc_role_seen - try: - owner, group, mode = stat_triplet(path) - except OSError: - etc_excluded.append(ExcludedFile(path=path, reason="unreadable")) - continue - - src_rel = path.lstrip("/") - try: - _copy_into_bundle(bundle_dir, etc_role_name, path, src_rel) - except OSError: - etc_excluded.append(ExcludedFile(path=path, reason="unreadable")) - continue - - etc_managed.append( - ManagedFile( - path=path, - src_rel=src_rel, - owner=owner, - group=group, - mode=mode, - reason="custom_unowned", - ) - ) - scanned += 1 - if scanned >= 2000: + if capture_file( + bundle_dir=bundle_dir, + role_name=role_for_copy, + abs_path=path, + reason=reason_for_role, + policy=policy, + path_filter=path_filter, + managed_out=managed_out, + excluded_out=excluded_out, + seen_role=role_seen, + seen_global=captured_global, + ): + scanned += 1 + if scanned >= system_paths.MAX_FILES_CAP: etc_notes.append( - "Reached file cap (2000) while scanning /etc for unowned files." + f"Reached file cap ({system_paths.MAX_FILES_CAP}) while scanning /etc for unowned files." ) break - if scanned >= 2000: + if scanned >= system_paths.MAX_FILES_CAP: break etc_custom_snapshot = EtcCustomSnapshot( @@ -647,17 +870,168 @@ def harvest(bundle_dir: str, policy: Optional[IgnorePolicy] = None) -> str: notes=etc_notes, ) + # ------------------------- + # usr_local_custom and extra_paths roles + # ------------------------- + already_all: Set[str] = set(already) + for mf in etc_managed: + already_all.add(mf.path) + + usr_local_custom_snapshot = UsrLocalCustomCollector( + context, + seen_by_role, + already_all, + ).collect() + + extra_paths_snapshot = ExtraPathsCollector( + context, + seen_by_role, + already_all, + include_paths=include_paths, + exclude_paths=exclude_paths, + ).collect() + + # ------------------------- + # Inventory: packages (SBOM-ish) + # ------------------------- + installed = installed_pkgs + + manual_set: Set[str] = set(manual_pkgs or []) + + pkg_units: Dict[str, Set[str]] = {} + pkg_roles_map: Dict[str, Set[str]] = {} + + for svc in service_snaps: + for p in svc.packages: + pkg_units.setdefault(p, set()).add(svc.unit) + pkg_roles_map.setdefault(p, set()).add(svc.role_name) + + pkg_role_names: Dict[str, List[str]] = {} + for ps in pkg_snaps: + pkg_roles_map.setdefault(ps.package, set()).add(ps.role_name) + pkg_role_names.setdefault(ps.package, []).append(ps.role_name) + + pkg_names: Set[str] = set() + pkg_names |= manual_set + pkg_names |= set(pkg_units.keys()) + pkg_names |= {ps.package for ps in pkg_snaps} + pkg_names |= set(firewall_runtime_snapshot.packages or []) + + packages_inventory: Dict[str, Dict[str, object]] = {} + for pkg in sorted(pkg_names): + installs = installed.get(pkg, []) or [] + arches = sorted({i.get("arch") for i in installs if i.get("arch")}) + vers = sorted({i.get("version") for i in installs if i.get("version")}) + version: Optional[str] = vers[0] if len(vers) == 1 else None + section = package_section_from_installations(installs) + + observed: List[Dict[str, str]] = [] + if pkg in manual_set: + observed.append({"kind": "user_installed"}) + for unit in sorted(pkg_units.get(pkg, set())): + observed.append({"kind": "systemd_unit", "ref": unit}) + for rn in sorted(set(pkg_role_names.get(pkg, []))): + observed.append({"kind": "package_role", "ref": rn}) + if pkg in set(firewall_runtime_snapshot.packages or []): + observed.append( + {"kind": "firewall_runtime", "ref": firewall_runtime_snapshot.role_name} + ) + pkg_roles_map.setdefault(pkg, set()).add( + firewall_runtime_snapshot.role_name + ) + + roles = sorted(pkg_roles_map.get(pkg, set())) + + packages_inventory[pkg] = { + "version": version, + "arches": arches, + "installations": installs, + "section": section, + "observed_via": observed, + "roles": roles, + } + + # Ensure every role has explicit managed_dirs for parent directories of managed files. + # This lets the manifest create directories with owner/group/mode (ansible-lint friendly) + # without a separate "mkdir without perms" task. + users_snapshot.managed_dirs = _merge_parent_dirs( + users_snapshot.managed_dirs, users_snapshot.managed_files, policy=policy + ) + for s in service_snaps: + s.managed_dirs = _merge_parent_dirs( + s.managed_dirs, + s.managed_files, + policy=policy, + extra_paths=[ml.path for ml in (s.managed_links or [])], + ) + for p in pkg_snaps: + p.managed_dirs = _merge_parent_dirs( + p.managed_dirs, + p.managed_files, + policy=policy, + extra_paths=[ml.path for ml in (p.managed_links or [])], + ) + + if apt_config_snapshot: + apt_config_snapshot.managed_dirs = _merge_parent_dirs( + apt_config_snapshot.managed_dirs, + apt_config_snapshot.managed_files, + policy=policy, + ) + if dnf_config_snapshot: + dnf_config_snapshot.managed_dirs = _merge_parent_dirs( + dnf_config_snapshot.managed_dirs, + dnf_config_snapshot.managed_files, + policy=policy, + ) + if etc_custom_snapshot: + etc_custom_snapshot.managed_dirs = _merge_parent_dirs( + etc_custom_snapshot.managed_dirs, + etc_custom_snapshot.managed_files, + policy=policy, + ) + if usr_local_custom_snapshot: + usr_local_custom_snapshot.managed_dirs = _merge_parent_dirs( + usr_local_custom_snapshot.managed_dirs, + usr_local_custom_snapshot.managed_files, + policy=policy, + ) + if extra_paths_snapshot: + extra_paths_snapshot.managed_dirs = _merge_parent_dirs( + extra_paths_snapshot.managed_dirs, + extra_paths_snapshot.managed_files, + policy=policy, + ) + state = { - "host": {"hostname": os.uname().nodename, "os": "debian"}, - "users": asdict(users_snapshot), - "services": [asdict(s) for s in service_snaps], - "manual_packages": manual_pkgs, - "manual_packages_skipped": manual_pkgs_skipped, - "package_roles": [asdict(p) for p in pkg_snaps], - "etc_custom": asdict(etc_custom_snapshot), + "enroll": { + "version": get_enroll_version(), + "harvest_time": time.time_ns(), + }, + "host": { + "hostname": os.uname().nodename, + "os": platform.os_family, + "pkg_backend": backend.name, + "os_release": platform.os_release, + }, + "inventory": { + "packages": packages_inventory, + }, + "roles": { + "users": asdict(users_snapshot), + "flatpak": asdict(flatpak_snapshot), + "snap": asdict(snap_snapshot), + "container_images": asdict(container_images_snapshot), + "services": [asdict(s) for s in service_snaps], + "packages": [asdict(p) for p in pkg_snaps], + "apt_config": asdict(apt_config_snapshot), + "dnf_config": asdict(dnf_config_snapshot), + "firewall_runtime": asdict(firewall_runtime_snapshot), + "sysctl": asdict(sysctl_snapshot), + "etc_custom": asdict(etc_custom_snapshot), + "usr_local_custom": asdict(usr_local_custom_snapshot), + "extra_paths": asdict(extra_paths_snapshot), + }, } - state_path = os.path.join(bundle_dir, "state.json") - with open(state_path, "w", encoding="utf-8") as f: - json.dump(state, f, indent=2, sort_keys=True) - return state_path + return str(write_state(bundle_dir, state)) diff --git a/enroll/harvest_collectors/__init__.py b/enroll/harvest_collectors/__init__.py new file mode 100644 index 0000000..dc8925f --- /dev/null +++ b/enroll/harvest_collectors/__init__.py @@ -0,0 +1,38 @@ +"""Harvest collector package exports""" + +from __future__ import annotations + +from importlib import import_module + +from .context import HarvestCollector, HarvestContext + +_COLLECTOR_EXPORTS = { + "CronLogrotateCollection": ".cron_logrotate", + "CronLogrotateCollector": ".cron_logrotate", + "ExtraPathsCollector": ".paths", + "PackageManagerConfigCollection": ".package_manager", + "PackageManagerConfigCollector": ".package_manager", + "RuntimeStateCollection": ".runtime", + "RuntimeStateCollector": ".runtime", + "ServicePackageCollection": ".services", + "ServicePackageCollector": ".services", + "UsersCollection": ".users", + "UsersCollector": ".users", + "UsrLocalCustomCollector": ".paths", +} + +__all__ = [ + "HarvestCollector", + "HarvestContext", + *_COLLECTOR_EXPORTS, +] + + +def __getattr__(name: str): + module_name = _COLLECTOR_EXPORTS.get(name) + if module_name is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module = import_module(module_name, __name__) + value = getattr(module, name) + globals()[name] = value + return value diff --git a/enroll/harvest_collectors/container_images.py b/enroll/harvest_collectors/container_images.py new file mode 100644 index 0000000..86129d9 --- /dev/null +++ b/enroll/harvest_collectors/container_images.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import json +import re +import shutil +import subprocess # nosec B404 +from collections.abc import ( + Iterable, +) # nosec - executes fixed docker/podman command arguments only +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from ..harvest_types import ContainerImagesSnapshot +from .context import HarvestCollector + +_DIGEST_RE = re.compile(r"@sha256:[0-9A-Fa-f]{32,}") +_SHA_ID_RE = re.compile(r"^(?:sha256:)?[0-9A-Fa-f]{64}$") + + +def _normalise_image_id(value: Any) -> Optional[str]: + s = str(value or "").strip() + if not s: + return None + if s.startswith("sha256:"): + return s + if _SHA_ID_RE.match(s): + return "sha256:" + s + return s + + +def _as_string_list(value: Any) -> List[str]: + if not value: + return [] + if isinstance(value, str): + values = [value] + elif isinstance(value, Iterable): + values = list(value) + else: + values = [value] + out: List[str] = [] + for item in values: + s = str(item or "").strip() + if not s or s in {"", ":"}: + continue + if s not in out: + out.append(s) + return out + + +def _pullable_digests(value: Any) -> List[str]: + return [s for s in _as_string_list(value) if _DIGEST_RE.search(s)] + + +def _split_tag_ref(ref: str) -> Optional[Dict[str, str]]: + """Split an image tag into repository/tag, preserving registry ports.""" + + s = str(ref or "").strip() + if not s or "@" in s or s == ":": + return None + last_slash = s.rfind("/") + last_colon = s.rfind(":") + if last_colon > last_slash: + repository = s[:last_colon] + tag = s[last_colon + 1 :] + else: + repository = s + tag = "latest" + if not repository or not tag: + return None + return {"ref": s, "repository": repository, "tag": tag} + + +def _tag_aliases(value: Any) -> List[Dict[str, str]]: + out: List[Dict[str, str]] = [] + seen = set() + for ref in _as_string_list(value): + item = _split_tag_ref(ref) + if not item: + continue + key = (item["repository"], item["tag"]) + if key in seen: + continue + seen.add(key) + out.append(item) + return out + + +def _platform_from_inspect( + item: Dict[str, Any], +) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]: + os_name = item.get("Os") or item.get("OS") + arch = item.get("Architecture") or item.get("Arch") + variant = item.get("Variant") + os_s = str(os_name).strip() if os_name not in (None, "") else None + arch_s = str(arch).strip() if arch not in (None, "") else None + variant_s = str(variant).strip() if variant not in (None, "") else None + platform = None + if os_s and arch_s: + platform = f"{os_s}/{arch_s}" + if variant_s: + platform = f"{platform}/{variant_s}" + return os_s, arch_s, variant_s, platform + + +def _run_command( + argv: Sequence[str], *, timeout: int = 20 +) -> subprocess.CompletedProcess[str]: + return subprocess.run( # nosec - argv is constructed from fixed binary names and image ids + list(argv), + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=timeout, + ) + + +def _chunks(items: Sequence[str], size: int) -> Iterable[List[str]]: + for i in range(0, len(items), size): + yield list(items[i : i + size]) + + +class ContainerImagesCollector(HarvestCollector): + """Collect local Docker and Podman image metadata. + + The harvest records pullable registry digests where present. Local image IDs + are kept as evidence but are not treated as pull references. + """ + + def collect(self) -> ContainerImagesSnapshot: + images: List[Dict[str, Any]] = [] + notes: List[str] = [] + + images.extend(self._collect_engine("docker", notes=notes)) + images.extend(self._collect_engine("podman", notes=notes)) + + if images: + digest_count = len([img for img in images if img.get("pull_ref")]) + notes.append( + f"Detected {len(images)} container image(s); {digest_count} have registry digests usable for exact pulls." + ) + + return ContainerImagesSnapshot( + role_name="container_images", + images=images, + notes=notes, + ) + + def _collect_engine(self, engine: str, *, notes: List[str]) -> List[Dict[str, Any]]: + exe = shutil.which(engine) + if not exe: + return [] + + try: + listed = _run_command([exe, "image", "ls", "-q", "--no-trunc"]) + except Exception as exc: + notes.append(f"Failed to list {engine} images: {exc!r}") + return [] + + if listed.returncode != 0: + detail = (listed.stderr or listed.stdout or "").strip() + if detail: + notes.append(f"Failed to list {engine} images: {detail}") + else: + notes.append( + f"Failed to list {engine} images: exit {listed.returncode}" + ) + return [] + + image_ids = [] + seen_ids = set() + for line in listed.stdout.splitlines(): + image_id = _normalise_image_id(line) + if not image_id or image_id in seen_ids: + continue + seen_ids.add(image_id) + image_ids.append(image_id) + + if not image_ids: + return [] + + out: List[Dict[str, Any]] = [] + for chunk in _chunks(image_ids, 40): + try: + inspected = _run_command([exe, "image", "inspect", *chunk]) + except Exception as exc: + notes.append(f"Failed to inspect {engine} images: {exc!r}") + continue + if inspected.returncode != 0: + detail = (inspected.stderr or inspected.stdout or "").strip() + notes.append( + f"Failed to inspect {engine} images {', '.join(chunk[:3])}: {detail or inspected.returncode}" + ) + continue + try: + data = json.loads(inspected.stdout or "[]") + except json.JSONDecodeError as exc: + notes.append(f"Failed to parse {engine} image inspect JSON: {exc}") + continue + if not isinstance(data, list): + notes.append(f"Unexpected {engine} image inspect JSON shape") + continue + for item in data: + if isinstance(item, dict): + normalised = self._normalise_inspect(engine, item) + if normalised is not None: + out.append(normalised) + return out + + def _normalise_inspect( + self, engine: str, item: Dict[str, Any] + ) -> Optional[Dict[str, Any]]: + image_id = _normalise_image_id(item.get("Id") or item.get("ID")) + repo_tags = _as_string_list(item.get("RepoTags")) + repo_digests = _pullable_digests(item.get("RepoDigests")) + pull_ref = sorted(repo_digests)[0] if repo_digests else None + os_name, arch, variant, platform = _platform_from_inspect(item) + + if not image_id and not repo_tags and not repo_digests: + return None + + notes: List[str] = [] + if not pull_ref: + if repo_tags: + notes.append( + "Image has tag(s) but no RepoDigest; exact digest-pinned pull cannot be rendered." + ) + else: + notes.append( + "Image has no tag or RepoDigest; local-only/dangling images cannot be pulled from a registry." + ) + + out: Dict[str, Any] = { + "engine": engine, + "scope": "system", + "user": None, + "home": None, + "image_id": image_id, + "repo_tags": repo_tags, + "repo_digests": repo_digests, + "pull_ref": pull_ref, + "tag_aliases": _tag_aliases(repo_tags), + "os": os_name, + "architecture": arch, + "variant": variant, + "platform": platform, + "size": item.get("Size"), + "created": item.get("Created"), + "source": f"{engine} image inspect", + "notes": notes, + } + return out diff --git a/enroll/harvest_collectors/context.py b/enroll/harvest_collectors/context.py new file mode 100644 index 0000000..7c5b5d9 --- /dev/null +++ b/enroll/harvest_collectors/context.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict, List, Set + +from ..ignore import IgnorePolicy +from ..pathfilter import PathFilter + + +@dataclass +class HarvestContext: + """Shared context passed to feature collectors.""" + + bundle_dir: str + policy: IgnorePolicy + path_filter: PathFilter + platform: Dict[str, Any] + backend: Any + installed_pkgs: Dict[str, Any] + installed_names: Set[str] + owned_etc: Set[str] + etc_owner_map: Dict[str, str] + topdir_to_pkgs: Dict[str, Set[str]] + pkg_to_etc_paths: Dict[str, List[str]] + captured_global: Set[str] + + +class HarvestCollector: + """Base class for harvest feature collectors.""" + + def __init__(self, context: HarvestContext) -> None: + self.context = context diff --git a/enroll/harvest_collectors/cron_logrotate.py b/enroll/harvest_collectors/cron_logrotate.py new file mode 100644 index 0000000..c40c4a1 --- /dev/null +++ b/enroll/harvest_collectors/cron_logrotate.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import List, Optional, Set + +from ..capture import capture_file +from ..harvest_types import ExcludedFile, ManagedFile, PackageSnapshot +from ..package_hints import package_section_from_installations +from ..system_paths import iter_matching_files +from .context import HarvestCollector + + +def _pick_installed(installed_names: Set[str], candidates: List[str]) -> Optional[str]: + for candidate in candidates: + if candidate in installed_names: + return candidate + return None + + +def _is_cron_path(path: str) -> bool: + return ( + path == "/etc/crontab" + or path == "/etc/anacrontab" + or path in ("/etc/cron.allow", "/etc/cron.deny") + or path.startswith("/etc/cron.") + or path.startswith("/etc/cron.d/") + or path.startswith("/etc/anacron/") + or path.startswith("/var/spool/cron/") + or path.startswith("/var/spool/crontabs/") + or path.startswith("/var/spool/anacron/") + ) + + +def _is_logrotate_path(path: str) -> bool: + return path == "/etc/logrotate.conf" or path.startswith("/etc/logrotate.d/") + + +_CRON_CAPTURE_GLOBS = [ + "/etc/crontab", + "/etc/cron.d/*", + "/etc/cron.hourly/*", + "/etc/cron.daily/*", + "/etc/cron.weekly/*", + "/etc/cron.monthly/*", + "/etc/cron.allow", + "/etc/cron.deny", + "/etc/anacrontab", + "/etc/anacron/*", + # user crontabs / spool state + "/var/spool/cron/*", + "/var/spool/cron/crontabs/*", + "/var/spool/crontabs/*", + "/var/spool/anacron/*", +] + +_LOGROTATE_CAPTURE_GLOBS = [ + "/etc/logrotate.conf", + "/etc/logrotate.d/*", +] + + +@dataclass +class CronLogrotateCollection: + cron_pkg: Optional[str] + logrotate_pkg: Optional[str] + cron_snapshot: Optional[PackageSnapshot] + logrotate_snapshot: Optional[PackageSnapshot] + + +class CronLogrotateCollector(HarvestCollector): + """Collect dedicated cron/logrotate package roles before general packages.""" + + cron_role_name = "cron" + logrotate_role_name = "logrotate" + + def collect(self) -> CronLogrotateCollection: + cron_pkg = _pick_installed( + self.context.installed_names, + ["cron", "cronie", "cronie-anacron", "vixie-cron", "fcron"], + ) + logrotate_pkg = _pick_installed(self.context.installed_names, ["logrotate"]) + + cron_snapshot = self._collect_cron_snapshot(cron_pkg) if cron_pkg else None + logrotate_snapshot = ( + self._collect_logrotate_snapshot(logrotate_pkg) if logrotate_pkg else None + ) + return CronLogrotateCollection( + cron_pkg=cron_pkg, + logrotate_pkg=logrotate_pkg, + cron_snapshot=cron_snapshot, + logrotate_snapshot=logrotate_snapshot, + ) + + def _collect_cron_snapshot(self, cron_pkg: str) -> PackageSnapshot: + managed: List[ManagedFile] = [] + excluded: List[ExcludedFile] = [] + notes: List[str] = [] + seen: Set[str] = set() + + for spec in _CRON_CAPTURE_GLOBS: + for path in iter_matching_files(spec): + if not os.path.isfile(path) or os.path.islink(path): + continue + capture_file( + bundle_dir=self.context.bundle_dir, + role_name=self.cron_role_name, + abs_path=path, + reason="system_cron", + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=managed, + excluded_out=excluded, + seen_role=seen, + seen_global=self.context.captured_global, + ) + + return PackageSnapshot( + package=cron_pkg, + role_name=self.cron_role_name, + section=package_section_from_installations( + self.context.installed_pkgs.get(cron_pkg, []) + ), + managed_files=managed, + excluded=excluded, + notes=notes, + ) + + def _collect_logrotate_snapshot(self, logrotate_pkg: str) -> PackageSnapshot: + managed: List[ManagedFile] = [] + excluded: List[ExcludedFile] = [] + notes: List[str] = [] + seen: Set[str] = set() + + for spec in _LOGROTATE_CAPTURE_GLOBS: + for path in iter_matching_files(spec): + if not os.path.isfile(path) or os.path.islink(path): + continue + capture_file( + bundle_dir=self.context.bundle_dir, + role_name=self.logrotate_role_name, + abs_path=path, + reason="system_logrotate", + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=managed, + excluded_out=excluded, + seen_role=seen, + seen_global=self.context.captured_global, + ) + + return PackageSnapshot( + package=logrotate_pkg, + role_name=self.logrotate_role_name, + section=package_section_from_installations( + self.context.installed_pkgs.get(logrotate_pkg, []) + ), + managed_files=managed, + excluded=excluded, + notes=notes, + ) diff --git a/enroll/harvest_collectors/package_manager.py b/enroll/harvest_collectors/package_manager.py new file mode 100644 index 0000000..0cbeb03 --- /dev/null +++ b/enroll/harvest_collectors/package_manager.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict, List, Set + +from ..capture import capture_file +from ..harvest_types import ( + AptConfigSnapshot, + DnfConfigSnapshot, + ExcludedFile, + ManagedFile, +) +from ..system_paths import iter_apt_capture_paths, iter_dnf_capture_paths +from .context import HarvestCollector, HarvestContext + + +@dataclass +class PackageManagerConfigCollection: + apt_config_snapshot: AptConfigSnapshot + dnf_config_snapshot: DnfConfigSnapshot + + +class PackageManagerConfigCollector(HarvestCollector): + """Collect package-manager configuration into existing role snapshots.""" + + def __init__( + self, context: HarvestContext, seen_by_role: Dict[str, Set[str]] + ) -> None: + super().__init__(context) + self.seen_by_role = seen_by_role + + def collect(self) -> PackageManagerConfigCollection: + apt_notes: List[str] = [] + apt_excluded: List[ExcludedFile] = [] + apt_managed: List[ManagedFile] = [] + dnf_notes: List[str] = [] + dnf_excluded: List[ExcludedFile] = [] + dnf_managed: List[ManagedFile] = [] + + apt_role_name = "apt_config" + dnf_role_name = "dnf_config" + + if self.context.backend.name == "dpkg": + apt_role_seen = self.seen_by_role.setdefault(apt_role_name, set()) + for path, reason in iter_apt_capture_paths(): + capture_file( + bundle_dir=self.context.bundle_dir, + role_name=apt_role_name, + abs_path=path, + reason=reason, + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=apt_managed, + excluded_out=apt_excluded, + seen_role=apt_role_seen, + seen_global=self.context.captured_global, + ) + elif self.context.backend.name == "rpm": + dnf_role_seen = self.seen_by_role.setdefault(dnf_role_name, set()) + for path, reason in iter_dnf_capture_paths(): + capture_file( + bundle_dir=self.context.bundle_dir, + role_name=dnf_role_name, + abs_path=path, + reason=reason, + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=dnf_managed, + excluded_out=dnf_excluded, + seen_role=dnf_role_seen, + seen_global=self.context.captured_global, + ) + + return PackageManagerConfigCollection( + apt_config_snapshot=AptConfigSnapshot( + role_name=apt_role_name, + managed_files=apt_managed, + excluded=apt_excluded, + notes=apt_notes, + ), + dnf_config_snapshot=DnfConfigSnapshot( + role_name=dnf_role_name, + managed_files=dnf_managed, + excluded=dnf_excluded, + notes=dnf_notes, + ), + ) diff --git a/enroll/harvest_collectors/paths.py b/enroll/harvest_collectors/paths.py new file mode 100644 index 0000000..f11896a --- /dev/null +++ b/enroll/harvest_collectors/paths.py @@ -0,0 +1,249 @@ +from __future__ import annotations + +import glob +import os +from typing import Dict, List, Optional, Set + +from .. import harvest as h +from ..capture import capture_file +from ..harvest_types import ( + ExcludedFile, + ExtraPathsSnapshot, + ManagedDir, + ManagedFile, + UsrLocalCustomSnapshot, +) +from ..system_paths import MAX_FILES_CAP +from ..pathfilter import expand_includes +from .context import HarvestCollector, HarvestContext + + +class UsrLocalCustomCollector(HarvestCollector): + """Collect selected /usr/local state into the usr_local_custom role.""" + + role_name = "usr_local_custom" + + def __init__( + self, + context: HarvestContext, + seen_by_role: Dict[str, Set[str]], + already_all: Set[str], + ) -> None: + super().__init__(context) + self.seen_by_role = seen_by_role + self.already_all = already_all + self.notes: List[str] = [] + self.excluded: List[ExcludedFile] = [] + self.managed: List[ManagedFile] = [] + + def collect(self) -> UsrLocalCustomSnapshot: + self._scan_tree( + "/usr/local/etc", + require_executable=False, + cap=MAX_FILES_CAP, + reason="usr_local_etc_custom", + ) + self._scan_tree( + "/usr/local/bin", + require_executable=True, + cap=MAX_FILES_CAP, + reason="usr_local_bin_script", + ) + return UsrLocalCustomSnapshot( + role_name=self.role_name, + managed_files=self.managed, + excluded=self.excluded, + notes=self.notes, + ) + + def _scan_tree( + self, + root: str, + *, + require_executable: bool, + cap: int, + reason: str, + ) -> None: + scanned = 0 + if not os.path.isdir(root): + return + role_seen = self.seen_by_role.setdefault(self.role_name, set()) + for dirpath, _, filenames in os.walk(root): + for filename in filenames: + path = os.path.join(dirpath, filename) + if path in self.already_all: + continue + if not os.path.isfile(path) or os.path.islink(path): + continue + try: + owner, group, mode = h.stat_triplet(path) + except OSError: + self.excluded.append(ExcludedFile(path=path, reason="unreadable")) + continue + + if require_executable: + try: + if (int(mode, 8) & 0o111) == 0: + continue + except ValueError: + continue + + if capture_file( + bundle_dir=self.context.bundle_dir, + role_name=self.role_name, + abs_path=path, + reason=reason, + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=self.managed, + excluded_out=self.excluded, + seen_role=role_seen, + seen_global=self.context.captured_global, + metadata=(owner, group, mode), + ): + self.already_all.add(path) + scanned += 1 + if scanned >= cap: + self.notes.append( + f"Reached file cap ({cap}) while scanning {root}." + ) + return + + +class ExtraPathsCollector(HarvestCollector): + """Collect user-requested include/exclude paths into extra_paths.""" + + role_name = "extra_paths" + + def __init__( + self, + context: HarvestContext, + seen_by_role: Dict[str, Set[str]], + already_all: Set[str], + *, + include_paths: Optional[List[str]] = None, + exclude_paths: Optional[List[str]] = None, + ) -> None: + super().__init__(context) + self.seen_by_role = seen_by_role + self.already_all = already_all + self.include_specs = list(include_paths or []) + self.exclude_specs = list(exclude_paths or []) + self.notes: List[str] = [] + self.excluded: List[ExcludedFile] = [] + self.managed: List[ManagedFile] = [] + self.managed_dirs: List[ManagedDir] = [] + self.dir_seen: Set[str] = set() + + def collect(self) -> ExtraPathsSnapshot: + self._collect_included_dirs() + if self.include_specs: + self.notes.append("User include patterns:") + self.notes.extend([f"- {p}" for p in self.include_specs]) + if self.exclude_specs: + self.notes.append("User exclude patterns:") + self.notes.extend([f"- {p}" for p in self.exclude_specs]) + + included_files: List[str] = [] + if self.include_specs: + files, inc_notes = expand_includes( + self.context.path_filter.iter_include_patterns(), + exclude=self.context.path_filter, + max_files=MAX_FILES_CAP, + ) + included_files = files + self.notes.extend(inc_notes) + + role_seen = self.seen_by_role.setdefault(self.role_name, set()) + for path in included_files: + if path in self.already_all: + continue + if capture_file( + bundle_dir=self.context.bundle_dir, + role_name=self.role_name, + abs_path=path, + reason="user_include", + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=self.managed, + excluded_out=self.excluded, + seen_role=role_seen, + seen_global=self.context.captured_global, + ): + self.already_all.add(path) + + return ExtraPathsSnapshot( + role_name=self.role_name, + include_patterns=self.include_specs, + exclude_patterns=self.exclude_specs, + managed_dirs=self.managed_dirs, + managed_files=self.managed, + excluded=self.excluded, + notes=self.notes, + ) + + def _collect_included_dirs(self) -> None: + for pat in self.context.path_filter.iter_include_patterns(): + if pat.kind == "prefix": + path = pat.value + if os.path.isdir(path) and not os.path.islink(path): + self._walk_and_capture_dirs(path) + elif pat.kind == "glob": + for hit in glob.glob(pat.value, recursive=True): + if os.path.isdir(hit) and not os.path.islink(hit): + self._walk_and_capture_dirs(hit) + + def _walk_and_capture_dirs(self, root: str) -> None: + root = os.path.normpath(root) + if not root.startswith("/"): + root = "/" + root + if not os.path.isdir(root) or os.path.islink(root): + return + for dirpath, dirnames, _ in os.walk(root, followlinks=False): + if len(self.managed_dirs) >= MAX_FILES_CAP: + self.notes.append( + f"Reached directory cap ({MAX_FILES_CAP}) while scanning {root}." + ) + return + dirpath = os.path.normpath(dirpath) + if not dirpath.startswith("/"): + dirpath = "/" + dirpath + if self.context.path_filter.is_excluded(dirpath): + dirnames[:] = [] + continue + if os.path.islink(dirpath) or not os.path.isdir(dirpath): + dirnames[:] = [] + continue + + if dirpath not in self.dir_seen: + deny = None + deny_dir = getattr(self.context.policy, "deny_reason_dir", None) + if callable(deny_dir): + deny = deny_dir(dirpath) + else: + deny = self.context.policy.deny_reason(dirpath) + if deny in ("not_regular_file", "not_file", "not_regular"): + deny = None + if not deny: + try: + owner, group, mode = h.stat_triplet(dirpath) + self.managed_dirs.append( + ManagedDir( + path=dirpath, + owner=owner, + group=group, + mode=mode, + reason="user_include_dir", + ) + ) + except OSError: + pass + self.dir_seen.add(dirpath) + + pruned: List[str] = [] + for dirname in dirnames: + path = os.path.join(dirpath, dirname) + if os.path.islink(path) or self.context.path_filter.is_excluded(path): + continue + pruned.append(dirname) + dirnames[:] = pruned diff --git a/enroll/harvest_collectors/runtime.py b/enroll/harvest_collectors/runtime.py new file mode 100644 index 0000000..c16f9da --- /dev/null +++ b/enroll/harvest_collectors/runtime.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import List, Optional + +from .. import harvest as h +from ..harvest_types import FirewallRuntimeSnapshot, SysctlSnapshot +from .context import HarvestCollector, HarvestContext + + +@dataclass +class RuntimeStateCollection: + firewall_runtime_snapshot: FirewallRuntimeSnapshot + sysctl_snapshot: SysctlSnapshot + + +class RuntimeStateCollector(HarvestCollector): + """Collect root-only live runtime state that has generated roles.""" + + def __init__( + self, + context: HarvestContext, + *, + persistent_ipset_files: Optional[List[str]] = None, + persistent_iptables_v4_files: Optional[List[str]] = None, + persistent_iptables_v6_files: Optional[List[str]] = None, + ) -> None: + super().__init__(context) + self.persistent_ipset_files = persistent_ipset_files or [] + self.persistent_iptables_v4_files = persistent_iptables_v4_files or [] + self.persistent_iptables_v6_files = persistent_iptables_v6_files or [] + + def collect(self) -> RuntimeStateCollection: + running_as_root = not hasattr(os, "geteuid") or os.geteuid() == 0 + if not running_as_root: + return RuntimeStateCollection( + firewall_runtime_snapshot=FirewallRuntimeSnapshot( + role_name="firewall_runtime", + notes=[ + "Live ipset/iptables runtime capture skipped because harvest " + "is not running as root." + ], + ), + sysctl_snapshot=SysctlSnapshot( + role_name="sysctl", + notes=[ + "Live sysctl runtime capture skipped because harvest is not " + "running as root." + ], + ), + ) + + firewall_runtime_snapshot = h._collect_firewall_runtime_snapshot( + self.context.bundle_dir, + persistent_ipset_files=self.persistent_ipset_files, + persistent_iptables_v4_files=self.persistent_iptables_v4_files, + persistent_iptables_v6_files=self.persistent_iptables_v6_files, + ) + sysctl_snapshot = h._collect_sysctl_snapshot(self.context.bundle_dir) + return RuntimeStateCollection( + firewall_runtime_snapshot=firewall_runtime_snapshot, + sysctl_snapshot=sysctl_snapshot, + ) diff --git a/enroll/harvest_collectors/services.py b/enroll/harvest_collectors/services.py new file mode 100644 index 0000000..2b087df --- /dev/null +++ b/enroll/harvest_collectors/services.py @@ -0,0 +1,541 @@ +from __future__ import annotations + +import glob +import os +from dataclasses import dataclass +from typing import Dict, List, Optional, Set + +from .. import harvest as h +from ..capture import capture_file, capture_link +from ..harvest_types import ExcludedFile, ManagedFile, PackageSnapshot, ServiceSnapshot +from ..package_hints import ( + SHARED_ETC_TOPDIRS, + add_pkgs_from_etc_topdirs, + hint_names, + maybe_add_specific_paths, + package_section_from_installations, + role_name_from_pkg, + role_name_from_unit, +) +from ..system_paths import ( + MAX_UNOWNED_FILES_PER_ROLE, + is_confish, + scan_unowned_under_roots, + topdirs_for_package, +) +from ..systemd import UnitQueryError +from .context import HarvestCollector, HarvestContext +from .cron_logrotate import CronLogrotateCollector, _is_cron_path, _is_logrotate_path + + +@dataclass +class ServicePackageCollection: + service_snaps: List[ServiceSnapshot] + pkg_snaps: List[PackageSnapshot] + manual_pkgs: List[str] + simple_packages: List[str] + manual_pkgs_skipped: List[str] + service_role_aliases: Dict[str, Set[str]] + seen_by_role: Dict[str, Set[str]] + + +class ServicePackageCollector(HarvestCollector): + """Collect service-attributed and manually-installed package snapshots.""" + + def __init__( + self, + context: HarvestContext, + *, + cron_snapshot: Optional[PackageSnapshot] = None, + logrotate_snapshot: Optional[PackageSnapshot] = None, + cron_pkg: Optional[str] = None, + logrotate_pkg: Optional[str] = None, + ) -> None: + super().__init__(context) + self.cron_snapshot = cron_snapshot + self.logrotate_snapshot = logrotate_snapshot + self.cron_pkg = cron_pkg + self.logrotate_pkg = logrotate_pkg + self.service_role_aliases: Dict[str, Set[str]] = {} + self.seen_by_role: Dict[str, Set[str]] = {} + self.managed_by_role: Dict[str, List[ManagedFile]] = {} + self.excluded_by_role: Dict[str, List[ExcludedFile]] = {} + + def collect(self) -> ServicePackageCollection: + service_snaps, timer_extra_by_pkg = self._collect_service_snapshots() + pkg_snaps, manual_pkgs, simple_packages, manual_pkgs_skipped = ( + self._collect_package_snapshots( + service_snaps, + timer_extra_by_pkg, + ) + ) + self._capture_common_enabled_symlinks(service_snaps, pkg_snaps) + return ServicePackageCollection( + service_snaps=service_snaps, + pkg_snaps=pkg_snaps, + manual_pkgs=manual_pkgs, + simple_packages=simple_packages, + manual_pkgs_skipped=manual_pkgs_skipped, + service_role_aliases=self.service_role_aliases, + seen_by_role=self.seen_by_role, + ) + + def _collect_service_snapshots( + self, + ) -> tuple[List[ServiceSnapshot], Dict[str, List[str]]]: + backend = self.context.backend + service_snaps: List[ServiceSnapshot] = [] + + enabled_services = h.list_enabled_services() + if self.cron_snapshot is not None or self.logrotate_snapshot is not None: + blocked_roles = set() + if self.cron_snapshot is not None: + blocked_roles.add(CronLogrotateCollector.cron_role_name) + if self.logrotate_snapshot is not None: + blocked_roles.add(CronLogrotateCollector.logrotate_role_name) + enabled_services = [ + u + for u in enabled_services + if role_name_from_unit(u) not in blocked_roles + ] + enabled_set = set(enabled_services) + + def service_sort_key(unit: str) -> tuple[int, str, str]: + base = unit.removesuffix(".service") + base = base.split("@", 1)[0] + return (base.count("-"), base.lower(), unit.lower()) + + def parent_service_unit(unit: str) -> Optional[str]: + if not unit.endswith(".service"): + return None + base = unit.removesuffix(".service") + base = base.split("@", 1)[0] + parts = base.split("-") + for i in range(len(parts) - 1, 0, -1): + cand = "-".join(parts[:i]) + ".service" + if cand in enabled_set: + return cand + return None + + parent_unit_for = { + u: pu for u in enabled_services if (pu := parent_service_unit(u)) + } + + for unit in sorted(enabled_services, key=service_sort_key): + role = role_name_from_unit(unit) + parent_unit = parent_unit_for.get(unit) + parent_role = role_name_from_unit(parent_unit) if parent_unit else None + + try: + ui = h.get_unit_info(unit) + except UnitQueryError as e: + self.service_role_aliases.setdefault( + role, hint_names(unit, set()) | {role} + ) + self.seen_by_role.setdefault(role, set()) + managed = self.managed_by_role.setdefault(role, []) + excluded = self.excluded_by_role.setdefault(role, []) + service_snaps.append( + ServiceSnapshot( + unit=unit, + role_name=role, + packages=[], + active_state=None, + sub_state=None, + unit_file_state=None, + condition_result=None, + managed_files=managed, + excluded=excluded, + notes=[str(e)], + ) + ) + continue + + pkgs: Set[str] = set() + notes: List[str] = [] + excluded = self.excluded_by_role.setdefault(role, []) + managed = self.managed_by_role.setdefault(role, []) + candidates: Dict[str, str] = {} + + if ui.fragment_path: + p = backend.owner_of_path(ui.fragment_path) + if p: + pkgs.add(p) + + for exe in ui.exec_paths: + p = backend.owner_of_path(exe) + if p: + pkgs.add(p) + + for pth in ui.dropin_paths: + if pth.startswith("/etc/"): + candidates[pth] = "systemd_dropin" + + for env_file in ui.env_files: + env_file = env_file.lstrip("-") + if any(ch in env_file for ch in "*?["): + for g in glob.glob(env_file): + if g.startswith("/etc/") and os.path.isfile(g): + candidates[g] = "systemd_envfile" + elif env_file.startswith("/etc/") and os.path.isfile(env_file): + candidates[env_file] = "systemd_envfile" + + hints = hint_names(unit, pkgs) + add_pkgs_from_etc_topdirs(hints, self.context.topdir_to_pkgs, pkgs) + self.service_role_aliases[role] = set(hints) | set(pkgs) | {role} + + for sp in maybe_add_specific_paths(hints, backend): + if not os.path.exists(sp): + continue + if sp in self.context.etc_owner_map: + pkgs.add(self.context.etc_owner_map[sp]) + else: + candidates.setdefault(sp, "custom_specific_path") + + for pkg in sorted(pkgs): + etc_paths = self.context.pkg_to_etc_paths.get(pkg, []) + for path, reason in backend.modified_paths(pkg, etc_paths).items(): + if not os.path.isfile(path) or os.path.islink(path): + continue + if self.cron_snapshot is not None and _is_cron_path(path): + continue + if self.logrotate_snapshot is not None and _is_logrotate_path(path): + continue + if backend.is_pkg_config_path(path): + continue + candidates.setdefault(path, reason) + + any_roots: List[str] = [] + confish_roots: List[str] = [] + for hint in hints: + roots_for_hint = [f"/etc/{hint}", f"/etc/{hint}.d"] + if hint in SHARED_ETC_TOPDIRS: + confish_roots.extend(roots_for_hint) + else: + any_roots.extend(roots_for_hint) + + found: List[str] = [] + found.extend( + scan_unowned_under_roots( + any_roots, + self.context.owned_etc, + limit=MAX_UNOWNED_FILES_PER_ROLE, + confish_only=False, + ) + ) + if len(found) < MAX_UNOWNED_FILES_PER_ROLE: + found.extend( + scan_unowned_under_roots( + confish_roots, + self.context.owned_etc, + limit=MAX_UNOWNED_FILES_PER_ROLE - len(found), + confish_only=True, + ) + ) + for pth in found: + candidates.setdefault(pth, "custom_unowned") + + if not pkgs and not candidates: + notes.append( + "No packages or /etc candidates detected (unexpected for enabled service)." + ) + + for path, reason in sorted(candidates.items()): + dest_role = role + if ( + parent_role + and path.startswith("/etc/") + and reason not in ("systemd_dropin", "systemd_envfile") + ): + dest_role = parent_role + + dest_managed = self.managed_by_role.setdefault(dest_role, []) + dest_excluded = self.excluded_by_role.setdefault(dest_role, []) + dest_seen = self.seen_by_role.setdefault(dest_role, set()) + capture_file( + bundle_dir=self.context.bundle_dir, + role_name=dest_role, + abs_path=path, + reason=reason, + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=dest_managed, + excluded_out=dest_excluded, + seen_role=dest_seen, + seen_global=self.context.captured_global, + ) + + service_snaps.append( + ServiceSnapshot( + unit=unit, + role_name=role, + packages=sorted(pkgs), + active_state=ui.active_state, + sub_state=ui.sub_state, + unit_file_state=ui.unit_file_state, + condition_result=ui.condition_result, + managed_files=managed, + excluded=excluded, + notes=notes, + ) + ) + + timer_extra_by_pkg = self._collect_timer_overrides(service_snaps) + return service_snaps, timer_extra_by_pkg + + def _collect_timer_overrides( + self, + service_snaps: List[ServiceSnapshot], + ) -> Dict[str, List[str]]: + backend = self.context.backend + timer_extra_by_pkg: Dict[str, List[str]] = {} + try: + enabled_timers = h.list_enabled_timers() + except Exception: + enabled_timers = [] + + service_snap_by_unit = {s.unit: s for s in service_snaps} + + for timer in sorted(enabled_timers): + try: + ti = h.get_timer_info(timer) + except Exception: # nosec + continue + + timer_paths: List[str] = [] + for pth in [ti.fragment_path, *ti.dropin_paths, *ti.env_files]: + if not pth: + continue + if not pth.startswith("/etc/"): + continue + if os.path.islink(pth) or not os.path.isfile(pth): + continue + timer_paths.append(pth) + + if not timer_paths: + continue + + snap = ( + service_snap_by_unit.get(ti.trigger_unit) if ti.trigger_unit else None + ) + if snap is not None: + role_seen = self.seen_by_role.setdefault(snap.role_name, set()) + for path in timer_paths: + capture_file( + bundle_dir=self.context.bundle_dir, + role_name=snap.role_name, + abs_path=path, + reason="related_timer", + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=snap.managed_files, + excluded_out=snap.excluded, + seen_role=role_seen, + seen_global=self.context.captured_global, + ) + continue + + pkgs: Set[str] = set() + if ti.fragment_path: + p = backend.owner_of_path(ti.fragment_path) + if p: + pkgs.add(p) + if ti.trigger_unit and ti.trigger_unit.endswith(".service"): + try: + ui = h.get_unit_info(ti.trigger_unit) + if ui.fragment_path: + p = backend.owner_of_path(ui.fragment_path) + if p: + pkgs.add(p) + for exe in ui.exec_paths: + p = backend.owner_of_path(exe) + if p: + pkgs.add(p) + except Exception: # nosec + pass + + for pkg in pkgs: + timer_extra_by_pkg.setdefault(pkg, []).extend(timer_paths) + + return timer_extra_by_pkg + + def _collect_package_snapshots( + self, + service_snaps: List[ServiceSnapshot], + timer_extra_by_pkg: Dict[str, List[str]], + ) -> tuple[List[PackageSnapshot], List[str], List[str], List[str]]: + backend = self.context.backend + manual_pkgs = backend.list_manual_packages() + covered_by_services: Set[str] = set() + for snap in service_snaps: + covered_by_services.update(snap.packages) + + manual_pkgs_skipped: List[str] = [] + pkg_snaps: List[PackageSnapshot] = [] + simple_packages: List[str] = [] + + if self.cron_snapshot is not None: + pkg_snaps.append(self.cron_snapshot) + if self.logrotate_snapshot is not None: + pkg_snaps.append(self.logrotate_snapshot) + + for pkg in sorted(manual_pkgs): + if pkg in covered_by_services: + manual_pkgs_skipped.append(pkg) + continue + if self.cron_snapshot is not None and pkg == self.cron_pkg: + manual_pkgs_skipped.append(pkg) + continue + if self.logrotate_snapshot is not None and pkg == self.logrotate_pkg: + manual_pkgs_skipped.append(pkg) + continue + + role = role_name_from_pkg(pkg) + notes: List[str] = [] + excluded: List[ExcludedFile] = [] + managed: List[ManagedFile] = [] + candidates: Dict[str, str] = {} + + for tpath in timer_extra_by_pkg.get(pkg, []): + candidates.setdefault(tpath, "related_timer") + + etc_paths = self.context.pkg_to_etc_paths.get(pkg, []) + for path, reason in backend.modified_paths(pkg, etc_paths).items(): + if not os.path.isfile(path) or os.path.islink(path): + continue + if self.cron_snapshot is not None and _is_cron_path(path): + continue + if self.logrotate_snapshot is not None and _is_logrotate_path(path): + continue + if backend.is_pkg_config_path(path): + continue + candidates.setdefault(path, reason) + + topdirs = topdirs_for_package(pkg, self.context.pkg_to_etc_paths) + roots: List[str] = [] + for topdir in sorted(topdirs): + if topdir in SHARED_ETC_TOPDIRS: + continue + if backend.is_pkg_config_path( + f"/etc/{topdir}/" + ) or backend.is_pkg_config_path(f"/etc/{topdir}"): + continue + roots.extend([f"/etc/{topdir}", f"/etc/{topdir}.d"]) + roots.extend(maybe_add_specific_paths(set(topdirs), backend)) + + for pth in scan_unowned_under_roots( + [r for r in roots if os.path.isdir(r)], + self.context.owned_etc, + confish_only=False, + ): + candidates.setdefault(pth, "custom_unowned") + + for root in roots: + if os.path.isfile(root) and not os.path.islink(root): + if root not in self.context.owned_etc and is_confish(root): + candidates.setdefault(root, "custom_specific_path") + + role_seen = self.seen_by_role.setdefault(role, set()) + for path, reason in sorted(candidates.items()): + capture_file( + bundle_dir=self.context.bundle_dir, + role_name=role, + abs_path=path, + reason=reason, + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=managed, + excluded_out=excluded, + seen_role=role_seen, + seen_global=self.context.captured_global, + ) + + has_config = bool(managed or excluded) + if not has_config: + notes.append( + "No changed or custom configuration detected for this package." + ) + simple_packages.append(pkg) + + pkg_snaps.append( + PackageSnapshot( + package=pkg, + role_name=role, + section=package_section_from_installations( + self.context.installed_pkgs.get(pkg, []) + ), + managed_files=managed, + managed_links=[], + excluded=excluded, + notes=notes, + has_config=has_config, + ) + ) + + return pkg_snaps, manual_pkgs, simple_packages, manual_pkgs_skipped + + def _find_role_snapshot( + self, + role_name: str, + service_snaps: List[ServiceSnapshot], + pkg_snaps: List[PackageSnapshot], + ): + for snap in service_snaps: + if snap.role_name == role_name: + return snap + for snap in pkg_snaps: + if snap.role_name == role_name: + return snap + return None + + def _capture_enabled_symlinks_for_role( + self, + role_name: str, + dirs: List[str], + service_snaps: List[ServiceSnapshot], + pkg_snaps: List[PackageSnapshot], + ) -> None: + snap = self._find_role_snapshot(role_name, service_snaps, pkg_snaps) + if snap is None: + return + + role_seen = self.seen_by_role.setdefault(role_name, set()) + for directory in dirs: + if not os.path.isdir(directory): + continue + for pth in sorted(glob.glob(os.path.join(directory, "*"))): + if not os.path.islink(pth): + continue + capture_link( + role_name=role_name, + abs_path=pth, + reason="enabled_symlink", + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=snap.managed_links, + excluded_out=snap.excluded, + seen_role=role_seen, + seen_global=self.context.captured_global, + ) + + def _capture_common_enabled_symlinks( + self, + service_snaps: List[ServiceSnapshot], + pkg_snaps: List[PackageSnapshot], + ) -> None: + self._capture_enabled_symlinks_for_role( + "nginx", + ["/etc/nginx/modules-enabled", "/etc/nginx/sites-enabled"], + service_snaps, + pkg_snaps, + ) + self._capture_enabled_symlinks_for_role( + "apache2", + [ + "/etc/apache2/conf-enabled", + "/etc/apache2/mods-enabled", + "/etc/apache2/sites-enabled", + ], + service_snaps, + pkg_snaps, + ) diff --git a/enroll/harvest_collectors/users.py b/enroll/harvest_collectors/users.py new file mode 100644 index 0000000..d1e86fe --- /dev/null +++ b/enroll/harvest_collectors/users.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any, Dict, List, Set + +from .. import harvest as h +from ..capture import capture_file, capture_user_shell_dotfiles +from ..harvest_types import ( + ExcludedFile, + FlatpakSnapshot, + ManagedFile, + SnapSnapshot, + UsersSnapshot, +) +from .context import HarvestCollector, HarvestContext + + +@dataclass +class UsersCollection: + users_snapshot: UsersSnapshot + flatpak_snapshot: FlatpakSnapshot + snap_snapshot: SnapSnapshot + + +class UsersCollector(HarvestCollector): + """Collect non-system users plus system/user Flatpak and Snap facts.""" + + def __init__( + self, context: HarvestContext, seen_by_role: Dict[str, Set[str]] + ) -> None: + super().__init__(context) + self.seen_by_role = seen_by_role + + def collect(self) -> UsersCollection: + users_notes: List[str] = [] + users_excluded: List[ExcludedFile] = [] + users_managed: List[ManagedFile] = [] + users_list: List[dict] = [] + + try: + user_records = h.collect_non_system_users() + except Exception as e: + user_records = [] + users_notes.append(f"Failed to enumerate users: {e!r}") + + # Detect system-wide Flatpaks/Snaps and configured Flatpak remotes. + from ..accounts import ( + find_system_flatpak_remotes, + find_system_flatpaks, + find_system_snaps, + find_user_flatpak_remotes, + ) + + system_flatpaks = [asdict(f) for f in find_system_flatpaks()] + system_snaps = [asdict(s) for s in find_system_snaps()] + system_flatpak_remotes = [asdict(r) for r in find_system_flatpak_remotes()] + flatpak_notes: List[str] = [] + snap_notes: List[str] = [] + if system_flatpaks: + flatpak_notes.append( + "System-wide flatpaks detected: " + + ", ".join(str(f.get("name")) for f in system_flatpaks) + ) + if system_snaps: + snap_notes.append( + "System-wide snaps detected: " + + ", ".join(str(s.get("name")) for s in system_snaps) + ) + + users_role_name = "users" + users_role_seen = self.seen_by_role.setdefault(users_role_name, set()) + + skel_dir = "/etc/skel" + auto_capture_user_dotfiles = bool( + getattr(self.context.policy, "dangerous", False) + ) + if user_records and not auto_capture_user_dotfiles: + users_notes.append( + "User shell dotfiles were not auto-harvested because --dangerous was not set; " + "use --dangerous for automatic shell-dotfile capture, or targeted " + "--include-path patterns for safe-mode review." + ) + + user_flatpaks_map: Dict[str, List[Dict[str, Any]]] = {} + user_flatpak_remotes: List[Dict[str, Any]] = [] + + for user in user_records: + users_list.append( + { + "name": user.name, + "uid": user.uid, + "gid": user.gid, + "gecos": user.gecos, + "home": user.home, + "shell": user.shell, + "primary_group": user.primary_group, + "supplementary_groups": user.supplementary_groups, + } + ) + + # Copy only safe SSH public material: authorized_keys + *.pub + for ssh_file in user.ssh_files: + reason = ( + "authorized_keys" + if ssh_file.endswith("/authorized_keys") + else "ssh_public_key" + ) + capture_file( + bundle_dir=self.context.bundle_dir, + role_name=users_role_name, + abs_path=ssh_file, + reason=reason, + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=users_managed, + excluded_out=users_excluded, + seen_role=users_role_seen, + seen_global=self.context.captured_global, + ) + + # Capture common per-user shell dotfiles only in dangerous mode. They + # often contain exported tokens or aliases/functions with embedded secrets. + home = (user.home or "").rstrip("/") + if home and home.startswith("/"): + capture_user_shell_dotfiles( + bundle_dir=self.context.bundle_dir, + role_name=users_role_name, + home=home, + skel_dir=skel_dir, + enabled=auto_capture_user_dotfiles, + policy=self.context.policy, + path_filter=self.context.path_filter, + managed_out=users_managed, + excluded_out=users_excluded, + seen_role=users_role_seen, + seen_global=self.context.captured_global, + ) + + # Collect per-user Flatpak applications and remotes. Snap packages are + # system-wide; ~/snap/* is user data, not an install source. + if user.flatpaks: + user_flatpaks_map[user.name] = [asdict(fp) for fp in user.flatpaks] + user_flatpak_remotes.extend( + asdict(r) for r in find_user_flatpak_remotes(home, user=user.name) + ) + + return UsersCollection( + users_snapshot=UsersSnapshot( + role_name="users", + users=users_list, + managed_files=users_managed, + excluded=users_excluded, + notes=users_notes, + user_flatpaks=user_flatpaks_map, + user_flatpak_remotes=user_flatpak_remotes, + ), + flatpak_snapshot=FlatpakSnapshot( + role_name="flatpak", + system_flatpaks=system_flatpaks, + remotes=system_flatpak_remotes, + notes=flatpak_notes, + ), + snap_snapshot=SnapSnapshot( + role_name="snap", + system_snaps=system_snaps, + notes=snap_notes, + ), + ) diff --git a/enroll/harvest_types.py b/enroll/harvest_types.py new file mode 100644 index 0000000..c88163e --- /dev/null +++ b/enroll/harvest_types.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + + +@dataclass +class ManagedFile: + path: str + src_rel: str + owner: str + group: str + mode: str + reason: str + + +@dataclass +class ManagedLink: + """A symlink we want to materialise on the target host. + + For configuration enablement patterns (e.g. sites-enabled), the symlink is + meaningful state even when the link target is captured elsewhere. + """ + + path: str + target: str + reason: str + + +@dataclass +class ManagedDir: + path: str + owner: str + group: str + mode: str + reason: str + + +@dataclass +class ExcludedFile: + path: str + reason: str + + +@dataclass +class ServiceSnapshot: + unit: str + role_name: str + packages: List[str] + active_state: Optional[str] + sub_state: Optional[str] + unit_file_state: Optional[str] + condition_result: Optional[str] + managed_dirs: List[ManagedDir] = field(default_factory=list) + managed_files: List[ManagedFile] = field(default_factory=list) + managed_links: List[ManagedLink] = field(default_factory=list) + excluded: List[ExcludedFile] = field(default_factory=list) + notes: List[str] = field(default_factory=list) + + +@dataclass +class PackageSnapshot: + package: str + role_name: str + section: Optional[str] = None + managed_dirs: List[ManagedDir] = field(default_factory=list) + managed_files: List[ManagedFile] = field(default_factory=list) + managed_links: List[ManagedLink] = field(default_factory=list) + excluded: List[ExcludedFile] = field(default_factory=list) + notes: List[str] = field(default_factory=list) + has_config: bool = True # False if package has no config/systemd/cron files + + +@dataclass +class UsersSnapshot: + role_name: str + users: List[dict] + managed_dirs: List[ManagedDir] = field(default_factory=list) + managed_files: List[ManagedFile] = field(default_factory=list) + excluded: List[ExcludedFile] = field(default_factory=list) + notes: List[str] = field(default_factory=list) + user_flatpaks: Dict[str, List[Dict[str, Any]]] = field(default_factory=dict) + user_flatpak_remotes: List[Dict[str, Any]] = field(default_factory=list) + + +@dataclass +class FlatpakSnapshot: + role_name: str + system_flatpaks: List[Dict[str, Any]] = field(default_factory=list) + remotes: List[Dict[str, Any]] = field(default_factory=list) + notes: List[str] = field(default_factory=list) + + +@dataclass +class SnapSnapshot: + role_name: str + system_snaps: List[Dict[str, Any]] = field(default_factory=list) + notes: List[str] = field(default_factory=list) + + +@dataclass +class ContainerImagesSnapshot: + role_name: str + images: List[Dict[str, Any]] = field(default_factory=list) + notes: List[str] = field(default_factory=list) + + +@dataclass +class AptConfigSnapshot: + role_name: str + managed_dirs: List[ManagedDir] = field(default_factory=list) + managed_files: List[ManagedFile] = field(default_factory=list) + excluded: List[ExcludedFile] = field(default_factory=list) + notes: List[str] = field(default_factory=list) + + +@dataclass +class DnfConfigSnapshot: + role_name: str + managed_dirs: List[ManagedDir] = field(default_factory=list) + managed_files: List[ManagedFile] = field(default_factory=list) + excluded: List[ExcludedFile] = field(default_factory=list) + notes: List[str] = field(default_factory=list) + + +@dataclass +class EtcCustomSnapshot: + role_name: str + managed_dirs: List[ManagedDir] = field(default_factory=list) + managed_files: List[ManagedFile] = field(default_factory=list) + excluded: List[ExcludedFile] = field(default_factory=list) + notes: List[str] = field(default_factory=list) + + +@dataclass +class UsrLocalCustomSnapshot: + role_name: str + managed_dirs: List[ManagedDir] = field(default_factory=list) + managed_files: List[ManagedFile] = field(default_factory=list) + excluded: List[ExcludedFile] = field(default_factory=list) + notes: List[str] = field(default_factory=list) + + +@dataclass +class ExtraPathsSnapshot: + role_name: str + include_patterns: List[str] = field(default_factory=list) + exclude_patterns: List[str] = field(default_factory=list) + managed_dirs: List[ManagedDir] = field(default_factory=list) + managed_files: List[ManagedFile] = field(default_factory=list) + managed_links: List[ManagedLink] = field(default_factory=list) + excluded: List[ExcludedFile] = field(default_factory=list) + notes: List[str] = field(default_factory=list) + + +@dataclass +class FirewallRuntimeSnapshot: + role_name: str + packages: List[str] = field(default_factory=list) + ipset_save: Optional[str] = None + ipset_sets: List[str] = field(default_factory=list) + iptables_v4_save: Optional[str] = None + iptables_v6_save: Optional[str] = None + notes: List[str] = field(default_factory=list) + + +@dataclass +class SysctlSnapshot: + role_name: str + managed_files: List[ManagedFile] = field(default_factory=list) + parameters: Dict[str, str] = field(default_factory=dict) + notes: List[str] = field(default_factory=list) diff --git a/enroll/ignore.py b/enroll/ignore.py index 217497f..a7bf297 100644 --- a/enroll/ignore.py +++ b/enroll/ignore.py @@ -23,6 +23,27 @@ DEFAULT_DENY_GLOBS = [ "/etc/gshadow", "/etc/*shadow", "/etc/letsencrypt/*", + "/usr/local/etc/ssl/private/*", + "/usr/local/etc/ssh/ssh_host_*", + "/usr/local/etc/*shadow", + "/usr/local/etc/*gshadow", + "/usr/local/etc/letsencrypt/*", +] + + +# Allow a small set of binary config artifacts that are commonly required to +# reproduce system configuration (notably APT keyrings). These are still subject +# to size and readability limits, but are exempt from the "binary_like" denial. +DEFAULT_ALLOW_BINARY_GLOBS = [ + "/etc/apt/trusted.gpg", + "/etc/apt/trusted.gpg.d/*.gpg", + "/etc/apt/keyrings/*.gpg", + "/etc/apt/keyrings/*.pgp", + "/etc/apt/keyrings/*.asc", + "/usr/share/keyrings/*.gpg", + "/usr/share/keyrings/*.pgp", + "/usr/share/keyrings/*.asc", + "/etc/pki/rpm-gpg/*", ] SENSITIVE_CONTENT_PATTERNS = [ @@ -31,21 +52,65 @@ SENSITIVE_CONTENT_PATTERNS = [ re.compile(rb"(?i)\b(pass|passwd|token|secret|api[_-]?key)\b"), ] +COMMENT_PREFIXES = (b"#", b";", b"//") +BLOCK_START = b"/*" +BLOCK_END = b"*/" + @dataclass class IgnorePolicy: - deny_globs: list[str] = None + deny_globs: Optional[list[str]] = None + allow_binary_globs: Optional[list[str]] = None max_file_bytes: int = 256_000 sample_bytes: int = 64_000 + # If True, be much less conservative about collecting potentially + # sensitive files. This disables deny globs (e.g. /etc/shadow, + # /etc/ssl/private/*) and skips heuristic content scanning. + dangerous: bool = False def __post_init__(self) -> None: if self.deny_globs is None: self.deny_globs = list(DEFAULT_DENY_GLOBS) + if self.allow_binary_globs is None: + self.allow_binary_globs = list(DEFAULT_ALLOW_BINARY_GLOBS) + + def iter_effective_lines(self, content: bytes): + in_block = False + for raw in content.splitlines(): + line = raw.lstrip() + + if in_block: + if BLOCK_END in line: + in_block = False + continue + + if not line: + continue + + if line.startswith(BLOCK_START): + in_block = True + continue + + if line.startswith(COMMENT_PREFIXES) or line.startswith(b"*"): + continue + + yield raw def deny_reason(self, path: str) -> Optional[str]: - for g in self.deny_globs: - if fnmatch.fnmatch(path, g): - return "denied_path" + # Always ignore plain *.log files (rarely useful as config, often noisy). + if path.endswith(".log"): + return "log_file" + # Ignore editor/backup files that end with a trailing tilde. + if path.endswith("~"): + return "backup_file" + # Ignore backup shadow files + if path.startswith("/etc/") and path.endswith("-"): + return "backup_file" + + if not self.dangerous: + for g in self.deny_globs or []: + if fnmatch.fnmatch(path, g): + return "denied_path" try: st = os.stat(path, follow_symlinks=True) @@ -65,10 +130,88 @@ class IgnorePolicy: return "unreadable" if b"\x00" in data: + for g in self.allow_binary_globs or []: + if fnmatch.fnmatch(path, g): + # Binary is acceptable for explicitly-allowed paths. + return None return "binary_like" - for pat in SENSITIVE_CONTENT_PATTERNS: - if pat.search(data): - return "sensitive_content" + if not self.dangerous: + for line in self.iter_effective_lines(data): + for pat in SENSITIVE_CONTENT_PATTERNS: + if pat.search(line): + return "sensitive_content" + + return None + + def deny_reason_dir(self, path: str) -> Optional[str]: + """Directory-specific deny logic. + + deny_reason() is file-oriented (it rejects directories as "not_regular_file"). + For directory metadata capture (so roles can recreate directory trees), we need + a lighter-weight check: + - apply deny_globs (unless dangerous) + - require the path to be a real directory (no symlink) + - ensure it's stat'able/readable + + No size checks or content scanning are performed for directories. + """ + if not self.dangerous: + for g in self.deny_globs or []: + if fnmatch.fnmatch(path, g): + return "denied_path" + + try: + os.stat(path, follow_symlinks=True) + except OSError: + return "unreadable" + + if os.path.islink(path): + return "symlink" + + if not os.path.isdir(path): + return "not_directory" + + return None + + def deny_reason_link(self, path: str) -> Optional[str]: + """Symlink-specific deny logic. + + Symlinks are meaningful configuration state (e.g. Debian-style + *-enabled directories). deny_reason() is file-oriented and rejects + symlinks as "not_regular_file". + + For symlinks we: + - apply the usual deny_globs (unless dangerous) + - ensure the path is a symlink and we can readlink() it + + No size checks or content scanning are performed for symlinks. + """ + + # Keep the same fast-path filename ignores as deny_reason(). + if path.endswith(".log"): + return "log_file" + if path.endswith("~"): + return "backup_file" + if path.startswith("/etc/") and path.endswith("-"): + return "backup_file" + + if not self.dangerous: + for g in self.deny_globs or []: + if fnmatch.fnmatch(path, g): + return "denied_path" + + try: + os.lstat(path) + except OSError: + return "unreadable" + + if not os.path.islink(path): + return "not_symlink" + + try: + os.readlink(path) + except OSError: + return "unreadable" return None diff --git a/enroll/jinjaturtle.py b/enroll/jinjaturtle.py new file mode 100644 index 0000000..7a2702e --- /dev/null +++ b/enroll/jinjaturtle.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import shutil +import subprocess # nosec +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + + +SYSTEMD_SUFFIXES = { + ".service", + ".socket", + ".target", + ".timer", + ".path", + ".mount", + ".automount", + ".slice", + ".swap", + ".scope", + ".link", + ".netdev", + ".network", +} + +SUPPORTED_SUFFIXES = { + ".ini", + ".cfg", + ".json", + ".toml", + ".yaml", + ".yml", + ".xml", + ".repo", +} | SYSTEMD_SUFFIXES + + +def infer_other_formats(dest_path: str) -> Optional[str]: + p = Path(dest_path) + name = p.name.lower() + suffix = p.suffix.lower() + # postfix + if name == "main.cf": + return "postfix" + # systemd units + if suffix in SYSTEMD_SUFFIXES: + return "systemd" + # OpenSSH system config files and snippets + parts = {part.lower() for part in p.parts} + if name in {"sshd_config", "ssh_config"}: + return "ssh" + if suffix == ".conf" and {"sshd_config.d", "ssh_config.d"} & parts: + return "ssh" + return None + + +@dataclass(frozen=True) +class JinjifyResult: + template_text: str + vars_text: str # YAML mapping text (no leading --- expected) + + +def find_jinjaturtle_cmd() -> Optional[str]: + """Return the executable path for jinjaturtle if found on PATH.""" + return shutil.which("jinjaturtle") + + +def can_jinjify_path(dest_path: str) -> bool: + p = Path(dest_path) + suffix = p.suffix.lower() + if infer_other_formats(dest_path): + return True + # allow unambiguous structured formats + if suffix in SUPPORTED_SUFFIXES: + return True + return False + + +def run_jinjaturtle( + jt_exe: str, + src_path: str, + *, + role_name: str, + force_format: Optional[str] = None, +) -> JinjifyResult: + """ + Run jinjaturtle against src_path and return (template, defaults-yaml). + Uses tempfiles and captures outputs. + + jinjaturtle CLI: + jinjaturtle -r [-f ] [-d ] [-t ] + """ + src = Path(src_path) + if not src.is_file(): + raise FileNotFoundError(src_path) + + with tempfile.TemporaryDirectory(prefix="enroll-jt-") as td: + td_path = Path(td) + defaults_out = td_path / "defaults.yml" + template_out = td_path / "template.j2" + + cmd = [ + jt_exe, + str(src), + "-r", + role_name, + "-d", + str(defaults_out), + "-t", + str(template_out), + ] + if force_format: + cmd.extend(["-f", force_format]) + + p = subprocess.run(cmd, text=True, capture_output=True) # nosec + if p.returncode != 0: + raise RuntimeError( + "jinjaturtle failed for %s (role=%s)\ncmd=%r\nstdout=%s\nstderr=%s" + % (src_path, role_name, cmd, p.stdout, p.stderr) + ) + + vars_text = defaults_out.read_text(encoding="utf-8").strip() + template_text = template_out.read_text(encoding="utf-8") + + # jinjaturtle outputs a YAML mapping; strip leading document marker if present + if vars_text.startswith("---"): + vars_text = "\n".join(vars_text.splitlines()[1:]).lstrip() + + return JinjifyResult( + template_text=template_text, vars_text=vars_text.rstrip() + "\n" + ) diff --git a/enroll/manifest.py b/enroll/manifest.py index b92f59c..32ea271 100644 --- a/enroll/manifest.py +++ b/enroll/manifest.py @@ -1,553 +1,259 @@ from __future__ import annotations -import json import os import shutil -from typing import Any, Dict, List - - -def _yaml_list(items: List[str], indent: int = 2) -> str: - pad = " " * indent - if not items: - return f"{pad}[]" - return "\n".join(f"{pad}- {x}" for x in items) - - -def _copy_artifacts(bundle_dir: str, role: str, role_dir: str) -> None: - artifacts_dir = os.path.join(bundle_dir, "artifacts", role) - if not os.path.isdir(artifacts_dir): - return - for root, _, files in os.walk(artifacts_dir): - for fn in files: - src = os.path.join(root, fn) - rel = os.path.relpath(src, artifacts_dir) - dst = os.path.join(role_dir, "files", rel) - os.makedirs(os.path.dirname(dst), exist_ok=True) - shutil.copy2(src, dst) - - -def _write_role_scaffold(role_dir: str) -> None: - os.makedirs(os.path.join(role_dir, "tasks"), exist_ok=True) - os.makedirs(os.path.join(role_dir, "handlers"), exist_ok=True) - os.makedirs(os.path.join(role_dir, "defaults"), exist_ok=True) - os.makedirs(os.path.join(role_dir, "meta"), exist_ok=True) - os.makedirs(os.path.join(role_dir, "files"), exist_ok=True) - - -def _write_playbook(path: str, roles: List[str]) -> None: - pb_lines = [ - "---", - "- name: Apply all roles on host", - " hosts: all", - " become: true", - " roles:", - ] - for r in roles: - pb_lines.append(f" - {r}") - with open(path, "w", encoding="utf-8") as f: - f.write("\n".join(pb_lines) + "\n") - - -def manifest(bundle_dir: str, out_dir: str) -> None: - state_path = os.path.join(bundle_dir, "state.json") - with open(state_path, "r", encoding="utf-8") as f: - state = json.load(f) - - services: List[Dict[str, Any]] = state.get("services", []) - package_roles: List[Dict[str, Any]] = state.get("package_roles", []) - users_snapshot: Dict[str, Any] = state.get("users", {}) - etc_custom_snapshot: Dict[str, Any] = state.get("etc_custom", {}) - - os.makedirs(out_dir, exist_ok=True) - roles_root = os.path.join(out_dir, "roles") - os.makedirs(roles_root, exist_ok=True) - - manifested_users_roles: List[str] = [] - manifested_etc_custom_roles: List[str] = [] - manifested_service_roles: List[str] = [] - manifested_pkg_roles: List[str] = [] - - # ------------------------- - # Users role (non-system users) - # ------------------------- - if users_snapshot and users_snapshot.get("users"): - role = users_snapshot.get("role_name", "users") - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - _copy_artifacts(bundle_dir, role, role_dir) - - users = users_snapshot.get("users", []) - managed_files = users_snapshot.get("managed_files", []) - excluded = users_snapshot.get("excluded", []) - notes = users_snapshot.get("notes", []) - - # Build group set from users - group_names = set() - for u in users: - pg = u.get("primary_group") - if pg: - group_names.add(pg) - for g in u.get("supplementary_groups", []) or []: - group_names.add(g) - group_names = sorted(group_names) - - # defaults: store users list (handy for later), but tasks are explicit for readability - defaults = """--- -users_accounts: -""" + ( - "\n".join([f" - name: {u.get('name')}" for u in users]) + "\n" - ) - with open( - os.path.join(role_dir, "defaults", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(defaults) - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\ndependencies: []\n") - - # tasks - lines: List[str] = ["---"] - # groups first (idempotent; safe even if already present) - for g in group_names: - lines.append(f"- name: Ensure group {g} exists") - lines.append(" ansible.builtin.group:") - lines.append(f" name: {g}") - lines.append(" state: present") - - # users - for u in users: - name = u["name"] - lines.append(f"- name: Ensure user {name} exists") - lines.append(" ansible.builtin.user:") - lines.append(f" name: {name}") - lines.append(f" uid: {u.get('uid')}") - lines.append(f" group: {u.get('primary_group')}") - supp = u.get("supplementary_groups") or [] - if supp: - lines.append(" groups: " + ",".join(sorted(supp))) - lines.append(" append: true") - lines.append(f" home: {u.get('home')}") - lines.append(" create_home: true") - if u.get("shell"): - lines.append(f" shell: {u.get('shell')}") - if u.get("gecos"): - # quote to avoid YAML surprises - gec = u.get("gecos").replace('"', '"') - lines.append(f' comment: "{gec}"') - lines.append(" state: present") - - # Ensure ~/.ssh - home = u.get("home") or f"/home/{name}" - sshdir = home.rstrip("/") + "/.ssh" - lines.append(f"- name: Ensure {name} .ssh directory exists") - lines.append(" ansible.builtin.file:") - lines.append(f" path: {sshdir}") - lines.append(" state: directory") - lines.append(f" owner: {name}") - lines.append(f" group: {u.get('primary_group')}") - lines.append(" mode: '0700'") - - # Copy harvested SSH public material (authorized_keys) - for mf in managed_files: - dest = mf["path"] - src = mf["src_rel"] - # Determine file owner from dest path: /home//... - owner = None - for u in users: - if dest.startswith((u.get("home") or "").rstrip("/") + "/"): - owner = u["name"] - group = u.get("primary_group") - break - if owner is None: - # fallback: try /home// - parts = dest.split("/") - owner = parts[2] if len(parts) > 2 and parts[1] == "home" else "root" - group = owner - - mode = "0600" if mf.get("reason") == "authorized_keys" else "0644" - lines.append(f"- name: Deploy {dest}") - lines.append(" ansible.builtin.copy:") - lines.append(f" src: {src}") - lines.append(f" dest: {dest}") - lines.append(f" owner: {owner}") - lines.append(f" group: {group}") - lines.append(f" mode: '{mode}'") - - tasks = "\n".join(lines).rstrip() + "\n" - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks) - - # handlers (none needed) - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\n") - - readme = ( - """# users - -Generated non-system user accounts and SSH public material. - -## Users -""" - + ( - "\n".join([f"- {u.get('name')} (uid {u.get('uid')})" for u in users]) - or "- (none)" - ) - + """\n -## Included SSH files -""" - + ( - "\n".join( - [f"- {mf.get('path')} ({mf.get('reason')})" for mf in managed_files] - ) - or "- (none)" - ) - + """\n -## Excluded -""" - + ( - "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) - or "- (none)" - ) - + """\n -## Notes -""" - + ("\n".join([f"- {n}" for n in notes]) or "- (none)") - + """\n""" - ) - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_users_roles.append(role) - - # ------------------------- - # etc_custom role (unowned /etc not already attributed) - # ------------------------- - if etc_custom_snapshot and etc_custom_snapshot.get("managed_files"): - role = etc_custom_snapshot.get("role_name", "etc_custom") - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - _copy_artifacts(bundle_dir, role, role_dir) - - managed_files = etc_custom_snapshot.get("managed_files", []) - excluded = etc_custom_snapshot.get("excluded", []) - notes = etc_custom_snapshot.get("notes", []) - - # tasks: just deploy files (no restarts) - lines: List[str] = ["---"] - for mf in managed_files: - dest = mf["path"] - src = mf["src_rel"] - lines.append(f"- name: Deploy {dest}") - lines.append(" ansible.builtin.copy:") - lines.append(f" src: {src}") - lines.append(f" dest: {dest}") - lines.append(f" owner: {mf.get('owner')}") - lines.append(f" group: {mf.get('group')}") - lines.append(f" mode: '{mf.get('mode')}'") - - tasks = "\n".join(lines).rstrip() + "\n" - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks) - - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\n") - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\ndependencies: []\n") - - readme = ( - """# etc_custom - -Unowned /etc config files not attributed to packages or services. - -## Managed files -""" - + ("\n".join([f"- {mf.get('path')}" for mf in managed_files]) or "- (none)") - + """\n -## Excluded -""" - + ( - "\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded]) - or "- (none)" - ) - + """\n -## Notes -""" - + ("\n".join([f"- {n}" for n in notes]) or "- (none)") - + """\n""" - ) - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_etc_custom_roles.append(role) - - # ------------------------- - # Service roles - # ------------------------- - for svc in services: - role = svc["role_name"] - unit = svc["unit"] - pkgs = svc["packages"] - managed_files = svc["managed_files"] - - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - _copy_artifacts(bundle_dir, role, role_dir) - - var_prefix = role - - was_active = svc.get("active_state") == "active" - defaults = f"""--- -{var_prefix}_packages: -{_yaml_list(pkgs, indent=2)} -{var_prefix}_active_state_at_harvest: "{svc.get("active_state")}" -{var_prefix}_start: {"true" if was_active else "false"} -""" - with open( - os.path.join(role_dir, "defaults", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(defaults) - - handlers = """--- -- name: Run systemd daemon-reload - ansible.builtin.systemd: - daemon_reload: true - -- name: Restart service - ansible.builtin.service: - name: "{{ unit_name }}" - state: restarted -""" - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(handlers) - - systemd_files = [ - mf for mf in managed_files if mf["path"].startswith("/etc/systemd/system/") - ] - other_files = [mf for mf in managed_files if mf not in systemd_files] - - def copy_task(mf: Dict[str, Any], notify: str | None) -> str: - notify_line = f" notify: {notify}\n" if notify else "" - return f"""- name: Deploy {mf["path"]} - ansible.builtin.copy: - src: "{mf["src_rel"]}" - dest: "{mf["path"]}" - owner: "{mf["owner"]}" - group: "{mf["group"]}" - mode: "{mf["mode"]}" -{notify_line}""" - - task_parts: List[str] = [] - task_parts.append( - f"""--- -- name: Set unit name - ansible.builtin.set_fact: - unit_name: "{unit}" - -- name: Install packages for {role} - ansible.builtin.apt: - name: "{{{{ {var_prefix}_packages }}}}" - state: present - update_cache: true - when: {var_prefix}_packages | length > 0 -""" - ) - - if systemd_files: - for mf in systemd_files: - task_parts.append(copy_task(mf, "[systemd daemon-reload]")) - task_parts.append( - """- name: Reload systemd to pick up unit changes - ansible.builtin.meta: flush_handlers -""" - ) - - for mf in other_files: - task_parts.append(copy_task(mf, "[Restart service]")) - - task_parts.append( - f""" -- name: Probe whether systemd unit exists and is manageable - ansible.builtin.systemd: - name: "{{{{ unit_name }}}}" - check_mode: true - register: _unit_probe - failed_when: false - changed_when: false - -- name: Ensure unit is enabled (preserve running state) - ansible.builtin.systemd: - name: "{{{{ unit_name }}}}" - enabled: true - when: _unit_probe is succeeded - -- name: Start unit if it was active at harvest time - ansible.builtin.systemd: - name: "{{{{ unit_name }}}}" - state: started - when: - - _unit_probe is succeeded - - {var_prefix}_start | bool -""" - ) - - tasks = "\n".join(task_parts).rstrip() + "\n" - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks) - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\ndependencies: []\n") - - excluded = svc.get("excluded", []) - notes = svc.get("notes", []) - readme = f"""# {role} - -Generated from `{unit}`. - -## Packages -{os.linesep.join("- " + p for p in pkgs) or "- (none detected)"} - -## Managed files -{os.linesep.join("- " + mf["path"] + " (" + mf["reason"] + ")" for mf in managed_files) or "- (none)"} - -## Excluded (possible secrets / unsafe) -{os.linesep.join("- " + e["path"] + " (" + e["reason"] + ")" for e in excluded) or "- (none)"} - -## Notes -{os.linesep.join("- " + n for n in notes) or "- (none)"} -""" - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_service_roles.append(role) - - # ------------------------- - # Manually installed package roles - # ------------------------- - for pr in package_roles: - role = pr["role_name"] - pkg = pr["package"] - managed_files = pr["managed_files"] - - role_dir = os.path.join(roles_root, role) - _write_role_scaffold(role_dir) - _copy_artifacts(bundle_dir, role, role_dir) - - var_prefix = role - - defaults = f"""--- -{var_prefix}_packages: - - {pkg} -""" - with open( - os.path.join(role_dir, "defaults", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(defaults) - - handlers = """--- -- name: Run systemd daemon-reload - ansible.builtin.systemd: - daemon_reload: true -""" - with open( - os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(handlers) - - systemd_files = [ - mf for mf in managed_files if mf["path"].startswith("/etc/systemd/system/") - ] - other_files = [mf for mf in managed_files if mf not in systemd_files] - - def copy_task(mf: Dict[str, Any], notify: str | None) -> str: - notify_line = f" notify: {notify}\n" if notify else "" - return f"""- name: Deploy {mf["path"]} - ansible.builtin.copy: - src: "{mf["src_rel"]}" - dest: "{mf["path"]}" - owner: "{mf["owner"]}" - group: "{mf["group"]}" - mode: "{mf["mode"]}" -{notify_line}""" - - task_parts: List[str] = [] - task_parts.append( - f"""--- -- name: Install package {pkg} - ansible.builtin.apt: - name: "{{{{ {var_prefix}_packages }}}}" - state: present - update_cache: true -""" - ) - - if systemd_files: - for mf in systemd_files: - task_parts.append(copy_task(mf, "[systemd daemon-reload]")) - task_parts.append( - """- name: Reload systemd to pick up unit changes - ansible.builtin.meta: flush_handlers -""" - ) - - for mf in other_files: - task_parts.append(copy_task(mf, None)) - - tasks = "\n".join(task_parts).rstrip() + "\n" - with open( - os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write(tasks) - - with open( - os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" - ) as f: - f.write("---\ndependencies: []\n") - - excluded = pr.get("excluded", []) - notes = pr.get("notes", []) - readme = f"""# {role} - -Generated for package `{pkg}`. - -## Managed files -{os.linesep.join("- " + mf["path"] + " (" + mf["reason"] + ")" for mf in managed_files) or "- (none)"} - -## Excluded (possible secrets / unsafe) -{os.linesep.join("- " + e["path"] + " (" + e["reason"] + ")" for e in excluded) or "- (none)"} - -## Notes -{os.linesep.join("- " + n for n in notes) or "- (none)"} - -> Note: package roles (those not discovered via a systemd service) do not attempt to restart or enable services automatically. -""" - with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: - f.write(readme) - - manifested_pkg_roles.append(role) - - # Playbooks - _write_playbook( - os.path.join(out_dir, "playbook.yml"), - manifested_pkg_roles - + manifested_service_roles - + manifested_etc_custom_roles - + manifested_users_roles, +import tarfile +import tempfile +from pathlib import Path +from typing import List, Optional + +from .ansible import manifest_from_bundle_dir as manifest_ansible_from_bundle_dir +from .puppet import manifest_from_bundle_dir as manifest_puppet_from_bundle_dir +from .remote import _safe_extract_tar +from .sopsutil import ( + decrypt_file_binary_to, + encrypt_file_binary, + require_sops_cmd, +) + + +def _prepare_bundle_dir( + bundle: str, + *, + sops_mode: bool, +) -> tuple[str, Optional[tempfile.TemporaryDirectory]]: + """Return (bundle_dir, tempdir). + + - In non-sops mode, `bundle` must be a directory. + - In sops mode, `bundle` may be a directory (already-decrypted) *or* + a SOPS-encrypted tarball. In the tarball case we decrypt+extract into + a secure temp directory. + """ + p = Path(bundle).expanduser() + + if p.is_dir(): + return str(p), None + + if not sops_mode: + raise RuntimeError(f"Harvest path is not a directory: {p}") + + if not p.exists(): + raise RuntimeError(f"Harvest path not found: {p}") + + # Ensure sops is available early for clear error messages. + require_sops_cmd() + + td = tempfile.TemporaryDirectory(prefix="enroll-harvest-") + td_path = Path(td.name) + try: + os.chmod(td_path, 0o700) + except OSError: + pass + + tar_path = td_path / "harvest.tar.gz" + out_dir = td_path / "bundle" + out_dir.mkdir(parents=True, exist_ok=True) + try: + os.chmod(out_dir, 0o700) + except OSError: + pass + + decrypt_file_binary_to(p, tar_path, mode=0o600) + + # Extract using the same safe extraction rules as remote harvesting. + with tarfile.open(tar_path, mode="r:gz") as tf: + _safe_extract_tar(tf, out_dir) + + return str(out_dir), td + + +def _resolve_sops_manifest_out_file(out: str) -> Path: + """Resolve an output *file* path for manifest --sops mode. + + If `out` looks like a directory (or points to an existing directory), we + place the encrypted manifest bundle inside it as manifest.tar.gz.sops. + """ + p = Path(out).expanduser() + if p.exists() and p.is_dir(): + return p / "manifest.tar.gz.sops" + # Heuristic: treat paths with a suffix as files; otherwise directories. + if p.suffix: + return p + return p / "manifest.tar.gz.sops" + + +def _tar_dir_to_with_progress( + src_dir: Path, tar_path: Path, *, desc: str = "tarring" +) -> None: + """Create a tar.gz of src_dir at tar_path, with a simple per-entry progress display.""" + src_dir = Path(src_dir) + tar_path = Path(tar_path) + tar_path.parent.mkdir(parents=True, exist_ok=True) + + # Collect paths (dirs + files) + paths: list[Path] = [src_dir] + for root, dirs, files in os.walk(str(src_dir)): + root_p = Path(root) + for d in sorted(dirs): + paths.append(root_p / d) + for f in sorted(files): + paths.append(root_p / f) + + total = len(paths) + is_tty = hasattr(os, "isatty") and os.isatty(2) + + def _print_progress(i: int, p: Path) -> None: + if not is_tty: + return + pct = (i / total * 100.0) if total else 100.0 + rel = "." + try: + rel = str(p.relative_to(src_dir)) + except Exception: + rel = str(p) + msg = f"{desc}: {i}/{total} ({pct:5.1f}%) {rel}" + try: + cols = shutil.get_terminal_size((80, 20)).columns + msg = msg[: cols - 1] + except Exception: + pass # nosec + os.write(2, ("\r" + msg).encode("utf-8", errors="replace")) + + with tarfile.open(tar_path, mode="w:gz") as tf: + prefix = Path("manifest") + + for i, p in enumerate(paths, start=1): + if p == src_dir: + arcname = str(prefix) + else: + rel = p.relative_to(src_dir) + arcname = str(prefix / rel) + tf.add(str(p), arcname=arcname, recursive=False) + _print_progress(i, p) + + if is_tty: + os.write(2, b"\n") + + +def _encrypt_manifest_out_dir_to_sops( + out_dir: Path, out_file: Path, fps: list[str] +) -> Path: + """Tar+encrypt the generated manifest output directory into a single .sops file.""" + require_sops_cmd() + out_file = Path(out_file) + out_file.parent.mkdir(parents=True, exist_ok=True) + + fd, tmp_tgz = tempfile.mkstemp( + prefix=".enroll-manifest-", + suffix=".tar.gz", + dir=str(out_file.parent), ) + os.close(fd) + try: + _tar_dir_to_with_progress( + Path(out_dir), Path(tmp_tgz), desc="Bundling manifest" + ) + encrypt_file_binary(Path(tmp_tgz), out_file, pgp_fingerprints=fps, mode=0o600) + finally: + try: + os.unlink(tmp_tgz) + except FileNotFoundError: + pass + + return out_file + + +def manifest( + bundle_dir: str, + out: str, + *, + fqdn: Optional[str] = None, + jinjaturtle: str = "auto", # auto|on|off + sops_fingerprints: Optional[List[str]] = None, + no_common_roles: bool = False, + target: str = "ansible", +) -> Optional[str]: + """Render a configuration-management manifest from a harvest. + + Plain mode: + - `bundle_dir` must be a directory + - `out` is a directory written in-place + + SOPS mode (when `sops_fingerprints` is provided): + - `bundle_dir` may be either a directory (already decrypted) or a SOPS + encrypted tarball (binary) produced by `harvest --sops` + - the manifest output is bundled (tar.gz) and encrypted into a single + SOPS file (binary) at the resolved output path. + + Returns: + - In SOPS mode: the path to the encrypted manifest bundle (.sops) + - In plain mode: None + """ + target = (target or "ansible").strip().lower() + if target not in {"ansible", "puppet"}: + raise ValueError(f"unsupported manifest target: {target!r}") + + sops_mode = bool(sops_fingerprints) + + # Decrypt/extract the harvest bundle if needed. + resolved_bundle_dir, td_bundle = _prepare_bundle_dir( + bundle_dir, sops_mode=sops_mode + ) + + td_out: Optional[tempfile.TemporaryDirectory] = None + try: + if not sops_mode: + if target == "puppet": + manifest_puppet_from_bundle_dir( + resolved_bundle_dir, + out, + fqdn=fqdn, + no_common_roles=no_common_roles, + ) + else: + manifest_ansible_from_bundle_dir( + resolved_bundle_dir, + out, + fqdn=fqdn, + jinjaturtle=jinjaturtle, + no_common_roles=no_common_roles, + ) + return None + + # SOPS mode: generate into a secure temp dir, then tar+encrypt into a single file. + out_file = _resolve_sops_manifest_out_file(out) + + td_out = tempfile.TemporaryDirectory(prefix="enroll-manifest-") + tmp_out = Path(td_out.name) / "out" + tmp_out.mkdir(parents=True, exist_ok=True) + try: + os.chmod(tmp_out, 0o700) + except OSError: + pass + + if target == "puppet": + manifest_puppet_from_bundle_dir( + resolved_bundle_dir, + str(tmp_out), + fqdn=fqdn, + no_common_roles=no_common_roles, + ) + else: + manifest_ansible_from_bundle_dir( + resolved_bundle_dir, + str(tmp_out), + fqdn=fqdn, + jinjaturtle=jinjaturtle, + no_common_roles=no_common_roles, + ) + + enc = _encrypt_manifest_out_dir_to_sops( + tmp_out, out_file, list(sops_fingerprints or []) + ) + return str(enc) + + finally: + if td_out is not None: + td_out.cleanup() + if td_bundle is not None: + td_bundle.cleanup() diff --git a/enroll/package_hints.py b/enroll/package_hints.py new file mode 100644 index 0000000..b710ed2 --- /dev/null +++ b/enroll/package_hints.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import re +from typing import Dict, List, Optional, Set + +from .role_names import avoid_reserved_role_name + + +# Directories that are shared across many packages. Never attribute all unowned +# files in these trees to one single package. +SHARED_ETC_TOPDIRS = { + "apparmor.d", + "apt", + "cron.d", + "cron.daily", + "cron.weekly", + "cron.monthly", + "cron.hourly", + "default", + "init.d", + "logrotate.d", + "modprobe.d", + "network", + "pam.d", + "ssh", + "ssl", + "sudoers.d", + "sysctl.d", + "systemd", + # RPM-family shared trees + "dnf", + "yum", + "yum.repos.d", + "sysconfig", + "pki", + "firewalld", +} + + +def safe_name(s: str) -> str: + out: List[str] = [] + for ch in s: + out.append(ch if ch.isalnum() or ch in ("_", "-") else "_") + return "".join(out).replace("-", "_") + + +def role_id(raw: str) -> str: + # normalise separators first + s = re.sub(r"[^A-Za-z0-9]+", "_", raw) + # split CamelCase -> snake_case + s = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s) + s = s.lower() + s = re.sub(r"_+", "_", s).strip("_") + if not re.match(r"^[a-z_]", s): + s = "r_" + s + return s + + +def role_name_from_unit(unit: str) -> str: + base = role_id(unit.removesuffix(".service")) + return avoid_reserved_role_name(safe_name(base), prefix="service") + + +def role_name_from_pkg(pkg: str) -> str: + return avoid_reserved_role_name(safe_name(pkg), prefix="package") + + +def package_section_from_installations( + installs: List[Dict[str, str]], +) -> Optional[str]: + """Return a stable package grouping label from installed package metadata.""" + + values: Set[str] = set() + for inst in installs or []: + value = (inst.get("section") or inst.get("group") or "").strip() + if not value: + continue + if value.lower() in {"(none)", "none", "unspecified"}: + continue + values.add(value) + + if not values: + return None + return sorted(values)[0] + + +def hint_names(unit: str, pkgs: Set[str]) -> Set[str]: + base = unit.removesuffix(".service") + hints = {base} + if "@" in base: + hints.add(base.split("@", 1)[0]) + hints |= set(pkgs) + hints |= {h.split(".", 1)[0] for h in list(hints) if "." in h} + return {h for h in hints if h} + + +def add_pkgs_from_etc_topdirs( + hints: Set[str], topdir_to_pkgs: Dict[str, Set[str]], pkgs: Set[str] +) -> None: + """Expand a service's package set using package-owned /etc top-level dirs.""" + + for h in hints: + for top in (h, f"{h}.d"): + if top in SHARED_ETC_TOPDIRS: + continue + for p in topdir_to_pkgs.get(top, set()): + pkgs.add(p) + + +def maybe_add_specific_paths(hints: Set[str], backend) -> List[str]: + # Delegate to backend-specific conventions (e.g. /etc/default on Debian, + # /etc/sysconfig on Fedora/RHEL). Always include sysctl.d. + try: + return backend.specific_paths_for_hints(hints) + except Exception: + # Best-effort fallback (Debian-ish). + paths: List[str] = [] + for h in hints: + paths.extend( + [ + f"/etc/default/{h}", + f"/etc/init.d/{h}", + f"/etc/sysctl.d/{h}.conf", + ] + ) + return paths diff --git a/enroll/pathfilter.py b/enroll/pathfilter.py new file mode 100644 index 0000000..680d390 --- /dev/null +++ b/enroll/pathfilter.py @@ -0,0 +1,293 @@ +from __future__ import annotations + +import glob +import os +import re +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import List, Optional, Sequence, Set, Tuple + + +_REGEX_PREFIXES = ("re:", "regex:") + + +def _has_glob_chars(s: str) -> bool: + return any(ch in s for ch in "*?[") + + +def _norm_abs(p: str) -> str: + """Normalise a path-ish string to an absolute POSIX path. + + We treat inputs that don't start with '/' as being relative to '/'. + """ + + p = p.strip() + if not p: + return "/" + if not p.startswith("/"): + p = "/" + p + # `normpath` keeps a leading '/' for absolute paths. + return os.path.normpath(p) + + +def _posix_match(path: str, pattern: str) -> bool: + """Path matching with glob semantics. + + Uses PurePosixPath.match which: + - treats '/' as a segment separator + - supports '**' for recursive matching + + Both `path` and `pattern` are treated as absolute paths. + """ + + # PurePosixPath.match is anchored and works best on relative strings. + p = path.lstrip("/") + pat = pattern.lstrip("/") + try: + return PurePosixPath(p).match(pat) + except Exception: + # If the pattern is somehow invalid, fail closed. + return False + + +def _regex_literal_prefix(regex: str) -> str: + """Best-effort literal prefix extraction for a regex. + + This lets us pick a starting directory to walk when expanding regex-based + include patterns. + """ + + s = regex + if s.startswith("^"): + s = s[1:] + out: List[str] = [] + escaped = False + meta = set(".^$*+?{}[]\\|()") + for ch in s: + if escaped: + out.append(ch) + escaped = False + continue + if ch == "\\": + escaped = True + continue + if ch in meta: + break + out.append(ch) + return "".join(out) + + +@dataclass(frozen=True) +class CompiledPathPattern: + raw: str + kind: str # 'prefix' | 'glob' | 'regex' + value: str + regex: Optional[re.Pattern[str]] = None + + def matches(self, path: str) -> bool: + p = _norm_abs(path) + + if self.kind == "regex": + if not self.regex: + return False + # Search (not match) so users can write unanchored patterns. + return self.regex.search(p) is not None + + if self.kind == "glob": + return _posix_match(p, self.value) + + # prefix + pref = self.value.rstrip("/") + return p == pref or p.startswith(pref + "/") + + +def compile_path_pattern(raw: str) -> CompiledPathPattern: + s = raw.strip() + for pre in _REGEX_PREFIXES: + if s.startswith(pre): + rex = s[len(pre) :].strip() + try: + return CompiledPathPattern( + raw=raw, kind="regex", value=rex, regex=re.compile(rex) + ) + except re.error: + # Treat invalid regexes as non-matching. + return CompiledPathPattern(raw=raw, kind="regex", value=rex, regex=None) + + # If the user explicitly says glob:, honour it. + if s.startswith("glob:"): + pat = s[len("glob:") :].strip() + return CompiledPathPattern(raw=raw, kind="glob", value=_norm_abs(pat)) + + # Heuristic: if it contains glob metacharacters, treat as a glob. + if _has_glob_chars(s) or "**" in s: + return CompiledPathPattern(raw=raw, kind="glob", value=_norm_abs(s)) + + # Otherwise treat as an exact path-or-prefix (dir subtree). + return CompiledPathPattern(raw=raw, kind="prefix", value=_norm_abs(s)) + + +@dataclass +class PathFilter: + """User-provided path filters. + + Semantics: + - exclude patterns always win + - include patterns are used only to expand *additional* files to harvest + (they do not restrict the default harvest set) + + Patterns: + - By default: glob-like (supports '**') + - Regex: prefix with 're:' or 'regex:' + - Force glob: prefix with 'glob:' + - A plain path without wildcards matches that path and everything under it + (directory-prefix behaviour). + + Examples: + --exclude-path /usr/local/bin/docker-* + --include-path /home/*/.bashrc + --include-path 're:^/home/[^/]+/.config/myapp/.*$' + """ + + include: Sequence[str] = () + exclude: Sequence[str] = () + + def __post_init__(self) -> None: + self._include = [ + compile_path_pattern(p) for p in self.include if str(p).strip() + ] + self._exclude = [ + compile_path_pattern(p) for p in self.exclude if str(p).strip() + ] + + def is_excluded(self, path: str) -> bool: + for pat in self._exclude: + if pat.matches(path): + return True + return False + + def iter_include_patterns(self) -> List[CompiledPathPattern]: + return list(self._include) + + +def expand_includes( + patterns: Sequence[CompiledPathPattern], + *, + exclude: Optional[PathFilter] = None, + max_files: int, +) -> Tuple[List[str], List[str]]: + """Expand include patterns into concrete file paths. + + Returns (paths, notes). The returned paths are absolute paths. + + This function is intentionally conservative: + - symlinks are ignored (both dirs and files) + - the number of collected files is capped + + Regex patterns are expanded by walking a best-effort inferred root. + """ + + out: List[str] = [] + notes: List[str] = [] + seen: Set[str] = set() + + def _maybe_add_file(p: str) -> None: + if len(out) >= max_files: + return + p = _norm_abs(p) + if exclude and exclude.is_excluded(p): + return + if p in seen: + return + if not os.path.isfile(p) or os.path.islink(p): + return + seen.add(p) + out.append(p) + + def _walk_dir(root: str, match: Optional[CompiledPathPattern] = None) -> None: + root = _norm_abs(root) + if not os.path.isdir(root) or os.path.islink(root): + return + for dirpath, dirnames, filenames in os.walk(root, followlinks=False): + # Prune excluded directories early. + if exclude: + dirnames[:] = [ + d + for d in dirnames + if not exclude.is_excluded(os.path.join(dirpath, d)) + and not os.path.islink(os.path.join(dirpath, d)) + ] + for fn in filenames: + if len(out) >= max_files: + return + p = os.path.join(dirpath, fn) + if os.path.islink(p) or not os.path.isfile(p): + continue + if exclude and exclude.is_excluded(p): + continue + if match is not None and not match.matches(p): + continue + if p in seen: + continue + seen.add(p) + out.append(_norm_abs(p)) + + for pat in patterns: + if len(out) >= max_files: + notes.append( + f"Include cap reached ({max_files}); some includes were not expanded." + ) + break + + matched_any = False + + if pat.kind == "prefix": + p = pat.value + if os.path.isfile(p) and not os.path.islink(p): + _maybe_add_file(p) + matched_any = True + elif os.path.isdir(p) and not os.path.islink(p): + before = len(out) + _walk_dir(p) + matched_any = len(out) > before + else: + # Still allow prefix patterns that don't exist now (e.g. remote different) + # by matching nothing rather than erroring. + matched_any = False + + elif pat.kind == "glob": + # Use glob for expansion; also walk directories that match. + gpat = pat.value + hits = glob.glob(gpat, recursive=True) + for h in hits: + if len(out) >= max_files: + break + h = _norm_abs(h) + if exclude and exclude.is_excluded(h): + continue + if os.path.isdir(h) and not os.path.islink(h): + before = len(out) + _walk_dir(h) + if len(out) > before: + matched_any = True + elif os.path.isfile(h) and not os.path.islink(h): + _maybe_add_file(h) + matched_any = True + + else: # regex + rex = pat.value + prefix = _regex_literal_prefix(rex) + # Determine a walk root. If we can infer an absolute prefix, use its + # directory; otherwise fall back to '/'. + if prefix.startswith("/"): + root = os.path.dirname(prefix) or "/" + else: + root = "/" + before = len(out) + _walk_dir(root, match=pat) + matched_any = len(out) > before + + if not matched_any: + notes.append(f"Include pattern matched no files: {pat.raw!r}") + + return out, notes diff --git a/enroll/platform.py b/enroll/platform.py new file mode 100644 index 0000000..3c1904b --- /dev/null +++ b/enroll/platform.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +import shutil +from dataclasses import dataclass +from typing import Dict, List, Optional, Set, Tuple + +from .fsutil import file_md5 + + +def _read_os_release(path: str = "/etc/os-release") -> Dict[str, str]: + out: Dict[str, str] = {} + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + for raw in f: + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + k = k.strip() + v = v.strip().strip('"') + out[k] = v + except OSError: + return {} + return out + + +@dataclass +class PlatformInfo: + os_family: str # debian|redhat|unknown + pkg_backend: str # dpkg|rpm|unknown + os_release: Dict[str, str] + + +def detect_platform() -> PlatformInfo: + """Detect platform family and package backend. + + Uses /etc/os-release when available, with a conservative fallback to + checking for dpkg/rpm binaries. + """ + + osr = _read_os_release() + os_id = (osr.get("ID") or "").strip().lower() + likes = (osr.get("ID_LIKE") or "").strip().lower().split() + + deb_ids = {"debian", "ubuntu", "linuxmint", "raspbian", "kali"} + rhel_ids = { + "fedora", + "rhel", + "centos", + "rocky", + "almalinux", + "ol", + "oracle", + "scientific", + } + + if os_id in deb_ids or "debian" in likes: + return PlatformInfo(os_family="debian", pkg_backend="dpkg", os_release=osr) + if os_id in rhel_ids or any( + x in likes for x in ("rhel", "fedora", "centos", "redhat") + ): + return PlatformInfo(os_family="redhat", pkg_backend="rpm", os_release=osr) + + # Fallback heuristics. + if shutil.which("dpkg"): + return PlatformInfo(os_family="debian", pkg_backend="dpkg", os_release=osr) + if shutil.which("rpm"): + return PlatformInfo(os_family="redhat", pkg_backend="rpm", os_release=osr) + return PlatformInfo(os_family="unknown", pkg_backend="unknown", os_release=osr) + + +class PackageBackend: + """Backend abstraction for package ownership, config detection, and manual package lists.""" + + name: str + pkg_config_prefixes: Tuple[str, ...] + + def owner_of_path(self, path: str) -> Optional[str]: # pragma: no cover + raise NotImplementedError + + def list_manual_packages(self) -> List[str]: # pragma: no cover + raise NotImplementedError + + def installed_packages(self) -> Dict[str, List[Dict[str, str]]]: # pragma: no cover + """Return mapping of package name -> installed instances. + + Each instance is a dict with at least: + - version: package version string + - arch: architecture string + + Backends should be best-effort and return an empty mapping on failure. + """ + raise NotImplementedError + + def build_etc_index( + self, + ) -> Tuple[ + Set[str], Dict[str, str], Dict[str, Set[str]], Dict[str, List[str]] + ]: # pragma: no cover + raise NotImplementedError + + def specific_paths_for_hints(self, hints: Set[str]) -> List[str]: + return [] + + def is_pkg_config_path(self, path: str) -> bool: + for pfx in self.pkg_config_prefixes: + if path == pfx or path.startswith(pfx): + return True + return False + + def modified_paths(self, pkg: str, etc_paths: List[str]) -> Dict[str, str]: + """Return a mapping of modified file paths -> reason label.""" + return {} + + +class DpkgBackend(PackageBackend): + name = "dpkg" + pkg_config_prefixes = ("/etc/apt/",) + + def __init__(self) -> None: + from .debian import parse_status_conffiles + + self._conffiles_by_pkg = parse_status_conffiles() + + def owner_of_path(self, path: str) -> Optional[str]: + from .debian import dpkg_owner + + return dpkg_owner(path) + + def list_manual_packages(self) -> List[str]: + from .debian import list_manual_packages + + return list_manual_packages() + + def installed_packages(self) -> Dict[str, List[Dict[str, str]]]: + from .debian import list_installed_packages + + return list_installed_packages() + + def build_etc_index(self): + from .debian import build_dpkg_etc_index + + return build_dpkg_etc_index() + + def specific_paths_for_hints(self, hints: Set[str]) -> List[str]: + paths: List[str] = [] + for h in hints: + paths.extend( + [ + f"/etc/default/{h}", + f"/etc/init.d/{h}", + f"/etc/sysctl.d/{h}.conf", + ] + ) + return paths + + def modified_paths(self, pkg: str, etc_paths: List[str]) -> Dict[str, str]: + from .debian import read_pkg_md5sums + + out: Dict[str, str] = {} + conff = self._conffiles_by_pkg.get(pkg, {}) + md5sums = read_pkg_md5sums(pkg) + + for path in etc_paths: + if not path.startswith("/etc/"): + continue + if self.is_pkg_config_path(path): + continue + if path in conff: + try: + current = file_md5(path) + except OSError: + continue + if current != conff[path]: + out[path] = "modified_conffile" + continue + + rel = path.lstrip("/") + baseline = md5sums.get(rel) + if baseline: + try: + current = file_md5(path) + except OSError: + continue + if current != baseline: + out[path] = "modified_packaged_file" + return out + + +class RpmBackend(PackageBackend): + name = "rpm" + pkg_config_prefixes = ( + "/etc/dnf/", + "/etc/yum/", + "/etc/yum.repos.d/", + "/etc/yum.conf", + ) + + def __init__(self) -> None: + self._modified_cache: Dict[str, Set[str]] = {} + self._config_cache: Dict[str, Set[str]] = {} + + def owner_of_path(self, path: str) -> Optional[str]: + from .rpm import rpm_owner + + return rpm_owner(path) + + def list_manual_packages(self) -> List[str]: + from .rpm import list_manual_packages + + return list_manual_packages() + + def installed_packages(self) -> Dict[str, List[Dict[str, str]]]: + from .rpm import list_installed_packages + + return list_installed_packages() + + def build_etc_index(self): + from .rpm import build_rpm_etc_index + + return build_rpm_etc_index() + + def specific_paths_for_hints(self, hints: Set[str]) -> List[str]: + paths: List[str] = [] + for h in hints: + paths.extend( + [ + f"/etc/sysconfig/{h}", + f"/etc/sysconfig/{h}.conf", + f"/etc/sysctl.d/{h}.conf", + ] + ) + return paths + + def _config_files(self, pkg: str) -> Set[str]: + if pkg in self._config_cache: + return self._config_cache[pkg] + from .rpm import rpm_config_files + + s = rpm_config_files(pkg) + self._config_cache[pkg] = s + return s + + def _modified_files(self, pkg: str) -> Set[str]: + if pkg in self._modified_cache: + return self._modified_cache[pkg] + from .rpm import rpm_modified_files + + s = rpm_modified_files(pkg) + self._modified_cache[pkg] = s + return s + + def modified_paths(self, pkg: str, etc_paths: List[str]) -> Dict[str, str]: + out: Dict[str, str] = {} + modified = self._modified_files(pkg) + if not modified: + return out + config = self._config_files(pkg) + + for path in etc_paths: + if not path.startswith("/etc/"): + continue + if self.is_pkg_config_path(path): + continue + if path not in modified: + continue + out[path] = ( + "modified_conffile" if path in config else "modified_packaged_file" + ) + return out + + +def get_backend(info: Optional[PlatformInfo] = None) -> PackageBackend: + info = info or detect_platform() + if info.pkg_backend == "dpkg": + return DpkgBackend() + if info.pkg_backend == "rpm": + return RpmBackend() + # Unknown: be conservative and use an rpm backend if rpm exists, otherwise dpkg. + if shutil.which("rpm"): + return RpmBackend() + return DpkgBackend() diff --git a/enroll/puppet.py b/enroll/puppet.py new file mode 100644 index 0000000..3f18d41 --- /dev/null +++ b/enroll/puppet.py @@ -0,0 +1,1231 @@ +from __future__ import annotations + +import hashlib +import json +import re +import shlex +import shutil +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple + +import yaml + +from .cm import ( + CMModule, + package_section_label, + resolve_catalog_conflicts, + role_order_key, + section_label_for_packages, +) +from .state import inventory_packages_from_state, roles_from_state + + +class PuppetRole(CMModule): + """Puppet-specific view of a renderer-neutral CMModule.""" + + def __init__(self, role_name: str) -> None: + super().__init__( + role_name=role_name, + module_name=_puppet_name(role_name, fallback="enroll_role"), + ) + self.container_images: List[Dict[str, Any]] = [] + + def has_resources(self) -> bool: + return super().has_resources() or bool(self.container_images) + + def add_package_snapshot(self, snap: Dict[str, Any]) -> None: + pkg = str(snap.get("package") or "").strip() + if pkg: + self.packages.add(pkg) + + def add_service_snapshot(self, snap: Dict[str, Any]) -> None: + for pkg in snap.get("packages", []) or []: + pkg_s = str(pkg or "").strip() + if pkg_s: + self.packages.add(pkg_s) + unit = str(snap.get("unit") or "").strip() + if unit: + unit_file_state = str(snap.get("unit_file_state") or "") + self.services[unit] = { + "name": unit, + "ensure": ( + "running" if snap.get("active_state") == "active" else "stopped" + ), + "enable": unit_file_state in ("enabled", "enabled-runtime"), + } + + def add_users_snapshot(self, snap: Dict[str, Any]) -> None: + for u in snap.get("users", []) or []: + if not isinstance(u, dict): + continue + name = str(u.get("name") or "").strip() + if not name: + continue + primary_group = str(u.get("primary_group") or name).strip() + if primary_group: + self.groups.add(primary_group) + supplementary = sorted( + { + str(g).strip() + for g in (u.get("supplementary_groups") or []) + if str(g).strip() + } + ) + self.groups.update(supplementary) + self.users[name] = { + "name": name, + "uid": u.get("uid"), + "gid": u.get("gid"), + "primary_group": primary_group or None, + "home": u.get("home") or f"/home/{name}", + "shell": u.get("shell"), + "gecos": u.get("gecos"), + "supplementary_groups": supplementary, + } + + if snap.get("user_flatpaks") or snap.get("user_flatpak_remotes"): + self.notes.append( + "Per-user Flatpak resources were detected but are not yet rendered as native Puppet resources." + ) + + def add_container_images_snapshot(self, snap: Dict[str, Any]) -> None: + for raw in snap.get("images", []) or []: + if not isinstance(raw, dict): + continue + engine = str(raw.get("engine") or "").strip().lower() + pull_ref = str(raw.get("pull_ref") or "").strip() + if engine not in {"docker", "podman"}: + continue + if not pull_ref: + tags = ", ".join(str(t) for t in (raw.get("repo_tags") or []) if t) + label = tags or str(raw.get("image_id") or "unknown image") + self.notes.append( + f"Container image {label} has no RepoDigest; exact Puppet pull resource was not rendered." + ) + continue + item = dict(raw) + item["engine"] = engine + item["pull_ref"] = pull_ref + item["scope"] = str(item.get("scope") or "system").strip() or "system" + image_name, image_digest = _split_digest_ref(pull_ref) + item["image"] = image_name + item["image_digest"] = image_digest + item["tag_aliases"] = [ + dict(alias) + for alias in (item.get("tag_aliases") or []) + if isinstance(alias, dict) and alias.get("ref") + ] + item["pull_cmd"] = _container_pull_cmd(engine, pull_ref) + item["pull_unless"] = _container_exists_cmd(engine, pull_ref) + for alias in item["tag_aliases"]: + alias_ref = str(alias.get("ref") or "") + alias["tag_cmd"] = _container_tag_cmd(engine, pull_ref, alias_ref) + alias["tag_unless"] = _container_exists_cmd(engine, alias_ref) + self.container_images.append(item) + for note in snap.get("notes", []) or []: + self.notes.append(str(note)) + + def add_managed_content( + self, + snap: Dict[str, Any], + *, + bundle_dir: str, + artifact_role: str, + module_files_dir: Path, + file_prefix: Optional[str] = None, + ) -> None: + for d in self.managed_dirs_from_snapshot(snap): + path = str(d.get("path") or "").strip() + self.add_managed_dir( + path, + owner=d.get("owner") or "root", + group=d.get("group") or "root", + mode=d.get("mode") or "0755", + reason=d.get("reason") or "managed_dir", + ) + + for mf in self.managed_files_from_snapshot(snap): + path = str(mf.get("path") or "").strip() + src_rel = str(mf.get("src_rel") or "").strip() + if not path or not src_rel: + continue + module_rel = _copy_artifact( + bundle_dir, + artifact_role, + src_rel, + module_files_dir, + dst_prefix=file_prefix, + ) + if not module_rel: + self.notes.append( + f"Skipped {path}: harvested artifact {artifact_role}/{src_rel} was not present." + ) + continue + self.add_managed_file( + path, + owner=mf.get("owner") or "root", + group=mf.get("group") or "root", + mode=mf.get("mode") or "0644", + source=_source_uri(self.module_name, module_rel), + reason=mf.get("reason") or "managed_file", + ) + + for ml in self.managed_links_from_snapshot(snap): + path = str(ml.get("path") or "").strip() + target = str(ml.get("target") or "").strip() + if not path or not target: + continue + self.add_managed_link( + path, + target=target, + reason=ml.get("reason") or "managed_link", + ) + + self.remove_directory_resource_conflicts() + + +# https://help.puppet.com/core/current/Content/PuppetCore/lang_reserved_words.htm +_RESERVED_PUPPET_NAMES = { + "and", + "application", + "attr", + "case", + "component", + "consumes", + "default", + "define", + "elsif", + "environment", + "false", + "function", + "if", + "import", + "in", + "init", + "inherits", + "node", + "or", + "private", + "produces", + "regexp", + "site", + "true", + "type", + "undef", + "unit", + "unless", +} + + +def _puppet_name(raw: str, *, fallback: str = "role") -> str: + s = re.sub(r"[^A-Za-z0-9_]+", "_", raw or fallback) + s = re.sub(r"_+", "_", s).strip("_").lower() + if not s: + s = fallback + if not re.match(r"^[a-z]", s): + s = f"{fallback}_{s}" + if s in _RESERVED_PUPPET_NAMES: + s = f"{fallback}_{s}" + return s + + +def _pp_quote(value: Any) -> str: + s = str(value) + s = s.replace("\\", "\\\\").replace("'", "\\'") + return f"'{s}'" + + +def _pp_bool(value: bool) -> str: + return "true" if bool(value) else "false" + + +def _shell_quote(value: Any) -> str: + return shlex.quote(str(value or "")) + + +def _split_digest_ref(value: Any) -> Tuple[str, Optional[str]]: + text = str(value or "").strip() + if "@" not in text: + return text, None + image, digest = text.split("@", 1) + return image, digest + + +def _container_pull_cmd(engine: str, pull_ref: str) -> str: + return f"{engine} pull {_shell_quote(pull_ref)}" + + +def _container_exists_cmd(engine: str, ref: str) -> str: + if engine == "podman": + return f"podman image exists {_shell_quote(ref)}" + return f"docker image inspect {_shell_quote(ref)} >/dev/null 2>&1" + + +def _container_tag_cmd(engine: str, pull_ref: str, tag_ref: str) -> str: + return f"{engine} tag {_shell_quote(pull_ref)} {_shell_quote(tag_ref)}" + + +def _pp_array(values: Iterable[Any]) -> str: + return "[" + ", ".join(_pp_quote(v) for v in values) + "]" + + +def _resource( + lines: List[str], rtype: str, title: str, attrs: List[Tuple[str, str]] +) -> None: + lines.append(f" {rtype} {{ {_pp_quote(title)}:") + for key, value in attrs: + lines.append(f" {key} => {value},") + lines.append(" }") + lines.append("") + + +def _state_title(prefix: str, value: Any) -> str: + safe = re.sub(r"[^A-Za-z0-9_.-]+", "-", str(value or "item")).strip("-._") + if not safe: + safe = "item" + if len(safe) > 64: + digest = hashlib.sha1( + str(value).encode("utf-8", errors="replace") + ).hexdigest()[ # nosec B324 + :8 + ] + safe = safe[:48] + "-" + digest + return f"enroll-{prefix}-{safe}" + + +def _copy_artifact( + bundle_dir: str, + role: str, + src_rel: str, + dst_files_dir: Path, + *, + dst_prefix: Optional[str] = None, +) -> Optional[str]: + if not role or not src_rel: + return None + src = Path(bundle_dir) / "artifacts" / role / src_rel + if not src.is_file(): + return None + module_rel = Path(dst_prefix or "") / src_rel + dst = dst_files_dir / module_rel + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + return module_rel.as_posix() + + +def _source_uri(module_name: str, module_rel: str) -> str: + return f"puppet:///modules/{module_name}/{module_rel}" + + +def _add_flatpak_snap_notes(roles: Dict[str, Any], out: Dict[str, PuppetRole]) -> None: + flatpak = roles.get("flatpak") or {} + if isinstance(flatpak, dict) and ( + flatpak.get("system_flatpaks") or flatpak.get("remotes") + ): + prole = out.setdefault("flatpak", PuppetRole("flatpak")) + prole.notes.append( + "Flatpak resources were detected but are not yet rendered as native Puppet resources." + ) + snap = roles.get("snap") or {} + if isinstance(snap, dict) and snap.get("system_snaps"): + prole = out.setdefault("snap", PuppetRole("snap")) + prole.notes.append( + "Snap resources were detected but are not yet rendered as native Puppet resources." + ) + + +def _node_data_filename(fqdn: str) -> str: + """Return a safe Hiera node-data filename for an FQDN/certname.""" + + name = str(fqdn or "").strip().replace("/", "_").replace("\\", "_") + return f"{name or 'node'}.yaml" + + +def _node_file_prefix(fqdn: str) -> str: + """Return a safe module-files prefix for node-specific artifacts.""" + + name = re.sub(r"[^A-Za-z0-9_.-]+", "_", str(fqdn or "").strip()) + name = name.strip("._-") or "node" + return f"nodes/{name}" + + +def _collect_puppet_roles( + state: Dict[str, Any], + bundle_dir: str, + modules_dir: Path, + *, + fqdn: Optional[str] = None, + no_common_roles: bool = False, +) -> List[PuppetRole]: + roles = roles_from_state(state) + inventory_packages = inventory_packages_from_state(state) + use_common_modules = not fqdn and not no_common_roles + node_file_prefix = _node_file_prefix(fqdn) if fqdn else None + out: Dict[str, PuppetRole] = {} + + def ensure_role(role_name: str) -> PuppetRole: + role_name = _puppet_name(role_name, fallback="enroll_role") + return out.setdefault(role_name, PuppetRole(role_name)) + + for key in ( + "apt_config", + "dnf_config", + "etc_custom", + "usr_local_custom", + "extra_paths", + "sysctl", + ): + snap = roles.get(key) or {} + if not isinstance(snap, dict): + continue + role_name = _puppet_name( + str(snap.get("role_name") or key), fallback="enroll_role" + ) + prole = ensure_role(role_name) + module_files_dir = modules_dir / prole.module_name / "files" + prole.add_managed_content( + snap, + bundle_dir=bundle_dir, + artifact_role=str(snap.get("role_name") or key), + module_files_dir=module_files_dir, + file_prefix=node_file_prefix, + ) + + users_snap = roles.get("users") or {} + if isinstance(users_snap, dict): + role_name = _puppet_name( + str(users_snap.get("role_name") or "users"), fallback="enroll_role" + ) + prole = ensure_role(role_name) + prole.add_users_snapshot(users_snap) + prole.add_managed_content( + users_snap, + bundle_dir=bundle_dir, + artifact_role=str(users_snap.get("role_name") or "users"), + module_files_dir=modules_dir / prole.module_name / "files", + file_prefix=node_file_prefix, + ) + + for svc in roles.get("services", []) or []: + if not isinstance(svc, dict): + continue + original_role_name = _puppet_name( + str(svc.get("role_name") or svc.get("unit") or "service"), + fallback="service", + ) + if use_common_modules: + role_name = _puppet_name( + section_label_for_packages( + [ + str(p).strip() + for p in (svc.get("packages") or []) + if str(p).strip() + ], + inventory_packages, + ), + fallback="package_group", + ) + else: + role_name = original_role_name + prole = ensure_role(role_name) + prole.add_service_snapshot(svc) + prole.add_managed_content( + svc, + bundle_dir=bundle_dir, + artifact_role=str(svc.get("role_name") or original_role_name), + module_files_dir=modules_dir / prole.module_name / "files", + file_prefix=node_file_prefix, + ) + + for pkg in roles.get("packages", []) or []: + if not isinstance(pkg, dict): + continue + original_role_name = _puppet_name( + str(pkg.get("role_name") or pkg.get("package") or "package"), + fallback="package", + ) + if use_common_modules: + role_name = _puppet_name( + package_section_label(pkg, inventory_packages), + fallback="package_group", + ) + else: + role_name = original_role_name + prole = ensure_role(role_name) + prole.add_package_snapshot(pkg) + prole.add_managed_content( + pkg, + bundle_dir=bundle_dir, + artifact_role=str(pkg.get("role_name") or original_role_name), + module_files_dir=modules_dir / prole.module_name / "files", + file_prefix=node_file_prefix, + ) + + container_images = roles.get("container_images") or {} + if isinstance(container_images, dict) and ( + container_images.get("images") or container_images.get("notes") + ): + prole = ensure_role( + str(container_images.get("role_name") or "container_images") + ) + prole.add_container_images_snapshot(container_images) + + fw = roles.get("firewall_runtime") or {} + if isinstance(fw, dict): + has_fw = ( + fw.get("ipset_save") + or fw.get("iptables_v4_save") + or fw.get("iptables_v6_save") + ) + packages = [ + str(p).strip() for p in (fw.get("packages") or []) if str(p).strip() + ] + if has_fw or packages: + prole = ensure_role(str(fw.get("role_name") or "firewall_runtime")) + prole.packages.update(packages) + if has_fw: + prole.notes.append( + "Live firewall runtime snapshots were detected but are not yet rendered as Puppet resources." + ) + + _add_flatpak_snap_notes(roles, out) + + puppet_roles = sorted(out.values(), key=lambda r: role_order_key(r.role_name)) + resolve_catalog_conflicts(puppet_roles) + return [r for r in puppet_roles if r.has_resources()] + + +def _render_role_class(prole: PuppetRole) -> str: + has_sysctl_conf = "/etc/sysctl.d/99-enroll.conf" in prole.files + if has_sysctl_conf: + lines: List[str] = [ + "# Generated by Enroll from harvest state.", + f"class {prole.module_name} (", + " Boolean $sysctl_apply = true,", + " Boolean $sysctl_ignore_apply_errors = true,", + ") {", + "", + ] + else: + lines = [ + "# Generated by Enroll from harvest state.", + f"class {prole.module_name} {{", + "", + ] + + for package in sorted(prole.packages): + _resource(lines, "package", package, [("ensure", _pp_quote("installed"))]) + + for group in sorted(prole.groups): + _resource(lines, "group", group, [("ensure", _pp_quote("present"))]) + + for user in [prole.users[k] for k in sorted(prole.users)]: + attrs: List[Tuple[str, str]] = [ + ("ensure", _pp_quote("present")), + ("managehome", _pp_bool(True)), + ] + if user.get("uid") is not None: + attrs.append(("uid", _pp_quote(user["uid"]))) + if user.get("primary_group"): + attrs.append(("gid", _pp_quote(user["primary_group"]))) + if user.get("home"): + attrs.append(("home", _pp_quote(user["home"]))) + if user.get("shell"): + attrs.append(("shell", _pp_quote(user["shell"]))) + if user.get("gecos"): + attrs.append(("comment", _pp_quote(user["gecos"]))) + if user.get("supplementary_groups"): + attrs.append(("groups", _pp_array(user["supplementary_groups"]))) + attrs.append(("membership", _pp_quote("minimum"))) + _resource(lines, "user", user["name"], attrs) + + for path, d in sorted(prole.dirs.items()): + _resource( + lines, + "file", + path, + [ + ("ensure", _pp_quote("directory")), + ("owner", _pp_quote(d.get("owner") or "root")), + ("group", _pp_quote(d.get("group") or "root")), + ("mode", _pp_quote(d.get("mode") or "0755")), + ], + ) + + for path, f in sorted(prole.files.items()): + _resource( + lines, + "file", + path, + [ + ("ensure", _pp_quote("file")), + ("source", _pp_quote(f.get("source") or "")), + ("owner", _pp_quote(f.get("owner") or "root")), + ("group", _pp_quote(f.get("group") or "root")), + ("mode", _pp_quote(f.get("mode") or "0644")), + ], + ) + + for path, lnk in sorted(prole.links.items()): + _resource( + lines, + "file", + path, + [ + ("ensure", _pp_quote("link")), + ("target", _pp_quote(lnk.get("target") or "")), + ], + ) + + for svc in [prole.services[k] for k in sorted(prole.services)]: + _resource( + lines, + "service", + svc["name"], + [ + ("ensure", _pp_quote(svc["ensure"])), + ("enable", _pp_bool(bool(svc["enable"]))), + ], + ) + + for image in prole.container_images: + engine = str(image.get("engine") or "").strip() + pull_ref = str(image.get("pull_ref") or "").strip() + if not engine or not pull_ref: + continue + if engine == "docker": + attrs: List[Tuple[str, str]] = [("ensure", _pp_quote("present"))] + if image.get("image"): + attrs.append(("image", _pp_quote(image["image"]))) + if image.get("image_digest"): + attrs.append(("image_digest", _pp_quote(image["image_digest"]))) + _resource(lines, "docker::image", pull_ref, attrs) + for alias in image.get("tag_aliases") or []: + tag_ref = str(alias.get("ref") or "").strip() + if not tag_ref: + continue + _resource( + lines, + "exec", + _state_title("docker-tag", tag_ref), + [ + ( + "command", + _pp_quote( + alias.get("tag_cmd") + or _container_tag_cmd(engine, pull_ref, tag_ref) + ), + ), + ( + "unless", + _pp_quote( + alias.get("tag_unless") + or _container_exists_cmd(engine, tag_ref) + ), + ), + ("path", "['/usr/bin', '/bin']"), + ("require", f"Docker::Image[{_pp_quote(pull_ref)}]"), + ], + ) + elif engine == "podman": + _resource( + lines, + "exec", + _state_title("podman-pull", pull_ref), + [ + ( + "command", + _pp_quote( + image.get("pull_cmd") + or _container_pull_cmd(engine, pull_ref) + ), + ), + ( + "unless", + _pp_quote( + image.get("pull_unless") + or _container_exists_cmd(engine, pull_ref) + ), + ), + ("path", "['/usr/bin', '/bin']"), + ], + ) + for alias in image.get("tag_aliases") or []: + tag_ref = str(alias.get("ref") or "").strip() + if not tag_ref: + continue + _resource( + lines, + "exec", + _state_title("podman-tag", tag_ref), + [ + ( + "command", + _pp_quote( + alias.get("tag_cmd") + or _container_tag_cmd(engine, pull_ref, tag_ref) + ), + ), + ( + "unless", + _pp_quote( + alias.get("tag_unless") + or _container_exists_cmd(engine, tag_ref) + ), + ), + ("path", "['/usr/bin', '/bin']"), + ( + "require", + f"Exec[{_pp_quote(_state_title('podman-pull', pull_ref))}]", + ), + ], + ) + + if has_sysctl_conf: + lines.append(" if $sysctl_apply {") + lines.append(" exec { 'enroll-apply-sysctl':") + lines.append(" command => $sysctl_ignore_apply_errors ? {") + lines.append( + " true => \"/bin/sh -c 'sysctl -e -p /etc/sysctl.d/99-enroll.conf || true'\"," + ) + lines.append(" default => 'sysctl -e -p /etc/sysctl.d/99-enroll.conf',") + lines.append(" },") + lines.append(" path => ['/sbin', '/usr/sbin', '/bin', '/usr/bin'],") + lines.append(" refreshonly => true,") + lines.append(" subscribe => File['/etc/sysctl.d/99-enroll.conf'],") + lines.append(" }") + lines.append(" }") + lines.append("") + + if prole.notes: + lines.append(" # Notes and limitations") + for note in prole.notes: + lines.append(f" # - {note}") + lines.append("") + + lines.append("}") + lines.append("") + return "\n".join(lines) + + +def _attrs_with_ensure( + attrs: Dict[str, Any], ensure: str, *, allowed: Set[str] +) -> Dict[str, Any]: + """Return only Puppet resource attributes, dropping Enroll metadata.""" + out = {"ensure": ensure} + for key in sorted(allowed): + if key in attrs and attrs[key] is not None: + out[key] = attrs[key] + return out + + +def _role_hiera_values(prole: PuppetRole) -> Dict[str, Any]: + """Return Automatic Parameter Lookup data for one generated module.""" + + data: Dict[str, Any] = {} + prefix = f"{prole.module_name}::" + + if prole.packages: + data[f"{prefix}packages"] = sorted(prole.packages) + + if prole.groups: + data[f"{prefix}groups"] = { + group: {"ensure": "present"} for group in sorted(prole.groups) + } + + if prole.users: + users: Dict[str, Dict[str, Any]] = {} + for name in sorted(prole.users): + user = prole.users[name] + attrs: Dict[str, Any] = {"ensure": "present", "managehome": True} + if user.get("uid") is not None: + attrs["uid"] = user["uid"] + if user.get("primary_group"): + attrs["gid"] = user["primary_group"] + if user.get("home"): + attrs["home"] = user["home"] + if user.get("shell"): + attrs["shell"] = user["shell"] + if user.get("gecos"): + attrs["comment"] = user["gecos"] + if user.get("supplementary_groups"): + attrs["groups"] = list(user["supplementary_groups"]) + attrs["membership"] = "minimum" + users[name] = attrs + data[f"{prefix}users"] = users + + if prole.dirs: + data[f"{prefix}dirs"] = { + path: _attrs_with_ensure( + prole.dirs[path], + "directory", + allowed={"owner", "group", "mode"}, + ) + for path in sorted(prole.dirs) + } + + if prole.files: + data[f"{prefix}files"] = { + path: _attrs_with_ensure( + prole.files[path], + "file", + allowed={"source", "owner", "group", "mode"}, + ) + for path in sorted(prole.files) + } + + if prole.links: + data[f"{prefix}links"] = { + path: _attrs_with_ensure( + prole.links[path], + "link", + allowed={"target"}, + ) + for path in sorted(prole.links) + } + + if prole.services: + data[f"{prefix}services"] = { + name: { + "ensure": prole.services[name].get("ensure") or "stopped", + "enable": bool(prole.services[name].get("enable")), + } + for name in sorted(prole.services) + } + + if prole.container_images: + data[f"{prefix}container_images"] = list(prole.container_images) + + if prole.notes: + data[f"{prefix}notes"] = list(prole.notes) + + if "/etc/sysctl.d/99-enroll.conf" in prole.files: + data[f"{prefix}sysctl_apply"] = True + data[f"{prefix}sysctl_ignore_apply_errors"] = True + + return data + + +def _render_hiera_role_class(prole: PuppetRole) -> str: + """Render a reusable, data-driven Puppet class for --fqdn/Hiera mode.""" + + lines: List[str] = [ + "# Generated by Enroll from harvest state.", + "# Resource data is supplied by Hiera Automatic Parameter Lookup.", + f"class {prole.module_name} (", + " Array[String] $packages = [],", + " Hash[String, Hash] $groups = {},", + " Hash[String, Hash] $users = {},", + " Hash[String, Hash] $dirs = {},", + " Hash[String, Hash] $files = {},", + " Hash[String, Hash] $links = {},", + " Hash[String, Hash] $services = {},", + " Array[Hash] $container_images = [],", + " Array[String] $notes = [],", + " Boolean $sysctl_apply = true,", + " Boolean $sysctl_ignore_apply_errors = true,", + ") {", + "", + " $packages.each |String $package_name| {", + " package { $package_name:", + " ensure => 'installed',", + " }", + " }", + "", + " $groups.each |String $resource_title, Hash $attrs| {", + " group { $resource_title:", + " * => $attrs,", + " }", + " }", + "", + " $users.each |String $resource_title, Hash $attrs| {", + " user { $resource_title:", + " * => $attrs,", + " }", + " }", + "", + " $dirs.each |String $resource_title, Hash $attrs| {", + " file { $resource_title:", + " * => $attrs,", + " }", + " }", + "", + " $files.each |String $resource_title, Hash $attrs| {", + " file { $resource_title:", + " * => $attrs,", + " }", + " }", + "", + " $links.each |String $resource_title, Hash $attrs| {", + " file { $resource_title:", + " * => $attrs,", + " }", + " }", + "", + " $services.each |String $resource_title, Hash $attrs| {", + " service { $resource_title:", + " * => $attrs,", + " }", + " }", + "", + " $container_images.each |Integer $idx, Hash $image| {", + " if $image['engine'] == 'docker' and $image['pull_ref'] {", + " docker::image { $image['pull_ref']:", + " ensure => 'present',", + " image => $image['image'],", + " image_digest => $image['image_digest'],", + " }", + " $image['tag_aliases'].each |Integer $tag_idx, Hash $alias| {", + ' exec { "enroll-docker-tag-${idx}-${tag_idx}":', + " command => $alias['tag_cmd'],", + " unless => $alias['tag_unless'],", + " path => ['/usr/bin', '/bin'],", + " require => Docker::Image[$image['pull_ref']],", + " }", + " }", + " } elsif $image['engine'] == 'podman' and $image['pull_ref'] {", + ' exec { "enroll-podman-pull-${idx}":', + " command => $image['pull_cmd'],", + " unless => $image['pull_unless'],", + " path => ['/usr/bin', '/bin'],", + " }", + " $image['tag_aliases'].each |Integer $tag_idx, Hash $alias| {", + ' exec { "enroll-podman-tag-${idx}-${tag_idx}":', + " command => $alias['tag_cmd'],", + " unless => $alias['tag_unless'],", + " path => ['/usr/bin', '/bin'],", + ' require => Exec["enroll-podman-pull-${idx}"],', + " }", + " }", + " }", + " }", + "", + " if $sysctl_apply and '/etc/sysctl.d/99-enroll.conf' in $files {", + " exec { 'enroll-apply-sysctl':", + " command => $sysctl_ignore_apply_errors ? {", + " true => \"/bin/sh -c 'sysctl -e -p /etc/sysctl.d/99-enroll.conf || true'\",", + " default => 'sysctl -e -p /etc/sysctl.d/99-enroll.conf',", + " },", + " path => ['/sbin', '/usr/sbin', '/bin', '/usr/bin'],", + " refreshonly => true,", + " subscribe => File['/etc/sysctl.d/99-enroll.conf'],", + " }", + " }", + "", + " # Generated notes are supplied through the $notes parameter for review.", + "}", + "", + ] + return "\n".join(lines) + + +def _render_site_pp(puppet_roles: List[PuppetRole], fqdn: Optional[str]) -> str: + node_name = _pp_quote(fqdn) if fqdn else "default" + if not puppet_roles: + return f"node {node_name} {{\n # No Puppet classes were generated from this harvest.\n}}\n" + includes = "\n".join(f" include {r.module_name}" for r in puppet_roles) + return f"node {node_name} {{\n{includes}\n}}\n" + + +def _render_hiera_site_pp(node_names: List[str]) -> str: + lines: List[str] = [ + "# Generated by Enroll from harvest state.", + "# Per-node class lists and resources are read from Hiera data.", + "", + ] + for node_name in node_names: + lines.extend( + [ + f"node {_pp_quote(node_name)} {{", + " $enroll_classes = lookup('enroll::classes', Array[String], 'unique', [])", + " $enroll_classes.each |String $enroll_class| {", + " include $enroll_class", + " }", + "}", + "", + ] + ) + lines.extend( + [ + "node default {", + " $enroll_classes = lookup('enroll::classes', Array[String], 'unique', [])", + " $enroll_classes.each |String $enroll_class| {", + " include $enroll_class", + " }", + "}", + "", + ] + ) + return "\n".join(lines) + + +def _render_hiera_yaml() -> str: + data = { + "version": 5, + "defaults": {"datadir": "data", "data_hash": "yaml_data"}, + "hierarchy": [ + { + "name": "Enroll trusted certname node data", + "path": "nodes/%{trusted.certname}.yaml", + }, + { + "name": "Enroll networking FQDN node data", + "path": "nodes/%{facts.networking.fqdn}.yaml", + }, + {"name": "Enroll common data", "path": "common.yaml"}, + ], + } + return yaml.safe_dump(data, sort_keys=False, explicit_start=True) + + +def _write_yaml(path: Path, data: Dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + yaml.safe_dump(data, sort_keys=True, explicit_start=True), + encoding="utf-8", + ) + + +def _write_hiera_node_data( + out: Path, fqdn: str, puppet_roles: List[PuppetRole] +) -> Path: + node_data: Dict[str, Any] = { + "enroll::classes": [r.module_name for r in puppet_roles] + } + for prole in puppet_roles: + node_data.update(_role_hiera_values(prole)) + node_path = out / "data" / "nodes" / _node_data_filename(fqdn) + _write_yaml(node_path, node_data) + common_path = out / "data" / "common.yaml" + if not common_path.exists(): + _write_yaml(common_path, {"enroll::classes": []}) + return node_path + + +def _hiera_node_names(out: Path) -> List[str]: + nodes_dir = out / "data" / "nodes" + if not nodes_dir.is_dir(): + return [] + out_names: Set[str] = set() + for path in nodes_dir.glob("*.yaml"): + out_names.add(path.name[: -len(".yaml")]) + return sorted(out_names) + + +def _write_metadata(module_dir: Path, module_name: str, prole: PuppetRole) -> None: + dependencies: List[Dict[str, str]] = [] + if any(img.get("engine") == "docker" for img in prole.container_images): + dependencies.append( + { + "name": "puppetlabs-docker", + "version_requirement": ">= 8.0.0 < 15.0.0", + } + ) + + (module_dir / "metadata.json").write_text( + json.dumps( + { + "name": f"enroll-{module_name}", + "version": "0.1.0", + "author": "Enroll", + "summary": f"Generated Enroll Puppet module for {module_name}", + "license": "UNLICENSED", + "source": "", + "dependencies": dependencies, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + + +def _render_readme( + state: Dict[str, Any], + puppet_roles: List[PuppetRole], + *, + fqdn: Optional[str] = None, + node_names: Optional[List[str]] = None, +) -> str: + host = state.get("host", {}) if isinstance(state.get("host"), dict) else {} + hostname = host.get("hostname") or "unknown" + hiera_mode = bool(fqdn) + role_lines = ( + "\n".join( + f"- `{r.module_name}` from Enroll role `{r.role_name}`" + for r in puppet_roles + ) + or "- None." + ) + node_lines = "\n".join(f"- `{n}`" for n in (node_names or [])) or "- None." + notes: List[str] = [] + for r in puppet_roles: + for note in r.notes: + notes.append(f"`{r.module_name}`: {note}") + notes_text = "\n".join(f"- {n}" for n in notes) or "- None." + if hiera_mode: + layout = f"""- `manifests/site.pp` declares node blocks and includes classes listed in Hiera key `enroll::classes`. +- `hiera.yaml` configures per-node lookup from `data/nodes/%{{trusted.certname}}.yaml` with a fallback to `data/common.yaml`. +- `data/nodes/{_node_data_filename(fqdn or '')}` contains this node's class list and class parameter data. +- `modules//manifests/init.pp` contains reusable, data-driven classes. +- `modules//files/nodes//...` contains node-specific harvested file artifacts, avoiding clashes between hosts.""" + apply = f"""Run from this generated output directory, passing the node certname so Hiera selects the right node data: + +```bash +sudo puppet apply --modulepath ./modules --hiera_config ./hiera.yaml --certname {fqdn} manifests/site.pp --noop +``` + +For Puppet agent/control-repo use, place this output where `hiera.yaml`, `data/`, `manifests/`, and `modules/` form the environment root. Re-running Enroll with another `--fqdn` into the same output directory adds or replaces that node's YAML without deleting existing node data.""" + else: + layout = """- `manifests/site.pp` declares a `node` block and includes the generated classes in manifest order. +- `modules//manifests/init.pp` contains resources for each generated Enroll role/snapshot or common package group. +- `modules//files/` contains harvested file artifacts for that role or group. +- Generated module names avoid Puppet reserved words such as `default`.""" + apply = """Run from this generated output directory so Puppet can find `./modules`, or pass an absolute module path: + +```bash +sudo puppet apply --modulepath ./modules manifests/site.pp --noop +``` + +```bash +sudo puppet apply --modulepath /path/to/generated/modules /path/to/generated/manifests/site.pp --noop +```""" + return f"""# Enroll Puppet manifest + +Generated by Enroll from harvest data for `{hostname}`. + +This Puppet target reuses the existing harvest state without changing harvesting behaviour. + +## Layout + +{layout} + +## Known nodes + +{node_lines if hiera_mode else '- Non-Hiera single-node output.'} + +## Generated modules + +{role_lines} + +## Apply / check + +{apply} + +## Generated resources + +- Native packages observed in package and service snapshots. +- Local users and groups from the users snapshot. +- Managed directories, files, and symlinks from harvested roles. +- Basic service enablement/running-state resources. +- `/etc/sysctl.d/99-enroll.conf` plus a refresh-only sysctl apply exec when present. +- Docker images by digest using the `puppetlabs-docker` module's `docker::image` defined type. +- Podman images by digest using guarded `podman pull` / `podman tag` exec resources. + +## Current limitations + +- Flatpak, Snap, and live firewall runtime snapshots are listed as notes when present rather than rendered as Puppet resources. +- Docker image resources require the `puppetlabs-docker` module to be installed in the Puppet environment. +- JinjaTurtle templating is currently Ansible-oriented and is not applied to Puppet output. +- Review generated resources before applying them broadly across unlike hosts. + +## Notes + +{notes_text} +""" + + +class PuppetManifestRenderer: + """Render Puppet modules and site manifest from a harvest bundle.""" + + def __init__( + self, + bundle_dir: str, + out_dir: str, + *, + fqdn: Optional[str] = None, + no_common_roles: bool = False, + ) -> None: + self.bundle_dir = bundle_dir + self.out_dir = out_dir + self.fqdn = fqdn + self.no_common_roles = no_common_roles + + def render(self) -> None: + """Render Puppet modules/site.pp from a harvest bundle.""" + + bundle_dir = self.bundle_dir + out_dir = self.out_dir + fqdn = self.fqdn + no_common_roles = self.no_common_roles + + state = PuppetRole.load_state(bundle_dir) + out = Path(out_dir) + hiera_mode = bool(fqdn) + if out.exists() and not hiera_mode: + shutil.rmtree(out) + manifests_dir = out / "manifests" + modules_dir = out / "modules" + manifests_dir.mkdir(parents=True, exist_ok=True) + modules_dir.mkdir(parents=True, exist_ok=True) + + puppet_roles = _collect_puppet_roles( + state, + bundle_dir, + modules_dir, + fqdn=fqdn, + no_common_roles=no_common_roles, + ) + for prole in puppet_roles: + module_dir = modules_dir / prole.module_name + module_manifests = module_dir / "manifests" + module_files = module_dir / "files" + module_manifests.mkdir(parents=True, exist_ok=True) + module_files.mkdir(parents=True, exist_ok=True) + (module_manifests / "init.pp").write_text( + ( + _render_hiera_role_class(prole) + if hiera_mode + else _render_role_class(prole) + ), + encoding="utf-8", + ) + _write_metadata(module_dir, prole.module_name, prole) + + node_names: List[str] = [] + if hiera_mode and fqdn: + (out / "hiera.yaml").write_text(_render_hiera_yaml(), encoding="utf-8") + _write_hiera_node_data(out, fqdn, puppet_roles) + node_names = _hiera_node_names(out) + (manifests_dir / "site.pp").write_text( + _render_hiera_site_pp(node_names), encoding="utf-8" + ) + else: + (manifests_dir / "site.pp").write_text( + _render_site_pp(puppet_roles, fqdn), encoding="utf-8" + ) + (out / "README.md").write_text( + _render_readme( + state, + puppet_roles, + fqdn=fqdn, + node_names=node_names, + ), + encoding="utf-8", + ) + + +def manifest_from_bundle_dir( + bundle_dir: str, + out_dir: str, + *, + fqdn: Optional[str] = None, + no_common_roles: bool = False, +) -> None: + PuppetManifestRenderer( + bundle_dir, + out_dir, + fqdn=fqdn, + no_common_roles=no_common_roles, + ).render() diff --git a/enroll/remote.py b/enroll/remote.py new file mode 100644 index 0000000..3e7b42d --- /dev/null +++ b/enroll/remote.py @@ -0,0 +1,680 @@ +from __future__ import annotations + +import getpass +import os +import shlex +import shutil +import sys +import time +import tarfile +import tempfile +import zipapp +from pathlib import Path +from pathlib import PurePosixPath +from typing import Optional, Callable, TextIO + + +class RemoteSudoPasswordRequired(RuntimeError): + """Raised when sudo requires a password but none was provided.""" + + +class RemoteSSHKeyPassphraseRequired(RuntimeError): + """Raised when SSH private key decryption needs a passphrase.""" + + +def _sudo_password_required(out: str, err: str) -> bool: + """Return True if sudo output indicates it needs a password/TTY.""" + blob = (out + "\n" + err).lower() + patterns = ( + "a password is required", + "password is required", + "a terminal is required to read the password", + "no tty present and no askpass program specified", + "must have a tty to run sudo", + "sudo: sorry, you must have a tty", + "askpass", + ) + return any(p in blob for p in patterns) + + +def _sudo_not_permitted(out: str, err: str) -> bool: + """Return True if sudo output indicates the user cannot sudo at all.""" + blob = (out + "\n" + err).lower() + patterns = ( + "is not in the sudoers file", + "not allowed to execute", + "may not run sudo", + "sorry, user", + ) + return any(p in blob for p in patterns) + + +def _sudo_tty_required(out: str, err: str) -> bool: + """Return True if sudo output indicates it requires a TTY (sudoers requiretty).""" + blob = (out + "\n" + err).lower() + patterns = ( + "must have a tty", + "sorry, you must have a tty", + "sudo: sorry, you must have a tty", + "must have a tty to run sudo", + ) + return any(p in blob for p in patterns) + + +def _resolve_become_password( + ask_become_pass: bool, + *, + prompt: str = "sudo password: ", + getpass_fn: Callable[[str], str] = getpass.getpass, +) -> Optional[str]: + if ask_become_pass: + return getpass_fn(prompt) + return None + + +def _resolve_ssh_key_passphrase( + ask_key_passphrase: bool, + *, + env_var: Optional[str] = None, + prompt: str = "SSH key passphrase: ", + getpass_fn: Callable[[str], str] = getpass.getpass, +) -> Optional[str]: + """Resolve SSH private-key passphrase from env and/or prompt. + + Precedence: + 1) --ssh-key-passphrase-env style input (env_var) + 2) --ask-key-passphrase style interactive prompt + 3) None + """ + if env_var: + val = os.environ.get(str(env_var)) + if val is None: + raise RuntimeError( + "SSH key passphrase environment variable is not set: " f"{env_var}" + ) + return val + + if ask_key_passphrase: + return getpass_fn(prompt) + + return None + + +def remote_harvest( + *, + ask_become_pass: bool = False, + ask_key_passphrase: bool = False, + ssh_key_passphrase_env: Optional[str] = None, + no_sudo: bool = False, + prompt: str = "sudo password: ", + key_prompt: str = "SSH key passphrase: ", + getpass_fn: Optional[Callable[[str], str]] = None, + stdin: Optional[TextIO] = None, + **kwargs, +): + """Call _remote_harvest, with a safe sudo password fallback. + + Behavior: + - Run without a password unless --ask-become-pass is set. + - If the remote sudo policy requires a password and none was provided, + prompt and retry when running interactively. + """ + + # Resolve defaults at call time (easier to test/monkeypatch, and avoids capturing + # sys.stdin / getpass.getpass at import time). + if getpass_fn is None: + getpass_fn = getpass.getpass + if stdin is None: + stdin = sys.stdin + + sudo_password = _resolve_become_password( + ask_become_pass and not no_sudo, + prompt=prompt, + getpass_fn=getpass_fn, + ) + ssh_key_passphrase = _resolve_ssh_key_passphrase( + ask_key_passphrase, + env_var=ssh_key_passphrase_env, + prompt=key_prompt, + getpass_fn=getpass_fn, + ) + + while True: + try: + return _remote_harvest( + sudo_password=sudo_password, + no_sudo=no_sudo, + ssh_key_passphrase=ssh_key_passphrase, + **kwargs, + ) + except RemoteSSHKeyPassphraseRequired: + # Already tried a passphrase and still failed. + if ssh_key_passphrase is not None: + raise RemoteSSHKeyPassphraseRequired( + "SSH private key could not be decrypted with the supplied " + "passphrase." + ) from None + + # Fallback prompt if interactive. + if stdin is not None and getattr(stdin, "isatty", lambda: False)(): + ssh_key_passphrase = getpass_fn(key_prompt) + continue + + raise RemoteSSHKeyPassphraseRequired( + "SSH private key is encrypted and needs a passphrase. " + "Re-run with --ask-key-passphrase or " + "--ssh-key-passphrase-env VAR." + ) + + except RemoteSudoPasswordRequired: + if sudo_password is not None: + raise + + # Fallback prompt if interactive. + if stdin is not None and getattr(stdin, "isatty", lambda: False)(): + sudo_password = getpass_fn(prompt) + continue + + raise RemoteSudoPasswordRequired( + "Remote sudo requires a password. Re-run with --ask-become-pass." + ) + + +def _safe_extract_tar(tar: tarfile.TarFile, dest: Path) -> None: + """Safely extract a tar archive into dest. + + Protects against path traversal (e.g. entries containing ../). + """ + # Note: tar member names use POSIX separators regardless of platform. + dest = dest.resolve() + + for m in tar.getmembers(): + name = m.name + + # Some tar implementations include a top-level '.' entry when created + # with `tar -C .`. That's harmless and should be allowed. + if name in {".", "./"}: + continue + + # Reject absolute paths and any '..' components up front. + p = PurePosixPath(name) + if p.is_absolute() or ".." in p.parts: + raise RuntimeError(f"Unsafe tar member path: {name}") + + # Refuse to extract links or device nodes from an untrusted archive. + # (A symlink can be used to redirect subsequent writes outside dest.) + if m.issym() or m.islnk() or m.isdev(): + raise RuntimeError(f"Refusing to extract special tar member: {name}") + + member_path = (dest / Path(*p.parts)).resolve() + if member_path != dest and not str(member_path).startswith(str(dest) + os.sep): + raise RuntimeError(f"Unsafe tar member path: {name}") + + # Extract members one-by-one after validation. Pass an explicit tarfile + # extraction filter on Python versions that support it so Python 3.12/3.13 + # do not warn about the Python 3.14 default changing. Keep the older call + # path for Python 3.10/3.11, where the filter argument is unavailable. + supports_filter = hasattr(tarfile, "data_filter") + for m in tar.getmembers(): + if m.name in {".", "./"}: + continue + if supports_filter: + tar.extract(m, path=dest, filter="data") + else: + tar.extract(m, path=dest) + + +def _build_enroll_pyz(tmpdir: Path) -> Path: + """Build a self-contained enroll zipapp (pyz) on the local machine. + + The resulting file is stdlib-only and can be executed on the remote host + as long as it has Python 3 available. + """ + import enroll as pkg + + pkg_dir = Path(pkg.__file__).resolve().parent + stage = tmpdir / "stage" + (stage / "enroll").mkdir(parents=True, exist_ok=True) + + def _ignore(d: str, names: list[str]) -> set[str]: + return { + n + for n in names + if n in {"__pycache__", ".pytest_cache"} or n.endswith(".pyc") + } + + shutil.copytree(pkg_dir, stage / "enroll", dirs_exist_ok=True, ignore=_ignore) + + pyz_path = tmpdir / "enroll.pyz" + zipapp.create_archive( + stage, + target=pyz_path, + main="enroll.cli:main", + compressed=True, + ) + return pyz_path + + +def _ssh_run( + ssh, + cmd: str, + *, + get_pty: bool = False, + stdin_text: Optional[str] = None, + close_stdin: bool = False, +) -> tuple[int, str, str]: + """Run a command over a Paramiko SSHClient. + + Paramiko's exec_command runs commands without a TTY by default. + Some hosts have sudoers "requiretty" enabled, which causes sudo to + fail even when passwordless sudo is configured. For those commands, + request a PTY. + + We do not request a PTY for commands that stream binary data + (e.g. tar/gzip output), as a PTY can corrupt the byte stream. + """ + stdin, stdout, stderr = ssh.exec_command(cmd, get_pty=get_pty) + # All three file-like objects share the same underlying Channel. + chan = stdout.channel + + if stdin_text is not None and stdin is not None: + try: + stdin.write(stdin_text) + stdin.flush() + except Exception: + # If the remote side closed stdin early, ignore. + pass # nosec + finally: + if close_stdin: + # For sudo -S, a wrong password causes sudo to re-prompt and wait + # forever for more input. We try hard to deliver EOF so sudo can + # fail fast. + try: + chan.shutdown_write() # sends EOF to the remote process + except Exception: + pass # nosec + try: + stdin.close() + except Exception: + pass # nosec + + # Read incrementally to avoid blocking forever on stdout.read()/stderr.read() + # if the remote process is waiting for more input (e.g. sudo password retry). + out_chunks: list[bytes] = [] + err_chunks: list[bytes] = [] + # Keep a small tail of stderr to detect sudo retry messages without + # repeatedly joining potentially large buffers. + err_tail = b"" + + while True: + progressed = False + if chan.recv_ready(): + out_chunks.append(chan.recv(1024 * 64)) + progressed = True + if chan.recv_stderr_ready(): + chunk = chan.recv_stderr(1024 * 64) + err_chunks.append(chunk) + err_tail = (err_tail + chunk)[-4096:] + progressed = True + + # If we just attempted sudo -S with a single password line and sudo is + # asking again, detect it and stop waiting. + if close_stdin and stdin_text is not None: + blob = err_tail.lower() + if b"sorry, try again" in blob or b"incorrect password" in blob: + try: + chan.close() + except Exception: + pass # nosec + break + + # Exit once the process has exited and we have drained the buffers. + if ( + chan.exit_status_ready() + and not chan.recv_ready() + and not chan.recv_stderr_ready() + ): + break + + if not progressed: + time.sleep(0.05) + + out = b"".join(out_chunks).decode("utf-8", errors="replace") + err = b"".join(err_chunks).decode("utf-8", errors="replace") + rc = chan.recv_exit_status() if chan.exit_status_ready() else 1 + return rc, out, err + + +def _ssh_run_sudo( + ssh, + cmd: str, + *, + sudo_password: Optional[str] = None, + get_pty: bool = True, +) -> tuple[int, str, str]: + """Run cmd via sudo with a safe non-interactive-first strategy. + + Strategy: + 1) Try `sudo -n`. + 2) If sudo reports a password is required and we have one, retry with + `sudo -S` and feed it via stdin. + 3) If sudo reports a password is required and we *don't* have one, raise + RemoteSudoPasswordRequired. + + We avoid requesting a PTY unless the remote sudo policy requires it. + This makes sudo -S behavior more reliable (wrong passwords fail fast + instead of blocking on a PTY). + """ + cmd_n = f"sudo -n -p '' -- {cmd}" + + # First try: never prompt, and prefer no PTY. + rc, out, err = _ssh_run(ssh, cmd_n, get_pty=False) + need_pty = False + + # Some sudoers configurations require a TTY even for passwordless sudo. + if get_pty and rc != 0 and _sudo_tty_required(out, err): + need_pty = True + rc, out, err = _ssh_run(ssh, cmd_n, get_pty=True) + + if rc == 0: + return rc, out, err + + if _sudo_not_permitted(out, err): + return rc, out, err + + if _sudo_password_required(out, err): + if sudo_password is None: + raise RemoteSudoPasswordRequired( + "Remote sudo requires a password, but none was provided." + ) + cmd_s = f"sudo -S -p '' -- {cmd}" + return _ssh_run( + ssh, + cmd_s, + get_pty=need_pty, + stdin_text=str(sudo_password) + "\n", + close_stdin=True, + ) + + return rc, out, err + + +def _remote_harvest( + *, + local_out_dir: Path, + remote_host: str, + remote_port: Optional[int] = None, + remote_user: Optional[str] = None, + remote_ssh_config: Optional[str] = None, + remote_python: str = "python3", + dangerous: bool = False, + no_sudo: bool = False, + sudo_password: Optional[str] = None, + ssh_key_passphrase: Optional[str] = None, + include_paths: Optional[list[str]] = None, + exclude_paths: Optional[list[str]] = None, +) -> Path: + """Run enroll harvest on a remote host via SSH and pull the bundle locally. + + Returns the local path to state.json inside local_out_dir. + """ + try: + import paramiko # type: ignore + except Exception as e: + raise RuntimeError( + "Remote harvesting requires the 'paramiko' package. " + "Install it with: pip install paramiko" + ) from e + + local_out_dir = Path(local_out_dir) + local_out_dir.mkdir(parents=True, exist_ok=True) + try: + os.chmod(local_out_dir, 0o700) + except OSError: + pass + + # Build a zipapp locally and upload it to the remote. + with tempfile.TemporaryDirectory(prefix="enroll-remote-") as td: + td_path = Path(td) + pyz = _build_enroll_pyz(td_path) + local_tgz = td_path / "bundle.tgz" + + ssh = paramiko.SSHClient() + ssh.load_system_host_keys() + # Default: refuse unknown host keys. + # Users should add the key to known_hosts. + ssh.set_missing_host_key_policy(paramiko.RejectPolicy()) + + # Resolve SSH connection parameters. + connect_host = remote_host + connect_port = int(remote_port) if remote_port is not None else 22 + connect_user = remote_user + key_filename = None + sock = None + hostkey_name = connect_host + + # Timeouts derived from ssh_config if set (ConnectTimeout). + # Used both for socket connect (when we create one) and Paramiko handshake/auth. + connect_timeout: Optional[float] = None + + if remote_ssh_config: + from paramiko.config import SSHConfig # type: ignore + from paramiko.proxy import ProxyCommand # type: ignore + import socket as _socket + + cfg_path = Path(str(remote_ssh_config)).expanduser() + if not cfg_path.exists(): + raise RuntimeError(f"SSH config file not found: {cfg_path}") + + cfg = SSHConfig() + with cfg_path.open("r", encoding="utf-8") as _fp: + cfg.parse(_fp) + hcfg = cfg.lookup(remote_host) + + connect_host = str(hcfg.get("hostname") or remote_host) + hostkey_name = str(hcfg.get("hostkeyalias") or connect_host) + + if remote_port is None and hcfg.get("port"): + try: + connect_port = int(str(hcfg.get("port"))) + except ValueError: + pass + if connect_user is None and hcfg.get("user"): + connect_user = str(hcfg.get("user")) + + ident = hcfg.get("identityfile") + if ident: + if isinstance(ident, (list, tuple)): + key_filename = [str(Path(p).expanduser()) for p in ident] + else: + key_filename = str(Path(str(ident)).expanduser()) + + # Honour OpenSSH ConnectTimeout (seconds) if present. + if hcfg.get("connecttimeout"): + try: + connect_timeout = float(str(hcfg.get("connecttimeout"))) + except (TypeError, ValueError): + connect_timeout = None + + proxycmd = hcfg.get("proxycommand") + + # AddressFamily support: inet (IPv4 only), inet6 (IPv6 only), any (default). + addrfam = str(hcfg.get("addressfamily") or "any").strip().lower() + family: Optional[int] = None + if addrfam == "inet": + family = _socket.AF_INET + elif addrfam == "inet6": + family = _socket.AF_INET6 + + if proxycmd: + # ProxyCommand provides the transport; AddressFamily doesn't apply here. + sock = ProxyCommand(str(proxycmd)) + elif family is not None: + # Enforce the requested address family by pre-connecting the socket and + # passing it into Paramiko via sock=. + last_err: Optional[OSError] = None + infos = _socket.getaddrinfo( + connect_host, connect_port, family, _socket.SOCK_STREAM + ) + for af, socktype, proto, _, sa in infos: + s = _socket.socket(af, socktype, proto) + if connect_timeout is not None: + s.settimeout(connect_timeout) + try: + s.connect(sa) + sock = s + break + except OSError as e: + last_err = e + try: + s.close() + except Exception: + pass # nosec + if sock is None and last_err is not None: + raise last_err + elif hostkey_name != connect_host: + # If HostKeyAlias is used, connect to HostName via a socket but + # use HostKeyAlias for known_hosts lookups. + sock = _socket.create_connection( + (connect_host, connect_port), timeout=connect_timeout + ) + + # If we created a socket (sock!=None), pass hostkey_name as hostname so + # known_hosts lookup uses HostKeyAlias (or whatever hostkey_name resolved to). + try: + ssh.connect( + hostname=hostkey_name if sock is not None else connect_host, + port=connect_port, + username=connect_user, + key_filename=key_filename, + sock=sock, + allow_agent=True, + look_for_keys=True, + timeout=connect_timeout, + banner_timeout=connect_timeout, + auth_timeout=connect_timeout, + passphrase=ssh_key_passphrase, + ) + except paramiko.PasswordRequiredException as e: # type: ignore[attr-defined] + raise RemoteSSHKeyPassphraseRequired( + "SSH private key is encrypted and no passphrase was provided." + ) from e + + # If no username was explicitly provided, SSH may have selected a default. + # We need a concrete username for the (sudo) chown step below. + resolved_user = remote_user + if not resolved_user: + rc, out, err = _ssh_run(ssh, "id -un") + if rc == 0 and out.strip(): + resolved_user = out.strip() + + sftp = ssh.open_sftp() + rtmp: Optional[str] = None + try: + rc, out, err = _ssh_run(ssh, "mktemp -d") + if rc != 0: + raise RuntimeError(f"Remote mktemp failed: {err.strip()}") + rtmp = out.strip() + + # Be explicit: restrict the remote staging area to the current user. + rc, out, err = _ssh_run(ssh, f"chmod 700 {rtmp}") + if rc != 0: + raise RuntimeError(f"Remote chmod failed: {err.strip()}") + + rapp = f"{rtmp}/enroll.pyz" + rbundle = f"{rtmp}/bundle" + + sftp.put(str(pyz), rapp) + + # Run remote harvest. + argv: list[str] = [ + remote_python, + rapp, + "harvest", + "--out", + rbundle, + ] + if dangerous: + argv.append("--dangerous") + for p in include_paths or []: + argv.extend(["--include-path", str(p)]) + for p in exclude_paths or []: + argv.extend(["--exclude-path", str(p)]) + + _cmd = " ".join(map(shlex.quote, argv)) + if not no_sudo: + # Prefer non-interactive sudo first; retry with -S only when needed. + rc, out, err = _ssh_run_sudo( + ssh, _cmd, sudo_password=sudo_password, get_pty=True + ) + cmd = f"sudo {_cmd}" + else: + cmd = _cmd + rc, out, err = _ssh_run(ssh, cmd, get_pty=False) + if rc != 0: + raise RuntimeError( + "Remote harvest failed.\n" + f"Command: {cmd}\n" + f"Exit code: {rc}\n" + f"Stdout: {out.strip()}\n" + f"Stderr: {err.strip()}" + ) + + if not no_sudo: + # Ensure user can read the files, before we tar it. + if not resolved_user: + raise RuntimeError( + "Unable to determine remote username for chown. " + "Pass --remote-user explicitly or use --no-sudo." + ) + chown_cmd = f"chown -R {resolved_user} {rbundle}" + rc, out, err = _ssh_run_sudo( + ssh, + chown_cmd, + sudo_password=sudo_password, + get_pty=True, + ) + if rc != 0: + raise RuntimeError( + "chown of harvest failed.\n" + f"Command: sudo {chown_cmd}\n" + f"Exit code: {rc}\n" + f"Stdout: {out.strip()}\n" + f"Stderr: {err.strip()}" + ) + + # Stream a tarball back to the local machine (avoid creating a tar file on the remote). + cmd = f"tar -cz -C {rbundle} ." + _stdin, stdout, stderr = ssh.exec_command(cmd) # nosec + with open(local_tgz, "wb") as f: + while True: + chunk = stdout.read(1024 * 128) + if not chunk: + break + f.write(chunk) + rc = stdout.channel.recv_exit_status() + err_text = stderr.read().decode("utf-8", errors="replace") + if rc != 0: + raise RuntimeError( + "Remote tar stream failed.\n" + f"Command: {cmd}\n" + f"Exit code: {rc}\n" + f"Stderr: {err_text.strip()}" + ) + + # Extract into the destination. + with tarfile.open(local_tgz, mode="r:gz") as tf: + _safe_extract_tar(tf, local_out_dir) + + finally: + # Cleanup remote tmpdir even on failure. + if rtmp: + _ssh_run(ssh, f"rm -rf {rtmp}") + try: + sftp.close() + ssh.close() + except Exception: + ssh.close() + raise RuntimeError("Something went wrong generating the harvest") + + return local_out_dir / "state.json" diff --git a/enroll/role_names.py b/enroll/role_names.py new file mode 100644 index 0000000..a6f4a32 --- /dev/null +++ b/enroll/role_names.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +RESERVED_SINGLETON_ROLE_NAMES = { + "users", + "flatpak", + "snap", + "container_images", + "apt_config", + "dnf_config", + "firewall_runtime", + "sysctl", + "etc_custom", + "usr_local_custom", + "extra_paths", + "common_packages", +} + + +def avoid_reserved_role_name(role_name: str, *, prefix: str) -> str: + """Return a role name that cannot collide with singleton roles. + + Singleton roles are generated once per manifest from dedicated top-level + state sections. Package and service roles can naturally have the same names + as those singletons, e.g. the OS package named ``flatpak``. Prefix those + generated package/service roles so they cannot overwrite singleton role + directories during manifestation. + """ + if role_name in RESERVED_SINGLETON_ROLE_NAMES: + return f"{prefix}_{role_name}" + return role_name diff --git a/enroll/rpm.py b/enroll/rpm.py new file mode 100644 index 0000000..a036814 --- /dev/null +++ b/enroll/rpm.py @@ -0,0 +1,327 @@ +from __future__ import annotations + +import os +import re +import shutil +import subprocess # nosec +from typing import Dict, List, Optional, Set, Tuple + + +def _run( + cmd: list[str], *, allow_fail: bool = False, merge_err: bool = False +) -> tuple[int, str]: + """Run a command and return (rc, stdout). + + If merge_err is True, stderr is merged into stdout to preserve ordering. + """ + p = subprocess.run( + cmd, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=(subprocess.STDOUT if merge_err else subprocess.PIPE), + ) # nosec + out = p.stdout or "" + if (not allow_fail) and p.returncode != 0: + err = "" if merge_err else (p.stderr or "") + raise RuntimeError(f"Command failed: {cmd}\n{err}{out}") + return p.returncode, out + + +def rpm_owner(path: str) -> Optional[str]: + """Return owning package name for a path, or None if unowned.""" + if not path: + return None + rc, out = _run( + ["rpm", "-qf", "--qf", "%{NAME}\n", path], allow_fail=True, merge_err=True + ) + if rc != 0: + return None + for line in out.splitlines(): + line = line.strip() + if not line: + continue + if "is not owned" in line: + return None + # With --qf we expect just the package name. + if re.match(r"^[A-Za-z0-9_.+:-]+$", line): + # Strip any accidental epoch/name-version-release output. + return line.split(":", 1)[-1].strip() if line else None + return None + + +_ARCH_SUFFIXES = { + "noarch", + "x86_64", + "i686", + "aarch64", + "armv7hl", + "ppc64le", + "s390x", + "riscv64", +} + + +def _strip_arch(token: str) -> str: + """Strip a trailing .ARCH from a yum/dnf package token.""" + t = token.strip() + if "." not in t: + return t + head, tail = t.rsplit(".", 1) + if tail in _ARCH_SUFFIXES: + return head + return t + + +def list_manual_packages() -> List[str]: + """Return packages considered "user-installed" on RPM-based systems. + + Best-effort: + 1) dnf repoquery --userinstalled + 2) dnf history userinstalled + 3) yum history userinstalled + + If none are available, returns an empty list. + """ + + def _dedupe(pkgs: List[str]) -> List[str]: + return sorted({p for p in (pkgs or []) if p}) + + if shutil.which("dnf"): + # Prefer a machine-friendly output. + for cmd in ( + ["dnf", "-q", "repoquery", "--userinstalled", "--qf", "%{name}\n"], + ["dnf", "-q", "repoquery", "--userinstalled"], + ): + rc, out = _run(cmd, allow_fail=True, merge_err=True) + if rc == 0 and out.strip(): + pkgs = [] + for line in out.splitlines(): + line = line.strip() + if not line or line.startswith("Loaded plugins"): + continue + pkgs.append(_strip_arch(line.split()[0])) + if pkgs: + return _dedupe(pkgs) + + # Fallback + rc, out = _run( + ["dnf", "-q", "history", "userinstalled"], allow_fail=True, merge_err=True + ) + if rc == 0 and out.strip(): + pkgs = [] + for line in out.splitlines(): + line = line.strip() + if not line or line.startswith("Installed") or line.startswith("Last"): + continue + # Often: "vim-enhanced.x86_64" + tok = line.split()[0] + pkgs.append(_strip_arch(tok)) + if pkgs: + return _dedupe(pkgs) + + if shutil.which("yum"): + rc, out = _run( + ["yum", "-q", "history", "userinstalled"], allow_fail=True, merge_err=True + ) + if rc == 0 and out.strip(): + pkgs = [] + for line in out.splitlines(): + line = line.strip() + if ( + not line + or line.startswith("Installed") + or line.startswith("Loaded") + ): + continue + tok = line.split()[0] + pkgs.append(_strip_arch(tok)) + if pkgs: + return _dedupe(pkgs) + + return [] + + +def list_installed_packages() -> Dict[str, List[Dict[str, str]]]: + """Return mapping of installed package name -> installed instances. + + Uses `rpm -qa` and is expected to work on RHEL/Fedora-like systems. + + Output format: + {"pkg": [{"version": "...", "arch": "...", "group": "..."}, ...], ...} + + The version string is formatted as: + - "-" for typical packages + - ":-" if a non-zero epoch is present + """ + + try: + _, out = _run( + [ + "rpm", + "-qa", + "--qf", + "%{NAME}\t%{EPOCHNUM}\t%{VERSION}\t%{RELEASE}\t%{ARCH}\t%{GROUP}\n", + ], + allow_fail=False, + merge_err=True, + ) + except Exception: + return {} + + pkgs: Dict[str, List[Dict[str, str]]] = {} + for raw in (out or "").splitlines(): + line = raw.strip("\n") + if not line: + continue + parts = line.split("\t") + if len(parts) < 5: + continue + name, epoch, ver, rel, arch = [p.strip() for p in parts[:5]] + if not name or not ver: + continue + + # Normalise epoch. + epoch = epoch.strip() + if epoch.lower() in ("(none)", "none", ""): + epoch = "0" + + v = f"{ver}-{rel}" if rel else ver + if epoch and epoch.isdigit() and epoch != "0": + v = f"{epoch}:{v}" + + instance = {"version": v, "arch": arch} + if len(parts) >= 6 and parts[5].strip(): + instance["group"] = parts[5].strip() + + pkgs.setdefault(name, []).append(instance) + + for k in list(pkgs.keys()): + pkgs[k] = sorted( + pkgs[k], key=lambda x: (x.get("arch") or "", x.get("version") or "") + ) + return pkgs + + +def _walk_etc_files() -> List[str]: + out: List[str] = [] + for dirpath, _, filenames in os.walk("/etc"): + for fn in filenames: + p = os.path.join(dirpath, fn) + if os.path.islink(p) or not os.path.isfile(p): + continue + out.append(p) + return out + + +def build_rpm_etc_index() -> ( + Tuple[Set[str], Dict[str, str], Dict[str, Set[str]], Dict[str, List[str]]] +): + """Best-effort equivalent of build_dpkg_etc_index for RPM systems. + + This builds indexes by walking the live /etc tree and querying RPM ownership + for each file. + + Returns: + owned_etc_paths: set of /etc paths owned by rpm + etc_owner_map: /etc/path -> pkg + topdir_to_pkgs: "nginx" -> {"nginx", ...} based on /etc//... + pkg_to_etc_paths: pkg -> list of owned /etc paths + """ + + owned: Set[str] = set() + owner: Dict[str, str] = {} + topdir_to_pkgs: Dict[str, Set[str]] = {} + pkg_to_etc: Dict[str, List[str]] = {} + + paths = _walk_etc_files() + + # Query in chunks to avoid excessive process spawns. + chunk_size = 250 + + not_owned_re = re.compile( + r"^file\s+(?P.+?)\s+is\s+not\s+owned\s+by\s+any\s+package", re.IGNORECASE + ) + + for i in range(0, len(paths), chunk_size): + chunk = paths[i : i + chunk_size] + rc, out = _run( + ["rpm", "-qf", "--qf", "%{NAME}\n", *chunk], + allow_fail=True, + merge_err=True, + ) + + lines = [ln.strip() for ln in out.splitlines() if ln.strip()] + # Heuristic: rpm prints one output line per input path. If that isn't + # true (warnings/errors), fall back to per-file queries for this chunk. + if len(lines) != len(chunk): + for p in chunk: + pkg = rpm_owner(p) + if not pkg: + continue + owned.add(p) + owner.setdefault(p, pkg) + pkg_to_etc.setdefault(pkg, []).append(p) + parts = p.split("/", 3) + if len(parts) >= 3 and parts[2]: + topdir_to_pkgs.setdefault(parts[2], set()).add(pkg) + continue + + for pth, line in zip(chunk, lines): + if not line: + continue + if not_owned_re.match(line) or "is not owned" in line: + continue + pkg = line.split()[0].strip() + if not pkg: + continue + owned.add(pth) + owner.setdefault(pth, pkg) + pkg_to_etc.setdefault(pkg, []).append(pth) + parts = pth.split("/", 3) + if len(parts) >= 3 and parts[2]: + topdir_to_pkgs.setdefault(parts[2], set()).add(pkg) + + for k, v in list(pkg_to_etc.items()): + pkg_to_etc[k] = sorted(set(v)) + + return owned, owner, topdir_to_pkgs, pkg_to_etc + + +def rpm_config_files(pkg: str) -> Set[str]: + """Return config files for a package (rpm -qc).""" + rc, out = _run(["rpm", "-qc", pkg], allow_fail=True, merge_err=True) + if rc != 0: + return set() + files: Set[str] = set() + for line in out.splitlines(): + line = line.strip() + if line.startswith("/"): + files.add(line) + return files + + +def rpm_modified_files(pkg: str) -> Set[str]: + """Return files reported as modified by rpm verification (rpm -V). + + rpm -V only prints lines for differences/missing files. + """ + rc, out = _run(["rpm", "-V", pkg], allow_fail=True, merge_err=True) + # rc is non-zero when there are differences; we still want the output. + files: Set[str] = set() + for raw in out.splitlines(): + line = raw.strip() + if not line: + continue + # Typical forms: + # S.5....T. c /etc/foo.conf + # missing /etc/bar + m = re.search(r"\s(/\S+)$", line) + if m: + files.add(m.group(1)) + continue + if line.startswith("missing"): + parts = line.split() + if parts and parts[-1].startswith("/"): + files.add(parts[-1]) + return files diff --git a/enroll/schema/__init__.py b/enroll/schema/__init__.py new file mode 100644 index 0000000..9d19c43 --- /dev/null +++ b/enroll/schema/__init__.py @@ -0,0 +1,4 @@ +"""Vendored JSON schemas. + +These are used by `enroll validate` so validation can run offline. +""" diff --git a/enroll/schema/state.schema.json b/enroll/schema/state.schema.json new file mode 100644 index 0000000..8806b2e --- /dev/null +++ b/enroll/schema/state.schema.json @@ -0,0 +1,1264 @@ +{ + "$defs": { + "AptConfigSnapshot": { + "allOf": [ + { + "$ref": "#/$defs/RoleCommon" + }, + { + "properties": { + "role_name": { + "const": "apt_config" + } + }, + "type": "object" + } + ], + "unevaluatedProperties": false + }, + "ContainerImageTagAlias": { + "additionalProperties": false, + "properties": { + "ref": { + "minLength": 1, + "type": "string" + }, + "repository": { + "minLength": 1, + "type": "string" + }, + "tag": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "ref", + "repository", + "tag" + ], + "type": "object" + }, + "ContainerImage": { + "additionalProperties": false, + "properties": { + "architecture": { + "type": [ + "string", + "null" + ] + }, + "created": { + "type": [ + "string", + "null" + ] + }, + "engine": { + "enum": [ + "docker", + "podman" + ], + "type": "string" + }, + "home": { + "type": [ + "string", + "null" + ] + }, + "image_id": { + "type": [ + "string", + "null" + ] + }, + "notes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "os": { + "type": [ + "string", + "null" + ] + }, + "platform": { + "type": [ + "string", + "null" + ] + }, + "pull_ref": { + "type": [ + "string", + "null" + ] + }, + "repo_digests": { + "items": { + "type": "string" + }, + "type": "array" + }, + "repo_tags": { + "items": { + "type": "string" + }, + "type": "array" + }, + "scope": { + "enum": [ + "system", + "user" + ], + "type": "string" + }, + "size": { + "type": [ + "integer", + "null" + ] + }, + "source": { + "type": "string" + }, + "tag_aliases": { + "items": { + "$ref": "#/$defs/ContainerImageTagAlias" + }, + "type": "array" + }, + "user": { + "type": [ + "string", + "null" + ] + }, + "variant": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "engine", + "scope", + "user", + "home", + "image_id", + "repo_tags", + "repo_digests", + "pull_ref", + "tag_aliases", + "os", + "architecture", + "variant", + "platform", + "size", + "created", + "source", + "notes" + ], + "type": "object" + }, + "ContainerImagesSnapshot": { + "additionalProperties": false, + "properties": { + "images": { + "items": { + "$ref": "#/$defs/ContainerImage" + }, + "type": "array" + }, + "notes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "role_name": { + "const": "container_images" + } + }, + "required": [ + "role_name", + "images", + "notes" + ], + "type": "object" + }, + "DnfConfigSnapshot": { + "allOf": [ + { + "$ref": "#/$defs/RoleCommon" + }, + { + "properties": { + "role_name": { + "const": "dnf_config" + } + }, + "type": "object" + } + ], + "unevaluatedProperties": false + }, + "EtcCustomSnapshot": { + "allOf": [ + { + "$ref": "#/$defs/RoleCommon" + }, + { + "properties": { + "role_name": { + "const": "etc_custom" + } + }, + "type": "object" + } + ], + "unevaluatedProperties": false + }, + "ExcludedFile": { + "additionalProperties": false, + "properties": { + "path": { + "minLength": 1, + "pattern": "^/.*", + "type": "string" + }, + "reason": { + "enum": [ + "user_excluded", + "unreadable", + "backup_file", + "log_file", + "denied_path", + "too_large", + "not_regular_file", + "not_symlink", + "binary_like", + "sensitive_content" + ], + "type": "string" + } + }, + "required": [ + "path", + "reason" + ], + "type": "object" + }, + "ExtraPathsSnapshot": { + "allOf": [ + { + "$ref": "#/$defs/RoleCommon" + }, + { + "properties": { + "exclude_patterns": { + "items": { + "type": "string" + }, + "type": "array" + }, + "include_patterns": { + "items": { + "type": "string" + }, + "type": "array" + }, + "role_name": { + "const": "extra_paths" + } + }, + "required": [ + "include_patterns", + "exclude_patterns" + ], + "type": "object" + } + ], + "unevaluatedProperties": false + }, + "InstalledPackageInstance": { + "additionalProperties": false, + "properties": { + "arch": { + "minLength": 1, + "type": "string" + }, + "group": { + "minLength": 1, + "type": "string" + }, + "section": { + "minLength": 1, + "type": "string" + }, + "version": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "version", + "arch" + ], + "type": "object" + }, + "ManagedDir": { + "additionalProperties": false, + "properties": { + "group": { + "minLength": 1, + "type": "string" + }, + "mode": { + "pattern": "^[0-7]{4}$", + "type": "string" + }, + "owner": { + "minLength": 1, + "type": "string" + }, + "path": { + "minLength": 1, + "pattern": "^/.*", + "type": "string" + }, + "reason": { + "enum": [ + "parent_of_managed_file", + "user_include_dir" + ], + "type": "string" + } + }, + "required": [ + "path", + "owner", + "group", + "mode", + "reason" + ], + "type": "object" + }, + "ManagedFile": { + "additionalProperties": false, + "properties": { + "group": { + "minLength": 1, + "type": "string" + }, + "mode": { + "pattern": "^[0-7]{4}$", + "type": "string" + }, + "owner": { + "minLength": 1, + "type": "string" + }, + "path": { + "minLength": 1, + "pattern": "^/.*", + "type": "string" + }, + "reason": { + "enum": [ + "apt_config", + "apt_keyring", + "apt_signed_by_keyring", + "apt_source", + "authorized_keys", + "cron_snippet", + "custom_specific_path", + "custom_unowned", + "dnf_config", + "logrotate_snippet", + "modified_conffile", + "modified_packaged_file", + "related_timer", + "rpm_gpg_key", + "ssh_public_key", + "system_cron", + "system_firewall", + "system_logrotate", + "system_modprobe", + "system_mounts", + "system_network", + "system_rc", + "system_security", + "system_sysctl", + "systemd_dropin", + "systemd_envfile", + "user_include", + "user_profile", + "user_shell_aliases", + "user_shell_logout", + "user_shell_rc", + "usr_local_bin_script", + "usr_local_etc_custom", + "yum_conf", + "yum_config", + "yum_repo" + ], + "type": "string" + }, + "src_rel": { + "minLength": 1, + "pattern": "^[^/].*", + "type": "string" + } + }, + "required": [ + "path", + "src_rel", + "owner", + "group", + "mode", + "reason" + ], + "type": "object" + }, + "ManagedLink": { + "additionalProperties": false, + "type": "object", + "properties": { + "path": { + "type": "string", + "minLength": 1, + "pattern": "^/.*" + }, + "target": { + "type": "string", + "minLength": 1 + }, + "reason": { + "type": "string", + "enum": [ + "enabled_symlink" + ] + } + }, + "required": [ + "path", + "target", + "reason" + ] + }, + "ObservedVia": { + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "user_installed" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "systemd_unit" + }, + "ref": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "kind", + "ref" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "package_role" + }, + "ref": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "kind", + "ref" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "firewall_runtime" + }, + "ref": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "kind", + "ref" + ], + "type": "object" + } + ] + }, + "PackageInventoryEntry": { + "additionalProperties": false, + "properties": { + "arches": { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array" + }, + "installations": { + "items": { + "$ref": "#/$defs/InstalledPackageInstance" + }, + "type": "array" + }, + "observed_via": { + "items": { + "$ref": "#/$defs/ObservedVia" + }, + "type": "array" + }, + "roles": { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array" + }, + "section": { + "type": [ + "string", + "null" + ] + }, + "version": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "version", + "arches", + "installations", + "observed_via", + "roles" + ], + "type": "object" + }, + "PackageSnapshot": { + "allOf": [ + { + "$ref": "#/$defs/RoleCommon" + }, + { + "properties": { + "package": { + "minLength": 1, + "type": "string" + }, + "has_config": { + "type": "boolean", + "default": true + }, + "section": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "package" + ], + "type": "object" + } + ], + "unevaluatedProperties": false + }, + "RoleCommon": { + "properties": { + "excluded": { + "items": { + "$ref": "#/$defs/ExcludedFile" + }, + "type": "array" + }, + "managed_dirs": { + "items": { + "$ref": "#/$defs/ManagedDir" + }, + "type": "array" + }, + "managed_files": { + "items": { + "$ref": "#/$defs/ManagedFile" + }, + "type": "array" + }, + "managed_links": { + "items": { + "$ref": "#/$defs/ManagedLink" + }, + "type": "array" + }, + "notes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "role_name": { + "minLength": 1, + "pattern": "^[A-Za-z0-9_]+$", + "type": "string" + } + }, + "required": [ + "role_name", + "managed_dirs", + "managed_files", + "excluded", + "notes" + ], + "type": "object" + }, + "ServiceSnapshot": { + "allOf": [ + { + "$ref": "#/$defs/RoleCommon" + }, + { + "properties": { + "active_state": { + "type": [ + "string", + "null" + ] + }, + "condition_result": { + "type": [ + "string", + "null" + ] + }, + "packages": { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array" + }, + "role_name": { + "minLength": 1, + "pattern": "^[a-z_][a-z0-9_]*$", + "type": "string" + }, + "sub_state": { + "type": [ + "string", + "null" + ] + }, + "unit": { + "minLength": 1, + "type": "string" + }, + "unit_file_state": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "unit", + "packages", + "active_state", + "sub_state", + "unit_file_state", + "condition_result" + ], + "type": "object" + } + ], + "unevaluatedProperties": false + }, + "UserEntry": { + "additionalProperties": false, + "properties": { + "gecos": { + "type": "string" + }, + "gid": { + "minimum": 0, + "type": "integer" + }, + "home": { + "type": "string" + }, + "name": { + "minLength": 1, + "type": "string" + }, + "primary_group": { + "minLength": 1, + "type": "string" + }, + "shell": { + "type": "string" + }, + "supplementary_groups": { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array" + }, + "uid": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "name", + "uid", + "gid", + "gecos", + "home", + "shell", + "primary_group", + "supplementary_groups" + ], + "type": "object" + }, + "UsersSnapshot": { + "allOf": [ + { + "$ref": "#/$defs/RoleCommon" + }, + { + "properties": { + "role_name": { + "const": "users" + }, + "users": { + "items": { + "$ref": "#/$defs/UserEntry" + }, + "type": "array" + }, + "user_flatpaks": { + "additionalProperties": { + "type": "array", + "items": { + "$ref": "#/$defs/FlatpakInstall" + } + }, + "type": "object" + }, + "user_flatpak_remotes": { + "type": "array", + "items": { + "$ref": "#/$defs/FlatpakRemote" + } + } + }, + "required": [ + "users" + ], + "type": "object" + } + ], + "unevaluatedProperties": false + }, + "UsrLocalCustomSnapshot": { + "allOf": [ + { + "$ref": "#/$defs/RoleCommon" + }, + { + "properties": { + "role_name": { + "const": "usr_local_custom" + } + }, + "type": "object" + } + ], + "unevaluatedProperties": false + }, + "FirewallRuntimeSnapshot": { + "additionalProperties": false, + "properties": { + "role_name": { + "const": "firewall_runtime" + }, + "packages": { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array" + }, + "ipset_save": { + "type": [ + "string", + "null" + ] + }, + "ipset_sets": { + "items": { + "minLength": 1, + "type": "string" + }, + "type": "array" + }, + "iptables_v4_save": { + "type": [ + "string", + "null" + ] + }, + "iptables_v6_save": { + "type": [ + "string", + "null" + ] + }, + "notes": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "role_name", + "packages", + "ipset_save", + "ipset_sets", + "iptables_v4_save", + "iptables_v6_save", + "notes" + ], + "type": "object" + }, + "SysctlSnapshot": { + "additionalProperties": false, + "properties": { + "role_name": { + "const": "sysctl" + }, + "managed_files": { + "items": { + "$ref": "#/$defs/ManagedFile" + }, + "type": "array" + }, + "parameters": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "notes": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "role_name", + "managed_files", + "parameters", + "notes" + ], + "type": "object" + }, + "FlatpakInstall": { + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "method": { + "type": "string", + "enum": [ + "system", + "user" + ] + }, + "remote": { + "type": [ + "string", + "null" + ] + }, + "branch": { + "type": [ + "string", + "null" + ] + }, + "arch": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "type": [ + "string", + "null" + ], + "enum": [ + "app", + "runtime", + null + ] + }, + "ref": { + "type": [ + "string", + "null" + ] + }, + "user": { + "type": [ + "string", + "null" + ] + }, + "home": { + "type": [ + "string", + "null" + ] + }, + "source": { + "type": "string" + }, + "from_url": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "name", + "method" + ], + "type": "object" + }, + "FlatpakRemote": { + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "method": { + "type": "string", + "enum": [ + "system", + "user" + ] + }, + "url": { + "type": "string", + "minLength": 1 + }, + "user": { + "type": [ + "string", + "null" + ] + }, + "home": { + "type": [ + "string", + "null" + ] + }, + "source": { + "type": "string" + } + }, + "required": [ + "name", + "method", + "url" + ], + "type": "object" + }, + "SnapInstall": { + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "channel": { + "type": [ + "string", + "null" + ] + }, + "revision": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "classic": { + "type": "boolean" + }, + "devmode": { + "type": "boolean" + }, + "dangerous": { + "type": "boolean" + }, + "notes": { + "type": "array", + "items": { + "type": "string" + } + }, + "source": { + "type": "string" + }, + "install_revision": { + "type": "boolean" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "FlatpakSnapshot": { + "additionalProperties": false, + "properties": { + "role_name": { + "const": "flatpak" + }, + "system_flatpaks": { + "type": "array", + "items": { + "$ref": "#/$defs/FlatpakInstall" + } + }, + "remotes": { + "type": "array", + "items": { + "$ref": "#/$defs/FlatpakRemote" + } + }, + "notes": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "role_name" + ], + "type": "object" + }, + "SnapSnapshot": { + "additionalProperties": false, + "properties": { + "role_name": { + "const": "snap" + }, + "system_snaps": { + "type": "array", + "items": { + "$ref": "#/$defs/SnapInstall" + } + }, + "notes": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "role_name" + ], + "type": "object" + } + }, + "$id": "https://enroll.sh/schema/state.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "enroll": { + "additionalProperties": false, + "properties": { + "harvest_time": { + "minimum": 0, + "type": "integer" + }, + "version": { + "type": "string" + } + }, + "required": [ + "version", + "harvest_time" + ], + "type": "object" + }, + "host": { + "additionalProperties": false, + "properties": { + "hostname": { + "minLength": 1, + "type": "string" + }, + "os": { + "enum": [ + "debian", + "redhat", + "unknown" + ], + "type": "string" + }, + "os_release": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "pkg_backend": { + "enum": [ + "dpkg", + "rpm" + ], + "type": "string" + } + }, + "required": [ + "hostname", + "os", + "pkg_backend", + "os_release" + ], + "type": "object" + }, + "inventory": { + "additionalProperties": false, + "properties": { + "packages": { + "additionalProperties": { + "$ref": "#/$defs/PackageInventoryEntry" + }, + "type": "object" + } + }, + "required": [ + "packages" + ], + "type": "object" + }, + "roles": { + "additionalProperties": false, + "properties": { + "apt_config": { + "$ref": "#/$defs/AptConfigSnapshot" + }, + "dnf_config": { + "$ref": "#/$defs/DnfConfigSnapshot" + }, + "etc_custom": { + "$ref": "#/$defs/EtcCustomSnapshot" + }, + "extra_paths": { + "$ref": "#/$defs/ExtraPathsSnapshot" + }, + "packages": { + "items": { + "$ref": "#/$defs/PackageSnapshot" + }, + "type": "array" + }, + "services": { + "items": { + "$ref": "#/$defs/ServiceSnapshot" + }, + "type": "array" + }, + "users": { + "$ref": "#/$defs/UsersSnapshot" + }, + "usr_local_custom": { + "$ref": "#/$defs/UsrLocalCustomSnapshot" + }, + "firewall_runtime": { + "$ref": "#/$defs/FirewallRuntimeSnapshot" + }, + "sysctl": { + "$ref": "#/$defs/SysctlSnapshot" + }, + "flatpak": { + "$ref": "#/$defs/FlatpakSnapshot" + }, + "snap": { + "$ref": "#/$defs/SnapSnapshot" + }, + "container_images": { + "$ref": "#/$defs/ContainerImagesSnapshot" + } + }, + "required": [ + "users", + "services", + "packages", + "apt_config", + "dnf_config", + "etc_custom", + "usr_local_custom", + "extra_paths" + ], + "type": "object" + } + }, + "required": [ + "enroll", + "host", + "inventory", + "roles" + ], + "title": "Enroll harvest state.json schema (latest)", + "type": "object" +} diff --git a/enroll/sopsutil.py b/enroll/sopsutil.py new file mode 100644 index 0000000..de36d4f --- /dev/null +++ b/enroll/sopsutil.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import os +import shutil +import subprocess # nosec +import tempfile +from pathlib import Path +from typing import Iterable, List, Optional + + +class SopsError(RuntimeError): + pass + + +def find_sops_cmd() -> Optional[str]: + """Return the `sops` executable path if present on PATH.""" + return shutil.which("sops") + + +def require_sops_cmd() -> str: + exe = find_sops_cmd() + if not exe: + raise SopsError( + "--sops was requested but `sops` was not found on PATH. " + "Install sops and ensure it is available as `sops`." + ) + return exe + + +def _pgp_arg(fingerprints: Iterable[str]) -> str: + fps = [f.strip() for f in fingerprints if f and f.strip()] + if not fps: + raise SopsError("No GPG fingerprints provided for --sops") + # sops accepts a comma-separated list for --pgp. + return ",".join(fps) + + +def encrypt_file_binary( + src_path: Path, + dst_path: Path, + *, + pgp_fingerprints: List[str], + mode: int = 0o600, +) -> None: + """Encrypt src_path with sops (binary) and write to dst_path atomically.""" + sops = require_sops_cmd() + src_path = Path(src_path) + dst_path = Path(dst_path) + dst_path.parent.mkdir(parents=True, exist_ok=True) + + res = subprocess.run( + [ + sops, + "--encrypt", + "--input-type", + "binary", + "--output-type", + "binary", + "--pgp", + _pgp_arg(pgp_fingerprints), + str(src_path), + ], + capture_output=True, + check=False, + ) # nosec + if res.returncode != 0: + raise SopsError( + "sops encryption failed:\n" + f" cmd: {sops} --encrypt ... {src_path}\n" + f" rc: {res.returncode}\n" + f" stderr: {res.stderr.decode('utf-8', errors='replace').strip()}" + ) + + # Write atomically in the destination directory. + fd, tmp = tempfile.mkstemp(prefix=".enroll-sops-", dir=str(dst_path.parent)) + try: + with os.fdopen(fd, "wb") as f: + f.write(res.stdout) + try: + os.chmod(tmp, mode) + except OSError: + pass + os.replace(tmp, dst_path) + finally: + try: + os.unlink(tmp) + except FileNotFoundError: + pass + + +def decrypt_file_binary_to( + src_path: Path, + dst_path: Path, + *, + mode: int = 0o600, +) -> None: + """Decrypt a sops-encrypted file (binary) into dst_path.""" + sops = require_sops_cmd() + src_path = Path(src_path) + dst_path = Path(dst_path) + dst_path.parent.mkdir(parents=True, exist_ok=True) + + res = subprocess.run( + [ + sops, + "--decrypt", + "--input-type", + "binary", + "--output-type", + "binary", + str(src_path), + ], + capture_output=True, + check=False, + ) # nosec + if res.returncode != 0: + raise SopsError( + "sops decryption failed:\n" + f" cmd: {sops} --decrypt ... {src_path}\n" + f" rc: {res.returncode}\n" + f" stderr: {res.stderr.decode('utf-8', errors='replace').strip()}" + ) + + fd, tmp = tempfile.mkstemp(prefix=".enroll-sops-", dir=str(dst_path.parent)) + try: + with os.fdopen(fd, "wb") as f: + f.write(res.stdout) + try: + os.chmod(tmp, mode) + except OSError: + pass + os.replace(tmp, dst_path) + finally: + try: + os.unlink(tmp) + except FileNotFoundError: + pass diff --git a/enroll/state.py b/enroll/state.py new file mode 100644 index 0000000..ed5a264 --- /dev/null +++ b/enroll/state.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, Mapping, Union + +BundlePath = Union[str, Path] +State = Dict[str, Any] + + +def state_path(bundle_dir: BundlePath) -> Path: + """Return the canonical state.json path for a harvest bundle.""" + + return Path(bundle_dir) / "state.json" + + +def load_state(bundle_dir: BundlePath) -> State: + """Load state.json from a harvest bundle directory.""" + + with open(state_path(bundle_dir), "r", encoding="utf-8") as f: + return json.load(f) + + +def write_state( + bundle_dir: BundlePath, + state: Mapping[str, Any], + *, + indent: int = 2, + sort_keys: bool = True, +) -> Path: + """Write state.json to a harvest bundle directory and return its path.""" + + path = state_path(bundle_dir) + with open(path, "w", encoding="utf-8") as f: + json.dump(state, f, indent=indent, sort_keys=sort_keys) + return path + + +def roles_from_state(state: Mapping[str, Any]) -> Dict[str, Any]: + """Return the roles mapping from a harvest state, or an empty mapping.""" + + roles = state.get("roles") + return dict(roles) if isinstance(roles, dict) else {} + + +def inventory_packages_from_state(state: Mapping[str, Any]) -> Dict[str, Any]: + """Return inventory.packages from a harvest state, or an empty mapping.""" + + inventory = state.get("inventory") + if not isinstance(inventory, dict): + return {} + packages = inventory.get("packages") + return dict(packages) if isinstance(packages, dict) else {} diff --git a/enroll/system_paths.py b/enroll/system_paths.py new file mode 100644 index 0000000..759d7b5 --- /dev/null +++ b/enroll/system_paths.py @@ -0,0 +1,313 @@ +from __future__ import annotations + +import glob +import os +import re +from typing import Dict, List, Set, Tuple + + +ALLOWED_UNOWNED_EXTS = { + ".cfg", + ".cnf", + ".conf", + ".ini", + ".json", + ".link", + ".mount", + ".netdev", + ".network", + ".path", + ".rules", + ".service", + ".socket", + ".target", + ".timer", + ".toml", + ".yaml", + ".yml", + "", # allow extensionless (common in /etc/default and /etc/init.d) +} + +MAX_FILES_CAP = 4000 +MAX_UNOWNED_FILES_PER_ROLE = 500 + + +def is_confish(path: str) -> bool: + base = os.path.basename(path) + _, ext = os.path.splitext(base) + return ext in ALLOWED_UNOWNED_EXTS + + +def scan_unowned_under_roots( + roots: List[str], + owned_etc: Set[str], + limit: int = MAX_UNOWNED_FILES_PER_ROLE, + *, + confish_only: bool = True, +) -> List[str]: + found: List[str] = [] + for root in roots: + if not os.path.isdir(root): + continue + for dirpath, _, filenames in os.walk(root): + if len(found) >= limit: + return found + for fn in filenames: + if len(found) >= limit: + return found + p = os.path.join(dirpath, fn) + if not p.startswith("/etc/"): + continue + if p in owned_etc: + continue + if not os.path.isfile(p) or os.path.islink(p): + continue + if confish_only and not is_confish(p): + continue + found.append(p) + return found + + +def topdirs_for_package(pkg: str, pkg_to_etc_paths: Dict[str, List[str]]) -> Set[str]: + topdirs: Set[str] = set() + for path in pkg_to_etc_paths.get(pkg, []): + parts = path.split("/", 3) + if len(parts) >= 3 and parts[1] == "etc" and parts[2]: + topdirs.add(parts[2]) + return topdirs + + +_APT_SOURCE_GLOBS = [ + "/etc/apt/sources.list", + "/etc/apt/sources.list.d/*.list", + "/etc/apt/sources.list.d/*.sources", +] + +_SYSTEM_CAPTURE_GLOBS: List[Tuple[str, str]] = [ + ("/etc/fstab", "system_mounts"), + ("/etc/crypttab", "system_mounts"), + ("/etc/sysctl.conf", "system_sysctl"), + ("/etc/sysctl.d/*", "system_sysctl"), + ("/etc/modprobe.d/*", "system_modprobe"), + ("/etc/modules", "system_modprobe"), + ("/etc/modules-load.d/*", "system_modprobe"), + ("/etc/netplan/*", "system_network"), + ("/etc/systemd/network/*", "system_network"), + ("/etc/network/interfaces", "system_network"), + ("/etc/network/interfaces.d/*", "system_network"), + ("/etc/resolvconf.conf", "system_network"), + ("/etc/resolvconf/resolv.conf.d/*", "system_network"), + ("/etc/NetworkManager/system-connections/*", "system_network"), + ("/etc/sysconfig/network*", "system_network"), + ("/etc/sysconfig/network-scripts/*", "system_network"), + ("/etc/nftables.conf", "system_firewall"), + ("/etc/nftables.d/*", "system_firewall"), + ("/etc/iptables/rules.v4", "system_firewall"), + ("/etc/iptables/rules.v6", "system_firewall"), + ("/etc/sysconfig/iptables", "system_firewall"), + ("/etc/sysconfig/ip6tables", "system_firewall"), + ("/etc/ipset.conf", "system_firewall"), + ("/etc/ipset/*", "system_firewall"), + ("/etc/ipset.d/*", "system_firewall"), + ("/etc/sysconfig/ipset", "system_firewall"), + ("/etc/default/ipset", "system_firewall"), + ("/etc/ufw/*", "system_firewall"), + ("/etc/default/ufw", "system_firewall"), + ("/etc/firewalld/*", "system_firewall"), + ("/etc/firewalld/zones/*", "system_firewall"), + ("/etc/selinux/config", "system_security"), + ("/etc/rc.local", "system_rc"), +] + +_PERSISTENT_IPTABLES_V4_GLOBS = [ + "/etc/iptables/rules.v4", + "/etc/sysconfig/iptables", +] + +_PERSISTENT_IPTABLES_V6_GLOBS = [ + "/etc/iptables/rules.v6", + "/etc/sysconfig/ip6tables", +] + +_PERSISTENT_IPSET_GLOBS = [ + "/etc/ipset.conf", + "/etc/ipset/*", + "/etc/ipset.d/*", + "/etc/sysconfig/ipset", +] + + +def persistent_ipset_globs() -> List[str]: + return list(_PERSISTENT_IPSET_GLOBS) + + +def persistent_iptables_v4_globs() -> List[str]: + return list(_PERSISTENT_IPTABLES_V4_GLOBS) + + +def persistent_iptables_v6_globs() -> List[str]: + return list(_PERSISTENT_IPTABLES_V6_GLOBS) + + +def persistent_firewall_files(globs: List[str]) -> List[str]: + """Return persistent firewall files matching ``globs``.""" + + seen: Set[str] = set() + out: List[str] = [] + for spec in globs: + for path in iter_matching_files(spec): + if path in seen: + continue + seen.add(path) + out.append(path) + return sorted(out) + + +def iter_matching_files(spec: str, *, cap: int = MAX_FILES_CAP) -> List[str]: + """Expand a glob spec and also walk directories to collect files.""" + + out: List[str] = [] + for p in glob.glob(spec): + if len(out) >= cap: + break + if os.path.islink(p): + continue + if os.path.isfile(p): + out.append(p) + continue + if os.path.isdir(p): + for dirpath, _, filenames in os.walk(p): + for fn in filenames: + if len(out) >= cap: + break + fp = os.path.join(dirpath, fn) + if os.path.islink(fp) or not os.path.isfile(fp): + continue + out.append(fp) + if len(out) >= cap: + break + return out + + +def parse_apt_signed_by(source_files: List[str]) -> Set[str]: + """Return absolute keyring paths referenced via signed-by / Signed-By.""" + + out: Set[str] = set() + re_signed_by = re.compile(r"signed-by\s*=\s*([^\]\s]+)", re.IGNORECASE) + re_signed_by_hdr = re.compile(r"^\s*Signed-By\s*:\s*(.+)$", re.IGNORECASE) + + for sf in source_files: + try: + with open(sf, "r", encoding="utf-8", errors="replace") as f: + for raw in f: + line = raw.strip() + if not line or line.startswith("#"): + continue + + m = re_signed_by_hdr.match(line) + if m: + val = m.group(1).strip() + if val.startswith("|"): + continue + toks = re.split(r"[\s,]+", val) + for t in toks: + if t.startswith("/"): + out.add(t) + continue + + if "[" in line and "]" in line: + bracket = line.split("[", 1)[1].split("]", 1)[0] + for mm in re_signed_by.finditer(bracket): + val = mm.group(1).strip().strip("\"'") + for t in re.split(r"[\s,]+", val): + if t.startswith("/"): + out.add(t) + continue + + for mm in re_signed_by.finditer(line): + val = mm.group(1).strip().strip("\"'") + for t in re.split(r"[\s,]+", val): + if t.startswith("/"): + out.add(t) + except OSError: + continue + + return out + + +def iter_apt_capture_paths() -> List[Tuple[str, str]]: + """Return (path, reason) pairs for APT configuration.""" + + reasons: Dict[str, str] = {} + + if os.path.isdir("/etc/apt"): + for dirpath, _, filenames in os.walk("/etc/apt"): + for fn in filenames: + p = os.path.join(dirpath, fn) + if os.path.islink(p) or not os.path.isfile(p): + continue + reasons.setdefault(p, "apt_config") + + apt_sources: List[str] = [] + for g in _APT_SOURCE_GLOBS: + apt_sources.extend(iter_matching_files(g)) + for p in sorted(set(apt_sources)): + reasons[p] = "apt_source" + + for g in ( + "/etc/apt/trusted.gpg", + "/etc/apt/trusted.gpg.d/*", + "/etc/apt/keyrings/*", + ): + for p in iter_matching_files(g): + reasons[p] = "apt_keyring" + + signed_by = parse_apt_signed_by(sorted(set(apt_sources))) + for p in sorted(signed_by): + if os.path.islink(p) or not os.path.isfile(p): + continue + if p.startswith("/etc/apt/"): + reasons[p] = "apt_keyring" + else: + reasons[p] = "apt_signed_by_keyring" + + return [(p, reasons[p]) for p in sorted(reasons.keys())] + + +def iter_dnf_capture_paths() -> List[Tuple[str, str]]: + """Return (path, reason) pairs for DNF/YUM configuration on RPM systems.""" + + reasons: Dict[str, str] = {} + + for root, tag in ( + ("/etc/dnf", "dnf_config"), + ("/etc/yum", "yum_config"), + ): + if os.path.isdir(root): + for dirpath, _, filenames in os.walk(root): + for fn in filenames: + p = os.path.join(dirpath, fn) + if os.path.islink(p) or not os.path.isfile(p): + continue + reasons.setdefault(p, tag) + + for p in iter_matching_files("/etc/yum.conf"): + reasons[p] = "yum_conf" + for p in iter_matching_files("/etc/yum.repos.d/*.repo"): + reasons[p] = "yum_repo" + for p in iter_matching_files("/etc/pki/rpm-gpg/*"): + reasons[p] = "rpm_gpg_key" + + return [(p, reasons[p]) for p in sorted(reasons.keys())] + + +def iter_system_capture_paths() -> List[Tuple[str, str]]: + out: List[Tuple[str, str]] = [] + seen: Set[str] = set() + for spec, reason in _SYSTEM_CAPTURE_GLOBS: + for path in iter_matching_files(spec): + if path in seen: + continue + seen.add(path) + out.append((path, reason)) + return sorted(out, key=lambda x: x[0]) diff --git a/enroll/systemd.py b/enroll/systemd.py index ae8ce8d..7081001 100644 --- a/enroll/systemd.py +++ b/enroll/systemd.py @@ -33,6 +33,19 @@ def _run(cmd: list[str]) -> str: return p.stdout +@dataclass +class TimerInfo: + name: str + fragment_path: Optional[str] + dropin_paths: List[str] + env_files: List[str] + trigger_unit: Optional[str] + active_state: Optional[str] + sub_state: Optional[str] + unit_file_state: Optional[str] + condition_result: Optional[str] + + def list_enabled_services() -> List[str]: out = _run( [ @@ -58,6 +71,31 @@ def list_enabled_services() -> List[str]: return sorted(set(units)) +def list_enabled_timers() -> List[str]: + out = _run( + [ + "systemctl", + "list-unit-files", + "--type=timer", + "--state=enabled", + "--no-legend", + ] + ) + units: List[str] = [] + for line in out.splitlines(): + parts = line.split() + if not parts: + continue + unit = parts[0].strip() + if not unit.endswith(".timer"): + continue + # Skip template units like "foo@.timer" + if unit.endswith("@.timer"): + continue + units.append(unit) + return sorted(set(units)) + + def get_unit_info(unit: str) -> UnitInfo: p = subprocess.run( [ @@ -117,3 +155,62 @@ def get_unit_info(unit: str) -> UnitInfo: unit_file_state=kv.get("UnitFileState") or None, condition_result=kv.get("ConditionResult") or None, ) + + +def get_timer_info(unit: str) -> TimerInfo: + p = subprocess.run( + [ + "systemctl", + "show", + unit, + "-p", + "FragmentPath", + "-p", + "DropInPaths", + "-p", + "EnvironmentFiles", + "-p", + "Unit", + "-p", + "ActiveState", + "-p", + "SubState", + "-p", + "UnitFileState", + "-p", + "ConditionResult", + ], + text=True, + capture_output=True, + ) # nosec + if p.returncode != 0: + raise RuntimeError(f"systemctl show failed for {unit}: {p.stderr}") + + kv: dict[str, str] = {} + for line in (p.stdout or "").splitlines(): + if "=" in line: + k, v = line.split("=", 1) + kv[k] = v.strip() + + fragment = kv.get("FragmentPath") or None + dropins = [pp for pp in (kv.get("DropInPaths", "") or "").split() if pp] + + env_files: List[str] = [] + for token in (kv.get("EnvironmentFiles", "") or "").split(): + token = token.lstrip("-") + if token: + env_files.append(token) + + trigger = kv.get("Unit") or None + + return TimerInfo( + name=unit, + fragment_path=fragment, + dropin_paths=dropins, + env_files=env_files, + trigger_unit=trigger, + active_state=kv.get("ActiveState") or None, + sub_state=kv.get("SubState") or None, + unit_file_state=kv.get("UnitFileState") or None, + condition_result=kv.get("ConditionResult") or None, + ) diff --git a/enroll/validate.py b/enroll/validate.py new file mode 100644 index 0000000..0a3e8cb --- /dev/null +++ b/enroll/validate.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +import json +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple + +import jsonschema + +from .diff import BundleRef, _bundle_from_input +from .state import load_state + + +@dataclass +class ValidationResult: + errors: List[str] + warnings: List[str] + + @property + def ok(self) -> bool: + return not self.errors + + def to_dict(self) -> Dict[str, Any]: + return { + "ok": self.ok, + "errors": list(self.errors), + "warnings": list(self.warnings), + } + + def to_text(self) -> str: + lines: List[str] = [] + if not self.errors and not self.warnings: + lines.append("OK: harvest bundle validated") + elif not self.errors and self.warnings: + lines.append(f"WARN: {len(self.warnings)} warning(s)") + else: + lines.append(f"ERROR: {len(self.errors)} validation error(s)") + + if self.errors: + lines.append("") + lines.append("Errors:") + for e in self.errors: + lines.append(f"- {e}") + if self.warnings: + lines.append("") + lines.append("Warnings:") + for w in self.warnings: + lines.append(f"- {w}") + return "\n".join(lines) + "\n" + + +def _default_schema_path() -> Path: + # Keep the schema vendored with the codebase so enroll can validate offline. + return Path(__file__).resolve().parent / "schema" / "state.schema.json" + + +def _load_schema(schema: Optional[str]) -> Dict[str, Any]: + """Load a JSON schema. + + If schema is None, load the vendored schema. + If schema begins with http(s)://, fetch it. + Otherwise, treat it as a local file path. + """ + + if not schema: + p = _default_schema_path() + with open(p, "r", encoding="utf-8") as f: + return json.load(f) + + if schema.startswith("http://") or schema.startswith("https://"): + with urllib.request.urlopen(schema, timeout=10) as resp: # nosec + data = resp.read() + return json.loads(data.decode("utf-8")) + + p = Path(schema).expanduser() + with open(p, "r", encoding="utf-8") as f: + return json.load(f) + + +def _json_pointer(err: jsonschema.ValidationError) -> str: + # Build a JSON pointer-ish path that is easy to read. + if err.absolute_path: + parts = [str(p) for p in err.absolute_path] + return "/" + "/".join(parts) + return "/" + + +def _iter_managed_files(state: Dict[str, Any]) -> List[Tuple[str, Dict[str, Any]]]: + """Return (role_name, managed_file_dict) tuples across all roles.""" + + roles = state.get("roles") or {} + out: List[Tuple[str, Dict[str, Any]]] = [] + + # Singleton roles + for rn in [ + "users", + "apt_config", + "dnf_config", + "sysctl", + "etc_custom", + "usr_local_custom", + "extra_paths", + ]: + snap = roles.get(rn) or {} + for mf in snap.get("managed_files") or []: + if isinstance(mf, dict): + out.append((rn, mf)) + + # Array roles + for s in roles.get("services") or []: + if not isinstance(s, dict): + continue + role_name = str(s.get("role_name") or "unknown") + for mf in s.get("managed_files") or []: + if isinstance(mf, dict): + out.append((role_name, mf)) + + for p in roles.get("packages") or []: + if not isinstance(p, dict): + continue + role_name = str(p.get("role_name") or "unknown") + for mf in p.get("managed_files") or []: + if isinstance(mf, dict): + out.append((role_name, mf)) + + return out + + +def validate_harvest( + harvest_input: str, + *, + sops_mode: bool = False, + schema: Optional[str] = None, + no_schema: bool = False, +) -> ValidationResult: + """Validate an enroll harvest bundle. + + Checks: + - state.json parses + - state.json validates against the schema (unless no_schema) + - every managed_file src_rel exists in artifacts// + """ + + errors: List[str] = [] + warnings: List[str] = [] + + bundle: BundleRef = _bundle_from_input(harvest_input, sops_mode=sops_mode) + try: + state_path = bundle.state_path + if not state_path.exists(): + return ValidationResult( + errors=[f"missing state.json at {state_path}"], warnings=[] + ) + + try: + state = load_state(bundle.dir) + except Exception as e: # noqa: BLE001 + return ValidationResult( + errors=[f"failed to parse state.json: {e!r}"], warnings=[] + ) + + if not no_schema: + try: + sch = _load_schema(schema) + validator = jsonschema.Draft202012Validator(sch) + for err in sorted(validator.iter_errors(state), key=str): + ptr = _json_pointer(err) + msg = err.message + errors.append(f"schema {ptr}: {msg}") + except Exception as e: # noqa: BLE001 + errors.append(f"failed to load/validate schema: {e!r}") + + # Artifact existence checks + artifacts_dir = bundle.dir / "artifacts" + referenced: Set[Tuple[str, str]] = set() + for role_name, mf in _iter_managed_files(state): + src_rel = str(mf.get("src_rel") or "") + if not src_rel: + errors.append( + f"managed_file missing src_rel for role {role_name} (path={mf.get('path')!r})" + ) + continue + if src_rel.startswith("/") or ".." in src_rel.split("/"): + errors.append( + f"managed_file has suspicious src_rel for role {role_name}: {src_rel!r}" + ) + continue + + referenced.add((role_name, src_rel)) + p = artifacts_dir / role_name / src_rel + if not p.exists(): + errors.append( + f"missing artifact for role {role_name}: artifacts/{role_name}/{src_rel}" + ) + continue + if not p.is_file(): + errors.append( + f"artifact is not a file for role {role_name}: artifacts/{role_name}/{src_rel}" + ) + + # Runtime firewall snapshots are generated artifacts rather than managed files. + fw = (state.get("roles") or {}).get("firewall_runtime") or {} + if isinstance(fw, dict): + for key in ("ipset_save", "iptables_v4_save", "iptables_v6_save"): + src_rel = str(fw.get(key) or "") + if not src_rel: + continue + if src_rel.startswith("/") or ".." in src_rel.split("/"): + errors.append( + f"firewall_runtime {key} has suspicious src_rel: {src_rel!r}" + ) + continue + referenced.add( + (str(fw.get("role_name") or "firewall_runtime"), src_rel) + ) + p = ( + artifacts_dir + / str(fw.get("role_name") or "firewall_runtime") + / src_rel + ) + if not p.exists(): + errors.append( + "missing firewall runtime artifact: " + f"artifacts/{fw.get('role_name') or 'firewall_runtime'}/{src_rel}" + ) + elif not p.is_file(): + errors.append( + "firewall runtime artifact is not a file: " + f"artifacts/{fw.get('role_name') or 'firewall_runtime'}/{src_rel}" + ) + + # Warn if there are extra files in artifacts not referenced. + if artifacts_dir.exists() and artifacts_dir.is_dir(): + for fp in artifacts_dir.rglob("*"): + if not fp.is_file(): + continue + try: + rel = fp.relative_to(artifacts_dir) + except ValueError: + continue + parts = rel.parts + if len(parts) < 2: + continue + role_name = parts[0] + src_rel = "/".join(parts[1:]) + if (role_name, src_rel) not in referenced: + warnings.append( + f"unreferenced artifact present: artifacts/{role_name}/{src_rel}" + ) + + return ValidationResult(errors=errors, warnings=warnings) + finally: + # Ensure any temp extraction dirs are cleaned up. + if bundle.tempdir is not None: + bundle.tempdir.cleanup() diff --git a/enroll/version.py b/enroll/version.py new file mode 100644 index 0000000..bbe78b6 --- /dev/null +++ b/enroll/version.py @@ -0,0 +1,32 @@ +from __future__ import annotations + + +def get_enroll_version() -> str: + """ + Best-effort version lookup that works when installed via: + - poetry/pip/wheel + - deb/rpm system packages + Falls back to "0+unknown" when running from an unpacked source tree. + """ + try: + from importlib.metadata import ( + packages_distributions, + version, + ) + except Exception: + # Very old Python or unusual environment + return "unknown" + + # Map import package -> dist(s) + dist_names = [] + try: + dist_names = (packages_distributions() or {}).get("enroll", []) or [] + except Exception: + dist_names = [] + + # Try mapped dists first, then a reasonable default + for dist in [*dist_names, "enroll"]: + try: + return version(dist) + except Exception: + return "unknown" diff --git a/poetry.lock b/poetry.lock index a056de4..b338a10 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,136 +1,335 @@ # This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +[[package]] +name = "attrs" +version = "26.1.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.9" +files = [ + {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, + {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, +] + +[[package]] +name = "bcrypt" +version = "5.0.0" +description = "Modern password hashing for your software and your servers" +optional = false +python-versions = ">=3.8" +files = [ + {file = "bcrypt-5.0.0-cp313-cp313t-macosx_10_12_universal2.whl", hash = "sha256:f3c08197f3039bec79cee59a606d62b96b16669cff3949f21e74796b6e3cd2be"}, + {file = "bcrypt-5.0.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:200af71bc25f22006f4069060c88ed36f8aa4ff7f53e67ff04d2ab3f1e79a5b2"}, + {file = "bcrypt-5.0.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:baade0a5657654c2984468efb7d6c110db87ea63ef5a4b54732e7e337253e44f"}, + {file = "bcrypt-5.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:c58b56cdfb03202b3bcc9fd8daee8e8e9b6d7e3163aa97c631dfcfcc24d36c86"}, + {file = "bcrypt-5.0.0-cp313-cp313t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4bfd2a34de661f34d0bda43c3e4e79df586e4716ef401fe31ea39d69d581ef23"}, + {file = "bcrypt-5.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:ed2e1365e31fc73f1825fa830f1c8f8917ca1b3ca6185773b349c20fd606cec2"}, + {file = "bcrypt-5.0.0-cp313-cp313t-manylinux_2_34_aarch64.whl", hash = "sha256:83e787d7a84dbbfba6f250dd7a5efd689e935f03dd83b0f919d39349e1f23f83"}, + {file = "bcrypt-5.0.0-cp313-cp313t-manylinux_2_34_x86_64.whl", hash = "sha256:137c5156524328a24b9fac1cb5db0ba618bc97d11970b39184c1d87dc4bf1746"}, + {file = "bcrypt-5.0.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:38cac74101777a6a7d3b3e3cfefa57089b5ada650dce2baf0cbdd9d65db22a9e"}, + {file = "bcrypt-5.0.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:d8d65b564ec849643d9f7ea05c6d9f0cd7ca23bdd4ac0c2dbef1104ab504543d"}, + {file = "bcrypt-5.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:741449132f64b3524e95cd30e5cd3343006ce146088f074f31ab26b94e6c75ba"}, + {file = "bcrypt-5.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:212139484ab3207b1f0c00633d3be92fef3c5f0af17cad155679d03ff2ee1e41"}, + {file = "bcrypt-5.0.0-cp313-cp313t-win32.whl", hash = "sha256:9d52ed507c2488eddd6a95bccee4e808d3234fa78dd370e24bac65a21212b861"}, + {file = "bcrypt-5.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f6984a24db30548fd39a44360532898c33528b74aedf81c26cf29c51ee47057e"}, + {file = "bcrypt-5.0.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9fffdb387abe6aa775af36ef16f55e318dcda4194ddbf82007a6f21da29de8f5"}, + {file = "bcrypt-5.0.0-cp314-cp314t-macosx_10_12_universal2.whl", hash = "sha256:4870a52610537037adb382444fefd3706d96d663ac44cbb2f37e3919dca3d7ef"}, + {file = "bcrypt-5.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48f753100931605686f74e27a7b49238122aa761a9aefe9373265b8b7aa43ea4"}, + {file = "bcrypt-5.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f70aadb7a809305226daedf75d90379c397b094755a710d7014b8b117df1ebbf"}, + {file = "bcrypt-5.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:744d3c6b164caa658adcb72cb8cc9ad9b4b75c7db507ab4bc2480474a51989da"}, + {file = "bcrypt-5.0.0-cp314-cp314t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a28bc05039bdf3289d757f49d616ab3efe8cf40d8e8001ccdd621cd4f98f4fc9"}, + {file = "bcrypt-5.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:7f277a4b3390ab4bebe597800a90da0edae882c6196d3038a73adf446c4f969f"}, + {file = "bcrypt-5.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:79cfa161eda8d2ddf29acad370356b47f02387153b11d46042e93a0a95127493"}, + {file = "bcrypt-5.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:a5393eae5722bcef046a990b84dff02b954904c36a194f6cfc817d7dca6c6f0b"}, + {file = "bcrypt-5.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f4c94dec1b5ab5d522750cb059bb9409ea8872d4494fd152b53cca99f1ddd8c"}, + {file = "bcrypt-5.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0cae4cb350934dfd74c020525eeae0a5f79257e8a201c0c176f4b84fdbf2a4b4"}, + {file = "bcrypt-5.0.0-cp314-cp314t-win32.whl", hash = "sha256:b17366316c654e1ad0306a6858e189fc835eca39f7eb2cafd6aaca8ce0c40a2e"}, + {file = "bcrypt-5.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:92864f54fb48b4c718fc92a32825d0e42265a627f956bc0361fe869f1adc3e7d"}, + {file = "bcrypt-5.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dd19cf5184a90c873009244586396a6a884d591a5323f0e8a5922560718d4993"}, + {file = "bcrypt-5.0.0-cp38-abi3-macosx_10_12_universal2.whl", hash = "sha256:fc746432b951e92b58317af8e0ca746efe93e66555f1b40888865ef5bf56446b"}, + {file = "bcrypt-5.0.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c2388ca94ffee269b6038d48747f4ce8df0ffbea43f31abfa18ac72f0218effb"}, + {file = "bcrypt-5.0.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:560ddb6ec730386e7b3b26b8b4c88197aaed924430e7b74666a586ac997249ef"}, + {file = "bcrypt-5.0.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d79e5c65dcc9af213594d6f7f1fa2c98ad3fc10431e7aa53c176b441943efbdd"}, + {file = "bcrypt-5.0.0-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2b732e7d388fa22d48920baa267ba5d97cca38070b69c0e2d37087b381c681fd"}, + {file = "bcrypt-5.0.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0c8e093ea2532601a6f686edbc2c6b2ec24131ff5c52f7610dd64fa4553b5464"}, + {file = "bcrypt-5.0.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5b1589f4839a0899c146e8892efe320c0fa096568abd9b95593efac50a87cb75"}, + {file = "bcrypt-5.0.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:89042e61b5e808b67daf24a434d89bab164d4de1746b37a8d173b6b14f3db9ff"}, + {file = "bcrypt-5.0.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:e3cf5b2560c7b5a142286f69bde914494b6d8f901aaa71e453078388a50881c4"}, + {file = "bcrypt-5.0.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f632fd56fc4e61564f78b46a2269153122db34988e78b6be8b32d28507b7eaeb"}, + {file = "bcrypt-5.0.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:801cad5ccb6b87d1b430f183269b94c24f248dddbbc5c1f78b6ed231743e001c"}, + {file = "bcrypt-5.0.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3cf67a804fc66fc217e6914a5635000259fbbbb12e78a99488e4d5ba445a71eb"}, + {file = "bcrypt-5.0.0-cp38-abi3-win32.whl", hash = "sha256:3abeb543874b2c0524ff40c57a4e14e5d3a66ff33fb423529c88f180fd756538"}, + {file = "bcrypt-5.0.0-cp38-abi3-win_amd64.whl", hash = "sha256:35a77ec55b541e5e583eb3436ffbbf53b0ffa1fa16ca6782279daf95d146dcd9"}, + {file = "bcrypt-5.0.0-cp38-abi3-win_arm64.whl", hash = "sha256:cde08734f12c6a4e28dc6755cd11d3bdfea608d93d958fffbe95a7026ebe4980"}, + {file = "bcrypt-5.0.0-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:0c418ca99fd47e9c59a301744d63328f17798b5947b0f791e9af3c1c499c2d0a"}, + {file = "bcrypt-5.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddb4e1500f6efdd402218ffe34d040a1196c072e07929b9820f363a1fd1f4191"}, + {file = "bcrypt-5.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7aeef54b60ceddb6f30ee3db090351ecf0d40ec6e2abf41430997407a46d2254"}, + {file = "bcrypt-5.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f0ce778135f60799d89c9693b9b398819d15f1921ba15fe719acb3178215a7db"}, + {file = "bcrypt-5.0.0-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a71f70ee269671460b37a449f5ff26982a6f2ba493b3eabdd687b4bf35f875ac"}, + {file = "bcrypt-5.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f8429e1c410b4073944f03bd778a9e066e7fad723564a52ff91841d278dfc822"}, + {file = "bcrypt-5.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:edfcdcedd0d0f05850c52ba3127b1fce70b9f89e0fe5ff16517df7e81fa3cbb8"}, + {file = "bcrypt-5.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:611f0a17aa4a25a69362dcc299fda5c8a3d4f160e2abb3831041feb77393a14a"}, + {file = "bcrypt-5.0.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:db99dca3b1fdc3db87d7c57eac0c82281242d1eabf19dcb8a6b10eb29a2e72d1"}, + {file = "bcrypt-5.0.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:5feebf85a9cefda32966d8171f5db7e3ba964b77fdfe31919622256f80f9cf42"}, + {file = "bcrypt-5.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3ca8a166b1140436e058298a34d88032ab62f15aae1c598580333dc21d27ef10"}, + {file = "bcrypt-5.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:61afc381250c3182d9078551e3ac3a41da14154fbff647ddf52a769f588c4172"}, + {file = "bcrypt-5.0.0-cp39-abi3-win32.whl", hash = "sha256:64d7ce196203e468c457c37ec22390f1a61c85c6f0b8160fd752940ccfb3a683"}, + {file = "bcrypt-5.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:64ee8434b0da054d830fa8e89e1c8bf30061d539044a39524ff7dec90481e5c2"}, + {file = "bcrypt-5.0.0-cp39-abi3-win_arm64.whl", hash = "sha256:f2347d3534e76bf50bca5500989d6c1d05ed64b440408057a37673282c654927"}, + {file = "bcrypt-5.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7edda91d5ab52b15636d9c30da87d2cc84f426c72b9dba7a9b4fe142ba11f534"}, + {file = "bcrypt-5.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:046ad6db88edb3c5ece4369af997938fb1c19d6a699b9c1b27b0db432faae4c4"}, + {file = "bcrypt-5.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:dcd58e2b3a908b5ecc9b9df2f0085592506ac2d5110786018ee5e160f28e0911"}, + {file = "bcrypt-5.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:6b8f520b61e8781efee73cba14e3e8c9556ccfb375623f4f97429544734545b4"}, + {file = "bcrypt-5.0.0.tar.gz", hash = "sha256:f748f7c2d6fd375cc93d3fba7ef4a9e3a092421b8dbf34d8d4dc06be9492dfdd"}, +] + +[package.extras] +tests = ["pytest (>=3.2.1,!=3.3.0)"] +typecheck = ["mypy"] + [[package]] name = "certifi" -version = "2025.11.12" +version = "2026.4.22" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" files = [ - {file = "certifi-2025.11.12-py3-none-any.whl", hash = "sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b"}, - {file = "certifi-2025.11.12.tar.gz", hash = "sha256:d8ab5478f2ecd78af242878415affce761ca6bc54a22a27e026d7c25357c3316"}, + {file = "certifi-2026.4.22-py3-none-any.whl", hash = "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a"}, + {file = "certifi-2026.4.22.tar.gz", hash = "sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580"}, ] +[[package]] +name = "cffi" +version = "2.0.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.9" +files = [ + {file = "cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44"}, + {file = "cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb"}, + {file = "cffi-2.0.0-cp310-cp310-win32.whl", hash = "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a"}, + {file = "cffi-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739"}, + {file = "cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe"}, + {file = "cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743"}, + {file = "cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5"}, + {file = "cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5"}, + {file = "cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d"}, + {file = "cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d"}, + {file = "cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba"}, + {file = "cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94"}, + {file = "cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187"}, + {file = "cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18"}, + {file = "cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5"}, + {file = "cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6"}, + {file = "cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb"}, + {file = "cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26"}, + {file = "cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c"}, + {file = "cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b"}, + {file = "cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27"}, + {file = "cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75"}, + {file = "cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91"}, + {file = "cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5"}, + {file = "cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775"}, + {file = "cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205"}, + {file = "cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1"}, + {file = "cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f"}, + {file = "cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25"}, + {file = "cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad"}, + {file = "cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9"}, + {file = "cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592"}, + {file = "cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512"}, + {file = "cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4"}, + {file = "cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e"}, + {file = "cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6"}, + {file = "cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9"}, + {file = "cffi-2.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf"}, + {file = "cffi-2.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322"}, + {file = "cffi-2.0.0-cp39-cp39-win32.whl", hash = "sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a"}, + {file = "cffi-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9"}, + {file = "cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529"}, +] + +[package.dependencies] +pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} + [[package]] name = "charset-normalizer" -version = "3.4.4" +version = "3.4.7" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" files = [ - {file = "charset_normalizer-3.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-win32.whl", hash = "sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-win_arm64.whl", hash = "sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-win32.whl", hash = "sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-win32.whl", hash = "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-win32.whl", hash = "sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-win32.whl", hash = "sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-win_arm64.whl", hash = "sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50"}, - {file = "charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f"}, - {file = "charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-win32.whl", hash = "sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-win_amd64.whl", hash = "sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-win_arm64.whl", hash = "sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-win32.whl", hash = "sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-win_amd64.whl", hash = "sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-win_arm64.whl", hash = "sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e5f4d355f0a2b1a31bc3edec6795b46324349c9cb25eed068049e4f472fb4259"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16d971e29578a5e97d7117866d15889a4a07befe0e87e703ed63cd90cb348c01"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dca4bbc466a95ba9c0234ef56d7dd9509f63da22274589ebd4ed7f1f4d4c54e3"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e80c8378d8f3d83cd3164da1ad2df9e37a666cdde7b1cb2298ed0b558064be30"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36836d6ff945a00b88ba1e4572d721e60b5b8c98c155d465f56ad19d68f23734"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux_2_31_armv7l.whl", hash = "sha256:bd9b23791fe793e4968dba0c447e12f78e425c59fc0e3b97f6450f4781f3ee60"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aef65cd602a6d0e0ff6f9930fcb1c8fec60dd2cfcb6facaf4bdb0e5873042db0"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:82b271f5137d07749f7bf32f70b17ab6eaabedd297e75dce75081a24f76eb545"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:1efde3cae86c8c273f1eb3b287be7d8499420cf2fe7585c41d370d3e790054a5"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:c593052c465475e64bbfe5dbd81680f64a67fdc752c56d7a0ae205dc8aeefe0f"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:af21eb4409a119e365397b2adbaca4c9ccab56543a65d5dbd9f920d6ac29f686"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:84c018e49c3bf790f9c2771c45e9313a08c2c2a6342b162cd650258b57817706"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dd915403e231e6b1809fe9b6d9fc55cf8fb5e02765ac625d9cd623342a7905d7"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-win32.whl", hash = "sha256:320ade88cfb846b8cd6b4ddf5ee9e80ee0c1f52401f2456b84ae1ae6a1a5f207"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-win_amd64.whl", hash = "sha256:1dc8b0ea451d6e69735094606991f32867807881400f808a106ee1d963c46a83"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:177a0ba5f0211d488e295aaf82707237e331c24788d8d76c96c5a41594723217"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e0d51f618228538a3e8f46bd246f87a6cd030565e015803691603f55e12afb5"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:14265bfe1f09498b9d8ec91e9ec9fa52775edf90fcbde092b25f4a33d444fea9"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87fad7d9ba98c86bcb41b2dc8dbb326619be2562af1f8ff50776a39e55721c5a"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f22dec1690b584cea26fade98b2435c132c1b5f68e39f5a0b7627cd7ae31f1dc"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:d61f00a0869d77422d9b2aba989e2d24afa6ffd552af442e0e58de4f35ea6d00"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6370e8686f662e6a3941ee48ed4742317cafbe5707e36406e9df792cdb535776"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:a6c5863edfbe888d9eff9c8b8087354e27618d9da76425c119293f11712a6319"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ed065083d0898c9d5b4bbec7b026fd755ff7454e6e8b73a67f8c744b13986e24"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:2cd4a60d0e2fb04537162c62bbbb4182f53541fe0ede35cdf270a1c1e723cc42"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:813c0e0132266c08eb87469a642cb30aaff57c5f426255419572aaeceeaa7bf4"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:07d9e39b01743c3717745f4c530a6349eadbfa043c7577eef86c502c15df2c67"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c0f081d69a6e58272819b70288d3221a6ee64b98df852631c80f293514d3b274"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-win32.whl", hash = "sha256:8751d2787c9131302398b11e6c8068053dcb55d5a8964e114b6e196cf16cb366"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-win_amd64.whl", hash = "sha256:12a6fff75f6bc66711b73a2f0addfc4c8c15a20e805146a02d147a318962c444"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-win_arm64.whl", hash = "sha256:bb8cc7534f51d9a017b93e3e85b260924f909601c3df002bcdb58ddb4dc41a5c"}, + {file = "charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d"}, + {file = "charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5"}, ] [[package]] @@ -146,103 +345,117 @@ files = [ [[package]] name = "coverage" -version = "7.13.0" +version = "7.14.0" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.10" files = [ - {file = "coverage-7.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:02d9fb9eccd48f6843c98a37bd6817462f130b86da8660461e8f5e54d4c06070"}, - {file = "coverage-7.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:367449cf07d33dc216c083f2036bb7d976c6e4903ab31be400ad74ad9f85ce98"}, - {file = "coverage-7.13.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cdb3c9f8fef0a954c632f64328a3935988d33a6604ce4bf67ec3e39670f12ae5"}, - {file = "coverage-7.13.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d10fd186aac2316f9bbb46ef91977f9d394ded67050ad6d84d94ed6ea2e8e54e"}, - {file = "coverage-7.13.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f88ae3e69df2ab62fb0bc5219a597cb890ba5c438190ffa87490b315190bb33"}, - {file = "coverage-7.13.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c4be718e51e86f553bcf515305a158a1cd180d23b72f07ae76d6017c3cc5d791"}, - {file = "coverage-7.13.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a00d3a393207ae12f7c49bb1c113190883b500f48979abb118d8b72b8c95c032"}, - {file = "coverage-7.13.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3a7b1cd820e1b6116f92c6128f1188e7afe421c7e1b35fa9836b11444e53ebd9"}, - {file = "coverage-7.13.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:37eee4e552a65866f15dedd917d5e5f3d59805994260720821e2c1b51ac3248f"}, - {file = "coverage-7.13.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:62d7c4f13102148c78d7353c6052af6d899a7f6df66a32bddcc0c0eb7c5326f8"}, - {file = "coverage-7.13.0-cp310-cp310-win32.whl", hash = "sha256:24e4e56304fdb56f96f80eabf840eab043b3afea9348b88be680ec5986780a0f"}, - {file = "coverage-7.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:74c136e4093627cf04b26a35dab8cbfc9b37c647f0502fc313376e11726ba303"}, - {file = "coverage-7.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0dfa3855031070058add1a59fdfda0192fd3e8f97e7c81de0596c145dea51820"}, - {file = "coverage-7.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4fdb6f54f38e334db97f72fa0c701e66d8479af0bc3f9bfb5b90f1c30f54500f"}, - {file = "coverage-7.13.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7e442c013447d1d8d195be62852270b78b6e255b79b8675bad8479641e21fd96"}, - {file = "coverage-7.13.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1ed5630d946859de835a85e9a43b721123a8a44ec26e2830b296d478c7fd4259"}, - {file = "coverage-7.13.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f15a931a668e58087bc39d05d2b4bf4b14ff2875b49c994bbdb1c2217a8daeb"}, - {file = "coverage-7.13.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:30a3a201a127ea57f7e14ba43c93c9c4be8b7d17a26e03bb49e6966d019eede9"}, - {file = "coverage-7.13.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7a485ff48fbd231efa32d58f479befce52dcb6bfb2a88bb7bf9a0b89b1bc8030"}, - {file = "coverage-7.13.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:22486cdafba4f9e471c816a2a5745337742a617fef68e890d8baf9f3036d7833"}, - {file = "coverage-7.13.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:263c3dbccc78e2e331e59e90115941b5f53e85cfcc6b3b2fbff1fd4e3d2c6ea8"}, - {file = "coverage-7.13.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e5330fa0cc1f5c3c4c3bb8e101b742025933e7848989370a1d4c8c5e401ea753"}, - {file = "coverage-7.13.0-cp311-cp311-win32.whl", hash = "sha256:0f4872f5d6c54419c94c25dd6ae1d015deeb337d06e448cd890a1e89a8ee7f3b"}, - {file = "coverage-7.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:51a202e0f80f241ccb68e3e26e19ab5b3bf0f813314f2c967642f13ebcf1ddfe"}, - {file = "coverage-7.13.0-cp311-cp311-win_arm64.whl", hash = "sha256:d2a9d7f1c11487b1c69367ab3ac2d81b9b3721f097aa409a3191c3e90f8f3dd7"}, - {file = "coverage-7.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0b3d67d31383c4c68e19a88e28fc4c2e29517580f1b0ebec4a069d502ce1e0bf"}, - {file = "coverage-7.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:581f086833d24a22c89ae0fe2142cfaa1c92c930adf637ddf122d55083fb5a0f"}, - {file = "coverage-7.13.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0a3a30f0e257df382f5f9534d4ce3d4cf06eafaf5192beb1a7bd066cb10e78fb"}, - {file = "coverage-7.13.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:583221913fbc8f53b88c42e8dbb8fca1d0f2e597cb190ce45916662b8b9d9621"}, - {file = "coverage-7.13.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f5d9bd30756fff3e7216491a0d6d520c448d5124d3d8e8f56446d6412499e74"}, - {file = "coverage-7.13.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a23e5a1f8b982d56fa64f8e442e037f6ce29322f1f9e6c2344cd9e9f4407ee57"}, - {file = "coverage-7.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9b01c22bc74a7fb44066aaf765224c0d933ddf1f5047d6cdfe4795504a4493f8"}, - {file = "coverage-7.13.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:898cce66d0836973f48dda4e3514d863d70142bdf6dfab932b9b6a90ea5b222d"}, - {file = "coverage-7.13.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:3ab483ea0e251b5790c2aac03acde31bff0c736bf8a86829b89382b407cd1c3b"}, - {file = "coverage-7.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1d84e91521c5e4cb6602fe11ece3e1de03b2760e14ae4fcf1a4b56fa3c801fcd"}, - {file = "coverage-7.13.0-cp312-cp312-win32.whl", hash = "sha256:193c3887285eec1dbdb3f2bd7fbc351d570ca9c02ca756c3afbc71b3c98af6ef"}, - {file = "coverage-7.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:4f3e223b2b2db5e0db0c2b97286aba0036ca000f06aca9b12112eaa9af3d92ae"}, - {file = "coverage-7.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:086cede306d96202e15a4b77ace8472e39d9f4e5f9fd92dd4fecdfb2313b2080"}, - {file = "coverage-7.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:28ee1c96109974af104028a8ef57cec21447d42d0e937c0275329272e370ebcf"}, - {file = "coverage-7.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d1e97353dcc5587b85986cda4ff3ec98081d7e84dd95e8b2a6d59820f0545f8a"}, - {file = "coverage-7.13.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:99acd4dfdfeb58e1937629eb1ab6ab0899b131f183ee5f23e0b5da5cba2fec74"}, - {file = "coverage-7.13.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ff45e0cd8451e293b63ced93161e189780baf444119391b3e7d25315060368a6"}, - {file = "coverage-7.13.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f4f72a85316d8e13234cafe0a9f81b40418ad7a082792fa4165bd7d45d96066b"}, - {file = "coverage-7.13.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:11c21557d0e0a5a38632cbbaca5f008723b26a89d70db6315523df6df77d6232"}, - {file = "coverage-7.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:76541dc8d53715fb4f7a3a06b34b0dc6846e3c69bc6204c55653a85dd6220971"}, - {file = "coverage-7.13.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6e9e451dee940a86789134b6b0ffbe31c454ade3b849bb8a9d2cca2541a8e91d"}, - {file = "coverage-7.13.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5c67dace46f361125e6b9cace8fe0b729ed8479f47e70c89b838d319375c8137"}, - {file = "coverage-7.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f59883c643cb19630500f57016f76cfdcd6845ca8c5b5ea1f6e17f74c8e5f511"}, - {file = "coverage-7.13.0-cp313-cp313-win32.whl", hash = "sha256:58632b187be6f0be500f553be41e277712baa278147ecb7559983c6d9faf7ae1"}, - {file = "coverage-7.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:73419b89f812f498aca53f757dd834919b48ce4799f9d5cad33ca0ae442bdb1a"}, - {file = "coverage-7.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:eb76670874fdd6091eedcc856128ee48c41a9bbbb9c3f1c7c3cf169290e3ffd6"}, - {file = "coverage-7.13.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6e63ccc6e0ad8986386461c3c4b737540f20426e7ec932f42e030320896c311a"}, - {file = "coverage-7.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:494f5459ffa1bd45e18558cd98710c36c0b8fbfa82a5eabcbe671d80ecffbfe8"}, - {file = "coverage-7.13.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:06cac81bf10f74034e055e903f5f946e3e26fc51c09fc9f584e4a1605d977053"}, - {file = "coverage-7.13.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f2ffc92b46ed6e6760f1d47a71e56b5664781bc68986dbd1836b2b70c0ce2071"}, - {file = "coverage-7.13.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0602f701057c6823e5db1b74530ce85f17c3c5be5c85fc042ac939cbd909426e"}, - {file = "coverage-7.13.0-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:25dc33618d45456ccb1d37bce44bc78cf269909aa14c4db2e03d63146a8a1493"}, - {file = "coverage-7.13.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:71936a8b3b977ddd0b694c28c6a34f4fff2e9dd201969a4ff5d5fc7742d614b0"}, - {file = "coverage-7.13.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:936bc20503ce24770c71938d1369461f0c5320830800933bc3956e2a4ded930e"}, - {file = "coverage-7.13.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:af0a583efaacc52ae2521f8d7910aff65cdb093091d76291ac5820d5e947fc1c"}, - {file = "coverage-7.13.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f1c23e24a7000da892a312fb17e33c5f94f8b001de44b7cf8ba2e36fbd15859e"}, - {file = "coverage-7.13.0-cp313-cp313t-win32.whl", hash = "sha256:5f8a0297355e652001015e93be345ee54393e45dc3050af4a0475c5a2b767d46"}, - {file = "coverage-7.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6abb3a4c52f05e08460bd9acf04fec027f8718ecaa0d09c40ffbc3fbd70ecc39"}, - {file = "coverage-7.13.0-cp313-cp313t-win_arm64.whl", hash = "sha256:3ad968d1e3aa6ce5be295ab5fe3ae1bf5bb4769d0f98a80a0252d543a2ef2e9e"}, - {file = "coverage-7.13.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:453b7ec753cf5e4356e14fe858064e5520c460d3bbbcb9c35e55c0d21155c256"}, - {file = "coverage-7.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af827b7cbb303e1befa6c4f94fd2bf72f108089cfa0f8abab8f4ca553cf5ca5a"}, - {file = "coverage-7.13.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9987a9e4f8197a1000280f7cc089e3ea2c8b3c0a64d750537809879a7b4ceaf9"}, - {file = "coverage-7.13.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3188936845cd0cb114fa6a51842a304cdbac2958145d03be2377ec41eb285d19"}, - {file = "coverage-7.13.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2bdb3babb74079f021696cb46b8bb5f5661165c385d3a238712b031a12355be"}, - {file = "coverage-7.13.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7464663eaca6adba4175f6c19354feea61ebbdd735563a03d1e472c7072d27bb"}, - {file = "coverage-7.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8069e831f205d2ff1f3d355e82f511eb7c5522d7d413f5db5756b772ec8697f8"}, - {file = "coverage-7.13.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:6fb2d5d272341565f08e962cce14cdf843a08ac43bd621783527adb06b089c4b"}, - {file = "coverage-7.13.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5e70f92ef89bac1ac8a99b3324923b4749f008fdbd7aa9cb35e01d7a284a04f9"}, - {file = "coverage-7.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4b5de7d4583e60d5fd246dd57fcd3a8aa23c6e118a8c72b38adf666ba8e7e927"}, - {file = "coverage-7.13.0-cp314-cp314-win32.whl", hash = "sha256:a6c6e16b663be828a8f0b6c5027d36471d4a9f90d28444aa4ced4d48d7d6ae8f"}, - {file = "coverage-7.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:0900872f2fdb3ee5646b557918d02279dc3af3dfb39029ac4e945458b13f73bc"}, - {file = "coverage-7.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:3a10260e6a152e5f03f26db4a407c4c62d3830b9af9b7c0450b183615f05d43b"}, - {file = "coverage-7.13.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9097818b6cc1cfb5f174e3263eba4a62a17683bcfe5c4b5d07f4c97fa51fbf28"}, - {file = "coverage-7.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0018f73dfb4301a89292c73be6ba5f58722ff79f51593352759c1790ded1cabe"}, - {file = "coverage-7.13.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:166ad2a22ee770f5656e1257703139d3533b4a0b6909af67c6b4a3adc1c98657"}, - {file = "coverage-7.13.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f6aaef16d65d1787280943f1c8718dc32e9cf141014e4634d64446702d26e0ff"}, - {file = "coverage-7.13.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e999e2dcc094002d6e2c7bbc1fb85b58ba4f465a760a8014d97619330cdbbbf3"}, - {file = "coverage-7.13.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:00c3d22cf6fb1cf3bf662aaaa4e563be8243a5ed2630339069799835a9cc7f9b"}, - {file = "coverage-7.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:22ccfe8d9bb0d6134892cbe1262493a8c70d736b9df930f3f3afae0fe3ac924d"}, - {file = "coverage-7.13.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:9372dff5ea15930fea0445eaf37bbbafbc771a49e70c0aeed8b4e2c2614cc00e"}, - {file = "coverage-7.13.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:69ac2c492918c2461bc6ace42d0479638e60719f2a4ef3f0815fa2df88e9f940"}, - {file = "coverage-7.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:739c6c051a7540608d097b8e13c76cfa85263ced467168dc6b477bae3df7d0e2"}, - {file = "coverage-7.13.0-cp314-cp314t-win32.whl", hash = "sha256:fe81055d8c6c9de76d60c94ddea73c290b416e061d40d542b24a5871bad498b7"}, - {file = "coverage-7.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:445badb539005283825959ac9fa4a28f712c214b65af3a2c464f1adc90f5fcbc"}, - {file = "coverage-7.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:de7f6748b890708578fc4b7bb967d810aeb6fcc9bff4bb77dbca77dab2f9df6a"}, - {file = "coverage-7.13.0-py3-none-any.whl", hash = "sha256:850d2998f380b1e266459ca5b47bc9e7daf9af1d070f66317972f382d46f1904"}, - {file = "coverage-7.13.0.tar.gz", hash = "sha256:a394aa27f2d7ff9bc04cf703817773a59ad6dfbd577032e690f961d2460ee936"}, + {file = "coverage-7.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:84c32d90bf4537f0e7b4dec9aaa9a938fb8205136b9d2ecf4d7629d5262dc075"}, + {file = "coverage-7.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7c843572c605ab51cfdb5c6b5f2586e2a8467c0d28eca4bdef4ec70c5fecbd82"}, + {file = "coverage-7.14.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0c451757d3fa2603354fdc789b5e58a0e327a117c370a40e3476ba4eabab228c"}, + {file = "coverage-7.14.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3fd43f0616e765ab78d069cf8358def7363957a45cee446d65c502dcfeea7893"}, + {file = "coverage-7.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:731e535b1498b27d13594a0527a79b0510867b0ad891532be41cb883f2128e20"}, + {file = "coverage-7.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c7492f2d493b976941c7ca050f273cbda2f43c381124f7586a3e3c16d1804fec"}, + {file = "coverage-7.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dc38367eaa2abb1b766ac333142bce7655335a73537f5c8b75aaa89c2b987757"}, + {file = "coverage-7.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0a951308cde22cf77f953955a754d04dccb57fe3bb8e345d685778ed9fc1632a"}, + {file = "coverage-7.14.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fab3877e4ebb06bd9d4d4d00ee53309ee5478e66873c66a382272e3ee33eb7ea"}, + {file = "coverage-7.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:b812eb847b19876ebf33fb6c4f11819af05ab6050b0bfa1bc53412ae81779adb"}, + {file = "coverage-7.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d9c8ef6ed820c433de075657d72dda1f89a2984955e58b8a75feb3f184250218"}, + {file = "coverage-7.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d128b1bba9361fbaaf6a19e179e6cfd6a9103ce0c0555876f72780acc93efd85"}, + {file = "coverage-7.14.0-cp310-cp310-win32.whl", hash = "sha256:65f267ca1370726ec2c1aa38bbe4df9a71a740f22878d2d4bf59d71a4cd8d323"}, + {file = "coverage-7.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:b34ece8065914f938ed7f2c5872bb865336977a52919149846eac3744327267a"}, + {file = "coverage-7.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6a78e2a9d9c5e3b8d4ab9b9d28c985ea66fced0a7d7c2aec1f216e03a2011480"}, + {file = "coverage-7.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a1816c505187592dcd1c5a5f226601a549f70365fbd00930ac88b0c225b76bb4"}, + {file = "coverage-7.14.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d8e1762f0e9cbc26ec315471e7b47855218e833cd5a032d706fbf43845d878c7"}, + {file = "coverage-7.14.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9336e23e8bb3a3925398261385e2a1533957d3e760e91070dcb0e98bfa514eed"}, + {file = "coverage-7.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd1169b2230f9cbe9c638ba38022ed7a2b1e641cc07f7cea0365e4be2a74980"}, + {file = "coverage-7.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d1bb3543b58fea74d2cd1abc4054cc927e4724687cb4560cd2ed88d2c7d820c0"}, + {file = "coverage-7.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a93bac2cb577ef60074999ed56d8a1535894398e2ed920d4185c3ec0c8864742"}, + {file = "coverage-7.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5904abf7e18cddc463219b17552229650c6b79e061d31a1059283051169cf7d5"}, + {file = "coverage-7.14.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:741f57cddc9004a8c81b084660215f33a6b597dbe62c31386b983ee26310e327"}, + {file = "coverage-7.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:664123feb0929d7affc135717dbd70d61d98688a08ab1e5ba464739620c6252d"}, + {file = "coverage-7.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:c83d2399a51bbec8429266905d33616f04bc5726b1138c35844d5fcd896b2e20"}, + {file = "coverage-7.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb2e855b87321259a037429288ae85216d191c74de3e79bf57cd2bc0761992c"}, + {file = "coverage-7.14.0-cp311-cp311-win32.whl", hash = "sha256:731dc15b385ac52289743d476245b61e1a2927e803bef655b52bc3b2a75a21f3"}, + {file = "coverage-7.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:bfb0ed8ec5d25e93face268115d7964db9df8b9aae8edcde9ec6b16c726a7cc1"}, + {file = "coverage-7.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:7ebb1c6df9f78046a1b1e0a89674cd4bf73b7c648914eebcf976a57fd99a5627"}, + {file = "coverage-7.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7ffd19fc8aed057fd686a17a4935eef5f9859d69208f96310e893e64b9b6ccf5"}, + {file = "coverage-7.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:829994cfe1aeb773ca27bf246d4badc1e764893e3bfb98fff820fcecd1ca4662"}, + {file = "coverage-7.14.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:b4f07cf7edcb7ec39431a5074d7ea83b29a9f71fcfc494f0f40af4e65180420f"}, + {file = "coverage-7.14.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ca3d9cf2c32b521bd9518385608787fa86f38daf993695307531822c3430ed67"}, + {file = "coverage-7.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92af52828e7f29d827346b0294e5a0853fa206db77db0395b282918d41e28db9"}, + {file = "coverage-7.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7b2bb6c9d7e769360d0f20a0f219603fd64f0c8f97de17ab25853261602be0fb"}, + {file = "coverage-7.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1c9ed6ef99f88fb8c14aa8e2bf8eb0fe55fa2edfea68f8675d78741df1a5ac0e"}, + {file = "coverage-7.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8231ade007f37959fbf58acc677f26b922c02eda6f0428ea307da0fd39681bf3"}, + {file = "coverage-7.14.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d8b013632cc1ce1d09dbe4f32667b4d320ec2f54fc326ebeffcd0b0bcc2bb6c4"}, + {file = "coverage-7.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1733198802d71ec4c524f322e2867ee05c62e9e75df86bdca545407a221827d1"}, + {file = "coverage-7.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:72a305291fa8ee01332f1aaf38b348ca34097f6aa0b0ef627eef2837e57bbba5"}, + {file = "coverage-7.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fcaba850dd317c65423a9d63d88f9573c53b00354d6dd95724576cc98a131595"}, + {file = "coverage-7.14.0-cp312-cp312-win32.whl", hash = "sha256:5ac83957a80d0701310e96d8bec68cdcf4f90a7674b7d13f15a344315b41ab27"}, + {file = "coverage-7.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:70390b0da32cb90b501953716302906e8bcce087cb283e70d8c97729f22e92b2"}, + {file = "coverage-7.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:91b993743d959b8be85b4abf9d5478216a69329c321efe5be0433c1a841d691d"}, + {file = "coverage-7.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f2bbb8254370eb4c628ff3d6fa8a7f74ddc40565394d4f7ab791d1fe568e37ef"}, + {file = "coverage-7.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:23b81107f46d3f21d0cbce30664fcec0f5d9f585638a67081750f99738f6bf66"}, + {file = "coverage-7.14.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:22a7e06a5f11a757cdfe79018e9095f9f69ae283c5cd8123774c788deec8717b"}, + {file = "coverage-7.14.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9d1aa57a1dc8e05bdc42e81c5d671d849577aeedf279f4c449d6d286f9ed88ca"}, + {file = "coverage-7.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90c1a51bcfddf645b3bb7ec333d9e94393a8e94f55642380fa8a9a5a9e636cb7"}, + {file = "coverage-7.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a841fae2fadcae4f438d43b6ccc4aac2ad609f47cdb6cfdce60cbb3fe5ca7bc2"}, + {file = "coverage-7.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c79d2319cabef1fe8e86df73371126931550804738f78ad7d31e3aad85a67367"}, + {file = "coverage-7.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1b23b0c6f0b1db6ad769b7050c8b641c0bf215ded26c1816955b17b7f26edfa9"}, + {file = "coverage-7.14.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:55d3089079ce181a4566b1065ab28d2575eb76d8ac8f81f4fcda2bf037fee087"}, + {file = "coverage-7.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:49c005cba1e2f9677fb2845dcdf9a2e72a52a17d63e8231aaaae35d9f50215ef"}, + {file = "coverage-7.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9117377b823daa28aa8635fbb08cda1cd6be3d7143257345459559aeef852d52"}, + {file = "coverage-7.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7b79d646cf46d5cf9a9f40281d4441df5849e445726e369006d2b117710b33fe"}, + {file = "coverage-7.14.0-cp313-cp313-win32.whl", hash = "sha256:fb609b3658479e33f9516d46f1a89dbb9b6c261366e3a11844a96ec487533dae"}, + {file = "coverage-7.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:0773d8329cf32b6fd222e4b52622c61fe8d503eb966cfc8d3c3c10c96266d50e"}, + {file = "coverage-7.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:b4e26a0f1b696faf283bffe5b8569e44e336c582439df5d53281ab89ee0cba96"}, + {file = "coverage-7.14.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:953f521ca9445300397e65fda3dca58b2dbd68fee983777420b57ac3c77e9f90"}, + {file = "coverage-7.14.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:98af83fd65ae24b1fdd03aaead967a9f523bcd2f1aab2d4f3ffda65bb568a6f1"}, + {file = "coverage-7.14.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:668b92e6958c4db7cf92e81caac328dfbbdbb215db2850ad28f0cbe1eea0bfbd"}, + {file = "coverage-7.14.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9fbd898551762dea00d3fef2b1c4f99afd2c6a3ff952ea07d60a9bd5ed4f34bc"}, + {file = "coverage-7.14.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68af363c07ecd8d4b7d4043d85cb376d7d227eceb54e5323ee45da73dbd3e426"}, + {file = "coverage-7.14.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6e57054a583da8ac55edf24117ea4c9133032cfc4cf72aa2d48c1e5d4b52f899"}, + {file = "coverage-7.14.0-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3499459bbcdd51a65b64c35ab7ed2764eaf3cba826e0df3f1d7fe2e102b70b"}, + {file = "coverage-7.14.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:45899ec2138a4346ed34d601dedf5076fb74edf2d1dd9dc76a78e82397edee90"}, + {file = "coverage-7.14.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:8767486808c436f05b23ab98eb963fb29185e32a9357a166971685cb3459900f"}, + {file = "coverage-7.14.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:a3b5ddfd6aa7ddad53ee3edb231e88a2151507a43229b7d71b953916deca127d"}, + {file = "coverage-7.14.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:63df0fe568e698e1045792399f8ab6da3a6c2dce3182813fb92afa2641087b47"}, + {file = "coverage-7.14.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:827d6397dbd95144939b18f89edf31f63e1f99633e8d5f32f22ba8bdda567477"}, + {file = "coverage-7.14.0-cp313-cp313t-win32.whl", hash = "sha256:7bf43e000d24012599b879791cff41589af90674722421ef11b11a5431920bab"}, + {file = "coverage-7.14.0-cp313-cp313t-win_amd64.whl", hash = "sha256:3f5549365af25d770e06b1f8f5682d9a5637d06eb494db91c6fa75d3950cc917"}, + {file = "coverage-7.14.0-cp313-cp313t-win_arm64.whl", hash = "sha256:6d160217ec6fe890f16ad3a9531761589443749e448f91986c972714fad361c8"}, + {file = "coverage-7.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9aed9fa983514ca032790f3fe0d1c0e42ca7e16b42432af1706b50a9a46bef5d"}, + {file = "coverage-7.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ba3b8390db29296dbbf49e91b6fe08f990743a90c8f447ba4c2ffc29670dfa63"}, + {file = "coverage-7.14.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3a5d8e876dfa2f102e970b183863d6dedd023d3c0eeca1fe7a9787bc5f28b212"}, + {file = "coverage-7.14.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5ebb8f4614a3787d567e610bbfdf96a4798dd69a1afb1bd8ad228d4111fe6ff3"}, + {file = "coverage-7.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b9bf47223dd8db3d4c4b2e443b02bace480d428f0822c3f991600448a176c97"}, + {file = "coverage-7.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3485a836550b303d006d57cc06e3d5afaabc642c77050b7c985a97b13e3776b8"}, + {file = "coverage-7.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3e7e88110bae996d199d1693ca8ec3fd52441d426401ae963437598667b4c5eb"}, + {file = "coverage-7.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:15228a6800ce7bdf1b74800595e56db7138cecb338fdbf044806e10dcf182dfe"}, + {file = "coverage-7.14.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:9d26ac7f5398bafc5b57421ad994e8a4749e8a7a0e62d05ec7d53014d5963bfa"}, + {file = "coverage-7.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2fb73254ff43c911c967a899e1359bc5049b4b115d6e8fbdde4937d0a2246cd5"}, + {file = "coverage-7.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:454a380af72c6adada298ed270d38c7a391288198dbfb8467f786f588751a90c"}, + {file = "coverage-7.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:65c86fb646d2bd2972e96bd1a8b45817ed907cee68655d6295fe7ec031d04cca"}, + {file = "coverage-7.14.0-cp314-cp314-win32.whl", hash = "sha256:6a6516b02a6101398e19a3f44820f69bab2590697f7def4331f668b14adaf828"}, + {file = "coverage-7.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:45e0f79d8351fa76e256716df91eab12890d32678b9590df7ae1042e4bd4cf5d"}, + {file = "coverage-7.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:4b899594a8b2d81e5cc064a0d7f9cac2081fed91049456cae7676787e41549c9"}, + {file = "coverage-7.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f580f8c80acd94ac72e863efe2cab791d8c38d153e0b463b92dfa000d5c84cd1"}, + {file = "coverage-7.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a2bd259c442cd43c49b30fbafc51776eb19ea396faf159d26a83e6a0a5f13b0c"}, + {file = "coverage-7.14.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a706b908dfa85538863504c624b237a3cc34232bf403c057414ebfdb3b4d9f84"}, + {file = "coverage-7.14.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7333cd944ee4393b9b3d3c1b598c936d4fc8d70573a4c7dacfec5590dd50e436"}, + {file = "coverage-7.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f162bc9a15b82d947b02651b0c7e1609d6f7a8735ca330cfadec8481dd97d5a"}, + {file = "coverage-7.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:362cb78e01a5dc82009d88004cf60f2e6b6d6fcbfdec05b05af73b0abf40118f"}, + {file = "coverage-7.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:acebd068fca5512c3a6fde9c045f901613478781a73f0e82b307b214daef23fb"}, + {file = "coverage-7.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:29fe3da551dface75deb2ccbf87b6b66e2e7ef38f6d89050b428be94afff3490"}, + {file = "coverage-7.14.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:b4cc4fce8672fffcb09b0eafc167b396b3ba53c4a7230f54b7aaffbf6c835fa9"}, + {file = "coverage-7.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:5d4a51aad8ba8bdcd2b8bd8f03d4aca19693fa2327a3470e4718a25b03481020"}, + {file = "coverage-7.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:9f323af3e1e4f68b60b7b247e37b8515563a61375518fa59de1af48ba28a3db6"}, + {file = "coverage-7.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1a0abc7342ea9711c469dd8b821c6c311e6bc6aac1442e5fbd6b27fae0a8f3db"}, + {file = "coverage-7.14.0-cp314-cp314t-win32.whl", hash = "sha256:a9f864ef57b7172e2db87a096642dd51e179e085ab6b2c371c29e885f65c8fb2"}, + {file = "coverage-7.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:29943e552fdc08e082eb51400fb2f58e118a83b5542bd06531214e084399b644"}, + {file = "coverage-7.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:742a73ea621953b012f2c4c2219b512180dd84489acf5b1596b0aafc55b9100b"}, + {file = "coverage-7.14.0-py3-none-any.whl", hash = "sha256:8de5b61163aee3d05c8a2beab6f47913df7981dad1baf82c414d99158c286ab1"}, + {file = "coverage-7.14.0.tar.gz", hash = "sha256:057a6af2f160a85384cde4ab36f0d2777bae1057bae255f95413cdd382aa5c74"}, ] [package.dependencies] @@ -251,6 +464,71 @@ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.1 [package.extras] toml = ["tomli"] +[[package]] +name = "cryptography" +version = "48.0.0" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = "!=3.9.0,!=3.9.1,>=3.9" +files = [ + {file = "cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c"}, + {file = "cryptography-48.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5"}, + {file = "cryptography-48.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321"}, + {file = "cryptography-48.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74"}, + {file = "cryptography-48.0.0-cp311-abi3-win32.whl", hash = "sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4"}, + {file = "cryptography-48.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7"}, + {file = "cryptography-48.0.0-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:3cb07a3ed6431663cd321ea8a000a1314c74211f823e4177fefa2255e057d1ec"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c7378637d7d88016fa6791c159f698b3d3eed28ebf844ac36b9dc04a14dae18"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc90c0b39b2e3c65ef52c804b72e3c58f8a04ab2a1871272798e5f9572c17d20"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:76341972e1eff8b4bea859f09c0d3e64b96ce931b084f9b9b7db8ef364c30eff"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:55b7718303bf06a5753dcdccf2f3945cf18ad7bffde41b61226e4db31ab89a9c"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:a64697c641c7b1b2178e573cbc31c7c6684cd56883a478d75143dbb7118036db"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:561215ea3879cb1cbbf272867e2efda62476f240fb58c64de6b393ae19246741"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ad64688338ed4bc1a6618076ba75fd7194a5f1797ac60b47afe926285adb3166"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:906cbf0670286c6e0044156bc7d4af9cbb0ef6db9f73e52c3ec56ba6bdde5336"}, + {file = "cryptography-48.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:ea8990436d914540a40ab24b6a77c0969695ed52f4a4874c5137ccf7045a7057"}, + {file = "cryptography-48.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c18684a7f0cc9a3cb60328f496b8e3372def7c5d2df39ac267878b05565aaaae"}, + {file = "cryptography-48.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9be5aafa5736574f8f15f262adc81b2a9869e2cfe9014d52a44633905b40d52c"}, + {file = "cryptography-48.0.0-cp314-cp314t-win32.whl", hash = "sha256:c17dfe85494deaeddc5ce251aebd1d60bbe6afc8b62071bb0b469431a000124f"}, + {file = "cryptography-48.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27241b1dc9962e056062a8eef1991d02c3a24569c95975bd2322a8a52c6e5e12"}, + {file = "cryptography-48.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a"}, + {file = "cryptography-48.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239"}, + {file = "cryptography-48.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c"}, + {file = "cryptography-48.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4"}, + {file = "cryptography-48.0.0-cp39-abi3-win32.whl", hash = "sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd"}, + {file = "cryptography-48.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8"}, + {file = "cryptography-48.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:84cf79f0dc8b36ac5da873481716e87aef31fcfa0444f9e1d8b4b2cece142855"}, + {file = "cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:fdfef35d751d510fcef5252703621574364fec16418c4a1e5e1055248401054b"}, + {file = "cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:0890f502ddf7d9c6426129c3f49f5c0a39278ed7cd6322c8755ffca6ee675a13"}, + {file = "cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:ecde28a596bead48b0cfd2a1b4416c3d43074c2d785e3a398d7ec1fc4d0f7fbb"}, + {file = "cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:4defde8685ae324a9eb9d818717e93b4638ef67070ac9bc15b8ca85f63048355"}, + {file = "cryptography-48.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:db63bf618e5dea46c07de12e900fe1cdd2541e6dc9dbae772a70b7d4d4765f6a"}, + {file = "cryptography-48.0.0.tar.gz", hash = "sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920"}, +] + +[package.dependencies] +cffi = {version = ">=2.0.0", markers = "platform_python_implementation != \"PyPy\""} +typing-extensions = {version = ">=4.13.2", markers = "python_full_version < \"3.11\""} + +[package.extras] +ssh = ["bcrypt (>=3.1.5)"] + [[package]] name = "desktop-entry-lib" version = "5.0" @@ -284,17 +562,17 @@ test = ["pytest (>=6)"] [[package]] name = "idna" -version = "3.11" +version = "3.15" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.8" files = [ - {file = "idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea"}, - {file = "idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902"}, + {file = "idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8"}, + {file = "idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc"}, ] [package.extras] -all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] +all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] [[package]] name = "iniconfig" @@ -307,17 +585,80 @@ files = [ {file = "iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730"}, ] +[[package]] +name = "invoke" +version = "3.0.3" +description = "Pythonic task execution" +optional = false +python-versions = ">=3.9" +files = [ + {file = "invoke-3.0.3-py3-none-any.whl", hash = "sha256:f11327165e5cbb89b2ad1d88d3292b5113332c43b8553b494da435d6ec6f5053"}, + {file = "invoke-3.0.3.tar.gz", hash = "sha256:437b6a622223824380bfb4e64f612711a6b648c795f565efc8625af66fb57f0c"}, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce"}, + {file = "jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +jsonschema-specifications = ">=2023.03.6" +referencing = ">=0.28.4" +rpds-py = ">=0.25.0" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "rfc3987-syntax (>=1.1.0)", "uri-template", "webcolors (>=24.6.0)"] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +optional = false +python-versions = ">=3.9" +files = [ + {file = "jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe"}, + {file = "jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d"}, +] + +[package.dependencies] +referencing = ">=0.31.0" + [[package]] name = "packaging" -version = "25.0" +version = "26.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, - {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, + {file = "packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e"}, + {file = "packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"}, ] +[[package]] +name = "paramiko" +version = "5.0.0" +description = "SSH2 protocol library" +optional = false +python-versions = ">=3.9" +files = [ + {file = "paramiko-5.0.0-py3-none-any.whl", hash = "sha256:b7044611c30140d9a75261653210e2002977b71a0497ff3ba0d98d7edbf62f7c"}, + {file = "paramiko-5.0.0.tar.gz", hash = "sha256:36763b5b95c2a0dcfdf1abc48e48156ee425b21efe2f0e787c2dd5a95c0e5e79"}, +] + +[package.dependencies] +bcrypt = ">=3.2" +cryptography = ">=3.3" +invoke = ">=2.0" +pynacl = ">=1.5" + [[package]] name = "pluggy" version = "1.6.0" @@ -333,20 +674,72 @@ files = [ dev = ["pre-commit", "tox"] testing = ["coverage", "pytest", "pytest-benchmark"] +[[package]] +name = "pycparser" +version = "3.0" +description = "C parser in Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, + {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, +] + [[package]] name = "pygments" -version = "2.19.2" +version = "2.20.0" description = "Pygments is a syntax highlighting package written in Python." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, - {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, + {file = "pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176"}, + {file = "pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f"}, ] [package.extras] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pynacl" +version = "1.6.2" +description = "Python binding to the Networking and Cryptography (NaCl) library" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pynacl-1.6.2-cp314-cp314t-macosx_10_10_universal2.whl", hash = "sha256:622d7b07cc5c02c666795792931b50c91f3ce3c2649762efb1ef0d5684c81594"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d071c6a9a4c94d79eb665db4ce5cedc537faf74f2355e4d502591d850d3913c0"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe9847ca47d287af41e82be1dd5e23023d3c31a951da134121ab02e42ac218c9"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:04316d1fc625d860b6c162fff704eb8426b1a8bcd3abacea11142cbd99a6b574"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44081faff368d6c5553ccf55322ef2819abb40e25afaec7e740f159f74813634"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:a9f9932d8d2811ce1a8ffa79dcbdf3970e7355b5c8eb0c1a881a57e7f7d96e88"}, + {file = "pynacl-1.6.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:bc4a36b28dd72fb4845e5d8f9760610588a96d5a51f01d84d8c6ff9849968c14"}, + {file = "pynacl-1.6.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3bffb6d0f6becacb6526f8f42adfb5efb26337056ee0831fb9a7044d1a964444"}, + {file = "pynacl-1.6.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2fef529ef3ee487ad8113d287a593fa26f48ee3620d92ecc6f1d09ea38e0709b"}, + {file = "pynacl-1.6.2-cp314-cp314t-win32.whl", hash = "sha256:a84bf1c20339d06dc0c85d9aea9637a24f718f375d861b2668b2f9f96fa51145"}, + {file = "pynacl-1.6.2-cp314-cp314t-win_amd64.whl", hash = "sha256:320ef68a41c87547c91a8b58903c9caa641ab01e8512ce291085b5fe2fcb7590"}, + {file = "pynacl-1.6.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d29bfe37e20e015a7d8b23cfc8bd6aa7909c92a1b8f41ee416bbb3e79ef182b2"}, + {file = "pynacl-1.6.2-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:c949ea47e4206af7c8f604b8278093b674f7c79ed0d4719cc836902bf4517465"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8845c0631c0be43abdd865511c41eab235e0be69c81dc66a50911594198679b0"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:22de65bb9010a725b0dac248f353bb072969c94fa8d6b1f34b87d7953cf7bbe4"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46065496ab748469cdd999246d17e301b2c24ae2fdf739132e580a0e94c94a87"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a66d6fb6ae7661c58995f9c6435bda2b1e68b54b598a6a10247bfcdadac996c"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:26bfcd00dcf2cf160f122186af731ae30ab120c18e8375684ec2670dccd28130"}, + {file = "pynacl-1.6.2-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:c8a231e36ec2cab018c4ad4358c386e36eede0319a0c41fed24f840b1dac59f6"}, + {file = "pynacl-1.6.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:68be3a09455743ff9505491220b64440ced8973fe930f270c8e07ccfa25b1f9e"}, + {file = "pynacl-1.6.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8b097553b380236d51ed11356c953bf8ce36a29a3e596e934ecabe76c985a577"}, + {file = "pynacl-1.6.2-cp38-abi3-win32.whl", hash = "sha256:5811c72b473b2f38f7e2a3dc4f8642e3a3e9b5e7317266e4ced1fba85cae41aa"}, + {file = "pynacl-1.6.2-cp38-abi3-win_amd64.whl", hash = "sha256:62985f233210dee6548c223301b6c25440852e13d59a8b81490203c3227c5ba0"}, + {file = "pynacl-1.6.2-cp38-abi3-win_arm64.whl", hash = "sha256:834a43af110f743a754448463e8fd61259cd4ab5bbedcf70f9dabad1d28a394c"}, + {file = "pynacl-1.6.2.tar.gz", hash = "sha256:018494d6d696ae03c7e656e5e74cdfd8ea1326962cc401bcf018f1ed8436811c"}, +] + +[package.dependencies] +cffi = {version = ">=2.0.0", markers = "platform_python_implementation != \"PyPy\" and python_version >= \"3.9\""} + +[package.extras] +docs = ["sphinx (<7)", "sphinx_rtd_theme"] +tests = ["hypothesis (>=3.27.0)", "pytest (>=7.4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] + [[package]] name = "pyproject-appimage" version = "4.2" @@ -405,75 +798,302 @@ pytest = ">=4.6" testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] [[package]] -name = "requests" -version = "2.32.5" -description = "Python HTTP for Humans." +name = "pyyaml" +version = "6.0.3" +description = "YAML parser and emitter for Python" optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" files = [ - {file = "requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6"}, - {file = "requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf"}, + {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, + {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, + {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, + {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, + {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, + {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, + {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, + {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, + {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, + {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, + {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, + {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, + {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, + {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, + {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, + {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, + {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, + {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, + {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, + {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, + {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, + {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, + {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, + {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, + {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, + {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, + {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, + {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, + {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, + {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, + {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, + {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, + {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, + {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, + {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, + {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, + {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, + {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, + {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, + {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, + {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, + {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, + {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, + {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, + {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, + {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, + {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, + {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, + {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, + {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, + {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, + {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, + {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, + {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, + {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, + {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, + {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, + {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, + {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, + {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, + {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, + {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, + {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, + {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, + {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, + {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, + {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, + {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, + {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, + {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, +] + +[[package]] +name = "referencing" +version = "0.37.0" +description = "JSON Referencing + Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231"}, + {file = "referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8"}, ] [package.dependencies] -certifi = ">=2017.4.17" +attrs = ">=22.2.0" +rpds-py = ">=0.7.0" +typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""} + +[[package]] +name = "requests" +version = "2.34.1" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.10" +files = [ + {file = "requests-2.34.1-py3-none-any.whl", hash = "sha256:bf38a3ff993960d3dd819c08862c40b3c703306eb7c744fcd9f4ddbb95b548f0"}, + {file = "requests-2.34.1.tar.gz", hash = "sha256:0fc5669f2b69704449fe1552360bd2a73a54512dfd03e65529157f1513322beb"}, +] + +[package.dependencies] +certifi = ">=2023.5.7" charset_normalizer = ">=2,<4" idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<3" +urllib3 = ">=1.26,<3" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] + +[[package]] +name = "rpds-py" +version = "0.30.0" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.10" +files = [ + {file = "rpds_py-0.30.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288"}, + {file = "rpds_py-0.30.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cc2206b76b4f576934f0ed374b10d7ca5f457858b157ca52064bdfc26b9fc00"}, + {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:389a2d49eded1896c3d48b0136ead37c48e221b391c052fba3f4055c367f60a6"}, + {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:32c8528634e1bf7121f3de08fa85b138f4e0dc47657866630611b03967f041d7"}, + {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f207f69853edd6f6700b86efb84999651baf3789e78a466431df1331608e5324"}, + {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:67b02ec25ba7a9e8fa74c63b6ca44cf5707f2fbfadae3ee8e7494297d56aa9df"}, + {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0e95f6819a19965ff420f65578bacb0b00f251fefe2c8b23347c37174271f3"}, + {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:a452763cc5198f2f98898eb98f7569649fe5da666c2dc6b5ddb10fde5a574221"}, + {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e0b65193a413ccc930671c55153a03ee57cecb49e6227204b04fae512eb657a7"}, + {file = "rpds_py-0.30.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:858738e9c32147f78b3ac24dc0edb6610000e56dc0f700fd5f651d0a0f0eb9ff"}, + {file = "rpds_py-0.30.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:da279aa314f00acbb803da1e76fa18666778e8a8f83484fba94526da5de2cba7"}, + {file = "rpds_py-0.30.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7c64d38fb49b6cdeda16ab49e35fe0da2e1e9b34bc38bd78386530f218b37139"}, + {file = "rpds_py-0.30.0-cp310-cp310-win32.whl", hash = "sha256:6de2a32a1665b93233cde140ff8b3467bdb9e2af2b91079f0333a0974d12d464"}, + {file = "rpds_py-0.30.0-cp310-cp310-win_amd64.whl", hash = "sha256:1726859cd0de969f88dc8673bdd954185b9104e05806be64bcd87badbe313169"}, + {file = "rpds_py-0.30.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a2bffea6a4ca9f01b3f8e548302470306689684e61602aa3d141e34da06cf425"}, + {file = "rpds_py-0.30.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dc4f992dfe1e2bc3ebc7444f6c7051b4bc13cd8e33e43511e8ffd13bf407010d"}, + {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:422c3cb9856d80b09d30d2eb255d0754b23e090034e1deb4083f8004bd0761e4"}, + {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07ae8a593e1c3c6b82ca3292efbe73c30b61332fd612e05abee07c79359f292f"}, + {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12f90dd7557b6bd57f40abe7747e81e0c0b119bef015ea7726e69fe550e394a4"}, + {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99b47d6ad9a6da00bec6aabe5a6279ecd3c06a329d4aa4771034a21e335c3a97"}, + {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33f559f3104504506a44bb666b93a33f5d33133765b0c216a5bf2f1e1503af89"}, + {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:946fe926af6e44f3697abbc305ea168c2c31d3e3ef1058cf68f379bf0335a78d"}, + {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:495aeca4b93d465efde585977365187149e75383ad2684f81519f504f5c13038"}, + {file = "rpds_py-0.30.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9a0ca5da0386dee0655b4ccdf46119df60e0f10da268d04fe7cc87886872ba7"}, + {file = "rpds_py-0.30.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8d6d1cc13664ec13c1b84241204ff3b12f9bb82464b8ad6e7a5d3486975c2eed"}, + {file = "rpds_py-0.30.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3896fa1be39912cf0757753826bc8bdc8ca331a28a7c4ae46b7a21280b06bb85"}, + {file = "rpds_py-0.30.0-cp311-cp311-win32.whl", hash = "sha256:55f66022632205940f1827effeff17c4fa7ae1953d2b74a8581baaefb7d16f8c"}, + {file = "rpds_py-0.30.0-cp311-cp311-win_amd64.whl", hash = "sha256:a51033ff701fca756439d641c0ad09a41d9242fa69121c7d8769604a0a629825"}, + {file = "rpds_py-0.30.0-cp311-cp311-win_arm64.whl", hash = "sha256:47b0ef6231c58f506ef0b74d44e330405caa8428e770fec25329ed2cb971a229"}, + {file = "rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad"}, + {file = "rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05"}, + {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28"}, + {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd"}, + {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f"}, + {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1"}, + {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23"}, + {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6"}, + {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51"}, + {file = "rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5"}, + {file = "rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e"}, + {file = "rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394"}, + {file = "rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf"}, + {file = "rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b"}, + {file = "rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e"}, + {file = "rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2"}, + {file = "rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8"}, + {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4"}, + {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136"}, + {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7"}, + {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2"}, + {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6"}, + {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e"}, + {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d"}, + {file = "rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7"}, + {file = "rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31"}, + {file = "rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95"}, + {file = "rpds_py-0.30.0-cp313-cp313-win32.whl", hash = "sha256:b40fb160a2db369a194cb27943582b38f79fc4887291417685f3ad693c5a1d5d"}, + {file = "rpds_py-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15"}, + {file = "rpds_py-0.30.0-cp313-cp313-win_arm64.whl", hash = "sha256:d96c2086587c7c30d44f31f42eae4eac89b60dabbac18c7669be3700f13c3ce1"}, + {file = "rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a"}, + {file = "rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e"}, + {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000"}, + {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db"}, + {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2"}, + {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa"}, + {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083"}, + {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9"}, + {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0"}, + {file = "rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94"}, + {file = "rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08"}, + {file = "rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27"}, + {file = "rpds_py-0.30.0-cp313-cp313t-win32.whl", hash = "sha256:3d4a69de7a3e50ffc214ae16d79d8fbb0922972da0356dcf4d0fdca2878559c6"}, + {file = "rpds_py-0.30.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f14fc5df50a716f7ece6a80b6c78bb35ea2ca47c499e422aa4463455dd96d56d"}, + {file = "rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0"}, + {file = "rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be"}, + {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f"}, + {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f"}, + {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87"}, + {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18"}, + {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad"}, + {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07"}, + {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f"}, + {file = "rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65"}, + {file = "rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f"}, + {file = "rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53"}, + {file = "rpds_py-0.30.0-cp314-cp314-win32.whl", hash = "sha256:ee454b2a007d57363c2dfd5b6ca4a5d7e2c518938f8ed3b706e37e5d470801ed"}, + {file = "rpds_py-0.30.0-cp314-cp314-win_amd64.whl", hash = "sha256:95f0802447ac2d10bcc69f6dc28fe95fdf17940367b21d34e34c737870758950"}, + {file = "rpds_py-0.30.0-cp314-cp314-win_arm64.whl", hash = "sha256:613aa4771c99f03346e54c3f038e4cc574ac09a3ddfb0e8878487335e96dead6"}, + {file = "rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb"}, + {file = "rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8"}, + {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7"}, + {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898"}, + {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e"}, + {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419"}, + {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551"}, + {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8"}, + {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5"}, + {file = "rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404"}, + {file = "rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856"}, + {file = "rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40"}, + {file = "rpds_py-0.30.0-cp314-cp314t-win32.whl", hash = "sha256:dc824125c72246d924f7f796b4f63c1e9dc810c7d9e2355864b3c3a73d59ade0"}, + {file = "rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c2262bdba0ad4fc6fb5545660673925c2d2a5d9e2e0fb603aad545427be0fc58"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ee6af14263f25eedc3bb918a3c04245106a42dfd4f5c2285ea6f997b1fc3f89a"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3adbb8179ce342d235c31ab8ec511e66c73faa27a47e076ccc92421add53e2bb"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:250fa00e9543ac9b97ac258bd37367ff5256666122c2d0f2bc97577c60a1818c"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9854cf4f488b3d57b9aaeb105f06d78e5529d3145b1e4a41750167e8c213c6d3"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:993914b8e560023bc0a8bf742c5f303551992dcb85e247b1e5c7f4a7d145bda5"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58edca431fb9b29950807e301826586e5bbf24163677732429770a697ffe6738"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:dea5b552272a944763b34394d04577cf0f9bd013207bc32323b5a89a53cf9c2f"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ba3af48635eb83d03f6c9735dfb21785303e73d22ad03d489e88adae6eab8877"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:dff13836529b921e22f15cb099751209a60009731a68519630a24d61f0b1b30a"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:1b151685b23929ab7beec71080a8889d4d6d9fa9a983d213f07121205d48e2c4"}, + {file = "rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ac37f9f516c51e5753f27dfdef11a88330f04de2d564be3991384b2f3535d02e"}, + {file = "rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84"}, +] [[package]] name = "tomli" -version = "2.3.0" +version = "2.4.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" files = [ - {file = "tomli-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45"}, - {file = "tomli-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:883b1c0d6398a6a9d29b508c331fa56adbcdff647f6ace4dfca0f50e90dfd0ba"}, - {file = "tomli-2.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1381caf13ab9f300e30dd8feadb3de072aeb86f1d34a8569453ff32a7dea4bf"}, - {file = "tomli-2.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0e285d2649b78c0d9027570d4da3425bdb49830a6156121360b3f8511ea3441"}, - {file = "tomli-2.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0a154a9ae14bfcf5d8917a59b51ffd5a3ac1fd149b71b47a3a104ca4edcfa845"}, - {file = "tomli-2.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:74bf8464ff93e413514fefd2be591c3b0b23231a77f901db1eb30d6f712fc42c"}, - {file = "tomli-2.3.0-cp311-cp311-win32.whl", hash = "sha256:00b5f5d95bbfc7d12f91ad8c593a1659b6387b43f054104cda404be6bda62456"}, - {file = "tomli-2.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:4dc4ce8483a5d429ab602f111a93a6ab1ed425eae3122032db7e9acf449451be"}, - {file = "tomli-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d7d86942e56ded512a594786a5ba0a5e521d02529b3826e7761a05138341a2ac"}, - {file = "tomli-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:73ee0b47d4dad1c5e996e3cd33b8a76a50167ae5f96a2607cbe8cc773506ab22"}, - {file = "tomli-2.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:792262b94d5d0a466afb5bc63c7daa9d75520110971ee269152083270998316f"}, - {file = "tomli-2.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f195fe57ecceac95a66a75ac24d9d5fbc98ef0962e09b2eddec5d39375aae52"}, - {file = "tomli-2.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e31d432427dcbf4d86958c184b9bfd1e96b5b71f8eb17e6d02531f434fd335b8"}, - {file = "tomli-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7b0882799624980785240ab732537fcfc372601015c00f7fc367c55308c186f6"}, - {file = "tomli-2.3.0-cp312-cp312-win32.whl", hash = "sha256:ff72b71b5d10d22ecb084d345fc26f42b5143c5533db5e2eaba7d2d335358876"}, - {file = "tomli-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:1cb4ed918939151a03f33d4242ccd0aa5f11b3547d0cf30f7c74a408a5b99878"}, - {file = "tomli-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5192f562738228945d7b13d4930baffda67b69425a7f0da96d360b0a3888136b"}, - {file = "tomli-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:be71c93a63d738597996be9528f4abe628d1adf5e6eb11607bc8fe1a510b5dae"}, - {file = "tomli-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4665508bcbac83a31ff8ab08f424b665200c0e1e645d2bd9ab3d3e557b6185b"}, - {file = "tomli-2.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4021923f97266babc6ccab9f5068642a0095faa0a51a246a6a02fccbb3514eaf"}, - {file = "tomli-2.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4ea38c40145a357d513bffad0ed869f13c1773716cf71ccaa83b0fa0cc4e42f"}, - {file = "tomli-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ad805ea85eda330dbad64c7ea7a4556259665bdf9d2672f5dccc740eb9d3ca05"}, - {file = "tomli-2.3.0-cp313-cp313-win32.whl", hash = "sha256:97d5eec30149fd3294270e889b4234023f2c69747e555a27bd708828353ab606"}, - {file = "tomli-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0c95ca56fbe89e065c6ead5b593ee64b84a26fca063b5d71a1122bf26e533999"}, - {file = "tomli-2.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:cebc6fe843e0733ee827a282aca4999b596241195f43b4cc371d64fc6639da9e"}, - {file = "tomli-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4c2ef0244c75aba9355561272009d934953817c49f47d768070c3c94355c2aa3"}, - {file = "tomli-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c22a8bf253bacc0cf11f35ad9808b6cb75ada2631c2d97c971122583b129afbc"}, - {file = "tomli-2.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0eea8cc5c5e9f89c9b90c4896a8deefc74f518db5927d0e0e8d4a80953d774d0"}, - {file = "tomli-2.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b74a0e59ec5d15127acdabd75ea17726ac4c5178ae51b85bfe39c4f8a278e879"}, - {file = "tomli-2.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b5870b50c9db823c595983571d1296a6ff3e1b88f734a4c8f6fc6188397de005"}, - {file = "tomli-2.3.0-cp314-cp314-win32.whl", hash = "sha256:feb0dacc61170ed7ab602d3d972a58f14ee3ee60494292d384649a3dc38ef463"}, - {file = "tomli-2.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:b273fcbd7fc64dc3600c098e39136522650c49bca95df2d11cf3b626422392c8"}, - {file = "tomli-2.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:940d56ee0410fa17ee1f12b817b37a4d4e4dc4d27340863cc67236c74f582e77"}, - {file = "tomli-2.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f85209946d1fe94416debbb88d00eb92ce9cd5266775424ff81bc959e001acaf"}, - {file = "tomli-2.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a56212bdcce682e56b0aaf79e869ba5d15a6163f88d5451cbde388d48b13f530"}, - {file = "tomli-2.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5f3ffd1e098dfc032d4d3af5c0ac64f6d286d98bc148698356847b80fa4de1b"}, - {file = "tomli-2.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e01decd096b1530d97d5d85cb4dff4af2d8347bd35686654a004f8dea20fc67"}, - {file = "tomli-2.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8a35dd0e643bb2610f156cca8db95d213a90015c11fee76c946aa62b7ae7e02f"}, - {file = "tomli-2.3.0-cp314-cp314t-win32.whl", hash = "sha256:a1f7f282fe248311650081faafa5f4732bdbfef5d45fe3f2e702fbc6f2d496e0"}, - {file = "tomli-2.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:70a251f8d4ba2d9ac2542eecf008b3c8a9fc5c3f9f02c56a9d7952612be2fdba"}, - {file = "tomli-2.3.0-py3-none-any.whl", hash = "sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b"}, - {file = "tomli-2.3.0.tar.gz", hash = "sha256:64be704a875d2a59753d80ee8a533c3fe183e3f06807ff7dc2232938ccb01549"}, + {file = "tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30"}, + {file = "tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a"}, + {file = "tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076"}, + {file = "tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9"}, + {file = "tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c"}, + {file = "tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc"}, + {file = "tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049"}, + {file = "tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e"}, + {file = "tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece"}, + {file = "tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a"}, + {file = "tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085"}, + {file = "tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9"}, + {file = "tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5"}, + {file = "tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585"}, + {file = "tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1"}, + {file = "tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917"}, + {file = "tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9"}, + {file = "tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257"}, + {file = "tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54"}, + {file = "tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a"}, + {file = "tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897"}, + {file = "tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f"}, + {file = "tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d"}, + {file = "tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5"}, + {file = "tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd"}, + {file = "tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36"}, + {file = "tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd"}, + {file = "tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf"}, + {file = "tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac"}, + {file = "tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662"}, + {file = "tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853"}, + {file = "tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15"}, + {file = "tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba"}, + {file = "tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6"}, + {file = "tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7"}, + {file = "tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232"}, + {file = "tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4"}, + {file = "tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c"}, + {file = "tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d"}, + {file = "tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41"}, + {file = "tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c"}, + {file = "tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f"}, + {file = "tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8"}, + {file = "tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26"}, + {file = "tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396"}, + {file = "tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe"}, + {file = "tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f"}, ] [[package]] @@ -489,13 +1109,13 @@ files = [ [[package]] name = "urllib3" -version = "2.6.2" +version = "2.7.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" files = [ - {file = "urllib3-2.6.2-py3-none-any.whl", hash = "sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd"}, - {file = "urllib3-2.6.2.tar.gz", hash = "sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797"}, + {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, + {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, ] [package.extras] @@ -507,4 +1127,4 @@ zstd = ["backports-zstd (>=1.0.0)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "84c06974dfe822257ef324807672e51d71c0a6197e037fa56e92d8369c40d341" +content-hash = "30e16396439f2cdd69005a5b7bdf8144aac33422a77a63accbc9eaa74151d851" diff --git a/pyproject.toml b/pyproject.toml index 6b4d1b1..a7a83d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,21 @@ [tool.poetry] name = "enroll" -version = "0.0.4" +version = "0.6.0" description = "Enroll a server's running state retrospectively into Ansible" authors = ["Miguel Jacq "] license = "GPL-3.0-or-later" readme = "README.md" packages = [{ include = "enroll" }] repository = "https://git.mig5.net/mig5/enroll" +include = [ + { path = "enroll/schema/state.schema.json", format = ["sdist", "wheel"] } +] [tool.poetry.dependencies] python = "^3.10" +pyyaml = "^6" +paramiko = ">=3.5" +jsonschema = "^4.23.0" [tool.poetry.scripts] enroll = "enroll.cli:main" diff --git a/pytests.sh b/pytests.sh new file mode 100755 index 0000000..d49d04b --- /dev/null +++ b/pytests.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -eou pipefail + +poetry run pytest -q tests -vvv --cov=enroll diff --git a/release.sh b/release.sh index fe99a52..d41c468 100755 --- a/release.sh +++ b/release.sh @@ -10,8 +10,7 @@ poetry build poetry publish # Make AppImage -poetry run pyproject-appimage -mv Enroll.AppImage dist/ +poetry run pyproject-appimage --output dist/Enroll.AppImage # Sign packages for file in `ls -1 dist/`; do qubes-gpg-client --batch --armor --detach-sign dist/$file > dist/$file.asc; done @@ -42,3 +41,52 @@ for dist in ${DISTS[@]}; do debfile=$(ls -1 dist/${release}/*.deb) reprepro -b /home/user/git/repo includedeb "${release}" "${debfile}" done + +# RPM +sudo apt-get -y install createrepo-c rpm +BUILD_OUTPUT="${HOME}/git/enroll/dist" +KEYID="54A91143AE0AB4F7743B01FE888ED1B423A3BC99" +REPO_ROOT="${HOME}/git/repo_rpm" +REMOTE="letessier.mig5.net:/opt/repo_rpm" + +DISTS=( + fedora:43 + fedora:42 +) + +for dist in ${DISTS[@]}; do + release=$(echo ${dist} | cut -d: -f2) + REPO_RELEASE_ROOT="${REPO_ROOT}/${release}" + RPM_REPO="${REPO_RELEASE_ROOT}/rpm/x86_64" + mkdir -p "$RPM_REPO" + + docker build \ + --no-cache \ + -f Dockerfile.rpmbuild \ + -t enroll-rpm:${release} \ + --progress=plain \ + --build-arg BASE_IMAGE=${dist} \ + . + + rm -rf "$PWD/dist/rpm"/* + mkdir -p "$PWD/dist/rpm" + + docker run --rm -v "$PWD":/src -v "$PWD/dist/rpm":/out enroll-rpm:${release} + sudo chown -R "${USER}" "$PWD/dist" + + for file in `ls -1 "${BUILD_OUTPUT}/rpm"`; do + rpmsign --addsign "${BUILD_OUTPUT}/rpm/$file" + done + + cp "${BUILD_OUTPUT}/rpm/"*.rpm "$RPM_REPO/" + + createrepo_c "$RPM_REPO" + + echo "==> Signing repomd.xml..." + qubes-gpg-client --local-user "$KEYID" --detach-sign --armor "$RPM_REPO/repodata/repomd.xml" > "$RPM_REPO/repodata/repomd.xml.asc" +done + +echo "==> Syncing repo to server..." +rsync -aHPvz --exclude=.git --delete "$REPO_ROOT/" "$REMOTE/" + +echo "Done!" diff --git a/rpm/enroll.spec b/rpm/enroll.spec new file mode 100644 index 0000000..0e83c84 --- /dev/null +++ b/rpm/enroll.spec @@ -0,0 +1,100 @@ +%global upstream_version 0.6.0 + +Name: enroll +Version: %{upstream_version} +Release: 1%{?dist}.enroll1 +Summary: Enroll a server's running state retrospectively into Ansible. + +License: GPL-3.0-or-later +URL: https://git.mig5.net/mig5/enroll +Source0: %{name}-%{version}.tar.gz + +BuildArch: noarch + +BuildRequires: pyproject-rpm-macros +BuildRequires: python3-devel +BuildRequires: python3-poetry-core + +Requires: python3-yaml +Requires: python3-paramiko +Requires: python3-jsonschema + +Recommends: jinjaturtle + +%description +Enroll a server's running state retrospectively into Ansible. + +%prep +%autosetup -n enroll + +%generate_buildrequires +%pyproject_buildrequires + +%build +%pyproject_wheel + +%install +%pyproject_install +%pyproject_save_files enroll + +%files -f %{pyproject_files} +%license LICENSE +%doc README.md CHANGELOG.md +%{_bindir}/enroll + +%changelog +* Thu May 14 2026 Miguel Jacq - %{version}-%{release} +- Add support for capturing ipset and iptables configuration files +- Add support for generating ipset and iptables configuration files from runtime, if the former weren't present ('firewall_runtime' role) +* Tue May 12 2026 Miguel Jacq - %{version}-%{release} +- Add ssh config support where JinjaTurtle is used +* Tue Feb 16 2026 Miguel Jacq - %{version}-%{release} +- Add capability to handle passphrases on encrypted SSH private keys. Prompting can be forced with `--ask-key-passphrase` or automated (e.g for CI) with `--ssh-key-passphrase env SOMEVAR` +* Fri Jan 16 2026 Miguel Jacq - %{version}-%{release} +- Add support for AddressFamily and ConnectTimeout in the .ssh/config when using `--remote-ssh-config`. +* Tue Jan 13 2026 Miguel Jacq - %{version}-%{release} +- Support `--remote-ssh-config [path-to-ssh-config]` as an argument in case extra params are required beyond `--remote-port` or `--remote-user`. Note: `--remote-host` must still be s +et, but it can be an 'alias' represented by the 'Host' value in the ssh config. +* Sun Jan 11 2026 Miguel Jacq - %{version}-%{release} +- Add interactive output when 'enroll diff --enforce' is invoking Ansible. +* Sat Jan 10 2026 Miguel Jacq - %{version}-%{release} +- Introduce `enroll validate` - a tool to validate a harvest against the state schema, or check for missing or orphaned obsolete artifacts in a harvest. +- Attempt to generate Jinja2 templates of systemd unit files and Postfix main.cf (now that JinjaTurtle supports it) +- Update pynacl dependency to resolve CVE-2025-69277 +- Add `--exclude-path` to `enroll diff` command, so that you can ignore certain churn from the diff (stuff you still wanted to harvest as a baseline but don't care if it changes day to day) +- Add `--ignore-package-versions` to `enroll diff` command, to optionally ignore package upgrades (e.g due to patching) from the diff. +- Add tags to the playbook for each role, to allow easier targeting of specific roles during play later. +- Add `--enforce` mode to `enroll diff`. If there is diff detected between the two harvests, and it can enforce restoring the state from the older harvest, it will manifest the state and apply it with ansible. +Only the specific roles that had diffed will be applied (via the new tags capability) +* Mon Jan 05 2026 Miguel Jacq - %{version}-%{release} +- Introduce `enroll explain` - a tool to analyze and explain what's in (or not in) a harvest and why. +- Centralise the cron and logrotate stuff into their respective roles, we had a bit of duplication between roles based on harvest discovery. +- Capture other files in the user's home directory such as `.bashrc`, `.bash_aliases`, `.profile`, if these files differ from the `/etc/skel` defaults +- Ignore files that end with a tilde or - (probably backup files generated by editors or shadow file changes) +- Manage certain symlinks e.g for apache2/nginx sites-enabled and so on +* Sun Jan 04 2026 Miguel Jacq - %{version}-%{release} +- Introduce --ask-become-pass or -K to support password-required sudo on remote hosts, just like Ansible. It will also fall back to this prompt if a password is required but the arg wasn't passed in. +* Sat Jan 03 2026 Miguel Jacq - %{version}-%{release} +- Fix stat() of parent directory so that we set directory perms correct on --include paths. +- Set pty for remote calls when sudo is required, to help systems with limits on sudo without pty +* Fri Jan 02 2026 Miguel Jacq - %{version}-%{release} +- Don't accidentally add extra_paths role to usr_local_custom list, resulting in extra_paths appearing twice in manifested playbook +- Ensure directories in the tree of anything included with --include are defined in the state and manifest so we make dirs before we try to create files +* Mon Dec 29 2025 Miguel Jacq - %{version}-%{release} +- Add version CLI arg +- Add ability to enroll RH-style systems (DNF5/DNF/RPM) +- Refactor harvest state to track package versions +* Sun Dec 28 2025 Miguel Jacq - %{version}-%{release} +- Fix an attribution bug for certain files ending up in the wrong package/role. +* Sun Dec 28 2025 Miguel Jacq - %{version}-%{release} +- DRY up some code logic +- More test coverage +* Sun Dec 28 2025 Miguel Jacq - %{version}-%{release} +- Consolidate logrotate and cron files into their main service/package roles if they exist. +- Standardise on MAX_FILES_CAP in one place +- Manage apt stuff in its own role, not in etc_custom +* Sat Dec 27 2025 Miguel Jacq - %{version}-%{release} +- Attempt to capture more stuff from /etc that might not be attributable to a specific package. This includes common singletons and systemd timers +- Avoid duplicate apt data in package-specific roles. +* Sat Dec 27 2025 Miguel Jacq - %{version}-%{release} +- Initial RPM packaging for Fedora 42 diff --git a/tests.sh b/tests.sh index f8d246c..1295daa 100755 --- a/tests.sh +++ b/tests.sh @@ -1,24 +1,182 @@ #!/bin/bash -set -eo pipefail +set -Eeuo pipefail -# Pytests -poetry run pytest -vvvv --cov=enroll --cov-report=term-missing --disable-warnings +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TMP_PARENT="${TMPDIR:-/tmp}" +KEEP_WORKDIR=0 +if [[ -n "${ENROLL_TEST_WORKDIR:-}" ]]; then + WORK_DIR="${ENROLL_TEST_WORKDIR}" + KEEP_WORKDIR=1 + mkdir -p "${WORK_DIR}" +else + WORK_DIR="$(mktemp -d "${TMP_PARENT%/}/enroll-tests.XXXXXX")" +fi -BUNDLE_DIR="/tmp/bundle" -ANSIBLE_DIR="/tmp/ansible" -rm -rf "${BUNDLE_DIR}" "${ANSIBLE_DIR}" +BUNDLE_DIR="${WORK_DIR}/bundle" +BUNDLE_DIFF_DIR="${WORK_DIR}/bundle-diff" +ANSIBLE_DIR="${WORK_DIR}/ansible" +ANSIBLE_NO_COMMON_DIR="${WORK_DIR}/ansible-no-common" +ANSIBLE_FQDN_DIR="${WORK_DIR}/ansible-fqdn" +PUPPET_DIR="${WORK_DIR}/puppet" +PUPPET_FQDN_DIR="${WORK_DIR}/puppet-fqdn" +TEST_FQDN="${ENROLL_TEST_FQDN:-enroll-ci.example.test}" -# Generate data -poetry run \ - enroll enroll \ - --harvest "${BUNDLE_DIR}" \ - --out "${ANSIBLE_DIR}" +cleanup() { + if [[ "${KEEP_WORKDIR}" -eq 0 ]]; then + rm -rf "${WORK_DIR}" + else + printf '\nKeeping ENROLL_TEST_WORKDIR: %s\n' "${WORK_DIR}" + fi +} +trap cleanup EXIT -builtin cd "${ANSIBLE_DIR}" +section() { + printf '\n================================================================================\n' + printf '%s\n' "$1" + printf '================================================================================\n' +} -# Lint -ansible-lint "${ANSIBLE_DIR}" +run() { + printf '+ ' + printf '%q ' "$@" + printf '\n' + "$@" +} -# Run -ansible-playbook playbook.yml -i "localhost," -c local --check --diff +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +require_root() { + if [[ "$(id -u)" -ne 0 ]]; then + fail "tests.sh must be run as root so harvest and CM noop tests can inspect/apply system state." + fi +} + +require_debian_ci() { + if [[ -r /etc/os-release ]]; then + # shellcheck disable=SC1091 + . /etc/os-release + if [[ "${ID:-}" != "debian" || "${VERSION_ID:-}" != "13" ]]; then + printf 'WARNING: tests.sh is maintained for Debian 13 CI; detected %s %s.\n' "${ID:-unknown}" "${VERSION_ID:-unknown}" >&2 + fi + fi +} + +apt_update_once() { + if [[ -z "${APT_UPDATED:-}" ]]; then + section "Setup: apt metadata" + run apt-get update + APT_UPDATED=1 + fi +} + +apt_install() { + apt_update_once + run env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "$@" +} + +apt_remove_purge() { + run env DEBIAN_FRONTEND=noninteractive apt-get remove -y --purge "$@" +} + +require_cmd() { + local cmd="$1" + local hint="$2" + if ! command -v "${cmd}" >/dev/null 2>&1; then + fail "Required command '${cmd}' was not found. ${hint}" + fi +} + +ensure_ansible() { + if ! command -v ansible-playbook >/dev/null 2>&1 || ! command -v ansible-lint >/dev/null 2>&1; then + apt_install ansible ansible-lint + fi + require_cmd ansible-playbook "Install the Debian ansible package." + require_cmd ansible-lint "Install the Debian ansible-lint package." +} + +ensure_puppet() { + if ! command -v puppet >/dev/null 2>&1; then + apt_install puppet || apt_install puppet-agent + fi + require_cmd puppet "Install Puppet before running the Puppet noop integration tests." +} + +run_pytests() { + section "Python unit tests" + cd "${PROJECT_ROOT}" + run poetry run pytest -vvvv --cov=enroll --cov-report=term-missing --disable-warnings +} + +prepare_harvest_fixture() { + section "Common harvest fixture and CLI smoke checks" + apt_install jq apache2 + + cd "${PROJECT_ROOT}" + rm -rf "${BUNDLE_DIR}" "${BUNDLE_DIFF_DIR}" + + run poetry run enroll harvest --out "${BUNDLE_DIR}" + run poetry run enroll explain "${BUNDLE_DIR}" + run bash -c "poetry run enroll explain '${BUNDLE_DIR}' --format json | jq" + run poetry run enroll validate --fail-on-warnings "${BUNDLE_DIR}" + + apt_install cowsay + run poetry run enroll harvest --out "${BUNDLE_DIFF_DIR}" + run poetry run enroll validate --fail-on-warnings "${BUNDLE_DIFF_DIR}" + run bash -c "poetry run enroll diff --old '${BUNDLE_DIR}' --new '${BUNDLE_DIFF_DIR}' --format json | jq" + apt_remove_purge cowsay +} + +run_ansible_noop_tests() { + section "Ansible manifest noop tests" + ensure_ansible + cd "${PROJECT_ROOT}" + rm -rf "${ANSIBLE_DIR}" "${ANSIBLE_NO_COMMON_DIR}" "${ANSIBLE_FQDN_DIR}" + + run poetry run enroll manifest --harvest "${BUNDLE_DIR}" --out "${ANSIBLE_DIR}" --target ansible + run ansible-lint "${ANSIBLE_DIR}" + cd "${ANSIBLE_DIR}" + run ansible-playbook playbook.yml -i "localhost," -c local --check --diff + + cd "${PROJECT_ROOT}" + run poetry run enroll manifest --harvest "${BUNDLE_DIR}" --out "${ANSIBLE_NO_COMMON_DIR}" --target ansible --no-common-roles + cd "${ANSIBLE_NO_COMMON_DIR}" + run ansible-playbook playbook.yml -i "localhost," -c local --check --diff + + cd "${PROJECT_ROOT}" + run poetry run enroll manifest --harvest "${BUNDLE_DIR}" --out "${ANSIBLE_FQDN_DIR}" --target ansible --fqdn "${TEST_FQDN}" + cd "${ANSIBLE_FQDN_DIR}" + run ansible-playbook "playbooks/${TEST_FQDN}.yml" -i inventory/hosts.ini -c local --limit "${TEST_FQDN}" --check --diff +} + +run_puppet_noop_tests() { + section "Puppet manifest noop tests" + ensure_puppet + cd "${PROJECT_ROOT}" + rm -rf "${PUPPET_DIR}" "${PUPPET_FQDN_DIR}" + + run poetry run enroll manifest --harvest "${BUNDLE_DIR}" --out "${PUPPET_DIR}" --target puppet + run puppet apply --modulepath "${PUPPET_DIR}/modules" "${PUPPET_DIR}/manifests/site.pp" --noop + + run poetry run enroll manifest --harvest "${BUNDLE_DIR}" --out "${PUPPET_FQDN_DIR}" --target puppet --fqdn "${TEST_FQDN}" + run puppet apply \ + --modulepath "${PUPPET_FQDN_DIR}/modules" \ + --hiera_config "${PUPPET_FQDN_DIR}/hiera.yaml" \ + --certname "${TEST_FQDN}" \ + "${PUPPET_FQDN_DIR}/manifests/site.pp" \ + --noop +} + +main() { + require_root + require_debian_ci + run_pytests + prepare_harvest_fixture + run_ansible_noop_tests + run_puppet_noop_tests +} + +main "$@" diff --git a/tests/test___main__.py b/tests/test___main__.py new file mode 100644 index 0000000..2e83ac1 --- /dev/null +++ b/tests/test___main__.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import runpy + + +def test_module_main_invokes_cli_main(monkeypatch): + import enroll.cli + + called = {"ok": False} + + def fake_main() -> None: + called["ok"] = True + + monkeypatch.setattr(enroll.cli, "main", fake_main) + + # Execute enroll.__main__ as if `python -m enroll`. + runpy.run_module("enroll.__main__", run_name="__main__") + assert called["ok"] is True diff --git a/tests/test_accounts.py b/tests/test_accounts.py new file mode 100644 index 0000000..a78c5f6 --- /dev/null +++ b/tests/test_accounts.py @@ -0,0 +1,496 @@ +from __future__ import annotations + +import os +from pathlib import Path + + +def test_parse_login_defs_parses_known_keys(tmp_path: Path): + from enroll.accounts import parse_login_defs + + p = tmp_path / "login.defs" + p.write_text( + """ + # comment + UID_MIN 1000 + UID_MAX 60000 + SYS_UID_MIN 100 + SYS_UID_MAX 999 + UID_MIN not_an_int + OTHER 123 + """, + encoding="utf-8", + ) + + vals = parse_login_defs(str(p)) + assert vals["UID_MIN"] == 1000 + assert vals["UID_MAX"] == 60000 + assert vals["SYS_UID_MIN"] == 100 + assert vals["SYS_UID_MAX"] == 999 + assert "OTHER" not in vals + + +def test_parse_passwd_and_group_and_ssh_files(tmp_path: Path): + from enroll.accounts import find_user_ssh_files, parse_group, parse_passwd + + passwd = tmp_path / "passwd" + passwd.write_text( + "\n".join( + [ + "root:x:0:0:root:/root:/bin/bash", + "# comment", + "alice:x:1000:1000:Alice:/home/alice:/bin/bash", + "bob:x:1001:1000:Bob:/home/bob:/usr/sbin/nologin", + "badline", + "cathy:x:notint:1000:Cathy:/home/cathy:/bin/bash", + "", + ] + ), + encoding="utf-8", + ) + + group = tmp_path / "group" + group.write_text( + "\n".join( + [ + "root:x:0:", + "users:x:1000:alice,bob", + "admins:x:1002:alice", + "badgroup:x:notint:alice", + "", + ] + ), + encoding="utf-8", + ) + + rows = parse_passwd(str(passwd)) + assert ("alice", 1000, 1000, "Alice", "/home/alice", "/bin/bash") in rows + assert all(r[0] != "cathy" for r in rows) # skipped invalid UID + + gid_to_name, name_to_gid, members = parse_group(str(group)) + assert gid_to_name[1000] == "users" + assert name_to_gid["admins"] == 1002 + assert "alice" in members["admins"] + + # ssh discovery: only authorized_keys, no symlinks + home = tmp_path / "home" / "alice" + sshdir = home / ".ssh" + sshdir.mkdir(parents=True) + ak = sshdir / "authorized_keys" + ak.write_text("ssh-ed25519 AAA...", encoding="utf-8") + # a symlink should be ignored + (sshdir / "authorized_keys2").write_text("x", encoding="utf-8") + os.symlink(str(sshdir / "authorized_keys2"), str(sshdir / "authorized_keys_link")) + assert find_user_ssh_files(str(home)) == [str(ak)] + + +def test_collect_non_system_users(monkeypatch, tmp_path: Path): + import enroll.accounts as a + + orig_parse_login_defs = a.parse_login_defs + orig_parse_passwd = a.parse_passwd + orig_parse_group = a.parse_group + + # Provide controlled passwd/group/login.defs inputs via monkeypatch. + passwd = tmp_path / "passwd" + passwd.write_text( + "\n".join( + [ + "root:x:0:0:root:/root:/bin/bash", + "nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin", + "alice:x:1000:1000:Alice:/home/alice:/bin/bash", + "sysuser:x:200:200:Sys:/home/sys:/bin/bash", + "bob:x:1001:1000:Bob:/home/bob:/bin/false", + "", + ] + ), + encoding="utf-8", + ) + group = tmp_path / "group" + group.write_text( + "\n".join( + [ + "users:x:1000:alice,bob", + "admins:x:1002:alice", + "", + ] + ), + encoding="utf-8", + ) + + defs = tmp_path / "login.defs" + defs.write_text("UID_MIN 1000\n", encoding="utf-8") + + monkeypatch.setattr( + a, "parse_login_defs", lambda path=str(defs): orig_parse_login_defs(path) + ) + monkeypatch.setattr( + a, "parse_passwd", lambda path=str(passwd): orig_parse_passwd(path) + ) + monkeypatch.setattr( + a, "parse_group", lambda path=str(group): orig_parse_group(path) + ) + + # Use a stable fake ssh discovery. + monkeypatch.setattr( + a, "find_user_ssh_files", lambda home: [f"{home}/.ssh/authorized_keys"] + ) + + users = a.collect_non_system_users() + assert [u.name for u in users] == ["alice"] + u = users[0] + assert u.primary_group == "users" + assert u.supplementary_groups == ["admins"] + assert u.ssh_files == ["/home/alice/.ssh/authorized_keys"] + + +def test_parse_login_defs_file_not_found(tmp_path: Path): + from enroll.accounts import parse_login_defs + + nonexistent = tmp_path / "nonexistent" / "login.defs" + vals = parse_login_defs(str(nonexistent)) + assert vals == {} + + +def test_parse_login_defs_handles_invalid_numbers(tmp_path: Path): + from enroll.accounts import parse_login_defs + + p = tmp_path / "login.defs" + p.write_text("UID_MIN not_a_number\nUID_MAX 60000\n", encoding="utf-8") + vals = parse_login_defs(str(p)) + assert "UID_MIN" not in vals + assert vals["UID_MAX"] == 60000 + + +def test_parse_group_handles_invalid_gid(tmp_path: Path): + from enroll.accounts import parse_group + + p = tmp_path / "group" + p.write_text( + "valid:x:1000:user1\n" "invalid_gid:x:notanint:user2\n", + encoding="utf-8", + ) + gid_to_name, name_to_gid, members = parse_group(str(p)) + assert 1000 in gid_to_name + assert gid_to_name[1000] == "valid" + assert "invalid_gid" not in name_to_gid + + +def test_parse_group_line_too_short(tmp_path: Path): + from enroll.accounts import parse_group + + p = tmp_path / "group" + p.write_text( + "valid:x:1000:user1\n" "shortline:x:1001\n", + encoding="utf-8", + ) + gid_to_name, name_to_gid, members = parse_group(str(p)) + assert 1000 in gid_to_name + assert 1001 not in gid_to_name + + +def test_is_human_user_filters_by_uid_and_shell(): + from enroll.accounts import is_human_user + + assert is_human_user(1000, "/bin/bash", 1000) is True + assert is_human_user(999, "/bin/bash", 1000) is False + assert is_human_user(1000, "/usr/sbin/nologin", 1000) is False + assert is_human_user(1000, "/usr/bin/nologin", 1000) is False + assert is_human_user(1000, "/bin/false", 1000) is False + assert is_human_user(1000, "", 1000) is True + + +def test_find_user_ssh_files_no_ssh_dir(tmp_path: Path): + from enroll.accounts import find_user_ssh_files + + home = tmp_path / "home" / "user" + home.mkdir(parents=True) + assert find_user_ssh_files(str(home)) == [] + + +def test_find_user_ssh_files_ignores_symlink(tmp_path: Path): + from enroll.accounts import find_user_ssh_files + + home = tmp_path / "home" / "user" + sshdir = home / ".ssh" + sshdir.mkdir(parents=True) + target = sshdir / "real_file" + target.write_text("x", encoding="utf-8") + os.symlink(str(target), str(sshdir / "authorized_keys")) + + result = find_user_ssh_files(str(home)) + assert result == [] + + +def test_find_user_ssh_files_handles_home_not_starting_with_slash(): + from enroll.accounts import find_user_ssh_files + + assert find_user_ssh_files("relative/path") == [] + assert find_user_ssh_files("") == [] + + +def test_collect_non_system_users_skips_nologin_users(tmp_path: Path): + import enroll.accounts as a + + orig_parse_login_defs = a.parse_login_defs + orig_parse_passwd = a.parse_passwd + orig_parse_group = a.parse_group + + passwd = tmp_path / "passwd" + passwd.write_text( + "root:x:0:0:root:/root:/bin/bash\n" + "alice:x:1000:1000:Alice:/home/alice:/bin/bash\n" + "nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin\n" + "sysuser:x:100:100:Sys:/home/sys:/bin/bash\n", + encoding="utf-8", + ) + group = tmp_path / "group" + group.write_text("users:x:1000:alice\n", encoding="utf-8") + defs = tmp_path / "login.defs" + defs.write_text("UID_MIN 1000\n", encoding="utf-8") + + monkeypatch_wrapper = lambda fn, p: lambda path=str(p): fn(path) + + a.parse_login_defs = monkeypatch_wrapper(orig_parse_login_defs, defs) + a.parse_passwd = monkeypatch_wrapper(orig_parse_passwd, passwd) + a.parse_group = monkeypatch_wrapper(orig_parse_group, group) + a.find_user_ssh_files = lambda home: [] + + users = a.collect_non_system_users() + assert [u.name for u in users] == ["alice"] + + +def test_collect_non_system_users_skips_below_uid_min(tmp_path: Path): + import enroll.accounts as a + + orig_parse_login_defs = a.parse_login_defs + orig_parse_passwd = a.parse_passwd + orig_parse_group = a.parse_group + + passwd = tmp_path / "passwd" + passwd.write_text( + "root:x:0:0:root:/root:/bin/bash\n" + "sysuser:x:999:999:Sys:/home/sys:/bin/bash\n" + "alice:x:1000:1000:Alice:/home/alice:/bin/bash\n", + encoding="utf-8", + ) + group = tmp_path / "group" + group.write_text("users:x:1000:alice\n", encoding="utf-8") + defs = tmp_path / "login.defs" + defs.write_text("UID_MIN 1000\n", encoding="utf-8") + + a.parse_login_defs = lambda path=str(defs): orig_parse_login_defs(path) + a.parse_passwd = lambda path=str(passwd): orig_parse_passwd(path) + a.parse_group = lambda path=str(group): orig_parse_group(path) + a.find_user_ssh_files = lambda home: [] + + users = a.collect_non_system_users() + assert [u.name for u in users] == ["alice"] + + +def test_parse_group_handles_empty_lines(tmp_path: Path): + from enroll.accounts import parse_group + + p = tmp_path / "group" + p.write_text( + "valid:x:1000:user1\n" "\n" "another:x:1001:user2\n", + encoding="utf-8", + ) + gid_to_name, name_to_gid, members = parse_group(str(p)) + assert 1000 in gid_to_name + assert 1001 in gid_to_name + + +def test_parse_group_handles_short_lines(tmp_path: Path): + from enroll.accounts import parse_group + + p = tmp_path / "group" + p.write_text( + "valid:x:1000:user1\n" "short:x:1001\n" "another:x:1002:user2\n", + encoding="utf-8", + ) + gid_to_name, name_to_gid, members = parse_group(str(p)) + assert 1000 in gid_to_name + assert 1001 not in gid_to_name # skipped due to short line + assert 1002 in gid_to_name + + +def test_find_flatpaks_in_root_detects_remote_branch_and_arch(tmp_path: Path): + import enroll.accounts as a + + root = tmp_path / "flatpak" + (root / "repo").mkdir(parents=True) + (root / "repo" / "config").write_text( + '[remote "acme"]\nurl=https://flatpak.example/repo/\n', + encoding="utf-8", + ) + ref = ( + root + / "repo" + / "refs" + / "remotes" + / "acme" + / "app" + / "com.example.App" + / "x86_64" + / "stable" + ) + ref.parent.mkdir(parents=True) + ref.write_text("checksum\n", encoding="utf-8") + active = root / "app" / "com.example.App" / "x86_64" / "stable" / "active" + active.mkdir(parents=True) + + remotes = a.find_flatpak_remotes(str(root), method="system") + assert [(r.name, r.url, r.method) for r in remotes] == [ + ("acme", "https://flatpak.example/repo/", "system") + ] + + apps = a._find_flatpaks_in_root(str(root), method="system") + assert len(apps) == 1 + assert apps[0].name == "com.example.App" + assert apps[0].remote == "acme" + assert apps[0].branch == "stable" + assert apps[0].arch == "x86_64" + + +def test_parse_snap_list_output_detects_channel_revision_and_modes(): + import enroll.accounts as a + + output = """Name Version Rev Tracking Publisher Notes +code abc 123 latest/stable vscode✓ classic +mydev 1.0 42 latest/edge example devmode,dangerous +bare 1.0 5 latest/stable canonical✓ base +""" + + snaps = {snap.name: snap for snap in a._parse_snap_list_output(output)} + assert snaps["code"].channel == "latest/stable" + assert snaps["code"].revision == 123 + assert snaps["code"].classic is True + assert snaps["mydev"].devmode is True + assert snaps["mydev"].dangerous is True + assert snaps["bare"].notes == ["base"] + + +def test_parse_flatpak_list_output_detects_system_refs(): + from enroll.accounts import _parse_flatpak_list_output + + output = "\n".join( + [ + "app/org.example.App/x86_64/stable\tflathub\tstable\tx86_64", + "runtime/org.freedesktop.Platform/x86_64/24.08\tflathub\t24.08\tx86_64", + ] + ) + + refs = _parse_flatpak_list_output( + output, method="system", columns=("ref", "origin", "branch", "arch") + ) + + assert [(r.kind, r.name, r.remote, r.branch, r.arch) for r in refs] == [ + ("app", "org.example.App", "flathub", "stable", "x86_64"), + ("runtime", "org.freedesktop.Platform", "flathub", "24.08", "x86_64"), + ] + assert refs[0].source == "flatpak-list" + + +def test_find_system_flatpaks_prefers_flatpak_list(monkeypatch): + import subprocess + import enroll.accounts as a + + calls = [] + + def fake_run(args, **kwargs): + calls.append(args) + if args == ["flatpak", "list", "--columns=help"]: + return subprocess.CompletedProcess( + args, + 0, + stdout="application\norigin\nbranch\narch\n", + stderr="", + ) + return subprocess.CompletedProcess( + args, + 0, + stdout="app/org.example.App/x86_64/stable\tacme\tstable\tx86_64\n", + stderr="", + ) + + monkeypatch.setattr(a.shutil, "which", lambda cmd: "/usr/bin/flatpak") + monkeypatch.setattr(a.subprocess, "run", fake_run) + monkeypatch.setattr( + a, + "_find_flatpaks_in_root", + lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("fallback used")), + ) + + refs = a.find_system_flatpaks() + + assert calls[0] == ["flatpak", "list", "--columns=help"] + assert calls[1][:3] == ["flatpak", "list", "--system"] + assert refs[0].name == "org.example.App" + assert refs[0].method == "system" + assert refs[0].remote == "acme" + + +def test_parse_flatpak_list_output_detects_application_columns(): + from enroll.accounts import _parse_flatpak_list_output + + output = "org.example.App\tflathub\tstable\tx86_64\n" + refs = _parse_flatpak_list_output( + output, method="system", columns=("application", "origin", "branch", "arch") + ) + + assert len(refs) == 1 + assert refs[0].name == "org.example.App" + assert refs[0].kind is None + assert refs[0].remote == "flathub" + assert refs[0].branch == "stable" + assert refs[0].arch == "x86_64" + + +def test_parse_plain_flatpak_list_output_like_default_table(): + from enroll.accounts import _parse_flatpak_list_output + + output = """Name Application ID Version Branch Installation +Mesa org.freedesktop.Platform.GL.default 26.0.6 25.08 system +Mesa (Extra) org.freedesktop.Platform.GL.default 26.0.6 25.08-extra system +Codecs Extra Extension org.freedesktop.Platform.codecs-extra 25.08-extra system +KDE Application Platform org.kde.Platform 6.10 system +OnionShare org.onionshare.OnionShare 2.6.4 stable system +""" + + refs = _parse_flatpak_list_output(output, method="system", columns=None) + by_name_branch = {(r.name, r.branch) for r in refs} + + assert ("org.onionshare.OnionShare", "stable") in by_name_branch + assert ("org.freedesktop.Platform.GL.default", "25.08") in by_name_branch + assert ("org.freedesktop.Platform.GL.default", "25.08-extra") in by_name_branch + assert ("org.kde.Platform", "6.10") in by_name_branch + + +def test_parse_flatpak_columns_help_handles_description_table(): + from enroll.accounts import _parse_flatpak_columns_help + + output = """ +Available columns: + application The application ID + branch The branch + installation The installation +""" + + assert _parse_flatpak_columns_help(output) >= { + "application", + "branch", + "installation", + } + + +def test_flatpak_list_attempts_respect_supported_columns(): + from enroll.accounts import _flatpak_list_attempts + + attempts = _flatpak_list_attempts( + "--system", {"application", "branch", "installation"} + ) + command_strings = [" ".join(args) for args, _columns in attempts] + + assert any("--columns=application,branch" in cmd for cmd in command_strings) + assert not any("origin" in cmd for cmd in command_strings) + assert command_strings[-1] == "flatpak list --system" diff --git a/tests/test_cache_security.py b/tests/test_cache_security.py new file mode 100644 index 0000000..4fda1e1 --- /dev/null +++ b/tests/test_cache_security.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + + +def test_ensure_dir_secure_refuses_symlink(tmp_path: Path): + from enroll.cache import _ensure_dir_secure + + target = tmp_path / "target" + target.mkdir() + link = tmp_path / "link" + link.symlink_to(target, target_is_directory=True) + + with pytest.raises(RuntimeError): + _ensure_dir_secure(link) + + +def test_ensure_dir_secure_ignores_chmod_failures(tmp_path: Path, monkeypatch): + from enroll.cache import _ensure_dir_secure + + d = tmp_path / "d" + + def boom(_path: str, _mode: int): + raise OSError("no") + + monkeypatch.setattr(os, "chmod", boom) + + # Should not raise. + _ensure_dir_secure(d) + assert d.exists() and d.is_dir() + + +def test_safe_component_returns_unknown_for_empty_string(): + from enroll.cache import _safe_component + + assert _safe_component("") == "unknown" + assert _safe_component(" ") == "unknown" + + +def test_safe_component_truncates_long_strings(): + from enroll.cache import _safe_component + + long_str = "a" * 100 + result = _safe_component(long_str) + assert len(result) <= 64 + + +def test_safe_component_replaces_special_chars(): + from enroll.cache import _safe_component + + result = _safe_component("hello world!") + assert result == "hello_world_" + + +def test_enroll_cache_dir_uses_xdg_cache_home(monkeypatch): + from enroll.cache import enroll_cache_dir + + monkeypatch.setenv("XDG_CACHE_HOME", "/custom/cache") + result = enroll_cache_dir() + assert str(result) == "/custom/cache/enroll" + + +def test_harvest_cache_state_json_property(): + from enroll.cache import HarvestCache + + cache_dir = HarvestCache(dir=Path("/tmp/test")) + assert cache_dir.state_json == Path("/tmp/test/state.json") + + +def test_new_harvest_cache_dir_chmod_fails(tmp_path: Path, monkeypatch): + from enroll.cache import new_harvest_cache_dir + + def fake_enroll_cache_dir(): + return tmp_path / "enroll" + + def fake_chmod(path, mode): + raise OSError("no") + + monkeypatch.setattr("enroll.cache.enroll_cache_dir", fake_enroll_cache_dir) + monkeypatch.setattr(os, "chmod", fake_chmod) + + # Should not raise even though chmod fails + cache = new_harvest_cache_dir(hint="test") + assert cache.dir.exists() + assert isinstance(cache.dir, Path) + + +def test_enroll_cache_dir_uses_default_when_xdg_not_set(monkeypatch): + from enroll.cache import enroll_cache_dir + + # Remove XDG_CACHE_HOME if it exists + monkeypatch.delenv("XDG_CACHE_HOME", raising=False) + result = enroll_cache_dir() + assert str(result).endswith("/.local/cache/enroll") diff --git a/tests/test_cli.py b/tests/test_cli.py index a93c509..ed3ffe6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,13 +1,29 @@ +from __future__ import annotations import sys +import pytest import enroll.cli as cli +from pathlib import Path + +from enroll.remote import RemoteSudoPasswordRequired +from enroll.sopsutil import SopsError + def test_cli_harvest_subcommand_calls_harvest(monkeypatch, capsys, tmp_path): called = {} - def fake_harvest(out: str): + def fake_harvest( + out: str, + dangerous: bool = False, + include_paths=None, + exclude_paths=None, + **_kwargs, + ): called["out"] = out + called["dangerous"] = dangerous + called["include_paths"] = include_paths or [] + called["exclude_paths"] = exclude_paths or [] return str(tmp_path / "state.json") monkeypatch.setattr(cli, "harvest", fake_harvest) @@ -15,6 +31,9 @@ def test_cli_harvest_subcommand_calls_harvest(monkeypatch, capsys, tmp_path): cli.main() assert called["out"] == str(tmp_path) + assert called["dangerous"] is False + assert called["include_paths"] == [] + assert called["exclude_paths"] == [] captured = capsys.readouterr() assert str(tmp_path / "state.json") in captured.out @@ -22,9 +41,14 @@ def test_cli_harvest_subcommand_calls_harvest(monkeypatch, capsys, tmp_path): def test_cli_manifest_subcommand_calls_manifest(monkeypatch, tmp_path): called = {} - def fake_manifest(harvest_dir: str, out_dir: str): + def fake_manifest(harvest_dir: str, out_dir: str, **kwargs): called["harvest"] = harvest_dir called["out"] = out_dir + # Common manifest args should be passed through by the CLI. + called["fqdn"] = kwargs.get("fqdn") + called["jinjaturtle"] = kwargs.get("jinjaturtle") + called["no_common_roles"] = kwargs.get("no_common_roles") + called["target"] = kwargs.get("target") monkeypatch.setattr(cli, "manifest", fake_manifest) monkeypatch.setattr( @@ -43,17 +67,96 @@ def test_cli_manifest_subcommand_calls_manifest(monkeypatch, tmp_path): cli.main() assert called["harvest"] == str(tmp_path / "bundle") assert called["out"] == str(tmp_path / "ansible") + assert called["fqdn"] is None + assert called["jinjaturtle"] == "auto" + assert called["no_common_roles"] is False + assert called["target"] == "ansible" + + +def test_cli_manifest_target_puppet_is_forwarded(monkeypatch, tmp_path): + called = {} + + def fake_manifest(harvest_dir: str, out_dir: str, **kwargs): + called["harvest"] = harvest_dir + called["out"] = out_dir + called["target"] = kwargs.get("target") + + monkeypatch.setattr(cli, "manifest", fake_manifest) + monkeypatch.setattr( + sys, + "argv", + [ + "enroll", + "manifest", + "--harvest", + str(tmp_path / "bundle"), + "--out", + str(tmp_path / "puppet"), + "--target", + "puppet", + ], + ) + + cli.main() + assert called["harvest"] == str(tmp_path / "bundle") + assert called["out"] == str(tmp_path / "puppet") + assert called["target"] == "puppet" + + +def test_cli_manifest_no_common_roles_is_forwarded(monkeypatch, tmp_path): + called = {} + + def fake_manifest(harvest_dir: str, out_dir: str, **kwargs): + called["harvest"] = harvest_dir + called["out"] = out_dir + called["no_common_roles"] = kwargs.get("no_common_roles") + + monkeypatch.setattr(cli, "manifest", fake_manifest) + monkeypatch.setattr( + sys, + "argv", + [ + "enroll", + "manifest", + "--harvest", + str(tmp_path / "bundle"), + "--out", + str(tmp_path / "ansible"), + "--no-common-roles", + ], + ) + + cli.main() + assert called["harvest"] == str(tmp_path / "bundle") + assert called["out"] == str(tmp_path / "ansible") + assert called["no_common_roles"] is True def test_cli_enroll_subcommand_runs_harvest_then_manifest(monkeypatch, tmp_path): calls = [] - def fake_harvest(bundle_dir: str): - calls.append(("harvest", bundle_dir)) + def fake_harvest( + bundle_dir: str, + dangerous: bool = False, + include_paths=None, + exclude_paths=None, + **_kwargs, + ): + calls.append( + ("harvest", bundle_dir, dangerous, include_paths or [], exclude_paths or []) + ) return str(tmp_path / "bundle" / "state.json") - def fake_manifest(bundle_dir: str, out_dir: str): - calls.append(("manifest", bundle_dir, out_dir)) + def fake_manifest(bundle_dir: str, out_dir: str, **kwargs): + calls.append( + ( + "manifest", + bundle_dir, + out_dir, + kwargs.get("fqdn"), + kwargs.get("jinjaturtle"), + ) + ) monkeypatch.setattr(cli, "harvest", fake_harvest) monkeypatch.setattr(cli, "manifest", fake_manifest) @@ -62,7 +165,7 @@ def test_cli_enroll_subcommand_runs_harvest_then_manifest(monkeypatch, tmp_path) "argv", [ "enroll", - "enroll", + "single-shot", "--harvest", str(tmp_path / "bundle"), "--out", @@ -72,6 +175,577 @@ def test_cli_enroll_subcommand_runs_harvest_then_manifest(monkeypatch, tmp_path) cli.main() assert calls == [ - ("harvest", str(tmp_path / "bundle")), - ("manifest", str(tmp_path / "bundle"), str(tmp_path / "ansible")), + ("harvest", str(tmp_path / "bundle"), False, [], []), + ("manifest", str(tmp_path / "bundle"), str(tmp_path / "ansible"), None, "auto"), ] + + +def test_cli_harvest_dangerous_flag_is_forwarded(monkeypatch, tmp_path): + called = {} + + def fake_harvest( + out: str, + dangerous: bool = False, + include_paths=None, + exclude_paths=None, + **_kwargs, + ): + called["out"] = out + called["dangerous"] = dangerous + called["include_paths"] = include_paths or [] + called["exclude_paths"] = exclude_paths or [] + return str(tmp_path / "state.json") + + monkeypatch.setattr(cli, "harvest", fake_harvest) + monkeypatch.setattr( + sys, "argv", ["enroll", "harvest", "--out", str(tmp_path), "--dangerous"] + ) + + cli.main() + assert called["dangerous"] is True + assert called["include_paths"] == [] + assert called["exclude_paths"] == [] + + +def test_cli_harvest_remote_calls_remote_harvest_and_uses_cache_dir( + monkeypatch, capsys, tmp_path +): + from enroll.cache import HarvestCache + + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + + called = {} + + def fake_cache_dir(*, hint=None): + called["hint"] = hint + return HarvestCache(dir=cache_dir) + + def fake_remote_harvest( + *, + local_out_dir, + remote_host, + remote_port, + remote_user, + dangerous, + no_sudo, + include_paths=None, + exclude_paths=None, + **_kwargs, + ): + called.update( + { + "local_out_dir": local_out_dir, + "remote_host": remote_host, + "remote_port": remote_port, + "remote_user": remote_user, + "dangerous": dangerous, + "no_sudo": no_sudo, + "include_paths": include_paths or [], + "exclude_paths": exclude_paths or [], + } + ) + return cache_dir / "state.json" + + monkeypatch.setattr(cli, "new_harvest_cache_dir", fake_cache_dir) + monkeypatch.setattr(cli, "remote_harvest", fake_remote_harvest) + monkeypatch.setattr( + sys, + "argv", + [ + "enroll", + "harvest", + "--remote-host", + "example.test", + "--remote-user", + "alice", + ], + ) + + cli.main() + out = capsys.readouterr().out + assert str(cache_dir / "state.json") in out + assert called["hint"] == "example.test" + assert called["local_out_dir"] == cache_dir + assert called["remote_host"] == "example.test" + assert called["remote_port"] == 22 + assert called["remote_user"] == "alice" + assert called["dangerous"] is False + assert called["no_sudo"] is False + assert called["include_paths"] == [] + assert called["exclude_paths"] == [] + + +def test_cli_single_shot_remote_without_harvest_prints_state_path( + monkeypatch, capsys, tmp_path +): + from enroll.cache import HarvestCache + + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + ansible_dir = tmp_path / "ansible" + + calls = [] + + def fake_cache_dir(*, hint=None): + return HarvestCache(dir=cache_dir) + + def fake_remote_harvest(**kwargs): + calls.append(("remote_harvest", kwargs)) + return cache_dir / "state.json" + + def fake_manifest(harvest_dir: str, out_dir: str, **kwargs): + calls.append(("manifest", harvest_dir, out_dir, kwargs.get("fqdn"))) + + monkeypatch.setattr(cli, "new_harvest_cache_dir", fake_cache_dir) + monkeypatch.setattr(cli, "remote_harvest", fake_remote_harvest) + monkeypatch.setattr(cli, "manifest", fake_manifest) + monkeypatch.setattr( + sys, + "argv", + [ + "enroll", + "single-shot", + "--remote-host", + "example.test", + "--remote-user", + "alice", + "--out", + str(ansible_dir), + "--fqdn", + "example.test", + ], + ) + + cli.main() + out = capsys.readouterr().out + + # It should print the derived state.json path for usability when --harvest + # wasn't provided. + assert str(cache_dir / "state.json") in out + + # And it should manifest using the cache dir. + assert ("manifest", str(cache_dir), str(ansible_dir), "example.test") in calls + + +def test_cli_harvest_remote_ask_become_pass_prompts_and_passes_password( + monkeypatch, tmp_path +): + from enroll.cache import HarvestCache + import enroll.remote as r + + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + + called = {} + + def fake_cache_dir(*, hint=None): + return HarvestCache(dir=cache_dir) + + def fake__remote_harvest(*, sudo_password=None, **kwargs): + called["sudo_password"] = sudo_password + return cache_dir / "state.json" + + monkeypatch.setattr(cli, "new_harvest_cache_dir", fake_cache_dir) + monkeypatch.setattr(r, "_remote_harvest", fake__remote_harvest) + monkeypatch.setattr(r.getpass, "getpass", lambda _prompt="": "pw123") + + monkeypatch.setattr( + sys, + "argv", + [ + "enroll", + "harvest", + "--remote-host", + "example.test", + "--ask-become-pass", + ], + ) + + cli.main() + assert called["sudo_password"] == "pw123" + + +def test_cli_harvest_remote_password_required_fallback_prompts_and_retries( + monkeypatch, tmp_path +): + from enroll.cache import HarvestCache + import enroll.remote as r + + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + + def fake_cache_dir(*, hint=None): + return HarvestCache(dir=cache_dir) + + calls = [] + + def fake__remote_harvest(*, sudo_password=None, **kwargs): + calls.append(sudo_password) + if sudo_password is None: + raise r.RemoteSudoPasswordRequired("pw required") + return cache_dir / "state.json" + + class _TTYStdin: + def isatty(self): + return True + + monkeypatch.setattr(cli, "new_harvest_cache_dir", fake_cache_dir) + monkeypatch.setattr(r, "_remote_harvest", fake__remote_harvest) + monkeypatch.setattr(r.getpass, "getpass", lambda _prompt="": "pw456") + monkeypatch.setattr(sys, "stdin", _TTYStdin()) + + monkeypatch.setattr( + sys, "argv", ["enroll", "harvest", "--remote-host", "example.test"] + ) + + cli.main() + assert calls == [None, "pw456"] + + +def test_cli_harvest_remote_password_required_noninteractive_errors( + monkeypatch, tmp_path +): + from enroll.cache import HarvestCache + import enroll.remote as r + + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + + def fake_cache_dir(*, hint=None): + return HarvestCache(dir=cache_dir) + + def fake__remote_harvest(*, sudo_password=None, **kwargs): + raise r.RemoteSudoPasswordRequired("pw required") + + class _NoTTYStdin: + def isatty(self): + return False + + monkeypatch.setattr(cli, "new_harvest_cache_dir", fake_cache_dir) + monkeypatch.setattr(r, "_remote_harvest", fake__remote_harvest) + monkeypatch.setattr(sys, "stdin", _NoTTYStdin()) + + monkeypatch.setattr( + sys, "argv", ["enroll", "harvest", "--remote-host", "example.test"] + ) + + with pytest.raises(SystemExit) as e: + cli.main() + assert "--ask-become-pass" in str(e.value) + + +def test_cli_manifest_common_args(monkeypatch, tmp_path): + """Ensure --fqdn and jinjaturtle mode flags are forwarded correctly.""" + + called = {} + + def fake_manifest(harvest_dir: str, out_dir: str, **kwargs): + called["harvest"] = harvest_dir + called["out"] = out_dir + called["fqdn"] = kwargs.get("fqdn") + called["jinjaturtle"] = kwargs.get("jinjaturtle") + + monkeypatch.setattr(cli, "manifest", fake_manifest) + monkeypatch.setattr( + sys, + "argv", + [ + "enroll", + "manifest", + "--harvest", + str(tmp_path / "bundle"), + "--out", + str(tmp_path / "ansible"), + "--fqdn", + "example.test", + "--no-jinjaturtle", + ], + ) + + cli.main() + assert called["fqdn"] == "example.test" + assert called["jinjaturtle"] == "off" + + +def test_cli_explain_passes_args_and_writes_stdout(monkeypatch, capsys, tmp_path): + called = {} + + def fake_explain_state( + harvest: str, + *, + sops_mode: bool = False, + fmt: str = "text", + max_examples: int = 3, + ): + called["harvest"] = harvest + called["sops_mode"] = sops_mode + called["fmt"] = fmt + called["max_examples"] = max_examples + return "EXPLAINED\n" + + monkeypatch.setattr(cli, "explain_state", fake_explain_state) + + monkeypatch.setattr( + sys, + "argv", + [ + "enroll", + "explain", + "--sops", + "--format", + "json", + "--max-examples", + "7", + str(tmp_path / "bundle" / "state.json"), + ], + ) + + cli.main() + out = capsys.readouterr().out + assert out == "EXPLAINED\n" + assert called["sops_mode"] is True + assert called["fmt"] == "json" + assert called["max_examples"] == 7 + + +def test_discover_config_path_missing_config_value_returns_none(monkeypatch): + # Covers the "--config" flag present with no value. + monkeypatch.delenv("ENROLL_CONFIG", raising=False) + monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) + assert cli._discover_config_path(["--config"]) is None + + +def test_discover_config_path_defaults_to_home_config(monkeypatch, tmp_path: Path): + # Covers the Path.home() / ".config" fallback. + monkeypatch.delenv("ENROLL_CONFIG", raising=False) + monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) + monkeypatch.setattr(cli.Path, "home", lambda: tmp_path) + monkeypatch.setattr(cli.Path, "cwd", lambda: tmp_path) + + cp = tmp_path / ".config" / "enroll" / "enroll.ini" + cp.parent.mkdir(parents=True) + cp.write_text("[enroll]\n", encoding="utf-8") + + assert cli._discover_config_path(["harvest"]) == cp + + +def test_cli_harvest_local_sops_encrypts_and_prints_path( + monkeypatch, tmp_path: Path, capsys +): + out_dir = tmp_path / "out" + out_dir.mkdir() + calls: dict[str, object] = {} + + def fake_harvest(bundle_dir: str, **kwargs): + calls["bundle"] = bundle_dir + # Create a minimal state.json so tooling that expects it won't break. + Path(bundle_dir).mkdir(parents=True, exist_ok=True) + (Path(bundle_dir) / "state.json").write_text("{}", encoding="utf-8") + return str(Path(bundle_dir) / "state.json") + + def fake_encrypt(bundle_dir: Path, out_file: Path, fps: list[str]): + calls["encrypt"] = (bundle_dir, out_file, fps) + out_file.write_text("encrypted", encoding="utf-8") + return out_file + + monkeypatch.setattr(cli, "harvest", fake_harvest) + monkeypatch.setattr(cli, "_encrypt_harvest_dir_to_sops", fake_encrypt) + + monkeypatch.setattr( + sys, + "argv", + [ + "enroll", + "harvest", + "--sops", + "ABCDEF", + "--out", + str(out_dir), + ], + ) + cli.main() + + printed = capsys.readouterr().out.strip() + assert printed.endswith("harvest.tar.gz.sops") + assert Path(printed).exists() + assert calls.get("encrypt") + + +def test_cli_harvest_remote_sops_encrypts_and_prints_path( + monkeypatch, tmp_path: Path, capsys +): + out_dir = tmp_path / "out" + out_dir.mkdir() + calls: dict[str, object] = {} + + def fake_remote_harvest(**kwargs): + calls["remote"] = kwargs + # Create a minimal state.json in the temp bundle. + out = Path(kwargs["local_out_dir"]) / "state.json" + out.write_text("{}", encoding="utf-8") + return out + + def fake_encrypt(bundle_dir: Path, out_file: Path, fps: list[str]): + calls["encrypt"] = (bundle_dir, out_file, fps) + out_file.write_text("encrypted", encoding="utf-8") + return out_file + + monkeypatch.setattr(cli, "remote_harvest", fake_remote_harvest) + monkeypatch.setattr(cli, "_encrypt_harvest_dir_to_sops", fake_encrypt) + + monkeypatch.setattr( + sys, + "argv", + [ + "enroll", + "harvest", + "--remote-host", + "example.com", + "--remote-user", + "root", + "--sops", + "ABCDEF", + "--out", + str(out_dir), + ], + ) + cli.main() + + printed = capsys.readouterr().out.strip() + assert printed.endswith("harvest.tar.gz.sops") + assert Path(printed).exists() + assert calls.get("remote") + assert calls.get("encrypt") + + +def test_cli_harvest_remote_password_required_exits_cleanly(monkeypatch): + def boom(**kwargs): + raise RemoteSudoPasswordRequired("pw required") + + monkeypatch.setattr(cli, "remote_harvest", boom) + monkeypatch.setattr( + sys, + "argv", + [ + "enroll", + "harvest", + "--remote-host", + "example.com", + "--remote-user", + "root", + ], + ) + with pytest.raises(SystemExit) as e: + cli.main() + assert "--ask-become-pass" in str(e.value) + + +def test_cli_runtime_error_is_wrapped_as_user_friendly_system_exit(monkeypatch): + def boom(*args, **kwargs): + raise RuntimeError("nope") + + monkeypatch.setattr(cli, "harvest", boom) + monkeypatch.setattr(sys, "argv", ["enroll", "harvest", "--out", "/tmp/x"]) + with pytest.raises(SystemExit) as e: + cli.main() + assert str(e.value) == "error: nope" + + +def test_cli_sops_error_is_wrapped_as_user_friendly_system_exit(monkeypatch): + def boom(*args, **kwargs): + raise SopsError("sops broke") + + monkeypatch.setattr(cli, "manifest", boom) + monkeypatch.setattr( + sys, "argv", ["enroll", "manifest", "--harvest", "/tmp/x", "--out", "/tmp/y"] + ) + with pytest.raises(SystemExit) as e: + cli.main() + assert str(e.value) == "error: sops broke" + + +def test_cli_diff_notifies_webhook_and_email_and_respects_exit_code( + monkeypatch, capsys +): + calls: dict[str, object] = {} + + def fake_compare(old, new, sops_mode=False, **kwargs): + calls["compare"] = (old, new, sops_mode) + return {"dummy": True}, True + + def fake_format(report, fmt="text"): + calls.setdefault("format", []).append((report, fmt)) + return "REPORT\n" + + def fake_post(url, body, headers=None): + calls["webhook"] = (url, body, headers) + return 200, b"ok" + + def fake_email(**kwargs): + calls["email"] = kwargs + + monkeypatch.setattr(cli, "compare_harvests", fake_compare) + monkeypatch.setattr(cli, "format_report", fake_format) + monkeypatch.setattr(cli, "post_webhook", fake_post) + monkeypatch.setattr(cli, "send_email", fake_email) + monkeypatch.setenv("SMTPPW", "secret") + + monkeypatch.setattr( + sys, + "argv", + [ + "enroll", + "diff", + "--old", + "/tmp/old", + "--new", + "/tmp/new", + "--webhook", + "https://example.invalid/h", + "--webhook-header", + "X-Test: ok", + "--email-to", + "a@example.com", + "--smtp-password-env", + "SMTPPW", + "--exit-code", + ], + ) + + with pytest.raises(SystemExit) as e: + cli.main() + assert e.value.code == 2 + + assert calls.get("compare") + assert calls.get("webhook") + assert calls.get("email") + # No report printed when exiting via --exit-code? (we still render and print). + _ = capsys.readouterr() + + +def test_cli_diff_webhook_http_error_raises_system_exit(monkeypatch): + def fake_compare(old, new, sops_mode=False, **kwargs): + return {"dummy": True}, True + + monkeypatch.setattr(cli, "compare_harvests", fake_compare) + monkeypatch.setattr(cli, "format_report", lambda report, fmt="text": "R\n") + monkeypatch.setattr(cli, "post_webhook", lambda url, body, headers=None: (500, b"")) + + monkeypatch.setattr( + sys, + "argv", + [ + "enroll", + "diff", + "--old", + "/tmp/old", + "--new", + "/tmp/new", + "--webhook", + "https://example.invalid/h", + ], + ) + with pytest.raises(SystemExit) as e: + cli.main() + assert "HTTP 500" in str(e.value) diff --git a/tests/test_cli_config_and_sops.py b/tests/test_cli_config_and_sops.py new file mode 100644 index 0000000..7e3fe5b --- /dev/null +++ b/tests/test_cli_config_and_sops.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import argparse +import configparser +import tarfile +from pathlib import Path + + +def test_discover_config_path_precedence(monkeypatch, tmp_path: Path): + from enroll.cli import _discover_config_path + + cfg = tmp_path / "cfg.ini" + cfg.write_text("[enroll]\n", encoding="utf-8") + + # --no-config always wins + monkeypatch.setenv("ENROLL_CONFIG", str(cfg)) + assert _discover_config_path(["--no-config", "harvest"]) is None + + # explicit --config wins + assert _discover_config_path(["--config", str(cfg), "harvest"]) == cfg + + # env var used when present + assert _discover_config_path(["harvest"]) == cfg + + +def test_discover_config_path_finds_local_and_xdg(monkeypatch, tmp_path: Path): + from enroll.cli import _discover_config_path + + # local file in cwd + cwd = tmp_path / "cwd" + cwd.mkdir() + local = cwd / "enroll.ini" + local.write_text("[enroll]\n", encoding="utf-8") + + monkeypatch.chdir(cwd) + monkeypatch.delenv("ENROLL_CONFIG", raising=False) + monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) + assert _discover_config_path(["harvest"]) == local + + # xdg config fallback + monkeypatch.chdir(tmp_path) + xdg = tmp_path / "xdg" + (xdg / "enroll").mkdir(parents=True) + xcfg = xdg / "enroll" / "enroll.ini" + xcfg.write_text("[enroll]\n", encoding="utf-8") + monkeypatch.setenv("XDG_CONFIG_HOME", str(xdg)) + assert _discover_config_path(["harvest"]) == xcfg + + +def test_section_to_argv_supports_bool_append_count_and_unknown(monkeypatch, capsys): + from enroll.cli import _section_to_argv + + ap = argparse.ArgumentParser(add_help=False) + ap.add_argument("--flag", action="store_true") + ap.add_argument("--no-flag", action="store_false", dest="flag2") + ap.add_argument("--item", action="append", default=[]) + ap.add_argument("-v", action="count", default=0) + + cfg = configparser.ConfigParser() + cfg.read_dict( + { + "enroll": { + "flag": "true", + "no_flag": "false", + "item": "a,b", + "v": "2", + "unknown_key": "zzz", + } + } + ) + + argv = _section_to_argv(ap, cfg, "enroll") + + # bools set + assert "--flag" in argv + assert "--no-flag" in argv + + # append expanded + assert argv.count("--item") == 2 + assert "a" in argv and "b" in argv + + # count flag expanded + assert argv.count("-v") == 2 + + # unknown key prints warning + err = capsys.readouterr().err + assert "unknown option" in err + + +def test_inject_config_argv_inserts_global_and_command_tokens(tmp_path: Path): + from enroll.cli import _inject_config_argv + + root = argparse.ArgumentParser(add_help=False) + root.add_argument("--root-flag", action="store_true") + sub = root.add_subparsers(dest="cmd", required=True) + p_h = sub.add_parser("harvest", add_help=False) + p_h.add_argument("--dangerous", action="store_true") + p_h.add_argument("--include-path", action="append", default=[]) + + cfg_path = tmp_path / "enroll.ini" + cfg_path.write_text( + """[enroll] +root-flag = true + +[harvest] +dangerous = true +include-path = /etc/one,/etc/two +""", + encoding="utf-8", + ) + + argv = ["harvest", "--include-path", "/etc/cli"] + injected = _inject_config_argv( + argv, + cfg_path=cfg_path, + root_parser=root, + subparsers={"harvest": p_h}, + ) + + # global inserted before cmd, subcommand tokens right after cmd + assert injected[:2] == ["--root-flag", "harvest"] + # include-path from config inserted before CLI include-path (CLI wins later if duplicates) + joined = " ".join(injected) + assert "--include-path /etc/one" in joined + assert "--include-path /etc/cli" in joined + + +def test_resolve_sops_out_file_and_encrypt_path(monkeypatch, tmp_path: Path): + from enroll import cli + + # directory output should yield harvest.tar.gz.sops inside + out_dir = tmp_path / "o" + out_dir.mkdir() + assert ( + cli._resolve_sops_out_file(str(out_dir), hint="h").name == "harvest.tar.gz.sops" + ) + + # file-like output retained + out_file = tmp_path / "x.sops" + assert cli._resolve_sops_out_file(str(out_file), hint="h") == out_file + + # None uses cache dir + class HC: + def __init__(self, d: Path): + self.dir = d + + monkeypatch.setattr( + cli, "new_harvest_cache_dir", lambda hint: HC(tmp_path / "cache") + ) + p = cli._resolve_sops_out_file(None, hint="h") + assert str(p).endswith("harvest.tar.gz.sops") + + # Cover _tar_dir_to quickly (writes a tarball) + bundle = tmp_path / "bundle" + bundle.mkdir() + (bundle / "state.json").write_text("{}", encoding="utf-8") + tar_path = tmp_path / "b.tar.gz" + cli._tar_dir_to(bundle, tar_path) + assert tar_path.exists() + with tarfile.open(tar_path, "r:gz") as tf: + names = tf.getnames() + assert "state.json" in names or "./state.json" in names + + +def test_encrypt_harvest_dir_to_sops_cleans_up_tmp_tgz(monkeypatch, tmp_path: Path): + from enroll.cli import _encrypt_harvest_dir_to_sops + + bundle = tmp_path / "bundle" + bundle.mkdir() + (bundle / "state.json").write_text("{}", encoding="utf-8") + out_file = tmp_path / "out.sops" + + seen = {} + + def fake_encrypt(src: Path, dst: Path, pgp_fingerprints, mode): # noqa: ARG001 + # write something so we can see output created + seen["src"] = src + dst.write_bytes(b"enc") + + monkeypatch.setattr("enroll.cli.encrypt_file_binary", fake_encrypt) + + # Make os.unlink raise FileNotFoundError to hit the except branch in finally. + monkeypatch.setattr( + "enroll.cli.os.unlink", lambda p: (_ for _ in ()).throw(FileNotFoundError()) + ) + + res = _encrypt_harvest_dir_to_sops(bundle, out_file, fps=["ABC"]) + assert res == out_file + assert out_file.read_bytes() == b"enc" diff --git a/tests/test_cli_helpers.py b/tests/test_cli_helpers.py new file mode 100644 index 0000000..264ff85 --- /dev/null +++ b/tests/test_cli_helpers.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import argparse +import configparser +import types +import textwrap +from pathlib import Path + + +def test_discover_config_path_precedence(tmp_path: Path, monkeypatch): + """_discover_config_path: --config > ENROLL_CONFIG > ./enroll.ini > XDG.""" + from enroll.cli import _discover_config_path + + cfg1 = tmp_path / "one.ini" + cfg1.write_text("[enroll]\n", encoding="utf-8") + + # Explicit --config should win. + assert _discover_config_path(["--config", str(cfg1)]) == cfg1 + + # --no-config disables config loading. + assert _discover_config_path(["--no-config", "--config", str(cfg1)]) is None + + monkeypatch.chdir(tmp_path) + + cfg2 = tmp_path / "two.ini" + cfg2.write_text("[enroll]\n", encoding="utf-8") + monkeypatch.setenv("ENROLL_CONFIG", str(cfg2)) + assert _discover_config_path([]) == cfg2 + + # Local ./enroll.ini fallback. + monkeypatch.delenv("ENROLL_CONFIG", raising=False) + local = tmp_path / "enroll.ini" + local.write_text("[enroll]\n", encoding="utf-8") + assert _discover_config_path([]) == local + + # XDG fallback. + local.unlink() + xdg = tmp_path / "xdg" + cfg3 = xdg / "enroll" / "enroll.ini" + cfg3.parent.mkdir(parents=True) + cfg3.write_text("[enroll]\n", encoding="utf-8") + monkeypatch.setenv("XDG_CONFIG_HOME", str(xdg)) + assert _discover_config_path([]) == cfg3 + + +def test_config_value_parsing_and_list_splitting(): + from enroll.cli import _parse_bool, _split_list_value + + assert _parse_bool("1") is True + assert _parse_bool("yes") is True + assert _parse_bool("false") is False + + assert _parse_bool("maybe") is None + + assert _split_list_value("a,b , c") == ["a", "b", "c"] + # When newlines are present, we split on lines (not commas within a line). + assert _split_list_value("a,b\nc") == ["a,b", "c"] + assert _split_list_value("a\n\n b\n") == ["a", "b"] + assert _split_list_value(" ") == [] + + +def test_section_to_argv_handles_types_and_unknown_keys(capsys): + from enroll.cli import _section_to_argv + + p = argparse.ArgumentParser(add_help=False) + p.add_argument("--dangerous", action="store_true") + p.add_argument("--no-color", dest="color", action="store_false") + p.add_argument("--include-path", dest="include_path", action="append") + p.add_argument("-v", action="count", default=0) + p.add_argument("--out") + + cfg = configparser.ConfigParser() + cfg.read_dict( + { + "harvest": { + "dangerous": "true", + # Keys are matched by argparse dest; store_false actions still use dest. + "color": "false", + "include-path": "a,b,c", + "v": "2", + "out": "/tmp/bundle", + "unknown": "ignored", + } + } + ) + + argv = _section_to_argv(p, cfg, "harvest") + + # Boolean store_true. + assert "--dangerous" in argv + + # Boolean store_false: include the flag only when config wants False. + assert "--no-color" in argv + + # Append: split lists and add one flag per item. + assert argv.count("--include-path") == 3 + assert "a" in argv and "b" in argv and "c" in argv + + # Count: repeats. + assert argv.count("-v") == 2 + + # Scalar. + assert "--out" in argv and "/tmp/bundle" in argv + + err = capsys.readouterr().err + assert "unknown option" in err + + +def test_inject_config_argv_inserts_global_and_subcommand(tmp_path: Path, capsys): + from enroll.cli import _inject_config_argv + + cfg = tmp_path / "enroll.ini" + cfg.write_text( + textwrap.dedent( + """ + [enroll] + dangerous = true + + [harvest] + include-path = /etc/foo + unknown = 1 + """ + ).strip() + + "\n", + encoding="utf-8", + ) + + root = argparse.ArgumentParser(add_help=False) + root.add_argument("--dangerous", action="store_true") + + harvest_p = argparse.ArgumentParser(add_help=False) + harvest_p.add_argument("--include-path", dest="include_path", action="append") + + argv = _inject_config_argv( + ["harvest", "--out", "x"], + cfg_path=cfg, + root_parser=root, + subparsers={"harvest": harvest_p}, + ) + + # Global tokens should appear before the subcommand. + assert argv[0] == "--dangerous" + assert argv[1] == "harvest" + + # Subcommand tokens should appear right after the subcommand. + assert argv[2:4] == ["--include-path", "/etc/foo"] + + # Unknown option should have produced a warning. + assert "unknown option" in capsys.readouterr().err + + +def test_resolve_sops_out_file(tmp_path: Path, monkeypatch): + from enroll import cli + + # Make a predictable cache dir for the default case. + fake_cache = types.SimpleNamespace(dir=tmp_path / "cache") + fake_cache.dir.mkdir(parents=True) + monkeypatch.setattr(cli, "new_harvest_cache_dir", lambda hint=None: fake_cache) + + # If out is a directory, use it directly. + out_dir = tmp_path / "out" + out_dir.mkdir() + # The output filename is fixed; hint is only used when creating a cache dir. + assert ( + cli._resolve_sops_out_file(out=out_dir, hint="bundle.tar.gz") + == out_dir / "harvest.tar.gz.sops" + ) + + # If out is a file path, keep it. + out_file = tmp_path / "x.sops" + assert cli._resolve_sops_out_file(out=out_file, hint="bundle.tar.gz") == out_file + + # None uses the cache dir, and the name is fixed. + assert ( + cli._resolve_sops_out_file(out=None, hint="bundle.tar.gz") + == fake_cache.dir / "harvest.tar.gz.sops" + ) diff --git a/tests/test_cm.py b/tests/test_cm.py new file mode 100644 index 0000000..89addbf --- /dev/null +++ b/tests/test_cm.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from enroll.cm import CMModule, resolve_catalog_conflicts + + +def test_resolve_catalog_conflicts_dedupes_before_rendering(): + first = CMModule(role_name="admin", module_name="admin") + first.packages.add("curl") + first.dirs["/etc/default"] = {"owner": "root"} + first.files["/etc/foo.conf"] = {"owner": "root"} + + second = CMModule(role_name="misc", module_name="misc") + second.packages.add("curl") + second.dirs["/etc/default"] = {"owner": "root"} + second.dirs["/etc/foo.conf"] = {"owner": "root"} + second.files["/etc/foo.conf"] = {"owner": "root"} + + resolve_catalog_conflicts([first, second]) + + assert first.packages == {"curl"} + assert "/etc/default" in first.dirs + assert "/etc/foo.conf" in first.files + + assert second.packages == set() + assert second.dirs == {} + assert second.files == {} + assert any("duplicate Package[curl]" in note for note in second.notes) + assert any("duplicate File[/etc/default]" in note for note in second.notes) + assert any("a file or link with the same path" in note for note in second.notes) + + +def test_cm_module_uses_shared_state_io(tmp_path): + state = {"roles": {"packages": []}} + + written = CMModule.write_state(tmp_path, state) + + assert written == tmp_path / "state.json" + assert CMModule.state_path(tmp_path) == written + assert CMModule.load_state(tmp_path) == state + assert CMModule._load_state(tmp_path) == state diff --git a/tests/test_debian.py b/tests/test_debian.py new file mode 100644 index 0000000..64fe420 --- /dev/null +++ b/tests/test_debian.py @@ -0,0 +1,538 @@ +from __future__ import annotations + +from pathlib import Path +import pytest + + +def test_dpkg_owner_parses_output(monkeypatch): + import enroll.debian as d + + class P: + def __init__(self, rc: int, out: str): + self.returncode = rc + self.stdout = out + self.stderr = "" + + def fake_run(cmd, text, capture_output): + assert cmd[:2] == ["dpkg", "-S"] + return P( + 0, + """ + diversion by foo from: /etc/something + nginx-common:amd64: /etc/nginx/nginx.conf + nginx-common, nginx: /etc/nginx/sites-enabled/default + """, + ) + + monkeypatch.setattr(d.subprocess, "run", fake_run) + assert d.dpkg_owner("/etc/nginx/nginx.conf") == "nginx-common" + + def fake_run_none(cmd, text, capture_output): + return P(1, "") + + monkeypatch.setattr(d.subprocess, "run", fake_run_none) + assert d.dpkg_owner("/missing") is None + + +def test_list_manual_packages_parses_and_sorts(monkeypatch): + import enroll.debian as d + + class P: + def __init__(self, rc: int, out: str): + self.returncode = rc + self.stdout = out + self.stderr = "" + + def fake_run(cmd, text, capture_output): + assert cmd == ["apt-mark", "showmanual"] + return P(0, "\n# comment\nnginx\nvim\nnginx\n") + + monkeypatch.setattr(d.subprocess, "run", fake_run) + assert d.list_manual_packages() == ["nginx", "vim"] + + +def test_build_dpkg_etc_index(tmp_path: Path): + import enroll.debian as d + + info = tmp_path / "info" + info.mkdir() + (info / "nginx.list").write_text( + "/etc/nginx/nginx.conf\n/etc/nginx/sites-enabled/default\n/usr/bin/nginx\n", + encoding="utf-8", + ) + (info / "vim:amd64.list").write_text( + "/etc/vim/vimrc\n/usr/bin/vim\n", + encoding="utf-8", + ) + + owned, owner_map, topdir_to_pkgs, pkg_to_etc = d.build_dpkg_etc_index(str(info)) + assert "/etc/nginx/nginx.conf" in owned + assert owner_map["/etc/nginx/nginx.conf"] == "nginx" + assert "nginx" in topdir_to_pkgs + assert topdir_to_pkgs["nginx"] == {"nginx"} + assert pkg_to_etc["vim"] == ["/etc/vim/vimrc"] + + +def test_parse_status_conffiles_handles_continuations(tmp_path: Path): + import enroll.debian as d + + status = tmp_path / "status" + status.write_text( + "\n".join( + [ + "Package: nginx", + "Version: 1", + "Conffiles:", + " /etc/nginx/nginx.conf abcdef", + " /etc/nginx/mime.types 123456", + "", + "Package: other", + "Version: 2", + "", + ] + ), + encoding="utf-8", + ) + m = d.parse_status_conffiles(str(status)) + assert m["nginx"]["/etc/nginx/nginx.conf"] == "abcdef" + assert m["nginx"]["/etc/nginx/mime.types"] == "123456" + assert "other" not in m + + +def test_dpkg_owner_returns_none_on_diversion_only(monkeypatch): + import enroll.debian as d + + class P: + def __init__(self, rc: int, out: str): + self.returncode = rc + self.stdout = out + self.stderr = "" + + def fake_run(cmd, text, capture_output): + return P(0, "diversion by foo from: /etc/something\n") + + monkeypatch.setattr(d.subprocess, "run", fake_run) + assert d.dpkg_owner("/etc/something") is None + + +def test_dpkg_owner_handles_line_without_colon(monkeypatch): + import enroll.debian as d + + class P: + def __init__(self, rc: int, out: str): + self.returncode = rc + self.stdout = out + self.stderr = "" + + def fake_run(cmd, text, capture_output): + return P(0, "invalid line without colon\n") + + monkeypatch.setattr(d.subprocess, "run", fake_run) + assert d.dpkg_owner("/etc/foo") is None + + +def test_list_manual_packages_returns_empty_on_error(monkeypatch): + import enroll.debian as d + + class P: + def __init__(self, rc: int, out: str): + self.returncode = rc + self.stdout = out + self.stderr = "" + + def fake_run(cmd, text, capture_output): + return P(1, "error") + + monkeypatch.setattr(d.subprocess, "run", fake_run) + assert d.list_manual_packages() == [] + + +def test_list_installed_packages_handles_exception(monkeypatch): + import enroll.debian as d + + def fake_run(*args, **kwargs): + raise Exception("simulated error") + + monkeypatch.setattr(d.subprocess, "run", fake_run) + assert d.list_installed_packages() == {} + + +def test_list_installed_packages_parses_output(): + import enroll.debian as d + + class P: + def __init__(self, rc: int, out: str): + self.returncode = rc + self.stdout = out + self.stderr = "" + + original_run = d.subprocess.run + + def fake_run(cmd, text, capture_output, check): + return P(0, "nginx\t1.18.0\tamd64\tweb\nvim\t8.2\tamd64\teditors\n") + + d.subprocess.run = fake_run + try: + result = d.list_installed_packages() + assert "nginx" in result + assert result["nginx"][0]["version"] == "1.18.0" + assert result["nginx"][0]["arch"] == "amd64" + assert result["nginx"][0]["section"] == "web" + assert "vim" in result + finally: + d.subprocess.run = original_run + + +def test_list_installed_packages_skips_invalid_lines(): + import enroll.debian as d + + class P: + def __init__(self, rc: int, out: str): + self.returncode = rc + self.stdout = out + self.stderr = "" + + original_run = d.subprocess.run + + def fake_run(cmd, text, capture_output, check): + return P(0, "nginx\t1.18.0\tamd64\ninvalid_line\n\t1.0\tamd64\n") + + d.subprocess.run = fake_run + try: + result = d.list_installed_packages() + assert "nginx" in result + assert "invalid_line" not in result + finally: + d.subprocess.run = original_run + + +def test_list_installed_packages_handles_empty_name(): + import enroll.debian as d + + class P: + def __init__(self, rc: int, out: str): + self.returncode = rc + self.stdout = out + self.stderr = "" + + original_run = d.subprocess.run + + def fake_run(cmd, text, capture_output, check): + return P(0, "\t1.0\tamd64\nnginx\t1.18.0\tamd64\n") + + d.subprocess.run = fake_run + try: + result = d.list_installed_packages() + assert "" not in result + assert "nginx" in result + finally: + d.subprocess.run = original_run + + +def test_list_installed_packages_sorts_output(): + import enroll.debian as d + + class P: + def __init__(self, rc: int, out: str): + self.returncode = rc + self.stdout = out + self.stderr = "" + + original_run = d.subprocess.run + + def fake_run(cmd, text, capture_output, check): + return P(0, "nginx\t1.18.0\tamd64\nnginx\t1.19.0\tarm64\n") + + d.subprocess.run = fake_run + try: + result = d.list_installed_packages() + assert len(result["nginx"]) == 2 + assert result["nginx"][0]["arch"] == "amd64" + assert result["nginx"][1]["arch"] == "arm64" + finally: + d.subprocess.run = original_run + + +def test_build_dpkg_etc_index_handles_missing_file(tmp_path: Path): + import enroll.debian as d + + info = tmp_path / "info" + info.mkdir() + # Don't create any .list files + + owned, owner_map, topdir_to_pkgs, pkg_to_etc = d.build_dpkg_etc_index(str(info)) + assert owned == set() + assert owner_map == {} + assert topdir_to_pkgs == {} + assert pkg_to_etc == {} + + +def test_build_dpkg_etc_index_skips_non_etc_paths(tmp_path: Path): + import enroll.debian as d + + info = tmp_path / "info" + info.mkdir() + (info / "foo.list").write_text("/usr/bin/foo\n/etc/bar\n", encoding="utf-8") + + owned, owner_map, topdir_to_pkgs, pkg_to_etc = d.build_dpkg_etc_index(str(info)) + assert "/usr/bin/foo" not in owned + assert "/etc/bar" in owned + assert "foo" not in topdir_to_pkgs + + +def test_parse_status_conffiles_handles_empty_status(tmp_path: Path): + import enroll.debian as d + + status = tmp_path / "status" + status.write_text("", encoding="utf-8") + m = d.parse_status_conffiles(str(status)) + assert m == {} + + +def test_parse_status_conffiles_handles_package_without_conffiles(tmp_path: Path): + import enroll.debian as d + + status = tmp_path / "status" + status.write_text( + "Package: nginx\nVersion: 1\nStatus: install ok installed\n", + encoding="utf-8", + ) + m = d.parse_status_conffiles(str(status)) + assert m == {} + + +def test_read_pkg_md5sums_returns_empty_if_file_not_exists(tmp_path: Path): + import enroll.debian as d + + result = d.read_pkg_md5sums("nonexistent_package") + assert result == {} + + +def test_read_pkg_md5sums_parses_md5sums_file(tmp_path: Path, monkeypatch): + import enroll.debian as d + + info_dir = tmp_path / "info" + info_dir.mkdir() + md5_file = info_dir / "nginx.md5sums" + md5_file.write_text( + "abcdef1234567890abcdef1234567890 etc/nginx/nginx.conf\n" + "1234567890abcdef1234567890abcdef etc/nginx/sites-enabled/default\n", + encoding="utf-8", + ) + + def fake_exists(path): + return str(path).endswith("nginx.md5sums") + + monkeypatch.setattr(d.os.path, "exists", fake_exists) + + original_open = open + + def fake_open(path, *args, **kwargs): + if "nginx.md5sums" in str(path): + return original_open(md5_file, *args, **kwargs) + return original_open(path, *args, **kwargs) + + monkeypatch.setattr("builtins.open", fake_open, raising=False) + + result = d.read_pkg_md5sums("nginx") + assert result["etc/nginx/nginx.conf"] == "abcdef1234567890abcdef1234567890" + assert ( + result["etc/nginx/sites-enabled/default"] == "1234567890abcdef1234567890abcdef" + ) + + +def test_dpkg_owner_raises_on_command_failure(monkeypatch): + """Test _run raises RuntimeError on non-zero exit.""" + import enroll.debian as d + + class P: + returncode = 1 + stdout = "" + stderr = "command failed" + + def fake_run(cmd, text, capture_output, check=False): + return P() + + monkeypatch.setattr(d.subprocess, "run", fake_run) + + with pytest.raises(RuntimeError) as exc_info: + d._run(["fake", "command"]) + + assert "Command failed" in str(exc_info.value) + assert "fake" in str(exc_info.value) + + +def test_build_dpkg_etc_index_skips_invalid_line_formats(tmp_path: Path): + """Test that lines with less than 3 parts are skipped.""" + import enroll.debian as d + + info = tmp_path / "info" + info.mkdir() + # Create a .list file with invalid format (missing tab-separated fields) + (info / "foo.list").write_text( + "/etc/foo/bar\n" # This is a path, not a tab-separated line + "/etc/foo/baz\n", + encoding="utf-8", + ) + + # Should handle gracefully + owned, owner_map, topdir_to_pkgs, pkg_to_etc = d.build_dpkg_etc_index(str(info)) + # The path lines should be processed normally + assert "/etc/foo/bar" in owned or "/etc/foo/baz" in owned + + +def test_build_dpkg_etc_index_handles_file_not_found(tmp_path: Path): + """Test that FileNotFoundError is handled gracefully.""" + import enroll.debian as d + + info = tmp_path / "info" + info.mkdir() + # Create a .list file that references a non-existent path + (info / "foo.list").write_text( + "/nonexistent/path\n", + encoding="utf-8", + ) + + # Should not raise + owned, owner_map, topdir_to_pkgs, pkg_to_etc = d.build_dpkg_etc_index(str(info)) + # The non-existent path should be skipped + assert "/nonexistent/path" not in owned + + +def test_parse_status_conffiles_skips_empty_lines(tmp_path: Path): + """Test that empty lines in conffiles are skipped.""" + import enroll.debian as d + + status = tmp_path / "status" + status.write_text( + "Package: nginx\n" + "Version: 1\n" + "Conffiles:\n" + " /etc/nginx/nginx.conf abcdef\n" + " /etc/nginx/mime.types 123456\n" + "\n", # Empty line to trigger flush + encoding="utf-8", + ) + + m = d.parse_status_conffiles(str(status)) + assert "/etc/nginx/nginx.conf" in m["nginx"] + assert "/etc/nginx/mime.types" in m["nginx"] + + +def test_read_pkg_md5sums_skips_invalid_md5_lines(tmp_path: Path, monkeypatch): + """Test that lines without proper MD5 format are skipped.""" + import enroll.debian as d + + info_dir = tmp_path / "info" + info_dir.mkdir() + md5_file = info_dir / "foo.md5sums" + md5_file.write_text( + "abcdef1234567890abcdef1234567890 etc/foo/bar\n" + "invalid line without proper format\n" + "1234567890abcdef1234567890abcdef etc/foo/baz\n", + encoding="utf-8", + ) + + def fake_exists(path): + return str(path).endswith("foo.md5sums") + + monkeypatch.setattr(d.os.path, "exists", fake_exists) + + original_open = open + + def fake_open(path, *args, **kwargs): + if "foo.md5sums" in str(path): + return original_open(md5_file, *args, **kwargs) + return original_open(path, *args, **kwargs) + + monkeypatch.setattr("builtins.open", fake_open, raising=False) + + result = d.read_pkg_md5sums("foo") + assert "etc/foo/bar" in result + assert "etc/foo/baz" in result + + +def test_build_dpkg_etc_index_skips_lines_without_tabs(tmp_path: Path): + """Test that lines without tab separators are skipped (parts < 3).""" + import enroll.debian as d + + info = tmp_path / "info" + info.mkdir() + # Create file with lines that don't have tab separators + (info / "foo.list").write_text( + "notabseparator\n" # No tab - should be skipped + "/etc/foo/bar\n", # This is a path line, processed differently + encoding="utf-8", + ) + + owned, owner_map, topdir_to_pkgs, pkg_to_etc = d.build_dpkg_etc_index(str(info)) + # Path lines are still processed + assert "/etc/foo/bar" in owned + + +def test_read_pkg_md5sums_skips_empty_lines(tmp_path: Path, monkeypatch): + """Test that empty lines in md5sums are skipped.""" + import enroll.debian as d + + info_dir = tmp_path / "info" + info_dir.mkdir() + md5_file = info_dir / "bar.md5sums" + md5_file.write_text( + "abcdef1234567890abcdef1234567890 etc/bar/file1\n" + "\n" # Empty line + "1234567890abcdef1234567890abcdef etc/bar/file2\n", + encoding="utf-8", + ) + + def fake_exists(path): + return str(path).endswith("bar.md5sums") + + monkeypatch.setattr(d.os.path, "exists", fake_exists) + + original_open = open + + def fake_open(path, *args, **kwargs): + if "bar.md5sums" in str(path): + return original_open(md5_file, *args, **kwargs) + return original_open(path, *args, **kwargs) + + monkeypatch.setattr("builtins.open", fake_open, raising=False) + + result = d.read_pkg_md5sums("bar") + assert "etc/bar/file1" in result + assert "etc/bar/file2" in result + + +def test_read_pkg_md5sums_skips_lines_not_starting_with_path( + tmp_path: Path, monkeypatch +): + """Test that lines not starting with / are skipped.""" + import enroll.debian as d + + info_dir = tmp_path / "info" + info_dir.mkdir() + md5_file = info_dir / "baz.md5sums" + md5_file.write_text( + "abcdef1234567890abcdef1234567890 etc/baz/file1\n" + "invalid line\n" # Doesn't start with / + "1234567890abcdef1234567890abcdef etc/baz/file2\n", + encoding="utf-8", + ) + + def fake_exists(path): + return str(path).endswith("baz.md5sums") + + monkeypatch.setattr(d.os.path, "exists", fake_exists) + + original_open = open + + def fake_open(path, *args, **kwargs): + if "baz.md5sums" in str(path): + return original_open(md5_file, *args, **kwargs) + return original_open(path, *args, **kwargs) + + monkeypatch.setattr("builtins.open", fake_open, raising=False) + + result = d.read_pkg_md5sums("baz") + assert "etc/baz/file1" in result + assert "etc/baz/file2" in result diff --git a/tests/test_diff_bundle.py b/tests/test_diff_bundle.py new file mode 100644 index 0000000..2895484 --- /dev/null +++ b/tests/test_diff_bundle.py @@ -0,0 +1,1373 @@ +from __future__ import annotations + +import os +import tarfile +from pathlib import Path + +import pytest + +from enroll.diff import ( + _Spinner, + _enforcement_plan, + has_enforceable_drift, + _role_tag, + _utc_now_iso, + _report_markdown, +) + + +def _make_bundle_dir(tmp_path: Path) -> Path: + b = tmp_path / "bundle" + (b / "artifacts").mkdir(parents=True) + (b / "state.json").write_text("{}\n", encoding="utf-8") + return b + + +def _tar_gz_of_dir(src: Path, out: Path) -> None: + with tarfile.open(out, mode="w:gz") as tf: + # tar -C src . semantics + for p in src.rglob("*"): + rel = p.relative_to(src) + tf.add(p, arcname=str(rel)) + + +def test_bundle_from_directory_and_statejson_path(tmp_path: Path): + import enroll.diff as d + + b = _make_bundle_dir(tmp_path) + + br1 = d._bundle_from_input(str(b), sops_mode=False) + assert br1.dir == b + assert br1.state_path.exists() + + br2 = d._bundle_from_input(str(b / "state.json"), sops_mode=False) + assert br2.dir == b + + +def test_bundle_from_tarball_extracts(tmp_path: Path): + import enroll.diff as d + + b = _make_bundle_dir(tmp_path) + tgz = tmp_path / "bundle.tgz" + _tar_gz_of_dir(b, tgz) + + br = d._bundle_from_input(str(tgz), sops_mode=False) + try: + assert br.dir.is_dir() + assert (br.dir / "state.json").exists() + finally: + if br.tempdir: + br.tempdir.cleanup() + + +def test_bundle_from_sops_like_file(monkeypatch, tmp_path: Path): + import enroll.diff as d + + b = _make_bundle_dir(tmp_path) + tgz = tmp_path / "bundle.tar.gz" + _tar_gz_of_dir(b, tgz) + + # Pretend the tarball is an encrypted bundle by giving it a .sops name. + sops_path = tmp_path / "bundle.tar.gz.sops" + sops_path.write_bytes(tgz.read_bytes()) + + # Stub out sops machinery: "decrypt" just copies through. + monkeypatch.setattr(d, "require_sops_cmd", lambda: "sops") + + def fake_decrypt(src: Path, dest: Path, mode: int): + dest.write_bytes(Path(src).read_bytes()) + try: + os.chmod(dest, mode) + except OSError: + pass + + monkeypatch.setattr(d, "decrypt_file_binary_to", fake_decrypt) + + br = d._bundle_from_input(str(sops_path), sops_mode=False) + try: + assert (br.dir / "state.json").exists() + finally: + if br.tempdir: + br.tempdir.cleanup() + + +def test_bundle_from_input_missing_path(tmp_path: Path): + import enroll.diff as d + + with pytest.raises(RuntimeError, match="not found"): + d._bundle_from_input(str(tmp_path / "nope"), sops_mode=False) + + +import json +import sys + + +from enroll.diff import ( + _bundle_from_input, + _file_index, + _iter_managed_files, + _load_state, + _pkg_version_display, + _pkg_version_key, + _progress_enabled, + _roles, + _service_units, + _sha256, + _users_by_name, + compare_harvests, +) +from enroll.sopsutil import SopsError + + +def test_progress_enabled_when_tty(monkeypatch): + monkeypatch.setattr(sys.stderr, "isatty", lambda: True) + monkeypatch.delenv("ENROLL_NO_PROGRESS", raising=False) + assert _progress_enabled() is True + + +def test_progress_enabled_when_not_tty(monkeypatch): + monkeypatch.setattr(sys.stderr, "isatty", lambda: False) + monkeypatch.delenv("ENROLL_NO_PROGRESS", raising=False) + assert _progress_enabled() is False + + +def test_progress_enabled_with_env_var(monkeypatch): + monkeypatch.setattr(sys.stderr, "isatty", lambda: True) + monkeypatch.setenv("ENROLL_NO_PROGRESS", "1") + assert _progress_enabled() is False + + monkeypatch.setenv("ENROLL_NO_PROGRESS", "true") + assert _progress_enabled() is False + + monkeypatch.setenv("ENROLL_NO_PROGRESS", "yes") + assert _progress_enabled() is False + + +def test_sha256(tmp_path: Path): + test_file = tmp_path / "test.txt" + test_file.write_text("hello world", encoding="utf-8") + hash_result = _sha256(test_file) + assert len(hash_result) == 64 + + +def test_sha256_empty_file(tmp_path: Path): + test_file = tmp_path / "empty.txt" + test_file.write_bytes(b"") + hash_result = _sha256(test_file) + assert ( + hash_result + == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ) + + +def test_bundle_from_input_directory(tmp_path: Path): + result = _bundle_from_input(str(tmp_path), sops_mode=False) + assert result.dir == tmp_path + assert result.tempdir is None + + +def test_bundle_from_input_state_json_path(tmp_path: Path): + state_file = tmp_path / "state.json" + state_file.write_text("{}", encoding="utf-8") + result = _bundle_from_input(str(state_file), sops_mode=False) + assert result.dir == tmp_path + assert result.tempdir is None + + +def test_bundle_from_input_not_found(): + with pytest.raises(RuntimeError) as exc_info: + _bundle_from_input("/nonexistent/path", sops_mode=False) + assert "not found" in str(exc_info.value).lower() + + +def test_bundle_from_input_tarball(tmp_path: Path): + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir() + state_file = bundle_dir / "state.json" + state_file.write_text("{}", encoding="utf-8") + + tar_path = tmp_path / "bundle.tar.gz" + with tarfile.open(tar_path, "w:gz") as tf: + tf.add(bundle_dir, arcname="bundle") + + result = _bundle_from_input(str(tar_path), sops_mode=False) + assert result.dir.exists() + assert result.tempdir is not None + result.tempdir.cleanup() + + +def test_bundle_from_input_invalid_type(tmp_path: Path): + test_file = tmp_path / "test.txt" + test_file.write_text("not a bundle", encoding="utf-8") + + with pytest.raises(RuntimeError) as exc_info: + _bundle_from_input(str(test_file), sops_mode=False) + assert "not a directory" in str(exc_info.value).lower() + + +def test_load_state(tmp_path: Path): + state_file = tmp_path / "state.json" + state_file.write_text('{"host": {"hostname": "test"}}', encoding="utf-8") + result = _load_state(tmp_path) + assert result["host"]["hostname"] == "test" + + +def test_roles_with_roles(): + state = {"roles": {"users": {}, "services": []}} + result = _roles(state) + assert "users" in result + + +def test_service_units_empty(): + assert _service_units({}) == {} + + +def test_service_units_with_services(): + state = { + "roles": { + "services": [ + {"unit": "nginx.service", "active_state": "active"}, + {"unit": "ssh.service", "active_state": "inactive"}, + ] + } + } + result = _service_units(state) + assert "nginx.service" in result + assert "ssh.service" in result + assert result["nginx.service"]["active_state"] == "active" + + +def test_users_by_name_empty(): + assert _users_by_name({}) == {} + + +def test_users_by_name_with_users(): + state = { + "roles": { + "users": { + "users": [ + {"name": "alice", "uid": 1000}, + {"name": "bob", "uid": 1001}, + ] + } + } + } + result = _users_by_name(state) + assert "alice" in result + assert "bob" in result + assert result["alice"]["uid"] == 1000 + + +def test_pkg_version_key_with_version(): + entry = {"version": "1.2.3"} + assert _pkg_version_key(entry) == "1.2.3" + + +def test_pkg_version_key_with_installations(): + entry = { + "installations": [ + {"arch": "x86_64", "version": "1.2.3"}, + {"arch": "aarch64", "version": "1.2.3"}, + ] + } + result = _pkg_version_key(entry) + assert "x86_64:1.2.3" in result + assert "aarch64:1.2.3" in result + + +def test_pkg_version_key_with_empty_version(): + entry = {"version": None} + assert _pkg_version_key(entry) is None + + +def test_pkg_version_key_with_invalid_installations(): + entry = {"installations": ["not_a_dict", {"arch": "x86_64", "version": "1.0"}]} + result = _pkg_version_key(entry) + assert "x86_64:1.0" in result + + +def test_pkg_version_display_with_version(): + entry = {"version": "1.2.3"} + assert _pkg_version_display(entry) == "1.2.3" + + +def test_pkg_version_display_with_installations(): + entry = { + "installations": [ + {"arch": "x86_64", "version": "1.2.3"}, + ] + } + assert _pkg_version_display(entry) == "1.2.3 (x86_64)" + + +def test_pkg_version_display_empty(): + assert _pkg_version_display({}) is None + + +def test_iter_managed_files_empty(): + state = {"roles": {}} + files = list(_iter_managed_files(state)) + assert files == [] + + +def test_iter_managed_files_services(): + state = { + "roles": { + "services": [ + { + "role_name": "nginx", + "managed_files": [ + {"path": "/etc/nginx/nginx.conf", "src_rel": "nginx.conf"} + ], + } + ] + } + } + files = list(_iter_managed_files(state)) + assert len(files) == 1 + assert files[0] == ( + "nginx", + {"path": "/etc/nginx/nginx.conf", "src_rel": "nginx.conf"}, + ) + + +def test_iter_managed_files_packages(): + state = { + "roles": { + "packages": [ + { + "role_name": "vim", + "managed_files": [{"path": "/usr/bin/vim", "src_rel": "bin/vim"}], + } + ] + } + } + files = list(_iter_managed_files(state)) + assert len(files) == 1 + assert files[0][0] == "vim" + + +def test_iter_managed_files_users(): + state = { + "roles": { + "users": { + "role_name": "users", + "managed_files": [{"path": "/etc/passwd", "src_rel": "passwd"}], + } + } + } + files = list(_iter_managed_files(state)) + assert len(files) == 1 + assert files[0][0] == "users" + + +def test_iter_managed_files_apt_config(): + state = { + "roles": { + "apt_config": { + "role_name": "apt_config", + "managed_files": [ + {"path": "/etc/apt/sources.list", "src_rel": "sources.list"} + ], + } + } + } + files = list(_iter_managed_files(state)) + assert len(files) == 1 + assert files[0][0] == "apt_config" + + +def test_iter_managed_files_etc_custom(): + state = { + "roles": { + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [ + {"path": "/etc/custom.conf", "src_rel": "custom.conf"} + ], + } + } + } + files = list(_iter_managed_files(state)) + assert len(files) == 1 + assert files[0][0] == "etc_custom" + + +def test_iter_managed_files_usr_local_custom(): + state = { + "roles": { + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [ + {"path": "/usr/local/bin/script", "src_rel": "bin/script"} + ], + } + } + } + files = list(_iter_managed_files(state)) + assert len(files) == 1 + assert files[0][0] == "usr_local_custom" + + +def test_iter_managed_files_extra_paths(): + state = { + "roles": { + "extra_paths": { + "role_name": "extra_paths", + "managed_files": [{"path": "/opt/app/config", "src_rel": "config"}], + } + } + } + files = list(_iter_managed_files(state)) + assert len(files) == 1 + assert files[0][0] == "extra_paths" + + +def test_file_index_empty(): + state = {"roles": {}} + index = _file_index(Path("/tmp"), state) + assert index == {} + + +def test_file_index_with_files(tmp_path: Path): + state = { + "roles": { + "users": { + "managed_files": [ + {"path": "/etc/passwd", "src_rel": "passwd", "owner": "root"}, + ] + } + } + } + index = _file_index(tmp_path, state) + assert "/etc/passwd" in index + assert index["/etc/passwd"].role == "users" + assert index["/etc/passwd"].owner == "root" + + +def test_file_index_duplicates_first_wins(tmp_path: Path): + state = { + "roles": { + "users": { + "managed_files": [ + {"path": "/etc/passwd", "src_rel": "passwd"}, + ] + }, + "etc_custom": { + "managed_files": [ + {"path": "/etc/passwd", "src_rel": "custom_passwd"}, + ] + }, + } + } + index = _file_index(tmp_path, state) + assert "/etc/passwd" in index + assert index["/etc/passwd"].src_rel == "passwd" + + +def test_file_index_skips_missing_path_or_src_rel(tmp_path: Path): + state = { + "roles": { + "users": { + "managed_files": [ + {"path": "/etc/passwd"}, # missing src_rel + {"src_rel": "passwd"}, # missing path + ] + } + } + } + index = _file_index(tmp_path, state) + assert index == {} + + +def test_compare_harvests_no_changes(tmp_path: Path): + old_bundle = tmp_path / "old" + old_bundle.mkdir() + (old_bundle / "state.json").write_text( + json.dumps( + { + "inventory": {"packages": {"vim": {"version": "1.0"}}}, + "roles": {}, + } + ), + encoding="utf-8", + ) + + new_bundle = tmp_path / "new" + new_bundle.mkdir() + (new_bundle / "state.json").write_text( + json.dumps( + { + "inventory": {"packages": {"vim": {"version": "1.0"}}}, + "roles": {}, + } + ), + encoding="utf-8", + ) + + report, has_changes = compare_harvests(str(old_bundle), str(new_bundle)) + assert has_changes is False + assert report["packages"]["added"] == [] + assert report["packages"]["removed"] == [] + + +def test_compare_harvests_package_added(tmp_path: Path): + old_bundle = tmp_path / "old" + old_bundle.mkdir() + (old_bundle / "state.json").write_text( + json.dumps({"inventory": {"packages": {}}, "roles": {}}), + encoding="utf-8", + ) + + new_bundle = tmp_path / "new" + new_bundle.mkdir() + (new_bundle / "state.json").write_text( + json.dumps( + {"inventory": {"packages": {"vim": {"version": "1.0"}}}, "roles": {}} + ), + encoding="utf-8", + ) + + report, has_changes = compare_harvests(str(old_bundle), str(new_bundle)) + assert has_changes is True + assert "vim" in report["packages"]["added"] + + +def test_compare_harvests_package_removed(tmp_path: Path): + old_bundle = tmp_path / "old" + old_bundle.mkdir() + (old_bundle / "state.json").write_text( + json.dumps( + {"inventory": {"packages": {"vim": {"version": "1.0"}}}, "roles": {}} + ), + encoding="utf-8", + ) + + new_bundle = tmp_path / "new" + new_bundle.mkdir() + (new_bundle / "state.json").write_text( + json.dumps({"inventory": {"packages": {}}, "roles": {}}), + encoding="utf-8", + ) + + report, has_changes = compare_harvests(str(old_bundle), str(new_bundle)) + assert has_changes is True + assert "vim" in report["packages"]["removed"] + + +def test_compare_harvests_package_version_changed(tmp_path: Path): + old_bundle = tmp_path / "old" + old_bundle.mkdir() + (old_bundle / "state.json").write_text( + json.dumps( + {"inventory": {"packages": {"vim": {"version": "1.0"}}}, "roles": {}} + ), + encoding="utf-8", + ) + + new_bundle = tmp_path / "new" + new_bundle.mkdir() + (new_bundle / "state.json").write_text( + json.dumps( + {"inventory": {"packages": {"vim": {"version": "2.0"}}}, "roles": {}} + ), + encoding="utf-8", + ) + + report, has_changes = compare_harvests(str(old_bundle), str(new_bundle)) + assert has_changes is True + assert len(report["packages"]["version_changed"]) == 1 + + +def test_compare_harvests_ignore_package_versions(tmp_path: Path): + old_bundle = tmp_path / "old" + old_bundle.mkdir() + (old_bundle / "state.json").write_text( + json.dumps( + {"inventory": {"packages": {"vim": {"version": "1.0"}}}, "roles": {}} + ), + encoding="utf-8", + ) + + new_bundle = tmp_path / "new" + new_bundle.mkdir() + (new_bundle / "state.json").write_text( + json.dumps( + {"inventory": {"packages": {"vim": {"version": "2.0"}}}, "roles": {}} + ), + encoding="utf-8", + ) + + report, has_changes = compare_harvests( + str(old_bundle), str(new_bundle), ignore_package_versions=True + ) + assert report["packages"]["version_changed_ignored_count"] == 1 + + +def test_compare_harvests_service_added(tmp_path: Path): + old_bundle = tmp_path / "old" + old_bundle.mkdir() + (old_bundle / "state.json").write_text( + json.dumps({"inventory": {"packages": {}}, "roles": {"services": []}}), + encoding="utf-8", + ) + + new_bundle = tmp_path / "new" + new_bundle.mkdir() + (new_bundle / "state.json").write_text( + json.dumps( + { + "inventory": {"packages": {}}, + "roles": {"services": [{"unit": "nginx.service"}]}, + } + ), + encoding="utf-8", + ) + + report, has_changes = compare_harvests(str(old_bundle), str(new_bundle)) + assert has_changes is True + assert "nginx.service" in report["services"]["enabled_added"] + + +def test_compare_harvests_user_added(tmp_path: Path): + old_bundle = tmp_path / "old" + old_bundle.mkdir() + (old_bundle / "state.json").write_text( + json.dumps({"inventory": {"packages": {}}, "roles": {"users": {"users": []}}}), + encoding="utf-8", + ) + + new_bundle = tmp_path / "new" + new_bundle.mkdir() + (new_bundle / "state.json").write_text( + json.dumps( + { + "inventory": {"packages": {}}, + "roles": {"users": {"users": [{"name": "alice", "uid": 1000}]}}, + } + ), + encoding="utf-8", + ) + + report, has_changes = compare_harvests(str(old_bundle), str(new_bundle)) + assert has_changes is True + assert "alice" in report["users"]["added"] + + +def test_compare_harvests_with_exclude_paths(tmp_path: Path): + old_bundle = tmp_path / "old" + old_bundle.mkdir() + old_artifacts = old_bundle / "artifacts" / "users" + old_artifacts.mkdir(parents=True) + (old_artifacts / "passwd").write_text("old", encoding="utf-8") + (old_bundle / "state.json").write_text( + json.dumps( + { + "inventory": {"packages": {}}, + "roles": { + "users": { + "managed_files": [{"path": "/etc/passwd", "src_rel": "passwd"}] + } + }, + } + ), + encoding="utf-8", + ) + + new_bundle = tmp_path / "new" + new_bundle.mkdir() + new_artifacts = new_bundle / "artifacts" / "users" + new_artifacts.mkdir(parents=True) + (new_artifacts / "passwd").write_text("new", encoding="utf-8") + (new_bundle / "state.json").write_text( + json.dumps( + { + "inventory": {"packages": {}}, + "roles": { + "users": { + "managed_files": [{"path": "/etc/passwd", "src_rel": "passwd"}] + } + }, + } + ), + encoding="utf-8", + ) + + report, has_changes = compare_harvests( + str(old_bundle), str(new_bundle), exclude_paths=["/etc/passwd"] + ) + assert "/etc/passwd" not in [f["path"] for f in report["files"]["added"]] + assert "/etc/passwd" not in [f["path"] for f in report["files"]["removed"]] + assert "/etc/passwd" not in [f["path"] for f in report["files"]["changed"]] + + +def test_utc_now_iso(): + result = _utc_now_iso() + assert "T" in result + assert "+" in result or "Z" in result + + +def test_spinner_stop_without_start(): + spinner = _Spinner("Test") + spinner.stop(final_line="Done") + # Should not raise + + +def test_spinner_run_exception(monkeypatch): + class FakeStderr: + def write(self, s): + raise Exception("Write error") + + def flush(self): + pass + + monkeypatch.setattr(sys, "stderr", FakeStderr()) + + spinner = _Spinner("Test") + spinner.start() + spinner.stop() + + +def test_spinner_double_start(): + spinner = _Spinner("Test") + spinner.start() + spinner.start() # Should not raise or spawn another thread + spinner.stop() + + +def test_role_tag_normal(): + assert _role_tag("nginx") == "role_nginx" + assert _role_tag("my-app") == "role_my-app" + + +def test_role_tag_with_special_chars(): + assert _role_tag("my.app") == "role_my_app" + assert _role_tag("my app") == "role_my_app" + + +def test_role_tag_empty(): + assert _role_tag("") == "role_other" + assert _role_tag(" ") == "role_other" + + +def test_has_enforceable_drift_packages_removed(): + report = {"packages": {"removed": ["vim"]}} + assert has_enforceable_drift(report) is True + + +def test_has_enforceable_drift_services_removed(): + report = {"services": {"enabled_removed": ["nginx.service"]}} + assert has_enforceable_drift(report) is True + + +def test_has_enforceable_drift_service_changed(): + report = { + "services": { + "changed": [ + { + "unit": "nginx.service", + "changes": {"active_state": {"old": "active", "new": "inactive"}}, + } + ] + } + } + assert has_enforceable_drift(report) is True + + +def test_has_enforceable_drift_service_package_only_changed(): + # Service changed only in packages - should NOT be enforceable + report = { + "services": { + "changed": [ + { + "unit": "nginx.service", + "changes": {"packages": {"added": ["nginx-extra"]}}, + } + ] + } + } + assert has_enforceable_drift(report) is False + + +def test_has_enforceable_drift_users_removed(): + report = {"users": {"removed": ["alice"]}} + assert has_enforceable_drift(report) is True + + +def test_has_enforceable_drift_users_changed(): + report = { + "users": { + "changed": [ + {"name": "alice", "changes": {"uid": {"old": 1000, "new": 1001}}} + ] + } + } + assert has_enforceable_drift(report) is True + + +def test_has_enforceable_drift_files_removed(): + report = { + "files": { + "removed": [{"path": "/etc/passwd", "role": "users", "reason": "conffile"}] + } + } + assert has_enforceable_drift(report) is True + + +def test_has_enforceable_drift_files_changed(): + report = { + "files": { + "changed": [ + { + "path": "/etc/passwd", + "changes": {"content": {"old": "sha1", "new": "sha2"}}, + } + ] + } + } + assert has_enforceable_drift(report) is True + + +def test_has_enforceable_drift_no_drift(): + report = { + "packages": {"added": ["newpkg"]}, + "services": {"enabled_added": ["new.service"]}, + "users": {"added": ["bob"]}, + "files": {"added": ["/opt/newfile"]}, + } + assert has_enforceable_drift(report) is False + + +def test_enforcement_plan_packages_removed(monkeypatch, tmp_path: Path): + old_state = { + "roles": { + "services": [{"role_name": "nginx", "packages": ["nginx"]}], + "packages": [{"role_name": "vim", "package": "vim"}], + } + } + report = {"packages": {"removed": ["nginx", "vim"]}} + + result = _enforcement_plan(report, old_state, tmp_path) + assert "nginx" in result.get("roles", []) + assert "vim" in result.get("roles", []) + assert "role_nginx" in result.get("tags", []) + + +def test_enforcement_plan_users_changed(): + old_state = { + "roles": {"users": {"role_name": "users", "users": [{"name": "alice"}]}} + } + report = {"users": {"changed": [{"name": "alice", "changes": {"uid": {}}}]}} + + result = _enforcement_plan(report, old_state, Path("/tmp")) + assert "users" in result.get("roles", []) + + +def test_enforcement_plan_files_removed(tmp_path: Path): + # Create the artifacts directory structure that _file_index expects + artifacts_dir = tmp_path / "artifacts" / "etc_custom" + artifacts_dir.mkdir(parents=True) + + old_state = { + "roles": { + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [ + {"path": "/etc/custom.conf", "src_rel": "custom.conf"} + ], + } + } + } + report = { + "files": {"removed": [{"path": "/etc/custom.conf", "role": "etc_custom"}]} + } + + result = _enforcement_plan(report, old_state, tmp_path) + assert "etc_custom" in result.get("roles", []) + + +def test_enforcement_plan_no_drift(): + old_state = {"roles": {}} + report = {"packages": {"added": ["newpkg"]}} + + result = _enforcement_plan(report, old_state, Path("/tmp")) + assert result.get("roles", []) == [] + + +def test_bundle_from_input_tgz(monkeypatch, tmp_path: Path): + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir() + state_file = bundle_dir / "state.json" + state_file.write_text("{}", encoding="utf-8") + + tar_path = tmp_path / "bundle.tgz" + with tarfile.open(tar_path, "w:gz") as tf: + tf.add(bundle_dir, arcname="bundle") + + result = _bundle_from_input(str(tar_path), sops_mode=False) + assert result.dir.exists() + assert result.tempdir is not None + result.tempdir.cleanup() + + +def test_bundle_from_input_sops_mode_no_sops(monkeypatch, tmp_path: Path): + # Create a fake .sops file + sops_file = tmp_path / "harvest.sops" + sops_file.write_bytes(b"encrypted") + + def fake_require(): + raise SopsError("sops not found") + + import enroll.diff as d + + monkeypatch.setattr(d, "require_sops_cmd", fake_require) + + with pytest.raises(SopsError): + _bundle_from_input(str(sops_file), sops_mode=True) + + +def test_report_markdown_basic(): + report = { + "generated_at": "2024-01-01T00:00:00Z", + "old": {"input": "old.tar.gz", "host": "host1"}, + "new": {"input": "new.tar.gz", "host": "host2"}, + "packages": {"added": ["vim"], "removed": [], "version_changed": []}, + "services": {"enabled_added": [], "enabled_removed": [], "changed": []}, + "users": {"added": [], "removed": [], "changed": []}, + "files": {"added": [], "removed": [], "changed": []}, + } + result = _report_markdown(report) + assert "## Packages" in result + assert "+ vim" in result + + +def test_report_markdown_with_enforcement_applied(): + report = { + "generated_at": "2024-01-01T00:00:00Z", + "old": {"input": "old.tar.gz"}, + "new": {"input": "new.tar.gz"}, + "packages": {"added": [], "removed": [], "version_changed": []}, + "services": {"enabled_added": [], "enabled_removed": [], "changed": []}, + "users": {"added": [], "removed": [], "changed": []}, + "files": {"added": [], "removed": [], "changed": []}, + "enforcement": { + "status": "applied", + "tags": ["role_users"], + "returncode": 0, + "finished_at": "2024-01-01T00:01:00Z", + }, + } + result = _report_markdown(report) + assert "Applied old harvest" in result + assert "role_users" in result + + +def test_report_markdown_with_enforcement_failed(): + report = { + "generated_at": "2024-01-01T00:00:00Z", + "old": {"input": "old.tar.gz"}, + "new": {"input": "new.tar.gz"}, + "packages": {"added": [], "removed": [], "version_changed": []}, + "services": {"enabled_added": [], "enabled_removed": [], "changed": []}, + "users": {"added": [], "removed": [], "changed": []}, + "files": {"added": [], "removed": [], "changed": []}, + "enforcement": { + "status": "failed", + "returncode": 1, + }, + } + result = _report_markdown(report) + assert "ansible-playbook failed" in result + + +def test_report_markdown_with_enforcement_skipped(): + report = { + "generated_at": "2024-01-01T00:00:00Z", + "old": {"input": "old.tar.gz"}, + "new": {"input": "new.tar.gz"}, + "packages": {"added": [], "removed": [], "version_changed": []}, + "services": {"enabled_added": [], "enabled_removed": [], "changed": []}, + "users": {"added": [], "removed": [], "changed": []}, + "files": {"added": [], "removed": [], "changed": []}, + "enforcement": { + "status": "skipped", + "reason": "no drift", + }, + } + result = _report_markdown(report) + assert "Skipped" in result + assert "no drift" in result + + +def test_report_markdown_with_version_ignored(): + report = { + "generated_at": "2024-01-01T00:00:00Z", + "old": {"input": "old.tar.gz"}, + "new": {"input": "new.tar.gz"}, + "packages": { + "added": [], + "removed": [], + "version_changed": [{"package": "vim", "old": "1.0", "new": "2.0"}], + "version_changed_ignored_count": 1, + }, + "services": {"enabled_added": [], "enabled_removed": [], "changed": []}, + "users": {"added": [], "removed": [], "changed": []}, + "files": {"added": [], "removed": [], "changed": []}, + } + result = _report_markdown(report) + assert "ignored 1" in result + + +def test_report_markdown_with_service_package_changes(): + report = { + "generated_at": "2024-01-01T00:00:00Z", + "old": {"input": "old.tar.gz"}, + "new": {"input": "new.tar.gz"}, + "packages": {"added": [], "removed": [], "version_changed": []}, + "services": { + "enabled_added": [], + "enabled_removed": [], + "changed": [ + { + "unit": "nginx.service", + "changes": {"packages": {"added": ["nginx-extra"], "removed": []}}, + } + ], + }, + "users": {"added": [], "removed": [], "changed": []}, + "files": {"added": [], "removed": [], "changed": []}, + } + result = _report_markdown(report) + assert "packages added" in result + + +def test_report_markdown_empty(): + report = { + "generated_at": "2024-01-01T00:00:00Z", + "old": {"input": "old.tar.gz"}, + "new": {"input": "new.tar.gz"}, + "packages": {}, + "services": {}, + "users": {}, + "files": {}, + } + result = _report_markdown(report) + assert "## Packages" in result + assert "## Services" in result + + +def test_spinner_start_stop(monkeypatch): + """Test spinner can be started and stopped.""" + import enroll.diff as d + + # Mock threading to avoid actual thread creation + class FakeThread: + def __init__(self, target, name, daemon): + self.target = target + self.daemon = daemon + + def start(self): + pass + + def join(self, timeout): + pass + + monkeypatch.setattr(d.threading, "Thread", FakeThread) + + spinner = d._Spinner("test message") + spinner.start() + spinner.stop() + + +def test_spinner_already_started(monkeypatch): + """Test spinner doesn't restart if already running.""" + import enroll.diff as d + + class FakeThread: + def __init__(self, target, name, daemon): + pass + + def start(self): + pass + + def join(self, timeout): + pass + + monkeypatch.setattr(d.threading, "Thread", FakeThread) + + spinner = d._Spinner("test message") + spinner.start() + spinner._thread = FakeThread(None, None, True) # Simulate already running + spinner.start() # Should return early + + +def test_spinner_stop_clears_line(monkeypatch, tmp_path): + """Test spinner stop clears the line.""" + import enroll.diff as d + import sys + + class FakeThread: + def __init__(self, target, name, daemon): + pass + + def start(self): + pass + + def join(self, timeout): + pass + + monkeypatch.setattr(d.threading, "Thread", FakeThread) + + # Capture stderr writes + writes = [] + original_write = sys.stderr.write + + def capture_write(s): + writes.append(s) + return original_write(s) + + monkeypatch.setattr(sys.stderr, "write", capture_write) + + spinner = d._Spinner("test message") + spinner._last_len = 20 + spinner.stop() + + # Should have written clearing sequence + assert any("\r" in w for w in writes) + + +def test_should_show_spinner_disabled_env(monkeypatch): + """Test spinner disabled via environment variable.""" + import enroll.diff as d + + monkeypatch.setenv("ENROLL_NO_PROGRESS", "1") + assert d._progress_enabled() is False + + monkeypatch.setenv("ENROLL_NO_PROGRESS", "true") + assert d._progress_enabled() is False + + monkeypatch.setenv("ENROLL_NO_PROGRESS", "yes") + assert d._progress_enabled() is False + + +def test_should_show_spinner_exception_on_isatty(monkeypatch): + """Test spinner returns False when isatty raises exception.""" + import enroll.diff as d + import sys + + original_stderr = sys.stderr + + class FakeStderr: + def isatty(self): + raise Exception("No tty") + + monkeypatch.setattr(sys, "stderr", FakeStderr()) + assert d._progress_enabled() is False + + # Restore + monkeypatch.setattr(sys, "stderr", original_stderr) + + +def test_all_packages_from_state(): + """Test _all_packages extracts sorted package list.""" + import enroll.diff as d + + state = { + "inventory": { + "packages": { + "nginx": [{"version": "1.0"}], + "vim": [{"version": "2.0"}], + "bash": [{"version": "3.0"}], + } + } + } + + result = d._all_packages(state) + assert result == ["bash", "nginx", "vim"] + + +def test_all_packages_empty_state(): + """Test _all_packages with empty state.""" + import enroll.diff as d + + state = {"inventory": {"packages": {}}} + result = d._all_packages(state) + assert result == [] + + +def test_roles_from_state(): + """Test _roles extracts roles from state.""" + import enroll.diff as d + + state = {"roles": {"web": {}, "db": {}}} + result = d._roles(state) + assert result == {"web": {}, "db": {}} + + +def test_roles_empty_state(): + """Test _roles with empty state.""" + import enroll.diff as d + + state = {} + result = d._roles(state) + assert result == {} + + +def test_pkg_version_key_with_multiple_versions(): + """Test _pkg_version_key handles multiple versions.""" + import enroll.diff as d + + entry = { + "installations": [ + {"version": "1.0", "arch": "amd64"}, + {"version": "2.0", "arch": "arm64"}, + ] + } + + result = d._pkg_version_key(entry) + # Just check it returns a non-None value with version info + assert result is not None + assert len(result) > 0 + + +def test_pkg_version_key_without_version(): + """Test _pkg_version_key skips entries without version.""" + import enroll.diff as d + + entry = { + "installations": [ + {"arch": "amd64"}, # No version + ] + } + + result = d._pkg_version_key(entry) + assert result is None + + +def test_pkg_version_key_with_empty_installations(): + """Test _pkg_version_key with empty installations.""" + import enroll.diff as d + + entry = {"installations": []} + result = d._pkg_version_key(entry) + assert result is None + + +def test_pkg_version_key_without_installations(): + """Test _pkg_version_key without installations key.""" + import enroll.diff as d + + entry = {} + result = d._pkg_version_key(entry) + assert result is None + + +def test_pkg_version_key_with_direct_version(): + """Test _pkg_version_key with direct version field.""" + import enroll.diff as d + + entry = {"version": "1.2.3"} + result = d._pkg_version_key(entry) + assert result == "1.2.3" + + +def test_report_text_with_exclude_paths(): + """Test _report_text includes exclude paths.""" + import enroll.diff as d + + report = { + "generated_at": "2024-01-01T00:00:00Z", + "old": {"input": "old.tar.gz", "host": "host1", "state_mtime": "mtime1"}, + "new": {"input": "new.tar.gz", "host": "host2", "state_mtime": "mtime2"}, + "filters": {"exclude_paths": ["/tmp/*", "/var/log/*"]}, + "packages": {"added": [], "removed": [], "version_changed": []}, + "services": {"enabled_added": [], "enabled_removed": [], "changed": []}, + "users": {"added": [], "removed": [], "changed": []}, + "files": {"added": [], "removed": [], "changed": []}, + } + result = d._report_text(report) + assert "file exclude patterns" in result + assert "/tmp/*" in result + + +def test_report_text_with_ignore_package_versions(): + """Test _report_text includes ignore package versions message.""" + import enroll.diff as d + + report = { + "generated_at": "2024-01-01T00:00:00Z", + "old": {"input": "old.tar.gz", "host": "host1", "state_mtime": "mtime1"}, + "new": {"input": "new.tar.gz", "host": "host2", "state_mtime": "mtime2"}, + "filters": {"ignore_package_versions": True}, + "packages": {"version_changed_ignored_count": 5}, + "services": {"enabled_added": [], "enabled_removed": [], "changed": []}, + "users": {"added": [], "removed": [], "changed": []}, + "files": {"added": [], "removed": [], "changed": []}, + } + result = d._report_text(report) + assert "package version drift: ignored" in result + assert "ignored 5 changes" in result + + +def test_report_text_with_enforcement_applied(): + """Test _report_text includes enforcement applied status.""" + import enroll.diff as d + + report = { + "generated_at": "2024-01-01T00:00:00Z", + "old": {"input": "old.tar.gz", "host": "host1", "state_mtime": "mtime1"}, + "new": {"input": "new.tar.gz", "host": "host2", "state_mtime": "mtime2"}, + "packages": {"added": [], "removed": [], "version_changed": []}, + "services": {"enabled_added": [], "enabled_removed": [], "changed": []}, + "users": {"added": [], "removed": [], "changed": []}, + "files": {"added": [], "removed": [], "changed": []}, + "enforcement": { + "status": "applied", + "returncode": 0, + "tags": ["test"], + "finished_at": "2024-01-01T01:00:00Z", + }, + } + result = d._report_text(report) + assert "Enforcement" in result + assert "applied old harvest via ansible-playbook" in result + assert "tags=test" in result + + +def test_report_text_with_enforcement_failed(): + """Test _report_text includes enforcement failed status.""" + import enroll.diff as d + + report = { + "generated_at": "2024-01-01T00:00:00Z", + "old": {"input": "old.tar.gz", "host": "host1", "state_mtime": "mtime1"}, + "new": {"input": "new.tar.gz", "host": "host2", "state_mtime": "mtime2"}, + "packages": {"added": [], "removed": [], "version_changed": []}, + "services": {"enabled_added": [], "enabled_removed": [], "changed": []}, + "users": {"added": [], "removed": [], "changed": []}, + "files": {"added": [], "removed": [], "changed": []}, + "enforcement": {"status": "failed", "returncode": 1}, + } + result = d._report_text(report) + assert "Enforcement" in result + assert "ansible-playbook failed" in result + + +def test_report_text_with_enforcement_skipped(): + """Test _report_text includes enforcement skipped status.""" + import enroll.diff as d + + report = { + "generated_at": "2024-01-01T00:00:00Z", + "old": {"input": "old.tar.gz", "host": "host1", "state_mtime": "mtime1"}, + "new": {"input": "new.tar.gz", "host": "host2", "state_mtime": "mtime2"}, + "packages": {"added": [], "removed": [], "version_changed": []}, + "services": {"enabled_added": [], "enabled_removed": [], "changed": []}, + "users": {"added": [], "removed": [], "changed": []}, + "files": {"added": [], "removed": [], "changed": []}, + "enforcement": {"status": "skipped", "reason": "no changes"}, + } + result = d._report_text(report) + assert "Enforcement" in result + assert "skipped" in result + assert "no changes" in result diff --git a/tests/test_diff_ignore_versions_exclude_enforce.py b/tests/test_diff_ignore_versions_exclude_enforce.py new file mode 100644 index 0000000..fd0524f --- /dev/null +++ b/tests/test_diff_ignore_versions_exclude_enforce.py @@ -0,0 +1,400 @@ +from __future__ import annotations + +import json +import sys +import types +from pathlib import Path + +import pytest + + +def _write_bundle( + root: Path, state: dict, artifacts: dict[str, bytes] | None = None +) -> None: + root.mkdir(parents=True, exist_ok=True) + (root / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + artifacts = artifacts or {} + for rel, data in artifacts.items(): + p = root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(data) + + +def _minimal_roles() -> dict: + """A small roles structure that's sufficient for enroll.diff file indexing.""" + return { + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [], + "packages": [], + "apt_config": { + "role_name": "apt_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + } + + +def test_diff_ignore_package_versions_suppresses_version_drift(tmp_path: Path): + from enroll.diff import compare_harvests + + old = tmp_path / "old" + new = tmp_path / "new" + + old_state = { + "schema_version": 3, + "host": {"hostname": "h1"}, + "inventory": { + "packages": { + "curl": { + "version": "1.0", + "installations": [{"version": "1.0", "arch": "amd64"}], + } + } + }, + "roles": _minimal_roles(), + } + new_state = { + **old_state, + "inventory": { + "packages": { + "curl": { + "version": "1.1", + "installations": [{"version": "1.1", "arch": "amd64"}], + } + } + }, + } + + _write_bundle(old, old_state) + _write_bundle(new, new_state) + + # Without ignore flag, version drift is reported and counts as changes. + report, has_changes = compare_harvests(str(old), str(new)) + assert has_changes is True + assert report["packages"]["version_changed"] + + # With ignore flag, version drift is suppressed and does not count as changes. + report2, has_changes2 = compare_harvests( + str(old), str(new), ignore_package_versions=True + ) + assert has_changes2 is False + assert report2["packages"]["version_changed"] == [] + assert report2["packages"]["version_changed_ignored_count"] == 1 + assert report2["filters"]["ignore_package_versions"] is True + + +def test_diff_exclude_path_filters_file_drift_and_affects_has_changes(tmp_path: Path): + from enroll.diff import compare_harvests + + old = tmp_path / "old" + new = tmp_path / "new" + + # Only file drift is under /var/anacron, which is excluded. + old_state = { + "schema_version": 3, + "host": {"hostname": "h1"}, + "inventory": {"packages": {}}, + "roles": { + **_minimal_roles(), + "extra_paths": { + **_minimal_roles()["extra_paths"], + "managed_files": [ + { + "path": "/var/anacron/daily.stamp", + "src_rel": "var/anacron/daily.stamp", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "extra_path", + } + ], + }, + }, + } + new_state = json.loads(json.dumps(old_state)) + + _write_bundle( + old, + old_state, + {"artifacts/extra_paths/var/anacron/daily.stamp": b"yesterday\n"}, + ) + _write_bundle( + new, + new_state, + {"artifacts/extra_paths/var/anacron/daily.stamp": b"today\n"}, + ) + + report, has_changes = compare_harvests( + str(old), str(new), exclude_paths=["/var/anacron"] + ) + assert has_changes is False + assert report["files"]["changed"] == [] + assert report["filters"]["exclude_paths"] == ["/var/anacron"] + + +def test_diff_exclude_path_only_filters_files_not_packages(tmp_path: Path): + from enroll.diff import compare_harvests + + old = tmp_path / "old" + new = tmp_path / "new" + + old_state = { + "schema_version": 3, + "host": {"hostname": "h1"}, + "inventory": {"packages": {"curl": {"version": "1.0"}}}, + "roles": { + **_minimal_roles(), + "extra_paths": { + **_minimal_roles()["extra_paths"], + "managed_files": [ + { + "path": "/var/anacron/daily.stamp", + "src_rel": "var/anacron/daily.stamp", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "extra_path", + } + ], + }, + }, + } + new_state = { + **old_state, + "inventory": { + "packages": { + "curl": {"version": "1.0"}, + "htop": {"version": "3.0"}, + } + }, + } + + _write_bundle( + old, + old_state, + {"artifacts/extra_paths/var/anacron/daily.stamp": b"yesterday\n"}, + ) + _write_bundle( + new, + new_state, + { + "artifacts/extra_paths/var/anacron/daily.stamp": b"today\n", + }, + ) + + report, has_changes = compare_harvests( + str(old), str(new), exclude_paths=["/var/anacron"] + ) + assert has_changes is True + # File drift is filtered, but package drift remains. + assert report["files"]["changed"] == [] + assert report["packages"]["added"] == ["htop"] + + +def test_enforce_old_harvest_requires_ansible_playbook(monkeypatch, tmp_path: Path): + import enroll.diff as d + + monkeypatch.setattr(d.shutil, "which", lambda name: None) + + old = tmp_path / "old" + _write_bundle(old, {"inventory": {"packages": {}}, "roles": _minimal_roles()}) + + with pytest.raises(RuntimeError, match="ansible-playbook not found"): + d.enforce_old_harvest(str(old)) + + +def test_enforce_old_harvest_runs_ansible_with_tags_from_file_drift( + monkeypatch, tmp_path: Path +): + import enroll.diff as d + import enroll.manifest as mf + + # Pretend ansible-playbook is installed. + monkeypatch.setattr(d.shutil, "which", lambda name: "/usr/bin/ansible-playbook") + + calls: dict[str, object] = {} + + # Stub manifest generation to only create playbook.yml (fast, no real roles needed). + def fake_manifest(_harvest_dir: str, out_dir: str, **_kwargs): + out = Path(out_dir) + (out / "playbook.yml").write_text( + "---\n- hosts: all\n gather_facts: false\n roles: []\n", + encoding="utf-8", + ) + + monkeypatch.setattr(mf, "manifest", fake_manifest) + + def fake_run( + argv, cwd=None, env=None, capture_output=False, text=False, check=False + ): + calls["argv"] = list(argv) + calls["cwd"] = cwd + return types.SimpleNamespace(returncode=0, stdout="ok", stderr="") + + monkeypatch.setattr(d.subprocess, "run", fake_run) + + old = tmp_path / "old" + old_state = { + "schema_version": 3, + "host": {"hostname": "h1"}, + "inventory": {"packages": {}}, + "roles": { + **_minimal_roles(), + "usr_local_custom": { + **_minimal_roles()["usr_local_custom"], + "managed_files": [ + { + "path": "/etc/myapp.conf", + "src_rel": "etc/myapp.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "custom", + } + ], + }, + }, + } + _write_bundle(old, old_state) + + # Minimal report containing enforceable drift: a baseline file is "removed". + report = { + "packages": {"added": [], "removed": [], "version_changed": []}, + "services": {"enabled_added": [], "enabled_removed": [], "changed": []}, + "users": {"added": [], "removed": [], "changed": []}, + "files": { + "added": [], + "removed": [{"path": "/etc/myapp.conf", "role": "usr_local_custom"}], + "changed": [], + }, + } + + info = d.enforce_old_harvest(str(old), report=report) + assert info["status"] == "applied" + assert "--tags" in info["command"] + assert "role_usr_local_custom" in ",".join(info.get("tags") or []) + + argv = calls.get("argv") + assert argv and argv[0].endswith("ansible-playbook") + assert "--tags" in argv + # Ensure we pass the computed tag. + i = argv.index("--tags") + assert "role_usr_local_custom" in str(argv[i + 1]) + + +def test_cli_diff_forwards_exclude_and_ignore_flags(monkeypatch, capsys): + import enroll.cli as cli + + calls: dict[str, object] = {} + + def fake_compare( + old, new, *, sops_mode=False, exclude_paths=None, ignore_package_versions=False + ): + calls["compare"] = { + "old": old, + "new": new, + "sops_mode": sops_mode, + "exclude_paths": exclude_paths, + "ignore_package_versions": ignore_package_versions, + } + # No changes -> should not try to enforce. + return {"packages": {}, "services": {}, "users": {}, "files": {}}, False + + monkeypatch.setattr(cli, "compare_harvests", fake_compare) + monkeypatch.setattr(cli, "format_report", lambda report, fmt="text": "R\n") + monkeypatch.setattr( + sys, + "argv", + [ + "enroll", + "diff", + "--old", + "/tmp/old", + "--new", + "/tmp/new", + "--exclude-path", + "/var/anacron", + "--ignore-package-versions", + ], + ) + + cli.main() + _ = capsys.readouterr() + assert calls["compare"]["exclude_paths"] == ["/var/anacron"] + assert calls["compare"]["ignore_package_versions"] is True + + +def test_cli_diff_enforce_skips_when_no_enforceable_drift(monkeypatch): + import enroll.cli as cli + + # Drift exists, but is not enforceable (only additions / version changes). + report = { + "packages": {"added": ["htop"], "removed": [], "version_changed": []}, + "services": { + "enabled_added": ["x.service"], + "enabled_removed": [], + "changed": [], + }, + "users": {"added": ["bob"], "removed": [], "changed": []}, + "files": {"added": [{"path": "/tmp/new"}], "removed": [], "changed": []}, + } + + monkeypatch.setattr(cli, "compare_harvests", lambda *a, **k: (report, True)) + monkeypatch.setattr(cli, "has_enforceable_drift", lambda r: False) + called = {"enforce": False} + monkeypatch.setattr( + cli, "enforce_old_harvest", lambda *a, **k: called.update({"enforce": True}) + ) + + captured = {} + + def fake_format(rep, fmt="text"): + captured["report"] = rep + return "R\n" + + monkeypatch.setattr(cli, "format_report", fake_format) + + monkeypatch.setattr( + sys, + "argv", + [ + "enroll", + "diff", + "--old", + "/tmp/old", + "--new", + "/tmp/new", + "--enforce", + ], + ) + + cli.main() + assert called["enforce"] is False + assert captured["report"]["enforcement"]["status"] == "skipped" diff --git a/tests/test_diff_notifications.py b/tests/test_diff_notifications.py new file mode 100644 index 0000000..53f6b57 --- /dev/null +++ b/tests/test_diff_notifications.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import types + +import pytest + + +def test_post_webhook_success(monkeypatch): + from enroll.diff import post_webhook + + class FakeResp: + status = 204 + + def read(self): + return b"OK" + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + monkeypatch.setattr( + "enroll.diff.urllib.request.urlopen", + lambda req, timeout=30: FakeResp(), + ) + + status, body = post_webhook("https://example.com", b"x") + assert status == 204 + assert body == "OK" + + +def test_post_webhook_raises_on_error(monkeypatch): + from enroll.diff import post_webhook + + def boom(_req, timeout=30): + raise OSError("nope") + + monkeypatch.setattr("enroll.diff.urllib.request.urlopen", boom) + + with pytest.raises(RuntimeError): + post_webhook("https://example.com", b"x") + + +def test_send_email_uses_sendmail_when_present(monkeypatch): + from enroll.diff import send_email + + calls = {} + + monkeypatch.setattr("enroll.diff.shutil.which", lambda name: "/usr/sbin/sendmail") + + def fake_run(argv, input=None, check=False, **_kwargs): + calls["argv"] = argv + calls["input"] = input + return types.SimpleNamespace(returncode=0) + + monkeypatch.setattr("enroll.diff.subprocess.run", fake_run) + + send_email( + subject="Subj", + body="Body", + from_addr="a@example.com", + to_addrs=["b@example.com"], + ) + + assert calls["argv"][0].endswith("sendmail") + msg = (calls["input"] or b"").decode("utf-8", errors="replace") + assert "Subject: Subj" in msg + assert "To: b@example.com" in msg + + +def test_send_email_raises_when_no_delivery_method(monkeypatch): + from enroll.diff import send_email + + monkeypatch.setattr("enroll.diff.shutil.which", lambda name: None) + + with pytest.raises(RuntimeError): + send_email( + subject="Subj", + body="Body", + from_addr="a@example.com", + to_addrs=["b@example.com"], + ) diff --git a/tests/test_diff_usr_local_custom.py b/tests/test_diff_usr_local_custom.py new file mode 100644 index 0000000..28ec57c --- /dev/null +++ b/tests/test_diff_usr_local_custom.py @@ -0,0 +1,152 @@ +import json +from pathlib import Path + +from enroll.diff import compare_harvests + + +def _write_bundle(root: Path, state: dict, artifacts: dict[str, bytes]) -> None: + root.mkdir(parents=True, exist_ok=True) + (root / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + for rel, data in artifacts.items(): + p = root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(data) + + +def test_diff_includes_usr_local_custom_files(tmp_path: Path): + old = tmp_path / "old" + new = tmp_path / "new" + + old_state = { + "schema_version": 3, + "host": {"hostname": "h1", "os": "debian", "pkg_backend": "dpkg"}, + "inventory": { + "packages": { + "curl": { + "version": "1.0", + "arches": [], + "installations": [{"version": "1.0", "arch": "amd64"}], + "observed_via": [{"kind": "user_installed"}], + "roles": [], + } + } + }, + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [], + "packages": [], + "apt_config": { + "role_name": "apt_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [ + { + "path": "/usr/local/etc/myapp.conf", + "src_rel": "usr/local/etc/myapp.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "usr_local_etc_custom", + } + ], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + }, + } + + new_state = { + **old_state, + "inventory": { + "packages": { + **old_state["inventory"]["packages"], + "htop": { + "version": "3.0", + "arches": [], + "installations": [{"version": "3.0", "arch": "amd64"}], + "observed_via": [{"kind": "user_installed"}], + "roles": [], + }, + } + }, + "roles": { + **old_state["roles"], + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [ + { + "path": "/usr/local/etc/myapp.conf", + "src_rel": "usr/local/etc/myapp.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "usr_local_etc_custom", + }, + { + "path": "/usr/local/bin/myscript", + "src_rel": "usr/local/bin/myscript", + "owner": "root", + "group": "root", + "mode": "0755", + "reason": "usr_local_bin_script", + }, + ], + "excluded": [], + "notes": [], + }, + }, + } + + _write_bundle( + old, + old_state, + { + "artifacts/usr_local_custom/usr/local/etc/myapp.conf": b"myapp=1\n", + }, + ) + _write_bundle( + new, + new_state, + { + "artifacts/usr_local_custom/usr/local/etc/myapp.conf": b"myapp=2\n", + "artifacts/usr_local_custom/usr/local/bin/myscript": b"#!/bin/sh\necho hi\n", + }, + ) + + report, has_changes = compare_harvests(str(old), str(new)) + assert has_changes is True + + # Packages: htop was added. + assert report["packages"]["added"] == ["htop"] + + # Files: /usr/local/etc/myapp.conf should be detected as changed (content sha differs). + changed_paths = {c["path"] for c in report["files"]["changed"]} + assert "/usr/local/etc/myapp.conf" in changed_paths + + # Files: new script was added. + added_paths = {a["path"] for a in report["files"]["added"]} + assert "/usr/local/bin/myscript" in added_paths diff --git a/tests/test_explain.py b/tests/test_explain.py new file mode 100644 index 0000000..69f4a88 --- /dev/null +++ b/tests/test_explain.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import enroll.explain as ex + + +def _write_state(bundle: Path, state: dict) -> Path: + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + return bundle / "state.json" + + +def test_explain_state_text_renders_roles_inventory_and_reasons(tmp_path: Path): + bundle = tmp_path / "bundle" + state = { + "schema_version": 3, + "host": {"hostname": "h1", "os": "debian", "pkg_backend": "dpkg"}, + "enroll": {"version": "0.0.0"}, + "inventory": { + "packages": { + "foo": { + "installations": [{"version": "1.0", "arch": "amd64"}], + "observed_via": [ + {"kind": "systemd_unit", "ref": "foo.service"}, + {"kind": "package_role", "ref": "foo"}, + ], + "roles": ["foo"], + }, + "bar": { + "installations": [{"version": "2.0", "arch": "amd64"}], + "observed_via": [{"kind": "user_installed", "ref": "manual"}], + "roles": ["bar"], + }, + } + }, + "roles": { + "users": { + "role_name": "users", + "users": [{"name": "alice"}], + "managed_files": [ + { + "path": "/home/alice/.ssh/authorized_keys", + "src_rel": "home/alice/.ssh/authorized_keys", + "owner": "alice", + "group": "alice", + "mode": "0600", + "reason": "authorized_keys", + } + ], + "managed_dirs": [ + { + "path": "/home/alice/.ssh", + "owner": "alice", + "group": "alice", + "mode": "0700", + "reason": "parent_of_managed_file", + } + ], + "excluded": [{"path": "/etc/shadow", "reason": "sensitive_content"}], + "notes": ["n1", "n2"], + }, + "services": [ + { + "unit": "foo.service", + "role_name": "foo", + "packages": ["foo"], + "managed_files": [ + { + "path": "/etc/foo.conf", + "src_rel": "etc/foo.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "modified_conffile", + }, + # Unknown reason should fall back to generic text. + { + "path": "/etc/odd.conf", + "src_rel": "etc/odd.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "mystery_reason", + }, + ], + "excluded": [], + "notes": [], + } + ], + "packages": [ + { + "package": "bar", + "role_name": "bar", + "managed_files": [], + "excluded": [], + "notes": [], + } + ], + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": ["/etc/a", "/etc/b"], + "exclude_patterns": ["/etc/x", "/etc/y"], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "apt_config": { + "role_name": "apt_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + }, + } + + state_path = _write_state(bundle, state) + + out = ex.explain_state(str(state_path), fmt="text", max_examples=1) + + assert "Enroll explained:" in out + assert "Host: h1" in out + assert "Inventory" in out + # observed_via summary should include both kinds (order not strictly guaranteed) + assert "observed_via" in out + assert "systemd_unit" in out + assert "user_installed" in out + + # extra_paths include/exclude patterns should be rendered with max_examples truncation. + assert "include_patterns:" in out + assert "/etc/a" in out + assert "exclude_patterns:" in out + + # Reasons section should mention known and unknown reasons. + assert "modified_conffile" in out + assert "mystery_reason" in out + assert "Captured with reason 'mystery_reason'" in out + + # Excluded paths section. + assert "Why paths were excluded" in out + assert "sensitive_content" in out + + +def test_explain_state_json_contains_structured_report(tmp_path: Path): + bundle = tmp_path / "bundle" + state = { + "schema_version": 3, + "host": {"hostname": "h2", "os": "rhel", "pkg_backend": "rpm"}, + "enroll": {"version": "1.2.3"}, + "inventory": {"packages": {}}, + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [], + "packages": [], + "apt_config": { + "role_name": "apt_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + }, + } + state_path = _write_state(bundle, state) + + raw = ex.explain_state(str(state_path), fmt="json", max_examples=2) + rep = json.loads(raw) + assert rep["host"]["hostname"] == "h2" + assert rep["enroll"]["version"] == "1.2.3" + assert rep["inventory"]["package_count"] == 0 + assert isinstance(rep["roles"], list) + assert "reasons" in rep diff --git a/tests/test_fsutil.py b/tests/test_fsutil.py new file mode 100644 index 0000000..ebe2224 --- /dev/null +++ b/tests/test_fsutil.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import hashlib +import os +from pathlib import Path + +from enroll.fsutil import file_md5, stat_triplet + + +def test_file_md5_matches_hashlib(tmp_path: Path): + p = tmp_path / "x" + p.write_bytes(b"hello world") + expected = hashlib.md5(b"hello world").hexdigest() # nosec + assert file_md5(str(p)) == expected + + +def test_stat_triplet_reports_mode(tmp_path: Path): + p = tmp_path / "x" + p.write_text("x", encoding="utf-8") + os.chmod(p, 0o600) + + owner, group, mode = stat_triplet(str(p)) + assert mode == "0600" + assert owner # non-empty string + assert group # non-empty string diff --git a/tests/test_fsutil_extra.py b/tests/test_fsutil_extra.py new file mode 100644 index 0000000..9b70a67 --- /dev/null +++ b/tests/test_fsutil_extra.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import os +from pathlib import Path + + +def test_stat_triplet_falls_back_to_numeric_ids(tmp_path: Path, monkeypatch): + """If uid/gid cannot be resolved, stat_triplet should return numeric strings.""" + from enroll.fsutil import stat_triplet + + p = tmp_path / "f" + p.write_text("x", encoding="utf-8") + os.chmod(p, 0o644) + + import grp + import pwd + + def _no_user(_uid): # pragma: no cover - executed via monkeypatch + raise KeyError + + def _no_group(_gid): # pragma: no cover - executed via monkeypatch + raise KeyError + + monkeypatch.setattr(pwd, "getpwuid", _no_user) + monkeypatch.setattr(grp, "getgrgid", _no_group) + + owner, group, mode = stat_triplet(str(p)) + + assert owner.isdigit() + assert group.isdigit() + assert mode == "0644" diff --git a/tests/test_harvest.py b/tests/test_harvest.py index 8e19fb4..a308bcf 100644 --- a/tests/test_harvest.py +++ b/tests/test_harvest.py @@ -1,8 +1,35 @@ import json +import os +import pytest + from pathlib import Path -import enroll.harvest as h +import enroll.harvest as harvest +import enroll.system_paths as system_paths +from enroll.platform import PlatformInfo from enroll.systemd import UnitInfo +from enroll.pathfilter import PathFilter +import enroll.capture as capture +from enroll.capture import ( + capture_file as _capture_file, + capture_link as _capture_link, + capture_user_shell_dotfiles, + files_differ, +) +from enroll.harvest_types import ExcludedFile, ManagedFile, ManagedLink +from enroll.ignore import IgnorePolicy +from enroll.package_hints import ( + add_pkgs_from_etc_topdirs, + hint_names as _hint_names, +) +from enroll.system_paths import ( + is_confish as _is_confish, + iter_matching_files as _iter_matching_files, + parse_apt_signed_by as _parse_apt_signed_by, + topdirs_for_package as _topdirs_for_package, +) + +from unittest.mock import MagicMock class AllowAllPolicy: @@ -10,6 +37,74 @@ class AllowAllPolicy: return None +class FakeBackend: + """Minimal backend stub for harvest tests. + + The real backends (dpkg/rpm) enumerate the live system (dpkg status, rpm + databases, etc). These tests instead control all backend behaviour. + """ + + def __init__( + self, + *, + name: str, + owned_etc: set[str], + etc_owner_map: dict[str, str], + topdir_to_pkgs: dict[str, set[str]], + pkg_to_etc_paths: dict[str, list[str]], + manual_pkgs: list[str], + owner_fn, + modified_by_pkg: dict[str, dict[str, str]] | None = None, + pkg_config_prefixes: tuple[str, ...] = ("/etc/apt/",), + installed: dict[str, list[dict[str, str]]] | None = None, + ): + self.name = name + self.pkg_config_prefixes = pkg_config_prefixes + self._owned_etc = owned_etc + self._etc_owner_map = etc_owner_map + self._topdir_to_pkgs = topdir_to_pkgs + self._pkg_to_etc_paths = pkg_to_etc_paths + self._manual = manual_pkgs + self._owner_fn = owner_fn + self._modified_by_pkg = modified_by_pkg or {} + self._installed = installed or {} + + def build_etc_index(self): + return ( + self._owned_etc, + self._etc_owner_map, + self._topdir_to_pkgs, + self._pkg_to_etc_paths, + ) + + def owner_of_path(self, path: str): + return self._owner_fn(path) + + def list_manual_packages(self): + return list(self._manual) + + def installed_packages(self): + """Return mapping package -> installations. + + The real backends return: + {"pkg": [{"version": "...", "arch": "..."}, ...]} + """ + return dict(self._installed) + + def specific_paths_for_hints(self, hints: set[str]): + return [] + + def is_pkg_config_path(self, path: str) -> bool: + for pfx in self.pkg_config_prefixes: + if path == pfx or path.startswith(pfx): + return True + return False + + def modified_paths(self, pkg: str, etc_paths: list[str]): + # Test-controlled; ignore etc_paths. + return dict(self._modified_by_pkg.get(pkg, {})) + + def test_harvest_dedup_manual_packages_and_builds_etc_custom( monkeypatch, tmp_path: Path ): @@ -22,31 +117,52 @@ def test_harvest_dedup_manual_packages_and_builds_etc_custom( real_exists = os.path.exists real_islink = os.path.islink - # Fake filesystem: two /etc files exist, only one is dpkg-owned. + # Fake filesystem: two /etc files exist, only one is package-owned. + # Also include some /usr/local files to populate usr_local_custom. files = { "/etc/openvpn/server.conf": b"server", "/etc/default/keyboard": b"kbd", + "/usr/local/etc/myapp.conf": b"myapp=1\n", + "/usr/local/bin/myscript": b"#!/bin/sh\necho hi\n", + # non-executable text under /usr/local/bin should be skipped + "/usr/local/bin/readme.txt": b"hello\n", + } + dirs = { + "/etc", + "/etc/openvpn", + "/etc/default", + "/usr", + "/usr/local", + "/usr/local/etc", + "/usr/local/bin", } - dirs = {"/etc", "/etc/openvpn", "/etc/default"} def fake_isfile(p: str) -> bool: if p.startswith("/etc/") or p == "/etc": return p in files + if p.startswith("/usr/local/"): + return p in files return real_isfile(p) def fake_isdir(p: str) -> bool: if p.startswith("/etc"): return p in dirs + if p.startswith("/usr/local") or p in ("/usr", "/usr/local"): + return p in dirs return real_isdir(p) def fake_islink(p: str) -> bool: if p.startswith("/etc"): return False + if p.startswith("/usr/local"): + return False return real_islink(p) def fake_exists(p: str) -> bool: if p.startswith("/etc"): return p in files or p in dirs + if p.startswith("/usr/local") or p in ("/usr", "/usr/local"): + return p in files or p in dirs return real_exists(p) def fake_walk(root: str): @@ -57,19 +173,24 @@ def test_harvest_dedup_manual_packages_and_builds_etc_custom( yield ("/etc/openvpn", [], ["server.conf"]) elif root == "/etc/default": yield ("/etc/default", [], ["keyboard"]) + elif root == "/usr/local/etc": + yield ("/usr/local/etc", [], ["myapp.conf"]) + elif root == "/usr/local/bin": + yield ("/usr/local/bin", [], ["myscript", "readme.txt"]) else: yield (root, [], []) - monkeypatch.setattr(h.os.path, "isfile", fake_isfile) - monkeypatch.setattr(h.os.path, "isdir", fake_isdir) - monkeypatch.setattr(h.os.path, "islink", fake_islink) - monkeypatch.setattr(h.os.path, "exists", fake_exists) - monkeypatch.setattr(h.os, "walk", fake_walk) + monkeypatch.setattr(harvest.os.path, "isfile", fake_isfile) + monkeypatch.setattr(harvest.os.path, "isdir", fake_isdir) + monkeypatch.setattr(harvest.os.path, "islink", fake_islink) + monkeypatch.setattr(harvest.os.path, "exists", fake_exists) + monkeypatch.setattr(harvest.os, "walk", fake_walk) # Avoid real system access - monkeypatch.setattr(h, "list_enabled_services", lambda: ["openvpn.service"]) + monkeypatch.setattr(harvest, "list_enabled_services", lambda: ["openvpn.service"]) + monkeypatch.setattr(harvest, "list_enabled_timers", lambda: []) monkeypatch.setattr( - h, + harvest, "get_unit_info", lambda unit: UnitInfo( name=unit, @@ -84,32 +205,58 @@ def test_harvest_dedup_manual_packages_and_builds_etc_custom( ), ) - # Debian package index: openvpn owns /etc/openvpn/server.conf; keyboard is unowned. - def fake_build_index(): - owned_etc = {"/etc/openvpn/server.conf"} - etc_owner_map = {"/etc/openvpn/server.conf": "openvpn"} - topdir_to_pkgs = {"openvpn": {"openvpn"}} - pkg_to_etc_paths = {"openvpn": ["/etc/openvpn/server.conf"], "curl": []} - return owned_etc, etc_owner_map, topdir_to_pkgs, pkg_to_etc_paths + # Package index: openvpn owns /etc/openvpn/server.conf; keyboard is unowned. + owned_etc = {"/etc/openvpn/server.conf"} + etc_owner_map = {"/etc/openvpn/server.conf": "openvpn"} + topdir_to_pkgs = {"openvpn": {"openvpn"}} + # curl has a package-owned /etc path, but no changed/custom harvested + # artifacts. That should still be considered a simple package role. + pkg_to_etc_paths = { + "openvpn": ["/etc/openvpn/server.conf"], + "curl": ["/etc/curl/curlrc"], + } - monkeypatch.setattr(h, "build_dpkg_etc_index", fake_build_index) - - # openvpn conffile hash mismatch => should be captured under service role - monkeypatch.setattr( - h, - "parse_status_conffiles", - lambda: {"openvpn": {"/etc/openvpn/server.conf": "old"}}, + backend = FakeBackend( + name="dpkg", + owned_etc=owned_etc, + etc_owner_map=etc_owner_map, + topdir_to_pkgs=topdir_to_pkgs, + pkg_to_etc_paths=pkg_to_etc_paths, + manual_pkgs=["openvpn", "curl"], + owner_fn=lambda p: "openvpn" if "openvpn" in (p or "") else None, + modified_by_pkg={ + "openvpn": {"/etc/openvpn/server.conf": "modified_conffile"}, + }, ) - monkeypatch.setattr(h, "read_pkg_md5sums", lambda pkg: {}) - monkeypatch.setattr(h, "file_md5", lambda path: "new") monkeypatch.setattr( - h, "dpkg_owner", lambda p: "openvpn" if "openvpn" in p else None + harvest, "detect_platform", lambda: PlatformInfo("debian", "dpkg", {}) ) - monkeypatch.setattr(h, "list_manual_packages", lambda: ["openvpn", "curl"]) - monkeypatch.setattr(h, "collect_non_system_users", lambda: []) + monkeypatch.setattr(harvest, "get_backend", lambda info=None: backend) - monkeypatch.setattr(h, "stat_triplet", lambda p: ("root", "root", "0644")) + monkeypatch.setattr(harvest, "collect_non_system_users", lambda: []) + + import enroll.accounts as accounts + + monkeypatch.setattr(accounts, "find_system_flatpaks", lambda: []) + monkeypatch.setattr(accounts, "find_system_flatpak_remotes", lambda: []) + monkeypatch.setattr( + accounts, "find_user_flatpak_remotes", lambda home, user=None: [] + ) + monkeypatch.setattr( + accounts, + "find_system_snaps", + lambda: [accounts.SnapInstall(name="code", channel="latest/stable")], + ) + + def fake_stat_triplet(p: str): + if p == "/usr/local/bin/myscript": + return ("root", "root", "0755") + # /usr/local/bin/readme.txt remains non-executable + return ("root", "root", "0644") + + monkeypatch.setattr(harvest, "stat_triplet", fake_stat_triplet) + monkeypatch.setattr(capture, "stat_triplet", fake_stat_triplet) # Avoid needing source files on disk by implementing our own bundle copier def fake_copy(bundle_dir: str, role_name: str, abs_path: str, src_rel: str): @@ -117,25 +264,879 @@ def test_harvest_dedup_manual_packages_and_builds_etc_custom( dst.parent.mkdir(parents=True, exist_ok=True) dst.write_bytes(files.get(abs_path, b"")) - monkeypatch.setattr(h, "_copy_into_bundle", fake_copy) + monkeypatch.setattr(capture, "copy_into_bundle", fake_copy) - state_path = h.harvest(str(bundle), policy=AllowAllPolicy()) + state_path = harvest.harvest(str(bundle), policy=AllowAllPolicy()) st = json.loads(Path(state_path).read_text(encoding="utf-8")) - assert "openvpn" in st["manual_packages"] - assert "curl" in st["manual_packages"] - assert "openvpn" in st["manual_packages_skipped"] - assert all(pr["package"] != "openvpn" for pr in st["package_roles"]) - assert any(pr["package"] == "curl" for pr in st["package_roles"]) + inv = st["inventory"]["packages"] + assert "openvpn" in inv + assert "curl" in inv + + # openvpn is managed by the service role, so it should NOT appear as a package role. + pkg_roles = st["roles"]["packages"] + assert all(pr["package"] != "openvpn" for pr in pkg_roles) + assert any(pr["package"] == "curl" for pr in pkg_roles) + curl_role = next(pr for pr in pkg_roles if pr["package"] == "curl") + assert curl_role["has_config"] is False + assert any("No changed or custom configuration" in n for n in curl_role["notes"]) + + # Inventory provenance: openvpn should be observed via systemd unit. + openvpn_obs = inv["openvpn"]["observed_via"] + assert any( + o.get("kind") == "systemd_unit" and o.get("ref") == "openvpn.service" + for o in openvpn_obs + ) + + assert st["roles"]["snap"]["role_name"] == "snap" + assert st["roles"]["snap"]["system_snaps"][0]["name"] == "code" # Service role captured modified conffile - svc = st["services"][0] + svc = st["roles"]["services"][0] assert svc["unit"] == "openvpn.service" assert "openvpn" in svc["packages"] assert any(mf["path"] == "/etc/openvpn/server.conf" for mf in svc["managed_files"]) # Unowned /etc/default/keyboard is attributed to etc_custom only - etc_custom = st["etc_custom"] + etc_custom = st["roles"]["etc_custom"] assert any( mf["path"] == "/etc/default/keyboard" for mf in etc_custom["managed_files"] ) + + # /usr/local content is attributed to usr_local_custom + ul = st["roles"]["usr_local_custom"] + assert any(mf["path"] == "/usr/local/etc/myapp.conf" for mf in ul["managed_files"]) + assert any(mf["path"] == "/usr/local/bin/myscript" for mf in ul["managed_files"]) + assert all(mf["path"] != "/usr/local/bin/readme.txt" for mf in ul["managed_files"]) + + +def test_shared_cron_snippet_prefers_matching_role_over_lexicographic( + monkeypatch, tmp_path: Path +): + """Regression test for shared snippet routing. + + When multiple service roles reference the same owning package, we prefer the + role whose name matches the snippet/package (e.g. ntpsec) rather than a + lexicographic tie-break that could incorrectly pick another role. + """ + + bundle = tmp_path / "bundle" + + files = {"/etc/cron.d/ntpsec": b"# cron\n"} + dirs = {"/etc", "/etc/cron.d"} + + monkeypatch.setattr(harvest.os.path, "isfile", lambda p: p in files) + monkeypatch.setattr(harvest.os.path, "islink", lambda p: False) + monkeypatch.setattr(harvest.os.path, "isdir", lambda p: p in dirs) + monkeypatch.setattr(harvest.os.path, "exists", lambda p: p in files or p in dirs) + monkeypatch.setattr( + harvest.os, "walk", lambda root: [("/etc/cron.d", [], ["ntpsec"])] + ) + + # Only include the cron snippet in the system capture set. + monkeypatch.setattr( + system_paths, + "iter_system_capture_paths", + lambda: [("/etc/cron.d/ntpsec", "system_cron")], + ) + + monkeypatch.setattr( + harvest, "list_enabled_services", lambda: ["apparmor.service", "ntpsec.service"] + ) + monkeypatch.setattr(harvest, "list_enabled_timers", lambda: []) + + def fake_unit_info(unit: str) -> UnitInfo: + if unit == "apparmor.service": + return UnitInfo( + name=unit, + fragment_path="/lib/systemd/system/apparmor.service", + dropin_paths=[], + env_files=[], + exec_paths=["/usr/sbin/apparmor"], + active_state="active", + sub_state="running", + unit_file_state="enabled", + condition_result=None, + ) + return UnitInfo( + name=unit, + fragment_path="/lib/systemd/system/ntpsec.service", + dropin_paths=[], + env_files=[], + exec_paths=["/usr/sbin/ntpd"], + active_state="active", + sub_state="running", + unit_file_state="enabled", + condition_result=None, + ) + + monkeypatch.setattr(harvest, "get_unit_info", fake_unit_info) + + # Make apparmor *also* claim the ntpsec package (simulates overly-broad + # package inference). The snippet routing should still prefer role 'ntpsec'. + def fake_owner(p: str): + if p == "/etc/cron.d/ntpsec": + return "ntpsec" + if "apparmor" in (p or ""): + return "ntpsec" # intentionally misleading + if "ntpsec" in (p or "") or "ntpd" in (p or ""): + return "ntpsec" + return None + + backend = FakeBackend( + name="dpkg", + owned_etc=set(), + etc_owner_map={}, + topdir_to_pkgs={}, + pkg_to_etc_paths={}, + manual_pkgs=[], + owner_fn=fake_owner, + modified_by_pkg={}, + ) + + monkeypatch.setattr( + harvest, "detect_platform", lambda: PlatformInfo("debian", "dpkg", {}) + ) + monkeypatch.setattr(harvest, "get_backend", lambda info=None: backend) + + monkeypatch.setattr(harvest, "stat_triplet", lambda p: ("root", "root", "0644")) + monkeypatch.setattr(capture, "stat_triplet", lambda p: ("root", "root", "0644")) + monkeypatch.setattr(harvest, "collect_non_system_users", lambda: []) + + def fake_copy(bundle_dir: str, role_name: str, abs_path: str, src_rel: str): + dst = Path(bundle_dir) / "artifacts" / role_name / src_rel + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_bytes(files[abs_path]) + + monkeypatch.setattr(capture, "copy_into_bundle", fake_copy) + + state_path = harvest.harvest(str(bundle), policy=AllowAllPolicy()) + st = json.loads(Path(state_path).read_text(encoding="utf-8")) + + # Cron snippet should end up attached to the ntpsec role, not apparmor. + svc_ntpsec = next(s for s in st["roles"]["services"] if s["role_name"] == "ntpsec") + assert any(mf["path"] == "/etc/cron.d/ntpsec" for mf in svc_ntpsec["managed_files"]) + + svc_apparmor = next( + s for s in st["roles"]["services"] if s["role_name"] == "apparmor" + ) + assert all( + mf["path"] != "/etc/cron.d/ntpsec" for mf in svc_apparmor["managed_files"] + ) + + +def test_files_differ_binary(tmp_path: Path): + file1 = tmp_path / "file1.bin" + file2 = tmp_path / "file2.bin" + file1.write_bytes(b"\x00\x01\x02\x03") + file2.write_bytes(b"\x00\x01\x02\x03") + assert files_differ(str(file1), str(file2)) is False + + +def test_files_differ_binary_different(tmp_path: Path): + file1 = tmp_path / "file1.bin" + file2 = tmp_path / "file2.bin" + file1.write_bytes(b"\x00\x01\x02\x03") + file2.write_bytes(b"\x00\x01\x02\x04") + assert files_differ(str(file1), str(file2)) is True + + +def test_files_differ_non_regular_a(tmp_path: Path): + directory = tmp_path / "dir" + directory.mkdir() + file1 = tmp_path / "file1.txt" + file1.write_text("content", encoding="utf-8") + assert files_differ(str(directory), str(file1)) is True + + +def test_topdirs_for_package_with_multiple_paths(): + pkg_to_etc_paths = { + "nginx": ["/etc/nginx/nginx.conf", "/etc/nginx/sites-enabled/default"], + } + result = _topdirs_for_package("nginx", pkg_to_etc_paths) + assert result == {"nginx"} + + +def test_topdirs_for_package_with_multiple_topdirs(): + pkg_to_etc_paths = { + "multi": ["/etc/nginx/nginx.conf", "/etc/ssh/sshd_config"], + } + result = _topdirs_for_package("multi", pkg_to_etc_paths) + assert result == {"nginx", "ssh"} + + +def test_topdirs_for_package_empty(): + result = _topdirs_for_package("empty", {}) + assert result == set() + + +def test_topdirs_for_package_no_etc(): + pkg_to_etc_paths = { + "other": ["/usr/share/doc/file"], + } + result = _topdirs_for_package("other", pkg_to_etc_paths) + assert result == set() + + +def test_files_differ_same_content(tmp_path: Path): + """Test that _files_differ returns False for identical content.""" + file_a = tmp_path / "a.txt" + file_b = tmp_path / "b.txt" + file_a.write_text("same content", encoding="utf-8") + file_b.write_text("same content", encoding="utf-8") + assert files_differ(str(file_a), str(file_b)) is False + + +def test_files_differ_different_content(tmp_path: Path): + """Test that _files_differ returns True for different content.""" + file_a = tmp_path / "a.txt" + file_b = tmp_path / "b.txt" + file_a.write_text("content a", encoding="utf-8") + file_b.write_text("content b", encoding="utf-8") + assert files_differ(str(file_a), str(file_b)) is True + + +def test_files_differ_missing_file(tmp_path: Path): + """Test that _files_differ returns True when one file is missing.""" + file_a = tmp_path / "a.txt" + file_a.write_text("content", encoding="utf-8") + file_b = tmp_path / "b.txt" + assert files_differ(str(file_a), str(file_b)) is True + + +def test_files_differ_both_missing(tmp_path: Path): + """Test that _files_differ returns True when both files are missing.""" + file_a = tmp_path / "a.txt" + file_b = tmp_path / "b.txt" + # Both missing - should return True (they differ in the sense that neither exists) + assert files_differ(str(file_a), str(file_b)) is True + + +def test_files_differ_non_regular_b(tmp_path: Path): + """Test that _files_differ handles non-regular file (symlink).""" + file_a = tmp_path / "a.txt" + file_a.write_text("content", encoding="utf-8") + link_b = tmp_path / "link" + link_b.symlink_to(file_a) + # Symlinks are followed, so content is the same + assert files_differ(str(file_a), str(link_b)) is False + + +def test_files_differ_oserror_on_read(tmp_path: Path, monkeypatch): + """Test that _files_differ returns True on OSError during read.""" + file_a = tmp_path / "a.txt" + file_b = tmp_path / "b.txt" + file_a.write_text("content", encoding="utf-8") + file_b.write_text("content", encoding="utf-8") + + def fake_open(path, *args, **kwargs): + raise OSError("Permission denied") + + monkeypatch.setattr("builtins.open", fake_open, raising=False) + assert files_differ(str(file_a), str(file_b)) is True + + +def test_files_differ_large_file_returns_true(tmp_path: Path): + """Test that _files_differ returns True for files larger than max_bytes.""" + file_a = tmp_path / "a.bin" + file_b = tmp_path / "b.bin" + # Create files larger than default max_bytes (2MB) + data = b"x" * 3_000_000 + file_a.write_bytes(data) + file_b.write_bytes(data) + # Should return True because files are too large + assert files_differ(str(file_a), str(file_b), max_bytes=1_000_000) is True + + +def test_files_differ_size_mismatch(tmp_path: Path): + """Test that _files_differ detects size mismatch quickly.""" + file_a = tmp_path / "a.txt" + file_b = tmp_path / "b.txt" + file_a.write_text("short", encoding="utf-8") + file_b.write_text("much longer content here", encoding="utf-8") + assert files_differ(str(file_a), str(file_b)) is True + + +def test_files_differ_large_files(tmp_path: Path): + """Test that _files_differ handles large files efficiently.""" + file_a = tmp_path / "a.bin" + file_b = tmp_path / "b.bin" + # Create files with same content but large + data = b"x" * 10000 + file_a.write_bytes(data) + file_b.write_bytes(data) + assert files_differ(str(file_a), str(file_b)) is False + + +def test_hint_names_with_unit_and_packages(): + """Test _hint_names extracts hints from unit and packages.""" + result = _hint_names("nginx.service", {"nginx-common", "nginx-core"}) + assert "nginx" in result + assert "nginx-common" in result + assert "nginx-core" in result + + +def test_hint_names_with_template_unit(): + """Test _hint_names handles template units.""" + result = _hint_names("getty@tty1.service", set()) + assert "getty" in result + assert "getty@tty1" in result + + +def test_hint_names_with_dotted_unit(): + """Test _hint_names handles dotted unit names.""" + result = _hint_names("nginx.service", set()) + assert "nginx" in result + + +def test_hint_names_empty(): + """Test _hint_names with empty inputs.""" + result = _hint_names("", set()) + assert result == set() + + +def test_add_pkgs_from_etc_topdirs(): + """Test _add_pkgs_from_etc_topdirs expands hints.""" + hints = {"nginx"} + topdir_to_pkgs = { + "nginx": {"nginx-common", "nginx-core"}, + "ssh": {"openssh-server"}, + } + pkgs = set() + add_pkgs_from_etc_topdirs(hints, topdir_to_pkgs, pkgs) + # Should add packages from matching topdirs + assert "nginx-common" in pkgs or "nginx-core" in pkgs + + +def test_add_pkgs_from_etc_topdirs_empty(): + """Test _add_pkgs_from_etc_topdirs with empty inputs.""" + hints = set() + topdir_to_pkgs = {} + pkgs = set() + add_pkgs_from_etc_topdirs(hints, topdir_to_pkgs, pkgs) + assert pkgs == set() + + +def test_is_confish_with_conf(tmp_path: Path): + """Test _is_confish recognizes .conf files.""" + file1 = tmp_path / "test.conf" + file1.write_text("[Unit]", encoding="utf-8") + assert _is_confish(str(file1)) is True + + +def test_is_confish_with_yaml(tmp_path: Path): + """Test _is_confish recognizes .yaml files.""" + file1 = tmp_path / "test.yaml" + file1.write_text("key: value", encoding="utf-8") + assert _is_confish(str(file1)) is True + + +def test_is_confish_with_json(tmp_path: Path): + """Test _is_confish recognizes .json files.""" + file1 = tmp_path / "test.json" + file1.write_text('{"key": "value"}', encoding="utf-8") + assert _is_confish(str(file1)) is True + + +def test_is_confish_with_service(tmp_path: Path): + """Test _is_confish recognizes .service files.""" + file1 = tmp_path / "test.service" + file1.write_text("[Unit]", encoding="utf-8") + assert _is_confish(str(file1)) is True + + +def test_is_confish_with_extensionless(tmp_path: Path): + """Test _is_confish recognizes extensionless config files.""" + file1 = tmp_path / "default" + file1.write_text("OPTIONS=", encoding="utf-8") + assert _is_confish(str(file1)) is True + + +def test_is_confish_not_config(tmp_path: Path): + """Test _is_confish rejects non-config files.""" + file1 = tmp_path / "test.log" + file1.write_text("log", encoding="utf-8") + assert _is_confish(str(file1)) is False + + +def test_is_confish_nonexistent(): + """Test _is_confish returns False for nonexistent files.""" + assert _is_confish("/nonexistent/file.xyz") is False + + +"""Additional coverage tests for harvest.py""" + + +class TestIsConfish: + """Tests for _is_confish function""" + + def test_is_confish_true_extensions(self, tmp_path): + """Test files with config extensions are detected.""" + for ext in [".conf", ".cfg", ".ini", ".yaml", ".json", ".cnf"]: + f = tmp_path / f"test{ext}" + f.write_text("test", encoding="utf-8") + assert _is_confish(str(f)) is True + + def test_is_confish_false(self, tmp_path): + """Test non-config files are not detected.""" + for name in ["data.txt", "script.sh"]: + f = tmp_path / name + f.write_text("test", encoding="utf-8") + assert _is_confish(str(f)) is False + + +class TestHintNames: + """Tests for _hint_names function""" + + def test_hint_names_simple(self): + """Test simple hint name extraction.""" + result = _hint_names("nginx", {"nginx"}) + assert "nginx" in result + + def test_hint_names_multiple(self): + """Test multiple hint names.""" + result = _hint_names("nginx", {"apache"}) + assert "nginx" in result + assert "apache" in result + + def test_hint_names_empty(self): + """Test empty hint names.""" + result = _hint_names("", set()) + assert result == set() + + def test_hint_names_with_service(self): + """Test hint names with .service suffix.""" + result = _hint_names("nginx.service", set()) + assert "nginx" in result + + def test_hint_names_with_template(self): + """Test hint names with template unit.""" + result = _hint_names("nginx@.service", set()) + assert "nginx" in result + + +class TestTopdirsForPackage: + """Tests for _topdirs_for_package function""" + + def test_topdirs_single_level(self): + """Test topdirs with single level paths.""" + pkg_to_etc = {"nginx": ["/etc/nginx/nginx.conf"]} + result = _topdirs_for_package("nginx", pkg_to_etc) + assert result == {"nginx"} + + def test_topdirs_multiple_paths(self): + """Test topdirs with multiple paths.""" + pkg_to_etc = {"nginx": ["/etc/nginx/nginx.conf", "/etc/nginx/sites-enabled"]} + result = _topdirs_for_package("nginx", pkg_to_etc) + assert result == {"nginx"} + + def test_topdirs_empty(self): + """Test topdirs with empty package.""" + result = _topdirs_for_package("nonexistent", {}) + assert result == set() + + +class TestIterMatchingFiles: + """Tests for _iter_matching_files function""" + + def test_iter_matching_files_glob(self, tmp_path): + """Test glob pattern matching.""" + (tmp_path / "a.txt").write_text("a", encoding="utf-8") + (tmp_path / "b.txt").write_text("b", encoding="utf-8") + (tmp_path / "c.py").write_text("c", encoding="utf-8") + + os.chdir(tmp_path) + result = _iter_matching_files("*.txt") + assert len(result) == 2 + assert any("a.txt" in p for p in result) + assert any("b.txt" in p for p in result) + + def test_iter_matching_files_directory_walk(self, tmp_path): + """Test directory walking.""" + subdir = tmp_path / "sub" + subdir.mkdir() + (tmp_path / "a.txt").write_text("a", encoding="utf-8") + (subdir / "b.txt").write_text("b", encoding="utf-8") + + os.chdir(tmp_path) + result = _iter_matching_files(str(tmp_path)) + assert len(result) == 2 + + def test_iter_matching_files_cap(self, tmp_path): + """Test file cap limit.""" + for i in range(100): + (tmp_path / f"file{i}.txt").write_text(str(i), encoding="utf-8") + + os.chdir(tmp_path) + result = _iter_matching_files("*.txt", cap=10) + assert len(result) == 10 + + +class TestParseAptSignedBy: + """Tests for _parse_apt_signed_by function""" + + def test_parse_apt_signed_by_bracket(self, tmp_path): + """Test parsing signed-by from bracket notation.""" + sources_list = tmp_path / "sources.list" + sources_list.write_text( + "deb [signed-by=/usr/share/keyrings/nginx.gpg] http://nginx.net stable main\n", + encoding="utf-8", + ) + result = _parse_apt_signed_by([str(sources_list)]) + assert "/usr/share/keyrings/nginx.gpg" in result + + def test_parse_apt_signed_by_header(self, tmp_path): + """Test parsing signed-by from header.""" + sources_file = tmp_path / "sources.list" + sources_file.write_text( + "Signed-By: /usr/share/keyrings/foo.gpg\n", encoding="utf-8" + ) + result = _parse_apt_signed_by([str(sources_file)]) + assert "/usr/share/keyrings/foo.gpg" in result + + def test_parse_apt_signed_by_multiple(self, tmp_path): + """Test parsing multiple signed-by paths.""" + sources_file = tmp_path / "sources.list" + sources_file.write_text( + "Signed-By: /usr/share/keyrings/a.gpg, /usr/share/keyrings/b.gpg\n", + encoding="utf-8", + ) + result = _parse_apt_signed_by([str(sources_file)]) + assert "/usr/share/keyrings/a.gpg" in result + assert "/usr/share/keyrings/b.gpg" in result + + def test_parse_apt_signed_by_oserror(self, tmp_path): + """Test handling of unreadable files.""" + result = _parse_apt_signed_by(["/nonexistent/file"]) + assert result == set() + + +class TestCaptureLink: + """Tests for _capture_link function""" + + def test_capture_link_basic(self, tmp_path): + """Test basic link capture.""" + target = tmp_path / "target.txt" + target.write_text("content", encoding="utf-8") + link = tmp_path / "link.txt" + link.symlink_to(target) + + policy = MagicMock(spec=IgnorePolicy) + policy.deny_reason_link = None + policy.deny_reason = MagicMock(return_value=None) + policy.deny_reason_link = None # No special link denial + + managed: list[ManagedLink] = [] + excluded: list[ExcludedFile] = [] + path_filter = PathFilter([], []) + + result = _capture_link( + role_name="test_role", + abs_path=str(link), + reason="test", + policy=policy, + path_filter=path_filter, + managed_out=managed, + excluded_out=excluded, + seen_role=set(), + seen_global=set(), + ) + assert result is True + assert len(managed) == 1 + assert managed[0].path == str(link) + + def test_capture_link_deny(self, tmp_path): + """Test link capture with deny policy.""" + target = tmp_path / "target.txt" + target.write_text("content", encoding="utf-8") + link = tmp_path / "link.txt" + link.symlink_to(target) + + policy = MagicMock(spec=IgnorePolicy) + policy.deny_reason_link = None + policy.deny_reason = MagicMock(return_value="policy_deny") + + managed: list[ManagedLink] = [] + excluded: list[ExcludedFile] = [] + path_filter = PathFilter([], []) + + result = _capture_link( + role_name="test_role", + abs_path=str(link), + reason="test", + policy=policy, + path_filter=path_filter, + managed_out=managed, + excluded_out=excluded, + seen_role=set(), + seen_global=set(), + ) + assert result is False + assert len(excluded) == 1 + + def test_capture_link_not_symlink(self, tmp_path): + """Test that regular files are rejected.""" + f = tmp_path / "file.txt" + f.write_text("content", encoding="utf-8") + + policy = MagicMock(spec=IgnorePolicy) + policy.deny_reason_link = None + policy.deny_reason = MagicMock(return_value=None) + + managed: list[ManagedLink] = [] + excluded: list[ExcludedFile] = [] + path_filter = PathFilter([], []) + + result = _capture_link( + role_name="test_role", + abs_path=str(f), + reason="test", + policy=policy, + path_filter=path_filter, + managed_out=managed, + excluded_out=excluded, + seen_role=set(), + seen_global=set(), + ) + assert result is False + assert len(excluded) == 1 + + def test_capture_link_seen_role(self, tmp_path): + """Test link capture with seen_role deduplication.""" + target = tmp_path / "target.txt" + target.write_text("content", encoding="utf-8") + link = tmp_path / "link.txt" + link.symlink_to(target) + + policy = MagicMock(spec=IgnorePolicy) + policy.deny_reason_link = None + policy.deny_reason = MagicMock(return_value=None) + + managed: list[ManagedLink] = [] + excluded: list[ExcludedFile] = [] + path_filter = PathFilter([], []) + seen_role = {str(link)} + + result = _capture_link( + role_name="test_role", + abs_path=str(link), + reason="test", + policy=policy, + path_filter=path_filter, + managed_out=managed, + excluded_out=excluded, + seen_role=seen_role, + seen_global=None, + ) + assert result is False + assert len(managed) == 0 + + def test_capture_link_seen_global(self, tmp_path): + """Test link capture with seen_global deduplication.""" + target = tmp_path / "target.txt" + target.write_text("content", encoding="utf-8") + link = tmp_path / "link.txt" + link.symlink_to(target) + + policy = MagicMock(spec=IgnorePolicy) + policy.deny_reason_link = None + policy.deny_reason = MagicMock(return_value=None) + + managed: list[ManagedLink] = [] + excluded: list[ExcludedFile] = [] + path_filter = PathFilter([], []) + seen_global = {str(link)} + + result = _capture_link( + role_name="test_role", + abs_path=str(link), + reason="test", + policy=policy, + path_filter=path_filter, + managed_out=managed, + excluded_out=excluded, + seen_role=None, + seen_global=seen_global, + ) + assert result is False + assert len(managed) == 0 + + +class TestCaptureFile: + """Tests for _capture_file function""" + + def test_capture_file_basic(self, tmp_path): + """Test basic file capture.""" + bundle = tmp_path / "bundle" + bundle.mkdir() + (bundle / "artifacts").mkdir() + + source = tmp_path / "source.txt" + source.write_text("content", encoding="utf-8") + + policy = MagicMock(spec=IgnorePolicy) + policy.deny_reason_link = None + policy.deny_reason = MagicMock(return_value=None) + + managed: list[ManagedFile] = [] + excluded: list[ExcludedFile] = [] + path_filter = PathFilter([], []) + + result = _capture_file( + bundle_dir=str(bundle), + role_name="test_role", + abs_path=str(source), + reason="test", + policy=policy, + path_filter=path_filter, + managed_out=managed, + excluded_out=excluded, + seen_role=set(), + seen_global=set(), + metadata=None, + ) + assert result is True + assert len(managed) == 1 + + def test_capture_file_seen_role(self, tmp_path): + """Test file capture with seen_role deduplication.""" + bundle = tmp_path / "bundle" + bundle.mkdir() + + source = tmp_path / "source.txt" + source.write_text("content", encoding="utf-8") + + policy = MagicMock(spec=IgnorePolicy) + policy.deny_reason_link = None + policy.deny_reason = MagicMock(return_value=None) + + managed: list[ManagedFile] = [] + excluded: list[ExcludedFile] = [] + path_filter = PathFilter([], []) + seen_role = {str(source)} + + result = _capture_file( + bundle_dir=str(bundle), + role_name="test_role", + abs_path=str(source), + reason="test", + policy=policy, + path_filter=path_filter, + managed_out=managed, + excluded_out=excluded, + seen_role=seen_role, + seen_global=None, + metadata=None, + ) + assert result is False + assert len(managed) == 0 + + def test_capture_file_seen_global(self, tmp_path): + """Test file capture with seen_global deduplication.""" + bundle = tmp_path / "bundle" + bundle.mkdir() + + source = tmp_path / "source.txt" + source.write_text("content", encoding="utf-8") + + policy = MagicMock(spec=IgnorePolicy) + policy.deny_reason_link = None + policy.deny_reason = MagicMock(return_value=None) + + managed: list[ManagedFile] = [] + excluded: list[ExcludedFile] = [] + path_filter = PathFilter([], []) + seen_global = {str(source)} + + result = _capture_file( + bundle_dir=str(bundle), + role_name="test_role", + abs_path=str(source), + reason="test", + policy=policy, + path_filter=path_filter, + managed_out=managed, + excluded_out=excluded, + seen_role=None, + seen_global=seen_global, + metadata=None, + ) + assert result is False + assert len(managed) == 0 + + +def test_user_shell_dotfiles_are_not_auto_captured_without_dangerous(tmp_path: Path): + home = tmp_path / "home" / "alice" + home.mkdir(parents=True) + (home / ".bashrc").write_text("export DEMO=value\n", encoding="utf-8") + (home / ".bash_aliases").write_text("alias ll='ls -la'\n", encoding="utf-8") + + managed: list[ManagedFile] = [] + excluded: list[ExcludedFile] = [] + + captured = capture_user_shell_dotfiles( + bundle_dir=str(tmp_path / "bundle"), + role_name="users", + home=str(home), + skel_dir=str(tmp_path / "skel"), + enabled=False, + policy=IgnorePolicy(dangerous=False), + path_filter=PathFilter(), + managed_out=managed, + excluded_out=excluded, + seen_role=set(), + seen_global=set(), + ) + + assert captured == 0 + assert managed == [] + assert excluded == [] + assert not (tmp_path / "bundle" / "artifacts" / "users").exists() + + +def test_user_shell_dotfiles_dangerous_captures_changed_files_only(tmp_path: Path): + skel = tmp_path / "skel" + home = tmp_path / "home" / "alice" + skel.mkdir(parents=True) + home.mkdir(parents=True) + + (skel / ".bashrc").write_text("# default bashrc\n", encoding="utf-8") + (home / ".bashrc").write_text("# customised bashrc\n", encoding="utf-8") + + (skel / ".profile").write_text("# default profile\n", encoding="utf-8") + (home / ".profile").write_text("# default profile\n", encoding="utf-8") + + (home / ".bash_aliases").write_text("alias ll='ls -la'\n", encoding="utf-8") + + target = home / "target" + target.write_text("# symlink target\n", encoding="utf-8") + os.symlink(target, home / ".bash_logout") + + managed: list[ManagedFile] = [] + excluded: list[ExcludedFile] = [] + + captured = capture_user_shell_dotfiles( + bundle_dir=str(tmp_path / "bundle"), + role_name="users", + home=str(home), + skel_dir=str(skel), + enabled=True, + policy=IgnorePolicy(dangerous=True), + path_filter=PathFilter(), + managed_out=managed, + excluded_out=excluded, + seen_role=set(), + seen_global=set(), + ) + + captured_paths = {mf.path for mf in managed} + assert captured == 2 + assert str(home / ".bashrc") in captured_paths + assert str(home / ".bash_aliases") in captured_paths + assert str(home / ".profile") not in captured_paths + assert str(home / ".bash_logout") not in captured_paths + assert excluded == [] + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_harvest_collectors.py b/tests/test_harvest_collectors.py new file mode 100644 index 0000000..e6e3228 --- /dev/null +++ b/tests/test_harvest_collectors.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +from enroll.harvest_collectors.context import HarvestContext +from enroll.harvest_collectors.runtime import RuntimeStateCollector +from enroll.harvest_types import FirewallRuntimeSnapshot, SysctlSnapshot +from enroll.ignore import IgnorePolicy +from enroll.pathfilter import PathFilter + + +class _Backend: + name = "dpkg" + + +def _context(tmp_path): + return HarvestContext( + bundle_dir=str(tmp_path), + policy=IgnorePolicy(), + path_filter=PathFilter(include=(), exclude=()), + platform={}, + backend=_Backend(), + installed_pkgs={}, + installed_names=set(), + owned_etc=set(), + etc_owner_map={}, + topdir_to_pkgs={}, + pkg_to_etc_paths={}, + captured_global=set(), + ) + + +def test_runtime_state_collector_preserves_non_root_skip_schema(monkeypatch, tmp_path): + monkeypatch.setattr("enroll.harvest.os.geteuid", lambda: 1000) + + result = RuntimeStateCollector(_context(tmp_path)).collect() + + assert isinstance(result.firewall_runtime_snapshot, FirewallRuntimeSnapshot) + assert isinstance(result.sysctl_snapshot, SysctlSnapshot) + assert result.firewall_runtime_snapshot.role_name == "firewall_runtime" + assert result.sysctl_snapshot.role_name == "sysctl" + assert "not running as root" in result.firewall_runtime_snapshot.notes[0] + assert "not running as root" in result.sysctl_snapshot.notes[0] + + +def test_container_images_collector_records_digest_pinned_docker_images( + monkeypatch, tmp_path +): + import json + import subprocess + + from enroll.harvest_collectors import container_images as ci + from enroll.harvest_collectors.container_images import ContainerImagesCollector + + def fake_which(cmd): + return f"/usr/bin/{cmd}" if cmd == "docker" else None + + def fake_run(argv, check=False, stdout=None, stderr=None, text=False, timeout=None): + if argv[:4] == ["/usr/bin/docker", "image", "ls", "-q"]: + return subprocess.CompletedProcess(argv, 0, "sha256:" + "a" * 64 + "\n", "") + if argv[:3] == ["/usr/bin/docker", "image", "inspect"]: + return subprocess.CompletedProcess( + argv, + 0, + json.dumps( + [ + { + "Id": "sha256:" + "a" * 64, + "RepoTags": ["docker.io/library/nginx:1.27"], + "RepoDigests": [ + "docker.io/library/nginx@sha256:" + "b" * 64 + ], + "Os": "linux", + "Architecture": "amd64", + "Size": 123, + "Created": "2026-01-01T00:00:00Z", + } + ] + ), + "", + ) + raise AssertionError(argv) + + monkeypatch.setattr(ci.shutil, "which", fake_which) + monkeypatch.setattr(ci.subprocess, "run", fake_run) + + result = ContainerImagesCollector(_context(tmp_path)).collect() + + assert result.role_name == "container_images" + assert len(result.images) == 1 + image = result.images[0] + assert image["engine"] == "docker" + assert image["pull_ref"] == "docker.io/library/nginx@sha256:" + "b" * 64 + assert image["platform"] == "linux/amd64" + assert image["tag_aliases"] == [ + { + "ref": "docker.io/library/nginx:1.27", + "repository": "docker.io/library/nginx", + "tag": "1.27", + } + ] + + +def test_container_images_collector_records_unpullable_tagged_images( + monkeypatch, tmp_path +): + import json + import subprocess + + from enroll.harvest_collectors import container_images as ci + from enroll.harvest_collectors.container_images import ContainerImagesCollector + + def fake_which(cmd): + return "/usr/bin/podman" if cmd == "podman" else None + + monkeypatch.setattr(ci.shutil, "which", fake_which) + + def fake_run(argv, check=False, stdout=None, stderr=None, text=False, timeout=None): + if argv[:4] == ["/usr/bin/podman", "image", "ls", "-q"]: + return subprocess.CompletedProcess(argv, 0, "c" * 64 + "\n", "") + if argv[:3] == ["/usr/bin/podman", "image", "inspect"]: + return subprocess.CompletedProcess( + argv, + 0, + json.dumps( + [ + { + "Id": "c" * 64, + "RepoTags": ["localhost/demo:latest"], + "RepoDigests": [], + "Os": "linux", + "Architecture": "amd64", + } + ] + ), + "", + ) + raise AssertionError(argv) + + monkeypatch.setattr(ci.subprocess, "run", fake_run) + + result = ContainerImagesCollector(_context(tmp_path)).collect() + + assert result.images[0]["pull_ref"] is None + assert "exact digest-pinned pull cannot be rendered" in result.images[0]["notes"][0] diff --git a/tests/test_harvest_cron_logrotate.py b/tests/test_harvest_cron_logrotate.py new file mode 100644 index 0000000..8e614b3 --- /dev/null +++ b/tests/test_harvest_cron_logrotate.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import enroll.harvest as h +import enroll.capture as capture +import enroll.harvest_collectors.cron_logrotate as cron_logrotate +from enroll.platform import PlatformInfo +from enroll.systemd import UnitInfo + + +class AllowAllPolicy: + def deny_reason(self, path: str): + return None + + +class FakeBackend: + def __init__( + self, + *, + name: str, + installed: dict[str, list[dict[str, str]]], + manual: list[str], + ): + self.name = name + self._installed = dict(installed) + self._manual = list(manual) + + def build_etc_index(self): + # No package ownership information needed for this test. + return set(), {}, {}, {} + + def installed_packages(self): + return dict(self._installed) + + def list_manual_packages(self): + return list(self._manual) + + def owner_of_path(self, path: str): + return None + + def specific_paths_for_hints(self, hints: set[str]): + return [] + + def is_pkg_config_path(self, path: str) -> bool: + return False + + def modified_paths(self, pkg: str, etc_paths: list[str]): + return {} + + +def test_harvest_unifies_cron_and_logrotate_into_dedicated_package_roles( + monkeypatch, tmp_path: Path +): + bundle = tmp_path / "bundle" + + # Fake files we want harvested. + files = { + "/etc/crontab": b"* * * * * root echo hi\n", + "/etc/cron.d/php": b"# php cron\n", + "/var/spool/cron/crontabs/alice": b"@daily echo user\n", + "/etc/logrotate.conf": b"weekly\n", + "/etc/logrotate.d/rsyslog": b"/var/log/syslog { rotate 7 }\n", + } + + monkeypatch.setattr(h.os.path, "islink", lambda p: False) + monkeypatch.setattr(h.os.path, "isfile", lambda p: p in files) + monkeypatch.setattr(h.os.path, "isdir", lambda p: False) + monkeypatch.setattr(h.os.path, "exists", lambda p: (p in files) or False) + + # Expand cron/logrotate globs deterministically. + def fake_iter_matching(spec: str, cap: int = 10000): + mapping = { + "/etc/crontab": ["/etc/crontab"], + "/etc/cron.d/*": ["/etc/cron.d/php"], + "/etc/cron.hourly/*": [], + "/etc/cron.daily/*": [], + "/etc/cron.weekly/*": [], + "/etc/cron.monthly/*": [], + "/etc/cron.allow": [], + "/etc/cron.deny": [], + "/etc/anacrontab": [], + "/etc/anacron/*": [], + "/var/spool/cron/*": [], + "/var/spool/cron/crontabs/*": ["/var/spool/cron/crontabs/alice"], + "/var/spool/crontabs/*": [], + "/var/spool/anacron/*": [], + "/etc/logrotate.conf": ["/etc/logrotate.conf"], + "/etc/logrotate.d/*": ["/etc/logrotate.d/rsyslog"], + } + return list(mapping.get(spec, []))[:cap] + + monkeypatch.setattr(cron_logrotate, "iter_matching_files", fake_iter_matching) + + # Avoid real system probing. + monkeypatch.setattr( + h, "detect_platform", lambda: PlatformInfo("debian", "dpkg", {}) + ) + backend = FakeBackend( + name="dpkg", + installed={ + "cron": [{"version": "1", "arch": "amd64"}], + "logrotate": [{"version": "1", "arch": "amd64"}], + }, + # Include cron/logrotate in manual packages to ensure they are skipped in the generic loop. + manual=["cron", "logrotate"], + ) + monkeypatch.setattr(h, "get_backend", lambda info=None: backend) + + # Include a service that would collide with cron role naming. + monkeypatch.setattr( + h, "list_enabled_services", lambda: ["cron.service", "foo.service"] + ) + monkeypatch.setattr(h, "list_enabled_timers", lambda: []) + monkeypatch.setattr( + h, + "get_unit_info", + lambda unit: UnitInfo( + name=unit, + fragment_path=None, + dropin_paths=[], + env_files=[], + exec_paths=[], + active_state="active", + sub_state="running", + unit_file_state="enabled", + condition_result=None, + ), + ) + monkeypatch.setattr(h, "collect_non_system_users", lambda: []) + monkeypatch.setattr( + capture, + "stat_triplet", + lambda p: ("alice" if "alice" in p else "root", "root", "0644"), + ) + + # Avoid needing real source files by implementing our own bundle copier. + def fake_copy(bundle_dir: str, role_name: str, abs_path: str, src_rel: str): + dst = Path(bundle_dir) / "artifacts" / role_name / src_rel + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_bytes(files.get(abs_path, b"")) + + monkeypatch.setattr(capture, "copy_into_bundle", fake_copy) + + state_path = h.harvest(str(bundle), policy=AllowAllPolicy()) + st = json.loads(Path(state_path).read_text(encoding="utf-8")) + + # cron.service must be skipped to avoid colliding with the dedicated "cron" package role. + svc_units = [s["unit"] for s in st["roles"]["services"]] + assert "cron.service" not in svc_units + assert "foo.service" in svc_units + + pkgs = st["roles"]["packages"] + cron = next(p for p in pkgs if p["role_name"] == "cron") + logrotate = next(p for p in pkgs if p["role_name"] == "logrotate") + + cron_paths = {mf["path"] for mf in cron["managed_files"]} + assert "/etc/crontab" in cron_paths + assert "/etc/cron.d/php" in cron_paths + # user crontab captured + assert "/var/spool/cron/crontabs/alice" in cron_paths + + lr_paths = {mf["path"] for mf in logrotate["managed_files"]} + assert "/etc/logrotate.conf" in lr_paths + assert "/etc/logrotate.d/rsyslog" in lr_paths diff --git a/tests/test_harvest_helpers.py b/tests/test_harvest_helpers.py new file mode 100644 index 0000000..07ae690 --- /dev/null +++ b/tests/test_harvest_helpers.py @@ -0,0 +1,398 @@ +from __future__ import annotations + +import os +from pathlib import Path + +import enroll.harvest as h +import enroll.system_paths as sp +from enroll.package_hints import role_name_from_pkg, role_name_from_unit + + +def test_iter_matching_files_skips_symlinks_and_walks_dirs(monkeypatch, tmp_path: Path): + # Layout: + # root/real.txt (file) + # root/sub/nested.txt + # root/link -> ... (ignored) + root = tmp_path / "root" + (root / "sub").mkdir(parents=True) + (root / "real.txt").write_text("a", encoding="utf-8") + (root / "sub" / "nested.txt").write_text("b", encoding="utf-8") + + paths = { + str(root): "dir", + str(root / "real.txt"): "file", + str(root / "sub"): "dir", + str(root / "sub" / "nested.txt"): "file", + str(root / "link"): "link", + } + + monkeypatch.setattr(sp.glob, "glob", lambda spec: [str(root), str(root / "link")]) + monkeypatch.setattr(sp.os.path, "islink", lambda p: paths.get(p) == "link") + monkeypatch.setattr(sp.os.path, "isfile", lambda p: paths.get(p) == "file") + monkeypatch.setattr(sp.os.path, "isdir", lambda p: paths.get(p) == "dir") + monkeypatch.setattr( + sp.os, + "walk", + lambda p: [ + (str(root), ["sub"], ["real.txt", "link"]), + (str(root / "sub"), [], ["nested.txt"]), + ], + ) + + out = sp.iter_matching_files("/whatever/*", cap=100) + assert str(root / "real.txt") in out + assert str(root / "sub" / "nested.txt") in out + assert str(root / "link") not in out + + +def test_parse_apt_signed_by_extracts_keyrings(tmp_path: Path): + f1 = tmp_path / "a.list" + f1.write_text( + "deb [signed-by=/usr/share/keyrings/foo.gpg] https://example.invalid stable main\n", + encoding="utf-8", + ) + f2 = tmp_path / "b.sources" + f2.write_text( + "Types: deb\nSigned-By: /etc/apt/keyrings/bar.gpg, /usr/share/keyrings/baz.gpg\n", + encoding="utf-8", + ) + f3 = tmp_path / "c.sources" + f3.write_text("Signed-By: | /bin/echo nope\n", encoding="utf-8") + + out = sp.parse_apt_signed_by([str(f1), str(f2), str(f3)]) + assert "/usr/share/keyrings/foo.gpg" in out + assert "/etc/apt/keyrings/bar.gpg" in out + assert "/usr/share/keyrings/baz.gpg" in out + + +def test_iter_apt_capture_paths_includes_signed_by_keyring(monkeypatch): + # Simulate: + # /etc/apt/apt.conf.d/00test + # /etc/apt/sources.list.d/test.list (signed-by outside /etc/apt) + # /usr/share/keyrings/ext.gpg + files = { + "/etc/apt/apt.conf.d/00test": "file", + "/etc/apt/sources.list.d/test.list": "file", + "/usr/share/keyrings/ext.gpg": "file", + } + + monkeypatch.setattr(sp.os.path, "isdir", lambda p: p in {"/etc/apt"}) + monkeypatch.setattr( + sp.os, + "walk", + lambda root: [ + ("/etc/apt", ["apt.conf.d", "sources.list.d"], []), + ("/etc/apt/apt.conf.d", [], ["00test"]), + ("/etc/apt/sources.list.d", [], ["test.list"]), + ], + ) + monkeypatch.setattr(sp.os.path, "islink", lambda p: False) + monkeypatch.setattr(sp.os.path, "isfile", lambda p: files.get(p) == "file") + + # Only treat the sources glob as having a hit. + def fake_iter_matching(spec: str, cap: int = 10000): + if spec == "/etc/apt/sources.list.d/*.list": + return ["/etc/apt/sources.list.d/test.list"] + return [] + + monkeypatch.setattr(sp, "iter_matching_files", fake_iter_matching) + + # Provide file contents for the sources file. + real_open = open + + def fake_open(path, *a, **k): + if path == "/etc/apt/sources.list.d/test.list": + return real_open(os.devnull, "r", encoding="utf-8") # placeholder + return real_open(path, *a, **k) + + # Easier: patch _parse_apt_signed_by directly to avoid filesystem reads. + monkeypatch.setattr( + sp, "parse_apt_signed_by", lambda sfs: {"/usr/share/keyrings/ext.gpg"} + ) + + out = sp.iter_apt_capture_paths() + paths = {p for p, _r in out} + reasons = {p: r for p, r in out} + assert "/etc/apt/apt.conf.d/00test" in paths + assert "/etc/apt/sources.list.d/test.list" in paths + assert "/usr/share/keyrings/ext.gpg" in paths + assert reasons["/usr/share/keyrings/ext.gpg"] == "apt_signed_by_keyring" + + +def test_iter_dnf_capture_paths(monkeypatch): + files = { + "/etc/dnf/dnf.conf": "file", + "/etc/yum/yum.conf": "file", + "/etc/yum.conf": "file", + "/etc/yum.repos.d/test.repo": "file", + "/etc/pki/rpm-gpg/RPM-GPG-KEY": "file", + } + + def isdir(p): + return p in {"/etc/dnf", "/etc/yum", "/etc/yum.repos.d", "/etc/pki/rpm-gpg"} + + def walk(root): + if root == "/etc/dnf": + return [("/etc/dnf", [], ["dnf.conf"])] + if root == "/etc/yum": + return [("/etc/yum", [], ["yum.conf"])] + if root == "/etc/pki/rpm-gpg": + return [("/etc/pki/rpm-gpg", [], ["RPM-GPG-KEY"])] + return [] + + monkeypatch.setattr(sp.os.path, "isdir", isdir) + monkeypatch.setattr(sp.os, "walk", walk) + monkeypatch.setattr(sp.os.path, "islink", lambda p: False) + monkeypatch.setattr(sp.os.path, "isfile", lambda p: files.get(p) == "file") + + def fake_iter_matching(spec: str, cap: int = 10000): + if spec == "/etc/yum.conf": + return ["/etc/yum.conf"] + if spec.endswith("*.repo"): + return ["/etc/yum.repos.d/test.repo"] + if spec == "/etc/pki/rpm-gpg/*": + return ["/etc/pki/rpm-gpg/RPM-GPG-KEY"] + return [] + + monkeypatch.setattr(sp, "iter_matching_files", fake_iter_matching) + + out = sp.iter_dnf_capture_paths() + paths = {p for p, _r in out} + assert "/etc/dnf/dnf.conf" in paths + assert "/etc/yum/yum.conf" in paths + assert "/etc/yum.conf" in paths + assert "/etc/yum.repos.d/test.repo" in paths + assert "/etc/pki/rpm-gpg/RPM-GPG-KEY" in paths + + +def test_iter_system_capture_paths_dedupes_first_reason(monkeypatch): + monkeypatch.setattr(sp, "_SYSTEM_CAPTURE_GLOBS", [("/a", "r1"), ("/b", "r2")]) + monkeypatch.setattr( + sp, + "iter_matching_files", + lambda spec, cap=10000: ["/dup"] if spec in {"/a", "/b"} else [], + ) + out = sp.iter_system_capture_paths() + assert out == [("/dup", "r1")] + + +def test_ipset_and_iptables_state_helpers(tmp_path: Path): + ipset_save = """create blocklist hash:ip family inet hashsize 1024 maxelem 65536 +add blocklist 203.0.113.10 +create nets hash:net family inet +""" + assert h._ipset_save_has_state(ipset_save) + assert h._parse_ipset_set_names(ipset_save) == ["blocklist", "nets"] + assert not h._ipset_save_has_state("# empty\n") + + empty_iptables = """*filter +:INPUT ACCEPT [0:0] +:FORWARD ACCEPT [0:0] +:OUTPUT ACCEPT [0:0] +COMMIT +""" + assert not h._iptables_save_has_state(empty_iptables) + + native_rule = empty_iptables.replace( + "COMMIT", "-A INPUT -p tcp --dport 22 -j ACCEPT\nCOMMIT" + ) + assert h._iptables_save_has_state(native_rule) + + changed_policy = empty_iptables.replace(":INPUT ACCEPT", ":INPUT DROP") + assert h._iptables_save_has_state(changed_policy) + + +def test_collect_firewall_runtime_snapshot_writes_generated_artifacts( + monkeypatch, tmp_path: Path +): + outputs = { + "ipset_save": ( + "create blocklist hash:ip family inet\nadd blocklist 203.0.113.10\n", + None, + ), + "iptables_v4_save": ( + "*filter\n:INPUT DROP [0:0]\n-A INPUT -m set --match-set blocklist src -j DROP\nCOMMIT\n", + None, + ), + "iptables_v6_save": ("*filter\n:INPUT ACCEPT [0:0]\nCOMMIT\n", None), + } + + def fake_run(command_key, *, timeout=10): + return outputs[command_key] + + monkeypatch.setattr(h, "_run_capture_command", fake_run) + + snap = h._collect_firewall_runtime_snapshot(str(tmp_path)) + assert snap.role_name == "firewall_runtime" + assert snap.packages == ["ipset", "iptables"] + assert snap.ipset_save == "firewall/ipset.save" + assert snap.ipset_sets == ["blocklist"] + assert snap.iptables_v4_save == "firewall/iptables.v4" + assert snap.iptables_v6_save is None + + assert ( + (tmp_path / "artifacts" / "firewall_runtime" / "firewall" / "ipset.save") + .read_text(encoding="utf-8") + .startswith("create blocklist") + ) + assert ( + (tmp_path / "artifacts" / "firewall_runtime" / "firewall" / "iptables.v4") + .read_text(encoding="utf-8") + .startswith("*filter") + ) + + +def test_collect_firewall_runtime_snapshot_is_per_family_fallback( + monkeypatch, tmp_path: Path +): + calls = [] + outputs = { + "ipset_save": ( + "create blocklist hash:ip family inet\nadd blocklist 203.0.113.10\n", + None, + ), + "iptables_v4_save": ( + "*filter\n:INPUT DROP [0:0]\n-A INPUT -p tcp --dport 22 -j ACCEPT\nCOMMIT\n", + None, + ), + "iptables_v6_save": ( + "*filter\n:INPUT DROP [0:0]\n-A INPUT -p tcp --dport 22 -j ACCEPT\nCOMMIT\n", + None, + ), + } + + def fake_run(command_key, *, timeout=10): + calls.append(command_key) + return outputs[command_key] + + monkeypatch.setattr(h, "_run_capture_command", fake_run) + + snap = h._collect_firewall_runtime_snapshot( + str(tmp_path), + persistent_ipset_files=["/etc/ipset.conf"], + persistent_iptables_v4_files=["/etc/iptables/rules.v4"], + persistent_iptables_v6_files=[], + ) + + assert "ipset_save" not in calls + assert "iptables_v4_save" not in calls + assert "iptables_v6_save" in calls + assert snap.ipset_save is None + assert snap.iptables_v4_save is None + assert snap.iptables_v6_save == "firewall/iptables.v6" + assert snap.packages == ["iptables"] + assert any("persistent ipset configuration" in note for note in snap.notes) + assert any("persistent IPv4 iptables configuration" in note for note in snap.notes) + assert not ( + tmp_path / "artifacts" / "firewall_runtime" / "firewall" / "ipset.save" + ).exists() + assert not ( + tmp_path / "artifacts" / "firewall_runtime" / "firewall" / "iptables.v4" + ).exists() + assert ( + tmp_path / "artifacts" / "firewall_runtime" / "firewall" / "iptables.v6" + ).exists() + + +def test_package_role_names_do_not_collide_with_singleton_roles(): + assert role_name_from_pkg("flatpak") == "package_flatpak" + assert role_name_from_pkg("snap") == "package_snap" + assert role_name_from_pkg("users") == "package_users" + assert role_name_from_pkg("nginx") == "nginx" + + +def test_service_role_names_do_not_collide_with_singleton_roles(): + assert role_name_from_unit("flatpak.service") == "service_flatpak" + assert role_name_from_unit("users.service") == "service_users" + assert role_name_from_unit("nginx.service") == "nginx" + + +def test_parse_sysctl_a_output_keeps_persistable_values(monkeypatch): + monkeypatch.setattr( + h, + "_sysctl_key_is_persistable", + lambda key: (key != "kernel.hostname", "test"), + ) + + params, skipped = h._parse_sysctl_a_output( + "net.ipv4.ip_forward = 1\n" + "kernel.hostname = example\n" + "malformed line\n" + "dev.cdrom.info = \n" + "net.ipv4.ip_forward = 0\n" + ) + + assert params == {"net.ipv4.ip_forward": "1"} + assert skipped["non_persistable"] == 1 + assert skipped["malformed"] == 1 + assert skipped["empty_value"] == 1 + assert skipped["duplicate"] == 1 + + +def test_sysctl_filter_skips_non_replayable_runtime_keys(monkeypatch): + for key in ( + "fs.binfmt_misc.status", + "fs.binfmt_misc.register", + "kernel.kexec_load_disabled", + "kernel.kexec_load_limit_panic", + "kernel.kexec_load_limit_reboot", + "kernel.max_rcu_stall_to_panic", + "kernel.modules_disabled", + "kernel.sched_domain.cpu0.domain0.flags", + ): + ok, reason = h._sysctl_key_is_persistable(key) + assert ok is False + assert reason == "volatile/action key" + + monkeypatch.setattr(h, "_sysctl_key_is_persistable", lambda key: (True, "")) + for key in ( + "vm.dirty_background_bytes", + "vm.dirty_background_ratio", + "vm.dirty_bytes", + "vm.dirty_ratio", + ): + ok, reason = h._sysctl_entry_is_persistable(key, "0") + assert ok is False + assert reason == "inactive mutually-exclusive zero value" + assert h._sysctl_entry_is_persistable(key, "10")[0] is True + + +def test_parse_sysctl_a_output_skips_non_replayable_values(monkeypatch): + monkeypatch.setattr( + h, + "_sysctl_key_is_persistable", + lambda key: (key != "kernel.modules_disabled", "volatile/action key"), + ) + + params, skipped = h._parse_sysctl_a_output( + "kernel.modules_disabled = 0\n" + "vm.dirty_background_bytes = 0\n" + "vm.dirty_ratio = 20\n" + "net.ipv4.ip_forward = 1\n" + ) + + assert params == {"net.ipv4.ip_forward": "1", "vm.dirty_ratio": "20"} + assert skipped["non_persistable"] == 2 + + +def test_collect_sysctl_snapshot_writes_generated_artifact(monkeypatch, tmp_path: Path): + monkeypatch.setattr( + h, + "_run_capture_command", + lambda command_key, *, timeout=10: ( + "net.ipv4.ip_forward = 1\nvm.swappiness = 10\n", + None, + ), + ) + monkeypatch.setattr(h, "_sysctl_key_is_persistable", lambda key: (True, "")) + + snap = h._collect_sysctl_snapshot(str(tmp_path)) + + assert snap.role_name == "sysctl" + assert snap.parameters == {"net.ipv4.ip_forward": "1", "vm.swappiness": "10"} + assert len(snap.managed_files) == 1 + assert snap.managed_files[0].path == "/etc/sysctl.d/99-enroll.conf" + conf = tmp_path / "artifacts" / "sysctl" / "sysctl" / "99-enroll.conf" + text = conf.read_text(encoding="utf-8") + assert "net.ipv4.ip_forward = 1" in text + assert "vm.swappiness = 10" in text diff --git a/tests/test_harvest_symlinks.py b/tests/test_harvest_symlinks.py new file mode 100644 index 0000000..c177cda --- /dev/null +++ b/tests/test_harvest_symlinks.py @@ -0,0 +1,265 @@ +import json +from pathlib import Path + +import enroll.harvest as h +import enroll.harvest_collectors.services as services +import enroll.capture as capture +from enroll.platform import PlatformInfo +from enroll.systemd import UnitInfo + + +class AllowAllPolicy: + def deny_reason(self, path: str): + return None + + def deny_reason_link(self, path: str): + return None + + +class FakeBackend: + """Minimal backend stub for harvest tests. + + Keep harvest deterministic and avoid enumerating the real system. + """ + + name = "dpkg" + + def build_etc_index(self): + return (set(), {}, {}, {}) + + def owner_of_path(self, path: str): + return None + + def list_manual_packages(self): + return [] + + def installed_packages(self): + return {} + + def specific_paths_for_hints(self, hints: set[str]): + return [] + + def is_pkg_config_path(self, path: str) -> bool: + return False + + def modified_paths(self, pkg: str, etc_paths: list[str]): + return {} + + +def _base_monkeypatches(monkeypatch, *, unit: str): + """Patch harvest to avoid live system access.""" + + monkeypatch.setattr( + h, "detect_platform", lambda: PlatformInfo("debian", "dpkg", {}) + ) + monkeypatch.setattr(h, "get_backend", lambda info=None: FakeBackend()) + + monkeypatch.setattr(h, "list_enabled_timers", lambda: []) + monkeypatch.setattr( + h, + "get_unit_info", + lambda u: UnitInfo( + name=u, + fragment_path=None, + dropin_paths=[], + env_files=[], + exec_paths=[], + active_state="inactive", + sub_state="dead", + unit_file_state="enabled", + condition_result=None, + ), + ) + + # Keep users empty and avoid touching /etc/skel. + monkeypatch.setattr(h, "collect_non_system_users", lambda: []) + + # Avoid warning spam from non-root test runs. + if hasattr(h.os, "geteuid"): + monkeypatch.setattr(h.os, "geteuid", lambda: 0) + + # Avoid walking the real filesystem. + monkeypatch.setattr(h.os, "walk", lambda root: iter(())) + monkeypatch.setattr(capture, "copy_into_bundle", lambda *a, **k: None) + + # Default to a "no files exist" view of the world unless a test overrides. + monkeypatch.setattr(h.os.path, "isfile", lambda p: False) + monkeypatch.setattr(h.os.path, "exists", lambda p: False) + + # Minimal enabled services list. + monkeypatch.setattr(h, "list_enabled_services", lambda: [unit] if unit else []) + + +def test_harvest_captures_nginx_enabled_symlinks(monkeypatch, tmp_path: Path): + bundle = tmp_path / "bundle" + unit = "nginx.service" + _base_monkeypatches(monkeypatch, unit=unit) + + # Fake filesystem for nginx enabled dirs. + dirs = { + "/etc", + "/etc/nginx", + "/etc/nginx/sites-enabled", + "/etc/nginx/modules-enabled", + } + links = { + "/etc/nginx/sites-enabled/default": "../sites-available/default", + "/etc/nginx/modules-enabled/mod-http": "../modules-available/mod-http", + } + + monkeypatch.setattr(h.os.path, "isdir", lambda p: p in dirs) + monkeypatch.setattr(h.os.path, "islink", lambda p: p in links) + monkeypatch.setattr(h.os, "readlink", lambda p: links[p]) + + def fake_glob(pat: str): + if pat == "/etc/nginx/sites-enabled/*": + return [ + "/etc/nginx/sites-enabled/default", + "/etc/nginx/sites-enabled/README", + ] + if pat == "/etc/nginx/modules-enabled/*": + return ["/etc/nginx/modules-enabled/mod-http"] + return [] + + monkeypatch.setattr(services.glob, "glob", fake_glob) + + state_path = h.harvest(str(bundle), policy=AllowAllPolicy()) + st = json.loads(Path(state_path).read_text(encoding="utf-8")) + + svc = next(s for s in st["roles"]["services"] if s["role_name"] == "nginx") + managed_links = svc.get("managed_links") or [] + assert {(ml["path"], ml["target"], ml["reason"]) for ml in managed_links} == { + ( + "/etc/nginx/sites-enabled/default", + "../sites-available/default", + "enabled_symlink", + ), + ( + "/etc/nginx/modules-enabled/mod-http", + "../modules-available/mod-http", + "enabled_symlink", + ), + } + + +def test_harvest_does_not_capture_enabled_symlinks_without_role( + monkeypatch, tmp_path: Path +): + bundle = tmp_path / "bundle" + _base_monkeypatches(monkeypatch, unit="") + + # Dirs exist but nginx isn't detected, so nothing should be captured. + monkeypatch.setattr( + h.os.path, + "isdir", + lambda p: p + in { + "/etc", + "/etc/nginx/sites-enabled", + "/etc/nginx/modules-enabled", + }, + ) + monkeypatch.setattr( + services.glob, "glob", lambda pat: ["/etc/nginx/sites-enabled/default"] + ) + monkeypatch.setattr(h.os.path, "islink", lambda p: True) + monkeypatch.setattr(h.os, "readlink", lambda p: "../sites-available/default") + + state_path = h.harvest(str(bundle), policy=AllowAllPolicy()) + st = json.loads(Path(state_path).read_text(encoding="utf-8")) + + # No services => no place to attach nginx links. + assert st["roles"]["services"] == [] + # And no package snapshots either. + assert st["roles"]["packages"] == [] + + +def test_harvest_symlink_capture_respects_ignore_policy(monkeypatch, tmp_path: Path): + bundle = tmp_path / "bundle" + _base_monkeypatches(monkeypatch, unit="nginx.service") + + dirs = {"/etc", "/etc/nginx/sites-enabled", "/etc/nginx/modules-enabled"} + links = { + "/etc/nginx/sites-enabled/default": "../sites-available/default", + "/etc/nginx/sites-enabled/ok": "../sites-available/ok", + } + + monkeypatch.setattr(h.os.path, "isdir", lambda p: p in dirs) + monkeypatch.setattr(h.os.path, "islink", lambda p: p in links) + monkeypatch.setattr(h.os, "readlink", lambda p: links[p]) + monkeypatch.setattr( + services.glob, + "glob", + lambda pat: ( + sorted(list(links.keys())) if pat == "/etc/nginx/sites-enabled/*" else [] + ), + ) + + calls: list[str] = [] + + class Policy: + def deny_reason(self, path: str): + return None + + def deny_reason_link(self, path: str): + calls.append(path) + if path.endswith("/default"): + return "denied_path" + return None + + state_path = h.harvest(str(bundle), policy=Policy()) + st = json.loads(Path(state_path).read_text(encoding="utf-8")) + + svc = next(s for s in st["roles"]["services"] if s["role_name"] == "nginx") + managed_links = svc.get("managed_links") or [] + excluded = svc.get("excluded") or [] + + assert any(p.endswith("/default") for p in calls) + assert any(p.endswith("/ok") for p in calls) + assert {ml["path"] for ml in managed_links} == {"/etc/nginx/sites-enabled/ok"} + assert {ex["path"] for ex in excluded} == {"/etc/nginx/sites-enabled/default"} + assert ( + next(ex["reason"] for ex in excluded if ex["path"].endswith("/default")) + == "denied_path" + ) + + +def test_harvest_captures_apache2_enabled_symlinks(monkeypatch, tmp_path: Path): + bundle = tmp_path / "bundle" + _base_monkeypatches(monkeypatch, unit="apache2.service") + + dirs = { + "/etc", + "/etc/apache2/conf-enabled", + "/etc/apache2/mods-enabled", + "/etc/apache2/sites-enabled", + } + links = { + "/etc/apache2/sites-enabled/000-default.conf": "../sites-available/000-default.conf", + "/etc/apache2/mods-enabled/rewrite.load": "../mods-available/rewrite.load", + "/etc/apache2/conf-enabled/security.conf": "../conf-available/security.conf", + } + + monkeypatch.setattr(h.os.path, "isdir", lambda p: p in dirs) + monkeypatch.setattr(h.os.path, "islink", lambda p: p in links) + monkeypatch.setattr(h.os, "readlink", lambda p: links[p]) + + def fake_glob(pat: str): + if pat == "/etc/apache2/sites-enabled/*": + return ["/etc/apache2/sites-enabled/000-default.conf"] + if pat == "/etc/apache2/mods-enabled/*": + return ["/etc/apache2/mods-enabled/rewrite.load"] + if pat == "/etc/apache2/conf-enabled/*": + return ["/etc/apache2/conf-enabled/security.conf"] + return [] + + monkeypatch.setattr(services.glob, "glob", fake_glob) + + state_path = h.harvest(str(bundle), policy=AllowAllPolicy()) + st = json.loads(Path(state_path).read_text(encoding="utf-8")) + + svc = next(s for s in st["roles"]["services"] if s["role_name"] == "apache2") + managed_links = svc.get("managed_links") or [] + assert {ml["path"] for ml in managed_links} == set(links.keys()) + assert {ml["target"] for ml in managed_links} == set(links.values()) + assert all(ml["reason"] == "enabled_symlink" for ml in managed_links) diff --git a/tests/test_ignore.py b/tests/test_ignore.py index bba9f06..2ba9a90 100644 --- a/tests/test_ignore.py +++ b/tests/test_ignore.py @@ -1,9 +1,250 @@ +from __future__ import annotations + +import os +from pathlib import Path + from enroll.ignore import IgnorePolicy def test_ignore_policy_denies_common_backup_files(): pol = IgnorePolicy() - assert pol.deny_reason("/etc/shadow-") == "denied_path" - assert pol.deny_reason("/etc/passwd-") == "denied_path" - assert pol.deny_reason("/etc/group-") == "denied_path" + assert pol.deny_reason("/etc/shadow-") == "backup_file" + assert pol.deny_reason("/etc/passwd-") == "backup_file" + assert pol.deny_reason("/etc/group-") == "backup_file" + assert pol.deny_reason("/etc/something~") == "backup_file" assert pol.deny_reason("/foobar") == "unreadable" + + +def test_deny_reason_dir_with_denied_path(): + pol = IgnorePolicy() + assert pol.deny_reason_dir("/etc/ssl/private/key") == "denied_path" + assert pol.deny_reason_dir("/etc/ssh/ssh_host_key") == "denied_path" + assert pol.deny_reason_dir("/etc/ssh") is None + + +def test_deny_reason_dir_unreadable(tmp_path: Path): + pol = IgnorePolicy() + nonexistent = tmp_path / "nonexistent" + assert pol.deny_reason_dir(str(nonexistent)) == "unreadable" + + +def test_deny_reason_dir_symlink(tmp_path: Path): + pol = IgnorePolicy() + real_dir = tmp_path / "real" + real_dir.mkdir() + link = tmp_path / "link" + os.symlink(str(real_dir), str(link)) + assert pol.deny_reason_dir(str(link)) == "symlink" + + +def test_deny_reason_dir_not_directory(tmp_path: Path): + pol = IgnorePolicy() + regular_file = tmp_path / "file.txt" + regular_file.write_text("content", encoding="utf-8") + assert pol.deny_reason_dir(str(regular_file)) == "not_directory" + + +def test_deny_reason_dir_dangerous_mode(tmp_path: Path): + pol = IgnorePolicy(dangerous=True) + real_dir = tmp_path / "private" + real_dir.mkdir() + assert pol.deny_reason_dir(str(real_dir)) is None + + +def test_deny_reason_link_basic(tmp_path: Path): + pol = IgnorePolicy() + real_file = tmp_path / "real" + real_file.write_text("content", encoding="utf-8") + link = tmp_path / "link" + os.symlink(str(real_file), str(link)) + assert pol.deny_reason_link(str(link)) is None + + +def test_deny_reason_link_denied_path(): + pol = IgnorePolicy() + assert pol.deny_reason_link("/etc/ssh/ssh_host_rsa_key") == "denied_path" + + +def test_deny_reason_link_unreadable(tmp_path: Path): + pol = IgnorePolicy() + # Create a symlink in a directory that doesn't exist + # This simulates an unreadable path + broken_link = tmp_path / "broken_link" + os.symlink("/nonexistent/target", str(broken_link)) + # Broken symlinks are still readable (we can readlink them) + # So they return None (allowed) unless they match deny globs + result = pol.deny_reason_link(str(broken_link)) + # Broken symlinks are allowed - we can still read the link target + assert result is None + + +def test_deny_reason_link_not_symlink(tmp_path: Path): + pol = IgnorePolicy() + regular_file = tmp_path / "file.txt" + regular_file.write_text("content", encoding="utf-8") + assert pol.deny_reason_link(str(regular_file)) == "not_symlink" + + +def test_deny_reason_link_log_file(): + pol = IgnorePolicy() + assert pol.deny_reason_link("/var/log/something.log") == "log_file" + + +def test_deny_reason_link_backup_file(): + pol = IgnorePolicy() + assert pol.deny_reason_link("/etc/passwd-") == "backup_file" + assert pol.deny_reason_link("/etc/something~") == "backup_file" + + +def test_deny_reason_link_dangerous_mode(tmp_path: Path): + pol = IgnorePolicy(dangerous=True) + real_file = tmp_path / "real" + real_file.write_text("content", encoding="utf-8") + link = tmp_path / "link" + os.symlink(str(real_file), str(link)) + assert pol.deny_reason_link(str(link)) is None + + +def test_iter_effective_lines_with_comments(): + pol = IgnorePolicy() + content = b""" +# This is a comment +; This is also a comment +* continuation +def main(): + pass +""" + lines = list(pol.iter_effective_lines(content)) + assert b"def main():" in lines + assert b"# This is a comment" not in lines + + +def test_iter_effective_lines_with_block_comments(): + pol = IgnorePolicy() + content = b""" +/* This is a block comment + spanning multiple lines */ +int x = 5; +""" + lines = list(pol.iter_effective_lines(content)) + assert b"int x = 5;" in lines + assert b"/*" not in lines + + +def test_iter_effective_lines_empty(): + pol = IgnorePolicy() + content = b"" + lines = list(pol.iter_effective_lines(content)) + assert lines == [] + + +def test_deny_reason_binary_not_allowed(tmp_path: Path): + pol = IgnorePolicy() + binary = tmp_path / "random.bin" + binary.write_bytes(b"\x00\x01\x02\x03") + reason = pol.deny_reason(str(binary)) + assert reason == "binary_like" + + +def test_deny_reason_sensitive_content(tmp_path: Path): + pol = IgnorePolicy() + config = tmp_path / "config.txt" + config.write_text("password=secret123", encoding="utf-8") + reason = pol.deny_reason(str(config)) + assert reason == "sensitive_content" + + +def test_deny_reason_sensitive_api_key(tmp_path: Path): + pol = IgnorePolicy() + config = tmp_path / "config.txt" + config.write_text("api_key=abc123", encoding="utf-8") + reason = pol.deny_reason(str(config)) + assert reason == "sensitive_content" + + +def test_deny_reason_private_key(tmp_path: Path): + pol = IgnorePolicy() + key = tmp_path / "key.pem" + key.write_text( + "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEA...", encoding="utf-8" + ) + reason = pol.deny_reason(str(key)) + assert reason == "sensitive_content" + + +def test_deny_reason_too_large(tmp_path: Path): + pol = IgnorePolicy(max_file_bytes=100) + large = tmp_path / "large.txt" + large.write_bytes(b"x" * 200) + reason = pol.deny_reason(str(large)) + assert reason == "too_large" + + +def test_deny_reason_unreadable(tmp_path: Path): + pol = IgnorePolicy() + nonexistent = tmp_path / "nonexistent" + reason = pol.deny_reason(str(nonexistent)) + assert reason == "unreadable" + + +def test_deny_reason_not_regular_file(tmp_path: Path): + pol = IgnorePolicy() + directory = tmp_path / "dir" + directory.mkdir() + reason = pol.deny_reason(str(directory)) + assert reason == "not_regular_file" + + +def test_deny_reason_symlink_file(tmp_path: Path): + pol = IgnorePolicy() + real_file = tmp_path / "real" + real_file.write_text("content", encoding="utf-8") + link = tmp_path / "link" + os.symlink(str(real_file), str(link)) + reason = pol.deny_reason(str(link)) + assert reason == "not_regular_file" + + +def test_deny_reason_logs(tmp_path: Path): + pol = IgnorePolicy() + log = tmp_path / "test.log" + log.write_text("log content", encoding="utf-8") + assert pol.deny_reason(str(log)) == "log_file" + + +def test_deny_reason_backup_file(tmp_path: Path): + pol = IgnorePolicy() + backup = tmp_path / "file~" + backup.write_text("backup", encoding="utf-8") + assert pol.deny_reason(str(backup)) == "backup_file" + + +def test_deny_reason_shadow_file(): + pol = IgnorePolicy() + assert pol.deny_reason("/etc/shadow") == "denied_path" + assert pol.deny_reason("/etc/gshadow") == "denied_path" + + +def test_deny_reason_ssl_private(): + pol = IgnorePolicy() + assert pol.deny_reason("/etc/ssl/private/key.pem") == "denied_path" + + +def test_deny_reason_ssh_host_keys(): + pol = IgnorePolicy() + assert pol.deny_reason("/etc/ssh/ssh_host_rsa_key") == "denied_path" + assert pol.deny_reason("/etc/ssh/ssh_host_ed25519_key") == "denied_path" + + +def test_deny_reason_letsencrypt(): + pol = IgnorePolicy() + assert ( + pol.deny_reason("/etc/letsencrypt/live/example.com/fullchain.pem") + == "denied_path" + ) + + +def test_deny_reason_shadow_backup(): + pol = IgnorePolicy() + assert pol.deny_reason("/etc/shadow-") == "backup_file" + assert pol.deny_reason("/etc/passwd-") == "backup_file" diff --git a/tests/test_ignore_dir.py b/tests/test_ignore_dir.py new file mode 100644 index 0000000..3066c92 --- /dev/null +++ b/tests/test_ignore_dir.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from pathlib import Path + + +def test_iter_effective_lines_skips_comments_and_block_comments(): + from enroll.ignore import IgnorePolicy + + policy = IgnorePolicy(deny_globs=[]) + + content = b""" +# comment +; semi +// slash +* c-star + +valid=1 +/* block +ignored=1 +*/ +valid=2 +""" + + lines = [l.strip() for l in policy.iter_effective_lines(content)] + assert lines == [b"valid=1", b"valid=2"] + + +def test_deny_reason_dir_behaviour(tmp_path: Path): + from enroll.ignore import IgnorePolicy + + # Use an absolute pattern matching our temporary path. + deny_glob = str(tmp_path / "deny") + "/*" + pol = IgnorePolicy(deny_globs=[deny_glob], dangerous=False) + + d = tmp_path / "dir" + d.mkdir() + f = tmp_path / "file" + f.write_text("x", encoding="utf-8") + link = tmp_path / "link" + link.symlink_to(d) + + assert pol.deny_reason_dir(str(d)) is None + assert pol.deny_reason_dir(str(link)) == "symlink" + assert pol.deny_reason_dir(str(f)) == "not_directory" + + # Denied by glob. + deny_path = tmp_path / "deny" / "x" + deny_path.mkdir(parents=True) + assert pol.deny_reason_dir(str(deny_path)) == "denied_path" + + # Missing/unreadable. + assert pol.deny_reason_dir(str(tmp_path / "missing")) == "unreadable" + + # Dangerous disables deny_globs. + pol2 = IgnorePolicy(deny_globs=[deny_glob], dangerous=True) + assert pol2.deny_reason_dir(str(deny_path)) is None diff --git a/tests/test_jinjaturtle.py b/tests/test_jinjaturtle.py new file mode 100644 index 0000000..7acb709 --- /dev/null +++ b/tests/test_jinjaturtle.py @@ -0,0 +1,150 @@ +import json +from pathlib import Path + +import enroll.manifest as manifest_mod +from enroll.ansible_renderer import context as ansible_context +from enroll.ansible_renderer import jinjaturtle as ansible_jt +from enroll.jinjaturtle import JinjifyResult + + +def test_manifest_uses_jinjaturtle_templates_and_does_not_copy_raw( + monkeypatch, tmp_path: Path +): + """If jinjaturtle can templatisize a file, we should store a template in the role + and avoid keeping the raw file copy in the destination files area. + + This test stubs out jinjaturtle execution so it doesn't depend on the external tool. + """ + + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + + # A jinjaturtle-compatible config file. + (bundle / "artifacts" / "foo" / "etc").mkdir(parents=True, exist_ok=True) + (bundle / "artifacts" / "foo" / "etc" / "foo.ini").write_text( + "[main]\nkey = 1\n", encoding="utf-8" + ) + + state = { + "schema_version": 3, + "host": {"hostname": "test", "os": "debian", "pkg_backend": "dpkg"}, + "inventory": { + "packages": { + "foo": { + "version": "1.0", + "arches": [], + "installations": [ + {"version": "1.0", "arch": "amd64", "section": "utils"} + ], + "section": "utils", + "observed_via": [{"kind": "systemd_unit", "ref": "foo.service"}], + "roles": ["foo"], + } + } + }, + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [ + { + "unit": "foo.service", + "role_name": "foo", + "packages": ["foo"], + "active_state": "inactive", + "sub_state": "dead", + "unit_file_state": "disabled", + "condition_result": "no", + "managed_files": [ + { + "path": "/etc/foo.ini", + "src_rel": "etc/foo.ini", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "modified_conffile", + } + ], + "excluded": [], + "notes": [], + } + ], + "packages": [], + "apt_config": { + "role_name": "apt_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + }, + } + + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + # Pretend jinjaturtle exists. + monkeypatch.setattr( + ansible_context, "find_jinjaturtle_cmd", lambda: "/usr/bin/jinjaturtle" + ) + + # Stub jinjaturtle output. + def fake_run_jinjaturtle( + jt_exe: str, src_path: str, *, role_name: str, force_format=None + ): + assert role_name == "foo" + return JinjifyResult( + template_text="[main]\nkey = {{ foo_key }}\n", + vars_text="foo_key: 1\n", + ) + + monkeypatch.setattr(ansible_jt, "run_jinjaturtle", fake_run_jinjaturtle) + + manifest_mod.manifest(str(bundle), str(out), jinjaturtle="on") + + role_dir = out / "roles" / "utils" + + # Template should exist in the grouped section role. + assert (role_dir / "templates" / "etc" / "foo.ini.j2").exists() + + # Raw file should NOT be copied into role files/ because it was templatised. + assert not (role_dir / "files" / "etc" / "foo.ini").exists() + + # Defaults should include jinjaturtle vars. + defaults = (role_dir / "defaults" / "main.yml").read_text(encoding="utf-8") + assert "foo_key: 1" in defaults + + +def test_openssh_paths_are_jinjaturtle_supported_and_forced_to_ssh() -> None: + from enroll.jinjaturtle import can_jinjify_path, infer_other_formats + + assert infer_other_formats("/etc/ssh/sshd_config") == "ssh" + assert infer_other_formats("/etc/ssh/ssh_config") == "ssh" + assert infer_other_formats("/etc/ssh/sshd_config.d/50-hardening.conf") == "ssh" + assert infer_other_formats("/etc/ssh/ssh_config.d/99-proxy.conf") == "ssh" + + assert can_jinjify_path("/etc/ssh/sshd_config") + assert can_jinjify_path("/etc/ssh/ssh_config") diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 09c66e1..fc9ea16 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -1,7 +1,91 @@ import json from pathlib import Path -from enroll.manifest import manifest +import os +import stat +import tarfile +import pytest + +import enroll.manifest as manifest +from enroll.ansible_renderer import context as ansible_context +from enroll.ansible_renderer import jinjaturtle as ansible_jt +from enroll.ansible_renderer import layout as ansible_layout +from enroll.ansible_renderer import tasks as ansible_tasks +from enroll.ansible_renderer import yamlutil as ansible_yaml + + +def _minimal_package_state(packages): + return { + "schema_version": 3, + "host": {"hostname": "test", "os": "debian", "pkg_backend": "dpkg"}, + "inventory": { + "packages": { + p["package"]: { + "version": "1.0", + "arches": ["amd64"], + "installations": [ + { + "version": "1.0", + "arch": "amd64", + "section": p.get("section") or "misc", + } + ], + "section": p.get("section") or "misc", + "observed_via": [{"kind": "package_role", "ref": p["role_name"]}], + "roles": [p["role_name"]], + } + for p in packages + } + }, + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [], + "packages": packages, + "apt_config": { + "role_name": "apt_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + }, + } + + +def _write_state(bundle: Path, state: dict) -> None: + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") def test_manifest_writes_roles_and_playbook_with_clean_when(tmp_path: Path): @@ -13,72 +97,136 @@ def test_manifest_writes_roles_and_playbook_with_clean_when(tmp_path: Path): ) state = { - "host": {"hostname": "test", "os": "debian"}, - "users": { - "role_name": "users", - "users": [ + "schema_version": 3, + "host": {"hostname": "test", "os": "debian", "pkg_backend": "dpkg"}, + "inventory": { + "packages": { + "foo": { + "version": "1.0", + "arches": [], + "installations": [{"version": "1.0", "arch": "amd64"}], + "observed_via": [{"kind": "systemd_unit", "ref": "foo.service"}], + "roles": ["foo"], + }, + "curl": { + "version": "8.0", + "arches": [], + "installations": [{"version": "8.0", "arch": "amd64"}], + "observed_via": [{"kind": "package_role", "ref": "curl"}], + "roles": ["curl"], + }, + } + }, + "roles": { + "users": { + "role_name": "users", + "users": [ + { + "name": "alice", + "uid": 1000, + "gid": 1000, + "gecos": "Alice", + "home": "/home/alice", + "shell": "/bin/bash", + "primary_group": "alice", + "supplementary_groups": ["docker", "qubes"], + } + ], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [ { - "name": "alice", - "uid": 1000, - "gid": 1000, - "gecos": "Alice", - "home": "/home/alice", - "shell": "/bin/bash", - "primary_group": "alice", - "supplementary_groups": ["docker", "qubes"], + "unit": "foo.service", + "role_name": "foo", + "packages": ["foo"], + "active_state": "inactive", + "sub_state": "dead", + "unit_file_state": "enabled", + "condition_result": "no", + "managed_files": [ + { + "path": "/etc/foo.conf", + "src_rel": "etc/foo.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "modified_conffile", + } + ], + "excluded": [], + "notes": [], } ], - "managed_files": [], - "excluded": [], - "notes": [], - }, - "etc_custom": { - "role_name": "etc_custom", - "managed_files": [ + "packages": [ { - "path": "/etc/default/keyboard", - "src_rel": "etc/default/keyboard", - "owner": "root", - "group": "root", - "mode": "0644", - "reason": "custom_unowned", + "package": "curl", + "role_name": "curl", + "managed_files": [], + "excluded": [], + "notes": [], } ], - "excluded": [], - "notes": [], - }, - "services": [ - { - "unit": "foo.service", - "role_name": "foo", - "packages": ["foo"], - "active_state": "inactive", - "sub_state": "dead", - "unit_file_state": "enabled", - "condition_result": "no", + "apt_config": { + "role_name": "apt_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", "managed_files": [ { - "path": "/etc/foo.conf", - "src_rel": "etc/foo.conf", + "path": "/etc/default/keyboard", + "src_rel": "etc/default/keyboard", "owner": "root", "group": "root", "mode": "0644", - "reason": "modified_conffile", + "reason": "custom_unowned", } ], "excluded": [], "notes": [], - } - ], - "package_roles": [ - { - "package": "curl", - "role_name": "curl", + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [ + { + "path": "/usr/local/etc/myapp.conf", + "src_rel": "usr/local/etc/myapp.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "usr_local_etc_custom", + }, + { + "path": "/usr/local/bin/myscript", + "src_rel": "usr/local/bin/myscript", + "owner": "root", + "group": "root", + "mode": "0755", + "reason": "usr_local_bin_script", + }, + ], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], "managed_files": [], "excluded": [], "notes": [], - } - ], + }, + }, } bundle.mkdir(parents=True, exist_ok=True) @@ -92,12 +240,39 @@ def test_manifest_writes_roles_and_playbook_with_clean_when(tmp_path: Path): "kbd", encoding="utf-8" ) - manifest(str(bundle), str(out)) + # Create artifacts for usr_local_custom files so copy works + (bundle / "artifacts" / "usr_local_custom" / "usr" / "local" / "etc").mkdir( + parents=True, exist_ok=True + ) + ( + bundle + / "artifacts" + / "usr_local_custom" + / "usr" + / "local" + / "etc" + / "myapp.conf" + ).write_text("myapp=1\n", encoding="utf-8") + (bundle / "artifacts" / "usr_local_custom" / "usr" / "local" / "bin").mkdir( + parents=True, exist_ok=True + ) + ( + bundle / "artifacts" / "usr_local_custom" / "usr" / "local" / "bin" / "myscript" + ).write_text("#!/bin/sh\necho hi\n", encoding="utf-8") - # Service role: conditional start must be a clean Ansible expression + manifest.manifest(str(bundle), str(out), no_common_roles=True) + + # Service role: systemd management should be gated on foo_manage_unit and a probe. tasks = (out / "roles" / "foo" / "tasks" / "main.yml").read_text(encoding="utf-8") - assert "when:\n - _unit_probe is succeeded\n - foo_start | bool\n" in tasks - # Ensure we didn't emit deprecated/broken '{{ }}' delimiters in when: + assert "- name: Probe whether systemd unit exists and is manageable" in tasks + assert 'no_log: "{{ enroll_hide_systemd_status | default(true) | bool }}"' in tasks + assert "when: foo_manage_unit | default(false)" in tasks + assert ( + "when:\n - foo_manage_unit | default(false)\n - _unit_probe is succeeded\n" + in tasks + ) + + # Ensure we didn't emit deprecated/broken '{{ }}' delimiters in when: lines. for line in tasks.splitlines(): if line.lstrip().startswith("when:"): assert "{{" not in line and "}}" not in line @@ -105,11 +280,1879 @@ def test_manifest_writes_roles_and_playbook_with_clean_when(tmp_path: Path): defaults = (out / "roles" / "foo" / "defaults" / "main.yml").read_text( encoding="utf-8" ) - assert "foo_start: false" in defaults + assert "foo_manage_unit: true" in defaults + assert "foo_systemd_enabled: true" in defaults + assert "foo_systemd_state: stopped" in defaults # Playbook should include users, etc_custom, packages, and services pb = (out / "playbook.yml").read_text(encoding="utf-8") - assert "- users" in pb - assert "- etc_custom" in pb - assert "- curl" in pb - assert "- foo" in pb + assert "role: users" in pb + assert "role: etc_custom" in pb + assert "role: usr_local_custom" in pb + assert "role: curl" in pb + assert "role: foo" in pb + + +def test_manifest_groups_simple_packages_by_section_by_default(tmp_path: Path): + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + state = _minimal_package_state( + [ + { + "package": "curl", + "role_name": "curl", + "section": "net", + "has_config": False, + "managed_files": [], + "managed_dirs": [], + "managed_links": [], + "excluded": [], + "notes": [], + }, + { + "package": "rsync", + "role_name": "rsync", + "section": "net", + "has_config": False, + "managed_files": [], + "managed_dirs": [], + "managed_links": [], + "excluded": [], + "notes": [], + }, + { + "package": "vim", + "role_name": "vim", + "section": "editors", + "has_config": False, + "managed_files": [], + "managed_dirs": [], + "managed_links": [], + "excluded": [], + "notes": [], + }, + { + "package": "nginx", + "role_name": "nginx", + "section": "httpd", + "has_config": True, + "managed_files": [], + "managed_dirs": [], + "managed_links": [], + "excluded": [], + "notes": [], + }, + ] + ) + _write_state(bundle, state) + + manifest.manifest(str(bundle), str(out)) + + assert (out / "roles" / "net").exists() + assert (out / "roles" / "editors").exists() + assert (out / "roles" / "httpd").exists() + assert not (out / "roles" / "curl").exists() + assert not (out / "roles" / "rsync").exists() + assert not (out / "roles" / "vim").exists() + assert not (out / "roles" / "nginx").exists() + + net_defaults = (out / "roles" / "net" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + assert "- curl" in net_defaults + assert "- rsync" in net_defaults + + pb = (out / "playbook.yml").read_text(encoding="utf-8") + assert "role: net" in pb + assert "role: editors" in pb + assert "role: httpd" in pb + + +def test_manifest_no_common_roles_preserves_package_roles(tmp_path: Path): + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + state = _minimal_package_state( + [ + { + "package": "curl", + "role_name": "curl", + "section": "net", + "has_config": False, + "managed_files": [], + "managed_dirs": [], + "managed_links": [], + "excluded": [], + "notes": [], + }, + { + "package": "vim", + "role_name": "vim", + "section": "editors", + "has_config": False, + "managed_files": [], + "managed_dirs": [], + "managed_links": [], + "excluded": [], + "notes": [], + }, + ] + ) + _write_state(bundle, state) + + manifest.manifest(str(bundle), str(out), no_common_roles=True) + + assert (out / "roles" / "curl").exists() + assert (out / "roles" / "vim").exists() + assert not (out / "roles" / "net").exists() + assert not (out / "roles" / "editors").exists() + + +def test_manifest_groups_excluded_package_paths_into_common_roles(tmp_path: Path): + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + state = _minimal_package_state( + [ + { + "package": "secret-agent", + "role_name": "secret_agent", + "section": "net", + "has_config": False, + "managed_files": [], + "managed_dirs": [], + "managed_links": [], + "excluded": [ + {"path": "/etc/secret-agent/key", "reason": "possible_secret"} + ], + "notes": [], + } + ] + ) + _write_state(bundle, state) + + manifest.manifest(str(bundle), str(out)) + + assert (out / "roles" / "net").exists() + assert not (out / "roles" / "secret_agent").exists() + readme = (out / "roles" / "net" / "README.md").read_text(encoding="utf-8") + assert "/etc/secret-agent/key" in readme + + +def test_manifest_groups_managed_package_config_into_common_role(tmp_path: Path): + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + (bundle / "artifacts" / "nginx" / "etc" / "nginx").mkdir( + parents=True, exist_ok=True + ) + (bundle / "artifacts" / "nginx" / "etc" / "nginx" / "nginx.conf").write_text( + "worker_processes auto;\n", encoding="utf-8" + ) + state = _minimal_package_state( + [ + { + "package": "nginx", + "role_name": "nginx", + "section": "httpd", + "has_config": True, + "managed_files": [ + { + "path": "/etc/nginx/nginx.conf", + "src_rel": "etc/nginx/nginx.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "modified_conffile", + } + ], + "managed_dirs": [ + { + "path": "/etc/nginx", + "owner": "root", + "group": "root", + "mode": "0755", + "reason": "parent_of_managed_file", + } + ], + "managed_links": [], + "excluded": [], + "notes": [], + } + ] + ) + _write_state(bundle, state) + + manifest.manifest(str(bundle), str(out)) + + assert (out / "roles" / "httpd").exists() + assert not (out / "roles" / "nginx").exists() + defaults = (out / "roles" / "httpd" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + assert "- nginx" in defaults + assert "dest: /etc/nginx/nginx.conf" in defaults + assert (out / "roles" / "httpd" / "files" / "etc" / "nginx" / "nginx.conf").exists() + + +def test_manifest_groups_systemd_units_into_common_role(tmp_path: Path): + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + (bundle / "artifacts" / "network_manager" / "etc" / "NetworkManager").mkdir( + parents=True, exist_ok=True + ) + ( + bundle + / "artifacts" + / "network_manager" + / "etc" + / "NetworkManager" + / "NetworkManager.conf" + ).write_text("[main]\n", encoding="utf-8") + + state = { + "schema_version": 3, + "host": {"hostname": "test", "os": "debian", "pkg_backend": "dpkg"}, + "inventory": { + "packages": { + "network-manager": { + "version": "1.0", + "arches": ["amd64"], + "installations": [ + {"version": "1.0", "arch": "amd64", "section": "net"} + ], + "section": "net", + "observed_via": [ + {"kind": "systemd_unit", "ref": "NetworkManager.service"} + ], + "roles": ["network_manager", "network_manager_dispatcher"], + } + } + }, + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [ + { + "unit": "NetworkManager.service", + "role_name": "network_manager", + "packages": ["network-manager"], + "active_state": "active", + "sub_state": "running", + "unit_file_state": "enabled", + "condition_result": "yes", + "managed_files": [ + { + "path": "/etc/NetworkManager/NetworkManager.conf", + "src_rel": "etc/NetworkManager/NetworkManager.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "modified_conffile", + } + ], + "managed_dirs": [], + "managed_links": [], + "excluded": [], + "notes": [], + }, + { + "unit": "NetworkManager-dispatcher.service", + "role_name": "network_manager_dispatcher", + "packages": ["network-manager"], + "active_state": "inactive", + "sub_state": "dead", + "unit_file_state": "enabled", + "condition_result": "no", + "managed_files": [], + "managed_dirs": [], + "managed_links": [], + "excluded": [], + "notes": [], + }, + ], + "packages": [], + "apt_config": { + "role_name": "apt_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + }, + } + _write_state(bundle, state) + + manifest.manifest(str(bundle), str(out)) + + assert (out / "roles" / "net").exists() + assert not (out / "roles" / "network_manager").exists() + assert not (out / "roles" / "network_manager_dispatcher").exists() + defaults = (out / "roles" / "net" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + assert "- network-manager" in defaults + assert "name: NetworkManager.service" in defaults + assert "name: NetworkManager-dispatcher.service" in defaults + assert "dest: /etc/NetworkManager/NetworkManager.conf" in defaults + tasks = (out / "roles" / "net" / "tasks" / "main.yml").read_text(encoding="utf-8") + assert "Ensure grouped unit enablement matches harvest" in tasks + assert 'no_log: "{{ enroll_hide_systemd_status | default(true) | bool }}"' in tasks + + +def test_manifest_fqdn_implies_no_common_roles(tmp_path: Path): + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + state = _minimal_package_state( + [ + { + "package": "curl", + "role_name": "curl", + "section": "net", + "has_config": False, + "managed_files": [], + "managed_dirs": [], + "managed_links": [], + "excluded": [], + "notes": [], + } + ] + ) + _write_state(bundle, state) + + manifest.manifest(str(bundle), str(out), fqdn="host1.example.test") + + assert (out / "roles" / "curl").exists() + assert not (out / "roles" / "net").exists() + + +def test_manifest_site_mode_creates_host_inventory_and_raw_files(tmp_path: Path): + """In --fqdn mode, host-specific state goes into inventory/host_vars.""" + + fqdn = "host1.example.test" + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + + # Artifacts for a service-managed file. + (bundle / "artifacts" / "foo" / "etc").mkdir(parents=True, exist_ok=True) + (bundle / "artifacts" / "foo" / "etc" / "foo.conf").write_text( + "x", encoding="utf-8" + ) + + # Artifacts for etc_custom file so copy works. + (bundle / "artifacts" / "etc_custom" / "etc" / "default").mkdir( + parents=True, exist_ok=True + ) + (bundle / "artifacts" / "etc_custom" / "etc" / "default" / "keyboard").write_text( + "kbd", encoding="utf-8" + ) + + state = { + "schema_version": 3, + "host": {"hostname": "test", "os": "debian", "pkg_backend": "dpkg"}, + "inventory": { + "packages": { + "foo": { + "version": "1.0", + "arches": [], + "installations": [{"version": "1.0", "arch": "amd64"}], + "observed_via": [{"kind": "systemd_unit", "ref": "foo.service"}], + "roles": ["foo"], + } + } + }, + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [ + { + "unit": "foo.service", + "role_name": "foo", + "packages": ["foo"], + "active_state": "active", + "sub_state": "running", + "unit_file_state": "enabled", + "condition_result": "yes", + "managed_files": [ + { + "path": "/etc/foo.conf", + "src_rel": "etc/foo.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "modified_conffile", + } + ], + "excluded": [], + "notes": [], + } + ], + "packages": [], + "apt_config": { + "role_name": "apt_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [ + { + "path": "/etc/default/keyboard", + "src_rel": "etc/default/keyboard", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "custom_unowned", + } + ], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [ + { + "path": "/usr/local/etc/myapp.conf", + "src_rel": "usr/local/etc/myapp.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "usr_local_etc_custom", + } + ], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + }, + } + + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + # Artifacts for usr_local_custom file so copy works. + (bundle / "artifacts" / "usr_local_custom" / "usr" / "local" / "etc").mkdir( + parents=True, exist_ok=True + ) + ( + bundle + / "artifacts" + / "usr_local_custom" + / "usr" + / "local" + / "etc" + / "myapp.conf" + ).write_text("myapp=1\n", encoding="utf-8") + + manifest.manifest(str(bundle), str(out), fqdn=fqdn) + + # Host playbook exists. + assert (out / "playbooks" / f"{fqdn}.yml").exists() + + # Role defaults are safe/host-agnostic in site mode. + foo_defaults = (out / "roles" / "foo" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + assert "foo_packages: []" in foo_defaults + assert "foo_managed_files: []" in foo_defaults + assert "foo_manage_unit: false" in foo_defaults + + # Host vars contain host-specific state. + foo_hostvars = (out / "inventory" / "host_vars" / fqdn / "foo.yml").read_text( + encoding="utf-8" + ) + assert "foo_packages" in foo_hostvars + assert "foo_managed_files" in foo_hostvars + assert "foo_manage_unit: true" in foo_hostvars + assert "foo_systemd_state: started" in foo_hostvars + + # Non-templated raw config is stored per-host under .files. + assert ( + out / "inventory" / "host_vars" / fqdn / "foo" / ".files" / "etc" / "foo.conf" + ).exists() + + +def test_copy2_replace_overwrites_readonly_destination(tmp_path: Path): + """Merging into an existing manifest should tolerate read-only files. + + Some harvested artifacts (e.g. private keys) may be mode 0400. If a previous + run copied them into the destination tree, a subsequent run must still be + able to update/replace them. + """ + + import os + import stat + + from enroll.ansible_renderer.layout import _copy2_replace + + src = tmp_path / "src" + dst = tmp_path / "dst" + src.write_text("new", encoding="utf-8") + dst.write_text("old", encoding="utf-8") + os.chmod(dst, 0o400) + + _copy2_replace(str(src), str(dst)) + + assert dst.read_text(encoding="utf-8") == "new" + mode = stat.S_IMODE(dst.stat().st_mode) + assert mode & stat.S_IWUSR # destination should remain mergeable + + +def test_manifest_includes_dnf_config_role_when_present(tmp_path: Path): + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + + # Create a dnf_config artifact. + (bundle / "artifacts" / "dnf_config" / "etc" / "dnf").mkdir( + parents=True, exist_ok=True + ) + (bundle / "artifacts" / "dnf_config" / "etc" / "dnf" / "dnf.conf").write_text( + "[main]\n", encoding="utf-8" + ) + + state = { + "schema_version": 3, + "host": {"hostname": "test", "os": "redhat", "pkg_backend": "rpm"}, + "inventory": { + "packages": { + "dnf": { + "version": "4.0", + "arches": [], + "installations": [{"version": "4.0", "arch": "x86_64"}], + "observed_via": [{"kind": "dnf_config"}], + "roles": [], + } + } + }, + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [], + "packages": [], + "apt_config": { + "role_name": "apt_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_files": [ + { + "path": "/etc/dnf/dnf.conf", + "src_rel": "etc/dnf/dnf.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "dnf_config", + } + ], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + }, + } + + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + manifest.manifest(str(bundle), str(out)) + + pb = (out / "playbook.yml").read_text(encoding="utf-8") + assert "role: dnf_config" in pb + + tasks = (out / "roles" / "dnf_config" / "tasks" / "main.yml").read_text( + encoding="utf-8" + ) + # Ensure the role exists and contains some file deployment logic. + assert "Deploy any other managed files" in tasks + + +def test_render_install_packages_tasks_uses_generic_package_provider(): + from enroll.ansible_renderer.tasks import _render_install_packages_tasks + + txt = _render_install_packages_tasks("role", "role") + assert "ansible.builtin.package" in txt + assert "ansible.builtin.apt" not in txt + assert "ansible.builtin.dnf" not in txt + assert "ansible.builtin.dnf5" not in txt + assert "pkg_mgr" not in txt + + +def test_manifest_orders_cron_and_logrotate_at_playbook_tail(tmp_path: Path): + """Cron/logrotate roles should appear at the end. + + The cron role may restore per-user crontabs under /var/spool, so it should + run after users have been created. + """ + + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + + state = { + "schema_version": 3, + "host": {"hostname": "test", "os": "debian", "pkg_backend": "dpkg"}, + "inventory": {"packages": {}}, + "roles": { + "users": { + "role_name": "users", + "users": [{"name": "alice"}], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [], + "packages": [ + { + "package": "curl", + "role_name": "curl", + "managed_files": [], + "excluded": [], + "notes": [], + }, + { + "package": "cron", + "role_name": "cron", + "managed_files": [ + { + "path": "/var/spool/cron/crontabs/alice", + "src_rel": "var/spool/cron/crontabs/alice", + "owner": "alice", + "group": "root", + "mode": "0600", + "reason": "system_cron", + } + ], + "excluded": [], + "notes": [], + }, + { + "package": "logrotate", + "role_name": "logrotate", + "managed_files": [ + { + "path": "/etc/logrotate.conf", + "src_rel": "etc/logrotate.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "system_logrotate", + } + ], + "excluded": [], + "notes": [], + }, + ], + "apt_config": { + "role_name": "apt_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + }, + } + + # Minimal artifacts for managed files. + (bundle / "artifacts" / "cron" / "var" / "spool" / "cron" / "crontabs").mkdir( + parents=True, exist_ok=True + ) + ( + bundle / "artifacts" / "cron" / "var" / "spool" / "cron" / "crontabs" / "alice" + ).write_text("@daily echo hi\n", encoding="utf-8") + (bundle / "artifacts" / "logrotate" / "etc").mkdir(parents=True, exist_ok=True) + (bundle / "artifacts" / "logrotate" / "etc" / "logrotate.conf").write_text( + "weekly\n", encoding="utf-8" + ) + + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + manifest.manifest(str(bundle), str(out)) + + pb = (out / "playbook.yml").read_text(encoding="utf-8").splitlines() + # Roles are emitted as indented list items under the `roles:` key. + roles = [ + ln.strip().removeprefix("- ").strip() for ln in pb if ln.startswith(" - ") + ] + + # Ensure the grouped role containing cron/logrotate is still ordered after users. + assert roles[-1] == "role: misc" + assert roles.index("role: users") < roles.index("role: misc") + assert "role: users" in roles + + +def test_yaml_helpers_fallback_when_yaml_unavailable(monkeypatch): + monkeypatch.setattr(ansible_yaml, "_try_yaml", lambda: None) + assert ansible_yaml._yaml_load_mapping("foo: 1\n") == {} + out = ansible_yaml._yaml_dump_mapping({"b": 2, "a": 1}) + # Best-effort fallback is key: repr(value) + assert out.splitlines()[0].startswith("a: ") + assert out.endswith("\n") + + +def test_copy2_replace_makes_readonly_sources_user_writable( + monkeypatch, tmp_path: Path +): + src = tmp_path / "src.txt" + dst = tmp_path / "dst.txt" + src.write_text("hello", encoding="utf-8") + # Make source read-only; copy2 preserves mode, so tmp will be read-only too. + os.chmod(src, 0o444) + + ansible_layout._copy2_replace(str(src), str(dst)) + + st = os.stat(dst, follow_symlinks=False) + assert stat.S_IMODE(st.st_mode) & stat.S_IWUSR + + +def test_prepare_bundle_dir_sops_decrypts_and_extracts(monkeypatch, tmp_path: Path): + enc = tmp_path / "harvest.tar.gz.sops" + enc.write_text("ignored", encoding="utf-8") + + def fake_require(): + return None + + def fake_decrypt(src: str, dst: str, *, mode: int = 0o600): + # Create a minimal tar.gz with a state.json file. + with tarfile.open(dst, "w:gz") as tf: + p = tmp_path / "state.json" + p.write_text("{}", encoding="utf-8") + tf.add(p, arcname="state.json") + + monkeypatch.setattr(manifest, "require_sops_cmd", fake_require) + monkeypatch.setattr(manifest, "decrypt_file_binary_to", fake_decrypt) + + bundle_dir, td = manifest._prepare_bundle_dir(str(enc), sops_mode=True) + try: + assert (Path(bundle_dir) / "state.json").exists() + finally: + td.cleanup() + + +def test_prepare_bundle_dir_rejects_non_dir_without_sops(tmp_path: Path): + fp = tmp_path / "bundle.tar.gz" + fp.write_text("x", encoding="utf-8") + with pytest.raises(RuntimeError): + manifest._prepare_bundle_dir(str(fp), sops_mode=False) + + +def test_tar_dir_to_with_progress_writes_progress_when_tty(monkeypatch, tmp_path: Path): + src = tmp_path / "dir" + src.mkdir() + (src / "a.txt").write_text("a", encoding="utf-8") + (src / "b.txt").write_text("b", encoding="utf-8") + + out = tmp_path / "out.tar.gz" + writes: list[bytes] = [] + + monkeypatch.setattr(manifest.os, "isatty", lambda fd: True) + monkeypatch.setattr(manifest.os, "write", lambda fd, b: writes.append(b) or len(b)) + + manifest._tar_dir_to_with_progress(str(src), str(out), desc="tarring") + assert out.exists() + assert writes # progress was written + assert writes[-1].endswith(b"\n") + + +def test_encrypt_manifest_out_dir_to_sops_handles_missing_tmp_cleanup( + monkeypatch, tmp_path: Path +): + src_dir = tmp_path / "manifest" + src_dir.mkdir() + (src_dir / "x.txt").write_text("x", encoding="utf-8") + + out = tmp_path / "manifest.tar.gz.sops" + + monkeypatch.setattr(manifest, "require_sops_cmd", lambda: None) + + def fake_encrypt(in_fp, out_fp, *args, **kwargs): + Path(out_fp).write_text("enc", encoding="utf-8") + + monkeypatch.setattr(manifest, "encrypt_file_binary", fake_encrypt) + # Simulate race where tmp tar is already removed. + monkeypatch.setattr( + manifest.os, "unlink", lambda p: (_ for _ in ()).throw(FileNotFoundError()) + ) + + res = manifest._encrypt_manifest_out_dir_to_sops(str(src_dir), str(out), ["ABC"]) # type: ignore[arg-type] + assert str(res).endswith(".sops") + assert out.exists() + + +def test_manifest_applies_jinjaturtle_to_jinjifyable_managed_file( + monkeypatch, tmp_path: Path +): + # Create a minimal bundle with just an apt_config snapshot. + bundle = tmp_path / "bundle" + (bundle / "artifacts" / "apt_config" / "etc" / "apt").mkdir(parents=True) + (bundle / "artifacts" / "apt_config" / "etc" / "apt" / "foo.ini").write_text( + "key=VALUE\n", encoding="utf-8" + ) + + state = { + "schema_version": 1, + "inventory": {"packages": {}}, + "roles": { + "services": [], + "packages": [], + "apt_config": { + "role_name": "apt_config", + "managed_files": [ + { + "path": "/etc/apt/foo.ini", + "src_rel": "etc/apt/foo.ini", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "apt_config", + } + ], + "managed_dirs": [], + "excluded": [], + "notes": [], + }, + }, + } + (bundle / "state.json").write_text( + __import__("json").dumps(state), encoding="utf-8" + ) + + monkeypatch.setattr(ansible_context, "find_jinjaturtle_cmd", lambda: "jinjaturtle") + + class _Res: + template_text = "key={{ foo }}\n" + vars_text = "foo: 123\n" + + monkeypatch.setattr(ansible_jt, "run_jinjaturtle", lambda *a, **k: _Res()) + + out_dir = tmp_path / "out" + manifest.manifest(str(bundle), str(out_dir), jinjaturtle="on") + + tmpl = out_dir / "roles" / "apt_config" / "templates" / "etc" / "apt" / "foo.ini.j2" + assert tmpl.exists() + assert "{{ foo }}" in tmpl.read_text(encoding="utf-8") + + defaults = out_dir / "roles" / "apt_config" / "defaults" / "main.yml" + txt = defaults.read_text(encoding="utf-8") + assert "foo: 123" in txt + # Non-templated file should not exist under files/. + assert not ( + out_dir / "roles" / "apt_config" / "files" / "etc" / "apt" / "foo.ini" + ).exists() + + +def test_manifest_writes_firewall_runtime_role(tmp_path: Path): + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + (bundle / "artifacts" / "firewall_runtime" / "firewall").mkdir( + parents=True, exist_ok=True + ) + (bundle / "artifacts" / "firewall_runtime" / "firewall" / "ipset.save").write_text( + "create blocklist hash:ip family inet\nadd blocklist 203.0.113.10\n", + encoding="utf-8", + ) + (bundle / "artifacts" / "firewall_runtime" / "firewall" / "iptables.v4").write_text( + "*filter\n:INPUT DROP [0:0]\n-A INPUT -m set --match-set blocklist src -j DROP\nCOMMIT\n", + encoding="utf-8", + ) + + state = { + "schema_version": 3, + "host": {"hostname": "test", "os": "debian", "pkg_backend": "dpkg"}, + "inventory": {"packages": {}}, + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [], + "packages": [], + "apt_config": { + "role_name": "apt_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "firewall_runtime": { + "role_name": "firewall_runtime", + "packages": ["ipset", "iptables"], + "ipset_save": "firewall/ipset.save", + "ipset_sets": ["blocklist"], + "iptables_v4_save": "firewall/iptables.v4", + "iptables_v6_save": None, + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + }, + } + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + manifest.manifest(str(bundle), str(out)) + + tasks = (out / "roles" / "firewall_runtime" / "tasks" / "main.yml").read_text( + encoding="utf-8" + ) + assert "ipset restore -exist" in tasks + assert "iptables-restore /etc/enroll/firewall/iptables.v4" in tasks + assert "ipset flush {{ item }}" in tasks + + defaults = (out / "roles" / "firewall_runtime" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + assert "firewall_runtime_ipset_sets:" in defaults + assert "- blocklist" in defaults + assert "firewall_runtime_restore_iptables: true" in defaults + + pb = (out / "playbook.yml").read_text(encoding="utf-8") + assert "role: firewall_runtime" in pb + assert ( + out / "roles" / "firewall_runtime" / "files" / "firewall" / "ipset.save" + ).exists() + + +def test_try_yaml_with_yaml_installed(): + result = ansible_yaml._try_yaml() + # PyYAML should be installed for tests + if result is None: + pytest.skip("PyYAML not installed") + assert hasattr(result, "safe_load") + assert hasattr(result, "dump") + + +def test_yaml_load_mapping_with_yaml(tmp_path: Path): + text = """ +key1: value1 +key2: + nested: value +list: + - item1 + - item2 +""" + result = ansible_yaml._yaml_load_mapping(text) + assert result["key1"] == "value1" + assert result["key2"]["nested"] == "value" + assert result["list"] == ["item1", "item2"] + + +def test_yaml_load_mapping_empty(): + result = ansible_yaml._yaml_load_mapping("") + assert result == {} + + +def test_yaml_load_mapping_invalid(): + result = ansible_yaml._yaml_load_mapping("invalid: yaml: :") + assert result == {} + + +def test_yaml_load_mapping_not_dict(): + result = ansible_yaml._yaml_load_mapping("- item1\n- item2") + assert result == {} + + +def test_yaml_load_mapping_none(): + result = ansible_yaml._yaml_load_mapping("~") + assert result == {} + + +def test_yaml_dump_mapping_with_yaml(tmp_path: Path): + obj = {"key1": "value1", "key2": 123} + result = ansible_yaml._yaml_dump_mapping(obj) + assert "key1: value1" in result + assert "key2:" in result + + +def test_yaml_dump_mapping_empty(): + result = ansible_yaml._yaml_dump_mapping({}) + # Empty dict produces '{}' + assert result.strip() == "{}" + + +def test_yaml_dump_mapping_with_nested(tmp_path: Path): + obj = {"key1": {"nested": "value"}} + result = ansible_yaml._yaml_dump_mapping(obj) + assert "nested:" in result + + +def test_merge_mappings_overwrite_simple(): + existing = {"key1": "old", "key2": "keep"} + incoming = {"key1": "new", "key3": "added"} + result = ansible_yaml._merge_mappings_overwrite(existing, incoming) + assert result["key1"] == "new" + assert result["key2"] == "keep" + assert result["key3"] == "added" + + +def test_merge_mappings_overwrite_nested(): + existing = {"key1": {"a": 1}} + incoming = {"key1": {"b": 2}} + result = ansible_yaml._merge_mappings_overwrite(existing, incoming) + # Nested dicts are replaced, not merged + assert result["key1"] == {"b": 2} + + +def test_merge_mappings_overwrite_empty(): + result = ansible_yaml._merge_mappings_overwrite({}, {"key": "value"}) + assert result == {"key": "value"} + + result = ansible_yaml._merge_mappings_overwrite({"key": "value"}, {}) + assert result == {"key": "value"} + + +def test_copy2_replace(tmp_path: Path): + src = tmp_path / "src.txt" + src.write_text("content", encoding="utf-8") + dst = tmp_path / "dst" / "subdir" / "dst.txt" + + ansible_layout._copy2_replace(str(src), str(dst)) + + assert dst.exists() + assert dst.read_text(encoding="utf-8") == "content" + + +def test_copy2_replace_preserves_metadata(tmp_path: Path): + src = tmp_path / "src.txt" + src.write_text("content", encoding="utf-8") + os.chmod(str(src), 0o644) + dst = tmp_path / "dst.txt" + + ansible_layout._copy2_replace(str(src), str(dst)) + + assert dst.exists() + st = dst.stat() + assert stat.S_IMODE(st.st_mode) == 0o644 + + +def test_copy2_replace_atomic(tmp_path: Path): + src = tmp_path / "src.txt" + src.write_text("content", encoding="utf-8") + dst = tmp_path / "dst.txt" + + # Write initial content + dst.write_text("old", encoding="utf-8") + + ansible_layout._copy2_replace(str(src), str(dst)) + + assert dst.read_text(encoding="utf-8") == "content" + + +def test_render_firewall_runtime_tasks_empty(): + result = ansible_tasks._render_firewall_runtime_tasks("firewall_runtime") + # Function always returns at least a basic playbook structure + assert isinstance(result, str) + assert len(result) > 0 + + +def test_render_firewall_runtime_tasks_with_iptables(): + result = ansible_tasks._render_firewall_runtime_tasks("firewall_runtime") + assert len(result) >= 1 + + +def test_render_firewall_runtime_tasks_with_ipset(): + result = ansible_tasks._render_firewall_runtime_tasks("firewall_runtime") + assert len(result) >= 1 + + +def test_render_firewall_runtime_tasks_with_ipv6(): + result = ansible_tasks._render_firewall_runtime_tasks("firewall_runtime") + assert len(result) >= 1 + + +def test_manifest_renders_flatpak_and_snap_details(tmp_path: Path): + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + state = { + "schema_version": 3, + "host": {"hostname": "test", "os": "debian", "pkg_backend": "dpkg"}, + "inventory": {"packages": {}}, + "roles": { + "users": { + "role_name": "users", + "users": [ + { + "name": "alice", + "uid": 1000, + "gid": 1000, + "gecos": "Alice", + "home": "/home/alice", + "shell": "/bin/bash", + "primary_group": "alice", + "supplementary_groups": [], + } + ], + "managed_files": [], + "excluded": [], + "notes": [], + "user_flatpak_remotes": [ + { + "name": "acme-user", + "method": "user", + "url": "https://flatpak.example/user-repo/", + "user": "alice", + "home": "/home/alice", + }, + ], + "user_flatpaks": { + "alice": [ + { + "name": "org.example.UserApp", + "method": "user", + "remote": "acme-user", + "branch": "stable", + "arch": "x86_64", + } + ] + }, + }, + "flatpak": { + "role_name": "flatpak", + "remotes": [ + { + "name": "acme", + "method": "system", + "url": "https://flatpak.example/repo/", + }, + ], + "system_flatpaks": [ + { + "name": "com.example.App", + "method": "system", + "remote": "acme", + "branch": "stable", + "arch": "x86_64", + } + ], + "notes": [], + }, + "snap": { + "role_name": "snap", + "system_snaps": [ + { + "name": "code", + "channel": "latest/stable", + "revision": 123, + "classic": True, + "notes": ["classic"], + } + ], + "notes": [], + }, + "services": [], + "packages": [], + "apt_config": { + "role_name": "apt_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + }, + } + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + manifest.manifest(str(bundle), str(out)) + + users_defaults = (out / "roles" / "users" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + users_tasks = (out / "roles" / "users" / "tasks" / "main.yml").read_text( + encoding="utf-8" + ) + users_readme = (out / "roles" / "users" / "README.md").read_text(encoding="utf-8") + flatpak_defaults = (out / "roles" / "flatpak" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + flatpak_tasks = (out / "roles" / "flatpak" / "tasks" / "main.yml").read_text( + encoding="utf-8" + ) + snap_defaults = (out / "roles" / "snap" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + snap_tasks = (out / "roles" / "snap" / "tasks" / "main.yml").read_text( + encoding="utf-8" + ) + + assert "users_flatpak_remotes:" in users_defaults + assert "remote: acme-user" in users_defaults + assert "community.general.snap" not in users_tasks + assert "Install system-wide snaps" not in users_tasks + assert "Install system-wide Flatpaks" not in users_tasks + assert "ansible-galaxy collection install -r requirements.yml" in users_readme + + assert "snap_system_snaps:" in snap_defaults + assert "channel: latest/stable" in snap_defaults + assert "classic: true" in snap_defaults + assert "community.general.snap" in snap_tasks + assert "Install system-wide snaps with full detected attributes" in snap_tasks + assert "Install system-wide snaps with compatibility options" in snap_tasks + assert "Install system-wide snaps with minimal options" in snap_tasks + assert "ignore_errors: true" in snap_tasks + + assert "flatpak_system_flatpaks:" in flatpak_defaults + assert "remote: acme" in flatpak_defaults + assert "community.general.flatpak" in flatpak_tasks + assert "Install system-wide Flatpaks" in flatpak_tasks + assert (out / "requirements.yml").exists() + + +def test_users_role_without_portable_apps_omits_community_general_tasks(tmp_path): + bundle = tmp_path / "bundle" + out = tmp_path / "out" + state = { + "roles": { + "users": { + "role_name": "users", + "users": [ + { + "name": "alice", + "uid": 1000, + "gid": 1000, + "gecos": "Alice", + "home": "/home/alice", + "shell": "/bin/bash", + "primary_group": "alice", + "supplementary_groups": [], + } + ], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [], + "packages": [], + }, + } + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + manifest.manifest(str(bundle), str(out)) + + users_tasks = (out / "roles" / "users" / "tasks" / "main.yml").read_text( + encoding="utf-8" + ) + users_meta = (out / "roles" / "users" / "meta" / "main.yml").read_text( + encoding="utf-8" + ) + + assert "community.general.flatpak" not in users_tasks + assert "community.general.snap" not in users_tasks + assert "collections:" not in users_meta + + +def test_users_role_only_creates_ssh_dir_when_managed_ssh_files_exist(tmp_path): + bundle = tmp_path / "bundle" + out = tmp_path / "out" + (bundle / "artifacts" / "users" / "alice" / ".ssh").mkdir( + parents=True, exist_ok=True + ) + (bundle / "artifacts" / "users" / "bob").mkdir(parents=True, exist_ok=True) + (bundle / "artifacts" / "users" / "alice" / ".ssh" / "authorized_keys").write_text( + "ssh-ed25519 example alice\n", encoding="utf-8" + ) + (bundle / "artifacts" / "users" / "bob" / ".bashrc").write_text( + "alias ll='ls -l'\n", encoding="utf-8" + ) + state = { + "roles": { + "users": { + "role_name": "users", + "users": [ + { + "name": "alice", + "uid": 1000, + "home": "/home/alice", + "primary_group": "alice", + "supplementary_groups": [], + }, + { + "name": "bob", + "uid": 1001, + "home": "/home/bob", + "primary_group": "bob", + "supplementary_groups": [], + }, + { + "name": "carol", + "uid": 1002, + "home": "/home/carol", + "primary_group": "carol", + "supplementary_groups": [], + }, + ], + "managed_files": [ + { + "path": "/home/alice/.ssh/authorized_keys", + "src_rel": "alice/.ssh/authorized_keys", + "mode": "0644", + "reason": "authorized_keys", + }, + { + "path": "/home/bob/.bashrc", + "src_rel": "bob/.bashrc", + "mode": "0644", + "reason": "dangerous_user_dotfile", + }, + ], + "excluded": [], + "notes": [], + }, + "services": [], + "packages": [], + }, + } + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + manifest.manifest(str(bundle), str(out)) + + users_defaults_text = (out / "roles" / "users" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + users_defaults = ansible_yaml._yaml_load_mapping(users_defaults_text) + users_tasks = (out / "roles" / "users" / "tasks" / "main.yml").read_text( + encoding="utf-8" + ) + + assert users_defaults["users_ssh_dirs"] == [ + { + "dest": "/home/alice/.ssh", + "group": "alice", + "mode": "0700", + "owner": "alice", + } + ] + assert 'loop: "{{ users_ssh_dirs | default([]) }}"' in users_tasks + assert 'path: "{{ item.ssh_dir }}"' not in users_tasks + assert "users_ssh_files" in users_defaults + + +def test_manifest_emits_flatpak_role_even_when_no_flatpaks(tmp_path): + bundle = tmp_path / "bundle" + out = tmp_path / "out" + state = { + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + "user_flatpaks": {}, + "user_flatpak_remotes": [], + }, + "flatpak": { + "role_name": "flatpak", + "system_flatpaks": [], + "remotes": [], + "notes": [], + }, + "services": [], + "packages": [], + } + } + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + manifest.manifest(str(bundle), str(out)) + + flatpak_tasks = (out / "roles" / "flatpak" / "tasks" / "main.yml").read_text( + encoding="utf-8" + ) + flatpak_defaults = (out / "roles" / "flatpak" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + + assert "flatpak_system_flatpaks: []" in flatpak_defaults + assert "flatpak_remotes: []" in flatpak_defaults + assert "Install system-wide Flatpaks" in flatpak_tasks + assert "Ensure system Flatpak remotes exist" in flatpak_tasks + + +def test_manifest_avoids_package_role_collision_with_flatpak_singleton(tmp_path): + bundle = tmp_path / "bundle" + out = tmp_path / "out" + state = { + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + "user_flatpaks": {}, + "user_flatpak_remotes": [], + }, + "flatpak": { + "role_name": "flatpak", + "remotes": [ + { + "name": "flathub", + "method": "system", + "url": "https://dl.flathub.org/repo/", + } + ], + "system_flatpaks": [ + { + "name": "org.onionshare.OnionShare", + "method": "system", + "remote": "flathub", + "branch": "stable", + "arch": "x86_64", + } + ], + "notes": [], + }, + "services": [], + "packages": [ + { + "package": "flatpak", + "role_name": "flatpak", + "managed_files": [], + "managed_dirs": [], + "managed_links": [], + "excluded": [], + "notes": [], + "has_config": True, + } + ], + } + } + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + manifest.manifest(str(bundle), str(out), no_common_roles=True) + + flatpak_defaults = (out / "roles" / "flatpak" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + playbook = (out / "playbook.yml").read_text(encoding="utf-8") + + assert "org.onionshare.OnionShare" in flatpak_defaults + assert (out / "roles" / "package_flatpak" / "tasks" / "main.yml").exists() + assert "role: flatpak" in playbook + assert "role: package_flatpak" in playbook + + +def test_manifest_writes_sysctl_role(tmp_path: Path): + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + (bundle / "artifacts" / "sysctl" / "sysctl").mkdir(parents=True, exist_ok=True) + (bundle / "artifacts" / "sysctl" / "sysctl" / "99-enroll.conf").write_text( + "net.ipv4.ip_forward = 1\n", + encoding="utf-8", + ) + + state = { + "schema_version": 3, + "host": {"hostname": "test", "os": "debian", "pkg_backend": "dpkg"}, + "inventory": {"packages": {}}, + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [], + "packages": [], + "apt_config": { + "role_name": "apt_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "sysctl": { + "role_name": "sysctl", + "managed_files": [ + { + "path": "/etc/sysctl.d/99-enroll.conf", + "src_rel": "sysctl/99-enroll.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "system_sysctl", + } + ], + "parameters": {"net.ipv4.ip_forward": "1"}, + "notes": ["Captured 1 live writable sysctl parameter(s)."], + }, + "firewall_runtime": { + "role_name": "firewall_runtime", + "packages": [], + "ipset_save": None, + "ipset_sets": [], + "iptables_v4_save": None, + "iptables_v6_save": None, + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + }, + } + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + manifest.manifest(str(bundle), str(out)) + + tasks = (out / "roles" / "sysctl" / "tasks" / "main.yml").read_text( + encoding="utf-8" + ) + assert "dest: /etc/sysctl.d/99-enroll.conf" in tasks + assert "notify: Apply captured sysctl configuration" in tasks + + handlers = (out / "roles" / "sysctl" / "handlers" / "main.yml").read_text( + encoding="utf-8" + ) + assert "- -p" in handlers + assert "- /etc/sysctl.d/99-enroll.conf" in handlers + + defaults = (out / "roles" / "sysctl" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + assert "sysctl_conf_src_rel: sysctl/99-enroll.conf" in defaults + assert "sysctl_ignore_apply_errors: true" in defaults + + pb = (out / "playbook.yml").read_text(encoding="utf-8") + assert "role: sysctl" in pb + assert (out / "roles" / "sysctl" / "files" / "sysctl" / "99-enroll.conf").exists() + + +def test_manifest_renders_container_image_role_for_ansible(tmp_path: Path): + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + digest = "docker.io/library/nginx@sha256:" + "a" * 64 + podman_digest = "quay.io/example/app@sha256:" + "b" * 64 + state = { + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [], + "packages": [], + "container_images": { + "role_name": "container_images", + "images": [ + { + "engine": "docker", + "scope": "system", + "user": None, + "home": None, + "image_id": "sha256:" + "c" * 64, + "repo_tags": ["docker.io/library/nginx:1.27"], + "repo_digests": [digest], + "pull_ref": digest, + "tag_aliases": [ + { + "ref": "docker.io/library/nginx:1.27", + "repository": "docker.io/library/nginx", + "tag": "1.27", + } + ], + "os": "linux", + "architecture": "amd64", + "variant": None, + "platform": "linux/amd64", + "size": 123, + "created": "2026-01-01T00:00:00Z", + "source": "docker image inspect", + "notes": [], + }, + { + "engine": "podman", + "scope": "system", + "user": None, + "home": None, + "image_id": "sha256:" + "d" * 64, + "repo_tags": ["quay.io/example/app:prod"], + "repo_digests": [podman_digest], + "pull_ref": podman_digest, + "tag_aliases": [ + { + "ref": "quay.io/example/app:prod", + "repository": "quay.io/example/app", + "tag": "prod", + } + ], + "os": "linux", + "architecture": "amd64", + "variant": None, + "platform": "linux/amd64", + "size": 456, + "created": "2026-01-01T00:00:00Z", + "source": "podman image inspect", + "notes": [], + }, + ], + "notes": [], + }, + } + } + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + manifest.manifest(str(bundle), str(out)) + + defaults = (out / "roles" / "container_images" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + tasks = (out / "roles" / "container_images" / "tasks" / "main.yml").read_text( + encoding="utf-8" + ) + meta = (out / "roles" / "container_images" / "meta" / "main.yml").read_text( + encoding="utf-8" + ) + requirements = (out / "requirements.yml").read_text(encoding="utf-8") + playbook = (out / "playbook.yml").read_text(encoding="utf-8") + + assert "container_images:" in defaults + assert digest in defaults + assert podman_digest in defaults + assert "community.docker.docker_image_pull" in tasks + assert "community.docker.docker_image_tag" in tasks + assert "containers.podman.podman_image" in tasks + assert "containers.podman.podman_tag" in tasks + assert "repository:" in tasks + assert "target_names:" in tasks + assert "community.docker" in meta + assert "containers.podman" in meta + assert "name: community.docker" in requirements + assert "name: containers.podman" in requirements + assert "role: container_images" in playbook + + +def test_manifest_writes_container_images_to_hostvars_in_fqdn_mode(tmp_path: Path): + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + digest = "docker.io/library/nginx@sha256:" + "a" * 64 + state = { + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [], + "packages": [], + "container_images": { + "role_name": "container_images", + "images": [ + { + "engine": "docker", + "scope": "system", + "user": None, + "home": None, + "image_id": "sha256:" + "c" * 64, + "repo_tags": ["docker.io/library/nginx:1.27"], + "repo_digests": [digest], + "pull_ref": digest, + "tag_aliases": [], + "os": "linux", + "architecture": "amd64", + "variant": None, + "platform": "linux/amd64", + "size": 123, + "created": "2026-01-01T00:00:00Z", + "source": "docker image inspect", + "notes": [], + } + ], + "notes": [], + }, + } + } + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + manifest.manifest(str(bundle), str(out), fqdn="host.example.test") + + defaults = (out / "roles" / "container_images" / "defaults" / "main.yml").read_text( + encoding="utf-8" + ) + hostvars = ( + out / "inventory" / "host_vars" / "host.example.test" / "container_images.yml" + ).read_text(encoding="utf-8") + playbook = (out / "playbooks" / "host.example.test.yml").read_text(encoding="utf-8") + + assert "container_images: []" in defaults + assert digest in hostvars + assert "role: container_images" in playbook diff --git a/tests/test_manifest_ansible_model.py b/tests/test_manifest_ansible_model.py new file mode 100644 index 0000000..696cb2c --- /dev/null +++ b/tests/test_manifest_ansible_model.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from enroll.cm import CMModule +from enroll.ansible_renderer.model import AnsibleRole + + +def test_ansible_role_extends_cm_module_and_normalises_service_snapshot(): + role = AnsibleRole("network") + + role.add_service_snapshot( + { + "role_name": "networking", + "unit": "networking.service", + "packages": ["ifupdown"], + "active_state": "active", + "unit_file_state": "enabled", + "managed_dirs": [ + { + "path": "/etc/network", + "owner": "root", + "group": "root", + "mode": "0755", + } + ], + "managed_files": [ + { + "path": "/etc/network/interfaces", + "src_rel": "etc/network/interfaces", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "service_config", + } + ], + "managed_links": [ + { + "path": "/etc/systemd/system/multi-user.target.wants/networking.service", + "target": "/usr/lib/systemd/system/networking.service", + } + ], + "excluded": [{"path": "/etc/network/secrets", "reason": "secret"}], + "notes": ["captured for test"], + } + ) + + assert isinstance(role, CMModule) + assert role.sorted_packages == ["ifupdown"] + assert role.dirs["/etc/network"]["mode"] == "0755" + assert role.files["/etc/network/interfaces"]["src_rel"] == "etc/network/interfaces" + assert ( + role.links["/etc/systemd/system/multi-user.target.wants/networking.service"][ + "src" + ] + == "/usr/lib/systemd/system/networking.service" + ) + assert role.systemd_units_var == [ + { + "name": "networking.service", + "manage": True, + "enabled": True, + "state": "started", + } + ] + assert role.excluded == [{"path": "/etc/network/secrets", "reason": "secret"}] + assert role.notes == ["captured for test"] + assert "service `networking.service` from role `networking`" in role.origin_lines + + +def test_ansible_role_normalises_package_snapshot(): + role = AnsibleRole("admin") + role.add_package_snapshot( + { + "role_name": "curl", + "package": "curl", + "managed_files": [ + { + "path": "/etc/curlrc", + "src_rel": "etc/curlrc", + "owner": "root", + "group": "root", + "mode": "0644", + } + ], + } + ) + + assert isinstance(role, CMModule) + assert role.sorted_packages == ["curl"] + assert role.files["/etc/curlrc"]["dest"] == "/etc/curlrc" + assert role.services == {} + assert role.origin_lines == ["package `curl` from role `curl`"] diff --git a/tests/test_manifest_puppet.py b/tests/test_manifest_puppet.py new file mode 100644 index 0000000..b025827 --- /dev/null +++ b/tests/test_manifest_puppet.py @@ -0,0 +1,714 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import yaml + +from enroll import manifest + + +def _write_state(bundle: Path, state: dict) -> None: + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + + +def test_manifest_puppet_writes_control_repo_style_output(tmp_path: Path): + bundle = tmp_path / "bundle" + out = tmp_path / "puppet" + artifact = bundle / "artifacts" / "foo" / "etc" / "foo.conf" + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_text("setting = true\n", encoding="utf-8") + sysctl_artifact = bundle / "artifacts" / "sysctl" / "sysctl" / "99-enroll.conf" + sysctl_artifact.parent.mkdir(parents=True, exist_ok=True) + sysctl_artifact.write_text("net.ipv4.ip_forward = 1\n", encoding="utf-8") + + state = { + "schema_version": 3, + "host": {"hostname": "test", "os": "debian", "pkg_backend": "dpkg"}, + "inventory": {"packages": {}}, + "roles": { + "users": { + "role_name": "users", + "users": [ + { + "name": "alice", + "uid": 1000, + "gid": 1000, + "gecos": "Alice Example", + "home": "/home/alice", + "shell": "/bin/bash", + "primary_group": "alice", + "supplementary_groups": ["docker"], + } + ], + "managed_dirs": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [ + { + "unit": "foo.service", + "role_name": "foo", + "packages": ["foo"], + "active_state": "active", + "sub_state": "running", + "unit_file_state": "enabled", + "condition_result": "yes", + "managed_dirs": [ + { + "path": "/etc/foo", + "owner": "root", + "group": "root", + "mode": "0755", + "reason": "parent_dir", + } + ], + "managed_files": [ + { + "path": "/etc/foo/foo.conf", + "src_rel": "etc/foo.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "modified_conffile", + } + ], + "managed_links": [], + "excluded": [], + "notes": [], + } + ], + "packages": [ + { + "package": "curl", + "role_name": "curl", + "section": "net", + "managed_dirs": [], + "managed_files": [], + "managed_links": [], + "excluded": [], + "notes": [], + } + ], + "apt_config": { + "role_name": "apt_config", + "managed_dirs": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_dirs": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "sysctl": { + "role_name": "sysctl", + "managed_dirs": [], + "managed_files": [ + { + "path": "/etc/sysctl.d/99-enroll.conf", + "src_rel": "sysctl/99-enroll.conf", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "system_sysctl", + } + ], + "parameters": {"net.ipv4.ip_forward": "1"}, + "notes": [], + }, + "firewall_runtime": { + "role_name": "firewall_runtime", + "packages": [], + "ipset_save": None, + "ipset_sets": [], + "iptables_v4_save": None, + "iptables_v6_save": None, + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_dirs": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_dirs": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], + "managed_dirs": [], + "managed_files": [], + "managed_links": [], + "excluded": [], + "notes": [], + }, + }, + } + _write_state(bundle, state) + + manifest.manifest(str(bundle), str(out), target="puppet", fqdn="test.example") + + site_pp = (out / "manifests" / "site.pp").read_text(encoding="utf-8") + assert "node 'test.example' {" in site_pp + assert "lookup('enroll::classes'" in site_pp + assert "$enroll_classes.each" in site_pp + assert "include $enroll_class" in site_pp + assert "node default {" in site_pp + + assert (out / "hiera.yaml").exists() + node_data = yaml.safe_load( + (out / "data" / "nodes" / "test.example.yaml").read_text(encoding="utf-8") + ) + assert node_data["enroll::classes"] == ["curl", "foo", "users", "sysctl"] + assert node_data["curl::packages"] == ["curl"] + assert node_data["foo::packages"] == ["foo"] + assert node_data["foo::files"]["/etc/foo/foo.conf"]["source"] == ( + "puppet:///modules/foo/nodes/test.example/etc/foo.conf" + ) + assert node_data["foo::services"]["foo.service"] == { + "ensure": "running", + "enable": True, + } + assert node_data["users::users"]["alice"]["comment"] == "Alice Example" + assert node_data["users::users"]["alice"]["groups"] == ["docker"] + assert node_data["sysctl::files"]["/etc/sysctl.d/99-enroll.conf"]["source"] == ( + "puppet:///modules/sysctl/nodes/test.example/sysctl/99-enroll.conf" + ) + + curl_pp = (out / "modules" / "curl" / "manifests" / "init.pp").read_text( + encoding="utf-8" + ) + assert "class curl" in curl_pp + assert "Array[String] $packages = []" in curl_pp + assert "package { $package_name:" in curl_pp + assert "package { 'curl':" not in curl_pp + + foo_pp = (out / "modules" / "foo" / "manifests" / "init.pp").read_text( + encoding="utf-8" + ) + assert "class foo" in foo_pp + assert "Hash[String, Hash] $files = {}" in foo_pp + assert "* => $attrs" in foo_pp + assert "package { 'foo':" not in foo_pp + assert "file { '/etc/foo/foo.conf':" not in foo_pp + + users_pp = (out / "modules" / "users" / "manifests" / "init.pp").read_text( + encoding="utf-8" + ) + assert "class users" in users_pp + assert "Hash[String, Hash] $users = {}" in users_pp + assert "user { 'alice':" not in users_pp + + sysctl_pp = (out / "modules" / "sysctl" / "manifests" / "init.pp").read_text( + encoding="utf-8" + ) + assert "class sysctl" in sysctl_pp + assert "Boolean $sysctl_apply = true" in sysctl_pp + assert "Boolean $sysctl_ignore_apply_errors = true" in sysctl_pp + assert "exec { 'enroll-apply-sysctl':" in sysctl_pp + assert ( + "if $sysctl_apply and '/etc/sysctl.d/99-enroll.conf' in $files {" in sysctl_pp + ) + + assert ( + out + / "modules" + / "foo" + / "files" + / "nodes" + / "test.example" + / "etc" + / "foo.conf" + ).exists() + assert ( + out + / "modules" + / "sysctl" + / "files" + / "nodes" + / "test.example" + / "sysctl" + / "99-enroll.conf" + ).exists() + + +def test_manifest_puppet_fqdn_mode_can_accumulate_separate_node_data( + tmp_path: Path, +): + out = tmp_path / "puppet" + + def write_bundle(name: str, content: str) -> Path: + bundle = tmp_path / name + artifact = bundle / "artifacts" / "foo" / "etc" / "foo.conf" + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_text(content, encoding="utf-8") + _write_state( + bundle, + { + "schema_version": 3, + "host": {"hostname": name, "os": "debian", "pkg_backend": "dpkg"}, + "inventory": {"packages": {}}, + "roles": { + "services": [ + { + "unit": "foo.service", + "role_name": "foo", + "packages": ["foo"], + "active_state": "active", + "unit_file_state": "enabled", + "managed_dirs": [], + "managed_files": [ + { + "path": "/etc/foo/foo.conf", + "src_rel": "etc/foo.conf", + "owner": "root", + "group": "root", + "mode": "0644", + } + ], + "managed_links": [], + } + ], + "packages": [], + "users": { + "role_name": "users", + "users": [], + "managed_dirs": [], + "managed_files": [], + }, + "apt_config": { + "role_name": "apt_config", + "managed_dirs": [], + "managed_files": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_dirs": [], + "managed_files": [], + }, + "sysctl": { + "role_name": "sysctl", + "managed_dirs": [], + "managed_files": [], + }, + "firewall_runtime": { + "role_name": "firewall_runtime", + "packages": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_dirs": [], + "managed_files": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_dirs": [], + "managed_files": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "managed_dirs": [], + "managed_files": [], + "managed_links": [], + }, + }, + }, + ) + return bundle + + first = write_bundle("first", "first = true\n") + second = write_bundle("second", "second = true\n") + + manifest.manifest(str(first), str(out), target="puppet", fqdn="first.example") + manifest.manifest(str(second), str(out), target="puppet", fqdn="second.example") + + assert (out / "data" / "nodes" / "first.example.yaml").exists() + assert (out / "data" / "nodes" / "second.example.yaml").exists() + + site_pp = (out / "manifests" / "site.pp").read_text(encoding="utf-8") + assert "node 'first.example' {" in site_pp + assert "node 'second.example' {" in site_pp + + first_artifact = ( + out + / "modules" + / "foo" + / "files" + / "nodes" + / "first.example" + / "etc" + / "foo.conf" + ) + second_artifact = ( + out + / "modules" + / "foo" + / "files" + / "nodes" + / "second.example" + / "etc" + / "foo.conf" + ) + assert first_artifact.read_text(encoding="utf-8") == "first = true\n" + assert second_artifact.read_text(encoding="utf-8") == "second = true\n" + + first_data = yaml.safe_load( + (out / "data" / "nodes" / "first.example.yaml").read_text(encoding="utf-8") + ) + second_data = yaml.safe_load( + (out / "data" / "nodes" / "second.example.yaml").read_text(encoding="utf-8") + ) + assert first_data["foo::files"]["/etc/foo/foo.conf"]["source"] == ( + "puppet:///modules/foo/nodes/first.example/etc/foo.conf" + ) + assert second_data["foo::files"]["/etc/foo/foo.conf"]["source"] == ( + "puppet:///modules/foo/nodes/second.example/etc/foo.conf" + ) + + +def test_manifest_puppet_uses_default_node_and_common_package_modules(tmp_path: Path): + bundle = tmp_path / "bundle" + out = tmp_path / "puppet" + artifact = bundle / "artifacts" / "foo" / "etc" / "foo.conf" + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_text("setting = true\n", encoding="utf-8") + + state = { + "schema_version": 3, + "host": {"hostname": "test", "os": "debian", "pkg_backend": "dpkg"}, + "inventory": { + "packages": { + "curl": {"section": "net"}, + "foo": {"installations": [{"section": "net"}]}, + } + }, + "roles": { + "services": [ + { + "unit": "foo.service", + "role_name": "foo", + "packages": ["foo"], + "active_state": "active", + "unit_file_state": "enabled", + "managed_dirs": [], + "managed_files": [ + { + "path": "/etc/foo/foo.conf", + "src_rel": "etc/foo.conf", + "owner": "root", + "group": "root", + "mode": "0644", + } + ], + "managed_links": [], + } + ], + "packages": [ + { + "package": "curl", + "role_name": "curl", + "section": "net", + "managed_dirs": [], + "managed_files": [], + "managed_links": [], + } + ], + "users": { + "role_name": "users", + "users": [], + "managed_dirs": [], + "managed_files": [], + }, + "apt_config": { + "role_name": "apt_config", + "managed_dirs": [], + "managed_files": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_dirs": [], + "managed_files": [], + }, + "sysctl": {"role_name": "sysctl", "managed_dirs": [], "managed_files": []}, + "firewall_runtime": {"role_name": "firewall_runtime", "packages": []}, + "etc_custom": { + "role_name": "etc_custom", + "managed_dirs": [], + "managed_files": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_dirs": [], + "managed_files": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "managed_dirs": [], + "managed_files": [], + "managed_links": [], + }, + }, + } + _write_state(bundle, state) + + manifest.manifest(str(bundle), str(out), target="puppet") + + site_pp = (out / "manifests" / "site.pp").read_text(encoding="utf-8") + assert site_pp == "node default {\n include net\n}\n" + + net_pp = (out / "modules" / "net" / "manifests" / "init.pp").read_text( + encoding="utf-8" + ) + assert "class net" in net_pp + assert "package { 'curl':" in net_pp + assert "package { 'foo':" in net_pp + assert "file { '/etc/foo/foo.conf':" in net_pp + assert "source => 'puppet:///modules/net/etc/foo.conf'" in net_pp + assert "service { 'foo.service':" in net_pp + assert (out / "modules" / "net" / "files" / "etc" / "foo.conf").exists() + assert not (out / "modules" / "curl").exists() + assert not (out / "modules" / "foo").exists() + + +def test_manifest_puppet_avoids_reserved_module_names_and_duplicate_resources( + tmp_path: Path, +): + bundle = tmp_path / "bundle" + out = tmp_path / "puppet" + state = { + "schema_version": 3, + "host": {"hostname": "test", "os": "debian", "pkg_backend": "dpkg"}, + "inventory": { + "packages": { + "alpha": {"section": "admin"}, + "beta": {"section": "misc"}, + "gamma": {"section": "default"}, + } + }, + "roles": { + "packages": [ + { + "package": "alpha", + "role_name": "alpha", + "section": "admin", + "managed_dirs": [ + { + "path": "/etc/default", + "owner": "root", + "group": "root", + "mode": "0755", + } + ], + "managed_files": [], + "managed_links": [], + }, + { + "package": "beta", + "role_name": "beta", + "section": "misc", + "managed_dirs": [ + { + "path": "/etc/default", + "owner": "root", + "group": "root", + "mode": "0755", + } + ], + "managed_files": [], + "managed_links": [], + }, + { + "package": "gamma", + "role_name": "gamma", + "section": "default", + "managed_dirs": [], + "managed_files": [], + "managed_links": [], + }, + ], + "users": { + "role_name": "users", + "users": [], + "managed_dirs": [], + "managed_files": [], + }, + "apt_config": { + "role_name": "apt_config", + "managed_dirs": [], + "managed_files": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_dirs": [], + "managed_files": [], + }, + "sysctl": {"role_name": "sysctl", "managed_dirs": [], "managed_files": []}, + "firewall_runtime": {"role_name": "firewall_runtime", "packages": []}, + "etc_custom": { + "role_name": "etc_custom", + "managed_dirs": [], + "managed_files": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_dirs": [], + "managed_files": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "managed_dirs": [], + "managed_files": [], + "managed_links": [], + }, + }, + } + _write_state(bundle, state) + + manifest.manifest(str(bundle), str(out), target="puppet") + + site_pp = (out / "manifests" / "site.pp").read_text(encoding="utf-8") + assert "include default\n" not in site_pp + assert "include package_group_default" in site_pp + assert ( + out / "modules" / "package_group_default" / "manifests" / "init.pp" + ).exists() + + init_pps = "\n".join( + p.read_text(encoding="utf-8") + for p in sorted((out / "modules").glob("*/manifests/init.pp")) + ) + assert init_pps.count("file { '/etc/default':") == 1 + + +def test_manifest_rejects_unknown_target(tmp_path: Path): + bundle = tmp_path / "bundle" + _write_state(bundle, {"roles": {}}) + + try: + manifest.manifest(str(bundle), str(tmp_path / "out"), target="chef") + except ValueError as e: + assert "unsupported manifest target" in str(e) + else: + raise AssertionError("expected ValueError") + + +def test_manifest_puppet_renders_container_images_static_and_hiera(tmp_path: Path): + digest = "docker.io/library/nginx@sha256:" + "a" * 64 + podman_digest = "quay.io/example/app@sha256:" + "b" * 64 + state = { + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_dirs": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [], + "packages": [], + "container_images": { + "role_name": "container_images", + "images": [ + { + "engine": "docker", + "scope": "system", + "user": None, + "home": None, + "image_id": "sha256:" + "c" * 64, + "repo_tags": ["docker.io/library/nginx:1.27"], + "repo_digests": [digest], + "pull_ref": digest, + "tag_aliases": [ + { + "ref": "docker.io/library/nginx:1.27", + "repository": "docker.io/library/nginx", + "tag": "1.27", + } + ], + "os": "linux", + "architecture": "amd64", + "variant": None, + "platform": "linux/amd64", + "size": 123, + "created": "2026-01-01T00:00:00Z", + "source": "docker image inspect", + "notes": [], + }, + { + "engine": "podman", + "scope": "system", + "user": None, + "home": None, + "image_id": "sha256:" + "d" * 64, + "repo_tags": ["quay.io/example/app:prod"], + "repo_digests": [podman_digest], + "pull_ref": podman_digest, + "tag_aliases": [], + "os": "linux", + "architecture": "amd64", + "variant": None, + "platform": "linux/amd64", + "size": 456, + "created": "2026-01-01T00:00:00Z", + "source": "podman image inspect", + "notes": [], + }, + ], + "notes": [], + }, + } + } + bundle = tmp_path / "bundle" + out = tmp_path / "puppet" + _write_state(bundle, state) + + manifest.manifest(str(bundle), str(out), target="puppet") + + site_pp = (out / "manifests" / "site.pp").read_text(encoding="utf-8") + assert "include container_images" in site_pp + pp = (out / "modules" / "container_images" / "manifests" / "init.pp").read_text( + encoding="utf-8" + ) + assert "docker::image" in pp + assert "image_digest => 'sha256:" + "a" * 64 + "'" in pp + assert "docker tag" in pp + assert "podman pull" in pp + metadata = json.loads( + (out / "modules" / "container_images" / "metadata.json").read_text( + encoding="utf-8" + ) + ) + assert metadata["dependencies"] == [ + {"name": "puppetlabs-docker", "version_requirement": ">= 8.0.0 < 15.0.0"} + ] + + fqdn_out = tmp_path / "puppet-fqdn" + manifest.manifest(str(bundle), str(fqdn_out), target="puppet", fqdn="node.example") + node_data = yaml.safe_load( + (fqdn_out / "data" / "nodes" / "node.example.yaml").read_text(encoding="utf-8") + ) + assert node_data["container_images::container_images"][0]["pull_ref"] == digest + fqdn_pp = ( + fqdn_out / "modules" / "container_images" / "manifests" / "init.pp" + ).read_text(encoding="utf-8") + assert "Array[Hash] $container_images = []" in fqdn_pp + assert "docker::image" in fqdn_pp + assert "enroll-podman-pull-${idx}" in fqdn_pp + assert "$image['pull_cmd']" in fqdn_pp + assert "podman pull" in ( + fqdn_out / "data" / "nodes" / "node.example.yaml" + ).read_text(encoding="utf-8") diff --git a/tests/test_manifest_symlinks.py b/tests/test_manifest_symlinks.py new file mode 100644 index 0000000..39ef9a0 --- /dev/null +++ b/tests/test_manifest_symlinks.py @@ -0,0 +1,108 @@ +import json +from pathlib import Path + +import enroll.manifest as manifest + + +def test_manifest_emits_symlink_tasks_and_vars(tmp_path: Path): + bundle = tmp_path / "bundle" + out = tmp_path / "ansible" + + state = { + "host": {"hostname": "test", "os": "debian", "pkg_backend": "dpkg"}, + "inventory": { + "packages": { + "nginx": { + "version": "1.0", + "arches": ["amd64"], + "installations": [ + {"version": "1.0", "arch": "amd64", "section": "httpd"} + ], + "section": "httpd", + "observed_via": [{"kind": "systemd_unit", "ref": "nginx.service"}], + "roles": ["nginx"], + } + } + }, + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [ + { + "unit": "nginx.service", + "role_name": "nginx", + "packages": ["nginx"], + "active_state": "active", + "sub_state": "running", + "unit_file_state": "enabled", + "condition_result": None, + "managed_files": [], + "managed_links": [ + { + "path": "/etc/nginx/sites-enabled/default", + "target": "../sites-available/default", + "reason": "enabled_symlink", + } + ], + "excluded": [], + "notes": [], + } + ], + "packages": [], + "apt_config": { + "role_name": "apt_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_files": [], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], + "managed_files": [], + "managed_links": [], + "excluded": [], + "notes": [], + }, + }, + } + + bundle.mkdir(parents=True, exist_ok=True) + (bundle / "artifacts").mkdir(parents=True, exist_ok=True) + (bundle / "state.json").write_text(json.dumps(state), encoding="utf-8") + + manifest.manifest(str(bundle), str(out)) + + role_dir = out / "roles" / "httpd" + tasks = (role_dir / "tasks" / "main.yml").read_text(encoding="utf-8") + assert "- name: Ensure managed symlinks exist" in tasks + assert 'loop: "{{ httpd_managed_links | default([]) }}"' in tasks + + defaults = (role_dir / "defaults" / "main.yml").read_text(encoding="utf-8") + # The grouped role defaults should include the converted link mapping. + assert "httpd_managed_links:" in defaults + assert "dest: /etc/nginx/sites-enabled/default" in defaults + assert "src: ../sites-available/default" in defaults diff --git a/tests/test_pathfilter.py b/tests/test_pathfilter.py new file mode 100644 index 0000000..08e4ae4 --- /dev/null +++ b/tests/test_pathfilter.py @@ -0,0 +1,340 @@ +from __future__ import annotations + +import os +from pathlib import Path + +import enroll.pathfilter as pf + + +def test_compile_and_match_prefix_glob_and_regex(tmp_path: Path): + from enroll.pathfilter import PathFilter, compile_path_pattern + + # prefix semantics: matches the exact path and subtree + p = compile_path_pattern("/etc/nginx") + assert p.kind == "prefix" + assert p.matches("/etc/nginx") + assert p.matches("/etc/nginx/nginx.conf") + assert not p.matches("/etc/nginx2/nginx.conf") + + # glob semantics + g = compile_path_pattern("/etc/**/*.conf") + assert g.kind == "glob" + assert g.matches("/etc/nginx/nginx.conf") + assert not g.matches("/var/etc/nginx.conf") + + # explicit glob + g2 = compile_path_pattern("glob:/home/*/.bashrc") + assert g2.kind == "glob" + assert g2.matches("/home/alice/.bashrc") + + # regex semantics (search, not match) + r = compile_path_pattern(r"re:/home/[^/]+/\.ssh/authorized_keys$") + assert r.kind == "regex" + assert r.matches("/home/alice/.ssh/authorized_keys") + assert not r.matches("/home/alice/.ssh/authorized_keys2") + + # invalid regex: never matches + bad = compile_path_pattern("re:[") + assert bad.kind == "regex" + assert not bad.matches("/etc/passwd") + + # exclude wins + pf = PathFilter(exclude=["/etc/nginx"], include=["/etc/nginx/nginx.conf"]) + assert pf.is_excluded("/etc/nginx/nginx.conf") + + +def test_expand_includes_respects_exclude_symlinks_and_caps(tmp_path: Path): + from enroll.pathfilter import PathFilter, compile_path_pattern, expand_includes + + root = tmp_path / "root" + (root / "a").mkdir(parents=True) + (root / "a" / "one.txt").write_text("1", encoding="utf-8") + (root / "a" / "two.txt").write_text("2", encoding="utf-8") + (root / "b").mkdir() + (root / "b" / "secret.txt").write_text("s", encoding="utf-8") + + # symlink file should be ignored + os.symlink(str(root / "a" / "one.txt"), str(root / "a" / "link.txt")) + + exclude = PathFilter(exclude=[str(root / "b")]) + + pats = [ + compile_path_pattern(str(root / "a")), + compile_path_pattern("glob:" + str(root / "**" / "*.txt")), + ] + + paths, notes = expand_includes(pats, exclude=exclude, max_files=2) + # cap should limit to 2 files + assert len(paths) == 2 + assert any("cap" in n.lower() for n in notes) + # excluded dir should not contribute + assert all("/b/" not in p for p in paths) + # symlink ignored + assert all(not p.endswith("link.txt") for p in paths) + + +def test_expand_includes_notes_on_no_matches(tmp_path: Path): + from enroll.pathfilter import compile_path_pattern, expand_includes + + pats = [compile_path_pattern(str(tmp_path / "does_not_exist"))] + paths, notes = expand_includes(pats, max_files=10) + assert paths == [] + assert any("matched no files" in n.lower() for n in notes) + + +def test_expand_includes_supports_regex_with_inferred_root(tmp_path: Path): + """Regex includes are expanded by walking an inferred literal prefix root.""" + from enroll.pathfilter import compile_path_pattern, expand_includes + + root = tmp_path / "root" + (root / "home" / "alice" / ".config" / "myapp").mkdir(parents=True) + target = root / "home" / "alice" / ".config" / "myapp" / "settings.ini" + target.write_text("x=1\n", encoding="utf-8") + + # This is anchored and begins with an absolute path, so expand_includes should + # infer a narrow walk root instead of scanning '/'. + rex = rf"re:^{root}/home/[^/]+/\.config/myapp/.*$" + pat = compile_path_pattern(rex) + paths, notes = expand_includes([pat], max_files=10) + assert str(target) in paths + assert notes == [] + + +def test_compile_path_pattern_normalises_relative_prefix(): + from enroll.pathfilter import compile_path_pattern + + p = compile_path_pattern("etc/ssh") + assert p.kind == "prefix" + assert p.value == "/etc/ssh" + + +def test_norm_abs_empty_string_is_root(): + assert pf._norm_abs("") == "/" + + +def test_posix_match_invalid_pattern_fails_closed(monkeypatch): + # Force PurePosixPath.match to raise to cover the exception handler. + real_match = pf.PurePosixPath.match + + def boom(self, pat): + raise ValueError("bad pattern") + + monkeypatch.setattr(pf.PurePosixPath, "match", boom) + try: + assert pf._posix_match("/etc/hosts", "[bad") is False + finally: + monkeypatch.setattr(pf.PurePosixPath, "match", real_match) + + +def test_regex_literal_prefix_handles_escapes(): + # Prefix stops at meta chars but includes escaped literals. + assert pf._regex_literal_prefix(r"^/etc/\./foo") == "/etc/./foo" + + +def test_expand_includes_maybe_add_file_skips_non_files(monkeypatch, tmp_path: Path): + # Drive the _maybe_add_file branch that rejects symlinks/non-files. + pats = [pf.compile_path_pattern(str(tmp_path / "missing"))] + + monkeypatch.setattr(pf.os.path, "isfile", lambda p: False) + monkeypatch.setattr(pf.os.path, "islink", lambda p: False) + monkeypatch.setattr(pf.os.path, "isdir", lambda p: False) + + paths, notes = pf.expand_includes(pats, max_files=10) + assert paths == [] + assert any("matched no files" in n for n in notes) + + +def test_expand_includes_prunes_excluded_dirs(monkeypatch): + include = [pf.compile_path_pattern("/root/**")] + exclude = pf.PathFilter(exclude=["/root/skip/**"]) + + # Simulate filesystem walk: + # /root has dirnames ['skip', 'keep'] but skip should be pruned. + monkeypatch.setattr( + pf.os.path, + "isdir", + lambda p: p in {"/root", "/root/keep", "/root/skip"}, + ) + monkeypatch.setattr(pf.os.path, "islink", lambda p: False) + monkeypatch.setattr(pf.os.path, "isfile", lambda p: True) + + def walk(root, followlinks=False): + assert root == "/root" + yield ("/root", ["skip", "keep"], []) + yield ("/root/keep", [], ["a.txt"]) + # If pruning works, we should never walk into /root/skip. + + monkeypatch.setattr(pf.os, "walk", walk) + + paths, _notes = pf.expand_includes(include, exclude=exclude, max_files=10) + assert "/root/keep/a.txt" in paths + assert not any(p.startswith("/root/skip") for p in paths) + + +def test_expand_includes_respects_max_files(monkeypatch): + include = [pf.compile_path_pattern("/root/**")] + monkeypatch.setattr(pf.os.path, "isdir", lambda p: p == "/root") + monkeypatch.setattr(pf.os.path, "islink", lambda p: False) + monkeypatch.setattr(pf.os.path, "isfile", lambda p: True) + monkeypatch.setattr( + pf.os, + "walk", + lambda root, followlinks=False: [("/root", [], ["a", "b", "c"])], + ) + paths, notes = pf.expand_includes(include, max_files=2) + assert len(paths) == 2 + assert "/root/c" not in paths + + +def test_has_glob_chars(): + assert pf._has_glob_chars("*.txt") is True + assert pf._has_glob_chars("file?.log") is True + assert pf._has_glob_chars("[abc]") is True + assert pf._has_glob_chars("file.txt") is False + assert pf._has_glob_chars("") is False + + +def test_compile_path_pattern_regex_valid(): + result = pf.compile_path_pattern("re:^/home/.*$") + assert result.kind == "regex" + assert result.regex is not None + assert result.regex.search("/home/user/file.txt") is not None + assert result.regex.search("/var/file.txt") is None + + +def test_compile_path_pattern_glob_forced(): + result = pf.compile_path_pattern("glob:/etc/*.conf") + assert result.kind == "glob" + assert result.value == "/etc/*.conf" + + +def test_compile_path_pattern_glob_heuristic(): + result = pf.compile_path_pattern("/etc/*.conf") + assert result.kind == "glob" + + +def test_compile_path_pattern_prefix(): + result = pf.compile_path_pattern("/etc/nginx") + assert result.kind == "prefix" + assert result.value == "/etc/nginx" + + +def test_compiled_pattern_matches_prefix(): + pat = pf.compile_path_pattern("/etc/nginx") + assert pat.matches("/etc/nginx") is True + assert pat.matches("/etc/nginx/conf.d") is True + assert pat.matches("/etc/ssh") is False + + +def test_compiled_pattern_matches_glob(): + pat = pf.compile_path_pattern("/etc/*.conf") + assert pat.matches("/etc/ssh.conf") is True + assert pat.matches("/etc/ssh/sshd.conf") is False + + +def test_compiled_pattern_matches_regex(): + pat = pf.compile_path_pattern("re:^/home/[^/]+/.bashrc$") + assert pat.matches("/home/alice/.bashrc") is True + assert pat.matches("/home/bob/.bashrc") is True + assert pat.matches("/home/alice/.profile") is False + assert pat.matches("/var/.bashrc") is False + + +def test_path_filter_is_excluded(): + pf_filter = pf.PathFilter(exclude=["/tmp/*", "/var/log"]) + assert pf_filter.is_excluded("/tmp/file.txt") is True + assert pf_filter.is_excluded("/var/log/syslog") is True + assert pf_filter.is_excluded("/etc/ssh") is False + + +def test_path_filter_empty(): + pf_filter = pf.PathFilter() + assert pf_filter.is_excluded("/anything") is False + assert pf_filter.iter_include_patterns() == [] + + +def test_expand_includes_prefix_existing(tmp_path: Path): + etc_dir = tmp_path / "etc" + etc_dir.mkdir() + (etc_dir / "file1.txt").write_text("a") + (etc_dir / "file2.txt").write_text("b") + + patterns = [pf.compile_path_pattern(str(etc_dir))] + paths, notes = pf.expand_includes(patterns, max_files=10) + + assert len(paths) == 2 + assert notes == [] + + +def test_expand_includes_prefix_nonexistent(): + patterns = [pf.compile_path_pattern("/nonexistent/path")] + paths, notes = pf.expand_includes(patterns, max_files=10) + + assert paths == [] + assert len(notes) == 1 + assert "matched no files" in notes[0] + + +def test_expand_includes_glob_no_matches(): + patterns = [pf.compile_path_pattern("/nonexistent/*.txt")] + paths, notes = pf.expand_includes(patterns, max_files=10) + + assert paths == [] + assert len(notes) == 1 + + +def test_expand_includes_skips_symlinks(tmp_path: Path): + real_file = tmp_path / "real.txt" + real_file.write_text("x") + link = tmp_path / "link.txt" + os.symlink(str(real_file), str(link)) + + patterns = [pf.compile_path_pattern(str(tmp_path))] + paths, notes = pf.expand_includes(patterns, max_files=10) + + assert len(paths) == 1 + assert paths[0].endswith("real.txt") + + +def test_expand_includes_excludes_pattern(tmp_path: Path): + etc_dir = tmp_path / "etc" + etc_dir.mkdir() + (etc_dir / "include.txt").write_text("a") + (etc_dir / "exclude.txt").write_text("b") + + patterns = [pf.compile_path_pattern(str(etc_dir))] + exclude = pf.PathFilter(exclude=["*exclude*"]) + paths, notes = pf.expand_includes(patterns, exclude=exclude, max_files=10) + + assert len(paths) == 1 + assert paths[0].endswith("include.txt") + + +def test_expand_includes_skips_directories(tmp_path: Path): + subdir = tmp_path / "subdir" + subdir.mkdir() + (tmp_path / "file.txt").write_text("x") + + patterns = [pf.compile_path_pattern(str(subdir))] + paths, notes = pf.expand_includes(patterns, max_files=10) + + assert paths == [] + + +def test_regex_literal_prefix_simple(): + assert pf._regex_literal_prefix("/etc/nginx/") == "/etc/nginx/" + + +def test_regex_literal_prefix_with_anchor(): + assert pf._regex_literal_prefix("^/etc/nginx/") == "/etc/nginx/" + + +def test_regex_literal_prefix_with_regex_chars(): + assert pf._regex_literal_prefix("^/etc/.*\\.conf$") == "/etc/" + + +def test_path_filter_with_include_patterns(): + pf_filter = pf.PathFilter(include=["/etc/*.conf"], exclude=["/etc/secret.conf"]) + patterns = pf_filter.iter_include_patterns() + assert len(patterns) == 1 + assert patterns[0].kind == "glob" diff --git a/tests/test_platform.py b/tests/test_platform.py new file mode 100644 index 0000000..c0c6b46 --- /dev/null +++ b/tests/test_platform.py @@ -0,0 +1,266 @@ +from __future__ import annotations + +from pathlib import Path + +import enroll.platform as platform + + +def test_read_os_release_parses_kv_and_strips_quotes(tmp_path: Path): + p = tmp_path / "os-release" + p.write_text( + """ +# comment +ID=fedora +ID_LIKE=\"rhel centos\" +NAME=\"Fedora Linux\" +EMPTY= +NOEQUALS +""", + encoding="utf-8", + ) + + osr = platform._read_os_release(str(p)) + assert osr["ID"] == "fedora" + assert osr["ID_LIKE"] == "rhel centos" + assert osr["NAME"] == "Fedora Linux" + assert osr["EMPTY"] == "" + assert "NOEQUALS" not in osr + + +def test_detect_platform_prefers_os_release(monkeypatch): + monkeypatch.setattr( + platform, + "_read_os_release", + lambda path="/etc/os-release": {"ID": "fedora", "ID_LIKE": "rhel"}, + ) + # If os-release is decisive we shouldn't need which() + monkeypatch.setattr(platform.shutil, "which", lambda exe: None) + + info = platform.detect_platform() + assert info.os_family == "redhat" + assert info.pkg_backend == "rpm" + + +def test_detect_platform_fallbacks_to_dpkg_when_unknown(monkeypatch): + monkeypatch.setattr(platform, "_read_os_release", lambda path="/etc/os-release": {}) + monkeypatch.setattr( + platform.shutil, "which", lambda exe: "/usr/bin/dpkg" if exe == "dpkg" else None + ) + + info = platform.detect_platform() + assert info.os_family == "debian" + assert info.pkg_backend == "dpkg" + + +def test_get_backend_unknown_prefers_rpm_if_present(monkeypatch): + monkeypatch.setattr( + platform.shutil, "which", lambda exe: "/usr/bin/rpm" if exe == "rpm" else None + ) + + b = platform.get_backend( + platform.PlatformInfo(os_family="unknown", pkg_backend="unknown", os_release={}) + ) + assert isinstance(b, platform.RpmBackend) + + +def test_rpm_backend_modified_paths_labels_conffiles(monkeypatch): + b = platform.RpmBackend() + + # Pretend rpm -V says both files changed, but only one is a config file. + monkeypatch.setattr(b, "_modified_files", lambda pkg: {"/etc/foo.conf", "/etc/bar"}) + monkeypatch.setattr(b, "_config_files", lambda pkg: {"/etc/foo.conf"}) + + out = b.modified_paths("mypkg", ["/etc/foo.conf", "/etc/bar", "/etc/dnf/dnf.conf"]) + assert out["/etc/foo.conf"] == "modified_conffile" + assert out["/etc/bar"] == "modified_packaged_file" + # Package-manager config paths are excluded. + assert "/etc/dnf/dnf.conf" not in out + + +def test_specific_paths_for_hints_differs_between_backends(): + # We can exercise this without instantiating DpkgBackend (which reads dpkg status) + class Dummy(platform.PackageBackend): + name = "dummy" + pkg_config_prefixes = ("/etc/apt/",) + + d = Dummy() + assert d.is_pkg_config_path("/etc/apt/sources.list") + assert not d.is_pkg_config_path("/etc/ssh/sshd_config") + + r = platform.RpmBackend() + paths = set(r.specific_paths_for_hints({"nginx"})) + assert "/etc/sysconfig/nginx" in paths + assert "/etc/sysconfig/nginx.conf" in paths + + +def test_read_os_release_file_not_found(tmp_path: Path): + result = platform._read_os_release(str(tmp_path / "nonexistent")) + assert result == {} + + +def test_read_os_release_handles_invalid_line(tmp_path: Path): + p = tmp_path / "os-release" + p.write_text( + "ID=ubuntu\n" "NO_EQUALS_SIGN\n" 'VERSION="22.04"\n', + encoding="utf-8", + ) + result = platform._read_os_release(str(p)) + assert result["ID"] == "ubuntu" + assert result["VERSION"] == "22.04" + assert "NO_EQUALS_SIGN" not in result + + +def test_detect_platform_debian(monkeypatch): + def fake_read_os_release(path: str = "/etc/os-release") -> dict: + return {"ID": "debian", "VERSION_ID": "11"} + + monkeypatch.setattr(platform, "_read_os_release", fake_read_os_release) + result = platform.detect_platform() + assert result.os_family == "debian" + assert result.pkg_backend == "dpkg" + + +def test_detect_platform_ubuntu(monkeypatch): + def fake_read_os_release(path: str = "/etc/os-release") -> dict: + return {"ID": "ubuntu", "VERSION_ID": "22.04"} + + monkeypatch.setattr(platform, "_read_os_release", fake_read_os_release) + result = platform.detect_platform() + assert result.os_family == "debian" + assert result.pkg_backend == "dpkg" + + +def test_detect_platform_fedora(monkeypatch): + def fake_read_os_release(path: str = "/etc/os-release") -> dict: + return {"ID": "fedora", "VERSION_ID": "38"} + + monkeypatch.setattr(platform, "_read_os_release", fake_read_os_release) + result = platform.detect_platform() + assert result.os_family == "redhat" + assert result.pkg_backend == "rpm" + + +def test_detect_platform_rocky(monkeypatch): + def fake_read_os_release(path: str = "/etc/os-release") -> dict: + return {"ID": "rocky", "VERSION_ID": "9"} + + monkeypatch.setattr(platform, "_read_os_release", fake_read_os_release) + result = platform.detect_platform() + assert result.os_family == "redhat" + assert result.pkg_backend == "rpm" + + +def test_detect_platform_unknown_fallback_to_dpkg(monkeypatch): + def fake_read_os_release(path: str = "/etc/os-release") -> dict: + return {"ID": "unknown"} + + monkeypatch.setattr(platform, "_read_os_release", fake_read_os_release) + monkeypatch.setattr(platform.shutil, "which", lambda x: x == "dpkg") + result = platform.detect_platform() + assert result.os_family == "debian" + assert result.pkg_backend == "dpkg" + + +def test_detect_platform_unknown_fallback_to_rpm(monkeypatch): + def fake_read_os_release(path: str = "/etc/os-release") -> dict: + return {"ID": "unknown"} + + monkeypatch.setattr(platform, "_read_os_release", fake_read_os_release) + monkeypatch.setattr(platform.shutil, "which", lambda x: x == "rpm") + result = platform.detect_platform() + assert result.os_family == "redhat" + assert result.pkg_backend == "rpm" + + +def test_detect_platform_completely_unknown(monkeypatch): + def fake_read_os_release(path: str = "/etc/os-release") -> dict: + return {"ID": "unknown"} + + monkeypatch.setattr(platform, "_read_os_release", fake_read_os_release) + monkeypatch.setattr(platform.shutil, "which", lambda x: False) + result = platform.detect_platform() + assert result.os_family == "unknown" + assert result.pkg_backend == "unknown" + + +def test_detect_platform_debian_like(monkeypatch): + def fake_read_os_release(path: str = "/etc/os-release") -> dict: + return {"ID": "linuxmint", "ID_LIKE": "debian"} + + monkeypatch.setattr(platform, "_read_os_release", fake_read_os_release) + result = platform.detect_platform() + assert result.os_family == "debian" + assert result.pkg_backend == "dpkg" + + +def test_detect_platform_rhel_like(monkeypatch): + def fake_read_os_release(path: str = "/etc/os-release") -> dict: + return {"ID": "centos", "ID_LIKE": "rhel fedora"} + + monkeypatch.setattr(platform, "_read_os_release", fake_read_os_release) + result = platform.detect_platform() + assert result.os_family == "redhat" + assert result.pkg_backend == "rpm" + + +def test_get_backend_returns_dpkg(monkeypatch): + info = platform.PlatformInfo(os_family="debian", pkg_backend="dpkg", os_release={}) + backend = platform.get_backend(info) + assert isinstance(backend, platform.DpkgBackend) + assert backend.name == "dpkg" + + +def test_get_backend_returns_rpm(monkeypatch): + info = platform.PlatformInfo(os_family="redhat", pkg_backend="rpm", os_release={}) + backend = platform.get_backend(info) + assert isinstance(backend, platform.RpmBackend) + assert backend.name == "rpm" + + +def test_get_backend_unknown_with_rpm(monkeypatch): + info = platform.PlatformInfo( + os_family="unknown", pkg_backend="unknown", os_release={} + ) + monkeypatch.setattr(platform.shutil, "which", lambda x: x == "rpm") + backend = platform.get_backend(info) + assert isinstance(backend, platform.RpmBackend) + + +def test_get_backend_unknown_with_dpkg(monkeypatch): + info = platform.PlatformInfo( + os_family="unknown", pkg_backend="unknown", os_release={} + ) + monkeypatch.setattr(platform.shutil, "which", lambda x: x == "dpkg") + backend = platform.get_backend(info) + assert isinstance(backend, platform.DpkgBackend) + + +def test_dpkg_backend_specific_paths(): + backend = platform.DpkgBackend() + paths = backend.specific_paths_for_hints({"nginx"}) + assert "/etc/default/nginx" in paths + assert "/etc/init.d/nginx" in paths + assert "/etc/sysctl.d/nginx.conf" in paths + + +def test_rpm_backend_specific_paths(): + backend = platform.RpmBackend() + paths = backend.specific_paths_for_hints({"nginx"}) + assert "/etc/sysconfig/nginx" in paths + assert "/etc/sysconfig/nginx.conf" in paths + assert "/etc/sysctl.d/nginx.conf" in paths + + +def test_is_pkg_config_path_dpkg(): + backend = platform.DpkgBackend() + assert backend.is_pkg_config_path("/etc/apt/sources.list") is True + assert backend.is_pkg_config_path("/etc/apt/trusted.gpg") is True + assert backend.is_pkg_config_path("/etc/ssh/sshd_config") is False + + +def test_is_pkg_config_path_rpm(): + backend = platform.RpmBackend() + assert backend.is_pkg_config_path("/etc/dnf/dnf.conf") is True + assert backend.is_pkg_config_path("/etc/yum.conf") is True + assert backend.is_pkg_config_path("/etc/yum.repos.d/custom.repo") is True + assert backend.is_pkg_config_path("/etc/ssh/sshd_config") is False diff --git a/tests/test_platform_backends.py b/tests/test_platform_backends.py new file mode 100644 index 0000000..6716d53 --- /dev/null +++ b/tests/test_platform_backends.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from collections import defaultdict + + +def test_dpkg_backend_modified_paths_marks_conffiles_and_packaged(monkeypatch): + from enroll.platform import DpkgBackend + + # Provide fake conffiles md5sums. + monkeypatch.setattr( + "enroll.debian.parse_status_conffiles", + lambda: {"mypkg": {"/etc/mypkg.conf": "aaaa"}}, + ) + monkeypatch.setattr( + "enroll.debian.read_pkg_md5sums", + lambda _pkg: {"etc/other.conf": "bbbb"}, + ) + + # Fake file_md5 values (avoids touching /etc). + def fake_md5(p: str): + if p == "/etc/mypkg.conf": + return "zzzz" # differs from conffile baseline + if p == "/etc/other.conf": + return "cccc" # differs from packaged baseline + if p == "/etc/apt/sources.list": + return "bbbb" + return None + + monkeypatch.setattr("enroll.platform.file_md5", fake_md5) + + b = DpkgBackend() + out = b.modified_paths( + "mypkg", + ["/etc/mypkg.conf", "/etc/other.conf", "/etc/apt/sources.list"], + ) + + assert out["/etc/mypkg.conf"] == "modified_conffile" + assert out["/etc/other.conf"] == "modified_packaged_file" + # pkg config paths (like /etc/apt/...) are excluded. + assert "/etc/apt/sources.list" not in out + + +def test_rpm_backend_modified_paths_caches_queries(monkeypatch): + from enroll.platform import RpmBackend + + calls = defaultdict(int) + + def fake_modified(_pkg=None): + calls["modified"] += 1 + return {"/etc/foo.conf", "/etc/bar.conf"} + + def fake_config(_pkg=None): + calls["config"] += 1 + return {"/etc/foo.conf"} + + monkeypatch.setattr("enroll.rpm.rpm_modified_files", fake_modified) + monkeypatch.setattr("enroll.rpm.rpm_config_files", fake_config) + + b = RpmBackend() + etc = ["/etc/foo.conf", "/etc/bar.conf", "/etc/baz.conf"] + + out1 = b.modified_paths("ignored", etc) + out2 = b.modified_paths("ignored", etc) + + assert out1 == out2 + assert out1["/etc/foo.conf"] == "modified_conffile" + assert out1["/etc/bar.conf"] == "modified_packaged_file" + assert "/etc/baz.conf" not in out1 + + # Caches should mean we only queried rpm once. + assert calls["modified"] == 1 + assert calls["config"] == 1 diff --git a/tests/test_remote.py b/tests/test_remote.py new file mode 100644 index 0000000..1c0bfd0 --- /dev/null +++ b/tests/test_remote.py @@ -0,0 +1,1023 @@ +from __future__ import annotations + +import io +import tarfile +import warnings +from pathlib import Path + +import pytest + + +def _make_tgz_bytes(files: dict[str, bytes]) -> bytes: + bio = io.BytesIO() + with tarfile.open(fileobj=bio, mode="w:gz") as tf: + for name, content in files.items(): + ti = tarfile.TarInfo(name=name) + ti.size = len(content) + tf.addfile(ti, io.BytesIO(content)) + return bio.getvalue() + + +def test_safe_extract_tar_rejects_path_traversal(tmp_path: Path): + from enroll.remote import _safe_extract_tar + + # Build an unsafe tar with ../ traversal + bio = io.BytesIO() + with tarfile.open(fileobj=bio, mode="w:gz") as tf: + ti = tarfile.TarInfo(name="../evil") + ti.size = 1 + tf.addfile(ti, io.BytesIO(b"x")) + + bio.seek(0) + with tarfile.open(fileobj=bio, mode="r:gz") as tf: + with pytest.raises(RuntimeError, match="Unsafe tar member path"): + _safe_extract_tar(tf, tmp_path) + + +def test_safe_extract_tar_rejects_symlinks(tmp_path: Path): + from enroll.remote import _safe_extract_tar + + bio = io.BytesIO() + with tarfile.open(fileobj=bio, mode="w:gz") as tf: + ti = tarfile.TarInfo(name="link") + ti.type = tarfile.SYMTYPE + ti.linkname = "/etc/passwd" + tf.addfile(ti) + + bio.seek(0) + with tarfile.open(fileobj=bio, mode="r:gz") as tf: + with pytest.raises(RuntimeError, match="Refusing to extract"): + _safe_extract_tar(tf, tmp_path) + + +def test_remote_harvest_happy_path(tmp_path: Path, monkeypatch): + import sys + + import enroll.remote as r + + # Avoid building a real zipapp; just create a file. + def fake_build(_td: Path) -> Path: + p = _td / "enroll.pyz" + p.write_bytes(b"PYZ") + return p + + monkeypatch.setattr(r, "_build_enroll_pyz", fake_build) + + # Prepare a tiny harvest bundle tar stream from the "remote". + tgz = _make_tgz_bytes({"state.json": b'{"ok": true}\n'}) + + # Track each SSH exec_command call with whether a PTY was requested. + calls: list[tuple[str, bool]] = [] + + class _Chan: + def __init__(self, out: bytes = b"", err: bytes = b"", rc: int = 0): + self._out = out + self._err = err + self._out_i = 0 + self._err_i = 0 + self._rc = rc + self._closed = False + + def recv_ready(self) -> bool: + return (not self._closed) and self._out_i < len(self._out) + + def recv(self, n: int) -> bytes: + if self._closed: + return b"" + chunk = self._out[self._out_i : self._out_i + n] + self._out_i += len(chunk) + return chunk + + def recv_stderr_ready(self) -> bool: + return (not self._closed) and self._err_i < len(self._err) + + def recv_stderr(self, n: int) -> bytes: + if self._closed: + return b"" + chunk = self._err[self._err_i : self._err_i + n] + self._err_i += len(chunk) + return chunk + + def exit_status_ready(self) -> bool: + return self._closed or ( + self._out_i >= len(self._out) and self._err_i >= len(self._err) + ) + + def recv_exit_status(self) -> int: + return self._rc + + def shutdown_write(self) -> None: + return + + def close(self) -> None: + self._closed = True + + class _Stdout: + def __init__(self, payload: bytes = b"", rc: int = 0, err: bytes = b""): + self._bio = io.BytesIO(payload) + # _ssh_run reads stdout/stderr via the underlying channel. + self.channel = _Chan(out=payload, err=err, rc=rc) + + def read(self, n: int = -1) -> bytes: + return self._bio.read(n) + + class _Stderr: + def __init__(self, payload: bytes = b""): + self._bio = io.BytesIO(payload) + + def read(self, n: int = -1) -> bytes: + return self._bio.read(n) + + class _SFTP: + def __init__(self): + self.put_calls: list[tuple[str, str]] = [] + + def put(self, local: str, remote: str) -> None: + self.put_calls.append((local, remote)) + + def close(self) -> None: + return + + class FakeSSH: + def __init__(self): + self._sftp = _SFTP() + + def load_system_host_keys(self): + return + + def set_missing_host_key_policy(self, _policy): + return + + def connect(self, **kwargs): + # Accept any connect parameters. + return + + def open_sftp(self): + return self._sftp + + def exec_command(self, cmd: str, *, get_pty: bool = False, **_kwargs): + calls.append((cmd, bool(get_pty))) + # The tar stream uses exec_command directly. + if cmd.startswith("tar -cz -C"): + return (None, _Stdout(tgz, rc=0), _Stderr(b"")) + + # _ssh_run path: id -un, mktemp -d, chmod, sudo harvest, sudo chown, rm -rf + if cmd == "id -un": + return (None, _Stdout(b"alice\n"), _Stderr()) + if cmd == "mktemp -d": + return (None, _Stdout(b"/tmp/enroll-remote-123\n"), _Stderr()) + if cmd.startswith("chmod 700"): + return (None, _Stdout(b""), _Stderr()) + if cmd.startswith("sudo -n") and " harvest " in cmd: + if not get_pty: + msg = b"sudo: sorry, you must have a tty to run sudo\n" + return (None, _Stdout(b"", rc=1, err=msg), _Stderr(msg)) + return (None, _Stdout(b"", rc=0), _Stderr(b"")) + if cmd.startswith("sudo -S") and " harvest " in cmd: + return (None, _Stdout(b""), _Stderr()) + if " harvest " in cmd: + return (None, _Stdout(b""), _Stderr()) + if cmd.startswith("sudo -n") and " chown -R" in cmd: + if not get_pty: + msg = b"sudo: sorry, you must have a tty to run sudo\n" + return (None, _Stdout(b"", rc=1, err=msg), _Stderr(msg)) + return (None, _Stdout(b"", rc=0), _Stderr(b"")) + if cmd.startswith("rm -rf"): + return (None, _Stdout(b""), _Stderr()) + + return (None, _Stdout(b""), _Stderr(b"unknown")) + + def close(self): + return + + import types + + class RejectPolicy: + pass + + FakeParamiko = types.SimpleNamespace(SSHClient=FakeSSH, RejectPolicy=RejectPolicy) + + # Provide a fake paramiko module. + monkeypatch.setitem(sys.modules, "paramiko", FakeParamiko) + + out_dir = tmp_path / "out" + state_path = r.remote_harvest( + ask_become_pass=False, + local_out_dir=out_dir, + remote_host="example.com", + remote_port=2222, + remote_user=None, + include_paths=["/etc/nginx/nginx.conf"], + exclude_paths=["/etc/shadow"], + dangerous=True, + no_sudo=False, + ) + + assert state_path == out_dir / "state.json" + assert state_path.exists() + assert b"ok" in state_path.read_bytes() + + # Ensure we attempted remote harvest with sudo and passed include/exclude and dangerous. + joined = "\n".join([c for c, _pty in calls]) + assert "sudo" in joined + assert "--dangerous" in joined + assert "--include-path" in joined + assert "--exclude-path" in joined + + # Ensure we fall back to PTY only when sudo reports it is required. + assert any(c == "id -un" and pty is False for c, pty in calls) + + sudo_harvest = [ + (c, pty) for c, pty in calls if c.startswith("sudo -n") and " harvest " in c + ] + assert any(pty is False for _c, pty in sudo_harvest) + assert any(pty is True for _c, pty in sudo_harvest) + + sudo_chown = [ + (c, pty) for c, pty in calls if c.startswith("sudo -n") and " chown -R" in c + ] + assert any(pty is False for _c, pty in sudo_chown) + assert any(pty is True for _c, pty in sudo_chown) + + assert any(c.startswith("tar -cz -C") and pty is False for c, pty in calls) + + +def test_remote_harvest_no_sudo_does_not_request_pty_or_chown( + tmp_path: Path, monkeypatch +): + """When --no-sudo is used we should not request a PTY nor run sudo chown.""" + import sys + + import enroll.remote as r + + monkeypatch.setattr( + r, + "_build_enroll_pyz", + lambda td: (Path(td) / "enroll.pyz").write_bytes(b"PYZ") + or (Path(td) / "enroll.pyz"), + ) + + tgz = _make_tgz_bytes({"state.json": b"{}"}) + calls: list[tuple[str, bool]] = [] + + class _Chan: + def __init__(self, out: bytes = b"", err: bytes = b"", rc: int = 0): + self._out = out + self._err = err + self._out_i = 0 + self._err_i = 0 + self._rc = rc + self._closed = False + + def recv_ready(self) -> bool: + return (not self._closed) and self._out_i < len(self._out) + + def recv(self, n: int) -> bytes: + if self._closed: + return b"" + chunk = self._out[self._out_i : self._out_i + n] + self._out_i += len(chunk) + return chunk + + def recv_stderr_ready(self) -> bool: + return (not self._closed) and self._err_i < len(self._err) + + def recv_stderr(self, n: int) -> bytes: + if self._closed: + return b"" + chunk = self._err[self._err_i : self._err_i + n] + self._err_i += len(chunk) + return chunk + + def exit_status_ready(self) -> bool: + return self._closed or ( + self._out_i >= len(self._out) and self._err_i >= len(self._err) + ) + + def recv_exit_status(self) -> int: + return self._rc + + def shutdown_write(self) -> None: + return + + def close(self) -> None: + self._closed = True + + class _Stdout: + def __init__(self, payload: bytes = b"", rc: int = 0, err: bytes = b""): + self._bio = io.BytesIO(payload) + # _ssh_run reads stdout/stderr via the underlying channel. + self.channel = _Chan(out=payload, err=err, rc=rc) + + def read(self, n: int = -1) -> bytes: + return self._bio.read(n) + + class _Stderr: + def __init__(self, payload: bytes = b""): + self._bio = io.BytesIO(payload) + + def read(self, n: int = -1) -> bytes: + return self._bio.read(n) + + class _SFTP: + def put(self, _local: str, _remote: str) -> None: + return + + def close(self) -> None: + return + + class FakeSSH: + def __init__(self): + self._sftp = _SFTP() + + def load_system_host_keys(self): + return + + def set_missing_host_key_policy(self, _policy): + return + + def connect(self, **_kwargs): + return + + def open_sftp(self): + return self._sftp + + def exec_command(self, cmd: str, *, get_pty: bool = False, **_kwargs): + calls.append((cmd, bool(get_pty))) + if cmd == "mktemp -d": + return (None, _Stdout(b"/tmp/enroll-remote-456\n"), _Stderr()) + if cmd.startswith("chmod 700"): + return (None, _Stdout(b""), _Stderr()) + if cmd.startswith("tar -cz -C"): + return (None, _Stdout(tgz, rc=0), _Stderr()) + if " harvest " in cmd: + return (None, _Stdout(b""), _Stderr()) + if cmd.startswith("rm -rf"): + return (None, _Stdout(b""), _Stderr()) + return (None, _Stdout(b""), _Stderr()) + + def close(self): + return + + import types + + class RejectPolicy: + pass + + monkeypatch.setitem( + sys.modules, + "paramiko", + types.SimpleNamespace(SSHClient=FakeSSH, RejectPolicy=RejectPolicy), + ) + + out_dir = tmp_path / "out" + r.remote_harvest( + ask_become_pass=False, + local_out_dir=out_dir, + remote_host="example.com", + remote_user="alice", + no_sudo=True, + ) + + joined = "\n".join([c for c, _pty in calls]) + assert "sudo" not in joined + assert "sudo chown" not in joined + assert any(" harvest " in c and pty is False for c, pty in calls) + + +def test_remote_harvest_sudo_password_retry_uses_sudo_s_and_writes_password( + tmp_path: Path, monkeypatch +): + """If sudo requires a password, we should fall back from -n to -S and feed stdin.""" + import sys + import types + + import enroll.remote as r + + # Avoid building a real zipapp; just create a file. + monkeypatch.setattr( + r, + "_build_enroll_pyz", + lambda td: (Path(td) / "enroll.pyz").write_bytes(b"PYZ") + or (Path(td) / "enroll.pyz"), + ) + + tgz = _make_tgz_bytes({"state.json": b'{"ok": true}\n'}) + calls: list[tuple[str, bool]] = [] + stdin_by_cmd: dict[str, list[str]] = {} + + class _Chan: + def __init__(self, out: bytes = b"", err: bytes = b"", rc: int = 0): + self._out = out + self._err = err + self._out_i = 0 + self._err_i = 0 + self._rc = rc + self._closed = False + + def recv_ready(self) -> bool: + return (not self._closed) and self._out_i < len(self._out) + + def recv(self, n: int) -> bytes: + if self._closed: + return b"" + chunk = self._out[self._out_i : self._out_i + n] + self._out_i += len(chunk) + return chunk + + def recv_stderr_ready(self) -> bool: + return (not self._closed) and self._err_i < len(self._err) + + def recv_stderr(self, n: int) -> bytes: + if self._closed: + return b"" + chunk = self._err[self._err_i : self._err_i + n] + self._err_i += len(chunk) + return chunk + + def exit_status_ready(self) -> bool: + return self._closed or ( + self._out_i >= len(self._out) and self._err_i >= len(self._err) + ) + + def recv_exit_status(self) -> int: + return self._rc + + def shutdown_write(self) -> None: + return + + def close(self) -> None: + self._closed = True + + class _Stdout: + def __init__(self, payload: bytes = b"", rc: int = 0, err: bytes = b""): + self._bio = io.BytesIO(payload) + # _ssh_run reads stdout/stderr via the underlying channel. + self.channel = _Chan(out=payload, err=err, rc=rc) + + def read(self, n: int = -1) -> bytes: + return self._bio.read(n) + + class _Stderr: + def __init__(self, payload: bytes = b""): + self._bio = io.BytesIO(payload) + + def read(self, n: int = -1) -> bytes: + return self._bio.read(n) + + class _Stdin: + def __init__(self, cmd: str): + self._cmd = cmd + stdin_by_cmd.setdefault(cmd, []) + + def write(self, s: str) -> None: + stdin_by_cmd[self._cmd].append(s) + + def flush(self) -> None: + return + + class _SFTP: + def put(self, _local: str, _remote: str) -> None: + return + + def close(self) -> None: + return + + class FakeSSH: + def __init__(self): + self._sftp = _SFTP() + + def load_system_host_keys(self): + return + + def set_missing_host_key_policy(self, _policy): + return + + def connect(self, **_kwargs): + return + + def open_sftp(self): + return self._sftp + + def exec_command(self, cmd: str, *, get_pty: bool = False, **_kwargs): + calls.append((cmd, bool(get_pty))) + + # Tar stream + if cmd.startswith("tar -cz -C"): + return (_Stdin(cmd), _Stdout(tgz, rc=0), _Stderr(b"")) + + if cmd == "mktemp -d": + return (_Stdin(cmd), _Stdout(b"/tmp/enroll-remote-789\n"), _Stderr()) + if cmd.startswith("chmod 700"): + return (_Stdin(cmd), _Stdout(b""), _Stderr()) + + # First attempt: sudo -n fails, prompting is not allowed. + if cmd.startswith("sudo -n") and " harvest " in cmd: + return ( + _Stdin(cmd), + _Stdout(b"", rc=1, err=b"sudo: a password is required\n"), + _Stderr(b"sudo: a password is required\n"), + ) + + # Retry: sudo -S succeeds and should have been fed the password via stdin. + if cmd.startswith("sudo -S") and " harvest " in cmd: + return (_Stdin(cmd), _Stdout(b"", rc=0), _Stderr(b"")) + + # chown succeeds passwordlessly (e.g., sudo timestamp is warm). + if cmd.startswith("sudo -n") and " chown -R" in cmd: + return (_Stdin(cmd), _Stdout(b"", rc=0), _Stderr(b"")) + + if cmd.startswith("rm -rf"): + return (_Stdin(cmd), _Stdout(b"", rc=0), _Stderr(b"")) + + # Fallback for unexpected commands. + return (_Stdin(cmd), _Stdout(b"", rc=0), _Stderr(b"")) + + def close(self): + return + + class RejectPolicy: + pass + + monkeypatch.setitem( + sys.modules, + "paramiko", + types.SimpleNamespace(SSHClient=FakeSSH, RejectPolicy=RejectPolicy), + ) + + out_dir = tmp_path / "out" + state_path = r.remote_harvest( + ask_become_pass=True, + getpass_fn=lambda _prompt="": "s3cr3t", + local_out_dir=out_dir, + remote_host="example.com", + remote_user="alice", + no_sudo=False, + ) + + assert state_path.exists() + assert b"ok" in state_path.read_bytes() + + # Ensure we attempted with sudo -n first, then sudo -S. + sudo_n = [c for c, _pty in calls if c.startswith("sudo -n") and " harvest " in c] + sudo_s = [c for c, _pty in calls if c.startswith("sudo -S") and " harvest " in c] + assert len(sudo_n) == 1 + assert len(sudo_s) == 1 + + # Ensure the password was written to stdin for the -S invocation. + assert stdin_by_cmd.get(sudo_s[0]) == ["s3cr3t\n"] + + +def test_sudo_password_required_detection(): + from enroll.remote import _sudo_password_required + + assert _sudo_password_required("", "a password is required") is True + assert _sudo_password_required("", "password is required") is True + assert ( + _sudo_password_required("", "a terminal is required to read the password") + is True + ) + assert ( + _sudo_password_required("", "no tty present and no askpass program specified") + is True + ) + assert _sudo_password_required("", "must have a tty to run sudo") is True + assert _sudo_password_required("", "sudo: sorry, you must have a tty") is True + assert _sudo_password_required("", "askpass") is True + assert _sudo_password_required("success", "") is False + + +def test_sudo_not_permitted_detection(): + from enroll.remote import _sudo_not_permitted + + assert _sudo_not_permitted("", "user is not in the sudoers file") is True + assert _sudo_not_permitted("", "not allowed to execute") is True + assert _sudo_not_permitted("", "may not run sudo") is True + assert _sudo_not_permitted("", "sorry, user") is True + assert _sudo_not_permitted("success", "") is False + + +def test_sudo_tty_required_detection(): + from enroll.remote import _sudo_tty_required + + assert _sudo_tty_required("", "must have a tty") is True + assert _sudo_tty_required("", "sorry, you must have a tty") is True + assert _sudo_tty_required("", "sudo: sorry, you must have a tty") is True + assert _sudo_tty_required("", "must have a tty to run sudo") is True + assert _sudo_tty_required("success", "") is False + + +def test_resolve_become_password_prompts_when_asked(monkeypatch): + from enroll.remote import _resolve_become_password + + prompted = [] + + def fake_getpass(prompt): + prompted.append(prompt) + return "secret" + + result = _resolve_become_password( + True, prompt="sudo password: ", getpass_fn=fake_getpass + ) + assert result == "secret" + assert len(prompted) == 1 + + +def test_resolve_become_password_returns_none_when_not_asked(): + from enroll.remote import _resolve_become_password + + result = _resolve_become_password(False) + assert result is None + + +def test_resolve_ssh_key_passphrase_from_env(monkeypatch): + from enroll.remote import _resolve_ssh_key_passphrase + + monkeypatch.setenv("SSH_KEY_PASS", "env_secret") + + result = _resolve_ssh_key_passphrase(False, env_var="SSH_KEY_PASS") + assert result == "env_secret" + + +def test_resolve_ssh_key_passphrase_raises_when_env_not_set(monkeypatch): + from enroll.remote import _resolve_ssh_key_passphrase + + monkeypatch.delenv("SSH_KEY_PASS", raising=False) + + with pytest.raises(RuntimeError, match="SSH key passphrase environment variable"): + _resolve_ssh_key_passphrase(False, env_var="SSH_KEY_PASS") + + +def test_resolve_ssh_key_passphrase_prompts_when_asked(monkeypatch): + from enroll.remote import _resolve_ssh_key_passphrase + + prompted = [] + + def fake_getpass(prompt): + prompted.append(prompt) + return "prompt_secret" + + result = _resolve_ssh_key_passphrase( + True, prompt="SSH key passphrase: ", getpass_fn=fake_getpass + ) + assert result == "prompt_secret" + assert len(prompted) == 1 + + +def test_resolve_ssh_key_passphrase_returns_none_when_not_asked(): + from enroll.remote import _resolve_ssh_key_passphrase + + result = _resolve_ssh_key_passphrase(False, env_var=None) + assert result is None + + +def test_safe_extract_tar_rejects_absolute_paths(tmp_path: Path): + from enroll.remote import _safe_extract_tar + + import io + import tarfile + + bio = io.BytesIO() + with tarfile.open(fileobj=bio, mode="w:gz") as tf: + ti = tarfile.TarInfo(name="/etc/passwd") + ti.size = 1 + tf.addfile(ti, io.BytesIO(b"x")) + + bio.seek(0) + with tarfile.open(fileobj=bio, mode="r:gz") as tf: + with pytest.raises(RuntimeError, match="Unsafe tar member path"): + _safe_extract_tar(tf, tmp_path) + + +def test_safe_extract_tar_rejects_hardlinks(tmp_path: Path): + from enroll.remote import _safe_extract_tar + + import io + import tarfile + + bio = io.BytesIO() + with tarfile.open(fileobj=bio, mode="w:gz") as tf: + ti = tarfile.TarInfo(name="hardlink") + ti.type = tarfile.LNKTYPE + ti.linkname = "/etc/passwd" + tf.addfile(ti) + + bio.seek(0) + with tarfile.open(fileobj=bio, mode="r:gz") as tf: + with pytest.raises(RuntimeError, match="Refusing to extract"): + _safe_extract_tar(tf, tmp_path) + + +def test_safe_extract_tar_rejects_device_nodes(tmp_path: Path): + from enroll.remote import _safe_extract_tar + + import io + import tarfile + + bio = io.BytesIO() + with tarfile.open(fileobj=bio, mode="w:gz") as tf: + ti = tarfile.TarInfo(name="device") + ti.type = tarfile.CHRTYPE + tf.addfile(ti) + + bio.seek(0) + with tarfile.open(fileobj=bio, mode="r:gz") as tf: + with pytest.raises(RuntimeError, match="Refusing to extract"): + _safe_extract_tar(tf, tmp_path) + + +def test_safe_extract_tar_accepts_dot_entry(tmp_path: Path): + from enroll.remote import _safe_extract_tar + + import io + import tarfile + + bio = io.BytesIO() + with tarfile.open(fileobj=bio, mode="w:gz") as tf: + ti = tarfile.TarInfo(name=".") + ti.size = 0 + tf.addfile(ti, io.BytesIO(b"")) + + bio.seek(0) + with tarfile.open(fileobj=bio, mode="r:gz") as tf: + _safe_extract_tar(tf, tmp_path) + + +def test_safe_extract_tar_accepts_valid_files(tmp_path: Path): + from enroll.remote import _safe_extract_tar + + import io + import tarfile + + bio = io.BytesIO() + with tarfile.open(fileobj=bio, mode="w:gz") as tf: + ti = tarfile.TarInfo(name="foo/bar.txt") + ti.size = 5 + tf.addfile(ti, io.BytesIO(b"hello")) + + bio.seek(0) + with tarfile.open(fileobj=bio, mode="r:gz") as tf: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", DeprecationWarning) + _safe_extract_tar(tf, tmp_path) + + assert not any( + "Python 3.14" in str(w.message) and issubclass(w.category, DeprecationWarning) + for w in caught + ) + assert (tmp_path / "foo" / "bar.txt").read_bytes() == b"hello" + + +def test_remote_harvest_ssh_key_passphrase_retry(monkeypatch, tmp_path: Path): + import sys + + import enroll.remote as r + + monkeypatch.setattr( + r, + "_build_enroll_pyz", + lambda td: (Path(td) / "enroll.pyz").write_bytes(b"PYZ") + or (Path(td) / "enroll.pyz"), + ) + + tgz = _make_tgz_bytes({"state.json": b'{"ok": true}\n'}) + + class _Chan: + def __init__(self, out: bytes = b"", err: bytes = b"", rc: int = 0): + self._out = out + self._err = err + self._out_i = 0 + self._err_i = 0 + self._rc = rc + self._closed = False + + def recv_ready(self) -> bool: + return (not self._closed) and self._out_i < len(self._out) + + def recv(self, n: int) -> bytes: + if self._closed: + return b"" + chunk = self._out[self._out_i : self._out_i + n] + self._out_i += len(chunk) + return chunk + + def recv_stderr_ready(self) -> bool: + return (not self._closed) and self._err_i < len(self._err) + + def recv_stderr(self, n: int) -> bytes: + if self._closed: + return b"" + chunk = self._err[self._err_i : self._err_i + n] + self._err_i += len(chunk) + return chunk + + def exit_status_ready(self) -> bool: + return self._closed or ( + self._out_i >= len(self._out) and self._err_i >= len(self._err) + ) + + def recv_exit_status(self) -> int: + return self._rc + + def shutdown_write(self) -> None: + return + + def close(self) -> None: + self._closed = True + + class _Stdout: + def __init__(self, payload: bytes = b"", rc: int = 0, err: bytes = b""): + self._bio = io.BytesIO(payload) + self.channel = _Chan(out=payload, err=err, rc=rc) + + def read(self, n: int = -1) -> bytes: + return self._bio.read(n) + + class _Stderr: + def __init__(self, payload: bytes = b""): + self._bio = io.BytesIO(payload) + + def read(self, n: int = -1) -> bytes: + return self._bio.read(n) + + class _Stdin: + def __init__(self, cmd: str): + self._cmd = cmd + + def write(self, s: str) -> None: + pass + + def flush(self) -> None: + return + + class _SFTP: + def put(self, _local: str, _remote: str) -> None: + return + + def close(self) -> None: + return + + class FakeSSH: + def __init__(self): + self._sftp = _SFTP() + + def load_system_host_keys(self): + return + + def set_missing_host_key_policy(self, _policy): + return + + def connect(self, **_kwargs): + return + + def open_sftp(self): + return self._sftp + + def exec_command(self, cmd: str, *, get_pty: bool = False, **_kwargs): + if cmd.startswith("tar -cz -C"): + return (_Stdin(cmd), _Stdout(tgz, rc=0), _Stderr(b"")) + if cmd == "mktemp -d": + return (_Stdin(cmd), _Stdout(b"/tmp/enroll-remote-789\n"), _Stderr()) + if cmd.startswith("chmod 700"): + return (_Stdin(cmd), _Stdout(b""), _Stderr()) + if " harvest " in cmd: + return (_Stdin(cmd), _Stdout(b""), _Stderr()) + if cmd.startswith("rm -rf"): + return (_Stdin(cmd), _Stdout(b""), _Stderr()) + return (_Stdin(cmd), _Stdout(b""), _Stderr()) + + def close(self): + return + + RejectPolicy4 = type("RejectPolicy", (), {}) + + class FakeParamiko: + SSHClient = FakeSSH + RejectPolicy = RejectPolicy4 # type: ignore + PasswordRequiredException = Exception # type: ignore + + monkeypatch.setitem(sys.modules, "paramiko", FakeParamiko) + + prompts = [] + + def fake_getpass(prompt): + prompts.append(prompt) + return "passphrase" + + out_dir = tmp_path / "out" + state_path = r.remote_harvest( + ask_key_passphrase=True, + getpass_fn=fake_getpass, + local_out_dir=out_dir, + remote_host="example.com", + remote_user="alice", + no_sudo=True, + ) + + assert state_path.exists() + assert len(prompts) == 1 + + +def test_remote_harvest_ssh_key_passphrase_raises_when_not_interactive( + monkeypatch, tmp_path: Path +): + import sys + + import enroll.remote as r + + monkeypatch.setattr( + r, + "_build_enroll_pyz", + lambda td: (Path(td) / "enroll.pyz").write_bytes(b"PYZ") + or (Path(td) / "enroll.pyz"), + ) + + class _Chan: + def __init__(self): + self._closed = False + + def recv_ready(self) -> bool: + return False + + def recv(self, n: int) -> bytes: + return b"" + + def recv_stderr_ready(self) -> bool: + return False + + def recv_stderr(self, n: int) -> bytes: + return b"" + + def exit_status_ready(self) -> bool: + return True + + def recv_exit_status(self) -> int: + return 0 + + def shutdown_write(self) -> None: + return + + def close(self) -> None: + self._closed = True + + class _Stdout: + def __init__(self): + self.channel = _Chan() + + def read(self, n: int = -1) -> bytes: + return b"" + + class _Stderr: + def read(self, n: int = -1) -> bytes: + return b"" + + class _SFTP: + def put(self, _local: str, _remote: str) -> None: + return + + def close(self) -> None: + return + + class FakeSSH: + def __init__(self): + self._sftp = _SFTP() + + def load_system_host_keys(self): + return + + def set_missing_host_key_policy(self, _policy): + return + + def connect(self, **_kwargs): + raise Exception("PasswordRequired") + + def open_sftp(self): + return self._sftp + + def exec_command(self, cmd: str, **_kwargs): + return (_Stdout(), _Stdout(), _Stderr()) + + def close(self): + return + + class RejectPolicy: + pass + + RejectPolicy3 = RejectPolicy + + class FakeParamiko: + SSHClient = FakeSSH + RejectPolicy = RejectPolicy3 # type: ignore + PasswordRequiredException = Exception # type: ignore + + monkeypatch.setitem(sys.modules, "paramiko", FakeParamiko) + + out_dir = tmp_path / "out" + + with pytest.raises(RuntimeError, match="SSH private key is encrypted"): + r.remote_harvest( + ask_key_passphrase=False, + local_out_dir=out_dir, + remote_host="example.com", + stdin=io.StringIO(), + ) diff --git a/tests/test_rpm.py b/tests/test_rpm.py new file mode 100644 index 0000000..ff8c1e7 --- /dev/null +++ b/tests/test_rpm.py @@ -0,0 +1,210 @@ +from __future__ import annotations +import pytest + +import enroll.rpm as rpm + + +def test_rpm_owner_returns_none_when_unowned(monkeypatch): + monkeypatch.setattr( + rpm, + "_run", + lambda cmd, allow_fail=False, merge_err=False: ( + 1, + "file /etc/x is not owned by any package\n", + ), + ) + assert rpm.rpm_owner("/etc/x") is None + + +def test_rpm_owner_parses_name(monkeypatch): + monkeypatch.setattr( + rpm, "_run", lambda cmd, allow_fail=False, merge_err=False: (0, "bash\n") + ) + assert rpm.rpm_owner("/bin/bash") == "bash" + + +def test_strip_arch_strips_known_arches(): + assert rpm._strip_arch("vim-enhanced.x86_64") == "vim-enhanced" + assert rpm._strip_arch("foo.noarch") == "foo" + assert rpm._strip_arch("weird.token") == "weird.token" + + +def test_list_manual_packages_prefers_dnf_repoquery(monkeypatch): + monkeypatch.setattr( + rpm.shutil, "which", lambda exe: "/usr/bin/dnf" if exe == "dnf" else None + ) + + def fake_run(cmd, allow_fail=False, merge_err=False): + # First repoquery form returns usable output. + if cmd[:3] == ["dnf", "-q", "repoquery"]: + return 0, "vim-enhanced.x86_64\nhtop\nvim-enhanced.x86_64\n" + raise AssertionError(f"unexpected cmd: {cmd}") + + monkeypatch.setattr(rpm, "_run", fake_run) + + pkgs = rpm.list_manual_packages() + assert pkgs == ["htop", "vim-enhanced"] + + +def test_list_manual_packages_falls_back_to_history(monkeypatch): + monkeypatch.setattr( + rpm.shutil, "which", lambda exe: "/usr/bin/dnf" if exe == "dnf" else None + ) + + def fake_run(cmd, allow_fail=False, merge_err=False): + # repoquery fails + if cmd[:3] == ["dnf", "-q", "repoquery"]: + return 1, "" + if cmd[:3] == ["dnf", "-q", "history"]: + return ( + 0, + "Installed Packages\nvim-enhanced.x86_64\nLast metadata expiration check: 0:01:00 ago\n", + ) + raise AssertionError(f"unexpected cmd: {cmd}") + + monkeypatch.setattr(rpm, "_run", fake_run) + + pkgs = rpm.list_manual_packages() + assert pkgs == ["vim-enhanced"] + + +def test_build_rpm_etc_index_uses_fallback_when_rpm_output_mismatches(monkeypatch): + # Two files in /etc, one owned, one unowned. + monkeypatch.setattr( + rpm, "_walk_etc_files", lambda: ["/etc/owned.conf", "/etc/unowned.conf"] + ) + + # Simulate chunk query producing unexpected extra line (mismatch) -> triggers per-file fallback. + monkeypatch.setattr( + rpm, + "_run", + lambda cmd, allow_fail=False, merge_err=False: (0, "ownedpkg\nEXTRA\nTHIRD\n"), + ) + monkeypatch.setattr( + rpm, "rpm_owner", lambda p: "ownedpkg" if p == "/etc/owned.conf" else None + ) + + owned, owner_map, topdir_to_pkgs, pkg_to_etc = rpm.build_rpm_etc_index() + + assert owned == {"/etc/owned.conf"} + assert owner_map["/etc/owned.conf"] == "ownedpkg" + assert "owned.conf" in topdir_to_pkgs + assert pkg_to_etc["ownedpkg"] == ["/etc/owned.conf"] + + +def test_build_rpm_etc_index_parses_chunk_output(monkeypatch): + monkeypatch.setattr( + rpm, "_walk_etc_files", lambda: ["/etc/ssh/sshd_config", "/etc/notowned"] + ) + + def fake_run(cmd, allow_fail=False, merge_err=False): + # One output line per input path. + return 0, "openssh-server\nfile /etc/notowned is not owned by any package\n" + + monkeypatch.setattr(rpm, "_run", fake_run) + + owned, owner_map, topdir_to_pkgs, pkg_to_etc = rpm.build_rpm_etc_index() + + assert "/etc/ssh/sshd_config" in owned + assert "/etc/notowned" not in owned + assert owner_map["/etc/ssh/sshd_config"] == "openssh-server" + assert "ssh" in topdir_to_pkgs + assert "openssh-server" in topdir_to_pkgs["ssh"] + assert pkg_to_etc["openssh-server"] == ["/etc/ssh/sshd_config"] + + +def test_rpm_config_files_and_modified_files_parsing(monkeypatch): + monkeypatch.setattr( + rpm, + "_run", + lambda cmd, allow_fail=False, merge_err=False: ( + 0, + "/etc/foo.conf\n/usr/bin/tool\n", + ), + ) + assert rpm.rpm_config_files("mypkg") == {"/etc/foo.conf", "/usr/bin/tool"} + + # rpm -V returns only changed/missing files + out = "S.5....T. c /etc/foo.conf\nmissing /etc/bar\n" + monkeypatch.setattr( + rpm, "_run", lambda cmd, allow_fail=False, merge_err=False: (1, out) + ) + assert rpm.rpm_modified_files("mypkg") == {"/etc/foo.conf", "/etc/bar"} + + +def test_list_manual_packages_uses_yum_fallback(monkeypatch): + # No dnf, yum present. + monkeypatch.setattr( + rpm.shutil, "which", lambda exe: "/usr/bin/yum" if exe == "yum" else None + ) + + def fake_run(cmd, allow_fail=False, merge_err=False): + assert cmd[:3] == ["yum", "-q", "history"] + return 0, "Installed Packages\nvim-enhanced.x86_64\nhtop\n" + + monkeypatch.setattr(rpm, "_run", fake_run) + + assert rpm.list_manual_packages() == ["htop", "vim-enhanced"] + + +def test_list_installed_packages_parses_epoch_and_sorts(monkeypatch): + out = ( + "bash\t0\t5.2.26\t1.el9\tx86_64\tSystem Environment/Shells\n" + "bash\t1\t5.2.26\t1.el9\taarch64\tSystem Environment/Shells\n" + "coreutils\t(none)\t9.1\t2.el9\tx86_64\tSystem Environment/Base\n" + ) + monkeypatch.setattr( + rpm, "_run", lambda cmd, allow_fail=False, merge_err=False: (0, out) + ) + pkgs = rpm.list_installed_packages() + assert pkgs["bash"][0]["arch"] == "aarch64" # sorted by arch then version + assert pkgs["bash"][0]["version"].startswith("1:") + assert pkgs["bash"][0]["group"] == "System Environment/Shells" + assert pkgs["coreutils"][0]["version"] == "9.1-2.el9" + + +def test_rpm_config_files_returns_empty_on_failure(monkeypatch): + monkeypatch.setattr( + rpm, "_run", lambda cmd, allow_fail=False, merge_err=False: (1, "") + ) + assert rpm.rpm_config_files("missing") == set() + + +def test_rpm_owner_strips_epoch_prefix_when_present(monkeypatch): + # Defensive: rpm output might include epoch-like token. + monkeypatch.setattr( + rpm, + "_run", + lambda cmd, allow_fail=False, merge_err=False: (0, "1:bash\n"), + ) + assert rpm.rpm_owner("/bin/bash") == "bash" + + +def test_strip_arch_no_suffix(): + assert rpm._strip_arch("vim") == "vim" + assert rpm._strip_arch("nginx ") == "nginx" + + +def test_strip_arch_with_unknown_suffix(): + assert rpm._strip_arch("package.unknown") == "package.unknown" + + +def test_run_command_raises_on_fail(): + with pytest.raises(RuntimeError): + rpm._run(["sh", "-c", "echo stderr >&2; exit 1"], allow_fail=False) + + +def test_rpm_owner_empty_path(): + assert rpm.rpm_owner("") is None + + +def test_rpm_modified_files_empty(monkeypatch): + monkeypatch.setattr( + rpm, "_run", lambda cmd, allow_fail=False, merge_err=False: (0, "") + ) + assert rpm.rpm_modified_files("vim") == set() + + +def test_list_manual_packages_no_commands_available(monkeypatch): + monkeypatch.setattr(rpm.shutil, "which", lambda exe: None) + assert rpm.list_manual_packages() == [] diff --git a/tests/test_rpm_run.py b/tests/test_rpm_run.py new file mode 100644 index 0000000..75b965e --- /dev/null +++ b/tests/test_rpm_run.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import types + +import pytest + +import enroll.rpm as rpm + + +def test_run_raises_on_nonzero_returncode_when_not_allow_fail(monkeypatch): + def fake_run(cmd, check, text, stdout, stderr): + return types.SimpleNamespace(returncode=1, stdout="OUT", stderr="ERR") + + monkeypatch.setattr(rpm.subprocess, "run", fake_run) + with pytest.raises(RuntimeError) as e: + rpm._run(["rpm", "-q"]) # type: ignore[attr-defined] + assert "Command failed" in str(e.value) + assert "ERR" in str(e.value) + assert "OUT" in str(e.value) + + +def test_run_merge_err_includes_stderr_in_stdout(monkeypatch): + def fake_run(cmd, check, text, stdout, stderr): + # When merge_err is True, stderr is redirected to STDOUT, so we only + # rely on stdout in our wrapper. + return types.SimpleNamespace(returncode=0, stdout="COMBINED", stderr=None) + + monkeypatch.setattr(rpm.subprocess, "run", fake_run) + rc, out = rpm._run(["rpm", "-q"], merge_err=True) + assert rc == 0 + assert out == "COMBINED" diff --git a/tests/test_sopsutil.py b/tests/test_sopsutil.py new file mode 100644 index 0000000..3aeffdc --- /dev/null +++ b/tests/test_sopsutil.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import pytest + +from pathlib import Path +from enroll.sopsutil import SopsError, _pgp_arg, find_sops_cmd, require_sops_cmd + + +def test_find_sops_cmd(): + result = find_sops_cmd() + if result is None: + pytest.skip("sops not installed") + assert result.endswith("sops") + + +def test_require_sops_cmd(): + exe = require_sops_cmd() + assert exe is not None + assert "sops" in exe + + +def test_require_sops_cmd_raises_when_not_found(monkeypatch): + import enroll.sopsutil as s + + def fake_find(): + return None + + monkeypatch.setattr(s, "find_sops_cmd", fake_find) + + with pytest.raises(SopsError) as exc_info: + require_sops_cmd() + + assert "sops" in str(exc_info.value).lower() + assert "not found" in str(exc_info.value).lower() + + +def test_pgp_arg_with_empty_fingerprints(): + with pytest.raises(SopsError) as exc_info: + _pgp_arg([]) + assert "No GPG fingerprints" in str(exc_info.value) + + +def test_pgp_arg_with_whitespace_fingerprints(): + result = _pgp_arg([" ", "ABC123", " DEF456 "]) + assert result == "ABC123,DEF456" + + +def test_pgp_arg_with_single_fingerprint(): + result = _pgp_arg(["ABC123DEF456"]) + assert result == "ABC123DEF456" + + +def test_pgp_arg_with_multiple_fingerprints(): + result = _pgp_arg(["ABC123", "DEF456", "GHI789"]) + assert result == "ABC123,DEF456,GHI789" + + +def test_encrypt_file_binary_success(monkeypatch, tmp_path: Path): + """Test successful encryption path.""" + # Create source file + src = tmp_path / "secret.txt" + src.write_text("secret data", encoding="utf-8") + dst = tmp_path / "encrypted.sops" + + # Mock subprocess.run to succeed + class Result: + returncode = 0 + stdout = b"encrypted data" + stderr = b"" + + def fake_run(cmd, capture_output, check): + return Result() + + # Mock require_sops_cmd to return a fake path + def fake_require(): + return "/fake/sops" + + monkeypatch.setattr("enroll.sopsutil.subprocess.run", fake_run) + monkeypatch.setattr("enroll.sopsutil.require_sops_cmd", fake_require) + + from enroll.sopsutil import encrypt_file_binary + + encrypt_file_binary(src, dst, pgp_fingerprints=["ABC123"]) + + assert dst.exists() + assert dst.read_bytes() == b"encrypted data" + + +def test_encrypt_file_binary_fails(monkeypatch, tmp_path: Path): + """Test encryption failure path.""" + src = tmp_path / "secret.txt" + src.write_text("secret data", encoding="utf-8") + dst = tmp_path / "encrypted.sops" + + class Result: + returncode = 1 + stdout = b"" + stderr = b"sops: gpg error" + + def fake_run(cmd, capture_output, check): + return Result() + + def fake_require(): + return "/fake/sops" + + monkeypatch.setattr("enroll.sopsutil.subprocess.run", fake_run) + monkeypatch.setattr("enroll.sopsutil.require_sops_cmd", fake_require) + + from enroll.sopsutil import encrypt_file_binary, SopsError + + with pytest.raises(SopsError) as exc_info: + encrypt_file_binary(src, dst, pgp_fingerprints=["ABC123"]) + + assert "encryption failed" in str(exc_info.value).lower() + + +def test_encrypt_file_binary_chmod_fails(monkeypatch, tmp_path: Path): + """Test when chmod fails but file is still written.""" + src = tmp_path / "secret.txt" + src.write_text("secret data", encoding="utf-8") + dst = tmp_path / "encrypted.sops" + + class Result: + returncode = 0 + stdout = b"encrypted data" + stderr = b"" + + def fake_run(cmd, capture_output, check): + return Result() + + def fake_require(): + return "/fake/sops" + + def fake_chmod(path, mode): + raise OSError("Permission denied") + + monkeypatch.setattr("enroll.sopsutil.subprocess.run", fake_run) + monkeypatch.setattr("enroll.sopsutil.require_sops_cmd", fake_require) + monkeypatch.setattr("enroll.sopsutil.os.chmod", fake_chmod) + + from enroll.sopsutil import encrypt_file_binary + + # Should not raise even though chmod fails + encrypt_file_binary(src, dst, pgp_fingerprints=["ABC123"]) + + assert dst.exists() + + +def test_decrypt_file_binary_to_success(monkeypatch, tmp_path: Path): + """Test successful decryption path.""" + src = tmp_path / "encrypted.sops" + src.write_bytes(b"encrypted data") + dst = tmp_path / "decrypted.txt" + + class Result: + returncode = 0 + stdout = b"decrypted data" + stderr = b"" + + def fake_run(cmd, capture_output, check): + return Result() + + def fake_require(): + return "/fake/sops" + + monkeypatch.setattr("enroll.sopsutil.subprocess.run", fake_run) + monkeypatch.setattr("enroll.sopsutil.require_sops_cmd", fake_require) + + from enroll.sopsutil import decrypt_file_binary_to + + decrypt_file_binary_to(src, dst) + + assert dst.exists() + assert dst.read_bytes() == b"decrypted data" + + +def test_decrypt_file_binary_to_fails(monkeypatch, tmp_path: Path): + """Test decryption failure path.""" + src = tmp_path / "encrypted.sops" + src.write_bytes(b"encrypted data") + dst = tmp_path / "decrypted.txt" + + class Result: + returncode = 1 + stdout = b"" + stderr = b"sops: decryption failed" + + def fake_run(cmd, capture_output, check): + return Result() + + def fake_require(): + return "/fake/sops" + + monkeypatch.setattr("enroll.sopsutil.subprocess.run", fake_run) + monkeypatch.setattr("enroll.sopsutil.require_sops_cmd", fake_require) + + from enroll.sopsutil import decrypt_file_binary_to, SopsError + + with pytest.raises(SopsError) as exc_info: + decrypt_file_binary_to(src, dst) + + assert "decryption failed" in str(exc_info.value).lower() + + +def test_decrypt_file_binary_to_chmod_fails(monkeypatch, tmp_path: Path): + """Test when chmod fails during decryption but file is still written.""" + src = tmp_path / "encrypted.sops" + src.write_bytes(b"encrypted data") + dst = tmp_path / "decrypted.txt" + + class Result: + returncode = 0 + stdout = b"decrypted data" + stderr = b"" + + def fake_run(cmd, capture_output, check): + return Result() + + def fake_require(): + return "/fake/sops" + + def fake_chmod(path, mode): + raise OSError("Permission denied") + + monkeypatch.setattr("enroll.sopsutil.subprocess.run", fake_run) + monkeypatch.setattr("enroll.sopsutil.require_sops_cmd", fake_require) + monkeypatch.setattr("enroll.sopsutil.os.chmod", fake_chmod) + + from enroll.sopsutil import decrypt_file_binary_to + + # Should not raise even though chmod fails + decrypt_file_binary_to(src, dst) + + assert dst.exists() diff --git a/tests/test_systemd.py b/tests/test_systemd.py new file mode 100644 index 0000000..16f8399 --- /dev/null +++ b/tests/test_systemd.py @@ -0,0 +1,321 @@ +from __future__ import annotations + +import pytest +import enroll.systemd as s + + +def test_list_enabled_services_and_timers_filters_templates(monkeypatch): + def fake_run(cmd: list[str]) -> str: + if "--type=service" in cmd: + return "\n".join( + [ + "nginx.service enabled", + "getty@.service enabled", # template + "foo@bar.service enabled", # instance units are included + "ssh.service enabled", + ] + ) + if "--type=timer" in cmd: + return "\n".join( + [ + "apt-daily.timer enabled", + "foo@.timer enabled", # template + ] + ) + raise AssertionError("unexpected") + + monkeypatch.setattr(s, "_run", fake_run) + assert s.list_enabled_services() == [ + "foo@bar.service", + "nginx.service", + "ssh.service", + ] + assert s.list_enabled_timers() == ["apt-daily.timer"] + + +def test_get_unit_info_parses_fields(monkeypatch): + class P: + def __init__(self, rc: int, out: str, err: str = ""): + self.returncode = rc + self.stdout = out + self.stderr = err + + def fake_run(cmd, check, text, capture_output): + assert cmd[0:2] == ["systemctl", "show"] + return P( + 0, + "\n".join( + [ + "FragmentPath=/lib/systemd/system/nginx.service", + "DropInPaths=/etc/systemd/system/nginx.service.d/override.conf /etc/systemd/system/nginx.service.d/extra.conf", + "EnvironmentFiles=-/etc/default/nginx /etc/nginx/env", + "ExecStart={ path=/usr/sbin/nginx ; argv[]=/usr/sbin/nginx -g daemon off; }", + "ActiveState=active", + "SubState=running", + "UnitFileState=enabled", + "ConditionResult=yes", + ] + ), + ) + + monkeypatch.setattr(s.subprocess, "run", fake_run) + ui = s.get_unit_info("nginx.service") + assert ui.fragment_path == "/lib/systemd/system/nginx.service" + assert "/etc/default/nginx" in ui.env_files + assert "/etc/nginx/env" in ui.env_files + assert "/usr/sbin/nginx" in ui.exec_paths + assert ui.active_state == "active" + + +def test_get_unit_info_raises_unit_query_error(monkeypatch): + class P: + def __init__(self, rc: int, out: str, err: str): + self.returncode = rc + self.stdout = out + self.stderr = err + + def fake_run(cmd, check, text, capture_output): + return P(1, "", "no such unit") + + monkeypatch.setattr(s.subprocess, "run", fake_run) + with pytest.raises(s.UnitQueryError) as ei: + s.get_unit_info("missing.service") + assert "missing.service" in str(ei.value) + assert ei.value.unit == "missing.service" + + +def test_get_timer_info_parses_fields(monkeypatch): + class P: + def __init__(self, rc: int, out: str, err: str = ""): + self.returncode = rc + self.stdout = out + self.stderr = err + + def fake_run(cmd, text, capture_output): + return P( + 0, + "\n".join( + [ + "FragmentPath=/lib/systemd/system/apt-daily.timer", + "DropInPaths=", + "EnvironmentFiles=-/etc/default/apt", + "Unit=apt-daily.service", + "ActiveState=active", + "SubState=waiting", + "UnitFileState=enabled", + "ConditionResult=yes", + ] + ), + ) + + monkeypatch.setattr(s.subprocess, "run", fake_run) + ti = s.get_timer_info("apt-daily.timer") + assert ti.trigger_unit == "apt-daily.service" + assert "/etc/default/apt" in ti.env_files + + +def test_list_enabled_services_empty_output(monkeypatch): + def fake_run(cmd: list[str]) -> str: + return "" + + monkeypatch.setattr(s, "_run", fake_run) + result = s.list_enabled_services() + assert result == [] + + +def test_list_enabled_timers_empty_output(monkeypatch): + def fake_run(cmd: list[str]) -> str: + return "" + + monkeypatch.setattr(s, "_run", fake_run) + result = s.list_enabled_timers() + assert result == [] + + +def test_list_enabled_services_with_only_templates(monkeypatch): + def fake_run(cmd: list[str]) -> str: + return "getty@.service enabled\n" + + monkeypatch.setattr(s, "_run", fake_run) + result = s.list_enabled_services() + assert result == [] + + +def test_list_enabled_timers_with_only_templates(monkeypatch): + def fake_run(cmd: list[str]) -> str: + return "foo@.timer enabled\n" + + monkeypatch.setattr(s, "_run", fake_run) + result = s.list_enabled_timers() + assert result == [] + + +def test_get_timer_info_raises_on_failure(monkeypatch): + class P: + def __init__(self, rc: int, out: str, err: str): + self.returncode = rc + self.stdout = out + self.stderr = err + + def fake_run(cmd, text, capture_output): + return P(1, "", "timer not found") + + monkeypatch.setattr(s.subprocess, "run", fake_run) + + with pytest.raises(RuntimeError) as exc_info: + s.get_timer_info("nonexistent.timer") + + assert "nonexistent.timer" in str(exc_info.value) + + +def test_get_timer_info_with_empty_fields(monkeypatch): + class P: + def __init__(self, rc: int, out: str, err: str = ""): + self.returncode = rc + self.stdout = out + self.stderr = err + + def fake_run(cmd, text, capture_output): + return P( + 0, + "\n".join( + [ + "FragmentPath=", + "DropInPaths=", + "EnvironmentFiles=", + "Unit=", + "ActiveState=", + "SubState=", + "UnitFileState=", + "ConditionResult=", + ] + ), + ) + + monkeypatch.setattr(s.subprocess, "run", fake_run) + ti = s.get_timer_info("empty.timer") + assert ti.fragment_path is None + assert ti.dropin_paths == [] + assert ti.env_files == [] + assert ti.trigger_unit is None + assert ti.active_state is None + + +def test_get_unit_info_with_empty_fields(monkeypatch): + class P: + def __init__(self, rc: int, out: str, err: str = ""): + self.returncode = rc + self.stdout = out + self.stderr = err + + def fake_run(cmd, check, text, capture_output): + return P( + 0, + "\n".join( + [ + "FragmentPath=", + "DropInPaths=", + "EnvironmentFiles=", + "ExecStart=", + "ActiveState=", + "SubState=", + "UnitFileState=", + "ConditionResult=", + ] + ), + ) + + monkeypatch.setattr(s.subprocess, "run", fake_run) + ui = s.get_unit_info("empty.service") + assert ui.fragment_path is None + assert ui.dropin_paths == [] + assert ui.env_files == [] + assert ui.exec_paths == [] + assert ui.active_state is None + + +def test_run_command_raises_on_error(monkeypatch): + """Test _run raises RuntimeError on non-zero exit.""" + + class P: + returncode = 1 + stdout = "" + stderr = "command failed" + + def fake_run(cmd, check, text, capture_output): + return P() + + monkeypatch.setattr(s.subprocess, "run", fake_run) + + with pytest.raises(RuntimeError) as exc_info: + s._run(["fake", "command"]) + + assert "Command failed" in str(exc_info.value) + assert "fake" in str(exc_info.value) + + +def test_list_enabled_services_filters_non_service_units(monkeypatch): + """Test that non-.service units are filtered out.""" + + def fake_run(cmd: list[str]) -> str: + return "\n".join( + [ + "nginx.service enabled", + "network.target enabled", # not a service + "multi-user.target enabled", # not a service + ] + ) + + monkeypatch.setattr(s, "_run", fake_run) + result = s.list_enabled_services() + assert result == ["nginx.service"] + + +def test_list_enabled_timers_filters_non_timer_units(monkeypatch): + """Test that non-.timer units are filtered out.""" + + def fake_run(cmd: list[str]) -> str: + return "\n".join( + [ + "apt-daily.timer enabled", + "some.service enabled", # not a timer + ] + ) + + monkeypatch.setattr(s, "_run", fake_run) + result = s.list_enabled_timers() + assert result == ["apt-daily.timer"] + + +def test_list_enabled_services_filters_empty_lines(monkeypatch): + """Test that empty lines are skipped.""" + + def fake_run(cmd: list[str]) -> str: + return "\n".join( + [ + "nginx.service enabled", + "", # empty line + "ssh.service enabled", + ] + ) + + monkeypatch.setattr(s, "_run", fake_run) + result = s.list_enabled_services() + assert result == ["nginx.service", "ssh.service"] + + +def test_list_enabled_timers_filters_empty_lines(monkeypatch): + """Test that empty lines are skipped.""" + + def fake_run(cmd: list[str]) -> str: + return "\n".join( + [ + "apt-daily.timer enabled", + "", # empty line + "daily.timer enabled", + ] + ) + + monkeypatch.setattr(s, "_run", fake_run) + result = s.list_enabled_timers() + assert result == ["apt-daily.timer", "daily.timer"] diff --git a/tests/test_validate.py b/tests/test_validate.py new file mode 100644 index 0000000..05ee88b --- /dev/null +++ b/tests/test_validate.py @@ -0,0 +1,413 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +import enroll.cli as cli +from enroll.validate import validate_harvest + + +def _base_state() -> dict: + return { + "enroll": {"version": "0.0.test", "harvest_time": 0}, + "host": { + "hostname": "testhost", + "os": "unknown", + "pkg_backend": "dpkg", + "os_release": {}, + }, + "inventory": {"packages": {}}, + "roles": { + "users": { + "role_name": "users", + "users": [], + "managed_dirs": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "services": [], + "packages": [], + "apt_config": { + "role_name": "apt_config", + "managed_dirs": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "dnf_config": { + "role_name": "dnf_config", + "managed_dirs": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "etc_custom": { + "role_name": "etc_custom", + "managed_dirs": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "usr_local_custom": { + "role_name": "usr_local_custom", + "managed_dirs": [], + "managed_files": [], + "excluded": [], + "notes": [], + }, + "extra_paths": { + "role_name": "extra_paths", + "include_patterns": [], + "exclude_patterns": [], + "managed_dirs": [], + "managed_files": [], + "managed_links": [], + "excluded": [], + "notes": [], + }, + }, + } + + +def _write_bundle(tmp_path: Path, state: dict) -> Path: + bundle = tmp_path / "bundle" + bundle.mkdir(parents=True) + (bundle / "artifacts").mkdir() + (bundle / "state.json").write_text(json.dumps(state, indent=2), encoding="utf-8") + return bundle + + +def test_validate_ok_bundle(tmp_path: Path): + state = _base_state() + state["roles"]["etc_custom"]["managed_files"].append( + { + "path": "/etc/hosts", + "src_rel": "etc/hosts", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "custom_specific_path", + } + ) + + bundle = _write_bundle(tmp_path, state) + art = bundle / "artifacts" / "etc_custom" / "etc" / "hosts" + art.parent.mkdir(parents=True, exist_ok=True) + art.write_text("127.0.0.1 localhost\n", encoding="utf-8") + + res = validate_harvest(str(bundle)) + assert res.ok + assert res.errors == [] + + +def test_validate_missing_artifact_is_error(tmp_path: Path): + state = _base_state() + state["roles"]["etc_custom"]["managed_files"].append( + { + "path": "/etc/hosts", + "src_rel": "etc/hosts", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "custom_specific_path", + } + ) + bundle = _write_bundle(tmp_path, state) + res = validate_harvest(str(bundle)) + assert not res.ok + assert any("missing artifact" in e for e in res.errors) + + +def test_validate_schema_error_is_reported(tmp_path: Path): + state = _base_state() + state["host"]["os"] = "not_a_real_os" + bundle = _write_bundle(tmp_path, state) + res = validate_harvest(str(bundle)) + assert not res.ok + assert any(e.startswith("schema /host/os") for e in res.errors) + + +def test_cli_validate_exits_1_on_validation_error(monkeypatch, tmp_path: Path): + state = _base_state() + state["roles"]["etc_custom"]["managed_files"].append( + { + "path": "/etc/hosts", + "src_rel": "etc/hosts", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "custom_specific_path", + } + ) + bundle = _write_bundle(tmp_path, state) + + monkeypatch.setattr(sys, "argv", ["enroll", "validate", str(bundle)]) + with pytest.raises(SystemExit) as e: + cli.main() + assert e.value.code == 1 + + +def test_cli_validate_exits_1_on_validation_warning_with_flag( + monkeypatch, tmp_path: Path +): + state = _base_state() + state["roles"]["etc_custom"]["managed_files"].append( + { + "path": "/etc/hosts", + "src_rel": "etc/hosts", + "owner": "root", + "group": "root", + "mode": "0644", + "reason": "custom_specific_path", + } + ) + + bundle = _write_bundle(tmp_path, state) + art = bundle / "artifacts" / "etc_custom" / "etc" / "hosts" + art.parent.mkdir(parents=True, exist_ok=True) + art.write_text("127.0.0.1 localhost\n", encoding="utf-8") + + art2 = bundle / "artifacts" / "etc_custom" / "etc" / "hosts2" + art2.write_text("hello\n", encoding="utf-8") + + monkeypatch.setattr( + sys, "argv", ["enroll", "validate", str(bundle), "--fail-on-warnings"] + ) + with pytest.raises(SystemExit) as e: + cli.main() + assert e.value.code == 1 + + +def test_validation_result_ok(): + from enroll.validate import ValidationResult + + result = ValidationResult(errors=[], warnings=[]) + assert result.ok is True + assert result.to_text() == "OK: harvest bundle validated\n" + + +def test_validation_result_with_errors(): + from enroll.validate import ValidationResult + + result = ValidationResult(errors=["error1", "error2"], warnings=[]) + assert result.ok is False + text = result.to_text() + assert "ERROR: 2 validation error(s)" in text + assert "error1" in text + assert "error2" in text + + +def test_validation_result_with_warnings(): + from enroll.validate import ValidationResult + + result = ValidationResult(errors=[], warnings=["warn1"]) + assert result.ok is True + text = result.to_text() + assert "WARN: 1 warning(s)" in text + assert "warn1" in text + + +def test_validation_result_to_dict(): + from enroll.validate import ValidationResult + + result = ValidationResult(errors=["e1"], warnings=["w1"]) + d = result.to_dict() + assert d["ok"] is False + assert d["errors"] == ["e1"] + assert d["warnings"] == ["w1"] + + +def test_iter_managed_files_singleton_roles(): + from enroll.validate import _iter_managed_files + + state = { + "roles": { + "users": {"managed_files": [{"path": "/etc/passwd", "src_rel": "passwd"}]}, + "packages": [ + { + "role_name": "vim", + "managed_files": [{"path": "/usr/bin/vim", "src_rel": "vim"}], + } + ], + } + } + files = _iter_managed_files(state) + assert len(files) == 2 + assert ("users", {"path": "/etc/passwd", "src_rel": "passwd"}) in files + + +def test_iter_managed_files_services_role(): + from enroll.validate import _iter_managed_files + + state = { + "roles": { + "services": [ + { + "role_name": "nginx", + "managed_files": [ + {"path": "/etc/nginx/nginx.conf", "src_rel": "nginx.conf"} + ], + } + ] + } + } + files = _iter_managed_files(state) + assert len(files) == 1 + assert files[0][0] == "nginx" + + +def test_iter_managed_files_handles_non_dict_items(): + from enroll.validate import _iter_managed_files + + state = { + "roles": { + "users": { + "managed_files": [ + "not_a_dict", + {"path": "/etc/passwd", "src_rel": "passwd"}, + ] + }, + "services": ["not_a_dict", {"role_name": "nginx", "managed_files": []}], + "packages": ["not_a_dict"], + } + } + files = _iter_managed_files(state) + assert len(files) == 1 + + +def test_iter_managed_files_empty_state(): + from enroll.validate import _iter_managed_files + + state = {"roles": {}} + files = _iter_managed_files(state) + assert files == [] + + +def test_validate_harvest_missing_state_json(tmp_path: Path): + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir() + result = validate_harvest(str(bundle_dir)) + assert result.ok is False + assert any("missing state.json" in e for e in result.errors) + + +def test_validate_harvest_invalid_json(tmp_path: Path): + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir() + state_file = bundle_dir / "state.json" + state_file.write_text("not valid json", encoding="utf-8") + result = validate_harvest(str(bundle_dir)) + assert result.ok is False + assert any("failed to parse" in e for e in result.errors) + + +def test_validate_harvest_schema_error(tmp_path: Path): + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir() + state_file = bundle_dir / "state.json" + state_file.write_text("{}", encoding="utf-8") + result = validate_harvest( + str(bundle_dir), schema="https://invalid.invalid/schema.json" + ) + assert result.ok is False + assert any("failed to load/validate schema" in e for e in result.errors) + + +def test_validate_harvest_missing_artifact(tmp_path: Path): + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir() + artifacts_dir = bundle_dir / "artifacts" + artifacts_dir.mkdir() + state = { + "roles": { + "users": {"managed_files": [{"path": "/etc/passwd", "src_rel": "passwd"}]} + } + } + state_file = bundle_dir / "state.json" + state_file.write_text(json.dumps(state), encoding="utf-8") + result = validate_harvest(str(bundle_dir)) + assert result.ok is False + assert any("missing artifact" in e for e in result.errors) + + +def test_validate_harvest_suspicious_src_rel(tmp_path: Path): + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir() + state = { + "roles": { + "users": { + "managed_files": [ + {"path": "/etc/passwd", "src_rel": "../../../etc/passwd"} + ] + } + } + } + state_file = bundle_dir / "state.json" + state_file.write_text(json.dumps(state), encoding="utf-8") + result = validate_harvest(str(bundle_dir)) + assert result.ok is False + assert any("suspicious src_rel" in e for e in result.errors) + + +def test_validate_harvest_missing_src_rel(tmp_path: Path): + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir() + state = {"roles": {"users": {"managed_files": [{"path": "/etc/passwd"}]}}} + state_file = bundle_dir / "state.json" + state_file.write_text(json.dumps(state), encoding="utf-8") + result = validate_harvest(str(bundle_dir)) + assert result.ok is False + assert any("missing src_rel" in e for e in result.errors) + + +def test_validate_harvest_firewall_runtime_missing(tmp_path: Path): + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir() + artifacts_dir = bundle_dir / "artifacts" + fw_dir = artifacts_dir / "firewall_runtime" + fw_dir.mkdir(parents=True) + state = { + "roles": { + "firewall_runtime": { + "role_name": "firewall_runtime", + "iptables_v4_save": "iptables.save", + } + } + } + state_file = bundle_dir / "state.json" + state_file.write_text(json.dumps(state), encoding="utf-8") + result = validate_harvest(str(bundle_dir)) + assert result.ok is False + assert any("missing firewall runtime artifact" in e for e in result.errors) + + +def test_validate_harvest_firewall_runtime_suspicious(tmp_path: Path): + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir() + state = { + "roles": { + "firewall_runtime": { + "role_name": "firewall_runtime", + "iptables_v4_save": "../../../etc/passwd", + } + } + } + state_file = bundle_dir / "state.json" + state_file.write_text(json.dumps(state), encoding="utf-8") + result = validate_harvest(str(bundle_dir)) + assert result.ok is False + assert any("suspicious src_rel" in e for e in result.errors) + + +def test_validate_harvest_no_schema_option(tmp_path: Path): + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir() + state_file = bundle_dir / "state.json" + state_file.write_text("invalid json", encoding="utf-8") + result = validate_harvest(str(bundle_dir), no_schema=True) + assert result.ok is False + assert any("failed to parse" in e for e in result.errors) diff --git a/tests/test_version_extra.py b/tests/test_version_extra.py new file mode 100644 index 0000000..67c1ce4 --- /dev/null +++ b/tests/test_version_extra.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +# The version module is hard to test fully because it uses importlib.metadata +# which is difficult to mock. We'll test what we can. + + +def test_get_enroll_version_returns_string(): + from enroll.version import get_enroll_version + + result = get_enroll_version() + assert isinstance(result, str) + assert len(result) > 0