Remove puppet and salt
This commit is contained in:
parent
88e9dba39a
commit
e9d7d74445
GPG key ID:
03906B4110AAD3B8
24 changed files with 256 additions and 7714 deletions
|
|
@ -35,13 +35,7 @@ jobs:
|
|||
apt-get update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl gnupg git tar gzip findutils bash nodejs procps \
|
||||
ansible ansible-lint python3 python3-venv python3-pip pipx systemctl python3-apt jq python3-jsonschema \
|
||||
puppet hiera
|
||||
curl -fsSL https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public | gpg --dearmor | tee /etc/apt/keyrings/salt-archive-keyring.pgp > /dev/null
|
||||
curl -fsSL https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources | tee /etc/apt/sources.list.d/salt.sources
|
||||
apt-get update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
salt-master salt-minion salt-ssh salt-syndic salt-cloud salt-api
|
||||
ansible ansible-lint python3 python3-venv python3-pip pipx systemctl python3-apt jq python3-jsonschema
|
||||
;;
|
||||
almalinux)
|
||||
dnf -y upgrade --refresh
|
||||
|
|
@ -49,15 +43,10 @@ jobs:
|
|||
ca-certificates curl-minimal gnupg2 git tar gzip findutils bash which jq nodejs procps-ng \
|
||||
dnf-plugins-core epel-release
|
||||
dnf -y config-manager --set-enabled crb || true
|
||||
curl -fsSL https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo > /etc/yum.repos.d/salt.repo
|
||||
dnf -y install https://yum.puppet.com/puppet8-release-el-9.noarch.rpm
|
||||
dnf -y makecache
|
||||
dnf -y install \
|
||||
python3.11 python3.11-devel python3.11-pip gcc make \
|
||||
ansible-core ansible-lint systemd rpm httpd \
|
||||
puppet-agent \
|
||||
salt-master salt-minion salt-ssh salt-syndic salt-cloud salt-api
|
||||
echo "/opt/puppetlabs/bin" >> "$GITHUB_PATH"
|
||||
ansible-core ansible-lint systemd rpm httpd
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported CI distro: ${DISTRO}" >&2
|
||||
|
|
|
|||
770
DEVELOPMENT.md
770
DEVELOPMENT.md
|
|
@ -21,25 +21,21 @@ Harvest bundle
|
|||
state.json
|
||||
artifacts/<role>/<path-relative-to-root>
|
||||
|
|
||||
| enroll manifest --target ansible|puppet|salt
|
||||
| enroll manifest
|
||||
v
|
||||
Generated configuration-management output
|
||||
Ansible roles/playbook
|
||||
Puppet modules/site.pp/Hiera data
|
||||
Salt states/pillar data
|
||||
```
|
||||
|
||||
The harvest bundle is deliberately target-neutral. Ansible, Puppet, and Salt renderers all consume the same `state.json` shape and the same harvested artifacts. Renderer code should translate harvest state into the target's idioms; it should not invent source facts that belong in the harvest.
|
||||
The harvest bundle is deliberately target-neutral. Ansible renderer consumes the same `state.json` shape and the same harvested artifacts. Renderer code should translate harvest state into the target's idioms; it should not invent source facts that belong in the harvest.
|
||||
|
||||
`enroll diff` is also built around harvest bundles. It compares two harvests and, when `--enforce` is requested, can generate a temporary manifest from the old harvest and apply it locally with the selected target:
|
||||
|
||||
```bash
|
||||
enroll diff --old ./baseline --new ./current --enforce --target ansible
|
||||
enroll diff --old ./baseline --new ./current --enforce --target puppet
|
||||
enroll diff --old ./baseline --new ./current --enforce --target salt
|
||||
enroll diff --old ./baseline --new ./current --enforce
|
||||
```
|
||||
|
||||
For enforcement, the user is responsible for having the chosen local apply tool on `PATH`: `ansible-playbook`, `puppet`, or `salt-call`.
|
||||
For enforcement, the user is responsible for having the chosen local apply tool on `PATH`: `ansible-playbook`.
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -67,8 +63,6 @@ enroll/
|
|||
|
||||
manifest.py target router and SOPS manifest wrapper
|
||||
ansible.py Ansible renderer
|
||||
puppet.py Puppet renderer
|
||||
salt.py Salt renderer
|
||||
cm.py renderer-neutral CMModule model and grouping helpers
|
||||
role_names.py reserved singleton role-name protection
|
||||
|
||||
|
|
@ -131,7 +125,7 @@ The supported subcommands are:
|
|||
|
||||
```text
|
||||
harvest collect a harvest bundle from a local or remote host
|
||||
manifest generate Ansible/Puppet/Salt output from a harvest bundle
|
||||
manifest generate Ansible output from a harvest bundle
|
||||
single-shot run harvest and manifest in one command
|
||||
diff compare two harvest bundles and optionally enforce old state
|
||||
explain produce a human/JSON explanation of a harvest
|
||||
|
|
@ -156,8 +150,8 @@ flowchart TD
|
|||
F --> G[diff.format_report]
|
||||
F --> H{--enforce?}
|
||||
H -->|yes| I[diff.enforce_old_harvest]
|
||||
I --> J[manifest.manifest target=ansible|puppet|salt]
|
||||
J --> K[ansible-playbook or puppet apply or salt-call]
|
||||
I --> J[manifest.manifest]
|
||||
J --> K[ansible-playbook]
|
||||
B -->|explain| L[explain.explain_state]
|
||||
B -->|validate| M[validate.validate_harvest]
|
||||
```
|
||||
|
|
@ -172,10 +166,10 @@ harvest.py
|
|||
depends on harvest_collectors, platform backends, capture policy, system scanners
|
||||
|
||||
manifest.py
|
||||
depends on ansible.py, puppet.py, salt.py
|
||||
depends on ansible.py
|
||||
|
||||
ansible.py / puppet.py / salt.py
|
||||
depend on state.py, cm.py, harvested artifacts, and target-specific helpers
|
||||
ansible.py
|
||||
depends on state.py, cm.py, harvested artifacts, and helpers
|
||||
```
|
||||
|
||||
---
|
||||
|
|
@ -362,8 +356,6 @@ This is one of the most important invariants in the project:
|
|||
|
||||
> A destination path should normally appear in only one generated role.
|
||||
|
||||
Puppet and Salt also run `cm.resolve_catalog_conflicts()` after renderer role collection because they compile a single global catalog and duplicate resources are hard failures.
|
||||
|
||||
---
|
||||
|
||||
## 7. File capture and safety policy
|
||||
|
|
@ -595,7 +587,7 @@ artifacts/firewall_runtime/firewall/iptables.v4
|
|||
artifacts/firewall_runtime/firewall/iptables.v6
|
||||
```
|
||||
|
||||
Renderers should only create a firewall runtime role when at least one runtime artifact exists. When firewall runtime is rendered, Ansible/Puppet/Salt also create an `enroll_runtime` role/module/state to own `/etc/enroll` before `/etc/enroll/firewall`.
|
||||
Renderers should only create a firewall runtime role when at least one runtime artifact exists. When firewall runtime is rendered, Ansible also creates an `enroll_runtime` role/module/state to own `/etc/enroll` before `/etc/enroll/firewall`.
|
||||
|
||||
### 9.2 `CronLogrotateCollector`
|
||||
|
||||
|
|
@ -795,9 +787,7 @@ manifest(
|
|||
Plain mode dispatches to:
|
||||
|
||||
```text
|
||||
target=ansible -> ansible.manifest_from_bundle_dir(..., jinjaturtle=..., no_common_roles=...)
|
||||
target=puppet -> puppet.manifest_from_bundle_dir(..., jinjaturtle=..., no_common_roles=...)
|
||||
target=salt -> salt.manifest_from_bundle_dir(..., jinjaturtle=..., no_common_roles=...)
|
||||
ansible.manifest_from_bundle_dir(..., jinjaturtle=..., no_common_roles=...)
|
||||
```
|
||||
|
||||
SOPS mode:
|
||||
|
|
@ -819,7 +809,7 @@ Before dispatching to a renderer, `manifest.manifest()` calls `validate.validate
|
|||
|
||||
File: `cm.py`
|
||||
|
||||
`CMModule` is the shared resource model used heavily by Puppet and Salt and partially by Ansible.
|
||||
`CMModule` is the shared resource model used by Ansible and perhaps other renderers in the future.
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
|
|
@ -866,22 +856,6 @@ normal manifest, no --no-common-roles: group package/service roles
|
|||
|
||||
`--fqdn` implies no common roles because host-specific output should preserve per-host state rather than merging unrelated resources into shared roles.
|
||||
|
||||
### 12.2 Catalog conflict resolution
|
||||
|
||||
`resolve_catalog_conflicts()` runs for Puppet and Salt.
|
||||
|
||||
It removes duplicates across generated modules/states for:
|
||||
|
||||
- packages,
|
||||
- groups,
|
||||
- users,
|
||||
- directories,
|
||||
- files,
|
||||
- symlinks,
|
||||
- services.
|
||||
|
||||
It also removes directory resources that conflict with a file or link at the same path. This matters because Puppet and Salt compile a single catalog; duplicates that Ansible might tolerate can fail hard there.
|
||||
|
||||
---
|
||||
|
||||
## 13. Ansible renderer
|
||||
|
|
@ -981,8 +955,6 @@ Ansible playbook roles are ordered intentionally:
|
|||
|
||||
Generated playbooks tag roles with `role_<safe_role_name>`. `diff --enforce --target ansible` uses these tags to narrow enforcement to roles relevant to the drift report when it can.
|
||||
|
||||
Puppet and Salt enforcement do not currently narrow to per-role tags; they run the full generated local manifest/state tree.
|
||||
|
||||
### 13.5 Ansible and JinjaTurtle
|
||||
|
||||
Ansible uses `jinjaturtle.jinjify_managed_files()`.
|
||||
|
|
@ -996,236 +968,7 @@ If JinjaTurtle is unavailable in `auto` mode, fails, emits missing variables, or
|
|||
|
||||
---
|
||||
|
||||
## 14. Puppet renderer
|
||||
|
||||
File: `puppet.py`
|
||||
|
||||
Entry point:
|
||||
|
||||
```python
|
||||
puppet.manifest_from_bundle_dir(
|
||||
bundle_dir,
|
||||
out_dir,
|
||||
fqdn=None,
|
||||
no_common_roles=False,
|
||||
jinjaturtle="auto",
|
||||
)
|
||||
```
|
||||
|
||||
It instantiates `PuppetManifestRenderer(...).render()`.
|
||||
|
||||
### 14.1 Puppet render flow
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[PuppetManifestRenderer.render] --> B[PuppetRole.load_state]
|
||||
B --> C[resolve_jinjaturtle_mode]
|
||||
C --> D[_collect_puppet_roles]
|
||||
D --> E[resolve_catalog_conflicts]
|
||||
E --> F[_sync_service_notifications]
|
||||
F --> G[write modules/<module>/manifests/init.pp]
|
||||
G --> H[write metadata.json]
|
||||
H --> I{fqdn?}
|
||||
I -->|no| J[write manifests/site.pp with node default]
|
||||
I -->|yes| K[write hiera.yaml]
|
||||
K --> L[write data/nodes/<fqdn>.yaml]
|
||||
L --> M[write Hiera-driven site.pp]
|
||||
J --> N[README.md]
|
||||
M --> N
|
||||
```
|
||||
|
||||
### 14.2 `PuppetRole`
|
||||
|
||||
`PuppetRole` extends `CMModule` and converts snapshots into Puppet-friendly resources. It handles:
|
||||
|
||||
- packages,
|
||||
- users and groups,
|
||||
- managed dirs/files/symlinks,
|
||||
- services,
|
||||
- sysctl apply execs,
|
||||
- Flatpak remotes/apps via guarded `exec`,
|
||||
- Snap installs via guarded `exec`,
|
||||
- Docker/Podman images by digest via guarded `exec`,
|
||||
- firewall runtime files and refresh-only restore execs,
|
||||
- JinjaTurtle ERB templates and class/Hiera parameter values.
|
||||
|
||||
`_puppet_name()` sanitises module names and avoids Puppet reserved words such as `default`, `class`, `node`, `site`, and `init`.
|
||||
|
||||
### 14.3 Output layout
|
||||
|
||||
Default mode:
|
||||
|
||||
```text
|
||||
<out>/
|
||||
manifests/site.pp
|
||||
README.md
|
||||
modules/
|
||||
<module>/
|
||||
metadata.json
|
||||
manifests/init.pp
|
||||
files/...
|
||||
templates/...
|
||||
```
|
||||
|
||||
Default `site.pp` includes generated classes in manifest order under a `node default` or named node block.
|
||||
|
||||
### 14.4 Puppet `--fqdn` / Hiera mode
|
||||
|
||||
When `--fqdn` is supplied, Puppet output switches to Hiera-style node data:
|
||||
|
||||
```text
|
||||
<out>/
|
||||
hiera.yaml
|
||||
manifests/site.pp
|
||||
data/
|
||||
common.yaml
|
||||
nodes/<fqdn>.yaml
|
||||
modules/
|
||||
<module>/
|
||||
metadata.json
|
||||
manifests/init.pp
|
||||
files/nodes/<fqdn>/...
|
||||
templates/...
|
||||
```
|
||||
|
||||
In this mode:
|
||||
|
||||
- `site.pp` includes classes from Hiera key `enroll::classes`,
|
||||
- `data/nodes/<fqdn>.yaml` contains class list and parameter data,
|
||||
- module classes are data-driven via Automatic Parameter Lookup,
|
||||
- node-specific raw file artifacts live under `modules/<module>/files/nodes/<fqdn>/...`,
|
||||
- JinjaTurtle ERB template values are written into node Hiera data.
|
||||
|
||||
Re-running Enroll with another `--fqdn` into the same output directory is intended to add or replace that node's YAML without deleting existing node data.
|
||||
|
||||
### 14.5 Puppet and JinjaTurtle
|
||||
|
||||
Puppet now participates in the shared JinjaTurtle integration.
|
||||
|
||||
When enabled, Puppet calls `jinjaturtle` with ERB-specific options:
|
||||
|
||||
```text
|
||||
--template-engine erb
|
||||
--puppet-class <module_name>
|
||||
```
|
||||
|
||||
The resulting template is written under:
|
||||
|
||||
```text
|
||||
modules/<module>/templates/<src_rel>.erb
|
||||
```
|
||||
|
||||
Static single-node mode renders class parameters with defaults and uses:
|
||||
|
||||
```puppet
|
||||
content => template('<module>/<src_rel>.erb')
|
||||
```
|
||||
|
||||
Hiera mode writes template parameter values into `data/nodes/<fqdn>.yaml` and renders data-driven file resources.
|
||||
|
||||
`jinjaturtle.missing_erb_template_vars()` checks that ERB instance variables such as `@main_key` have matching context/Hiera data. If variables are missing, Enroll falls back to raw file copying rather than emitting a broken Puppet template.
|
||||
|
||||
---
|
||||
|
||||
## 15. Salt renderer
|
||||
|
||||
File: `salt.py`
|
||||
|
||||
Entry point:
|
||||
|
||||
```python
|
||||
salt.manifest_from_bundle_dir(
|
||||
bundle_dir,
|
||||
out_dir,
|
||||
fqdn=None,
|
||||
no_common_roles=False,
|
||||
jinjaturtle="auto",
|
||||
)
|
||||
```
|
||||
|
||||
It instantiates `SaltManifestRenderer(...).render()`.
|
||||
|
||||
### 15.1 Salt render flow
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[SaltManifestRenderer.render] --> B[SaltRole.load_state]
|
||||
B --> C[resolve_jinjaturtle_mode]
|
||||
C --> D[_collect_salt_roles]
|
||||
D --> E[resolve_catalog_conflicts]
|
||||
E --> F[write states/roles/<role>/init.sls]
|
||||
F --> G{fqdn?}
|
||||
G -->|no| H[write states/top.sls target '*']
|
||||
G -->|yes| I[write pillar node data]
|
||||
I --> J[write states/top.sls and pillar/top.sls]
|
||||
H --> K[write config/master.d/enroll.conf]
|
||||
J --> K
|
||||
K --> L[README.md]
|
||||
```
|
||||
|
||||
### 15.2 `SaltRole`
|
||||
|
||||
`SaltRole` extends `CMModule` and changes `managed_owner_attr` to `user`, because Salt `file.managed` uses `user` rather than `owner`.
|
||||
|
||||
It prepares:
|
||||
|
||||
- packages as `pkg.installed`,
|
||||
- groups as `group.present`,
|
||||
- users as `user.present`,
|
||||
- dirs/files/symlinks as Salt `file.*` states,
|
||||
- services as `service.running` or `service.dead`,
|
||||
- Flatpaks/Snaps via guarded `cmd.run`,
|
||||
- Docker/Podman images via guarded `cmd.run`,
|
||||
- firewall runtime restore commands,
|
||||
- optional Jinja templates for managed files.
|
||||
|
||||
### 15.3 Output layout
|
||||
|
||||
Default mode:
|
||||
|
||||
```text
|
||||
<out>/
|
||||
README.md
|
||||
config/master.d/enroll.conf
|
||||
states/
|
||||
top.sls
|
||||
roles/<role>/
|
||||
init.sls
|
||||
files/...
|
||||
templates/...
|
||||
```
|
||||
|
||||
`--fqdn` mode:
|
||||
|
||||
```text
|
||||
<out>/
|
||||
states/
|
||||
top.sls
|
||||
roles/<role>/init.sls
|
||||
pillar/
|
||||
top.sls
|
||||
nodes/<sanitised-fqdn>_<digest>.sls
|
||||
```
|
||||
|
||||
The Salt renderer can accumulate node data in `--fqdn` mode and preserves existing top data where appropriate.
|
||||
|
||||
### 15.4 Salt and JinjaTurtle
|
||||
|
||||
Salt uses `jinjaturtle.jinjify_artifact()` directly. When successful, a managed file becomes a Salt `file.managed` with:
|
||||
|
||||
```yaml
|
||||
source: salt://roles/<role>/templates/<src_rel>.j2
|
||||
template: jinja
|
||||
context: {...}
|
||||
```
|
||||
|
||||
Salt has one additional compatibility step: `_saltify_jinjaturtle_template()` rewrites Ansible-oriented `to_json(...)` filters emitted by JinjaTurtle into Salt-safe context variables or `tojson` filters.
|
||||
|
||||
If templating fails or is unsupported, the renderer falls back to a literal file copy under `files/`.
|
||||
|
||||
---
|
||||
|
||||
## 16. Shared JinjaTurtle integration
|
||||
## 14. JinjaTurtle integration
|
||||
|
||||
File: `jinjaturtle.py`
|
||||
|
||||
|
|
@ -1270,27 +1013,24 @@ jinjify_artifact(
|
|||
template_root,
|
||||
jt_exe=...,
|
||||
jt_enabled=...,
|
||||
template_engine="jinja2" | "erb",
|
||||
puppet_class=..., # Puppet only
|
||||
)
|
||||
```
|
||||
|
||||
Ansible uses `jinjify_managed_files()` because it merges variables into role defaults or host vars. Salt uses `jinjify_artifact()` directly because context lives with each `file.managed`. Puppet uses `jinjify_artifact(..., template_engine="erb", puppet_class=<module>)` so variables line up with Puppet class/Hiera names.
|
||||
Ansible uses `jinjify_managed_files()` because it merges variables into role defaults or host vars.
|
||||
|
||||
Safety checks:
|
||||
|
||||
- `missing_jinja_template_vars()` rejects Jinja2 templates that reference absent variables.
|
||||
- `missing_erb_template_vars()` rejects ERB templates that reference absent Puppet/Hiera variables.
|
||||
|
||||
When checks fail, Enroll deletes obsolete generated templates when appropriate and falls back to raw file copying.
|
||||
|
||||
---
|
||||
|
||||
## 17. Diff, notifications, and enforcement
|
||||
## 15. Diff, notifications, and enforcement
|
||||
|
||||
File: `diff.py`
|
||||
|
||||
### 17.1 Inputs
|
||||
### 15.1 Inputs
|
||||
|
||||
`compare_harvests()` accepts:
|
||||
|
||||
|
|
@ -1301,7 +1041,7 @@ File: `diff.py`
|
|||
|
||||
Bundle resolution is handled by `_bundle_from_input()`, which reuses `remote._safe_extract_tar()` for tarball extraction.
|
||||
|
||||
### 17.2 What diff compares
|
||||
### 15.2 What diff compares
|
||||
|
||||
`compare_harvests()` compares:
|
||||
|
||||
|
|
@ -1322,7 +1062,7 @@ Reports are formatted by:
|
|||
format_report(report, fmt="text" | "markdown" | "json")
|
||||
```
|
||||
|
||||
### 17.3 Enforcement decision
|
||||
### 15.3 Enforcement decision
|
||||
|
||||
`has_enforceable_drift()` is intentionally conservative.
|
||||
|
||||
|
|
@ -1343,29 +1083,7 @@ Not enforceable:
|
|||
|
||||
This keeps `--enforce` focused on restoring baseline state rather than deleting unknown current state or downgrading packages.
|
||||
|
||||
### 17.4 Target-selected enforcement
|
||||
|
||||
`enforce_old_harvest()` now accepts `target="ansible" | "puppet" | "salt"`.
|
||||
|
||||
It performs:
|
||||
|
||||
1. resolve the old/baseline harvest,
|
||||
2. build a best-effort enforcement plan from the diff report,
|
||||
3. generate a temporary manifest from the old harvest using the selected target,
|
||||
4. run the matching local apply tool,
|
||||
5. attach enforcement metadata to the diff report.
|
||||
|
||||
Target commands:
|
||||
|
||||
```text
|
||||
ansible -> ansible-playbook -i localhost, -c local playbook.yml
|
||||
puppet -> puppet apply --modulepath ./modules [--hiera_config ./hiera.yaml] manifests/site.pp
|
||||
salt -> salt-call --local --file-root ./states [--pillar-root ./pillar] state.apply
|
||||
```
|
||||
|
||||
Only Ansible uses generated per-role tags to narrow the apply scope. Puppet and Salt enforcement deliberately run the full generated local manifest/state tree for now. The JSON report keeps target-specific compatibility fields such as `ansible_playbook`, `puppet`, or `salt_call`.
|
||||
|
||||
### 17.5 Notifications
|
||||
### 15.4 Notifications
|
||||
|
||||
`diff.py` also supports webhooks and email notifications:
|
||||
|
||||
|
|
@ -1376,9 +1094,9 @@ CLI notification options are only sent when differences exist unless `--notify-a
|
|||
|
||||
---
|
||||
|
||||
## 18. Explanation and validation
|
||||
## 16. Explanation and validation
|
||||
|
||||
### 18.1 `explain.py`
|
||||
### 16.1 `explain.py`
|
||||
|
||||
`explain_state()` reads a harvest and produces text or JSON explaining:
|
||||
|
||||
|
|
@ -1395,7 +1113,7 @@ CLI notification options are only sent when differences exist unless `--notify-a
|
|||
|
||||
This is intended to answer “what did Enroll collect and why?”
|
||||
|
||||
### 18.2 `validate.py`
|
||||
### 16.2 `validate.py`
|
||||
|
||||
`validate_harvest()` checks:
|
||||
|
||||
|
|
@ -1411,7 +1129,7 @@ This is intended to answer “what did Enroll collect and why?”
|
|||
`validate_harvest()` is used in three important contexts:
|
||||
|
||||
- `enroll validate` exposes the checks directly to users.
|
||||
- `manifest.manifest()` validates before rendering Ansible/Puppet/Salt output.
|
||||
- `manifest.manifest()` validates before rendering Ansible output.
|
||||
- `diff.compare_harvests()` validates both input bundles before comparing them, using `no_schema=True` so older harvests can still be inspected while artifact safety checks remain active.
|
||||
|
||||
`diff --enforce` renders the old harvest through `manifest.manifest()`, so enforcement also passes through manifest-time validation before a local apply tool is invoked.
|
||||
|
|
@ -1422,7 +1140,7 @@ The CLI supports local schema override with `--schema`, warning failure with `--
|
|||
|
||||
---
|
||||
|
||||
## 19. Remote harvesting
|
||||
## 17. Remote harvesting
|
||||
|
||||
File: `remote.py`
|
||||
|
||||
|
|
@ -1441,7 +1159,7 @@ It wraps `_remote_harvest()` and handles:
|
|||
- retrying when remote sudo requires a password,
|
||||
- retrying when an encrypted SSH private key needs a passphrase.
|
||||
|
||||
### 19.1 Remote harvest flow
|
||||
### 17.1 Remote harvest flow
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
|
|
@ -1460,7 +1178,7 @@ flowchart TD
|
|||
|
||||
`_build_enroll_pyz()` packages the local `enroll` Python package into a zipapp and uses `enroll.cli:main` as its entry point.
|
||||
|
||||
### 19.2 SSH config support
|
||||
### 17.2 SSH config support
|
||||
|
||||
`--remote-ssh-config` enables Paramiko `SSHConfig` support for settings such as:
|
||||
|
||||
|
|
@ -1475,7 +1193,7 @@ flowchart TD
|
|||
|
||||
Unknown host keys are rejected by default through Paramiko's reject policy. Users should have valid host keys in known hosts.
|
||||
|
||||
### 19.3 Safe tar extraction
|
||||
### 17.3 Safe tar extraction
|
||||
|
||||
`_safe_extract_tar()` validates tar members before extraction and rejects:
|
||||
|
||||
|
|
@ -1490,13 +1208,13 @@ This helper is reused by remote harvest, manifest SOPS extraction, validate/diff
|
|||
|
||||
---
|
||||
|
||||
## 20. SOPS support
|
||||
## 18. SOPS support
|
||||
|
||||
File: `sopsutil.py`
|
||||
|
||||
SOPS support is binary tarball encryption, not field-level YAML encryption.
|
||||
|
||||
### 20.1 Harvest SOPS mode
|
||||
### 18.1 Harvest SOPS mode
|
||||
|
||||
`enroll harvest --sops <fingerprint...>`:
|
||||
|
||||
|
|
@ -1505,7 +1223,7 @@ SOPS support is binary tarball encryption, not field-level YAML encryption.
|
|||
3. encrypts it with SOPS binary mode,
|
||||
4. writes `harvest.tar.gz.sops` or the requested output file.
|
||||
|
||||
### 20.2 Manifest SOPS mode
|
||||
### 18.2 Manifest SOPS mode
|
||||
|
||||
`enroll manifest --sops <fingerprint...>`:
|
||||
|
||||
|
|
@ -1514,7 +1232,7 @@ SOPS support is binary tarball encryption, not field-level YAML encryption.
|
|||
3. tars the generated output,
|
||||
4. encrypts it as a single SOPS file.
|
||||
|
||||
### 20.3 Helpers
|
||||
### 18.3 Helpers
|
||||
|
||||
`sopsutil.py` provides:
|
||||
|
||||
|
|
@ -1527,7 +1245,7 @@ Encryption/decryption helpers write via temp files and default to mode `0600`.
|
|||
|
||||
---
|
||||
|
||||
## 21. Configuration file support
|
||||
## 19. Configuration file support
|
||||
|
||||
`cli.py` supports optional INI config files.
|
||||
|
||||
|
|
@ -1551,33 +1269,21 @@ The translation is argparse-driven, so new flags often gain config-file support
|
|||
|
||||
---
|
||||
|
||||
## 22. CLI flags that affect multiple layers
|
||||
## 20. CLI flags that affect multiple layers
|
||||
|
||||
### 22.1 `--target`
|
||||
|
||||
`--target ansible|puppet|salt` exists for:
|
||||
|
||||
- `enroll manifest`,
|
||||
- `enroll single-shot`,
|
||||
- `enroll diff --enforce`.
|
||||
|
||||
For `manifest` and `single-shot`, it chooses the output renderer. For `diff --enforce`, it chooses both the temporary manifest target and the local apply tool.
|
||||
|
||||
### 22.2 `--fqdn`
|
||||
### 20.1 `--fqdn`
|
||||
|
||||
`--fqdn` changes output semantics, not just filenames:
|
||||
|
||||
- Ansible: uses inventory/host_vars and host-specific artifacts.
|
||||
- Puppet: uses Hiera node data and Hiera-driven classes.
|
||||
- Salt: uses pillar node data and minion-targeted top files.
|
||||
|
||||
`--fqdn` implies no common role grouping.
|
||||
|
||||
### 22.3 `--no-common-roles`
|
||||
### 20.2 `--no-common-roles`
|
||||
|
||||
Disables the default grouping of package/service snapshots by Debian Section or RPM Group. This preserves one generated role/module/state per package or unit snapshot.
|
||||
|
||||
### 22.4 `--jinjaturtle` / `--no-jinjaturtle`
|
||||
### 20.3 `--jinjaturtle` / `--no-jinjaturtle`
|
||||
|
||||
The CLI maps these to renderer mode strings:
|
||||
|
||||
|
|
@ -1587,11 +1293,9 @@ no flag -> auto
|
|||
--no-jinjaturtle -> off
|
||||
```
|
||||
|
||||
All three manifest targets receive this mode. Puppet uses ERB when JinjaTurtle is enabled; Ansible and Salt use Jinja2.
|
||||
|
||||
---
|
||||
|
||||
## 23. Tests and how to navigate them
|
||||
## 21. Tests and how to navigate them
|
||||
|
||||
Run tests with:
|
||||
|
||||
|
|
@ -1606,402 +1310,8 @@ or the repository helper when appropriate:
|
|||
./tests.sh
|
||||
```
|
||||
|
||||
Important test files:
|
||||
|
||||
| Test file | What it covers |
|
||||
|---|---|
|
||||
| `test_cli.py` | argparse dispatch, remote flags, manifest target forwarding, single-shot flow. |
|
||||
| `test_cli_config_and_sops.py`, `test_cli_helpers.py` | config-file injection and SOPS output helpers. |
|
||||
| `test_harvest.py`, `test_harvest_helpers.py` | harvest orchestration, sysctl/firewall helpers, role naming. |
|
||||
| `test_harvest_collectors.py` | runtime and container image collectors. |
|
||||
| `test_harvest_cron_logrotate.py` | cron/logrotate unification. |
|
||||
| `test_harvest_symlinks.py` | nginx/apache enabled symlink capture. |
|
||||
| `test_accounts.py` | users, Flatpak, Snap parsing/discovery. |
|
||||
| `test_ignore.py`, `test_ignore_dir.py` | secret/noise policy. |
|
||||
| `test_pathfilter.py` | include/exclude matching and expansion. |
|
||||
| `test_platform.py`, `test_platform_backends.py` | platform detection and backend behaviour. |
|
||||
| `test_debian.py`, `test_rpm.py`, `test_rpm_run.py` | package manager helpers. |
|
||||
| `test_manifest.py`, `test_manifest_ansible.py` | Ansible rendering and role behaviour. |
|
||||
| `test_manifest_puppet.py` | Puppet rendering, Hiera mode, reserved names, firewall/container/Flatpak/Snap/JinjaTurtle support. |
|
||||
| `test_manifest_salt.py` | Salt rendering, pillar mode, JinjaTurtle, firewall/container/Flatpak/Snap support. |
|
||||
| `test_manifest_symlinks.py` | symlink manifest output. |
|
||||
| `test_jinjaturtle.py` | shared template generation and fallback safety. |
|
||||
| `test_diff_bundle.py`, `test_diff_ignore_versions_exclude_enforce.py`, `test_diff_notifications.py` | diff, bundle resolution, target-selected enforcement, notifications. |
|
||||
| `test_remote.py` | remote harvest, SSH/sudo prompts, safe tar extraction. |
|
||||
| `test_explain.py` | harvest explanation output. |
|
||||
| `test_validate.py` | schema/artifact validation. |
|
||||
| `test_cm.py` | `CMModule` conflict resolution and service-package helpers. |
|
||||
| `test_fsutil.py`, `test_fsutil_extra.py` | file hashing and stat metadata helpers. |
|
||||
|
||||
When changing behaviour, extend the closest specific tests rather than relying only on broad integration tests.
|
||||
|
||||
---
|
||||
|
||||
## 24. Common maintenance tasks
|
||||
|
||||
### 24.1 Add a new thing to harvest
|
||||
|
||||
1. Add or extend a dataclass in `harvest_types.py` if existing snapshots cannot represent it.
|
||||
2. Add a collector under `harvest_collectors/` if it is a distinct feature.
|
||||
3. Add the collector to the sequence in `harvest.harvest()`.
|
||||
4. Add the snapshot to the `state = {...}` object in `harvest.harvest()`.
|
||||
5. Update `schema/state.schema.json`.
|
||||
6. Update renderers that should emit the new resource.
|
||||
7. Update `explain.py` and `validate.py` if users need visibility or artifact checks.
|
||||
8. Add tests for harvest and each renderer.
|
||||
|
||||
### 24.2 Add a new renderer target
|
||||
|
||||
1. Create `<target>.py` with `manifest_from_bundle_dir()`.
|
||||
2. Load state via `CMModule.load_state()` or `state.load_state()`.
|
||||
3. Consume `roles_from_state()` and `inventory_packages_from_state()`.
|
||||
4. Convert snapshots into renderer-specific role/module/state objects.
|
||||
5. Reuse `CMModule.package_service_entries()` for package/service grouping.
|
||||
6. Run conflict resolution if the target compiles a global catalog.
|
||||
7. Write target output and README.
|
||||
8. Add the target to `manifest.manifest()` validation and dispatch.
|
||||
9. Add CLI choices in `_add_common_manifest_args()` and diff enforcement if applicable.
|
||||
10. Add tests.
|
||||
|
||||
### 24.3 Add a new CLI flag
|
||||
|
||||
For harvest-affecting flags:
|
||||
|
||||
1. add the flag to `cli.py` for `harvest` and possibly `single-shot`,
|
||||
2. forward it to `harvest.harvest()` or `remote.remote_harvest()`,
|
||||
3. forward it through remote command construction if remote mode needs it,
|
||||
4. check whether config-file injection handles it,
|
||||
5. add tests in `test_cli.py` and feature-specific tests.
|
||||
|
||||
For manifest-affecting flags:
|
||||
|
||||
1. add it to `_add_common_manifest_args()` if all manifest-like commands need it,
|
||||
2. forward it through `manifest.manifest()`,
|
||||
3. forward it to target renderers,
|
||||
4. add tests for forwarding and output.
|
||||
|
||||
For diff enforcement flags:
|
||||
|
||||
1. add argparse support under the `diff` subparser,
|
||||
2. pass values to `compare_harvests()` or `enforce_old_harvest()`,
|
||||
3. update report formatting if new fields appear,
|
||||
4. add tests in `test_diff_ignore_versions_exclude_enforce.py` or `test_diff_notifications.py`.
|
||||
|
||||
### 24.4 Change file safety rules
|
||||
|
||||
Modify `ignore.py` and add tests in `test_ignore.py` / `test_ignore_dir.py`.
|
||||
|
||||
Be careful:
|
||||
|
||||
- relaxing safety affects secret exposure risk,
|
||||
- tightening safety can make expected config disappear,
|
||||
- binary allowance matters for APT/RPM keyrings,
|
||||
- `--dangerous` must remain explicit for risky harvesting.
|
||||
|
||||
### 24.5 Change service/package attribution
|
||||
|
||||
Most logic is in:
|
||||
|
||||
- `harvest_collectors/services.py`,
|
||||
- `package_hints.py`,
|
||||
- `system_paths.py`,
|
||||
- package backend `modified_paths()` implementations.
|
||||
|
||||
Preserve these invariants:
|
||||
|
||||
- cron/logrotate should stay unified when installed,
|
||||
- shared directories should not be attributed too broadly,
|
||||
- package-manager config belongs in `apt_config`/`dnf_config`,
|
||||
- `captured_global` should prevent duplicates,
|
||||
- stopped services should not receive broad restart notifications.
|
||||
|
||||
### 24.6 Change manifest role grouping
|
||||
|
||||
Common grouping uses:
|
||||
|
||||
- `CMModule.package_service_entries()`,
|
||||
- `package_section_label()`,
|
||||
- `section_label_for_packages()`.
|
||||
|
||||
Remember:
|
||||
|
||||
- default non-`--fqdn` output groups package/service roles unless `--no-common-roles` is set,
|
||||
- `--fqdn` implies per-role output,
|
||||
- Ansible, Puppet, and Salt grouping should stay conceptually aligned,
|
||||
- Puppet/Salt need `resolve_catalog_conflicts()` after grouping.
|
||||
|
||||
### 24.7 Change JinjaTurtle support
|
||||
|
||||
Shared path support and safety checks belong in `jinjaturtle.py`.
|
||||
|
||||
Renderer-specific behaviour belongs in the renderer:
|
||||
|
||||
- Ansible: variables in defaults or host vars, templates under role `templates/`.
|
||||
- Puppet: ERB templates, class params or Hiera values.
|
||||
- Salt: `file.managed` context and Salt-safe Jinja rewrites.
|
||||
|
||||
Fallback-to-raw-copy is part of the product contract unless JinjaTurtle was explicitly required and missing.
|
||||
|
||||
### 24.8 Change diff enforcement
|
||||
|
||||
`diff --enforce` now has a target dimension.
|
||||
|
||||
When changing it, keep these distinctions clear:
|
||||
|
||||
- `has_enforceable_drift()` decides whether enforcement should run.
|
||||
- `_enforcement_plan()` finds relevant baseline roles.
|
||||
- Ansible uses role tags from the plan.
|
||||
- Puppet and Salt currently run a full manifest/state apply.
|
||||
- `_enforcement_command()` is the source of truth for local apply commands.
|
||||
- `cli.py` attaches enforcement metadata to the report and formats it.
|
||||
|
||||
Do not make enforcement delete newly added packages/users/files/services unless the safety model is explicitly redesigned.
|
||||
|
||||
---
|
||||
|
||||
## 25. Important maintenance hazards
|
||||
|
||||
### 25.1 Renderer output is downstream of harvest state
|
||||
|
||||
If a renderer needs information, first ask whether that information belongs in `state.json`. Avoid papering over missing harvest facts inside a renderer.
|
||||
|
||||
### 25.2 `--fqdn` mode is not cosmetic
|
||||
|
||||
`--fqdn` changes where variables and artifacts live and how target inclusion works.
|
||||
|
||||
A change that works in default mode can still break:
|
||||
|
||||
- Ansible host vars,
|
||||
- Puppet Hiera node data,
|
||||
- Salt pillar node data.
|
||||
|
||||
### 25.3 Puppet and Salt are stricter about duplicates
|
||||
|
||||
Ansible often tolerates repeated packages or tasks. Puppet and Salt compile catalogs where duplicate resources can fail. Keep `resolve_catalog_conflicts()` in mind whenever adding resources.
|
||||
|
||||
### 25.4 Secret avoidance is part of the product contract
|
||||
|
||||
Default harvest should avoid likely secrets. `--dangerous` exists because useful files may contain secrets. Do not silently make risky harvesting the default.
|
||||
|
||||
### 25.5 Runtime state should not override persistent config
|
||||
|
||||
Firewall runtime capture is skipped when persistent firewall config exists. Preserve this principle for future runtime snapshots.
|
||||
|
||||
### 25.6 JinjaTurtle is best-effort except when explicitly required
|
||||
|
||||
`auto` mode should not make manifest generation fail merely because templating failed. `on` should require the executable; unsupported or unsafe individual files should still fall back to raw copy unless code explicitly changes that contract.
|
||||
|
||||
### 25.7 Role names must be sanitised
|
||||
|
||||
Raw package/service names can be invalid or reserved in Ansible roles, Puppet classes, or Salt SLS names. Use role-name helpers and singleton collision protection.
|
||||
|
||||
### 25.8 Tests encode edge cases
|
||||
|
||||
Many behaviours exist because of previously found edge cases:
|
||||
|
||||
- non-root/no-sudo harvests,
|
||||
- Puppet reserved words,
|
||||
- Salt Docker module availability limitations,
|
||||
- symlink capture,
|
||||
- JinjaTurtle missing variables,
|
||||
- Salt JSON filter compatibility,
|
||||
- file caps,
|
||||
- SOPS secure temp files,
|
||||
- tar path traversal,
|
||||
- target-selected diff enforcement.
|
||||
|
||||
Before simplifying logic, search the tests.
|
||||
|
||||
---
|
||||
|
||||
## 26. Troubleshooting guide
|
||||
|
||||
### 26.1 Generated manifest references a missing artifact
|
||||
|
||||
Likely causes:
|
||||
|
||||
- `managed_files[*].src_rel` was added without copying into `artifacts/`,
|
||||
- a renderer used the generated role/module name instead of the artifact role,
|
||||
- a role was renamed after harvest but before artifact lookup,
|
||||
- `--fqdn` file prefixes are wrong.
|
||||
|
||||
Start with:
|
||||
or (just pytests, no root required)
|
||||
|
||||
```bash
|
||||
enroll validate /path/to/harvest
|
||||
./pytests.sh
|
||||
```
|
||||
|
||||
Then inspect:
|
||||
|
||||
```text
|
||||
state.json roles.*.managed_files[*]
|
||||
artifacts/<role>/<src_rel>
|
||||
```
|
||||
|
||||
### 26.2 Puppet fails with duplicate resources
|
||||
|
||||
Check:
|
||||
|
||||
- `_collect_puppet_roles()`,
|
||||
- `resolve_catalog_conflicts()`,
|
||||
- `role_order_key()`,
|
||||
- whether a new resource type needs conflict resolution,
|
||||
- whether a directory resource conflicts with a file/link of the same path.
|
||||
|
||||
### 26.3 Salt fails with duplicate IDs or missing modules
|
||||
|
||||
Check:
|
||||
|
||||
- `_state_id()` naming,
|
||||
- `_collect_salt_roles()` grouping,
|
||||
- `resolve_catalog_conflicts()`,
|
||||
- guarded `cmd.run` fallbacks for Docker/Podman/Snap/Flatpak.
|
||||
|
||||
Salt uses guarded shell commands for some resources because native states/modules are not consistently available across Salt installations.
|
||||
|
||||
### 26.4 Ansible check mode reports unexpected changes
|
||||
|
||||
Check:
|
||||
|
||||
- role ordering,
|
||||
- grouped mode versus `--fqdn` / `--no-common-roles`,
|
||||
- handler notifications,
|
||||
- whether runtime roles were emitted without runtime artifacts,
|
||||
- harvested directory/file mode normalisation.
|
||||
|
||||
Grouped and per-role output can legitimately produce different numbers of reported changes.
|
||||
|
||||
### 26.5 A file was not harvested
|
||||
|
||||
Check, in order:
|
||||
|
||||
1. Was it excluded by `--exclude-path`?
|
||||
2. Was it denied by `IgnorePolicy`?
|
||||
3. Was it too large?
|
||||
4. Did it look binary?
|
||||
5. Did it contain sensitive-looking content?
|
||||
6. Was it already captured by another role via `captured_global`?
|
||||
7. Is it outside known scanned locations?
|
||||
8. Would `--include-path` collect it?
|
||||
9. Does it require `--dangerous`?
|
||||
|
||||
`enroll explain` can show notes and exclusion reasons.
|
||||
|
||||
### 26.6 `diff --enforce` fails
|
||||
|
||||
Check:
|
||||
|
||||
- whether the selected `--target` tool is on `PATH`,
|
||||
- `ansible-playbook` for Ansible,
|
||||
- `puppet` for Puppet,
|
||||
- `salt-call` for Salt,
|
||||
- whether the generated temp manifest has the expected target entrypoint,
|
||||
- whether the report contains enforceable drift,
|
||||
- whether package drift is only version changes or additions, which enforcement skips.
|
||||
|
||||
### 26.7 Remote harvest fails with sudo or SSH key prompts
|
||||
|
||||
Relevant flags:
|
||||
|
||||
- `--ask-become-pass`,
|
||||
- `--ask-key-passphrase`,
|
||||
- `--ssh-key-passphrase-env`,
|
||||
- `--no-sudo`,
|
||||
- `--remote-ssh-config`.
|
||||
|
||||
Interactive sessions can prompt and retry. Non-interactive sessions should pass explicit flags or environment variables.
|
||||
|
||||
---
|
||||
|
||||
## 27. Practical code-reading map
|
||||
|
||||
| Feature/question | Start with | Then read |
|
||||
|---|---|---|
|
||||
| CLI option behaviour | `cli.py` | called module for `args.cmd` |
|
||||
| Local harvest ordering | `harvest.py:harvest()` | `harvest_collectors/` |
|
||||
| Why a file was skipped | `capture.py`, `ignore.py`, `pathfilter.py` | `explain.py` |
|
||||
| File metadata/hash helpers | `fsutil.py` | `debian.py`, `capture.py` |
|
||||
| Service/package attribution | `harvest_collectors/services.py` | `package_hints.py`, `platform.py` |
|
||||
| APT/DNF config capture | `harvest_collectors/package_manager.py` | `system_paths.py` |
|
||||
| Users and SSH keys | `harvest_collectors/users.py` | `accounts.py` |
|
||||
| Flatpak/Snap parsing | `accounts.py` | renderer Flatpak/Snap helpers |
|
||||
| Docker/Podman images | `harvest_collectors/container_images.py` | renderer container image helpers |
|
||||
| Runtime firewall | `harvest_collectors/runtime.py`, `harvest.py` | renderer firewall helpers |
|
||||
| Sysctl | `harvest.py` sysctl helpers | renderer sysctl role functions |
|
||||
| Ansible output | `ansible.py:AnsibleManifestRenderer.render()` | `_render_*` helpers |
|
||||
| Puppet output | `puppet.py:PuppetManifestRenderer.render()` | `_collect_puppet_roles()` |
|
||||
| Salt output | `salt.py:SaltManifestRenderer.render()` | `_collect_salt_roles()` |
|
||||
| Grouping/common roles | `cm.py` | renderer collection functions |
|
||||
| JinjaTurtle | `jinjaturtle.py` | renderer managed-content code |
|
||||
| Diff/enforce | `diff.py` | `manifest.py`, target renderer |
|
||||
| Validation | `validate.py` | schema file and `state.json` |
|
||||
| Remote mode | `remote.py` | `cli.py` remote branches |
|
||||
| SOPS | `sopsutil.py` | `cli.py`, `manifest.py`, `diff.py` |
|
||||
|
||||
---
|
||||
|
||||
## 28. Glossary
|
||||
|
||||
**Harvest bundle**
|
||||
A directory or encrypted tarball containing `state.json` and `artifacts/`.
|
||||
|
||||
**Snapshot**
|
||||
A structured object under `roles` in `state.json`, such as a `ServiceSnapshot` or `PackageSnapshot`.
|
||||
|
||||
**Managed file**
|
||||
A file Enroll intends generated CM code to recreate. It has a destination path and a matching artifact file.
|
||||
|
||||
**Managed link**
|
||||
A symlink Enroll intends generated CM code to recreate.
|
||||
|
||||
**Managed dir**
|
||||
A directory Enroll intends generated CM code to ensure exists with recorded metadata.
|
||||
|
||||
**Role**
|
||||
The Enroll logical group for related resources. In Ansible it usually maps to an Ansible role. In Puppet it maps to a module/class. In Salt it maps to an SLS role.
|
||||
|
||||
**Artifact role**
|
||||
The role directory under `artifacts/` that contains a harvested file. This can differ from the generated renderer role when grouping is enabled.
|
||||
|
||||
**Common/grouped role**
|
||||
A generated role/module/state that merges multiple package/service snapshots by Debian Section or RPM Group.
|
||||
|
||||
**Site mode / `--fqdn` mode**
|
||||
Host-specific output mode. Ansible uses host vars, Puppet uses Hiera node data, and Salt uses pillar node data.
|
||||
|
||||
**Dangerous mode**
|
||||
Explicit opt-in mode that relaxes safety checks and enables risky capture such as user shell dotfiles.
|
||||
|
||||
**JinjaTurtle**
|
||||
Optional external tool used to convert recognised config files into Jinja2 or ERB templates plus variable defaults/context.
|
||||
|
||||
**Enforcement target**
|
||||
The config manager chosen for `diff --enforce` with `--target ansible|puppet|salt`.
|
||||
|
||||
---
|
||||
|
||||
## 29. Final maintenance model
|
||||
|
||||
Most changes should preserve this pipeline:
|
||||
|
||||
```text
|
||||
Collect facts and files safely
|
||||
-> represent them in target-neutral state.json
|
||||
-> keep artifact references consistent
|
||||
-> let each renderer translate the same state into its own idioms
|
||||
-> validate the bundle and test each target
|
||||
```
|
||||
|
||||
Before changing code, ask:
|
||||
|
||||
1. Is this a harvest concern or renderer concern?
|
||||
2. Does `state.json` or the schema need to change?
|
||||
3. Does this affect `--fqdn` mode?
|
||||
4. Does this introduce duplicate ownership of a path/resource?
|
||||
5. Does this weaken default secret avoidance?
|
||||
6. Do Puppet and Salt need conflict handling?
|
||||
7. Does JinjaTurtle fallback still behave safely?
|
||||
8. Does `diff --enforce --target ...` still do the conservative thing?
|
||||
9. Do existing tests explain why the current behaviour exists?
|
||||
|
||||
Keeping those boundaries clear is the main way to maintain Enroll without creating subtle cross-target regressions.
|
||||
|
|
|
|||
82
README.md
82
README.md
|
|
@ -4,7 +4,7 @@
|
|||
<img src="https://git.mig5.net/mig5/enroll/raw/branch/main/enroll.svg" alt="Enroll logo" width="240" />
|
||||
</div>
|
||||
|
||||
**enroll** inspects a Linux machine (Debian-like or RedHat-like) and generates configuration-management code from it, as Ansible (default), Puppet or Salt.
|
||||
**enroll** inspects a Linux machine (Debian-like or RedHat-like) and generates Ansible configuration-management code from it.
|
||||
|
||||
- Detects packages that have been installed.
|
||||
- Detects package ownership of `/etc` files where possible
|
||||
|
|
@ -27,7 +27,7 @@
|
|||
`enroll` works in two phases:
|
||||
|
||||
1) **Harvest**: collect host facts + relevant files into a harvest bundle (`state.json` + harvested artifacts)
|
||||
2) **Manifest**: turn that harvest into configuration-management code such as Ansible, Puppet or Salt.
|
||||
2) **Manifest**: turn that harvest into Ansible configuration-management code.
|
||||
|
||||
Additionally, some other functionalities exist:
|
||||
|
||||
|
|
@ -38,8 +38,6 @@ Additionally, some other functionalities exist:
|
|||
|
||||
## Output modes: single-site vs multi-site (`--fqdn`)
|
||||
|
||||
`enroll manifest` (and `enroll single-shot`) support multiple output targets. Ansible is the default target and supports two distinct output styles.
|
||||
|
||||
### Single-site mode (default: *no* `--fqdn`)
|
||||
Use when enrolling **one server** (or generating a “golden” role set you intend to reuse).
|
||||
|
||||
|
|
@ -124,7 +122,7 @@ enroll single-shot --remote-host myhost.example.com --remote-user myuser --ssh-k
|
|||
---
|
||||
|
||||
### `enroll manifest`
|
||||
Generate configuration-management output from an existing harvest bundle. Ansible remains the default; use `--target puppet` for Puppet output or `--target salt` for Salt output.
|
||||
Generate Ansible output from an existing harvest bundle.
|
||||
|
||||
**Inputs**
|
||||
- `--harvest /path/to/harvest` (directory)
|
||||
|
|
@ -132,13 +130,10 @@ Generate configuration-management output from an existing harvest bundle. Ansibl
|
|||
|
||||
**Output**
|
||||
- In plaintext Ansible mode: an Ansible repo-like directory structure (roles/playbooks, and inventory in multi-site mode).
|
||||
- In plaintext Puppet mode: a Puppet control-repo style layout with `manifests/site.pp` and generated modules under `modules/`. By default, package and service resources are grouped by Debian Section/RPM Group where possible; `--fqdn` or `--no-common-roles` preserves one generated module per Enroll role/snapshot.
|
||||
- In plaintext Salt mode: a Salt state tree under `states/`, plus `pillar/` data in `--fqdn` mode. By default, package and service resources are grouped by Debian Section/RPM Group where possible; `--fqdn` or `--no-common-roles` preserves one generated SLS role per Enroll role/snapshot.
|
||||
- In `--sops` mode: a single encrypted file `manifest.tar.gz.sops` containing the generated output.
|
||||
|
||||
**Common flags**
|
||||
- `--target ansible|puppet|salt`: choose the manifest target (`ansible` is the default).
|
||||
- `--fqdn <host>`: enables **multi-site** output style for Ansible, emits Puppet Hiera/node output, or emits Salt top/pillar output targeted at that minion ID. Without `--fqdn`, Puppet emits `node default { ... }` and Salt targets `*` in `states/top.sls`.
|
||||
- `--fqdn <host>`: enables **multi-site** output style for Ansible (host-specific state lives in inventory `host_vars`).
|
||||
- `--no-common-roles`: disables the default grouping of package and systemd-unit roles into Debian Section/RPM Group roles, preserving one generated role per package/unit. `--fqdn` implies this behaviour.
|
||||
|
||||
**Role tags**
|
||||
|
|
@ -159,7 +154,7 @@ Convenience wrapper that runs **harvest → manifest** in one command.
|
|||
|
||||
Use this when you want “get me something workable ASAP”.
|
||||
|
||||
Supports the same general flags as harvest/manifest, including `--target`, `--fqdn`, `--no-common-roles`, remote harvest flags, and `--sops`.
|
||||
Supports the same general flags as harvest/manifest, including `--fqdn`, `--no-common-roles`, remote harvest flags, and `--sops`.
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -178,14 +173,14 @@ Compare two harvest bundles and report what changed.
|
|||
- `--exclude-path <PATTERN>` (repeatable) to ignore file/dir drift under matching paths (same pattern syntax as harvest)
|
||||
- `--ignore-package-versions` to ignore package version-only drift (upgrades/downgrades)
|
||||
- `--enforce` to apply the **old** harvest state locally (requires the relevant config manager tool on `PATH` - defaults to `ansible-playbook`)
|
||||
- `--target` when using `--enforce`, to set the desired config manager tool to manifest to and run)
|
||||
- `--enforce` runs `ansible-playbook` against the regenerated manifest)
|
||||
|
||||
**Noise suppression**
|
||||
- `--exclude-path` is useful for things that change often but you still want in the harvest baseline (e.g. `/var/anacron`).
|
||||
- `--ignore-package-versions` keeps routine upgrades from alerting; package add/remove drift is still reported.
|
||||
|
||||
**Enforcement (`--enforce` (`--target`))**
|
||||
If a diff exists and the config manager defined by `--target` (default: ansible) is on the PATH, Enroll will:
|
||||
**Enforcement (`--enforce`)**
|
||||
If a diff exists and `ansible-playbook` is on the PATH, Enroll will:
|
||||
1) generate a manifest from the **old** harvest into a temporary directory
|
||||
2) run the config manager tool against that manifest
|
||||
3) record in the diff report that the old harvest was enforced
|
||||
|
|
@ -316,16 +311,6 @@ For Ansible:
|
|||
- single-site: `roles/<role>/defaults/main.yml`
|
||||
- multi-site: `inventory/host_vars/<fqdn>/<role>.yml`
|
||||
|
||||
For Salt:
|
||||
- Templates live in `states/roles/<role>/templates/...`
|
||||
- `file.managed` uses `template: jinja` with per-file `context` values
|
||||
- In `--fqdn` mode, template context values are written to pillar with the file metadata
|
||||
|
||||
For Puppet:
|
||||
- JinjaTurtle will use its 'erb' mode if you are running a recent-enough version.
|
||||
- Templates will be stored in `modules/<module>/templates/<file>.erb`
|
||||
- In `--fqdn` mode, template context values are written to Hiera data.
|
||||
|
||||
You can force template generation on with `--jinjaturtle` or disable it with `--no-jinjaturtle`.
|
||||
|
||||
---
|
||||
|
|
@ -473,53 +458,6 @@ If Docker or Podman is available during harvest, Enroll records local image-cach
|
|||
|
||||
For Ansible, digest-pinned Docker images are pulled with `community.docker.docker_image_pull` and digest-pinned Podman images are pulled with `containers.podman.podman_image`; harvested tag aliases are re-applied where possible. The generated `requirements.yml` includes `community.docker` and `containers.podman` alongside any other required collections. In `--fqdn` mode the image list is host-specific inventory data.
|
||||
|
||||
### Puppet target
|
||||
```bash
|
||||
enroll manifest --harvest /tmp/enroll-harvest --out /tmp/enroll-puppet --target puppet
|
||||
```
|
||||
|
||||
The Puppet target renders native packages, users/groups, managed directories/files/symlinks, basic service state, and the generated sysctl file/apply exec when present. Without `--fqdn`, `site.pp` uses `node default { ... }`; with `--fqdn`, it uses `node '<host>' { ... }`. Run from the generated output directory with the generated modules on Puppet's module path, for example:
|
||||
|
||||
```bash
|
||||
cd /tmp/enroll-puppet
|
||||
sudo puppet apply --modulepath ./modules manifests/site.pp --noop
|
||||
```
|
||||
|
||||
Or with absolute paths:
|
||||
|
||||
```bash
|
||||
sudo puppet apply --modulepath /tmp/enroll-puppet/modules /tmp/enroll-puppet/manifests/site.pp --noop
|
||||
```
|
||||
|
||||
Docker images with registry digests are currently managed with `exec` statements. I know that's nasty, but the `puppetlabs-docker` module is even nastier and creates non-idempotent bash scripts for executing on every run. Worse, if you then reharvest that host that has Puppet installed, you'll get a File resource collision with that very shell script. Believe me, for the simple use case of 'make sure this Docker image is installed', this simple solution is better.
|
||||
|
||||
### Salt target
|
||||
```bash
|
||||
enroll manifest --harvest /tmp/enroll-harvest --out /tmp/enroll-salt --target salt
|
||||
```
|
||||
|
||||
The Salt target renders native packages, users/groups, managed directories/files/symlinks, basic service state, and the generated sysctl file/apply command when present. Without `--fqdn`, it writes a self-contained state tree under `states/` and targets all minions in `states/top.sls`:
|
||||
|
||||
```bash
|
||||
cd /tmp/enroll-salt
|
||||
sudo salt-call --local --file-root ./states state.apply test=True
|
||||
```
|
||||
|
||||
With `--fqdn`, it uses Salt's state/pillar split: `states/top.sls` targets the minion ID to reusable generated role SLS files, while `pillar/top.sls` targets the same minion ID to node-specific data under `pillar/nodes/`. Host-specific file artifacts are stored under `states/roles/<role>/files/nodes/<fqdn>/...` and referenced through `salt://` URLs:
|
||||
|
||||
```bash
|
||||
enroll manifest --harvest /tmp/enroll-harvest --out /tmp/enroll-salt --target salt --fqdn host.example.com
|
||||
cd /tmp/enroll-salt
|
||||
sudo salt-call --local --file-root ./states --pillar-root ./pillar --id host.example.com state.apply test=True
|
||||
```
|
||||
|
||||
Re-running Salt `--fqdn` output into the same directory adds or replaces that minion's top/pillar data without deleting other generated minions.
|
||||
|
||||
Docker and Podman images with registry digests are rendered as guarded `cmd.run` states that use the local `docker`/`podman` CLI directly (`pull`, `image inspect`, and `tag`).
|
||||
|
||||
This is because Salt Stack, in 3008, does not have proper Docker extensions that actually work. Wow. It's a bit like Puppet. Seriously, you should probably just be using Ansible.
|
||||
|
||||
|
||||
### Manifest with `--sops`
|
||||
```bash
|
||||
# Generate encrypted manifest bundle (writes /tmp/enroll-ansible/manifest.tar.gz.sops)
|
||||
|
|
@ -705,14 +643,12 @@ exclude_path = /usr/local/bin/docker-*, /usr/local/bin/some-tool
|
|||
# you can set defaults here too, e.g.
|
||||
no_jinjaturtle = true
|
||||
sops = 54A91143AE0AB4F7743B01FE888ED1B423A3BC99
|
||||
# target = ansible (the default), or salt, or puppet
|
||||
|
||||
[diff]
|
||||
# ignore noisy drift
|
||||
exclude_path = /var/anacron
|
||||
ignore_package_versions = true
|
||||
# enforce = true # requires the target config manager on PATH
|
||||
# target = puppet (for example, as per manifest)
|
||||
# enforce = true # requires ansible-playbook on PATH
|
||||
|
||||
[single-shot]
|
||||
# if you use single-shot, put its defaults here.
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ In particular:
|
|||
* If an `enroll.ini` configuration file is loaded, its location and contents are assumed to be owned, selected, and understood by the operator.
|
||||
* The operator is expected to understand the implications of options such as `--dangerous`, `--assume-safe-path`, `--sops`, `--enforce`, `--remote-host`, and `--remote-ssh-config`.
|
||||
* Harvest bundles used for `manifest`, `diff`, or `diff --enforce` are assumed to come from a trusted source unless the operator is deliberately inspecting untrusted input without applying it.
|
||||
* Configuration-management tools invoked by Enroll, such as Ansible, Puppet, Salt, SOPS, SSH, `sudo`, Docker, Podman, Flatpak, Snap, package managers, and system utilities, are assumed to be the trusted tools the operator intended to use.
|
||||
* Configuration-management tools invoked by Enroll, such as Ansible, SOPS, SSH, `sudo`, Docker, Podman, Flatpak, Snap, package managers, and system utilities, are assumed to be the trusted tools the operator intended to use.
|
||||
|
||||
## What is in scope
|
||||
|
||||
|
|
@ -31,7 +31,7 @@ In-scope security concerns include:
|
|||
* Writing plaintext harvest outputs into private directories by default.
|
||||
* Hardening root-run output path handling so Enroll does not accidentally write through attacker-prepared symlinks or unsafe parent directories.
|
||||
* Refusing to continue non-interactively when run as root with an unsafe `PATH`, unless the operator explicitly confirms with `--assume-safe-path`.
|
||||
* Avoiding shell injection in generated manifests where harvested values are embedded into Ansible, Puppet, or Salt output.
|
||||
* Avoiding shell injection in generated manifests where harvested values are embedded into Ansible output.
|
||||
* Rejecting unknown SSH host keys by default during remote harvests.
|
||||
|
||||
These measures are defense-in-depth. They are intended to reduce the chance of accidental exposure, unsafe filesystem writes, path traversal, command injection, or dangerous behavior when Enroll is used normally by an administrator.
|
||||
|
|
@ -45,7 +45,7 @@ The following are generally out of scope and should not be reported as Enroll vu
|
|||
* A root user passing `--dangerous` and then observing that Enroll may collect sensitive information.
|
||||
* A root user passing `--assume-safe-path` and then observing that Enroll does not prompt about `PATH` safety.
|
||||
* A root user enforcing a malicious or manually edited harvest bundle with `diff --enforce`.
|
||||
* A user applying generated Ansible, Puppet, or Salt manifests from an untrusted harvest.
|
||||
* A user applying generated Ansible manifests from an untrusted harvest.
|
||||
* A user configuring a webhook, email target, SSH proxy command, SOPS binary, package manager, or configuration-management tool that they do not trust.
|
||||
* A compromised system where an attacker already controls root-owned files, root’s shell, root’s configuration, or the privileged tools Enroll invokes.
|
||||
* Reports that amount to “if root runs this tool with malicious options, root can make the system do dangerous things.”
|
||||
|
|
@ -79,7 +79,7 @@ Useful vulnerability reports include issues where Enroll behaves unsafely despit
|
|||
* Enroll follows a symlink or hardlink in a way that causes privileged file disclosure or overwrite.
|
||||
* Enroll extracts a tar member outside the intended harvest directory.
|
||||
* Enroll accepts a malicious harvest artifact that escapes the artifact root.
|
||||
* Enroll generates an Ansible, Puppet, or Salt manifest where ordinary harvested data can cause command injection.
|
||||
* Enroll generates an Ansible manifest where ordinary harvested data can cause command injection.
|
||||
* Enroll writes root-run output into an unsafe attacker-controlled path despite its safety checks.
|
||||
* Enroll silently ignores a failed safety check and proceeds anyway.
|
||||
* Enroll accepts an unknown SSH host key unexpectedly.
|
||||
|
|
|
|||
|
|
@ -724,7 +724,7 @@ def _write_ansible_role_vars(
|
|||
*,
|
||||
site_defaults: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
"""Write role variables using the same mode split as Puppet Hiera/Salt Pillar."""
|
||||
"""Write role variables using the single-site/site-mode split."""
|
||||
|
||||
if ctx.site_mode:
|
||||
_write_role_defaults(role_dir, site_defaults or {})
|
||||
|
|
|
|||
|
|
@ -458,15 +458,9 @@ def _encrypt_harvest_dir_to_sops(
|
|||
|
||||
|
||||
def _add_common_manifest_args(p: argparse.ArgumentParser) -> None:
|
||||
p.add_argument(
|
||||
"--target",
|
||||
choices=["ansible", "puppet", "salt"],
|
||||
default="ansible",
|
||||
help="Manifest target to generate (default: ansible).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--fqdn",
|
||||
help="Host FQDN/name for site-mode output (creates target-specific host inventory/data such as Ansible host_vars, Puppet Hiera, or Salt pillar).",
|
||||
help="Host FQDN/name for site-mode output (creates Ansible host_vars for that host).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--no-common-roles",
|
||||
|
|
@ -808,15 +802,6 @@ def main() -> None:
|
|||
"Enroll does not attempt to downgrade packages; if the only drift is package version upgrades (or newly installed packages), enforcement is skipped."
|
||||
),
|
||||
)
|
||||
d.add_argument(
|
||||
"--target",
|
||||
choices=["ansible", "puppet", "salt"],
|
||||
default="ansible",
|
||||
help=(
|
||||
"Configuration-management target to use with --enforce (default: ansible). "
|
||||
"Requires ansible-playbook, puppet, or salt-call on PATH as appropriate."
|
||||
),
|
||||
)
|
||||
d.add_argument(
|
||||
"--out",
|
||||
help="Write the report to this file instead of stdout.",
|
||||
|
|
@ -1119,7 +1104,6 @@ def main() -> None:
|
|||
jinjaturtle=_jt_mode(args),
|
||||
sops_fingerprints=getattr(args, "sops", None),
|
||||
no_common_roles=bool(getattr(args, "no_common_roles", False)),
|
||||
target=getattr(args, "target", "ansible"),
|
||||
)
|
||||
if getattr(args, "sops", None) and out_enc:
|
||||
print(str(out_enc))
|
||||
|
|
@ -1135,7 +1119,7 @@ def main() -> None:
|
|||
)
|
||||
|
||||
# Optional enforcement: if drift is detected, attempt to restore the
|
||||
# system to the *old* (baseline) state using the selected target.
|
||||
# system to the *old* (baseline) state using ansible.
|
||||
if bool(getattr(args, "enforce", False)):
|
||||
if has_changes:
|
||||
if not has_enforceable_drift(report):
|
||||
|
|
@ -1153,7 +1137,6 @@ def main() -> None:
|
|||
args.old,
|
||||
sops_mode=bool(getattr(args, "sops", False)),
|
||||
report=report,
|
||||
target=getattr(args, "target", "ansible"),
|
||||
)
|
||||
except Exception as e:
|
||||
raise SystemExit(
|
||||
|
|
@ -1258,7 +1241,6 @@ def main() -> None:
|
|||
jinjaturtle=_jt_mode(args),
|
||||
sops_fingerprints=list(sops_fps),
|
||||
no_common_roles=bool(getattr(args, "no_common_roles", False)),
|
||||
target=getattr(args, "target", "ansible"),
|
||||
)
|
||||
if not args.harvest:
|
||||
print(str(out_file))
|
||||
|
|
@ -1291,7 +1273,6 @@ def main() -> None:
|
|||
fqdn=args.fqdn,
|
||||
jinjaturtle=_jt_mode(args),
|
||||
no_common_roles=bool(getattr(args, "no_common_roles", False)),
|
||||
target=getattr(args, "target", "ansible"),
|
||||
)
|
||||
# For usability (when --harvest wasn't provided), print the harvest path.
|
||||
if not args.harvest:
|
||||
|
|
@ -1324,7 +1305,6 @@ def main() -> None:
|
|||
jinjaturtle=_jt_mode(args),
|
||||
sops_fingerprints=list(sops_fps),
|
||||
no_common_roles=bool(getattr(args, "no_common_roles", False)),
|
||||
target=getattr(args, "target", "ansible"),
|
||||
)
|
||||
if not args.harvest:
|
||||
print(str(out_file))
|
||||
|
|
@ -1345,7 +1325,6 @@ def main() -> None:
|
|||
fqdn=args.fqdn,
|
||||
jinjaturtle=_jt_mode(args),
|
||||
no_common_roles=bool(getattr(args, "no_common_roles", False)),
|
||||
target=getattr(args, "target", "ansible"),
|
||||
)
|
||||
except RemoteSudoPasswordRequired:
|
||||
raise SystemExit(
|
||||
|
|
|
|||
15
enroll/cm.py
15
enroll/cm.py
|
|
@ -22,9 +22,9 @@ from .state import load_state, state_path, write_state
|
|||
class CMModule:
|
||||
"""Renderer-neutral configuration-management resource group.
|
||||
|
||||
A CMModule is intentionally small: it captures the resources that a target
|
||||
renderer can turn into Ansible tasks, Puppet resources, Salt states, etc.
|
||||
The renderer may still decide how to name/include/order the group.
|
||||
A CMModule is intentionally small: it captures the resources that the
|
||||
renderer turns into Ansible tasks. The renderer may still decide how to
|
||||
name/include/order the group.
|
||||
"""
|
||||
|
||||
role_name: str
|
||||
|
|
@ -806,12 +806,11 @@ def _drop_duplicate_mapping_items(
|
|||
|
||||
|
||||
def resolve_catalog_conflicts(modules: Iterable[CMModule]) -> None:
|
||||
"""Resolve global catalog conflicts before renderer output.
|
||||
"""Resolve global catalog conflicts in the shared model.
|
||||
|
||||
Puppet and Salt compile a single resource catalog. Ansible can tolerate the
|
||||
same package, service, or parent directory appearing in more than one role;
|
||||
catalog targets cannot. Resolve those conflicts in the shared model rather
|
||||
than deleting renderer output after the fact.
|
||||
Deduplicates the same package, service, or parent directory appearing in
|
||||
more than one role. The Ansible renderer tolerates such duplicates, but this
|
||||
helper remains available for any catalog-style consumer of the shared model.
|
||||
"""
|
||||
|
||||
ordered = list(modules)
|
||||
|
|
|
|||
|
|
@ -228,6 +228,10 @@ def read_pkg_md5sums(pkg: str) -> Dict[str, str]:
|
|||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
md5, rel = line.split(None, 1)
|
||||
parts = line.split(None, 1)
|
||||
if len(parts) != 2:
|
||||
# Skip malformed/truncated lines instead of aborting the harvest.
|
||||
continue
|
||||
md5, rel = parts
|
||||
m[rel.strip()] = md5.strip()
|
||||
return m
|
||||
|
|
|
|||
175
enroll/diff.py
175
enroll/diff.py
|
|
@ -682,40 +682,7 @@ def _role_tag(role: str) -> str:
|
|||
return f"role_{safe}"
|
||||
|
||||
|
||||
def _normalise_enforcement_target(target: str) -> str:
|
||||
t = str(target or "ansible").strip().lower()
|
||||
if t not in {"ansible", "puppet", "salt"}:
|
||||
raise ValueError(f"unsupported enforcement target: {target!r}")
|
||||
return t
|
||||
|
||||
|
||||
def _enforcement_tool(target: str) -> Tuple[str, str]:
|
||||
"""Return (binary-name, human-label) for a local enforcement target."""
|
||||
if target == "puppet":
|
||||
return "puppet", "puppet apply"
|
||||
if target == "salt":
|
||||
return "salt-call", "salt-call"
|
||||
return "ansible-playbook", "ansible-playbook"
|
||||
|
||||
|
||||
def _require_enforcement_tool(target: str) -> Tuple[str, str]:
|
||||
binary, label = _enforcement_tool(target)
|
||||
exe = shutil.which(binary)
|
||||
if not exe:
|
||||
install_hint = {
|
||||
"ansible": "Ansible",
|
||||
"puppet": "Puppet",
|
||||
"salt": "Salt",
|
||||
}.get(target, target)
|
||||
raise RuntimeError(
|
||||
f"{binary} not found on PATH "
|
||||
f"(cannot enforce with target {target}; install {install_hint})"
|
||||
)
|
||||
return exe, label
|
||||
|
||||
|
||||
def _enforcement_command(
|
||||
target: str,
|
||||
exe: str,
|
||||
manifest_dir: Path,
|
||||
*,
|
||||
|
|
@ -724,69 +691,27 @@ def _enforcement_command(
|
|||
"""Return the local apply command and environment for a rendered manifest."""
|
||||
env = dict(os.environ)
|
||||
|
||||
if target == "ansible":
|
||||
playbook = manifest_dir / "playbook.yml"
|
||||
if not playbook.exists():
|
||||
raise RuntimeError(
|
||||
f"manifest did not produce expected playbook.yml at {playbook}"
|
||||
)
|
||||
playbook = manifest_dir / "playbook.yml"
|
||||
if not playbook.exists():
|
||||
raise RuntimeError(
|
||||
f"manifest did not produce expected playbook.yml at {playbook}"
|
||||
)
|
||||
|
||||
cfg = manifest_dir / "ansible.cfg"
|
||||
if cfg.exists():
|
||||
env["ANSIBLE_CONFIG"] = str(cfg)
|
||||
cfg = manifest_dir / "ansible.cfg"
|
||||
if cfg.exists():
|
||||
env["ANSIBLE_CONFIG"] = str(cfg)
|
||||
|
||||
cmd = [
|
||||
exe,
|
||||
"-i",
|
||||
"localhost,",
|
||||
"-c",
|
||||
"local",
|
||||
str(playbook),
|
||||
]
|
||||
if tags:
|
||||
cmd.extend(["--tags", ",".join(tags)])
|
||||
return cmd, env
|
||||
|
||||
if target == "puppet":
|
||||
site_pp = manifest_dir / "manifests" / "site.pp"
|
||||
if not site_pp.exists():
|
||||
raise RuntimeError(
|
||||
f"manifest did not produce expected Puppet site.pp at {site_pp}"
|
||||
)
|
||||
|
||||
cmd = [
|
||||
exe,
|
||||
"apply",
|
||||
"--modulepath",
|
||||
str(manifest_dir / "modules"),
|
||||
]
|
||||
hiera_config = manifest_dir / "hiera.yaml"
|
||||
if hiera_config.exists():
|
||||
cmd.extend(["--hiera_config", str(hiera_config)])
|
||||
cmd.append(str(site_pp))
|
||||
return cmd, env
|
||||
|
||||
if target == "salt":
|
||||
states_dir = manifest_dir / "states"
|
||||
top_sls = states_dir / "top.sls"
|
||||
if not top_sls.exists():
|
||||
raise RuntimeError(
|
||||
f"manifest did not produce expected Salt top.sls at {top_sls}"
|
||||
)
|
||||
|
||||
cmd = [
|
||||
exe,
|
||||
"--local",
|
||||
"--file-root",
|
||||
str(states_dir),
|
||||
]
|
||||
pillar_dir = manifest_dir / "pillar"
|
||||
if pillar_dir.exists():
|
||||
cmd.extend(["--pillar-root", str(pillar_dir)])
|
||||
cmd.extend(["state.apply"])
|
||||
return cmd, env
|
||||
|
||||
raise ValueError(f"unsupported enforcement target: {target!r}")
|
||||
cmd = [
|
||||
exe,
|
||||
"-i",
|
||||
"localhost,",
|
||||
"-c",
|
||||
"local",
|
||||
str(playbook),
|
||||
]
|
||||
if tags:
|
||||
cmd.extend(["--tags", ",".join(tags)])
|
||||
return cmd, env
|
||||
|
||||
|
||||
def _enforcement_plan(
|
||||
|
|
@ -898,22 +823,18 @@ def enforce_old_harvest(
|
|||
*,
|
||||
sops_mode: bool = False,
|
||||
report: Optional[Dict[str, Any]] = None,
|
||||
target: str = "ansible",
|
||||
) -> Dict[str, Any]:
|
||||
"""Enforce the *old* (baseline) harvest state on the current machine.
|
||||
|
||||
This renders a temporary manifest from the old harvest using the requested
|
||||
target, then runs the target's local apply command:
|
||||
- ansible: ansible-playbook -i localhost, -c local playbook.yml
|
||||
- puppet: puppet apply --modulepath ./modules manifests/site.pp
|
||||
- salt: salt-call --local --file-root ./states state.apply
|
||||
This renders a temporary Ansible manifest from the old harvest, then runs
|
||||
the local apply command:
|
||||
- ansible-playbook -i localhost, -c local playbook.yml
|
||||
|
||||
Returns a dict suitable for attaching to the diff report under
|
||||
report['enforcement'].
|
||||
"""
|
||||
|
||||
target = _normalise_enforcement_target(target)
|
||||
tool_exe, tool_label = _require_enforcement_tool(target)
|
||||
tool_exe = "ansible-playbook"
|
||||
|
||||
# Import lazily to avoid heavy import cost and potential CLI cycles.
|
||||
from .manifest import manifest
|
||||
|
|
@ -933,12 +854,10 @@ def enforce_old_harvest(
|
|||
if report is not None:
|
||||
plan = _enforcement_plan(report, old_state, old_b.dir)
|
||||
roles = list(plan.get("roles") or [])
|
||||
# Only Ansible has generated per-role tags that can safely narrow
|
||||
# the apply scope. Puppet and Salt enforcement deliberately run the
|
||||
# full generated local manifest/catalog for now.
|
||||
if target == "ansible":
|
||||
t = list(plan.get("tags") or [])
|
||||
tags = t if t else None
|
||||
# Ansible has generated per-role tags that can safely narrow the
|
||||
# apply scope.
|
||||
t = list(plan.get("tags") or [])
|
||||
tags = t if t else None
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="enroll-enforce-") as td:
|
||||
td_path = Path(td)
|
||||
|
|
@ -951,11 +870,10 @@ def enforce_old_harvest(
|
|||
# refuses to write into an existing destination, so use a fresh
|
||||
# child path under the secure temporary directory.
|
||||
manifest_dir = td_path / "manifest"
|
||||
manifest(str(old_b.dir), str(manifest_dir), target=target)
|
||||
manifest(str(old_b.dir), str(manifest_dir))
|
||||
|
||||
# 2) Apply it locally.
|
||||
cmd, env = _enforcement_command(
|
||||
target,
|
||||
tool_exe,
|
||||
manifest_dir,
|
||||
tags=tags,
|
||||
|
|
@ -967,12 +885,12 @@ def enforce_old_harvest(
|
|||
if _progress_enabled():
|
||||
if tags:
|
||||
sys.stderr.write(
|
||||
f"Enforce: running {tool_label} (tags: {','.join(tags)})\n",
|
||||
f"Enforce: running {tool_exe} tags: {','.join(tags)})\n",
|
||||
)
|
||||
else:
|
||||
sys.stderr.write(f"Enforce: running {tool_label}\n")
|
||||
sys.stderr.write(f"Enforce: running {tool_exe}\n")
|
||||
sys.stderr.flush()
|
||||
spinner = _Spinner(f" {tool_label}")
|
||||
spinner = _Spinner(f" {tool_exe}")
|
||||
spinner.start()
|
||||
|
||||
try:
|
||||
|
|
@ -990,7 +908,7 @@ def enforce_old_harvest(
|
|||
rc = p.returncode if p is not None else None
|
||||
spinner.stop(
|
||||
final_line=(
|
||||
f"Enforce: {tool_label} finished in {elapsed:0.1f}s"
|
||||
f"Enforce: {tool_exe} finished in {elapsed:0.1f}s"
|
||||
+ (f" (rc={rc})" if rc is not None else "")
|
||||
),
|
||||
)
|
||||
|
|
@ -999,22 +917,15 @@ def enforce_old_harvest(
|
|||
|
||||
info: Dict[str, Any] = {
|
||||
"status": "applied" if p.returncode == 0 else "failed",
|
||||
"target": target,
|
||||
"tool": tool_label,
|
||||
"executable": tool_exe,
|
||||
"started_at": started_at,
|
||||
"finished_at": finished_at,
|
||||
"command": cmd,
|
||||
"returncode": int(p.returncode),
|
||||
}
|
||||
# Keep the original Ansible-specific field for compatibility with
|
||||
# existing consumers of the JSON report.
|
||||
if target == "ansible":
|
||||
info["ansible_playbook"] = tool_exe
|
||||
elif target == "puppet":
|
||||
info["puppet"] = tool_exe
|
||||
elif target == "salt":
|
||||
info["salt_call"] = tool_exe
|
||||
# Keep the Ansible-specific field for compatibility with existing
|
||||
# consumers of the JSON report.
|
||||
info["ansible_playbook"] = tool_exe
|
||||
|
||||
info["roles"] = roles
|
||||
info["tags"] = list(tags or [])
|
||||
|
|
@ -1024,7 +935,7 @@ def enforce_old_harvest(
|
|||
if p.returncode != 0:
|
||||
err = (p.stderr or p.stdout or "").strip()
|
||||
raise RuntimeError(
|
||||
f"{tool_label} failed"
|
||||
f"{tool_exe} failed"
|
||||
+ (f" (rc={p.returncode})" if p.returncode is not None else "")
|
||||
+ (f": {err}" if err else "")
|
||||
)
|
||||
|
|
@ -1069,9 +980,6 @@ def _report_text(report: Dict[str, Any]) -> str:
|
|||
if enf:
|
||||
lines.append("\nEnforcement")
|
||||
status = str(enf.get("status") or "").strip().lower()
|
||||
tool = str(enf.get("tool") or "ansible-playbook")
|
||||
target = str(enf.get("target") or "ansible")
|
||||
via = f"{tool} ({target})" if target and target not in tool else tool
|
||||
if status == "applied":
|
||||
extra = ""
|
||||
tags = enf.get("tags") or []
|
||||
|
|
@ -1081,7 +989,7 @@ def _report_text(report: Dict[str, Any]) -> str:
|
|||
elif scope:
|
||||
extra = f" ({scope})"
|
||||
lines.append(
|
||||
f" applied old harvest via {via} (rc={enf.get('returncode')})"
|
||||
f" applied old harvest (rc={enf.get('returncode')})"
|
||||
+ extra
|
||||
+ (
|
||||
f" (finished {enf.get('finished_at')})"
|
||||
|
|
@ -1091,7 +999,7 @@ def _report_text(report: Dict[str, Any]) -> str:
|
|||
)
|
||||
elif status == "failed":
|
||||
lines.append(
|
||||
f" attempted enforcement but {via} failed (rc={enf.get('returncode')})"
|
||||
f" attempted enforcement but failed (rc={enf.get('returncode')})"
|
||||
)
|
||||
elif status == "skipped":
|
||||
r = enf.get("reason")
|
||||
|
|
@ -1231,9 +1139,6 @@ def _report_markdown(report: Dict[str, Any]) -> str:
|
|||
if enf:
|
||||
out.append("\n## Enforcement\n")
|
||||
status = str(enf.get("status") or "").strip().lower()
|
||||
tool = str(enf.get("tool") or "ansible-playbook")
|
||||
target = str(enf.get("target") or "ansible")
|
||||
via = f"{tool} ({target})" if target and target not in tool else tool
|
||||
if status == "applied":
|
||||
extra = ""
|
||||
tags = enf.get("tags") or []
|
||||
|
|
@ -1243,7 +1148,7 @@ def _report_markdown(report: Dict[str, Any]) -> str:
|
|||
elif scope:
|
||||
extra = f" ({scope})"
|
||||
out.append(
|
||||
f"- ✅ Applied old harvest via {via}"
|
||||
"- ✅ Applied old harvest"
|
||||
+ extra
|
||||
+ (
|
||||
f" (rc={enf.get('returncode')})"
|
||||
|
|
@ -1259,7 +1164,7 @@ def _report_markdown(report: Dict[str, Any]) -> str:
|
|||
)
|
||||
elif status == "failed":
|
||||
out.append(
|
||||
f"- ⚠️ Attempted enforcement but {via} failed"
|
||||
"- ⚠️ Attempted enforcement but failed"
|
||||
+ (
|
||||
f" (rc={enf.get('returncode')})"
|
||||
if enf.get("returncode") is not None
|
||||
|
|
|
|||
|
|
@ -27,6 +27,9 @@ DEFAULT_DENY_GLOBS = [
|
|||
"/etc/gshadow",
|
||||
"/etc/*shadow",
|
||||
"/etc/letsencrypt/*",
|
||||
"/etc/ppp/chap-secrets",
|
||||
"/etc/ppp/pap-secrets",
|
||||
"/etc/ppp/*-secrets",
|
||||
"/usr/local/etc/ssl/private/*",
|
||||
"/usr/local/etc/ssh/ssh_host_*",
|
||||
"/usr/local/etc/*shadow",
|
||||
|
|
@ -91,7 +94,20 @@ SENSITIVE_CONTENT_PATTERNS = [
|
|||
\s*[:=]
|
||||
"""
|
||||
),
|
||||
re.compile(rb"(?i)\b(pass|passwd|token|secret|api[_-]?key)\b"),
|
||||
re.compile(
|
||||
rb"(?i)\b(pass|passwd|password|passphrase|token|secret|"
|
||||
rb"credentials?|api[_-]?key)\b"
|
||||
),
|
||||
# Credentials embedded in connection-string URIs, e.g.
|
||||
# postgres://user:pass@host, redis://:pass@host, amqp://u:p@host
|
||||
# The keyword regex above keys on assignment-style names and misses these.
|
||||
re.compile(rb"(?i)[a-z][a-z0-9+.-]*://[^/\s:@]*:[^/\s@]+@[^/\s]"),
|
||||
# HTTP(S) Authorization / Proxy-Authorization header values carrying a
|
||||
# bearer/basic/digest credential.
|
||||
re.compile(
|
||||
rb"(?im)^\s*(?:proxy-)?authorization\s*:\s*"
|
||||
rb"(?:bearer|basic|token|digest)\s+\S"
|
||||
),
|
||||
]
|
||||
|
||||
COMMENT_PREFIXES = (b"#", b";", b"//")
|
||||
|
|
|
|||
|
|
@ -82,7 +82,6 @@ _JINJA_FOR_RE = re.compile(
|
|||
r"{%\s*for\s+([A-Za-z_][A-Za-z0-9_]*)\s+in\s+([A-Za-z_][A-Za-z0-9_]*)\b"
|
||||
)
|
||||
_JINJA_SPECIAL_VARS = {"loop", "true", "false", "none", "True", "False", "None"}
|
||||
_ERB_INSTANCE_VAR_RE = re.compile(r"<%=?[^%]*@([A-Za-z_][A-Za-z0-9_]*)", re.S)
|
||||
|
||||
|
||||
def _find_undeclared_jinja_vars(template_text: str) -> Set[str]:
|
||||
|
|
@ -121,21 +120,6 @@ def missing_jinja_template_vars(
|
|||
return {name for name in referenced if name not in context}
|
||||
|
||||
|
||||
def missing_erb_template_vars(template_text: str, context: Dict[str, Any]) -> Set[str]:
|
||||
"""Return ERB ``@param`` references absent from Puppet Hiera/class data."""
|
||||
|
||||
local_names: Set[str] = set()
|
||||
for key in context:
|
||||
text = str(key)
|
||||
if "::" in text:
|
||||
local_names.add(text.split("::", 1)[1])
|
||||
else:
|
||||
local_names.add(text)
|
||||
|
||||
referenced = set(_ERB_INSTANCE_VAR_RE.findall(template_text))
|
||||
return {name for name in referenced if name not in local_names}
|
||||
|
||||
|
||||
def jinjify_artifact(
|
||||
bundle_dir: str | Path,
|
||||
artifact_role: str,
|
||||
|
|
@ -147,14 +131,8 @@ def jinjify_artifact(
|
|||
jt_enabled: bool,
|
||||
overwrite_templates: bool = True,
|
||||
role_name: Optional[str] = None,
|
||||
template_engine: str = "jinja2",
|
||||
puppet_class: Optional[str] = None,
|
||||
) -> Optional[JinjifiedArtifact]:
|
||||
"""Best-effort conversion of one harvested artifact into a template.
|
||||
|
||||
Ansible/Salt use Jinja2 output. Puppet uses ERB output with Puppet Hiera
|
||||
keys when a new enough JinjaTurtle is available.
|
||||
"""
|
||||
"""Best-effort conversion of one harvested artifact into a Jinja2 template."""
|
||||
if not (jt_enabled and jt_exe and can_jinjify_path(dest_path)):
|
||||
return None
|
||||
|
||||
|
|
@ -164,31 +142,20 @@ def jinjify_artifact(
|
|||
return None
|
||||
|
||||
try:
|
||||
run_kwargs: Dict[str, Any] = {
|
||||
"role_name": role_name or artifact_role,
|
||||
"force_format": infer_other_formats(dest_path),
|
||||
}
|
||||
# Keep the historical call shape for Ansible/Salt and for tests that
|
||||
# monkeypatch run_jinjaturtle with the old signature. Puppet/ERB is
|
||||
# the only path that needs the newer JinjaTurtle CLI switches.
|
||||
if template_engine != "jinja2":
|
||||
run_kwargs["template_engine"] = template_engine
|
||||
if puppet_class:
|
||||
run_kwargs["puppet_class"] = puppet_class
|
||||
result = run_jinjaturtle(jt_exe, str(artifact_path), **run_kwargs)
|
||||
result = run_jinjaturtle(
|
||||
jt_exe,
|
||||
str(artifact_path),
|
||||
role_name=role_name or artifact_role,
|
||||
force_format=infer_other_formats(dest_path),
|
||||
)
|
||||
except Exception:
|
||||
return None # nosec - best-effort template generation
|
||||
|
||||
ext = "erb" if template_engine == "erb" else "j2"
|
||||
template_rel = Path(src_rel).as_posix() + f".{ext}"
|
||||
template_rel = Path(src_rel).as_posix() + ".j2"
|
||||
template_dst = Path(template_root) / template_rel
|
||||
|
||||
context = yaml_load_mapping(result.vars_text)
|
||||
missing = (
|
||||
missing_erb_template_vars(result.template_text, context)
|
||||
if template_engine == "erb"
|
||||
else missing_jinja_template_vars(result.template_text, context)
|
||||
)
|
||||
missing = missing_jinja_template_vars(result.template_text, context)
|
||||
if missing:
|
||||
# If this role was generated into an existing output directory, avoid
|
||||
# leaving an obsolete template behind after falling back to a raw copy.
|
||||
|
|
@ -243,10 +210,7 @@ def jinjify_managed_files(
|
|||
) -> Tuple[Set[str], str]:
|
||||
"""Jinjify a list of managed files and return Ansible-style vars text.
|
||||
|
||||
The return shape intentionally matches the historical Ansible helper:
|
||||
``(templated_src_rels, combined_vars_text)``. Salt uses
|
||||
:func:`jinjify_artifact` directly because it stores variables as a context
|
||||
map per managed file.
|
||||
The return shape is ``(templated_src_rels, combined_vars_text)``.
|
||||
"""
|
||||
templated: Set[str] = set()
|
||||
vars_map: Dict[str, Any] = {}
|
||||
|
|
@ -340,8 +304,6 @@ def run_jinjaturtle(
|
|||
*,
|
||||
role_name: str,
|
||||
force_format: Optional[str] = None,
|
||||
template_engine: str = "jinja2",
|
||||
puppet_class: Optional[str] = None,
|
||||
) -> JinjifyResult:
|
||||
"""
|
||||
Run jinjaturtle against src_path and return (template, defaults-yaml).
|
||||
|
|
@ -349,9 +311,6 @@ def run_jinjaturtle(
|
|||
|
||||
jinjaturtle CLI:
|
||||
jinjaturtle <config> -r <role> [-f <format>] [-d <defaults-output>] [-t <template-output>]
|
||||
|
||||
Newer JinjaTurtle versions also support ``--template-engine erb`` and
|
||||
``--puppet-class`` for Puppet/Hiera output.
|
||||
"""
|
||||
src = Path(src_path)
|
||||
if not src.is_file():
|
||||
|
|
@ -360,9 +319,7 @@ def run_jinjaturtle(
|
|||
with tempfile.TemporaryDirectory(prefix="enroll-jt-") as td:
|
||||
td_path = Path(td)
|
||||
defaults_out = td_path / "defaults.yml"
|
||||
template_out = td_path / (
|
||||
"template.erb" if template_engine == "erb" else "template.j2"
|
||||
)
|
||||
template_out = td_path / "template.j2"
|
||||
|
||||
cmd = [
|
||||
jt_exe,
|
||||
|
|
@ -376,10 +333,6 @@ def run_jinjaturtle(
|
|||
]
|
||||
if force_format:
|
||||
cmd.extend(["-f", force_format])
|
||||
if template_engine != "jinja2":
|
||||
cmd.extend(["--template-engine", template_engine])
|
||||
if puppet_class:
|
||||
cmd.extend(["--puppet-class", puppet_class])
|
||||
|
||||
p = subprocess.run(cmd, text=True, capture_output=True) # nosec
|
||||
if p.returncode != 0:
|
||||
|
|
|
|||
|
|
@ -8,8 +8,6 @@ from pathlib import Path
|
|||
from typing import List, Optional
|
||||
|
||||
from .ansible import manifest_from_bundle_dir as manifest_ansible_from_bundle_dir
|
||||
from .puppet import manifest_from_bundle_dir as manifest_puppet_from_bundle_dir
|
||||
from .salt import manifest_from_bundle_dir as manifest_salt_from_bundle_dir
|
||||
from .harvest_safety import ensure_safe_output_parent
|
||||
from .manifest_safety import validate_site_fqdn
|
||||
from .remote import _safe_extract_tar
|
||||
|
|
@ -175,7 +173,6 @@ def manifest(
|
|||
jinjaturtle: str = "auto", # auto|on|off
|
||||
sops_fingerprints: Optional[List[str]] = None,
|
||||
no_common_roles: bool = False,
|
||||
target: str = "ansible",
|
||||
) -> Optional[str]:
|
||||
"""Render a configuration-management manifest from a harvest.
|
||||
|
||||
|
|
@ -193,9 +190,6 @@ def manifest(
|
|||
- In SOPS mode: the path to the encrypted manifest bundle (.sops)
|
||||
- In plain mode: None
|
||||
"""
|
||||
target = (target or "ansible").strip().lower()
|
||||
if target not in {"ansible", "puppet", "salt"}:
|
||||
raise ValueError(f"unsupported manifest target: {target!r}")
|
||||
fqdn = validate_site_fqdn(fqdn)
|
||||
|
||||
sops_mode = bool(sops_fingerprints)
|
||||
|
|
@ -216,30 +210,13 @@ def manifest(
|
|||
)
|
||||
|
||||
if not sops_mode:
|
||||
if target == "puppet":
|
||||
manifest_puppet_from_bundle_dir(
|
||||
resolved_bundle_dir,
|
||||
out,
|
||||
fqdn=fqdn,
|
||||
no_common_roles=no_common_roles,
|
||||
jinjaturtle=jinjaturtle,
|
||||
)
|
||||
elif target == "salt":
|
||||
manifest_salt_from_bundle_dir(
|
||||
resolved_bundle_dir,
|
||||
out,
|
||||
fqdn=fqdn,
|
||||
no_common_roles=no_common_roles,
|
||||
jinjaturtle=jinjaturtle,
|
||||
)
|
||||
else:
|
||||
manifest_ansible_from_bundle_dir(
|
||||
resolved_bundle_dir,
|
||||
out,
|
||||
fqdn=fqdn,
|
||||
jinjaturtle=jinjaturtle,
|
||||
no_common_roles=no_common_roles,
|
||||
)
|
||||
manifest_ansible_from_bundle_dir(
|
||||
resolved_bundle_dir,
|
||||
out,
|
||||
fqdn=fqdn,
|
||||
jinjaturtle=jinjaturtle,
|
||||
no_common_roles=no_common_roles,
|
||||
)
|
||||
return None
|
||||
|
||||
# SOPS mode: generate into a secure temp dir, then tar+encrypt into a single file.
|
||||
|
|
@ -248,30 +225,13 @@ def manifest(
|
|||
td_out = tempfile.TemporaryDirectory(prefix="enroll-manifest-")
|
||||
tmp_out = Path(td_out.name) / "out"
|
||||
|
||||
if target == "puppet":
|
||||
manifest_puppet_from_bundle_dir(
|
||||
resolved_bundle_dir,
|
||||
str(tmp_out),
|
||||
fqdn=fqdn,
|
||||
no_common_roles=no_common_roles,
|
||||
jinjaturtle=jinjaturtle,
|
||||
)
|
||||
elif target == "salt":
|
||||
manifest_salt_from_bundle_dir(
|
||||
resolved_bundle_dir,
|
||||
str(tmp_out),
|
||||
fqdn=fqdn,
|
||||
no_common_roles=no_common_roles,
|
||||
jinjaturtle=jinjaturtle,
|
||||
)
|
||||
else:
|
||||
manifest_ansible_from_bundle_dir(
|
||||
resolved_bundle_dir,
|
||||
str(tmp_out),
|
||||
fqdn=fqdn,
|
||||
jinjaturtle=jinjaturtle,
|
||||
no_common_roles=no_common_roles,
|
||||
)
|
||||
manifest_ansible_from_bundle_dir(
|
||||
resolved_bundle_dir,
|
||||
str(tmp_out),
|
||||
fqdn=fqdn,
|
||||
jinjaturtle=jinjaturtle,
|
||||
no_common_roles=no_common_roles,
|
||||
)
|
||||
|
||||
enc = _encrypt_manifest_out_dir_to_sops(
|
||||
tmp_out, out_file, list(sops_fingerprints or [])
|
||||
|
|
|
|||
1840
enroll/puppet.py
1840
enroll/puppet.py
File diff suppressed because it is too large
Load diff
|
|
@ -1,7 +1,5 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping, Set as AbstractSet
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -50,183 +48,3 @@ def ansible_unsafe_data(value: Any) -> Any:
|
|||
if isinstance(value, AbstractSet):
|
||||
return sorted(ansible_unsafe_data(item) for item in value)
|
||||
return value
|
||||
|
||||
|
||||
def escape_puppet_hiera_interpolation(value: str) -> str:
|
||||
"""Preserve literal ``%{`` text in Puppet Hiera data sources.
|
||||
|
||||
Hiera treats ``%{...}`` in data values as interpolation. Enroll's Hiera
|
||||
data is generated from harvested values, not authored Hiera expressions, so
|
||||
any literal interpolation opener is escaped with Hiera's documented
|
||||
``literal('%')`` helper.
|
||||
"""
|
||||
|
||||
return str(value).replace("%{", "%{literal('%')}{")
|
||||
|
||||
|
||||
def puppet_hiera_safe_data(value: Any) -> Any:
|
||||
"""Recursively escape Hiera interpolation openers in harvested data."""
|
||||
|
||||
if isinstance(value, Mapping):
|
||||
return {
|
||||
escape_puppet_hiera_interpolation(str(key)): puppet_hiera_safe_data(inner)
|
||||
for key, inner in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [puppet_hiera_safe_data(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return [puppet_hiera_safe_data(item) for item in value]
|
||||
if isinstance(value, AbstractSet):
|
||||
return sorted(puppet_hiera_safe_data(item) for item in value)
|
||||
if isinstance(value, str):
|
||||
return escape_puppet_hiera_interpolation(value)
|
||||
return value
|
||||
|
||||
|
||||
def _plain_json_data(value: Any) -> Any:
|
||||
if isinstance(value, Mapping):
|
||||
return {str(key): _plain_json_data(inner) for key, inner in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_plain_json_data(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return [_plain_json_data(item) for item in value]
|
||||
if isinstance(value, AbstractSet):
|
||||
return sorted(_plain_json_data(item) for item in value)
|
||||
return value
|
||||
|
||||
|
||||
def _escape_braces_inside_json_strings(text: str) -> str:
|
||||
"""Replace literal braces only while scanning JSON string tokens."""
|
||||
|
||||
out: list[str] = []
|
||||
in_string = False
|
||||
escaped = False
|
||||
for ch in text:
|
||||
if not in_string:
|
||||
out.append(ch)
|
||||
if ch == '"':
|
||||
in_string = True
|
||||
continue
|
||||
|
||||
if escaped:
|
||||
out.append(ch)
|
||||
escaped = False
|
||||
elif ch == "\\":
|
||||
out.append(ch)
|
||||
escaped = True
|
||||
elif ch == '"':
|
||||
out.append(ch)
|
||||
in_string = False
|
||||
elif ch == "{":
|
||||
out.append("\\u007b")
|
||||
elif ch == "}":
|
||||
out.append("\\u007d")
|
||||
else:
|
||||
out.append(ch)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def salt_sls_json_quote(value: Any) -> str:
|
||||
"""Return a double-quoted YAML/JSON scalar safe for Salt's Jinja pass.
|
||||
|
||||
Salt state and pillar SLS files normally use the ``jinja|yaml`` renderer
|
||||
pipeline. YAML/JSON quoting alone does not stop ``{{ ... }}``, ``{% ... %}``
|
||||
or ``{# ... #}`` inside harvested values from being evaluated before YAML is
|
||||
parsed. JSON/YAML double-quoted scalars decode ``\u007b`` and ``\u007d``
|
||||
after Jinja has run, so encode braces inside string tokens as Unicode escapes.
|
||||
"""
|
||||
|
||||
dumped = json.dumps(str(value), ensure_ascii=False)
|
||||
return _escape_braces_inside_json_strings(dumped)
|
||||
|
||||
|
||||
_PLAIN_YAML_KEY_RE = re.compile(r"^[A-Za-z0-9_./:-]+$")
|
||||
|
||||
|
||||
def _salt_yaml_key(value: Any) -> str:
|
||||
text = str(value)
|
||||
if text and _PLAIN_YAML_KEY_RE.match(text) and not text.startswith(("-", "?", ":")):
|
||||
return text
|
||||
return salt_sls_json_quote(text)
|
||||
|
||||
|
||||
def _salt_yaml_scalar(value: Any) -> str:
|
||||
if value is None:
|
||||
return "null"
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, int) and not isinstance(value, bool):
|
||||
return str(value)
|
||||
if isinstance(value, float):
|
||||
return json.dumps(value, allow_nan=False)
|
||||
return salt_sls_json_quote(value)
|
||||
|
||||
|
||||
def _salt_yaml_lines(
|
||||
value: Any, indent: int = 0, *, sort_keys: bool = True
|
||||
) -> list[str]:
|
||||
prefix = " " * indent
|
||||
if isinstance(value, Mapping):
|
||||
if not value:
|
||||
return [prefix + "{}"]
|
||||
keys = sorted(value, key=lambda item: str(item)) if sort_keys else list(value)
|
||||
lines: list[str] = []
|
||||
for key in keys:
|
||||
inner = value[key]
|
||||
key_text = _salt_yaml_key(key)
|
||||
if isinstance(inner, Mapping):
|
||||
if not inner:
|
||||
lines.append(f"{prefix}{key_text}: {{}}")
|
||||
else:
|
||||
lines.append(f"{prefix}{key_text}:")
|
||||
lines.extend(
|
||||
_salt_yaml_lines(inner, indent + 2, sort_keys=sort_keys)
|
||||
)
|
||||
elif isinstance(inner, (list, tuple, set)):
|
||||
seq = list(inner) if not isinstance(inner, set) else sorted(inner)
|
||||
if not seq:
|
||||
lines.append(f"{prefix}{key_text}: []")
|
||||
else:
|
||||
lines.append(f"{prefix}{key_text}:")
|
||||
lines.extend(_salt_yaml_lines(seq, indent + 2, sort_keys=sort_keys))
|
||||
else:
|
||||
lines.append(f"{prefix}{key_text}: {_salt_yaml_scalar(inner)}")
|
||||
return lines
|
||||
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
seq = list(value) if not isinstance(value, set) else sorted(value)
|
||||
if not seq:
|
||||
return [prefix + "[]"]
|
||||
lines = []
|
||||
for item in seq:
|
||||
if isinstance(item, Mapping):
|
||||
if not item:
|
||||
lines.append(prefix + "- {}")
|
||||
else:
|
||||
lines.append(prefix + "-")
|
||||
lines.extend(
|
||||
_salt_yaml_lines(item, indent + 2, sort_keys=sort_keys)
|
||||
)
|
||||
elif isinstance(item, (list, tuple, set)):
|
||||
lines.append(prefix + "-")
|
||||
lines.extend(_salt_yaml_lines(item, indent + 2, sort_keys=sort_keys))
|
||||
else:
|
||||
lines.append(f"{prefix}- {_salt_yaml_scalar(item)}")
|
||||
return lines
|
||||
|
||||
return [prefix + _salt_yaml_scalar(value)]
|
||||
|
||||
|
||||
def salt_sls_yaml_dump(
|
||||
value: Any,
|
||||
*,
|
||||
sort_keys: bool = True,
|
||||
explicit_start: bool = False,
|
||||
) -> str:
|
||||
"""Dump block YAML whose string braces cannot form Salt Jinja delimiters."""
|
||||
|
||||
lines = _salt_yaml_lines(_plain_json_data(value), sort_keys=sort_keys)
|
||||
rendered = "\n".join(lines).rstrip() + "\n"
|
||||
if explicit_start:
|
||||
rendered = "---\n" + rendered
|
||||
return rendered
|
||||
|
|
|
|||
1759
enroll/salt.py
1759
enroll/salt.py
File diff suppressed because it is too large
Load diff
|
|
@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api"
|
|||
[project]
|
||||
name = "enroll"
|
||||
version = "0.7.0b7"
|
||||
description = "Enroll a server's running state retrospectively into Ansible, Puppet or Salt"
|
||||
description = "Enroll a server's running state retrospectively into Ansible"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
license = "GPL-3.0-or-later"
|
||||
|
|
|
|||
140
tests.sh
140
tests.sh
|
|
@ -2,10 +2,6 @@
|
|||
|
||||
set -Eeuo pipefail
|
||||
|
||||
if [[ -d /opt/puppetlabs/bin ]]; then
|
||||
export PATH="/opt/puppetlabs/bin:${PATH}"
|
||||
fi
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TMP_PARENT="${TMPDIR:-/tmp}"
|
||||
KEEP_WORKDIR=0
|
||||
|
|
@ -22,16 +18,8 @@ BUNDLE_DIFF_DIR="${WORK_DIR}/bundle-diff"
|
|||
ANSIBLE_DIR="${WORK_DIR}/ansible"
|
||||
ANSIBLE_NO_COMMON_DIR="${WORK_DIR}/ansible-no-common"
|
||||
ANSIBLE_FQDN_DIR="${WORK_DIR}/ansible-fqdn"
|
||||
PUPPET_DIR="${WORK_DIR}/puppet"
|
||||
PUPPET_FQDN_DIR="${WORK_DIR}/puppet-fqdn"
|
||||
SALT_DIR="${WORK_DIR}/salt"
|
||||
SALT_FQDN_DIR="${WORK_DIR}/salt-fqdn"
|
||||
ANSIBLE_JINJATURTLE_DIR="${WORK_DIR}/ansible-jinjaturtle"
|
||||
ANSIBLE_NO_JINJATURTLE_DIR="${WORK_DIR}/ansible-no-jinjaturtle"
|
||||
PUPPET_JINJATURTLE_DIR="${WORK_DIR}/puppet-jinjaturtle"
|
||||
PUPPET_NO_JINJATURTLE_DIR="${WORK_DIR}/puppet-no-jinjaturtle"
|
||||
SALT_JINJATURTLE_DIR="${WORK_DIR}/salt-jinjaturtle"
|
||||
SALT_NO_JINJATURTLE_DIR="${WORK_DIR}/salt-no-jinjaturtle"
|
||||
TEST_FQDN="${ENROLL_TEST_FQDN:-enroll-ci.example.test}"
|
||||
JINJATURTLE_FIXTURE="${WORK_DIR}/enroll-tests-jinjaturtle.ini"
|
||||
ANSIBLE_PLAYBOOK_EXTRA_ARGS=()
|
||||
|
|
@ -176,7 +164,6 @@ translate_packages() {
|
|||
gnupg) translated+=(gnupg2) ;;
|
||||
curl) translated+=(curl-minimal) ;;
|
||||
lsb-release) translated+=(redhat-lsb-core) ;;
|
||||
puppet) translated+=(puppet-agent) ;;
|
||||
python3-apt) ;;
|
||||
python3-jsonschema) translated+=(python3-jsonschema) ;;
|
||||
python3-venv) ;;
|
||||
|
|
@ -232,42 +219,6 @@ ensure_epel_repo() {
|
|||
DNF_UPDATED=
|
||||
}
|
||||
|
||||
ensure_salt_repo() {
|
||||
if is_debian; then
|
||||
if [[ -e /etc/apt/sources.list.d/salt.sources ]]; then
|
||||
return
|
||||
fi
|
||||
section "Setup: Salt apt repository"
|
||||
pkg_install ca-certificates curl gnupg
|
||||
run mkdir -m 755 -p /etc/apt/keyrings
|
||||
run bash -c "curl -fsSL https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public | gpg --dearmor --yes -o /etc/apt/keyrings/salt-archive-keyring.pgp"
|
||||
run bash -c "curl -fsSL https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources > /etc/apt/sources.list.d/salt.sources"
|
||||
APT_UPDATED=
|
||||
elif is_rpm_family; then
|
||||
if [[ -e /etc/yum.repos.d/salt.repo ]]; then
|
||||
return
|
||||
fi
|
||||
section "Setup: Salt dnf repository"
|
||||
pkg_install ca-certificates curl
|
||||
run bash -c "curl -fsSL https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo > /etc/yum.repos.d/salt.repo"
|
||||
DNF_UPDATED=
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_puppet_repo() {
|
||||
if ! is_rpm_family; then
|
||||
return
|
||||
fi
|
||||
if rpm -q puppet8-release >/dev/null 2>&1 || [[ -e /etc/yum.repos.d/puppet8-release.repo ]]; then
|
||||
return
|
||||
fi
|
||||
section "Setup: Puppet dnf repository"
|
||||
local major
|
||||
major="$(os_version_major)"
|
||||
run dnf -y install "https://yum.puppet.com/puppet8-release-el-${major}.noarch.rpm"
|
||||
DNF_UPDATED=
|
||||
}
|
||||
|
||||
ensure_jinjaturtle() {
|
||||
section "Setup: JinjaTurtle package"
|
||||
if command -v jinjaturtle >/dev/null 2>&1; then
|
||||
|
|
@ -311,22 +262,6 @@ ensure_ansible() {
|
|||
require_cmd ansible-lint "Install the ansible-lint package."
|
||||
}
|
||||
|
||||
ensure_puppet() {
|
||||
ensure_puppet_repo
|
||||
if ! command -v puppet >/dev/null 2>&1; then
|
||||
pkg_install puppet || pkg_install puppet-agent
|
||||
fi
|
||||
require_cmd puppet "Install Puppet before running the Puppet noop integration tests."
|
||||
}
|
||||
|
||||
ensure_salt() {
|
||||
ensure_salt_repo
|
||||
if ! command -v salt-call >/dev/null 2>&1; then
|
||||
pkg_install salt-minion || true
|
||||
fi
|
||||
require_cmd salt-call "Install Salt's salt-call binary before running the Salt noop integration tests. This may require configuring the upstream Salt/Broadcom package repository first."
|
||||
}
|
||||
|
||||
run_pytests() {
|
||||
section "Python unit tests"
|
||||
cd "${PROJECT_ROOT}"
|
||||
|
|
@ -396,34 +331,6 @@ run_ansible_jinjaturtle_variant() {
|
|||
run ansible-playbook playbook.yml -i "localhost," -c local --check --diff "${ANSIBLE_PLAYBOOK_EXTRA_ARGS[@]}"
|
||||
}
|
||||
|
||||
run_puppet_jinjaturtle_variant() {
|
||||
local out_dir="$1"
|
||||
local expected="$2"
|
||||
local label="$3"
|
||||
shift 3
|
||||
|
||||
ensure_puppet
|
||||
cd "${PROJECT_ROOT}"
|
||||
rm -rf "${out_dir}"
|
||||
run poetry run enroll manifest --harvest "${BUNDLE_DIR}" --out "${out_dir}" --target puppet "$@"
|
||||
assert_template_files "${out_dir}" "erb" "${expected}" "${label}"
|
||||
run puppet apply --modulepath "${out_dir}/modules" "${out_dir}/manifests/site.pp" --noop
|
||||
}
|
||||
|
||||
run_salt_jinjaturtle_variant() {
|
||||
local out_dir="$1"
|
||||
local expected="$2"
|
||||
local label="$3"
|
||||
shift 3
|
||||
|
||||
ensure_salt
|
||||
cd "${PROJECT_ROOT}"
|
||||
rm -rf "${out_dir}"
|
||||
run poetry run enroll manifest --harvest "${BUNDLE_DIR}" --out "${out_dir}" --target salt "$@"
|
||||
assert_template_files "${out_dir}" "j2" "${expected}" "${label}"
|
||||
run salt-call --local --retcode-passthrough --file-root "${out_dir}/states" state.apply test=True
|
||||
}
|
||||
|
||||
run_jinjaturtle_manifest_tests() {
|
||||
if is_rpm_family ; then
|
||||
section "JinjaTurtle integration matrix"
|
||||
|
|
@ -437,14 +344,6 @@ run_jinjaturtle_manifest_tests() {
|
|||
section "Ansible JinjaTurtle manifest noop tests"
|
||||
run_ansible_jinjaturtle_variant "${ANSIBLE_JINJATURTLE_DIR}" present "Ansible with JinjaTurtle on PATH"
|
||||
run_ansible_jinjaturtle_variant "${ANSIBLE_NO_JINJATURTLE_DIR}" absent "Ansible with --no-jinjaturtle" --no-jinjaturtle
|
||||
|
||||
section "Puppet JinjaTurtle manifest noop tests"
|
||||
run_puppet_jinjaturtle_variant "${PUPPET_JINJATURTLE_DIR}" present "Puppet with JinjaTurtle on PATH"
|
||||
run_puppet_jinjaturtle_variant "${PUPPET_NO_JINJATURTLE_DIR}" absent "Puppet with --no-jinjaturtle" --no-jinjaturtle
|
||||
|
||||
section "Salt JinjaTurtle manifest noop tests"
|
||||
run_salt_jinjaturtle_variant "${SALT_JINJATURTLE_DIR}" present "Salt with JinjaTurtle on PATH"
|
||||
run_salt_jinjaturtle_variant "${SALT_NO_JINJATURTLE_DIR}" absent "Salt with --no-jinjaturtle" --no-jinjaturtle
|
||||
}
|
||||
|
||||
run_ansible_noop_tests() {
|
||||
|
|
@ -472,43 +371,6 @@ run_ansible_noop_tests() {
|
|||
run ansible-playbook "playbooks/${TEST_FQDN}.yml" -i inventory/hosts.ini -c local --limit "${TEST_FQDN}" --check --diff "${ANSIBLE_PLAYBOOK_EXTRA_ARGS[@]}"
|
||||
}
|
||||
|
||||
run_puppet_noop_tests() {
|
||||
section "Puppet manifest noop tests"
|
||||
ensure_puppet
|
||||
cd "${PROJECT_ROOT}"
|
||||
rm -rf "${PUPPET_DIR}" "${PUPPET_FQDN_DIR}"
|
||||
|
||||
run poetry run enroll manifest --harvest "${BUNDLE_DIR}" --out "${PUPPET_DIR}" --target puppet
|
||||
run puppet apply --modulepath "${PUPPET_DIR}/modules" "${PUPPET_DIR}/manifests/site.pp" --noop
|
||||
|
||||
run poetry run enroll manifest --harvest "${BUNDLE_DIR}" --out "${PUPPET_FQDN_DIR}" --target puppet --fqdn "${TEST_FQDN}"
|
||||
run puppet apply \
|
||||
--modulepath "${PUPPET_FQDN_DIR}/modules" \
|
||||
--hiera_config "${PUPPET_FQDN_DIR}/hiera.yaml" \
|
||||
--certname "${TEST_FQDN}" \
|
||||
"${PUPPET_FQDN_DIR}/manifests/site.pp" \
|
||||
--noop
|
||||
}
|
||||
|
||||
run_salt_noop_tests() {
|
||||
section "Salt manifest noop tests"
|
||||
ensure_salt
|
||||
cd "${PROJECT_ROOT}"
|
||||
rm -rf "${SALT_DIR}" "${SALT_FQDN_DIR}"
|
||||
|
||||
run poetry run enroll manifest --harvest "${BUNDLE_DIR}" --out "${SALT_DIR}" --target salt
|
||||
run salt-call --local --retcode-passthrough --file-root "${SALT_DIR}/states" state.apply test=True
|
||||
|
||||
run poetry run enroll manifest --harvest "${BUNDLE_DIR}" --out "${SALT_FQDN_DIR}" --target salt --fqdn "${TEST_FQDN}"
|
||||
run salt-call \
|
||||
--local \
|
||||
--retcode-passthrough \
|
||||
--id "${TEST_FQDN}" \
|
||||
--file-root "${SALT_FQDN_DIR}/states" \
|
||||
--pillar-root "${SALT_FQDN_DIR}/pillar" \
|
||||
state.apply test=True
|
||||
}
|
||||
|
||||
main() {
|
||||
require_root
|
||||
require_supported_ci_os
|
||||
|
|
@ -516,8 +378,6 @@ main() {
|
|||
prepare_harvest_fixture
|
||||
configure_ansible_playbook_extra_args
|
||||
run_ansible_noop_tests
|
||||
run_puppet_noop_tests
|
||||
run_salt_noop_tests
|
||||
run_jinjaturtle_manifest_tests
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -48,7 +48,6 @@ def test_cli_manifest_subcommand_calls_manifest(monkeypatch, tmp_path):
|
|||
called["fqdn"] = kwargs.get("fqdn")
|
||||
called["jinjaturtle"] = kwargs.get("jinjaturtle")
|
||||
called["no_common_roles"] = kwargs.get("no_common_roles")
|
||||
called["target"] = kwargs.get("target")
|
||||
|
||||
monkeypatch.setattr(cli, "manifest", fake_manifest)
|
||||
monkeypatch.setattr(
|
||||
|
|
@ -70,7 +69,6 @@ def test_cli_manifest_subcommand_calls_manifest(monkeypatch, tmp_path):
|
|||
assert called["fqdn"] is None
|
||||
assert called["jinjaturtle"] == "auto"
|
||||
assert called["no_common_roles"] is False
|
||||
assert called["target"] == "ansible"
|
||||
|
||||
|
||||
def test_cli_force_unsafe_path_before_subcommand_reaches_guard(monkeypatch, tmp_path):
|
||||
|
|
@ -131,36 +129,6 @@ def test_cli_force_unsafe_path_after_subcommand_reaches_guard(monkeypatch, tmp_p
|
|||
assert seen["force"] is True
|
||||
|
||||
|
||||
def test_cli_manifest_target_puppet_is_forwarded(monkeypatch, tmp_path):
|
||||
called = {}
|
||||
|
||||
def fake_manifest(harvest_dir: str, out_dir: str, **kwargs):
|
||||
called["harvest"] = harvest_dir
|
||||
called["out"] = out_dir
|
||||
called["target"] = kwargs.get("target")
|
||||
|
||||
monkeypatch.setattr(cli, "manifest", fake_manifest)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"enroll",
|
||||
"manifest",
|
||||
"--harvest",
|
||||
str(tmp_path / "bundle"),
|
||||
"--out",
|
||||
str(tmp_path / "puppet"),
|
||||
"--target",
|
||||
"puppet",
|
||||
],
|
||||
)
|
||||
|
||||
cli.main()
|
||||
assert called["harvest"] == str(tmp_path / "bundle")
|
||||
assert called["out"] == str(tmp_path / "puppet")
|
||||
assert called["target"] == "puppet"
|
||||
|
||||
|
||||
def test_cli_manifest_no_common_roles_is_forwarded(monkeypatch, tmp_path):
|
||||
called = {}
|
||||
|
||||
|
|
|
|||
|
|
@ -1025,7 +1025,7 @@ def test_report_markdown_with_enforcement_failed():
|
|||
},
|
||||
}
|
||||
result = _report_markdown(report)
|
||||
assert "ansible-playbook failed" in result
|
||||
assert "but failed" in result
|
||||
|
||||
|
||||
def test_report_markdown_with_enforcement_skipped():
|
||||
|
|
@ -1378,7 +1378,7 @@ def test_report_text_with_enforcement_applied():
|
|||
}
|
||||
result = d._report_text(report)
|
||||
assert "Enforcement" in result
|
||||
assert "applied old harvest via ansible-playbook" in result
|
||||
assert "applied old harvest" in result
|
||||
assert "tags=test" in result
|
||||
|
||||
|
||||
|
|
@ -1398,7 +1398,7 @@ def test_report_text_with_enforcement_failed():
|
|||
}
|
||||
result = d._report_text(report)
|
||||
assert "Enforcement" in result
|
||||
assert "ansible-playbook failed" in result
|
||||
assert "failed" in result
|
||||
|
||||
|
||||
def test_report_text_with_enforcement_skipped():
|
||||
|
|
|
|||
|
|
@ -218,18 +218,6 @@ def test_diff_exclude_path_only_filters_files_not_packages(tmp_path: Path):
|
|||
assert report["packages"]["added"] == ["htop"]
|
||||
|
||||
|
||||
def test_enforce_old_harvest_requires_ansible_playbook(monkeypatch, tmp_path: Path):
|
||||
import enroll.diff as d
|
||||
|
||||
monkeypatch.setattr(d.shutil, "which", lambda name: None)
|
||||
|
||||
old = tmp_path / "old"
|
||||
_write_bundle(old, {"inventory": {"packages": {}}, "roles": _minimal_roles()})
|
||||
|
||||
with pytest.raises(RuntimeError, match="ansible-playbook not found"):
|
||||
d.enforce_old_harvest(str(old))
|
||||
|
||||
|
||||
def test_enforce_old_harvest_runs_ansible_with_tags_from_file_drift(
|
||||
monkeypatch, tmp_path: Path
|
||||
):
|
||||
|
|
@ -310,121 +298,6 @@ def test_enforce_old_harvest_runs_ansible_with_tags_from_file_drift(
|
|||
assert "role_usr_local_custom" in str(argv[i + 1])
|
||||
|
||||
|
||||
def test_enforce_old_harvest_runs_puppet_target(monkeypatch, tmp_path: Path):
|
||||
import enroll.diff as d
|
||||
import enroll.manifest as mf
|
||||
|
||||
monkeypatch.setattr(
|
||||
d.shutil,
|
||||
"which",
|
||||
lambda name: "/usr/bin/puppet" if name == "puppet" else None,
|
||||
)
|
||||
|
||||
calls: dict[str, object] = {}
|
||||
|
||||
def fake_manifest(_harvest_dir: str, out_dir: str, **kwargs):
|
||||
calls["manifest_target"] = kwargs.get("target")
|
||||
out = Path(out_dir)
|
||||
(out / "manifests").mkdir(parents=True)
|
||||
(out / "modules").mkdir(parents=True)
|
||||
(out / "manifests" / "site.pp").write_text(
|
||||
"node default { }\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(mf, "manifest", fake_manifest)
|
||||
|
||||
def fake_run(
|
||||
argv, cwd=None, env=None, capture_output=False, text=False, check=False
|
||||
):
|
||||
calls["argv"] = list(argv)
|
||||
calls["cwd"] = cwd
|
||||
return types.SimpleNamespace(returncode=0, stdout="ok", stderr="")
|
||||
|
||||
monkeypatch.setattr(d.subprocess, "run", fake_run)
|
||||
|
||||
old = tmp_path / "old"
|
||||
_write_bundle(old, {"inventory": {"packages": {}}, "roles": _minimal_roles()})
|
||||
|
||||
report = {
|
||||
"packages": {"added": [], "removed": ["curl"], "version_changed": []},
|
||||
"services": {"enabled_added": [], "enabled_removed": [], "changed": []},
|
||||
"users": {"added": [], "removed": [], "changed": []},
|
||||
"files": {"added": [], "removed": [], "changed": []},
|
||||
}
|
||||
|
||||
info = d.enforce_old_harvest(str(old), report=report, target="puppet")
|
||||
|
||||
assert info["status"] == "applied"
|
||||
assert info["target"] == "puppet"
|
||||
assert info["tool"] == "puppet apply"
|
||||
assert info["scope"] == "full_manifest"
|
||||
assert info["tags"] == []
|
||||
assert calls["manifest_target"] == "puppet"
|
||||
|
||||
argv = calls.get("argv")
|
||||
assert argv and argv[:2] == ["/usr/bin/puppet", "apply"]
|
||||
assert "--modulepath" in argv
|
||||
assert any(
|
||||
str(Path(calls["cwd"]) / "manifest" / "manifests" / "site.pp") == str(a)
|
||||
for a in argv
|
||||
)
|
||||
|
||||
|
||||
def test_enforce_old_harvest_runs_salt_target(monkeypatch, tmp_path: Path):
|
||||
import enroll.diff as d
|
||||
import enroll.manifest as mf
|
||||
|
||||
monkeypatch.setattr(
|
||||
d.shutil,
|
||||
"which",
|
||||
lambda name: "/usr/bin/salt-call" if name == "salt-call" else None,
|
||||
)
|
||||
|
||||
calls: dict[str, object] = {}
|
||||
|
||||
def fake_manifest(_harvest_dir: str, out_dir: str, **kwargs):
|
||||
calls["manifest_target"] = kwargs.get("target")
|
||||
out = Path(out_dir)
|
||||
(out / "states").mkdir(parents=True)
|
||||
(out / "states" / "top.sls").write_text("base:\n '*': []\n", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(mf, "manifest", fake_manifest)
|
||||
|
||||
def fake_run(
|
||||
argv, cwd=None, env=None, capture_output=False, text=False, check=False
|
||||
):
|
||||
calls["argv"] = list(argv)
|
||||
calls["cwd"] = cwd
|
||||
return types.SimpleNamespace(returncode=0, stdout="ok", stderr="")
|
||||
|
||||
monkeypatch.setattr(d.subprocess, "run", fake_run)
|
||||
|
||||
old = tmp_path / "old"
|
||||
_write_bundle(old, {"inventory": {"packages": {}}, "roles": _minimal_roles()})
|
||||
|
||||
report = {
|
||||
"packages": {"added": [], "removed": ["curl"], "version_changed": []},
|
||||
"services": {"enabled_added": [], "enabled_removed": [], "changed": []},
|
||||
"users": {"added": [], "removed": [], "changed": []},
|
||||
"files": {"added": [], "removed": [], "changed": []},
|
||||
}
|
||||
|
||||
info = d.enforce_old_harvest(str(old), report=report, target="salt")
|
||||
|
||||
assert info["status"] == "applied"
|
||||
assert info["target"] == "salt"
|
||||
assert info["tool"] == "salt-call"
|
||||
assert info["scope"] == "full_manifest"
|
||||
assert calls["manifest_target"] == "salt"
|
||||
|
||||
argv = calls.get("argv")
|
||||
assert argv and argv[0] == "/usr/bin/salt-call"
|
||||
assert "--local" in argv
|
||||
assert "--file-root" in argv
|
||||
assert "state.apply" in argv
|
||||
assert str(Path(calls["cwd"]) / "manifest" / "states") in argv
|
||||
|
||||
|
||||
def test_cli_diff_enforce_forwards_target(monkeypatch):
|
||||
import enroll.cli as cli
|
||||
|
||||
|
|
@ -458,17 +331,38 @@ def test_cli_diff_enforce_forwards_target(monkeypatch):
|
|||
"--new",
|
||||
"/tmp/new",
|
||||
"--enforce",
|
||||
"--target",
|
||||
"puppet",
|
||||
],
|
||||
)
|
||||
|
||||
cli.main()
|
||||
assert calls["old"] == "/tmp/old"
|
||||
assert calls["target"] == "puppet"
|
||||
assert calls["report"] is report
|
||||
|
||||
|
||||
def test_cli_diff_enforce_rejects_non_ansible_target(monkeypatch):
|
||||
"""Salt and Puppet targets have been removed from --enforce."""
|
||||
import enroll.cli as cli
|
||||
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"enroll",
|
||||
"diff",
|
||||
"--old",
|
||||
"/tmp/old",
|
||||
"--new",
|
||||
"/tmp/new",
|
||||
"--enforce",
|
||||
"--target",
|
||||
"puppet",
|
||||
],
|
||||
)
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
cli.main()
|
||||
|
||||
|
||||
def test_cli_diff_forwards_exclude_and_ignore_flags(monkeypatch, capsys):
|
||||
import enroll.cli as cli
|
||||
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ def test_ansible_static_marks_harvested_jinja_values_unsafe(tmp_path: Path):
|
|||
payload = "{{ lookup('pipe','touch /tmp/PWNED_BY_ENROLL_ANSIBLE') }}"
|
||||
write_schema_state(bundle, _ansible_jinja_payload_state(payload))
|
||||
|
||||
manifest.manifest(str(bundle), str(out), target="ansible")
|
||||
manifest.manifest(str(bundle), str(out))
|
||||
|
||||
defaults = out / "roles" / "users" / "defaults" / "main.yml"
|
||||
text = defaults.read_text(encoding="utf-8")
|
||||
|
|
@ -152,7 +152,7 @@ def test_ansible_fqdn_marks_harvested_jinja_values_unsafe(tmp_path: Path):
|
|||
payload = "{{ lookup('pipe','touch /tmp/PWNED_BY_ENROLL_ANSIBLE') }}"
|
||||
write_schema_state(bundle, _ansible_jinja_payload_state(payload))
|
||||
|
||||
manifest.manifest(str(bundle), str(out), target="ansible", fqdn="host.example.test")
|
||||
manifest.manifest(str(bundle), str(out), fqdn="host.example.test")
|
||||
|
||||
hostvars = out / "inventory" / "host_vars" / "host.example.test" / "users.yml"
|
||||
text = hostvars.read_text(encoding="utf-8")
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
74
tests/test_secret_detection.py
Normal file
74
tests/test_secret_detection.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
"""Regression test: safe-mode content sniff must catch URI-embedded credentials
|
||||
and HTTP Authorization headers, not just assignment-style credential keys."""
|
||||
|
||||
from enroll.ignore import IgnorePolicy
|
||||
|
||||
|
||||
def test_uri_embedded_credentials_denied():
|
||||
pol = IgnorePolicy(dangerous=False)
|
||||
leaking = [
|
||||
b"DATABASE_URL=postgresql://admin:S3cr3tPass@db:5432/app\n",
|
||||
b"REDIS_URL=redis://:mypassword@localhost:6379/0\n",
|
||||
b"broker_url = amqp://guest:guest@rabbit:5672//\n",
|
||||
b"uri: mongodb+srv://user:p%40ss@cluster0.mongodb.net\n",
|
||||
]
|
||||
for data in leaking:
|
||||
assert (
|
||||
pol._content_deny_reason("/etc/svc/app.conf", data) == "sensitive_content"
|
||||
), data
|
||||
|
||||
|
||||
def test_authorization_headers_denied():
|
||||
pol = IgnorePolicy(dangerous=False)
|
||||
for data in [
|
||||
b"Authorization: Bearer eyJhbGciOiJIUzI1NiIs\n",
|
||||
b"Authorization: Basic dXNlcjpwYXNzd29yZA==\n",
|
||||
b"Proxy-Authorization: Bearer xyz\n",
|
||||
]:
|
||||
assert (
|
||||
pol._content_deny_reason("/etc/svc/app.conf", data) == "sensitive_content"
|
||||
), data
|
||||
|
||||
|
||||
def test_benign_urls_not_false_positive():
|
||||
pol = IgnorePolicy(dangerous=False)
|
||||
for data in [
|
||||
b"homepage = https://example.com/docs\n",
|
||||
b"server = http://localhost:8080/api\n",
|
||||
b"origin = git://github.com/u/repo.git\n",
|
||||
b"bind = http://[::1]:9000/\n",
|
||||
b"contact = mailto:admin@example.com\n",
|
||||
b"log_level = info\nworkers = 4\n",
|
||||
]:
|
||||
assert pol._content_deny_reason("/etc/svc/app.conf", data) is None, data
|
||||
|
||||
|
||||
def test_bare_word_password_denied():
|
||||
"""Credential lines using the word password/passphrase/credentials without
|
||||
an assignment delimiter (netrc, ppp, space/tab separated) must be denied."""
|
||||
pol = IgnorePolicy(dangerous=False)
|
||||
for data in [
|
||||
b"machine api.example.com login bob password s3cret\n",
|
||||
b"passphrase mysecretphrase\n",
|
||||
b"credentials /run/creds/db\n",
|
||||
]:
|
||||
assert (
|
||||
pol._content_deny_reason("/etc/svc.conf", data) == "sensitive_content"
|
||||
), data
|
||||
|
||||
|
||||
def test_ppp_secrets_paths_denied():
|
||||
pol = IgnorePolicy(dangerous=False)
|
||||
for p in ("/etc/ppp/chap-secrets", "/etc/ppp/pap-secrets", "/etc/ppp/my-secrets"):
|
||||
assert pol._path_deny_reason(p) == "denied_path", p
|
||||
|
||||
|
||||
def test_password_keyword_no_false_positives():
|
||||
pol = IgnorePolicy(dangerous=False)
|
||||
for data in [
|
||||
b"passive_mode = true\n",
|
||||
b"PassengerRoot /usr/lib\n",
|
||||
b"description = reset the account\n",
|
||||
b"compression = gzip\n",
|
||||
]:
|
||||
assert pol._content_deny_reason("/etc/app.conf", data) is None, data
|
||||
Loading…
Add table
Add a link
Reference in a new issue