Merge branch 'erb'

This commit is contained in:
Miguel Jacq 2026-06-21 09:03:33 +10:00
commit 97b64522c6
Signed by: mig5
GPG key ID: 03906B4110AAD3B8
8 changed files with 959 additions and 75 deletions

View file

@ -295,7 +295,7 @@ If you intend to keep harvests/manifests long-term (especially in git), strongly
## JinjaTurtle integration ## JinjaTurtle integration
If [JinjaTurtle](https://git.mig5.net/mig5/jinjaturtle) is installed, `enroll` can generate Jinja2 templates for ini/json/xml/toml-style config in renderers that consume Jinja templates. If [JinjaTurtle](https://git.mig5.net/mig5/jinjaturtle) is installed, `enroll` can generate templates for ini/json/xml/toml-style config in renderers.
For Ansible: For Ansible:
- Templates live in `roles/<role>/templates/...` - Templates live in `roles/<role>/templates/...`
@ -308,7 +308,10 @@ For Salt:
- `file.managed` uses `template: jinja` with per-file `context` values - `file.managed` uses `template: jinja` with per-file `context` values
- In `--fqdn` mode, template context values are written to pillar with the file metadata - In `--fqdn` mode, template context values are written to pillar with the file metadata
Puppet output does not use `.erb` templates at this time. For Puppet:
- JinjaTurtle will use its 'erb' mode if you are running a recent-enough version.
- Templates will be stored in `modules/<module>/templates/<file>.erb`
- In `--fqdn` mode, template context values are written to Hiera data.
You can force template generation on with `--jinjaturtle` or disable it with `--no-jinjaturtle`. You can force template generation on with `--jinjaturtle` or disable it with `--no-jinjaturtle`.

View file

@ -816,7 +816,7 @@ def _render_readme(
- `roles/<role>/files/...` and `roles/<role>/templates/...` contain reusable role artifacts where applicable.""" - `roles/<role>/files/...` and `roles/<role>/templates/...` contain reusable role artifacts where applicable."""
apply = f"""```bash apply = f"""```bash
ansible-galaxy collection install -r requirements.yml ansible-galaxy collection install -r requirements.yml
ansible-playbook -i inventory/hosts.ini playbooks/{fqdn}.yml --check ansible-playbook -i inventory/hosts.ini playbooks/{fqdn}.yml --check --diff
```""" ```"""
else: else:
layout = """- `playbook.yml` applies the generated roles to the current inventory. layout = """- `playbook.yml` applies the generated roles to the current inventory.

View file

@ -81,6 +81,7 @@ _JINJA_FOR_RE = re.compile(
r"{%\s*for\s+([A-Za-z_][A-Za-z0-9_]*)\s+in\s+([A-Za-z_][A-Za-z0-9_]*)\b" r"{%\s*for\s+([A-Za-z_][A-Za-z0-9_]*)\s+in\s+([A-Za-z_][A-Za-z0-9_]*)\b"
) )
_JINJA_SPECIAL_VARS = {"loop", "true", "false", "none", "True", "False", "None"} _JINJA_SPECIAL_VARS = {"loop", "true", "false", "none", "True", "False", "None"}
_ERB_INSTANCE_VAR_RE = re.compile(r"<%=?[^%]*@([A-Za-z_][A-Za-z0-9_]*)", re.S)
def _find_undeclared_jinja_vars(template_text: str) -> Set[str]: def _find_undeclared_jinja_vars(template_text: str) -> Set[str]:
@ -119,6 +120,21 @@ def missing_jinja_template_vars(
return {name for name in referenced if name not in context} return {name for name in referenced if name not in context}
def missing_erb_template_vars(template_text: str, context: Dict[str, Any]) -> Set[str]:
"""Return ERB ``@param`` references absent from Puppet Hiera/class data."""
local_names: Set[str] = set()
for key in context:
text = str(key)
if "::" in text:
local_names.add(text.split("::", 1)[1])
else:
local_names.add(text)
referenced = set(_ERB_INSTANCE_VAR_RE.findall(template_text))
return {name for name in referenced if name not in local_names}
def jinjify_artifact( def jinjify_artifact(
bundle_dir: str | Path, bundle_dir: str | Path,
artifact_role: str, artifact_role: str,
@ -130,14 +146,13 @@ def jinjify_artifact(
jt_enabled: bool, jt_enabled: bool,
overwrite_templates: bool = True, overwrite_templates: bool = True,
role_name: Optional[str] = None, role_name: Optional[str] = None,
template_engine: str = "jinja2",
puppet_class: Optional[str] = None,
) -> Optional[JinjifiedArtifact]: ) -> Optional[JinjifiedArtifact]:
"""Best-effort conversion of one harvested artifact into a Jinja2 template. """Best-effort conversion of one harvested artifact into a template.
Puppet does not use JinjaTurtle, but Salt and Ansible both have the same Ansible/Salt use Jinja2 output. Puppet uses ERB output with Puppet Hiera
philosophical operation: take ``artifacts/<role>/<src_rel>``, ask keys when a new enough JinjaTurtle is available.
JinjaTurtle for a template and variable mapping, and write that template
under the renderer's template directory. Keeping that here prevents Salt
and Ansible from reimplementing the same probing/format/error handling.
""" """
if not (jt_enabled and jt_exe and can_jinjify_path(dest_path)): if not (jt_enabled and jt_exe and can_jinjify_path(dest_path)):
return None return None
@ -147,22 +162,34 @@ def jinjify_artifact(
return None return None
try: try:
result = run_jinjaturtle( run_kwargs: Dict[str, Any] = {
jt_exe, "role_name": role_name or artifact_role,
str(artifact_path), "force_format": infer_other_formats(dest_path),
role_name=role_name or artifact_role, }
force_format=infer_other_formats(dest_path), # Keep the historical call shape for Ansible/Salt and for tests that
) # monkeypatch run_jinjaturtle with the old signature. Puppet/ERB is
# the only path that needs the newer JinjaTurtle CLI switches.
if template_engine != "jinja2":
run_kwargs["template_engine"] = template_engine
if puppet_class:
run_kwargs["puppet_class"] = puppet_class
result = run_jinjaturtle(jt_exe, str(artifact_path), **run_kwargs)
except Exception: except Exception:
return None # nosec - best-effort template generation return None # nosec - best-effort template generation
template_rel = Path(src_rel).as_posix() + ".j2" ext = "erb" if template_engine == "erb" else "j2"
template_rel = Path(src_rel).as_posix() + f".{ext}"
template_dst = Path(template_root) / template_rel template_dst = Path(template_root) / template_rel
context = yaml_load_mapping(result.vars_text) context = yaml_load_mapping(result.vars_text)
if missing_jinja_template_vars(result.template_text, context): missing = (
missing_erb_template_vars(result.template_text, context)
if template_engine == "erb"
else missing_jinja_template_vars(result.template_text, context)
)
if missing:
# If this role was generated into an existing output directory, avoid # If this role was generated into an existing output directory, avoid
# leaving an obsolete .j2 behind after falling back to a raw copy. # leaving an obsolete template behind after falling back to a raw copy.
if overwrite_templates and template_dst.exists(): if overwrite_templates and template_dst.exists():
template_dst.unlink() template_dst.unlink()
return None return None
@ -311,6 +338,8 @@ def run_jinjaturtle(
*, *,
role_name: str, role_name: str,
force_format: Optional[str] = None, force_format: Optional[str] = None,
template_engine: str = "jinja2",
puppet_class: Optional[str] = None,
) -> JinjifyResult: ) -> JinjifyResult:
""" """
Run jinjaturtle against src_path and return (template, defaults-yaml). Run jinjaturtle against src_path and return (template, defaults-yaml).
@ -318,6 +347,9 @@ def run_jinjaturtle(
jinjaturtle CLI: jinjaturtle CLI:
jinjaturtle <config> -r <role> [-f <format>] [-d <defaults-output>] [-t <template-output>] jinjaturtle <config> -r <role> [-f <format>] [-d <defaults-output>] [-t <template-output>]
Newer JinjaTurtle versions also support ``--template-engine erb`` and
``--puppet-class`` for Puppet/Hiera output.
""" """
src = Path(src_path) src = Path(src_path)
if not src.is_file(): if not src.is_file():
@ -326,7 +358,9 @@ def run_jinjaturtle(
with tempfile.TemporaryDirectory(prefix="enroll-jt-") as td: with tempfile.TemporaryDirectory(prefix="enroll-jt-") as td:
td_path = Path(td) td_path = Path(td)
defaults_out = td_path / "defaults.yml" defaults_out = td_path / "defaults.yml"
template_out = td_path / "template.j2" template_out = td_path / (
"template.erb" if template_engine == "erb" else "template.j2"
)
cmd = [ cmd = [
jt_exe, jt_exe,
@ -340,6 +374,10 @@ def run_jinjaturtle(
] ]
if force_format: if force_format:
cmd.extend(["-f", force_format]) cmd.extend(["-f", force_format])
if template_engine != "jinja2":
cmd.extend(["--template-engine", template_engine])
if puppet_class:
cmd.extend(["--puppet-class", puppet_class])
p = subprocess.run(cmd, text=True, capture_output=True) # nosec p = subprocess.run(cmd, text=True, capture_output=True) # nosec
if p.returncode != 0: if p.returncode != 0:

View file

@ -210,6 +210,7 @@ def manifest(
out, out,
fqdn=fqdn, fqdn=fqdn,
no_common_roles=no_common_roles, no_common_roles=no_common_roles,
jinjaturtle=jinjaturtle,
) )
elif target == "salt": elif target == "salt":
manifest_salt_from_bundle_dir( manifest_salt_from_bundle_dir(
@ -246,6 +247,7 @@ def manifest(
str(tmp_out), str(tmp_out),
fqdn=fqdn, fqdn=fqdn,
no_common_roles=no_common_roles, no_common_roles=no_common_roles,
jinjaturtle=jinjaturtle,
) )
elif target == "salt": elif target == "salt":
manifest_salt_from_bundle_dir( manifest_salt_from_bundle_dir(

View file

@ -17,6 +17,12 @@ from .cm import (
markdown_list, markdown_list,
) )
from .state import inventory_packages_from_state, roles_from_state from .state import inventory_packages_from_state, roles_from_state
from .jinjaturtle import (
can_jinjify_path,
jinjify_artifact,
managed_file_var_prefix,
resolve_jinjaturtle_mode,
)
class PuppetRole(CMModule): class PuppetRole(CMModule):
@ -31,6 +37,7 @@ class PuppetRole(CMModule):
self.flatpak_remotes: List[Dict[str, Any]] = [] self.flatpak_remotes: List[Dict[str, Any]] = []
self.flatpaks: List[Dict[str, Any]] = [] self.flatpaks: List[Dict[str, Any]] = []
self.snaps: List[Dict[str, Any]] = [] self.snaps: List[Dict[str, Any]] = []
self.template_hiera: Dict[str, Any] = {}
def has_resources(self) -> bool: def has_resources(self) -> bool:
return self.has_resources_or_attrs( return self.has_resources_or_attrs(
@ -133,8 +140,12 @@ class PuppetRole(CMModule):
bundle_dir: str, bundle_dir: str,
artifact_role: str, artifact_role: str,
module_files_dir: Path, module_files_dir: Path,
module_templates_dir: Optional[Path] = None,
file_prefix: Optional[str] = None, file_prefix: Optional[str] = None,
notify_services: Optional[List[str]] = None, notify_services: Optional[List[str]] = None,
jt_exe: Optional[str] = None,
jt_enabled: bool = False,
overwrite_templates: bool = True,
) -> None: ) -> None:
for d in self.managed_dirs_from_snapshot(snap): for d in self.managed_dirs_from_snapshot(snap):
path = str(d.get("path") or "").strip() path = str(d.get("path") or "").strip()
@ -146,11 +157,55 @@ class PuppetRole(CMModule):
reason=d.get("reason") or "managed_dir", reason=d.get("reason") or "managed_dir",
) )
for mf in self.managed_files_from_snapshot(snap): managed_files = list(self.managed_files_from_snapshot(snap))
candidates = [
mf
for mf in managed_files
if str(mf.get("path") or "")
and str(mf.get("src_rel") or "")
and can_jinjify_path(str(mf.get("path") or ""))
]
namespace_by_file = len(candidates) > 1
for mf in managed_files:
path = str(mf.get("path") or "").strip() path = str(mf.get("path") or "").strip()
src_rel = str(mf.get("src_rel") or "").strip() src_rel = str(mf.get("src_rel") or "").strip()
if not path or not src_rel: if not path or not src_rel:
continue continue
template_rel: Optional[str] = None
if module_templates_dir is not None:
role_prefix = (
managed_file_var_prefix(self.module_name, src_rel)
if namespace_by_file
else self.module_name
)
converted = jinjify_artifact(
bundle_dir,
artifact_role,
src_rel,
path,
module_templates_dir,
jt_exe=jt_exe,
jt_enabled=jt_enabled,
overwrite_templates=overwrite_templates,
role_name=role_prefix,
template_engine="erb",
puppet_class=self.module_name,
)
if converted is not None:
template_rel = converted.template_rel
self.template_hiera.update(converted.context)
attrs: Dict[str, Any] = {
"owner": mf.get("owner") or "root",
"group": mf.get("group") or "root",
"mode": mf.get("mode") or "0644",
"reason": mf.get("reason") or "managed_file",
}
if template_rel is not None:
attrs["template"] = f"{self.module_name}/{template_rel}"
else:
module_rel = _copy_artifact( module_rel = _copy_artifact(
bundle_dir, bundle_dir,
artifact_role, artifact_role,
@ -163,16 +218,14 @@ class PuppetRole(CMModule):
f"Skipped {path}: harvested artifact {artifact_role}/{src_rel} was not present." f"Skipped {path}: harvested artifact {artifact_role}/{src_rel} was not present."
) )
continue continue
attrs: Dict[str, Any] = { attrs["source"] = _source_uri(self.module_name, module_rel)
"owner": mf.get("owner") or "root",
"group": mf.get("group") or "root",
"mode": mf.get("mode") or "0644",
"source": _source_uri(self.module_name, module_rel),
"reason": mf.get("reason") or "managed_file",
}
if notify_services and not path.startswith("/etc/systemd/system/"): if notify_services and not path.startswith("/etc/systemd/system/"):
refs = [f"Service[{_pp_quote(unit)}]" for unit in notify_services] notify_units = [unit for unit in notify_services if str(unit).strip()]
attrs["notify"] = refs[0] if len(refs) == 1 else f"[{', '.join(refs)}]" notify_value = _service_notify_value(notify_units)
if notify_value:
attrs["notify"] = notify_value
attrs["notify_services"] = notify_units
attrs["_notify_services"] = notify_units
self.add_managed_file(path, **attrs) self.add_managed_file(path, **attrs)
for ml in self.managed_links_from_snapshot(snap): for ml in self.managed_links_from_snapshot(snap):
@ -380,6 +433,43 @@ def _pp_array(values: Iterable[Any]) -> str:
return "[" + ", ".join(_pp_quote(v) for v in values) + "]" return "[" + ", ".join(_pp_quote(v) for v in values) + "]"
def _pp_value(value: Any) -> str:
"""Render a conservative Puppet literal for generated class defaults."""
if value is None:
return "undef"
if isinstance(value, bool):
return _pp_bool(value)
if isinstance(value, int) and not isinstance(value, bool):
return str(value)
if isinstance(value, float):
return repr(value)
if isinstance(value, list):
return "[" + ", ".join(_pp_value(v) for v in value) + "]"
if isinstance(value, dict):
parts = []
for key in sorted(value, key=lambda k: str(k)):
parts.append(f"{_pp_quote(key)} => {_pp_value(value[key])}")
return "{" + ", ".join(parts) + "}"
return _pp_quote(value)
def _template_param_defaults(prole: PuppetRole) -> Dict[str, Any]:
prefix = f"{prole.module_name}::"
out: Dict[str, Any] = {}
for key, value in prole.template_hiera.items():
key_s = str(key)
if key_s.startswith(prefix):
local = key_s[len(prefix) :]
elif "::" in key_s:
local = key_s.split("::", 1)[1]
else:
local = key_s
if local:
out[local] = value
return out
def _puppet_exec_attrs( def _puppet_exec_attrs(
command: str, command: str,
unless: str, unless: str,
@ -469,6 +559,65 @@ def _render_firewall_runtime_execs(
lines.append("") lines.append("")
def _active_service_snapshots_by_unit(
entries: Iterable[Dict[str, Any]],
) -> Dict[str, Dict[str, Any]]:
"""Return active service snapshots keyed by systemd unit name."""
by_unit: Dict[str, Dict[str, Any]] = {}
for entry in entries:
if str(entry.get("kind") or "package") != "service":
continue
snap = entry.get("snapshot") or {}
if not isinstance(snap, dict):
continue
unit = str(snap.get("unit") or "").strip()
if not unit or str(snap.get("active_state") or "") != "active":
continue
by_unit.setdefault(unit, snap)
return by_unit
def _service_notify_value(units: Iterable[str]) -> Optional[str]:
refs = [f"Service[{_pp_quote(unit)}]" for unit in units if str(unit).strip()]
if not refs:
return None
return refs[0] if len(refs) == 1 else f"[{', '.join(refs)}]"
def _sync_service_notifications(puppet_roles: Iterable[PuppetRole]) -> None:
"""Remove generated service notifications that do not target this catalog."""
roles = list(puppet_roles)
declared_services = {unit for role in roles for unit in role.services}
for role in roles:
for path, attrs in role.files.items():
notify_units = [
str(unit).strip()
for unit in (attrs.get("_notify_services") or [])
if str(unit).strip()
]
if not notify_units:
attrs.pop("_notify_services", None)
continue
kept = [unit for unit in notify_units if unit in declared_services]
missing = sorted(set(notify_units) - set(kept))
if missing:
role.notes.append(
"Skipped service notification for "
f"{path}: no generated Service resource for "
f"{', '.join(missing)}."
)
notify_value = _service_notify_value(kept)
if notify_value:
attrs["notify"] = notify_value
attrs["notify_services"] = kept
else:
attrs.pop("notify", None)
attrs.pop("notify_services", None)
attrs.pop("_notify_services", None)
def _copy_artifact( def _copy_artifact(
bundle_dir: str, bundle_dir: str,
role: str, role: str,
@ -515,6 +664,8 @@ def _collect_puppet_roles(
*, *,
fqdn: Optional[str] = None, fqdn: Optional[str] = None,
no_common_roles: bool = False, no_common_roles: bool = False,
jt_exe: Optional[str] = None,
jt_enabled: bool = False,
) -> List[PuppetRole]: ) -> List[PuppetRole]:
roles = roles_from_state(state) roles = roles_from_state(state)
inventory_packages = inventory_packages_from_state(state) inventory_packages = inventory_packages_from_state(state)
@ -541,13 +692,18 @@ def _collect_puppet_roles(
str(snap.get("role_name") or key), fallback="enroll_role" str(snap.get("role_name") or key), fallback="enroll_role"
) )
prole = ensure_role(role_name) prole = ensure_role(role_name)
module_files_dir = modules_dir / prole.module_name / "files" module_dir = modules_dir / prole.module_name
module_files_dir = module_dir / "files"
prole.add_managed_content( prole.add_managed_content(
snap, snap,
bundle_dir=bundle_dir, bundle_dir=bundle_dir,
artifact_role=str(snap.get("role_name") or key), artifact_role=str(snap.get("role_name") or key),
module_files_dir=module_files_dir, module_files_dir=module_files_dir,
module_templates_dir=module_dir / "templates",
file_prefix=node_file_prefix, file_prefix=node_file_prefix,
jt_exe=jt_exe,
jt_enabled=jt_enabled,
overwrite_templates=not bool(fqdn),
) )
users_snap = roles.get("users") or {} users_snap = roles.get("users") or {}
@ -557,12 +713,17 @@ def _collect_puppet_roles(
) )
prole = ensure_role(role_name) prole = ensure_role(role_name)
prole.add_users_snapshot(users_snap) prole.add_users_snapshot(users_snap)
module_dir = modules_dir / prole.module_name
prole.add_managed_content( prole.add_managed_content(
users_snap, users_snap,
bundle_dir=bundle_dir, bundle_dir=bundle_dir,
artifact_role=str(users_snap.get("role_name") or "users"), artifact_role=str(users_snap.get("role_name") or "users"),
module_files_dir=modules_dir / prole.module_name / "files", module_files_dir=module_dir / "files",
module_templates_dir=module_dir / "templates",
file_prefix=node_file_prefix, file_prefix=node_file_prefix,
jt_exe=jt_exe,
jt_enabled=jt_enabled,
overwrite_templates=not bool(fqdn),
) )
package_service_entries = list( package_service_entries = list(
@ -573,6 +734,9 @@ def _collect_puppet_roles(
service_units_by_package = CMModule.active_service_units_by_package( service_units_by_package = CMModule.active_service_units_by_package(
package_service_entries package_service_entries
) )
service_snapshots_by_unit = _active_service_snapshots_by_unit(
package_service_entries
)
for entry in package_service_entries: for entry in package_service_entries:
snap = entry.get("snapshot") or {} snap = entry.get("snapshot") or {}
@ -598,13 +762,22 @@ def _collect_puppet_roles(
notify_services = CMModule.active_service_units_for_package_snapshot( notify_services = CMModule.active_service_units_for_package_snapshot(
snap, service_units_by_package snap, service_units_by_package
) )
for unit in notify_services:
service_snap = service_snapshots_by_unit.get(unit)
if service_snap is not None:
prole.add_service_snapshot(service_snap)
module_dir = modules_dir / prole.module_name
prole.add_managed_content( prole.add_managed_content(
snap, snap,
bundle_dir=bundle_dir, bundle_dir=bundle_dir,
artifact_role=str(snap.get("role_name") or original_role_name), artifact_role=str(snap.get("role_name") or original_role_name),
module_files_dir=modules_dir / prole.module_name / "files", module_files_dir=module_dir / "files",
module_templates_dir=module_dir / "templates",
file_prefix=node_file_prefix, file_prefix=node_file_prefix,
notify_services=notify_services, notify_services=notify_services,
jt_exe=jt_exe,
jt_enabled=jt_enabled,
overwrite_templates=not bool(fqdn),
) )
container_images = roles.get("container_images") or {} container_images = roles.get("container_images") or {}
@ -656,17 +829,29 @@ def _collect_puppet_roles(
puppet_roles = sorted(out.values(), key=lambda r: role_order_key(r.role_name)) puppet_roles = sorted(out.values(), key=lambda r: role_order_key(r.role_name))
resolve_catalog_conflicts(puppet_roles) resolve_catalog_conflicts(puppet_roles)
_sync_service_notifications(puppet_roles)
return [r for r in puppet_roles if r.has_resources()] return [r for r in puppet_roles if r.has_resources()]
def _render_role_class(prole: PuppetRole) -> str: def _render_role_class(prole: PuppetRole) -> str:
has_sysctl_conf = "/etc/sysctl.d/99-enroll.conf" in prole.files has_sysctl_conf = "/etc/sysctl.d/99-enroll.conf" in prole.files
template_defaults = _template_param_defaults(prole)
params: List[str] = []
if has_sysctl_conf: if has_sysctl_conf:
params.extend(
[
" Boolean $sysctl_apply = true,",
" Boolean $sysctl_ignore_apply_errors = true,",
]
)
for name, value in sorted(template_defaults.items()):
params.append(f" Any ${name} = {_pp_value(value)},")
if params:
lines: List[str] = [ lines: List[str] = [
"# Generated by Enroll from harvest state.", "# Generated by Enroll from harvest state.",
f"class {prole.module_name} (", f"class {prole.module_name} (",
" Boolean $sysctl_apply = true,", *params,
" Boolean $sysctl_ignore_apply_errors = true,",
") {", ") {",
"", "",
] ]
@ -718,19 +903,20 @@ def _render_role_class(prole: PuppetRole) -> str:
) )
for path, f in sorted(prole.files.items()): for path, f in sorted(prole.files.items()):
_resource( file_attrs: List[Tuple[str, str]] = [("ensure", _pp_quote("file"))]
lines, if f.get("template"):
"file", file_attrs.append(("content", f"template({_pp_quote(f.get('template'))})"))
path, else:
file_attrs.append(("source", _pp_quote(f.get("source") or "")))
file_attrs.extend(
[ [
("ensure", _pp_quote("file")),
("source", _pp_quote(f.get("source") or "")),
("owner", _pp_quote(f.get("owner") or "root")), ("owner", _pp_quote(f.get("owner") or "root")),
("group", _pp_quote(f.get("group") or "root")), ("group", _pp_quote(f.get("group") or "root")),
("mode", _pp_quote(f.get("mode") or "0644")), ("mode", _pp_quote(f.get("mode") or "0644")),
*([("notify", str(f.get("notify")))] if f.get("notify") else []), *([("notify", str(f.get("notify")))] if f.get("notify") else []),
], ]
) )
_resource(lines, "file", path, file_attrs)
for path, lnk in sorted(prole.links.items()): for path, lnk in sorted(prole.links.items()):
_resource( _resource(
@ -1031,7 +1217,14 @@ def _role_hiera_values(prole: PuppetRole) -> Dict[str, Any]:
path: _attrs_with_ensure( path: _attrs_with_ensure(
prole.files[path], prole.files[path],
"file", "file",
allowed={"source", "owner", "group", "mode", "notify"}, allowed={
"source",
"template",
"owner",
"group",
"mode",
"notify_services",
},
) )
for path in sorted(prole.files) for path in sorted(prole.files)
} }
@ -1069,6 +1262,8 @@ def _role_hiera_values(prole: PuppetRole) -> Dict[str, Any]:
if prole.notes: if prole.notes:
data[f"{prefix}notes"] = list(prole.notes) data[f"{prefix}notes"] = list(prole.notes)
data.update(prole.template_hiera)
if "/etc/sysctl.d/99-enroll.conf" in prole.files: if "/etc/sysctl.d/99-enroll.conf" in prole.files:
data[f"{prefix}sysctl_apply"] = True data[f"{prefix}sysctl_apply"] = True
data[f"{prefix}sysctl_ignore_apply_errors"] = True data[f"{prefix}sysctl_ignore_apply_errors"] = True
@ -1098,6 +1293,10 @@ def _render_hiera_role_class(prole: PuppetRole) -> str:
" Array[String] $notes = [],", " Array[String] $notes = [],",
" Boolean $sysctl_apply = true,", " Boolean $sysctl_apply = true,",
" Boolean $sysctl_ignore_apply_errors = true,", " Boolean $sysctl_ignore_apply_errors = true,",
*[
f" Any ${name} = undef,"
for name in sorted(_template_param_defaults(prole))
],
") {", ") {",
"", "",
" $packages.each |String $package_name| {", " $packages.each |String $package_name| {",
@ -1124,24 +1323,50 @@ def _render_hiera_role_class(prole: PuppetRole) -> str:
" }", " }",
" }", " }",
"", "",
" $files.each |String $resource_title, Hash $attrs| {", " # Declare services before files so file notify relationships can",
" file { $resource_title:", " # resolve in Hiera-driven classes.",
" $services.each |String $resource_title, Hash $attrs| {",
" service { $resource_title:",
" * => $attrs,", " * => $attrs,",
" }", " }",
" }", " }",
"", "",
" $files.each |String $resource_title, Hash $attrs| {",
" $file_attrs = $attrs.filter |$key, $value| {",
" $key != 'template' and $key != 'notify_services'",
" }",
" if $attrs['notify_services'] {",
" $notify_targets = $attrs['notify_services'].map |String $unit| { Service[$unit] }",
" if $attrs['template'] {",
" file { $resource_title:",
" * => $file_attrs,",
" content => template($attrs['template']),",
" notify => $notify_targets,",
" }",
" } else {",
" file { $resource_title:",
" * => $file_attrs,",
" notify => $notify_targets,",
" }",
" }",
" } elsif $attrs['template'] {",
" file { $resource_title:",
" * => $file_attrs,",
" content => template($attrs['template']),",
" }",
" } else {",
" file { $resource_title:",
" * => $file_attrs,",
" }",
" }",
" }",
"",
" $links.each |String $resource_title, Hash $attrs| {", " $links.each |String $resource_title, Hash $attrs| {",
" file { $resource_title:", " file { $resource_title:",
" * => $attrs,", " * => $attrs,",
" }", " }",
" }", " }",
"", "",
" $services.each |String $resource_title, Hash $attrs| {",
" service { $resource_title:",
" * => $attrs,",
" }",
" }",
"",
" $flatpak_remotes.each |Integer $idx, Hash $remote| {", " $flatpak_remotes.each |Integer $idx, Hash $remote| {",
" exec { $remote['state_id']:", " exec { $remote['state_id']:",
" command => $remote['add_cmd'],", " command => $remote['add_cmd'],",
@ -1385,7 +1610,8 @@ def _render_readme(
- `hiera.yaml` configures per-node lookup from `data/nodes/%{{trusted.certname}}.yaml` with a fallback to `data/common.yaml`. - `hiera.yaml` configures per-node lookup from `data/nodes/%{{trusted.certname}}.yaml` with a fallback to `data/common.yaml`.
- `data/nodes/{_node_data_filename(fqdn or '')}` contains this node's class list and class parameter data. - `data/nodes/{_node_data_filename(fqdn or '')}` contains this node's class list and class parameter data.
- `modules/<role>/manifests/init.pp` contains reusable, data-driven classes. - `modules/<role>/manifests/init.pp` contains reusable, data-driven classes.
- `modules/<role>/files/nodes/<fqdn>/...` contains node-specific harvested file artifacts, avoiding clashes between hosts.""" - `modules/<role>/files/nodes/<fqdn>/...` contains node-specific harvested raw file artifacts, avoiding clashes between hosts.
- `modules/<role>/templates/` contains ERB templates when JinjaTurtle can convert a harvested config file."""
apply = f"""Run from this generated output directory, passing the node certname so Hiera selects the right node data: apply = f"""Run from this generated output directory, passing the node certname so Hiera selects the right node data:
```bash ```bash
@ -1395,14 +1621,15 @@ sudo puppet apply --modulepath ./modules --hiera_config ./hiera.yaml --certname
If you depend on other pre-installed Puppet modules, you may need to pass in other modulepaths as well, e.g: If you depend on other pre-installed Puppet modules, you may need to pass in other modulepaths as well, e.g:
```bash ```bash
sudo puppet apply --modulepath ./modules:/etc/puppet/code/modules --hiera_config ./hiera.yaml --certname {fqdn} manifests/site.pp --noop sudo puppet apply --modulepath ./modules:/etc/puppet/code/modules --hiera_config ./hiera.yaml --certname {fqdn} manifests/site.pp --noop --test
``` ```
For Puppet agent/control-repo use, place this output where `hiera.yaml`, `data/`, `manifests/`, and `modules/` form the environment root. Re-running Enroll with another `--fqdn` into the same output directory adds or replaces that node's YAML without deleting existing node data.""" For Puppet agent/control-repo use, place this output where `hiera.yaml`, `data/`, `manifests/`, and `modules/` form the environment root. Re-running Enroll with another `--fqdn` into the same output directory adds or replaces that node's YAML without deleting existing node data."""
else: else:
layout = """- `manifests/site.pp` declares a `node` block and includes the generated classes in manifest order. layout = """- `manifests/site.pp` declares a `node` block and includes the generated classes in manifest order.
- `modules/<role>/manifests/init.pp` contains resources for each generated Enroll role/snapshot or common package group. - `modules/<role>/manifests/init.pp` contains resources for each generated Enroll role/snapshot or common package group.
- `modules/<role>/files/` contains harvested file artifacts for that role or group. - `modules/<role>/files/` contains harvested raw file artifacts for that role or group.
- `modules/<role>/templates/` contains ERB templates when JinjaTurtle can convert a harvested config file.
- Generated module names avoid Puppet reserved words such as `default`.""" - Generated module names avoid Puppet reserved words such as `default`."""
apply = """Run from this generated output directory so Puppet can find `./modules`, or pass an absolute module path: apply = """Run from this generated output directory so Puppet can find `./modules`, or pass an absolute module path:
@ -1413,7 +1640,7 @@ sudo puppet apply --modulepath ./modules manifests/site.pp --noop --test
If you depend on other pre-installed Puppet modules, you may need to pass in other modulepaths as well, e.g: If you depend on other pre-installed Puppet modules, you may need to pass in other modulepaths as well, e.g:
```bash ```bash
sudo puppet apply --modulepath ./modules:/etc/puppet/code/modules manifests/site.pp --noop sudo puppet apply --modulepath ./modules:/etc/puppet/code/modules manifests/site.pp --noop --test
```""" ```"""
return f"""# Enroll Puppet manifest return f"""# Enroll Puppet manifest
@ -1449,7 +1676,7 @@ This Puppet target reuses the existing harvest state without changing harvesting
## Current limitations ## Current limitations
- JinjaTurtle templating is currently Ansible/Salt-oriented and is not applied to Puppet output - there are no erb templates, just raw files. - JinjaTurtle/ERB templating is best-effort. Files that JinjaTurtle cannot parse are copied as raw module files.
- Review generated resources before applying them broadly across unlike hosts. - Review generated resources before applying them broadly across unlike hosts.
## Notes ## Notes
@ -1468,11 +1695,13 @@ class PuppetManifestRenderer:
*, *,
fqdn: Optional[str] = None, fqdn: Optional[str] = None,
no_common_roles: bool = False, no_common_roles: bool = False,
jinjaturtle: str = "auto",
) -> None: ) -> None:
self.bundle_dir = bundle_dir self.bundle_dir = bundle_dir
self.out_dir = out_dir self.out_dir = out_dir
self.fqdn = fqdn self.fqdn = fqdn
self.no_common_roles = no_common_roles self.no_common_roles = no_common_roles
self.jinjaturtle = jinjaturtle
def render(self) -> None: def render(self) -> None:
"""Render Puppet modules/site.pp from a harvest bundle.""" """Render Puppet modules/site.pp from a harvest bundle."""
@ -1492,12 +1721,16 @@ class PuppetManifestRenderer:
manifests_dir.mkdir(parents=True, exist_ok=True) manifests_dir.mkdir(parents=True, exist_ok=True)
modules_dir.mkdir(parents=True, exist_ok=True) modules_dir.mkdir(parents=True, exist_ok=True)
jt_exe, jt_enabled = resolve_jinjaturtle_mode(self.jinjaturtle)
puppet_roles = _collect_puppet_roles( puppet_roles = _collect_puppet_roles(
state, state,
bundle_dir, bundle_dir,
modules_dir, modules_dir,
fqdn=fqdn, fqdn=fqdn,
no_common_roles=no_common_roles, no_common_roles=no_common_roles,
jt_exe=jt_exe,
jt_enabled=jt_enabled,
) )
for prole in puppet_roles: for prole in puppet_roles:
module_dir = modules_dir / prole.module_name module_dir = modules_dir / prole.module_name
@ -1544,10 +1777,12 @@ def manifest_from_bundle_dir(
*, *,
fqdn: Optional[str] = None, fqdn: Optional[str] = None,
no_common_roles: bool = False, no_common_roles: bool = False,
jinjaturtle: str = "auto",
) -> None: ) -> None:
PuppetManifestRenderer( PuppetManifestRenderer(
bundle_dir, bundle_dir,
out_dir, out_dir,
fqdn=fqdn, fqdn=fqdn,
no_common_roles=no_common_roles, no_common_roles=no_common_roles,
jinjaturtle=jinjaturtle,
).render() ).render()

View file

@ -6,7 +6,7 @@ import re
import shlex import shlex
import shutil import shutil
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple
import yaml import yaml
@ -150,6 +150,7 @@ class SaltRole(CMModule):
jt_enabled: bool = False, jt_enabled: bool = False,
overwrite_templates: bool = True, overwrite_templates: bool = True,
watch_services: Optional[List[str]] = None, watch_services: Optional[List[str]] = None,
watch_service_states: Optional[List[str]] = None,
) -> None: ) -> None:
for d in self.managed_dirs_from_snapshot(snap): for d in self.managed_dirs_from_snapshot(snap):
path = str(d.get("path") or "").strip() path = str(d.get("path") or "").strip()
@ -162,6 +163,12 @@ class SaltRole(CMModule):
reason=d.get("reason") or "managed_dir", reason=d.get("reason") or "managed_dir",
) )
watch_state_ids = _service_watch_state_ids(
self.module_name,
watch_services=watch_services,
watch_service_states=watch_service_states,
)
for mf in self.managed_files_from_snapshot(snap): for mf in self.managed_files_from_snapshot(snap):
path = str(mf.get("path") or "").strip() path = str(mf.get("path") or "").strip()
src_rel = str(mf.get("src_rel") or "").strip() src_rel = str(mf.get("src_rel") or "").strip()
@ -190,10 +197,9 @@ class SaltRole(CMModule):
"makedirs": True, "makedirs": True,
"reason": mf.get("reason") or "managed_file", "reason": mf.get("reason") or "managed_file",
} }
if watch_services and not path.startswith("/etc/systemd/system/"): if watch_state_ids and not path.startswith("/etc/systemd/system/"):
attrs["watch_in"] = [ attrs["watch_in"] = [
{"service": _state_id("service", unit, role=self.module_name)} {"service": state_id} for state_id in watch_state_ids
for unit in watch_services
] ]
self.add_managed_file(path, **attrs) self.add_managed_file(path, **attrs)
continue continue
@ -218,10 +224,9 @@ class SaltRole(CMModule):
"makedirs": True, "makedirs": True,
"reason": mf.get("reason") or "managed_file", "reason": mf.get("reason") or "managed_file",
} }
if watch_services and not path.startswith("/etc/systemd/system/"): if watch_state_ids and not path.startswith("/etc/systemd/system/"):
attrs["watch_in"] = [ attrs["watch_in"] = [
{"service": _state_id("service", unit, role=self.module_name)} {"service": state_id} for state_id in watch_state_ids
for unit in watch_services
] ]
self.add_managed_file(path, **attrs) self.add_managed_file(path, **attrs)
@ -271,6 +276,105 @@ def _state_id(prefix: str, value: Any, *, role: str = "") -> str:
return "_".join(parts) return "_".join(parts)
def _plain_salt_data(value: Any) -> Any:
"""Return data made from plain JSON/YAML-safe containers.
Salt's Jinja ``yaml_encode`` filter cannot represent Salt/PyYAML
``OrderedDict`` values. Normalise generated template contexts before we
write static SLS or pillar data, and before passing context to file.managed.
"""
if isinstance(value, Mapping):
return {str(key): _plain_salt_data(inner) for key, inner in value.items()}
if isinstance(value, list):
return [_plain_salt_data(item) for item in value]
if isinstance(value, tuple):
return [_plain_salt_data(item) for item in value]
if isinstance(value, set):
return sorted(_plain_salt_data(item) for item in value)
return value
_TO_JSON_FILTER_RE = re.compile(
r"{{\s*([A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)?)\s*"
r"\|\s*to_json\s*\([^)]*\)\s*}}"
)
def _saltify_jinjaturtle_template(
template_text: str, context: Dict[str, Any]
) -> Tuple[str, Dict[str, Any]]:
"""Translate JinjaTurtle's Ansible-oriented Jinja into Salt-safe Jinja.
JinjaTurtle emits Ansible's ``to_json`` filter for JSON/TOML values. Salt's
Jinja environment does not provide that filter. For ordinary generated
context variables, pre-render a JSON string and substitute a plain variable
reference. For loop-local expressions such as ``item`` or ``item.name`` we
fall back to Jinja's built-in ``tojson`` filter.
"""
salt_context = _plain_salt_data(context)
def replace(match: re.Match[str]) -> str:
expr = match.group(1)
if "." not in expr and expr in salt_context:
json_var = f"{expr}__enroll_json"
salt_context[json_var] = json.dumps(salt_context[expr], ensure_ascii=False)
return "{{ " + json_var + " }}"
return "{{ " + expr + " | tojson }}"
return _TO_JSON_FILTER_RE.sub(replace, template_text), salt_context
def _service_watch_state_ids(
role_name: str,
*,
watch_services: Optional[Iterable[str]] = None,
watch_service_states: Optional[Iterable[str]] = None,
) -> List[str]:
"""Return de-duplicated Salt service state ids for watch_in requisites."""
out: List[str] = []
seen = set()
for state_id in watch_service_states or []:
value = str(state_id or "").strip()
if value and value not in seen:
seen.add(value)
out.append(value)
for unit in watch_services or []:
unit_s = str(unit or "").strip()
if not unit_s:
continue
value = _state_id("service", unit_s, role=role_name)
if value not in seen:
seen.add(value)
out.append(value)
return out
def _active_service_state_ids_by_unit(
entries: Iterable[Dict[str, Any]],
) -> Dict[str, str]:
"""Return generated Salt service state ids keyed by active systemd unit."""
by_unit: Dict[str, str] = {}
for entry in entries:
if str(entry.get("kind") or "package") != "service":
continue
snap = entry.get("snapshot") or {}
if not isinstance(snap, dict):
continue
unit = str(snap.get("unit") or "").strip()
if not unit or str(snap.get("active_state") or "") != "active":
continue
source_label = str(snap.get("role_name") or snap.get("unit") or "service")
role_name = _salt_name(
str(entry.get("role_label") or source_label), fallback="service"
)
by_unit.setdefault(unit, _state_id("service", unit, role=role_name))
return by_unit
def _yaml_quote(value: Any) -> str: def _yaml_quote(value: Any) -> str:
return json.dumps(str(value), ensure_ascii=False) return json.dumps(str(value), ensure_ascii=False)
@ -547,7 +651,24 @@ def _jinjify_managed_file(
) )
if converted is None: if converted is None:
return None return None
return converted.template_rel, converted.context
template_text, context = _saltify_jinjaturtle_template(
converted.template_text, converted.context
)
template_path = role_dir / "templates" / converted.template_rel
if template_text != converted.template_text:
existing = (
template_path.read_text(encoding="utf-8") if template_path.exists() else ""
)
if (
overwrite_templates
or not template_path.exists()
or _TO_JSON_FILTER_RE.search(existing)
):
template_path.parent.mkdir(parents=True, exist_ok=True)
template_path.write_text(template_text, encoding="utf-8")
return converted.template_rel, context
def _node_file_prefix(fqdn: str) -> str: def _node_file_prefix(fqdn: str) -> str:
@ -639,6 +760,9 @@ def _collect_salt_roles(
service_units_by_package = CMModule.active_service_units_by_package( service_units_by_package = CMModule.active_service_units_by_package(
package_service_entries package_service_entries
) )
service_state_ids_by_unit = _active_service_state_ids_by_unit(
package_service_entries
)
for entry in package_service_entries: for entry in package_service_entries:
snap = entry.get("snapshot") or {} snap = entry.get("snapshot") or {}
@ -654,6 +778,7 @@ def _collect_salt_roles(
) )
srole = ensure_role(role_name) srole = ensure_role(role_name)
watch_services: List[str] = [] watch_services: List[str] = []
watch_service_states: List[str] = []
if kind == "service": if kind == "service":
srole.add_service_snapshot(snap) srole.add_service_snapshot(snap)
unit = str(snap.get("unit") or "").strip() unit = str(snap.get("unit") or "").strip()
@ -664,6 +789,13 @@ def _collect_salt_roles(
watch_services = CMModule.active_service_units_for_package_snapshot( watch_services = CMModule.active_service_units_for_package_snapshot(
snap, service_units_by_package snap, service_units_by_package
) )
watch_service_states = [
service_state_ids_by_unit[unit]
for unit in watch_services
if unit in service_state_ids_by_unit
]
if watch_service_states:
watch_services = []
srole.add_managed_content( srole.add_managed_content(
snap, snap,
bundle_dir=bundle_dir, bundle_dir=bundle_dir,
@ -674,6 +806,7 @@ def _collect_salt_roles(
jt_enabled=jt_enabled, jt_enabled=jt_enabled,
overwrite_templates=not bool(fqdn), overwrite_templates=not bool(fqdn),
watch_services=watch_services, watch_services=watch_services,
watch_service_states=watch_service_states,
) )
container_images = roles.get("container_images") or {} container_images = roles.get("container_images") or {}
@ -732,7 +865,7 @@ def _append_yaml_value(lines: List[str], key: str, value: Any, *, indent: int) -
prefix = " " * indent prefix = " " * indent
if isinstance(value, dict): if isinstance(value, dict):
dumped = yaml.safe_dump( dumped = yaml.safe_dump(
value, sort_keys=True, default_flow_style=False _plain_salt_data(value), sort_keys=True, default_flow_style=False
).rstrip() ).rstrip()
if not dumped: if not dumped:
lines.append(f"{prefix}- {key}: {{}}") lines.append(f"{prefix}- {key}: {{}}")
@ -1090,7 +1223,11 @@ def _role_pillar_values(srole: SaltRole) -> Dict[str, Any]:
**( **(
{"template": attrs.get("template")} if attrs.get("template") else {} {"template": attrs.get("template")} if attrs.get("template") else {}
), ),
**({"context": attrs.get("context")} if attrs.get("context") else {}), **(
{"context": _plain_salt_data(attrs.get("context"))}
if attrs.get("context")
else {}
),
**( **(
{"watch_in": attrs.get("watch_in")} if attrs.get("watch_in") else {} {"watch_in": attrs.get("watch_in")} if attrs.get("watch_in") else {}
), ),
@ -1211,7 +1348,7 @@ def _render_pillar_role(srole: SaltRole) -> str:
" - template: {{ attrs.get('template')|yaml_dquote }}", " - template: {{ attrs.get('template')|yaml_dquote }}",
"{% endif %}", "{% endif %}",
"{% if attrs.get('context') %}", "{% if attrs.get('context') %}",
" - context: {{ attrs.get('context')|yaml_encode }}", " - context: {{ attrs.get('context')|tojson }}",
"{% endif %}", "{% endif %}",
"{% if attrs.get('watch_in') %}", "{% if attrs.get('watch_in') %}",
" - watch_in:", " - watch_in:",

View file

@ -184,9 +184,9 @@ def test_manifest_puppet_writes_control_repo_style_output(tmp_path: Path):
assert node_data["foo::files"]["/etc/foo/foo.conf"]["source"] == ( assert node_data["foo::files"]["/etc/foo/foo.conf"]["source"] == (
"puppet:///modules/foo/nodes/test.example/etc/foo.conf" "puppet:///modules/foo/nodes/test.example/etc/foo.conf"
) )
assert node_data["foo::files"]["/etc/foo/foo.conf"]["notify"] == ( assert node_data["foo::files"]["/etc/foo/foo.conf"]["notify_services"] == [
"Service['foo.service']" "foo.service"
) ]
assert node_data["foo::services"]["foo.service"] == { assert node_data["foo::services"]["foo.service"] == {
"ensure": "running", "ensure": "running",
"enable": True, "enable": True,
@ -254,6 +254,111 @@ def test_manifest_puppet_writes_control_repo_style_output(tmp_path: Path):
).exists() ).exists()
def test_manifest_puppet_fqdn_package_notify_service_declared_in_same_role(
tmp_path: Path,
):
bundle = tmp_path / "bundle"
out = tmp_path / "puppet"
artifact = bundle / "artifacts" / "apparmor" / "etc" / "apparmor" / "parser.conf"
artifact.parent.mkdir(parents=True, exist_ok=True)
artifact.write_text("cache-loc /var/cache/apparmor\n", encoding="utf-8")
state = {
"schema_version": 3,
"host": {"hostname": "vpn-ssh", "os": "debian", "pkg_backend": "dpkg"},
"inventory": {"packages": {"apparmor": {"section": "admin"}}},
"roles": {
"services": [
{
"unit": "apparmor.service",
"role_name": "apparmor_service",
"packages": ["apparmor"],
"active_state": "active",
"unit_file_state": "enabled",
"managed_dirs": [],
"managed_files": [],
"managed_links": [],
}
],
"packages": [
{
"package": "apparmor",
"role_name": "apparmor",
"section": "admin",
"managed_dirs": [],
"managed_files": [
{
"path": "/etc/apparmor/parser.conf",
"src_rel": "etc/apparmor/parser.conf",
"owner": "root",
"group": "root",
"mode": "0644",
}
],
"managed_links": [],
}
],
"users": {
"role_name": "users",
"users": [],
"managed_dirs": [],
"managed_files": [],
},
"apt_config": {
"role_name": "apt_config",
"managed_dirs": [],
"managed_files": [],
},
"dnf_config": {
"role_name": "dnf_config",
"managed_dirs": [],
"managed_files": [],
},
"sysctl": {"role_name": "sysctl", "managed_dirs": [], "managed_files": []},
"firewall_runtime": {"role_name": "firewall_runtime", "packages": []},
"etc_custom": {
"role_name": "etc_custom",
"managed_dirs": [],
"managed_files": [],
},
"usr_local_custom": {
"role_name": "usr_local_custom",
"managed_dirs": [],
"managed_files": [],
},
"extra_paths": {
"role_name": "extra_paths",
"managed_dirs": [],
"managed_files": [],
"managed_links": [],
},
},
}
_write_state(bundle, state)
manifest.manifest(str(bundle), str(out), target="puppet", fqdn="vpn-ssh")
node_data = yaml.safe_load(
(out / "data" / "nodes" / "vpn-ssh.yaml").read_text(encoding="utf-8")
)
assert node_data["apparmor::files"]["/etc/apparmor/parser.conf"][
"notify_services"
] == ["apparmor.service"]
assert node_data["apparmor::services"]["apparmor.service"] == {
"ensure": "running",
"enable": True,
}
apparmor_pp = (out / "modules" / "apparmor" / "manifests" / "init.pp").read_text(
encoding="utf-8"
)
assert "Hash[String, Hash] $services = {}" in apparmor_pp
assert "service { $resource_title:" in apparmor_pp
assert apparmor_pp.index("$services.each") < apparmor_pp.index("$files.each")
assert "$attrs['notify_services'].map" in apparmor_pp
assert "notify => $notify_targets" in apparmor_pp
def test_manifest_puppet_fqdn_mode_can_accumulate_separate_node_data( def test_manifest_puppet_fqdn_mode_can_accumulate_separate_node_data(
tmp_path: Path, tmp_path: Path,
): ):
@ -999,3 +1104,224 @@ def test_puppet_names_are_sanitised_for_target_reserved_words() -> None:
assert _puppet_name("123") == "role_123" assert _puppet_name("123") == "role_123"
assert _puppet_name("node") == "role_node" assert _puppet_name("node") == "role_node"
assert _puppet_name("web-app") == "web_app" assert _puppet_name("web-app") == "web_app"
def test_manifest_puppet_uses_jinjaturtle_erb_templates(monkeypatch, tmp_path: Path):
import enroll.jinjaturtle as jinjaturtle_mod
from enroll.jinjaturtle import JinjifyResult
bundle = tmp_path / "bundle"
out = tmp_path / "puppet"
artifact = bundle / "artifacts" / "foo" / "etc" / "foo.ini"
artifact.parent.mkdir(parents=True, exist_ok=True)
artifact.write_text("[main]\nkey = 1\n", encoding="utf-8")
state = {
"schema_version": 3,
"host": {"hostname": "test", "os": "debian", "pkg_backend": "dpkg"},
"inventory": {"packages": {}},
"roles": {
"services": [
{
"unit": "foo.service",
"role_name": "foo",
"packages": ["foo"],
"active_state": "active",
"unit_file_state": "enabled",
"managed_dirs": [],
"managed_files": [
{
"path": "/etc/foo.ini",
"src_rel": "etc/foo.ini",
"owner": "root",
"group": "root",
"mode": "0644",
}
],
"managed_links": [],
}
],
"packages": [],
"users": {
"role_name": "users",
"users": [],
"managed_dirs": [],
"managed_files": [],
},
"apt_config": {
"role_name": "apt_config",
"managed_dirs": [],
"managed_files": [],
},
"dnf_config": {
"role_name": "dnf_config",
"managed_dirs": [],
"managed_files": [],
},
"sysctl": {"role_name": "sysctl", "managed_dirs": [], "managed_files": []},
"firewall_runtime": {"role_name": "firewall_runtime", "packages": []},
"etc_custom": {
"role_name": "etc_custom",
"managed_dirs": [],
"managed_files": [],
},
"usr_local_custom": {
"role_name": "usr_local_custom",
"managed_dirs": [],
"managed_files": [],
},
"extra_paths": {
"role_name": "extra_paths",
"managed_dirs": [],
"managed_files": [],
"managed_links": [],
},
},
}
_write_state(bundle, state)
monkeypatch.setattr(
jinjaturtle_mod, "find_jinjaturtle_cmd", lambda: "/usr/bin/jinjaturtle"
)
monkeypatch.setattr(jinjaturtle_mod, "can_jinjify_path", lambda _path: True)
calls = []
def fake_run_jinjaturtle(
jt_exe: str,
src_path: str,
*,
role_name: str,
force_format=None,
template_engine: str = "jinja2",
puppet_class=None,
):
calls.append((role_name, template_engine, puppet_class))
assert template_engine == "erb"
assert puppet_class == "foo"
return JinjifyResult(
template_text="[main]\nkey = <%= @main_key %>\n",
vars_text="foo::main_key: 1\n",
)
monkeypatch.setattr(jinjaturtle_mod, "run_jinjaturtle", fake_run_jinjaturtle)
manifest.manifest(
str(bundle),
str(out),
target="puppet",
jinjaturtle="on",
no_common_roles=True,
)
assert calls == [("foo", "erb", "foo")]
assert (out / "modules" / "foo" / "templates" / "etc" / "foo.ini.erb").read_text(
encoding="utf-8"
) == "[main]\nkey = <%= @main_key %>\n"
assert not (out / "modules" / "foo" / "files" / "etc" / "foo.ini").exists()
init_pp = (out / "modules" / "foo" / "manifests" / "init.pp").read_text(
encoding="utf-8"
)
assert "Any $main_key = 1," in init_pp
assert "content => template('foo/etc/foo.ini.erb')" in init_pp
assert "source =>" not in init_pp
def test_manifest_puppet_fqdn_writes_erb_template_values_to_node_hiera(
monkeypatch, tmp_path: Path
):
import enroll.jinjaturtle as jinjaturtle_mod
from enroll.jinjaturtle import JinjifyResult
bundle = tmp_path / "bundle"
out = tmp_path / "puppet"
artifact = bundle / "artifacts" / "foo" / "etc" / "foo.ini"
artifact.parent.mkdir(parents=True, exist_ok=True)
artifact.write_text("[main]\nkey = 1\n", encoding="utf-8")
state = {
"schema_version": 3,
"host": {"hostname": "test", "os": "debian", "pkg_backend": "dpkg"},
"inventory": {"packages": {}},
"roles": {
"services": [
{
"unit": "foo.service",
"role_name": "foo",
"packages": ["foo"],
"active_state": "active",
"unit_file_state": "enabled",
"managed_dirs": [],
"managed_files": [
{"path": "/etc/foo.ini", "src_rel": "etc/foo.ini"}
],
"managed_links": [],
}
],
"packages": [],
"users": {
"role_name": "users",
"users": [],
"managed_dirs": [],
"managed_files": [],
},
"apt_config": {
"role_name": "apt_config",
"managed_dirs": [],
"managed_files": [],
},
"dnf_config": {
"role_name": "dnf_config",
"managed_dirs": [],
"managed_files": [],
},
"sysctl": {"role_name": "sysctl", "managed_dirs": [], "managed_files": []},
"firewall_runtime": {"role_name": "firewall_runtime", "packages": []},
"etc_custom": {
"role_name": "etc_custom",
"managed_dirs": [],
"managed_files": [],
},
"usr_local_custom": {
"role_name": "usr_local_custom",
"managed_dirs": [],
"managed_files": [],
},
"extra_paths": {
"role_name": "extra_paths",
"managed_dirs": [],
"managed_files": [],
"managed_links": [],
},
},
}
_write_state(bundle, state)
monkeypatch.setattr(
jinjaturtle_mod, "find_jinjaturtle_cmd", lambda: "/usr/bin/jinjaturtle"
)
monkeypatch.setattr(jinjaturtle_mod, "can_jinjify_path", lambda _path: True)
monkeypatch.setattr(
jinjaturtle_mod,
"run_jinjaturtle",
lambda *a, **k: JinjifyResult(
template_text="[main]\nkey = <%= @main_key %>\n",
vars_text="foo::main_key: 1\n",
),
)
manifest.manifest(
str(bundle), str(out), target="puppet", fqdn="test.example", jinjaturtle="on"
)
node_data = yaml.safe_load(
(out / "data" / "nodes" / "test.example.yaml").read_text(encoding="utf-8")
)
assert node_data["foo::main_key"] == 1
assert node_data["foo::files"]["/etc/foo.ini"]["template"] == "foo/etc/foo.ini.erb"
assert "source" not in node_data["foo::files"]["/etc/foo.ini"]
init_pp = (out / "modules" / "foo" / "manifests" / "init.pp").read_text(
encoding="utf-8"
)
assert "Any $main_key = undef," in init_pp
assert "content => template($attrs['template'])" in init_pp

View file

@ -1,12 +1,20 @@
from __future__ import annotations from __future__ import annotations
import json import json
from collections import OrderedDict
from pathlib import Path from pathlib import Path
import yaml import yaml
from enroll import manifest from enroll import manifest
from enroll.salt import SaltRole, _render_static_role, _role_pillar_values, _salt_name from enroll.salt import (
SaltRole,
_render_pillar_role,
_render_static_role,
_role_pillar_values,
_salt_name,
_state_id,
)
def _write_state(bundle: Path, state: dict) -> None: def _write_state(bundle: Path, state: dict) -> None:
@ -188,6 +196,76 @@ def test_manifest_salt_writes_single_site_state_tree(tmp_path: Path):
assert (out / "config" / "master.d" / "enroll.conf").exists() assert (out / "config" / "master.d" / "enroll.conf").exists()
def test_manifest_salt_fqdn_package_watch_targets_declared_service_role(
tmp_path: Path,
):
bundle = tmp_path / "bundle"
out = tmp_path / "salt"
artifact = bundle / "artifacts" / "apparmor" / "etc" / "apparmor" / "parser.conf"
artifact.parent.mkdir(parents=True, exist_ok=True)
artifact.write_text("cache-loc /var/cache/apparmor\n", encoding="utf-8")
state = _sample_state()
state["inventory"] = {"packages": {"apparmor": {"section": "admin"}}}
state["roles"]["services"] = [
{
"unit": "apparmor.service",
"role_name": "apparmor_service",
"packages": ["apparmor"],
"active_state": "active",
"unit_file_state": "enabled",
"managed_dirs": [],
"managed_files": [],
"managed_links": [],
}
]
state["roles"]["packages"] = [
{
"package": "apparmor",
"role_name": "apparmor",
"section": "admin",
"managed_dirs": [],
"managed_files": [
{
"path": "/etc/apparmor/parser.conf",
"src_rel": "etc/apparmor/parser.conf",
"owner": "root",
"group": "root",
"mode": "0644",
}
],
"managed_links": [],
}
]
state["roles"]["sysctl"] = {
"role_name": "sysctl",
"managed_dirs": [],
"managed_files": [],
"managed_links": [],
}
_write_state(bundle, state)
manifest.manifest(str(bundle), str(out), target="salt", fqdn="vpn-ssh")
pillar_top = yaml.safe_load(
(out / "pillar" / "top.sls").read_text(encoding="utf-8")
)
node_sls = pillar_top["base"]["vpn-ssh"][0]
pillar_path = out / "pillar" / Path(*node_sls.split("."))
pillar = yaml.safe_load(pillar_path.with_suffix(".sls").read_text(encoding="utf-8"))
roles = pillar["enroll"]["roles"]
expected_service_state = _state_id(
"service", "apparmor.service", role="apparmor_service"
)
assert roles["apparmor"]["files"]["/etc/apparmor/parser.conf"]["watch_in"] == [
{"service": expected_service_state}
]
assert roles["apparmor_service"]["services"]["apparmor.service"]["state_id"] == (
expected_service_state
)
def test_manifest_salt_fqdn_mode_uses_pillar_and_accumulates_nodes(tmp_path: Path): def test_manifest_salt_fqdn_mode_uses_pillar_and_accumulates_nodes(tmp_path: Path):
out = tmp_path / "salt" out = tmp_path / "salt"
@ -540,6 +618,71 @@ def test_manifest_salt_uses_jinjaturtle_templates(monkeypatch, tmp_path: Path):
assert file_data["context"] == {"foo_setting": True} assert file_data["context"] == {"foo_setting": True}
def test_manifest_salt_rewrites_jinjaturtle_json_filters(monkeypatch, tmp_path: Path):
import enroll.jinjaturtle as jinjaturtle_mod
from enroll.jinjaturtle import JinjifyResult
bundle = tmp_path / "bundle"
out = tmp_path / "salt"
state = _sample_state()
_write_sample_artifacts(bundle)
_write_state(bundle, state)
monkeypatch.setattr(
jinjaturtle_mod, "find_jinjaturtle_cmd", lambda: "/usr/bin/jinjaturtle"
)
monkeypatch.setattr(jinjaturtle_mod, "can_jinjify_path", lambda _path: True)
def fake_run_jinjaturtle(
jt_exe: str, src_path: str, *, role_name: str, force_format=None
):
return JinjifyResult(
template_text='{ "setting": {{ foo_setting | to_json(ensure_ascii=False) }} }\n',
vars_text='foo_setting: "alpha"\n',
)
monkeypatch.setattr(jinjaturtle_mod, "run_jinjaturtle", fake_run_jinjaturtle)
manifest.manifest(str(bundle), str(out), target="salt", jinjaturtle="on")
template_text = (
out / "states" / "roles" / "net" / "templates" / "etc" / "foo.conf.j2"
).read_text(encoding="utf-8")
assert "to_json" not in template_text
assert "foo_setting__enroll_json" in template_text
sls = (out / "states" / "roles" / "net" / "init.sls").read_text(encoding="utf-8")
assert "foo_setting__enroll_json:" in sls
assert '"alpha"' in sls
def test_manifest_salt_pillar_role_uses_json_for_template_context() -> None:
role = SaltRole("foo")
role.add_managed_file(
"/etc/foo.json",
source="salt://roles/foo/templates/etc/foo.json.j2",
user="root",
group="root",
mode="0644",
makedirs=True,
template="jinja",
context=OrderedDict(
[("foo_name", "alpha"), ("foo_nested", OrderedDict([("x", 1)]))]
),
)
pillar = _role_pillar_values(role)
assert type(pillar["files"]["/etc/foo.json"]["context"]) is dict
assert type(pillar["files"]["/etc/foo.json"]["context"]["foo_nested"]) is dict
rendered = _render_static_role(role)
assert "foo_nested:" in rendered
context_block = (
_render_pillar_role(role).split("context:", 1)[1].split("{% endif %}", 1)[0]
)
assert "|yaml_encode" not in context_block
assert "|tojson" in _render_pillar_role(role)
def test_manifest_salt_renders_firewall_runtime_states(tmp_path: Path): def test_manifest_salt_renders_firewall_runtime_states(tmp_path: Path):
bundle = tmp_path / "bundle" bundle = tmp_path / "bundle"
out = tmp_path / "salt" out = tmp_path / "salt"