diff --git a/CHANGELOG.md b/CHANGELOG.md index ef94a82..a6b840d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 0.7.0 + + * Add --merge-simple-packages to reduce the number of roles, for packages that have no config files or services to maintain. + # 0.6.0 * Add support for capturing ipset and iptables configuration files diff --git a/README.md b/README.md index d2d51ad..96e228f 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,7 @@ Generate Ansible output from an existing harvest bundle. **Common flags** - `--fqdn `: enables **multi-site** output style +- `--merge-simple-packages`: Puts all packages that don't have config files or services to maintain, in a `common_packages` role, to reduce the number of overall Ansible roles to run. **Role tags** Generated playbooks tag each role so you can target just the parts you need: @@ -148,7 +149,7 @@ Convenience wrapper that runs **harvest → manifest** in one command. Use this when you want “get me something workable ASAP”. -Supports the same general flags as harvest/manifest, including `--fqdn`, remote harvest flags, and `--sops`. +Supports the same general flags as harvest/manifest, including `--fqdn`, remote harvest flags, `--merge-simple-packages`, and `--sops`. --- diff --git a/enroll/cli.py b/enroll/cli.py index 44de047..2483a24 100644 --- a/enroll/cli.py +++ b/enroll/cli.py @@ -459,7 +459,6 @@ def main() -> None: "Excludes apply to all harvesting, including defaults." ), ) - h.add_argument( "--sops", nargs="+", @@ -504,6 +503,11 @@ def main() -> None: "(binary) using the given GPG fingerprint(s). Requires `sops` on PATH." ), ) + m.add_argument( + "--merge-simple-packages", + action="store_true", + help="Merge packages with no configuration files into a single 'common_packages' role.", + ) _add_common_manifest_args(m) s = sub.add_parser( @@ -543,7 +547,6 @@ def main() -> None: "Excludes apply to all harvesting, including defaults." ), ) - s.add_argument( "--sops", nargs="+", @@ -567,6 +570,11 @@ def main() -> None: "or a file path." ), ) + s.add_argument( + "--merge-simple-packages", + action="store_true", + help="Merge packages with no configuration files into a single 'common_packages' role.", + ) _add_common_manifest_args(s) d = sub.add_parser("diff", help="Compare two harvests and report differences") @@ -913,6 +921,7 @@ def main() -> None: fqdn=args.fqdn, jinjaturtle=_jt_mode(args), sops_fingerprints=getattr(args, "sops", None), + merge_simple_packages=getattr(args, "merge_simple_packages", False), ) if getattr(args, "sops", None) and out_enc: print(str(out_enc)) @@ -1050,6 +1059,9 @@ def main() -> None: fqdn=args.fqdn, jinjaturtle=_jt_mode(args), sops_fingerprints=list(sops_fps), + merge_simple_packages=getattr( + args, "merge_simple_packages", False + ), ) if not args.harvest: print(str(out_file)) @@ -1080,6 +1092,9 @@ def main() -> None: args.out, fqdn=args.fqdn, jinjaturtle=_jt_mode(args), + merge_simple_packages=getattr( + args, "merge_simple_packages", False + ), ) # For usability (when --harvest wasn't provided), print the harvest path. if not args.harvest: @@ -1110,6 +1125,9 @@ def main() -> None: fqdn=args.fqdn, jinjaturtle=_jt_mode(args), sops_fingerprints=list(sops_fps), + merge_simple_packages=getattr( + args, "merge_simple_packages", False + ), ) if not args.harvest: print(str(out_file)) @@ -1129,6 +1147,9 @@ def main() -> None: args.out, fqdn=args.fqdn, jinjaturtle=_jt_mode(args), + merge_simple_packages=getattr( + args, "merge_simple_packages", False + ), ) except RemoteSudoPasswordRequired: raise SystemExit( diff --git a/enroll/harvest.py b/enroll/harvest.py index b64862e..9e5d320 100644 --- a/enroll/harvest.py +++ b/enroll/harvest.py @@ -90,6 +90,7 @@ class PackageSnapshot: managed_links: List[ManagedLink] = field(default_factory=list) excluded: List[ExcludedFile] = field(default_factory=list) notes: List[str] = field(default_factory=list) + has_config: bool = True # False if package has no config/systemd/cron files @dataclass @@ -366,10 +367,6 @@ def _role_name_from_unit(unit: str) -> str: return _safe_name(base) -def _role_name_from_pkg(pkg: str) -> str: - return _safe_name(pkg) - - def _copy_into_bundle( bundle_dir: str, role_name: str, abs_path: str, src_rel: str ) -> None: @@ -1625,6 +1622,7 @@ def harvest( manual_pkgs_skipped: List[str] = [] pkg_snaps: List[PackageSnapshot] = [] + simple_packages: List[str] = [] # Packages with no config/systemd/cron files # Add dedicated cron/logrotate roles (if detected) as package roles. # These roles centralise all cron/logrotate managed files so they aren't scattered @@ -1634,16 +1632,18 @@ def harvest( if logrotate_snapshot is not None: pkg_snaps.append(logrotate_snapshot) for pkg in sorted(manual_pkgs): + # Skip packages that are already managed by service roles + if pkg in covered_by_services: + manual_pkgs_skipped.append(pkg) + continue + # Skip cron/logrotate packages (they have dedicated roles) if cron_snapshot is not None and pkg == cron_pkg: manual_pkgs_skipped.append(pkg) continue if logrotate_snapshot is not None and pkg == logrotate_pkg: manual_pkgs_skipped.append(pkg) continue - if pkg in covered_by_services: - manual_pkgs_skipped.append(pkg) - continue - role = _role_name_from_pkg(pkg) + notes: List[str] = [] excluded: List[ExcludedFile] = [] managed: List[ManagedFile] = [] @@ -1708,8 +1708,13 @@ def harvest( seen_global=captured_global, ) - if not pkg_to_etc_paths.get(pkg, []) and not managed: - notes.append("No /etc files detected for this package.") + has_config = bool(pkg_to_etc_paths.get(pkg, []) or managed) + + if not has_config: + notes.append( + "No /etc files or custom configuration detected for this package." + ) + simple_packages.append(pkg) pkg_snaps.append( PackageSnapshot( @@ -1719,6 +1724,7 @@ def harvest( managed_links=[], excluded=excluded, notes=notes, + has_config=has_config, ) ) diff --git a/enroll/manifest.py b/enroll/manifest.py index 99adbb7..97f8caf 100644 --- a/enroll/manifest.py +++ b/enroll/manifest.py @@ -825,6 +825,7 @@ def _manifest_from_bundle_dir( *, fqdn: Optional[str] = None, jinjaturtle: str = "auto", # auto|on|off + merge_simple_packages: bool = False, ) -> None: state_path = os.path.join(bundle_dir, "state.json") with open(state_path, "r", encoding="utf-8") as f: @@ -2055,9 +2056,101 @@ Generated from `{unit}`. manifested_service_roles.append(role) + # ------------------------- + # Merge simple packages (if --merge-simple-packages is set) + # + # Packages with no configuration files, systemd units, or cron jobs + # are merged into a single 'common_packages' role to reduce role count. + # ------------------------- + simple_packages_list: List[str] = [] + if merge_simple_packages: + filtered_package_roles: List[Dict[str, Any]] = [] + for pr in package_roles: + has_config = pr.get("has_config", True) + managed_files = pr.get("managed_files", []) or [] + # A package is "simple" if it has no config files AND no managed files + if not has_config and not managed_files: + pkg = pr.get("package") + if pkg: + simple_packages_list.append(pkg) + else: + filtered_package_roles.append(pr) + package_roles = filtered_package_roles + # ------------------------- # Manually installed package roles # ------------------------- + # First, create the common_packages role if we have simple packages to merge + if simple_packages_list: + role = "common_packages" + role_dir = os.path.join(roles_root, role) + _write_role_scaffold(role_dir) + + var_prefix = role + + # No managed files for common_packages - just package installation + files_var: List[Dict[str, Any]] = [] + links_var: List[Dict[str, Any]] = [] + dirs_var: List[Dict[str, Any]] = [] + + base_vars: Dict[str, Any] = { + f"{var_prefix}_packages": simple_packages_list, + f"{var_prefix}_managed_files": files_var, + f"{var_prefix}_managed_dirs": dirs_var, + f"{var_prefix}_managed_links": links_var, + } + + if site_mode: + _write_role_defaults( + role_dir, + { + f"{var_prefix}_packages": [], + f"{var_prefix}_managed_files": [], + f"{var_prefix}_managed_dirs": [], + f"{var_prefix}_managed_links": [], + }, + ) + _write_hostvars(out_dir, fqdn or "", role, base_vars) + else: + _write_role_defaults(role_dir, base_vars) + + handlers = "---\n" + with open( + os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(handlers) + + task_parts: List[str] = [] + task_parts.append("---\n" + _render_install_packages_tasks(role, var_prefix)) + + tasks = "\n".join(task_parts).rstrip() + "\n" + with open( + os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write(tasks) + + with open( + os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8" + ) as f: + f.write("---\ndependencies: []\n") + + readme = f"""# {role} + +Common packages with no configuration files. + +This role was created by merging simple packages using the `--merge-simple-packages` flag. + +## Packages +{os.linesep.join("- " + p for p in simple_packages_list) or "- (none)"} + +> Note: This role only installs packages; it does not manage any configuration files or services. +""" + with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f: + f.write(readme) + + manifested_pkg_roles.append(role) + + # Process package roles (those with configuration files) for pr in package_roles: role = pr["role_name"] pkg = pr.get("package") or "" @@ -2221,6 +2314,7 @@ def manifest( fqdn: Optional[str] = None, jinjaturtle: str = "auto", # auto|on|off sops_fingerprints: Optional[List[str]] = None, + merge_simple_packages: bool = False, ) -> Optional[str]: """Render an Ansible manifest from a harvest. @@ -2249,7 +2343,11 @@ def manifest( try: if not sops_mode: _manifest_from_bundle_dir( - resolved_bundle_dir, out, fqdn=fqdn, jinjaturtle=jinjaturtle + resolved_bundle_dir, + out, + fqdn=fqdn, + jinjaturtle=jinjaturtle, + merge_simple_packages=merge_simple_packages, ) return None @@ -2265,7 +2363,11 @@ def manifest( pass _manifest_from_bundle_dir( - resolved_bundle_dir, str(tmp_out), fqdn=fqdn, jinjaturtle=jinjaturtle + resolved_bundle_dir, + str(tmp_out), + fqdn=fqdn, + jinjaturtle=jinjaturtle, + merge_simple_packages=merge_simple_packages, ) enc = _encrypt_manifest_out_dir_to_sops( diff --git a/enroll/schema/state.schema.json b/enroll/schema/state.schema.json index d0bde52..d8c136a 100644 --- a/enroll/schema/state.schema.json +++ b/enroll/schema/state.schema.json @@ -390,6 +390,10 @@ "package": { "minLength": 1, "type": "string" + }, + "has_config": { + "type": "boolean", + "default": true } }, "required": [ diff --git a/tests.sh b/tests.sh index 126a87b..68f7007 100755 --- a/tests.sh +++ b/tests.sh @@ -51,3 +51,14 @@ ansible-lint "${ANSIBLE_DIR}" # Run ansible-playbook playbook.yml -i "localhost," -c local --check --diff + +# Common simple packages mode +poetry run \ + enroll manifest \ + --harvest "${BUNDLE_DIR}2" \ + --out "${ANSIBLE_DIR}2" \ + --merge-simple-packages + +builtin cd "${ANSIBLE_DIR}2" +ls "${ANSIBLE_DIR}2/roles" +ansible-playbook playbook.yml -i "localhost," -c local --check --diff