Include files from /usr/local/bin and /usr/local/etc in harvest (assuming they aren't binaries or symlinks) and store in usr_local_custom role, similar to etc_custom.
All checks were successful
CI / test (push) Successful in 5m43s
Lint / test (push) Successful in 30s
Trivy / test (push) Successful in 19s

This commit is contained in:
Miguel Jacq 2025-12-18 17:11:04 +11:00
parent b5d2b99174
commit 4660a0703e
Signed by: mig5
GPG key ID: 59B3F0C24135C6A9
11 changed files with 551 additions and 3 deletions

View file

@ -190,6 +190,12 @@ def _iter_managed_files(state: Dict[str, Any]) -> Iterable[Tuple[str, Dict[str,
for mf in ec.get("managed_files", []) or []:
yield str(ec_role), mf
# usr_local_custom
ul = state.get("usr_local_custom") or {}
ul_role = ul.get("role_name") or "usr_local_custom"
for mf in ul.get("managed_files", []) or []:
yield str(ul_role), mf
def _file_index(bundle_dir: Path, state: Dict[str, Any]) -> Dict[str, FileRec]:
"""Return mapping of absolute path -> FileRec.

View file

@ -78,6 +78,14 @@ class EtcCustomSnapshot:
notes: List[str]
@dataclass
class UsrLocalCustomSnapshot:
role_name: str
managed_files: List[ManagedFile]
excluded: List[ExcludedFile]
notes: List[str]
ALLOWED_UNOWNED_EXTS = {
".conf",
".cfg",
@ -701,6 +709,103 @@ def harvest(
notes=etc_notes,
)
# -------------------------
# usr_local_custom role (/usr/local/etc + /usr/local/bin scripts)
# -------------------------
ul_notes: List[str] = []
ul_excluded: List[ExcludedFile] = []
ul_managed: List[ManagedFile] = []
ul_role_name = "usr_local_custom"
# Extend the already-captured set with etc_custom.
already_all: Set[str] = set(already)
for mf in etc_managed:
already_all.add(mf.path)
def _scan_usr_local_tree(
root: str, *, require_executable: bool, cap: int, reason: str
) -> None:
scanned = 0
if not os.path.isdir(root):
return
for dirpath, _, filenames in os.walk(root):
for fn in filenames:
path = os.path.join(dirpath, fn)
if path in already_all:
continue
if not os.path.isfile(path) or os.path.islink(path):
continue
if require_executable:
try:
owner, group, mode = stat_triplet(path)
except OSError:
ul_excluded.append(ExcludedFile(path=path, reason="unreadable"))
continue
try:
if (int(mode, 8) & 0o111) == 0:
continue
except ValueError:
# If mode parsing fails, be conservative and skip.
continue
else:
try:
owner, group, mode = stat_triplet(path)
except OSError:
ul_excluded.append(ExcludedFile(path=path, reason="unreadable"))
continue
deny = policy.deny_reason(path)
if deny:
ul_excluded.append(ExcludedFile(path=path, reason=deny))
continue
src_rel = path.lstrip("/")
try:
_copy_into_bundle(bundle_dir, ul_role_name, path, src_rel)
except OSError:
ul_excluded.append(ExcludedFile(path=path, reason="unreadable"))
continue
ul_managed.append(
ManagedFile(
path=path,
src_rel=src_rel,
owner=owner,
group=group,
mode=mode,
reason=reason,
)
)
already_all.add(path)
scanned += 1
if scanned >= cap:
ul_notes.append(f"Reached file cap ({cap}) while scanning {root}.")
return
# /usr/local/etc: capture all non-binary regular files (filtered by IgnorePolicy)
_scan_usr_local_tree(
"/usr/local/etc",
require_executable=False,
cap=2000,
reason="usr_local_etc_custom",
)
# /usr/local/bin: capture executable scripts only (skip non-executable text)
_scan_usr_local_tree(
"/usr/local/bin",
require_executable=True,
cap=2000,
reason="usr_local_bin_script",
)
usr_local_custom_snapshot = UsrLocalCustomSnapshot(
role_name=ul_role_name,
managed_files=ul_managed,
excluded=ul_excluded,
notes=ul_notes,
)
state = {
"host": {"hostname": os.uname().nodename, "os": "debian"},
"users": asdict(users_snapshot),
@ -709,6 +814,7 @@ def harvest(
"manual_packages_skipped": manual_pkgs_skipped,
"package_roles": [asdict(p) for p in pkg_snaps],
"etc_custom": asdict(etc_custom_snapshot),
"usr_local_custom": asdict(usr_local_custom_snapshot),
}
state_path = os.path.join(bundle_dir, "state.json")

View file

@ -23,6 +23,11 @@ DEFAULT_DENY_GLOBS = [
"/etc/gshadow",
"/etc/*shadow",
"/etc/letsencrypt/*",
"/usr/local/etc/ssl/private/*",
"/usr/local/etc/ssh/ssh_host_*",
"/usr/local/etc/*shadow",
"/usr/local/etc/*gshadow",
"/usr/local/etc/letsencrypt/*",
]
SENSITIVE_CONTENT_PATTERNS = [

View file

@ -629,6 +629,7 @@ def _manifest_from_bundle_dir(
package_roles: List[Dict[str, Any]] = state.get("package_roles", [])
users_snapshot: Dict[str, Any] = state.get("users", {})
etc_custom_snapshot: Dict[str, Any] = state.get("etc_custom", {})
usr_local_custom_snapshot: Dict[str, Any] = state.get("usr_local_custom", {})
site_mode = fqdn is not None and fqdn != ""
@ -661,6 +662,7 @@ def _manifest_from_bundle_dir(
manifested_users_roles: List[str] = []
manifested_etc_custom_roles: List[str] = []
manifested_usr_local_custom_roles: List[str] = []
manifested_service_roles: List[str] = []
manifested_pkg_roles: List[str] = []
@ -999,6 +1001,105 @@ Unowned /etc config files not attributed to packages or services.
# -------------------------
# -------------------------
# -------------------------
# usr_local_custom role (/usr/local/etc + /usr/local/bin scripts)
# -------------------------
if usr_local_custom_snapshot and usr_local_custom_snapshot.get("managed_files"):
role = usr_local_custom_snapshot.get("role_name", "usr_local_custom")
role_dir = os.path.join(roles_root, role)
_write_role_scaffold(role_dir)
var_prefix = role
managed_files = usr_local_custom_snapshot.get("managed_files", [])
excluded = usr_local_custom_snapshot.get("excluded", [])
notes = usr_local_custom_snapshot.get("notes", [])
templated, jt_vars = _jinjify_managed_files(
bundle_dir,
role,
role_dir,
managed_files,
jt_exe=jt_exe,
jt_enabled=jt_enabled,
overwrite_templates=not site_mode,
)
# Copy only the non-templated artifacts (templates live in the role).
if site_mode:
_copy_artifacts(
bundle_dir,
role,
_host_role_files_dir(out_dir, fqdn or "", role),
exclude_rels=templated,
)
else:
_copy_artifacts(
bundle_dir,
role,
os.path.join(role_dir, "files"),
exclude_rels=templated,
)
files_var = _build_managed_files_var(
managed_files,
templated,
notify_other=None,
notify_systemd=None,
)
jt_map = _yaml_load_mapping(jt_vars) if jt_vars.strip() else {}
vars_map: Dict[str, Any] = {f"{var_prefix}_managed_files": files_var}
vars_map = _merge_mappings_overwrite(vars_map, jt_map)
if site_mode:
_write_role_defaults(role_dir, {f"{var_prefix}_managed_files": []})
_write_hostvars(out_dir, fqdn or "", role, vars_map)
else:
_write_role_defaults(role_dir, vars_map)
tasks = "---\n" + _render_generic_files_tasks(
var_prefix, include_restart_notify=False
)
with open(
os.path.join(role_dir, "tasks", "main.yml"), "w", encoding="utf-8"
) as f:
f.write(tasks.rstrip() + "\n")
# No handlers needed for this role, but keep a valid YAML document.
with open(
os.path.join(role_dir, "handlers", "main.yml"), "w", encoding="utf-8"
) as f:
f.write("---\n")
with open(
os.path.join(role_dir, "meta", "main.yml"), "w", encoding="utf-8"
) as f:
f.write("---\ndependencies: []\n")
readme = (
"""# usr_local_custom\n\n"""
"Unowned /usr/local files (scripts in /usr/local/bin and config under /usr/local/etc).\n\n"
"## Managed files\n"
+ ("\n".join([f"- {mf.get('path')}" for mf in managed_files]) or "- (none)")
+ "\n\n## Excluded\n"
+ (
"\n".join([f"- {e.get('path')} ({e.get('reason')})" for e in excluded])
or "- (none)"
)
+ "\n\n## Notes\n"
+ ("\n".join([f"- {n}" for n in notes]) or "- (none)")
+ "\n"
)
with open(os.path.join(role_dir, "README.md"), "w", encoding="utf-8") as f:
f.write(readme)
manifested_usr_local_custom_roles.append(role)
# -------------------------
# -------------------------
# Service roles
# -------------------------
@ -1310,6 +1411,7 @@ Generated for package `{pkg}`.
manifested_pkg_roles
+ manifested_service_roles
+ manifested_etc_custom_roles
+ manifested_usr_local_custom_roles
+ manifested_users_roles
)