mig5/jinjaturtle
1
0
Fork 0
Code Issues Pull requests Projects Releases 8 Packages Wiki Activity Actions

Compare commits

base: mig5:main
Branches Tags
mig5:main
mig5:0.3.3
mig5:0.3.2
mig5:0.3.1
mig5:0.1.4
mig5:0.1.3
mig5:0.1.2
mig5:0.1.1
mig5:0.1.0
..
compare: mig5:0.1.0
Branches Tags
mig5:main
mig5:0.3.3
mig5:0.3.2
mig5:0.3.1
mig5:0.1.4
mig5:0.1.3
mig5:0.1.2
mig5:0.1.1
mig5:0.1.0

No commits in common. "main" and "0.1.0" have entirely different histories.
main ... 0.1.0

43 changed files with 1106 additions and 5745 deletions
Download patch file Download diff file Expand all files Collapse all files

15
.forgejo/workflows/ci.yml
View file

@ -24,22 +24,9 @@ jobs:
- name: Install project deps (including test extras)
run: |
poetry install --with dev
poetry install --with test
- name: Run test script
run: |
./tests.sh
# Notify if any previous step in this job failed
- name: Notify on failure
if: ${{ failure() }}
env:
WEBHOOK_URL: ${{ secrets.NODERED_WEBHOOK_URL }}
REPOSITORY: ${{ forgejo.repository }}
RUN_NUMBER: ${{ forgejo.run_number }}
SERVER_URL: ${{ forgejo.server_url }}
run: |
curl -X POST \
-H "Content-Type: application/json" \
-d "{\"repository\":\"$REPOSITORY\",\"run_number\":\"$RUN_NUMBER\",\"status\":\"failure\",\"url\":\"$SERVER_URL/$REPOSITORY/actions/runs/$RUN_NUMBER\"}" \
"$WEBHOOK_URL"

17
.forgejo/workflows/lint.yml
View file

@ -15,7 +15,7 @@ jobs:
run: |
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
black pyflakes3 python3-bandit vulture
black pyflakes3 python3-bandit
- name: Run linters
run: |
@ -24,18 +24,3 @@ jobs:
pyflakes3 src/*
pyflakes3 tests/*
bandit -s B110 -r src/
vulture .
# Notify if any previous step in this job failed
- name: Notify on failure
if: ${{ failure() }}
env:
WEBHOOK_URL: ${{ secrets.NODERED_WEBHOOK_URL }}
REPOSITORY: ${{ forgejo.repository }}
RUN_NUMBER: ${{ forgejo.run_number }}
SERVER_URL: ${{ forgejo.server_url }}
run: |
curl -X POST \
-H "Content-Type: application/json" \
-d "{\"repository\":\"$REPOSITORY\",\"run_number\":\"$RUN_NUMBER\",\"status\":\"failure\",\"url\":\"$SERVER_URL/$REPOSITORY/actions/runs/$RUN_NUMBER\"}" \
"$WEBHOOK_URL"

14
.forgejo/workflows/trivy.yml
View file

@ -24,17 +24,3 @@ jobs:
- name: Run trivy
run: |
trivy fs --no-progress --ignore-unfixed --format table --disable-telemetry .
# Notify if any previous step in this job failed
- name: Notify on failure
if: ${{ failure() }}
env:
WEBHOOK_URL: ${{ secrets.NODERED_WEBHOOK_URL }}
REPOSITORY: ${{ forgejo.repository }}
RUN_NUMBER: ${{ forgejo.run_number }}
SERVER_URL: ${{ forgejo.server_url }}
run: |
curl -X POST \
-H "Content-Type: application/json" \
-d "{\"repository\":\"$REPOSITORY\",\"run_number\":\"$RUN_NUMBER\",\"status\":\"failure\",\"url\":\"$SERVER_URL/$REPOSITORY/actions/runs/$RUN_NUMBER\"}" \
"$WEBHOOK_URL"

4
.gitignore vendored
View file

@ -4,7 +4,3 @@ __pycache__
.pytest_cache
dist
.coverage
*.yml
*.j2
*.toml
regenerated_*

85
Dockerfile.debbuild
View file

@ -1,85 +0,0 @@
# syntax=docker/dockerfile:1
ARG BASE_IMAGE=debian:bookworm
FROM ${BASE_IMAGE}
ENV DEBIAN_FRONTEND=noninteractive
# If Ubuntu, ensure Universe is enabled.
RUN set -eux; \
. /etc/os-release; \
if [ "${ID:-}" = "ubuntu" ]; then \
apt-get update; \
apt-get install -y --no-install-recommends software-properties-common ca-certificates; \
add-apt-repository -y universe; \
fi; \
if [ "${VERSION_CODENAME:-}" = "jammy" ]; then \
apt-get update; \
apt-get install -y --no-install-recommends python3-tomli; \
fi
# Build deps
RUN set -eux; \
apt-get update; \
apt-get install -y --no-install-recommends \
build-essential \
devscripts \
debhelper \
dh-python \
pybuild-plugin-pyproject \
python3-all \
python3-poetry-core \
python3-yaml \
python3-defusedxml \
python3-jinja2 \
python3-toml \
rsync \
ca-certificates \
; \
rm -rf /var/lib/apt/lists/*
# Build runner script
RUN set -eux; \
cat > /usr/local/bin/build-deb <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
SRC="${SRC:-/src}"
WORKROOT="${WORKROOT:-/work}"
WORK="${WORKROOT}/src"
OUT="${OUT:-/out}"
mkdir -p "$WORK" "$OUT"
rsync -a --delete \
--exclude '.git' \
--exclude '.venv' \
--exclude 'dist' \
--exclude 'build' \
--exclude '__pycache__' \
--exclude '.pytest_cache' \
--exclude '.mypy_cache' \
"${SRC}/" "${WORK}/"
cd "${WORK}"
if [ -n "${SUITE:-}" ]; then
export DEBEMAIL="mig@mig5.net"
export DEBFULLNAME="Miguel Jacq"
dch --distribution "$SUITE" --local "~${SUITE}" "CI build for $SUITE"
fi
dpkg-buildpackage -us -uc -b
shopt -s nullglob
cp -v "${WORKROOT}"/*.deb \
"${WORKROOT}"/*.changes \
"${WORKROOT}"/*.buildinfo \
"${WORKROOT}"/*.dsc \
"${WORKROOT}"/*.tar.* \
"${OUT}/" || true
echo "Artifacts copied to ${OUT}"
EOF
RUN chmod +x /usr/local/bin/build-deb
WORKDIR /work
ENTRYPOINT ["/usr/local/bin/build-deb"]

51
README.md
View file

@ -1,11 +1,8 @@
# JinjaTurtle
<div align="center">
<img src="https://git.mig5.net/mig5/jinjaturtle/raw/branch/main/jinjaturtle.svg" alt="JinjaTurtle logo" width="240" />
</div>
JinjaTurtle is a command-line tool to help you generate Jinja2 templates and
Ansible inventory from a native configuration file of a piece of software.
Ansible `defaults/main.yml` files from a native configuration file of a piece
of software.
## How it works
@ -16,40 +13,14 @@ Ansible inventory from a native configuration file of a piece of software.
role.
* A Jinja2 file is generated from the file with those parameter key names
injected as the `{{ variable }}` names.
* An Ansible inventory YAML file is generated with those key names and the
*values* taken from the original config file as the defaults.
* A `defaults/main.yml` is generated with those key names and the *values*
taken from the original config file as the defaults.
By default, the Jinja2 template and the Ansible inventory are printed to
By default, the Jinja2 template and the `defaults/main.yml` are printed to
stdout. However, it is possible to output the results to new files.
## What sort of config files can it handle?
TOML, YAML, INI, JSON and XML-style config files should be okay. There are always
going to be some edge cases in very complex files that are difficult to work
with, though, so you may still find that you need to tweak the results.
For XML and YAML files, JinjaTurtle will attempt to generate 'for' loops
and lists in the Ansible yaml if the config file looks homogenous enough to
support it. However, if it lacks the confidence in this, it will fall back to
using scalar-style flattened attributes.
You may need or wish to tidy up the config to suit your needs.
The goal here is really to *speed up* converting files into Ansible/Jinja2,
but not necessarily to make it perfect.
## How to install it
### Ubuntu/Debian apt repository
```bash
sudo mkdir -p /usr/share/keyrings
curl -fsSL https://mig5.net/static/mig5.asc | sudo gpg --dearmor -o /usr/share/keyrings/mig5.gpg
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/mig5.gpg] https://apt.mig5.net $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/mig5.list
sudo apt update
sudo apt install jinjaturtle
```
### From PyPi
```
@ -84,9 +55,9 @@ jinjaturtle php.ini \
## Full usage info
```
usage: jinjaturtle [-h] -r ROLE_NAME [-f {json,ini,toml,yaml,xml}] [-d DEFAULTS_OUTPUT] [-t TEMPLATE_OUTPUT] config
usage: jinjaturtle [-h] -r ROLE_NAME [-f {ini,toml}] [-d DEFAULTS_OUTPUT] [-t TEMPLATE_OUTPUT] config
Convert a config file into Ansible inventory and a Jinja2 template.
Convert a config file into an Ansible defaults file and Jinja2 template.
positional arguments:
config Path to the source configuration file (TOML or INI-style).
@ -95,7 +66,7 @@ options:
-h, --help show this help message and exit
-r, --role-name ROLE_NAME
Ansible role name, used as variable prefix (e.g. cometbft).
-f, --format {ini,json,toml,xml}
-f, --format {ini,toml}
Force config format instead of auto-detecting from filename.
-d, --defaults-output DEFAULTS_OUTPUT
Path to write defaults/main.yml. If omitted, defaults YAML is printed to stdout.
@ -103,9 +74,3 @@ options:
Path to write the Jinja2 config template. If omitted, template is printed to stdout.
```
## Found a bug, have a suggestion?
You can e-mail me (see the pyproject.toml for details) or contact me on the Fediverse:
https://goto.mig5.net/@mig5

11
debian/changelog vendored
View file

@ -1,11 +0,0 @@
jinjaturtle (0.3.3) unstable; urgency=medium
* Fixes for tomli on Ubuntu 22
-- Miguel Jacq <mig@mig5.net> Mon, 15 Dec 2025 14:00:00 +0000
jinjaturtle (0.3.2) unstable; urgency=medium
* Initial package
-- Miguel Jacq <mig@mig5.net> Mon, 15 Dec 2025 12:00:00 +0000

28
debian/control vendored
View file

@ -1,28 +0,0 @@
Source: jinjaturtle
Section: admin
Priority: optional
Maintainer: Miguel Jacq <mig@mig5.net>
Rules-Requires-Root: no
Build-Depends:
debhelper-compat (= 13),
dh-python,
pybuild-plugin-pyproject,
python3-all,
python3-poetry-core,
python3-yaml,
python3-toml,
python3-defusedxml,
python3-jinja2
Standards-Version: 4.6.2
Homepage: https://git.mig5.net/mig5/jinjaturtle
Package: jinjaturtle
Architecture: all
Depends:
${misc:Depends},
${python3:Depends},
python3-yaml,
python3-toml,
python3-defusedxml,
python3-jinja2
Description: Convert config files into Ansible defaults and Jinja2 templates.

6
debian/rules vendored
View file

@ -1,6 +0,0 @@
#!/usr/bin/make -f
export PYBUILD_NAME=jinjaturtle
export PYBUILD_SYSTEM=pyproject
%:
dh $@ --with python3 --buildsystem=pybuild

1
debian/source/format vendored
View file

@ -1 +0,0 @@
3.0 (quilt)

6
debian/source/options vendored
View file

@ -1,6 +0,0 @@
tar-ignore = ".git"
tar-ignore = ".venv"
tar-ignore = "__pycache__"
tar-ignore = ".pytest_cache"
tar-ignore = "dist"
tar-ignore = "build"

59
jinjaturtle.svg
View file

@ -1,59 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 128 128"
role="img"
aria-label="JinjaTurtle logo">
<!-- Turtle shell -->
<ellipse cx="56" cy="68" rx="32" ry="24"
fill="#F97316"
stroke="#111827"
stroke-width="4"/>
<!-- Center braces block (in the middle of the shell) -->
<text x="56" y="71"
text-anchor="middle"
font-family="monospace"
font-size="10"
fill="#111827">{{ }}</text>
<!-- Head -->
<circle cx="96" cy="64" r="10"
fill="#16A34A"
stroke="#111827"
stroke-width="4"/>
<!-- Eyes -->
<circle cx="93" cy="61" r="2" fill="#111827"/>
<circle cx="99" cy="61" r="2" fill="#111827"/>
<!-- Smile! -->
<path d="M92 67 Q96 70 100 67"
fill="none"
stroke="#111827"
stroke-width="2"
stroke-linecap="round"/>
<!-- Legs -->
<ellipse cx="40" cy="88" rx="8" ry="5"
fill="#16A34A"
stroke="#111827"
stroke-width="4"/>
<ellipse cx="68" cy="88" rx="8" ry="5"
fill="#16A34A"
stroke="#111827"
stroke-width="4"/>
<ellipse cx="40" cy="48" rx="8" ry="5"
fill="#16A34A"
stroke="#111827"
stroke-width="4"/>
<ellipse cx="68" cy="48" rx="8" ry="5"
fill="#16A34A"
stroke="#111827"
stroke-width="4"/>
<!-- Tail -->
<path d="M24 68 L16 64 L24 60 Z"
fill="#16A34A"
stroke="#111827"
stroke-width="3"
stroke-linejoin="round"/>
</svg>
Side by side

Before

Width:  |  Height:  |  Size: 1.5 KiB

911
poetry.lock generated
View file

File diff suppressed because it is too large Load diff

18
pyproject.toml
View file

@ -1,13 +1,13 @@
[tool.poetry]
name = "jinjaturtle"
version = "0.3.3"
version = "0.1.0"
description = "Convert config files into Ansible defaults and Jinja2 templates."
authors = ["Miguel Jacq <mig@mig5.net>"]
license = "GPL-3.0-or-later"
readme = "README.md"
packages = [{ include = "jinjaturtle", from = "src" }]
keywords = ["ansible", "jinja2", "config", "toml", "ini", "yaml", "json", "devops"]
keywords = ["ansible", "jinja2", "config", "toml", "ini", "devops"]
homepage = "https://git.mig5.net/mig5/jinjaturtle"
repository = "https://git.mig5.net/mig5/jinjaturtle"
@ -16,8 +16,13 @@ repository = "https://git.mig5.net/mig5/jinjaturtle"
python = "^3.10"
PyYAML = "^6.0"
tomli = { version = "^2.0.0", python = "<3.11" }
defusedxml = "^0.7.1"
jinja2 = "^3.1.6"
[tool.poetry.group.dev.dependencies]
pytest = "^7.0"
pytest-cov = "^4.0"
build = "^1.0"
twine = "^5.0"
pyproject-appimage = "^4.2"
[tool.poetry.scripts]
jinjaturtle = "jinjaturtle.cli:main"
@ -29,8 +34,3 @@ build-backend = "poetry.core.masonry.api"
[tool.pyproject-appimage]
script = "jinjaturtle"
output = "JinjaTurtle.AppImage"
[tool.poetry.dev-dependencies]
pytest = "^8"
pytest-cov = "^5"
pyproject-appimage = "^4.2"

30
release.sh
View file

@ -2,8 +2,7 @@
set -eo pipefail
# Clean caches etc
filedust -y .
rm -rf dist
# Publish to Pypi
poetry build
@ -15,30 +14,3 @@ mv JinjaTurtle.AppImage dist/
# Sign packages
for file in `ls -1 dist/`; do qubes-gpg-client --batch --armor --detach-sign dist/$file > dist/$file.asc; done
# Deb stuff
DISTS=(
debian:bookworm
debian:trixie
ubuntu:jammy
ubuntu:noble
)
for dist in ${DISTS[@]}; do
release=$(echo ${dist} | cut -d: -f2)
mkdir -p dist/${release}
docker build -f Dockerfile.debbuild -t jinjaturtle-deb:${release} \
--no-cache \
--progress=plain \
--build-arg BASE_IMAGE=${dist} .
docker run --rm \
-e SUITE="${release}" \
-v "$PWD":/src \
-v "$PWD/dist/${release}":/out \
jinjaturtle-deb:${release}
debfile=$(ls -1 dist/${release}/*.deb)
reprepro -b /home/user/git/repo includedeb "${release}" "${debfile}"
done

46
src/jinjaturtle/cli.py
View file

@ -2,15 +2,13 @@ from __future__ import annotations
import argparse
import sys
from defusedxml import defuse_stdlib
from pathlib import Path
from .core import (
parse_config,
analyze_loops,
flatten_config,
generate_ansible_yaml,
generate_jinja2_template,
generate_defaults_yaml,
generate_template,
)
@ -21,7 +19,7 @@ def _build_arg_parser() -> argparse.ArgumentParser:
)
ap.add_argument(
"config",
help="Path to the source configuration file (TOML, YAML, JSON or INI-style).",
help="Path to the source configuration file (TOML or INI-style).",
)
ap.add_argument(
"-r",
@ -32,7 +30,7 @@ def _build_arg_parser() -> argparse.ArgumentParser:
ap.add_argument(
"-f",
"--format",
choices=["ini", "json", "toml", "yaml", "xml"],
choices=["ini", "toml"],
help="Force config format instead of auto-detecting from filename.",
)
ap.add_argument(
@ -49,39 +47,20 @@ def _build_arg_parser() -> argparse.ArgumentParser:
def _main(argv: list[str] | None = None) -> int:
defuse_stdlib()
parser = _build_arg_parser()
args = parser.parse_args(argv)
config_path = Path(args.config)
config_text = config_path.read_text(encoding="utf-8")
# Parse the config
fmt, parsed = parse_config(config_path, args.format)
# Analyze for loops
loop_candidates = analyze_loops(fmt, parsed)
# Flatten config (excluding loop paths if loops are detected)
flat_items = flatten_config(fmt, parsed, loop_candidates)
# Generate defaults YAML (with loop collections if detected)
ansible_yaml = generate_ansible_yaml(args.role_name, flat_items, loop_candidates)
# Generate template (with loops if detected)
template_str = generate_jinja2_template(
fmt,
parsed,
args.role_name,
original_text=config_text,
loop_candidates=loop_candidates,
)
flat_items = flatten_config(fmt, parsed)
defaults_yaml = generate_defaults_yaml(args.role_name, flat_items)
template_str = generate_template(fmt, parsed, args.role_name)
if args.defaults_output:
Path(args.defaults_output).write_text(ansible_yaml, encoding="utf-8")
Path(args.defaults_output).write_text(defaults_yaml, encoding="utf-8")
else:
print("# defaults/main.yml")
print(ansible_yaml, end="")
print(defaults_yaml, end="")
if args.template_output:
Path(args.template_output).write_text(template_str, encoding="utf-8")
@ -89,11 +68,14 @@ def _main(argv: list[str] | None = None) -> int:
print("# config.j2")
print(template_str, end="")
return True
return 0
def main() -> None:
"""
Console-script entry point.
Defined in pyproject.toml as:
jinjaturtle = jinjaturtle.cli:main
"""
_main(sys.argv[1:])
raise SystemExit(_main(sys.argv[1:]))

347
src/jinjaturtle/core.py
View file

@ -1,78 +1,23 @@
from __future__ import annotations
import configparser
from pathlib import Path
from typing import Any, Iterable
import datetime
import yaml
from .loop_analyzer import LoopAnalyzer, LoopCandidate
from .handlers import (
BaseHandler,
IniHandler,
JsonHandler,
TomlHandler,
YamlHandler,
XmlHandler,
)
class QuotedString(str):
"""
Marker type for strings that must be double-quoted in YAML output.
"""
pass
def _fallback_str_representer(dumper: yaml.SafeDumper, data: Any):
"""
Fallback for objects the dumper doesn't know about.
"""
return dumper.represent_scalar("tag:yaml.org,2002:str", str(data))
class _TurtleDumper(yaml.SafeDumper):
"""
Custom YAML dumper that always double-quotes QuotedString values.
"""
pass
def _quoted_str_representer(dumper: yaml.SafeDumper, data: QuotedString):
return dumper.represent_scalar("tag:yaml.org,2002:str", str(data), style='"')
_TurtleDumper.add_representer(QuotedString, _quoted_str_representer)
# Use our fallback for any unknown object types
_TurtleDumper.add_representer(None, _fallback_str_representer)
_HANDLERS: dict[str, BaseHandler] = {}
_INI_HANDLER = IniHandler()
_JSON_HANDLER = JsonHandler()
_TOML_HANDLER = TomlHandler()
_YAML_HANDLER = YamlHandler()
_XML_HANDLER = XmlHandler()
_HANDLERS["ini"] = _INI_HANDLER
_HANDLERS["json"] = _JSON_HANDLER
_HANDLERS["toml"] = _TOML_HANDLER
_HANDLERS["yaml"] = _YAML_HANDLER
_HANDLERS["xml"] = _XML_HANDLER
def make_var_name(role_prefix: str, path: Iterable[str]) -> str:
"""
Wrapper for :meth:`BaseHandler.make_var_name`.
"""
return BaseHandler.make_var_name(role_prefix, path)
try:
import tomllib # Python 3.11+
except ModuleNotFoundError: # pragma: no cover
try:
import tomli as tomllib # type: ignore
except ModuleNotFoundError: # pragma: no cover
tomllib = None # type: ignore
def detect_format(path: Path, explicit: str | None = None) -> str:
"""
Determine config format from argument or filename.
Determine config format (toml vs ini-ish) from argument or filename.
"""
if explicit:
return explicit
@ -80,168 +25,214 @@ def detect_format(path: Path, explicit: str | None = None) -> str:
name = path.name.lower()
if suffix == ".toml":
return "toml"
if suffix in {".yaml", ".yml"}:
return "yaml"
if suffix == ".json":
return "json"
if suffix in {".ini", ".cfg", ".conf"} or name.endswith(".ini"):
return "ini"
if suffix == ".xml":
return "xml"
# Fallback: treat as INI-ish
return "ini"
def parse_config(path: Path, fmt: str | None = None) -> tuple[str, Any]:
"""
Parse config file into a Python object.
Parse config file into a Python object:
TOML -> nested dict
INI -> configparser.ConfigParser
"""
fmt = detect_format(path, fmt)
handler = _HANDLERS.get(fmt)
if handler is None:
raise ValueError(f"Unsupported config format: {fmt}")
parsed = handler.parse(path)
# Make sure datetime objects are treated as strings (TOML, YAML)
parsed = _stringify_timestamps(parsed)
return fmt, parsed
if fmt == "toml":
if tomllib is None:
raise RuntimeError(
"tomllib/tomli is required to parse TOML files but is not installed"
)
with path.open("rb") as f:
data = tomllib.load(f)
return fmt, data
if fmt == "ini":
parser = configparser.ConfigParser()
parser.optionxform = str # preserve key case
with path.open("r", encoding="utf-8") as f:
parser.read_file(f)
return fmt, parser
raise ValueError(f"Unsupported config format: {fmt}")
def analyze_loops(fmt: str, parsed: Any) -> list[LoopCandidate]:
def flatten_config(fmt: str, parsed: Any) -> list[tuple[tuple[str, ...], Any]]:
"""
Analyze parsed config to find loop opportunities.
Flatten parsed config into a list of (path_tuple, value).
Examples:
TOML: [server.tls] enabled = true
-> (("server", "tls", "enabled"), True)
INI: [somesection] foo = "bar"
-> (("somesection", "foo"), "bar")
For INI, values are processed as strings (quotes stripped when obvious).
"""
analyzer = LoopAnalyzer()
candidates = analyzer.analyze(parsed, fmt)
items: list[tuple[tuple[str, ...], Any]] = []
# Filter by confidence threshold
return [c for c in candidates if c.confidence >= LoopAnalyzer.MIN_CONFIDENCE]
if fmt == "toml":
def _walk(obj: Any, path: tuple[str, ...] = ()) -> None:
if isinstance(obj, dict):
for k, v in obj.items():
_walk(v, path + (str(k),))
else:
items.append((path, obj))
def flatten_config(
fmt: str, parsed: Any, loop_candidates: list[LoopCandidate] | None = None
) -> list[tuple[tuple[str, ...], Any]]:
"""
Flatten parsed config into (path, value) pairs.
_walk(parsed)
If loop_candidates is provided, paths within those loops are excluded
from flattening (they'll be handled via loops in the template).
"""
handler = _HANDLERS.get(fmt)
if handler is None:
elif fmt == "ini":
parser: configparser.ConfigParser = parsed
for section in parser.sections():
for key, value in parser.items(section, raw=True):
raw = value.strip()
# Strip surrounding quotes from INI values for defaults
if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {'"', "'"}:
processed: Any = raw[1:-1]
else:
processed = raw
items.append(((section, key), processed))
else: # pragma: no cover
raise ValueError(f"Unsupported format: {fmt}")
all_items = handler.flatten(parsed)
if not loop_candidates:
return all_items
# Build set of paths to exclude (anything under a loop path)
excluded_prefixes = {candidate.path for candidate in loop_candidates}
# Filter out items that fall under loop paths
filtered_items = []
for item_path, value in all_items:
# Check if this path starts with any loop path
is_excluded = False
for loop_path in excluded_prefixes:
if _path_starts_with(item_path, loop_path):
is_excluded = True
break
if not is_excluded:
filtered_items.append((item_path, value))
return filtered_items
return items
def _path_starts_with(path: tuple[str, ...], prefix: tuple[str, ...]) -> bool:
"""Check if path starts with prefix."""
if len(path) < len(prefix):
return False
return path[: len(prefix)] == prefix
def make_var_name(role_prefix: str, path: Iterable[str]) -> str:
"""
Build an Ansible var name like:
role_prefix_section_subsection_key
Sanitises parts to lowercase [a-z0-9_] and strips extras.
"""
role_prefix = role_prefix.strip().lower()
clean_parts: list[str] = []
for part in path:
part = str(part).strip()
part = part.replace(" ", "_")
cleaned_chars: list[str] = []
for c in part:
if c.isalnum() or c == "_":
cleaned_chars.append(c.lower())
else:
cleaned_chars.append("_")
cleaned_part = "".join(cleaned_chars).strip("_")
if cleaned_part:
clean_parts.append(cleaned_part)
if clean_parts:
return role_prefix + "_" + "_".join(clean_parts)
return role_prefix
def generate_ansible_yaml(
role_prefix: str,
flat_items: list[tuple[tuple[str, ...], Any]],
loop_candidates: list[LoopCandidate] | None = None,
def generate_defaults_yaml(
role_prefix: str, flat_items: list[tuple[tuple[str, ...], Any]]
) -> str:
"""
Create Ansible YAML for defaults/main.yml.
Create YAML for defaults/main.yml from flattened items.
"""
defaults: dict[str, Any] = {}
# Add scalar variables
for path, value in flat_items:
var_name = make_var_name(role_prefix, path)
defaults[var_name] = value # No normalization - keep original types
defaults[var_name] = value
# Add loop collections
if loop_candidates:
for candidate in loop_candidates:
var_name = make_var_name(role_prefix, candidate.path)
defaults[var_name] = candidate.items
return yaml.dump(
return yaml.safe_dump(
defaults,
Dumper=_TurtleDumper,
sort_keys=True,
default_flow_style=False,
allow_unicode=True,
explicit_start=True,
indent=2,
)
def generate_jinja2_template(
fmt: str,
parsed: Any,
role_prefix: str,
original_text: str | None = None,
loop_candidates: list[LoopCandidate] | None = None,
) -> str:
def _generate_toml_template(role_prefix: str, data: dict[str, Any]) -> str:
"""
Generate a Jinja2 template for the config.
Generate a TOML Jinja2 template from parsed TOML dict.
Values become Jinja placeholders, with quoting preserved for strings:
foo = "bar" -> foo = "{{ prefix_foo }}"
port = 8080 -> port = {{ prefix_port }}
"""
handler = _HANDLERS.get(fmt)
lines: list[str] = []
if handler is None:
raise ValueError(f"Unsupported format: {fmt}")
def emit_kv(path: tuple[str, ...], key: str, value: Any) -> None:
var_name = make_var_name(role_prefix, path + (key,))
if isinstance(value, str):
lines.append(f'{key} = "{{{{ {var_name} }}}}"')
else:
lines.append(f"{key} = {{{{ {var_name} }}}}")
# Check if handler supports loop-aware generation
if hasattr(handler, "generate_jinja2_template_with_loops") and loop_candidates:
return handler.generate_jinja2_template_with_loops(
parsed, role_prefix, original_text, loop_candidates
)
def walk(obj: dict[str, Any], path: tuple[str, ...] = ()) -> None:
scalar_items = {k: v for k, v in obj.items() if not isinstance(v, dict)}
nested_items = {k: v for k, v in obj.items() if isinstance(v, dict)}
# Fallback to original scalar-only generation
return handler.generate_jinja2_template(
parsed, role_prefix, original_text=original_text
)
if path:
header = ".".join(path)
lines.append(f"[{header}]")
for key, val in scalar_items.items():
emit_kv(path, str(key), val)
if scalar_items:
lines.append("")
for key, val in nested_items.items():
walk(val, path + (str(key),))
# Root scalars (no table header)
root_scalars = {k: v for k, v in data.items() if not isinstance(v, dict)}
for key, val in root_scalars.items():
emit_kv((), str(key), val)
if root_scalars:
lines.append("")
# Tables
for key, val in data.items():
if isinstance(val, dict):
walk(val, (str(key),))
return "\n".join(lines).rstrip() + "\n"
def _stringify_timestamps(obj: Any) -> Any:
def _generate_ini_template(role_prefix: str, parser: configparser.ConfigParser) -> str:
"""
Recursively walk a parsed config and turn any datetime/date/time objects
into plain strings in ISO-8601 form.
Generate an INI-style Jinja2 template from a ConfigParser.
This prevents Python datetime objects from leaking into YAML/Jinja, which
would otherwise reformat the value (e.g. replacing 'T' with a space).
This commonly occurs otherwise with TOML and YAML files, which sees
Python automatically convert those sorts of strings into datetime objects.
Quoting heuristic:
foo = "bar" -> foo = "{{ prefix_section_foo }}"
num = 42 -> num = {{ prefix_section_num }}
"""
if isinstance(obj, dict):
return {k: _stringify_timestamps(v) for k, v in obj.items()}
if isinstance(obj, list):
return [_stringify_timestamps(v) for v in obj]
lines: list[str] = []
# TOML & YAML both use the standard datetime types
if isinstance(obj, datetime.datetime):
# Use default ISO-8601: 'YYYY-MM-DDTHH:MM:SS±HH:MM' (with 'T')
return obj.isoformat()
if isinstance(obj, (datetime.date, datetime.time)):
return obj.isoformat()
for section in parser.sections():
lines.append(f"[{section}]")
for key, value in parser.items(section, raw=True):
path = (section, key)
var_name = make_var_name(role_prefix, path)
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}:
lines.append(f'{key} = "{{{{ {var_name} }}}}"')
else:
lines.append(f"{key} = {{{{ {var_name} }}}}")
lines.append("")
return obj
return "\n".join(lines).rstrip() + "\n"
def generate_template(fmt: str, parsed: Any, role_prefix: str) -> str:
"""
Dispatch to the appropriate template generator.
"""
if fmt == "toml":
if not isinstance(parsed, dict):
raise TypeError("TOML parser result must be a dict")
return _generate_toml_template(role_prefix, parsed)
if fmt == "ini":
if not isinstance(parsed, configparser.ConfigParser):
raise TypeError("INI parser result must be a ConfigParser")
return _generate_ini_template(role_prefix, parsed)
raise ValueError(f"Unsupported format: {fmt}")

19
src/jinjaturtle/handlers/__init__.py
View file

@ -1,19 +0,0 @@
from __future__ import annotations
from .base import BaseHandler
from .dict import DictLikeHandler
from .ini import IniHandler
from .json import JsonHandler
from .toml import TomlHandler
from .yaml import YamlHandler
from .xml import XmlHandler
__all__ = [
"BaseHandler",
"DictLikeHandler",
"IniHandler",
"JsonHandler",
"TomlHandler",
"YamlHandler",
"XmlHandler",
]

79
src/jinjaturtle/handlers/base.py
View file

@ -1,79 +0,0 @@
from __future__ import annotations
from pathlib import Path
from typing import Any, Iterable
class BaseHandler:
"""
Base class for a config format handler.
Each handler is responsible for:
- parse(path) -> parsed object
- flatten(parsed) -> list[(path_tuple, value)]
- generate_jinja2_template(parsed, role_prefix, original_text=None) -> str
"""
fmt: str # e.g. "ini", "yaml", ...
def parse(self, path: Path) -> Any:
raise NotImplementedError
def flatten(self, parsed: Any) -> list[tuple[tuple[str, ...], Any]]:
raise NotImplementedError
def generate_jinja2_template(
self,
parsed: Any,
role_prefix: str,
original_text: str | None = None,
) -> str:
raise NotImplementedError
def _split_inline_comment(
self, text: str, comment_chars: set[str]
) -> tuple[str, str]:
"""
Split 'value # comment' into (value_part, comment_part), where
comment_part starts at the first unquoted comment character.
comment_chars is e.g. {'#'} for TOML/YAML, {'#', ';'} for INI.
"""
in_single = False
in_double = False
for i, ch in enumerate(text):
if ch == "'" and not in_double:
in_single = not in_single
elif ch == '"' and not in_single:
in_double = not in_double
elif ch in comment_chars and not in_single and not in_double:
return text[:i], text[i:]
return text, ""
@staticmethod
def make_var_name(role_prefix: str, path: Iterable[str]) -> str:
"""
Build an Ansible var name like:
role_prefix_section_subsection_key
Sanitises parts to lowercase [a-z0-9_] and strips extras.
"""
role_prefix = role_prefix.strip().lower()
clean_parts: list[str] = []
for part in path:
part = str(part).strip()
part = part.replace(" ", "_")
cleaned_chars: list[str] = []
for c in part:
if c.isalnum() or c == "_":
cleaned_chars.append(c.lower())
else:
cleaned_chars.append("_")
cleaned_part = "".join(cleaned_chars).strip("_")
if cleaned_part:
clean_parts.append(cleaned_part)
if clean_parts:
return role_prefix + "_" + "_".join(clean_parts)
return role_prefix

31
src/jinjaturtle/handlers/dict.py
View file

@ -1,31 +0,0 @@
from __future__ import annotations
from typing import Any
from . import BaseHandler
class DictLikeHandler(BaseHandler):
"""
Base for TOML/YAML/JSON: nested dict/list structures.
Subclasses control whether lists are flattened.
"""
flatten_lists: bool = False # override in subclasses
def flatten(self, parsed: Any) -> list[tuple[tuple[str, ...], Any]]:
items: list[tuple[tuple[str, ...], Any]] = []
def _walk(obj: Any, path: tuple[str, ...] = ()) -> None:
if isinstance(obj, dict):
for k, v in obj.items():
_walk(v, path + (str(k),))
elif isinstance(obj, list) and self.flatten_lists:
for i, v in enumerate(obj):
_walk(v, path + (str(i),))
else:
items.append((path, obj))
_walk(parsed)
return items

153
src/jinjaturtle/handlers/ini.py
View file

@ -1,153 +0,0 @@
from __future__ import annotations
import configparser
from pathlib import Path
from typing import Any
from . import BaseHandler
class IniHandler(BaseHandler):
fmt = "ini"
def parse(self, path: Path) -> configparser.ConfigParser:
parser = configparser.ConfigParser()
parser.optionxform = str # noqa
with path.open("r", encoding="utf-8") as f:
parser.read_file(f)
return parser
def flatten(self, parsed: Any) -> list[tuple[tuple[str, ...], Any]]:
if not isinstance(parsed, configparser.ConfigParser):
raise TypeError("INI parser result must be a ConfigParser")
parser: configparser.ConfigParser = parsed
items: list[tuple[tuple[str, ...], Any]] = []
for section in parser.sections():
for key, value in parser.items(section, raw=True):
raw = value.strip()
if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {'"', "'"}:
processed: Any = raw[1:-1]
else:
processed = raw
items.append(((section, key), processed))
return items
def generate_jinja2_template(
self,
parsed: Any,
role_prefix: str,
original_text: str | None = None,
) -> str:
if original_text is not None:
return self._generate_ini_template_from_text(role_prefix, original_text)
if not isinstance(parsed, configparser.ConfigParser):
raise TypeError("INI parser result must be a ConfigParser")
return self._generate_ini_template(role_prefix, parsed)
def _generate_ini_template(
self, role_prefix: str, parser: configparser.ConfigParser
) -> str:
"""
Generate an INI-style Jinja2 template from a ConfigParser.
Quoting heuristic:
foo = "bar" -> foo = "{{ prefix_section_foo }}"
num = 42 -> num = {{ prefix_section_num }}
"""
lines: list[str] = []
for section in parser.sections():
lines.append(f"[{section}]")
for key, value in parser.items(section, raw=True):
path = (section, key)
var_name = self.make_var_name(role_prefix, path)
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}:
lines.append(f'{key} = "{{{{ {var_name} }}}}"')
else:
lines.append(f"{key} = {{{{ {var_name} }}}}")
lines.append("")
return "\n".join(lines).rstrip() + "\n"
def _generate_ini_template_from_text(self, role_prefix: str, text: str) -> str:
"""
Generate a Jinja2 template for an INI/php.ini-style file, preserving
comments, blank lines, and section headers by patching values in-place.
"""
lines = text.splitlines(keepends=True)
current_section: str | None = None
out_lines: list[str] = []
for raw_line in lines:
line = raw_line
stripped = line.lstrip()
# Blank or pure comment: keep as-is
if not stripped or stripped[0] in {"#", ";"}:
out_lines.append(raw_line)
continue
# Section header
if stripped.startswith("[") and "]" in stripped:
header_inner = stripped[1 : stripped.index("]")]
current_section = header_inner.strip()
out_lines.append(raw_line)
continue
# Work without newline so we can re-attach it exactly
newline = ""
content = raw_line
if content.endswith("\r\n"):
newline = "\r\n"
content = content[:-2]
elif content.endswith("\n"):
newline = content[-1]
content = content[:-1]
eq_index = content.find("=")
if eq_index == -1:
# Not a simple key=value line: leave untouched
out_lines.append(raw_line)
continue
before_eq = content[:eq_index]
after_eq = content[eq_index + 1 :]
key = before_eq.strip()
if not key:
out_lines.append(raw_line)
continue
# Whitespace after '='
value_ws_len = len(after_eq) - len(after_eq.lstrip(" \t"))
leading_ws = after_eq[:value_ws_len]
value_and_comment = after_eq[value_ws_len:]
value_part, comment_part = self._split_inline_comment(
value_and_comment, {"#", ";"}
)
raw_value = value_part.strip()
path = (key,) if current_section is None else (current_section, key)
var_name = self.make_var_name(role_prefix, path)
# Was the original value quoted?
use_quotes = (
len(raw_value) >= 2
and raw_value[0] == raw_value[-1]
and raw_value[0] in {'"', "'"}
)
if use_quotes:
quote_char = raw_value[0]
replacement_value = f"{quote_char}{{{{ {var_name} }}}}{quote_char}"
else:
replacement_value = f"{{{{ {var_name} }}}}"
new_content = (
before_eq + "=" + leading_ws + replacement_value + comment_part
)
out_lines.append(new_content + newline)
return "".join(out_lines)

191
src/jinjaturtle/handlers/json.py
View file

@ -1,191 +0,0 @@
from __future__ import annotations
import json
import re
from pathlib import Path
from typing import Any
from . import DictLikeHandler
from ..loop_analyzer import LoopCandidate
class JsonHandler(DictLikeHandler):
fmt = "json"
flatten_lists = True
def parse(self, path: Path) -> Any:
with path.open("r", encoding="utf-8") as f:
return json.load(f)
def generate_jinja2_template(
self,
parsed: Any,
role_prefix: str,
original_text: str | None = None,
) -> str:
"""Original scalar-only template generation."""
if not isinstance(parsed, (dict, list)):
raise TypeError("JSON parser result must be a dict or list")
# As before: ignore original_text and rebuild structurally
return self._generate_json_template(role_prefix, parsed)
def generate_jinja2_template_with_loops(
self,
parsed: Any,
role_prefix: str,
original_text: str | None,
loop_candidates: list[LoopCandidate],
) -> str:
"""Generate template with Jinja2 for loops where appropriate."""
if not isinstance(parsed, (dict, list)):
raise TypeError("JSON parser result must be a dict or list")
# Build loop path set for quick lookup
loop_paths = {candidate.path for candidate in loop_candidates}
return self._generate_json_template_with_loops(
role_prefix, parsed, loop_paths, loop_candidates
)
def _generate_json_template(self, role_prefix: str, data: Any) -> str:
"""
Generate a JSON Jinja2 template from parsed JSON data.
All scalar values are replaced with Jinja expressions whose names are
derived from the path, similar to TOML/YAML.
Uses | tojson filter to preserve types (numbers, booleans, null).
"""
def _walk(obj: Any, path: tuple[str, ...] = ()) -> Any:
if isinstance(obj, dict):
return {k: _walk(v, path + (str(k),)) for k, v in obj.items()}
if isinstance(obj, list):
return [_walk(v, path + (str(i),)) for i, v in enumerate(obj)]
# scalar - use marker that will be replaced with tojson
var_name = self.make_var_name(role_prefix, path)
return f"__SCALAR__{var_name}__"
templated = _walk(data)
json_str = json.dumps(templated, indent=2, ensure_ascii=False)
# Replace scalar markers with Jinja expressions using tojson filter
# This preserves types (numbers stay numbers, booleans stay booleans)
json_str = re.sub(
r'"__SCALAR__([a-zA-Z_][a-zA-Z0-9_]*)__"', r"{{ \1 | tojson }}", json_str
)
return json_str + "\n"
def _generate_json_template_with_loops(
self,
role_prefix: str,
data: Any,
loop_paths: set[tuple[str, ...]],
loop_candidates: list[LoopCandidate],
path: tuple[str, ...] = (),
) -> str:
"""
Generate a JSON Jinja2 template with for loops where appropriate.
"""
def _walk(obj: Any, current_path: tuple[str, ...] = ()) -> Any:
# Check if this path is a loop candidate
if current_path in loop_paths:
# Find the matching candidate
candidate = next(c for c in loop_candidates if c.path == current_path)
collection_var = self.make_var_name(role_prefix, candidate.path)
item_var = candidate.loop_var
if candidate.item_schema == "scalar":
# Simple list of scalars - use special marker that we'll replace
return f"__LOOP_SCALAR__{collection_var}__{item_var}__"
elif candidate.item_schema in ("simple_dict", "nested"):
# List of dicts - use special marker
return f"__LOOP_DICT__{collection_var}__{item_var}__"
if isinstance(obj, dict):
return {k: _walk(v, current_path + (str(k),)) for k, v in obj.items()}
if isinstance(obj, list):
# Check if this list is a loop candidate
if current_path in loop_paths:
# Already handled above
return _walk(obj, current_path)
return [_walk(v, current_path + (str(i),)) for i, v in enumerate(obj)]
# scalar - use marker to preserve type
var_name = self.make_var_name(role_prefix, current_path)
return f"__SCALAR__{var_name}__"
templated = _walk(data, path)
# Convert to JSON string
json_str = json.dumps(templated, indent=2, ensure_ascii=False)
# Replace scalar markers with Jinja expressions using tojson filter
json_str = re.sub(
r'"__SCALAR__([a-zA-Z_][a-zA-Z0-9_]*)__"', r"{{ \1 | tojson }}", json_str
)
# Post-process to replace loop markers with actual Jinja loops
for candidate in loop_candidates:
collection_var = self.make_var_name(role_prefix, candidate.path)
item_var = candidate.loop_var
if candidate.item_schema == "scalar":
# Replace scalar loop marker with Jinja for loop
marker = f'"__LOOP_SCALAR__{collection_var}__{item_var}__"'
replacement = self._generate_json_scalar_loop(
collection_var, item_var, candidate
)
json_str = json_str.replace(marker, replacement)
elif candidate.item_schema in ("simple_dict", "nested"):
# Replace dict loop marker with Jinja for loop
marker = f'"__LOOP_DICT__{collection_var}__{item_var}__"'
replacement = self._generate_json_dict_loop(
collection_var, item_var, candidate
)
json_str = json_str.replace(marker, replacement)
return json_str + "\n"
def _generate_json_scalar_loop(
self, collection_var: str, item_var: str, candidate: LoopCandidate
) -> str:
"""Generate a Jinja for loop for a scalar list in JSON."""
# Use tojson filter to properly handle strings (quotes them) and other types
# Include array brackets around the loop
return (
f"[{{% for {item_var} in {collection_var} %}}"
f"{{{{ {item_var} | tojson }}}}"
f"{{% if not loop.last %}}, {{% endif %}}"
f"{{% endfor %}}]"
)
def _generate_json_dict_loop(
self, collection_var: str, item_var: str, candidate: LoopCandidate
) -> str:
"""Generate a Jinja for loop for a dict list in JSON."""
if not candidate.items:
return "[]"
# Get first item as template
sample_item = candidate.items[0]
# Build the dict template - use tojson for all values to handle types correctly
fields = []
for key, value in sample_item.items():
if key == "_key":
continue
# Use tojson filter to properly serialize all types (strings, numbers, booleans)
fields.append(f'"{key}": {{{{ {item_var}.{key} | tojson }}}}')
dict_template = "{" + ", ".join(fields) + "}"
return (
f"{{% for {item_var} in {collection_var} %}}"
f"{dict_template}"
f"{{% if not loop.last %}}, {{% endif %}}"
f"{{% endfor %}}"
)

547
src/jinjaturtle/handlers/toml.py
View file

@ -1,547 +0,0 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
from . import DictLikeHandler
from ..loop_analyzer import LoopCandidate
try:
import tomllib
except Exception:
import tomli as tomllib
class TomlHandler(DictLikeHandler):
fmt = "toml"
flatten_lists = False # keep lists as scalars
def parse(self, path: Path) -> Any:
if tomllib is None:
raise RuntimeError(
"tomllib/tomli is required to parse TOML files but is not installed"
)
with path.open("rb") as f:
return tomllib.load(f)
def generate_jinja2_template(
self,
parsed: Any,
role_prefix: str,
original_text: str | None = None,
) -> str:
"""Original scalar-only template generation."""
if original_text is not None:
return self._generate_toml_template_from_text(role_prefix, original_text)
if not isinstance(parsed, dict):
raise TypeError("TOML parser result must be a dict")
return self._generate_toml_template(role_prefix, parsed)
def generate_jinja2_template_with_loops(
self,
parsed: Any,
role_prefix: str,
original_text: str | None,
loop_candidates: list[LoopCandidate],
) -> str:
"""Generate template with Jinja2 for loops where appropriate."""
if original_text is not None:
return self._generate_toml_template_with_loops_from_text(
role_prefix, original_text, loop_candidates
)
if not isinstance(parsed, dict):
raise TypeError("TOML parser result must be a dict")
return self._generate_toml_template_with_loops(
role_prefix, parsed, loop_candidates
)
def _generate_toml_template(self, role_prefix: str, data: dict[str, Any]) -> str:
"""
Generate a TOML Jinja2 template from parsed TOML dict.
Values become Jinja placeholders, with quoting preserved for strings:
foo = "bar" -> foo = "{{ prefix_foo }}"
port = 8080 -> port = {{ prefix_port }}
"""
lines: list[str] = []
def emit_kv(path: tuple[str, ...], key: str, value: Any) -> None:
var_name = self.make_var_name(role_prefix, path + (key,))
if isinstance(value, str):
lines.append(f'{key} = "{{{{ {var_name} }}}}"')
elif isinstance(value, bool):
# Booleans need | lower filter (Python True/False → TOML true/false)
lines.append(f"{key} = {{{{ {var_name} | lower }}}}")
else:
lines.append(f"{key} = {{{{ {var_name} }}}}")
def walk(obj: dict[str, Any], path: tuple[str, ...] = ()) -> None:
scalar_items = {k: v for k, v in obj.items() if not isinstance(v, dict)}
nested_items = {k: v for k, v in obj.items() if isinstance(v, dict)}
if path:
header = ".".join(path)
lines.append(f"[{header}]")
for key, val in scalar_items.items():
emit_kv(path, str(key), val)
if scalar_items:
lines.append("")
for key, val in nested_items.items():
walk(val, path + (str(key),))
# Root scalars (no table header)
root_scalars = {k: v for k, v in data.items() if not isinstance(v, dict)}
for key, val in root_scalars.items():
emit_kv((), str(key), val)
if root_scalars:
lines.append("")
# Tables
for key, val in data.items():
if isinstance(val, dict):
walk(val, (str(key),))
return "\n".join(lines).rstrip() + "\n"
def _generate_toml_template_with_loops(
self,
role_prefix: str,
data: dict[str, Any],
loop_candidates: list[LoopCandidate],
) -> str:
"""
Generate a TOML Jinja2 template with for loops where appropriate.
"""
lines: list[str] = []
loop_paths = {candidate.path for candidate in loop_candidates}
def emit_kv(path: tuple[str, ...], key: str, value: Any) -> None:
var_name = self.make_var_name(role_prefix, path + (key,))
if isinstance(value, str):
lines.append(f'{key} = "{{{{ {var_name} }}}}"')
elif isinstance(value, bool):
# Booleans need | lower filter (Python True/False → TOML true/false)
lines.append(f"{key} = {{{{ {var_name} | lower }}}}")
elif isinstance(value, list):
# Check if this list is a loop candidate
if path + (key,) in loop_paths:
# Find the matching candidate
candidate = next(
c for c in loop_candidates if c.path == path + (key,)
)
collection_var = self.make_var_name(role_prefix, candidate.path)
item_var = candidate.loop_var
if candidate.item_schema == "scalar":
# Scalar list loop
lines.append(
f"{key} = ["
f"{{% for {item_var} in {collection_var} %}}"
f"{{{{ {item_var} }}}}"
f"{{% if not loop.last %}}, {{% endif %}}"
f"{{% endfor %}}"
f"]"
)
elif candidate.item_schema in ("simple_dict", "nested"):
# Dict list loop - TOML array of tables
# This is complex for TOML, using simplified approach
lines.append(f"{key} = {{{{ {var_name} | tojson }}}}")
else:
# Not a loop, treat as regular variable
lines.append(f"{key} = {{{{ {var_name} }}}}")
else:
lines.append(f"{key} = {{{{ {var_name} }}}}")
def walk(obj: dict[str, Any], path: tuple[str, ...] = ()) -> None:
scalar_items = {k: v for k, v in obj.items() if not isinstance(v, dict)}
nested_items = {k: v for k, v in obj.items() if isinstance(v, dict)}
if path:
header = ".".join(path)
lines.append(f"[{header}]")
for key, val in scalar_items.items():
emit_kv(path, str(key), val)
if scalar_items:
lines.append("")
for key, val in nested_items.items():
walk(val, path + (str(key),))
# Root scalars (no table header)
root_scalars = {k: v for k, v in data.items() if not isinstance(v, dict)}
for key, val in root_scalars.items():
emit_kv((), str(key), val)
if root_scalars:
lines.append("")
# Tables
for key, val in data.items():
if isinstance(val, dict):
walk(val, (str(key),))
return "\n".join(lines).rstrip() + "\n"
def _generate_toml_template_from_text(self, role_prefix: str, text: str) -> str:
"""
Generate a Jinja2 template for a TOML file, preserving comments,
blank lines, and table headers by patching values in-place.
Handles inline tables like:
temp_targets = { cpu = 79.5, case = 72.0 }
by mapping them to:
temp_targets = { cpu = {{ prefix_database_temp_targets_cpu }},
case = {{ prefix_database_temp_targets_case }} }
"""
lines = text.splitlines(keepends=True)
current_table: tuple[str, ...] = ()
out_lines: list[str] = []
for raw_line in lines:
line = raw_line
stripped = line.lstrip()
# Blank or pure comment
if not stripped or stripped.startswith("#"):
out_lines.append(raw_line)
continue
# Table header: [server] or [server.tls] or [[array.of.tables]]
if stripped.startswith("[") and "]" in stripped:
header = stripped
first_bracket = header.find("[")
closing_bracket = header.find("]", first_bracket + 1)
if first_bracket != -1 and closing_bracket != -1:
inner = header[first_bracket + 1 : closing_bracket].strip()
inner = inner.strip("[]") # handle [[table]] as well
parts = [p.strip() for p in inner.split(".") if p.strip()]
current_table = tuple(parts)
out_lines.append(raw_line)
continue
# Try key = value
newline = ""
content = raw_line
if content.endswith("\r\n"):
newline = "\r\n"
content = content[:-2]
elif content.endswith("\n"):
newline = content[-1]
content = content[:-1]
eq_index = content.find("=")
if eq_index == -1:
out_lines.append(raw_line)
continue
before_eq = content[:eq_index]
after_eq = content[eq_index + 1 :]
key = before_eq.strip()
if not key:
out_lines.append(raw_line)
continue
# Whitespace after '='
value_ws_len = len(after_eq) - len(after_eq.lstrip(" \t"))
leading_ws = after_eq[:value_ws_len]
value_and_comment = after_eq[value_ws_len:]
value_part, comment_part = self._split_inline_comment(
value_and_comment, {"#"}
)
raw_value = value_part.strip()
# Path for this key (table + key)
path = current_table + (key,)
# Special case: inline table
if (
raw_value.startswith("{")
and raw_value.endswith("}")
and tomllib is not None
):
try:
# Parse the inline table as a tiny TOML document
mini_source = "table = " + raw_value + "\n"
mini_data = tomllib.loads(mini_source)["table"]
except Exception:
mini_data = None
if isinstance(mini_data, dict):
inner_bits: list[str] = []
for sub_key, sub_val in mini_data.items():
nested_path = path + (sub_key,)
nested_var = self.make_var_name(role_prefix, nested_path)
if isinstance(sub_val, str):
inner_bits.append(f'{sub_key} = "{{{{ {nested_var} }}}}"')
elif isinstance(sub_val, bool):
inner_bits.append(
f"{sub_key} = {{{{ {nested_var} | lower }}}}"
)
else:
inner_bits.append(f"{sub_key} = {{{ {nested_var} }}}")
replacement_value = "{ " + ", ".join(inner_bits) + " }"
new_content = (
before_eq + "=" + leading_ws + replacement_value + comment_part
)
out_lines.append(new_content + newline)
continue
# If parsing fails, fall through to normal handling
# Normal scalar value handling (including bools, numbers, strings)
var_name = self.make_var_name(role_prefix, path)
use_quotes = (
len(raw_value) >= 2
and raw_value[0] == raw_value[-1]
and raw_value[0] in {'"', "'"}
)
# Check if value is a boolean in the text
is_bool = raw_value.strip().lower() in ("true", "false")
if use_quotes:
quote_char = raw_value[0]
replacement_value = f"{quote_char}{{{{ {var_name} }}}}{quote_char}"
elif is_bool:
replacement_value = f"{{{{ {var_name} | lower }}}}"
else:
replacement_value = f"{{{{ {var_name} }}}}"
new_content = (
before_eq + "=" + leading_ws + replacement_value + comment_part
)
out_lines.append(new_content + newline)
return "".join(out_lines)
def _generate_toml_template_with_loops_from_text(
self, role_prefix: str, text: str, loop_candidates: list[LoopCandidate]
) -> str:
"""
Generate a Jinja2 template for a TOML file with loop support.
"""
loop_paths = {candidate.path for candidate in loop_candidates}
lines = text.splitlines(keepends=True)
current_table: tuple[str, ...] = ()
out_lines: list[str] = []
skip_until_next_table = (
False # Track when we're inside a looped array-of-tables
)
for raw_line in lines:
line = raw_line
stripped = line.lstrip()
# Blank or pure comment
if not stripped or stripped.startswith("#"):
# Only output if we're not skipping
if not skip_until_next_table:
out_lines.append(raw_line)
continue
# Table header: [server] or [server.tls] or [[array.of.tables]]
if stripped.startswith("[") and "]" in stripped:
header = stripped
# Check if it's array-of-tables ([[name]]) or regular table ([name])
is_array_table = header.startswith("[[") and "]]" in header
if is_array_table:
# Extract content between [[ and ]]
start = header.find("[[") + 2
end = header.find("]]", start)
inner = header[start:end].strip() if end != -1 else ""
else:
# Extract content between [ and ]
start = header.find("[") + 1
end = header.find("]", start)
inner = header[start:end].strip() if end != -1 else ""
if inner:
parts = [p.strip() for p in inner.split(".") if p.strip()]
table_path = tuple(parts)
# Check if this is an array-of-tables that's a loop candidate
if is_array_table and table_path in loop_paths:
# If we're already skipping this table, this is a subsequent occurrence
if skip_until_next_table and current_table == table_path:
# This is a duplicate [[table]] - skip it
continue
# This is the first occurrence - generate the loop
current_table = table_path
candidate = next(
c for c in loop_candidates if c.path == table_path
)
# Generate the loop header
collection_var = self.make_var_name(role_prefix, candidate.path)
item_var = candidate.loop_var
# Get sample item to build template
if candidate.items:
sample_item = candidate.items[0]
# Build loop
out_lines.append(
f"{{% for {item_var} in {collection_var} %}}\n"
)
out_lines.append(f"[[{'.'.join(table_path)}]]\n")
# Add fields from sample item
for key, value in sample_item.items():
if key == "_key":
continue
if isinstance(value, str):
out_lines.append(
f'{key} = "{{{{ {item_var}.{key} }}}}"\n'
)
else:
out_lines.append(
f"{key} = {{{{ {item_var}.{key} }}}}\n"
)
out_lines.append("{% endfor %}\n")
# Skip all content until the next different table
skip_until_next_table = True
continue
else:
# Regular table or non-loop array - reset skip flag if it's a different table
if current_table != table_path:
skip_until_next_table = False
current_table = table_path
out_lines.append(raw_line)
continue
# If we're inside a skipped array-of-tables section, skip this line
if skip_until_next_table:
continue
# Try key = value
newline = ""
content = raw_line
if content.endswith("\r\n"):
newline = "\r\n"
content = content[:-2]
elif content.endswith("\n"):
newline = content[-1]
content = content[:-1]
eq_index = content.find("=")
if eq_index == -1:
out_lines.append(raw_line)
continue
before_eq = content[:eq_index]
after_eq = content[eq_index + 1 :]
key = before_eq.strip()
if not key:
out_lines.append(raw_line)
continue
# Whitespace after '='
value_ws_len = len(after_eq) - len(after_eq.lstrip(" \t"))
leading_ws = after_eq[:value_ws_len]
value_and_comment = after_eq[value_ws_len:]
value_part, comment_part = self._split_inline_comment(
value_and_comment, {"#"}
)
raw_value = value_part.strip()
# Path for this key (table + key)
path = current_table + (key,)
# Check if this path is a loop candidate
if path in loop_paths:
candidate = next(c for c in loop_candidates if c.path == path)
collection_var = self.make_var_name(role_prefix, candidate.path)
item_var = candidate.loop_var
if candidate.item_schema == "scalar":
# Scalar list loop
replacement_value = (
f"["
f"{{% for {item_var} in {collection_var} %}}"
f"{{{{ {item_var} }}}}"
f"{{% if not loop.last %}}, {{% endif %}}"
f"{{% endfor %}}"
f"]"
)
else:
# Dict/nested loop - use tojson filter for complex arrays
replacement_value = f"{{{{ {collection_var} | tojson }}}}"
new_content = (
before_eq + "=" + leading_ws + replacement_value + comment_part
)
out_lines.append(new_content + newline)
continue
# Special case: inline table
if (
raw_value.startswith("{")
and raw_value.endswith("}")
and tomllib is not None
):
try:
# Parse the inline table as a tiny TOML document
mini_source = "table = " + raw_value + "\n"
mini_data = tomllib.loads(mini_source)["table"]
except Exception:
mini_data = None
if isinstance(mini_data, dict):
inner_bits: list[str] = []
for sub_key, sub_val in mini_data.items():
nested_path = path + (sub_key,)
nested_var = self.make_var_name(role_prefix, nested_path)
if isinstance(sub_val, str):
inner_bits.append(f'{sub_key} = "{{{{ {nested_var} }}}}"')
elif isinstance(sub_val, bool):
inner_bits.append(
f"{sub_key} = {{{{ {nested_var} | lower }}}}"
)
else:
inner_bits.append(f"{sub_key} = {{{{ {nested_var} }}}}")
replacement_value = "{ " + ", ".join(inner_bits) + " }"
new_content = (
before_eq + "=" + leading_ws + replacement_value + comment_part
)
out_lines.append(new_content + newline)
continue
# If parsing fails, fall through to normal handling
# Normal scalar value handling (including bools, numbers, strings)
var_name = self.make_var_name(role_prefix, path)
use_quotes = (
len(raw_value) >= 2
and raw_value[0] == raw_value[-1]
and raw_value[0] in {'"', "'"}
)
# Check if value is a boolean in the text
is_bool = raw_value.strip().lower() in ("true", "false")
if use_quotes:
quote_char = raw_value[0]
replacement_value = f"{quote_char}{{{{ {var_name} }}}}{quote_char}"
elif is_bool:
replacement_value = f"{{{{ {var_name} | lower }}}}"
else:
replacement_value = f"{{{{ {var_name} }}}}"
new_content = (
before_eq + "=" + leading_ws + replacement_value + comment_part
)
out_lines.append(new_content + newline)
return "".join(out_lines)

459
src/jinjaturtle/handlers/xml.py
View file

@ -1,459 +0,0 @@
from __future__ import annotations
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any
import xml.etree.ElementTree as ET # nosec
from .base import BaseHandler
from ..loop_analyzer import LoopCandidate
class XmlHandler(BaseHandler):
"""
XML handler that can generate both scalar templates and loop-based templates.
"""
fmt = "xml"
def parse(self, path: Path) -> ET.Element:
text = path.read_text(encoding="utf-8")
parser = ET.XMLParser(
target=ET.TreeBuilder(insert_comments=False)
) # nosec B314
parser.feed(text)
root = parser.close()
return root
def flatten(self, parsed: Any) -> list[tuple[tuple[str, ...], Any]]:
if not isinstance(parsed, ET.Element):
raise TypeError("XML parser result must be an Element")
return self._flatten_xml(parsed)
def generate_jinja2_template(
self,
parsed: Any,
role_prefix: str,
original_text: str | None = None,
) -> str:
"""Original scalar-only template generation."""
if original_text is not None:
return self._generate_xml_template_from_text(role_prefix, original_text)
if not isinstance(parsed, ET.Element):
raise TypeError("XML parser result must be an Element")
xml_str = ET.tostring(parsed, encoding="unicode")
return self._generate_xml_template_from_text(role_prefix, xml_str)
def generate_jinja2_template_with_loops(
self,
parsed: Any,
role_prefix: str,
original_text: str | None,
loop_candidates: list[LoopCandidate],
) -> str:
"""Generate template with Jinja2 for loops where appropriate."""
if original_text is not None:
return self._generate_xml_template_with_loops_from_text(
role_prefix, original_text, loop_candidates
)
if not isinstance(parsed, ET.Element):
raise TypeError("XML parser result must be an Element")
xml_str = ET.tostring(parsed, encoding="unicode")
return self._generate_xml_template_with_loops_from_text(
role_prefix, xml_str, loop_candidates
)
def _flatten_xml(self, root: ET.Element) -> list[tuple[tuple[str, ...], Any]]:
"""Flatten an XML tree into (path, value) pairs."""
items: list[tuple[tuple[str, ...], Any]] = []
def walk(elem: ET.Element, path: tuple[str, ...]) -> None:
# Attributes
for attr_name, attr_val in elem.attrib.items():
attr_path = path + (f"@{attr_name}",)
items.append((attr_path, attr_val))
# Children
children = [c for c in list(elem) if isinstance(c.tag, str)]
# Text content
text = (elem.text or "").strip()
if text:
if not elem.attrib and not children:
items.append((path, text))
else:
items.append((path + ("value",), text))
# Repeated siblings get an index; singletons just use the tag
counts = Counter(child.tag for child in children)
index_counters: dict[str, int] = defaultdict(int)
for child in children:
tag = child.tag
if counts[tag] > 1:
idx = index_counters[tag]
index_counters[tag] += 1
child_path = path + (tag, str(idx))
else:
child_path = path + (tag,)
walk(child, child_path)
walk(root, ())
return items
def _split_xml_prolog(self, text: str) -> tuple[str, str]:
"""Split XML into (prolog, body)."""
i = 0
n = len(text)
prolog_parts: list[str] = []
while i < n:
while i < n and text[i].isspace():
prolog_parts.append(text[i])
i += 1
if i >= n:
break
if text.startswith("<?", i):
end = text.find("?>", i + 2)
if end == -1:
break
prolog_parts.append(text[i : end + 2])
i = end + 2
continue
if text.startswith("<!--", i):
end = text.find("-->", i + 4)
if end == -1:
break
prolog_parts.append(text[i : end + 3])
i = end + 3
continue
if text.startswith("<!DOCTYPE", i):
end = text.find(">", i + 9)
if end == -1:
break
prolog_parts.append(text[i : end + 1])
i = end + 1
continue
if text[i] == "<":
break
break
return "".join(prolog_parts), text[i:]
def _apply_jinja_to_xml_tree(
self,
role_prefix: str,
root: ET.Element,
loop_candidates: list[LoopCandidate] | None = None,
) -> None:
"""
Mutate XML tree in-place, replacing values with Jinja expressions.
If loop_candidates is provided, repeated elements matching a candidate
will be replaced with a {% for %} loop.
"""
# Build a map of loop paths for quick lookup
loop_paths = {}
if loop_candidates:
for candidate in loop_candidates:
loop_paths[candidate.path] = candidate
def walk(elem: ET.Element, path: tuple[str, ...]) -> None:
# Attributes (unless this element is in a loop)
for attr_name in list(elem.attrib.keys()):
attr_path = path + (f"@{attr_name}",)
var_name = self.make_var_name(role_prefix, attr_path)
elem.set(attr_name, f"{{{{ {var_name} }}}}")
# Children
children = [c for c in list(elem) if isinstance(c.tag, str)]
# Text content
text = (elem.text or "").strip()
if text:
if not elem.attrib and not children:
text_path = path
else:
text_path = path + ("value",)
var_name = self.make_var_name(role_prefix, text_path)
elem.text = f"{{{{ {var_name} }}}}"
# Handle children - check for loops first
counts = Counter(child.tag for child in children)
index_counters: dict[str, int] = defaultdict(int)
# Check each tag to see if it's a loop candidate
processed_tags = set()
for child in children:
tag = child.tag
# Skip if we've already processed this tag as a loop
if tag in processed_tags:
continue
child_path = path + (tag,)
# Check if this is a loop candidate
if child_path in loop_paths:
# Mark this tag as processed
processed_tags.add(tag)
# Remove all children with this tag
for child_to_remove in [c for c in children if c.tag == tag]:
elem.remove(child_to_remove)
# Create a loop comment/marker
# We'll handle the actual loop generation in text processing
loop_marker = ET.Comment(f"LOOP:{tag}")
elem.append(loop_marker)
elif counts[tag] > 1:
# Multiple children but not a loop candidate - use indexed paths
idx = index_counters[tag]
index_counters[tag] += 1
indexed_path = path + (tag, str(idx))
walk(child, indexed_path)
else:
# Single child
walk(child, child_path)
walk(root, ())
def _generate_xml_template_from_text(self, role_prefix: str, text: str) -> str:
"""Generate scalar-only Jinja2 template."""
prolog, body = self._split_xml_prolog(text)
parser = ET.XMLParser(target=ET.TreeBuilder(insert_comments=True)) # nosec B314
parser.feed(body)
root = parser.close()
self._apply_jinja_to_xml_tree(role_prefix, root)
indent = getattr(ET, "indent", None)
if indent is not None:
indent(root, space=" ") # type: ignore[arg-type]
xml_body = ET.tostring(root, encoding="unicode")
return prolog + xml_body
def _generate_xml_template_with_loops_from_text(
self,
role_prefix: str,
text: str,
loop_candidates: list[LoopCandidate],
) -> str:
"""Generate Jinja2 template with for loops."""
prolog, body = self._split_xml_prolog(text)
# Parse with comments preserved
parser = ET.XMLParser(target=ET.TreeBuilder(insert_comments=True)) # nosec B314
parser.feed(body)
root = parser.close()
# Apply Jinja transformations (including loop markers)
self._apply_jinja_to_xml_tree(role_prefix, root, loop_candidates)
# Convert to string
indent = getattr(ET, "indent", None)
if indent is not None:
indent(root, space=" ") # type: ignore[arg-type]
xml_body = ET.tostring(root, encoding="unicode")
# Post-process to replace loop markers with actual Jinja loops
xml_body = self._insert_xml_loops(xml_body, role_prefix, loop_candidates, root)
return prolog + xml_body
def _insert_xml_loops(
self,
xml_str: str,
role_prefix: str,
loop_candidates: list[LoopCandidate],
root: ET.Element,
) -> str:
"""
Post-process XML string to insert Jinja2 for loops.
This replaces <!--LOOP:tagname--> markers with actual loop constructs.
"""
# Build a sample element for each loop to use as template
lines = xml_str.split("\n")
result_lines = []
for line in lines:
# Check if this line contains a loop marker
if "<!--LOOP:" in line:
# Extract tag name from marker
start = line.find("<!--LOOP:") + 9
end = line.find("-->", start)
tag_name = line[start:end].strip()
# Find matching loop candidate
candidate = None
for cand in loop_candidates:
if cand.path and cand.path[-1] == tag_name:
candidate = cand
break
if candidate:
# Get indentation from current line
indent_level = len(line) - len(line.lstrip())
indent_str = " " * indent_level
# Generate loop variable name
collection_var = self.make_var_name(role_prefix, candidate.path)
item_var = candidate.loop_var
# Create sample element with ALL possible fields from ALL items
if candidate.items:
# Merge all items to get the union of all fields
merged_dict = self._merge_dicts_for_template(candidate.items)
sample_elem = self._dict_to_xml_element(
tag_name, merged_dict, item_var
)
# Apply indentation to the sample element
ET.indent(sample_elem, space=" ")
# Convert sample to string
sample_str = ET.tostring(
sample_elem, encoding="unicode"
).strip()
# Add proper indentation to each line of the sample
sample_lines = sample_str.split("\n")
# Build loop
result_lines.append(
f"{indent_str}{{% for {item_var} in {collection_var} %}}"
)
# Add each line of the sample with proper indentation
for sample_line in sample_lines:
result_lines.append(f"{indent_str} {sample_line}")
result_lines.append(f"{indent_str}{{% endfor %}}")
else:
# Keep the marker if we can't find the candidate
result_lines.append(line)
else:
result_lines.append(line)
# Post-process to replace <!--IF:...--> and <!--ENDIF:...--> with Jinja2 conditionals
final_lines = []
for line in result_lines:
# Replace <!--IF:var.field--> with {% if var.field is defined %}
if "<!--IF:" in line:
start = line.find("<!--IF:") + 7
end = line.find("-->", start)
condition = line[start:end]
indent = len(line) - len(line.lstrip())
final_lines.append(f"{' ' * indent}{{% if {condition} is defined %}}")
# Replace <!--ENDIF:field--> with {% endif %}
elif "<!--ENDIF:" in line:
indent = len(line) - len(line.lstrip())
final_lines.append(f"{' ' * indent}{{% endif %}}")
else:
final_lines.append(line)
return "\n".join(final_lines)
def _merge_dicts_for_template(self, items: list[dict[str, Any]]) -> dict[str, Any]:
"""
Merge all dicts to get the union of all possible keys.
This is used to generate XML templates that include ALL possible child
elements, even if they only appear in some items (like OSSEC rules where
different rules have different optional elements).
Args:
items: List of dict representations of XML elements
Returns:
Merged dict with all possible keys, using first occurrence as example
"""
merged: dict[str, Any] = {}
for item in items:
for key, value in item.items():
if key not in merged:
merged[key] = value
return merged
def _dict_to_xml_element(
self, tag: str, data: dict[str, Any], loop_var: str
) -> ET.Element:
"""
Convert a dict to an XML element with Jinja2 variable references.
For heterogeneous XML (like OSSEC rules), this generates conditional
Jinja2 for optional child elements.
Args:
tag: Element tag name
data: Dict representing element structure (merged from all items)
loop_var: Loop variable name to use in Jinja expressions
"""
elem = ET.Element(tag)
# Handle attributes and child elements
for key, value in data.items():
if key.startswith("@"):
# Attribute - these come from element attributes
attr_name = key[1:] # Remove @ prefix
# Use simple variable reference - attributes should always exist
elem.set(attr_name, f"{{{{ {loop_var}.{attr_name} }}}}")
elif key == "_text":
# Simple text content - use ._text accessor for dict-based items
elem.text = f"{{{{ {loop_var}._text }}}}"
elif key == "value":
# Text with attributes/children
elem.text = f"{{{{ {loop_var}.value }}}}"
elif key == "_key":
# This is the dict key (for dict collections), skip in XML
pass
elif isinstance(value, dict):
# Nested element - wrap in conditional since it might not exist in all items
# Create a conditional wrapper comment
child = ET.Element(key)
if "_text" in value:
child.text = f"{{{{ {loop_var}.{key}._text }}}}"
else:
# More complex nested structure
for sub_key, sub_val in value.items():
if not sub_key.startswith("_"):
grandchild = ET.SubElement(child, sub_key)
grandchild.text = f"{{{{ {loop_var}.{key}.{sub_key} }}}}"
# Wrap the child in a Jinja if statement (will be done via text replacement)
# For now, add a marker comment before the element
marker = ET.Comment(f"IF:{loop_var}.{key}")
elem.append(marker)
elem.append(child)
end_marker = ET.Comment(f"ENDIF:{key}")
elem.append(end_marker)
elif not isinstance(value, list):
# Simple child element (scalar value) - also wrap in conditional
marker = ET.Comment(f"IF:{loop_var}.{key}")
elem.append(marker)
child = ET.SubElement(elem, key)
child.text = f"{{{{ {loop_var}.{key} }}}}"
end_marker = ET.Comment(f"ENDIF:{key}")
elem.append(end_marker)
return elem

451
src/jinjaturtle/handlers/yaml.py
View file

@ -1,451 +0,0 @@
from __future__ import annotations
import yaml
from pathlib import Path
from typing import Any
from .dict import DictLikeHandler
from ..loop_analyzer import LoopCandidate
class YamlHandler(DictLikeHandler):
"""
YAML handler that can generate both scalar templates and loop-based templates.
"""
fmt = "yaml"
flatten_lists = True
def parse(self, path: Path) -> Any:
text = path.read_text(encoding="utf-8")
return yaml.safe_load(text) or {}
def generate_jinja2_template(
self,
parsed: Any,
role_prefix: str,
original_text: str | None = None,
) -> str:
"""Original scalar-only template generation."""
if original_text is not None:
return self._generate_yaml_template_from_text(role_prefix, original_text)
if not isinstance(parsed, (dict, list)):
raise TypeError("YAML parser result must be a dict or list")
dumped = yaml.safe_dump(parsed, sort_keys=False)
return self._generate_yaml_template_from_text(role_prefix, dumped)
def generate_jinja2_template_with_loops(
self,
parsed: Any,
role_prefix: str,
original_text: str | None,
loop_candidates: list[LoopCandidate],
) -> str:
"""Generate template with Jinja2 for loops where appropriate."""
# Build loop path set for quick lookup
loop_paths = {candidate.path for candidate in loop_candidates}
if original_text is not None:
return self._generate_yaml_template_with_loops_from_text(
role_prefix, original_text, loop_candidates, loop_paths
)
if not isinstance(parsed, (dict, list)):
raise TypeError("YAML parser result must be a dict or list")
dumped = yaml.safe_dump(parsed, sort_keys=False)
return self._generate_yaml_template_with_loops_from_text(
role_prefix, dumped, loop_candidates, loop_paths
)
def _generate_yaml_template_from_text(
self,
role_prefix: str,
text: str,
) -> str:
"""Original scalar-only template generation (unchanged from base)."""
lines = text.splitlines(keepends=True)
out_lines: list[str] = []
stack: list[tuple[int, tuple[str, ...], str]] = []
seq_counters: dict[tuple[str, ...], int] = {}
def current_path() -> tuple[str, ...]:
return stack[-1][1] if stack else ()
for raw_line in lines:
stripped = raw_line.lstrip()
indent = len(raw_line) - len(stripped)
if not stripped or stripped.startswith("#"):
out_lines.append(raw_line)
continue
while stack and indent < stack[-1][0]:
stack.pop()
if ":" in stripped and not stripped.lstrip().startswith("- "):
key_part, rest = stripped.split(":", 1)
key = key_part.strip()
if not key:
out_lines.append(raw_line)
continue
rest_stripped = rest.lstrip(" \t")
value_candidate, _ = self._split_inline_comment(rest_stripped, {"#"})
has_value = bool(value_candidate.strip())
if stack and stack[-1][0] == indent and stack[-1][2] == "map":
stack.pop()
path = current_path() + (key,)
stack.append((indent, path, "map"))
if not has_value:
out_lines.append(raw_line)
continue
value_part, comment_part = self._split_inline_comment(
rest_stripped, {"#"}
)
raw_value = value_part.strip()
var_name = self.make_var_name(role_prefix, path)
use_quotes = (
len(raw_value) >= 2
and raw_value[0] == raw_value[-1]
and raw_value[0] in {'"', "'"}
)
if use_quotes:
q = raw_value[0]
replacement = f"{q}{{{{ {var_name} }}}}{q}"
else:
replacement = f"{{{{ {var_name} }}}}"
leading = rest[: len(rest) - len(rest.lstrip(" \t"))]
new_rest = f"{leading}{replacement}{comment_part}"
new_stripped = f"{key}:{new_rest}"
out_lines.append(
" " * indent
+ new_stripped
+ ("\n" if raw_line.endswith("\n") else "")
)
continue
if stripped.startswith("- "):
if not stack or stack[-1][0] != indent or stack[-1][2] != "seq":
parent_path = current_path()
stack.append((indent, parent_path, "seq"))
parent_path = stack[-1][1]
content = stripped[2:]
index = seq_counters.get(parent_path, 0)
seq_counters[parent_path] = index + 1
path = parent_path + (str(index),)
value_part, comment_part = self._split_inline_comment(content, {"#"})
raw_value = value_part.strip()
var_name = self.make_var_name(role_prefix, path)
use_quotes = (
len(raw_value) >= 2
and raw_value[0] == raw_value[-1]
and raw_value[0] in {'"', "'"}
)
if use_quotes:
q = raw_value[0]
replacement = f"{q}{{{{ {var_name} }}}}{q}"
else:
replacement = f"{{{{ {var_name} }}}}"
new_stripped = f"- {replacement}{comment_part}"
out_lines.append(
" " * indent
+ new_stripped
+ ("\n" if raw_line.endswith("\n") else "")
)
continue
out_lines.append(raw_line)
return "".join(out_lines)
def _generate_yaml_template_with_loops_from_text(
self,
role_prefix: str,
text: str,
loop_candidates: list[LoopCandidate],
loop_paths: set[tuple[str, ...]],
) -> str:
"""
Generate YAML template with Jinja2 for loops.
Strategy:
1. Parse YAML line-by-line maintaining context
2. When we encounter a path that's a loop candidate:
- Replace that section with a {% for %} loop
- Use the first item as template structure
3. Everything else gets scalar variable replacement
"""
lines = text.splitlines(keepends=True)
out_lines: list[str] = []
stack: list[tuple[int, tuple[str, ...], str]] = []
seq_counters: dict[tuple[str, ...], int] = {}
# Track which lines are part of loop sections (to skip them)
skip_until_indent: int | None = None
def current_path() -> tuple[str, ...]:
return stack[-1][1] if stack else ()
for raw_line in lines:
stripped = raw_line.lstrip()
indent = len(raw_line) - len(stripped)
# If we're skipping lines (inside a loop section), check if we can stop
if skip_until_indent is not None:
if (
indent <= skip_until_indent
and stripped
and not stripped.startswith("#")
):
skip_until_indent = None
else:
continue # Skip this line
# Blank or comment lines
if not stripped or stripped.startswith("#"):
out_lines.append(raw_line)
continue
# Adjust stack based on indent
while stack and indent < stack[-1][0]:
stack.pop()
# --- Handle mapping key lines: "key:" or "key: value"
if ":" in stripped and not stripped.lstrip().startswith("- "):
key_part, rest = stripped.split(":", 1)
key = key_part.strip()
if not key:
out_lines.append(raw_line)
continue
rest_stripped = rest.lstrip(" \t")
value_candidate, _ = self._split_inline_comment(rest_stripped, {"#"})
has_value = bool(value_candidate.strip())
if stack and stack[-1][0] == indent and stack[-1][2] == "map":
stack.pop()
path = current_path() + (key,)
stack.append((indent, path, "map"))
# Check if this path is a loop candidate
if path in loop_paths:
# Find the matching candidate
candidate = next(c for c in loop_candidates if c.path == path)
# Generate loop
loop_str = self._generate_yaml_loop(candidate, role_prefix, indent)
out_lines.append(loop_str)
# Skip subsequent lines that are part of this collection
skip_until_indent = indent
continue
if not has_value:
out_lines.append(raw_line)
continue
# Scalar value - replace with variable
value_part, comment_part = self._split_inline_comment(
rest_stripped, {"#"}
)
raw_value = value_part.strip()
var_name = self.make_var_name(role_prefix, path)
use_quotes = (
len(raw_value) >= 2
and raw_value[0] == raw_value[-1]
and raw_value[0] in {'"', "'"}
)
if use_quotes:
q = raw_value[0]
replacement = f"{q}{{{{ {var_name} }}}}{q}"
else:
replacement = f"{{{{ {var_name} }}}}"
leading = rest[: len(rest) - len(rest.lstrip(" \t"))]
new_rest = f"{leading}{replacement}{comment_part}"
new_stripped = f"{key}:{new_rest}"
out_lines.append(
" " * indent
+ new_stripped
+ ("\n" if raw_line.endswith("\n") else "")
)
continue
# --- Handle list items: "- value" or "- key: value"
if stripped.startswith("- "):
if not stack or stack[-1][0] != indent or stack[-1][2] != "seq":
parent_path = current_path()
stack.append((indent, parent_path, "seq"))
parent_path = stack[-1][1]
# Check if parent path is a loop candidate
if parent_path in loop_paths:
# Find the matching candidate
candidate = next(
c for c in loop_candidates if c.path == parent_path
)
# Generate loop (with indent for the '-' items)
loop_str = self._generate_yaml_loop(
candidate, role_prefix, indent, is_list=True
)
out_lines.append(loop_str)
# Skip subsequent items
skip_until_indent = indent - 1 if indent > 0 else None
continue
content = stripped[2:]
index = seq_counters.get(parent_path, 0)
seq_counters[parent_path] = index + 1
path = parent_path + (str(index),)
value_part, comment_part = self._split_inline_comment(content, {"#"})
raw_value = value_part.strip()
var_name = self.make_var_name(role_prefix, path)
use_quotes = (
len(raw_value) >= 2
and raw_value[0] == raw_value[-1]
and raw_value[0] in {'"', "'"}
)
if use_quotes:
q = raw_value[0]
replacement = f"{q}{{{{ {var_name} }}}}{q}"
else:
replacement = f"{{{{ {var_name} }}}}"
new_stripped = f"- {replacement}{comment_part}"
out_lines.append(
" " * indent
+ new_stripped
+ ("\n" if raw_line.endswith("\n") else "")
)
continue
out_lines.append(raw_line)
return "".join(out_lines)
def _generate_yaml_loop(
self,
candidate: LoopCandidate,
role_prefix: str,
indent: int,
is_list: bool = False,
) -> str:
"""
Generate a Jinja2 for loop for a YAML collection.
Args:
candidate: Loop candidate with items and metadata
role_prefix: Variable prefix
indent: Indentation level in spaces
is_list: True if this is a YAML list, False if dict
Returns:
YAML string with Jinja2 loop
"""
indent_str = " " * indent
collection_var = self.make_var_name(role_prefix, candidate.path)
item_var = candidate.loop_var
lines = []
if not is_list:
# Dict-style: key: {% for ... %}
key = candidate.path[-1] if candidate.path else "items"
lines.append(f"{indent_str}{key}:")
lines.append(f"{indent_str} {{% for {item_var} in {collection_var} -%}}")
else:
# List-style: just the loop
lines.append(f"{indent_str}{{% for {item_var} in {collection_var} -%}}")
# Generate template for item structure
if candidate.items:
sample_item = candidate.items[0]
item_indent = indent + 2 if not is_list else indent
if candidate.item_schema == "scalar":
# Simple list of scalars
if is_list:
lines.append(f"{indent_str}- {{{{ {item_var} }}}}")
else:
lines.append(f"{indent_str} - {{{{ {item_var} }}}}")
elif candidate.item_schema in ("simple_dict", "nested"):
# List of dicts or complex items - these are ALWAYS list items in YAML
item_lines = self._dict_to_yaml_lines(
sample_item, item_var, item_indent, is_list_item=True
)
lines.extend(item_lines)
# Close loop
close_indent = indent + 2 if not is_list else indent
lines.append(f"{' ' * close_indent}{{% endfor %}}")
return "\n".join(lines) + "\n"
def _dict_to_yaml_lines(
self,
data: dict[str, Any],
loop_var: str,
indent: int,
is_list_item: bool = False,
) -> list[str]:
"""
Convert a dict to YAML lines with Jinja2 variable references.
Args:
data: Dict representing item structure
loop_var: Loop variable name
indent: Base indentation level
is_list_item: True if this should start with '-'
Returns:
List of YAML lines
"""
lines = []
indent_str = " " * indent
first_key = True
for key, value in data.items():
if key == "_key":
# Special key for dict collections - output as comment or skip
continue
if first_key and is_list_item:
# First key gets the list marker
lines.append(f"{indent_str}- {key}: {{{{ {loop_var}.{key} }}}}")
first_key = False
else:
# Subsequent keys are indented
sub_indent = indent + 2 if is_list_item else indent
lines.append(f"{' ' * sub_indent}{key}: {{{{ {loop_var}.{key} }}}}")
return lines

470
src/jinjaturtle/loop_analyzer.py
View file

@ -1,470 +0,0 @@
"""
Loop detection and analysis for intelligent Jinja2 template generation.
This module determines when config structures should use Jinja2 'for' loops
instead of flattened scalar variables.
"""
from __future__ import annotations
from collections import Counter
from typing import Any, Literal
class LoopCandidate:
"""
Represents a detected loop opportunity in the config structure.
Attributes:
path: Path to the collection (e.g. ("servers",) or ("config", "endpoints"))
loop_var: Variable name for loop items (e.g. "server", "endpoint")
items: The actual list/dict items that will be looped over
item_schema: Structure of each item ("scalar", "simple_dict", "nested")
confidence: How confident we are this should be a loop (0.0 to 1.0)
"""
def __init__(
self,
path: tuple[str, ...],
loop_var: str,
items: list[Any] | dict[str, Any],
item_schema: Literal["scalar", "simple_dict", "nested"],
confidence: float = 1.0,
):
self.path = path
self.loop_var = loop_var
self.items = items
self.item_schema = item_schema
self.confidence = confidence
def __repr__(self) -> str:
path_str = ".".join(self.path) if self.path else "<root>"
return (
f"LoopCandidate(path={path_str}, var={self.loop_var}, "
f"count={len(self.items)}, schema={self.item_schema}, "
f"confidence={self.confidence:.2f})"
)
class LoopAnalyzer:
"""
Analyzes parsed config structures to detect loop opportunities.
Strategy:
1. Detect homogeneous lists (all items same type/structure)
2. Detect dict collections where all values have similar structure
3. Assign confidence scores based on:
- Homogeneity of items
- Number of items (2+ for loops to make sense)
- Depth and complexity (too nested -> fallback to scalars)
- Structural patterns (e.g., repeated XML elements)
"""
# Configuration thresholds
MIN_ITEMS_FOR_LOOP = 2 # Need at least 2 items to justify a loop
MAX_NESTING_DEPTH = 3 # Beyond this, use scalar fallback
MIN_CONFIDENCE = 0.7 # Minimum confidence to use a loop
def __init__(self):
self.candidates: list[LoopCandidate] = []
def analyze(self, parsed: Any, fmt: str) -> list[LoopCandidate]:
"""
Analyze a parsed config structure and return loop candidates.
Args:
parsed: The parsed config (dict, list, or ET.Element for XML)
fmt: Format type ("yaml", "json", "toml", "xml", "ini")
Returns:
List of LoopCandidate objects, sorted by path depth (shallowest first)
"""
self.candidates = []
if fmt == "xml":
self._analyze_xml(parsed)
elif fmt in ("yaml", "json", "toml"):
self._analyze_dict_like(parsed, path=())
elif fmt == "ini":
# INI files are typically flat key-value, not suitable for loops
pass
# Sort by path depth (process parent structures before children)
self.candidates.sort(key=lambda c: len(c.path))
return self.candidates
def _analyze_dict_like(
self,
obj: Any,
path: tuple[str, ...],
depth: int = 0,
parent_is_list: bool = False,
) -> None:
"""Recursively analyze dict/list structures."""
# Safety: don't go too deep
if depth > self.MAX_NESTING_DEPTH:
return
if isinstance(obj, dict):
# Check if this dict's values form a homogeneous collection
if len(obj) >= self.MIN_ITEMS_FOR_LOOP:
candidate = self._check_dict_collection(obj, path)
if candidate:
self.candidates.append(candidate)
# Don't recurse into items we've marked as a loop
return
# Recurse into dict values
for key, value in obj.items():
self._analyze_dict_like(
value, path + (str(key),), depth + 1, parent_is_list=False
)
elif isinstance(obj, list):
# Don't create loop candidates for nested lists (lists inside lists)
# These are too complex for clean template generation and should fall back to scalar handling
if parent_is_list:
return
# Check if this list is homogeneous
if len(obj) >= self.MIN_ITEMS_FOR_LOOP:
candidate = self._check_list_collection(obj, path)
if candidate:
self.candidates.append(candidate)
# Don't recurse into items we've marked as a loop
return
# If not a good loop candidate, recurse into items
# Pass parent_is_list=True so nested lists won't create loop candidates
for i, item in enumerate(obj):
self._analyze_dict_like(
item, path + (str(i),), depth + 1, parent_is_list=True
)
def _check_list_collection(
self, items: list[Any], path: tuple[str, ...]
) -> LoopCandidate | None:
"""Check if a list should be a loop."""
if not items:
return None
# Analyze item types and structures
item_types = [type(item).__name__ for item in items]
type_counts = Counter(item_types)
# Must be homogeneous (all same type)
if len(type_counts) != 1:
return None
item_type = item_types[0]
# Scalar list (strings, numbers, bools)
if item_type in ("str", "int", "float", "bool", "NoneType"):
return LoopCandidate(
path=path,
loop_var=self._derive_loop_var(path, singular=True),
items=items,
item_schema="scalar",
confidence=1.0,
)
# List of dicts - check structural homogeneity
if item_type == "dict":
schema = self._analyze_dict_schema(items)
if schema == "simple_dict":
return LoopCandidate(
path=path,
loop_var=self._derive_loop_var(path, singular=True),
items=items,
item_schema="simple_dict",
confidence=0.95,
)
elif schema == "homogeneous":
return LoopCandidate(
path=path,
loop_var=self._derive_loop_var(path, singular=True),
items=items,
item_schema="simple_dict",
confidence=0.85,
)
# If too complex/heterogeneous, return None (use scalar fallback)
return None
def _check_dict_collection(
self, obj: dict[str, Any], path: tuple[str, ...]
) -> LoopCandidate | None:
"""
Check if a dict's values form a collection suitable for looping.
Example: {"server1": {...}, "server2": {...}} where all values
have the same structure.
NOTE: Currently disabled for TOML compatibility. TOML's dict-of-tables
syntax ([servers.alpha], [servers.beta]) cannot be easily converted to
loops without restructuring the entire TOML format. To maintain consistency
between Ansible YAML and Jinja2 templates, we treat these as scalars.
"""
# TODO: Re-enable this if we implement proper dict-of-tables loop generation
# For now, return None to use scalar handling
return None
# Original logic preserved below for reference:
# if not obj:
# return None
#
# values = list(obj.values())
#
# # Check type homogeneity
# value_types = [type(v).__name__ for v in values]
# type_counts = Counter(value_types)
#
# if len(type_counts) != 1:
# return None
#
# value_type = value_types[0]
#
# # Only interested in dict values for dict collections
# # (scalar-valued dicts stay as scalars)
# if value_type != "dict":
# return None
#
# # Check structural homogeneity
# schema = self._analyze_dict_schema(values)
# if schema in ("simple_dict", "homogeneous"):
# confidence = 0.9 if schema == "simple_dict" else 0.8
#
# # Convert dict to list of items with 'key' added
# items_with_keys = [{"_key": k, **v} for k, v in obj.items()]
#
# return LoopCandidate(
# path=path,
# loop_var=self._derive_loop_var(path, singular=True),
# items=items_with_keys,
# item_schema="simple_dict",
# confidence=confidence,
# )
#
# return None
def _analyze_dict_schema(
self, dicts: list[dict[str, Any]]
) -> Literal["simple_dict", "homogeneous", "heterogeneous"]:
"""
Analyze a list of dicts to determine their structural homogeneity.
Returns:
"simple_dict": All dicts have same keys, all values are scalars
"homogeneous": All dicts have same keys, may have nested structures
"heterogeneous": Dicts have different structures
"""
if not dicts:
return "heterogeneous"
# Get key sets from each dict
key_sets = [set(d.keys()) for d in dicts]
# Check if all have the same keys
first_keys = key_sets[0]
if not all(ks == first_keys for ks in key_sets):
# Allow minor variations (80% key overlap)
all_keys = set().union(*key_sets)
common_keys = set.intersection(*key_sets)
if len(common_keys) / len(all_keys) < 0.8:
return "heterogeneous"
# Check if values are all scalars
all_scalars = True
for d in dicts:
for v in d.values():
if isinstance(v, (dict, list)):
all_scalars = False
break
if not all_scalars:
break
if all_scalars:
return "simple_dict"
else:
return "homogeneous"
def _derive_loop_var(self, path: tuple[str, ...], singular: bool = True) -> str:
"""
Derive a sensible loop variable name from the path.
Examples:
("servers",) -> "server" (singular)
("config", "endpoints") -> "endpoint"
("users",) -> "user"
("databases",) -> "database"
"""
if not path:
return "item"
last_part = path[-1].lower()
if singular:
# Simple English pluralization rules (order matters - most specific first)
if last_part.endswith("sses"):
return last_part[:-2] # "classes" -> "class"
elif last_part.endswith("xes"):
return last_part[:-2] # "boxes" -> "box"
elif last_part.endswith("ches"):
return last_part[:-2] # "watches" -> "watch"
elif last_part.endswith("shes"):
return last_part[:-2] # "dishes" -> "dish"
elif last_part.endswith("ies"):
return last_part[:-3] + "y" # "entries" -> "entry"
elif last_part.endswith("oes"):
return last_part[:-2] # "tomatoes" -> "tomato"
elif last_part.endswith("ses") and not last_part.endswith("sses"):
# Only for words ending in "se": "databases" -> "database"
# But NOT for "sses" which we already handled
if len(last_part) > 3 and last_part[-4] not in "aeiou":
# "databases" -> "database" (consonant before 's')
return last_part[:-1]
else:
# "houses" -> "house", "causes" -> "cause"
return last_part[:-1]
elif last_part.endswith("s") and not last_part.endswith("ss"):
return last_part[:-1] # "servers" -> "server"
return last_part
def _analyze_xml(self, root: Any) -> None:
"""
Analyze XML structure for loop opportunities.
XML is particularly suited for loops when we have repeated sibling elements.
"""
import xml.etree.ElementTree as ET # nosec B405
if not isinstance(root, ET.Element):
return
self._walk_xml_element(root, path=())
def _walk_xml_element(self, elem: Any, path: tuple[str, ...]) -> None:
"""Recursively walk XML elements looking for repeated siblings."""
children = [c for c in list(elem) if isinstance(c.tag, str)]
# Count sibling elements by tag
tag_counts = Counter(child.tag for child in children)
# Find repeated tags
for tag, count in tag_counts.items():
if count >= self.MIN_ITEMS_FOR_LOOP:
# Get all elements with this tag
tagged_elements = [c for c in children if c.tag == tag]
# Check homogeneity
if self._are_xml_elements_homogeneous(tagged_elements):
# Convert to dict representation for easier handling
items = [self._xml_elem_to_dict(el) for el in tagged_elements]
# Determine schema
if all(self._is_scalar_dict(item) for item in items):
schema = "simple_dict"
confidence = 1.0
else:
schema = "nested"
confidence = 0.8
candidate = LoopCandidate(
path=path + (tag,),
loop_var=self._derive_loop_var((tag,), singular=True),
items=items,
item_schema=schema,
confidence=confidence,
)
self.candidates.append(candidate)
# Recurse into unique children (non-repeated ones will be processed normally)
for tag, count in tag_counts.items():
if count == 1:
child = next(c for c in children if c.tag == tag)
self._walk_xml_element(child, path + (tag,))
def _are_xml_elements_homogeneous(self, elements: list[Any]) -> bool:
"""Check if XML elements have similar structure."""
if not elements:
return False
# Compare attribute sets
attr_sets = [set(el.attrib.keys()) for el in elements]
first_attrs = attr_sets[0]
if not all(attrs == first_attrs for attrs in attr_sets):
# Allow some variation
all_attrs = set().union(*attr_sets)
common_attrs = set.intersection(*attr_sets) if attr_sets else set()
# Very permissive for attributes - 20% overlap is OK
if len(common_attrs) / max(len(all_attrs), 1) < 0.2:
return False
# Compare child element tags
child_tag_sets = [
set(c.tag for c in el if hasattr(c, "tag")) for el in elements
]
if child_tag_sets:
first_tags = child_tag_sets[0]
if not all(tags == first_tags for tags in child_tag_sets):
# Allow significant variation for XML - just need SOME commonality
# This is important for cases like OSSEC rules where each rule
# has different optional child elements (if_sid, url_pcre2, etc.)
all_tags = set().union(*child_tag_sets)
common_tags = (
set.intersection(*child_tag_sets) if child_tag_sets else set()
)
# Lower threshold to 20% - if they share at least 20% of tags, consider them similar
# Even if they just share 'description' or 'id' fields, that's enough
if len(common_tags) / max(len(all_tags), 1) < 0.2:
return False
return True
def _xml_elem_to_dict(self, elem: Any) -> dict[str, Any]:
"""Convert an XML element to a dict representation."""
result: dict[str, Any] = {}
# Add attributes
for attr_name, attr_val in elem.attrib.items():
result[f"@{attr_name}"] = attr_val
# Add text content
text = (elem.text or "").strip()
if text:
children = [c for c in list(elem) if hasattr(c, "tag")]
if not elem.attrib and not children:
result["_text"] = text
else:
result["value"] = text
# Add child elements
for child in elem:
if hasattr(child, "tag"):
child_dict = self._xml_elem_to_dict(child)
if child.tag in result:
# Multiple children with same tag - convert to list
if not isinstance(result[child.tag], list):
result[child.tag] = [result[child.tag]]
result[child.tag].append(child_dict)
else:
result[child.tag] = child_dict
return result
def _is_scalar_dict(self, obj: dict[str, Any]) -> bool:
"""Check if a dict contains only scalar values (no nested dicts/lists)."""
for v in obj.values():
if isinstance(v, (dict, list)):
return False
return True

8
tests.sh
View file

@ -1,11 +1,3 @@
#!/bin/bash
set -eo pipefail
# Run pytests
poetry run pytest -vvvv --cov=jinjaturtle --cov-report=term-missing --disable-warnings
# Ensure we test the CLI like a human
for file in `ls -1 tests/samples/*`; do
poetry run jinjaturtle -r test $file -d test.yml -t test.j2
done

7
tests/samples/bar.yaml
View file

@ -1,7 +0,0 @@
---
# Top comment
foo: "bar"
blah:
- something
- else

11
tests/samples/foo.json
View file

@ -1,11 +0,0 @@
{
"foo": "bar",
"nested": {
"a": 1,
"b": true
},
"list": [
10,
20
]
}

225
tests/samples/ossec.xml
View file

@ -1,225 +0,0 @@
<!-- @(#) $Id: ./etc/rules/web_rules.xml, 2013/02/28 dcid Exp $
-
- Official Web access rules for OSSEC.
-
- Copyright (C) 2009 Trend Micro Inc.
- All rights reserved.
-
- This program is a free software; you can redistribute it
- and/or modify it under the terms of the GNU General Public
- License (version 2) as published by the FSF - Free Software
- Foundation.
-
- License details: http://www.ossec.net/en/licensing.html
-->
<group name="web,accesslog,">
<rule id="31100" level="0">
<category>web-log</category>
<description>Access log messages grouped.</description>
</rule>
<rule id="31108" level="0">
<if_sid>31100</if_sid>
<id_pcre2>^2|^3</id_pcre2>
<compiled_rule>is_simple_http_request</compiled_rule>
<description>Ignored URLs (simple queries).</description>
</rule>
<rule id="31101" level="5">
<if_sid>31100</if_sid>
<id_pcre2>^4</id_pcre2>
<description>Web server 400 error code.</description>
</rule>
<rule id="31102" level="0">
<if_sid>31101</if_sid>
<url_pcre2>\.jpg$|\.gif$|favicon\.ico$|\.png$|robots\.txt$|\.css$|\.js$|\.jpeg$</url_pcre2>
<compiled_rule>is_simple_http_request</compiled_rule>
<description>Ignored extensions on 400 error codes.</description>
</rule>
<rule id="31103" level="6">
<if_sid>31100,31108</if_sid>
<url_pcre2>=select%20|select\+|insert%20|%20from%20|%20where%20|union%20|</url_pcre2>
<url_pcre2>union\+|where\+|null,null|xp_cmdshell</url_pcre2>
<description>SQL injection attempt.</description>
<group>attack,sql_injection,</group>
</rule>
<rule id="31104" level="6">
<if_sid>31100</if_sid>
<!-- Attempt to do directory transversal, simple sql injections,
- or access to the etc or bin directory (unix). -->
<url_pcre2>%027|%00|%01|%7f|%2E%2E|%0A|%0D|\.\./\.\.|\.\.\\\.\.|echo;|</url_pcre2>
<url_pcre2>cmd\.exe|root\.exe|_mem_bin|msadc|/winnt/|/boot\.ini|</url_pcre2>
<url_pcre2>/x90/|default\.ida|/sumthin|nsiislog\.dll|chmod%|wget%|cd%20|</url_pcre2>
<url_pcre2>exec%20|\.\./\.\.//|%5C\.\./%5C|\./\./\./\./|2e%2e%5c%2e|\\x5C\\x5C</url_pcre2>
<description>Common web attack.</description>
<group>attack,</group>
</rule>
<rule id="31105" level="6">
<if_sid>31100</if_sid>
<url_pcre2>%3Cscript|%3C%2Fscript|script>|script%3E|SRC=javascript|IMG%20|</url_pcre2>
<url_pcre2>%20ONLOAD=|INPUT%20|iframe%20</url_pcre2>
<description>XSS (Cross Site Scripting) attempt.</description>
<group>attack,</group>
</rule>
<rule id="31106" level="6">
<if_sid>31103, 31104, 31105</if_sid>
<id_pcre2>^200</id_pcre2>
<description>A web attack returned code 200 (success).</description>
<group>attack,</group>
</rule>
<rule id="31110" level="6">
<if_sid>31100</if_sid>
<url_pcre2>\?-d|\?-s|\?-a|\?-b|\?-w</url_pcre2>
<description>PHP CGI-bin vulnerability attempt.</description>
<group>attack,</group>
</rule>
<rule id="31109" level="6">
<if_sid>31100</if_sid>
<url_pcre2>\+as\+varchar</url_pcre2>
<pcre2>%2Bchar\(\d+\)%2Bchar\(\d+\)%2Bchar\(\d+\)%2Bchar\(\d+\)%2Bchar\(\d+\)%2Bchar\(\d+\)</pcre2>
<description>MSSQL Injection attempt (/ur.php, urchin.js)</description>
<group>attack,</group>
</rule>
<!-- If your site have a search engine, you may need to ignore
- it in here.
-->
<rule id="31107" level="0">
<if_sid>31103, 31104, 31105</if_sid>
<url_pcre2>^/search\.php\?search=|^/index\.php\?searchword=</url_pcre2>
<description>Ignored URLs for the web attacks</description>
</rule>
<rule id="31115" level="13" maxsize="7900">
<if_sid>31100</if_sid>
<description>URL too long. Higher than allowed on most </description>
<description>browsers. Possible attack.</description>
<group>invalid_access,</group>
</rule>
<!-- 500 error codes, server error
- http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
-->
<rule id="31120" level="5">
<if_sid>31100</if_sid>
<id_pcre2>^50</id_pcre2>
<description>Web server 500 error code (server error).</description>
</rule>
<rule id="31121" level="4">
<if_sid>31120</if_sid>
<id_pcre2>^501</id_pcre2>
<description>Web server 501 error code (Not Implemented).</description>
</rule>
<rule id="31122" level="5">
<if_sid>31120</if_sid>
<id_pcre2>^500</id_pcre2>
<options>alert_by_email</options>
<description>Web server 500 error code (Internal Error).</description>
<group>system_error,</group>
</rule>
<rule id="31123" level="4">
<if_sid>31120</if_sid>
<id_pcre2>^503</id_pcre2>
<options>alert_by_email</options>
<description>Web server 503 error code (Service unavailable).</description>
</rule>
<!-- Rules to ignore crawlers -->
<rule id="31140" level="0">
<if_sid>31101</if_sid>
<compiled_rule>is_valid_crawler</compiled_rule>
<description>Ignoring google/msn/yahoo bots.</description>
</rule>
<!-- Ignoring nginx 499's -->
<rule id="31141" level="0">
<if_sid>31101</if_sid>
<id_pcre2>^499</id_pcre2>
<description>Ignored 499's on nginx.</description>
</rule>
<rule id="31151" level="10" frequency="12" timeframe="90">
<if_matched_sid>31101</if_matched_sid>
<same_source_ip />
<description>Multiple web server 400 error codes </description>
<description>from same source ip.</description>
<group>web_scan,recon,</group>
</rule>
<rule id="31152" level="10" frequency="6" timeframe="120">
<if_matched_sid>31103</if_matched_sid>
<same_source_ip />
<description>Multiple SQL injection attempts from same </description>
<description>source ip.</description>
<group>attack,sql_injection,</group>
</rule>
<rule id="31153" level="10" frequency="8" timeframe="120">
<if_matched_sid>31104</if_matched_sid>
<same_source_ip />
<description>Multiple common web attacks from same source ip.</description>
<group>attack,</group>
</rule>
<rule id="31154" level="10" frequency="8" timeframe="120">
<if_matched_sid>31105</if_matched_sid>
<same_source_ip />
<description>Multiple XSS (Cross Site Scripting) attempts </description>
<description>from same source ip.</description>
<group>attack,</group>
</rule>
<rule id="31161" level="10" frequency="12" timeframe="120">
<if_matched_sid>31121</if_matched_sid>
<same_source_ip />
<description>Multiple web server 501 error code (Not Implemented).</description>
<group>web_scan,recon,</group>
</rule>
<rule id="31162" level="10" frequency="12" timeframe="120">
<if_matched_sid>31122</if_matched_sid>
<same_source_ip />
<description>Multiple web server 500 error code (Internal Error).</description>
<group>system_error,</group>
</rule>
<rule id="31163" level="10" frequency="12" timeframe="120">
<if_matched_sid>31123</if_matched_sid>
<same_source_ip />
<description>Multiple web server 503 error code (Service unavailable).</description>
<group>web_scan,recon,</group>
</rule>
<rule id="31164" level="6">
<if_sid>31100</if_sid>
<url_pcre2>=%27|select%2B|insert%2B|%2Bfrom%2B|%2Bwhere%2B|%2Bunion%2B</url_pcre2>
<description>SQL injection attempt.</description>
<group>attack,sqlinjection,</group>
</rule>
<rule id="31165" level="6">
<if_sid>31100</if_sid>
<url_pcre2>%EF%BC%87|%EF%BC%87|%EF%BC%87|%2531|%u0053%u0045</url_pcre2>
<description>SQL injection attempt.</description>
<group>attack,sqlinjection,</group>
</rule>
</group> <!-- Web access log -->

34
tests/test_base_handler.py
View file

@ -1,34 +0,0 @@
from __future__ import annotations
from pathlib import Path
import pytest
from jinjaturtle.handlers.base import BaseHandler
def test_split_inline_comment_handles_quoted_hash():
# The '#' inside quotes should not start a comment; the one outside should.
text = " 'foo # not comment' # real"
handler = BaseHandler()
value, comment = handler._split_inline_comment(text, {"#"})
assert "not comment" in value
assert comment.strip() == "# real"
def test_base_handler_abstract_methods_raise_not_implemented(tmp_path: Path):
"""
Ensure the abstract methods on BaseHandler all raise NotImplementedError.
This covers the stub implementations.
"""
handler = BaseHandler()
dummy_path = tmp_path / "dummy.cfg"
with pytest.raises(NotImplementedError):
handler.parse(dummy_path)
with pytest.raises(NotImplementedError):
handler.flatten(object())
with pytest.raises(NotImplementedError):
handler.generate_jinja2_template(parsed=object(), role_prefix="role")

25
tests/test_cli.py
View file

@ -1,6 +1,10 @@
from __future__ import annotations
import sys
from pathlib import Path
import pytest
from jinjaturtle import cli
SAMPLES_DIR = Path(__file__).parent / "samples"
@ -14,7 +18,7 @@ def test_cli_stdout_toml(capsys):
cfg_path = SAMPLES_DIR / "tom.toml"
exit_code = cli._main([str(cfg_path), "-r", "jinjaturtle"])
assert exit_code
assert exit_code == 0
captured = capsys.readouterr()
out = captured.out
@ -48,7 +52,7 @@ def test_cli_writes_output_files(tmp_path, capsys):
]
)
assert exit_code
assert exit_code == 0
assert defaults_path.is_file()
assert template_path.is_file()
@ -62,3 +66,20 @@ def test_cli_writes_output_files(tmp_path, capsys):
# When writing to files, we shouldn't print the big headers
assert "# defaults/main.yml" not in captured.out
assert "# config.j2" not in captured.out
def test_main_wrapper_exits_with_zero(monkeypatch):
"""
Cover the main() wrapper that raises SystemExit.
"""
cfg_path = SAMPLES_DIR / "tom.toml"
monkeypatch.setattr(
sys,
"argv",
["jinjaturtle", str(cfg_path), "-r", "jinjaturtle"],
)
with pytest.raises(SystemExit) as exc:
cli.main()
assert exc.value.code == 0

191
tests/test_core.py Normal file
View file

@ -0,0 +1,191 @@
from __future__ import annotations
from pathlib import Path
import pytest
import yaml
import jinjaturtle.core as core
from jinjaturtle.core import (
detect_format,
parse_config,
flatten_config,
generate_defaults_yaml,
generate_template,
make_var_name,
)
SAMPLES_DIR = Path(__file__).parent / "samples"
def test_make_var_name_basic():
# simple sanity checks on the naming rules
assert (
make_var_name("jinjaturtle", ("somesection", "foo"))
== "jinjaturtle_somesection_foo"
)
assert (
make_var_name("JinjaTurtle", ("Other-Section", "some value"))
== "jinjaturtle_other_section_some_value"
)
# no trailing underscores, all lowercase, no spaces
name = make_var_name("MyRole", (" Section Name ", "Key-Name "))
assert name == name.lower()
assert " " not in name
assert not name.endswith("_")
def test_make_var_name_empty_path_returns_prefix():
# Cover the branch where there are no path components.
assert make_var_name("MyRole", ()) == "myrole"
def test_detect_format_explicit_overrides_suffix(tmp_path: Path):
# Explicit format should win over file suffix.
cfg_path = tmp_path / "config.ini"
cfg_path.write_text("[section]\nkey=value\n", encoding="utf-8")
fmt = detect_format(cfg_path, explicit="toml")
assert fmt == "toml"
def test_detect_format_fallback_ini(tmp_path: Path):
# Unknown suffix should fall back to "ini".
cfg_path = tmp_path / "weird.cnf"
cfg_path.write_text("[section]\nkey=value\n", encoding="utf-8")
fmt, parsed = parse_config(cfg_path) # no explicit fmt
assert fmt == "ini"
# parsed should be an INI ConfigParser with our section/key
flat = flatten_config(fmt, parsed)
assert any(path == ("section", "key") for path, _ in flat)
def test_toml_sample_roundtrip():
toml_path = SAMPLES_DIR / "tom.toml"
assert toml_path.is_file(), f"Missing sample TOML file: {toml_path}"
fmt, parsed = parse_config(toml_path)
assert fmt == "toml"
flat_items = flatten_config(fmt, parsed)
assert flat_items, "Expected at least one flattened item from TOML sample"
defaults_yaml = generate_defaults_yaml("jinjaturtle", flat_items)
defaults = yaml.safe_load(defaults_yaml)
# defaults should be a non-empty dict
assert isinstance(defaults, dict)
assert defaults, "Expected non-empty defaults for TOML sample"
# all keys should be lowercase, start with prefix, and have no spaces
for key in defaults:
assert key.startswith("jinjaturtle_")
assert key == key.lower()
assert " " not in key
# template generation
template = generate_template(fmt, parsed, "jinjaturtle")
assert isinstance(template, str)
assert template.strip(), "Template for TOML sample should not be empty"
# each default variable name should appear in the template as a Jinja placeholder
for var_name in defaults:
assert (
var_name in template
), f"Variable {var_name} not referenced in TOML template"
def test_ini_php_sample_roundtrip():
ini_path = SAMPLES_DIR / "php.ini"
assert ini_path.is_file(), f"Missing sample INI file: {ini_path}"
fmt, parsed = parse_config(ini_path)
assert fmt == "ini"
flat_items = flatten_config(fmt, parsed)
assert flat_items, "Expected at least one flattened item from php.ini sample"
defaults_yaml = generate_defaults_yaml("php", flat_items)
defaults = yaml.safe_load(defaults_yaml)
# defaults should be a non-empty dict
assert isinstance(defaults, dict)
assert defaults, "Expected non-empty defaults for php.ini sample"
# all keys should be lowercase, start with prefix, and have no spaces
for key in defaults:
assert key.startswith("php_")
assert key == key.lower()
assert " " not in key
# template generation
template = generate_template(fmt, parsed, "php")
assert isinstance(template, str)
assert template.strip(), "Template for php.ini sample should not be empty"
# each default variable name should appear in the template as a Jinja placeholder
for var_name in defaults:
assert (
var_name in template
), f"Variable {var_name} not referenced in INI template"
def test_formats_match_expected_extensions():
"""
Sanity check that format detection lines up with the filenames
we’re using for the samples.
"""
toml_path = SAMPLES_DIR / "tom.toml"
ini_path = SAMPLES_DIR / "php.ini"
fmt_toml, _ = parse_config(toml_path)
fmt_ini, _ = parse_config(ini_path)
assert fmt_toml == "toml"
assert fmt_ini == "ini"
def test_parse_config_toml_missing_tomllib(monkeypatch):
"""
Force tomllib to None to hit the RuntimeError branch when parsing TOML.
"""
toml_path = SAMPLES_DIR / "tom.toml"
# Simulate an environment without tomllib/tomli
monkeypatch.setattr(core, "tomllib", None)
with pytest.raises(RuntimeError) as exc:
core.parse_config(toml_path, fmt="toml")
assert "tomllib/tomli is required" in str(exc.value)
def test_parse_config_unsupported_format(tmp_path: Path):
"""
Hit the ValueError in parse_config when fmt is neither 'toml' nor 'ini'.
"""
cfg_path = tmp_path / "config.whatever"
cfg_path.write_text("", encoding="utf-8")
with pytest.raises(ValueError):
parse_config(cfg_path, fmt="yaml")
def test_generate_template_type_and_format_errors():
"""
Exercise the error branches in generate_template:
- toml with non-dict parsed
- ini with non-ConfigParser parsed
- completely unsupported fmt
"""
# wrong type for TOML
with pytest.raises(TypeError):
generate_template("toml", parsed="not a dict", role_prefix="role")
# wrong type for INI
with pytest.raises(TypeError):
generate_template("ini", parsed={"not": "a configparser"}, role_prefix="role")
# unsupported format
with pytest.raises(ValueError):
generate_template("yaml", parsed=None, role_prefix="role")

193
tests/test_core_utils.py
View file

@ -1,193 +0,0 @@
from __future__ import annotations
from pathlib import Path
import pytest
import yaml
import jinjaturtle.core as core
from jinjaturtle.core import (
detect_format,
parse_config,
flatten_config,
generate_ansible_yaml,
generate_jinja2_template,
make_var_name,
)
SAMPLES_DIR = Path(__file__).parent / "samples"
def test_make_var_name_basic():
# simple sanity checks on the naming rules
assert (
make_var_name("jinjaturtle", ("somesection", "foo"))
== "jinjaturtle_somesection_foo"
)
assert (
make_var_name("JinjaTurtle", ("Other-Section", "some value"))
== "jinjaturtle_other_section_some_value"
)
# no trailing underscores, all lowercase, no spaces
name = make_var_name("MyRole", (" Section Name ", "Key-Name "))
assert name == name.lower()
assert " " not in name
assert not name.endswith("_")
def test_make_var_name_empty_path_returns_prefix():
# Cover the branch where there are no path components.
assert make_var_name("MyRole", ()) == "myrole"
def test_detect_format_explicit_overrides_suffix(tmp_path: Path):
# Explicit format should win over file suffix.
cfg_path = tmp_path / "config.ini"
cfg_path.write_text("[section]\nkey=value\n", encoding="utf-8")
fmt = detect_format(cfg_path, explicit="toml")
assert fmt == "toml"
def test_detect_format_fallback_ini(tmp_path: Path):
# Unknown suffix should fall back to "ini".
cfg_path = tmp_path / "weird.cnf"
cfg_path.write_text("[section]\nkey=value\n", encoding="utf-8")
fmt, parsed = parse_config(cfg_path) # no explicit fmt
assert fmt == "ini"
# parsed should be an INI ConfigParser with our section/key
flat = flatten_config(fmt, parsed)
assert any(path == ("section", "key") for path, _ in flat)
def test_formats_match_expected_extensions():
"""
Sanity check that format detection lines up with the filenames
we’re using for the samples.
"""
toml_path = SAMPLES_DIR / "tom.toml"
ini_path = SAMPLES_DIR / "php.ini"
xml_path = SAMPLES_DIR / "ossec.xml"
fmt_toml, _ = parse_config(toml_path)
fmt_ini, _ = parse_config(ini_path)
fmt_xml, _ = parse_config(xml_path)
assert fmt_toml == "toml"
assert fmt_ini == "ini"
assert fmt_xml == "xml"
def test_parse_config_unsupported_format(tmp_path: Path):
"""
Hit the ValueError in parse_config when fmt is not a supported format.
"""
cfg_path = tmp_path / "config.whatever"
cfg_path.write_text("", encoding="utf-8")
with pytest.raises(ValueError):
parse_config(cfg_path, fmt="bogus")
def test_generate_jinja2_template_type_and_format_errors():
"""
Exercise the error branches in generate_jinja2_template:
- toml with non-dict parsed
- ini with non-ConfigParser parsed
- yaml with wrong parsed type
- json with wrong parsed type
- completely unsupported fmt (with and without original_text)
"""
# wrong type for TOML
with pytest.raises(TypeError):
generate_jinja2_template("toml", parsed="not a dict", role_prefix="role")
# wrong type for INI
with pytest.raises(TypeError):
generate_jinja2_template(
"ini", parsed={"not": "a configparser"}, role_prefix="role"
)
# wrong type for YAML
with pytest.raises(TypeError):
generate_jinja2_template("yaml", parsed=None, role_prefix="role")
# wrong type for JSON
with pytest.raises(TypeError):
generate_jinja2_template("json", parsed=None, role_prefix="role")
# unsupported format, no original_text
with pytest.raises(ValueError):
generate_jinja2_template("bogusfmt", parsed=None, role_prefix="role")
# unsupported format, with original_text
with pytest.raises(ValueError):
generate_jinja2_template(
"bogusfmt",
parsed=None,
role_prefix="role",
original_text="foo=bar",
)
def test_normalize_default_value_true_false_strings():
# 'true'/'false' strings should be preserved as strings and double-quoted in YAML.
flat_items = [
(("section", "foo"), "true"),
(("section", "bar"), "FALSE"),
]
ansible_yaml = generate_ansible_yaml("role", flat_items)
data = yaml.safe_load(ansible_yaml)
assert data["role_section_foo"] == "true"
assert data["role_section_bar"] == "FALSE"
def test_fallback_str_representer_for_unknown_type():
"""
Ensure that the _fallback_str_representer is used for objects that
PyYAML doesn't know how to represent.
"""
class Weird:
def __str__(self) -> str:
return "weird-value"
data = {"foo": Weird()}
dumped = yaml.dump(
data,
Dumper=core._TurtleDumper,
sort_keys=False,
default_flow_style=False,
)
# It should serialize without error, and the string form should appear.
assert "weird-value" in dumped
def test_normalize_default_value_bool_inputs_are_stringified():
"""
Boolean values are now preserved as booleans in YAML (not stringified).
This supports proper type preservation for JSON and other formats.
"""
flat_items = [
(("section", "flag_true"), True),
(("section", "flag_false"), False),
]
ansible_yaml = generate_ansible_yaml("role", flat_items)
data = yaml.safe_load(ansible_yaml)
# Booleans are now preserved as booleans
assert data["role_section_flag_true"] is True
assert data["role_section_flag_false"] is False
def test_flatten_config_unsupported_format():
"""
Calling flatten_config with an unknown fmt should raise ValueError.
"""
with pytest.raises(ValueError) as exc:
flatten_config("bogusfmt", parsed=None)
assert "Unsupported format" in str(exc.value)

93
tests/test_ini_handler.py
View file

@ -1,93 +0,0 @@
from __future__ import annotations
from pathlib import Path
import configparser
import pytest
import yaml
from jinjaturtle.core import (
parse_config,
flatten_config,
generate_ansible_yaml,
generate_jinja2_template,
)
from jinjaturtle.handlers.ini import IniHandler
SAMPLES_DIR = Path(__file__).parent / "samples"
def test_ini_php_sample_roundtrip():
ini_path = SAMPLES_DIR / "php.ini"
assert ini_path.is_file(), f"Missing sample INI file: {ini_path}"
fmt, parsed = parse_config(ini_path)
assert fmt == "ini"
flat_items = flatten_config(fmt, parsed)
assert flat_items, "Expected at least one flattened item from php.ini sample"
ansible_yaml = generate_ansible_yaml("php", flat_items)
defaults = yaml.safe_load(ansible_yaml)
# defaults should be a non-empty dict
assert isinstance(defaults, dict)
assert defaults, "Expected non-empty defaults for php.ini sample"
# all keys should be lowercase, start with prefix, and have no spaces
for key in defaults:
assert key.startswith("php_")
assert key == key.lower()
assert " " not in key
# template generation
original_text = ini_path.read_text(encoding="utf-8")
template = generate_jinja2_template(fmt, parsed, "php", original_text=original_text)
assert "; About this file" in template
assert isinstance(template, str)
assert template.strip(), "Template for php.ini sample should not be empty"
# each default variable name should appear in the template as a Jinja placeholder
for var_name in defaults:
assert (
var_name in template
), f"Variable {var_name} not referenced in INI template"
def test_generate_jinja2_template_fallback_ini():
"""
When original_text is not provided, generate_jinja2_template should use the
structural fallback path for INI configs.
"""
parser = configparser.ConfigParser()
# foo is quoted in the INI text to hit the "preserve quotes" branch
parser["section"] = {"foo": '"bar"', "num": "42"}
tmpl_ini = generate_jinja2_template("ini", parsed=parser, role_prefix="role")
assert "[section]" in tmpl_ini
assert "role_section_foo" in tmpl_ini
assert '"{{ role_section_foo }}"' in tmpl_ini # came from quoted INI value
def test_generate_ini_template_from_text_edge_cases():
# Cover CRLF newlines, lines without '=', and lines with no key before '='.
text = "[section]\r\nkey=value\r\nnoequals\r\n = bare\r\n"
handler = IniHandler()
tmpl = handler._generate_ini_template_from_text("role", text)
# We don't care about exact formatting here, just that it runs and
# produces some reasonable output.
assert "[section]" in tmpl
assert "role_section_key" in tmpl
# The "noequals" line should be preserved as-is.
assert "noequals" in tmpl
# The " = bare" line has no key and should be left untouched.
assert " = bare" in tmpl
def test_ini_handler_flatten_type_error():
"""
Passing a non-ConfigParser into IniHandler.flatten should raise TypeError.
"""
handler = IniHandler()
with pytest.raises(TypeError):
handler.flatten(parsed={"not": "a configparser"})

63
tests/test_json_handler.py
View file

@ -1,63 +0,0 @@
from __future__ import annotations
from pathlib import Path
import pytest
import yaml
from jinjaturtle.core import (
parse_config,
flatten_config,
generate_ansible_yaml,
analyze_loops,
generate_jinja2_template,
)
from jinjaturtle.handlers.json import JsonHandler
SAMPLES_DIR = Path(__file__).parent / "samples"
def test_json_roundtrip():
json_path = SAMPLES_DIR / "foo.json"
assert json_path.is_file(), f"Missing sample JSON file: {json_path}"
fmt, parsed = parse_config(json_path)
assert fmt == "json"
# With loop detection
loop_candidates = analyze_loops(fmt, parsed)
flat_items = flatten_config(fmt, parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("foobar", flat_items, loop_candidates)
defaults = yaml.safe_load(ansible_yaml)
# Defaults: nested keys
assert defaults["foobar_foo"] == "bar"
assert defaults["foobar_nested_a"] == 1
# Booleans are now preserved as booleans (not stringified)
assert defaults["foobar_nested_b"] is True
# List should be a list (not flattened to scalars)
assert defaults["foobar_list"] == [10, 20]
# Template generation with loops
template = generate_jinja2_template("json", parsed, "foobar", None, loop_candidates)
# Template should use | tojson for type preservation
assert "{{ foobar_foo | tojson }}" in template
assert "{{ foobar_nested_a | tojson }}" in template
assert "{{ foobar_nested_b | tojson }}" in template
# List should use loop (not scalar indices)
assert "{% for" in template
assert "foobar_list" in template
# Should NOT have scalar indices
assert "foobar_list_0" not in template
assert "foobar_list_1" not in template
def test_generate_jinja2_template_json_type_error():
"""
Wrong type for JSON in JsonHandler.generate_jinja2_template should raise TypeError.
"""
handler = JsonHandler()
with pytest.raises(TypeError):
handler.generate_jinja2_template(parsed="not a dict", role_prefix="role")

566
tests/test_roundtrip.py
View file

@ -1,566 +0,0 @@
"""
Roundtrip tests: Generate config → template/YAML → regenerate config → compare.
These tests verify that:
1. Generated Jinja2 template + Ansible YAML can reproduce the original config
2. The regenerated config is semantically equivalent (allowing whitespace differences)
3. No data loss occurs during the template generation process
This is the ultimate validation - if the roundtrip works, the templates are correct.
"""
from __future__ import annotations
import json
import yaml
from pathlib import Path
from typing import Any
from jinja2 import Environment, StrictUndefined
import pytest
from jinjaturtle.core import (
parse_config,
analyze_loops,
flatten_config,
generate_ansible_yaml,
generate_jinja2_template,
)
def render_template(template: str, variables: dict[str, Any]) -> str:
"""Render a Jinja2 template with variables."""
env = Environment(undefined=StrictUndefined)
jinja_template = env.from_string(template)
return jinja_template.render(variables)
class TestRoundtripJSON:
"""Roundtrip tests for JSON files."""
def test_foo_json_roundtrip(self):
"""Test foo.json can be perfectly regenerated from template."""
samples_dir = Path(__file__).parent / "samples"
json_file = samples_dir / "foo.json"
if not json_file.exists():
pytest.skip("foo.json not found")
# Read original
original_text = json_file.read_text()
original_data = json.loads(original_text)
# Generate template and YAML
fmt, parsed = parse_config(json_file)
loop_candidates = analyze_loops(fmt, parsed)
flat_items = flatten_config(fmt, parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
template = generate_jinja2_template(fmt, parsed, "test", None, loop_candidates)
# Load variables from YAML
variables = yaml.safe_load(ansible_yaml)
# Render template
regenerated_text = render_template(template, variables)
regenerated_data = json.loads(regenerated_text)
# Compare data structures (should match exactly)
assert regenerated_data == original_data, (
f"Regenerated JSON differs from original\n"
f"Original: {json.dumps(original_data, indent=2, sort_keys=True)}\n"
f"Regenerated: {json.dumps(regenerated_data, indent=2, sort_keys=True)}"
)
def test_json_all_types_roundtrip(self):
"""Test JSON with all data types roundtrips perfectly."""
json_text = """
{
"string": "value",
"number": 42,
"float": 3.14,
"boolean": true,
"false_val": false,
"null_value": null,
"array": [1, 2, 3],
"object": {
"nested": "data"
}
}
"""
original_data = json.loads(json_text)
# Generate template and YAML
loop_candidates = analyze_loops("json", original_data)
flat_items = flatten_config("json", original_data, loop_candidates)
ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
template = generate_jinja2_template(
"json", original_data, "test", None, loop_candidates
)
# Render template
variables = yaml.safe_load(ansible_yaml)
regenerated_text = render_template(template, variables)
regenerated_data = json.loads(regenerated_text)
# Should match exactly
assert regenerated_data == original_data
class TestRoundtripYAML:
"""Roundtrip tests for YAML files."""
def test_bar_yaml_roundtrip(self):
"""Test bar.yaml can be regenerated from template."""
samples_dir = Path(__file__).parent / "samples"
yaml_file = samples_dir / "bar.yaml"
if not yaml_file.exists():
pytest.skip("bar.yaml not found")
# Read original
original_text = yaml_file.read_text()
original_data = yaml.safe_load(original_text)
# Generate template and YAML
fmt, parsed = parse_config(yaml_file)
loop_candidates = analyze_loops(fmt, parsed)
flat_items = flatten_config(fmt, parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
template = generate_jinja2_template(
fmt, parsed, "test", original_text, loop_candidates
)
# Load variables from YAML
variables = yaml.safe_load(ansible_yaml)
# Render template
regenerated_text = render_template(template, variables)
regenerated_data = yaml.safe_load(regenerated_text)
# Compare data structures
assert regenerated_data == original_data, (
f"Regenerated YAML differs from original\n"
f"Original: {original_data}\n"
f"Regenerated: {regenerated_data}"
)
def test_yaml_with_lists_roundtrip(self):
"""Test YAML with various list structures."""
yaml_text = """
name: myapp
simple_list:
- item1
- item2
- item3
list_of_dicts:
- name: first
value: 1
- name: second
value: 2
nested:
inner_list:
- a
- b
"""
original_data = yaml.safe_load(yaml_text)
# Generate template and YAML
loop_candidates = analyze_loops("yaml", original_data)
flat_items = flatten_config("yaml", original_data, loop_candidates)
ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
template = generate_jinja2_template(
"yaml", original_data, "test", yaml_text, loop_candidates
)
# Render template
variables = yaml.safe_load(ansible_yaml)
regenerated_text = render_template(template, variables)
regenerated_data = yaml.safe_load(regenerated_text)
# Compare
assert regenerated_data == original_data
class TestRoundtripTOML:
"""Roundtrip tests for TOML files."""
def test_tom_toml_roundtrip(self):
"""Test tom.toml can be regenerated from template."""
samples_dir = Path(__file__).parent / "samples"
toml_file = samples_dir / "tom.toml"
if not toml_file.exists():
pytest.skip("tom.toml not found")
# Read original
original_text = toml_file.read_text()
import tomllib
original_data = tomllib.loads(original_text)
# Generate template and YAML
fmt, parsed = parse_config(toml_file)
loop_candidates = analyze_loops(fmt, parsed)
flat_items = flatten_config(fmt, parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
template = generate_jinja2_template(
fmt, parsed, "test", original_text, loop_candidates
)
# Load variables from YAML
variables = yaml.safe_load(ansible_yaml)
# Render template
regenerated_text = render_template(template, variables)
regenerated_data = tomllib.loads(regenerated_text)
# Compare data structures
# Note: TOML datetime objects need special handling
assert _compare_toml_data(regenerated_data, original_data), (
f"Regenerated TOML differs from original\n"
f"Original: {original_data}\n"
f"Regenerated: {regenerated_data}"
)
def test_toml_with_arrays_roundtrip(self):
"""Test TOML with inline arrays and array-of-tables."""
toml_text = """
name = "test"
ports = [8080, 8081, 8082]
[[database]]
host = "db1.example.com"
port = 5432
[[database]]
host = "db2.example.com"
port = 5433
"""
import tomllib
original_data = tomllib.loads(toml_text)
# Generate template and YAML
loop_candidates = analyze_loops("toml", original_data)
flat_items = flatten_config("toml", original_data, loop_candidates)
ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
template = generate_jinja2_template(
"toml", original_data, "test", toml_text, loop_candidates
)
# Render template
variables = yaml.safe_load(ansible_yaml)
regenerated_text = render_template(template, variables)
regenerated_data = tomllib.loads(regenerated_text)
# Compare
assert regenerated_data == original_data
class TestRoundtripXML:
"""Roundtrip tests for XML files."""
def test_xml_simple_roundtrip(self):
"""Test simple XML can be regenerated."""
xml_text = """<?xml version="1.0"?>
<config>
<name>test</name>
<port>8080</port>
<server>server1</server>
<server>server2</server>
<server>server3</server>
</config>
"""
import xml.etree.ElementTree as ET
original_root = ET.fromstring(xml_text)
# Generate template and YAML
fmt = "xml"
loop_candidates = analyze_loops(fmt, original_root)
flat_items = flatten_config(fmt, original_root, loop_candidates)
ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
template = generate_jinja2_template(
fmt, original_root, "test", xml_text, loop_candidates
)
# Render template
variables = yaml.safe_load(ansible_yaml)
regenerated_text = render_template(template, variables)
# Parse regenerated XML
regenerated_root = ET.fromstring(regenerated_text)
# Compare XML structures (ignore insignificant whitespace)
assert _xml_elements_equal(
original_root, regenerated_root, ignore_whitespace=True
), (
f"Regenerated XML differs from original\n"
f"Original: {ET.tostring(original_root, encoding='unicode')}\n"
f"Regenerated: {ET.tostring(regenerated_root, encoding='unicode')}"
)
def test_ossec_xml_roundtrip(self):
"""Test ossec.xml (complex real-world XML) roundtrip."""
samples_dir = Path(__file__).parent / "samples"
xml_file = samples_dir / "ossec.xml"
if not xml_file.exists():
pytest.skip("ossec.xml not found")
# Read original
original_text = xml_file.read_text()
import xml.etree.ElementTree as ET
original_root = ET.fromstring(original_text)
# Generate template and YAML
fmt, parsed = parse_config(xml_file)
loop_candidates = analyze_loops(fmt, parsed)
flat_items = flatten_config(fmt, parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
template = generate_jinja2_template(
fmt, parsed, "test", original_text, loop_candidates
)
# Load variables and render
variables = yaml.safe_load(ansible_yaml)
regenerated_text = render_template(template, variables)
# Parse regenerated
regenerated_root = ET.fromstring(regenerated_text)
# Compare - for complex XML, we compare structure not exact text
assert _xml_elements_equal(
original_root, regenerated_root, ignore_whitespace=True
)
class TestRoundtripINI:
"""Roundtrip tests for INI files."""
def test_ini_simple_roundtrip(self):
"""Test simple INI can be regenerated."""
ini_text = """[section1]
key1 = value1
key2 = value2
[section2]
key3 = value3
"""
from configparser import ConfigParser
original_config = ConfigParser()
original_config.read_string(ini_text)
# Generate template and YAML
fmt = "ini"
loop_candidates = analyze_loops(fmt, original_config)
flat_items = flatten_config(fmt, original_config, loop_candidates)
ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
template = generate_jinja2_template(
fmt, original_config, "test", ini_text, loop_candidates
)
# Render template
variables = yaml.safe_load(ansible_yaml)
regenerated_text = render_template(template, variables)
# Parse regenerated
regenerated_config = ConfigParser()
regenerated_config.read_string(regenerated_text)
# Compare
assert _ini_configs_equal(original_config, regenerated_config)
class TestRoundtripEdgeCases:
"""Roundtrip tests for edge cases and special scenarios."""
def test_empty_lists_roundtrip(self):
"""Test handling of empty lists."""
json_text = '{"items": []}'
original_data = json.loads(json_text)
loop_candidates = analyze_loops("json", original_data)
flat_items = flatten_config("json", original_data, loop_candidates)
ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
template = generate_jinja2_template(
"json", original_data, "test", None, loop_candidates
)
variables = yaml.safe_load(ansible_yaml)
regenerated_text = render_template(template, variables)
regenerated_data = json.loads(regenerated_text)
assert regenerated_data == original_data
def test_special_characters_roundtrip(self):
"""Test handling of special characters."""
json_data = {
"quote": 'He said "hello"',
"backslash": "path\\to\\file",
"newline": "line1\nline2",
"unicode": "emoji: 🚀",
}
loop_candidates = analyze_loops("json", json_data)
flat_items = flatten_config("json", json_data, loop_candidates)
ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
template = generate_jinja2_template(
"json", json_data, "test", None, loop_candidates
)
variables = yaml.safe_load(ansible_yaml)
regenerated_text = render_template(template, variables)
regenerated_data = json.loads(regenerated_text)
assert regenerated_data == json_data
def test_numeric_types_roundtrip(self):
"""Test preservation of numeric types."""
json_data = {
"int": 42,
"float": 3.14159,
"negative": -100,
"zero": 0,
"large": 9999999999,
}
loop_candidates = analyze_loops("json", json_data)
flat_items = flatten_config("json", json_data, loop_candidates)
ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
template = generate_jinja2_template(
"json", json_data, "test", None, loop_candidates
)
variables = yaml.safe_load(ansible_yaml)
regenerated_text = render_template(template, variables)
regenerated_data = json.loads(regenerated_text)
assert regenerated_data == json_data
def test_boolean_preservation_roundtrip(self):
"""Test that booleans are preserved correctly."""
yaml_text = """
enabled: true
disabled: false
"""
original_data = yaml.safe_load(yaml_text)
loop_candidates = analyze_loops("yaml", original_data)
flat_items = flatten_config("yaml", original_data, loop_candidates)
ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
template = generate_jinja2_template(
"yaml", original_data, "test", yaml_text, loop_candidates
)
variables = yaml.safe_load(ansible_yaml)
regenerated_text = render_template(template, variables)
regenerated_data = yaml.safe_load(regenerated_text)
# Both should be actual booleans
assert regenerated_data["enabled"] is True
assert regenerated_data["disabled"] is False
# Helper functions
def _compare_toml_data(data1: Any, data2: Any) -> bool:
"""Compare TOML data, handling datetime objects."""
import datetime
if type(data1) != type(data2):
return False
if isinstance(data1, dict):
if set(data1.keys()) != set(data2.keys()):
return False
return all(_compare_toml_data(data1[k], data2[k]) for k in data1.keys())
elif isinstance(data1, list):
if len(data1) != len(data2):
return False
return all(_compare_toml_data(v1, v2) for v1, v2 in zip(data1, data2))
elif isinstance(data1, datetime.datetime):
# Compare datetime objects
return data1 == data2
else:
return data1 == data2
def _xml_elements_equal(elem1, elem2, ignore_whitespace: bool = False) -> bool:
"""Compare two XML elements for equality."""
# Compare tags
if elem1.tag != elem2.tag:
return False
# Compare attributes
if elem1.attrib != elem2.attrib:
return False
# Compare text
text1 = (elem1.text or "").strip() if ignore_whitespace else (elem1.text or "")
text2 = (elem2.text or "").strip() if ignore_whitespace else (elem2.text or "")
if text1 != text2:
return False
# Compare tail
tail1 = (elem1.tail or "").strip() if ignore_whitespace else (elem1.tail or "")
tail2 = (elem2.tail or "").strip() if ignore_whitespace else (elem2.tail or "")
if tail1 != tail2:
return False
# Compare children
children1 = list(elem1)
children2 = list(elem2)
if len(children1) != len(children2):
return False
return all(
_xml_elements_equal(c1, c2, ignore_whitespace)
for c1, c2 in zip(children1, children2)
)
def _ini_configs_equal(config1, config2) -> bool:
"""Compare two ConfigParser objects for equality."""
if set(config1.sections()) != set(config2.sections()):
return False
for section in config1.sections():
if set(config1.options(section)) != set(config2.options(section)):
return False
for option in config1.options(section):
if config1.get(section, option) != config2.get(section, option):
return False
return True
if __name__ == "__main__":
pytest.main([__file__, "-v"])

114
tests/test_toml_handler.py
View file

@ -1,114 +0,0 @@
from __future__ import annotations
from pathlib import Path
import pytest
import yaml
from jinjaturtle.core import (
parse_config,
flatten_config,
generate_ansible_yaml,
generate_jinja2_template,
)
from jinjaturtle.handlers.toml import TomlHandler
import jinjaturtle.handlers.toml as toml_module
SAMPLES_DIR = Path(__file__).parent / "samples"
def test_toml_sample_roundtrip():
toml_path = SAMPLES_DIR / "tom.toml"
assert toml_path.is_file(), f"Missing sample TOML file: {toml_path}"
fmt, parsed = parse_config(toml_path)
assert fmt == "toml"
flat_items = flatten_config(fmt, parsed)
assert flat_items
ansible_yaml = generate_ansible_yaml("jinjaturtle", flat_items)
defaults = yaml.safe_load(ansible_yaml)
# defaults should be a non-empty dict
assert isinstance(defaults, dict)
assert defaults, "Expected non-empty defaults for TOML sample"
# all keys should be lowercase, start with prefix, and have no spaces
for key in defaults:
assert key.startswith("jinjaturtle_")
assert key == key.lower()
assert " " not in key
# template generation – **now with original_text**
original_text = toml_path.read_text(encoding="utf-8")
template = generate_jinja2_template(
fmt, parsed, "jinjaturtle", original_text=original_text
)
assert isinstance(template, str)
assert template.strip()
# comments from the original file should now be preserved
assert "# This is a TOML document" in template
# each default variable name should appear in the template as a Jinja placeholder
for var_name in defaults:
assert (
var_name in template
), f"Variable {var_name} not referenced in TOML template"
def test_parse_config_toml_missing_tomllib(monkeypatch):
"""
Force tomllib to None to hit the RuntimeError branch when parsing TOML.
"""
toml_path = SAMPLES_DIR / "tom.toml"
# Simulate an environment without tomllib/tomli
monkeypatch.setattr(toml_module, "tomllib", None)
with pytest.raises(RuntimeError) as exc:
parse_config(toml_path, fmt="toml")
assert "tomllib/tomli is required" in str(exc.value)
def test_generate_jinja2_template_fallback_toml():
"""
When original_text is not provided, generate_jinja2_template should use the
structural fallback path for TOML configs.
"""
parsed_toml = {
"title": "Example",
"server": {"port": 8080, "host": "127.0.0.1"},
"logging": {
"file": {"path": "/tmp/app.log"}
}, # nested table to hit recursive walk
}
tmpl_toml = generate_jinja2_template("toml", parsed=parsed_toml, role_prefix="role")
assert "[server]" in tmpl_toml
assert "role_server_port" in tmpl_toml
assert "[logging]" in tmpl_toml or "[logging.file]" in tmpl_toml
def test_generate_toml_template_from_text_edge_cases():
# Cover CRLF newlines, lines without '=', empty keys, and inline tables
# that both parse successfully and fail parsing.
text = (
"# comment\r\n"
"[table]\r\n"
"noequals\r\n"
" = 42\r\n"
'inline_good = { name = "abc", value = 1 }\r\n'
"inline_bad = { invalid = }\r\n"
)
handler = TomlHandler()
tmpl = handler._generate_toml_template_from_text("role", text)
# The good inline table should expand into two separate variables.
assert "role_table_inline_good_name" in tmpl
assert "role_table_inline_good_value" in tmpl
# The bad inline table should fall back to scalar handling.
assert "role_table_inline_bad" in tmpl
# Ensure the lines without '=' / empty key were handled without exploding.
assert "[table]" in tmpl
assert "noequals" in tmpl

232
tests/test_xml_handler.py
View file

@ -1,232 +0,0 @@
from __future__ import annotations
from pathlib import Path
import textwrap
import xml.etree.ElementTree as ET
import pytest
import yaml
from jinjaturtle.core import (
parse_config,
flatten_config,
generate_ansible_yaml,
generate_jinja2_template,
)
from jinjaturtle.handlers.xml import XmlHandler
SAMPLES_DIR = Path(__file__).parent / "samples"
def test_xml_roundtrip_ossec_web_rules():
xml_path = SAMPLES_DIR / "ossec.xml"
assert xml_path.is_file(), f"Missing sample XML file: {xml_path}"
fmt, parsed = parse_config(xml_path)
assert fmt == "xml"
flat_items = flatten_config(fmt, parsed)
assert flat_items, "Expected at least one flattened item from XML sample"
ansible_yaml = generate_ansible_yaml("ossec", flat_items)
defaults = yaml.safe_load(ansible_yaml)
# defaults should be a non-empty dict
assert isinstance(defaults, dict)
assert defaults, "Expected non-empty defaults for XML sample"
# all keys should be lowercase, start with prefix, and have no spaces
for key in defaults:
assert key.startswith("ossec_")
assert key == key.lower()
assert " " not in key
# Root <group name="web,accesslog,"> attribute should flatten to ossec_name
assert defaults["ossec_name"] == "web,accesslog,"
# There should be at least one default for rule id="31100"
id_keys = [k for k, v in defaults.items() if v == "31100"]
assert id_keys, "Expected to find a default for rule id 31100"
# At least one of them should be the rule *id* attribute
assert any(
key.startswith("ossec_rule_") and key.endswith("_id") for key in id_keys
), f"Expected at least one *_id var for value 31100, got: {id_keys}"
# Template generation (preserving comments)
original_text = xml_path.read_text(encoding="utf-8")
template = generate_jinja2_template(
fmt, parsed, "ossec", original_text=original_text
)
assert isinstance(template, str)
assert template.strip(), "Template for XML sample should not be empty"
# Top-of-file and mid-file comments should be preserved
assert "Official Web access rules for OSSEC." in template
assert "Rules to ignore crawlers" in template
# Each default variable name should appear in the template as a Jinja placeholder
for var_name in defaults:
assert (
var_name in template
), f"Variable {var_name} not referenced in XML template"
def test_generate_xml_template_from_text_edge_cases():
"""
Exercise XML text edge cases:
- XML declaration and DOCTYPE in prolog
- top-level and inner comments
- repeated child elements (indexing)
- attributes and text content
"""
text = textwrap.dedent(
"""\
<?xml version="1.0"?>
<!-- top comment -->
<!DOCTYPE something>
<root attr="1">
<!-- inner comment -->
<child attr="2">text</child>
<child>other</child>
</root>
"""
)
handler = XmlHandler()
tmpl = handler._generate_xml_template_from_text("role", text)
# Prolog and comments preserved
assert "<?xml version" in tmpl
assert "top comment" in tmpl
assert "inner comment" in tmpl
# Root attribute becomes a variable (path ("@attr",) -> role_attr)
assert "role_attr" in tmpl
# Repeated <child> elements should be indexed in both attr and text
assert "role_child_0_attr" in tmpl
assert "role_child_0" in tmpl
assert "role_child_1" in tmpl
def test_generate_jinja2_template_xml_type_error():
"""
Wrong type for XML in XmlHandler.generate_jinja2_template should raise TypeError.
"""
handler = XmlHandler()
with pytest.raises(TypeError):
handler.generate_jinja2_template(parsed="not an element", role_prefix="role")
def test_flatten_config_xml_type_error():
"""
Wrong type for XML in flatten_config should raise TypeError.
"""
with pytest.raises(TypeError):
flatten_config("xml", parsed="not-an-element")
def test_generate_jinja2_template_xml_structural_fallback():
"""
When original_text is not provided for XML, generate_jinja2_template should use
the structural fallback path (ET.tostring + handler processing).
"""
xml_text = textwrap.dedent(
"""\
<root attr="1">
<child>2</child>
<node attr="x">text</node>
</root>
"""
)
root = ET.fromstring(xml_text)
tmpl = generate_jinja2_template("xml", parsed=root, role_prefix="role")
# Root attribute path ("@attr",) -> role_attr
assert "role_attr" in tmpl
# Simple child element text ("child",) -> role_child
assert "role_child" in tmpl
# Element with both attr and text:
# - attr -> ("node", "@attr") -> role_node_attr
# - text -> ("node", "value") -> role_node_value
assert "role_node_attr" in tmpl
assert "role_node_value" in tmpl
def test_split_xml_prolog_only_whitespace():
"""
Whitespace-only input: prolog is the whitespace, body is empty.
Exercises the 'if i >= n: break' path.
"""
text = " \n\t"
handler = XmlHandler()
prolog, body = handler._split_xml_prolog(text)
assert prolog == text
assert body == ""
def test_split_xml_prolog_unterminated_declaration():
"""
Unterminated XML declaration should hit the 'end == -1' branch and
treat the whole string as body.
"""
text = "<?xml version='1.0'"
handler = XmlHandler()
prolog, body = handler._split_xml_prolog(text)
assert prolog == ""
assert body == text
def test_split_xml_prolog_unterminated_comment():
"""
Unterminated comment should likewise hit its 'end == -1' branch.
"""
text = "<!-- no end"
handler = XmlHandler()
prolog, body = handler._split_xml_prolog(text)
assert prolog == ""
assert body == text
def test_split_xml_prolog_unterminated_doctype():
"""
Unterminated DOCTYPE should hit the DOCTYPE 'end == -1' branch.
"""
text = "<!DOCTYPE foo"
handler = XmlHandler()
prolog, body = handler._split_xml_prolog(text)
assert prolog == ""
assert body == text
def test_split_xml_prolog_unexpected_content():
"""
Non-XML content at the start should trigger the 'unexpected content'
break and be returned entirely as body.
"""
text = "garbage<root/>"
handler = XmlHandler()
prolog, body = handler._split_xml_prolog(text)
assert prolog == ""
assert body == text
def test_flatten_xml_text_with_attributes_uses_value_suffix():
"""
When an element has both attributes and text, _flatten_xml should store
the text at path + ('value',), not just path.
"""
xml_text = "<root><node attr='x'>text</node></root>"
root = ET.fromstring(xml_text)
items = flatten_config("xml", root)
# Attribute path: ("node", "@attr") -> "x"
assert (("node", "@attr"), "x") in items
# Text-with-attrs path: ("node", "value") -> "text"
assert (("node", "value"), "text") in items

102
tests/test_yaml_handler.py
View file

@ -1,102 +0,0 @@
from __future__ import annotations
from pathlib import Path
import textwrap
import yaml
from jinjaturtle.core import (
parse_config,
flatten_config,
generate_ansible_yaml,
generate_jinja2_template,
)
from jinjaturtle.handlers.yaml import YamlHandler
SAMPLES_DIR = Path(__file__).parent / "samples"
def test_yaml_roundtrip_with_list_and_comment():
yaml_path = SAMPLES_DIR / "bar.yaml"
assert yaml_path.is_file(), f"Missing sample YAML file: {yaml_path}"
fmt, parsed = parse_config(yaml_path)
assert fmt == "yaml"
flat_items = flatten_config(fmt, parsed)
ansible_yaml = generate_ansible_yaml("foobar", flat_items)
defaults = yaml.safe_load(ansible_yaml)
# Defaults: keys are flattened with indices
assert defaults["foobar_foo"] == "bar"
assert defaults["foobar_blah_0"] == "something"
assert defaults["foobar_blah_1"] == "else"
# Template generation (preserving comments)
original_text = yaml_path.read_text(encoding="utf-8")
template = generate_jinja2_template(
fmt, parsed, "foobar", original_text=original_text
)
# Comment preserved
assert "# Top comment" in template
# Scalar replacement
assert "foo:" in template
assert "foobar_foo" in template
# List items use indexed vars, not "item"
assert "foobar_blah_0" in template
assert "foobar_blah_1" in template
assert "{{ foobar_blah }}" not in template
assert "foobar_blah_item" not in template
def test_generate_yaml_template_from_text_edge_cases():
"""
Exercise YAML text edge cases:
- indentation dedent (stack pop)
- empty key before ':'
- quoted and unquoted list items
"""
text = textwrap.dedent(
"""
root:
child: 1
other: 2
: 3
list:
- "quoted"
- unquoted
"""
)
handler = YamlHandler()
tmpl = handler._generate_yaml_template_from_text("role", text)
# Dedent from "root -> child" back to "other" exercises the stack-pop path.
# Just check the expected variable names appear.
assert "role_root_child" in tmpl
assert "role_other" in tmpl
# The weird " : 3" line has no key and should be left untouched.
assert " : 3" in tmpl
# The list should generate indexed variables for each item.
# First item is quoted (use_quotes=True), second is unquoted.
assert "role_list_0" in tmpl
assert "role_list_1" in tmpl
def test_generate_jinja2_template_yaml_structural_fallback():
"""
When original_text is not provided for YAML, generate_jinja2_template should use
the structural fallback path (yaml.safe_dump + handler processing).
"""
parsed = {"outer": {"inner": "val"}}
tmpl = generate_jinja2_template("yaml", parsed=parsed, role_prefix="role")
# We don't care about exact formatting, just that the expected variable
# name shows up, proving we went through the structural path.
assert "role_outer_inner" in tmpl

558
tests/test_yaml_template_consistency.py
View file

@ -1,558 +0,0 @@
"""
Tests to ensure all Jinja2 template variables exist in the Ansible YAML.
These tests catch the bug where templates reference variables that don't exist
because the YAML has a list but the template uses scalar references (or vice versa).
"""
from __future__ import annotations
import re
from pathlib import Path
from typing import Set
import yaml
import pytest
from jinjaturtle.core import (
parse_config,
analyze_loops,
flatten_config,
generate_ansible_yaml,
generate_jinja2_template,
)
def extract_jinja_variables(template: str) -> Set[str]:
"""
Extract all Jinja2 variable names from a template that must exist in YAML.
Extracts variables from:
- {{ variable_name }}
- {{ variable.field }}
- {% for item in collection %}
Returns only the base variable names that must be defined in YAML.
Filters out loop variables (the 'item' part of 'for item in collection').
"""
variables = set()
# First, find all loop variables (these are defined by the template, not YAML)
loop_vars = set()
for_pattern = r"\{%\s*for\s+(\w+)\s+in\s+([a-zA-Z_][a-zA-Z0-9_]*)"
for match in re.finditer(for_pattern, template):
loop_var = match.group(1) # The item
collection = match.group(2) # The collection
loop_vars.add(loop_var)
variables.add(collection) # Collection must exist in YAML
# Pattern 1: {{ variable_name }} or {{ variable.field }}
# Captures the first part before any dots or filters
var_pattern = r"\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)"
for match in re.finditer(var_pattern, template):
var_name = match.group(1)
# Only add if it's not a loop variable
if var_name not in loop_vars:
variables.add(var_name)
return variables
def extract_yaml_variables(ansible_yaml: str) -> Set[str]:
"""
Extract all variable names from Ansible YAML.
Returns the top-level keys from the YAML document.
"""
data = yaml.safe_load(ansible_yaml)
if not isinstance(data, dict):
return set()
return set(data.keys())
class TestTemplateYamlConsistency:
"""Tests that verify template variables exist in YAML."""
def test_simple_json_consistency(self):
"""Simple JSON with scalars and lists."""
json_text = """
{
"name": "test",
"values": [1, 2, 3]
}
"""
fmt = "json"
import json
parsed = json.loads(json_text)
loop_candidates = analyze_loops(fmt, parsed)
flat_items = flatten_config(fmt, parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
template = generate_jinja2_template(fmt, parsed, "app", None, loop_candidates)
yaml_vars = extract_yaml_variables(ansible_yaml)
template_vars = extract_jinja_variables(template)
# Every variable in template must exist in YAML
missing_vars = template_vars - yaml_vars
assert not missing_vars, (
f"Template references variables not in YAML: {missing_vars}\n"
f"YAML vars: {yaml_vars}\n"
f"Template vars: {template_vars}\n"
f"Template:\n{template}\n"
f"YAML:\n{ansible_yaml}"
)
def test_toml_inline_array_consistency(self):
"""TOML with inline array should use loops consistently."""
import tomllib
toml_text = """
name = "myapp"
servers = ["server1", "server2", "server3"]
"""
parsed = tomllib.loads(toml_text)
loop_candidates = analyze_loops("toml", parsed)
flat_items = flatten_config("toml", parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
template = generate_jinja2_template(
"toml", parsed, "app", toml_text, loop_candidates
)
yaml_vars = extract_yaml_variables(ansible_yaml)
template_vars = extract_jinja_variables(template)
missing_vars = template_vars - yaml_vars
assert not missing_vars, (
f"Template references variables not in YAML: {missing_vars}\n"
f"Template:\n{template}\n"
f"YAML:\n{ansible_yaml}"
)
def test_toml_array_of_tables_consistency(self):
"""TOML with [[array.of.tables]] should use loops consistently."""
import tomllib
toml_text = """
[[database]]
host = "db1.example.com"
port = 5432
[[database]]
host = "db2.example.com"
port = 5433
"""
parsed = tomllib.loads(toml_text)
loop_candidates = analyze_loops("toml", parsed)
flat_items = flatten_config("toml", parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
template = generate_jinja2_template(
"toml", parsed, "app", toml_text, loop_candidates
)
yaml_vars = extract_yaml_variables(ansible_yaml)
template_vars = extract_jinja_variables(template)
missing_vars = template_vars - yaml_vars
assert not missing_vars, (
f"Template references variables not in YAML: {missing_vars}\n"
f"Template:\n{template}\n"
f"YAML:\n{ansible_yaml}"
)
# Additionally verify that if YAML has a list, template uses a loop
defaults = yaml.safe_load(ansible_yaml)
for var_name, value in defaults.items():
if isinstance(value, list) and len(value) > 1:
# YAML has a list - template should use {% for %}
assert "{% for" in template, (
f"YAML has list variable '{var_name}' but template doesn't use loops\n"
f"Template:\n{template}"
)
def test_yaml_list_consistency(self):
"""YAML with lists should use loops consistently."""
yaml_text = """
name: myapp
servers:
- server1
- server2
- server3
databases:
- host: db1
port: 5432
- host: db2
port: 5433
"""
parsed = yaml.safe_load(yaml_text)
loop_candidates = analyze_loops("yaml", parsed)
flat_items = flatten_config("yaml", parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
template = generate_jinja2_template(
"yaml", parsed, "app", yaml_text, loop_candidates
)
yaml_vars = extract_yaml_variables(ansible_yaml)
template_vars = extract_jinja_variables(template)
missing_vars = template_vars - yaml_vars
assert not missing_vars, (
f"Template references variables not in YAML: {missing_vars}\n"
f"Template:\n{template}\n"
f"YAML:\n{ansible_yaml}"
)
def test_mixed_scalars_and_loops_consistency(self):
"""Config with both scalars and loops should be consistent."""
import tomllib
toml_text = """
name = "myapp"
version = "1.0"
ports = [8080, 8081, 8082]
[database]
host = "localhost"
port = 5432
[[servers]]
name = "web1"
ip = "10.0.0.1"
[[servers]]
name = "web2"
ip = "10.0.0.2"
"""
parsed = tomllib.loads(toml_text)
loop_candidates = analyze_loops("toml", parsed)
flat_items = flatten_config("toml", parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
template = generate_jinja2_template(
"toml", parsed, "app", toml_text, loop_candidates
)
yaml_vars = extract_yaml_variables(ansible_yaml)
template_vars = extract_jinja_variables(template)
missing_vars = template_vars - yaml_vars
assert not missing_vars, (
f"Template references variables not in YAML: {missing_vars}\n"
f"Template:\n{template}\n"
f"YAML:\n{ansible_yaml}"
)
def test_no_orphaned_scalar_references(self):
"""
When YAML has a list variable, template must NOT reference scalar indices.
This catches the bug where:
- YAML has: app_list: [1, 2, 3]
- Template incorrectly uses: {{ app_list_0 }}, {{ app_list_1 }}
"""
import json
json_text = '{"items": [1, 2, 3, 4, 5]}'
parsed = json.loads(json_text)
loop_candidates = analyze_loops("json", parsed)
flat_items = flatten_config("json", parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
template = generate_jinja2_template(
"json", parsed, "app", None, loop_candidates
)
defaults = yaml.safe_load(ansible_yaml)
# Check each list variable in YAML
for var_name, value in defaults.items():
if isinstance(value, list):
# Template should NOT reference app_items_0, app_items_1, etc.
for i in range(len(value)):
scalar_ref = f"{var_name}_{i}"
assert scalar_ref not in template, (
f"Template incorrectly uses scalar reference '{scalar_ref}' "
f"when YAML has '{var_name}' as a list\n"
f"Template should use loops, not scalar indices\n"
f"Template:\n{template}"
)
def test_all_sample_files_consistency(self):
"""Test all sample files for consistency."""
samples_dir = Path(__file__).parent / "samples"
sample_files = [
("foo.json", "json"),
("bar.yaml", "yaml"),
("tom.toml", "toml"),
]
for filename, fmt in sample_files:
file_path = samples_dir / filename
if not file_path.exists():
pytest.skip(f"Sample file {filename} not found")
original_text = file_path.read_text()
fmt_detected, parsed = parse_config(file_path)
loop_candidates = analyze_loops(fmt_detected, parsed)
flat_items = flatten_config(fmt_detected, parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
template = generate_jinja2_template(
fmt_detected, parsed, "test", original_text, loop_candidates
)
yaml_vars = extract_yaml_variables(ansible_yaml)
template_vars = extract_jinja_variables(template)
missing_vars = template_vars - yaml_vars
assert not missing_vars, (
f"File: {filename}\n"
f"Template references variables not in YAML: {missing_vars}\n"
f"YAML vars: {yaml_vars}\n"
f"Template vars: {template_vars}\n"
f"Template:\n{template}\n"
f"YAML:\n{ansible_yaml}"
)
class TestStructuralConsistency:
"""Tests that verify structural consistency between YAML and templates."""
def test_list_in_yaml_means_loop_in_template(self):
"""When YAML has a list (len > 1), template should use {% for %}."""
import json
json_text = """
{
"scalar": "value",
"list": [1, 2, 3]
}
"""
parsed = json.loads(json_text)
loop_candidates = analyze_loops("json", parsed)
flat_items = flatten_config("json", parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
template = generate_jinja2_template(
"json", parsed, "app", None, loop_candidates
)
defaults = yaml.safe_load(ansible_yaml)
# Find list variables in YAML
list_vars = [
k for k, v in defaults.items() if isinstance(v, list) and len(v) > 1
]
if list_vars:
# Template must contain for loops
assert "{% for" in template, (
f"YAML has list variables {list_vars} but template has no loops\n"
f"Template:\n{template}"
)
# Each list variable should be used in a for loop
for var_name in list_vars:
# Look for "{% for ... in var_name %}"
for_pattern = (
r"\{%\s*for\s+\w+\s+in\s+" + re.escape(var_name) + r"\s*%\}"
)
assert re.search(for_pattern, template), (
f"List variable '{var_name}' not used in a for loop\n"
f"Template:\n{template}"
)
def test_scalar_in_yaml_means_no_loop_in_template(self):
"""When YAML has scalars, template should use {{ var }}, not loops."""
import json
json_text = """
{
"name": "test",
"port": 8080,
"enabled": true
}
"""
parsed = json.loads(json_text)
loop_candidates = analyze_loops("json", parsed)
flat_items = flatten_config("json", parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
template = generate_jinja2_template(
"json", parsed, "app", None, loop_candidates
)
defaults = yaml.safe_load(ansible_yaml)
# All variables are scalars - template should NOT have loops
scalar_vars = [
k for k, v in defaults.items() if not isinstance(v, (list, dict))
]
# Check that scalar vars are used directly, not in loops
for var_name in scalar_vars:
# Should appear in {{ var_name }}, not {% for ... in var_name %}
direct_ref = f"{{{{ {var_name}"
loop_ref = f"for .* in {var_name}"
assert direct_ref in template, (
f"Scalar variable '{var_name}' should be directly referenced\n"
f"Template:\n{template}"
)
assert not re.search(loop_ref, template), (
f"Scalar variable '{var_name}' incorrectly used in a loop\n"
f"Template:\n{template}"
)
def test_no_undefined_variable_errors(self):
"""
Simulate Ansible template rendering to catch undefined variables.
This is the ultimate test - actually render the template with the YAML
and verify no undefined variable errors occur.
"""
from jinja2 import Environment, StrictUndefined
import json
json_text = """
{
"name": "myapp",
"servers": ["web1", "web2"],
"database": {
"host": "localhost",
"port": 5432
}
}
"""
parsed = json.loads(json_text)
loop_candidates = analyze_loops("json", parsed)
flat_items = flatten_config("json", parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
template = generate_jinja2_template(
"json", parsed, "app", None, loop_candidates
)
# Load variables from YAML
variables = yaml.safe_load(ansible_yaml)
# Try to render the template
env = Environment(undefined=StrictUndefined)
try:
jinja_template = env.from_string(template)
rendered = jinja_template.render(variables)
# Successfully rendered - this is what we want!
assert rendered, "Template rendered successfully"
except Exception as e:
pytest.fail(
f"Template rendering failed with variables from YAML\n"
f"Error: {e}\n"
f"Template:\n{template}\n"
f"Variables:\n{ansible_yaml}"
)
class TestRegressionBugs:
"""Tests for specific bugs that were found and fixed."""
def test_toml_array_of_tables_no_scalar_refs(self):
"""
Regression test: TOML [[array]] should not generate scalar references.
Bug: Template had {{ app_database_host }} when YAML had app_database as list.
"""
import tomllib
toml_text = """
[[database]]
host = "db1"
port = 5432
[[database]]
host = "db2"
port = 5433
"""
parsed = tomllib.loads(toml_text)
loop_candidates = analyze_loops("toml", parsed)
flat_items = flatten_config("toml", parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
template = generate_jinja2_template(
"toml", parsed, "app", toml_text, loop_candidates
)
# YAML should have app_database as a list
defaults = yaml.safe_load(ansible_yaml)
assert isinstance(
defaults.get("app_database"), list
), f"Expected app_database to be a list in YAML\n{ansible_yaml}"
# Template should NOT have app_database_host or app_database_port
assert (
"app_database_host" not in template
), f"Template incorrectly uses scalar 'app_database_host'\n{template}"
assert (
"app_database_port" not in template
), f"Template incorrectly uses scalar 'app_database_port'\n{template}"
# Template SHOULD use a loop
assert "{% for" in template, f"Template should use a loop\n{template}"
assert (
"app_database" in template
), f"Template should reference app_database\n{template}"
def test_json_array_no_index_refs(self):
"""
Regression test: JSON arrays should not generate index references.
Bug: Template had {{ app_list_0 }}, {{ app_list_1 }} when YAML had app_list as list.
"""
import json
json_text = '{"items": [1, 2, 3]}'
parsed = json.loads(json_text)
loop_candidates = analyze_loops("json", parsed)
flat_items = flatten_config("json", parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
template = generate_jinja2_template(
"json", parsed, "app", None, loop_candidates
)
# YAML should have app_items as a list
defaults = yaml.safe_load(ansible_yaml)
assert isinstance(defaults.get("app_items"), list)
# Template should NOT have app_items_0, app_items_1, app_items_2
for i in range(3):
assert (
f"app_items_{i}" not in template
), f"Template incorrectly uses scalar 'app_items_{i}'\n{template}"
# Template SHOULD use a loop
assert "{% for" in template
assert "app_items" in template
if __name__ == "__main__":
pytest.main([__file__, "-v"])

218
utils/diff_configs.py
View file

@ -1,218 +0,0 @@
#!/usr/bin/env python3
"""
Side-by-side comparison of original vs regenerated config.
Usage:
./diff_configs.py tests/samples/foo.json
./diff_configs.py tests/samples/tom.toml --context 5
"""
import argparse
import sys
from pathlib import Path
import difflib
import yaml
from jinja2 import Environment, StrictUndefined
# Add parent directory to path for imports
sys.path.insert(0, str(Path(__file__).parent))
from jinjaturtle.core import (
parse_config,
analyze_loops,
flatten_config,
generate_ansible_yaml,
generate_jinja2_template,
)
def colorize(text: str, color: str) -> str:
"""Add ANSI color codes."""
colors = {
"red": "\033[91m",
"green": "\033[92m",
"yellow": "\033[93m",
"blue": "\033[94m",
"reset": "\033[0m",
}
return f"{colors.get(color, '')}{text}{colors['reset']}"
def side_by_side_diff(original: str, regenerated: str, width: int = 80):
"""Print side-by-side diff."""
orig_lines = original.splitlines()
regen_lines = regenerated.splitlines()
# Calculate column width
col_width = width // 2 - 3
print(
colorize("ORIGINAL".center(col_width), "blue")
+ " | "
+ colorize("REGENERATED".center(col_width), "green")
)
print("-" * col_width + "-+-" + "-" * col_width)
max_lines = max(len(orig_lines), len(regen_lines))
for i in range(max_lines):
orig_line = orig_lines[i] if i < len(orig_lines) else ""
regen_line = regen_lines[i] if i < len(regen_lines) else ""
# Truncate if too long
if len(orig_line) > col_width - 2:
orig_line = orig_line[: col_width - 5] + "..."
if len(regen_line) > col_width - 2:
regen_line = regen_line[: col_width - 5] + "..."
# Color lines if different
if orig_line != regen_line:
orig_display = colorize(orig_line.ljust(col_width), "red")
regen_display = colorize(regen_line.ljust(col_width), "green")
else:
orig_display = orig_line.ljust(col_width)
regen_display = regen_line.ljust(col_width)
print(f"{orig_display} | {regen_display}")
def unified_diff(original: str, regenerated: str, filename: str, context: int = 3):
"""Print unified diff."""
orig_lines = original.splitlines(keepends=True)
regen_lines = regenerated.splitlines(keepends=True)
diff = difflib.unified_diff(
orig_lines,
regen_lines,
fromfile=f"{filename} (original)",
tofile=f"{filename} (regenerated)",
n=context,
)
for line in diff:
if line.startswith("+++") or line.startswith("---"):
print(colorize(line.rstrip(), "blue"))
elif line.startswith("@@"):
print(colorize(line.rstrip(), "cyan"))
elif line.startswith("+"):
print(colorize(line.rstrip(), "green"))
elif line.startswith("-"):
print(colorize(line.rstrip(), "red"))
else:
print(line.rstrip())
def main():
parser = argparse.ArgumentParser(
description="Compare original config with regenerated version",
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument("file", type=Path, help="Config file to check")
parser.add_argument(
"--mode",
choices=["side-by-side", "unified", "both"],
default="both",
help="Comparison mode (default: both)",
)
parser.add_argument(
"--context",
type=int,
default=3,
help="Number of context lines for unified diff (default: 3)",
)
parser.add_argument(
"--width",
type=int,
default=160,
help="Terminal width for side-by-side (default: 160)",
)
args = parser.parse_args()
if not args.file.exists():
print(colorize(f"❌ File not found: {args.file}", "red"))
return 1
print(colorize(f"\n{'=' * 80}", "blue"))
print(colorize(f" Comparing: {args.file}", "blue"))
print(colorize(f"{'=' * 80}\n", "blue"))
# Read and regenerate
try:
original_text = args.file.read_text()
fmt, parsed = parse_config(args.file)
loop_candidates = analyze_loops(fmt, parsed)
flat_items = flatten_config(fmt, parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
template = generate_jinja2_template(
fmt, parsed, "app", original_text, loop_candidates
)
variables = yaml.safe_load(ansible_yaml)
env = Environment(undefined=StrictUndefined)
jinja_template = env.from_string(template)
regenerated_text = jinja_template.render(variables)
# Check if identical
if original_text.strip() == regenerated_text.strip():
print(colorize("✅ Files are IDENTICAL (text comparison)\n", "green"))
else:
# Show diff
if args.mode in ("unified", "both"):
print(colorize("\n--- UNIFIED DIFF ---\n", "yellow"))
unified_diff(
original_text, regenerated_text, args.file.name, args.context
)
if args.mode in ("side-by-side", "both"):
print(colorize("\n--- SIDE-BY-SIDE COMPARISON ---\n", "yellow"))
side_by_side_diff(original_text, regenerated_text, args.width)
# Try semantic comparison
print(colorize(f"\n{'=' * 80}", "cyan"))
print(colorize(" Semantic Comparison", "cyan"))
print(colorize(f"{'=' * 80}", "cyan"))
try:
if fmt == "json":
import json
if json.loads(original_text) == json.loads(regenerated_text):
print(colorize("✅ JSON data structures are IDENTICAL", "green"))
else:
print(colorize("⚠️ JSON data structures DIFFER", "yellow"))
elif fmt == "yaml":
if yaml.safe_load(original_text) == yaml.safe_load(regenerated_text):
print(colorize("✅ YAML data structures are IDENTICAL", "green"))
else:
print(colorize("⚠️ YAML data structures DIFFER", "yellow"))
elif fmt == "toml":
try:
import tomllib
except Exception:
import tomli as tomllib
if tomllib.loads(original_text) == tomllib.loads(regenerated_text):
print(colorize("✅ TOML data structures are IDENTICAL", "green"))
else:
print(colorize("⚠️ TOML data structures DIFFER", "yellow"))
except Exception as e:
print(colorize(f"ℹ️ Could not compare semantically: {e}", "yellow"))
except Exception as e:
print(colorize(f"❌ ERROR: {e}", "red"))
import traceback
traceback.print_exc()
return 1
return 0
if __name__ == "__main__":
sys.exit(main())

162
utils/regenerate.py
View file

@ -1,162 +0,0 @@
#!/usr/bin/env python3
"""
Regenerate config files and save all intermediate files.
Creates:
- original.{ext}
- defaults/main.yml
- templates/config.j2
- regenerated.{ext}
Usage:
./regenerate.py tests/samples/foo.json
./regenerate.py tests/samples/tom.toml --output-dir tmp/toml_test
"""
import argparse
import sys
from pathlib import Path
import yaml
from jinja2 import Environment, StrictUndefined
# Add parent directory to path for imports
sys.path.insert(0, str(Path(__file__).parent))
from jinjaturtle.core import (
parse_config,
analyze_loops,
flatten_config,
generate_ansible_yaml,
generate_jinja2_template,
)
def regenerate_and_save(config_file: Path, output_dir: Path, role_prefix: str = "app"):
"""
Regenerate config and save all intermediate files.
"""
output_dir.mkdir(parents=True, exist_ok=True)
# Read original
original_text = config_file.read_text()
fmt, parsed = parse_config(config_file)
# Determine extension
ext = config_file.suffix
# Save original
original_out = output_dir / f"original{ext}"
original_out.write_text(original_text)
print(f"📄 Saved: {original_out}")
# Generate Ansible files
loop_candidates = analyze_loops(fmt, parsed)
flat_items = flatten_config(fmt, parsed, loop_candidates)
ansible_yaml = generate_ansible_yaml(role_prefix, flat_items, loop_candidates)
template = generate_jinja2_template(
fmt, parsed, role_prefix, original_text, loop_candidates
)
# Save Ansible YAML
defaults_dir = output_dir / "defaults"
defaults_dir.mkdir(exist_ok=True)
defaults_file = defaults_dir / "main.yml"
defaults_file.write_text(ansible_yaml)
print(f"📄 Saved: {defaults_file}")
# Save template
templates_dir = output_dir / "templates"
templates_dir.mkdir(exist_ok=True)
template_file = templates_dir / "config.j2"
template_file.write_text(template)
print(f"📄 Saved: {template_file}")
# Render template
variables = yaml.safe_load(ansible_yaml)
env = Environment(undefined=StrictUndefined)
jinja_template = env.from_string(template)
regenerated_text = jinja_template.render(variables)
# Save regenerated
regenerated_out = output_dir / f"regenerated{ext}"
regenerated_out.write_text(regenerated_text)
print(f"📄 Saved: {regenerated_out}")
# Summary
print(f"\n✅ All files saved to: {output_dir}")
print("\n📊 Statistics:")
print(f" Format: {fmt}")
print(f" Loop candidates: {len(loop_candidates)}")
if loop_candidates:
print(" Loops detected:")
for c in loop_candidates:
print(f" - {'.'.join(c.path)}: {len(c.items)} items")
# Check if identical
if original_text.strip() == regenerated_text.strip():
print("\n✅ Original and regenerated are IDENTICAL (text comparison)")
else:
print("\n⚠️ Original and regenerated differ in whitespace/formatting")
print(f" Run: diff {original_out} {regenerated_out}")
return output_dir
def main():
parser = argparse.ArgumentParser(
description="Regenerate config and save all intermediate files",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
%(prog)s tests/samples/foo.json
%(prog)s tests/samples/tom.toml -o tmp/toml_output
%(prog)s tests/samples/bar.yaml --role-prefix myapp
""",
)
parser.add_argument("file", type=Path, help="Config file to process")
parser.add_argument(
"-o",
"--output-dir",
type=Path,
help="Output directory (default: regenerated_<filename>)",
)
parser.add_argument(
"-r",
"--role-prefix",
default="app",
help="Ansible role prefix for variables (default: app)",
)
args = parser.parse_args()
if not args.file.exists():
print(f"❌ File not found: {args.file}")
return 1
# Determine output directory
if args.output_dir:
output_dir = args.output_dir
else:
output_dir = Path(f"regenerated_{args.file.stem}")
print(f"🔄 Regenerating: {args.file}")
print(f"📁 Output directory: {output_dir}")
print(f"🏷️ Role prefix: {args.role_prefix}\n")
try:
regenerate_and_save(args.file, output_dir, args.role_prefix)
return 0
except Exception as e:
print(f"\n❌ ERROR: {e}")
import traceback
traceback.print_exc()
return 1
if __name__ == "__main__":
sys.exit(main())