Make templates more faithful to the original file in terms of indentation, newlines, no deserialisation of things like < or >. More test coverage
This commit is contained in:
parent
3d53d4fb30
commit
49fee7afe4
GPG key ID:
03906B4110AAD3B8
12 changed files with 579 additions and 52 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -8,3 +8,5 @@ dist
|
|||
*.j2
|
||||
*.toml
|
||||
regenerated_*
|
||||
*.orig
|
||||
*.rej
|
||||
|
|
|
|||
|
|
@ -276,7 +276,15 @@ def parse_config(path: Path, fmt: str | None = None) -> tuple[str, Any]:
|
|||
def analyze_loops(fmt: str, parsed: Any) -> list[LoopCandidate]:
|
||||
"""
|
||||
Analyze parsed config to find loop opportunities.
|
||||
|
||||
JSON files are intentionally kept scalar/index-based instead of being
|
||||
collapsed into generated loops. JSON is commonly checked byte-for-byte by
|
||||
configuration management tools, and preserving inline arrays/objects is
|
||||
more valuable than reducing variable count.
|
||||
"""
|
||||
if fmt == "json":
|
||||
return []
|
||||
|
||||
analyzer = LoopAnalyzer()
|
||||
candidates = analyzer.analyze(parsed, fmt)
|
||||
|
||||
|
|
|
|||
|
|
@ -23,13 +23,15 @@ class JsonHandler(DictLikeHandler):
|
|||
role_prefix: str,
|
||||
original_text: str | None = None,
|
||||
) -> str:
|
||||
"""Original scalar-only template generation."""
|
||||
"""Generate a scalar JSON template while preserving source formatting."""
|
||||
if not isinstance(parsed, (dict, list)):
|
||||
raise TypeError("JSON parser result must be a dict or list")
|
||||
# As before: ignore original_text and rebuild structurally
|
||||
if original_text is not None:
|
||||
return self._generate_json_template_from_text(role_prefix, original_text)
|
||||
return self._generate_json_template(role_prefix, parsed)
|
||||
|
||||
JSON_INDENT = 2
|
||||
JSON_VALUE_FILTER = "to_json(ensure_ascii=False)"
|
||||
|
||||
def _leading_indent(self, s: str, idx: int) -> int:
|
||||
"""Return the number of leading spaces on the line containing idx."""
|
||||
|
|
@ -75,6 +77,129 @@ class JsonHandler(DictLikeHandler):
|
|||
role_prefix, parsed, loop_paths, loop_candidates
|
||||
)
|
||||
|
||||
def _json_value_expr(self, var_name: str) -> str:
|
||||
"""Return a Jinja expression for a JSON value.
|
||||
|
||||
Jinja's built-in ``tojson`` filter is HTML-safe and therefore escapes
|
||||
characters such as ``<`` and ``>`` as ``\u003c``/``\u003e``. That is
|
||||
useful in HTML, but noisy in configuration files. JinjaTurtle generates
|
||||
Ansible templates, so use Ansible's ``to_json`` filter instead.
|
||||
"""
|
||||
return f"{{{{ {var_name} | {self.JSON_VALUE_FILTER} }}}}"
|
||||
|
||||
def _generate_json_template_from_text(self, role_prefix: str, text: str) -> str:
|
||||
"""Replace JSON scalar values in-place, preserving original formatting.
|
||||
|
||||
The older JSON path parsed the file and wrote it back with
|
||||
``json.dumps(indent=2)``, which caused cosmetic diffs such as changing
|
||||
four-space indentation to two-space indentation and adding a final
|
||||
newline to files that intentionally lacked one. This scanner walks the
|
||||
original JSON source and only replaces scalar value tokens with Jinja2
|
||||
expressions; all whitespace, object/list indentation, key ordering, and
|
||||
final newline state are left untouched.
|
||||
"""
|
||||
spans = self._collect_json_scalar_spans(text)
|
||||
if spans is None:
|
||||
# Should be rare because the caller has already parsed the JSON, but
|
||||
# keep the structural fallback rather than failing template creation.
|
||||
parsed = json.loads(text)
|
||||
return self._generate_json_template(role_prefix, parsed)
|
||||
|
||||
chunks: list[str] = []
|
||||
pos = 0
|
||||
for path, start, end in spans:
|
||||
chunks.append(text[pos:start])
|
||||
chunks.append(self._json_value_expr(self.make_var_name(role_prefix, path)))
|
||||
pos = end
|
||||
chunks.append(text[pos:])
|
||||
return "".join(chunks)
|
||||
|
||||
def _collect_json_scalar_spans(
|
||||
self, text: str
|
||||
) -> list[tuple[tuple[str, ...], int, int]] | None:
|
||||
"""Return source spans for JSON scalar *values*.
|
||||
|
||||
Keys are parsed to determine the current path but are not returned.
|
||||
"""
|
||||
decoder = json.JSONDecoder()
|
||||
spans: list[tuple[tuple[str, ...], int, int]] = []
|
||||
|
||||
def skip_ws(i: int) -> int:
|
||||
while i < len(text) and text[i] in " \t\r\n":
|
||||
i += 1
|
||||
return i
|
||||
|
||||
def raw_decode_at(i: int) -> tuple[Any, int]:
|
||||
return decoder.raw_decode(text, i)
|
||||
|
||||
def parse_value(i: int, path: tuple[str, ...]) -> int:
|
||||
i = skip_ws(i)
|
||||
if i >= len(text):
|
||||
raise ValueError("unexpected end of JSON")
|
||||
ch = text[i]
|
||||
if ch == "{":
|
||||
return parse_object(i, path)
|
||||
if ch == "[":
|
||||
return parse_array(i, path)
|
||||
|
||||
_value, end = raw_decode_at(i)
|
||||
spans.append((path, i, end))
|
||||
return end
|
||||
|
||||
def parse_object(i: int, path: tuple[str, ...]) -> int:
|
||||
i += 1 # {
|
||||
i = skip_ws(i)
|
||||
if i < len(text) and text[i] == "}":
|
||||
return i + 1
|
||||
|
||||
while True:
|
||||
i = skip_ws(i)
|
||||
if i >= len(text) or text[i] != '"':
|
||||
raise ValueError("expected JSON object key")
|
||||
key, i = raw_decode_at(i)
|
||||
if not isinstance(key, str):
|
||||
raise ValueError("expected JSON object key string")
|
||||
|
||||
i = skip_ws(i)
|
||||
if i >= len(text) or text[i] != ":":
|
||||
raise ValueError("expected ':' after JSON object key")
|
||||
i = parse_value(i + 1, path + (key,))
|
||||
i = skip_ws(i)
|
||||
|
||||
if i < len(text) and text[i] == ",":
|
||||
i += 1
|
||||
continue
|
||||
if i < len(text) and text[i] == "}":
|
||||
return i + 1
|
||||
raise ValueError("expected ',' or '}' in JSON object")
|
||||
|
||||
def parse_array(i: int, path: tuple[str, ...]) -> int:
|
||||
i += 1 # [
|
||||
i = skip_ws(i)
|
||||
if i < len(text) and text[i] == "]":
|
||||
return i + 1
|
||||
|
||||
index = 0
|
||||
while True:
|
||||
i = parse_value(i, path + (str(index),))
|
||||
index += 1
|
||||
i = skip_ws(i)
|
||||
|
||||
if i < len(text) and text[i] == ",":
|
||||
i += 1
|
||||
continue
|
||||
if i < len(text) and text[i] == "]":
|
||||
return i + 1
|
||||
raise ValueError("expected ',' or ']' in JSON array")
|
||||
|
||||
try:
|
||||
end = parse_value(0, ())
|
||||
if skip_ws(end) != len(text):
|
||||
return None
|
||||
return spans
|
||||
except (json.JSONDecodeError, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
def _generate_json_template(self, role_prefix: str, data: Any) -> str:
|
||||
"""
|
||||
Generate a JSON Jinja2 template from parsed JSON data.
|
||||
|
|
@ -82,7 +207,7 @@ class JsonHandler(DictLikeHandler):
|
|||
All scalar values are replaced with Jinja expressions whose names are
|
||||
derived from the path, similar to TOML/YAML.
|
||||
|
||||
Uses | tojson filter to preserve types (numbers, booleans, null).
|
||||
Uses | to_json filter to preserve types (numbers, booleans, null).
|
||||
"""
|
||||
|
||||
def _walk(obj: Any, path: tuple[str, ...] = ()) -> Any:
|
||||
|
|
@ -90,17 +215,19 @@ class JsonHandler(DictLikeHandler):
|
|||
return {k: _walk(v, path + (str(k),)) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [_walk(v, path + (str(i),)) for i, v in enumerate(obj)]
|
||||
# scalar - use marker that will be replaced with tojson
|
||||
# scalar - use marker that will be replaced with to_json
|
||||
var_name = self.make_var_name(role_prefix, path)
|
||||
return f"__SCALAR__{var_name}__"
|
||||
|
||||
templated = _walk(data)
|
||||
json_str = json.dumps(templated, indent=2, ensure_ascii=False)
|
||||
|
||||
# Replace scalar markers with Jinja expressions using tojson filter
|
||||
# Replace scalar markers with Jinja expressions using to_json filter
|
||||
# This preserves types (numbers stay numbers, booleans stay booleans)
|
||||
json_str = re.sub(
|
||||
r'"__SCALAR__([a-zA-Z_][a-zA-Z0-9_]*)__"', r"{{ \1 | tojson }}", json_str
|
||||
r'"__SCALAR__([a-zA-Z_][a-zA-Z0-9_]*)__"',
|
||||
lambda m: self._json_value_expr(m.group(1)),
|
||||
json_str,
|
||||
)
|
||||
|
||||
return json_str + "\n"
|
||||
|
|
@ -150,9 +277,11 @@ class JsonHandler(DictLikeHandler):
|
|||
# Convert to JSON string
|
||||
json_str = json.dumps(templated, indent=2, ensure_ascii=False)
|
||||
|
||||
# Replace scalar markers with Jinja expressions using tojson filter
|
||||
# Replace scalar markers with Jinja expressions using to_json filter
|
||||
json_str = re.sub(
|
||||
r'"__SCALAR__([a-zA-Z_][a-zA-Z0-9_]*)__"', r"{{ \1 | tojson }}", json_str
|
||||
r'"__SCALAR__([a-zA-Z_][a-zA-Z0-9_]*)__"',
|
||||
lambda m: self._json_value_expr(m.group(1)),
|
||||
json_str,
|
||||
)
|
||||
|
||||
# Post-process to replace loop markers with actual Jinja loops (indent-aware)
|
||||
|
|
@ -201,7 +330,8 @@ class JsonHandler(DictLikeHandler):
|
|||
# a blank line between iterations under default Jinja whitespace settings.
|
||||
return (
|
||||
f"[\n"
|
||||
f"{{% for {item_var} in {collection_var} %}}{inner}{{{{ {item_var} | tojson }}}}"
|
||||
f"{{% for {item_var} in {collection_var} %}}"
|
||||
f"{inner}{{{{ {item_var} | to_json(ensure_ascii=False) }}}}"
|
||||
f"{{% if not loop.last %}},{{% endif %}}\n"
|
||||
f"{{% endfor %}}{base}]"
|
||||
)
|
||||
|
|
@ -234,7 +364,8 @@ class JsonHandler(DictLikeHandler):
|
|||
for i, key in enumerate(keys):
|
||||
comma = "," if i < len(keys) - 1 else ""
|
||||
dict_lines.append(
|
||||
f'{field}"{key}": {{{{ {item_var}.{key} | tojson }}}}{comma}'
|
||||
f'{field}"{key}": '
|
||||
f"{{{{ {item_var}.{key} | to_json(ensure_ascii=False) }}}}{comma}"
|
||||
)
|
||||
# Comma between *items* goes after the closing brace.
|
||||
dict_lines.append(f"{inner}}}{{% if not loop.last %}},{{% endif %}}")
|
||||
|
|
|
|||
|
|
@ -148,7 +148,10 @@ class TomlHandler(DictLikeHandler):
|
|||
elif candidate.item_schema in ("simple_dict", "nested"):
|
||||
# Dict list loop - TOML array of tables
|
||||
# This is complex for TOML, using simplified approach
|
||||
lines.append(f"{key} = {{{{ {var_name} | tojson }}}}")
|
||||
lines.append(
|
||||
f"{key} = "
|
||||
f"{{{{ {var_name} | to_json(ensure_ascii=False) }}}}"
|
||||
)
|
||||
else:
|
||||
# Not a loop, treat as regular variable
|
||||
lines.append(f"{key} = {{{{ {var_name} }}}}")
|
||||
|
|
@ -477,8 +480,10 @@ class TomlHandler(DictLikeHandler):
|
|||
f"]"
|
||||
)
|
||||
else:
|
||||
# Dict/nested loop - use tojson filter for complex arrays
|
||||
replacement_value = f"{{{{ {collection_var} | tojson }}}}"
|
||||
# Dict/nested loop - use to_json filter for complex arrays
|
||||
replacement_value = (
|
||||
f"{{{{ {collection_var} | to_json(ensure_ascii=False) }}}}"
|
||||
)
|
||||
|
||||
new_content = (
|
||||
before_eq + "=" + leading_ws + replacement_value + comment_part
|
||||
|
|
|
|||
|
|
@ -81,6 +81,15 @@ class YamlHandler(DictLikeHandler):
|
|||
return f"{{{{ 'null' if {value_expr} is none else {value_expr} }}}}"
|
||||
return f"{{{{ {value_expr} }}}}"
|
||||
|
||||
def _surrounding_quote(self, raw_value: str) -> str | None:
|
||||
if (
|
||||
len(raw_value) >= 2
|
||||
and raw_value[0] == raw_value[-1]
|
||||
and raw_value[0] in {'"', "'"}
|
||||
):
|
||||
return raw_value[0]
|
||||
return None
|
||||
|
||||
def _generate_yaml_template_from_text(
|
||||
self,
|
||||
role_prefix: str,
|
||||
|
|
@ -226,7 +235,26 @@ class YamlHandler(DictLikeHandler):
|
|||
def current_path() -> tuple[str, ...]:
|
||||
return stack[-1][1] if stack else ()
|
||||
|
||||
for raw_line in lines:
|
||||
def first_sequence_item_quote(
|
||||
start_index: int, parent_indent: int
|
||||
) -> str | None:
|
||||
for future_line in lines[start_index + 1 :]:
|
||||
future_stripped = future_line.lstrip()
|
||||
future_indent = len(future_line) - len(future_stripped)
|
||||
if not future_stripped or future_stripped.startswith("#"):
|
||||
continue
|
||||
if future_indent < parent_indent:
|
||||
return None
|
||||
if future_stripped.startswith("- "):
|
||||
value_part, _comment_part = self._split_inline_comment(
|
||||
future_stripped[2:], {"#"}
|
||||
)
|
||||
return self._surrounding_quote(value_part.strip())
|
||||
if future_indent <= parent_indent:
|
||||
return None
|
||||
return None
|
||||
|
||||
for line_index, raw_line in enumerate(lines):
|
||||
stripped = raw_line.lstrip()
|
||||
indent = len(raw_line) - len(stripped)
|
||||
|
||||
|
|
@ -287,8 +315,14 @@ class YamlHandler(DictLikeHandler):
|
|||
# Find the matching candidate
|
||||
candidate = next(c for c in loop_candidates if c.path == path)
|
||||
|
||||
scalar_quote = None
|
||||
if candidate.item_schema == "scalar":
|
||||
scalar_quote = first_sequence_item_quote(line_index, indent)
|
||||
|
||||
# Generate loop
|
||||
loop_str = self._generate_yaml_loop(candidate, role_prefix, indent)
|
||||
loop_str = self._generate_yaml_loop(
|
||||
candidate, role_prefix, indent, scalar_quote=scalar_quote
|
||||
)
|
||||
out_lines.append(loop_str)
|
||||
|
||||
# Skip subsequent lines that are part of this collection
|
||||
|
|
@ -335,6 +369,10 @@ class YamlHandler(DictLikeHandler):
|
|||
stack.append((indent, parent_path, "seq"))
|
||||
|
||||
parent_path = stack[-1][1]
|
||||
content = stripped[2:]
|
||||
value_part, _comment_part = self._split_inline_comment(content, {"#"})
|
||||
raw_value = value_part.strip()
|
||||
scalar_quote = self._surrounding_quote(raw_value)
|
||||
|
||||
# Check if parent path is a loop candidate
|
||||
if parent_path in loop_paths:
|
||||
|
|
@ -345,7 +383,11 @@ class YamlHandler(DictLikeHandler):
|
|||
|
||||
# Generate loop (with indent for the '-' items)
|
||||
loop_str = self._generate_yaml_loop(
|
||||
candidate, role_prefix, indent, is_list=True
|
||||
candidate,
|
||||
role_prefix,
|
||||
indent,
|
||||
is_list=True,
|
||||
scalar_quote=scalar_quote,
|
||||
)
|
||||
out_lines.append(loop_str)
|
||||
|
||||
|
|
@ -353,7 +395,6 @@ class YamlHandler(DictLikeHandler):
|
|||
skip_until_indent = indent - 1 if indent > 0 else None
|
||||
continue
|
||||
|
||||
content = stripped[2:]
|
||||
index = seq_counters.get(parent_path, 0)
|
||||
seq_counters[parent_path] = index + 1
|
||||
|
||||
|
|
@ -393,6 +434,7 @@ class YamlHandler(DictLikeHandler):
|
|||
role_prefix: str,
|
||||
indent: int,
|
||||
is_list: bool = False,
|
||||
scalar_quote: str | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Generate a Jinja2 for loop for a YAML collection.
|
||||
|
|
@ -423,9 +465,10 @@ class YamlHandler(DictLikeHandler):
|
|||
item_indent_str = " " * item_indent
|
||||
|
||||
if candidate.item_schema == "scalar":
|
||||
item_lines.append(
|
||||
f"{item_indent_str}- {self._yaml_value_expr(item_var, sample_item)}"
|
||||
)
|
||||
value_expr = self._yaml_value_expr(item_var, sample_item)
|
||||
if scalar_quote and isinstance(sample_item, str):
|
||||
value_expr = f"{scalar_quote}{{{{ {item_var} }}}}{scalar_quote}"
|
||||
item_lines.append(f"{item_indent_str}- {value_expr}")
|
||||
elif candidate.item_schema in ("simple_dict", "nested"):
|
||||
item_lines = self._dict_to_yaml_lines(
|
||||
sample_item, item_var, item_indent, is_list_item=True
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ Notes:
|
|||
* If the folder contains *multiple* formats, we generate one template per
|
||||
format (e.g. config.yaml.j2, config.xml.j2) and emit one list variable per
|
||||
format in the defaults YAML.
|
||||
* JSON union templates are emitted using a simple `{{ data | tojson }}`
|
||||
* JSON union templates are emitted using a simple `{{ data | to_json }}`
|
||||
approach to avoid comma-management complexity for optional keys.
|
||||
"""
|
||||
|
||||
|
|
@ -624,7 +624,7 @@ def process_directory(
|
|||
if multiple_formats
|
||||
else f"{role_prefix}_items"
|
||||
)
|
||||
template = "{{ data | tojson(indent=2) }}\n"
|
||||
template = "{{ data | to_json(indent=2, ensure_ascii=False) }}\n"
|
||||
items: list[dict[str, Any]] = []
|
||||
for rid, parsed in zip(rel_ids, parsed_list):
|
||||
items.append({"id": rid, "data": parsed})
|
||||
|
|
|
|||
|
|
@ -62,3 +62,73 @@ def test_cli_writes_output_files(tmp_path, capsys):
|
|||
# When writing to files, we shouldn't print the big headers
|
||||
assert "# defaults/main.yml" not in captured.out
|
||||
assert "# config.j2" not in captured.out
|
||||
|
||||
|
||||
def test_cli_folder_stdout(tmp_path, capsys):
|
||||
"""Folder mode without output paths prints defaults and all templates."""
|
||||
(tmp_path / "a.json").write_text('{"name": "one"}\n', encoding="utf-8")
|
||||
(tmp_path / "b.yaml").write_text("name: two\n", encoding="utf-8")
|
||||
|
||||
exit_code = cli._main([str(tmp_path), "-r", "role"])
|
||||
|
||||
assert exit_code == 0
|
||||
captured = capsys.readouterr()
|
||||
assert "# defaults/main.yml" in captured.out
|
||||
assert "# config.json.j2" in captured.out
|
||||
assert "# config.yaml.j2" in captured.out
|
||||
assert "role_json_items" in captured.out
|
||||
assert "role_yaml_items" in captured.out
|
||||
|
||||
|
||||
def test_cli_folder_writes_template_directory(tmp_path, capsys):
|
||||
"""Folder mode writes multiple templates when template output is a directory."""
|
||||
src_dir = tmp_path / "src"
|
||||
src_dir.mkdir()
|
||||
(src_dir / "a.json").write_text('{"name": "one"}\n', encoding="utf-8")
|
||||
(src_dir / "b.yaml").write_text("name: two\n", encoding="utf-8")
|
||||
defaults_path = tmp_path / "defaults.yml"
|
||||
template_dir = tmp_path / "templates"
|
||||
|
||||
exit_code = cli._main(
|
||||
[
|
||||
str(src_dir),
|
||||
"-r",
|
||||
"role",
|
||||
"--defaults-output",
|
||||
str(defaults_path),
|
||||
"--template-output",
|
||||
str(template_dir),
|
||||
]
|
||||
)
|
||||
|
||||
assert exit_code == 0
|
||||
assert defaults_path.is_file()
|
||||
assert (template_dir / "config.json.j2").is_file()
|
||||
assert (template_dir / "config.yaml.j2").is_file()
|
||||
captured = capsys.readouterr()
|
||||
assert captured.out == ""
|
||||
|
||||
|
||||
def test_cli_folder_single_output_file_when_one_format(tmp_path):
|
||||
src_dir = tmp_path / "src"
|
||||
src_dir.mkdir()
|
||||
(src_dir / "a.json").write_text('{"name": "one"}\n', encoding="utf-8")
|
||||
defaults_path = tmp_path / "defaults.yml"
|
||||
template_path = tmp_path / "config.j2"
|
||||
|
||||
exit_code = cli._main(
|
||||
[
|
||||
str(src_dir),
|
||||
"-r",
|
||||
"role",
|
||||
"--defaults-output",
|
||||
str(defaults_path),
|
||||
"--template-output",
|
||||
str(template_path),
|
||||
]
|
||||
)
|
||||
|
||||
assert exit_code == 0
|
||||
assert defaults_path.is_file()
|
||||
assert template_path.is_file()
|
||||
assert "to_json" in template_path.read_text(encoding="utf-8")
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import json
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from jinja2 import Environment
|
||||
|
||||
from jinjaturtle.core import (
|
||||
parse_config,
|
||||
|
|
@ -13,6 +15,7 @@ from jinjaturtle.core import (
|
|||
generate_jinja2_template,
|
||||
)
|
||||
from jinjaturtle.handlers.json import JsonHandler
|
||||
from jinjaturtle.loop_analyzer import LoopCandidate
|
||||
|
||||
SAMPLES_DIR = Path(__file__).parent / "samples"
|
||||
|
||||
|
|
@ -35,23 +38,136 @@ def test_json_roundtrip():
|
|||
assert defaults["foobar_nested_a"] == 1
|
||||
# Booleans are now preserved as booleans (not stringified)
|
||||
assert defaults["foobar_nested_b"] is True
|
||||
# List should be a list (not flattened to scalars)
|
||||
assert defaults["foobar_list"] == [10, 20]
|
||||
# JSON stays scalar/index-based so source formatting can be preserved.
|
||||
assert loop_candidates == []
|
||||
assert defaults["foobar_list_0"] == 10
|
||||
assert defaults["foobar_list_1"] == 20
|
||||
|
||||
# Template generation with loops
|
||||
template = generate_jinja2_template("json", parsed, "foobar", None, loop_candidates)
|
||||
|
||||
# Template should use | tojson for type preservation
|
||||
assert "{{ foobar_foo | tojson }}" in template
|
||||
assert "{{ foobar_nested_a | tojson }}" in template
|
||||
assert "{{ foobar_nested_b | tojson }}" in template
|
||||
# Template should use Ansible | to_json for type preservation without
|
||||
# HTML-safe escaping.
|
||||
assert "{{ foobar_foo | to_json(ensure_ascii=False) }}" in template
|
||||
assert "{{ foobar_nested_a | to_json(ensure_ascii=False) }}" in template
|
||||
assert "{{ foobar_nested_b | to_json(ensure_ascii=False) }}" in template
|
||||
assert "{{ foobar_list_0 | to_json(ensure_ascii=False) }}" in template
|
||||
assert "{{ foobar_list_1 | to_json(ensure_ascii=False) }}" in template
|
||||
assert "{% for" not in template
|
||||
|
||||
# List should use loop (not scalar indices)
|
||||
assert "{% for" in template
|
||||
assert "foobar_list" in template
|
||||
# Should NOT have scalar indices
|
||||
assert "foobar_list_0" not in template
|
||||
assert "foobar_list_1" not in template
|
||||
|
||||
def test_json_template_preserves_original_indentation_and_final_newline_state(
|
||||
tmp_path: Path,
|
||||
):
|
||||
json_text = (
|
||||
"{\n"
|
||||
' "default-runtime": "runsc",\n'
|
||||
' "runtimes": {\n'
|
||||
' "runsc": {\n'
|
||||
' "path": "/usr/bin/runsc"\n'
|
||||
" }\n"
|
||||
" }\n"
|
||||
"}"
|
||||
)
|
||||
path = tmp_path / "daemon.json"
|
||||
path.write_text(json_text, encoding="utf-8")
|
||||
|
||||
fmt, parsed = parse_config(path)
|
||||
template = generate_jinja2_template(fmt, parsed, "docker", json_text, [])
|
||||
|
||||
assert ' "default-runtime"' in template
|
||||
assert ' "runsc"' in template
|
||||
assert ' "path"' in template
|
||||
assert not template.endswith("\n")
|
||||
|
||||
|
||||
def test_json_template_uses_non_html_safe_json_filter_for_angle_brackets(
|
||||
tmp_path: Path,
|
||||
):
|
||||
json_text = '{"version_requirement": ">= 8.0.0 < 9.0.0"}\n'
|
||||
path = tmp_path / "metadata.json"
|
||||
path.write_text(json_text, encoding="utf-8")
|
||||
|
||||
fmt, parsed = parse_config(path)
|
||||
template = generate_jinja2_template(fmt, parsed, "puppet", json_text, [])
|
||||
|
||||
assert "to_json(ensure_ascii=False)" in template
|
||||
assert "tojson" not in template
|
||||
|
||||
|
||||
def test_json_inline_object_array_roundtrips_without_expanding(tmp_path: Path):
|
||||
json_text = (
|
||||
"{\n"
|
||||
' "description": "Gets the last boot time of a Linux or Windows system",\n'
|
||||
' "implementations": [\n'
|
||||
' {"name": "last_boot_time_nix.sh", "requirements": ["shell"]},\n'
|
||||
' {"name": "last_boot_time_win.ps1", "requirements": ["powershell"]}\n'
|
||||
" ]\n"
|
||||
"}\n"
|
||||
)
|
||||
path = tmp_path / "metadata.json"
|
||||
path.write_text(json_text, encoding="utf-8")
|
||||
|
||||
fmt, parsed = parse_config(path)
|
||||
loop_candidates = analyze_loops(fmt, parsed)
|
||||
flat_items = flatten_config(fmt, parsed, loop_candidates)
|
||||
defaults = yaml.safe_load(
|
||||
generate_ansible_yaml("fact", flat_items, loop_candidates)
|
||||
)
|
||||
template = generate_jinja2_template(fmt, parsed, "fact", json_text, loop_candidates)
|
||||
|
||||
env = Environment(keep_trailing_newline=True)
|
||||
env.filters["to_json"] = lambda value, **kwargs: json.dumps(value, **kwargs)
|
||||
rendered = env.from_string(template).render(**defaults)
|
||||
|
||||
assert loop_candidates == []
|
||||
assert '{"name": {{ fact_implementations_0_name' in template
|
||||
assert rendered == json_text
|
||||
|
||||
|
||||
def test_json_direct_loop_generation_renders_valid_json_without_joined_objects():
|
||||
items = [
|
||||
{"name": "last_boot_time_nix.sh", "requirements": ["shell"]},
|
||||
{"name": "last_boot_time_win.ps1", "requirements": ["powershell"]},
|
||||
]
|
||||
parsed = {"implementations": items}
|
||||
candidate = LoopCandidate(
|
||||
path=("implementations",),
|
||||
loop_var="implementation",
|
||||
items=items,
|
||||
item_schema="simple_dict",
|
||||
)
|
||||
handler = JsonHandler()
|
||||
template = handler.generate_jinja2_template_with_loops(
|
||||
parsed, "fact", None, [candidate]
|
||||
)
|
||||
|
||||
env = Environment(keep_trailing_newline=True)
|
||||
env.filters["to_json"] = lambda value, **kwargs: json.dumps(value, **kwargs)
|
||||
rendered = env.from_string(template).render(fact_implementations=items)
|
||||
|
||||
assert "}, {" not in rendered
|
||||
assert json.loads(rendered) == parsed
|
||||
|
||||
|
||||
def test_json_scalar_span_collector_empty_containers_and_invalid_json():
|
||||
handler = JsonHandler()
|
||||
|
||||
assert handler._collect_json_scalar_spans('{"empty": [], "obj": {}}') == []
|
||||
assert handler._collect_json_scalar_spans('{"unterminated": [1,}') is None
|
||||
|
||||
|
||||
def test_json_template_falls_back_when_source_scanner_cannot_collect_spans(monkeypatch):
|
||||
handler = JsonHandler()
|
||||
monkeypatch.setattr(handler, "_collect_json_scalar_spans", lambda _text: None)
|
||||
|
||||
template = handler.generate_jinja2_template(
|
||||
{"answer": 42}, "role", original_text='{"answer": 42}'
|
||||
)
|
||||
|
||||
assert (
|
||||
template
|
||||
== '{\n "answer": {{ role_answer | to_json(ensure_ascii=False) }}\n}\n'
|
||||
)
|
||||
|
||||
|
||||
def test_generate_jinja2_template_json_type_error():
|
||||
|
|
|
|||
125
tests/test_multi.py
Normal file
125
tests/test_multi.py
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from jinjaturtle.multi import (
|
||||
_collect_dict_like_paths,
|
||||
_merge_union,
|
||||
defined_var_name,
|
||||
is_supported_file,
|
||||
iter_supported_files,
|
||||
process_directory,
|
||||
)
|
||||
|
||||
|
||||
def test_iter_supported_files_handles_files_dirs_and_missing_paths(tmp_path: Path):
|
||||
json_file = tmp_path / "config.json"
|
||||
text_file = tmp_path / "notes.txt"
|
||||
nested = tmp_path / "nested"
|
||||
nested.mkdir()
|
||||
nested_yaml = nested / "config.yaml"
|
||||
|
||||
json_file.write_text('{"name": "one"}\n', encoding="utf-8")
|
||||
text_file.write_text("ignore me\n", encoding="utf-8")
|
||||
nested_yaml.write_text("name: two\n", encoding="utf-8")
|
||||
|
||||
assert is_supported_file(json_file)
|
||||
assert not is_supported_file(text_file)
|
||||
assert iter_supported_files(json_file, recursive=False) == [json_file]
|
||||
assert iter_supported_files(text_file, recursive=False) == []
|
||||
assert iter_supported_files(tmp_path, recursive=False) == [json_file]
|
||||
assert iter_supported_files(tmp_path, recursive=True) == [json_file, nested_yaml]
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
iter_supported_files(tmp_path / "missing", recursive=False)
|
||||
|
||||
|
||||
def test_merge_union_and_collect_dict_like_paths():
|
||||
merged = _merge_union(
|
||||
{"name": "one", "ports": [80], "nested": {"a": 1}},
|
||||
{"ports": [80, 443], "nested": {"b": 2}, "enabled": True},
|
||||
)
|
||||
|
||||
assert merged == {
|
||||
"name": "one",
|
||||
"ports": [80, 443],
|
||||
"nested": {"a": 1, "b": 2},
|
||||
"enabled": True,
|
||||
}
|
||||
|
||||
containers, leaves = _collect_dict_like_paths(merged)
|
||||
assert ("nested",) in containers
|
||||
assert ("nested", "a") in containers
|
||||
assert ("ports", "1") in leaves
|
||||
|
||||
|
||||
def test_process_directory_multiple_formats(tmp_path: Path):
|
||||
(tmp_path / "a.json").write_text('{"name": "one"}\n', encoding="utf-8")
|
||||
(tmp_path / "b.yaml").write_text("name: one\nextra: true\n", encoding="utf-8")
|
||||
(tmp_path / "c.toml").write_text('name = "one"\n', encoding="utf-8")
|
||||
(tmp_path / "d.ini").write_text("[main]\nname = one\n", encoding="utf-8")
|
||||
(tmp_path / "e.xml").write_text("<root><name>one</name></root>", encoding="utf-8")
|
||||
|
||||
defaults_yaml, outputs = process_directory(
|
||||
tmp_path, recursive=False, role_prefix="role"
|
||||
)
|
||||
defaults = yaml.safe_load(defaults_yaml)
|
||||
by_fmt = {output.fmt: output for output in outputs}
|
||||
|
||||
assert set(by_fmt) == {"ini", "json", "toml", "xml", "yaml"}
|
||||
assert set(defaults) == {
|
||||
"role_ini_items",
|
||||
"role_json_items",
|
||||
"role_toml_items",
|
||||
"role_xml_items",
|
||||
"role_yaml_items",
|
||||
}
|
||||
assert defaults["role_json_items"][0]["data"] == {"name": "one"}
|
||||
assert (
|
||||
by_fmt["json"].template
|
||||
== "{{ data | to_json(indent=2, ensure_ascii=False) }}\n"
|
||||
)
|
||||
assert "{{ role_main_name }}" in by_fmt["ini"].template
|
||||
assert "{{ role_name }}" in by_fmt["xml"].template
|
||||
|
||||
|
||||
def test_process_directory_yaml_union_marks_optional_keys(tmp_path: Path):
|
||||
(tmp_path / "one.yaml").write_text("name: one\nextra: yes\n", encoding="utf-8")
|
||||
(tmp_path / "two.yaml").write_text("name: two\n", encoding="utf-8")
|
||||
|
||||
defaults_yaml, outputs = process_directory(
|
||||
tmp_path, recursive=False, role_prefix="role"
|
||||
)
|
||||
defaults = yaml.safe_load(defaults_yaml)
|
||||
|
||||
assert len(outputs) == 1
|
||||
assert outputs[0].fmt == "yaml"
|
||||
assert "{% if role_defined_extra is defined %}" in outputs[0].template
|
||||
assert defaults["role_items"][0][defined_var_name("role", ("extra",))] is True
|
||||
assert defined_var_name("role", ("extra",)) not in defaults["role_items"][1]
|
||||
|
||||
|
||||
def test_process_directory_ini_union_marks_optional_sections_and_keys(tmp_path: Path):
|
||||
(tmp_path / "one.ini").write_text(
|
||||
"[main]\nname = one\n[extra]\nflag = yes\n", encoding="utf-8"
|
||||
)
|
||||
(tmp_path / "two.ini").write_text("[main]\nname = two\n", encoding="utf-8")
|
||||
|
||||
defaults_yaml, outputs = process_directory(
|
||||
tmp_path, recursive=False, role_prefix="role"
|
||||
)
|
||||
defaults = yaml.safe_load(defaults_yaml)
|
||||
|
||||
assert len(outputs) == 1
|
||||
assert outputs[0].fmt == "ini"
|
||||
assert "{% if role_defined_extra is defined %}" in outputs[0].template
|
||||
assert defaults["role_items"][0][defined_var_name("role", ("extra",))] is True
|
||||
assert defined_var_name("role", ("extra",)) not in defaults["role_items"][1]
|
||||
|
||||
|
||||
def test_process_directory_rejects_empty_folder(tmp_path: Path):
|
||||
with pytest.raises(ValueError, match="No supported config files"):
|
||||
process_directory(tmp_path, recursive=False, role_prefix="role")
|
||||
|
|
@ -31,6 +31,7 @@ from jinjaturtle.core import (
|
|||
def render_template(template: str, variables: dict[str, Any]) -> str:
|
||||
"""Render a Jinja2 template with variables."""
|
||||
env = Environment(undefined=StrictUndefined)
|
||||
env.filters["to_json"] = lambda value, **kwargs: json.dumps(value, **kwargs)
|
||||
jinja_template = env.from_string(template)
|
||||
return jinja_template.render(variables)
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import yaml
|
|||
|
||||
from jinjaturtle.core import (
|
||||
parse_config,
|
||||
analyze_loops,
|
||||
flatten_config,
|
||||
generate_ansible_yaml,
|
||||
generate_jinja2_template,
|
||||
|
|
@ -204,6 +205,36 @@ def test_yaml_loop_preserves_following_top_level_comments(tmp_path: Path):
|
|||
assert yaml.safe_load(rendered) == yaml.safe_load(text)
|
||||
|
||||
|
||||
def test_yaml_scalar_loop_preserves_quoted_list_items(tmp_path: Path):
|
||||
from jinja2 import Template
|
||||
|
||||
text = textwrap.dedent(
|
||||
"""
|
||||
files:
|
||||
- 'spec/unit/puppet/provider/dsc_base_provider/dsc_base_provider_spec.rb'
|
||||
- 'spec/unit/pwsh/util_spec.rb'
|
||||
- 'spec/unit/pwsh/windows_powershell_spec.rb'
|
||||
"""
|
||||
).lstrip()
|
||||
path = tmp_path / "quoted-list.yaml"
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
fmt, parsed = parse_config(path)
|
||||
loop_candidates = analyze_loops(fmt, parsed)
|
||||
flat_items = flatten_config(fmt, parsed, loop_candidates)
|
||||
defaults = yaml.safe_load(
|
||||
generate_ansible_yaml("role", flat_items, loop_candidates)
|
||||
)
|
||||
template = generate_jinja2_template(
|
||||
fmt, parsed, "role", original_text=text, loop_candidates=loop_candidates
|
||||
)
|
||||
rendered = Template(template).render(**defaults)
|
||||
|
||||
assert loop_candidates
|
||||
assert "- '{{" in template
|
||||
assert rendered == text
|
||||
|
||||
|
||||
def test_yaml_bool_scalars_render_with_yaml_spelling(tmp_path: Path):
|
||||
from jinja2 import Template
|
||||
|
||||
|
|
|
|||
|
|
@ -454,6 +454,7 @@ class TestStructuralConsistency:
|
|||
|
||||
# Try to render the template
|
||||
env = Environment(undefined=StrictUndefined)
|
||||
env.filters["to_json"] = lambda value, **kwargs: json.dumps(value, **kwargs)
|
||||
try:
|
||||
jinja_template = env.from_string(template)
|
||||
rendered = jinja_template.render(variables)
|
||||
|
|
@ -520,11 +521,9 @@ class TestRegressionBugs:
|
|||
"app_database" in template
|
||||
), f"Template should reference app_database\n{template}"
|
||||
|
||||
def test_json_array_no_index_refs(self):
|
||||
def test_json_array_uses_index_refs_for_source_fidelity(self):
|
||||
"""
|
||||
Regression test: JSON arrays should not generate index references.
|
||||
|
||||
Bug: Template had {{ app_list_0 }}, {{ app_list_1 }} when YAML had app_list as list.
|
||||
JSON arrays stay index-based so original inline formatting can survive.
|
||||
"""
|
||||
import json
|
||||
|
||||
|
|
@ -536,22 +535,18 @@ class TestRegressionBugs:
|
|||
|
||||
ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
|
||||
template = generate_jinja2_template(
|
||||
"json", parsed, "app", None, loop_candidates
|
||||
"json", parsed, "app", json_text, loop_candidates
|
||||
)
|
||||
|
||||
# YAML should have app_items as a list
|
||||
defaults = yaml.safe_load(ansible_yaml)
|
||||
assert isinstance(defaults.get("app_items"), list)
|
||||
|
||||
# Template should NOT have app_items_0, app_items_1, app_items_2
|
||||
for i in range(3):
|
||||
assert (
|
||||
f"app_items_{i}" not in template
|
||||
), f"Template incorrectly uses scalar 'app_items_{i}'\n{template}"
|
||||
|
||||
# Template SHOULD use a loop
|
||||
assert "{% for" in template
|
||||
assert "app_items" in template
|
||||
assert loop_candidates == []
|
||||
assert defaults["app_items_0"] == 1
|
||||
assert defaults["app_items_1"] == 2
|
||||
assert defaults["app_items_2"] == 3
|
||||
assert "{% for" not in template
|
||||
assert "app_items_0" in template
|
||||
assert "app_items_1" in template
|
||||
assert "app_items_2" in template
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue