From 49fee7afe4ab691bb5c72adfab80b834277b4e15 Mon Sep 17 00:00:00 2001 From: Miguel Jacq Date: Sat, 20 Jun 2026 15:28:48 +1000 Subject: [PATCH] Make templates more faithful to the original file in terms of indentation, newlines, no deserialisation of things like < or >. More test coverage --- .gitignore | 2 + src/jinjaturtle/core.py | 8 ++ src/jinjaturtle/handlers/json.py | 151 ++++++++++++++++++++++-- src/jinjaturtle/handlers/toml.py | 11 +- src/jinjaturtle/handlers/yaml.py | 57 +++++++-- src/jinjaturtle/multi.py | 4 +- tests/test_cli.py | 70 +++++++++++ tests/test_json_handler.py | 142 ++++++++++++++++++++-- tests/test_multi.py | 125 ++++++++++++++++++++ tests/test_roundtrip.py | 1 + tests/test_yaml_handler.py | 31 +++++ tests/test_yaml_template_consistency.py | 29 ++--- 12 files changed, 579 insertions(+), 52 deletions(-) create mode 100644 tests/test_multi.py diff --git a/.gitignore b/.gitignore index dedc5da..5a5117f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ dist *.j2 *.toml regenerated_* +*.orig +*.rej diff --git a/src/jinjaturtle/core.py b/src/jinjaturtle/core.py index f854c1e..ff6a586 100644 --- a/src/jinjaturtle/core.py +++ b/src/jinjaturtle/core.py @@ -276,7 +276,15 @@ def parse_config(path: Path, fmt: str | None = None) -> tuple[str, Any]: def analyze_loops(fmt: str, parsed: Any) -> list[LoopCandidate]: """ Analyze parsed config to find loop opportunities. + + JSON files are intentionally kept scalar/index-based instead of being + collapsed into generated loops. JSON is commonly checked byte-for-byte by + configuration management tools, and preserving inline arrays/objects is + more valuable than reducing variable count. """ + if fmt == "json": + return [] + analyzer = LoopAnalyzer() candidates = analyzer.analyze(parsed, fmt) diff --git a/src/jinjaturtle/handlers/json.py b/src/jinjaturtle/handlers/json.py index a0b0017..9d763a4 100644 --- a/src/jinjaturtle/handlers/json.py +++ b/src/jinjaturtle/handlers/json.py @@ -23,13 +23,15 @@ class JsonHandler(DictLikeHandler): role_prefix: str, original_text: str | None = None, ) -> str: - """Original scalar-only template generation.""" + """Generate a scalar JSON template while preserving source formatting.""" if not isinstance(parsed, (dict, list)): raise TypeError("JSON parser result must be a dict or list") - # As before: ignore original_text and rebuild structurally + if original_text is not None: + return self._generate_json_template_from_text(role_prefix, original_text) return self._generate_json_template(role_prefix, parsed) JSON_INDENT = 2 + JSON_VALUE_FILTER = "to_json(ensure_ascii=False)" def _leading_indent(self, s: str, idx: int) -> int: """Return the number of leading spaces on the line containing idx.""" @@ -75,6 +77,129 @@ class JsonHandler(DictLikeHandler): role_prefix, parsed, loop_paths, loop_candidates ) + def _json_value_expr(self, var_name: str) -> str: + """Return a Jinja expression for a JSON value. + + Jinja's built-in ``tojson`` filter is HTML-safe and therefore escapes + characters such as ``<`` and ``>`` as ``\u003c``/``\u003e``. That is + useful in HTML, but noisy in configuration files. JinjaTurtle generates + Ansible templates, so use Ansible's ``to_json`` filter instead. + """ + return f"{{{{ {var_name} | {self.JSON_VALUE_FILTER} }}}}" + + def _generate_json_template_from_text(self, role_prefix: str, text: str) -> str: + """Replace JSON scalar values in-place, preserving original formatting. + + The older JSON path parsed the file and wrote it back with + ``json.dumps(indent=2)``, which caused cosmetic diffs such as changing + four-space indentation to two-space indentation and adding a final + newline to files that intentionally lacked one. This scanner walks the + original JSON source and only replaces scalar value tokens with Jinja2 + expressions; all whitespace, object/list indentation, key ordering, and + final newline state are left untouched. + """ + spans = self._collect_json_scalar_spans(text) + if spans is None: + # Should be rare because the caller has already parsed the JSON, but + # keep the structural fallback rather than failing template creation. + parsed = json.loads(text) + return self._generate_json_template(role_prefix, parsed) + + chunks: list[str] = [] + pos = 0 + for path, start, end in spans: + chunks.append(text[pos:start]) + chunks.append(self._json_value_expr(self.make_var_name(role_prefix, path))) + pos = end + chunks.append(text[pos:]) + return "".join(chunks) + + def _collect_json_scalar_spans( + self, text: str + ) -> list[tuple[tuple[str, ...], int, int]] | None: + """Return source spans for JSON scalar *values*. + + Keys are parsed to determine the current path but are not returned. + """ + decoder = json.JSONDecoder() + spans: list[tuple[tuple[str, ...], int, int]] = [] + + def skip_ws(i: int) -> int: + while i < len(text) and text[i] in " \t\r\n": + i += 1 + return i + + def raw_decode_at(i: int) -> tuple[Any, int]: + return decoder.raw_decode(text, i) + + def parse_value(i: int, path: tuple[str, ...]) -> int: + i = skip_ws(i) + if i >= len(text): + raise ValueError("unexpected end of JSON") + ch = text[i] + if ch == "{": + return parse_object(i, path) + if ch == "[": + return parse_array(i, path) + + _value, end = raw_decode_at(i) + spans.append((path, i, end)) + return end + + def parse_object(i: int, path: tuple[str, ...]) -> int: + i += 1 # { + i = skip_ws(i) + if i < len(text) and text[i] == "}": + return i + 1 + + while True: + i = skip_ws(i) + if i >= len(text) or text[i] != '"': + raise ValueError("expected JSON object key") + key, i = raw_decode_at(i) + if not isinstance(key, str): + raise ValueError("expected JSON object key string") + + i = skip_ws(i) + if i >= len(text) or text[i] != ":": + raise ValueError("expected ':' after JSON object key") + i = parse_value(i + 1, path + (key,)) + i = skip_ws(i) + + if i < len(text) and text[i] == ",": + i += 1 + continue + if i < len(text) and text[i] == "}": + return i + 1 + raise ValueError("expected ',' or '}' in JSON object") + + def parse_array(i: int, path: tuple[str, ...]) -> int: + i += 1 # [ + i = skip_ws(i) + if i < len(text) and text[i] == "]": + return i + 1 + + index = 0 + while True: + i = parse_value(i, path + (str(index),)) + index += 1 + i = skip_ws(i) + + if i < len(text) and text[i] == ",": + i += 1 + continue + if i < len(text) and text[i] == "]": + return i + 1 + raise ValueError("expected ',' or ']' in JSON array") + + try: + end = parse_value(0, ()) + if skip_ws(end) != len(text): + return None + return spans + except (json.JSONDecodeError, ValueError, TypeError): + return None + def _generate_json_template(self, role_prefix: str, data: Any) -> str: """ Generate a JSON Jinja2 template from parsed JSON data. @@ -82,7 +207,7 @@ class JsonHandler(DictLikeHandler): All scalar values are replaced with Jinja expressions whose names are derived from the path, similar to TOML/YAML. - Uses | tojson filter to preserve types (numbers, booleans, null). + Uses | to_json filter to preserve types (numbers, booleans, null). """ def _walk(obj: Any, path: tuple[str, ...] = ()) -> Any: @@ -90,17 +215,19 @@ class JsonHandler(DictLikeHandler): return {k: _walk(v, path + (str(k),)) for k, v in obj.items()} if isinstance(obj, list): return [_walk(v, path + (str(i),)) for i, v in enumerate(obj)] - # scalar - use marker that will be replaced with tojson + # scalar - use marker that will be replaced with to_json var_name = self.make_var_name(role_prefix, path) return f"__SCALAR__{var_name}__" templated = _walk(data) json_str = json.dumps(templated, indent=2, ensure_ascii=False) - # Replace scalar markers with Jinja expressions using tojson filter + # Replace scalar markers with Jinja expressions using to_json filter # This preserves types (numbers stay numbers, booleans stay booleans) json_str = re.sub( - r'"__SCALAR__([a-zA-Z_][a-zA-Z0-9_]*)__"', r"{{ \1 | tojson }}", json_str + r'"__SCALAR__([a-zA-Z_][a-zA-Z0-9_]*)__"', + lambda m: self._json_value_expr(m.group(1)), + json_str, ) return json_str + "\n" @@ -150,9 +277,11 @@ class JsonHandler(DictLikeHandler): # Convert to JSON string json_str = json.dumps(templated, indent=2, ensure_ascii=False) - # Replace scalar markers with Jinja expressions using tojson filter + # Replace scalar markers with Jinja expressions using to_json filter json_str = re.sub( - r'"__SCALAR__([a-zA-Z_][a-zA-Z0-9_]*)__"', r"{{ \1 | tojson }}", json_str + r'"__SCALAR__([a-zA-Z_][a-zA-Z0-9_]*)__"', + lambda m: self._json_value_expr(m.group(1)), + json_str, ) # Post-process to replace loop markers with actual Jinja loops (indent-aware) @@ -201,7 +330,8 @@ class JsonHandler(DictLikeHandler): # a blank line between iterations under default Jinja whitespace settings. return ( f"[\n" - f"{{% for {item_var} in {collection_var} %}}{inner}{{{{ {item_var} | tojson }}}}" + f"{{% for {item_var} in {collection_var} %}}" + f"{inner}{{{{ {item_var} | to_json(ensure_ascii=False) }}}}" f"{{% if not loop.last %}},{{% endif %}}\n" f"{{% endfor %}}{base}]" ) @@ -234,7 +364,8 @@ class JsonHandler(DictLikeHandler): for i, key in enumerate(keys): comma = "," if i < len(keys) - 1 else "" dict_lines.append( - f'{field}"{key}": {{{{ {item_var}.{key} | tojson }}}}{comma}' + f'{field}"{key}": ' + f"{{{{ {item_var}.{key} | to_json(ensure_ascii=False) }}}}{comma}" ) # Comma between *items* goes after the closing brace. dict_lines.append(f"{inner}}}{{% if not loop.last %}},{{% endif %}}") diff --git a/src/jinjaturtle/handlers/toml.py b/src/jinjaturtle/handlers/toml.py index fe071bd..ba4bbb4 100644 --- a/src/jinjaturtle/handlers/toml.py +++ b/src/jinjaturtle/handlers/toml.py @@ -148,7 +148,10 @@ class TomlHandler(DictLikeHandler): elif candidate.item_schema in ("simple_dict", "nested"): # Dict list loop - TOML array of tables # This is complex for TOML, using simplified approach - lines.append(f"{key} = {{{{ {var_name} | tojson }}}}") + lines.append( + f"{key} = " + f"{{{{ {var_name} | to_json(ensure_ascii=False) }}}}" + ) else: # Not a loop, treat as regular variable lines.append(f"{key} = {{{{ {var_name} }}}}") @@ -477,8 +480,10 @@ class TomlHandler(DictLikeHandler): f"]" ) else: - # Dict/nested loop - use tojson filter for complex arrays - replacement_value = f"{{{{ {collection_var} | tojson }}}}" + # Dict/nested loop - use to_json filter for complex arrays + replacement_value = ( + f"{{{{ {collection_var} | to_json(ensure_ascii=False) }}}}" + ) new_content = ( before_eq + "=" + leading_ws + replacement_value + comment_part diff --git a/src/jinjaturtle/handlers/yaml.py b/src/jinjaturtle/handlers/yaml.py index be1bd8d..7d35c9c 100644 --- a/src/jinjaturtle/handlers/yaml.py +++ b/src/jinjaturtle/handlers/yaml.py @@ -81,6 +81,15 @@ class YamlHandler(DictLikeHandler): return f"{{{{ 'null' if {value_expr} is none else {value_expr} }}}}" return f"{{{{ {value_expr} }}}}" + def _surrounding_quote(self, raw_value: str) -> str | None: + if ( + len(raw_value) >= 2 + and raw_value[0] == raw_value[-1] + and raw_value[0] in {'"', "'"} + ): + return raw_value[0] + return None + def _generate_yaml_template_from_text( self, role_prefix: str, @@ -226,7 +235,26 @@ class YamlHandler(DictLikeHandler): def current_path() -> tuple[str, ...]: return stack[-1][1] if stack else () - for raw_line in lines: + def first_sequence_item_quote( + start_index: int, parent_indent: int + ) -> str | None: + for future_line in lines[start_index + 1 :]: + future_stripped = future_line.lstrip() + future_indent = len(future_line) - len(future_stripped) + if not future_stripped or future_stripped.startswith("#"): + continue + if future_indent < parent_indent: + return None + if future_stripped.startswith("- "): + value_part, _comment_part = self._split_inline_comment( + future_stripped[2:], {"#"} + ) + return self._surrounding_quote(value_part.strip()) + if future_indent <= parent_indent: + return None + return None + + for line_index, raw_line in enumerate(lines): stripped = raw_line.lstrip() indent = len(raw_line) - len(stripped) @@ -287,8 +315,14 @@ class YamlHandler(DictLikeHandler): # Find the matching candidate candidate = next(c for c in loop_candidates if c.path == path) + scalar_quote = None + if candidate.item_schema == "scalar": + scalar_quote = first_sequence_item_quote(line_index, indent) + # Generate loop - loop_str = self._generate_yaml_loop(candidate, role_prefix, indent) + loop_str = self._generate_yaml_loop( + candidate, role_prefix, indent, scalar_quote=scalar_quote + ) out_lines.append(loop_str) # Skip subsequent lines that are part of this collection @@ -335,6 +369,10 @@ class YamlHandler(DictLikeHandler): stack.append((indent, parent_path, "seq")) parent_path = stack[-1][1] + content = stripped[2:] + value_part, _comment_part = self._split_inline_comment(content, {"#"}) + raw_value = value_part.strip() + scalar_quote = self._surrounding_quote(raw_value) # Check if parent path is a loop candidate if parent_path in loop_paths: @@ -345,7 +383,11 @@ class YamlHandler(DictLikeHandler): # Generate loop (with indent for the '-' items) loop_str = self._generate_yaml_loop( - candidate, role_prefix, indent, is_list=True + candidate, + role_prefix, + indent, + is_list=True, + scalar_quote=scalar_quote, ) out_lines.append(loop_str) @@ -353,7 +395,6 @@ class YamlHandler(DictLikeHandler): skip_until_indent = indent - 1 if indent > 0 else None continue - content = stripped[2:] index = seq_counters.get(parent_path, 0) seq_counters[parent_path] = index + 1 @@ -393,6 +434,7 @@ class YamlHandler(DictLikeHandler): role_prefix: str, indent: int, is_list: bool = False, + scalar_quote: str | None = None, ) -> str: """ Generate a Jinja2 for loop for a YAML collection. @@ -423,9 +465,10 @@ class YamlHandler(DictLikeHandler): item_indent_str = " " * item_indent if candidate.item_schema == "scalar": - item_lines.append( - f"{item_indent_str}- {self._yaml_value_expr(item_var, sample_item)}" - ) + value_expr = self._yaml_value_expr(item_var, sample_item) + if scalar_quote and isinstance(sample_item, str): + value_expr = f"{scalar_quote}{{{{ {item_var} }}}}{scalar_quote}" + item_lines.append(f"{item_indent_str}- {value_expr}") elif candidate.item_schema in ("simple_dict", "nested"): item_lines = self._dict_to_yaml_lines( sample_item, item_var, item_indent, is_list_item=True diff --git a/src/jinjaturtle/multi.py b/src/jinjaturtle/multi.py index 20cf544..d223b7e 100644 --- a/src/jinjaturtle/multi.py +++ b/src/jinjaturtle/multi.py @@ -16,7 +16,7 @@ Notes: * If the folder contains *multiple* formats, we generate one template per format (e.g. config.yaml.j2, config.xml.j2) and emit one list variable per format in the defaults YAML. - * JSON union templates are emitted using a simple `{{ data | tojson }}` + * JSON union templates are emitted using a simple `{{ data | to_json }}` approach to avoid comma-management complexity for optional keys. """ @@ -624,7 +624,7 @@ def process_directory( if multiple_formats else f"{role_prefix}_items" ) - template = "{{ data | tojson(indent=2) }}\n" + template = "{{ data | to_json(indent=2, ensure_ascii=False) }}\n" items: list[dict[str, Any]] = [] for rid, parsed in zip(rel_ids, parsed_list): items.append({"id": rid, "data": parsed}) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2f299a1..c0d5470 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -62,3 +62,73 @@ def test_cli_writes_output_files(tmp_path, capsys): # When writing to files, we shouldn't print the big headers assert "# defaults/main.yml" not in captured.out assert "# config.j2" not in captured.out + + +def test_cli_folder_stdout(tmp_path, capsys): + """Folder mode without output paths prints defaults and all templates.""" + (tmp_path / "a.json").write_text('{"name": "one"}\n', encoding="utf-8") + (tmp_path / "b.yaml").write_text("name: two\n", encoding="utf-8") + + exit_code = cli._main([str(tmp_path), "-r", "role"]) + + assert exit_code == 0 + captured = capsys.readouterr() + assert "# defaults/main.yml" in captured.out + assert "# config.json.j2" in captured.out + assert "# config.yaml.j2" in captured.out + assert "role_json_items" in captured.out + assert "role_yaml_items" in captured.out + + +def test_cli_folder_writes_template_directory(tmp_path, capsys): + """Folder mode writes multiple templates when template output is a directory.""" + src_dir = tmp_path / "src" + src_dir.mkdir() + (src_dir / "a.json").write_text('{"name": "one"}\n', encoding="utf-8") + (src_dir / "b.yaml").write_text("name: two\n", encoding="utf-8") + defaults_path = tmp_path / "defaults.yml" + template_dir = tmp_path / "templates" + + exit_code = cli._main( + [ + str(src_dir), + "-r", + "role", + "--defaults-output", + str(defaults_path), + "--template-output", + str(template_dir), + ] + ) + + assert exit_code == 0 + assert defaults_path.is_file() + assert (template_dir / "config.json.j2").is_file() + assert (template_dir / "config.yaml.j2").is_file() + captured = capsys.readouterr() + assert captured.out == "" + + +def test_cli_folder_single_output_file_when_one_format(tmp_path): + src_dir = tmp_path / "src" + src_dir.mkdir() + (src_dir / "a.json").write_text('{"name": "one"}\n', encoding="utf-8") + defaults_path = tmp_path / "defaults.yml" + template_path = tmp_path / "config.j2" + + exit_code = cli._main( + [ + str(src_dir), + "-r", + "role", + "--defaults-output", + str(defaults_path), + "--template-output", + str(template_path), + ] + ) + + assert exit_code == 0 + assert defaults_path.is_file() + assert template_path.is_file() + assert "to_json" in template_path.read_text(encoding="utf-8") diff --git a/tests/test_json_handler.py b/tests/test_json_handler.py index dd502b1..b2bbe75 100644 --- a/tests/test_json_handler.py +++ b/tests/test_json_handler.py @@ -1,9 +1,11 @@ from __future__ import annotations from pathlib import Path +import json import pytest import yaml +from jinja2 import Environment from jinjaturtle.core import ( parse_config, @@ -13,6 +15,7 @@ from jinjaturtle.core import ( generate_jinja2_template, ) from jinjaturtle.handlers.json import JsonHandler +from jinjaturtle.loop_analyzer import LoopCandidate SAMPLES_DIR = Path(__file__).parent / "samples" @@ -35,23 +38,136 @@ def test_json_roundtrip(): assert defaults["foobar_nested_a"] == 1 # Booleans are now preserved as booleans (not stringified) assert defaults["foobar_nested_b"] is True - # List should be a list (not flattened to scalars) - assert defaults["foobar_list"] == [10, 20] + # JSON stays scalar/index-based so source formatting can be preserved. + assert loop_candidates == [] + assert defaults["foobar_list_0"] == 10 + assert defaults["foobar_list_1"] == 20 - # Template generation with loops template = generate_jinja2_template("json", parsed, "foobar", None, loop_candidates) - # Template should use | tojson for type preservation - assert "{{ foobar_foo | tojson }}" in template - assert "{{ foobar_nested_a | tojson }}" in template - assert "{{ foobar_nested_b | tojson }}" in template + # Template should use Ansible | to_json for type preservation without + # HTML-safe escaping. + assert "{{ foobar_foo | to_json(ensure_ascii=False) }}" in template + assert "{{ foobar_nested_a | to_json(ensure_ascii=False) }}" in template + assert "{{ foobar_nested_b | to_json(ensure_ascii=False) }}" in template + assert "{{ foobar_list_0 | to_json(ensure_ascii=False) }}" in template + assert "{{ foobar_list_1 | to_json(ensure_ascii=False) }}" in template + assert "{% for" not in template - # List should use loop (not scalar indices) - assert "{% for" in template - assert "foobar_list" in template - # Should NOT have scalar indices - assert "foobar_list_0" not in template - assert "foobar_list_1" not in template + +def test_json_template_preserves_original_indentation_and_final_newline_state( + tmp_path: Path, +): + json_text = ( + "{\n" + ' "default-runtime": "runsc",\n' + ' "runtimes": {\n' + ' "runsc": {\n' + ' "path": "/usr/bin/runsc"\n' + " }\n" + " }\n" + "}" + ) + path = tmp_path / "daemon.json" + path.write_text(json_text, encoding="utf-8") + + fmt, parsed = parse_config(path) + template = generate_jinja2_template(fmt, parsed, "docker", json_text, []) + + assert ' "default-runtime"' in template + assert ' "runsc"' in template + assert ' "path"' in template + assert not template.endswith("\n") + + +def test_json_template_uses_non_html_safe_json_filter_for_angle_brackets( + tmp_path: Path, +): + json_text = '{"version_requirement": ">= 8.0.0 < 9.0.0"}\n' + path = tmp_path / "metadata.json" + path.write_text(json_text, encoding="utf-8") + + fmt, parsed = parse_config(path) + template = generate_jinja2_template(fmt, parsed, "puppet", json_text, []) + + assert "to_json(ensure_ascii=False)" in template + assert "tojson" not in template + + +def test_json_inline_object_array_roundtrips_without_expanding(tmp_path: Path): + json_text = ( + "{\n" + ' "description": "Gets the last boot time of a Linux or Windows system",\n' + ' "implementations": [\n' + ' {"name": "last_boot_time_nix.sh", "requirements": ["shell"]},\n' + ' {"name": "last_boot_time_win.ps1", "requirements": ["powershell"]}\n' + " ]\n" + "}\n" + ) + path = tmp_path / "metadata.json" + path.write_text(json_text, encoding="utf-8") + + fmt, parsed = parse_config(path) + loop_candidates = analyze_loops(fmt, parsed) + flat_items = flatten_config(fmt, parsed, loop_candidates) + defaults = yaml.safe_load( + generate_ansible_yaml("fact", flat_items, loop_candidates) + ) + template = generate_jinja2_template(fmt, parsed, "fact", json_text, loop_candidates) + + env = Environment(keep_trailing_newline=True) + env.filters["to_json"] = lambda value, **kwargs: json.dumps(value, **kwargs) + rendered = env.from_string(template).render(**defaults) + + assert loop_candidates == [] + assert '{"name": {{ fact_implementations_0_name' in template + assert rendered == json_text + + +def test_json_direct_loop_generation_renders_valid_json_without_joined_objects(): + items = [ + {"name": "last_boot_time_nix.sh", "requirements": ["shell"]}, + {"name": "last_boot_time_win.ps1", "requirements": ["powershell"]}, + ] + parsed = {"implementations": items} + candidate = LoopCandidate( + path=("implementations",), + loop_var="implementation", + items=items, + item_schema="simple_dict", + ) + handler = JsonHandler() + template = handler.generate_jinja2_template_with_loops( + parsed, "fact", None, [candidate] + ) + + env = Environment(keep_trailing_newline=True) + env.filters["to_json"] = lambda value, **kwargs: json.dumps(value, **kwargs) + rendered = env.from_string(template).render(fact_implementations=items) + + assert "}, {" not in rendered + assert json.loads(rendered) == parsed + + +def test_json_scalar_span_collector_empty_containers_and_invalid_json(): + handler = JsonHandler() + + assert handler._collect_json_scalar_spans('{"empty": [], "obj": {}}') == [] + assert handler._collect_json_scalar_spans('{"unterminated": [1,}') is None + + +def test_json_template_falls_back_when_source_scanner_cannot_collect_spans(monkeypatch): + handler = JsonHandler() + monkeypatch.setattr(handler, "_collect_json_scalar_spans", lambda _text: None) + + template = handler.generate_jinja2_template( + {"answer": 42}, "role", original_text='{"answer": 42}' + ) + + assert ( + template + == '{\n "answer": {{ role_answer | to_json(ensure_ascii=False) }}\n}\n' + ) def test_generate_jinja2_template_json_type_error(): diff --git a/tests/test_multi.py b/tests/test_multi.py new file mode 100644 index 0000000..2076655 --- /dev/null +++ b/tests/test_multi.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from jinjaturtle.multi import ( + _collect_dict_like_paths, + _merge_union, + defined_var_name, + is_supported_file, + iter_supported_files, + process_directory, +) + + +def test_iter_supported_files_handles_files_dirs_and_missing_paths(tmp_path: Path): + json_file = tmp_path / "config.json" + text_file = tmp_path / "notes.txt" + nested = tmp_path / "nested" + nested.mkdir() + nested_yaml = nested / "config.yaml" + + json_file.write_text('{"name": "one"}\n', encoding="utf-8") + text_file.write_text("ignore me\n", encoding="utf-8") + nested_yaml.write_text("name: two\n", encoding="utf-8") + + assert is_supported_file(json_file) + assert not is_supported_file(text_file) + assert iter_supported_files(json_file, recursive=False) == [json_file] + assert iter_supported_files(text_file, recursive=False) == [] + assert iter_supported_files(tmp_path, recursive=False) == [json_file] + assert iter_supported_files(tmp_path, recursive=True) == [json_file, nested_yaml] + + with pytest.raises(FileNotFoundError): + iter_supported_files(tmp_path / "missing", recursive=False) + + +def test_merge_union_and_collect_dict_like_paths(): + merged = _merge_union( + {"name": "one", "ports": [80], "nested": {"a": 1}}, + {"ports": [80, 443], "nested": {"b": 2}, "enabled": True}, + ) + + assert merged == { + "name": "one", + "ports": [80, 443], + "nested": {"a": 1, "b": 2}, + "enabled": True, + } + + containers, leaves = _collect_dict_like_paths(merged) + assert ("nested",) in containers + assert ("nested", "a") in containers + assert ("ports", "1") in leaves + + +def test_process_directory_multiple_formats(tmp_path: Path): + (tmp_path / "a.json").write_text('{"name": "one"}\n', encoding="utf-8") + (tmp_path / "b.yaml").write_text("name: one\nextra: true\n", encoding="utf-8") + (tmp_path / "c.toml").write_text('name = "one"\n', encoding="utf-8") + (tmp_path / "d.ini").write_text("[main]\nname = one\n", encoding="utf-8") + (tmp_path / "e.xml").write_text("one", encoding="utf-8") + + defaults_yaml, outputs = process_directory( + tmp_path, recursive=False, role_prefix="role" + ) + defaults = yaml.safe_load(defaults_yaml) + by_fmt = {output.fmt: output for output in outputs} + + assert set(by_fmt) == {"ini", "json", "toml", "xml", "yaml"} + assert set(defaults) == { + "role_ini_items", + "role_json_items", + "role_toml_items", + "role_xml_items", + "role_yaml_items", + } + assert defaults["role_json_items"][0]["data"] == {"name": "one"} + assert ( + by_fmt["json"].template + == "{{ data | to_json(indent=2, ensure_ascii=False) }}\n" + ) + assert "{{ role_main_name }}" in by_fmt["ini"].template + assert "{{ role_name }}" in by_fmt["xml"].template + + +def test_process_directory_yaml_union_marks_optional_keys(tmp_path: Path): + (tmp_path / "one.yaml").write_text("name: one\nextra: yes\n", encoding="utf-8") + (tmp_path / "two.yaml").write_text("name: two\n", encoding="utf-8") + + defaults_yaml, outputs = process_directory( + tmp_path, recursive=False, role_prefix="role" + ) + defaults = yaml.safe_load(defaults_yaml) + + assert len(outputs) == 1 + assert outputs[0].fmt == "yaml" + assert "{% if role_defined_extra is defined %}" in outputs[0].template + assert defaults["role_items"][0][defined_var_name("role", ("extra",))] is True + assert defined_var_name("role", ("extra",)) not in defaults["role_items"][1] + + +def test_process_directory_ini_union_marks_optional_sections_and_keys(tmp_path: Path): + (tmp_path / "one.ini").write_text( + "[main]\nname = one\n[extra]\nflag = yes\n", encoding="utf-8" + ) + (tmp_path / "two.ini").write_text("[main]\nname = two\n", encoding="utf-8") + + defaults_yaml, outputs = process_directory( + tmp_path, recursive=False, role_prefix="role" + ) + defaults = yaml.safe_load(defaults_yaml) + + assert len(outputs) == 1 + assert outputs[0].fmt == "ini" + assert "{% if role_defined_extra is defined %}" in outputs[0].template + assert defaults["role_items"][0][defined_var_name("role", ("extra",))] is True + assert defined_var_name("role", ("extra",)) not in defaults["role_items"][1] + + +def test_process_directory_rejects_empty_folder(tmp_path: Path): + with pytest.raises(ValueError, match="No supported config files"): + process_directory(tmp_path, recursive=False, role_prefix="role") diff --git a/tests/test_roundtrip.py b/tests/test_roundtrip.py index 850cc9a..1a6d47b 100644 --- a/tests/test_roundtrip.py +++ b/tests/test_roundtrip.py @@ -31,6 +31,7 @@ from jinjaturtle.core import ( def render_template(template: str, variables: dict[str, Any]) -> str: """Render a Jinja2 template with variables.""" env = Environment(undefined=StrictUndefined) + env.filters["to_json"] = lambda value, **kwargs: json.dumps(value, **kwargs) jinja_template = env.from_string(template) return jinja_template.render(variables) diff --git a/tests/test_yaml_handler.py b/tests/test_yaml_handler.py index 41edc07..184dfbc 100644 --- a/tests/test_yaml_handler.py +++ b/tests/test_yaml_handler.py @@ -7,6 +7,7 @@ import yaml from jinjaturtle.core import ( parse_config, + analyze_loops, flatten_config, generate_ansible_yaml, generate_jinja2_template, @@ -204,6 +205,36 @@ def test_yaml_loop_preserves_following_top_level_comments(tmp_path: Path): assert yaml.safe_load(rendered) == yaml.safe_load(text) +def test_yaml_scalar_loop_preserves_quoted_list_items(tmp_path: Path): + from jinja2 import Template + + text = textwrap.dedent( + """ + files: + - 'spec/unit/puppet/provider/dsc_base_provider/dsc_base_provider_spec.rb' + - 'spec/unit/pwsh/util_spec.rb' + - 'spec/unit/pwsh/windows_powershell_spec.rb' + """ + ).lstrip() + path = tmp_path / "quoted-list.yaml" + path.write_text(text, encoding="utf-8") + + fmt, parsed = parse_config(path) + loop_candidates = analyze_loops(fmt, parsed) + flat_items = flatten_config(fmt, parsed, loop_candidates) + defaults = yaml.safe_load( + generate_ansible_yaml("role", flat_items, loop_candidates) + ) + template = generate_jinja2_template( + fmt, parsed, "role", original_text=text, loop_candidates=loop_candidates + ) + rendered = Template(template).render(**defaults) + + assert loop_candidates + assert "- '{{" in template + assert rendered == text + + def test_yaml_bool_scalars_render_with_yaml_spelling(tmp_path: Path): from jinja2 import Template diff --git a/tests/test_yaml_template_consistency.py b/tests/test_yaml_template_consistency.py index 22a3c69..5a48502 100644 --- a/tests/test_yaml_template_consistency.py +++ b/tests/test_yaml_template_consistency.py @@ -454,6 +454,7 @@ class TestStructuralConsistency: # Try to render the template env = Environment(undefined=StrictUndefined) + env.filters["to_json"] = lambda value, **kwargs: json.dumps(value, **kwargs) try: jinja_template = env.from_string(template) rendered = jinja_template.render(variables) @@ -520,11 +521,9 @@ class TestRegressionBugs: "app_database" in template ), f"Template should reference app_database\n{template}" - def test_json_array_no_index_refs(self): + def test_json_array_uses_index_refs_for_source_fidelity(self): """ - Regression test: JSON arrays should not generate index references. - - Bug: Template had {{ app_list_0 }}, {{ app_list_1 }} when YAML had app_list as list. + JSON arrays stay index-based so original inline formatting can survive. """ import json @@ -536,22 +535,18 @@ class TestRegressionBugs: ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates) template = generate_jinja2_template( - "json", parsed, "app", None, loop_candidates + "json", parsed, "app", json_text, loop_candidates ) - # YAML should have app_items as a list defaults = yaml.safe_load(ansible_yaml) - assert isinstance(defaults.get("app_items"), list) - - # Template should NOT have app_items_0, app_items_1, app_items_2 - for i in range(3): - assert ( - f"app_items_{i}" not in template - ), f"Template incorrectly uses scalar 'app_items_{i}'\n{template}" - - # Template SHOULD use a loop - assert "{% for" in template - assert "app_items" in template + assert loop_candidates == [] + assert defaults["app_items_0"] == 1 + assert defaults["app_items_1"] == 2 + assert defaults["app_items_2"] == 3 + assert "{% for" not in template + assert "app_items_0" in template + assert "app_items_1" in template + assert "app_items_2" in template if __name__ == "__main__":