Make templates more faithful to the original file in terms of indentation, newlines, no deserialisation of things like < or >. More test coverage
This commit is contained in:
parent
3d53d4fb30
commit
49fee7afe4
GPG key ID:
03906B4110AAD3B8
12 changed files with 579 additions and 52 deletions
|
|
@ -276,7 +276,15 @@ def parse_config(path: Path, fmt: str | None = None) -> tuple[str, Any]:
|
|||
def analyze_loops(fmt: str, parsed: Any) -> list[LoopCandidate]:
|
||||
"""
|
||||
Analyze parsed config to find loop opportunities.
|
||||
|
||||
JSON files are intentionally kept scalar/index-based instead of being
|
||||
collapsed into generated loops. JSON is commonly checked byte-for-byte by
|
||||
configuration management tools, and preserving inline arrays/objects is
|
||||
more valuable than reducing variable count.
|
||||
"""
|
||||
if fmt == "json":
|
||||
return []
|
||||
|
||||
analyzer = LoopAnalyzer()
|
||||
candidates = analyzer.analyze(parsed, fmt)
|
||||
|
||||
|
|
|
|||
|
|
@ -23,13 +23,15 @@ class JsonHandler(DictLikeHandler):
|
|||
role_prefix: str,
|
||||
original_text: str | None = None,
|
||||
) -> str:
|
||||
"""Original scalar-only template generation."""
|
||||
"""Generate a scalar JSON template while preserving source formatting."""
|
||||
if not isinstance(parsed, (dict, list)):
|
||||
raise TypeError("JSON parser result must be a dict or list")
|
||||
# As before: ignore original_text and rebuild structurally
|
||||
if original_text is not None:
|
||||
return self._generate_json_template_from_text(role_prefix, original_text)
|
||||
return self._generate_json_template(role_prefix, parsed)
|
||||
|
||||
JSON_INDENT = 2
|
||||
JSON_VALUE_FILTER = "to_json(ensure_ascii=False)"
|
||||
|
||||
def _leading_indent(self, s: str, idx: int) -> int:
|
||||
"""Return the number of leading spaces on the line containing idx."""
|
||||
|
|
@ -75,6 +77,129 @@ class JsonHandler(DictLikeHandler):
|
|||
role_prefix, parsed, loop_paths, loop_candidates
|
||||
)
|
||||
|
||||
def _json_value_expr(self, var_name: str) -> str:
|
||||
"""Return a Jinja expression for a JSON value.
|
||||
|
||||
Jinja's built-in ``tojson`` filter is HTML-safe and therefore escapes
|
||||
characters such as ``<`` and ``>`` as ``\u003c``/``\u003e``. That is
|
||||
useful in HTML, but noisy in configuration files. JinjaTurtle generates
|
||||
Ansible templates, so use Ansible's ``to_json`` filter instead.
|
||||
"""
|
||||
return f"{{{{ {var_name} | {self.JSON_VALUE_FILTER} }}}}"
|
||||
|
||||
def _generate_json_template_from_text(self, role_prefix: str, text: str) -> str:
|
||||
"""Replace JSON scalar values in-place, preserving original formatting.
|
||||
|
||||
The older JSON path parsed the file and wrote it back with
|
||||
``json.dumps(indent=2)``, which caused cosmetic diffs such as changing
|
||||
four-space indentation to two-space indentation and adding a final
|
||||
newline to files that intentionally lacked one. This scanner walks the
|
||||
original JSON source and only replaces scalar value tokens with Jinja2
|
||||
expressions; all whitespace, object/list indentation, key ordering, and
|
||||
final newline state are left untouched.
|
||||
"""
|
||||
spans = self._collect_json_scalar_spans(text)
|
||||
if spans is None:
|
||||
# Should be rare because the caller has already parsed the JSON, but
|
||||
# keep the structural fallback rather than failing template creation.
|
||||
parsed = json.loads(text)
|
||||
return self._generate_json_template(role_prefix, parsed)
|
||||
|
||||
chunks: list[str] = []
|
||||
pos = 0
|
||||
for path, start, end in spans:
|
||||
chunks.append(text[pos:start])
|
||||
chunks.append(self._json_value_expr(self.make_var_name(role_prefix, path)))
|
||||
pos = end
|
||||
chunks.append(text[pos:])
|
||||
return "".join(chunks)
|
||||
|
||||
def _collect_json_scalar_spans(
|
||||
self, text: str
|
||||
) -> list[tuple[tuple[str, ...], int, int]] | None:
|
||||
"""Return source spans for JSON scalar *values*.
|
||||
|
||||
Keys are parsed to determine the current path but are not returned.
|
||||
"""
|
||||
decoder = json.JSONDecoder()
|
||||
spans: list[tuple[tuple[str, ...], int, int]] = []
|
||||
|
||||
def skip_ws(i: int) -> int:
|
||||
while i < len(text) and text[i] in " \t\r\n":
|
||||
i += 1
|
||||
return i
|
||||
|
||||
def raw_decode_at(i: int) -> tuple[Any, int]:
|
||||
return decoder.raw_decode(text, i)
|
||||
|
||||
def parse_value(i: int, path: tuple[str, ...]) -> int:
|
||||
i = skip_ws(i)
|
||||
if i >= len(text):
|
||||
raise ValueError("unexpected end of JSON")
|
||||
ch = text[i]
|
||||
if ch == "{":
|
||||
return parse_object(i, path)
|
||||
if ch == "[":
|
||||
return parse_array(i, path)
|
||||
|
||||
_value, end = raw_decode_at(i)
|
||||
spans.append((path, i, end))
|
||||
return end
|
||||
|
||||
def parse_object(i: int, path: tuple[str, ...]) -> int:
|
||||
i += 1 # {
|
||||
i = skip_ws(i)
|
||||
if i < len(text) and text[i] == "}":
|
||||
return i + 1
|
||||
|
||||
while True:
|
||||
i = skip_ws(i)
|
||||
if i >= len(text) or text[i] != '"':
|
||||
raise ValueError("expected JSON object key")
|
||||
key, i = raw_decode_at(i)
|
||||
if not isinstance(key, str):
|
||||
raise ValueError("expected JSON object key string")
|
||||
|
||||
i = skip_ws(i)
|
||||
if i >= len(text) or text[i] != ":":
|
||||
raise ValueError("expected ':' after JSON object key")
|
||||
i = parse_value(i + 1, path + (key,))
|
||||
i = skip_ws(i)
|
||||
|
||||
if i < len(text) and text[i] == ",":
|
||||
i += 1
|
||||
continue
|
||||
if i < len(text) and text[i] == "}":
|
||||
return i + 1
|
||||
raise ValueError("expected ',' or '}' in JSON object")
|
||||
|
||||
def parse_array(i: int, path: tuple[str, ...]) -> int:
|
||||
i += 1 # [
|
||||
i = skip_ws(i)
|
||||
if i < len(text) and text[i] == "]":
|
||||
return i + 1
|
||||
|
||||
index = 0
|
||||
while True:
|
||||
i = parse_value(i, path + (str(index),))
|
||||
index += 1
|
||||
i = skip_ws(i)
|
||||
|
||||
if i < len(text) and text[i] == ",":
|
||||
i += 1
|
||||
continue
|
||||
if i < len(text) and text[i] == "]":
|
||||
return i + 1
|
||||
raise ValueError("expected ',' or ']' in JSON array")
|
||||
|
||||
try:
|
||||
end = parse_value(0, ())
|
||||
if skip_ws(end) != len(text):
|
||||
return None
|
||||
return spans
|
||||
except (json.JSONDecodeError, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
def _generate_json_template(self, role_prefix: str, data: Any) -> str:
|
||||
"""
|
||||
Generate a JSON Jinja2 template from parsed JSON data.
|
||||
|
|
@ -82,7 +207,7 @@ class JsonHandler(DictLikeHandler):
|
|||
All scalar values are replaced with Jinja expressions whose names are
|
||||
derived from the path, similar to TOML/YAML.
|
||||
|
||||
Uses | tojson filter to preserve types (numbers, booleans, null).
|
||||
Uses | to_json filter to preserve types (numbers, booleans, null).
|
||||
"""
|
||||
|
||||
def _walk(obj: Any, path: tuple[str, ...] = ()) -> Any:
|
||||
|
|
@ -90,17 +215,19 @@ class JsonHandler(DictLikeHandler):
|
|||
return {k: _walk(v, path + (str(k),)) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [_walk(v, path + (str(i),)) for i, v in enumerate(obj)]
|
||||
# scalar - use marker that will be replaced with tojson
|
||||
# scalar - use marker that will be replaced with to_json
|
||||
var_name = self.make_var_name(role_prefix, path)
|
||||
return f"__SCALAR__{var_name}__"
|
||||
|
||||
templated = _walk(data)
|
||||
json_str = json.dumps(templated, indent=2, ensure_ascii=False)
|
||||
|
||||
# Replace scalar markers with Jinja expressions using tojson filter
|
||||
# Replace scalar markers with Jinja expressions using to_json filter
|
||||
# This preserves types (numbers stay numbers, booleans stay booleans)
|
||||
json_str = re.sub(
|
||||
r'"__SCALAR__([a-zA-Z_][a-zA-Z0-9_]*)__"', r"{{ \1 | tojson }}", json_str
|
||||
r'"__SCALAR__([a-zA-Z_][a-zA-Z0-9_]*)__"',
|
||||
lambda m: self._json_value_expr(m.group(1)),
|
||||
json_str,
|
||||
)
|
||||
|
||||
return json_str + "\n"
|
||||
|
|
@ -150,9 +277,11 @@ class JsonHandler(DictLikeHandler):
|
|||
# Convert to JSON string
|
||||
json_str = json.dumps(templated, indent=2, ensure_ascii=False)
|
||||
|
||||
# Replace scalar markers with Jinja expressions using tojson filter
|
||||
# Replace scalar markers with Jinja expressions using to_json filter
|
||||
json_str = re.sub(
|
||||
r'"__SCALAR__([a-zA-Z_][a-zA-Z0-9_]*)__"', r"{{ \1 | tojson }}", json_str
|
||||
r'"__SCALAR__([a-zA-Z_][a-zA-Z0-9_]*)__"',
|
||||
lambda m: self._json_value_expr(m.group(1)),
|
||||
json_str,
|
||||
)
|
||||
|
||||
# Post-process to replace loop markers with actual Jinja loops (indent-aware)
|
||||
|
|
@ -201,7 +330,8 @@ class JsonHandler(DictLikeHandler):
|
|||
# a blank line between iterations under default Jinja whitespace settings.
|
||||
return (
|
||||
f"[\n"
|
||||
f"{{% for {item_var} in {collection_var} %}}{inner}{{{{ {item_var} | tojson }}}}"
|
||||
f"{{% for {item_var} in {collection_var} %}}"
|
||||
f"{inner}{{{{ {item_var} | to_json(ensure_ascii=False) }}}}"
|
||||
f"{{% if not loop.last %}},{{% endif %}}\n"
|
||||
f"{{% endfor %}}{base}]"
|
||||
)
|
||||
|
|
@ -234,7 +364,8 @@ class JsonHandler(DictLikeHandler):
|
|||
for i, key in enumerate(keys):
|
||||
comma = "," if i < len(keys) - 1 else ""
|
||||
dict_lines.append(
|
||||
f'{field}"{key}": {{{{ {item_var}.{key} | tojson }}}}{comma}'
|
||||
f'{field}"{key}": '
|
||||
f"{{{{ {item_var}.{key} | to_json(ensure_ascii=False) }}}}{comma}"
|
||||
)
|
||||
# Comma between *items* goes after the closing brace.
|
||||
dict_lines.append(f"{inner}}}{{% if not loop.last %}},{{% endif %}}")
|
||||
|
|
|
|||
|
|
@ -148,7 +148,10 @@ class TomlHandler(DictLikeHandler):
|
|||
elif candidate.item_schema in ("simple_dict", "nested"):
|
||||
# Dict list loop - TOML array of tables
|
||||
# This is complex for TOML, using simplified approach
|
||||
lines.append(f"{key} = {{{{ {var_name} | tojson }}}}")
|
||||
lines.append(
|
||||
f"{key} = "
|
||||
f"{{{{ {var_name} | to_json(ensure_ascii=False) }}}}"
|
||||
)
|
||||
else:
|
||||
# Not a loop, treat as regular variable
|
||||
lines.append(f"{key} = {{{{ {var_name} }}}}")
|
||||
|
|
@ -477,8 +480,10 @@ class TomlHandler(DictLikeHandler):
|
|||
f"]"
|
||||
)
|
||||
else:
|
||||
# Dict/nested loop - use tojson filter for complex arrays
|
||||
replacement_value = f"{{{{ {collection_var} | tojson }}}}"
|
||||
# Dict/nested loop - use to_json filter for complex arrays
|
||||
replacement_value = (
|
||||
f"{{{{ {collection_var} | to_json(ensure_ascii=False) }}}}"
|
||||
)
|
||||
|
||||
new_content = (
|
||||
before_eq + "=" + leading_ws + replacement_value + comment_part
|
||||
|
|
|
|||
|
|
@ -81,6 +81,15 @@ class YamlHandler(DictLikeHandler):
|
|||
return f"{{{{ 'null' if {value_expr} is none else {value_expr} }}}}"
|
||||
return f"{{{{ {value_expr} }}}}"
|
||||
|
||||
def _surrounding_quote(self, raw_value: str) -> str | None:
|
||||
if (
|
||||
len(raw_value) >= 2
|
||||
and raw_value[0] == raw_value[-1]
|
||||
and raw_value[0] in {'"', "'"}
|
||||
):
|
||||
return raw_value[0]
|
||||
return None
|
||||
|
||||
def _generate_yaml_template_from_text(
|
||||
self,
|
||||
role_prefix: str,
|
||||
|
|
@ -226,7 +235,26 @@ class YamlHandler(DictLikeHandler):
|
|||
def current_path() -> tuple[str, ...]:
|
||||
return stack[-1][1] if stack else ()
|
||||
|
||||
for raw_line in lines:
|
||||
def first_sequence_item_quote(
|
||||
start_index: int, parent_indent: int
|
||||
) -> str | None:
|
||||
for future_line in lines[start_index + 1 :]:
|
||||
future_stripped = future_line.lstrip()
|
||||
future_indent = len(future_line) - len(future_stripped)
|
||||
if not future_stripped or future_stripped.startswith("#"):
|
||||
continue
|
||||
if future_indent < parent_indent:
|
||||
return None
|
||||
if future_stripped.startswith("- "):
|
||||
value_part, _comment_part = self._split_inline_comment(
|
||||
future_stripped[2:], {"#"}
|
||||
)
|
||||
return self._surrounding_quote(value_part.strip())
|
||||
if future_indent <= parent_indent:
|
||||
return None
|
||||
return None
|
||||
|
||||
for line_index, raw_line in enumerate(lines):
|
||||
stripped = raw_line.lstrip()
|
||||
indent = len(raw_line) - len(stripped)
|
||||
|
||||
|
|
@ -287,8 +315,14 @@ class YamlHandler(DictLikeHandler):
|
|||
# Find the matching candidate
|
||||
candidate = next(c for c in loop_candidates if c.path == path)
|
||||
|
||||
scalar_quote = None
|
||||
if candidate.item_schema == "scalar":
|
||||
scalar_quote = first_sequence_item_quote(line_index, indent)
|
||||
|
||||
# Generate loop
|
||||
loop_str = self._generate_yaml_loop(candidate, role_prefix, indent)
|
||||
loop_str = self._generate_yaml_loop(
|
||||
candidate, role_prefix, indent, scalar_quote=scalar_quote
|
||||
)
|
||||
out_lines.append(loop_str)
|
||||
|
||||
# Skip subsequent lines that are part of this collection
|
||||
|
|
@ -335,6 +369,10 @@ class YamlHandler(DictLikeHandler):
|
|||
stack.append((indent, parent_path, "seq"))
|
||||
|
||||
parent_path = stack[-1][1]
|
||||
content = stripped[2:]
|
||||
value_part, _comment_part = self._split_inline_comment(content, {"#"})
|
||||
raw_value = value_part.strip()
|
||||
scalar_quote = self._surrounding_quote(raw_value)
|
||||
|
||||
# Check if parent path is a loop candidate
|
||||
if parent_path in loop_paths:
|
||||
|
|
@ -345,7 +383,11 @@ class YamlHandler(DictLikeHandler):
|
|||
|
||||
# Generate loop (with indent for the '-' items)
|
||||
loop_str = self._generate_yaml_loop(
|
||||
candidate, role_prefix, indent, is_list=True
|
||||
candidate,
|
||||
role_prefix,
|
||||
indent,
|
||||
is_list=True,
|
||||
scalar_quote=scalar_quote,
|
||||
)
|
||||
out_lines.append(loop_str)
|
||||
|
||||
|
|
@ -353,7 +395,6 @@ class YamlHandler(DictLikeHandler):
|
|||
skip_until_indent = indent - 1 if indent > 0 else None
|
||||
continue
|
||||
|
||||
content = stripped[2:]
|
||||
index = seq_counters.get(parent_path, 0)
|
||||
seq_counters[parent_path] = index + 1
|
||||
|
||||
|
|
@ -393,6 +434,7 @@ class YamlHandler(DictLikeHandler):
|
|||
role_prefix: str,
|
||||
indent: int,
|
||||
is_list: bool = False,
|
||||
scalar_quote: str | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Generate a Jinja2 for loop for a YAML collection.
|
||||
|
|
@ -423,9 +465,10 @@ class YamlHandler(DictLikeHandler):
|
|||
item_indent_str = " " * item_indent
|
||||
|
||||
if candidate.item_schema == "scalar":
|
||||
item_lines.append(
|
||||
f"{item_indent_str}- {self._yaml_value_expr(item_var, sample_item)}"
|
||||
)
|
||||
value_expr = self._yaml_value_expr(item_var, sample_item)
|
||||
if scalar_quote and isinstance(sample_item, str):
|
||||
value_expr = f"{scalar_quote}{{{{ {item_var} }}}}{scalar_quote}"
|
||||
item_lines.append(f"{item_indent_str}- {value_expr}")
|
||||
elif candidate.item_schema in ("simple_dict", "nested"):
|
||||
item_lines = self._dict_to_yaml_lines(
|
||||
sample_item, item_var, item_indent, is_list_item=True
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ Notes:
|
|||
* If the folder contains *multiple* formats, we generate one template per
|
||||
format (e.g. config.yaml.j2, config.xml.j2) and emit one list variable per
|
||||
format in the defaults YAML.
|
||||
* JSON union templates are emitted using a simple `{{ data | tojson }}`
|
||||
* JSON union templates are emitted using a simple `{{ data | to_json }}`
|
||||
approach to avoid comma-management complexity for optional keys.
|
||||
"""
|
||||
|
||||
|
|
@ -624,7 +624,7 @@ def process_directory(
|
|||
if multiple_formats
|
||||
else f"{role_prefix}_items"
|
||||
)
|
||||
template = "{{ data | tojson(indent=2) }}\n"
|
||||
template = "{{ data | to_json(indent=2, ensure_ascii=False) }}\n"
|
||||
items: list[dict[str, Any]] = []
|
||||
for rid, parsed in zip(rel_ids, parsed_list):
|
||||
items.append({"id": rid, "data": parsed})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue