Refactor and add much more robust tests (both automated and manual) to ensure loops and things work ok

tests/test_cli.py

@@ -1,10 +1,6 @@
 from __future__ import annotations
 
-import sys
 from pathlib import Path
-
-import pytest
-
 from jinjaturtle import cli
 
 SAMPLES_DIR = Path(__file__).parent / "samples"

tests/test_core_utils.py

@@ -168,8 +168,8 @@ def test_fallback_str_representer_for_unknown_type():
 
 def test_normalize_default_value_bool_inputs_are_stringified():
     """
-    Real boolean values should be turned into quoted 'true'/'false' strings
-    by _normalize_default_value via generate_ansible_yaml.
+    Boolean values are now preserved as booleans in YAML (not stringified).
+    This supports proper type preservation for JSON and other formats.
     """
     flat_items = [
         (("section", "flag_true"), True),
@@ -178,8 +178,9 @@ def test_normalize_default_value_bool_inputs_are_stringified():
     ansible_yaml = generate_ansible_yaml("role", flat_items)
     data = yaml.safe_load(ansible_yaml)
 
-    assert data["role_section_flag_true"] == "true"
-    assert data["role_section_flag_false"] == "false"
+    # Booleans are now preserved as booleans
+    assert data["role_section_flag_true"] is True
+    assert data["role_section_flag_false"] is False
 
 
 def test_flatten_config_unsupported_format():

tests/test_json_handler.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 from pathlib import Path
 
-import json
 import pytest
 import yaml
 
@@ -10,6 +9,8 @@ from jinjaturtle.core import (
     parse_config,
     flatten_config,
     generate_ansible_yaml,
+    analyze_loops,
+    generate_jinja2_template,
 )
 from jinjaturtle.handlers.json import JsonHandler
 
@@ -23,30 +24,34 @@ def test_json_roundtrip():
     fmt, parsed = parse_config(json_path)
     assert fmt == "json"
 
-    flat_items = flatten_config(fmt, parsed)
-    ansible_yaml = generate_ansible_yaml("foobar", flat_items)
+    # With loop detection
+    loop_candidates = analyze_loops(fmt, parsed)
+    flat_items = flatten_config(fmt, parsed, loop_candidates)
+    ansible_yaml = generate_ansible_yaml("foobar", flat_items, loop_candidates)
     defaults = yaml.safe_load(ansible_yaml)
 
-    # Defaults: nested keys and list indices
+    # Defaults: nested keys
     assert defaults["foobar_foo"] == "bar"
     assert defaults["foobar_nested_a"] == 1
-    # Bool normalized to string "true"
-    assert defaults["foobar_nested_b"] == "true"
-    assert defaults["foobar_list_0"] == 10
-    assert defaults["foobar_list_1"] == 20
+    # Booleans are now preserved as booleans (not stringified)
+    assert defaults["foobar_nested_b"] is True
+    # List should be a list (not flattened to scalars)
+    assert defaults["foobar_list"] == [10, 20]
 
-    # Template generation is done via JsonHandler.generate_jinja2_template; we just
-    # make sure it produces a structure with the expected placeholders.
-    handler = JsonHandler()
-    templated = json.loads(
-        handler.generate_jinja2_template(parsed, role_prefix="foobar")
-    )
+    # Template generation with loops
+    template = generate_jinja2_template("json", parsed, "foobar", None, loop_candidates)
 
-    assert templated["foo"] == "{{ foobar_foo }}"
-    assert "foobar_nested_a" in str(templated)
-    assert "foobar_nested_b" in str(templated)
-    assert "foobar_list_0" in str(templated)
-    assert "foobar_list_1" in str(templated)
+    # Template should use | tojson for type preservation
+    assert "{{ foobar_foo | tojson }}" in template
+    assert "{{ foobar_nested_a | tojson }}" in template
+    assert "{{ foobar_nested_b | tojson }}" in template
+
+    # List should use loop (not scalar indices)
+    assert "{% for" in template
+    assert "foobar_list" in template
+    # Should NOT have scalar indices
+    assert "foobar_list_0" not in template
+    assert "foobar_list_1" not in template
 
 
 def test_generate_jinja2_template_json_type_error():

tests/test_roundtrip.py

@@ -0,0 +1,566 @@
+"""
+Roundtrip tests: Generate config → template/YAML → regenerate config → compare.
+
+These tests verify that:
+1. Generated Jinja2 template + Ansible YAML can reproduce the original config
+2. The regenerated config is semantically equivalent (allowing whitespace differences)
+3. No data loss occurs during the template generation process
+
+This is the ultimate validation - if the roundtrip works, the templates are correct.
+"""
+
+from __future__ import annotations
+
+import json
+import yaml
+from pathlib import Path
+from typing import Any
+from jinja2 import Environment, StrictUndefined
+
+import pytest
+
+from jinjaturtle.core import (
+    parse_config,
+    analyze_loops,
+    flatten_config,
+    generate_ansible_yaml,
+    generate_jinja2_template,
+)
+
+
+def render_template(template: str, variables: dict[str, Any]) -> str:
+    """Render a Jinja2 template with variables."""
+    env = Environment(undefined=StrictUndefined)
+    jinja_template = env.from_string(template)
+    return jinja_template.render(variables)
+
+
+class TestRoundtripJSON:
+    """Roundtrip tests for JSON files."""
+
+    def test_foo_json_roundtrip(self):
+        """Test foo.json can be perfectly regenerated from template."""
+        samples_dir = Path(__file__).parent / "samples"
+        json_file = samples_dir / "foo.json"
+
+        if not json_file.exists():
+            pytest.skip("foo.json not found")
+
+        # Read original
+        original_text = json_file.read_text()
+        original_data = json.loads(original_text)
+
+        # Generate template and YAML
+        fmt, parsed = parse_config(json_file)
+        loop_candidates = analyze_loops(fmt, parsed)
+        flat_items = flatten_config(fmt, parsed, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
+        template = generate_jinja2_template(fmt, parsed, "test", None, loop_candidates)
+
+        # Load variables from YAML
+        variables = yaml.safe_load(ansible_yaml)
+
+        # Render template
+        regenerated_text = render_template(template, variables)
+        regenerated_data = json.loads(regenerated_text)
+
+        # Compare data structures (should match exactly)
+        assert regenerated_data == original_data, (
+            f"Regenerated JSON differs from original\n"
+            f"Original: {json.dumps(original_data, indent=2, sort_keys=True)}\n"
+            f"Regenerated: {json.dumps(regenerated_data, indent=2, sort_keys=True)}"
+        )
+
+    def test_json_all_types_roundtrip(self):
+        """Test JSON with all data types roundtrips perfectly."""
+        json_text = """
+        {
+          "string": "value",
+          "number": 42,
+          "float": 3.14,
+          "boolean": true,
+          "false_val": false,
+          "null_value": null,
+          "array": [1, 2, 3],
+          "object": {
+            "nested": "data"
+          }
+        }
+        """
+
+        original_data = json.loads(json_text)
+
+        # Generate template and YAML
+        loop_candidates = analyze_loops("json", original_data)
+        flat_items = flatten_config("json", original_data, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "json", original_data, "test", None, loop_candidates
+        )
+
+        # Render template
+        variables = yaml.safe_load(ansible_yaml)
+        regenerated_text = render_template(template, variables)
+        regenerated_data = json.loads(regenerated_text)
+
+        # Should match exactly
+        assert regenerated_data == original_data
+
+
+class TestRoundtripYAML:
+    """Roundtrip tests for YAML files."""
+
+    def test_bar_yaml_roundtrip(self):
+        """Test bar.yaml can be regenerated from template."""
+        samples_dir = Path(__file__).parent / "samples"
+        yaml_file = samples_dir / "bar.yaml"
+
+        if not yaml_file.exists():
+            pytest.skip("bar.yaml not found")
+
+        # Read original
+        original_text = yaml_file.read_text()
+        original_data = yaml.safe_load(original_text)
+
+        # Generate template and YAML
+        fmt, parsed = parse_config(yaml_file)
+        loop_candidates = analyze_loops(fmt, parsed)
+        flat_items = flatten_config(fmt, parsed, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            fmt, parsed, "test", original_text, loop_candidates
+        )
+
+        # Load variables from YAML
+        variables = yaml.safe_load(ansible_yaml)
+
+        # Render template
+        regenerated_text = render_template(template, variables)
+        regenerated_data = yaml.safe_load(regenerated_text)
+
+        # Compare data structures
+        assert regenerated_data == original_data, (
+            f"Regenerated YAML differs from original\n"
+            f"Original: {original_data}\n"
+            f"Regenerated: {regenerated_data}"
+        )
+
+    def test_yaml_with_lists_roundtrip(self):
+        """Test YAML with various list structures."""
+        yaml_text = """
+        name: myapp
+        simple_list:
+          - item1
+          - item2
+          - item3
+        list_of_dicts:
+          - name: first
+            value: 1
+          - name: second
+            value: 2
+        nested:
+          inner_list:
+            - a
+            - b
+        """
+
+        original_data = yaml.safe_load(yaml_text)
+
+        # Generate template and YAML
+        loop_candidates = analyze_loops("yaml", original_data)
+        flat_items = flatten_config("yaml", original_data, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "yaml", original_data, "test", yaml_text, loop_candidates
+        )
+
+        # Render template
+        variables = yaml.safe_load(ansible_yaml)
+        regenerated_text = render_template(template, variables)
+        regenerated_data = yaml.safe_load(regenerated_text)
+
+        # Compare
+        assert regenerated_data == original_data
+
+
+class TestRoundtripTOML:
+    """Roundtrip tests for TOML files."""
+
+    def test_tom_toml_roundtrip(self):
+        """Test tom.toml can be regenerated from template."""
+        samples_dir = Path(__file__).parent / "samples"
+        toml_file = samples_dir / "tom.toml"
+
+        if not toml_file.exists():
+            pytest.skip("tom.toml not found")
+
+        # Read original
+        original_text = toml_file.read_text()
+        import tomllib
+
+        original_data = tomllib.loads(original_text)
+
+        # Generate template and YAML
+        fmt, parsed = parse_config(toml_file)
+        loop_candidates = analyze_loops(fmt, parsed)
+        flat_items = flatten_config(fmt, parsed, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            fmt, parsed, "test", original_text, loop_candidates
+        )
+
+        # Load variables from YAML
+        variables = yaml.safe_load(ansible_yaml)
+
+        # Render template
+        regenerated_text = render_template(template, variables)
+        regenerated_data = tomllib.loads(regenerated_text)
+
+        # Compare data structures
+        # Note: TOML datetime objects need special handling
+        assert _compare_toml_data(regenerated_data, original_data), (
+            f"Regenerated TOML differs from original\n"
+            f"Original: {original_data}\n"
+            f"Regenerated: {regenerated_data}"
+        )
+
+    def test_toml_with_arrays_roundtrip(self):
+        """Test TOML with inline arrays and array-of-tables."""
+        toml_text = """
+        name = "test"
+        ports = [8080, 8081, 8082]
+        
+        [[database]]
+        host = "db1.example.com"
+        port = 5432
+        
+        [[database]]
+        host = "db2.example.com"
+        port = 5433
+        """
+
+        import tomllib
+
+        original_data = tomllib.loads(toml_text)
+
+        # Generate template and YAML
+        loop_candidates = analyze_loops("toml", original_data)
+        flat_items = flatten_config("toml", original_data, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "toml", original_data, "test", toml_text, loop_candidates
+        )
+
+        # Render template
+        variables = yaml.safe_load(ansible_yaml)
+        regenerated_text = render_template(template, variables)
+        regenerated_data = tomllib.loads(regenerated_text)
+
+        # Compare
+        assert regenerated_data == original_data
+
+
+class TestRoundtripXML:
+    """Roundtrip tests for XML files."""
+
+    def test_xml_simple_roundtrip(self):
+        """Test simple XML can be regenerated."""
+        xml_text = """<?xml version="1.0"?>
+<config>
+  <name>test</name>
+  <port>8080</port>
+  <server>server1</server>
+  <server>server2</server>
+  <server>server3</server>
+</config>
+"""
+
+        import xml.etree.ElementTree as ET
+
+        original_root = ET.fromstring(xml_text)
+
+        # Generate template and YAML
+        fmt = "xml"
+        loop_candidates = analyze_loops(fmt, original_root)
+        flat_items = flatten_config(fmt, original_root, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            fmt, original_root, "test", xml_text, loop_candidates
+        )
+
+        # Render template
+        variables = yaml.safe_load(ansible_yaml)
+        regenerated_text = render_template(template, variables)
+
+        # Parse regenerated XML
+        regenerated_root = ET.fromstring(regenerated_text)
+
+        # Compare XML structures (ignore insignificant whitespace)
+        assert _xml_elements_equal(
+            original_root, regenerated_root, ignore_whitespace=True
+        ), (
+            f"Regenerated XML differs from original\n"
+            f"Original: {ET.tostring(original_root, encoding='unicode')}\n"
+            f"Regenerated: {ET.tostring(regenerated_root, encoding='unicode')}"
+        )
+
+    def test_ossec_xml_roundtrip(self):
+        """Test ossec.xml (complex real-world XML) roundtrip."""
+        samples_dir = Path(__file__).parent / "samples"
+        xml_file = samples_dir / "ossec.xml"
+
+        if not xml_file.exists():
+            pytest.skip("ossec.xml not found")
+
+        # Read original
+        original_text = xml_file.read_text()
+        import xml.etree.ElementTree as ET
+
+        original_root = ET.fromstring(original_text)
+
+        # Generate template and YAML
+        fmt, parsed = parse_config(xml_file)
+        loop_candidates = analyze_loops(fmt, parsed)
+        flat_items = flatten_config(fmt, parsed, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            fmt, parsed, "test", original_text, loop_candidates
+        )
+
+        # Load variables and render
+        variables = yaml.safe_load(ansible_yaml)
+        regenerated_text = render_template(template, variables)
+
+        # Parse regenerated
+        regenerated_root = ET.fromstring(regenerated_text)
+
+        # Compare - for complex XML, we compare structure not exact text
+        assert _xml_elements_equal(
+            original_root, regenerated_root, ignore_whitespace=True
+        )
+
+
+class TestRoundtripINI:
+    """Roundtrip tests for INI files."""
+
+    def test_ini_simple_roundtrip(self):
+        """Test simple INI can be regenerated."""
+        ini_text = """[section1]
+key1 = value1
+key2 = value2
+
+[section2]
+key3 = value3
+"""
+
+        from configparser import ConfigParser
+
+        original_config = ConfigParser()
+        original_config.read_string(ini_text)
+
+        # Generate template and YAML
+        fmt = "ini"
+        loop_candidates = analyze_loops(fmt, original_config)
+        flat_items = flatten_config(fmt, original_config, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            fmt, original_config, "test", ini_text, loop_candidates
+        )
+
+        # Render template
+        variables = yaml.safe_load(ansible_yaml)
+        regenerated_text = render_template(template, variables)
+
+        # Parse regenerated
+        regenerated_config = ConfigParser()
+        regenerated_config.read_string(regenerated_text)
+
+        # Compare
+        assert _ini_configs_equal(original_config, regenerated_config)
+
+
+class TestRoundtripEdgeCases:
+    """Roundtrip tests for edge cases and special scenarios."""
+
+    def test_empty_lists_roundtrip(self):
+        """Test handling of empty lists."""
+        json_text = '{"items": []}'
+        original_data = json.loads(json_text)
+
+        loop_candidates = analyze_loops("json", original_data)
+        flat_items = flatten_config("json", original_data, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "json", original_data, "test", None, loop_candidates
+        )
+
+        variables = yaml.safe_load(ansible_yaml)
+        regenerated_text = render_template(template, variables)
+        regenerated_data = json.loads(regenerated_text)
+
+        assert regenerated_data == original_data
+
+    def test_special_characters_roundtrip(self):
+        """Test handling of special characters."""
+        json_data = {
+            "quote": 'He said "hello"',
+            "backslash": "path\\to\\file",
+            "newline": "line1\nline2",
+            "unicode": "emoji: 🚀",
+        }
+
+        loop_candidates = analyze_loops("json", json_data)
+        flat_items = flatten_config("json", json_data, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "json", json_data, "test", None, loop_candidates
+        )
+
+        variables = yaml.safe_load(ansible_yaml)
+        regenerated_text = render_template(template, variables)
+        regenerated_data = json.loads(regenerated_text)
+
+        assert regenerated_data == json_data
+
+    def test_numeric_types_roundtrip(self):
+        """Test preservation of numeric types."""
+        json_data = {
+            "int": 42,
+            "float": 3.14159,
+            "negative": -100,
+            "zero": 0,
+            "large": 9999999999,
+        }
+
+        loop_candidates = analyze_loops("json", json_data)
+        flat_items = flatten_config("json", json_data, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "json", json_data, "test", None, loop_candidates
+        )
+
+        variables = yaml.safe_load(ansible_yaml)
+        regenerated_text = render_template(template, variables)
+        regenerated_data = json.loads(regenerated_text)
+
+        assert regenerated_data == json_data
+
+    def test_boolean_preservation_roundtrip(self):
+        """Test that booleans are preserved correctly."""
+        yaml_text = """
+        enabled: true
+        disabled: false
+        """
+
+        original_data = yaml.safe_load(yaml_text)
+
+        loop_candidates = analyze_loops("yaml", original_data)
+        flat_items = flatten_config("yaml", original_data, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "yaml", original_data, "test", yaml_text, loop_candidates
+        )
+
+        variables = yaml.safe_load(ansible_yaml)
+        regenerated_text = render_template(template, variables)
+        regenerated_data = yaml.safe_load(regenerated_text)
+
+        # Both should be actual booleans
+        assert regenerated_data["enabled"] is True
+        assert regenerated_data["disabled"] is False
+
+
+# Helper functions
+
+
+def _compare_toml_data(data1: Any, data2: Any) -> bool:
+    """Compare TOML data, handling datetime objects."""
+    import datetime
+
+    if type(data1) != type(data2):
+        return False
+
+    if isinstance(data1, dict):
+        if set(data1.keys()) != set(data2.keys()):
+            return False
+        return all(_compare_toml_data(data1[k], data2[k]) for k in data1.keys())
+
+    elif isinstance(data1, list):
+        if len(data1) != len(data2):
+            return False
+        return all(_compare_toml_data(v1, v2) for v1, v2 in zip(data1, data2))
+
+    elif isinstance(data1, datetime.datetime):
+        # Compare datetime objects
+        return data1 == data2
+
+    else:
+        return data1 == data2
+
+
+def _xml_elements_equal(elem1, elem2, ignore_whitespace: bool = False) -> bool:
+    """Compare two XML elements for equality."""
+    # Compare tags
+    if elem1.tag != elem2.tag:
+        return False
+
+    # Compare attributes
+    if elem1.attrib != elem2.attrib:
+        return False
+
+    # Compare text
+    text1 = (elem1.text or "").strip() if ignore_whitespace else (elem1.text or "")
+    text2 = (elem2.text or "").strip() if ignore_whitespace else (elem2.text or "")
+    if text1 != text2:
+        return False
+
+    # Compare tail
+    tail1 = (elem1.tail or "").strip() if ignore_whitespace else (elem1.tail or "")
+    tail2 = (elem2.tail or "").strip() if ignore_whitespace else (elem2.tail or "")
+    if tail1 != tail2:
+        return False
+
+    # Compare children
+    children1 = list(elem1)
+    children2 = list(elem2)
+
+    if len(children1) != len(children2):
+        return False
+
+    return all(
+        _xml_elements_equal(c1, c2, ignore_whitespace)
+        for c1, c2 in zip(children1, children2)
+    )
+
+
+def _ini_configs_equal(config1, config2) -> bool:
+    """Compare two ConfigParser objects for equality."""
+    if set(config1.sections()) != set(config2.sections()):
+        return False
+
+    for section in config1.sections():
+        if set(config1.options(section)) != set(config2.options(section)):
+            return False
+
+        for option in config1.options(section):
+            if config1.get(section, option) != config2.get(section, option):
+                return False
+
+    return True
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

tests/test_yaml_template_consistency.py

@@ -0,0 +1,558 @@
+"""
+Tests to ensure all Jinja2 template variables exist in the Ansible YAML.
+
+These tests catch the bug where templates reference variables that don't exist
+because the YAML has a list but the template uses scalar references (or vice versa).
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Set
+import yaml
+import pytest
+
+from jinjaturtle.core import (
+    parse_config,
+    analyze_loops,
+    flatten_config,
+    generate_ansible_yaml,
+    generate_jinja2_template,
+)
+
+
+def extract_jinja_variables(template: str) -> Set[str]:
+    """
+    Extract all Jinja2 variable names from a template that must exist in YAML.
+
+    Extracts variables from:
+    - {{ variable_name }}
+    - {{ variable.field }}
+    - {% for item in collection %}
+
+    Returns only the base variable names that must be defined in YAML.
+    Filters out loop variables (the 'item' part of 'for item in collection').
+    """
+    variables = set()
+
+    # First, find all loop variables (these are defined by the template, not YAML)
+    loop_vars = set()
+    for_pattern = r"\{%\s*for\s+(\w+)\s+in\s+([a-zA-Z_][a-zA-Z0-9_]*)"
+    for match in re.finditer(for_pattern, template):
+        loop_var = match.group(1)  # The item
+        collection = match.group(2)  # The collection
+        loop_vars.add(loop_var)
+        variables.add(collection)  # Collection must exist in YAML
+
+    # Pattern 1: {{ variable_name }} or {{ variable.field }}
+    # Captures the first part before any dots or filters
+    var_pattern = r"\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)"
+    for match in re.finditer(var_pattern, template):
+        var_name = match.group(1)
+        # Only add if it's not a loop variable
+        if var_name not in loop_vars:
+            variables.add(var_name)
+
+    return variables
+
+
+def extract_yaml_variables(ansible_yaml: str) -> Set[str]:
+    """
+    Extract all variable names from Ansible YAML.
+
+    Returns the top-level keys from the YAML document.
+    """
+    data = yaml.safe_load(ansible_yaml)
+    if not isinstance(data, dict):
+        return set()
+    return set(data.keys())
+
+
+class TestTemplateYamlConsistency:
+    """Tests that verify template variables exist in YAML."""
+
+    def test_simple_json_consistency(self):
+        """Simple JSON with scalars and lists."""
+        json_text = """
+        {
+          "name": "test",
+          "values": [1, 2, 3]
+        }
+        """
+
+        fmt = "json"
+        import json
+
+        parsed = json.loads(json_text)
+
+        loop_candidates = analyze_loops(fmt, parsed)
+        flat_items = flatten_config(fmt, parsed, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
+        template = generate_jinja2_template(fmt, parsed, "app", None, loop_candidates)
+
+        yaml_vars = extract_yaml_variables(ansible_yaml)
+        template_vars = extract_jinja_variables(template)
+
+        # Every variable in template must exist in YAML
+        missing_vars = template_vars - yaml_vars
+        assert not missing_vars, (
+            f"Template references variables not in YAML: {missing_vars}\n"
+            f"YAML vars: {yaml_vars}\n"
+            f"Template vars: {template_vars}\n"
+            f"Template:\n{template}\n"
+            f"YAML:\n{ansible_yaml}"
+        )
+
+    def test_toml_inline_array_consistency(self):
+        """TOML with inline array should use loops consistently."""
+        import tomllib
+
+        toml_text = """
+        name = "myapp"
+        servers = ["server1", "server2", "server3"]
+        """
+
+        parsed = tomllib.loads(toml_text)
+        loop_candidates = analyze_loops("toml", parsed)
+        flat_items = flatten_config("toml", parsed, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "toml", parsed, "app", toml_text, loop_candidates
+        )
+
+        yaml_vars = extract_yaml_variables(ansible_yaml)
+        template_vars = extract_jinja_variables(template)
+
+        missing_vars = template_vars - yaml_vars
+        assert not missing_vars, (
+            f"Template references variables not in YAML: {missing_vars}\n"
+            f"Template:\n{template}\n"
+            f"YAML:\n{ansible_yaml}"
+        )
+
+    def test_toml_array_of_tables_consistency(self):
+        """TOML with [[array.of.tables]] should use loops consistently."""
+        import tomllib
+
+        toml_text = """
+        [[database]]
+        host = "db1.example.com"
+        port = 5432
+        
+        [[database]]
+        host = "db2.example.com"
+        port = 5433
+        """
+
+        parsed = tomllib.loads(toml_text)
+        loop_candidates = analyze_loops("toml", parsed)
+        flat_items = flatten_config("toml", parsed, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "toml", parsed, "app", toml_text, loop_candidates
+        )
+
+        yaml_vars = extract_yaml_variables(ansible_yaml)
+        template_vars = extract_jinja_variables(template)
+
+        missing_vars = template_vars - yaml_vars
+        assert not missing_vars, (
+            f"Template references variables not in YAML: {missing_vars}\n"
+            f"Template:\n{template}\n"
+            f"YAML:\n{ansible_yaml}"
+        )
+
+        # Additionally verify that if YAML has a list, template uses a loop
+        defaults = yaml.safe_load(ansible_yaml)
+        for var_name, value in defaults.items():
+            if isinstance(value, list) and len(value) > 1:
+                # YAML has a list - template should use {% for %}
+                assert "{% for" in template, (
+                    f"YAML has list variable '{var_name}' but template doesn't use loops\n"
+                    f"Template:\n{template}"
+                )
+
+    def test_yaml_list_consistency(self):
+        """YAML with lists should use loops consistently."""
+        yaml_text = """
+        name: myapp
+        servers:
+          - server1
+          - server2
+          - server3
+        databases:
+          - host: db1
+            port: 5432
+          - host: db2
+            port: 5433
+        """
+
+        parsed = yaml.safe_load(yaml_text)
+        loop_candidates = analyze_loops("yaml", parsed)
+        flat_items = flatten_config("yaml", parsed, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "yaml", parsed, "app", yaml_text, loop_candidates
+        )
+
+        yaml_vars = extract_yaml_variables(ansible_yaml)
+        template_vars = extract_jinja_variables(template)
+
+        missing_vars = template_vars - yaml_vars
+        assert not missing_vars, (
+            f"Template references variables not in YAML: {missing_vars}\n"
+            f"Template:\n{template}\n"
+            f"YAML:\n{ansible_yaml}"
+        )
+
+    def test_mixed_scalars_and_loops_consistency(self):
+        """Config with both scalars and loops should be consistent."""
+        import tomllib
+
+        toml_text = """
+        name = "myapp"
+        version = "1.0"
+        ports = [8080, 8081, 8082]
+        
+        [database]
+        host = "localhost"
+        port = 5432
+        
+        [[servers]]
+        name = "web1"
+        ip = "10.0.0.1"
+        
+        [[servers]]
+        name = "web2"
+        ip = "10.0.0.2"
+        """
+
+        parsed = tomllib.loads(toml_text)
+        loop_candidates = analyze_loops("toml", parsed)
+        flat_items = flatten_config("toml", parsed, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "toml", parsed, "app", toml_text, loop_candidates
+        )
+
+        yaml_vars = extract_yaml_variables(ansible_yaml)
+        template_vars = extract_jinja_variables(template)
+
+        missing_vars = template_vars - yaml_vars
+        assert not missing_vars, (
+            f"Template references variables not in YAML: {missing_vars}\n"
+            f"Template:\n{template}\n"
+            f"YAML:\n{ansible_yaml}"
+        )
+
+    def test_no_orphaned_scalar_references(self):
+        """
+        When YAML has a list variable, template must NOT reference scalar indices.
+
+        This catches the bug where:
+        - YAML has: app_list: [1, 2, 3]
+        - Template incorrectly uses: {{ app_list_0 }}, {{ app_list_1 }}
+        """
+        import json
+
+        json_text = '{"items": [1, 2, 3, 4, 5]}'
+        parsed = json.loads(json_text)
+
+        loop_candidates = analyze_loops("json", parsed)
+        flat_items = flatten_config("json", parsed, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "json", parsed, "app", None, loop_candidates
+        )
+
+        defaults = yaml.safe_load(ansible_yaml)
+
+        # Check each list variable in YAML
+        for var_name, value in defaults.items():
+            if isinstance(value, list):
+                # Template should NOT reference app_items_0, app_items_1, etc.
+                for i in range(len(value)):
+                    scalar_ref = f"{var_name}_{i}"
+                    assert scalar_ref not in template, (
+                        f"Template incorrectly uses scalar reference '{scalar_ref}' "
+                        f"when YAML has '{var_name}' as a list\n"
+                        f"Template should use loops, not scalar indices\n"
+                        f"Template:\n{template}"
+                    )
+
+    def test_all_sample_files_consistency(self):
+        """Test all sample files for consistency."""
+        samples_dir = Path(__file__).parent / "samples"
+
+        sample_files = [
+            ("foo.json", "json"),
+            ("bar.yaml", "yaml"),
+            ("tom.toml", "toml"),
+        ]
+
+        for filename, fmt in sample_files:
+            file_path = samples_dir / filename
+            if not file_path.exists():
+                pytest.skip(f"Sample file {filename} not found")
+
+            original_text = file_path.read_text()
+            fmt_detected, parsed = parse_config(file_path)
+
+            loop_candidates = analyze_loops(fmt_detected, parsed)
+            flat_items = flatten_config(fmt_detected, parsed, loop_candidates)
+
+            ansible_yaml = generate_ansible_yaml("test", flat_items, loop_candidates)
+            template = generate_jinja2_template(
+                fmt_detected, parsed, "test", original_text, loop_candidates
+            )
+
+            yaml_vars = extract_yaml_variables(ansible_yaml)
+            template_vars = extract_jinja_variables(template)
+
+            missing_vars = template_vars - yaml_vars
+            assert not missing_vars, (
+                f"File: {filename}\n"
+                f"Template references variables not in YAML: {missing_vars}\n"
+                f"YAML vars: {yaml_vars}\n"
+                f"Template vars: {template_vars}\n"
+                f"Template:\n{template}\n"
+                f"YAML:\n{ansible_yaml}"
+            )
+
+
+class TestStructuralConsistency:
+    """Tests that verify structural consistency between YAML and templates."""
+
+    def test_list_in_yaml_means_loop_in_template(self):
+        """When YAML has a list (len > 1), template should use {% for %}."""
+        import json
+
+        json_text = """
+        {
+          "scalar": "value",
+          "list": [1, 2, 3]
+        }
+        """
+
+        parsed = json.loads(json_text)
+        loop_candidates = analyze_loops("json", parsed)
+        flat_items = flatten_config("json", parsed, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "json", parsed, "app", None, loop_candidates
+        )
+
+        defaults = yaml.safe_load(ansible_yaml)
+
+        # Find list variables in YAML
+        list_vars = [
+            k for k, v in defaults.items() if isinstance(v, list) and len(v) > 1
+        ]
+
+        if list_vars:
+            # Template must contain for loops
+            assert "{% for" in template, (
+                f"YAML has list variables {list_vars} but template has no loops\n"
+                f"Template:\n{template}"
+            )
+
+            # Each list variable should be used in a for loop
+            for var_name in list_vars:
+                # Look for "{% for ... in var_name %}"
+                for_pattern = (
+                    r"\{%\s*for\s+\w+\s+in\s+" + re.escape(var_name) + r"\s*%\}"
+                )
+                assert re.search(for_pattern, template), (
+                    f"List variable '{var_name}' not used in a for loop\n"
+                    f"Template:\n{template}"
+                )
+
+    def test_scalar_in_yaml_means_no_loop_in_template(self):
+        """When YAML has scalars, template should use {{ var }}, not loops."""
+        import json
+
+        json_text = """
+        {
+          "name": "test",
+          "port": 8080,
+          "enabled": true
+        }
+        """
+
+        parsed = json.loads(json_text)
+        loop_candidates = analyze_loops("json", parsed)
+        flat_items = flatten_config("json", parsed, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "json", parsed, "app", None, loop_candidates
+        )
+
+        defaults = yaml.safe_load(ansible_yaml)
+
+        # All variables are scalars - template should NOT have loops
+        scalar_vars = [
+            k for k, v in defaults.items() if not isinstance(v, (list, dict))
+        ]
+
+        # Check that scalar vars are used directly, not in loops
+        for var_name in scalar_vars:
+            # Should appear in {{ var_name }}, not {% for ... in var_name %}
+            direct_ref = f"{{{{ {var_name}"
+            loop_ref = f"for .* in {var_name}"
+
+            assert direct_ref in template, (
+                f"Scalar variable '{var_name}' should be directly referenced\n"
+                f"Template:\n{template}"
+            )
+
+            assert not re.search(loop_ref, template), (
+                f"Scalar variable '{var_name}' incorrectly used in a loop\n"
+                f"Template:\n{template}"
+            )
+
+    def test_no_undefined_variable_errors(self):
+        """
+        Simulate Ansible template rendering to catch undefined variables.
+
+        This is the ultimate test - actually render the template with the YAML
+        and verify no undefined variable errors occur.
+        """
+        from jinja2 import Environment, StrictUndefined
+        import json
+
+        json_text = """
+        {
+          "name": "myapp",
+          "servers": ["web1", "web2"],
+          "database": {
+            "host": "localhost",
+            "port": 5432
+          }
+        }
+        """
+
+        parsed = json.loads(json_text)
+        loop_candidates = analyze_loops("json", parsed)
+        flat_items = flatten_config("json", parsed, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "json", parsed, "app", None, loop_candidates
+        )
+
+        # Load variables from YAML
+        variables = yaml.safe_load(ansible_yaml)
+
+        # Try to render the template
+        env = Environment(undefined=StrictUndefined)
+        try:
+            jinja_template = env.from_string(template)
+            rendered = jinja_template.render(variables)
+
+            # Successfully rendered - this is what we want!
+            assert rendered, "Template rendered successfully"
+
+        except Exception as e:
+            pytest.fail(
+                f"Template rendering failed with variables from YAML\n"
+                f"Error: {e}\n"
+                f"Template:\n{template}\n"
+                f"Variables:\n{ansible_yaml}"
+            )
+
+
+class TestRegressionBugs:
+    """Tests for specific bugs that were found and fixed."""
+
+    def test_toml_array_of_tables_no_scalar_refs(self):
+        """
+        Regression test: TOML [[array]] should not generate scalar references.
+
+        Bug: Template had {{ app_database_host }} when YAML had app_database as list.
+        """
+        import tomllib
+
+        toml_text = """
+        [[database]]
+        host = "db1"
+        port = 5432
+        
+        [[database]]
+        host = "db2"
+        port = 5433
+        """
+
+        parsed = tomllib.loads(toml_text)
+        loop_candidates = analyze_loops("toml", parsed)
+        flat_items = flatten_config("toml", parsed, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "toml", parsed, "app", toml_text, loop_candidates
+        )
+
+        # YAML should have app_database as a list
+        defaults = yaml.safe_load(ansible_yaml)
+        assert isinstance(
+            defaults.get("app_database"), list
+        ), f"Expected app_database to be a list in YAML\n{ansible_yaml}"
+
+        # Template should NOT have app_database_host or app_database_port
+        assert (
+            "app_database_host" not in template
+        ), f"Template incorrectly uses scalar 'app_database_host'\n{template}"
+        assert (
+            "app_database_port" not in template
+        ), f"Template incorrectly uses scalar 'app_database_port'\n{template}"
+
+        # Template SHOULD use a loop
+        assert "{% for" in template, f"Template should use a loop\n{template}"
+        assert (
+            "app_database" in template
+        ), f"Template should reference app_database\n{template}"
+
+    def test_json_array_no_index_refs(self):
+        """
+        Regression test: JSON arrays should not generate index references.
+
+        Bug: Template had {{ app_list_0 }}, {{ app_list_1 }} when YAML had app_list as list.
+        """
+        import json
+
+        json_text = '{"items": [1, 2, 3]}'
+        parsed = json.loads(json_text)
+
+        loop_candidates = analyze_loops("json", parsed)
+        flat_items = flatten_config("json", parsed, loop_candidates)
+
+        ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
+        template = generate_jinja2_template(
+            "json", parsed, "app", None, loop_candidates
+        )
+
+        # YAML should have app_items as a list
+        defaults = yaml.safe_load(ansible_yaml)
+        assert isinstance(defaults.get("app_items"), list)
+
+        # Template should NOT have app_items_0, app_items_1, app_items_2
+        for i in range(3):
+            assert (
+                f"app_items_{i}" not in template
+            ), f"Template incorrectly uses scalar 'app_items_{i}'\n{template}"
+
+        # Template SHOULD use a loop
+        assert "{% for" in template
+        assert "app_items" in template
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])