Refactor and add much more robust tests (both automated and manual) to ensure loops and things work ok
This commit is contained in:
parent
3af628e22e
commit
d7c71f6349
GPG key ID:
59B3F0C24135C6A9
17 changed files with 2126 additions and 91 deletions
216
utils/diff_configs.py
Normal file
216
utils/diff_configs.py
Normal file
|
|
@ -0,0 +1,216 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Side-by-side comparison of original vs regenerated config.
|
||||
|
||||
Usage:
|
||||
./diff_configs.py tests/samples/foo.json
|
||||
./diff_configs.py tests/samples/tom.toml --context 5
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import difflib
|
||||
import yaml
|
||||
from jinja2 import Environment, StrictUndefined
|
||||
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from jinjaturtle.core import (
|
||||
parse_config,
|
||||
analyze_loops,
|
||||
flatten_config,
|
||||
generate_ansible_yaml,
|
||||
generate_jinja2_template,
|
||||
)
|
||||
|
||||
|
||||
def colorize(text: str, color: str) -> str:
|
||||
"""Add ANSI color codes."""
|
||||
colors = {
|
||||
"red": "\033[91m",
|
||||
"green": "\033[92m",
|
||||
"yellow": "\033[93m",
|
||||
"blue": "\033[94m",
|
||||
"reset": "\033[0m",
|
||||
}
|
||||
return f"{colors.get(color, '')}{text}{colors['reset']}"
|
||||
|
||||
|
||||
def side_by_side_diff(original: str, regenerated: str, width: int = 80):
|
||||
"""Print side-by-side diff."""
|
||||
orig_lines = original.splitlines()
|
||||
regen_lines = regenerated.splitlines()
|
||||
|
||||
# Calculate column width
|
||||
col_width = width // 2 - 3
|
||||
|
||||
print(
|
||||
colorize("ORIGINAL".center(col_width), "blue")
|
||||
+ " | "
|
||||
+ colorize("REGENERATED".center(col_width), "green")
|
||||
)
|
||||
print("-" * col_width + "-+-" + "-" * col_width)
|
||||
|
||||
max_lines = max(len(orig_lines), len(regen_lines))
|
||||
|
||||
for i in range(max_lines):
|
||||
orig_line = orig_lines[i] if i < len(orig_lines) else ""
|
||||
regen_line = regen_lines[i] if i < len(regen_lines) else ""
|
||||
|
||||
# Truncate if too long
|
||||
if len(orig_line) > col_width - 2:
|
||||
orig_line = orig_line[: col_width - 5] + "..."
|
||||
if len(regen_line) > col_width - 2:
|
||||
regen_line = regen_line[: col_width - 5] + "..."
|
||||
|
||||
# Color lines if different
|
||||
if orig_line != regen_line:
|
||||
orig_display = colorize(orig_line.ljust(col_width), "red")
|
||||
regen_display = colorize(regen_line.ljust(col_width), "green")
|
||||
else:
|
||||
orig_display = orig_line.ljust(col_width)
|
||||
regen_display = regen_line.ljust(col_width)
|
||||
|
||||
print(f"{orig_display} | {regen_display}")
|
||||
|
||||
|
||||
def unified_diff(original: str, regenerated: str, filename: str, context: int = 3):
|
||||
"""Print unified diff."""
|
||||
orig_lines = original.splitlines(keepends=True)
|
||||
regen_lines = regenerated.splitlines(keepends=True)
|
||||
|
||||
diff = difflib.unified_diff(
|
||||
orig_lines,
|
||||
regen_lines,
|
||||
fromfile=f"{filename} (original)",
|
||||
tofile=f"{filename} (regenerated)",
|
||||
n=context,
|
||||
)
|
||||
|
||||
for line in diff:
|
||||
if line.startswith("+++") or line.startswith("---"):
|
||||
print(colorize(line.rstrip(), "blue"))
|
||||
elif line.startswith("@@"):
|
||||
print(colorize(line.rstrip(), "cyan"))
|
||||
elif line.startswith("+"):
|
||||
print(colorize(line.rstrip(), "green"))
|
||||
elif line.startswith("-"):
|
||||
print(colorize(line.rstrip(), "red"))
|
||||
else:
|
||||
print(line.rstrip())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Compare original config with regenerated version",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
|
||||
parser.add_argument("file", type=Path, help="Config file to check")
|
||||
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
choices=["side-by-side", "unified", "both"],
|
||||
default="both",
|
||||
help="Comparison mode (default: both)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--context",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Number of context lines for unified diff (default: 3)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--width",
|
||||
type=int,
|
||||
default=160,
|
||||
help="Terminal width for side-by-side (default: 160)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.file.exists():
|
||||
print(colorize(f"❌ File not found: {args.file}", "red"))
|
||||
return 1
|
||||
|
||||
print(colorize(f"\n{'=' * 80}", "blue"))
|
||||
print(colorize(f" Comparing: {args.file}", "blue"))
|
||||
print(colorize(f"{'=' * 80}\n", "blue"))
|
||||
|
||||
# Read and regenerate
|
||||
try:
|
||||
original_text = args.file.read_text()
|
||||
|
||||
fmt, parsed = parse_config(args.file)
|
||||
loop_candidates = analyze_loops(fmt, parsed)
|
||||
flat_items = flatten_config(fmt, parsed, loop_candidates)
|
||||
|
||||
ansible_yaml = generate_ansible_yaml("app", flat_items, loop_candidates)
|
||||
template = generate_jinja2_template(
|
||||
fmt, parsed, "app", original_text, loop_candidates
|
||||
)
|
||||
|
||||
variables = yaml.safe_load(ansible_yaml)
|
||||
env = Environment(undefined=StrictUndefined)
|
||||
jinja_template = env.from_string(template)
|
||||
regenerated_text = jinja_template.render(variables)
|
||||
|
||||
# Check if identical
|
||||
if original_text.strip() == regenerated_text.strip():
|
||||
print(colorize("✅ Files are IDENTICAL (text comparison)\n", "green"))
|
||||
else:
|
||||
# Show diff
|
||||
if args.mode in ("unified", "both"):
|
||||
print(colorize("\n--- UNIFIED DIFF ---\n", "yellow"))
|
||||
unified_diff(
|
||||
original_text, regenerated_text, args.file.name, args.context
|
||||
)
|
||||
|
||||
if args.mode in ("side-by-side", "both"):
|
||||
print(colorize("\n--- SIDE-BY-SIDE COMPARISON ---\n", "yellow"))
|
||||
side_by_side_diff(original_text, regenerated_text, args.width)
|
||||
|
||||
# Try semantic comparison
|
||||
print(colorize(f"\n{'=' * 80}", "cyan"))
|
||||
print(colorize(" Semantic Comparison", "cyan"))
|
||||
print(colorize(f"{'=' * 80}", "cyan"))
|
||||
|
||||
try:
|
||||
if fmt == "json":
|
||||
import json
|
||||
|
||||
if json.loads(original_text) == json.loads(regenerated_text):
|
||||
print(colorize("✅ JSON data structures are IDENTICAL", "green"))
|
||||
else:
|
||||
print(colorize("⚠️ JSON data structures DIFFER", "yellow"))
|
||||
elif fmt == "yaml":
|
||||
if yaml.safe_load(original_text) == yaml.safe_load(regenerated_text):
|
||||
print(colorize("✅ YAML data structures are IDENTICAL", "green"))
|
||||
else:
|
||||
print(colorize("⚠️ YAML data structures DIFFER", "yellow"))
|
||||
elif fmt == "toml":
|
||||
import tomllib
|
||||
|
||||
if tomllib.loads(original_text) == tomllib.loads(regenerated_text):
|
||||
print(colorize("✅ TOML data structures are IDENTICAL", "green"))
|
||||
else:
|
||||
print(colorize("⚠️ TOML data structures DIFFER", "yellow"))
|
||||
except Exception as e:
|
||||
print(colorize(f"ℹ️ Could not compare semantically: {e}", "yellow"))
|
||||
|
||||
except Exception as e:
|
||||
print(colorize(f"❌ ERROR: {e}", "red"))
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
162
utils/regenerate.py
Normal file
162
utils/regenerate.py
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Regenerate config files and save all intermediate files.
|
||||
|
||||
Creates:
|
||||
- original.{ext}
|
||||
- defaults/main.yml
|
||||
- templates/config.j2
|
||||
- regenerated.{ext}
|
||||
|
||||
Usage:
|
||||
./regenerate.py tests/samples/foo.json
|
||||
./regenerate.py tests/samples/tom.toml --output-dir tmp/toml_test
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
from jinja2 import Environment, StrictUndefined
|
||||
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from jinjaturtle.core import (
|
||||
parse_config,
|
||||
analyze_loops,
|
||||
flatten_config,
|
||||
generate_ansible_yaml,
|
||||
generate_jinja2_template,
|
||||
)
|
||||
|
||||
|
||||
def regenerate_and_save(config_file: Path, output_dir: Path, role_prefix: str = "app"):
|
||||
"""
|
||||
Regenerate config and save all intermediate files.
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Read original
|
||||
original_text = config_file.read_text()
|
||||
fmt, parsed = parse_config(config_file)
|
||||
|
||||
# Determine extension
|
||||
ext = config_file.suffix
|
||||
|
||||
# Save original
|
||||
original_out = output_dir / f"original{ext}"
|
||||
original_out.write_text(original_text)
|
||||
print(f"📄 Saved: {original_out}")
|
||||
|
||||
# Generate Ansible files
|
||||
loop_candidates = analyze_loops(fmt, parsed)
|
||||
flat_items = flatten_config(fmt, parsed, loop_candidates)
|
||||
|
||||
ansible_yaml = generate_ansible_yaml(role_prefix, flat_items, loop_candidates)
|
||||
template = generate_jinja2_template(
|
||||
fmt, parsed, role_prefix, original_text, loop_candidates
|
||||
)
|
||||
|
||||
# Save Ansible YAML
|
||||
defaults_dir = output_dir / "defaults"
|
||||
defaults_dir.mkdir(exist_ok=True)
|
||||
defaults_file = defaults_dir / "main.yml"
|
||||
defaults_file.write_text(ansible_yaml)
|
||||
print(f"📄 Saved: {defaults_file}")
|
||||
|
||||
# Save template
|
||||
templates_dir = output_dir / "templates"
|
||||
templates_dir.mkdir(exist_ok=True)
|
||||
template_file = templates_dir / "config.j2"
|
||||
template_file.write_text(template)
|
||||
print(f"📄 Saved: {template_file}")
|
||||
|
||||
# Render template
|
||||
variables = yaml.safe_load(ansible_yaml)
|
||||
env = Environment(undefined=StrictUndefined)
|
||||
jinja_template = env.from_string(template)
|
||||
regenerated_text = jinja_template.render(variables)
|
||||
|
||||
# Save regenerated
|
||||
regenerated_out = output_dir / f"regenerated{ext}"
|
||||
regenerated_out.write_text(regenerated_text)
|
||||
print(f"📄 Saved: {regenerated_out}")
|
||||
|
||||
# Summary
|
||||
print(f"\n✅ All files saved to: {output_dir}")
|
||||
print("\n📊 Statistics:")
|
||||
print(f" Format: {fmt}")
|
||||
print(f" Loop candidates: {len(loop_candidates)}")
|
||||
if loop_candidates:
|
||||
print(" Loops detected:")
|
||||
for c in loop_candidates:
|
||||
print(f" - {'.'.join(c.path)}: {len(c.items)} items")
|
||||
|
||||
# Check if identical
|
||||
if original_text.strip() == regenerated_text.strip():
|
||||
print("\n✅ Original and regenerated are IDENTICAL (text comparison)")
|
||||
else:
|
||||
print("\n⚠️ Original and regenerated differ in whitespace/formatting")
|
||||
print(f" Run: diff {original_out} {regenerated_out}")
|
||||
|
||||
return output_dir
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Regenerate config and save all intermediate files",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
%(prog)s tests/samples/foo.json
|
||||
%(prog)s tests/samples/tom.toml -o tmp/toml_output
|
||||
%(prog)s tests/samples/bar.yaml --role-prefix myapp
|
||||
""",
|
||||
)
|
||||
|
||||
parser.add_argument("file", type=Path, help="Config file to process")
|
||||
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output-dir",
|
||||
type=Path,
|
||||
help="Output directory (default: regenerated_<filename>)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-r",
|
||||
"--role-prefix",
|
||||
default="app",
|
||||
help="Ansible role prefix for variables (default: app)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.file.exists():
|
||||
print(f"❌ File not found: {args.file}")
|
||||
return 1
|
||||
|
||||
# Determine output directory
|
||||
if args.output_dir:
|
||||
output_dir = args.output_dir
|
||||
else:
|
||||
output_dir = Path(f"regenerated_{args.file.stem}")
|
||||
|
||||
print(f"🔄 Regenerating: {args.file}")
|
||||
print(f"📁 Output directory: {output_dir}")
|
||||
print(f"🏷️ Role prefix: {args.role_prefix}\n")
|
||||
|
||||
try:
|
||||
regenerate_and_save(args.file, output_dir, args.role_prefix)
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"\n❌ ERROR: {e}")
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue