From dce3d07f32125700c6f78f6e55846c9d3fe5915b Mon Sep 17 00:00:00 2001 From: PDD Bot Date: Fri, 30 Jan 2026 07:53:04 +0000 Subject: [PATCH] Add failing tests for issue #437: null hash fields in fingerprint metadata This commit adds comprehensive test coverage to detect the bug where fingerprint metadata files have all hash fields set to null instead of containing actual content hashes. Test files added: - Unit test in tests/test_operation_log.py: Tests the @log_operation decorator to verify it creates fingerprints with null hashes when the 'paths' parameter is not passed to save_fingerprint() - E2E test in tests/test_e2e_issue_437_null_hashes.py: Tests the full 'pdd generate' command flow to verify the bug at integration level Both tests are verified to fail on the current codebase and will pass once the fix is implemented. Root cause: pdd/operation_log.py:338 - @log_operation decorator calls save_fingerprint() without the required 'paths' parameter. Co-Authored-By: Claude Sonnet 4.5 --- tests/test_e2e_issue_437_null_hashes.py | 237 ++++++++++++++++++++++++ tests/test_operation_log.py | 116 +++++++++++- 2 files changed, 352 insertions(+), 1 deletion(-) create mode 100644 tests/test_e2e_issue_437_null_hashes.py diff --git a/tests/test_e2e_issue_437_null_hashes.py b/tests/test_e2e_issue_437_null_hashes.py new file mode 100644 index 000000000..9a4d7067f --- /dev/null +++ b/tests/test_e2e_issue_437_null_hashes.py @@ -0,0 +1,237 @@ +""" +E2E Test for Issue #437: Fingerprint metadata files have null hash fields after generate + +This test exercises the full CLI path from `pdd generate` to verify that after a successful +generate command, the fingerprint metadata files in `.pdd/meta/` contain actual content hashes +instead of null values. + +The bug: After running `pdd generate`, the fingerprint metadata files have all hash fields +(prompt_hash, code_hash, example_hash, test_hash) set to `null` instead of containing +actual SHA-256 content hashes. + +Bug location: +- pdd/operation_log.py:338 - The @log_operation decorator calls save_fingerprint() without + the required `paths` parameter, causing all hashes to be None. + +This E2E test: +1. Creates a test project with a prompt file +2. Runs `pdd generate` command through Click's CliRunner +3. Verifies that the resulting fingerprint metadata file contains actual hash values (not null) + +The test should FAIL on buggy code (all hashes are null) and PASS once the fix is applied. + +Issue: https://github.com/promptdriven/pdd/issues/437 +""" + +import json +import pytest +from pathlib import Path +from unittest.mock import patch, MagicMock +from click.testing import CliRunner + + +class TestIssue437NullHashesE2E: + """ + E2E tests for Issue #437: Verify CLI generate command creates fingerprints + with actual content hashes instead of null values. + """ + + def test_pdd_generate_creates_fingerprint_with_valid_hashes(self, tmp_path, monkeypatch): + """ + E2E Test: `pdd generate` should create fingerprint metadata with actual hash values + + This test runs the full CLI path and verifies that when generating code from a prompt, + the resulting fingerprint metadata file contains valid SHA-256 hashes for all fields + instead of null values. + + Expected behavior (after fix): + - Fingerprint file should contain valid 64-character hex hashes for prompt_hash, code_hash, etc. + - Hashes should be different from null/None + + Bug behavior (Issue #437): + - All hash fields are null: {"prompt_hash": null, "code_hash": null, ...} + - This breaks fingerprint-based change detection feature + """ + # 1. Set up test project structure + project_dir = tmp_path / "test_project" + project_dir.mkdir() + monkeypatch.chdir(project_dir) + + # Create a simple prompt file + prompt_file = project_dir / "hello_Python.prompt" + prompt_content = """# Prompt: Hello World Function + +Create a simple Python function that prints "Hello, World!". + +## Requirements +- Function should be named `say_hello` +- Function should print "Hello, World!" to stdout +- Include a docstring + +## Example +```python +def say_hello(): + \"\"\"Print a friendly greeting.\"\"\" + print("Hello, World!") +``` +""" + prompt_file.write_text(prompt_content) + + # 2. Mock the LLM call to avoid actual API calls and costs + # We need to mock code_generator_main to return a successful result + mock_result = { + "basename": "hello", + "language": "Python", + "code_file": str(project_dir / "hello.py"), + "success": True, + } + + def mock_code_generator_main(*args, **kwargs): + """Mock code generator that creates output files and returns success.""" + # Create the output code file + code_file = project_dir / "hello.py" + code_content = '''"""Hello World module.""" + +def say_hello(): + """Print a friendly greeting.""" + print("Hello, World!") + + +if __name__ == "__main__": + say_hello() +''' + code_file.write_text(code_content) + + # Create example file (optional but often generated) + example_file = project_dir / "hello_example.py" + example_content = '''"""Example usage of hello module.""" +from hello import say_hello + +say_hello() +''' + example_file.write_text(example_content) + + # Return success tuple: (message, cost, model) + return ("Generated hello.py successfully", 0.001, "mock-gpt-4") + + # 3. Patch the code generator and run the CLI command + with patch("pdd.commands.generate.code_generator_main", side_effect=mock_code_generator_main): + from pdd.cli import cli + + runner = CliRunner() + result = runner.invoke( + cli, + ["generate", str(prompt_file)], + catch_exceptions=False, # Let exceptions propagate for debugging + ) + + # 5. THE KEY ASSERTIONS + + # Check that command succeeded + assert result.exit_code == 0, ( + f"Command failed with exit code {result.exit_code}\n" + f"Output: {result.output}\n" + f"Exception: {result.exception}" + ) + + # 6. Verify the fingerprint metadata file exists + meta_dir = project_dir / ".pdd" / "meta" + fingerprint_file = meta_dir / "hello_Python.json" + + assert fingerprint_file.exists(), ( + f"Fingerprint metadata file not found at {fingerprint_file}\n" + f"Contents of .pdd/meta/: {list(meta_dir.glob('*')) if meta_dir.exists() else 'directory does not exist'}" + ) + + # 7. Read and parse the fingerprint metadata + fingerprint_data = json.loads(fingerprint_file.read_text()) + + # 8. THE BUG CHECK: Verify hash fields are NOT null + hash_fields = ["prompt_hash", "code_hash", "example_hash", "test_hash"] + null_hashes = [] + + for field in hash_fields: + if field in fingerprint_data: + value = fingerprint_data[field] + if value is None: + null_hashes.append(field) + + if null_hashes: + pytest.fail( + f"BUG DETECTED (Issue #437): Fingerprint metadata has null hash fields!\n\n" + f"Null fields: {null_hashes}\n\n" + f"Full fingerprint data:\n{json.dumps(fingerprint_data, indent=2)}\n\n" + f"Root cause: The @log_operation decorator at pdd/operation_log.py:338\n" + f"calls save_fingerprint() without the required 'paths' parameter,\n" + f"causing calculate_current_hashes() to return an empty dict.\n\n" + f"Expected: All hash fields should contain 64-character hex SHA-256 hashes.\n" + f"Actual: Hash fields are null, breaking fingerprint-based change detection.\n\n" + f"This breaks PDD's core feature of detecting which files have changed\n" + f"and intelligently deciding when to regenerate code." + ) + + # 9. Additional validation: Verify hashes are valid SHA-256 format + # SHA-256 hashes are 64 hexadecimal characters + for field in hash_fields: + if field in fingerprint_data and fingerprint_data[field] is not None: + hash_value = fingerprint_data[field] + assert isinstance(hash_value, str), ( + f"Hash field {field} should be a string, got {type(hash_value)}" + ) + assert len(hash_value) == 64, ( + f"Hash field {field} should be 64 characters (SHA-256), got {len(hash_value)}" + ) + assert all(c in "0123456789abcdef" for c in hash_value.lower()), ( + f"Hash field {field} should be hexadecimal, got {hash_value}" + ) + + def test_pdd_generate_with_mock_llm_minimal(self, tmp_path, monkeypatch): + """ + Minimal E2E test for Issue #437 with simpler setup. + + This test focuses purely on the fingerprint hash bug without + requiring full project setup. + """ + project_dir = tmp_path / "minimal_project" + project_dir.mkdir() + monkeypatch.chdir(project_dir) + + # Create minimal prompt + prompt_file = project_dir / "test_Python.prompt" + prompt_file.write_text("# Test\nCreate a test function.") + + # Mock to create minimal output + def mock_generator(*args, **kwargs): + (project_dir / "test.py").write_text("def test(): pass") + return ("Success", 0.0, "mock-model") + + with patch("pdd.commands.generate.code_generator_main", side_effect=mock_generator): + from pdd.cli import cli + runner = CliRunner() + result = runner.invoke(cli, ["generate", str(prompt_file)], catch_exceptions=False) + + assert result.exit_code == 0, f"Command failed: {result.output}" + + # Check fingerprint + fingerprint_file = project_dir / ".pdd" / "meta" / "test_Python.json" + assert fingerprint_file.exists(), "Fingerprint file not created" + + fingerprint_data = json.loads(fingerprint_file.read_text()) + + # THE BUG: Check if prompt_hash is null + if fingerprint_data.get("prompt_hash") is None: + pytest.fail( + f"BUG Issue #437: prompt_hash is null!\n" + f"Fingerprint: {json.dumps(fingerprint_data, indent=2)}\n\n" + f"The @log_operation decorator doesn't pass 'paths' to save_fingerprint(),\n" + f"causing all hash fields to be null instead of containing SHA-256 hashes." + ) + + # Verify prompt_hash is a valid SHA-256 hash + prompt_hash = fingerprint_data.get("prompt_hash") + assert prompt_hash is not None, "prompt_hash should not be None" + assert isinstance(prompt_hash, str), "prompt_hash should be a string" + assert len(prompt_hash) == 64, f"prompt_hash should be 64 chars, got {len(prompt_hash)}" + assert all(c in "0123456789abcdef" for c in prompt_hash.lower()), ( + f"prompt_hash should be hex, got {prompt_hash}" + ) diff --git a/tests/test_operation_log.py b/tests/test_operation_log.py index 22fddf76b..891de5e0e 100644 --- a/tests/test_operation_log.py +++ b/tests/test_operation_log.py @@ -623,4 +623,118 @@ def test_fingerprint_hash_compatibility_with_sync(tmp_path): assert result.command == "generate" # Verify pdd_version is set - assert result.pdd_version is not None, "pdd_version should be set" \ No newline at end of file + assert result.pdd_version is not None, "pdd_version should be set" + + +# -------------------------------------------------------------------------------- +# BUG REPRODUCTION TEST: Issue #437 - Null Hashes in Fingerprint +# -------------------------------------------------------------------------------- + +def test_log_operation_decorator_null_hashes_bug_issue_437(temp_pdd_env, tmp_path): + """ + Regression test for Issue #437: Decorator creates fingerprints with null hashes. + + Bug: The @log_operation decorator calls save_fingerprint() WITHOUT the required + 'paths' parameter, causing all hash fields (prompt_hash, code_hash, example_hash, + test_hash) to be set to None/null instead of containing actual content hashes. + + This test verifies that when a command decorated with @log_operation runs + successfully with updates_fingerprint=True, the resulting fingerprint file + contains actual hash values (64-character hex strings) rather than null. + + Root cause: pdd/operation_log.py:338 calls: + save_fingerprint(basename, language, operation=operation, cost=cost, model=model) + Missing the 'paths' parameter that calculate_current_hashes() needs. + + When fixed, the decorator should either: + 1. Return paths from the decorated function for the decorator to use + 2. Store paths in Click context for the decorator to access + 3. Pass paths explicitly when calling decorated functions + """ + # Create actual files with content so hashes can be calculated + prompts_dir = tmp_path / "prompts" + prompts_dir.mkdir() + src_dir = tmp_path / "src" + src_dir.mkdir() + examples_dir = tmp_path / "examples" + examples_dir.mkdir() + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + + basename = "test_module" + language = "python" + + # Create files with actual content + prompt_file = prompts_dir / f"{basename}_{language}.prompt" + prompt_file.write_text("# Test prompt content\nThis is a test prompt.") + + code_file = src_dir / f"{basename}.py" + code_file.write_text("def test_function():\n return 'hello'\n") + + example_file = examples_dir / f"{basename}_{language}.example" + example_file.write_text("# Example usage\ntest_function()\n") + + test_file = tests_dir / f"test_{basename}.py" + test_file.write_text("def test_test_function():\n assert True\n") + + # Mock function decorated with @log_operation + @operation_log.log_operation( + operation="generate", + updates_fingerprint=True + ) + def mock_generate_command(prompt_file: str): + """Simulates a generate command that should update fingerprints.""" + # In a real command, this would generate code and return cost/model info + # For this test, we just return the tuple structure the decorator expects + return {"status": "generated"}, 0.25, "gpt-4" + + # Run the decorated command + prompt_path = f"prompts/{basename}_{language}.prompt" + result = mock_generate_command(prompt_file=prompt_path) + + # Verify the command executed successfully + assert result[0]["status"] == "generated" + + # Load the fingerprint file that was created + fp_path = operation_log.get_fingerprint_path(basename, language) + assert fp_path.exists(), "Fingerprint file should be created" + + with open(fp_path) as f: + fp_data = json.load(f) + + # BUG DETECTION: With the current bug, all hash fields will be null + # This is what we're testing for - this assertion SHOULD FAIL on buggy code + # and PASS after the fix + + # Verify basic structure exists + assert "prompt_hash" in fp_data + assert "code_hash" in fp_data + assert "example_hash" in fp_data + assert "test_hash" in fp_data + + # THE CRITICAL ASSERTIONS: These will FAIL with the bug, PASS after fix + # With the bug, all hashes are None. After fix, they should be SHA-256 hex strings. + assert fp_data["prompt_hash"] is not None, ( + "Bug Issue #437: prompt_hash is null! The decorator doesn't pass 'paths' to save_fingerprint()" + ) + assert fp_data["code_hash"] is not None, ( + "Bug Issue #437: code_hash is null! The decorator doesn't pass 'paths' to save_fingerprint()" + ) + + # After fix, verify they are valid SHA-256 hashes (64 hex characters) + if fp_data["prompt_hash"] is not None: + assert len(fp_data["prompt_hash"]) == 64, "prompt_hash should be SHA-256 (64 hex chars)" + assert all(c in "0123456789abcdef" for c in fp_data["prompt_hash"]), ( + "prompt_hash should be hexadecimal" + ) + + if fp_data["code_hash"] is not None: + assert len(fp_data["code_hash"]) == 64, "code_hash should be SHA-256 (64 hex chars)" + assert all(c in "0123456789abcdef" for c in fp_data["code_hash"]), ( + "code_hash should be hexadecimal" + ) + + # Verify other fingerprint fields are correctly populated + assert fp_data["command"] == "generate" + assert "pdd_version" in fp_data + assert "timestamp" in fp_data \ No newline at end of file