Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 14 additions & 9 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ TEST_OUTPUTS := $(patsubst $(PDD_DIR)/%.py,$(TESTS_DIR)/test_%.py,$(PY_OUTPUTS))
# All Example files in context directory (recursive)
EXAMPLE_FILES := $(shell find $(CONTEXT_DIR) -name "*_example.py" 2>/dev/null)

.PHONY: all clean test requirements production coverage staging regression sync-regression all-regression cloud-regression install build analysis fix crash update update-extension generate run-examples verify detect change lint publish publish-public publish-public-cap public-ensure public-update public-import public-diff sync-public
.PHONY: all clean test requirements production coverage staging regression sync-regression all-regression cloud-regression install build analysis fix crash update update-extension generate run-examples verify detect change lint publish publish-public publish-public-cap public-ensure public-update public-import public-diff sync-public ensure-dev-deps

all: $(PY_OUTPUTS) $(MAKEFILE_OUTPUT) $(CSV_OUTPUTS) $(EXAMPLE_OUTPUTS) $(TEST_OUTPUTS)

Expand Down Expand Up @@ -190,20 +190,25 @@ run-examples: $(EXAMPLE_FILES)
)
@echo "All examples ran successfully"

# Ensure dev dependencies are installed before running tests
ensure-dev-deps:
@echo "Updating pdd conda environment with dev dependencies"
@conda run -n pdd --no-capture-output pip install -e '.[dev]'

# Run tests
test:
test: ensure-dev-deps
@echo "Running staging tests"
@cd $(STAGING_DIR)
@conda run -n pdd --no-capture-output PDD_RUN_REAL_LLM_TESTS=1 PDD_RUN_LLM_TESTS=1 PDD_PATH=$(abspath $(PDD_DIR)) PYTHONPATH=$(PDD_DIR):$$PYTHONPATH python -m pytest -vv -n auto $(TESTS_DIR)

# Run tests with coverage
coverage:
coverage: ensure-dev-deps
@echo "Running tests with coverage"
@cd $(STAGING_DIR)
@conda run -n pdd --no-capture-output PDD_PATH=$(STAGING_DIR) PYTHONPATH=$(PDD_DIR):$$PYTHONPATH python -m pytest --cov=$(PDD_DIR) --cov-report=term-missing --cov-report=html $(TESTS_DIR)

# Run pylint
lint:
lint: ensure-dev-deps
@echo "Running pylint"
@conda run -n pdd --no-capture-output pylint pdd tests

Expand Down Expand Up @@ -467,7 +472,7 @@ production:
@cp -r $(TESTS_DIR) .
@cp $(MAKEFILE_OUTPUT) .

regression:
regression: ensure-dev-deps
@echo "Running regression tests"
@mkdir -p staging/regression
@find staging/regression -type f ! -name ".*" -delete
Expand All @@ -480,7 +485,7 @@ endif

SYNC_PARALLEL ?= 1

sync-regression:
sync-regression: ensure-dev-deps
@echo "Running sync regression tests"
ifdef TEST_NUM
@echo "Running specific sync test: $(TEST_NUM)"
Expand All @@ -498,7 +503,7 @@ endif
all-regression: regression sync-regression cloud-regression
@echo "All regression test suites completed."

cloud-regression:
cloud-regression: ensure-dev-deps
@echo "Running cloud regression tests"
@mkdir -p staging/cloud_regression
ifdef TEST_NUM
Expand All @@ -510,7 +515,7 @@ endif

# Automated test runner with Infisical for CI/CD
.PHONY: test-all-ci
test-all-ci:
test-all-ci: ensure-dev-deps
@echo "Running all test suites with result capture for CI/CD"
@mkdir -p test_results
ifdef PR_NUMBER
Expand All @@ -525,7 +530,7 @@ endif

# Run all tests with Infisical (for local development and CI)
.PHONY: test-all-with-infisical
test-all-with-infisical:
test-all-with-infisical: ensure-dev-deps
@echo "Running all test suites with Infisical"
@if ! command -v infisical &> /dev/null; then \
echo "Error: Infisical CLI not found. Please install it:"; \
Expand Down
142 changes: 142 additions & 0 deletions tests/test_code_generator_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -2223,3 +2223,145 @@ def test_json_serialization_after_unwrap(self):

expected = '[\n {\n "key": "value"\n }\n]'
assert generated_code_content == expected


# =============================================================================
# Issue #154 Regression Tests: File Overwrite Not Happening
# =============================================================================
#
# These tests verify the bug where "generation overwrite a file doesn't always
# happen". The root cause is an architectural disconnect between:
# 1. construct_paths() - handles overwrite confirmation
# 2. code_generator_main() - handles actual file write at line 1087
#
# When generate_output_paths() returns {} (e.g., due to empty basename),
# output_path becomes falsy and the write block is skipped silently.
#
# TEST STRATEGY:
# These tests assert the EXPECTED FIXED behavior, so they will FAIL on the
# current buggy code and PASS once the bug is fixed.
# =============================================================================


class TestIssue154OverwriteNotHappening:
"""
Regression tests for GitHub Issue #154: File overwrite not always happening.

The bug manifests when:
1. construct_paths() returns empty/falsy output_file_paths
2. LLM successfully generates code
3. The file write block at line 1087 is silently skipped due to `if output_path:`

These tests assert the EXPECTED behavior after the fix, so they will
FAIL on the current buggy code.
"""

def test_issue_154_empty_output_path_should_raise_error(
self, mock_ctx, temp_dir_setup, mock_construct_paths_fixture,
mock_local_generator_fixture, mock_rich_console_fixture, mock_env_vars
):
"""
FAILING TEST for issue #154: Empty output_path should raise an error.

Current behavior (BUG): Code generates, write is silently skipped.
Expected behavior (FIX): Should raise click.UsageError when output_path is empty.

This test FAILS on the current buggy code because no error is raised.
"""
mock_ctx.obj['local'] = True
mock_ctx.obj['quiet'] = False
prompt_file_path = temp_dir_setup["prompts_dir"] / "issue154_prompt.prompt"
create_file(prompt_file_path, "Test prompt for issue 154")

# Simulate what happens when generate_output_paths returns {}
# due to empty basename - output_file_paths.get("output") returns ""
mock_construct_paths_fixture.return_value = (
{}, # resolved_config
{"prompt_file": "Test prompt for issue 154"},
{"output": ""}, # Empty string - falsy output path (triggers the bug)
"python"
)

# EXPECTED FIX: Should raise click.UsageError when output_path is empty
# CURRENT BUG: No error raised, write is silently skipped
with pytest.raises(click.UsageError) as excinfo:
code_generator_main(
mock_ctx, str(prompt_file_path), "", None, False
)

# Verify the error message is helpful
assert "output" in str(excinfo.value).lower() or "path" in str(excinfo.value).lower(), \
"Error message should mention output path issue"

def test_issue_154_none_output_path_should_raise_error(
self, mock_ctx, temp_dir_setup, mock_construct_paths_fixture,
mock_local_generator_fixture, mock_rich_console_fixture, mock_env_vars
):
"""
FAILING TEST for issue #154: None output_path should raise an error.

When construct_paths returns None for output path (e.g., when
generate_output_paths returns {} and .get("output") returns None),
an error should be raised instead of silently skipping the write.

This test FAILS on the current buggy code.
"""
mock_ctx.obj['local'] = True
mock_ctx.obj['quiet'] = False
prompt_file_path = temp_dir_setup["prompts_dir"] / "issue154_none_prompt.prompt"
create_file(prompt_file_path, "Test prompt for issue 154 - None case")

# Simulate output_file_paths.get("output") returning None
mock_construct_paths_fixture.return_value = (
{}, # resolved_config
{"prompt_file": "Test prompt for issue 154 - None case"},
{"output": None}, # None output path - triggers the bug
"python"
)

# EXPECTED FIX: Should raise click.UsageError when output_path is None
# CURRENT BUG: No error, write silently skipped, console shows warning
with pytest.raises(click.UsageError) as excinfo:
code_generator_main(
mock_ctx, str(prompt_file_path), None, None, False
)

assert "output" in str(excinfo.value).lower() or "path" in str(excinfo.value).lower()

def test_issue_154_llm_should_not_be_called_with_invalid_output_path(
self, mock_ctx, temp_dir_setup, mock_construct_paths_fixture,
mock_local_generator_fixture, mock_env_vars
):
"""
FAILING TEST for issue #154: LLM should NOT be called when output_path is invalid.

The architectural bug is that output_path validation happens AFTER the
expensive LLM call. The LLM is called (costing money/time) even when
output_path is invalid/empty.

Expected fix: Validate output_path BEFORE calling the LLM.
This test FAILS on the current buggy code because the LLM IS called.
"""
mock_ctx.obj['local'] = True
prompt_file_path = temp_dir_setup["prompts_dir"] / "issue154_llm_call.prompt"
create_file(prompt_file_path, "Test prompt")

# Empty output path - simulates generate_output_paths returning {}
mock_construct_paths_fixture.return_value = (
{},
{"prompt_file": "Test prompt"},
{"output": ""}, # Empty - should be caught BEFORE LLM call
"python"
)

# Expect an error to be raised BEFORE the LLM is called
try:
code_generator_main(mock_ctx, str(prompt_file_path), "", None, False)
except click.UsageError:
pass # This is the expected behavior after fix

# EXPECTED FIX: LLM should NOT be called when output_path is invalid
# CURRENT BUG: LLM IS called, wasting resources
assert not mock_local_generator_fixture.called, \
"LLM should NOT be called when output_path is empty/invalid. " \
"The output path should be validated BEFORE expensive LLM operations."
Loading