diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5eb932a..72ee996 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,3 @@ repos: - id: ruff args: [--fix] - id: ruff-format - - repo: https://github.com/numpy/numpydoc - rev: "v1.8.0" - hooks: - - id: numpydoc-validation diff --git a/data/SConscript b/data/SConscript index 29cdc7d..d308b96 100644 --- a/data/SConscript +++ b/data/SConscript @@ -102,7 +102,7 @@ rc2 = ( where="instrument='HSC'", extra_qg_args=["--data-id-table", "resource://lsst.ci.middleware/data/data_ids.ecsv"], ) - .add("step1", where="instrument='HSC' AND exposure NOT IN (18202, 96980)") + .add("step1", where="instrument='HSC' AND exposure NOT IN (18202, 96980)", ingest_graph=False) # Add more side runs to check --extend-run and --clobber-outputs. .add_side_run( "test-clobber-without-skip", @@ -139,6 +139,7 @@ rc2 = ( where="skymap='ci_mw' AND tract=0", group="attempt1", fail=['''consolidateForcedSourceTable::"skymap='ci_mw' AND tract=0"'''], + ingest_graph=False, ) .add("step5", where="skymap='ci_mw' AND tract=0", group="attempt2", skip_existing_in_last=True) .add("step6") @@ -166,8 +167,9 @@ prod = ( group="i-attempt1", where="band='i'", fail=['''calibrateImage::"instrument='HSC' AND visit=18202"'''], + ingest_graph=False ) - .add("step1", group="i-attempt2", where="band='i'", skip_existing_in_last=True) + .add("step1", group="i-attempt2", where="band='i'", skip_existing_in_last=True, ingest_graph=False) .add("step2a", group="r", where="band='r'") .add("step2a", group="i", where="band='i'") .add("step2b", group="even", where="skymap='ci_mw' AND tract IN (0, 2)") diff --git a/python/lsst/ci/middleware/scons.py b/python/lsst/ci/middleware/scons.py index 0514fc8..4bf09d4 100644 --- a/python/lsst/ci/middleware/scons.py +++ b/python/lsst/ci/middleware/scons.py @@ -187,6 +187,7 @@ def add( fail: Sequence[str] = (), skip_existing_in_last: bool = False, auto_retry_mem: tuple[str, str] | None = None, + ingest_graph: bool = True, ) -> PipelineCommands: """Add a new QuantumGraph and its execution to the build. @@ -217,6 +218,10 @@ def add( ``fail`` setting that causes the first attempt for one or more quanta to simulate an out-of-memory failure for the first value only, effectively testing BPS auto-retry logic. + ingest_graph : `bool`, optional + Whether to ingest the provenance quantum graph (only for QBB right + now). Must be `False` if a side run with ``extend_run=True`` will + be added. Returns ------- @@ -252,6 +257,7 @@ def add( output_run=output_run, expect_failure=bool(fail), auto_retry_mem=auto_retry_mem, + ingest_graph=ingest_graph, ) self.last_output_run = output_run return self @@ -626,6 +632,7 @@ def _add_qbb( pre_exec_init: bool = True, auto_retry_mem: tuple[str, str] | None = None, raise_on_partial_outputs: bool = False, + ingest_graph: bool = True, ) -> File: """Make an SCons target for direct execution of the quantum graph with ``pipetask run-qbb`` and `lsst.daf.butler.QuantumBackedButler`. @@ -648,6 +655,9 @@ def _add_qbb( raise_on_partial_outputs : `bool`, optional If `True`, pass ``--raise-on-partial-outputs`` to ``pipetask run-qbb``. + ingest_graph : `bool`, optional + Whether to ingest the provenance quantum graph. Must be `False` if + a side run will use ``extend_run=True``. Returns ------- @@ -718,6 +728,23 @@ def _add_qbb( "-o", "${TARGETS[2]}", "--mock-storage-classes", + *(["--promise-ingest-graph"] if ingest_graph else []), + ), + python_cmd( + BUTLER_BIN, + "--long-log", + "--log-level", + "VERBOSE", + "--log-file", + f"data/{log}", + "--no-log-tty", + "ingest-graph", + repo_in_cmd, + "${TARGETS[2]}", + "--batch-size", + "50", + "-t", + "copy", ), tar_repo_cmd(repo_in_cmd, "${TARGETS[0]}"), ] diff --git a/tests/test_ci_hsc_outputs.py b/tests/test_ci_hsc_outputs.py index e95d8d1..9b53c95 100644 --- a/tests/test_ci_hsc_outputs.py +++ b/tests/test_ci_hsc_outputs.py @@ -54,7 +54,7 @@ def setUp(self) -> None: def test_direct_qbb_equivalence(self) -> None: """Test that the direct and QBB runs produce exactly the same - collections, dataset types, and datasets.""" + collections, dataset types, and datasets (except for provenance).""" self.assertEqual( list( self.direct.butler.registry.queryCollections( @@ -69,7 +69,8 @@ def test_direct_qbb_equivalence(self) -> None: ) self.assertEqual( set(self.direct.butler.registry.queryDatasetTypes(...)), - set(self.qbb.butler.registry.queryDatasetTypes(...)), + set(self.qbb.butler.registry.queryDatasetTypes(...)) + - {self.qbb.butler.get_dataset_type("run_provenance")}, ) self.assertEqual( set(self.direct.butler.registry.queryDatasets(get_mock_name("isr_config"))), diff --git a/tests/test_prod_outputs.py b/tests/test_prod_outputs.py index 22796e6..94c4f40 100644 --- a/tests/test_prod_outputs.py +++ b/tests/test_prod_outputs.py @@ -240,7 +240,8 @@ def check_step1_qpg(self, helper: OutputRepoTests) -> None: for message in quantum_summary.messages: self.assertIsInstance(message, str) self.assertTrue( - message.startswith("Execution of task '_mock_calibrateImage' on quantum") + message.startswith("Execution of task '_mock_calibrateImage' on quantum"), + message, ) self.assertIn( "Exception ValueError: Simulated failure: task=_mock_calibrateImage", message