DM-52738: Run 'butler ingest-graph' in QBB runs.

.pre-commit-config.yaml

@@ -13,7 +13,3 @@ repos:
       - id: ruff
         args: [--fix]
       - id: ruff-format
-  - repo: https://github.com/numpy/numpydoc
-    rev: "v1.8.0"
-    hooks:
-      - id: numpydoc-validation

data/SConscript

@@ -102,7 +102,7 @@ rc2 = (
         where="instrument='HSC'",
         extra_qg_args=["--data-id-table", "resource://lsst.ci.middleware/data/data_ids.ecsv"],
     )
-    .add("step1", where="instrument='HSC' AND exposure NOT IN (18202, 96980)")
+    .add("step1", where="instrument='HSC' AND exposure NOT IN (18202, 96980)", ingest_graph=False)
     # Add more side runs to check --extend-run and --clobber-outputs.
     .add_side_run(
         "test-clobber-without-skip",
@@ -139,6 +139,7 @@ rc2 = (
         where="skymap='ci_mw' AND tract=0",
         group="attempt1",
         fail=['''consolidateForcedSourceTable::"skymap='ci_mw' AND tract=0"'''],
+        ingest_graph=False,
     )
     .add("step5", where="skymap='ci_mw' AND tract=0", group="attempt2", skip_existing_in_last=True)
     .add("step6")
@@ -166,8 +167,9 @@ prod = (
         group="i-attempt1",
         where="band='i'",
         fail=['''calibrateImage::"instrument='HSC' AND visit=18202"'''],
+        ingest_graph=False
     )
-    .add("step1", group="i-attempt2", where="band='i'", skip_existing_in_last=True)
+    .add("step1", group="i-attempt2", where="band='i'", skip_existing_in_last=True, ingest_graph=False)
     .add("step2a", group="r", where="band='r'")
     .add("step2a", group="i", where="band='i'")
     .add("step2b", group="even", where="skymap='ci_mw' AND tract IN (0, 2)")

python/lsst/ci/middleware/scons.py

@@ -187,6 +187,7 @@ def add(
         fail: Sequence[str] = (),
         skip_existing_in_last: bool = False,
         auto_retry_mem: tuple[str, str] | None = None,
+        ingest_graph: bool = True,
     ) -> PipelineCommands:
         """Add a new QuantumGraph and its execution to the build.
 
@@ -217,6 +218,10 @@ def add(
             ``fail`` setting that causes the first attempt for one or more
             quanta to simulate an out-of-memory failure for the first value
             only, effectively testing BPS auto-retry logic.
+        ingest_graph : `bool`, optional
+            Whether to ingest the provenance quantum graph (only for QBB right
+            now).  Must be `False` if a side run with ``extend_run=True`` will
+            be added.
 
         Returns
         -------
@@ -252,6 +257,7 @@ def add(
             output_run=output_run,
             expect_failure=bool(fail),
             auto_retry_mem=auto_retry_mem,
+            ingest_graph=ingest_graph,
         )
         self.last_output_run = output_run
         return self
@@ -626,6 +632,7 @@ def _add_qbb(
         pre_exec_init: bool = True,
         auto_retry_mem: tuple[str, str] | None = None,
         raise_on_partial_outputs: bool = False,
+        ingest_graph: bool = True,
     ) -> File:
         """Make an SCons target for direct execution of the quantum graph
         with ``pipetask run-qbb`` and `lsst.daf.butler.QuantumBackedButler`.
@@ -648,6 +655,9 @@ def _add_qbb(
         raise_on_partial_outputs : `bool`, optional
             If `True`, pass ``--raise-on-partial-outputs`` to
             ``pipetask run-qbb``.
+        ingest_graph : `bool`, optional
+            Whether to ingest the provenance quantum graph.  Must be `False` if
+            a side run will use ``extend_run=True``.
 
         Returns
         -------
@@ -718,6 +728,23 @@ def _add_qbb(
                 "-o",
                 "${TARGETS[2]}",
                 "--mock-storage-classes",
+                *(["--promise-ingest-graph"] if ingest_graph else []),
+            ),
+            python_cmd(
+                BUTLER_BIN,
+                "--long-log",
+                "--log-level",
+                "VERBOSE",
+                "--log-file",
+                f"data/{log}",
+                "--no-log-tty",
+                "ingest-graph",
+                repo_in_cmd,
+                "${TARGETS[2]}",
+                "--batch-size",
+                "50",
+                "-t",
+                "copy",
             ),
             tar_repo_cmd(repo_in_cmd, "${TARGETS[0]}"),
         ]

tests/test_ci_hsc_outputs.py

@@ -54,7 +54,7 @@ def setUp(self) -> None:
 
     def test_direct_qbb_equivalence(self) -> None:
         """Test that the direct and QBB runs produce exactly the same
-        collections, dataset types, and datasets."""
+        collections, dataset types, and datasets (except for provenance)."""
         self.assertEqual(
             list(
                 self.direct.butler.registry.queryCollections(
@@ -69,7 +69,8 @@ def test_direct_qbb_equivalence(self) -> None:
         )
         self.assertEqual(
             set(self.direct.butler.registry.queryDatasetTypes(...)),
-            set(self.qbb.butler.registry.queryDatasetTypes(...)),
+            set(self.qbb.butler.registry.queryDatasetTypes(...))
+            - {self.qbb.butler.get_dataset_type("run_provenance")},
         )
         self.assertEqual(
             set(self.direct.butler.registry.queryDatasets(get_mock_name("isr_config"))),

tests/test_prod_outputs.py

@@ -240,7 +240,8 @@ def check_step1_qpg(self, helper: OutputRepoTests) -> None:
                         for message in quantum_summary.messages:
                             self.assertIsInstance(message, str)
                             self.assertTrue(
-                                message.startswith("Execution of task '_mock_calibrateImage' on quantum")
+                                message.startswith("Execution of task '_mock_calibrateImage' on quantum"),
+                                message,
                             )
                             self.assertIn(
                                 "Exception ValueError: Simulated failure: task=_mock_calibrateImage", message