Refactor: Convert all tests to use multiprocessing queues

tests/workers/conftest.py

@@ -10,6 +10,7 @@
 import platform
 import queue
 import stat
+import sys
 import typing
 
 import cruizlib.workers.api as workers_api
@@ -308,22 +309,39 @@ def _the_fixture(
         watcher_thread: TestableThread,
         context: typing.Optional[multiprocessing.context.SpawnContext],
     ) -> None:
+        def _have_conan_symbols_leaked() -> None:
+            assert "conans" not in sys.modules
+
         if context is None:
             # abusing the type system, as the API used for queue.Queue is the same
             # as for multiprocessing.Queue
             worker(reply_queue, params)
         else:
+            _have_conan_symbols_leaked()
             process = context.Process(target=worker, args=(reply_queue, params))
             process.start()
             process.join()
 
         # tell the watcher thread that there is no more information coming
-        workers_api.endmessagethread.invoke(reply_queue)  # type: ignore[arg-type]
+        if context is not None:
+            # this must be done in a separate process because it closes the other side
+            # of the queue
+            process = context.Process(
+                target=workers_api.endmessagethread.invoke, args=(reply_queue,)
+            )
+            process.start()
+            process.join()
+        else:
+            # no need to close ths single process queue
+            reply_queue.put(End())
 
         watcher_thread.join(timeout=5.0)
         if watcher_thread.is_alive():
             raise texceptions.WatcherThreadTimeoutError()
 
+        if context is not None:
+            _have_conan_symbols_leaked()
+
     return _the_fixture
 
 

tests/workers/single_process/test_arbitrary_conan_cmd.py

@@ -22,7 +22,7 @@
 import texceptions
 
 if typing.TYPE_CHECKING:
-    from ttypes import RunWorkerFixture, SingleprocessReplyQueueFixture
+    from ttypes import MultiprocessReplyQueueFixture, RunWorkerFixture
 
 
 @pytest.mark.parametrize(
@@ -63,7 +63,7 @@
 )
 # pylint: disable=too-many-arguments, too-many-positional-arguments
 def test_arbitrary_conan_command(
-    reply_queue_fixture: SingleprocessReplyQueueFixture,
+    multiprocess_reply_queue_fixture: MultiprocessReplyQueueFixture,
     run_worker: RunWorkerFixture,
     conan_local_cache: typing.Dict[str, str],
     verb: str,
@@ -81,7 +81,7 @@ def test_arbitrary_conan_command(
     if args:
         params.arguments.extend(args)
 
-    reply_queue, replies, watcher_thread, context = reply_queue_fixture()
+    reply_queue, replies, watcher_thread, context = multiprocess_reply_queue_fixture()
     with expectation:
         run_worker(worker, reply_queue, params, watcher_thread, context)
 

tests/workers/single_process/test_cmake_build_tool.py

@@ -25,7 +25,7 @@
 import texceptions
 
 if typing.TYPE_CHECKING:
-    from ttypes import RunWorkerFixture, SingleprocessReplyQueueFixture
+    from ttypes import MultiprocessReplyQueueFixture, RunWorkerFixture
 
 
 LOGGER = logging.getLogger(__name__)
@@ -37,7 +37,7 @@
     strict=True,
 )
 def test_cmake_no_cache(
-    reply_queue_fixture: SingleprocessReplyQueueFixture,
+    multiprocess_reply_queue_fixture: MultiprocessReplyQueueFixture,
     run_worker: RunWorkerFixture,
     tmp_path: pathlib.Path,
     caplog: pytest.LogCaptureFixture,
@@ -49,7 +49,7 @@ def test_cmake_no_cache(
     params = CommandParameters("cmakebuild", worker)
     params.cwd = tmp_path
     # params.added_environment = conan_local_cache
-    reply_queue, _, watcher_thread, context = reply_queue_fixture()
+    reply_queue, _, watcher_thread, context = multiprocess_reply_queue_fixture()
     with pytest.raises(
         texceptions.FailedMessageTestError, match="Error: could not load cache"
     ):
@@ -73,7 +73,7 @@ def fixture_custom_cmake_command(tmp_path: pathlib.Path) -> pathlib.Path:
     strict=True,
 )
 def test_cmake_custom_program(
-    reply_queue_fixture: SingleprocessReplyQueueFixture,
+    multiprocess_reply_queue_fixture: MultiprocessReplyQueueFixture,
     run_worker: RunWorkerFixture,
     tmp_path: pathlib.Path,
     caplog: pytest.LogCaptureFixture,
@@ -86,7 +86,7 @@ def test_cmake_custom_program(
     params = CommandParameters("cmakebuild", worker)
     params.cwd = tmp_path
     params.added_environment = {"CONAN_CMAKE_PROGRAM": os.fspath(custom_cmake_command)}
-    reply_queue, replies, watcher_thread, context = reply_queue_fixture()
+    reply_queue, replies, watcher_thread, context = multiprocess_reply_queue_fixture()
     run_worker(worker, reply_queue, params, watcher_thread, context)
 
     assert replies
@@ -102,7 +102,7 @@ def test_cmake_custom_program(
     strict=True,
 )
 def test_cmake_custom_build_tool(
-    reply_queue_fixture: SingleprocessReplyQueueFixture,
+    multiprocess_reply_queue_fixture: MultiprocessReplyQueueFixture,
     run_worker: RunWorkerFixture,
     tmp_path: pathlib.Path,
     caplog: pytest.LogCaptureFixture,
@@ -114,7 +114,7 @@ def test_cmake_custom_build_tool(
     params = CommandParameters("cmakebuild", worker)
     params.cwd = tmp_path
     params.added_environment = {"CONAN_MAKE_PROGRAM": "another_make"}
-    reply_queue, _, watcher_thread, context = reply_queue_fixture()
+    reply_queue, _, watcher_thread, context = multiprocess_reply_queue_fixture()
     with pytest.raises(
         texceptions.FailedMessageTestError, match="Error: could not load cache"
     ):
@@ -127,7 +127,7 @@ def test_cmake_custom_build_tool(
     strict=True,
 )
 def test_cmake_use_ninja_generator(
-    reply_queue_fixture: SingleprocessReplyQueueFixture,
+    multiprocess_reply_queue_fixture: MultiprocessReplyQueueFixture,
     run_worker: RunWorkerFixture,
     tmp_path: pathlib.Path,
     caplog: pytest.LogCaptureFixture,
@@ -139,7 +139,7 @@ def test_cmake_use_ninja_generator(
     params = CommandParameters("cmakebuild", worker)
     params.cwd = tmp_path
     params.added_environment = {"CONAN_CMAKE_GENERATOR": "Ninja"}
-    reply_queue, _, watcher_thread, context = reply_queue_fixture()
+    reply_queue, _, watcher_thread, context = multiprocess_reply_queue_fixture()
     with pytest.raises(
         texceptions.FailedMessageTestError, match="Error: could not load cache"
     ):
@@ -153,7 +153,7 @@ def test_cmake_use_ninja_generator(
 )
 @pytest.mark.parametrize("generator", [None, "Ninja"])
 def test_cmake_verbose_output(
-    reply_queue_fixture: SingleprocessReplyQueueFixture,
+    multiprocess_reply_queue_fixture: MultiprocessReplyQueueFixture,
     run_worker: RunWorkerFixture,
     tmp_path: pathlib.Path,
     caplog: pytest.LogCaptureFixture,
@@ -168,7 +168,7 @@ def test_cmake_verbose_output(
     if generator:
         params.added_environment = {"CONAN_CMAKE_GENERATOR": generator}
     params.arguments.append("verbose")
-    reply_queue, _, watcher_thread, context = reply_queue_fixture()
+    reply_queue, _, watcher_thread, context = multiprocess_reply_queue_fixture()
     with pytest.raises(
         texceptions.FailedMessageTestError, match="Error: could not load cache"
     ):
@@ -181,7 +181,7 @@ def test_cmake_verbose_output(
     strict=True,
 )
 def test_cmake_set_cpu_count(
-    reply_queue_fixture: SingleprocessReplyQueueFixture,
+    multiprocess_reply_queue_fixture: MultiprocessReplyQueueFixture,
     run_worker: RunWorkerFixture,
     tmp_path: pathlib.Path,
     caplog: pytest.LogCaptureFixture,
@@ -193,7 +193,7 @@ def test_cmake_set_cpu_count(
     params = CommandParameters("cmakebuild", worker)
     params.cwd = tmp_path
     params.added_environment = {"CONAN_CPU_COUNT": "1"}
-    reply_queue, _, watcher_thread, context = reply_queue_fixture()
+    reply_queue, _, watcher_thread, context = multiprocess_reply_queue_fixture()
     with pytest.raises(
         texceptions.FailedMessageTestError, match="Error: could not load cache"
     ):

tests/workers/single_process/test_conan_autotools_helper.py

@@ -23,7 +23,7 @@
 import pytest
 
 if typing.TYPE_CHECKING:
-    from ttypes import RunWorkerFixture, SingleprocessReplyQueueFixture
+    from ttypes import MultiprocessReplyQueueFixture, RunWorkerFixture
 
 LOGGER = logging.getLogger(__name__)
 
@@ -60,7 +60,7 @@
 )
 # pylint: disable=too-many-arguments, too-many-positional-arguments  # noqa: E501
 def test_conan_autotoolsbuildhelper_configure(
-    reply_queue_fixture: SingleprocessReplyQueueFixture,
+    multiprocess_reply_queue_fixture: MultiprocessReplyQueueFixture,
     run_worker: RunWorkerFixture,
     conan_autotoolsbuildenvironment_configure_recipe: pathlib.Path,
     conan_local_cache: typing.Dict[str, str],
@@ -81,7 +81,9 @@ def test_conan_autotoolsbuildhelper_configure(
         params.recipe_path = conan_autotoolsbuildenvironment_configure_recipe
         params.cwd = conan_autotoolsbuildenvironment_configure_recipe.parent
         params.profile = "default"
-        reply_queue, replies, watcher_thread, context = reply_queue_fixture()
+        reply_queue, replies, watcher_thread, context = (
+            multiprocess_reply_queue_fixture()
+        )
         run_worker(worker, reply_queue, params, watcher_thread, context)
 
     if env_key and env_value:
@@ -93,7 +95,7 @@ def test_conan_autotoolsbuildhelper_configure(
     params.added_environment = conan_local_cache
     params.recipe_path = conan_autotoolsbuildenvironment_configure_recipe
     params.cwd = conan_autotoolsbuildenvironment_configure_recipe.parent
-    reply_queue, replies, watcher_thread, context = reply_queue_fixture()
+    reply_queue, replies, watcher_thread, context = multiprocess_reply_queue_fixture()
     run_worker(worker, reply_queue, params, watcher_thread, context)
 
     assert replies
@@ -114,7 +116,7 @@ def test_conan_autotoolsbuildhelper_configure(
 )
 # pylint: disable=too-many-arguments, too-many-positional-arguments  # noqa: E501
 def test_conan_autotoolsbuildhelper_make(
-    reply_queue_fixture: SingleprocessReplyQueueFixture,
+    multiprocess_reply_queue_fixture: MultiprocessReplyQueueFixture,
     run_worker: RunWorkerFixture,
     conan_autotoolsbuildenvironment_make_recipe: pathlib.Path,
     conan_local_cache: typing.Dict[str, str],
@@ -139,7 +141,9 @@ def test_conan_autotoolsbuildhelper_make(
         params.recipe_path = conan_autotoolsbuildenvironment_make_recipe
         params.cwd = conan_autotoolsbuildenvironment_make_recipe.parent
         params.profile = "default"
-        reply_queue, replies, watcher_thread, context = reply_queue_fixture()
+        reply_queue, replies, watcher_thread, context = (
+            multiprocess_reply_queue_fixture()
+        )
         run_worker(worker, reply_queue, params, watcher_thread, context)
 
     if env_key and env_value:
@@ -151,7 +155,7 @@ def test_conan_autotoolsbuildhelper_make(
     params.added_environment["CONAN_MAKE_PROGRAM"] = os.fspath(custom_make_command)
     params.recipe_path = conan_autotoolsbuildenvironment_make_recipe
     params.cwd = conan_autotoolsbuildenvironment_make_recipe.parent
-    reply_queue, replies, watcher_thread, context = reply_queue_fixture()
+    reply_queue, replies, watcher_thread, context = multiprocess_reply_queue_fixture()
     run_worker(worker, reply_queue, params, watcher_thread, context)
 
     assert replies

tests/workers/single_process/test_conan_build.py

@@ -21,7 +21,7 @@
 import pytest
 
 if typing.TYPE_CHECKING:
-    from ttypes import RunWorkerFixture, SingleprocessReplyQueueFixture
+    from ttypes import MultiprocessReplyQueueFixture, RunWorkerFixture
 
 
 LOGGER = logging.getLogger(__name__)
@@ -67,7 +67,7 @@
 )
 # pylint: disable=too-many-arguments, too-many-positional-arguments  # noqa: E501
 def test_conan_build(
-    reply_queue_fixture: SingleprocessReplyQueueFixture,
+    multiprocess_reply_queue_fixture: MultiprocessReplyQueueFixture,
     run_worker: RunWorkerFixture,
     conan_recipe: pathlib.Path,
     conan_local_cache: typing.Dict[str, str],
@@ -92,7 +92,9 @@ def test_conan_build(
             # in the cwd for the install artifacts
             assert isinstance(value, str)
             params.install_folder = tmp_path / value
-        reply_queue, replies, watcher_thread, context = reply_queue_fixture()
+        reply_queue, replies, watcher_thread, context = (
+            multiprocess_reply_queue_fixture()
+        )
         run_worker(worker, reply_queue, params, watcher_thread, context)
 
     worker = workers_api.build.invoke
@@ -113,7 +115,7 @@ def test_conan_build(
         elif arg == "package_folder":
             assert isinstance(value, str)
             params.package_folder = tmp_path / value
-    reply_queue, replies, watcher_thread, context = reply_queue_fixture()
+    reply_queue, replies, watcher_thread, context = multiprocess_reply_queue_fixture()
     run_worker(worker, reply_queue, params, watcher_thread, context)
 
     assert replies

tests/workers/single_process/test_conan_cmake_helper.py

@@ -22,7 +22,7 @@
 import pytest
 
 if typing.TYPE_CHECKING:
-    from ttypes import RunWorkerFixture, SingleprocessReplyQueueFixture
+    from ttypes import MultiprocessReplyQueueFixture, RunWorkerFixture
 
 
 LOGGER = logging.getLogger(__name__)
@@ -44,7 +44,7 @@
 )
 # pylint: disable=too-many-arguments, too-many-positional-arguments  # noqa: E501
 def test_conan_cmake_helper(
-    reply_queue_fixture: SingleprocessReplyQueueFixture,
+    multiprocess_reply_queue_fixture: MultiprocessReplyQueueFixture,
     run_worker: RunWorkerFixture,
     conan_cmake_helper_recipe: pathlib.Path,
     conan_local_cache: typing.Dict[str, str],
@@ -65,7 +65,9 @@ def test_conan_cmake_helper(
         params.recipe_path = conan_cmake_helper_recipe
         params.cwd = conan_cmake_helper_recipe.parent
         params.profile = "default"
-        reply_queue, replies, watcher_thread, context = reply_queue_fixture()
+        reply_queue, replies, watcher_thread, context = (
+            multiprocess_reply_queue_fixture()
+        )
         run_worker(worker, reply_queue, params, watcher_thread, context)
 
     if env_key and env_value:
@@ -76,7 +78,7 @@ def test_conan_cmake_helper(
     params.added_environment = conan_local_cache
     params.recipe_path = conan_cmake_helper_recipe
     params.cwd = conan_cmake_helper_recipe.parent
-    reply_queue, replies, watcher_thread, context = reply_queue_fixture()
+    reply_queue, replies, watcher_thread, context = multiprocess_reply_queue_fixture()
     run_worker(worker, reply_queue, params, watcher_thread, context)
 
     assert replies