From f38601f989d58b2a193b6eff616f9e4eb7689b31 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 29 Jan 2026 13:55:19 -0800 Subject: [PATCH 01/14] try to log skipped tests --- .github/workflows/ci.yml | 34 +++++++++++ scripts/test_runner.py | 122 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100755 scripts/test_runner.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e319e063..19118d014 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,9 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: + - uses: actions/checkout@v4 + with: + submodules: true - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -68,6 +71,14 @@ jobs: run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + - name: Track skipped tests + run: | + if [ -d ".venv-builder" ]; then + source .venv-builder/bin/activate + python scripts/test_runner.py --top-level-directory . --verbose --start-directory test 2>/dev/null | grep -A 1000 "SKIPPED TESTS SUMMARY" || echo "No skipped tests found or test runner not available" + else + echo "Virtual environment not found, skipping skipped test tracking" + fi manylinux2014-arm64: runs-on: ubuntu-24.04-arm @@ -84,6 +95,9 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: + - uses: actions/checkout@v4 + with: + submodules: true - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -93,6 +107,14 @@ jobs: run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-aarch64 build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + - name: Track skipped tests + run: | + if [ -d ".venv-builder" ]; then + source .venv-builder/bin/activate + python scripts/test_runner.py --top-level-directory . --verbose --start-directory test 2>/dev/null | grep -A 1000 "SKIPPED TESTS SUMMARY" || echo "No skipped tests found or test runner not available" + else + echo "Virtual environment not found, skipping skipped test tracking" + fi musllinux-1-1: runs-on: ubuntu-24.04 # latest @@ -226,6 +248,9 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: + - uses: actions/checkout@v4 + with: + submodules: true - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -249,6 +274,15 @@ jobs: USES_LIBCRYPTO_SO=`echo "$LINKED_AGAINST" | grep 'libcrypto*.so' | head -1` test -n "$USES_LIBCRYPTO_SO" + - name: Track skipped tests + run: | + if [ -d "aws-crt-python/.venv-builder" ]; then + cd aws-crt-python + source .venv-builder/bin/activate + python ../scripts/test_runner.py --top-level-directory . --verbose --start-directory test 2>/dev/null | grep -A 1000 "SKIPPED TESTS SUMMARY" || echo "No skipped tests found or test runner not available" + else + echo "Virtual environment not found, skipping skipped test tracking" + fi windows: # Currently, setup.py explicitly tries to use Windows SDK 10.0.17763.0. diff --git a/scripts/test_runner.py b/scripts/test_runner.py new file mode 100755 index 000000000..39af8cb93 --- /dev/null +++ b/scripts/test_runner.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +""" +Custom test runner that tracks and reports skipped tests. +""" + +import sys +import unittest +import argparse +from io import StringIO + + +class SkippedTestTracker(unittest.TextTestResult): + """Custom test result that tracks skipped tests with their reasons.""" + + def __init__(self, stream, descriptions, verbosity): + super().__init__(stream, descriptions, verbosity) + self.skipped_tests = [] + + def addSkip(self, test, reason): + """Override to track skipped tests.""" + super().addSkip(test, reason) + self.skipped_tests.append({ + 'test': str(test), + 'reason': reason, + 'class': test.__class__.__name__, + 'method': test._testMethodName + }) + + +class SkippedTestRunner(unittest.TextTestRunner): + """Custom test runner that uses our SkippedTestTracker.""" + + def __init__(self, stream=None, descriptions=True, verbosity=1, + failfast=False, buffer=False, resultclass=None, warnings=None, *, tb_locals=False): + if resultclass is None: + resultclass = SkippedTestTracker + super().__init__(stream, descriptions, verbosity, failfast, buffer, + resultclass, warnings, tb_locals=tb_locals) + + def run(self, test): + """Run tests and report skipped tests at the end.""" + result = super().run(test) + + # Print summary of skipped tests + if hasattr(result, 'skipped_tests') and result.skipped_tests: + self.stream.write('\n' + '='*70 + '\n') + self.stream.write(f'SKIPPED TESTS SUMMARY ({len(result.skipped_tests)} total)\n') + self.stream.write('='*70 + '\n') + + # Group by class for better organization + by_class = {} + for skip in result.skipped_tests: + class_name = skip['class'] + if class_name not in by_class: + by_class[class_name] = [] + by_class[class_name].append(skip) + + for class_name, skips in sorted(by_class.items()): + self.stream.write(f'\n{class_name}:\n') + for skip in skips: + self.stream.write(f' - {skip["method"]}: {skip["reason"]}\n') + + self.stream.write('\n' + '='*70 + '\n') + else: + self.stream.write('\nNo tests were skipped.\n') + + return result + + +def main(): + """Main entry point for the custom test runner.""" + parser = argparse.ArgumentParser(description='Run tests with skipped test tracking') + parser.add_argument('--start-directory', '-s', default='test', + help='Directory to start discovery (default: test)') + parser.add_argument('--pattern', '-p', default='test*.py', + help='Pattern to match test files (default: test*.py)') + parser.add_argument('--top-level-directory', '-t', default=None, + help='Top level directory of project') + parser.add_argument('--verbose', '-v', action='count', default=1, + help='Verbose output (can be used multiple times)') + parser.add_argument('--failfast', '-f', action='store_true', + help='Stop on first failure') + parser.add_argument('--buffer', '-b', action='store_true', + help='Buffer stdout and stderr during tests') + + args = parser.parse_args() + + # Discover tests + loader = unittest.TestLoader() + + try: + suite = loader.discover( + start_dir=args.start_directory, + pattern=args.pattern, + top_level_dir=args.top_level_directory + ) + except ImportError as e: + print(f"Error discovering tests: {e}") + return 1 + + # Run tests with our custom runner + runner = SkippedTestRunner( + verbosity=args.verbose, + failfast=args.failfast, + buffer=args.buffer + ) + + result = runner.run(suite) + + # Return appropriate exit code + if result.wasSuccessful(): + return 0 + else: + return 1 + + +if __name__ == '__main__': + sys.exit(main()) From f4b92e2a2242dff242ceea8cc480ba0179521e77 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 29 Jan 2026 14:09:57 -0800 Subject: [PATCH 02/14] remove tracking --- .github/workflows/ci.yml | 34 ----------- scripts/test_runner.py | 122 --------------------------------------- 2 files changed, 156 deletions(-) delete mode 100755 scripts/test_runner.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19118d014..9e319e063 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,9 +59,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - - uses: actions/checkout@v4 - with: - submodules: true - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -71,14 +68,6 @@ jobs: run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python - - name: Track skipped tests - run: | - if [ -d ".venv-builder" ]; then - source .venv-builder/bin/activate - python scripts/test_runner.py --top-level-directory . --verbose --start-directory test 2>/dev/null | grep -A 1000 "SKIPPED TESTS SUMMARY" || echo "No skipped tests found or test runner not available" - else - echo "Virtual environment not found, skipping skipped test tracking" - fi manylinux2014-arm64: runs-on: ubuntu-24.04-arm @@ -95,9 +84,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - - uses: actions/checkout@v4 - with: - submodules: true - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -107,14 +93,6 @@ jobs: run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-aarch64 build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python - - name: Track skipped tests - run: | - if [ -d ".venv-builder" ]; then - source .venv-builder/bin/activate - python scripts/test_runner.py --top-level-directory . --verbose --start-directory test 2>/dev/null | grep -A 1000 "SKIPPED TESTS SUMMARY" || echo "No skipped tests found or test runner not available" - else - echo "Virtual environment not found, skipping skipped test tracking" - fi musllinux-1-1: runs-on: ubuntu-24.04 # latest @@ -248,9 +226,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - - uses: actions/checkout@v4 - with: - submodules: true - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -274,15 +249,6 @@ jobs: USES_LIBCRYPTO_SO=`echo "$LINKED_AGAINST" | grep 'libcrypto*.so' | head -1` test -n "$USES_LIBCRYPTO_SO" - - name: Track skipped tests - run: | - if [ -d "aws-crt-python/.venv-builder" ]; then - cd aws-crt-python - source .venv-builder/bin/activate - python ../scripts/test_runner.py --top-level-directory . --verbose --start-directory test 2>/dev/null | grep -A 1000 "SKIPPED TESTS SUMMARY" || echo "No skipped tests found or test runner not available" - else - echo "Virtual environment not found, skipping skipped test tracking" - fi windows: # Currently, setup.py explicitly tries to use Windows SDK 10.0.17763.0. diff --git a/scripts/test_runner.py b/scripts/test_runner.py deleted file mode 100755 index 39af8cb93..000000000 --- a/scripts/test_runner.py +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0. - -""" -Custom test runner that tracks and reports skipped tests. -""" - -import sys -import unittest -import argparse -from io import StringIO - - -class SkippedTestTracker(unittest.TextTestResult): - """Custom test result that tracks skipped tests with their reasons.""" - - def __init__(self, stream, descriptions, verbosity): - super().__init__(stream, descriptions, verbosity) - self.skipped_tests = [] - - def addSkip(self, test, reason): - """Override to track skipped tests.""" - super().addSkip(test, reason) - self.skipped_tests.append({ - 'test': str(test), - 'reason': reason, - 'class': test.__class__.__name__, - 'method': test._testMethodName - }) - - -class SkippedTestRunner(unittest.TextTestRunner): - """Custom test runner that uses our SkippedTestTracker.""" - - def __init__(self, stream=None, descriptions=True, verbosity=1, - failfast=False, buffer=False, resultclass=None, warnings=None, *, tb_locals=False): - if resultclass is None: - resultclass = SkippedTestTracker - super().__init__(stream, descriptions, verbosity, failfast, buffer, - resultclass, warnings, tb_locals=tb_locals) - - def run(self, test): - """Run tests and report skipped tests at the end.""" - result = super().run(test) - - # Print summary of skipped tests - if hasattr(result, 'skipped_tests') and result.skipped_tests: - self.stream.write('\n' + '='*70 + '\n') - self.stream.write(f'SKIPPED TESTS SUMMARY ({len(result.skipped_tests)} total)\n') - self.stream.write('='*70 + '\n') - - # Group by class for better organization - by_class = {} - for skip in result.skipped_tests: - class_name = skip['class'] - if class_name not in by_class: - by_class[class_name] = [] - by_class[class_name].append(skip) - - for class_name, skips in sorted(by_class.items()): - self.stream.write(f'\n{class_name}:\n') - for skip in skips: - self.stream.write(f' - {skip["method"]}: {skip["reason"]}\n') - - self.stream.write('\n' + '='*70 + '\n') - else: - self.stream.write('\nNo tests were skipped.\n') - - return result - - -def main(): - """Main entry point for the custom test runner.""" - parser = argparse.ArgumentParser(description='Run tests with skipped test tracking') - parser.add_argument('--start-directory', '-s', default='test', - help='Directory to start discovery (default: test)') - parser.add_argument('--pattern', '-p', default='test*.py', - help='Pattern to match test files (default: test*.py)') - parser.add_argument('--top-level-directory', '-t', default=None, - help='Top level directory of project') - parser.add_argument('--verbose', '-v', action='count', default=1, - help='Verbose output (can be used multiple times)') - parser.add_argument('--failfast', '-f', action='store_true', - help='Stop on first failure') - parser.add_argument('--buffer', '-b', action='store_true', - help='Buffer stdout and stderr during tests') - - args = parser.parse_args() - - # Discover tests - loader = unittest.TestLoader() - - try: - suite = loader.discover( - start_dir=args.start_directory, - pattern=args.pattern, - top_level_dir=args.top_level_directory - ) - except ImportError as e: - print(f"Error discovering tests: {e}") - return 1 - - # Run tests with our custom runner - runner = SkippedTestRunner( - verbosity=args.verbose, - failfast=args.failfast, - buffer=args.buffer - ) - - result = runner.run(suite) - - # Return appropriate exit code - if result.wasSuccessful(): - return 0 - else: - return 1 - - -if __name__ == '__main__': - sys.exit(main()) From 8880193310842032cb76238846d0aa9454993f1f Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Thu, 29 Jan 2026 14:14:15 -0800 Subject: [PATCH 03/14] double_client_id_failure test --- test/test_mqtt5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index 83ee5928a..5022cd4c9 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -628,8 +628,8 @@ def test_connect_with_incorrect_basic_authentication_credentials(self): # test_websocket_handshake_failure : tested in the SDK def test_double_client_id_failure(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_HOST") - input_port = int(_get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_PORT")) + input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") + input_port = int(8883) shared_client_id = create_client_id() connect_options = mqtt5.ConnectPacket(client_id=shared_client_id) From d6788c1a503974a431bf54d3ba131c00807acba1 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 30 Jan 2026 11:31:41 -0800 Subject: [PATCH 04/14] cert and key added to test --- test/test_mqtt5.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index 5022cd4c9..9ca32f7e7 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -629,6 +629,8 @@ def test_connect_with_incorrect_basic_authentication_credentials(self): def test_double_client_id_failure(self): input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") + input_cert = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_RSA_CERT") + input_key = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_RSA_KEY") input_port = int(8883) shared_client_id = create_client_id() @@ -638,6 +640,11 @@ def test_double_client_id_failure(self): port=input_port, connect_options=connect_options ) + tls_ctx_options = io.TlsContextOptions.create_client_with_mtls_from_path( + input_cert, + input_key + ) + client_options.tls_ctx = io.ClientTlsContext(tls_ctx_options) callbacks = Mqtt5TestCallbacks() client1 = self._create_client(client_options=client_options, callbacks=callbacks) From 87aa1fcff8576e4a19c6d51046ade41badcef246 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 30 Jan 2026 12:12:58 -0800 Subject: [PATCH 05/14] expand client creation for test --- test/test_mqtt5.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index 9ca32f7e7..3d5c1d5e3 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -109,17 +109,30 @@ def on_lifecycle_disconnection(self, lifecycle_disconnect_data: mqtt5.LifecycleD class Mqtt5ClientTest(NativeResourceTest): + def _create_tls_context(): + cert = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_RSA_CERT") + key = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_RSA_KEY") + return io.ClientTlsContext( + io.TlsContextOptions.create_client_with_mtls_from_path(cert, key)) + def _create_client( self, client_options: mqtt5.ClientOptions = None, callbacks: Mqtt5TestCallbacks = None): default_host = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") + if client_options is None: client_options = mqtt5.ClientOptions( host_name=default_host, - port=8883 - ) + port=8883, + tls_ctx = _create_tls_context()) + + if (client_options.host_name == _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") and + client_options.tls_ctx is None): + client_options.port = 8883 + client_options.tls_ctx = _create_tls_context() + if client_options.connect_options is None: client_options.connect_options = mqtt5.ConnectPacket() client_options.connect_options.client_id = create_client_id() @@ -629,22 +642,13 @@ def test_connect_with_incorrect_basic_authentication_credentials(self): def test_double_client_id_failure(self): input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") - input_cert = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_RSA_CERT") - input_key = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_RSA_KEY") - input_port = int(8883) shared_client_id = create_client_id() connect_options = mqtt5.ConnectPacket(client_id=shared_client_id) client_options = mqtt5.ClientOptions( host_name=input_host_name, - port=input_port, connect_options=connect_options ) - tls_ctx_options = io.TlsContextOptions.create_client_with_mtls_from_path( - input_cert, - input_key - ) - client_options.tls_ctx = io.ClientTlsContext(tls_ctx_options) callbacks = Mqtt5TestCallbacks() client1 = self._create_client(client_options=client_options, callbacks=callbacks) From 817b06506d8c84bd5c2e22552842698fb59ee4a5 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 30 Jan 2026 12:20:11 -0800 Subject: [PATCH 06/14] self --- test/test_mqtt5.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index 3d5c1d5e3..147a4ca05 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -109,7 +109,7 @@ def on_lifecycle_disconnection(self, lifecycle_disconnect_data: mqtt5.LifecycleD class Mqtt5ClientTest(NativeResourceTest): - def _create_tls_context(): + def _create_tls_context(self): cert = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_RSA_CERT") key = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_RSA_KEY") return io.ClientTlsContext( @@ -126,12 +126,12 @@ def _create_client( client_options = mqtt5.ClientOptions( host_name=default_host, port=8883, - tls_ctx = _create_tls_context()) + tls_ctx = self._create_tls_context()) if (client_options.host_name == _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") and client_options.tls_ctx is None): client_options.port = 8883 - client_options.tls_ctx = _create_tls_context() + client_options.tls_ctx = self._create_tls_context() if client_options.connect_options is None: client_options.connect_options = mqtt5.ConnectPacket() From 30ccbbb8876527cb2b85153563355b85fa6e1fc9 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 30 Jan 2026 12:52:00 -0800 Subject: [PATCH 07/14] invalid port test --- test/test_mqtt5.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index 147a4ca05..946ea3843 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -121,17 +121,19 @@ def _create_client( callbacks: Mqtt5TestCallbacks = None): default_host = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") - + if client_options is None: client_options = mqtt5.ClientOptions( host_name=default_host, port=8883, - tls_ctx = self._create_tls_context()) - + tls_ctx = self._create_tls_context()) + if (client_options.host_name == _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") and - client_options.tls_ctx is None): - client_options.port = 8883 + client_options.tls_ctx is None): client_options.tls_ctx = self._create_tls_context() + + if client_options.port is None: + client_options.port = 8883 if client_options.connect_options is None: client_options.connect_options = mqtt5.ConnectPacket() @@ -577,7 +579,7 @@ def test_connect_with_invalid_host_name(self): callbacks.future_stopped.result(TIMEOUT) def test_connect_with_invalid_port(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_HOST") + input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") client_options = mqtt5.ClientOptions( host_name=input_host_name, port=444 From 99e9b903c84f5064a58aba212ee5bcc5da6aca63 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 30 Jan 2026 12:58:26 -0800 Subject: [PATCH 08/14] enable remaining skipped tests --- test/test_mqtt5.py | 41 +++++++++++++---------------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index 946ea3843..3a94a1c96 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -213,12 +213,10 @@ def test_client_creation_maximum(self): # ============================================================== def _test_direct_connect_minimum(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_HOST") - input_port = int(_get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_PORT")) + input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") client_options = mqtt5.ClientOptions( - host_name=input_host_name, - port=input_port + host_name=input_host_name ) callbacks = Mqtt5TestCallbacks() client = self._create_client(client_options=client_options, callbacks=callbacks) @@ -336,8 +334,7 @@ def test_direct_connect_http_proxy_tls(self): test_retry_wrapper(self._test_direct_connect_http_proxy_tls) def _test_direct_connect_maximum(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_HOST") - input_port = int(_get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_PORT")) + input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") user_properties = [] user_properties.append(mqtt5.UserProperty(name="name1", value="value1")) @@ -592,7 +589,7 @@ def test_connect_with_invalid_port(self): callbacks.future_stopped.result(TIMEOUT) def test_connect_with_invalid_port_for_websocket_connection(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT5_WS_MQTT_HOST") + input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") client_options = mqtt5.ClientOptions( host_name=input_host_name, port=1883 @@ -906,15 +903,13 @@ def test_negative_subscribe_packet_properties(self): # ============================================================== def _test_negotiated_settings_minimal_settings(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_HOST") - input_port = int(_get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_PORT")) + input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") connect_options = mqtt5.ConnectPacket( session_expiry_interval_sec=600000 ) client_options = mqtt5.ClientOptions( host_name=input_host_name, - port=input_port, connect_options=connect_options ) callbacks = Mqtt5TestCallbacks() @@ -932,8 +927,7 @@ def test_negotiated_settings_minimal_settings(self): test_retry_wrapper(self._test_negotiated_settings_minimal_settings) def _test_negotiated_settings_maximum_settings(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_HOST") - input_port = int(_get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_PORT")) + input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") client_id = create_client_id() connect_options = mqtt5.ConnectPacket( @@ -944,7 +938,6 @@ def _test_negotiated_settings_maximum_settings(self): client_options = mqtt5.ClientOptions( host_name=input_host_name, - port=input_port, connect_options=connect_options ) callbacks = Mqtt5TestCallbacks() @@ -976,8 +969,7 @@ def test_negotiated_settings_maximum_settings(self): test_retry_wrapper(self._test_negotiated_settings_maximum_settings) def _test_negotiated_settings_server_limit(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_HOST") - input_port = int(_get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_PORT")) + input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") uint32_max = 4294967295 uint16_max = 65535 @@ -991,7 +983,6 @@ def _test_negotiated_settings_server_limit(self): client_options = mqtt5.ClientOptions( host_name=input_host_name, - port=input_port, connect_options=connect_options ) callbacks = Mqtt5TestCallbacks() @@ -1413,12 +1404,10 @@ def test_operation_publish_correlation_data_bytes_binary_precedence(self): # ============================================================== def _test_operation_error_null_publish(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_HOST") - input_port = int(_get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_PORT")) + input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") client_options = mqtt5.ClientOptions( - host_name=input_host_name, - port=input_port + host_name=input_host_name ) callbacks = Mqtt5TestCallbacks() client = self._create_client(client_options=client_options, callbacks=callbacks) @@ -1435,12 +1424,10 @@ def test_operation_error_null_publish(self): test_retry_wrapper(self._test_operation_error_null_publish) def _test_operation_error_null_subscribe(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_HOST") - input_port = int(_get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_PORT")) + input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") client_options = mqtt5.ClientOptions( - host_name=input_host_name, - port=input_port + host_name=input_host_name ) callbacks = Mqtt5TestCallbacks() client = self._create_client(client_options=client_options, callbacks=callbacks) @@ -1457,12 +1444,10 @@ def test_operation_error_null_subscribe(self): test_retry_wrapper(self._test_operation_error_null_subscribe) def _test_operation_error_null_unsubscribe(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_HOST") - input_port = int(_get_env_variable("AWS_TEST_MQTT5_DIRECT_MQTT_PORT")) + input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") client_options = mqtt5.ClientOptions( - host_name=input_host_name, - port=input_port + host_name=input_host_name ) callbacks = Mqtt5TestCallbacks() client = self._create_client(client_options=client_options, callbacks=callbacks) From f78b89c2a243cafa456f5b673f7763c399b974eb Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 30 Jan 2026 13:00:52 -0800 Subject: [PATCH 09/14] lint --- test/test_mqtt5.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index 3a94a1c96..c1a1a5bb7 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -126,14 +126,14 @@ def _create_client( client_options = mqtt5.ClientOptions( host_name=default_host, port=8883, - tls_ctx = self._create_tls_context()) + tls_ctx = self._create_tls_context()) - if (client_options.host_name == _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") and - client_options.tls_ctx is None): + if (client_options.host_name == _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") and + client_options.tls_ctx is None): client_options.tls_ctx = self._create_tls_context() - + if client_options.port is None: - client_options.port = 8883 + client_options.port = 8883 if client_options.connect_options is None: client_options.connect_options = mqtt5.ConnectPacket() From 49ee929c9342f01f558c3598a58bcb181308c1b2 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 30 Jan 2026 13:02:22 -0800 Subject: [PATCH 10/14] omg lint --- test/test_mqtt5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index c1a1a5bb7..08ce2e652 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -126,7 +126,7 @@ def _create_client( client_options = mqtt5.ClientOptions( host_name=default_host, port=8883, - tls_ctx = self._create_tls_context()) + tls_ctx=self._create_tls_context()) if (client_options.host_name == _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") and client_options.tls_ctx is None): From 595e95e790dda90713e8001e88f72a6be6b4faa4 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 30 Jan 2026 13:10:28 -0800 Subject: [PATCH 11/14] port --- test/test_mqtt5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index 08ce2e652..1b19d9f9a 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -367,7 +367,7 @@ def _test_direct_connect_maximum(self): ) client_options = mqtt5.ClientOptions( host_name=input_host_name, - port=input_port, + port=8883, connect_options=connect_options, session_behavior=mqtt5.ClientSessionBehaviorType.CLEAN, extended_validation_and_flow_control_options=mqtt5.ExtendedValidationAndFlowControlOptions.AWS_IOT_CORE_DEFAULTS, From d6b6ba79f93b401431097203ea3977c75c536a44 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 30 Jan 2026 13:32:58 -0800 Subject: [PATCH 12/14] negotiated settings results should be checked against expected IoT Core output now --- test/test_mqtt5.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index 1b19d9f9a..a5dc7749d 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -951,13 +951,13 @@ def _test_negotiated_settings_maximum_settings(self): self.assertEqual(callbacks.negotiated_settings.session_expiry_interval_sec, 600) self.assertEqual(callbacks.negotiated_settings.server_keep_alive_sec, 1000) self.assertEqual(callbacks.negotiated_settings.maximum_qos, mqtt5.QoS.AT_LEAST_ONCE) - self.assertEqual(callbacks.negotiated_settings.receive_maximum_from_server, 10) - self.assertEqual(callbacks.negotiated_settings.maximum_packet_size_to_server, 268435460) - self.assertEqual(callbacks.negotiated_settings.topic_alias_maximum_to_server, 10) + self.assertEqual(callbacks.negotiated_settings.receive_maximum_from_server, 100) + self.assertEqual(callbacks.negotiated_settings.maximum_packet_size_to_server, 149504) + self.assertEqual(callbacks.negotiated_settings.topic_alias_maximum_to_server, 8) self.assertEqual(callbacks.negotiated_settings.topic_alias_maximum_to_client, 0) self.assertTrue(callbacks.negotiated_settings.retain_available) self.assertTrue(callbacks.negotiated_settings.wildcard_subscriptions_available) - self.assertTrue(callbacks.negotiated_settings.subscription_identifiers_available) + self.assertFalse(callbacks.negotiated_settings.subscription_identifiers_available) self.assertTrue(callbacks.negotiated_settings.shared_subscriptions_available) self.assertFalse(callbacks.negotiated_settings.rejoined_session) self.assertEqual(callbacks.negotiated_settings.client_id, client_id) From 4d20c5aec39232bc9ced1562a5552ba893ae51e5 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 30 Jan 2026 13:42:41 -0800 Subject: [PATCH 13/14] minimal test also needs to use AWS IoT Core defaults --- test/test_mqtt5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index a5dc7749d..bd2ab658b 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -918,7 +918,7 @@ def _test_negotiated_settings_minimal_settings(self): callbacks.future_connection_success.result(TIMEOUT) self.assertIsNotNone(callbacks.negotiated_settings) - self.assertEqual(callbacks.negotiated_settings.session_expiry_interval_sec, 600000) + self.assertEqual(callbacks.negotiated_settings.session_expiry_interval_sec, 3600) client.stop() callbacks.future_stopped.result(TIMEOUT) From 17bf443503b60958b597ebd3e80074478f393f71 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Fri, 30 Jan 2026 13:49:13 -0800 Subject: [PATCH 14/14] again, IoT Core limits on checks --- test/test_mqtt5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_mqtt5.py b/test/test_mqtt5.py index bd2ab658b..4d8f7d0e2 100644 --- a/test/test_mqtt5.py +++ b/test/test_mqtt5.py @@ -993,8 +993,8 @@ def _test_negotiated_settings_server_limit(self): self.assertIsNotNone(callbacks.negotiated_settings) self.assertNotEqual(callbacks.negotiated_settings.receive_maximum_from_server, uint16_max) self.assertNotEqual(callbacks.negotiated_settings.maximum_packet_size_to_server, uint32_max) - self.assertEqual(callbacks.negotiated_settings.server_keep_alive_sec, uint16_max) - self.assertEqual(callbacks.negotiated_settings.session_expiry_interval_sec, uint32_max) + self.assertEqual(callbacks.negotiated_settings.server_keep_alive_sec, 1200) + self.assertEqual(callbacks.negotiated_settings.session_expiry_interval_sec, 3600) client.stop() callbacks.future_stopped.result(TIMEOUT)