Added trace logging support

pyproject.toml

@@ -26,14 +26,14 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
 dependencies = [
-#    "open-mpic-core @ git+https://github.com/open-mpic/open-mpic-core-python.git@main",
+    #"open-mpic-core @ git+https://github.com/open-mpic/open-mpic-core-python.git@ds-trace-logging",
     "pyyaml==6.0.1",
     "requests>=2.32.3",
     "dnspython==2.6.1",
     "pydantic==2.8.2",
     "aiohttp==3.11.11",
     "aws-lambda-powertools[parser]==3.2.0",
-    "open-mpic-core==4.4.0",
+    "open-mpic-core==4.5.0",
     "aioboto3~=13.3.0",
 ]
 

src/aws_lambda_mpic/mpic_caa_checker_lambda/mpic_caa_checker_lambda_function.py

@@ -1,17 +1,28 @@
+import os
 import asyncio
 
 from aws_lambda_powertools.utilities.parser import event_parser
 
 from open_mpic_core.common_domain.check_request import CaaCheckRequest
 from open_mpic_core.mpic_caa_checker.mpic_caa_checker import MpicCaaChecker
-import os
+from open_mpic_core.common_util.trace_level_logger import get_logger
+
+logger = get_logger(__name__)
 
 
 class MpicCaaCheckerLambdaHandler:
     def __init__(self):
         self.perspective_code = os.environ['AWS_REGION']
         self.default_caa_domain_list = os.environ['default_caa_domains'].split("|")
-        self.caa_checker = MpicCaaChecker(self.default_caa_domain_list, self.perspective_code)
+        self.log_level = os.environ['log_level'] if 'log_level' in os.environ else None
+
+        self.logger = logger.getChild(self.__class__.__name__)
+        if self.log_level:
+            self.logger.setLevel(self.log_level)
+
+        self.caa_checker = MpicCaaChecker(default_caa_domain_list=self.default_caa_domain_list,
+                                          perspective_code=self.perspective_code,
+                                          log_level=self.logger.level)
 
     def process_invocation(self, caa_request: CaaCheckRequest):
         try:

src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py

@@ -1,15 +1,18 @@
+import logging
 import os
 import json
+import traceback
+
 import yaml
 import asyncio
 import aioboto3
 
 from asyncio import Queue
 from collections import defaultdict
 from importlib import resources
-
-from aws_lambda_powertools.utilities.parser import event_parser, envelopes
 from pydantic import TypeAdapter, ValidationError, BaseModel
+from aws_lambda_powertools.utilities.parser import event_parser, envelopes
+
 from open_mpic_core.common_domain.check_request import BaseCheckRequest
 from open_mpic_core.common_domain.check_response import CheckResponse
 from open_mpic_core.mpic_coordinator.domain.mpic_request import MpicRequest
@@ -18,6 +21,9 @@
 from open_mpic_core.mpic_coordinator.mpic_coordinator import MpicCoordinator, MpicCoordinatorConfiguration
 from open_mpic_core.common_domain.enum.check_type import CheckType
 from open_mpic_core.mpic_coordinator.domain.remote_perspective import RemotePerspective
+from open_mpic_core.common_util.trace_level_logger import get_logger
+
+logger = get_logger(__name__)
 
 
 class PerspectiveEndpointInfo(BaseModel):
@@ -37,6 +43,11 @@ def __init__(self):
         self.default_perspective_count = int(os.environ['default_perspective_count'])
         self.global_max_attempts = int(os.environ['absolute_max_attempts']) if 'absolute_max_attempts' in os.environ else None
         self.hash_secret = os.environ['hash_secret']
+        self.log_level = os.environ['log_level'] if 'log_level' in os.environ else None
+
+        self.logger = logger.getChild(self.__class__.__name__)
+        if self.log_level:
+            self.logger.setLevel(self.log_level)
 
         self.remotes_per_perspective_per_check_type = {
             CheckType.DCV: {perspective_code: perspective_config.dcv_endpoint_info for perspective_code, perspective_config in perspectives.items()},
@@ -54,10 +65,7 @@ def __init__(self):
             self.hash_secret
         )
 
-        self.mpic_coordinator = MpicCoordinator(
-            self.call_remote_perspective,
-            self.mpic_coordinator_configuration
-        )
+        self.mpic_coordinator = MpicCoordinator(self.call_remote_perspective, self.mpic_coordinator_configuration, self.logger.level)
 
         # for correct deserialization of responses based on discriminator field (check type)
         self.mpic_request_adapter = TypeAdapter(MpicRequest)
@@ -125,7 +133,7 @@ async def call_remote_perspective(self, perspective: RemotePerspective, check_ty
             response_payload = json.loads(await response['Payload'].read())
             return self.check_response_adapter.validate_json(response_payload['body'])
         except ValidationError as ve:
-            # We might want to handle this differently later.
+            self.logger.log(level=logging.ERROR, msg=f"Validation error in response from {perspective.code}: {ve}")
             raise ve
         finally:
             await self.release_lambda_client(perspective.code, client)
@@ -184,6 +192,9 @@ def wrapper(*args, **kwargs):
         except ValidationError as validation_error:
             return build_400_response(MpicRequestValidationMessages.REQUEST_VALIDATION_FAILED.key, validation_error.errors())
         except Exception as e:
+            logger.error(f"An error occurred: {str(e)}")
+            print(traceback.format_exc())
+            print(f"BOY HOWDY error occurred: {str(e)}")
             return {
                 'statusCode': 500,
                 'headers': {'Content-Type': 'application/json'},

src/aws_lambda_mpic/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda_function.py

@@ -1,16 +1,25 @@
+import os
 import asyncio
 
 from aws_lambda_powertools.utilities.parser import event_parser
 
 from open_mpic_core.common_domain.check_request import DcvCheckRequest
 from open_mpic_core.mpic_dcv_checker.mpic_dcv_checker import MpicDcvChecker
-import os
+from open_mpic_core.common_util.trace_level_logger import get_logger
+
+logger = get_logger(__name__)
 
 
 class MpicDcvCheckerLambdaHandler:
     def __init__(self):
         self.perspective_code = os.environ['AWS_REGION']
-        self.dcv_checker = MpicDcvChecker(self.perspective_code)
+        self.log_level = os.environ['log_level'] if 'log_level' in os.environ else None
+
+        self.logger = logger.getChild(self.__class__.__name__)
+        if self.log_level:
+            self.logger.setLevel(self.log_level)
+
+        self.dcv_checker = MpicDcvChecker(perspective_code=self.perspective_code, log_level=self.logger.level)
 
     def process_invocation(self, dcv_request: DcvCheckRequest):
         try:

tests/integration/test_deployed_mpic_api.py

@@ -1,6 +1,7 @@
 import json
 import sys
 import pytest
+
 from pydantic import TypeAdapter
 
 from open_mpic_core.common_domain.check_parameters import CaaCheckParameters, DcvWebsiteChangeValidationDetails

tests/unit/aws_lambda_mpic/conftest.py

@@ -0,0 +1,21 @@
+# conftest.py
+import logging
+from io import StringIO
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def setup_logging():
+    # Clear existing handlers
+    root = logging.getLogger()
+    for handler in root.handlers[:]:
+        root.removeHandler(handler)
+
+    log_output = StringIO()  # to be able to inspect what gets logged
+    handler = logging.StreamHandler(log_output)
+    handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
+
+    # Configure fresh logging
+    logging.basicConfig(handlers=[handler])
+
+    yield log_output

tests/unit/aws_lambda_mpic/test_caa_checker_lambda.py

@@ -1,19 +1,23 @@
 import time
 
+import dns
 import pytest
 
 import aws_lambda_mpic.mpic_caa_checker_lambda.mpic_caa_checker_lambda_function as mpic_caa_checker_lambda_function
 from open_mpic_core.common_domain.check_response import CaaCheckResponse, CaaCheckResponseDetails
+from open_mpic_core_test.test_util.mock_dns_object_creator import MockDnsObjectCreator
 from open_mpic_core_test.test_util.valid_check_creator import ValidCheckCreator
 
 
+# noinspection PyMethodMayBeStatic
 class TestCaaCheckerLambda:
     @staticmethod
     @pytest.fixture(scope='class')
     def set_env_variables():
         envvars = {
             'AWS_REGION': 'us-east-1',
-            'default_caa_domains': 'ca1.com|ca2.org|ca3.net'
+            'default_caa_domains': 'ca1.com|ca2.org|ca3.net',
+            'log_level': 'TRACE'
         }
         with pytest.MonkeyPatch.context() as class_scoped_monkeypatch:
             for k, v in envvars.items():
@@ -33,6 +37,21 @@ def lambda_handler__should_do_caa_check_using_configured_caa_checker(self, set_e
         result = mpic_caa_checker_lambda_function.lambda_handler(caa_check_request, None)
         assert result == mock_return_value
 
+    def lambda_handler__should_set_log_level_of_caa_checker(self, set_env_variables, setup_logging, mocker):
+        caa_check_request = ValidCheckCreator.create_valid_caa_check_request()
+
+        records = [MockDnsObjectCreator.create_caa_record(0, 'issue', 'ca1.org')]
+        mock_rrset = MockDnsObjectCreator.create_rrset(dns.rdatatype.CAA, *records)
+        mock_domain = dns.name.from_text(caa_check_request.domain_or_ip_target)
+        mock_return = (mock_rrset, mock_domain)
+        mocker.patch('open_mpic_core.mpic_caa_checker.mpic_caa_checker.MpicCaaChecker.find_caa_records_and_domain',
+                     return_value=mock_return)
+
+        result = mpic_caa_checker_lambda_function.lambda_handler(caa_check_request, None)
+        assert result['statusCode'] == 200
+        log_contents = setup_logging.getvalue()
+        assert all(text in log_contents for text in ['MpicCaaChecker', 'TRACE'])  # Verify the log level was set
+
     @staticmethod
     def create_caa_check_response():
         return CaaCheckResponse(perspective_code='us-east-1', check_passed=True,

tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py

@@ -1,4 +1,5 @@
 import time
+
 import pytest
 
 import aws_lambda_mpic.mpic_dcv_checker_lambda.mpic_dcv_checker_lambda_function as mpic_dcv_checker_lambda_function
@@ -8,13 +9,17 @@
 from open_mpic_core.common_domain.check_response import DcvCheckResponse
 from open_mpic_core_test.test_util.valid_check_creator import ValidCheckCreator
 
+from unit.aws_lambda_mpic.conftest import setup_logging
+
 
+# noinspection PyMethodMayBeStatic
 class TestDcvCheckerLambda:
     @staticmethod
     @pytest.fixture(scope='class')
     def set_env_variables():
         envvars = {
             'AWS_REGION': 'us-east-1',
+            'log_level': 'TRACE'
         }
         with pytest.MonkeyPatch.context() as class_scoped_monkeypatch:
             for k, v in envvars.items():
@@ -53,6 +58,15 @@ def lambda_handler__should_return_appropriate_status_code_given_errors_in_respon
         result = mpic_dcv_checker_lambda_function.lambda_handler(dcv_check_request, None)
         assert result == mock_return_value
 
+    def lambda_handler__should_set_log_level_of_dcv_checker(self, set_env_variables, mocker, setup_logging):
+        dcv_check_request = ValidCheckCreator.create_valid_http_check_request()
+        mocker.patch('open_mpic_core.mpic_dcv_checker.mpic_dcv_checker.MpicDcvChecker.perform_http_based_validation',
+                     return_value=TestDcvCheckerLambda.create_dcv_check_response())
+        result = mpic_dcv_checker_lambda_function.lambda_handler(dcv_check_request, None)
+        assert result['statusCode'] == 200
+        log_contents = setup_logging.getvalue()
+        assert all(text in log_contents for text in ['MpicDcvChecker', 'TRACE'])  # Verify the log level was set
+
     @staticmethod
     def create_dcv_check_response():
         return DcvCheckResponse(perspective_code='us-east-1', check_passed=True,

tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py

@@ -12,9 +12,9 @@
 from pydantic import TypeAdapter
 
 from open_mpic_core.common_domain.check_request import DcvCheckRequest
-from open_mpic_core.common_domain.check_response import DcvCheckResponse
+from open_mpic_core.common_domain.check_response import DcvCheckResponse, CaaCheckResponse
 from open_mpic_core.common_domain.enum.check_type import CheckType
-from open_mpic_core.common_domain.check_response_details import DcvDnsCheckResponseDetails
+from open_mpic_core.common_domain.check_response_details import DcvDnsCheckResponseDetails, CaaCheckResponseDetails
 from open_mpic_core.common_domain.enum.dcv_validation_method import DcvValidationMethod
 from open_mpic_core.mpic_coordinator.domain.mpic_orchestration_parameters import MpicEffectiveOrchestrationParameters
 from open_mpic_core.mpic_coordinator.domain.mpic_response import MpicCaaResponse
@@ -49,7 +49,8 @@ def set_env_variables():
         envvars = {
             'perspectives': json.dumps({k: v.model_dump() for k, v in perspectives_as_dict.items()}),
             'default_perspective_count': '3',
-            'hash_secret': 'test_secret'
+            'hash_secret': 'test_secret',
+            'log_level': 'TRACE'
         }
         with pytest.MonkeyPatch.context() as class_scoped_monkeypatch:
             for k, v in envvars.items():
@@ -153,6 +154,28 @@ def lambda_handler__should_coordinate_mpic_using_configured_mpic_coordinator(sel
         result = mpic_coordinator_lambda_function.lambda_handler(api_request, None)
         assert result == expected_response
 
+    def lambda_handler__should_set_log_level_for_coordinator(self, set_env_variables, setup_logging, mocker):
+        mpic_request = ValidMpicRequestCreator.create_valid_mpic_request(CheckType.CAA)
+        api_request = TestMpicCoordinatorLambda.create_api_gateway_request()
+        api_request.body = mpic_request.model_dump_json()
+        mocked_perspective_responses = [
+            CaaCheckResponse(perspective_code='us-east-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)),
+            CaaCheckResponse(perspective_code='us-west-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)),
+            CaaCheckResponse(perspective_code='eu-west-2', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)),
+            CaaCheckResponse(perspective_code='eu-central-2', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)),
+            CaaCheckResponse(perspective_code='ap-northeast-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)),
+            CaaCheckResponse(perspective_code='ap-south-2', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)),
+        ]
+        mocked_validity_per_perspective = {response.perspective_code: response.check_passed for response in mocked_perspective_responses}
+        mock_return = (mocked_perspective_responses, mocked_validity_per_perspective)
+
+        mocker.patch('open_mpic_core.mpic_coordinator.mpic_coordinator.MpicCoordinator.issue_async_calls_and_collect_responses', return_value=mock_return)
+        # noinspection PyTypeChecker
+        result = mpic_coordinator_lambda_function.lambda_handler(api_request, None)
+        assert result['statusCode'] == 200
+        log_contents = setup_logging.getvalue()
+        assert all(text in log_contents for text in ['MpicCoordinator', 'TRACE'])  # Verify the log level was set
+
     def load_aws_region_config__should_return_dict_of_aws_regions_with_proximity_info_by_region_code(self):
         mpic_coordinator_lambda_handler = MpicCoordinatorLambdaHandler()
         loaded_aws_regions = mpic_coordinator_lambda_handler.load_aws_region_config()
@@ -213,6 +236,14 @@ def create_caa_mpic_response():
             caa_check_parameters=caa_request.caa_check_parameters
         )
 
+    @staticmethod
+    def create_caa_perspective_response(*args, **kwargs) -> CaaCheckResponse:
+        return CaaCheckResponse(
+            perspective_code=kwargs['perspective'].code,
+            check_passed=True,
+            details=CaaCheckResponseDetails(caa_record_present=False),
+        )
+
     @staticmethod
     def create_api_gateway_request():
         request = APIGatewayProxyEventModel(