diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index a7275d1..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "open_mpic_core_python"] - path = open_mpic_core_python - url = https://github.com/open-mpic/open-mpic-core-python.git diff --git a/README.md b/README.md index 088159c..15a7c30 100644 --- a/README.md +++ b/README.md @@ -13,15 +13,16 @@ All requirements for running the API are packaged and uploaded to AWS as a lambd - Hatch (https://hatch.pypa.io/) for building and running the project. This is a Python project manager that can be installed via `pip install hatch`. ## Deployment Steps -1. Init and update the git submodule using: `git submodule init` followed by `git submodule update` in the root of the project. -2. Install layer dependencies. cd to the `layer` directory. Run `./1-install.sh` to create a virtual Python environment and install the project dependencies via pip. -3. Package the AWS layer. In the `layer` directory, run `./2-package.sh`. This will make a file called `layer_content.zip` which will later be referenced by Open Tofu. -4. Zip all functions. AWS Lambda functions are usually deployed from zip files. cd to the main project directory and then run `./zip-all.sh` -5. Create `config.yaml` in the root directory of the repo to contain the proper values needed for the deployment. A default config.yaml for a 6-perspective deployment with the controller in us-east-2 is included in this repo as `config.example.yaml`. This config can be made the active config by running `cp config.example.yaml config.yaml` in the root directory. -6. Run `hatch run ./configure.py` from the root directory of the repo to generate Open Tofu files from templates. -7. Deploy the entire package with Open Tofu. cd to the `open-tofu` directory where .tf files are located. Then run `tofu init`. Then run `tofu apply` and type `yes` at the confirmation prompt. -8. Get the URL of the deployed API endpoint by running `hatch run ./get_api_url.py` in the root directory. -9. Get the API Key generated by AWS by running `hatch run ./get_api_key.py` in the root directory. The deployment is configured to reject any API call that does not have this key passed via the `x-api-key` HTTP header. +1. From the project root directory, run `hatch run lambda-layer:install`. This will create a virtual Python environment in the `layer` directory and install the project dependencies via pip. +2. Package the AWS layer. In the `layer` directory, run `./package.sh`. This will make two files: `python311_layer_content.zip` and `mpic_coordinator_layer_content.zip` which will later be referenced by Open Tofu. +3. Zip all functions. AWS Lambda functions are usually deployed from zip files. cd to the main project directory and then run `./zip-all.sh` +4. Create `config.yaml` in the root directory of the repo to contain the proper values needed for the deployment. A default config.yaml for a 6-perspective deployment with the controller in us-east-2 is included in this repo as `config.example.yaml`. This config can be made the active config by running `cp config.example.yaml config.yaml` in the root directory. +5. Run `hatch run ./configure.py` from the root directory of the repo to generate Open Tofu files from templates. +6. Deploy the entire package with Open Tofu. cd to the `open-tofu` directory where .tf files are located. Then run `tofu init`. Then run `tofu apply` and type `yes` at the confirmation prompt. This provides a standard install with DNSSEC enabled which causes the system to incur expenses even when it is not in use (due to the AWS VPC NAT Gateways needed). To reduce the AWS bill, DNSSEC can also be disabled by appending `-var="dnssec_enabled=false"` to `tofu apply` (i.e., `tofu apply -var="dnssec_enabled=false"`). +7. Get the URL of the deployed API endpoint by running `hatch run ./get_api_url.py` in the root directory. +8. Get the API Key generated by AWS by running `hatch run ./get_api_key.py` in the root directory. The deployment is configured to reject any API call that does not have this key passed via the `x-api-key` HTTP header. + +For convenience `./deploy.sh` in the project root will perform all of these steps (using `-var="dnssec_enabled=false"`) with the exception of copying over the example config to the operational config and running `tofu init` in the open-tofu dir. ## Testing The following is an example of a test API call that uses bash command substitution to fill in the proper values for the API URL and the API key. @@ -40,7 +41,7 @@ The above sample must be run from the root directory of a deployed Open MPIC aws The API is compliant with the [Open MPIC Specification](https://github.com/open-mpic/open-mpic-specification). -Documentation based on the API specification can be viewed [here](https://open-mpic.org/documentation.html). +Documentation based on the API specification used in this version can be viewed [here](https://open-mpic.org/documentation.html?commit=f763382c38a867dda3253afded017f9e3a24ead5). ## Development Code changes can easily be deployed by editing the .py files and then rezipping the project via `./zip-all.sh` and `./2-package.sh` in the `layer` directory. Then, running `tofu apply` run from the open-tofu directory will update only on the required resources and leave the others unchanged. If any `.tf.template` files are changed or `config.yaml` is edited, `hatch run ./configure.py` must be rerun followed by `tofu apply` in the open-tofu directory. diff --git a/clean.sh b/clean.sh index a3c871a..cf0d5c1 100755 --- a/clean.sh +++ b/clean.sh @@ -10,15 +10,9 @@ rm open-tofu/*.generated.tf rm -r layer/create_layer_virtualenv rm -r layer/python311_layer_content rm -r layer/mpic_coordinator_layer_content -rm -r layer/mpic_caa_checker_layer_content -rm -r layer/mpic_dcv_checker_layer_content -rm -r layer/mpic_common_layer_content rm layer/python311_layer_content.zip rm layer/mpic_coordinator_layer_content.zip -rm layer/mpic_caa_checker_layer_content.zip -rm layer/mpic_dcv_checker_layer_content.zip -rm layer/mpic_common_layer_content.zip rm "${FUNCTIONS_DIR}"/mpic_coordinator_lambda/mpic_coordinator_lambda.zip rm "${FUNCTIONS_DIR}"/mpic_caa_checker_lambda/mpic_caa_checker_lambda.zip diff --git a/configure.py b/configure.py index 9430f1c..a3be590 100755 --- a/configure.py +++ b/configure.py @@ -38,7 +38,6 @@ def main(raw_args=None): stream.write(deployment_id_to_write) # Read the deployment id. - deployment_id = 0 with open(args.deployment_id_file) as stream: deployment_id = int(stream.read()) @@ -48,14 +47,14 @@ def main(raw_args=None): try: config = yaml.safe_load(stream) except yaml.YAMLError as exc: - print(f"Error loading YAML config at {args.config}. Project not configured. Error details: {exec}.") + print(f"Error loading YAML config at {args.config}. Project not configured. Error details: {exc}.") exit() aws_available_regions = {} with open(args.available_regions) as stream: try: aws_available_regions = yaml.safe_load(stream)['aws-available-regions'] except yaml.YAMLError as exc: - print(f"Error loading YAML config at {args.available_regions}. Project not configured. Error details: {exec}.") + print(f"Error loading YAML config at {args.available_regions}. Project not configured. Error details: {exc}.") exit() # Remove all old files. @@ -96,7 +95,6 @@ def main(raw_args=None): main_tf_string = main_tf_string.replace("{{absolut-max-attempts-with-key}}", f"absolute_max_attempts = \"{config['absolute-max-attempts']}\"") else: main_tf_string = main_tf_string.replace("{{absolut-max-attempts-with-key}}", "") - # Replace enforce distinct rir regions. main_tf_string = main_tf_string.replace("{{enforce-distinct-rir-regions}}", f"\"{1 if config['enforce-distinct-rir-regions'] else 0}\"") @@ -154,7 +152,6 @@ def main(raw_args=None): # Set the RIR region to load into env variables. aws_perspective_tf_region = aws_perspective_tf_region.replace("{{rir-region}}", f"{rir_region}") - if not args.aws_perspective_tf_template.endswith(".tf.template"): print(f"Error: invalid tf template name: {args.aws_perspective_tf_template}. Make sure all tf template files end in '.tf.template'.") exit() @@ -166,4 +163,4 @@ def main(raw_args=None): # Main module init for direct invocation. if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/deploy.sh b/deploy.sh index 9437a95..3d0de2b 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,2 +1,2 @@ #!/bin/bash -./clean.sh; cd layer; ./1-install.sh; ./2-package.sh; cd ..; hatch run ./configure.py; ./zip-all.sh; cd open-tofu; tofu apply -auto-approve; cd .. +./clean.sh; hatch run lambda-layer:install; cd layer; ./package.sh; cd ..; hatch run ./configure.py; ./zip-all.sh; cd open-tofu; tofu apply -var="dnssec_enabled=false" -auto-approve; cd .. diff --git a/layer/1-install.sh b/layer/1-install.sh deleted file mode 100755 index 3d0a538..0000000 --- a/layer/1-install.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -python3.11 -m venv --clear create_layer_virtualenv -source create_layer_virtualenv/bin/activate -# need to explicitly set target directory to install dependencies when explicitly specifying platform -pip install -r requirements.txt --platform manylinux2014_aarch64 --only-binary=:all: --target "$VIRTUAL_ENV/lib/python3.11/site-packages" diff --git a/layer/2-package.sh b/layer/2-package.sh deleted file mode 100755 index ae56758..0000000 --- a/layer/2-package.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# make common python3.11 layer for all lambda functions -mkdir -p python311_layer_content/python -cd python311_layer_content -cp -r ../create_layer_virtualenv/lib python/ -zip -r ../python311_layer_content.zip python -cd .. # should be at layer directory - -py_exclude=('*.pyc' '*__pycache__*') - -# make mpic_common lambda layer for all lambda functions -mkdir -p mpic_common_layer_content/python/open_mpic_core -cp -r ../open_mpic_core_python/src/open_mpic_core/common_domain mpic_common_layer_content/python/open_mpic_core/common_domain -cd mpic_common_layer_content -zip -r ../mpic_common_layer_content.zip python -x "${py_exclude[@]}" # Zip the mpic_common lambda layer -rm -r python # clean up, mostly not to bother the IDE which will find this duplicate code! -cd .. # should be at layer directory - -# make mpic_coordinator lambda layer for mpic coordinator lambda function -mkdir -p mpic_coordinator_layer_content/python/open_mpic_core -cp -r ../open_mpic_core_python/src/open_mpic_core/mpic_coordinator mpic_coordinator_layer_content/python/open_mpic_core/mpic_coordinator -cp -r ../resources mpic_coordinator_layer_content/python/resources # TODO consider a more elegant approach -cd mpic_coordinator_layer_content -zip -r ../mpic_coordinator_layer_content.zip python -x "${py_exclude[@]}" # Zip the mpic_coordinator lambda layer -rm -r python # clean up, mostly not to bother the IDE which will find this duplicate code! -cd .. # should be at layer directory - -# make mpic_caa_checker lambda layer for mpic caa checker lambda function -mkdir -p mpic_caa_checker_layer_content/python/open_mpic_core -cp -r ../open_mpic_core_python/src/open_mpic_core/mpic_caa_checker mpic_caa_checker_layer_content/python/open_mpic_core/mpic_caa_checker -cd mpic_caa_checker_layer_content -zip -r ../mpic_caa_checker_layer_content.zip python -x "${py_exclude[@]}" # Zip the mpic_caa_checker lambda layer -rm -r python # clean up, mostly not to bother the IDE which will find this duplicate code! -cd .. # should be at layer directory - -# make mpic_dcv_checker lambda layer for mpic dcv checker lambda function -mkdir -p mpic_dcv_checker_layer_content/python/open_mpic_core -cp -r ../open_mpic_core_python/src/open_mpic_core/mpic_dcv_checker mpic_dcv_checker_layer_content/python/open_mpic_core/mpic_dcv_checker -cd mpic_dcv_checker_layer_content -zip -r ../mpic_dcv_checker_layer_content.zip python -x "${py_exclude[@]}" # Zip the mpic_dcv_checker lambda layer -rm -r python # clean up, mostly not to bother the IDE which will find this duplicate code! -cd .. # should be at layer directory - diff --git a/layer/package.sh b/layer/package.sh new file mode 100755 index 0000000..c9841f1 --- /dev/null +++ b/layer/package.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# make common python3.11 layer for all lambda functions +mkdir -p python311_layer_content/python +cd python311_layer_content +cp -r ../create_layer_virtualenv/lib python/ +zip -r ../python311_layer_content.zip python +cd .. # should be at layer directory + +py_exclude=('*.pyc' '*__pycache__*') + +# make mpic_coordinator lambda layer for mpic coordinator lambda function +mkdir -p mpic_coordinator_layer_content/python +cp -r ../resources mpic_coordinator_layer_content/python/resources # TODO consider a more elegant approach +cd mpic_coordinator_layer_content +zip -r ../mpic_coordinator_layer_content.zip python -x "${py_exclude[@]}" # Zip the mpic_coordinator lambda layer +rm -r python # clean up, mostly not to bother the IDE which will find this duplicate code! +cd .. # should be at layer directory + + diff --git a/layer/requirements.txt b/layer/requirements.txt deleted file mode 100644 index d0c479a..0000000 --- a/layer/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -requests>=2.32.0 -dnspython==2.6.1 -pydantic==2.8.2 -pyyaml==6.0.1 -aws-lambda-powertools==3.2.0 \ No newline at end of file diff --git a/open-tofu/aws-perspective.tf.template b/open-tofu/aws-perspective.tf.template index 904fada..3c3b3db 100644 --- a/open-tofu/aws-perspective.tf.template +++ b/open-tofu/aws-perspective.tf.template @@ -7,30 +7,6 @@ resource "aws_lambda_layer_version" "python311_open_mpic_layer_{{region}}" { provider = aws.{{region}} } -resource "aws_lambda_layer_version" "mpic_common_layer_{{region}}" { - filename = "../layer/mpic_common_layer_content.zip" - layer_name = "mpic_common_layer_{{region}}_{{deployment-id}}" - source_code_hash = "${filebase64sha256("../layer/mpic_common_layer_content.zip")}" - compatible_runtimes = ["python3.11"] - provider = aws.{{region}} -} - -resource "aws_lambda_layer_version" "mpic_caa_checker_layer_{{region}}" { - filename = "../layer/mpic_caa_checker_layer_content.zip" - layer_name = "mpic_caa_checker_layer_{{region}}_{{deployment-id}}" - source_code_hash = "${filebase64sha256("../layer/mpic_caa_checker_layer_content.zip")}" - compatible_runtimes = ["python3.11"] - provider = aws.{{region}} -} - -resource "aws_lambda_layer_version" "mpic_dcv_checker_layer_{{region}}" { - filename = "../layer/mpic_dcv_checker_layer_content.zip" - layer_name = "mpic_dcv_checker_layer_{{region}}_{{deployment-id}}" - source_code_hash = "${filebase64sha256("../layer/mpic_dcv_checker_layer_content.zip")}" - compatible_runtimes = ["python3.11"] - provider = aws.{{region}} -} - variable "vpc_cidr_block_{{region}}" { type = string description = "VPC CIDR" @@ -50,6 +26,7 @@ variable "subnet_private_cidr_block_{{region}}" { } resource "aws_vpc" "vpc_{{region}}" { + count = var.dnssec_enabled ? 1 : 0 cidr_block = var.vpc_cidr_block_{{region}} tags = { Name = "mpic-{{region}}-vpc" @@ -60,7 +37,8 @@ resource "aws_vpc" "vpc_{{region}}" { } resource "aws_subnet" "subnet_public_{{region}}" { - vpc_id = aws_vpc.vpc_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + vpc_id = aws_vpc.vpc_{{region}}[count.index].id cidr_block = var.subnet_public_cidr_block_{{region}} depends_on = [aws_vpc.vpc_{{region}}] map_public_ip_on_launch = true @@ -71,7 +49,8 @@ resource "aws_subnet" "subnet_public_{{region}}" { } resource "aws_internet_gateway" "internet_gateway_{{region}}" { - vpc_id = aws_vpc.vpc_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + vpc_id = aws_vpc.vpc_{{region}}[count.index].id tags = { Name = "mpic-{{region}}-internet-gateway" @@ -80,11 +59,12 @@ resource "aws_internet_gateway" "internet_gateway_{{region}}" { } resource "aws_route_table" "route_table_public_{{region}}" { - vpc_id = aws_vpc.vpc_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + vpc_id = aws_vpc.vpc_{{region}}[count.index].id route { cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.internet_gateway_{{region}}.id + gateway_id = aws_internet_gateway.internet_gateway_{{region}}[count.index].id } tags = { @@ -94,12 +74,14 @@ resource "aws_route_table" "route_table_public_{{region}}" { } resource "aws_route_table_association" "route_table_association_public_{{region}}" { - subnet_id = aws_subnet.subnet_public_{{region}}.id - route_table_id = aws_route_table.route_table_public_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + subnet_id = aws_subnet.subnet_public_{{region}}[count.index].id + route_table_id = aws_route_table.route_table_public_{{region}}[count.index].id provider = aws.{{region}} } resource "aws_eip" "eip_{{region}}" { + count = var.dnssec_enabled ? 1 : 0 domain = "vpc" depends_on = [aws_internet_gateway.internet_gateway_{{region}}] tags = { @@ -109,8 +91,9 @@ resource "aws_eip" "eip_{{region}}" { } resource "aws_nat_gateway" "nat_gateway_{{region}}" { - allocation_id = aws_eip.eip_{{region}}.id - subnet_id = aws_subnet.subnet_public_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + allocation_id = aws_eip.eip_{{region}}[count.index].id + subnet_id = aws_subnet.subnet_public_{{region}}[count.index].id tags = { Name = "mpic-{{region}}-nat-gateway" @@ -119,7 +102,8 @@ resource "aws_nat_gateway" "nat_gateway_{{region}}" { } resource "aws_subnet" "subnet_private_{{region}}" { - vpc_id = aws_vpc.vpc_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + vpc_id = aws_vpc.vpc_{{region}}[count.index].id cidr_block = var.subnet_private_cidr_block_{{region}} map_public_ip_on_launch = false depends_on = [aws_vpc.vpc_{{region}}] @@ -129,13 +113,13 @@ resource "aws_subnet" "subnet_private_{{region}}" { provider = aws.{{region}} } - resource "aws_route_table" "route_table_private_{{region}}" { - vpc_id = aws_vpc.vpc_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + vpc_id = aws_vpc.vpc_{{region}}[count.index].id route { cidr_block = "0.0.0.0/0" - nat_gateway_id = aws_nat_gateway.nat_gateway_{{region}}.id + nat_gateway_id = aws_nat_gateway.nat_gateway_{{region}}[count.index].id } tags = { @@ -145,14 +129,16 @@ resource "aws_route_table" "route_table_private_{{region}}" { } resource "aws_route_table_association" "route_table_association_private_{{region}}" { - subnet_id = aws_subnet.subnet_private_{{region}}.id - route_table_id = aws_route_table.route_table_private_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + subnet_id = aws_subnet.subnet_private_{{region}}[count.index].id + route_table_id = aws_route_table.route_table_private_{{region}}[count.index].id provider = aws.{{region}} } resource "aws_default_network_acl" "default_network_acl_{{region}}" { - default_network_acl_id = aws_vpc.vpc_{{region}}.default_network_acl_id - subnet_ids = [aws_subnet.subnet_public_{{region}}.id, aws_subnet.subnet_private_{{region}}.id] + count = var.dnssec_enabled ? 1 : 0 + default_network_acl_id = aws_vpc.vpc_{{region}}[count.index].default_network_acl_id + subnet_ids = [aws_subnet.subnet_public_{{region}}[count.index].id, aws_subnet.subnet_private_{{region}}[count.index].id] ingress { protocol = -1 @@ -178,9 +164,9 @@ resource "aws_default_network_acl" "default_network_acl_{{region}}" { provider = aws.{{region}} } - resource "aws_default_security_group" "default_security_group_{{region}}" { - vpc_id = aws_vpc.vpc_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + vpc_id = aws_vpc.vpc_{{region}}[count.index].id ingress { protocol = -1 @@ -194,7 +180,7 @@ resource "aws_default_security_group" "default_security_group_{{region}}" { to_port = 0 protocol = "-1" cidr_blocks = ["0.0.0.0/0"] - # cidr_blocks = ["127.0.0.1/32"] + # cidr_blocks = ["127.0.0.1/32"] } tags = { @@ -204,7 +190,8 @@ resource "aws_default_security_group" "default_security_group_{{region}}" { } resource "aws_route53_resolver_dnssec_config" "dnssec_config_{{region}}" { - resource_id = aws_vpc.vpc_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + resource_id = aws_vpc.vpc_{{region}}[count.index].id provider = aws.{{region}} } @@ -212,7 +199,12 @@ resource "aws_lambda_function" "mpic_dcv_checker_lambda_{{region}}" { filename = "../{{source-path}}/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda.zip" function_name = "open_mpic_dcv_checker_lambda_{{region}}_{{deployment-id}}" role = aws_iam_role.open_mpic_lambda_role.arn - depends_on = [aws_iam_role.open_mpic_lambda_role, aws_iam_role_policy_attachment.basic-execution-policy-attach, aws_iam_role_policy_attachment.vpc-policy-attach, aws_iam_role_policy_attachment.invoke-lambda-policy-attach] + depends_on = [ + aws_iam_role.open_mpic_lambda_role, + aws_iam_role_policy_attachment.basic-execution-policy-attach, + aws_iam_role_policy_attachment.invoke-lambda-policy-attach, + aws_iam_role_policy_attachment.vpc-policy-attach + ] handler = "mpic_dcv_checker_lambda_function.lambda_handler" source_code_hash = filebase64sha256("../{{source-path}}/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda.zip") timeout = 60 @@ -220,8 +212,6 @@ resource "aws_lambda_function" "mpic_dcv_checker_lambda_{{region}}" { architectures = ["arm64"] layers = [ aws_lambda_layer_version.python311_open_mpic_layer_{{region}}.arn, - aws_lambda_layer_version.mpic_common_layer_{{region}}.arn, - aws_lambda_layer_version.mpic_dcv_checker_layer_{{region}}.arn, ] environment { variables = { @@ -229,8 +219,8 @@ resource "aws_lambda_function" "mpic_dcv_checker_lambda_{{region}}" { } } vpc_config { - subnet_ids = [aws_subnet.subnet_private_{{region}}.id] - security_group_ids = [aws_default_security_group.default_security_group_{{region}}.id] + subnet_ids = [for s in aws_subnet.subnet_private_{{region}} : s.id] + security_group_ids = [for s in aws_default_security_group.default_security_group_{{region}} : s.id] } provider = aws.{{region}} } @@ -239,7 +229,12 @@ resource "aws_lambda_function" "mpic_caa_checker_lambda_{{region}}" { filename = "../{{source-path}}/mpic_caa_checker_lambda/mpic_caa_checker_lambda.zip" function_name = "open_mpic_caa_checker_lambda_{{region}}_{{deployment-id}}" role = aws_iam_role.open_mpic_lambda_role.arn - depends_on = [aws_iam_role.open_mpic_lambda_role, aws_iam_role_policy_attachment.basic-execution-policy-attach, aws_iam_role_policy_attachment.vpc-policy-attach, aws_iam_role_policy_attachment.invoke-lambda-policy-attach] + depends_on = [ + aws_iam_role.open_mpic_lambda_role, + aws_iam_role_policy_attachment.basic-execution-policy-attach, + aws_iam_role_policy_attachment.invoke-lambda-policy-attach, + aws_iam_role_policy_attachment.vpc-policy-attach + ] handler = "mpic_caa_checker_lambda_function.lambda_handler" source_code_hash = filebase64sha256("../{{source-path}}/mpic_caa_checker_lambda/mpic_caa_checker_lambda.zip") timeout = 60 @@ -247,12 +242,10 @@ resource "aws_lambda_function" "mpic_caa_checker_lambda_{{region}}" { architectures = ["arm64"] layers = [ aws_lambda_layer_version.python311_open_mpic_layer_{{region}}.arn, - aws_lambda_layer_version.mpic_common_layer_{{region}}.arn, - aws_lambda_layer_version.mpic_caa_checker_layer_{{region}}.arn, ] vpc_config { - subnet_ids = [aws_subnet.subnet_private_{{region}}.id] - security_group_ids = [aws_default_security_group.default_security_group_{{region}}.id] + subnet_ids = [for s in aws_subnet.subnet_private_{{region}} : s.id] + security_group_ids = [for s in aws_default_security_group.default_security_group_{{region}} : s.id] } provider = aws.{{region}} environment { diff --git a/open-tofu/main.tf.template b/open-tofu/main.tf.template index 88697c1..e6cbd4e 100644 --- a/open-tofu/main.tf.template +++ b/open-tofu/main.tf.template @@ -12,14 +12,6 @@ resource "aws_lambda_layer_version" "python311_open_mpic_layer" { compatible_runtimes = ["python3.11"] } -# Mpic Common layer for all lambdas (contains supporting first-party source code) -resource "aws_lambda_layer_version" "mpic_common_layer" { - filename = "../layer/mpic_common_layer_content.zip" - layer_name = "mpic_common_layer_{{deployment-id}}" - source_code_hash = "${filebase64sha256("../layer/mpic_common_layer_content.zip")}" - compatible_runtimes = ["python3.11"] -} - # Mpic Coordinator layer for the mpic coordinator lambda (contains supporting first-party source code) resource "aws_lambda_layer_version" "mpic_coordinator_layer" { filename = "../layer/mpic_coordinator_layer_content.zip" @@ -61,6 +53,7 @@ resource "aws_iam_role_policy_attachment" "basic-execution-policy-attach" { } resource "aws_iam_role_policy_attachment" "vpc-policy-attach" { + count = var.dnssec_enabled ? 1 : 0 role = "${aws_iam_role.open_mpic_lambda_role.name}" policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" } @@ -83,7 +76,6 @@ resource "aws_lambda_function" "mpic_coordinator_lambda" { timeout = 60 layers = [ aws_lambda_layer_version.python311_open_mpic_layer.arn, - aws_lambda_layer_version.mpic_common_layer.arn, aws_lambda_layer_version.mpic_coordinator_layer.arn, ] environment { diff --git a/open-tofu/variables.tf b/open-tofu/variables.tf new file mode 100644 index 0000000..05a1c85 --- /dev/null +++ b/open-tofu/variables.tf @@ -0,0 +1,5 @@ +variable "dnssec_enabled" { + type = bool + description = "Enable DNSSEC" + default = true +} diff --git a/open_mpic_core_python b/open_mpic_core_python deleted file mode 160000 index 4f3824a..0000000 --- a/open_mpic_core_python +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4f3824a450e452f18796d8fd55c3be934b0ae498 diff --git a/pyproject.toml b/pyproject.toml index 0ad477c..1bd4198 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "dnspython==2.6.1", "pydantic==2.8.2", "aws-lambda-powertools[parser]==3.2.0", + "open-mpic-core==2.2.0", ] [project.optional-dependencies] @@ -39,7 +40,6 @@ test = [ "pytest-cov==5.0.0", "pytest-mock==3.14.0", "pytest-html==4.1.1", - "pytest-spec==3.2.0", ] [project.urls] @@ -52,37 +52,59 @@ Source = "https://github.com/open-mpic/aws-lambda-python" [tool.hatch] version.path = "src/aws_lambda_mpic/__about__.py" -build.sources = ["src", "resources", "open_mpic_core_python/src"] -build.targets.wheel.packages = ["src/aws_lambda_mpic", "open_mpic_core_python/src/open_mpic_core"] +build.sources = ["src", "resources"] +build.targets.wheel.packages = ["src/aws_lambda_mpic"] [tool.hatch.envs.default] skip-install = false type="virtual" path="venv" +[tool.hatch.envs.default.env-vars] +PIP_INDEX_URL = "https://pypi.org/simple/" +PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" # FIXME here temporarily to test open-mpic-core packaging +PIP_VERBOSE = "1" + +[tool.hatch.envs.lambda-layer] +skip-install = true +python = "3.11" +type="virtual" +path="layer/create_layer_virtualenv" + +[tool.hatch.envs.lambda-layer.env-vars] +PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" +PIP_ONLY_BINARY = ":all:" +#PIP_PLATFORM = "manylinux2014_aarch64" +#PIP_TARGET = "layer/create_layer_virtualenv2/lib/python3.11/site-packages" # does not work... bug in pip 24.2? + +[tool.hatch.envs.lambda-layer.scripts] +install = "pip install . --platform manylinux2014_aarch64 --only-binary=:all: --target layer/create_layer_virtualenv/lib/python3.11/site-packages" + [tool.hatch.envs.test] +skip-install = false features = [ "test", "provided" ] +installer = "pip" + +[tool.hatch.envs.test.env-vars] +PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" [tool.hatch.envs.test.scripts] +pre-install = "python -m ensurepip" unit = "pytest" unit-html = "pytest --html=testreports/index.html" # generate html report (warning: uses an aging plugin, 11-2023) integration = "pytest tests/integration" -coverage = "pytest --cov=src/aws_lambda_mpic --cov=src/open_mpic_core --cov-report=term-missing --cov-report=html" +coverage = "pytest --cov=src/aws_lambda_mpic --cov-report=term-missing --cov-report=html" [tool.hatch.envs.hatch-test] -features = [ - "test", - "provided" -] default-args = ["tests/unit"] randomize = true [tool.pytest.ini_options] pythonpath = [ - "src", "tests", "open_mpic_core_python/src", "." # need root directory because it has some python utility files we are importing for integration tests + "src", "tests", "." # need root directory because it has some python utility files we are importing for integration tests ] testpaths = [ "tests/unit" @@ -96,7 +118,6 @@ markers = [ ] addopts = [ "--import-mode=prepend", # explicit default, as the tests rely on it for proper import resolution - "--spec" # show test names in a more readable format in the console (warning: uses an aging plugin, 5-2021) ] spec_header_format = "Spec for {test_case} ({path}):" spec_test_format = "{result} {docstring_summary}" # defaults to {name} if docstring is not present in test diff --git a/tests/integration/test_deployed_mpic_api.py b/tests/integration/test_deployed_mpic_api.py index 569dd87..e4e6999 100644 --- a/tests/integration/test_deployed_mpic_api.py +++ b/tests/integration/test_deployed_mpic_api.py @@ -17,6 +17,7 @@ MPIC_REQUEST_PATH = "/mpic" + # noinspection PyMethodMayBeStatic @pytest.mark.integration class TestDeployedMpicApi: @@ -69,8 +70,8 @@ def api_should_return_200_and_passed_corroboration_given_successful_caa_check(se ('sub1.cname-deny.basic.caatestsuite.com', 'Tests handling of CNAME, where parent is CNAME and CAA record is at target', False), ('deny.permit.basic.caatestsuite.com', 'Tests rejection when parent name contains a permissible CAA record set', False), ('ipv6only.caatestsuite.com', 'Tests handling of record at IPv6-only authoritative name server', False), - ('expired.caatestsuite-dnssec.com', 'Tests rejection when expired DNSSEC signatures', False), - ('missing.caatestsuite-dnssec.com', 'Tests rejection when missing DNSSEC signatures', False), + #('expired.caatestsuite-dnssec.com', 'Tests rejection when expired DNSSEC signatures', False), # DNSSEC SHOULD be enabled in production but is not a current requirement for MPIC + #('missing.caatestsuite-dnssec.com', 'Tests rejection when missing DNSSEC signatures', False), # DNSSEC SHOULD be enabled in production but is not a current requirement for MPIC ('blackhole.caatestsuite-dnssec.com', 'Tests rejection when DNSSEC chain goes to non-responsive server', False), ('servfail.caatestsuite-dnssec.com', 'Tests rejection when DNSSEC chain goes to server returning SERVFAIL', False), ('refused.caatestsuite-dnssec.com', 'Tests rejection when DNSSEC chain goes to server returning REFUSED', False), @@ -146,7 +147,7 @@ def api_should_return_200_given_valid_dcv_validation(self, api_client): orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), dcv_check_parameters=DcvCheckParameters( validation_details=DcvWebsiteChangeValidationDetails(http_token_path='/', - challenge_value='test') + challenge_value='test') ) ) @@ -162,7 +163,7 @@ def api_should_return_200_and_failed_corroboration_given_failed_dcv_check(self, domain_or_ip_target='ifconfig.me', dcv_check_parameters=DcvCheckParameters( validation_details=DcvWebsiteChangeValidationDetails(http_token_path='/', - challenge_value='test') + challenge_value='test') ) ) diff --git a/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py b/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py index 8e54815..1709681 100644 --- a/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py @@ -2,11 +2,11 @@ import pytest import aws_lambda_mpic.mpic_dcv_checker_lambda.mpic_dcv_checker_lambda_function as mpic_dcv_checker_lambda_function -from open_mpic_core.common_domain.check_parameters import DcvWebsiteChangeValidationDetails -from open_mpic_core.common_domain.check_response import DcvCheckResponse, DcvCheckResponseDetails -from open_mpic_core.common_domain.check_response_details import DcvWebsiteChangeResponseDetails from open_mpic_core.common_domain.validation_error import MpicValidationError from unit.test_util.valid_check_creator import ValidCheckCreator +from open_mpic_core.common_domain.check_response_details import DcvHttpCheckResponseDetails +from open_mpic_core.common_domain.enum.dcv_validation_method import DcvValidationMethod +from open_mpic_core.common_domain.check_response import DcvCheckResponse class TestDcvCheckerLambda: @@ -58,7 +58,7 @@ def lambda_handler__should_return_appropriate_status_code_given_errors_in_respon @staticmethod def create_dcv_check_response(): return DcvCheckResponse(perspective_code='us-east-1', check_passed=True, - details=DcvWebsiteChangeResponseDetails(), + details=DcvHttpCheckResponseDetails(validation_method=DcvValidationMethod.WEBSITE_CHANGE_V2), timestamp_ns=time.time_ns()) diff --git a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py index 8b6c632..da4e2fe 100644 --- a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py @@ -12,9 +12,10 @@ from open_mpic_core.common_domain.check_request import DcvCheckRequest from open_mpic_core.common_domain.check_response import DcvCheckResponse -from open_mpic_core.common_domain.check_response_details import DcvDnsChangeResponseDetails from open_mpic_core.common_domain.enum.check_type import CheckType from open_mpic_core.common_domain.remote_perspective import RemotePerspective +from open_mpic_core.common_domain.check_response_details import DcvDnsCheckResponseDetails +from open_mpic_core.common_domain.enum.dcv_validation_method import DcvValidationMethod from open_mpic_core.mpic_coordinator.domain.mpic_orchestration_parameters import MpicEffectiveOrchestrationParameters from open_mpic_core.mpic_coordinator.domain.mpic_response import MpicCaaResponse from aws_lambda_mpic.mpic_coordinator_lambda.mpic_coordinator_lambda_function import MpicCoordinatorLambdaHandler @@ -130,7 +131,7 @@ def create_successful_boto3_api_call_response_for_dcv_check(self, lambda_method, check_request = DcvCheckRequest.model_validate_json(lambda_configuration['Payload']) # hijacking the value of 'perspective' to verify that the right arguments got passed to the call expected_response_body = DcvCheckResponse(perspective_code=check_request.domain_or_ip_target, - check_passed=True, details=DcvDnsChangeResponseDetails()) + check_passed=True, details=DcvDnsCheckResponseDetails(validation_method=DcvValidationMethod.ACME_DNS_01)) expected_response = {'statusCode': 200, 'body': expected_response_body.model_dump_json()} json_bytes = json.dumps(expected_response).encode('utf-8') file_like_response = io.BytesIO(json_bytes)