Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ poetry bundle venv .build/.venv --without dev
package-python-function .build/.venv --output-dir .build/lambda
```

The output will be a .zip file with the same name as your project from your pyproject.toml file.
The output will be a .zip file with the same name as your project from your pyproject.toml file (with dashes replaced
with underscores).

## Installation
Use [pipx](https://github.com/pypa/pipx) to install:
Expand All @@ -40,7 +41,7 @@ One of the following must be specified:
- `--output`: The full output path of the final zip file.

- `--output-dir`: The output directory for the final zip file. The name of the zip file will be based on the project's
name in the pyproject.toml file.
name in the pyproject.toml file (with dashes replaced with underscores).



2 changes: 1 addition & 1 deletion package_python_function/packager.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def __init__(self, venv_path: Path, project_path: Path, output_dir: Path, output
self.venv_path = venv_path

self.output_dir = output_file.parent if output_file else output_dir
self.output_file = output_file if output_file else output_dir / f'{self.project.name}.zip'
self.output_file = output_file if output_file else output_dir / f'{self.project.distribution_name}.zip'

self._uncompressed_bytes = 0

Expand Down
13 changes: 12 additions & 1 deletion package_python_function/python_project.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from pathlib import Path
from typing import Optional
import tomllib
import re


class PythonProject:
Expand All @@ -16,14 +17,24 @@ def name(self) -> str:
('tool', 'poetry', 'name'),
))

"""
Get the normalized name of the distribution, according to the Python Packaging Authority (PyPa) guidelines.
This is used to create the name of the zip file.
The name is normalized by replacing any non-alphanumeric characters with underscores.
https://peps.python.org/pep-0427/#escaping-and-unicode
"""
@cached_property
def distribution_name(self) -> str:
return re.sub("[^\w\d.]+", "_", self.name, re.UNICODE)

@cached_property
def entrypoint_package_name(self) -> str:
"""
The subdirectory name in the source virtual environment's site-packages that contains the function's entrypoint
code.
"""
# TODO : Parse out the project's package dir(s) if defined. Use the first one if there are multiple.
return self.name.replace('-', '_')
return self.distribution_name

def find_value(self, paths: tuple[tuple[str]]) -> str:
for path in paths:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,8 @@
import logging

logger = logging.getLogger(__name__)

logger.info("Load")

def other_package_module():
print("other_package_module")
logger.info("Hello from other_package_module")
6 changes: 5 additions & 1 deletion scripts/poc/inner_package/zip_in_zip_test/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
# This file represents the original module's __init__.py file that gets renamed when creating the innner ZIP.

print("__init__ original")
import logging

logger = logging.getLogger(__name__ + "(__init__.py ORIGINAL)")

logger.info("Hello, I am the original pacakge __init__.py")

GLOBAL_VALUE_IN_INIT_ORIGINAL = "This global is defined in the original __init__.py"

Expand Down
10 changes: 7 additions & 3 deletions scripts/poc/inner_package/zip_in_zip_test/main.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
print("main.py: Load")
import logging

logger = logging.getLogger(__name__)

logger.info("main.py: Load")

from zip_in_zip_test import GLOBAL_VALUE_IN_INIT_ORIGINAL, other_module_function
from other_package.other_package_module import other_package_module

def main():
print("Hello from main!")
print(GLOBAL_VALUE_IN_INIT_ORIGINAL)
logger.info("Hello from main!")
logger.info(GLOBAL_VALUE_IN_INIT_ORIGINAL)
other_module_function()
other_package_module()
8 changes: 7 additions & 1 deletion scripts/poc/inner_package/zip_in_zip_test/other_module.py
Original file line number Diff line number Diff line change
@@ -1,2 +1,8 @@
import logging

logger = logging.getLogger(__name__)

logger.info("Load")

def other_module_function():
print("I'm in other_module_function")
logger.info("I'm in other_module_function")
15 changes: 13 additions & 2 deletions scripts/poc/lambda-runner.py
Original file line number Diff line number Diff line change
@@ -1,24 +1,35 @@
# This is my best attempt at simulating what AWS Lambda does
# Instead of messing with zipping and unzipping in this experiment, I just copy the files to the .test directory.

import logging
from pathlib import Path
import shutil
import sys

print('[lambda-runner]')
print('sys.path:', sys.path)
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
logger = logging.getLogger("lambda-runner.py")

logger.info('BEGIN')

module_path = Path(__file__).parent
TEST_DIR = module_path / ".test"
PACKAGE_NAME = "zip_in_zip_test"
TEST_PACKAGE_DIR = TEST_DIR / PACKAGE_NAME

shutil.rmtree(TEST_DIR, ignore_errors=True)

# Copy the stub `zip_in_zip_test` package to the .test directory. This simulates the outer ZIP extraction
# that lambda will do. This is the module that Lambda will import, and where we will do the inner ZIP extraction.
shutil.copytree(str(module_path / PACKAGE_NAME), str(TEST_PACKAGE_DIR))

# Copy the inner package to the .test directory. This simulates the inner ZIP file, but without actually dealing
# with zip/unzip in this experiemen.
shutil.copytree(str(module_path / "inner_package"), str(TEST_PACKAGE_DIR / ".inner_package"))

sys.path.insert(0, str(TEST_DIR))

logger.info('--- Importing entrypoint module ---')
import importlib
module = importlib.import_module('zip_in_zip_test.main')
logger.info('--- Calling entryoint function ---')
module.__dict__['main']()
20 changes: 14 additions & 6 deletions scripts/poc/zip_in_zip_test/__init__.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,20 @@
# This works perfectly!

print('zip_in_zip_test.__init__: BEGIN. This is the loader.')
print("module_path:", __file__)

"""
Demonstrate how we can swap out the content an entire package during the __init__.py load process of that package.
"""
from pathlib import Path
import importlib
import sys
import logging

logger = logging.getLogger(__name__ + "(__init__.py loader)")

logger.info(f'BEGIN. This is the loader. {__file__}')

module_path = Path(__file__).parent

# This is where we would unzip the inner ZIP file. For this experiment, we can skip actually doing that and pretend
# that it was extracted to .inner_package/

# This works if I insert at zero.
# Why does the serverless-python-requirements insist on inserting at 1?
# From https://docs.aws.amazon.com/lambda/latest/dg/python-package.html#python-package-searchpath:
Expand All @@ -17,6 +23,7 @@

# This also works. I am thinking this is the best way, because we need to unmount the original decompressed directory
# since it contains the load __init__.py.
previous_sys_path_root = sys.path[0]
sys.path[0] = str(module_path / ".inner_package")


Expand All @@ -30,6 +37,7 @@
# importlib.import_module(__name__)

# This also works. I think this is the best way.
logger.info(f'Reloading {__name__} after switching {previous_sys_path_root} to {sys.path[0]}.')
importlib.reload(sys.modules[__name__])

print('zip_in_zip_test.__init__: END')
logger.info('END')
2 changes: 1 addition & 1 deletion tests/test_package_python_function.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,4 @@ def test_package_python_function(tmp_path: Path) -> None:
]
main()

assert (output_dir_path / 'project-1.zip').exists()
assert (output_dir_path / 'project_1.zip').exists()