From 45f53d93a7c204f4dc5a20531b825ef3e875475b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 5 Mar 2025 06:55:37 -0600 Subject: [PATCH 1/6] PYTHON-4953 Add contributing guide and improve helper scripts --- .github/workflows/dist-python.yml | 4 +- .pre-commit-config.yaml | 12 +++ bindings/python/CONTRIBUTING.md | 34 +++++++++ bindings/python/pymongocrypt/binding.py | 3 +- .../{ => scripts}/build-manylinux-wheel.sh | 0 .../{ => scripts}/libmongocrypt-version.txt | 0 bindings/python/{ => scripts}/release.sh | 10 ++- bindings/python/{ => scripts}/synchro.py | 25 +++--- bindings/python/scripts/synchro.sh | 8 ++ .../update-version.sh} | 19 ++++- bindings/python/scripts/update_binding.py | 76 +++++++++++++++++++ bindings/python/strip_header.py | 50 ------------ 12 files changed, 174 insertions(+), 67 deletions(-) create mode 100644 bindings/python/CONTRIBUTING.md rename bindings/python/{ => scripts}/build-manylinux-wheel.sh (100%) rename bindings/python/{ => scripts}/libmongocrypt-version.txt (100%) rename bindings/python/{ => scripts}/release.sh (95%) rename bindings/python/{ => scripts}/synchro.py (69%) create mode 100755 bindings/python/scripts/synchro.sh rename bindings/python/{update-sbom.sh => scripts/update-version.sh} (50%) create mode 100644 bindings/python/scripts/update_binding.py delete mode 100644 bindings/python/strip_header.py diff --git a/.github/workflows/dist-python.yml b/.github/workflows/dist-python.yml index dd278d4ea..b25b84c18 100644 --- a/.github/workflows/dist-python.yml +++ b/.github/workflows/dist-python.yml @@ -54,9 +54,9 @@ jobs: - name: Build and test dist files run: | - export LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt) + export LIBMONGOCRYPT_VERSION=$(cat ./scripts/libmongocrypt-version.txt) git fetch origin $LIBMONGOCRYPT_VERSION - bash ./release.sh + bash ./scripts/release.sh - uses: actions/upload-artifact@v4 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bbd2bbbd1..9d3a41a30 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,6 +53,18 @@ repos: language: system types: [shell] +- repo: local + hooks: + - id: synchro + name: synchro + entry: bash ./bindings/python/scripts/synchro.sh + language: python + require_serial: true + fail_fast: true + additional_dependencies: + - ruff==0.1.3 + - unasync + - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. rev: v0.1.3 diff --git a/bindings/python/CONTRIBUTING.md b/bindings/python/CONTRIBUTING.md new file mode 100644 index 000000000..a1a598368 --- /dev/null +++ b/bindings/python/CONTRIBUTING.md @@ -0,0 +1,34 @@ +# Contributing to PyMongoCrypt + +## Asyncio considerations +PyMongoCrypt adds asyncio capability by modifying the source files in */asynchronous to */synchronous using [unasync](https://github.com/python-trio/unasync/) and some custom transforms. + +Where possible, edit the code in `*/asynchronous/*.py` and not the synchronous files. You can run `pre-commit run --all-files synchro` before running tests if you are testing synchronous code. + +To prevent the synchro hook from accidentally overwriting code, it first checks to see whether a sync version of a file is changing and not its async counterpart, and will fail. In the unlikely scenario that you want to override this behavior, first export `OVERRIDE_SYNCHRO_CHECK=1`. + +Sometimes, the synchro hook will fail and introduce changes many previously unmodified files. This is due to static Python errors, such as missing imports, incorrect syntax, or other fatal typos. To resolve these issues, run `pre-commit run --all-files --hook-stage manual ruff` and fix all reported errors before running the synchro hook again. + +## Updating the libmongocrypt bindings + +To update the libmongocrypt bindings in `pymongocrypt/binding.py`, run the following script: + +```bash +python scripts/update_binding.py +``` + +## Update the bundled version of libmongocrypt + +To update the bundled version of libmongocrypt, run the following script: + +```bash +bash script/update-version.sh +``` + +This will set the version in `scripts/libmongocrypt-version.sh` and update `sbom.json` to reflect +the new vendored version of `libmongocrypt`. + +## Building wheels + +To build wheels, run `scripts/release.sh`. It will build the appropriate wheel for the current system +on Windows and MacOS. If docker is available on Linux or MacOS, it will build the manylinux wheels. diff --git a/bindings/python/pymongocrypt/binding.py b/bindings/python/pymongocrypt/binding.py index fe371a52a..c1bd543cf 100644 --- a/bindings/python/pymongocrypt/binding.py +++ b/bindings/python/pymongocrypt/binding.py @@ -29,7 +29,7 @@ def _parse_version(version): ffi = cffi.FFI() -# Generated with strip_header.py +# Start embedding from update_binding.py ffi.cdef( """/* * Copyright 2019-present MongoDB, Inc. @@ -1468,6 +1468,7 @@ def _parse_version(version): // DEPRECATED: Support "rangePreview" has been removed in favor of "range". """ ) +# End embedding from update_binding.py def _to_string(cdata): diff --git a/bindings/python/build-manylinux-wheel.sh b/bindings/python/scripts/build-manylinux-wheel.sh similarity index 100% rename from bindings/python/build-manylinux-wheel.sh rename to bindings/python/scripts/build-manylinux-wheel.sh diff --git a/bindings/python/libmongocrypt-version.txt b/bindings/python/scripts/libmongocrypt-version.txt similarity index 100% rename from bindings/python/libmongocrypt-version.txt rename to bindings/python/scripts/libmongocrypt-version.txt diff --git a/bindings/python/release.sh b/bindings/python/scripts/release.sh similarity index 95% rename from bindings/python/release.sh rename to bindings/python/scripts/release.sh index 684b172b0..2f49ff35e 100755 --- a/bindings/python/release.sh +++ b/bindings/python/scripts/release.sh @@ -15,14 +15,19 @@ set -o xtrace # Write all commands first to stderr set -o errexit # Exit the script with error if any of the commands fail +SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) + # The libmongocrypt git revision release to embed in our wheels. -LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt) +LIBMONGOCRYPT_VERSION=$(cat $SCRIPT_DIR/libmongocrypt-version.txt) REVISION=$(git rev-list -n 1 $LIBMONGOCRYPT_VERSION) # The libmongocrypt release branch. -BRANCH="r1.12" +MINOR_VERSION=$(echo $LIBMONGOCRYPT_VERSION | cut -d. -f1,2) +BRANCH="r${MINOR_VERSION}" # The python executable to use. PYTHON=${PYTHON:-python} +pushd $SCRIPT_DIR/.. + # Clean slate. rm -rf dist .venv build libmongocrypt pymongocrypt/*.so pymongocrypt/*.dll pymongocrypt/*.dylib @@ -126,3 +131,4 @@ if [ $(command -v docker) ]; then fi ls -ltr dist +popd diff --git a/bindings/python/synchro.py b/bindings/python/scripts/synchro.py similarity index 69% rename from bindings/python/synchro.py rename to bindings/python/scripts/synchro.py index a9a6ba686..26737ccbc 100644 --- a/bindings/python/synchro.py +++ b/bindings/python/scripts/synchro.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import sys from os import listdir from pathlib import Path @@ -27,12 +29,14 @@ "aclose": "close", } -_base = "pymongocrypt" +ROOT = Path(__file__).absolute().parent.parent + +_base = ROOT / "pymongocrypt" async_files = [ - f"./{_base}/asynchronous/{f}" - for f in listdir("pymongocrypt/asynchronous") - if (Path(_base) / "asynchronous" / f).is_file() + f"{_base}/asynchronous/{f}" + for f in listdir(f"{_base}/asynchronous") + if (_base / "asynchronous" / f).is_file() ] @@ -40,20 +44,23 @@ async_files, [ Rule( - fromdir="/pymongocrypt/asynchronous/", - todir="/pymongocrypt/synchronous/", + fromdir=f"{_base}/asynchronous/", + todir=f"{_base}/synchronous/", additional_replacements=replacements, ) ], ) sync_files = [ - f"./{_base}/synchronous/{f}" - for f in listdir("pymongocrypt/synchronous") - if (Path(_base) / "synchronous" / f).is_file() + f"{_base}/synchronous/{f}" + for f in listdir(f"{_base}/synchronous") + if (_base / "synchronous" / f).is_file() ] +modified_files = [f"./{f}" for f in sys.argv[1:]] for file in sync_files: + if file in modified_files and "OVERRIDE_SYNCHRO_CHECK" not in os.environ: + raise ValueError(f"Refusing to overwrite {file}") with open(file, "r+") as f: lines = f.readlines() for i in range(len(lines)): diff --git a/bindings/python/scripts/synchro.sh b/bindings/python/scripts/synchro.sh new file mode 100755 index 000000000..70efcadac --- /dev/null +++ b/bindings/python/scripts/synchro.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -eu + +SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) + +python $SCRIPT_DIR/synchro.py "$@" +python -m ruff check $SCRIPT_DIR/../pymongocrypt/synchronous --fix --silent diff --git a/bindings/python/update-sbom.sh b/bindings/python/scripts/update-version.sh similarity index 50% rename from bindings/python/update-sbom.sh rename to bindings/python/scripts/update-version.sh index e0c903149..3bdb7b05e 100755 --- a/bindings/python/update-sbom.sh +++ b/bindings/python/scripts/update-version.sh @@ -1,8 +1,19 @@ #!/bin/bash -set -eux +set -eu -LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt) +SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) + +if [ -z "${1:-}" ]; then + echo "Provide the new version of libmongocrypt!" + exit 1 +fi + +LIBMONGOCRYPT_VERSION=$1 + +echo $LIBMONGOCRYPT_VERSION > libmongocrypt-version.txt + +pushd $SCRIPT_DIR/.. if [ $(command -v podman) ]; then DOCKER=podman else @@ -10,5 +21,7 @@ else fi echo "pkg:github/mongodb/libmongocrypt@$LIBMONGOCRYPT_VERSION" > purls.txt -$DOCKER run --platform="linux/amd64" -it --rm -v $(pwd):$(pwd) artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:1.0 update --purls=$(pwd)/purls.txt -o $(pwd)/sbom.json +$DOCKER run --platform="linux/amd64" -it --rm -v $(pwd):$(pwd) artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0 update --purls=$(pwd)/purls.txt -o $(pwd)/sbom.json rm purls.txt + +popd diff --git a/bindings/python/scripts/update_binding.py b/bindings/python/scripts/update_binding.py new file mode 100644 index 000000000..0f1c7c89c --- /dev/null +++ b/bindings/python/scripts/update_binding.py @@ -0,0 +1,76 @@ +# Copyright 2019-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Update pymongocrypt/bindings.py using mongocrypt.h. +""" + +import re +from pathlib import Path + +DROP_RE = re.compile(r"^\s*(#|MONGOCRYPT_EXPORT)") +HERE = Path(__file__).absolute().parent + + +# itertools.pairwise backport for Python 3.9 support. +def pairwise(iterable): + # pairwise('ABCDEFG') → AB BC CD DE EF FG + + iterator = iter(iterable) + a = next(iterator, None) + + for b in iterator: + yield a, b + a = b + + +def strip_file(content): + fold = content.replace("\\\n", " ") + all_lines = [*fold.split("\n"), ""] + keep_lines = (line for line in all_lines if not DROP_RE.match(line)) + fin = "" + for line, peek in pairwise(keep_lines): + if peek == "" and line == "": + # Drop adjacent empty lines + continue + yield line + fin = peek + yield fin + + +def update_bindings(): + header_file = HERE.parent.parent.parent / "src/mongocrypt.h" + with header_file.open(encoding="utf-8") as fp: + header_lines = strip_file(fp.read()) + + target = HERE.parent / "pymongocrypt/binding.py" + source_lines = target.read_text().splitlines() + new_lines = [] + skip = False + for line in source_lines: + if not skip: + new_lines.append(line) + if line.strip() == "# Start embedding from update_binding.py": + skip = True + new_lines.append("ffi.cdef(") + new_lines.append('"""') + new_lines.extend(header_lines) + if line.strip() == "# End embedding from update_binding.py": + new_lines.append('"""') + new_lines.append(")") + new_lines.append(line) + skip = False + + +if __name__ == "__main__": + update_bindings() diff --git a/bindings/python/strip_header.py b/bindings/python/strip_header.py deleted file mode 100644 index fcc5426fc..000000000 --- a/bindings/python/strip_header.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2019-present MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Generate a CFFI.cdef() string from a C header file - -Usage (on macOS):: python strip_header.py ../../src/mongocrypt.h | pbcopy -""" - -import itertools -import re -import sys - -DROP_RE = re.compile(r"^\s*(#|MONGOCRYPT_EXPORT)") - - -def strip_file(content): - fold = content.replace("\\\n", " ") - all_lines = [*fold.split("\n"), ""] - keep_lines = (line for line in all_lines if not DROP_RE.match(line)) - fin = "" - for line, peek in itertools.pairwise(keep_lines): - if peek == "" and line == "": - # Drop adjacent empty lines - continue - yield line - fin = peek - yield fin - - -def strip(hdr): - with open(hdr) as fp: - out = strip_file(fp.read()) - print("\n".join(out)) # noqa: T201 - - -if __name__ == "__main__": - if len(sys.argv) != 2: - raise Exception("Usage: strip_header.py header.h") - strip(sys.argv[1]) From 7b8419ff6b13337c26c63e985a7a0d46dacd3285 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 5 Mar 2025 06:57:57 -0600 Subject: [PATCH 2/6] fix workflow --- .github/workflows/test-python.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 961127312..b818692be 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -40,6 +40,7 @@ jobs: if: github.repository_owner == 'mongodb' runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] python-version: ["3.8", "3.13"] @@ -59,6 +60,6 @@ jobs: if [ "${{ matrix.python-version }}" == "3.13" ]; then export PIP_PRE=1 fi - export LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt) + export LIBMONGOCRYPT_VERSION=$(cat ./scripts/libmongocrypt-version.txt) git fetch origin $LIBMONGOCRYPT_VERSION - bash ./release.sh + bash ./scripts/release.sh From a186b6c8e0da5ff9ef10355c510c897369c0be94 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 5 Mar 2025 07:33:03 -0600 Subject: [PATCH 3/6] fix linux wheel build --- bindings/python/scripts/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/scripts/release.sh b/bindings/python/scripts/release.sh index 2f49ff35e..37bdced4e 100755 --- a/bindings/python/scripts/release.sh +++ b/bindings/python/scripts/release.sh @@ -53,7 +53,7 @@ function build_wheel() { function build_manylinux_wheel() { python -m pip install unasync docker pull $1 - docker run --rm -v `pwd`:/python $1 /python/build-manylinux-wheel.sh + docker run --rm -v `pwd`:/python $1 /python/scripts/build-manylinux-wheel.sh # Sudo is needed to remove the files created by docker. sudo rm -rf build libmongocrypt pymongocrypt/*.so pymongocrypt/*.dll pymongocrypt/*.dylib } From cf732c3f0a3ae9f7b807b5120ac20556381bc81a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 5 Mar 2025 07:37:30 -0600 Subject: [PATCH 4/6] fix codeql --- .github/workflows/codeql-python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-python.yml b/.github/workflows/codeql-python.yml index b8964e2a0..2453b94f5 100644 --- a/.github/workflows/codeql-python.yml +++ b/.github/workflows/codeql-python.yml @@ -55,9 +55,9 @@ jobs: - name: Install package run: | cd bindings/python - export LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt) + export LIBMONGOCRYPT_VERSION=$(cat ./scripts/libmongocrypt-version.txt) git fetch origin $LIBMONGOCRYPT_VERSION - bash release.sh + bash ./scripts/release.sh pip install dist/*.whl - name: Perform CodeQL Analysis From 3840aeb1b5a493e3cc70640bcf95d4609f13cd5d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 5 Mar 2025 08:43:23 -0600 Subject: [PATCH 5/6] fix test --- bindings/python/.evergreen/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/.evergreen/test.sh b/bindings/python/.evergreen/test.sh index 0e5b68222..e49577c6f 100755 --- a/bindings/python/.evergreen/test.sh +++ b/bindings/python/.evergreen/test.sh @@ -86,7 +86,7 @@ for PYTHON_BINARY in "${PYTHONS[@]}"; do done # Verify the sbom file -LIBMONGOCRYPT_VERSION=$(cat ./libmongocrypt-version.txt) +LIBMONGOCRYPT_VERSION=$(cat ./scripts/libmongocrypt-version.txt) EXPECTED="pkg:github/mongodb/libmongocrypt@$LIBMONGOCRYPT_VERSION" if grep -q $EXPECTED sbom.json; then echo "SBOM is up to date!" From c34bcaa7771c0e69dfb913861f63da71bbb3a953 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 5 Mar 2025 08:46:33 -0600 Subject: [PATCH 6/6] fix test --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 5e2d97bcd..1e1f711c5 100755 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -306,7 +306,7 @@ functions: script: | set -ex cd ./libmongocrypt/bindings/python - PYTHON=${PYTHON} ./release.sh + PYTHON=${PYTHON} ./scripts/release.sh "upload python release": - command: archive.targz_pack