From 8fd05bbe43d46d969d32bdaeb32306fcbfb58ab5 Mon Sep 17 00:00:00 2001 From: Vito Di Benedetto <55766483+vitodb@users.noreply.github.com> Date: Thu, 7 Dec 2023 15:47:58 -0600 Subject: [PATCH 1/8] Add decisionengine as decisionengine_modules requirement --- setup.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7cd88ab4..f16ff247 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,8 @@ ] rpm_require.extend(__base_pip_requires) + + # This metadata can be read out with: # import importlib.metadata # dir(importlib.metadata.metadata('decisionengine_modules')) @@ -78,6 +80,19 @@ # importlib_resources on python < 3.9 # # Much of it comes out of decisionengine_modules.about.py + +# for decisionengine dev we have version in the form X.Y.Z.devN +# while for tag we have version in the form X.Y.Z +verarr=about.__version__.split(".")[:4] +if len(verarr)>3: + # this is a dev version + ver='.'.join(verarr[:3]+[verarr[3][:3]]) +else: + # this is a tag + ver='.'.join(verarr[:3]) +# string to add decisionengine as decisionengine_modules requirement +de_req=[f"decisionengine>="+ver] + setup( setup_requires=["setuptools >= 51.2", "wheel >= 0.36.2", "setuptools_scm >= 6.3.1"], python_requires=">3.7.0", @@ -89,7 +104,7 @@ license=about.__license__, package_dir={"": "src"}, packages=find_packages(where="src", exclude=("tests", "*.tests", "*.tests.*", "build.*", "doc.*")), - install_requires=runtime_require, + install_requires=runtime_require+de_req, extras_require={ "develop": devel_req, }, From fb0621992db2f88715509453554fef717068275f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Dec 2023 22:24:49 +0000 Subject: [PATCH 2/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- setup.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index f16ff247..0a467bb4 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,6 @@ rpm_require.extend(__base_pip_requires) - # This metadata can be read out with: # import importlib.metadata # dir(importlib.metadata.metadata('decisionengine_modules')) @@ -83,15 +82,15 @@ # for decisionengine dev we have version in the form X.Y.Z.devN # while for tag we have version in the form X.Y.Z -verarr=about.__version__.split(".")[:4] -if len(verarr)>3: +verarr = about.__version__.split(".")[:4] +if len(verarr) > 3: # this is a dev version - ver='.'.join(verarr[:3]+[verarr[3][:3]]) + ver = ".".join(verarr[:3] + [verarr[3][:3]]) else: # this is a tag - ver='.'.join(verarr[:3]) + ver = ".".join(verarr[:3]) # string to add decisionengine as decisionengine_modules requirement -de_req=[f"decisionengine>="+ver] +de_req = [f"decisionengine>=" + ver] setup( setup_requires=["setuptools >= 51.2", "wheel >= 0.36.2", "setuptools_scm >= 6.3.1"], @@ -104,7 +103,7 @@ license=about.__license__, package_dir={"": "src"}, packages=find_packages(where="src", exclude=("tests", "*.tests", "*.tests.*", "build.*", "doc.*")), - install_requires=runtime_require+de_req, + install_requires=runtime_require + de_req, extras_require={ "develop": devel_req, }, From 148e2fe692a0670c291d86a03b8dae89228a2d01 Mon Sep 17 00:00:00 2001 From: Vito Di Benedetto <55766483+vitodb@users.noreply.github.com> Date: Thu, 28 Dec 2023 15:48:58 -0600 Subject: [PATCH 3/8] In Jenkinsfile for EL9 add build_whl job and remove rpmbuild job --- .Jenkins/workflows/Jenkinsfile_EL9 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.Jenkins/workflows/Jenkinsfile_EL9 b/.Jenkins/workflows/Jenkinsfile_EL9 index 919369b6..895c8c08 100644 --- a/.Jenkins/workflows/Jenkinsfile_EL9 +++ b/.Jenkins/workflows/Jenkinsfile_EL9 @@ -104,7 +104,7 @@ pipeline { } } } - stage('rpmbuild') { + stage('build_whl') { agent { node { label 'docker' @@ -117,7 +117,7 @@ pipeline { steps { script { // DOCKER_IMAGE is defined through Jenkins project - rpmbuildStageDockerImage="${DOCKER_IMAGE}_${BUILD_NUMBER}_${STAGE_NAME}" + buildwhlStageDockerImage="${DOCKER_IMAGE}_${BUILD_NUMBER}_${STAGE_NAME}" } echo "cleanup workspace" sh 'for f in $(ls -A); do rm -rf ${f}; done' @@ -135,16 +135,16 @@ pipeline { fi cd .. ''' - echo "prepare docker image ${rpmbuildStageDockerImage}" - sh "docker build --pull --tag ${rpmbuildStageDockerImage} --build-arg BASEIMAGE=hepcloud/decision-engine-ci-el9:${BRANCH} --build-arg UID=\$(id -u) --build-arg GID=\$(id -g) -f decisionengine_modules/package/ci/EL9/Dockerfile decisionengine_modules/package/ci/EL9/" + echo "prepare docker image ${buildwhlStageDockerImage}" + sh "docker build --pull --tag ${buildwhlStageDockerImage} --build-arg BASEIMAGE=hepcloud/decision-engine-ci-el9:${BRANCH} --build-arg UID=\$(id -u) --build-arg GID=\$(id -g) -f decisionengine_modules/package/ci/EL9/Dockerfile decisionengine_modules/package/ci/EL9/" echo "Run ${STAGE_NAME} tests" - sh "docker run --rm --env GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER} --env PYTEST_TIMEOUT=${PYTEST_TIMEOUT} -v ${WORKSPACE}/decisionengine_modules:${WORKSPACE}/decisionengine_modules -w ${WORKSPACE}/decisionengine_modules ${rpmbuildStageDockerImage} \"setup.py bdist_rpm\" \"rpmbuild.log\" \"${BRANCH}\"" + sh "docker run --rm --env GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER} --env PYTEST_TIMEOUT=${PYTEST_TIMEOUT} -v ${WORKSPACE}/decisionengine_modules:${WORKSPACE}/decisionengine_modules -w ${WORKSPACE}/decisionengine_modules ${buildwhlStageDockerImage} \"setup.py bdist_wheel\" \"build_whl.log\" \"${BRANCH}\"" } post { always { - archiveArtifacts artifacts: "decisionengine_modules/rpmbuild.log,decisionengine_modules/dist/*.rpm" - echo "cleanup docker image ${rpmbuildStageDockerImage}" - sh "docker rmi ${rpmbuildStageDockerImage}" + archiveArtifacts artifacts: "decisionengine_modules/build_whl.log,decisionengine_modules/dist/*.whl" + echo "cleanup docker image ${buildwhlStageDockerImage}" + sh "docker rmi ${buildwhlStageDockerImage}" } } } From fee44612fef6a71b65930467079c956f2741b0c9 Mon Sep 17 00:00:00 2001 From: Vito Di Benedetto <55766483+vitodb@users.noreply.github.com> Date: Thu, 28 Dec 2023 15:49:38 -0600 Subject: [PATCH 4/8] Add build_whl job in GitHub actions to build DE modules whl --- .github/workflows/ci.yaml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 53eba063..c49842d5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -173,3 +173,42 @@ jobs: run: | cd doc make rst html latexpdf + + build_whl: + if: ${{ github.ref != 'refs/heads/1.7' }} + name: Build DE Modules whl + runs-on: ubuntu-latest + needs: pytest + steps: + - name: make date tag + id: mkdatetag + run: echo "::set-output name=dtag::$(date +%Y%m%d_%H%M%S)" + + - name: make ref tag case 1 + id: mkreftag1 + if: ${{ github.event.inputs.ref != '' }} + run: echo "::set-output name=reftag::${{github.event.inputs.ref}}" + + - name: make ref tag case 2 + id: mkreftag2 + if: ${{ github.event.inputs.ref == '' }} + run: echo "::set-output name=reftag::${GITHUB_BASE_REF:+PR}"$(awk -F"/" '{print $3}' <<< ${GITHUB_REF}) + + - name: checkout code tree + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{github.event.inputs.ref}} + + - name: Run the build in a container (EL9/HEPCloud-CI) + uses: ./.github/actions/python-command-in-el9-container + with: + python-command: "setup.py bdist_wheel" + logfile: "build_whl.log" + + - name: Archive whl + uses: actions/upload-artifact@v3 + with: + name: whl-DE-EL9-${{steps.mkreftag1.outputs.reftag}}${{steps.mkreftag2.outputs.reftag}}-${{steps.mkdatetag.outputs.dtag}} + path: dist/*.whl + if-no-files-found: error From f936473fb608fb495002f4c33e89219539c2e319 Mon Sep 17 00:00:00 2001 From: Vito Di Benedetto <55766483+vitodb@users.noreply.github.com> Date: Thu, 28 Dec 2023 15:56:27 -0600 Subject: [PATCH 5/8] Add build_whl job in GitHub actions to build DE modules whl --- .../action.yaml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/actions/python-command-in-el9-container/action.yaml diff --git a/.github/actions/python-command-in-el9-container/action.yaml b/.github/actions/python-command-in-el9-container/action.yaml new file mode 100644 index 00000000..5d5d071e --- /dev/null +++ b/.github/actions/python-command-in-el9-container/action.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: 2017 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + +name: "Python Action" +description: "Run python3 action in preconfigured EL9 container image" +inputs: + python-command: + description: "What to run" + required: true + default: "-m pytest" + logfile: + description: "Where to log output" + required: true + default: "pytest.log" +runs: + using: "docker" + image: "../../../package/ci/EL9/Dockerfile" + args: + - ${{ inputs.python-command }} + - ${{ inputs.logfile }} From 12dcfa320f98dfcff30ace1ed7927c8dcae8772a Mon Sep 17 00:00:00 2001 From: Vito Di Benedetto <55766483+vitodb@users.noreply.github.com> Date: Thu, 28 Dec 2023 16:04:34 -0600 Subject: [PATCH 6/8] Add build_whl job in GitHub actions to build DE modules whl --- .../action.yaml | 20 ++++++++++ .github/workflows/ci.yaml | 39 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 .github/actions/python-command-in-el9-container/action.yaml diff --git a/.github/actions/python-command-in-el9-container/action.yaml b/.github/actions/python-command-in-el9-container/action.yaml new file mode 100644 index 00000000..5d5d071e --- /dev/null +++ b/.github/actions/python-command-in-el9-container/action.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: 2017 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + +name: "Python Action" +description: "Run python3 action in preconfigured EL9 container image" +inputs: + python-command: + description: "What to run" + required: true + default: "-m pytest" + logfile: + description: "Where to log output" + required: true + default: "pytest.log" +runs: + using: "docker" + image: "../../../package/ci/EL9/Dockerfile" + args: + - ${{ inputs.python-command }} + - ${{ inputs.logfile }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 53eba063..c49842d5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -173,3 +173,42 @@ jobs: run: | cd doc make rst html latexpdf + + build_whl: + if: ${{ github.ref != 'refs/heads/1.7' }} + name: Build DE Modules whl + runs-on: ubuntu-latest + needs: pytest + steps: + - name: make date tag + id: mkdatetag + run: echo "::set-output name=dtag::$(date +%Y%m%d_%H%M%S)" + + - name: make ref tag case 1 + id: mkreftag1 + if: ${{ github.event.inputs.ref != '' }} + run: echo "::set-output name=reftag::${{github.event.inputs.ref}}" + + - name: make ref tag case 2 + id: mkreftag2 + if: ${{ github.event.inputs.ref == '' }} + run: echo "::set-output name=reftag::${GITHUB_BASE_REF:+PR}"$(awk -F"/" '{print $3}' <<< ${GITHUB_REF}) + + - name: checkout code tree + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{github.event.inputs.ref}} + + - name: Run the build in a container (EL9/HEPCloud-CI) + uses: ./.github/actions/python-command-in-el9-container + with: + python-command: "setup.py bdist_wheel" + logfile: "build_whl.log" + + - name: Archive whl + uses: actions/upload-artifact@v3 + with: + name: whl-DE-EL9-${{steps.mkreftag1.outputs.reftag}}${{steps.mkreftag2.outputs.reftag}}-${{steps.mkdatetag.outputs.dtag}} + path: dist/*.whl + if-no-files-found: error From 244f858eb62bcfa549e17c1cc7dfa99561e69872 Mon Sep 17 00:00:00 2001 From: Vito Di Benedetto <55766483+vitodb@users.noreply.github.com> Date: Thu, 4 Jan 2024 16:11:51 -0600 Subject: [PATCH 7/8] GlideinWMS production branch is master --- package/ci/EL9/python3-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/ci/EL9/python3-entrypoint.sh b/package/ci/EL9/python3-entrypoint.sh index d1af2f52..3365a62c 100755 --- a/package/ci/EL9/python3-entrypoint.sh +++ b/package/ci/EL9/python3-entrypoint.sh @@ -27,7 +27,7 @@ git clone -b ${DE_BRANCH} https://github.com/HEPCloud/decisionengine.git # checkout GlideinWMS for python3 rm -rf glideinwms -git clone -b branch_v3_9 https://github.com/glideinWMS/glideinwms.git +git clone -b master https://github.com/glideinWMS/glideinwms.git # Install dependencies for GlideinWMS python3 -m pip install --upgrade pip From 57846bde122184e28653897ce5a32808e468a2e4 Mon Sep 17 00:00:00 2001 From: Vito Di Benedetto <55766483+vitodb@users.noreply.github.com> Date: Mon, 10 Feb 2025 21:08:32 -0600 Subject: [PATCH 8/8] In build_whl stage use podman to solve a merge conflict --- .Jenkins/workflows/Jenkinsfile_EL9 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.Jenkins/workflows/Jenkinsfile_EL9 b/.Jenkins/workflows/Jenkinsfile_EL9 index 895c8c08..1da636fe 100644 --- a/.Jenkins/workflows/Jenkinsfile_EL9 +++ b/.Jenkins/workflows/Jenkinsfile_EL9 @@ -135,16 +135,16 @@ pipeline { fi cd .. ''' - echo "prepare docker image ${buildwhlStageDockerImage}" - sh "docker build --pull --tag ${buildwhlStageDockerImage} --build-arg BASEIMAGE=hepcloud/decision-engine-ci-el9:${BRANCH} --build-arg UID=\$(id -u) --build-arg GID=\$(id -g) -f decisionengine_modules/package/ci/EL9/Dockerfile decisionengine_modules/package/ci/EL9/" + echo "prepare podman image ${buildwhlStageDockerImage}" + sh "podman build --pull --tag ${buildwhlStageDockerImage} --build-arg BASEIMAGE=hepcloud/decision-engine-ci-el9:${BRANCH} --build-arg UID=\$(id -u) --build-arg GID=\$(id -g) -f decisionengine_modules/package/ci/EL9/Dockerfile decisionengine_modules/package/ci/EL9/" echo "Run ${STAGE_NAME} tests" - sh "docker run --rm --env GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER} --env PYTEST_TIMEOUT=${PYTEST_TIMEOUT} -v ${WORKSPACE}/decisionengine_modules:${WORKSPACE}/decisionengine_modules -w ${WORKSPACE}/decisionengine_modules ${buildwhlStageDockerImage} \"setup.py bdist_wheel\" \"build_whl.log\" \"${BRANCH}\"" + sh "podman run --userns keep-id:uid=\$(id -u),gid=\$(id -g) --rm --env GITHUB_PR_NUMBER=${GITHUB_PR_NUMBER} --env PYTEST_TIMEOUT=${PYTEST_TIMEOUT} -v ${WORKSPACE}/decisionengine_modules:${WORKSPACE}/decisionengine_modules -w ${WORKSPACE}/decisionengine_modules ${buildwhlStageDockerImage} \"setup.py bdist_wheel\" \"build_whl.log\" \"${BRANCH}\"" } post { always { archiveArtifacts artifacts: "decisionengine_modules/build_whl.log,decisionengine_modules/dist/*.whl" - echo "cleanup docker image ${buildwhlStageDockerImage}" - sh "docker rmi ${buildwhlStageDockerImage}" + echo "cleanup podman image ${buildwhlStageDockerImage}" + sh "podman rmi ${buildwhlStageDockerImage}" } } }