From de7dd1d94ca8dce8dab7bb3dd1471a51795d3c63 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 30 Aug 2019 10:47:31 -0400 Subject: [PATCH 1/7] Update packaging standards Signed-off-by: Brian J. Murrell --- Dockerfile.centos.7 | 77 ------- Jenkinsfile | 127 +++++++++-- Makefile-rpm.mk | 105 ++------- mpich.spec | 5 +- packaging/Dockerfile.centos.7 | 34 +++ packaging/Dockerfile.leap.15 | 45 ++++ packaging/Dockerfile.leap.42.3 | 46 ++++ packaging/Dockerfile.sles.12.3 | 46 ++++ packaging/Dockerfile.ubuntu.18.04 | 28 +++ packaging/Dockerfile.ubuntu.18.10 | 24 +++ packaging/Makefile_packaging.mk | 341 ++++++++++++++++++++++++++++++ 11 files changed, 692 insertions(+), 186 deletions(-) delete mode 100644 Dockerfile.centos.7 create mode 100644 packaging/Dockerfile.centos.7 create mode 100755 packaging/Dockerfile.leap.15 create mode 100644 packaging/Dockerfile.leap.42.3 create mode 100755 packaging/Dockerfile.sles.12.3 create mode 100644 packaging/Dockerfile.ubuntu.18.04 create mode 100644 packaging/Dockerfile.ubuntu.18.10 create mode 100644 packaging/Makefile_packaging.mk diff --git a/Dockerfile.centos.7 b/Dockerfile.centos.7 deleted file mode 100644 index 180c833ed44..00000000000 --- a/Dockerfile.centos.7 +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright 2019, Intel Corporation -# -# 'recipe' for Docker to build an RPM -# - -# Pull base image -FROM centos:7 -MAINTAINER Brian J. Murrell - -# use same UID as host and default value of 1000 if not specified -ARG UID=1000 - -# Update distribution -#Nothing to do for CentOS - -# Install basic tools -RUN yum install -y epel-release -RUN yum install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ - git - -# Add build user (to keep rpmbuild happy) -ENV USER build -ENV PASSWD build -RUN useradd -u $UID -ms /bin/bash $USER -RUN echo "$USER:$PASSWD" | chpasswd -# add the user to the mock group so it can run mock -RUN usermod -a -G mock $USER - -# mock in Docker needs to use the old-chroot option -RUN grep use_nspawn || \ - echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg - -ARG JENKINS_URL="" - -RUN echo -e "config_opts['yum.conf'] += \"\"\"\n" >> /etc/mock/default.cfg; \ - for repo in openpa libfabric pmix ompi mercury spdk isa-l fio dpdk \ - protobuf-c fuse pmdk argobots raft cart@daos_devel1 daos \ - automake libtool; do \ - branch="master"; \ - build_number="lastSuccessfulBuild"; \ - if [[ $repo = *@* ]]; then \ - branch="${repo#*@}"; \ - repo="${repo%@*}"; \ - if [[ $branch = *:* ]]; then \ - build_number="${branch#*:}"; \ - branch="${branch%:*}"; \ - fi; \ - fi; \ - echo -e "[$repo:$branch:$build_number]\n\ -name=$repo:$branch:$build_number\n\ -baseurl=${JENKINS_URL}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/centos7/\n\ -enabled=1\n\ -gpgcheck = False\n" >> /etc/mock/default.cfg; \ - done; \ - echo -e "[jhli-ipmctl]\n\ -name=Copr repo for ipmctl owned by jhli\n\ -baseurl=https://copr-be.cloud.fedoraproject.org/results/jhli/ipmctl/epel-7-\$basearch/\n\ -type=rpm-md\n\ -skip_if_unavailable=True\n\ -gpgcheck=1\n\ -gpgkey=https://copr-be.cloud.fedoraproject.org/results/jhli/ipmctl/pubkey.gpg\n\ -repo_gpgcheck=0\n\ -enabled=1\n\ -enabled_metadata=1\n\n\ -[jhli-safeclib]\n\ -name=Copr repo for safeclib owned by jhli\n\ -baseurl=https://copr-be.cloud.fedoraproject.org/results/jhli/safeclib/epel-7-\$basearch/\n\ -type=rpm-md\n\ -skip_if_unavailable=True\n\ -gpgcheck=1\n\ -gpgkey=https://copr-be.cloud.fedoraproject.org/results/jhli/safeclib/pubkey.gpg\n\ -repo_gpgcheck=0\n\ -enabled=1\n\ -enabled_metadata=1\n\"\"\"" >> /etc/mock/default.cfg - -RUN yum -y install patchutils autoconf automake libtool diff --git a/Jenkinsfile b/Jenkinsfile index 8ad36d1677c..3592fa1aea3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,68 +1,151 @@ +#!/usr/bin/groovy +/* Copyright (C) 2019 Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted for any purpose (including commercial purposes) + * provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions, and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions, and the following disclaimer in the + * documentation and/or materials provided with the distribution. + * + * 3. In addition, redistributions of modified forms of the source or binary + * code must carry prominent notices stating that the original code was + * changed and the date of the change. + * + * 4. All publications or advertising materials mentioning features or use of + * this software are asked, but not required, to acknowledge that it was + * developed by Intel Corporation and credit the contributors. + * + * 5. Neither the name of Intel Corporation, nor the name of any Contributor + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +// To use a test branch (i.e. PR) until it lands to master +// I.e. for testing library changes +//@Library(value="pipeline-lib@your_branch") _ + pipeline { agent none stages { + stage('Cancel Previous Builds') { + when { changeRequest() } + steps { + cancelPreviousBuilds() + } + } stage('Lint') { - stages { + parallel { stage('RPM Lint') { agent { dockerfile { - filename 'Dockerfile.centos.7' + filename 'packaging/Dockerfile.centos.7' label 'docker_runner' + args '--group-add mock' + + ' --cap-add=SYS_ADMIN' + + ' --privileged=true' additionalBuildArgs '--build-arg UID=$(id -u)' - args '--group-add mock --cap-add=SYS_ADMIN --privileged=true' } } steps { sh 'make -f Makefile-rpm.mk rpmlint' } } - } - } + stage('Check Packaging') { + agent { label 'lightweight' } + steps { + checkoutScm url: 'https://github.com/daos-stack/packaging.git', + checkoutDir: 'packaging-module' + catchError(stageResult: 'UNSTABLE', buildResult: 'SUCCESS') { + sh '''make -f Makefile-rpm.mk \ + PACKAGING_CHECK_DIR=packaging-module \ + packaging_check''' + } + } + post { + unsuccessful { + emailext body: 'Packaging out of date for ' + jobName() + '.\n' + + "You should update it and submit your PR again.", + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + subject: 'Packaging is out of date for ' + jobName() + + } + } + } //stage('Check Packaging') + } // parallel + } //stage('Lint') stage('Build') { parallel { stage('Build on CentOS 7') { agent { dockerfile { - filename 'Dockerfile.centos.7' + filename 'packaging/Dockerfile.centos.7' label 'docker_runner' - additionalBuildArgs '--build-arg UID=$(id -u) --build-arg JENKINS_URL=' + + args '--group-add mock' + + ' --cap-add=SYS_ADMIN' + + ' --privileged=true' + additionalBuildArgs '--build-arg UID=$(id -u)' + + ' --build-arg JENKINS_URL=' + env.JENKINS_URL - args '--group-add mock --cap-add=SYS_ADMIN --privileged=true' } } steps { - sh '''rm -rf artifacts/centos7/ + sh label: "Build package", + script: '''rm -rf artifacts/centos7/ mkdir -p artifacts/centos7/ - make -f Makefile-rpm.mk srpm - make -f Makefile-rpm.mk mockbuild''' + make -f Makefile-rpm.mk chrootbuild''' } post { success { - sh '''(cd /var/lib/mock/epel-7-x86_64/result/ && + sh label: "Collect artifacts", + script: '''(cd /var/lib/mock/epel-7-x86_64/result/ && cp -r . $OLDPWD/artifacts/centos7/) createrepo artifacts/centos7/''' - archiveArtifacts artifacts: 'artifacts/centos7/**' } - failure { - sh '''cp -af _topdir/SRPMS artifacts/centos7/ - (cd /var/lib/mock/epel-7-x86_64/result/ && - cp -r . $OLDPWD/artifacts/centos7/) - (cd /var/lib/mock/epel-7-x86_64/root/builddir/build/BUILD/*/ + unsuccessful { + sh label: "Collect artifacts", + script: '''mockroot=/var/lib/mock/epel-7-x86_64 + artdir=$PWD/artifacts/centos7 + cp -af _topdir/SRPMS $artdir + (cd $mockroot/result/ && + cp -r . $artdir) + (if cd $mockroot/root/builddir/build/BUILD/*/; then find . -name configure -printf %h\\\\n | \ while read dir; do if [ ! -f $dir/config.log ]; then continue fi - tdir="$OLDPWD/artifacts/centos7/autoconf-logs/$dir" + tdir="$artdir/autoconf-logs/$dir" mkdir -p $tdir cp -a $dir/config.log $tdir/ - done)''' + done + fi)''' + } + cleanup { archiveArtifacts artifacts: 'artifacts/centos7/**' } } - } - stage('Build on Ubuntu 16.04') { + } //stage('Build on CentOS 7') + stage('Build on Ubuntu 18.04') { agent { label 'docker_runner' } diff --git a/Makefile-rpm.mk b/Makefile-rpm.mk index f22ee0e7637..0ea35f5eea1 100644 --- a/Makefile-rpm.mk +++ b/Makefile-rpm.mk @@ -6,6 +6,10 @@ PATCHES := mpich.macros mpich.pth.py2 mpich.pth.py3 mpich-modules.patch fix-vers # daos_adio-hwloc.patch daos_adio-izem.patch daos_adio-ucx.patch \ # daos_adio-libfabric.patch # daos_adio-all.patch +ADD_REPOS := openpa libfabric pmix ompi mercury spdk isa-l fio dpdk \ + protobuf-c fuse pmdk argobots raft cart@daos_devel1 daos \ + automake libtool + GIT_TAG := v3.3 TOPDIR := $(shell echo $$PWD) @@ -33,7 +37,6 @@ daos_adio-all.patch: --exclude \*.gz \ pristine/* . > $@ || true echo rm -rf pristine - # so instead we get a patch for each submodule define gen_submod_patch set -e; \ @@ -81,42 +84,13 @@ daos_adio.patch: -x src/mpid/ch4/netmod/ucx/ucx \ > $@ -COMMON_RPM_ARGS := --define "%_topdir $$PWD/_topdir" -DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) -ifeq ($(DIST),) -SED_EXPR := 1p -else -SED_EXPR := 1s/$(DIST)//p -endif -SPEC := $(NAME).spec -VERSION := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{version}\n' $(SPEC) | sed -n '1p') -RELEASE := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{release}\n' $(SPEC) | sed -n '$(SED_EXPR)') -SRPM := _topdir/SRPMS/$(NAME)-$(VERSION)-$(RELEASE)$(DIST).src.rpm -RPMS := $(addsuffix .rpm,$(addprefix _topdir/RPMS/x86_64/,$(shell rpm --specfile $(SPEC)))) -SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES)) -TARGETS := $(RPMS) $(SRPM) +include packaging/Makefile_packaging.mk -all: $(TARGETS) +GIT_COMMIT := $(shell git rev-parse --short HEAD) -%/: - mkdir -p $@ - -_topdir/SOURCES/%: % | _topdir/SOURCES/ - rm -f $@ - ln $< $@ - -$(NAME)-$(VERSION).tar.$(SRC_EXT).asc: - curl -f -L -O '$(SOURCE).asc' - -%.gz: % - rm -f $@ - gzip $< - -#$(NAME)-$(VERSION).tar.$(SRC_EXT): -# curl -f -L -O '$(SOURCE)' -$(NAME)-$(VERSION).tar: Makefile-rpm.mk +$(NAME)-$(DL_VERSION)-$(GIT_COMMIT).tar: mkdir -p rpmbuild - git archive --prefix=$(subst .tar,,$@)/ -o $@ HEAD + git archive --prefix=$(subst -$(GIT_COMMIT).tar,,$@)/ -o $@ HEAD git submodule update --init set -x; p=$$PWD && (echo .; git submodule foreach) | \ while read junk path; do \ @@ -124,59 +98,18 @@ $(NAME)-$(VERSION).tar: Makefile-rpm.mk temp="$${temp#\'}"; \ path=$$temp; \ [ "$$path" = "" ] && continue; \ - (cd $$path && git archive --prefix=$(subst .tar,,$@)/$$path/ HEAD \ + (cd $$path && git archive --prefix=$(subst -$(GIT_COMMIT).tar,,$@)/$$path/ HEAD \ > $$p/rpmbuild/tmp.tar && \ tar --concatenate --file=$$p/$@ \ $$p/rpmbuild/tmp.tar && rm $$p/rpmbuild/tmp.tar); \ done - tar tvf $@ - -v$(VERSION).tar.$(SRC_EXT): - curl -f -L -O '$(SOURCE)' - -$(VERSION).tar.$(SRC_EXT): - curl -f -L -O '$(SOURCE)' - -# see https://stackoverflow.com/questions/2973445/ for why we subst -# the "rpm" for "%" to effectively turn this into a multiple matching -# target pattern rule -$(subst rpm,%,$(RPMS)): $(SPEC) $(SOURCES) - rpmbuild -bb $(COMMON_RPM_ARGS) $(RPM_BUILD_OPTIONS) $(SPEC) - -$(SRPM): $(SPEC) $(SOURCES) - rpmbuild -bs $(COMMON_RPM_ARGS) $(SPEC) - -srpm: $(SRPM) $(NAME).spec - -$(RPMS): Makefile-rpm.mk - -rpms: $(RPMS) - -ls: $(TARGETS) - ls -ld $^ - -mockbuild: $(SRPM) Makefile-rpm.mk - mock $(MOCK_OPTIONS) $< - -rpmlint: $(SPEC) - rpmlint $< - -show_version: - @echo $(VERSION) - -show_release: - @echo $(RELEASE) - -show_rpms: - @echo $(RPMS) - -show_source: - @echo $(SOURCE) - -show_sources: - @echo $(SOURCES) - -show_spec: - @ls -l $(SPEC) - -.PHONY: srpm rpms ls mockbuild rpmlint FORCE show_version show_release show_rpms show_source show_sources + #tar tvf $@ + +$(NAME)-$(DL_VERSION).tar.$(SRC_EXT): $(NAME)-$(DL_VERSION)-$(GIT_COMMIT).tar + older_tarballs=$$(ls $(NAME)-$(DL_VERSION)-*.tar | \ + grep -v $(NAME)-$(DL_VERSION)-$(GIT_COMMIT).tar) ; \ + if [ -n "$$older_tarballs" ]; then \ + rm -f "$$older_tarballs"; \ + fi + rm -f $@ + gzip < $< > $@ diff --git a/mpich.spec b/mpich.spec index a8be26529e7..4ccd2ccc1cf 100644 --- a/mpich.spec +++ b/mpich.spec @@ -1,7 +1,7 @@ Summary: A high-performance implementation of MPI Name: mpich Version: 3.3 -Release: 0.03%{?dist} +Release: 1%{?dist} License: MIT URL: http://www.mpich.org/ @@ -334,6 +334,9 @@ make check VERBOSE=1 %{python3_sitearch}/%{name}.pth %changelog +* Fri Aug 30 2019 Brian J. Murrell - 3.3-1 +- Update packaging standards + * Fri Aug 30 2019 Brian J. Murrell - 3.3-0.04 - Fix ABI version after upstream master merge diff --git a/packaging/Dockerfile.centos.7 b/packaging/Dockerfile.centos.7 new file mode 100644 index 00000000000..ab1a1fd78b0 --- /dev/null +++ b/packaging/Dockerfile.centos.7 @@ -0,0 +1,34 @@ +# +# Copyright 2018-2019, Intel Corporation +# +# 'recipe' for Docker to build an RPM +# + +# Pull base image +FROM centos:7 +MAINTAINER daos-stack + +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 + +# Update distribution +#Nothing to do for CentOS + +# Install basic tools +RUN yum install -y epel-release +RUN yum install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ + git + +# Add build user (to keep rpmbuild happy) +ENV USER build +ENV PASSWD build +RUN useradd -u $UID -ms /bin/bash $USER +RUN echo "$USER:$PASSWD" | chpasswd +# add the user to the mock group so it can run mock +RUN usermod -a -G mock $USER + +# mock in Docker needs to use the old-chroot option +RUN grep use_nspawn || \ + echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg + +RUN chmod g+w /etc/mock/default.cfg diff --git a/packaging/Dockerfile.leap.15 b/packaging/Dockerfile.leap.15 new file mode 100755 index 00000000000..6dfb538aa72 --- /dev/null +++ b/packaging/Dockerfile.leap.15 @@ -0,0 +1,45 @@ +# +# Copyright 2019, Intel Corporation +# +# 'recipe' for Docker to build an RPM +# + +# Pull base image +FROM opensuse/leap:15 +MAINTAINER daos-stack + +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 + +# Add build user (to keep rpmbuild happy) +ENV USER build +RUN useradd -u $UID -ms /bin/bash $USER +RUN groupadd -g $UID $USER + +# Install basic tools +RUN zypper --non-interactive up + +# basic building +RUN zypper --non-interactive in autoconf automake \ + ca-certificates-mozilla curl createrepo \ + git libtool lsb-release make \ + perl-libwww-perl perl-LWP-Protocol-https \ + perl-XML-Parser rpm-build sudo + +# need to run the build command as root, as it needs to chroot +RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ + echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ + fi; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/build" > /etc/sudoers.d/build; \ + chmod 0440 /etc/sudoers.d/build; \ + visudo -c; \ + sudo -l -U build + +ENV OPENSUSE_REPO="https://download.opensuse.org/repositories" +ENV TOOLS="${OPENSUSE_REPO}/openSUSE:Tools/openSUSE_15.1/openSUSE:Tools.repo" + +RUN set -x; zypper --non-interactive ar ${TOOLS}; \ + zypper --non-interactive --gpg-auto-import-keys ref \ + 'openSUSE.org tools (openSUSE_15.1)' + +RUN zypper --non-interactive in build diff --git a/packaging/Dockerfile.leap.42.3 b/packaging/Dockerfile.leap.42.3 new file mode 100644 index 00000000000..135317fca8d --- /dev/null +++ b/packaging/Dockerfile.leap.42.3 @@ -0,0 +1,46 @@ +# +# Copyright 2019, Intel Corporation +# +# 'recipe' for Docker to build an RPM +# + +# Pull base image +FROM opensuse/leap:42.3 +MAINTAINER daos-stack + +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 + +# Add build user (to keep rpmbuild happy) +ENV USER build +RUN useradd -u $UID -ms /bin/bash $USER +RUN groupadd -g $UID $USER + +# Install basic tools +RUN zypper --non-interactive up + +# basic building +RUN zypper --non-interactive in autoconf automake \ + ca-certificates-mozilla curl createrepo \ + git libtool lsb-release make \ + perl-libwww-perl perl-LWP-Protocol-https \ + perl-XML-Parser rpm-build sudo + +# need to run the build command as root, as it needs to chroot +RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ + echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ + fi; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/build" > /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /bin/rpm" >> /etc/sudoers.d/build; \ + chmod 0440 /etc/sudoers.d/build; \ + visudo -c; \ + sudo -l -U build + +ENV OPENSUSE_REPO="https://download.opensuse.org/repositories" +ENV TOOLS="${OPENSUSE_REPO}/openSUSE:Tools/openSUSE_42.3/openSUSE:Tools.repo" + +RUN set -x; zypper --non-interactive ar ${TOOLS}; \ + zypper --non-interactive --gpg-auto-import-keys ref \ + 'openSUSE.org tools (openSUSE_42.3)' + +RUN zypper --non-interactive in build diff --git a/packaging/Dockerfile.sles.12.3 b/packaging/Dockerfile.sles.12.3 new file mode 100755 index 00000000000..9894e371fe7 --- /dev/null +++ b/packaging/Dockerfile.sles.12.3 @@ -0,0 +1,46 @@ +# +# Copyright 2019, Intel Corporation +# +# 'recipe' for Docker to build an RPM +# + +# Pull base image +FROM suse/sles:12.3 +MAINTAINER daos-stack + +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 + +# Add build user (to keep rpmbuild happy) +ENV USER build +RUN useradd -u $UID -ms /bin/bash $USER +RUN groupadd -g $UID $USER + +# Install basic tools +RUN zypper --non-interactive up + +# basic building +RUN zypper --non-interactive in autoconf automake \ + ca-certificates-mozilla curl createrepo \ + git libtool lsb-release make \ + perl-libwww-perl perl-LWP-Protocol-https \ + perl-XML-Parser rpm-build sudo + +# need to run the build command as root, as it needs to chroot +RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ + echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ + fi; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/build" > /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /bin/rpm" >> /etc/sudoers.d/build; \ + chmod 0440 /etc/sudoers.d/build; \ + visudo -c; \ + sudo -l -U build + +ENV OPENSUSE_REPO="https://download.opensuse.org/repositories" +ENV TOOLS="${OPENSUSE_REPO}/openSUSE:Tools/SLE_12_SP3/openSUSE:Tools.repo" + +RUN set -x; zypper --non-interactive ar ${TOOLS}; \ + zypper --non-interactive --gpg-auto-import-keys ref \ + 'openSUSE.org tools (SLE_12_SP3)' + +RUN zypper --non-interactive in build diff --git a/packaging/Dockerfile.ubuntu.18.04 b/packaging/Dockerfile.ubuntu.18.04 new file mode 100644 index 00000000000..8518651fb74 --- /dev/null +++ b/packaging/Dockerfile.ubuntu.18.04 @@ -0,0 +1,28 @@ +# +# Copyright 2019, Intel Corporation +# +# 'recipe' for Docker to build an Debian package +# +# Pull base image +FROM ubuntu:18.04 +Maintainer John.E.Malmberg + +# Install basic tools +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + autoconf bash curl debhelper dh-make dpkg-dev doxygen gcc \ + git git-buildpackage locales make patch rpm wget + +# libfabric +RUN apt-get install -y \ + libibverbs-dev libpsm-infinipath1-dev librdmacm-dev + +ENV UBUNTU_URL="https://launchpad.net/ubuntu/+archive/primary/+files" +RUN curl -sS -f -L -O ${UBUNTU_URL}/libpsm2-2_10.3.58-1_amd64.deb ; \ + curl -sS -f -L -O ${UBUNTU_URL}/libpsm2-dev_10.3.58-1_amd64.deb ; \ + apt-get install -y \ + ./libpsm2-2_10.3.58-1_amd64.deb ./libpsm2-dev_10.3.58-1_amd64.deb + +# force an upgrade to get any newly built packages +ARG CACHEBUST=1 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y; \ + apt-get autoremove -y diff --git a/packaging/Dockerfile.ubuntu.18.10 b/packaging/Dockerfile.ubuntu.18.10 new file mode 100644 index 00000000000..5a3c766f96c --- /dev/null +++ b/packaging/Dockerfile.ubuntu.18.10 @@ -0,0 +1,24 @@ +# +# Copyright 2019, Intel Corporation +# +# 'recipe' for Docker to build an Debian package +# +# Pull base image +FROM ubuntu:18.10 +Maintainer John.E.Malmberg + +# Install basic tools +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + autoconf bash curl debhelper dh-make dpkg-dev doxygen gcc \ + git git-buildpackage locales make patch rpm wget + +# libfabric +RUN apt-get install -y \ + libibverbs-dev libpsm-infinipath1-dev librdmacm-dev \ + libpsm2-dev + +# force an upgrade to get any newly built packages +ARG CACHEBUST=1 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y; \ + apt-get autoremove -y + diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk new file mode 100644 index 00000000000..e3da977c7c5 --- /dev/null +++ b/packaging/Makefile_packaging.mk @@ -0,0 +1,341 @@ +# Common Makefile for including +# Needs the following variables set at a minium: +# NAME := +# SRC_EXT := +# SOURCE = + +ifeq ($(DEB_NAME),) +DEB_NAME := $(NAME) +endif + +CALLING_MAKEFILE := $(word 1, $(MAKEFILE_LIST)) + +# Find out what we are +ID_LIKE := $(shell . /etc/os-release; echo $$ID_LIKE) +# Of course that does not work for SLES-12 +ID := $(shell . /etc/os-release; echo $$ID) +VERSION_ID := $(shell . /etc/os-release; echo $$VERSION_ID) +ifeq ($(findstring opensuse,$(ID)),opensuse) +ID_LIKE := suse +DISTRO_ID := sl$(VERSION_ID) +endif +ifeq ($(ID),sles) +# SLES-12 or 15 detected. +ID_LIKE := suse +DISTRO_ID := sle$(VERSION_ID) +endif + +BUILD_OS ?= leap.42.3 +PACKAGING_CHECK_DIR ?= ../packaging +COMMON_RPM_ARGS := --define "%_topdir $$PWD/_topdir" +DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) +ifeq ($(DIST),) +SED_EXPR := 1p +else +SED_EXPR := 1s/$(DIST)//p +endif +SPEC := $(NAME).spec +VERSION := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{version}\n' $(SPEC) | sed -n '1p') +DOT := . +DEB_VERS := $(subst rc,~rc,$(VERSION)) +DEB_RVERS := $(subst $(DOT),\$(DOT),$(DEB_VERS)) +DEB_BVERS := $(basename $(subst ~rc,$(DOT)rc,$(DEB_VERS))) +RELEASE := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{release}\n' $(SPEC) | sed -n '$(SED_EXPR)') +SRPM := _topdir/SRPMS/$(NAME)-$(VERSION)-$(RELEASE)$(DIST).src.rpm +RPMS := $(addsuffix .rpm,$(addprefix _topdir/RPMS/x86_64/,$(shell rpm --specfile $(SPEC)))) +DEB_TOP := _topdir/BUILD +DEB_BUILD := $(DEB_TOP)/$(NAME)-$(DEB_VERS) +DEB_TARBASE := $(DEB_TOP)/$(DEB_NAME)_$(DEB_VERS) +SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES)) +ifeq ($(ID_LIKE),debian) +DEBS := $(addsuffix _$(DEB_VERS)-1_amd64.deb,$(shell sed -n '/-udeb/b; s,^Package:[[:blank:]],$(DEB_TOP)/,p' debian/control)) +#Ubuntu Containers do not set a UTF-8 environment by default. +ifndef LANG +export LANG = C.UTF-8 +endif +ifndef LC_ALL +export LC_ALL = C.UTF-8 +endif +TARGETS := $(DEBS) +else +# CentOS/Suse packages that want a locale set need this. +ifndef LANG +export LANG = en_US.utf8 +endif +ifndef LC_ALL +export LC_ALL = en_US.utf8 +endif +TARGETS := $(RPMS) $(SRPM) +endif +all: $(TARGETS) + +%/: + mkdir -p $@ + +%.gz: % + rm -f $@ + gzip $< + +_topdir/SOURCES/%: % | _topdir/SOURCES/ + rm -f $@ + ln $< $@ + +# At least one spec file, SLURM (sles), has a different version for the +# download file than the version in the spec file. +ifeq ($(DL_VERSION),) +DL_VERSION = $(VERSION) +endif + +$(NAME)-$(DL_VERSION).tar.$(SRC_EXT).asc: $(NAME).spec $(CALLING_MAKEFILE) + rm -f ./$(NAME)-*.tar.{gz,bz*,xz}.asc + curl -f -L -O '$(SOURCE).asc' + +$(NAME)-$(DL_VERSION).tar.$(SRC_EXT): $(NAME).spec $(CALLING_MAKEFILE) + rm -f ./$(NAME)-*.tar.{gz,bz*,xz} + curl -f -L -O '$(SOURCE)' + +v$(DL_VERSION).tar.$(SRC_EXT): $(NAME).spec $(CALLING_MAKEFILE) + rm -f ./v*.tar.{gz,bz*,xz} + curl -f -L -O '$(SOURCE)' + +$(DL_VERSION).tar.$(SRC_EXT): $(NAME).spec $(CALLING_MAKEFILE) + rm -f ./*.tar.{gz,bz*,xz} + curl -f -L -O '$(SOURCE)' + +$(DEB_TOP)/%: % | $(DEB_TOP)/ + +$(DEB_BUILD)/%: % | $(DEB_BUILD)/ + +$(DEB_BUILD).tar.$(SRC_EXT): $(notdir $(SOURCE)) | $(DEB_TOP)/ + ln -f $< $@ + +$(DEB_TARBASE).orig.tar.$(SRC_EXT) : $(DEB_BUILD).tar.$(SRC_EXT) + rm -f $(DEB_TOP)/*.orig.tar.* + ln -f $< $@ + +$(DEB_TOP)/.detar: $(notdir $(SOURCE)) $(DEB_TARBASE).orig.tar.$(SRC_EXT) + # Unpack tarball + rm -rf ./$(DEB_BUILD)/* + mkdir -p $(DEB_BUILD) + tar -C $(DEB_BUILD) --strip-components=1 -xpf $< + touch $@ + +# Extract patches for Debian +$(DEB_TOP)/.patched: $(PATCHES) check-env $(DEB_TOP)/.detar | \ + $(DEB_BUILD)/debian/ + mkdir -p ${DEB_BUILD}/debian/patches + mkdir -p $(DEB_TOP)/patches + for f in $(PATCHES); do \ + rm -f $(DEB_TOP)/patches/*; \ + if git mailsplit -o$(DEB_TOP)/patches < "$$f" ;then \ + fn=$$(basename "$$f"); \ + for f1 in $(DEB_TOP)/patches/*;do \ + [ -e "$$f1" ] || continue; \ + f1n=$$(basename "$$f1"); \ + echo "$${fn}_$${f1n}" >> $(DEB_BUILD)/debian/patches/series ; \ + mv "$$f1" $(DEB_BUILD)/debian/patches/$${fn}_$${f1n}; \ + done; \ + else \ + fb=$$(basename "$$f"); \ + cp "$$f" $(DEB_BUILD)/debian/patches/ ; \ + echo "$$fb" >> $(DEB_BUILD)/debian/patches/series ; \ + if ! grep -q "^Description:\|^Subject:" "$$f" ;then \ + sed -i '1 iSubject: Auto added patch' \ + "$(DEB_BUILD)/debian/patches/$$fb" ;fi ; \ + if ! grep -q "^Origin:\|^Author:\|^From:" "$$f" ;then \ + sed -i '1 iOrigin: other' \ + "$(DEB_BUILD)/debian/patches/$$fb" ;fi ; \ + fi ; \ + done + touch $@ + + +# Move the debian files into the Debian directory. +ifeq ($(ID_LIKE),debian) +$(DEB_TOP)/.deb_files : $(shell find debian -type f) \ + $(DEB_TOP)/.detar | \ + $(DEB_BUILD)/debian/ + find debian -maxdepth 1 -type f -exec cp '{}' '$(DEB_BUILD)/{}' ';' + if [ -e debian/source ]; then \ + cp -r debian/source $(DEB_BUILD)/debian; fi + if [ -e debian/local ]; then \ + cp -r debian/local $(DEB_BUILD)/debian; fi + if [ -e debian/examples ]; then \ + cp -r debian/examples $(DEB_BUILD)/debian; fi + if [ -e debian/upstream ]; then \ + cp -r debian/upstream $(DEB_BUILD)/debian; fi + if [ -e debian/tests ]; then \ + cp -r debian/tests $(DEB_BUILD)/debian; fi + rm -f $(DEB_BUILD)/debian/*.ex $(DEB_BUILD)/debian/*.EX + rm -f $(DEB_BUILD)/debian/*.orig + touch $@ +endif + +# see https://stackoverflow.com/questions/2973445/ for why we subst +# the "rpm" for "%" to effectively turn this into a multiple matching +# target pattern rule +$(subst rpm,%,$(RPMS)): $(SPEC) $(SOURCES) + rpmbuild -bb $(COMMON_RPM_ARGS) $(RPM_BUILD_OPTIONS) $(SPEC) + +$(subst deb,%,$(DEBS)): $(DEB_BUILD).tar.$(SRC_EXT) \ + $(DEB_TOP)/.deb_files $(DEB_TOP)/.detar $(DEB_TOP)/.patched + rm -f $(DEB_TOP)/*.deb $(DEB_TOP)/*.ddeb $(DEB_TOP)/*.dsc \ + $(DEB_TOP)/*.dsc $(DEB_TOP)/*.build* $(DEB_TOP)/*.changes \ + $(DEB_TOP)/*.debian.tar.* + rm -rf $(DEB_TOP)/*-tmp + cd $(DEB_BUILD); debuild --no-lintian -b -us -uc + cd $(DEB_BUILD); debuild -- clean + git status + rm -rf $(DEB_TOP)/$(NAME)-tmp + lfile1=$(shell echo $(DEB_TOP)/$(NAME)[0-9]*_$(DEB_VERS)-1_amd64.deb);\ + lfile=$$(ls $${lfile1}); \ + lfile2=$${lfile##*/}; lname=$${lfile2%%_*}; \ + dpkg-deb -R $${lfile} \ + $(DEB_TOP)/$(NAME)-tmp; \ + if [ -e $(DEB_TOP)/$(NAME)-tmp/DEBIAN/symbols ]; then \ + sed 's/$(DEB_RVERS)-1/$(DEB_BVERS)/' \ + $(DEB_TOP)/$(NAME)-tmp/DEBIAN/symbols \ + > $(DEB_BUILD)/debian/$${lname}.symbols; fi + cd $(DEB_BUILD); debuild -us -uc + rm $(DEB_BUILD).tar.$(SRC_EXT) + for f in $(DEB_TOP)/*.deb; do \ + echo $$f; dpkg -c $$f; done + +$(SRPM): $(SPEC) $(SOURCES) + rpmbuild -bs $(COMMON_RPM_ARGS) $(RPM_BUILD_OPTIONS) $(SPEC) + +srpm: $(SRPM) + +$(RPMS): $(SRPM) $(CALLING_MAKEFILE) + +rpms: $(RPMS) + +$(DEBS): $(CALLING_MAKEFILE) + +debs: $(DEBS) + +ls: $(TARGETS) + ls -ld $^ + +ifeq ($(ID_LIKE),rhel fedora) +chrootbuild: $(SRPM) $(CALLING_MAKEFILE) + if [ -w /etc/mock/default.cfg ]; then \ + echo -e "config_opts['yum.conf'] += \"\"\"\n" >> /etc/mock/default.cfg; \ + for repo in $(ADD_REPOS); do \ + if [[ $$repo = *@* ]]; then \ + branch="$${repo#*@}"; \ + repo="$${repo%@*}"; \ + else \ + branch="master"; \ + fi; \ + echo -e "[$$repo:$$branch:lastSuccessful]\n\ +name=$$repo:$$branch:lastSuccessful\n\ +baseurl=$${JENKINS_URL}job/daos-stack/job/$$repo/job/$$branch/lastSuccessfulBuild/artifact/artifacts/centos7/\n\ +enabled=1\n\ +gpgcheck = False\n" >> /etc/mock/default.cfg; \ + done; \ + echo "\"\"\"" >> /etc/mock/default.cfg; \ + else \ + echo "Unable to update /etc/mock/default.cfg."; \ + echo "You need to make sure it has the needed repos in it yourself."; \ + fi + mock $(MOCK_OPTIONS) $(RPM_BUILD_OPTIONS) $< +else +sle12_REPOS += --repo https://download.opensuse.org/repositories/science:/HPC/openSUSE_Leap_42.3/ \ + --repo http://cobbler/cobbler/repo_mirror/sdkupdate-sles12.3-x86_64/ \ + --repo http://cobbler/cobbler/repo_mirror/sdk-sles12.3-x86_64 \ + --repo http://download.opensuse.org/repositories/openSUSE:/Backports:/SLE-12/standard/ \ + --repo http://cobbler/cobbler/repo_mirror/updates-sles12.3-x86_64 \ + --repo http://cobbler/cobbler/pub/SLES-12.3-x86_64/ + +sl42_REPOS += --repo https://download.opensuse.org/repositories/science:/HPC/openSUSE_Leap_42.3 \ + --repo http://download.opensuse.org/update/leap/42.3/oss/ \ + --repo http://download.opensuse.org/distribution/leap/42.3/repo/oss/suse/ + +sl15_REPOS += --repo http://download.opensuse.org/update/leap/15.1/oss/ \ + --repo http://download.opensuse.org/distribution/leap/15.1/repo/oss/ + +chrootbuild: $(SRPM) $(CALLING_MAKEFILE) + add_repos=""; \ + for repo in $(ADD_REPOS); do \ + if [[ $$repo = *@* ]]; then \ + branch="$${repo#*@}"; \ + repo="$${repo%@*}"; \ + else \ + branch="master"; \ + fi; \ + case $(DISTRO_ID) in \ + sle12.3) distro="sles12.3"; \ + ;; \ + sl42.3) distro="leap42.3"; \ + ;; \ + sl15.1) distro="leap15.1"; \ + ;; \ + esac; \ + baseurl=$${JENKINS_URL}job/daos-stack/job/$$repo/job/$$branch/; \ + baseurl+=lastSuccessfulBuild/artifact/artifacts/$$distro/; \ + add_repos+=" --repo $$baseurl"; \ + done; \ + curl -O http://download.opensuse.org/repositories/science:/HPC/openSUSE_Leap_42.3/repodata/repomd.xml.key; \ + sudo rpm --import repomd.xml.key; \ + sudo build $(BUILD_OPTIONS) $$add_repos \ + $($(basename $(DISTRO_ID))_REPOS) \ + --dist $(DISTRO_ID) $(RPM_BUILD_OPTIONS) $(SRPM) +endif + +docker_chrootbuild: + docker build --build-arg UID=$$(id -u) -t $(BUILD_OS)-chrootbuild \ + -f packaging/Dockerfile.$(BUILD_OS) . + docker run --privileged=true -w $$PWD -v=$$PWD:$$PWD \ + -it $(BUILD_OS)-chrootbuild bash -c "make chrootbuild" + +rpmlint: $(SPEC) + rpmlint $< + +packaging_check: + diff --exclude \*.sw? \ + --exclude debian \ + --exclude .git \ + --exclude Jenkinsfile \ + --exclude libfabric.spec \ + --exclude Makefile \ + --exclude README.md \ + --exclude _topdir \ + --exclude \*.tar.\* \ + -bur $(PACKAGING_CHECK_DIR)/ packaging/ + +check-env: +ifndef DEBEMAIL + $(error DEBEMAIL is undefined) +endif +ifndef DEBFULLNAME + $(error DEBFULLNAME is undefined) +endif + +show_version: + @echo $(VERSION) + +show_release: + @echo $(RELEASE) + +show_rpms: + @echo $(RPMS) + +show_source: + @echo $(SOURCE) + +show_sources: + @echo $(SOURCES) + +show_targets: + @echo $(TARGETS) + +show_makefiles: + @echo $(MAKEFILE_LIST) + +show_calling_makefile: + @echo $(CALLING_MAKEFILE) + +.PHONY: srpm rpms debs ls chrootbuild rpmlint FORCE \ + show_version show_release show_rpms show_source show_sources \ + show_targets check-env From 25e015e665d23254029a749c2ac12b66d70d8733 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 3 Sep 2019 11:32:53 -0400 Subject: [PATCH 2/7] CORCI-635 test: Create romio tarball So that an RPM can be created to install the ROMIO tests. Signed-off-by: Brian J. Murrell --- Jenkinsfile | 4 +++- Makefile-rpm.mk | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3592fa1aea3..350082ad52d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -112,14 +112,16 @@ pipeline { sh label: "Build package", script: '''rm -rf artifacts/centos7/ mkdir -p artifacts/centos7/ - make -f Makefile-rpm.mk chrootbuild''' + make -f Makefile-rpm.mk chrootbuild romio.tar.gz''' } post { success { sh label: "Collect artifacts", script: '''(cd /var/lib/mock/epel-7-x86_64/result/ && cp -r . $OLDPWD/artifacts/centos7/) + cp romio.tar.gz artifacts/ createrepo artifacts/centos7/''' + archiveArtifacts artifacts: 'artifacts/romio.tar.gz' } unsuccessful { sh label: "Collect artifacts", diff --git a/Makefile-rpm.mk b/Makefile-rpm.mk index 0ea35f5eea1..eb158fd16da 100644 --- a/Makefile-rpm.mk +++ b/Makefile-rpm.mk @@ -113,3 +113,17 @@ $(NAME)-$(DL_VERSION).tar.$(SRC_EXT): $(NAME)-$(DL_VERSION)-$(GIT_COMMIT).tar fi rm -f $@ gzip < $< > $@ + +romio.tar.gz: + set -e; \ + romio_dir=_topdir/BUILD/mpich-$(VERSION)/src/mpi/romio; \ + if grep "MPI_LIB = /" $$romio_dir/test/Makefile; then \ + trap 'mv $$romio_dir/test/Makefile{.old,}' EXIT; \ + mv $$romio_dir/test/Makefile{,.old}; \ + sed -e 's/\(MPI_LIB = \)\/\(.*\)/\1-L\/\2/' \ + < $$romio_dir/test/Makefile.old \ + > $$romio_dir/test/Makefile; \ + fi; \ + make -C $$romio_dir clean; \ + tar -C $$romio_dir/.. --exclude Makefile.old \ + -czf romio.tar.gz romio From 05ee51088762ade3e40705729b3a87cc92538ae8 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 3 Sep 2019 12:38:15 -0400 Subject: [PATCH 3/7] Updates Signed-off-by: Brian J. Murrell --- Jenkinsfile | 3 ++- Makefile-rpm.mk | 35 ++++++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 350082ad52d..c7bd9233ed6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -112,7 +112,8 @@ pipeline { sh label: "Build package", script: '''rm -rf artifacts/centos7/ mkdir -p artifacts/centos7/ - make -f Makefile-rpm.mk chrootbuild romio.tar.gz''' + make CHROOT=true -f Makefile-rpm.mk chrootbuild \ + romio.tar.gz''' } post { success { diff --git a/Makefile-rpm.mk b/Makefile-rpm.mk index eb158fd16da..d26f68aed4e 100644 --- a/Makefile-rpm.mk +++ b/Makefile-rpm.mk @@ -9,6 +9,7 @@ PATCHES := mpich.macros mpich.pth.py2 mpich.pth.py3 mpich-modules.patch fix-vers ADD_REPOS := openpa libfabric pmix ompi mercury spdk isa-l fio dpdk \ protobuf-c fuse pmdk argobots raft cart@daos_devel1 daos \ automake libtool +ROMIO_SRC ?= _topdir GIT_TAG := v3.3 @@ -114,16 +115,28 @@ $(NAME)-$(DL_VERSION).tar.$(SRC_EXT): $(NAME)-$(DL_VERSION)-$(GIT_COMMIT).tar rm -f $@ gzip < $< > $@ +CHROOT ?= false + romio.tar.gz: - set -e; \ - romio_dir=_topdir/BUILD/mpich-$(VERSION)/src/mpi/romio; \ - if grep "MPI_LIB = /" $$romio_dir/test/Makefile; then \ - trap 'mv $$romio_dir/test/Makefile{.old,}' EXIT; \ - mv $$romio_dir/test/Makefile{,.old}; \ - sed -e 's/\(MPI_LIB = \)\/\(.*\)/\1-L\/\2/' \ - < $$romio_dir/test/Makefile.old \ - > $$romio_dir/test/Makefile; \ - fi; \ - make -C $$romio_dir clean; \ - tar -C $$romio_dir/.. --exclude Makefile.old \ + set -e; \ + if $(CHROOT); then \ + romio_prefix=/var/lib/mock/epel-7-x86_64/root/builddir/build; \ + chroot_romio_prefix=/builddir/build; \ + else \ + romio_prefix=_topdir; \ + fi; \ + romio_dir=BUILD/mpich-$(VERSION)/src/mpi/romio; \ + if grep "MPI_LIB = /" $$romio_prefix/$$romio_dir/test/Makefile; then \ + trap 'mv $$romio_prefix/$$romio_dir/test/Makefile{.old,}' EXIT; \ + mv $$romio_prefix/$$romio_dir/test/Makefile{,.old}; \ + sed -e 's/\(MPI_LIB = \)\/\(.*\)/\1-L\/\2/' \ + < $$romio_prefix/$$romio_dir/test/Makefile.old \ + > $$romio_prefix/$$romio_dir/test/Makefile; \ + fi; \ + if $(CHROOT); then \ + mock --chroot "make -C $$chroot_romio_prefix/$$romio_dir clean"; \ + else \ + make -C $$romio_prefix/$$romio_dir clean; \ + fi; \ + tar -C $$romio_prefix/$$romio_dir/.. --exclude Makefile.old \ -czf romio.tar.gz romio From 3c5328ec48779a69bf4d4bce6028a1a26020a8fe Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 3 Sep 2019 16:37:05 -0400 Subject: [PATCH 4/7] Updates Signed-off-by: Brian J. Murrell --- Jenkinsfile | 15 +++------------ Makefile-rpm.mk | 8 ++++++-- mpich.spec | 2 ++ 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c7bd9233ed6..c6c6e17bedc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -87,7 +87,6 @@ pipeline { [$class: 'RequesterRecipientProvider'] ], subject: 'Packaging is out of date for ' + jobName() - } } } //stage('Check Packaging') @@ -113,16 +112,16 @@ pipeline { script: '''rm -rf artifacts/centos7/ mkdir -p artifacts/centos7/ make CHROOT=true -f Makefile-rpm.mk chrootbuild \ - romio.tar.gz''' + romio-tarball''' } post { success { sh label: "Collect artifacts", script: '''(cd /var/lib/mock/epel-7-x86_64/result/ && cp -r . $OLDPWD/artifacts/centos7/) - cp romio.tar.gz artifacts/ + cp romio-*.tar.gz artifacts/ createrepo artifacts/centos7/''' - archiveArtifacts artifacts: 'artifacts/romio.tar.gz' + archiveArtifacts artifacts: 'artifacts/romio-*.tar.gz' } unsuccessful { sh label: "Collect artifacts", @@ -148,14 +147,6 @@ pipeline { } } } //stage('Build on CentOS 7') - stage('Build on Ubuntu 18.04') { - agent { - label 'docker_runner' - } - steps { - echo "Building on Ubuntu is not implemented for the moment" - } - } } } } diff --git a/Makefile-rpm.mk b/Makefile-rpm.mk index d26f68aed4e..05670a0b762 100644 --- a/Makefile-rpm.mk +++ b/Makefile-rpm.mk @@ -117,7 +117,7 @@ $(NAME)-$(DL_VERSION).tar.$(SRC_EXT): $(NAME)-$(DL_VERSION)-$(GIT_COMMIT).tar CHROOT ?= false -romio.tar.gz: +romio-$(VERSION).tar.gz: set -e; \ if $(CHROOT); then \ romio_prefix=/var/lib/mock/epel-7-x86_64/root/builddir/build; \ @@ -139,4 +139,8 @@ romio.tar.gz: make -C $$romio_prefix/$$romio_dir clean; \ fi; \ tar -C $$romio_prefix/$$romio_dir/.. --exclude Makefile.old \ - -czf romio.tar.gz romio + -czf romio-$(VERSION).tar.gz romio + +romio-tarball: romio-$(VERSION).tar.gz + +.PHONY: romio-tarball diff --git a/mpich.spec b/mpich.spec index 4ccd2ccc1cf..4952d37328e 100644 --- a/mpich.spec +++ b/mpich.spec @@ -91,6 +91,7 @@ Provides: %{name}-devel-static = %{version}-%{release} Requires: %{name} = %{version}-%{release} Requires: pkgconfig Requires: gcc-gfortran +Requires: daos-devel %if (0%{?fedora} >= 30) Requires: rpm-mpi-hooks %endif @@ -336,6 +337,7 @@ make check VERBOSE=1 %changelog * Fri Aug 30 2019 Brian J. Murrell - 3.3-1 - Update packaging standards +- mpich-devel should Requires: daos-devel * Fri Aug 30 2019 Brian J. Murrell - 3.3-0.04 - Fix ABI version after upstream master merge From cd0e0c4064f8a3101db5607a5a6f9b6bf363606b Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 13 Sep 2019 12:35:36 -0400 Subject: [PATCH 5/7] Use packageBuildingPipeline Signed-off-by: Brian J. Murrell --- Jenkinsfile | 118 +++------------------------------------------------- 1 file changed, 6 insertions(+), 112 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c6c6e17bedc..4945a67a369 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,116 +38,10 @@ */ // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes -//@Library(value="pipeline-lib@your_branch") _ +@Library(value="pipeline-lib@package-building-pipeline") _ -pipeline { - agent none - - stages { - stage('Cancel Previous Builds') { - when { changeRequest() } - steps { - cancelPreviousBuilds() - } - } - stage('Lint') { - parallel { - stage('RPM Lint') { - agent { - dockerfile { - filename 'packaging/Dockerfile.centos.7' - label 'docker_runner' - args '--group-add mock' + - ' --cap-add=SYS_ADMIN' + - ' --privileged=true' - additionalBuildArgs '--build-arg UID=$(id -u)' - } - } - steps { - sh 'make -f Makefile-rpm.mk rpmlint' - } - } - stage('Check Packaging') { - agent { label 'lightweight' } - steps { - checkoutScm url: 'https://github.com/daos-stack/packaging.git', - checkoutDir: 'packaging-module' - catchError(stageResult: 'UNSTABLE', buildResult: 'SUCCESS') { - sh '''make -f Makefile-rpm.mk \ - PACKAGING_CHECK_DIR=packaging-module \ - packaging_check''' - } - } - post { - unsuccessful { - emailext body: 'Packaging out of date for ' + jobName() + '.\n' + - "You should update it and submit your PR again.", - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - subject: 'Packaging is out of date for ' + jobName() - } - } - } //stage('Check Packaging') - } // parallel - } //stage('Lint') - stage('Build') { - parallel { - stage('Build on CentOS 7') { - agent { - dockerfile { - filename 'packaging/Dockerfile.centos.7' - label 'docker_runner' - args '--group-add mock' + - ' --cap-add=SYS_ADMIN' + - ' --privileged=true' - additionalBuildArgs '--build-arg UID=$(id -u)' + - ' --build-arg JENKINS_URL=' + - env.JENKINS_URL - } - } - steps { - sh label: "Build package", - script: '''rm -rf artifacts/centos7/ - mkdir -p artifacts/centos7/ - make CHROOT=true -f Makefile-rpm.mk chrootbuild \ - romio-tarball''' - } - post { - success { - sh label: "Collect artifacts", - script: '''(cd /var/lib/mock/epel-7-x86_64/result/ && - cp -r . $OLDPWD/artifacts/centos7/) - cp romio-*.tar.gz artifacts/ - createrepo artifacts/centos7/''' - archiveArtifacts artifacts: 'artifacts/romio-*.tar.gz' - } - unsuccessful { - sh label: "Collect artifacts", - script: '''mockroot=/var/lib/mock/epel-7-x86_64 - artdir=$PWD/artifacts/centos7 - cp -af _topdir/SRPMS $artdir - (cd $mockroot/result/ && - cp -r . $artdir) - (if cd $mockroot/root/builddir/build/BUILD/*/; then - find . -name configure -printf %h\\\\n | \ - while read dir; do - if [ ! -f $dir/config.log ]; then - continue - fi - tdir="$artdir/autoconf-logs/$dir" - mkdir -p $tdir - cp -a $dir/config.log $tdir/ - done - fi)''' - } - cleanup { - archiveArtifacts artifacts: 'artifacts/centos7/**' - } - } - } //stage('Build on CentOS 7') - } - } - } -} +packageBuildingPipeline(['distros' : 'centos7', + 'name' : 'mpich' + 'make args' : 'CHROOT=true -f Makefile-rpm.mk', + 'add_make_targets' : 'romio-tarball', + 'add_archiving_cmds' : 'cp romio-*.tar.gz artifacts/']) From 3506bae4871caa1ce31bb9885977f4675e08dec9 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 13 Sep 2019 12:45:04 -0400 Subject: [PATCH 6/7] Missing , Signed-off-by: Brian J. Murrell --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4945a67a369..45678d661ae 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,7 +41,7 @@ @Library(value="pipeline-lib@package-building-pipeline") _ packageBuildingPipeline(['distros' : 'centos7', - 'name' : 'mpich' + 'name' : 'mpich', 'make args' : 'CHROOT=true -f Makefile-rpm.mk', 'add_make_targets' : 'romio-tarball', 'add_archiving_cmds' : 'cp romio-*.tar.gz artifacts/']) From 5c0594babdd0fa283911ca97426e8ab48d3f50d6 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 13 Sep 2019 12:54:59 -0400 Subject: [PATCH 7/7] Archive romio tarball Signed-off-by: Brian J. Murrell --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 45678d661ae..25bd174af49 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,4 +44,5 @@ packageBuildingPipeline(['distros' : 'centos7', 'name' : 'mpich', 'make args' : 'CHROOT=true -f Makefile-rpm.mk', 'add_make_targets' : 'romio-tarball', - 'add_archiving_cmds' : 'cp romio-*.tar.gz artifacts/']) + 'add_archiving_cmds' : 'cp romio-*.tar.gz artifacts/', + 'add_artifacts': 'artifacts/romio-*.tar.gz'])