daos-stack · brianjmurrell · Aug 30, 2019 · Sep 3, 2019 · Sep 3, 2019 · Sep 3, 2019
diff --git a/Dockerfile.centos.7 b/Dockerfile.centos.7
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,76 +1,48 @@
-pipeline {
-    agent none
+#!/usr/bin/groovy
+/* Copyright (C) 2019 Intel Corporation
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted for any purpose (including commercial purposes)
+ * provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions, and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions, and the following disclaimer in the
+ *    documentation and/or materials provided with the distribution.
+ *
+ * 3. In addition, redistributions of modified forms of the source or binary
+ *    code must carry prominent notices stating that the original code was
+ *    changed and the date of the change.
+ *
+ *  4. All publications or advertising materials mentioning features or use of
+ *     this software are asked, but not required, to acknowledge that it was
+ *     developed by Intel Corporation and credit the contributors.
+ *
+ * 5. Neither the name of Intel Corporation, nor the name of any Contributor
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+// To use a test branch (i.e. PR) until it lands to master
+// I.e. for testing library changes
+@Library(value="pipeline-lib@package-building-pipeline") _
 
-    stages {
-        stage('Lint') {
-            stages {
-                stage('RPM Lint') {
-                    agent {
-                        dockerfile {
-                            filename 'Dockerfile.centos.7'
-                            label 'docker_runner'
-                            additionalBuildArgs  '--build-arg UID=$(id -u)'
-                            args  '--group-add mock --cap-add=SYS_ADMIN --privileged=true'
-                        }
-                    }
-                    steps {
-                        sh 'make -f Makefile-rpm.mk rpmlint'
-                    }
-                }
-            }
-        }
-        stage('Build') {
-            parallel {
-                stage('Build on CentOS 7') {
-                    agent {
-                        dockerfile {
-                            filename 'Dockerfile.centos.7'
-                            label 'docker_runner'
-                            additionalBuildArgs '--build-arg UID=$(id -u) --build-arg JENKINS_URL=' +
-                                                env.JENKINS_URL
-                            args  '--group-add mock --cap-add=SYS_ADMIN --privileged=true'
-                        }
-                    }
-                    steps {
-                        sh '''rm -rf artifacts/centos7/
-                              mkdir -p artifacts/centos7/
-                              make -f Makefile-rpm.mk srpm
-                              make -f Makefile-rpm.mk mockbuild'''
-                    }
-                    post {
-                        success {
-                            sh '''(cd /var/lib/mock/epel-7-x86_64/result/ &&
-                                   cp -r . $OLDPWD/artifacts/centos7/)
-                                  createrepo artifacts/centos7/'''
-                            archiveArtifacts artifacts: 'artifacts/centos7/**'
-                        }
-                        failure {
-                            sh '''cp -af _topdir/SRPMS artifacts/centos7/
-                                  (cd /var/lib/mock/epel-7-x86_64/result/ &&
-                                   cp -r . $OLDPWD/artifacts/centos7/)
-                                  (cd /var/lib/mock/epel-7-x86_64/root/builddir/build/BUILD/*/
-                                   find . -name configure -printf %h\\\\n | \
-                                   while read dir; do
-                                       if [ ! -f $dir/config.log ]; then
-                                           continue
-                                       fi
-                                       tdir="$OLDPWD/artifacts/centos7/autoconf-logs/$dir"
-                                       mkdir -p $tdir
-                                       cp -a $dir/config.log $tdir/
-                                   done)'''
-                            archiveArtifacts artifacts: 'artifacts/centos7/**'
-                        }
-                    }
-                }
-                stage('Build on Ubuntu 16.04') {
-                    agent {
-                        label 'docker_runner'
-                    }
-                    steps {
-                        echo "Building on Ubuntu is not implemented for the moment"
-                    }
-                }
-            }
-        }
-    }
-}
+packageBuildingPipeline(['distros' : 'centos7',
+                         'name' : 'mpich',
+                         'make args' : 'CHROOT=true -f Makefile-rpm.mk',
+                         'add_make_targets' : 'romio-tarball',
+                         'add_archiving_cmds' : 'cp romio-*.tar.gz artifacts/',
+                         'add_artifacts': 'artifacts/romio-*.tar.gz'])
diff --git a/Makefile-rpm.mk b/Makefile-rpm.mk
@@ -6,6 +6,11 @@ PATCHES := mpich.macros mpich.pth.py2 mpich.pth.py3 mpich-modules.patch fix-vers
 #	   daos_adio-hwloc.patch daos_adio-izem.patch daos_adio-ucx.patch      \
 #	   daos_adio-libfabric.patch
 # daos_adio-all.patch
+ADD_REPOS := openpa libfabric pmix ompi mercury spdk isa-l fio dpdk   \
+	     protobuf-c fuse pmdk argobots raft cart@daos_devel1 daos \
+	     automake libtool
+ROMIO_SRC ?= _topdir
+
 GIT_TAG := v3.3
 
 TOPDIR  := $(shell echo $$PWD)
@@ -33,7 +38,6 @@ daos_adio-all.patch:
 	         --exclude \*.gz            \
 	         pristine/* . > $@ || true
 	echo rm -rf pristine
-
 # so instead we get a patch for each submodule
 define gen_submod_patch
 	set -e;                                              \
@@ -81,102 +85,62 @@ daos_adio.patch:
 	    -x src/mpid/ch4/netmod/ucx/ucx                 \
 	    > $@
 
-COMMON_RPM_ARGS := --define "%_topdir $$PWD/_topdir"
-DIST    := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist})
-ifeq ($(DIST),)
-SED_EXPR := 1p
-else
-SED_EXPR := 1s/$(DIST)//p
-endif
-SPEC    := $(NAME).spec
-VERSION := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{version}\n' $(SPEC) | sed -n '1p')
-RELEASE := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{release}\n' $(SPEC) | sed -n '$(SED_EXPR)')
-SRPM    := _topdir/SRPMS/$(NAME)-$(VERSION)-$(RELEASE)$(DIST).src.rpm
-RPMS    := $(addsuffix .rpm,$(addprefix _topdir/RPMS/x86_64/,$(shell rpm --specfile $(SPEC))))
-SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES))
-TARGETS := $(RPMS) $(SRPM)
-
-all: $(TARGETS)
-
-%/:
-	mkdir -p $@
-
-_topdir/SOURCES/%: % | _topdir/SOURCES/
-	rm -f $@
-	ln $< $@
-
-$(NAME)-$(VERSION).tar.$(SRC_EXT).asc:
-	curl -f -L -O '$(SOURCE).asc'
+include packaging/Makefile_packaging.mk
 
-%.gz: %
-	rm -f $@
-	gzip $<
+GIT_COMMIT := $(shell git rev-parse --short HEAD)
 
-#$(NAME)-$(VERSION).tar.$(SRC_EXT):
-#	curl -f -L -O '$(SOURCE)'
-$(NAME)-$(VERSION).tar: Makefile-rpm.mk
+$(NAME)-$(DL_VERSION)-$(GIT_COMMIT).tar:
 	mkdir -p rpmbuild
-	git archive --prefix=$(subst .tar,,$@)/ -o $@ HEAD
+	git archive --prefix=$(subst -$(GIT_COMMIT).tar,,$@)/ -o $@ HEAD
 	git submodule update --init
 	set -x; p=$$PWD && (echo .; git submodule foreach) |                          \
 	    while read junk path; do                                          \
 	    temp="$${path%\'}";                                               \
 	    temp="$${temp#\'}";                                               \
 	    path=$$temp;                                                      \
 	    [ "$$path" = "" ] && continue;                                    \
-	    (cd $$path && git archive --prefix=$(subst .tar,,$@)/$$path/ HEAD \
+	    (cd $$path && git archive --prefix=$(subst -$(GIT_COMMIT).tar,,$@)/$$path/ HEAD \
 	      > $$p/rpmbuild/tmp.tar &&                                       \
 	     tar --concatenate --file=$$p/$@                                  \
 	       $$p/rpmbuild/tmp.tar && rm $$p/rpmbuild/tmp.tar);              \
 	done
-	tar tvf $@
-
-v$(VERSION).tar.$(SRC_EXT):
-	curl -f -L -O '$(SOURCE)'
-
-$(VERSION).tar.$(SRC_EXT):
-	curl -f -L -O '$(SOURCE)'
-
-# see https://stackoverflow.com/questions/2973445/ for why we subst
-# the "rpm" for "%" to effectively turn this into a multiple matching
-# target pattern rule
-$(subst rpm,%,$(RPMS)): $(SPEC) $(SOURCES)
-	rpmbuild -bb $(COMMON_RPM_ARGS) $(RPM_BUILD_OPTIONS) $(SPEC)
-
-$(SRPM): $(SPEC) $(SOURCES)
-	rpmbuild -bs $(COMMON_RPM_ARGS) $(SPEC)
-
-srpm: $(SRPM) $(NAME).spec
-
-$(RPMS): Makefile-rpm.mk
-
-rpms: $(RPMS)
-
-ls: $(TARGETS)
-	ls -ld $^
-
-mockbuild: $(SRPM) Makefile-rpm.mk
-	mock $(MOCK_OPTIONS) $<
-
-rpmlint: $(SPEC)
-	rpmlint $<
-
-show_version:
-	@echo $(VERSION)
-
-show_release:
-	@echo $(RELEASE)
-
-show_rpms:
-	@echo $(RPMS)
-
-show_source:
-	@echo $(SOURCE)
-
-show_sources:
-	@echo $(SOURCES)
-
-show_spec:
-	@ls -l $(SPEC)
-
-.PHONY: srpm rpms ls mockbuild rpmlint FORCE show_version show_release show_rpms show_source show_sources
+	#tar tvf $@
+
+$(NAME)-$(DL_VERSION).tar.$(SRC_EXT): $(NAME)-$(DL_VERSION)-$(GIT_COMMIT).tar
+	older_tarballs=$$(ls $(NAME)-$(DL_VERSION)-*.tar |                   \
+	                  grep -v $(NAME)-$(DL_VERSION)-$(GIT_COMMIT).tar) ; \
+	if [ -n "$$older_tarballs" ]; then                                   \
+	    rm -f "$$older_tarballs";                                        \
+	fi
+	rm -f $@
+	gzip < $< > $@
+
+CHROOT ?= false
+
+romio-$(VERSION).tar.gz:
+	set -e;                                                              \
+	if $(CHROOT); then                                                   \
+	    romio_prefix=/var/lib/mock/epel-7-x86_64/root/builddir/build;    \
+	    chroot_romio_prefix=/builddir/build;                             \
+	else                                                                 \
+	    romio_prefix=_topdir;                                            \
+	fi;                                                                  \
+	romio_dir=BUILD/mpich-$(VERSION)/src/mpi/romio;                      \
+	if grep "MPI_LIB = /" $$romio_prefix/$$romio_dir/test/Makefile; then \
+	    trap 'mv $$romio_prefix/$$romio_dir/test/Makefile{.old,}' EXIT;  \
+	    mv $$romio_prefix/$$romio_dir/test/Makefile{,.old};              \
+	    sed -e 's/\(MPI_LIB = \)\/\(.*\)/\1-L\/\2/'                      \
+	        < $$romio_prefix/$$romio_dir/test/Makefile.old               \
+	        > $$romio_prefix/$$romio_dir/test/Makefile;                  \
+	fi;                                                                  \
+	if $(CHROOT); then                                                   \
+	    mock --chroot "make -C $$chroot_romio_prefix/$$romio_dir clean"; \
+	else                                                                 \
+	    make -C $$romio_prefix/$$romio_dir clean;                        \
+	fi;                                                                  \
+	tar -C $$romio_prefix/$$romio_dir/.. --exclude Makefile.old          \
+	    -czf romio-$(VERSION).tar.gz romio
+
+romio-tarball: romio-$(VERSION).tar.gz
+
+.PHONY: romio-tarball
diff --git a/mpich.spec b/mpich.spec
@@ -1,7 +1,7 @@
 Summary:        A high-performance implementation of MPI
 Name:           mpich
 Version:        3.3
-Release:        0.03%{?dist}
+Release:        1%{?dist}
 License:        MIT
 URL:            http://www.mpich.org/
 
@@ -91,6 +91,7 @@ Provides:       %{name}-devel-static = %{version}-%{release}
 Requires:       %{name} = %{version}-%{release}
 Requires:       pkgconfig
 Requires:       gcc-gfortran
+Requires:       daos-devel
 %if (0%{?fedora} >= 30)
 Requires:       rpm-mpi-hooks
 %endif
@@ -334,6 +335,10 @@ make check VERBOSE=1
 %{python3_sitearch}/%{name}.pth
 
 %changelog
+* Fri Aug 30 2019 Brian J. Murrell <brian.murrell@intel.com> - 3.3-1
+- Update packaging standards
+- mpich-devel should Requires: daos-devel
+
 * Fri Aug 30 2019 Brian J. Murrell <brian.murrell@intel.com> - 3.3-0.04
 - Fix ABI version after upstream master merge