From 3b4d3a21d94d2a98574c5e07b39b3bf0725b218c Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sun, 17 Nov 2019 09:37:58 +0100 Subject: [PATCH 01/20] 1st shot at build workflow --- .github/workflows/build-and-sign.yml | 109 +++++++++++++++++++++++++++ .gpghome/pubring.gpg | Bin 0 -> 988 bytes .gpghome/secring.gpg | Bin 0 -> 1063 bytes 3 files changed, 109 insertions(+) create mode 100644 .github/workflows/build-and-sign.yml create mode 100644 .gpghome/pubring.gpg create mode 100644 .gpghome/secring.gpg diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml new file mode 100644 index 0000000..97c9e41 --- /dev/null +++ b/.github/workflows/build-and-sign.yml @@ -0,0 +1,109 @@ +on: deployment +env: + USE_DOCKER: 1 +jobs: + prepare: + name: Prepare Gitian build environment + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + - name: Base VM cache + id: base-cache + uses: actions/cache@v1 + with: + path: vendor/gitian-builder/var + key: base-cache + - name: Prepare + if: steps.base-cache.outputs.cache-hit != 'true' + run: | + vendor/gitian-builder/bin/make-base-vm --docker --suite bionic + vendor/gitian-builder/bin/make-base-vm --docker --suite xenial + linux: + name: Build and sign linux binaries + needs: prepare + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + - name: Load Base Cache + uses: actions/cache@v1 + with: + path: vendor/gitian-builder/var + key: base-cache + - name: Load Build Cache + uses: actions/cache@v1 + with: + path: vendor/gitian-builder/cache/bitshares-core-linux + - name: Build and Sign + run: | + sudo apt-get install ruby + export GNUPGHOME=`pwd`/.gpghome + echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ + | ./run-gitian -b -s BitShares-Gitian-Auto-Build-Signer \ + -p "gpg --passphrase-fd 3" -O linux ${{ github.event.deployment.payload.coreversion }} -j 1 + - name: Upload build artifacts + uses: actions/upload-artifact@master + with: + name: Linux-Binaries + path: vendor/gitian-builder/build/out + mac: + name: Build and sign mac binaries + needs: prepare + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + - name: Load Base Cache + uses: actions/cache@v1 + with: + path: vendor/gitian-builder/var + key: base-cache + - name: Load Build Cache + uses: actions/cache@v1 + with: + path: vendor/gitian-builder/cache/bitshares-core-osx + - name: Build and Sign + run: | + sudo apt-get install ruby + export GNUPGHOME=`pwd`/.gpghome + echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ + | ./run-gitian -b -s BitShares-Gitian-Auto-Build-Signer \ + -p "gpg --passphrase-fd 3" -O linux ${{ github.event.deployment.payload.coreversion }} -j 1 + - name: Upload build artifacts + uses: actions/upload-artifact@master + with: + name: Mac Binaries + path: vendor/gitian-builder/build/out + windows: + name: Build and sign windows binaries + needs: prepare + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + - name: Load Base Cache + uses: actions/cache@v1 + with: + path: vendor/gitian-builder/var + key: base-cache + - name: Load Build Cache + uses: actions/cache@v1 + with: + path: vendor/gitian-builder/cache/bitshares-core-win + - name: Build and Sign + run: | + sudo apt-get install ruby + export GNUPGHOME=`pwd`/.gpghome + echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ + | ./run-gitian -b -s BitShares-Gitian-Auto-Build-Signer \ + -p "gpg --passphrase-fd 3" -O linux ${{ github.event.deployment.payload.coreversion }} -j 1 + - name: Upload build artifacts + uses: actions/upload-artifact@master + with: + name: Windows Binaries + path: vendor/gitian-builder/build/out diff --git a/.gpghome/pubring.gpg b/.gpghome/pubring.gpg new file mode 100644 index 0000000000000000000000000000000000000000..f829d4642cc3a0515076aeeb431f48a99e83a976 GIT binary patch literal 988 zcmV<210(#I11G=RSo0qTH-b_Y+V|``_}2qdit*4)Zte9tKPs6C5u%p7sT? zC6+XNZ|P z4hi)E&(NBJEP@u@89AymVW}ciqH)9BQ7t`FmGoi$a@yce@UBTOa8u3)$>`pwX!Po_ zo}%k%=0Qc?31<+?y4dtX5A0YE*@!%x(I%6y(}{bCWk@3V_&=o+$s*QXbB>YB0ozaU zwNv}{@xyu%3RDyowJQMtziklh;xRVUnU)|TUMF1c2^+m`qdO@C`1O<==AO!B2mrsG zY5yL+8Faimkhj&Ytb=O-26Ub*3Ky84QIBwIcT}fER-VXq&LQOXSlIn%4iYpo+Nz~a z;+9ZTOpO+m+U`QrwGr-Sx|ose#+m)q>zR6SR3;6&(->1dv#O*pZ8vN9ELg&r_e}p? z*`CfpyD_&NVgfglq5bUW7c_k8b*mjJbNKRS_f;2G2tj?~LJhldbv9JsEb|fu;h+t5 z=&0OztH0pcvc!|w-I_n_Zm<%xMpk&1om_~^Ie~>pBY|n>gv7$l)hl>aRpdoe4 znk0KFhk3RjJgANzVW%0?9iG2yvSAa3m;KEsJqZ1ND6Go`=?(TkARBM;#@{Mt#hcA{ zNcr-=1>r$tARCCt5!D3;ZQ1VulK&ug2HiY!E_;3OBMQi$jPiKg%5lOYl9SU*_^3SM zXH*n9Y20dm5wqD>UFQ(#AhPHUi{XZ%gS4Ch_^>)_!wmOVP7zI$O9HYoqTOu(+F8qt zsHiR(jgXX`A(!*z!y<3hV$^jY%1#^<>@-G1U*7uyOqvKz!ZtMR&G)>^1 zr92BE$*qcYaA*lQ$E7|J0PqQ|S-`;S(PU6k4)R}mI(4?tq}Sf8C4EZQv?4-jbW>Kughx|wq|H4o{tW+cWeSgf=ez5xC&m7rvAkM-m|A^cL4Dm^(Mv#fRI@xko&My+a? Ka?B^N0ssTQPs_Og literal 0 HcmV?d00001 diff --git a/.gpghome/secring.gpg b/.gpghome/secring.gpg new file mode 100644 index 0000000000000000000000000000000000000000..4217c5fce636ee616d9f25372471ddd38e2dcdfb GIT binary patch literal 1063 zcmV+?1laqP19=2p(FGL|2mr!22%s*d%Jm6R2Ai9R7Pow(a#07Sj@U@lw3$JQyVOmP z=C+t8S!6)rX1DNQGYA>G=RSo0qTH-b_Y+V|``_}2qdit*4)Zte9tKPs6C5u%p7sT? zC6+XNZ|P z4hi)E&(NBJEP@u@89AymVW}ciqH)9BQ7t`FmGoi$a@yce@UBTOa8u3)$>`pwX!Po_ zo}%k%=0Qc?31<+?y4dtX5A0YE*@!%x(I%6y(}{bCWk@3V_&=o+$s*QXbB>YB0ozaU zwNv}{@xyu%3RDyowJQMtziklh;xRVUnU)|TUMF1c2^+m`qdO@C`1O<==AO!B2mrsG zY5yL+8Faimkhj&Ytb=O-26Ub*3Ky84QIBwIcT}fER-VXq&LQOXSlIn%4iYpo+Nz~a z;+9ZTOpO+m+U`QrwGr-Sx|ose#+m)q>zR6SR3;6&(->1dv#O*pZ8vN9ELg&r_e}p? z*`CfpyD_&NVgfglq5bUW7c_k8b*mjJbNKRS_f;2G2tj?~LJhldbv9JsEb|fu;h+t5 z=&0OztH0pcvc!|w-I_n_Zm<%xMpk&1om_~^Ie~>pBY|n>gv7$l)hl>aRpdoe4 znk0KFhk3RjJgANzVW%0?9iG2yvSAa3m;KEsJqZ1ND6Go`=?(TkARBM;#@{Mt#hcA{ zNcr-=1>r$tARCCt5!D3;ZQ1VulK&ug2HiY!E_;3OBMQi$jPiKg%5lOYl9SU*_^3SM zXH*n9Y20dm5wqD>UFQ(#AhPHUi{XZ%gS4Ch_^>)_!wmOVP7zI$O9HYoqTOu(+F8qt zsHiR(jgXX`A(!*z!y<3hV$^jY%1#^<>@-G1U*7uyOqvKz!ZtMR&G)>^1 zr92BE$*qcYaA*lQ$E7|J0PqQ|S-`;S(PU6k4)R}mI(4?tq}Sf8C4EZQ{sRL7-?<#mHfj#cPI|Z#lL!OwxN_A@5}ZY@D|bZ;#}b!lv5EmLV{Ze? Date: Sun, 17 Nov 2019 10:45:26 +0100 Subject: [PATCH 02/20] Fixed caching --- .github/workflows/build-and-sign.yml | 37 +++++++++++----------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index 97c9e41..82e70a5 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -9,17 +9,14 @@ jobs: - uses: actions/checkout@v1 with: submodules: recursive - - name: Base VM cache - id: base-cache - uses: actions/cache@v1 - with: - path: vendor/gitian-builder/var - key: base-cache - name: Prepare if: steps.base-cache.outputs.cache-hit != 'true' run: | - vendor/gitian-builder/bin/make-base-vm --docker --suite bionic - vendor/gitian-builder/bin/make-base-vm --docker --suite xenial + for i in bionic xenial; do + if ! docker image ls | grep -q "base-$i-amd64"; then + vendor/gitian-builder/bin/make-base-vm --docker --suite "$i" + fi + done linux: name: Build and sign linux binaries needs: prepare @@ -28,15 +25,13 @@ jobs: - uses: actions/checkout@v1 with: submodules: recursive - - name: Load Base Cache - uses: actions/cache@v1 - with: - path: vendor/gitian-builder/var - key: base-cache - name: Load Build Cache uses: actions/cache@v1 with: path: vendor/gitian-builder/cache/bitshares-core-linux + key: build-cache-linux-${{ github.event.deployment.payload.coreversion }} + restore-keys: | + build-cache-linux- - name: Build and Sign run: | sudo apt-get install ruby @@ -57,15 +52,13 @@ jobs: - uses: actions/checkout@v1 with: submodules: recursive - - name: Load Base Cache - uses: actions/cache@v1 - with: - path: vendor/gitian-builder/var - key: base-cache - name: Load Build Cache uses: actions/cache@v1 with: path: vendor/gitian-builder/cache/bitshares-core-osx + key: build-cache-osx-${{ github.event.deployment.payload.coreversion }} + restore-keys: | + build-cache-osx- - name: Build and Sign run: | sudo apt-get install ruby @@ -86,15 +79,13 @@ jobs: - uses: actions/checkout@v1 with: submodules: recursive - - name: Load Base Cache - uses: actions/cache@v1 - with: - path: vendor/gitian-builder/var - key: base-cache - name: Load Build Cache uses: actions/cache@v1 with: path: vendor/gitian-builder/cache/bitshares-core-win + key: build-cache-win-${{ github.event.deployment.payload.coreversion }} + restore-keys: | + build-cache-win- - name: Build and Sign run: | sudo apt-get install ruby From b25df7e294bbd54ff1476fe864693a3a1aa58463 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sun, 17 Nov 2019 12:27:42 +0100 Subject: [PATCH 03/20] Fixed gpg parameter --- .github/workflows/build-and-sign.yml | 12 +++++++++--- run-gitian | 11 +++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index 82e70a5..4964aa6 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -38,7 +38,9 @@ jobs: export GNUPGHOME=`pwd`/.gpghome echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ | ./run-gitian -b -s BitShares-Gitian-Auto-Build-Signer \ - -p "gpg --passphrase-fd 3" -O linux ${{ github.event.deployment.payload.coreversion }} -j 1 + -O linux -p "gpg --passphrase-fd 3" \ + ${{ github.event.deployment.payload.coreversion }} \ + -j 1 - name: Upload build artifacts uses: actions/upload-artifact@master with: @@ -65,7 +67,9 @@ jobs: export GNUPGHOME=`pwd`/.gpghome echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ | ./run-gitian -b -s BitShares-Gitian-Auto-Build-Signer \ - -p "gpg --passphrase-fd 3" -O linux ${{ github.event.deployment.payload.coreversion }} -j 1 + -O osx -p "gpg --passphrase-fd 3" \ + ${{ github.event.deployment.payload.coreversion }} \ + -j 1 - name: Upload build artifacts uses: actions/upload-artifact@master with: @@ -92,7 +96,9 @@ jobs: export GNUPGHOME=`pwd`/.gpghome echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ | ./run-gitian -b -s BitShares-Gitian-Auto-Build-Signer \ - -p "gpg --passphrase-fd 3" -O linux ${{ github.event.deployment.payload.coreversion }} -j 1 + -O win -p "gpg --passphrase-fd 3" \ + ${{ github.event.deployment.payload.coreversion }} \ + -j 1 - name: Upload build artifacts uses: actions/upload-artifact@master with: diff --git a/run-gitian b/run-gitian index 8a49802..9925231 100755 --- a/run-gitian +++ b/run-gitian @@ -11,12 +11,14 @@ BUILD= SIGN= VERIFY= OS=linux -while [ "$1" = "-b" -o "$1" = "-s" -o "$1" = "-v" -o "$1" = "-O" ]; do +GPG= +while [ "$1" = "-b" -o "$1" = "-s" -o "$1" = "-v" -o "$1" = "-O" -o "$1" = "-p" ]; do case "$1" in -b) BUILD=1; ;; -s) SIGN=1; KEY_ID="$2"; shift; ;; -v) VERIFY=1; ;; -O) OS="$2"; shift;; + -p) GPG="$1 $2"; shift;; esac shift done @@ -76,12 +78,13 @@ _EOL_ fi if [ "$#" -lt 1 ]; then - echo "Usage:\t$0 [-b | -s KEY_ID | -v] [-O linux|win|osx] VERSION [builder-options]" 1>&2 + echo "Usage:\t$0 [-b | -s KEY_ID | -v] [-O linux|win|osx] [-p GPG] VERSION [builder-options]" 1>&2 echo "\t$0 KEY_ID VERSION [builder-options]" 1>&2 echo "-b\tbuild" 1>&2 echo "-s\tsign" 1>&2 echo "-v\tverify" 1>&2 echo "-O\tchoose target operating system linux or win, defaults to linux" 1>&2 + echo "-p\tcommand to use for signing" 1>&2 echo "\t(will build and sign with KEY_ID if neither option is present)" 1>&2 echo "VERSION\tis the BitShares release version to build" 1>&2 echo "KEY_ID\tis the GPG key ID to use for signing" 1>&2 @@ -89,7 +92,7 @@ if [ "$#" -lt 1 ]; then echo "builder-options:" 1>&2 gbuild --help 2>&1 | tail -n +2 1>&2 echo "" 1>&2 - echo "You should allow at least 4G RAM for the build (-m 4096), more if" 1>&2 + echo "You should allow at least 5G RAM for the build (-m 4096), more if" 1>&2 echo "you want to build with more than one core!" 1>&2 exit 1 fi @@ -98,7 +101,7 @@ VERSION="$1" shift [ -z "$BUILD" ] || gbuild "../../descriptors/bitshares-core-$OS.yml" -c bitshares="$VERSION" "$@" -[ -z "$SIGN" ] || gsign --signer "$KEY_ID" --destination ../../signatures --release "$VERSION"-"$OS" \ +[ -z "$SIGN" ] || gsign --signer "$KEY_ID" $GPG --destination ../../signatures --release "$VERSION"-"$OS" \ "../../descriptors/bitshares-core-$OS.yml" [ -z "$VERIFY" ] || gverify --destination ../../signatures --release "$VERSION"-"$OS" \ "../../descriptors/bitshares-core-$OS.yml" From d97d7a748f3b853c48e3cae7c9506ef94063fb5e Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sun, 17 Nov 2019 14:02:02 +0100 Subject: [PATCH 04/20] Cache docker base images --- .github/workflows/build-and-sign.yml | 61 +++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index 4964aa6..373f2aa 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -2,29 +2,55 @@ on: deployment env: USE_DOCKER: 1 jobs: - prepare: - name: Prepare Gitian build environment + prepare-bionic: + name: Prepare "bionic" Gitian build environment runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 with: submodules: recursive + - name: Load Prep Cache + uses: actions/cache@v1 + with: + path: docker + key: prep-cache-bionic - name: Prepare - if: steps.base-cache.outputs.cache-hit != 'true' run: | - for i in bionic xenial; do - if ! docker image ls | grep -q "base-$i-amd64"; then - vendor/gitian-builder/bin/make-base-vm --docker --suite "$i" - fi - done + if [ ! -r docker/base-bionic-amd64.tar.xz ]; then + vendor/gitian-builder/bin/make-base-vm --docker --suite bionic + docker save base-bionic-amd64 | xz >docker/base-bionic-amd64.tar.xz + fi + prepare-xenial: + name: Prepare "xenial" Gitian build environment + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + - name: Load Prep Cache + uses: actions/cache@v1 + with: + path: docker + key: prep-cache-xenial + - name: Prepare + run: | + if [ ! -r docker/base-xenial-amd64.tar.xz ]; then + vendor/gitian-builder/bin/make-base-vm --docker --suite xenial + docker save base-xenial-amd64 | xz >docker/base-xenial-amd64.tar.xz + fi linux: name: Build and sign linux binaries - needs: prepare + needs: prepare-xenial runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 with: submodules: recursive + - name: Load Prep Cache + uses: actions/cache@v1 + with: + path: docker + key: prep-cache-bionic - name: Load Build Cache uses: actions/cache@v1 with: @@ -34,6 +60,7 @@ jobs: build-cache-linux- - name: Build and Sign run: | + docker load -i docker/base-xenial-amd64.tar.xz sudo apt-get install ruby export GNUPGHOME=`pwd`/.gpghome echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ @@ -48,12 +75,17 @@ jobs: path: vendor/gitian-builder/build/out mac: name: Build and sign mac binaries - needs: prepare + needs: prepare-bionic runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 with: submodules: recursive + - name: Load Prep Cache + uses: actions/cache@v1 + with: + path: docker + key: prep-cache-bionic - name: Load Build Cache uses: actions/cache@v1 with: @@ -63,6 +95,7 @@ jobs: build-cache-osx- - name: Build and Sign run: | + docker load -i docker/base-bionic-amd64.tar.xz sudo apt-get install ruby export GNUPGHOME=`pwd`/.gpghome echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ @@ -77,12 +110,17 @@ jobs: path: vendor/gitian-builder/build/out windows: name: Build and sign windows binaries - needs: prepare + needs: prepare-bionic runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 with: submodules: recursive + - name: Load Prep Cache + uses: actions/cache@v1 + with: + path: docker + key: prep-cache-xenial - name: Load Build Cache uses: actions/cache@v1 with: @@ -92,6 +130,7 @@ jobs: build-cache-win- - name: Build and Sign run: | + docker load -i docker/base-bionic-amd64.tar.xz sudo apt-get install ruby export GNUPGHOME=`pwd`/.gpghome echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ From 4030e4ff5c0928db42cb953b65938a519bd25fbc Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sun, 17 Nov 2019 14:53:56 +0100 Subject: [PATCH 05/20] Dont rely on caching :-( --- .github/workflows/build-and-sign.yml | 48 +++++++++++++++++----------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index 373f2aa..c4acfb2 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -13,12 +13,13 @@ jobs: uses: actions/cache@v1 with: path: docker - key: prep-cache-bionic + key: prep-cache-bioni - name: Prepare run: | if [ ! -r docker/base-bionic-amd64.tar.xz ]; then - vendor/gitian-builder/bin/make-base-vm --docker --suite bionic - docker save base-bionic-amd64 | xz >docker/base-bionic-amd64.tar.xz + echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" + #vendor/gitian-builder/bin/make-base-vm --docker --suite bionic + #docker save base-bionic-amd64 | xz >docker/base-bionic-amd64.tar.xz fi prepare-xenial: name: Prepare "xenial" Gitian build environment @@ -31,12 +32,13 @@ jobs: uses: actions/cache@v1 with: path: docker - key: prep-cache-xenial + key: prep-cache-xenia - name: Prepare run: | if [ ! -r docker/base-xenial-amd64.tar.xz ]; then - vendor/gitian-builder/bin/make-base-vm --docker --suite xenial - docker save base-xenial-amd64 | xz >docker/base-xenial-amd64.tar.xz + echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" + #vendor/gitian-builder/bin/make-base-vm --docker --suite xenial + #docker save base-xenial-amd64 | xz >docker/base-xenial-amd64.tar.xz fi linux: name: Build and sign linux binaries @@ -46,11 +48,6 @@ jobs: - uses: actions/checkout@v1 with: submodules: recursive - - name: Load Prep Cache - uses: actions/cache@v1 - with: - path: docker - key: prep-cache-bionic - name: Load Build Cache uses: actions/cache@v1 with: @@ -58,9 +55,15 @@ jobs: key: build-cache-linux-${{ github.event.deployment.payload.coreversion }} restore-keys: | build-cache-linux- + - name: Prepare + run: | + if [ ! -r docker/base-xenial-amd64.tar.xz ]; then + echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" + vendor/gitian-builder/bin/make-base-vm --docker --suite xenial + fi - name: Build and Sign run: | - docker load -i docker/base-xenial-amd64.tar.xz + #docker load -i docker/base-xenial-amd64.tar.xz sudo apt-get install ruby export GNUPGHOME=`pwd`/.gpghome echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ @@ -93,9 +96,15 @@ jobs: key: build-cache-osx-${{ github.event.deployment.payload.coreversion }} restore-keys: | build-cache-osx- + - name: Prepare + run: | + if [ ! -r docker/base-bionic-amd64.tar.xz ]; then + echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" + vendor/gitian-builder/bin/make-base-vm --docker --suite bionic + fi - name: Build and Sign run: | - docker load -i docker/base-bionic-amd64.tar.xz + #docker load -i docker/base-bionic-amd64.tar.xz sudo apt-get install ruby export GNUPGHOME=`pwd`/.gpghome echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ @@ -116,11 +125,6 @@ jobs: - uses: actions/checkout@v1 with: submodules: recursive - - name: Load Prep Cache - uses: actions/cache@v1 - with: - path: docker - key: prep-cache-xenial - name: Load Build Cache uses: actions/cache@v1 with: @@ -128,9 +132,15 @@ jobs: key: build-cache-win-${{ github.event.deployment.payload.coreversion }} restore-keys: | build-cache-win- + - name: Prepare + run: | + if [ ! -r docker/base-bionic-amd64.tar.xz ]; then + echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" + vendor/gitian-builder/bin/make-base-vm --docker --suite bionic + fi - name: Build and Sign run: | - docker load -i docker/base-bionic-amd64.tar.xz + #docker load -i docker/base-bionic-amd64.tar.xz sudo apt-get install ruby export GNUPGHOME=`pwd`/.gpghome echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ From 6ae398911a9d7164e1f4a7884bd96e8a2d6e4663 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sun, 17 Nov 2019 17:10:04 +0100 Subject: [PATCH 06/20] Add log output on failure --- .github/workflows/build-and-sign.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index c4acfb2..007237b 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -70,7 +70,10 @@ jobs: | ./run-gitian -b -s BitShares-Gitian-Auto-Build-Signer \ -O linux -p "gpg --passphrase-fd 3" \ ${{ github.event.deployment.payload.coreversion }} \ - -j 1 + -j 1 \ + || ( echo Install log: && cat vendor/gitian-builder/var/install.log \ + && echo Build log: && cat vendor/gitian-builder/var/build.log \ + && false ) - name: Upload build artifacts uses: actions/upload-artifact@master with: @@ -111,7 +114,10 @@ jobs: | ./run-gitian -b -s BitShares-Gitian-Auto-Build-Signer \ -O osx -p "gpg --passphrase-fd 3" \ ${{ github.event.deployment.payload.coreversion }} \ - -j 1 + -j 1 \ + || ( echo Install log: && cat vendor/gitian-builder/var/install.log \ + && echo Build log: && cat vendor/gitian-builder/var/build.log \ + && false ) - name: Upload build artifacts uses: actions/upload-artifact@master with: @@ -147,7 +153,10 @@ jobs: | ./run-gitian -b -s BitShares-Gitian-Auto-Build-Signer \ -O win -p "gpg --passphrase-fd 3" \ ${{ github.event.deployment.payload.coreversion }} \ - -j 1 + -j 1 \ + || ( echo Install log: && cat vendor/gitian-builder/var/install.log \ + && echo Build log: && cat vendor/gitian-builder/var/build.log \ + && false ) - name: Upload build artifacts uses: actions/upload-artifact@master with: From 8e858d96afc856a247b3fac2006a89425640b1d0 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sun, 17 Nov 2019 17:34:00 +0100 Subject: [PATCH 07/20] Add explicit `submodule update` --- descriptors/bitshares-core-linux.yml | 1 + descriptors/bitshares-core-osx.yml | 1 + descriptors/bitshares-core-win.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/descriptors/bitshares-core-linux.yml b/descriptors/bitshares-core-linux.yml index 8269eea..621be47 100644 --- a/descriptors/bitshares-core-linux.yml +++ b/descriptors/bitshares-core-linux.yml @@ -80,6 +80,7 @@ script: | popd cd bitshares + GIT_SSL_NO_VERIFY=true git submodule update --init --recursive cp libraries/wallet/wallet.cpp{,.orig} sed -i '/__DATE__/d' libraries/wallet/wallet.cpp # Workaround for #1791 diff --git a/descriptors/bitshares-core-osx.yml b/descriptors/bitshares-core-osx.yml index 62b1f12..8d8c097 100644 --- a/descriptors/bitshares-core-osx.yml +++ b/descriptors/bitshares-core-osx.yml @@ -126,6 +126,7 @@ script: | popd cd bitshares + GIT_SSL_NO_VERIFY=true git submodule update --init --recursive sed -i '/__DATE__/d' libraries/wallet/wallet.cpp sed -i '/hardfork.hpp/d' libraries/chain/CMakeLists.txt sed -i "/[ep].configure/s/$/ --host=${DARWIN%-} CC=${DARWIN}clang LD=${DARWIN}clang/" libraries/fc/CMakeLists.txt diff --git a/descriptors/bitshares-core-win.yml b/descriptors/bitshares-core-win.yml index cfdd7b7..0c2fd4e 100644 --- a/descriptors/bitshares-core-win.yml +++ b/descriptors/bitshares-core-win.yml @@ -116,6 +116,7 @@ script: | popd cd bitshares + GIT_SSL_NO_VERIFY=true git submodule update --init --recursive sed -i '/__DATE__/d' libraries/wallet/wallet.cpp sed -i '/hardfork.hpp/d' libraries/chain/CMakeLists.txt sed -i "/add_executable/alist( APPEND PLATFORM_SPECIFIC_LIBS $LIBS/lib/libcurl.a -lws2_32 -lpthread -lcrypt32 )" \ From 9375d9344ca94c9aa400f7a3553e3d7197be52eb Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sun, 17 Nov 2019 17:41:57 +0100 Subject: [PATCH 08/20] Skip mac build --- .github/workflows/build-and-sign.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index 007237b..d44bcd7 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -103,21 +103,22 @@ jobs: run: | if [ ! -r docker/base-bionic-amd64.tar.xz ]; then echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" - vendor/gitian-builder/bin/make-base-vm --docker --suite bionic + #vendor/gitian-builder/bin/make-base-vm --docker --suite bionic fi - name: Build and Sign run: | #docker load -i docker/base-bionic-amd64.tar.xz sudo apt-get install ruby export GNUPGHOME=`pwd`/.gpghome - echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ - | ./run-gitian -b -s BitShares-Gitian-Auto-Build-Signer \ - -O osx -p "gpg --passphrase-fd 3" \ - ${{ github.event.deployment.payload.coreversion }} \ - -j 1 \ - || ( echo Install log: && cat vendor/gitian-builder/var/install.log \ - && echo Build log: && cat vendor/gitian-builder/var/build.log \ - && false ) + echo "Skipping Mac build due to missing SDK" + #echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ + # | ./run-gitian -b -s BitShares-Gitian-Auto-Build-Signer \ + # -O osx -p "gpg --passphrase-fd 3" \ + # ${{ github.event.deployment.payload.coreversion }} \ + # -j 1 \ + # || ( echo Install log: && cat vendor/gitian-builder/var/install.log \ + # && echo Build log: && cat vendor/gitian-builder/var/build.log \ + # && false ) - name: Upload build artifacts uses: actions/upload-artifact@master with: From 0afc4349614b1ee291b75a0019f41e5f010e48fb Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sun, 17 Nov 2019 18:38:08 +0100 Subject: [PATCH 09/20] Fixed signing --- .github/gpg-wrapper | 3 +++ .github/workflows/build-and-sign.yml | 40 ++++++++++++++++------------ 2 files changed, 26 insertions(+), 17 deletions(-) create mode 100755 .github/gpg-wrapper diff --git a/.github/gpg-wrapper b/.github/gpg-wrapper new file mode 100755 index 0000000..a825b70 --- /dev/null +++ b/.github/gpg-wrapper @@ -0,0 +1,3 @@ +#!/bin/bash + +exec gpg --batch --passphrase-fd 3 --pinentry-mode loopback --detach-sign "$@" 3<<<"${PGP_PASS}" diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index d44bcd7..029d8ee 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -66,14 +66,16 @@ jobs: #docker load -i docker/base-xenial-amd64.tar.xz sudo apt-get install ruby export GNUPGHOME=`pwd`/.gpghome - echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ - | ./run-gitian -b -s BitShares-Gitian-Auto-Build-Signer \ - -O linux -p "gpg --passphrase-fd 3" \ - ${{ github.event.deployment.payload.coreversion }} \ - -j 1 \ + ./run-gitian -b -O linux \ + ${{ github.event.deployment.payload.coreversion }} \ + -j 1 \ || ( echo Install log: && cat vendor/gitian-builder/var/install.log \ && echo Build log: && cat vendor/gitian-builder/var/build.log \ && false ) + PGP_PASS="${{ secrets.PGP_PASSPHRASE }}" \ + ./run-gitian -s BitShares-Gitian-Auto-Build-Signer \ + -O linux -p `pwd`/.github/gpg-wrapper \ + ${{ github.event.deployment.payload.coreversion }} - name: Upload build artifacts uses: actions/upload-artifact@master with: @@ -109,16 +111,18 @@ jobs: run: | #docker load -i docker/base-bionic-amd64.tar.xz sudo apt-get install ruby - export GNUPGHOME=`pwd`/.gpghome echo "Skipping Mac build due to missing SDK" - #echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ - # | ./run-gitian -b -s BitShares-Gitian-Auto-Build-Signer \ - # -O osx -p "gpg --passphrase-fd 3" \ - # ${{ github.event.deployment.payload.coreversion }} \ - # -j 1 \ + #./run-gitian -b -O osx \ + # ${{ github.event.deployment.payload.coreversion }} \ + # -j 1 \ # || ( echo Install log: && cat vendor/gitian-builder/var/install.log \ # && echo Build log: && cat vendor/gitian-builder/var/build.log \ # && false ) + export GNUPGHOME=`pwd`/.gpghome + #PGP_PASS="${{ secrets.PGP_PASSPHRASE }}" \ + # ./run-gitian -s BitShares-Gitian-Auto-Build-Signer \ + # -O osx -p `pwd`/.github/gpg-wrapper \ + # ${{ github.event.deployment.payload.coreversion }} - name: Upload build artifacts uses: actions/upload-artifact@master with: @@ -149,15 +153,17 @@ jobs: run: | #docker load -i docker/base-bionic-amd64.tar.xz sudo apt-get install ruby - export GNUPGHOME=`pwd`/.gpghome - echo ${{ secrets.PGP_PASSPHRASE }} 1>&3 \ - | ./run-gitian -b -s BitShares-Gitian-Auto-Build-Signer \ - -O win -p "gpg --passphrase-fd 3" \ - ${{ github.event.deployment.payload.coreversion }} \ - -j 1 \ + ./run-gitian -b -O win \ + ${{ github.event.deployment.payload.coreversion }} \ + -j 1 \ || ( echo Install log: && cat vendor/gitian-builder/var/install.log \ && echo Build log: && cat vendor/gitian-builder/var/build.log \ && false ) + export GNUPGHOME=`pwd`/.gpghome + PGP_PASS="${{ secrets.PGP_PASSPHRASE }}" \ + ./run-gitian -s BitShares-Gitian-Auto-Build-Signer \ + -O win -p `pwd`/.github/gpg-wrapper \ + ${{ github.event.deployment.payload.coreversion }} - name: Upload build artifacts uses: actions/upload-artifact@master with: From f9e32a26aee358decffdd62d7ebe88d86baeae9a Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Mon, 25 Nov 2019 16:31:18 +0100 Subject: [PATCH 10/20] Extract MacOS SDK --- .github/workflows/build-and-sign.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index 029d8ee..c80496c 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -40,6 +40,25 @@ jobs: #vendor/gitian-builder/bin/make-base-vm --docker --suite xenial #docker save base-xenial-amd64 | xz >docker/base-xenial-amd64.tar.xz fi + prepare-mac: + name: Scrape mac-SDK + runs-on: macos-latest + steps: + - uses: actions/checkout@v1 + with: + repository: tpoechtrager/osxcross + ref: e0a171828a72a0d7ad4409489033536590008ebf + - name: Package + run: | + pushd ../osxcross + tools/gen_sdk_package.sh + popd + mv ../osxcross/*MacOSX* . + - name: Upload SDK + uses: actions/upload-artifact@master + with: + name: MacOS-SDK + path: MacOSX10.15.sdk.tar.xz linux: name: Build and sign linux binaries needs: prepare-xenial From 375ed297b56e01318019ea39a7d30484ba89ce6d Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Wed, 27 Nov 2019 16:55:57 +0100 Subject: [PATCH 11/20] Re-enable caches and osx build --- .github/workflows/build-and-sign.yml | 147 +++++++++++++++++---------- 1 file changed, 96 insertions(+), 51 deletions(-) diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index c80496c..7b629a0 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -6,64 +6,90 @@ jobs: name: Prepare "bionic" Gitian build environment runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - with: - submodules: recursive - name: Load Prep Cache + id: cache-bionic uses: actions/cache@v1 with: path: docker - key: prep-cache-bioni + key: prep-cache-bionic + - uses: actions/checkout@v1 + if: steps.cache-bionic.outputs.cache-hit != 'true' + with: + submodules: recursive - name: Prepare + if: steps.cache-bionic.outputs.cache-hit != 'true' run: | - if [ ! -r docker/base-bionic-amd64.tar.xz ]; then - echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" - #vendor/gitian-builder/bin/make-base-vm --docker --suite bionic - #docker save base-bionic-amd64 | xz >docker/base-bionic-amd64.tar.xz - fi + mkdir -p docker + echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" + vendor/gitian-builder/bin/make-base-vm --docker --suite bionic + docker save base-bionic-amd64 | xz >docker/base-bionic-amd64.tar.xz prepare-xenial: name: Prepare "xenial" Gitian build environment runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - with: - submodules: recursive - name: Load Prep Cache + id: cache-xenial uses: actions/cache@v1 with: path: docker - key: prep-cache-xenia + key: prep-cache-xenial + - uses: actions/checkout@v1 + if: steps.cache-xenial.outputs.cache-hit != 'true' + with: + submodules: recursive - name: Prepare + if: steps.cache-xenial.outputs.cache-hit != 'true' run: | - if [ ! -r docker/base-xenial-amd64.tar.xz ]; then - echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" - #vendor/gitian-builder/bin/make-base-vm --docker --suite xenial - #docker save base-xenial-amd64 | xz >docker/base-xenial-amd64.tar.xz - fi + mkdir -p docker + echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" + vendor/gitian-builder/bin/make-base-vm --docker --suite xenial + docker save base-xenial-amd64 | xz >docker/base-xenial-amd64.tar.xz prepare-mac: name: Scrape mac-SDK runs-on: macos-latest steps: + - name: Load Prep Cache + id: cache-osx + uses: actions/cache@v1 + with: + path: osx + key: prep-cache-osx - uses: actions/checkout@v1 + if: steps.cache-osx.outputs.cache-hit != 'true' with: repository: tpoechtrager/osxcross ref: e0a171828a72a0d7ad4409489033536590008ebf - name: Package + if: steps.cache-osx.outputs.cache-hit != 'true' run: | + echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" pushd ../osxcross tools/gen_sdk_package.sh popd - mv ../osxcross/*MacOSX* . + mkdir -p osx + mv ../osxcross/*MacOSX* osx - name: Upload SDK + if: steps.cache-osx.outputs.cache-hit != 'true' uses: actions/upload-artifact@master with: name: MacOS-SDK - path: MacOSX10.15.sdk.tar.xz + path: osx/MacOSX10.15.sdk.tar.xz linux: name: Build and sign linux binaries needs: prepare-xenial runs-on: ubuntu-latest steps: + - name: Load Prep Cache + id: cache-xenial + uses: actions/cache@v1 + with: + path: docker + key: prep-cache-xenial + - name: Check cache + if: steps.cache-xenial.outputs.cache-hit != 'true' + run: | + echo "Need to prep the cache." + false - uses: actions/checkout@v1 with: submodules: recursive @@ -76,13 +102,9 @@ jobs: build-cache-linux- - name: Prepare run: | - if [ ! -r docker/base-xenial-amd64.tar.xz ]; then - echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" - vendor/gitian-builder/bin/make-base-vm --docker --suite xenial - fi + docker load -i docker/base-xenial-amd64.tar.xz - name: Build and Sign run: | - #docker load -i docker/base-xenial-amd64.tar.xz sudo apt-get install ruby export GNUPGHOME=`pwd`/.gpghome ./run-gitian -b -O linux \ @@ -102,17 +124,36 @@ jobs: path: vendor/gitian-builder/build/out mac: name: Build and sign mac binaries - needs: prepare-bionic + needs: + - prepare-bionic + - prepare-mac runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - with: - submodules: recursive - - name: Load Prep Cache + - name: Load Docker Prep Cache + id: cache-bionic uses: actions/cache@v1 with: path: docker key: prep-cache-bionic + - name: Check cache + if: steps.cache-bionic.outputs.cache-hit != 'true' + run: | + echo "Need to prep the cache." + false + - name: Load OSX Prep Cache + id: cache-osx + uses: actions/cache@v1 + with: + path: osx + key: prep-cache-osx + - name: Check cache + if: steps.cache-osx.outputs.cache-hit != 'true' + run: | + echo "Need to prep the cache." + false + - uses: actions/checkout@v1 + with: + submodules: recursive - name: Load Build Cache uses: actions/cache@v1 with: @@ -122,26 +163,23 @@ jobs: build-cache-osx- - name: Prepare run: | - if [ ! -r docker/base-bionic-amd64.tar.xz ]; then - echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" - #vendor/gitian-builder/bin/make-base-vm --docker --suite bionic - fi + docker load -i docker/base-bionic-amd64.tar.xz - name: Build and Sign run: | - #docker load -i docker/base-bionic-amd64.tar.xz sudo apt-get install ruby - echo "Skipping Mac build due to missing SDK" - #./run-gitian -b -O osx \ - # ${{ github.event.deployment.payload.coreversion }} \ - # -j 1 \ - # || ( echo Install log: && cat vendor/gitian-builder/var/install.log \ - # && echo Build log: && cat vendor/gitian-builder/var/build.log \ - # && false ) + mkdir -p vendor/gitian-builder/inputs + mv osx/* vendor/gitian-builder/inputs + ./run-gitian -b -O osx \ + ${{ github.event.deployment.payload.coreversion }} \ + -j 1 \ + || ( echo Install log: && cat vendor/gitian-builder/var/install.log \ + && echo Build log: && cat vendor/gitian-builder/var/build.log \ + && false ) export GNUPGHOME=`pwd`/.gpghome - #PGP_PASS="${{ secrets.PGP_PASSPHRASE }}" \ - # ./run-gitian -s BitShares-Gitian-Auto-Build-Signer \ - # -O osx -p `pwd`/.github/gpg-wrapper \ - # ${{ github.event.deployment.payload.coreversion }} + PGP_PASS="${{ secrets.PGP_PASSPHRASE }}" \ + ./run-gitian -s BitShares-Gitian-Auto-Build-Signer \ + -O osx -p `pwd`/.github/gpg-wrapper \ + ${{ github.event.deployment.payload.coreversion }} - name: Upload build artifacts uses: actions/upload-artifact@master with: @@ -155,6 +193,17 @@ jobs: - uses: actions/checkout@v1 with: submodules: recursive + - name: Load Docker Prep Cache + id: cache-bionic + uses: actions/cache@v1 + with: + path: docker + key: prep-cache-bionic + - name: Check cache + if: steps.cache-bionic.outputs.cache-hit != 'true' + run: | + echo "Need to prep the cache." + false - name: Load Build Cache uses: actions/cache@v1 with: @@ -164,13 +213,9 @@ jobs: build-cache-win- - name: Prepare run: | - if [ ! -r docker/base-bionic-amd64.tar.xz ]; then - echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" - vendor/gitian-builder/bin/make-base-vm --docker --suite bionic - fi + docker load -i docker/base-bionic-amd64.tar.xz - name: Build and Sign run: | - #docker load -i docker/base-bionic-amd64.tar.xz sudo apt-get install ruby ./run-gitian -b -O win \ ${{ github.event.deployment.payload.coreversion }} \ From 60b63517ce00946e7e42a6aa26fd5c7250de9be3 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Wed, 27 Nov 2019 17:24:34 +0100 Subject: [PATCH 12/20] (ab)use artifacts as cache --- .github/workflows/build-and-sign.yml | 56 ++++++++++++++++++---------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index 7b629a0..87a1d50 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -23,6 +23,12 @@ jobs: echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" vendor/gitian-builder/bin/make-base-vm --docker --suite bionic docker save base-bionic-amd64 | xz >docker/base-bionic-amd64.tar.xz + - name: Upload build artifacts + if: steps.cache-bionic.outputs.cache-hit != 'true' + uses: actions/upload-artifact@master + with: + name: docker-bionic + path: docker/base-bionic-amd64.tar.xz prepare-xenial: name: Prepare "xenial" Gitian build environment runs-on: ubuntu-latest @@ -44,6 +50,12 @@ jobs: echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" vendor/gitian-builder/bin/make-base-vm --docker --suite xenial docker save base-xenial-amd64 | xz >docker/base-xenial-amd64.tar.xz + - name: Upload build artifacts + if: steps.cache-xenial.outputs.cache-hit != 'true' + uses: actions/upload-artifact@master + with: + name: docker-xenial + path: docker/base-xenial-amd64.tar.xz prepare-mac: name: Scrape mac-SDK runs-on: macos-latest @@ -79,20 +91,21 @@ jobs: needs: prepare-xenial runs-on: ubuntu-latest steps: + - uses: actions/checkout@v1 + with: + submodules: recursive - name: Load Prep Cache id: cache-xenial uses: actions/cache@v1 with: path: docker key: prep-cache-xenial - - name: Check cache + - name: Poor man's cache if: steps.cache-xenial.outputs.cache-hit != 'true' - run: | - echo "Need to prep the cache." - false - - uses: actions/checkout@v1 + uses: actions/download-artifact@master with: - submodules: recursive + name: docker-xenial + path: docker/ - name: Load Build Cache uses: actions/cache@v1 with: @@ -129,31 +142,33 @@ jobs: - prepare-mac runs-on: ubuntu-latest steps: + - uses: actions/checkout@v1 + with: + submodules: recursive - name: Load Docker Prep Cache id: cache-bionic uses: actions/cache@v1 with: path: docker key: prep-cache-bionic - - name: Check cache + - name: Poor man's docker cache if: steps.cache-bionic.outputs.cache-hit != 'true' - run: | - echo "Need to prep the cache." - false + uses: actions/download-artifact@master + with: + name: docker-bionic + path: docker/ - name: Load OSX Prep Cache id: cache-osx uses: actions/cache@v1 with: path: osx key: prep-cache-osx - - name: Check cache + - name: Poor man's OSX cache if: steps.cache-osx.outputs.cache-hit != 'true' - run: | - echo "Need to prep the cache." - false - - uses: actions/checkout@v1 + uses: actions/download-artifact@master with: - submodules: recursive + name: MacOS-SDK + path: osx/ - name: Load Build Cache uses: actions/cache@v1 with: @@ -199,11 +214,12 @@ jobs: with: path: docker key: prep-cache-bionic - - name: Check cache + - name: Poor man's cache if: steps.cache-bionic.outputs.cache-hit != 'true' - run: | - echo "Need to prep the cache." - false + uses: actions/download-artifact@master + with: + name: docker-bionic + path: docker/ - name: Load Build Cache uses: actions/cache@v1 with: From 95b9f1e9b866290d404a9eaf3fab233e368ac6ac Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Wed, 27 Nov 2019 17:56:37 +0100 Subject: [PATCH 13/20] Switched to MacOS SDK 10.15 --- README.md | 6 +++--- descriptors/bitshares-core-osx.yml | 2 +- run-gitian | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5f082f6..091ab7f 100644 --- a/README.md +++ b/README.md @@ -44,10 +44,10 @@ You must have GnuPG installed and on your path as `gpg`. Instructions on how to install required software on some OSes and prepare a gitian base environment can be found [here](https://github.com/devrandom/gitian-builder/blob/master/README.md). You should follow the described steps until you have completed the "Sanity-testing" section successfully. Be sure to use the "bionic" suite for your base image. -If you want to build build executables for Mac you'll need to download MacOSX SDK 10.14. -It is contained in the Xcode 10.3 distribution, which is available at https://developer.apple.com/xcode/resources/ under "Command Line Tools & Older Versions of Xcode". . +If you want to build build executables for Mac you'll need to download MacOSX SDK 10.15. +It is contained in the Xcode 11.1 distribution, which is available at https://developer.apple.com/xcode/resources/ under "Command Line Tools & Older Versions of Xcode". . After downloading Xcode, you can extract the SDK as described [here](https://github.com/tpoechtrager/osxcross#packaging-the-sdk). -The resulting file `MacOSX10.14.sdk.tar.xz` must be put in the `vendor/gitian-builder/inputs` subdirectory. +The resulting file `MacOSX10.15.sdk.tar.xz` must be put in the `vendor/gitian-builder/inputs` subdirectory. ### Example for Docker diff --git a/descriptors/bitshares-core-osx.yml b/descriptors/bitshares-core-osx.yml index 8d8c097..fe2b921 100644 --- a/descriptors/bitshares-core-osx.yml +++ b/descriptors/bitshares-core-osx.yml @@ -26,7 +26,7 @@ files: - openssl-1.1.1c.tar.gz - curl-7.65.0.tar.bz2 - boost_1_69_0.tar.bz2 -- MacOSX10.14.sdk.tar.xz +- MacOSX10.15.sdk.tar.xz - e0a171828a72a0d7ad4409489033536590008ebf.tar.gz script: | set -e -o pipefail diff --git a/run-gitian b/run-gitian index 9925231..0dd2811 100755 --- a/run-gitian +++ b/run-gitian @@ -59,8 +59,8 @@ _EOL_ cat <<_EOL_ https://github.com/tpoechtrager/osxcross/archive/e0a171828a72a0d7ad4409489033536590008ebf.tar.gz 7ef00c27b76745d4b44e13f291df60318588aa7b5d1788aeba5aca569ac7e989 _EOL_ - if [ ! -r inputs/MacOSX10.14.sdk.tar.xz ]; then - echo "Please download MacOSX10.14.sdk.tar.xz and place it in `pwd`/inputs." 1>&2 + if [ ! -r inputs/MacOSX10.15.sdk.tar.xz ]; then + echo "Please download MacOSX10.15.sdk.tar.xz and place it in `pwd`/inputs." 1>&2 echo "Also be sure to read and understand Apple's licensing terms." 1>&2 exit 1 fi From 1784b1d7e0bd928816c4b4d9d331792569e7fec1 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Thu, 28 Nov 2019 10:06:00 +0100 Subject: [PATCH 14/20] Added script for triggering the build --- .github/trigger-workflow.sh | 18 ++++++++++++++++++ .gitignore | 1 + 2 files changed, 19 insertions(+) create mode 100755 .github/trigger-workflow.sh diff --git a/.github/trigger-workflow.sh b/.github/trigger-workflow.sh new file mode 100755 index 0000000..0084004 --- /dev/null +++ b/.github/trigger-workflow.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +if [ "$#" != 1 ]; then + echo "Usage: $0 " 1>&2 + exit 1 +fi + +if [ ! -r "$(dirname $0)/token" ]; then + echo "Create an API token on github with the 'deployment' privilege and" 1>&2 + echo "put it into '$(dirname $0)/token'." 1>&2 + exit 1 +fi + +curl -H "Authorization: token $(head -c 40 "$(dirname $0)/token")" \ + -H "Content-Type: application/json" \ + --data '{"ref":"action-test","required_contexts":[],"payload":{"coreversion":"'"$1"'"}}' \ + https://api.github.com/repos/pmconrad/bitshares-gitian/deployments + diff --git a/.gitignore b/.gitignore index d0768eb..df40e9a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /docker /var +.github/token From d39e36f6d74c84d7b18f4d95e91d2e6a143eea8e Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Thu, 28 Nov 2019 10:34:02 +0100 Subject: [PATCH 15/20] Add steps for pushing signatures --- .github/workflows/build-and-sign.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index 87a1d50..0b24be3 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -130,6 +130,15 @@ jobs: ./run-gitian -s BitShares-Gitian-Auto-Build-Signer \ -O linux -p `pwd`/.github/gpg-wrapper \ ${{ github.event.deployment.payload.coreversion }} + - name: Push Signatures + run: | + git config add user.name "BitShares Github Autobuild" + git config add user.email "bga@noreply" + BRANCH="autobuild-signatures-linux-$(date -u +%Y-%m-%dT%H:%M:%SZ)" + git checkout -b "$BRANCH" + git add signatures + git commit -m "Add Linux autobuild signature for ${{ github.event.deployment.payload.coreversion }}" + git push "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$BRANCH" - name: Upload build artifacts uses: actions/upload-artifact@master with: @@ -195,6 +204,15 @@ jobs: ./run-gitian -s BitShares-Gitian-Auto-Build-Signer \ -O osx -p `pwd`/.github/gpg-wrapper \ ${{ github.event.deployment.payload.coreversion }} + - name: Push Signatures + run: | + git config add user.name "BitShares Github Autobuild" + git config add user.email "bga@noreply" + BRANCH="autobuild-signatures-osx-$(date -u +%Y-%m-%dT%H:%M:%SZ)" + git checkout -b "$BRANCH" + git add signatures + git commit -m "Add OSX autobuild signature for ${{ github.event.deployment.payload.coreversion }}" + git push "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$BRANCH" - name: Upload build artifacts uses: actions/upload-artifact@master with: @@ -244,6 +262,15 @@ jobs: ./run-gitian -s BitShares-Gitian-Auto-Build-Signer \ -O win -p `pwd`/.github/gpg-wrapper \ ${{ github.event.deployment.payload.coreversion }} + - name: Push Signatures + run: | + git config add user.name "BitShares Github Autobuild" + git config add user.email "bga@noreply" + BRANCH="autobuild-signatures-win-$(date -u +%Y-%m-%dT%H:%M:%SZ)" + git checkout -b "$BRANCH" + git add signatures + git commit -m "Add win autobuild signature for ${{ github.event.deployment.payload.coreversion }}" + git push "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$BRANCH" - name: Upload build artifacts uses: actions/upload-artifact@master with: From bd6ace5c8ed6643592d2aa4a64a6c58a7c4e55b0 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Fri, 29 Nov 2019 10:47:59 +0100 Subject: [PATCH 16/20] Added missing apt-get update --- .github/workflows/build-and-sign.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index 0b24be3..1eda870 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -118,6 +118,7 @@ jobs: docker load -i docker/base-xenial-amd64.tar.xz - name: Build and Sign run: | + sudo apt-get update sudo apt-get install ruby export GNUPGHOME=`pwd`/.gpghome ./run-gitian -b -O linux \ @@ -190,6 +191,7 @@ jobs: docker load -i docker/base-bionic-amd64.tar.xz - name: Build and Sign run: | + sudo apt-get update sudo apt-get install ruby mkdir -p vendor/gitian-builder/inputs mv osx/* vendor/gitian-builder/inputs @@ -250,6 +252,7 @@ jobs: docker load -i docker/base-bionic-amd64.tar.xz - name: Build and Sign run: | + sudo apt-get update sudo apt-get install ruby ./run-gitian -b -O win \ ${{ github.event.deployment.payload.coreversion }} \ From 9f90bb22327ed6dc9223b922d6cb4d4ddd830507 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Fri, 29 Nov 2019 15:03:02 +0100 Subject: [PATCH 17/20] fixup --- .github/workflows/build-and-sign.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index 1eda870..dbc252d 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -133,8 +133,8 @@ jobs: ${{ github.event.deployment.payload.coreversion }} - name: Push Signatures run: | - git config add user.name "BitShares Github Autobuild" - git config add user.email "bga@noreply" + git config --add user.name "BitShares Github Autobuild" + git config --add user.email "bga@noreply" BRANCH="autobuild-signatures-linux-$(date -u +%Y-%m-%dT%H:%M:%SZ)" git checkout -b "$BRANCH" git add signatures @@ -208,8 +208,8 @@ jobs: ${{ github.event.deployment.payload.coreversion }} - name: Push Signatures run: | - git config add user.name "BitShares Github Autobuild" - git config add user.email "bga@noreply" + git config --add user.name "BitShares Github Autobuild" + git config --add user.email "bga@noreply" BRANCH="autobuild-signatures-osx-$(date -u +%Y-%m-%dT%H:%M:%SZ)" git checkout -b "$BRANCH" git add signatures @@ -267,8 +267,8 @@ jobs: ${{ github.event.deployment.payload.coreversion }} - name: Push Signatures run: | - git config add user.name "BitShares Github Autobuild" - git config add user.email "bga@noreply" + git config --add user.name "BitShares Github Autobuild" + git config --add user.email "bga@noreply" BRANCH="autobuild-signatures-win-$(date -u +%Y-%m-%dT%H:%M:%SZ)" git checkout -b "$BRANCH" git add signatures From 8f2947cf4a08326bbb892a864f3f064bb0be03b2 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Fri, 29 Nov 2019 17:39:17 +0100 Subject: [PATCH 18/20] fixup_branch --- .github/workflows/build-and-sign.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index dbc252d..c7f4e0a 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -135,7 +135,7 @@ jobs: run: | git config --add user.name "BitShares Github Autobuild" git config --add user.email "bga@noreply" - BRANCH="autobuild-signatures-linux-$(date -u +%Y-%m-%dT%H:%M:%SZ)" + BRANCH="autobuild-signatures-linux-$(date -u +%Y%m%dT%H%M%SZ)" git checkout -b "$BRANCH" git add signatures git commit -m "Add Linux autobuild signature for ${{ github.event.deployment.payload.coreversion }}" @@ -210,7 +210,7 @@ jobs: run: | git config --add user.name "BitShares Github Autobuild" git config --add user.email "bga@noreply" - BRANCH="autobuild-signatures-osx-$(date -u +%Y-%m-%dT%H:%M:%SZ)" + BRANCH="autobuild-signatures-osx-$(date -u +%Y%m%dT%H%M%SZ)" git checkout -b "$BRANCH" git add signatures git commit -m "Add OSX autobuild signature for ${{ github.event.deployment.payload.coreversion }}" @@ -269,7 +269,7 @@ jobs: run: | git config --add user.name "BitShares Github Autobuild" git config --add user.email "bga@noreply" - BRANCH="autobuild-signatures-win-$(date -u +%Y-%m-%dT%H:%M:%SZ)" + BRANCH="autobuild-signatures-win-$(date -u +%Y%m%dT%H%M%SZ)" git checkout -b "$BRANCH" git add signatures git commit -m "Add win autobuild signature for ${{ github.event.deployment.payload.coreversion }}" From cbd102ed1ecc2dc8e5108f909d59dd1d6c8716ae Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sat, 30 Nov 2019 12:44:59 +0100 Subject: [PATCH 19/20] Fixup token --- .github/workflows/build-and-sign.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index c7f4e0a..21abf20 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -139,7 +139,7 @@ jobs: git checkout -b "$BRANCH" git add signatures git commit -m "Add Linux autobuild signature for ${{ github.event.deployment.payload.coreversion }}" - git push "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$BRANCH" + git push "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${GITHUB_REPOSITORY}.git" "$BRANCH" - name: Upload build artifacts uses: actions/upload-artifact@master with: @@ -214,7 +214,7 @@ jobs: git checkout -b "$BRANCH" git add signatures git commit -m "Add OSX autobuild signature for ${{ github.event.deployment.payload.coreversion }}" - git push "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$BRANCH" + git push "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${GITHUB_REPOSITORY}.git" "$BRANCH" - name: Upload build artifacts uses: actions/upload-artifact@master with: @@ -273,7 +273,7 @@ jobs: git checkout -b "$BRANCH" git add signatures git commit -m "Add win autobuild signature for ${{ github.event.deployment.payload.coreversion }}" - git push "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$BRANCH" + git push "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${GITHUB_REPOSITORY}.git" "$BRANCH" - name: Upload build artifacts uses: actions/upload-artifact@master with: From 643482750fed9f10bcef0b38a874fddba6c82e2f Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sun, 1 Dec 2019 10:03:57 +0100 Subject: [PATCH 20/20] Use artifacts from past run as a shortcut --- .github/workflows/build-and-sign.yml | 31 +++++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml index 21abf20..e553c00 100644 --- a/.github/workflows/build-and-sign.yml +++ b/.github/workflows/build-and-sign.yml @@ -21,8 +21,13 @@ jobs: run: | mkdir -p docker echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" - vendor/gitian-builder/bin/make-base-vm --docker --suite bionic - docker save base-bionic-amd64 | xz >docker/base-bionic-amd64.tar.xz + # Attempt short-cut... + curl -L -o docker/base-bionic-amd64.zip https://github.com/pmconrad/bitshares-gitian/suites/335677056/artifacts/497179 + unzip -j -d docker docker/base-bionic-amd64.zip + if [ ! -r docker/base-bionic-amd64.tar.xz ]; then + vendor/gitian-builder/bin/make-base-vm --docker --suite bionic + docker save base-bionic-amd64 | xz >docker/base-bionic-amd64.tar.xz + fi - name: Upload build artifacts if: steps.cache-bionic.outputs.cache-hit != 'true' uses: actions/upload-artifact@master @@ -48,8 +53,13 @@ jobs: run: | mkdir -p docker echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" - vendor/gitian-builder/bin/make-base-vm --docker --suite xenial - docker save base-xenial-amd64 | xz >docker/base-xenial-amd64.tar.xz + # Attempt short-cut... + curl -L -o docker/base-xenial-amd64.zip https://github.com/pmconrad/bitshares-gitian/suites/335677056/artifacts/497178 + unzip -j -d docker docker/base-xenial-amd64.zip + if [ ! -r docker/base-xenial-amd64.tar.xz ]; then + vendor/gitian-builder/bin/make-base-vm --docker --suite xenial + docker save base-xenial-amd64 | xz >docker/base-xenial-amd64.tar.xz + fi - name: Upload build artifacts if: steps.cache-xenial.outputs.cache-hit != 'true' uses: actions/upload-artifact@master @@ -75,11 +85,16 @@ jobs: if: steps.cache-osx.outputs.cache-hit != 'true' run: | echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" - pushd ../osxcross - tools/gen_sdk_package.sh - popd mkdir -p osx - mv ../osxcross/*MacOSX* osx + # Attempt short-cut... + curl -L -o osx/MacOSX10.15.zip https://github.com/pmconrad/bitshares-gitian/suites/335677056/artifacts/497177 + unzip -j -d osx osx/MacOSX10.15.zip + if [ ! -r osx/MacOSX10.15.sdk.tar.xz ]; then + pushd ../osxcross + tools/gen_sdk_package.sh + popd + mv ../osxcross/*MacOSX* osx + fi - name: Upload SDK if: steps.cache-osx.outputs.cache-hit != 'true' uses: actions/upload-artifact@master