diff --git a/.github/gpg-wrapper b/.github/gpg-wrapper new file mode 100755 index 0000000..a825b70 --- /dev/null +++ b/.github/gpg-wrapper @@ -0,0 +1,3 @@ +#!/bin/bash + +exec gpg --batch --passphrase-fd 3 --pinentry-mode loopback --detach-sign "$@" 3<<<"${PGP_PASS}" diff --git a/.github/trigger-workflow.sh b/.github/trigger-workflow.sh new file mode 100755 index 0000000..0084004 --- /dev/null +++ b/.github/trigger-workflow.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +if [ "$#" != 1 ]; then + echo "Usage: $0 " 1>&2 + exit 1 +fi + +if [ ! -r "$(dirname $0)/token" ]; then + echo "Create an API token on github with the 'deployment' privilege and" 1>&2 + echo "put it into '$(dirname $0)/token'." 1>&2 + exit 1 +fi + +curl -H "Authorization: token $(head -c 40 "$(dirname $0)/token")" \ + -H "Content-Type: application/json" \ + --data '{"ref":"action-test","required_contexts":[],"payload":{"coreversion":"'"$1"'"}}' \ + https://api.github.com/repos/pmconrad/bitshares-gitian/deployments + diff --git a/.github/workflows/build-and-sign.yml b/.github/workflows/build-and-sign.yml new file mode 100644 index 0000000..e553c00 --- /dev/null +++ b/.github/workflows/build-and-sign.yml @@ -0,0 +1,296 @@ +on: deployment +env: + USE_DOCKER: 1 +jobs: + prepare-bionic: + name: Prepare "bionic" Gitian build environment + runs-on: ubuntu-latest + steps: + - name: Load Prep Cache + id: cache-bionic + uses: actions/cache@v1 + with: + path: docker + key: prep-cache-bionic + - uses: actions/checkout@v1 + if: steps.cache-bionic.outputs.cache-hit != 'true' + with: + submodules: recursive + - name: Prepare + if: steps.cache-bionic.outputs.cache-hit != 'true' + run: | + mkdir -p docker + echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" + # Attempt short-cut... + curl -L -o docker/base-bionic-amd64.zip https://github.com/pmconrad/bitshares-gitian/suites/335677056/artifacts/497179 + unzip -j -d docker docker/base-bionic-amd64.zip + if [ ! -r docker/base-bionic-amd64.tar.xz ]; then + vendor/gitian-builder/bin/make-base-vm --docker --suite bionic + docker save base-bionic-amd64 | xz >docker/base-bionic-amd64.tar.xz + fi + - name: Upload build artifacts + if: steps.cache-bionic.outputs.cache-hit != 'true' + uses: actions/upload-artifact@master + with: + name: docker-bionic + path: docker/base-bionic-amd64.tar.xz + prepare-xenial: + name: Prepare "xenial" Gitian build environment + runs-on: ubuntu-latest + steps: + - name: Load Prep Cache + id: cache-xenial + uses: actions/cache@v1 + with: + path: docker + key: prep-cache-xenial + - uses: actions/checkout@v1 + if: steps.cache-xenial.outputs.cache-hit != 'true' + with: + submodules: recursive + - name: Prepare + if: steps.cache-xenial.outputs.cache-hit != 'true' + run: | + mkdir -p docker + echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" + # Attempt short-cut... + curl -L -o docker/base-xenial-amd64.zip https://github.com/pmconrad/bitshares-gitian/suites/335677056/artifacts/497178 + unzip -j -d docker docker/base-xenial-amd64.zip + if [ ! -r docker/base-xenial-amd64.tar.xz ]; then + vendor/gitian-builder/bin/make-base-vm --docker --suite xenial + docker save base-xenial-amd64 | xz >docker/base-xenial-amd64.tar.xz + fi + - name: Upload build artifacts + if: steps.cache-xenial.outputs.cache-hit != 'true' + uses: actions/upload-artifact@master + with: + name: docker-xenial + path: docker/base-xenial-amd64.tar.xz + prepare-mac: + name: Scrape mac-SDK + runs-on: macos-latest + steps: + - name: Load Prep Cache + id: cache-osx + uses: actions/cache@v1 + with: + path: osx + key: prep-cache-osx + - uses: actions/checkout@v1 + if: steps.cache-osx.outputs.cache-hit != 'true' + with: + repository: tpoechtrager/osxcross + ref: e0a171828a72a0d7ad4409489033536590008ebf + - name: Package + if: steps.cache-osx.outputs.cache-hit != 'true' + run: | + echo "Argh - caching only works for push and pull events. See https://github.com/actions/cache/issues/63" + mkdir -p osx + # Attempt short-cut... + curl -L -o osx/MacOSX10.15.zip https://github.com/pmconrad/bitshares-gitian/suites/335677056/artifacts/497177 + unzip -j -d osx osx/MacOSX10.15.zip + if [ ! -r osx/MacOSX10.15.sdk.tar.xz ]; then + pushd ../osxcross + tools/gen_sdk_package.sh + popd + mv ../osxcross/*MacOSX* osx + fi + - name: Upload SDK + if: steps.cache-osx.outputs.cache-hit != 'true' + uses: actions/upload-artifact@master + with: + name: MacOS-SDK + path: osx/MacOSX10.15.sdk.tar.xz + linux: + name: Build and sign linux binaries + needs: prepare-xenial + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + - name: Load Prep Cache + id: cache-xenial + uses: actions/cache@v1 + with: + path: docker + key: prep-cache-xenial + - name: Poor man's cache + if: steps.cache-xenial.outputs.cache-hit != 'true' + uses: actions/download-artifact@master + with: + name: docker-xenial + path: docker/ + - name: Load Build Cache + uses: actions/cache@v1 + with: + path: vendor/gitian-builder/cache/bitshares-core-linux + key: build-cache-linux-${{ github.event.deployment.payload.coreversion }} + restore-keys: | + build-cache-linux- + - name: Prepare + run: | + docker load -i docker/base-xenial-amd64.tar.xz + - name: Build and Sign + run: | + sudo apt-get update + sudo apt-get install ruby + export GNUPGHOME=`pwd`/.gpghome + ./run-gitian -b -O linux \ + ${{ github.event.deployment.payload.coreversion }} \ + -j 1 \ + || ( echo Install log: && cat vendor/gitian-builder/var/install.log \ + && echo Build log: && cat vendor/gitian-builder/var/build.log \ + && false ) + PGP_PASS="${{ secrets.PGP_PASSPHRASE }}" \ + ./run-gitian -s BitShares-Gitian-Auto-Build-Signer \ + -O linux -p `pwd`/.github/gpg-wrapper \ + ${{ github.event.deployment.payload.coreversion }} + - name: Push Signatures + run: | + git config --add user.name "BitShares Github Autobuild" + git config --add user.email "bga@noreply" + BRANCH="autobuild-signatures-linux-$(date -u +%Y%m%dT%H%M%SZ)" + git checkout -b "$BRANCH" + git add signatures + git commit -m "Add Linux autobuild signature for ${{ github.event.deployment.payload.coreversion }}" + git push "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${GITHUB_REPOSITORY}.git" "$BRANCH" + - name: Upload build artifacts + uses: actions/upload-artifact@master + with: + name: Linux-Binaries + path: vendor/gitian-builder/build/out + mac: + name: Build and sign mac binaries + needs: + - prepare-bionic + - prepare-mac + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + - name: Load Docker Prep Cache + id: cache-bionic + uses: actions/cache@v1 + with: + path: docker + key: prep-cache-bionic + - name: Poor man's docker cache + if: steps.cache-bionic.outputs.cache-hit != 'true' + uses: actions/download-artifact@master + with: + name: docker-bionic + path: docker/ + - name: Load OSX Prep Cache + id: cache-osx + uses: actions/cache@v1 + with: + path: osx + key: prep-cache-osx + - name: Poor man's OSX cache + if: steps.cache-osx.outputs.cache-hit != 'true' + uses: actions/download-artifact@master + with: + name: MacOS-SDK + path: osx/ + - name: Load Build Cache + uses: actions/cache@v1 + with: + path: vendor/gitian-builder/cache/bitshares-core-osx + key: build-cache-osx-${{ github.event.deployment.payload.coreversion }} + restore-keys: | + build-cache-osx- + - name: Prepare + run: | + docker load -i docker/base-bionic-amd64.tar.xz + - name: Build and Sign + run: | + sudo apt-get update + sudo apt-get install ruby + mkdir -p vendor/gitian-builder/inputs + mv osx/* vendor/gitian-builder/inputs + ./run-gitian -b -O osx \ + ${{ github.event.deployment.payload.coreversion }} \ + -j 1 \ + || ( echo Install log: && cat vendor/gitian-builder/var/install.log \ + && echo Build log: && cat vendor/gitian-builder/var/build.log \ + && false ) + export GNUPGHOME=`pwd`/.gpghome + PGP_PASS="${{ secrets.PGP_PASSPHRASE }}" \ + ./run-gitian -s BitShares-Gitian-Auto-Build-Signer \ + -O osx -p `pwd`/.github/gpg-wrapper \ + ${{ github.event.deployment.payload.coreversion }} + - name: Push Signatures + run: | + git config --add user.name "BitShares Github Autobuild" + git config --add user.email "bga@noreply" + BRANCH="autobuild-signatures-osx-$(date -u +%Y%m%dT%H%M%SZ)" + git checkout -b "$BRANCH" + git add signatures + git commit -m "Add OSX autobuild signature for ${{ github.event.deployment.payload.coreversion }}" + git push "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${GITHUB_REPOSITORY}.git" "$BRANCH" + - name: Upload build artifacts + uses: actions/upload-artifact@master + with: + name: Mac Binaries + path: vendor/gitian-builder/build/out + windows: + name: Build and sign windows binaries + needs: prepare-bionic + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + - name: Load Docker Prep Cache + id: cache-bionic + uses: actions/cache@v1 + with: + path: docker + key: prep-cache-bionic + - name: Poor man's cache + if: steps.cache-bionic.outputs.cache-hit != 'true' + uses: actions/download-artifact@master + with: + name: docker-bionic + path: docker/ + - name: Load Build Cache + uses: actions/cache@v1 + with: + path: vendor/gitian-builder/cache/bitshares-core-win + key: build-cache-win-${{ github.event.deployment.payload.coreversion }} + restore-keys: | + build-cache-win- + - name: Prepare + run: | + docker load -i docker/base-bionic-amd64.tar.xz + - name: Build and Sign + run: | + sudo apt-get update + sudo apt-get install ruby + ./run-gitian -b -O win \ + ${{ github.event.deployment.payload.coreversion }} \ + -j 1 \ + || ( echo Install log: && cat vendor/gitian-builder/var/install.log \ + && echo Build log: && cat vendor/gitian-builder/var/build.log \ + && false ) + export GNUPGHOME=`pwd`/.gpghome + PGP_PASS="${{ secrets.PGP_PASSPHRASE }}" \ + ./run-gitian -s BitShares-Gitian-Auto-Build-Signer \ + -O win -p `pwd`/.github/gpg-wrapper \ + ${{ github.event.deployment.payload.coreversion }} + - name: Push Signatures + run: | + git config --add user.name "BitShares Github Autobuild" + git config --add user.email "bga@noreply" + BRANCH="autobuild-signatures-win-$(date -u +%Y%m%dT%H%M%SZ)" + git checkout -b "$BRANCH" + git add signatures + git commit -m "Add win autobuild signature for ${{ github.event.deployment.payload.coreversion }}" + git push "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${GITHUB_REPOSITORY}.git" "$BRANCH" + - name: Upload build artifacts + uses: actions/upload-artifact@master + with: + name: Windows Binaries + path: vendor/gitian-builder/build/out diff --git a/.gitignore b/.gitignore index d0768eb..df40e9a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /docker /var +.github/token diff --git a/.gpghome/pubring.gpg b/.gpghome/pubring.gpg new file mode 100644 index 0000000..f829d46 Binary files /dev/null and b/.gpghome/pubring.gpg differ diff --git a/.gpghome/secring.gpg b/.gpghome/secring.gpg new file mode 100644 index 0000000..4217c5f Binary files /dev/null and b/.gpghome/secring.gpg differ diff --git a/README.md b/README.md index 5f082f6..091ab7f 100644 --- a/README.md +++ b/README.md @@ -44,10 +44,10 @@ You must have GnuPG installed and on your path as `gpg`. Instructions on how to install required software on some OSes and prepare a gitian base environment can be found [here](https://github.com/devrandom/gitian-builder/blob/master/README.md). You should follow the described steps until you have completed the "Sanity-testing" section successfully. Be sure to use the "bionic" suite for your base image. -If you want to build build executables for Mac you'll need to download MacOSX SDK 10.14. -It is contained in the Xcode 10.3 distribution, which is available at https://developer.apple.com/xcode/resources/ under "Command Line Tools & Older Versions of Xcode". . +If you want to build build executables for Mac you'll need to download MacOSX SDK 10.15. +It is contained in the Xcode 11.1 distribution, which is available at https://developer.apple.com/xcode/resources/ under "Command Line Tools & Older Versions of Xcode". . After downloading Xcode, you can extract the SDK as described [here](https://github.com/tpoechtrager/osxcross#packaging-the-sdk). -The resulting file `MacOSX10.14.sdk.tar.xz` must be put in the `vendor/gitian-builder/inputs` subdirectory. +The resulting file `MacOSX10.15.sdk.tar.xz` must be put in the `vendor/gitian-builder/inputs` subdirectory. ### Example for Docker diff --git a/descriptors/bitshares-core-linux.yml b/descriptors/bitshares-core-linux.yml index 8269eea..621be47 100644 --- a/descriptors/bitshares-core-linux.yml +++ b/descriptors/bitshares-core-linux.yml @@ -80,6 +80,7 @@ script: | popd cd bitshares + GIT_SSL_NO_VERIFY=true git submodule update --init --recursive cp libraries/wallet/wallet.cpp{,.orig} sed -i '/__DATE__/d' libraries/wallet/wallet.cpp # Workaround for #1791 diff --git a/descriptors/bitshares-core-osx.yml b/descriptors/bitshares-core-osx.yml index 62b1f12..fe2b921 100644 --- a/descriptors/bitshares-core-osx.yml +++ b/descriptors/bitshares-core-osx.yml @@ -26,7 +26,7 @@ files: - openssl-1.1.1c.tar.gz - curl-7.65.0.tar.bz2 - boost_1_69_0.tar.bz2 -- MacOSX10.14.sdk.tar.xz +- MacOSX10.15.sdk.tar.xz - e0a171828a72a0d7ad4409489033536590008ebf.tar.gz script: | set -e -o pipefail @@ -126,6 +126,7 @@ script: | popd cd bitshares + GIT_SSL_NO_VERIFY=true git submodule update --init --recursive sed -i '/__DATE__/d' libraries/wallet/wallet.cpp sed -i '/hardfork.hpp/d' libraries/chain/CMakeLists.txt sed -i "/[ep].configure/s/$/ --host=${DARWIN%-} CC=${DARWIN}clang LD=${DARWIN}clang/" libraries/fc/CMakeLists.txt diff --git a/descriptors/bitshares-core-win.yml b/descriptors/bitshares-core-win.yml index cfdd7b7..0c2fd4e 100644 --- a/descriptors/bitshares-core-win.yml +++ b/descriptors/bitshares-core-win.yml @@ -116,6 +116,7 @@ script: | popd cd bitshares + GIT_SSL_NO_VERIFY=true git submodule update --init --recursive sed -i '/__DATE__/d' libraries/wallet/wallet.cpp sed -i '/hardfork.hpp/d' libraries/chain/CMakeLists.txt sed -i "/add_executable/alist( APPEND PLATFORM_SPECIFIC_LIBS $LIBS/lib/libcurl.a -lws2_32 -lpthread -lcrypt32 )" \ diff --git a/run-gitian b/run-gitian index 8a49802..0dd2811 100755 --- a/run-gitian +++ b/run-gitian @@ -11,12 +11,14 @@ BUILD= SIGN= VERIFY= OS=linux -while [ "$1" = "-b" -o "$1" = "-s" -o "$1" = "-v" -o "$1" = "-O" ]; do +GPG= +while [ "$1" = "-b" -o "$1" = "-s" -o "$1" = "-v" -o "$1" = "-O" -o "$1" = "-p" ]; do case "$1" in -b) BUILD=1; ;; -s) SIGN=1; KEY_ID="$2"; shift; ;; -v) VERIFY=1; ;; -O) OS="$2"; shift;; + -p) GPG="$1 $2"; shift;; esac shift done @@ -57,8 +59,8 @@ _EOL_ cat <<_EOL_ https://github.com/tpoechtrager/osxcross/archive/e0a171828a72a0d7ad4409489033536590008ebf.tar.gz 7ef00c27b76745d4b44e13f291df60318588aa7b5d1788aeba5aca569ac7e989 _EOL_ - if [ ! -r inputs/MacOSX10.14.sdk.tar.xz ]; then - echo "Please download MacOSX10.14.sdk.tar.xz and place it in `pwd`/inputs." 1>&2 + if [ ! -r inputs/MacOSX10.15.sdk.tar.xz ]; then + echo "Please download MacOSX10.15.sdk.tar.xz and place it in `pwd`/inputs." 1>&2 echo "Also be sure to read and understand Apple's licensing terms." 1>&2 exit 1 fi @@ -76,12 +78,13 @@ _EOL_ fi if [ "$#" -lt 1 ]; then - echo "Usage:\t$0 [-b | -s KEY_ID | -v] [-O linux|win|osx] VERSION [builder-options]" 1>&2 + echo "Usage:\t$0 [-b | -s KEY_ID | -v] [-O linux|win|osx] [-p GPG] VERSION [builder-options]" 1>&2 echo "\t$0 KEY_ID VERSION [builder-options]" 1>&2 echo "-b\tbuild" 1>&2 echo "-s\tsign" 1>&2 echo "-v\tverify" 1>&2 echo "-O\tchoose target operating system linux or win, defaults to linux" 1>&2 + echo "-p\tcommand to use for signing" 1>&2 echo "\t(will build and sign with KEY_ID if neither option is present)" 1>&2 echo "VERSION\tis the BitShares release version to build" 1>&2 echo "KEY_ID\tis the GPG key ID to use for signing" 1>&2 @@ -89,7 +92,7 @@ if [ "$#" -lt 1 ]; then echo "builder-options:" 1>&2 gbuild --help 2>&1 | tail -n +2 1>&2 echo "" 1>&2 - echo "You should allow at least 4G RAM for the build (-m 4096), more if" 1>&2 + echo "You should allow at least 5G RAM for the build (-m 4096), more if" 1>&2 echo "you want to build with more than one core!" 1>&2 exit 1 fi @@ -98,7 +101,7 @@ VERSION="$1" shift [ -z "$BUILD" ] || gbuild "../../descriptors/bitshares-core-$OS.yml" -c bitshares="$VERSION" "$@" -[ -z "$SIGN" ] || gsign --signer "$KEY_ID" --destination ../../signatures --release "$VERSION"-"$OS" \ +[ -z "$SIGN" ] || gsign --signer "$KEY_ID" $GPG --destination ../../signatures --release "$VERSION"-"$OS" \ "../../descriptors/bitshares-core-$OS.yml" [ -z "$VERIFY" ] || gverify --destination ../../signatures --release "$VERSION"-"$OS" \ "../../descriptors/bitshares-core-$OS.yml"