Add error handling and configurable retries to curl calls

README.md

@@ -43,7 +43,8 @@ Parameters:
  * `description` - a short description of the status
  * `description_path` - path to an input file whose data is the value of `description`
  * `target_url` - the target URL to associate with the status (default: concourse build link)
-
+ * `retry_count` - the number of times to retry for the status to be ready (default: 5)
+ * `retry_delay` - the amount of time in seconds to wait while retrying for the status (default: 3)
 
 ## Example
 

bin/common.sh

@@ -50,5 +50,55 @@ curlgh () {
   else
     skip_verify_arg=""
   fi
-  curl $skip_verify_arg -s -H "Authorization: token $source_access_token" $@
+
+  REMAINING_TRIES="${retry_count:-5}"
+
+  while true; do
+    # Output the response headers and body to two separate files so that we can easily work on them both
+    curl $skip_verify_arg -s -D/tmp/responseheaders -H "Authorization: token $source_access_token" $@ > /tmp/rawresponse
+
+    http_status=$(head -n1 /tmp/responseheaders | sed 's|HTTP.* \([0-9]*\) .*|\1|')
+    # If HTTP status is OK, break the retry loop now to carry on (skip all error handling & retries)
+    if [ "$http_status" -eq "200" ]; then
+      break;
+    fi
+
+    if [ "$http_status" -ge 400 ]; then
+      if [ "$http_status" -le 499 ]; then # 400-499 range
+        if [ $(grep -i 'rate-limit' /tmp/rawresponse || echo '0') -ge 1 ]; then
+          now=$(date "+%s")
+          ratelimit_reset=$(cat /tmp/responseheaders | sed -n 's|X-RateLimit-Reset: \([0-9]*\)|\1|p')
+
+          sleep_duration="$((ratelimit_reset - now))"
+          # If our system clock is in advance to GitHub's the result of sleep_duration might be a negative number
+          if [[ "$sleep_duration" -lt 1 ]]; then
+            sleep_duration="1"
+          fi
+          echo "Limited by the API rate limit. Script will retry at $( date -d@$((now + sleep_duration)) )" >&2
+        else
+          fatal "Authentication error against the GitHub API"
+        fi
+      else # 500+ range
+        echo "Unexpected HTTP $(echo $http_status) when querying the GitHub API" >&2
+        sleep_duration="${retry_delay:-3}"
+      fi
+    else # Other status code that's not 200 OK, nor in the 400+ range
+      fatal "Unexpected HTTP status code when querying the GitHub API: $(echo $http_status)"
+    fi
+
+
+    # Exit if we have reach the maximum number of attemps, or sleep and retry otherwise
+    if [ "$REMAINING_TRIES" -le 0 ]; then
+      fatal "Maximum number of attempts reached while trying to query the GitHub API"
+    fi
+
+    echo "Will retry in $sleep_duration seconds" >&2
+
+    REMAINING_TRIES=$(($REMAINING_TRIES - 1))
+
+    sleep $sleep_duration
+
+  done
+
+  cat /tmp/rawresponse
 }

bin/in

@@ -11,6 +11,13 @@ eval $( jq -r '{
   "version_status": .version.status
 } | to_entries[] | .key + "=" + @sh "\(.value)"' < /tmp/stdin )
 
+#
+# check retry counter
+#
+
+REMAINING_TRIES="${retry_count:-5}"
+
+while true; do
 
 #
 # lookup
@@ -25,14 +32,29 @@ curlgh "$source_endpoint/repos/$source_repository/commits/$version_commit/status
     }' \
     > /tmp/status
 
-
 #
 # validate
 #
+set +e
+jq -e '.status' < /tmp/status > /dev/null
+check_status="$?"
+set -e
+
+if [ "$check_status" -eq 0 ]; then
+  break
+elif [ "$check_status" -gt 0 ] && [ "$REMAINING_TRIES" -le 0 ]; then
+  fatal "Status not found on $( jq -r '.sha' < /tmp/status )"
+fi
+
+#
+# decrease retry counter and loop
+#
+
+REMAINING_TRIES=$(($REMAINING_TRIES - 1))
 
-jq -e '.status' < /tmp/status > /dev/null \
-  || fatal "Status not found on $( jq -r '.sha' < /tmp/status )"
+sleep "${retry_delay:-3}"
 
+done
 
 #
 # concourse

bin/out

@@ -91,9 +91,9 @@ jq -c -n \
 # check retry counter
 #
 
-REMAINING_TRIES=5
+REMAINING_TRIES="${retry_count:-5}"
 
-while [[ $REMAINING_TRIES -gt 0 ]]; do
+while true; do
 
 #
 # lookup
@@ -112,17 +112,24 @@ curlgh "$source_endpoint/repos/$source_repository/commits/$source_branch/status"
 # validate
 #
 
-[[ -s /tmp/status ]] \
-  && jq -e '.status' < /tmp/status > /dev/null \
-  && break
+set +e
+jq -e '.status' < /tmp/status > /dev/null
+check_status="$?"
+set -e
+
+if [ "$check_status" -eq 0 ]; then
+  break
+elif [ "$check_status" -gt 0 ] && [ "$REMAINING_TRIES" -le 0 ]; then
+  fatal "Status not found on $( jq -r '.sha' < /tmp/status )"
+fi
 
 #
 # decrease retry counter and loop
 #
 
 REMAINING_TRIES=$(($REMAINING_TRIES - 1))
 
-sleep 1
+sleep "${retry_delay:-3}"
 
 done
 

test/in/error-503.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+set -eu
+
+DIR=$( dirname "$0" )/../..
+
+cat <<EOF | nc -l -s 127.0.0.1 -p 9192 > $TMPDIR/http.req-$$ &
+HTTP/1.0 503 Service Unavailable
+
+EOF
+
+in_dir=$TMPDIR/status-$$
+
+mkdir $in_dir
+
+set +e
+
+$DIR/bin/in "$in_dir" > $TMPDIR/resource-$$ 2>&1 <<EOF
+{
+  "version": {
+    "commit": "6dcb09b5b57875f334f61aebed695e2e4193db5e",
+    "status": "2"
+  },
+  "source": {
+    "access_token": "test-token",
+    "context": "test-context",
+    "endpoint": "http://127.0.0.1:9192",
+    "repository": "dpb587/test-repo"
+  }
+}
+EOF
+
+exitcode=$?
+
+set -e
+
+if ! [ "1" == "$exitcode" ] ; then
+  echo "FAILURE: Expected exit code 1"
+  exit 1
+fi
+
+if ! grep -q 'Status not found on 6dcb09b5b57875f334f61aebed695e2e4193db5e' $TMPDIR/resource-$$ ; then
+  echo "FAILURE: Unexpected failure message"
+  cat $TMPDIR/resource-$$
+  exit 1
+fi

test/in/missing-status.sh

@@ -19,7 +19,7 @@ mkdir $in_dir
 
 set +e
 
-$DIR/bin/in "$in_dir" > $TMPDIR/resource-$$ 2>&1 <<EOF
+retry_count=0 $DIR/bin/in "$in_dir" > $TMPDIR/resource-$$ 2>&1 <<EOF
 {
   "version": {
     "commit": "6dcb09b5b57875f334f61aebed695e2e4193db5e",