#!/usr/bin/env bash
# Trustico® CaaS for Google Cloud : Optional Extra. An alert when a run of the job fails.
#
# Purpose
# The job does not send a message when a run fails. This script sets up an alert in your project that e-mails you, or notifies you in another way you choose, whenever a run of the job fails. The message names the job and says where to read why. Running the script again changes nothing : to change the alert, or to take a new version of this script, delete the alert under Monitoring, Alerting, Policies and run the script again.
#
# Before You Start
# Create the place the alert goes to : in the console, open Monitoring, Alerting, Edit notification channels, and add an e-mail address or another kind of channel.
#
# The Questions
# Your project. Your job, picked from a list. The channel the alert goes to, picked from a list.
#
# Running the Script
# Open Cloud Shell in the Google Cloud console and run these two commands.
# curl -O https://cdn.trustico.com/caas/gcloud/create-failed-execution-alert.sh
# bash create-failed-execution-alert.sh
# Answer the questions. Type yes when it shows you your answers.
#
# The Result
# It prints CREATED with the name of the alert, or EXISTS ALREADY if it was set up before. The alert is listed in the console under Monitoring, Alerting, Policies.
#
# Afterwards
# When a run fails, your channel gets a message naming the job and pointing you to its log under Cloud Run, Jobs, the job, History. The alert closes on its own half an hour after the last failed run. To remove it, delete it under Monitoring, Alerting, Policies.
#
# If you prefer, give the answers on the command line, in this order : project, job name, and the channel by its resource name (projects/YOUR-PROJECT-ID/notificationChannels/NUMBER, shown by gcloud beta monitoring channels list). Several channels can be given, separated by commas.
# The metric and its label values are Cloud Run's own : run.googleapis.com/job/completed_execution_count, resource cloud_run_job, label result in {succeeded, failed}.
set -euo pipefail

fail() { echo "$1" >&2; exit 1; }

# ask VARIABLE "question" "default" : one question, Enter keeps the default when there is one.
ask() {
  local var="$1" question="$2" default="${3:-}" answer
  if [[ -n "$default" ]]; then
    read -r -p "  $question [$default] : " answer
    answer="${answer:-$default}"
  else
    read -r -p "  $question : " answer
  fi
  printf -v "$var" '%s' "$answer"
}

# choose VARIABLE "what" item... : one item is offered as the default ; several are numbered to pick from, or another can be typed. An item may be "value<TAB>label" : the label is shown, the value is kept.
choose() {
  local var="$1" what="$2"
  shift 2
  local items=("$@") answer i
  if (( ${#items[@]} == 0 )); then
    ask "$var" "$what"
    return
  fi
  if (( ${#items[@]} == 1 )); then
    ask "$var" "$what" "${items[0]%%$'\t'*}"
    return
  fi
  echo "  $what :"
  for i in "${!items[@]}"; do
    echo "    $((i + 1))) ${items[$i]#*$'\t'}"
  done
  while true; do
    read -r -p "  the number of your choice, or type another : " answer
    if [[ "$answer" =~ ^[0-9]+$ ]]; then
      if (( 10#$answer >= 1 && 10#$answer <= ${#items[@]} )); then
        printf -v "$var" '%s' "${items[$((10#$answer - 1))]%%$'\t'*}"
        return
      fi
      echo "  there is no choice $answer : type a number from 1 to ${#items[@]}"
      continue
    fi
    if [[ -n "$answer" ]]; then
      printf -v "$var" '%s' "$answer"
      return
    fi
  done
}

if (( $# >= 3 )); then
  PROJECT="$1"; JOB="$2"; CHANNEL_LIST="$3"
elif (( $# == 0 )); then
  echo "=== YOUR VALUES"
  ask PROJECT "Your project ID" "$(gcloud config get-value project 2>/dev/null || true)"
  mapfile -t JOBS < <(gcloud run jobs list --project="$PROJECT" --format="value(metadata.name)" 2>/dev/null | sort || true)
  choose JOB "Your job" "${JOBS[@]}"
  # Each channel is shown by its display name and its kind ; its resource name is what the policy takes.
  mapfile -t CHANNELS_FOUND < <(gcloud beta monitoring channels list --project="$PROJECT" --format="value(name,displayName,type)" 2>/dev/null | awk -F'\t' '{ printf "%s\t%s (%s)\n", $1, $2, $3 }' || true)
  choose CHANNEL_LIST "The notification channel the alert goes to" "${CHANNELS_FOUND[@]}"
  echo
  echo "  project $PROJECT, job $JOB, channel $CHANNEL_LIST"
  read -r -p "  Create it now ? Type yes to continue : " GO
  [[ "$GO" == "yes" ]] || { echo "NOTHING CREATED"; exit 0; }
else
  echo "USAGE : bash $0 (and answer the questions), or bash $0 YOUR-PROJECT-ID YOUR-JOB-NAME YOUR-CHANNEL-RESOURCE-NAME"
  exit 2
fi

echo "=== GUARDS"
gcloud projects describe "$PROJECT" --format="value(projectId)" >/dev/null 2>&1 || fail "PROJECT NOT FOUND, OR NO ACCESS TO IT : $PROJECT"
[[ "$CHANNEL_LIST" == projects/* ]] || fail "THE CHANNEL MUST BE GIVEN BY ITS RESOURCE NAME, projects/YOUR-PROJECT-ID/notificationChannels/... : $CHANNEL_LIST"
gcloud run jobs list --project="$PROJECT" --format="value(metadata.name)" 2>/dev/null | grep -qx -- "$JOB" || fail "JOB NOT FOUND : $JOB (gcloud run jobs list shows the jobs of the project)"
# The comma-separated channels as the JSON list the policy takes.
CHANNELS="$(printf '%s' "$CHANNEL_LIST" | awk -F, '{ for (i = 1; i <= NF; i++) { gsub(/^ +| +$/, "", $i); printf "%s\"%s\"", (i > 1 ? ", " : ""), $i } }')"
CHANNELS="[${CHANNELS}]"

NAME="CaaS job : ${JOB} failed execution"
# The lookup's own failure (a lapsed login, the wrong project, the API not enabled) must stop the script, not read as "no policy yet" and create a second copy on a re-run : its status is checked on its own line, and set -e stops here on a non-zero one.
EXISTING="$(gcloud monitoring policies list --project="$PROJECT" --filter="displayName=\"${NAME}\"" --format="value(name)")"
if [[ -n "$EXISTING" ]]; then
  echo "EXISTS ALREADY : ${NAME}"
  exit 0
fi

echo "=== THE ALERT"
WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT
cat > "${WORK}/policy.json" <<JSON
{
  "displayName": "${NAME}",
  "combiner": "OR",
  "conditions": [{
    "displayName": "An execution of ${JOB} completed with result failed",
    "conditionThreshold": {
      "filter": "metric.type=\"run.googleapis.com/job/completed_execution_count\" AND resource.type=\"cloud_run_job\" AND resource.label.job_name=\"${JOB}\" AND metric.label.result=\"failed\"",
      "aggregations": [{
        "alignmentPeriod": "300s",
        "perSeriesAligner": "ALIGN_SUM",
        "crossSeriesReducer": "REDUCE_SUM",
        "groupByFields": []
      }],
      "comparison": "COMPARISON_GT",
      "thresholdValue": 0,
      "duration": "0s",
      "trigger": { "count": 1 }
    }
  }],
  "notificationChannels": ${CHANNELS},
  "alertStrategy": { "autoClose": "1800s", "notificationPrompts": ["OPENED", "CLOSED"] },
  "documentation": {
    "mimeType": "text/markdown",
    "content": "An execution of the Cloud Run job ${JOB} ended with exit 1 : an SSL Certificate is not current or not installed. The job's own log says which SSL Certificate and why (the SSL CERTIFICATE NOT CURRENT OR NOT INSTALLED line and the lines above it) : Cloud Run, Jobs, ${JOB}, History, the execution. The next scheduled run retries from scratch ; a rejection by Google is in the INSTALL FAILED line with Google's own message."
  }
}
JSON
gcloud monitoring policies create --project="$PROJECT" --policy-from-file="${WORK}/policy.json" >/dev/null
echo "CREATED : ${NAME}"