#!/usr/bin/env bash

set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly ROOT
readonly IMAGE="postgres:18.1-alpine@sha256:aa6eb304ddb6dd26df23d05db4e5cb05af8951cda3e0dc57731b771e0ef4ab29"
readonly NETWORK="ops-04-network"
readonly PRIMARY="ops-04-primary"
readonly STANDBY="ops-04-standby"
readonly RECOVERY="ops-04-recovery"
readonly END_RECOVERY="ops-04-end-recovery"
readonly MISSING_RECOVERY="ops-04-missing-recovery"
readonly BASE_VOLUME="ops-04-base"
readonly STANDBY_VOLUME="ops-04-standby-data"
readonly RECOVERY_VOLUME="ops-04-recovery-data"
readonly END_VOLUME="ops-04-end-data"
readonly MISSING_VOLUME="ops-04-missing-data"
readonly RESULTS="${ROOT}/results"

cleanup() {
  docker rm --force "${PRIMARY}" "${STANDBY}" "${RECOVERY}" \
    "${END_RECOVERY}" "${MISSING_RECOVERY}" \
    >/dev/null 2>&1 || true
  docker network rm "${NETWORK}" >/dev/null 2>&1 || true
  docker volume rm "${BASE_VOLUME}" "${STANDBY_VOLUME}" \
    "${RECOVERY_VOLUME}" "${END_VOLUME}" "${MISSING_VOLUME}" \
    >/dev/null 2>&1 || true
}
trap cleanup EXIT INT TERM

wait_ready() {
  local container="$1"
  local user="${2:-fixture}"
  local attempt
  for attempt in $(seq 1 120); do
    if docker exec "${container}" pg_isready --username "${user}" \
      --dbname recovery >/dev/null 2>&1; then
      return 0
    fi
    sleep 0.25
  done
  echo "PostgreSQL did not become ready in ${container}" >&2
  return 1
}

psql_command() {
  local container="$1"
  local command="$2"
  docker exec "${container}" psql --username fixture --dbname recovery \
    --no-psqlrc --set ON_ERROR_STOP=1 --tuples-only --no-align \
    --command "${command}" | tr -d '\n'
}

psql_file() {
  local container="$1"
  local file="$2"
  docker exec -i "${container}" psql --username fixture --dbname recovery \
    --no-psqlrc --set ON_ERROR_STOP=1 --tuples-only --no-align <"${file}"
}

cleanup
if [[ "${RESULTS}" != "${ROOT}/results" ]]; then
  echo "refusing to replace unexpected results directory" >&2
  exit 65
fi
if [[ -d "${RESULTS}" ]]; then
  chmod -R u+rwX "${RESULTS}"
fi
rm -rf "${RESULTS}"
mkdir -p "${RESULTS}/wal" "${RESULTS}/controls"
chmod 0777 "${RESULTS}/wal"

docker network create "${NETWORK}" >/dev/null
docker volume create "${BASE_VOLUME}" >/dev/null
docker volume create "${STANDBY_VOLUME}" >/dev/null
docker volume create "${RECOVERY_VOLUME}" >/dev/null
docker volume create "${END_VOLUME}" >/dev/null
docker volume create "${MISSING_VOLUME}" >/dev/null

docker run --detach --name "${PRIMARY}" --network "${NETWORK}" \
  --hostname primary --cpus=1 --memory=512m \
  --env POSTGRES_DB=recovery --env POSTGRES_USER=fixture \
  --env POSTGRES_PASSWORD=fixture \
  --volume "${RESULTS}/wal:/archive" \
  "${IMAGE}" postgres -c wal_level=replica -c max_wal_senders=10 \
  -c max_replication_slots=10 -c archive_mode=on \
  -c "archive_command=test ! -f /archive/%f && cp %p /archive/%f" \
  >/dev/null
wait_ready "${PRIMARY}"

psql_file "${PRIMARY}" "${ROOT}/schema.sql" >/dev/null
psql_file "${PRIMARY}" "${ROOT}/seed.sql" >/dev/null
psql_command "${PRIMARY}" \
  "create role replicator with replication login password 'replicator'" \
  >/dev/null
docker exec --user root "${PRIMARY}" sh -c \
  "printf '%s\n' 'host replication replicator all scram-sha-256' >> \"\$PGDATA/pg_hba.conf\""
psql_command "${PRIMARY}" "select pg_reload_conf()" >/dev/null

for volume in "${BASE_VOLUME}" "${STANDBY_VOLUME}"; do
  docker run --rm --user root --volume "${volume}:/backup" "${IMAGE}" \
    chown postgres:postgres /backup
  docker run --rm --user postgres --network "${NETWORK}" \
    --env PGPASSWORD=replicator --volume "${volume}:/backup" "${IMAGE}" \
    pg_basebackup --host primary --username replicator --pgdata /backup \
    --format plain --wal-method stream --checkpoint fast --write-recovery-conf
done

docker run --detach --name "${STANDBY}" --network "${NETWORK}" \
  --hostname standby --cpus=1 --memory=512m \
  --env PGDATA=/var/lib/postgresql/data \
  --volume "${STANDBY_VOLUME}:/var/lib/postgresql/data" \
  "${IMAGE}" >/dev/null
wait_ready "${STANDBY}"

restore_point_lsn="$(psql_command "${PRIMARY}" \
  "select pg_create_restore_point('before_destructive_delete')")"
psql_file "${PRIMARY}" "${ROOT}/post-target.sql" >/dev/null
psql_file "${PRIMARY}" "${ROOT}/delete.sql" >/dev/null
delete_lsn="$(psql_command "${PRIMARY}" "select pg_current_wal_lsn()")"
psql_command "${PRIMARY}" "select pg_switch_wal()" >/dev/null

for attempt in $(seq 1 120); do
  replay_lsn="$(psql_command "${STANDBY}" "select pg_last_wal_replay_lsn()")"
  replayed="$(psql_command "${PRIMARY}" \
    "select '${replay_lsn}'::pg_lsn >= '${delete_lsn}'::pg_lsn")"
  [[ "${replayed}" == "t" ]] && break
  sleep 0.25
done
if [[ "${replayed}" != "t" ]]; then
  echo "standby did not replay the destructive transaction" >&2
  exit 1
fi

primary_health="$(psql_command "${PRIMARY}" 'select 1')"
standby_health="$(psql_command "${STANDBY}" 'select 1')"
standby_row_count="$(psql_command "${STANDBY}" 'select count(*) from shipments')"
standby_known_count="$(psql_command "${STANDBY}" \
  "select count(*) from shipments where external_id='shipment-100'")"
set +e
psql_file "${STANDBY}" "${ROOT}/probe.sql" \
  >"${RESULTS}/controls/standby-probe.stdout" \
  2>"${RESULTS}/controls/standby-probe.stderr"
standby_probe_exit=$?
set -e
[[ ${standby_probe_exit} -ne 0 ]] || {
  echo "standby unexpectedly retained deleted shipment" >&2
  exit 1
}

psql_command "${STANDBY}" 'select pg_promote(true, 60)' >/dev/null
promoted_role="$(psql_command "${STANDBY}" 'select pg_is_in_recovery()')"
set +e
psql_file "${STANDBY}" "${ROOT}/probe.sql" \
  >"${RESULTS}/controls/promoted-probe.stdout" \
  2>"${RESULTS}/controls/promoted-probe.stderr"
promoted_probe_exit=$?
set -e
[[ ${promoted_probe_exit} -ne 0 && "${promoted_role}" == "f" ]] || {
  echo "promotion control did not produce an available wrong primary" >&2
  exit 1
}

# Recovering to the end of retained WAL is available but repeats the delete.
docker run --rm --user root --volume "${BASE_VOLUME}:/from:ro" \
  --volume "${END_VOLUME}:/to" "${IMAGE}" sh -c \
  'cp -a /from/. /to/ && touch /to/recovery.signal && printf "%s\n" "restore_command = '\''cp /archive/%f %p'\''" "recovery_target_action = '\''promote'\''" >> /to/postgresql.auto.conf'
docker run --detach --name "${END_RECOVERY}" --network "${NETWORK}" \
  --hostname end-recovery --cpus=1 --memory=512m \
  --env PGDATA=/var/lib/postgresql/data \
  --volume "${END_VOLUME}:/var/lib/postgresql/data" \
  --volume "${RESULTS}/wal:/archive:ro" "${IMAGE}" >/dev/null
wait_ready "${END_RECOVERY}"
set +e
psql_file "${END_RECOVERY}" "${ROOT}/probe.sql" \
  >"${RESULTS}/controls/end-of-wal-probe.stdout" \
  2>"${RESULTS}/controls/end-of-wal-probe.stderr"
end_probe_exit=$?
set -e
[[ ${end_probe_exit} -ne 0 ]] || {
  echo "end-of-WAL recovery unexpectedly avoided the delete" >&2
  exit 1
}

# Removing the WAL segment containing the named point prevents that target.
target_wal_file="$(psql_command "${PRIMARY}" \
  "select pg_walfile_name('${restore_point_lsn}'::pg_lsn)")"
mkdir -p "${RESULTS}/controls/missing-wal-archive"
cp "${RESULTS}/wal/"* "${RESULTS}/controls/missing-wal-archive/"
rm "${RESULTS}/controls/missing-wal-archive/${target_wal_file}"
chmod -R 0555 "${RESULTS}/controls/missing-wal-archive"
docker run --rm --user root --volume "${BASE_VOLUME}:/from:ro" \
  --volume "${MISSING_VOLUME}:/to" "${IMAGE}" sh -c \
  'cp -a /from/. /to/ && touch /to/recovery.signal && printf "%s\n" "restore_command = '\''cp /archive/%f %p'\''" "recovery_target_name = '\''before_destructive_delete'\''" "recovery_target_action = '\''promote'\''" >> /to/postgresql.auto.conf'
docker run --detach --name "${MISSING_RECOVERY}" --network "${NETWORK}" \
  --hostname missing-recovery --cpus=1 --memory=512m \
  --env PGDATA=/var/lib/postgresql/data \
  --volume "${MISSING_VOLUME}:/var/lib/postgresql/data" \
  --volume "${RESULTS}/controls/missing-wal-archive:/archive:ro" \
  "${IMAGE}" >/dev/null
sleep 2
set +e
docker exec "${MISSING_RECOVERY}" pg_isready --username fixture \
  --dbname recovery >/dev/null 2>&1
missing_wal_ready=$?
set -e
docker logs "${MISSING_RECOVERY}" \
  >"${RESULTS}/controls/missing-wal.log" 2>&1
[[ ${missing_wal_ready} -ne 0 ]] || {
  echo "recovery unexpectedly reached target without required WAL" >&2
  exit 1
}

docker run --rm --user root --volume "${BASE_VOLUME}:/from:ro" \
  --volume "${RECOVERY_VOLUME}:/to" "${IMAGE}" sh -c \
  'cp -a /from/. /to/ && touch /to/recovery.signal && printf "%s\n" "restore_command = '\''cp /archive/%f %p'\''" "recovery_target_name = '\''before_destructive_delete'\''" "recovery_target_action = '\''promote'\''" >> /to/postgresql.auto.conf'

docker run --detach --name "${RECOVERY}" --network "${NETWORK}" \
  --hostname recovery --cpus=1 --memory=512m \
  --env PGDATA=/var/lib/postgresql/data \
  --volume "${RECOVERY_VOLUME}:/var/lib/postgresql/data" \
  --volume "${RESULTS}/wal:/archive:ro" "${IMAGE}" >/dev/null
wait_ready "${RECOVERY}"

recovery_role="$(psql_command "${RECOVERY}" 'select pg_is_in_recovery()')"
psql_file "${RECOVERY}" "${ROOT}/probe.sql" \
  >"${RESULTS}/recovery-probe.txt"
recovered_known="$(psql_command "${RECOVERY}" \
  "select count(*) from shipments where external_id='shipment-100'")"
excluded_later="$(psql_command "${RECOVERY}" \
  "select count(*) from shipments where external_id='shipment-102'")"
recovery_timeline="$(psql_command "${RECOVERY}" \
  "select timeline_id from pg_control_checkpoint()")"
wal_files="$(find "${RESULTS}/wal" -type f | wc -l | tr -d ' ')"

docker run --rm --volume "${BASE_VOLUME}:/backup:ro" "${IMAGE}" \
  sha256sum /backup/backup_manifest >"${RESULTS}/base-manifest.sha256"
docker run --rm --volume "${BASE_VOLUME}:/backup:ro" "${IMAGE}" \
  cat /backup/backup_manifest >"${RESULTS}/backup_manifest"

jq -n \
  --arg image "${IMAGE}" \
  --arg restore_point_lsn "${restore_point_lsn}" \
  --arg delete_lsn "${delete_lsn}" \
  --arg replay_lsn "${replay_lsn}" \
  --argjson standby_probe_exit "${standby_probe_exit}" \
  --argjson promoted_probe_exit "${promoted_probe_exit}" \
  --arg primary_health "${primary_health}" \
  --arg standby_health "${standby_health}" \
  --arg standby_row_count "${standby_row_count}" \
  --arg standby_known_count "${standby_known_count}" \
  --arg promoted_role "${promoted_role}" \
  --arg recovery_role "${recovery_role}" \
  --arg recovered_known "${recovered_known}" \
  --arg excluded_later "${excluded_later}" \
  --arg recovery_timeline "${recovery_timeline}" \
  --argjson wal_files "${wal_files}" \
  --argjson end_probe_exit "${end_probe_exit}" \
  --argjson missing_wal_ready "${missing_wal_ready}" \
  --arg target_wal_file "${target_wal_file}" \
  '{image: $image, restore_point_lsn: $restore_point_lsn,
    destructive_transaction_lsn: $delete_lsn, standby_replay_lsn: $replay_lsn,
    availability: {primary: $primary_health, standby: $standby_health},
    controls: {standby_probe_exit: $standby_probe_exit,
      promoted_probe_exit: $promoted_probe_exit,
      promoted_is_in_recovery: $promoted_role,
      row_count: ($standby_row_count | tonumber),
      known_shipment_count: ($standby_known_count | tonumber),
      end_of_wal_probe_exit: $end_probe_exit,
      missing_wal_ready_exit: $missing_wal_ready,
      removed_target_wal_file: $target_wal_file},
    recovery: {is_in_recovery: $recovery_role,
      required_shipment_count: ($recovered_known | tonumber),
      post_target_shipment_count: ($excluded_later | tonumber),
      observed_booking_loss: 1, timeline: ($recovery_timeline | tonumber)},
    archived_wal_files: $wal_files}' >"${RESULTS}/summary.json"

jq -e '
  .availability.primary == "1" and
  .availability.standby == "1" and
  .controls.standby_probe_exit != 0 and
  .controls.promoted_probe_exit != 0 and
  .controls.promoted_is_in_recovery == "f" and
  .controls.row_count == 2 and
  .controls.known_shipment_count == 0 and
  .controls.end_of_wal_probe_exit != 0 and
  .controls.missing_wal_ready_exit != 0 and
  .recovery.is_in_recovery == "f" and
  .recovery.required_shipment_count == 1 and
  .recovery.post_target_shipment_count == 0 and
  .recovery.observed_booking_loss == 1 and
  .archived_wal_files > 0
' "${RESULTS}/summary.json" >/dev/null

node "${ROOT}/source-evidence.mjs"
node "${ROOT}/source-manifest.mjs"
node "${ROOT}/verify-results.mjs"

chmod -R u=rwX,go=rX "${RESULTS}"
printf 'OPS-04 replication and PITR fixture passed\n'