#!/usr/bin/env bash
# apollo-update: installs Apollo release files (tools/release/build-release.sh) on a robot.
# installed root-owned at /usr/local/sbin/apollo-update; the apollo server may run it through sudo.
#
#   apollo-update verify <file>       check signature + checksums, print the release as JSON
#   apollo-update install <file>      verify, then install in the background (unit apollo-update)
#   apollo-update rollback [version]  switch to a version kept on this robot (the previous one, or "baseline"
#                                     for the original install; default: previous), in the background
#   apollo-update status              JSON: current, previous, installed versions, last result
#   apollo-update log                 JSON: every log line of the most recent install / go-back
#   apollo-update restore-baseline    undo everything: put the original git checkouts back
#   apollo-update recover             at boot (apollo-update-recover.service): undo a switch a power cut interrupted
#   apollo-update setup-boot          install + enable that boot unit (the manifest and every install call it)
#
# layout: every version is unpacked into /opt/apollo/releases/<version>/{cloud,ros}; /opt/apollo/current
# and /opt/apollo/previous point at two of them; /home/vmx/Apollo-Cloud and /home/vmx/ROS2 are links to
# current/cloud and current/ros, so every existing path keeps working. the first install moves the robot's
# own checkouts to releases/baseline (never pruned) — restore-baseline puts them back exactly.
# students' data is in /var/lib/apollo and is never touched, apart from a database backup before each switch.
# a release may carry a system part (system.tar.zst): system-step.sh plus .deb packages, run as root before the
# switch to give the new version what it needs from the operating system — offline, and additive only.
set -Eeuo pipefail
umask 022

APOLLO_USER=vmx
HOME_DIR=/home/$APOLLO_USER
HOME_CLOUD=$HOME_DIR/Apollo-Cloud
HOME_ROS=$HOME_DIR/ROS2
OPT=/opt/apollo
RELEASES=$OPT/releases
CURRENT=$OPT/current
PREVIOUS=$OPT/previous
DATA=/var/lib/apollo
UPDATES=$DATA/updates
STATUS=$UPDATES/status.json
PUBKEY=/etc/apollo/update.pub
SERVER_ENV=/etc/apollo/server.env
UNIT=apollo-update
LOG=/var/log/apollo-update.log
HEALTH_SECONDS=90
# root-only note that exists only while versions are being switched; at boot, `recover` undoes a switch a power cut interrupted
RECOVERY=$OPT/recovery.json
BOOT_UNIT=apollo-update-recover
# raise this whenever this file changes: an install only replaces the installer with one at least this new,
# so installing an older release never takes features (like the power-cut recovery) away again
HELPER_VERSION=5

log() { printf '%s %s\n' "$(date -Is)" "$*" >> "$LOG"; printf '%s\n' "$*" >&2; }
die() { log "ERROR: $*"; exit 1; }

# status.json is what the Updates page shows while an install runs
write_status() { # state step message [version]
  python3 - "$STATUS" "$@" <<'PY'
import json, os, sys, datetime
path, state, step, message = sys.argv[1:5]
try:
    old = json.load(open(path))
except Exception:
    old = {}
# a version given (even an empty one, e.g. going back to the original install) replaces the old one
version = sys.argv[5] if len(sys.argv) > 5 else old.get("version", "")
now = datetime.datetime.now().astimezone().isoformat(timespec="seconds")
data = {"state": state, "step": step, "message": message,
        "version": version,
        "started": now if step == "queued" or not old.get("started") or old.get("finished") else old["started"]}
if state in ("done", "failed", "rolled-back", "idle"):
    data["finished"] = now
tmp = path + ".tmp"
with open(tmp, "w") as f:
    json.dump(data, f)
os.chmod(tmp, 0o644)
os.replace(tmp, path)
PY
  log "[$1] $2: $3"
}

release_field() { # file.json key
  python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); print(d.get(sys.argv[2], ""))' "$1" "$2"
}

# "0.9.4", or "your original install" for the checkouts from before the first update
current_label() {
  local v
  v="$(release_field "$(readlink -f "$CURRENT" 2>/dev/null || echo /nonexistent)/release.json" version 2>/dev/null || true)"
  [[ "$v" == baseline ]] && echo "your original install" || echo "${v:-the previous version}"
}

# only files that were uploaded or copied into the updates folder
checked_path() {
  local real
  real="$(realpath -e -- "$1" 2>/dev/null)" || die "no such file: $1"
  [[ "$real" == "$UPDATES"/* && "$real" == *.apollo && -f "$real" ]] || die "refused: $1 is not an .apollo file in $UPDATES"
  printf '%s\n' "$real"
}

# verify <file> <dir>: leaves the checked release.json in <dir>
verify_into() {
  local file="$1" dir="$2" members expected name want got size
  [[ -r "$PUBKEY" ]] || die "no update key at $PUBKEY"
  members="$(tar -tf "$file" 2>/dev/null | LC_ALL=C sort | tr '\n' ' ')" || die "not a release file (unreadable tar)"
  [[ " $members" == *" release.json "* && " $members" == *" release.json.sig "* ]] || die "not a release file (no release.json)"
  tar -xf "$file" -C "$dir" release.json release.json.sig
  openssl pkeyutl -verify -pubin -inkey "$PUBKEY" -rawin -in "$dir/release.json" -sigfile "$dir/release.json.sig" >/dev/null 2>&1 ||
    die "signature check failed: this file was not made by Studica, or it was changed"
  python3 - "$dir/release.json" "$(uname -m)" "$(. /etc/os-release; echo "$VERSION_CODENAME")" \
      "$("$(server_node)" -v 2>/dev/null || true)" <<'PY' || exit 1
import json, re, sys
r = json.load(open(sys.argv[1]))
def fail(msg):
    print(msg, file=sys.stderr); sys.exit(1)
if r.get("format") != 1: fail("unsupported release format")
if not re.fullmatch(r"\d+\.\d+\.\d+([.-][0-9A-Za-z.-]+)?", str(r.get("version", ""))): fail("bad version")
if r.get("arch") != sys.argv[2]: fail(f"built for {r.get('arch')}, this robot is {sys.argv[2]}")
if r.get("os") != sys.argv[3]: fail(f"built for Ubuntu {r.get('os')}, this robot runs {sys.argv[3]}")
# the server ships as bytecode that only runs on the Node version it was made with
if r.get("node") and r["node"] != sys.argv[4]: fail(f"built for Node {r['node']}, this robot has {sys.argv[4] or 'no Node'}")
PY
  # exactly the files release.json describes (signed), nothing more, nothing less
  expected="$(python3 -c 'import json,sys; f=json.load(open(sys.argv[1])).get("files",{}); print(" ".join(sorted(["release.json","release.json.sig",*f])))' "$dir/release.json") "
  [[ "$members" == "$expected" ]] || die "not a release file (its contents don't match its description)"
  [[ " $members" == *" cloud.tar.zst "* && " $members" == *" ros.tar.zst "* ]] || die "not a release file (Apollo or ROS part missing)"
  for name in $(python3 -c 'import json,sys; print(" ".join(json.load(open(sys.argv[1]))["files"]))' "$dir/release.json"); do
    want="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["files"][sys.argv[2]]["sha256"])' "$dir/release.json" "$name")"
    got="$(tar -xOf "$file" "$name" | sha256sum | cut -d' ' -f1)"
    [[ "$want" == "$got" ]] || die "checksum mismatch in $name: the file is damaged"
  done
}

cmd_verify() {
  local file dir
  file="$(checked_path "$1")"
  dir="$(mktemp -d)"; trap 'rm -rf "$dir"' RETURN
  verify_into "$file" "$dir"
  python3 -c 'import json,sys; r=json.load(open(sys.argv[1])); r["ok"]=True; r["file"]=sys.argv[2]; print(json.dumps(r))' "$dir/release.json" "$file"
}

set_link() { # target link — replaced in one rename so the link is never missing
  ln -sfn "$1" "$2.new"
  mv -Tf "$2.new" "$2"
}

stop_services() {
  systemctl stop apollo-project.service 2>/dev/null || true
  systemctl stop apollo-server.service controller-helper.service 2>/dev/null || true
}
start_services() {
  systemctl start controller-helper.service 2>/dev/null || true
  systemctl start apollo-server.service
}
healthy() {
  local port i restarts_before
  port="$(sed -nE 's/^PORT=([0-9]+).*/\1/p' "$SERVER_ENV" 2>/dev/null | tail -n1)"
  restarts_before="$(systemctl show -p NRestarts --value apollo-server.service 2>/dev/null || echo 0)"
  for ((i = 0; i < HEALTH_SECONDS; i += 3)); do
    curl -fsS "http://127.0.0.1:${port:-80}/health" 2>/dev/null | grep -q '"status":"ok"' && return 0
    # a server that keeps crashing won't come up; no need to wait out the whole timeout
    (( $(systemctl show -p NRestarts --value apollo-server.service 2>/dev/null || echo 0) - restarts_before >= 3 )) && return 1
    sleep 3
  done
  return 1
}

# puts a database backup back as the apollo user's file: the backups are root's (cp -a kept that, and the
# server could then no longer write its own database); side files of the failed run are dropped
restore_database() {
  local db="$DATA/db/apollo.db"
  install -m 0644 -o "$APOLLO_USER" -g "$APOLLO_USER" "$1" "$db.restore"
  rm -f "$db-journal" "$db-wal" "$db-shm"
  mv -f "$db.restore" "$db"
  runuser -u "$APOLLO_USER" -- test -w "$db" || log "WARNING: the restored database is not writable by $APOLLO_USER"
}

# what make install + the manifest's secure staging do, from the active ros folder
install_native() {
  local ros="$1" staged=/opt/apollo-hardware/lib
  install -d -m 0755 /usr/local/include/studica_drivers /usr/local/lib/studica_drivers
  cp "$ros"/drivers/*.hpp /usr/local/include/studica_drivers/
  cp "$ros/drivers/libstudica_drivers.so" /usr/local/lib/studica_drivers/
  ldconfig
  rm -rf "$staged.new"
  install -d -m 0755 "$staged.new"
  cp -rL "$ros/install/studica_control/lib/." "$staged.new/"
  chown -R root:root "$staged.new"; chmod -R go-w "$staged.new"
  [[ -e "$staged" ]] && mv "$staged" "$staged.old"
  mv "$staged.new" "$staged"
  rm -rf "$staged.old"
}

# the links data-move.sh leaves in the code folder, recreated in a fresh release
link_data() {
  local cloud="$1" server="$1/Teddy/server"
  [[ -d "$DATA/projects" ]] && ln -sfn "$DATA/projects" "$cloud/Teddy/projects"
  [[ -d "$DATA/logs" ]] && ln -sfn "$DATA/logs" "$server/logs"
  [[ -f "$SERVER_ENV" ]] && ln -sfn "$SERVER_ENV" "$server/.env"
  [[ -e "$DATA/db/apollo.db" ]] && ln -sfn "$DATA/db/apollo.db" "$server/prisma/dev.db"
  local f
  for f in wifi_settings.json client_wifi_settings.json; do
    [[ -e "$DATA/settings/$f" ]] && ln -sfn "$DATA/settings/$f" "$server/$f"
  done
  find "$cloud" -maxdepth 4 -type l -user root -exec chown -h "$APOLLO_USER:$APOLLO_USER" {} + 2>/dev/null || true
}

# first install only: the robot's own checkouts become releases/baseline and the home paths become links
adopt_baseline() {
  [[ -L "$HOME_CLOUD" && -L "$HOME_ROS" ]] && return 0
  [[ -d "$HOME_CLOUD" && ! -L "$HOME_CLOUD" && -d "$HOME_ROS" && ! -L "$HOME_ROS" ]] ||
    die "expected $HOME_CLOUD and $HOME_ROS to be folders (or links made by apollo-update)"
  [[ -e "$RELEASES/baseline" ]] && die "$RELEASES/baseline already exists"
  log "keeping the robot's own checkouts as releases/baseline"
  install -d -m 0755 "$RELEASES/baseline"
  # git is asked as the checkouts' owner: as root (from the update unit) it refuses someone else's repository
  python3 - "$RELEASES/baseline/release.json" \
    "$(runuser -u "$APOLLO_USER" -- git -C "$HOME_CLOUD" rev-parse --short=12 HEAD 2>/dev/null || echo unknown)" \
    "$(runuser -u "$APOLLO_USER" -- git -C "$HOME_ROS" rev-parse --short=12 HEAD 2>/dev/null || echo unknown)" <<'PY'
import json, sys, datetime
json.dump({"format": 1, "version": "baseline", "baseline": True,
           "notes": "the robot's own Apollo-Cloud and ROS2 checkouts from before the first update",
           "built": datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
           "cloud": {"commit": sys.argv[2]}, "ros": {"commit": sys.argv[3]}}, open(sys.argv[1], "w"), indent=2)
PY
  mv "$HOME_CLOUD" "$RELEASES/baseline/cloud"
  ln -s "$CURRENT/cloud" "$HOME_CLOUD"
  mv "$HOME_ROS" "$RELEASES/baseline/ros"
  ln -s "$CURRENT/ros" "$HOME_ROS"
  chown -h "$APOLLO_USER:$APOLLO_USER" "$HOME_CLOUD" "$HOME_ROS"
  set_link "$RELEASES/baseline" "$CURRENT"
}

node_bin() { ls -d /opt/node-v*/bin 2>/dev/null | head -n1; }
# the node the server is started with (the manifest writes its path into the start script)
server_node() {
  local n
  n="$(sed -nE 's#^exec ([^ ]+) .*dist/app\.js.*#\1#p' /usr/local/libexec/apollo-server-start 2>/dev/null | head -n1)"
  [[ -x "$n" ]] && echo "$n" || echo "$(node_bin)/node"
}
migrate_database() {
  runuser -u "$APOLLO_USER" -- env HOME="$HOME_DIR" PATH="$(node_bin):/usr/bin:/bin" PRISMA_HIDE_UPDATE_MESSAGE=1 \
    bash -c 'cd "$1/Teddy/server" && npx --no-install prisma migrate deploy' bash "$CURRENT/cloud"
}

switch_to() { # release dir
  local target="$1" now
  now="$(readlink -f "$CURRENT")"
  [[ "$now" == "$target" ]] && return 0
  set_link "$now" "$PREVIOUS"
  set_link "$target" "$CURRENT"
}

# note_switch kind target back previous db_backup version label: written (and flushed) right before a switch
note_switch() {
  python3 - "$RECOVERY" "$@" <<'PY'
import json, os, sys
path, kind, target, back, previous, db, version, label = sys.argv[1:]
with open(path + ".tmp", "w") as f:
    json.dump({"kind": kind, "target": target, "back": back, "previous": previous,
               "db_backup": db, "version": version, "label": label}, f)
os.replace(path + ".tmp", path)
PY
  sync
}
clear_switch_note() { rm -f "$RECOVERY"; sync; }

# the boot unit that runs `recover` before Apollo starts; ordering only, so Apollo still starts if it fails
setup_boot_unit() {
  local unit="/etc/systemd/system/$BOOT_UNIT.service" want
  want="[Unit]
Description=Apollo: undo an update that a power cut interrupted
After=local-fs.target
Before=apollo-server.service controller-helper.service

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/apollo-update recover
TimeoutStartSec=300

[Install]
WantedBy=multi-user.target"
  if [[ "$(cat "$unit" 2>/dev/null)" != "$want" ]]; then
    printf '%s\n' "$want" > "$unit"
    systemctl daemon-reload
  fi
  systemctl is-enabled --quiet "$BOOT_UNIT.service" 2>/dev/null || systemctl enable --quiet "$BOOT_UNIT.service"
}

prune() {
  local keep_current keep_previous dir
  keep_current="$(readlink -f "$CURRENT")"; keep_previous="$(readlink -f "$PREVIOUS" 2>/dev/null || true)"
  for dir in "$RELEASES"/*; do
    [[ -d "$dir" ]] || continue
    case "$dir" in "$keep_current"|"$keep_previous"|"$RELEASES/baseline") continue ;; esac
    log "removing old version $(basename "$dir")"
    rm -rf "$dir"
  done
}

run_install() {
  local file="$1" tmp version dir from db_backup="" switched=false prev_before
  [[ -n "${INVOCATION_ID:-}" ]] || die "run-install only runs from the $UNIT unit (use: apollo-update install)"
  tmp="$(mktemp -d)"
  setup_boot_unit || log "WARNING: could not set up $BOOT_UNIT.service (power-cut recovery)"
  write_status installing verify "Checking the update file"
  if ! (verify_into "$file" "$tmp") 2>>"$LOG"; then
    write_status failed verify "$(tail -n1 "$LOG" | sed 's/^[^ ]* ERROR: //')"
    exit 1
  fi
  version="$(release_field "$tmp/release.json" version)"
  dir="$RELEASES/$version"
  [[ "$(readlink -f "$CURRENT" 2>/dev/null)" == "$dir" ]] && { write_status failed verify "Version $version is already installed" "$version"; exit 1; }
  prev_before="$(readlink -f "$PREVIOUS" 2>/dev/null || true)"

  # anything failing after the switch goes back to the version that was running
  on_error() {
    local why="$1"
    if [[ "$switched" == true ]]; then
      {
        echo "--- apollo-server output (last 40 lines) ---"
        journalctl -u apollo-server.service -n 40 --no-pager -o cat 2>/dev/null || true
        echo "--- end of apollo-server output ---"
      } >> "$LOG"
      write_status rolling-back restore "$why — going back to ${from:-the previous version}" "$version"
      # nothing may be writing the database while it's put back (the failed version may still be restarting)
      stop_services
      set_link "$(readlink -f "$PREVIOUS")" "$CURRENT" || true
      # previous goes back to what it was, and the version that failed is removed
      [[ -n "$prev_before" ]] && set_link "$prev_before" "$PREVIOUS" || true
      [[ "$(readlink -f "$CURRENT")" != "$dir" ]] && rm -rf "$dir" || true
      if [[ -n "$db_backup" && -f "$db_backup" ]]; then
        restore_database "$db_backup" || log "WARNING: could not put back the database backup $db_backup"
      fi
      install_native "$CURRENT/ros" >>"$LOG" 2>&1 || true
      start_services || true
      if healthy; then write_status rolled-back done "Update to $version failed ($why); Apollo is back on ${from:-the previous version}" "$version"
      else write_status failed done "Update to $version failed ($why) and the previous version did not come back — run: sudo apollo-update restore-baseline" "$version"; fi
      clear_switch_note
    else
      clear_switch_note
      rm -rf "$dir.partial"
      [[ "$(readlink -f "$CURRENT" 2>/dev/null)" != "$dir" ]] && rm -rf "$dir" || true
      start_services || true
      write_status failed install "$why" "$version"
    fi
    exit 1
  }
  trap 'on_error "step failed (see /var/log/apollo-update.log)"' ERR

  write_status installing prepare "Preparing version $version" "$version"
  install -d -m 0755 "$RELEASES"
  # a release unpacks to about 4-5x its download; stop before filling the disk
  local packed need avail
  packed="$(python3 -c 'import json,sys; print(sum(f["size"] for f in json.load(open(sys.argv[1]))["files"].values()))' "$tmp/release.json")"
  need=$(( packed * 6 + 300 * 1024 * 1024 ))
  avail="$(df --output=avail -B1 "$OPT" 2>/dev/null | tail -n1 || df --output=avail -B1 / | tail -n1)"
  if (( avail < need )); then
    write_status failed prepare "Not enough free space: version $version needs about $(( need / 1048576 )) MB, $(( avail / 1048576 )) MB free" "$version"
    exit 1
  fi
  adopt_baseline
  from="$(current_label)"
  write_status installing unpack "Unpacking version $version" "$version"
  rm -rf "$dir.partial"
  install -d -m 0755 "$dir.partial/cloud" "$dir.partial/ros"
  tar -xOf "$file" cloud.tar.zst | zstd -dcq | tar --no-same-owner -C "$dir.partial/cloud" -xpf -
  tar -xOf "$file" ros.tar.zst | zstd -dcq | tar --no-same-owner -C "$dir.partial/ros" -xpf -
  if tar -tf "$file" system.tar.zst >/dev/null 2>&1; then
    install -d -m 0755 "$dir.partial/system"
    tar -xOf "$file" system.tar.zst | zstd -dcq | tar --no-same-owner -C "$dir.partial/system" -xpf -
  fi
  cp "$tmp/release.json" "$dir.partial/release.json"
  chown -R "$APOLLO_USER:$APOLLO_USER" "$dir.partial/cloud" "$dir.partial/ros"
  link_data "$dir.partial/cloud"
  [[ -e "$dir" ]] && rm -rf "$dir"
  mv "$dir.partial" "$dir"

  if [[ -f "$dir/system/system-step.sh" ]]; then
    write_status installing system "Installing what version $version needs from the system" "$version"
    echo "--- system step ---" >> "$LOG"
    RELEASE_DIR="$dir" bash "$dir/system/system-step.sh" >>"$LOG" 2>&1 ||
      on_error "the system step of version $version failed (see Details)"
    echo "--- end of system step ---" >> "$LOG"
  fi

  write_status installing stop "Stopping Apollo" "$version"
  stop_services
  if [[ -f "$DATA/db/apollo.db" ]]; then
    install -d -m 0700 "$DATA/backups"
    db_backup="$DATA/backups/db-before-$version-$(date -u +%Y%m%dT%H%M%SZ).db"
    sqlite3 "$DATA/db/apollo.db" ".backup '$db_backup'"
  fi
  write_status installing switch "Switching to version $version" "$version"
  note_switch install "$dir" "$(readlink -f "$CURRENT")" "$prev_before" "$db_backup" "$version" "$from"
  switch_to "$dir"; switched=true
  write_status installing migrate "Updating the project database" "$version"
  migrate_database >>"$LOG" 2>&1
  write_status installing hardware "Installing the hardware drivers" "$version"
  install_native "$CURRENT/ros" >>"$LOG" 2>&1
  write_status installing start "Starting Apollo $version" "$version"
  start_services
  healthy || on_error "Apollo $version did not start"
  trap - ERR
  clear_switch_note
  prune
  rm -rf "$tmp"
  # installed: the file isn't needed any more (the version stays as current/previous for going back)
  rm -f "$file"
  local shipped="$CURRENT/cloud/Teddy/server/scripts/apollo-update" shipped_version
  # installers from before HELPER_VERSION count as 0 and are never put back
  shipped_version="$(sed -nE 's/^HELPER_VERSION=([0-9]+).*/\1/p' "$shipped" 2>/dev/null | head -n1)"
  if [[ -f "$shipped" ]] && (( ${shipped_version:-0} >= HELPER_VERSION )) && ! cmp -s "$shipped" /usr/local/sbin/apollo-update; then
    install -m 0755 -o root -g root "$shipped" /usr/local/sbin/apollo-update && log "updated the installer itself from version $version"
  fi
  write_status done done "Updated to $version" "$version"
}

# switching to a version that is still unpacked here (previous or the original install): no file needed; the
# version left becomes previous. one that doesn't start is switched away from again, like a failed install
run_rollback() { # [version]
  [[ -n "${INVOCATION_ID:-}" ]] || die "run-rollback only runs from the $UNIT unit (use: apollo-update rollback)"
  local back ver to leaving prev_before from
  [[ -z "${1:-}" || "$1" == baseline || "$1" =~ ^[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$ ]] || die "not a version: $1"
  if [[ -n "${1:-}" ]]; then back="$RELEASES/$1"; else back="$(readlink -f "$PREVIOUS" 2>/dev/null || true)"; fi
  leaving="$(readlink -f "$CURRENT" 2>/dev/null || true)"
  prev_before="$(readlink -f "$PREVIOUS" 2>/dev/null || true)"
  [[ -n "$back" && -d "$back" && -f "$back/release.json" ]] || { write_status failed rollback "That version is not kept on this robot any more; install it from its file"; exit 1; }
  [[ "$back" != "$leaving" ]] || { write_status failed rollback "That version is already running"; exit 1; }
  ver="$(release_field "$back/release.json" version)"
  to="$ver"
  [[ "$ver" == baseline ]] && { to="your original install"; ver=""; }
  from="$(current_label)"
  setup_boot_unit || log "WARNING: could not set up $BOOT_UNIT.service (power-cut recovery)"
  write_status rolling-back stop "Switching to $to" "$ver"
  stop_services
  note_switch rollback "$back" "$leaving" "$prev_before" "" "$ver" "$to"
  switch_to "$back"
  install_native "$CURRENT/ros" >>"$LOG" 2>&1 || true
  start_services
  if healthy; then
    prune
    write_status done done "Switched to $to (projects kept as they are now)" "$ver"
  else
    {
      echo "--- apollo-server output (last 40 lines) ---"
      journalctl -u apollo-server.service -n 40 --no-pager -o cat 2>/dev/null || true
      echo "--- end of apollo-server output ---"
    } >> "$LOG"
    write_status rolling-back restore "${to^} did not start — going back to $from" "$ver"
    stop_services
    # a power cut from here on finishes going back to the version that was running
    note_switch rollback "$leaving" "$back" "$prev_before" "" "$ver" "$from"
    set_link "$leaving" "$CURRENT"
    if [[ -n "$prev_before" ]]; then set_link "$prev_before" "$PREVIOUS"; else rm -f "$PREVIOUS"; fi
    install_native "$CURRENT/ros" >>"$LOG" 2>&1 || true
    start_services || true
    if healthy; then write_status rolled-back done "${to^} did not start; Apollo is back on $from" "$ver"
    else write_status failed done "${to^} did not start and $from did not come back — run: sudo apollo-update restore-baseline" "$ver"; fi
  fi
  clear_switch_note
}

# at boot, before Apollo starts (apollo-update-recover.service): an install or go-back that a power cut stopped is
# finished cleanly. stopped after the switch (the note is there): the same undo as a failed install. stopped
# before it: nothing changed, the robot is already on the version it had
cmd_recover() {
  # at boot nothing is running; by hand it must never undo a switch the running installer is in the middle of
  systemctl is-active --quiet "$UNIT" && die "an update is running; recover only runs when none is"
  local state
  state="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("state",""))' "$STATUS" 2>/dev/null || true)"
  if [[ "$state" == installing || "$state" == rolling-back ]]; then
    # a package install that was cut short leaves dpkg half done; finish it so apt keeps working
    DEBIAN_FRONTEND=noninteractive dpkg --configure -a --force-confdef --force-confold >>"$LOG" 2>&1 ||
      log "WARNING: dpkg --configure -a failed"
    rm -rf "$RELEASES"/*.partial
  fi
  if [[ ! -f "$RECOVERY" ]]; then
    if [[ "$state" == installing || "$state" == rolling-back ]]; then
      write_status failed done "The last update was interrupted before Apollo switched versions; it is still on $(current_label). You can try the update again."
    fi
    return 0
  fi

  local kind target back previous db version label
  { read -r kind; read -r target; read -r back; read -r previous; read -r db; read -r version; read -r label; } < <(
    python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); [print(str(d.get(k,"")).replace("\n"," ")) for k in ("kind","target","back","previous","db_backup","version","label")]' "$RECOVERY") || true
  # the note is root's, but only ever act on release folders and database backups
  local p
  for p in "$target" "$back" "$previous"; do
    [[ -z "$p" || ( "$p" == "$RELEASES"/* && -d "$p" ) ]] || { log "ERROR: recovery note points outside $RELEASES ($p); ignored"; clear_switch_note; return 0; }
  done
  [[ -z "$db" || "$db" == "$DATA/backups/"* ]] || db=""

  # at boot Apollo isn't up yet: its start waits for this unit, and stopping it here would cancel that start.
  # only a hand-run `recover` finds it running, and then it's stopped and started again around the undo
  local was_running=false
  if systemctl is-active --quiet apollo-server.service; then was_running=true; stop_services; fi
  if [[ "$kind" == install && -n "$back" ]]; then
    log "an update to $version was interrupted after switching versions; going back to ${label:-the previous version}"
    set_link "$back" "$CURRENT"
    [[ -n "$previous" ]] && set_link "$previous" "$PREVIOUS"
    [[ "$target" != "$RELEASES/baseline" && "$(readlink -f "$CURRENT")" != "$target" &&
       "$(readlink -f "$PREVIOUS" 2>/dev/null)" != "$target" ]] && rm -rf "$target"
    if [[ -n "$db" && -f "$db" ]]; then
      restore_database "$db" || log "WARNING: could not put back the database backup $db"
    fi
    install_native "$CURRENT/ros" >>"$LOG" 2>&1 || log "WARNING: could not reinstall the hardware drivers"
    write_status rolled-back done "The update to $version was interrupted (the robot lost power); Apollo went back to ${label:-the previous version}" "$version"
  elif [[ "$kind" == rollback && -n "$target" ]]; then
    log "going back to ${label:-the previous version} was interrupted; finishing it"
    [[ "$(readlink -f "$CURRENT")" == "$target" ]] || switch_to "$target"
    install_native "$CURRENT/ros" >>"$LOG" 2>&1 || log "WARNING: could not reinstall the hardware drivers"
    write_status done done "Going back was interrupted (the robot lost power) and has been finished: Apollo is on ${label:-the previous version}" "$version"
  fi
  clear_switch_note
  [[ "$was_running" == true ]] && start_services
  # at boot apollo-server starts after this unit as usual
  return 0
}

cmd_restore_baseline() {
  [[ -d "$RELEASES/baseline/cloud" && -d "$RELEASES/baseline/ros" ]] || die "no baseline to restore (nothing was ever installed)"
  [[ -L "$HOME_CLOUD" && -L "$HOME_ROS" ]] || die "$HOME_CLOUD / $HOME_ROS are not apollo-update links; not touching them"
  log "restoring the original checkouts"
  stop_services
  rm "$HOME_CLOUD" "$HOME_ROS"
  mv "$RELEASES/baseline/cloud" "$HOME_CLOUD"
  mv "$RELEASES/baseline/ros" "$HOME_ROS"
  rm -f "$CURRENT" "$PREVIOUS" "$RELEASES/baseline/release.json"
  rmdir "$RELEASES/baseline"
  install_native "$HOME_ROS" >>"$LOG" 2>&1
  start_services
  healthy && write_status idle done "Restored the original Apollo checkouts" "" || die "restored, but Apollo did not answer /health"
  log "restored; other versions are still in $RELEASES (remove them by hand if not needed)"
}

# every log line of the most recent run (from its "queued" line), as JSON for the Updates page
cmd_log() {
  python3 - "$LOG" <<'PY'
import json, sys
try:
    lines = open(sys.argv[1], errors="replace").read().splitlines()
except Exception:
    lines = []
start = max((i for i, line in enumerate(lines) if "] queued: " in line), default=max(len(lines) - 60, 0))
print(json.dumps({"lines": lines[start:][-500:]}))
PY
}

cmd_status() {
  python3 - "$CURRENT" "$PREVIOUS" "$RELEASES" "$STATUS" "$(systemctl is-active "$UNIT" 2>/dev/null || true)" \
    "$(git -C "$HOME_CLOUD" rev-parse --short=12 HEAD 2>/dev/null || true)" <<'PY'
import json, os, sys
current, previous, releases, status, unit, git = sys.argv[1:]
def info(link):
    try:
        d = json.load(open(os.path.join(os.path.realpath(link), "release.json")))
        return {k: d.get(k) for k in ("version", "built", "notes", "cloud", "ros", "baseline")}
    except Exception:
        return None
try:
    last = json.load(open(status))
except Exception:
    last = None
running = unit in ("active", "activating")
if last and last.get("state") in ("installing", "rolling-back") and not running:
    # the background job is gone (power cut, crash) without writing a result
    last = {**last, "state": "failed", "message": "The last update was interrupted before it finished. "
            "Apollo is running the version shown below; you can try the update again."}
cur = info(current)
if cur is None:
    # never updated: the robot runs its own git checkout
    cur = {"version": "development", "notes": "not installed from a release file", "cloud": {"commit": git or "unknown"}}
installed = sorted(n for n in os.listdir(releases) if os.path.isdir(os.path.join(releases, n)) and not n.endswith(".partial")) if os.path.isdir(releases) else []
# the original install stays until restore-baseline; the page always lists it
print(json.dumps({"installer": True, "current": cur, "previous": info(previous), "installed": installed,
                  "original": info(os.path.join(releases, "baseline")), "updating": running, "last": last}))
PY
}

launch() { # run-install <file> | run-rollback
  systemctl is-active --quiet "$UNIT" && die "an update is already running"
  systemctl reset-failed "$UNIT" 2>/dev/null || true
  systemd-run --unit="$UNIT" --collect --quiet --property=Type=exec /usr/local/sbin/apollo-update "$@"
}

[[ $EUID -eq 0 ]] || { echo "apollo-update must run as root (sudo)" >&2; exit 1; }
touch "$LOG"; chmod 0640 "$LOG"
install -d -m 0755 -o "$APOLLO_USER" -g "$APOLLO_USER" "$UPDATES"
case "${1:-}" in
  verify)  [[ $# -eq 2 ]] || die "usage: apollo-update verify <file>"; cmd_verify "$2" ;;
  install) [[ $# -eq 2 ]] || die "usage: apollo-update install <file>"; file="$(checked_path "$2")"
           systemctl is-active --quiet "$UNIT" && die "an update is already running"
           write_status installing queued "Starting the update" ""
           launch run-install "$file" || { write_status failed queued "Could not start the update (see /var/log/apollo-update.log)"; exit 1; }
           echo '{"started":true}' ;;
  rollback) [[ $# -le 2 ]] || die "usage: apollo-update rollback [version|baseline]"
           [[ -z "${2:-}" || "$2" == baseline || "$2" =~ ^[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$ ]] || die "not a version: $2"
           systemctl is-active --quiet "$UNIT" && die "an update is already running"
           write_status rolling-back queued "Switching versions" ""
           launch run-rollback ${2:+"$2"} || { write_status failed queued "Could not start switching versions (see /var/log/apollo-update.log)"; exit 1; }
           echo '{"started":true}' ;;
  status)  cmd_status ;;
  log)     cmd_log ;;
  restore-baseline) cmd_restore_baseline ;;
  run-install) run_install "$(checked_path "$2")" ;;
  run-rollback) run_rollback "${2:-}" ;;
  recover) cmd_recover ;;
  setup-boot) setup_boot_unit ;;
  *) echo "usage: apollo-update verify|install <file> | rollback | status | restore-baseline | recover | setup-boot" >&2; exit 64 ;;
esac
