#!/usr/bin/env bash # ╭────────────────────────────────────────────────────╮ # │ _ _ _ _ owo │ # │ | |__ ___| |_ __ ___ _ __| |__ ___ | |_ │ # │ | '_ \ / _ \ | '_ \ / _ \ '__| '_ \ / _ \| __| │ # │ | | | | __/ | |_) | __/ | | |_) | (_) | |_ │ # │ |_| |_|\___|_| .__/ \___|_| |_.__/ \___/ \__| │ # │ |_| │ # ╰────────────────────────────────────────────────────╯ # helperbot - synth.download's all in one script for managing everything. beep! # ============================================================================= # exit immediately if an error occurs somewhere to prevent Fucked Up Shit set -e # ╭──────────────────────────────────────╮ # │ functions and variables - start here │ # ╰──────────────────────────────────────╯ # beware of function spam following this point :3 # unset everything - ensure we're working with a clean state all_known_variables=("synth_current_system" "synth_args_exist") for variable in "${all_known_variables[@]}"; do unset $variable done # set variable to determine if we actually received any arguments if [ -n "$1" ]; then synth_args_exist=1 fi # attempt to detect the system based on hostname function detect_system { local valid_systems=("phosphorus" "neptunium" "cerium" "synthnix") local current_hostname=$(hostname) for variable in "${all_known_variables[@]}"; do if [ "$current_hostname" = "$system" ]; then synth_current_system=$system echo "Detected ${blue}${system}${normal}." return 0 fi done # report if no valid system was found echo "${red}Failed to detect system.${normal}" echo "We're most likely being run in an environment we don't know of." echo "Exiting..." exit 1 } # defining text formatting for text output if [[ -t 1 ]]; then red=$(tput setaf 1); green=$(tput setaf 2); yellow=$(tput setaf 3); blue=$(tput setaf 4); pink=$(tput setaf 5); cyan=$(tput setaf 6); gray=$(tput setaf 8); bold=$(tput bold) underline=$(tput smul) normal=$( tput sgr 0); fi # ============================================================================= # ╭───────────────────╮ # │ defining messages │ # ╰───────────────────╯ # yes i know spamming echo is a bad way of doing this but other actually efficient requires me to remove indenting which makes it harder to read this code for me 💔 # header function header { echo "╭────────────────╮" echo "│ helperbot! owo │" echo "╰────────────────╯" echo sleep 1 # grace period } # help info function info_help { echo "${pink}${bold}${underline}Usage:${normal} ${bold}helperbot${normal} [-h|-u|-b|-v] [--update-email-certs|--sync-blocklists|--update-frontends]" echo echo "${blue}${bold}${underline}Options:${normal}" echo "${bold}-h${normal}, ${bold}--help${normal}" echo " Show this help page." echo echo "${green}${bold}${underline}System maintenance:${normal}" echo "${bold}-u${normal}, ${bold}--upgrade${normal}" echo " Perform a full system upgrade, including containers and services." echo echo "${bold}-b${normal}, ${bold}--backup${normal}" echo " Perform a backup of all known services." echo echo "${bold}-v${normal}, ${bold}--vacuum${normal}" echo " Vacuum the postgresql databases." echo echo "${bold}--update-email-certs${normal}" echo " Pull the email/XMPP certificates from Caddy into ${underline}/etc/certs${normal}." echo echo "${cyan}${bold}${underline}Fediverse:${normal}" echo "${bold}--sync-blocklists${normal}" echo " Import blocklist from Sharkey -> Iceshrimp" echo echo "${bold}--update-frontends${normal}" echo " Update standalone fediverse frontends." echo echo "helperbot automatically knows what to do for some actions based on this system's hostname. Beep!" echo echo "${yellow}${bold}This script is still generally a work-in-progress.${normal}" echo "Report breakage or suggestions or improvments or whatever to here:" echo "${blue}https://forged.synth.download/synth.download/synth.download${normal}" echo } # invalid command message function invalid_command { echo "${red}Error:${normal} Invalid option \""$1"\"." echo "\"helperbot is very confused... >~<\"" echo echo "Run with --help to see all options." } # attempt to detect the system based on hostname function detect_system { local valid_systems=("phosphorus" "neptunium" "cerium" "synthnix") local current_hostname=$(hostname) for system in "${valid_systems[@]}"; do if [ "$current_hostname" = "$system" ]; then synth_current_system=$system echo "Detected ${blue}${system}${normal}." return 0 fi done # report if no valid system was found echo "${red}Failed to detect system.${normal}" echo "We're most likely being run in an environment we don't know of." echo "Exiting..." exit 1 } # root check function root_check { if [[ ${UID} != 0 ]]; then echo "${red}helperbot must be run as root or with sudo permissions to perform this action!${normal} Beep!" exit 1 fi } # ╭─────────────────╮ # │ upgrade related │ # ╰─────────────────╯ # base system upgrade - generic steps for debian/ubuntu based systems function base_system_upgrade { echo "${cyan}Upgrading base system.${normal}" echo "${blue}Doing standard apt upgrade...${normal}" apt update apt upgrade -y echo "${blue}Try upgrading distro base...${normal}" apt dist-upgrade -y echo "${blue}Apt cleanup...${normal}" apt clean echo "${green}Base system upgraded!.${normal}" } # docker container updates # reusable steps to update containers - upgrade_docker_container [/srv/docker] [name_of_service_or_folder] [compose.yaml] function upgrade_docker_container { if [ -d "$1/$2" ]; then # pull the container cd "$1"/"$2" && docker compose -f "$1/$2/$3" pull docker compose -f "$1/$2/$3" down && docker compose -f "$1/$2/$3" up -d else echo "${red}docker:${normal} Folder $1/$2 does not exist." fi } # ╭──────────────╮ # │ upgrade step │ # ╰──────────────╯ function system_upgrade { #timestamp=$(date +'%Y%m%d%H%M%S') #synth_upgrade_log=/tmp/upgrade-output-${timestamp}.txt echo "${blue}upgrade:${normal} Running full system upgrade for ${green}${synth_current_system}${normal}." #echo "Upgrade will be logged into ${yellow}${synth_upgrade_log}${normal} if needed." # logging doesn't work properly - check on later if [ "$synth_current_system" = "phosphorus" ]; then # phosphorus # apt/system related upgrade base_system_upgrade # docker upgrade_docker_container "/srv/docker" "sharkey" "compose.yaml" upgrade_docker_container "/srv/docker" "iceshrimp" "compose.yaml" upgrade_docker_container "/srv/docker" "mastodon" "compose.yaml" upgrade_docker_container "/srv/docker" "pds" "compose.yaml" # done echo "${green}System upgrade finished! beep!~${normal}" elif [ "$synth_current_system" = "neptunium" ]; then # neptunium # apt/system related upgrade base_system_upgrade # docker upgrade_docker_container "/srv/docker" "mailserver" "compose.yaml" upgrade_docker_container "/srv/docker" "ejabberd" "compose.yaml" upgrade_docker_container "/srv/docker" "zitadel" "compose.yaml" upgrade_docker_container "/srv/docker" "forgejo" "compose.yaml" upgrade_docker_container "/srv/docker" "forgejo" "compose-runner.yaml" upgrade_docker_container "/srv/docker" "freshrss" "compose.yaml" upgrade_docker_container "/srv/docker" "vaultwarden" "compose.yaml" upgrade_docker_container "/srv/docker" "ask-js" "compose.yaml" # done echo "${green}System upgrade finished! beep!~${normal}" elif [ "$synth_current_system" = "cerium" ]; then # cerium # apt/system related upgrade base_system_upgrade # docker upgrade_docker_container "/srv/docker" "redlib" "compose.yaml" upgrade_docker_container "/srv/docker" "safetwitch" "compose.yaml" # done echo "${green}System upgrade finished! beep!~${normal}" echo "${red}Rebooting system.${normal}" sleep 1 && systemctl reboot elif [ "$synth_current_system" = "synthnix" ]; then # synthnix # apt/system related upgrade base_system_upgrade # done echo "${green}System upgrade finished! beep!~${normal}" fi } # ╭────────────────╮ # │ backup related │ # ╰────────────────╯ # mostly just symlinks to commands because i think it looks less ugly (and easier to update) # psql vacuuming # reusable step to vacuum databases - postgres_vacuum [postgres-db-1] [user_and_db_name] [password] function postgres_vacuum { # load postgres passwords if [ -f /etc/secrets/postgres.env ]; then export $(grep -v '^#' /etc/secrets/postgres.env | xargs) else echo "${red}postgres_vacuum:${normal} Postgresql Secrets don't exist. Exiting..." exit 1 fi # vacuum docker exec -it "$1" /bin/bash -c "POSTGRES_PASSWORD="$3" psql -U "$2" -d "$2" -c 'VACUUM ANALYZE;'" # unset secrets unset $(grep -v '^#' /etc/secrets/postgres.env | sed -E 's/(.*)=.*/\1/' | xargs) } # postgres_vacuum_self function postgres_vacuum_self { docker exec -it postgres-db-1 /bin/bash -c "psql -U postgres -c 'VACUUM ANALYZE;'" } # psql backup # reusable step to backup databases - postgres_backup [postgres-db-1] [user_and_db_name] [output_name] [$backup_working_directory] function postgres_backup { # for some reason, doing a dump *doesn't* require a password apparently. huh docker exec "$1" /bin/bash -c "pg_dump "$2" --username "$2" > "$3".sql" docker cp "$1":/$3.sql $4/$3/$3.sql docker exec "$1" /bin/bash -c "rm "$3".sql" } # redis snapshot # tells redis to make a snapshot - redis_snapshot [whatever-redis-1] function redis_snapshot { docker exec $1 redis-cli SAVE } # b2 upload # load secrets then start uploading to backblaze b2 - b2_upload [$backup_working_directory] [$backup_output_tar] function b2_upload { # load in secrets from external file if [ -f /etc/secrets/b2.env ]; then export $(grep -v '^#' /etc/secrets/b2.env | xargs) else echo "${red}b2_upload:${normal} B2 Secrets don't exist. Exiting..." exit 1 fi # upload file specified backblaze-b2 authorize-account $B2_KEYID $B2_SECRET backblaze-b2 upload-file $B2_BACKUP_BUCKET ""$1"/"$2".zst" ""$2".zst" backblaze-b2 clear-account # just to ensure we won't stay authenticated afterwards # clear out secrets unset $(grep -v '^#' /etc/secrets/b2.env | sed -E 's/(.*)=.*/\1/' | xargs) } # ╭─────────────╮ # │ backup step │ # ╰─────────────╯ function system_backup { echo "${blue}backup:${normal} Running full system backup for ${green}${synth_current_system}${normal}." if [ "$synth_current_system" = "phosphorus" ]; then # phosphorus # variables - could probably be set locally but unsure how much this will dynamically change between systems backup_local_folder=/srv/docker backup_working_directory=/var/backups/phosphorus backup_output_tar=phosphorus.tar backup_media_output_tar=fedi_media_backups.tar # ============================================================================= # initial steps - cleanup then create rm -fr $backup_working_directory/* mkdir -p $backup_working_directory # ============================================================================= # call in database vacuuming function echo "${blue}Calling in vacuuming...${normal}" system_vacuum # ============================================================================= # backup files - sharkey echo "${blue}Pulling in Sharkey...${normal}" mkdir -p $backup_working_directory/sharkey/.config # database postgres_backup postgres-db-1 misskey sharkey $backup_working_directory # redis redis_snapshot sharkey-redis-1 cp -r $backup_local_folder/sharkey/redis $backup_working_directory/sharkey # configs, extra cp -r $backup_local_folder/sharkey/compose.yaml $backup_working_directory/sharkey cp -r $backup_local_folder/sharkey/.config $backup_working_directory/sharkey # ============================================================================= # iceshrimp echo "${blue}Pulling in Iceshrimp...${normal}" mkdir -p $backup_working_directory/iceshrimp/config # database postgres_backup postgres-db-1 iceshrimp iceshrimp $backup_working_directory # configs, extra cp -r $backup_local_folder/iceshrimp/compose.yaml $backup_working_directory/iceshrimp cp -r $backup_local_folder/iceshrimp/config $backup_working_directory/iceshrimp # ============================================================================= # mastodon echo "${blue}Pulling in Mastodon...${normal}" mkdir -p $backup_working_directory/mastodon/.config # database postgres_backup postgres-db-1 mastodon mastodon $backup_working_directory # redis redis_snapshot mastodon-redis-1 cp -r $backup_local_folder/mastodon/redis $backup_working_directory/mastodon # configs, extra cp -r $backup_local_folder/mastodon/compose.yaml $backup_working_directory/mastodon cp -r $backup_local_folder/mastodon/.config $backup_working_directory/mastodon # ============================================================================= # pds echo "${blue}Pulling in PDS...${normal}" mkdir -p $backup_working_directory/pds # there isn't a native way to "backup" the pds, so we shut it off and copy it docker compose -f $backup_local_folder/pds/compose.yaml down cp -r $backup_local_folder/pds/pds $backup_working_directory/pds docker compose -f $backup_local_folder/pds/compose.yaml up -d # configs, extra cp -r $backup_local_folder/pds/compose.yaml $backup_working_directory/pds # ============================================================================= # pull in any other common configs and secrets echo "${blue}Pulling in other configurations...${normal}" mkdir -p $backup_working_directory/other/etc/caddy mkdir -p $backup_working_directory/other/etc/secrets cp /etc/caddy/Caddyfile $backup_working_directory/other/etc/caddy/Caddyfile cp -r /etc/secrets/* $backup_working_directory/other/etc/secrets/ # ============================================================================= # archive and compress everything echo "${blue}Compressing everything into one archive...${normal}" tar -cf "$backup_working_directory/$backup_output_tar" $backup_working_directory # create the archive zstd -z -T3 -9 --rm "$backup_working_directory/$backup_output_tar" # compress the archive # TODO: it may be possible to combine these steps so tar automatically compresses the archive with zstd instead of doing it separately # ============================================================================= # upload backup to backblaze - secrets used here are fetched from b2.env echo "${blue}Uploading backup...${normal}" b2_upload $backup_working_directory $backup_output_tar # ============================================================================= # cleanup echo "${blue}Cleaning up...${normal}" rm -fr ${backup_working_directory}/${backup_output_tar}.zst $backup_working_directory/* # ============================================================================= # unload secrets - we already unload them for each vacuum/upload step, but we want to ensure they are unset $(grep -v '^#' /etc/secrets/b2.env | sed -E 's/(.*)=.*/\1/' | xargs) unset $(grep -v '^#' /etc/secrets/postgres.env | sed -E 's/(.*)=.*/\1/' | xargs) elif [ "$synth_current_system" = "neptunium" ]; then # neptunium postgres_vacuum_self elif [ "$synth_current_system" = "cerium" ]; then # cerium postgres_vacuum_self elif [ "$synth_current_system" = "synthnix" ]; then # synthnix # as synthnix doesn't really include much and serves as a place for members # we just need to back up the home directory here # # WIP echo "wip" fi echo "${green}System backup finished! beep!~${normal}" } # backup - create folder and copy # step that combines the process of making folders and copying files for backup # backup_create_copy ["source files"] [subpath/to/folder] [$backup_working_directory] function backup_create_copy { mkdir -p $3/$2 cp -r $1 $3/$2 } # ╭─────────────╮ # │ vacuum step │ # ╰─────────────╯ function system_vacuum { echo "${blue}vacuum:${normal} Running database vacuums for ${green}${synth_current_system}${normal}." # external files containing secrets if [ -f /etc/secrets/postgres.env ]; then export $(grep -v '^#' /etc/secrets/postgres.env | xargs) else echo "${red}vacuum:${normal} Secrets don't exist. Exiting..." exit 1 fi # vacuum if [ "$synth_current_system" = "phosphorus" ]; then # phosphorus postgres_vacuum_self postgres_vacuum postgres-db-1 misskey ${SHARKEY_POSTGRES_PASSWORD} postgres_vacuum postgres-db-1 iceshrimp ${ICESHRIMP_POSTGRES_PASSWORD} postgres_vacuum postgres-db-1 mastodon ${MASTODON_POSTGRES_PASSWORD} elif [ "$synth_current_system" = "neptunium" ]; then # neptunium postgres_vacuum_self elif [ "$synth_current_system" = "cerium" ]; then # cerium postgres_vacuum_self elif [ "$synth_current_system" = "synthnix" ]; then # synthnix # as synthnix doesn't really include much and serves as a place for members # we just need to back up the home directory here # # WIP echo "wip" fi # unload secrets - if we pass that they do exist, no need to check if they exist here again unset $(grep -v '^#' /etc/secrets/postgres.env | sed -E 's/(.*)=.*/\1/' | xargs) echo "${green}Vacuuming complete! Beep!~${normal}${normal}" } # ╭────────────────────────────────────╮ # │ functions and variables - end here │ # ╰────────────────────────────────────╯ # ============================================================================= # ╭──────────────╮ # │ main program │ # ╰──────────────╯ # display the header header # evaluate arguments and set environment variables to enable each command and see what should be executed while [ -n "$1" ]; do case "$1" in -h | --help) # display help info info_help exit 0;; -u | --upgrade) # upgrade system root_check if [ ! -v synth_current_system ]; then detect_system fi system_upgrade;; -b | --backup) # backup system root_check if [ ! -v synth_current_system ]; then detect_system fi system_backup;; -v | --vacuum) # vacuum database root_check if [ ! -v synth_current_system ]; then detect_system fi system_vacuum;; *) # invalid option was given invalid_command $1 exit 1;; esac shift 1 done # show help if we didn't recieve commands either if [ ! -v synth_args_exist ]; then info_help exit 0 fi # unset everything for variable in "${all_known_variables[@]}"; do unset $variable done