work on helperbot more, .gitignore

This commit is contained in:
Ruben 2025-07-06 22:51:45 -05:00
commit 47d504da96
No known key found for this signature in database
GPG key ID: 8EA836555FB6D9A5
2 changed files with 328 additions and 183 deletions

1
.gitignore vendored Normal file
View file

@ -0,0 +1 @@
testing.sh

504
helperbot
View file

@ -12,50 +12,152 @@
# =============================================================================
# ╭─────────────────────────╮
# │ functions and variables │
# ╰─────────────────────────╯
# unset everything - ensure we're working with a clean state
unset synth_help && unset synth_upgrade && unset synth_backup && unset synth_vacuum && unset synth_invalid && unset synth_current_system
# exit immediately if an error occurs somewhere to prevent Fucked Up Shit
set -e
# defining colors for text output
if [[ -t 1 ]]; then
red=$( tput setaf 1 );
green=$( tput setaf 2 );
yellow=$( tput setaf 3 );
blue=$( tput setaf 4 );
pink=$( tput setaf 5 );
cyan=$( tput setaf 6 );
gray=$( tput setaf 8 );
normal=$( tput sgr 0 );
# ╭──────────────────────────────────────╮
# │ functions and variables - start here │
# ╰──────────────────────────────────────╯
# beware of function spam following this point :3
# unset everything - ensure we're working with a clean state
all_known_variables=("synth_current_system" "synth_args_exist")
for variable in "${all_known_variables[@]}"; do
unset $variable
done
# set variable to determine if we actually received any arguments
if [ -n "$1" ]; then
synth_args_exist=1
fi
# attempt to detect the system based on hostname
function detect_system {
if [ "$(hostname)" = "phosphorus" ]; then
synth_current_system=phosphorus
echo "Detected ${blue}phosphorus${normal}."
elif [ "$(hostname)" = "neptunium" ]; then
synth_current_system=neptunium
echo "Detected ${blue}neptunium${normal}."
elif [ "$(hostname)" = "cerium" ]; then
synth_current_system=cerium
echo "Detected ${blue}cerium${normal}."
elif [ "$(hostname)" = "synthnix" ]; then
synth_current_system=synthnix
echo "Detected ${blue}synthnix${normal}."
else
local valid_systems=("phosphorus" "neptunium" "cerium" "synthnix")
local current_hostname=$(hostname)
for variable in "${all_known_variables[@]}"; do
if [ "$current_hostname" = "$system" ]; then
synth_current_system=$system
echo "Detected ${blue}${system}${normal}."
return 0
fi
done
# report if no valid system was found
echo "${red}Failed to detect system.${normal}"
echo "We're most likely being run in an environment we don't know of."
echo "Exiting..."
exit 1
}
# defining text formatting for text output
if [[ -t 1 ]]; then
red=$(tput setaf 1);
green=$(tput setaf 2);
yellow=$(tput setaf 3);
blue=$(tput setaf 4);
pink=$(tput setaf 5);
cyan=$(tput setaf 6);
gray=$(tput setaf 8);
bold=$(tput bold)
underline=$(tput smul)
normal=$( tput sgr 0);
fi
# =============================================================================
# ╭───────────────────╮
# │ defining messages │
# ╰───────────────────╯
# yes i know spamming echo is a bad way of doing this but other actually efficient requires me to remove indenting which makes it harder to read this code for me 💔
# header
function header {
echo "╭────────────────╮"
echo "│ helperbot! owo │"
echo "╰────────────────╯"
echo
sleep 1 # grace period
}
# help info
function info_help {
echo "${pink}${bold}${underline}Usage:${normal} ${bold}helperbot${normal} [-h|-u|-b|-v] [--update-email-certs|--sync-blocklists|--update-frontends]"
echo
echo "${blue}${bold}${underline}Options:${normal}"
echo "${bold}-h${normal}, ${bold}--help${normal}"
echo " Show this help page."
echo
echo "${green}${bold}${underline}System maintenance:${normal}"
echo "${bold}-u${normal}, ${bold}--upgrade${normal}"
echo " Perform a full system upgrade, including containers and services."
echo
echo "${bold}-b${normal}, ${bold}--backup${normal}"
echo " Perform a backup of all known services."
echo
echo "${bold}-v${normal}, ${bold}--vacuum${normal}"
echo " Vacuum the postgresql databases."
echo
echo "${bold}--update-email-certs${normal}"
echo " Pull the email/XMPP certificates from Caddy into ${underline}/etc/certs${normal}."
echo
echo "${cyan}${bold}${underline}Fediverse:${normal}"
echo "${bold}--sync-blocklists${normal}"
echo " Import blocklist from Sharkey -> Iceshrimp"
echo
echo "${bold}--update-frontends${normal}"
echo " Update standalone fediverse frontends."
echo
echo "helperbot automatically knows what to do for some actions based on this system's hostname. Beep!"
echo
echo "${yellow}${bold}This script is still generally a work-in-progress.${normal}"
echo "Report breakage or suggestions or improvments or whatever to here:"
echo "${blue}https://forged.synth.download/synth.download/synth.download${normal}"
echo
}
# invalid command message
function invalid_command {
echo "${red}Error:${normal} Invalid option \""$1"\"."
echo "\"helperbot is very confused... >~<\""
echo
echo "Run with --help to see all options."
}
# attempt to detect the system based on hostname
function detect_system {
local valid_systems=("phosphorus" "neptunium" "cerium" "synthnix")
local current_hostname=$(hostname)
for system in "${valid_systems[@]}"; do
if [ "$current_hostname" = "$system" ]; then
synth_current_system=$system
echo "Detected ${blue}${system}${normal}."
return 0
fi
done
# report if no valid system was found
echo "${red}Failed to detect system.${normal}"
echo "We're most likely being run in an environment we don't know of."
echo "Exiting..."
exit 1
}
# root check
function root_check {
if [[ ${UID} != 0 ]]; then
echo "${red}helperbot must be run as root or with sudo permissions to perform this action!${normal} Beep!"
exit 1
fi
}
# ╭─────────────────╮
# │ upgrade related │
# ╰─────────────────╯
# base system upgrade - generic steps for debian/ubuntu based systems
function base_system_upgrade {
echo "${cyan}Upgrading base system.${normal}"
@ -81,131 +183,10 @@ function upgrade_docker_container {
fi
}
# psql vacuuming
# reusable step to vacuum databases - postgres_vacuum [postgres-db-1] [user_and_db_name] [password]
function postgres_vacuum {
docker exec -it "$1" /bin/bash -c "POSTGRES_PASSWORD="$3" psql -U "$2" -d "$2" -c 'VACUUM ANALYZE;'"
}
# postgres_vacuum_self [postgres-db-1]
function postgres_vacuum_self {
docker exec -it "$1" /bin/bash -c "psql -U postgres -c 'VACUUM ANALYZE;'"
}
# psql backup
# reusable step to backup databases - postgres_backup [postgres-db-1] [user_and_db_name] [output_name] [$backup_working_directory]
function postgres_backup {
docker exec "$1" /bin/bash -c "pg_dump "$2" --username "$2" > "$3".sql"
docker cp "$1":/$3.sql $4/$3/$3.sql
docker exec "$1" /bin/bash -c "rm "$3".sql"
}
# redis snapshot
# tells redis to make a snapshot - redis_snapshot [whatever-redis-1]
function redis_snapshot {
docker exec $1 redis-cli SAVE
}
# backup - create folder and copy
# step that combines the process of making folders and copying files for backup
# backup_create_copy ["source files"] [subpath/to/folder] [$backup_working_directory]
function backup_create_copy {
mkdir -p $3/$2
cp -r $1 $3/$2
}
# ╭───────────────────╮
# │ defining messages │
# ╰───────────────────╯
# header
function header {
echo "╭────────────────╮"
echo "│ helperbot! owo │"
echo "╰────────────────╯"
echo
sleep 1 # grace period
}
# help info
function info_help {
echo "${blue}Usage:${normal} helperbot [-h|-u|-b|-v]"
echo
echo "${green}Options:${normal}"
echo "-h, --help Print this help page."
echo "-u, --upgrade Update the system."
echo "-b, --backup Backup the system."
echo "-v, --vacuum Vacuum the postgresql databases."
echo
echo "helperbot automatically knows what to do based on this system's hostname! Beep!"
echo
echo "${yellow}This script is still generally a work-in-progress.${normal}"
echo "Report breakage or suggestions or improvments or whatever to here:"
echo "https://forged.synth.download/synth.download/synth.download"
echo
}
# =============================================================================
# ╭──────────────╮
# │ main program │
# ╰──────────────╯
# check to see if we're running as root
#if [[ ${UID} != 0 ]]; then
# echo "${red}helperbot must be run as root or with sudo permissions!${normal} thanks!"
# exit 1
#fi
# display the header
header
# evaluate arguments and set environment variables to enable each command and see what should be executed
while [ -n "$1" ]; do
case "$1" in
-h | --help) # display help info
synth_help=1;;
-u | --upgrade) # upgrade system
synth_upgrade=1
if [ ! -v synth_current_system ]; then
detect_system
fi;;
-b | --backup) # backup system
synth_backup=1
if [ ! -v synth_current_system ]; then
detect_system
fi;;
-v | --vacuum) # vacuum database
synth_vacuum=1
if [ ! -v synth_current_system ]; then
detect_system
fi;;
*) # invalid option was given
synth_invalid=1;;
esac
shift 1
done
# say invalid option if we get an invalid option (duh)
if [ -v synth_invalid ]; then
echo "${red}Error:${normal} Invalid option."
echo "\"helperbot is very confused... >~<\""
echo
echo "Run with --help to see all options."
exit 1
fi
# runs if no option was specified; throw up the help menu
# otherwise: also run if specified
if [[ ! -v synth_args_exist || -v synth_help ]]; then
info_help
exit 0
fi
# ╭──────────────╮
# │ upgrade step │
# ╰──────────────╯
if [ -v synth_upgrade ]; then
function system_upgrade {
#timestamp=$(date +'%Y%m%d%H%M%S')
#synth_upgrade_log=/tmp/upgrade-output-${timestamp}.txt
echo "${blue}upgrade:${normal} Running full system upgrade for ${green}${synth_current_system}${normal}."
@ -230,7 +211,7 @@ if [ -v synth_upgrade ]; then
upgrade_docker_container "/srv/docker" "forgejo" "compose.yaml"
upgrade_docker_container "/srv/docker" "forgejo" "compose-runner.yaml"
upgrade_docker_container "/srv/docker" "freshrss" "compose.yaml"
upgrade_docker_container "/srv/docker" "vaultwarden" "compose-runner.yaml"
upgrade_docker_container "/srv/docker" "vaultwarden" "compose.yaml"
upgrade_docker_container "/srv/docker" "ask-js" "compose.yaml"
# done
echo "${green}System upgrade finished! beep!~${normal}"
@ -250,34 +231,86 @@ if [ -v synth_upgrade ]; then
# done
echo "${green}System upgrade finished! beep!~${normal}"
fi
fi
}
# ╭────────────────╮
# │ backup related │
# ╰────────────────╯
# mostly just symlinks to commands because i think it looks less ugly (and easier to update)
# psql vacuuming
# reusable step to vacuum databases - postgres_vacuum [postgres-db-1] [user_and_db_name] [password]
function postgres_vacuum {
# load postgres passwords
if [ -f /etc/secrets/postgres.env ]; then
export $(grep -v '^#' /etc/secrets/postgres.env | xargs)
else
echo "${red}postgres_vacuum:${normal} Postgresql Secrets don't exist. Exiting..."
exit 1
fi
# vacuum
docker exec -it "$1" /bin/bash -c "POSTGRES_PASSWORD="$3" psql -U "$2" -d "$2" -c 'VACUUM ANALYZE;'"
# unset secrets
unset $(grep -v '^#' /etc/secrets/postgres.env | sed -E 's/(.*)=.*/\1/' | xargs)
}
# postgres_vacuum_self
function postgres_vacuum_self {
docker exec -it postgres-db-1 /bin/bash -c "psql -U postgres -c 'VACUUM ANALYZE;'"
}
# psql backup
# reusable step to backup databases - postgres_backup [postgres-db-1] [user_and_db_name] [output_name] [$backup_working_directory]
function postgres_backup {
# for some reason, doing a dump *doesn't* require a password apparently. huh
docker exec "$1" /bin/bash -c "pg_dump "$2" --username "$2" > "$3".sql"
docker cp "$1":/$3.sql $4/$3/$3.sql
docker exec "$1" /bin/bash -c "rm "$3".sql"
}
# redis snapshot
# tells redis to make a snapshot - redis_snapshot [whatever-redis-1]
function redis_snapshot {
docker exec $1 redis-cli SAVE
}
# b2 upload
# load secrets then start uploading to backblaze b2 - b2_upload [$backup_working_directory] [$backup_output_tar]
function b2_upload {
# load in secrets from external file
if [ -f /etc/secrets/b2.env ]; then
export $(grep -v '^#' /etc/secrets/b2.env | xargs)
else
echo "${red}b2_upload:${normal} B2 Secrets don't exist. Exiting..."
exit 1
fi
# upload file specified
backblaze-b2 authorize-account $B2_KEYID $B2_SECRET
backblaze-b2 upload-file $B2_BACKUP_BUCKET ""$1"/"$2".zst" ""$2".zst"
backblaze-b2 clear-account # just to ensure we won't stay authenticated afterwards
# clear out secrets
unset $(grep -v '^#' /etc/secrets/b2.env | sed -E 's/(.*)=.*/\1/' | xargs)
}
# ╭─────────────╮
# │ backup step │
# ╰─────────────╯
if [ -v synth_backup ]; then
if [ -v synth_vacuum ]; then
echo "${yellow}NOTICE:${normal} You've also passed in the --vacuum command. Note that upgrading also automatically vacuums the databases beforehand."
sleep 1
fi
function system_backup {
echo "${blue}backup:${normal} Running full system backup for ${green}${synth_current_system}${normal}."
if [ "$synth_current_system" = "phosphorus" ]; then # phosphorus
# variables
# variables - could probably be set locally but unsure how much this will dynamically change between systems
backup_local_folder=/srv/docker
backup_working_directory=/var/backups/phosphorus
backup_output_tar=phosphorus.tar
backup_media_output_tar=media_backups.tar # refers to the old local fedi media before s3 migration
# external files containing secrets
export $(grep -v '^#' /etc/secrets/b2.env | xargs)
export $(grep -v '^#' /etc/secrets/postgres.env | xargs)
# initial
backup_media_output_tar=fedi_media_backups.tar
# =============================================================================
# initial steps - cleanup then create
rm -fr $backup_working_directory/*
mkdir -p $backup_working_directory
# database vacuuming
echo "${blue}Vacuuming postgres databases...${normal}"
postgres_vacuum_self postgres-db-1
postgres_vacuum postgres-db-1 misskey ${SHARKEY_POSTGRES_PASSWORD}
postgres_vacuum postgres-db-1 iceshrimp ${ICESHRIMP_POSTGRES_PASSWORD}
postgres_vacuum postgres-db-1 mastodon ${MASTODON_POSTGRES_PASSWORD}
# =============================================================================
# call in database vacuuming function
echo "${blue}Calling in vacuuming...${normal}"
system_vacuum
# =============================================================================
# backup files - sharkey
echo "${blue}Pulling in Sharkey...${normal}"
@ -322,26 +355,137 @@ if [ -v synth_backup ]; then
# configs, extra
cp -r $backup_local_folder/pds/compose.yaml $backup_working_directory/pds
# =============================================================================
# unset secrets
# pull in any other common configs and secrets
echo "${blue}Pulling in other configurations...${normal}"
mkdir -p $backup_working_directory/other/etc/caddy
mkdir -p $backup_working_directory/other/etc/secrets
cp /etc/caddy/Caddyfile $backup_working_directory/other/etc/caddy/Caddyfile
cp -r /etc/secrets/* $backup_working_directory/other/etc/secrets/
# =============================================================================
# archive and compress everything
echo "${blue}Compressing everything into one archive...${normal}"
tar -cf "$backup_working_directory/$backup_output_tar" $backup_working_directory # create the archive
zstd -z -T3 -9 --rm "$backup_working_directory/$backup_output_tar" # compress the archive
# TODO: it may be possible to combine these steps so tar automatically compresses the archive with zstd instead of doing it separately
# =============================================================================
# upload backup to backblaze - secrets used here are fetched from b2.env
echo "${blue}Uploading backup...${normal}"
b2_upload $backup_working_directory $backup_output_tar
# =============================================================================
# cleanup
echo "${blue}Cleaning up...${normal}"
rm -fr ${backup_working_directory}/${backup_output_tar}.zst $backup_working_directory/*
# =============================================================================
# unload secrets - we already unload them for each vacuum/upload step, but we want to ensure they are
unset $(grep -v '^#' /etc/secrets/b2.env | sed -E 's/(.*)=.*/\1/' | xargs)
unset $(grep -v '^#' /etc/secrets/postgres.env | sed -E 's/(.*)=.*/\1/' | xargs)
elif [ "$synth_current_system" = "neptunium" ]; then # neptunium
postgres_vacuum_self postgres-db-1
postgres_vacuum_self
elif [ "$synth_current_system" = "cerium" ]; then # cerium
postgres_vacuum_self postgres-db-1
postgres_vacuum_self
elif [ "$synth_current_system" = "synthnix" ]; then # synthnix
# as synthnix doesn't really include much and serves as a place for members
# we just need to back up the home directory here
#
# WIP
echo "wip"
fi
echo "${green}System backup finished! beep!~${normal}"
}
# backup - create folder and copy
# step that combines the process of making folders and copying files for backup
# backup_create_copy ["source files"] [subpath/to/folder] [$backup_working_directory]
function backup_create_copy {
mkdir -p $3/$2
cp -r $1 $3/$2
}
# ╭─────────────╮
# │ vacuum step │
# ╰─────────────╯
function system_vacuum {
echo "${blue}vacuum:${normal} Running database vacuums for ${green}${synth_current_system}${normal}."
# external files containing secrets
if [ -f /etc/secrets/postgres.env ]; then
export $(grep -v '^#' /etc/secrets/postgres.env | xargs)
else
echo "${red}vacuum:${normal} Secrets don't exist. Exiting..."
exit 1
fi
# vacuum
if [ "$synth_current_system" = "phosphorus" ]; then # phosphorus
postgres_vacuum_self
postgres_vacuum postgres-db-1 misskey ${SHARKEY_POSTGRES_PASSWORD}
postgres_vacuum postgres-db-1 iceshrimp ${ICESHRIMP_POSTGRES_PASSWORD}
postgres_vacuum postgres-db-1 mastodon ${MASTODON_POSTGRES_PASSWORD}
elif [ "$synth_current_system" = "neptunium" ]; then # neptunium
postgres_vacuum_self
elif [ "$synth_current_system" = "cerium" ]; then # cerium
postgres_vacuum_self
elif [ "$synth_current_system" = "synthnix" ]; then # synthnix
# as synthnix doesn't really include much and serves as a place for members
# we just need to back up the home directory here
#
# WIP
echo "wip"
fi
# unload secrets - if we pass that they do exist, no need to check if they exist here again
unset $(grep -v '^#' /etc/secrets/postgres.env | sed -E 's/(.*)=.*/\1/' | xargs)
echo "${green}Vacuuming complete! Beep!~${normal}${normal}"
}
# ╭────────────────────────────────────╮
# │ functions and variables - end here │
# ╰────────────────────────────────────╯
# =============================================================================
# ╭──────────────╮
# │ main program │
# ╰──────────────╯
# display the header
header
# evaluate arguments and set environment variables to enable each command and see what should be executed
while [ -n "$1" ]; do
case "$1" in
-h | --help) # display help info
info_help
exit 0;;
-u | --upgrade) # upgrade system
root_check
if [ ! -v synth_current_system ]; then
detect_system
fi
system_upgrade;;
-b | --backup) # backup system
root_check
if [ ! -v synth_current_system ]; then
detect_system
fi
system_backup;;
-v | --vacuum) # vacuum database
root_check
if [ ! -v synth_current_system ]; then
detect_system
fi
system_vacuum;;
*) # invalid option was given
invalid_command $1
exit 1;;
esac
shift 1
done
# show help if we didn't recieve commands either
if [ ! -v synth_args_exist ]; then
info_help
exit 0
fi
#if [[ -v system_vacuum ]]; then
# echo "${yellow}NOTICE:${normal} You've also passed in the --vacuum command. Note that upgrading also automatically vacuums the databases beforehand."
# sleep 1
#fi
# unset everything
unset synth_help && unset synth_upgrade && unset synth_backup && unset synth_vacuum && unset synth_invalid && unset synth_current_system
for variable in "${all_known_variables[@]}"; do
unset $variable
done