From 96eda14b7a1caff77da000a9a957b20ab1a1bb6e Mon Sep 17 00:00:00 2001 From: David Walter Seikel Date: Tue, 8 Nov 2016 15:49:56 +1000 Subject: Lots of hacking up the scripts to work with the new setup, and more. --- scripts/backup-inventories.sh | 42 ++++++++++++ scripts/backup-inventory | 1 + scripts/backup-sims.sh | 30 +++++++++ scripts/fix_var_run.sh | 6 ++ scripts/gitAR.sh | 118 ++++++++++++++++++++++++++++++++ scripts/opensim-crontab.txt | 16 +++++ scripts/opensim-monit.conf | 36 ++++++++++ scripts/opensim.tmux.conf | 46 +++++++++++++ scripts/show-console | 3 + scripts/start-sim | 152 ++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 450 insertions(+) create mode 100755 scripts/backup-inventories.sh create mode 120000 scripts/backup-inventory create mode 100755 scripts/backup-sims.sh create mode 100755 scripts/fix_var_run.sh create mode 100755 scripts/gitAR.sh create mode 100644 scripts/opensim-crontab.txt create mode 100644 scripts/opensim-monit.conf create mode 100644 scripts/opensim.tmux.conf create mode 100755 scripts/show-console create mode 100755 scripts/start-sim (limited to 'scripts') diff --git a/scripts/backup-inventories.sh b/scripts/backup-inventories.sh new file mode 100755 index 0000000..642d71d --- /dev/null +++ b/scripts/backup-inventories.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Figure out where we are, most of this mess is to troll through soft links. +PRG="$0" +while [ -h "${PRG}" ] ; do + ls=$(ls -ld "${PRG}") + link=`expr "${ls}" : '.*-> \(.*\)$'` + if expr "${link}" : '.*/.*' > /dev/null; then + PRG="${link}" + else + PRG=$(dirname "${PRG}")/"${link}" + fi +done +PRGDIR=$(dirname "${PRG}") +pushd ${PRGDIR} >/dev/null +PRGDIR=$(pwd) +popd >/dev/null + +# Get the database credentials. +declare -A creds +while read -d ';' p; do + k=$(echo ${p} | cut -d '=' -f 1) + v=$(echo ${p} | cut -d '=' -f 2) + creds[${k}]="${v}" +done < <(grep ConnectionString ${PRGDIR}/../config/config.ini | cut -d '"' -f 2) +# The above seems the best way to get bash to let the creds assignments survive outside the loop. + +# Only backup those that have not logged on since their last backup, but returning prims from sims will bypass this check. +timestamp=$(ls -o --time-style="+%s" ${PRGDIR}/../backups/.keep | cut -d ' ' -f 5) +touch ${PRGDIR}/../backups/.keep + +# Get the user names, and back 'em up. +mysql --host="${creds[Data Source]}" "${creds[Database]}" --user="${creds[User ID]}" --password="${creds[Password]}" \ + -e "select FirstName,LastName from UserAccounts,GridUser where UserAccounts.PrincipalID=GridUser.UserID and GridUser.Logout>${timestamp};" -ss | while read user; do + # Replace tab with space + user=${user// / } + ${PRGDIR}/backup-inventory "${user}" + # Sleep for a while, so that there is plenty of time to do the backup, + # and we are not keeping the computer very busy if there are lots of users. + # My big arsed 1 GB OAR takes about ten minutes to create, and maybe an hour to gitIOR! + sleep 200 +done diff --git a/scripts/backup-inventory b/scripts/backup-inventory new file mode 120000 index 0000000..d326a74 --- /dev/null +++ b/scripts/backup-inventory @@ -0,0 +1 @@ +start-sim \ No newline at end of file diff --git a/scripts/backup-sims.sh b/scripts/backup-sims.sh new file mode 100755 index 0000000..fb18c18 --- /dev/null +++ b/scripts/backup-sims.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Figure out where we are, most of this mess is to troll through soft links. +PRG="$0" +while [ -h "${PRG}" ] ; do + ls=$(ls -ld "${PRG}") + link=`expr "${ls}" : '.*-> \(.*\)$'` + if expr "${link}" : '.*/.*' > /dev/null; then + PRG="${link}" + else + PRG=$(dirname "${PRG}")/"${link}" + fi +done +PRGDIR=$(dirname "${PRG}") +pushd ${PRGDIR} >/dev/null +PRGDIR=$(pwd) +popd >/dev/null + +for i in $(seq 99) +do + j=$(printf "sim%02d" $i) + if [ -e "${PRGDIR}/../config/$j" ] + then + cd ${PRGDIR}/../config/$j + ./backup-sim + # Sleep for a while, so that there is plenty of time to do the backup, + # and we are not keeping the computer very busy if there are lots of sims. + sleep 200 + fi +done diff --git a/scripts/fix_var_run.sh b/scripts/fix_var_run.sh new file mode 100755 index 0000000..97567cb --- /dev/null +++ b/scripts/fix_var_run.sh @@ -0,0 +1,6 @@ +# Create the opensim run dir. +mkdir -p /var/run/opensim +chown opensim:opensim /var/run/opensim +chmod ug+rwx /var/run/opensim +chmod o-rwx /var/run/opensim +chmod g+s /var/run/opensim diff --git a/scripts/gitAR.sh b/scripts/gitAR.sh new file mode 100755 index 0000000..2a067be --- /dev/null +++ b/scripts/gitAR.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +# Work around OpenSims slow database corruption bug by using git to store all old backups. +# Try to squeeze every last byte out of the tarballs. Seems to cut the total storage size down to one third the size of just the raw I/OAR files. +# Saves even more if there's been no changes. +# On the other hand, these backup files will grow indefinately, the more changes, the faster it grows. I can live with that for more reliable backups that go back further. +# Tries to avoid loosing data if things go wrong. I think the main remaining problem would be running out of space, in which case you have bigger problems to deal with. + +# Strategy - unpack the last one, unpack and commit any old I/OARs, pack up the result, delete it's working directory, THEN run the save i/oar. +# Avoids having to sync with OpenSim finishing the current I/OAR, and as a bonus, an easy to deliver latest I/OAR for people that want it. + +# Not really meant to be called by users, so don't bother validating the input and such. + +type=$1 +title=$2 + +# Figure out where we are, most of this mess is to troll through soft links. +PRG="$0" +while [ -h "${PRG}" ] ; do + ls=$(ls -ld "${PRG}") + link=`expr "${ls}" : '.*-> \(.*\)$'` + if expr "${link}" : '.*/.*' > /dev/null; then + PRG="${link}" + else + PRG=$(dirname "${PRG}")/"${link}" + fi +done +PRGDIR=$(dirname "${PRG}") +pushd ${PRGDIR} >/dev/null +PRGDIR=$(pwd) +popd >/dev/null + +date=$(date '+%F_%T') + +# Sanitize the name. Not removing [ or ], couldn't get that to work, only important for Windows. +name=$(echo "${title}" | sed -e 's/[\\/:\*\?"<>\|@#$%&\0\x01-\x1F\x27\x40\x60\x7F. ]/_/g' -e 's/^$/NONAME/') +# Convert the type to uppercase. +gar="_git$(echo -n ${type} | tr '[:lower:]' '[:upper:]')AR" + +if [ -d ${PRGDIR}/../../backups/temp_backup${type}_${name} ]; then + echo "WARNING - Mess left over from last backup, not gonna run!" + mv ${PRGDIR}/../../backups/temp_backup${type}_${name}/*.oar ${PRGDIR}/../backups + exit 1 +fi + +mkdir -p ${PRGDIR}/../../backups/temp_backup${type}_${name} +pushd ${PRGDIR}/../../backups/temp_backup${type}_${name} >/dev/null +if [ -f ../${name}${gar}.tar.xz ]; then + nice -n 19 tar -xf ../${name}${gar}.tar.xz +else + mkdir -p ${name}${gar} + git init ${name}${gar} >log +fi + +pushd ${name}${gar} >/dev/null + +# Make sure stuff that's already compressed doesn't get compressed by git. +# Also tries to protect binaries from mangling. +cat >.gitattributes <<- zzzzEOFzzzz +*.bvh -delta -diff -text +*.jp2 -delta -diff -text +*.jpg -delta -diff -text +*.llmesh -delta -diff -text +*.ogg -delta -diff -text +*.png -delta -diff -text +*.r32 -delta -diff -text +*.tga -delta -diff -text +zzzzEOFzzzz +git add .gitattributes &>>../log +# Coz git insists. +git config user.email "opensim@$(hostname -A | cut -d ' ' -f 1)" +git config user.name "opensim" + +# Looping through them in case there's a bunch of I/OARs from previous versions of this script. +find ../.. -maxdepth 1 -type f -name ${name}-*.${type}ar | sort | while read file; do + # Deal with deletions in the inventory / sim, easy method, which becomes a nop for files that stay in the git add below. + git rm -fr * &>>../log && \ + nice -n 19 tar -xzf "${file}" || echo "ERROR - Could not unpack ${file} !" >>../errors + if [ ! -f ../errors ]; then + git add * &>>../log && git add */\* &>>../log + # Magic needed to figure out if there's anything to commit. + # After all the pain to get this to work, there's an ever changing timestamp in archive.xml that screws it up. + # Like this system didn't have enough timestamps in it already. lol + # TODO - I could sed out that timestamp, and put it back again based on the OAR file name when extracting. + # IARs don't seem to have the timestamp. + if t=$(git status --porcelain) && [ -z "${t}" ]; then + true + else + # Note this commit message has to be just the file name, as the ungitAR script uses it. + git commit -qm "$(basename ${file})" &>>../log || echo "ERROR - Could not commit ${file} !" >>../errors + fi + if [ ! -f ../errors ]; then + mv ${file} .. + fi + fi + if [ -f ../errors ]; then + exit 1 # Seems to only exit from this loop, not the script. Makes me want to rewrite this in a real language. lol + fi +done + +#git gc --aggressive --prune=now # Takes a long time, doesn't gain much. Even worse, it increases the size of the resulting tarball. lol + +popd >/dev/null + +if [ ! -f errors ]; then + XZ_OPT="-9e" nice -n 19 tar -c --xz ${name}${gar} -f ../${name}${gar}.tar.xz || echo "ERROR - Could not pack gitAR!" >>errors +fi + +popd >/dev/null + +if [ -f ${PRGDIR}/../../backups/temp_backup${type}_${name}/errors ]; then + echo "NOT cleaning up coz - " + cat ${PRGDIR}/../../backups/temp_backup${type}_${name}/errors +else + rm -fr ${PRGDIR}/../../backups/temp_backup${type}_${name} +fi + +echo -n ${name} diff --git a/scripts/opensim-crontab.txt b/scripts/opensim-crontab.txt new file mode 100644 index 0000000..f26b25d --- /dev/null +++ b/scripts/opensim-crontab.txt @@ -0,0 +1,16 @@ +# to see current crontab for opensim +# $ sudo crontab -u opensim -l +# +# to install this crontab for opensim: +# $ cat /opt/opensim/current/scripts/opensim-crontab.txt | sudo crontab -u opensim - +# +# at midnight, remove old logs, files created 3 or more days ago +#0 0 * * * find /var/log/opensim -ctime +1 -delete +# experimental version using savelog -c cycles all opensim log files over 7 days +0 0 * * * /usr/bin/savelog -c 7 /var/log/opensim/*.log > /dev/null + +# Backup the inventories and sims every 6 hours. +0 */6 * * * /opt/opensim/current/scripts/backup-sims.sh + +# Only backup inventories on the grid server itself. +#55 */6 * * * /opt/opensim/current/scripts/backup-inventories.sh diff --git a/scripts/opensim-monit.conf b/scripts/opensim-monit.conf new file mode 100644 index 0000000..7785056 --- /dev/null +++ b/scripts/opensim-monit.conf @@ -0,0 +1,36 @@ +# manage the OpenSim process for Your Sim +# +# usage: +# monit start your_sim +# monit stop your_sim +# monit restart your_sim +# +# see 'daemon' setting in /etc/monit/monitrc for the cycle length. +# on ubuntu/debian, this is overridden by the CHECK_INTERVALS var in +# /etc/default/monit . the below assumes you've set it to 30 seconds. +# +# if process dies, will restart sim within 30 seconds. if process +# dies 5 times in as many tries, will stop trying and send email +# alert. +# +# if SimFPS drops to 0 for 2 minutes, restart. +# +# if process CPU usage stays above 300% for 2 minutes, restart. +# +# see ../README for configuration instructions. +# +# Hmmmm, seems that monit changing to a user and running bash -l is different from sudo changing to a user and running bash -l. +# +check process sim01 with pidfile /var/run/opensim/sim01.pid + start program = "/usr/bin/sudo -Hu opensim /bin/bash -lc 'cd /opt/opensim/config/sim01 && /opt/opensim/config/sim01/start-sim -q'" + as uid root and gid root + stop program = "/usr/bin/sudo -Hu opensim /bin/bash -lc 'cd /opt/opensim/config/sim01 && /opt/opensim/config/sim01/stop-sim'" with timeout 600 seconds + if cpu usage > 50% for 4 cycles then restart +# if 5 restarts within 5 cycles then timeout +# if failed url http://127.0.0.1:9005/jsonSimStats/ +# and content != '"SimFPS":0.0,' for 4 cycles +# then restart +# if failed url http://127.0.0.1:9008/jsonSimStats/ +# and content == '"SimFPS":' for 4 cycles +# then restart + diff --git a/scripts/opensim.tmux.conf b/scripts/opensim.tmux.conf new file mode 100644 index 0000000..ecd0b04 --- /dev/null +++ b/scripts/opensim.tmux.conf @@ -0,0 +1,46 @@ +set-option -g prefix C-a +unbind-key C-b +bind-key C-a send-prefix + +# r reloads the configuration, handy +bind r source-file ~/.tmux.conf + +unbind % # Remove default binding since we’re replacing them. +bind | split-window -h +bind - split-window -v + +set-option -g bell-action any +set-option -g bell-on-alert on + +set-option -g history-limit 100000 + +set-option -g mouse-resize-pane on +set-option -g mouse-select-pane on +set-option -g mouse-select-window on +set-option -g mouse-utf8 on + +#set-option -g set-remain-on-exit on +set-option -g set-titles on +set-option -g set-titles-string '#W' + +set-option -g status-interval 1 +set-option -g status-justify centre +set-option -g status-utf8 on + +set-option -g status-left-length 80 +set-option -g status-left '[#H #S #F]' +set-option -g status-right-length 80 +set-option -g status-right '%F #(uptime)'<-----># &F is ISO date, uptime starts with the current time, and ends with the load average. B-) + +set-option -g visual-activity on +set-option -g visual-content on +set-option -g visual-silence on + +set-option -gw clock-mode-style 24 +set-option -gw mode-mouse on + +# Highlight active window +set-option -gw window-status-current-bg red + +# Set window notifications +set-option -gw monitor-activity on diff --git a/scripts/show-console b/scripts/show-console new file mode 100755 index 0000000..ddeba86 --- /dev/null +++ b/scripts/show-console @@ -0,0 +1,3 @@ +#!/bin/bash + +tmux -S /var/run/opensim/opensim-tmux.socket select-window -t "OpenSim_console" \; attach-session -t "OpenSim_console" diff --git a/scripts/start-sim b/scripts/start-sim new file mode 100755 index 0000000..cd28b43 --- /dev/null +++ b/scripts/start-sim @@ -0,0 +1,152 @@ +#!/bin/bash + +# Figure out where we are, most of this mess is to troll through soft links. +PRG="$0" +while [ -h "${PRG}" ] ; do + ls=$(ls -ld "${PRG}") + link=`expr "${ls}" : '.*-> \(.*\)$'` + if expr "${link}" : '.*/.*' > /dev/null; then + PRG="${link}" + else + PRG=$(dirname "${PRG}")/"${link}" + fi +done +PRGDIR=$(dirname "${PRG}") +pushd ${PRGDIR} >/dev/null +PRGDIR=$(pwd) +popd >/dev/null + + +USER=$(whoami) +console_name="OpenSim_console" +tmux_command="tmux -S /var/run/opensim/opensim-tmux.socket" +tmux_session=${console_name} +tmux_window="${tmux_command} select-window -t ${tmux_session}" +tmux_send="${tmux_command} send-keys -t ${tmux_session}" +bindir="${PRGDIR}/../bin" +date=$(date '+%F_%T') +quiet="" +inventory="" + +if [ $USER = "opensim" ] +then + SUDO="" +else + SUDO="sudo -Hu opensim" +fi + +if [ "$1" = "-q" ] +then + quiet="true" + shift 1 +fi + +if [ "x$1" = "x" ]; then + tgt=$(basename $(pwd)) + num=$(echo ${tgt} | cut -c 4-) +elif [ -d "./$1" ]; then + tgt=$1 +elif [ -d "./sim$1" ]; then + num=$1 + tgt="./sim${num}" +else + tgt=$1 + inventory=$1 +fi + +if [ "x$tgt" = "x" ]; then + echo "usage:" + echo " $ $(basename $0) " + echo "where is one of: " robust sim[0-9][0-9] + exit 1 +fi + +cd ${bindir} + +if ( ${tmux_command} -q list-sessions 2>/dev/null | grep -q ${console_name}: ); then + true +else + # The sudo is only so that the session is owned by opensim, otherwise it's owned by whoever ran this script, which is a likely security hole. + # After the session is created, we rely on the /var/run/opensim directory to be group sticky, so that anyone in the opensim group can attach to the tmux socket. + $SUDO ${tmux_command} new-session -d -s ${console_name} -n "Server" +fi + + +if [ "x$tgt" = "xROBUST" ]; then + exe="Robust" + title="ROBUST" +elif [ "x$inventory" = "x" ]; then + exe="OpenSim" + # Grab the first Section line of the sims .xml file, cut it down to the name. + title=$(grep "
../config/Robust.ini + ${tmux_command} split-window -vp 30 -t "${tmux_session}:" "${cmd}" + else + ${tmux_command} new-window -dn "[${title}]" -t "${tmux_session}:${num}" "${cmd}" + fi + fi + + if [ "x$quiet" = "x" ] + then + if [ "x$tgt" = "xROBUST" ]; then + ${tmux_window} \; attach-session -t "${tmux_session}" + fi + fi + ;; + + "backup-inventory") + user=$($SUDO ${PRGDIR}/gitAR.sh i "${inventory}") + # Add the full date and time to create the IAR file name. + cmd="save iar -c ${inventory} / password ${PRGDIR}/../../backups/${user}-${date}.iar" + # Do it in the highest numbered window. + ${tmux_send}:"$" "${cmd}" Enter + ${tmux_send}:"$" "force gc" Enter + ;; + + "backup-sim") + sim=$($SUDO ${PRGDIR}/gitAR.sh o "${title}") + # Add the full date and time to create the OAR file name. + cmd="save oar --all ${PRGDIR}/../../backups/${sim}-${date}.oar" + if [ -e /var/run/opensim/${tgt}.pid ]; then + ${tmux_send}:"[${title}]" "${cmd}" Enter + # Wait a bit, then generate the map tiles, coz the good generator leaks memory badly if you leave it turned on. + sleep 30 + ${tmux_send}:"[${title}]" "generate map" Enter + ${tmux_send}:"[${title}]" "force gc" Enter + else + echo "No OAR created for ${title}, it's not running." + fi + ;; + + "stop-sim") + if [ -e /var/run/opensim/${tgt}.pid ]; then + ${tmux_send}:"[${title}]" "alert WARNING - Emergency shutdown in one minute!" Enter + ${tmux_send}:"[${title}]" "alert WARNING - Emergency shutdown in one minute!" Enter + sleep 30 + ${tmux_send}:"[${title}]" "alert WARNING - Emergency shutdown in thirty seconds!" Enter + ${tmux_send}:"[${title}]" "alert WARNING - Emergency shutdown in thirty seconds!" Enter + sleep 30 + ${tmux_send}:"[${title}]" "shutdown" Enter + else + echo "Sim ${title} is not running, so not stopping." + fi + ;; +esac -- cgit v1.1