Lots of hacking up the scripts to work with the new setup, and more.

11 files changed, 450 insertions, 48 deletions

diff --git a/config/Robust.ini b/config/Robust.ini.template
index 9926ed7..1bbccd5 100644
--- a/config/Robust.ini
+++ b/config/Robust.ini.template
@@ -9,55 +9,7 @@
 ; * Additional OpenSim.Server.Handlers.dll:AssetServiceConnector and 
 ; *            OpenSim.Server.Handlers.dll:XInventoryInConnector 
 ; * are started in port 8002, outside the firewall
-; *
-; **
-; *
-; * The Const section allows us to define some basic information that we
-; * will use throughout our configuration. We will provide examples for 
-; * setting the base url of the Robust server and the public and private ports
-; * it uses. Changing the values of the constants will set the operating
-; * parameters thoughout the configuration. Other constants that may prove
-; * to be useful may be added to the followin section. They may be 
-; * referenced anywhere in the configuration by using ${Const|Name}. One
-; * such use is providing a base path for setting locations that Robust
-; * uses to write data.
-; *
-
-[Includes]
-    ; Define your server specific constants in this file.
-;    Include-Common = /opt/opensim/config/constants.ini
-
-
-[Const]
-    MOTD = "Welcome to virtual 'Gabba."
-
-    Base = "../.."
-    AssetsPath = "../../AssetFiles"
-    CachePath = "../../caches"
-    ConfigPath = "../../config"
-    IncludePath = "../../config-include"
-
-    GridName = "onefangWorld"
-
-    ; For a grid these will usually be the externally accessible IP/DNS
-    ; name and use default public port 8002 and default private port 8003
-    ; For a standalone this will usually be the externally accessible IP/DNS
-    ; name and use default public port 9000. The private port is not used
-    ; in the configuration for a standalone.
-
-    ;# {BaseURL} {} {BaseURL} {"http://example.com" "http://127.0.0.1"} "http://127.0.0.1"
-    HostName = "localhost"
-    BaseURL = http://127.0.0.1
-
-    ;# {PublicPort} {} {PublicPort} {8002 9000} "8002"
-    PublicPort = "8002"
-
-    ;# {PrivatePort} {} {PrivatePort} {8003} "8003"
-    PrivatePort = "8003"
-
 
-    DataProvider = "OpenSim.Data.MySQL.dll"
-    ConnectionString = "Data Source=localhost;Database=$MYSQL_DB;User ID=$MYSQL_USER;Password=$MYSQL_PASSWORD;Old Guids=true;"
 
 ; * The startup section lists all the connectors to start up in this server
 ; * instance. This may be only one, or it may be the entire server suite.
diff --git a/scripts/backup-inventories.sh b/scripts/backup-inventories.sh
new file mode 100755
index 0000000..642d71d
--- /dev/null
+++ b/scripts/backup-inventories.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Figure out where we are, most of this mess is to troll through soft links.
+PRG="$0"
+while [ -h "${PRG}" ] ; do
+  ls=$(ls -ld "${PRG}")
+  link=`expr "${ls}" : '.*-> \(.*\)$'`
+  if expr "${link}" : '.*/.*' > /dev/null; then
+    PRG="${link}"
+  else
+    PRG=$(dirname "${PRG}")/"${link}"
+  fi
+done
+PRGDIR=$(dirname "${PRG}")
+pushd ${PRGDIR} >/dev/null
+PRGDIR=$(pwd)
+popd >/dev/null
+
+# Get the database credentials.
+declare -A creds
+while read -d ';' p; do
+  k=$(echo ${p} | cut -d '=' -f 1)
+  v=$(echo ${p} | cut -d '=' -f 2)
+  creds[${k}]="${v}"
+done < <(grep ConnectionString ${PRGDIR}/../config/config.ini | cut -d '"' -f 2)
+# The above seems the best way to get bash to let the creds assignments survive outside the loop.
+
+# Only backup those that have not logged on since their last backup, but returning prims from sims will bypass this check.
+timestamp=$(ls -o --time-style="+%s" ${PRGDIR}/../backups/.keep | cut -d ' ' -f 5)
+touch ${PRGDIR}/../backups/.keep
+
+# Get the user names, and back 'em up.
+mysql --host="${creds[Data Source]}" "${creds[Database]}" --user="${creds[User ID]}" --password="${creds[Password]}" \
+  -e "select FirstName,LastName from UserAccounts,GridUser where UserAccounts.PrincipalID=GridUser.UserID and GridUser.Logout>${timestamp};" -ss | while read user; do
+  # Replace tab with space
+  user=${user// / }
+  ${PRGDIR}/backup-inventory "${user}"
+  # Sleep for a while, so that there is plenty of time to do the backup,
+  # and we are not keeping the computer very busy if there are lots of users.
+  # My big arsed 1 GB OAR takes about ten minutes to create, and maybe an hour to gitIOR!
+  sleep 200
+done
diff --git a/scripts/backup-inventory b/scripts/backup-inventory
new file mode 120000
index 0000000..d326a74
--- /dev/null
+++ b/scripts/backup-inventory
@@ -0,0 +1 @@
+start-sim
\ No newline at end of file
diff --git a/scripts/backup-sims.sh b/scripts/backup-sims.sh
new file mode 100755
index 0000000..fb18c18
--- /dev/null
+++ b/scripts/backup-sims.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Figure out where we are, most of this mess is to troll through soft links.
+PRG="$0"
+while [ -h "${PRG}" ] ; do
+  ls=$(ls -ld "${PRG}")
+  link=`expr "${ls}" : '.*-> \(.*\)$'`
+  if expr "${link}" : '.*/.*' > /dev/null; then
+    PRG="${link}"
+  else
+    PRG=$(dirname "${PRG}")/"${link}"
+  fi
+done
+PRGDIR=$(dirname "${PRG}")
+pushd ${PRGDIR} >/dev/null
+PRGDIR=$(pwd)
+popd >/dev/null
+
+for i in $(seq 99)
+do
+    j=$(printf "sim%02d" $i)
+    if [ -e "${PRGDIR}/../config/$j" ]
+    then
+        cd ${PRGDIR}/../config/$j
+        ./backup-sim
+        # Sleep for a while, so that there is plenty of time to do the backup,
+        # and we are not keeping the computer very busy if there are lots of sims.
+        sleep 200
+    fi
+done
diff --git a/scripts/fix_var_run.sh b/scripts/fix_var_run.sh
new file mode 100755
index 0000000..97567cb
--- /dev/null
+++ b/scripts/fix_var_run.sh
@@ -0,0 +1,6 @@
+# Create the opensim run dir.
+mkdir -p /var/run/opensim
+chown opensim:opensim /var/run/opensim
+chmod ug+rwx  /var/run/opensim
+chmod o-rwx  /var/run/opensim
+chmod g+s  /var/run/opensim
diff --git a/scripts/gitAR.sh b/scripts/gitAR.sh
new file mode 100755
index 0000000..2a067be
--- /dev/null
+++ b/scripts/gitAR.sh
@@ -0,0 +1,118 @@
+#!/bin/bash
+
+# Work around OpenSims slow database corruption bug by using git to store all old backups.
+# Try to squeeze every last byte out of the tarballs.  Seems to cut the total storage size down to one third the size of just the raw I/OAR files.
+# Saves even more if there's been no changes.
+# On the other hand, these backup files will grow indefinately, the more changes, the faster it grows.  I can live with that for more reliable backups that go back further.
+# Tries to avoid loosing data if things go wrong.  I think the main remaining problem would be running out of space, in which case you have bigger problems to deal with.
+
+# Strategy - unpack the last one, unpack and commit any old I/OARs, pack up the result, delete it's working directory, THEN run the save i/oar.
+# Avoids having to sync with OpenSim finishing the current I/OAR, and as a bonus, an easy to deliver latest I/OAR for people that want it.
+
+# Not really meant to be called by users, so don't bother validating the input and such.
+
+type=$1
+title=$2
+
+# Figure out where we are, most of this mess is to troll through soft links.
+PRG="$0"
+while [ -h "${PRG}" ] ; do
+  ls=$(ls -ld "${PRG}")
+  link=`expr "${ls}" : '.*-> \(.*\)$'`
+  if expr "${link}" : '.*/.*' > /dev/null; then
+    PRG="${link}"
+  else
+    PRG=$(dirname "${PRG}")/"${link}"
+  fi
+done
+PRGDIR=$(dirname "${PRG}")
+pushd ${PRGDIR} >/dev/null
+PRGDIR=$(pwd)
+popd >/dev/null
+
+date=$(date '+%F_%T')
+
+# Sanitize the name.  Not removing [ or ], couldn't get that to work, only important for Windows.
+name=$(echo "${title}" | sed -e 's/[\\/:\*\?"<>\|@#$%&\0\x01-\x1F\x27\x40\x60\x7F. ]/_/g' -e 's/^$/NONAME/')
+# Convert the type to uppercase.
+gar="_git$(echo -n ${type} | tr '[:lower:]' '[:upper:]')AR"
+
+if [ -d ${PRGDIR}/../../backups/temp_backup${type}_${name} ]; then
+  echo "WARNING - Mess left over from last backup, not gonna run!"
+  mv ${PRGDIR}/../../backups/temp_backup${type}_${name}/*.oar ${PRGDIR}/../backups
+  exit 1
+fi
+
+mkdir -p ${PRGDIR}/../../backups/temp_backup${type}_${name}
+pushd ${PRGDIR}/../../backups/temp_backup${type}_${name} >/dev/null
+if [ -f ../${name}${gar}.tar.xz ]; then
+  nice -n 19 tar -xf ../${name}${gar}.tar.xz
+else
+  mkdir -p ${name}${gar}
+  git init ${name}${gar} >log
+fi
+
+pushd ${name}${gar} >/dev/null
+
+# Make sure stuff that's already compressed doesn't get compressed by git.
+# Also tries to protect binaries from mangling.
+cat >.gitattributes <<- zzzzEOFzzzz
+*.bvh -delta -diff -text
+*.jp2 -delta -diff -text
+*.jpg -delta -diff -text
+*.llmesh -delta -diff -text
+*.ogg -delta -diff -text
+*.png -delta -diff -text
+*.r32 -delta -diff -text
+*.tga -delta -diff -text
+zzzzEOFzzzz
+git add .gitattributes &>>../log
+# Coz git insists.
+git config user.email "opensim@$(hostname -A | cut -d ' ' -f 1)"
+git config user.name  "opensim"
+
+# Looping through them in case there's a bunch of I/OARs from previous versions of this script.
+find ../.. -maxdepth 1 -type f -name ${name}-*.${type}ar | sort | while read file;  do
+  # Deal with deletions in the inventory / sim, easy method, which becomes a nop for files that stay in the git add below.
+  git rm -fr * &>>../log && \
+  nice -n 19 tar -xzf "${file}" || echo "ERROR - Could not unpack ${file} !" >>../errors
+  if [ ! -f ../errors ]; then
+    git add * &>>../log && git add */\* &>>../log
+    # Magic needed to figure out if there's anything to commit.
+    # After all the pain to get this to work, there's an ever changing timestamp in archive.xml that screws it up.
+    # Like this system didn't have enough timestamps in it already.  lol
+    # TODO - I could sed out that timestamp, and put it back again based on the OAR file name when extracting.
+    #        IARs don't seem to have the timestamp.
+    if t=$(git status --porcelain) && [ -z "${t}" ]; then
+      true
+    else
+      # Note this commit message has to be just the file name, as the ungitAR script uses it.
+      git commit -qm "$(basename ${file})" &>>../log || echo "ERROR - Could not commit ${file} !" >>../errors
+    fi
+    if [ ! -f ../errors ]; then
+      mv ${file} ..
+    fi
+  fi
+  if [ -f ../errors ]; then
+    exit 1      # Seems to only exit from this loop, not the script.  Makes me want to rewrite this in a real language.  lol
+  fi
+done
+
+#git gc --aggressive --prune=now                # Takes a long time, doesn't gain much.  Even worse, it increases the size of the resulting tarball.  lol
+
+popd >/dev/null
+
+if [ ! -f errors ]; then
+  XZ_OPT="-9e" nice -n 19 tar -c --xz ${name}${gar} -f ../${name}${gar}.tar.xz || echo "ERROR - Could not pack gitAR!" >>errors
+fi
+
+popd >/dev/null
+
+if [ -f ${PRGDIR}/../../backups/temp_backup${type}_${name}/errors ]; then
+  echo "NOT cleaning up coz - "
+  cat ${PRGDIR}/../../backups/temp_backup${type}_${name}/errors
+else
+  rm -fr ${PRGDIR}/../../backups/temp_backup${type}_${name}
+fi
+
+echo -n ${name}
diff --git a/scripts/opensim-crontab.txt b/scripts/opensim-crontab.txt
new file mode 100644
index 0000000..f26b25d
--- /dev/null
+++ b/scripts/opensim-crontab.txt
@@ -0,0 +1,16 @@
+# to see current crontab for opensim
+#  $ sudo crontab -u opensim -l
+#
+# to install this crontab for opensim:
+#  $ cat /opt/opensim/current/scripts/opensim-crontab.txt | sudo crontab -u opensim -
+#
+# at midnight, remove old logs, files created 3 or more days ago
+#0 0 * * *       find /var/log/opensim -ctime +1 -delete
+# experimental version using savelog -c cycles all opensim log files over 7 days
+0 0 * * *       /usr/bin/savelog -c 7 /var/log/opensim/*.log > /dev/null
+
+# Backup the inventories and sims every 6 hours.
+0 */6 * * *       /opt/opensim/current/scripts/backup-sims.sh
+
+# Only backup inventories on the grid server itself.
+#55 */6 * * *       /opt/opensim/current/scripts/backup-inventories.sh
diff --git a/scripts/opensim-monit.conf b/scripts/opensim-monit.conf
new file mode 100644
index 0000000..7785056
--- /dev/null
+++ b/scripts/opensim-monit.conf
@@ -0,0 +1,36 @@
+# manage the OpenSim process for Your Sim
+#
+# usage:
+#     monit start your_sim
+#     monit stop your_sim
+#     monit restart your_sim
+#
+# see 'daemon' setting in /etc/monit/monitrc for the cycle length.
+# on ubuntu/debian, this is overridden by the CHECK_INTERVALS var in
+# /etc/default/monit .  the below assumes you've set it to 30 seconds.
+#
+# if process dies, will restart sim within 30 seconds.  if process
+# dies 5 times in as many tries, will stop trying and send email
+# alert.
+#
+# if SimFPS drops to 0 for 2 minutes, restart.
+#
+# if process CPU usage stays above 300% for 2 minutes, restart.
+#
+# see ../README for configuration instructions.
+#
+# Hmmmm, seems that monit changing to a user and running bash -l is different from sudo changing to a user and running bash -l.
+#
+check process sim01 with pidfile /var/run/opensim/sim01.pid
+    start program = "/usr/bin/sudo -Hu opensim /bin/bash -lc 'cd /opt/opensim/config/sim01 && /opt/opensim/config/sim01/start-sim -q'"
+        as uid root and gid root
+    stop program = "/usr/bin/sudo -Hu opensim /bin/bash -lc 'cd /opt/opensim/config/sim01 && /opt/opensim/config/sim01/stop-sim'" with timeout 600 seconds
+    if cpu usage > 50% for 4 cycles then restart
+#    if 5 restarts within 5 cycles then timeout
+#    if failed url http://127.0.0.1:9005/jsonSimStats/
+#        and content != '"SimFPS":0.0,' for 4 cycles
+#        then restart
+#    if failed url http://127.0.0.1:9008/jsonSimStats/
+#        and content == '"SimFPS":' for 4 cycles
+#        then restart
+
diff --git a/scripts/opensim.tmux.conf b/scripts/opensim.tmux.conf
new file mode 100644
index 0000000..ecd0b04
--- /dev/null
+++ b/scripts/opensim.tmux.conf
@@ -0,0 +1,46 @@
+set-option -g prefix C-a
+unbind-key C-b
+bind-key C-a send-prefix
+
+# r reloads the configuration, handy
+bind r source-file ~/.tmux.conf
+
+unbind % # Remove default binding since we’re replacing them.
+bind | split-window -h
+bind - split-window -v
+
+set-option -g bell-action any
+set-option -g bell-on-alert on
+
+set-option -g history-limit 100000
+
+set-option -g mouse-resize-pane on
+set-option -g mouse-select-pane on
+set-option -g mouse-select-window on
+set-option -g mouse-utf8 on
+
+#set-option -g set-remain-on-exit on
+set-option -g set-titles on
+set-option -g set-titles-string '#W'
+
+set-option -g status-interval 1
+set-option -g status-justify centre
+set-option -g status-utf8 on
+
+set-option -g status-left-length 80
+set-option -g status-left '[#H #S #F]'
+set-option -g status-right-length 80
+set-option -g status-right '%F #(uptime)'<-----># &F is ISO date, uptime starts with the current time, and ends with the load average.  B-)
+
+set-option -g visual-activity on
+set-option -g visual-content on
+set-option -g visual-silence on
+
+set-option -gw clock-mode-style 24
+set-option -gw mode-mouse on
+
+# Highlight active window
+set-option -gw window-status-current-bg red
+
+# Set window notifications
+set-option -gw monitor-activity on
diff --git a/scripts/show-console b/scripts/show-console
new file mode 100755
index 0000000..ddeba86
--- /dev/null
+++ b/scripts/show-console
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+tmux -S /var/run/opensim/opensim-tmux.socket select-window -t "OpenSim_console" \; attach-session -t "OpenSim_console"
diff --git a/scripts/start-sim b/scripts/start-sim
new file mode 100755
index 0000000..cd28b43
--- /dev/null
+++ b/scripts/start-sim
@@ -0,0 +1,152 @@
+#!/bin/bash
+
+# Figure out where we are, most of this mess is to troll through soft links.
+PRG="$0"
+while [ -h "${PRG}" ] ; do
+  ls=$(ls -ld "${PRG}")
+  link=`expr "${ls}" : '.*-> \(.*\)$'`
+  if expr "${link}" : '.*/.*' > /dev/null; then
+    PRG="${link}"
+  else
+    PRG=$(dirname "${PRG}")/"${link}"
+  fi
+done
+PRGDIR=$(dirname "${PRG}")
+pushd ${PRGDIR} >/dev/null
+PRGDIR=$(pwd)
+popd >/dev/null
+
+
+USER=$(whoami)
+console_name="OpenSim_console"
+tmux_command="tmux -S /var/run/opensim/opensim-tmux.socket"
+tmux_session=${console_name}
+tmux_window="${tmux_command} select-window -t ${tmux_session}"
+tmux_send="${tmux_command} send-keys -t ${tmux_session}"
+bindir="${PRGDIR}/../bin"
+date=$(date '+%F_%T')
+quiet=""
+inventory=""
+
+if [ $USER = "opensim" ]
+then
+    SUDO=""
+else
+    SUDO="sudo -Hu opensim"
+fi
+
+if [ "$1" = "-q" ]
+then
+    quiet="true"
+    shift 1
+fi
+   
+if [ "x$1" = "x" ]; then
+    tgt=$(basename $(pwd))
+    num=$(echo ${tgt} | cut -c 4-)
+elif [ -d "./$1" ]; then
+    tgt=$1
+elif [ -d "./sim$1" ]; then
+    num=$1
+    tgt="./sim${num}"
+else
+    tgt=$1
+    inventory=$1
+fi
+
+if [ "x$tgt" = "x" ]; then
+    echo "usage:"
+    echo "    $ $(basename $0) <sim>"
+    echo "where <sim> is one of: " robust sim[0-9][0-9]
+    exit 1
+fi
+
+cd ${bindir}
+
+if ( ${tmux_command} -q list-sessions 2>/dev/null | grep -q ${console_name}: ); then
+    true
+else
+    # The sudo is only so that the session is owned by opensim, otherwise it's owned by whoever ran this script, which is a likely security hole.
+    # After the session is created, we rely on the /var/run/opensim directory to be group sticky, so that anyone in the opensim group can attach to the tmux socket.
+    $SUDO ${tmux_command} new-session -d -s ${console_name} -n "Server"
+fi
+
+
+if [ "x$tgt" = "xROBUST" ]; then
+    exe="Robust"
+    title="ROBUST"
+elif [ "x$inventory" = "x" ]; then
+    exe="OpenSim"
+    # Grab the first Section line of the sims .xml file, cut it down to the name.
+    title=$(grep "<Section " ../config/${tgt}/*.xml | head -n 1 | cut -d '"' -f 2 )
+fi
+
+case $(basename $0) in 
+    "start-sim")
+        cmd="mono ${exe}.exe -inidirectory=${tgt} "
+
+        # Check if it's already running.
+        if [ -e /var/run/opensim/${tgt}.pid ]
+        then
+            # Double check if it's REALLY running.
+            if [ "x$(ps -p $(cat /var/run/opensim/${tgt}.pid) --no-headers -o comm)" = "x" ]; then
+                $SUDO rm -f /var/run/opensim/${tgt}.pid
+            fi
+        fi
+        # Now see if it's really really running.  lol
+        if [ ! -e /var/run/opensim/${tgt}.pid ]
+        then
+            if [ "x$tgt" = "xROBUST" ]; then
+                $SUDO cat ../config/config.ini ../config/Robust.ini.template >../config/Robust.ini
+                ${tmux_command} split-window -vp 30 -t "${tmux_session}:" "${cmd}"
+            else
+                ${tmux_command} new-window -dn "[${title}]" -t "${tmux_session}:${num}" "${cmd}"
+            fi
+        fi
+
+        if [ "x$quiet" = "x" ]
+        then
+            if [ "x$tgt" = "xROBUST" ]; then
+                ${tmux_window} \; attach-session -t "${tmux_session}"
+            fi
+        fi
+        ;;
+
+    "backup-inventory")
+        user=$($SUDO ${PRGDIR}/gitAR.sh i "${inventory}")
+        # Add the full date and time to create the IAR file name.
+        cmd="save iar -c ${inventory} / password ${PRGDIR}/../../backups/${user}-${date}.iar"
+        # Do it in the highest numbered window.
+        ${tmux_send}:"$" "${cmd}" Enter
+        ${tmux_send}:"$" "force gc" Enter
+        ;;
+
+    "backup-sim")
+        sim=$($SUDO ${PRGDIR}/gitAR.sh o "${title}")
+        # Add the full date and time to create the OAR file name.
+        cmd="save oar --all ${PRGDIR}/../../backups/${sim}-${date}.oar"
+        if [ -e /var/run/opensim/${tgt}.pid ]; then
+            ${tmux_send}:"[${title}]" "${cmd}" Enter
+            # Wait a bit, then generate the map tiles, coz the good generator leaks memory badly if you leave it turned on.
+            sleep 30
+            ${tmux_send}:"[${title}]" "generate map" Enter
+            ${tmux_send}:"[${title}]" "force gc" Enter
+        else
+            echo "No OAR created for ${title}, it's not running."
+        fi
+        ;;
+
+    "stop-sim")
+        if [ -e /var/run/opensim/${tgt}.pid ]; then
+            ${tmux_send}:"[${title}]" "alert WARNING - Emergency shutdown in one minute!" Enter
+            ${tmux_send}:"[${title}]" "alert WARNING - Emergency shutdown in one minute!" Enter
+            sleep 30
+            ${tmux_send}:"[${title}]" "alert WARNING - Emergency shutdown in thirty seconds!" Enter
+            ${tmux_send}:"[${title}]" "alert WARNING - Emergency shutdown in thirty seconds!" Enter
+            sleep 30
+            ${tmux_send}:"[${title}]" "shutdown" Enter
+        else
+            echo "Sim ${title} is not running, so not stopping."
+        fi
+        ;;
+esac