aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authorDavid Walter Seikel2016-11-08 15:49:56 +1000
committerDavid Walter Seikel2016-11-08 15:49:56 +1000
commit96eda14b7a1caff77da000a9a957b20ab1a1bb6e (patch)
treeceb91518776a8b36af8f08c155074fb77f5a437a
parentReduce default logging level to INFO. (diff)
downloadopensim-SC-96eda14b7a1caff77da000a9a957b20ab1a1bb6e.zip
opensim-SC-96eda14b7a1caff77da000a9a957b20ab1a1bb6e.tar.gz
opensim-SC-96eda14b7a1caff77da000a9a957b20ab1a1bb6e.tar.bz2
opensim-SC-96eda14b7a1caff77da000a9a957b20ab1a1bb6e.tar.xz
Lots of hacking up the scripts to work with the new setup, and more.
Diffstat (limited to '')
-rw-r--r--config/Robust.ini.template (renamed from config/Robust.ini)48
-rwxr-xr-xscripts/backup-inventories.sh42
l---------scripts/backup-inventory1
-rwxr-xr-xscripts/backup-sims.sh30
-rwxr-xr-xscripts/fix_var_run.sh6
-rwxr-xr-xscripts/gitAR.sh118
-rw-r--r--scripts/opensim-crontab.txt16
-rw-r--r--scripts/opensim-monit.conf36
-rw-r--r--scripts/opensim.tmux.conf46
-rwxr-xr-xscripts/show-console3
-rwxr-xr-xscripts/start-sim152
11 files changed, 450 insertions, 48 deletions
diff --git a/config/Robust.ini b/config/Robust.ini.template
index 9926ed7..1bbccd5 100644
--- a/config/Robust.ini
+++ b/config/Robust.ini.template
@@ -9,55 +9,7 @@
9; * Additional OpenSim.Server.Handlers.dll:AssetServiceConnector and 9; * Additional OpenSim.Server.Handlers.dll:AssetServiceConnector and
10; * OpenSim.Server.Handlers.dll:XInventoryInConnector 10; * OpenSim.Server.Handlers.dll:XInventoryInConnector
11; * are started in port 8002, outside the firewall 11; * are started in port 8002, outside the firewall
12; *
13; **
14; *
15; * The Const section allows us to define some basic information that we
16; * will use throughout our configuration. We will provide examples for
17; * setting the base url of the Robust server and the public and private ports
18; * it uses. Changing the values of the constants will set the operating
19; * parameters thoughout the configuration. Other constants that may prove
20; * to be useful may be added to the followin section. They may be
21; * referenced anywhere in the configuration by using ${Const|Name}. One
22; * such use is providing a base path for setting locations that Robust
23; * uses to write data.
24; *
25
26[Includes]
27 ; Define your server specific constants in this file.
28; Include-Common = /opt/opensim/config/constants.ini
29
30
31[Const]
32 MOTD = "Welcome to virtual 'Gabba."
33
34 Base = "../.."
35 AssetsPath = "../../AssetFiles"
36 CachePath = "../../caches"
37 ConfigPath = "../../config"
38 IncludePath = "../../config-include"
39
40 GridName = "onefangWorld"
41
42 ; For a grid these will usually be the externally accessible IP/DNS
43 ; name and use default public port 8002 and default private port 8003
44 ; For a standalone this will usually be the externally accessible IP/DNS
45 ; name and use default public port 9000. The private port is not used
46 ; in the configuration for a standalone.
47
48 ;# {BaseURL} {} {BaseURL} {"http://example.com" "http://127.0.0.1"} "http://127.0.0.1"
49 HostName = "localhost"
50 BaseURL = http://127.0.0.1
51
52 ;# {PublicPort} {} {PublicPort} {8002 9000} "8002"
53 PublicPort = "8002"
54
55 ;# {PrivatePort} {} {PrivatePort} {8003} "8003"
56 PrivatePort = "8003"
57
58 12
59 DataProvider = "OpenSim.Data.MySQL.dll"
60 ConnectionString = "Data Source=localhost;Database=$MYSQL_DB;User ID=$MYSQL_USER;Password=$MYSQL_PASSWORD;Old Guids=true;"
61 13
62; * The startup section lists all the connectors to start up in this server 14; * The startup section lists all the connectors to start up in this server
63; * instance. This may be only one, or it may be the entire server suite. 15; * instance. This may be only one, or it may be the entire server suite.
diff --git a/scripts/backup-inventories.sh b/scripts/backup-inventories.sh
new file mode 100755
index 0000000..642d71d
--- /dev/null
+++ b/scripts/backup-inventories.sh
@@ -0,0 +1,42 @@
1#!/bin/bash
2
3# Figure out where we are, most of this mess is to troll through soft links.
4PRG="$0"
5while [ -h "${PRG}" ] ; do
6 ls=$(ls -ld "${PRG}")
7 link=`expr "${ls}" : '.*-> \(.*\)$'`
8 if expr "${link}" : '.*/.*' > /dev/null; then
9 PRG="${link}"
10 else
11 PRG=$(dirname "${PRG}")/"${link}"
12 fi
13done
14PRGDIR=$(dirname "${PRG}")
15pushd ${PRGDIR} >/dev/null
16PRGDIR=$(pwd)
17popd >/dev/null
18
19# Get the database credentials.
20declare -A creds
21while read -d ';' p; do
22 k=$(echo ${p} | cut -d '=' -f 1)
23 v=$(echo ${p} | cut -d '=' -f 2)
24 creds[${k}]="${v}"
25done < <(grep ConnectionString ${PRGDIR}/../config/config.ini | cut -d '"' -f 2)
26# The above seems the best way to get bash to let the creds assignments survive outside the loop.
27
28# Only backup those that have not logged on since their last backup, but returning prims from sims will bypass this check.
29timestamp=$(ls -o --time-style="+%s" ${PRGDIR}/../backups/.keep | cut -d ' ' -f 5)
30touch ${PRGDIR}/../backups/.keep
31
32# Get the user names, and back 'em up.
33mysql --host="${creds[Data Source]}" "${creds[Database]}" --user="${creds[User ID]}" --password="${creds[Password]}" \
34 -e "select FirstName,LastName from UserAccounts,GridUser where UserAccounts.PrincipalID=GridUser.UserID and GridUser.Logout>${timestamp};" -ss | while read user; do
35 # Replace tab with space
36 user=${user// / }
37 ${PRGDIR}/backup-inventory "${user}"
38 # Sleep for a while, so that there is plenty of time to do the backup,
39 # and we are not keeping the computer very busy if there are lots of users.
40 # My big arsed 1 GB OAR takes about ten minutes to create, and maybe an hour to gitIOR!
41 sleep 200
42done
diff --git a/scripts/backup-inventory b/scripts/backup-inventory
new file mode 120000
index 0000000..d326a74
--- /dev/null
+++ b/scripts/backup-inventory
@@ -0,0 +1 @@
start-sim \ No newline at end of file
diff --git a/scripts/backup-sims.sh b/scripts/backup-sims.sh
new file mode 100755
index 0000000..fb18c18
--- /dev/null
+++ b/scripts/backup-sims.sh
@@ -0,0 +1,30 @@
1#!/bin/bash
2
3# Figure out where we are, most of this mess is to troll through soft links.
4PRG="$0"
5while [ -h "${PRG}" ] ; do
6 ls=$(ls -ld "${PRG}")
7 link=`expr "${ls}" : '.*-> \(.*\)$'`
8 if expr "${link}" : '.*/.*' > /dev/null; then
9 PRG="${link}"
10 else
11 PRG=$(dirname "${PRG}")/"${link}"
12 fi
13done
14PRGDIR=$(dirname "${PRG}")
15pushd ${PRGDIR} >/dev/null
16PRGDIR=$(pwd)
17popd >/dev/null
18
19for i in $(seq 99)
20do
21 j=$(printf "sim%02d" $i)
22 if [ -e "${PRGDIR}/../config/$j" ]
23 then
24 cd ${PRGDIR}/../config/$j
25 ./backup-sim
26 # Sleep for a while, so that there is plenty of time to do the backup,
27 # and we are not keeping the computer very busy if there are lots of sims.
28 sleep 200
29 fi
30done
diff --git a/scripts/fix_var_run.sh b/scripts/fix_var_run.sh
new file mode 100755
index 0000000..97567cb
--- /dev/null
+++ b/scripts/fix_var_run.sh
@@ -0,0 +1,6 @@
1# Create the opensim run dir.
2mkdir -p /var/run/opensim
3chown opensim:opensim /var/run/opensim
4chmod ug+rwx /var/run/opensim
5chmod o-rwx /var/run/opensim
6chmod g+s /var/run/opensim
diff --git a/scripts/gitAR.sh b/scripts/gitAR.sh
new file mode 100755
index 0000000..2a067be
--- /dev/null
+++ b/scripts/gitAR.sh
@@ -0,0 +1,118 @@
1#!/bin/bash
2
3# Work around OpenSims slow database corruption bug by using git to store all old backups.
4# Try to squeeze every last byte out of the tarballs. Seems to cut the total storage size down to one third the size of just the raw I/OAR files.
5# Saves even more if there's been no changes.
6# On the other hand, these backup files will grow indefinately, the more changes, the faster it grows. I can live with that for more reliable backups that go back further.
7# Tries to avoid loosing data if things go wrong. I think the main remaining problem would be running out of space, in which case you have bigger problems to deal with.
8
9# Strategy - unpack the last one, unpack and commit any old I/OARs, pack up the result, delete it's working directory, THEN run the save i/oar.
10# Avoids having to sync with OpenSim finishing the current I/OAR, and as a bonus, an easy to deliver latest I/OAR for people that want it.
11
12# Not really meant to be called by users, so don't bother validating the input and such.
13
14type=$1
15title=$2
16
17# Figure out where we are, most of this mess is to troll through soft links.
18PRG="$0"
19while [ -h "${PRG}" ] ; do
20 ls=$(ls -ld "${PRG}")
21 link=`expr "${ls}" : '.*-> \(.*\)$'`
22 if expr "${link}" : '.*/.*' > /dev/null; then
23 PRG="${link}"
24 else
25 PRG=$(dirname "${PRG}")/"${link}"
26 fi
27done
28PRGDIR=$(dirname "${PRG}")
29pushd ${PRGDIR} >/dev/null
30PRGDIR=$(pwd)
31popd >/dev/null
32
33date=$(date '+%F_%T')
34
35# Sanitize the name. Not removing [ or ], couldn't get that to work, only important for Windows.
36name=$(echo "${title}" | sed -e 's/[\\/:\*\?"<>\|@#$%&\0\x01-\x1F\x27\x40\x60\x7F. ]/_/g' -e 's/^$/NONAME/')
37# Convert the type to uppercase.
38gar="_git$(echo -n ${type} | tr '[:lower:]' '[:upper:]')AR"
39
40if [ -d ${PRGDIR}/../../backups/temp_backup${type}_${name} ]; then
41 echo "WARNING - Mess left over from last backup, not gonna run!"
42 mv ${PRGDIR}/../../backups/temp_backup${type}_${name}/*.oar ${PRGDIR}/../backups
43 exit 1
44fi
45
46mkdir -p ${PRGDIR}/../../backups/temp_backup${type}_${name}
47pushd ${PRGDIR}/../../backups/temp_backup${type}_${name} >/dev/null
48if [ -f ../${name}${gar}.tar.xz ]; then
49 nice -n 19 tar -xf ../${name}${gar}.tar.xz
50else
51 mkdir -p ${name}${gar}
52 git init ${name}${gar} >log
53fi
54
55pushd ${name}${gar} >/dev/null
56
57# Make sure stuff that's already compressed doesn't get compressed by git.
58# Also tries to protect binaries from mangling.
59cat >.gitattributes <<- zzzzEOFzzzz
60*.bvh -delta -diff -text
61*.jp2 -delta -diff -text
62*.jpg -delta -diff -text
63*.llmesh -delta -diff -text
64*.ogg -delta -diff -text
65*.png -delta -diff -text
66*.r32 -delta -diff -text
67*.tga -delta -diff -text
68zzzzEOFzzzz
69git add .gitattributes &>>../log
70# Coz git insists.
71git config user.email "opensim@$(hostname -A | cut -d ' ' -f 1)"
72git config user.name "opensim"
73
74# Looping through them in case there's a bunch of I/OARs from previous versions of this script.
75find ../.. -maxdepth 1 -type f -name ${name}-*.${type}ar | sort | while read file; do
76 # Deal with deletions in the inventory / sim, easy method, which becomes a nop for files that stay in the git add below.
77 git rm -fr * &>>../log && \
78 nice -n 19 tar -xzf "${file}" || echo "ERROR - Could not unpack ${file} !" >>../errors
79 if [ ! -f ../errors ]; then
80 git add * &>>../log && git add */\* &>>../log
81 # Magic needed to figure out if there's anything to commit.
82 # After all the pain to get this to work, there's an ever changing timestamp in archive.xml that screws it up.
83 # Like this system didn't have enough timestamps in it already. lol
84 # TODO - I could sed out that timestamp, and put it back again based on the OAR file name when extracting.
85 # IARs don't seem to have the timestamp.
86 if t=$(git status --porcelain) && [ -z "${t}" ]; then
87 true
88 else
89 # Note this commit message has to be just the file name, as the ungitAR script uses it.
90 git commit -qm "$(basename ${file})" &>>../log || echo "ERROR - Could not commit ${file} !" >>../errors
91 fi
92 if [ ! -f ../errors ]; then
93 mv ${file} ..
94 fi
95 fi
96 if [ -f ../errors ]; then
97 exit 1 # Seems to only exit from this loop, not the script. Makes me want to rewrite this in a real language. lol
98 fi
99done
100
101#git gc --aggressive --prune=now # Takes a long time, doesn't gain much. Even worse, it increases the size of the resulting tarball. lol
102
103popd >/dev/null
104
105if [ ! -f errors ]; then
106 XZ_OPT="-9e" nice -n 19 tar -c --xz ${name}${gar} -f ../${name}${gar}.tar.xz || echo "ERROR - Could not pack gitAR!" >>errors
107fi
108
109popd >/dev/null
110
111if [ -f ${PRGDIR}/../../backups/temp_backup${type}_${name}/errors ]; then
112 echo "NOT cleaning up coz - "
113 cat ${PRGDIR}/../../backups/temp_backup${type}_${name}/errors
114else
115 rm -fr ${PRGDIR}/../../backups/temp_backup${type}_${name}
116fi
117
118echo -n ${name}
diff --git a/scripts/opensim-crontab.txt b/scripts/opensim-crontab.txt
new file mode 100644
index 0000000..f26b25d
--- /dev/null
+++ b/scripts/opensim-crontab.txt
@@ -0,0 +1,16 @@
1# to see current crontab for opensim
2# $ sudo crontab -u opensim -l
3#
4# to install this crontab for opensim:
5# $ cat /opt/opensim/current/scripts/opensim-crontab.txt | sudo crontab -u opensim -
6#
7# at midnight, remove old logs, files created 3 or more days ago
8#0 0 * * * find /var/log/opensim -ctime +1 -delete
9# experimental version using savelog -c cycles all opensim log files over 7 days
100 0 * * * /usr/bin/savelog -c 7 /var/log/opensim/*.log > /dev/null
11
12# Backup the inventories and sims every 6 hours.
130 */6 * * * /opt/opensim/current/scripts/backup-sims.sh
14
15# Only backup inventories on the grid server itself.
16#55 */6 * * * /opt/opensim/current/scripts/backup-inventories.sh
diff --git a/scripts/opensim-monit.conf b/scripts/opensim-monit.conf
new file mode 100644
index 0000000..7785056
--- /dev/null
+++ b/scripts/opensim-monit.conf
@@ -0,0 +1,36 @@
1# manage the OpenSim process for Your Sim
2#
3# usage:
4# monit start your_sim
5# monit stop your_sim
6# monit restart your_sim
7#
8# see 'daemon' setting in /etc/monit/monitrc for the cycle length.
9# on ubuntu/debian, this is overridden by the CHECK_INTERVALS var in
10# /etc/default/monit . the below assumes you've set it to 30 seconds.
11#
12# if process dies, will restart sim within 30 seconds. if process
13# dies 5 times in as many tries, will stop trying and send email
14# alert.
15#
16# if SimFPS drops to 0 for 2 minutes, restart.
17#
18# if process CPU usage stays above 300% for 2 minutes, restart.
19#
20# see ../README for configuration instructions.
21#
22# Hmmmm, seems that monit changing to a user and running bash -l is different from sudo changing to a user and running bash -l.
23#
24check process sim01 with pidfile /var/run/opensim/sim01.pid
25 start program = "/usr/bin/sudo -Hu opensim /bin/bash -lc 'cd /opt/opensim/config/sim01 && /opt/opensim/config/sim01/start-sim -q'"
26 as uid root and gid root
27 stop program = "/usr/bin/sudo -Hu opensim /bin/bash -lc 'cd /opt/opensim/config/sim01 && /opt/opensim/config/sim01/stop-sim'" with timeout 600 seconds
28 if cpu usage > 50% for 4 cycles then restart
29# if 5 restarts within 5 cycles then timeout
30# if failed url http://127.0.0.1:9005/jsonSimStats/
31# and content != '"SimFPS":0.0,' for 4 cycles
32# then restart
33# if failed url http://127.0.0.1:9008/jsonSimStats/
34# and content == '"SimFPS":' for 4 cycles
35# then restart
36
diff --git a/scripts/opensim.tmux.conf b/scripts/opensim.tmux.conf
new file mode 100644
index 0000000..ecd0b04
--- /dev/null
+++ b/scripts/opensim.tmux.conf
@@ -0,0 +1,46 @@
1set-option -g prefix C-a
2unbind-key C-b
3bind-key C-a send-prefix
4
5# r reloads the configuration, handy
6bind r source-file ~/.tmux.conf
7
8unbind % # Remove default binding since we’re replacing them.
9bind | split-window -h
10bind - split-window -v
11
12set-option -g bell-action any
13set-option -g bell-on-alert on
14
15set-option -g history-limit 100000
16
17set-option -g mouse-resize-pane on
18set-option -g mouse-select-pane on
19set-option -g mouse-select-window on
20set-option -g mouse-utf8 on
21
22#set-option -g set-remain-on-exit on
23set-option -g set-titles on
24set-option -g set-titles-string '#W'
25
26set-option -g status-interval 1
27set-option -g status-justify centre
28set-option -g status-utf8 on
29
30set-option -g status-left-length 80
31set-option -g status-left '[#H #S #F]'
32set-option -g status-right-length 80
33set-option -g status-right '%F #(uptime)'<-----># &F is ISO date, uptime starts with the current time, and ends with the load average. B-)
34
35set-option -g visual-activity on
36set-option -g visual-content on
37set-option -g visual-silence on
38
39set-option -gw clock-mode-style 24
40set-option -gw mode-mouse on
41
42# Highlight active window
43set-option -gw window-status-current-bg red
44
45# Set window notifications
46set-option -gw monitor-activity on
diff --git a/scripts/show-console b/scripts/show-console
new file mode 100755
index 0000000..ddeba86
--- /dev/null
+++ b/scripts/show-console
@@ -0,0 +1,3 @@
1#!/bin/bash
2
3tmux -S /var/run/opensim/opensim-tmux.socket select-window -t "OpenSim_console" \; attach-session -t "OpenSim_console"
diff --git a/scripts/start-sim b/scripts/start-sim
new file mode 100755
index 0000000..cd28b43
--- /dev/null
+++ b/scripts/start-sim
@@ -0,0 +1,152 @@
1#!/bin/bash
2
3# Figure out where we are, most of this mess is to troll through soft links.
4PRG="$0"
5while [ -h "${PRG}" ] ; do
6 ls=$(ls -ld "${PRG}")
7 link=`expr "${ls}" : '.*-> \(.*\)$'`
8 if expr "${link}" : '.*/.*' > /dev/null; then
9 PRG="${link}"
10 else
11 PRG=$(dirname "${PRG}")/"${link}"
12 fi
13done
14PRGDIR=$(dirname "${PRG}")
15pushd ${PRGDIR} >/dev/null
16PRGDIR=$(pwd)
17popd >/dev/null
18
19
20USER=$(whoami)
21console_name="OpenSim_console"
22tmux_command="tmux -S /var/run/opensim/opensim-tmux.socket"
23tmux_session=${console_name}
24tmux_window="${tmux_command} select-window -t ${tmux_session}"
25tmux_send="${tmux_command} send-keys -t ${tmux_session}"
26bindir="${PRGDIR}/../bin"
27date=$(date '+%F_%T')
28quiet=""
29inventory=""
30
31if [ $USER = "opensim" ]
32then
33 SUDO=""
34else
35 SUDO="sudo -Hu opensim"
36fi
37
38if [ "$1" = "-q" ]
39then
40 quiet="true"
41 shift 1
42fi
43
44if [ "x$1" = "x" ]; then
45 tgt=$(basename $(pwd))
46 num=$(echo ${tgt} | cut -c 4-)
47elif [ -d "./$1" ]; then
48 tgt=$1
49elif [ -d "./sim$1" ]; then
50 num=$1
51 tgt="./sim${num}"
52else
53 tgt=$1
54 inventory=$1
55fi
56
57if [ "x$tgt" = "x" ]; then
58 echo "usage:"
59 echo " $ $(basename $0) <sim>"
60 echo "where <sim> is one of: " robust sim[0-9][0-9]
61 exit 1
62fi
63
64cd ${bindir}
65
66if ( ${tmux_command} -q list-sessions 2>/dev/null | grep -q ${console_name}: ); then
67 true
68else
69 # The sudo is only so that the session is owned by opensim, otherwise it's owned by whoever ran this script, which is a likely security hole.
70 # After the session is created, we rely on the /var/run/opensim directory to be group sticky, so that anyone in the opensim group can attach to the tmux socket.
71 $SUDO ${tmux_command} new-session -d -s ${console_name} -n "Server"
72fi
73
74
75if [ "x$tgt" = "xROBUST" ]; then
76 exe="Robust"
77 title="ROBUST"
78elif [ "x$inventory" = "x" ]; then
79 exe="OpenSim"
80 # Grab the first Section line of the sims .xml file, cut it down to the name.
81 title=$(grep "<Section " ../config/${tgt}/*.xml | head -n 1 | cut -d '"' -f 2 )
82fi
83
84case $(basename $0) in
85 "start-sim")
86 cmd="mono ${exe}.exe -inidirectory=${tgt} "
87
88 # Check if it's already running.
89 if [ -e /var/run/opensim/${tgt}.pid ]
90 then
91 # Double check if it's REALLY running.
92 if [ "x$(ps -p $(cat /var/run/opensim/${tgt}.pid) --no-headers -o comm)" = "x" ]; then
93 $SUDO rm -f /var/run/opensim/${tgt}.pid
94 fi
95 fi
96 # Now see if it's really really running. lol
97 if [ ! -e /var/run/opensim/${tgt}.pid ]
98 then
99 if [ "x$tgt" = "xROBUST" ]; then
100 $SUDO cat ../config/config.ini ../config/Robust.ini.template >../config/Robust.ini
101 ${tmux_command} split-window -vp 30 -t "${tmux_session}:" "${cmd}"
102 else
103 ${tmux_command} new-window -dn "[${title}]" -t "${tmux_session}:${num}" "${cmd}"
104 fi
105 fi
106
107 if [ "x$quiet" = "x" ]
108 then
109 if [ "x$tgt" = "xROBUST" ]; then
110 ${tmux_window} \; attach-session -t "${tmux_session}"
111 fi
112 fi
113 ;;
114
115 "backup-inventory")
116 user=$($SUDO ${PRGDIR}/gitAR.sh i "${inventory}")
117 # Add the full date and time to create the IAR file name.
118 cmd="save iar -c ${inventory} / password ${PRGDIR}/../../backups/${user}-${date}.iar"
119 # Do it in the highest numbered window.
120 ${tmux_send}:"$" "${cmd}" Enter
121 ${tmux_send}:"$" "force gc" Enter
122 ;;
123
124 "backup-sim")
125 sim=$($SUDO ${PRGDIR}/gitAR.sh o "${title}")
126 # Add the full date and time to create the OAR file name.
127 cmd="save oar --all ${PRGDIR}/../../backups/${sim}-${date}.oar"
128 if [ -e /var/run/opensim/${tgt}.pid ]; then
129 ${tmux_send}:"[${title}]" "${cmd}" Enter
130 # Wait a bit, then generate the map tiles, coz the good generator leaks memory badly if you leave it turned on.
131 sleep 30
132 ${tmux_send}:"[${title}]" "generate map" Enter
133 ${tmux_send}:"[${title}]" "force gc" Enter
134 else
135 echo "No OAR created for ${title}, it's not running."
136 fi
137 ;;
138
139 "stop-sim")
140 if [ -e /var/run/opensim/${tgt}.pid ]; then
141 ${tmux_send}:"[${title}]" "alert WARNING - Emergency shutdown in one minute!" Enter
142 ${tmux_send}:"[${title}]" "alert WARNING - Emergency shutdown in one minute!" Enter
143 sleep 30
144 ${tmux_send}:"[${title}]" "alert WARNING - Emergency shutdown in thirty seconds!" Enter
145 ${tmux_send}:"[${title}]" "alert WARNING - Emergency shutdown in thirty seconds!" Enter
146 sleep 30
147 ${tmux_send}:"[${title}]" "shutdown" Enter
148 else
149 echo "Sim ${title} is not running, so not stopping."
150 fi
151 ;;
152esac