1 files changed, 325 insertions, 471 deletions

diff --git a/src/sledjchisl/sledjchisl.c b/src/sledjchisl/sledjchisl.c
index c068d57..12c03cd 100644
--- a/src/sledjchisl/sledjchisl.c
+++ b/src/sledjchisl/sledjchisl.c
@@ -1090,6 +1090,10 @@ struct _simList
   int len, num;
   char **sims;
   qtreetbl_t *byTab, *simsLua;
+  // Stuff for the looping through sims doing things and waiting.
+  char *target;
+  float  la;
+  int doWait;
 };
 simList *ourSims = NULL;
 
@@ -1435,6 +1439,305 @@ int checkSimIsRunning(char *sim)
   return ret;
 }
 
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// Sim wrangling loop.
+////////////////////////////////////////////////////////////////////////////////////////////////////
+typedef void (*simFunction)(simData *simd, char *type, int count, int window, int panes, int pane);
+void forEachSim(char *verb, simFunction func)
+{
+  qtreetbl_obj_t obj0, obj1;
+  qLua *q0, *q1;
+  int count = 0, window = 0, panes = 4, pane = 0;
+
+  ourSims->doWait = 1;
+  memset((void*)&obj0, 0, sizeof(obj0));
+  ourSims->simsLua->lock(ourSims->simsLua);
+  while(ourSims->simsLua->getnext(ourSims->simsLua, &obj0, false) == true)
+  {
+    q0 = obj0.data;
+    char *type = "unsorted";
+
+    if (LUA_TTABLE == q0->type)
+    {
+      panes = 4;
+      q1 = q0->v.t->get(q0->v.t, "number",      NULL, false);  if (NULL != q1)  window  = q1->v.f - 1;
+      q1 = q0->v.t->get(q0->v.t, "panes",       NULL, false);  if (NULL != q1)  panes   = q1->v.f;
+      q1 = q0->v.t->get(q0->v.t, "type",        NULL, false);  if (NULL != q1)  type    = q1->v.s;
+      if (0 == panes)
+      {
+        pane = 2;
+        window = 0;
+        type = Ttab;
+      }
+      else if (0 != pane)
+      {
+        pane = 0;
+        window++;
+      }
+
+      if (verb)
+        V("%s sims of type %s, window %d, %d panes per window.", verb, type, window, panes);
+      memset((void*)&obj1, 0, sizeof(obj1));
+      q0->v.t->lock(q0->v.t);
+      while(q0->v.t->getnext(q0->v.t, &obj1, false) == true)
+      {
+        q1 = obj1.data;
+
+        if ((strcmp("number", obj1.name) != 0) && (strcmp("panes", obj1.name) != 0) && (strcmp("type", obj1.name) != 0))
+        {
+          simData *simd = ourSims->byTab->get(ourSims->byTab, q1->v.s, NULL, false);
+
+          if (NULL == simd)
+            E("Sim %s not found in ini list!", q1->v.s);
+          else
+          {
+            func(simd, type, count, window, panes, pane);
+            count++;
+            pane++;
+            if (pane >= panes)
+            {
+              pane = 0;
+              window++;
+            }
+          }
+        }
+      }
+      q0->v.t->unlock(q0->v.t);
+    }
+  }
+  ourSims->simsLua->unlock(ourSims->simsLua);
+}
+
+void prepSims(simData *simd, char *type, int count, int window, int panes, int pane)
+{
+  char *path = xmprintf("%s/%s.shini", scEtc, simd->tab);
+  char *newPath = xmprintf("%s/sim%d/%s.ini", scTemp, count, simd->tab);        // Coz OpenSim sucks.
+  char *cmd;
+
+  simd->window = window;
+  simd->pane = pane;
+
+  // Make sure all tmux windows and panes are startned.
+  // The bash invocations are so that the PATH is properly propagated.
+  if (0 == panes)
+    doTmuxCmd("split-window -hp 50 -d -t '%s:0.1' bash", Tconsole);
+  else if (0 == pane)
+  {
+    doTmuxCmd("new-window -dc '%s/current/bin' -n '%s' -t '%s:%d' bash", scRoot, type, Tconsole, window);
+    if (2 <= panes)
+      doTmuxCmd("split-window -hp 50 -d -t '%s:%d.0' bash", Tconsole, window);
+    if (4 <= panes)
+    {
+      doTmuxCmd("split-window -vp 50 -d -t '%s:%d.1' bash", Tconsole, window);
+      doTmuxCmd("split-window -vp 50 -d -t '%s:%d.0' bash", Tconsole, window);
+    }
+    if (8 <= panes)
+    {
+      doTmuxCmd("split-window -hp 50 -d -t '%s:%d.3' bash", Tconsole, window);
+      doTmuxCmd("split-window -hp 50 -d -t '%s:%d.2' bash", Tconsole, window);
+      doTmuxCmd("split-window -hp 50 -d -t '%s:%d.1' bash", Tconsole, window);
+      doTmuxCmd("split-window -hp 50 -d -t '%s:%d.0' bash", Tconsole, window);
+    }
+  }
+
+  // Create the temporary .ini files.
+  snprintf(toybuf, sizeof(toybuf), "echo 'IDs={' >%s/IDs_%d.lua", scTemp, window);
+  system(toybuf);
+  doTmuxCmd("list-panes -t %d -F \"['%d #{pane_index}'] = '#{window_id}.#{pane_id}',\" >> %s/IDs_%d.lua", window, window, scTemp, window);
+  snprintf(toybuf, sizeof(toybuf), "echo '}\nreturn IDs' >>%s/IDs_%d.lua", scTemp, window);
+  system(toybuf);
+
+  snprintf(toybuf, sizeof(toybuf), "%s/IDs_%d.lua", scTemp, window);
+  qtreetbl_t *IDs = Lua2tree(toybuf, "IDs");
+  snprintf(toybuf, sizeof(toybuf), "%d %d", window, pane);
+  simd->paneID = xmprintf("%s", qLuaGet(IDs, toybuf)->v.s);
+  freeLuaTree(IDs);
+
+  simd->portH = 8004 + count * 2;
+  simd->portU = 8005 + count * 2;
+  cmd = xmprintf("rm -fr %s/sim%d; mkdir -p %s/sim%d; sed -E"
+    " -e 's@\\[Region\\]@"
+    "    paneID = \"\\%s\"\\n"
+    "\\[Startup\\]\\n"
+    "    PIDFile = \"\\$\\{Paths\\|CachePath\\}\\/\\$\\{Const\\|mysim\\}\\.pid\"\\n"
+    "    LogFile = \"\\$\\{Paths\\|LogPath\\}\\/OpenSim_\\$\\{Const\\|mysim\\}\\.log\"\\n"
+    "    StatsLogFile = \"\\$\\{Paths\\|LogPath\\}\\/OpenSimStats_\\$\\{Const\\|mysim\\}\\.log\"\\n"
+    "    ConsoleHistoryFile = \"\\$\\{Paths\\|LogPath\\}\\/OpenSimConsoleHistory_\\$\\{Const\\|mysim\\}\\.txt\"\\n"
+    "\\n\\[Region\\]@'"
+    " -e 's/InternalPort[[:space:]]*=[[:space:]]*[[:digit:]]+/InternalPort = %d/'"
+    " -e 's/InternalPort[[:space:]]*=[[:space:]]*\"[[:digit:]]+\"/InternalPort = %d/'"
+    " -e 's/http_listener_port[[:space:]]*=[[:space:]]*[[:digit:]]+/http_listener_port = %d/'"
+    " -e 's/http_listener_port[[:space:]]*=[[:space:]]*\"[[:digit:]]+\"/http_listener_port = %d/'"
+    " %s >%s",
+    scTemp, count,
+    scTemp, count,
+    simd->paneID,
+    simd->portU,
+    simd->portU,
+    simd->portH,
+    simd->portH,
+    path, newPath);
+  D("Writing .ini file %s with ports %d %d", newPath, simd->portH, simd->portU);
+  if (!WIFEXITED(system(cmd)))
+    E("sed command failed!");
+  free(cmd);
+  free(newPath);
+  free(path);
+}
+
+// Figure out where the sims are running.
+void findSimsTmux(simData *simd, char *type, int count, int window, int panes, int pane)
+{
+  simd->window = window;
+  simd->pane = pane;
+  snprintf(toybuf, sizeof(toybuf), "%s/IDs_%d.lua", scTemp, window);
+  qtreetbl_t *IDs = Lua2tree(toybuf, "IDs");
+  snprintf(toybuf, sizeof(toybuf), "%d %d", window, pane);
+  simd->paneID = xmprintf("%s", qLuaGet(IDs, toybuf)->v.s);
+  freeLuaTree(IDs);
+}
+
+void doSimsThing(simData *simd, char *type, int count, int window, int panes, int pane)
+{
+  // Check if only doing a single sim.
+  int cont = FALSE;
+  if (NULL != ourSims->target)
+  {
+    cont = TRUE;
+    snprintf(toybuf, sizeof(toybuf), "%s.shini", simd->tab);
+    if
+    (
+      (strcmp(ourSims->target, simd->name) == 0) ||
+      (strcmp(ourSims->target, simd->tab) == 0)   ||
+      (strcmp(ourSims->target, toybuf) == 0)
+    )
+    cont = FALSE;
+    snprintf(toybuf, sizeof(toybuf), "%s.ini", simd->tab);
+    if (strcmp(ourSims->target, toybuf) == 0)
+      cont = FALSE;
+    snprintf(toybuf, sizeof(toybuf), "sim%d.ini", count);
+    if (strcmp(ourSims->target, toybuf) == 0)
+      cont = FALSE;
+    if (cont)
+      return;
+  }
+
+  switch (currentMode)
+  {
+    case START :        // "start sim01"  "start 'Welcome sim'" "start Welcome.ini" "start Welcome"  "start" start everything
+    {
+      if (!checkSimIsRunning(simd->tab))
+      {
+        I("Tmux tab [%d:%s](pane %d) tmux ID %s, from %s/sim%d - %s is starting.", window, type, pane, simd->paneID, scTemp, count, simd->name);
+        doTmuxCmd("select-pane -t %s:%s -T '%s'", Tconsole, simd->paneID, simd->tab);
+        snprintf(toybuf, sizeof(toybuf), "mono OpenSim.exe -inidirectory=%s/sim%d", scTemp, count);
+        sendTmuxCmd(simd->paneID, toybuf);
+        if (0 == ourSims->doWait)
+        {
+          snprintf(toybuf, sizeof(toybuf), "INITIALIZATION COMPLETE FOR %s", simd->name);
+          waitTmuxText(simd->paneID, toybuf);
+          I("%s is done starting up.", simd->name);
+          ourSims->la = waitLoadAverage(ourSims->la, loadAverageInc, simTimeOut);
+          if (1 < bulkSims)
+            bulkSims = bulkSims / 2;
+        }
+        else
+        {
+// TODO - some compromise, do a premptive load check if we are not doing a wait anyway.
+        }
+        ourSims->doWait = ((count + 1) % bulkSims);
+      }
+      break;
+    }
+
+    case BACKUP :       // "backup 'onefang rejected'"  "backup sim01"  "backup 'Welcome Sim'"  "backup" backup everything
+    {           // TODO - If it's not a sim code, and not a sim name, it's an account inventory.
+      if (checkSimIsRunning(simd->tab))
+      {
+        struct timeval tv;
+        time_t curtime;
+        char date[DATE_TIME_LEN];
+
+// TODO - should collect names of existing backups, both tmux / ini name and proper name.
+//          Scan backups directory once before this for loop, add details to sims list.
+//            strip off the last bit of file name (YYYY-mm-dd_HH:MM:SS.oar) to get the name
+//              keep in mind some old files had munged names like "Tiffanie_s_Paradise-2021-06-23_05:11:38.oar"
+        gettimeofday(&tv, NULL);
+        curtime = tv.tv_sec;
+        strftime(date, DATE_TIME_LEN, "%F_%T", localtime(&curtime));
+        I("%s is being backed up to %s/backups/%s-%s.oar.", simd->name, scRoot, simd->tab, date);
+        snprintf(toybuf, sizeof(toybuf), "save oar --all %s/backups/%s-%s.oar", scRoot, simd->tab, date);
+        sendTmuxCmd(simd->paneID, toybuf);
+//      if (0 == doWait)
+        {
+          memset(toybuf, 0, sizeof(toybuf));
+          snprintf(toybuf, sizeof(toybuf), "Finished writing out OAR for %s", simd->name);
+          waitTmuxText(simd->paneID, toybuf);
+          I("%s is done backing up.", simd->name);
+// TODO - should delete / gitAR the old ones now.
+//          Have a config option for delete / gitAR.
+          ourSims->la = waitLoadAverage(ourSims->la, loadAverageInc, simTimeOut);
+        }
+      }
+      break;
+    }
+
+    case CREATE :       // "create name x,y size"
+    {
+      break;
+    }
+
+    case GITAR :        // "gitAR i name"  "gitAR o name"
+    {
+      break;
+    }
+
+    case STATUS :
+    {
+      break;
+    }
+
+    case STOP : // "stop sim01"  "stop 'Welcome Sim'"  "stop" stop everything
+    {
+      if (checkSimIsRunning(simd->tab))
+      {
+        // Leave empty panes as is.
+        sendTmuxCmd(simd->paneID, "quit");
+        // There's three things that might happen -
+        //   Sim will quit, tmux pane will go away before we can see the shutdown message.
+        //      pane-exited hook might be useful here.
+        //   Sim will quit, tmux pane wont go away.  Dunno yet if waitTmuxText() will still work.
+        //      pane-died hook might be useful here.
+        //   Sim fails to quit, error message on the pane that we want to read, wont see the shutdown message.
+        // Also sim might not have finished starting up, and may even not be accepting comands yet.
+// TODO - cater for and test them all.
+//         Could also loop on checkSimIsRunning(sim)
+//      snprintf(toybuf, sizeof(toybuf), "[SHUTDOWN]: Shutdown processing on main thread complete.  Exiting...");
+//      waitTmuxText(nm, toybuf);
+//      I("%s is done stopping.", simd->name);
+//      la = waitLoadAverage(la, loadAverageInc, simTimeOut);
+      }
+      break;
+     }
+
+    default :
+    {
+      E("Unknown command! %s", toys.optargs[0]);
+      break;
+    }
+  }
+}
+
+void stopSims(simData *simd, char *type, int count, int window, int panes, int pane)
+{
+  // NOTE - these sleeps are guesses, seems to work on my super desktop during testing.
+  while (checkSimIsRunning(simd->tab))
+    usleep(100000);
+  usleep(10000);
+  doTmuxCmd("kill-pane -t %s", simd->paneID);
+  I("Tmux tab [%d:%s](pane %d) tmux ID %s, from %s/sim%d - %s has stopped.", window, type, pane, simd->paneID, scTemp, count, simd->name);
+}
+
 
 static void PrintEnv(qgrow_t *reply, char *label, char **envp)
 {
@@ -7740,18 +8043,15 @@ fcgiDone:
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 // Start of OpenSim wrangling section.
 ////////////////////////////////////////////////////////////////////////////////////////////////////
-  char *target = NULL;
   struct sysinfo info;
-  float  la;
   // TODO - See https://stackoverflow.com/questions/2693948/how-do-i-retrieve-the-number-of-processors-on-c-linux, there are more portable ways, this seems to be GNU specific.
   int cpus = (int) sysconf(_SC_NPROCESSORS_CONF), cpusOnline = (int) sysconf(_SC_NPROCESSORS_ONLN);
-  int doWait = 1;
 
   if (0 == bulkSims)
     bulkSims = (cpusOnline / 3) - 1;
-  I("There are %i CPUs, %i of them are online.  Doing %i sims at once.", cpus, cpusOnline, bulkSims);
   sysinfo(&info);
-  la = info.loads[0]/65536.0;
+  ourSims->la = info.loads[0]/65536.0;
+  I("There are %i CPUs, %i of them are online.  Doing %i sims at once.  LoadAverage = %f", cpus, cpusOnline, bulkSims, ourSims->la);
 
   if (0 == toys.optc)
     ;
@@ -7768,19 +8068,12 @@ fcgiDone:
   else if (strcmp(toys.optargs[0], "stop") == 0)
     currentMode = STOP;
   else
-    target = toys.optargs[0];
+    ourSims->target = toys.optargs[0];
   if (2 == toys.optc)
-    target = toys.optargs[1];
-
-//T("ARGS - %i |%s|  |%s|", currentMode, toys.optargs[0],  target);
+    ourSims->target = toys.optargs[1];
+V("ARGS - %d %d %i |%s|  %s  |%s|", toys.optc, toys.optflags, currentMode, toys.optargs[0], toys.optargs[1], ourSims->target);
 
-  qtreetbl_obj_t obj0, obj1;
-  qLua *q0, *q1;
-  int count = 0, window = 0, panes = 4, pane = 0;
-
-////////////////////////////////////////////////////////////////////////////////////////////////////
-// Start ROBUST or join the tmux session.
-////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Start ROBUST or join the tmux session, or just figure out where the sims are running in tmux.
   if ((START == currentMode) && !checkSimIsRunning("ROBUST"))
   {
     char *d = xmprintf("%s.{right}", Ttab);
@@ -7794,136 +8087,8 @@ fcgiDone:
     free(c);
     doTmuxCmd("select-pane -t 0 -T 'ROBUST'", Tcmd, scRun, Tsocket);
 
-////////////////////////////////////////////////////////////////////////////////////////////////////
-// Create all the tmux windows, panes, and temporary .ini files, coz OpenSim sucketh.
-////////////////////////////////////////////////////////////////////////////////////////////////////
-    memset((void*)&obj0, 0, sizeof(obj0));
-    ourSims->simsLua->lock(ourSims->simsLua);
-    while(ourSims->simsLua->getnext(ourSims->simsLua, &obj0, false) == true)
-    {
-      q0 = obj0.data;
-      char *type = "unsorted";
-
-      if (LUA_TTABLE == q0->type)
-      {
-        panes = 4;
-        q1 = q0->v.t->get(q0->v.t, "number",    NULL, false);  if (NULL != q1)  window  = q1->v.f - 1;
-        q1 = q0->v.t->get(q0->v.t, "panes",     NULL, false);  if (NULL != q1)  panes   = q1->v.f;
-        q1 = q0->v.t->get(q0->v.t, "type",      NULL, false);  if (NULL != q1)  type    = q1->v.s;
-        if (0 == panes)
-        {
-          pane = 2;
-          window = 0;
-          type = Ttab;
-        }
-        else if (0 != pane)
-        {
-          pane = 0;
-          window++;
-        }
-
-        V("Prepping sims of type %s, window %d, %d panes per window.", type, window, panes);
-
-        memset((void*)&obj1, 0, sizeof(obj1));
-        q0->v.t->lock(q0->v.t);
-        while(q0->v.t->getnext(q0->v.t, &obj1, false) == true)
-        {
-          q1 = obj1.data;
-
-          if ((strcmp("number", obj1.name) != 0) && (strcmp("panes", obj1.name) != 0) && (strcmp("type", obj1.name) != 0))
-          {
-            simData *simd = ourSims->byTab->get(ourSims->byTab, q1->v.s, NULL, false);
-
-            if (NULL == simd)
-              E("Sim %s not found in ini list!", q1->v.s);
-            else
-            {
-                char *path = xmprintf("%s/%s.shini", scEtc, simd->tab);
-                char *newPath = xmprintf("%s/sim%d/%s.ini", scTemp, count, simd->tab);  // Coz OpenSim.
-                char *cmd;
-
-                simd->window = window;
-                simd->pane = pane;
-
-                // The bash invocations are so that the PATH is properly propagated.
-                if (0 == panes)
-                  doTmuxCmd("split-window -hp 50 -d -t '%s:0.1' bash", Tconsole);
-                else if (0 == pane)
-                {
-                  doTmuxCmd("new-window -dc '%s/current/bin' -n '%s' -t '%s:%d' bash", scRoot, type, Tconsole, window);
-                  if (2 <= panes)
-                    doTmuxCmd("split-window -hp 50 -d -t '%s:%d.0' bash", Tconsole, window);
-                  if (4 <= panes)
-                  {
-                    doTmuxCmd("split-window -vp 50 -d -t '%s:%d.1' bash", Tconsole, window);
-                    doTmuxCmd("split-window -vp 50 -d -t '%s:%d.0' bash", Tconsole, window);
-                  }
-                  if (8 <= panes)
-                  {
-                    doTmuxCmd("split-window -hp 50 -d -t '%s:%d.3' bash", Tconsole, window);
-                    doTmuxCmd("split-window -hp 50 -d -t '%s:%d.2' bash", Tconsole, window);
-                    doTmuxCmd("split-window -hp 50 -d -t '%s:%d.1' bash", Tconsole, window);
-                    doTmuxCmd("split-window -hp 50 -d -t '%s:%d.0' bash", Tconsole, window);
-                  }
-                }
-                snprintf(toybuf, sizeof(toybuf), "echo 'IDs={' >%s/IDs_%d.lua", scTemp, window);
-                system(toybuf);
-                doTmuxCmd("list-panes -t %d -F \"['%d #{pane_index}'] = '#{window_id}.#{pane_id}',\" >> %s/IDs_%d.lua", window, window, scTemp, window);
-                snprintf(toybuf, sizeof(toybuf), "echo '}\nreturn IDs' >>%s/IDs_%d.lua", scTemp, window);
-                system(toybuf);
-
-                snprintf(toybuf, sizeof(toybuf), "%s/IDs_%d.lua", scTemp, window);
-                qtreetbl_t *IDs = Lua2tree(toybuf, "IDs");
-                snprintf(toybuf, sizeof(toybuf), "%d %d", window, pane);
-                simd->paneID = xmprintf("%s", qLuaGet(IDs, toybuf)->v.s);
-                freeLuaTree(IDs);
-
-                simd->portH = 8004 + count * 2;
-                simd->portU = 8005 + count * 2;
-                cmd = xmprintf("rm -fr %s/sim%d; mkdir -p %s/sim%d; sed -E"
-                  " -e 's@\\[Region\\]@"
-                  "    paneID = \"\\%s\"\\n"
-                  "\\[Startup\\]\\n"
-                  "    PIDFile = \"\\$\\{Paths\\|CachePath\\}\\/\\$\\{Const\\|mysim\\}\\.pid\"\\n"
-                  "    LogFile = \"\\$\\{Paths\\|LogPath\\}\\/OpenSim_\\$\\{Const\\|mysim\\}\\.log\"\\n"
-                  "    StatsLogFile = \"\\$\\{Paths\\|LogPath\\}\\/OpenSimStats_\\$\\{Const\\|mysim\\}\\.log\"\\n"
-                  "    ConsoleHistoryFile = \"\\$\\{Paths\\|LogPath\\}\\/OpenSimConsoleHistory_\\$\\{Const\\|mysim\\}\\.txt\"\\n"
-                  "\\n\\[Region\\]@'"
-                  " -e 's/InternalPort[[:space:]]*=[[:space:]]*[[:digit:]]+/InternalPort = %d/'"
-                  " -e 's/InternalPort[[:space:]]*=[[:space:]]*\"[[:digit:]]+\"/InternalPort = %d/'"
-                  " -e 's/http_listener_port[[:space:]]*=[[:space:]]*[[:digit:]]+/http_listener_port = %d/'"
-                  " -e 's/http_listener_port[[:space:]]*=[[:space:]]*\"[[:digit:]]+\"/http_listener_port = %d/'"
-                  " %s >%s",
-                  scTemp, count,
-                  scTemp, count,
-                  simd->paneID,
-                  simd->portU,
-                  simd->portU,
-                  simd->portH,
-                  simd->portH,
-                  path, newPath);
-                D("Writing .ini file %s with ports %d %d", newPath, simd->portH, simd->portU);
-                int j = system(cmd);
-                if (!WIFEXITED(j))
-                  E("sed command failed!");
-                free(cmd);
-                free(newPath);
-                free(path);
-                count++;
-                pane++;
-                if (pane >= panes)
-                {
-                  pane = 0;
-                  window++;
-                }
-              }
-            }
-        }
-        q0->v.t->unlock(q0->v.t);
-      }
-    }
-    ourSims->simsLua->unlock(ourSims->simsLua);
-
+    // Create all the tmux windows, panes, and temporary .ini files coz OpenSim sucketh.
+    forEachSim("Prepping", prepSims);
     waitTmuxText(d, "INITIALIZATION COMPLETE FOR ROBUST");
     I("ROBUST is done starting up.");
     free(d);
@@ -7934,269 +8099,17 @@ fcgiDone:
     doTmuxCmd("select-window -t '%s' \\; attach-session -t '%s'", Tconsole, Tconsole);
     goto finished;
   }
-  else
-  {
-    // Find out where the sims are in tmux.
-    memset((void*)&obj0, 0, sizeof(obj0));
-    ourSims->simsLua->lock(ourSims->simsLua);
-    while(ourSims->simsLua->getnext(ourSims->simsLua, &obj0, false) == true)
-    {
-      q0 = obj0.data;
-      char *type = "unsorted";
-
-      if (LUA_TTABLE == q0->type)
-      {
-        panes = 4;
-        q1 = q0->v.t->get(q0->v.t, "number",    NULL, false);  if (NULL != q1)  window  = q1->v.f - 1;
-        q1 = q0->v.t->get(q0->v.t, "panes",     NULL, false);  if (NULL != q1)  panes   = q1->v.f;
-        q1 = q0->v.t->get(q0->v.t, "type",      NULL, false);  if (NULL != q1)  type    = q1->v.s;
-        if (0 == panes)
-        {
-          pane = 2;
-          window = 0;
-          type = Ttab;
-        }
-        else if (0 != pane)
-        {
-          pane = 0;
-          window++;
-        }
-
-        memset((void*)&obj1, 0, sizeof(obj1));
-        q0->v.t->lock(q0->v.t);
-        while(q0->v.t->getnext(q0->v.t, &obj1, false) == true)
-        {
-          q1 = obj1.data;
-
-          if ((strcmp("number", obj1.name) != 0) && (strcmp("panes", obj1.name) != 0) && (strcmp("type", obj1.name) != 0))
-          {
-            simData *simd = ourSims->byTab->get(ourSims->byTab, q1->v.s, NULL, false);
-
-            if (NULL == simd)
-              E("Sim %s not found in ini list!", q1->v.s);
-            else
-            {
-                simd->window = window;
-                simd->pane = pane;
-
-                char *newPath = xmprintf("%s/sim%d/%s.ini", scTemp, count, simd->tab);  // Coz OpenSim.
-
-                snprintf(toybuf, sizeof(toybuf), "%s/IDs_%d.lua", scTemp, window);
-                qtreetbl_t *IDs = Lua2tree(toybuf, "IDs");
-                snprintf(toybuf, sizeof(toybuf), "%d %d", window, pane);
-                simd->paneID = xmprintf("%s", qLuaGet(IDs, toybuf)->v.s);
-                freeLuaTree(IDs);
-                free(newPath);
-                count++;
-                pane++;
-                if (pane >= panes)
-                {
-                  pane = 0;
-                  window++;
-                }
-              }
-            }
-        }
-        q0->v.t->unlock(q0->v.t);
-      }
-    }
-    ourSims->simsLua->unlock(ourSims->simsLua);
-  }
-
-
-////////////////////////////////////////////////////////////////////////////////////////////////////
-// Sim wrangling loop.
-////////////////////////////////////////////////////////////////////////////////////////////////////
-  count = 0;  window = 0;  panes = 4;  pane = 0;
-  memset((void*)&obj0, 0, sizeof(obj0));
-  ourSims->simsLua->lock(ourSims->simsLua);
-  while(ourSims->simsLua->getnext(ourSims->simsLua, &obj0, false) == true)
-  {
-    q0 = obj0.data;
-    char *type = "unsorted";
-
-    if (LUA_TTABLE == q0->type)
-    {
-      panes = 4;
-      q1 = q0->v.t->get(q0->v.t, "number",      NULL, false);  if (NULL != q1)  window  = q1->v.f - 1;
-      q1 = q0->v.t->get(q0->v.t, "panes",       NULL, false);  if (NULL != q1)  panes   = q1->v.f;
-      q1 = q0->v.t->get(q0->v.t, "type",        NULL, false);  if (NULL != q1)  type    = q1->v.s;
-      if (0 == panes)
-      {
-        pane = 2;
-        window = 0;
-        type = Ttab;
-      }
-      else if (0 != pane)
-      {
-        pane = 0;
-        window++;
-      }
-
-      V("Doing sims of type %s, window %d, %d panes per window.", type, window, panes);
-      memset((void*)&obj1, 0, sizeof(obj1));
-      q0->v.t->lock(q0->v.t);
-      while(q0->v.t->getnext(q0->v.t, &obj1, false) == true)
-      {
-        q1 = obj1.data;
-
-        if ((strcmp("number", obj1.name) != 0) && (strcmp("panes", obj1.name) != 0) && (strcmp("type", obj1.name) != 0))
-        {
-          simData *simd = ourSims->byTab->get(ourSims->byTab, q1->v.s, NULL, false);
-
-          if (NULL == simd)
-            E("Sim %s not found in ini list!", 5);
-          else
-          {
-            // Check if only doing a single sim.
-            int cont = FALSE;
-            if (NULL != target)
-            {
-              cont = TRUE;
-              snprintf(toybuf, sizeof(toybuf), "%s.shini", simd->tab);
-              if
-              (
-                (strcmp(target, q1->v.s) == 0)  ||
-                (strcmp(target, simd->name) == 0) ||
-                (strcmp(target, simd->tab) == 0)   ||
-                (strcmp(target, toybuf) == 0)
-              )
-              cont = FALSE;
-              snprintf(toybuf, sizeof(toybuf), "%s.ini", simd->tab);
-              if (strcmp(target, toybuf) == 0)
-                cont = FALSE;
-              snprintf(toybuf, sizeof(toybuf), "sim%d.ini", count);
-              if (strcmp(target, toybuf) == 0)
-                cont = FALSE;
-            }
-            if (!cont)
-            {
-                switch (currentMode)
-                {
-                  case START :  // "start sim01"  "start 'Welcome sim'" "start Welcome.ini" "start Welcome"  "start" start everything
-                  {
-                    if (!checkSimIsRunning(simd->tab))
-                    {
-                      I("Tmux tab [%d:%s](pane %d) tmux ID %s, from %s/sim%d - %s is starting.", window, type, pane, simd->paneID, scTemp, count, simd->name);
-                      doTmuxCmd("select-pane -t %s:%s -T '%s'", Tconsole, simd->paneID, simd->tab);
-                      snprintf(toybuf, sizeof(toybuf), "mono OpenSim.exe -inidirectory=%s/sim%d", scTemp, count);
-                      sendTmuxCmd(simd->paneID, toybuf);
-// TODO - some compromise, do a premptive load check if we are not doing a wait anyway.
-                      if (0 == doWait)
-                      {
-//                      doTmuxCmd("select-window -t %s:%d", Tconsole, simd->window);
-                        snprintf(toybuf, sizeof(toybuf), "INITIALIZATION COMPLETE FOR %s", simd->name);
-                        waitTmuxText(simd->paneID, toybuf);
-                        I("%s is done starting up.", simd->name);
-                        la = waitLoadAverage(la, loadAverageInc, simTimeOut);
-                        if (1 < bulkSims)
-                            bulkSims = bulkSims / 2;
-                      }
-                      doWait = ((count + 1) % bulkSims);
-                    }
-                    break;
-                  }
-
-                  case BACKUP : // "backup 'onefang rejected'"  "backup sim01"  "backup 'Welcome Sim'"  "backup" backup everything
-                  {             // TODO - If it's not a sim code, and not a sim name, it's an account inventory.
-                    if (checkSimIsRunning(simd->tab))
-                    {
-                      struct timeval tv;
-                      time_t curtime;
-                      char date[DATE_TIME_LEN];
-
-// TODO - should collect names of existing backups, both tmux / ini name and proper name.
-//          Scan backups directory once before this for loop, add details to sims list.
-//            strip off the last bit of file name (YYYY-mm-dd_HH:MM:SS.oar) to get the name
-//              keep in mind some old files had munged names like "Tiffanie_s_Paradise-2021-06-23_05:11:38.oar"
-                      gettimeofday(&tv, NULL);
-                      curtime = tv.tv_sec;
-                      strftime(date, DATE_TIME_LEN, "%F_%T", localtime(&curtime));
-                      I("%s is being backed up to %s/backups/%s-%s.oar.", simd->name, scRoot, simd->tab, date);
-                      snprintf(toybuf, sizeof(toybuf), "save oar --all %s/backups/%s-%s.oar", scRoot, simd->tab, date);
-                      sendTmuxCmd(simd->paneID, toybuf);
-//                      if (0 == doWait)
-                      {
-                        memset(toybuf, 0, sizeof(toybuf));
-                        snprintf(toybuf, sizeof(toybuf), "Finished writing out OAR for %s", simd->name);
-                        waitTmuxText(cmd, toybuf);
-                        I("%s is done backing up.", simd->name);
-// TODO - should delete / gitAR the old ones now.
-//          Have a config option for delete / gitAR.
-                        la = waitLoadAverage(la, loadAverageInc, simTimeOut);
-                      }
-                    }
-                    break;
-                  }
-
-                  case CREATE : // "create name x,y size"
-                  {
-                    break;
-                  }
-
-                  case GITAR :  // "gitAR i name"  "gitAR o name"
-                  {
-                    break;
-                  }
-
-                  case STATUS :
-                  {
-                    break;
-                  }
-
-                  case STOP :   // "stop sim01"  "stop 'Welcome Sim'"  "stop" stop everything
-                  {
-                    if (checkSimIsRunning(simd->tab))
-                    {
-                      // Leave empty panes as is.
-                      sendTmuxCmd(simd->paneID, "quit");
-                      // There's three things that might happen -
-                      //   Sim will quit, tmux pane will go away before we can see the shutdown message.
-                      //        pane-exited hook might be useful here.
-                      //   Sim will quit, tmux pane wont go away.  Dunno yet if waitTmuxText() will still work.
-                      //        pane-died hook might be useful here.
-                      //   Sim fails to quit, error message on the pane that we want to read, wont see the shutdown message.
-                      // Also sim might not have finished starting up, and may even not be accepting comands yet.
-// TODO - cater for and test them all.
-                      //        Could also loop on checkSimIsRunning(sim)
-//                    snprintf(toybuf, sizeof(toybuf), "[SHUTDOWN]: Shutdown processing on main thread complete.  Exiting...");
-//                    waitTmuxText(nm, toybuf);
-//                    I("%s is done stopping.", simd->name);
-//                    la = waitLoadAverage(la, loadAverageInc, simTimeOut);
-                    }
-                    break;
-                  }
-
-                  default :
-                  {
-                    E("Unknown command! %s", toys.optargs[0]);
-                    break;
-                  }
-                }
-            }
-
-          }
-
-          count++;
-          pane++;
-          if (pane >= panes)
-          {
-            pane = 0;
-            window++;
-          }
-        }
-      }
-      q0->v.t->unlock(q0->v.t);
-    }
-  }
-  ourSims->simsLua->unlock(ourSims->simsLua);
+  else  // Find out where the sims are in tmux.
+    forEachSim(NULL, findSimsTmux);
 
+  // Do stuff with the sims.
+  forEachSim("Doing", doSimsThing);
 
   if (START == currentMode)
   {
-    if (checkSimIsRunning("ROBUST") && (NULL == target))
+    if (checkSimIsRunning("ROBUST") && (NULL == ourSims->target))
     {
-      // Start up the web stuff.  TODO - remove this once we handle the fcgi stuff ourselves.
+      // TODO - remove this once we handle the fcgi stuff ourselves.
       I("Starting the web stuff.");
       char *c = xmprintf("spawn-fcgi -n -u %s -s %s/sledjchisl.socket -M 0660 -G www-data  -- "
         "/usr/bin/valgrind --leak-check=full sledjchisl",
@@ -8207,85 +8120,27 @@ fcgiDone:
     else
       I("NOT Starting the web stuff.");
   }
-  else if ((STOP == currentMode) && (NULL == target))
+  else if ((STOP == currentMode) && (NULL == ourSims->target))
   {
-    memset((void*)&obj0, 0, sizeof(obj0));
-    ourSims->simsLua->lock(ourSims->simsLua);
-    while(ourSims->simsLua->getnext(ourSims->simsLua, &obj0, false) == true)
-    {
-      q0 = obj0.data;
-      char *type = "unsorted";
-
-      if (LUA_TTABLE == q0->type)
-      {
-        panes = 4;
-        q1 = q0->v.t->get(q0->v.t, "number",    NULL, false);  if (NULL != q1)  window  = q1->v.f - 1;
-        q1 = q0->v.t->get(q0->v.t, "panes",     NULL, false);  if (NULL != q1)  panes   = q1->v.f;
-        q1 = q0->v.t->get(q0->v.t, "type",      NULL, false);  if (NULL != q1)  type    = q1->v.s;
-        if (0 == panes)
-        {
-          pane = 2;
-          window = 0;
-          type = Ttab;
-        }
-        else if (0 != pane)
-        {
-          pane = 0;
-          window++;
-        }
-
-        memset((void*)&obj1, 0, sizeof(obj1));
-        q0->v.t->lock(q0->v.t);
-        while(q0->v.t->getnext(q0->v.t, &obj1, false) == true)
-        {
-          q1 = obj1.data;
-
-          if ((strcmp("number", obj1.name) != 0) && (strcmp("panes", obj1.name) != 0) && (strcmp("type", obj1.name) != 0))
-          {
-            simData *simd = ourSims->byTab->get(ourSims->byTab, q1->v.s, NULL, false);
-
-            if (NULL == simd)
-              E("Sim %s not found in ini list!", q1->v.s);
-            else
-            {
-                // NOTE - these sleeps are guesses, seems to work on my super desktop during testing.
-                while (checkSimIsRunning(simd->tab))
-                  usleep(100000);
-                usleep(10000);
-                doTmuxCmd("kill-pane -t %s", simd->paneID);
-                I("Tmux tab [%d:%s](pane %d) tmux ID %s, from %s/sim%d - %s has stopped.", window, type, pane, simd->paneID, scTemp, count, simd->name);
-
-                count++;
-                pane++;
-                if (pane >= panes)
-                {
-                  pane = 0;
-                  window++;
-                }
-              }
-            }
-        }
-        q0->v.t->unlock(q0->v.t);
-      }
-    }
-    ourSims->simsLua->unlock(ourSims->simsLua);
-
-    // Kill the rest of the panes that are not on the original window.
-    I("Closing all the panes.");
+    // Stop all the sims.
+    forEachSim(NULL, stopSims);
+    I("Closing all the other windows.");
+    // First figure out what panes and windows are left.
     snprintf(toybuf, sizeof(toybuf), "echo 'IDs={' >%s/IDs_ALL.lua", scTemp);
     system(toybuf);
     doTmuxCmd("list-panes -s -F '\"#{window_id}.#{pane_id}\",' >> %s/IDs_ALL.lua", scTemp);
     snprintf(toybuf, sizeof(toybuf), "echo '}\nreturn IDs' >>%s/IDs_ALL.lua", scTemp);
     system(toybuf);
-
     snprintf(toybuf, sizeof(toybuf), "%s/IDs_ALL.lua", scTemp);
     qtreetbl_t *IDs = Lua2tree(toybuf, "IDs");
 
+    qtreetbl_obj_t obj0;
     memset((void*)&obj0, 0, sizeof(obj0));
     IDs->lock(IDs);
     while(IDs->getnext(IDs, &obj0, false) == true)
     {
-      q0 = obj0.data;
+      qLua *q0 = obj0.data;
+
       if ('0' != q0->v.s[1])
         doTmuxCmd("kill-pane -t %s", q0->v.s);
     }
@@ -8296,7 +8151,6 @@ fcgiDone:
     if (!WIFEXITED(system(c)))
       E("rm command failed!  %s", c);
     free(c);
-
   }
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 // End of OpenSim wrangling section.