Add rsync and gitar stuff to sledjchisl backups.

1 files changed, 255 insertions, 21 deletions

diff --git a/src/sledjchisl/sledjchisl.c b/src/sledjchisl/sledjchisl.c
index c2ec573..e4c4359 100644
--- a/src/sledjchisl/sledjchisl.c
+++ b/src/sledjchisl/sledjchisl.c
@@ -522,6 +522,8 @@ char *Tsocket = "opensim-tmux.socket";
 char *Ttab = "SC";
 char *Tcmd = "tmux -S";
 char *backupIARsim = "Sandbox";
+char *rSync = "";
+int  rSyncPort = 0;
 char *webRoot = "/var/www/html";
 char *URL = "fcgi-bin/sledjchisl.fcgi";
 char *ToS = "Be good.";
@@ -1883,6 +1885,8 @@ void startSim(simData *simd, char *sim, char *type, int count, int window, int p
       snprintf(toybuf, sizeof(toybuf), "INITIALIZATION COMPLETE FOR %s", simd->name);
       waitTmuxText(simd->paneID, toybuf);
       I("%s is done starting up.", simd->name);
+      sendTmuxCmd(simd->paneID, "");
+      sendTmuxCmd(simd->paneID, "");
       ourSims->la = waitLoadAverage(ourSims->la, loadAverageInc, simTimeOut);
       if (1 < bulkSims)
         bulkSims = bulkSims / 2;
@@ -1947,16 +1951,51 @@ void stopSim(simData *simd, char *sim, char *type, int count, int window, int pa
   }
 }
 
+typedef struct _ARList ARList;
+struct _ARList
+{
+  int len, num;
+  char **ARs, *this;
+};
+
+static int filterARs(struct dirtree *node)
+{
+    if (!node->parent) return DIRTREE_RECURSE | DIRTREE_SHUTUP;
+    ARList *list = (ARList *) node->parent->extra;
+    int l0 = strlen(node->name), l1 = strlen(list->this);
+
+// TODO - ignore zero sized ones.
+    if ((4 < l0) && ((strncmp(&(node->name[l0 - 4]), ".iar", 4) == 0) || (strncmp(&(node->name[l0 - 4]), ".oar", 4) == 0)) && (strncmp(node->name, list->this, l1) == 0))
+    {
+      if ((list->num + 1) > list->len)
+      {
+        list->len = list->len + 1;
+        list->ARs = xrealloc(list->ARs, list->len * sizeof(char *));
+      }
+      list->ARs[list->num] = xstrdup(node->name);
+      list->num++;
+    }
+    return 0;
+}
+
 // Forward declare this.
 my_ulonglong dbCount(char *table, char *where);
 void doSimsThing(simData *simd, char *sim, char *type, int count, int window, int panes, int pane)
 {
   // Check if only doing a single sim.
-  int cont = FALSE;
+  int cont = FALSE, member = false;
+  char *last = strchr(sim, ' ');
 
   if (FLAG(m))
   {
-// TODO - check if it's a real user.
+    // check if it's a real user.
+    last[0] = '\0';  last++;
+    // Double check it's a real member.
+    snprintf(toybuf, sizeof(toybuf), "FirstName='%s' and LastName='%s'", sim, last);
+    if (1 == dbCount("UserAccounts", toybuf))
+      member = TRUE;
+    else
+      E("Can't find member %s %s.", sim, last);
   }
   else if (NULL != ourSims->target)
   {
@@ -1997,7 +2036,7 @@ byTab has the short name as the key, simData as the value.
     }
 
     case BACKUP :       // "backup -m 'onefang rejected'"  "backup 'Welcome sim'" "backup Welcome.shini" "Welcome.shini backup" "backup Welcome.ini" "backup Welcome"  "backup" backup everything
-    {           // TODO - If it's not a sim code, and not a sim name, it's an account inventory.
+    {
       struct timeval tv;
       time_t curtime;
       char date[DATE_TIME_LEN];
@@ -2007,17 +2046,14 @@ byTab has the short name as the key, simData as the value.
       strftime(date, DATE_TIME_LEN, "%F_%T", localtime(&curtime));
       if (FLAG(m))
       {
-        char *last = strchr(sim, ' ');
-
-        last[0] = '\0';  last++;
-        // Double check it's a real member.
-        snprintf(toybuf, sizeof(toybuf), "FirstName='%s' and LastName='%s'", sim, last);
-        if (1 == dbCount("UserAccounts", toybuf))
+        if (member)
         {
-          I("Member %s %s is being backed up to %s/backups/%s_%s-%s.iar in tmux windo %s.", sim, last, scRoot, sim, last, date, ourSims->backup);
-          snprintf(toybuf, sizeof(toybuf), "save iar -c %s %s / password %s/backups/%s_%s-%s.iar", sim, last, scRoot, sim, last, date);
+          I("Member %s %s is being backed up to %s/%s_%s-%s.iar in tmux windo %s.", sim, last, scBackup, sim, last, date, ourSims->backup);
+          snprintf(toybuf, sizeof(toybuf), "save iar -c %s %s / password %s/%s_%s-%s.iar", sim, last, scBackup, sim, last, date);
           if (NULL != ourSims->backup)
           {
+            if ('\0' == rSync[0])
+              shellMeFail("%s/current/bin/sledjchisl gitar -m %s %s %s", scRoot, FLAG(v) ? "-v" : "", sim, last);
             sendTmuxCmd(ourSims->backup, toybuf);
 //          if (0 == do)
             {
@@ -2026,16 +2062,18 @@ byTab has the short name as the key, simData as the value.
               I("%s %s is done backing up.", sim, last);
               sendTmuxCmd(ourSims->backup, "");
               sendTmuxCmd(ourSims->backup, "");
-// TODO - should delete / gitAR the old ones now.
-//          Have a config option for delete / gitAR / rsync.
+              if ('\0' != rSync[0])
+              {
+                if (shellMeFail("time ionice -c3 nice -n 19 rsync -Ha -R --modify-window=2 --partial --port=%d --remove-source-files %s/*.iar %s",
+                     rSyncPort, scBackup, rSync))
+                  E("rsync failed");
+              }
               ourSims->la = waitLoadAverage(ourSims->la, loadAverageInc, simTimeOut);
             }
           }
           else
             E("Can't find backup sim.");
         }
-        else
-          E("Can't find member %s %s.", sim, last);
       }
       else if (checkSimIsRunning(simd->tab))
       {
@@ -2044,15 +2082,23 @@ byTab has the short name as the key, simData as the value.
 //            strip off the last bit of file name (YYYY-mm-dd_HH:MM:SS.oar) to get the name
 //              keep in mind some old files had munged names like "Tiffanie_s_Paradise-2021-06-23_05:11:38.oar"
         I("Sim %s is being backed up to %s/backups/%s-%s.oar.", simd->name, scRoot, simd->tab, date);
-        snprintf(toybuf, sizeof(toybuf), "save oar --all %s/backups/%s-%s.oar", scRoot, simd->tab, date);
+        if ('\0' == rSync[0])
+          shellMeFail("%s/current/bin/sledjchisl gitar %s %s", scRoot, FLAG(v) ? "-v" : "", simd->tab);
+        snprintf(toybuf, sizeof(toybuf), "save oar --all %s/%s-%s.oar", scBackup, simd->tab, date);
         sendTmuxCmd(simd->paneID, toybuf);
 //      if (0 == do)
         {
           snprintf(toybuf, sizeof(toybuf), "Finished writing out OAR for %s", simd->name);
           waitTmuxText(simd->paneID, toybuf);
           I("%s is done backing up.", simd->name);
-// TODO - should delete / gitAR the old ones now.
-//          Have a config option for delete / gitAR / rsync.
+          sendTmuxCmd(simd->paneID, "");
+          sendTmuxCmd(simd->paneID, "");
+          if ('\0' != rSync[0])
+          {
+            if (shellMeFail("time ionice -c3 nice -n 19 rsync -Ha -R --modify-window=2 --partial --port=%d --remove-source-files %s/*.oar %s",
+                rSyncPort, scBackup, rSync))
+              E("rsync failed");
+          }
           ourSims->la = waitLoadAverage(ourSims->la, loadAverageInc, simTimeOut);
         }
       }
@@ -2061,6 +2107,189 @@ byTab has the short name as the key, simData as the value.
 
     case GITAR :        // "gitAR -m avatar name"  "gitAR sim name"
     {
+      /* Work around OpenSims slow database corruption bug by using git to store all old backups.
+        Try to squeeze every last byte out of the tarballs.  Seems to cut the total storage size down to one third the size of just the raw I/OAR files.
+        Saves even more if there's been no changes.
+        On the other hand, these backup files will grow indefinately, the more changes, the faster it grows.  I can live with that for more reliable backups that go back further.
+        Tries to avoid loosing data if things go wrong.  I think the main remaining problem would be running out of space, in which case you have bigger problems to deal with.
+
+        Strategy - unpack the last one, unpack and commit any old I/OARs, pack up the result, delete it's working directory, THEN run the save i/oar.
+        Avoids having to sync with OpenSim finishing the current I/OAR, and as a bonus, an easy to deliver latest I/OAR for people that want it.
+      */
+      char *name = xstrdup(sim);
+
+      if (FLAG(m))
+      {
+        if (member)
+        {
+          free(name);
+          name = xmprintf("%s_%s", sim, last);
+        }
+        else
+        {
+          free(name);
+          return;
+        }
+      }
+
+      char type  = FLAG(m) ? 'I' : 'O';
+      char *gar  = xmprintf("%s_git%cAR", name, type);
+      char *gtr  = xmprintf("%s/%s-git%cAR.tar.xz", scBackup, name, type);
+      char *dir  = xmprintf("%s/temp_backups%c_%s", scBackup, type, name);
+      // Make sure stuff that's already compressed doesn't get compressed by git.
+      // Also tries to protect binaries from mangling.
+      char *gab = xmprintf("%s/%s/.gitattributes", dir, gar);
+      char *ga  = xmprintf("%s/%s", dir, gar);
+      char *gal  = xmprintf("%s/log", scBackup);
+      char *gad =
+        "*.bvh -delta -diff -text\n"
+        "*.jp2 -delta -diff -text\n"
+        "*.jpg -delta -diff -text\n"
+        "*.llmesh -delta -diff -text\n"
+        "*.ogg -delta -diff -text\n"
+        "*.png -delta -diff -text\n"
+        "*.r32 -delta -diff -text\n"
+        "*.tga -delta -diff -text\n";
+      ARList *ourARs = xzalloc(sizeof(ARList));
+
+      if (qfile_exist(dir))
+      {
+        if (shellMeFail("echo 'Mess left over from last backup in %d, not gonna run!' >>%s/errors", dir))
+          E("Failed cleaning up the mess!");
+        if (shellMeFail("mv %s/*.%car %s 2>&1 >/dev/null", dir, tolower(type), scBackup))
+          E("Failed cleaning up the mess!");
+        goto gitARend;
+      }
+
+      // Either unpack the old gitAR, or create a new one.
+      qfile_mkdir(dir, S_IRWXU | S_IRGRP | S_IXGRP, true);
+      if (qfile_exist(gtr))
+      {
+        I("Unpacking %s", gtr);
+        if (shellMeFail("cd %s; ionice -c3 nice -n 19 tar -xf %s", dir, gtr))
+          E("Failed to unpack %s!", gtr);
+      }
+      else
+      {
+        qfile_mkdir(gar, S_IRWXU | S_IRGRP | S_IXGRP, true);
+        // git will create gar for us.
+        if (shellMeFail("cd %s; git init %s >%s", dir, gar, gal))
+          E("Failed to git init %s/%s!", dir, gar);
+
+        // Coz git insists.
+        if (shellMeFail("cd %s/%s; git config user.email \"opensim@$(hostname -A | cut -d ' ' -f 1)\"", dir, gar))
+          E("Failed to git config user.email!");
+        if (shellMeFail("cd %s/%s; git config user.name  \"opensim\"", dir, gar))
+          E("Failed to git config user.name!");
+
+        // Coz git insists on having something comitted before it can change the branch name.
+        if (-1 == qfile_save(gab, gad, strlen(gad), false))
+          E("Faild to write %s file!", ga);
+        if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gal))
+          E("Failed to git add!");
+        V("Committing initial git.");
+        if (shellMeFail("cd %s/%s; git commit -qm \"Initial commit\" >>%s || echo \"ERROR - Could not commit!\" >>%s/errors ", 
+            dir, gar, gal, dir))
+          E("Failed to git commit!");
+        if (shellMeFail("cd %s/%s; git branch -m master Domme >>%s", dir, gar, gal))
+          E("Failed to git branch -m master Domme!");
+      }
+
+      // Git is such a pedantic bitch, let's just fucking ignore any errors it gives due to lack of anything to do.
+      // Even worse the OpenSim devs breaking logout tracking gives git plenty of nothing to do.  lol
+
+      // Loop through the .iar / .oar files in backups, ignoring zero sized files.
+      struct dirtree *new = dirtree_add_node(0, scBackup, 0);
+      int i;
+
+      ourARs->num = 0;
+      ourARs->this = xmprintf("%s-", name);
+      new->extra = (long) ourARs;
+      dirtree_handle_callback(new, filterARs);
+      qsort(ourARs->ARs, ourARs->num, sizeof(char *), qstrcmp);
+      free(ga);
+      ga = xmprintf("%s/errors", dir);
+      for (i = 0; i < ourARs->num; i++)
+      {
+        I("Adding %s to %s", ourARs->ARs[i], gtr);
+        // Deal with deletions in the inventory / sim, easy method, which becomes a nop for files that stay in the git add below.
+        if (shellMeFail("cd %s/%s; rm -fr * >>%s", dir, gar, gal))
+          E("Failed to rm!");
+        if (shellMeFail("cd %s/%s; ionice -c3 nice -n 19 tar -xzf \"%s/%s\" || echo \"ERROR - Could not unpack %s !\" >>%s",
+          dir, gar, scBackup, ourARs->ARs[i], ourARs->ARs[i], ga))
+        E("Failed to unpack %s!", ourARs->ARs[i]);
+        if (!qfile_exist(ga))
+        {
+          if (shellMeFail("cd %s/%s; git add * >>%s", dir, gar, gal))
+            E("Failed to git add!");
+          // The \\* bit is to escape the \ from snprintf, which itself escapes the * from the shell.
+          if (shellMeFail("cd %s/%s; git add */\\* >>%s", dir, gar, gal))
+            E("Failed to git add!");
+          // Gotta add this again, coz of the rm.  Apparently rm removes dotfiles, but add doesn't!
+          if (-1 == qfile_save(gab, gad, strlen(gad), false))
+            E("Faild to write %s file!", ga);
+          if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gal))
+            E("Failed to git add!");
+
+          // Magic needed to figure out if there's anything to commit.
+          // After all the pain to get this to work, there's an ever changing timestamp in archive.xml that screws it up.
+          // Like this system didn't have enough timestamps in it already.  lol
+          // TODO - I could sed out that timestamp, and put it back again based on the OAR file name when extracting.
+          //        IARs don't seem to have the timestamp.
+          int j = shellMe("cd %s/%s; t=$(git status --porcelain) && [ -z \"${t}\" ]", dir, gar);
+          if (!WIFEXITED(j))
+            E("git status failed!");
+          else if (1 == WEXITSTATUS(j))
+          {
+            V("Committing changes from %s", ourARs->ARs[i]);
+            // Note this commit message has to be just the file name, as the ungitAR script uses it.
+            if (shellMeFail("cd %s/%s; git commit -a -qm \"%s\" >>%s || echo \"ERROR - Could not commit %s !\" >>%s ", 
+                            dir, gar, ourARs->ARs[i], ourARs->ARs[i], gal, ga))
+              E("Failed to git commit!");
+            if (qfile_exist(ga))
+            {
+              free(ga);
+              free(ourARs->ARs[i]);
+              goto gitARend;
+            }
+          }
+          else
+            V("No changes to commit from %s.", ourARs->ARs[i]);
+        }
+        if (!qfile_exist(ga))
+        {
+          if (shellMeFail("mv %s/%s %s", scBackup, ourARs->ARs[i], dir))
+            E("Failed to move %s!", ourARs->ARs[i]);
+        }
+        free(ourARs->ARs[i]);
+      }
+
+      if (!qfile_exist(ga))
+      {
+        I("Compressing gitAR %s", gtr);
+        if (shellMeFail("cd %s; XZ_OPT='-9e' ionice -c3 nice -n 19 tar -c --xz %s -f %s || echo 'ERROR - Could not pack gitAR!' >>%s", 
+            dir, gar, gtr, ga))
+          E("Failed to git add!");
+      }
+gitARend:
+      free(ga);
+      ga = xmprintf("%s/errors", dir);
+      if (qfile_exist(ga))
+        E("Failed to process the archives, look in %s for errors!", ga);
+      else
+      {
+        if (shellMeFail("rm -fr %s", dir))
+          E("Failed to rm!");
+      }
+      free(ga);
+      free(ourARs->ARs);
+      free(ourARs->this);
+      free(ourARs);
+      free(gab);
+      free(dir);
+      free(gtr);
+      free(gar);
+      free(name);
       break;
     }
 
@@ -7525,12 +7754,16 @@ void forEachMember(char *verb, simFunction func, simFunction not)
   {
     char *file = xmprintf("%s/.lastTime", scBackup);
 
+    free(last);
     last = (char *) qfile_load(file, NULL);
-    if (NULL == last)
-      last = xstrdup("0");
     if (shellMeFail("date +%%s > %s", file))
       E("date command failed!");
     free(file);
+    if ((NULL == last) || (NULL != ourSims->target))
+    {
+      free(last);
+      last = xstrdup("0");
+    }
   }
 
   static dbRequest *users = NULL;
@@ -7552,7 +7785,6 @@ void forEachMember(char *verb, simFunction func, simFunction not)
   {
     qhashtbl_t *row;
 
-d("Number of rows returned %d", (int) rows->rows->size(rows->rows));
     while (NULL != (row = rows->rows->getat(rows->rows, 0, NULL, true)))
     {
       char *firstName = row->getstr(row, "FirstName", false);
@@ -7862,6 +8094,8 @@ Other possibilities -
     if ((tmp = configs->getstr(configs, "Tsocket",  false))             != NULL)  {Tsocket              = tmp;                          V("Setting Tsocket = %s", Tsocket);}
     if ((tmp = configs->getstr(configs, "Ttab",     false))             != NULL)  {Ttab                 = tmp;                          V("Setting Ttab = %s", Ttab);}
     if ((tmp = configs->getstr(configs, "backupIARsim",         false)) != NULL)  {backupIARsim         = tmp;                          V("Setting backupIARsim = %s", backupIARsim);}
+    if ((tmp = configs->getstr(configs, "rsync",                false)) != NULL)  {rSync                = tmp;                          V("Setting rsync = %s", rSync);}
+    if ((vd  = configs->getstr(configs, "rsyncPort",            false)) != NULL)  {rSyncPort            = (int) *((float *) vd);        V("Setting rsyncPort = %s", rSyncPort);}
     if ((tmp = configs->getstr(configs, "webRoot",  false))             != NULL)  {webRoot              = tmp;                          V("Setting webRoot = %s", webRoot);}
     if ((tmp = configs->getstr(configs, "URL",      false))             != NULL)  {URL                  = tmp;                          V("Setting URL = %s", URL);}
     if ((vd  = configs->get   (configs, "seshRenew",      NULL, false)) != NULL)  {seshRenew            = (int) *((float *) vd);        V("Setting seshRenew = %d", seshRenew);}