Refactor GITAR, so we can call it direct from BACKUP.

1 files changed, 181 insertions, 174 deletions

diff --git a/src/sledjchisl/sledjchisl.c b/src/sledjchisl/sledjchisl.c
index aca04d6..b74e419 100644
--- a/src/sledjchisl/sledjchisl.c
+++ b/src/sledjchisl/sledjchisl.c
@@ -2271,6 +2271,182 @@ static int filterARs(struct dirtree *node)
     return 0;
 }
 
+void gitar(simData *simd, char *sim, int count, int window, int panes, int pane, int m, int member, char *last)
+{
+  /* Work around OpenSims slow database corruption bug by using git to store all old backups.
+    Try to squeeze every last byte out of the tarballs.  Seems to cut the total storage size down to one third the size of just the raw I/OAR files.
+    Saves even more if there's been no changes.
+    On the other hand, these backup files will grow indefinately, the more changes, the faster it grows.  I can live with that for more reliable backups that go back further.
+    Tries to avoid loosing data if things go wrong.  I think the main remaining problem would be running out of space, in which case you have bigger problems to deal with.
+
+    Strategy - unpack the last one, unpack and commit any old I/OARs, pack up the result, delete it's working directory, THEN run the save i/oar.
+    Avoids having to sync with OpenSim finishing the current I/OAR, and as a bonus, an easy to deliver latest I/OAR for people that want it.
+  */
+  char *name = xstrdup(sim);
+
+  if (m)
+  {
+    if (member)
+    {
+      free(name);
+      name = xmprintf("%s_%s", sim, last);
+    }
+    else
+    {
+      free(name);
+      return;
+    }
+  }
+
+  char type  = m ? 'I' : 'O';
+  char *gar  = xmprintf("%s-git%cAR", name, type), *gtr  = xmprintf("%s/%s.tar.xz", scBackup, gar);
+  char *dir  = xmprintf("%s/temp_%c_%s", scBackup, type, name);
+  char *glog = xmprintf("%s/log.log", scBackup), *gerr = xmprintf("%s/errors", dir);
+  // Make sure stuff that's already compressed doesn't get compressed by git.
+  // Also tries to protect binaries from mangling.
+  char *gab = xmprintf("%s/%s/.gitattributes", dir, gar), *gad =
+    "*.dat -diff -text\n"
+    "*.bvh -delta -diff -text\n"
+    "*.j2c -delta -diff -text\n"        // OpenSim uses all four JPEG 2000 extensions.  In the same directories.  lol
+    "*.jp2 -delta -diff -text\n"
+    "*.jpc -delta -diff -text\n"
+    "*.jpg -delta -diff -text\n"
+    "*.llmesh -delta -diff -text\n"
+    "*.ogg -delta -diff -text\n"
+    "*.png -delta -diff -text\n"
+    "*.r32 -delta -diff -text\n"
+    "*.tga -delta -diff -text\n";
+  ARList *ourARs = xzalloc(sizeof(ARList));
+
+  if (qfile_exist(dir))
+  {
+    if (shellMeFail("echo 'Mess left over from last backup in %d, not gonna run!' >>%s", dir, gerr))            E("Cleaning up the mess!");
+    if (shellMeFail("mv %s/*.%car %s 2>&1 >>%s", dir, tolower(type), scBackup, gerr))                           E("Failed cleaning up the mess!");
+    goto gitARend;
+  }
+
+  // Either unpack the old gitAR, or create a new one.
+  qfile_mkdir(dir, S_IRWXU | S_IRGRP | S_IXGRP, true);
+  if (qfile_exist(gtr))
+  {
+    I("Unpacking %s", gtr);
+    if (shellMeFail("cd %s; ionice -c3 nice -n 19 tar -xf %s >>%s", dir, gtr, gerr))                            E("Failed to unpack %s!", gtr);
+    char *t = xmprintf("%s/%s_git%cAR", dir, name, type);
+    // Changed from _ to -, but deal with old archives.
+    if (qfile_exist(t))
+    {
+      if (shellMeFail("mv %s %s/%s", t, dir, gar))                                                              E("Failed to move %s!", t);
+    }
+    free(t);
+  }
+  else
+  {
+    // git will create gar for us.
+    if (shellMeFail("cd %s; git init --quiet %s >>%s", dir, gar, gerr))                                         E("Failed to git init %s/%s!", dir, gar);
+
+    // Coz git insists.
+    if (shellMeFail("cd %s/%s; git config user.email \"opensim@$(hostname -A | cut -d ' ' -f 1)\"", dir, gar))  E("Failed to git config user.email!");
+    if (shellMeFail("cd %s/%s; git config user.name  \"opensim\"", dir, gar))                                   E("Failed to git config user.name!");
+
+    // Coz git insists on having something comitted before it can change the branch name.
+    if (-1 == qfile_save(gab, gad, strlen(gad), false))                                                         E("Faild to write %s file!", gad);
+    if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gerr))                                   E("Failed to git add!");
+    V("Committing initial git.");
+    if (shellMeFail("cd %s/%s; git commit -qm \"Initial commit\" >>%s || echo \"ERROR - Could not commit!\" >>%s",
+        dir, gar, glog, gerr))                                                                                  E("Failed to git commit!");
+    if (shellMeFail("cd %s/%s; git branch -m master Domme >>%s", dir, gar, gerr))                               E("Failed to git branch -m master Domme!");
+  }
+  // Doing these each time, to catch those old ones that didn't have them.
+  // Coz otherwise git commit starts a gc run IN THE BACKGROUD, which screws up trying to tarball it.
+  if (shellMeFail("cd %s/%s; git config gc.autodetach false", dir, gar))                                        E("Failed to git config gc.autodetach!");
+  // Don't want it running in the foreground either, coz it dumps on the console, we do a gc --quiet later anyway.
+  if (shellMeFail("cd %s/%s; git config gc.auto 0", dir, gar))                                                  E("Failed to git config gc.auto!");
+
+  // Git is such a pedantic bitch, let's just fucking ignore any errors it gives due to lack of anything to do.
+  // Even worse the OpenSim devs breaking logout tracking gives git plenty of nothing to do.  lol
+
+  // Loop through the .iar / .oar files in backups, ignoring zero sized files.
+  struct dirtree *new = dirtree_add_node(0, scBackup, 0);
+  int i;
+
+  ourARs->num = 0;
+  ourARs->this = xmprintf("%s-", name);
+  new->extra = (long) ourARs;
+  dirtree_handle_callback(new, filterARs);
+  qsort(ourARs->ARs, ourARs->num, sizeof(char *), qstrcmp);
+  for (i = 0; i < ourARs->num; i++)
+  {
+    I("Adding %s to %s", ourARs->ARs[i], gtr);
+    // Deal with deletions in the inventory / sim, easy method, which becomes a nop for files that stay in the git add below.
+    if (shellMeFail("cd %s/%s; rm -fr * >>%s", dir, gar, gerr))                                                 E("Failed to rm!");
+    if (shellMeFail("cd %s/%s; ionice -c3 nice -n 19 tar -xzf \"%s/%s\" || echo \"ERROR - Could not unpack %s !\" >>%s",
+      dir, gar, scBackup, ourARs->ARs[i], ourARs->ARs[i], gerr))                                                E("Failed to unpack %s!", ourARs->ARs[i]);
+    if (!hasContents(gerr))
+    {
+      if (shellMeFail("cd %s/%s; git add * >>%s", dir, gar, glog))                                              E("Failed to git add!");
+      // The \\* bit is to escape the \ from snprintf, which itself escapes the * from the shell.
+      if (shellMeFail("cd %s/%s; git add */\\* >>%s", dir, gar, gerr))                                          E("Failed to git add!");
+      // Gotta add this again, coz of the rm.  Apparently rm removes dotfiles, but add doesn't!
+      if (-1 == qfile_save(gab, gad, strlen(gad), false))                                                       E("Faild to write %s file!", gab);
+      if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gerr))                                 E("Failed to git add!");
+
+      // Magic needed to figure out if there's anything to commit.
+      // After all the pain to get this to work, there's an ever changing timestamp in archive.xml that screws it up.
+      // Like this system didn't have enough timestamps in it already.  lol
+      // TODO - I could sed out that timestamp, and put it back again based on the OAR file name when extracting.
+      //        IARs don't seem to have the timestamp.
+      int j = shellMe("cd %s/%s; t=$(git status --porcelain) && [ -z \"${t}\" ]", dir, gar);
+      if (!WIFEXITED(j))                                                                                        E("git status failed!");
+      else if (1 == WEXITSTATUS(j))
+      {
+        V("Committing changes from %s", ourARs->ARs[i]);
+        // Note this commit message has to be just the file name, as the ungitAR script uses it.
+        if (shellMeFail("cd %s/%s; git commit -a -qm \"%s\" >>%s || echo \"ERROR - Could not commit %s !\" >>%s ", 
+            dir, gar, ourARs->ARs[i], ourARs->ARs[i], glog, gerr))                                              E("Failed to git commit!");
+        if (hasContents(gerr))
+        {
+          free(ourARs->ARs[i]);
+          goto gitARend;
+        }
+      }
+      else
+        V("No changes to commit from %s.", ourARs->ARs[i]);
+    }
+    if (!hasContents(gerr))
+    {
+      if (shellMeFail("mv %s/%s %s", scBackup, ourARs->ARs[i], dir))                                            E("Failed to move %s!", ourARs->ARs[i]);
+    }
+    free(ourARs->ARs[i]);
+  }
+
+  if (!hasContents(gerr))
+  {
+    I("Checking git repo %s", gtr);
+    if (shellMeFail("cd %s/%s; git fsck --strict --lost-found --no-progress >>%s || echo \"ERROR - Problem with git fsck %s !\" >>%s ", 
+            dir, gar, glog, gtr, gerr))                                                                         E("Failed to git fsck!");
+    if (shellMeFail("cd %s/%s; git gc --aggressive --prune=all --quiet >>%s || echo \"ERROR - Problem with git gc %s !\" >>%s ", 
+            dir, gar, glog, gtr, gerr))                                                                         E("Failed to git gc!");
+    I("Compressing %s", gtr);
+    if (shellMeFail("cd %s; XZ_OPT='-9e' ionice -c3 nice -n 19 tar -c --xz %s -f %s || echo 'ERROR - Could not pack gitAR!' >>%s", 
+        dir, gar, gtr, gerr))                                                                                   E("Failed to git add!");
+  }
+gitARend:
+  if (hasContents(gerr))                                                                                        E("Failed to process the archives, look in %s for errors!", gerr);
+  else
+  {
+    if (shellMeFail("rm -fr %s", dir))                                                                          E("Failed to rm!");
+  }
+  free(ourARs->ARs);
+  free(ourARs->this);
+  free(ourARs);
+  free(gab);
+  free(gerr);
+  free(glog);
+  free(dir);
+  free(gtr);
+  free(gar);
+  free(name);
+}
 
 // Forward declare this.
 my_ulonglong dbCount(char *table, char *where);
@@ -2349,7 +2525,7 @@ byTab has the short name as the key, simData as the value.
             if ('\0' == rSync[0])
             {
               I("Running gitar on %s %s", sim, last);
-              shellMeFail("%s/current/bin/sledjchisl gitar -m %s %s %s", scRoot, FLAG(v) ? "-v" : "", sim, last);
+              gitar(simd, sim, count, window, panes, pane, true, member, last);
             }
             sendTmuxCmd(ourSims->backup, toybuf);
 //          if (0 == do)
@@ -2382,7 +2558,7 @@ byTab has the short name as the key, simData as the value.
         if ('\0' == rSync[0])
         {
           I("Running gitar on %s", simd->tab);
-          shellMeFail("%s/current/bin/sledjchisl gitar %s %s", scRoot, FLAG(v) ? "-v" : "", simd->tab);
+            gitar(simd, sim, count, window, panes, pane, false, member, last);
         }
         snprintf(toybuf, sizeof(toybuf), "save oar --all %s/%s-%s.oar", scBackup, simd->tab, date);
         sendTmuxCmd(simd->paneID, toybuf);
@@ -2407,179 +2583,10 @@ byTab has the short name as the key, simData as the value.
 
     case GITAR :        // "gitAR -m avatar name"  "gitAR sim name"
     {
-      /* Work around OpenSims slow database corruption bug by using git to store all old backups.
-        Try to squeeze every last byte out of the tarballs.  Seems to cut the total storage size down to one third the size of just the raw I/OAR files.
-        Saves even more if there's been no changes.
-        On the other hand, these backup files will grow indefinately, the more changes, the faster it grows.  I can live with that for more reliable backups that go back further.
-        Tries to avoid loosing data if things go wrong.  I think the main remaining problem would be running out of space, in which case you have bigger problems to deal with.
-
-        Strategy - unpack the last one, unpack and commit any old I/OARs, pack up the result, delete it's working directory, THEN run the save i/oar.
-        Avoids having to sync with OpenSim finishing the current I/OAR, and as a bonus, an easy to deliver latest I/OAR for people that want it.
-      */
-      char *name = xstrdup(sim);
-
-      if (FLAG(m))
-      {
-        if (member)
-        {
-          free(name);
-          name = xmprintf("%s_%s", sim, last);
-        }
-        else
-        {
-          free(name);
-          return;
-        }
-      }
-
-      char type  = FLAG(m) ? 'I' : 'O';
-      char *gar  = xmprintf("%s-git%cAR", name, type), *gtr  = xmprintf("%s/%s.tar.xz", scBackup, gar);
-      char *dir  = xmprintf("%s/temp_%c_%s", scBackup, type, name);
-      char *glog = xmprintf("%s/log.log", scBackup), *gerr = xmprintf("%s/errors", dir);
-      // Make sure stuff that's already compressed doesn't get compressed by git.
-      // Also tries to protect binaries from mangling.
-      char *gab = xmprintf("%s/%s/.gitattributes", dir, gar), *gad =
-        "*.dat -diff -text\n"
-        "*.bvh -delta -diff -text\n"
-        "*.j2c -delta -diff -text\n"    // OpenSim uses all four JPEG 2000 extensions.  In the same directories.  lol
-        "*.jp2 -delta -diff -text\n"
-        "*.jpc -delta -diff -text\n"
-        "*.jpg -delta -diff -text\n"
-        "*.llmesh -delta -diff -text\n"
-        "*.ogg -delta -diff -text\n"
-        "*.png -delta -diff -text\n"
-        "*.r32 -delta -diff -text\n"
-        "*.tga -delta -diff -text\n";
-      ARList *ourARs = xzalloc(sizeof(ARList));
-
-      if (qfile_exist(dir))
-      {
-        if (shellMeFail("echo 'Mess left over from last backup in %d, not gonna run!' >>%s", dir, gerr))                E("Cleaning up the mess!");
-        if (shellMeFail("mv %s/*.%car %s 2>&1 >>%s", dir, tolower(type), scBackup, gerr))                               E("Failed cleaning up the mess!");
-        goto gitARend;
-      }
-
-      // Either unpack the old gitAR, or create a new one.
-      qfile_mkdir(dir, S_IRWXU | S_IRGRP | S_IXGRP, true);
-      if (qfile_exist(gtr))
-      {
-        I("Unpacking %s", gtr);
-        if (shellMeFail("cd %s; ionice -c3 nice -n 19 tar -xf %s >>%s", dir, gtr, gerr))                                E("Failed to unpack %s!", gtr);
-        char *t = xmprintf("%s/%s_git%cAR", dir, name, type);
-        // Changed from _ to -, but deal with old archives.
-        if (qfile_exist(t))
-        {
-          if (shellMeFail("mv %s %s/%s", t, dir, gar))                                          E("Failed to move %s!", t);
-        }
-        free(t);
-      }
-      else
-      {
-        // git will create gar for us.
-        if (shellMeFail("cd %s; git init --quiet %s >>%s", dir, gar, gerr))                                             E("Failed to git init %s/%s!", dir, gar);
-
-        // Coz git insists.
-        if (shellMeFail("cd %s/%s; git config user.email \"opensim@$(hostname -A | cut -d ' ' -f 1)\"", dir, gar))      E("Failed to git config user.email!");
-        if (shellMeFail("cd %s/%s; git config user.name  \"opensim\"", dir, gar))                                       E("Failed to git config user.name!");
-
-        // Coz git insists on having something comitted before it can change the branch name.
-        if (-1 == qfile_save(gab, gad, strlen(gad), false))                                                             E("Faild to write %s file!", gad);
-        if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gerr))                                       E("Failed to git add!");
-        V("Committing initial git.");
-        if (shellMeFail("cd %s/%s; git commit -qm \"Initial commit\" >>%s || echo \"ERROR - Could not commit!\" >>%s",
-            dir, gar, glog, gerr))                                                                                      E("Failed to git commit!");
-        if (shellMeFail("cd %s/%s; git branch -m master Domme >>%s", dir, gar, gerr))                                   E("Failed to git branch -m master Domme!");
-      }
-      // Doing these each time, to catch those old ones that didn't have them.
-      // Coz otherwise git commit starts a gc run IN THE BACKGROUD, which screws up trying to tarball it.
-      if (shellMeFail("cd %s/%s; git config gc.autodetach false", dir, gar))                                            E("Failed to git config gc.autodetach!");
-      // Don't want it running in the foreground either, coz it dumps on the console, we do a gc --quiet later anyway.
-      if (shellMeFail("cd %s/%s; git config gc.auto 0", dir, gar))                                                      E("Failed to git config gc.auto!");
-
-      // Git is such a pedantic bitch, let's just fucking ignore any errors it gives due to lack of anything to do.
-      // Even worse the OpenSim devs breaking logout tracking gives git plenty of nothing to do.  lol
-
-      // Loop through the .iar / .oar files in backups, ignoring zero sized files.
-      struct dirtree *new = dirtree_add_node(0, scBackup, 0);
-      int i;
-
-      ourARs->num = 0;
-      ourARs->this = xmprintf("%s-", name);
-      new->extra = (long) ourARs;
-      dirtree_handle_callback(new, filterARs);
-      qsort(ourARs->ARs, ourARs->num, sizeof(char *), qstrcmp);
-      for (i = 0; i < ourARs->num; i++)
-      {
-        I("Adding %s to %s", ourARs->ARs[i], gtr);
-        // Deal with deletions in the inventory / sim, easy method, which becomes a nop for files that stay in the git add below.
-        if (shellMeFail("cd %s/%s; rm -fr * >>%s", dir, gar, gerr))                                                     E("Failed to rm!");
-        if (shellMeFail("cd %s/%s; ionice -c3 nice -n 19 tar -xzf \"%s/%s\" || echo \"ERROR - Could not unpack %s !\" >>%s",
-          dir, gar, scBackup, ourARs->ARs[i], ourARs->ARs[i], gerr))                                                    E("Failed to unpack %s!", ourARs->ARs[i]);
-        if (!hasContents(gerr))
-        {
-          if (shellMeFail("cd %s/%s; git add * >>%s", dir, gar, glog))                                                  E("Failed to git add!");
-          // The \\* bit is to escape the \ from snprintf, which itself escapes the * from the shell.
-          if (shellMeFail("cd %s/%s; git add */\\* >>%s", dir, gar, gerr))                                              E("Failed to git add!");
-          // Gotta add this again, coz of the rm.  Apparently rm removes dotfiles, but add doesn't!
-          if (-1 == qfile_save(gab, gad, strlen(gad), false))                                                           E("Faild to write %s file!", gab);
-          if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gerr))                                     E("Failed to git add!");
-
-          // Magic needed to figure out if there's anything to commit.
-          // After all the pain to get this to work, there's an ever changing timestamp in archive.xml that screws it up.
-          // Like this system didn't have enough timestamps in it already.  lol
-          // TODO - I could sed out that timestamp, and put it back again based on the OAR file name when extracting.
-          //        IARs don't seem to have the timestamp.
-          int j = shellMe("cd %s/%s; t=$(git status --porcelain) && [ -z \"${t}\" ]", dir, gar);
-          if (!WIFEXITED(j))                                                                                            E("git status failed!");
-          else if (1 == WEXITSTATUS(j))
-          {
-            V("Committing changes from %s", ourARs->ARs[i]);
-            // Note this commit message has to be just the file name, as the ungitAR script uses it.
-            if (shellMeFail("cd %s/%s; git commit -a -qm \"%s\" >>%s || echo \"ERROR - Could not commit %s !\" >>%s ", 
-                            dir, gar, ourARs->ARs[i], ourARs->ARs[i], glog, gerr))                                      E("Failed to git commit!");
-            if (hasContents(gerr))
-            {
-              free(ourARs->ARs[i]);
-              goto gitARend;
-            }
-          }
-          else
-            V("No changes to commit from %s.", ourARs->ARs[i]);
-        }
-        if (!hasContents(gerr))
-        {
-          if (shellMeFail("mv %s/%s %s", scBackup, ourARs->ARs[i], dir))                                                E("Failed to move %s!", ourARs->ARs[i]);
-        }
-        free(ourARs->ARs[i]);
-      }
+      gitar(simd, sim, count, window, panes, pane, FLAG(m), member, last);
+      break;
+    }
 
-      if (!hasContents(gerr))
-      {
-        I("Checking git repo %s", gtr);
-        if (shellMeFail("cd %s/%s; git fsck --strict --lost-found --no-progress >>%s || echo \"ERROR - Problem with git fsck %s !\" >>%s ", 
-                    dir, gar, glog, gtr, gerr))                                                                         E("Failed to git fsck!");
-        if (shellMeFail("cd %s/%s; git gc --aggressive --prune=all --quiet >>%s || echo \"ERROR - Problem with git gc %s !\" >>%s ", 
-                    dir, gar, glog, gtr, gerr))                                                                         E("Failed to git gc!");
-        I("Compressing %s", gtr);
-        if (shellMeFail("cd %s; XZ_OPT='-9e' ionice -c3 nice -n 19 tar -c --xz %s -f %s || echo 'ERROR - Could not pack gitAR!' >>%s", 
-            dir, gar, gtr, gerr))                                                                                       E("Failed to git add!");
-      }
-gitARend:
-      if (hasContents(gerr))                                                                                            E("Failed to process the archives, look in %s for errors!", gerr);
-      else
-      {
-        if (shellMeFail("rm -fr %s", dir))                                                                              E("Failed to rm!");
-      }
-      free(ourARs->ARs);
-      free(ourARs->this);
-      free(ourARs);
-      free(gab);
-      free(gerr);
-      free(glog);
-      free(dir);
-      free(gtr);
-      free(gar);
-      free(name);
       break;
     }