From 9a8e3b5a93ff3985a37fc4fc5bd4611a5f42b806 Mon Sep 17 00:00:00 2001 From: onefang Date: Thu, 14 Oct 2021 02:52:49 +1000 Subject: More tweaking of the backup system. --- src/sledjchisl/sledjchisl.c | 103 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 87 insertions(+), 16 deletions(-) diff --git a/src/sledjchisl/sledjchisl.c b/src/sledjchisl/sledjchisl.c index d8a7594..8e3a724 100644 --- a/src/sledjchisl/sledjchisl.c +++ b/src/sledjchisl/sledjchisl.c @@ -2266,7 +2266,10 @@ static int filterARs(struct dirtree *node) int l0 = strlen(node->name), l1 = strlen(list->this); // TODO - ignore zero sized ones. - if ((4 < l0) && ((strncmp(&(node->name[l0 - 4]), ".iar", 4) == 0) || (strncmp(&(node->name[l0 - 4]), ".oar", 4) == 0)) && (strncmp(node->name, list->this, l1) == 0)) + if (( + ((4 < l0) && ((strncmp(&(node->name[l0 - 4]), ".iar", 4) == 0) || (strncmp(&(node->name[l0 - 4]), ".oar", 4) == 0))) + || ((8 < l0) && ((strncmp(&(node->name[l0 - 8]), ".iar.tgz", 8) == 0) || (strncmp(&(node->name[l0 - 8]), ".oar.tgz", 8) == 0))) + ) && (strncmp(node->name, list->this, l1) == 0)) { if ((list->num + 1) > list->len) { @@ -2279,6 +2282,7 @@ static int filterARs(struct dirtree *node) return 0; } +void ungitar(simData *simd, char *sim, int count, int window, int panes, int pane, int m, int member, char *last); void gitar(simData *simd, char *sim, int count, int window, int panes, int pane, int m, int member, char *last) { /* Work around OpenSims slow database corruption bug by using git to store all old backups. @@ -2289,6 +2293,10 @@ void gitar(simData *simd, char *sim, int count, int window, int panes, int pane, Strategy - unpack the last one, unpack and commit any old I/OARs, pack up the result, delete it's working directory, THEN run the save i/oar. Avoids having to sync with OpenSim finishing the current I/OAR, and as a bonus, an easy to deliver latest I/OAR for people that want it. + +TODO - + Keep in mind some old files had munged names like "Tiffanie_s_Paradise-2021-06-23_05:11:38.oar" + Maybe deal with those manually? */ char *name = xstrdup(sim); @@ -2306,9 +2314,10 @@ void gitar(simData *simd, char *sim, int count, int window, int panes, int pane, } } - char type = m ? 'I' : 'O'; - char *gar = xmprintf("%s-git%cAR", name, type), *gtr = xmprintf("%s/%s.tar.xz", scBackup, gar); - char *dir = xmprintf("%s/temp_%c_%s", scBackup, type, name); + char TYPE = m ? 'I' : 'O'; + char type = m ? 'i' : 'o'; + char *gar = xmprintf("%s-git%cAR", name, TYPE), *gtr = xmprintf("%s/%s.tar.xz", scBackup, gar); + char *dir = xmprintf("%s/temp_%c_%s", scBackup, TYPE, name); char *glog = xmprintf("%s/log.log", scBackup), *gerr = xmprintf("%s/errors", dir); // Make sure stuff that's already compressed doesn't get compressed by git. // Also tries to protect binaries from mangling. @@ -2329,17 +2338,55 @@ void gitar(simData *simd, char *sim, int count, int window, int panes, int pane, if (qfile_exist(dir)) { if (shellMeFail("echo 'Mess left over from last backup in %d, not gonna run!' >>%s", dir, gerr)) E("Cleaning up the mess!"); - if (shellMeFail("mv %s/*.%car %s 2>&1 >>%s", dir, tolower(type), scBackup, gerr)) E("Failed cleaning up the mess!"); + if (shellMeFail("mv %s/*.%car %s 2>&1 >>%s", dir, type, scBackup, gerr)) E("Failed cleaning up the mess!"); goto gitARend; } + // Look for ancient ones first. + char *fullName = qstrreplace("tn", simd->name, " ", "_"); + char *gar1 = xmprintf("%s_git%cAR", fullName, TYPE), *gtr1 = xmprintf("%s/%s.tar.xz", scBackup, gar1); + +// TODO - refactor this. + if (qfile_exist(gtr1)) + { + I("Found ancient gitar %s", gtr1); + if (strcmp(fullName, sim) != 0) + ungitar(simd, fullName, count, window, panes, pane, FLAG(m), member, last); + ungitar(simd, sim, count, window, panes, pane, FLAG(m), member, last); + if (!qfile_mkdir(dir, S_IRWXU | S_IRGRP | S_IXGRP, true)) E("Faild to make directory %s!", dir); + if (shellMeFail("mv %s %s", gtr1, dir)) E("Failed to move %s!", gtr1); + if (qfile_exist(gtr)) + { + if (shellMeFail("mv %s %s", gtr, dir)) E("Failed to move %s!", gtr); + } + } + free(gtr1); + free(gar1); + gar1 = xmprintf("%s_git%cAR", name, TYPE); gtr1 = xmprintf("%s/%s.tar.xz", scBackup, gar1); + if (qfile_exist(gtr1)) + { + I("Found ancient gitar %s", gtr1); + ungitar(simd, sim, count, window, panes, pane, FLAG(m), member, last); + if (!qfile_mkdir(dir, S_IRWXU | S_IRGRP | S_IXGRP, true)) E("Faild to make directory %s!", dir); + if (shellMeFail("mv %s %s", gtr1, dir)) E("Failed to move %s!", gtr1); + if (qfile_exist(gtr)) + { + if (shellMeFail("mv %s %s", gtr, dir)) E("Failed to move %s!", gtr); + } + } + free(gtr1); + free(gar1); + // Either unpack the old gitAR, or create a new one. - qfile_mkdir(dir, S_IRWXU | S_IRGRP | S_IXGRP, true); + if (!qfile_exist(dir)) + { + if (!qfile_mkdir(dir, S_IRWXU | S_IRGRP | S_IXGRP, true)) E("Faild to make directory %s!", dir); + } if (qfile_exist(gtr)) { I("Unpacking %s", gtr); if (shellMeFail("cd %s; ionice -c3 nice -n 19 tar -xf %s >>%s", dir, gtr, gerr)) E("Failed to unpack %s!", gtr); - char *t = xmprintf("%s/%s_git%cAR", dir, name, type); + char *t = xmprintf("%s/%s_git%cAR", dir, name, TYPE); // Changed from _ to -, but deal with old archives. if (qfile_exist(t)) { @@ -2377,13 +2424,40 @@ void gitar(simData *simd, char *sim, int count, int window, int panes, int pane, struct dirtree *new = dirtree_add_node(0, scBackup, 0); int i; + new->extra = (long) ourARs; ourARs->num = 0; + if (strcmp(sim, fullName) != 0) + { + struct dirtree *new1 = dirtree_add_node(0, scBackup, 0); + + new1->extra = (long) ourARs; + ourARs->this = xmprintf("%s-", fullName); + dirtree_handle_callback(new1, filterARs); + for (i = 0; i < ourARs->num; i++) + { + char *t = ourARs->ARs[i]; + int l0 = strlen(t), l1 = strlen(ourARs->this); + + V("Renaming %s -> %s-%s.tgz", t, name, &t[l1]); + if (shellMeFail("cd %s; mv %s %s-%s.tgz", scBackup, t, name, &t[l1])) E("Failed to mv %s -> %s-%s.tgz!", t, name, &t[l1]); + } + free(ourARs->ARs); + ourARs->ARs = NULL; + ourARs->len = 0; + free(ourARs->this); + ourARs->num = 0; + } ourARs->this = xmprintf("%s-", name); - new->extra = (long) ourARs; dirtree_handle_callback(new, filterARs); qsort(ourARs->ARs, ourARs->num, sizeof(char *), qstrcmp); for (i = 0; i < ourARs->num; i++) { + char *t = xstrdup(ourARs->ARs[i]); + int l = strlen(t); + + if (strcmp(".tgz", &t[l - 4]) == 0) {t[l - 4] = '\0'; l -= 4;} + if (strcmp(".iar", &t[l - 4]) == 0) {t[l - 4] = '\0'; l -= 4;} + if (strcmp(".oar", &t[l - 4]) == 0) {t[l - 4] = '\0'; l -= 4;} I("Adding %s to %s", ourARs->ARs[i], gtr); // Deal with deletions in the inventory / sim, easy method, which becomes a nop for files that stay in the git add below. if (shellMeFail("cd %s/%s; rm -fr * >>%s", dir, gar, gerr)) E("Failed to rm!"); @@ -2410,7 +2484,7 @@ void gitar(simData *simd, char *sim, int count, int window, int panes, int pane, V("Committing changes from %s", ourARs->ARs[i]); // Note this commit message has to be just the file name, as the ungitAR script uses it. if (shellMeFail("cd %s/%s; git commit -a -qm \"%s\" >>%s || echo \"ERROR - Could not commit %s !\" >>%s ", - dir, gar, ourARs->ARs[i], ourARs->ARs[i], glog, gerr)) E("Failed to git commit!"); + dir, gar, t, glog, t, gerr)) E("Failed to git commit!"); if (hasContents(gerr)) { free(ourARs->ARs[i]); @@ -2422,8 +2496,9 @@ void gitar(simData *simd, char *sim, int count, int window, int panes, int pane, } if (!hasContents(gerr)) { - if (shellMeFail("mv %s/%s %s", scBackup, ourARs->ARs[i], dir)) E("Failed to move %s!", ourARs->ARs[i]); + if (shellMeFail("mv %s/%s %s/", scBackup, ourARs->ARs[i], dir)) E("Failed to move %s!", ourARs->ARs[i]); } + free(t); free(ourARs->ARs[i]); } @@ -2444,6 +2519,7 @@ gitARend: { if (shellMeFail("rm -fr %s", dir)) E("Failed to rm!"); } + free(fullName); free(ourARs->ARs); free(ourARs->this); free(ourARs); @@ -2555,14 +2631,13 @@ void ungitar(simData *simd, char *sim, int count, int window, int panes, int pan { I("Extracting %s", prev); free(prev); - prev = xmprintf("%s/%s", dir, out); + prev = xmprintf("%s/%s-%s", scBackup, simd->tab, &out[l - 24]); // Converting any old full names to new names. // OpenSim insists on the archive.xml file being the very first in the tarball. if (shellMeFail("cd %s/%s; ionice -c3 nice -n 19 tar -c archive.xml -f %s", dir, gar, prev)) E("Failed to tar %s!", prev); if (shellMeFail("cd %s/%s; ionice -c3 nice -n 19 tar -r -f %s --exclude='.git*' --exclude='archive.xml' * ", dir, gar, prev)) E("Failed to tar %s!", prev); // Using gzip instead of something that compresses better, coz OpenSim only knows how to deal with gzipped files. if (shellMeFail("cd %s/%s; ionice -c3 nice -n 19 gzip -S .tgz %s", dir, gar, prev)) E("Failed to gzip %s!", prev); - if (shellMeFail("cd %s/%s; mv %s.tgz ../..", dir, gar, prev)) E("Failed to rename %s!", prev); } else W("NOT extracting %s, it already exists.", prev); @@ -2690,10 +2765,6 @@ byTab has the short name as the key, simData as the value. } else if (checkSimIsRunning(simd->tab)) { -// TODO - should collect names of existing backups, both tmux / ini name and proper name. -// Scan backups directory once before this for loop, add details to sims list. -// strip off the last bit of file name (YYYY-mm-dd_HH:MM:SS.oar) to get the name -// keep in mind some old files had munged names like "Tiffanie_s_Paradise-2021-06-23_05:11:38.oar" I("Sim %s is being backed up to %s/backups/%s-%s.oar.", simd->name, scRoot, simd->tab, date); if ('\0' == rSync[0]) { -- cgit v1.1