From 13ea9a731ea344524c724a9c285fb3f8f74ae1c3 Mon Sep 17 00:00:00 2001 From: onefang Date: Fri, 8 Oct 2021 00:23:39 +1000 Subject: Refactor GITAR, so we can call it direct from BACKUP. --- src/sledjchisl/sledjchisl.c | 355 ++++++++++++++++++++++---------------------- 1 file changed, 181 insertions(+), 174 deletions(-) (limited to 'src') diff --git a/src/sledjchisl/sledjchisl.c b/src/sledjchisl/sledjchisl.c index aca04d6..b74e419 100644 --- a/src/sledjchisl/sledjchisl.c +++ b/src/sledjchisl/sledjchisl.c @@ -2271,6 +2271,182 @@ static int filterARs(struct dirtree *node) return 0; } +void gitar(simData *simd, char *sim, int count, int window, int panes, int pane, int m, int member, char *last) +{ + /* Work around OpenSims slow database corruption bug by using git to store all old backups. + Try to squeeze every last byte out of the tarballs. Seems to cut the total storage size down to one third the size of just the raw I/OAR files. + Saves even more if there's been no changes. + On the other hand, these backup files will grow indefinately, the more changes, the faster it grows. I can live with that for more reliable backups that go back further. + Tries to avoid loosing data if things go wrong. I think the main remaining problem would be running out of space, in which case you have bigger problems to deal with. + + Strategy - unpack the last one, unpack and commit any old I/OARs, pack up the result, delete it's working directory, THEN run the save i/oar. + Avoids having to sync with OpenSim finishing the current I/OAR, and as a bonus, an easy to deliver latest I/OAR for people that want it. + */ + char *name = xstrdup(sim); + + if (m) + { + if (member) + { + free(name); + name = xmprintf("%s_%s", sim, last); + } + else + { + free(name); + return; + } + } + + char type = m ? 'I' : 'O'; + char *gar = xmprintf("%s-git%cAR", name, type), *gtr = xmprintf("%s/%s.tar.xz", scBackup, gar); + char *dir = xmprintf("%s/temp_%c_%s", scBackup, type, name); + char *glog = xmprintf("%s/log.log", scBackup), *gerr = xmprintf("%s/errors", dir); + // Make sure stuff that's already compressed doesn't get compressed by git. + // Also tries to protect binaries from mangling. + char *gab = xmprintf("%s/%s/.gitattributes", dir, gar), *gad = + "*.dat -diff -text\n" + "*.bvh -delta -diff -text\n" + "*.j2c -delta -diff -text\n" // OpenSim uses all four JPEG 2000 extensions. In the same directories. lol + "*.jp2 -delta -diff -text\n" + "*.jpc -delta -diff -text\n" + "*.jpg -delta -diff -text\n" + "*.llmesh -delta -diff -text\n" + "*.ogg -delta -diff -text\n" + "*.png -delta -diff -text\n" + "*.r32 -delta -diff -text\n" + "*.tga -delta -diff -text\n"; + ARList *ourARs = xzalloc(sizeof(ARList)); + + if (qfile_exist(dir)) + { + if (shellMeFail("echo 'Mess left over from last backup in %d, not gonna run!' >>%s", dir, gerr)) E("Cleaning up the mess!"); + if (shellMeFail("mv %s/*.%car %s 2>&1 >>%s", dir, tolower(type), scBackup, gerr)) E("Failed cleaning up the mess!"); + goto gitARend; + } + + // Either unpack the old gitAR, or create a new one. + qfile_mkdir(dir, S_IRWXU | S_IRGRP | S_IXGRP, true); + if (qfile_exist(gtr)) + { + I("Unpacking %s", gtr); + if (shellMeFail("cd %s; ionice -c3 nice -n 19 tar -xf %s >>%s", dir, gtr, gerr)) E("Failed to unpack %s!", gtr); + char *t = xmprintf("%s/%s_git%cAR", dir, name, type); + // Changed from _ to -, but deal with old archives. + if (qfile_exist(t)) + { + if (shellMeFail("mv %s %s/%s", t, dir, gar)) E("Failed to move %s!", t); + } + free(t); + } + else + { + // git will create gar for us. + if (shellMeFail("cd %s; git init --quiet %s >>%s", dir, gar, gerr)) E("Failed to git init %s/%s!", dir, gar); + + // Coz git insists. + if (shellMeFail("cd %s/%s; git config user.email \"opensim@$(hostname -A | cut -d ' ' -f 1)\"", dir, gar)) E("Failed to git config user.email!"); + if (shellMeFail("cd %s/%s; git config user.name \"opensim\"", dir, gar)) E("Failed to git config user.name!"); + + // Coz git insists on having something comitted before it can change the branch name. + if (-1 == qfile_save(gab, gad, strlen(gad), false)) E("Faild to write %s file!", gad); + if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gerr)) E("Failed to git add!"); + V("Committing initial git."); + if (shellMeFail("cd %s/%s; git commit -qm \"Initial commit\" >>%s || echo \"ERROR - Could not commit!\" >>%s", + dir, gar, glog, gerr)) E("Failed to git commit!"); + if (shellMeFail("cd %s/%s; git branch -m master Domme >>%s", dir, gar, gerr)) E("Failed to git branch -m master Domme!"); + } + // Doing these each time, to catch those old ones that didn't have them. + // Coz otherwise git commit starts a gc run IN THE BACKGROUD, which screws up trying to tarball it. + if (shellMeFail("cd %s/%s; git config gc.autodetach false", dir, gar)) E("Failed to git config gc.autodetach!"); + // Don't want it running in the foreground either, coz it dumps on the console, we do a gc --quiet later anyway. + if (shellMeFail("cd %s/%s; git config gc.auto 0", dir, gar)) E("Failed to git config gc.auto!"); + + // Git is such a pedantic bitch, let's just fucking ignore any errors it gives due to lack of anything to do. + // Even worse the OpenSim devs breaking logout tracking gives git plenty of nothing to do. lol + + // Loop through the .iar / .oar files in backups, ignoring zero sized files. + struct dirtree *new = dirtree_add_node(0, scBackup, 0); + int i; + + ourARs->num = 0; + ourARs->this = xmprintf("%s-", name); + new->extra = (long) ourARs; + dirtree_handle_callback(new, filterARs); + qsort(ourARs->ARs, ourARs->num, sizeof(char *), qstrcmp); + for (i = 0; i < ourARs->num; i++) + { + I("Adding %s to %s", ourARs->ARs[i], gtr); + // Deal with deletions in the inventory / sim, easy method, which becomes a nop for files that stay in the git add below. + if (shellMeFail("cd %s/%s; rm -fr * >>%s", dir, gar, gerr)) E("Failed to rm!"); + if (shellMeFail("cd %s/%s; ionice -c3 nice -n 19 tar -xzf \"%s/%s\" || echo \"ERROR - Could not unpack %s !\" >>%s", + dir, gar, scBackup, ourARs->ARs[i], ourARs->ARs[i], gerr)) E("Failed to unpack %s!", ourARs->ARs[i]); + if (!hasContents(gerr)) + { + if (shellMeFail("cd %s/%s; git add * >>%s", dir, gar, glog)) E("Failed to git add!"); + // The \\* bit is to escape the \ from snprintf, which itself escapes the * from the shell. + if (shellMeFail("cd %s/%s; git add */\\* >>%s", dir, gar, gerr)) E("Failed to git add!"); + // Gotta add this again, coz of the rm. Apparently rm removes dotfiles, but add doesn't! + if (-1 == qfile_save(gab, gad, strlen(gad), false)) E("Faild to write %s file!", gab); + if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gerr)) E("Failed to git add!"); + + // Magic needed to figure out if there's anything to commit. + // After all the pain to get this to work, there's an ever changing timestamp in archive.xml that screws it up. + // Like this system didn't have enough timestamps in it already. lol + // TODO - I could sed out that timestamp, and put it back again based on the OAR file name when extracting. + // IARs don't seem to have the timestamp. + int j = shellMe("cd %s/%s; t=$(git status --porcelain) && [ -z \"${t}\" ]", dir, gar); + if (!WIFEXITED(j)) E("git status failed!"); + else if (1 == WEXITSTATUS(j)) + { + V("Committing changes from %s", ourARs->ARs[i]); + // Note this commit message has to be just the file name, as the ungitAR script uses it. + if (shellMeFail("cd %s/%s; git commit -a -qm \"%s\" >>%s || echo \"ERROR - Could not commit %s !\" >>%s ", + dir, gar, ourARs->ARs[i], ourARs->ARs[i], glog, gerr)) E("Failed to git commit!"); + if (hasContents(gerr)) + { + free(ourARs->ARs[i]); + goto gitARend; + } + } + else + V("No changes to commit from %s.", ourARs->ARs[i]); + } + if (!hasContents(gerr)) + { + if (shellMeFail("mv %s/%s %s", scBackup, ourARs->ARs[i], dir)) E("Failed to move %s!", ourARs->ARs[i]); + } + free(ourARs->ARs[i]); + } + + if (!hasContents(gerr)) + { + I("Checking git repo %s", gtr); + if (shellMeFail("cd %s/%s; git fsck --strict --lost-found --no-progress >>%s || echo \"ERROR - Problem with git fsck %s !\" >>%s ", + dir, gar, glog, gtr, gerr)) E("Failed to git fsck!"); + if (shellMeFail("cd %s/%s; git gc --aggressive --prune=all --quiet >>%s || echo \"ERROR - Problem with git gc %s !\" >>%s ", + dir, gar, glog, gtr, gerr)) E("Failed to git gc!"); + I("Compressing %s", gtr); + if (shellMeFail("cd %s; XZ_OPT='-9e' ionice -c3 nice -n 19 tar -c --xz %s -f %s || echo 'ERROR - Could not pack gitAR!' >>%s", + dir, gar, gtr, gerr)) E("Failed to git add!"); + } +gitARend: + if (hasContents(gerr)) E("Failed to process the archives, look in %s for errors!", gerr); + else + { + if (shellMeFail("rm -fr %s", dir)) E("Failed to rm!"); + } + free(ourARs->ARs); + free(ourARs->this); + free(ourARs); + free(gab); + free(gerr); + free(glog); + free(dir); + free(gtr); + free(gar); + free(name); +} // Forward declare this. my_ulonglong dbCount(char *table, char *where); @@ -2349,7 +2525,7 @@ byTab has the short name as the key, simData as the value. if ('\0' == rSync[0]) { I("Running gitar on %s %s", sim, last); - shellMeFail("%s/current/bin/sledjchisl gitar -m %s %s %s", scRoot, FLAG(v) ? "-v" : "", sim, last); + gitar(simd, sim, count, window, panes, pane, true, member, last); } sendTmuxCmd(ourSims->backup, toybuf); // if (0 == do) @@ -2382,7 +2558,7 @@ byTab has the short name as the key, simData as the value. if ('\0' == rSync[0]) { I("Running gitar on %s", simd->tab); - shellMeFail("%s/current/bin/sledjchisl gitar %s %s", scRoot, FLAG(v) ? "-v" : "", simd->tab); + gitar(simd, sim, count, window, panes, pane, false, member, last); } snprintf(toybuf, sizeof(toybuf), "save oar --all %s/%s-%s.oar", scBackup, simd->tab, date); sendTmuxCmd(simd->paneID, toybuf); @@ -2407,179 +2583,10 @@ byTab has the short name as the key, simData as the value. case GITAR : // "gitAR -m avatar name" "gitAR sim name" { - /* Work around OpenSims slow database corruption bug by using git to store all old backups. - Try to squeeze every last byte out of the tarballs. Seems to cut the total storage size down to one third the size of just the raw I/OAR files. - Saves even more if there's been no changes. - On the other hand, these backup files will grow indefinately, the more changes, the faster it grows. I can live with that for more reliable backups that go back further. - Tries to avoid loosing data if things go wrong. I think the main remaining problem would be running out of space, in which case you have bigger problems to deal with. - - Strategy - unpack the last one, unpack and commit any old I/OARs, pack up the result, delete it's working directory, THEN run the save i/oar. - Avoids having to sync with OpenSim finishing the current I/OAR, and as a bonus, an easy to deliver latest I/OAR for people that want it. - */ - char *name = xstrdup(sim); - - if (FLAG(m)) - { - if (member) - { - free(name); - name = xmprintf("%s_%s", sim, last); - } - else - { - free(name); - return; - } - } - - char type = FLAG(m) ? 'I' : 'O'; - char *gar = xmprintf("%s-git%cAR", name, type), *gtr = xmprintf("%s/%s.tar.xz", scBackup, gar); - char *dir = xmprintf("%s/temp_%c_%s", scBackup, type, name); - char *glog = xmprintf("%s/log.log", scBackup), *gerr = xmprintf("%s/errors", dir); - // Make sure stuff that's already compressed doesn't get compressed by git. - // Also tries to protect binaries from mangling. - char *gab = xmprintf("%s/%s/.gitattributes", dir, gar), *gad = - "*.dat -diff -text\n" - "*.bvh -delta -diff -text\n" - "*.j2c -delta -diff -text\n" // OpenSim uses all four JPEG 2000 extensions. In the same directories. lol - "*.jp2 -delta -diff -text\n" - "*.jpc -delta -diff -text\n" - "*.jpg -delta -diff -text\n" - "*.llmesh -delta -diff -text\n" - "*.ogg -delta -diff -text\n" - "*.png -delta -diff -text\n" - "*.r32 -delta -diff -text\n" - "*.tga -delta -diff -text\n"; - ARList *ourARs = xzalloc(sizeof(ARList)); - - if (qfile_exist(dir)) - { - if (shellMeFail("echo 'Mess left over from last backup in %d, not gonna run!' >>%s", dir, gerr)) E("Cleaning up the mess!"); - if (shellMeFail("mv %s/*.%car %s 2>&1 >>%s", dir, tolower(type), scBackup, gerr)) E("Failed cleaning up the mess!"); - goto gitARend; - } - - // Either unpack the old gitAR, or create a new one. - qfile_mkdir(dir, S_IRWXU | S_IRGRP | S_IXGRP, true); - if (qfile_exist(gtr)) - { - I("Unpacking %s", gtr); - if (shellMeFail("cd %s; ionice -c3 nice -n 19 tar -xf %s >>%s", dir, gtr, gerr)) E("Failed to unpack %s!", gtr); - char *t = xmprintf("%s/%s_git%cAR", dir, name, type); - // Changed from _ to -, but deal with old archives. - if (qfile_exist(t)) - { - if (shellMeFail("mv %s %s/%s", t, dir, gar)) E("Failed to move %s!", t); - } - free(t); - } - else - { - // git will create gar for us. - if (shellMeFail("cd %s; git init --quiet %s >>%s", dir, gar, gerr)) E("Failed to git init %s/%s!", dir, gar); - - // Coz git insists. - if (shellMeFail("cd %s/%s; git config user.email \"opensim@$(hostname -A | cut -d ' ' -f 1)\"", dir, gar)) E("Failed to git config user.email!"); - if (shellMeFail("cd %s/%s; git config user.name \"opensim\"", dir, gar)) E("Failed to git config user.name!"); - - // Coz git insists on having something comitted before it can change the branch name. - if (-1 == qfile_save(gab, gad, strlen(gad), false)) E("Faild to write %s file!", gad); - if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gerr)) E("Failed to git add!"); - V("Committing initial git."); - if (shellMeFail("cd %s/%s; git commit -qm \"Initial commit\" >>%s || echo \"ERROR - Could not commit!\" >>%s", - dir, gar, glog, gerr)) E("Failed to git commit!"); - if (shellMeFail("cd %s/%s; git branch -m master Domme >>%s", dir, gar, gerr)) E("Failed to git branch -m master Domme!"); - } - // Doing these each time, to catch those old ones that didn't have them. - // Coz otherwise git commit starts a gc run IN THE BACKGROUD, which screws up trying to tarball it. - if (shellMeFail("cd %s/%s; git config gc.autodetach false", dir, gar)) E("Failed to git config gc.autodetach!"); - // Don't want it running in the foreground either, coz it dumps on the console, we do a gc --quiet later anyway. - if (shellMeFail("cd %s/%s; git config gc.auto 0", dir, gar)) E("Failed to git config gc.auto!"); - - // Git is such a pedantic bitch, let's just fucking ignore any errors it gives due to lack of anything to do. - // Even worse the OpenSim devs breaking logout tracking gives git plenty of nothing to do. lol - - // Loop through the .iar / .oar files in backups, ignoring zero sized files. - struct dirtree *new = dirtree_add_node(0, scBackup, 0); - int i; - - ourARs->num = 0; - ourARs->this = xmprintf("%s-", name); - new->extra = (long) ourARs; - dirtree_handle_callback(new, filterARs); - qsort(ourARs->ARs, ourARs->num, sizeof(char *), qstrcmp); - for (i = 0; i < ourARs->num; i++) - { - I("Adding %s to %s", ourARs->ARs[i], gtr); - // Deal with deletions in the inventory / sim, easy method, which becomes a nop for files that stay in the git add below. - if (shellMeFail("cd %s/%s; rm -fr * >>%s", dir, gar, gerr)) E("Failed to rm!"); - if (shellMeFail("cd %s/%s; ionice -c3 nice -n 19 tar -xzf \"%s/%s\" || echo \"ERROR - Could not unpack %s !\" >>%s", - dir, gar, scBackup, ourARs->ARs[i], ourARs->ARs[i], gerr)) E("Failed to unpack %s!", ourARs->ARs[i]); - if (!hasContents(gerr)) - { - if (shellMeFail("cd %s/%s; git add * >>%s", dir, gar, glog)) E("Failed to git add!"); - // The \\* bit is to escape the \ from snprintf, which itself escapes the * from the shell. - if (shellMeFail("cd %s/%s; git add */\\* >>%s", dir, gar, gerr)) E("Failed to git add!"); - // Gotta add this again, coz of the rm. Apparently rm removes dotfiles, but add doesn't! - if (-1 == qfile_save(gab, gad, strlen(gad), false)) E("Faild to write %s file!", gab); - if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gerr)) E("Failed to git add!"); - - // Magic needed to figure out if there's anything to commit. - // After all the pain to get this to work, there's an ever changing timestamp in archive.xml that screws it up. - // Like this system didn't have enough timestamps in it already. lol - // TODO - I could sed out that timestamp, and put it back again based on the OAR file name when extracting. - // IARs don't seem to have the timestamp. - int j = shellMe("cd %s/%s; t=$(git status --porcelain) && [ -z \"${t}\" ]", dir, gar); - if (!WIFEXITED(j)) E("git status failed!"); - else if (1 == WEXITSTATUS(j)) - { - V("Committing changes from %s", ourARs->ARs[i]); - // Note this commit message has to be just the file name, as the ungitAR script uses it. - if (shellMeFail("cd %s/%s; git commit -a -qm \"%s\" >>%s || echo \"ERROR - Could not commit %s !\" >>%s ", - dir, gar, ourARs->ARs[i], ourARs->ARs[i], glog, gerr)) E("Failed to git commit!"); - if (hasContents(gerr)) - { - free(ourARs->ARs[i]); - goto gitARend; - } - } - else - V("No changes to commit from %s.", ourARs->ARs[i]); - } - if (!hasContents(gerr)) - { - if (shellMeFail("mv %s/%s %s", scBackup, ourARs->ARs[i], dir)) E("Failed to move %s!", ourARs->ARs[i]); - } - free(ourARs->ARs[i]); - } + gitar(simd, sim, count, window, panes, pane, FLAG(m), member, last); + break; + } - if (!hasContents(gerr)) - { - I("Checking git repo %s", gtr); - if (shellMeFail("cd %s/%s; git fsck --strict --lost-found --no-progress >>%s || echo \"ERROR - Problem with git fsck %s !\" >>%s ", - dir, gar, glog, gtr, gerr)) E("Failed to git fsck!"); - if (shellMeFail("cd %s/%s; git gc --aggressive --prune=all --quiet >>%s || echo \"ERROR - Problem with git gc %s !\" >>%s ", - dir, gar, glog, gtr, gerr)) E("Failed to git gc!"); - I("Compressing %s", gtr); - if (shellMeFail("cd %s; XZ_OPT='-9e' ionice -c3 nice -n 19 tar -c --xz %s -f %s || echo 'ERROR - Could not pack gitAR!' >>%s", - dir, gar, gtr, gerr)) E("Failed to git add!"); - } -gitARend: - if (hasContents(gerr)) E("Failed to process the archives, look in %s for errors!", gerr); - else - { - if (shellMeFail("rm -fr %s", dir)) E("Failed to rm!"); - } - free(ourARs->ARs); - free(ourARs->this); - free(ourARs); - free(gab); - free(gerr); - free(glog); - free(dir); - free(gtr); - free(gar); - free(name); break; } -- cgit v1.1