From 84402e4fad71cd1c7490eaee829815111629d981 Mon Sep 17 00:00:00 2001 From: onefang Date: Fri, 20 Aug 2021 21:38:12 +1000 Subject: More work on gitAR code. More polish, more working around git things. --- src/sledjchisl/sledjchisl.c | 133 +++++++++++++++++++++++--------------------- 1 file changed, 70 insertions(+), 63 deletions(-) diff --git a/src/sledjchisl/sledjchisl.c b/src/sledjchisl/sledjchisl.c index 175cf4d..e076654 100644 --- a/src/sledjchisl/sledjchisl.c +++ b/src/sledjchisl/sledjchisl.c @@ -701,6 +701,19 @@ int shellMe(char *format, ...) } +int hasContents(char *file) +{ + struct stat st; + + if (0 == lstat(file, &st)) + { + if (0 != st.st_size) + return TRUE; + } + return FALSE; +} + + static void addStrL(qlist_t *list, char *s) { list->addlast(list, s, strlen(s) + 1); @@ -2053,7 +2066,10 @@ byTab has the short name as the key, simData as the value. if (NULL != ourSims->backup) { if ('\0' == rSync[0]) + { + I("Running gitar on %s %s", sim, last); shellMeFail("%s/current/bin/sledjchisl gitar -m %s %s %s", scRoot, FLAG(v) ? "-v" : "", sim, last); + } sendTmuxCmd(ourSims->backup, toybuf); // if (0 == do) { @@ -2083,7 +2099,10 @@ byTab has the short name as the key, simData as the value. // keep in mind some old files had munged names like "Tiffanie_s_Paradise-2021-06-23_05:11:38.oar" I("Sim %s is being backed up to %s/backups/%s-%s.oar.", simd->name, scRoot, simd->tab, date); if ('\0' == rSync[0]) + { + I("Running gitar on %s", simd->tab); shellMeFail("%s/current/bin/sledjchisl gitar %s %s", scRoot, FLAG(v) ? "-v" : "", simd->tab); + } snprintf(toybuf, sizeof(toybuf), "save oar --all %s/%s-%s.oar", scBackup, simd->tab, date); sendTmuxCmd(simd->paneID, toybuf); // if (0 == do) @@ -2133,15 +2152,12 @@ byTab has the short name as the key, simData as the value. } char type = FLAG(m) ? 'I' : 'O'; - char *gar = xmprintf("%s_git%cAR", name, type); - char *gtr = xmprintf("%s/%s-git%cAR.tar.xz", scBackup, name, type); - char *dir = xmprintf("%s/temp_backups%c_%s", scBackup, type, name); + char *gar = xmprintf("%s-git%cAR", name, type), *gtr = xmprintf("%s/%s.tar.xz", scBackup, gar); + char *dir = xmprintf("%s/temp_%c_%s", scBackup, type, name); + char *glog = xmprintf("%s/log.log", scBackup), *gerr = xmprintf("%s/errors", dir); // Make sure stuff that's already compressed doesn't get compressed by git. // Also tries to protect binaries from mangling. - char *gab = xmprintf("%s/%s/.gitattributes", dir, gar); - char *ga = xmprintf("%s/%s", dir, gar); - char *gal = xmprintf("%s/log", scBackup); - char *gad = + char *gab = xmprintf("%s/%s/.gitattributes", dir, gar), *gad = "*.bvh -delta -diff -text\n" "*.jp2 -delta -diff -text\n" "*.jpg -delta -diff -text\n" @@ -2154,10 +2170,8 @@ byTab has the short name as the key, simData as the value. if (qfile_exist(dir)) { - if (shellMeFail("echo 'Mess left over from last backup in %d, not gonna run!' >>%s/errors", dir)) - E("Failed cleaning up the mess!"); - if (shellMeFail("mv %s/*.%car %s 2>&1 >/dev/null", dir, tolower(type), scBackup)) - E("Failed cleaning up the mess!"); + if (shellMeFail("echo 'Mess left over from last backup in %d, not gonna run!' >>%s", dir, gerr)) E("Cleaning up the mess!"); + if (shellMeFail("mv %s/*.%car %s 2>&1 >>%s", dir, tolower(type), scBackup, gerr)) E("Failed cleaning up the mess!"); goto gitARend; } @@ -2166,34 +2180,38 @@ byTab has the short name as the key, simData as the value. if (qfile_exist(gtr)) { I("Unpacking %s", gtr); - if (shellMeFail("cd %s; ionice -c3 nice -n 19 tar -xf %s", dir, gtr)) - E("Failed to unpack %s!", gtr); + if (shellMeFail("cd %s; ionice -c3 nice -n 19 tar -xf %s >>%s", dir, gtr, gerr)) E("Failed to unpack %s!", gtr); + char *t = xmprintf("%s/%s_git%cAR", dir, name, type); + // Changed from _ to -, but deal with old archives. + if (qfile_exist(t)) + { + if (shellMeFail("mv %s %s/%s", t, dir, gar)) E("Failed to move %s!", t); + } + free(t); } else { qfile_mkdir(gar, S_IRWXU | S_IRGRP | S_IXGRP, true); // git will create gar for us. - if (shellMeFail("cd %s; git init %s >%s", dir, gar, gal)) - E("Failed to git init %s/%s!", dir, gar); + if (shellMeFail("cd %s; git init --quiet %s >>%s", dir, gar, gerr)) E("Failed to git init %s/%s!", dir, gar); // Coz git insists. - if (shellMeFail("cd %s/%s; git config user.email \"opensim@$(hostname -A | cut -d ' ' -f 1)\"", dir, gar)) - E("Failed to git config user.email!"); - if (shellMeFail("cd %s/%s; git config user.name \"opensim\"", dir, gar)) - E("Failed to git config user.name!"); + if (shellMeFail("cd %s/%s; git config user.email \"opensim@$(hostname -A | cut -d ' ' -f 1)\"", dir, gar)) E("Failed to git config user.email!"); + if (shellMeFail("cd %s/%s; git config user.name \"opensim\"", dir, gar)) E("Failed to git config user.name!"); // Coz git insists on having something comitted before it can change the branch name. - if (-1 == qfile_save(gab, gad, strlen(gad), false)) - E("Faild to write %s file!", ga); - if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gal)) - E("Failed to git add!"); + if (-1 == qfile_save(gab, gad, strlen(gad), false)) E("Faild to write %s file!", gad); + if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gerr)) E("Failed to git add!"); V("Committing initial git."); - if (shellMeFail("cd %s/%s; git commit -qm \"Initial commit\" >>%s || echo \"ERROR - Could not commit!\" >>%s/errors ", - dir, gar, gal, dir)) - E("Failed to git commit!"); - if (shellMeFail("cd %s/%s; git branch -m master Domme >>%s", dir, gar, gal)) - E("Failed to git branch -m master Domme!"); + if (shellMeFail("cd %s/%s; git commit -qm \"Initial commit\" >>%s || echo \"ERROR - Could not commit!\" >>%s", + dir, gar, glog, gerr)) E("Failed to git commit!"); + if (shellMeFail("cd %s/%s; git branch -m master Domme >>%s", dir, gar, gerr)) E("Failed to git branch -m master Domme!"); } + // Doing these each time, to catch those old ones that didn't have them. + // Coz otherwise git commit starts a gc run IN THE BACKGROUD, which screws up trying to tarball it. + if (shellMeFail("cd %s/%s; git config gc.autodetach false", dir, gar)) E("Failed to git config gc.autodetach!"); + // Don't want it running in the foreground either, coz it dumps on the console, we do a gc --quiet later anyway. + if (shellMeFail("cd %s/%s; git config gc.auto 0", dir, gar)) E("Failed to git config gc.auto!"); // Git is such a pedantic bitch, let's just fucking ignore any errors it gives due to lack of anything to do. // Even worse the OpenSim devs breaking logout tracking gives git plenty of nothing to do. lol @@ -2207,29 +2225,21 @@ byTab has the short name as the key, simData as the value. new->extra = (long) ourARs; dirtree_handle_callback(new, filterARs); qsort(ourARs->ARs, ourARs->num, sizeof(char *), qstrcmp); - free(ga); - ga = xmprintf("%s/errors", dir); for (i = 0; i < ourARs->num; i++) { I("Adding %s to %s", ourARs->ARs[i], gtr); // Deal with deletions in the inventory / sim, easy method, which becomes a nop for files that stay in the git add below. - if (shellMeFail("cd %s/%s; rm -fr * >>%s", dir, gar, gal)) - E("Failed to rm!"); + if (shellMeFail("cd %s/%s; rm -fr * >>%s", dir, gar, gerr)) E("Failed to rm!"); if (shellMeFail("cd %s/%s; ionice -c3 nice -n 19 tar -xzf \"%s/%s\" || echo \"ERROR - Could not unpack %s !\" >>%s", - dir, gar, scBackup, ourARs->ARs[i], ourARs->ARs[i], ga)) - E("Failed to unpack %s!", ourARs->ARs[i]); - if (!qfile_exist(ga)) + dir, gar, scBackup, ourARs->ARs[i], ourARs->ARs[i], gerr)) E("Failed to unpack %s!", ourARs->ARs[i]); + if (!hasContents(gerr)) { - if (shellMeFail("cd %s/%s; git add * >>%s", dir, gar, gal)) - E("Failed to git add!"); + if (shellMeFail("cd %s/%s; git add * >>%s", dir, gar, glog)) E("Failed to git add!"); // The \\* bit is to escape the \ from snprintf, which itself escapes the * from the shell. - if (shellMeFail("cd %s/%s; git add */\\* >>%s", dir, gar, gal)) - E("Failed to git add!"); + if (shellMeFail("cd %s/%s; git add */\\* >>%s", dir, gar, gerr)) E("Failed to git add!"); // Gotta add this again, coz of the rm. Apparently rm removes dotfiles, but add doesn't! - if (-1 == qfile_save(gab, gad, strlen(gad), false)) - E("Faild to write %s file!", ga); - if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gal)) - E("Failed to git add!"); + if (-1 == qfile_save(gab, gad, strlen(gad), false)) E("Faild to write %s file!", gab); + if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gerr)) E("Failed to git add!"); // Magic needed to figure out if there's anything to commit. // After all the pain to get this to work, there's an ever changing timestamp in archive.xml that screws it up. @@ -2237,18 +2247,15 @@ byTab has the short name as the key, simData as the value. // TODO - I could sed out that timestamp, and put it back again based on the OAR file name when extracting. // IARs don't seem to have the timestamp. int j = shellMe("cd %s/%s; t=$(git status --porcelain) && [ -z \"${t}\" ]", dir, gar); - if (!WIFEXITED(j)) - E("git status failed!"); + if (!WIFEXITED(j)) E("git status failed!"); else if (1 == WEXITSTATUS(j)) { V("Committing changes from %s", ourARs->ARs[i]); // Note this commit message has to be just the file name, as the ungitAR script uses it. if (shellMeFail("cd %s/%s; git commit -a -qm \"%s\" >>%s || echo \"ERROR - Could not commit %s !\" >>%s ", - dir, gar, ourARs->ARs[i], ourARs->ARs[i], gal, ga)) - E("Failed to git commit!"); - if (qfile_exist(ga)) + dir, gar, ourARs->ARs[i], ourARs->ARs[i], glog, gerr)) E("Failed to git commit!"); + if (hasContents(gerr)) { - free(ga); free(ourARs->ARs[i]); goto gitARend; } @@ -2256,36 +2263,36 @@ byTab has the short name as the key, simData as the value. else V("No changes to commit from %s.", ourARs->ARs[i]); } - if (!qfile_exist(ga)) + if (!hasContents(gerr)) { - if (shellMeFail("mv %s/%s %s", scBackup, ourARs->ARs[i], dir)) - E("Failed to move %s!", ourARs->ARs[i]); + if (shellMeFail("mv %s/%s %s", scBackup, ourARs->ARs[i], dir)) E("Failed to move %s!", ourARs->ARs[i]); } free(ourARs->ARs[i]); } - if (!qfile_exist(ga)) + if (!hasContents(gerr)) { - I("Compressing gitAR %s", gtr); + I("Checking git repo %s", gtr); + if (shellMeFail("cd %s/%s; git fsck --strict --lost-found --no-progress >>%s || echo \"ERROR - Problem with git fsck %s !\" >>%s ", + dir, gar, glog, gtr, gerr)) E("Failed to git fsck!"); + if (shellMeFail("cd %s/%s; git gc --aggressive --prune=all --quiet >>%s || echo \"ERROR - Problem with git gc %s !\" >>%s ", + dir, gar, glog, gtr, gerr)) E("Failed to git gc!"); + I("Compressing %s", gtr); if (shellMeFail("cd %s; XZ_OPT='-9e' ionice -c3 nice -n 19 tar -c --xz %s -f %s || echo 'ERROR - Could not pack gitAR!' >>%s", - dir, gar, gtr, ga)) - E("Failed to git add!"); + dir, gar, gtr, gerr)) E("Failed to git add!"); } gitARend: - free(ga); - ga = xmprintf("%s/errors", dir); - if (qfile_exist(ga)) - E("Failed to process the archives, look in %s for errors!", ga); + if (hasContents(gerr)) E("Failed to process the archives, look in %s for errors!", gerr); else { - if (shellMeFail("rm -fr %s", dir)) - E("Failed to rm!"); + if (shellMeFail("rm -fr %s", dir)) E("Failed to rm!"); } - free(ga); free(ourARs->ARs); free(ourARs->this); free(ourARs); free(gab); + free(gerr); + free(glog); free(dir); free(gtr); free(gar); -- cgit v1.1