From 550e010789bcd9d79a803df4cc2828922dad3838 Mon Sep 17 00:00:00 2001 From: onefang Date: Thu, 19 Aug 2021 21:23:43 +1000 Subject: Add rsync and gitar stuff to sledjchisl backups. --- src/sledjchisl/sledjchisl.c | 276 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 255 insertions(+), 21 deletions(-) diff --git a/src/sledjchisl/sledjchisl.c b/src/sledjchisl/sledjchisl.c index c2ec573..e4c4359 100644 --- a/src/sledjchisl/sledjchisl.c +++ b/src/sledjchisl/sledjchisl.c @@ -522,6 +522,8 @@ char *Tsocket = "opensim-tmux.socket"; char *Ttab = "SC"; char *Tcmd = "tmux -S"; char *backupIARsim = "Sandbox"; +char *rSync = ""; +int rSyncPort = 0; char *webRoot = "/var/www/html"; char *URL = "fcgi-bin/sledjchisl.fcgi"; char *ToS = "Be good."; @@ -1883,6 +1885,8 @@ void startSim(simData *simd, char *sim, char *type, int count, int window, int p snprintf(toybuf, sizeof(toybuf), "INITIALIZATION COMPLETE FOR %s", simd->name); waitTmuxText(simd->paneID, toybuf); I("%s is done starting up.", simd->name); + sendTmuxCmd(simd->paneID, ""); + sendTmuxCmd(simd->paneID, ""); ourSims->la = waitLoadAverage(ourSims->la, loadAverageInc, simTimeOut); if (1 < bulkSims) bulkSims = bulkSims / 2; @@ -1947,16 +1951,51 @@ void stopSim(simData *simd, char *sim, char *type, int count, int window, int pa } } +typedef struct _ARList ARList; +struct _ARList +{ + int len, num; + char **ARs, *this; +}; + +static int filterARs(struct dirtree *node) +{ + if (!node->parent) return DIRTREE_RECURSE | DIRTREE_SHUTUP; + ARList *list = (ARList *) node->parent->extra; + int l0 = strlen(node->name), l1 = strlen(list->this); + +// TODO - ignore zero sized ones. + if ((4 < l0) && ((strncmp(&(node->name[l0 - 4]), ".iar", 4) == 0) || (strncmp(&(node->name[l0 - 4]), ".oar", 4) == 0)) && (strncmp(node->name, list->this, l1) == 0)) + { + if ((list->num + 1) > list->len) + { + list->len = list->len + 1; + list->ARs = xrealloc(list->ARs, list->len * sizeof(char *)); + } + list->ARs[list->num] = xstrdup(node->name); + list->num++; + } + return 0; +} + // Forward declare this. my_ulonglong dbCount(char *table, char *where); void doSimsThing(simData *simd, char *sim, char *type, int count, int window, int panes, int pane) { // Check if only doing a single sim. - int cont = FALSE; + int cont = FALSE, member = false; + char *last = strchr(sim, ' '); if (FLAG(m)) { -// TODO - check if it's a real user. + // check if it's a real user. + last[0] = '\0'; last++; + // Double check it's a real member. + snprintf(toybuf, sizeof(toybuf), "FirstName='%s' and LastName='%s'", sim, last); + if (1 == dbCount("UserAccounts", toybuf)) + member = TRUE; + else + E("Can't find member %s %s.", sim, last); } else if (NULL != ourSims->target) { @@ -1997,7 +2036,7 @@ byTab has the short name as the key, simData as the value. } case BACKUP : // "backup -m 'onefang rejected'" "backup 'Welcome sim'" "backup Welcome.shini" "Welcome.shini backup" "backup Welcome.ini" "backup Welcome" "backup" backup everything - { // TODO - If it's not a sim code, and not a sim name, it's an account inventory. + { struct timeval tv; time_t curtime; char date[DATE_TIME_LEN]; @@ -2007,17 +2046,14 @@ byTab has the short name as the key, simData as the value. strftime(date, DATE_TIME_LEN, "%F_%T", localtime(&curtime)); if (FLAG(m)) { - char *last = strchr(sim, ' '); - - last[0] = '\0'; last++; - // Double check it's a real member. - snprintf(toybuf, sizeof(toybuf), "FirstName='%s' and LastName='%s'", sim, last); - if (1 == dbCount("UserAccounts", toybuf)) + if (member) { - I("Member %s %s is being backed up to %s/backups/%s_%s-%s.iar in tmux windo %s.", sim, last, scRoot, sim, last, date, ourSims->backup); - snprintf(toybuf, sizeof(toybuf), "save iar -c %s %s / password %s/backups/%s_%s-%s.iar", sim, last, scRoot, sim, last, date); + I("Member %s %s is being backed up to %s/%s_%s-%s.iar in tmux windo %s.", sim, last, scBackup, sim, last, date, ourSims->backup); + snprintf(toybuf, sizeof(toybuf), "save iar -c %s %s / password %s/%s_%s-%s.iar", sim, last, scBackup, sim, last, date); if (NULL != ourSims->backup) { + if ('\0' == rSync[0]) + shellMeFail("%s/current/bin/sledjchisl gitar -m %s %s %s", scRoot, FLAG(v) ? "-v" : "", sim, last); sendTmuxCmd(ourSims->backup, toybuf); // if (0 == do) { @@ -2026,16 +2062,18 @@ byTab has the short name as the key, simData as the value. I("%s %s is done backing up.", sim, last); sendTmuxCmd(ourSims->backup, ""); sendTmuxCmd(ourSims->backup, ""); -// TODO - should delete / gitAR the old ones now. -// Have a config option for delete / gitAR / rsync. + if ('\0' != rSync[0]) + { + if (shellMeFail("time ionice -c3 nice -n 19 rsync -Ha -R --modify-window=2 --partial --port=%d --remove-source-files %s/*.iar %s", + rSyncPort, scBackup, rSync)) + E("rsync failed"); + } ourSims->la = waitLoadAverage(ourSims->la, loadAverageInc, simTimeOut); } } else E("Can't find backup sim."); } - else - E("Can't find member %s %s.", sim, last); } else if (checkSimIsRunning(simd->tab)) { @@ -2044,15 +2082,23 @@ byTab has the short name as the key, simData as the value. // strip off the last bit of file name (YYYY-mm-dd_HH:MM:SS.oar) to get the name // keep in mind some old files had munged names like "Tiffanie_s_Paradise-2021-06-23_05:11:38.oar" I("Sim %s is being backed up to %s/backups/%s-%s.oar.", simd->name, scRoot, simd->tab, date); - snprintf(toybuf, sizeof(toybuf), "save oar --all %s/backups/%s-%s.oar", scRoot, simd->tab, date); + if ('\0' == rSync[0]) + shellMeFail("%s/current/bin/sledjchisl gitar %s %s", scRoot, FLAG(v) ? "-v" : "", simd->tab); + snprintf(toybuf, sizeof(toybuf), "save oar --all %s/%s-%s.oar", scBackup, simd->tab, date); sendTmuxCmd(simd->paneID, toybuf); // if (0 == do) { snprintf(toybuf, sizeof(toybuf), "Finished writing out OAR for %s", simd->name); waitTmuxText(simd->paneID, toybuf); I("%s is done backing up.", simd->name); -// TODO - should delete / gitAR the old ones now. -// Have a config option for delete / gitAR / rsync. + sendTmuxCmd(simd->paneID, ""); + sendTmuxCmd(simd->paneID, ""); + if ('\0' != rSync[0]) + { + if (shellMeFail("time ionice -c3 nice -n 19 rsync -Ha -R --modify-window=2 --partial --port=%d --remove-source-files %s/*.oar %s", + rSyncPort, scBackup, rSync)) + E("rsync failed"); + } ourSims->la = waitLoadAverage(ourSims->la, loadAverageInc, simTimeOut); } } @@ -2061,6 +2107,189 @@ byTab has the short name as the key, simData as the value. case GITAR : // "gitAR -m avatar name" "gitAR sim name" { + /* Work around OpenSims slow database corruption bug by using git to store all old backups. + Try to squeeze every last byte out of the tarballs. Seems to cut the total storage size down to one third the size of just the raw I/OAR files. + Saves even more if there's been no changes. + On the other hand, these backup files will grow indefinately, the more changes, the faster it grows. I can live with that for more reliable backups that go back further. + Tries to avoid loosing data if things go wrong. I think the main remaining problem would be running out of space, in which case you have bigger problems to deal with. + + Strategy - unpack the last one, unpack and commit any old I/OARs, pack up the result, delete it's working directory, THEN run the save i/oar. + Avoids having to sync with OpenSim finishing the current I/OAR, and as a bonus, an easy to deliver latest I/OAR for people that want it. + */ + char *name = xstrdup(sim); + + if (FLAG(m)) + { + if (member) + { + free(name); + name = xmprintf("%s_%s", sim, last); + } + else + { + free(name); + return; + } + } + + char type = FLAG(m) ? 'I' : 'O'; + char *gar = xmprintf("%s_git%cAR", name, type); + char *gtr = xmprintf("%s/%s-git%cAR.tar.xz", scBackup, name, type); + char *dir = xmprintf("%s/temp_backups%c_%s", scBackup, type, name); + // Make sure stuff that's already compressed doesn't get compressed by git. + // Also tries to protect binaries from mangling. + char *gab = xmprintf("%s/%s/.gitattributes", dir, gar); + char *ga = xmprintf("%s/%s", dir, gar); + char *gal = xmprintf("%s/log", scBackup); + char *gad = + "*.bvh -delta -diff -text\n" + "*.jp2 -delta -diff -text\n" + "*.jpg -delta -diff -text\n" + "*.llmesh -delta -diff -text\n" + "*.ogg -delta -diff -text\n" + "*.png -delta -diff -text\n" + "*.r32 -delta -diff -text\n" + "*.tga -delta -diff -text\n"; + ARList *ourARs = xzalloc(sizeof(ARList)); + + if (qfile_exist(dir)) + { + if (shellMeFail("echo 'Mess left over from last backup in %d, not gonna run!' >>%s/errors", dir)) + E("Failed cleaning up the mess!"); + if (shellMeFail("mv %s/*.%car %s 2>&1 >/dev/null", dir, tolower(type), scBackup)) + E("Failed cleaning up the mess!"); + goto gitARend; + } + + // Either unpack the old gitAR, or create a new one. + qfile_mkdir(dir, S_IRWXU | S_IRGRP | S_IXGRP, true); + if (qfile_exist(gtr)) + { + I("Unpacking %s", gtr); + if (shellMeFail("cd %s; ionice -c3 nice -n 19 tar -xf %s", dir, gtr)) + E("Failed to unpack %s!", gtr); + } + else + { + qfile_mkdir(gar, S_IRWXU | S_IRGRP | S_IXGRP, true); + // git will create gar for us. + if (shellMeFail("cd %s; git init %s >%s", dir, gar, gal)) + E("Failed to git init %s/%s!", dir, gar); + + // Coz git insists. + if (shellMeFail("cd %s/%s; git config user.email \"opensim@$(hostname -A | cut -d ' ' -f 1)\"", dir, gar)) + E("Failed to git config user.email!"); + if (shellMeFail("cd %s/%s; git config user.name \"opensim\"", dir, gar)) + E("Failed to git config user.name!"); + + // Coz git insists on having something comitted before it can change the branch name. + if (-1 == qfile_save(gab, gad, strlen(gad), false)) + E("Faild to write %s file!", ga); + if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gal)) + E("Failed to git add!"); + V("Committing initial git."); + if (shellMeFail("cd %s/%s; git commit -qm \"Initial commit\" >>%s || echo \"ERROR - Could not commit!\" >>%s/errors ", + dir, gar, gal, dir)) + E("Failed to git commit!"); + if (shellMeFail("cd %s/%s; git branch -m master Domme >>%s", dir, gar, gal)) + E("Failed to git branch -m master Domme!"); + } + + // Git is such a pedantic bitch, let's just fucking ignore any errors it gives due to lack of anything to do. + // Even worse the OpenSim devs breaking logout tracking gives git plenty of nothing to do. lol + + // Loop through the .iar / .oar files in backups, ignoring zero sized files. + struct dirtree *new = dirtree_add_node(0, scBackup, 0); + int i; + + ourARs->num = 0; + ourARs->this = xmprintf("%s-", name); + new->extra = (long) ourARs; + dirtree_handle_callback(new, filterARs); + qsort(ourARs->ARs, ourARs->num, sizeof(char *), qstrcmp); + free(ga); + ga = xmprintf("%s/errors", dir); + for (i = 0; i < ourARs->num; i++) + { + I("Adding %s to %s", ourARs->ARs[i], gtr); + // Deal with deletions in the inventory / sim, easy method, which becomes a nop for files that stay in the git add below. + if (shellMeFail("cd %s/%s; rm -fr * >>%s", dir, gar, gal)) + E("Failed to rm!"); + if (shellMeFail("cd %s/%s; ionice -c3 nice -n 19 tar -xzf \"%s/%s\" || echo \"ERROR - Could not unpack %s !\" >>%s", + dir, gar, scBackup, ourARs->ARs[i], ourARs->ARs[i], ga)) + E("Failed to unpack %s!", ourARs->ARs[i]); + if (!qfile_exist(ga)) + { + if (shellMeFail("cd %s/%s; git add * >>%s", dir, gar, gal)) + E("Failed to git add!"); + // The \\* bit is to escape the \ from snprintf, which itself escapes the * from the shell. + if (shellMeFail("cd %s/%s; git add */\\* >>%s", dir, gar, gal)) + E("Failed to git add!"); + // Gotta add this again, coz of the rm. Apparently rm removes dotfiles, but add doesn't! + if (-1 == qfile_save(gab, gad, strlen(gad), false)) + E("Faild to write %s file!", ga); + if (shellMeFail("cd %s/%s; git add .gitattributes >>%s", dir, gar, gal)) + E("Failed to git add!"); + + // Magic needed to figure out if there's anything to commit. + // After all the pain to get this to work, there's an ever changing timestamp in archive.xml that screws it up. + // Like this system didn't have enough timestamps in it already. lol + // TODO - I could sed out that timestamp, and put it back again based on the OAR file name when extracting. + // IARs don't seem to have the timestamp. + int j = shellMe("cd %s/%s; t=$(git status --porcelain) && [ -z \"${t}\" ]", dir, gar); + if (!WIFEXITED(j)) + E("git status failed!"); + else if (1 == WEXITSTATUS(j)) + { + V("Committing changes from %s", ourARs->ARs[i]); + // Note this commit message has to be just the file name, as the ungitAR script uses it. + if (shellMeFail("cd %s/%s; git commit -a -qm \"%s\" >>%s || echo \"ERROR - Could not commit %s !\" >>%s ", + dir, gar, ourARs->ARs[i], ourARs->ARs[i], gal, ga)) + E("Failed to git commit!"); + if (qfile_exist(ga)) + { + free(ga); + free(ourARs->ARs[i]); + goto gitARend; + } + } + else + V("No changes to commit from %s.", ourARs->ARs[i]); + } + if (!qfile_exist(ga)) + { + if (shellMeFail("mv %s/%s %s", scBackup, ourARs->ARs[i], dir)) + E("Failed to move %s!", ourARs->ARs[i]); + } + free(ourARs->ARs[i]); + } + + if (!qfile_exist(ga)) + { + I("Compressing gitAR %s", gtr); + if (shellMeFail("cd %s; XZ_OPT='-9e' ionice -c3 nice -n 19 tar -c --xz %s -f %s || echo 'ERROR - Could not pack gitAR!' >>%s", + dir, gar, gtr, ga)) + E("Failed to git add!"); + } +gitARend: + free(ga); + ga = xmprintf("%s/errors", dir); + if (qfile_exist(ga)) + E("Failed to process the archives, look in %s for errors!", ga); + else + { + if (shellMeFail("rm -fr %s", dir)) + E("Failed to rm!"); + } + free(ga); + free(ourARs->ARs); + free(ourARs->this); + free(ourARs); + free(gab); + free(dir); + free(gtr); + free(gar); + free(name); break; } @@ -7525,12 +7754,16 @@ void forEachMember(char *verb, simFunction func, simFunction not) { char *file = xmprintf("%s/.lastTime", scBackup); + free(last); last = (char *) qfile_load(file, NULL); - if (NULL == last) - last = xstrdup("0"); if (shellMeFail("date +%%s > %s", file)) E("date command failed!"); free(file); + if ((NULL == last) || (NULL != ourSims->target)) + { + free(last); + last = xstrdup("0"); + } } static dbRequest *users = NULL; @@ -7552,7 +7785,6 @@ void forEachMember(char *verb, simFunction func, simFunction not) { qhashtbl_t *row; -d("Number of rows returned %d", (int) rows->rows->size(rows->rows)); while (NULL != (row = rows->rows->getat(rows->rows, 0, NULL, true))) { char *firstName = row->getstr(row, "FirstName", false); @@ -7862,6 +8094,8 @@ Other possibilities - if ((tmp = configs->getstr(configs, "Tsocket", false)) != NULL) {Tsocket = tmp; V("Setting Tsocket = %s", Tsocket);} if ((tmp = configs->getstr(configs, "Ttab", false)) != NULL) {Ttab = tmp; V("Setting Ttab = %s", Ttab);} if ((tmp = configs->getstr(configs, "backupIARsim", false)) != NULL) {backupIARsim = tmp; V("Setting backupIARsim = %s", backupIARsim);} + if ((tmp = configs->getstr(configs, "rsync", false)) != NULL) {rSync = tmp; V("Setting rsync = %s", rSync);} + if ((vd = configs->getstr(configs, "rsyncPort", false)) != NULL) {rSyncPort = (int) *((float *) vd); V("Setting rsyncPort = %s", rSyncPort);} if ((tmp = configs->getstr(configs, "webRoot", false)) != NULL) {webRoot = tmp; V("Setting webRoot = %s", webRoot);} if ((tmp = configs->getstr(configs, "URL", false)) != NULL) {URL = tmp; V("Setting URL = %s", URL);} if ((vd = configs->get (configs, "seshRenew", NULL, false)) != NULL) {seshRenew = (int) *((float *) vd); V("Setting seshRenew = %d", seshRenew);} -- cgit v1.1