aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/SuckIt
diff options
context:
space:
mode:
Diffstat (limited to '')
-rwxr-xr-xSuckIt69
-rwxr-xr-xSuckItFos68
-rwxr-xr-xSuckItPm72
3 files changed, 69 insertions, 140 deletions
diff --git a/SuckIt b/SuckIt
new file mode 100755
index 0000000..57aa88b
--- /dev/null
+++ b/SuckIt
@@ -0,0 +1,69 @@
1#!/bin/bash
2
3TIMEFORMAT=" took %lR using %P%% CPU"
4time {
5pushd /opt/merged
6
7rm -fr Foswiki/*
8cp -r /opt/merged_EMPTY/Foswiki .
9rm -fr PmWiki/*
10cp -r /opt/merged_EMPTY/PmWiki .
11
12
13filter="
14 -name _default -prune -o \
15 -name _empty -prune -o \
16 -name System -prune -o \
17 -name Trash -prune -o \
18 -name TWiki -prune -o \
19"
20URL="https://fos.wiki.devuan.org"
21time find /opt/Foswiki/data ${filter} \
22-name "*.txt" -type f,l -printf "%P\n" | while read line
23do
24 base=`echo "${line}" | cut -d '/' -f 1`
25 file=`echo "${line}" | cut -d '/' -f 2- | rev | cut -b 5- | rev`
26 if [[ ! $file =~ (AdminGroup|AdminUser|AdminUserLeftBar|EditorGroup|GroupTemplate|GroupViewTemplate|NobodyGroup|PatternSkinUserViewTemplate|ProjectContributor|RegistrationAgent|SitePreferences|UnprocessedRegistrations|UnprocessedRegistrationsLog|UserHomepageHeader|UserList|UserListByDateJoined|UserListByLocation|UserList|UserListHeader|WebAtom|WebChanges|WebCreateNewTopic|WebHome|WebIndex|WebLeftBar|WebLeftBarExample|WebNotify|WebPreferences|WebRss|WebSearch|WebSearchAdvanced|WebTopicList|WikiGroups|WikiUsers)$ ]]; then
27 time=`date --rfc-3339=seconds -ur /opt/Foswiki/data/${base}/${file}.txt | cut -d '+' -f 1`
28 mkdir -p Foswiki/$base
29 mkdir -p Foswiki/${base}/`dirname ${file}`
30 echo -e "ogWiki=Foswiki\nogURL=${URL}\nogBase=${base}\nogFile=${file}\ntimestamp=${time}\n" > Foswiki/${base}/${file}.md.md
31 echo "downloading ${URL}/${base}/${file}?cover=print"
32 # Doesn't help with redownloads, coz natch a dynamic site isn't cached. But I can at least comment out the curl command during testing to save time.
33 curl --silent --no-progress-meter ${URL}/${base}/${file}?cover=print -o Foswiki/${base}/${file}.HTM
34 fi
35done
36
37
38filter="
39 -not -name "*~" -a \
40 -not -name ".flock" -a \
41 -not -name ".htaccess" -a \
42 -not -name ".lastmod" -a \
43 -not -name ".pageindex" -a \
44"
45URL="https://wiki.devuan.org"
46time find /opt/pmwiki/wiki.d ${filter} \
47-name "*.*" -type f,l -printf "%P\n" | while read line
48do
49 base=`echo "${line}" | cut -d '.' -f 1`
50 if [[ "${base}" != "Site" ]]; then
51 file=`echo "${line}" | cut -d '.' -f 2`
52 time=`date --rfc-3339=seconds -ur /opt/pmwiki/wiki.d/${base}.${file} | cut -d '+' -f 1`
53 mkdir -p PmWiki/$base
54 echo -e "ogWiki=PmWiki\nogURL=${URL}\nogBase=${base}\nogFile=${file}\ntimestamp=${time}\n" > PmWiki/${base}/${file}.md.md
55# echo "downloading ${URL}/?n=${base}.${file}?action=markdown"
56# curl --no-progress-meter ${URL}/?n=${base}.${file}?action=markdown -o PmWiki/${base}/${file}.MARKDOWN
57 echo "downloading ${URL}/?n=${base}.${file}?action=print"
58 curl --no-progress-meter ${URL}/?n=${base}.${file}?action=print -o PmWiki/${base}/${file}.HTM
59
60# pandoc -f markdown -t commonmark_x --self-contained PmWiki//${base}/${file}.MD >PmWiki/${base}/${file}.md
61# pandoc -f html -t commonmark_x --self-contained PmWiki//${base}/${file}.HTM >PmWiki/${base}/${file}.md
62 fi
63done
64
65
66time notYetAnotherWiki.lua
67
68popd
69}
diff --git a/SuckItFos b/SuckItFos
deleted file mode 100755
index dc65505..0000000
--- a/SuckItFos
+++ /dev/null
@@ -1,68 +0,0 @@
1#!/bin/bash
2
3URL="https://fos.wiki.devuan.org"
4
5filter="
6 -name _default -prune -o \
7 -name _empty -prune -o \
8 -name System -prune -o \
9 -name Trash -prune -o \
10 -name TWiki -prune -o \
11"
12
13pushd /opt/mergedWork
14
15find /opt/Foswiki/data ${filter} \
16-name "*.txt" -type f,l -printf "%P\n" | while read line
17do
18 base=`echo "${line}" | cut -d '/' -f 1`
19 file=`echo "${line}" | cut -d '/' -f 2- | rev | cut -b 5- | rev`
20 time=`date --rfc-3339=seconds -ur /opt/Foswiki/data/${base}/${file}.txt | cut -d '+' -f 1`
21 mkdir -p Foswiki/$base
22 mkdir -p Foswiki/${base}/`dirname ${file}`
23 mkdir -p combined/$base
24 mkdir -p combined/${base}/`dirname ${file}`
25 echo "Converting ${URL}/${base}/${file}?cover=print -> Foswiki/${base}/${file}.md"
26 echo -e "ogWiki=Foswiki\nogURL=${URL}\nogBase=${base}\nogFile=${file}\ntimestamp=${time}\n" > Foswiki/${base}/${file}.md.md
27 # Doesn't help with redownloads, coz natch a dynamic site isn't cached. But I can at least comment out the curl command during testing to save time.
28 curl --silent --no-progress-meter ${URL}/${base}/${file}?cover=print -o Foswiki/${base}/${file}.HTM
29 cp Foswiki/${base}/${file}.HTM Foswiki/${base}/${file}.HTM_ORIGINAL
30 csplit -ks Foswiki/${base}/${file}.HTM '%<div id="patternMainContents">%' '/<div class="foswikiAttachments foswikiFormStep" style="overflow:auto">/'
31 if [ -f xx00 ]; then
32 rm Foswiki/${base}/${file}.HTM
33 mv xx00 Foswiki/${base}/${file}.HTM
34 fi
35 sed -i -E Foswiki/${base}/${file}.HTM \
36 -e "s/rel='nofollow'//g" \
37 -e 's/rel="nofollow"//g' \
38 -e "s/target='_blank'//g" \
39 -e "s/class='foswiki[[:alpha:]]*'//g" \
40 -e 's/class="foswikiTopic"/class="FoswikiTopic"/g' \
41 -e 's/class="foswiki[[:alpha:]]*"//g' \
42 -e "s/style='.*;'//g"
43
44 pandoc -f html -t commonmark_x --self-contained Foswiki//${base}/${file}.HTM >Foswiki/${base}/${file}.md
45 cp Foswiki/${base}/${file}.md Foswiki/${base}/${file}.md_ORIGINAL
46
47 csplit -ks Foswiki/${base}/${file}.md '%::: {.FoswikiTopic}%' '/::: {.patternInfo}/'
48 if [ -f xx00 ]; then
49 rm Foswiki/${base}/${file}.md
50 mv xx00 Foswiki/${base}/${file}.md
51 fi
52 if [ -f xx01 ]; then
53 rm xx01
54 fi
55
56 # Attempt to clean things up, badly.
57 sed -i -E Foswiki/${base}/${file}.md \
58 -e 's/\$/\$dlr\$/g' \
59 -e 's/\{#.*\}//g' \
60 -e '/^:::/d' \
61 -e '/^<!-- -->/d' \
62# -e 's/\{\.pattern.*\}//g' \
63# -e 's/\{\.pattern.*//g' \
64
65 ln -frs Foswiki/${base}/${file}.md combined/${base}/${file}.md
66done
67
68popd
diff --git a/SuckItPm b/SuckItPm
deleted file mode 100755
index 246f485..0000000
--- a/SuckItPm
+++ /dev/null
@@ -1,72 +0,0 @@
1#!/bin/bash
2
3URL="https://wiki.devuan.org"
4
5filter="
6 -not -name "*~" -a \
7 -not -name ".flock" -a \
8 -not -name ".htaccess" -a \
9 -not -name ".lastmod" -a \
10 -not -name ".pageindex" -a \
11"
12
13pushd /opt/mergedWork
14
15find /opt/pmwiki/wiki.d ${filter} \
16-name "*.*" -type f,l -printf "%P\n" | while read line
17do
18 base=`echo "${line}" | cut -d '.' -f 1`
19 file=`echo "${line}" | cut -d '.' -f 2`
20 time=`date --rfc-3339=seconds -ur /opt/pmwiki/wiki.d/${base}.${file} | cut -d '+' -f 1`
21 mkdir -p PmWiki/$base
22 mkdir -p combined/$base
23 echo "Converting ${URL}/?n=${base}.${file}?action=print -> PmWiki/${base}/${file}.md"
24 echo -e "ogWiki=PmWiki\nogURL=${URL}\nogBase=${base}\nogFile=${file}\ntimestamp=${time}\n" > PmWiki/${base}/${file}.md.md
25 # Doesn't help with redownloads, coz natch a dynamic site isn't cached. But I can at least comment out the curl command during testing to save time.
26# curl --no-progress-meter ${URL}/?n=${base}.${file}?action=markdown -o PmWiki/${base}/${file}.MD
27 curl --no-progress-meter ${URL}/?n=${base}.${file}?action=print -o PmWiki/${base}/${file}.HTM
28 cp PmWiki/${base}/${file}.HTM PmWiki/${base}/${file}.HTM_ORIGINAL
29 csplit -ks PmWiki/${base}/${file}.HTM '%<!--PageText-->%' '/<!--HTMLFooter-->/'
30 if [ -f xx00 ]; then
31 rm PmWiki/${base}/${file}.HTM
32 mv xx00 PmWiki/${base}/${file}.HTM
33 fi
34 if [ -f xx01 ]; then
35 rm xx01
36 fi
37 sed -i -E PmWiki/${base}/${file}.HTM \
38 -e "s/rel='nofollow'//g" \
39 -e "s/target='_blank'//g" \
40 -e "s/class='categorylink'//g" \
41 -e "s/class='createlink'//g" \
42 -e "s/class='createlinktext'//g" \
43 -e "s/class='escaped'//g" \
44 -e "s/class='diffmarkup'//g" \
45 -e "s/class='selflink'//g" \
46 -e "s/class='urllink'//g" \
47 -e "s/class='vspace'//g" \
48 -e "s/class='wikilink'//g" \
49 -e "s/style='.*;'//g"
50
51# pandoc -f markdown -t commonmark_x --self-contained PmWiki//${base}/${file}.MD >PmWiki/${base}/${file}.md
52 pandoc -f html -t commonmark_x --self-contained PmWiki//${base}/${file}.HTM >PmWiki/${base}/${file}.md
53 cp PmWiki/${base}/${file}.md PmWiki/${base}/${file}.md_ORIGINAL
54
55 # Attempt to clean things up, badly.
56 sed -i -E PmWiki/${base}/${file}.md \
57 -e 's/\$/\$dlr\$/g' \
58 -e 's/\{#.*\}//g' \
59 -e '/^:::/d' \
60# -e '/\[Site$/d' \
61# -e '/^\[\]/d' \
62# -e "s/\`<a id='trailstart'>\`\{=html\}\`<\/a>\`\{=html\}//g" \
63# -e "s/^\`<img /<img /g" \
64# -e "s/^\`\`\`\{=html\}//g" \
65# -e "s/^\`\`\`//g" \
66# -e "s/\`\{=html\}//g"
67
68 ln -frs PmWiki/${base}/${file}.md combined/${base}/${file}.md
69 ln -frs PmWiki/${base}/${file}.md.md combined/${base}/${file}.md.md
70done
71
72popd
generated by cgit v1.1 at 2025-03-13 15:59:18 +0000