From 5a671268340c107ee94f8244f01b6cb8ee9decd0 Mon Sep 17 00:00:00 2001 From: dvs1 Date: Sat, 25 Jan 2025 03:14:10 +1000 Subject: YAR see description for details. New everything page, lists all the pages and where they came from, with links. Now we know where the page came from, type and URL. Rejigged how metadata works. Do our own metadata substitution. Fixed up a bunch of bugs. General cleanups. --- .md.md | 2 + SuckItFos | 9 +-- SuckItPm | 12 ++- TODO.md | 8 +- default.template | 1 - notYetAnotherWiki.lua | 206 +++++++++++++++++++++++++++++++------------------- 6 files changed, 146 insertions(+), 92 deletions(-) diff --git a/.md.md b/.md.md index 3c5cb8a..4a55e50 100644 --- a/.md.md +++ b/.md.md @@ -1,3 +1,5 @@ +favicon=nYAW_icon.png +logo=nYAW.png sourcecode=https://sledjhamr.org/cgit/notYetAnotherWiki/ pagehistory=https://sledjhamr.org/cgit/notYetAnotherWiki/log feedatom=https://sledjhamr.org/cgit/notYetAnotherWiki/atom diff --git a/SuckItFos b/SuckItFos index 1e6e48d..0f6f8a8 100755 --- a/SuckItFos +++ b/SuckItFos @@ -22,6 +22,7 @@ do mkdir -p combined/$base mkdir -p combined/${base}/`dirname ${file}` echo "Converting ${URL}/${base}/${file}?cover=print -> Foswiki/${base}/${file}.md" + echo -e "ogWiki=Foswiki\nogURL=${URL}/${base}/${file}\n" > Foswiki/${base}/${file}.md.md # Doesn't help with redownloads, coz natch a dynamic site isn't cached. But I can at least comment out the curl command during testing to save time. curl --silent --no-progress-meter ${URL}/${base}/${file}?cover=print -o Foswiki/${base}/${file}.HTM cp Foswiki/${base}/${file}.HTM Foswiki/${base}/${file}.HTM_ORIGINAL @@ -47,6 +48,9 @@ do rm Foswiki/${base}/${file}.md mv xx00 Foswiki/${base}/${file}.md fi + if [ -f xx01 ]; then + rm xx01 + fi # Attempt to clean things up, badly. sed -i -E Foswiki/${base}/${file}.md \ @@ -57,12 +61,7 @@ do # -e 's/\{\.pattern.*\}//g' \ # -e 's/\{\.pattern.*//g' \ - echo -e "****\n[Original page](${URL}/${base}/${file}) where maybe you can edit it." >> Foswiki/${base}/${file}.md ln -frs Foswiki/${base}/${file}.md combined/${base}/${file}.md - - if [ -f xx01 ]; then - rm xx01 - fi done popd diff --git a/SuckItPm b/SuckItPm index fe7efb3..fd0b048 100755 --- a/SuckItPm +++ b/SuckItPm @@ -20,6 +20,7 @@ do mkdir -p PmWiki/$base mkdir -p combined/$base echo "Converting ${URL}/?n=${base}.${file}?action=print -> PmWiki/${base}/${file}.md" + echo -e "ogWiki=PmWiki\nogURL=${URL}/?n=${base}.${file}\n" > PmWiki/${base}/${file}.md.md # Doesn't help with redownloads, coz natch a dynamic site isn't cached. But I can at least comment out the curl command during testing to save time. # curl --no-progress-meter ${URL}/?n=${base}.${file}?action=markdown -o PmWiki/${base}/${file}.MD curl --no-progress-meter ${URL}/?n=${base}.${file}?action=print -o PmWiki/${base}/${file}.HTM @@ -29,6 +30,9 @@ do rm PmWiki/${base}/${file}.HTM mv xx00 PmWiki/${base}/${file}.HTM fi + if [ -f xx01 ]; then + rm xx01 + fi sed -i -E PmWiki/${base}/${file}.HTM \ -e "s/rel='nofollow'//g" \ -e "s/target='_blank'//g" \ @@ -53,7 +57,6 @@ do -e 's/\{#.*\}//g' \ -e '/^:::/d' \ # -e '/\[Site$/d' \ -# -e '/^Page last modified on /d' \ # -e '/^\[\]/d' \ # -e "s/\`\`\{=html\}\`<\/a>\`\{=html\}//g" \ # -e "s/^\`> PmWiki/${base}/${file}.md ln -frs PmWiki/${base}/${file}.md combined/${base}/${file}.md - - if [ -f xx01 ]; then - rm xx01 - fi + ln -frs PmWiki/${base}/${file}.md.md combined/${base}/${file}.md.md done popd diff --git a/TODO.md b/TODO.md index 60a2eca..84f62b8 100644 --- a/TODO.md +++ b/TODO.md @@ -6,26 +6,27 @@ Bugs - - /users/Plentyn/WebHome.HTML   has that twisty thing which looks not simple to remove. - /users/dunno/Devuan Cluster.HTML   A very lengthy and complex document, I'll likely miss something, but chip away at the obvious. - /users/Debdog.HTML   pandoc can't handle the background table cell colours in the "Background colours" table, which is kinda the point of it. -- PmWiki in it's current config needs that ?n=foo.bar nonsense for the Original page link. Which I'm currently neatly sidestepping, the scraped page has a similar thing. Check the timestamps on the files, only update if source is newer than destination. Meh, it's already 600 times faster than the pandoc version. + - One quirk to watch for is if a URL path changes, the docs that have that URL need to be redone. - pandoc is a lot slower though, so do this for sure when dealing with that. - When scraping the web sites, they tend to be dynamically generated with no useful timestamp on them. - The web site scrape happens locally anyway, I can compare source file timestamps. Add atom feed for single page. Alas cgit only seems to have ATOM feed on the whole repo, not individual files. + - git.devuan.org might have usable per page history. - However, once timestamps are sorted, I can use that code to generate RSS and ATOM feeds, and create page histories using diffs. Deal with complex directory trees. + - /testing/even should display as even/deeper on the testing page, coz even/ has no files, but even/deeper does. - On the other hand, I could just create an empty page with the sub directories shown as usual. - Scanning /usr/share/doc on my super desktop with looots of software installed will be fun. - On the gripping hand, this is where we want empty directories to vanish. - ## Maybe not Extract title from Fos and Pm, maybe even pagetitle if possible. Title is pointless, both of them just use the file name. @@ -34,12 +35,13 @@ Extract title from Fos and Pm, maybe even pagetitle if possible. Title is point ## Try out +Lua pattern matching sucks in general. Might be why people use lpeg instead. Try it. + htmx cgit has Lua - ## User system Reuse the user system from SledjChisl. diff --git a/default.template b/default.template index 9f2490b..bedfdcf 100644 --- a/default.template +++ b/default.template @@ -2,7 +2,6 @@ - $title$ diff --git a/notYetAnotherWiki.lua b/notYetAnotherWiki.lua index 8c9a5c9..dc64fa2 100755 --- a/notYetAnotherWiki.lua +++ b/notYetAnotherWiki.lua @@ -64,10 +64,11 @@ local globalData = { ['devuanDeepSeaDark'] = '#132f40', ['devuanDeepSeaLight'] = '#1a4562', ['devuanSaphireDark'] = '#004489', ['devuanSaphireLight'] = '#00509f', ['karenPurple'] = '#8800ff', ['onefangPurple'] = '#cc00ff', - favicon = 'nYAW_icon.png', logo = 'nYAW.png', header = '', home = '', --menu = '', - history = '', footer = 'Powered by notYetAnotherWiki version 0.0.  ', + favicon = 'nYAW_icon.png', logo = 'nYAW.png', + header = '', home = '', history = '', footer = 'Powered by notYetAnotherWiki version 0.0.  ', + --menu = '', } -local Sites, Files, Subs = {}, {}, {} +local Files, Subs = {}, {} @@ -86,7 +87,7 @@ local derefTable = function(t, strip) end --- String together the bits array into a path string. +-- String together the bits array into a path string. Or the other way around. lol local stringBits = function(l) local bits = {} local last = 1 @@ -102,7 +103,19 @@ end -- Put a value into the Files or Subs table, creating things if needed. local toFile = function(name, key, value) - if nil == Files[name] then Files[name] = {} end + if nil == Files[name] then + local bits, bit = stringBits(name) + local path = '' + Files[name] = {} + Files[name].bits = bits + Files[name].bit = bit + for i, d in ipairs(bits) do + if '' ~= path then path = path .. '/' end + path = path .. d + end + Files[name].path = path +-- Files[name].body = '' + end if nil ~= key then Files[name][key] = value end for i, v in ipairs{'metadata', 'bits', } do if nil == Files[name][v] then Files[name][v] = {} end @@ -120,9 +133,15 @@ end -- Actually start doing things. +-- Create the base of everything.md here, so it gets picked up as usual in the file scan. +local body, h = '', io.open('everything.md', 'w') +h:write('# All the pages\n') +h:close() + + -- Scan the subdirectories looking for .md files. local directory = arg[1] -toSub('') --Subs[''] = {files = {}, subs = {}, bits = {}} +toSub('') if nil == directory then directory = '.' end if '.' ~= directory then for l in io.popen('find . -name "*.md" -type f,l -printf "%P\n"'):lines() do @@ -135,39 +154,60 @@ for l in io.popen('find ' .. directory .. ' -name "*.md" -type f,l -printf "%P\n if nil == Files[n] then toFile(n) end end --- Gotta figure out all the files and subs first. File and sub metadata comes along for the ride. + +-- Gotta figure out all the files and subs first. File and sub metadata comes along for the ride, coz we need them later. +local newMeta = {} for name, file in pairs(Files) do local bitter, path = '', '' - local bits, bit = stringBits(name) + local bits, bit = file.bits, file.bit local ln = #bits local body, metadata = '', {} -- Go through our bits, construct Subs with bits. - Files[name].bits = bits - Files[name].bit = bit if ln > 0 then bitter = bits[1] end if '' ~= bitter then Subs[''].subs[bitter] = bitter end -- "bitter end" was entirely by accident, I'm keeping it. B-) for i, d in ipairs(bits) do if '' ~= path then path = path .. '/' end path = path .. d toSub(path, 'bits', derefiTable(bits, true)) - if i < ln then Subs[path].subs[bits[i + 1]] = bits[i + 1] end - if i < ln then table.remove(Subs[path].bits, #bits) end + if i < ln then + Subs[path].subs[bits[i + 1]] = bits[i + 1] + table.remove(Subs[path].bits, #bits) + end end if '.md' == string.sub(name, -3, -1) then -- This is a metadata only file, no content, stash the matadata. for l in io.open(name .. '.md'):lines() do - for k, v in string.gmatch(l, "(%w+)%s*=%s*(.+)") do metadata[k] = v end + for k, v in string.gmatch(l, "(%w+)%s*=%s*(.+)") do + if nil == v then + print(name .. ' ' .. k) + else + metadata[k] = v + end + end end if '.md' == name then toSub(path, 'metadata', metadata) elseif '/.md' == string.sub(name, -4, -1) then toSub(path, 'metadata', metadata) - else toFile(string.sub(name, 1, -4), 'metadata', metadata) +-- else toFile(string.sub(name, 1, -4), 'metadata', metadata) + else newMeta[string.sub(name, 1, -4)] = metadata end Files[name] = nil - else + end +end - -- Start the file parsing here, coz we need it's metadata. +-- FIXTHEM - Lua doesn't like modifying the thing you are pair()ing, like we want to do in the last loop. +for name, file in pairs(newMeta) do + if nil == Files[name] then toFile(name) end + if nil == Files[name].metadata then Files[name].metadata = {} end + for k, v in pairs(file) do + if nil == Files[name].metadata[k] then Files[name].metadata[k] = v end + end +end + +-- Open the files and do the iniital cleanups. +for name, file in pairs(Files) do + if '.md' ~= string.sub(name, -3, -1) then -- print('Parsing ' .. name .. '.md') h = io.open(name .. '.md', 'r') if nil ~= h then @@ -201,13 +241,31 @@ for name, file in pairs(Files) do end until fail == f0 result = result .. string.sub(body, start) + body = result + end + Files[name].body = body + table.insert(Subs[Files[name].path].files, Files[name].bit) + end +end - body = parse(result) +-- Create an "everything" page, for URL links to every file.HTML. +-- TODO - sort the list of pages. +local bdy, h = Files['everything'].body, io.open('everything.md', 'a+') +bdy = bdy .. '\n\n| page | converted | Foswiki | PmWiki | \n| ---- | --------- | ------- | ------ | ' +for name, file in pairs(Files) do + local metadata = derefTable(Files[name].metadata, true) + if 'everything' ~= name then + local ln, fw, pw = 'DUNNO', '', '' + if 'PmWiki' == metadata.ogWiki then pw = '[' .. metadata.ogURL .. '](' .. metadata.ogURL .. ')' end + if 'Foswiki' == metadata.ogWiki then fw = '[' .. metadata.ogURL .. '](' .. metadata.ogURL .. ')' end + if nil ~= file.bit then ln = file.bit end - toFile(name, 'body', body) - table.insert(Subs[path].files, bit) + bdy = bdy .. '\n| ' .. name .. ' | [' .. ln .. '](<' .. name .. '.HTML>) | ' .. fw .. ' | ' .. pw .. ' |' end end +h:write(bdy) +h:close() +toFile('everything', 'body', parse(bdy)) @@ -252,16 +310,20 @@ local linkFrom = function(source, dest) break end end - depth = #(Subs[source].bits) - depth - depth = depth + 1 - link = string.rep('../', depth) - if (0 == depth) or (depth > #(Subs[dest].bits)) then + + if #(Subs[dest].bits) >= #(Subs[source].bits) then + depth = #(Subs[source].bits) for i, v in ipairs(Subs[dest].bits) do - if i >= depth then + if i > depth then if '' ~= link then link = link .. '/' end link = link .. Subs[dest].bits[i] end end + if '' ~= link then link = link .. '/' end + else + depth = #(Subs[source].bits) - depth + depth = depth + 1 + link = string.rep('../', depth) end end return link @@ -269,12 +331,28 @@ end --- TODO - loop through Subs, doing whichPage and inheritance. - - - -- More of this actually doing things nonsense. +-- Loop through Subs, doing whichPage and inheritance. +for name, sub in pairs(Subs) do + sub.whichPage = whichPage(name) + local metadata = sub.metadata + for i, s in pairs(sub.subs) do + local nm = i + if '' ~= name then nm = name .. '/' .. i end + ss = Subs[nm] + for k, v in pairs(metadata) do + if nil == ss.metadata[k] then + if ('favicon' == k) or ('logo' == k) then + ss.metadata[k] = linkFrom(nm, name) .. v + else + ss.metadata[k] = v + end + end + end + end +end + -- Loop through the files we found and actually create their HTML files. for name, file in pairs(Files) do local path, result = '', '' @@ -285,42 +363,31 @@ for name, file in pairs(Files) do path = table.concat(bits, '/', 1, ln) if '' ~= body then - -- Copy any metadata found in parent directories. - local pth = '' - for i, d in ipairs(bits) do - if '' ~= pth then pth = pth .. '/' end - pth = pth .. d - if nil ~= Subs[pth] then - if nil ~= Subs[pth].metadata then - for m, x in pairs(Subs[pth].metadata) do - if nil == metadata[m] then - metadata[m] = x - end - end - end + -- Inherit stuff from sub and global. + local mdata = Subs[path].metadata + for k, v in pairs(mdata) do + if nil == file.metadata[k] then + file.metadata[k] = v end end - -- Root directory needs to be handled separately, for now. - if nil ~= Subs[''].metadata then - for m, x in pairs(Subs[''].metadata) do if nil == metadata[m] then metadata[m] = x end end - end - + Files[name].metadata = file.metadata + metadata = derefTable(Files[name].metadata, true) for m, x in pairs(globalData) do if nil == metadata[m] then metadata[m] = x end end - -- Inherit these images from most recent parent directory that defines them. - for n, y in ipairs{'favicon', 'logo'} do - local pith = '' - if nil ~= metadata[y] then - local pth, found = '', false - if (nil ~= Subs[''].metadata) and (nil ~= Subs[''].metadata[y]) then pith = '' ; found = true end - for m, x in ipairs(bits) do - if '' ~= pth then pth = pth .. '/' end - pth = pth .. x - if (nil ~= Subs[pth].metadata) and (nil ~= Subs[pth].metadata[y]) then pith = pth ; found = true end - end - if found then metadata[y] = linkFrom(path, pith) .. metadata[y] - else metadata[y] = linkFrom(path, pith) .. globalData[y] - end + if nil ~= metadata.ogURL then + body = body .. '\n\n---\n\n[Original page](' .. metadata.ogURL .. '), maybe you can edit it.\n' + end + + -- Figure out this pages trail links. + metadata.home = linkFrom(path, '') .. Subs[''].whichPage + metadata.trail = '' + for i, b in ipairs(bits) do + local p = table.concat(bits, '/', 1, i) + if i < #bits then + metadata.trail = metadata.trail .. '' .. b .. ' 👣   ' + linkFrom(path, table.concat(bits, '/', 1, i)) + else + metadata.trail = metadata.trail .. b .. '   ' end end @@ -334,8 +401,7 @@ for name, file in pairs(Files) do for i, f in ipairs(subs) do local pth = path if '' ~= path then pth = path .. '/' end - local fl = whichPage(pth .. f) - metadata.header = metadata.header .. '' .. f .. '   ' + metadata.header = metadata.header .. '' .. f .. '   ' end -- Figure out this pages menu links. @@ -360,18 +426,6 @@ for name, file in pairs(Files) do end end - -- Figure out this pages trail links. - metadata.home = linkFrom(path, '') .. whichPage('') - metadata.trail = '' - for i, b in ipairs(bits) do - if i < #bits then - metadata.trail = metadata.trail .. '' .. b .. ' 👣   ' - linkFrom(path, table.concat(bits, '/', 1, i)) - else - metadata.trail = metadata.trail .. b .. '   ' - end - end - -- Figure out this pages footer links. if nil ~= metadata.pagehistory then metadata.history = '

Page history

' end if nil ~= metadata.sourcecode then metadata.footer = 'source code     ' .. metadata.footer end @@ -382,12 +436,12 @@ for name, file in pairs(Files) do -- Do our own metadata replacement, it's simple and works better. local temp = template local start = 1 - metadata.body = nil local f0, f1, token -- Toss the body in first, so the scan can deal with it to. f0, f1, token = string.find(temp, '%$(body)%$') if fail ~= f0 then - temp = string.sub(temp, 1, f0 - 1) .. body .. string.sub(temp, f1 + 1) + -- NOTE - this is where we actually parse the markup into HTML. + temp = string.sub(temp, 1, f0 - 1) .. parse(body) .. string.sub(temp, f1 + 1) end -- The actual metadata replacement scan. result = '' -- cgit v1.1