6 files changed, 146 insertions, 92 deletions

diff --git a/.md.md b/.md.md
index 3c5cb8a..4a55e50 100644
--- a/.md.md
+++ b/.md.md
@@ -1,3 +1,5 @@
+favicon=nYAW_icon.png
+logo=nYAW.png
 sourcecode=https://sledjhamr.org/cgit/notYetAnotherWiki/
 pagehistory=https://sledjhamr.org/cgit/notYetAnotherWiki/log
 feedatom=https://sledjhamr.org/cgit/notYetAnotherWiki/atom
diff --git a/SuckItFos b/SuckItFos
index 1e6e48d..0f6f8a8 100755
--- a/SuckItFos
+++ b/SuckItFos
@@ -22,6 +22,7 @@ do
     mkdir -p combined/$base
     mkdir -p combined/${base}/`dirname ${file}`
     echo "Converting ${URL}/${base}/${file}?cover=print -> Foswiki/${base}/${file}.md"
+    echo -e "ogWiki=Foswiki\nogURL=${URL}/${base}/${file}\n" > Foswiki/${base}/${file}.md.md 
     # Doesn't help with redownloads, coz natch a dynamic site isn't cached.  But I can at least comment out the curl command during testing to save time.
     curl --silent --no-progress-meter ${URL}/${base}/${file}?cover=print -o Foswiki/${base}/${file}.HTM
     cp Foswiki/${base}/${file}.HTM Foswiki/${base}/${file}.HTM_ORIGINAL
@@ -47,6 +48,9 @@ do
         rm Foswiki/${base}/${file}.md
         mv xx00 Foswiki/${base}/${file}.md
     fi
+    if [ -f xx01 ]; then
+        rm xx01
+    fi
 
     # Attempt to clean things up, badly.
     sed -i -E Foswiki/${base}/${file}.md \
@@ -57,12 +61,7 @@ do
 #       -e 's/\{\.pattern.*\}//g' \
 #       -e 's/\{\.pattern.*//g' \
 
-    echo -e "****\n[Original page](${URL}/${base}/${file}) where maybe you can edit it." >> Foswiki/${base}/${file}.md
     ln -frs Foswiki/${base}/${file}.md combined/${base}/${file}.md
-
-    if [ -f xx01 ]; then
-        rm xx01
-    fi
 done
 
 popd
diff --git a/SuckItPm b/SuckItPm
index fe7efb3..fd0b048 100755
--- a/SuckItPm
+++ b/SuckItPm
@@ -20,6 +20,7 @@ do
     mkdir -p PmWiki/$base
     mkdir -p combined/$base
     echo "Converting ${URL}/?n=${base}.${file}?action=print -> PmWiki/${base}/${file}.md"
+    echo -e "ogWiki=PmWiki\nogURL=${URL}/?n=${base}.${file}\n" > PmWiki/${base}/${file}.md.md
     # Doesn't help with redownloads, coz natch a dynamic site isn't cached.  But I can at least comment out the curl command during testing to save time.
 #    curl --no-progress-meter ${URL}/?n=${base}.${file}?action=markdown -o PmWiki/${base}/${file}.MD
     curl --no-progress-meter ${URL}/?n=${base}.${file}?action=print -o PmWiki/${base}/${file}.HTM
@@ -29,6 +30,9 @@ do
         rm PmWiki/${base}/${file}.HTM
         mv xx00 PmWiki/${base}/${file}.HTM
     fi
+    if [ -f xx01 ]; then
+        rm xx01
+    fi
     sed -i -E PmWiki/${base}/${file}.HTM \
         -e "s/rel='nofollow'//g" \
         -e "s/target='_blank'//g" \
@@ -53,7 +57,6 @@ do
         -e 's/\{#.*\}//g' \
         -e '/^:::/d' \
 #       -e '/\[Site$/d' \
-#       -e '/^Page last modified on /d' \
 #       -e '/^\[\]/d' \
 #       -e "s/\`<a id='trailstart'>\`\{=html\}\`<\/a>\`\{=html\}//g" \
 #       -e "s/^\`<img /<img /g" \
@@ -61,13 +64,8 @@ do
 #       -e "s/^\`\`\`//g" \
 #       -e "s/\`\{=html\}//g"
 
-    # Don't need this, the parts we are grabbing already include that link at the bottom.
-#    echo -e "****\n[Original page](${URL}/${base}/${page}) where maybe you can edit it." >> PmWiki/${base}/${file}.md
     ln -frs PmWiki/${base}/${file}.md combined/${base}/${file}.md
-
-    if [ -f xx01 ]; then
-        rm xx01
-    fi
+    ln -frs PmWiki/${base}/${file}.md.md combined/${base}/${file}.md.md
 done
 
 popd
diff --git a/TODO.md b/TODO.md
index 60a2eca..84f62b8 100644
--- a/TODO.md
+++ b/TODO.md
@@ -6,26 +6,27 @@ Bugs -
 - /users/Plentyn/WebHome.HTML &nbsp;                    has that twisty thing which looks not simple to remove.
 - /users/dunno/Devuan Cluster.HTML &nbsp;               A very lengthy and complex document, I'll likely miss something, but chip away at the obvious.
 - /users/Debdog.HTML &nbsp;                             pandoc can't handle the background table cell colours in the "Background colours" table, which is kinda the point of it.
-- PmWiki in it's current config needs that ?n=foo.bar nonsense for the Original page link.  Which I'm currently neatly sidestepping, the scraped page has a similar thing.
 
 Check the timestamps on the files, only update if source is newer than destination.  Meh, it's already 600 times faster than the pandoc version.
+
 - One quirk to watch for is if a URL path changes, the docs that have that URL need to be redone.
 - pandoc is a lot slower though, so do this for sure when dealing with that.
 - When scraping the web sites, they tend to be dynamically generated with no useful timestamp on them.
 - The web site scrape happens locally anyway, I can compare source file timestamps.
 
 Add atom feed for single page.  Alas cgit only seems to have ATOM feed on the whole repo, not individual files.
+
 - git.devuan.org might have usable per page history.
 - However, once timestamps are sorted, I can use that code to generate RSS and ATOM feeds, and create page histories using diffs.
 
 Deal with complex directory trees.
+
 - /testing/even should display as even/deeper on the testing page, coz even/ has no files, but even/deeper does.
 - On the other hand, I could just create an empty page with the sub directories shown as usual.
 - Scanning /usr/share/doc on my super desktop with looots of software installed will be fun.
 - On the gripping hand, this is where we want empty directories to vanish.
 
 
-
 ## Maybe not
 
 Extract title from Fos and Pm, maybe even pagetitle if possible.  Title is pointless, both of them just use the file name.
@@ -34,12 +35,13 @@ Extract title from Fos and Pm, maybe even pagetitle if possible.  Title is point
 
 ## Try out
 
+Lua pattern matching sucks in general.  Might be why people use lpeg instead.  Try it.
+
 htmx
 
 cgit has Lua
 
 
-
 ## User system
 
 Reuse the user system from SledjChisl.
diff --git a/default.template b/default.template
index 9f2490b..bedfdcf 100644
--- a/default.template
+++ b/default.template
@@ -2,7 +2,6 @@
 <html>
   <head>
     <meta charset="utf-8">
-    <meta name="generator" contents="lcmark">
     <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
     <title>$title$</title>
     <link rel="icon" type="image/png" href="$favicon$" />
diff --git a/notYetAnotherWiki.lua b/notYetAnotherWiki.lua
index 8c9a5c9..dc64fa2 100755
--- a/notYetAnotherWiki.lua
+++ b/notYetAnotherWiki.lua
@@ -64,10 +64,11 @@ local globalData = {
     ['devuanDeepSeaDark']    = '#132f40',    ['devuanDeepSeaLight']   = '#1a4562',
     ['devuanSaphireDark']    = '#004489',    ['devuanSaphireLight']   = '#00509f',
     ['karenPurple']          = '#8800ff',    ['onefangPurple']        = '#cc00ff',
-    favicon = 'nYAW_icon.png', logo = 'nYAW.png', header = '', home = '', --menu = '',
-    history = '',  footer = 'Powered by <a href="https://sledjhamr.org/cgit/notYetAnotherWiki/about/">notYetAnotherWiki</a> version 0.0. &nbsp;',
+    favicon = 'nYAW_icon.png', logo = 'nYAW.png',
+    header = '', home = '', history = '',  footer = 'Powered by <a href="https://sledjhamr.org/cgit/notYetAnotherWiki/about/">notYetAnotherWiki</a> version 0.0. &nbsp;',
+     --menu = '',
 }
-local Sites, Files, Subs = {}, {}, {}
+local Files, Subs = {}, {}
 
 
 
@@ -86,7 +87,7 @@ local derefTable = function(t, strip)
 end
 
 
--- String together the bits array into a path string.
+-- String together the bits array into a path string.  Or the other way around.  lol
 local stringBits = function(l)
     local bits = {}
     local last = 1
@@ -102,7 +103,19 @@ end
 
 -- Put a value into the Files or Subs table, creating things if needed.
 local toFile = function(name, key, value)
-    if nil == Files[name] then Files[name] = {} end
+    if nil == Files[name] then
+        local bits, bit = stringBits(name)
+        local path = ''
+        Files[name] = {}
+        Files[name].bits = bits
+        Files[name].bit = bit
+        for i, d in ipairs(bits) do
+            if '' ~= path then path = path .. '/' end
+            path = path .. d
+        end
+        Files[name].path = path
+--      Files[name].body = ''
+    end
     if nil ~= key         then Files[name][key] = value end
     for i, v in ipairs{'metadata', 'bits', } do
         if nil == Files[name][v] then Files[name][v] = {} end
@@ -120,9 +133,15 @@ end
 
 -- Actually start doing things.
 
+-- Create the base of everything.md here, so it gets picked up as usual in the file scan.
+local body, h = '', io.open('everything.md', 'w')
+h:write('# All the pages\n')
+h:close()
+
+
 -- Scan the subdirectories looking for .md files.
 local directory = arg[1]
-toSub('')  --Subs[''] = {files = {}, subs = {}, bits = {}}
+toSub('')
 if nil == directory then directory = '.' end
 if '.' ~= directory then
     for l in io.popen('find . -name "*.md" -type f,l -printf "%P\n"'):lines() do
@@ -135,39 +154,60 @@ for l in io.popen('find ' .. directory .. ' -name "*.md" -type f,l -printf "%P\n
     if nil == Files[n] then toFile(n) end
 end
 
--- Gotta figure out all the files and subs first.  File and sub metadata comes along for the ride.
+
+-- Gotta figure out all the files and subs first.  File and sub metadata comes along for the ride, coz we need them later.
+local newMeta = {}
 for name, file in pairs(Files) do
     local bitter, path = '', ''
-    local bits, bit = stringBits(name)
+    local bits, bit = file.bits, file.bit
     local ln = #bits
     local body, metadata = '', {}
 
     -- Go through our bits, construct Subs with bits.
-    Files[name].bits = bits
-    Files[name].bit = bit
     if ln > 0 then bitter = bits[1] end
     if '' ~= bitter then Subs[''].subs[bitter] = bitter end     -- "bitter end" was entirely by accident, I'm keeping it.  B-)
     for i, d in ipairs(bits) do
         if '' ~= path then path = path .. '/' end
         path = path .. d
         toSub(path, 'bits', derefiTable(bits, true))
-        if i < ln               then Subs[path].subs[bits[i + 1]] = bits[i + 1] end
-        if i < ln               then table.remove(Subs[path].bits, #bits) end
+        if i < ln               then
+            Subs[path].subs[bits[i + 1]] = bits[i + 1]
+            table.remove(Subs[path].bits, #bits)
+        end
     end
 
     if '.md' == string.sub(name, -3, -1) then
         -- This is a metadata only file, no content, stash the matadata.
         for l in io.open(name .. '.md'):lines() do
-            for k, v in string.gmatch(l, "(%w+)%s*=%s*(.+)") do metadata[k] = v end
+            for k, v in string.gmatch(l, "(%w+)%s*=%s*(.+)") do 
+                if nil == v then
+                    print(name .. ' ' .. k)
+                else
+                    metadata[k] = v
+                end
+            end
         end
         if '.md' == name                          then toSub(path, 'metadata', metadata)
         elseif '/.md' == string.sub(name, -4, -1) then toSub(path, 'metadata', metadata)
-        else                                           toFile(string.sub(name, 1, -4), 'metadata', metadata)
+--      else                                           toFile(string.sub(name, 1, -4), 'metadata', metadata)
+        else                                            newMeta[string.sub(name, 1, -4)] = metadata
         end
         Files[name] = nil
-    else
+    end
+end
 
-    -- Start the file parsing here, coz we need it's metadata.
+-- FIXTHEM - Lua doesn't like modifying the thing you are pair()ing, like we want to do in the last loop.
+for name, file in pairs(newMeta) do
+    if nil == Files[name] then toFile(name) end
+    if nil == Files[name].metadata then Files[name].metadata = {} end
+    for k, v in pairs(file) do
+        if nil == Files[name].metadata[k] then Files[name].metadata[k] = v end
+    end
+end
+
+-- Open the files and do the iniital cleanups.
+for name, file in pairs(Files) do
+    if '.md' ~= string.sub(name, -3, -1) then
 --      print('Parsing ' .. name .. '.md')
         h = io.open(name .. '.md', 'r')
         if nil ~= h then
@@ -201,13 +241,31 @@ for name, file in pairs(Files) do
                 end
             until fail == f0
             result = result .. string.sub(body, start)
+            body = result
+        end
+        Files[name].body = body
+        table.insert(Subs[Files[name].path].files, Files[name].bit)
+    end
+end
 
-            body = parse(result)
+-- Create an "everything" page, for URL links to every file.HTML.
+-- TODO - sort the list of pages.
+local bdy, h = Files['everything'].body, io.open('everything.md', 'a+')
+bdy = bdy .. '\n\n| page | converted | Foswiki | PmWiki | \n| ---- | --------- | ------- | ------ | '
+for name, file in pairs(Files) do
+    local metadata = derefTable(Files[name].metadata, true)
+    if 'everything' ~= name then
+        local ln, fw, pw = 'DUNNO', '', ''
+        if 'PmWiki'  == metadata.ogWiki then pw = '[' .. metadata.ogURL .. '](' .. metadata.ogURL .. ')' end
+        if 'Foswiki' == metadata.ogWiki then fw = '[' .. metadata.ogURL .. '](' .. metadata.ogURL .. ')' end
+        if nil ~= file.bit   then ln = file.bit
         end
-        toFile(name, 'body', body)
-        table.insert(Subs[path].files, bit)
+        bdy = bdy .. '\n| ' .. name .. ' | [' .. ln .. '](<' .. name .. '.HTML>) | ' .. fw .. ' | ' .. pw .. ' |'
     end
 end
+h:write(bdy)
+h:close()
+toFile('everything', 'body', parse(bdy))
 
 
 
@@ -252,16 +310,20 @@ local linkFrom = function(source, dest)
                 break
             end
         end
-        depth = #(Subs[source].bits) - depth
-        depth = depth + 1
-        link = string.rep('../', depth)
-        if (0 == depth) or (depth > #(Subs[dest].bits)) then
+
+        if #(Subs[dest].bits) >= #(Subs[source].bits) then
+            depth = #(Subs[source].bits)
             for i, v in ipairs(Subs[dest].bits) do
-                if i >= depth then
+                if i > depth then
                     if '' ~= link then link = link .. '/' end
                     link = link .. Subs[dest].bits[i]
                 end
             end
+            if '' ~= link then link = link .. '/' end
+        else
+            depth = #(Subs[source].bits) - depth
+            depth = depth + 1
+            link = string.rep('../', depth)
         end
     end
     return link
@@ -269,12 +331,28 @@ end
 
 
 
--- TODO - loop through Subs, doing whichPage and inheritance.
-
-
-
 -- More of this actually doing things nonsense.
 
+-- Loop through Subs, doing whichPage and inheritance.
+for name, sub in pairs(Subs) do
+    sub.whichPage = whichPage(name)
+    local metadata = sub.metadata
+    for i, s in pairs(sub.subs) do
+        local nm = i
+        if '' ~= name then nm = name .. '/' .. i end
+        ss = Subs[nm]
+        for k, v in pairs(metadata) do
+            if nil == ss.metadata[k] then
+                if ('favicon' == k) or ('logo' == k) then
+                    ss.metadata[k] = linkFrom(nm, name) .. v
+                else
+                    ss.metadata[k] = v
+                end
+            end
+        end
+    end
+end
+
 -- Loop through the files we found and actually create their HTML files.
 for name, file in pairs(Files) do
     local path, result = '', ''
@@ -285,42 +363,31 @@ for name, file in pairs(Files) do
     path = table.concat(bits, '/', 1, ln)
 
     if '' ~= body then
-        -- Copy any metadata found in parent directories.
-        local pth = ''
-        for i, d in ipairs(bits) do
-            if '' ~= pth then pth = pth .. '/' end
-            pth = pth .. d
-            if nil ~= Subs[pth] then
-                if nil ~= Subs[pth].metadata then
-                    for m, x in pairs(Subs[pth].metadata) do
-                        if nil == metadata[m] then
-                            metadata[m] = x
-                        end
-                    end
-                end
+        -- Inherit stuff from sub and global.
+        local mdata = Subs[path].metadata
+        for k, v in pairs(mdata) do
+            if nil == file.metadata[k] then
+                file.metadata[k] = v
             end
         end
-        -- Root directory needs to be handled separately, for now.
-        if nil ~= Subs[''].metadata then
-            for m, x in pairs(Subs[''].metadata)        do if nil == metadata[m] then metadata[m] = x end end
-        end
-
+        Files[name].metadata = file.metadata
+        metadata = derefTable(Files[name].metadata, true)
         for             m, x in pairs(globalData)               do if nil == metadata[m] then metadata[m] = x end end
 
-        -- Inherit these images from most recent parent directory that defines them.
-        for n, y in ipairs{'favicon', 'logo'} do
-            local pith = ''
-            if nil ~= metadata[y] then
-                local pth, found = '', false
-                if (nil ~= Subs[''].metadata) and (nil ~= Subs[''].metadata[y]) then pith = '' ; found = true end
-                for m, x in ipairs(bits) do
-                    if '' ~= pth then pth = pth .. '/' end
-                    pth = pth .. x
-                    if (nil ~= Subs[pth].metadata) and (nil ~= Subs[pth].metadata[y]) then pith = pth ; found = true end
-                end
-                if found then           metadata[y] = linkFrom(path, pith) .. metadata[y]
-                else                    metadata[y] = linkFrom(path, pith) .. globalData[y]
-                end
+        if nil ~= metadata.ogURL then
+            body = body .. '\n\n---\n\n[Original page](' .. metadata.ogURL .. '), maybe you can edit it.\n'
+        end
+
+        -- Figure out this pages trail links.
+        metadata.home = linkFrom(path, '') .. Subs[''].whichPage
+        metadata.trail = ''
+        for i, b in ipairs(bits) do
+            local p = table.concat(bits, '/', 1, i)
+            if i < #bits then
+                metadata.trail = metadata.trail .. '<a href="' .. linkFrom(path, p) .. Subs[p].whichPage .. '">' .. b .. '</a> &#x1f463; &nbsp; '
+                linkFrom(path, table.concat(bits, '/', 1, i))
+            else
+                metadata.trail = metadata.trail ..  b .. ' &nbsp; '
             end
         end
 
@@ -334,8 +401,7 @@ for name, file in pairs(Files) do
         for i, f in ipairs(subs) do
             local pth = path
             if '' ~= path then pth = path .. '/' end
-            local fl = whichPage(pth .. f)
-            metadata.header = metadata.header .. '<a href="' .. f .. '/' .. fl .. '">' .. f .. '</a> &nbsp; '
+            metadata.header = metadata.header .. '<a href="' .. f .. '/' .. whichPage(pth .. f) .. '">' .. f .. '</a> &nbsp; '
         end
 
         -- Figure out this pages menu links.
@@ -360,18 +426,6 @@ for name, file in pairs(Files) do
             end
         end
 
-        -- Figure out this pages trail links.
-        metadata.home = linkFrom(path, '') .. whichPage('')
-        metadata.trail = ''
-        for i, b in ipairs(bits) do
-            if i < #bits then
-                metadata.trail = metadata.trail .. '<a href="' .. linkFrom(path, table.concat(bits, '/', 1, i)) .. whichPage(b) .. '">' .. b .. '</a> &#x1f463; &nbsp; '
-                linkFrom(path, table.concat(bits, '/', 1, i))
-            else
-                metadata.trail = metadata.trail ..  b .. ' &nbsp; '
-            end
-        end
-
         -- Figure out this pages footer links.
         if nil ~= metadata.pagehistory then metadata.history = '<p>Page&nbsp;<a href="' .. metadata.pagehistory .. '/' .. name .. '.md">history</a></p>' end
         if nil ~= metadata.sourcecode   then metadata.footer = '<a href="' .. metadata.sourcecode       .. '">source code</a> &nbsp; &nbsp; ' .. metadata.footer    end
@@ -382,12 +436,12 @@ for name, file in pairs(Files) do
         -- Do our own metadata replacement, it's simple and works better.
         local temp = template
         local start = 1
-        metadata.body = nil
         local f0, f1, token
         -- Toss the body in first, so the scan can deal with it to.
         f0, f1, token = string.find(temp, '%$(body)%$')
         if fail ~= f0 then
-            temp = string.sub(temp, 1, f0 - 1) .. body .. string.sub(temp, f1 + 1)
+            -- NOTE - this is where we actually parse the markup into HTML.
+            temp = string.sub(temp, 1, f0 - 1) .. parse(body) .. string.sub(temp, f1 + 1)
         end
         -- The actual metadata replacement scan.
         result = ''