YAR see description for details.

New everything page, lists all the pages and where they came from, with
links.

Now we know where the page came from, type and URL.

Rejigged how metadata works.

Do our own metadata substitution.

Fixed up a bunch of bugs.

General cleanups.

6 files changed, 146 insertions, 92 deletions

diff --git a/.md.md b/.md.md
index 3c5cb8a..4a55e50 100644
--- a/.md.md
+++ b/.md.md
@@ -1,3 +1,5 @@
+favicon=nYAW_icon.png
+logo=nYAW.png
 sourcecode=https://sledjhamr.org/cgit/notYetAnotherWiki/
 pagehistory=https://sledjhamr.org/cgit/notYetAnotherWiki/log
 feedatom=https://sledjhamr.org/cgit/notYetAnotherWiki/atom
diff --git a/SuckItFos b/SuckItFos
index 1e6e48d..0f6f8a8 100755
--- a/SuckItFos
+++ b/SuckItFos
@@ -22,6 +22,7 @@ do
     mkdir -p combined/$base
     mkdir -p combined/${base}/`dirname ${file}`
     echo "Converting ${URL}/${base}/${file}?cover=print -> Foswiki/${base}/${file}.md"
+    echo -e "ogWiki=Foswiki\nogURL=${URL}/${base}/${file}\n" > Foswiki/${base}/${file}.md.md 
     # Doesn't help with redownloads, coz natch a dynamic site isn't cached.  But I can at least comment out the curl command during testing to save time.
     curl --silent --no-progress-meter ${URL}/${base}/${file}?cover=print -o Foswiki/${base}/${file}.HTM
     cp Foswiki/${base}/${file}.HTM Foswiki/${base}/${file}.HTM_ORIGINAL
@@ -47,6 +48,9 @@ do
         rm Foswiki/${base}/${file}.md
         mv xx00 Foswiki/${base}/${file}.md
     fi
+    if [ -f xx01 ]; then
+        rm xx01
+    fi
 
     # Attempt to clean things up, badly.
     sed -i -E Foswiki/${base}/${file}.md \
@@ -57,12 +61,7 @@ do
 #       -e 's/\{\.pattern.*\}//g' \
 #       -e 's/\{\.pattern.*//g' \
 
-    echo -e "****\n[Original page](${URL}/${base}/${file}) where maybe you can edit it." >> Foswiki/${base}/${file}.md
     ln -frs Foswiki/${base}/${file}.md combined/${base}/${file}.md
-
-    if [ -f xx01 ]; then
-        rm xx01
-    fi
 done
 
 popd
diff --git a/SuckItPm b/SuckItPm
index fe7efb3..fd0b048 100755
--- a/SuckItPm
+++ b/SuckItPm
@@ -20,6 +20,7 @@ do
     mkdir -p PmWiki/$base
     mkdir -p combined/$base
     echo "Converting ${URL}/?n=${base}.${file}?action=print -> PmWiki/${base}/${file}.md"
+    echo -e "ogWiki=PmWiki\nogURL=${URL}/?n=${base}.${file}\n" > PmWiki/${base}/${file}.md.md
     # Doesn't help with redownloads, coz natch a dynamic site isn't cached.  But I can at least comment out the curl command during testing to save time.
 #    curl --no-progress-meter ${URL}/?n=${base}.${file}?action=markdown -o PmWiki/${base}/${file}.MD
     curl --no-progress-meter ${URL}/?n=${base}.${file}?action=print -o PmWiki/${base}/${file}.HTM
@@ -29,6 +30,9 @@ do
         rm PmWiki/${base}/${file}.HTM
         mv xx00 PmWiki/${base}/${file}.HTM
     fi
+    if [ -f xx01 ]; then
+        rm xx01
+    fi
     sed -i -E PmWiki/${base}/${file}.HTM \
         -e "s/rel='nofollow'//g" \
         -e "s/target='_blank'//g" \
@@ -53,7 +57,6 @@ do
         -e 's/\{#.*\}//g' \
         -e '/^:::/d' \
 #       -e '/\[Site$/d' \
-#       -e '/^Page last modified on /d' \
 #       -e '/^\[\]/d' \
 #       -e "s/\`<a id='trailstart'>\`\{=html\}\`<\/a>\`\{=html\}//g" \
 #       -e "s/^\`<img /<img /g" \
@@ -61,13 +64,8 @@ do
 #       -e "s/^\`\`\`//g" \
 #       -e "s/\`\{=html\}//g"
 
-    # Don't need this, the parts we are grabbing already include that link at the bottom.
-#    echo -e "****\n[Original page](${URL}/${base}/${page}) where maybe you can edit it." >> PmWiki/${base}/${file}.md
     ln -frs PmWiki/${base}/${file}.md combined/${base}/${file}.md
-
-    if [ -f xx01 ]; then
-        rm xx01
-    fi
+    ln -frs PmWiki/${base}/${file}.md.md combined/${base}/${file}.md.md
 done
 
 popd
diff --git a/TODO.md b/TODO.md
index 60a2eca..84f62b8 100644
--- a/TODO.md
+++ b/TODO.md
@@ -6,26 +6,27 @@ Bugs -
 - /users/Plentyn/WebHome.HTML &nbsp;                    has that twisty thing which looks not simple to remove.
 - /users/dunno/Devuan Cluster.HTML &nbsp;               A very lengthy and complex document, I'll likely miss something, but chip away at the obvious.
 - /users/Debdog.HTML &nbsp;                             pandoc can't handle the background table cell colours in the "Background colours" table, which is kinda the point of it.
-- PmWiki in it's current config needs that ?n=foo.bar nonsense for the Original page link.  Which I'm currently neatly sidestepping, the scraped page has a similar thing.
 
 Check the timestamps on the files, only update if source is newer than destination.  Meh, it's already 600 times faster than the pandoc version.
+
 - One quirk to watch for is if a URL path changes, the docs that have that URL need to be redone.
 - pandoc is a lot slower though, so do this for sure when dealing with that.
 - When scraping the web sites, they tend to be dynamically generated with no useful timestamp on them.
 - The web site scrape happens locally anyway, I can compare source file timestamps.
 
 Add atom feed for single page.  Alas cgit only seems to have ATOM feed on the whole repo, not individual files.
+
 - git.devuan.org might have usable per page history.
 - However, once timestamps are sorted, I can use that code to generate RSS and ATOM feeds, and create page histories using diffs.
 
 Deal with complex directory trees.
+
 - /testing/even should display as even/deeper on the testing page, coz even/ has no files, but even/deeper does.
 - On the other hand, I could just create an empty page with the sub directories shown as usual.
 - Scanning /usr/share/doc on my super desktop with looots of software installed will be fun.
 - On the gripping hand, this is where we want empty directories to vanish.
 
 
-
 ## Maybe not
 
 Extract title from Fos and Pm, maybe even pagetitle if possible.  Title is pointless, both of them just use the file name.
@@ -34,12 +35,13 @@ Extract title from Fos and Pm, maybe even pagetitle if possible.  Title is point
 
 ## Try out
 
+Lua pattern matching sucks in general.  Might be why people use lpeg instead.  Try it.
+
 htmx
 
 cgit has Lua
 
 
-
 ## User system
 
 Reuse the user system from SledjChisl.
diff --git a/default.template b/default.template
index 9f2490b..bedfdcf 100644
--- a/default.template
+++ b/default.template
@@ -2,7 +2,6 @@
 <html>
   <head>
     <meta charset="utf-8">
-    <meta name="generator" contents="lcmark">
     <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
     <title>$title$</title>
     <link rel="icon" type="image/png" href="$favicon$" />
diff --git a/notYetAnotherWiki.lua b/notYetAnotherWiki.lua
index 8c9a5c9..dc64fa2 100755
--- a/notYetAnotherWiki.lua
+++ b/notYetAnotherWiki.lua
@@ -64,10 +64,11 @@ local globalData = {
     ['devuanDeepSeaDark']    = '#132f40',    ['devuanDeepSeaLight']   = '#1a4562',
     ['devuanSaphireDark']    = '#004489',    ['devuanSaphireLight']   = '#00509f',
     ['karenPurple']          = '#8800ff',    ['onefangPurple']        = '#cc00ff',
-    favicon = 'nYAW_icon.png', logo = 'nYAW.png', header = '', home = '', --menu = '',
-    history = '',  footer = 'Powered by <a href="https://sledjhamr.org/cgit/notYetAnotherWiki/about/">notYetAnotherWiki</a> version 0.0. &nbsp;',
+    favicon = 'nYAW_icon.png', logo = 'nYAW.png',
+    header = '', home = '', history = '',  footer = 'Powered by <a href="https://sledjhamr.org/cgit/notYetAnotherWiki/about/">notYetAnotherWiki</a> version 0.0. &nbsp;',
+     --menu = '',
 }
-local Sites, Files, Subs = {}, {}, {}
+local Files, Subs = {}, {}
 
 
 
@@ -86,7 +87,7 @@ local derefTable = function(t, strip)
 end
 
 
--- String together the bits array into a path string.
+-- String together the bits array into a path string.  Or the other way around.  lol
 local stringBits = function(l)
     local bits = {}
     local last = 1
@@ -102,7 +103,19 @@ end
 
 -- Put a value into the Files or Subs table, creating things if needed.
 local toFile = function(name, key, value)
-    if nil == Files[name] then Files[name] = {} end
+    if nil == Files[name] then
+        local bits, bit = stringBits(name)
+        local path = ''
+        Files[name] = {}
+        Files[name].bits = bits
+        Files[name].bit = bit
+        for i, d in ipairs(bits) do
+            if '' ~= path then path = path .. '/' end
+            path = path .. d
+        end
+        Files[name].path = path
+--      Files[name].body = ''
+    end
     if nil ~= key         then Files[name][key] = value end
     for i, v in ipairs{'metadata', 'bits', } do
         if nil == Files[name][v] then Files[name][v] = {} end
@@ -120,9 +133,15 @@ end
 
 -- Actually start doing things.
 
+-- Create the base of everything.md here, so it gets picked up as usual in the file scan.
+local body, h = '', io.open('everything.md', 'w')
+h:write('# All the pages\n')
+h:close()
+
+
 -- Scan the subdirectories looking for .md files.
 local directory = arg[1]
-toSub('')  --Subs[''] = {files = {}, subs = {}, bits = {}}
+toSub('')
 if nil == directory then directory = '.' end
 if '.' ~= directory then
     for l in io.popen('find . -name "*.md" -type f,l -printf "%P\n"'):lines() do
@@ -135,39 +154,60 @@ for l in io.popen('find ' .. directory .. ' -name "*.md" -type f,l -printf "%P\n
     if nil == Files[n] then toFile(n) end
 end
 
--- Gotta figure out all the files and subs first.  File and sub metadata comes along for the ride.
+
+-- Gotta figure out all the files and subs first.  File and sub metadata comes along for the ride, coz we need them later.
+local newMeta = {}
 for name, file in pairs(Files) do
     local bitter, path = '', ''
-    local bits, bit = stringBits(name)
+    local bits, bit = file.bits, file.bit
     local ln = #bits
     local body, metadata = '', {}
 
     -- Go through our bits, construct Subs with bits.
-    Files[name].bits = bits
-    Files[name].bit = bit
     if ln > 0 then bitter = bits[1] end
     if '' ~= bitter then Subs[''].subs[bitter] = bitter end     -- "bitter end" was entirely by accident, I'm keeping it.  B-)
     for i, d in ipairs(bits) do
         if '' ~= path then path = path .. '/' end
         path = path .. d
         toSub(path, 'bits', derefiTable(bits, true))
-        if i < ln               then Subs[path].subs[bits[i + 1]] = bits[i + 1] end
-        if i < ln               then table.remove(Subs[path].bits, #bits) end
+        if i < ln               then
+            Subs[path].subs[bits[i + 1]] = bits[i + 1]
+            table.remove(Subs[path].bits, #bits)
+        end
     end
 
     if '.md' == string.sub(name, -3, -1) then
         -- This is a metadata only file, no content, stash the matadata.
         for l in io.open(name .. '.md'):lines() do
-            for k, v in string.gmatch(l, "(%w+)%s*=%s*(.+)") do metadata[k] = v end
+            for k, v in string.gmatch(l, "(%w+)%s*=%s*(.+)") do 
+                if nil == v then
+                    print(name .. ' ' .. k)
+                else
+                    metadata[k] = v
+                end
+            end
         end
         if '.md' == name                          then toSub(path, 'metadata', metadata)
         elseif '/.md' == string.sub(name, -4, -1) then toSub(path, 'metadata', metadata)
-        else                                           toFile(string.sub(name, 1, -4), 'metadata', metadata)
+--      else                                           toFile(string.sub(name, 1, -4), 'metadata', metadata)
+        else                                            newMeta[string.sub(name, 1, -4)] = metadata
         end
         Files[name] = nil
-    else
+    end
+end
 
-    -- Start the file parsing here, coz we need it's metadata.
+-- FIXTHEM - Lua doesn't like modifying the thing you are pair()ing, like we want to do in the last loop.
+for name, file in pairs(newMeta) do
+    if nil == Files[name] then toFile(name) end
+    if nil == Files[name].metadata then Files[name].metadata = {} end
+    for k, v in pairs(file) do
+        if nil == Files[name].metadata[k] then Files[name].metadata[k] = v end
+    end
+end
+
+-- Open the files and do the iniital cleanups.
+for name, file in pairs(Files) do
+    if '.md' ~= string.sub(name, -3, -1) then
 --      print('Parsing ' .. name .. '.md')
         h = io.open(name .. '.md', 'r')
         if nil ~= h then
@@ -201,13 +241,31 @@ for name, file in pairs(Files) do
                 end
             until fail == f0
             result = result .. string.sub(body, start)
+            body = result
+        end
+        Files[name].body = body
+        table.insert(Subs[Files[name].path].files, Files[name].bit)
+    end
+end
 
-            body = parse(result)
+-- Create an "everything" page, for URL links to every file.HTML.
+-- TODO - sort the list of pages.
+local bdy, h = Files['everything'].body, io.open('everything.md', 'a+')
+bdy = bdy .. '\n\n| page | converted | Foswiki | PmWiki | \n| ---- | --------- | ------- | ------ | '
+for name, file in pairs(Files) do
+    local metadata = derefTable(Files[name].metadata, true)
+    if 'everything' ~= name then
+        local ln, fw, pw = 'DUNNO', '', ''
+        if 'PmWiki'  == metadata.ogWiki then pw = '[' .. metadata.ogURL .. '](' .. metadata.ogURL .. ')' end
+        if 'Foswiki' == metadata.ogWiki then fw = '[' .. metadata.ogURL .. '](' .. metadata.ogURL .. ')' end
+        if nil ~= file.bit   then ln = file.bit
         end
-        toFile(name, 'body', body)
-        table.insert(Subs[path].files, bit)
+        bdy = bdy .. '\n| ' .. name .. ' | [' .. ln .. '](<' .. name .. '.HTML>) | ' .. fw .. ' | ' .. pw .. ' |'
     end
 end
+h:write(bdy)
+h:close()
+toFile('everything', 'body', parse(bdy))
 
 
 
@@ -252,16 +310,20 @@ local linkFrom = function(source, dest)
                 break
             end
         end
-        depth = #(Subs[source].bits) - depth
-        depth = depth + 1
-        link = string.rep('../', depth)
-        if (0 == depth) or (depth > #(Subs[dest].bits)) then
+
+        if #(Subs[dest].bits) >= #(Subs[source].bits) then
+            depth = #(Subs[source].bits)
             for i, v in ipairs(Subs[dest].bits) do
-                if i >= depth then
+                if i > depth then
                     if '' ~= link then link = link .. '/' end
                     link = link .. Subs[dest].bits[i]
                 end
             end
+            if '' ~= link then link = link .. '/' end
+        else
+            depth = #(Subs[source].bits) - depth
+            depth = depth + 1
+            link = string.rep('../', depth)
         end
     end
     return link
@@ -269,12 +331,28 @@ end
 
 
 
--- TODO - loop through Subs, doing whichPage and inheritance.
-
-
-
 -- More of this actually doing things nonsense.
 
+-- Loop through Subs, doing whichPage and inheritance.
+for name, sub in pairs(Subs) do
+    sub.whichPage = whichPage(name)
+    local metadata = sub.metadata
+    for i, s in pairs(sub.subs) do
+        local nm = i
+        if '' ~= name then nm = name .. '/' .. i end
+        ss = Subs[nm]
+        for k, v in pairs(metadata) do
+            if nil == ss.metadata[k] then
+                if ('favicon' == k) or ('logo' == k) then
+                    ss.metadata[k] = linkFrom(nm, name) .. v
+                else
+                    ss.metadata[k] = v
+                end
+            end
+        end
+    end
+end
+
 -- Loop through the files we found and actually create their HTML files.
 for name, file in pairs(Files) do
     local path, result = '', ''
@@ -285,42 +363,31 @@ for name, file in pairs(Files) do
     path = table.concat(bits, '/', 1, ln)
 
     if '' ~= body then
-        -- Copy any metadata found in parent directories.
-        local pth = ''
-        for i, d in ipairs(bits) do
-            if '' ~= pth then pth = pth .. '/' end
-            pth = pth .. d
-            if nil ~= Subs[pth] then
-                if nil ~= Subs[pth].metadata then
-                    for m, x in pairs(Subs[pth].metadata) do
-                        if nil == metadata[m] then
-                            metadata[m] = x
-                        end
-                    end
-                end
+        -- Inherit stuff from sub and global.
+        local mdata = Subs[path].metadata
+        for k, v in pairs(mdata) do
+            if nil == file.metadata[k] then
+                file.metadata[k] = v
             end
         end
-        -- Root directory needs to be handled separately, for now.
-        if nil ~= Subs[''].metadata then
-            for m, x in pairs(Subs[''].metadata)        do if nil == metadata[m] then metadata[m] = x end end
-        end
-
+        Files[name].metadata = file.metadata
+        metadata = derefTable(Files[name].metadata, true)
         for             m, x in pairs(globalData)               do if nil == metadata[m] then metadata[m] = x end end
 
-        -- Inherit these images from most recent parent directory that defines them.
-        for n, y in ipairs{'favicon', 'logo'} do
-            local pith = ''
-            if nil ~= metadata[y] then
-                local pth, found = '', false
-                if (nil ~= Subs[''].metadata) and (nil ~= Subs[''].metadata[y]) then pith = '' ; found = true end
-                for m, x in ipairs(bits) do
-                    if '' ~= pth then pth = pth .. '/' end
-                    pth = pth .. x
-                    if (nil ~= Subs[pth].metadata) and (nil ~= Subs[pth].metadata[y]) then pith = pth ; found = true end
-                end
-                if found then           metadata[y] = linkFrom(path, pith) .. metadata[y]
-                else                    metadata[y] = linkFrom(path, pith) .. globalData[y]
-                end
+        if nil ~= metadata.ogURL then
+            body = body .. '\n\n---\n\n[Original page](' .. metadata.ogURL .. '), maybe you can edit it.\n'
+        end
+
+        -- Figure out this pages trail links.
+        metadata.home = linkFrom(path, '') .. Subs[''].whichPage
+        metadata.trail = ''
+        for i, b in ipairs(bits) do
+            local p = table.concat(bits, '/', 1, i)
+            if i < #bits then
+                metadata.trail = metadata.trail .. '<a href="' .. linkFrom(path, p) .. Subs[p].whichPage .. '">' .. b .. '</a> &#x1f463; &nbsp; '
+                linkFrom(path, table.concat(bits, '/', 1, i))
+            else
+                metadata.trail = metadata.trail ..  b .. ' &nbsp; '
             end
         end
 
@@ -334,8 +401,7 @@ for name, file in pairs(Files) do
         for i, f in ipairs(subs) do
             local pth = path
             if '' ~= path then pth = path .. '/' end
-            local fl = whichPage(pth .. f)
-            metadata.header = metadata.header .. '<a href="' .. f .. '/' .. fl .. '">' .. f .. '</a> &nbsp; '
+            metadata.header = metadata.header .. '<a href="' .. f .. '/' .. whichPage(pth .. f) .. '">' .. f .. '</a> &nbsp; '
         end
 
         -- Figure out this pages menu links.
@@ -360,18 +426,6 @@ for name, file in pairs(Files) do
             end
         end
 
-        -- Figure out this pages trail links.
-        metadata.home = linkFrom(path, '') .. whichPage('')
-        metadata.trail = ''
-        for i, b in ipairs(bits) do
-            if i < #bits then
-                metadata.trail = metadata.trail .. '<a href="' .. linkFrom(path, table.concat(bits, '/', 1, i)) .. whichPage(b) .. '">' .. b .. '</a> &#x1f463; &nbsp; '
-                linkFrom(path, table.concat(bits, '/', 1, i))
-            else
-                metadata.trail = metadata.trail ..  b .. ' &nbsp; '
-            end
-        end
-
         -- Figure out this pages footer links.
         if nil ~= metadata.pagehistory then metadata.history = '<p>Page&nbsp;<a href="' .. metadata.pagehistory .. '/' .. name .. '.md">history</a></p>' end
         if nil ~= metadata.sourcecode   then metadata.footer = '<a href="' .. metadata.sourcecode       .. '">source code</a> &nbsp; &nbsp; ' .. metadata.footer    end
@@ -382,12 +436,12 @@ for name, file in pairs(Files) do
         -- Do our own metadata replacement, it's simple and works better.
         local temp = template
         local start = 1
-        metadata.body = nil
         local f0, f1, token
         -- Toss the body in first, so the scan can deal with it to.
         f0, f1, token = string.find(temp, '%$(body)%$')
         if fail ~= f0 then
-            temp = string.sub(temp, 1, f0 - 1) .. body .. string.sub(temp, f1 + 1)
+            -- NOTE - this is where we actually parse the markup into HTML.
+            temp = string.sub(temp, 1, f0 - 1) .. parse(body) .. string.sub(temp, f1 + 1)
         end
         -- The actual metadata replacement scan.
         result = ''