From 98dcc0be5f0df60ba4c4e6c25c495ee3d83374b2 Mon Sep 17 00:00:00 2001 From: onefang Date: Sat, 28 Dec 2019 22:49:01 +1000 Subject: Add bandwidth argument and it's aliases, to choose how hard to hit the 'net. --- apt-panopticommon.lua | 31 +++++++++++++++++++++++++++++++ apt-panopticon.lua | 34 ++++++++++++++++++++++------------ 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/apt-panopticommon.lua b/apt-panopticommon.lua index d9293f6..b1f39d7 100644 --- a/apt-panopticommon.lua +++ b/apt-panopticommon.lua @@ -26,6 +26,19 @@ APT.keep = false APT.options = { + bandwidth = + { + typ = "number", + help = ' 0 = low - HTTP tests for all IPs of all mirrors (HEAD tests), but not URL sanity tests.\n' .. + ' 1 = medium - Also HTTP(S) redirect tests, other protocols, download Release files over HTTP, and check them.\n' .. + ' 2 = high - Also download Packages.xz files that changed, unpack and check them.\n' .. + ' Also download and check InRelease files\n.' .. + ' Pick a few small packages, download them, check their SHA512.\n' .. + ' 3 = more - Also pick more packages, some for each arch.\n' .. + ' 4 = all - Do absolutely everything.\n' .. + ' Actually download some Contents files, and some more Packages.xz, and package files.', + value = 2, + }, referenceSite = { typ = "string", @@ -127,6 +140,24 @@ APT.parseArgs = function(args) elseif "-r" == a then APT.redir = true sendArgs = sendArgs .. a .. " " + elseif "--low" == a then + APT.options.bandwidth.value = 0 + APT.options.timeout.value = 2 + APT.options.timeouts.value = 1 + APT.options.retries.value = 1 + sendArgs = sendArgs .. a .. " " + elseif "--medium" == a then + APT.options.bandwidth.value = 1 + sendArgs = sendArgs .. a .. " " + elseif "--high" == a then + APT.options.bandwidth.value = 2 + sendArgs = sendArgs .. a .. " " + elseif "--more" == a then + APT.options.bandwidth.value = 3 + sendArgs = sendArgs .. a .. " " + elseif "--all" == a then + APT.options.bandwidth.value = 4 + sendArgs = sendArgs .. a .. " " elseif "--" == a:sub(1, 2) then local s, e = a:find("=") if nil == s then e = -1 end diff --git a/apt-panopticon.lua b/apt-panopticon.lua index c2b44ee..b45ca45 100755 --- a/apt-panopticon.lua +++ b/apt-panopticon.lua @@ -350,7 +350,7 @@ checkHEAD = function (host, URL, r, retry, sanity) end I(" " .. code .. " " .. cstr .. ". " .. check .. " " .. host .. " -> " .. URL) -- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through. - if nil ~= location then + if (1 <= APT.options.bandwidth.value) and (nil ~= location) then pu = url.parse(location, defaultURL) if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change. if APT.testing("Protocol") then @@ -397,7 +397,7 @@ end local checkTimeouts = function(host, scheme, URL) totalTimeouts = totalTimeouts + timeouts; timeouts = 0 checkHEAD(host, scheme .. "://" .. URL) - if APT.testing("URLSanity") then + if (1 <= APT.options.bandwidth.value) and APT.testing("URLSanity") then URL = URL:gsub("/", "///") URL = URL:gsub("///", "/", 1) checkHEAD(host, scheme .. "://" .. URL, 0, 0, true) @@ -424,21 +424,31 @@ local checkFiles = function (host, ip, path, file) if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end else I(" Checking IP " .. host .. " -> " .. ip .. " " .. path) + if 1 <= APT.options.bandwidth.value then + -- Do these first, coz they are likely to fork off a different server. + for i, s in pairs(referenceDebs) do + if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end + if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end + end + end for i, s in pairs(releases) do for j, k in pairs(releaseFiles) do if repoExists(s .. k) then if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end - if checkTimeouts(host, "https", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end + if 1 <= APT.options.bandwidth.value then + if checkTimeouts(host, "https", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end + else + break + end end end + if 0 == APT.options.bandwidth.value then break end end - for i, s in pairs(referenceDebs) do - if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end - if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end - end - for i, s in pairs(referenceDevs) do - if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end - if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end + if 1 <= APT.options.bandwidth.value then + for i, s in pairs(referenceDevs) do + if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end + if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end + end end end end @@ -843,7 +853,7 @@ local parseRelease = function(host) -- TODO - like we do with debs, pick just the smallest Packages.xz that has changed. -- Though we are sorting Release by name, so we can do the diff with the one from results_old, so we'll need to sort by size to. -- pkgmaster still needs to download all the changed Packages.xz files though. - if updated or APT.testing("Integrity") or (APT.options.referenceSite.value == host) then + if (2 <= APT.options.bandwidth.value) and (updated or APT.testing("Integrity") or (APT.options.referenceSite.value == host)) then local dfile, e = io.open(outFile, "r") if nil == dfile then W("opening " .. outFile .. " file - " .. e) else for l in dfile:lines() do @@ -1024,7 +1034,7 @@ if 0 < #arg then end if not APT.redir then - if APT.testing("Integrity") or APT.testing("Updated") then + if (1 <= APT.options.bandwidth.value) and (APT.testing("Integrity") or APT.testing("Updated")) then if APT.origin and (APT.options.roundRobin.value ~= pu.host) then I("Starting file downloads for " .. pu.host) D('*>* About to create coroutine.') -- cgit v1.1