From 13e6c8ef80778d99e88ddbc2a2f2d822a5c71062 Mon Sep 17 00:00:00 2001 From: Pierce Lally Date: Sat, 12 Aug 2023 17:50:24 -0400 Subject: [PATCH] Handle IDN domains and add tests (#27) * add tests and move files to shared * use CFCHTTP.ParseURL in getAddress * change url in tests * add tests to github actions * rename test file and update function calls to moved functions * minor config loader refactor * whitespace changes * rename file -> configFile * add invalid date test cases for address parsing functions --- .github/workflows/gluatest.yml | 8 ++ lua/.luarc.json | 5 + lua/autorun/cfc_http_whitelist_init.lua | 12 ++- .../client/wrap_functions.lua | 4 +- lua/cfc_http_restrictions/config_loader.lua | 64 ------------- lua/cfc_http_restrictions/default_config.lua | 15 ++- .../shared/config_loader.lua | 95 +++++++++++++++++++ .../{client => shared}/filetypes.lua | 6 +- .../{client => shared}/filetypes/html.lua | 19 +--- .../{client => shared}/filetypes/m3u.lua | 0 .../{client => shared}/filetypes/pls.lua | 15 +-- .../{client => shared}/list_manager.lua | 23 +---- lua/cfc_http_restrictions/shared/url.lua | 53 +++++++++++ lua/tests/cfc_cl_http_whitelist/url.lua | 54 +++++++++++ 14 files changed, 247 insertions(+), 126 deletions(-) create mode 100644 .github/workflows/gluatest.yml create mode 100644 lua/.luarc.json delete mode 100644 lua/cfc_http_restrictions/config_loader.lua create mode 100644 lua/cfc_http_restrictions/shared/config_loader.lua rename lua/cfc_http_restrictions/{client => shared}/filetypes.lua (85%) rename lua/cfc_http_restrictions/{client => shared}/filetypes/html.lua (56%) rename lua/cfc_http_restrictions/{client => shared}/filetypes/m3u.lua (100%) rename lua/cfc_http_restrictions/{client => shared}/filetypes/pls.lua (85%) rename lua/cfc_http_restrictions/{client => shared}/list_manager.lua (84%) create mode 100644 lua/cfc_http_restrictions/shared/url.lua create mode 100644 lua/tests/cfc_cl_http_whitelist/url.lua diff --git a/.github/workflows/gluatest.yml b/.github/workflows/gluatest.yml new file mode 100644 index 0000000..ed04a8a --- /dev/null +++ b/.github/workflows/gluatest.yml @@ -0,0 +1,8 @@ +name: GLuaTest Runner + +on: + pull_request: + +jobs: + run-tests: + uses: CFC-Servers/GLuaTest/.github/workflows/gluatest.yml@main diff --git a/lua/.luarc.json b/lua/.luarc.json new file mode 100644 index 0000000..8a11435 --- /dev/null +++ b/lua/.luarc.json @@ -0,0 +1,5 @@ +{ + "diagnostics.globals": [ + "expect" + ] +} diff --git a/lua/autorun/cfc_http_whitelist_init.lua b/lua/autorun/cfc_http_whitelist_init.lua index 789aa57..a17e079 100644 --- a/lua/autorun/cfc_http_whitelist_init.lua +++ b/lua/autorun/cfc_http_whitelist_init.lua @@ -1,5 +1,7 @@ AddCSLuaFile() +CFCHTTP = CFCHTTP or {} + local function includeClient( f ) if SERVER then AddCSLuaFile( f ) @@ -12,9 +14,11 @@ local function includeShared( f ) AddCSLuaFile( f ) include( f ) end -include( "cfc_http_restrictions/config_loader.lua" ) -includeShared( "cfc_http_restrictions/client/filetypes.lua" ) -includeClient( "cfc_http_restrictions/config_loader.lua" ) -includeClient( "cfc_http_restrictions/client/list_manager.lua" ) + +includeShared( "cfc_http_restrictions/shared/config_loader.lua" ) +includeShared( "cfc_http_restrictions/shared/filetypes.lua" ) +includeShared( "cfc_http_restrictions/shared/list_manager.lua" ) +includeShared( "cfc_http_restrictions/shared/url.lua" ) + includeClient( "cfc_http_restrictions/client/list_view.lua" ) includeClient( "cfc_http_restrictions/client/wrap_functions.lua" ) diff --git a/lua/cfc_http_restrictions/client/wrap_functions.lua b/lua/cfc_http_restrictions/client/wrap_functions.lua index b02e7e0..a5b14e4 100644 --- a/lua/cfc_http_restrictions/client/wrap_functions.lua +++ b/lua/cfc_http_restrictions/client/wrap_functions.lua @@ -20,7 +20,7 @@ local function logRequest( method, url, fileLocation, allowed, noisy ) if not url then url = "unknown" elseif isVerbose == false then - local address = CFCHTTP.getAddress( url ) + local address = CFCHTTP.GetAddress( url ) if noisy then return end url = address @@ -182,7 +182,7 @@ local function wrapHTMLPanel( panelName ) logRequest( "GET", options.combinedUri, stack[3], isAllowed ) if not isAllowed then - html = [[

BLOCKED

]] + html = [[

BLOCKED By CFC HTTP Whitelist

]] end return _G[setHTML]( self, html, ... ) diff --git a/lua/cfc_http_restrictions/config_loader.lua b/lua/cfc_http_restrictions/config_loader.lua deleted file mode 100644 index 295595f..0000000 --- a/lua/cfc_http_restrictions/config_loader.lua +++ /dev/null @@ -1,64 +0,0 @@ -CFCHTTP = CFCHTTP or {} -CFCHTTP.config = include( "default_config.lua" ) - -function CFCHTTP.LoadConfigs() - CFCHTTP.config = include( "default_config.lua" ) - CFCHTTP.loadLuaConfigs() - - if CLIENT then - local fileConfig = CFCHTTP.readFileConfig() - if fileConfig then - CFCHTTP.config = CFCHTTP.mergeConfigs( CFCHTTP.config, fileConfig ) - end - end -end - --- LoadLuaConfigs loads the default config and then any lua files in the cfc_http_restrictions/configs directory -function CFCHTTP.loadLuaConfigs() - local files = file.Find( "cfc_http_restrictions/configs/*.lua", "LUA" ) - for _, fil in pairs( files ) do - AddCSLuaFile( "cfc_http_restrictions/configs/" .. fil ) - local newConfig = include( "cfc_http_restrictions/configs/" .. fil ) - CFCHTTP.config = CFCHTTP.mergeConfigs( CFCHTTP.config, newConfig ) - end -end - -function CFCHTTP.mergeConfigs( old, new ) - if new.version == "1" then - if new.wrapHTMLPanels ~= nil then old.wrapHTMLPanels = new.wrapHTMLPanels end - if new.defaultOptions ~= nil then old.defaultOptions = new.defaultOptions end - if new.defaultAssetURIOption ~= nil then old.defaultAssetURIOption = new.defaultAssetURIOption end - - for domain, options in pairs( new.addresses ) do - local currentOptions = old.addresses[domain] - if currentOptions and currentOptions.permanent then - print( "[CFC HTTP Restrictions] Skipping " .. domain .. " because it is permanent" ) - else - old.addresses[domain] = options - end - end - else - ErrorNoHalt( "[CFC HTTP Restrictions] Invalid config version: " .. tostring( new.version ) ) - end - - return old -end - -function CFCHTTP.copyConfig( cfg ) - return util.JSONToTable( util.TableToJSON( cfg ) ) -end - -function CFCHTTP.saveFileConfig( config ) - file.Write( "cfc_cl_http_whitelist_config.json", util.TableToJSON( config, true ) ) - - notification.AddLegacy( "Saved http whitelist", NOTIFY_GENERIC, 5 ) -end - -function CFCHTTP.readFileConfig() - local fileData = file.Read( "cfc_cl_http_whitelist_config.json" ) - if not fileData then return end - - return util.JSONToTable( fileData ) -end - -CFCHTTP.LoadConfigs() \ No newline at end of file diff --git a/lua/cfc_http_restrictions/default_config.lua b/lua/cfc_http_restrictions/default_config.lua index a256f80..d6b4173 100644 --- a/lua/cfc_http_restrictions/default_config.lua +++ b/lua/cfc_http_restrictions/default_config.lua @@ -1,9 +1,17 @@ AddCSLuaFile() -return { +---@alias WhitelistAddressOption { allowed: boolean|nil, noisy: boolean|nil, permanent: boolean|nil } + +---@class WhitelistConfig +---@field version string +---@field wrapHTMLPanels boolean|nil +---@field defaultAssetURIOptions WhitelistAddressOption +---@field defaultOptions WhitelistAddressOption +---@field addresses table +local config = { version = "1", -- this field allows backwards compatibility if the config structure is ever updated - wrapHTMLPanels = false, + wrapHTMLPanels = true, defaultAssetURIOptions = { allowed = true @@ -83,6 +91,7 @@ return { ["(%w+)%.keybase.pub"] = { allowed = true, pattern = true }, ["tts.cyzon.us"] = { allowed = true }, - } } + +return config diff --git a/lua/cfc_http_restrictions/shared/config_loader.lua b/lua/cfc_http_restrictions/shared/config_loader.lua new file mode 100644 index 0000000..4669823 --- /dev/null +++ b/lua/cfc_http_restrictions/shared/config_loader.lua @@ -0,0 +1,95 @@ +---@package +function CFCHTTP.loadClientFileConfig() + local fileConfig = CFCHTTP.ReadFileConfig() + if fileConfig then + CFCHTTP.config = CFCHTTP.mergeConfigs( CFCHTTP.config, fileConfig ) + end +end + +---@package +---@param dir string|nil +function CFCHTTP.loadLuaConfigs( dir ) + dir = dir or "cfc_http_restrictions/configs/" + local files = file.Find( dir .. "*.lua", "LUA" ) + for _, fil in pairs( files ) do + local newConfig = include( dir .. fil ) + CFCHTTP.config = CFCHTTP.mergeConfigs( CFCHTTP.config, newConfig ) + end +end + +---@package +---@param dir string|nil +function CFCHTTP.addCSLuaConfigs( dir ) + dir = dir or "cfc_http_restrictions/configs/" + local files = file.Find( dir .. "*.lua", "LUA" ) + for _, fil in pairs( files ) do + AddCSLuaFile( dir .. fil ) + end +end + +---@package +---@param configFile string|nil +function CFCHTTP.loadDefaultConfg( configFile ) + configFile = configFile or "cfc_http_restrictions/default_config.lua" + CFCHTTP.config = include( configFile ) +end + +function CFCHTTP.LoadConfigsClient() + CFCHTTP.loadDefaultConfg() + CFCHTTP.loadLuaConfigs() + CFCHTTP.loadLuaConfigs( "cfc_http_restrictions/configs/client/" ) + CFCHTTP.loadClientFileConfig() +end + +function CFCHTTP.LoadConfigsServer() + CFCHTTP.loadDefaultConfg() + CFCHTTP.loadLuaConfigs() + CFCHTTP.loadLuaConfigs( "cfc_http_restrictions/configs/server/" ) +end + +function CFCHTTP.mergeConfigs( old, new ) + if new.version == "1" then + if new.wrapHTMLPanels ~= nil then old.wrapHTMLPanels = new.wrapHTMLPanels end + if new.defaultOptions ~= nil then old.defaultOptions = new.defaultOptions end + if new.defaultAssetURIOption ~= nil then old.defaultAssetURIOption = new.defaultAssetURIOption end + + for domain, options in pairs( new.addresses ) do + local currentOptions = old.addresses[domain] + if currentOptions and currentOptions.permanent then + print( "[CFC HTTP Restrictions] Skipping " .. domain .. " because it is permanent" ) + else + old.addresses[domain] = options + end + end + else + ErrorNoHalt( "[CFC HTTP Restrictions] Invalid config version: " .. tostring( new.version ) ) + end + + return old +end + +function CFCHTTP.CopyConfig( cfg ) + return util.JSONToTable( util.TableToJSON( cfg ) ) +end + +function CFCHTTP.SaveFileConfig( config ) + file.Write( "cfc_cl_http_whitelist_config.json", util.TableToJSON( config, true ) ) + + notification.AddLegacy( "Saved http whitelist", NOTIFY_GENERIC, 5 ) +end + +function CFCHTTP.ReadFileConfig() + local fileData = file.Read( "cfc_cl_http_whitelist_config.json" ) + if not fileData then return end + + return util.JSONToTable( fileData ) +end + +if CLIENT then + CFCHTTP.LoadConfigsClient() +else + CFCHTTP.addCSLuaConfigs() + CFCHTTP.addCSLuaConfigs( "cfc_http_restrictions/configs/client/" ) + + CFCHTTP.LoadConfigsServer() +end diff --git a/lua/cfc_http_restrictions/client/filetypes.lua b/lua/cfc_http_restrictions/shared/filetypes.lua similarity index 85% rename from lua/cfc_http_restrictions/client/filetypes.lua rename to lua/cfc_http_restrictions/shared/filetypes.lua index 8ac7056..a878aa1 100644 --- a/lua/cfc_http_restrictions/client/filetypes.lua +++ b/lua/cfc_http_restrictions/shared/filetypes.lua @@ -1,9 +1,9 @@ CFCHTTP.FileTypes = CFCHTTP.FIleTypes or {} -local files, _ = file.Find( "cfc_http_restrictions/client/filetypes/*.lua", "LUA" ) +local files, _ = file.Find( "cfc_http_restrictions/shared/filetypes/*.lua", "LUA" ) for _, f in pairs( files ) do - include( "cfc_http_restrictions/client/filetypes/" .. f ) - AddCSLuaFile( "cfc_http_restrictions/client/filetypes/" .. f ) + include( "cfc_http_restrictions/shared/filetypes/" .. f ) + AddCSLuaFile( "cfc_http_restrictions/shared/filetypes/" .. f ) end ---@param data string diff --git a/lua/cfc_http_restrictions/client/filetypes/html.lua b/lua/cfc_http_restrictions/shared/filetypes/html.lua similarity index 56% rename from lua/cfc_http_restrictions/client/filetypes/html.lua rename to lua/cfc_http_restrictions/shared/filetypes/html.lua index 1c5bd6e..9497d02 100644 --- a/lua/cfc_http_restrictions/client/filetypes/html.lua +++ b/lua/cfc_http_restrictions/shared/filetypes/html.lua @@ -5,10 +5,10 @@ local HTML = { } CFCHTTP.FileTypes.HTML = HTML --- Not implemented ----@param body string +---@param _body string ---@return boolean -function HTML.IsFileData( body ) +---@diagnostic disable-next-line: unused-local +function HTML.IsFileData( _body ) return false end @@ -19,21 +19,10 @@ function HTML.IsFileURL( url ) return false end -local function getUrlsFromText( text ) - local pattern = "%a+://[%a%d%.-]+:?%d*/?[a-zA-Z0-9%.]*" - - local urls = {} - for url in string.gmatch( text, pattern ) do - table.insert( urls, url ) - end - - return urls -end - ---@param body string ---@return string[] urls ---@return string|nil error function HTML.GetURLSFromData( body ) - local urls = getUrlsFromText( body ) + local urls = CFCHTTP.FindURLs( body ) return urls, nil end diff --git a/lua/cfc_http_restrictions/client/filetypes/m3u.lua b/lua/cfc_http_restrictions/shared/filetypes/m3u.lua similarity index 100% rename from lua/cfc_http_restrictions/client/filetypes/m3u.lua rename to lua/cfc_http_restrictions/shared/filetypes/m3u.lua diff --git a/lua/cfc_http_restrictions/client/filetypes/pls.lua b/lua/cfc_http_restrictions/shared/filetypes/pls.lua similarity index 85% rename from lua/cfc_http_restrictions/client/filetypes/pls.lua rename to lua/cfc_http_restrictions/shared/filetypes/pls.lua index 27a3b08..1917c6d 100644 --- a/lua/cfc_http_restrictions/client/filetypes/pls.lua +++ b/lua/cfc_http_restrictions/shared/filetypes/pls.lua @@ -50,19 +50,6 @@ function PLS.IsFileURL( url ) return false end ----@param text string ----@return string[] urls -local function getUrlsFromText( text ) - local pattern = "%a+://[%a%d%.-]+:?%d*/?[a-zA-Z0-9%.]*" - - local urls = {} - for url in string.gmatch( text, pattern ) do - table.insert( urls, url ) - end - - return urls -end - ---@param body string ---@return string[] urls ---@return string|nil error @@ -70,7 +57,7 @@ function PLS.GetURLSFromData( body ) if not PLS.allowed then return {}, "pls files are not allowed" end if #body > PLS.maxFileSize then return {}, "body too large" end - local urls = getUrlsFromText( body ) + local urls = CFCHTTP.FindURLs( body ) local plsData = loadPLSFile( body ) if not plsData.playlist then diff --git a/lua/cfc_http_restrictions/client/list_manager.lua b/lua/cfc_http_restrictions/shared/list_manager.lua similarity index 84% rename from lua/cfc_http_restrictions/client/list_manager.lua rename to lua/cfc_http_restrictions/shared/list_manager.lua index ff1757c..9f9bf74 100644 --- a/lua/cfc_http_restrictions/client/list_manager.lua +++ b/lua/cfc_http_restrictions/shared/list_manager.lua @@ -1,24 +1,5 @@ CFCHTTP = CFCHTTP or {} - -local parsedAddressCache = {} ----@parm url string ----@return string -function CFCHTTP.getAddress( url ) - local cached = parsedAddressCache[url] - if cached then return cached end - - local pattern = "(%a+)://([%a%d%.-]+):?(%d*)/?.*" - local _, _, _, addr, _ = string.find( url, pattern ) - parsedAddressCache[url] = addr - - return addr -end - -function CFCHTTP.isAssetURI( url ) - return string.StartWith( url, "asset://" ) -end - -- escapes all lua pattern characters and allows the use of * as a wildcard local escapedCache = {} local function escapeAddr( addr ) @@ -37,9 +18,9 @@ end function CFCHTTP.GetOptionsForURL( url ) if not url then return CFCHTTP.config.defaultOptions end - if CFCHTTP.isAssetURI( url ) then return CFCHTTP.config.defaultAssetURIOptions end + if CFCHTTP.IsAssetURI( url ) then return CFCHTTP.config.defaultAssetURIOptions end - local address = CFCHTTP.getAddress( url ) + local address = CFCHTTP.GetAddress( url ) if not address then return CFCHTTP.config.defaultOptions end local options = CFCHTTP.config.addresses[address] diff --git a/lua/cfc_http_restrictions/shared/url.lua b/lua/cfc_http_restrictions/shared/url.lua new file mode 100644 index 0000000..3191f68 --- /dev/null +++ b/lua/cfc_http_restrictions/shared/url.lua @@ -0,0 +1,53 @@ +---@class URLData +---@field protocol string +---@field address string +---@field port number +---@field path string + + +CFCHTTP.URLPattern = "(%a+)://([^:/ \t]+):?(%d*)/?.*" +CFCHTTP.URLPatternNoGroups = "%a+://[^:/ \t\"]+:?%d*/?[^\n\" ]*" + +---@param url string +---@return URLData +function CFCHTTP.ParseURL( url ) + local pattern = CFCHTTP.URLPattern + local _, _, protocol, address, port, remainder = string.find( url, pattern ) + return { + protocol = protocol, + address = address, + port = tonumber( port ), + path = remainder + } +end + +---@param text string +---@return string[] +function CFCHTTP.FindURLs( text ) + local pattern = CFCHTTP.URLPatternNoGroups + + local urls = {} + for url in string.gmatch( text, pattern ) do + table.insert( urls, url ) + end + + return urls +end + +local parsedAddressCache = {} +---@parm url string +---@return string|nil +function CFCHTTP.GetAddress( url ) + if not url then return end + local cached = parsedAddressCache[url] + if cached then return cached end + + local data = CFCHTTP.ParseURL( url ) + parsedAddressCache[url] = data.address + + return data.address +end + +function CFCHTTP.IsAssetURI( url ) + return string.StartWith( url, "asset://" ) +end diff --git a/lua/tests/cfc_cl_http_whitelist/url.lua b/lua/tests/cfc_cl_http_whitelist/url.lua new file mode 100644 index 0000000..59f1067 --- /dev/null +++ b/lua/tests/cfc_cl_http_whitelist/url.lua @@ -0,0 +1,54 @@ +local testUrls = { + { url = "http://лиса.рф/static/img/test.png", address = "лиса.рф" }, + { url = "https://store.fox.pics/0d875a97-2ab3-489c-b7db-d9d9f026504e.jpg", address = "store.fox.pics" }, + { url = "https://fox.pics:8080/0d875a97-2ab3-489c-b7db-d9d9f026504e.jpg", address = "fox.pics" }, + { url = "https://cfcservers.org", address = "cfcservers.org" }, + { url = "https://24.321.483.222", address = "24.321.483.222" }, + { url = "nil", address = nil }, + { url = nil, address = nil }, +} + +local htmlBlobs = [[ + + + Test + + + + + +]] + +return { + groupName = "CFC HTTP Whitelist Domains", + cases = { + { + timeout = 3, + async = false, + name = "Should get addresses from urls", + func = function() + for _, urlData in pairs( testUrls ) do + local html = htmlBlobs:format( urlData.url ) + local urls = CFCHTTP.FileTypes.HTML.GetURLSFromData( html ) + if urlData.address then + expect( #urls ).to.equal( 1 ) + expect( urls[1] ).to.equal( urlData.url ) + else + expect( #urls ).to.equal( 0 ) + end + end + end + }, + { + timeout = 3, + async = false, + name = "Get address should return expected data", + func = function() + for _, urlData in pairs( testUrls ) do + local address = CFCHTTP.GetAddress( urlData.url ) + expect( address ).to.equal( urlData.address ) + end + end + }, + } +}