Handle IDN domains and add tests (#27)

* add tests and move files to shared

* use CFCHTTP.ParseURL in getAddress

* change url in tests

* add tests to github actions

* rename test file and update function calls to moved functions

* minor config loader refactor

* whitespace changes

* rename file -> configFile

* add invalid date test cases for address parsing functions

.github/workflows/gluatest.yml

@@ -0,0 +1,8 @@
+name: GLuaTest Runner
+
+on:
+  pull_request:
+
+jobs:
+  run-tests:
+    uses: CFC-Servers/GLuaTest/.github/workflows/gluatest.yml@main

lua/.luarc.json

@@ -0,0 +1,5 @@
+{
+    "diagnostics.globals": [
+        "expect"
+    ]
+}

lua/autorun/cfc_http_whitelist_init.lua

@@ -1,5 +1,7 @@
 AddCSLuaFile()
 
+CFCHTTP = CFCHTTP or {}
+
 local function includeClient( f )
     if SERVER then
         AddCSLuaFile( f )
@@ -12,9 +14,11 @@ local function includeShared( f )
     AddCSLuaFile( f )
     include( f )
 end
-include( "cfc_http_restrictions/config_loader.lua" )
-includeShared( "cfc_http_restrictions/client/filetypes.lua" )
-includeClient( "cfc_http_restrictions/config_loader.lua" )
-includeClient( "cfc_http_restrictions/client/list_manager.lua" )
+
+includeShared( "cfc_http_restrictions/shared/config_loader.lua" )
+includeShared( "cfc_http_restrictions/shared/filetypes.lua" )
+includeShared( "cfc_http_restrictions/shared/list_manager.lua" )
+includeShared( "cfc_http_restrictions/shared/url.lua" )
+
 includeClient( "cfc_http_restrictions/client/list_view.lua" )
 includeClient( "cfc_http_restrictions/client/wrap_functions.lua" )

lua/cfc_http_restrictions/client/wrap_functions.lua

@@ -20,7 +20,7 @@ local function logRequest( method, url, fileLocation, allowed, noisy )
     if not url then
         url = "unknown"
     elseif isVerbose == false then
-        local address = CFCHTTP.getAddress( url )
+        local address = CFCHTTP.GetAddress( url )
         if noisy then return end
 
         url = address
@@ -182,7 +182,7 @@ local function wrapHTMLPanel( panelName )
         logRequest( "GET", options.combinedUri, stack[3], isAllowed )
 
         if not isAllowed then
-            html = [[<h1> BLOCKED </h1>]]
+            html = [[<h1>BLOCKED By CFC HTTP Whitelist</h1>]]
         end
 
         return _G[setHTML]( self, html, ... )

lua/cfc_http_restrictions/config_loader.lua

@@ -1,64 +0,0 @@
-CFCHTTP = CFCHTTP or {}
-CFCHTTP.config = include( "default_config.lua" )
-
-function CFCHTTP.LoadConfigs()
-    CFCHTTP.config = include( "default_config.lua" )
-    CFCHTTP.loadLuaConfigs()
-
-    if CLIENT then
-        local fileConfig = CFCHTTP.readFileConfig()
-        if fileConfig then
-            CFCHTTP.config = CFCHTTP.mergeConfigs( CFCHTTP.config, fileConfig )
-        end
-    end
-end
-
--- LoadLuaConfigs loads the default config and then any lua files in the cfc_http_restrictions/configs directory
-function CFCHTTP.loadLuaConfigs()
-    local files = file.Find( "cfc_http_restrictions/configs/*.lua", "LUA" )
-    for _, fil in pairs( files ) do
-        AddCSLuaFile( "cfc_http_restrictions/configs/" .. fil )
-        local newConfig = include( "cfc_http_restrictions/configs/" .. fil )
-        CFCHTTP.config = CFCHTTP.mergeConfigs( CFCHTTP.config, newConfig )
-    end
-end
-
-function CFCHTTP.mergeConfigs( old, new )
-    if new.version == "1" then
-        if new.wrapHTMLPanels ~= nil then old.wrapHTMLPanels = new.wrapHTMLPanels end
-        if new.defaultOptions ~= nil then old.defaultOptions = new.defaultOptions end
-        if new.defaultAssetURIOption ~= nil then old.defaultAssetURIOption = new.defaultAssetURIOption end
-
-        for domain, options in pairs( new.addresses ) do
-            local currentOptions = old.addresses[domain]
-            if currentOptions and currentOptions.permanent then
-                print( "[CFC HTTP Restrictions] Skipping " .. domain .. " because it is permanent" )
-            else
-                old.addresses[domain] = options
-            end
-        end
-    else
-        ErrorNoHalt( "[CFC HTTP Restrictions] Invalid config version: " .. tostring( new.version ) )
-    end
-
-    return old
-end
-
-function CFCHTTP.copyConfig( cfg )
-    return util.JSONToTable( util.TableToJSON( cfg ) )
-end
-
-function CFCHTTP.saveFileConfig( config )
-    file.Write( "cfc_cl_http_whitelist_config.json", util.TableToJSON( config, true ) )
-
-    notification.AddLegacy( "Saved http whitelist", NOTIFY_GENERIC, 5 )
-end
-
-function CFCHTTP.readFileConfig()
-    local fileData = file.Read( "cfc_cl_http_whitelist_config.json" )
-    if not fileData then return end
-
-    return util.JSONToTable( fileData )
-end
-
-CFCHTTP.LoadConfigs()

lua/cfc_http_restrictions/default_config.lua

@@ -1,9 +1,17 @@
 AddCSLuaFile()
 
-return {
+---@alias WhitelistAddressOption { allowed: boolean|nil, noisy: boolean|nil, permanent: boolean|nil }
+
+---@class WhitelistConfig
+---@field version string
+---@field wrapHTMLPanels boolean|nil
+---@field defaultAssetURIOptions WhitelistAddressOption
+---@field defaultOptions WhitelistAddressOption
+---@field addresses table<string, WhitelistAddressOption>
+local config = {
     version = "1", -- this field allows backwards compatibility if the config structure is ever updated
 
-    wrapHTMLPanels = false,
+    wrapHTMLPanels = true,
 
     defaultAssetURIOptions = {
         allowed = true
@@ -83,6 +91,7 @@ return {
 
         ["(%w+)%.keybase.pub"] = { allowed = true, pattern = true },
         ["tts.cyzon.us"] = { allowed = true },
-
     }
 }
+
+return config

lua/cfc_http_restrictions/shared/config_loader.lua

@@ -0,0 +1,95 @@
+---@package
+function CFCHTTP.loadClientFileConfig()
+    local fileConfig = CFCHTTP.ReadFileConfig()
+    if fileConfig then
+        CFCHTTP.config = CFCHTTP.mergeConfigs( CFCHTTP.config, fileConfig )
+    end
+end
+
+---@package
+---@param dir string|nil
+function CFCHTTP.loadLuaConfigs( dir )
+    dir = dir or "cfc_http_restrictions/configs/"
+    local files = file.Find( dir .. "*.lua", "LUA" )
+    for _, fil in pairs( files ) do
+        local newConfig = include( dir .. fil )
+        CFCHTTP.config = CFCHTTP.mergeConfigs( CFCHTTP.config, newConfig )
+    end
+end
+
+---@package
+---@param dir string|nil
+function CFCHTTP.addCSLuaConfigs( dir )
+    dir = dir or "cfc_http_restrictions/configs/"
+    local files = file.Find( dir .. "*.lua", "LUA" )
+    for _, fil in pairs( files ) do
+        AddCSLuaFile( dir .. fil )
+    end
+end
+
+---@package
+---@param configFile string|nil
+function CFCHTTP.loadDefaultConfg( configFile )
+    configFile = configFile or "cfc_http_restrictions/default_config.lua"
+    CFCHTTP.config = include( configFile )
+end
+
+function CFCHTTP.LoadConfigsClient()
+    CFCHTTP.loadDefaultConfg()
+    CFCHTTP.loadLuaConfigs()
+    CFCHTTP.loadLuaConfigs( "cfc_http_restrictions/configs/client/" )
+    CFCHTTP.loadClientFileConfig()
+end
+
+function CFCHTTP.LoadConfigsServer()
+    CFCHTTP.loadDefaultConfg()
+    CFCHTTP.loadLuaConfigs()
+    CFCHTTP.loadLuaConfigs( "cfc_http_restrictions/configs/server/" )
+end
+
+function CFCHTTP.mergeConfigs( old, new )
+    if new.version == "1" then
+        if new.wrapHTMLPanels ~= nil then old.wrapHTMLPanels = new.wrapHTMLPanels end
+        if new.defaultOptions ~= nil then old.defaultOptions = new.defaultOptions end
+        if new.defaultAssetURIOption ~= nil then old.defaultAssetURIOption = new.defaultAssetURIOption end
+
+        for domain, options in pairs( new.addresses ) do
+            local currentOptions = old.addresses[domain]
+            if currentOptions and currentOptions.permanent then
+                print( "[CFC HTTP Restrictions] Skipping " .. domain .. " because it is permanent" )
+            else
+                old.addresses[domain] = options
+            end
+        end
+    else
+        ErrorNoHalt( "[CFC HTTP Restrictions] Invalid config version: " .. tostring( new.version ) )
+    end
+
+    return old
+end
+
+function CFCHTTP.CopyConfig( cfg )
+    return util.JSONToTable( util.TableToJSON( cfg ) )
+end
+
+function CFCHTTP.SaveFileConfig( config )
+    file.Write( "cfc_cl_http_whitelist_config.json", util.TableToJSON( config, true ) )
+
+    notification.AddLegacy( "Saved http whitelist", NOTIFY_GENERIC, 5 )
+end
+
+function CFCHTTP.ReadFileConfig()
+    local fileData = file.Read( "cfc_cl_http_whitelist_config.json" )
+    if not fileData then return end
+
+    return util.JSONToTable( fileData )
+end
+
+if CLIENT then
+    CFCHTTP.LoadConfigsClient()
+else
+    CFCHTTP.addCSLuaConfigs()
+    CFCHTTP.addCSLuaConfigs( "cfc_http_restrictions/configs/client/" )
+
+    CFCHTTP.LoadConfigsServer()
+end

lua/cfc_http_restrictions/shared/filetypes.lua

@@ -1,9 +1,9 @@
 CFCHTTP.FileTypes = CFCHTTP.FIleTypes or {}
 
-local files, _ = file.Find( "cfc_http_restrictions/client/filetypes/*.lua", "LUA" )
+local files, _ = file.Find( "cfc_http_restrictions/shared/filetypes/*.lua", "LUA" )
 for _, f in pairs( files ) do
-    include( "cfc_http_restrictions/client/filetypes/" .. f )
-    AddCSLuaFile( "cfc_http_restrictions/client/filetypes/" .. f )
+    include( "cfc_http_restrictions/shared/filetypes/" .. f )
+    AddCSLuaFile( "cfc_http_restrictions/shared/filetypes/" .. f )
 end
 
 ---@param data string

lua/cfc_http_restrictions/shared/filetypes/html.lua

@@ -5,10 +5,10 @@ local HTML = {
 }
 CFCHTTP.FileTypes.HTML = HTML
 
--- Not implemented
----@param body string
+---@param _body string
 ---@return boolean
-function HTML.IsFileData( body )
+---@diagnostic disable-next-line: unused-local
+function HTML.IsFileData( _body )
     return false
 end
 
@@ -19,21 +19,10 @@ function HTML.IsFileURL( url )
     return false
 end
 
-local function getUrlsFromText( text )
-    local pattern = "%a+://[%a%d%.-]+:?%d*/?[a-zA-Z0-9%.]*"
-
-    local urls = {}
-    for url in string.gmatch( text, pattern ) do
-        table.insert( urls, url )
-    end
-
-    return urls
-end
-
 ---@param body string
 ---@return string[] urls
 ---@return string|nil error
 function HTML.GetURLSFromData( body )
-    local urls = getUrlsFromText( body )
+    local urls = CFCHTTP.FindURLs( body )
     return urls, nil
 end

lua/cfc_http_restrictions/shared/filetypes/pls.lua

@@ -50,19 +50,6 @@ function PLS.IsFileURL( url )
     return false
 end
 
----@param text string
----@return string[] urls
-local function getUrlsFromText( text )
-    local pattern = "%a+://[%a%d%.-]+:?%d*/?[a-zA-Z0-9%.]*"
-
-    local urls = {}
-    for url in string.gmatch( text, pattern ) do
-        table.insert( urls, url )
-    end
-
-    return urls
-end
-
 ---@param body string
 ---@return string[] urls
 ---@return string|nil error
@@ -70,7 +57,7 @@ function PLS.GetURLSFromData( body )
     if not PLS.allowed then return {}, "pls files are not allowed" end
 
     if #body > PLS.maxFileSize then return {}, "body too large" end
-    local urls = getUrlsFromText( body )
+    local urls = CFCHTTP.FindURLs( body )
 
     local plsData = loadPLSFile( body )
     if not plsData.playlist then

lua/cfc_http_restrictions/shared/list_manager.lua

@@ -1,24 +1,5 @@
 CFCHTTP = CFCHTTP or {}
 
-
-local parsedAddressCache = {}
----@parm url string
----@return string
-function CFCHTTP.getAddress( url )
-    local cached = parsedAddressCache[url]
-    if cached then return cached end
-
-    local pattern = "(%a+)://([%a%d%.-]+):?(%d*)/?.*"
-    local _, _, _, addr, _ = string.find( url, pattern )
-    parsedAddressCache[url] = addr
-
-    return addr
-end
-
-function CFCHTTP.isAssetURI( url )
-    return string.StartWith( url, "asset://" )
-end
-
 -- escapes all lua pattern characters and allows the use of * as a wildcard
 local escapedCache = {}
 local function escapeAddr( addr )
@@ -37,9 +18,9 @@ end
 function CFCHTTP.GetOptionsForURL( url )
     if not url then return CFCHTTP.config.defaultOptions end
 
-    if CFCHTTP.isAssetURI( url ) then return CFCHTTP.config.defaultAssetURIOptions end
+    if CFCHTTP.IsAssetURI( url ) then return CFCHTTP.config.defaultAssetURIOptions end
 
-    local address = CFCHTTP.getAddress( url )
+    local address = CFCHTTP.GetAddress( url )
     if not address then return CFCHTTP.config.defaultOptions end
 
     local options = CFCHTTP.config.addresses[address]

lua/cfc_http_restrictions/shared/url.lua

@@ -0,0 +1,53 @@
+---@class URLData
+---@field protocol string
+---@field address string
+---@field port number
+---@field path string
+
+
+CFCHTTP.URLPattern = "(%a+)://([^:/ \t]+):?(%d*)/?.*"
+CFCHTTP.URLPatternNoGroups = "%a+://[^:/ \t\"]+:?%d*/?[^\n\" ]*"
+
+---@param url string
+---@return URLData
+function CFCHTTP.ParseURL( url )
+    local pattern = CFCHTTP.URLPattern
+    local _, _, protocol, address, port, remainder = string.find( url, pattern )
+    return {
+        protocol = protocol,
+        address = address,
+        port = tonumber( port ),
+        path = remainder
+    }
+end
+
+---@param text string
+---@return string[]
+function CFCHTTP.FindURLs( text )
+    local pattern = CFCHTTP.URLPatternNoGroups
+
+    local urls = {}
+    for url in string.gmatch( text, pattern ) do
+        table.insert( urls, url )
+    end
+
+    return urls
+end
+
+local parsedAddressCache = {}
+---@parm url string
+---@return string|nil
+function CFCHTTP.GetAddress( url )
+    if not url then return end
+    local cached = parsedAddressCache[url]
+    if cached then return cached end
+
+    local data = CFCHTTP.ParseURL( url )
+    parsedAddressCache[url] = data.address
+
+    return data.address
+end
+
+function CFCHTTP.IsAssetURI( url )
+    return string.StartWith( url, "asset://" )
+end

lua/tests/cfc_cl_http_whitelist/url.lua

@@ -0,0 +1,54 @@
+local testUrls = {
+    { url = "http://лиса.рф/static/img/test.png", address = "лиса.рф" },
+    { url = "https://store.fox.pics/0d875a97-2ab3-489c-b7db-d9d9f026504e.jpg", address = "store.fox.pics" },
+    { url = "https://fox.pics:8080/0d875a97-2ab3-489c-b7db-d9d9f026504e.jpg", address = "fox.pics" },
+    { url = "https://cfcservers.org", address = "cfcservers.org" },
+    { url = "https://24.321.483.222", address = "24.321.483.222" },
+    { url = "nil", address = nil },
+    { url = nil, address = nil },
+}
+
+local htmlBlobs = [[
+    <html>
+        <head>
+            <title>Test</title>
+        </head>
+        <body>
+            <img src="%s" />
+        </body>
+    </html>
+]]
+
+return {
+    groupName = "CFC HTTP Whitelist Domains",
+    cases = {
+        {
+            timeout = 3,
+            async = false,
+            name = "Should get addresses from urls",
+            func = function()
+                for _, urlData in pairs( testUrls ) do
+                    local html = htmlBlobs:format( urlData.url )
+                    local urls = CFCHTTP.FileTypes.HTML.GetURLSFromData( html )
+                    if urlData.address then
+                        expect( #urls ).to.equal( 1 )
+                        expect( urls[1] ).to.equal( urlData.url )
+                    else
+                        expect( #urls ).to.equal( 0 )
+                    end
+                end
+            end
+        },
+        {
+            timeout = 3,
+            async = false,
+            name = "Get address should return expected data",
+            func = function()
+                for _, urlData in pairs( testUrls ) do
+                    local address = CFCHTTP.GetAddress( urlData.url )
+                    expect( address ).to.equal( urlData.address )
+                end
+            end
+        },
+    }
+}