Networked config (#50)

* create config file functions to load networked configs

* fix http stack

* split config into client server and shared, allow receiving config
networked from the server

* formatting

* Update README

* fix typo

* update title

* add known issues

* whitespace changes

* update toc

* Update README

* update README

README.md

@@ -1,30 +1,59 @@
-# cfc_cl_http_whitelist
-Attempts to block any http request on clients that are not in the defined whitelist
+# CFC HTTP Whitelist
+Blocks http requests and references in HTML to addresses not in the allowed list of addresses.
 
+* [Configuring](#configuring)
+  * [How it works](#how-it-works)
+  * [Configuring the addon](#configuring-the-addon)
+  * [Client convars](#clientside-convars)
+* [Integrating with your addon](#integrating-with-your-addon)
+* [Known Issues](#known-issues)
 
-## Configuring 
-You can create a file in `lua/cfc_http_restrictions/configs` to add your own default domains to your server.
-e.g. `lua/cfc_http_restrictions/config/myserver_config.lua`. See [CFC's Whitelist config addon](https://github.com/CFC-Servers/cfc_cl_http_whitelist_configs) for an exmaple
-
-Configuration is loaded from lua files and a data file on in the clients data folder
-Each config thats loaded will overwrite values in the previous config, unless permanent=true is set on that config option
+# Configuring 
+## How it works
+Clients are allowed to allow or deny urls for their clientside whitelist using the options menu in sandbox. Before loading the clientside editable config defaults are loaded from lua files or networked from the server.
+Configs are layered so one config doesnt not completely overwrite an earlier config, just adds to it and overwrites any conflicting addresses (unless they are set to permanent)
 
 Configuration load order on client
-- lua/cfc_http_retrictions/default_config.lua
-- lua/cfc_http_restrictions/configs/*.lua
-- data/cfc_cl_http_whitelist_config.json
+- Loads default config from `lua/cfc_http_retrictions/default_config.lua`
+- Loads additional configs from `lua/cfc_http_restrictions/configs/*.lua`
+- Loads additional configs from `lua/cfc_http_restrictions/configs/client/*.lua`
+- Loads networked config if it exists
+- Loads clientside config from `data/cfc_cl_http_whitelist_config.json`
 
-#### Configuration options
+## Configuring the addon
+The best way to configure this addon is using lua files.   
+*do NOT edit default_config.lua or any other file in this addon to change the config*
+
+- Fork or copy the template repo found here https://github.com/cfc-servers/cfc_http_whitelist_config_template
+- Modify the repo to allow or deny any domains you need. See [Configuration Options](##configuration-options) for a list of options you can use in your address config 
+- Put that repo in the addons folder on your server
+
+
+## Configuration options
 | name | type | description |
 | ----- | ---- | --------------------------------------------------------------------------------------------------------------------------------- |
 | noisy | bool | mark the domain as noisy, hiding it from logs, this can be used for internal domains that will be called frequently on the client |
 | allowed|bool| Is the domain allowed, if false block the domain, if true allow the domain |
-|permanent|bool|Is the domain permanent, if true the domain can not be removed by the users own config|
-|_edited|bool|DO NOT SET, internal field used to track if a config option has been edited by the client|
+|permanent|bool|Is the domain permanent, if true the domain can not be removed from the config| 
+|pattern|bool|Should the address be treated as a lua pattern|
 
-#### Clientside Convars
+## Clientside Convars
 | name | default | description |
 | ---- | ------- | ----------- |
 | cfc_http_restrictions_log_allows | 1 | Should log allowed HTTP requests? |
 | cfc_http_restrictions_log_blocks | 1 | Should log blocked HTTP requests |
 | cfc_http_restrictions_log_verbose | 0 | Should the logs include verbose messages? noisy domains and full urls. |
+
+# Known Issues
+- Some filetypes will not work with sound.playURL. This is intentional and will likely not be fixed. these filetypes would allow you to bypass the whitelist.
+Wav, mp3, and any other audio filetype should work. If a filetype that should be allowed is being blocked please create a github issue with a link to the file.
+- Radio streams will not work. This may be fixed, this is an unfortunate side effect of checking the file content of audio files before playing with sound.PlayURL.
+
+# Integrating with your addon
+Anything not listed here has no guarantee of backwards compatability
+## Functions
+- `CFCHTTP.GetOptionsForURL(url)` gets the config options for a url to check if its allowed
+- `CFCHTTP.LogRequest( tbl )` logs a request in the console
+## Variables
+`CFCHTTP.BASS_ERROR_BLOCKED_URI = 11001` custom error code returned by sound.PlayURL
+`CFCHTTP.BASS_ERROR_BLOCKED_CONTENT = 11002` custom error code returned by sound.PlayURL

lua/autorun/cfc_http_whitelist_init.lua

@@ -10,11 +10,25 @@ local function includeClient( f )
     end
 end
 
+local function includeServer( f )
+    if SERVER then
+        include( f )
+    end
+end
+
 local function includeShared( f )
     AddCSLuaFile( f )
     include( f )
 end
 
+local function runOnDir( dir, action )
+    local files = file.Find( dir .. "/*.lua", "LUA" )
+    for _, filename in pairs( files ) do
+        action( dir .. "/" .. filename )
+    end
+end
+
+includeShared( "cfc_http_restrictions/shared/netmiddleware.lua" )
 includeShared( "cfc_http_restrictions/shared/logging.lua" )
 includeShared( "cfc_http_restrictions/shared/config_loader.lua" )
 includeShared( "cfc_http_restrictions/shared/filetypes.lua" )
@@ -23,11 +37,8 @@ includeShared( "cfc_http_restrictions/shared/url.lua" )
 
 includeClient( "cfc_http_restrictions/client/list_view.lua" )
 includeClient( "cfc_http_restrictions/client/wrap_functions.lua" )
-
-
-AddCSLuaFile( "cfc_http_restrictions/wraps/http.lua" )
-AddCSLuaFile( "cfc_http_restrictions/wraps/html.lua" )
-AddCSLuaFile( "cfc_http_restrictions/wraps/playURL.lua" )
-
+includeClient( "cfc_http_restrictions/client/config_loader.lua" )
 includeClient( "cfc_http_restrictions/client/integrations.lua")
 
+runOnDir( "cfc_http_restrictions/wraps", AddCSLuaFile )
+includeServer( "cfc_http_restrictions/server/config_loader.lua" )

lua/cfc_http_restrictions/client/config_loader.lua

@@ -0,0 +1,25 @@
+local function requestNetworkedConfig()
+    net.Start( "CFCHTTP_RequestConfig" )
+    net.SendToServer()
+end
+
+local function loadConfigsClient()
+    CFCHTTP.LoadConfig( {
+        CFCHTTP.LuaDirectorySources( CFCHTTP.filenames.sharedConfigsDir ),
+        CFCHTTP.LuaDirectorySources( CFCHTTP.filenames.clientConfigsDir ),
+        CFCHTTP.LuaTableSources( CFCHTTP.networkedConfig ),
+        CFCHTTP.FileSource( CFCHTTP.filenames.defaultJsonConfig ),
+    } )
+end
+
+net.Receive( "CFCHTTP_ConfigUpdate", function()
+    local l = net.ReadDouble()
+    local config = util.JSONToTable( util.Decompress( net.ReadData( l ) ) )
+    CFCHTTP.networkedConfig = config
+
+    loadConfigsClient()
+    CFCHTTP.repopulateListPanel()
+end )
+
+loadConfigsClient()
+hook.Add("InitPostEntity", "CFCHTTP_Init_Config", requestNetworkedConfig)

lua/cfc_http_restrictions/client/list_view.lua

@@ -9,6 +9,17 @@ local function removeByValue( listView, value )
     end
 end
 
+function CFCHTTP.repopulateListPanel()
+    local list = CFCHTTP.listPanel
+    if not IsValid( list ) then return end
+
+    list:Clear()
+
+    for k, v in pairs( CFCHTTP.config.addresses ) do
+        list:AddLine( k, (v and v.allowed) and "yes" or "no" )
+    end
+end
+
 local function populatePanel( form )
     local warning = vgui.Create( "DLabel" )
     warning:SetText( "Adding a domain here could expose your ip to other players (and other vulnerabilities)" )
@@ -30,6 +41,7 @@ local function populatePanel( form )
     list:AddColumn( "Allowed" )
     list:SetTall( 300 )
     form:AddItem( list )
+    CFCHTTP.listPanel = list
 
     for k, v in pairs( CFCHTTP.config.addresses ) do
         list:AddLine( k, (v and v.allowed) and "yes" or "no" )
@@ -37,6 +49,7 @@ local function populatePanel( form )
 
     local textEntry, _ = form:TextEntry( "Address" )
 
+    ---@diagnostic disable-next-line: inject-field
     list.OnRowSelected = function( _, _, pnl )
         textEntry:SetValue( pnl:GetColumnText( 1 ) )
     end
@@ -75,7 +88,7 @@ local function populatePanel( form )
                 conf.addresses[addr] = nil
             end
         end
-        CFCHTTP.SaveFileConfig( {
+        CFCHTTP.SaveFileConfig( "cfc_cl_http_whitelist_config.json", {
             version = "1",
             addresses = conf.addresses
         } )

lua/cfc_http_restrictions/default_config.lua

@@ -1,6 +1,6 @@
 AddCSLuaFile()
 
----@alias WhitelistAddressOption { allowed: boolean|nil, noisy: boolean|nil, permanent: boolean|nil, pattern: boolean|nil }
+---@alias WhitelistAddressOption { allowed: boolean|nil, noisy: boolean|nil, permanent: boolean|nil, pattern: boolean|nil, _edited: boolean|nil }
 
 ---@class WhitelistConfig
 ---@field version string

lua/cfc_http_restrictions/server/config_loader.lua

@@ -0,0 +1,38 @@
+util.AddNetworkString( "CFCHTTP_ConfigUpdate" )
+util.AddNetworkString( "CFCHTTP_RequestConfig" )
+
+local function sendClientConfig( ply )
+    local data = util.Compress( util.TableToJSON( CFCHTTP.networkedClientConfig ) )
+    net.Start( "CFCHTTP_ConfigUpdate" )
+    net.WriteDouble( #data )
+    net.WriteData( data, #data )
+    net.Send( ply )
+end
+
+CFCHTTP.Net.receiveWithMiddleware( "CFCHTTP_RequestConfig", function( _, ply )
+    sendClientConfig( ply )
+end, CFCHTTP.Net.rateLimit( "CFCHTTP_RequestConfig", 2, 10 ) )
+
+local function loadConfigsServer()
+    CFCHTTP.LoadConfig( {
+        CFCHTTP.LuaDirectorySources( CFCHTTP.filenames.sharedConfigsDir ),
+        CFCHTTP.LuaDirectorySources( CFCHTTP.filenames.serverConfigsDir ),
+        CFCHTTP.FileSource( CFCHTTP.filenames.defaultJsonConfig ),
+    } )
+
+    local data = file.Read( CFCHTTP.filenames.serverClientJsonConfig, "DATA" )
+    CFCHTTP.networkedClientConfig = data and util.JSONToTable( data ) or {}
+end
+
+local function addCSLuaConfigs( dir )
+    local files = file.Find( dir .. "*.lua", "LUA" )
+    for _, fil in pairs( files ) do
+        AddCSLuaFile( dir .. fil )
+    end
+end
+
+
+AddCSLuaFile( CFCHTTP.filenames.defaultLuaConfig )
+addCSLuaConfigs( CFCHTTP.filenames.sharedConfigsDir )
+addCSLuaConfigs( CFCHTTP.filenames.clientConfigsDir )
+loadConfigsServer()

lua/cfc_http_restrictions/shared/config_loader.lua

@@ -1,52 +1,58 @@
----@package
-function CFCHTTP.loadClientFileConfig()
-    local fileConfig = CFCHTTP.ReadFileConfig()
-    if fileConfig then
-        CFCHTTP.config = CFCHTTP.mergeConfigs( CFCHTTP.config, fileConfig )
-    end
-end
+CFCHTTP.filenames = {
+    defaultLuaConfig = "cfc_http_restrictions/default_config.lua",
+    defaultJsonConfig = "cfc_cl_http_whitelist_config.json",
+    serverClientJsonConfig = "cfchttp_client_config.json",
+    sharedConfigsDir = "cfc_http_restrictions/configs/",
+    serverConfigsDir = "cfc_http_restrictions/configs/server/",
+    clientConfigsDir = "cfc_http_restrictions/configs/client/",
+}
 
----@package
----@param dir string|nil
-function CFCHTTP.loadLuaConfigs( dir )
-    dir = dir or "cfc_http_restrictions/configs/"
-    local files = file.Find( dir .. "*.lua", "LUA" )
-    for _, fil in pairs( files ) do
-        local newConfig = include( dir .. fil )
-        if newConfig then
-            CFCHTTP.config = CFCHTTP.mergeConfigs( CFCHTTP.config, newConfig )
+---@param sources (fun(): WhitelistConfig)[]
+function CFCHTTP.LoadConfig( sources )
+    CFCHTTP.config = include( CFCHTTP.filenames.defaultLuaConfig )
+    for _, source in pairs( sources ) do
+        local config = source()
+        if config then
+            CFCHTTP.config = CFCHTTP.mergeConfigs( CFCHTTP.config, config )
         end
     end
 end
 
----@package
----@param dir string|nil
-function CFCHTTP.addCSLuaConfigs( dir )
-    dir = dir or "cfc_http_restrictions/configs/"
-    local files = file.Find( dir .. "*.lua", "LUA" )
-    for _, fil in pairs( files ) do
-        AddCSLuaFile( dir .. fil )
+---@param filename string
+---@return fun(): WhitelistConfig|nil
+function CFCHTTP.FileSource( filename )
+    return function()
+        return CFCHTTP.ReadFileConfig( filename )
     end
 end
 
----@package
----@param configFile string|nil
-function CFCHTTP.loadDefaultConfg( configFile )
-    configFile = configFile or "cfc_http_restrictions/default_config.lua"
-    CFCHTTP.config = include( configFile )
+---@param filename string
+---@return fun(): WhitelistConfig
+function CFCHTTP.LuaFileSource( filename )
+    return function()
+        return include( filename )
+    end
 end
 
-function CFCHTTP.LoadConfigsClient()
-    CFCHTTP.loadDefaultConfg()
-    CFCHTTP.loadLuaConfigs()
-    CFCHTTP.loadLuaConfigs( "cfc_http_restrictions/configs/client/" )
-    CFCHTTP.loadClientFileConfig()
+---@param dir string
+---@return fun(): WhitelistConfig ...
+function CFCHTTP.LuaDirectorySources( dir )
+    local funcs = {}
+    local files = file.Find( dir .. "*.lua", "LUA" )
+
+    for _, fil in pairs( files ) do
+        table.insert( funcs, CFCHTTP.LuaFileSource( dir .. fil ) )
+    end
+
+    return unpack( funcs )
 end
 
-function CFCHTTP.LoadConfigsServer()
-    CFCHTTP.loadDefaultConfg()
-    CFCHTTP.loadLuaConfigs()
-    CFCHTTP.loadLuaConfigs( "cfc_http_restrictions/configs/server/" )
+---@param tbl WhitelistConfig
+---@return fun(): WhitelistConfig
+function CFCHTTP.LuaTableSources( tbl )
+    return function()
+        return tbl
+    end
 end
 
 ---@param old any
@@ -79,26 +85,21 @@ function CFCHTTP.CopyConfig( cfg )
     return util.JSONToTable( util.TableToJSON( cfg ) )
 end
 
+---@param filename string
 ---@param config WhitelistConfig
-function CFCHTTP.SaveFileConfig( config )
-    file.Write( "cfc_cl_http_whitelist_config.json", util.TableToJSON( config, true ) )
+function CFCHTTP.SaveFileConfig( filename, config )
+    file.Write( filename, util.TableToJSON( config, true ) )
 
-    notification.AddLegacy( "Saved http whitelist", NOTIFY_GENERIC, 5 )
+    if CLIENT then
+        notification.AddLegacy( "Saved http whitelist", NOTIFY_GENERIC, 5 )
+    end
 end
 
+---@param filename string
 ---@return WhitelistConfig|nil
-function CFCHTTP.ReadFileConfig()
-    local fileData = file.Read( "cfc_cl_http_whitelist_config.json" )
+function CFCHTTP.ReadFileConfig( filename )
+    local fileData = file.Read( filename )
     if not fileData then return nil end
 
     return util.JSONToTable( fileData )
 end
-
-if CLIENT then
-    CFCHTTP.LoadConfigsClient()
-else
-    CFCHTTP.addCSLuaConfigs()
-    CFCHTTP.addCSLuaConfigs( "cfc_http_restrictions/configs/client/" )
-
-    CFCHTTP.LoadConfigsServer()
-end

lua/cfc_http_restrictions/shared/netmiddleware.lua

@@ -0,0 +1,79 @@
+CFCHTTP.Net = CFCHTTP.Net or {}
+
+function CFCHTTP.newRateLimitBucket( capacity, rate )
+    local bucket = {
+        capacity = capacity,
+        rate = rate,
+        tokens = capacity,
+        last = SysTime()
+    }
+
+    function bucket:consume( amount )
+        local now = SysTime()
+        local delta = now - self.last
+
+        self.last = now
+        self.tokens = math.min( self.tokens + delta * self.rate, self.capacity )
+
+        if self.tokens < amount then
+            return false
+        end
+
+        self.tokens = self.tokens - amount
+        return true
+    end
+
+    return bucket
+end
+
+function CFCHTTP.Net.receiveWithMiddleware( name, cb, ... )
+    for _, v in pairs( { ... } ) do
+        cb = v( cb )
+    end
+
+    net.Receive( name, cb )
+end
+
+CFCHTTP.Net._rateLimitBuckets = CFCHTTP.Net._rateLimitBuckets or {}
+
+function CFCHTTP.Net.rateLimit( name, capacity, rate )
+    return function( cb )
+        return function( n, ply )
+            if not IsValid( ply ) then return end
+
+            local identifier = ply:SteamID() .. name
+            local bucket = CFCHTTP.Net._rateLimitBuckets[identifier]
+            if not bucket then
+                bucket = CFCHTTP.newRateLimitBucket( capacity, rate )
+                CFCHTTP.Net._rateLimitBuckets[identifier] = bucket
+                ply:CallOnRemove( identifier .. "_cleanup", function()
+                    CFCHTTP.Net._rateLimitBuckets[identifier] = nil
+                end )
+            end
+
+            if not bucket:consume( 1 ) then
+                -- print( "CFCHTTP: Rate limit exceeded for " .. name .. " from " .. ply:Nick() )
+                -- TODO add system to print rate limit violaters while not allowing console spamming
+                return
+            end
+            cb( n, ply )
+        end
+    end
+end
+
+function CFCHTTP.Net.requirePermission( perm )
+    return function( cb )
+        return function( n, ply )
+            if not CAMI then
+                if not ply:IsSuperAdmin() then return end
+                return cb( n, ply )
+            end
+
+            CAMI.PlayerHasAccess( ply, perm, function( b )
+                if not b then return end
+                cb( n, ply )
+            end )
+        end
+    end
+end
+

lua/cfc_http_restrictions/wraps/http.lua

@@ -1,3 +1,16 @@
+
+local function getSourceFromStack(stack)
+   local s = stack[3]
+   for i = 4, 5 do
+      if not stack[i] then break end
+      s = stack[i]
+
+      if not string.EndsWith(s, "/http.lua") then break end
+   end
+
+   return s
+end
+
 local function wrapHTTP()
     _HTTP = _HTTP or HTTP
     print( "HTTP wrapped, original function at '_G._HTTP'" )
@@ -11,7 +24,7 @@ local function wrapHTTP()
         CFCHTTP.LogRequest( {
             noisy = noisy,
             method = req.method,
-            fileLocation = stack[3],
+            fileLocation = getSourceFromStack( stack ),
             urls = { { url = req.url, status = isAllowed and "allowed" or "blocked" } },
         } )