merge main into feature/html-panels

This commit is contained in:
Pierce 2022-08-21 14:49:57 -04:00
commit e1b617bb7b
No known key found for this signature in database
GPG Key ID: EC79465B0E865E47
9 changed files with 384 additions and 252 deletions

View File

@ -1 +1,28 @@
# cfc_cl_http_whitelist
# cfc_cl_http_whitelist
## Configuring
You can create a file in `lua/cfc_http_restrictions/configs` to add your own default domains to your server.
e.g. `lua/cfc_http_restrictions/config/myserver_config.lua`. See `lua/cfc_http_retrictions/default_config.lua` for an example.
Configuration is loaded from lua files and a data file on in the clients data folder
Each config thats loaded will overwrite values in the previous config, unless permanent=true is set on that config option
Configuration load order on client
- lua/cfc_http_retrictions/default_config.lua
- lua/cfc_http_restrictions/configs/*.lua
- data/cfc_cl_http_whitelist_config.json
#### Configuration options
| name | type | description |
| ----- | ---- | --------------------------------------------------------------------------------------------------------------------------------- |
| noisy | bool | mark the domain as noisy, hiding it from logs, this can be used for internal domains that will be called frequently on the client |
| allowed|bool| Is the domain allowed, if false block the domain, if true allow the domain |
|permanent|bool|Is the domain permanent, if true the domain can not be removed by the users own config|
|_edited|bool|DO NOT SET, internal field used to track if a config option has been edited by the client|
#### Clientside Convars
| name | default | description |
| ---- | ------- | ----------- |
| cfc_http_restrictions_log_allows | 1 | Should log allowed HTTP requests? |
| cfc_http_restrictions_log_blocks | 1 | Should log blocked HTTP requests |
| cfc_http_restrictions_log_verbose | 0 | Should the logs include verbose messages? noisy domains and full urls. |

View File

@ -8,6 +8,9 @@ local function includeClient( f )
end
end
includeClient( "cfc_http_restrictions/list_manager.lua" )
includeClient( "cfc_http_restrictions/list_view.lua" )
includeClient( "cfc_http_restrictions/wrap_functions.lua" )
include("cfc_http_restrictions/config_loader.lua")
includeClient("cfc_http_restrictions/config_loader.lua")
includeClient( "cfc_http_restrictions/client/list_manager.lua" )
includeClient( "cfc_http_restrictions/client/list_view.lua" )
includeClient( "cfc_http_restrictions/client/wrap_functions.lua" )

View File

@ -0,0 +1,148 @@
CFCHTTP = CFCHTTP or {}
local parsedAddressCache = {}
function CFCHTTP.getAddress( url )
local cached = parsedAddressCache[url]
if cached then return cached end
local pattern = "(%a+)://([%a%d%.-]+):?(%d*)/?.*"
local _, _, protocol, addr, port = string.find( url, pattern )
parsedAddressCache[url] = addr
return addr
end
function CFCHTTP.isAssetURI(url)
return string.StartWith(url, "asset://")
end
-- escapes all lua pattern characters and allows the use of * as a wildcard
local escapedCache = {}
local function escapeAddr( addr )
if escapedCache[addr] then return escapedCache[addr] end
local split = string.Split( addr, "*" )
for i=1, #split do
split[i] = string.PatternSafe( split[i] )
end
escapedCache[addr] = table.concat( split, ".*" )
return escapedCache[addr]
end
function CFCHTTP.getOptionsForURI(url)
if not url then return end
if CFCHTTP.isAssetURI(url) then return CFCHTTP.config.defaultAssetURIConfig end
local address = CFCHTTP.getAddress( url )
if not address then return end
local options = CFCHTTP.config.addresses[address]
if options and not options.pattern then
return options
end
for allowedAddr, options in pairs( CFCHTTP.config.addresses ) do
if not options.pattern then
options = escapeAddr( allowedAddr )
end
if string.match( address, "^"..allowedAddr.."$" ) then
return options
end
end
end
-- TODO reimmplement caching
function CFCHTTP.getOptionsForURI(url)
if not url then return CFCHTTP.config.defaultOptions end
if CFCHTTP.isAssetURI(url) then return CFCHTTP.config.defaultAssetURIOptions end
local address = CFCHTTP.getAddress( url )
if not address then return CFCHTTP.config.defaultOptions end
local options = CFCHTTP.config.addresses[address]
if options and not options.pattern then
return options
end
for allowedAddr, options in pairs( CFCHTTP.config.addresses ) do
if not options.pattern then
options = escapeAddr( allowedAddr )
end
if string.match( address, "^"..allowedAddr.."$" ) then
return options
end
end
return CFCHTTP.config.defaultOptions
end
local function getUrlsInHTML( html )
local pattern = "%a+://[%a%d%.-]+:?%d*/?[a-zA-Z0-9%.]*"
local urls = {}
for url in string.gmatch(html, pattern) do
table.insert(urls, url)
end
return urls
end
function CFCHTTP.isHTMLAllowed( html )
local urls = getUrlsInHTML( html )
for _, url in pairs(urls) do
local options = CFCHTTP.getOptionsForURI(url)
if options and not options.allowed then
return false, url
end
end
return true, ""
end
-- file based config functions
function CFCHTTP.allowAddress( addr )
if CFCHTTP.config.addresses[addr] ~= nil and CFCHTTP.config.addresses[addr].permanent then
notification.AddLegacy( "You cant change this address", NOTIFY_ERROR, 5 )
return false
end
CFCHTTP.config.addresses[addr] = {
_edited=true,
allowed=true,
}
return true
end
function CFCHTTP.blockAddress( addr )
if CFCHTTP.config.addresses[addr] ~= nil and CFCHTTP.config.addresses[addr].permanent then
notification.AddLegacy( "You cant change this address", NOTIFY_ERROR, 5 )
return false
end
CFCHTTP.config.addresses[addr] = {
_edited=true,
allowed=false,
}
return true
end
function CFCHTTP.removeAddress( addr )
if CFCHTTP.adresses[addr] ~= nil and CFCHTTP.config.addresses[addr].permanent then
notification.AddLegacy( "You cant change this address", NOTIFY_ERROR, 5 )
return false
end
CFCHTTP.config.addresses[addr] = nil
return true
end

View File

@ -29,13 +29,12 @@ local function populatePanel( form )
list:SetTall(300)
form:AddItem( list )
for k, v in pairs( CFCHTTP.allowedAddresses ) do
for k, v in pairs( CFCHTTP.config.addresses ) do
list:AddLine( k, v and "yes" or "no" )
end
local textEntry, _ = form:TextEntry( "Address" )
list.OnRowSelected = function( self, index, pnl )
textEntry:SetValue( pnl:GetColumnText( 1 ) )
end
@ -58,6 +57,7 @@ local function populatePanel( form )
list:AddLine( v, "no" )
end
-- TODO when a config is removed we should reload the default values from the lua based configs instead of just removing it from the entire list
local remove = form:Button("Remove")
remove.DoClick = function()
local v = textEntry:GetValue()
@ -67,9 +67,14 @@ local function populatePanel( form )
local save = form:Button("Save")
save.DoClick = function()
CFCHTTP.saveList()
local conf = CFCHTTP.copyConfig(CFCHTTP.config)
for addr, options in pairs(conf.addresses) do
if not options._edited then
conf.addresses[addr] = nil
end
end
CFCHTTP.saveFileConfig(conf)
end
end
hook.Add( "AddToolMenuCategories", "CFC_HTTP_ListManager", function()

View File

@ -1,8 +1,6 @@
local shouldLogAllows = CreateConVar( "cfc_http_restrictions_log_allows", 1, FCVAR_ARCHIVE, "Should the HTTP restrictions log allowed HTTP requests?", 0, 1 )
local shouldLogBlocks = CreateConVar( "cfc_http_restrictions_log_blocks", 1, FCVAR_ARCHIVE, "Should the HTTP restrictions log blocked HTTP requests?", 0, 1 )
local verboseLogging = CreateConVar( "cfc_http_restrictions_log_verbose", 0, FCVAR_ARCHIVE, "Should the HTTP restrictions log include verbose messages?", 0, 1 )
local getAddress = CFCHTTP.getAddress
local noisyDomains = CFCHTTP.noisyDomains
local COLORS = {
RED = Color( 255, 0, 0 ),
@ -11,7 +9,7 @@ local COLORS = {
YELLOW = Color( 235, 226, 52 )
}
local function logRequest( method, url, fileLocation, allowed )
local function logRequest( method, url, fileLocation, allowed, noisy )
if allowed and not shouldLogAllows:GetBool() then return end
if not shouldLogBlocks:GetBool() then return end
@ -20,8 +18,8 @@ local function logRequest( method, url, fileLocation, allowed )
local requestColor = allowed and COLORS.GREEN or COLORS.RED
if isVerbose == false then
local address = getAddress( url )
if noisyDomains[address] then return end
local address = CFCHTTP.getAddress( url )
if noisy then return end
url = address
end
@ -44,9 +42,12 @@ local function wrapHTTP()
print( "HTTP wrapped, original function at '_G._HTTP'" )
HTTP = function( req )
local isAllowed = CFCHTTP.isAllowed( req.url )
local options = CFCHTTP.getOptionsForURI(req.url)
local isAllowed = options and options.allowed
local noisy = options and options.noisy
local stack = string.Split(debug.traceback(), "\n")
logRequest( req.method, req.url, stack[3], isAllowed )
logRequest( req.method, req.url, stack[3], isAllowed, noisy)
local onFailure = req.failed
if not isAllowed then
if onFailure then onFailure( "URL is not whitelisted" ) end
@ -61,9 +62,12 @@ local function wrapFetch()
print( "http.Fetch wrapped, original function at '_http_Fetch'" )
http.Fetch = function( url, onSuccess, onFailure, headers )
local isAllowed = CFCHTTP.isAllowed( url )
local options = CFCHTTP.getOptionsForURI(url)
local isAllowed = options and options.allowed
local noisy = options and options.noisy
local stack = string.Split(debug.traceback(), "\n")
logRequest( "GET", url, stack[3], isAllowed )
logRequest( "GET", url, stack[3], isAllowed, noisy )
if not isAllowed then
if onFailure then onFailure( "URL is not whitelisted" ) end
return
@ -78,9 +82,12 @@ local function wrapPost()
print( "http.Post wrapped, original function at '_http_Post'" )
http.Post = function( url, params, onSuccess, onFailure, headers )
local isAllowed = CFCHTTP.isAllowed( url )
local options = CFCHTTP.getOptionsForURI(url)
local isAllowed = options and options.allowed
local noisy = options and options.noisy
local stack = string.Split(debug.traceback(), "\n")
logRequest( "POST", url, stack[3], isAllowed )
logRequest( "POST", url, stack[3], isAllowed, noisy )
if not isAllowed then
if onFailure then onFailure( "URL is not whitelisted" ) end
return
@ -96,9 +103,12 @@ local function wrapPlayURL()
print( "sound.PlayURL wrapped, original function at _sound_PlayUrl" )
sound.PlayURL = function( url, flags, callback )
local isAllowed = CFCHTTP.isAllowed( url )
local options = CFCHTTP.getOptionsForURI(url)
local isAllowed = options and options.allowed
local noisy = options and options.noisy
local stack = string.Split( debug.traceback(), "\n" )
logRequest( "GET", url, stack[3], isAllowed )
logRequest( "GET", url, stack[3], isAllowed, noisy )
if not isAllowed then
if callback then callback( nil, BASS_ERROR_ILLPARAM, "BASS_ERROR_ILLPARAM" ) end
return

View File

@ -0,0 +1,60 @@
CFCHTTP = CFCHTTP or {}
CFCHTTP.config = include("default_config.lua")
function CFCHTTP.LoadConfigs()
CFCHTTP.config = include("default_config.lua")
CFCHTTP.loadLuaConfigs()
if CLIENT then
local fileConfig = CFCHTTP.readFileConfig()
if fileConfig then
CFCHTTP.config = CFCHTTP.mergeConfigs(CFCHTTP.config, fileConfig)
end
end
end
-- LoadLuaConfigs loads the default config and then any lua files in the cfc_http_restrictions/configs directory
function CFCHTTP.loadLuaConfigs()
local files = file.Find("cfc_http_restrictions/configs/*.lua", "LUA")
for _, file in pairs(files) do
AddCSLuaFile("cfc_http_restrictions/configs/" .. file)
local newConfig = include("cfc_http_restrictions/configs/" .. file)
CFCHTTP.config = CFCHTTP.mergeConfigs(CFCHTTP.config, newConfig)
end
end
function CFCHTTP.mergeConfigs(old, new)
if new.version == "1" then
for domain, options in pairs(new.addresses) do
local currentOptions = old.addresses[domain]
if currentOptions and currentOptions.permanent then
print("[CFC HTTP Restrictions] Skipping " .. domain .. " because it is permanent")
else
old.addresses[domain] = options
end
end
else
ErrorNoHalt("[CFC HTTP Restrictions] Invalid config version: " .. new.version)
end
return old
end
function CFCHTTP.copyConfig(cfg)
return util.JSONToTable(util.TableToJSON(cfg))
end
function CFCHTTP.saveFileConfig(config)
file.Write( "cfc_cl_http_whitelist_config.json", util.TableToJSON( config, true ) )
notification.AddLegacy( "Saved http whitelist", NOTIFY_GENERIC, 5 )
end
function CFCHTTP.readFileConfig()
local fileData = file.Read( "cfc_cl_http_whitelist_config.json" )
if not fileData then return end
return util.JSONToTable( fileData )
end
CFCHTTP.LoadConfigs()

View File

@ -0,0 +1,31 @@
AddCSLuaFile()
-- TODO this should be in its own repo
return {
version="1", -- this field allows backwards compatibility if the config structure is ever updated
wrapHTMLPanels = true,
addresses = {
-- TODO can we delete these, we have *.cfcservers.org?
["nanny.cfcservers.org"] = {allowed=true, noisy=true, permanent=true},
["paste.cfcservers.org"] = {allowed=true, noisy=true, permanent=true},
["cdn.cfcservers.org"] = {allowed=true, noisy=true, permanent=true},
["*.cfcservers.org"] = {allowed=true, permanent=true},
["cfcservers.org"] = {allowed=true, permanent=true},
["google.com"] = {allowed=true, permanent=true},
["www.google.com"] = {allowed=true, permanent=true},
["api.mixpanel.com"] = {allowed=true},
-- fox pictures
["fox.pics"] = {allowed=true},
["*.fox.pics"] = {allowed=true},
-- media player
["samuelmaddock.github.io"] = {allowed=true, isPermanent=true} -- Media player
}
}

View File

@ -0,0 +1,79 @@
AddCSLuaFile()
return {
version="1", -- this field allows backwards compatibility if the config structure is ever updated
wrapHTMLPanels = false,
defaultAssetURIOptions = {
allowed=true
},
defaultOptions = {
allowed=false,
},
addresses = {
["google.com"] = {allowed=true, noisy=true},
["www.google.com"] = {allowed=true, noisy=true},
["steamcommunity.com"] = {allowed=true},
["api.github.com"] = {allowed=true},
["github.com"] = {allowed=true},
["thegrb93.github.io"] = {allowed=true},
-- dropbox
["dl.dropboxusercontent.com"] = {allowed=true},
["dl.dropbox.com"] = {allowed=true},
["www.dropbox.com"] = {allowed=true},
-- onedrive
["onedrive.live.com"] = {allowed=true},
["api.onedrive.com"] = {allowed=true},
-- google drive
["docs.google.com"] = {allowed=true},
["drive.google.com"] = {allowed=true},
-- youtube
["youtube.com"] = {allowed=true},
["youtu.be"] = {allowed=true},
["raw.githubusercontent.com"] = {allowed=true},
["gist.githubusercontent.com"] = {allowed=true},
["gitlab.com"] = {allowed=true},
["bitbucket.org"] = {allowed=true},
["u.teknik.io"] = {allowed=true},
["i.imgur.com"] = {allowed=true},
["pastebin.com"] = {allowed=true},
["p.teknik.io"] = {allowed=true},
["paste.ee"] = {allowed=true},
["hastebin.com"] = {allowed=true},
["hastebin.nl"] = {allowed=true},
["puu.sh"] = {allowed=true},
["images.akamai.steamusercontent.com"] = {allowed=true},
["steamcdn-a.akamaihd.net"] = {allowed=true},
["facepunch.com"] = {allowed=true},
["*.facepunch.com"] = {allowed=true},
["i.redditmedia.com"] = {allowed=true},
["i.redd.it"] = {allowed=true},
["api.wolframalpha.com"] = {allowed=true},
["text-to-speech-demo.ng.bluemix.net"] = {allowed=true},
["translate.google.com"] = {allowed=true},
["cdn[%w-_]*.discordapp%.com"] = {allowed=true, pattern=true},
["images-([%w%-]+)%.discordapp%.net"] = {allowed=true, pattern=true},
["i([%w-_]+)%.tinypic%.com"] = {allowed=true, pattern=true}
}
}

View File

@ -1,231 +0,0 @@
CFCHTTP = CFCHTTP or {}
CFCHTTP.noisyDomains = {
["nanny.cfcservers.org"] = true,
["google.com"] = true,
["www.google.com"] = true,
["cdn.cfcservers.org"] = true,
["api.mixpanel.com"] = true,
["paste.cfcservers.org"] = true
}
CFCHTTP.allowedAddresses = {
["steamcommunity.com"] = {allowed=true},
["api.github.com"] = {allowed=true},
["github.com"] = {allowed=true},
["thegrb93.github.io"] = {allowed=true},
-- dropbox
["dl.dropboxusercontent.com"] = {allowed=true},
["dl.dropbox.com"] = {allowed=true},
["www.dropbox.com"] = {allowed=true},
-- onedrive
["onedrive.live.com"] = {allowed=true},
["api.onedrive.com"] = {allowed=true},
-- google drive
["docs.google.com"] = {allowed=true},
["drive.google.com"] = {allowed=true},
-- youtube
["youtube.com"] = {allowed=true},
["youtu.be"] = {allowed=true},
["raw.githubusercontent.com"] = {allowed=true},
["gist.githubusercontent.com"] = {allowed=true},
["gitlab.com"] = {allowed=true},
["bitbucket.org"] = {allowed=true},
["u.teknik.io"] = {allowed=true},
["i.imgur.com"] = {allowed=true},
["pastebin.com"] = {allowed=true},
["p.teknik.io"] = {allowed=true},
["paste.ee"] = {allowed=true},
["hastebin.com"] = {allowed=true},
["hastebin.nl"] = {allowed=true},
["puu.sh"] = {allowed=true},
["images.akamai.steamusercontent.com"] = {allowed=true},
["steamcdn-a.akamaihd.net"] = {allowed=true},
["i.redditmedia.com"] = {allowed=true},
["i.redd.it"] = {allowed=true},
["api.wolframalpha.com"] = {allowed=true},
["text-to-speech-demo.ng.bluemix.net"] = {allowed=true},
["translate.google.com"] = {allowed=true},
["api.foxorsomething.net"] = {allowed=true},
["cdn[%w-_]*.discordapp%.com"] = {allowed=true, isPattern=true},
["images-([%w%-]+)%.discordapp%.net"] = {allowed=true, isPattern=true},
["i([%w-_]+)%.tinypic%.com"] = {allowed=true, isPattern=true},
["api.mixpanel.com"] = {allowed=true},
["*.cfcservers.org"] = {allowed=true, isPermanent=true},
["cfcservers.org"] = {allowed=true, isPermanent=true},
["google.com"] = {allowed=true, isPermanent=true},
["www.google.com"] = {allowed=true, isPermanent=true},
["samuelmaddock.github.io"] = {allowed=true, isPermanent=true} -- Media player
}
AddressCache = {}
ParsedAddressCache = {}
function CFCHTTP.getAddress( url )
local cached = ParsedAddressCache[url]
if cached then return cached end
local pattern = "(%a+)://([%a%d%.-]+):?(%d*)/?.*"
local _, _, protocol, addr, port = string.find( url, pattern )
ParsedAddressCache[url] = addr
return addr
end
-- escapes all lua pattern characters and allows the use of * as a wildcard
local escaped = {}
local function escapeAddr( addr )
if escaped[addr] then return escaped[addr] end
local split = string.Split( addr, "*" )
for i=1, #split do
split[i] = string.PatternSafe( split[i] )
end
escaped[addr] = table.concat( split, ".*" )
return escaped[addr]
end
function CFCHTTP.checkAllowed( url )
if not url then return end
local address = CFCHTTP.getAddress( url )
if not address then return end
local allowedEntry = CFCHTTP.allowedAddresses[address]
if allowedEntry and allowedEntry.allowed and not allowedEntry.isPattern then
return allowedEntry.allowed
end
for allowedAddr, allowedEntry in pairs( CFCHTTP.allowedAddresses ) do
if not allowedEntry.isPattern then
allowedAddr = escapeAddr( allowedAddr )
end
if string.match( address, "^"..allowedAddr.."$" ) then
return allowedEntry.allowed
end
end
end
function CFCHTTP.isAllowed( url )
local cached = AddressCache[url]
if cached ~= nil then return cached end
local isAllowed = CFCHTTP.checkAllowed( url )
AddressCache[url] = isAllowed
return isAllowed
end
local function getUrlsInHTML( html )
local pattern = "%a+://[%a%d%.-]+:?%d*/[a-zA-Z0-9%.]+"
local urls = {}
for url in string.gmatch(html, pattern) do
table.insert(urls, url)
end
return urls
end
function CFCHTTP.isHTMLAllowed( html )
local urls = getUrlsInHTML( html )
for _, url in pairs(urls) do
if url and not CFCHTTP.isAllowed( url ) then
return false, url
end
end
return true, ""
end
function CFCHTTP.allowAddress( addr, isPattern, isPermanent )
if CFCHTTP.allowedAddresses[addr] ~= nil and CFCHTTP.allowedAddresses[addr].isPermanent then
notification.AddLegacy( "You cant change this address", NOTIFY_ERROR, 5 )
return false
end
CFCHTTP.allowedAddresses[addr] = {
allowed=true,
isPattern=isPattern,
isPermanent=isPermanent
}
AddressCache = {}
return true
end
function CFCHTTP.blockAddress( addr )
if CFCHTTP.allowedAddresses[addr] ~= nil and CFCHTTP.allowedAddresses[addr].isPermanent then
notification.AddLegacy( "You cant change this address", NOTIFY_ERROR, 5 )
return false
end
CFCHTTP.allowedAddresses[addr] = {
allowed=false,
isPattern=isPattern,
isPermanent=isPermanent
}
AddressCache = {}
return true
end
function CFCHTTP.removeAddress( addr )
if CFCHTTP.allowedAddresses[addr] ~= nil and CFCHTTP.allowedAddresses[addr].isPermanent then
notification.AddLegacy( "You cant change this address", NOTIFY_ERROR, 5 )
return false
end
CFCHTTP.allowedAddresses[addr] = nil
AddressCache = {}
return true
end
function CFCHTTP.saveList()
file.CreateDir( "cfc" )
file.Write( "cfc/http_whitelist.json", util.TableToJSON( CFCHTTP.allowedAddresses ) )
notification.AddLegacy( "Saved http whitelist", NOTIFY_GENERIC, 5 )
end
function CFCHTTP.readList()
local fileData = file.Read( "cfc/http_whitelist.json" )
if not fileData then return end
local loadedWhitelist = util.JSONToTable( fileData )
for address, entryData in pairs( CFCHTTP.allowedAddresses ) do
if entryData.isPermanent then
loadedWhitelist[address] = entryData
end
end
CFCHTTP.allowedAddresses = loadedWhitelist
end
CFCHTTP.readList()