Patch methods used to bypass whitelist (#24)

* add code to handle pls files

* add PlayURL file data inspecting before playing

* add filetypes.lua

* fix bugs with file url fetching

* naming changes

* allow non recognized filetypes

* update config

* change function names

* dont redefine locals

* hot fix html wrapping

* wrap RunJavascript and apply html url checks

* Remove debug print

* dont log javascript when no urls are found

---------

Co-authored-by: Brandon Sturgeon <brandon@brandonsturgeon.com>

lua/autorun/cfc_http_whitelist_init.lua

@@ -8,9 +8,13 @@ local function includeClient( f )
     end
 end
 
+local function includeShared( f )
+    AddCSLuaFile( f )
+    include( f )
+end
 include( "cfc_http_restrictions/config_loader.lua" )
+includeShared( "cfc_http_restrictions/client/filetypes.lua" )
 includeClient( "cfc_http_restrictions/config_loader.lua" )
-
 includeClient( "cfc_http_restrictions/client/list_manager.lua" )
 includeClient( "cfc_http_restrictions/client/list_view.lua" )
 includeClient( "cfc_http_restrictions/client/wrap_functions.lua" )

lua/cfc_http_restrictions/client/filetypes.lua

@@ -0,0 +1,45 @@
+CFCHTTP.FileTypes = CFCHTTP.FIleTypes or {}
+
+local files, _ = file.Find( "cfc_http_restrictions/client/filetypes/*.lua", "LUA" )
+for _, f in pairs( files ) do
+    include( "cfc_http_restrictions/client/filetypes/" .. f )
+    AddCSLuaFile( "cfc_http_restrictions/client/filetypes/" .. f )
+end
+
+---@param data string
+---@return table|nil
+function CFCHTTP.getFileType( data )
+    for _, f in pairs( CFCHTTP.FileTypes ) do
+        if f.IsFileData( data ) then
+            return f
+        end
+    end
+    return nil
+end
+
+---@param url string
+---@param callback fun( urls: string[], err: string|nil)
+function CFCHTTP.GetFileDataURLS( url, callback )
+    http.Fetch( url, function( body, _, _, code )
+        if code < 200 or code > 299 then
+            callback( {}, "HTTP request returned status code " .. code )
+            return
+        end
+
+        local filetype = CFCHTTP.getFileType( body )
+        if filetype then
+            local urls, err = filetype.GetURLSFromData( body )
+            if err then
+                callback( {}, err )
+            else
+                if #urls == 0 then
+                    callback( {}, "No URLs found in file" )
+                    return
+                end
+                callback( urls, nil )
+            end
+        else
+            callback( {}, nil )
+        end
+    end )
+end

lua/cfc_http_restrictions/client/filetypes/html.lua

@@ -0,0 +1,39 @@
+local HTML = {
+    name = "HTML",
+    extension = "html",
+    maxFileSize = 0,
+}
+CFCHTTP.FileTypes.HTML = HTML
+
+-- Not implemented
+---@param body string
+---@return boolean
+function HTML.IsFileData( body )
+    return false
+end
+
+---@param url string
+---@return boolean
+function HTML.IsFileURL( url )
+    if string.EndsWith( url, "." .. HTML.extension ) then return true end
+    return false
+end
+
+local function getUrlsFromText( text )
+    local pattern = "%a+://[%a%d%.-]+:?%d*/?[a-zA-Z0-9%.]*"
+
+    local urls = {}
+    for url in string.gmatch( text, pattern ) do
+        table.insert( urls, url )
+    end
+
+    return urls
+end
+
+---@param body string
+---@return string[] urls
+---@return string|nil error
+function HTML.GetURLSFromData( body )
+    local urls = getUrlsFromText( body )
+    return urls, nil
+end

lua/cfc_http_restrictions/client/filetypes/m3u.lua

@@ -0,0 +1,32 @@
+CFCHTTP.FileTypes.M3U = {
+    name = "M3U",
+    allowed = false,
+    extension = "m3u",
+    maxFileSize = 0,
+}
+local M3U = CFCHTTP.FileTypes.M3U
+
+---@param body string
+---@return boolean
+function M3U.IsFileData( body )
+    local endPos = string.find( body, "\n" )
+    local firstLine = string.sub( body, 1, endPos )
+    if string.find( firstLine, "#EXTM3U" ) then return true end
+    return false
+end
+
+---@param url string
+---@return boolean
+function M3U.IsFileURL( url )
+    if string.EndsWith( url, "." .. M3U.extension ) then return true end
+    return false
+end
+
+---@param _ string
+---@return string[] urls
+---@return string|nil error
+function M3U.GetURLSFromData( _ )
+    return {}, "m3u parsing not implemented"
+end
+
+return M3U

lua/cfc_http_restrictions/client/filetypes/pls.lua

@@ -0,0 +1,90 @@
+CFCHTTP.FileTypes.PLS = {
+    name = "PLS",
+    allowed = false,
+    extension = ".pls",
+    maxFileSize = 10000,
+}
+local PLS = CFCHTTP.FileTypes.PLS
+
+
+---@param body string
+---@return table data
+local function loadPLSFile( body )
+    body = string.Replace( body, "\r\n", "\n" )
+    body = string.Replace( body, "\r", "\n" )
+
+    local lines = string.Split( body, "\n" )
+    local section;
+    local data = {}
+
+    for _, line in ipairs( lines ) do
+        local tempSection = line:match( "^%[([^%[%]]+)%]$" );
+        if tempSection then
+            section = tonumber( tempSection ) and tonumber( tempSection ) or tempSection;
+            data[section] = data[section] or {};
+        end
+
+        local param, value = line:match( "^([%w|_]+)%s-=%s-(.+)$" );
+        if param and value ~= nil then
+            if tonumber( value ) then
+                value = tonumber( value );
+            end
+            data[section][param] = value;
+        end
+    end
+
+    return data;
+end
+
+---@param body string
+---@return boolean
+function PLS.IsFileData( body )
+    if string.find( body, "%[playlist%]" ) then return true end
+    return false
+end
+
+---@param url string
+---@return boolean
+function PLS.IsFileURL( url )
+    if string.EndsWith( url, "." .. PLS.extension ) then return true end
+    return false
+end
+
+---@param text string
+---@return string[] urls
+local function getUrlsFromText( text )
+    local pattern = "%a+://[%a%d%.-]+:?%d*/?[a-zA-Z0-9%.]*"
+
+    local urls = {}
+    for url in string.gmatch( text, pattern ) do
+        table.insert( urls, url )
+    end
+
+    return urls
+end
+
+---@param body string
+---@return string[] urls
+---@return string|nil error
+function PLS.GetURLSFromData( body )
+    if not PLS.allowed then return {}, "pls files are not allowed" end
+
+    if #body > PLS.maxFileSize then return {}, "body too large" end
+    local urls = getUrlsFromText( body )
+
+    local plsData = loadPLSFile( body )
+    if not plsData.playlist then
+        return urls, "no playlist section"
+    end
+
+    for i = 1, 150 do
+        local f = plsData.playlist["File" .. i]
+        if not f then
+            return urls, nil
+        end
+        table.insert( urls, f )
+    end
+    return urls, "too many files"
+end
+
+return PLS

lua/cfc_http_restrictions/client/list_manager.lua

@@ -2,12 +2,14 @@ CFCHTTP = CFCHTTP or {}
 
 
 local parsedAddressCache = {}
+---@parm url string
+---@return string
 function CFCHTTP.getAddress( url )
     local cached = parsedAddressCache[url]
     if cached then return cached end
 
     local pattern = "(%a+)://([%a%d%.-]+):?(%d*)/?.*"
-    local  _,  _, _, addr, _ = string.find( url, pattern )
+    local _, _, _, addr, _ = string.find( url, pattern )
     parsedAddressCache[url] = addr
 
     return addr
@@ -32,7 +34,7 @@ local function escapeAddr( addr )
 end
 
 -- TODO reimmplement caching
-function CFCHTTP.getOptionsForURI( url )
+function CFCHTTP.GetOptionsForURL( url )
     if not url then return CFCHTTP.config.defaultOptions end
 
     if CFCHTTP.isAssetURI( url ) then return CFCHTTP.config.defaultAssetURIOptions end
@@ -58,31 +60,32 @@ function CFCHTTP.getOptionsForURI( url )
     return CFCHTTP.config.defaultOptions
 end
 
-local function getUrlsInHTML( html )
-    local pattern = "%a+://[%a%d%.-]+:?%d*/?[a-zA-Z0-9%.]*"
-
-    local urls = {}
-    for url in string.gmatch( html, pattern ) do
-        table.insert( urls, url )
-    end
-
-    return urls
-end
-
-function CFCHTTP.isHTMLAllowed( html )
-    local urls = getUrlsInHTML( html )
+--- Returns the options for a list of URLs
+---@param urls string[]
+---@return {options: table<string, table>, combined: table|nil, combinedUri: string|nil}
+function CFCHTTP.GetOptionsForURLs( urls )
+    local out = {
+        combined = nil,
+        options = {},
+    }
     for _, url in pairs( urls ) do
-        local options = CFCHTTP.getOptionsForURI( url )
-
+        local options = CFCHTTP.GetOptionsForURL( url )
+        out.options[url] = options
         if options and not options.allowed then
-            return false, url
+            out.combined = options
+            out.combinedUri = url
+        elseif not out.combined then
+            out.combined = options
+            out.combinedUri = url
         end
     end
+    if out.combined == nil then
+        out.combined = CFCHTTP.config.defaultOptions
+    end
 
-    return true, ""
+    return out
 end
 
-
 -- file based config functions
 function CFCHTTP.allowAddress( addr )
     if CFCHTTP.config.addresses[addr] ~= nil and CFCHTTP.config.addresses[addr].permanent then

lua/cfc_http_restrictions/client/wrap_functions.lua

@@ -17,7 +17,9 @@ local function logRequest( method, url, fileLocation, allowed, noisy )
     local requestStatus = allowed and "ALLOWED" or "BLOCKED"
     local requestColor = allowed and COLORS.GREEN or COLORS.RED
 
-    if isVerbose == false then
+    if not url then
+        url = "unknown"
+    elseif isVerbose == false then
         local address = CFCHTTP.getAddress( url )
         if noisy then return end
 
@@ -42,7 +44,7 @@ local function wrapHTTP()
     print( "HTTP wrapped, original function at '_G._HTTP'" )
 
     HTTP = function( req )
-        local options = CFCHTTP.getOptionsForURI( req.url )
+        local options = CFCHTTP.GetOptionsForURL( req.url )
         local isAllowed = options and options.allowed
         local noisy = options and options.noisy
 
@@ -62,7 +64,7 @@ local function wrapFetch()
     print( "http.Fetch wrapped, original function at '_http_Fetch'" )
 
     http.Fetch = function( url, onSuccess, onFailure, headers )
-        local options = CFCHTTP.getOptionsForURI( url )
+        local options = CFCHTTP.GetOptionsForURL( url )
         local isAllowed = options and options.allowed
         local noisy = options and options.noisy
 
@@ -82,7 +84,7 @@ local function wrapPost()
     print( "http.Post wrapped, original function at '_http_Post'" )
 
     http.Post = function( url, params, onSuccess, onFailure, headers )
-        local options = CFCHTTP.getOptionsForURI( url )
+        local options = CFCHTTP.GetOptionsForURL( url )
         local isAllowed = options and options.allowed
         local noisy = options and options.noisy
 
@@ -107,18 +109,44 @@ local function wrapPlayURL()
     _sound_PlayURL = _sound_PlayURL or sound.PlayURL
     print( "sound.PlayURL wrapped, original function at _sound_PlayUrl" )
 
+    ---@diagnostic disable-next-line: duplicate-set-field
     sound.PlayURL = function( url, flags, callback )
-        local options = CFCHTTP.getOptionsForURI( url )
+        local stack = string.Split( debug.traceback(), "\n" )
+
+        local options = CFCHTTP.GetOptionsForURL( url )
         local isAllowed = options and options.allowed
         local noisy = options and options.noisy
 
-        local stack = string.Split( debug.traceback(), "\n" )
-        logRequest( "GET", url, stack[3], isAllowed, noisy )
         if not isAllowed then
+            logRequest( "GET", url, stack[3], isAllowed, noisy )
             if callback then callback( nil, CFCHTTP.BASS_ERROR_BLOCKED_URI, "BASS_ERROR_BLOCKED_URI" ) end
             return
         end
-        _sound_PlayURL( url, flags, callback )
+
+        CFCHTTP.GetFileDataURLS( url, function( uris, err )
+            if err ~= nil then
+                print( "Error getting URLs: " .. err )
+                if callback then callback( nil, CFCHTTP.BASS_ERROR_BLOCKED_CONTENT, "BASS_ERROR_BLOCKED_CONTENT" ) end
+                return
+            end
+
+            if #uris == 0 then
+                logRequest( "GET", url, stack[3], isAllowed, noisy )
+                _sound_PlayURL( url, flags, callback )
+                return
+            end
+
+            options = CFCHTTP.GetOptionsForURLs( uris )
+            isAllowed = options.combined.allowed
+
+            logRequest( "GET", url, stack[3], isAllowed, noisy )
+            if not isAllowed then
+                if callback then callback( nil, CFCHTTP.BASS_ERROR_BLOCKED_CONTENT, "BASS_ERROR_BLOCKED_CONTENT" ) end
+                return
+            end
+
+            _sound_PlayURL( url, flags, callback )
+        end )
     end
 end
 
@@ -133,25 +161,56 @@ local function wrapHTMLPanel( panelName )
 
     local setHTML = funcName( "SetHTML" )
     local openURL = funcName( "OpenURL" )
+    local runJavascript = funcName( "RunJavascript" )
 
     _G[setHTML] = _G[setHTML] or controlTable.SetHTML
     _G[openURL] = _G[openURL] or controlTable.OpenURL
+    _G[runJavascript] = _G[runJavascript] or controlTable.RunJavascript
 
     controlTable.SetHTML = function( self, html, ... )
-        local isAllowed, url = CFCHTTP.isHTMLAllowed( html )
+        local urls, err = CFCHTTP.FileTypes.HTML.GetURLSFromData( html )
+        local options = CFCHTTP.GetOptionsForURLs( urls )
+
+        local isAllowed
+        if #urls == 0 then
+            isAllowed = true
+        else
+            isAllowed = err == nil and options.combined and options.combined.allowed
+        end
 
         local stack = string.Split( debug.traceback(), "\n" )
-        logRequest( "GET", url, stack[3], isAllowed )
+        logRequest( "GET", options.combinedUri, stack[3], isAllowed )
 
         if not isAllowed then
             html = [[<h1> BLOCKED </h1>]]
         end
 
-        _G[setHTML]( self, html, ... )
+        return _G[setHTML]( self, html, ... )
+    end
+
+    controlTable.RunJavascript = function( self, js )
+        local urls, err = CFCHTTP.FileTypes.HTML.GetURLSFromData( js )
+        local options = CFCHTTP.GetOptionsForURLs( urls )
+
+        local isAllowed
+        if #urls == 0 then
+            return _G[runJavascript]( self, js )
+        else
+            isAllowed = err == nil and options.combined and options.combined.allowed
+        end
+
+        local stack = string.Split( debug.traceback(), "\n" )
+        logRequest( "GET", options.combinedUri, stack[3], isAllowed )
+
+        if not isAllowed then
+            return
+        end
+
+        return _G[runJavascript]( self, js )
     end
 
     controlTable.OpenURL = function( self, url, ... )
-        local options = CFCHTTP.getOptionsForURI( url )
+        local options = CFCHTTP.GetOptionsForURL( url )
         local isAllowed = options and options.allowed
         local noisy = options and options.noisy
 
@@ -160,7 +219,7 @@ local function wrapHTMLPanel( panelName )
 
         if not isAllowed then return end
 
-        _G[openURL]( self, url, ... )
+        return _G[openURL]( self, url, ... )
     end
 end
 

lua/cfc_http_restrictions/default_config.lua

@@ -12,7 +12,7 @@ return {
         allowed = false,
     },
     addresses = {
-        ["google.com"] =  { allowed = true, noisy = true },
+        ["google.com"] = { allowed = true, noisy = true },
         ["www.google.com"] = { allowed = true, noisy = true },
 
         ["api.steampowered.com"] = { allowed = true },