diff --git a/plugins/openresty/waf/conf/global.json b/plugins/openresty/waf/conf/global.json deleted file mode 100644 index e53bfe670..000000000 --- a/plugins/openresty/waf/conf/global.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "waf": { - "state": "on", - "mode": "protection", - "secret": "qwer1234" - }, - "redis": { - "state": "off", - "host": "127.0.0.1", - "port": 6379, - "password": "Calong@2015", - "ssl": false, - "poolSize": 10 - }, - "ipWhite": { - "state": "on", - "type": "ipWhite", - "action": "allow" - }, - "ipBlack": { - "state": "on", - "code": 403, - "action": "deny", - "type": "ipBlack", - "res": "ip" - }, - "urlWhite": { - "type": "urlWhite", - "state": "on", - "action": "allow" - }, - "urlBlack": { - "type": "urlBlack", - "state": "on", - "code": 403, - "action": "deny" - }, - "uaWhite": { - "type": "uaWhite", - "state": "off", - "action": "allow" - }, - "uaBlack": { - "type": "uaBlack", - "state": "on", - "code": 403, - "action": "deny" - }, - "notFoundCount": { - "state": "off", - "type": "notFoundCount", - "threshold": 10, - "duration": 60, - "action": "deny", - "ipBlock": "on", - "ipBlockTime": 600 - }, - "methodWhite": { - "type": "methodWhite", - "state": "on", - "code": 444, - "action": "deny" - }, - "bot": { - "state": "on", - "type": "bot", - "uri": "/1pwaf/bot/trap", - "action": "REDIRECT_JS", - "ipBlock": "on", - "ipBlockTime": 600 - }, - "geoRestrict": { - "state": "on", - "rules": [], - "code": 444, - "action": "deny", - "type": "geoRestrict" - }, - "defaultIpBlack": { - "state": "on", - "type": "defaultIpBlack", - "code": 403, - "action": "deny" - }, - "xss": { - "state": "on", - "type": "xss", - "code": 403, - "action": "deny" - }, - "sql": { - "state": "on", - "type": "sql", - "code": 403, - "action": "deny" - }, - "cc": { - "state": "off", - "type": "cc", - "tokenTimeOut": 1800, - "threshold": 120, - "duration": 60, - "action": "deny", - "ipBlock": "on", - "ipBlockTime": 600 - }, - "ccurl": { - "state": "off", - "type": "urlcc", - "action": "deny", - "ipBlock": "on", - "ipBlockTime": 600 - }, - "attackCount": { - "state": "off", - "type": "attackCount", - "threshold": 20, - "duration": 60, - "action": "deny", - "ipBlock": "on", - "ipBlockTime": 600 - }, - "fileExt": { - "state": "on", - "action": "deny", - "code": 403, - "type": "fileExtCheck" - }, - "cookie": { - "type": "cookie", - "state": "on", - "code": 403, - "action": "deny" - }, - "header": { - "state": "on", - "type": "header", - "code": 403, - "action": "deny" - }, - "defaultUaBlack": { - "type": "defaultUaBlack", - "state": "on", - "code": 403, - "action": "deny" - }, - "defaultUrlBlack": { - "type": "defaultUrlBlack", - "state": "on", - "code": 403, - "action": "deny" - }, - "args": { - "type": "args", - "state": "on", - "code": 403, - "action": "deny" - } -} diff --git a/plugins/openresty/waf/conf/siteConfig.json b/plugins/openresty/waf/conf/siteConfig.json deleted file mode 100644 index 25e78eec3..000000000 --- a/plugins/openresty/waf/conf/siteConfig.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "waf": { - "state": "on", - "mode": "protection", - "secret": "qwer1234" - }, - "args": { - "state": "on", - "type": "args", - "code": 403, - "action": "deny" - }, - "defaultUaBlack": { - "type": "defaultUaBlack", - "state": "on", - "code": 403, - "action": "deny" - }, - "cookie": { - "state": "on", - "type": "cookie", - "code": 403, - "action": "deny" - }, - "bot": { - "type": "bot", - "state": "on", - "uri": "/1pwaf/bot/trap", - "action": "deny", - "ipBlock": "off", - "ipBlockTime": 600 - }, - "geoRestrict": { - "state": "off", - "rules": [ - "CN" - ], - "action": "allow" - }, - "defaultIpBlack": { - "state": "on", - "type": "defaultIpBlack", - "code": 403, - "action": "deny" - }, - "xss": { - "state": "on", - "type": "xss", - "code": 403, - "action": "deny" - }, - "sql": { - "state": "on", - "type": "sql", - "code": 403, - "action": "deny" - }, - "cc": { - "state": "on", - "type": "cc", - "rule": "cc", - "tokenTimeOut": 1800, - "threshold": 300, - "duration": 60, - "action": "deny", - "ipBlock": "on", - "ipBlockTime": 600 - }, - "ccurl": { - "state": "on", - "type": "ccurl", - "action": "deny", - "ipBlock": "off", - "ipBlockTime": 600 - }, - "fileExt": { - "state": "on", - "action": "deny", - "code": 403, - "type": "fileExtCheck" - }, - "header": { - "state": "on", - "type": "header", - "code": 403, - "action": "deny" - }, - "defaultUrlBlack": { - "type": "defaultUrlBlack", - "state": "on", - "code": 403, - "action": "deny" - } -} \ No newline at end of file diff --git a/plugins/openresty/waf/conf/token b/plugins/openresty/waf/conf/token deleted file mode 100644 index e69de29bb..000000000 diff --git a/plugins/openresty/waf/conf/waf.conf b/plugins/openresty/waf/conf/waf.conf deleted file mode 100644 index 2d69226c7..000000000 --- a/plugins/openresty/waf/conf/waf.conf +++ /dev/null @@ -1,13 +0,0 @@ -lua_shared_dict dict_locks 100k; -lua_shared_dict waf_req_count 10m; -lua_shared_dict waf 30m; -lua_shared_dict waf_block_ip 10m; -lua_shared_dict waf_ip_arr 10m; -lua_shared_dict waf_limit 10m; -lua_shared_dict ipc_shared_dict 10m; - -lua_package_path "/usr/local/openresty/1pwaf/?.lua;/usr/local/openresty/1pwaf/lib/?.lua;;"; -init_by_lua_file /usr/local/openresty/1pwaf/init.lua; -access_by_lua_file /usr/local/openresty/1pwaf/waf.lua; -log_by_lua_file /usr/local/openresty/1pwaf/log_and_traffic.lua; -init_worker_by_lua_file /usr/local/openresty/1pwaf/worker.lua; diff --git a/plugins/openresty/waf/config.lua b/plugins/openresty/waf/config.lua deleted file mode 100644 index 0e9cb3056..000000000 --- a/plugins/openresty/waf/config.lua +++ /dev/null @@ -1,215 +0,0 @@ -local file_utils = require "file" -local lfs = require "lfs" -local utils = require "utils" -local cjson = require "cjson" - - -local read_rule = file_utils.read_rule -local read_file2string = file_utils.read_file2string -local read_file2table = file_utils.read_file2table -local set_content_to_file = file_utils.set_content_to_file -local read_list2table = file_utils.read_list2table -local list_dir = lfs.dir -local attributes = lfs.attributes -local match_str = string.match - -local waf_dir = "/usr/local/openresty/1pwaf/" -local config_dir = waf_dir .. 'conf/' -local global_rule_dir = waf_dir .. 'rules/' -local site_dir = waf_dir .. 'sites/' -local ip_group_dir = global_rule_dir .. 'ip_group/' - -local _M = {} -local config = {} -local global_config = {} - -local function init_sites_config() - local site_config = {} - local site_rules = {} - for entry in list_dir(site_dir) do - if entry ~= "." and entry ~= ".." then - local site_path = site_dir .. entry .. "/" - if attributes(site_path, "mode") == "directory" then - local site_key = entry - for s_entry in list_dir(site_path) do - local s_entry_path = site_path .. s_entry - if attributes(s_entry_path, "mode") == "file" and s_entry == "config.json" then - local s_config = read_file2table(s_entry_path) - site_config[site_key] = s_config - end - if attributes(s_entry_path, "mode") == "directory" and s_entry == "rules" then - local s_rules = {} - local rule_dir = s_entry_path .. "/" - for r_file in list_dir(rule_dir) do - if r_file ~= "." and r_file ~= ".." then - local rule_path = rule_dir .. r_file - local rule_type = match_str(r_file, "(.-)%.json$") - if attributes(rule_path, "mode") == "file" then - local s_rule = nil - if rule_type == "methodWhite" then - s_rule = read_rule(rule_dir, rule_type, true) - - else - s_rule = read_rule(rule_dir, rule_type) - end - s_rules[rule_type] = s_rule - end - end - end - site_rules[site_key] = s_rules - end - end - end - end - end - config.site_config = site_config - config.site_rules = site_rules -end - -local function ini_waf_info() - local waf_info = read_file2table(waf_dir .. 'waf.json') - if waf_info then - ngx.log(ngx.NOTICE, "Load " .. waf_info.name .. " Version:" .. waf_info.version) - end -end - -local function load_ip_group() - local ip_group_list = {} - for entry in list_dir(ip_group_dir) do - if entry ~= "." and entry ~= ".." then - local group_path = ip_group_dir .. entry - local group_value = read_list2table(group_path) - ip_group_list[entry] = group_value - end - end - local ok, err = cache:set("ip_group_list", { - ipc_shm = "ipc_shared_dict", - },ip_group_list) - if not ok then - ngx.log(ngx.ERR, "Failed to set config",err) - end -end - -local function init_global_config() - local global_config_file = config_dir .. 'global.json' - global_config = file_utils.read_file2table(global_config_file) - config.global_config = global_config - config.isProtectionMode = global_config["mode"] == "protection" and true or false - - _M.get_token() - - local rules = {} - rules.uaBlack = read_rule(global_rule_dir, "uaBlack") - rules.uaWhite = read_rule(global_rule_dir, "uaWhite") - rules.urlBlack = read_rule(global_rule_dir, "urlBlack") - rules.urlWhite = read_rule(global_rule_dir, "urlWhite") - rules.ipWhite = read_rule(global_rule_dir, "ipWhite") - rules.args = read_rule(global_rule_dir, "args") - rules.cookie = read_rule(global_rule_dir, "cookie") - rules.defaultUaBlack = read_rule(global_rule_dir, "defaultUaBlack") - rules.defaultUrlBlack = read_rule(global_rule_dir, "defaultUrlBlack") - rules.header = read_rule(global_rule_dir, "header") - rules.ipBlack = read_rule(global_rule_dir, "ipBlack") - - config.global_rules = rules - - local html_res = {} - local htmDir = waf_dir .. "html/" - html_res.slide = read_file2string(htmDir .. "slide.html") - html_res.slide_js = read_file2string(htmDir .. "slide.js") - html_res.five_second = read_file2string(htmDir .. "5s.html") - html_res.five_second_js = read_file2string(htmDir .. "5s.js") - html_res.redirect = read_file2string(htmDir .. "redirect.html") - html_res.ip = read_file2string(htmDir .. "ip.html") - - config.html_res = html_res - - _M.waf_dir = waf_dir - _M.waf_db_dir = waf_dir .. "db/" - _M.waf_db_path = _M.waf_db_dir .. "1pwaf.db" - _M.waf_log_db_path = _M.waf_db_dir .. "req_log.db" - _M.config_dir = config_dir - -end - -function _M.load_config_file() - ini_waf_info() - init_global_config() - init_sites_config() - load_ip_group() - - local ok, err = cache:set("config", { - ipc_shm = "ipc_shared_dict", - },config) - if not ok then - ngx.log(ngx.ERR, "Failed to set config",err) - end -end - -local function get_config() - local cache_config = cache:get("config", { - ipc_shm = "ipc_shared_dict", - }) - if not cache_config then - return config - end - return cache_config -end - -function _M.get_site_config(website_key) - return get_config().site_config[website_key] -end - -function _M.get_site_rules(website_key) - return get_config().site_rules[website_key] -end - -function _M.get_global_config(name) - return get_config().global_config[name] -end - -function _M.get_global_rules(name) - return get_config().global_rules[name] -end - -function _M.is_global_state_on(name) - return get_config().global_config[name]["state"] == "on" and true or false -end - -function _M.is_site_state_on(name) - return get_config().site_config[name]["state"] == "on" and true or false -end - -function _M.get_redis_config() - return get_config().global_config["redis"] -end - -function _M.get_html_res(name) - return get_config().html_res[name] -end - -function _M.is_waf_on() - return _M.is_global_state_on("waf") -end - -function _M.is_redis_on() - return _M.is_global_state_on("redis") -end - -function _M.get_secret() - return get_config().global_config["waf"]["secret"] -end - -function _M.get_token() - local waf_dict = ngx.shared.waf - local token = waf_dict:get("token") - if not token then - token = utils.random_string(20) - waf_dict:set("token", token, 86400) - local token_path = config_dir .. 'token' - set_content_to_file(token,token_path) - end - return token -end - -return _M \ No newline at end of file diff --git a/plugins/openresty/waf/db.lua b/plugins/openresty/waf/db.lua deleted file mode 100644 index 792030adb..000000000 --- a/plugins/openresty/waf/db.lua +++ /dev/null @@ -1,119 +0,0 @@ -local config = require "config" - -local open_file = io.open -local exec = os.execute -local pcall = pcall - -local _M = {} - -local function init_dir(path) - local file = open_file(path, "rb") - if not file then - exec("mkdir -p " .. path) - end -end - -local function check_table(table_name,wafdb) - if wafdb == nil then - return false - end - local stmt = wafdb:prepare("SELECT COUNT(*) FROM sqlite_master where type='table' and name=?") - local rows = 0 - if stmt ~= nil then - stmt:bind_values(table_name) - stmt:step() - rows = stmt:get_uvalues() - stmt:finalize() - end - return rows > 0 -end - -local function init_db_config(db_path) - local ok, sqlite3 = pcall(function() - return require "lsqlite3" - end) - if not ok then - return false - end - local wafdb = sqlite3.open(db_path) - if wafdb == nil then - return false - end - wafdb:exec([[PRAGMA journal_mode = wal]]) - wafdb:exec([[PRAGMA synchronous = OFF]]) - wafdb:exec([[PRAGMA page_size = 8192]]) - wafdb:exec([[PRAGMA journal_size_limit = 2147483648]]) - return wafdb -end - -function _M.init() - init_dir(config.waf_db_dir) - local wafdb = init_db_config(config.waf_db_path) - if not wafdb then - return false - end - - local status = {} - if not check_table("waf_stat",wafdb) then - status = wafdb:exec([[ - CREATE TABLE waf_stat ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - day TEXT, - req_count INTEGER, - attack_count INTEGER, - count4xx INTEGER, - count5xx INTEGER, - create_date DATETIME - )]]) - ngx.log(ngx.ERR, "init waf_stat status"..status) - end - - local logdb = init_db_config(config.waf_log_db_path) - if not check_table("req_logs",logdb) then - status = logdb:exec([[ - CREATE TABLE req_logs ( - id TEXT PRIMARY KEY, - ip TEXT, - ip_iso TEXT, - ip_country_zh TEXT, - ip_country_en TEXT, - ip_province_zh TEXT, - ip_province_en TEXT, - ip_longitude TEXT, - ip_latitude TEXT, - localtime DATETIME, - server_name TEXT, - website_key TEXT, - host TEXT, - method TEXT, - uri TEXT, - user_agent TEXT, - exec_rule TEXT, - rule_type TEXT, - match_rule TEXT, - match_value TEXT, - nginx_log TEXT, - blocking_time INTEGER, - action TEXT, - is_block INTEGER, - is_attack INTEGER - )]]) - end - - if not check_table("block_ips",logdb) then - status = logdb:exec([[ - CREATE TABLE block_ips ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ip TEXT, - is_block INTEGER, - blocking_time INTEGER, - req_log_id TEXT, - create_date DATETIME - )]]) - ngx.log(ngx.ERR, "init block_ip status"..status) - end - - ngx.log(ngx.ERR, "init db success") -end - -return _M \ No newline at end of file diff --git a/plugins/openresty/waf/html/5s.html b/plugins/openresty/waf/html/5s.html deleted file mode 100644 index 245ad7708..000000000 --- a/plugins/openresty/waf/html/5s.html +++ /dev/null @@ -1,21 +0,0 @@ - - - -5s - - - -
-
正在验证...
-
验证成功
-
-
- - - diff --git a/plugins/openresty/waf/html/5s.js b/plugins/openresty/waf/html/5s.js deleted file mode 100644 index 7be80bc94..000000000 --- a/plugins/openresty/waf/html/5s.js +++ /dev/null @@ -1,24 +0,0 @@ -window.onload = function () { - setTimeout(function () { - showSuccess(); - verifySucc(); - }, 5000); - - function showSuccess() { - document.getElementById("loadingText").style.display = "none"; - document.getElementById("loadingSuccess").style.display = "block"; - document.querySelector(".loadingSpinner").style.display = "none"; - } - - function verifySucc() { - let xhr = new XMLHttpRequest(); - xhr.onreadystatechange = function () { - if (xhr.readyState === 4 && xhr.status === 200) { - window.location.reload(); - } - }; - const requestUrl = "%s-%s-%s"; - xhr.open("GET", requestUrl, true); - xhr.send(); - } -} \ No newline at end of file diff --git a/plugins/openresty/waf/html/ip.html b/plugins/openresty/waf/html/ip.html deleted file mode 100644 index 95b3c4f02..000000000 --- a/plugins/openresty/waf/html/ip.html +++ /dev/null @@ -1,29 +0,0 @@ - - - - 访问被拒绝 - - - -
-
无法访问
-
-

很抱歉，您的 IP 已被禁止访问

-
    -
  1. 如被误封，请联系网站管理员解封
    2. -
-
-
此防护来自 1Panel
-
- - diff --git a/plugins/openresty/waf/html/redirect.html b/plugins/openresty/waf/html/redirect.html deleted file mode 100644 index 64c776b47..000000000 --- a/plugins/openresty/waf/html/redirect.html +++ /dev/null @@ -1,24 +0,0 @@ - - - - 网站防火墙 - - - -
-
网站防火墙
-
-

您的请求不合法，已被拒绝

-
-
此网站防护来自 1Panel
-
- - diff --git a/plugins/openresty/waf/html/slide.html b/plugins/openresty/waf/html/slide.html deleted file mode 100644 index 2b62dfbad..000000000 --- a/plugins/openresty/waf/html/slide.html +++ /dev/null @@ -1,26 +0,0 @@ - - - -滑动认证 - - - -
-
-
-
-
-
-
- - - diff --git a/plugins/openresty/waf/html/slide.js b/plugins/openresty/waf/html/slide.js deleted file mode 100644 index d5df91929..000000000 --- a/plugins/openresty/waf/html/slide.js +++ /dev/null @@ -1 +0,0 @@ -window.onload=function(){(function(){const dragContainer=document.getElementById("dragContainer");const dragBg=document.getElementById("dragBg");const dragText=document.getElementById("dragText");const dragHandler=document.getElementById("dragHandler");const maxHandleOffset=dragContainer.clientWidth-dragHandler.clientWidth;let isVertifySucc=false;initDrag();function initDrag(){dragText.textContent="拖动滑块验证";dragHandler.addEventListener("mousedown",onDragStart);dragHandler.addEventListener("touchstart",onDragStart)}function onDragStart(a){a.preventDefault();if(a.type==="mousedown"||(a.type==="touchstart"&&a.touches.length===1)){document.addEventListener("mousemove",onDragMove);document.addEventListener("touchmove",onDragMove);document.addEventListener("mouseup",onDragEnd);document.addEventListener("touchend",onDragEnd)}}function onDragMove(a){let clientX;if(a.type==="mousemove"){clientX=a.clientX}else if(a.type==="touchmove"&&a.touches.length===1){clientX=a.touches[0].clientX}let containerOffsetX=clientX-dragContainer.getBoundingClientRect().left;let left=containerOffsetX-dragHandler.clientWidth/2;if(left<0){left=0}else if(left>maxHandleOffset){left=maxHandleOffset}dragHandler.style.left=left+"px";dragBg.style.width=dragHandler.style.left}function onDragEnd(){document.removeEventListener("mousemove",onDragMove);document.removeEventListener("touchmove",onDragMove);document.removeEventListener("mouseup",onDragEnd);document.removeEventListener("touchend",onDragEnd);if(!isVertifySucc){let left=dragHandler.offsetLeft;if(left>=maxHandleOffset){verifySucc()}else{dragHandler.style.left="0px";dragBg.style.width="0px"}}}function verifySucc(){isVertifySucc=true;dragText.textContent="验证通过";dragText.style.color="white";dragHandler.setAttribute("class","dragHandlerOkBg");dragHandler.removeEventListener("mousedown",onDragStart);dragHandler.removeEventListener("touchstart",onDragStart);let xhr=new XMLHttpRequest();xhr.onreadystatechange=function(){if(xhr.readyState===4&&xhr.status===200){window.location.reload()}};const requestUrl="%s-%s-%s";xhr.open("GET",requestUrl,true);xhr.send()}})()}; \ No newline at end of file diff --git a/plugins/openresty/waf/init.lua b/plugins/openresty/waf/init.lua deleted file mode 100644 index e856ce256..000000000 --- a/plugins/openresty/waf/init.lua +++ /dev/null @@ -1,21 +0,0 @@ -local db = require "db" -local config = require "config" -local mlcache = require "resty.mlcache" - -local cache, err = mlcache.new("config", "waf", { - lru_size = 1000, - ipc_shm = "ipc_shared_dict", -}) -if not cache then - error("could not create mlcache: " .. err) -end -_G.cache = cache - - -config.load_config_file() -db.init() - - - - - diff --git a/plugins/openresty/waf/lib/action.lua b/plugins/openresty/waf/lib/action.lua deleted file mode 100644 index d5313ea8d..000000000 --- a/plugins/openresty/waf/lib/action.lua +++ /dev/null @@ -1,167 +0,0 @@ -local config = require "config" -local redis_util = require "redis_util" -local format_str = string.format - -local _M = {} - -local function deny(status_code, res) - if status_code == nil then - status_code = 403 - end - - ngx.status = status_code - if res ~= nil and res ~= "" then - ngx.header.content_type = "text/html; charset=UTF-8" - ngx.say(config.get_html_res(res)) - end - ngx.exit(ngx.status) -end - -local function redirect(status_code) - ngx.header.content_type = "text/html; charset=UTF-8" - ngx.say(config.get_html_res("redirect")) - ngx.status = status_code - ngx.exit(ngx.status) -end - -local function slide() - ngx.header.content_type = "text/html; charset=UTF-8" - ngx.header.Cache_Control = "no-cache" - ngx.status = 200 - ngx.say(format_str(config.get_html_res("slide"), ngx.md5(ngx.ctx.ip))) - ngx.exit(ngx.status) -end - -local function five_second() - ngx.header.content_type = "text/html; charset=UTF-8" - ngx.header.Cache_Control = "no-cache" - ngx.status = 200 - ngx.say(format_str(config.get_html_res("five_second"), ngx.md5(ngx.ctx.ip))) - ngx.exit(ngx.status) -end - -function _M.block_ip(ip, rule) - local ok, err = nil, nil - local msg = "拉黑IP : " .. ip .. "国家 " .. ngx.ctx.ip_location.country["zh"] - if rule then - msg = msg .. " 规则 " .. rule.type - end - ngx.log(ngx.ERR, msg) - - if config.is_redis_on() then - local red, err1 = redis_util.get_conn() - if not red then - return nil, err1 - end - local key = "black_ip:" .. ip - local exists = red:exists(key) - if exists == 0 then - ok, err = red:set(key, 1) - end - if rule.ipBlockTime > 0 then - ok, err = red:expire(key, rule.ipBlockTime) - end - if not ok then - ngx.log(ngx.ERR, "failed to expire redis key " .. key, err) - end - redis_util.close_conn(red) - else - local block_ip_dict = ngx.shared.waf_block_ip - local exists = block_ip_dict:get(ip) - if not exists then - ok, err = block_ip_dict:set(ip, 1, rule.ipBlockTime) - elseif rule.ipBlockTime > 0 then - ok, err = block_ip_dict:expire(ip, rule.ipBlockTime) - end - if not ok then - ngx.log(ngx.ERR, "failed to block ip " .. ip, err) - end - end - - return ok -end - -local function attack_count(config_type) - if config_type == "ipBlack" then - return - end - if config.is_global_state_on("attackCount") then - local ip = ngx.ctx.ip - local attack_config = config.get_global_config("attackCount") - local key = ip - - if config.is_redis_on() then - key = "cc_attack_count:" .. key - local count, _ = redis_util.incr(key, attack_config.duration) - if not count then - redis_util.set(key, 1, attack_config.duration) - elseif count >= attack_config.threshold then - _M.block_ip(ip, attack_config) - return - end - else - key = ip .. "attack" - local limit = ngx.shared.waf_limit - local count, _ = limit:incr(key, 1, 0, attack_config.duration) - - if not count then - limit:set(key, 1, attack_config.duration) - elseif count >= attack_config.threshold then - _M.block_ip(ip, attack_config) - return - end - end - end -end - -function _M.exec_action(rule_config, match_rule, data) - local action = rule_config.action - if match_rule then - rule_config.match_rule = match_rule - end - ngx.ctx.exec_rule = rule_config - ngx.ctx.hitData = data - ngx.ctx.is_attack = true - ngx.ctx.ip_blocked= false - - if rule_config.ipBlock and rule_config.ipBlock == 'on' then - if _M.block_ip(ngx.ctx.ip, rule_config) then - ngx.ctx.ip_blocked= true - end - end - - attack_count(rule_config.type) - - local msg = "访问 IP " .. ngx.ctx.ip .. " 访问 URL" .. ngx.var.uri .. " 触发动作 " .. action .. " 规则类型 " .. rule_config.type - if match_rule then - if match_rule.type then - msg = msg .. " 触发规则类型 " .. match_rule.type - else - msg = msg .. " 触发规则 " .. match_rule.rule - end - end - - ngx.log(ngx.ERR, msg) - if action == "allow" then - return - - elseif action == "deny" then - if rule_config.code and rule_config.res then - deny(rule_config.code, rule_config.res) - else - ngx.exit(403) - end - - elseif action == "slide" then - slide() - - elseif action == "fives" then - five_second() - - else - redirect(403) - end - -end - -return _M diff --git a/plugins/openresty/waf/lib/cc.lua b/plugins/openresty/waf/lib/cc.lua deleted file mode 100644 index 2c96e4d1b..000000000 --- a/plugins/openresty/waf/lib/cc.lua +++ /dev/null @@ -1,67 +0,0 @@ -local config = require "config" -local redis_util = require "redis_util" -local utils = require "utils" - -local _M = {} - -function _M.set_access_token(k, v) - local secret = config.get_secret() - local key = ngx.md5(ngx.ctx.ip .. ngx.var.server_name .. ngx.ctx.website_key - .. ngx.ctx.ua .. ngx.ctx.today .. secret) - local value = ngx.md5(ngx.time() .. ngx.ctx.ip) - --TODO check value - if key ~= k then - ngx.exit(444) - end - ngx.log(ngx.ERR, "set cc key: ", key) - if config.redis_on then - --local prefix = "ac_token:" - --redis_util.set(prefix .. accesstoken, accesstoken, timeout) - else - local limit = ngx.shared.waf_limit - limit:set(key, value, 7200) - end - - local cookie_expire = ngx.cookie_time(ngx.time() + 86400) - ngx.header['Set-Cookie'] = { key .. '=' .. value .. '; path=/; Expires=' .. cookie_expire } - ngx.exit(200) -end - -function _M.check_access_token() - local secret = config.get_secret() - local key = ngx.md5(ngx.ctx.ip .. ngx.var.server_name .. ngx.ctx.website_key - .. ngx.ctx.ua .. ngx.ctx.today .. secret) - if not ngx.var.http_cookie then - return false - end - local cookies = utils.get_cookie_list(ngx.var.http_cookie) - if not cookies then - return false - end - if not cookies[key] then - return false - end - local accesstoken = cookies[key] - local value = nil - - if config.redis_on then - local prefix = "ac_token:" - value = redis_util.get(prefix .. key) - if value and value == accesstoken then - return true - end - else - local limit = ngx.shared.waf_limit - value = limit:get(key) - end - if value and value == accesstoken then - return true - end - return false -end - -function _M.clear_access_token() - ngx.header['Set-Cookie'] = { 'a_token=; path=/; Expires=Thu, 01-Jan-1970 00:00:00 GMT' } -end - -return _M diff --git a/plugins/openresty/waf/lib/file.lua b/plugins/openresty/waf/lib/file.lua deleted file mode 100644 index 81eb1d324..000000000 --- a/plugins/openresty/waf/lib/file.lua +++ /dev/null @@ -1,122 +0,0 @@ -local cjson = require "cjson" -local pairs = pairs -local insert_table = table.insert -local lower_str = string.lower -local open_file = io.open -local gsub_str = string.gsub -local decode = cjson.decode - -local _M = {} - -function _M.read_rule(file_path, file_name, read_all) - local file, err = open_file(file_path .. file_name .. ".json", "r") - if not file then - ngx.log(ngx.ERR, "Failed to open file ", err) - return - end - - local rules_table = {} - local other_table = {} - local text = file:read('*a') - - file:close() - - if #text > 0 then - local result = decode(text) - - if result then - for key, value in pairs(result) do - if key == "rules" then - for _, r in pairs(value) do - if read_all then - r.hits = 0 - r.totalHits = 0 - insert_table(rules_table, r) - else - if lower_str(r.state) == 'on' then - r.hits = 0 - r.totalHits = 0 - insert_table(rules_table, r) - end - end - end - else - other_table[key] = value - end - end - end - end - - return rules_table, other_table -end - -function _M.read_file2table(file_path) - local file = open_file(file_path, 'r') - if file == nil then - return nil - end - local str = file:read("*a") - file:close() - return decode(str) -end - -function _M.read_list2table(filePath) - local file, err = open_file(filePath, "r") - if not file then - ngx.log(ngx.ERR, "Failed to open file ", err) - return - end - - local t = {} - for line in file:lines() do - line = gsub_str(line, "[\r\n]", "") - insert_table(t, line) - end - file:close() - return t -end - -function _M.set_content_to_file(data, file_path) - if data == nil or file_path == nil then - return - end - local file = open_file(file_path, "w") - if file then - file:write(data) - file:close() - end -end - - -function _M.read_file2string(file_path, binary) - if not file_path then - ngx.log(ngx.ERR, "No file found ", file_path) - return - end - - local mode = "r" - if binary == true then - mode = "rb" - end - - local file, err = open_file(file_path, mode) - if not file then - ngx.log(ngx.ERR, "Failed to open file ", err) - return - end - - local content = "" - repeat - local chunk = file:read(8192) -- 读取 8KB 的块 - if chunk then - content = content .. chunk - else - break - end - until not chunk - - file:close() - return content -end - -return _M \ No newline at end of file diff --git a/plugins/openresty/waf/lib/geoip.lua b/plugins/openresty/waf/lib/geoip.lua deleted file mode 100644 index 6f2edeb79..000000000 --- a/plugins/openresty/waf/lib/geoip.lua +++ /dev/null @@ -1,51 +0,0 @@ -local geo = require "resty.maxminddb" - -local pcall = pcall - -local _M = {} -local geo_ip_file = "/usr/local/openresty/1pwaf/data/GeoIP.mmdb" -local black_ip_file = "/usr/local/openresty/1pwaf/data/BlackIP.mmdb" - -function _M.init() - if not geo.initted() then - geo.init({ - geo_ip = geo_ip_file, - black_ip = black_ip_file - }) - end -end - -function _M.is_default_black_ip(ip) - local pass, res, err = pcall(geo.lookup, "black_ip", ip) - if not pass then - ngx.log(ngx.ERR, 'failed to lookup black ip,reason:', err) - elseif res and res['isBlack'] then - return true - end - return false -end - -function _M.lookup(ip) - local geo_res = { - iso = "", - country = "", - city = "", - longitude = 0, - latitude = 0, - province = "" - } - local pass, res, err = pcall(geo.lookup, "geo_ip", ip) - if not pass then - ngx.log(ngx.ERR, 'failed to lookup by ip,reason:', err) - elseif res and res['iso'] then - geo_res.iso = res['iso'] - geo_res.country = res['country'] - geo_res.province = res['province'] - geo_res.longitude = res['longitude'] - geo_res.latitude = res['latitude'] - return geo_res - end - return geo_res -end - -return _M diff --git a/plugins/openresty/waf/lib/lib.lua b/plugins/openresty/waf/lib/lib.lua deleted file mode 100644 index f1e8c7924..000000000 --- a/plugins/openresty/waf/lib/lib.lua +++ /dev/null @@ -1,720 +0,0 @@ -local redis_util = require "redis_util" -local action = require "action" -local cc = require "cc" -local file_utils = require "file" -local ck = require "resty.cookie" -local geo = require "geoip" -local libinjection = require "resty.libinjection" -local config = require "config" -local utils = require "utils" -local ipmatcher = require "resty.ipmatcher" -local cjson = require "cjson" - -local pairs = pairs -local ipairs = ipairs -local tostring = tostring -local type = type -local next = next -local concat_table = table.concat -local ngx_re_find = ngx.re.find -local ngx_re_gmatch = ngx.re.gmatch -local ngx_re_match = ngx.re.match -local ipv4_to_int = utils.ipv4_to_int -local is_ip_in_array = utils.is_ip_in_array -local is_ipv6 = utils.is_ipv6 - -local exec_action = action.exec_action - -local _M = {} - -local function is_global_state_on(name) - return config.is_global_state_on(name) -end - -local function is_site_state_on(name) - local site_config = config.get_site_config(ngx.ctx.website_key) - if site_config ~= nil then - return site_config[name]["state"] == "on" and true or false - end - return true -end - -local function is_state_on(name) - return is_site_state_on(name) and is_global_state_on(name) -end - -local function get_site_config(name) - local site_config = config.get_site_config(ngx.ctx.website_key) - if site_config ~= nil then - return site_config[name] - end - return config.get_global_config(name) -end - -local function get_site_rule(name) - local site_rules = config.get_site_rules(ngx.ctx.website_key) - if site_rules ~= nil then - return site_rules[name] - end - return config.get_global_rules(name) -end - -local function get_global_rules(name) - return config.get_global_rules(name) -end - -local function get_global_config(name) - return config.get_global_config(name) -end - -local function is_rule_state_on(rule_table) - return rule_table["state"] == "on" and true or false -end - -local function matches(input, regex, ctx, nth) - if not nth then - nth = 0 - end - return ngx_re_find(input, regex, "isjo", ctx, nth) -end - -local function match_rule(rule_table, str) - if str == nil or next(rule_table) == nil then - return false - end - for _, t in ipairs(rule_table) do - if matches(str, t.rule) then - return true, t - end - end - - return false -end - -local function match_ip(ip_rule, ip, ipn) - if ip_rule == nil or ip == nil then - return false - end - if is_rule_state_on(ip_rule) == false then - return false - end - local ip_rule_type = ip_rule.type - if is_ipv6(ip) and ip_rule_type == "ipv6" then - if ip == ip_rule.ipv6 then - return true - end - return false - end - - if ip_rule.type == "ipv4" then - if ipn == ipv4_to_int(ip_rule.ipv4) then - return true - end - - elseif ip_rule.type == "ipArr" then - local ip_start_n = ipv4_to_int(ip_rule.ipStart) - local ip_end_n = ipv4_to_int(ip_rule.ipEnd) - if is_ip_in_array(ipn, ip_start_n, ip_end_n) then - return true - end - - elseif ip_rule.type == "ipGroup" then - if ip_rule.ipGroup == nil or ip_rule.ipGroup == "" then - return false - end - local ip_group_list = cache:get("ip_group_list", { - ipc_shm = "ipc_shared_dict", - }) - if ip_group_list == nil then - return false - end - local ip_group = ip_group_list[ip_rule.ipGroup] - if ip_group == nil then - return false - end - local ip_matcher = ipmatcher.new(ip_group) - local ok = ip_matcher:match(ip) - if ok then - return true - end - end - - return false -end - -local function get_boundary() - local header = utils.get_headers()["content-type"] - if not header then - return nil - end - - if type(header) == "table" then - header = header[1] - end - - local m = ngx_re_match(header, ";%s*boundary=\"([^\"]+)\"") - if m then - return m - end - - return ngx_re_match(header, ";%s*boundary=([^\",;]+)") -end - - -local function xss_and_sql_check(kv) - if type(kv) ~= 'string' then - return - end - if is_site_state_on("xss") then - local is_xss, fingerprint = libinjection.xss(tostring(kv)) - local xss_config = get_site_config("xss") - if is_xss then - exec_action(xss_config, { rule = kv }) - return - end - end - if is_site_state_on("sql") then - local is_sqli, fingerprint = libinjection.sqli(tostring(kv)) - local sql_config = get_site_config("sql") - if is_sqli then - exec_action(sql_config, { rule = kv }) - return - end - end -end - - -local function get_request_body() - ngx.req.read_body() - local body_data = ngx.req.get_body_data() - if not body_data then - local body_file = ngx.req.get_body_file() - if body_file then - body_data = file_utils.read_file2string(body_file, true) - end - end - return body_data -end - -function _M.is_white_ip() - if is_global_state_on("ipWhite") then - local ip = ngx.ctx.ip - if ip == "unknown" then - return false - end - if ip == "127.0.0.1" then - return true - end - local ipn = utils.ipv4_to_int(ip) - local ip_rules = get_global_rules("ipWhite") - for _, ip_rule in pairs(ip_rules) do - if match_ip(ip_rule, ip, ipn) then - return true - end - end - end - return false -end - -function _M.allow_location_check() - if is_state_on("geoRestrict") then - local ip_location = ngx.ctx.ip_location - if ip_location and ip_location.iso and ip_location.iso ~= "" then - local iso = ip_location.iso - local geo_config = get_site_config("geoRestrict") - local exist = false - for _, rule in ipairs(geo_config.rules) do - if iso == rule then - exist = true - break - - end - end - local default_geo_config = { - action = "deny", - code = 444, - type = "geoRestrict", - state = "on", - rule = iso - } - if exist then - if geo_config.action == "allow" then - return true - end - if geo_config.action == "deny" then - exec_action(default_geo_config, default_geo_config) - return false - end - else - if geo_config.action == "allow" then - exec_action(default_geo_config, default_geo_config) - return false - end - end - end - end -end - -function _M.default_ip_black() - if is_state_on("defaultIpBlack") then - if geo.is_default_black_ip(ngx.ctx.ip) then - exec_action(get_site_config("defaultIpBlack"), { rule = ngx.ctx.ip }) - end - end -end - -function _M.black_ip() - local ip = ngx.ctx.ip - if ip == "unknown" then - return false - end - local exists = false - if config.is_redis_on() then - exists = redis_util.get("black_ip:" .. ip) - else - local block_ip_dict = ngx.shared.waf_block_ip - exists = block_ip_dict:get(ip) - end - - if exists then - ngx.exit(444) - return true - end - - if is_global_state_on("ipBlack") then - local ip_black_list = get_global_rules("ipBlack") - local ipn = ipv4_to_int(ip) - for _, ip_rule in pairs(ip_black_list) do - if match_ip(ip_rule, ip, ipn) then - exists = true - break - end - end - - if exists then - exec_action(get_global_config("ipBlack")) - end - end -end - -function _M.method_check() - local method = ngx.req.get_method() - local method_white_list = get_site_rule("methodWhite") - for _, method_rule in ipairs(method_white_list) do - if method_rule.rule == method and method_rule.state == 'off' then - local method_config = get_global_config("methodWhite") - exec_action(method_config, method_rule) - return false - end - end - return true -end - -function _M.bot_check() - if is_state_on("bot") then - local ruri = ngx.var.request_uri - local uri = ngx.var.uri - local bot_rule = get_site_config("bot") - if uri == bot_rule.uri or ruri == bot_rule.uri then - exec_action(bot_rule) - end - end -end - -function _M.black_ua() - if is_global_state_on("uaBlack") then - if type(ngx.ctx.ua) ~= 'string' then - ngx.exit(200) - end - local m, mr = match_rule(get_global_rules("uaBlack"), ngx.ctx.ua) - if m then - exec_action(get_global_config("uaBlack"), mr) - end - end -end - -function _M.default_ua_black() - if is_state_on("defaultUaBlack") then - if type(ngx.ctx.ua) ~= 'string' then - ngx.exit(200) - end - local m, mr = match_rule(get_global_rules('defaultUaBlack'), ngx.ctx.ua) - if m then - exec_action(get_global_config('defaultUaBlack'), mr) - end - end -end - -function _M.is_white_ua() - if is_global_state_on("uaWhite") then - local ua = utils.get_header("user-agent") - if not ua then - return false - end - if type(ua) ~= 'string' then - ngx.exit(200) - end - for _, wa in ipairs(get_global_rules("uaWhite")) do - if ngx.ctx.ua == wa then - return true - end - end - end - return false -end - -function _M.cc() - if is_state_on("cc") then - if cc.check_access_token() then - return - end - local ip = ngx.ctx.ip - local cc_config = get_site_config("cc") - local key = ip - - if config.is_redis_on() then - key = "cc_req_count:" .. key - local exist = redis_util.get(key) - if exist then - ngx.exit(444) - return - end - local count, _ = redis_util.incr(key, cc_config.duration) - if not count then - redis_util.set(key, 1, cc_config.duration) - elseif count >= cc_config.threshold then - exec_action(cc_config) - return - end - else - local block_ip_dict = ngx.shared.waf_block_ip - local exists = block_ip_dict:get(ip) - if exists then - ngx.exit(444) - end - local limit = ngx.shared.waf_limit - local count, _ = limit:incr(key, 1, 0, cc_config.duration) - if not count then - limit:set(key, 1, cc_config.duration) - elseif count >= cc_config.threshold then - exec_action(cc_config) - return - end - end - end -end - -function _M.cc_url() - if is_state_on("ccurl") then - local ip = ngx.ctx.ip - local key = ip - local urlcc_rules = get_site_rule("ccurl") - local urlcc_config = get_site_config("ccurl") - local uri = ngx.var.uri - - local m, mr = match_rule(urlcc_rules, uri) - if not m or not mr then - return - end - key = uri .. key - if config.is_redis_on() then - key = "url_cc_req_count:" .. key - local count, _ = redis_util.incr(key, mr.duration) - if not count then - redis_util.set(key, 1, mr.duration) - elseif count > mr.threshold then - exec_action(urlcc_config, { rule = mr.rule }) - return - end - else - local limit = ngx.shared.waf_limit - local count, _ = limit:incr(key, 1, 0, mr.duration) - if not count then - limit:set(key, 1, urlcc_config.duration) - elseif count > mr.threshold then - exec_action(urlcc_config, { rule = mr.rule }) - return - end - end - end -end - -function _M.is_white_url() - if is_global_state_on("urlWhite") then - local url = ngx.var.uri - if url == nil or url == " " then - return false - end - local m, _ = match_rule(get_global_rules("urlWhite"), url) - if m then - return true - end - return false - end - - return false -end - -function _M.black_url() - if is_global_state_on("urlBlack") then - local url = ngx.var.uri - if url == nil or url == "" then - return false - end - local m, mr = match_rule(get_global_rules("urlBlack"), url) - if m then - exec_action(get_global_config("urlBlack"), mr) - return - end - end -end - -function _M.default_url_black() - if is_state_on("defaultUrlBlack") then - local url = ngx.var.uri - if url == nil or url == "" then - return false - end - local m, mr = match_rule(get_global_rules('defaultUrlBlack'), url) - if m then - exec_action(get_global_config('defaultUrlBlack'), mr) - return - end - end -end - -function _M.args_check() - if is_state_on("args") then - local args = ngx.req.get_uri_args() - if args then - local args_list = get_global_rules("args") - for _, val in pairs(args) do - local val_arr = val - if type(val) == "table" then - val_arr = concat_table(val, ", ") - end - if val_arr and type(val_arr) ~= "boolean" and val_arr ~= "" then - local check_value = utils.unescape_uri(val_arr) - xss_and_sql_check(check_value) - local m, mr = match_rule(args_list,check_value) - if m then - exec_action(get_global_config("args"), mr) - return - end - end - end - end - end -end - -function _M.cookie_check() - local cookie = ngx.var.http_cookie - if cookie and is_state_on("cookie") then - local cookieList = get_site_rule('cookie') - local m, mr = match_rule(cookieList, cookie) - if m then - exec_action(get_global_config('cookie'), mr) - return true - end - end - return false -end - -function _M.header_check() - if is_state_on("header") then - local headers_rule = get_site_rule("header") - local headers_config = get_site_config("header") - local referer = ngx.var.http_referer - if referer and referer ~= "" then - local check_value = utils.unescape_uri(referer) - local m = match_rule(headers_rule, check_value) - if m then - exec_action(headers_config) - end - xss_and_sql_check(check_value) - end - local headers = utils.get_headers() - if headers then - for _, v in pairs(headers) do - local m, mr = match_rule(headers_rule, v) - if m then - exec_action(headers_config, mr) - end - end - end - end -end - -function _M.post_check() - local content_type = ngx.ctx.content_type - local content_length = ngx.ctx.content_length - - if ngx.ctx.method == "GET" or not content_type or type(content_type) ~= 'string' then - return - end - - if content_length == nil or content_length == 0 then - return - end - - local boundary = get_boundary() - - if boundary and is_state_on('fileExt') then - if not ngx_re_match(content_type, '^multipart/form-data; boundary=') or not ngx_re_find(content_type, [[multipart]], 'ijo')then - return - end - local boundary_value = ngx_re_match(content_type, '^multipart/form-data; boundary=(.+)') - if boundary_value == nil then - return - end - local data = get_request_body() - if data == nil then - return - end - local iterator = ngx_re_gmatch(data, [[Content-Disposition.+filename=.+]], 'ijo') - if not iterator then - return - end - - local rule = get_site_rule("fileExt") - while true do - local m = iterator() - if m then - local match = ngx_re_match(m[0], 'Content-Disposition: form-data; (.+)filename="(.+)\\.(.*)"', 'ijo') - if match then - local extension = match[3] - for _, ext in ipairs(rule.rules) do - if extension == ext then - exec_action(rule) - end - end - end - else - break - end - end - else - ngx.req.read_body() - local body_obj = ngx.req.get_post_args() - if not body_obj then - return - end - - for key, val in pairs(body_obj) do - if is_global_state_on("xss") or is_global_state_on("sql") then - xss_and_sql_check(key) - xss_and_sql_check(val) - end - if is_state_on("args") then - local post_rules = get_global_rules("args") - local m, mr = match_rule(post_rules, val) - if m then - exec_action(get_global_config("args"), mr) - return - end - end - end - end - -end - -local function match_acl_rule(match_value, pattern,rule) - if pattern == "eq" then - if match_value == rule then - return true - end - - elseif pattern == "notEq" then - if match_value ~= rule then - return true - end - - elseif pattern == "regex" then - if matches(match_value, rule) then - return true - end - - elseif pattern == "contain" then - if ngx_re_find(match_value, rule, "isjo") then - return true - end - end -end - -function _M.acl() - local rules = get_site_rule("acl") - for _, rule in pairs(rules) do - if rule.state == nil or rule.state == "off" then - goto continue - end - local conditions = rule.conditions - local match = true - local condition_rule = "" - for _, condition in pairs(conditions) do - local field = condition.field - local field_name = condition.name - local pattern = condition.pattern - condition_rule = condition.rule - local match_value = '' - if field == 'URL' then - match_value = ngx.var.request_uri - - elseif field == 'Cookie' then - if field_name ~= nil and field_name ~= '' then - local cookies, _ = ck:new() - if not cookies then - match = false - break - else - match_value, _ = cookies:get(field_name) - end - else - match_value = ngx.var.http_cookie - end - - elseif field == 'Header' then - local headers = ngx.req.get_headers() - if headers then - if field_name ~= nil and field_name ~= '' then - match_value = headers[field_name] - else - match_value = concat_table(headers, '') - end - else - match = false - break - end - - elseif field == 'Referer' then - match_value = ngx.var.http_referer - - elseif field == 'User-Agent' then - match_value = ngx.var.http_user_agent - - elseif field == 'IP' then - match_value = ngx.ctx.ip - end - - if pattern == '' then - match = false - break - end - - if not match_acl_rule(match_value, pattern,condition_rule) then - match = false - break - end - end - if match then - rule.type = "acl" - local mr = { - type = rule.name, - rule = condition_rule - } - exec_action(rule,mr) - end - :: continue :: - end -end - -return _M diff --git a/plugins/openresty/waf/lib/logger.lua b/plugins/openresty/waf/lib/logger.lua deleted file mode 100644 index c79f05448..000000000 --- a/plugins/openresty/waf/lib/logger.lua +++ /dev/null @@ -1,148 +0,0 @@ -local type = type -local concat_table = table.concat -local new_table = table.new -local tostring = tostring -local setmetatable = setmetatable -local open_file = io.open -local ngx_timer_at = ngx.timer.at - -local _M = {} - -local mt = { __index = _M } - -function _M:new(log_path, host, rolling) - local t = { - flush_limit = 4096, -- 4kb - flush_timeout = 1, - - buffered_size = 0, - buffer_index = 0, - buffer_data = new_table(20000, 0), - - log_path = log_path, - prefix = log_path .. host .. '_', - rolling = rolling or false, - host = host, - timer = nil } - - setmetatable(t, mt) - return t -end - -local function needFlush(self) - if self.buffered_size > 0 then - return true - end - - return false -end - -local function flush_lock(self) - local dic_lock = ngx.shared.dict_locks - local locked = dic_lock:get(self.host) - if not locked then - local succ, err = dic_lock:set(self.host, true) - if not succ then - ngx.log(ngx.ERR, "failed to lock logfile " .. self.host .. ": ", err) - end - return succ - end - return false -end - -local function flush_unlock(self) - local dic_lock = ngx.shared.dict_locks - local success, err = dic_lock:set(self.host, false) - if not success then - ngx.log(ngx.ERR, "failed to unlock logfile " .. self.host .. ": ", err) - end - return success -end - -local function write_file(self, value) - local file_name = '' - if self.rolling then - file_name = self.prefix .. ngx.today() .. ".log" - else - file_name = self.log_path - end - - local file = open_file(file_name, "a+") - - if file == nil or value == nil then - return - end - - file:write(value) - file:flush() - file:close() - - return -end - -local function flushBuffer(self) - if not needFlush(self) then - return true - end - - if not flush_lock(self) then - return true - end - - local buffer = concat_table(self.buffer_data, "", 1, self.buffer_index) - write_file(self, buffer) - - self.buffered_size = 0 - self.buffer_index = 0 - self.buffer_data = new_table(20000, 0) - - flush_unlock(self) -end - -local function flushPeriod(premature, self) - flushBuffer(self) - self.timer = false -end - -local function writeBuffer(self, msg, msg_len) - self.buffer_index = self.buffer_index + 1 - - self.buffer_data[self.buffer_index] = msg - - self.buffered_size = self.buffered_size + msg_len - - return self.buffered_size -end - -local function startTimer(self) - if not self.timer then - local ok, err = ngx_timer_at(self.flush_timeout, flushPeriod, self) - if not ok then - ngx.log(ngx.ERR, "failed to create the timer: ", err) - return - end - if ok then - self.timer = true - end - end - return self.timer -end - -function _M:log(msg) - if type(msg) ~= "string" then - msg = tostring(msg) - end - - local msg_len = #msg - local len = msg_len + self.buffered_size - - if len < self.flush_limit then - writeBuffer(self, msg, msg_len) - startTimer(self) - elseif len >= self.flush_limit then - writeBuffer(self, msg, msg_len) - flushBuffer(self) - end -end - -return _M \ No newline at end of file diff --git a/plugins/openresty/waf/lib/logger_factory.lua b/plugins/openresty/waf/lib/logger_factory.lua deleted file mode 100644 index 506bb08e8..000000000 --- a/plugins/openresty/waf/lib/logger_factory.lua +++ /dev/null @@ -1,16 +0,0 @@ -local logger = require "logger" - -local loggers = {} - -local _M = {} - -function _M.get_logger(log_path, host, rolling) - local host_logger = loggers[host] - if not host_logger then - host_logger = logger:new(log_path, host, rolling) - loggers[host] = host_logger - end - return host_logger -end - -return _M \ No newline at end of file diff --git a/plugins/openresty/waf/lib/redis_util.lua b/plugins/openresty/waf/lib/redis_util.lua deleted file mode 100644 index e260e7422..000000000 --- a/plugins/openresty/waf/lib/redis_util.lua +++ /dev/null @@ -1,131 +0,0 @@ -local redis = require "resty.redis" -local config = require "config" - -local _M = {} - -local connect_timeout, send_timeout, read_timeout = 1000, 1000, 1000 - -function _M.get_conn() - local red, err1 = redis:new() - if not red then - ngx.log(ngx.ERR, "failed to new redis:", err1) - return nil, err1 - end - - local redis_config = config.get_redis_config() - - red:set_timeouts(connect_timeout, send_timeout, read_timeout) - - local ok, err = red:connect(redis_config.host, redis_config.port, { ssl = redis_config.ssl, pool_size = redis_config.poolSize }) - - if not ok then - ngx.log(ngx.ERR, "failed to connect: ", err .. "\n") - return nil, err - end - - if redis_config.password ~= nil and #redis_config.password ~= 0 then - local times = 0 - times, err = red:get_reused_times() - - if times == 0 then - local res, err2 = red:auth(redis_config.password) - if not res then - ngx.log(ngx.ERR, "failed to authenticate: ", err2) - return nil, err2 - end - end - end - - return red, err -end - -function _M.close_conn(red) - local ok, err = red:set_keepalive(10000, 100) - if not ok then - ngx.log(ngx.ERR, "failed to set keepalive: ", err) - end - - return ok, err -end - -function _M.set(key, value, expire_time) - local red, _ = _M.get_conn() - local ok, err = nil, nil - if red then - ok, err = red:set(key, value) - if not ok then - ngx.log(ngx.ERR, "failed to set key: " .. key .. " ", err) - elseif expire_time and expire_time > 0 then - red:expire(key, expire_time) - end - - _M.close_conn(red) - end - - return ok, err -end - -function _M.bath_set(keyTable, value, keyPrefix) - local red, _ = _M.get_conn() - local results, err = nil, nil - if red then - red:init_pipeline() - - if keyPrefix then - for _, ip in ipairs(keyTable) do - red:set(keyPrefix .. ip, value) - end - else - for _, ip in ipairs(keyTable) do - red:set(ip, value) - end - end - - results, err = red:commit_pipeline() - if not results then - ngx.log(ngx.ERR, "failed to set keys: ", err) - end - - _M.close_conn(red) - end - - return results, err -end - -function _M.get(key) - local red, err = _M.get_conn() - local value = nil - if red then - value, err = red:get(key) - if not value then - ngx.log(ngx.ERR, "failed to get key: " .. key, err) - return value, err - end - if value == ngx.null then - value = nil - end - - _M.close_conn(red) - end - - return value, err -end - -function _M.incr(key, expire_time) - local red, err = _M.get_conn() - local res = nil - if red then - res, err = red:incr(key) - if not res then - ngx.log(ngx.ERR, "failed to incr key: " .. key, err) - elseif res == 1 and expire_time and expire_time > 0 then - red:expire(key, expire_time) - end - - _M.close_conn(red) - end - - return res, err -end - -return _M diff --git a/plugins/openresty/waf/lib/resty/cookie.lua b/plugins/openresty/waf/lib/resty/cookie.lua deleted file mode 100644 index f4c81ab0f..000000000 --- a/plugins/openresty/waf/lib/resty/cookie.lua +++ /dev/null @@ -1,213 +0,0 @@ --- Copyright (C) 2013-2016 Jiale Zhi (calio), CloudFlare Inc. --- See RFC6265 http://tools.ietf.org/search/rfc6265 --- require "luacov" - -local type = type -local byte = string.byte -local sub = string.sub -local format = string.format -local log = ngx.log -local ERR = ngx.ERR -local WARN = ngx.WARN -local ngx_header = ngx.header - -local EQUAL = byte("=") -local SEMICOLON = byte(";") -local SPACE = byte(" ") -local HTAB = byte("\t") - --- table.new(narr, nrec) -local ok, new_tab = pcall(require, "table.new") -if not ok then - new_tab = function () return {} end -end - -local ok, clear_tab = pcall(require, "table.clear") -if not ok then - clear_tab = function(tab) for k, _ in pairs(tab) do tab[k] = nil end end -end - -local _M = new_tab(0, 2) - -_M._VERSION = '0.01' - - -local function get_cookie_table(text_cookie) - if type(text_cookie) ~= "string" then - log(ERR, format("expect text_cookie to be \"string\" but found %s", - type(text_cookie))) - return {} - end - - local EXPECT_KEY = 1 - local EXPECT_VALUE = 2 - local EXPECT_SP = 3 - - local n = 0 - local len = #text_cookie - - for i=1, len do - if byte(text_cookie, i) == SEMICOLON then - n = n + 1 - end - end - - local cookie_table = new_tab(0, n + 1) - - local state = EXPECT_SP - local i = 1 - local j = 1 - local key, value - - while j <= len do - if state == EXPECT_KEY then - if byte(text_cookie, j) == EQUAL then - key = sub(text_cookie, i, j - 1) - state = EXPECT_VALUE - i = j + 1 - end - elseif state == EXPECT_VALUE then - if byte(text_cookie, j) == SEMICOLON - or byte(text_cookie, j) == SPACE - or byte(text_cookie, j) == HTAB - then - value = sub(text_cookie, i, j - 1) - cookie_table[key] = value - - key, value = nil, nil - state = EXPECT_SP - i = j + 1 - end - elseif state == EXPECT_SP then - if byte(text_cookie, j) ~= SPACE - and byte(text_cookie, j) ~= HTAB - then - state = EXPECT_KEY - i = j - j = j - 1 - end - end - j = j + 1 - end - - if key ~= nil and value == nil then - cookie_table[key] = sub(text_cookie, i) - end - - return cookie_table -end - -function _M.new(self) - local _cookie = ngx.var.http_cookie - --if not _cookie then - --return nil, "no cookie found in current request" - --end - return setmetatable({ _cookie = _cookie, set_cookie_table = new_tab(4, 0) }, - { __index = self }) -end - -function _M.get(self, key) - if not self._cookie then - return nil, "no cookie found in the current request" - end - if self.cookie_table == nil then - self.cookie_table = get_cookie_table(self._cookie) - end - - return self.cookie_table[key] -end - -function _M.get_all(self) - if not self._cookie then - return nil, "no cookie found in the current request" - end - - if self.cookie_table == nil then - self.cookie_table = get_cookie_table(self._cookie) - end - - return self.cookie_table -end - -function _M.get_cookie_size(self) - if not self._cookie then - return 0 - end - - return string.len(self._cookie) -end - -local function bake(cookie) - if not cookie.key or not cookie.value then - return nil, 'missing cookie field "key" or "value"' - end - - if cookie["max-age"] then - cookie.max_age = cookie["max-age"] - end - - if (cookie.samesite) then - local samesite = cookie.samesite - - -- if we don't have a valid-looking attribute, ignore the attribute - if (samesite ~= "Strict" and samesite ~= "Lax" and samesite ~= "None") then - log(WARN, "SameSite value must be 'Strict', 'Lax' or 'None'") - cookie.samesite = nil - end - end - - local str = cookie.key .. "=" .. cookie.value - .. (cookie.expires and "; Expires=" .. cookie.expires or "") - .. (cookie.max_age and "; Max-Age=" .. cookie.max_age or "") - .. (cookie.domain and "; Domain=" .. cookie.domain or "") - .. (cookie.path and "; Path=" .. cookie.path or "") - .. (cookie.secure and "; Secure" or "") - .. (cookie.httponly and "; HttpOnly" or "") - .. (cookie.samesite and "; SameSite=" .. cookie.samesite or "") - .. (cookie.extension and "; " .. cookie.extension or "") - return str -end - -function _M.set(self, cookie) - local cookie_str, err = bake(cookie) - if not cookie_str then - return nil, err - end - - local set_cookie = ngx_header['Set-Cookie'] - local set_cookie_type = type(set_cookie) - local t = self.set_cookie_table - clear_tab(t) - - if set_cookie_type == "string" then - -- only one cookie has been setted - if set_cookie ~= cookie_str then - t[1] = set_cookie - t[2] = cookie_str - ngx_header['Set-Cookie'] = t - end - elseif set_cookie_type == "table" then - -- more than one cookies has been setted - local size = #set_cookie - - -- we can not set cookie like ngx.header['Set-Cookie'][3] = val - -- so create a new table, copy all the values, and then set it back - for i=1, size do - t[i] = ngx_header['Set-Cookie'][i] - if t[i] == cookie_str then - -- new cookie is duplicated - return true - end - end - t[size + 1] = cookie_str - ngx_header['Set-Cookie'] = t - else - -- no cookie has been setted - ngx_header['Set-Cookie'] = cookie_str - end - return true -end - -_M.get_cookie_string = bake - -return _M diff --git a/plugins/openresty/waf/lib/resty/ipmatcher.lua b/plugins/openresty/waf/lib/resty/ipmatcher.lua deleted file mode 100644 index 880d78d0a..000000000 --- a/plugins/openresty/waf/lib/resty/ipmatcher.lua +++ /dev/null @@ -1,407 +0,0 @@ -local base = require("resty.core.base") -local bit = require("bit") -local clear_tab = require("table.clear") -local nkeys = require("table.nkeys") -local new_tab = base.new_tab -local find_str = string.find -local tonumber = tonumber -local ipairs = ipairs -local pairs = pairs -local ffi = require "ffi" -local ffi_cdef = ffi.cdef -local ffi_copy = ffi.copy -local ffi_new = ffi.new -local C = ffi.C -local insert_tab = table.insert -local sort_tab = table.sort -local string = string -local setmetatable=setmetatable -local type = type -local error = error -local str_sub = string.sub -local str_byte = string.byte -local cur_level = ngx.config.subsystem == "http" and - require "ngx.errlog" .get_sys_filter_level() - -local AF_INET = 2 -local AF_INET6 = 10 -if ffi.os == "OSX" then - AF_INET6 = 30 -elseif ffi.os == "BSD" then - AF_INET6 = 28 -elseif ffi.os == "Windows" then - AF_INET6 = 23 -end - - -local _M = {_VERSION = 0.3} - - -ffi_cdef[[ - int inet_pton(int af, const char * restrict src, void * restrict dst); - uint32_t ntohl(uint32_t netlong); -]] - - -local parse_ipv4 -do - local inet = ffi_new("unsigned int [1]") - - function parse_ipv4(ip) - if not ip then - return false - end - - if C.inet_pton(AF_INET, ip, inet) ~= 1 then - return false - end - - return C.ntohl(inet[0]) - end -end -_M.parse_ipv4 = parse_ipv4 - -local parse_bin_ipv4 -do - local inet = ffi_new("unsigned int [1]") - - function parse_bin_ipv4(ip) - if not ip or #ip ~= 4 then - return false - end - - ffi_copy(inet, ip, 4) - return C.ntohl(inet[0]) - end -end - -local parse_ipv6 -do - local inets = ffi_new("unsigned int [4]") - - function parse_ipv6(ip) - if not ip then - return false - end - - if str_byte(ip, 1, 1) == str_byte('[') - and str_byte(ip, #ip) == str_byte(']') then - - -- strip square brackets around IPv6 literal if present - ip = str_sub(ip, 2, #ip - 1) - end - - if C.inet_pton(AF_INET6, ip, inets) ~= 1 then - return false - end - - local inets_arr = new_tab(4, 0) - for i = 0, 3 do - insert_tab(inets_arr, C.ntohl(inets[i])) - end - return inets_arr - end -end -_M.parse_ipv6 = parse_ipv6 - -local parse_bin_ipv6 -do - local inets = ffi_new("unsigned int [4]") - - function parse_bin_ipv6(ip) - if not ip or #ip ~= 16 then - return false - end - - ffi_copy(inets, ip, 16) - local inets_arr = new_tab(4, 0) - for i = 0, 3 do - insert_tab(inets_arr, C.ntohl(inets[i])) - end - return inets_arr - end -end - - -local mt = {__index = _M} - - -local ngx_log = ngx.log -local ngx_INFO = ngx.INFO -local function log_info(...) - if cur_level and ngx_INFO > cur_level then - return - end - - return ngx_log(ngx_INFO, ...) -end - - -local function split_ip(ip_addr_org) - local idx = find_str(ip_addr_org, "/", 1, true) - if not idx then - return ip_addr_org - end - - local ip_addr = str_sub(ip_addr_org, 1, idx - 1) - local ip_addr_mask = str_sub(ip_addr_org, idx + 1) - return ip_addr, tonumber(ip_addr_mask) -end -_M.split_ip = split_ip - - -local idxs = {} -local function gen_ipv6_idxs(inets_ipv6, mask) - clear_tab(idxs) - - for _, inet in ipairs(inets_ipv6) do - local valid_mask = mask - if valid_mask > 32 then - valid_mask = 32 - end - - if valid_mask == 32 then - insert_tab(idxs, inet) - else - insert_tab(idxs, bit.rshift(inet, 32 - valid_mask)) - end - - mask = mask - 32 - if mask <= 0 then - break - end - end - - return idxs -end - - -local function cmp(x, y) - return x > y -end - - -local function new(ips, with_value) - if not ips or type(ips) ~= "table" then - error("missing valid ip argument", 2) - end - - local parsed_ipv4s = {} - local parsed_ipv4s_mask = {} - local ipv4_match_all_value - - local parsed_ipv6s = {} - local parsed_ipv6s_mask = {} - local ipv6_values = {} - local ipv6s_values_idx = 1 - local ipv6_match_all_value - - local iter = with_value and pairs or ipairs - for a, b in iter(ips) do - local ip_addr_org, value - if with_value then - ip_addr_org = a - value = b - - else - ip_addr_org = b - value = true - end - - local ip_addr, ip_addr_mask = split_ip(ip_addr_org) - - local inet_ipv4 = parse_ipv4(ip_addr) - if inet_ipv4 then - ip_addr_mask = ip_addr_mask or 32 - if ip_addr_mask == 32 then - parsed_ipv4s[inet_ipv4] = value - - elseif ip_addr_mask == 0 then - ipv4_match_all_value = value - - else - local valid_inet_addr = bit.rshift(inet_ipv4, 32 - ip_addr_mask) - - parsed_ipv4s_mask[ip_addr_mask] = parsed_ipv4s_mask[ip_addr_mask] or {} - parsed_ipv4s_mask[ip_addr_mask][valid_inet_addr] = value - log_info("ipv4 mask: ", ip_addr_mask, - " valid inet: ", valid_inet_addr) - end - - goto continue - end - - local inets_ipv6 = parse_ipv6(ip_addr) - if inets_ipv6 then - ip_addr_mask = ip_addr_mask or 128 - if ip_addr_mask == 128 then - parsed_ipv6s[ip_addr] = value - - elseif ip_addr_mask == 0 then - ipv6_match_all_value = value - end - - parsed_ipv6s[ip_addr_mask] = parsed_ipv6s[ip_addr_mask] or {} - - local inets_idxs = gen_ipv6_idxs(inets_ipv6, ip_addr_mask) - local node = parsed_ipv6s[ip_addr_mask] - for i, inet in ipairs(inets_idxs) do - if i == #inets_idxs then - if with_value then - ipv6_values[ipv6s_values_idx] = value - node[inet] = ipv6s_values_idx - ipv6s_values_idx = ipv6s_values_idx + 1 - else - node[inet] = true - end - end - node[inet] = node[inet] or {} - node = node[inet] - end - - parsed_ipv6s_mask[ip_addr_mask] = true - - goto continue - end - - if not inet_ipv4 and not inets_ipv6 then - return nil, "invalid ip address: " .. ip_addr - end - - ::continue:: - end - - local ipv4_mask_arr = new_tab(nkeys(parsed_ipv4s_mask), 0) - local i = 1 - for k, _ in pairs(parsed_ipv4s_mask) do - ipv4_mask_arr[i] = k - i = i + 1 - end - - sort_tab(ipv4_mask_arr, cmp) - - local ipv6_mask_arr = new_tab(nkeys(parsed_ipv6s_mask), 0) - - i = 1 - for k, _ in pairs(parsed_ipv6s_mask) do - ipv6_mask_arr[i] = k - i = i + 1 - end - - sort_tab(ipv6_mask_arr, cmp) - - return setmetatable({ - ipv4 = parsed_ipv4s, - ipv4_mask = parsed_ipv4s_mask, - ipv4_mask_arr = ipv4_mask_arr, - ipv4_match_all_value = ipv4_match_all_value, - - ipv6 = parsed_ipv6s, - ipv6_mask = parsed_ipv6s_mask, - ipv6_mask_arr = ipv6_mask_arr, - ipv6_values = ipv6_values, - ipv6_match_all_value = ipv6_match_all_value, - }, mt) -end - -function _M.new(ips) - return new(ips, false) -end - -function _M.new_with_value(ips) - return new(ips, true) -end - - -local function match_ipv4(self, ip) - local ipv4s = self.ipv4 - local value = ipv4s[ip] - if value ~= nil then - return value - end - - local ipv4_mask = self.ipv4_mask - if self.ipv4_match_all_value ~= nil then - return self.ipv4_match_all_value -- match any ip - end - - for _, mask in ipairs(self.ipv4_mask_arr) do - local valid_inet_addr = bit.rshift(ip, 32 - mask) - - log_info("ipv4 mask: ", mask, - " valid inet: ", valid_inet_addr) - - value = ipv4_mask[mask][valid_inet_addr] - if value ~= nil then - return value - end - end - - return false -end - -local function match_ipv6(self, ip) - local ipv6s = self.ipv6 - if self.ipv6_match_all_value ~= nil then - return self.ipv6_match_all_value -- match any ip - end - - for _, mask in ipairs(self.ipv6_mask_arr) do - local node = ipv6s[mask] - local inet_idxs = gen_ipv6_idxs(ip, mask) - for _, inet in ipairs(inet_idxs) do - if not node[inet] then - break - else - node = node[inet] - if node == true then - return true - end - if type(node) == "number" then - -- fetch with the ipv6s_values_idx - return self.ipv6_values[node] - end - end - end - end - - return false -end - -function _M.match(self, ip) - local inet_ipv4 = parse_ipv4(ip) - if inet_ipv4 then - return match_ipv4(self, inet_ipv4) - end - - local inets_ipv6 = parse_ipv6(ip) - if not inets_ipv6 then - return false, "invalid ip address, not ipv4 and ipv6" - end - - local ipv6s = self.ipv6 - local value = ipv6s[ip] - if value ~= nil then - return value - end - - return match_ipv6(self, inets_ipv6) -end - - -function _M.match_bin(self, bin_ip) - local inet_ipv4 = parse_bin_ipv4(bin_ip) - if inet_ipv4 then - return match_ipv4(self, inet_ipv4) - end - - local inets_ipv6 = parse_bin_ipv6(bin_ip) - if not inets_ipv6 then - return false, "invalid ip address, not ipv4 and ipv6" - end - - return match_ipv6(self, inets_ipv6) -end - - -return _M \ No newline at end of file diff --git a/plugins/openresty/waf/lib/resty/libinjection.lua b/plugins/openresty/waf/lib/resty/libinjection.lua deleted file mode 100644 index 587604ae3..000000000 --- a/plugins/openresty/waf/lib/resty/libinjection.lua +++ /dev/null @@ -1,326 +0,0 @@ -local _M = {} - -local bit = require "bit" -local ffi = require "ffi" - -local ffi_new = ffi.new -local ffi_string = ffi.string - --- enum sqli_flags -local FLAG_NONE = 0 -local FLAG_QUOTE_NONE = 1 -local FLAG_QUOTE_SINGLE = 2 -local FLAG_QUOTE_DOUBLE = 4 -local FLAG_SQL_ANSI = 8 -local FLAG_SQL_MYSQL = 16 - --- enum lookup_type -local LOOKUP_FINGERPRINT = 4 - --- enum html5_flags -local DATA_STATE = 0 -local VALUE_NO_QUOTE = 1 -local VALUE_SINGLE_QUOTE = 2 -local VALUE_DOUBLE_QUOTE = 3 -local VALUE_BACK_QUOTE = 4 - --- cached ORs -local QUOTE_NONE_SQL_ANSI = bit.bor(FLAG_QUOTE_NONE, FLAG_SQL_ANSI) -local QUOTE_NONE_SQL_MYSQL = bit.bor(FLAG_QUOTE_NONE, FLAG_SQL_MYSQL) -local QUOTE_SINGLE_SQL_ANSI = bit.bor(FLAG_QUOTE_SINGLE, FLAG_SQL_ANSI) -local QUOTE_SINGLE_SQL_MYSQL = bit.bor(FLAG_QUOTE_SINGLE, FLAG_SQL_MYSQL) -local QUOTE_DOUBLE_SQL_MYSQL = bit.bor(FLAG_QUOTE_DOUBLE, FLAG_SQL_MYSQL) - --- libibjection.so -ffi.cdef [[ -const char* libinjection_sqli_fingerprint(struct libinjection_sqli_state* sql_state, int flags); - -struct libinjection_sqli_token { - char type; - char str_open; - char str_close; - size_t pos; - size_t len; - int count; - char val[32]; -}; - -typedef char (*ptr_lookup_fn)(struct libinjection_sqli_state*, int lookuptype, const char* word, size_t len); - -struct libinjection_sqli_state { - const char *s; - size_t slen; - ptr_lookup_fn lookup; - void* userdata; - int flags; - size_t pos; - struct libinjection_sqli_token tokenvec[8]; - struct libinjection_sqli_token *current; - char fingerprint[8]; - int reason; - int stats_comment_ddw; - int stats_comment_ddx; - int stats_comment_c; - int stats_comment_hash; - int stats_folds; - int stats_tokens; -}; - -void libinjection_sqli_init(struct libinjection_sqli_state * sf, const char *s, size_t len, int flags); -int libinjection_is_sqli(struct libinjection_sqli_state* sql_state); - -int libinjection_sqli(const char* s, size_t slen, char fingerprint[]); - -int libinjection_is_xss(const char* s, size_t len, int flags); -int libinjection_xss(const char* s, size_t slen); -]] - -_M.version = "0.1.1" - -local state_type = ffi.typeof("struct libinjection_sqli_state[1]") -local lib, loaded - --- "borrowed" from CF aho-corasick lib -local function _loadlib() - if (not loaded) then - local path, so_path - local libname = "libinjection.so" - - for k, v in string.gmatch(package.cpath, "[^;]+") do - so_path = string.match(k, "(.*/)") - if so_path then - -- "so_path" could be nil. e.g, the dir path component is "." - so_path = so_path .. libname - - -- Don't get me wrong, the only way to know if a file exist is - -- trying to open it. - local f = io.open(so_path) - if f ~= nil then - io.close(f) - path = so_path - break - end - end - end - - path = "/usr/local/openresty/1pwaf/data/libinjection.so" - - lib = ffi.load(path) - - if (lib) then - loaded = true - return true - else - return false - end - else - return true - end -end - --- this function is not publicly exposed so we need to emulate it here. not great but not a measurable perf hit -local function _reparse_as_mysql(sqli_state) - return sqli_state[0].stats_comment_ddx ~= 0 or sqli_state[0].stats_comment_hash ~= 0 -end - ---[[ -Secondary API: detects SQLi in a string, given a context. Given a string, returns a list of - -* boolean indicating a match -* SQLi fingerprint ---]] -local function _sqli_contextwrapper(string, char, flag1, flag2) - if (char and not string.find(string, char, 1, true)) then - return false, nil - end - - if (not loaded) then - if (not _loadlib()) then - return false, nil - end - end - - local issqli, lookup, sqli_state - - -- allocate a new libinjection_sqli_state struct - sqli_state = ffi_new(state_type) - - -- init the state - lib.libinjection_sqli_init( - sqli_state, - string, - #string, - FLAG_NONE - ) - - -- initial fingerprint - lib.libinjection_sqli_fingerprint( - sqli_state, - flag1 - ) - - -- lookup - lookup = sqli_state[0].lookup( - sqli_state, - LOOKUP_FINGERPRINT, - sqli_state[0].fingerprint, - #ffi.string(sqli_state[0].fingerprint) - ) - - -- match? great, we're done - if (lookup > 0) then - return true, ffi_string(sqli_state[0].fingerprint) - end - - -- no? reparse, fingerprint and lookup again - if (flag2 and _reparse_as_mysql(sqli_state)) then - lib.libinjection_sqli_fingerprint( - sqli_state, - flag2 - ) - - lookup = sqli_state[0].lookup( - sqli_state, - LOOKUP_FINGERPRINT, - sqli_state[0].fingerprint, - #ffi.string(sqli_state[0].fingerprint) - ) - - if (lookup > 0) then - return true, ffi_string(sqli_state[0].fingerprint) - end - end - - return false, nil -end - ---[[ -Wrapper for second-level API with no char context ---]] -function _M.sqli_noquote(string) - return _sqli_contextwrapper( - string, - nil, - QUOTE_NONE_SQL_ANSI, - QUOTE_NONE_SQL_MYSQL - ) -end - ---[[ -Wrapper for second-level API with CHAR_SINGLE context ---]] -function _M.sqli_singlequote(string) - return _sqli_contextwrapper( - string, - "'", - QUOTE_SINGLE_SQL_ANSI, - QUOTE_SINGLE_SQL_MYSQL - ) -end - ---[[ -Wrapper for second-level API with CHAR_DOUBLE context ---]] -function _M.sqli_doublequote(string) - return _sqli_contextwrapper( - string, - '"', - QUOTE_DOUBLE_SQL_MYSQL - ) -end - ---[[ -Simple API. Given a string, returns a list of - -* boolean indicating a match -* SQLi fingerprint ---]] -function _M.sqli(string) - if (not loaded) then - if (not _loadlib()) then - return false, nil - end - end - - local fingerprint = ffi_new("char [8]") - - return lib.libinjection_sqli(string, #string, fingerprint) == 1, ffi_string(fingerprint) -end - ---[[ -Secondary API: detects XSS in a string, given a context. Given a string, returns a boolean denoting if XSS was detected ---]] -local function _xss_contextwrapper(string, flag) - if (not loaded) then - if (not _loadlib()) then - return false - end - end - - return lib.libinjection_is_xss(string, #string, flag) == 1 -end - ---[[ -Wrapper for second-level API with DATA_STATE flag ---]] -function _M.xss_data_state(string) - return _xss_contextwrapper( - string, - DATA_STATE - ) -end - ---[[ -Wrapper for second-level API with VALUE_NO_QUOTE flag ---]] -function _M.xss_noquote(string) - return _xss_contextwrapper( - string, - VALUE_NO_QUOTE - ) -end - ---[[ -Wrapper for second-level API with VALUE_SINGLE_QUOTE flag ---]] -function _M.xss_singlequote(string) - return _xss_contextwrapper( - string, - VALUE_SINGLE_QUOTE - ) -end - ---[[ -Wrapper for second-level API with VALUE_DOUBLE_QUOTE flag ---]] -function _M.xss_doublequote(string) - return _xss_contextwrapper( - string, - VALUE_DOUBLE_QUOTE - ) -end - ---[[ -Wrapper for second-level API with VALUE_BACK_QUOTE flag ---]] -function _M.xss_backquote(string) - return _xss_contextwrapper( - string, - VALUE_BACK_QUOTE - ) -end - ---[[ -ALPHA version of XSS detector. Given a string, returns a boolean denoting if XSS was detected ---]] -function _M.xss(string) - if (not loaded) then - if (not _loadlib()) then - return false - end - end - - return lib.libinjection_xss(string, #string) == 1 -end - -return _M \ No newline at end of file diff --git a/plugins/openresty/waf/lib/resty/maxminddb.lua b/plugins/openresty/waf/lib/resty/maxminddb.lua deleted file mode 100644 index ddf82032f..000000000 --- a/plugins/openresty/waf/lib/resty/maxminddb.lua +++ /dev/null @@ -1,372 +0,0 @@ ---[[ - Copyright 2017-now anjia (anjia0532@gmail.com) - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -]] - --- copy from https://github.com/lilien1010/lua-resty-maxminddb/blob/f96633e2428f8f7bcc1e2a7a28b747b33233a8db/resty/maxminddb.lua#L5-L12 - -local ffi = require('ffi') -local ffi_new = ffi.new -local ffi_str = ffi.string -local ffi_cast = ffi.cast -local ffi_gc = ffi.gc -local C = ffi.C - -local _M = {} -local _D = {} - -_M._VERSION = '1.3.3' -local mt = { __index = _M } - --- copy from https://github.com/lilien1010/lua-resty-maxminddb/blob/f96633e2428f8f7bcc1e2a7a28b747b33233a8db/resty/maxminddb.lua#L36-L126 -ffi.cdef [[ - -typedef long int ssize_t; - -typedef unsigned int mmdb_uint128_t __attribute__ ((__mode__(TI))); - -typedef struct MMDB_entry_s { - struct MMDB_s *mmdb; - uint32_t offset; -} MMDB_entry_s; - -typedef struct MMDB_lookup_result_s { - bool found_entry; - MMDB_entry_s entry; - uint16_t netmask; -} MMDB_lookup_result_s; - -typedef struct MMDB_entry_data_s { - bool has_data; - union { - uint32_t pointer; - const char *utf8_string; - double double_value; - const uint8_t *bytes; - uint16_t uint16; - uint32_t uint32; - int32_t int32; - uint64_t uint64; - mmdb_uint128_t uint128; - bool boolean; - float float_value; - }; - - uint32_t offset; - uint32_t offset_to_next; - uint32_t data_size; - uint32_t type; -} MMDB_entry_data_s; - -typedef struct MMDB_entry_data_list_s { - MMDB_entry_data_s entry_data; - struct MMDB_entry_data_list_s *next; -} MMDB_entry_data_list_s; - -typedef struct MMDB_description_s { - const char *language; - const char *description; -} MMDB_description_s; - -typedef struct MMDB_metadata_s { - uint32_t node_count; - uint16_t record_size; - uint16_t ip_version; - const char *database_type; - struct { - size_t count; - const char **names; - } languages; - uint16_t binary_format_major_version; - uint16_t binary_format_minor_version; - uint64_t build_epoch; - struct { - size_t count; - MMDB_description_s **descriptions; - } description; -} MMDB_metadata_s; - -typedef struct MMDB_ipv4_start_node_s { - uint16_t netmask; - uint32_t node_value; -} MMDB_ipv4_start_node_s; - -typedef struct MMDB_s { - uint32_t flags; - const char *filename; - ssize_t file_size; - const uint8_t *file_content; - const uint8_t *data_section; - uint32_t data_section_size; - const uint8_t *metadata_section; - uint32_t metadata_section_size; - uint16_t full_record_byte_size; - uint16_t depth; - MMDB_ipv4_start_node_s ipv4_start_node; - MMDB_metadata_s metadata; -} MMDB_s; - -typedef char * pchar; - -MMDB_lookup_result_s MMDB_lookup_string(MMDB_s *const mmdb, const char *const ipstr, int *const gai_error,int *const mmdb_error); -int MMDB_open(const char *const filename, uint32_t flags, MMDB_s *const mmdb); -int MMDB_aget_value(MMDB_entry_s *const start, MMDB_entry_data_s *const entry_data, const char *const *const path); -char *MMDB_strerror(int error_code); - -int MMDB_get_entry_data_list(MMDB_entry_s *start, MMDB_entry_data_list_s **const entry_data_list); -void MMDB_free_entry_data_list(MMDB_entry_data_list_s *const entry_data_list); -void MMDB_close(MMDB_s *const mmdb); -const char *gai_strerror(int errcode); -]] - --- error codes --- https://github.com/maxmind/libmaxminddb/blob/master/include/maxminddb.h#L66 -local MMDB_SUCCESS = 0 -local MMDB_FILE_OPEN_ERROR = 1 -local MMDB_CORRUPT_SEARCH_TREE_ERROR = 2 -local MMDB_INVALID_METADATA_ERROR = 3 -local MMDB_IO_ERROR = 4 -local MMDB_OUT_OF_MEMORY_ERROR = 5 -local MMDB_UNKNOWN_DATABASE_FORMAT_ERROR = 6 -local MMDB_INVALID_DATA_ERROR = 7 -local MMDB_INVALID_LOOKUP_PATH_ERROR = 8 -local MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR = 9 -local MMDB_INVALID_NODE_NUMBER_ERROR = 10 -local MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR = 11 - --- data type --- https://github.com/maxmind/libmaxminddb/blob/master/include/maxminddb.h#L40 -local MMDB_DATA_TYPE_EXTENDED = 0 -local MMDB_DATA_TYPE_POINTER = 1 -local MMDB_DATA_TYPE_UTF8_STRING = 2 -local MMDB_DATA_TYPE_DOUBLE = 3 -local MMDB_DATA_TYPE_BYTES = 4 -local MMDB_DATA_TYPE_UINT16 = 5 -local MMDB_DATA_TYPE_UINT32 = 6 -local MMDB_DATA_TYPE_MAP = 7 -local MMDB_DATA_TYPE_INT32 = 8 -local MMDB_DATA_TYPE_UINT64 = 9 -local MMDB_DATA_TYPE_UINT128 = 10 -local MMDB_DATA_TYPE_ARRAY = 11 -local MMDB_DATA_TYPE_CONTAINER = 12 -local MMDB_DATA_TYPE_END_MARKER = 13 -local MMDB_DATA_TYPE_BOOLEAN = 14 -local MMDB_DATA_TYPE_FLOAT = 15 - --- copy from https://github.com/lilien1010/lua-resty-maxminddb/blob/f96633e2428f8f7bcc1e2a7a28b747b33233a8db/resty/maxminddb.lua#L136-L138 - -local initted = false - -local function mmdb_strerror(profile, rc) - return ffi_str(_D[profile].maxm.MMDB_strerror(rc)) -end - -local function gai_strerror(rc) - return ffi_str(C.gai_strerror(rc)) -end - -function _M.init(profiles) - for profile, location in pairs(profiles) do - _D[profile] = {} - _D[profile].maxm = ffi.load('/usr/local/openresty/1pwaf/data/libmaxminddb.so') - _D[profile].mmdb = ffi_new('MMDB_s') - local maxmind_ready = _D[profile].maxm.MMDB_open(location, 0, _D[profile].mmdb) - if maxmind_ready ~= MMDB_SUCCESS then - return nil, mmdb_strerror(profile, maxmind_ready) - end - ffi_gc(_D[profile].mmdb, _D[profile].maxm.MMDB_close) - end - - --if not initted then - -- local maxmind_ready = maxm.MMDB_open(dbfile, 0, mmdb) - -- - -- if maxmind_ready ~= MMDB_SUCCESS then - -- return nil, mmdb_strerror(maxmind_ready) - -- end - -- - -- - -- - -- ffi_gc(mmdb, maxm.MMDB_close) - --end - initted = true - return initted -end - -function _M.initted() - return initted -end - --- https://github.com/maxmind/libmaxminddb/blob/master/src/maxminddb.c#L1938 --- LOCAL MMDB_entry_data_list_s *dump_entry_data_list( FILE *stream, MMDB_entry_data_list_s *entry_data_list, int indent, int *status) -local function _dump_entry_data_list(entry_data_list, status) - - if not entry_data_list then - return nil, MMDB_INVALID_DATA_ERROR - end - - local entry_data_item = entry_data_list[0].entry_data - local data_type = entry_data_item.type - local data_size = entry_data_item.data_size - local result - - if data_type == MMDB_DATA_TYPE_MAP then - result = {} - - local size = entry_data_item.data_size - - entry_data_list = entry_data_list[0].next - - while (size > 0 and entry_data_list) - do - entry_data_item = entry_data_list[0].entry_data - data_type = entry_data_item.type - data_size = entry_data_item.data_size - - if MMDB_DATA_TYPE_UTF8_STRING ~= data_type then - return nil, MMDB_INVALID_DATA_ERROR - end - - local key = ffi_str(entry_data_item.utf8_string, data_size) - - if not key then - return nil, MMDB_OUT_OF_MEMORY_ERROR - end - - local val - entry_data_list = entry_data_list[0].next - entry_data_list, status, val = _dump_entry_data_list(entry_data_list) - - if status ~= MMDB_SUCCESS then - return nil, status - end - - result[key] = val - - size = size - 1 - end - - - elseif entry_data_list[0].entry_data.type == MMDB_DATA_TYPE_ARRAY then - local size = entry_data_list[0].entry_data.data_size - result = {} - - entry_data_list = entry_data_list[0].next - - local i = 1 - while (i <= size and entry_data_list) - do - local val - entry_data_list, status, val = _dump_entry_data_list(entry_data_list) - - if status ~= MMDB_SUCCESS then - return nil, nil, val - end - - result[i] = val - i = i + 1 - end - - - else - entry_data_item = entry_data_list[0].entry_data - data_type = entry_data_item.type - data_size = entry_data_item.data_size - - local val - -- string type "key":"val" - -- other type "key":val - -- default other type - if data_type == MMDB_DATA_TYPE_UTF8_STRING then - val = ffi_str(entry_data_item.utf8_string, data_size) - if not val then - status = MMDB_OUT_OF_MEMORY_ERROR - return nil, status - end - elseif data_type == MMDB_DATA_TYPE_BYTES then - val = ffi_str(ffi_cast('char * ', entry_data_item.bytes), data_size) - if not val then - status = MMDB_OUT_OF_MEMORY_ERROR - return nil, status - end - elseif data_type == MMDB_DATA_TYPE_DOUBLE then - val = entry_data_item.double_value - elseif data_type == MMDB_DATA_TYPE_FLOAT then - val = entry_data_item.float_value - elseif data_type == MMDB_DATA_TYPE_UINT16 then - val = entry_data_item.uint16 - elseif data_type == MMDB_DATA_TYPE_UINT32 then - val = entry_data_item.uint32 - elseif data_type == MMDB_DATA_TYPE_BOOLEAN then - val = entry_data_item.boolean - elseif data_type == MMDB_DATA_TYPE_UINT64 then - val = entry_data_item.uint64 - elseif data_type == MMDB_DATA_TYPE_INT32 then - val = entry_data_item.int32 - else - return nil, MMDB_INVALID_DATA_ERROR - end - - result = val - entry_data_list = entry_data_list[0].next - end - - status = MMDB_SUCCESS - return entry_data_list, status, result -end - -function _M.lookup(profile, ip) - - if not initted then - return nil, "not initialized" - end - - -- copy from https://github.com/lilien1010/lua-resty-maxminddb/blob/f96633e2428f8f7bcc1e2a7a28b747b33233a8db/resty/maxminddb.lua#L159-L176 - local gai_error = ffi_new('int[1]') - local mmdb_error = ffi_new('int[1]') - - local result = _D[profile].maxm.MMDB_lookup_string(_D[profile].mmdb, ip, gai_error, mmdb_error) - - if mmdb_error[0] ~= MMDB_SUCCESS then - return nil, 'lookup failed: ' .. mmdb_strerror(profile, mmdb_error[0]) - end - - if gai_error[0] ~= MMDB_SUCCESS then - return nil, 'lookup failed: ' .. gai_strerror(gai_error[0]) - end - - if true ~= result.found_entry then - return nil, 'not found' - end - - local entry_data_list = ffi_cast('MMDB_entry_data_list_s **const', ffi_new("MMDB_entry_data_list_s")) - - local status = _D[profile].maxm.MMDB_get_entry_data_list(result.entry, entry_data_list) - - if status ~= MMDB_SUCCESS then - return nil, 'get entry data failed: ' .. mmdb_strerror(profile, status) - end - - local head = entry_data_list[0] -- Save so this can be passed to free fn. - local _, status, result = _dump_entry_data_list(entry_data_list) - _D[profile].maxm.MMDB_free_entry_data_list(head) - - if status ~= MMDB_SUCCESS then - return nil, 'dump entry data failed: ' .. mmdb_strerror(profile, status) - end - - return result -end - --- copy from https://github.com/lilien1010/lua-resty-maxminddb/blob/master/resty/maxminddb.lua#L208 --- https://www.maxmind.com/en/geoip2-databases you should download the mmdb file from maxmind - -return _M; diff --git a/plugins/openresty/waf/lib/resty/mlcache.lua b/plugins/openresty/waf/lib/resty/mlcache.lua deleted file mode 100644 index 0c1e5b2e2..000000000 --- a/plugins/openresty/waf/lib/resty/mlcache.lua +++ /dev/null @@ -1,1441 +0,0 @@ --- vim: ts=4 sts=4 sw=4 et: - -local new_tab = require "table.new" -local lrucache = require "resty.lrucache" -local resty_lock = require "resty.lock" -local tablepool -do - local pok - pok, tablepool = pcall(require, "tablepool") - if not pok then - -- fallback for OpenResty < 1.15.8.1 - tablepool = { - fetch = function(_, narr, nrec) - return new_tab(narr, nrec) - end, - release = function(_, _, _) - -- nop (obj will be subject to GC) - end, - } - end -end -local codec -do - local pok - pok, codec = pcall(require, "string.buffer") - if not pok then - codec = require "cjson" - end -end - - -local now = ngx.now -local min = math.min -local ceil = math.ceil -local fmt = string.format -local sub = string.sub -local find = string.find -local type = type -local pcall = pcall -local xpcall = xpcall -local traceback = debug.traceback -local error = error -local tostring = tostring -local tonumber = tonumber -local encode = codec.encode -local decode = codec.decode -local thread_spawn = ngx.thread.spawn -local thread_wait = ngx.thread.wait -local setmetatable = setmetatable -local shared = ngx.shared -local ngx_log = ngx.log -local WARN = ngx.WARN -local ERR = ngx.ERR - - -local CACHE_MISS_SENTINEL_LRU = {} -local LOCK_KEY_PREFIX = "lua-resty-mlcache:lock:" -local LRU_INSTANCES = setmetatable({}, { __mode = "v" }) -local SHM_SET_DEFAULT_TRIES = 3 -local BULK_DEFAULT_CONCURRENCY = 3 - - -local TYPES_LOOKUP = { - number = 1, - boolean = 2, - string = 3, - table = 4, -} - - -local SHM_FLAGS = { - stale = 0x00000001, -} - - -local marshallers = { - shm_value = function(str_value, value_type, at, ttl) - return fmt("%d:%f:%f:%s", value_type, at, ttl, str_value) - end, - - shm_nil = function(at, ttl) - return fmt("0:%f:%f:", at, ttl) - end, - - [1] = function(number) -- number - return tostring(number) - end, - - [2] = function(bool) -- boolean - return bool and "true" or "false" - end, - - [3] = function(str) -- string - return str - end, - - [4] = function(t) -- table - local pok, str = pcall(encode, t) - if not pok then - return nil, "could not encode table value: " .. str - end - - return str - end, -} - - -local unmarshallers = { - shm_value = function(marshalled) - -- split our shm marshalled value by the hard-coded ":" tokens - -- "type:at:ttl:value" - -- 1:1501831735.052000:0.500000:123 - local ttl_last = find(marshalled, ":", 21, true) - 1 - - local value_type = sub(marshalled, 1, 1) -- n:... - local at = sub(marshalled, 3, 19) -- n:1501831160 - local ttl = sub(marshalled, 21, ttl_last) - local str_value = sub(marshalled, ttl_last + 2) - - return str_value, tonumber(value_type), tonumber(at), tonumber(ttl) - end, - - [0] = function() -- nil - return nil - end, - - [1] = function(str) -- number - return tonumber(str) - end, - - [2] = function(str) -- boolean - return str == "true" - end, - - [3] = function(str) -- string - return str - end, - - [4] = function(str) -- table - local pok, t = pcall(decode, str) - if not pok then - return nil, "could not decode table value: " .. t - end - - return t - end, -} - - -local function rebuild_lru(self) - if self.lru then - if self.lru.flush_all then - self.lru:flush_all() - return - end - - -- fallback for OpenResty < 1.13.6.2 - -- Invalidate the entire LRU by GC-ing it. - LRU_INSTANCES[self.name] = nil - self.lru = nil - end - - -- Several mlcache instances can have the same name and hence, the same - -- lru instance. We need to GC such LRU instance when all mlcache instances - -- using them are GC'ed. We do this with a weak table. - local lru = LRU_INSTANCES[self.name] - if not lru then - lru = lrucache.new(self.lru_size) - LRU_INSTANCES[self.name] = lru - end - - self.lru = lru -end - - -local _M = { - _VERSION = "2.6.1", - _AUTHOR = "Thibault Charbonnier", - _LICENSE = "MIT", - _URL = "https://github.com/thibaultcha/lua-resty-mlcache", -} -local mt = { __index = _M } - - -function _M.new(name, shm, opts) - if type(name) ~= "string" then - error("name must be a string", 2) - end - - if type(shm) ~= "string" then - error("shm must be a string", 2) - end - - if opts ~= nil then - if type(opts) ~= "table" then - error("opts must be a table", 2) - end - - if opts.lru_size ~= nil and type(opts.lru_size) ~= "number" then - error("opts.lru_size must be a number", 2) - end - - if opts.ttl ~= nil then - if type(opts.ttl) ~= "number" then - error("opts.ttl must be a number", 2) - end - - if opts.ttl < 0 then - error("opts.ttl must be >= 0", 2) - end - end - - if opts.neg_ttl ~= nil then - if type(opts.neg_ttl) ~= "number" then - error("opts.neg_ttl must be a number", 2) - end - - if opts.neg_ttl < 0 then - error("opts.neg_ttl must be >= 0", 2) - end - end - - if opts.resurrect_ttl ~= nil then - if type(opts.resurrect_ttl) ~= "number" then - error("opts.resurrect_ttl must be a number", 2) - end - - if opts.resurrect_ttl < 0 then - error("opts.resurrect_ttl must be >= 0", 2) - end - end - - if opts.resty_lock_opts ~= nil - and type(opts.resty_lock_opts) ~= "table" - then - error("opts.resty_lock_opts must be a table", 2) - end - - if opts.ipc_shm ~= nil and type(opts.ipc_shm) ~= "string" then - error("opts.ipc_shm must be a string", 2) - end - - if opts.ipc ~= nil then - if opts.ipc_shm then - error("cannot specify both of opts.ipc_shm and opts.ipc", 2) - end - - if type(opts.ipc) ~= "table" then - error("opts.ipc must be a table", 2) - end - - if type(opts.ipc.register_listeners) ~= "function" then - error("opts.ipc.register_listeners must be a function", 2) - end - - if type(opts.ipc.broadcast) ~= "function" then - error("opts.ipc.broadcast must be a function", 2) - end - - if opts.ipc.poll ~= nil and type(opts.ipc.poll) ~= "function" then - error("opts.ipc.poll must be a function", 2) - end - end - - if opts.l1_serializer ~= nil - and type(opts.l1_serializer) ~= "function" - then - error("opts.l1_serializer must be a function", 2) - end - - if opts.shm_set_tries ~= nil then - if type(opts.shm_set_tries) ~= "number" then - error("opts.shm_set_tries must be a number", 2) - end - - if opts.shm_set_tries < 1 then - error("opts.shm_set_tries must be >= 1", 2) - end - end - - if opts.shm_miss ~= nil and type(opts.shm_miss) ~= "string" then - error("opts.shm_miss must be a string", 2) - end - - if opts.shm_locks ~= nil and type(opts.shm_locks) ~= "string" then - error("opts.shm_locks must be a string", 2) - end - else - opts = {} - end - - local dict = shared[shm] - if not dict then - return nil, "no such lua_shared_dict: " .. shm - end - - local dict_miss - if opts.shm_miss then - dict_miss = shared[opts.shm_miss] - if not dict_miss then - return nil, "no such lua_shared_dict for opts.shm_miss: " - .. opts.shm_miss - end - end - - if opts.shm_locks then - local dict_locks = shared[opts.shm_locks] - if not dict_locks then - return nil, "no such lua_shared_dict for opts.shm_locks: " - .. opts.shm_locks - end - end - - local self = { - name = name, - dict = dict, - shm = shm, - dict_miss = dict_miss, - shm_miss = opts.shm_miss, - shm_locks = opts.shm_locks or shm, - ttl = opts.ttl or 30, - neg_ttl = opts.neg_ttl or 5, - resurrect_ttl = opts.resurrect_ttl, - lru_size = opts.lru_size or 100, - resty_lock_opts = opts.resty_lock_opts, - l1_serializer = opts.l1_serializer, - shm_set_tries = opts.shm_set_tries or SHM_SET_DEFAULT_TRIES, - debug = opts.debug, - } - - if opts.ipc_shm or opts.ipc then - self.events = { - ["invalidation"] = { - channel = fmt("mlcache:invalidations:%s", name), - handler = function(key) - self.lru:delete(key) - end, - }, - ["purge"] = { - channel = fmt("mlcache:purge:%s", name), - handler = function() - rebuild_lru(self) - end, - } - } - - if opts.ipc_shm then - local mlcache_ipc = require "resty.mlcache.ipc" - - local ipc, err = mlcache_ipc.new(opts.ipc_shm, opts.debug) - if not ipc then - return nil, "failed to initialize mlcache IPC " .. - "(could not instantiate mlcache.ipc): " .. err - end - - for _, ev in pairs(self.events) do - ipc:subscribe(ev.channel, ev.handler) - end - - self.broadcast = function(channel, data) - return ipc:broadcast(channel, data) - end - - self.poll = function(timeout) - return ipc:poll(timeout) - end - - self.ipc = ipc - - else - -- opts.ipc - local ok, err = opts.ipc.register_listeners(self.events) - if not ok and err ~= nil then - return nil, "failed to initialize custom IPC " .. - "(opts.ipc.register_listeners returned an error): " - .. err - end - - self.broadcast = opts.ipc.broadcast - self.poll = opts.ipc.poll - - self.ipc = true - end - end - - if opts.lru then - self.lru = opts.lru - - else - rebuild_lru(self) - end - - return setmetatable(self, mt) -end - - -local function l1_serialize(value, l1_serializer) - if value ~= nil and l1_serializer then - local ok, err - ok, value, err = pcall(l1_serializer, value) - if not ok then - return nil, "l1_serializer threw an error: " .. value - end - - if err then - return nil, err - end - - if value == nil then - return nil, "l1_serializer returned a nil value" - end - end - - return value -end - - -local function set_lru(self, key, value, ttl, neg_ttl, l1_serializer) - local value, err = l1_serialize(value, l1_serializer) - if err then - return nil, err - end - - if value == nil then - value = CACHE_MISS_SENTINEL_LRU - ttl = neg_ttl - end - - if ttl == 0 then - -- indefinite ttl for lua-resty-lrucache is 'nil' - ttl = nil - end - - self.lru:set(key, value, ttl) - - return value -end - - -local function marshall_for_shm(value, ttl, neg_ttl) - local at = now() - - if value == nil then - return marshallers.shm_nil(at, neg_ttl), nil, true -- is_nil - end - - -- serialize insertion time + Lua types for shm storage - - local value_type = TYPES_LOOKUP[type(value)] - - if not marshallers[value_type] then - error("cannot cache value of type " .. type(value)) - end - - local str_marshalled, err = marshallers[value_type](value) - if not str_marshalled then - return nil, "could not serialize value for lua_shared_dict insertion: " - .. err - end - - return marshallers.shm_value(str_marshalled, value_type, at, ttl) -end - - -local function unmarshall_from_shm(shm_v) - local str_serialized, value_type, at, ttl = unmarshallers.shm_value(shm_v) - - local value, err = unmarshallers[value_type](str_serialized) - if err then - return nil, err - end - - return value, nil, at, ttl -end - - -local function set_shm(self, shm_key, value, ttl, neg_ttl, flags, shm_set_tries, - throw_no_mem) - local shm_value, err, is_nil = marshall_for_shm(value, ttl, neg_ttl) - if not shm_value then - return nil, err - end - - local shm = self.shm - local dict = self.dict - - if is_nil then - ttl = neg_ttl - - if self.dict_miss then - shm = self.shm_miss - dict = self.dict_miss - end - end - - -- we will call `set()` N times to work around potential shm fragmentation. - -- when the shm is full, it will only evict about 30 to 90 items (via - -- LRU), which could lead to a situation where `set()` still does not - -- have enough memory to store the cached value, in which case we - -- try again to try to trigger more LRU evictions. - - local tries = 0 - local ok, err - - while tries < shm_set_tries do - tries = tries + 1 - - ok, err = dict:set(shm_key, shm_value, ttl, flags or 0) - if ok or err and err ~= "no memory" then - break - end - end - - if not ok then - if err ~= "no memory" or throw_no_mem then - return nil, "could not write to lua_shared_dict '" .. shm - .. "': " .. err - end - - ngx_log(WARN, "could not write to lua_shared_dict '", - shm, "' after ", tries, " tries (no memory), ", - "it is either fragmented or cannot allocate more ", - "memory, consider increasing 'opts.shm_set_tries'") - end - - return true -end - - -local function set_shm_set_lru(self, key, shm_key, value, ttl, neg_ttl, flags, - shm_set_tries, l1_serializer, throw_no_mem) - - local ok, err = set_shm(self, shm_key, value, ttl, neg_ttl, flags, - shm_set_tries, throw_no_mem) - if not ok then - return nil, err - end - - return set_lru(self, key, value, ttl, neg_ttl, l1_serializer) -end - - -local function get_shm_set_lru(self, key, shm_key, l1_serializer) - local v, shmerr, went_stale = self.dict:get_stale(shm_key) - if v == nil and shmerr then - -- shmerr can be 'flags' upon successful get_stale() calls, so we - -- also check v == nil - return nil, "could not read from lua_shared_dict: " .. shmerr - end - - if self.shm_miss and v == nil then - -- if we cache misses in another shm, maybe it is there - v, shmerr, went_stale = self.dict_miss:get_stale(shm_key) - if v == nil and shmerr then - -- shmerr can be 'flags' upon successful get_stale() calls, so we - -- also check v == nil - return nil, "could not read from lua_shared_dict: " .. shmerr - end - end - - if v ~= nil then - local value, err, at, ttl = unmarshall_from_shm(v) - if err then - return nil, "could not deserialize value after lua_shared_dict " .. - "retrieval: " .. err - end - - if went_stale then - value, err = l1_serialize(value, l1_serializer) - if err then - return nil, err - end - - return value, nil, went_stale - end - - -- 'shmerr' is 'flags' on :get_stale() success - local is_stale = shmerr == SHM_FLAGS.stale - - local remaining_ttl - if ttl == 0 then - -- indefinite ttl, keep '0' as it means 'forever' - remaining_ttl = 0 - - else - -- compute elapsed time to get remaining ttl for LRU caching - remaining_ttl = ttl - (now() - at) - - if remaining_ttl <= 0 then - -- value has less than 1ms of lifetime in the shm, avoid - -- setting it in LRU which would be wasteful and could - -- indefinitely cache the value when ttl == 0 - value, err = l1_serialize(value, l1_serializer) - if err then - return nil, err - end - - return value, nil, nil, is_stale - end - end - - value, err = set_lru(self, key, value, remaining_ttl, remaining_ttl, - l1_serializer) - if err then - return nil, err - end - - return value, nil, nil, is_stale - end -end - - -local function check_opts(self, opts) - local ttl - local neg_ttl - local resurrect_ttl - local l1_serializer - local shm_set_tries - local resty_lock_opts - - if opts ~= nil then - if type(opts) ~= "table" then - error("opts must be a table", 3) - end - - ttl = opts.ttl - if ttl ~= nil then - if type(ttl) ~= "number" then - error("opts.ttl must be a number", 3) - end - - if ttl < 0 then - error("opts.ttl must be >= 0", 3) - end - end - - neg_ttl = opts.neg_ttl - if neg_ttl ~= nil then - if type(neg_ttl) ~= "number" then - error("opts.neg_ttl must be a number", 3) - end - - if neg_ttl < 0 then - error("opts.neg_ttl must be >= 0", 3) - end - end - - resurrect_ttl = opts.resurrect_ttl - if resurrect_ttl ~= nil then - if type(resurrect_ttl) ~= "number" then - error("opts.resurrect_ttl must be a number", 3) - end - - if resurrect_ttl < 0 then - error("opts.resurrect_ttl must be >= 0", 3) - end - end - - l1_serializer = opts.l1_serializer - if l1_serializer ~= nil and type(l1_serializer) ~= "function" then - error("opts.l1_serializer must be a function", 3) - end - - shm_set_tries = opts.shm_set_tries - if shm_set_tries ~= nil then - if type(shm_set_tries) ~= "number" then - error("opts.shm_set_tries must be a number", 3) - end - - if shm_set_tries < 1 then - error("opts.shm_set_tries must be >= 1", 3) - end - end - - resty_lock_opts = opts.resty_lock_opts - if resty_lock_opts ~= nil then - if type(resty_lock_opts) ~= "table" then - error("opts.resty_lock_opts must be a table", 3) - end - end - end - - if not ttl then - ttl = self.ttl - end - - if not neg_ttl then - neg_ttl = self.neg_ttl - end - - if not resurrect_ttl then - resurrect_ttl = self.resurrect_ttl - end - - if not l1_serializer then - l1_serializer = self.l1_serializer - end - - if not shm_set_tries then - shm_set_tries = self.shm_set_tries - end - - if not resty_lock_opts then - resty_lock_opts = self.resty_lock_opts - end - - return ttl, neg_ttl, resurrect_ttl, l1_serializer, shm_set_tries, - resty_lock_opts -end - - -local function unlock_and_ret(lock, res, err, hit_lvl) - local ok, lerr = lock:unlock() - if not ok and lerr ~= "unlocked" then - return nil, "could not unlock callback: " .. lerr - end - - return res, err, hit_lvl -end - - -local function run_callback(self, key, shm_key, data, ttl, neg_ttl, - went_stale, l1_serializer, resurrect_ttl, shm_set_tries, rlock_opts, cb, ...) - - local lock, err = resty_lock:new(self.shm_locks, rlock_opts) - if not lock then - return nil, "could not create lock: " .. err - end - - local elapsed, lerr = lock:lock(LOCK_KEY_PREFIX .. shm_key) - if not elapsed and lerr ~= "timeout" then - return nil, "could not acquire callback lock: " .. lerr - end - - do - -- check for another worker's success at running the callback, but - -- do not return data if it is still the same stale value (this is - -- possible if the value was still not evicted between the first - -- get() and this one) - - local data2, err, went_stale2, stale2 = get_shm_set_lru(self, key, - shm_key, - l1_serializer) - if err then - return unlock_and_ret(lock, nil, err) - end - - if data2 ~= nil and not went_stale2 then - -- we got a fresh item from shm: other worker succeeded in running - -- the callback - if data2 == CACHE_MISS_SENTINEL_LRU then - data2 = nil - end - - return unlock_and_ret(lock, data2, nil, stale2 and 4 or 2) - end - end - - -- we are either the 1st worker to hold the lock, or - -- a subsequent worker whose lock has timed out before the 1st one - -- finished to run the callback - - if lerr == "timeout" then - local errmsg = "could not acquire callback lock: timeout" - - -- no stale data nor desire to resurrect it - if not went_stale or not resurrect_ttl then - return nil, errmsg - end - - -- do not resurrect the value here (another worker is running the - -- callback and will either get the new value, or resurrect it for - -- us if the callback fails) - - ngx_log(WARN, errmsg) - - -- went_stale is true, hence the value cannot be set in the LRU - -- cache, and cannot be CACHE_MISS_SENTINEL_LRU - - return data, nil, 4 - end - - -- still not in shm, we are the 1st worker to hold the lock, and thus - -- responsible for running the callback - - local pok, perr, err, new_ttl = xpcall(cb, traceback, ...) - if not pok then - return unlock_and_ret(lock, nil, "callback threw an error: " .. - tostring(perr)) - end - - if err then - -- callback returned nil + err - - -- be resilient in case callbacks return wrong error type - err = tostring(err) - - -- no stale data nor desire to resurrect it - if not went_stale or not resurrect_ttl then - return unlock_and_ret(lock, perr, err) - end - - -- we got 'data' from the shm, even though it is stale - -- 1. log as warn that the callback returned an error - -- 2. resurrect: insert it back into shm if 'resurrect_ttl' - -- 3. signify the staleness with a high hit_lvl of '4' - - ngx_log(WARN, "callback returned an error (", err, ") but stale ", - "value found in shm will be resurrected for ", - resurrect_ttl, "s (resurrect_ttl)") - - local res_data, res_err = set_shm_set_lru(self, key, shm_key, - data, resurrect_ttl, - resurrect_ttl, - SHM_FLAGS.stale, - shm_set_tries, l1_serializer) - if res_err then - ngx_log(WARN, "could not resurrect stale data (", res_err, ")") - end - - if res_data == CACHE_MISS_SENTINEL_LRU then - res_data = nil - end - - return unlock_and_ret(lock, res_data, nil, 4) - end - - -- successful callback run returned 'data, nil, new_ttl?' - - data = perr - - -- override ttl / neg_ttl - - if type(new_ttl) == "number" then - if new_ttl < 0 then - -- bypass cache - return unlock_and_ret(lock, data, nil, 3) - end - - if data == nil then - neg_ttl = new_ttl - - else - ttl = new_ttl - end - end - - data, err = set_shm_set_lru(self, key, shm_key, data, ttl, neg_ttl, nil, - shm_set_tries, l1_serializer) - if err then - return unlock_and_ret(lock, nil, err) - end - - if data == CACHE_MISS_SENTINEL_LRU then - data = nil - end - - -- unlock and return - - return unlock_and_ret(lock, data, nil, 3) -end - - -function _M:get(key, opts, cb, ...) - if type(key) ~= "string" then - error("key must be a string", 2) - end - - if cb ~= nil and type(cb) ~= "function" then - error("callback must be nil or a function", 2) - end - - -- worker LRU cache retrieval - - local data = self.lru:get(key) - if data == CACHE_MISS_SENTINEL_LRU then - return nil, nil, 1 - end - - if data ~= nil then - return data, nil, 1 - end - - -- not in worker's LRU cache, need shm lookup - - -- restrict this key to the current namespace, so we isolate this - -- mlcache instance from potential other instances using the same - -- shm - local namespaced_key = self.name .. key - - -- opts validation - - local ttl, neg_ttl, resurrect_ttl, l1_serializer, shm_set_tries, - rlock_opts = check_opts(self, opts) - - local err, went_stale, is_stale - data, err, went_stale, is_stale = get_shm_set_lru(self, key, namespaced_key, - l1_serializer) - if err then - return nil, err - end - - if data ~= nil and not went_stale then - if data == CACHE_MISS_SENTINEL_LRU then - data = nil - end - - return data, nil, is_stale and 4 or 2 - end - - -- not in shm either - - if cb == nil then - -- no L3 callback, early exit - return nil, nil, -1 - end - - -- L3 callback, single worker to run it - - return run_callback(self, key, namespaced_key, data, ttl, neg_ttl, - went_stale, l1_serializer, resurrect_ttl, - shm_set_tries, rlock_opts, cb, ...) -end - - -do - local function run_thread(self, ops, from, to) - for i = from, to do - local ctx = ops[i] - - ctx.data, ctx.err, ctx.hit_lvl = run_callback(self, ctx.key, - ctx.shm_key, ctx.data, - ctx.ttl, ctx.neg_ttl, - ctx.went_stale, - ctx.l1_serializer, - ctx.resurrect_ttl, - ctx.shm_set_tries, - ctx.rlock_opts, - ctx.cb, ctx.arg) - end - end - - - local bulk_mt = {} - bulk_mt.__index = bulk_mt - - - function _M.new_bulk(n_ops) - local bulk = new_tab((n_ops or 2) * 4, 1) -- 4 slots per op - bulk.n = 0 - - return setmetatable(bulk, bulk_mt) - end - - - function bulk_mt:add(key, opts, cb, arg) - local i = (self.n * 4) + 1 - self[i] = key - self[i + 1] = opts - self[i + 2] = cb - self[i + 3] = arg - self.n = self.n + 1 - end - - - local function bulk_res_iter(res, i) - local idx = i * 3 + 1 - if idx > res.n then - return - end - - i = i + 1 - - local data = res[idx] - local err = res[idx + 1] - local hit_lvl = res[idx + 2] - - return i, data, err, hit_lvl - end - - - function _M.each_bulk_res(res) - if not res.n then - error("res must have res.n field; is this a get_bulk() result?", 2) - end - - return bulk_res_iter, res, 0 - end - - - function _M:get_bulk(bulk, opts) - if type(bulk) ~= "table" then - error("bulk must be a table", 2) - end - - if not bulk.n then - error("bulk must have n field", 2) - end - - if opts then - if type(opts) ~= "table" then - error("opts must be a table", 2) - end - - if opts.concurrency then - if type(opts.concurrency) ~= "number" then - error("opts.concurrency must be a number", 2) - end - - if opts.concurrency <= 0 then - error("opts.concurrency must be > 0", 2) - end - end - end - - local n_bulk = bulk.n * 4 - local res = new_tab(n_bulk - n_bulk / 4, 1) - local res_idx = 1 - - -- only used if running L3 callbacks - local n_cbs = 0 - local cb_ctxs - - -- bulk - -- { "key", opts, cb, arg } - -- - -- res - -- { data, "err", hit_lvl } - - for i = 1, n_bulk, 4 do - local b_key = bulk[i] - local b_opts = bulk[i + 1] - local b_cb = bulk[i + 2] - - if type(b_key) ~= "string" then - error("key at index " .. i .. " must be a string for operation " .. - ceil(i / 4) .. " (got " .. type(b_key) .. ")", 2) - end - - if type(b_cb) ~= "function" then - error("callback at index " .. i + 2 .. " must be a function " .. - "for operation " .. ceil(i / 4) .. " (got " .. type(b_cb) .. - ")", 2) - end - - -- worker LRU cache retrieval - - local data = self.lru:get(b_key) - if data ~= nil then - if data == CACHE_MISS_SENTINEL_LRU then - data = nil - end - - res[res_idx] = data - --res[res_idx + 1] = nil - res[res_idx + 2] = 1 - - else - local pok, ttl, neg_ttl, resurrect_ttl, l1_serializer, - shm_set_tries, rlock_opts = pcall(check_opts, self, b_opts) - if not pok then - -- strip the stacktrace - local err = ttl:match("mlcache%.lua:%d+:%s(.*)") - error("options at index " .. i + 1 .. " for operation " .. - ceil(i / 4) .. " are invalid: " .. err, 2) - end - - -- not in worker's LRU cache, need shm lookup - -- we will prepare a task for each cache miss - local namespaced_key = self.name .. b_key - - local err, went_stale, is_stale - data, err, went_stale, is_stale = get_shm_set_lru(self, b_key, - namespaced_key, - l1_serializer) - if err then - --res[res_idx] = nil - res[res_idx + 1] = err - --res[res_idx + 2] = nil - - elseif data ~= nil and not went_stale then - if data == CACHE_MISS_SENTINEL_LRU then - data = nil - end - - res[res_idx] = data - --res[res_idx + 1] = nil - res[res_idx + 2] = is_stale and 4 or 2 - - else - -- not in shm either, we have to prepare a task to run the - -- L3 callback - - n_cbs = n_cbs + 1 - - if n_cbs == 1 then - cb_ctxs = tablepool.fetch("bulk_cb_ctxs", 1, 0) - end - - local ctx = tablepool.fetch("bulk_cb_ctx", 0, 15) - ctx.res_idx = res_idx - ctx.cb = b_cb - ctx.arg = bulk[i + 3] -- arg - ctx.key = b_key - ctx.shm_key = namespaced_key - ctx.data = data - ctx.ttl = ttl - ctx.neg_ttl = neg_ttl - ctx.went_stale = went_stale - ctx.l1_serializer = l1_serializer - ctx.resurrect_ttl = resurrect_ttl - ctx.shm_set_tries = shm_set_tries - ctx.rlock_opts = rlock_opts - ctx.data = data - ctx.err = nil - ctx.hit_lvl = nil - - cb_ctxs[n_cbs] = ctx - end - end - - res_idx = res_idx + 3 - end - - if n_cbs == 0 then - -- no callback to run, all items were in L1/L2 - res.n = res_idx - 1 - return res - end - - -- some L3 callbacks have to run - -- schedule threads as per our concurrency settings - -- we will use this thread as well - - local concurrency - if opts then - concurrency = opts.concurrency - end - - if not concurrency then - concurrency = BULK_DEFAULT_CONCURRENCY - end - - local threads - local threads_idx = 0 - - do - -- spawn concurrent threads - local thread_size - local n_threads = min(n_cbs, concurrency) - 1 - - if n_threads > 0 then - threads = tablepool.fetch("bulk_threads", n_threads, 0) - thread_size = ceil(n_cbs / concurrency) - end - - if self.debug then - ngx.log(ngx.DEBUG, "spawning ", n_threads, " threads to run ", - n_cbs, " callbacks") - end - - local from = 1 - local rest = n_cbs - - for i = 1, n_threads do - local to - if rest >= thread_size then - rest = rest - thread_size - to = from + thread_size - 1 - else - rest = 0 - to = from - end - - if self.debug then - ngx.log(ngx.DEBUG, "thread ", i, " running callbacks ", from, - " to ", to) - end - - threads_idx = threads_idx + 1 - threads[i] = thread_spawn(run_thread, self, cb_ctxs, from, to) - - from = from + thread_size - - if rest == 0 then - break - end - end - - if rest > 0 then - -- use this thread as one of our concurrent threads - local to = from + rest - 1 - - if self.debug then - ngx.log(ngx.DEBUG, "main thread running callbacks ", from, - " to ", to) - end - - run_thread(self, cb_ctxs, from, to) - end - end - - -- wait for other threads - - for i = 1, threads_idx do - local ok, err = thread_wait(threads[i]) - if not ok then - -- when thread_wait() fails, we don't get res_idx, and thus - -- cannot populate the appropriate res indexes with the - -- error - ngx_log(ERR, "failed to wait for thread number ", i, ": ", err) - end - end - - for i = 1, n_cbs do - local ctx = cb_ctxs[i] - local ctx_res_idx = ctx.res_idx - - res[ctx_res_idx] = ctx.data - res[ctx_res_idx + 1] = ctx.err - res[ctx_res_idx + 2] = ctx.hit_lvl - - tablepool.release("bulk_cb_ctx", ctx, true) -- no clear tab - end - - tablepool.release("bulk_cb_ctxs", cb_ctxs) - - if threads then - tablepool.release("bulk_threads", threads) - end - - res.n = res_idx - 1 - - return res - end - - -end -- get_bulk() - - -function _M:peek(key, stale) - if type(key) ~= "string" then - error("key must be a string", 2) - end - - -- restrict this key to the current namespace, so we isolate this - -- mlcache instance from potential other instances using the same - -- shm - local namespaced_key = self.name .. key - - local v, err, went_stale = self.dict:get_stale(namespaced_key) - if v == nil and err then - -- err can be 'flags' upon successful get_stale() calls, so we - -- also check v == nil - return nil, "could not read from lua_shared_dict: " .. err - end - - -- if we specified shm_miss, it might be a negative hit cached - -- there - if self.dict_miss and v == nil then - v, err, went_stale = self.dict_miss:get_stale(namespaced_key) - if v == nil and err then - -- err can be 'flags' upon successful get_stale() calls, so we - -- also check v == nil - return nil, "could not read from lua_shared_dict: " .. err - end - end - - if went_stale and not stale then - return nil - end - - if v ~= nil then - local value, err, at, ttl = unmarshall_from_shm(v) - if err then - return nil, "could not deserialize value after lua_shared_dict " .. - "retrieval: " .. err - end - - local remaining_ttl = 0 - - if ttl > 0 then - remaining_ttl = ttl - (now() - at) - - if remaining_ttl == 0 then - -- guarantee a non-zero remaining_ttl if ttl is set - remaining_ttl = 0.001 - end - end - - return remaining_ttl, nil, value, went_stale - end -end - - -function _M:set(key, opts, value) - if not self.broadcast then - error("no ipc to propagate update, specify opts.ipc_shm or opts.ipc", 2) - end - - if type(key) ~= "string" then - error("key must be a string", 2) - end - - do - -- restrict this key to the current namespace, so we isolate this - -- mlcache instance from potential other instances using the same - -- shm - local ttl, neg_ttl, _, l1_serializer, shm_set_tries = check_opts(self, - opts) - local namespaced_key = self.name .. key - - if self.dict_miss then - -- since we specified a separate shm for negative caches, we - -- must make sure that we clear any value that may have been - -- set in the other shm - local dict = value == nil and self.dict or self.dict_miss - - -- TODO: there is a potential race-condition here between this - -- :delete() and the subsequent :set() in set_shm() - local ok, err = dict:delete(namespaced_key) - if not ok then - return nil, "could not delete from shm: " .. err - end - end - - local _, err = set_shm_set_lru(self, key, namespaced_key, value, ttl, - neg_ttl, nil, shm_set_tries, - l1_serializer, true) - if err then - return nil, err - end - end - - local _, err = self.broadcast(self.events.invalidation.channel, key) - if err then - return nil, "could not broadcast update: " .. err - end - - return true -end - - -function _M:delete(key) - if not self.broadcast then - error("no ipc to propagate deletion, specify opts.ipc_shm or opts.ipc", - 2) - end - - if type(key) ~= "string" then - error("key must be a string", 2) - end - - -- delete from shm first - do - -- restrict this key to the current namespace, so we isolate this - -- mlcache instance from potential other instances using the same - -- shm - local namespaced_key = self.name .. key - - local ok, err = self.dict:delete(namespaced_key) - if not ok then - return nil, "could not delete from shm: " .. err - end - - -- instance uses shm_miss for negative caches, since we don't know - -- where the cached value is (is it nil or not?), we must remove it - -- from both - if self.dict_miss then - ok, err = self.dict_miss:delete(namespaced_key) - if not ok then - return nil, "could not delete from shm: " .. err - end - end - end - - -- delete from LRU and propagate - self.lru:delete(key) - - local _, err = self.broadcast(self.events.invalidation.channel, key) - if err then - return nil, "could not broadcast deletion: " .. err - end - - return true -end - - -function _M:purge(flush_expired) - if not self.broadcast then - error("no ipc to propagate purge, specify opts.ipc_shm or opts.ipc", 2) - end - - if not self.lru.flush_all and LRU_INSTANCES[self.name] ~= self.lru then - error("cannot purge when using custom LRU cache with " .. - "OpenResty < 1.13.6.2", 2) - end - - -- clear shm first - self.dict:flush_all() - - -- clear negative caches shm if specified - if self.dict_miss then - self.dict_miss:flush_all() - end - - if flush_expired then - self.dict:flush_expired() - - if self.dict_miss then - self.dict_miss:flush_expired() - end - end - - -- clear LRU content and propagate - rebuild_lru(self) - - local _, err = self.broadcast(self.events.purge.channel, "") - if err then - return nil, "could not broadcast purge: " .. err - end - - return true -end - - -function _M:update(timeout) - if not self.poll then - error("no polling configured, specify opts.ipc_shm or opts.ipc.poll", 2) - end - - local _, err = self.poll(timeout) - if err then - return nil, "could not poll ipc events: " .. err - end - - return true -end - - -return _M \ No newline at end of file diff --git a/plugins/openresty/waf/lib/resty/mlcache/ipc.lua b/plugins/openresty/waf/lib/resty/mlcache/ipc.lua deleted file mode 100644 index 55cf0113b..000000000 --- a/plugins/openresty/waf/lib/resty/mlcache/ipc.lua +++ /dev/null @@ -1,257 +0,0 @@ --- vim: ts=4 sts=4 sw=4 et: - -local ERR = ngx.ERR -local WARN = ngx.WARN -local INFO = ngx.INFO -local sleep = ngx.sleep -local shared = ngx.shared -local worker_pid = ngx.worker.pid -local ngx_log = ngx.log -local fmt = string.format -local sub = string.sub -local find = string.find -local min = math.min -local type = type -local pcall = pcall -local error = error -local insert = table.insert -local tonumber = tonumber -local setmetatable = setmetatable - - -local INDEX_KEY = "lua-resty-ipc:index" -local FORCIBLE_KEY = "lua-resty-ipc:forcible" -local POLL_SLEEP_RATIO = 2 - - -local function marshall(worker_pid, channel, data) - return fmt("%d:%d:%s%s", worker_pid, #data, channel, data) -end - - -local function unmarshall(str) - local sep_1 = find(str, ":", nil , true) - local sep_2 = find(str, ":", sep_1 + 1, true) - - local pid = tonumber(sub(str, 1 , sep_1 - 1)) - local data_len = tonumber(sub(str, sep_1 + 1, sep_2 - 1)) - - local channel_last_pos = #str - data_len - - local channel = sub(str, sep_2 + 1, channel_last_pos) - local data = sub(str, channel_last_pos + 1) - - return pid, channel, data -end - - -local function log(lvl, ...) - return ngx_log(lvl, "[ipc] ", ...) -end - - -local _M = {} -local mt = { __index = _M } - - -function _M.new(shm, debug) - local dict = shared[shm] - if not dict then - return nil, "no such lua_shared_dict: " .. shm - end - - local self = { - dict = dict, - pid = debug and 0 or worker_pid(), - idx = 0, - callbacks = {}, - } - - return setmetatable(self, mt) -end - - -function _M:subscribe(channel, cb) - if type(channel) ~= "string" then - error("channel must be a string", 2) - end - - if type(cb) ~= "function" then - error("callback must be a function", 2) - end - - if not self.callbacks[channel] then - self.callbacks[channel] = { cb } - - else - insert(self.callbacks[channel], cb) - end -end - - -function _M:broadcast(channel, data) - if type(channel) ~= "string" then - error("channel must be a string", 2) - end - - if type(data) ~= "string" then - error("data must be a string", 2) - end - - local marshalled_event = marshall(worker_pid(), channel, data) - - local idx, err = self.dict:incr(INDEX_KEY, 1, 0) - if not idx then - return nil, "failed to increment index: " .. err - end - - local ok, err, forcible = self.dict:set(idx, marshalled_event) - if not ok then - return nil, "failed to insert event in shm: " .. err - end - - if forcible then - -- take note that eviction has started - -- we repeat this flagging to avoid this key from ever being - -- evicted itself - local ok, err = self.dict:set(FORCIBLE_KEY, true) - if not ok then - return nil, "failed to set forcible flag in shm: " .. err - end - end - - return true -end - - --- Note: if this module were to be used by users (that is, users can implement --- their own pub/sub events and thus, callbacks), this method would then need --- to consider the time spent in callbacks to prevent long running callbacks --- from penalizing the worker. --- Since this module is currently only used by mlcache, whose callback is an --- shm operation, we only worry about the time spent waiting for events --- between the 'incr()' and 'set()' race condition. -function _M:poll(timeout) - if timeout ~= nil and type(timeout) ~= "number" then - error("timeout must be a number", 2) - end - - local shm_idx, err = self.dict:get(INDEX_KEY) - if err then - return nil, "failed to get index: " .. err - end - - if shm_idx == nil then - -- no events to poll yet - return true - end - - if type(shm_idx) ~= "number" then - return nil, "index is not a number, shm tampered with" - end - - if not timeout then - timeout = 0.3 - end - - if self.idx == 0 then - local forcible, err = self.dict:get(FORCIBLE_KEY) - if err then - return nil, "failed to get forcible flag from shm: " .. err - end - - if forcible then - -- shm lru eviction occurred, we are likely a new worker - -- skip indexes that may have been evicted and resume current - -- polling idx - self.idx = shm_idx - 1 - end - - else - -- guard: self.idx <= shm_idx - self.idx = min(self.idx, shm_idx) - end - - local elapsed = 0 - - for _ = self.idx, shm_idx - 1 do - -- fetch event from shm with a retry policy in case - -- we run our :get() in between another worker's - -- :incr() and :set() - - local v - local idx = self.idx + 1 - - do - local perr - local pok = true - local sleep_step = 0.001 - - while elapsed < timeout do - v, err = self.dict:get(idx) - if v ~= nil or err then - break - end - - if pok then - log(INFO, "no event data at index '", idx, "', ", - "retrying in: ", sleep_step, "s") - - -- sleep is not available in all ngx_lua contexts - -- if we fail once, never retry to sleep - pok, perr = pcall(sleep, sleep_step) - if not pok then - log(WARN, "could not sleep before retry: ", perr, - " (note: it is safer to call this function ", - "in contexts that support the ngx.sleep() ", - "API)") - end - end - - elapsed = elapsed + sleep_step - sleep_step = min(sleep_step * POLL_SLEEP_RATIO, - timeout - elapsed) - end - end - - -- fetch next event on next iteration - -- even if we timeout, we might miss 1 event (we return in timeout and - -- we don't retry that event), but it's better than being stuck forever - -- on an event that might have been evicted from the shm. - self.idx = idx - - if elapsed >= timeout then - return nil, "timeout" - end - - if err then - log(ERR, "could not get event at index '", self.idx, "': ", err) - - elseif type(v) ~= "string" then - log(ERR, "event at index '", self.idx, "' is not a string, ", - "shm tampered with") - - else - local pid, channel, data = unmarshall(v) - - if self.pid ~= pid then - -- coming from another worker - local cbs = self.callbacks[channel] - if cbs then - for j = 1, #cbs do - local pok, perr = pcall(cbs[j], data) - if not pok then - log(ERR, "callback for channel '", channel, - "' threw a Lua error: ", perr) - end - end - end - end - end - end - - return true -end - - -return _M \ No newline at end of file diff --git a/plugins/openresty/waf/lib/resty/uuid.lua b/plugins/openresty/waf/lib/resty/uuid.lua deleted file mode 100644 index 2d73b43c2..000000000 --- a/plugins/openresty/waf/lib/resty/uuid.lua +++ /dev/null @@ -1,427 +0,0 @@ --- vim:set ts=4 sts=4 sw=4 et: - ---- jit-uuid --- Fast and dependency-free UUID library for LuaJIT/ngx_lua. --- @module jit-uuid --- @author Thibault Charbonnier --- @license MIT --- @release 0.0.7 - - -local bit = require 'bit' - - -local tohex = bit.tohex -local band = bit.band -local bor = bit.bor - - -local _M = { - _VERSION = '0.0.7' -} - - ----------- --- seeding ----------- - - ---- Seed the random number generator. --- Under the hood, this function calls `math.randomseed`. --- It makes sure to use the most appropriate seeding technique for --- the current environment, guaranteeing a unique seed. --- --- To guarantee unique UUIDs, you must have correctly seeded --- the Lua pseudo-random generator (with `math.randomseed`). --- You are free to seed it any way you want, but this function --- can do it for you if you'd like, with some added guarantees. --- --- @param[type=number] seed (Optional) A seed to use. If none given, will --- generate one trying to use the most appropriate technique. --- @treturn number `seed`: the seed given to `math.randomseed`. --- @usage --- local uuid = require 'resty.jit-uuid' --- uuid.seed() --- --- -- in ngx_lua, seed in the init_worker context: --- init_worker_by_lua { --- local uuid = require 'resty.jit-uuid' --- uuid.seed() --- } -function _M.seed(seed) - if not seed then - if ngx then - seed = ngx.time() + ngx.worker.pid() - - elseif package.loaded['socket'] and package.loaded['socket'].gettime then - seed = package.loaded['socket'].gettime()*10000 - - else - seed = os.time() - end - end - - math.randomseed(seed) - - return seed -end - - -------------- --- validation -------------- - - -do - if ngx and string.find(ngx.config.nginx_configure(),'--with-pcre-jit',nil,true) then - local type = type - local re_find = ngx.re.find - local regex = '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$' - - - --- Validate a string as a UUID. - -- To be considered valid, a UUID must be given in its canonical - -- form (hexadecimal digits including the hyphen characters). - -- This function validates UUIDs disregarding their generation algorithm, - -- and in a case-insensitive manner, but checks the variant field. - -- - -- Use JIT PCRE if available in OpenResty or fallbacks on Lua patterns. - -- - -- @param[type=string] str String to verify. - -- @treturn boolean `valid`: true if valid UUID, false otherwise. - -- @usage - -- local uuid = require 'resty.jit-uuid' - -- - -- uuid.is_valid 'cbb297c0-a956-486d-ad1d-f9bZZZZZZZZZ' --> false - -- uuid.is_valid 'cbb297c0-a956-486d-dd1d-f9b42df9465a' --> false (invalid variant) - -- uuid.is_valid 'cbb297c0a956486dad1df9b42df9465a' --> false (no dashes) - -- uuid.is_valid 'cbb297c0-a956-486d-ad1d-f9b42df9465a' --> true - function _M.is_valid(str) - -- it has proven itself efficient to first check the length with an - -- evenly distributed set of valid and invalid uuid lengths. - if type(str) ~= 'string' or #str ~= 36 then - return false - end - - return re_find(str, regex, 'ioj') ~= nil - end - - else - local match = string.match - local d = '[0-9a-fA-F]' - local p = '^' .. table.concat({ - d:rep(8), - d:rep(4), - d:rep(4), - '[89ab]' .. d:rep(3), - d:rep(12) - }, '%-') .. '$' - - - function _M.is_valid(str) - if type(str) ~= 'string' or #str ~= 36 then - return false - end - - return match(str, p) ~= nil - end - end -end - - ----------------- --- v4 generation ----------------- - - -do - local fmt = string.format - local random = math.random - - - --- Generate a v4 UUID. - -- v4 UUIDs are created from randomly generated numbers. - -- - -- @treturn string `uuid`: a v4 (randomly generated) UUID. - -- @usage - -- local uuid = require 'resty.jit-uuid' - -- - -- local u1 = uuid() ---> __call metamethod - -- local u2 = uuid.generate_v4() - function _M.generate_v4() - return (fmt('%s%s%s%s-%s%s-%s%s-%s%s-%s%s%s%s%s%s', - tohex(random(0, 255), 2), - tohex(random(0, 255), 2), - tohex(random(0, 255), 2), - tohex(random(0, 255), 2), - - tohex(random(0, 255), 2), - tohex(random(0, 255), 2), - - tohex(bor(band(random(0, 255), 0x0F), 0x40), 2), - tohex(random(0, 255), 2), - - tohex(bor(band(random(0, 255), 0x3F), 0x80), 2), - tohex(random(0, 255), 2), - - tohex(random(0, 255), 2), - tohex(random(0, 255), 2), - tohex(random(0, 255), 2), - tohex(random(0, 255), 2), - tohex(random(0, 255), 2), - tohex(random(0, 255), 2))) - end -end - - ----------------- --- v3/v5 generation ----------------- - - -do - if ngx then - local ffi = require 'ffi' - - - local tonumber = tonumber - local assert = assert - local error = error - local concat = table.concat - local type = type - local char = string.char - local fmt = string.format - local sub = string.sub - local gmatch = ngx.re.gmatch - local sha1_bin = ngx.sha1_bin - local md5 = ngx.md5 - local C = ffi.C - local ffi_new = ffi.new - local ffi_str = ffi.string - local ffi_cast = ffi.cast - local new_tab - do - local ok - ok, new_tab = pcall(require, 'table.new') - if not ok then - new_tab = function(narr, nrec) return {} end - end - end - - - ffi.cdef [[ - typedef unsigned char u_char; - typedef intptr_t ngx_int_t; - - u_char * ngx_hex_dump(u_char *dst, const u_char *src, size_t len); - ngx_int_t ngx_hextoi(u_char *line, size_t n); - ]] - - - local str_type = ffi.typeof('uint8_t[?]') - local u_char_type = ffi.typeof('u_char *') - - - local function bin_tohex(s) - local slen = #s - local blen = slen * 2 - local buf = ffi_new(str_type, blen) - - C.ngx_hex_dump(buf, s, slen) - - return ffi_str(buf, blen) - end - - - local function hex_to_i(s) - local buf = ffi_cast(u_char_type, s) - - local n = tonumber(C.ngx_hextoi(buf, #s)) - if n == -1 then - error("could not convert hex to number") - end - - return n - end - - - local buf = new_tab(16, 0) - - - local function factory(namespace, hash_fn) - if not _M.is_valid(namespace) then - return nil, 'namespace must be a valid UUID' - end - - local i = 0 - local iter, err = gmatch(namespace, [[([\da-f][\da-f])]]) - if not iter then - return nil, 'could not create iter: ' .. err - end - - while true do - local m, err = iter() - if err then - return nil, err - end - - if not m then - break - end - - i = i + 1 - buf[i] = char(tonumber(m[0], 16)) - end - - assert(i == 16, "invalid binary namespace buffer length") - local ns = concat(buf) - - return function(name) - if type(name) ~= 'string' then - return nil, 'name must be a string' - end - - local hash, ver, var = hash_fn(ns, name) - - return (fmt('%s-%s-%s%s-%s%s-%s', sub(hash, 1, 8), - sub(hash, 9, 12), - ver, - sub(hash, 15, 16), - var, - sub(hash, 19, 20), - sub(hash, 21, 32))) - end - end - - - local function v3_hash(binary, name) - local hash = md5(binary .. name) - - return hash, - tohex(bor(band(hex_to_i(sub(hash, 13, 14)), 0x0F), 0x30), 2), - tohex(bor(band(hex_to_i(sub(hash, 17, 18)), 0x3F), 0x80), 2) - end - - - local function v5_hash(binary, name) - local hash = bin_tohex(sha1_bin(binary .. name)) - - return hash, - tohex(bor(band(hex_to_i(sub(hash, 13, 14)), 0x0F), 0x50), 2), - tohex(bor(band(hex_to_i(sub(hash, 17, 18)), 0x3F), 0x80), 2) - end - - - --- Instanciate a v3 UUID factory. - -- @function factory_v3 - -- Creates a closure generating namespaced v3 UUIDs. - -- @param[type=string] namespace (must be a valid UUID according to `is_valid`) - -- @treturn function `factory`: a v3 UUID generator. - -- @treturn string `err`: a string describing an error - -- @usage - -- local uuid = require 'resty.jit-uuid' - -- - -- local fact = assert(uuid.factory_v3('e6ebd542-06ae-11e6-8e82-bba81706b27d')) - -- - -- local u1 = fact('hello') - -- ---> 3db7a435-8c56-359d-a563-1b69e6802c78 - -- - -- local u2 = fact('foobar') - -- ---> e8d3eeba-7723-3b72-bbc5-8f598afa6773 - function _M.factory_v3(namespace) - return factory(namespace, v3_hash) - end - - - --- Instanciate a v5 UUID factory. - -- @function factory_v5 - -- Creates a closure generating namespaced v5 UUIDs. - -- @param[type=string] namespace (must be a valid UUID according to `is_valid`) - -- @treturn function `factory`: a v5 UUID generator. - -- @treturn string `err`: a string describing an error - -- @usage - -- local uuid = require 'resty.jit-uuid' - -- - -- local fact = assert(uuid.factory_v5('e6ebd542-06ae-11e6-8e82-bba81706b27d')) - -- - -- local u1 = fact('hello') - -- ---> 4850816f-1658-5890-8bfd-1ed14251f1f0 - -- - -- local u2 = fact('foobar') - -- ---> c9be99fc-326b-5066-bdba-dcd31a6d01ab - function _M.factory_v5(namespace) - return factory(namespace, v5_hash) - end - - - --- Generate a v3 UUID. - -- v3 UUIDs are created from a namespace and a name (a UUID and a string). - -- The same name and namespace result in the same UUID. The same name and - -- different namespaces result in different UUIDs, and vice-versa. - -- The resulting UUID is derived using MD5 hashing. - -- - -- This is a sugar function which instanciates a short-lived v3 UUID factory. - -- It is an expensive operation, and intensive generation using the same - -- namespaces should prefer allocating their own long-lived factory with - -- `factory_v3`. - -- - -- @param[type=string] namespace (must be a valid UUID according to `is_valid`) - -- @param[type=string] name - -- @treturn string `uuid`: a v3 (namespaced) UUID. - -- @treturn string `err`: a string describing an error - -- @usage - -- local uuid = require 'resty.jit-uuid' - -- - -- local u = uuid.generate_v3('e6ebd542-06ae-11e6-8e82-bba81706b27d', 'hello') - -- ---> 3db7a435-8c56-359d-a563-1b69e6802c78 - function _M.generate_v3(namespace, name) - local fact, err = _M.factory_v3(namespace) - if not fact then - return nil, err - end - - return fact(name) - end - - - --- Generate a v5 UUID. - -- v5 UUIDs are created from a namespace and a name (a UUID and a string). - -- The same name and namespace result in the same UUID. The same name and - -- different namespaces result in different UUIDs, and vice-versa. - -- The resulting UUID is derived using SHA-1 hashing. - -- - -- This is a sugar function which instanciates a short-lived v5 UUID factory. - -- It is an expensive operation, and intensive generation using the same - -- namespaces should prefer allocating their own long-lived factory with - -- `factory_v5`. - -- - -- @param[type=string] namespace (must be a valid UUID according to `is_valid`) - -- @param[type=string] name - -- @treturn string `uuid`: a v5 (namespaced) UUID. - -- @treturn string `err`: a string describing an error - -- @usage - -- local uuid = require 'resty.jit-uuid' - -- - -- local u = uuid.generate_v5('e6ebd542-06ae-11e6-8e82-bba81706b27d', 'hello') - -- ---> 4850816f-1658-5890-8bfd-1ed14251f1f0 - function _M.generate_v5(namespace, name) - local fact, err = _M.factory_v5(namespace) - if not fact then - return nil, err - end - - return fact(name) - end - - else - function _M.factory_v3() error('v3 UUID generation only supported in ngx_lua', 2) end - function _M.generate_v3() error('v3 UUID generation only supported in ngx_lua', 2) end - function _M.factory_v5() error('v5 UUID generation only supported in ngx_lua', 2) end - function _M.generate_v5() error('v5 UUID generation only supported in ngx_lua', 2) end - end -end - - -return setmetatable(_M, { - __call = _M.generate_v4 -}) \ No newline at end of file diff --git a/plugins/openresty/waf/lib/stringutf8.lua b/plugins/openresty/waf/lib/stringutf8.lua deleted file mode 100644 index 921757392..000000000 --- a/plugins/openresty/waf/lib/stringutf8.lua +++ /dev/null @@ -1,149 +0,0 @@ -local error = error -local str_len = string.len -local new_table = table.new -local concat_table = table.concat -local insert_table = table.insert -local byte_str = string.byte -local sub_str = string.sub -local type = type -local abs = math.abs -local match_str = string.match -local ngx_re_gsub = ngx.re.gsub - -local _M = {} - -local INDEX_OUT_OF_RANGE = "String index out of range: " -local NOT_NUMBER = "number expected, got " -local NOT_STRING = "string expected, got " -local NOT_STRING_NIL = "string expected, got nil" - -function _M.to_char_array(str) - local array - if str then - local length = str_len(str) - array = new_table(length, 0) - - local byteLength = 1 - local i, j = 1, 1 - while i <= length do - local firstByte = byte_str(str, i) - if firstByte >= 0 and firstByte < 128 then - byteLength = 1 - - elseif firstByte > 191 and firstByte < 224 then - byteLength = 2 - - elseif firstByte > 223 and firstByte < 240 then - byteLength = 3 - - elseif firstByte > 239 and firstByte < 248 then - byteLength = 4 - end - - j = i + byteLength - local char = sub_str(str, i, j - 1) - i = j - insert_table(array, char) - end - end - - return array -end - -function _M.sub(str, i, j) - local str_sub - if str then - if i == nil then - i = 1 - end - - if type(i) ~= "number" then - error(NOT_NUMBER .. type(i)) - end - - if i < 1 then - error(INDEX_OUT_OF_RANGE .. i) - end - - if j then - if type(j) ~= "number" then - error(NOT_NUMBER .. type(j)) - end - end - - local array = _M.to_char_array(str) - if array then - local length = #array - local subLen = length - i - if subLen < 0 then - error(INDEX_OUT_OF_RANGE .. subLen) - end - - if not j then - str_sub = concat_table(array, "", i) - else - if abs(j) > length then - error(INDEX_OUT_OF_RANGE .. j) - end - if j < 0 then - j = length + j + 1 - end - str_sub = concat_table(array, "", i, j) - end - end - end - - return str_sub -end - -function _M.trim(str) - if str then - str = ngx_re_gsub(str, "^\\s*|\\s*$", "", "jo") - end - - return str -end - -function _M.len(str) - local str_length = 0 - if str then - if type(str) ~= "string" then - error(NOT_STRING .. type(str)) - end - - local length = str_len(str) - - local i = 1 - while i <= length do - local firstByte = byte_str(str, i) - if firstByte >= 0 and firstByte < 128 then - i = i + 1 - - elseif firstByte > 191 and firstByte < 224 then - i = i + 2 - - elseif firstByte > 223 and firstByte < 240 then - i = i + 3 - - elseif firstByte > 239 and firstByte < 248 then - i = i + 4 - end - - str_length = str_length + 1 - end - else - error(NOT_STRING_NIL) - end - - return str_length -end - -function _M.default_if_blank(str, default_str) - if str == nil or match_str(str, "^%s*$") then - return default_str - end - - return str -end - -return _M diff --git a/plugins/openresty/waf/lib/utils.lua b/plugins/openresty/waf/lib/utils.lua deleted file mode 100644 index b9b8ad251..000000000 --- a/plugins/openresty/waf/lib/utils.lua +++ /dev/null @@ -1,200 +0,0 @@ -local geoip = require "geoip" -local sub_str = string.sub -local pairs = pairs -local insert_table = table.insert -local tonumber = tonumber -local ipairs = ipairs -local type = type -local find_str = string.find -local gmatch_str = string.gmatch -local pcall = pcall -local random = math.random -local unescape_uri = ngx.unescape_uri - -local _M = {} - -function _M.split(input_string, delimiter) - local result = {} - for part in input_string:gmatch("([^" .. delimiter .. "]+)") do - insert_table(result, part) - end - return result -end - -function _M.get_cookie_list(cookie_str) - local cookies = {} - for cookie in cookie_str:gmatch("([^;]+)") do - local key, value = cookie:match("^%s*([^=]+)=(.*)$") - if key and value then - cookies[key] = value - end - end - return cookies -end - -function _M.unescape_uri(str) - local newStr = str - for t = 1, 2 do - local temp = unescape_uri(newStr) - if not temp then - break - end - newStr = temp - end - return newStr -end - -function _M.get_expire_time() - local localtime = ngx.localtime() - local hour = sub_str(localtime, 12, 13) - local expire_time = (24 - tonumber(hour)) * 3600 - return expire_time -end - -function _M.get_date_hour() - local localtime = ngx.localtime() - local hour = sub_str(localtime, 1, 13) - return hour -end - -function _M.getHours() - local hours = {} - local today = ngx.today() - local hour = nil - for i = 0, 23 do - if i < 10 then - hour = today .. ' 0' .. i - else - hour = today .. ' ' .. i - end - hours[i + 1] = hour - end - - return hours -end - -function _M.ipv4_to_int(ip) - local ipInt = 0 - for i, octet in ipairs({ ip:match("(%d+)%.(%d+)%.(%d+)%.(%d+)") }) do - ipInt = ipInt + tonumber(octet) * 256 ^ (4 - i) - end - return ipInt -end - -function _M.is_ipv6(ip) - if find_str(ip, ':') then - return true - end - return false -end - -function _M.is_ip_in_array(ip, ipStart, ipEnd) - if ip >= ipStart and ip <= ipEnd then - return true - end - return false -end - -function _M.get_real_ip() - local var = ngx.var - local ips = { - var.http_x_forwarded_for, - var.http_proxy_client_ip, - var.http_wl_proxy_client_ip, - var.http_http_client_ip, - var.http_http_x_forwarded_for, - var.remote_addr - } - - for _, ip in pairs(ips) do - if ip and ip ~= "" then - if type(ip) == "table" then - ip = ip[1] - end - return ip - end - end - - return "unknown" -end - -function _M.get_ip_location(ip) - if _M.is_intranet_address(ip) then - return { - country = { ["zh"] = "内网", ["en"] = "Intranet" }, - longitude = 0, - latitude = 0, - iso = "Local" - } - else - geoip.init() - local geo_res = geoip.lookup(ip) - --local msg = "访问 IP " .. ip - --if geo_res.country then - -- msg = msg .. " 国家 " .. cjson.encode(geo_res.country) - --end - --if geo_res.province then - -- msg = msg .. " 省份 " .. cjson.encode(geo_res.province) - --end - --ngx.log(ngx.ERR, msg) - return geo_res - - end -end - -function _M.get_header(headerKey) - return ngx.req.get_headers(20000)[headerKey] -end - -function _M.get_headers() - return ngx.req.get_headers(20000) -end - -function _M.is_intranet_address(ip_addr) - if not ip_addr then - return false - end - if ip_addr == "unknown" then - return false - end - if find_str(ip_addr, ':') then - return false - end - - local parts = {} - for part in gmatch_str(ip_addr, "%d+") do - insert_table(parts, tonumber(part)) - end - if parts[1] == 10 or - (parts[1] == 192 and parts[2] == 168) or - (parts[1] == 172 and parts[2] >= 16 and parts[2] <= 31) then - return true - else - return false - end -end - -function _M.get_wafdb(waf_db_path) - local ok, sqlite3 = pcall(function() - return require "lsqlite3" - end) - if not ok then - return nil - end - return sqlite3.open(waf_db_path) -end - - -math.randomseed(os.time()) - -function _M.random_string(length) - local charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" - local str = "" - for i = 1, length do - local rand_index = random(1, #charset) - str = str .. sub_str(charset, rand_index, rand_index) - end - return str -end - -return _M diff --git a/plugins/openresty/waf/log_and_traffic.lua b/plugins/openresty/waf/log_and_traffic.lua deleted file mode 100644 index 3d0bca317..000000000 --- a/plugins/openresty/waf/log_and_traffic.lua +++ /dev/null @@ -1,240 +0,0 @@ -local utils = require "utils" -local config = require "config" -local redis_util = require "redis_util" -local action = require "action" -local uuid = require"resty.uuid" - -local upper_str = string.upper -local tonumber = tonumber -local pairs = pairs -local type = type -local concat_table = table.concat - -local function write_req_log(attack) - local wafdb = utils.get_wafdb(config.waf_log_db_path) - if not wafdb then - ngx.log(ngx.ERR, "get log db failed") - return - end - - local real_ip = ngx.ctx.ip - local ip_location = ngx.ctx.ip_location - local country - local province - local longitude = 0.0 - local latitude = 0.0 - local iso = "CN" - if ip_location then - country = ip_location.country or { - ["zh"] = "unknown", - ["en"] = "unknown" - } - province = ip_location.province or { - ["zh"] = "", - ["en"] = "" - } - longitude = ip_location.longitude - latitude = ip_location.latitude - iso = ip_location.iso - end - - local exec_rule = {} - local rule_action = "" - local exec_rule_type = "" - local match_rule_detail = "" - local match_rule_type = "" - local is_attack = 0 - local is_block = 0 - local blocking_time = 0 - - local method = "" - local uri = "" - local ua = "" - local host = "" - local protocol = "" - local website_key = "" - local logs_str = "" - - if attack then - exec_rule = ngx.ctx.exec_rule - rule_action = exec_rule.action - exec_rule_type = exec_rule.type - is_attack = 1 - method = ngx.req.get_method() - uri = ngx.var.request_uri - ua = ngx.ctx.ua - host = ngx.var.server_name - protocol = ngx.var.server_protocol or "" - website_key = ngx.ctx.website_key - - if exec_rule.match_rule then - match_rule_detail = exec_rule.match_rule.rule - match_rule_type = exec_rule.match_rule.type - end - - if ngx.ctx.ip_blocked then - is_block = 1 - blocking_time = tonumber(exec_rule.ipBlockTime) - end - - logs_str = method .. " " .. uri .. " "..protocol.."\n" - local headers = ngx.req.get_headers(20000) - for k, v in pairs(headers) do - local value = "" - if v then - if type(v) == "table" then - value = concat_table(v, ",") - else - value = v - end - end - logs_str = logs_str .. upper_str(k) .. ": " .. value .. "\n" - end - end - - local log_id = uuid() - local insertQuery = [[ - INSERT INTO req_logs ( - id, ip, ip_iso, ip_country_zh, ip_country_en, - ip_province_zh, ip_province_en, ip_longitude, ip_latitude, - localtime, server_name, website_key, host, method, - uri, user_agent, exec_rule, rule_type, match_rule, match_value, - nginx_log, blocking_time, action, is_block,is_attack - ) VALUES ( - :id, :real_ip, :iso, :country_zh, :country_en, - :province_zh, :province_en,:longitude, :latitude, - DATETIME('now'), :server_name,:host, :website_key, :method, - :uri, :ua, :exec_rule, :rule_type, :match_rule, :match_value, - :logs_str, :blocking_time, :action, :is_block, :is_attack - ) - ]] - - wafdb:execute([[BEGIN TRANSACTION]]) - - local stmt = wafdb:prepare(insertQuery) - stmt:bind_names { - id = log_id, - iso = iso, - real_ip = real_ip, - country_zh = country["zh"], - country_en = country["en"], - province_zh = province["zh"], - province_en = province["en"], - longitude = longitude, - latitude = latitude, - host = host, - server_name = host, - website_key = website_key, - method = method, - uri = uri, - ua = ua, - exec_rule = exec_rule_type, - rule_type = match_rule_type, - match_rule = match_rule_detail, - match_value = "", - logs_str = logs_str, - blocking_time = blocking_time or 0, - action = rule_action, - is_block = is_block, - is_attack = is_attack - } - stmt:step() - stmt:finalize() - - local code2 = 101 - if ngx.ctx.ip_blocked then - local insertBlockIp = [[ - INSERT INTO block_ips (ip, is_block, blocking_time, req_log_id,create_date) - VALUES (:ip, :is_block, :blocking_time, :req_log_id, DATETIME('now')) - ]] - stmt = wafdb:prepare(insertBlockIp) - stmt:bind_names { - ip=real_ip, - is_block = is_block, - blocking_time = blocking_time or 0, - req_log_id = log_id - } - code2 = stmt:step() - stmt:finalize() - end - - wafdb:execute([[COMMIT]]) - - --local error_msg = wafdb:errmsg() - --if error_msg then - -- ngx.log(ngx.ERR, "insert attack_log error ", error_msg .. " ") - --end - -end - -local function count_not_found() - if ngx.status ~= 404 then - return - end - if config.is_global_state_on("notFoundCount") then - local ip = ngx.ctx.ip - local not_found_config = config.get_global_config("notFoundCount") - local key = ip - - if config.is_redis_on() then - key = "cc_attack_count:" .. key - local count, _ = redis_util.incr(key, not_found_config.duration) - if not count then - redis_util.set(key, 1, not_found_config.duration) - elseif count >= not_found_config.threshold then - action.block_ip(ip, not_found_config) - return - end - else - key = ip .. "not_found" - local limit = ngx.shared.waf_limit - local count, _ = limit:incr(key, 1, 0, not_found_config.duration) - if not count then - limit:set(key, 1, not_found_config.duration) - elseif count >= not_found_config.threshold then - action.block_ip(ip, not_found_config) - return - end - end - end -end - -local add_count = function(shared_dict,key) - local count, _ = shared_dict:incr(key, 1) - if not count then - shared_dict:set(key, 1) - end -end - -local function count_req_status(is_attack) - local status = ngx.status - local req_count = ngx.shared.waf_req_count - add_count(req_count, "req_count") - if (status >= 400 and status < 500) then - add_count(req_count, "count_4xx") - end - if (status >= 500) then - add_count(req_count, "count_5xx") - end - if is_attack then - add_count(req_count, "attack_count") - end -end - -if config.is_waf_on() then - if ngx.ctx.is_waf_url then - return - end - count_not_found() - local is_attack = ngx.ctx.is_attack - - if not ngx.ctx.ip then - ngx.ctx.ip = utils.get_real_ip() - ngx.ctx.ip_location = utils.get_ip_location(ngx.ctx.ip) - end - - count_req_status(is_attack) - if is_attack then - write_req_log(is_attack) - end -end diff --git a/plugins/openresty/waf/rules/acl.json b/plugins/openresty/waf/rules/acl.json deleted file mode 100644 index 218d999e1..000000000 --- a/plugins/openresty/waf/rules/acl.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "rules": [ - { - "state": "on", - "name": "no cookie", - "conditions": [ - { - "field": "URL", - "pattern": "eq", - "rule": "/test/\\d+\\.html" - }, - { - "field": "Cookie", - "pattern": "eq", - "rule": "" - } - ], - "action": "deny", - "code": 403, - "res": "", - "ipBlock": "off", - "ipBlockTime": 60, - "description": "拦截不带Cookie的请求" - } - ] -} \ No newline at end of file diff --git a/plugins/openresty/waf/rules/args.json b/plugins/openresty/waf/rules/args.json deleted file mode 100644 index bc44ba4c1..000000000 --- a/plugins/openresty/waf/rules/args.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "rules": [ - { - "state": "on", - "name": "sqlInject1", - "rule": "select.+(from|limit)", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject2", - "rule": "(?:(union(.*?)select))", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject3", - "rule": "having|rongjitest", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject4", - "rule": "sleep\\((\\s*)(\\d*)(\\s*)\\)", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject5", - "rule": "benchmark\\((.*)\\,(.*)\\)", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject6", - "rule": "group\\s+by.+\\(", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject7", - "rule": "(?:from\\W+information_schema\\W)", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject8", - "rule": "(?:(?:current_)user|database|schema|connection_id)\\s*\\(", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject9", - "rule": "into(\\s+)+(?:dump|out)file\\s*", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject10", - "rule": "\\s+(or|xor|and)\\s+.*(=|<|>|'|\")", - "type": "sqlInject" - }, - { - "state": "on", - "name": "args1", - "rule": "xwork.MethodAccessor", - "type": "args", - "description": "Struts 恶意参数过滤" - }, - { - "state": "on", - "name": "args2", - "rule": "xwork\\.MethodAccessor", - "type": "args", - "description": "Struts 恶意参数过滤" - }, - { - "state": "on", - "name": "oneWordTrojan1", - "rule": "(?:define|eval|file_get_contents|include|require|require_once|shell_exec|phpinfo|system|passthru|preg_\\w+|execute|echo|print|print_r|var_dump|(fp)open|alert|showmodaldialog)\\(", - "type": "oneWordTrojan" - }, - { - "state": "on", - "name": "oneWordTrojan2", - "rule": "\\$_(GET|post|cookie|files|session|env|phplib|GLOBALS|SERVER)\\[", - "type": "oneWordTrojan" - }, - { - "state": "on", - "name": "protocolFilter1", - "rule": "(gopher|doc|php|glob|file|phar|zlib|ftp|ldap|dict|ogg|data)\\:\\/", - "type": "protocolFilter", - "description": "协议过滤" - }, - { - "state": "on", - "name": "dirFilter1", - "rule": "(?:etc\\/\\W*passwd)", - "type": "dirFilter" - }, - { - "state": "on", - "name": "dirFilter2", - "rule": "java\\.lang", - "type": "dirFilter" - }, - { - "state": "on", - "name": "xss1", - "rule": "\\<(iframe|script|body|img|layer|div|meta|style|base|object|input)", - "type": "xss" - }, - { - "state": "on", - "name": "xss2", - "rule": "(onmouseover|onerror|onload)\\=", - "type": "xss" - }, - { - "state": "on", - "name": "xss3", - "rule": "base64_decode\\(", - "type": "xss" - }, - { - "state": "on", - "name": "webshell1", - "rule": "/shell?cd+/tmp;\\s*rm+-rf\\+\\*;\\s*wget", - "type": "webshell" - }, - { - "state": "on", - "name": "phpExec1", - "rule": "/systembc/password.php", - "type": "phpExec" - }, - { - "state":"on", - "name": "scannerFilter1", - "rule":"(Acunetix-Aspect|Acunetix-Aspect-Password|Acunetix-Aspect-Queries|X-WIPP|X-RequestManager-Memo|X-Request-Memo|X-Scan-Memo)", - "type": "scannerFilter" - } - ] -} \ No newline at end of file diff --git a/plugins/openresty/waf/rules/ccurl.json b/plugins/openresty/waf/rules/ccurl.json deleted file mode 100644 index 544b7b4dd..000000000 --- a/plugins/openresty/waf/rules/ccurl.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - -} \ No newline at end of file diff --git a/plugins/openresty/waf/rules/cookie.json b/plugins/openresty/waf/rules/cookie.json deleted file mode 100644 index 147a05eec..000000000 --- a/plugins/openresty/waf/rules/cookie.json +++ /dev/null @@ -1,138 +0,0 @@ -{ - "rules": [ - { - "state": "on", - "name": "dirFilter1", - "rule": "\\.\\./", - "type": "dirFilter" - }, - { - "state": "on", - "name": "dirFilter2", - "rule": "\\:\\$", - "type": "dirFilter" - }, - { - "state": "on", - "name": "dirFilter3", - "rule": "\\$\\{", - "type": "dirFilter" - }, - { - "state": "on", - "name": "dirFilter4", - "rule": "(?:etc\\/\\W*passwd)", - "type": "dirFilter" - }, - { - "state": "on", - "name": "dirFilter5", - "rule": "java\\.lang", - "type": "dirFilter" - }, - { - "state": "on", - "name": "sqlInject1", - "rule": "select.+(from|limit)", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject2", - "rule": "(?:(union(.*?)select))", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject3", - "rule": "having|rongjitest", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject4", - "rule": "sleep\\((\\s*)(\\d*)(\\s*)\\)", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject5", - "rule": "benchmark\\((.*)\\,(.*)\\)", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject6", - "rule": "group\\s+by.+\\(", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject7", - "rule": "(?:from\\W+information_schema\\W)", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject8", - "rule": "(?:(?:current_)user|database|schema|connection_id)\\s*\\(", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject9", - "rule": "into(\\s+)+(?:dump|out)file\\s*", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject10", - "rule": "\\s+(or|xor|and)\\s+.*(=|<|>|'|\")", - "type": "sqlInject" - }, - { - "state": "on", - "name": "args1", - "rule": "xwork.MethodAccessor", - "type": "args", - "description": "Struts 恶意参数过滤" - }, - { - "state": "on", - "name": "args2", - "rule": "xwork\\.MethodAccessor", - "type": "args", - "description": "Struts 恶意参数过滤" - }, - { - "state": "on", - "name": "oneWordTrojan1", - "rule": "(?:define|eval|file_get_contents|include|require|require_once|shell_exec|phpinfo|system|passthru|preg_\\w+|execute|echo|print|print_r|var_dump|(fp)open|alert|showmodaldialog)\\(", - "type": "oneWordTrojan" - }, - { - "state": "on", - "name": "oneWordTrojan2", - "rule": "\\$_(GET|post|cookie|files|session|env|phplib|GLOBALS|SERVER)\\[", - "type": "oneWordTrojan" - }, - { - "state": "on", - "name": "protocolFilter1", - "rule": "(gopher|doc|php|glob|file|phar|zlib|ftp|ldap|dict|ogg|data)\\:\\/", - "type": "protocolFilter" - }, - { - "state":"on", - "name":"scannerFilter1", - "rule":"(CustomCookie|acunetixCookie)", - "type": "scannerFilter" - }, - { - "state": "on", - "name": "xss1", - "rule": "base64_decode\\(", - "type": "xss" - } - ] -} \ No newline at end of file diff --git a/plugins/openresty/waf/rules/defaultUaBlack.json b/plugins/openresty/waf/rules/defaultUaBlack.json deleted file mode 100644 index b5b984b8f..000000000 --- a/plugins/openresty/waf/rules/defaultUaBlack.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "rules": [ - { - "state": "on", - "name": "uaBlock1", - "rule": "HTTrack|Apache-HttpClient|harvest|audit|dirbuster|pangolin|nmap|sqln|hydra|Parser|libwww|BBBike|sqlmap|w3af|owasp|Nikto|fimap|havij|zmeu|BabyKrokodil|netsparker|httperf| SF/", - "type": "scannerFilter" - } - ] -} \ No newline at end of file diff --git a/plugins/openresty/waf/rules/defaultUrlBlack.json b/plugins/openresty/waf/rules/defaultUrlBlack.json deleted file mode 100644 index 4e189066f..000000000 --- a/plugins/openresty/waf/rules/defaultUrlBlack.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "rules": [ - { - "state": "on", - "rule": "\\.(htaccess|mysql_history|bash_history|DS_Store|idea|user\\.ini)", - "name": "dirFilter1", - "type": "dirFilter" - }, - { - "state": "on", - "rule": "\\.(bak|inc|old|mdb|sql|backup|java|class)$", - "name": "dirFilter2", - "type": "dirFilter" - }, - { - "state": "on", - "rule": "^/(vhost|bbs|host|wwwroot|www|site|root|backup|data|ftp|db|admin|website|web).*\\.(rar|sql|zip|tar\\.gz|tar)$", - "name": "dirFilter3", - "type": "dirFilter" - }, - { - "state": "on", - "rule": "java\\.lang", - "name": "dirFilter4", - "type": "dirFilter" - }, - { - "state": "on", - "rule": "/(hack|shell|spy|phpspy)\\.php$", - "name": "phpExec1", - "type": "phpExec" - }, - { - "state": "on", - "rule": "/(attachments|upimg|images|css|uploadfiles|html|uploads|templets|static|template|data|inc|forumdata|upload|includes|cache|avatar)/(\\\\w+).(php|jsp)", - "name": "phpExec2", - "type": "phpExec" - }, - { - "state": "on", - "rule": "(?:phpMyAdmin2|phpMyAdmin|phpmyadmin|dbadmin|pma|myadmin|admin|mysql)/scripts/setup%.php", - "name": "phpExec3", - "type": "phpExec" - }, - { - "state": "on", - "rule": "(?:define|eval|file_get_contents|include|require|require_once|shell_exec|phpinfo|system|passthru|preg_\\w+|execute|echo|print|print_r|var_dump|(fp)open|alert|showmodaldialog)\\(", - "name": "oneWordTrojan1", - "type": "oneWordTrojan" - }, - { - "state": "on", - "rule": "(phpmyadmin|jmx-console|jmxinvokerservlet)", - "name": "appFilter1", - "type": "appFilter" - }, - { - "state": "on", - "rule": "wp-includes/wlwmanifest.xml", - "name": "appFilter2", - "type": "appFilter" - }, - { - "state": "on", - "rule": "die(@md5(HelloThinkCMF))", - "name": "appFilter3", - "type": "appFilter" - }, - { - "state": "on", - "rule": "/boaform/admin/formLogin", - "name": "appFilter4", - "type": "appFilter" - }, - { - "state": "on", - "rule": "/password_change.cgi", - "name": "appFilter5", - "type": "appFilter" - }, - { - "state": "on", - "rule": "/service/extdirect", - "name": "appFilter6", - "type": "appFilter" - }, - { - "state": "on", - "rule": "/api/jsonws/invoke", - "name": "appFilter7", - "type": "appFilter" - }, - { - "state": "on", - "rule": "/jars/upload", - "name": "appFilter8", - "type": "appFilter" - }, - { - "state": "on", - "rule": "/example/tree/a/search", - "name": "appFilter9", - "type": "appFilter" - }, - { - "state": "on", - "rule": "/actuator/gateway/routes/hacktest", - "name": "appFilter10", - "type": "appFilter" - }, - { - "state": "on", - "rule": "/api/v1/method.callAnon/getPasswordPolicy", - "name": "appFilter11", - "type": "appFilter" - }, - { - "state": "on", - "rule": "/functionRouter", - "name": "appFilter12", - "type": "appFilter" - }, - { - "state": "on", - "rule": "(?:(union(.*?)select))", - "name": "sqlInject1", - "type": "sqlInject" - } - ] -} \ No newline at end of file diff --git a/plugins/openresty/waf/rules/fileExt.json b/plugins/openresty/waf/rules/fileExt.json deleted file mode 100644 index c0cc78781..000000000 --- a/plugins/openresty/waf/rules/fileExt.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "rules": [ - { - "state": "on", - "rule": "php", - "name": "php", - "type": "fileExt" - }, - { - "state": "on", - "rule": "jsp", - "name": "jsp", - "type": "fileExt" - }, - { - "state": "on", - "rule": "asp", - "name": "asp", - "type": "fileExt" - }, - { - "state": "on", - "rule": "exe", - "name": "exe", - "type": "fileExt" - }, - { - "state": "on", - "rule": "sh", - "name": "sh", - "type": "fileExt" - } - ] -} diff --git a/plugins/openresty/waf/rules/geoRestrict.json b/plugins/openresty/waf/rules/geoRestrict.json deleted file mode 100644 index 7769a81db..000000000 --- a/plugins/openresty/waf/rules/geoRestrict.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "rules": [ - { - "state": "on", - "name": "appFilter1", - "rule": "/TomcatBypass/Command/Base64", - "type": "appFilter" - }, - { - "state": "on", - "name": "appFilter2", - "rule": "j\\S*ndi\\S*:\\S*(?:dap|dns)\\S+", - "type": "appFilter" - }, - ] -} \ No newline at end of file diff --git a/plugins/openresty/waf/rules/header.json b/plugins/openresty/waf/rules/header.json deleted file mode 100644 index 1c1511c3c..000000000 --- a/plugins/openresty/waf/rules/header.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "rules": [ - { - "state": "on", - "name": "appFilter1", - "rule": "/TomcatBypass/Command/Base64", - "type": "appFilter" - }, - { - "state": "on", - "name": "appFilter2", - "rule": "j\\S*ndi\\S*:\\S*(?:dap|dns)\\S+", - "type": "appFilter" - }, - { - "state": "on", - "name": "scannerFilter1", - "rule": "(/acunetix-wvs-test-for-some-inexistent-file|netsparker|acunetix_wvs_security_test|AppScan|XSS@HERE)", - "type": "scannerFilter" - } - ] -} \ No newline at end of file diff --git a/plugins/openresty/waf/rules/ipBlack.json b/plugins/openresty/waf/rules/ipBlack.json deleted file mode 100644 index e9fc917d4..000000000 --- a/plugins/openresty/waf/rules/ipBlack.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "rules": [ - { - "name": "拦截IP", - "state": "on", - "type": "ipGroup", - "ipGroup": "test", - "description": "拦截IP" - } - ] -} \ No newline at end of file diff --git a/plugins/openresty/waf/rules/ipWhite.json b/plugins/openresty/waf/rules/ipWhite.json deleted file mode 100644 index b7fca6ded..000000000 --- a/plugins/openresty/waf/rules/ipWhite.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "rules": [ - ] -} \ No newline at end of file diff --git a/plugins/openresty/waf/rules/ip_group/test b/plugins/openresty/waf/rules/ip_group/test deleted file mode 100644 index 3fc5c1775..000000000 --- a/plugins/openresty/waf/rules/ip_group/test +++ /dev/null @@ -1 +0,0 @@ -192.168.1.1 \ No newline at end of file diff --git a/plugins/openresty/waf/rules/methodWhite.json b/plugins/openresty/waf/rules/methodWhite.json deleted file mode 100644 index 93fcad090..000000000 --- a/plugins/openresty/waf/rules/methodWhite.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "rules": [ - { - "state": "on", - "rule": "GET", - "name": "GET", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "POST", - "name": "POST", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "PUT", - "name": "PUT", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "DELETE", - "name": "DELETE", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "PATCH", - "name": "PATCH", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "HEAD", - "name": "HEAD", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "OPTIONS", - "name": "OPTIONS", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "TRACE", - "name": "TRACE", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "CONNECT", - "name": "CONNECT", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "PROPFIND", - "name": "PROPFIND", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "PROPPATCH", - "name": "PROPPATCH", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "MKCOL", - "name": "MKCOL", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "COPY", - "name": "COPY", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "MOVE", - "name": "MOVE", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "LOCK", - "name": "LOCK", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "UNLOCK", - "name": "UNLOCK", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "LINK", - "name": "LINK", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "UNLINK", - "name": "UNLINK", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "WRAPPED", - "name": "WRAPPED", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "PROPFIND", - "name": "PROPFIND", - "type": "httpMethod" - }, - { - "state": "on", - "rule": "SRARCH", - "name": "SRARCH", - "type": "httpMethod" - } - ] -} diff --git a/plugins/openresty/waf/rules/post.json b/plugins/openresty/waf/rules/post.json deleted file mode 100644 index 2722b88ff..000000000 --- a/plugins/openresty/waf/rules/post.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "rules": [ - { - "state": "on", - "name": "sqlInject1", - "rule": "select.+(from|limit)", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject2", - "rule": "(?:(union(.*?)select))", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject3", - "rule": "having|rongjitest", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject4", - "rule": "sleep\\((\\s*)(\\d*)(\\s*)\\)", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject5", - "rule": "benchmark\\((.*)\\,(.*)\\)", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject6", - "rule": "group\\s+by.+\\(", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject7", - "rule": "(?:from\\W+information_schema\\W)", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject8", - "rule": "(?:(?:current_)user|database|schema|connection_id)\\s*\\(", - "type": "sqlInject" - }, - { - "state": "on", - "name": "sqlInject9", - "rule": "into(\\s+)+(?:dump|out)file\\s*", - "type": "sqlInject" - }, - { - "state": "on", - "name": "args1", - "rule": "xwork.MethodAccessor", - "type": "args", - "description": "Struts 恶意参数过滤" - }, - { - "state": "on", - "name": "args2", - "rule": "xwork\\.MethodAccessor", - "type": "args", - "description": "Struts 恶意参数过滤" - }, - { - "state": "on", - "name": "oneWordTrojan1", - "rule": "(?:define|eval|file_get_contents|include|require|require_once|shell_exec|phpinfo|system|passthru|preg_\\w+|execute|echo|print|print_r|var_dump|(fp)open|alert|showmodaldialog)\\(", - "type": "oneWordTrojan" - }, - { - "state": "on", - "name": "protocolFilter1", - "rule": "(gopher|doc|php|glob|file|phar|zlib|ftp|ldap|dict|ogg|data)\\:\\/", - "type": "protocolFilter", - "description": "协议过滤" - }, - { - "state": "on", - "name": "oneWordTrojan2", - "rule": "\\$_(GET|post|cookie|files|session|env|phplib|GLOBALS|SERVER)\\[", - "type": "oneWordTrojan" - }, - { - "state": "on", - "name": "xss1", - "rule": "\\<(iframe|script|body|img|layer|div|meta|style|base|object|input)", - "type": "xss" - }, - { - "state": "on", - "name": "xss2", - "rule": "(onmouseover|onerror|onload)\\=", - "type": "xss" - }, - { - "state": "on", - "name": "xss3", - "rule": "base64_decode\\(", - "type": "xss" - }, - { - "state": "on", - "name": "dirFilter1", - "rule": "(?:etc\\/\\W*passwd)", - "type": "dirFilter" - }, - { - "state": "on", - "name": "dirFilter2", - "rule": "java\\.lang", - "type": "dirFilter" - } - ] -} \ No newline at end of file diff --git a/plugins/openresty/waf/rules/uaBlack.json b/plugins/openresty/waf/rules/uaBlack.json deleted file mode 100644 index 45b403002..000000000 --- a/plugins/openresty/waf/rules/uaBlack.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "rules": [ - { - "name": "UaBlack", - "state": "on", - "action": "deny", - "rule": "ua-blacklist", - "description": "测试" - } - ] -} \ No newline at end of file diff --git a/plugins/openresty/waf/rules/uaWhite.json b/plugins/openresty/waf/rules/uaWhite.json deleted file mode 100644 index f70da9920..000000000 --- a/plugins/openresty/waf/rules/uaWhite.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "rules": [] -} diff --git a/plugins/openresty/waf/rules/urlBlack.json b/plugins/openresty/waf/rules/urlBlack.json deleted file mode 100644 index 7d00d1f4a..000000000 --- a/plugins/openresty/waf/rules/urlBlack.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "rules": [] -} \ No newline at end of file diff --git a/plugins/openresty/waf/rules/urlWhite.json b/plugins/openresty/waf/rules/urlWhite.json deleted file mode 100644 index 7d00d1f4a..000000000 --- a/plugins/openresty/waf/rules/urlWhite.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "rules": [] -} \ No newline at end of file diff --git a/plugins/openresty/waf/test.lua b/plugins/openresty/waf/test.lua deleted file mode 100644 index 3318b9b17..000000000 --- a/plugins/openresty/waf/test.lua +++ /dev/null @@ -1,33 +0,0 @@ -function ip_to_int(ip) - local ip_int = 0 - for i, octet in ipairs({ ip:match("(%d+)%.(%d+)%.(%d+)%.(%d+)") }) do - ip_int = ip_int + tonumber(octet) * 256 ^ (4 - i) - end - return ip_int -end - ------- 示例 -local ip_address = "222.249.139.98" -local ip_number = ip_to_int(ip_address) -print(ip_number) - ---local geoip = require "lib.resty.maxminddb" ---local cjson = require("cjson") --- ---geoip.init("/Users/wangzhengkun/Downloads/blackIP.mmdb") --- ---local geo = geoip.lookup("165.154.132.251") --- ---print(cjson.encode(geo)) - ---local fileUtils = require "lib.file" ---local read_file2string = fileUtils.read_file2string --- ---local slideHtml = read_file2string("./html/" .. "slide.html") --- ---print(string.format(slideHtml, "1", "2")) - - ---local today = os.date("%Y-%m-%d") ---print(today) - diff --git a/plugins/openresty/waf/waf.json b/plugins/openresty/waf/waf.json deleted file mode 100644 index ab7668f0c..000000000 --- a/plugins/openresty/waf/waf.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "1Panel WAF", - "version": "1.0.0" -} \ No newline at end of file diff --git a/plugins/openresty/waf/waf.lua b/plugins/openresty/waf/waf.lua deleted file mode 100644 index 1b7075498..000000000 --- a/plugins/openresty/waf/waf.lua +++ /dev/null @@ -1,187 +0,0 @@ -local lib = require "lib" -local file_utils = require "file" -local config = require "config" -local cc = require "cc" -local utils = require "utils" -local cjson = require "cjson" - -local ipairs = ipairs -local sub_str = string.sub -local find_str = string.find -local split_str = utils.split -local encode = cjson.encode -local read_file2table = file_utils.read_file2table -local tonumber = tonumber -local date = os.date -local format_str = string.format - -local function get_website_key() - local s_name = ngx.var.server_name - local website_key = ngx.shared.waf:get(s_name) - if website_key then - return website_key - end - local websites = read_file2table(config.config_dir .. '/websites.json') - if not websites then - return s_name - end - for _, v in ipairs(websites) - do - for _, domain in ipairs(v['domains']) - do - if s_name == domain then - ngx.shared.waf:set(s_name, v['key'], 3600) - return v['key'] - end - end - end - if s_name == '_' then - s_name = "unknown" - end - return s_name -end - - - -local function init() - local ip = utils.get_real_ip() - ngx.ctx.ip = ip - local ua = utils.get_header("user-agent") - if not ua then - ua = "" - end - - ngx.ctx.ua = ua - ngx.ctx.ip_location = utils.get_ip_location(ip) - ngx.ctx.website_key = get_website_key() - ngx.ctx.method = ngx.req.get_method() - ngx.ctx.content_type = utils.get_header("content-type") - if ngx.ctx.content_type then - ngx.ctx.content_length = tonumber(utils.get_header("content-length")) - end - ngx.ctx.today = date("%Y-%m-%d") -end - -local function return_js(js_type) - ngx.header.content_type = "text/html;charset=utf8" - ngx.header.Cache_Control = "no-cache" - local host = ngx.var.scheme .. "://" .. ngx.var.host - local set_access_url = host .. "/set_access_token" - local secret = config.get_secret() - local key = ngx.md5(ngx.ctx.ip .. ngx.var.server_name .. ngx.ctx.website_key - .. ngx.ctx.ua .. ngx.ctx.today .. secret) - local value = ngx.md5(ngx.time() .. ngx.ctx.ip) - local js = config.get_html_res(js_type) - ngx.say(format_str(js, set_access_url, key, value)) - ngx.status = 200 - ngx.exit(200) -end - -local function return_json(data) - ngx.header.content_type = "application/json;" - ngx.header.Cache_Control = "no-cache" - ngx.status = 200 - ngx.say(data) - ngx.exit(200) -end - -local function waf_api() - local uri = ngx.var.uri - local prefix = sub_str(uri, 1, 15) - if find_str(prefix, "/set_access_token") then - local kvs = split_str(uri, "-") - if kvs[2] and kvs[3] then - cc.set_access_token(kvs[2], kvs[3]) - else - ngx.exit(444) - end - end - if uri == "/slide_check_" .. ngx.md5(ngx.ctx.ip) .. ".js" then - ngx.ctx.is_waf_url = true - return_js("slide_js") - end - - if uri == "/5s_check_" .. ngx.md5(ngx.ctx.ip) .. ".js" then - ngx.ctx.is_waf_url = true - return_js("five_second_js") - end - local method = ngx.req.get_method() - if method ~= 'POST' then - return false - end - if ngx.var.remote_addr ~= '127.0.0.1' then - return false - end - ngx.req.read_body() - local body_data = ngx.req.get_body_data() - if not body_data then - return false - end - local args - if body_data then - args = cjson.decode(body_data) - end - if args == nil or args.token == nil then - return false - end - if args.token ~= config.get_token() then - return false - end - ngx.ctx.is_waf_url = true - if uri == '/reload_waf_config' then - config.load_config_file() - ngx.exit(200) - end - if uri == '/get_block_ip' then - --TODO 从 redis 获取黑名单 - local block_ip_dict = ngx.shared.waf_block_ip - local data = block_ip_dict:get_keys(0) - return_json(encode(data)) - end - if uri == '/remove_block_ip' and args.ip then - local block_ip_dict = ngx.shared.waf_block_ip - block_ip_dict:delete(args.ip) - ngx.exit(200) - end -end - - -if config.is_waf_on() then - init() - waf_api() - - if ngx.ctx.website_key == "unknown" then - ngx.exit(403) - return - end - - if lib.is_white_ip() then - return true - end - lib.black_ip() - lib.default_ip_black() - - if lib.is_white_ua() then - return true - end - lib.black_ua() - lib.default_ua_black() - - --lib.cc_url() - lib.cc() - - if lib.is_white_url() then - return true - end - lib.black_url() - lib.default_url_black() - - lib.allow_location_check() - lib.method_check() - lib.acl() - --lib.bot_check() - lib.args_check() - lib.cookie_check() - lib.post_check() - lib.header_check() -end \ No newline at end of file diff --git a/plugins/openresty/waf/worker.lua b/plugins/openresty/waf/worker.lua deleted file mode 100644 index 96688f8e3..000000000 --- a/plugins/openresty/waf/worker.lua +++ /dev/null @@ -1,64 +0,0 @@ -local uuid = require 'resty.uuid' -local utils = require "utils" -local config = require "config" - -uuid.seed() - -local update_req_count = function() - local req_count = ngx.shared.waf_req_count - local req_count_update = req_count:get("req_count") or 0 - req_count:set("req_count", 0) - local count_4xx_update = req_count:get("count_4xx") or 0 - req_count:set("count_4xx", 0) - local count_5xx_update = req_count:get("count_5xx") or 0 - req_count:set("count_5xx", 0) - local attack_count_update = req_count:get("attack_count") or 0 - req_count:set("attack_count", 0) - - if req_count_update == 0 and count_4xx_update == 0 and count_5xx_update == 0 and attack_count_update == 0 then - return - end - - local today = ngx.today() - local wafdb = utils.get_wafdb(config.waf_db_path) - if not wafdb then - ngx.log(ngx.ERR, "get log db failed") - return - end - - wafdb:execute([[BEGIN TRANSACTION]]) - - local stmt_exist = wafdb:prepare("SELECT COUNT(*) FROM waf_stat WHERE day = ?") - stmt_exist:bind_values(today) - stmt_exist:step() - local count = stmt_exist:get_uvalues() - stmt_exist:finalize() - - local code = 0 - if count > 0 then - local stmt = wafdb:prepare("UPDATE waf_stat SET req_count = req_count + ?, count4xx = count4xx + ?, count5xx = count5xx + ?, attack_count = attack_count + ? WHERE day = ?") - stmt:bind_values(req_count_update, count_4xx_update, count_5xx_update, attack_count_update, today) - code = stmt:step() - stmt:finalize() - else - local stmt = wafdb:prepare("INSERT INTO waf_stat (day, req_count, count4xx, count5xx, attack_count,create_date) VALUES (?, ?, ?, ?, ?,DATETIME('now'))") - stmt:bind_values(today, req_count_update, count_4xx_update, count_5xx_update, attack_count_update) - code = stmt:step() - stmt:finalize() - end - - wafdb:execute([[COMMIT]]) - - --local error_msg = wafdb:errmsg() - --if error_msg then - -- ngx.log(ngx.ERR, "update waf_stat error ", error_msg .. " ") - --end -end - -if 0 == ngx.worker.id() then - local ok, err = ngx.timer.every(2, update_req_count) - if not ok then - ngx.log(ngx.ERR, "failed to create the timer: ", err) - return - end -end \ No newline at end of file