openresty|openresty+redis实现 redis存储upstream redis|lua|balancer|openresty

任务描述

upstream的server实例存储在redis中，为openresty设置一个location，来触发加载server的动作
访问upstream时，轮询内部的server
不包括健康检测，我认为既然你的server都存储在redis里面了，健康检测也应该独立于openresty，发现坏节点，重新加载

技术描述 ngx.balancerngx.shared.DICTresty.redis

部署环境描述

http://10.161.4.61:80/后端实例
http://10.161.4.63:80/后端实例
http://10.161.4.63:8000/openresty
10.161.4.63 6379redis

步骤
部署redis 启动一个redis实例，没有特殊要求，能访问就行
设置openresty
【openresty|openresty+redis实现 redis存储upstream】修改配置文件

worker_processes3; error_log logs/error.log; events { worker_connections30; } http { lua_shared_dictservers 10m; #全局变量，存储upstream，根据你的后端节点数量，设置大小 lua_shared_dictnow_server 1m; #全局变量，用于轮询，记录现在是哪个节点服务 lua_shared_dictlens 1m; #全局变量，记录后端节点的数量 upstreamtomcat17 {#这个upstream用于对比测试 server10.161.4.63:80 weight=1; server10.161.4.61:80 weight=1; keepalive 10; } upstream test{#动态加载的upstream server192.168.0.18:80 weight=1; #此处必须有一个server，否则会启动失败，可以胡乱写一个 balancer_by_lua_block { local set_peer = require "set_peer" set_peer.run_peer() } keepalive 10; } #init_worker_by_lua_block { #local set_peer = require "set_peer" #set_peer.insert_server() #} server{ listen 8000; location ~ ^/api/([-_a-zA-Z0-9/]+) { access_by_lua_file work/lua/check_access.lua; content_by_lua_file work/lua/$1.lua; } location = /tomcat17 {#用于对比测试的location proxy_pass http://tomcat17/; } location = /insert {#用于加载server的location content_by_lua_block{ local set_peer = require "set_peer" set_peer.insert_server() } } location = /test_run {#用于测试的upstream proxy_pass http://test/; }location = /status { stub_status; } } }

lualib或者你自己的lua目录下创建，set_peer.lua 脚本（刚学lua没多久，写的有点水）

local balancer = require "ngx.balancer" local now_server = ngx.shared.now_server; local servers = ngx.shared.servers; local server_len = ngx.shared.lens; local redis = require "resty.redis"local _M = { _VERSION = '1.0' }local function con_redis() local red = redis:new() red:set_timeout(1000) local ok, err = red:connect("127.0.0.1", 6379)--注意此处 if not ok then ngx.log(ngx.ERR,"failed to connect: ", err) return ngx.exit(500) else return red end endlocal function set_peer(host,port) if not host and not port then ngx.log(ngx.ERR, " host and port can not be nil") return ngx.exit(500) end local ok, err = balancer.set_current_peer(host, port) if not ok then ngx.log(ngx.ERR, "failed to set the current peer: ", err) return ngx.exit(500) end endlocal function split( str,reps) local resultStrList = {} string.gsub(str,'[^'..reps..']+',function ( w ) table.insert(resultStrList,w) end) return resultStrList endlocal function test_server() local value, err = servers:get(1) ifnot value then ngx.log(ngx.ERR, "no server found: ", err) ngx.status=ngx.HTTP_INTERNAL_SERVER_ERROR return ngx.say("no server found") else return true end endlocal function fail_say( fail_str ) -- body ngx.log(ngx.ERR, fail_str) ngx.status = ngx.HTTP_INTERNAL_SERVER_ERROR return ngx.say(fail_str) endlocal function get_lens() -- body local len, flags = server_len:get("len") if not len then local fail_str = "failed to get len "..flags fail_say(fail_str) end return len endlocal function get_nums() local value = https://www.it610.com/article/now_server:get("nums") if not value then local fail_str = 'resetnumsto 1' ngx.log(ngx.ERR, fail_str) local ok, err = now_server:set("nums",1) if not ok then fail_str = 'failed to set nums '..err fail_say(fail_str) else return -1 end end return value endlocal function round_server() -- body local value = https://www.it610.com/article/get_nums() if not value then return end --ngx.log(ngx.ERR,'---->','ago',value) local len = get_lens() if not len then return end if value+1 >= len then local ok, err = now_server:set("nums",1) if not ok then fail_str = 'failed to set nums '..err fail_say(fail_str) else return 1 end else local ok, err = now_server:set( "nums", 2+value ) if not ok then fail_str = 'failed to set nums '..err fail_say(fail_str) else return 2+value end end endlocal functionhandle_redis(red) if not red then return end local res, err = red:smembers("servers") if not res then ngx.log(ngx.ERR,"failed to smembers servers: ", err) return ngx.exit(500) else return res endendfunction _M.insert_server() -- body local red = con_redis() if not red then return end local red_result = handle_redis(red) --ngx.log(ngx.ERR,'-->',red_result[1]) if not red_result then red:close() return end local nums = 0 fork,v in pairs(red_result) do local resultStrList = split(v,':') --ngx.log(ngx.ERR,'-->', resultStrList[1], resultStrList[2]) ifresultStrList[1] andresultStrList[2] then local res, err = servers:set(2*k-1, resultStrList[1]) if not res then ngx.log(ngx.ERR,"failed to set servers first: ", err) red:close() return ngx.exit(500) end --ngx.log(ngx.ERR,'-->',2*k-1,'--->',resultStrList[1]) --ngx.log(ngx.ERR,'-->',2*k,'--->',resultStrList[2]) local res1, err1 = servers:set(2*k, resultStrList[2]) if not res1 then ngx.log(ngx.ERR,"failed to set servers second: ", err1) local res2, err2 = servers:rpop(k) if not res2 then ngx.log(ngx.ERR,"failed to rpop servers first: ", err2) end red:close() return ngx.exit(500) else nums = nums+1 --ngx.log(ngx.ERR,'-->', nums) end end end local res, err = server_len:set('len', nums*2) if not res then ngx.log(ngx.ERR,"failed to set servers first: ", err) return ngx.exit(500) else ngx.say('ok,all is ', nums) return true end endfunction _M.run_peer() local ok = test_server() if not ok then return end local len = round_server() if not len then return end ngx.log(ngx.ERR,'--->', len)--可以去掉，调试时使用 local host ,err = servers:get(len) if not host then ngx.log(ngx.ERR,"failed to get server-host: ", err) return ngx.exit(500) end --ngx.log(ngx.ERR,"get server: ", host, '----', len+1) local port ,err = servers:get(len+1) if not port then ngx.log(ngx.ERR,"failed to get server-port: ", err) return ngx.exit(500) end ngx.log(ngx.ERR, "set peer ", host, ':', port) --可以去掉，调试时使用 set_peer(host, port) endreturn _M

重启openresty 重启一下openresty，启动时监控一下errlog,确保启动成功
在redis内插入upstream的server

SADD servers '10.161.4.63:80' '10.161.4.61:80'

动态加载 curl -X GET http://10.161.4.63:8000/insert
正常的时候会得到 OK，all is 2 的句子
访问测试
连续两次执行，curl -X GET http://10.161.4.63:8000/test_run，查看结果是不是轮询

openresty|openresty+redis实现 redis存储upstream

文章图片

简单的压力测试

文章图片

文章图片

简单测试了一下，觉得性能差不多，仅供参考呀
----------------------------------------------------------------------------------------------------------------------------------
就写到这了，如果有问题，欢迎来qq群 630300475 一起聊聊