介绍
对于大多数语言开发,redis都是必备的,其中一个作用就是减少查询DB,提高系统查询效率。OpenResty也不例外,也会配合redis来使用。
Openresty连接mysql
用过java、C#等面向对象语言,对于重复代码肯定要封装,来隐藏设置连接池,取连接池和放连接池等操作。我们下面来看一个封装代码redis-util.lua:
local cjson = require "cjson"
local redis_c = require("resty.redis")
local ok, new_tab = pcall(require, "table.new")
if not ok or type(new_tab) ~= "function" then
new_tab = function (narr, nrec) return {} end
end
local _M = new_tab(0, 54)
_M._VERSION = '0.07'
local mt = {__index = _M}
local ngx_log = ngx.log
local debug = ngx.config.debug
local DEBUG = ngx.DEBUG
local CRIT = ngx.CRIT
local MAX_PORT = 65535
-- redis host,default: 127.0.0.1
local host = '127.0.0.1'
-- redis port,default:6379
local port = 6379
-- redis库索引(默认0-15 共16个库),默认的就是0库(建议用不同端口开不同实例或者用不同前缀,因为换库需要用select命令),default:0
local db_index = 0
-- redis auth 认证的密码
local password = nil
-- reids连接超时时间,default: 1000 (1s)
local keepalive = 60000 --60s
-- redis连接池的最大闲置时间, default: 60000 (1m)
local pool_size = 100
-- redis连接池大小, default: 100
local timeout = 3000 --3s --modify by hirryli
-- if res is ngx.null or nil or type(res) is table and all value is ngx.null return true else false
local function _is_null(res)
if res == ngx.null or res ==nil then
return true
elseif type(res) == "table" then
for _, v in pairs(res) do
if v ~= ngx.null then
return false
end
end
return true
end
return false
end
local function _debug_err(msg,err)
if debug then
ngx_log(DEBUG, msg ,err)
end
end
-- encapsulation redis connect
local function _connect_mod(self,redis)
-- set timeout -- add by hirryli
-- ngx.say("timeout:", timeout)
if timeout then
redis:set_timeout(timeout)
else
redis:set_timeout(3000)
end
local ok, err
-- set redis unix socket
if host:find("unix:/", 1, true) == 1 then
ok, err = redis:connect(host)
-- set redis host,port
else
ok, err = redis:connect(host, port)
end
if not ok or err then
_debug_err("previous connection not finished,reason::",err)
return nil, err
end
-- set auth
if password then
local times, err = redis:get_reused_times()
if times == 0 then
local ok, err = redis:auth(password)
if not ok or err then
_debug_err("failed to set redis password,reason::",err)
return nil, err
end
elseif err then
_debug_err( "failed to get this connect reused times,reason::",err)
return nil, err
end
end
if db_index >0 then
local ok, err = redis:select(db_index)
if not ok or err then
_debug_err( "failed to select redis databse index to" , db_index , ",reason::",err)
return nil, err
end
end
return redis, nil
end
local function _init_connect()
-- init redis
local redis, err = redis_c:new()
if not redis then
_debug_err( "failed to init redis,reason::",err)
return nil, err
end
-- get connect
local ok, err = _connect_mod(self,redis)
if not ok or err then
_debug_err( "failed to create redis connection,reason::",err)
return nil, err
end
return redis,nil
end
-- put it into the connection pool of size (default 100), with max idle time (default 60s)
local function _set_keepalive_mod(self,redis )
return redis:set_keepalive(keepalive, pool_size)
end
-- encapsulation subscribe
function _M.subscribe( self, channel )
-- init redis
local redis, err = _init_connect()
if not redis then
_debug_err( "failed to init redis,reason::",err)
return nil, err
end
-- sub channel
local res, err = redis:subscribe(channel)
if not res then
_debug_err("failed to subscribe channel,reason:",err)
return nil, err
end
ngx.say("1: subscribe: ", cjson.encode(res))
local function do_read_func ( do_read )
if do_read == nil or do_read == true then
res, err = redis:read_reply()
if not res then
_debug_err("failed to read subscribe channel reply,reason:",err)
return nil, err
end
return res
end
-- if do_read is false
redis:unsubscribe(channel)
_set_keepalive_mod(self,redis)
return
end
return do_read_func
end
-- init pipeline,default cmds num is 4
function _M.init_pipeline(self, n)
self._reqs = new_tab(n or 4, 0)
end
-- cancel pipeline
function _M.cancel_pipeline(self)
self._reqs = nil
end
-- commit pipeline
function _M.commit_pipeline(self)
-- get cache cmds
local _reqs = rawget(self, "_reqs")
if not _reqs then
_debug_err("failed to commit pipeline,reason:no pipeline")
return nil, "no pipeline"
end
self._reqs = nil
-- init redis
local redis, err = _init_connect()
if not redis then
_debug_err( "failed to init redis,reason::",err)
return nil, err
end
redis:init_pipeline()
--redis command like set/get ...
for _, vals in ipairs(_reqs) do
-- vals[1] is redis cmd
local fun = redis[vals[1]]
-- get params without cmd
table.remove(vals , 1)
-- invoke redis cmd
fun(redis, unpack(vals))
end
-- commit pipeline
local results, err = redis:commit_pipeline()
if not results or err then
_debug_err( "failed to commit pipeline,reason:",err)
return {}, err
end
-- check null
if _is_null(results) then
results = {}
ngx.log(ngx.WARN, "redis result is null")
end
-- put it into the connection pool
_set_keepalive_mod(self,redis)
-- if null set default value nil
for i,value in ipairs(results) do
if _is_null(value) then
results[i] = nil
end
end
return results, err
end
-- common method
local function do_command(self, cmd, ...)
-- pipeline reqs
local _reqs = rawget(self, "_reqs")
if _reqs then
-- append reqs
_reqs[#_reqs + 1] = {cmd,...}
return
end
-- init redis
local redis, err = _init_connect()
if not redis then
_debug_err( "failed to init redis,reason::",err)
return nil, err
end
-- exec redis cmd
local method = redis[cmd]
local result, err = method(redis, ...)
if not result or err then
return nil, err
end
-- check null
if _is_null(result) then
result = nil
end
-- put it into the connection pool
local ok, err = _set_keepalive_mod(self,redis)
if not ok or err then
return nil, err
end
return result, nil
end
-- init options
function _M.new(self, opts)
opts = opts or {}
if (type(opts) ~= "table") then
return nil, "user_config must be a table"
end
for k, v in pairs(opts) do
if k == "host" then
if type(v) ~= "string" then
return nil, '"host" must be a string'
end
host = v
elseif k == "port" then
if type(v) ~= "number" then
return nil, '"port" must be a number'
end
if v < 0 or v > MAX_PORT then
return nil, ('"port" out of range 0~%s'):format(MAX_PORT)
end
port = v
elseif k == "password" then
if type(v) ~= "string" then
return nil, '"password" must be a string'
end
password = v
elseif k == "db_index" then
if type(v) ~= "number" then
return nil, '"db_index" must be a number'
end
if v < 0 then
return nil, '"db_index" must be >= 0'
end
db_index = v
elseif k == "timeout" then
if type(v) ~= "number" or v < 0 then
return nil, 'invalid "timeout"'
end
timeout = v
elseif k == "keepalive" then
if type(v) ~= "number" or v < 0 then
return nil, 'invalid "keepalive"'
end
keepalive = v
elseif k == "pool_size" then
if type(v) ~= "number" or v < 0 then
return nil, 'invalid "pool_size"'
end
pool_size = v
end
end
if not (host and port) then
return nil, "no redis server configured. "host"/"port" is required."
end
return setmetatable({},mt)
end
-- dynamic cmd
setmetatable(_M, {__index = function(self, cmd)
local method =
function (self, ...)
return do_command(self, cmd, ...)
end
-- cache the lazily generated method in our
-- module table
_M[cmd] = method
return method
end})
return _M
复制代码
上面代码github上的一位作者的封装redis-util,其中redis配置可以抽取出来放在配置文件中在OpenResty启动的时候进行加载。上面对redis的密码认证,连接创建,连接释放还有连接的池化等进行了封装。我们接下来举例几个简单的demo来对上面的封装进行测试下。
简单测试
set and get
test_set.lua
-- 依赖库
local redis = require "libs.redis-util"
-- 初始化
local red = redis:new();
-- 插入键值
local ok,err = red:set("dog","an cute animal")
-- 判断结果
if not ok then
ngx.say("failed to set dog:",err)
return
end
local res, err = red:get("dog")
if not res then
ngx.say("failed to get dog: ", err)
return
end
if res == ngx.null then
ngx.say("dog not found.")
return
end
-- 页面打印结果
ngx.say("get dog: ", res)
复制代码
上面的redis:new()等同于:
local red2 = redis:new({
host='127.0.0.1',
port=6379,
db_index=0,
password=nil,
timeout=1000,
keepalive=60000,
pool_size=100
});
复制代码
new(opts)中的参数opts是一个table,设置了redis连接相关配置,new返回的是redis-util自身. 可能你会问了,redis-util里面并没有get和set方法,怎么会调到的那?
这里面就用到动态语言可以动态生成方法的特性。此封装示例中是在运行时按需生成的动态方法,并且这种还是懒加载的方式。
-- dynamic cmd
setmetatable(_M, {__index = function(self, cmd)
local method =
function (self, ...)
return do_command(self, cmd, ...)
end
-- cache the lazily generated method in our
-- module table
_M[cmd] = method
return method
end})
复制代码
生成的动态方法的function内容为do_command方法:
-- common method
local function do_command(self, cmd, ...)
-- pipeline reqs
local _reqs = rawget(self, "_reqs")
if _reqs then
-- append reqs
_reqs[#_reqs + 1] = {cmd,...}
return
end
-- init redis
local redis, err = _init_connect()
if not redis then
_debug_err( "failed to init redis,reason::",err)
return nil, err
end
-- exec redis cmd
local method = redis[cmd]
local result, err = method(redis, ...)
if not result or err then
return nil, err
end
-- check null
if _is_null(result) then
result = nil
end
-- put it into the connection pool
local ok, err = _set_keepalive_mod(self,redis)
if not ok or err then
return nil, err
end
return result, nil
end
复制代码
上面方法主要做的就是连接redis->执行命令->收回连接->返回命令执行结果。则我们在执行get或者set的时候会在运行时动态生成相应命令执行代码进行execute。可见实现了和resty.redis相同的命令执行。大部分命令都可以这种方式生成,除了一些特殊的命令需要额外处理,如:subscribe.
我们curl进行请求运行上面代码,会输出如下内容:
wukongdeMacBook-Pro:redis wukong$ curl http://127.0.0.1:8080/redis/set
get dog: an cute animal
复制代码
pipeline
test_pipeline.lua
local cjson = require "cjson"
local redis = require "libs.redis-util"
local red = redis:new();
red:init_pipeline()
red:set("cat", "Marry")
red:set("horse", "Bob")
red:get("cat")
red:get("horse")
local results, err = red:commit_pipeline()
if not results then
ngx.say("failed to commit the pipelined requests: ", err)
return
else
ngx.say("pipeline",cjson.encode(results))
end
复制代码
批量操作封装中主要用到了init_pipeline和commit_pipeline接口,首先调用init_pipeline生成存储待执行的批量命令,然后将redis命令进行打包后批量执行。执行完成后释放连接到连接池。
我们curl进行请求运行上面代码,会输出如下内容:
wukongdeMacBook-Pro:redis wukong$ curl http://127.0.0.1:8080/redis/pipeline
pipeline["OK","OK","Marry","Bob"]
复制代码
subscribe and publish
test_sub_publish.lua
--[[发布-订阅简单示例]]
local cjson = require "cjson"
local redis = require "libs.redis-util"
local red = redis:new()
local red2 = redis:new()
local func = red:subscribe("dog")
if not func then
ngx.say("1: failed to subscribe: ", err)
return
end
-- 判断是否成功订阅
if not func then
return nil
end
res, err = red2:publish("dog", "Hello")
if not res then
ngx.say("2: failed to publish: ", err)
return
end
ngx.say("2: publish: ", cjson.encode(res))
-- 获取值
local res, err = func() --func()=func(true)
-- 如果失败,取消订阅
if err then
func(false)
end
-- 如果取到结果,进行页面输出
if res then
ngx.say("receive: ", cjson.encode(res))
end
-- 循环获取
--[[while true do
local res, err = func()
if err then
func(false)
end
-- ...
end]]
复制代码
发布订阅的示例进行了简单subscribe,然后进行publish。正常我们应该使用注释的循环获取的方式获取数据。其实这个里面还是有一些问题:如下订阅代码所示:
-- encapsulation subscribe
function _M.subscribe( self, channel )
-- init redis
local redis, err = _init_connect()
if not redis then
_debug_err( "failed to init redis,reason::",err)
return nil, err
end
-- sub channel
local res, err = redis:subscribe(channel)
if not res then
_debug_err("failed to subscribe channel,reason:",err)
return nil, err
end
ngx.say("1: subscribe: ", cjson.encode(res))
local function do_read_func ( do_read )
if do_read == nil or do_read == true then
res, err = redis:read_reply()
if not res then
_debug_err("failed to read subscribe channel reply,reason:",err)
return nil, err
end
return res
end
-- if do_read is false
redis:unsubscribe(channel)
_set_keepalive_mod(self,redis)
return
end
return do_read_func
end
复制代码
问题就是当执行redis:unsubscribe(channel)以后,只是退订了对应的频道,并不会把当前接收到的数据清空。如果要想复用该连接,我们就需要保证清空当前读取到的数据。这样的话我们就需要在redis:unsubscribe(channel)代码下加上如下逻辑:
if not res then
ngx.log(ngx.ERR, err)
return
else
-- redis 推送的消息格式,可能是
-- {"message", ...} 或
-- {"unsubscribe", $channel_name, $remain_channel_num}
-- 如果返回的是前者,说明我们还在读取 Redis 推送过的数据
if res[1] ~= "unsubscribe" then
repeat
-- 需要抽空已经接收到的消息
res, err = red:read_reply()
if not res then
ngx.log(ngx.ERR, err)
return
end
until res[1] == "unsubscribe"
end
end
复制代码
这样的话后面再执行_set_keepalive_mod(self,redis)就可以安全复用连接了。此问题可具体查看OpenResty最佳实践 里面的具体说明。
我们curl进行请求运行上面代码,会输出如下内容:
wukongdeMacBook-Pro:redis wukong$ curl http://127.0.0.1:8080/redis/sub_publish
1: subscribe: ["subscribe","dog",1]
2: publish: 1
receive: ["message","dog","Hello"]
复制代码
script
test_script.lua
local redis = require "libs.redis-util"
local red = redis:new();
-- 插入键值
local ok,err = red:set("1","{"gid":2}")
-- 判断结果
if not ok then
ngx.say("failed to set dog:",err)
return
end
-- 插入键值
local ok,err = red:set("2","Hello World")
-- 判断结果
if not ok then
ngx.say("failed to set dog:",err)
return
end
local id = 1
local res, err = red:eval([[
-- 注意在 Redis 执行脚本的时候,从 KEYS/ARGV 取出来的值类型为 string
local info = redis.call('get', KEYS[1])
info = cjson.decode(info)
local g_id = info.gid
local g_info = redis.call('get', g_id)
return g_info
]], 1, id)
if not res then
ngx.say("failed to get the group info: ", err)
return
end
ngx.say("script:",res)
复制代码
上面简单演示了redis执行lua脚本的执行。redis引入lua脚本执行,主要是为了满足若干指令能够原子性执行,因为这使用原生命令是无法完成的。redis会单线程原子化执行lua脚本,执行中不会被其它请求所打断,没有竞争,那么也就不需要事务。另外lua脚本是可以复用的,对于客户端发送过来的脚本redis会保存起来。
我们curl进行请求运行上面代码,会输出如下内容:
wukongdeMacBook-Pro:redis wukong$ curl http://127.0.0.1:8080/redis/script
script:Hello World
复制代码
上面是几个简单的测试示例,对应的nginx.conf如下所示:
worker_processes 1;
error_log logs/error.log;
events {
worker_connections 1024;
}
http {
lua_package_path "$prefix/lua/?.lua;$prefix/libs/?.lua;;";
server {
server_name localhost;
listen 8080;
charset utf-8;
set $LESSON_ROOT lua/;
error_log logs/error.log;
access_log logs/access.log;
location /redis/set {
default_type text/html;
content_by_lua_file $LESSON_ROOT/test_set.lua;
}
location /redis/pipeline {
default_type text/html;
content_by_lua_file $LESSON_ROOT/test_pipeline.lua;
}
location /redis/script {
default_type text/html;
content_by_lua_file $LESSON_ROOT/test_script.lua;
}
location /redis/sub_publish {
default_type text/html;
content_by_lua_file $LESSON_ROOT/test_sub_publish.lua;
}
}
}
复制代码
总结
具体测试代码可查看github