OpenResty连接redis

介绍

对于大多数语言开发，redis都是必备的，其中一个作用就是减少查询DB，提高系统查询效率。OpenResty也不例外，也会配合redis来使用。

Openresty连接mysql

用过java、C#等面向对象语言，对于重复代码肯定要封装，来隐藏设置连接池，取连接池和放连接池等操作。我们下面来看一个封装代码redis-util.lua：

local cjson = require "cjson"
local redis_c = require("resty.redis")

local ok, new_tab = pcall(require, "table.new")
if not ok or type(new_tab) ~= "function" then
    new_tab = function (narr, nrec) return {} end
end

local _M = new_tab(0, 54)

_M._VERSION = '0.07'

local mt = {__index = _M}


local ngx_log               = ngx.log
local debug                 = ngx.config.debug

local DEBUG                 = ngx.DEBUG
local CRIT                  = ngx.CRIT

local MAX_PORT              = 65535

-- redis host,default: 127.0.0.1
local host                  = '127.0.0.1'
-- redis port,default:6379
local port                  = 6379
-- redis库索引(默认0-15 共16个库)，默认的就是0库(建议用不同端口开不同实例或者用不同前缀，因为换库需要用select命令),default:0
local db_index              = 0
-- redis auth 认证的密码
local password              = nil
-- reids连接超时时间,default: 1000 (1s)
local keepalive             = 60000 --60s
-- redis连接池的最大闲置时间, default: 60000 (1m)
local pool_size             = 100
-- redis连接池大小, default: 100
local timeout             = 3000 --3s   --modify by hirryli


-- if res is ngx.null or nil or type(res) is table and all value is ngx.null return true else false
local function _is_null(res)
    if res == ngx.null or res ==nil then
        return true
    elseif type(res) == "table" then
        for _, v in pairs(res) do
            if v ~= ngx.null then
                return false
            end
        end
        return true
    end
    return false
end


local function _debug_err(msg,err)
    if debug then
        ngx_log(DEBUG, msg ,err)
    end
end

-- encapsulation redis connect
local function _connect_mod(self,redis)
    -- set timeout -- add by hirryli
    -- ngx.say("timeout:", timeout)
    if timeout then
        redis:set_timeout(timeout)
    else
        redis:set_timeout(3000)
    end

    local ok, err
    -- set redis unix socket
    if host:find("unix:/", 1, true) == 1 then
        ok, err = redis:connect(host)
        -- set redis host,port
    else
        ok, err = redis:connect(host, port)
    end
    if not ok or err then

        _debug_err("previous connection not finished,reason::",err)

        return nil, err
    end

    -- set auth
    if password then
        local times, err = redis:get_reused_times()

        if times == 0 then

            local ok, err = redis:auth(password)
            if not ok or err then
                _debug_err("failed to set redis password,reason::",err)
                return nil, err
            end
        elseif err then
            _debug_err( "failed to get this connect reused times,reason::",err)
            return nil, err
        end
    end

    if db_index >0 then
        local ok, err = redis:select(db_index)
        if not ok or err then
            _debug_err( "failed to select redis databse index to" , db_index , ",reason::",err)
            return nil, err
        end
    end

    return redis, nil
end


local function _init_connect()
    -- init redis
    local redis, err = redis_c:new()
    if not redis then
        _debug_err( "failed to init redis,reason::",err)
        return nil, err
    end

    -- get connect
    local ok, err = _connect_mod(self,redis)
    if not ok or err then
        _debug_err( "failed to create redis connection,reason::",err)
        return nil, err
    end
    return redis,nil
end

-- put it into the connection pool of size (default 100), with max idle time (default 60s)
local function _set_keepalive_mod(self,redis )
    return redis:set_keepalive(keepalive, pool_size)
end

-- encapsulation subscribe
function _M.subscribe( self, channel )

    -- init redis
    local redis, err = _init_connect()
    if not redis then
        _debug_err( "failed to init redis,reason::",err)
        return nil, err
    end
    -- sub channel
    local res, err = redis:subscribe(channel)

    if not res then
        _debug_err("failed to subscribe channel,reason:",err)
        return nil, err
    end
    ngx.say("1: subscribe: ", cjson.encode(res))
    local function do_read_func ( do_read )
        if do_read == nil or do_read == true then
            res, err = redis:read_reply()
            if not res then
                _debug_err("failed to read subscribe channel reply,reason:",err)
                return nil, err
            end
            return res
        end

        -- if do_read is false
        redis:unsubscribe(channel)
        _set_keepalive_mod(self,redis)
        return
    end

    return do_read_func
end

-- init pipeline,default cmds num is 4
function _M.init_pipeline(self, n)
    self._reqs = new_tab(n or 4, 0)
end

-- cancel pipeline
function _M.cancel_pipeline(self)
    self._reqs = nil
end

-- commit pipeline
function _M.commit_pipeline(self)
    -- get cache cmds
    local _reqs = rawget(self, "_reqs")
    if not _reqs then
        _debug_err("failed to commit pipeline,reason:no pipeline")
        return nil, "no pipeline"
    end

    self._reqs = nil

    -- init redis
    local redis, err = _init_connect()
    if not redis then
        _debug_err( "failed to init redis,reason::",err)
        return nil, err
    end

    redis:init_pipeline()

    --redis command like set/get ...
    for _, vals in ipairs(_reqs) do
        -- vals[1] is redis cmd
        local fun = redis[vals[1]]
        -- get params without cmd
        table.remove(vals , 1)
        -- invoke redis cmd
        fun(redis, unpack(vals))
    end

    -- commit pipeline
    local results, err = redis:commit_pipeline()
    if not results or err then
        _debug_err( "failed to commit pipeline,reason:",err)
        return {}, err
    end

    -- check null
    if _is_null(results) then
        results = {}
        ngx.log(ngx.WARN, "redis result is null")
    end

    -- put it into the connection pool
    _set_keepalive_mod(self,redis)

    -- if null set default value nil
    for i,value in ipairs(results) do
        if _is_null(value) then
            results[i] = nil
        end
    end

    return results, err
end

-- common method
local function do_command(self, cmd, ...)

    -- pipeline reqs
    local _reqs = rawget(self, "_reqs")
    if _reqs then
        -- append reqs
        _reqs[#_reqs + 1] = {cmd,...}
        return
    end

    -- init redis
    local redis, err = _init_connect()
    if not redis then
        _debug_err( "failed to init redis,reason::",err)
        return nil, err
    end

    -- exec redis cmd
    local method = redis[cmd]
    local result, err = method(redis, ...)
    if not result or err then
        return nil, err
    end

    -- check null
    if _is_null(result) then
        result = nil
    end

    -- put it into the connection pool
    local ok, err = _set_keepalive_mod(self,redis)
    if not ok or err then
        return nil, err
    end

    return result, nil
end

-- init options
function _M.new(self, opts)
    opts = opts or {}
    if (type(opts) ~= "table") then
        return nil, "user_config must be a table"
    end

    for k, v in pairs(opts) do
        if k == "host" then
            if type(v) ~= "string" then
                return nil, '"host" must be a string'
            end
            host = v
        elseif k == "port" then
            if type(v) ~= "number" then
                return nil, '"port" must be a number'
            end
            if v < 0 or v > MAX_PORT then
                return nil, ('"port" out of range 0~%s'):format(MAX_PORT)
            end
            port = v
        elseif k == "password" then
            if type(v) ~= "string" then
                return nil, '"password" must be a string'
            end
            password = v
        elseif k == "db_index" then
            if type(v) ~= "number" then
                return nil, '"db_index" must be a number'
            end
            if v < 0 then
                return nil, '"db_index" must be >= 0'
            end
            db_index = v
        elseif k == "timeout" then
            if type(v) ~= "number" or v < 0 then
                return nil, 'invalid "timeout"'
            end
            timeout = v
        elseif k == "keepalive" then
            if type(v) ~= "number" or v < 0 then
                return nil, 'invalid "keepalive"'
            end
            keepalive = v
        elseif k == "pool_size" then
            if type(v) ~= "number" or v < 0 then
                return nil, 'invalid "pool_size"'
            end
            pool_size = v
        end
    end

    if not (host and port) then
        return nil, "no redis server configured. "host"/"port" is required."
    end

    return setmetatable({},mt)
end

-- dynamic cmd
setmetatable(_M, {__index = function(self, cmd)
    local method =
    function (self, ...)
        return do_command(self, cmd, ...)
    end

    -- cache the lazily generated method in our
    -- module table
    _M[cmd] = method
    return method
end})

return _M
复制代码

上面代码github上的一位作者的封装redis-util,其中redis配置可以抽取出来放在配置文件中在OpenResty启动的时候进行加载。上面对redis的密码认证，连接创建，连接释放还有连接的池化等进行了封装。我们接下来举例几个简单的demo来对上面的封装进行测试下。

简单测试

set and get

test_set.lua

-- 依赖库
local redis = require "libs.redis-util"
-- 初始化
local red = redis:new();
-- 插入键值
local ok,err = red:set("dog","an cute animal")
-- 判断结果
if not ok then
    ngx.say("failed to set dog:",err)
    return
end
local res, err = red:get("dog")
if not res then
    ngx.say("failed to get dog: ", err)
    return
end
if res == ngx.null then
    ngx.say("dog not found.")
    return
end
-- 页面打印结果
ngx.say("get dog: ", res)
复制代码

上面的redis:new()等同于:

local red2 = redis:new({
                            host='127.0.0.1',
                            port=6379,
                            db_index=0,
                            password=nil,
                            timeout=1000,
                            keepalive=60000,
                            pool_size=100
                        });
复制代码

new(opts)中的参数opts是一个table，设置了redis连接相关配置，new返回的是redis-util自身. 可能你会问了，redis-util里面并没有get和set方法,怎么会调到的那？

这里面就用到动态语言可以动态生成方法的特性。此封装示例中是在运行时按需生成的动态方法，并且这种还是懒加载的方式。

-- dynamic cmd
setmetatable(_M, {__index = function(self, cmd)
    local method =
    function (self, ...)
        return do_command(self, cmd, ...)
    end

    -- cache the lazily generated method in our
    -- module table
    _M[cmd] = method
    return method
end})
复制代码

生成的动态方法的function内容为do_command方法：

-- common method
local function do_command(self, cmd, ...)

    -- pipeline reqs
    local _reqs = rawget(self, "_reqs")
    if _reqs then
        -- append reqs
        _reqs[#_reqs + 1] = {cmd,...}
        return
    end

    -- init redis
    local redis, err = _init_connect()
    if not redis then
        _debug_err( "failed to init redis,reason::",err)
        return nil, err
    end

    -- exec redis cmd
    local method = redis[cmd]
    local result, err = method(redis, ...)
    if not result or err then
        return nil, err
    end

    -- check null
    if _is_null(result) then
        result = nil
    end

    -- put it into the connection pool
    local ok, err = _set_keepalive_mod(self,redis)
    if not ok or err then
        return nil, err
    end

    return result, nil
end
复制代码

上面方法主要做的就是连接redis->执行命令->收回连接->返回命令执行结果。则我们在执行get或者set的时候会在运行时动态生成相应命令执行代码进行execute。可见实现了和resty.redis相同的命令执行。大部分命令都可以这种方式生成，除了一些特殊的命令需要额外处理，如:subscribe.

我们curl进行请求运行上面代码,会输出如下内容：

wukongdeMacBook-Pro:redis wukong$ curl http://127.0.0.1:8080/redis/set

get dog: an cute animal
复制代码

pipeline

test_pipeline.lua

local cjson = require "cjson"
local redis = require "libs.redis-util"

local red = redis:new();

red:init_pipeline()

red:set("cat", "Marry")
red:set("horse", "Bob")
red:get("cat")
red:get("horse")

local results, err = red:commit_pipeline()

if not results then
    ngx.say("failed to commit the pipelined requests: ", err)
    return
else
    ngx.say("pipeline",cjson.encode(results))
end
复制代码

批量操作封装中主要用到了init_pipeline和commit_pipeline接口,首先调用init_pipeline生成存储待执行的批量命令，然后将redis命令进行打包后批量执行。执行完成后释放连接到连接池。

我们curl进行请求运行上面代码,会输出如下内容：

wukongdeMacBook-Pro:redis wukong$ curl http://127.0.0.1:8080/redis/pipeline

pipeline["OK","OK","Marry","Bob"]
复制代码

subscribe and publish

test_sub_publish.lua

--[[发布-订阅简单示例]]
local cjson = require "cjson"
local redis = require "libs.redis-util"

local red = redis:new()
local red2 = redis:new()


local func = red:subscribe("dog")
if not func then
    ngx.say("1: failed to subscribe: ", err)
    return
end

-- 判断是否成功订阅
if not func then
    return nil
end

res, err = red2:publish("dog", "Hello")
if not res then
    ngx.say("2: failed to publish: ", err)
    return
end

ngx.say("2: publish: ", cjson.encode(res))

-- 获取值
local res, err = func() --func()=func(true)
-- 如果失败，取消订阅
if err then
    func(false)
end

-- 如果取到结果，进行页面输出
if res then
    ngx.say("receive: ", cjson.encode(res))
end

-- 循环获取
--[[while true do
    local res, err = func()
    if err then
        func(false)
    end
--    ...

end]]
复制代码

发布订阅的示例进行了简单subscribe,然后进行publish。正常我们应该使用注释的循环获取的方式获取数据。其实这个里面还是有一些问题:如下订阅代码所示：

-- encapsulation subscribe
function _M.subscribe( self, channel )

    -- init redis
    local redis, err = _init_connect()
    if not redis then
        _debug_err( "failed to init redis,reason::",err)
        return nil, err
    end
    -- sub channel
    local res, err = redis:subscribe(channel)

    if not res then
        _debug_err("failed to subscribe channel,reason:",err)
        return nil, err
    end
    ngx.say("1: subscribe: ", cjson.encode(res))
    local function do_read_func ( do_read )
        if do_read == nil or do_read == true then
            res, err = redis:read_reply()
            if not res then
                _debug_err("failed to read subscribe channel reply,reason:",err)
                return nil, err
            end
            return res
        end

        -- if do_read is false
        redis:unsubscribe(channel)
        _set_keepalive_mod(self,redis)
        return
    end

    return do_read_func
end
复制代码

问题就是当执行redis:unsubscribe(channel)以后，只是退订了对应的频道，并不会把当前接收到的数据清空。如果要想复用该连接，我们就需要保证清空当前读取到的数据。这样的话我们就需要在redis:unsubscribe(channel)代码下加上如下逻辑：

if not res then
    ngx.log(ngx.ERR, err)
    return
else
    -- redis 推送的消息格式，可能是
    -- {"message", ...} 或
    -- {"unsubscribe", $channel_name, $remain_channel_num}
    -- 如果返回的是前者，说明我们还在读取 Redis 推送过的数据
    if res[1] ~= "unsubscribe" then
        repeat
            -- 需要抽空已经接收到的消息
            res, err = red:read_reply()
            if not res then
                ngx.log(ngx.ERR, err)
                return
            end
        until res[1] == "unsubscribe"
    end
end
复制代码

这样的话后面再执行_set_keepalive_mod(self,redis)就可以安全复用连接了。此问题可具体查看OpenResty最佳实践 里面的具体说明。

我们curl进行请求运行上面代码,会输出如下内容：

wukongdeMacBook-Pro:redis wukong$ curl http://127.0.0.1:8080/redis/sub_publish

1: subscribe: ["subscribe","dog",1]

2: publish: 1

receive: ["message","dog","Hello"]
复制代码

script

test_script.lua

local redis = require "libs.redis-util"

local red = redis:new();
-- 插入键值
local ok,err = red:set("1","{"gid":2}")
-- 判断结果
if not ok then
    ngx.say("failed to set dog:",err)
    return
end

-- 插入键值
local ok,err = red:set("2","Hello World")
-- 判断结果
if not ok then
    ngx.say("failed to set dog:",err)
    return
end

local id = 1
local res, err = red:eval([[
        -- 注意在 Redis 执行脚本的时候，从 KEYS/ARGV 取出来的值类型为 string
        local info = redis.call('get', KEYS[1])
        info = cjson.decode(info)
        local g_id = info.gid

        local g_info = redis.call('get', g_id)
        return g_info
        ]], 1, id)

if not res then
    ngx.say("failed to get the group info: ", err)
    return
end

ngx.say("script:",res)
复制代码

上面简单演示了redis执行lua脚本的执行。redis引入lua脚本执行，主要是为了满足若干指令能够原子性执行，因为这使用原生命令是无法完成的。redis会单线程原子化执行lua脚本，执行中不会被其它请求所打断，没有竞争，那么也就不需要事务。另外lua脚本是可以复用的，对于客户端发送过来的脚本redis会保存起来。

我们curl进行请求运行上面代码,会输出如下内容：

wukongdeMacBook-Pro:redis wukong$ curl http://127.0.0.1:8080/redis/script

script:Hello World
复制代码

上面是几个简单的测试示例，对应的nginx.conf如下所示：

worker_processes 1;
error_log  logs/error.log;

events {
  worker_connections 1024;
}

http {
  lua_package_path "$prefix/lua/?.lua;$prefix/libs/?.lua;;";
  server {
    server_name localhost;
    listen 8080;
    charset utf-8;
    set $LESSON_ROOT lua/;
    error_log  logs/error.log;
    access_log logs/access.log;
    location /redis/set {
      default_type text/html;
      content_by_lua_file $LESSON_ROOT/test_set.lua;
    }

    location /redis/pipeline {
              default_type text/html;
              content_by_lua_file $LESSON_ROOT/test_pipeline.lua;
    }

    location /redis/script {
                  default_type text/html;
                  content_by_lua_file $LESSON_ROOT/test_script.lua;
    }

    location /redis/sub_publish {
                      default_type text/html;
                      content_by_lua_file $LESSON_ROOT/test_sub_publish.lua;
    }

  }

}
复制代码

总结

具体测试代码可查看github