Redis fixed-length queue (support redis cluster)

Preface

In some business scenarios, it is necessary to implement an atomic, reliable and distributed fixed-length queue. It must have the following functions:

• The queue has an expiration date.
• The maximum length of the queue is n, and when the elements are full, it refuses to add new data.
• The maximum length of the queue is n. When the elements are full, the oldest data will be popped and new data will be accepted.
• Distributed api can be accessed and used at low cost.
• Support redis cluster

Implementation analysis

• Use redis to achieve
• Use expire mechanism to achieve expiration date
• Use lua scripts to achieve CAS atomicity
• Use a single key to ensure that the redis cluster supports the eval command

Source code

package redistool

import (
	"github.com/fwhezfwhez/errorx"
	"github.com/garyburd/redigo/redis"
)

// 定长list
type RedisLimitList struct {
    
    
	expireSeconds int
	maxLen        int
	scheme        int // 1时，list打满时，将拒绝新的push。 2时，list打满时，将pop掉一个最早的，再将新的压进来。
}

func NewLimitList(expireSeconds int, maxLen int, scheme int) RedisLimitList {
    
    
	return RedisLimitList{
    
    
		expireSeconds: expireSeconds,
		maxLen:        maxLen,
		scheme:        scheme,
	}
}

var BeyondErr = errorx.NewServiceError("超出list最大长度限制", 1)

// key list的key
// value 压入的值
// maxLength 最大长度
func (rll RedisLimitList) LPush(conn redis.Conn, key string, value []byte) (int, error) {
    
    
	return rll.LPushScheme1(conn, key, value)
}

// key list的key
// value 压入的值
// maxLength 最大长度
// list打满后，将不再接受新的数据，除非随后pop腾出了位置。
func (rll RedisLimitList) LPushScheme1(conn redis.Conn, key string, value []byte) (int, error) {
    
    
	var keyNum = 1 // eval的key的数量,为1
	var arg1 = value

	var arg2 = rll.maxLen        // 队列最大长度
	var arg3 = rll.expireSeconds // 队列key失效时间
	var scriptf = `
    local num = redis.call('llen',KEYS[1]);
    if tonumber(ARGV[2])>0 and tonumber(num) >= tonumber(ARGV[2]) then
        return -3
    end
    redis.call('lpush',KEYS[1],ARGV[1])
    if tonumber(ARGV[3]) > 0 then
        redis.call('expire', KEYS[1], ARGV[3])
    end
    local result = redis.call('llen',KEYS[1])
    return result
`

	vint, e := redis.Int(conn.Do("eval", scriptf, keyNum, key, arg1, arg2, arg3))
	if e != nil {
    
    
		return 0, errorx.Wrap(e)
	}
	if vint == -3 {
    
    
		return rll.maxLen, BeyondErr
	}
	return vint, nil
}

// key list的key
// value 压入的值
// maxLength 最大长度
// list打满后，将Pop出最老的，再push进新数据
func (rll RedisLimitList) LPushScheme2(conn redis.Conn, key string, value []byte) (int, error) {
    
    
	var keyNum = 1 // eval的key的数量,为1
	var arg1 = value

	var arg2 = rll.maxLen        // 队列最大长度
	var arg3 = rll.expireSeconds // 队列key失效时间
	var scriptf = `
    local num = redis.call('llen',KEYS[1]);
    if tonumber(ARGV[2])>0 and tonumber(num) >= tonumber(ARGV[2]) then
        redis.call('rpop', KEYS[1])
    end
    redis.call('lpush',KEYS[1],ARGV[1])
    if tonumber(ARGV[3]) > 0 then
        redis.call('expire', KEYS[1], ARGV[3])
    end
    local result = redis.call('llen',KEYS[1])
    return result
`

	vint, e := redis.Int(conn.Do("eval", scriptf, keyNum, key, arg1, arg2, arg3))
	if e != nil {
    
    
		return 0, errorx.Wrap(e)
	}

	return vint, nil
}

func (rll RedisLimitList) LLen(conn redis.Conn, key string) int {
    
    
	l, e := redis.Int(conn.Do("llen", key))

	if e != nil {
    
    
		return 0
	}

	return l
}

func (rll RedisLimitList) LRANGE(conn redis.Conn, key string, start int, stop int) ([] []byte, error) {
    
    
	rsI, e := conn.Do("lrange", key, start, stop)
	//fmt.Printf("%s\n", reflect.TypeOf(rsI).Name())
	//fmt.Printf("%v\n", rsI)
	return redis.ByteSlices(rsI, e)
}

func (rll RedisLimitList) RPOP(conn redis.Conn, key string) ([]byte, error) {
    
    
	return redis.Bytes(conn.Do("rpop", key))
}

Test, use case

package redistool

import (
	"fmt"
	"testing"
)

func TestLimitList(t *testing.T) {
    
    
	var ll =NewLimitList(20, 3,1)
	conn := RedisPool.Get()
	defer conn.Close()
	// lenth, e := ll.LPushScheme2(conn, "test_limit_list", []byte(fmt.Sprintf("msg%d", i)))

	lenth, e := ll.LPushScheme1(conn, "test_limit_list", []byte(fmt.Sprintf("msg%d", i)))
	if e != nil && e != BeyondErr {
    
    
		if e == BeyondErr {
    
    
			fmt.Println(lenth, e.Error())
			return
		}
		panic(e)
	}
	fmt.Println(lenth)

	rs, e := ll.LRANGE(conn, "test_limit_list", 0, -1)
	if e != nil {
    
    
		panic(e)
	}
	for _, v := range rs {
    
    
		fmt.Println(string(v))
	}
}

Output

• No matter how many times it is inserted in 20 seconds, these three values ​​are the same

msg2
msg1
msg0

If you use scheme2 (pop the oldest to insert the newest), then output

• Inserted 5 times, the last 2 times were too long, so msg0 and msg1 were ejected

msg4
msg3
msg2