布隆过滤器调研与设计Go实现

var bloomInstance *RedisBloomFilter

const RedisMaxLength = 8 * 512 * 1024 * 1024

type BloomFilter interface {
   Put([]byte) error
   PutString(string) error
   Has([]byte) (bool, error)
   HasString(string) (bool, error)
   getKeyAndOffset(offset uint) (string, uint)
}

func BloomInstance()  *RedisBloomFilter {
   if bloomInstance != nil{
      return bloomInstance
   }
   Init()
   return bloomInstance
}

type RedisBloomFilter struct {
   keyPrefix string
   n         uint
   k         uint
}
//对数据进行哈希计算
func HashData(data []byte) [4]uint64 {
   a1 := []byte{1} // to grab another bit of data
   hasher := murmur3.New128()
   hasher.Write(data) // #nosec
   v1, v2 := hasher.Sum128()
   hasher.Write(a1) // #nosec
   v3, v4 := hasher.Sum128()
   return [4]uint64{
      v1, v2, v3, v4,
   }
}
//创建一个redis布隆过滤器
func NewRedisBloomFilter(m uint, p float64, kp string) *RedisBloomFilter {
   n, k := bloom.EstimateParameters(m, p)
   filter := &RedisBloomFilter{
      n:         n,
      k:         k,
      keyPrefix: kp,
   }
   cli := redis.RedisInstance()
   key, value := filter.getKeyAndOffset(n)
   cli.Do("SETBIT", key, value, 0)
   return filter
}

func Init(){

   fmt.Println(os.Args[0])
   key := conf.GetConfig().MustValue("bloom","key_prefix","")
   if key == ""{
      panic("bloom:get key fail")
   }
   data_number := conf.GetConfig().MustValue("bloom","data_number","")
   false_positive := conf.GetConfig().MustValue("bloom","false_positive","")
   m,_ :=strconv.ParseUint(data_number,10,64)
   p,_ :=strconv.ParseFloat(false_positive,64)

   bloomInstance = NewRedisBloomFilter(uint(m),p,key)
   fmt.Println("redis init successful")

}
//将数据放入布隆过滤器
func (filter *RedisBloomFilter) Put(data []byte) error {
   cli := redis.RedisInstance()
   h := HashData(data)
   for i := uint(0); i < filter.k; i++ {
      key, value := filter.getKeyAndOffset(uint(location(h, i) % uint64(filter.n)))
      _, err := cli.Do("SETBIT", key, value, 1)
      if err != nil {
         return err
      }
   }
   return nil
}
//将字符串放入布隆过滤器
func (filter *RedisBloomFilter) PutString(data string) error {
   return filter.Put([]byte(data))
}
//判断数据是否在布隆过滤器中
func (filter *RedisBloomFilter) Has(data []byte) (bool, error) {
   cli := redis.RedisInstance()
   h := HashData(data)
   for i := uint(0); i < filter.k; i++ {
      key, value := filter.getKeyAndOffset(uint(location(h, i) % uint64(filter.n)))
      bitValue,err := cli.GetBitInt(key, value)
      if err != nil {
         return false, err
      }
      if bitValue == 0 {
         return false, nil
      }
   }
   return true, nil
}
//判断字符串是否在布隆过滤器中
func (filter *RedisBloomFilter) HasString(data string) (bool, error) {
   return filter.Has([]byte(data))
}

func (filter *RedisBloomFilter) getKeyAndOffset(offset uint) (string, uint) {
   n := uint(offset / RedisMaxLength)
   thisOffset := offset - n*RedisMaxLength
   key := fmt.Sprintf("%s:%d", filter.keyPrefix, n)
   return key, thisOffset
}

func location(h [4]uint64, i uint) uint64 {
   ii := uint64(i)
   return h[ii%2] + ii*h[2+(((ii+(ii%2))%4)/2)]
}

func EstimateParameters(n uint, p float64) (m uint, k uint) {
   m = uint(math.Ceil(-1 * float64(n) * math.Log(p) / math.Pow(math.Log(2), 2)))
   k = uint(math.Ceil(math.Log(2) * float64(m) / float64(n)))
   return
}