SpringBoot集成BloomFilter、补充击穿问题

1.依赖包

    <!--使用Redis-->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-data-redis</artifactId>
    </dependency>
    <!--借助guava的布隆过滤器-->
    <dependency>
        <groupId>com.google.guava</groupId>
        <artifactId>guava</artifactId>
        <version>19.0</version>
    </dependency>

2.配置RedisConfig.class

package com.my.equipment.config.RedisCluster;

import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
import com.google.common.hash.Funnel;
import com.my.equipment.config.BloomFilter.BloomFilterHelper;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.core.env.Environment;
import org.springframework.core.env.MapPropertySource;
import org.springframework.data.redis.connection.RedisClusterConfiguration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.connection.lettuce.LettucePoolingClientConfiguration;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;

import java.util.HashMap;
import java.util.Map;

@Configuration
public class RedisConfig {

    @Autowired
    private Environment environment;

    /**
     * 配置 lettuce 连接池
     * @return
     */
    @Bean
    @Primary
    @ConfigurationProperties(prefix = "spring.redis.lettuce.pool")
    public GenericObjectPoolConfig redisPool() {
        return new GenericObjectPoolConfig();
    }

    @Primary
    @Bean("redisClusterConfig")
    public RedisClusterConfiguration redisClusterConfig(){
        Map<String,Object> source=new HashMap<>();
        source.put("spring.redis.cluster.nodes",environment.getProperty("spring.redis.cluster.nodes"));
        RedisClusterConfiguration redisClusterConfiguration=new RedisClusterConfiguration(new MapPropertySource("RedisClusterConfiguration", source));
        redisClusterConfiguration.setPassword(environment.getProperty("spring.redis.password"));
        return redisClusterConfiguration;
    }

    /**
     * 配置(第一个)数据源的连接工厂
     * 这里注意：需要添加@Primary 指定bean的名称，目的是为了创建两个不同名称的LettuceConnectionFactory
     * @param redisPool
     * @param redisClusterConfig
     * @return
     */
    @Bean("lettuceConnectionFactory")
    @Primary
    public LettuceConnectionFactory lettuceConnectionFactory(GenericObjectPoolConfig redisPool, @Qualifier("redisClusterConfig") RedisClusterConfiguration redisClusterConfig){
        LettuceClientConfiguration clientConfiguration = LettucePoolingClientConfiguration.builder().poolConfig(redisPool).build();
        return new LettuceConnectionFactory(redisClusterConfig,clientConfiguration);
    }


    @Bean("redisTemplate")
    @Primary
    public RedisTemplate redisTemplate(@Qualifier("lettuceConnectionFactory") RedisConnectionFactory redisConnectionFactory){
        return getRedisTemplate(redisConnectionFactory);
    }


    private RedisTemplate getRedisTemplate(RedisConnectionFactory factory){
        RedisTemplate<String,Object>  template=new RedisTemplate<>();
        template.setConnectionFactory(factory);

        Jackson2JsonRedisSerializer jackson2JsonRedisSerializer=new Jackson2JsonRedisSerializer(Object.class);
        ObjectMapper om = new ObjectMapper();
        // 指定要序列化的域，field,get和set,以及修饰符范围，ANY是都有包括private和public
        om.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY);
        //  指定序列化输入的类型，类必须是非final修饰的，final修饰的类，比如String,Integer等会跑出异常
        // 存储到redis里的数据将是有类型的json数据
        om.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL);
        jackson2JsonRedisSerializer.setObjectMapper(om);

        StringRedisSerializer stringRedisSerializer=new StringRedisSerializer();
        // key采用String的序列化方式
        template.setKeySerializer(stringRedisSerializer);
        // hash的key也采用String的序列化方式
        template.setHashKeySerializer(stringRedisSerializer);
        // value序列化方式采用jackson
        template.setValueSerializer(jackson2JsonRedisSerializer);
        // hash的value也采用jackson
        template.setHashValueSerializer(jackson2JsonRedisSerializer);


        //非spring注入需要调用该函数,使template初始化生效
        template.afterPropertiesSet();

        return template;

    }

    @Bean
    public BloomFilterHelper<Integer> initBloomFilterHelper(){
        return new BloomFilterHelper<>((Funnel<Integer>)(from,into)->into.putInt(from), 1000000, 0.01);
    }

//    @Bean
//    public BloomFilterHelper<String> initBloomFilterHelper() {
//        return new BloomFilterHelper<>((Funnel<String>) (from, into) -> into.putString(from, Charsets.UTF_8), 1000000, 0.01);
//    }

}

3.配置BloomFilter

package com.my.equipment.config.BloomFilter;

import com.google.common.base.Preconditions;
import com.google.common.hash.Funnel;
import com.google.common.hash.Hashing;
import org.springframework.stereotype.Component;



public class BloomFilterHelper<T> {

    /**
     * hash 函数的个数
     */
    private int numHashFunctions;

    /**
     * bits 数据的长度
     */
    private int bitSize;

    /**
     * 用于将任意类型T的输入数据转化为Java基本类型的数据（byte、int、char等等）。这里是会转化为byte。
     */
    private Funnel<T> funnel;


    public BloomFilterHelper(Funnel<T> funnel,int expectedInsertions,double fpp){
        Preconditions.checkArgument(funnel != null, "funnel不能为空");
        this.funnel = funnel;
        // 计算bit数组长度
        bitSize = optimalNumOfBits(expectedInsertions, fpp);
        // 计算hash方法执行次数
        numHashFunctions = optimalNumOfHashFunctions(expectedInsertions, bitSize);

    }



    public int[] murmurHashOffset(T value) {
        int[] offset = new int[numHashFunctions];

        long hash64 = Hashing.murmur3_128().hashObject(value, funnel).asLong();
        int hash1 = (int) hash64;
        int hash2 = (int) (hash64 >>> 32);
        for (int i = 1; i <= numHashFunctions; i++) {
            int nextHash = hash1 + i * hash2;
            if (nextHash < 0) {
                nextHash = ~nextHash;
            }
            offset[i - 1] = nextHash % bitSize;
        }

        return offset;
    }

    /**
     * 计算hash方法执行次数
     * @param n
     * @param m
     * @return
     */
    private int optimalNumOfHashFunctions(long n, long m) {
        int countOfHash = Math.max(1, (int) Math.round((double) m / n * Math.log(2)));
        return countOfHash;
    }

    /**
     * 计算bit数组长度
     * @param n
     * @param p
     * @return
     */
    private int optimalNumOfBits(long n, double p) {
        if (p==0){
            //设定最小期望长度
            p = Double.MIN_VALUE;
        }

        int sizeOfBitArray = (int) (-n * Math.log(p) / (Math.log(2) * Math.log(2)));
        return sizeOfBitArray;
    }
}

package com.my.equipment.config.BloomFilter;

import com.google.common.base.Preconditions;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;

@Service
public class RedisBloomFilter {

    @Autowired
    private RedisTemplate redisTemplate;


    /**
     * 根据给定的布隆过滤器添加值
     * @param bloomFilterHelper
     * @param key
     * @param value
     * @param <T>
     */
    public <T> void addByBloomFilter(BloomFilterHelper<T> bloomFilterHelper,String key,T value){
        Preconditions.checkArgument(bloomFilterHelper != null,"bloomFilterHelper不能为空");
        int[] offset = bloomFilterHelper.murmurHashOffset(value);
        for (int i : offset) {
//            System.out.println("key : " + key + " " + "value : " + i);
            redisTemplate.opsForValue().setBit(key,i,true);
        }
    }

    /**
     * 注意布隆过滤器对于不存值一定不存在，但是对于存在值存在误判
     * @param bloomFilterHelper
     * @param key
     * @param value
     * @param <T>
     * @return
     */
    public <T> boolean includeByBloomFilter(BloomFilterHelper<T> bloomFilterHelper,String key,T value){
        Preconditions.checkArgument(bloomFilterHelper != null, "bloomFilterHelper不能为空");
        int[] offset = bloomFilterHelper.murmurHashOffset(value);
        for (int i : offset) {
//            System.out.println("key : " + key + " " + "value : " + i);
            if (!redisTemplate.opsForValue().getBit(key,i)){
                return false;
            }
        }
        return true;
    }
}

4.测试使用

package com.my.equipment.utils.RedisUsing;

import com.my.equipment.config.BloomFilter.BloomFilterHelper;
import com.my.equipment.config.BloomFilter.RedisBloomFilter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

@Component
public class BloomFilterUtils {


    @Autowired
    private RedisBloomFilter redisBloomFilter;

    @Autowired
    private BloomFilterHelper bloomFilterHelper;


    public <T> boolean addBloomFilter(String key,T value){

        try {
            redisBloomFilter.addByBloomFilter(bloomFilterHelper,"bloom:"+key,value);
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }

        return true;
    }

    public <T> boolean checkBloomFilter(String key,T value){
        boolean exist= redisBloomFilter.includeByBloomFilter(bloomFilterHelper,"bloom:"+key,value);
        return exist;
    }



}

• 4.1 将所有数据库数据存入bloomFilter

@Override
public void save(SimData simData) {
    simDataMapper.insertSelective(simData);
    int autoId= simData.getId();
    //将所有新增的数据添加到布隆过滤器中
    bloomFilterUtils.addBloomFilter(String.valueOf(autoId),autoId);
}

• 4.2 检验数据

 @Override
    @Transactional
//    @Cacheable(key = "#id")
    public SlaveSimData getSlaveData(int id) throws BloomException {

        boolean hasKey = bloomFilterUtils.checkBloomFilter(String.valueOf(id), id);
        if (!hasKey) {
            //如果不存在，说明在数据库中必不存在该数据，直接返回错误
            throw new BloomException();
        } else {
            SlaveSimData slaveSimData = null;
            if (redisTemplate.hasKey(id + "")) {
                slaveSimData = (SlaveSimData) redisTemplate.opsForValue().get(id + "");
            } else {
                slaveSimData = slaveSimDataMapper.selectByPrimaryKey(id);
                redisTemplate.opsForValue().set(id + "", slaveSimData);
                System.out.println("生成数据缓存" + id);
            }
            return slaveSimData;
        }
    }

补充

击穿问题

package com.my.equipment.utils.RedisUsing;

import com.my.equipment.web.seSlaveDao.SlaveSimDataMapper;
import com.my.equipment.web.seSlavePojo.SlaveSimData;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Component;

import java.time.Duration;
import java.util.concurrent.TimeUnit;

@Component
public class RedisTemplateUtil {

    private final RedisTemplate redisTemplate;

    @Autowired
    private SlaveSimDataMapper slaveSimDataMapper;

    /**
     * 将配置文件中RedisTemplate自动注入
     *
     * @param redisTemplate
     */
    public RedisTemplateUtil(RedisTemplate redisTemplate) {
        this.redisTemplate = redisTemplate;
    }

    /**
     * String类型的value 防止缓存击穿
     * @param key
     * @return
     */
    public Object getHotkey(String key) {

        Object value = redisTemplate.opsForValue().get(key);
        if (value == null) {
            Object key_mutex = "lock";
            if (redisTemplate.opsForValue().setIfAbsent(key_mutex, "1",180,TimeUnit.SECONDS)) {
                System.out.println("开启分布式锁");
                value = slaveSimDataMapper.selectByPrimaryKey(Integer.valueOf(key));
                redisTemplate.opsForValue().set(key, value,3600, TimeUnit.MILLISECONDS);
                redisTemplate.delete(key_mutex);
            } else {
                //其他线程休息
                try {
                    Thread.sleep(50);
                    System.out.println("其他线程阻塞");
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
                getHotkey(key);
            }
        }
        return value;
    }

}