千万级别字符串找出新出现的

38 阅读1分钟
package com.hzmc.dsc.epc.plugin.audit.tag.filter;

import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.hzmc.common.epc.model.AuditEventBO;
import com.hzmc.dsc.epc.plugin.audit.filter.alert.cache.AlertRuleInfo;
import com.hzmc.dsc.epc.plugin.audit.tag.filter.base.AbstracDefaultOneFilter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;

import java.util.List;
import java.util.concurrent.TimeUnit;

import static com.hzmc.dsc.epc.plugin.audit.enums.ExtraDataPojoEnum.NEEDOUT;

@Slf4j
public abstract class BitSetBloomFilter extends AbstracDefaultOneFilter {
    /**
     * 初始化Hutool BitSetBloomFilter
     */
    int c = 1000;
    cn.hutool.bloomfilter.BitSetBloomFilter bloomFilter = new cn.hutool.bloomfilter.BitSetBloomFilter(c, c / 2, 7);
    Object object = new Object();
    /**
     * 初始化Caffeine缓存
     */

    static Cache<String, Object> cache = Caffeine.newBuilder()
            .maximumSize(1000)
            .expireAfterWrite(10, TimeUnit.MINUTES)
            .build();


    boolean checkFindKey(List<String> keys) {
        if (CollectionUtils.isEmpty(keys)) {
            return false;
        }

        int caffeineCount = 0;
        boolean bloomCount = false;

        if (bloomFilter.getFalsePositiveProbability() > 0.1) {
            synchronized (BitSetBloomFilter.class) {
                // 布隆过滤器扩容
                c *= 10;
                bloomFilter = new cn.hutool.bloomfilter.BitSetBloomFilter(c, c / 2, 7);
                log.info("newCount:{}", c);
            }

        }
        for (String key : keys) {
            Object cachedResult = cache.getIfPresent(key);
            if (cachedResult == null) {
                caffeineCount++;
            }
            if (!bloomFilter.contains(key)) {
                bloomCount = true;
            }
        }
        for (String key : keys) {
            cache.put(key, object);
            bloomFilter.add(key);
        }
        return bloomCount || caffeineCount >= 2;


    }

    @Override
    public boolean check(AlertRuleInfo alertRuleInfo) {
        return true;
    }
}