布隆过滤器：提高效率与降低成本的秘密

<dependency>
    <groupId>com.google.guava</groupId>
    <artifactId>guava</artifactId>
    <version>23.0</version>
</dependency>
1.
2.
3.
4.
5.

private static void GuavaBloomFilter() {
    // 创建布隆过滤器对象
    BloomFilter bloomFilter = BloomFilter.create(Funnels.stringFunnel(Charset.defaultCharset()),EXPECTED_INSERTIONS,FALSE_PROBABILITY);
    // 向过滤器中添加元素
    bloomFilter.put("element001");
    bloomFilter.put("element003");
    // 判断元素是否存在
    System.out.println(bloomFilter.mightContain("element001"));//true
    System.out.println(bloomFilter.mightContain("element002"));//false
    // 已添加到Bloom过滤器的元素的数量
    System.out.println(bloomFilter.approximateElementCount());// 2
    // 返回元素存在的错误概率
    System.out.println(bloomFilter.expectedFpp());
}
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.

<dependency>
    <groupId>org.redisson</groupId>
    <artifactId>redisson</artifactId>
    <version>3.22.1</version>
</dependency>
1.
2.
3.
4.
5.

private static void RedissonBloomFilter() {
    Config config = new Config();
    config.useSingleServer().setAddress("redis://" + REDIS_IP + ":" + REDIS_PORT);
    config.useSingleServer().setPassword(REDIS_PASSWORD);
    // 获取客户端
    RedissonClient redissonClient = Redisson.create(config);
    RBloomFilter<String> bloomFilter = redissonClient.getBloomFilter(BLOOM_FILTER_NAME);
    // 初始化布隆过滤器：预期插入量为100000000L,预期错误概率为1%
    bloomFilter.tryInit(EXPECTED_INSERTIONS, FALSE_PROBABILITY);
    // 插入数据
    bloomFilter.add("element001");
    bloomFilter.add("element003");

    // 判断下面元素是否在布隆过滤器中
    System.out.println(bloomFilter.contains("element002"));//false
    System.out.println(bloomFilter.contains("element001"));//true

    // 已添加到Bloom过滤器的元素的数量
    System.out.println(bloomFilter.count());//2
    // 预期插入元素的个数
    System.out.println(bloomFilter.getExpectedInsertions());//1000000
    // 元素存在的错误概率
    System.out.println(bloomFilter.getFalseProbability());//0.01
    // 每个元素使用的哈希迭代次数
    System.out.println(bloomFilter.getHashIterations());
    // 实例所需Redis内存的位数
    System.out.println(bloomFilter.getSize());
}
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.

<dependency>
        <groupId>redis.clients</groupId>
        <artifactId>jedis</artifactId>
        <version>4.2.0</version>
    </dependency>
1.
2.
3.
4.
5.

private static void RedisBloomFilter() {
    // 建立连接
    BloomFilterCommands bloomFilterCommands = new JedisPooled(REDIS_IP, REDIS_PORT, "", REDIS_PASSWORD);
    // 构建布隆过滤器参数
    BFReserveParams bfReserveParams = new BFReserveParams();
    bfReserveParams.expansion(2);

    // 创建一个过滤器
    String test = bloomFilterCommands.bfReserve(BLOOM_FILTER_NAME, FALSE_PROBABILITY, EXPECTED_INSERTIONS, bfReserveParams);

    // 向过滤器中添加元素
    bloomFilterCommands.bfAdd(BLOOM_FILTER_NAME, "element001");
    bloomFilterCommands.bfAdd(BLOOM_FILTER_NAME, "element003");

    // 判断元素是否存在
    System.out.println(bloomFilterCommands.bfExists(BLOOM_FILTER_NAME, "element001"));//true
    System.out.println(bloomFilterCommands.bfExists(BLOOM_FILTER_NAME, "element002"));//false
}
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.

public class BloomFilterUtils {

    private static final String BF_KEY_PREFIX = "bf_";

    private long numApproxElements;
    private double falseProbability;
    // hash个数
    private int numHashFunctions;
    // 数组长度
    private int bitmapLength;

    private JedisResourcePool jedisResourcePool;

    /**
     * 构造布隆过滤器。注意：在同一业务场景下，三个参数务必相同
     *
     * @param numApproxElements 预估元素数量
     * @param fpp               可接受的最大误差
     * @param jedisResourcePool Codis专用的Jedis连接池
     */
    public BloomFilterUtils(Long numApproxElements, double fpp, JedisResourcePool jedisResourcePool) {
        this.numApproxElements = numApproxElements;
        this.falseProbability = fpp;
        this.jedisResourcePool = jedisResourcePool;
        // 数组长度 m = (n * lnp)/ln2^2
        bitmapLength = (int) (-numApproxElements * Math.log(fpp) / (Math.log(2) * Math.log(2)));
        // hash个数 k = (n / m ) * ln2
        numHashFunctions = Math.max(1, (int) Math.round((double) bitmapLength / numApproxElements * Math.log(2)));
    }

    /**
     * 取得预估元素数量
     */
    public long getExpectedInsertions() {
        return numApproxElements;
    }

    /**
     * 返回元素存在的错误概率
     */
    public double getFalseProbability() {
        return falseProbability;
    }

    /**
     * 取得自动计算的最优哈希函数个数
     */
    public int getNumHashFunctions() {
        return numHashFunctions;
    }

    /**
     * 取得自动计算的最优Bitmap长度
     */
    public int getBitmapLength() {
        return bitmapLength;
    }

    /**
     * 计算一个元素值哈希后映射到Bitmap的哪些bit上
     *
     * @param element 元素值
     * @return bit下标的数组
     */
    private long[] getBitIndices(String element) {
        long[] indices = new long[numHashFunctions];

        // 元素  使用MurMurHash3 128位Hash算法转换值
        byte[] bytes = Hashing.murmur3_128()
                .hashObject(element, Funnels.stringFunnel(Charset.forName("UTF-8")))
                .asBytes();

        // 低8位转Long值
        long hash1 = Longs.fromBytes(
                bytes[7], bytes[6], bytes[5], bytes[4], bytes[3], bytes[2], bytes[1], bytes[0]
        );
        // 高8位转Long值
        long hash2 = Longs.fromBytes(
                bytes[15], bytes[14], bytes[13], bytes[12], bytes[11], bytes[10], bytes[9], bytes[8]
        );

        long combinedHash = hash1;
        // 双重哈希进行散列
        for (int i = 0; i  < numHashFunctions; i++) {
            indices[i] = (combinedHash & Long.MAX_VALUE) % bitmapLength;
            combinedHash += hash2;
        }
        return indices;
    }


    /**
     * 插入元素
     *
     * @param key       原始Redis键，会自动加上'bf_'前缀
     * @param element   元素值，字符串类型
     * @param expireSec 过期时间（秒）
     */
    public void add(String key, String element, int expireSec) {
        if (key == null || element == null) {
            throw new RuntimeException("键值均不能为空");
        }
        String actualKey = BF_KEY_PREFIX.concat(key);

        try (Jedis jedis = jedisResourcePool.getResource()) {
            try (Pipeline pipeline = jedis.pipelined()) {
                // 遍历元素所有hash结果的bit位置
                for (long index : getBitIndices(element)) {
                    pipeline.setbit(actualKey, index, true);
                }
                pipeline.syncAndReturnAll();
            }
            jedis.expire(actualKey, expireSec);
        }
    }

    /**
     * 检查元素在集合中是否（可能）存在
     *
     * @param key     原始Redis键，会自动加上'bf_'前缀
     * @param element 元素值，字符串类型
     */
    public boolean contains(String key, String element) {
        if (key == null || element == null) {
            throw new RuntimeException("键值均不能为空");
        }
        String actualKey = BF_KEY_PREFIX.concat(key);
        boolean result = false;

        try (Jedis jedis = jedisResourcePool.getResource()) {
            // 遍历元素所有hash结果的bit位置
            try (Pipeline pipeline = jedis.pipelined()) {
                for (long index : getBitIndices(element)) {
                    pipeline.getbit(actualKey, index);
                }
                result = !pipeline.syncAndReturnAll().contains(false);
            }
        }
        return result;
    }

    public static void main(String[] args) {
        String path = Path.getCurrentPath() + "/config/zzjodis.properties";
        ConfigReadUtil configReadUtil = new ConfigReadUtil(path);
        try {
            JedisResourcePool jedisResourcePool = RoundRobinJedisPool.
                    create()
                    .curatorClient(configReadUtil.getString("jodisZkStr"), 5000)
                    .zkProxyDir(configReadUtil.getString("zkProxyDir"))
                    .team(configReadUtil.getString("team"))
                    .connectionTimeoutMs(configReadUtil.getInt("connectionTimeoutMs"))
                    .soTimeoutMs(configReadUtil.getInt("soTimeoutMs"))
                    .appKey(configReadUtil.getString("appKey"))
                    .password("".equals(configReadUtil.getString("password")) ? null : configReadUtil.getString("password"))
                    .build();
            BloomFilterUtils bloomFilterUtils = new BloomFilterUtils(10000, 0.01, jedisResourcePool);
            bloomFilterUtils.add("filter01", "element001", 30 * 60);
            System.out.println(bloomFilterUtils.contains("filter01", "element001"));  // true
            System.out.println(bloomFilterUtils.contains("filter01", "element002"));  // false
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

}
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.
59.
60.
61.
62.
63.
64.
65.
66.
67.
68.
69.
70.
71.
72.
73.
74.
75.
76.
77.
78.
79.
80.
81.
82.
83.
84.
85.
86.
87.
88.
89.
90.
91.
92.
93.
94.
95.
96.
97.
98.
99.
100.
101.
102.
103.
104.
105.
106.
107.
108.
109.
110.
111.
112.
113.
114.
115.
116.
117.
118.
119.
120.
121.
122.
123.
124.
125.
126.
127.
128.
129.
130.
131.
132.
133.
134.
135.
136.
137.
138.
139.
140.
141.
142.
143.
144.
145.
146.
147.
148.
149.
150.
151.
152.
153.
154.
155.
156.
157.
158.
159.
160.
161.
162.
163.
164.
165.