Algorithm Search⌘K

Stable Bloom filter in Python

Bloom variant for unbounded streams that randomly decrements counters to keep a stable false positive rate.

File
stable_bloom_filter.py, 74 lines
Level
I0 not checked yet
Toolchain
Python 3.14, pytest 9
Test framework
pytest
Run the tests
python3 -m pytest
stable_bloom_filter.pyRawGitHub
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/stable-bloom-filter/. The authors are listed in prov/XYQ2CR.jsonl.

"""Stable Bloom filter (Deng and Rafiei 2006) for duplicate detection in a stream."""

import hashlib
import random

Key = str | bytes | int


class StableBloomFilter:
    """m cells of d bits, K hash functions, and P random decrements per element.

    process(x) reports whether x looks like a duplicate of a recent element,
    then forgets a little (P random cells lose 1) and records x (its K cells
    are set to Max = 2^d - 1). Both false positives and false negatives occur;
    the fraction of zero cells tends to a constant, so the error rates stay
    stable on an unbounded stream.
    """

    def __init__(self, m: int, K: int, d: int = 3, P: int = 10, seed: int = 0, rng_seed: int = 0) -> None:
        if not 1 <= K <= m or d < 1 or not 0 <= P <= m:
            raise ValueError("need 1 <= K <= m, d >= 1 and 0 <= P <= m")
        self.m, self.K, self.P, self.seed = m, K, P, seed
        self.max = (1 << d) - 1
        self.rng = random.Random(rng_seed)
        self.cells = [0] * m  # @line L1

    def _positions(self, key: Key) -> list[int]:
        """K distinct cells of key, from 64-bit words of BLAKE2b."""
        out: list[int] = []
        t = 0
        while len(out) < self.K:
            d = hashlib.blake2b(
                repr(key).encode(), key=self.seed.to_bytes(8, "little"), salt=t.to_bytes(16, "little")
            ).digest()
            for w in range(0, 64, 8):
                p = int.from_bytes(d[w : w + 8], "little") % self.m
                if p not in out and len(out) < self.K:
                    out.append(p)
            t += 1
        return out

    def _victims(self) -> list[int]:
        """P distinct cells chosen uniformly at random."""
        return self.rng.sample(range(self.m), self.P)

    def query(self, key: Key) -> bool:
        """True if all K cells of key are nonzero."""
        return all(self.cells[p] for p in self._positions(key))

    def process(self, key: Key) -> bool:
        """Report whether key looks like a recent duplicate, then record it."""
        ps = self._positions(key)
        dup = all(self.cells[p] for p in ps)  # @line L2
        for j in self._victims():  # @line L3
            if self.cells[j] > 0:
                self.cells[j] -= 1
        for p in ps:  # @line L4
            self.cells[p] = self.max
        return dup  # @line L5

    def zero_fraction(self) -> float:
        """The fraction of cells that are 0."""
        return self.cells.count(0) / self.m

    def stable_zero_fraction(self) -> float:
        """The limit (1 / (1 + 1 / (P (1/K - 1/m))))^Max of the zero fraction."""
        if self.P == 0:
            return 0.0
        if self.K == self.m:
            return 0.0
        return (1 / (1 + 1 / (self.P * (1 / self.K - 1 / self.m)))) ** self.max
test_stable_bloom_filter.py 82 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/stable-bloom-filter/. The authors are listed in prov/XYQ2CR.jsonl.

import random

import pytest

from stable_bloom_filter import Key, StableBloomFilter

POS: dict[Key, list[int]] = {"a": [0, 3], "b": [1, 5], "c": [3, 6], "d": [0, 5], "e": [2, 4], "f": [2, 7]}
STREAM = "abacdefc"
VICTIMS = [[2, 7], [0, 4], [3, 5], [0, 1], [6, 3], [6, 7], [6, 1], [5, 0]]


class FixedSBF(StableBloomFilter):
    def __init__(self) -> None:
        super().__init__(8, 2, d=2, P=2)
        self.script = iter(VICTIMS)

    def _positions(self, key: Key) -> list[int]:
        return POS[key]

    def _victims(self) -> list[int]:
        return next(self.script)


def test_example() -> None:
    f = FixedSBF()
    states = []
    out = []
    for x in STREAM:
        out.append(f.process(x))
        states.append(list(f.cells))
    assert out == [False, False, True, False, True, False, False, False]
    assert states[4] == [3, 2, 0, 2, 0, 3, 2, 0]  # after d, a false positive
    assert states[6] == [3, 1, 3, 2, 3, 3, 0, 3]  # cell 6 of c is now 0
    assert states[7] == [2, 1, 3, 3, 3, 2, 3, 3]  # c was a false negative


def test_edges() -> None:
    f = StableBloomFilter(1, 1, d=1, P=0)
    assert not f.process("x") and f.process("y")  # one cell, never cleared
    assert f.stable_zero_fraction() == 0.0
    g = StableBloomFilter(4, 1, d=1, P=4)
    g.process("x")
    assert g.cells.count(1) == 1  # every cell was decremented, then x was set
    for bad in [dict(m=2, K=3), dict(m=2, K=1, d=0), dict(m=2, K=1, P=3)]:
        with pytest.raises(ValueError):
            StableBloomFilter(**bad)


def test_random_against_model() -> None:
    rng = random.Random(2006)
    for _ in range(500):
        m = rng.randint(1, 30)
        K = rng.randint(1, m)
        d = rng.randint(1, 3)
        P = rng.randint(0, m)
        f = StableBloomFilter(m, K, d, P, seed=rng.randrange(100), rng_seed=rng.randrange(100))
        shadow = random.Random(0)
        shadow.setstate(f.rng.getstate())
        model = [0] * m
        for _ in range(rng.randint(0, 40)):
            x = rng.randrange(15)
            ps = f._positions(x)
            assert len(set(ps)) == K
            want = all(model[p] for p in ps)
            for j in shadow.sample(range(m), P):
                model[j] = max(0, model[j] - 1)
            for p in ps:
                model[p] = f.max
            assert f.process(x) == want
            assert f.cells == model
            assert f.query(x)  # just recorded


def test_stable_fraction() -> None:
    f = StableBloomFilter(2000, 3, d=3, P=50, seed=1, rng_seed=1)
    for x in range(20000):
        f.process(x)
    assert abs(f.zero_fraction() - f.stable_zero_fraction()) < 0.03
    assert abs(f.stable_zero_fraction() - 0.6637) < 0.001

What the next level needs

  • the tests did not run in this build

Read the full page about Stable Bloom filter