Stable Bloom filter in Python
Bloom variant for unbounded streams that randomly decrements counters to keep a stable false positive rate.
- File
stable_bloom_filter.py, 74 lines- Level
- I0 not checked yet
- Toolchain
- Python 3.14, pytest 9
- Test framework
- pytest
- Run the tests
python3 -m pytest
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/stable-bloom-filter/. The authors are listed in prov/XYQ2CR.jsonl.
"""Stable Bloom filter (Deng and Rafiei 2006) for duplicate detection in a stream."""
import hashlib
import random
Key = str | bytes | int
class StableBloomFilter:
"""m cells of d bits, K hash functions, and P random decrements per element.
process(x) reports whether x looks like a duplicate of a recent element,
then forgets a little (P random cells lose 1) and records x (its K cells
are set to Max = 2^d - 1). Both false positives and false negatives occur;
the fraction of zero cells tends to a constant, so the error rates stay
stable on an unbounded stream.
"""
def __init__(self, m: int, K: int, d: int = 3, P: int = 10, seed: int = 0, rng_seed: int = 0) -> None:
if not 1 <= K <= m or d < 1 or not 0 <= P <= m:
raise ValueError("need 1 <= K <= m, d >= 1 and 0 <= P <= m")
self.m, self.K, self.P, self.seed = m, K, P, seed
self.max = (1 << d) - 1
self.rng = random.Random(rng_seed)
self.cells = [0] * m # @line L1
def _positions(self, key: Key) -> list[int]:
"""K distinct cells of key, from 64-bit words of BLAKE2b."""
out: list[int] = []
t = 0
while len(out) < self.K:
d = hashlib.blake2b(
repr(key).encode(), key=self.seed.to_bytes(8, "little"), salt=t.to_bytes(16, "little")
).digest()
for w in range(0, 64, 8):
p = int.from_bytes(d[w : w + 8], "little") % self.m
if p not in out and len(out) < self.K:
out.append(p)
t += 1
return out
def _victims(self) -> list[int]:
"""P distinct cells chosen uniformly at random."""
return self.rng.sample(range(self.m), self.P)
def query(self, key: Key) -> bool:
"""True if all K cells of key are nonzero."""
return all(self.cells[p] for p in self._positions(key))
def process(self, key: Key) -> bool:
"""Report whether key looks like a recent duplicate, then record it."""
ps = self._positions(key)
dup = all(self.cells[p] for p in ps) # @line L2
for j in self._victims(): # @line L3
if self.cells[j] > 0:
self.cells[j] -= 1
for p in ps: # @line L4
self.cells[p] = self.max
return dup # @line L5
def zero_fraction(self) -> float:
"""The fraction of cells that are 0."""
return self.cells.count(0) / self.m
def stable_zero_fraction(self) -> float:
"""The limit (1 / (1 + 1 / (P (1/K - 1/m))))^Max of the zero fraction."""
if self.P == 0:
return 0.0
if self.K == self.m:
return 0.0
return (1 / (1 + 1 / (self.P * (1 / self.K - 1 / self.m)))) ** self.max
test_stable_bloom_filter.py 82 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/stable-bloom-filter/. The authors are listed in prov/XYQ2CR.jsonl.
import random
import pytest
from stable_bloom_filter import Key, StableBloomFilter
POS: dict[Key, list[int]] = {"a": [0, 3], "b": [1, 5], "c": [3, 6], "d": [0, 5], "e": [2, 4], "f": [2, 7]}
STREAM = "abacdefc"
VICTIMS = [[2, 7], [0, 4], [3, 5], [0, 1], [6, 3], [6, 7], [6, 1], [5, 0]]
class FixedSBF(StableBloomFilter):
def __init__(self) -> None:
super().__init__(8, 2, d=2, P=2)
self.script = iter(VICTIMS)
def _positions(self, key: Key) -> list[int]:
return POS[key]
def _victims(self) -> list[int]:
return next(self.script)
def test_example() -> None:
f = FixedSBF()
states = []
out = []
for x in STREAM:
out.append(f.process(x))
states.append(list(f.cells))
assert out == [False, False, True, False, True, False, False, False]
assert states[4] == [3, 2, 0, 2, 0, 3, 2, 0] # after d, a false positive
assert states[6] == [3, 1, 3, 2, 3, 3, 0, 3] # cell 6 of c is now 0
assert states[7] == [2, 1, 3, 3, 3, 2, 3, 3] # c was a false negative
def test_edges() -> None:
f = StableBloomFilter(1, 1, d=1, P=0)
assert not f.process("x") and f.process("y") # one cell, never cleared
assert f.stable_zero_fraction() == 0.0
g = StableBloomFilter(4, 1, d=1, P=4)
g.process("x")
assert g.cells.count(1) == 1 # every cell was decremented, then x was set
for bad in [dict(m=2, K=3), dict(m=2, K=1, d=0), dict(m=2, K=1, P=3)]:
with pytest.raises(ValueError):
StableBloomFilter(**bad)
def test_random_against_model() -> None:
rng = random.Random(2006)
for _ in range(500):
m = rng.randint(1, 30)
K = rng.randint(1, m)
d = rng.randint(1, 3)
P = rng.randint(0, m)
f = StableBloomFilter(m, K, d, P, seed=rng.randrange(100), rng_seed=rng.randrange(100))
shadow = random.Random(0)
shadow.setstate(f.rng.getstate())
model = [0] * m
for _ in range(rng.randint(0, 40)):
x = rng.randrange(15)
ps = f._positions(x)
assert len(set(ps)) == K
want = all(model[p] for p in ps)
for j in shadow.sample(range(m), P):
model[j] = max(0, model[j] - 1)
for p in ps:
model[p] = f.max
assert f.process(x) == want
assert f.cells == model
assert f.query(x) # just recorded
def test_stable_fraction() -> None:
f = StableBloomFilter(2000, 3, d=3, P=50, seed=1, rng_seed=1)
for x in range(20000):
f.process(x)
assert abs(f.zero_fraction() - f.stable_zero_fraction()) < 0.03
assert abs(f.stable_zero_fraction() - 0.6637) < 0.001
What the next level needs
- the tests did not run in this build