Cuckoo filter in Python
Approximate set using cuckoo hashing of key fingerprints, supporting deletion.
- File
cuckoo_filter.py, 97 lines- Level
- I0 not checked yet
- Toolchain
- Python 3.14, pytest 9
- Test framework
- pytest
- Run the tests
python3 -m pytest
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/cuckoo-filter/. The authors are listed in prov/131BXM.jsonl.
"""Cuckoo filter (Fan, Andersen, Kaminsky and Mitzenmacher 2014)."""
import hashlib
import random
Key = str | bytes | int
class CuckooFilter:
"""m buckets of b slots that hold f-bit fingerprints, with partial-key cuckoo hashing.
The value 0 marks an empty slot, so fingerprints are in 1 .. 2^f - 1.
m must be a power of two, so that i XOR (h(fp) mod m) is a bucket index.
delete(x) is allowed only for a key x that is present.
"""
def __init__(self, m: int, b: int = 4, f: int = 8, max_kicks: int = 500, seed: int = 0) -> None:
if m < 1 or m & (m - 1):
raise ValueError("m must be a power of two")
if b < 1 or f < 1:
raise ValueError("b and f must be positive")
self.m, self.b, self.f, self.max_kicks, self.seed = m, b, f, max_kicks, seed
self.table: list[list[int]] = [[] for _ in range(m)] # @line L1
self.victim: tuple[int, int] | None = None
self.rng = random.Random(seed)
def _h(self, data: bytes) -> int:
d = hashlib.blake2b(data, digest_size=8, key=self.seed.to_bytes(8, "little")).digest()
return int.from_bytes(d, "little")
def _fp_index(self, key: Key) -> tuple[int, int]:
"""The fingerprint of key (never 0) and its first bucket."""
v = self._h(repr(key).encode())
fp = (v >> 32) % ((1 << self.f) - 1) + 1 if self.f > 1 else 1
return fp, (v & 0xFFFFFFFF) % self.m
def _alt(self, i: int, fp: int) -> int:
"""The other bucket of fingerprint fp in bucket i. _alt(_alt(i, fp), fp) == i."""
return i ^ (self._h(b"fp" + fp.to_bytes(8, "little")) % self.m)
def _choose(self, n: int) -> int:
return self.rng.randrange(n)
def _place(self, i: int, fp: int) -> None:
"""Store fp in bucket i or its other bucket, kicking entries if needed."""
for j in (i, self._alt(i, fp)):
if len(self.table[j]) < self.b: # @line L4
self.table[j].append(fp)
return
i = (i, self._alt(i, fp))[self._choose(2)] # @line L5
for _ in range(self.max_kicks): # @line L6
s = self._choose(self.b)
fp, self.table[i][s] = self.table[i][s], fp # @line L7
i = self._alt(i, fp) # @line L8
if len(self.table[i]) < self.b: # @line L9
self.table[i].append(fp)
return
self.victim = (i, fp) # @line L10: the table is now full
def insert(self, key: Key) -> bool:
"""Add key. False, with no change, if the filter is full."""
if self.victim is not None: # @line L2
return False
fp, i1 = self._fp_index(key) # @line L3
self._place(i1, fp)
return True
def query(self, key: Key) -> bool:
"""False if key is surely absent; True if it may be present."""
fp, i1 = self._fp_index(key) # @line L11
i2 = self._alt(i1, fp)
if fp in self.table[i1] or fp in self.table[i2]: # @line L12
return True
return self.victim is not None and self.victim[1] == fp and self.victim[0] in (i1, i2)
def delete(self, key: Key) -> bool:
"""Remove one copy of key. Precondition: key is present."""
fp, i1 = self._fp_index(key) # @line L13
i2 = self._alt(i1, fp)
for j in (i1, i2):
if fp in self.table[j]: # @line L14
self.table[j].remove(fp)
if self.victim is not None: # @line L15: there is room again
vi, vfp = self.victim
self.victim = None
self._place(vi, vfp)
return True
if self.victim is not None and self.victim[1] == fp and self.victim[0] in (i1, i2): # @line L16
self.victim = None
return True
return False # @line L17
def __len__(self) -> int:
return sum(len(t) for t in self.table) + (self.victim is not None)
test_cuckoo_filter.py 117 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/cuckoo-filter/. The authors are listed in prov/131BXM.jsonl.
import random
from collections import Counter
import pytest
from cuckoo_filter import CuckooFilter, Key
TABLE: dict[Key, tuple[int, int]] = {
"a": (5, 0), "b": (6, 0), "c": (9, 0), "d": (7, 1), "e": (13, 0), "x": (7, 2), "y": (3, 3),
}
class FixedCF(CuckooFilter):
def __init__(self) -> None:
super().__init__(4, b=2, f=4)
def _fp_index(self, key: Key) -> tuple[int, int]:
return TABLE[key]
def _alt(self, i: int, fp: int) -> int:
return i ^ (fp % self.m)
def _choose(self, n: int) -> int:
return 0
def test_example() -> None:
f = FixedCF()
for k in "bacd":
assert f.insert(k)
assert f.table == [[6, 5], [9, 7], [], []]
assert f.insert("e") # kicks 6 from bucket 0 to bucket 0 ^ 2 = 2
assert f.table == [[13, 5], [9, 7], [6], []]
assert all(f.query(k) for k in "abcde")
assert f.query("x") # same fingerprint and buckets as d: a false positive
assert not f.query("y")
assert f.delete("a")
assert f.table == [[13], [9, 7], [6], []]
assert not f.query("a")
def test_victim() -> None:
f = FixedCF()
f.max_kicks = 3
for k in "bacd":
f.insert(k)
f.table[2] = [1, 1]
f.table[3] = [1, 1] # fill the table so that e cannot be placed
assert f.insert("e") and f.victim is not None
assert not f.insert("y") # full: no change
assert all(f.query(k) for k in "abcde")
assert f.delete("d") # room again: the victim goes back into the table
assert f.victim is None and all(f.query(k) for k in "abce")
def test_edges() -> None:
with pytest.raises(ValueError):
CuckooFilter(3)
f = CuckooFilter(1, b=1, f=1)
assert not f.query(0) and f.insert(0) and f.query(0)
assert f.delete(0) and not f.query(0) and not f.delete(0)
def tokens_of_table(f: CuckooFilter) -> Counter[tuple[int, frozenset[int]]]:
t: Counter[tuple[int, frozenset[int]]] = Counter()
for j, bucket in enumerate(f.table):
for fp in bucket:
t[(fp, frozenset((j, f._alt(j, fp))))] += 1
if f.victim is not None:
j, fp = f.victim
t[(fp, frozenset((j, f._alt(j, fp))))] += 1
return t
def tokens_of_keys(f: CuckooFilter, live: Counter[int]) -> Counter[tuple[int, frozenset[int]]]:
t: Counter[tuple[int, frozenset[int]]] = Counter()
for x, c in live.items():
fp, i1 = f._fp_index(x)
if c:
t[(fp, frozenset((i1, f._alt(i1, fp))))] += c
return t
def test_random_against_multiset() -> None:
rng = random.Random(2014)
for _ in range(500):
m = 1 << rng.randint(0, 4)
b = rng.randint(1, 4)
f = CuckooFilter(m, b, f=rng.randint(2, 10), max_kicks=rng.randint(1, 30), seed=rng.randrange(1000))
live: Counter[int] = Counter()
for _ in range(rng.randint(0, 80)):
x = rng.randrange(40)
r = rng.random()
if r < 0.5:
if f.insert(x):
live[x] += 1
elif r < 0.8 and live[x] > 0:
assert f.delete(x)
live[x] -= 1
else:
assert f.query(x) or live[x] == 0
assert tokens_of_table(f) == tokens_of_keys(f, live)
assert all(f.query(y) for y in live if live[y])
assert all(len(t) <= b for t in f.table)
def test_load_and_rate() -> None:
f = CuckooFilter(1 << 10, b=4, f=12, seed=7)
n = 0
while f.insert(("k", n)):
n += 1
assert n >= 0.9 * 4 * (1 << 10)
fp = sum(f.query(("q", i)) for i in range(20000))
assert fp / 20000 <= 2 * 4 / ((1 << 12) - 1) + 0.002
What the next level needs
- the tests did not run in this build