Algorithm Search⌘K

Cuckoo filter in Python

Approximate set using cuckoo hashing of key fingerprints, supporting deletion.

File
cuckoo_filter.py, 97 lines
Level
I0 not checked yet
Toolchain
Python 3.14, pytest 9
Test framework
pytest
Run the tests
python3 -m pytest
cuckoo_filter.pyRawGitHub
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/cuckoo-filter/. The authors are listed in prov/131BXM.jsonl.

"""Cuckoo filter (Fan, Andersen, Kaminsky and Mitzenmacher 2014)."""

import hashlib
import random

Key = str | bytes | int


class CuckooFilter:
    """m buckets of b slots that hold f-bit fingerprints, with partial-key cuckoo hashing.

    The value 0 marks an empty slot, so fingerprints are in 1 .. 2^f - 1.
    m must be a power of two, so that i XOR (h(fp) mod m) is a bucket index.
    delete(x) is allowed only for a key x that is present.
    """

    def __init__(self, m: int, b: int = 4, f: int = 8, max_kicks: int = 500, seed: int = 0) -> None:
        if m < 1 or m & (m - 1):
            raise ValueError("m must be a power of two")
        if b < 1 or f < 1:
            raise ValueError("b and f must be positive")
        self.m, self.b, self.f, self.max_kicks, self.seed = m, b, f, max_kicks, seed
        self.table: list[list[int]] = [[] for _ in range(m)]  # @line L1
        self.victim: tuple[int, int] | None = None
        self.rng = random.Random(seed)

    def _h(self, data: bytes) -> int:
        d = hashlib.blake2b(data, digest_size=8, key=self.seed.to_bytes(8, "little")).digest()
        return int.from_bytes(d, "little")

    def _fp_index(self, key: Key) -> tuple[int, int]:
        """The fingerprint of key (never 0) and its first bucket."""
        v = self._h(repr(key).encode())
        fp = (v >> 32) % ((1 << self.f) - 1) + 1 if self.f > 1 else 1
        return fp, (v & 0xFFFFFFFF) % self.m

    def _alt(self, i: int, fp: int) -> int:
        """The other bucket of fingerprint fp in bucket i. _alt(_alt(i, fp), fp) == i."""
        return i ^ (self._h(b"fp" + fp.to_bytes(8, "little")) % self.m)

    def _choose(self, n: int) -> int:
        return self.rng.randrange(n)

    def _place(self, i: int, fp: int) -> None:
        """Store fp in bucket i or its other bucket, kicking entries if needed."""
        for j in (i, self._alt(i, fp)):
            if len(self.table[j]) < self.b:  # @line L4
                self.table[j].append(fp)
                return
        i = (i, self._alt(i, fp))[self._choose(2)]  # @line L5
        for _ in range(self.max_kicks):  # @line L6
            s = self._choose(self.b)
            fp, self.table[i][s] = self.table[i][s], fp  # @line L7
            i = self._alt(i, fp)  # @line L8
            if len(self.table[i]) < self.b:  # @line L9
                self.table[i].append(fp)
                return
        self.victim = (i, fp)  # @line L10: the table is now full

    def insert(self, key: Key) -> bool:
        """Add key. False, with no change, if the filter is full."""
        if self.victim is not None:  # @line L2
            return False
        fp, i1 = self._fp_index(key)  # @line L3
        self._place(i1, fp)
        return True

    def query(self, key: Key) -> bool:
        """False if key is surely absent; True if it may be present."""
        fp, i1 = self._fp_index(key)  # @line L11
        i2 = self._alt(i1, fp)
        if fp in self.table[i1] or fp in self.table[i2]:  # @line L12
            return True
        return self.victim is not None and self.victim[1] == fp and self.victim[0] in (i1, i2)

    def delete(self, key: Key) -> bool:
        """Remove one copy of key. Precondition: key is present."""
        fp, i1 = self._fp_index(key)  # @line L13
        i2 = self._alt(i1, fp)
        for j in (i1, i2):
            if fp in self.table[j]:  # @line L14
                self.table[j].remove(fp)
                if self.victim is not None:  # @line L15: there is room again
                    vi, vfp = self.victim
                    self.victim = None
                    self._place(vi, vfp)
                return True
        if self.victim is not None and self.victim[1] == fp and self.victim[0] in (i1, i2):  # @line L16
            self.victim = None
            return True
        return False  # @line L17

    def __len__(self) -> int:
        return sum(len(t) for t in self.table) + (self.victim is not None)
test_cuckoo_filter.py 117 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/cuckoo-filter/. The authors are listed in prov/131BXM.jsonl.

import random
from collections import Counter

import pytest

from cuckoo_filter import CuckooFilter, Key

TABLE: dict[Key, tuple[int, int]] = {
    "a": (5, 0), "b": (6, 0), "c": (9, 0), "d": (7, 1), "e": (13, 0), "x": (7, 2), "y": (3, 3),
}


class FixedCF(CuckooFilter):
    def __init__(self) -> None:
        super().__init__(4, b=2, f=4)

    def _fp_index(self, key: Key) -> tuple[int, int]:
        return TABLE[key]

    def _alt(self, i: int, fp: int) -> int:
        return i ^ (fp % self.m)

    def _choose(self, n: int) -> int:
        return 0


def test_example() -> None:
    f = FixedCF()
    for k in "bacd":
        assert f.insert(k)
    assert f.table == [[6, 5], [9, 7], [], []]
    assert f.insert("e")  # kicks 6 from bucket 0 to bucket 0 ^ 2 = 2
    assert f.table == [[13, 5], [9, 7], [6], []]
    assert all(f.query(k) for k in "abcde")
    assert f.query("x")  # same fingerprint and buckets as d: a false positive
    assert not f.query("y")
    assert f.delete("a")
    assert f.table == [[13], [9, 7], [6], []]
    assert not f.query("a")


def test_victim() -> None:
    f = FixedCF()
    f.max_kicks = 3
    for k in "bacd":
        f.insert(k)
    f.table[2] = [1, 1]
    f.table[3] = [1, 1]  # fill the table so that e cannot be placed
    assert f.insert("e") and f.victim is not None
    assert not f.insert("y")  # full: no change
    assert all(f.query(k) for k in "abcde")
    assert f.delete("d")  # room again: the victim goes back into the table
    assert f.victim is None and all(f.query(k) for k in "abce")


def test_edges() -> None:
    with pytest.raises(ValueError):
        CuckooFilter(3)
    f = CuckooFilter(1, b=1, f=1)
    assert not f.query(0) and f.insert(0) and f.query(0)
    assert f.delete(0) and not f.query(0) and not f.delete(0)


def tokens_of_table(f: CuckooFilter) -> Counter[tuple[int, frozenset[int]]]:
    t: Counter[tuple[int, frozenset[int]]] = Counter()
    for j, bucket in enumerate(f.table):
        for fp in bucket:
            t[(fp, frozenset((j, f._alt(j, fp))))] += 1
    if f.victim is not None:
        j, fp = f.victim
        t[(fp, frozenset((j, f._alt(j, fp))))] += 1
    return t


def tokens_of_keys(f: CuckooFilter, live: Counter[int]) -> Counter[tuple[int, frozenset[int]]]:
    t: Counter[tuple[int, frozenset[int]]] = Counter()
    for x, c in live.items():
        fp, i1 = f._fp_index(x)
        if c:
            t[(fp, frozenset((i1, f._alt(i1, fp))))] += c
    return t


def test_random_against_multiset() -> None:
    rng = random.Random(2014)
    for _ in range(500):
        m = 1 << rng.randint(0, 4)
        b = rng.randint(1, 4)
        f = CuckooFilter(m, b, f=rng.randint(2, 10), max_kicks=rng.randint(1, 30), seed=rng.randrange(1000))
        live: Counter[int] = Counter()
        for _ in range(rng.randint(0, 80)):
            x = rng.randrange(40)
            r = rng.random()
            if r < 0.5:
                if f.insert(x):
                    live[x] += 1
            elif r < 0.8 and live[x] > 0:
                assert f.delete(x)
                live[x] -= 1
            else:
                assert f.query(x) or live[x] == 0
            assert tokens_of_table(f) == tokens_of_keys(f, live)
            assert all(f.query(y) for y in live if live[y])
            assert all(len(t) <= b for t in f.table)


def test_load_and_rate() -> None:
    f = CuckooFilter(1 << 10, b=4, f=12, seed=7)
    n = 0
    while f.insert(("k", n)):
        n += 1
    assert n >= 0.9 * 4 * (1 << 10)
    fp = sum(f.query(("q", i)) for i in range(20000))
    assert fp / 20000 <= 2 * 4 / ((1 << 12) - 1) + 0.002

What the next level needs

  • the tests did not run in this build

Read the full page about Cuckoo filter