Algorithm Search⌘K

FKS perfect hashing in Python

Two-level universal hashing that gives O(n) space and worst-case O(1) static lookup.

File
fks_perfect_hashing.py, 91 lines
Level
I0 not checked yet
Toolchain
Python 3.14, pytest 9
Test framework
pytest
Run the tests
python3 -m pytest
fks_perfect_hashing.pyRawGitHub
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/fks-perfect-hashing/. The authors are listed in prov/VWBFM0.jsonl.

import random
from dataclasses import dataclass, field

P61 = (1 << 61) - 1  # a Mersenne prime


def h_ab(a: int, b: int, p: int, m: int, x: int) -> int:
    """The Carter-Wegman hash ((a x + b) mod p) mod m."""
    return ((a * x + b) % p) % m


@dataclass
class FKSTable:
    """A two-level perfect hash table for a fixed set of keys."""

    p: int
    m: int
    a: int
    b: int
    sizes: list[int] = field(default_factory=list)  # m_j for each bucket
    params: list[tuple[int, int]] = field(default_factory=list)  # (a_j, b_j)
    slots: list[list[int | None]] = field(default_factory=list)  # T_j

    def first(self, x: int) -> int:
        return h_ab(self.a, self.b, self.p, self.m, x)

    def second(self, j: int, x: int) -> int:
        a_j, b_j = self.params[j]
        return h_ab(a_j, b_j, self.p, self.sizes[j], x)


def fks_build(keys: list[int], rng: random.Random | None = None, p: int = P61) -> FKSTable:
    """Build an FKS table for the distinct keys, each in [0, p), p prime.

    Las Vegas: the result is always correct; the expected time is O(n), and
    the second level has at most 4n slots. Raises ValueError on a duplicate
    key or a key out of range.
    """
    rng = rng if rng is not None else random.Random()
    n = len(keys)
    if len(set(keys)) != n:
        raise ValueError("duplicate key")
    if any(not 0 <= x < p for x in keys):
        raise ValueError("key out of range")
    m = max(n, 1)  # @line L1
    while True:  # @line L2
        a, b = rng.randrange(1, p), rng.randrange(p)  # @line L3
        buckets: list[list[int]] = [[] for _ in range(m)]
        for x in keys:
            buckets[h_ab(a, b, p, m, x)].append(x)  # @line L4
        if sum(len(s) ** 2 for s in buckets) <= 4 * n:  # @line L5
            break
    t = FKSTable(p, m, a, b)
    for s in buckets:  # @line L6
        m_j = len(s) ** 2  # @line L7
        slots: list[int | None] = [None] * m_j
        a_j, b_j = 1, 0
        if m_j > 0:  # @line L8
            while True:
                a_j, b_j = rng.randrange(1, p), rng.randrange(p)  # @line L9
                if _place(s, slots, a_j, b_j, p, m_j):  # @line L10
                    break  # @line L11: _place has stored each key
        t.sizes.append(m_j)
        t.params.append((a_j, b_j))
        t.slots.append(slots)
    return t


def _place(s: list[int], slots: list[int | None], a: int, b: int, p: int, m: int) -> bool:
    """Write the keys of s into slots by h_{a,b,m}; on a collision undo and return False."""
    used: list[int] = []
    for x in s:
        i = h_ab(a, b, p, m, x)
        if slots[i] is not None:
            for k in used:
                slots[k] = None
            return False
        slots[i] = x
        used.append(i)
    return True


def fks_search(t: FKSTable, x: int) -> bool:
    """Return True if x is a key of the table. O(1) time in the worst case."""
    j = t.first(x)  # @line L12
    if t.sizes[j] == 0:  # @line L13
        return False
    return t.slots[j][t.second(j, x)] == x  # @line L14
test_fks_perfect_hashing.py 66 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/fks-perfect-hashing/. The authors are listed in prov/VWBFM0.jsonl.

import random

import pytest

from fks_perfect_hashing import P61, fks_build, fks_search, h_ab

S = [10, 22, 37, 40, 52, 60, 70, 72, 75]


def test_page_example_numbers() -> None:
    p, m = 101, 9
    buckets = [[x for x in S if h_ab(3, 42, p, m, x) == j] for j in range(m)]
    assert buckets == [[10], [], [60, 72, 75], [], [], [70], [], [22, 37, 40, 52], []]
    assert sum(len(b) ** 2 for b in buckets) == 27
    assert [h_ab(3, 1, p, 9, x) for x in [60, 72, 75]] == [8, 6, 6]
    assert [h_ab(10, 18, p, 9, x) for x in [60, 72, 75]] == [3, 4, 7]
    assert [h_ab(10, 18, p, 16, x) for x in [22, 37, 40, 52]] == [4, 5, 14, 1]
    assert h_ab(3, 42, p, m, 50) == 1 and h_ab(3, 42, p, m, 13) == 0


def test_page_example_build() -> None:
    t = fks_build(S, random.Random(7), p=101)
    assert all(fks_search(t, x) for x in S)
    assert not any(fks_search(t, x) for x in range(101) if x not in S)
    assert sum(t.sizes) <= 4 * len(S)


def test_edges() -> None:
    t = fks_build([], random.Random(1))
    assert not fks_search(t, 0) and not fks_search(t, 5)
    t = fks_build([0], random.Random(1))
    assert fks_search(t, 0) and not fks_search(t, 1)
    big = [P61 - 1, 0, 2**60, 2**32 + 7]
    t = fks_build(big, random.Random(2))
    assert all(fks_search(t, x) for x in big) and not fks_search(t, 1)
    with pytest.raises(ValueError):
        fks_build([3, 3])
    with pytest.raises(ValueError):
        fks_build([P61])


def test_injective_second_level() -> None:
    rng = random.Random(3)
    keys = rng.sample(range(10**6), 300)
    t = fks_build(keys, rng)
    for j in range(t.m):
        stored = [x for x in t.slots[j] if x is not None]
        assert all(t.first(x) == j for x in stored)
        assert sorted(stored) == sorted(x for x in keys if t.first(x) == j)
    assert sum(t.sizes) <= 4 * len(keys)


def test_random_against_set() -> None:
    rng = random.Random(1984)
    for _ in range(600):
        p = rng.choice([101, 1009, P61])
        universe = min(p, 400)
        keys = rng.sample(range(universe), rng.randint(0, 40))
        t = fks_build(keys, rng, p=p)
        model = set(keys)
        assert sum(t.sizes) <= 4 * len(keys)
        for x in range(universe):
            assert fks_search(t, x) == (x in model)

What the next level needs

  • the tests did not run in this build

Read the full page about FKS perfect hashing