FKS perfect hashing in Python
Two-level universal hashing that gives O(n) space and worst-case O(1) static lookup.
- File
fks_perfect_hashing.py, 91 lines- Level
- I0 not checked yet
- Toolchain
- Python 3.14, pytest 9
- Test framework
- pytest
- Run the tests
python3 -m pytest
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/fks-perfect-hashing/. The authors are listed in prov/VWBFM0.jsonl.
import random
from dataclasses import dataclass, field
P61 = (1 << 61) - 1 # a Mersenne prime
def h_ab(a: int, b: int, p: int, m: int, x: int) -> int:
"""The Carter-Wegman hash ((a x + b) mod p) mod m."""
return ((a * x + b) % p) % m
@dataclass
class FKSTable:
"""A two-level perfect hash table for a fixed set of keys."""
p: int
m: int
a: int
b: int
sizes: list[int] = field(default_factory=list) # m_j for each bucket
params: list[tuple[int, int]] = field(default_factory=list) # (a_j, b_j)
slots: list[list[int | None]] = field(default_factory=list) # T_j
def first(self, x: int) -> int:
return h_ab(self.a, self.b, self.p, self.m, x)
def second(self, j: int, x: int) -> int:
a_j, b_j = self.params[j]
return h_ab(a_j, b_j, self.p, self.sizes[j], x)
def fks_build(keys: list[int], rng: random.Random | None = None, p: int = P61) -> FKSTable:
"""Build an FKS table for the distinct keys, each in [0, p), p prime.
Las Vegas: the result is always correct; the expected time is O(n), and
the second level has at most 4n slots. Raises ValueError on a duplicate
key or a key out of range.
"""
rng = rng if rng is not None else random.Random()
n = len(keys)
if len(set(keys)) != n:
raise ValueError("duplicate key")
if any(not 0 <= x < p for x in keys):
raise ValueError("key out of range")
m = max(n, 1) # @line L1
while True: # @line L2
a, b = rng.randrange(1, p), rng.randrange(p) # @line L3
buckets: list[list[int]] = [[] for _ in range(m)]
for x in keys:
buckets[h_ab(a, b, p, m, x)].append(x) # @line L4
if sum(len(s) ** 2 for s in buckets) <= 4 * n: # @line L5
break
t = FKSTable(p, m, a, b)
for s in buckets: # @line L6
m_j = len(s) ** 2 # @line L7
slots: list[int | None] = [None] * m_j
a_j, b_j = 1, 0
if m_j > 0: # @line L8
while True:
a_j, b_j = rng.randrange(1, p), rng.randrange(p) # @line L9
if _place(s, slots, a_j, b_j, p, m_j): # @line L10
break # @line L11: _place has stored each key
t.sizes.append(m_j)
t.params.append((a_j, b_j))
t.slots.append(slots)
return t
def _place(s: list[int], slots: list[int | None], a: int, b: int, p: int, m: int) -> bool:
"""Write the keys of s into slots by h_{a,b,m}; on a collision undo and return False."""
used: list[int] = []
for x in s:
i = h_ab(a, b, p, m, x)
if slots[i] is not None:
for k in used:
slots[k] = None
return False
slots[i] = x
used.append(i)
return True
def fks_search(t: FKSTable, x: int) -> bool:
"""Return True if x is a key of the table. O(1) time in the worst case."""
j = t.first(x) # @line L12
if t.sizes[j] == 0: # @line L13
return False
return t.slots[j][t.second(j, x)] == x # @line L14
test_fks_perfect_hashing.py 66 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/fks-perfect-hashing/. The authors are listed in prov/VWBFM0.jsonl.
import random
import pytest
from fks_perfect_hashing import P61, fks_build, fks_search, h_ab
S = [10, 22, 37, 40, 52, 60, 70, 72, 75]
def test_page_example_numbers() -> None:
p, m = 101, 9
buckets = [[x for x in S if h_ab(3, 42, p, m, x) == j] for j in range(m)]
assert buckets == [[10], [], [60, 72, 75], [], [], [70], [], [22, 37, 40, 52], []]
assert sum(len(b) ** 2 for b in buckets) == 27
assert [h_ab(3, 1, p, 9, x) for x in [60, 72, 75]] == [8, 6, 6]
assert [h_ab(10, 18, p, 9, x) for x in [60, 72, 75]] == [3, 4, 7]
assert [h_ab(10, 18, p, 16, x) for x in [22, 37, 40, 52]] == [4, 5, 14, 1]
assert h_ab(3, 42, p, m, 50) == 1 and h_ab(3, 42, p, m, 13) == 0
def test_page_example_build() -> None:
t = fks_build(S, random.Random(7), p=101)
assert all(fks_search(t, x) for x in S)
assert not any(fks_search(t, x) for x in range(101) if x not in S)
assert sum(t.sizes) <= 4 * len(S)
def test_edges() -> None:
t = fks_build([], random.Random(1))
assert not fks_search(t, 0) and not fks_search(t, 5)
t = fks_build([0], random.Random(1))
assert fks_search(t, 0) and not fks_search(t, 1)
big = [P61 - 1, 0, 2**60, 2**32 + 7]
t = fks_build(big, random.Random(2))
assert all(fks_search(t, x) for x in big) and not fks_search(t, 1)
with pytest.raises(ValueError):
fks_build([3, 3])
with pytest.raises(ValueError):
fks_build([P61])
def test_injective_second_level() -> None:
rng = random.Random(3)
keys = rng.sample(range(10**6), 300)
t = fks_build(keys, rng)
for j in range(t.m):
stored = [x for x in t.slots[j] if x is not None]
assert all(t.first(x) == j for x in stored)
assert sorted(stored) == sorted(x for x in keys if t.first(x) == j)
assert sum(t.sizes) <= 4 * len(keys)
def test_random_against_set() -> None:
rng = random.Random(1984)
for _ in range(600):
p = rng.choice([101, 1009, P61])
universe = min(p, 400)
keys = rng.sample(range(universe), rng.randint(0, 40))
t = fks_build(keys, rng, p=p)
model = set(keys)
assert sum(t.sizes) <= 4 * len(keys)
for x in range(universe):
assert fks_search(t, x) == (x in model)
What the next level needs
- the tests did not run in this build