Algorithm Search⌘K

Linear hashing in Python

Dynamic hash table that grows one bucket at a time, without a full rehash.

File
linear_hashing.py, 119 lines
Level
I0 not checked yet
Toolchain
Python 3.14, pytest 9
Test framework
pytest
Run the tests
python3 -m pytest
linear_hashing.pyRawGitHub
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/linear-hashing/. The authors are listed in prov/EA0KDC.jsonl.

from collections.abc import Hashable
from typing import Generic, TypeVar

K = TypeVar("K", bound=Hashable)
V = TypeVar("V")


class LinearHashing(Generic[K, V]):
    """A map from keys to values by Litwin's linear hashing, with chained buckets.

    The table has N * 2**L + p buckets. A key k lives in bucket h(k) mod N * 2**L,
    or in bucket h(k) mod N * 2**(L+1) when the first value is below the split
    pointer p. Here h is Python's hash. An insertion that makes the load factor
    n / (number of buckets) exceed max_load splits bucket p, then moves p one
    step; a deletion that makes it fall below min_load undoes the last split.
    Each operation costs O(1) expected amortized time when the hash values
    behave like independent uniform values and max_load is a constant.
    """

    def __init__(self, initial_buckets: int = 2, max_load: float = 1.0, min_load: float | None = None) -> None:
        """Make an empty table. Needs initial_buckets >= 1, max_load > 0 and 0 <= min_load <= max_load / 2."""
        if min_load is None:
            min_load = max_load / 4
        if initial_buckets < 1 or max_load <= 0 or not 0 <= min_load <= max_load / 2:
            raise ValueError("need initial_buckets >= 1, max_load > 0, 0 <= min_load <= max_load / 2")
        self.n0 = initial_buckets  # N in the text
        self.max_load = max_load
        self.min_load = min_load
        self.level = 0  # L
        self.p = 0  # the split pointer
        self.n = 0  # the number of pairs
        self.buckets: list[list[tuple[K, V]]] = [[] for _ in range(initial_buckets)]

    def __len__(self) -> int:
        return self.n

    def address(self, key: K) -> int:
        """Return the index of the bucket that holds key, if it is present."""
        hk = hash(key)
        a = hk % (self.n0 << self.level)  # @line L1
        if a < self.p:  # @line L2
            a = hk % (self.n0 << (self.level + 1))
        return a  # @line L3

    def search(self, key: K) -> V | None:
        """Return the value of key, or None when key is absent."""
        for k, v in self.buckets[self.address(key)]:  # @line L4
            if k == key:  # @line L5
                return v
        return None  # @line L6

    def insert(self, key: K, value: V) -> None:
        """Map key to value; replace the old value when key is present."""
        a = self.address(key)  # @line L7
        bucket = self.buckets[a]
        for i, (k, _) in enumerate(bucket):  # @line L8
            if k == key:
                bucket[i] = (key, value)
                return
        bucket.append((key, value))  # @line L9
        self.n += 1
        while self.n > self.max_load * len(self.buckets):  # @line L10
            self.split()

    def split(self) -> None:
        """Split the bucket at the split pointer and move the pointer one step."""
        half = self.n0 << self.level
        self.buckets.append([])  # @line L11: the new bucket gets index p + half
        old = self.buckets[self.p]  # @line L12
        self.buckets[self.p] = []
        wide = half << 1
        for k, v in old:  # @line L13
            self.buckets[hash(k) % wide].append((k, v))  # @line L14
        self.p += 1  # @line L15
        if self.p == half:  # @line L16
            self.level += 1
            self.p = 0

    def delete(self, key: K) -> bool:
        """Remove key. Return True when it was present."""
        bucket = self.buckets[self.address(key)]  # D1
        for i, (k, _) in enumerate(bucket):  # D2
            if k == key:
                bucket[i] = bucket[-1]  # D3: order inside a bucket does not matter
                bucket.pop()
                self.n -= 1
                while len(self.buckets) > self.n0 and self.n < self.min_load * len(self.buckets):  # D4
                    self.merge()
                return True  # D5
        return False

    def merge(self) -> None:
        """Undo the last split: merge the last bucket into its partner."""
        if self.p == 0:  # M1
            self.level -= 1
            self.p = self.n0 << self.level
        self.p -= 1  # M2
        last = self.buckets.pop()  # M3, M4: the last bucket has index p + N * 2**L
        self.buckets[self.p].extend(last)

    def items(self) -> list[tuple[K, V]]:
        """Return all pairs, in no particular order."""
        return [pair for bucket in self.buckets for pair in bucket]

    def check_invariant(self) -> None:
        """Raise AssertionError when an invariant of the page fails."""
        half = self.n0 << self.level
        assert self.level >= 0 and 0 <= self.p < half  # I1
        assert len(self.buckets) == half + self.p >= self.n0  # I1
        keys = []
        for j, bucket in enumerate(self.buckets):
            for k, _ in bucket:
                assert self.address(k) == j  # I2
                keys.append(k)
        assert len(keys) == len(set(keys)) == self.n  # I3
        assert self.n <= self.max_load * len(self.buckets)  # I4
test_linear_hashing.py 101 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/linear-hashing/. The authors are listed in prov/EA0KDC.jsonl.

import random

import pytest

from linear_hashing import LinearHashing


def layout(t: LinearHashing) -> list[list[int]]:
    return [sorted(k for k, _ in b) for b in t.buckets]


def test_example() -> None:
    t: LinearHashing[int, str] = LinearHashing(2, 1.0, 0.5)
    states = []
    for k in [5, 12, 7, 20, 3, 18, 9]:
        t.insert(k, str(k))
        t.check_invariant()
        states.append((t.level, t.p, layout(t)))
    assert states[2] == (0, 1, [[12], [5, 7], []])
    assert states[3] == (1, 0, [[12, 20], [5], [], [7]])
    assert states[4] == (1, 1, [[], [5], [], [3, 7], [12, 20]])
    assert states[6] == (1, 3, [[], [9], [18], [3, 7], [12, 20], [5], []])
    assert t.address(20) == 4 and t.address(5) == 5 and t.address(7) == 3
    for k in [12, 20, 9]:
        assert t.delete(k)
        t.check_invariant()
    assert (t.level, t.p, len(t.buckets)) == (1, 3, 7)
    assert t.delete(5)
    t.check_invariant()
    assert (t.level, t.p, layout(t)) == (1, 2, [[], [], [18], [3, 7], [], []])
    assert t.search(18) == "18" and t.search(5) is None


def test_edges() -> None:
    t: LinearHashing[int, int] = LinearHashing()
    assert len(t) == 0 and t.search(1) is None and not t.delete(1)
    t.insert(1, 10)
    t.insert(1, 11)  # replace, not a second pair
    assert len(t) == 1 and t.search(1) == 11
    assert t.delete(1) and len(t) == 0 and not t.delete(1)
    t.check_invariant()
    # negative keys, large keys and keys that are not integers
    for k in [-1, -2, -(2**63), 2**63 - 1, 2**64 + 7]:
        t.insert(k, k)
    t.insert("abc", 1)
    t.insert((1, 2), 2)
    t.check_invariant()
    assert t.search(-(2**63)) == -(2**63) and t.search("abc") == 1 and t.search((1, 2)) == 2
    # all keys in one bucket: the structure stays correct
    u: LinearHashing[int, int] = LinearHashing(1, 2.0)
    for i in range(200):
        u.insert(i * 2**20, i)
        u.check_invariant()
    assert all(u.search(i * 2**20) == i for i in range(200))
    # one initial bucket and a load factor below 1, which needs several splits per insert
    w: LinearHashing[int, int] = LinearHashing(1, 0.3, 0.1)
    for i in range(100):
        w.insert(i, i)
        w.check_invariant()
    assert len(w.buckets) >= 100 / 0.3
    for i in range(100):
        assert w.delete(i)
        w.check_invariant()
    assert len(w.buckets) == 1 and w.level == 0 and w.p == 0


def test_bad_parameters() -> None:
    with pytest.raises(ValueError):
        LinearHashing(0)
    with pytest.raises(ValueError):
        LinearHashing(2, 0.0)
    with pytest.raises(ValueError):
        LinearHashing(2, 1.0, 0.6)


def test_random_against_dict() -> None:
    rng = random.Random(1980)
    for _ in range(500):
        n0 = rng.randint(1, 5)
        hi = rng.choice([0.5, 1.0, 2.0, 3.0])
        t: LinearHashing[int, int] = LinearHashing(n0, hi, hi * rng.choice([0.0, 0.25, 0.5]))
        model: dict[int, int] = {}
        span = rng.choice([5, 30, 200])
        for _ in range(rng.randint(0, 80)):
            k = rng.randint(-span, span)
            op = rng.random()
            if op < 0.5:
                v = rng.randint(0, 9)
                t.insert(k, v)
                model[k] = v
            elif op < 0.8:
                assert t.delete(k) == (k in model)
                model.pop(k, None)
            else:
                assert t.search(k) == model.get(k)
            t.check_invariant()
        assert len(t) == len(model)
        assert sorted(t.items()) == sorted(model.items())

What the next level needs

  • the tests did not run in this build

Read the full page about Linear hashing