Algorithm Search⌘K

Self-organizing list in Python

List that reorders its elements on access to reduce expected search cost.

File
self_organizing_list.py, 135 lines
Level
I0 not checked yet
Toolchain
Python 3.14, pytest 9
Test framework
pytest
Run the tests
python3 -m pytest
self_organizing_list.pyRawGitHub
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/self-organizing-list/. The authors are listed in prov/2WXV0M.jsonl.

from collections.abc import Hashable, Iterator
from typing import Generic, TypeVar

K = TypeVar("K", bound=Hashable)
V = TypeVar("V")

RULES = ("mtf", "transpose", "count")


class _Node(Generic[K, V]):
    __slots__ = ("key", "value", "next", "count")

    def __init__(self, key: K, value: V) -> None:
        self.key = key
        self.value = value
        self.next: _Node[K, V] | None = None
        self.count = 1


class SelfOrganizingList(Generic[K, V]):
    """A dictionary stored as a singly linked list that reorders itself on each hit.

    The rule is "mtf" (move to front), "transpose" (one place forward) or
    "count" (keep the list sorted by access count, ties in order of arrival).
    get and put of a key at position i cost Theta(i); a miss and the put of
    a new key cost Theta(n); delete costs Theta(i). Space Theta(n).
    """

    def __init__(self, rule: str = "mtf") -> None:
        if rule not in RULES:
            raise ValueError(f"rule must be one of {RULES}")
        self.rule = rule
        self.head: _Node[K, V] | None = None
        self.n = 0

    def _access(self, k: K) -> "_Node[K, V] | None":
        pp: _Node[K, V] | None = None
        prev: _Node[K, V] | None = None
        cur = self.head  # @line L1
        while cur is not None and cur.key != k:  # @line L2
            pp, prev, cur = prev, cur, cur.next  # @line L3
        if cur is None:  # @line L4
            return None
        if prev is None:  # @line L5
            self.head = cur.next
        else:
            prev.next = cur.next
        p: _Node[K, V] | None
        if self.rule == "mtf":  # @line L6
            p = None
        elif self.rule == "transpose":  # @line L7
            p = pp
        else:  # @line L8
            cur.count += 1  # @line L9
            p, q = None, self.head
            while q is not None and q.count >= cur.count:  # @line L10
                p, q = q, q.next
        if p is None:  # @line L11
            cur.next = self.head
            self.head = cur
        else:
            cur.next = p.next
            p.next = cur
        return cur  # @line L12

    def get(self, k: K) -> V | None:
        """Return the value of k, or None if k is absent. Reorders on a hit."""
        x = self._access(k)
        return None if x is None else x.value

    def put(self, k: K, v: V) -> None:
        """Associate v with k. Theta(n) for a new key."""
        x = self._access(k)
        if x is not None:
            x.value = v
            return
        x = _Node(k, v)
        self.n += 1
        if self.rule == "mtf" or self.head is None:
            x.next = self.head
            self.head = x
            return
        last = self.head
        while last.next is not None:
            last = last.next
        last.next = x

    def delete(self, k: K) -> None:
        """Remove k and its value, if present. Does not reorder the other keys."""
        prev: _Node[K, V] | None = None
        cur = self.head
        while cur is not None and cur.key != k:
            prev, cur = cur, cur.next
        if cur is None:
            return
        if prev is None:
            self.head = cur.next
        else:
            prev.next = cur.next
        self.n -= 1

    def __len__(self) -> int:
        return self.n

    def keys(self) -> Iterator[K]:
        """The keys from front to back, without reordering."""
        cur = self.head
        while cur is not None:
            yield cur.key
            cur = cur.next

    def counts(self) -> list[int]:
        """The access counts from front to back."""
        out, cur = [], self.head
        while cur is not None:
            out.append(cur.count)
            cur = cur.next
        return out

    def check_invariant(self) -> None:
        """Assert (I1), the distinct keys of (I2), and (I3) under the rule count."""
        seen, cur, steps = set(), self.head, 0
        while cur is not None:
            assert steps < self.n, "cycle or wrong size"
            assert cur.key not in seen, "duplicate key"
            seen.add(cur.key)
            steps += 1
            cur = cur.next
        assert steps == self.n
        if self.rule == "count":
            c = self.counts()
            assert all(c[j] >= c[j + 1] for j in range(len(c) - 1))
test_self_organizing_list.py 110 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/self-organizing-list/. The authors are listed in prov/2WXV0M.jsonl.

import random

import pytest

from self_organizing_list import SelfOrganizingList


def build(rule: str, keys: str) -> SelfOrganizingList[str, int]:
    lst: SelfOrganizingList[str, int] = SelfOrganizingList(rule)
    for v, k in enumerate(keys):
        lst.put(k, v)
    if rule == "mtf":  # new keys go to the front; rebuild the order a, b, c, ...
        lst = SelfOrganizingList(rule)
        for v, k in reversed(list(enumerate(keys))):
            lst.put(k, v)
    return lst


def positions(lst: SelfOrganizingList[str, int], reqs: str) -> list[int]:
    out = []
    for k in reqs:
        out.append(list(lst.keys()).index(k) + 1)
        lst.get(k)
        lst.check_invariant()
    return out


def test_example() -> None:
    m = build("mtf", "abcde")
    assert list(m.keys()) == list("abcde")
    assert sum(positions(m, "dbdeadb")) == 25 and "".join(m.keys()) == "bdaec"
    t = build("transpose", "abcde")
    assert positions(t, "dbdeadb") == [4, 2, 3, 5, 3, 3, 1] and "".join(t.keys()) == "bdaec"
    c = build("count", "abcde")
    assert positions(c, "dbdeadb") == [4, 3, 1, 5, 4, 1, 2]
    assert "".join(c.keys()) == "dbeac" and c.counts() == [4, 3, 2, 2, 1]
    t2 = build("transpose", "abcde")
    assert sum(positions(t2, "ed" * 5)) == 50
    m2 = build("mtf", "abcde")
    assert sum(positions(m2, "ed" * 5)) == 26


def test_edges() -> None:
    for rule in ("mtf", "transpose", "count"):
        lst: SelfOrganizingList[int, str] = SelfOrganizingList(rule)
        assert lst.get(1) is None and len(lst) == 0
        lst.delete(1)
        lst.put(1, "x")
        assert lst.get(1) == "x" and lst.get(2) is None
        lst.put(1, "y")
        assert lst.get(1) == "y" and len(lst) == 1
        lst.delete(1)
        assert len(lst) == 0 and list(lst.keys()) == []
        lst.check_invariant()
    with pytest.raises(ValueError):
        SelfOrganizingList("random")


def model_access(rule: str, order: list[list], k: int) -> None:
    """Independent model: a Python list of [key, count] pairs."""
    i = [e[0] for e in order].index(k)
    e = order.pop(i)
    if rule == "mtf":
        order.insert(0, e)
    elif rule == "transpose":
        order.insert(max(i - 1, 0), e)
    else:
        e[1] += 1
        j = 0
        while j < len(order) and order[j][1] >= e[1]:
            j += 1
        order.insert(j, e)


def test_random_against_model() -> None:
    rng = random.Random(1976)
    for trial in range(600):
        rule = ("mtf", "transpose", "count")[trial % 3]
        lst: SelfOrganizingList[int, int] = SelfOrganizingList(rule)
        values: dict[int, int] = {}
        order: list[list] = []
        for _ in range(rng.randint(0, 40)):
            op, k = rng.random(), rng.randrange(8)
            if op < 0.5:
                assert lst.get(k) == values.get(k)
                if k in values:
                    model_access(rule, order, k)
            elif op < 0.85:
                v = rng.randrange(100)
                lst.put(k, v)
                if k in values:
                    model_access(rule, order, k)
                elif rule == "mtf":
                    order.insert(0, [k, 1])
                else:
                    order.append([k, 1])
                values[k] = v
            else:
                lst.delete(k)
                if k in values:
                    del values[k]
                    order = [e for e in order if e[0] != k]
            lst.check_invariant()
            assert list(lst.keys()) == [e[0] for e in order]
            assert len(lst) == len(values)
            if rule == "count":
                assert lst.counts() == [e[1] for e in order]

What the next level needs

  • the tests did not run in this build

Read the full page about Self-organizing list