Self-organizing list in Python
List that reorders its elements on access to reduce expected search cost.
- File
self_organizing_list.py, 135 lines- Level
- I0 not checked yet
- Toolchain
- Python 3.14, pytest 9
- Test framework
- pytest
- Run the tests
python3 -m pytest
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/self-organizing-list/. The authors are listed in prov/2WXV0M.jsonl.
from collections.abc import Hashable, Iterator
from typing import Generic, TypeVar
K = TypeVar("K", bound=Hashable)
V = TypeVar("V")
RULES = ("mtf", "transpose", "count")
class _Node(Generic[K, V]):
__slots__ = ("key", "value", "next", "count")
def __init__(self, key: K, value: V) -> None:
self.key = key
self.value = value
self.next: _Node[K, V] | None = None
self.count = 1
class SelfOrganizingList(Generic[K, V]):
"""A dictionary stored as a singly linked list that reorders itself on each hit.
The rule is "mtf" (move to front), "transpose" (one place forward) or
"count" (keep the list sorted by access count, ties in order of arrival).
get and put of a key at position i cost Theta(i); a miss and the put of
a new key cost Theta(n); delete costs Theta(i). Space Theta(n).
"""
def __init__(self, rule: str = "mtf") -> None:
if rule not in RULES:
raise ValueError(f"rule must be one of {RULES}")
self.rule = rule
self.head: _Node[K, V] | None = None
self.n = 0
def _access(self, k: K) -> "_Node[K, V] | None":
pp: _Node[K, V] | None = None
prev: _Node[K, V] | None = None
cur = self.head # @line L1
while cur is not None and cur.key != k: # @line L2
pp, prev, cur = prev, cur, cur.next # @line L3
if cur is None: # @line L4
return None
if prev is None: # @line L5
self.head = cur.next
else:
prev.next = cur.next
p: _Node[K, V] | None
if self.rule == "mtf": # @line L6
p = None
elif self.rule == "transpose": # @line L7
p = pp
else: # @line L8
cur.count += 1 # @line L9
p, q = None, self.head
while q is not None and q.count >= cur.count: # @line L10
p, q = q, q.next
if p is None: # @line L11
cur.next = self.head
self.head = cur
else:
cur.next = p.next
p.next = cur
return cur # @line L12
def get(self, k: K) -> V | None:
"""Return the value of k, or None if k is absent. Reorders on a hit."""
x = self._access(k)
return None if x is None else x.value
def put(self, k: K, v: V) -> None:
"""Associate v with k. Theta(n) for a new key."""
x = self._access(k)
if x is not None:
x.value = v
return
x = _Node(k, v)
self.n += 1
if self.rule == "mtf" or self.head is None:
x.next = self.head
self.head = x
return
last = self.head
while last.next is not None:
last = last.next
last.next = x
def delete(self, k: K) -> None:
"""Remove k and its value, if present. Does not reorder the other keys."""
prev: _Node[K, V] | None = None
cur = self.head
while cur is not None and cur.key != k:
prev, cur = cur, cur.next
if cur is None:
return
if prev is None:
self.head = cur.next
else:
prev.next = cur.next
self.n -= 1
def __len__(self) -> int:
return self.n
def keys(self) -> Iterator[K]:
"""The keys from front to back, without reordering."""
cur = self.head
while cur is not None:
yield cur.key
cur = cur.next
def counts(self) -> list[int]:
"""The access counts from front to back."""
out, cur = [], self.head
while cur is not None:
out.append(cur.count)
cur = cur.next
return out
def check_invariant(self) -> None:
"""Assert (I1), the distinct keys of (I2), and (I3) under the rule count."""
seen, cur, steps = set(), self.head, 0
while cur is not None:
assert steps < self.n, "cycle or wrong size"
assert cur.key not in seen, "duplicate key"
seen.add(cur.key)
steps += 1
cur = cur.next
assert steps == self.n
if self.rule == "count":
c = self.counts()
assert all(c[j] >= c[j + 1] for j in range(len(c) - 1))
test_self_organizing_list.py 110 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/self-organizing-list/. The authors are listed in prov/2WXV0M.jsonl.
import random
import pytest
from self_organizing_list import SelfOrganizingList
def build(rule: str, keys: str) -> SelfOrganizingList[str, int]:
lst: SelfOrganizingList[str, int] = SelfOrganizingList(rule)
for v, k in enumerate(keys):
lst.put(k, v)
if rule == "mtf": # new keys go to the front; rebuild the order a, b, c, ...
lst = SelfOrganizingList(rule)
for v, k in reversed(list(enumerate(keys))):
lst.put(k, v)
return lst
def positions(lst: SelfOrganizingList[str, int], reqs: str) -> list[int]:
out = []
for k in reqs:
out.append(list(lst.keys()).index(k) + 1)
lst.get(k)
lst.check_invariant()
return out
def test_example() -> None:
m = build("mtf", "abcde")
assert list(m.keys()) == list("abcde")
assert sum(positions(m, "dbdeadb")) == 25 and "".join(m.keys()) == "bdaec"
t = build("transpose", "abcde")
assert positions(t, "dbdeadb") == [4, 2, 3, 5, 3, 3, 1] and "".join(t.keys()) == "bdaec"
c = build("count", "abcde")
assert positions(c, "dbdeadb") == [4, 3, 1, 5, 4, 1, 2]
assert "".join(c.keys()) == "dbeac" and c.counts() == [4, 3, 2, 2, 1]
t2 = build("transpose", "abcde")
assert sum(positions(t2, "ed" * 5)) == 50
m2 = build("mtf", "abcde")
assert sum(positions(m2, "ed" * 5)) == 26
def test_edges() -> None:
for rule in ("mtf", "transpose", "count"):
lst: SelfOrganizingList[int, str] = SelfOrganizingList(rule)
assert lst.get(1) is None and len(lst) == 0
lst.delete(1)
lst.put(1, "x")
assert lst.get(1) == "x" and lst.get(2) is None
lst.put(1, "y")
assert lst.get(1) == "y" and len(lst) == 1
lst.delete(1)
assert len(lst) == 0 and list(lst.keys()) == []
lst.check_invariant()
with pytest.raises(ValueError):
SelfOrganizingList("random")
def model_access(rule: str, order: list[list], k: int) -> None:
"""Independent model: a Python list of [key, count] pairs."""
i = [e[0] for e in order].index(k)
e = order.pop(i)
if rule == "mtf":
order.insert(0, e)
elif rule == "transpose":
order.insert(max(i - 1, 0), e)
else:
e[1] += 1
j = 0
while j < len(order) and order[j][1] >= e[1]:
j += 1
order.insert(j, e)
def test_random_against_model() -> None:
rng = random.Random(1976)
for trial in range(600):
rule = ("mtf", "transpose", "count")[trial % 3]
lst: SelfOrganizingList[int, int] = SelfOrganizingList(rule)
values: dict[int, int] = {}
order: list[list] = []
for _ in range(rng.randint(0, 40)):
op, k = rng.random(), rng.randrange(8)
if op < 0.5:
assert lst.get(k) == values.get(k)
if k in values:
model_access(rule, order, k)
elif op < 0.85:
v = rng.randrange(100)
lst.put(k, v)
if k in values:
model_access(rule, order, k)
elif rule == "mtf":
order.insert(0, [k, 1])
else:
order.append([k, 1])
values[k] = v
else:
lst.delete(k)
if k in values:
del values[k]
order = [e for e in order if e[0] != k]
lst.check_invariant()
assert list(lst.keys()) == [e[0] for e in order]
assert len(lst) == len(values)
if rule == "count":
assert lst.counts() == [e[1] for e in order]
What the next level needs
- the tests did not run in this build