Cache-oblivious matrix multiplication in Python
Halves the largest of the three dimensions until the pieces are tiny; without knowing M or B it makes O(n^2/B + n^3/(B sqrt(M))) cache misses, the optimal count for the classical product.
- File
cache_oblivious_matrix_multiplication.py, 120 lines- Level
- I0 not checked yet
- Toolchain
- Python 3.14, pytest 9
- Test framework
- pytest
- Run the tests
python3 -m pytest
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/cache-oblivious-matrix-multiplication/. The authors are listed in prov/M2GPXZ.jsonl.
from collections import OrderedDict
from typing import Callable, Optional
Matrix = list[list[int]]
def cache_oblivious_matrix_multiplication(a: Matrix, b: Matrix, p: Optional[int] = None,
mod: Optional[int] = None, base: int = 1) -> Matrix:
"""Return a * b by halving the largest of the three dimensions recursively.
a is m x n and b is n x p (lists of rows); pass p when n = 0. The ring is
Z, or Z/mod Z when mod is set; commutativity is not used. Subproblems with
all three sides at most `base` use the triple loop. Cost: m n p
multiplications; recursion depth at most lg m + lg n + lg p + 3.
"""
if base < 1:
raise ValueError("the base size must be at least 1")
m, n = len(a), len(b)
if p is None:
p = len(b[0]) if n > 0 else 0
if any(len(row) != n for row in a) or any(len(row) != p for row in b):
raise ValueError("the shapes do not match")
c = [[0] * p for _ in range(m)]
if m and n and p:
_rec(a, b, c, 0, m, 0, n, 0, p, base)
if mod is not None:
c = [[x % mod for x in row] for row in c]
return c
def _rec(a: Matrix, b: Matrix, c: Matrix, i0: int, i1: int, k0: int, k1: int,
j0: int, j1: int, base: int) -> None:
"""C[i0:i1, j0:j1] += A[i0:i1, k0:k1] * B[k0:k1, j0:j1]."""
m, n, p = i1 - i0, k1 - k0, j1 - j0
if max(m, n, p) <= base: # @line L1
for i in range(i0, i1):
ai, ci = a[i], c[i]
for k in range(k0, k1):
aik, bk = ai[k], b[k]
for j in range(j0, j1):
ci[j] += aik * bk[j]
return
if m >= n and m >= p: # @line L2
h = i0 + m // 2
_rec(a, b, c, i0, h, k0, k1, j0, j1, base)
_rec(a, b, c, h, i1, k0, k1, j0, j1, base)
elif n >= p: # @line L3
h = k0 + n // 2
_rec(a, b, c, i0, i1, k0, h, j0, j1, base)
_rec(a, b, c, i0, i1, h, k1, j0, j1, base)
else: # @line L4
h = j0 + p // 2
_rec(a, b, c, i0, i1, k0, k1, j0, h, base)
_rec(a, b, c, i0, i1, k0, k1, h, j1, base)
def recursion_depth(m: int, n: int, p: int, base: int = 1) -> int:
"""Depth of the recursion tree of the method on an m x n by n x p product."""
if max(m, n, p) <= base:
return 0
if m >= n and m >= p:
return 1 + recursion_depth(m - m // 2, n, p, base)
if n >= p:
return 1 + recursion_depth(m, n - n // 2, p, base)
return 1 + recursion_depth(m, n, p - p // 2, base)
def _trace(n: int, base: int, emit: Callable[[int], None]) -> None:
"""Word addresses read by the recursive method on n x n row-major matrices A, B, C."""
off_b, off_c = n * n, 2 * n * n
def rec(i0: int, i1: int, k0: int, k1: int, j0: int, j1: int) -> None:
m, q, p = i1 - i0, k1 - k0, j1 - j0
if max(m, q, p) <= base:
for i in range(i0, i1):
for k in range(k0, k1):
emit(i * n + k)
for j in range(j0, j1):
emit(off_b + k * n + j)
emit(off_c + i * n + j)
return
if m >= q and m >= p:
h = i0 + m // 2
rec(i0, h, k0, k1, j0, j1)
rec(h, i1, k0, k1, j0, j1)
elif q >= p:
h = k0 + q // 2
rec(i0, i1, k0, h, j0, j1)
rec(i0, i1, h, k1, j0, j1)
else:
h = j0 + p // 2
rec(i0, i1, k0, k1, j0, h)
rec(i0, i1, k0, k1, h, j1)
rec(0, n, 0, n, 0, n)
def lru_misses(n: int, line: int, cache_words: int, base: int = 1) -> int:
"""Misses of a fully associative LRU cache with cache_words words and lines of
`line` words, for the recursive method on n x n row-major matrices."""
cap = cache_words // line
cache: OrderedDict[int, None] = OrderedDict()
misses = 0
def emit(addr: int) -> None:
nonlocal misses
blk = addr // line
if blk in cache:
cache.move_to_end(blk)
else:
misses += 1
cache[blk] = None
if len(cache) > cap:
cache.popitem(last=False)
_trace(n, base, emit)
return misses
test_cache_oblivious_matrix_multiplication.py 83 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/cache-oblivious-matrix-multiplication/. The authors are listed in prov/M2GPXZ.jsonl.
import math
import random
import pytest
from cache_oblivious_matrix_multiplication import cache_oblivious_matrix_multiplication as mul
from cache_oblivious_matrix_multiplication import lru_misses, recursion_depth
def schoolbook(a, b, p, mod=None):
n, m = len(a), len(b)
c = [[0] * p for _ in range(n)]
for i in range(n):
for j in range(p):
s = 0
for k in range(m):
s += a[i][k] * b[k][j]
c[i][j] = s % mod if mod is not None else s
return c
def rand(rng, r, c, lo=-9, hi=9):
return [[rng.randint(lo, hi) for _ in range(c)] for _ in range(r)]
def test_edges() -> None:
assert mul([], []) == []
assert mul([[5]], [[-3]]) == [[-15]]
assert mul([[], []], [], p=2) == [[0, 0], [0, 0]]
assert mul([[1, 2, 3]], [[1], [1], [1]]) == [[6]]
with pytest.raises(ValueError):
mul([[1, 2]], [[1, 2]])
with pytest.raises(ValueError):
mul([[1]], [[1]], base=0)
def test_identity_and_zero() -> None:
rng = random.Random(21)
for n in (1, 3, 6, 13):
a = rand(rng, n, n)
eye = [[int(i == j) for j in range(n)] for i in range(n)]
zero = [[0] * n for _ in range(n)]
assert mul(a, eye) == a
assert mul(eye, a, base=4) == a
assert mul(zero, a, base=2) == zero
def test_random_against_schoolbook() -> None:
rng = random.Random(22)
for _ in range(600):
n, m, p = rng.randint(1, 9), rng.randint(1, 9), rng.randint(1, 9)
a, b = rand(rng, n, m), rand(rng, m, p)
assert mul(a, b, base=rng.randint(1, 5)) == schoolbook(a, b, p)
def test_mod_small_primes() -> None:
rng = random.Random(23)
for q in (2, 3, 5, 7):
for _ in range(40):
n, m, p = rng.randint(1, 12), rng.randint(1, 12), rng.randint(1, 12)
a, b = rand(rng, n, m, 0, q - 1), rand(rng, m, p, 0, q - 1)
assert mul(a, b, mod=q) == schoolbook(a, b, p, q)
def test_depth_is_logarithmic() -> None:
assert recursion_depth(64, 64, 64) == 18
for m, n, p in ((100, 7, 33), (1, 1, 1000), (999, 999, 999)):
bound = sum(math.ceil(math.log2(x)) for x in (m, n, p))
assert recursion_depth(m, n, p) <= bound
a = [[1] * 3 for _ in range(1)]
b = [[1] * 3000 for _ in range(3)]
assert mul(a, b) == [[3] * 3000]
def test_lru_observed_counts() -> None:
# n = 32, lines of 4 words; the recursion is never tuned to M
counts = {cache: lru_misses(32, 4, cache) for cache in (48, 192, 384)}
assert counts == {48: 5888, 192: 3040, 384: 2112}
# i, k, j order makes 17408 misses at 48 words and 8704 at 192 and 384
assert counts[48] < 17408 and counts[192] < 8704
What the next level needs
- the tests did not run in this build
Read the full page about Cache-oblivious matrix multiplication