Algorithm Search⌘K

Cache-oblivious matrix multiplication in Python

Halves the largest of the three dimensions until the pieces are tiny; without knowing M or B it makes O(n^2/B + n^3/(B sqrt(M))) cache misses, the optimal count for the classical product.

File
cache_oblivious_matrix_multiplication.py, 120 lines
Level
I0 not checked yet
Toolchain
Python 3.14, pytest 9
Test framework
pytest
Run the tests
python3 -m pytest
cache_oblivious_matrix_multiplication.pyRawGitHub
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/cache-oblivious-matrix-multiplication/. The authors are listed in prov/M2GPXZ.jsonl.

from collections import OrderedDict
from typing import Callable, Optional

Matrix = list[list[int]]


def cache_oblivious_matrix_multiplication(a: Matrix, b: Matrix, p: Optional[int] = None,
                                          mod: Optional[int] = None, base: int = 1) -> Matrix:
    """Return a * b by halving the largest of the three dimensions recursively.

    a is m x n and b is n x p (lists of rows); pass p when n = 0. The ring is
    Z, or Z/mod Z when mod is set; commutativity is not used. Subproblems with
    all three sides at most `base` use the triple loop. Cost: m n p
    multiplications; recursion depth at most lg m + lg n + lg p + 3.
    """
    if base < 1:
        raise ValueError("the base size must be at least 1")
    m, n = len(a), len(b)
    if p is None:
        p = len(b[0]) if n > 0 else 0
    if any(len(row) != n for row in a) or any(len(row) != p for row in b):
        raise ValueError("the shapes do not match")
    c = [[0] * p for _ in range(m)]
    if m and n and p:
        _rec(a, b, c, 0, m, 0, n, 0, p, base)
    if mod is not None:
        c = [[x % mod for x in row] for row in c]
    return c


def _rec(a: Matrix, b: Matrix, c: Matrix, i0: int, i1: int, k0: int, k1: int,
         j0: int, j1: int, base: int) -> None:
    """C[i0:i1, j0:j1] += A[i0:i1, k0:k1] * B[k0:k1, j0:j1]."""
    m, n, p = i1 - i0, k1 - k0, j1 - j0
    if max(m, n, p) <= base:  # @line L1
        for i in range(i0, i1):
            ai, ci = a[i], c[i]
            for k in range(k0, k1):
                aik, bk = ai[k], b[k]
                for j in range(j0, j1):
                    ci[j] += aik * bk[j]
        return
    if m >= n and m >= p:  # @line L2
        h = i0 + m // 2
        _rec(a, b, c, i0, h, k0, k1, j0, j1, base)
        _rec(a, b, c, h, i1, k0, k1, j0, j1, base)
    elif n >= p:  # @line L3
        h = k0 + n // 2
        _rec(a, b, c, i0, i1, k0, h, j0, j1, base)
        _rec(a, b, c, i0, i1, h, k1, j0, j1, base)
    else:  # @line L4
        h = j0 + p // 2
        _rec(a, b, c, i0, i1, k0, k1, j0, h, base)
        _rec(a, b, c, i0, i1, k0, k1, h, j1, base)


def recursion_depth(m: int, n: int, p: int, base: int = 1) -> int:
    """Depth of the recursion tree of the method on an m x n by n x p product."""
    if max(m, n, p) <= base:
        return 0
    if m >= n and m >= p:
        return 1 + recursion_depth(m - m // 2, n, p, base)
    if n >= p:
        return 1 + recursion_depth(m, n - n // 2, p, base)
    return 1 + recursion_depth(m, n, p - p // 2, base)


def _trace(n: int, base: int, emit: Callable[[int], None]) -> None:
    """Word addresses read by the recursive method on n x n row-major matrices A, B, C."""
    off_b, off_c = n * n, 2 * n * n

    def rec(i0: int, i1: int, k0: int, k1: int, j0: int, j1: int) -> None:
        m, q, p = i1 - i0, k1 - k0, j1 - j0
        if max(m, q, p) <= base:
            for i in range(i0, i1):
                for k in range(k0, k1):
                    emit(i * n + k)
                    for j in range(j0, j1):
                        emit(off_b + k * n + j)
                        emit(off_c + i * n + j)
            return
        if m >= q and m >= p:
            h = i0 + m // 2
            rec(i0, h, k0, k1, j0, j1)
            rec(h, i1, k0, k1, j0, j1)
        elif q >= p:
            h = k0 + q // 2
            rec(i0, i1, k0, h, j0, j1)
            rec(i0, i1, h, k1, j0, j1)
        else:
            h = j0 + p // 2
            rec(i0, i1, k0, k1, j0, h)
            rec(i0, i1, k0, k1, h, j1)

    rec(0, n, 0, n, 0, n)


def lru_misses(n: int, line: int, cache_words: int, base: int = 1) -> int:
    """Misses of a fully associative LRU cache with cache_words words and lines of
    `line` words, for the recursive method on n x n row-major matrices."""
    cap = cache_words // line
    cache: OrderedDict[int, None] = OrderedDict()
    misses = 0

    def emit(addr: int) -> None:
        nonlocal misses
        blk = addr // line
        if blk in cache:
            cache.move_to_end(blk)
        else:
            misses += 1
            cache[blk] = None
            if len(cache) > cap:
                cache.popitem(last=False)

    _trace(n, base, emit)
    return misses
test_cache_oblivious_matrix_multiplication.py 83 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/cache-oblivious-matrix-multiplication/. The authors are listed in prov/M2GPXZ.jsonl.

import math
import random

import pytest

from cache_oblivious_matrix_multiplication import cache_oblivious_matrix_multiplication as mul
from cache_oblivious_matrix_multiplication import lru_misses, recursion_depth


def schoolbook(a, b, p, mod=None):
    n, m = len(a), len(b)
    c = [[0] * p for _ in range(n)]
    for i in range(n):
        for j in range(p):
            s = 0
            for k in range(m):
                s += a[i][k] * b[k][j]
            c[i][j] = s % mod if mod is not None else s
    return c


def rand(rng, r, c, lo=-9, hi=9):
    return [[rng.randint(lo, hi) for _ in range(c)] for _ in range(r)]


def test_edges() -> None:
    assert mul([], []) == []
    assert mul([[5]], [[-3]]) == [[-15]]
    assert mul([[], []], [], p=2) == [[0, 0], [0, 0]]
    assert mul([[1, 2, 3]], [[1], [1], [1]]) == [[6]]
    with pytest.raises(ValueError):
        mul([[1, 2]], [[1, 2]])
    with pytest.raises(ValueError):
        mul([[1]], [[1]], base=0)


def test_identity_and_zero() -> None:
    rng = random.Random(21)
    for n in (1, 3, 6, 13):
        a = rand(rng, n, n)
        eye = [[int(i == j) for j in range(n)] for i in range(n)]
        zero = [[0] * n for _ in range(n)]
        assert mul(a, eye) == a
        assert mul(eye, a, base=4) == a
        assert mul(zero, a, base=2) == zero


def test_random_against_schoolbook() -> None:
    rng = random.Random(22)
    for _ in range(600):
        n, m, p = rng.randint(1, 9), rng.randint(1, 9), rng.randint(1, 9)
        a, b = rand(rng, n, m), rand(rng, m, p)
        assert mul(a, b, base=rng.randint(1, 5)) == schoolbook(a, b, p)


def test_mod_small_primes() -> None:
    rng = random.Random(23)
    for q in (2, 3, 5, 7):
        for _ in range(40):
            n, m, p = rng.randint(1, 12), rng.randint(1, 12), rng.randint(1, 12)
            a, b = rand(rng, n, m, 0, q - 1), rand(rng, m, p, 0, q - 1)
            assert mul(a, b, mod=q) == schoolbook(a, b, p, q)


def test_depth_is_logarithmic() -> None:
    assert recursion_depth(64, 64, 64) == 18
    for m, n, p in ((100, 7, 33), (1, 1, 1000), (999, 999, 999)):
        bound = sum(math.ceil(math.log2(x)) for x in (m, n, p))
        assert recursion_depth(m, n, p) <= bound
    a = [[1] * 3 for _ in range(1)]
    b = [[1] * 3000 for _ in range(3)]
    assert mul(a, b) == [[3] * 3000]


def test_lru_observed_counts() -> None:
    # n = 32, lines of 4 words; the recursion is never tuned to M
    counts = {cache: lru_misses(32, 4, cache) for cache in (48, 192, 384)}
    assert counts == {48: 5888, 192: 3040, 384: 2112}
    # i, k, j order makes 17408 misses at 48 words and 8704 at 192 and 384
    assert counts[48] < 17408 and counts[192] < 8704

What the next level needs

  • the tests did not run in this build

Read the full page about Cache-oblivious matrix multiplication