External merge sort in Python
Form sorted runs that fit in memory, then repeatedly merge them until one sorted file remains.
- File
external_merge_sort.py, 84 lines- Level
- I0 not checked yet
- Toolchain
- Python 3.14, pytest 9
- Test framework
- pytest
- Run the tests
python3 -m pytest
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/external-merge-sort/. The authors are listed in prov/V917JH.jsonl.
import heapq
from typing import Protocol, TypeVar
class Ordered(Protocol):
def __lt__(self, other: "Ordered", /) -> bool: ...
T = TypeVar("T", bound=Ordered)
def blocks(length: int, b: int) -> int:
"""The number of blocks of b items that hold a run of the given length."""
return -(-length // b)
class _Head:
"""The head of run i at position j. Equal keys order by the run number."""
__slots__ = ("x", "i", "j")
def __init__(self, x: T, i: int, j: int) -> None:
self.x, self.i, self.j = x, i, j
def __lt__(self, other: "_Head") -> bool:
if self.x < other.x:
return True
if other.x < self.x:
return False
return self.i < other.i
def k_way_merge(runs: list[list[T]]) -> list[T]:
"""Merge sorted runs into one sorted run. Equal items keep the order of their runs."""
heap = [_Head(run[0], i, 0) for i, run in enumerate(runs) if run]
heapq.heapify(heap)
out: list[T] = []
while heap:
h = heap[0]
out.append(h.x)
if h.j + 1 < len(runs[h.i]):
heapq.heapreplace(heap, _Head(runs[h.i][h.j + 1], h.i, h.j + 1))
else:
heapq.heappop(heap)
return out
def external_merge_sort(f: list[T], m: int = 16, b: int = 4) -> int:
"""Sort the list f in place as an external merge sort with a memory of m items and blocks of b items.
The disk is simulated: a run is a list, and each read or write of a block
of a run counts as one I/O. Precondition: b >= 1 and m >= 3 * b, so that
the fan-in k = m // b - 1 is at least 2. Returns the number of block I/Os. When b divides m, this number is at most
2 * ceil(n / b) * (1 + passes) with passes = ceil(log_k ceil(n / m)).
The in-memory time is O(n log n). The sort is stable.
"""
if b < 1 or m < 3 * b:
raise ValueError("need b >= 1 and m >= 3 * b")
k = m // b - 1 # @line L1
ios = 0
runs: list[list[T]] = [] # @line L2
for start in range(0, len(f), m): # @line L3
load = f[start:start + m] # @line L4
ios += blocks(len(load), b)
load.sort() # @line L5: a stable sort in memory
ios += blocks(len(load), b)
runs.append(load)
while len(runs) > 1: # @line L6
merged: list[list[T]] = [] # @line L7
for g in range(0, len(runs), k): # @line L8
group = runs[g:g + k]
if len(group) == 1: # @line L9: a single run is kept, with no I/O
merged.append(group[0])
continue
ios += sum(blocks(len(r), b) for r in group) # @line L10
out = k_way_merge(group)
ios += blocks(len(out), b)
merged.append(out)
runs = merged # @line L11
f[:] = runs[0] if runs else [] # @line L12
return ios
test_external_merge_sort.py 71 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/external-merge-sort/. The authors are listed in prov/V917JH.jsonl.
import random
import pytest
from external_merge_sort import external_merge_sort
def passes(n: int, m: int, b: int) -> int:
k = m // b - 1
r = -(-n // m)
p = 0
while r > 1:
r = -(-r // k)
p += 1
return p
def test_example() -> None:
f = [41, 7, 63, 18, 90, 2, 55, 34, 12, 77, 29, 5, 68, 23, 81, 46, 9, 60]
want = sorted(f)
ios = external_merge_sort(f, 6, 2)
assert f == want
assert ios == 18 + 12 + 18
def test_edges() -> None:
a: list[int] = []
assert external_merge_sort(a, 6, 2) == 0 and a == []
a = [5]
assert external_merge_sort(a, 6, 2) == 2 and a == [5]
a = [3] * 7
external_merge_sort(a, 6, 2)
assert a == [3] * 7
a = list(range(20, 0, -1))
external_merge_sort(a, 6, 2)
assert a == list(range(1, 21))
a = list(range(50))
assert external_merge_sort(a) > 0 and a == list(range(50))
with pytest.raises(ValueError):
external_merge_sort([1, 2], 4, 2)
def test_stable() -> None:
class Key:
def __init__(self, k: int, tag: int) -> None:
self.k, self.tag = k, tag
def __lt__(self, other: "Key") -> bool:
return self.k < other.k
rng = random.Random(7)
a = [Key(rng.randint(0, 3), i) for i in range(40)]
want = sorted((x.k, x.tag) for x in a)
external_merge_sort(a, 6, 2)
assert [(x.k, x.tag) for x in a] == want
def test_random_against_sorted() -> None:
rng = random.Random(4417)
for _ in range(600):
n = rng.randint(0, 120)
b = rng.randint(1, 4)
m = b * rng.randint(3, 6)
f = [rng.randint(-30, 30) for _ in range(n)]
want = sorted(f)
ios = external_merge_sort(f, m, b)
assert f == want
assert ios <= 2 * (-(-n // b)) * (1 + passes(n, m, b))
What the next level needs
- the tests did not run in this build