Grid file in Python
Disk-oriented adaptive grid whose directory maps cells to buckets with at most two disk accesses per lookup.
- File
grid_file.py, 189 lines- Level
- I0 not checked yet
- Toolchain
- Python 3.14, pytest 9
- Test framework
- pytest
- Run the tests
python3 -m pytest
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/grid-file/. The authors are listed in prov/0WX5ZW.jsonl.
from bisect import bisect_right, insort
Point = tuple[int, int]
class Bucket:
"""A bucket of at most B points. Its region is the box of grid cells
with columns c0 to c1 - 1 and rows r0 to r1 - 1."""
__slots__ = ("points", "c0", "c1", "r0", "r1")
def __init__(self, c0: int, c1: int, r0: int, r1: int) -> None:
self.points: list[Point] = []
self.c0, self.c1, self.r0, self.r1 = c0, c1, r0, r1
class GridFile:
"""A two-dimensional grid file with bucket capacity B.
The scales xs and ys are sorted lists of split values: column c holds the
x with c = bisect_right(xs, x), and rows are the same with ys. The directory
grid[c][r] names the bucket of each cell. Invariant: the regions of the
buckets are boxes of cells that partition the grid, the directory agrees
with them, each point lies in a cell of its bucket's region, and each
bucket holds at most B points.
"""
def __init__(self, capacity: int = 2) -> None:
if capacity < 1:
raise ValueError("need capacity >= 1")
self.capacity = capacity
self.xs: list[int] = []
self.ys: list[int] = []
first = Bucket(0, 1, 0, 1)
self.buckets: list[Bucket] = [first]
self.grid: list[list[Bucket]] = [[first]]
self.size = 0
def __len__(self) -> int:
return self.size
def cell(self, p: Point) -> tuple[int, int]:
return bisect_right(self.xs, p[0]), bisect_right(self.ys, p[1])
def bucket_of(self, p: Point) -> Bucket:
"""The bucket whose region holds p: two binary searches and one directory read."""
c = bisect_right(self.xs, p[0]) # @line L1
r = bisect_right(self.ys, p[1]) # @line L2
return self.grid[c][r] # @line L3
def __contains__(self, p: Point) -> bool:
p = (int(p[0]), int(p[1]))
return p in self.bucket_of(p).points
def insert(self, p: Point) -> bool:
"""Add p. Returns False if p was stored."""
p = (int(p[0]), int(p[1]))
b = self.bucket_of(p) # @line L4
if p in b.points:
return False
b.points.append(p) # @line L5
self.size += 1
while len(b.points) > self.capacity: # @line L6
if b.c1 - b.c0 > 1 or b.r1 - b.r0 > 1: # @line L7
axis = 0 if b.c1 - b.c0 >= b.r1 - b.r0 else 1
at = (b.c0 + b.c1) // 2 if axis == 0 else (b.r0 + b.r1) // 2 # @line L8
else:
axis, at = self._refine(b) # @line L9
b = self._split(b, axis, at) # @line L10
return True # @line L11
def _refine(self, b: Bucket) -> tuple[int, int]:
"""Add a split value inside the single cell of b, between its points; return the new line.
The axis is the one with fewer split values so far, if the points of b
differ on it, with x first on a tie; the value is the median of the
distinct coordinates, so both sides get a point.
"""
dx = sorted({q[0] for q in b.points})
dy = sorted({q[1] for q in b.points})
if len(dy) < 2 or (len(dx) >= 2 and len(self.xs) <= len(self.ys)):
v = dx[len(dx) // 2]
insort(self.xs, v)
c = b.c0
self.grid.insert(c + 1, list(self.grid[c]))
for u in self.buckets:
if u.c0 > c:
u.c0 += 1
if u.c1 > c:
u.c1 += 1
return 0, c + 1
v = dy[len(dy) // 2]
insort(self.ys, v)
r = b.r0
for col in self.grid:
col.insert(r + 1, col[r])
for u in self.buckets:
if u.r0 > r:
u.r0 += 1
if u.r1 > r:
u.r1 += 1
return 1, r + 1
def _split(self, b: Bucket, axis: int, at: int) -> Bucket:
"""Split b at the grid line at, on the axis; return the half that holds the most points."""
if axis == 0:
nb = Bucket(at, b.c1, b.r0, b.r1)
b.c1 = at
cut = self.xs[at - 1]
keep = [q for q in b.points if q[0] < cut]
nb.points = [q for q in b.points if q[0] >= cut]
else:
nb = Bucket(b.c0, b.c1, at, b.r1)
b.r1 = at
cut = self.ys[at - 1]
keep = [q for q in b.points if q[1] < cut]
nb.points = [q for q in b.points if q[1] >= cut]
b.points = keep
self.buckets.append(nb)
for c in range(nb.c0, nb.c1):
for r in range(nb.r0, nb.r1):
self.grid[c][r] = nb
return b if len(b.points) >= len(nb.points) else nb
def delete(self, p: Point) -> bool:
"""Remove p, then merge its bucket with a neighbor if their union is a box that fits."""
p = (int(p[0]), int(p[1]))
b = self.bucket_of(p) # @line L12
if p not in b.points:
return False
b.points.remove(p) # @line L13
self.size -= 1
for u in self.buckets: # @line L14
if u is b or len(u.points) + len(b.points) > self.capacity:
continue
same_rows = (u.r0, u.r1) == (b.r0, b.r1) and (u.c1 == b.c0 or u.c0 == b.c1)
same_cols = (u.c0, u.c1) == (b.c0, b.c1) and (u.r1 == b.r0 or u.r0 == b.r1)
if same_rows or same_cols:
b.points.extend(u.points)
b.c0, b.c1 = min(b.c0, u.c0), max(b.c1, u.c1)
b.r0, b.r1 = min(b.r0, u.r0), max(b.r1, u.r1)
for c in range(u.c0, u.c1):
for r in range(u.r0, u.r1):
self.grid[c][r] = b
self.buckets.remove(u)
break
return True
def range_search(self, lo: Point, hi: Point) -> list[Point]:
"""The points in the closed box [lo, hi], reading each bucket of the cells it meets once."""
c0, r0 = self.cell(lo) # @line L15
c1, r1 = self.cell(hi)
seen: set[int] = set()
out: list[Point] = []
for c in range(c0, c1 + 1): # @line L16
for r in range(r0, r1 + 1):
b = self.grid[c][r]
if id(b) in seen:
continue
seen.add(id(b))
out.extend(q for q in b.points if lo[0] <= q[0] <= hi[0] and lo[1] <= q[1] <= hi[1]) # @line L17
return out
def points(self) -> list[Point]:
return [q for b in self.buckets for q in b.points]
def check_invariant(self) -> None:
nc, nr = len(self.xs) + 1, len(self.ys) + 1
assert self.xs == sorted(set(self.xs)) and self.ys == sorted(set(self.ys)), "scales not sorted"
assert len(self.grid) == nc and all(len(col) == nr for col in self.grid), "directory has the wrong size"
owner: dict[tuple[int, int], Bucket] = {}
total = 0
for b in self.buckets:
assert 0 <= b.c0 < b.c1 <= nc and 0 <= b.r0 < b.r1 <= nr, "bad region"
assert len(b.points) <= self.capacity, "bucket overflow"
total += len(b.points)
for c in range(b.c0, b.c1):
for r in range(b.r0, b.r1):
assert (c, r) not in owner, "regions overlap"
owner[(c, r)] = b
assert self.grid[c][r] is b, "directory disagrees"
for q in b.points:
c, r = self.cell(q)
assert b.c0 <= c < b.c1 and b.r0 <= r < b.r1, "point outside its region"
assert len(owner) == nc * nr, "regions do not cover the grid"
assert total == self.size and len(set(self.points())) == total, "size is wrong"
test_grid_file.py 81 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/grid-file/. The authors are listed in prov/0WX5ZW.jsonl.
import random
import pytest
from grid_file import GridFile
def regions(g: GridFile) -> list:
return sorted(((b.c0, b.c1, b.r0, b.r1), sorted(b.points)) for b in g.buckets)
def test_page_example() -> None:
g = GridFile(2)
for p in [(10, 10), (60, 20), (30, 70), (80, 80), (40, 40), (70, 60), (20, 30)]:
assert g.insert(p)
g.check_invariant()
assert not g.insert((40, 40))
assert g.xs == [30, 60] and g.ys == [70]
assert regions(g) == [
((0, 1, 0, 2), [(10, 10), (20, 30)]),
((1, 2, 0, 1), [(40, 40)]),
((1, 3, 1, 2), [(30, 70), (80, 80)]),
((2, 3, 0, 1), [(60, 20), (70, 60)]),
]
assert g.bucket_of((90, 90)) is g.bucket_of((40, 75))
assert sorted(g.range_search((25, 25), (65, 65))) == [(40, 40)]
assert g.delete((80, 80)) and len(g.buckets) == 4
assert g.delete((70, 60)) and not g.delete((70, 60))
g.check_invariant()
assert regions(g) == [
((0, 1, 0, 2), [(10, 10), (20, 30)]),
((1, 3, 0, 1), [(40, 40), (60, 20)]),
((1, 3, 1, 2), [(30, 70)]),
]
def test_edges() -> None:
g = GridFile(1)
assert g.range_search((0, 0), (5, 5)) == []
assert not g.delete((1, 1))
for i in range(50):
assert g.insert((i, 0))
g.check_invariant()
assert len(g.buckets) == 50 and g.ys == []
assert sorted(g.range_search((10, -1), (12, 1))) == [(10, 0), (11, 0), (12, 0)]
h = GridFile(3)
for i in range(20):
h.insert((7, i))
h.check_invariant()
assert h.xs == [] and len(h.ys) > 0
with pytest.raises(ValueError):
GridFile(0)
def test_random_against_oracle() -> None:
rng = random.Random(20261104)
for case in range(600):
cap = rng.randint(1, 5)
g = GridFile(cap)
live: set = set()
span = rng.choice([4, 10, 100, 1000])
for _ in range(rng.randint(1, 60)):
p = (rng.randrange(span), rng.randrange(span))
if live and rng.random() < 0.35:
d = rng.choice(sorted(live))
assert d in g and g.delete(d)
live.discard(d)
assert g.insert(p) == (p not in live)
live.add(p)
g.check_invariant()
assert sorted(g.points()) == sorted(live)
for _ in range(3):
x0, x1 = sorted(rng.randrange(-1, span + 1) for _ in range(2))
y0, y1 = sorted(rng.randrange(-1, span + 1) for _ in range(2))
want = sorted(q for q in live if x0 <= q[0] <= x1 and y0 <= q[1] <= y1)
assert sorted(g.range_search((x0, y0), (x1, y1))) == want
q = (rng.randrange(span), rng.randrange(span))
assert (q in g) == (q in live)
What the next level needs
- the tests did not run in this build