RRR bit vector in Python
An entropy-compressed bit vector with rank and select, using nH0 + o(n) bits.
- File
rrr_bitvector.py, 178 lines- Level
- I0 not checked yet
- Toolchain
- Python 3.14, pytest 9
- Test framework
- pytest
- Run the tests
python3 -m pytest
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/rrr-bitvector/. The authors are listed in prov/5SBQH9.jsonl.
from collections.abc import Sequence
W = 64
class RRRBitVector:
"""A static bit vector stored as blocks of b bits, each one a class (its
number of 1s) and an offset (its rank among the blocks of that class).
access and rank decode one block; select does two binary searches and one
decode. Decoding here is a loop of b steps; with b = lg(n)/2 it is a table
read, and then access and rank take O(1) time.
"""
def __init__(self, bits: Sequence[int], b: int = 15, k: int = 8) -> None:
if b < 1 or k < 1:
raise ValueError("b and k must be positive")
self.n = len(bits)
self.b, self.k = b, k
self.C = [[0] * (b + 2) for _ in range(b + 1)]
for j in range(b + 1):
self.C[j][0] = 1
for c in range(1, j + 1):
self.C[j][c] = self.C[j - 1][c - 1] + self.C[j - 1][c]
self.L = [(self.C[b][c] - 1).bit_length() for c in range(b + 1)]
nblocks = (self.n + b - 1) // b
self.nblocks = nblocks
last = self.n - (nblocks - 1) * b # the true length of the last block
self.L_last = [(self.C[last][c] - 1).bit_length() if c <= last else 0
for c in range(b + 1)] if nblocks else []
self.K: list[int] = []
self.R: list[int] = []
self.P: list[int] = []
self.RB: list[int] = []
self.PB: list[int] = []
self.words: list[int] = []
ones = pos = 0
for t in range(nblocks + 1): # block nblocks is a sentinel for the counts
if t % k == 0:
self.R.append(ones)
self.P.append(pos)
self.RB.append(ones - self.R[-1])
self.PB.append(pos - self.P[-1])
if t == nblocks:
break
x = 0
for j in range(b):
i = t * b + j
if i < self.n and bits[i]:
x |= 1 << j
c, o = self.encode(x)
self.K.append(c)
self._append(pos, o, self._len(t, c))
ones += c
pos += self._len(t, c)
self.m = ones
def _len(self, t: int, c: int) -> int:
"""Return the offset length of block t of class c; the last block may be short."""
return self.L_last[c] if t == self.nblocks - 1 else self.L[c]
def _append(self, p: int, value: int, length: int) -> None:
while len(self.words) * W < p + length:
self.words.append(0)
if length:
self.words[p // W] |= (value << (p % W)) & ((1 << W) - 1)
if p % W + length > W:
self.words[p // W + 1] |= value >> (W - p % W)
def _read(self, p: int, length: int) -> int:
if length == 0:
return 0
x = self.words[p // W] >> (p % W)
if p % W + length > W:
x |= self.words[p // W + 1] << (W - p % W)
return x & ((1 << length) - 1)
def encode(self, x: int) -> tuple[int, int]:
"""Return (class, offset) of a block x of b bits."""
c = o = 0 # @line L1
for j in range(self.b): # @line L2
if (x >> j) & 1: # @line L3
c += 1
o += self.C[j][c]
return c, o # @line L4
def decode(self, c: int, o: int) -> int:
"""Return the block of class c and offset o."""
x = 0 # @line L5
for j in range(self.b - 1, -1, -1): # @line L6
if c > 0 and o >= self.C[j][c]: # @line L7
x |= 1 << j # @line L8
o -= self.C[j][c]
c -= 1
return x # @line L9
def _block(self, t: int) -> int:
q = t // self.k
c = self.K[t]
return self.decode(c, self._read(self.P[q] + self.PB[t], self._len(t, c)))
def __len__(self) -> int:
return self.n
def access(self, i: int) -> int:
"""Return B[i] for 0 <= i < n."""
if not 0 <= i < self.n:
raise IndexError(i)
t = i // self.b # @line L10
return (self._block(t) >> (i % self.b)) & 1 # @line L11
def rank1(self, i: int) -> int:
"""Return the number of 1s in B[0..i-1], for 0 <= i <= n."""
if not 0 <= i <= self.n:
raise IndexError(i)
t = i // self.b # @line L12
q = t // self.k
base = self.R[q] + self.RB[t]
if i % self.b == 0: # @line L13
return base
x = self._block(t) # @line L14
return base + (x & ((1 << (i % self.b)) - 1)).bit_count() # @line L15
def rank0(self, i: int) -> int:
"""Return the number of 0s in B[0..i-1], for 0 <= i <= n."""
return i - self.rank1(i)
def _select(self, j: int, bit: int) -> int:
b, k = self.b, self.k
def before_super(q: int) -> int:
return self.R[q] if bit else q * k * b - self.R[q]
def before_block(t: int) -> int:
q = t // k
r = self.R[q] + self.RB[t]
return r if bit else t * b - r
lo, hi = 0, (self.nblocks - 1) // k # @line L16
while lo < hi:
mid = (lo + hi + 1) // 2
if before_super(mid) < j:
lo = mid
else:
hi = mid - 1
lo, hi = lo * k, min(self.nblocks - 1, lo * k + k - 1) # @line L17
while lo < hi:
mid = (lo + hi + 1) // 2
if before_block(mid) < j:
lo = mid
else:
hi = mid - 1
t = lo
x = self._block(t) # @line L18
if not bit:
x = ~x & ((1 << b) - 1)
for _ in range(j - before_block(t) - 1): # @line L19
x &= x - 1
return t * b + (x & -x).bit_length() - 1
def select1(self, j: int) -> int:
"""Return the position of the j-th 1, for 1 <= j <= m."""
if not 1 <= j <= self.m:
raise IndexError(j)
return self._select(j, 1)
def select0(self, j: int) -> int:
"""Return the position of the j-th 0, for 1 <= j <= n - m."""
if not 1 <= j <= self.n - self.m:
raise IndexError(j)
return self._select(j, 0)
def offset_bits(self) -> int:
"""Return the length of the offset stream in bits."""
return self.P[-1] + self.PB[-1]
test_rrr_bitvector.py 83 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/rrr-bitvector/. The authors are listed in prov/5SBQH9.jsonl.
import math
import random
import pytest
from rrr_bitvector import RRRBitVector
EXAMPLE = [int(c) for c in "1011001011000111"]
def check_all(bits: list[int], b: int, k: int) -> None:
v = RRRBitVector(bits, b, k)
n = len(bits)
r = 0
for i in range(n + 1):
assert v.rank1(i) == r
assert v.rank0(i) == i - r
if i < n:
assert v.access(i) == bits[i]
r += bits[i]
ones = [i for i, x in enumerate(bits) if x]
zeros = [i for i, x in enumerate(bits) if not x]
for j, p in enumerate(ones, 1):
assert v.select1(j) == p
for j, p in enumerate(zeros, 1):
assert v.select0(j) == p
m = len(ones)
bound = math.log2(math.comb(n, m)) if n else 0
assert v.offset_bits() <= bound + (n + b - 1) // b + 1
def test_page_example() -> None:
v = RRRBitVector(EXAMPLE, b=4, k=2)
assert v.K == [3, 1, 2, 3]
assert v.L == [0, 2, 3, 2, 0]
assert [v.encode(v._block(t))[1] for t in range(4)] == [2, 2, 0, 3]
assert v.R == [0, 4, 9] and v.P == [0, 4, 9]
assert v.RB[:4] == [0, 3, 0, 2] and v.PB[:4] == [0, 2, 0, 3]
assert v.decode(3, 3) == 0b1110
assert v.decode(2, 0) == 0b0011
assert v.rank1(13) == 6
assert v.rank1(10) == 6
assert v.select1(5) == 8
assert v.offset_bits() == 9
check_all(EXAMPLE, 4, 2)
def test_encode_decode_bijection() -> None:
v = RRRBitVector([], b=8, k=1)
seen: dict[int, set[int]] = {}
for x in range(256):
c, o = v.encode(x)
assert c == x.bit_count()
assert 0 <= o < math.comb(8, c)
assert o not in seen.setdefault(c, set())
seen[c].add(o)
assert v.decode(c, o) == x
def test_edges() -> None:
check_all([], 4, 2)
check_all([0], 1, 1)
check_all([1], 1, 1)
check_all([0] * 70, 15, 3)
check_all([1] * 70, 15, 3)
check_all([1] * 64 + [0] * 64, 63, 2)
v = RRRBitVector([1, 0], 3, 1)
with pytest.raises(IndexError):
v.select1(2)
with pytest.raises(IndexError):
v.rank1(3)
def test_random_against_lists() -> None:
rng = random.Random(2007)
for _ in range(500):
n = rng.randrange(0, 300)
p = rng.choice([0.02, 0.1, 0.5, 0.9, 0.98])
bits = [1 if rng.random() < p else 0 for _ in range(n)]
check_all(bits, rng.randrange(1, 64), rng.randrange(1, 6))
What the next level needs
- the tests did not run in this build