Skip to content

Commit

Permalink
Added thread speed tests
Browse files Browse the repository at this point in the history
  • Loading branch information
gentlegiantJGC committed Nov 3, 2024
1 parent 3d5410b commit dc9cc19
Showing 1 changed file with 111 additions and 0 deletions.
111 changes: 111 additions & 0 deletions tests/test_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import glob
import os
import weakref
import time
from concurrent.futures import ThreadPoolExecutor

num_keys = [struct.pack("<Q", i) for i in range(10_000)]
num_db = dict(zip(num_keys, num_keys))
Expand Down Expand Up @@ -412,6 +414,115 @@ def test_iterator(self) -> None:

db.close()

def test_thread_write(self) -> None:
count = 10_000
data = [
[
struct.pack(">i", i + j)
for j in range(count)
]
for i in range(0, count * 10, count)
]
m = {k: k for k in sum(data, [])}

with TemporaryDirectory() as path:
t1 = -time.time()
db = LevelDB(path, True)
for l in data:
for v in l:
db.put(v, v)
db.close()
t1 += time.time()

db = LevelDB(path)
m1 = dict(db)
db.close()
self.assertEqual(m, m1)

with TemporaryDirectory() as path:
t2 = -time.time()
db = LevelDB(path, True)

def add(values) -> None:
for v in values:
db.put(v, v)

with ThreadPoolExecutor() as executor:
executor.map(add, data)

db.close()
t2 += time.time()

db = LevelDB(path)
m2 = dict(db)
db.close()
self.assertEqual(m, m2)

self.assertLess(t2, t1)

def test_thread_read(self) -> None:
count = 10_000
data = [
[
struct.pack(">i", i + j)
for j in range(count)
]
for i in range(0, count * 10, count)
]
m = {k: k for k in sum(data, [])}

with TemporaryDirectory() as path:
db = LevelDB(path, True)

def add(values) -> None:
for v in values:
db.put(v, v)

with ThreadPoolExecutor() as executor:
executor.map(add, data)

db.close()

# Validate the database
db = LevelDB(path)
m2 = dict(db)
db.close()
self.assertEqual(m, m2)

# # Read serial
db = LevelDB(path)
m1 = {}
t1 = -time.time()
it = db.create_iterator()
it.seek_to_first()
while it.valid():
m1[it.key()] = it.value()
it.next()
t1 += time.time()
db.close()
self.assertEqual(m, m1)

# Read parallel
db = LevelDB(path)
m2 = {}
t2 = -time.time()

def read(values) -> None:
it = db.create_iterator()
it.seek(values[0])
for _ in range(count):
m2[it.key()] = it.value()
it.next()

with ThreadPoolExecutor() as executor:
executor.map(read, data)

t2 += time.time()
db.close()
self.assertEqual(m, m2)

self.assertLess(t2, t1)
print(t1, t2)

if __name__ == "__main__":
unittest.main()

0 comments on commit dc9cc19

Please sign in to comment.