Baguette
/
dodb.cr
3
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

Specs are fixed & reworked for the most part. New benchmarks.

toying-with-ramdb
Philippe PITTOLI 2024-05-09 18:41:20 +02:00
parent 2a71254533
commit c475c4f584
5 changed files with 99 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
TODO.md
View File

@ -1,8 +1,3 @@
# API
Cached indexes (index, partition, tags) should be used by default.
Uncached indexes should be an option, through a new function `add_uncached_index` or something.
# Performance # Performance
Search with some kind of "pagination" system: ask entries with a limit on the number of elements and an offset. Search with some kind of "pagination" system: ask entries with a limit on the number of elements and an offset.

212
spec/benchmark-cars.cr
View File

@ -1,181 +1,109 @@
require "benchmark" require "benchmark"
require "./benchmark-utilities.cr" require "./benchmark-utilities.cr"
require "./cars.cr"
require "../src/dodb.cr" cars_ram = DODB::RAMOnlySpecDataBase(Car).new
require "./test-data.cr" cars_cached = DODB::CachedSpecDataBase(Car).new
cars_semi = DODB::SpecDataBase(Car).new "-semi"
cars_uncached = DODB::SpecDataBase(Car).new
class DODBCachedCars < DODB::CachedDataBase(Car) ram_searchby_name, ram_searchby_color, ram_searchby_keywords = ram_indexes cars_ram
property storage_dir : String cached_searchby_name, cached_searchby_color, cached_searchby_keywords = cached_indexes cars_cached
def initialize(storage_ext = "", remove_previous_data = true) semi_searchby_name, semi_searchby_color, semi_searchby_keywords = cached_indexes cars_semi
@storage_dir = "test-storage-cars-cached#{storage_ext}" uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = uncached_indexes cars_uncached
if remove_previous_data
::FileUtils.rm_rf storage_dir
end
super storage_dir
end
def rm_storage_dir
::FileUtils.rm_rf @storage_dir
end
end
class DODBUnCachedCars < DODB::DataBase(Car)
property storage_dir : String
def initialize(storage_ext = "", remove_previous_data = true)
@storage_dir = "test-storage-cars-uncached#{storage_ext}"
if remove_previous_data
::FileUtils.rm_rf storage_dir
end
super storage_dir
end
def rm_storage_dir
::FileUtils.rm_rf @storage_dir
end
end
class DODBSemiCachedCars < DODB::DataBase(Car)
property storage_dir : String
def initialize(storage_ext = "", remove_previous_data = true)
@storage_dir = "test-storage-cars-semi#{storage_ext}"
if remove_previous_data
::FileUtils.rm_rf storage_dir
end
super storage_dir
end
def rm_storage_dir
::FileUtils.rm_rf @storage_dir
end
end
def init_indexes(storage : DODB::Storage)
n = storage.new_index "name", &.name
c = storage.new_partition "color", &.color
k = storage.new_tags "keyword", &.keywords
return n, c, k
end
def init_uncached_indexes(storage : DODB::Storage)
n = storage.new_uncached_index "name", &.name
c = storage.new_uncached_partition "color", &.color
k = storage.new_uncached_tags "keyword", &.keywords
return n, c, k
end
def add_cars(storage : DODB::Storage, nb_iterations : Int32)
i = 0
car1 = Car.new "Corvet", "red", [ "shiny", "impressive", "fast", "elegant" ]
car2 = Car.new "Bullet-GT", "blue", [ "shiny", "fast", "expensive" ]
car3 = Car.new "Deudeuche", "beige", [ "curvy", "sublime" ]
car4 = Car.new "Ford-5", "red", [ "unknown" ]
car5 = Car.new "C-MAX", "gray", [ "spacious", "affordable" ]
while i < nb_iterations
car1.name = "Corvet-#{i}"
car2.name = "Bullet-GT-#{i}"
car3.name = "Deudeuche-#{i}"
car4.name = "Ford-5-#{i}"
car5.name = "C-MAX-#{i}"
storage << car1
storage << car2
storage << car3
storage << car4
storage << car5
i += 1
STDOUT.write "\radding value #{i}".to_slice
end
puts ""
end
cars_cached = DODBCachedCars.new
cars_uncached = DODBUnCachedCars.new
cars_semi = DODBSemiCachedCars.new
cached_searchby_name, cached_searchby_color, cached_searchby_keywords = init_indexes cars_cached
uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = init_uncached_indexes cars_uncached
semi_searchby_name, semi_searchby_color, semi_searchby_keywords = init_indexes cars_semi
add_cars cars_ram, 1_000
add_cars cars_cached, 1_000 add_cars cars_cached, 1_000
add_cars cars_uncached, 1_000
add_cars cars_semi, 1_000 add_cars cars_semi, 1_000
add_cars cars_uncached, 1_000
# Searching for data with an index. # Searching for data with an index.
puts "search by index 'Corvet-500': get a single value"
Benchmark.ips do |x| Benchmark.ips do |x|
x.report("(cars db) searching a data with an index (with a cache)") do x.report("(ram db and index) searching a data with an index") do
corvet = ram_searchby_name.get "Corvet-500"
end
x.report("(cached db and index) searching a data with an index") do
corvet = cached_searchby_name.get "Corvet-500" corvet = cached_searchby_name.get "Corvet-500"
end end
x.report("(cars db) searching a data with an index (semi: cache is only on index)") do x.report("(semi: uncached db but cached index) searching a data with an index") do
corvet = semi_searchby_name.get "Corvet-500" corvet = semi_searchby_name.get "Corvet-500"
end end
x.report("(cars db) searching a data with an index (without a cache)") do x.report("(uncached db and index) searching a data with an index") do
corvet = uncached_searchby_name.get "Corvet-500" corvet = uncached_searchby_name.get "Corvet-500"
end end
end end
# Searching for data with a partition. # Searching for data with a partition.
puts ""
puts "search by partition 'red': get #{ram_searchby_color.get("red").size} values"
Benchmark.ips do |x| Benchmark.ips do |x|
x.report("(cars db) searching a data with a partition (with a cache)") do x.report("(ram db and partition) searching a data with a partition") do
red_cars = cached_searchby_color.get "red" corvet = ram_searchby_color.get "red"
end end
x.report("(cars db) searching a data with a partition (semi: cache is only on partition)") do x.report("(cached db and partition) searching a data with a partition") do
red_cars = semi_searchby_color.get "red" corvet = cached_searchby_color.get "red"
end end
x.report("(cars db) searching a data with a partition (without a cache)") do x.report("(semi: uncached db but cached partition) searching a data with a partition") do
red_cars = uncached_searchby_color.get "red" corvet = semi_searchby_color.get "red"
end
x.report("(uncached db and partition) searching a data with a partition") do
corvet = uncached_searchby_color.get "red"
end end
end end
# Searching for data with a tag. # Searching for data with a tag.
puts ""
puts "search by tag 'spacious': get #{ram_searchby_keywords.get("spacious").size} values"
Benchmark.ips do |x| Benchmark.ips do |x|
x.report("(cars db) searching a data with a tag (with a cache)") do x.report("(ram db and tag) searching a data with a tag") do
red_cars = cached_searchby_keywords.get "spacious" corvet = ram_searchby_keywords.get "spacious"
end end
x.report("(cars db) searching a data with a tag (semi: cache is only on tags)") do x.report("(cached db and tag) searching a data with a tag") do
red_cars = semi_searchby_keywords.get "spacious" corvet = cached_searchby_keywords.get "spacious"
end end
x.report("(cars db) searching a data with a tag (without a cache)") do x.report("(semi: uncached db but cached tag) searching a data with a tag") do
red_cars = uncached_searchby_keywords.get "spacious" corvet = semi_searchby_keywords.get "spacious"
end
x.report("(uncached db and tag) searching a data with a tag") do
corvet = uncached_searchby_keywords.get "spacious"
end end
end end
cars_cached.rm_storage_dir #cars_ram.rm_storage_dir
cars_uncached.rm_storage_dir #cars_cached.rm_storage_dir
#cars_semi.rm_storage_dir
#cars_uncached.rm_storage_dir
cars_cached = DODBCachedCars.new #cars_cached = DODB::CachedSpecDataBase(Car).new
cars_uncached = DODBUnCachedCars.new #cars_uncached = DODB::SpecDataBase(Car).new
#
#init_indexes cars_cached ##init_indexes cars_cached
#init_indexes cars_uncached ##init_indexes cars_uncached
cached_searchby_name, cached_searchby_color, cached_searchby_keywords = init_indexes cars_cached #cached_searchby_name, cached_searchby_color, cached_searchby_keywords = cached_indexes cars_cached
uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = init_uncached_indexes cars_uncached #uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = uncached_indexes cars_uncached
#
add_cars cars_cached, 1_000 #add_cars cars_cached, 1_000
add_cars cars_uncached, 1_000 #add_cars cars_uncached, 1_000
#
nb_run = 1000 #nb_run = 1000
#
perform_benchmark_average_verbose "(cached) search db with an index", nb_run, do #perform_benchmark_average_verbose "(cached) search db with an index", nb_run, do
cached_searchby_name.get "Corvet-500" # cached_searchby_name.get "Corvet-500"
end #end
#
perform_benchmark_average_verbose "(uncached) search db with an index", nb_run, do #perform_benchmark_average_verbose "(uncached) search db with an index", nb_run, do
uncached_searchby_name.get "Corvet-500" # uncached_searchby_name.get "Corvet-500"
end #end
#
cars_cached.rm_storage_dir #cars_cached.rm_storage_dir
cars_uncached.rm_storage_dir #cars_uncached.rm_storage_dir
cars_semi.rm_storage_dir

0
spec/benchmark.cr → spec/benchmark-todo.cr
View File

6
spec/spec-database.cr
View File

@ -1,7 +1,7 @@
class DODB::SpecDataBase(V) < DODB::DataBase(V) class DODB::SpecDataBase(V) < DODB::DataBase(V)
property storage_dir : String property storage_dir : String
def initialize(storage_ext = "", remove_previous_data = true) def initialize(storage_ext = "", remove_previous_data = true)
@storage_dir = "test-spec-storage#{storage_ext}" @storage_dir = "specdb-storage-uncached#{storage_ext}"
if remove_previous_data if remove_previous_data
::FileUtils.rm_rf storage_dir ::FileUtils.rm_rf storage_dir
@ -18,7 +18,7 @@ end
class DODB::CachedSpecDataBase(V) < DODB::CachedDataBase(V) class DODB::CachedSpecDataBase(V) < DODB::CachedDataBase(V)
property storage_dir : String property storage_dir : String
def initialize(storage_ext = "", remove_previous_data = true) def initialize(storage_ext = "", remove_previous_data = true)
@storage_dir = "test-spec-storage-cached#{storage_ext}" @storage_dir = "specdb-storage-cached#{storage_ext}"
if remove_previous_data if remove_previous_data
::FileUtils.rm_rf storage_dir ::FileUtils.rm_rf storage_dir
@ -35,7 +35,7 @@ end
class DODB::RAMOnlySpecDataBase(V) < DODB::RAMOnlyDataBase(V) class DODB::RAMOnlySpecDataBase(V) < DODB::RAMOnlyDataBase(V)
property storage_dir : String property storage_dir : String
def initialize(storage_ext = "", remove_previous_data = true) def initialize(storage_ext = "", remove_previous_data = true)
@storage_dir = "test-spec-storage-ram#{storage_ext}" @storage_dir = "specdb-storage-ram#{storage_ext}"
if remove_previous_data if remove_previous_data
::FileUtils.rm_rf storage_dir ::FileUtils.rm_rf storage_dir

17
spec/test-cars.cr
View File

@ -4,8 +4,7 @@ require "./cars.cr"
corvet0 = Car.new "Corvet-0", "red", [ "shiny", "impressive", "fast", "elegant" ] corvet0 = Car.new "Corvet-0", "red", [ "shiny", "impressive", "fast", "elegant" ]
describe "DODB car" do describe "uncached, cached and ram indexes" do
describe "behavior of uncached, cached and ram indexes:" do
it "RAM DB - add items, add indexes, search, reindex, search" do it "RAM DB - add items, add indexes, search, reindex, search" do
cars_ram0 = DODB::RAMOnlySpecDataBase(Car).new "-0" cars_ram0 = DODB::RAMOnlySpecDataBase(Car).new "-0"
@ -28,18 +27,16 @@ describe "DODB car" do
cars_ram1.reindex_everything! cars_ram1.reindex_everything!
cars_ram2.reindex_everything! cars_ram2.reindex_everything!
# Cannot get the value since it's not written on disk. # Get the value even if not written on the disk since the index was written on the disk.
# FIXME: should only retrieve the key with the index, not the actual value. # The value is retrieved by the database, the index only reads its key in the database.
# So, this should work. uncached_searchby_name.get?("Corvet-0").should eq corvet0
uncached_searchby_name.get?("Corvet-0").should be_nil
# Both cached and RAM indexes can retrieve the value since they store the key. # Both cached and RAM indexes can retrieve the value since they store the key.
cached_searchby_name.get?("Corvet-0").should eq corvet0 cached_searchby_name.get?("Corvet-0").should eq corvet0
ram_searchby_name.get?("Corvet-0").should eq corvet0 ram_searchby_name.get?("Corvet-0").should eq corvet0
cars_ram0.rm_storage_dir # cars_ram0.rm_storage_dir
cars_ram1.rm_storage_dir # cars_ram1.rm_storage_dir
cars_ram2.rm_storage_dir # cars_ram2.rm_storage_dir
end
end end
end end