Specs are fixed & reworked for the most part. New benchmarks.
parent
2a71254533
commit
c475c4f584
5
TODO.md
5
TODO.md
|
|
@ -1,8 +1,3 @@
|
||||||
# API
|
|
||||||
|
|
||||||
Cached indexes (index, partition, tags) should be used by default.
|
|
||||||
Uncached indexes should be an option, through a new function `add_uncached_index` or something.
|
|
||||||
|
|
||||||
# Performance
|
# Performance
|
||||||
|
|
||||||
Search with some kind of "pagination" system: ask entries with a limit on the number of elements and an offset.
|
Search with some kind of "pagination" system: ask entries with a limit on the number of elements and an offset.
|
||||||
|
|
|
||||||
|
|
@ -1,181 +1,109 @@
|
||||||
require "benchmark"
|
require "benchmark"
|
||||||
require "./benchmark-utilities.cr"
|
require "./benchmark-utilities.cr"
|
||||||
|
require "./cars.cr"
|
||||||
|
|
||||||
require "../src/dodb.cr"
|
cars_ram = DODB::RAMOnlySpecDataBase(Car).new
|
||||||
require "./test-data.cr"
|
cars_cached = DODB::CachedSpecDataBase(Car).new
|
||||||
|
cars_semi = DODB::SpecDataBase(Car).new "-semi"
|
||||||
|
cars_uncached = DODB::SpecDataBase(Car).new
|
||||||
|
|
||||||
class DODBCachedCars < DODB::CachedDataBase(Car)
|
ram_searchby_name, ram_searchby_color, ram_searchby_keywords = ram_indexes cars_ram
|
||||||
property storage_dir : String
|
cached_searchby_name, cached_searchby_color, cached_searchby_keywords = cached_indexes cars_cached
|
||||||
def initialize(storage_ext = "", remove_previous_data = true)
|
semi_searchby_name, semi_searchby_color, semi_searchby_keywords = cached_indexes cars_semi
|
||||||
@storage_dir = "test-storage-cars-cached#{storage_ext}"
|
uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = uncached_indexes cars_uncached
|
||||||
|
|
||||||
if remove_previous_data
|
|
||||||
::FileUtils.rm_rf storage_dir
|
|
||||||
end
|
|
||||||
|
|
||||||
super storage_dir
|
|
||||||
end
|
|
||||||
|
|
||||||
def rm_storage_dir
|
|
||||||
::FileUtils.rm_rf @storage_dir
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
class DODBUnCachedCars < DODB::DataBase(Car)
|
|
||||||
property storage_dir : String
|
|
||||||
def initialize(storage_ext = "", remove_previous_data = true)
|
|
||||||
@storage_dir = "test-storage-cars-uncached#{storage_ext}"
|
|
||||||
|
|
||||||
if remove_previous_data
|
|
||||||
::FileUtils.rm_rf storage_dir
|
|
||||||
end
|
|
||||||
|
|
||||||
super storage_dir
|
|
||||||
end
|
|
||||||
|
|
||||||
def rm_storage_dir
|
|
||||||
::FileUtils.rm_rf @storage_dir
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
class DODBSemiCachedCars < DODB::DataBase(Car)
|
|
||||||
property storage_dir : String
|
|
||||||
def initialize(storage_ext = "", remove_previous_data = true)
|
|
||||||
@storage_dir = "test-storage-cars-semi#{storage_ext}"
|
|
||||||
|
|
||||||
if remove_previous_data
|
|
||||||
::FileUtils.rm_rf storage_dir
|
|
||||||
end
|
|
||||||
|
|
||||||
super storage_dir
|
|
||||||
end
|
|
||||||
|
|
||||||
def rm_storage_dir
|
|
||||||
::FileUtils.rm_rf @storage_dir
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def init_indexes(storage : DODB::Storage)
|
|
||||||
n = storage.new_index "name", &.name
|
|
||||||
c = storage.new_partition "color", &.color
|
|
||||||
k = storage.new_tags "keyword", &.keywords
|
|
||||||
return n, c, k
|
|
||||||
end
|
|
||||||
|
|
||||||
def init_uncached_indexes(storage : DODB::Storage)
|
|
||||||
n = storage.new_uncached_index "name", &.name
|
|
||||||
c = storage.new_uncached_partition "color", &.color
|
|
||||||
k = storage.new_uncached_tags "keyword", &.keywords
|
|
||||||
return n, c, k
|
|
||||||
end
|
|
||||||
|
|
||||||
def add_cars(storage : DODB::Storage, nb_iterations : Int32)
|
|
||||||
i = 0
|
|
||||||
car1 = Car.new "Corvet", "red", [ "shiny", "impressive", "fast", "elegant" ]
|
|
||||||
car2 = Car.new "Bullet-GT", "blue", [ "shiny", "fast", "expensive" ]
|
|
||||||
car3 = Car.new "Deudeuche", "beige", [ "curvy", "sublime" ]
|
|
||||||
car4 = Car.new "Ford-5", "red", [ "unknown" ]
|
|
||||||
car5 = Car.new "C-MAX", "gray", [ "spacious", "affordable" ]
|
|
||||||
|
|
||||||
while i < nb_iterations
|
|
||||||
car1.name = "Corvet-#{i}"
|
|
||||||
car2.name = "Bullet-GT-#{i}"
|
|
||||||
car3.name = "Deudeuche-#{i}"
|
|
||||||
car4.name = "Ford-5-#{i}"
|
|
||||||
car5.name = "C-MAX-#{i}"
|
|
||||||
|
|
||||||
storage << car1
|
|
||||||
storage << car2
|
|
||||||
storage << car3
|
|
||||||
storage << car4
|
|
||||||
storage << car5
|
|
||||||
i += 1
|
|
||||||
STDOUT.write "\radding value #{i}".to_slice
|
|
||||||
end
|
|
||||||
puts ""
|
|
||||||
end
|
|
||||||
|
|
||||||
cars_cached = DODBCachedCars.new
|
|
||||||
cars_uncached = DODBUnCachedCars.new
|
|
||||||
cars_semi = DODBSemiCachedCars.new
|
|
||||||
|
|
||||||
cached_searchby_name, cached_searchby_color, cached_searchby_keywords = init_indexes cars_cached
|
|
||||||
uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = init_uncached_indexes cars_uncached
|
|
||||||
semi_searchby_name, semi_searchby_color, semi_searchby_keywords = init_indexes cars_semi
|
|
||||||
|
|
||||||
|
add_cars cars_ram, 1_000
|
||||||
add_cars cars_cached, 1_000
|
add_cars cars_cached, 1_000
|
||||||
add_cars cars_uncached, 1_000
|
|
||||||
add_cars cars_semi, 1_000
|
add_cars cars_semi, 1_000
|
||||||
|
add_cars cars_uncached, 1_000
|
||||||
|
|
||||||
# Searching for data with an index.
|
# Searching for data with an index.
|
||||||
|
puts "search by index 'Corvet-500': get a single value"
|
||||||
Benchmark.ips do |x|
|
Benchmark.ips do |x|
|
||||||
x.report("(cars db) searching a data with an index (with a cache)") do
|
x.report("(ram db and index) searching a data with an index") do
|
||||||
|
corvet = ram_searchby_name.get "Corvet-500"
|
||||||
|
end
|
||||||
|
|
||||||
|
x.report("(cached db and index) searching a data with an index") do
|
||||||
corvet = cached_searchby_name.get "Corvet-500"
|
corvet = cached_searchby_name.get "Corvet-500"
|
||||||
end
|
end
|
||||||
|
|
||||||
x.report("(cars db) searching a data with an index (semi: cache is only on index)") do
|
x.report("(semi: uncached db but cached index) searching a data with an index") do
|
||||||
corvet = semi_searchby_name.get "Corvet-500"
|
corvet = semi_searchby_name.get "Corvet-500"
|
||||||
end
|
end
|
||||||
|
|
||||||
x.report("(cars db) searching a data with an index (without a cache)") do
|
x.report("(uncached db and index) searching a data with an index") do
|
||||||
corvet = uncached_searchby_name.get "Corvet-500"
|
corvet = uncached_searchby_name.get "Corvet-500"
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
# Searching for data with a partition.
|
# Searching for data with a partition.
|
||||||
|
puts ""
|
||||||
|
puts "search by partition 'red': get #{ram_searchby_color.get("red").size} values"
|
||||||
Benchmark.ips do |x|
|
Benchmark.ips do |x|
|
||||||
x.report("(cars db) searching a data with a partition (with a cache)") do
|
x.report("(ram db and partition) searching a data with a partition") do
|
||||||
red_cars = cached_searchby_color.get "red"
|
corvet = ram_searchby_color.get "red"
|
||||||
end
|
end
|
||||||
|
|
||||||
x.report("(cars db) searching a data with a partition (semi: cache is only on partition)") do
|
x.report("(cached db and partition) searching a data with a partition") do
|
||||||
red_cars = semi_searchby_color.get "red"
|
corvet = cached_searchby_color.get "red"
|
||||||
end
|
end
|
||||||
|
|
||||||
x.report("(cars db) searching a data with a partition (without a cache)") do
|
x.report("(semi: uncached db but cached partition) searching a data with a partition") do
|
||||||
red_cars = uncached_searchby_color.get "red"
|
corvet = semi_searchby_color.get "red"
|
||||||
|
end
|
||||||
|
|
||||||
|
x.report("(uncached db and partition) searching a data with a partition") do
|
||||||
|
corvet = uncached_searchby_color.get "red"
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
# Searching for data with a tag.
|
# Searching for data with a tag.
|
||||||
|
puts ""
|
||||||
|
puts "search by tag 'spacious': get #{ram_searchby_keywords.get("spacious").size} values"
|
||||||
Benchmark.ips do |x|
|
Benchmark.ips do |x|
|
||||||
x.report("(cars db) searching a data with a tag (with a cache)") do
|
x.report("(ram db and tag) searching a data with a tag") do
|
||||||
red_cars = cached_searchby_keywords.get "spacious"
|
corvet = ram_searchby_keywords.get "spacious"
|
||||||
end
|
end
|
||||||
|
|
||||||
x.report("(cars db) searching a data with a tag (semi: cache is only on tags)") do
|
x.report("(cached db and tag) searching a data with a tag") do
|
||||||
red_cars = semi_searchby_keywords.get "spacious"
|
corvet = cached_searchby_keywords.get "spacious"
|
||||||
end
|
end
|
||||||
|
|
||||||
x.report("(cars db) searching a data with a tag (without a cache)") do
|
x.report("(semi: uncached db but cached tag) searching a data with a tag") do
|
||||||
red_cars = uncached_searchby_keywords.get "spacious"
|
corvet = semi_searchby_keywords.get "spacious"
|
||||||
|
end
|
||||||
|
|
||||||
|
x.report("(uncached db and tag) searching a data with a tag") do
|
||||||
|
corvet = uncached_searchby_keywords.get "spacious"
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
cars_cached.rm_storage_dir
|
#cars_ram.rm_storage_dir
|
||||||
cars_uncached.rm_storage_dir
|
#cars_cached.rm_storage_dir
|
||||||
|
#cars_semi.rm_storage_dir
|
||||||
|
#cars_uncached.rm_storage_dir
|
||||||
|
|
||||||
cars_cached = DODBCachedCars.new
|
#cars_cached = DODB::CachedSpecDataBase(Car).new
|
||||||
cars_uncached = DODBUnCachedCars.new
|
#cars_uncached = DODB::SpecDataBase(Car).new
|
||||||
|
#
|
||||||
#init_indexes cars_cached
|
##init_indexes cars_cached
|
||||||
#init_indexes cars_uncached
|
##init_indexes cars_uncached
|
||||||
cached_searchby_name, cached_searchby_color, cached_searchby_keywords = init_indexes cars_cached
|
#cached_searchby_name, cached_searchby_color, cached_searchby_keywords = cached_indexes cars_cached
|
||||||
uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = init_uncached_indexes cars_uncached
|
#uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = uncached_indexes cars_uncached
|
||||||
|
#
|
||||||
add_cars cars_cached, 1_000
|
#add_cars cars_cached, 1_000
|
||||||
add_cars cars_uncached, 1_000
|
#add_cars cars_uncached, 1_000
|
||||||
|
#
|
||||||
nb_run = 1000
|
#nb_run = 1000
|
||||||
|
#
|
||||||
perform_benchmark_average_verbose "(cached) search db with an index", nb_run, do
|
#perform_benchmark_average_verbose "(cached) search db with an index", nb_run, do
|
||||||
cached_searchby_name.get "Corvet-500"
|
# cached_searchby_name.get "Corvet-500"
|
||||||
end
|
#end
|
||||||
|
#
|
||||||
perform_benchmark_average_verbose "(uncached) search db with an index", nb_run, do
|
#perform_benchmark_average_verbose "(uncached) search db with an index", nb_run, do
|
||||||
uncached_searchby_name.get "Corvet-500"
|
# uncached_searchby_name.get "Corvet-500"
|
||||||
end
|
#end
|
||||||
|
#
|
||||||
cars_cached.rm_storage_dir
|
#cars_cached.rm_storage_dir
|
||||||
cars_uncached.rm_storage_dir
|
#cars_uncached.rm_storage_dir
|
||||||
cars_semi.rm_storage_dir
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
class DODB::SpecDataBase(V) < DODB::DataBase(V)
|
class DODB::SpecDataBase(V) < DODB::DataBase(V)
|
||||||
property storage_dir : String
|
property storage_dir : String
|
||||||
def initialize(storage_ext = "", remove_previous_data = true)
|
def initialize(storage_ext = "", remove_previous_data = true)
|
||||||
@storage_dir = "test-spec-storage#{storage_ext}"
|
@storage_dir = "specdb-storage-uncached#{storage_ext}"
|
||||||
|
|
||||||
if remove_previous_data
|
if remove_previous_data
|
||||||
::FileUtils.rm_rf storage_dir
|
::FileUtils.rm_rf storage_dir
|
||||||
|
|
@ -18,7 +18,7 @@ end
|
||||||
class DODB::CachedSpecDataBase(V) < DODB::CachedDataBase(V)
|
class DODB::CachedSpecDataBase(V) < DODB::CachedDataBase(V)
|
||||||
property storage_dir : String
|
property storage_dir : String
|
||||||
def initialize(storage_ext = "", remove_previous_data = true)
|
def initialize(storage_ext = "", remove_previous_data = true)
|
||||||
@storage_dir = "test-spec-storage-cached#{storage_ext}"
|
@storage_dir = "specdb-storage-cached#{storage_ext}"
|
||||||
|
|
||||||
if remove_previous_data
|
if remove_previous_data
|
||||||
::FileUtils.rm_rf storage_dir
|
::FileUtils.rm_rf storage_dir
|
||||||
|
|
@ -35,7 +35,7 @@ end
|
||||||
class DODB::RAMOnlySpecDataBase(V) < DODB::RAMOnlyDataBase(V)
|
class DODB::RAMOnlySpecDataBase(V) < DODB::RAMOnlyDataBase(V)
|
||||||
property storage_dir : String
|
property storage_dir : String
|
||||||
def initialize(storage_ext = "", remove_previous_data = true)
|
def initialize(storage_ext = "", remove_previous_data = true)
|
||||||
@storage_dir = "test-spec-storage-ram#{storage_ext}"
|
@storage_dir = "specdb-storage-ram#{storage_ext}"
|
||||||
|
|
||||||
if remove_previous_data
|
if remove_previous_data
|
||||||
::FileUtils.rm_rf storage_dir
|
::FileUtils.rm_rf storage_dir
|
||||||
|
|
|
||||||
|
|
@ -4,42 +4,39 @@ require "./cars.cr"
|
||||||
|
|
||||||
corvet0 = Car.new "Corvet-0", "red", [ "shiny", "impressive", "fast", "elegant" ]
|
corvet0 = Car.new "Corvet-0", "red", [ "shiny", "impressive", "fast", "elegant" ]
|
||||||
|
|
||||||
describe "DODB car" do
|
describe "uncached, cached and ram indexes" do
|
||||||
describe "behavior of uncached, cached and ram indexes:" do
|
it "RAM DB - add items, add indexes, search, reindex, search" do
|
||||||
it "RAM DB - add items, add indexes, search, reindex, search" do
|
|
||||||
|
|
||||||
cars_ram0 = DODB::RAMOnlySpecDataBase(Car).new "-0"
|
cars_ram0 = DODB::RAMOnlySpecDataBase(Car).new "-0"
|
||||||
cars_ram1 = DODB::RAMOnlySpecDataBase(Car).new "-1"
|
cars_ram1 = DODB::RAMOnlySpecDataBase(Car).new "-1"
|
||||||
cars_ram2 = DODB::RAMOnlySpecDataBase(Car).new "-2"
|
cars_ram2 = DODB::RAMOnlySpecDataBase(Car).new "-2"
|
||||||
|
|
||||||
add_cars cars_ram0, 1
|
add_cars cars_ram0, 1
|
||||||
add_cars cars_ram1, 1
|
add_cars cars_ram1, 1
|
||||||
add_cars cars_ram2, 1
|
add_cars cars_ram2, 1
|
||||||
|
|
||||||
uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = uncached_indexes cars_ram0
|
uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = uncached_indexes cars_ram0
|
||||||
cached_searchby_name, cached_searchby_color, cached_searchby_keywords = cached_indexes cars_ram1
|
cached_searchby_name, cached_searchby_color, cached_searchby_keywords = cached_indexes cars_ram1
|
||||||
ram_searchby_name, ram_searchby_color, ram_searchby_keywords = ram_indexes cars_ram2
|
ram_searchby_name, ram_searchby_color, ram_searchby_keywords = ram_indexes cars_ram2
|
||||||
|
|
||||||
uncached_searchby_name.get?("Corvet-0").should be_nil
|
uncached_searchby_name.get?("Corvet-0").should be_nil
|
||||||
cached_searchby_name.get?("Corvet-0").should be_nil
|
cached_searchby_name.get?("Corvet-0").should be_nil
|
||||||
ram_searchby_name.get?("Corvet-0").should be_nil
|
ram_searchby_name.get?("Corvet-0").should be_nil
|
||||||
|
|
||||||
cars_ram0.reindex_everything!
|
cars_ram0.reindex_everything!
|
||||||
cars_ram1.reindex_everything!
|
cars_ram1.reindex_everything!
|
||||||
cars_ram2.reindex_everything!
|
cars_ram2.reindex_everything!
|
||||||
|
|
||||||
# Cannot get the value since it's not written on disk.
|
# Get the value even if not written on the disk since the index was written on the disk.
|
||||||
# FIXME: should only retrieve the key with the index, not the actual value.
|
# The value is retrieved by the database, the index only reads its key in the database.
|
||||||
# So, this should work.
|
uncached_searchby_name.get?("Corvet-0").should eq corvet0
|
||||||
uncached_searchby_name.get?("Corvet-0").should be_nil
|
|
||||||
|
|
||||||
# Both cached and RAM indexes can retrieve the value since they store the key.
|
# Both cached and RAM indexes can retrieve the value since they store the key.
|
||||||
cached_searchby_name.get?("Corvet-0").should eq corvet0
|
cached_searchby_name.get?("Corvet-0").should eq corvet0
|
||||||
ram_searchby_name.get?("Corvet-0").should eq corvet0
|
ram_searchby_name.get?("Corvet-0").should eq corvet0
|
||||||
|
|
||||||
cars_ram0.rm_storage_dir
|
# cars_ram0.rm_storage_dir
|
||||||
cars_ram1.rm_storage_dir
|
# cars_ram1.rm_storage_dir
|
||||||
cars_ram2.rm_storage_dir
|
# cars_ram2.rm_storage_dir
|
||||||
end
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue