Baguette/dodb.cr
3
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

FIFO implementation improved: let's see the results.

Browse Source
This commit is contained in:
Philippe PITTOLI 2024-05-23 14:18:16 +02:00
parent e2841a263b
commit 09678965e0
6 changed files with 97 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bin/extract-final-data.sh
View File

@ -19,6 +19,6 @@ awk '{ print $1 }' < $d/ram_index.d > it
mkdir data
echo "from truncated data (.t) to graphed data data/XXX.d"
paste it $d/ram_index.t $d/cached_index.t $d/semi_index.t $d/uncached_index.t > ./data/index.d
paste it $d/ram_partitions.t $d/cached_partitions.t $d/semi_partitions.t $d/uncached_partitions.t > ./data/partitions.d
paste it $d/ram_tags.t $d/cached_tags.t $d/semi_tags.t $d/uncached_tags.t > ./data/tags.d
paste it $d/ram_index.t $d/cached_index.t $d/fifo_index.t $d/semi_index.t $d/uncached_index.t > ./data/index.d
paste it $d/ram_partitions.t $d/cached_partitions.t $d/fifo_partitions.t $d/semi_partitions.t $d/uncached_partitions.t > ./data/partitions.d
paste it $d/ram_tags.t $d/cached_tags.t $d/fifo_tags.t $d/semi_tags.t $d/uncached_tags.t > ./data/tags.d

25
spec/benchmark-cars.cr
View File

@ -86,13 +86,15 @@ def prepare_env(storage, name, s_index, s_partition, s_tags, &)
end
def batch()
cars_ram = DODB::RAMOnlySpecDataBase(Car).new
cars_cached = DODB::CachedSpecDataBase(Car).new
cars_semi = DODB::SpecDataBase(Car).new "-semi"
cars_uncached = DODB::SpecDataBase(Car).new
cars_ram = SPECDB::RAMOnly(Car).new
cars_cached = SPECDB::Cached(Car).new
cars_fifo = DODB::FIFOSpecDataBase(Car).new
cars_semi = SPECDB::Uncached(Car).new "-semi"
cars_uncached = SPECDB::Uncached(Car).new
ram_Sby_name, ram_Sby_color, ram_Sby_keywords = ram_indexes cars_ram
cached_Sby_name, cached_Sby_color, cached_Sby_keywords = cached_indexes cars_cached
fifo_Sby_name, fifo_Sby_color, fifo_Sby_keywords = cached_indexes cars_fifo
semi_Sby_name, semi_Sby_color, semi_Sby_keywords = cached_indexes cars_semi
uncached_Sby_name, uncached_Sby_color, uncached_Sby_keywords = uncached_indexes cars_uncached
@ -123,9 +125,10 @@ def batch()
end
}
prepare_env cars_cached, "cached", cached_Sby_name, cached_Sby_color, cached_Sby_keywords, &fn
prepare_env cars_ram, "ram", ram_Sby_name, ram_Sby_color, ram_Sby_keywords, &fn
prepare_env cars_semi, "semi", semi_Sby_name, semi_Sby_color, semi_Sby_keywords, &fn
prepare_env cars_ram, "ram", ram_Sby_name, ram_Sby_color, ram_Sby_keywords, &fn
prepare_env cars_cached, "cached", cached_Sby_name, cached_Sby_color, cached_Sby_keywords, &fn
prepare_env cars_fifo, "fifo", fifo_Sby_name, fifo_Sby_color, fifo_Sby_keywords, &fn
prepare_env cars_semi, "semi", semi_Sby_name, semi_Sby_color, semi_Sby_keywords, &fn
prepare_env cars_uncached, "uncached", uncached_Sby_name, uncached_Sby_color, uncached_Sby_keywords, &fn
end
@ -141,10 +144,10 @@ def perform_add(storage : DODB::Storage(Car))
end
def batch_add()
cars_ram = DODB::RAMOnlySpecDataBase(Car).new
cars_cached = DODB::CachedSpecDataBase(Car).new
cars_semi = DODB::SpecDataBase(Car).new "-semi"
cars_uncached = DODB::SpecDataBase(Car).new
cars_ram = SPECDB::RAMOnly(Car).new
cars_cached = SPECDB::Cached(Car).new
cars_semi = SPECDB::Uncached(Car).new "-semi"
cars_uncached = SPECDB::Uncached(Car).new
ram_indexes cars_ram
cached_indexes cars_cached

37
spec/spec-database.cr
View File

@ -1,12 +1,8 @@
class DODB::SpecDataBase(V) < DODB::Storage::Basic(V)
class SPECDB::Uncached(V) < DODB::Storage::Basic(V)
property storage_dir : String
def initialize(storage_ext = "", remove_previous_data = true)
@storage_dir = "specdb-storage-uncached#{storage_ext}"
if remove_previous_data
::FileUtils.rm_rf storage_dir
end
::FileUtils.rm_rf storage_dir if remove_previous_data
super storage_dir
end
@ -15,15 +11,11 @@ class DODB::SpecDataBase(V) < DODB::Storage::Basic(V)
end
end
class DODB::CachedSpecDataBase(V) < DODB::Storage::Cached(V)
class SPECDB::Cached(V) < DODB::Storage::Cached(V)
property storage_dir : String
def initialize(storage_ext = "", remove_previous_data = true)
@storage_dir = "specdb-storage-cached#{storage_ext}"
if remove_previous_data
::FileUtils.rm_rf storage_dir
end
::FileUtils.rm_rf storage_dir if remove_previous_data
super storage_dir
end
@ -32,15 +24,24 @@ class DODB::CachedSpecDataBase(V) < DODB::Storage::Cached(V)
end
end
class DODB::RAMOnlySpecDataBase(V) < DODB::Storage::RAMOnly(V)
class DODB::FIFOSpecDataBase(V) < DODB::Storage::Stacked(V)
property storage_dir : String
def initialize(storage_ext = "", remove_previous_data = true)
@storage_dir = "specdb-storage-fifo#{storage_ext}"
::FileUtils.rm_rf storage_dir if remove_previous_data
super storage_dir
end
def rm_storage_dir
::FileUtils.rm_rf @storage_dir
end
end
class SPECDB::RAMOnly(V) < DODB::Storage::RAMOnly(V)
property storage_dir : String
def initialize(storage_ext = "", remove_previous_data = true)
@storage_dir = "specdb-storage-ram#{storage_ext}"
if remove_previous_data
::FileUtils.rm_rf storage_dir
end
::FileUtils.rm_rf storage_dir if remove_previous_data
super storage_dir
end

12
spec/test-cars.cr
View File

@ -6,9 +6,9 @@ corvet0 = Car.new "Corvet-0", "red", [ "shiny", "impressive", "fast", "elegant"
describe "uncached, cached and ram indexes" do
it "RAM DB - add items, add indexes, search, reindex, search" do
cars_ram0 = DODB::RAMOnlySpecDataBase(Car).new "-0"
cars_ram1 = DODB::RAMOnlySpecDataBase(Car).new "-1"
cars_ram2 = DODB::RAMOnlySpecDataBase(Car).new "-2"
cars_ram0 = SPECDB::RAMOnly(Car).new "-0"
cars_ram1 = SPECDB::RAMOnly(Car).new "-1"
cars_ram2 = SPECDB::RAMOnly(Car).new "-2"
add_cars cars_ram0, 1
add_cars cars_ram1, 1
@ -42,9 +42,9 @@ end
describe "tracking inconsistencies between implementations" do
it "index - partitions - tags" do
cars_ram0 = DODB::RAMOnlySpecDataBase(Car).new "-0"
cars_ram1 = DODB::RAMOnlySpecDataBase(Car).new "-1"
cars_ram2 = DODB::RAMOnlySpecDataBase(Car).new "-2"
cars_ram0 = SPECDB::RAMOnly(Car).new "-0"
cars_ram1 = SPECDB::RAMOnly(Car).new "-1"
cars_ram2 = SPECDB::RAMOnly(Car).new "-2"
uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = uncached_indexes cars_ram0
cached_searchby_name, cached_searchby_color, cached_searchby_keywords = cached_indexes cars_ram1

90
spec/test-ships.cr
View File

@ -9,7 +9,7 @@ end
describe "DODB::Storage::Basic" do
describe "basics" do
it "store and get data" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
Ship.all_ships.each do |ship|
db << ship
@ -21,7 +21,7 @@ describe "DODB::Storage::Basic" do
end
it "rewrite already stored data" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
ship = Ship.all_ships[0]
key = db << ship
@ -35,7 +35,7 @@ describe "DODB::Storage::Basic" do
end
it "properly remove data" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
Ship.all_ships.each do |ship|
db << ship
@ -58,12 +58,12 @@ describe "DODB::Storage::Basic" do
end
it "preserves data on reopening" do
db1 = DODB::SpecDataBase(Ship).new
db1 = SPECDB::Uncached(Ship).new
db1 << Ship.kisaragi
db1.to_a.size.should eq(1)
db2 = DODB::SpecDataBase(Ship).new remove_previous_data: false
db2 = SPECDB::Uncached(Ship).new remove_previous_data: false
db2 << Ship.mutsuki
db1.to_a.size.should eq(2)
@ -73,7 +73,7 @@ describe "DODB::Storage::Basic" do
end
it "iterates in normal and reversed order" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
Ship.all_ships.each do |ship|
db << ship
@ -95,7 +95,7 @@ describe "DODB::Storage::Basic" do
end
it "respects the provided offsets if any" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
Ship.all_ships.each do |ship|
db << ship
@ -115,7 +115,7 @@ describe "DODB::Storage::Basic" do
describe "indices" do
it "do basic indexing" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
db_ships_by_name = db.new_index "name", &.name
@ -131,7 +131,7 @@ describe "DODB::Storage::Basic" do
end
it "raise on index overload" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
db_ships_by_name = db.new_index "name", &.name
@ -147,7 +147,7 @@ describe "DODB::Storage::Basic" do
end
it "properly deindex" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
db_ships_by_name = db.new_index "name", &.name
@ -167,7 +167,7 @@ describe "DODB::Storage::Basic" do
end
it "properly reindex" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
db_ships_by_name = db.new_index "name", &.name
@ -187,7 +187,7 @@ describe "DODB::Storage::Basic" do
end
it "properly updates" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
db_ships_by_name = db.new_index "name", &.name
@ -211,7 +211,7 @@ describe "DODB::Storage::Basic" do
describe "partitions" do
it "do basic partitioning" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
db_ships_by_class = db.new_partition "class", &.klass
@ -241,7 +241,7 @@ describe "DODB::Storage::Basic" do
end
it "removes select elements from partitions" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
db_ships_by_class = db.new_partition "class", &.klass
@ -263,7 +263,7 @@ describe "DODB::Storage::Basic" do
describe "tags" do
it "do basic tagging" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
db_ships_by_tags = db.new_tags "tags", &.tags
@ -286,7 +286,7 @@ describe "DODB::Storage::Basic" do
end
it "properly removes tags" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
db_ships_by_tags = db.new_tags "tags", &.tags
@ -312,7 +312,7 @@ describe "DODB::Storage::Basic" do
end
it "gets items that have multiple tags" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
db_ships_by_tags = db.new_tags "tags", &.tags
@ -335,7 +335,7 @@ describe "DODB::Storage::Basic" do
describe "atomic operations" do
it "safe_get and safe_get?" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
db_ships_by_name = db.new_index "name", &.name
@ -359,7 +359,7 @@ describe "DODB::Storage::Basic" do
describe "tools" do
it "rebuilds indexes" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
db_ships_by_name = db.new_index "name", &.name
db_ships_by_class = db.new_partition "class", &.klass
@ -380,7 +380,7 @@ describe "DODB::Storage::Basic" do
end
it "migrates properly" do
old_db = DODB::SpecDataBase(PrimitiveShip).new "-migration-origin"
old_db = SPECDB::Uncached(PrimitiveShip).new "-migration-origin"
old_ships_by_name = old_db.new_index "name", &.name
old_ships_by_class = old_db.new_partition "class", &.class_name
@ -392,7 +392,7 @@ describe "DODB::Storage::Basic" do
# At this point, the “old” DB is filled. Now we need to convert
# to the new DB.
new_db = DODB::SpecDataBase(Ship).new "-migration-target"
new_db = SPECDB::Uncached(Ship).new "-migration-target"
new_ships_by_name = new_db.new_index "name", &.name
new_ships_by_class = new_db.new_partition "class", &.klass
@ -432,7 +432,7 @@ describe "DODB::Storage::Basic" do
entries_per_fork = 100
it "works for pushing values" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
processes = [] of Process
@ -454,7 +454,7 @@ describe "DODB::Storage::Basic" do
end
it "works for updating values" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
db_entries_by_name = db.new_index "name", &.name
# First pass, creating data.
@ -494,7 +494,7 @@ describe "DODB::Storage::Basic" do
end
it "does parallel-safe updates" do
db = DODB::SpecDataBase(Ship).new
db = SPECDB::Uncached(Ship).new
db_entries_by_name = db.new_index "name", &.name
# We’ll be storing an integer in the "klass" field, and incrementing
@ -528,7 +528,7 @@ end
describe "DODB::Storage::Cached" do
describe "basics" do
it "store and get data" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
Ship.all_ships.each do |ship|
db << ship
@ -540,7 +540,7 @@ describe "DODB::Storage::Cached" do
end
it "rewrite already stored data" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
ship = Ship.all_ships[0]
key = db << ship
@ -554,7 +554,7 @@ describe "DODB::Storage::Cached" do
end
it "properly remove data" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
Ship.all_ships.each do |ship|
db << ship
@ -577,12 +577,12 @@ describe "DODB::Storage::Cached" do
end
it "preserves data on reopening" do
db1 = DODB::CachedSpecDataBase(Ship).new
db1 = SPECDB::Cached(Ship).new
db1 << Ship.kisaragi
db1.to_a.size.should eq(1)
db2 = DODB::CachedSpecDataBase(Ship).new remove_previous_data: false
db2 = SPECDB::Cached(Ship).new remove_previous_data: false
db2 << Ship.mutsuki
# Only difference with DODB::Storage::Basic: concurrent DB cannot coexists.
@ -593,7 +593,7 @@ describe "DODB::Storage::Cached" do
end
it "iterates in normal and reversed order" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
Ship.all_ships.each do |ship|
db << ship
@ -615,7 +615,7 @@ describe "DODB::Storage::Cached" do
end
it "respects the provided offsets if any" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
Ship.all_ships.each do |ship|
db << ship
@ -635,7 +635,7 @@ describe "DODB::Storage::Cached" do
describe "indices" do
it "do basic indexing" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
db_ships_by_name = db.new_index "name", &.name
@ -651,7 +651,7 @@ describe "DODB::Storage::Cached" do
end
it "raise on index overload" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
db_ships_by_name = db.new_index "name", &.name
@ -667,7 +667,7 @@ describe "DODB::Storage::Cached" do
end
it "properly deindex" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
db_ships_by_name = db.new_index "name", &.name
@ -687,7 +687,7 @@ describe "DODB::Storage::Cached" do
end
it "properly reindex" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
db_ships_by_name = db.new_index "name", &.name
@ -707,7 +707,7 @@ describe "DODB::Storage::Cached" do
end
it "properly updates" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
db_ships_by_name = db.new_index "name", &.name
@ -731,7 +731,7 @@ describe "DODB::Storage::Cached" do
describe "partitions" do
it "do basic partitioning" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
db_ships_by_class = db.new_partition "class", &.klass
@ -761,7 +761,7 @@ describe "DODB::Storage::Cached" do
end
it "removes select elements from partitions" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
db_ships_by_class = db.new_partition "class", &.klass
@ -783,7 +783,7 @@ describe "DODB::Storage::Cached" do
describe "tags" do
it "do basic tagging" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
db_ships_by_tags = db.new_tags "tags", &.tags
@ -806,7 +806,7 @@ describe "DODB::Storage::Cached" do
end
it "properly removes tags" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
db_ships_by_tags = db.new_tags "tags", &.tags
@ -833,7 +833,7 @@ describe "DODB::Storage::Cached" do
end
it "gets items that have multiple tags" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
db_ships_by_tags = db.new_tags "tags", &.tags
@ -856,7 +856,7 @@ describe "DODB::Storage::Cached" do
describe "atomic operations" do
it "safe_get and safe_get?" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
db_ships_by_name = db.new_index "name", &.name
@ -880,7 +880,7 @@ describe "DODB::Storage::Cached" do
describe "tools" do
it "rebuilds indexes" do
db = DODB::CachedSpecDataBase(Ship).new
db = SPECDB::Cached(Ship).new
db_ships_by_name = db.new_index "name", &.name
db_ships_by_class = db.new_partition "class", &.klass
@ -901,7 +901,7 @@ describe "DODB::Storage::Cached" do
end
it "migrates properly" do
old_db = DODB::CachedSpecDataBase(PrimitiveShip).new "-migration-origin"
old_db = SPECDB::Cached(PrimitiveShip).new "-migration-origin"
old_ships_by_name = old_db.new_index "name", &.name
old_ships_by_class = old_db.new_partition "class", &.class_name
@ -913,7 +913,7 @@ describe "DODB::Storage::Cached" do
# At this point, the “old” DB is filled. Now we need to convert
# to the new DB.
new_db = DODB::CachedSpecDataBase(Ship).new "-migration-target"
new_db = SPECDB::Cached(Ship).new "-migration-target"
new_ships_by_name = new_db.new_index "name", &.name
new_ships_by_class = new_db.new_partition "class", &.klass

13
src/dodb/storage/stacked.cr
View File

@ -38,9 +38,16 @@ class DODB::Storage::Stacked(V) < DODB::Storage::Cached(V)
property stack : FIFO(Int32)
# Initializes the `StackedDataBase` with a maximum number of entries in the cache.
def initialize(directory_name : String, @max_entries : Int32 = 100_000)
super directory_name
@stack = FIFO(Int32).new @max_entries
def initialize(@directory_name : String, max_entries : UInt32 = 100_000)
@stack = FIFO(Int32).new max_entries
Dir.mkdir_p data_path
Dir.mkdir_p locks_directory
begin
self.last_key
rescue
self.last_key = -1
end
end
def [](key : Int32) : V