From 09678965e07a349fadc0175050adbbdb31ca2286 Mon Sep 17 00:00:00 2001 From: Philippe PITTOLI Date: Thu, 23 May 2024 14:18:16 +0200 Subject: [PATCH] FIFO implementation improved: let's see the results. --- bin/extract-final-data.sh | 6 +-- spec/benchmark-cars.cr | 25 ++++++----- spec/spec-database.cr | 37 +++++++-------- spec/test-cars.cr | 12 ++--- spec/test-ships.cr | 90 ++++++++++++++++++------------------- src/dodb/storage/stacked.cr | 13 ++++-- 6 files changed, 97 insertions(+), 86 deletions(-) diff --git a/bin/extract-final-data.sh b/bin/extract-final-data.sh index 697b2d6..034c79e 100755 --- a/bin/extract-final-data.sh +++ b/bin/extract-final-data.sh @@ -19,6 +19,6 @@ awk '{ print $1 }' < $d/ram_index.d > it mkdir data echo "from truncated data (.t) to graphed data data/XXX.d" -paste it $d/ram_index.t $d/cached_index.t $d/semi_index.t $d/uncached_index.t > ./data/index.d -paste it $d/ram_partitions.t $d/cached_partitions.t $d/semi_partitions.t $d/uncached_partitions.t > ./data/partitions.d -paste it $d/ram_tags.t $d/cached_tags.t $d/semi_tags.t $d/uncached_tags.t > ./data/tags.d +paste it $d/ram_index.t $d/cached_index.t $d/fifo_index.t $d/semi_index.t $d/uncached_index.t > ./data/index.d +paste it $d/ram_partitions.t $d/cached_partitions.t $d/fifo_partitions.t $d/semi_partitions.t $d/uncached_partitions.t > ./data/partitions.d +paste it $d/ram_tags.t $d/cached_tags.t $d/fifo_tags.t $d/semi_tags.t $d/uncached_tags.t > ./data/tags.d diff --git a/spec/benchmark-cars.cr b/spec/benchmark-cars.cr index a564d01..eb8ce1f 100644 --- a/spec/benchmark-cars.cr +++ b/spec/benchmark-cars.cr @@ -86,13 +86,15 @@ def prepare_env(storage, name, s_index, s_partition, s_tags, &) end def batch() - cars_ram = DODB::RAMOnlySpecDataBase(Car).new - cars_cached = DODB::CachedSpecDataBase(Car).new - cars_semi = DODB::SpecDataBase(Car).new "-semi" - cars_uncached = DODB::SpecDataBase(Car).new + cars_ram = SPECDB::RAMOnly(Car).new + cars_cached = SPECDB::Cached(Car).new + cars_fifo = DODB::FIFOSpecDataBase(Car).new + cars_semi = SPECDB::Uncached(Car).new "-semi" + cars_uncached = SPECDB::Uncached(Car).new ram_Sby_name, ram_Sby_color, ram_Sby_keywords = ram_indexes cars_ram cached_Sby_name, cached_Sby_color, cached_Sby_keywords = cached_indexes cars_cached + fifo_Sby_name, fifo_Sby_color, fifo_Sby_keywords = cached_indexes cars_fifo semi_Sby_name, semi_Sby_color, semi_Sby_keywords = cached_indexes cars_semi uncached_Sby_name, uncached_Sby_color, uncached_Sby_keywords = uncached_indexes cars_uncached @@ -123,9 +125,10 @@ def batch() end } - prepare_env cars_cached, "cached", cached_Sby_name, cached_Sby_color, cached_Sby_keywords, &fn - prepare_env cars_ram, "ram", ram_Sby_name, ram_Sby_color, ram_Sby_keywords, &fn - prepare_env cars_semi, "semi", semi_Sby_name, semi_Sby_color, semi_Sby_keywords, &fn + prepare_env cars_ram, "ram", ram_Sby_name, ram_Sby_color, ram_Sby_keywords, &fn + prepare_env cars_cached, "cached", cached_Sby_name, cached_Sby_color, cached_Sby_keywords, &fn + prepare_env cars_fifo, "fifo", fifo_Sby_name, fifo_Sby_color, fifo_Sby_keywords, &fn + prepare_env cars_semi, "semi", semi_Sby_name, semi_Sby_color, semi_Sby_keywords, &fn prepare_env cars_uncached, "uncached", uncached_Sby_name, uncached_Sby_color, uncached_Sby_keywords, &fn end @@ -141,10 +144,10 @@ def perform_add(storage : DODB::Storage(Car)) end def batch_add() - cars_ram = DODB::RAMOnlySpecDataBase(Car).new - cars_cached = DODB::CachedSpecDataBase(Car).new - cars_semi = DODB::SpecDataBase(Car).new "-semi" - cars_uncached = DODB::SpecDataBase(Car).new + cars_ram = SPECDB::RAMOnly(Car).new + cars_cached = SPECDB::Cached(Car).new + cars_semi = SPECDB::Uncached(Car).new "-semi" + cars_uncached = SPECDB::Uncached(Car).new ram_indexes cars_ram cached_indexes cars_cached diff --git a/spec/spec-database.cr b/spec/spec-database.cr index 57ddb90..fc0cbfb 100644 --- a/spec/spec-database.cr +++ b/spec/spec-database.cr @@ -1,12 +1,8 @@ -class DODB::SpecDataBase(V) < DODB::Storage::Basic(V) +class SPECDB::Uncached(V) < DODB::Storage::Basic(V) property storage_dir : String def initialize(storage_ext = "", remove_previous_data = true) @storage_dir = "specdb-storage-uncached#{storage_ext}" - - if remove_previous_data - ::FileUtils.rm_rf storage_dir - end - + ::FileUtils.rm_rf storage_dir if remove_previous_data super storage_dir end @@ -15,15 +11,11 @@ class DODB::SpecDataBase(V) < DODB::Storage::Basic(V) end end -class DODB::CachedSpecDataBase(V) < DODB::Storage::Cached(V) +class SPECDB::Cached(V) < DODB::Storage::Cached(V) property storage_dir : String def initialize(storage_ext = "", remove_previous_data = true) @storage_dir = "specdb-storage-cached#{storage_ext}" - - if remove_previous_data - ::FileUtils.rm_rf storage_dir - end - + ::FileUtils.rm_rf storage_dir if remove_previous_data super storage_dir end @@ -32,15 +24,24 @@ class DODB::CachedSpecDataBase(V) < DODB::Storage::Cached(V) end end -class DODB::RAMOnlySpecDataBase(V) < DODB::Storage::RAMOnly(V) +class DODB::FIFOSpecDataBase(V) < DODB::Storage::Stacked(V) + property storage_dir : String + def initialize(storage_ext = "", remove_previous_data = true) + @storage_dir = "specdb-storage-fifo#{storage_ext}" + ::FileUtils.rm_rf storage_dir if remove_previous_data + super storage_dir + end + + def rm_storage_dir + ::FileUtils.rm_rf @storage_dir + end +end + +class SPECDB::RAMOnly(V) < DODB::Storage::RAMOnly(V) property storage_dir : String def initialize(storage_ext = "", remove_previous_data = true) @storage_dir = "specdb-storage-ram#{storage_ext}" - - if remove_previous_data - ::FileUtils.rm_rf storage_dir - end - + ::FileUtils.rm_rf storage_dir if remove_previous_data super storage_dir end diff --git a/spec/test-cars.cr b/spec/test-cars.cr index 2cee998..1313c28 100644 --- a/spec/test-cars.cr +++ b/spec/test-cars.cr @@ -6,9 +6,9 @@ corvet0 = Car.new "Corvet-0", "red", [ "shiny", "impressive", "fast", "elegant" describe "uncached, cached and ram indexes" do it "RAM DB - add items, add indexes, search, reindex, search" do - cars_ram0 = DODB::RAMOnlySpecDataBase(Car).new "-0" - cars_ram1 = DODB::RAMOnlySpecDataBase(Car).new "-1" - cars_ram2 = DODB::RAMOnlySpecDataBase(Car).new "-2" + cars_ram0 = SPECDB::RAMOnly(Car).new "-0" + cars_ram1 = SPECDB::RAMOnly(Car).new "-1" + cars_ram2 = SPECDB::RAMOnly(Car).new "-2" add_cars cars_ram0, 1 add_cars cars_ram1, 1 @@ -42,9 +42,9 @@ end describe "tracking inconsistencies between implementations" do it "index - partitions - tags" do - cars_ram0 = DODB::RAMOnlySpecDataBase(Car).new "-0" - cars_ram1 = DODB::RAMOnlySpecDataBase(Car).new "-1" - cars_ram2 = DODB::RAMOnlySpecDataBase(Car).new "-2" + cars_ram0 = SPECDB::RAMOnly(Car).new "-0" + cars_ram1 = SPECDB::RAMOnly(Car).new "-1" + cars_ram2 = SPECDB::RAMOnly(Car).new "-2" uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = uncached_indexes cars_ram0 cached_searchby_name, cached_searchby_color, cached_searchby_keywords = cached_indexes cars_ram1 diff --git a/spec/test-ships.cr b/spec/test-ships.cr index f96cc72..361d8ed 100644 --- a/spec/test-ships.cr +++ b/spec/test-ships.cr @@ -9,7 +9,7 @@ end describe "DODB::Storage::Basic" do describe "basics" do it "store and get data" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new Ship.all_ships.each do |ship| db << ship @@ -21,7 +21,7 @@ describe "DODB::Storage::Basic" do end it "rewrite already stored data" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new ship = Ship.all_ships[0] key = db << ship @@ -35,7 +35,7 @@ describe "DODB::Storage::Basic" do end it "properly remove data" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new Ship.all_ships.each do |ship| db << ship @@ -58,12 +58,12 @@ describe "DODB::Storage::Basic" do end it "preserves data on reopening" do - db1 = DODB::SpecDataBase(Ship).new + db1 = SPECDB::Uncached(Ship).new db1 << Ship.kisaragi db1.to_a.size.should eq(1) - db2 = DODB::SpecDataBase(Ship).new remove_previous_data: false + db2 = SPECDB::Uncached(Ship).new remove_previous_data: false db2 << Ship.mutsuki db1.to_a.size.should eq(2) @@ -73,7 +73,7 @@ describe "DODB::Storage::Basic" do end it "iterates in normal and reversed order" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new Ship.all_ships.each do |ship| db << ship @@ -95,7 +95,7 @@ describe "DODB::Storage::Basic" do end it "respects the provided offsets if any" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new Ship.all_ships.each do |ship| db << ship @@ -115,7 +115,7 @@ describe "DODB::Storage::Basic" do describe "indices" do it "do basic indexing" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new db_ships_by_name = db.new_index "name", &.name @@ -131,7 +131,7 @@ describe "DODB::Storage::Basic" do end it "raise on index overload" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new db_ships_by_name = db.new_index "name", &.name @@ -147,7 +147,7 @@ describe "DODB::Storage::Basic" do end it "properly deindex" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new db_ships_by_name = db.new_index "name", &.name @@ -167,7 +167,7 @@ describe "DODB::Storage::Basic" do end it "properly reindex" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new db_ships_by_name = db.new_index "name", &.name @@ -187,7 +187,7 @@ describe "DODB::Storage::Basic" do end it "properly updates" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new db_ships_by_name = db.new_index "name", &.name @@ -211,7 +211,7 @@ describe "DODB::Storage::Basic" do describe "partitions" do it "do basic partitioning" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new db_ships_by_class = db.new_partition "class", &.klass @@ -241,7 +241,7 @@ describe "DODB::Storage::Basic" do end it "removes select elements from partitions" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new db_ships_by_class = db.new_partition "class", &.klass @@ -263,7 +263,7 @@ describe "DODB::Storage::Basic" do describe "tags" do it "do basic tagging" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new db_ships_by_tags = db.new_tags "tags", &.tags @@ -286,7 +286,7 @@ describe "DODB::Storage::Basic" do end it "properly removes tags" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new db_ships_by_tags = db.new_tags "tags", &.tags @@ -312,7 +312,7 @@ describe "DODB::Storage::Basic" do end it "gets items that have multiple tags" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new db_ships_by_tags = db.new_tags "tags", &.tags @@ -335,7 +335,7 @@ describe "DODB::Storage::Basic" do describe "atomic operations" do it "safe_get and safe_get?" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new db_ships_by_name = db.new_index "name", &.name @@ -359,7 +359,7 @@ describe "DODB::Storage::Basic" do describe "tools" do it "rebuilds indexes" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new db_ships_by_name = db.new_index "name", &.name db_ships_by_class = db.new_partition "class", &.klass @@ -380,7 +380,7 @@ describe "DODB::Storage::Basic" do end it "migrates properly" do - old_db = DODB::SpecDataBase(PrimitiveShip).new "-migration-origin" + old_db = SPECDB::Uncached(PrimitiveShip).new "-migration-origin" old_ships_by_name = old_db.new_index "name", &.name old_ships_by_class = old_db.new_partition "class", &.class_name @@ -392,7 +392,7 @@ describe "DODB::Storage::Basic" do # At this point, the “old” DB is filled. Now we need to convert # to the new DB. - new_db = DODB::SpecDataBase(Ship).new "-migration-target" + new_db = SPECDB::Uncached(Ship).new "-migration-target" new_ships_by_name = new_db.new_index "name", &.name new_ships_by_class = new_db.new_partition "class", &.klass @@ -432,7 +432,7 @@ describe "DODB::Storage::Basic" do entries_per_fork = 100 it "works for pushing values" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new processes = [] of Process @@ -454,7 +454,7 @@ describe "DODB::Storage::Basic" do end it "works for updating values" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new db_entries_by_name = db.new_index "name", &.name # First pass, creating data. @@ -494,7 +494,7 @@ describe "DODB::Storage::Basic" do end it "does parallel-safe updates" do - db = DODB::SpecDataBase(Ship).new + db = SPECDB::Uncached(Ship).new db_entries_by_name = db.new_index "name", &.name # We’ll be storing an integer in the "klass" field, and incrementing @@ -528,7 +528,7 @@ end describe "DODB::Storage::Cached" do describe "basics" do it "store and get data" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new Ship.all_ships.each do |ship| db << ship @@ -540,7 +540,7 @@ describe "DODB::Storage::Cached" do end it "rewrite already stored data" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new ship = Ship.all_ships[0] key = db << ship @@ -554,7 +554,7 @@ describe "DODB::Storage::Cached" do end it "properly remove data" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new Ship.all_ships.each do |ship| db << ship @@ -577,12 +577,12 @@ describe "DODB::Storage::Cached" do end it "preserves data on reopening" do - db1 = DODB::CachedSpecDataBase(Ship).new + db1 = SPECDB::Cached(Ship).new db1 << Ship.kisaragi db1.to_a.size.should eq(1) - db2 = DODB::CachedSpecDataBase(Ship).new remove_previous_data: false + db2 = SPECDB::Cached(Ship).new remove_previous_data: false db2 << Ship.mutsuki # Only difference with DODB::Storage::Basic: concurrent DB cannot coexists. @@ -593,7 +593,7 @@ describe "DODB::Storage::Cached" do end it "iterates in normal and reversed order" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new Ship.all_ships.each do |ship| db << ship @@ -615,7 +615,7 @@ describe "DODB::Storage::Cached" do end it "respects the provided offsets if any" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new Ship.all_ships.each do |ship| db << ship @@ -635,7 +635,7 @@ describe "DODB::Storage::Cached" do describe "indices" do it "do basic indexing" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new db_ships_by_name = db.new_index "name", &.name @@ -651,7 +651,7 @@ describe "DODB::Storage::Cached" do end it "raise on index overload" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new db_ships_by_name = db.new_index "name", &.name @@ -667,7 +667,7 @@ describe "DODB::Storage::Cached" do end it "properly deindex" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new db_ships_by_name = db.new_index "name", &.name @@ -687,7 +687,7 @@ describe "DODB::Storage::Cached" do end it "properly reindex" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new db_ships_by_name = db.new_index "name", &.name @@ -707,7 +707,7 @@ describe "DODB::Storage::Cached" do end it "properly updates" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new db_ships_by_name = db.new_index "name", &.name @@ -731,7 +731,7 @@ describe "DODB::Storage::Cached" do describe "partitions" do it "do basic partitioning" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new db_ships_by_class = db.new_partition "class", &.klass @@ -761,7 +761,7 @@ describe "DODB::Storage::Cached" do end it "removes select elements from partitions" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new db_ships_by_class = db.new_partition "class", &.klass @@ -783,7 +783,7 @@ describe "DODB::Storage::Cached" do describe "tags" do it "do basic tagging" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new db_ships_by_tags = db.new_tags "tags", &.tags @@ -806,7 +806,7 @@ describe "DODB::Storage::Cached" do end it "properly removes tags" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new db_ships_by_tags = db.new_tags "tags", &.tags @@ -833,7 +833,7 @@ describe "DODB::Storage::Cached" do end it "gets items that have multiple tags" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new db_ships_by_tags = db.new_tags "tags", &.tags @@ -856,7 +856,7 @@ describe "DODB::Storage::Cached" do describe "atomic operations" do it "safe_get and safe_get?" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new db_ships_by_name = db.new_index "name", &.name @@ -880,7 +880,7 @@ describe "DODB::Storage::Cached" do describe "tools" do it "rebuilds indexes" do - db = DODB::CachedSpecDataBase(Ship).new + db = SPECDB::Cached(Ship).new db_ships_by_name = db.new_index "name", &.name db_ships_by_class = db.new_partition "class", &.klass @@ -901,7 +901,7 @@ describe "DODB::Storage::Cached" do end it "migrates properly" do - old_db = DODB::CachedSpecDataBase(PrimitiveShip).new "-migration-origin" + old_db = SPECDB::Cached(PrimitiveShip).new "-migration-origin" old_ships_by_name = old_db.new_index "name", &.name old_ships_by_class = old_db.new_partition "class", &.class_name @@ -913,7 +913,7 @@ describe "DODB::Storage::Cached" do # At this point, the “old” DB is filled. Now we need to convert # to the new DB. - new_db = DODB::CachedSpecDataBase(Ship).new "-migration-target" + new_db = SPECDB::Cached(Ship).new "-migration-target" new_ships_by_name = new_db.new_index "name", &.name new_ships_by_class = new_db.new_partition "class", &.klass diff --git a/src/dodb/storage/stacked.cr b/src/dodb/storage/stacked.cr index eb8cfd3..23630c9 100644 --- a/src/dodb/storage/stacked.cr +++ b/src/dodb/storage/stacked.cr @@ -38,9 +38,16 @@ class DODB::Storage::Stacked(V) < DODB::Storage::Cached(V) property stack : FIFO(Int32) # Initializes the `StackedDataBase` with a maximum number of entries in the cache. - def initialize(directory_name : String, @max_entries : Int32 = 100_000) - super directory_name - @stack = FIFO(Int32).new @max_entries + def initialize(@directory_name : String, max_entries : UInt32 = 100_000) + @stack = FIFO(Int32).new max_entries + Dir.mkdir_p data_path + Dir.mkdir_p locks_directory + + begin + self.last_key + rescue + self.last_key = -1 + end end def [](key : Int32) : V