From f8c0222593d042767332fbeb590fe3b47fbd1297 Mon Sep 17 00:00:00 2001 From: Karchnu Date: Sat, 18 Jul 2020 00:11:11 +0200 Subject: [PATCH] CachedDB OK --- spec/cached.cr | 583 +++++++++++++++++++++--------------------- spec/test.cr | 89 +------ src/cached.cr | 8 +- src/dodb.cr | 40 +-- src/dodb/index.cr | 2 +- src/dodb/partition.cr | 2 +- 6 files changed, 330 insertions(+), 394 deletions(-) diff --git a/spec/cached.cr b/spec/cached.cr index 8c8bc6d..7a09cee 100644 --- a/spec/cached.cr +++ b/spec/cached.cr @@ -5,7 +5,7 @@ require "../src/dodb.cr" require "./test-data.cr" -class DODB::SpecDataBase < DODB::DataBase::Cached(Ship) +class DODB::SpecDataBase < DODB::CachedDataBase(Ship) def initialize(storage_ext = "", remove_previous_data = true) storage_dir = "test-storage#{storage_ext}" @@ -62,6 +62,18 @@ describe "DODB::DataBase::Cached" do end end + it "preserves data on reopening" do + db1 = DODB::SpecDataBase.new + db1 << Ship.kisaragi + + db1.to_a.size.should eq(1) + + db2 = DODB::SpecDataBase.new remove_previous_data: false + db2 << Ship.mutsuki + + db2.to_a.size.should eq(2) + end + it "iterates in normal and reversed order" do db = DODB::SpecDataBase.new @@ -99,292 +111,291 @@ describe "DODB::DataBase::Cached" do end end -# describe "indices" do -# it "do basic indexing" do -# db = DODB::SpecDataBase.new -# -# db_ships_by_name = db.new_index "name", &.name -# -# Ship.all_ships.each do |ship| -# db << ship -# end -# -# Ship.all_ships.each_with_index do |ship| -# db_ships_by_name.get?(ship.name).should eq(ship) -# end -# end -# end + describe "indices" do + it "do basic indexing" do + db = DODB::SpecDataBase.new -# it "raise on index overload" do -# db = DODB::SpecDataBase.new -# -# db_ships_by_name = db.new_index "name", &.name -# -# db << Ship.kisaragi -# -# # Should not be allowed to store an entry whose “name” field -# # already exists. -# expect_raises(DODB::IndexOverload) do -# db << Ship.kisaragi + db_ships_by_name = db.new_index "name", &.name + + Ship.all_ships.each do |ship| + db << ship + end + + Ship.all_ships.each_with_index do |ship| + db_ships_by_name.get?(ship.name).should eq(ship) + end + end + + it "raise on index overload" do + db = DODB::SpecDataBase.new + + db_ships_by_name = db.new_index "name", &.name + + db << Ship.kisaragi + + # Should not be allowed to store an entry whose “name” field + # already exists. + expect_raises(DODB::IndexOverload) do + db << Ship.kisaragi + end + end + + it "properly deindex" do + db = DODB::SpecDataBase.new + + db_ships_by_name = db.new_index "name", &.name + + Ship.all_ships.each do |ship| + db << ship + end + + Ship.all_ships.each_with_index do |ship, i| + db.delete i + end + + Ship.all_ships.each do |ship| + db_ships_by_name.get?(ship.name).should be_nil + end + end + + it "properly reindex" do + db = DODB::SpecDataBase.new + + db_ships_by_name = db.new_index "name", &.name + + key = db << Ship.kisaragi + + # We give the old id to the new ship, to get it replaced in + # the database. + some_new_ship = Ship.all_ships[2].clone + + db[key] = some_new_ship + + db[key].should eq(some_new_ship) + + db_ships_by_name.get?(some_new_ship.name).should eq(some_new_ship) + end + + it "properly updates" do + db = DODB::SpecDataBase.new + + db_ships_by_name = db.new_index "name", &.name + + Ship.all_ships.each do |ship| + db << ship + end + + new_kisaragi = Ship.kisaragi.clone.tap do |s| + s.name = "Kisaragi Kai" # Don’t think about it too much. + end + + # We’re changing an indexed value on purpose. + db_ships_by_name.update "Kisaragi", new_kisaragi + + db_ships_by_name.get?("Kisaragi").should be_nil + db_ships_by_name.get?(new_kisaragi.name).should eq new_kisaragi + end + end + + describe "partitions" do + it "do basic partitioning" do + db = DODB::SpecDataBase.new + + db_ships_by_class = db.new_partition "class", &.klass + + Ship.all_ships.each do |ship| + db << ship + end + + Ship.all_ships.each do |ship| + db_ships_by_class.get(ship.klass).should contain(ship) + end + + # We extract the possible classes to do test on them. + ship_classes = Ship.all_ships.map(&.klass).uniq + ship_classes.each do |klass| + partition = db_ships_by_class.get klass + + # A partition on “class” should contain entries that all + # share the same value of “class”. + partition.map(&.klass.==(klass)).reduce { |a, b| + a && b + }.should be_true + end + + db_ships_by_class.get("does-not-exist").should eq [] of Ship + end + + it "removes select elements from partitions" do + db = DODB::SpecDataBase.new + + db_ships_by_class = db.new_partition "class", &.klass + + Ship.all_ships.each do |ship| + db << ship + end + + db_ships_by_class.delete "Mutsuki", &.name.==("Kisaragi") + + Ship.all_ships.map(&.klass).uniq.each do |klass| + partition = db_ships_by_class.get klass + + partition.any?(&.name.==("Kisaragi")).should be_false + end + end + end + + describe "tags" do + it "do basic tagging" do + db = DODB::SpecDataBase.new + + db_ships_by_tags = db.new_tags "tags", &.tags + + Ship.all_ships.each do |ship| + db << ship + end + + db_ships_by_tags.get("flagship").should eq([Ship.flagship]) + + # All returned entries should have the requested tag. + db_ships_by_tags.get("name ship") + .map(&.tags.includes?("name ship")) + .reduce { |a, e| a && e } + .should be_true + + # There shouldn’t be one in our data about WWII Japanese warships… + db_ships_by_tags.get("starship").should eq([] of Ship) + end + + it "properly removes tags" do + db = DODB::SpecDataBase.new + + db_ships_by_tags = db.new_tags "tags", &.tags + + Ship.all_ships.each do |ship| + db << ship + end + + # Removing the “flagship” tag, brace for impact. + flagship, index = db_ships_by_tags.get_with_indices("flagship")[0] + flagship.tags = [] of String + db[index] = flagship + + + +# ship, index = db_ships_by_tags.update(tag: "flagship") do |ship, index| +# ship.tags = [] of String +# db[index] = ship # end -# end -# -# it "properly deindex" do -# db = DODB::SpecDataBase.new -# -# db_ships_by_name = db.new_index "name", &.name -# -# Ship.all_ships.each do |ship| -# db << ship -# end -# -# Ship.all_ships.each_with_index do |ship, i| -# db.delete i -# end -# -# Ship.all_ships.each do |ship| -# db_ships_by_name.get?(ship.name).should be_nil -# end -# end -# -# it "properly reindex" do -# db = DODB::SpecDataBase.new -# -# db_ships_by_name = db.new_index "name", &.name -# -# key = db << Ship.kisaragi -# -# # We give the old id to the new ship, to get it replaced in -# # the database. -# some_new_ship = Ship.all_ships[2].clone -# -# db[key] = some_new_ship -# -# db[key].should eq(some_new_ship) -# -# db_ships_by_name.get?(some_new_ship.name).should eq(some_new_ship) -# end -# -# it "properly updates" do -# db = DODB::SpecDataBase.new -# -# db_ships_by_name = db.new_index "name", &.name -# -# Ship.all_ships.each do |ship| -# db << ship -# end -# -# new_kisaragi = Ship.kisaragi.clone.tap do |s| -# s.name = "Kisaragi Kai" # Don’t think about it too much. -# end -# -# # We’re changing an indexed value on purpose. -# db_ships_by_name.update "Kisaragi", new_kisaragi -# -# db_ships_by_name.get?("Kisaragi").should be_nil -# db_ships_by_name.get?(new_kisaragi.name).should eq new_kisaragi -# end -# end -# -# describe "partitions" do -# it "do basic partitioning" do -# db = DODB::SpecDataBase.new -# -# db_ships_by_class = db.new_partition "class", &.klass -# -# Ship.all_ships.each do |ship| -# db << ship -# end -# -# Ship.all_ships.each do |ship| -# db_ships_by_class.get(ship.klass).should contain(ship) -# end -# -# # We extract the possible classes to do test on them. -# ship_classes = Ship.all_ships.map(&.klass).uniq -# ship_classes.each do |klass| -# partition = db_ships_by_class.get klass -# -# # A partition on “class” should contain entries that all -# # share the same value of “class”. -# partition.map(&.klass.==(klass)).reduce { |a, b| -# a && b -# }.should be_true -# end -# -# db_ships_by_class.get("does-not-exist").should eq [] of Ship -# end -# -# it "removes select elements from partitions" do -# db = DODB::SpecDataBase.new -# -# db_ships_by_class = db.new_partition "class", &.klass -# -# Ship.all_ships.each do |ship| -# db << ship -# end -# -# db_ships_by_class.delete "Mutsuki", &.name.==("Kisaragi") -# -# Ship.all_ships.map(&.klass).uniq.each do |klass| -# partition = db_ships_by_class.get klass -# -# partition.any?(&.name.==("Kisaragi")).should be_false -# end -# end -# end -# -# describe "tags" do -# it "do basic tagging" do -# db = DODB::SpecDataBase.new -# -# db_ships_by_tags = db.new_tags "tags", &.tags -# -# Ship.all_ships.each do |ship| -# db << ship -# end -# -# db_ships_by_tags.get("flagship").should eq([Ship.flagship]) -# -# # All returned entries should have the requested tag. -# db_ships_by_tags.get("name ship") -# .map(&.tags.includes?("name ship")) -# .reduce { |a, e| a && e } -# .should be_true -# -# # There shouldn’t be one in our data about WWII Japanese warships… -# db_ships_by_tags.get("starship").should eq([] of Ship) -# end -# -# it "properly removes tags" do -# db = DODB::SpecDataBase.new -# -# db_ships_by_tags = db.new_tags "tags", &.tags -# -# Ship.all_ships.each do |ship| -# db << ship -# end -# -# # Removing the “flagship” tag, brace for impact. -# flagship, index = db_ships_by_tags.get_with_indices("flagship")[0] -# flagship.tags = [] of String -# db[index] = flagship -# -# -# -## ship, index = db_ships_by_tags.update(tag: "flagship") do |ship, index| -## ship.tags = [] of String -## db[index] = ship -## end -# -# db_ships_by_tags.get("flagship").should eq([] of Ship) -# end -# -# it "gets items that have multiple tags" do -# db = DODB::SpecDataBase.new -# -# db_ships_by_tags = db.new_tags "tags", &.tags -# -# Ship.all_ships.each do |ship| -# db << ship -# end -# -# results = db_ships_by_tags.get(["flagship", "name ship"]) -# results.should eq([Ship.yamato]) -# -# results = db_ships_by_tags.get(["name ship", "flagship"]) -# results.should eq([Ship.yamato]) -# -# results = db_ships_by_tags.get(["flagship"]) -# results.should eq([Ship.yamato]) -# end -# end -# -# describe "atomic operations" do -# it "safe_get and safe_get?" do -# db = DODB::SpecDataBase.new -# -# db_ships_by_name = db.new_index "name", &.name -# -# Ship.all_ships.each do |ship| -# db << ship -# end -# -# Ship.all_ships.each do |ship| -# db_ships_by_name.safe_get ship.name do |results| -# results.should eq(ship) -# end -# -# db_ships_by_name.safe_get? ship.name do |results| -# results.should eq(ship) -# end -# end -# end -# end -# -# describe "tools" do -# it "rebuilds indexes" do -# db = DODB::SpecDataBase.new -# -# db_ships_by_name = db.new_index "name", &.name -# db_ships_by_class = db.new_partition "class", &.klass -# db_ships_by_tags = db.new_tags "tags", &.tags -# -# Ship.all_ships.each do |ship| -# db << ship -# end -# -# db.reindex_everything! -# -# Ship.all_ships.each do |ship| -# db_ships_by_name.get?(ship.name).should eq(ship) -# db_ships_by_class.get(ship.klass).should contain(ship) -# end -# end -# -# it "migrates properly" do -# ::FileUtils.rm_rf "test-storage-migration-origin" -# old_db = DODB::DataBase(PrimitiveShip).new "test-storage-migration-origin" -# -# old_ships_by_name = old_db.new_index "name", &.name -# old_ships_by_class = old_db.new_partition "class", &.class_name -# -# PrimitiveShip.all_ships.each do |ship| -# old_db << ship -# end -# -# # At this point, the “old” DB is filled. Now we need to convert -# # to the new DB. -# -# new_db = DODB::SpecDataBase.new "-migration-target" -# -# new_ships_by_name = new_db.new_index "name", &.name -# new_ships_by_class = new_db.new_partition "class", &.klass -# new_ships_by_tags = new_db.new_tags "tags", &.tags -# -# old_db.each_with_index do |ship, index| -# new_ship = Ship.new ship.name, -# klass: ship.class_name, -# id: ship.id, -# tags: Array(String).new.tap { |tags| -# tags << "name ship" if ship.name == ship.class_name -# } -# -# new_db[index] = new_ship -# end -# -# # At this point, the conversion is done, so… we’re making a few -# # arbitrary tests on the new data. -# -# old_db.each_with_index do |old_ship, old_index| -# ship = new_db[old_index] -# -# ship.id.should eq(old_ship.id) -# ship.klass.should eq(old_ship.class_name) -# -# ship.tags.any?(&.==("name ship")).should be_true if ship.name == ship.klass -# end -# end -# end + + db_ships_by_tags.get("flagship").should eq([] of Ship) + end + + it "gets items that have multiple tags" do + db = DODB::SpecDataBase.new + + db_ships_by_tags = db.new_tags "tags", &.tags + + Ship.all_ships.each do |ship| + db << ship + end + + results = db_ships_by_tags.get(["flagship", "name ship"]) + results.should eq([Ship.yamato]) + + results = db_ships_by_tags.get(["name ship", "flagship"]) + results.should eq([Ship.yamato]) + + results = db_ships_by_tags.get(["flagship"]) + results.should eq([Ship.yamato]) + end + end + + describe "atomic operations" do + it "safe_get and safe_get?" do + db = DODB::SpecDataBase.new + + db_ships_by_name = db.new_index "name", &.name + + Ship.all_ships.each do |ship| + db << ship + end + + Ship.all_ships.each do |ship| + db_ships_by_name.safe_get ship.name do |results| + results.should eq(ship) + end + + db_ships_by_name.safe_get? ship.name do |results| + results.should eq(ship) + end + end + end + end + + describe "tools" do + it "rebuilds indexes" do + db = DODB::SpecDataBase.new + + db_ships_by_name = db.new_index "name", &.name + db_ships_by_class = db.new_partition "class", &.klass + db_ships_by_tags = db.new_tags "tags", &.tags + + Ship.all_ships.each do |ship| + db << ship + end + + db.reindex_everything! + + Ship.all_ships.each do |ship| + db_ships_by_name.get?(ship.name).should eq(ship) + db_ships_by_class.get(ship.klass).should contain(ship) + end + end + + it "migrates properly" do + ::FileUtils.rm_rf "test-storage-migration-origin" + old_db = DODB::DataBase(PrimitiveShip).new "test-storage-migration-origin" + + old_ships_by_name = old_db.new_index "name", &.name + old_ships_by_class = old_db.new_partition "class", &.class_name + + PrimitiveShip.all_ships.each do |ship| + old_db << ship + end + + # At this point, the “old” DB is filled. Now we need to convert + # to the new DB. + + new_db = DODB::SpecDataBase.new "-migration-target" + + new_ships_by_name = new_db.new_index "name", &.name + new_ships_by_class = new_db.new_partition "class", &.klass + new_ships_by_tags = new_db.new_tags "tags", &.tags + + old_db.each_with_index do |ship, index| + new_ship = Ship.new ship.name, + klass: ship.class_name, + id: ship.id, + tags: Array(String).new.tap { |tags| + tags << "name ship" if ship.name == ship.class_name + } + + new_db[index] = new_ship + end + + # At this point, the conversion is done, so… we’re making a few + # arbitrary tests on the new data. + + old_db.each_with_index do |old_ship, old_index| + ship = new_db[old_index] + + ship.id.should eq(old_ship.id) + ship.klass.should eq(old_ship.class_name) + + ship.tags.any?(&.==("name ship")).should be_true if ship.name == ship.klass + end + end + end end diff --git a/spec/test.cr b/spec/test.cr index 43863d1..3c4d4ce 100644 --- a/spec/test.cr +++ b/spec/test.cr @@ -1,94 +1,9 @@ require "spec" require "file_utils" -require "json" -require "uuid" -require "../src/*" +require "../src/dodb.cr" +require "./test-data.cr" -# FIXME: Split the test data in separate files. We don’t care about those here. - -class Ship - include JSON::Serializable - - def_clone - - property id : String - property klass : String - property name : String - property tags : Array(String) - - def initialize(@name, @klass = "", @id = UUID.random.to_s, @tags = [] of String) - end - - # Makes testing arrays of this class easier. - def <=>(other) - @name <=> other.name - end - - # Common, reusable test data. - # Those data can be indexed, partitioned or tagged on different parameters, - # and can easily be extended. - - class_getter kisaragi = Ship.new("Kisaragi", "Mutsuki") - class_getter mutsuki = Ship.new("Mutsuki", "Mutsuki", tags: ["name ship"]) - class_getter yayoi = Ship.new("Yayoi", "Mutsuki") - class_getter destroyers = [ - @@mutsuki, - @@kisaragi, - @@yayoi, - Ship.new("Uzuki", "Mutsuki"), - Ship.new("Satsuki", "Mutsuki"), - - Ship.new("Shiratsuyu", "Shiratsuyu", tags: ["name ship"]), - Ship.new("Murasame", "Shiratsuyu"), - Ship.new("Yuudachi", "Shiratsuyu") - ] - - class_getter yamato = - Ship.new("Yamato", "Yamato", tags: ["name ship", "flagship"]) - class_getter flagship : Ship = yamato - class_getter battleships = [ - @@yamato, - Ship.new("Kongou", "Kongou", tags: ["name ship"]), - Ship.new("Haruna", "Kongou"), - Ship.new("Kirishima", "Kongou"), - Ship.new("Hiei" , "Kongou"), - Ship.new("Musashi", "Yamato"), - Ship.new("Shinano", "Yamato") - ] - - class_getter all_ships : Array(Ship) = @@destroyers + @@battleships - - # Equality is true if every property is identical. - def ==(other) - @id == other.id && @klass == other.klass && @name == other.name && - @tags == other.tags - end -end - -# This will be used for migration testing, but basically it’s a variant of -# the class above, a few extra fields, a few missing ones. -class PrimitiveShip - include JSON::Serializable - - property id : String - property name : String - property wooden : Bool = false # Will be removed. - property class_name : String # Will be renamed - property flagship : Bool = false # Will be moved to tags. - - def initialize(@name, @class_name = "", @id = UUID.random.to_s, @flagship = false) - end - - class_getter kamikaze = - PrimitiveShip.new("Kamikaze", "Kamikaze") - class_getter asakaze = - PrimitiveShip.new("Asakaze", "Kamikaze") - class_getter all_ships : Array(PrimitiveShip) = [ - @@kamikaze, - @@asakaze - ] -end class DODB::SpecDataBase < DODB::DataBase(Ship) def initialize(storage_ext = "", remove_previous_data = true) diff --git a/src/cached.cr b/src/cached.cr index e1a3de6..d20fe69 100644 --- a/src/cached.cr +++ b/src/cached.cr @@ -16,7 +16,7 @@ class Hash(K,V) end end -class DODB::DataBase::Cached(V) +class DODB::CachedDataBase(V) < DODB::Storage(V) @indexers = [] of Indexer(V) property data = Hash(Int32, V).new @@ -31,6 +31,10 @@ class DODB::DataBase::Cached(V) end # TODO: load the database in RAM at start-up + DODB::DataBase(V).new(@directory_name).each_with_index do |v, index| + puts "loading value #{v} at index #{index}" + self[index] = v + end end # Data should be written on disk AND in RAM. @@ -38,7 +42,6 @@ class DODB::DataBase::Cached(V) index = last_index + 1 self[index] = item self.last_index = index - @data[self.last_index] = item self.last_index end @@ -282,6 +285,7 @@ class DODB::DataBase::Cached(V) private def remove_data! FileUtils.rm_rf data_path Dir.mkdir_p data_path + @data = Hash(Int32, V).new end private def remove_indexing! diff --git a/src/dodb.cr b/src/dodb.cr index 4042ef0..c8f87ce 100644 --- a/src/dodb.cr +++ b/src/dodb.cr @@ -3,7 +3,29 @@ require "json" require "./dodb/*" -class DODB::DataBase(V) +abstract class DODB::Storage(V) + def request_lock(name) + r = -1 + file_path = get_lock_file_path name + file_perms = 0o644 + + flags = LibC::O_EXCL | LibC::O_CREAT + while (r = LibC.open file_path, flags, file_perms) == -1 + sleep 1.milliseconds + end + + LibC.close r + end + def release_lock(name) + File.delete get_lock_file_path name + end + + abstract def [](key : Int32) + abstract def <<(item : V) + abstract def delete(key : Int32) +end + +class DODB::DataBase(V) < DODB::Storage(V) @indexers = [] of Indexer(V) def initialize(@directory_name : String) @@ -44,22 +66,6 @@ class DODB::DataBase(V) end end - def request_lock(name) - r = -1 - file_path = get_lock_file_path name - file_perms = 0o644 - - flags = LibC::O_EXCL | LibC::O_CREAT - while (r = LibC.open file_path, flags, file_perms) == -1 - sleep 1.milliseconds - end - - LibC.close r - end - def release_lock(name) - File.delete get_lock_file_path name - end - ## # name is the name that will be used on the file system. def new_partition(name : String, &block : Proc(V, String)) diff --git a/src/dodb/index.cr b/src/dodb/index.cr index 5c518f3..4e83185 100644 --- a/src/dodb/index.cr +++ b/src/dodb/index.cr @@ -9,7 +9,7 @@ class DODB::Index(V) < DODB::Indexer(V) property key_proc : Proc(V, String) getter storage_root : String - @storage : DODB::DataBase(V) + @storage : DODB::Storage(V) def initialize(@storage, @storage_root, @name, @key_proc) Dir.mkdir_p indexing_directory diff --git a/src/dodb/partition.cr b/src/dodb/partition.cr index ed3849e..d0d4954 100644 --- a/src/dodb/partition.cr +++ b/src/dodb/partition.cr @@ -8,7 +8,7 @@ class DODB::Partition(V) < DODB::Indexer(V) property key_proc : Proc(V, String) getter storage_root : String - @storage : DODB::DataBase(V) + @storage : DODB::Storage(V) def initialize(@storage, @storage_root, @name, @key_proc) ::Dir.mkdir_p indexing_directory