From 3f53034b5aadd4f264cc9f4ffc9d26bfad184431 Mon Sep 17 00:00:00 2001 From: Luka Vandervelden Date: Wed, 18 Dec 2019 03:43:09 +0100 Subject: [PATCH] DataBase(K, V) becomes DataBase(V). --- spec/test.cr | 111 ++++++++++++++++--------------- src/dodb.cr | 145 ++++++++++++++++++++++++++++++++--------- src/dodb/exceptions.cr | 2 +- src/dodb/tags.cr | 17 +++-- 4 files changed, 186 insertions(+), 89 deletions(-) diff --git a/spec/test.cr b/spec/test.cr index af9b566..bc917c2 100644 --- a/spec/test.cr +++ b/spec/test.cr @@ -13,11 +13,16 @@ class Ship def_clone property id : String - property class : String + property klass : String property name : String property tags : Array(String) - def initialize(@name, @class = "", @id = UUID.random.to_s, @tags = [] of String) + def initialize(@name, @klass = "", @id = UUID.random.to_s, @tags = [] of String) + end + + # Makes testing arrays of this class easier. + def <=>(other) + @name <=> other.name end # Common, reusable test data. @@ -54,7 +59,7 @@ class Ship # Equality is true if every property is identical. def ==(other) - @id == other.id && @class == other.class && @name == other.name && + @id == other.id && @klass == other.klass && @name == other.name && @tags == other.tags end end @@ -99,42 +104,42 @@ describe "DODB::DataBase" do db = DODB::SpecDataBase.new Ship.all_ships.each do |ship| - db[ship.id] = ship + db << ship end - Ship.all_ships.each do |ship| - db[ship.id].should eq(ship) - end + db.to_a.sort.should eq(Ship.all_ships.sort) end it "rewrite already stored data" do db = DODB::SpecDataBase.new ship = Ship.all_ships[0] - db[ship.id] = Ship.new "broken", id: ship.id - db[ship.id] = ship + key = db << ship - db[ship.id].should eq(ship) + db[key] = Ship.new "broken" + db[key] = ship + + db[key].should eq(ship) end it "properly remove data" do db = DODB::SpecDataBase.new Ship.all_ships.each do |ship| - db[ship.id] = ship + db << ship end Ship.all_ships.each do |ship| - db.delete ship.id + db.pop end - Ship.all_ships.each do |ship| + Ship.all_ships.each_with_index do |ship, i| # FIXME: Should it raise a particular exception? expect_raises DODB::MissingEntry do - db[ship.id] + db[i] end - db[ship.id]?.should be_nil + db[i]?.should be_nil end end end @@ -146,10 +151,10 @@ describe "DODB::DataBase" do db_ships_by_name = db.new_index "name", &.name Ship.all_ships.each do |ship| - db[ship.id] = ship + db << ship end - Ship.all_ships.each do |ship| + Ship.all_ships.each_with_index do |ship| db_ships_by_name.get?(ship.name).should eq(ship) end end @@ -159,14 +164,12 @@ describe "DODB::DataBase" do db_ships_by_name = db.new_index "name", &.name - some_ship = Ship.kisaragi - - db[some_ship.id] = some_ship + db << Ship.kisaragi # Should not be allowed to store an entry whose “name” field # already exists. expect_raises(DODB::IndexOverload) do - db["another id"] = some_ship + db << Ship.kisaragi end end @@ -176,11 +179,11 @@ describe "DODB::DataBase" do db_ships_by_name = db.new_index "name", &.name Ship.all_ships.each do |ship| - db[ship.id] = ship + db << ship end - Ship.all_ships.each do |ship| - db.delete ship.id + Ship.all_ships.each_with_index do |ship, i| + db.delete i end Ship.all_ships.each do |ship| @@ -193,18 +196,15 @@ describe "DODB::DataBase" do db_ships_by_name = db.new_index "name", &.name - some_ship = Ship.kisaragi - - db[some_ship.id] = some_ship + key = db << Ship.kisaragi # We give the old id to the new ship, to get it replaced in # the database. some_new_ship = Ship.all_ships[2].clone - some_new_ship.id = some_ship.id - db[some_new_ship.id] = some_new_ship + db[key] = some_new_ship - db[some_new_ship.id].should eq(some_new_ship) + db[key].should eq(some_new_ship) db_ships_by_name.get?(some_new_ship.name).should eq(some_new_ship) end @@ -214,24 +214,24 @@ describe "DODB::DataBase" do it "do basic partitioning" do db = DODB::SpecDataBase.new - db_ships_by_class = db.new_partition "class", &.class + db_ships_by_class = db.new_partition "class", &.klass Ship.all_ships.each do |ship| - db[ship.id] = ship + db << ship end Ship.all_ships.each do |ship| - db_ships_by_class.get(ship.class).should contain(ship) + db_ships_by_class.get(ship.klass).should contain(ship) end # We extract the possible classes to do test on them. - ship_classes = Ship.all_ships.map(&.class).uniq + ship_classes = Ship.all_ships.map(&.klass).uniq ship_classes.each do |klass| partition = db_ships_by_class.get klass # A partition on “class” should contain entries that all # share the same value of “class”. - partition.map(&.class.==(klass)).reduce { |a, b| + partition.map(&.klass.==(klass)).reduce { |a, b| a && b }.should be_true end @@ -245,7 +245,7 @@ describe "DODB::DataBase" do db_ships_by_tags = db.new_tags "tags", &.tags Ship.all_ships.each do |ship| - db[ship.id] = ship + db << ship end db_ships_by_tags.get("flagship").should eq([Ship.flagship]) @@ -266,13 +266,20 @@ describe "DODB::DataBase" do db_ships_by_tags = db.new_tags "tags", &.tags Ship.all_ships.each do |ship| - db[ship.id] = ship + db << ship end # Removing the “flagship” tag, brace for impact. - flagship = db_ships_by_tags.get("flagship")[0].clone + flagship, index = db_ships_by_tags.get_with_indices("flagship")[0] flagship.tags = [] of String - db[flagship.id] = flagship + db[index] = flagship + + + +# ship, index = db_ships_by_tags.update(tag: "flagship") do |ship, index| +# ship.tags = [] of String +# db[index] = ship +# end db_ships_by_tags.get("flagship").should eq([] of Ship) end @@ -283,18 +290,18 @@ describe "DODB::DataBase" do db = DODB::SpecDataBase.new db_ships_by_name = db.new_index "name", &.name - db_ships_by_class = db.new_partition "class", &.class + db_ships_by_class = db.new_partition "class", &.klass db_ships_by_tags = db.new_tags "tags", &.tags Ship.all_ships.each do |ship| - db[ship.id] = ship + db << ship end db.reindex_everything! Ship.all_ships.each do |ship| db_ships_by_name.get?(ship.name).should eq(ship) - db_ships_by_class.get(ship.class).should contain(ship) + db_ships_by_class.get(ship.klass).should contain(ship) end end @@ -305,7 +312,7 @@ describe "DODB::DataBase" do old_ships_by_class = old_db.new_partition "class", &.class_name PrimitiveShip.all_ships.each do |ship| - old_db[ship.id] = ship + old_db << ship end # At this point, the “old” DB is filled. Now we need to convert @@ -313,31 +320,31 @@ describe "DODB::DataBase" do new_db = DODB::SpecDataBase.new "-migration-target" - new_ships_by_class = new_db.new_partition "class", &.class - new_ships_by_tags = new_db.new_tags "tags", &.tags + new_ships_by_name = new_db.new_index "name", &.name + new_ships_by_class = new_db.new_partition "class", &.klass new_ships_by_tags = new_db.new_tags "tags", &.tags - old_db.each do |id, ship| + old_db.each_with_index do |ship, index| new_ship = Ship.new ship.name, - class: ship.class_name, + klass: ship.class_name, id: ship.id, tags: Array(String).new.tap { |tags| tags << "name ship" if ship.name == ship.class_name } - new_db[new_ship.id] = new_ship + new_db[index] = new_ship end # At this point, the conversion is done, so… we’re making a few # arbitrary tests on the new data. - old_db.each do |old_id, old_ship| - ship = new_db[old_id] + old_db.each_with_index do |old_ship, old_index| + ship = new_db[old_index] ship.id.should eq(old_ship.id) - ship.class.should eq(old_ship.class_name) + ship.klass.should eq(old_ship.class_name) - ship.tags.any?(&.==("name ship")).should be_true if ship.name == ship.class + ship.tags.any?(&.==("name ship")).should be_true if ship.name == ship.klass end end end diff --git a/src/dodb.cr b/src/dodb.cr index 8849590..db95daa 100644 --- a/src/dodb.cr +++ b/src/dodb.cr @@ -8,6 +8,35 @@ class DODB::DataBase(K, V) def initialize(@directory_name : String) Dir.mkdir_p data_path + + self.last_index = -1 + end + + private def index_file + "#{@directory_name}/last-index" + end + def last_index : Int32 + File.read(index_file).to_i + end + def last_index=(x : Int32) + file = File.open(index_file, "w") + + file << stringify_key(x) + + file.close + + x + rescue + raise Exception.new "could not update index file" + end + + def stringify_key(key : Int32) + # Negative numbers give strange results with Crystal’s printf. + if key >= 0 + "%010i" % key + else + key.to_s + end end ## @@ -51,48 +80,82 @@ class DODB::DataBase(K, V) partition.not_nil!.as(DODB::Tags).get name, key end - def []?(key : K) : V? + def <<(item : V) + index = last_index + 1 + + self[index] = item + + self.last_index = index + end + + def []?(key : Int32) : V? self[key] rescue MissingEntry # FIXME: Only rescue JSON and “no such file” errors. return nil end - def [](key : K) : V + def [](key : Int32) : V raise MissingEntry.new(key) unless ::File.exists? file_path key read file_path key end - def []=(key : K, value : V) - old_value = self.[key]? + def []=(index : Int32, value : V) + old_value = self.[index]? - check_collisions! key, value, old_value + check_collisions! index, value, old_value # Removes any old indices or partitions pointing to a value about # to be replaced. if old_value - remove_partitions key, old_value + remove_partitions index, old_value end # Avoids corruption in case the application crashes while writing. - file_path(key).tap do |path| + file_path(index).tap do |path| ::File.write "#{path}.new", value.to_json ::FileUtils.mv "#{path}.new", path end - write_partitions key, value + write_partitions index, value + + if index > last_index + self.last_index = index + end end - def check_collisions!(key : K, value : V, old_value : V?) - @indexers.each &.check!(key, value, old_value) + def check_collisions!(key : Int32, value : V, old_value : V?) + @indexers.each &.check!(stringify_key(key), value, old_value) end - def write_partitions(key : K, value : V) - @indexers.each &.index(key, value) + def write_partitions(key : Int32, value : V) + @indexers.each &.index(stringify_key(key), value) end - def delete(key : K) + def pop + index = last_index + + # Some entries may have been removed. We’ll skip over those. + # Not the most efficient if a large number of indices are empty. + while index >= 0 && self[index]?.nil? + index = index - 1 + end + + if index < 0 + return nil + end + + poped = self[index] + + self.delete index + + last_index = index - 1 + + poped + end + + def delete(key : Int32) value = self[key]? return if value.nil? @@ -108,14 +171,14 @@ class DODB::DataBase(K, V) value end - def remove_partitions(key : K, value : V) - @indexers.each &.deindex(key, value) + def remove_partitions(key : Int32, value : V) + @indexers.each &.deindex(stringify_key(key), value) end ## # CAUTION: Very slow. Try not to use. # Can be useful for making dumps or to restore a database, however. - def each + def each_with_index dirname = data_path Dir.each_child dirname do |child| next if child.match /^\./ @@ -129,20 +192,36 @@ class DODB::DataBase(K, V) next end - # FIXME: Will only work for String. :( - key = child.gsub /\.json$/, "" + key = child.gsub(/\.json$/, "").to_i - yield key, field + yield field, key + end + end + def each + each_with_index do |item, index| + yield item end end ## # CAUTION: Very slow. Try not to use. - def to_h - hash = ::Hash(K, V).new + def to_a + array = ::Array(V).new - each do |key, value| - hash[key] = value + each do |value| + array << value + end + + array + end + + ## + # CAUTION: Very slow. Try not to use. + def to_h + hash = ::Hash(Int32, V).new + + each_with_index do |element, index| + hash[index] = element end hash @@ -152,25 +231,29 @@ class DODB::DataBase(K, V) "#{@directory_name}/data" end - private def file_path(key : K) - "#{data_path}/#{key.to_s}.json" + private def file_path(key : Int32) + "#{data_path}/%010i.json" % key end private def read(file_path : String) V.from_json ::File.read file_path end + private def remove_data! + FileUtils.rm_rf data_path + Dir.mkdir_p data_path + end + # A very slow operation that removes all indices and then rewrites # them all. + # FIXME: Is this really useful in its current form? We should remove the + # index directories, not the indices based on our current (and + # possiblly different from what’s stored) data. def reindex_everything! old_data = to_h - old_data.each do |key, value| - self.delete key - end - - old_data.each do |key, value| - self[key] = value + old_data.each do |index, item| + self[index] = item end end end diff --git a/src/dodb/exceptions.cr b/src/dodb/exceptions.cr index 0628304..0b8ce63 100644 --- a/src/dodb/exceptions.cr +++ b/src/dodb/exceptions.cr @@ -1,7 +1,7 @@ class DODB::MissingEntry < Exception getter index : String? - getter key : String + getter key : String | Int32 def initialize(@index, @key) super "no entry in index '#{@index}' for key '#{@key}''" diff --git a/src/dodb/tags.cr b/src/dodb/tags.cr index 434c811..ca6ce55 100644 --- a/src/dodb/tags.cr +++ b/src/dodb/tags.cr @@ -14,7 +14,7 @@ class DODB::Tags(V) < DODB::Indexer(V) indices = key_proc.call value indices.each do |index| - symlink = get_tagged_entry_path(key.to_s, index) + symlink = get_tagged_entry_path(key, index) Dir.mkdir_p ::File.dirname symlink @@ -38,20 +38,27 @@ class DODB::Tags(V) < DODB::Indexer(V) return true # Tags don’t have collisions or overloads. end - def get(key) : Array(V) - r_value = Array(V).new + def get_with_indices(key) : Array(Tuple(V, Int32)) + r_value = Array(Tuple(V, Int32)).new partition_directory = "#{get_tag_directory}/#{key}" return r_value unless Dir.exists? partition_directory Dir.each_child partition_directory do |child| - r_value << V.from_json ::File.read "#{partition_directory}/#{child}" + r_value << { + V.from_json(::File.read("#{partition_directory}/#{child}")), + File.basename(child).gsub(/\.json$/, "").to_i + } end r_value end + def get(key) : Array(V) + get_with_indices(key).map &.[0] + end + private def get_tag_directory "#{@storage_root}/by_tags/by_#{@name}" end @@ -60,7 +67,7 @@ class DODB::Tags(V) < DODB::Indexer(V) "#{get_tag_directory}/#{index_key}/#{key}.json" end - private def get_data_symlink(key) + private def get_data_symlink(key : String) "../../../data/#{key}.json" end end