forked from Baguette/dodb.cr
DataBase(K, V) becomes DataBase(V).
parent
c4030d4179
commit
3f53034b5a
111
spec/test.cr
111
spec/test.cr
|
|
@ -13,11 +13,16 @@ class Ship
|
|||
def_clone
|
||||
|
||||
property id : String
|
||||
property class : String
|
||||
property klass : String
|
||||
property name : String
|
||||
property tags : Array(String)
|
||||
|
||||
def initialize(@name, @class = "<unknown>", @id = UUID.random.to_s, @tags = [] of String)
|
||||
def initialize(@name, @klass = "<unknown>", @id = UUID.random.to_s, @tags = [] of String)
|
||||
end
|
||||
|
||||
# Makes testing arrays of this class easier.
|
||||
def <=>(other)
|
||||
@name <=> other.name
|
||||
end
|
||||
|
||||
# Common, reusable test data.
|
||||
|
|
@ -54,7 +59,7 @@ class Ship
|
|||
|
||||
# Equality is true if every property is identical.
|
||||
def ==(other)
|
||||
@id == other.id && @class == other.class && @name == other.name &&
|
||||
@id == other.id && @klass == other.klass && @name == other.name &&
|
||||
@tags == other.tags
|
||||
end
|
||||
end
|
||||
|
|
@ -99,42 +104,42 @@ describe "DODB::DataBase" do
|
|||
db = DODB::SpecDataBase.new
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
db[ship.id] = ship
|
||||
db << ship
|
||||
end
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
db[ship.id].should eq(ship)
|
||||
end
|
||||
db.to_a.sort.should eq(Ship.all_ships.sort)
|
||||
end
|
||||
|
||||
it "rewrite already stored data" do
|
||||
db = DODB::SpecDataBase.new
|
||||
ship = Ship.all_ships[0]
|
||||
|
||||
db[ship.id] = Ship.new "broken", id: ship.id
|
||||
db[ship.id] = ship
|
||||
key = db << ship
|
||||
|
||||
db[ship.id].should eq(ship)
|
||||
db[key] = Ship.new "broken"
|
||||
db[key] = ship
|
||||
|
||||
db[key].should eq(ship)
|
||||
end
|
||||
|
||||
it "properly remove data" do
|
||||
db = DODB::SpecDataBase.new
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
db[ship.id] = ship
|
||||
db << ship
|
||||
end
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
db.delete ship.id
|
||||
db.pop
|
||||
end
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
Ship.all_ships.each_with_index do |ship, i|
|
||||
# FIXME: Should it raise a particular exception?
|
||||
expect_raises DODB::MissingEntry do
|
||||
db[ship.id]
|
||||
db[i]
|
||||
end
|
||||
|
||||
db[ship.id]?.should be_nil
|
||||
db[i]?.should be_nil
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -146,10 +151,10 @@ describe "DODB::DataBase" do
|
|||
db_ships_by_name = db.new_index "name", &.name
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
db[ship.id] = ship
|
||||
db << ship
|
||||
end
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
Ship.all_ships.each_with_index do |ship|
|
||||
db_ships_by_name.get?(ship.name).should eq(ship)
|
||||
end
|
||||
end
|
||||
|
|
@ -159,14 +164,12 @@ describe "DODB::DataBase" do
|
|||
|
||||
db_ships_by_name = db.new_index "name", &.name
|
||||
|
||||
some_ship = Ship.kisaragi
|
||||
|
||||
db[some_ship.id] = some_ship
|
||||
db << Ship.kisaragi
|
||||
|
||||
# Should not be allowed to store an entry whose “name” field
|
||||
# already exists.
|
||||
expect_raises(DODB::IndexOverload) do
|
||||
db["another id"] = some_ship
|
||||
db << Ship.kisaragi
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -176,11 +179,11 @@ describe "DODB::DataBase" do
|
|||
db_ships_by_name = db.new_index "name", &.name
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
db[ship.id] = ship
|
||||
db << ship
|
||||
end
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
db.delete ship.id
|
||||
Ship.all_ships.each_with_index do |ship, i|
|
||||
db.delete i
|
||||
end
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
|
|
@ -193,18 +196,15 @@ describe "DODB::DataBase" do
|
|||
|
||||
db_ships_by_name = db.new_index "name", &.name
|
||||
|
||||
some_ship = Ship.kisaragi
|
||||
|
||||
db[some_ship.id] = some_ship
|
||||
key = db << Ship.kisaragi
|
||||
|
||||
# We give the old id to the new ship, to get it replaced in
|
||||
# the database.
|
||||
some_new_ship = Ship.all_ships[2].clone
|
||||
some_new_ship.id = some_ship.id
|
||||
|
||||
db[some_new_ship.id] = some_new_ship
|
||||
db[key] = some_new_ship
|
||||
|
||||
db[some_new_ship.id].should eq(some_new_ship)
|
||||
db[key].should eq(some_new_ship)
|
||||
|
||||
db_ships_by_name.get?(some_new_ship.name).should eq(some_new_ship)
|
||||
end
|
||||
|
|
@ -214,24 +214,24 @@ describe "DODB::DataBase" do
|
|||
it "do basic partitioning" do
|
||||
db = DODB::SpecDataBase.new
|
||||
|
||||
db_ships_by_class = db.new_partition "class", &.class
|
||||
db_ships_by_class = db.new_partition "class", &.klass
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
db[ship.id] = ship
|
||||
db << ship
|
||||
end
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
db_ships_by_class.get(ship.class).should contain(ship)
|
||||
db_ships_by_class.get(ship.klass).should contain(ship)
|
||||
end
|
||||
|
||||
# We extract the possible classes to do test on them.
|
||||
ship_classes = Ship.all_ships.map(&.class).uniq
|
||||
ship_classes = Ship.all_ships.map(&.klass).uniq
|
||||
ship_classes.each do |klass|
|
||||
partition = db_ships_by_class.get klass
|
||||
|
||||
# A partition on “class” should contain entries that all
|
||||
# share the same value of “class”.
|
||||
partition.map(&.class.==(klass)).reduce { |a, b|
|
||||
partition.map(&.klass.==(klass)).reduce { |a, b|
|
||||
a && b
|
||||
}.should be_true
|
||||
end
|
||||
|
|
@ -245,7 +245,7 @@ describe "DODB::DataBase" do
|
|||
db_ships_by_tags = db.new_tags "tags", &.tags
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
db[ship.id] = ship
|
||||
db << ship
|
||||
end
|
||||
|
||||
db_ships_by_tags.get("flagship").should eq([Ship.flagship])
|
||||
|
|
@ -266,13 +266,20 @@ describe "DODB::DataBase" do
|
|||
db_ships_by_tags = db.new_tags "tags", &.tags
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
db[ship.id] = ship
|
||||
db << ship
|
||||
end
|
||||
|
||||
# Removing the “flagship” tag, brace for impact.
|
||||
flagship = db_ships_by_tags.get("flagship")[0].clone
|
||||
flagship, index = db_ships_by_tags.get_with_indices("flagship")[0]
|
||||
flagship.tags = [] of String
|
||||
db[flagship.id] = flagship
|
||||
db[index] = flagship
|
||||
|
||||
|
||||
|
||||
# ship, index = db_ships_by_tags.update(tag: "flagship") do |ship, index|
|
||||
# ship.tags = [] of String
|
||||
# db[index] = ship
|
||||
# end
|
||||
|
||||
db_ships_by_tags.get("flagship").should eq([] of Ship)
|
||||
end
|
||||
|
|
@ -283,18 +290,18 @@ describe "DODB::DataBase" do
|
|||
db = DODB::SpecDataBase.new
|
||||
|
||||
db_ships_by_name = db.new_index "name", &.name
|
||||
db_ships_by_class = db.new_partition "class", &.class
|
||||
db_ships_by_class = db.new_partition "class", &.klass
|
||||
db_ships_by_tags = db.new_tags "tags", &.tags
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
db[ship.id] = ship
|
||||
db << ship
|
||||
end
|
||||
|
||||
db.reindex_everything!
|
||||
|
||||
Ship.all_ships.each do |ship|
|
||||
db_ships_by_name.get?(ship.name).should eq(ship)
|
||||
db_ships_by_class.get(ship.class).should contain(ship)
|
||||
db_ships_by_class.get(ship.klass).should contain(ship)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -305,7 +312,7 @@ describe "DODB::DataBase" do
|
|||
old_ships_by_class = old_db.new_partition "class", &.class_name
|
||||
|
||||
PrimitiveShip.all_ships.each do |ship|
|
||||
old_db[ship.id] = ship
|
||||
old_db << ship
|
||||
end
|
||||
|
||||
# At this point, the “old” DB is filled. Now we need to convert
|
||||
|
|
@ -313,31 +320,31 @@ describe "DODB::DataBase" do
|
|||
|
||||
new_db = DODB::SpecDataBase.new "-migration-target"
|
||||
|
||||
new_ships_by_class = new_db.new_partition "class", &.class
|
||||
new_ships_by_tags = new_db.new_tags "tags", &.tags
|
||||
new_ships_by_name = new_db.new_index "name", &.name
|
||||
new_ships_by_class = new_db.new_partition "class", &.klass
|
||||
new_ships_by_tags = new_db.new_tags "tags", &.tags
|
||||
|
||||
old_db.each do |id, ship|
|
||||
old_db.each_with_index do |ship, index|
|
||||
new_ship = Ship.new ship.name,
|
||||
class: ship.class_name,
|
||||
klass: ship.class_name,
|
||||
id: ship.id,
|
||||
tags: Array(String).new.tap { |tags|
|
||||
tags << "name ship" if ship.name == ship.class_name
|
||||
}
|
||||
|
||||
new_db[new_ship.id] = new_ship
|
||||
new_db[index] = new_ship
|
||||
end
|
||||
|
||||
# At this point, the conversion is done, so… we’re making a few
|
||||
# arbitrary tests on the new data.
|
||||
|
||||
old_db.each do |old_id, old_ship|
|
||||
ship = new_db[old_id]
|
||||
old_db.each_with_index do |old_ship, old_index|
|
||||
ship = new_db[old_index]
|
||||
|
||||
ship.id.should eq(old_ship.id)
|
||||
ship.class.should eq(old_ship.class_name)
|
||||
ship.klass.should eq(old_ship.class_name)
|
||||
|
||||
ship.tags.any?(&.==("name ship")).should be_true if ship.name == ship.class
|
||||
ship.tags.any?(&.==("name ship")).should be_true if ship.name == ship.klass
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
145
src/dodb.cr
145
src/dodb.cr
|
|
@ -8,6 +8,35 @@ class DODB::DataBase(K, V)
|
|||
|
||||
def initialize(@directory_name : String)
|
||||
Dir.mkdir_p data_path
|
||||
|
||||
self.last_index = -1
|
||||
end
|
||||
|
||||
private def index_file
|
||||
"#{@directory_name}/last-index"
|
||||
end
|
||||
def last_index : Int32
|
||||
File.read(index_file).to_i
|
||||
end
|
||||
def last_index=(x : Int32)
|
||||
file = File.open(index_file, "w")
|
||||
|
||||
file << stringify_key(x)
|
||||
|
||||
file.close
|
||||
|
||||
x
|
||||
rescue
|
||||
raise Exception.new "could not update index file"
|
||||
end
|
||||
|
||||
def stringify_key(key : Int32)
|
||||
# Negative numbers give strange results with Crystal’s printf.
|
||||
if key >= 0
|
||||
"%010i" % key
|
||||
else
|
||||
key.to_s
|
||||
end
|
||||
end
|
||||
|
||||
##
|
||||
|
|
@ -51,48 +80,82 @@ class DODB::DataBase(K, V)
|
|||
partition.not_nil!.as(DODB::Tags).get name, key
|
||||
end
|
||||
|
||||
def []?(key : K) : V?
|
||||
def <<(item : V)
|
||||
index = last_index + 1
|
||||
|
||||
self[index] = item
|
||||
|
||||
self.last_index = index
|
||||
end
|
||||
|
||||
def []?(key : Int32) : V?
|
||||
self[key]
|
||||
rescue MissingEntry
|
||||
# FIXME: Only rescue JSON and “no such file” errors.
|
||||
return nil
|
||||
end
|
||||
|
||||
def [](key : K) : V
|
||||
def [](key : Int32) : V
|
||||
raise MissingEntry.new(key) unless ::File.exists? file_path key
|
||||
|
||||
read file_path key
|
||||
end
|
||||
|
||||
def []=(key : K, value : V)
|
||||
old_value = self.[key]?
|
||||
def []=(index : Int32, value : V)
|
||||
old_value = self.[index]?
|
||||
|
||||
check_collisions! key, value, old_value
|
||||
check_collisions! index, value, old_value
|
||||
|
||||
# Removes any old indices or partitions pointing to a value about
|
||||
# to be replaced.
|
||||
if old_value
|
||||
remove_partitions key, old_value
|
||||
remove_partitions index, old_value
|
||||
end
|
||||
|
||||
# Avoids corruption in case the application crashes while writing.
|
||||
file_path(key).tap do |path|
|
||||
file_path(index).tap do |path|
|
||||
::File.write "#{path}.new", value.to_json
|
||||
::FileUtils.mv "#{path}.new", path
|
||||
end
|
||||
|
||||
write_partitions key, value
|
||||
write_partitions index, value
|
||||
|
||||
if index > last_index
|
||||
self.last_index = index
|
||||
end
|
||||
end
|
||||
|
||||
def check_collisions!(key : K, value : V, old_value : V?)
|
||||
@indexers.each &.check!(key, value, old_value)
|
||||
def check_collisions!(key : Int32, value : V, old_value : V?)
|
||||
@indexers.each &.check!(stringify_key(key), value, old_value)
|
||||
end
|
||||
|
||||
def write_partitions(key : K, value : V)
|
||||
@indexers.each &.index(key, value)
|
||||
def write_partitions(key : Int32, value : V)
|
||||
@indexers.each &.index(stringify_key(key), value)
|
||||
end
|
||||
|
||||
def delete(key : K)
|
||||
def pop
|
||||
index = last_index
|
||||
|
||||
# Some entries may have been removed. We’ll skip over those.
|
||||
# Not the most efficient if a large number of indices are empty.
|
||||
while index >= 0 && self[index]?.nil?
|
||||
index = index - 1
|
||||
end
|
||||
|
||||
if index < 0
|
||||
return nil
|
||||
end
|
||||
|
||||
poped = self[index]
|
||||
|
||||
self.delete index
|
||||
|
||||
last_index = index - 1
|
||||
|
||||
poped
|
||||
end
|
||||
|
||||
def delete(key : Int32)
|
||||
value = self[key]?
|
||||
|
||||
return if value.nil?
|
||||
|
|
@ -108,14 +171,14 @@ class DODB::DataBase(K, V)
|
|||
value
|
||||
end
|
||||
|
||||
def remove_partitions(key : K, value : V)
|
||||
@indexers.each &.deindex(key, value)
|
||||
def remove_partitions(key : Int32, value : V)
|
||||
@indexers.each &.deindex(stringify_key(key), value)
|
||||
end
|
||||
|
||||
##
|
||||
# CAUTION: Very slow. Try not to use.
|
||||
# Can be useful for making dumps or to restore a database, however.
|
||||
def each
|
||||
def each_with_index
|
||||
dirname = data_path
|
||||
Dir.each_child dirname do |child|
|
||||
next if child.match /^\./
|
||||
|
|
@ -129,20 +192,36 @@ class DODB::DataBase(K, V)
|
|||
next
|
||||
end
|
||||
|
||||
# FIXME: Will only work for String. :(
|
||||
key = child.gsub /\.json$/, ""
|
||||
key = child.gsub(/\.json$/, "").to_i
|
||||
|
||||
yield key, field
|
||||
yield field, key
|
||||
end
|
||||
end
|
||||
def each
|
||||
each_with_index do |item, index|
|
||||
yield item
|
||||
end
|
||||
end
|
||||
|
||||
##
|
||||
# CAUTION: Very slow. Try not to use.
|
||||
def to_h
|
||||
hash = ::Hash(K, V).new
|
||||
def to_a
|
||||
array = ::Array(V).new
|
||||
|
||||
each do |key, value|
|
||||
hash[key] = value
|
||||
each do |value|
|
||||
array << value
|
||||
end
|
||||
|
||||
array
|
||||
end
|
||||
|
||||
##
|
||||
# CAUTION: Very slow. Try not to use.
|
||||
def to_h
|
||||
hash = ::Hash(Int32, V).new
|
||||
|
||||
each_with_index do |element, index|
|
||||
hash[index] = element
|
||||
end
|
||||
|
||||
hash
|
||||
|
|
@ -152,25 +231,29 @@ class DODB::DataBase(K, V)
|
|||
"#{@directory_name}/data"
|
||||
end
|
||||
|
||||
private def file_path(key : K)
|
||||
"#{data_path}/#{key.to_s}.json"
|
||||
private def file_path(key : Int32)
|
||||
"#{data_path}/%010i.json" % key
|
||||
end
|
||||
|
||||
private def read(file_path : String)
|
||||
V.from_json ::File.read file_path
|
||||
end
|
||||
|
||||
private def remove_data!
|
||||
FileUtils.rm_rf data_path
|
||||
Dir.mkdir_p data_path
|
||||
end
|
||||
|
||||
# A very slow operation that removes all indices and then rewrites
|
||||
# them all.
|
||||
# FIXME: Is this really useful in its current form? We should remove the
|
||||
# index directories, not the indices based on our current (and
|
||||
# possiblly different from what’s stored) data.
|
||||
def reindex_everything!
|
||||
old_data = to_h
|
||||
|
||||
old_data.each do |key, value|
|
||||
self.delete key
|
||||
end
|
||||
|
||||
old_data.each do |key, value|
|
||||
self[key] = value
|
||||
old_data.each do |index, item|
|
||||
self[index] = item
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
|
||||
class DODB::MissingEntry < Exception
|
||||
getter index : String?
|
||||
getter key : String
|
||||
getter key : String | Int32
|
||||
|
||||
def initialize(@index, @key)
|
||||
super "no entry in index '#{@index}' for key '#{@key}''"
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ class DODB::Tags(V) < DODB::Indexer(V)
|
|||
indices = key_proc.call value
|
||||
|
||||
indices.each do |index|
|
||||
symlink = get_tagged_entry_path(key.to_s, index)
|
||||
symlink = get_tagged_entry_path(key, index)
|
||||
|
||||
Dir.mkdir_p ::File.dirname symlink
|
||||
|
||||
|
|
@ -38,20 +38,27 @@ class DODB::Tags(V) < DODB::Indexer(V)
|
|||
return true # Tags don’t have collisions or overloads.
|
||||
end
|
||||
|
||||
def get(key) : Array(V)
|
||||
r_value = Array(V).new
|
||||
def get_with_indices(key) : Array(Tuple(V, Int32))
|
||||
r_value = Array(Tuple(V, Int32)).new
|
||||
|
||||
partition_directory = "#{get_tag_directory}/#{key}"
|
||||
|
||||
return r_value unless Dir.exists? partition_directory
|
||||
|
||||
Dir.each_child partition_directory do |child|
|
||||
r_value << V.from_json ::File.read "#{partition_directory}/#{child}"
|
||||
r_value << {
|
||||
V.from_json(::File.read("#{partition_directory}/#{child}")),
|
||||
File.basename(child).gsub(/\.json$/, "").to_i
|
||||
}
|
||||
end
|
||||
|
||||
r_value
|
||||
end
|
||||
|
||||
def get(key) : Array(V)
|
||||
get_with_indices(key).map &.[0]
|
||||
end
|
||||
|
||||
private def get_tag_directory
|
||||
"#{@storage_root}/by_tags/by_#{@name}"
|
||||
end
|
||||
|
|
@ -60,7 +67,7 @@ class DODB::Tags(V) < DODB::Indexer(V)
|
|||
"#{get_tag_directory}/#{index_key}/#{key}.json"
|
||||
end
|
||||
|
||||
private def get_data_symlink(key)
|
||||
private def get_data_symlink(key : String)
|
||||
"../../../data/#{key}.json"
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Reference in New Issue