CachedDB OK

remotes/1708606157012549584/ramdb
Karchnu 2020-07-18 00:11:11 +02:00
parent ae2b4801c1
commit f8c0222593
6 changed files with 330 additions and 394 deletions

View File

@ -5,7 +5,7 @@ require "../src/dodb.cr"
require "./test-data.cr"
class DODB::SpecDataBase < DODB::DataBase::Cached(Ship)
class DODB::SpecDataBase < DODB::CachedDataBase(Ship)
def initialize(storage_ext = "", remove_previous_data = true)
storage_dir = "test-storage#{storage_ext}"
@ -62,6 +62,18 @@ describe "DODB::DataBase::Cached" do
end
end
it "preserves data on reopening" do
db1 = DODB::SpecDataBase.new
db1 << Ship.kisaragi
db1.to_a.size.should eq(1)
db2 = DODB::SpecDataBase.new remove_previous_data: false
db2 << Ship.mutsuki
db2.to_a.size.should eq(2)
end
it "iterates in normal and reversed order" do
db = DODB::SpecDataBase.new
@ -99,292 +111,291 @@ describe "DODB::DataBase::Cached" do
end
end
# describe "indices" do
# it "do basic indexing" do
# db = DODB::SpecDataBase.new
#
# db_ships_by_name = db.new_index "name", &.name
#
# Ship.all_ships.each do |ship|
# db << ship
# end
#
# Ship.all_ships.each_with_index do |ship|
# db_ships_by_name.get?(ship.name).should eq(ship)
# end
# end
# end
describe "indices" do
it "do basic indexing" do
db = DODB::SpecDataBase.new
# it "raise on index overload" do
# db = DODB::SpecDataBase.new
#
# db_ships_by_name = db.new_index "name", &.name
#
# db << Ship.kisaragi
#
# # Should not be allowed to store an entry whose “name” field
# # already exists.
# expect_raises(DODB::IndexOverload) do
# db << Ship.kisaragi
db_ships_by_name = db.new_index "name", &.name
Ship.all_ships.each do |ship|
db << ship
end
Ship.all_ships.each_with_index do |ship|
db_ships_by_name.get?(ship.name).should eq(ship)
end
end
it "raise on index overload" do
db = DODB::SpecDataBase.new
db_ships_by_name = db.new_index "name", &.name
db << Ship.kisaragi
# Should not be allowed to store an entry whose “name” field
# already exists.
expect_raises(DODB::IndexOverload) do
db << Ship.kisaragi
end
end
it "properly deindex" do
db = DODB::SpecDataBase.new
db_ships_by_name = db.new_index "name", &.name
Ship.all_ships.each do |ship|
db << ship
end
Ship.all_ships.each_with_index do |ship, i|
db.delete i
end
Ship.all_ships.each do |ship|
db_ships_by_name.get?(ship.name).should be_nil
end
end
it "properly reindex" do
db = DODB::SpecDataBase.new
db_ships_by_name = db.new_index "name", &.name
key = db << Ship.kisaragi
# We give the old id to the new ship, to get it replaced in
# the database.
some_new_ship = Ship.all_ships[2].clone
db[key] = some_new_ship
db[key].should eq(some_new_ship)
db_ships_by_name.get?(some_new_ship.name).should eq(some_new_ship)
end
it "properly updates" do
db = DODB::SpecDataBase.new
db_ships_by_name = db.new_index "name", &.name
Ship.all_ships.each do |ship|
db << ship
end
new_kisaragi = Ship.kisaragi.clone.tap do |s|
s.name = "Kisaragi Kai" # Dont think about it too much.
end
# Were changing an indexed value on purpose.
db_ships_by_name.update "Kisaragi", new_kisaragi
db_ships_by_name.get?("Kisaragi").should be_nil
db_ships_by_name.get?(new_kisaragi.name).should eq new_kisaragi
end
end
describe "partitions" do
it "do basic partitioning" do
db = DODB::SpecDataBase.new
db_ships_by_class = db.new_partition "class", &.klass
Ship.all_ships.each do |ship|
db << ship
end
Ship.all_ships.each do |ship|
db_ships_by_class.get(ship.klass).should contain(ship)
end
# We extract the possible classes to do test on them.
ship_classes = Ship.all_ships.map(&.klass).uniq
ship_classes.each do |klass|
partition = db_ships_by_class.get klass
# A partition on “class” should contain entries that all
# share the same value of “class”.
partition.map(&.klass.==(klass)).reduce { |a, b|
a && b
}.should be_true
end
db_ships_by_class.get("does-not-exist").should eq [] of Ship
end
it "removes select elements from partitions" do
db = DODB::SpecDataBase.new
db_ships_by_class = db.new_partition "class", &.klass
Ship.all_ships.each do |ship|
db << ship
end
db_ships_by_class.delete "Mutsuki", &.name.==("Kisaragi")
Ship.all_ships.map(&.klass).uniq.each do |klass|
partition = db_ships_by_class.get klass
partition.any?(&.name.==("Kisaragi")).should be_false
end
end
end
describe "tags" do
it "do basic tagging" do
db = DODB::SpecDataBase.new
db_ships_by_tags = db.new_tags "tags", &.tags
Ship.all_ships.each do |ship|
db << ship
end
db_ships_by_tags.get("flagship").should eq([Ship.flagship])
# All returned entries should have the requested tag.
db_ships_by_tags.get("name ship")
.map(&.tags.includes?("name ship"))
.reduce { |a, e| a && e }
.should be_true
# There shouldnt be one in our data about WWII Japanese warships…
db_ships_by_tags.get("starship").should eq([] of Ship)
end
it "properly removes tags" do
db = DODB::SpecDataBase.new
db_ships_by_tags = db.new_tags "tags", &.tags
Ship.all_ships.each do |ship|
db << ship
end
# Removing the “flagship” tag, brace for impact.
flagship, index = db_ships_by_tags.get_with_indices("flagship")[0]
flagship.tags = [] of String
db[index] = flagship
# ship, index = db_ships_by_tags.update(tag: "flagship") do |ship, index|
# ship.tags = [] of String
# db[index] = ship
# end
# end
#
# it "properly deindex" do
# db = DODB::SpecDataBase.new
#
# db_ships_by_name = db.new_index "name", &.name
#
# Ship.all_ships.each do |ship|
# db << ship
# end
#
# Ship.all_ships.each_with_index do |ship, i|
# db.delete i
# end
#
# Ship.all_ships.each do |ship|
# db_ships_by_name.get?(ship.name).should be_nil
# end
# end
#
# it "properly reindex" do
# db = DODB::SpecDataBase.new
#
# db_ships_by_name = db.new_index "name", &.name
#
# key = db << Ship.kisaragi
#
# # We give the old id to the new ship, to get it replaced in
# # the database.
# some_new_ship = Ship.all_ships[2].clone
#
# db[key] = some_new_ship
#
# db[key].should eq(some_new_ship)
#
# db_ships_by_name.get?(some_new_ship.name).should eq(some_new_ship)
# end
#
# it "properly updates" do
# db = DODB::SpecDataBase.new
#
# db_ships_by_name = db.new_index "name", &.name
#
# Ship.all_ships.each do |ship|
# db << ship
# end
#
# new_kisaragi = Ship.kisaragi.clone.tap do |s|
# s.name = "Kisaragi Kai" # Dont think about it too much.
# end
#
# # Were changing an indexed value on purpose.
# db_ships_by_name.update "Kisaragi", new_kisaragi
#
# db_ships_by_name.get?("Kisaragi").should be_nil
# db_ships_by_name.get?(new_kisaragi.name).should eq new_kisaragi
# end
# end
#
# describe "partitions" do
# it "do basic partitioning" do
# db = DODB::SpecDataBase.new
#
# db_ships_by_class = db.new_partition "class", &.klass
#
# Ship.all_ships.each do |ship|
# db << ship
# end
#
# Ship.all_ships.each do |ship|
# db_ships_by_class.get(ship.klass).should contain(ship)
# end
#
# # We extract the possible classes to do test on them.
# ship_classes = Ship.all_ships.map(&.klass).uniq
# ship_classes.each do |klass|
# partition = db_ships_by_class.get klass
#
# # A partition on “class” should contain entries that all
# # share the same value of “class”.
# partition.map(&.klass.==(klass)).reduce { |a, b|
# a && b
# }.should be_true
# end
#
# db_ships_by_class.get("does-not-exist").should eq [] of Ship
# end
#
# it "removes select elements from partitions" do
# db = DODB::SpecDataBase.new
#
# db_ships_by_class = db.new_partition "class", &.klass
#
# Ship.all_ships.each do |ship|
# db << ship
# end
#
# db_ships_by_class.delete "Mutsuki", &.name.==("Kisaragi")
#
# Ship.all_ships.map(&.klass).uniq.each do |klass|
# partition = db_ships_by_class.get klass
#
# partition.any?(&.name.==("Kisaragi")).should be_false
# end
# end
# end
#
# describe "tags" do
# it "do basic tagging" do
# db = DODB::SpecDataBase.new
#
# db_ships_by_tags = db.new_tags "tags", &.tags
#
# Ship.all_ships.each do |ship|
# db << ship
# end
#
# db_ships_by_tags.get("flagship").should eq([Ship.flagship])
#
# # All returned entries should have the requested tag.
# db_ships_by_tags.get("name ship")
# .map(&.tags.includes?("name ship"))
# .reduce { |a, e| a && e }
# .should be_true
#
# # There shouldnt be one in our data about WWII Japanese warships…
# db_ships_by_tags.get("starship").should eq([] of Ship)
# end
#
# it "properly removes tags" do
# db = DODB::SpecDataBase.new
#
# db_ships_by_tags = db.new_tags "tags", &.tags
#
# Ship.all_ships.each do |ship|
# db << ship
# end
#
# # Removing the “flagship” tag, brace for impact.
# flagship, index = db_ships_by_tags.get_with_indices("flagship")[0]
# flagship.tags = [] of String
# db[index] = flagship
#
#
#
## ship, index = db_ships_by_tags.update(tag: "flagship") do |ship, index|
## ship.tags = [] of String
## db[index] = ship
## end
#
# db_ships_by_tags.get("flagship").should eq([] of Ship)
# end
#
# it "gets items that have multiple tags" do
# db = DODB::SpecDataBase.new
#
# db_ships_by_tags = db.new_tags "tags", &.tags
#
# Ship.all_ships.each do |ship|
# db << ship
# end
#
# results = db_ships_by_tags.get(["flagship", "name ship"])
# results.should eq([Ship.yamato])
#
# results = db_ships_by_tags.get(["name ship", "flagship"])
# results.should eq([Ship.yamato])
#
# results = db_ships_by_tags.get(["flagship"])
# results.should eq([Ship.yamato])
# end
# end
#
# describe "atomic operations" do
# it "safe_get and safe_get?" do
# db = DODB::SpecDataBase.new
#
# db_ships_by_name = db.new_index "name", &.name
#
# Ship.all_ships.each do |ship|
# db << ship
# end
#
# Ship.all_ships.each do |ship|
# db_ships_by_name.safe_get ship.name do |results|
# results.should eq(ship)
# end
#
# db_ships_by_name.safe_get? ship.name do |results|
# results.should eq(ship)
# end
# end
# end
# end
#
# describe "tools" do
# it "rebuilds indexes" do
# db = DODB::SpecDataBase.new
#
# db_ships_by_name = db.new_index "name", &.name
# db_ships_by_class = db.new_partition "class", &.klass
# db_ships_by_tags = db.new_tags "tags", &.tags
#
# Ship.all_ships.each do |ship|
# db << ship
# end
#
# db.reindex_everything!
#
# Ship.all_ships.each do |ship|
# db_ships_by_name.get?(ship.name).should eq(ship)
# db_ships_by_class.get(ship.klass).should contain(ship)
# end
# end
#
# it "migrates properly" do
# ::FileUtils.rm_rf "test-storage-migration-origin"
# old_db = DODB::DataBase(PrimitiveShip).new "test-storage-migration-origin"
#
# old_ships_by_name = old_db.new_index "name", &.name
# old_ships_by_class = old_db.new_partition "class", &.class_name
#
# PrimitiveShip.all_ships.each do |ship|
# old_db << ship
# end
#
# # At this point, the “old” DB is filled. Now we need to convert
# # to the new DB.
#
# new_db = DODB::SpecDataBase.new "-migration-target"
#
# new_ships_by_name = new_db.new_index "name", &.name
# new_ships_by_class = new_db.new_partition "class", &.klass
# new_ships_by_tags = new_db.new_tags "tags", &.tags
#
# old_db.each_with_index do |ship, index|
# new_ship = Ship.new ship.name,
# klass: ship.class_name,
# id: ship.id,
# tags: Array(String).new.tap { |tags|
# tags << "name ship" if ship.name == ship.class_name
# }
#
# new_db[index] = new_ship
# end
#
# # At this point, the conversion is done, so… were making a few
# # arbitrary tests on the new data.
#
# old_db.each_with_index do |old_ship, old_index|
# ship = new_db[old_index]
#
# ship.id.should eq(old_ship.id)
# ship.klass.should eq(old_ship.class_name)
#
# ship.tags.any?(&.==("name ship")).should be_true if ship.name == ship.klass
# end
# end
# end
db_ships_by_tags.get("flagship").should eq([] of Ship)
end
it "gets items that have multiple tags" do
db = DODB::SpecDataBase.new
db_ships_by_tags = db.new_tags "tags", &.tags
Ship.all_ships.each do |ship|
db << ship
end
results = db_ships_by_tags.get(["flagship", "name ship"])
results.should eq([Ship.yamato])
results = db_ships_by_tags.get(["name ship", "flagship"])
results.should eq([Ship.yamato])
results = db_ships_by_tags.get(["flagship"])
results.should eq([Ship.yamato])
end
end
describe "atomic operations" do
it "safe_get and safe_get?" do
db = DODB::SpecDataBase.new
db_ships_by_name = db.new_index "name", &.name
Ship.all_ships.each do |ship|
db << ship
end
Ship.all_ships.each do |ship|
db_ships_by_name.safe_get ship.name do |results|
results.should eq(ship)
end
db_ships_by_name.safe_get? ship.name do |results|
results.should eq(ship)
end
end
end
end
describe "tools" do
it "rebuilds indexes" do
db = DODB::SpecDataBase.new
db_ships_by_name = db.new_index "name", &.name
db_ships_by_class = db.new_partition "class", &.klass
db_ships_by_tags = db.new_tags "tags", &.tags
Ship.all_ships.each do |ship|
db << ship
end
db.reindex_everything!
Ship.all_ships.each do |ship|
db_ships_by_name.get?(ship.name).should eq(ship)
db_ships_by_class.get(ship.klass).should contain(ship)
end
end
it "migrates properly" do
::FileUtils.rm_rf "test-storage-migration-origin"
old_db = DODB::DataBase(PrimitiveShip).new "test-storage-migration-origin"
old_ships_by_name = old_db.new_index "name", &.name
old_ships_by_class = old_db.new_partition "class", &.class_name
PrimitiveShip.all_ships.each do |ship|
old_db << ship
end
# At this point, the “old” DB is filled. Now we need to convert
# to the new DB.
new_db = DODB::SpecDataBase.new "-migration-target"
new_ships_by_name = new_db.new_index "name", &.name
new_ships_by_class = new_db.new_partition "class", &.klass
new_ships_by_tags = new_db.new_tags "tags", &.tags
old_db.each_with_index do |ship, index|
new_ship = Ship.new ship.name,
klass: ship.class_name,
id: ship.id,
tags: Array(String).new.tap { |tags|
tags << "name ship" if ship.name == ship.class_name
}
new_db[index] = new_ship
end
# At this point, the conversion is done, so… were making a few
# arbitrary tests on the new data.
old_db.each_with_index do |old_ship, old_index|
ship = new_db[old_index]
ship.id.should eq(old_ship.id)
ship.klass.should eq(old_ship.class_name)
ship.tags.any?(&.==("name ship")).should be_true if ship.name == ship.klass
end
end
end
end

View File

@ -1,94 +1,9 @@
require "spec"
require "file_utils"
require "json"
require "uuid"
require "../src/*"
require "../src/dodb.cr"
require "./test-data.cr"
# FIXME: Split the test data in separate files. We dont care about those here.
class Ship
include JSON::Serializable
def_clone
property id : String
property klass : String
property name : String
property tags : Array(String)
def initialize(@name, @klass = "<unknown>", @id = UUID.random.to_s, @tags = [] of String)
end
# Makes testing arrays of this class easier.
def <=>(other)
@name <=> other.name
end
# Common, reusable test data.
# Those data can be indexed, partitioned or tagged on different parameters,
# and can easily be extended.
class_getter kisaragi = Ship.new("Kisaragi", "Mutsuki")
class_getter mutsuki = Ship.new("Mutsuki", "Mutsuki", tags: ["name ship"])
class_getter yayoi = Ship.new("Yayoi", "Mutsuki")
class_getter destroyers = [
@@mutsuki,
@@kisaragi,
@@yayoi,
Ship.new("Uzuki", "Mutsuki"),
Ship.new("Satsuki", "Mutsuki"),
Ship.new("Shiratsuyu", "Shiratsuyu", tags: ["name ship"]),
Ship.new("Murasame", "Shiratsuyu"),
Ship.new("Yuudachi", "Shiratsuyu")
]
class_getter yamato =
Ship.new("Yamato", "Yamato", tags: ["name ship", "flagship"])
class_getter flagship : Ship = yamato
class_getter battleships = [
@@yamato,
Ship.new("Kongou", "Kongou", tags: ["name ship"]),
Ship.new("Haruna", "Kongou"),
Ship.new("Kirishima", "Kongou"),
Ship.new("Hiei" , "Kongou"),
Ship.new("Musashi", "Yamato"),
Ship.new("Shinano", "Yamato")
]
class_getter all_ships : Array(Ship) = @@destroyers + @@battleships
# Equality is true if every property is identical.
def ==(other)
@id == other.id && @klass == other.klass && @name == other.name &&
@tags == other.tags
end
end
# This will be used for migration testing, but basically its a variant of
# the class above, a few extra fields, a few missing ones.
class PrimitiveShip
include JSON::Serializable
property id : String
property name : String
property wooden : Bool = false # Will be removed.
property class_name : String # Will be renamed
property flagship : Bool = false # Will be moved to tags.
def initialize(@name, @class_name = "<unknown>", @id = UUID.random.to_s, @flagship = false)
end
class_getter kamikaze =
PrimitiveShip.new("Kamikaze", "Kamikaze")
class_getter asakaze =
PrimitiveShip.new("Asakaze", "Kamikaze")
class_getter all_ships : Array(PrimitiveShip) = [
@@kamikaze,
@@asakaze
]
end
class DODB::SpecDataBase < DODB::DataBase(Ship)
def initialize(storage_ext = "", remove_previous_data = true)

View File

@ -16,7 +16,7 @@ class Hash(K,V)
end
end
class DODB::DataBase::Cached(V)
class DODB::CachedDataBase(V) < DODB::Storage(V)
@indexers = [] of Indexer(V)
property data = Hash(Int32, V).new
@ -31,6 +31,10 @@ class DODB::DataBase::Cached(V)
end
# TODO: load the database in RAM at start-up
DODB::DataBase(V).new(@directory_name).each_with_index do |v, index|
puts "loading value #{v} at index #{index}"
self[index] = v
end
end
# Data should be written on disk AND in RAM.
@ -38,7 +42,6 @@ class DODB::DataBase::Cached(V)
index = last_index + 1
self[index] = item
self.last_index = index
@data[self.last_index] = item
self.last_index
end
@ -282,6 +285,7 @@ class DODB::DataBase::Cached(V)
private def remove_data!
FileUtils.rm_rf data_path
Dir.mkdir_p data_path
@data = Hash(Int32, V).new
end
private def remove_indexing!

View File

@ -3,7 +3,29 @@ require "json"
require "./dodb/*"
class DODB::DataBase(V)
abstract class DODB::Storage(V)
def request_lock(name)
r = -1
file_path = get_lock_file_path name
file_perms = 0o644
flags = LibC::O_EXCL | LibC::O_CREAT
while (r = LibC.open file_path, flags, file_perms) == -1
sleep 1.milliseconds
end
LibC.close r
end
def release_lock(name)
File.delete get_lock_file_path name
end
abstract def [](key : Int32)
abstract def <<(item : V)
abstract def delete(key : Int32)
end
class DODB::DataBase(V) < DODB::Storage(V)
@indexers = [] of Indexer(V)
def initialize(@directory_name : String)
@ -44,22 +66,6 @@ class DODB::DataBase(V)
end
end
def request_lock(name)
r = -1
file_path = get_lock_file_path name
file_perms = 0o644
flags = LibC::O_EXCL | LibC::O_CREAT
while (r = LibC.open file_path, flags, file_perms) == -1
sleep 1.milliseconds
end
LibC.close r
end
def release_lock(name)
File.delete get_lock_file_path name
end
##
# name is the name that will be used on the file system.
def new_partition(name : String, &block : Proc(V, String))

View File

@ -9,7 +9,7 @@ class DODB::Index(V) < DODB::Indexer(V)
property key_proc : Proc(V, String)
getter storage_root : String
@storage : DODB::DataBase(V)
@storage : DODB::Storage(V)
def initialize(@storage, @storage_root, @name, @key_proc)
Dir.mkdir_p indexing_directory

View File

@ -8,7 +8,7 @@ class DODB::Partition(V) < DODB::Indexer(V)
property key_proc : Proc(V, String)
getter storage_root : String
@storage : DODB::DataBase(V)
@storage : DODB::Storage(V)
def initialize(@storage, @storage_root, @name, @key_proc)
::Dir.mkdir_p indexing_directory