From 031175a90a99a418dbd239cbf16a5e26dfaae83e Mon Sep 17 00:00:00 2001 From: Philippe PITTOLI Date: Wed, 8 May 2024 20:29:53 +0200 Subject: [PATCH] TODO: RAMOnlyTags + tests tests tests. --- src/cached.cr | 60 +++++++++++++++++++++++++++++++++++++--- src/dodb.cr | 24 ++++++++++++++++ src/dodb/index.cr | 27 ++++++++++++++++++ src/dodb/partition.cr | 64 +++++++++++++++++++++++++++++++++++++++++-- src/dodb/tags.cr | 52 +++++++++++++++++++++++++++++++++++ 5 files changed, 221 insertions(+), 6 deletions(-) diff --git a/src/cached.cr b/src/cached.cr index ad1f6c1..987410b 100644 --- a/src/cached.cr +++ b/src/cached.cr @@ -30,9 +30,9 @@ class DODB::CachedDataBase(V) < DODB::Storage(V) self.last_index = -1 end - # TODO: load the database in RAM at start-up + # Load the database in RAM at start-up. DODB::DataBase(V).new(@directory_name).each_with_index do |v, index| - puts "loading value #{v} at index #{index}" + puts "\rloading data from #{@directory_name} at index #{index}" self[index] = v end end @@ -98,8 +98,7 @@ class DODB::CachedDataBase(V) < DODB::Storage(V) begin ::File.delete file_path key - rescue - # FIXME: Only intercept “no such file" errors + rescue File::NotFoundError end remove_indexes key, value @@ -113,3 +112,56 @@ class DODB::CachedDataBase(V) < DODB::Storage(V) @data = Hash(Int32, V).new end end + +# `DODB::RAMOnlyDataBase` is a database without a file-system representation, +# enabling the use of DODB to store data which have the same lifetime as the application. +# Indexing (indexes, partitions, tags) will behave the same way. + +class DODB::RAMOnlyDataBase(V) < DODB::CachedDataBase(V) + # Initialization still uses a directory name and creates a few paths. + # This is an implementation detail to re-use code of `DODB::Storage` and to get the indexers to work. + def initialize(@directory_name : String) + Dir.mkdir_p data_path + Dir.mkdir_p locks_directory + self.last_index = -1 + end + + # WARNING: takes `[]?` and `[]` implementations from `CachedDataBase`. + # This will lead to errors in case the implementations change, be aware. + + def []=(index : Int32, value : V) + old_value = self.[index]? + + check_collisions! index, value, old_value + + # Removes any old indices or partitions pointing to a value about + # to be replaced. + if old_value + remove_indexes index, old_value + end + + write_partitions index, value + + if index > last_index + self.last_index = index + end + + @data[index] = value + end + + def delete(key : Int32) + value = self[key]? + + return if value.nil? + + remove_indexes key, value + + @data.delete key + value + end + + private def remove_data! + super + @data = Hash(Int32, V).new + end +end diff --git a/src/dodb.cr b/src/dodb.cr index 82dc5b8..fd28695 100644 --- a/src/dodb.cr +++ b/src/dodb.cr @@ -135,6 +135,18 @@ abstract class DODB::Storage(V) end end + def new_RAM_index(name : String, &block : Proc(V, String)) + RAMOnlyIndex(V).new(self, @directory_name, name, block).tap do |indexer| + @indexers << indexer + end + end + + def new_nilable_RAM_index(name : String, &block : Proc(V, String | DODB::NoIndex)) + RAMOnlyIndex(V).new(self, @directory_name, name, block).tap do |indexer| + @indexers << indexer + end + end + def get_index(name : String, key) index = @indexers.find &.name.==(name) @@ -155,6 +167,12 @@ abstract class DODB::Storage(V) end end + def new_RAM_partition(name : String, &block : Proc(V, String)) + RAMOnlyPartition(V).new(self, @directory_name, name, block).tap do |table| + @indexers << table + end + end + def get_partition(table_name : String, partition_name : String) partition = @indexers.find &.name.==(table_name) @@ -177,6 +195,12 @@ abstract class DODB::Storage(V) end end + def new_RAM_tags(name : String, &block : Proc(V, Array(String))) + RAMOnlyTags(V).new(self, @directory_name, name, block).tap do |tags| + @indexers << tags + end + end + def get_tags(name, key : String) tag = @indexers.find &.name.==(name) diff --git a/src/dodb/index.cr b/src/dodb/index.cr index c559532..2227a6e 100644 --- a/src/dodb/index.cr +++ b/src/dodb/index.cr @@ -200,3 +200,30 @@ class DODB::CachedIndex(V) < DODB::Index(V) end end end + +# `DODB::RAMOnlyIndex` enables the flexibility of indexes without a file-system representation. +# Absolute efficiency, exactly as easy to use as the other index implementations. + +class DODB::RAMOnlyIndex(V) < DODB::CachedIndex(V) + def index(key, value) + index_key = key_proc.call value + return if index_key.is_a? NoIndex + @data[index_key] = key.to_i + end + + def deindex(key, value) + index_key = key_proc.call value + return if index_key.is_a? NoIndex + @data.delete index_key + end + + # Get the key (ex: 343) for an entry in the DB. + # With a RAM only index, the key is necessarily stored in the hash. + def get_key(index : String) : Int32 + if k = @data[index]? + k + else + raise MissingEntry.new(@name, index) + end + end +end diff --git a/src/dodb/partition.cr b/src/dodb/partition.cr index 53d68e1..34cd920 100644 --- a/src/dodb/partition.cr +++ b/src/dodb/partition.cr @@ -130,7 +130,7 @@ class DODB::CachedPartition(V) < DODB::Partition(V) end end - def get(partition) + def get_with_indexes(partition) : Array(Tuple(V, Int32)) r_value = Array(Tuple(V, Int32)).new if keys = @data[partition]? @@ -148,6 +148,66 @@ class DODB::CachedPartition(V) < DODB::Partition(V) @data[partition] = r_value.map &.[1] end - r_value.map &.[0] + r_value + end + + def get(partition) : Array(V) + get_with_indexes(partition).map &.[0] + end + + def delete(partition, &matcher) + # Use `get_with_indexes` to retrieve data on-disk, if necessary. + new_partition = get_with_indexes(partition).map(&.[1]).select do |key| + item = @storage[key] + ! yield item + end + + @data[partition] = new_partition + + super(partition, matcher) + end +end + +class DODB::RAMOnlyPartition(V) < DODB::CachedPartition(V) + def index(key, value) + partition = key_proc.call value + + array = if v = @data[partition]? + v + else + Array(Int32).new + end + array << key.to_i + + @data[partition] = array + end + + def deindex(key, value) + partition = key_proc.call value + + if v = @data[partition]? + v.delete key.to_i + @data[partition] = v + end + end + + def get_with_indexes(partition) : Array(Tuple(V, Int32)) + r_value = Array(Tuple(V, Int32)).new + + if keys = @data[partition]? + keys.each do |data_key| + r_value << { @storage[data_key], data_key } + end + end + r_value + end + + def delete(partition, &matcher) + new_partition = @data[partition].select do |key| + item = @storage[key] + ! yield item + end + + @data[partition] = new_partition end end diff --git a/src/dodb/tags.cr b/src/dodb/tags.cr index 5ef7df0..6258774 100644 --- a/src/dodb/tags.cr +++ b/src/dodb/tags.cr @@ -172,3 +172,55 @@ class DODB::CachedTags(V) < DODB::Tags(V) r_value end end + +# TODO +class DODB::RAMOnlyTags(V) < DODB::CachedTags(V) + def index(key, value) + indices = key_proc.call value + + indices.each do |tag| + array = if v = @data[tag]? + v + else + Array(Int32).new + end + array << key.to_i + + @data[tag] = array + end + end + + def deindex(key, value) + super(key, value) + indices = key_proc.call value + + indices.each do |tag| + if v = @data[tag]? + v.delete key.to_i + @data[tag] = v + end + end + end + + def get_with_indice(tag : String) : Array(Tuple(V, Int32)) + r_value = Array(Tuple(V, Int32)).new + + if keys = @data[tag]? + keys.each do |data_key| + r_value << { @storage[data_key], data_key } + end + else + # Get the key from the database representation on the file-system. + tag_directory = indexing_directory tag + raise MissingEntry.new(@name, tag) unless Dir.exists? tag_directory + + Dir.each_child tag_directory do |child| + r_value << { @storage[get_key child], get_key child } + end + + @data[tag] = r_value.map &.[1] + end + + r_value + end +end