From fd73bc437ff0b691cb38762a99dc7d0086691afc Mon Sep 17 00:00:00 2001 From: Karchnu Date: Sat, 18 Jul 2020 00:37:29 +0200 Subject: [PATCH] Factoring. --- src/cached.cr | 206 +--------------------------------- src/dodb.cr | 303 +++++++++++++++++++++++++------------------------- 2 files changed, 154 insertions(+), 355 deletions(-) diff --git a/src/cached.cr b/src/cached.cr index d20fe69..85eb8a6 100644 --- a/src/cached.cr +++ b/src/cached.cr @@ -37,14 +37,6 @@ class DODB::CachedDataBase(V) < DODB::Storage(V) end end - # Data should be written on disk AND in RAM. - def <<(item : V) - index = last_index + 1 - self[index] = item - self.last_index = index - self.last_index - end - # Getting data from the hash in RAM. def []?(key : Int32) : V? @data[key] @@ -85,8 +77,7 @@ class DODB::CachedDataBase(V) < DODB::Storage(V) end ## - # CAUTION: Very slow. Try not to use. - # Can be useful for making dumps or to restore a database, however. + # Can be useful for making dumps or to restore a database. def each_with_index(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil) i = -1 # do not trust key to be the right index (reversed ? @data.reverse : @data).each do |index, v| @@ -97,149 +88,6 @@ class DODB::CachedDataBase(V) < DODB::Storage(V) yield v, index end end - def each(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil) - each_with_index( - reversed: reversed, - start_offset: start_offset, - end_offset: end_offset - ) do |item, index| - yield item - end - end - - def to_a(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil) - res = Array(V).new - each(reversed, start_offset, end_offset) do |v| - res << v - end - res - end - - def to_h(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil) - res = Hash(Int32, V).new - - (reversed ? @data.reverse : @data).each do |index, v| - next if start_offset > index - break unless end_offset.nil? || index < end_offset - - res[index] = v - end - - res - end - - private def index_file - "#{@directory_name}/last-index" - end - def last_index : Int32 - File.read(index_file).to_i - end - def last_index=(x : Int32) - file = File.open(index_file, "w") - - file << x.to_s - - file.close - - x - rescue - raise Exception.new "could not update index file" - end - - def stringify_key(key : Int32) - # Negative numbers give strange results with Crystal’s printf. - if key >= 0 - "%010i" % key - else - key.to_s - end - end - - def request_lock(name) - r = -1 - file_path = get_lock_file_path name - file_perms = 0o644 - - flags = LibC::O_EXCL | LibC::O_CREAT - while (r = LibC.open file_path, flags, file_perms) == -1 - sleep 1.milliseconds - end - - LibC.close r - end - def release_lock(name) - File.delete get_lock_file_path name - end - - ## - # name is the name that will be used on the file system. - def new_partition(name : String, &block : Proc(V, String)) - Partition(V).new(self, @directory_name, name, block).tap do |table| - @indexers << table - end - end - - ## - # name is the name that will be used on the file system. - def new_index(name : String, &block : Proc(V, String)) - Index(V).new(self, @directory_name, name, block).tap do |indexer| - @indexers << indexer - end - end - - def new_tags(name : String, &block : Proc(V, Array(String))) - Tags(V).new(@directory_name, name, block).tap do |tags| - @indexers << tags - end - end - - def get_index(name : String, key) - index = @indexers.find &.name.==(name) - - index.not_nil!.as(DODB::Index).get key - end - - def get_partition(table_name : String, partition_name : String) - partition = @indexers.find &.name.==(table_name) - - partition.not_nil!.as(DODB::Partition).get partition_name - end - - def get_tags(name, key : String) - partition = @indexers.find &.name.==(name) - - partition.not_nil!.as(DODB::Tags).get name, key - end - - def check_collisions!(key : Int32, value : V, old_value : V?) - @indexers.each &.check!(stringify_key(key), value, old_value) - end - - def write_partitions(key : Int32, value : V) - @indexers.each &.index(stringify_key(key), value) - end - - def pop - index = last_index - - # Some entries may have been removed. We’ll skip over those. - # Not the most efficient if a large number of indices are empty. - while index >= 0 && self[index]?.nil? - index = index - 1 - end - - if index < 0 - return nil - end - - poped = self[index] - - self.delete index - - last_index = index - 1 - - poped - end def delete(key : Int32) value = self[key]? @@ -257,56 +105,4 @@ class DODB::CachedDataBase(V) < DODB::Storage(V) @data.delete key value end - - def remove_partitions(key : Int32, value : V) - @indexers.each &.deindex(stringify_key(key), value) - end - - private def data_path - "#{@directory_name}/data" - end - - private def file_path(key : Int32) - "#{data_path}/%010i.json" % key - end - - private def locks_directory : String - "#{@directory_name}/locks" - end - - private def get_lock_file_path(name : String) - "#{locks_directory}/#{name}.lock" - end - - private def read(file_path : String) - V.from_json ::File.read file_path - end - - private def remove_data! - FileUtils.rm_rf data_path - Dir.mkdir_p data_path - @data = Hash(Int32, V).new - end - - private def remove_indexing! - @indexers.each do |indexer| - FileUtils.rm_rf indexer.indexing_directory - end - end - - # A very slow operation that removes all indices and then rewrites - # them all. - # FIXME: Is this really useful in its current form? We should remove the - # index directories, not the indices based on our current (and - # possiblly different from what’s stored) data. - def reindex_everything! - old_data = to_h - - remove_indexing! - remove_data! - - old_data.each do |index, item| - self[index] = item - end - end end diff --git a/src/dodb.cr b/src/dodb.cr index c8f87ce..9e69dee 100644 --- a/src/dodb.cr +++ b/src/dodb.cr @@ -4,6 +4,11 @@ require "json" require "./dodb/*" abstract class DODB::Storage(V) + property directory_name : String + + def initialize(@directory_name : String) + end + def request_lock(name) r = -1 file_path = get_lock_file_path name @@ -20,25 +25,6 @@ abstract class DODB::Storage(V) File.delete get_lock_file_path name end - abstract def [](key : Int32) - abstract def <<(item : V) - abstract def delete(key : Int32) -end - -class DODB::DataBase(V) < DODB::Storage(V) - @indexers = [] of Indexer(V) - - def initialize(@directory_name : String) - Dir.mkdir_p data_path - Dir.mkdir_p locks_directory - - begin - self.last_index - rescue - self.last_index = -1 - end - end - private def index_file "#{@directory_name}/last-index" end @@ -66,14 +52,54 @@ class DODB::DataBase(V) < DODB::Storage(V) end end - ## - # name is the name that will be used on the file system. - def new_partition(name : String, &block : Proc(V, String)) - Partition(V).new(self, @directory_name, name, block).tap do |table| - @indexers << table + def <<(item : V) + index = last_index + 1 + self[index] = item + self.last_index = index + end + + def each(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil) + each_with_index( + reversed: reversed, + start_offset: start_offset, + end_offset: end_offset + ) do |item, index| + yield item end end + ## + # CAUTION: Very slow. Try not to use. + def to_a(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil) + array = ::Array(V).new + + each( + reversed: reversed, + start_offset: start_offset, + end_offset: end_offset + ) do |value| + array << value + end + + array + end + + ## + # CAUTION: Very slow. Try not to use. + def to_h(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil) + hash = ::Hash(Int32, V).new + + each_with_index( + reversed: reversed, + start_offset: start_offset, + end_offset: end_offset + ) do |element, index| + hash[index] = element + end + + hash + end + ## # name is the name that will be used on the file system. def new_index(name : String, &block : Proc(V, String)) @@ -82,6 +108,14 @@ class DODB::DataBase(V) < DODB::Storage(V) end end + ## + # name is the name that will be used on the file system. + def new_partition(name : String, &block : Proc(V, String)) + Partition(V).new(self, @directory_name, name, block).tap do |table| + @indexers << table + end + end + def new_tags(name : String, &block : Proc(V, Array(String))) Tags(V).new(@directory_name, name, block).tap do |tags| @indexers << tags @@ -106,12 +140,103 @@ class DODB::DataBase(V) < DODB::Storage(V) partition.not_nil!.as(DODB::Tags).get name, key end - def <<(item : V) - index = last_index + 1 + def check_collisions!(key : Int32, value : V, old_value : V?) + @indexers.each &.check!(stringify_key(key), value, old_value) + end - self[index] = item + def write_partitions(key : Int32, value : V) + @indexers.each &.index(stringify_key(key), value) + end - self.last_index = index + def pop + index = last_index + + # Some entries may have been removed. We’ll skip over those. + # Not the most efficient if a large number of indices are empty. + while index >= 0 && self[index]?.nil? + index = index - 1 + end + + if index < 0 + return nil + end + + poped = self[index] + + self.delete index + + last_index = index - 1 + + poped + end + + private def data_path + "#{@directory_name}/data" + end + + private def file_path(key : Int32) + "#{data_path}/%010i.json" % key + end + + private def locks_directory : String + "#{@directory_name}/locks" + end + + private def get_lock_file_path(name : String) + "#{locks_directory}/#{name}.lock" + end + + private def read(file_path : String) + V.from_json ::File.read file_path + end + + private def remove_data! + FileUtils.rm_rf data_path + Dir.mkdir_p data_path + end + + private def remove_indexing! + @indexers.each do |indexer| + FileUtils.rm_rf indexer.indexing_directory + end + end + + # A very slow operation that removes all indices and then rewrites + # them all. + # FIXME: Is this really useful in its current form? We should remove the + # index directories, not the indices based on our current (and + # possiblly different from what’s stored) data. + def reindex_everything! + old_data = to_h + + remove_indexing! + remove_data! + + old_data.each do |index, item| + self[index] = item + end + end + + def remove_partitions(key : Int32, value : V) + @indexers.each &.deindex(stringify_key(key), value) + end + + abstract def [](key : Int32) + abstract def delete(key : Int32) +end + +class DODB::DataBase(V) < DODB::Storage(V) + @indexers = [] of Indexer(V) + + def initialize(@directory_name : String) + Dir.mkdir_p data_path + Dir.mkdir_p locks_directory + + begin + self.last_index + rescue + self.last_index = -1 + end end def []?(key : Int32) : V? @@ -151,36 +276,6 @@ class DODB::DataBase(V) < DODB::Storage(V) end end - def check_collisions!(key : Int32, value : V, old_value : V?) - @indexers.each &.check!(stringify_key(key), value, old_value) - end - - def write_partitions(key : Int32, value : V) - @indexers.each &.index(stringify_key(key), value) - end - - def pop - index = last_index - - # Some entries may have been removed. We’ll skip over those. - # Not the most efficient if a large number of indices are empty. - while index >= 0 && self[index]?.nil? - index = index - 1 - end - - if index < 0 - return nil - end - - poped = self[index] - - self.delete index - - last_index = index - 1 - - poped - end - def delete(key : Int32) value = self[key]? @@ -197,10 +292,6 @@ class DODB::DataBase(V) < DODB::Storage(V) value end - def remove_partitions(key : Int32, value : V) - @indexers.each &.deindex(stringify_key(key), value) - end - private def each_key(reversed = false) start = 0 _end = last_index @@ -252,94 +343,6 @@ class DODB::DataBase(V) < DODB::Storage(V) yield field, key end end - def each(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil) - each_with_index( - reversed: reversed, - start_offset: start_offset, - end_offset: end_offset - ) do |item, index| - yield item - end - end - - ## - # CAUTION: Very slow. Try not to use. - def to_a(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil) - array = ::Array(V).new - - each( - reversed: reversed, - start_offset: start_offset, - end_offset: end_offset - ) do |value| - array << value - end - - array - end - - ## - # CAUTION: Very slow. Try not to use. - def to_h(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil) - hash = ::Hash(Int32, V).new - - each_with_index( - reversed: reversed, - start_offset: start_offset, - end_offset: end_offset - ) do |element, index| - hash[index] = element - end - - hash - end - - private def data_path - "#{@directory_name}/data" - end - - private def file_path(key : Int32) - "#{data_path}/%010i.json" % key - end - - private def locks_directory : String - "#{@directory_name}/locks" - end - - private def get_lock_file_path(name : String) - "#{locks_directory}/#{name}.lock" - end - - private def read(file_path : String) - V.from_json ::File.read file_path - end - - private def remove_data! - FileUtils.rm_rf data_path - Dir.mkdir_p data_path - end - - private def remove_indexing! - @indexers.each do |indexer| - FileUtils.rm_rf indexer.indexing_directory - end - end - - # A very slow operation that removes all indices and then rewrites - # them all. - # FIXME: Is this really useful in its current form? We should remove the - # index directories, not the indices based on our current (and - # possiblly different from what’s stored) data. - def reindex_everything! - old_data = to_h - - remove_indexing! - remove_data! - - old_data.each do |index, item| - self[index] = item - end - end end require "./cached.cr"