From 8ba15394b786d0dea5740732af0f522b68d43053 Mon Sep 17 00:00:00 2001 From: Luka Vandervelden Date: Wed, 11 Dec 2019 18:10:09 +0100 Subject: [PATCH] File split, grooming. --- src/fsdb.cr | 202 +----------------------------------------- src/fsdb/index.cr | 73 +++++++++++++++ src/fsdb/indexer.cr | 8 ++ src/fsdb/partition.cr | 67 ++++++++++++++ src/fsdb/tags.cr | 67 ++++++++++++++ 5 files changed, 216 insertions(+), 201 deletions(-) create mode 100644 src/fsdb/index.cr create mode 100644 src/fsdb/indexer.cr create mode 100644 src/fsdb/partition.cr create mode 100644 src/fsdb/tags.cr diff --git a/src/fsdb.cr b/src/fsdb.cr index dd61ebd..098881d 100644 --- a/src/fsdb.cr +++ b/src/fsdb.cr @@ -1,207 +1,7 @@ - require "file_utils" require "json" -abstract class FSDB::Indexer(V) - abstract def index (key : String, value : V) - abstract def deindex (key : String, value : V) - abstract def check! (key : String, value : V, old_value : V?) - abstract def name : String -end - -class FSDB::Partition(V) < FSDB::Indexer(V) - property name : String - property key_proc : Proc(V, String) - getter storage_root : String - - def initialize(@storage_root, @name, @key_proc) - ::Dir.mkdir_p get_partition_directory - end - - def check!(key, value, old_value) - return true # Partitions don’t have collisions or overloads. - end - - def index(key, value) - partition = key_proc.call value - - symlink = get_partition_symlink(partition, key) - - Dir.mkdir_p ::File.dirname symlink - - # FIXME: Should not happen anymore. Should we remove this? - ::File.delete symlink if ::File.exists? symlink - - ::File.symlink get_data_symlink(key), symlink - end - - def deindex(key, value) - partition = key_proc.call value - - symlink = get_partition_symlink(partition, key) - - ::File.delete symlink - end - - def get(partition) - r_value = Array(V).new - - partition_directory = get_partition_directory partition - Dir.each_child partition_directory do |child| - r_value << V.from_json ::File.read "#{partition_directory}/#{child}" - end - - r_value - end - - private def get_partition_directory - "#{@storage_root}/partitions/by_#{@name}" - end - - private def get_partition_directory(partition) - "#{get_partition_directory}/#{partition}" - end - - private def get_partition_symlink(partition : String, key : String) - "#{get_partition_directory partition}/#{key}.json" - end - - private def get_data_symlink(key : String) - "../../../data/#{key}.json" - end -end - -class FSDB::Index(V) < FSDB::Indexer(V) - property name : String - property key_proc : Proc(V, String) - getter storage_root : String - - def initialize(@storage_root, @name, @key_proc) - Dir.mkdir_p dir_path_indices - end - - def check!(key, value, old_value) - index_key = key_proc.call value - - symlink = file_path_index index_key.to_s - - # FIXME: Check it’s not pointing to “old_value”, if any, before raising. - if ::File.exists? symlink - if old_value - old_key = key_proc.call old_value - return if symlink == file_path_index old_key.to_s - end - - raise IndexOverload.new "Index '#{@name}' is overloaded for key '#{key}'" - end - end - - def index(key, value) - index_key = key_proc.call value - - symlink = file_path_index index_key - - Dir.mkdir_p ::File.dirname symlink - - # FIXME: Now that this is done in check!, can we remove it? - if ::File.exists? symlink - raise Exception.new "symlink already exists: #{symlink}" - end - - ::File.symlink get_data_symlink_index(key), symlink - end - - def deindex(key, value) - index_key = key_proc.call value - - symlink = file_path_index index_key - - ::File.delete symlink - end - - def get(index : String) : V? - V.from_json ::File.read "#{file_path_index index}" - end - - private def dir_path_indices - "#{@storage_root}/indices/by_#{@name}" - end - - private def file_path_index(index_key : String) - "#{dir_path_indices}/#{index_key}.json" - end - - private def get_data_symlink_index(key : String) - "../../data/#{key}.json" - end -end - -class FSDB::Tags(V) < FSDB::Indexer(V) - property name : String - property key_proc : Proc(V, Array(String)) - getter storage_root : String - - def initialize(@storage_root, @name, @key_proc) - ::Dir.mkdir_p get_tag_directory - end - - def index(key, value) - indices = key_proc.call value - - indices.each do |index| - symlink = get_tagged_entry_path(key.to_s, index) - - Dir.mkdir_p ::File.dirname symlink - - ::File.delete symlink if ::File.exists? symlink - - ::File.symlink get_data_symlink(key), symlink - end - end - - def deindex(key, value) - indices = key_proc.call value - - indices.each do |index_key| - symlink = get_tagged_entry_path(key, index_key) - - ::File.delete symlink - end - end - - def check!(key, value, old_value) - return true # Tags don’t have collisions or overloads. - end - - def get(name, key) : Array(V) - r_value = Array(V).new - - partition_directory = "#{get_tag_directory}/#{key}" - - return r_value unless Dir.exists? partition_directory - - Dir.each_child partition_directory do |child| - r_value << V.from_json ::File.read "#{partition_directory}/#{child}" - end - - r_value - end - - private def get_tag_directory - "#{@storage_root}/by_tags/by_#{@name}" - end - - private def get_tagged_entry_path(key : String, index_key : String) - "#{get_tag_directory}/#{index_key}/#{key}.json" - end - - private def get_data_symlink(key) - "../../../data/#{key}.json" - end -end - -class FSDB::IndexOverload < Exception -end +require "./fsdb/*" class FSDB::DataBase(K, V) @indexers = [] of Indexer(V) diff --git a/src/fsdb/index.cr b/src/fsdb/index.cr new file mode 100644 index 0000000..e02ffcb --- /dev/null +++ b/src/fsdb/index.cr @@ -0,0 +1,73 @@ +require "file_utils" +require "json" + +require "./indexer.cr" + +class FSDB::Index(V) < FSDB::Indexer(V) + property name : String + property key_proc : Proc(V, String) + getter storage_root : String + + def initialize(@storage_root, @name, @key_proc) + Dir.mkdir_p dir_path_indices + end + + def check!(key, value, old_value) + index_key = key_proc.call value + + symlink = file_path_index index_key.to_s + + # FIXME: Check it’s not pointing to “old_value”, if any, before raising. + if ::File.exists? symlink + if old_value + old_key = key_proc.call old_value + return if symlink == file_path_index old_key.to_s + end + + raise IndexOverload.new "Index '#{@name}' is overloaded for key '#{key}'" + end + end + + def index(key, value) + index_key = key_proc.call value + + symlink = file_path_index index_key + + Dir.mkdir_p ::File.dirname symlink + + # FIXME: Now that this is done in check!, can we remove it? + if ::File.exists? symlink + raise Exception.new "symlink already exists: #{symlink}" + end + + ::File.symlink get_data_symlink_index(key), symlink + end + + def deindex(key, value) + index_key = key_proc.call value + + symlink = file_path_index index_key + + ::File.delete symlink + end + + def get(index : String) : V? + V.from_json ::File.read "#{file_path_index index}" + end + + private def dir_path_indices + "#{@storage_root}/indices/by_#{@name}" + end + + private def file_path_index(index_key : String) + "#{dir_path_indices}/#{index_key}.json" + end + + private def get_data_symlink_index(key : String) + "../../data/#{key}.json" + end +end + +class FSDB::IndexOverload < Exception +end + diff --git a/src/fsdb/indexer.cr b/src/fsdb/indexer.cr new file mode 100644 index 0000000..c6fa90b --- /dev/null +++ b/src/fsdb/indexer.cr @@ -0,0 +1,8 @@ + +abstract class FSDB::Indexer(V) + abstract def index (key : String, value : V) + abstract def deindex (key : String, value : V) + abstract def check! (key : String, value : V, old_value : V?) + abstract def name : String +end + diff --git a/src/fsdb/partition.cr b/src/fsdb/partition.cr new file mode 100644 index 0000000..345c195 --- /dev/null +++ b/src/fsdb/partition.cr @@ -0,0 +1,67 @@ +require "file_utils" +require "json" + +require "./indexer.cr" + +class FSDB::Partition(V) < FSDB::Indexer(V) + property name : String + property key_proc : Proc(V, String) + getter storage_root : String + + def initialize(@storage_root, @name, @key_proc) + ::Dir.mkdir_p get_partition_directory + end + + def check!(key, value, old_value) + return true # Partitions don’t have collisions or overloads. + end + + def index(key, value) + partition = key_proc.call value + + symlink = get_partition_symlink(partition, key) + + Dir.mkdir_p ::File.dirname symlink + + # FIXME: Should not happen anymore. Should we remove this? + ::File.delete symlink if ::File.exists? symlink + + ::File.symlink get_data_symlink(key), symlink + end + + def deindex(key, value) + partition = key_proc.call value + + symlink = get_partition_symlink(partition, key) + + ::File.delete symlink + end + + def get(partition) + r_value = Array(V).new + + partition_directory = get_partition_directory partition + Dir.each_child partition_directory do |child| + r_value << V.from_json ::File.read "#{partition_directory}/#{child}" + end + + r_value + end + + private def get_partition_directory + "#{@storage_root}/partitions/by_#{@name}" + end + + private def get_partition_directory(partition) + "#{get_partition_directory}/#{partition}" + end + + private def get_partition_symlink(partition : String, key : String) + "#{get_partition_directory partition}/#{key}.json" + end + + private def get_data_symlink(key : String) + "../../../data/#{key}.json" + end +end + diff --git a/src/fsdb/tags.cr b/src/fsdb/tags.cr new file mode 100644 index 0000000..9ca153e --- /dev/null +++ b/src/fsdb/tags.cr @@ -0,0 +1,67 @@ +require "file_utils" +require "json" + +class FSDB::Tags(V) < FSDB::Indexer(V) + property name : String + property key_proc : Proc(V, Array(String)) + getter storage_root : String + + def initialize(@storage_root, @name, @key_proc) + ::Dir.mkdir_p get_tag_directory + end + + def index(key, value) + indices = key_proc.call value + + indices.each do |index| + symlink = get_tagged_entry_path(key.to_s, index) + + Dir.mkdir_p ::File.dirname symlink + + ::File.delete symlink if ::File.exists? symlink + + ::File.symlink get_data_symlink(key), symlink + end + end + + def deindex(key, value) + indices = key_proc.call value + + indices.each do |index_key| + symlink = get_tagged_entry_path(key, index_key) + + ::File.delete symlink + end + end + + def check!(key, value, old_value) + return true # Tags don’t have collisions or overloads. + end + + def get(name, key) : Array(V) + r_value = Array(V).new + + partition_directory = "#{get_tag_directory}/#{key}" + + return r_value unless Dir.exists? partition_directory + + Dir.each_child partition_directory do |child| + r_value << V.from_json ::File.read "#{partition_directory}/#{child}" + end + + r_value + end + + private def get_tag_directory + "#{@storage_root}/by_tags/by_#{@name}" + end + + private def get_tagged_entry_path(key : String, index_key : String) + "#{get_tag_directory}/#{index_key}/#{key}.json" + end + + private def get_data_symlink(key) + "../../../data/#{key}.json" + end +end +