TODO.md

@@ -5,11 +5,6 @@
 - RAMOnly: do not read/write the `last_entry` file.
 - Use the `#unsafe_add` function when possible.
 
-# Memory management
-
-- When a value is removed, the related partitions (and tags) may be empty, leaving both an empty array
-  in memory and a directory on the file-system. Should they be removed?
-
 # Documentation
 
 - Write the API documentation.

spec/test-ships.cr

@@ -16,8 +16,6 @@ describe "DODB::DataBase" do
 			end
 
 			db.to_a.sort.should eq(Ship.all_ships.sort)
-
-			db.rm_storage_dir
 		end
 
 		it "rewrite already stored data" do
@@ -30,8 +28,6 @@ describe "DODB::DataBase" do
 			db[key] = ship
 
 			db[key].should eq(ship)
-
-			db.rm_storage_dir
 		end
 
 		it "properly remove data" do
@@ -53,8 +49,6 @@ describe "DODB::DataBase" do
 
 				db[i]?.should be_nil
 			end
-
-			db.rm_storage_dir
 		end
 
 		it "preserves data on reopening" do
@@ -67,9 +61,6 @@ describe "DODB::DataBase" do
 			db2 << Ship.mutsuki
 
 			db1.to_a.size.should eq(2)
-
-			db1.rm_storage_dir
-			db2.rm_storage_dir
 		end
 
 		it "iterates in normal and reversed order" do
@@ -90,8 +81,6 @@ describe "DODB::DataBase" do
 
 			# Actual reversal is tested here.
 			db.to_a(reversed: true).should eq db.to_a.reverse
-
-			db.rm_storage_dir
 		end
 
 		it "respects the provided offsets if any" do
@@ -108,8 +97,6 @@ describe "DODB::DataBase" do
 			db.to_a(offset: 0, limit: 3).should eq [
 				Ship.mutsuki, Ship.kisaragi, Ship.yayoi
 			]
-
-			db.rm_storage_dir
 		end
 	end
 
@@ -126,8 +113,6 @@ describe "DODB::DataBase" do
 			Ship.all_ships.each_with_index do |ship|
 				db_ships_by_name.get?(ship.name).should eq(ship)
 			end
-
-			db.rm_storage_dir
 		end
 
 		it "raise on index overload" do
@@ -142,8 +127,6 @@ describe "DODB::DataBase" do
 			expect_raises(DODB::IndexOverload) do
 				db << Ship.kisaragi
 			end
-
-			db.rm_storage_dir
 		end
 
 		it "properly deindex" do
@@ -162,8 +145,6 @@ describe "DODB::DataBase" do
 			Ship.all_ships.each do |ship|
 				db_ships_by_name.get?(ship.name).should be_nil
 			end
-
-			db.rm_storage_dir
 		end
 
 		it "properly reindex" do
@@ -182,8 +163,6 @@ describe "DODB::DataBase" do
 			db[key].should eq(some_new_ship)
 
 			db_ships_by_name.get?(some_new_ship.name).should eq(some_new_ship)
-
-			db.rm_storage_dir
 		end
 
 		it "properly updates" do
@@ -204,8 +183,6 @@ describe "DODB::DataBase" do
 
 			db_ships_by_name.get?("Kisaragi").should be_nil
 			db_ships_by_name.get?(new_kisaragi.name).should eq new_kisaragi
-
-			db.rm_storage_dir
 		end
 	end
 
@@ -236,8 +213,6 @@ describe "DODB::DataBase" do
 			end
 
 			db_ships_by_class.get?("does-not-exist").should be_nil
-
-			db.rm_storage_dir
 		end
 
 		it "removes select elements from partitions" do
@@ -256,8 +231,6 @@ describe "DODB::DataBase" do
 
 				partition.any?(&.name.==("Kisaragi")).should be_false
 			end
-
-			db.rm_storage_dir
 		end
 	end
 
@@ -281,8 +254,6 @@ describe "DODB::DataBase" do
 
 			# There shouldn’t be one in our data about WWII Japanese warships…
 			db_ships_by_tags.get?("starship").should be_nil
-
-			db.rm_storage_dir
 		end
 
 		it "properly removes tags" do
@@ -307,8 +278,6 @@ describe "DODB::DataBase" do
 #			end
 
 			db_ships_by_tags.get("flagship").should eq([] of Ship)
-
-			db.rm_storage_dir
 		end
 
 		it "gets items that have multiple tags" do
@@ -328,8 +297,6 @@ describe "DODB::DataBase" do
 
 			results = db_ships_by_tags.get(["flagship"])
 			results.should eq([Ship.yamato])
-
-			db.rm_storage_dir
 		end
 	end
 
@@ -352,8 +319,6 @@ describe "DODB::DataBase" do
 					results.should eq(ship)
 				end
 			end
-
-			db.rm_storage_dir
 		end
 	end
 
@@ -375,12 +340,11 @@ describe "DODB::DataBase" do
 				db_ships_by_name.get?(ship.name).should eq(ship)
 				db_ships_by_class.get(ship.klass).should contain(ship)
 			end
-
-			db.rm_storage_dir
 		end
 
 		it "migrates properly" do
-			old_db = DODB::SpecDataBase(PrimitiveShip).new "-migration-origin"
+			::FileUtils.rm_rf "test-storage-migration-origin"
+			old_db = DODB::DataBase(PrimitiveShip).new "test-storage-migration-origin"
 
 			old_ships_by_name  = old_db.new_index     "name", &.name
 			old_ships_by_class = old_db.new_partition "class", &.class_name
@@ -420,9 +384,6 @@ describe "DODB::DataBase" do
 
 				ship.tags.any?(&.==("name ship")).should be_true if ship.name == ship.klass
 			end
-
-			old_db.rm_storage_dir
-			new_db.rm_storage_dir
 		end
 	end
 
@@ -449,8 +410,6 @@ describe "DODB::DataBase" do
 			dump = db.to_a
 
 			dump.size.should eq fork_count * entries_per_fork
-
-			db.rm_storage_dir
 		end
 
 		it "works for updating values" do
@@ -489,8 +448,6 @@ describe "DODB::DataBase" do
 					entry.tags.should eq ["updated"]
 				end
 			end
-
-			db.rm_storage_dir
 		end
 
 		it "does parallel-safe updates" do
@@ -516,8 +473,6 @@ describe "DODB::DataBase" do
 			processes.each &.wait
 
 			db_entries_by_name.get("test").klass.should eq((fork_count * entries_per_fork).to_s)
-
-			db.rm_storage_dir
 		end
 	end
 end

src/dodb/index.cr

@@ -6,10 +6,6 @@ require "./indexer.cr"
 # Basic indexes for 1-to-1 relations.
 # Uncached version.
 #
-# ```
-# cars_by_name = car_database.new_uncached_index "name", &.name
-# ```
-#
 # This index provides a file-system representation, enabling the administrators to
 # select a value based on its index. The following example presents an index named "id"
 # with some data indexed by an UUID attribute.
@@ -31,27 +27,16 @@ require "./indexer.cr"
 # NOTE: see `CachedIndex` for a cached version, faster for retrieval.
 # NOTE: for fast operations without fs representation, see `RAMOnlyIndex`.
 class DODB::Index(V) < DODB::Indexer(V)
-	# Name of the index, such as *id* or *color* for example.
-	# This is an arbitrary value, mostly to create the index directory.
-	#
-	# NOTE: used for internal operations.
 	property name         : String
-
+	property key_proc     : Proc(V, String | NoIndex) | Proc(V, String)
-	# Procedure to retrieve the index attribute from the value, used for **internal operations**.
-	property key_proc     : Proc(V, String | NoIndex)
-
-	# Root database directory, used for **internal operations**.
 	getter   storage_root : String
 
-	# Reference to the database instance, used for **internal operations**.
 	@storage : DODB::Storage(V)
 
 	# To create an index from a database, use `DODB::Storage#new_index` to create
 	# a cached index, `DODB::Storage#new_uncached_index` for an uncached index or
 	#  `DODB::Storage#new_RAM_index` for a RAM-only index.
-	#
+	def initialize(@storage, @storage_root, @name, @key_proc)
-	# WARNING: this is an internal operation, do not instanciate an index by hand.
-	def initialize(@storage : DODB::Storage(V), @storage_root : String, @name : String, @key_proc : Proc(V, String | NoIndex))
 		Dir.mkdir_p indexing_directory
 	end
 
@@ -99,7 +84,7 @@ class DODB::Index(V) < DODB::Indexer(V)
 		end
 	end
 
-	# Gets the key (ex: 343) for an entry in the DB from an indexed value, used for **internal operations**.
+	# Gets the key (ex: 343) for an entry in the DB from an indexed value.
 	#
 	# Reads the link in `db/indices/by_#{name}/<index>`.
 	#
@@ -174,7 +159,7 @@ class DODB::Index(V) < DODB::Indexer(V)
 		yield nil
 	end
 
-	# Reads the indexed symlink to find its related key, used for **internal operations**.
+	# Reads the indexed symlink to find its related key.
 	#
 	# For example, for a car indexed by its name:
 	#
@@ -190,8 +175,6 @@ class DODB::Index(V) < DODB::Indexer(V)
 	# `#get_key_on_fs` reads the *storage/indices/by_name/Corvet* symlink and gets
 	# the name of the data file ("000000343") and converts it in an integer,
 	# which is the key in the database.
-	#
-	# NOTE: used for internal operations.
 	def get_key_on_fs(index : String) : Int32
 		file_path = file_path_index index
 		raise MissingEntry.new(@name, index) unless ::File.symlink? file_path
@@ -284,10 +267,6 @@ end
 # Basic indexes for 1-to-1 relations.
 # Cached version.
 #
-# ```
-# cars_by_name = car_database.new_index "name", &.name
-# ```
-#
 # The cache makes this index fast and since the index doesn't contain
 # the full value but just an attribute and a key, memory usage is still reasonable.
 #
@@ -312,8 +291,7 @@ end
 # NOTE: see `Index` for an uncached version, even less memory-hungry.
 # NOTE: for fast operations without fs representation, see `RAMOnlyIndex`.
 class DODB::CachedIndex(V) < DODB::Index(V)
-	# This hash contains the relation between the index key and the data key, used for
+	# This hash contains the relation between the index key and the data key.
-	# **internal operations**.
 	#
 	# WARNING: used for internal operations, do not change its content or access it directly.
 	property data = Hash(String, Int32).new
@@ -334,7 +312,8 @@ class DODB::CachedIndex(V) < DODB::Index(V)
 		end
 	end
 
-	# Clears the cache and removes the `#indexing_directory`.
+	# Clears the cache.
+	# :inherit:
 	def nuke_index
 		super
 		data.clear
@@ -351,8 +330,7 @@ class DODB::CachedIndex(V) < DODB::Index(V)
 		@data[index_key] = key.to_i
 	end
 
-	# Removes the index of a value on the file-system as `DODB::Index#deindex` but also from
+	# Removes the index of a value on the file-system as `DODB::Index#deindex` but also from the cache.
-	# the cache, used for **internal operations**.
 	#
 	# NOTE: used for internal operations.
 	def deindex(key, value)
@@ -382,10 +360,6 @@ end
 # Basic indexes for 1-to-1 relations.
 # RAM-only version, no file-system representation.
 #
-# ```
-# cars_by_name = car_database.new_RAM_index "name", &.name
-# ```
-#
 # Since there is no file-system operations, all the operations are fast.
 # `DODB::RAMOnlyIndex` enables the flexibility of indexes without a file-system representation
 # for absolute efficiency.
@@ -423,9 +397,4 @@ class DODB::RAMOnlyIndex(V) < DODB::CachedIndex(V)
 			raise MissingEntry.new(@name, index)
 		end
 	end
-
-	# Clears the index.
-	def nuke_index
-		data.clear
-	end
 end

src/dodb/partition.cr

@@ -5,10 +5,6 @@ require "./indexer.cr"
 # Partitions for 1-to-n relations.
 # Uncached version.
 #
-# ```
-# cars_by_color = car_database.new_uncached_partition "color", &.color
-# ```
-#
 # This (partition) index provides a file-system representation, enabling the administrators to
 # select a value based on its index. The following example presents an index named "color"
 # with some data indexed by a color attribute.
@@ -32,35 +28,22 @@ require "./indexer.cr"
 # NOTE: see `CachedPartition` for a cached version, faster for retrieval.
 # NOTE: for fast operations without fs representation, see `RAMOnlyPartition`.
 class DODB::Partition(V) < DODB::Indexer(V)
-	# Name of the index, such as *id* or *color* for example.
-	# This is an arbitrary value, mostly to create the index directory.
-	#
-	# NOTE: used for internal operations.
 	property name         : String
-
+	property key_proc     : Proc(V, String | NoIndex) | Proc(V, String)
-	# Procedure to retrieve the index attribute from the value.
-	property key_proc     : Proc(V, String | NoIndex)
-
-	# Root database directory.
 	getter   storage_root : String
 
-	# Reference to the database instance.
+	# Required to remove an entry in the DB.
 	@storage : DODB::Storage(V)
 
-	# To create a partition from a database, use `DODB::Storage#new_partition` to create
+	def initialize(@storage, @storage_root, @name, @key_proc)
-	# a cached partition, `DODB::Storage#new_uncached_partition` for an uncached partition or
-	# `DODB::Storage#new_RAM_partition` for a RAM-only partition.
-	#
-	# WARNING: this is an internal operation, do not instanciate a partition by hand.
-	def initialize(@storage : DODB::Storage(V), @storage_root : String, @name : String, @key_proc : Proc(V, String | NoIndex))
 		::Dir.mkdir_p indexing_directory
 	end
 
-	def check!(key : String, value : V, old_value : V?)
+	def check!(key, value, old_value)
 		return true # Partitions don’t have collisions or overloads.
 	end
 
-	def index(key : String, value : V)
+	def index(key, value)
 		partition = key_proc.call value
 		return if partition.is_a? NoIndex
 
@@ -71,7 +54,7 @@ class DODB::Partition(V) < DODB::Indexer(V)
 		::File.symlink get_data_symlink(key), symlink
 	end
 
-	def deindex (key : String, value : V)
+	def deindex(key, value)
 		partition = key_proc.call value
 		return if partition.is_a? NoIndex
 
@@ -83,15 +66,7 @@ class DODB::Partition(V) < DODB::Indexer(V)
 		end
 	end
 
-	# Gets data from an indexed value (throws an exception on a missing entry).
+	def get(partition) : Array(V)
-	#
-	# ```
-	# red_cars = cars_by_color.get "red"  # No red cars = MissingEntry exception
-	# ```
-	#
-	# WARNING: throws an exception if no value is found.
-	# NOTE: for a safe version, use `#get?`.
-	def get(partition : String) : Array(V)
 		partition_directory = indexing_directory partition
 		raise MissingEntry.new(@name, partition) unless Dir.exists? partition_directory
 
@@ -104,36 +79,16 @@ class DODB::Partition(V) < DODB::Indexer(V)
 		r_value
 	end
 
-	# Safe version of `#get`, gets data and returns a *nil* value in case of
+	def get?(partition) : Array(V)?
-	# a missing entry instead of an exception.
-	#
-	# ```
-	# red_cars = cars_by_color.get? "red"  # No red cars = nil
-	# ```
-	def get?(partition : String) : Array(V)?
 		get partition
 	rescue MissingEntry
 		nil
 	end
 
-	# Deletes all entries within the provided partition.
+	def delete(partition)
-	#
-	# ```
-	# cars_by_color.delete "red"  # Deletes all red cars
-	# ```
-	def delete(partition : String)
 		delete partition, do true end
 	end
 
-	# Deletes entries within the provided partition and matching the provided block of code.
-	#
-	# ```
-	# # Deletes all red Corvets.
-	# cars_by_color.delete "red", do |car|
-	#   car.name == "Corvet"
-	# end
-	# ```
-	# TODO: in case the partition is left empty, should the partition's directory be removed?
 	def delete(partition, &matcher : Proc(V, Bool))
 		partition_directory = indexing_directory partition
 
@@ -149,7 +104,6 @@ class DODB::Partition(V) < DODB::Indexer(V)
 		end
 	end
 
-	# :inherit:
 	def indexing_directory : String
 		"#{@storage_root}/partitions/by_#{@name}"
 	end
@@ -174,10 +128,6 @@ end
 # Partitions for 1-to-n relations.
 # Cached version.
 #
-# ```
-# cars_by_color = car_database.new_partition "color", &.color
-# ```
-#
 # This (partition) index provides a file-system representation, enabling the administrators to
 # select a value based on its index. The following example presents an index named "color"
 # with some data indexed by a color attribute.
@@ -202,22 +152,15 @@ end
 # NOTE: see `Partition` for an uncached version, even less memory-hungry.
 # NOTE: for fast operations without fs representation, see `RAMOnlyPartition`.
 class DODB::CachedPartition(V) < DODB::Partition(V)
-	# This hash contains the relation between the index key and the data key, used for
+	# This hash contains the relation between the index key and the data keys.
-	# **internal operations**.
-	#
-	# WARNING: used for internal operations, do not change its content or access it directly.
 	property data = Hash(String, Array(Int32)).new
 
-	# Clears the cache and removes the `#indexing_directory`.
 	def nuke_index
 		super
 		data.clear
 	end
 
-	# Indexes the value on the file-system as `DODB::Partition#index` but also puts the index in a cache.
+	def index(key, value)
-	#
-	# NOTE: used for internal operations.
-	def index(key : String, value : V)
 		partition = key_proc.call value
 		return if partition.is_a? NoIndex
 		super(key, value)
@@ -232,11 +175,7 @@ class DODB::CachedPartition(V) < DODB::Partition(V)
 		@data[partition] = array
 	end
 
-	# Removes the index of a value on the file-system as `DODB::Partition#deindex` but also from
+	def deindex(key, value)
-	# the cache, used for **internal operations**.
-	#
-	# NOTE: used for internal operations.
-	def deindex(key : String, value : V)
 		partition = key_proc.call value
 		return if partition.is_a? NoIndex
 		super(key, value)
@@ -247,21 +186,9 @@ class DODB::CachedPartition(V) < DODB::Partition(V)
 		end
 	end
 
-	# Gets a partition entries and the database key for each entry.
+	def get_with_indexes(partition) : Array(Tuple(V, Int32))
-	#
-	# ```
-	# # For example, get all red cars.
-	# cars_by_color.get_with_indexes "red"
-	# # Will return something like:
-	# # [ (@storage[42], 42)
-	# # , (@storage[91], 91)
-	# # ]
-	# # Each tuple is composed of a car and its key in the database.
-	# ```
-	def get_with_indexes(partition : String) : Array(Tuple(V, Int32))
 		r_value = Array(Tuple(V, Int32)).new
 
-		# In case the partition is cached.
 		if keys = @data[partition]?
 			keys.each do |data_key|
 				r_value << { @storage[data_key], data_key }
@@ -280,51 +207,11 @@ class DODB::CachedPartition(V) < DODB::Partition(V)
 		r_value
 	end
 
-	# Gets a partition entries.
+	def get(partition) : Array(V)
-	#
+		get_with_indexes(partition).map &.[0]
-	# ```
-	# # For example, get all red cars.
-	# cars_by_color.get "red"
-	# ```
-	# NOTE: returns an empty list on empty or non-existing partition.
-	def get(partition : String) : Array(V)
-		r_value = Array(V).new
-
-		# In case the partition is cached.
-		if keys = @data[partition]?
-			keys.each do |data_key|
-				r_value << @storage[data_key]
-			end
-		else
-			# The keys to put in the partition.
-			p_value = Array(Int32).new
-
-			# Get the key from the database representation on the file-system.
-			partition_directory = indexing_directory partition
-			raise MissingEntry.new(@name, partition) unless Dir.exists? partition_directory
-
-			Dir.each_child partition_directory do |child|
-				key = get_key child
-				r_value << @storage[key]
-				p_value << key
 	end
 
-			@data[partition] = p_value
+	def delete(partition, &matcher : Proc(V, Bool))
-		end
-		r_value
-	end
-
-	# Deletes entries within the provided partition and matching the provided block of code,
-	# both from the file-system representation and from the cache.
-	#
-	# ```
-	# # Deletes all red Corvets.
-	# cars_by_color.delete "red", do |car|
-	#   car.name == "Corvet"
-	# end
-	# ```
-	# TODO: in case the partition is left empty, should the partition be removed from the cache?
-	def delete(partition : String, &matcher : Proc(V, Bool))
 		# Use `get_with_indexes` to retrieve data on-disk, if necessary.
 		new_partition = get_with_indexes(partition).map(&.[1]).select do |key|
 			item = @storage[key]
@@ -340,10 +227,6 @@ end
 # Partitions for 1-to-n relations.
 # RAM-only version.
 #
-# ```
-# cars_by_color = car_database.new_RAM_partition "color", &.color
-# ```
-#
 # Since there is no file-system operations, all the operations are fast.
 # `DODB::RAMOnlyPartition` enables the flexibility of partitions without a file-system representation.
 # Absolute efficiency, exactly as easy to use as the other partition implementations.
@@ -353,7 +236,7 @@ end
 # NOTE: see `Partition` for an uncached version, even less memory-hungry.
 # NOTE: for an fs representation but still fast for retrieval, see `CachedPartition`.
 class DODB::RAMOnlyPartition(V) < DODB::CachedPartition(V)
-	def index(key : String, value : V)
+	def index(key, value)
 		partition = key_proc.call value
 		return if partition.is_a? NoIndex
 
@@ -367,7 +250,7 @@ class DODB::RAMOnlyPartition(V) < DODB::CachedPartition(V)
 		@data[partition] = array
 	end
 
-	def deindex(key : String, value : V)
+	def deindex(key, value)
 		partition = key_proc.call value
 		return if partition.is_a? NoIndex
 
@@ -377,7 +260,7 @@ class DODB::RAMOnlyPartition(V) < DODB::CachedPartition(V)
 		end
 	end
 
-	def get_with_indexes(partition : String) : Array(Tuple(V, Int32))
+	def get_with_indexes(partition) : Array(Tuple(V, Int32))
 		r_value = Array(Tuple(V, Int32)).new
 
 		if keys = @data[partition]?
@@ -388,15 +271,6 @@ class DODB::RAMOnlyPartition(V) < DODB::CachedPartition(V)
 		r_value
 	end
 
-	# Deletes entries within the provided partition and matching the provided block of code.
-	#
-	# ```
-	# # Deletes all red Corvets.
-	# cars_by_color.delete "red", do |car|
-	#   car.name == "Corvet"
-	# end
-	# ```
-	# TODO: in case the partition is left empty, should the partition be removed from the cache?
 	def delete(partition, &matcher : Proc(V, Bool))
 		if keys = @data[partition]?
 			new_partition = keys.select do |key|
@@ -406,9 +280,4 @@ class DODB::RAMOnlyPartition(V) < DODB::CachedPartition(V)
 			@data[partition] = new_partition
 		end
 	end
-
-	# Clears the cache.
-	def nuke_index
-		data.clear
-	end
 end