More tests for the Common database.

spec/test-common.cr

@@ -1,6 +1,10 @@
 require "spec"
 require "./db-cars.cr"
 
+# NOTE: the following tests access both database and triggers' caches directly.
+# It must not be done in real applications.
+# `data` and `lru` properties are implementation details.
+
 describe "SPECDB::Common" do
 	it "basics, 3 values" do
 		car0 = Car.new "Corvet-0", "red", [] of String
@@ -30,4 +34,90 @@ describe "SPECDB::Common" do
 		db.data.keys.sort.should eq([0, 3] of Int32)
 		db.lru.to_s.should eq "[ 3, 0 ]"
 	end
+
+	it "add some values, reindex, check the values" do
+		car0 = Car.new "Corvet-0", "red", [] of String
+		car1 = Car.new "Corvet-1", "red", [] of String
+		car2 = Car.new "Corvet-2", "red", [] of String
+		car3 = Car.new "Corvet-3", "red", [] of String
+
+		db = SPECDB::Common(Car).new "reindex", 2
+		index_name  = db.new_index     "name",  &.name
+		index_color = db.new_partition "color", &.color
+
+		db.data.keys.sort.should eq([] of Int32)
+
+		db << car0
+		db.data.keys.sort.should eq([0] of Int32)
+
+		db << car1
+		db.data.keys.sort.should eq([0, 1] of Int32)
+		db.lru.to_s.should eq "[ 1, 0 ]"
+
+		db.data.keys.sort.should eq([0, 1] of Int32)
+		db.lru.to_s.should eq "[ 1, 0 ]"
+
+		db << car2
+		db.data.keys.sort.should eq([1, 2] of Int32)
+		db.lru.to_s.should eq "[ 2, 1 ]"
+		db[0] # Let's use the first value, it shouldn't be the one to be dropped.
+
+		db << car3
+		db.data.keys.sort.should eq([0, 3] of Int32)
+		db.lru.to_s.should eq "[ 3, 0 ]"
+
+		# Actually remove the value indexed "2".
+		db.delete 2
+		# The "delete" action actually puts the value in the cache before removing it.
+		db.data.keys.sort.should eq([3] of Int32)
+		db.lru.to_s.should eq "[ 3 ]"
+
+		# db2: further tests for the LRU behavior, with `reindex_everything!` for example.
+		db2 = SPECDB::Common(Car).new "reindex", 2, false
+		index2_name  = db2.new_index     "name",  &.name
+
+		# For now, we didn't use any value in the database.
+		# Also, triggers should have empty caches.
+		db2.data.keys.sort.should eq([] of Int32)
+		db2.lru.to_s.should eq "[  ]"
+		index2_name.get?("Corvet-0").should eq car0
+
+		# Reindex removes all triggers (hard rm -rf trigger-dir) then reads all
+		# data from the file-system to run the triggers. Data is put into the cache
+		# of the `Common` database.
+		db2.reindex_everything!
+
+		# Values 1 and 3 are the last recently used values following `reindex_everything!`.
+		db2.data.keys.sort.should eq([1, 3] of Int32)
+		db2.lru.to_s.should eq "[ 3, 1 ]"
+
+		total_values = 0
+		db2.each_with_key do |v, k|
+			total_values += 1
+		end
+		total_values.should eq 3
+
+		# db3: no need for a reindex to access values from the database,
+		# with `each` or `each_with_key` for example.
+		db3 = SPECDB::Common(Car).new "reindex", 2, false
+		total_values = 0
+		db3.each do |v|
+			total_values += 1
+		end
+		total_values.should eq 3
+
+		# db4: no need for a reindex to access values from triggers.
+		db4 = SPECDB::Common(Car).new "reindex", 2, false
+		index4_name  = db4.new_index     "name",  &.name
+		index4_color = db4.new_partition "color", &.color
+
+		index4_name.data.size.should eq 0 # trigger cache is empty
+		index4_name.get?("Corvet-3").should eq car3
+		index4_name.data.size.should eq 1 # trigger cache: one value, Corvet-3
+
+		index4_color.data.size.should eq 0 # trigger cache is empty
+		index4_color.get("red").size.should eq 3 # read from the disk
+		index4_color.data.size.should eq 1 # trigger cache: 1, the color "red"
+		index4_color.data["red"].size.should eq 3 # 3 values in "red"
+	end
 end

src/dodb/storage.cr

@@ -344,7 +344,7 @@ abstract class DODB::Storage(V)
 		Dir.mkdir_p data_path
 	end
 
-	private def remove_triggers!
+	private def nuke_triggers!
 		@triggers.each do |trigger|
 			trigger.nuke_trigger
 		end
@@ -354,7 +354,7 @@ abstract class DODB::Storage(V)
 	#
 	# WARNING: slow operation.
 	def reindex_everything!
-		remove_triggers!
+		nuke_triggers!
 
 		each_with_key() do |item, key|
 			run_triggers key, item
@@ -436,7 +436,7 @@ abstract class DODB::Storage(V)
 	end
 
 	# Lists all the keys in the database.
-	private def each_key(reversed = false)
+	def each_key_from_fs(reversed = false)
 		# Removes the first two "." and ".." directories.
 		keys = Dir.children(data_path).map(&.to_i).sort
 		(reversed ? keys.reverse : keys).each do |key|
@@ -455,7 +455,7 @@ abstract class DODB::Storage(V)
 			-1
 		end
 
-		each_key(reversed) do |key|
+		each_key_from_fs(reversed) do |key|
 			offset -= 1 if offset >= 0
 			next if offset >= 0
 

src/dodb/storage/common.cr

@@ -83,7 +83,35 @@ class DODB::Storage::Common(V) < DODB::Storage::Cached(V)
 
 	private def push_lru(key : Int32)
 		if entry_to_remove = @lru << key
+			# Remove the data from the cache but doesn't remove the indexes.
 			@data.delete entry_to_remove
 		end
 	end
+
+	# Function `each_with_key` is a rewrite of the parent.
+	# The `Common` database doesn't keep all entries in memory, data has to be read from disk.
+	def each_with_key(reversed : Bool = false, offset = 0, limit : Int32? = -1)
+		limit = if l = limit
+			l
+		else
+			-1
+		end
+
+		each_key_from_fs(reversed) do |key|
+			offset -= 1 if offset >= 0
+			next if offset >= 0
+
+			return if limit == 0
+			limit -= 1 if limit > 0
+
+			begin
+				# FIXME: Only intercept JSON parsing errors.
+				value = self[key]
+			rescue
+				next
+			end
+
+			yield value, key
+		end
+	end
 end