Adding cached operations DODB::CachedDataBase(V).

spec/benchmark.cr

@@ -0,0 +1,83 @@
+require "benchmark"
+
+require "../src/dodb.cr"
+require "./test-data.cr"
+
+class DODBCached < DODB::CachedDataBase(Ship)
+	def initialize(storage_ext = "", remove_previous_data = true)
+		storage_dir = "test-storage#{storage_ext}"
+
+		if remove_previous_data
+			::FileUtils.rm_rf storage_dir
+		end
+
+		super storage_dir
+	end
+end
+
+class DODBUnCached < DODB::DataBase(Ship)
+	def initialize(storage_ext = "", remove_previous_data = true)
+		storage_dir = "test-storage#{storage_ext}"
+
+		if remove_previous_data
+			::FileUtils.rm_rf storage_dir
+		end
+
+		super storage_dir
+	end
+end
+
+cached   = DODBCached.new
+uncached = DODBUnCached.new
+
+Benchmark.ips do |x|
+	x.report("adding values with a cache") do
+		Ship.all_ships.each do |ship|
+			cached << ship
+		end
+	end
+
+	x.report("adding values without cache") do
+		Ship.all_ships.each do |ship|
+			uncached << ship
+		end
+	end
+end
+
+cached   = DODBCached.new
+uncached = DODBUnCached.new
+
+Ship.all_ships.each do |ship|
+	cached << ship
+	uncached << ship
+end
+
+Benchmark.ips do |x|
+	x.report("to_a with a cache") do
+		cached.to_a
+	end
+
+	x.report("to_a without cache") do
+		uncached.to_a
+	end
+end
+
+Benchmark.ips do |x|
+	x.report("to_h with a cache") do
+		cached.to_h
+	end
+
+	x.report("to_h without cache") do
+		uncached.to_h
+	end
+end
+
+Benchmark.ips do |x|
+	x.report("[0] with a cache") do
+		cached[0]
+	end
+
+	x.report("[0] without cache") do
+		uncached[0]
+	end
+end

spec/cached.cr

@@ -0,0 +1,402 @@
+require "spec"
+require "file_utils"
+
+require "../src/dodb.cr"
+require "./test-data.cr"
+
+
+class DODB::SpecDataBase < DODB::CachedDataBase(Ship)
+	def initialize(storage_ext = "", remove_previous_data = true)
+		storage_dir = "test-storage#{storage_ext}"
+
+		if remove_previous_data
+			::FileUtils.rm_rf storage_dir
+		end
+
+		super storage_dir
+	end
+end
+
+describe "DODB::DataBase::Cached" do
+	describe "basics" do
+		it "store and get data" do
+			db = DODB::SpecDataBase.new
+
+			Ship.all_ships.each do |ship|
+				db << ship
+			end
+
+			db.to_a.sort.should eq(Ship.all_ships.sort)
+		end
+
+		it "rewrite already stored data" do
+			db = DODB::SpecDataBase.new
+			ship = Ship.all_ships[0]
+
+			key = db << ship
+
+			db[key] = Ship.new "broken"
+			db[key] = ship
+
+			db[key].should eq(ship)
+		end
+
+		it "properly remove data" do
+			db = DODB::SpecDataBase.new
+
+			Ship.all_ships.each do |ship|
+				db << ship
+			end
+
+			Ship.all_ships.each do |ship|
+				db.pop
+			end
+
+			Ship.all_ships.each_with_index do |ship, i|
+				# FIXME: Should it raise a particular exception?
+				expect_raises DODB::MissingEntry do
+					db[i]
+				end
+
+				db[i]?.should be_nil
+			end
+		end
+
+		it "preserves data on reopening" do
+			db1 = DODB::SpecDataBase.new
+			db1 << Ship.kisaragi
+
+			db1.to_a.size.should eq(1)
+
+			db2 = DODB::SpecDataBase.new remove_previous_data: false
+			db2 << Ship.mutsuki
+
+			# Only difference with DODB::DataBase: for now, concurrent DB cannot coexists.
+			db2.to_a.size.should eq(2)
+		end
+
+		it "iterates in normal and reversed order" do
+			db = DODB::SpecDataBase.new
+
+			Ship.all_ships.each do |ship|
+				db << ship
+			end
+
+			# The two #each test iteration.
+			db.each_with_index do |item, index|
+				item.should eq Ship.all_ships[index]
+			end
+
+			db.each_with_index(reversed: true) do |item, index|
+				item.should eq Ship.all_ships[index]
+			end
+
+			# Actual reversal is tested here.
+			db.to_a(reversed: true).should eq db.to_a.reverse
+		end
+
+		it "respects the provided offsets if any" do
+			db = DODB::SpecDataBase.new
+
+			Ship.all_ships.each do |ship|
+				db << ship
+			end
+
+			db.to_a(start_offset: 0, end_offset: 0)[0]?.should eq Ship.mutsuki
+			db.to_a(start_offset: 1, end_offset: 1)[0]?.should eq Ship.kisaragi
+			db.to_a(start_offset: 2, end_offset: 2)[0]?.should eq Ship.yayoi
+
+			db.to_a(start_offset: 0, end_offset: 2).should eq [
+				Ship.mutsuki, Ship.kisaragi, Ship.yayoi
+			]
+		end
+	end
+
+	describe "indices" do
+		it "do basic indexing" do
+			db = DODB::SpecDataBase.new
+
+			db_ships_by_name = db.new_index "name", &.name
+
+			Ship.all_ships.each do |ship|
+				db << ship
+			end
+
+			Ship.all_ships.each_with_index do |ship|
+				db_ships_by_name.get?(ship.name).should eq(ship)
+			end
+		end
+
+		it "raise on index overload" do
+			db = DODB::SpecDataBase.new
+
+			db_ships_by_name = db.new_index "name", &.name
+
+			db << Ship.kisaragi
+
+			# Should not be allowed to store an entry whose “name” field
+			# already exists.
+			expect_raises(DODB::IndexOverload) do
+				db << Ship.kisaragi
+			end
+		end
+
+		it "properly deindex" do
+			db = DODB::SpecDataBase.new
+
+			db_ships_by_name = db.new_index "name", &.name
+
+			Ship.all_ships.each do |ship|
+				db << ship
+			end
+
+			Ship.all_ships.each_with_index do |ship, i|
+				db.delete i
+			end
+
+			Ship.all_ships.each do |ship|
+				db_ships_by_name.get?(ship.name).should be_nil
+			end
+		end
+
+		it "properly reindex" do
+			db = DODB::SpecDataBase.new
+
+			db_ships_by_name = db.new_index "name", &.name
+
+			key = db << Ship.kisaragi
+
+			# We give the old id to the new ship, to get it replaced in
+			# the database.
+			some_new_ship = Ship.all_ships[2].clone
+
+			db[key] = some_new_ship
+
+			db[key].should eq(some_new_ship)
+
+			db_ships_by_name.get?(some_new_ship.name).should eq(some_new_ship)
+		end
+
+		it "properly updates" do
+			db = DODB::SpecDataBase.new
+
+			db_ships_by_name = db.new_index "name", &.name
+
+			Ship.all_ships.each do |ship|
+				db << ship
+			end
+
+			new_kisaragi = Ship.kisaragi.clone.tap do |s|
+				s.name = "Kisaragi Kai" # Don’t think about it too much.
+			end
+
+			# We’re changing an indexed value on purpose.
+			db_ships_by_name.update "Kisaragi", new_kisaragi
+
+			db_ships_by_name.get?("Kisaragi").should be_nil
+			db_ships_by_name.get?(new_kisaragi.name).should eq new_kisaragi
+		end
+	end
+
+	describe "partitions" do
+		it "do basic partitioning" do
+			db = DODB::SpecDataBase.new
+
+			db_ships_by_class = db.new_partition "class", &.klass
+
+			Ship.all_ships.each do |ship|
+				db << ship
+			end
+
+			Ship.all_ships.each do |ship|
+				db_ships_by_class.get(ship.klass).should contain(ship)
+			end
+
+			# We extract the possible classes to do test on them.
+			ship_classes = Ship.all_ships.map(&.klass).uniq
+			ship_classes.each do |klass|
+				partition = db_ships_by_class.get klass
+
+				# A partition on “class” should contain entries that all
+				# share the same value of “class”.
+				partition.map(&.klass.==(klass)).reduce { |a, b|
+					a && b
+				}.should be_true
+			end
+
+			db_ships_by_class.get("does-not-exist").should eq [] of Ship
+		end
+
+		it "removes select elements from partitions" do
+			db = DODB::SpecDataBase.new
+
+			db_ships_by_class = db.new_partition "class", &.klass
+
+			Ship.all_ships.each do |ship|
+				db << ship
+			end
+
+			db_ships_by_class.delete "Mutsuki", &.name.==("Kisaragi")
+
+			Ship.all_ships.map(&.klass).uniq.each do |klass|
+				partition = db_ships_by_class.get klass
+
+				partition.any?(&.name.==("Kisaragi")).should be_false
+			end
+		end
+	end
+
+	describe "tags" do
+		it "do basic tagging" do
+			db = DODB::SpecDataBase.new
+
+			db_ships_by_tags = db.new_tags "tags", &.tags
+
+			Ship.all_ships.each do |ship|
+				db << ship
+			end
+
+			db_ships_by_tags.get("flagship").should eq([Ship.flagship])
+
+			# All returned entries should have the requested tag.
+			db_ships_by_tags.get("name ship")
+				.map(&.tags.includes?("name ship"))
+				.reduce { |a, e| a && e }
+				.should be_true
+
+			# There shouldn’t be one in our data about WWII Japanese warships…
+			db_ships_by_tags.get("starship").should eq([] of Ship)
+		end
+
+		it "properly removes tags" do
+			db = DODB::SpecDataBase.new
+
+			db_ships_by_tags = db.new_tags "tags", &.tags
+
+			Ship.all_ships.each do |ship|
+				db << ship
+			end
+
+			# Removing the “flagship” tag, brace for impact.
+			flagship, index = db_ships_by_tags.get_with_indices("flagship")[0]
+			flagship.tags = [] of String
+			db[index] = flagship
+
+
+
+#			ship, index = db_ships_by_tags.update(tag: "flagship") do |ship, index|
+#				ship.tags = [] of String
+#				db[index] = ship
+#			end
+
+			db_ships_by_tags.get("flagship").should eq([] of Ship)
+		end
+
+		it "gets items that have multiple tags" do
+			db = DODB::SpecDataBase.new
+
+			db_ships_by_tags = db.new_tags "tags", &.tags
+
+			Ship.all_ships.each do |ship|
+				db << ship
+			end
+
+			results = db_ships_by_tags.get(["flagship", "name ship"])
+			results.should eq([Ship.yamato])
+
+			results = db_ships_by_tags.get(["name ship", "flagship"])
+			results.should eq([Ship.yamato])
+
+			results = db_ships_by_tags.get(["flagship"])
+			results.should eq([Ship.yamato])
+		end
+	end
+
+	describe "atomic operations" do
+		it "safe_get and safe_get?" do
+			db = DODB::SpecDataBase.new
+
+			db_ships_by_name = db.new_index "name", &.name
+
+			Ship.all_ships.each do |ship|
+				db << ship
+			end
+
+			Ship.all_ships.each do |ship|
+				db_ships_by_name.safe_get ship.name do |results|
+					results.should eq(ship)
+				end
+
+				db_ships_by_name.safe_get? ship.name do |results|
+					results.should eq(ship)
+				end
+			end
+		end
+	end
+
+	describe "tools" do
+		it "rebuilds indexes" do
+			db = DODB::SpecDataBase.new
+
+			db_ships_by_name  = db.new_index     "name", &.name
+			db_ships_by_class = db.new_partition "class", &.klass
+			db_ships_by_tags  = db.new_tags      "tags", &.tags
+
+			Ship.all_ships.each do |ship|
+				db << ship
+			end
+
+			db.reindex_everything!
+
+			Ship.all_ships.each do |ship|
+				db_ships_by_name.get?(ship.name).should eq(ship)
+				db_ships_by_class.get(ship.klass).should contain(ship)
+			end
+		end
+
+		it "migrates properly" do
+			::FileUtils.rm_rf "test-storage-migration-origin"
+			old_db = DODB::DataBase(PrimitiveShip).new "test-storage-migration-origin"
+
+			old_ships_by_name  = old_db.new_index     "name", &.name
+			old_ships_by_class = old_db.new_partition "class", &.class_name
+
+			PrimitiveShip.all_ships.each do |ship|
+				old_db << ship
+			end
+
+			# At this point, the “old” DB is filled. Now we need to convert
+			# to the new DB.
+
+			new_db = DODB::SpecDataBase.new "-migration-target"
+
+			new_ships_by_name  = new_db.new_index     "name", &.name
+			new_ships_by_class = new_db.new_partition "class", &.klass
+			new_ships_by_tags  = new_db.new_tags      "tags", &.tags
+
+			old_db.each_with_index do |ship, index|
+				new_ship = Ship.new ship.name,
+					klass: ship.class_name,
+					id: ship.id,
+					tags: Array(String).new.tap { |tags|
+						tags << "name ship" if ship.name == ship.class_name
+					}
+
+				new_db[index] = new_ship
+			end
+
+			# At this point, the conversion is done, so… we’re making a few
+			# arbitrary tests on the new data.
+
+			old_db.each_with_index do |old_ship, old_index|
+				ship = new_db[old_index]
+
+				ship.id.should eq(old_ship.id)
+				ship.klass.should eq(old_ship.class_name)
+
+				ship.tags.any?(&.==("name ship")).should be_true if ship.name == ship.klass
+			end
+		end
+	end
+end
+

spec/test-data.cr

@@ -0,0 +1,87 @@
+require "uuid"
+require "json"
+
+# FIXME: Split the test data in separate files. We don’t care about those here.
+
+class Ship
+	include JSON::Serializable
+
+	def_clone
+
+	property id    : String
+	property klass : String
+	property name  : String
+	property tags  : Array(String)
+
+	def initialize(@name, @klass = "<unknown>", @id = UUID.random.to_s, @tags = [] of String)
+	end
+
+	# Makes testing arrays of this class easier.
+	def <=>(other)
+		@name <=> other.name
+	end
+
+	# Common, reusable test data.
+	# Those data can be indexed, partitioned or tagged on different parameters,
+	# and can easily be extended.
+
+	class_getter kisaragi = Ship.new("Kisaragi", "Mutsuki")
+	class_getter mutsuki  = Ship.new("Mutsuki",  "Mutsuki", tags: ["name ship"])
+	class_getter yayoi    = Ship.new("Yayoi",    "Mutsuki")
+	class_getter destroyers = [
+		@@mutsuki,
+		@@kisaragi,
+		@@yayoi,
+		Ship.new("Uzuki",    "Mutsuki"),
+		Ship.new("Satsuki",  "Mutsuki"),
+
+		Ship.new("Shiratsuyu", "Shiratsuyu", tags: ["name ship"]),
+		Ship.new("Murasame",   "Shiratsuyu"),
+		Ship.new("Yuudachi",   "Shiratsuyu")
+	]
+
+	class_getter yamato = 
+		Ship.new("Yamato",    "Yamato",      tags: ["name ship", "flagship"])
+	class_getter flagship : Ship = yamato
+	class_getter battleships = [
+		@@yamato,
+		Ship.new("Kongou",    "Kongou",      tags: ["name ship"]),
+		Ship.new("Haruna",    "Kongou"),
+		Ship.new("Kirishima", "Kongou"),
+		Ship.new("Hiei"     , "Kongou"),
+		Ship.new("Musashi",   "Yamato"),
+		Ship.new("Shinano",   "Yamato")
+	]
+
+	class_getter all_ships : Array(Ship) = @@destroyers + @@battleships
+
+	# Equality is true if every property is identical.
+	def ==(other)
+		@id == other.id && @klass == other.klass && @name == other.name &&
+			@tags == other.tags
+	end
+end
+
+# This will be used for migration testing, but basically it’s a variant of
+# the class above, a few extra fields, a few missing ones.
+class PrimitiveShip
+	include JSON::Serializable
+
+	property id         : String
+	property name       : String
+	property wooden     : Bool   = false # Will be removed.
+	property class_name : String         # Will be renamed
+	property flagship   : Bool   = false # Will be moved to tags.
+
+	def initialize(@name, @class_name = "<unknown>", @id = UUID.random.to_s, @flagship = false)
+	end
+
+	class_getter kamikaze =
+		PrimitiveShip.new("Kamikaze", "Kamikaze")
+	class_getter asakaze =
+		PrimitiveShip.new("Asakaze",  "Kamikaze")
+	class_getter all_ships : Array(PrimitiveShip) = [
+		@@kamikaze,
+		@@asakaze
+	]
+end

spec/test.cr

@@ -1,94 +1,9 @@
 require "spec"
 require "file_utils"
-require "json"
-require "uuid"
 
-require "../src/*"
+require "../src/dodb.cr"
+require "./test-data.cr"
 
-# FIXME: Split the test data in separate files. We don’t care about those here.
-
-class Ship
-	include JSON::Serializable
-
-	def_clone
-
-	property id    : String
-	property klass : String
-	property name  : String
-	property tags  : Array(String)
-
-	def initialize(@name, @klass = "<unknown>", @id = UUID.random.to_s, @tags = [] of String)
-	end
-
-	# Makes testing arrays of this class easier.
-	def <=>(other)
-		@name <=> other.name
-	end
-
-	# Common, reusable test data.
-	# Those data can be indexed, partitioned or tagged on different parameters,
-	# and can easily be extended.
-
-	class_getter kisaragi = Ship.new("Kisaragi", "Mutsuki")
-	class_getter mutsuki  = Ship.new("Mutsuki",  "Mutsuki", tags: ["name ship"])
-	class_getter yayoi    = Ship.new("Yayoi",    "Mutsuki")
-	class_getter destroyers = [
-		@@mutsuki,
-		@@kisaragi,
-		@@yayoi,
-		Ship.new("Uzuki",    "Mutsuki"),
-		Ship.new("Satsuki",  "Mutsuki"),
-
-		Ship.new("Shiratsuyu", "Shiratsuyu", tags: ["name ship"]),
-		Ship.new("Murasame",   "Shiratsuyu"),
-		Ship.new("Yuudachi",   "Shiratsuyu")
-	]
-
-	class_getter yamato = 
-		Ship.new("Yamato",    "Yamato",      tags: ["name ship", "flagship"])
-	class_getter flagship : Ship = yamato
-	class_getter battleships = [
-		@@yamato,
-		Ship.new("Kongou",    "Kongou",      tags: ["name ship"]),
-		Ship.new("Haruna",    "Kongou"),
-		Ship.new("Kirishima", "Kongou"),
-		Ship.new("Hiei"     , "Kongou"),
-		Ship.new("Musashi",   "Yamato"),
-		Ship.new("Shinano",   "Yamato")
-	]
-
-	class_getter all_ships : Array(Ship) = @@destroyers + @@battleships
-
-	# Equality is true if every property is identical.
-	def ==(other)
-		@id == other.id && @klass == other.klass && @name == other.name &&
-			@tags == other.tags
-	end
-end
-
-# This will be used for migration testing, but basically it’s a variant of
-# the class above, a few extra fields, a few missing ones.
-class PrimitiveShip
-	include JSON::Serializable
-
-	property id         : String
-	property name       : String
-	property wooden     : Bool   = false # Will be removed.
-	property class_name : String         # Will be renamed
-	property flagship   : Bool   = false # Will be moved to tags.
-
-	def initialize(@name, @class_name = "<unknown>", @id = UUID.random.to_s, @flagship = false)
-	end
-
-	class_getter kamikaze =
-		PrimitiveShip.new("Kamikaze", "Kamikaze")
-	class_getter asakaze =
-		PrimitiveShip.new("Asakaze",  "Kamikaze")
-	class_getter all_ships : Array(PrimitiveShip) = [
-		@@kamikaze,
-		@@asakaze
-	]
-end
 
 class DODB::SpecDataBase < DODB::DataBase(Ship)
 	def initialize(storage_ext = "", remove_previous_data = true)

src/cached.cr

@@ -0,0 +1,113 @@
+require "file_utils"
+require "json"
+
+class Hash(K,V)
+	def reverse
+		rev = Array(Tuple(K,V)).new
+		keys = Array(K).new
+		each_key do |k|
+			keys << k
+		end
+		keys.reverse.each do |k|
+			rev << {k, self.[k]}
+		end
+
+		rev
+	end
+end
+
+class DODB::CachedDataBase(V) < DODB::Storage(V)
+	@indexers = [] of Indexer(V)
+	property data = Hash(Int32, V).new
+
+	def initialize(@directory_name : String)
+		Dir.mkdir_p data_path
+		Dir.mkdir_p locks_directory
+
+		begin
+			self.last_index
+		rescue
+			self.last_index = -1
+		end
+
+		# TODO: load the database in RAM at start-up
+		DODB::DataBase(V).new(@directory_name).each_with_index do |v, index|
+			puts "loading value #{v} at index #{index}"
+			self[index] = v
+		end
+	end
+
+	# Getting data from the hash in RAM.
+	def []?(key : Int32) : V?
+		@data[key]
+	rescue e
+		# FIXME: rescues any error the same way.
+		return nil
+	end
+	def [](key : Int32) : V
+		# raise MissingEntry.new(key) unless ::File.exists? file_path key
+		# read file_path key
+		@data[key] rescue raise MissingEntry.new(key)
+	end
+
+	def []=(index : Int32, value : V)
+		old_value = self.[index]?
+
+		check_collisions! index, value, old_value
+
+		# Removes any old indices or partitions pointing to a value about
+		# to be replaced.
+		if old_value
+			remove_partitions index, old_value
+		end
+
+		# Avoids corruption in case the application crashes while writing.
+		file_path(index).tap do |path|
+			::File.write "#{path}.new", value.to_json
+			::FileUtils.mv "#{path}.new", path
+		end
+
+		write_partitions index, value
+
+		if index > last_index
+			self.last_index = index
+		end
+
+		@data[index] = value
+	end
+
+	##
+	# Can be useful for making dumps or to restore a database.
+	def each_with_index(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
+		i = -1 # do not trust key to be the right index
+		(reversed ? @data.reverse : @data).each do |index, v|
+			i += 1
+			next if start_offset > i
+			break unless end_offset.nil? || i <= end_offset
+
+			yield v, index
+		end
+	end
+
+	def delete(key : Int32)
+		value = self[key]?
+
+		return if value.nil?
+
+		begin
+			::File.delete file_path key
+		rescue
+			# FIXME: Only intercept “no such file" errors
+		end
+
+		remove_partitions key, value
+
+		@data.delete key
+		value
+	end
+
+	private def remove_data!
+		super
+		@data = Hash(Int32, V).new
+	end
+end

src/dodb.cr

@@ -3,18 +3,26 @@ require "json"
 
 require "./dodb/*"
 
-class DODB::DataBase(V)
+abstract class DODB::Storage(V)
-	@indexers = [] of Indexer(V)
+	property directory_name : String
 
 	def initialize(@directory_name : String)
-		Dir.mkdir_p data_path
-		Dir.mkdir_p locks_directory
-
-		begin
-			self.last_index
-		rescue
-			self.last_index = -1
 	end
+
+	def request_lock(name)
+		r = -1
+		file_path = get_lock_file_path name
+		file_perms = 0o644
+
+		flags = LibC::O_EXCL | LibC::O_CREAT
+		while (r = LibC.open file_path, flags, file_perms) == -1
+			sleep 1.milliseconds
+		end
+
+		LibC.close r
+	end
+	def release_lock(name)
+		File.delete get_lock_file_path name
 	end
 
 	private def index_file
@@ -44,28 +52,52 @@ class DODB::DataBase(V)
 		end
 	end
 
-	def request_lock(name)
+	def <<(item : V)
-		r = -1
+		index = last_index + 1
-		file_path = get_lock_file_path name
+		self[index] = item
-		file_perms = 0o644
+		self.last_index = index
-
-		flags = LibC::O_EXCL | LibC::O_CREAT
-		while (r = LibC.open file_path, flags, file_perms) == -1
-			sleep 1.milliseconds
 	end
 
-		LibC.close r
+	def each(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
+		each_with_index(
+			reversed: reversed,
+			start_offset: start_offset,
+			end_offset: end_offset
+		) do |item, index|
+			yield item
 		end
-	def release_lock(name)
-		File.delete get_lock_file_path name
 	end
 
 	##
-	# name is the name that will be used on the file system.
+	# CAUTION: Very slow. Try not to use.
-	def new_partition(name : String, &block : Proc(V, String))
+	def to_a(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
-		Partition(V).new(self, @directory_name, name, block).tap do |table|
+		array = ::Array(V).new
-			@indexers << table
+
+		each(
+			reversed: reversed,
+			start_offset: start_offset,
+			end_offset: end_offset
+		) do |value|
+			array << value
 		end
+
+		array
+	end
+
+	##
+	# CAUTION: Very slow. Try not to use.
+	def to_h(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
+		hash = ::Hash(Int32, V).new
+
+		each_with_index(
+			reversed: reversed,
+			start_offset: start_offset,
+			end_offset: end_offset
+		) do |element, index|
+			hash[index] = element
+		end
+
+		hash
 	end
 
 	##
@@ -76,6 +108,14 @@ class DODB::DataBase(V)
 		end
 	end
 
+	##
+	# name is the name that will be used on the file system.
+	def new_partition(name : String, &block : Proc(V, String))
+		Partition(V).new(self, @directory_name, name, block).tap do |table|
+			@indexers << table
+		end
+	end
+
 	def new_tags(name : String, &block : Proc(V, Array(String)))
 		Tags(V).new(@directory_name, name, block).tap do |tags|
 			@indexers << tags
@@ -100,12 +140,103 @@ class DODB::DataBase(V)
 		partition.not_nil!.as(DODB::Tags).get name, key
 	end
 
-	def <<(item : V)
+	def check_collisions!(key : Int32, value : V, old_value : V?)
-		index = last_index + 1
+		@indexers.each &.check!(stringify_key(key), value, old_value)
+	end
 
+	def write_partitions(key : Int32, value : V)
+		@indexers.each &.index(stringify_key(key), value)
+	end
+
+	def pop
+		index = last_index
+
+		# Some entries may have been removed. We’ll skip over those.
+		# Not the most efficient if a large number of indices are empty.
+		while index >= 0 && self[index]?.nil?
+			index = index - 1
+		end
+
+		if index < 0
+			return nil
+		end
+
+		poped = self[index]
+
+		self.delete index
+
+		last_index = index - 1
+
+		poped
+	end
+
+	private def data_path
+		"#{@directory_name}/data"
+	end
+
+	private def file_path(key : Int32)
+		"#{data_path}/%010i.json" % key
+	end
+
+	private def locks_directory : String
+		"#{@directory_name}/locks"
+	end
+
+	private def get_lock_file_path(name : String)
+		"#{locks_directory}/#{name}.lock"
+	end
+
+	private def read(file_path : String)
+		V.from_json ::File.read file_path
+	end
+
+	private def remove_data!
+		FileUtils.rm_rf data_path
+		Dir.mkdir_p data_path
+	end
+
+	private def remove_indexing!
+		@indexers.each do |indexer|
+			FileUtils.rm_rf indexer.indexing_directory
+		end
+	end
+
+	# A very slow operation that removes all indices and then rewrites
+	# them all.
+	# FIXME: Is this really useful in its current form? We should remove the
+	#        index directories, not the indices based on our current (and
+	#        possiblly different from what’s stored) data.
+	def reindex_everything!
+		old_data = to_h
+
+		remove_indexing!
+		remove_data!
+
+		old_data.each do |index, item|
 			self[index] = item
+		end
+	end
 
-		self.last_index = index
+	def remove_partitions(key : Int32, value : V)
+		@indexers.each &.deindex(stringify_key(key), value)
+	end
+
+	abstract def [](key : Int32)
+	abstract def delete(key : Int32)
+end
+
+class DODB::DataBase(V) < DODB::Storage(V)
+	@indexers = [] of Indexer(V)
+
+	def initialize(@directory_name : String)
+		Dir.mkdir_p data_path
+		Dir.mkdir_p locks_directory
+
+		begin
+			self.last_index
+		rescue
+			self.last_index = -1
+		end
 	end
 
 	def []?(key : Int32) : V?
@@ -145,36 +276,6 @@ class DODB::DataBase(V)
 		end
 	end
 
-	def check_collisions!(key : Int32, value : V, old_value : V?)
-		@indexers.each &.check!(stringify_key(key), value, old_value)
-	end
-
-	def write_partitions(key : Int32, value : V)
-		@indexers.each &.index(stringify_key(key), value)
-	end
-
-	def pop
-		index = last_index
-
-		# Some entries may have been removed. We’ll skip over those.
-		# Not the most efficient if a large number of indices are empty.
-		while index >= 0 && self[index]?.nil?
-			index = index - 1
-		end
-
-		if index < 0
-			return nil
-		end
-
-		poped = self[index]
-
-		self.delete index
-
-		last_index = index - 1
-
-		poped
-	end
-
 	def delete(key : Int32)
 		value = self[key]?
 
@@ -191,10 +292,6 @@ class DODB::DataBase(V)
 		value
 	end
 
-	def remove_partitions(key : Int32, value : V)
-		@indexers.each &.deindex(stringify_key(key), value)
-	end
-
 	private def each_key(reversed = false)
 		start = 0
 		_end = last_index
@@ -246,93 +343,6 @@ class DODB::DataBase(V)
 			yield field, key
 		end
 	end
-	def each(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
-		each_with_index(
-			reversed: reversed,
-			start_offset: start_offset,
-			end_offset: end_offset
-		) do |item, index|
-			yield item
-		end
-	end
-
-	##
-	# CAUTION: Very slow. Try not to use.
-	def to_a(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
-		array = ::Array(V).new
-
-		each(
-			reversed: reversed,
-			start_offset: start_offset,
-			end_offset: end_offset
-		) do |value|
-			array << value
-		end
-
-		array
-	end
-
-	##
-	# CAUTION: Very slow. Try not to use.
-	def to_h(reversed : Bool = false, start_offset = 0, end_offset : Int32? = nil)
-		hash = ::Hash(Int32, V).new
-
-		each_with_index(
-			reversed: reversed,
-			start_offset: start_offset,
-			end_offset: end_offset
-		) do |element, index|
-			hash[index] = element
-		end
-
-		hash
-	end
-
-	private def data_path
-		"#{@directory_name}/data"
-	end
-
-	private def file_path(key : Int32)
-		"#{data_path}/%010i.json" % key
-	end
-
-	private def locks_directory : String
-		"#{@directory_name}/locks"
-	end
-
-	private def get_lock_file_path(name : String)
-		"#{locks_directory}/#{name}.lock"
-	end
-
-	private def read(file_path : String)
-		V.from_json ::File.read file_path
-	end
-
-	private def remove_data!
-		FileUtils.rm_rf data_path
-		Dir.mkdir_p data_path
-	end
-
-	private def remove_indexing!
-		@indexers.each do |indexer|
-			FileUtils.rm_rf indexer.indexing_directory
-		end
-	end
-
-	# A very slow operation that removes all indices and then rewrites
-	# them all.
-	# FIXME: Is this really useful in its current form? We should remove the
-	#        index directories, not the indices based on our current (and
-	#        possiblly different from what’s stored) data.
-	def reindex_everything!
-		old_data = to_h
-
-		remove_indexing!
-		remove_data!
-
-		old_data.each do |index, item|
-			self[index] = item
-		end
-	end
 end
 
+require "./cached.cr"

src/dodb/index.cr

@@ -9,7 +9,7 @@ class DODB::Index(V) < DODB::Indexer(V)
 	property key_proc     : Proc(V, String)
 	getter   storage_root : String
 
-	@storage : DODB::DataBase(V)
+	@storage : DODB::Storage(V)
 
 	def initialize(@storage, @storage_root, @name, @key_proc)
 		Dir.mkdir_p indexing_directory

src/dodb/partition.cr

@@ -8,7 +8,7 @@ class DODB::Partition(V) < DODB::Indexer(V)
 	property key_proc     : Proc(V, String)
 	getter   storage_root : String
 
-	@storage : DODB::DataBase(V)
+	@storage : DODB::Storage(V)
 
 	def initialize(@storage, @storage_root, @name, @key_proc)
 		::Dir.mkdir_p indexing_directory