Rewriting the benchmarks to get better data.

spec/benchmark-cars.cr

@@ -2,6 +2,43 @@ require "benchmark"
 require "./utilities.cr"
 require "./db-cars.cr"
 
+# List of environment variables and default values:
+# ENV["CARNAME"]          rescue "Corvet-#{(db_size/2).to_i}"
+# ENV["CARCOLOR"]         rescue "red"
+# ENV["CARKEYWORD"]       rescue "spacious"
+# ENV["DBSIZE"]           rescue 1_000
+# ENV["DBSIZE_START"]     rescue 5_000
+# ENV["DBSIZE_INCREMENT"] rescue 5_000
+# ENV["REPORT_DIR"]       rescue "results"
+
+class DODB::Storage(V)
+	def empty_db
+		while pop
+		end
+	end
+end
+
+class Context
+	class_property report_dir = "results"
+end
+
+def report(storage, db_size, nb_run, name, title, &block)
+	#add_cars storage, db_size
+
+	durations = run_n_times nb_run, &block
+	File.open("#{Context.report_dir}/#{name}", "w") do |file|
+		durations.each do |d|
+			file.puts d
+		end
+	end
+	avr = durations.reduce { |a, b| a + b }  / nb_run
+	puts "#{title}: #{avr}"
+	#storage.empty_db
+
+	avr
+end
+
+def batch(db_size, nb_run)
 	cars_ram      = DODB::RAMOnlySpecDataBase(Car).new
 	cars_cached   = DODB::CachedSpecDataBase(Car).new
 	cars_semi     = DODB::SpecDataBase(Car).new "-semi"
@@ -12,92 +49,97 @@ cached_searchby_name,   cached_searchby_color,   cached_searchby_keywords   = ca
 	semi_searchby_name,     semi_searchby_color,     semi_searchby_keywords     = cached_indexes   cars_semi
 	uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = uncached_indexes cars_uncached
 
-db_size               = ENV["DBSIZE"].to_i rescue 1_000
+	puts "add some values"
-car_name_to_search    = ENV["CARNAME"]     rescue "Corvet-#{(db_size/2).to_i}"
-car_color_to_search   = ENV["CARCOLOR"]    rescue "red"
-car_keyword_to_search = ENV["CARKEYWORD"]  rescue "spacious"
-
 	add_cars cars_ram,      db_size
 	add_cars cars_cached,   db_size
 	add_cars cars_semi,     db_size
 	add_cars cars_uncached, db_size
 
 	# Searching for data with an index.
+	car_name_to_search = ENV["CARNAME"] rescue "Corvet-#{(db_size/2).to_i}"
 	puts "search by index '#{car_name_to_search}': get a single value"
-Benchmark.ips do |x|
+
-	x.report("(ram db and index) searching with an index") do
+	report(cars_ram, db_size, nb_run, "ram_#{db_size}_index", "(ram db and index) searching with an index") do
 		corvet = ram_searchby_name.get car_name_to_search
 	end
 
-	x.report("(cached db and index) searching with an index") do
+	report(cars_cached, db_size, nb_run, "cached_#{db_size}_index", "(cached db and index) searching with an index") do
 		corvet = cached_searchby_name.get car_name_to_search
 	end
 
-	x.report("(semi: uncached db but cached index) searching with an index") do
+	report(cars_semi, db_size, nb_run, "semi_#{db_size}_index", "(semi: uncached db but cached index) searching with an index") do
 		corvet = semi_searchby_name.get car_name_to_search
 	end
 
-	x.report("(uncached db and index) searching with an index") do
+	report(cars_uncached, db_size, nb_run, "uncached_#{db_size}_index", "(uncached db and index) searching with an index") do
 		corvet = uncached_searchby_name.get car_name_to_search
 	end
-end
 
-# Searching for data with a partition.
+#	# Searching for data with a partition.
-puts ""
+#	car_color_to_search = ENV["CARCOLOR"] rescue "red"
-puts "search by partition #{car_color_to_search}: get #{ram_searchby_color.get(car_color_to_search).size} values"
+#	puts ""
-Benchmark.ips do |x|
+#	puts "search by partition #{car_color_to_search}: get #{ram_searchby_color.get(car_color_to_search).size} values"
-	x.report("(ram db and partition) searching with a partition") do
+#	Benchmark.ips do |x|
-		corvet = ram_searchby_color.get car_color_to_search
+#		add_cars cars_ram, db_size
-	end
+#		x.report("(ram db and partition) searching with a partition") do
-
+#			corvet = ram_searchby_color.get car_color_to_search
-	x.report("(cached db and partition) searching with a partition") do
+#		end
-		corvet = cached_searchby_color.get car_color_to_search
+#		cars_ram.empty_db
-	end
+#
-
+#		add_cars cars_cached, db_size
-	x.report("(semi: uncached db but cached partition) searching with a partition") do
+#		x.report("(cached db and partition) searching with a partition") do
-		corvet = semi_searchby_color.get car_color_to_search
+#			corvet = cached_searchby_color.get car_color_to_search
-	end
+#		end
-
+#		cars_cached.empty_db
-	x.report("(uncached db and partition) searching with a partition") do
+#
-		corvet = uncached_searchby_color.get car_color_to_search
+#		add_cars cars_semi, db_size
-	end
+#		x.report("(semi: uncached db but cached partition) searching with a partition") do
-end
+#			corvet = semi_searchby_color.get car_color_to_search
-
+#		end
-# Searching for data with a tag.
+#		cars_semi.empty_db
-puts ""
+#
-puts "search by tag #{car_keyword_to_search}: get #{ram_searchby_keywords.get(car_keyword_to_search).size} values"
+#		add_cars cars_uncached, db_size
-Benchmark.ips do |x|
+#		x.report("(uncached db and partition) searching with a partition") do
-	x.report("(ram db and tag) searching with a tag") do
+#			corvet = uncached_searchby_color.get car_color_to_search
-		corvet = ram_searchby_keywords.get car_keyword_to_search
+#		end
-	end
+#		cars_uncached.empty_db
-
+#	end
-	x.report("(cached db and tag) searching with a tag") do
+#
-		corvet = cached_searchby_keywords.get car_keyword_to_search
+#	# Searching for data with a tag.
-	end
+#	car_keyword_to_search = ENV["CARKEYWORD"]  rescue "spacious"
-
+#	puts ""
-	x.report("(semi: uncached db but cached tag) searching with a tag") do
+#	puts "search by tag #{car_keyword_to_search}: get #{ram_searchby_keywords.get(car_keyword_to_search).size} values"
-		corvet = semi_searchby_keywords.get car_keyword_to_search
+#	Benchmark.ips do |x|
-	end
+#		add_cars cars_ram, db_size
-
+#		x.report("(ram db and tag) searching with a tag") do
-	x.report("(uncached db and tag) searching with a tag") do
+#			corvet = ram_searchby_keywords.get car_keyword_to_search
-		corvet = uncached_searchby_keywords.get car_keyword_to_search
+#		end
-	end
+#		cars_ram.empty_db
-end
+#
+#		add_cars cars_cached, db_size
+#		x.report("(cached db and tag) searching with a tag") do
+#			corvet = cached_searchby_keywords.get car_keyword_to_search
+#		end
+#		cars_cached.empty_db
+#
+#		add_cars cars_semi, db_size
+#		x.report("(semi: uncached db but cached tag) searching with a tag") do
+#			corvet = semi_searchby_keywords.get car_keyword_to_search
+#		end
+#		cars_semi.empty_db
+#
+#		add_cars cars_uncached, db_size
+#		x.report("(uncached db and tag) searching with a tag") do
+#			corvet = uncached_searchby_keywords.get car_keyword_to_search
+#		end
+#		cars_uncached.empty_db
+#	end
 
 	cars_ram.rm_storage_dir
 	cars_cached.rm_storage_dir
 	cars_semi.rm_storage_dir
 	cars_uncached.rm_storage_dir
-
+end
-cars_ram      = DODB::RAMOnlySpecDataBase(Car).new
-cars_cached   = DODB::CachedSpecDataBase(Car).new
-cars_semi     = DODB::SpecDataBase(Car).new "-semi"
-cars_uncached = DODB::SpecDataBase(Car).new
-
-ram_searchby_name,      ram_searchby_color,      ram_searchby_keywords      = ram_indexes      cars_ram
-cached_searchby_name,   cached_searchby_color,   cached_searchby_keywords   = cached_indexes   cars_cached
-semi_searchby_name,     semi_searchby_color,     semi_searchby_keywords     = cached_indexes   cars_semi
-uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = uncached_indexes cars_uncached
 
 def perform_add(storage : DODB::Storage(Car), nb_run)
 	corvet0 = Car.new "Corvet", "red", [ "shiny", "impressive", "fast", "elegant" ]
@@ -110,7 +152,17 @@ def perform_add(storage : DODB::Storage(Car), nb_run)
 	end
 end
 
-nb_run = 1000
+def batch_add(nb_run)
+	cars_ram      = DODB::RAMOnlySpecDataBase(Car).new
+	cars_cached   = DODB::CachedSpecDataBase(Car).new
+	cars_semi     = DODB::SpecDataBase(Car).new "-semi"
+	cars_uncached = DODB::SpecDataBase(Car).new
+
+	ram_searchby_name,      ram_searchby_color,      ram_searchby_keywords      = ram_indexes      cars_ram
+	cached_searchby_name,   cached_searchby_color,   cached_searchby_keywords   = cached_indexes   cars_cached
+	semi_searchby_name,     semi_searchby_color,     semi_searchby_keywords     = cached_indexes   cars_semi
+	uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = uncached_indexes cars_uncached
+
 	avr = perform_add(cars_ram, nb_run)
 	puts "(ram db and indexes) add a value (average on #{nb_run} tries): #{avr}"
 
@@ -127,3 +179,19 @@ cars_ram.rm_storage_dir
 	cars_cached.rm_storage_dir
 	cars_semi.rm_storage_dir
 	cars_uncached.rm_storage_dir
+end
+
+ENV["REPORT_DIR"]?.try do |report_dir|
+	Context.report_dir = report_dir
+end
+Dir.mkdir_p Context.report_dir
+
+nb_run = 1000
+db_size_max = ENV["DBSIZE"].to_i rescue 1_000
+current_size = ENV["DBSIZE_START"].to_i rescue 5_000
+size_increment = ENV["DBSIZE_INCREMENT"].to_i rescue 5_000
+while current_size <= db_size_max
+	batch current_size, nb_run
+	current_size += size_increment
+end
+batch_add nb_run

spec/utilities.cr

@@ -12,6 +12,21 @@ def perform_benchmark_average(ntimes : Int32, &block)
 		sum += elapsed_time
 		i += 1
 	end
-
 	sum / ntimes
 end
+
+def run_n_times(ntimes : Int32, &block)
+	i = 1
+	durations = Array(Float64).new
+	while i <= ntimes
+		elapsed_time = perform_something &block
+		durations << elapsed_time.total_nanoseconds
+		i += 1
+	end
+	durations
+end
+
+# TODO
+def should_nb_files(path : String, expected_nb_files : UInt32)
+	raise Exception.new "should_nb_files: not implemented yet"
+end