From 36ca2f5941396b02200dcf3a207ca3a713c3643a Mon Sep 17 00:00:00 2001 From: Philippe PITTOLI Date: Fri, 10 May 2024 17:57:07 +0200 Subject: [PATCH] Rewriting the benchmarks to get better data. --- spec/benchmark-cars.cr | 260 ++++++++++++++++++++++++++--------------- spec/utilities.cr | 17 ++- 2 files changed, 180 insertions(+), 97 deletions(-) diff --git a/spec/benchmark-cars.cr b/spec/benchmark-cars.cr index de86475..1b930a6 100644 --- a/spec/benchmark-cars.cr +++ b/spec/benchmark-cars.cr @@ -2,103 +2,145 @@ require "benchmark" require "./utilities.cr" require "./db-cars.cr" -cars_ram = DODB::RAMOnlySpecDataBase(Car).new -cars_cached = DODB::CachedSpecDataBase(Car).new -cars_semi = DODB::SpecDataBase(Car).new "-semi" -cars_uncached = DODB::SpecDataBase(Car).new +# List of environment variables and default values: +# ENV["CARNAME"] rescue "Corvet-#{(db_size/2).to_i}" +# ENV["CARCOLOR"] rescue "red" +# ENV["CARKEYWORD"] rescue "spacious" +# ENV["DBSIZE"] rescue 1_000 +# ENV["DBSIZE_START"] rescue 5_000 +# ENV["DBSIZE_INCREMENT"] rescue 5_000 +# ENV["REPORT_DIR"] rescue "results" -ram_searchby_name, ram_searchby_color, ram_searchby_keywords = ram_indexes cars_ram -cached_searchby_name, cached_searchby_color, cached_searchby_keywords = cached_indexes cars_cached -semi_searchby_name, semi_searchby_color, semi_searchby_keywords = cached_indexes cars_semi -uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = uncached_indexes cars_uncached +class DODB::Storage(V) + def empty_db + while pop + end + end +end -db_size = ENV["DBSIZE"].to_i rescue 1_000 -car_name_to_search = ENV["CARNAME"] rescue "Corvet-#{(db_size/2).to_i}" -car_color_to_search = ENV["CARCOLOR"] rescue "red" -car_keyword_to_search = ENV["CARKEYWORD"] rescue "spacious" +class Context + class_property report_dir = "results" +end -add_cars cars_ram, db_size -add_cars cars_cached, db_size -add_cars cars_semi, db_size -add_cars cars_uncached, db_size +def report(storage, db_size, nb_run, name, title, &block) + #add_cars storage, db_size -# Searching for data with an index. -puts "search by index '#{car_name_to_search}': get a single value" -Benchmark.ips do |x| - x.report("(ram db and index) searching with an index") do + durations = run_n_times nb_run, &block + File.open("#{Context.report_dir}/#{name}", "w") do |file| + durations.each do |d| + file.puts d + end + end + avr = durations.reduce { |a, b| a + b } / nb_run + puts "#{title}: #{avr}" + #storage.empty_db + + avr +end + +def batch(db_size, nb_run) + cars_ram = DODB::RAMOnlySpecDataBase(Car).new + cars_cached = DODB::CachedSpecDataBase(Car).new + cars_semi = DODB::SpecDataBase(Car).new "-semi" + cars_uncached = DODB::SpecDataBase(Car).new + + ram_searchby_name, ram_searchby_color, ram_searchby_keywords = ram_indexes cars_ram + cached_searchby_name, cached_searchby_color, cached_searchby_keywords = cached_indexes cars_cached + semi_searchby_name, semi_searchby_color, semi_searchby_keywords = cached_indexes cars_semi + uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = uncached_indexes cars_uncached + + puts "add some values" + add_cars cars_ram, db_size + add_cars cars_cached, db_size + add_cars cars_semi, db_size + add_cars cars_uncached, db_size + + # Searching for data with an index. + car_name_to_search = ENV["CARNAME"] rescue "Corvet-#{(db_size/2).to_i}" + puts "search by index '#{car_name_to_search}': get a single value" + + report(cars_ram, db_size, nb_run, "ram_#{db_size}_index", "(ram db and index) searching with an index") do corvet = ram_searchby_name.get car_name_to_search end - x.report("(cached db and index) searching with an index") do + report(cars_cached, db_size, nb_run, "cached_#{db_size}_index", "(cached db and index) searching with an index") do corvet = cached_searchby_name.get car_name_to_search end - x.report("(semi: uncached db but cached index) searching with an index") do + report(cars_semi, db_size, nb_run, "semi_#{db_size}_index", "(semi: uncached db but cached index) searching with an index") do corvet = semi_searchby_name.get car_name_to_search end - x.report("(uncached db and index) searching with an index") do + report(cars_uncached, db_size, nb_run, "uncached_#{db_size}_index", "(uncached db and index) searching with an index") do corvet = uncached_searchby_name.get car_name_to_search end + +# # Searching for data with a partition. +# car_color_to_search = ENV["CARCOLOR"] rescue "red" +# puts "" +# puts "search by partition #{car_color_to_search}: get #{ram_searchby_color.get(car_color_to_search).size} values" +# Benchmark.ips do |x| +# add_cars cars_ram, db_size +# x.report("(ram db and partition) searching with a partition") do +# corvet = ram_searchby_color.get car_color_to_search +# end +# cars_ram.empty_db +# +# add_cars cars_cached, db_size +# x.report("(cached db and partition) searching with a partition") do +# corvet = cached_searchby_color.get car_color_to_search +# end +# cars_cached.empty_db +# +# add_cars cars_semi, db_size +# x.report("(semi: uncached db but cached partition) searching with a partition") do +# corvet = semi_searchby_color.get car_color_to_search +# end +# cars_semi.empty_db +# +# add_cars cars_uncached, db_size +# x.report("(uncached db and partition) searching with a partition") do +# corvet = uncached_searchby_color.get car_color_to_search +# end +# cars_uncached.empty_db +# end +# +# # Searching for data with a tag. +# car_keyword_to_search = ENV["CARKEYWORD"] rescue "spacious" +# puts "" +# puts "search by tag #{car_keyword_to_search}: get #{ram_searchby_keywords.get(car_keyword_to_search).size} values" +# Benchmark.ips do |x| +# add_cars cars_ram, db_size +# x.report("(ram db and tag) searching with a tag") do +# corvet = ram_searchby_keywords.get car_keyword_to_search +# end +# cars_ram.empty_db +# +# add_cars cars_cached, db_size +# x.report("(cached db and tag) searching with a tag") do +# corvet = cached_searchby_keywords.get car_keyword_to_search +# end +# cars_cached.empty_db +# +# add_cars cars_semi, db_size +# x.report("(semi: uncached db but cached tag) searching with a tag") do +# corvet = semi_searchby_keywords.get car_keyword_to_search +# end +# cars_semi.empty_db +# +# add_cars cars_uncached, db_size +# x.report("(uncached db and tag) searching with a tag") do +# corvet = uncached_searchby_keywords.get car_keyword_to_search +# end +# cars_uncached.empty_db +# end + + cars_ram.rm_storage_dir + cars_cached.rm_storage_dir + cars_semi.rm_storage_dir + cars_uncached.rm_storage_dir end -# Searching for data with a partition. -puts "" -puts "search by partition #{car_color_to_search}: get #{ram_searchby_color.get(car_color_to_search).size} values" -Benchmark.ips do |x| - x.report("(ram db and partition) searching with a partition") do - corvet = ram_searchby_color.get car_color_to_search - end - - x.report("(cached db and partition) searching with a partition") do - corvet = cached_searchby_color.get car_color_to_search - end - - x.report("(semi: uncached db but cached partition) searching with a partition") do - corvet = semi_searchby_color.get car_color_to_search - end - - x.report("(uncached db and partition) searching with a partition") do - corvet = uncached_searchby_color.get car_color_to_search - end -end - -# Searching for data with a tag. -puts "" -puts "search by tag #{car_keyword_to_search}: get #{ram_searchby_keywords.get(car_keyword_to_search).size} values" -Benchmark.ips do |x| - x.report("(ram db and tag) searching with a tag") do - corvet = ram_searchby_keywords.get car_keyword_to_search - end - - x.report("(cached db and tag) searching with a tag") do - corvet = cached_searchby_keywords.get car_keyword_to_search - end - - x.report("(semi: uncached db but cached tag) searching with a tag") do - corvet = semi_searchby_keywords.get car_keyword_to_search - end - - x.report("(uncached db and tag) searching with a tag") do - corvet = uncached_searchby_keywords.get car_keyword_to_search - end -end - -cars_ram.rm_storage_dir -cars_cached.rm_storage_dir -cars_semi.rm_storage_dir -cars_uncached.rm_storage_dir - -cars_ram = DODB::RAMOnlySpecDataBase(Car).new -cars_cached = DODB::CachedSpecDataBase(Car).new -cars_semi = DODB::SpecDataBase(Car).new "-semi" -cars_uncached = DODB::SpecDataBase(Car).new - -ram_searchby_name, ram_searchby_color, ram_searchby_keywords = ram_indexes cars_ram -cached_searchby_name, cached_searchby_color, cached_searchby_keywords = cached_indexes cars_cached -semi_searchby_name, semi_searchby_color, semi_searchby_keywords = cached_indexes cars_semi -uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = uncached_indexes cars_uncached - def perform_add(storage : DODB::Storage(Car), nb_run) corvet0 = Car.new "Corvet", "red", [ "shiny", "impressive", "fast", "elegant" ] i = 0 @@ -110,20 +152,46 @@ def perform_add(storage : DODB::Storage(Car), nb_run) end end +def batch_add(nb_run) + cars_ram = DODB::RAMOnlySpecDataBase(Car).new + cars_cached = DODB::CachedSpecDataBase(Car).new + cars_semi = DODB::SpecDataBase(Car).new "-semi" + cars_uncached = DODB::SpecDataBase(Car).new + + ram_searchby_name, ram_searchby_color, ram_searchby_keywords = ram_indexes cars_ram + cached_searchby_name, cached_searchby_color, cached_searchby_keywords = cached_indexes cars_cached + semi_searchby_name, semi_searchby_color, semi_searchby_keywords = cached_indexes cars_semi + uncached_searchby_name, uncached_searchby_color, uncached_searchby_keywords = uncached_indexes cars_uncached + + avr = perform_add(cars_ram, nb_run) + puts "(ram db and indexes) add a value (average on #{nb_run} tries): #{avr}" + + avr = perform_add(cars_cached, nb_run) + puts "(cached db and indexes) add a value (average on #{nb_run} tries): #{avr}" + + avr = perform_add(cars_semi, nb_run) + puts "(uncached db but cached indexes) add a value (average on #{nb_run} tries): #{avr}" + + avr = perform_add(cars_uncached, nb_run) + puts "(uncached db and indexes) add a value (average on #{nb_run} tries): #{avr}" + + cars_ram.rm_storage_dir + cars_cached.rm_storage_dir + cars_semi.rm_storage_dir + cars_uncached.rm_storage_dir +end + +ENV["REPORT_DIR"]?.try do |report_dir| + Context.report_dir = report_dir +end +Dir.mkdir_p Context.report_dir + nb_run = 1000 -avr = perform_add(cars_ram, nb_run) -puts "(ram db and indexes) add a value (average on #{nb_run} tries): #{avr}" - -avr = perform_add(cars_cached, nb_run) -puts "(cached db and indexes) add a value (average on #{nb_run} tries): #{avr}" - -avr = perform_add(cars_semi, nb_run) -puts "(uncached db but cached indexes) add a value (average on #{nb_run} tries): #{avr}" - -avr = perform_add(cars_uncached, nb_run) -puts "(uncached db and indexes) add a value (average on #{nb_run} tries): #{avr}" - -cars_ram.rm_storage_dir -cars_cached.rm_storage_dir -cars_semi.rm_storage_dir -cars_uncached.rm_storage_dir +db_size_max = ENV["DBSIZE"].to_i rescue 1_000 +current_size = ENV["DBSIZE_START"].to_i rescue 5_000 +size_increment = ENV["DBSIZE_INCREMENT"].to_i rescue 5_000 +while current_size <= db_size_max + batch current_size, nb_run + current_size += size_increment +end +batch_add nb_run diff --git a/spec/utilities.cr b/spec/utilities.cr index e69cb28..ed77284 100644 --- a/spec/utilities.cr +++ b/spec/utilities.cr @@ -12,6 +12,21 @@ def perform_benchmark_average(ntimes : Int32, &block) sum += elapsed_time i += 1 end - sum / ntimes end + +def run_n_times(ntimes : Int32, &block) + i = 1 + durations = Array(Float64).new + while i <= ntimes + elapsed_time = perform_something &block + durations << elapsed_time.total_nanoseconds + i += 1 + end + durations +end + +# TODO +def should_nb_files(path : String, expected_nb_files : UInt32) + raise Exception.new "should_nb_files: not implemented yet" +end