From 9375f5ff50712e42c3cd2a988ade7972a0fcc2b1 Mon Sep 17 00:00:00 2001 From: Philippe PITTOLI Date: Thu, 30 Jan 2020 01:27:06 +0100 Subject: [PATCH] tests, code split, better transfer handler --- src/common/utils.cr | 43 +++++++++++++++++++++++++++ src/server/context.cr | 3 +- src/server/handlers.cr | 60 ++++++++++++-------------------------- src/tests/context-tests.cr | 36 +++++++++++++++++++++-- src/tests/dodb-tests.cr | 49 +++++++++++++++++++++++++++++++ 5 files changed, 145 insertions(+), 46 deletions(-) create mode 100644 src/common/utils.cr create mode 100644 src/tests/dodb-tests.cr diff --git a/src/common/utils.cr b/src/common/utils.cr new file mode 100644 index 0000000..36b4aa7 --- /dev/null +++ b/src/common/utils.cr @@ -0,0 +1,43 @@ + +def remove_chunk_from_db(transfer_info : TransferInfo, chunk_number : Int32) + transfer_info.chunks.delete chunk_number + Context.db_by_filedigest.update transfer_info.file_info.digest, transfer_info +end + +def write_a_chunk(userid : String, file_info : FileInfo, chunk_number : Int32, data : Bytes) + + # storage: Context.storage_directory/userid/fileuuid.bin + dir = "#{Context.storage_directory}/#{userid}" + + FileUtils.mkdir_p dir + + path = "#{dir}/#{file_info.digest}.bin" + # Create file if non existant + File.open(path, "a+") do |file| + end + + # Write in it + File.open(path, "ab") do |file| + offset = chunk_number * FileStorage.message_buffer_size + file.seek(offset, IO::Seek::Set) + file.write data + end +end + +### # TODO: +### # why getting the file_info here? We could check for the transfer_info right away +### # it has more info, and we'll get it later eventually +### +### file_info = nil +### begin +### file_info = user.uploads.select do |v| +### v.file.digest == message.filedigest +### end.first.file +### +### pp! file_info +### rescue e : IndexError +### puts "No recorded upload request for file #{message.filedigest}" +### +### rescue e +### puts "Unexpected error: #{e}" +### end diff --git a/src/server/context.cr b/src/server/context.cr index d547d28..2bba0eb 100644 --- a/src/server/context.cr +++ b/src/server/context.cr @@ -25,7 +25,7 @@ class TransferInfo property chunks : Array(Int32) def initialize(@owner, @file_info) - @chunks = [0...@file_info.nb_chunks] + @chunks = (0...@file_info.nb_chunks).to_a end end @@ -43,7 +43,6 @@ class Context def self.db_reconnect # In case file_info_directory changes: database reinstanciation - @@db = DODB::DataBase(TransferInfo).new @@file_info_directory # recreate indexes, partitions and tags objects, too diff --git a/src/server/handlers.cr b/src/server/handlers.cr index 10cf503..4028514 100644 --- a/src/server/handlers.cr +++ b/src/server/handlers.cr @@ -2,6 +2,13 @@ require "dodb" require "base64" +require "../common/utils" + +# XXX TODO FIXME: architectural questions +# wonder why I should keep the user upload and download requests +# the server can be just for uploads, delegating downloads to HTTP + + # reception of a file chunk def hdl_transfer(message : FileStorage::Transfer, user : User) : FileStorage::Response @@ -10,56 +17,25 @@ def hdl_transfer(message : FileStorage::Transfer, user : User) : FileStorage::Re mid = message.mid mid ||= "no message id" - # pp! message + # Get the transfer info from the db + transfer_info = Context.db_by_filedigest.get message.filedigest - file_info = nil - begin - file_info = user.uploads.select do |v| - v.file.digest == message.filedigest - end.first.file - - pp! file_info - rescue e : IndexError - puts "No recorded upload request for file #{message.filedigest}" - - rescue e - puts "Unexpected error: #{e}" + # TODO: if we don't have the information + if transfer_info.nil? + # TODO end - # Get the transfer info from the db - # is the file info recorded? - by_digest = Context.db.get_index "filedigest" - transfer_info = by_digest.get message.filedigest - - if - - by_owner = Context.db.get_partition "owner" - pp! by_owner + # TODO: verify that the chunk sent was really missing + chunk_number = message.chunk.n - 1 # TODO: verify the digest - # storage: Context.storage_directory/userid/fileuuid.bin - dir = "#{Context.storage_directory}/#{user.uid}" - - FileUtils.mkdir_p dir - - path = "#{dir}/#{file_info.digest}.bin" - # Create file if non existant - File.open(path, "a+") do |file| - end - - # Write in it - File.open(path, "ab") do |file| - # TODO: store the file - offset = (message.chunk.n - 1) * FileStorage.message_buffer_size - file.seek(offset, IO::Seek::Set) - data = Base64.decode message.data - file.write data - end + data = Base64.decode message.data + write_a_chunk user.uid.to_s, transfer_info.file_info, chunk_number, data # TODO: register the file with dodb, with its tags - - Context.db << + + remove_chunk_from_db transfer_info, message.chunk.n FileStorage::Response.new mid, "Ok" diff --git a/src/tests/context-tests.cr b/src/tests/context-tests.cr index 65c692a..987d483 100644 --- a/src/tests/context-tests.cr +++ b/src/tests/context-tests.cr @@ -2,8 +2,12 @@ require "../common/filestorage.cr" require "ipc" require "option_parser" +require "../server/context.cr" + filename = "./README.md" +tags = "readme example" + OptionParser.parse do |parser| parser.on "-f file-to-transfer", "--file to-transfer", @@ -11,11 +15,39 @@ OptionParser.parse do |parser| filename = opt end + parser.on "-d database-directory", + "--db-dir directory", + "DB directory" do |opt| + Context.file_info_directory = opt + Context.db_reconnect + end + + parser.on "-t tags", + "--tags tags", + "Tags, example: 'fruit bio comestible'" do |opt| + tags = opt + end + parser.unknown_args do |args| pp! args end end -require "../server/context.cr" - pp! Context + +fileinfo : FileStorage::FileInfo? = nil + +File.open(filename) do |file| + fileinfo = FileStorage::FileInfo.new file, tags.split(' ') +end + +pp! fileinfo + +transfer_info = TransferInfo.new 1000, fileinfo.not_nil! + +Context.db << transfer_info + +Context.db.each do |ti| + pp! ti +end + diff --git a/src/tests/dodb-tests.cr b/src/tests/dodb-tests.cr new file mode 100644 index 0000000..6786120 --- /dev/null +++ b/src/tests/dodb-tests.cr @@ -0,0 +1,49 @@ +require "dodb" +require "json" +require "../common/filestorage.cr" + +# this is a copy of User and TransferInfo classes from src/server/context.cr +class User + property uid : Int32 + property token : FileStorage::Token + property uploads : Array(FileStorage::UploadRequest) + property downloads : Array(FileStorage::DownloadRequest) + + def initialize(@token, + @uploads = Array(FileStorage::UploadRequest).new, + @downloads = Array(FileStorage::DownloadRequest).new) + @uid = token.uid + end +end + +class TransferInfo + include JSON::Serializable + + property owner : Int32 + property file_info : FileStorage::FileInfo + property chunks : Hash(Int32, Bool) + + def initialize(@owner, @file_info) + @chunks = Hash(Int32, Bool).new + @file_info.nb_chunks.times do |n| + @chunks[n] = false + end + end +end + + +file_info_directory = "./file-infos" + +def init_db(file_info_directory : String) + db = DODB::DataBase(TransferInfo).new file_info_directory + + # search file informations by their index, owner and tags + pp! db_by_filedigest = db.new_index "filedigest", &.file_info.digest + pp! db_by_owner = db.new_partition "owner", &.owner.to_s + pp! db_by_tags = db.new_tags "tags", &.file_info.tags.not_nil! + + db +end + +db = init_db file_info_directory +pp! db