From bfa43f2667e1c9bf596e3d4b2389f9555af663ca Mon Sep 17 00:00:00 2001 From: Karchnu Date: Wed, 21 Oct 2020 19:59:57 +0200 Subject: [PATCH] Upload and download OK, on both server and client. --- src/client/lib.cr | 47 ++++++++++++++++++++++++++++++++++++++++ src/client/main.cr | 43 +++++++++++++++++++++++++++++------- src/requests/download.cr | 4 ++-- src/server/main.cr | 13 ++++++++--- src/server/storage.cr | 26 +++++++++------------- 5 files changed, 104 insertions(+), 29 deletions(-) diff --git a/src/client/lib.cr b/src/client/lib.cr index 9c42606..9bff707 100644 --- a/src/client/lib.cr +++ b/src/client/lib.cr @@ -83,6 +83,53 @@ class FileStorage::Client < IPC::Client parse_message [ FileStorage::Response::Download ], read end + def get_chunks(dl_response : FileStorage::Response::Download, path : String = ".") + file_path = "#{path}/#{dl_response.file_info.name}" + Baguette::Log.debug "Getting #{file_path}" + + digest = dl_response.file_info.digest + buffer_size = FileStorage.message_buffer_size + + counter = 0 + size = 0 + + while counter < dl_response.file_info.nb_chunks + Baguette::Log.debug "Getting #{file_path}: chunk #{counter+1}/#{dl_response.file_info.nb_chunks}" + get_chunk_message = FileStorage::Request::GetChunk.new digest, counter + send_now @server_fd.not_nil!, get_chunk_message + response = parse_message [ FileStorage::Response::GetChunk ], read + # TODO: write the file + pp! response + + case response + when FileStorage::Response::GetChunk + b64_decoded_data = Base64.decode response.data + write_chunk file_path, buffer_size, response.chunk.n, b64_decoded_data + else + Baguette::Log.error "#{response}" + raise "wrong response: #{response}" + end + counter += 1 + end + end + + # Reception of a file chunk. + def write_chunk(file_path : String, + chunk_size : Int32, + offset : Int32, + data : Bytes + ) + + pp! file_path, chunk_size, offset, data.size + + # Create the file if non existant. + File.open(file_path, "a+") do |file| + file.seek (offset * chunk_size) + file.write data + end + end + + def upload(file : String) file_info : FileStorage::FileInfo File.open(file) do |f| diff --git a/src/client/main.cr b/src/client/main.cr index 816d711..693a341 100644 --- a/src/client/main.cr +++ b/src/client/main.cr @@ -21,7 +21,9 @@ class Context class_property authd_login : String = "test" class_property authd_pass : String = "test" - class_property to_transfer = Array(String).new + class_property path : String = "." + + class_property args = Array(String).new class_property service_name = "filestorage" class_property command = "unknown" @@ -29,7 +31,7 @@ end opt_unknown_args = ->(parser : OptionParser) { parser.unknown_args do |arg| - Context.to_transfer = arg + Context.args = arg end } @@ -64,6 +66,13 @@ OptionParser.parse do |parser| Context.authd_pass = pass end + parser.on "-d directory", + "--directory path", + "Path where to put downloaded files." do |path| + Context.path = path + end + + parser.on "-v verbosity", "--verbosity level", "Verbosity. From 0 to 4." do |v| @@ -85,8 +94,7 @@ def put(client : FileStorage::Client) files = [] of String - Baguette::Log.debug "files and directories to transfer" - Context.to_transfer.each do |f| + Context.args.each do |f| if File.directory? f # TODO Baguette::Log.warning "Directories not supported, for now" @@ -104,15 +112,34 @@ def put(client : FileStorage::Client) end files.each do |file| - Baguette::Log.info "upload: #{file}" - pp! client.upload file - Baguette::Log.debug "transfer" + response = client.upload file + if response.is_a?(FileStorage::Errors::FileFullyUploaded) + file_info = File.open(file) do |f| + FileStorage::FileInfo.new f + end + Baguette::Log.warning "file #{file} already uploaded, digest: #{file_info.digest}" + next + end + Baguette::Log.info "transfering: #{file}" client.transfer file end end def get(client : FileStorage::Client) - Baguette::Log.error "get command not available, yet" + Baguette::Log.warning "get command not complete, yet" + + files = Context.args + files.each do |filedigest| + response = client.download filedigest + case response + when FileStorage::Response::Download + Baguette::Log.info "downloading file #{filedigest} with #{response.file_info.nb_chunks} chunks" + client.get_chunks response, Context.path + else + Baguette::Log.error ">> #{response.class.name}" + next + end + end end diff --git a/src/requests/download.cr b/src/requests/download.cr index 4662470..572314b 100644 --- a/src/requests/download.cr +++ b/src/requests/download.cr @@ -28,8 +28,8 @@ end class FileStorage::Response IPC::JSON.message Download, 30 do property mid : String - property nb_chunks : Int32 - def initialize(@mid, @nb_chunks) + property file_info : FileInfo + def initialize(@mid, @file_info) end end end diff --git a/src/server/main.cr b/src/server/main.cr index 705a99b..74d579b 100644 --- a/src/server/main.cr +++ b/src/server/main.cr @@ -64,9 +64,9 @@ class FileStorage::Service < IPC::Server @auth : AuthD::Client @auth_key : String - def initialize(storage_directory, @auth_key) + def initialize(storage_directory, @auth_key, reindex : Bool) # Data and metadata storage directory. - @storage = FileStorage::Storage.new storage_directory + @storage = FileStorage::Storage.new storage_directory, reindex @logged_users = Hash(Int32, AuthD::User::Public).new @all_connections = Array(Int32).new @@ -204,6 +204,8 @@ class FileStorage::Service < IPC::Server key = "nico-nico-nii" # Default authd key, as per the specs. :eyes: timer = 30_000 # Default timer: 30 seconds. + reindex = false + OptionParser.parse do |parser| parser.banner = "usage: filestoraged [options]" @@ -225,6 +227,11 @@ class FileStorage::Service < IPC::Server Baguette::Context.verbosity = v.to_i end + parser.on "-I", + "--re-index", + "Reindex the database." do + reindex = true + end parser.on "-h", "--help", @@ -241,7 +248,7 @@ class FileStorage::Service < IPC::Server end end - service = ::FileStorage::Service.new storage_directory, key + service = ::FileStorage::Service.new storage_directory, key, reindex service.base_timer = timer service.timer = timer diff --git a/src/server/storage.cr b/src/server/storage.cr index 9793e2c..127dd96 100644 --- a/src/server/storage.cr +++ b/src/server/storage.cr @@ -45,7 +45,7 @@ class FileStorage::Storage # - meta/ : DODB TransferInfo # - users/ : DODB UserData (for later use: quotas, rights) - def initialize(@root) + def initialize(@root, reindex : Bool = false) @db = DODB::DataBase(TransferInfo).new "#{@root}/meta" # Where to store uploaded files. @@ -58,6 +58,11 @@ class FileStorage::Storage @user_data = DODB::DataBase(UserData).new "#{@root}/users" @user_data_per_user = @user_data.new_index "uid", &.uid.to_s + + if reindex + @db.reindex_everything! + @user_data.reindex_everything! + end end # Path part of the URL. @@ -127,9 +132,6 @@ class FileStorage::Storage digest = transfer_info.file_info.digest FileStorage::Response::PutChunk.new mid, digest, chunk_number - rescue e - Baguette::Log.error "Error handling write_chunk: #{e.message}" - FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error: #{e.message}" end # Provide a file chunk to the client. @@ -166,9 +168,6 @@ class FileStorage::Storage chunk = Chunk.new chunk_number, transfer_info.file_info.nb_chunks, b64_encoded_data FileStorage::Response::GetChunk.new mid, digest, chunk, b64_encoded_data - rescue e - Baguette::Log.error "Error handling read_chunk: #{e.message}" - FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error: #{e.message}" end # the client sent an upload request @@ -216,9 +215,6 @@ class FileStorage::Storage # TODO: store upload request in UserData? FileStorage::Response::Upload.new request.mid, path - rescue e - Baguette::Log.error "Error handling upload request: #{e.message}" - FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error in upload request: #{e.message}" end # The client sent a download request. @@ -235,7 +231,7 @@ class FileStorage::Storage # This is acceptation. # Return some useful values: number of chunks. - return FileStorage::Response::Download.new mid, file_transfer.file_info.nb_chunks + return FileStorage::Response::Download.new mid, file_transfer.file_info else return FileStorage::Errors::GenericError.new mid, "Unknown file digest: #{file_digest}" end @@ -247,9 +243,6 @@ class FileStorage::Storage # Should have returned by now: file wasn't found. FileStorage::Errors::GenericError.new mid, "File not found with provided parameters." - rescue e - Baguette::Log.error "Error handling download request: #{e.message}" - FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error in download request: #{e.message}" end # Entry point for request management @@ -283,8 +276,9 @@ class FileStorage::Storage path = get_fs_path file_digest real_size = 0 - File.open(path, "rb").read_at offset, chunk_size do |buffer| - real_size = buffer.read buffer_data + File.open(path, "rb") do |file| + file.seek offset + real_size = file.read buffer_data end buffer_data[0..real_size-1]