From b4f09f2efd49a94aac3cd61345559d96761fa482 Mon Sep 17 00:00:00 2001 From: Karchnu Date: Sat, 6 Jun 2020 20:43:14 +0200 Subject: [PATCH] =?UTF-8?q?Naming,=20download,=20errors,=20consistency?= =?UTF-8?q?=E2=80=A6=20pretty=20much=20rewrite=20of=20the=20whole=20thing.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/requests/client.cr | 64 ++++++++++ src/common/requests/download.cr | 23 +--- src/common/requests/errors.cr | 54 ++++++++ src/common/requests/login.cr | 18 --- src/common/requests/transfer.cr | 57 ++++++--- src/common/requests/upload.cr | 20 +-- src/common/utils.cr | 43 ------- src/server/main.cr | 66 ++-------- src/server/network.cr | 1 + src/server/storage.cr | 213 ++++++++++++++++++++++++-------- src/server/storage/file_info.cr | 1 - src/structures | 45 +++++++ 12 files changed, 380 insertions(+), 225 deletions(-) create mode 100644 src/common/requests/client.cr create mode 100644 src/common/requests/errors.cr delete mode 100644 src/common/utils.cr create mode 100644 src/structures diff --git a/src/common/requests/client.cr b/src/common/requests/client.cr new file mode 100644 index 0000000..d12d875 --- /dev/null +++ b/src/common/requests/client.cr @@ -0,0 +1,64 @@ + +class FileStorage::Client + + def login(token : String) + request = FileStorage::Request::Login.new token + send request + + response = parse_message [ FileStorage::Response::Login, FileStorage::Response::Error ], read + + case response + when FileStorage::Response::Login + # Received favorites, likes, etc. + when FileStorage::Response::Error + raise "user was not logged in: #{response.reason}" + end + + response + end + + def transfer(file_info : FileInfo, count, bindata) + request = FileStorage::Request::PutChunk.new file_info, count, bindata + send request + + response = parse_message [ FileStorage::Response::PutChunk, FileStorage::Response::Error ], read + + case response + when FileStorage::Response::PutChunk + when FileStorage::Response::Error + raise "File chunk was not transfered: #{response.reason}" + end + + response + end + + def download(filedigest = nil, name = nil, tags = nil) + request = FileStorage::Request::Download.new filedigest, name, tags + send request + + response = parse_message [ FileStorage::Response::Download, FileStorage::Response::Error ], read + + case response + when FileStorage::Response::Download + when FileStorage::Response::Error + raise "Download request denied: #{response.reason}" + end + + response + end + + def upload(token : String) + request = FileStorage::Request::Upload.new token + send request + + response = parse_message [ FileStorage::Response::Upload, FileStorage::Response::Error ], read + + case response + when FileStorage::Response::Upload + when FileStorage::Response::Error + raise "Upload request failed: #{response.reason}" + end + + response + end +end diff --git a/src/common/requests/download.cr b/src/common/requests/download.cr index faf2415..4b8cd40 100644 --- a/src/common/requests/download.cr +++ b/src/common/requests/download.cr @@ -27,28 +27,11 @@ class FileStorage::Request FileStorage.requests << Download end -class FileStorage::Client - def download(filedigest = nil, name = nil, tags = nil) - request = FileStorage::Request::Download.new filedigest, name, tags - send request - - response = parse_message [ FileStorage::Response::Download, FileStorage::Response::Error ], read - - case response - when FileStorage::Response::Download - when FileStorage::Response::Error - raise "Download request denied: #{response.reason}" - end - - response - end -end - - class FileStorage::Response JSONIPC.request Download, 30 do - property mid : String - def initialize(@mid) + property mid : String + property nb_chunks : Int32 + def initialize(@mid, @nb_chunks) end end end diff --git a/src/common/requests/errors.cr b/src/common/requests/errors.cr new file mode 100644 index 0000000..d4c3c01 --- /dev/null +++ b/src/common/requests/errors.cr @@ -0,0 +1,54 @@ +class FileStorage::Errors + JSONIPC.request GenericError, 200 do + property mid : String + property reason : String + def initialize(@mid, @reason) + end + end + FileStorage.errors << GenericError + + # When uploading a chunk already present in the DB. + JSONIPC.request ChunkAlreadyUploaded, 201 do + property mid : String + property reason = "Chunk already present" + property filedigest : String + property chunk : Chunk + + def initialize(@mid, @filedigest, @chunk) + end + end + FileStorage.errors << ChunkAlreadyUploaded + + # You upload a chunk, but you are not the owner of the file. + JSONIPC.request ChunkUploadDenied, 202 do + property mid : String + property reason = "This file is not yours" + property filedigest : String + + def initialize(@mid, @filedigest) + end + end + FileStorage.errors << ChunkUploadDenied + + # When uploading a file already present in the DB. + JSONIPC.request FileExists, 203 do + property mid : String + property reason = "file already present" + property filedigest : String + + def initialize(@mid, @filedigest) + end + end + FileStorage.errors << FileExists + + # When transfering a chunk for an inexistent file. + JSONIPC.request FileDoesNotExist, 204 do + property mid : String + property reason = "file does not exist" + property filedigest : String + + def initialize(@mid, @filedigest) + end + end + FileStorage.errors << FileDoesNotExist +end diff --git a/src/common/requests/login.cr b/src/common/requests/login.cr index 36a4808..538534f 100644 --- a/src/common/requests/login.cr +++ b/src/common/requests/login.cr @@ -34,24 +34,6 @@ class FileStorage::Request FileStorage.requests << Login end -class FileStorage::Client - def login(token : String) - request = FileStorage::Request::Login.new token - send request - - response = parse_message [ FileStorage::Response::Login, FileStorage::Response::Error ], read - - case response - when FileStorage::Response::Login - # Received favorites, likes, etc. - when FileStorage::Response::Error - raise "user was not logged in: #{response.reason}" - end - - response - end -end - class FileStorage::Response JSONIPC.request Login, 5 do property mid : String diff --git a/src/common/requests/transfer.cr b/src/common/requests/transfer.cr index dcc9d20..95c81a1 100644 --- a/src/common/requests/transfer.cr +++ b/src/common/requests/transfer.cr @@ -1,11 +1,11 @@ class FileStorage::Request - JSONIPC.request Transfer, 40 do + JSONIPC.request PutChunk, 40 do property mid : String # autogenerated property filedigest : String # SHA256 digest of the entire file # Chunk: # - n : Int32 => chunk number # - on : Int32 => number of chunks - # - digest : String => 1024-byte data in base64 format + # - digest : String => digest of the chunk property chunk : Chunk # For now, just the counter in a string property data : String # base64 slice def initialize(file_info : FileInfo, count, bindata) @@ -27,38 +27,59 @@ class FileStorage::Request user_data = filestoraged.get_user_data user.uid - filestoraged.storage.transfer self, user_data + filestoraged.storage.write_chunk self, user_data rescue e return Response::Error.new @mid, "unauthorized" end end - FileStorage.requests << Transfer -end + FileStorage.requests << PutChunk -class FileStorage::Client - def transfer(file_info : FileInfo, count, bindata) - request = FileStorage::Request::Transfer.new file_info, count, bindata - send request + JSONIPC.request GetChunk, 41 do + property mid : String # autogenerated + property filedigest : String # SHA256 digest of the entire file + property n : Int32 # chunk number - response = parse_message [ FileStorage::Response::Transfer, FileStorage::Response::Error ], read - - case response - when FileStorage::Response::Transfer - when FileStorage::Response::Error - raise "File chunk was not transfered: #{response.reason}" + def initialize(@filedigest, @n) + @mid = UUID.random.to_s end - response + def handle(filestoraged : FileStorage::Service, event : IPC::Event::Events) + user = filestoraged.get_logged_user event + + raise Exception.new "unauthorized" if user.nil? + + # FIXME: Maybe this should be moved to FileStorage::Service + fd = event.connection.fd + + user_data = filestoraged.get_user_data user.uid + + filestoraged.storage.read_chunk self, user_data + rescue e + return Response::Error.new @mid, "unauthorized" + end end + FileStorage.requests << GetChunk end - class FileStorage::Response - JSONIPC.request Transfer, 40 do + JSONIPC.request PutChunk, 40 do property mid : String property file_digest : String property n : Int32 # chunk number def initialize(@mid, @file_digest, @n) end end + + JSONIPC.request GetChunk, 41 do + property mid : String + property file_digest : String + # Chunk: + # - n : Int32 => chunk number + # - on : Int32 => number of chunks + # - digest : String => digest of the chunk + property chunk : Chunk # Currently: info about the chunk + property data : String # base64 slice + def initialize(@mid, @file_digest, @chunk, @data) + end + end end diff --git a/src/common/requests/upload.cr b/src/common/requests/upload.cr index bacfe0a..5828c69 100644 --- a/src/common/requests/upload.cr +++ b/src/common/requests/upload.cr @@ -25,23 +25,6 @@ class FileStorage::Request FileStorage.requests << Upload end -class FileStorage::Client - def upload(token : String) - request = FileStorage::Request::Upload.new token - send request - - response = parse_message [ FileStorage::Response::Upload, FileStorage::Response::Error ], read - - case response - when FileStorage::Response::Upload - when FileStorage::Response::Error - raise "Upload request failed: #{response.reason}" - end - - response - end -end - class FileStorage::Response JSONIPC.request Upload, 20 do property mid : String @@ -49,10 +32,11 @@ class FileStorage::Response def initialize(@mid, @path) end end + FileStorage.responses << Upload # JSONIPC.request Responses, 100 do # property mid : String -# property responses : Array(Response) # a response for each request +# property responses : Array(Response | Errors) # a response for each request # property response : String # property reason : String? # diff --git a/src/common/utils.cr b/src/common/utils.cr deleted file mode 100644 index 8a4b4c4..0000000 --- a/src/common/utils.cr +++ /dev/null @@ -1,43 +0,0 @@ - -def remove_chunk_from_db(transfer_info : TransferInfo, chunk_number : Int32) - transfer_info.chunks.delete chunk_number - Context.db_by_filedigest.update transfer_info.file_info.digest, transfer_info -end - -def write_a_chunk(userid : String, file_info : FileStorage::FileInfo, chunk_number : Int32, data : Bytes) - - # storage: Context.storage_directory/userid/fileuuid.bin - dir = "#{Context.storage_directory}/#{userid}" - - FileUtils.mkdir_p dir - - path = "#{dir}/#{file_info.digest}.bin" - # Create file if non existant - File.open(path, "a+") do |file| - end - - # Write in it - File.open(path, "ab") do |file| - offset = chunk_number * FileStorage.message_buffer_size - file.seek(offset, IO::Seek::Set) - file.write data - end -end - -### # TODO: -### # why getting the file_info here? We could check for the transfer_info right away -### # it has more info, and we'll get it later eventually -### -### file_info = nil -### begin -### file_info = user.uploads.select do |v| -### v.file.digest == message.filedigest -### end.first.file -### -### pp! file_info -### rescue e : IndexError -### puts "No recorded upload request for file #{message.filedigest}" -### -### rescue e -### puts "Unexpected error: #{e}" -### end diff --git a/src/server/main.cr b/src/server/main.cr index 74b1d6c..700bd55 100644 --- a/src/server/main.cr +++ b/src/server/main.cr @@ -19,23 +19,14 @@ require "../common/colors" # upload or download its files. # TODO: -# * Authd integration. # * Elegantly handling errors. -# * Store the file, /files/userid/UID.bin for example: /files/1002/UID.bin. -# * Metadata should be in a dodb. -# /storage/partitions/by_uid/UID.json -> content: -# data: /files/uid/UID.bin (storing raw files) -# uid: 1002 -# name: "The File About Things" -# size: 1500 -# tags: thing1 thing2 +# * Store the file, @root/files/UID for example: ./files/UID. # * Knowing which parts of the files are still to be sent. # * Rights. # * Quotas. require "./storage.cr" require "./network.cr" -#### require "./sample.cr" require "dodb" require "json" @@ -58,12 +49,9 @@ class FileStorage::Service < IPC::Service @auth : AuthD::Client @auth_key : String - def initialize(storage_directory, file_info_directory, @auth_key) - # Data and metadata storage directories. - # storage_directory : String - # file_info_directory : String - - @storage = FileStorage::Storage.new storage_directory, file_info_directory + def initialize(storage_directory, @auth_key) + # Data and metadata storage directory. + @storage = FileStorage::Storage.new storage_directory @logged_users = Hash(Int32, AuthD::User::Public).new @logged_connections = Hash(Int32, IPC::Connection).new @@ -164,14 +152,14 @@ class FileStorage::Service < IPC::Service info "<< #{request.class.name.sub /^FileStorage::Request::/, ""}" response = request.handle self, event + response_type = response.class.name - if response.is_a? FileStorage::Response::Error - warning ">> #{response.class.name.sub /^FileStorage::Response::/, ""} (#{response.reason})" + if response.responds_to?(:reason) + warning ">> #{response_type.sub /^FileStorage::Errors::/, ""} (#{response.reason})" else info ">> #{response.class.name.sub /^FileStorage::Response::/, ""}" end - ################################################################# # THERE START ################################################################# @@ -229,7 +217,7 @@ class FileStorage::Service < IPC::Service # raise "The user isn't recorded in the users_status structure" # end # -# transfer = FileStorage::Transfer.from_json( +# transfer = FileStorage::PutChunk.from_json( # String.new event.message.payload # ) # response = hdl_transfer transfer, Context.users_status[userid] @@ -254,10 +242,6 @@ class FileStorage::Service < IPC::Service warning "unhandled IPC event: #{event.class}" end - # TODOOOOOOOOOO TODO FIXME - - - rescue exception error "exception: #{typeof(exception)} - #{exception.message}" end @@ -265,25 +249,18 @@ class FileStorage::Service < IPC::Service end def self.from_cli - storage_directory = "file-data-storage" - file_info_directory = "file-info-storage" + storage_directory = "files/" key = "nico-nico-nii" # Default authd key, as per the specs. :eyes: OptionParser.parse do |parser| parser.banner = "usage: filestoraged [options]" - parser.on "-d storage-directory", - "--storage-directory storage-directory", - "The directory where to put uploaded files." do |opt| + parser.on "-r root-directory", + "--root-directory dir", + "The root directory for FileStoraged." do |opt| storage_directory = opt end - parser.on "-m metadata-directory", - "--metadata-directory storage-directory", - "The directory where to put metadata of uploaded files." do |opt| - file_info_directory = opt - end - parser.on "-h", "--help", "Displays this help and exits." do @@ -299,25 +276,8 @@ class FileStorage::Service < IPC::Service end end - ::FileStorage::Service.new storage_directory, file_info_directory, key + ::FileStorage::Service.new storage_directory, key end - - # TODO: this will probably be dropped at some point. -# def do_response(event : IPC::Event::Message, -# response : FileStorage::Message) -# -# case response -# when FileStorage::Response -# event.connection.send FileStorage::MessageType::Response.to_u8, response.to_json -# when FileStorage::Responses -# event.connection.send FileStorage::MessageType::Responses.to_u8, response.to_json -# when FileStorage::Error -# event.connection.send FileStorage::MessageType::Error.to_u8, response.to_json -# else -# puts "response should not happen: #{response}" -# pp! response -# end -# end end FileStorage::Service.from_cli.run diff --git a/src/server/network.cr b/src/server/network.cr index 4e8c06d..0a6d422 100644 --- a/src/server/network.cr +++ b/src/server/network.cr @@ -36,6 +36,7 @@ end class FileStorage class_getter requests = [] of JSONIPC.class class_getter responses = [] of JSONIPC.class + class_getter errors = [] of JSONIPC.class end class FileStorage::Client < IPC::Client diff --git a/src/server/storage.cr b/src/server/storage.cr index 950e39b..487a7ed 100644 --- a/src/server/storage.cr +++ b/src/server/storage.cr @@ -22,8 +22,8 @@ end # XXX TODO FIXME: architectural questions # Why keeping upload and download requests? -# The server can be just for uploads, delegating downloads to HTTP. -# In environment without HTTP integration, this could still be pertinent. +# The server can be just for uploads, delegating downloads to HTTP, +# but in environment without HTTP integration, this could still be relevant. class FileStorage::Storage property db : DODB::DataBase(TransferInfo) @@ -33,47 +33,77 @@ class FileStorage::Storage property db_by_owner : DODB::Partition(TransferInfo) property db_by_tags : DODB::Tags(TransferInfo) - # Where to store the data. + # Where to store data: files, users informations, files metadata. property root : String getter user_data : DODB::DataBase(UserData) getter user_data_per_user : DODB::Index(UserData) - def initialize(@root, file_info_directory) - @db = DODB::DataBase(TransferInfo).new file_info_directory + # FileStorage::Storage constructor takes a `root directory` as parameter + # which is used to create 3 sub-dirs: + # - files/ : actual files stored on the file-system + # - meta/ : DODB TransferInfo + # - users/ : DODB UserData (for later use: quotas, rights) + + def initialize(@root) + @db = DODB::DataBase(TransferInfo).new "'#{@root}/meta" + + # Where to store uploaded files. + FileUtils.mkdir_p "#{@root}/files" # Create indexes, partitions and tags objects. @db_by_filedigest = @db.new_index "filedigest", &.file_info.digest @db_by_owner = @db.new_partition "owner", &.owner.to_s @db_by_tags = @db.new_tags "tags", &.file_info.tags - @user_data = DODB::DataBase(UserData).new "#{@root}/user-data" + @user_data = DODB::DataBase(UserData).new "#{@root}/users" @user_data_per_user = @user_data.new_index "uid", &.uid.to_s end + # Path part of the URL. + def get_path(file_digest : String) + "/files/#{file_digest}" + end + + # Path on the file-system. + def get_fs_path(file_digest : String) + "#{@root}#{get_path file_digest}" + end + # Reception of a file chunk. - def transfer(message : FileStorage::Request::Transfer, user : UserData) + def write_chunk(message : FileStorage::Request::PutChunk, user : UserData) # We received a message containing a chunk of file. mid = message.mid mid ||= "no message id" - # Get the transfer info from the db + # Get the transfer info from the db. transfer_info = @db_by_filedigest.get message.filedigest + file_digest = transfer_info.file_info.digest + if transfer_info.nil? - # The user has to send an upload request before sending anything. - # If not the case, it should be discarded. raise "file not recorded" end - chunk_number = message.chunk.n + if transfer_info.nil? + # The user did not ask permission to upload the file (upload request). + return FileStorage::Errors::FileDoesNotExist.new mid, file_digest + end + # Verify the user had a granted upload request. + if transfer_info.owner != user.uid + return FileStorage::Errors::ChunkUploadDenied.new mid, file_digest + end + + # TODO: this should be dynamic (per file) in the future. + chunk_size = FileStorage.message_buffer_size + chunk_number = message.chunk.n data = Base64.decode message.data # Verify that the chunk sent was really missing. if transfer_info.chunks.select do |v| v == chunk_number end.size == 1 - write_a_chunk user.uid.to_s, transfer_info.file_info, chunk_number, data + write_a_chunk file_digest, chunk_size, chunk_number, data else # TODO: send the remaining chunks to upload. raise "non existent chunk or already uploaded" @@ -84,10 +114,49 @@ class FileStorage::Storage # TODO: verify the digest, if no more chunks. digest = transfer_info.file_info.digest - FileStorage::Response::Transfer.new mid, digest, chunk_number + FileStorage::Response::PutChunk.new mid, digest, chunk_number rescue e - puts "Error handling transfer: #{e.message}" - FileStorage::Response::Error.new mid.not_nil!, "Unexpected error: #{e.message}" + puts "Error handling write_chunk: #{e.message}" + FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error: #{e.message}" + end + + # Provide a file chunk to the client. + def read_chunk(message : FileStorage::Request::GetChunk, user : UserData) + + # We received a message containing a chunk of file. + mid = message.mid + mid ||= "no message id" + + file_digest = message.filedigest + # TODO: this should be dynamic (per file) in the future. + chunk_size = FileStorage.message_buffer_size + chunk_number = message.n + transfer_info = @db_by_filedigest.get file_digest + + if transfer_info.nil? + # The user is asking for an inexistant file. + return FileStorage::Errors::FileDoesNotExist.new mid, file_digest + end + + # Verify that the chunk is already present. + if transfer_info.chunks.select do |v| v == chunk_number end.size != 0 + raise "non existent chunk or not yet uploaded" + end + + # b64 data + data = read_a_chunk file_digest, chunk_size, chunk_number + b64_encoded_data = Base64.encode data + + # whole file digest + digest = transfer_info.file_info.digest + + # about the transfered chunk + chunk = Chunk.new chunk_number, transfer_info.file_info.nb_chunks, b64_encoded_data + + FileStorage::Response::GetChunk.new mid, digest, chunk, b64_encoded_data + rescue e + puts "Error handling read_chunk: #{e.message}" + FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error: #{e.message}" end # the client sent an upload request @@ -99,62 +168,101 @@ class FileStorage::Storage puts "hdl upload: mid=#{request.mid}" pp! request + # The final path of the file. + file_digest = request.file.digest + path = get_path file_digest + # TODO: verify the rights and quotas of the user # file_info attributes: name, size, nb_chunks, digest, tags - # First: check if the file already exists - transfer_info = @db_by_filedigest.get? request.file.digest + # First: check if the file already exists. + transfer_info = @db_by_filedigest.get? file_digest if transfer_info.nil? # In case file informations aren't already registered - # which is normal at this point - transfer_info = TransferInfo.new user.uid, request.file - @db << transfer_info + # which is normal at this point. + @db << TransferInfo.new user.uid, request.file else - # File information already exists, request may be duplicated - # In this case: ignore the upload request + # File information already exists, request may be duplicated, + # in this case: ignore the upload request. + return FileStorage::Errors::FileExists.new mid, file_digest end - path = "/files/#{user.uid}/#{request.file.digest}.bin" + # TODO: store upload request in UserData? + FileStorage::Response::Upload.new request.mid, path rescue e - puts "Error handling upload: #{e.message}" - FileStorage::Response::Error.new mid.not_nil!, "Unexpected error: #{e.message}" + puts "Error handling upload request: #{e.message}" + FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error in upload request: #{e.message}" end - # TODO # The client sent a download request. def download(request : FileStorage::Request::Download, user : UserData) - puts "hdl download: mid=#{request.mid}" - pp! request + mid = request.mid + mid ||= "no message id" + puts "hdl download: mid=#{mid}" - FileStorage::Response::Download.new request.mid + unless (file_digest = request.filedigest).nil? + unless (file_transfer = @db_by_filedigest.get? file_digest).nil? + # The file exists. + # TODO: verify rights here. + + # This is acceptation. + # Return some useful values: number of chunks. + return FileStorage::Response::Download.new mid, file_transfer.file_info.nb_chunks + else + return FileStorage::Errors::GenericError.new mid, "Unknown file digest: #{file_digest}" + end + end + + # TODO: search a file by its name and tags + + # TODO: store download request in UserData? + + # Should have returned by now: file wasn't found. + FileStorage::Errors::GenericError.new mid, "File not found with provided parameters." + rescue e + puts "Error handling download request: #{e.message}" + FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error in download request: #{e.message}" end # Entry point for request management # Each request should have a response. # Then, responses are sent in a single message. - def requests(requests : Array(FileStorage::Request), - user : UserData, - event : IPC::Event::Message) : Array(FileStorage::Response) +# def requests(requests : Array(FileStorage::Request), +# user : UserData, +# event : IPC::Event::Message) : Array(FileStorage::Response) +# +# puts "hdl request" +# responses = Array(FileStorage::Response | FileStorage::Errors).new +# +# requests.each do |request| +# case request +# when FileStorage::DownloadRequest +# responses << download request, user +# when FileStorage::UploadRequest +# responses << upload request, user +# else +# raise "request not understood" +# end +# +# puts +# end +# +# responses +# end - puts "hdl request" - responses = Array(FileStorage::Response).new + def read_a_chunk(file_digest : String, chunk_size : Int32, chunk_number : Int32) + offset = chunk_number * chunk_size + buffer_data = Bytes.new chunk_size - requests.each do |request| - case request - when FileStorage::DownloadRequest - responses << download request, user - when FileStorage::UploadRequest - responses << upload request, user - else - raise "request not understood" - end - - puts + path = get_fs_path file_digest + real_size = 0 + File.open(path, "rb").read_at offset, chunk_size do |buffer| + real_size = buffer.read buffer_data end - responses + buffer_data[0..real_size-1] end def remove_chunk_from_db(transfer_info : TransferInfo, chunk_number : Int32) @@ -162,24 +270,21 @@ class FileStorage::Storage @db_by_filedigest.update transfer_info.file_info.digest, transfer_info end - def write_a_chunk(userid : String, - file_info : FileStorage::FileInfo, + def write_a_chunk(digest : String, + chunk_size : Int32, chunk_number : Int32, data : Bytes) - # storage: @root/files/userid/fileuuid.bin - dir = "#{@root}/files/#{userid}" + # storage: @root/files/digest + path = get_fs_path digest - FileUtils.mkdir_p dir - - path = "#{dir}/#{file_info.digest}.bin" # Create file if non existant File.open(path, "a+") do |file| end # Write in it File.open(path, "ab") do |file| - offset = chunk_number * FileStorage.message_buffer_size + offset = chunk_number * chunk_size file.seek(offset, IO::Seek::Set) file.write data end diff --git a/src/server/storage/file_info.cr b/src/server/storage/file_info.cr index cf69ed6..0c4f9c1 100644 --- a/src/server/storage/file_info.cr +++ b/src/server/storage/file_info.cr @@ -24,7 +24,6 @@ class FileStorage # private function def self.data_digest(data : Bytes) - iodata = IO::Memory.new data, false buffer = Bytes.new FileStorage.file_reading_buffer_size diff --git a/src/structures b/src/structures new file mode 100644 index 0000000..28db681 --- /dev/null +++ b/src/structures @@ -0,0 +1,45 @@ +class FileStorage + def self.message_buffer_size + def self.file_reading_buffer_size + def self.data_digest(data : Bytes) + def self.file_digest(file : File) +end + +class FileStorage::Chunk + n : Int32 # chunk's number + on : Int32 # number of chunks + digest : String # digest of the current chunk + + initialize(@n, @on, data) +end + +class FileStorage::FileInfo + name : String + size : UInt64 + nb_chunks : Int32 + digest : String + tags : Array(String) +end + +class TransferInfo + owner : Int32 + file_info : FileStorage::FileInfo + chunks : Array(Int32) + + initialize(@owner, @file_info) +end + +# Keep track of connected users and their requests. +class FileStorage::UserData + property uid : Int32 + property uploads : Array(Upload) # NOT USED. + property downloads : Array(Download) # NOT USED. + + initialize(@uid, @uploads = Array(Upload).new, @downloads = Array(Download).new) +end + +root/ + files/ : actual files + meta/ : DODB TransferInfo + users/ : DODP UserData (for later use: quotas, rights) +