Baguette/filestoraged
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Upload and download OK, on both server and client.

Browse Source
This commit is contained in:
Karchnu 2020-10-21 19:59:57 +02:00
parent 3491a394a9
commit bfa43f2667
5 changed files with 104 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
src/client/lib.cr
View File

@ -83,6 +83,53 @@ class FileStorage::Client < IPC::Client
parse_message [ FileStorage::Response::Download ], read
end
def get_chunks(dl_response : FileStorage::Response::Download, path : String = ".")
file_path = "#{path}/#{dl_response.file_info.name}"
Baguette::Log.debug "Getting #{file_path}"
digest = dl_response.file_info.digest
buffer_size = FileStorage.message_buffer_size
counter = 0
size = 0
while counter < dl_response.file_info.nb_chunks
Baguette::Log.debug "Getting #{file_path}: chunk #{counter+1}/#{dl_response.file_info.nb_chunks}"
get_chunk_message = FileStorage::Request::GetChunk.new digest, counter
send_now @server_fd.not_nil!, get_chunk_message
response = parse_message [ FileStorage::Response::GetChunk ], read
# TODO: write the file
pp! response
case response
when FileStorage::Response::GetChunk
b64_decoded_data = Base64.decode response.data
write_chunk file_path, buffer_size, response.chunk.n, b64_decoded_data
else
Baguette::Log.error "#{response}"
raise "wrong response: #{response}"
end
counter += 1
end
end
# Reception of a file chunk.
def write_chunk(file_path : String,
chunk_size : Int32,
offset : Int32,
data : Bytes
)
pp! file_path, chunk_size, offset, data.size
# Create the file if non existant.
File.open(file_path, "a+") do |file|
file.seek (offset * chunk_size)
file.write data
end
end
def upload(file : String)
file_info : FileStorage::FileInfo
File.open(file) do |f|

43
src/client/main.cr
View File

@ -21,7 +21,9 @@ class Context
class_property authd_login : String = "test"
class_property authd_pass : String = "test"
class_property to_transfer = Array(String).new
class_property path : String = "."
class_property args = Array(String).new
class_property service_name = "filestorage"
class_property command = "unknown"
@ -29,7 +31,7 @@ end
opt_unknown_args = ->(parser : OptionParser) {
parser.unknown_args do |arg|
Context.to_transfer = arg
Context.args = arg
end
}
@ -64,6 +66,13 @@ OptionParser.parse do |parser|
Context.authd_pass = pass
end
parser.on "-d directory",
"--directory path",
"Path where to put downloaded files." do |path|
Context.path = path
end
parser.on "-v verbosity",
"--verbosity level",
"Verbosity. From 0 to 4." do |v|
@ -85,8 +94,7 @@ def put(client : FileStorage::Client)
files = [] of String
Baguette::Log.debug "files and directories to transfer"
Context.to_transfer.each do |f|
Context.args.each do |f|
if File.directory? f
# TODO
Baguette::Log.warning "Directories not supported, for now"
@ -104,15 +112,34 @@ def put(client : FileStorage::Client)
end
files.each do |file|
Baguette::Log.info "upload: #{file}"
pp! client.upload file
Baguette::Log.debug "transfer"
response = client.upload file
if response.is_a?(FileStorage::Errors::FileFullyUploaded)
file_info = File.open(file) do |f|
FileStorage::FileInfo.new f
end
Baguette::Log.warning "file #{file} already uploaded, digest: #{file_info.digest}"
next
end
Baguette::Log.info "transfering: #{file}"
client.transfer file
end
end
def get(client : FileStorage::Client)
Baguette::Log.error "get command not available, yet"
Baguette::Log.warning "get command not complete, yet"
files = Context.args
files.each do |filedigest|
response = client.download filedigest
case response
when FileStorage::Response::Download
Baguette::Log.info "downloading file #{filedigest} with #{response.file_info.nb_chunks} chunks"
client.get_chunks response, Context.path
else
Baguette::Log.error ">> #{response.class.name}"
next
end
end
end

4
src/requests/download.cr
View File

@ -28,8 +28,8 @@ end
class FileStorage::Response
IPC::JSON.message Download, 30 do
property mid : String
property nb_chunks : Int32
def initialize(@mid, @nb_chunks)
property file_info : FileInfo
def initialize(@mid, @file_info)
end
end
end

13
src/server/main.cr
View File

@ -64,9 +64,9 @@ class FileStorage::Service < IPC::Server
@auth : AuthD::Client
@auth_key : String
def initialize(storage_directory, @auth_key)
def initialize(storage_directory, @auth_key, reindex : Bool)
# Data and metadata storage directory.
@storage = FileStorage::Storage.new storage_directory
@storage = FileStorage::Storage.new storage_directory, reindex
@logged_users = Hash(Int32, AuthD::User::Public).new
@all_connections = Array(Int32).new
@ -204,6 +204,8 @@ class FileStorage::Service < IPC::Server
key = "nico-nico-nii" # Default authd key, as per the specs. :eyes:
timer = 30_000 # Default timer: 30 seconds.
reindex = false
OptionParser.parse do |parser|
parser.banner = "usage: filestoraged [options]"
@ -225,6 +227,11 @@ class FileStorage::Service < IPC::Server
Baguette::Context.verbosity = v.to_i
end
parser.on "-I",
"--re-index",
"Reindex the database." do
reindex = true
end
parser.on "-h",
"--help",
@ -241,7 +248,7 @@ class FileStorage::Service < IPC::Server
end
end
service = ::FileStorage::Service.new storage_directory, key
service = ::FileStorage::Service.new storage_directory, key, reindex
service.base_timer = timer
service.timer = timer

26
src/server/storage.cr
View File

@ -45,7 +45,7 @@ class FileStorage::Storage
# - meta/ : DODB TransferInfo
# - users/ : DODB UserData (for later use: quotas, rights)
def initialize(@root)
def initialize(@root, reindex : Bool = false)
@db = DODB::DataBase(TransferInfo).new "#{@root}/meta"
# Where to store uploaded files.
@ -58,6 +58,11 @@ class FileStorage::Storage
@user_data = DODB::DataBase(UserData).new "#{@root}/users"
@user_data_per_user = @user_data.new_index "uid", &.uid.to_s
if reindex
@db.reindex_everything!
@user_data.reindex_everything!
end
end
# Path part of the URL.
@ -127,9 +132,6 @@ class FileStorage::Storage
digest = transfer_info.file_info.digest
FileStorage::Response::PutChunk.new mid, digest, chunk_number
rescue e
Baguette::Log.error "Error handling write_chunk: #{e.message}"
FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error: #{e.message}"
end
# Provide a file chunk to the client.
@ -166,9 +168,6 @@ class FileStorage::Storage
chunk = Chunk.new chunk_number, transfer_info.file_info.nb_chunks, b64_encoded_data
FileStorage::Response::GetChunk.new mid, digest, chunk, b64_encoded_data
rescue e
Baguette::Log.error "Error handling read_chunk: #{e.message}"
FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error: #{e.message}"
end
# the client sent an upload request
@ -216,9 +215,6 @@ class FileStorage::Storage
# TODO: store upload request in UserData?
FileStorage::Response::Upload.new request.mid, path
rescue e
Baguette::Log.error "Error handling upload request: #{e.message}"
FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error in upload request: #{e.message}"
end
# The client sent a download request.
@ -235,7 +231,7 @@ class FileStorage::Storage
# This is acceptation.
# Return some useful values: number of chunks.
return FileStorage::Response::Download.new mid, file_transfer.file_info.nb_chunks
return FileStorage::Response::Download.new mid, file_transfer.file_info
else
return FileStorage::Errors::GenericError.new mid, "Unknown file digest: #{file_digest}"
end
@ -247,9 +243,6 @@ class FileStorage::Storage
# Should have returned by now: file wasn't found.
FileStorage::Errors::GenericError.new mid, "File not found with provided parameters."
rescue e
Baguette::Log.error "Error handling download request: #{e.message}"
FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error in download request: #{e.message}"
end
# Entry point for request management
@ -283,8 +276,9 @@ class FileStorage::Storage
path = get_fs_path file_digest
real_size = 0
File.open(path, "rb").read_at offset, chunk_size do |buffer|
real_size = buffer.read buffer_data
File.open(path, "rb") do |file|
file.seek offset
real_size = file.read buffer_data
end
buffer_data[0..real_size-1]