Upload and download OK, on both server and client.
parent
3491a394a9
commit
bfa43f2667
|
|
@ -83,6 +83,53 @@ class FileStorage::Client < IPC::Client
|
|||
parse_message [ FileStorage::Response::Download ], read
|
||||
end
|
||||
|
||||
def get_chunks(dl_response : FileStorage::Response::Download, path : String = ".")
|
||||
file_path = "#{path}/#{dl_response.file_info.name}"
|
||||
Baguette::Log.debug "Getting #{file_path}"
|
||||
|
||||
digest = dl_response.file_info.digest
|
||||
buffer_size = FileStorage.message_buffer_size
|
||||
|
||||
counter = 0
|
||||
size = 0
|
||||
|
||||
while counter < dl_response.file_info.nb_chunks
|
||||
Baguette::Log.debug "Getting #{file_path}: chunk #{counter+1}/#{dl_response.file_info.nb_chunks}"
|
||||
get_chunk_message = FileStorage::Request::GetChunk.new digest, counter
|
||||
send_now @server_fd.not_nil!, get_chunk_message
|
||||
response = parse_message [ FileStorage::Response::GetChunk ], read
|
||||
# TODO: write the file
|
||||
pp! response
|
||||
|
||||
case response
|
||||
when FileStorage::Response::GetChunk
|
||||
b64_decoded_data = Base64.decode response.data
|
||||
write_chunk file_path, buffer_size, response.chunk.n, b64_decoded_data
|
||||
else
|
||||
Baguette::Log.error "#{response}"
|
||||
raise "wrong response: #{response}"
|
||||
end
|
||||
counter += 1
|
||||
end
|
||||
end
|
||||
|
||||
# Reception of a file chunk.
|
||||
def write_chunk(file_path : String,
|
||||
chunk_size : Int32,
|
||||
offset : Int32,
|
||||
data : Bytes
|
||||
)
|
||||
|
||||
pp! file_path, chunk_size, offset, data.size
|
||||
|
||||
# Create the file if non existant.
|
||||
File.open(file_path, "a+") do |file|
|
||||
file.seek (offset * chunk_size)
|
||||
file.write data
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
def upload(file : String)
|
||||
file_info : FileStorage::FileInfo
|
||||
File.open(file) do |f|
|
||||
|
|
|
|||
|
|
@ -21,7 +21,9 @@ class Context
|
|||
class_property authd_login : String = "test"
|
||||
class_property authd_pass : String = "test"
|
||||
|
||||
class_property to_transfer = Array(String).new
|
||||
class_property path : String = "."
|
||||
|
||||
class_property args = Array(String).new
|
||||
|
||||
class_property service_name = "filestorage"
|
||||
class_property command = "unknown"
|
||||
|
|
@ -29,7 +31,7 @@ end
|
|||
|
||||
opt_unknown_args = ->(parser : OptionParser) {
|
||||
parser.unknown_args do |arg|
|
||||
Context.to_transfer = arg
|
||||
Context.args = arg
|
||||
end
|
||||
}
|
||||
|
||||
|
|
@ -64,6 +66,13 @@ OptionParser.parse do |parser|
|
|||
Context.authd_pass = pass
|
||||
end
|
||||
|
||||
parser.on "-d directory",
|
||||
"--directory path",
|
||||
"Path where to put downloaded files." do |path|
|
||||
Context.path = path
|
||||
end
|
||||
|
||||
|
||||
parser.on "-v verbosity",
|
||||
"--verbosity level",
|
||||
"Verbosity. From 0 to 4." do |v|
|
||||
|
|
@ -85,8 +94,7 @@ def put(client : FileStorage::Client)
|
|||
|
||||
files = [] of String
|
||||
|
||||
Baguette::Log.debug "files and directories to transfer"
|
||||
Context.to_transfer.each do |f|
|
||||
Context.args.each do |f|
|
||||
if File.directory? f
|
||||
# TODO
|
||||
Baguette::Log.warning "Directories not supported, for now"
|
||||
|
|
@ -104,15 +112,34 @@ def put(client : FileStorage::Client)
|
|||
end
|
||||
|
||||
files.each do |file|
|
||||
Baguette::Log.info "upload: #{file}"
|
||||
pp! client.upload file
|
||||
Baguette::Log.debug "transfer"
|
||||
response = client.upload file
|
||||
if response.is_a?(FileStorage::Errors::FileFullyUploaded)
|
||||
file_info = File.open(file) do |f|
|
||||
FileStorage::FileInfo.new f
|
||||
end
|
||||
Baguette::Log.warning "file #{file} already uploaded, digest: #{file_info.digest}"
|
||||
next
|
||||
end
|
||||
Baguette::Log.info "transfering: #{file}"
|
||||
client.transfer file
|
||||
end
|
||||
end
|
||||
|
||||
def get(client : FileStorage::Client)
|
||||
Baguette::Log.error "get command not available, yet"
|
||||
Baguette::Log.warning "get command not complete, yet"
|
||||
|
||||
files = Context.args
|
||||
files.each do |filedigest|
|
||||
response = client.download filedigest
|
||||
case response
|
||||
when FileStorage::Response::Download
|
||||
Baguette::Log.info "downloading file #{filedigest} with #{response.file_info.nb_chunks} chunks"
|
||||
client.get_chunks response, Context.path
|
||||
else
|
||||
Baguette::Log.error ">> #{response.class.name}"
|
||||
next
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ end
|
|||
class FileStorage::Response
|
||||
IPC::JSON.message Download, 30 do
|
||||
property mid : String
|
||||
property nb_chunks : Int32
|
||||
def initialize(@mid, @nb_chunks)
|
||||
property file_info : FileInfo
|
||||
def initialize(@mid, @file_info)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -64,9 +64,9 @@ class FileStorage::Service < IPC::Server
|
|||
@auth : AuthD::Client
|
||||
@auth_key : String
|
||||
|
||||
def initialize(storage_directory, @auth_key)
|
||||
def initialize(storage_directory, @auth_key, reindex : Bool)
|
||||
# Data and metadata storage directory.
|
||||
@storage = FileStorage::Storage.new storage_directory
|
||||
@storage = FileStorage::Storage.new storage_directory, reindex
|
||||
|
||||
@logged_users = Hash(Int32, AuthD::User::Public).new
|
||||
@all_connections = Array(Int32).new
|
||||
|
|
@ -204,6 +204,8 @@ class FileStorage::Service < IPC::Server
|
|||
key = "nico-nico-nii" # Default authd key, as per the specs. :eyes:
|
||||
timer = 30_000 # Default timer: 30 seconds.
|
||||
|
||||
reindex = false
|
||||
|
||||
OptionParser.parse do |parser|
|
||||
parser.banner = "usage: filestoraged [options]"
|
||||
|
||||
|
|
@ -225,6 +227,11 @@ class FileStorage::Service < IPC::Server
|
|||
Baguette::Context.verbosity = v.to_i
|
||||
end
|
||||
|
||||
parser.on "-I",
|
||||
"--re-index",
|
||||
"Reindex the database." do
|
||||
reindex = true
|
||||
end
|
||||
|
||||
parser.on "-h",
|
||||
"--help",
|
||||
|
|
@ -241,7 +248,7 @@ class FileStorage::Service < IPC::Server
|
|||
end
|
||||
end
|
||||
|
||||
service = ::FileStorage::Service.new storage_directory, key
|
||||
service = ::FileStorage::Service.new storage_directory, key, reindex
|
||||
service.base_timer = timer
|
||||
service.timer = timer
|
||||
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ class FileStorage::Storage
|
|||
# - meta/ : DODB TransferInfo
|
||||
# - users/ : DODB UserData (for later use: quotas, rights)
|
||||
|
||||
def initialize(@root)
|
||||
def initialize(@root, reindex : Bool = false)
|
||||
@db = DODB::DataBase(TransferInfo).new "#{@root}/meta"
|
||||
|
||||
# Where to store uploaded files.
|
||||
|
|
@ -58,6 +58,11 @@ class FileStorage::Storage
|
|||
|
||||
@user_data = DODB::DataBase(UserData).new "#{@root}/users"
|
||||
@user_data_per_user = @user_data.new_index "uid", &.uid.to_s
|
||||
|
||||
if reindex
|
||||
@db.reindex_everything!
|
||||
@user_data.reindex_everything!
|
||||
end
|
||||
end
|
||||
|
||||
# Path part of the URL.
|
||||
|
|
@ -127,9 +132,6 @@ class FileStorage::Storage
|
|||
|
||||
digest = transfer_info.file_info.digest
|
||||
FileStorage::Response::PutChunk.new mid, digest, chunk_number
|
||||
rescue e
|
||||
Baguette::Log.error "Error handling write_chunk: #{e.message}"
|
||||
FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error: #{e.message}"
|
||||
end
|
||||
|
||||
# Provide a file chunk to the client.
|
||||
|
|
@ -166,9 +168,6 @@ class FileStorage::Storage
|
|||
chunk = Chunk.new chunk_number, transfer_info.file_info.nb_chunks, b64_encoded_data
|
||||
|
||||
FileStorage::Response::GetChunk.new mid, digest, chunk, b64_encoded_data
|
||||
rescue e
|
||||
Baguette::Log.error "Error handling read_chunk: #{e.message}"
|
||||
FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error: #{e.message}"
|
||||
end
|
||||
|
||||
# the client sent an upload request
|
||||
|
|
@ -216,9 +215,6 @@ class FileStorage::Storage
|
|||
# TODO: store upload request in UserData?
|
||||
|
||||
FileStorage::Response::Upload.new request.mid, path
|
||||
rescue e
|
||||
Baguette::Log.error "Error handling upload request: #{e.message}"
|
||||
FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error in upload request: #{e.message}"
|
||||
end
|
||||
|
||||
# The client sent a download request.
|
||||
|
|
@ -235,7 +231,7 @@ class FileStorage::Storage
|
|||
|
||||
# This is acceptation.
|
||||
# Return some useful values: number of chunks.
|
||||
return FileStorage::Response::Download.new mid, file_transfer.file_info.nb_chunks
|
||||
return FileStorage::Response::Download.new mid, file_transfer.file_info
|
||||
else
|
||||
return FileStorage::Errors::GenericError.new mid, "Unknown file digest: #{file_digest}"
|
||||
end
|
||||
|
|
@ -247,9 +243,6 @@ class FileStorage::Storage
|
|||
|
||||
# Should have returned by now: file wasn't found.
|
||||
FileStorage::Errors::GenericError.new mid, "File not found with provided parameters."
|
||||
rescue e
|
||||
Baguette::Log.error "Error handling download request: #{e.message}"
|
||||
FileStorage::Errors::GenericError.new mid.not_nil!, "Unexpected error in download request: #{e.message}"
|
||||
end
|
||||
|
||||
# Entry point for request management
|
||||
|
|
@ -283,8 +276,9 @@ class FileStorage::Storage
|
|||
|
||||
path = get_fs_path file_digest
|
||||
real_size = 0
|
||||
File.open(path, "rb").read_at offset, chunk_size do |buffer|
|
||||
real_size = buffer.read buffer_data
|
||||
File.open(path, "rb") do |file|
|
||||
file.seek offset
|
||||
real_size = file.read buffer_data
|
||||
end
|
||||
|
||||
buffer_data[0..real_size-1]
|
||||
|
|
|
|||
Loading…
Reference in New Issue