youngkingdom/s3fs-fuse
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored multipart put head request

Browse Source
This commit is contained in:
Takeshi Nakatani
2024-07-15 06:40:05 +00:00
committed by Andrew Gaul
parent a1e47bc287
commit 86b5c9d88e
7 changed files with 297 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
src/s3fs.cpp
View File

@ -44,6 +44,7 @@
#include "fdcache_stat.h"
#include "curl.h"
#include "curl_multi.h"
#include "curl_util.h"
#include "s3objlist.h"
#include "cache.h"
#include "addhead.h"
@ -110,11 +111,6 @@ static bool update_parent_dir_stat= false; // default not updating parent direc
static fsblkcnt_t bucket_block_count; // advertised block count of the bucket
static unsigned long s3fs_block_size = 16 * 1024 * 1024; // s3fs block size is 16MB
//-------------------------------------------------------------------
// Global functions : prototype
//-------------------------------------------------------------------
int put_headers(const char* path, headers_t& meta, bool is_copy, bool use_st_size = true); // [NOTE] global function because this is called from FdEntity class
//-------------------------------------------------------------------
// Static functions : prototype
//-------------------------------------------------------------------
@ -849,7 +845,7 @@ static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char*
// create or update s3 meta
// @return fuse return code
//
int put_headers(const char* path, headers_t& meta, bool is_copy, bool use_st_size)
int put_headers(const char* path, const headers_t& meta, bool is_copy, bool use_st_size)
{
int result;
off_t size;
@ -876,11 +872,7 @@ int put_headers(const char* path, headers_t& meta, bool is_copy, bool use_st_siz
}
if(!nocopyapi && !nomultipart && size >= multipart_threshold){
// [TODO]
// This object will be removed after removing S3fsMultiCurl
//
S3fsCurl s3fscurl(true);
if(0 != (result = s3fscurl.MultipartHeadRequest(strpath.c_str(), size, meta))){
if(0 != (result = multipart_put_head_request(strpath, strpath, size, meta))){
return result;
}
}else{
@ -1560,14 +1552,9 @@ static int rename_large_object(const char* from, const char* to)
return result;
}
// [TODO]
// This object will be removed after removing S3fsMultiCurl
//
S3fsCurl s3fscurl(true);
if(0 != (result = s3fscurl.MultipartRenameRequest(from, to, meta, buf.st_size))){
if(0 != (result = multipart_put_head_request(std::string(from), std::string(to), buf.st_size, meta))){
return result;
}
s3fscurl.DestroyCurlHandle();
// Rename cache file
FdManager::get()->Rename(from, to);
@ -3124,27 +3111,10 @@ static int readdir_multi_head(const char* path, const S3ObjList& head, void* buf
continue;
}
// parameter for thread worker
auto* thargs = new multi_head_req_thparam; // free in multi_head_req_threadworker
thargs->psyncfiller = &syncfiller;
thargs->pthparam_lock = &thparam_lock; // for pretrycount and presult member
thargs->pretrycount = &retrycount;
thargs->pnotfound_list = &notfound_list;
thargs->use_wtf8 = use_wtf8;
thargs->path = disppath;
thargs->presult = &req_result;
// make parameter for thread pool
thpoolman_param ppoolparam;
ppoolparam.args = thargs;
ppoolparam.psem = &multi_head_sem;
ppoolparam.pfunc = multi_head_req_threadworker;
// setup instruction
if(!ThreadPoolMan::Instruct(ppoolparam)){
S3FS_PRN_ERR("failed setup instruction for one header request.");
delete thargs;
return -EIO;
// set one head request
int result;
if(0 != (result = multi_head_request(disppath, syncfiller, thparam_lock, retrycount, notfound_list, use_wtf8, req_result, multi_head_sem))){
return result;
}
++req_count;
}