youngkingdom/s3fs-fuse
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved stat cache efficiency by not delete but only register

Browse Source
This commit is contained in:
Takeshi Nakatani
2025-08-01 06:14:02 +00:00
committed by Andrew Gaul
parent b7b5a108c2
commit 60fb557f14
7 changed files with 389 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
src/cache.cpp
View File

@ -113,18 +113,10 @@ bool StatCache::GetStat(const std::string& key, struct stat* pstbuf, headers_t*
return true;
}
bool StatCache::AddStat(const std::string& key, const struct stat& stbuf, const headers_t& meta, objtype_t type, bool notruncate)
bool StatCache::AddStatHasLock(const std::string& key, const struct stat* pstbuf, const headers_t* pmeta, objtype_t type, bool notruncate)
{
// [NOTE]
// If notruncate=true, force caching
//
if(GetCacheSize() < 1 && !notruncate){
return true;
}
const std::lock_guard<std::mutex> lock(StatCache::stat_cache_lock);
// Add(overwrite) new cache
if(!pMountPointDir->Add(key, &stbuf, &meta, type, notruncate)){
if(!pMountPointDir->Add(key, pstbuf, pmeta, type, notruncate)){
S3FS_PRN_DBG("failed to add stat cache entry[path=%s]", key.c_str());
return false;
}
@ -141,6 +133,32 @@ bool StatCache::AddStat(const std::string& key, const struct stat& stbuf, const
return true;
}
bool StatCache::AddStat(const std::string& key, const struct stat& stbuf, const headers_t& meta, objtype_t type, bool notruncate)
{
// [NOTE]
// If notruncate=true, force caching
//
if(GetCacheSize() < 1 && !notruncate){
return true;
}
const std::lock_guard<std::mutex> lock(StatCache::stat_cache_lock);
return AddStatHasLock(key, &stbuf, &meta, type, notruncate);
}
bool StatCache::AddStat(const std::string& key, const struct stat& stbuf, objtype_t type, bool notruncate)
{
// [NOTE]
// If notruncate=true, force caching
//
if(GetCacheSize() < 1 && !notruncate){
return true;
}
const std::lock_guard<std::mutex> lock(StatCache::stat_cache_lock);
return AddStatHasLock(key, &stbuf, nullptr, type, notruncate);
}
// [NOTE]
// Updates only meta data if cached data exists.
// And when these are updated, it also updates the cache time.

2
src/cache.h
View File

@ -58,6 +58,7 @@ class StatCache
StatCache();
~StatCache();
bool AddStatHasLock(const std::string& key, const struct stat* pstbuf, const headers_t* pmeta, objtype_t type, bool notruncate) REQUIRES(StatCache::stat_cache_lock);
bool TruncateCacheHasLock(bool check_only_oversize_case = true) REQUIRES(StatCache::stat_cache_lock);
bool DelStatHasLock(const std::string& key) REQUIRES(StatCache::stat_cache_lock);
bool RawGetChildStats(const std::string& dir, s3obj_list_t* plist, s3obj_type_map_t* pobjmap);
@ -103,6 +104,7 @@ class StatCache
// Add stat cache
bool AddStat(const std::string& key, const struct stat& stbuf, const headers_t& meta, objtype_t type, bool notruncate = false);
bool AddStat(const std::string& key, const struct stat& stbuf, objtype_t type, bool notruncate = false);
bool AddNegativeStat(const std::string& key);
// Update meta stats

16
src/fdcache_entity.cpp
View File

@ -1002,6 +1002,22 @@ bool FdEntity::SetContentType(const char* path)
return true;
}
//
// Converts the internal meta header into a stat structure and returns it.
//
bool FdEntity::GetStatsFromMeta(struct stat& st) const
{
const std::lock_guard<std::mutex> lock(fdent_lock);
if(!convert_header_to_stat(path, orgmeta, st, false)){
return false;
}
const std::lock_guard<std::mutex> data_lock(fdent_data_lock);
st.st_size = pagelist.Size(); // set current file size
return true;
}
bool FdEntity::SetAllStatus(bool is_loaded)
{
S3FS_PRN_INFO3("[path=%s][physical_fd=%d][%s]", path.c_str(), physical_fd, is_loaded ? "loaded" : "unloaded");

5
src/fdcache_entity.h
View File

@ -172,6 +172,10 @@ class FdEntity : public std::enable_shared_from_this<FdEntity>
const std::lock_guard<std::mutex> lock_data(fdent_data_lock);
return UploadPendingHasLock(fd);
}
bool HaveUploadPending(){
const std::lock_guard<std::mutex> lock_data(fdent_data_lock);
return (pending_status_t::NO_UPDATE_PENDING != pending_status);
}
bool GetStats(struct stat& st) const {
const std::lock_guard<std::mutex> lock(fdent_lock);
@ -219,6 +223,7 @@ class FdEntity : public std::enable_shared_from_this<FdEntity>
}
bool SetGIdHasLock(gid_t gid) REQUIRES(FdEntity::fdent_lock);
bool SetContentType(const char* path);
bool GetStatsFromMeta(struct stat& st) const;
int Load(off_t start, off_t size, bool is_modified_flag = false) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); // size=0 means loading to end

45
src/metaheader.cpp
View File

@ -260,6 +260,51 @@ bool is_dir_fmt(const headers_t& meta)
return S_ISDIR(convert_meta_to_mode_fmt(meta));
}
// [NOTE]
// For directory types, detailed judgment is not possible.
// DIR_NORMAL is always returned.
//
// Objects uploaded using clients other than s3fs has a Content-Type
// of application/unknown and dose not have x-amz-meta-mode header.
// In this case, you can specify objtype_t as default_type.
//
objtype_t derive_object_type(const std::string& strpath, const headers_t& meta, objtype_t default_type)
{
mode_t mode = convert_meta_to_mode_fmt(meta);
if(S_ISDIR(mode)){
if('/' != *strpath.rbegin()){
return objtype_t::DIR_NOT_TERMINATE_SLASH;
}else if(std::string::npos != strpath.find("_$folder$", 0)){
return objtype_t::DIR_FOLDER_SUFFIX;
}else{
// [NOTE]
// It returns DIR_NORMAL, although it could be DIR_NOT_EXIST_OBJECT.
//
return objtype_t::DIR_NORMAL;
}
}else if(S_ISLNK(mode)){
return objtype_t::SYMLINK;
}else if(S_ISREG(mode)){
return objtype_t::FILE;
}else if(0 == mode){
// If the x-amz-meta-mode header is not present, mode is 0.
headers_t::const_iterator iter;
if(meta.cend() != (iter = meta.find("Content-Type"))){
std::string strConType = iter->second;
// Leave just the mime type, remove any optional parameters (eg charset)
std::string::size_type pos = strConType.find(';');
if(std::string::npos != pos){
strConType.erase(pos);
}
if(strConType == "application/unknown"){
return default_type;
}
}
}
return objtype_t::UNKNOWN;
}
uid_t get_uid(const char *s)
{
return static_cast<uid_t>(cvt_strtoofft(s, /*base=*/ 0));

1
src/metaheader.h
View File

@ -45,6 +45,7 @@ mode_t get_mode(const headers_t& meta, const std::string& strpath, bool checkdir
bool is_reg_fmt(const headers_t& meta);
bool is_symlink_fmt(const headers_t& meta);
bool is_dir_fmt(const headers_t& meta);
objtype_t derive_object_type(const std::string& strpath, const headers_t& meta, objtype_t default_type = objtype_t::UNKNOWN);
uid_t get_uid(const char *s);
uid_t get_uid(const headers_t& meta);
gid_t get_gid(const char *s);

395
src/s3fs.cpp
View File

@ -116,8 +116,8 @@ static int readdir_multi_head(const std::string& strpath, const S3ObjList& head,
static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, bool check_content_only = false);
static int directory_empty(const char* path);
static int rename_large_object(const char* from, const char* to);
static int create_file_object(const char* path, mode_t mode, uid_t uid, gid_t gid);
static int create_directory_object(const char* path, mode_t mode, const struct timespec& ts_atime, const struct timespec& ts_mtime, const struct timespec& ts_ctime, uid_t uid, gid_t gid, const char* pxattrvalue);
static int create_file_object(const char* path, mode_t mode, uid_t uid, gid_t gid, struct stat* pstbuf = nullptr);
static int create_directory_object(const char* path, mode_t mode, const struct timespec& ts_atime, const struct timespec& ts_mtime, const struct timespec& ts_ctime, uid_t uid, gid_t gid, const char* pxattrvalue, struct stat* pstbuf = nullptr);
static int rename_object(const char* from, const char* to, bool update_ctime);
static int rename_object_nocopy(const char* from, const char* to, bool update_ctime);
static int clone_directory_object(const char* from, const char* to, bool update_ctime, const char* pxattrvalue);
@ -254,7 +254,8 @@ static int remove_old_type_dir(const std::string& path, objtype_t ObjType)
if(0 != (result = delete_request(path))){
return result;
}
// succeed removing or not found the directory
// succeed removing or not found the directory, then remove stat cache as well
StatCache::getStatCacheData()->DelStat(path);
}else{
// nothing to do
}
@ -287,8 +288,10 @@ static int get_object_attribute(const char* path, struct stat* pstbuf, headers_t
struct stat tmpstbuf = {};
headers_t tmpHead;
objtype_t tmpObjType = objtype_t::UNKNOWN;
bool is_mountpoint = false; // path is the mount point
bool is_bucket_mountpoint = false; // path is the mount point which is the bucket root
bool need_meta = (nullptr != pmeta); // required for stat cache checking
objtype_t* parm_pObjType = pObjType; // keep origin for using at stat cache checking
bool is_mountpoint = false; // path is the mount point
bool is_bucket_mountpoint = false; // path is the mount point which is the bucket root
std::string::size_type Pos;
//
@ -349,16 +352,30 @@ static int get_object_attribute(const char* path, struct stat* pstbuf, headers_t
// [NOTE]
// For mount points("/"), the Stat cache key name is "/".
//
if(StatCache::getStatCacheData()->GetStat(strpath, pstbuf, pmeta, pObjType)){
return 0;
}else{
if(!need_meta){
// [NOTE]
// In the case of a Negative Cache, GetStat() returns false and
// objtype_t::NEGATIVE is set.
// Normally, if the object's stat cache exists but has only stat
// structure(no meta header), a HEAD request will be sent for
// loading meta header.
// But if this function is called with pmeta as nullptr, we can
// suppress the HEAD request by setting the meta header pointer
// to nullptr and making GetStat() call.
//
if(objtype_t::NEGATIVE == *pObjType){
// When not hit cache and the reason is negative cache.
return -ENOENT;
if(StatCache::getStatCacheData()->GetStat(strpath, pstbuf, nullptr, parm_pObjType)){
return 0;
}
}else{
if(StatCache::getStatCacheData()->GetStat(strpath, pstbuf, pmeta, pObjType)){
return 0;
}else{
// [NOTE]
// In the case of a Negative Cache, GetStat() returns false and
// objtype_t::NEGATIVE is set.
//
if(objtype_t::NEGATIVE == *pObjType){
// When not hit cache and the reason is negative cache.
return -ENOENT;
}
}
}
@ -932,7 +949,6 @@ static int s3fs_readlink(const char* _path, char* buf, size_t size)
std::string strValue;
objtype_t ObjType = objtype_t::UNKNOWN;
struct stat stbuf = {};
headers_t meta;
FUSE_CTX_INFO("[path=%s]", strPath.c_str());
@ -943,7 +959,7 @@ static int s3fs_readlink(const char* _path, char* buf, size_t size)
// check if there is a valid cache
bool found_cache = false;
if(StatCache::getStatCacheData()->GetStat(strPath, &stbuf, &meta, &ObjType)){
if(StatCache::getStatCacheData()->GetStat(strPath, &stbuf, nullptr, &ObjType)){
if(IS_SYMLINK_OBJ(ObjType)){
// found symlink stat cache
found_cache = true;
@ -993,7 +1009,8 @@ static int s3fs_readlink(const char* _path, char* buf, size_t size)
}
// make meta and stat
bool add_cache = true;
headers_t meta;
bool add_cache = true;
if(!found_cache){
// make headers for symlink
std::string strnow = s3fs_str_realtime();
@ -1027,8 +1044,12 @@ static int s3fs_readlink(const char* _path, char* buf, size_t size)
return 0;
}
// common function for creation of a plain object
static int create_file_object(const char* path, mode_t mode, uid_t uid, gid_t gid)
// Common function for creation of a plain object
//
// [NOTE]
// Currently this is only called by s3fs_mknod.
//
static int create_file_object(const char* path, mode_t mode, uid_t uid, gid_t gid, struct stat* pstbuf)
{
S3FS_PRN_INFO2("[path=%s][mode=%04o]", path, mode);
@ -1046,6 +1067,13 @@ static int create_file_object(const char* path, mode_t mode, uid_t uid, gid_t gi
if(0 != (result = put_request(SAFESTRPTR(path), meta, -1, true/* ahbe */))){
return result;
}
if(pstbuf){
if(!convert_header_to_stat(std::string(path), meta, *pstbuf)){
S3FS_PRN_ERR("failed convert headers to stat[path=%s]", path);
return -EIO;
}
}
return 0;
}
@ -1054,6 +1082,7 @@ static int s3fs_mknod(const char *_path, mode_t mode, dev_t rdev)
WTF8_ENCODE(path)
int result;
struct fuse_context* pcxt;
struct stat stbuf;
FUSE_CTX_INFO("[path=%s][mode=%04o][dev=%llu]", path, mode, (unsigned long long)rdev);
@ -1061,13 +1090,15 @@ static int s3fs_mknod(const char *_path, mode_t mode, dev_t rdev)
return -EIO;
}
if(0 != (result = create_file_object(path, mode, pcxt->uid, pcxt->gid))){
if(0 != (result = create_file_object(path, mode, pcxt->uid, pcxt->gid, &stbuf))){
S3FS_PRN_ERR("could not create object for special file(result=%d)", result);
return result;
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(path);
// Add only stat structure to stat cache
if(!StatCache::getStatCacheData()->AddStat(std::string(path), stbuf, objtype_t::FILE)){
S3FS_PRN_ERR("succeed to mknod the file(%s), but could not add stat cache.", path);
}
// update parent directory timestamp
int update_result;
@ -1154,7 +1185,7 @@ static int s3fs_create(const char* _path, mode_t mode, struct fuse_file_info* fi
return 0;
}
static int create_directory_object(const char* path, mode_t mode, const struct timespec& ts_atime, const struct timespec& ts_mtime, const struct timespec& ts_ctime, uid_t uid, gid_t gid, const char* pxattrvalue)
static int create_directory_object(const char* path, mode_t mode, const struct timespec& ts_atime, const struct timespec& ts_mtime, const struct timespec& ts_ctime, uid_t uid, gid_t gid, const char* pxattrvalue, struct stat* pstbuf)
{
S3FS_PRN_INFO1("[path=%s][mode=%04o][atime=%s][mtime=%s][ctime=%s][uid=%u][gid=%u]", path, mode, str(ts_atime).c_str(), str(ts_mtime).c_str(), str(ts_ctime).c_str(), (unsigned int)uid, (unsigned int)gid);
@ -1185,6 +1216,13 @@ static int create_directory_object(const char* path, mode_t mode, const struct t
if(0 != (result = put_request(tpath, meta, -1, false/* ahbe */))){
return result;
}
if(pstbuf){
if(!convert_header_to_stat(tpath, meta, *pstbuf, true)){ // There is no need to worry if the tpath is "//"
S3FS_PRN_ERR("failed convert headers to stat[path=%s]", path);
return -EIO;
}
}
return 0;
}
@ -1193,6 +1231,7 @@ static int s3fs_mkdir(const char* _path, mode_t mode)
WTF8_ENCODE(path)
int result;
struct fuse_context* pcxt;
struct stat stbuf;
FUSE_CTX_INFO("[path=%s][mode=%04o]", path, mode);
@ -1221,10 +1260,12 @@ static int s3fs_mkdir(const char* _path, mode_t mode)
struct timespec now;
s3fs_realtime(now);
result = create_directory_object(path, mode, now, now, now, pcxt->uid, pcxt->gid, pxattrvalue);
result = create_directory_object(path, mode, now, now, now, pcxt->uid, pcxt->gid, pxattrvalue, &stbuf);
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(path);
// Add only stat structure to stat cache
if(!StatCache::getStatCacheData()->AddStat(std::string(path), stbuf, objtype_t::DIR_NORMAL)){
S3FS_PRN_ERR("succeed to create the directory(%s), but could not add stat cache.", path);
}
// update parent directory timestamp
int update_result;
@ -1456,7 +1497,7 @@ static int rename_object(const char* from, const char* to, bool update_ctime)
{
int result;
headers_t meta;
struct stat buf;
struct stat stbuf;
S3FS_PRN_INFO1("[from=%s][to=%s]", from , to);
@ -1468,7 +1509,7 @@ static int rename_object(const char* from, const char* to, bool update_ctime)
// not permit removing "from" object parent dir.
return result;
}
if(0 != (result = get_object_attribute(from, &buf, &meta))){
if(0 != (result = get_object_attribute(from, &stbuf, &meta))){
return result;
}
@ -1526,7 +1567,7 @@ static int rename_object(const char* from, const char* to, bool update_ctime)
// This does not affect the rename process, but the cache information in
// the "modified" state remains, making it impossible to read the file correctly.
//
ent = autoent.Open(from, &meta, buf.st_size, mtime, O_RDONLY, false, true, false);
ent = autoent.Open(from, &meta, stbuf.st_size, mtime, O_RDONLY, false, true, false);
}
if(ent){
if(0 != (result = ent->SetMCtime(mtime, ctime))){
@ -1549,8 +1590,19 @@ static int rename_object(const char* from, const char* to, bool update_ctime)
// Remove file
result = s3fs_unlink(from);
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(to);
// Add only stat structure to stat cache
std::string strTo = to;
stbuf = {};
if(convert_header_to_stat(strTo, meta, stbuf)){
objtype_t ObjType = derive_object_type(strTo, meta, objtype_t::FILE);
if(!StatCache::getStatCacheData()->AddStat(strTo, stbuf, ObjType)){
S3FS_PRN_ERR("succeed to rename %s to %s, but could not add stat cache. So remove stat cache.", from, to);
StatCache::getStatCacheData()->DelStat(strTo);
}
}else{
S3FS_PRN_ERR("failed convert headers to stat[path=%s]. So remove stat cache.", to);
StatCache::getStatCacheData()->DelStat(strTo);
}
return result;
}
@ -1603,7 +1655,9 @@ static int rename_object_nocopy(const char* from, const char* to, bool update_ct
// Remove file
result = s3fs_unlink(from);
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
// [NOTE]
// Not have a copy API (nocopy), the stat cache will be removed.
//
StatCache::getStatCacheData()->DelStat(to);
return result;
@ -1612,7 +1666,7 @@ static int rename_object_nocopy(const char* from, const char* to, bool update_ct
static int rename_large_object(const char* from, const char* to)
{
int result;
struct stat buf;
struct stat stbuf;
headers_t meta;
S3FS_PRN_INFO1("[from=%s][to=%s]", from , to);
@ -1625,11 +1679,11 @@ static int rename_large_object(const char* from, const char* to)
// not permit removing "from" object parent dir.
return result;
}
if(0 != (result = get_object_attribute(from, &buf, &meta, false))){
if(0 != (result = get_object_attribute(from, &stbuf, &meta, false))){
return result;
}
if(0 != (result = multipart_put_head_request(from, to, buf.st_size, meta))){
if(0 != (result = multipart_put_head_request(from, to, stbuf.st_size, meta))){
return result;
}
@ -1639,8 +1693,18 @@ static int rename_large_object(const char* from, const char* to)
// Remove file
result = s3fs_unlink(from);
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(to);
// Add only stat structure to stat cache
stbuf = {};
if(convert_header_to_stat(std::string(to), meta, stbuf)){
objtype_t ObjType = derive_object_type(std::string(to), meta, objtype_t::FILE);
if(!StatCache::getStatCacheData()->AddStat(std::string(to), stbuf, ObjType)){
S3FS_PRN_ERR("succeed to rename %s to %s, but could not add stat cache. So remove stat cache.", from, to);
StatCache::getStatCacheData()->DelStat(std::string(to));
}
}else{
S3FS_PRN_ERR("failed convert headers to stat[path=%s]. So remove stat cache.", to);
StatCache::getStatCacheData()->DelStat(std::string(to));
}
return result;
}
@ -1649,6 +1713,7 @@ static int clone_directory_object(const char* from, const char* to, bool update_
{
int result = -1;
struct stat stbuf;
struct stat newbuf;
S3FS_PRN_INFO1("[from=%s][to=%s]", from, to);
@ -1667,10 +1732,13 @@ static int clone_directory_object(const char* from, const char* to, bool update_
}else{
set_stat_to_timespec(stbuf, stat_time_type::CTIME, ts_ctime);
}
result = create_directory_object(to, stbuf.st_mode, ts_atime, ts_mtime, ts_ctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue);
result = create_directory_object(to, stbuf.st_mode, ts_atime, ts_mtime, ts_ctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue, &newbuf);
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(to);
// Add only stat structure to stat cache
if(!StatCache::getStatCacheData()->AddStat(std::string(to), newbuf, objtype_t::DIR_NORMAL)){
S3FS_PRN_ERR("succeed to rename %s to %s directory, but could not add stat cache. So remove stat cache.", from, to);
StatCache::getStatCacheData()->DelStat(std::string(to));
}
return result;
}
@ -1909,6 +1977,7 @@ static int s3fs_chmod(const char* _path, mode_t mode)
std::string normpath;
headers_t meta;
struct stat stbuf;
struct stat newstbuf;
objtype_t ObjType = objtype_t::UNKNOWN;
FUSE_CTX_INFO("[path=%s][mode=%04o]", path, mode);
@ -1949,9 +2018,6 @@ static int s3fs_chmod(const char* _path, mode_t mode)
}
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(normpath);
// Make new directory object("dir/")
struct timespec ts_atime;
struct timespec ts_mtime;
@ -1960,9 +2026,15 @@ static int s3fs_chmod(const char* _path, mode_t mode)
set_stat_to_timespec(stbuf, stat_time_type::MTIME, ts_mtime);
s3fs_realtime(ts_ctime);
if(0 != (result = create_directory_object(normpath.c_str(), mode, ts_atime, ts_mtime, ts_ctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue))){
if(0 != (result = create_directory_object(normpath.c_str(), mode, ts_atime, ts_mtime, ts_ctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue, &newstbuf))){
return result;
}
// Add only stat structure to stat cache
if(!StatCache::getStatCacheData()->AddStat(normpath, newstbuf, objtype_t::DIR_NORMAL)){
S3FS_PRN_ERR("succeed to change mode for directory(%s), but could not add stat cache. So remove stat cache.", normpath.c_str());
StatCache::getStatCacheData()->DelStat(normpath);
}
}else{
// normal object or directory object of newer version
std::string strSourcePath = (mount_prefix.empty() && "/" == curpath) ? "//" : curpath;
@ -2026,8 +2098,17 @@ static int s3fs_chmod(const char* _path, mode_t mode)
return result;
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(normpath);
// Add only stat structure to stat cache
if(convert_header_to_stat(curpath, meta, newstbuf)){
ObjType = derive_object_type(curpath, meta, objtype_t::FILE);
if(!StatCache::getStatCacheData()->AddStat(curpath, newstbuf, ObjType)){
S3FS_PRN_ERR("succeed to change mode for %s, but could not add stat cache. So remove stat cache.", curpath.c_str());
StatCache::getStatCacheData()->DelStat(curpath);
}
}else{
S3FS_PRN_ERR("failed convert headers to stat[path=%s]. So remove stat cache.", curpath.c_str());
StatCache::getStatCacheData()->DelStat(curpath);
}
}
}
@ -2041,6 +2122,7 @@ static int s3fs_chmod_nocopy(const char* _path, mode_t mode)
std::string curpath;
std::string normpath;
struct stat stbuf;
struct stat newstbuf;
objtype_t ObjType = objtype_t::UNKNOWN;
FUSE_CTX_INFO1("[path=%s][mode=%04o]", path, mode);
@ -2083,9 +2165,6 @@ static int s3fs_chmod_nocopy(const char* _path, mode_t mode)
}
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(normpath);
// Make new directory object("dir/")
struct timespec ts_atime;
struct timespec ts_mtime;
@ -2094,9 +2173,15 @@ static int s3fs_chmod_nocopy(const char* _path, mode_t mode)
set_stat_to_timespec(stbuf, stat_time_type::MTIME, ts_mtime);
s3fs_realtime(ts_ctime);
if(0 != (result = create_directory_object(normpath.c_str(), mode, ts_atime, ts_mtime, ts_ctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue))){
if(0 != (result = create_directory_object(normpath.c_str(), mode, ts_atime, ts_mtime, ts_ctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue, &newstbuf))){
return result;
}
// Add only stat structure to stat cache
if(!StatCache::getStatCacheData()->AddStat(normpath, newstbuf, objtype_t::DIR_NORMAL)){
S3FS_PRN_ERR("succeed to change mode for directory(%s), but could not add stat cache. So remove stat cache.", normpath.c_str());
StatCache::getStatCacheData()->DelStat(normpath);
}
}else{
// normal object or directory object of newer version
@ -2121,7 +2206,9 @@ static int s3fs_chmod_nocopy(const char* _path, mode_t mode)
return result;
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
// [NOTE]
// Not have a copy API (nocopy), the stat cache will be removed.
//
StatCache::getStatCacheData()->DelStat(normpath);
}
@ -2136,6 +2223,7 @@ static int s3fs_chown(const char* _path, uid_t uid, gid_t gid)
std::string normpath;
headers_t meta;
struct stat stbuf;
struct stat newstbuf;
objtype_t ObjType = objtype_t::UNKNOWN;
FUSE_CTX_INFO("[path=%s][uid=%u][gid=%u]", path, (unsigned int)uid, (unsigned int)gid);
@ -2182,9 +2270,6 @@ static int s3fs_chown(const char* _path, uid_t uid, gid_t gid)
}
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(normpath);
// Make new directory object("dir/")
struct timespec ts_atime;
struct timespec ts_mtime;
@ -2193,9 +2278,15 @@ static int s3fs_chown(const char* _path, uid_t uid, gid_t gid)
set_stat_to_timespec(stbuf, stat_time_type::MTIME, ts_mtime);
s3fs_realtime(ts_ctime);
if(0 != (result = create_directory_object(normpath.c_str(), stbuf.st_mode, ts_atime, ts_mtime, ts_ctime, uid, gid, pxattrvalue))){
if(0 != (result = create_directory_object(normpath.c_str(), stbuf.st_mode, ts_atime, ts_mtime, ts_ctime, uid, gid, pxattrvalue, &newstbuf))){
return result;
}
// Add only stat structure to stat cache
if(!StatCache::getStatCacheData()->AddStat(normpath, newstbuf, objtype_t::DIR_NORMAL)){
S3FS_PRN_ERR("succeed to chown for directory(%s), but could not add stat cache. So remove stat cache.", normpath.c_str());
StatCache::getStatCacheData()->DelStat(normpath);
}
}else{
std::string strSourcePath = (mount_prefix.empty() && "/" == curpath) ? "//" : curpath;
headers_t updatemeta;
@ -2259,8 +2350,17 @@ static int s3fs_chown(const char* _path, uid_t uid, gid_t gid)
return result;
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(normpath);
// Add only stat structure to stat cache
if(convert_header_to_stat(curpath, meta, newstbuf)){
ObjType = derive_object_type(curpath, meta, objtype_t::FILE);
if(!StatCache::getStatCacheData()->AddStat(curpath, newstbuf, ObjType)){
S3FS_PRN_ERR("succeed to chown for %s, but could not add stat cache. So remove stat cache.", curpath.c_str());
StatCache::getStatCacheData()->DelStat(curpath);
}
}else{
S3FS_PRN_ERR("failed convert headers to stat[path=%s]. So remove stat cache.", curpath.c_str());
StatCache::getStatCacheData()->DelStat(curpath);
}
}
}
@ -2274,6 +2374,7 @@ static int s3fs_chown_nocopy(const char* _path, uid_t uid, gid_t gid)
std::string curpath;
std::string normpath;
struct stat stbuf;
struct stat newstbuf;
objtype_t ObjType = objtype_t::UNKNOWN;
FUSE_CTX_INFO1("[path=%s][uid=%u][gid=%u]", path, (unsigned int)uid, (unsigned int)gid);
@ -2322,9 +2423,6 @@ static int s3fs_chown_nocopy(const char* _path, uid_t uid, gid_t gid)
}
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(normpath);
// Make new directory object("dir/")
struct timespec ts_atime;
struct timespec ts_mtime;
@ -2333,9 +2431,15 @@ static int s3fs_chown_nocopy(const char* _path, uid_t uid, gid_t gid)
set_stat_to_timespec(stbuf, stat_time_type::MTIME, ts_mtime);
s3fs_realtime(ts_ctime);
if(0 != (result = create_directory_object(normpath.c_str(), stbuf.st_mode, ts_atime, ts_mtime, ts_ctime, uid, gid, pxattrvalue))){
if(0 != (result = create_directory_object(normpath.c_str(), stbuf.st_mode, ts_atime, ts_mtime, ts_ctime, uid, gid, pxattrvalue, &newstbuf))){
return result;
}
// Add only stat structure to stat cache
if(!StatCache::getStatCacheData()->AddStat(normpath, newstbuf, objtype_t::DIR_NORMAL)){
S3FS_PRN_ERR("succeed to chown for directory(%s), but could not add stat cache. So remove stat cache.", normpath.c_str());
StatCache::getStatCacheData()->DelStat(normpath);
}
}else{
// normal object or directory object of newer version
@ -2361,7 +2465,9 @@ static int s3fs_chown_nocopy(const char* _path, uid_t uid, gid_t gid)
return result;
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
// [NOTE]
// Not have a copy API (nocopy), the stat cache will be removed.
//
StatCache::getStatCacheData()->DelStat(normpath);
}
@ -2394,6 +2500,7 @@ static int update_mctime_parent_directory(const char* _path)
std::string normpath; // normalized directory path(ex. "dir/")
headers_t meta;
struct stat stbuf;
struct stat newstbuf;
struct timespec mctime;
struct timespec atime;
objtype_t ObjType = objtype_t::UNKNOWN;
@ -2450,14 +2557,15 @@ static int update_mctime_parent_directory(const char* _path)
}
}
if(!normpath.empty()){
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(normpath);
// Make new directory object("dir/")
if(0 != (result = create_directory_object(normpath.c_str(), stbuf.st_mode, atime, mctime, mctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue, &newstbuf))){
return result;
}
// Make new directory object("dir/")
if(0 != (result = create_directory_object(normpath.c_str(), stbuf.st_mode, atime, mctime, mctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue))){
return result;
// Add only stat structure to stat cache
if(!StatCache::getStatCacheData()->AddStat(normpath, newstbuf, objtype_t::DIR_NORMAL)){
S3FS_PRN_ERR("succeed to change mctime for parent(%s), but could not add stat cache. So remove stat cache.", normpath.c_str());
StatCache::getStatCacheData()->DelStat(normpath);
}
}else{
std::string strSourcePath = (mount_prefix.empty() && "/" == curpath) ? "//" : curpath;
@ -2475,8 +2583,16 @@ static int update_mctime_parent_directory(const char* _path)
return result;
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(normpath);
if(convert_header_to_stat(curpath, meta, newstbuf)){
ObjType = derive_object_type(curpath, meta, objtype_t::DIR_NORMAL);
if(!StatCache::getStatCacheData()->AddStat(curpath, newstbuf, ObjType)){
S3FS_PRN_ERR("succeed to change mctime for parent(%s), but could not add stat cache. So remove stat cache.", curpath.c_str());
StatCache::getStatCacheData()->DelStat(curpath);
}
}else{
S3FS_PRN_ERR("failed convert headers to stat[path=%s]. So remove stat cache.", curpath.c_str());
StatCache::getStatCacheData()->DelStat(curpath);
}
}
return 0;
@ -2490,6 +2606,7 @@ static int s3fs_utimens(const char* _path, const struct timespec ts[2])
std::string normpath;
headers_t meta;
struct stat stbuf;
struct stat newstbuf;
objtype_t ObjType = objtype_t::UNKNOWN;
FUSE_CTX_INFO("[path=%s][mtime=%s][ctime/atime=%s]", path, str(ts[1]).c_str(), str(ts[0]).c_str());
@ -2546,13 +2663,16 @@ static int s3fs_utimens(const char* _path, const struct timespec ts[2])
}
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(normpath);
// Make new directory object("dir/")
if(0 != (result = create_directory_object(normpath.c_str(), stbuf.st_mode, atime, mtime, ctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue))){
if(0 != (result = create_directory_object(normpath.c_str(), stbuf.st_mode, atime, mtime, ctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue, &newstbuf))){
return result;
}
// Add only stat structure to stat cache
if(!StatCache::getStatCacheData()->AddStat(normpath, newstbuf, objtype_t::DIR_NORMAL)){
S3FS_PRN_ERR("succeed to set utimens directory(%s), but could not add stat cache. So remove stat cache.", normpath.c_str());
StatCache::getStatCacheData()->DelStat(normpath);
}
}else{
std::string strSourcePath = (mount_prefix.empty() && "/" == curpath) ? "//" : curpath;
headers_t updatemeta;
@ -2630,8 +2750,17 @@ static int s3fs_utimens(const char* _path, const struct timespec ts[2])
return result;
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(normpath);
// Add only stat structure to stat cache
if(convert_header_to_stat(curpath, meta, newstbuf)){
ObjType = derive_object_type(curpath, meta, objtype_t::FILE);
if(!StatCache::getStatCacheData()->AddStat(curpath, newstbuf, ObjType)){
S3FS_PRN_ERR("succeed to set utimens for %s, but could not add stat cache. So remove stat cache.", curpath.c_str());
StatCache::getStatCacheData()->DelStat(curpath);
}
}else{
S3FS_PRN_ERR("failed convert headers to stat[path=%s]. So remove stat cache.", curpath.c_str());
StatCache::getStatCacheData()->DelStat(curpath);
}
if(keep_mtime){
if(!ent->SetHoldingMtime(mtime)){
@ -2651,6 +2780,7 @@ static int s3fs_utimens_nocopy(const char* _path, const struct timespec ts[2])
std::string curpath;
std::string normpath;
struct stat stbuf;
struct stat newstbuf;
objtype_t ObjType = objtype_t::UNKNOWN;
FUSE_CTX_INFO1("[path=%s][mtime=%s][atime/ctime=%s]", path, str(ts[1]).c_str(), str(ts[0]).c_str());
@ -2709,13 +2839,16 @@ static int s3fs_utimens_nocopy(const char* _path, const struct timespec ts[2])
}
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(normpath);
// Make new directory object("dir/")
if(0 != (result = create_directory_object(normpath.c_str(), stbuf.st_mode, atime, mtime, ctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue))){
if(0 != (result = create_directory_object(normpath.c_str(), stbuf.st_mode, atime, mtime, ctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue, &newstbuf))){
return result;
}
// Add only stat structure to stat cache
if(!StatCache::getStatCacheData()->AddStat(normpath, newstbuf, objtype_t::DIR_NORMAL)){
S3FS_PRN_ERR("succeed to set utimens directory(%s), but could not add stat cache. So remove stat cache.", normpath.c_str());
StatCache::getStatCacheData()->DelStat(normpath);
}
}else{
// normal object or directory object of newer version
@ -2745,7 +2878,9 @@ static int s3fs_utimens_nocopy(const char* _path, const struct timespec ts[2])
return result;
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
// [NOTE]
// Not have a copy API (nocopy), the stat cache will be removed.
//
StatCache::getStatCacheData()->DelStat(normpath);
}
@ -2820,8 +2955,6 @@ static int s3fs_truncate(const char* _path, off_t size)
S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result);
return result;
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(path);
}
#endif
@ -2849,8 +2982,6 @@ static int s3fs_truncate(const char* _path, off_t size)
S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result);
return result;
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(path);
}
@ -3098,7 +3229,18 @@ static int s3fs_flush(const char* _path, struct fuse_file_info* fi)
return result;
}
}
StatCache::getStatCacheData()->DelStat(path);
//
// Cache only stat structures to stat cache
//
std::string strpath = path;
struct stat stbuf = {};
if(!nocopyapi && ent->GetStatsFromMeta(stbuf) && StatCache::getStatCacheData()->AddStat(strpath, stbuf, objtype_t::FILE)){
S3FS_PRN_DBG("Updated stats(without meta) cache(path=%s) from internal meta header in fdentity.", path);
}else{
S3FS_PRN_DBG("Could not cache only stat structures, so the stat cache(path=%s) will be deleted.", path);
StatCache::getStatCacheData()->DelStat(strpath);
}
if(is_new_file){
// update parent directory timestamp
@ -3200,23 +3342,44 @@ static int s3fs_release(const char* _path, struct fuse_file_info* fi)
// All opened file's stats is cached with no truncate flag.
// Thus we unset it here.
//
StatCache::getStatCacheData()->ClearNoTruncateFlag(std::string(path));
std::string strpath = path;
StatCache::getStatCacheData()->ClearNoTruncateFlag(strpath);
// [NOTICE]
// At first, we remove stats cache.
// Because fuse does not wait for response from "release" function. :-(
// And fuse runs next command before this function returns.
// Thus we call deleting stats function ASAP.
// Thus we call updating or deleting stats function ASAP.
//
if((fi->flags & O_RDWR) || (fi->flags & O_WRONLY)){
StatCache::getStatCacheData()->DelStat(path);
//
// Cache only stat structures to stat cache
//
struct stat stbuf = {};
if(!nocopyapi && ent->GetStatsFromMeta(stbuf) && StatCache::getStatCacheData()->AddStat(strpath, stbuf, objtype_t::FILE)){
S3FS_PRN_DBG("Updated stats(without meta) cache(path=%s) from internal meta header in fdentity.", path);
}else{
S3FS_PRN_DBG("Could not cache only stat structures, so the stat cache(path=%s) will be deleted.", path);
StatCache::getStatCacheData()->DelStat(strpath);
}
}
bool is_new_file = ent->IsDirtyNewFile();
if(0 != (result = ent->UploadPending(static_cast<int>(fi->fh)))){
S3FS_PRN_ERR("could not upload pending data(meta, etc) for pseudo_fd(%llu) / path(%s)", (unsigned long long)(fi->fh), path);
return result;
if(ent->HaveUploadPending()){
if(0 != (result = ent->UploadPending(static_cast<int>(fi->fh)))){
S3FS_PRN_ERR("could not upload pending data(meta, etc) for pseudo_fd(%llu) / path(%s)", (unsigned long long)(fi->fh), path);
return result;
}
// If updated, cache only stat structures to stat cache again.
struct stat stbuf = {};
if(!nocopyapi && ent->GetStatsFromMeta(stbuf) && StatCache::getStatCacheData()->AddStat(strpath, stbuf, objtype_t::FILE)){
S3FS_PRN_DBG("Updated stats(without meta) cache(path=%s) from internal meta header in fdentity.", path);
}else{
S3FS_PRN_ERR("Could not get current stats(path=%s) from internal meta header in fdentity. So remove stat cache.", path);
StatCache::getStatCacheData()->DelStat(strpath);
}
}
if(is_new_file){
@ -3813,6 +3976,7 @@ static int s3fs_setxattr(const char* _path, const char* name, const char* value,
std::string normpath;
headers_t meta;
struct stat stbuf;
struct stat newstbuf;
objtype_t ObjType = objtype_t::UNKNOWN;
FUSE_CTX_INFO("[path=%s][name=%s][value=%p][size=%zu][flags=0x%x]", path, name, value, size, flags);
@ -3846,9 +4010,6 @@ static int s3fs_setxattr(const char* _path, const char* name, const char* value,
}
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(normpath);
// Make new directory object("dir/")
struct timespec ts_atime;
struct timespec ts_mtime;
@ -3857,10 +4018,16 @@ static int s3fs_setxattr(const char* _path, const char* name, const char* value,
set_stat_to_timespec(stbuf, stat_time_type::MTIME, ts_mtime);
set_stat_to_timespec(stbuf, stat_time_type::CTIME, ts_ctime);
if(0 != (result = create_directory_object(normpath.c_str(), stbuf.st_mode, ts_atime, ts_mtime, ts_ctime, stbuf.st_uid, stbuf.st_gid, nullptr))){
if(0 != (result = create_directory_object(normpath.c_str(), stbuf.st_mode, ts_atime, ts_mtime, ts_ctime, stbuf.st_uid, stbuf.st_gid, nullptr, &newstbuf))){
return result;
}
// Add only stat structure to stat cache
if(!StatCache::getStatCacheData()->AddStat(normpath, newstbuf, objtype_t::DIR_NORMAL)){
S3FS_PRN_ERR("succeed to set xattrs directory(%s), but could not add stat cache. So remove stat cache.", normpath.c_str());
StatCache::getStatCacheData()->DelStat(normpath);
}
// need to set xattr header for directory.
curpath = normpath;
}
@ -3947,8 +4114,17 @@ static int s3fs_setxattr(const char* _path, const char* name, const char* value,
return result;
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(normpath);
// Add only stat structure to stat cache
if(convert_header_to_stat(curpath, meta, newstbuf)){
ObjType = derive_object_type(curpath, meta, objtype_t::FILE);
if(!StatCache::getStatCacheData()->AddStat(curpath, newstbuf, ObjType)){
S3FS_PRN_ERR("succeed to set xattrs for %s, but could not add stat cache. So remove stat cache.", curpath.c_str());
StatCache::getStatCacheData()->DelStat(curpath);
}
}else{
S3FS_PRN_ERR("failed convert headers to stat[path=%s]. So remove stat cache.", curpath.c_str());
StatCache::getStatCacheData()->DelStat(curpath);
}
}
return 0;
}
@ -4100,6 +4276,7 @@ static int s3fs_removexattr(const char* _path, const char* name)
headers_t meta;
xattrs_t xattrs;
struct stat stbuf;
struct stat newstbuf;
objtype_t ObjType = objtype_t::UNKNOWN;
FUSE_CTX_INFO("[path=%s][name=%s]", path, name);
@ -4157,9 +4334,6 @@ static int s3fs_removexattr(const char* _path, const char* name)
return result;
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(normpath);
// Make new directory object("dir/")
struct timespec ts_atime;
struct timespec ts_mtime;
@ -4168,10 +4342,16 @@ static int s3fs_removexattr(const char* _path, const char* name)
set_stat_to_timespec(stbuf, stat_time_type::MTIME, ts_mtime);
set_stat_to_timespec(stbuf, stat_time_type::CTIME, ts_ctime);
if(0 != (result = create_directory_object(normpath.c_str(), stbuf.st_mode, ts_atime, ts_mtime, ts_ctime, stbuf.st_uid, stbuf.st_gid, nullptr))){
if(0 != (result = create_directory_object(normpath.c_str(), stbuf.st_mode, ts_atime, ts_mtime, ts_ctime, stbuf.st_uid, stbuf.st_gid, nullptr, &newstbuf))){
return result;
}
// Add only stat structure to stat cache
if(!StatCache::getStatCacheData()->AddStat(normpath, newstbuf, objtype_t::DIR_NORMAL)){
S3FS_PRN_ERR("succeed to remove xattrs directory(%s), but could not add stat cache. So remove stat cache.", normpath.c_str());
StatCache::getStatCacheData()->DelStat(normpath);
}
// need to set xattr header for directory.
curpath = normpath;
}
@ -4268,8 +4448,17 @@ static int s3fs_removexattr(const char* _path, const char* name)
return result;
}
// [TODO][TBD] Plan to overwrite the temporary stat(no meta header) as a cache.
StatCache::getStatCacheData()->DelStat(normpath);
// Add only stat structure to stat cache
if(convert_header_to_stat(curpath, meta, newstbuf)){
ObjType = derive_object_type(curpath, meta, objtype_t::FILE);
if(!StatCache::getStatCacheData()->AddStat(curpath, newstbuf, ObjType)){
S3FS_PRN_ERR("succeed to remove xattrs for %s, but could not add stat cache. So remove stat cache.", curpath.c_str());
StatCache::getStatCacheData()->DelStat(curpath);
}
}else{
S3FS_PRN_ERR("failed convert headers to stat[path=%s]. So remove stat cache.", curpath.c_str());
StatCache::getStatCacheData()->DelStat(curpath);
}
}
return 0;
}